From 5812d6a387658cca6ee239dd782e9ae305721517 Mon Sep 17 00:00:00 2001 From: Tyler <53561637+im-tyler@users.noreply.github.com> Date: Tue, 22 Sep 2026 17:33:49 -0700 Subject: [PATCH 1/8] feat(deploy,state): durable readiness receipts, predecessor snapshots, honest degraded outcomes (C01-4/5/10) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Readiness receipts (exact candidate IDs + probe outcomes) persist at the attempt namespace the moment the gate passes, before the traffic switch — un-collapsing ReadinessPassed from CandidatesRunning: recovery now distinguishes COMPENSATE (receipt present) from INSPECT (absent) via the unchanged Decide(). Predecessor snapshots persist before any new container starts, so a crashed attempt's compensation restores exactly the recorded container IDs. logDeploy records degraded outcomes (success + degraded + reason) when predecessor retirement partially fails — fleet consumers keying on clean success no longer skip a host that needs attention. C01-1/2/3/8/9 remain deferred with ADR rationales. --- AUDIT_OPEN.md | 89 ++++++ docs/C01_RECOVERY_STATE_TABLE.md | 36 ++- internal/cli/log.go | 14 +- internal/cli/log_degraded_test.go | 92 ++++++ internal/deploy/degraded_log_test.go | 148 ++++++++++ internal/deploy/deploy.go | 139 ++++++++-- internal/deploy/journal.go | 338 +++++++++++++++++++++++ internal/deploy/journal_receipt_test.go | 260 +++++++++++++++++ internal/deploy/journal_snapshot_test.go | 206 ++++++++++++++ internal/deploy/recovery/recovery.go | 14 +- internal/deploy/recovery_a_test.go | 8 +- internal/state/state.go | 11 + 12 files changed, 1311 insertions(+), 44 deletions(-) create mode 100644 internal/cli/log_degraded_test.go create mode 100644 internal/deploy/degraded_log_test.go create mode 100644 internal/deploy/journal.go create mode 100644 internal/deploy/journal_receipt_test.go create mode 100644 internal/deploy/journal_snapshot_test.go diff --git a/AUDIT_OPEN.md b/AUDIT_OPEN.md index aecfde6..e21817d 100644 --- a/AUDIT_OPEN.md +++ b/AUDIT_OPEN.md @@ -1340,3 +1340,92 @@ containers), the enforcement TIMER (nothing server-side schedules pruning — `preview prune` is cron-able but teploy ships no daemon, by design), the same-version shared-alias window above, and Dash-side changes. + +## Programme slice (2026-09-22, latest) — C01 implementation: attempt journal + honest degraded outcome + +Three contained C01 findings landed as their own coherent changes (the +spec is docs/C01_RECOVERY_STATE_TABLE.md's findings list; the decision +function internal/deploy/recovery.Decide is UNCHANGED — its exhaustive +tests pass untouched; this slice produces the EVIDENCE its inputs model). +Base revision `c0efd26`; changes left uncommitted for review. New file +`internal/deploy/journal.go` is the attempt journal: receipts persisted +into the F08 attempt namespace (`meta/att/./`, write-once, +0600 atomic, identity-validated on read — T56 parity). + +- **C01-10 — durable predecessor snapshot.** `predecessors.json` + (exact container IDs/names/labels + the predecessor release identity + + same-version flag) is persisted at the RENAME PHASE — after the 6b + listing, before the recreate displacement or any new container starts + (test pins the write's call index below the first `docker stop` and + `docker run`). On the recovery paths the instruction names + (`restoreDisplacedAndStarted`, `abortStateCommit`), when the in-memory + displaced list is absent, `displacedFromSnapshot` reads the receipt and + restores exactly the recorded web containers that are no longer running + (blue/green predecessors and same-version `_replaced` renames inspect + as running and are skipped by construction). abortStateCommit takes the + attempt as a parameter for this. TDD red: both tests failed "no + predecessor snapshot persisted" before journal.go existed. Mutation: + removing the write fails both tests for that reason (reverted). The + write-then-crash-then-recover test drives a NEW executor seeded with + only the crashed attempt's file state and asserts the exact recorded + name is recreated while a stopped same-release WORKER the name-derived + fallback would touch is never inspected. +- **C01-4 — durable readiness receipt.** `readiness.json` (exact + candidate container IDs from docker run + names, per-replica probe + host/port/path, outcome, timestamp) is written EXACTLY when the health + gate passes and BEFORE the traffic switch begins (test pins the + ordering: after the last probe, before the Caddyfile transaction's + first command; a health-failing deploy leaves no receipt). The + Decide-side wiring is the evidence derivation `attemptReadinessState` + (receipt present → recovery.ReadinessPassed; confirmed absent → + CandidatesRunning — the ADR's collapse, un-collapsed) and + `candidateAttribution` (running candidate-shaped containers are + PROVABLY the crashed attempt's only on receipt ID/name match → Present; + without a receipt, or with IDs that provably belong to another attempt + of the same release, → Unknown — R4's never-auto-decide class). Tests + assert through recovery.Decide: the crash-after-readiness world + (traffic switched, uncommitted, predecessor serving) is COMPENSATE with + the receipt and INSPECT without. Mutations: writing the receipt before + the gate fails both ordering tests; receipt-independent attribution + fails the Decide distinction ("without the receipt the same world must + INSPECT, got COMPENSATE") — both reverted. In-tree consumers are the + derivation helpers; the recovery OWNER that reads them on lock + acquisition is C01-1's slice (recorded). +- **C01-5 — honest degraded outcome.** `state.LogEntry` gains + `Degraded` + `DegradedReason` (omitempty — old entries parse + unchanged). Step-14 retirement collects its incompleteness + (stopPredecessorSnapshot now returns what escaped — stop/remove + failures, fence-loss interruptions, skipped cleanup, name-fallback + errors) and `logDeploy(ctx, cfg, true, reason, start)` records + Success=true AND Degraded=true: traffic IS switched (not a deploy + failure) but the outcome is not clean success. The Success-filtering + consumer in-repo, `teploy log` rendering (internal/cli/log.go), shows + DEGRADED + reason, and --json carries the field for dash/machine + readers; the consumer test also models the fleet-rollback selector + (Success alone targets the degraded host as clean; Success && !Degraded + separates it). TDD red: tests failed to compile against the field-less + LogEntry; mutation: emptying the degraded population at the success + call site fails "DegradedReason must name the escaped container" + (reverted — note this check ran before an accidental `git checkout` + required re-applying the same edits; the re-applied code is identical + and all tests re-ran green). + +Gates: `go vet ./...` clean; `go vet -tags integration +./internal/deploy/recovery` clean; `go test ./... -race` all 25 packages +ok; recovery exhaustive suite green and byte-identical semantics; gofmt +clean on touched files; contract probes 5/5 PASS. No push performed. + +**Residual C01 list (explicit):** C01-1 lock acquisition is treated as +quiescence — the replacement owner must run Decide over observed +evidence after a stale break (the locking-protocol redesign). C01-2 +pre-commit effects are check-then-act, not guarded — a broken holder's +candidate/route effects can land inside the new owner's window. C01-3 +the shared Caddy lock is ownerless/unfenced — conflicting-route evidence +has no producer/consumer. C01-8 same-version running `_replaced` stays +MANUAL — deliberate A08 containment; automating the INSPECT→adopt +continuation needs generation-scoped identities. C01-9 candidate names +are version-keyed, not attempt-keyed — two attempts of one hash are not +attributable by evidence (F04/A09). C01-6 (record-write convergence has +no reconciler) and C01-7 (compensation reconstructs the predecessor +route instead of using a receipt) also remain, with their register items +(A12/T05 standing for C01-7). diff --git a/docs/C01_RECOVERY_STATE_TABLE.md b/docs/C01_RECOVERY_STATE_TABLE.md index 6652858..187917d 100644 --- a/docs/C01_RECOVERY_STATE_TABLE.md +++ b/docs/C01_RECOVERY_STATE_TABLE.md @@ -41,7 +41,7 @@ that dies inside the transition's window. Also encoded as data in |---|---|---|---|---| | 1 | admitted → prepared | attempt dir `/deployments//meta/att/./` (`internal/releasemeta/attempt.go:107-116`); owner token in `.lock/info` (`internal/state/lock.go:86-92`, `internal/state/state.go:522-533`) | **RETRY** | attempt paths are random-id write-once; a fresh attempt collides with nothing | | 2 | prepared → candidates-running | container IDs from docker run (`internal/deploy/deploy.go:577-607`); names `{app}-{process}-{version}[-{index}]` + `teploy.*` labels (`internal/docker/docker.go:82-117,160-165`) | **INSPECT** | a running candidate with no receipt is never success; corpses are reconciled by the next attempt (`deploy.go:1282-1292`) | -| 3 | candidates-running → readiness-passed | **none — no durable receipt exists** (`internal/deploy/health.go` probes are ephemeral) | **INSPECT** | unobservable post-crash; the owner must re-probe (finding C01-4) | +| 3 | candidates-running → readiness-passed | readiness receipt `meta/att/./readiness.json` — exact candidate IDs + probes + outcome, written on pass before the switch (`internal/deploy/journal.go`, **LANDED 2026-09-22, C01-4**) | **INSPECT** | without the receipt the window is unobservable post-crash; the owner must re-probe. With it, `attemptReadinessState`/`candidateAttribution` derive `ReadinessPassed` + attributable candidates for `Decide` | | 4 | readiness-passed → traffic-switched | managed marker block `# TEPLOY BEGIN `…`END` naming candidate upstreams (`internal/caddy/caddy.go:20-21,584-625`); reload receipt (`caddy.go:29-32`); delivery verification md5 host-vs-container (`caddy.go:523-550`) | **COMPENSATE** | traffic on an uncommitted generation; undo via the recorded/serving predecessor (`abortStateCommit`, `deploy.go:1036-1095`). MANUAL when the predecessor is gone | | 5 | traffic-switched → authoritative-state-committed | fenced rename of `state.json` naming the release (`internal/state/lock.go:353-381`); `Generation`/`OperationID` (`internal/state/state.go:68-95`) | **COMPENSATE** | the commit is the single fenced atomic effect; before it, traffic is uncommitted | | 6 | authoritative-state-committed → predecessor-retired | predecessor snapshot stopped (`internal/deploy/deploy.go:839-861,963-989`); absence in the label inventory (`internal/docker/docker.go:568-571`) | **RETRY** | retirement re-derives from the inventory; failures reported, never silent | @@ -156,7 +156,11 @@ table's, with the register item it belongs to. ReadinessPassed is unobservable post-crash and its recovery disposition collapses into CandidatesRunning's INSPECT. A receipt (attempt-scoped marker recording the probed port/time/result) is a design obligation - for the helper/journal slice. + for the helper/journal slice. **LANDED 2026-09-22** (see AUDIT_OPEN's + C01 implementation slice): `meta/att/./readiness.json`, + written exactly on pass before the switch, with the + `attemptReadinessState`/`candidateAttribution` evidence derivation + asserted through `recovery.Decide` (COMPENSATE with, INSPECT without). 5. **C01-5 — The terminal receipt records success on incomplete retirement.** Predecessor stop/remove failures are warnings @@ -166,7 +170,10 @@ table's, with the register item it belongs to. degraded/partial field. The table (and the multi-host rule below) requires recorded outcomes to be the real outcomes — a fleet rollback decision keyed on that log would skip a host that is still running the - superseded generation. + superseded generation. **LANDED 2026-09-22** (see AUDIT_OPEN's C01 + implementation slice): `LogEntry.Degraded`/`DegradedReason` populated + from step-14 retirement incompleteness; `teploy log` renders DEGRADED + and the JSON carries the field for log-keyed consumers. 6. **C01-6 — Record-write failure degrades silently.** `recordRelease` warns (`internal/deploy/deploy.go:1230-1232`) and nothing schedules @@ -209,7 +216,11 @@ table's, with the register item it belongs to. but a crash loses it; retirement re-derives via `selectPredecessors` (TCL-02-correct) at the cost of the removed-worker capture property. The journal slice should persist the snapshot with the attempt - artifacts. + artifacts. **LANDED 2026-09-22** (see AUDIT_OPEN's C01 implementation + slice): `meta/att/./predecessors.json` at the rename phase + (before any new container starts); `restoreDisplacedAndStarted` and + `abortStateCommit` read it when the in-memory displaced list is + absent and compensate exactly the recorded identities. ## Multi-host rule @@ -305,8 +316,15 @@ Scenarios (each prints a scenario × observed × decision × correctness row): Landed in this slice: the table (tested, exhaustive), this ADR, the harness (compiles, unit-tested decision logic, skips without a fixture). -Open: **execution against a real fixture** (next slice, once the fixture -host exists), and the ten disagreement findings above feed C01's -implementation slices (recovery owner on acquisition, guarded pre-commit -effects, readiness receipt, honest terminal receipts, receipt-driven -compensation, attempt-scoped identities). +Executed against a real fixture 2026-09-21 (see AUDIT_OPEN). + +Implementation slices: **C01-4, C01-5, C01-10 landed 2026-09-22** +(attempt-journal receipts + honest degraded log outcome; evidence in +AUDIT_OPEN's C01 implementation-slice section). Remaining findings: +C01-1/2/3 (the locking-protocol redesign — replacement-owner +reconciliation on acquisition, guarded pre-commit effects, fenced shared +Caddy lock), C01-8 (same-version `_replaced` MANUAL — deliberate A08 +containment until F04 generation identities exist), C01-9 +(attempt-scoped candidate identities — F04/A09), and C01-6/C01-7 +(record-write convergence reconciler; receipt-driven route compensation) +which stay with their register items. diff --git a/internal/cli/log.go b/internal/cli/log.go index 33ad289..58ac87a 100644 --- a/internal/cli/log.go +++ b/internal/cli/log.go @@ -90,11 +90,17 @@ func writeLogEntries(out io.Writer, entries []state.LogEntry, jsonOutput bool, e return nil } - fmt.Fprintf(out, "%-20s %-10s %-8s %-7s %s\n", "TIMESTAMP", "TYPE", "VERSION", "STATUS", "DURATION") + fmt.Fprintf(out, "%-20s %-10s %-8s %-8s %s\n", "TIMESTAMP", "TYPE", "VERSION", "STATUS", "DURATION") for _, e := range entries { status := "ok" if !e.Success { status = "FAILED" + } else if e.Degraded { + // C01-5: traffic switched and the app serves, but predecessor + // retirement partially failed — neither clean nor failed, and + // Success-filtering consumers (fleet rollback targeting keyed + // on the log) must be able to tell it apart from "ok". + status = "DEGRADED" } ts := e.Timestamp.Format("2006-01-02 15:04:05") hash := e.Hash @@ -105,7 +111,11 @@ func writeLogEntries(out io.Writer, entries []state.LogEntry, jsonOutput bool, e if e.DurationMs == 0 { dur = "-" } - fmt.Fprintf(out, "%-20s %-10s %-8s %-7s %s\n", ts, e.Type, hash, status, dur) + fmt.Fprintf(out, "%-20s %-10s %-8s %-8s %s", ts, e.Type, hash, status, dur) + if e.DegradedReason != "" { + fmt.Fprintf(out, " (%s)", e.DegradedReason) + } + fmt.Fprintln(out) } return nil } diff --git a/internal/cli/log_degraded_test.go b/internal/cli/log_degraded_test.go new file mode 100644 index 0000000..6d9f0b3 --- /dev/null +++ b/internal/cli/log_degraded_test.go @@ -0,0 +1,92 @@ +package cli + +import ( + "bytes" + "encoding/json" + "strings" + "testing" + "time" + + "github.com/useteploy/teploy/internal/state" +) + +// TestWriteLogEntries_DegradedIsDistinctFromCleanAndFailed is the C01-5 +// consumer regression: every consumer that filters LogEntry on Success +// must be able to tell a DEGRADED success (traffic switched, predecessor +// retirement incomplete) from a clean one. `teploy log` rendering is the +// in-repo consumer; the modeled rollback-selection filter demonstrates +// the fleet decision the finding describes — a rollback keyed on +// Success alone targets the degraded host as "cleanly on the new +// generation", while Success && !Degraded excludes it for attention. +func TestWriteLogEntries_DegradedIsDistinctFromCleanAndFailed(t *testing.T) { + entries := []state.LogEntry{ + {Timestamp: time.Date(2026, 9, 22, 12, 0, 0, 0, time.UTC), App: "app", Type: "deploy", Hash: "aaa111", Success: true, DurationMs: 1000}, + {Timestamp: time.Date(2026, 9, 22, 12, 1, 0, 0, time.UTC), App: "app", Type: "deploy", Hash: "bbb222", Success: true, Degraded: true, DegradedReason: "stop app-web-aaa111: connection refused", DurationMs: 1200}, + {Timestamp: time.Date(2026, 9, 22, 12, 2, 0, 0, time.UTC), App: "app", Type: "deploy", Hash: "ccc333", Success: false, DurationMs: 300}, + } + + var out bytes.Buffer + if err := writeLogEntries(&out, entries, false, ""); err != nil { + t.Fatalf("writeLogEntries: %v", err) + } + text := out.String() + + for _, line := range strings.Split(text, "\n") { + switch { + case strings.Contains(line, "bbb222"): + if !strings.Contains(line, "DEGRADED") { + t.Errorf("degraded entry must render status DEGRADED, got line: %s", line) + } + if strings.Contains(line, " FAILED") { + t.Errorf("degraded is not a failure (traffic switched), got line: %s", line) + } + if !strings.Contains(line, "app-web-aaa111") { + t.Errorf("degraded entry must surface the reason, got line: %s", line) + } + case strings.Contains(line, "aaa111"): + if !strings.Contains(line, " ok ") { + t.Errorf("clean entry must render status ok, got line: %s", line) + } + case strings.Contains(line, "ccc333"): + if !strings.Contains(line, "FAILED") { + t.Errorf("failed entry must render status FAILED, got line: %s", line) + } + } + } + + // The distinction a log-keyed consumer needs: the degraded host is a + // success (it serves the new generation and is compensatable like one) + // but NOT a clean success (part of the superseded generation remains). + rolledBackTargets := 0 + cleanHosts := 0 + for _, e := range entries { + if e.Success { + rolledBackTargets++ + } + if e.Success && !e.Degraded { + cleanHosts++ + } + } + if rolledBackTargets != 2 || cleanHosts != 1 { + t.Errorf("success-filtering consumers must see the degraded host: %d successes (want 2), %d clean (want 1)", rolledBackTargets, cleanHosts) + } + + // JSON (the dash/machine surface) must carry the field. + var jsonOut bytes.Buffer + if err := writeLogEntries(&jsonOut, entries, true, ""); err != nil { + t.Fatalf("writeLogEntries json: %v", err) + } + var decoded []map[string]any + if err := json.Unmarshal(jsonOut.Bytes(), &decoded); err != nil { + t.Fatalf("invalid log JSON: %v", err) + } + if len(decoded) != 3 { + t.Fatalf("want 3 decoded entries, got %d", len(decoded)) + } + if decoded[1]["degraded"] != true { + t.Errorf("degraded entry must carry degraded=true in JSON, got %v", decoded[1]["degraded"]) + } + if decoded[0]["degraded"] != nil { + t.Errorf("clean entry must not carry a degraded key (omitempty), got %v", decoded[0]["degraded"]) + } +} diff --git a/internal/deploy/degraded_log_test.go b/internal/deploy/degraded_log_test.go new file mode 100644 index 0000000..fca85da --- /dev/null +++ b/internal/deploy/degraded_log_test.go @@ -0,0 +1,148 @@ +package deploy + +import ( + "bytes" + "context" + "encoding/json" + "strings" + "testing" + "time" + + "github.com/useteploy/teploy/internal/ssh" + "github.com/useteploy/teploy/internal/state" +) + +// predecessorInventoryJSON renders one running predecessor web container +// of release old123 in docker.ListContainers' custom --format shape +// (labels as a JSON object, audit T15). +const predecessorInventoryJSON = `{"ID":"cid-old123","Names":"myapp-web-old123","Image":"myapp:old","State":"running","Status":"Up 2 hours","CreatedAt":"2026-09-20 10:00:00 +0000 UTC","Labels":{"teploy.app":"myapp","teploy.process":"web","teploy.version":"old123"}}` + "\n" + +// TestDeploy_LogsDegradedWhenRetirementPartiallyFails is the C01-5 core +// regression: a deploy whose traffic switched and committed, but whose +// predecessor retirement partially failed, must be logged as an outcome +// that is NEITHER clean success NOR deploy failure — Success stays true +// (the app serves the new generation) and Degraded becomes true with the +// escaped retirement itemized. The old log recorded clean success, so a +// fleet rollback keyed on the log would skip a host still running part +// of the superseded generation. +func TestDeploy_LogsDegradedWhenRetirementPartiallyFails(t *testing.T) { + existingState := "current_port=49152\ncurrent_hash=old123\nprevious_port=0\nprevious_hash=\n" + + mock := ssh.NewMockExecutor("1.2.3.4", + ssh.MockCommand{Match: "mkdir -p /deployments/myapp", Output: ""}, + ssh.MockCommand{Match: "mkdir /deployments/myapp/.lock", Output: ""}, + ssh.MockCommand{Match: "if [ ! -e '/deployments/myapp/state.json' ]", Output: "absent"}, + ssh.MockCommand{Match: "if [ ! -e '/deployments/myapp/state' ]", Output: "present\n" + existingState}, + ssh.MockCommand{Match: "ss -tln", Output: ssOutput}, + // The predecessor snapshot listing (step 6b) sees old123 running. + ssh.MockCommand{Match: "docker ps --all --filter label=teploy.app='myapp'", Output: predecessorInventoryJSON}, + ssh.MockCommand{Match: "docker run", Output: "newcontainer123"}, + ssh.MockCommand{Match: "docker inspect", Output: "running"}, + ssh.MockCommand{Match: "curl -s -o /dev/null", Output: "200"}, + ssh.MockCommand{Match: "curl -sf http://localhost:2019/config/apps/http/servers/srv0", Output: `{"listen":[":80",":443"]}`}, + ssh.MockCommand{Match: "curl -sf -X PATCH", Err: errBoom}, + ssh.MockCommand{Match: "curl -sf -X POST http://localhost:2019/config/apps/http/servers/srv0/routes", Output: ""}, + ssh.MockCommand{Match: "rm -f /tmp/teploy_caddy", Output: ""}, + ssh.MockCommand{Match: "cat /deployments/caddy/Caddyfile", Output: "{\n\tadmin 0.0.0.0:2019\n}\n"}, + ssh.MockCommand{Match: "mv /tmp/teploy_caddyfile.tmp", Output: ""}, + ssh.MockCommand{Match: "mkdir /deployments/caddy/.lock", Output: ""}, + ssh.MockCommand{Match: "a=$(docker exec caddy md5sum", Output: "TEPLOY_CADDY_OK"}, + ssh.MockCommand{Match: "docker exec caddy caddy reload", Output: ""}, + ssh.MockCommand{Match: "rmdir /deployments/caddy/.lock", Output: ""}, + // Post-commit retirement of the snapshotted predecessor FAILS. + ssh.MockCommand{Match: "docker stop", Err: errBoom}, + ssh.MockCommand{Match: "printf %s", Output: ""}, + ssh.MockCommand{Match: "rm -rf /deployments/myapp/.lock", Output: ""}, + ) + + var buf bytes.Buffer + deployer := NewDeployer(mock, &buf) + + err := deployer.Deploy(context.Background(), Config{ + App: "myapp", + Domain: "myapp.com", + Image: "myapp:v2", + Version: "new456", + Health: HealthConfig{Timeout: 5 * time.Second, Interval: 10 * time.Millisecond}, + }) + if err != nil { + t.Fatalf("a partial retirement failure is degraded, not a failed deploy: %v", err) + } + + logData, ok := logEntryFromCalls(mock) + if !ok { + t.Fatal("log entry not written") + } + var logEntry state.LogEntry + if err := json.Unmarshal(logData, &logEntry); err != nil { + t.Fatalf("parsing log entry: %v", err) + } + if !logEntry.Success { + t.Error("Success must stay true: traffic switched and the app serves (this is not a deploy failure)") + } + if !logEntry.Degraded { + t.Error("Degraded must be true: the predecessor escaped retirement and the log must not record clean success") + } + if !strings.Contains(logEntry.DegradedReason, "myapp-web-old123") { + t.Errorf("DegradedReason must name the escaped container, got %q", logEntry.DegradedReason) + } + if !strings.Contains(buf.String(), "myapp-web-old123") { + t.Error("the deploy output must still report the failed retirement") + } +} + +// TestDeploy_LogsCleanSuccessWhenRetirementCompletes pins the other side +// of C01-5: a fully retired predecessor logs Success WITHOUT the degraded +// flag — the new outcome class must not leak into clean deploys. +func TestDeploy_LogsCleanSuccessWhenRetirementCompletes(t *testing.T) { + existingState := "current_port=49152\ncurrent_hash=old123\nprevious_port=0\nprevious_hash=\n" + + mock := ssh.NewMockExecutor("1.2.3.4", + ssh.MockCommand{Match: "mkdir -p /deployments/myapp", Output: ""}, + ssh.MockCommand{Match: "mkdir /deployments/myapp/.lock", Output: ""}, + ssh.MockCommand{Match: "if [ ! -e '/deployments/myapp/state.json' ]", Output: "absent"}, + ssh.MockCommand{Match: "if [ ! -e '/deployments/myapp/state' ]", Output: "present\n" + existingState}, + ssh.MockCommand{Match: "ss -tln", Output: ssOutput}, + ssh.MockCommand{Match: "docker ps --all --filter label=teploy.app='myapp'", Output: predecessorInventoryJSON}, + ssh.MockCommand{Match: "docker run", Output: "newcontainer123"}, + ssh.MockCommand{Match: "docker inspect", Output: "running"}, + ssh.MockCommand{Match: "curl -s -o /dev/null", Output: "200"}, + ssh.MockCommand{Match: "curl -sf http://localhost:2019/config/apps/http/servers/srv0", Output: `{"listen":[":80",":443"]}`}, + ssh.MockCommand{Match: "curl -sf -X PATCH", Err: errBoom}, + ssh.MockCommand{Match: "curl -sf -X POST http://localhost:2019/config/apps/http/servers/srv0/routes", Output: ""}, + ssh.MockCommand{Match: "rm -f /tmp/teploy_caddy", Output: ""}, + ssh.MockCommand{Match: "cat /deployments/caddy/Caddyfile", Output: "{\n\tadmin 0.0.0.0:2019\n}\n"}, + ssh.MockCommand{Match: "mv /tmp/teploy_caddyfile.tmp", Output: ""}, + ssh.MockCommand{Match: "mkdir /deployments/caddy/.lock", Output: ""}, + ssh.MockCommand{Match: "a=$(docker exec caddy md5sum", Output: "TEPLOY_CADDY_OK"}, + ssh.MockCommand{Match: "docker exec caddy caddy reload", Output: ""}, + ssh.MockCommand{Match: "rmdir /deployments/caddy/.lock", Output: ""}, + ssh.MockCommand{Match: "docker stop", Output: ""}, + ssh.MockCommand{Match: "printf %s", Output: ""}, + ssh.MockCommand{Match: "rm -rf /deployments/myapp/.lock", Output: ""}, + ) + + var buf bytes.Buffer + deployer := NewDeployer(mock, &buf) + if err := deployer.Deploy(context.Background(), Config{ + App: "myapp", + Domain: "myapp.com", + Image: "myapp:v2", + Version: "new456", + Health: HealthConfig{Timeout: 5 * time.Second, Interval: 10 * time.Millisecond}, + }); err != nil { + t.Fatalf("Deploy: %v", err) + } + + logData, ok := logEntryFromCalls(mock) + if !ok { + t.Fatal("log entry not written") + } + var logEntry state.LogEntry + if err := json.Unmarshal(logData, &logEntry); err != nil { + t.Fatalf("parsing log entry: %v", err) + } + if !logEntry.Success || logEntry.Degraded { + t.Errorf("a clean deploy must log clean success, got success=%v degraded=%v", logEntry.Success, logEntry.Degraded) + } +} diff --git a/internal/deploy/deploy.go b/internal/deploy/deploy.go index 1f1d700..9e95b11 100644 --- a/internal/deploy/deploy.go +++ b/internal/deploy/deploy.go @@ -472,6 +472,23 @@ func (d *Deployer) DeployFenced(ctx context.Context, cfg Config, lk *state.Lock) } } + // 6c. Persist the predecessor snapshot into this attempt's immutable + // artifact namespace (C01-10), still BEFORE any new container starts + // and before the recreate-strategy displacement stops the fixed-port + // workload: a crash after candidates start loses the in-memory + // snapshot, and with it the exact knowledge of what this attempt must + // compensate or retire — re-derivation from a post-crash inventory + // selects by the NEW authoritative release (TCL-02's hazard) and the + // name fallback cannot see removed workers (T63). The attempt dir is + // write-once per F08, so the snapshot is immutable once written. A + // persistence failure degrades crash-safety only (warned); the + // in-memory snapshot keeps this deploy correct. + if predecessorsListed { + if err := d.persistPredecessorSnapshot(ctx, assetAttempt, current.CurrentHash, sameVersion, predecessors); err != nil { + fmt.Fprintf(d.out, "Warning: could not persist the predecessor snapshot for crash recovery: %v\n", err) + } + } + // Host ingress and publish-apps recreate rather than blue/green: the new // container reuses the old one's fixed host port(s), so stop the running // web container first to free them. Keep the stopped container until @@ -516,6 +533,10 @@ func (d *Deployer) DeployFenced(ctx context.Context, cfg Config, lk *state.Lock) // returned without either, orphaning the first replica and leaving a // host-ingress app down (audit F05). var started []string + // candidateIDs collects the container IDs docker run returned for the + // web candidates — the exact identities the readiness receipt records + // (C01-4) and the strongest attribution evidence a recovery owner has. + candidateIDs := make([]string, replicas) webContainerNames := make([]string, replicas) restoreDisplacedAndStarted := func(reason error) error { // Cleanup runs detached from the (possibly cancelled) deploy context, @@ -536,8 +557,17 @@ func (d *Deployer) DeployFenced(ctx context.Context, cfg Config, lk *state.Lock) cleanupFailures = append(cleanupFailures, fmt.Sprintf("remove %s: %v", n, err)) } } - restored := len(displacedHostWeb) == 0 - for _, old := range displacedHostWeb { + // The displaced list is the in-memory one when this process did + // the displacing; a process recovering a crashed attempt arrives + // with it empty, and the durable predecessor snapshot (C01-10) + // restores exactly that knowledge — snapshot web containers that + // are no longer running were displaced by the attempt. + displaced := displacedHostWeb + if len(displaced) == 0 { + displaced = d.displacedFromSnapshot(recoveryCtx, assetAttempt) + } + restored := len(displaced) == 0 + for _, old := range displaced { if err := d.docker.Restart(recoveryCtx, old, nil); err != nil { cleanupFailures = append(cleanupFailures, fmt.Sprintf("restore %s: %v", old, err)) fmt.Fprintf(d.out, " WARNING: could not restore displaced container %s: %v\n", old, err) @@ -553,8 +583,8 @@ func (d *Deployer) DeployFenced(ctx context.Context, cfg Config, lk *state.Lock) if len(cleanupFailures) > 0 { fmt.Fprintf(d.out, " WARNING: cleanup incomplete after failure — %s\n", strings.Join(cleanupFailures, "; ")) } - d.logDeploy(recoveryCtx, cfg, false, start) - if len(displacedHostWeb) > 0 && !restored { + d.logDeploy(recoveryCtx, cfg, false, "", start) + if len(displaced) > 0 && !restored { return fmt.Errorf("%w — recovery also failed: no predecessor could be restarted; %s needs manual attention (%s)", reason, cfg.App, strings.Join(cleanupFailures, "; ")) } if len(cleanupFailures) > 0 { @@ -603,6 +633,7 @@ func (d *Deployer) DeployFenced(ctx context.Context, cfg Config, lk *state.Lock) return restoreDisplacedAndStarted(fmt.Errorf("starting container %s: %w", name, err)) } started = append(started, name) + candidateIDs[i] = containerID fmt.Fprintf(d.out, " Container %s started\n", containerID[:min(12, len(containerID))]) } @@ -657,6 +688,30 @@ func (d *Deployer) DeployFenced(ctx context.Context, cfg Config, lk *state.Lock) fmt.Fprintln(d.out, " Health check passed") } + // 9b. Persist the readiness receipt (C01-4) — exactly now: the gate + // has passed, the traffic switch has NOT begun. The receipt (exact + // candidate container IDs + what was probed + when) is what lets a + // recovery owner distinguish "crashed during readiness" (no receipt: + // INSPECT) from "readiness held, crash before/while switching + // traffic" (the COMPENSATE class) — see attemptReadinessState / + // candidateAttribution for the Decide-side derivation. A write + // failure warns: the receipt is recovery evidence, not a deploy + // precondition. + { + probeHost := healthProbeHost(webBindHost) + cands := make([]receiptCandidate, len(webContainerNames)) + probes := make([]readinessProbe, len(ports)) + for i, name := range webContainerNames { + cands[i] = receiptCandidate{Name: name, ID: candidateIDs[i]} + } + for i, p := range ports { + probes[i] = readinessProbe{Container: webContainerNames[i], Host: probeHost, Port: p, Path: healthCfg.Path} + } + if err := d.persistReadinessReceipt(ctx, assetAttempt, cfg.Version, cands, probes); err != nil { + fmt.Fprintf(d.out, "Warning: could not persist the readiness receipt for crash recovery: %v\n", err) + } + } + // 10. Start non-web process containers (workers, etc. — no replicas, one each). if err := lk.Check(ctx, d.exec); err != nil { return fail(err) @@ -769,7 +824,7 @@ func (d *Deployer) DeployFenced(ctx context.Context, cfg Config, lk *state.Lock) // this deploy authoritative is a guarded effect, so a broken holder // commits nothing. if err := state.WriteFenced(ctx, d.exec, cfg.App, newState, lk); err != nil { - return d.abortStateCommit(ctx, cfg, current, started, displacedHostWeb, start, err) + return d.abortStateCommit(ctx, cfg, current, started, displacedHostWeb, assetAttempt, start, err) } // 13b. Record the release metadata (F14). The containers are live and @@ -836,8 +891,15 @@ func (d *Deployer) DeployFenced(ctx context.Context, cfg Config, lk *state.Lock) // reported success. Only a still-failing inventory degrades to names — // now with every stop/remove failure reported (T63's honest-retirement // half). + // + // Every incomplete retirement is collected (C01-5): the deploy stays + // successful (traffic is switched, the app serves), but the terminal + // log entry records the degraded outcome instead of clean success — + // a fleet rollback keyed on the log must not skip a host still + // running part of the superseded generation. + var retireIncomplete []string if predecessorsListed { - d.stopPredecessorSnapshot(ctx, predecessors, sameVersion, stopTimeout, lk) + retireIncomplete = d.stopPredecessorSnapshot(ctx, predecessors, sameVersion, stopTimeout, lk) } else if current != nil && current.CurrentHash != "" { // Fence (F16): post-commit cleanup never interleaves with a new // holder. @@ -845,16 +907,18 @@ func (d *Deployer) DeployFenced(ctx context.Context, cfg Config, lk *state.Lock) if lk != nil { if err := lk.Check(ctx, d.exec); err != nil { fmt.Fprintf(d.out, "Warning: predecessor cleanup skipped — %v\n", err) + retireIncomplete = append(retireIncomplete, fmt.Sprintf("cleanup skipped: %v", err)) fenceOK = false } } if fenceOK { if inv, invErr := d.docker.ListContainers(ctx, cfg.App); invErr == nil { - d.stopPredecessorSnapshot(ctx, selectPredecessors(inv, current, sameVersion), sameVersion, stopTimeout, lk) + retireIncomplete = append(retireIncomplete, d.stopPredecessorSnapshot(ctx, selectPredecessors(inv, current, sameVersion), sameVersion, stopTimeout, lk)...) } else { fmt.Fprintf(d.out, "Warning: container inventory still unreadable (%v) — cleaning up by derived names; a removed worker process may escape retirement\n", invErr) if err := stopOldWorkloadsByName(ctx, d.docker, d.out, cfg, current, processes, stopTimeout); err != nil { fmt.Fprintf(d.out, "Warning: name-based cleanup incomplete: %v\n", err) + retireIncomplete = append(retireIncomplete, err.Error()) } } } @@ -930,8 +994,13 @@ func (d *Deployer) DeployFenced(ctx context.Context, cfg Config, lk *state.Lock) } } - // 16. Log success. - d.logDeploy(ctx, cfg, true, start) + // 16. Log the real outcome (C01-5): clean success only when + // retirement completed; a partial retirement is a degraded success. + degradedReason := strings.Join(retireIncomplete, "; ") + d.logDeploy(ctx, cfg, true, degradedReason, start) + if degradedReason != "" { + fmt.Fprintf(d.out, "Warning: deployed, but predecessor retirement is incomplete — %s\n", degradedReason) + } duration := time.Since(start) fmt.Fprintf(d.out, "\nDeployed %s version %s in %s\n", cfg.App, cfg.Version, duration.Round(time.Millisecond)) @@ -960,15 +1029,19 @@ func selectPredecessors(inv []docker.Container, current *state.AppState, sameVer return out } -// stopPredecessorSnapshot retires exactly the snapshotted predecessor set. +// stopPredecessorSnapshot retires exactly the snapshotted predecessor set +// and returns what escaped retirement (C01-5: the caller records it as a +// degraded outcome — never clean success, never a failed deploy). // Fence checks precede each stop: the deploy is already committed, and a // fence loss mid-cleanup means another operation owns the app — refuse // further stops (loudly) rather than interleaving. -func (d *Deployer) stopPredecessorSnapshot(ctx context.Context, predecessors []docker.Container, sameVersion bool, stopTimeout int, lk *state.Lock) { +func (d *Deployer) stopPredecessorSnapshot(ctx context.Context, predecessors []docker.Container, sameVersion bool, stopTimeout int, lk *state.Lock) []string { + var incomplete []string for _, ct := range predecessors { if lk != nil { if err := lk.Check(ctx, d.exec); err != nil { fmt.Fprintf(d.out, "Warning: predecessor cleanup stopped — %v\n", err) + incomplete = append(incomplete, fmt.Sprintf("cleanup interrupted before %s (fence lost)", ct.Name)) break } } @@ -979,14 +1052,17 @@ func (d *Deployer) stopPredecessorSnapshot(ctx context.Context, predecessors []d // deploy — but it must be reported, never silent (TCL-19): // a leftover old worker keeps consuming jobs. fmt.Fprintf(d.out, "Warning: could not stop old container %s: %v\n", ct.Name, err) + incomplete = append(incomplete, fmt.Sprintf("stop %s: %v", ct.Name, err)) continue } if sameVersion { if err := d.docker.Remove(ctx, ct.Name); err != nil { fmt.Fprintf(d.out, "Warning: could not remove old container %s: %v\n", ct.Name, err) + incomplete = append(incomplete, fmt.Sprintf("remove %s: %v", ct.Name, err)) } } } + return incomplete } // stopOldWorkloadsByName is the name-derived fallback for old-workload @@ -1033,20 +1109,29 @@ func stopOldWorkloadsByName(ctx context.Context, dk *docker.Client, out io.Write return errors.Join(failures...) } -func (d *Deployer) abortStateCommit(ctx context.Context, cfg Config, current *state.AppState, started, displacedHostWeb []string, start time.Time, commitErr error) error { +func (d *Deployer) abortStateCommit(ctx context.Context, cfg Config, current *state.AppState, started, displacedHostWeb []string, att releasemeta.Attempt, start time.Time, commitErr error) error { // Compensation runs on a DETACHED bounded context (A11): if the commit // failed because the deploy context was cancelled, reusing that context // would skip the very stops/restarts/route restores that undo the // deploy — leaving the app dark while the error text claims recovery. recoveryCtx, cancel := context.WithTimeout(context.Background(), 90*time.Second) defer cancel() - d.logDeploy(recoveryCtx, cfg, false, start) + d.logDeploy(recoveryCtx, cfg, false, "", start) + + // The displaced fixed-port workload: the in-memory list when this + // process did the displacing; a process recovering a crashed attempt + // reads it from the durable predecessor snapshot (C01-10) — exactly + // the recorded container identities, never a re-derivation. + displaced := displacedHostWeb + if len(displaced) == 0 { + displaced = d.displacedFromSnapshot(recoveryCtx, att) + } if cfg.ingressHost() || len(cfg.Publish) > 0 { for _, name := range started { d.docker.Stop(recoveryCtx, name, 5) } - for _, old := range displacedHostWeb { + for _, old := range displaced { if err := d.docker.Restart(recoveryCtx, old, nil); err != nil { for _, name := range started { d.docker.Start(recoveryCtx, name) @@ -1072,7 +1157,7 @@ func (d *Deployer) abortStateCommit(ctx context.Context, cfg Config, current *st for _, name := range started { d.docker.Remove(recoveryCtx, name) } - if len(displacedHostWeb) == 0 { + if len(displaced) == 0 { return fmt.Errorf("committing authoritative applied state after starting the first host-ingress workload: %w; the uncommitted workload was stopped and removed", commitErr) } return fmt.Errorf("committing authoritative applied state after replacing the fixed-port host workload: %w; the original workload was restored and the uncommitted workload was removed", commitErr) @@ -1136,15 +1221,23 @@ func (d *Deployer) restorePreviousRoute(ctx context.Context, cfg Config, current return d.caddy.SetRoute(ctx, cfg.App, domain, names[0], primaryPort, tls, cfg.CaddyExtra, cfg.Cache, cfg.Firewall, cfg.Access) } -func (d *Deployer) logDeploy(ctx context.Context, cfg Config, success bool, start time.Time) { +// logDeploy appends the terminal receipt for a deploy attempt. success +// records whether traffic switched and committed; degradedReason (empty +// for clean outcomes and failures) itemizes post-commit retirement that +// partially failed — a degraded success is still serving the new +// generation, and the log must let Success-filtering consumers tell it +// apart from a clean one (C01-5). +func (d *Deployer) logDeploy(ctx context.Context, cfg Config, success bool, degradedReason string, start time.Time) { state.AppendLog(ctx, d.exec, state.LogEntry{ - Timestamp: time.Now().UTC(), - App: cfg.App, - Type: "deploy", - Hash: cfg.Version, - Image: cfg.Image, - Success: success, - DurationMs: time.Since(start).Milliseconds(), + Timestamp: time.Now().UTC(), + App: cfg.App, + Type: "deploy", + Hash: cfg.Version, + Image: cfg.Image, + Success: success, + Degraded: success && degradedReason != "", + DegradedReason: degradedReason, + DurationMs: time.Since(start).Milliseconds(), }) } diff --git a/internal/deploy/journal.go b/internal/deploy/journal.go new file mode 100644 index 0000000..c59aab6 --- /dev/null +++ b/internal/deploy/journal.go @@ -0,0 +1,338 @@ +// The deploy attempt journal (programme workstream C01): small durable +// receipts persisted into the F08 attempt namespace +// (/deployments//meta/att/./) so that a crash anywhere in +// the deploy lifecycle leaves the NEXT process — a recovery owner, an +// operator, a rollback — with the evidence the crash-recovery state table +// (internal/deploy/recovery, docs/C01_RECOVERY_STATE_TABLE.md) reasons +// over. The namespace is write-once per attempt (random id, immutable), +// which is exactly the durability contract these receipts need: nothing +// rewrites a landed receipt, and a fresh attempt can never collide with +// one. +// +// Receipts in this file: +// +// - predecessors.json (C01-10): the exact predecessor workload the +// attempt must retire or compensate, snapshotted at the rename phase +// before any new container starts. +// - readiness.json (C01-4): the health gate's receipt, written the +// moment readiness passes and before the traffic switch begins. +// +// Every receipt carries the writing attempt's identity (app, release, +// attempt name) and is validated against the requested key on read (T56 +// parity: a copied or corrupted-but-valid receipt must not drive effects +// at a different attempt's world). +package deploy + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "strings" + "time" + + "github.com/useteploy/teploy/internal/deploy/recovery" + "github.com/useteploy/teploy/internal/docker" + "github.com/useteploy/teploy/internal/releasemeta" + "github.com/useteploy/teploy/internal/ssh" + "github.com/useteploy/teploy/internal/state" +) + +// journalSchemaVersion is the schema of the attempt-journal receipts. +const journalSchemaVersion = 1 + +// predecessorSnapshotFile is the durable predecessor snapshot's name in +// the attempt namespace. +const predecessorSnapshotFile = "predecessors.json" + +// readinessReceiptFile is the durable readiness receipt's name in the +// attempt namespace (C01-4). +const readinessReceiptFile = "readiness.json" + +// predecessorSnapshot is the durable form of the in-memory predecessor +// snapshot deploy takes after the same-version renames and before any new +// container starts (C01-10). Release is the predecessor's releasemeta +// identity (the release whose record describes these containers); +// Containers are the EXACT identities docker reported — retirement and +// compensation address these names/IDs, never a re-derivation. +type predecessorSnapshot struct { + SchemaVersion int `json:"schema_version"` + App string `json:"app"` + Release string `json:"release"` + Attempt string `json:"attempt"` + SameVersion bool `json:"same_version,omitempty"` + Containers []predecessorContainer `json:"containers"` + WrittenAt time.Time `json:"written_at"` +} + +// predecessorContainer mirrors docker.Container's identity fields. +type predecessorContainer struct { + ID string `json:"id"` + Name string `json:"name"` + Image string `json:"image,omitempty"` + State string `json:"state,omitempty"` + Labels map[string]string `json:"labels,omitempty"` +} + +// predecessorSnapshotPath is the snapshot's location in the attempt +// namespace. +func predecessorSnapshotPath(att releasemeta.Attempt) string { + return att.Dir() + "/" + predecessorSnapshotFile +} + +// persistPredecessorSnapshot writes the snapshot atomically into the +// attempt's immutable namespace (0600, sibling temp + rename — the same +// discipline releasemeta.Write uses). +func (d *Deployer) persistPredecessorSnapshot(ctx context.Context, att releasemeta.Attempt, release string, sameVersion bool, containers []docker.Container) error { + snap := predecessorSnapshot{ + SchemaVersion: journalSchemaVersion, + App: att.App, + Release: release, + Attempt: att.Name(), + SameVersion: sameVersion, + WrittenAt: time.Now().UTC(), + } + for _, c := range containers { + snap.Containers = append(snap.Containers, predecessorContainer{ + ID: c.ID, + Name: c.Name, + Image: c.Image, + State: c.State, + Labels: c.Labels, + }) + } + data, err := json.Marshal(snap) + if err != nil { + return fmt.Errorf("marshaling the predecessor snapshot: %w", err) + } + // The path segments are grammar-validated (app via releasemeta's + // ValidateName, hash via validHash, random hex id), so — like + // releasemeta.Write's own meta mkdir — the path needs no quoting. + if _, err := d.exec.Run(ctx, "mkdir -p "+att.Dir()); err != nil { + return fmt.Errorf("creating the attempt directory: %w", err) + } + return ssh.UploadAtomic(ctx, d.exec, bytes.NewReader(data), predecessorSnapshotPath(att), "0600") +} + +// readPredecessorSnapshot loads the attempt's predecessor snapshot. A +// confirmed-missing file returns (nil, nil); every other failure +// (transport, malformed JSON, wrong schema, identity mismatch) is an +// error — recovery must not guess on unreadable compensation knowledge. +func readPredecessorSnapshot(ctx context.Context, exec ssh.Executor, att releasemeta.Attempt) (*predecessorSnapshot, error) { + data, present, err := state.ReadRemoteFile(ctx, exec, predecessorSnapshotPath(att)) + if err != nil { + return nil, fmt.Errorf("reading the predecessor snapshot for %s: %w", att.Name(), err) + } + if !present { + return nil, nil + } + var snap predecessorSnapshot + if err := json.Unmarshal(data, &snap); err != nil { + return nil, fmt.Errorf("parsing the predecessor snapshot for %s: %w", att.Name(), err) + } + if snap.SchemaVersion != journalSchemaVersion { + return nil, fmt.Errorf("unsupported predecessor-snapshot schema version %d for %s", snap.SchemaVersion, att.Name()) + } + if snap.App != att.App || snap.Attempt != att.Name() { + return nil, fmt.Errorf("predecessor snapshot identity mismatch: requested %s@%s, snapshot describes %s@%s — refusing to use it", att.App, att.Name(), snap.App, snap.Attempt) + } + return &snap, nil +} + +// displacedFromSnapshot recovers the displaced fixed-port workload from +// the durable predecessor snapshot when the in-memory list is absent +// (C01-10's crash-recovery entry, shared by restoreDisplacedAndStarted +// and abortStateCommit): every snapshot web container that is no longer +// running was stopped by this attempt's displacement. A blue/green +// predecessor was never stopped (inspect still shows it running, as does +// a same-version rename under its _replaced name), so only the actually +// displaced containers come back — exactly the recorded identities, never +// a re-derivation from the live inventory (which a post-commit world +// cannot attribute, TCL-02) or from derived names (which cannot see +// removed workers, T63). +func (d *Deployer) displacedFromSnapshot(ctx context.Context, att releasemeta.Attempt) []string { + snap, err := readPredecessorSnapshot(ctx, d.exec, att) + if err != nil { + fmt.Fprintf(d.out, "Warning: could not read the predecessor snapshot for displaced-workload recovery: %v\n", err) + return nil + } + if snap == nil { + return nil + } + var displaced []string + for _, c := range snap.Containers { + if c.Labels["teploy.process"] != "web" { + continue // displacement stops only the fixed-port web workload + } + out, err := d.exec.Run(ctx, "docker inspect -f '{{.State.Status}}' "+ssh.ShellQuote(c.Name)+" 2>/dev/null || true") + if err != nil || strings.TrimSpace(out) == "" { + continue // gone or unreadable: nothing restorable under this name + } + if strings.TrimSpace(out) != "running" { + displaced = append(displaced, c.Name) + } + } + return displaced +} + +// readinessReceipt is the health gate's durable receipt (C01-4): +// persisted into the attempt namespace the moment readiness passes and +// BEFORE the traffic switch begins, so a recovery owner reading the +// journal can distinguish "crashed during readiness probing" (no +// receipt; the CandidatesRunning window — INSPECT) from "readiness held +// for these exact containers, crash before/while switching traffic" (the +// ReadinessPassed window — the COMPENSATE class). Containers records the +// exact candidate identities docker run returned; Probes records what +// was probed (host/port/path per replica); Outcome is "passed" (the +// receipt is never written otherwise). +type readinessReceipt struct { + SchemaVersion int `json:"schema_version"` + App string `json:"app"` + Release string `json:"release"` + Attempt string `json:"attempt"` + Containers []receiptCandidate `json:"containers"` + Probes []readinessProbe `json:"probes"` + Outcome string `json:"outcome"` + ProbedAt time.Time `json:"probed_at"` +} + +// receiptCandidate is one readiness-passed candidate container: its exact +// name and the ID docker run returned — the strongest attribution +// evidence available (names alone are version-keyed and shared by every +// attempt of the same release, register F04/A09). +type receiptCandidate struct { + Name string `json:"name"` + ID string `json:"id,omitempty"` +} + +// readinessProbe records one readiness probe the gate ran. +type readinessProbe struct { + Container string `json:"container,omitempty"` + Host string `json:"host,omitempty"` + Port int `json:"port"` + Path string `json:"path,omitempty"` +} + +// readinessReceiptPath is the receipt's location in the attempt +// namespace. +func readinessReceiptPath(att releasemeta.Attempt) string { + return att.Dir() + "/" + readinessReceiptFile +} + +// persistReadinessReceipt writes the receipt atomically (0600, sibling +// temp + rename). Called exactly once per attempt, after the health gate +// passes and before the traffic switch begins. +func (d *Deployer) persistReadinessReceipt(ctx context.Context, att releasemeta.Attempt, release string, candidates []receiptCandidate, probes []readinessProbe) error { + r := readinessReceipt{ + SchemaVersion: journalSchemaVersion, + App: att.App, + Release: release, + Attempt: att.Name(), + Containers: candidates, + Probes: probes, + Outcome: "passed", + ProbedAt: time.Now().UTC(), + } + data, err := json.Marshal(r) + if err != nil { + return fmt.Errorf("marshaling the readiness receipt: %w", err) + } + if _, err := d.exec.Run(ctx, "mkdir -p "+att.Dir()); err != nil { + return fmt.Errorf("creating the attempt directory: %w", err) + } + return ssh.UploadAtomic(ctx, d.exec, bytes.NewReader(data), readinessReceiptPath(att), "0600") +} + +// readReadinessReceipt loads the attempt's readiness receipt. Confirmed +// absent returns (nil, nil) — that absence IS the evidence (the gate +// never passed); every other failure is an error. +func readReadinessReceipt(ctx context.Context, exec ssh.Executor, att releasemeta.Attempt) (*readinessReceipt, error) { + data, present, err := state.ReadRemoteFile(ctx, exec, readinessReceiptPath(att)) + if err != nil { + return nil, fmt.Errorf("reading the readiness receipt for %s: %w", att.Name(), err) + } + if !present { + return nil, nil + } + var r readinessReceipt + if err := json.Unmarshal(data, &r); err != nil { + return nil, fmt.Errorf("parsing the readiness receipt for %s: %w", att.Name(), err) + } + if r.SchemaVersion != journalSchemaVersion { + return nil, fmt.Errorf("unsupported readiness-receipt schema version %d for %s", r.SchemaVersion, att.Name()) + } + if r.App != att.App || r.Attempt != att.Name() { + return nil, fmt.Errorf("readiness receipt identity mismatch: requested %s@%s, receipt describes %s@%s — refusing to use it", att.App, att.Name(), r.App, r.Attempt) + } + return &r, nil +} + +// attemptReadinessState derives the lifecycle state a recovery owner may +// assume about a crashed attempt from its readiness receipt (C01-4): the +// receipt is written exactly when the gate passes (before the traffic +// switch begins), so its confirmed presence proves +// recovery.ReadinessPassed; its confirmed absence collapses the state to +// recovery.CandidatesRunning — the ADR's "ReadinessPassed is unobservable +// post-crash" finding, un-collapsed for owners that read the journal. +// Decide's semantics are unchanged; this is the evidence producer its +// inputs always modeled. +func attemptReadinessState(receipt *readinessReceipt) recovery.State { + if receipt != nil { + return recovery.ReadinessPassed + } + return recovery.CandidatesRunning +} + +// candidateAttribution derives the recovery.Candidates evidence class +// for a crashed ATTEMPT from its readiness receipt and the live +// inventory: is a running workload provably THIS attempt's candidate? +// Nothing candidate-shaped running is provably Absent. A running +// candidate-shaped container is PROVABLY the attempt's only when its +// identity matches the receipt (the ID docker run returned; the name +// alone only when no ID exists to compare) — Present. Without a receipt, +// or when the running containers provably belong to a different attempt +// of the same release (version-keyed names are shared by every attempt +// of one release, register F04/A09 — the release-scoped reading of the +// class is that finding's deferred identity work), attribution is +// Unknown — the never-auto-decide class (R4: INSPECT). +func candidateAttribution(receipt *readinessReceipt, inv []docker.Container, app, release string) recovery.Evidence { + prefix := fmt.Sprintf("%s-web-%s", app, release) + isRunningCandidate := func(c docker.Container) bool { + return c.State == "running" && strings.HasPrefix(c.Name, prefix) + } + running := false + for _, c := range inv { + if isRunningCandidate(c) { + running = true + break + } + } + if !running { + return recovery.Absent + } + if receipt == nil { + return recovery.Unknown + } + ids := map[string]bool{} + names := map[string]bool{} + for _, c := range receipt.Containers { + if c.ID != "" { + ids[c.ID] = true + } + names[c.Name] = true + } + for _, c := range inv { + if !isRunningCandidate(c) { + continue + } + switch { + case ids[c.ID]: + return recovery.Present + case c.ID == "" && names[c.Name]: + // No ID on either side to disagree: the name is all the + // evidence that exists. + return recovery.Present + } + } + return recovery.Unknown +} diff --git a/internal/deploy/journal_receipt_test.go b/internal/deploy/journal_receipt_test.go new file mode 100644 index 0000000..c36f66e --- /dev/null +++ b/internal/deploy/journal_receipt_test.go @@ -0,0 +1,260 @@ +package deploy + +import ( + "bytes" + "context" + "encoding/json" + "strings" + "testing" + "time" + + "github.com/useteploy/teploy/internal/deploy/recovery" + "github.com/useteploy/teploy/internal/docker" + "github.com/useteploy/teploy/internal/releasemeta" + "github.com/useteploy/teploy/internal/ssh" +) + +// receiptUnderTest finds and parses the readiness receipt a deploy wrote. +func receiptUnderTest(t *testing.T, mock *ssh.MockExecutor, app string) (string, readinessReceipt) { + t.Helper() + for p := range mock.Files { + if strings.HasPrefix(p, "/deployments/"+app+"/meta/att/") && strings.HasSuffix(p, "/readiness.json") { + var r readinessReceipt + if err := json.Unmarshal(mock.Files[p], &r); err != nil { + t.Fatalf("parsing receipt %s: %v", p, err) + } + return p, r + } + } + t.Fatalf("no readiness receipt persisted under /deployments/%s/meta/att/", app) + return "", readinessReceipt{} +} + +// TestReadinessReceipt_WrittenExactlyOnReadinessPass is the C01-4 +// write-side regression: the receipt lands in the attempt namespace the +// moment the health gate passes — AFTER the last readiness probe and +// BEFORE the traffic switch issues its first command — and records the +// candidate identities plus what was probed. +func TestReadinessReceipt_WrittenExactlyOnReadinessPass(t *testing.T) { + existingState := "current_port=49152\ncurrent_hash=old123\nprevious_port=0\nprevious_hash=\n" + mock := ssh.NewMockExecutor("1.2.3.4", + ssh.MockCommand{Match: "mkdir -p /deployments/myapp", Output: ""}, + ssh.MockCommand{Match: "mkdir /deployments/myapp/.lock", Output: ""}, + ssh.MockCommand{Match: "if [ ! -e '/deployments/myapp/state.json' ]", Output: "absent"}, + ssh.MockCommand{Match: "if [ ! -e '/deployments/myapp/state' ]", Output: "present\n" + existingState}, + ssh.MockCommand{Match: "docker ps --all --filter label=teploy.app='myapp'", Output: predecessorInventoryJSON}, + ssh.MockCommand{Match: "ss -tln", Output: ssOutput}, + ssh.MockCommand{Match: "docker run", Output: "newcontainer123"}, + ssh.MockCommand{Match: "docker inspect", Output: "running"}, + ssh.MockCommand{Match: "curl -s -o /dev/null", Output: "200"}, + ssh.MockCommand{Match: "curl -sf http://localhost:2019/config/apps/http/servers/srv0", Output: `{"listen":[":80",":443"]}`}, + ssh.MockCommand{Match: "curl -sf -X PATCH", Err: errBoom}, + ssh.MockCommand{Match: "curl -sf -X POST http://localhost:2019/config/apps/http/servers/srv0/routes", Output: ""}, + ssh.MockCommand{Match: "rm -f /tmp/teploy_caddy", Output: ""}, + ssh.MockCommand{Match: "cat /deployments/caddy/Caddyfile", Output: "{\n\tadmin 0.0.0.0:2019\n}\n"}, + ssh.MockCommand{Match: "mv /tmp/teploy_caddyfile.tmp", Output: ""}, + ssh.MockCommand{Match: "mkdir /deployments/caddy/.lock", Output: ""}, + ssh.MockCommand{Match: "a=$(docker exec caddy md5sum", Output: "TEPLOY_CADDY_OK"}, + ssh.MockCommand{Match: "docker exec caddy caddy reload", Output: ""}, + ssh.MockCommand{Match: "rmdir /deployments/caddy/.lock", Output: ""}, + ssh.MockCommand{Match: "docker stop", Output: ""}, + ssh.MockCommand{Match: "printf %s", Output: ""}, + ssh.MockCommand{Match: "rm -rf /deployments/myapp/.lock", Output: ""}, + ) + + var buf bytes.Buffer + if err := NewDeployer(mock, &buf).Deploy(context.Background(), Config{ + App: "myapp", + Domain: "myapp.com", + Image: "myapp:v2", + Version: "new456", + Health: HealthConfig{Timeout: 5 * time.Second, Interval: 10 * time.Millisecond}, + }); err != nil { + t.Fatalf("Deploy: %v", err) + } + + _, r := receiptUnderTest(t, mock, "myapp") + if r.App != "myapp" || r.Release != "new456" || r.Outcome != "passed" { + t.Errorf("receipt identity/outcome: app=%q release=%q outcome=%q", r.App, r.Release, r.Outcome) + } + if len(r.Containers) != 1 || r.Containers[0].Name != "myapp-web-new456" || r.Containers[0].ID != "newcontainer123" { + t.Errorf("receipt must record the exact candidate identities (docker run's IDs), got %+v", r.Containers) + } + if len(r.Probes) != 1 || r.Probes[0].Port != 49152 || r.Probes[0].Path != "/health" || r.Probes[0].Host != "localhost" { + t.Errorf("receipt must record what was probed, got %+v", r.Probes) + } + if r.ProbedAt.IsZero() { + t.Error("receipt must carry the probe timestamp") + } + + // Ordering: after the LAST readiness probe, before the FIRST edge + // command (the traffic switch begins with the Caddyfile transaction: + // the caddy lock + read of the current file). + commitIdx, lastProbeIdx, firstEdgeIdx := -1, -1, -1 + for i, c := range mock.Calls { + if strings.HasPrefix(c, "mv -fT -- ") && strings.Contains(c, "/readiness.json.tmp-") { + commitIdx = i + } + if strings.HasPrefix(c, "curl -s -o /dev/null") { + lastProbeIdx = i + } + if firstEdgeIdx < 0 && strings.HasPrefix(c, "cat /deployments/caddy/Caddyfile") { + firstEdgeIdx = i + } + } + if commitIdx < 0 || lastProbeIdx < 0 || firstEdgeIdx < 0 { + t.Fatalf("missing calls for ordering (receipt=%d probe=%d edge=%d)", commitIdx, lastProbeIdx, firstEdgeIdx) + } + if commitIdx < lastProbeIdx { + t.Errorf("receipt must not exist before readiness passes: receipt=%d lastProbe=%d", commitIdx, lastProbeIdx) + } + if commitIdx > firstEdgeIdx { + t.Errorf("receipt must be durable before the traffic switch begins: receipt=%d firstEdge=%d", commitIdx, firstEdgeIdx) + } +} + +// TestReadinessReceipt_NotWrittenWhenReadinessFails pins the other side: +// a deploy whose health gate never passes leaves NO receipt — its crash +// window is CandidatesRunning's, and a recovery owner must see exactly +// that. +func TestReadinessReceipt_NotWrittenWhenReadinessFails(t *testing.T) { + mock := ssh.NewMockExecutor("1.2.3.4", + ssh.MockCommand{Match: "mkdir -p /deployments/myapp", Output: ""}, + ssh.MockCommand{Match: "mkdir /deployments/myapp/.lock", Output: ""}, + ssh.MockCommand{Match: "if [ ! -e '/deployments/myapp/state.json' ]", Output: "absent"}, + ssh.MockCommand{Match: "if [ ! -e '/deployments/myapp/state' ]", Output: "absent"}, + ssh.MockCommand{Match: "ss -tln", Output: ssOutput}, + ssh.MockCommand{Match: "docker run", Output: "failcontainer"}, + ssh.MockCommand{Match: "docker inspect -f '{{json .State}}'", Output: `{"Status":"running","Running":true}`}, + ssh.MockCommand{Match: "docker inspect -f", Output: "running"}, + ssh.MockCommand{Match: "curl -s -o /dev/null", Err: errBoom}, + ssh.MockCommand{Match: "docker logs", Output: "Error: app crashed on startup"}, + ssh.MockCommand{Match: "docker stop", Output: ""}, + ssh.MockCommand{Match: "docker rm", Output: ""}, + ssh.MockCommand{Match: "printf %s", Output: ""}, + ssh.MockCommand{Match: "rm -rf /deployments/myapp/.lock", Output: ""}, + ) + if err := NewDeployer(mock, &bytes.Buffer{}).Deploy(context.Background(), Config{ + App: "myapp", + Domain: "myapp.com", + Image: "myapp:latest", + Version: "abc123", + Health: HealthConfig{Timeout: 300 * time.Millisecond, Interval: 50 * time.Millisecond}, + }); err == nil { + t.Fatal("expected the health-failing deploy to fail") + } + for p := range mock.Files { + if strings.HasSuffix(p, "/readiness.json") { + t.Fatalf("no readiness receipt may exist when the gate never passed, found %s", p) + } + } +} + +// TestReadinessReceipt_ReadValidatesIdentity covers the read helper: +// confirmed absent is (nil, nil); a foreign-identity receipt is refused. +func TestReadinessReceipt_ReadValidatesIdentity(t *testing.T) { + att := releasemeta.MustAttempt("myapp", "new456") + path := readinessReceiptPath(att) + + mock := ssh.NewMockExecutor("1.2.3.4") + r, err := readReadinessReceipt(context.Background(), mock, att) + if err != nil || r != nil { + t.Fatalf("confirmed-absent receipt must be (nil, nil), got (%v, %v)", r, err) + } + + foreign, _ := json.Marshal(readinessReceipt{SchemaVersion: journalSchemaVersion, App: "otherapp", Release: "new456", Attempt: att.Name(), Outcome: "passed"}) + mock.Files[path] = foreign + if _, err := readReadinessReceipt(context.Background(), mock, att); err == nil { + t.Fatal("a receipt describing another app/attempt must be refused, not used") + } +} + +// TestReadinessReceipt_DecideDistinguishesCompensateFromInspect is the +// C01-4 recovery wiring proof, asserted through the recovery package's +// Decide: the world a crash-after-readiness leaves (traffic switched onto +// an uncommitted generation) takes the COMPENSATE branch when the +// attempt's receipt lets the owner attribute the running candidates, and +// collapses to INSPECT without it — the receipt is the evidence that +// readiness held for THESE containers; Decide's own semantics are +// unchanged (it already models this evidence; the receipt is its +// producer). +func TestReadinessReceipt_DecideDistinguishesCompensateFromInspect(t *testing.T) { + app, attempted, predecessor := "myapp", "new456", "old123" + receipt := &readinessReceipt{ + SchemaVersion: journalSchemaVersion, + App: app, + Release: attempted, + Outcome: "passed", + Containers: []receiptCandidate{ + {Name: "myapp-web-new456", ID: "cid-new-1"}, + }, + Probes: []readinessProbe{{Container: "myapp-web-new456", Host: "localhost", Port: 49152, Path: "/health"}}, + ProbedAt: time.Now().UTC(), + } + // The crashed world: the candidate runs under the receipt's exact ID, + // the edge names it, state.json still names the predecessor + // (crash-before-commit), and the predecessor still serves elsewhere. + inv := []docker.Container{ + {ID: "cid-new-1", Name: "myapp-web-new456", State: "running", Labels: map[string]string{"teploy.app": app, "teploy.version": attempted}}, + {ID: "cid-old-1", Name: "myapp-web-old123", State: "running", Labels: map[string]string{"teploy.app": app, "teploy.version": predecessor}}, + } + observation := func(candidates recovery.Evidence) recovery.Observation { + return recovery.Observation{ + Candidates: candidates, + CandidateCorpses: recovery.Absent, + ForeignCandidates: recovery.Absent, + RouteToCandidate: recovery.Present, + RouteToPredecessor: recovery.Absent, + StateToCandidate: recovery.Absent, + StateToPredecessor: recovery.Present, + PredecessorServing: recovery.Present, + PredecessorStopped: recovery.Absent, + ReleaseRecord: recovery.Absent, + } + } + + // With the receipt: the owner's state is ReadinessPassed (proven) and + // the candidates are attributable (exact IDs) -> COMPENSATE. + if got := recovery.Decide(attemptReadinessState(receipt), observation(candidateAttribution(receipt, inv, app, attempted))); got != recovery.Compensate { + t.Errorf("with the readiness receipt, crash-after-readiness must COMPENSATE, got %s", got) + } + // Without it (confirmed absent): the state collapses to + // CandidatesRunning and the running candidates are not attributable + // to the attempt -> INSPECT, never auto-decided. + noReceiptState := attemptReadinessState(nil) + if noReceiptState != recovery.CandidatesRunning { + t.Errorf("without a receipt the state must collapse to CandidatesRunning, got %s", noReceiptState) + } + if got := recovery.Decide(noReceiptState, observation(candidateAttribution(nil, inv, app, attempted))); got != recovery.Inspect { + t.Errorf("without the readiness receipt the same world must INSPECT, got %s", got) + } +} + +// TestCandidateAttribution pins the evidence derivation's truth table. +func TestCandidateAttribution(t *testing.T) { + app, release := "myapp", "new456" + running := docker.Container{ID: "cid-1", Name: "myapp-web-new456", State: "running"} + receipt := &readinessReceipt{Containers: []receiptCandidate{{Name: "myapp-web-new456", ID: "cid-1"}}} + + if got := candidateAttribution(receipt, []docker.Container{running}, app, release); got != recovery.Present { + t.Errorf("running container matching the receipt's ID: want Present, got %s", got) + } + if got := candidateAttribution(receipt, nil, app, release); got != recovery.Absent { + t.Errorf("nothing candidate-shaped running: want Absent, got %s", got) + } + if got := candidateAttribution(nil, []docker.Container{running}, app, release); got != recovery.Unknown { + t.Errorf("running candidate without a receipt is not attributable to the attempt: want Unknown, got %s", got) + } + // Another attempt's candidates of the same release (version-keyed + // names, register F04/A09): provably not the receipt's — still never + // auto-decided. + other := docker.Container{ID: "cid-other", Name: "myapp-web-new456", State: "running"} + if got := candidateAttribution(receipt, []docker.Container{other}, app, release); got != recovery.Unknown { + t.Errorf("running candidate provably not the receipt's: want Unknown, got %s", got) + } + // Stopped corpses are not the Candidates class. + stopped := docker.Container{ID: "cid-1", Name: "myapp-web-new456", State: "exited"} + if got := candidateAttribution(receipt, []docker.Container{stopped}, app, release); got != recovery.Absent { + t.Errorf("only running containers are Candidates: want Absent, got %s", got) + } +} diff --git a/internal/deploy/journal_snapshot_test.go b/internal/deploy/journal_snapshot_test.go new file mode 100644 index 0000000..6600d60 --- /dev/null +++ b/internal/deploy/journal_snapshot_test.go @@ -0,0 +1,206 @@ +package deploy + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "strings" + "testing" + "time" + + "github.com/useteploy/teploy/internal/releasemeta" + "github.com/useteploy/teploy/internal/ssh" + "github.com/useteploy/teploy/internal/state" +) + +// snapshotUnderTest finds the predecessor snapshot a deploy wrote in the +// mock's recorded file state and parses it. The attempt id is random, so +// the path is discovered by prefix/suffix, not constructed. +func snapshotUnderTest(t *testing.T, mock *ssh.MockExecutor, app string) (path string, snap predecessorSnapshot, att releasemeta.Attempt) { + t.Helper() + for p := range mock.Files { + if strings.HasPrefix(p, "/deployments/"+app+"/meta/att/") && strings.HasSuffix(p, "/predecessors.json") { + path = p + break + } + } + if path == "" { + t.Fatalf("no predecessor snapshot persisted under /deployments/%s/meta/att/", app) + } + if err := json.Unmarshal(mock.Files[path], &snap); err != nil { + t.Fatalf("parsing snapshot %s: %v", path, err) + } + // /./predecessors.json + dir := path[:strings.LastIndex(path, "/")] + name := dir[strings.LastIndex(dir, "/")+1:] + hash, id, ok := strings.Cut(name, ".") + if !ok { + t.Fatalf("attempt dir name %q does not parse as .", name) + } + return path, snap, releasemeta.Attempt{App: app, Hash: hash, ID: id} +} + +// restoreFixtureInspect is a minimal recreate-able container inspect for +// the displaced workload (the shape docker.InspectRecreate consumes). +func restoreFixtureInspect(name, version string) string { + return fmt.Sprintf(`[{ + "Image": "sha256:%s", + "Config": {"Image": "web:old", "Cmd": ["npm", "start"], "Labels": {"teploy.app": "web", "teploy.process": "web", "teploy.version": %q}}, + "HostConfig": {"NetworkMode": "teploy", "PortBindings": {"3000/tcp": [{"HostIp": "0.0.0.0", "HostPort": "3000"}]}, "RestartPolicy": {"Name": "no"}}, + "NetworkSettings": {"Networks": {"teploy": {"Aliases": ["web"]}}} +}]`, strings.Repeat("b", 64), version) +} + +// TestPredecessorSnapshot_PersistedAtRenamePhaseBeforeAnyContainerStarts is +// the C01-10 write-side regression: the predecessor snapshot (exact +// container identities) must be durable in the attempt's immutable +// artifact namespace from the rename phase onward — written after the +// same-version renames / predecessor listing, BEFORE the recreate +// displacement stops anything and before any candidate container starts. +func TestPredecessorSnapshot_PersistedAtRenamePhaseBeforeAnyContainerStarts(t *testing.T) { + existingState := "current_port=3000\ncurrent_hash=old123\n" + mock := ssh.NewMockExecutor("1.2.3.4", + ssh.MockCommand{Match: "mkdir -p /deployments/web", Output: ""}, + ssh.MockCommand{Match: "mkdir /deployments/web/.lock", Output: ""}, + ssh.MockCommand{Match: "if [ ! -e '/deployments/web/state.json' ]", Output: "absent"}, + ssh.MockCommand{Match: "if [ ! -e '/deployments/web/state' ]", Output: "present\n" + existingState}, + // Step 6b listing: the predecessor web workload of old123, running. + ssh.MockCommand{Match: "docker ps --all --filter label=teploy.app='web'", Output: `{"ID":"cid-old123","Names":"web-web-old123","Image":"web:old","State":"running","Status":"Up","CreatedAt":"2026-09-20 10:00:00 +0000 UTC","Labels":{"teploy.app":"web","teploy.process":"web","teploy.version":"old123"}}` + "\n"}, + // Recreate displacement query + stop. + ssh.MockCommand{Match: "docker ps --filter label=teploy.app=web", Output: "web-web-old123"}, + ssh.MockCommand{Match: "docker stop", Output: ""}, + ssh.MockCommand{Match: "docker run", Output: "newcid"}, + ssh.MockCommand{Match: "docker inspect", Output: "running"}, + ssh.MockCommand{Match: "curl -s -o /dev/null", Output: "200"}, + ssh.MockCommand{Match: "printf %s", Output: ""}, + ssh.MockCommand{Match: "rm -rf /deployments/web/.lock", Output: ""}, + ) + + var buf bytes.Buffer + if err := NewDeployer(mock, &buf).Deploy(context.Background(), Config{ + App: "web", + Image: "web:new", + Version: "new456", + Ingress: "host", + ContainerPort: 3000, + Health: HealthConfig{Timeout: 5 * time.Second, Interval: 10 * time.Millisecond}, + }); err != nil { + t.Fatalf("Deploy: %v", err) + } + + _, snap, att := snapshotUnderTest(t, mock, "web") + if snap.App != "web" || snap.Release != "old123" || snap.Attempt != att.Name() { + t.Errorf("snapshot identity: app=%q release=%q attempt=%q (want web/old123/%s)", snap.App, snap.Release, snap.Attempt, att.Name()) + } + if len(snap.Containers) != 1 || snap.Containers[0].ID != "cid-old123" || snap.Containers[0].Name != "web-web-old123" { + t.Errorf("snapshot must record the exact predecessor identities, got %+v", snap.Containers) + } + if snap.Containers[0].Labels["teploy.version"] != "old123" { + t.Errorf("snapshot must carry the releasemeta-record identity of the predecessor, got labels %+v", snap.Containers[0].Labels) + } + + // Ordering: the snapshot's atomic rename landed BEFORE the first + // displacement stop and BEFORE the first candidate docker run. + commitIdx, stopIdx, runIdx := -1, -1, -1 + for i, c := range mock.Calls { + if commitIdx < 0 && strings.HasPrefix(c, "mv -fT -- ") && strings.Contains(c, "/predecessors.json.tmp-") { + commitIdx = i + } + if stopIdx < 0 && strings.HasPrefix(c, "docker stop") { + stopIdx = i + } + if runIdx < 0 && strings.HasPrefix(c, "docker run") { + runIdx = i + } + } + if commitIdx < 0 || stopIdx < 0 || runIdx < 0 { + t.Fatalf("missing calls for ordering (snapshot=%d stop=%d run=%d)", commitIdx, stopIdx, runIdx) + } + if commitIdx > stopIdx || commitIdx > runIdx { + t.Errorf("snapshot must be durable before any container starts or stops: snapshot=%d stop=%d run=%d", commitIdx, stopIdx, runIdx) + } +} + +// TestPredecessorSnapshot_CrashRecoveryCompensatesExactlyRecordedIDs is +// the C01-10 recover-side regression: a NEW process (no in-memory +// displaced list) recovers a crashed attempt by reading the snapshot from +// disk and compensating EXACTLY the recorded container identities — not a +// re-derivation. The live world also carries a stopped old123 WORKER +// that the name-derived fallback would touch; the snapshot does not name +// it, so recovery must not. +func TestPredecessorSnapshot_CrashRecoveryCompensatesExactlyRecordedIDs(t *testing.T) { + existingState := "current_port=3000\ncurrent_hash=old123\n" + + // Phase 1 — the crashed attempt: the deploy writes the snapshot, then + // dies during the recreate displacement (the stop fails; the process + // "crashes" with the error, abandoning all in-memory knowledge). + crashed := ssh.NewMockExecutor("1.2.3.4", + ssh.MockCommand{Match: "mkdir -p /deployments/web", Output: ""}, + ssh.MockCommand{Match: "mkdir /deployments/web/.lock", Output: ""}, + ssh.MockCommand{Match: "if [ ! -e '/deployments/web/state.json' ]", Output: "absent"}, + ssh.MockCommand{Match: "if [ ! -e '/deployments/web/state' ]", Output: "present\n" + existingState}, + ssh.MockCommand{Match: "docker ps --all --filter label=teploy.app='web'", Output: `{"ID":"cid-old123","Names":"web-web-old123","Image":"web:old","State":"running","Status":"Up","CreatedAt":"2026-09-20 10:00:00 +0000 UTC","Labels":{"teploy.app":"web","teploy.process":"web","teploy.version":"old123"}}` + "\n"}, + ssh.MockCommand{Match: "docker ps --filter label=teploy.app=web", Output: "web-web-old123"}, + ssh.MockCommand{Match: "docker stop", Err: errBoom}, + ssh.MockCommand{Match: "printf %s", Output: ""}, + ssh.MockCommand{Match: "rm -rf /deployments/web/.lock", Output: ""}, + ) + var crashedOut bytes.Buffer + if err := NewDeployer(crashed, &crashedOut).Deploy(context.Background(), Config{ + App: "web", Image: "web:new", Version: "new456", Ingress: "host", ContainerPort: 3000, + }); err == nil { + t.Fatal("the crashed attempt must fail (its displacement stop failed)") + } + _, snap, att := snapshotUnderTest(t, crashed, "web") + if len(snap.Containers) != 1 { + t.Fatalf("crashed attempt's snapshot must name the predecessor, got %+v", snap.Containers) + } + + // Phase 2 — a NEW process recovers: fresh executor sharing only the + // durable file state (the snapshot on disk). It has no in-memory + // displaced list. The live docker world: the snapshot's web container + // is STOPPED (displaced before the crash), and a stopped old123 + // worker exists that the snapshot does NOT name. + snapshotPath := fmt.Sprintf("/deployments/web/meta/att/%s/predecessors.json", att.Name()) + recovered := ssh.NewMockExecutor("1.2.3.4", + // The snapshot's web container: stopped -> restorable. + ssh.MockCommand{Match: "docker inspect -f '{{.State.Status}}' 'web-web-old123'", Output: "exited"}, + ssh.MockCommand{Match: "docker inspect 'web-web-old123'", Output: restoreFixtureInspect("web-web-old123", "old123")}, + ssh.MockCommand{Match: "docker rm -f", Output: ""}, + ssh.MockCommand{Match: "docker run", Output: "restored-old123"}, + ssh.MockCommand{Match: "printf %s", Output: ""}, + ) + recovered.Files[snapshotPath] = crashed.Files[snapshotPath] + + var out bytes.Buffer + d := NewDeployer(recovered, &out) + err := d.abortStateCommit(context.Background(), Config{ + App: "web", Image: "web:new", Version: "new456", Ingress: "host", ContainerPort: 3000, + }, &state.AppState{SchemaVersion: 2, CurrentHash: "old123", IngressMode: "host"}, nil, nil, att, time.Now(), errBoom) + if err == nil { + t.Fatal("expected the commit error to surface") + } + if !strings.Contains(err.Error(), "original workload was restored") { + t.Errorf("recovery must report the restored workload, got: %v", err) + } + + // Exactly the recorded container was restored. + restored := false + for _, c := range recovered.Calls { + if strings.HasPrefix(c, "docker run") && strings.Contains(c, "--name 'web-web-old123'") { + restored = true + } + } + if !restored { + t.Fatalf("the snapshot's displaced web container was not restored: %v", recovered.Calls) + } + // Nothing outside the snapshot was touched: the stopped old123 worker + // (which the name-derived fallback stopOldWorkloadsByName derives + // from the process list) was never inspected, stopped, or restored. + for _, c := range recovered.Calls { + if strings.Contains(c, "worker") { + t.Errorf("recovery touched a container the snapshot does not name: %s", c) + } + } +} diff --git a/internal/deploy/recovery/recovery.go b/internal/deploy/recovery/recovery.go index e0ee4df..f9411be 100644 --- a/internal/deploy/recovery/recovery.go +++ b/internal/deploy/recovery/recovery.go @@ -45,9 +45,11 @@ const ( // {app}-{process}-{version}[-{index}] with teploy.* labels // (internal/docker/docker.go:82-117, internal/deploy/deploy.go:573-607). CandidatesRunning - // ReadinessPassed: the health gate passed. NO durable receipt exists - // today (internal/deploy/health.go probes are ephemeral) — see ADR - // finding on the readiness receipt. + // ReadinessPassed: the health gate passed. The durable receipt is the + // attempt-scoped readiness.json (internal/deploy/journal.go, C01-4), + // written exactly on pass and before the traffic switch; the evidence + // derivation that turns it into this state and Candidates attribution + // lives alongside it (attemptReadinessState / candidateAttribution). ReadinessPassed // TrafficSwitched: the edge routes name the candidates — the managed // marker block in /deployments/caddy/Caddyfile plus reload + delivery @@ -410,10 +412,10 @@ func Lattice() []Transition { { From: CandidatesRunning, To: ReadinessPassed, Evidence: []string{ - "NONE DURABLE — health-gate results are ephemeral (deploy/health.go)", + "readiness receipt /deployments//meta/att/./readiness.json — exact candidate IDs + probes + outcome, written on pass before the switch (deploy/journal.go, C01-4; internal/deploy/health.go probes remain ephemeral)", }, CrashDisposition: Inspect, - Note: "the only transition with no receipt today; a recovery owner must re-probe. ADR finding: readiness receipt is a design obligation", + Note: "receipt LANDED (C01-4): its presence derives ReadinessPassed and attributes the running candidates (attemptReadinessState/candidateAttribution → Decide); its absence keeps the window INSPECT. Wiring a recovery OWNER that reads it on acquisition is the C01-1 slice", }, { From: ReadinessPassed, To: TrafficSwitched, @@ -441,7 +443,7 @@ func Lattice() []Transition { "absence of predecessor names in docker ps label inventory (docker/docker.go:568-571)", }, CrashDisposition: Retry, - Note: "retryable: retirement re-derives from the inventory; failures are reported, never silent. ADR finding: the success log entry records no degraded flag", + Note: "retryable: retirement re-derives from the inventory; failures are reported, never silent — and since C01-5 the terminal log entry records them as a DEGRADED success instead of clean success; the durable predecessor snapshot (deploy/journal.go, C01-10) preserves the exact retirement set across crashes", }, { From: PredecessorRetired, To: TerminalReceiptPersisted, diff --git a/internal/deploy/recovery_a_test.go b/internal/deploy/recovery_a_test.go index 8bfbc33..276991e 100644 --- a/internal/deploy/recovery_a_test.go +++ b/internal/deploy/recovery_a_test.go @@ -9,6 +9,7 @@ import ( "testing" "time" + "github.com/useteploy/teploy/internal/releasemeta" "github.com/useteploy/teploy/internal/ssh" "github.com/useteploy/teploy/internal/state" ) @@ -113,7 +114,7 @@ func TestAbortStateCommit_CancelledContextStillRunsCompensation(t *testing.T) { ContainerPort: 8080, } started := []string{"fency-web-abc123"} - err := d.abortStateCommit(ctx, cfg, nil, started, nil, time.Now(), errBoom) + err := d.abortStateCommit(ctx, cfg, nil, started, nil, releasemeta.MustAttempt(app, cfg.Version), time.Now(), errBoom) if err == nil { t.Fatal("expected the commit error to be returned") } @@ -159,7 +160,7 @@ func TestAbortStateCommit_CaddyPublishAppRestoresRoute(t *testing.T) { Health: HealthConfig{Timeout: 5 * time.Second, Interval: 10 * time.Millisecond}, } started := []string{"fency-web-abc123"} - err := d.abortStateCommit(context.Background(), cfg, current, started, nil, time.Now(), errBoom) + err := d.abortStateCommit(context.Background(), cfg, current, started, nil, releasemeta.MustAttempt(app, cfg.Version), time.Now(), errBoom) if err == nil { t.Fatal("expected the commit error to surface") } @@ -228,7 +229,7 @@ func TestLogDeploy_RecordsImage(t *testing.T) { ssh.MockCommand{Match: "printf %s", Output: ""}, ) d := &Deployer{exec: mock, out: &bytes.Buffer{}} - d.logDeploy(context.Background(), Config{App: "myapp", Image: "myapp:latest", Version: "abc123"}, true, time.Now()) + d.logDeploy(context.Background(), Config{App: "myapp", Image: "myapp:latest", Version: "abc123"}, true, "", time.Now()) var line string for _, c := range mock.Calls { if strings.HasPrefix(c, "printf %s '") { @@ -251,4 +252,3 @@ func TestLogDeploy_RecordsImage(t *testing.T) { t.Errorf("log entry image: got %q want myapp:latest", entry.Image) } } - diff --git a/internal/state/state.go b/internal/state/state.go index 381962a..5155978 100644 --- a/internal/state/state.go +++ b/internal/state/state.go @@ -111,6 +111,17 @@ type LogEntry struct { // for entries with no image (heal, lifecycle, static), so old log lines // parse unchanged. Image string `json:"image,omitempty"` + // Degraded marks a deploy that switched traffic and committed + // successfully but whose post-commit predecessor retirement (or its + // fallback) partially failed (C01-5): the app serves the new + // generation, yet a superseded container escaped retirement. Success + // stays true — this is not a deploy failure — but consumers filtering + // on Success alone (fleet rollback targeting keyed on the log, log + // rendering, dash reads) must not mistake it for a clean outcome: a + // degraded host is still running part of the superseded generation. + Degraded bool `json:"degraded,omitempty"` + // DegradedReason itemizes what escaped retirement when Degraded is set. + DegradedReason string `json:"degraded_reason,omitempty"` } // ReadRemoteFile returns the file's contents and whether it exists. Absence From cce0726445dd02b949f6cc3e907a394662f1fc60 Mon Sep 17 00:00:00 2001 From: Tyler <53561637+im-tyler@users.noreply.github.com> Date: Tue, 22 Sep 2026 17:55:06 -0700 Subject: [PATCH 2/8] docs: changelog v0.1.37 (commit-pinned builds, blue/green previews, crash-recovery evidence) --- CHANGELOG.md | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8fab40c..12e51a5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,38 @@ All notable changes to teploy are documented here. Format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). +## [0.1.37] - 2026-09-22 + +### Fixed + +- **Webhook deploys build the authenticated commit.** A delivery now + pins the build to the payload's exact commit (fetch-verify-reset); + if that commit was force-pushed away the deploy fails loudly naming + both commits instead of silently building the moving branch tip. + The pin rides the durable admission ledger through supersede and + crash-resume. +- **Preview updates no longer take the preview down.** A preview update + now runs blue/green: the candidate starts under a version-suffixed + name with its own network alias, passes a readiness gate, the route + switches, and only then is the predecessor retired — a failed + candidate leaves the old preview serving. `teploy preview prune` + prunes expired previews across all apps (both record eras, + idempotent, 72h default TTL) and is cron-able; the deploy-time prune + uses the same core. +- **SSH host-key mismatches now name what was presented and what + known_hosts holds** (key algorithms included), instead of a bare + mismatch error (found via ship's delivery provisioning). + +### Added + +- **Crash-recovery evidence for deploys (C01 design obligations):** + readiness receipts (exact candidate IDs + probe outcomes) and + predecessor snapshots (exact container IDs) persist per attempt at + the moment they become true, so recovery can distinguish + compensable states from inspect-only ones and restore exactly what + was displaced; the deploy log records DEGRADED outcomes (traffic + switched but retirement partially failed) instead of clean success. + ## [0.1.36] - 2026-09-22 ### Fixed From a914631ef01842c1b8549e34535b19029ad8d60f Mon Sep 17 00:00:00 2001 From: Tyler <53561637+im-tyler@users.noreply.github.com> Date: Tue, 22 Sep 2026 19:24:22 -0700 Subject: [PATCH 3/8] feat(deploy,cli): repair-debt reconciler + receipt-driven route compensation (C01-6/7) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A record-write failure after the live commit now persists a repair-debt marker (never rolling back live traffic); the next deploy reconciles it under the app lock — backfill, clear, report — and teploy status shows outstanding debt in text and JSON. Route compensation renders from the predecessor's recorded F14 receipt (record is authoritative for what was switched away from); reconstruct-from-inspect survives only as an announced fallback. Remaining C01: -1/2/3 (locking protocol) and -8/9 (F04 identities). --- AUDIT_OPEN.md | 81 ++++++++ docs/C01_RECOVERY_STATE_TABLE.md | 32 ++- internal/cli/status.go | 56 +++-- internal/cli/status_test.go | 89 ++++++++ internal/deploy/deploy.go | 115 ++++++++++- internal/deploy/repairdebt.go | 208 +++++++++++++++++++ internal/deploy/repairdebt_test.go | 284 ++++++++++++++++++++++++++ internal/deploy/route_receipt_test.go | 178 ++++++++++++++++ 8 files changed, 1017 insertions(+), 26 deletions(-) create mode 100644 internal/cli/status_test.go create mode 100644 internal/deploy/repairdebt.go create mode 100644 internal/deploy/repairdebt_test.go create mode 100644 internal/deploy/route_receipt_test.go diff --git a/AUDIT_OPEN.md b/AUDIT_OPEN.md index e21817d..bb08503 100644 --- a/AUDIT_OPEN.md +++ b/AUDIT_OPEN.md @@ -1429,3 +1429,84 @@ attributable by evidence (F04/A09). C01-6 (record-write convergence has no reconciler) and C01-7 (compensation reconstructs the predecessor route instead of using a receipt) also remain, with their register items (A12/T05 standing for C01-7). + +## Programme slice (2026-09-22, latest) — C01 implementation: record-repair debt + receipt-driven route compensation + +The two remaining contained C01 findings landed as bounded slices (spec: +docs/C01_RECOVERY_STATE_TABLE.md findings 6 and 7; recovery.Decide and its +exhaustive suite untouched). Base revision `cce0726` (v0.1.37); changes +left uncommitted for review. + +- **C01-6 — record-write failure now converges.** A releasemeta record + write that fails after the live commit still never fails the deploy + (deliberate degradation — record failure must not roll back live + traffic), but the debt is now DURABLE and visible: + `internal/deploy/repairdebt.go` persists a marker at + `/deployments//repair-debt.json` (atomic 0600, identity-validated + on read, T56 parity) naming app, release, attempt, what failed, when, + and the failed write/repair attempt count. The NEXT deploy repairs it + BEFORE its own work (DeployFenced step 1b, under the app lock): if the + record already exists (a rollback/backfill converged it meanwhile) the + marker is just cleared; otherwise the record is rebuilt from the live + containers via releasemeta.Backfill — the table's transition-7 + convergence — and the marker cleared on success (reported in output); + a repeated failure keeps the marker with an incremented count and says + so (never a deploy failure — the debt describes the previous deploy). + A re-failing record write of the SAME release bumps the existing + marker instead of resetting its history. `teploy status` (writeStatus, + extracted from runStatus for testability) reports outstanding debt in + text and JSON (`repair_debt`), an unreadable marker is reported rather + than hidden, and no marker means zero output noise. +- **C01-7 — route compensation uses the recorded receipt.** + `restorePreviousRoute` (deploy.go, both abortStateCommit call sites) + now renders the previous route from the predecessor release's F14 + RECORD — domain, replica upstream names, the recorded primary + container port (TCL-14), TLS/caddy_extra/cache/firewall/access, and + the LB health path — consulting zero live inspect (the record is + authoritative for what teploy switched away FROM; compensating from + cfg+inspect compensates to the wrong block exactly when config + drifted). Reconstruct-from-inspection survives only as the documented + fallback for legacy installs (no record), unreadable records, or + records with no designated primary port — and every fallback is + announced in the output ("restoring the previous route from live + inspection"). rollback's restoreRollbackRoute (the failed-ROLLBACK + compensation over running containers) is NOT this path and stays with + the A12/T05 register item, as does the exact-block compare-and-swap + design on ParseSites/ExtractPolicy. + +Evidence — TDD red first per finding: C01-6's tests failed at compile +(RepairDebt absent) and behaviorally after stubbing (marker never +written; next deploy never repaired; count never bumped); C01-7's four +tests failed against the inspect-driven base for the finding's own +reasons (route rendered from inspect against a record, silent fallback, +a SUCCEEDING disagreeing inspect winning 8080-over-3000). New coverage: +marker content/order (app, release, attempt id, reason, count, timing), +next-deploy repair + clear + report ordering (repair precedes +"Deploying"), persistent-failure count bump, no-marker silence, status +text+JSON+unreadable, receipt rendering (exact hosts/upstream/TLS/health +path from the record, `_replaced` same-version naming, multi-replica +upstreams, zero NetworkSettings inspects), loud legacy fallback. +Mutation checks (in-place, reverted): removing the DeployFenced repair +call fails the repair test ("must rebuild the failed record") and the +count test ("got 1"); swapping restorePreviousRoute precedence to +inspect-first fails all three receipt tests (route from inspect, +disagreeing inspect wins, fallback message fires on the record path). +Gates after revert: `go vet ./...` clean; `go test ./... -race -count=1` +all 25 packages ok; gofmt clean on touched files; contract probes 5/5 +PASS. No push performed. + +**Residual C01 list (explicit, updated):** C01-1 lock acquisition is +treated as quiescence — the replacement owner must run Decide over +observed evidence after a stale break (ADR: the locking-protocol +redesign). C01-2 pre-commit effects are check-then-act, not guarded — a +broken holder's candidate/route effects can land inside the new owner's +window (ADR: guarded single-command effects, WriteFenced's shape +generalized). C01-3 the shared Caddy lock is ownerless/unfenced — +conflicting-route evidence has no producer/consumer (ADR: fenced +short-lived proxy-commit lock or an owner-tagged equivalent). C01-8 +same-version running `_replaced` stays MANUAL — deliberate A08 +containment; the INSPECT→adopt continuation needs F04 generation +identities (ADR: attempt-keyed container identities). C01-9 candidate +names are version-keyed, not attempt-keyed — two attempts of one hash +are not attributable by evidence (ADR: F08 attempt ids as the keying +surface for candidate names). C01-6 and C01-7 are LANDED (this slice). diff --git a/docs/C01_RECOVERY_STATE_TABLE.md b/docs/C01_RECOVERY_STATE_TABLE.md index 187917d..cdb0ad2 100644 --- a/docs/C01_RECOVERY_STATE_TABLE.md +++ b/docs/C01_RECOVERY_STATE_TABLE.md @@ -181,7 +181,13 @@ table's, with the register item it belongs to. (`internal/deploy/rollback.go:582-587`). The table says transition 7 is RETRY-convergent — correct — but no reconciler exists: `status`/`drift` do not heal a missing record, so the convergence the table promises is - latent until the next deploy. + latent until the next deploy. **LANDED 2026-09-22** (see AUDIT_OPEN's + C01 record-repair-debt slice): a repair-debt marker + (`/deployments//repair-debt.json`) is persisted on the post-commit + record-write failure; the NEXT deploy repairs it before its own work + (record rebuilt from live containers via Backfill, marker cleared, + reported — repeated failure keeps the marker with an incremented count); + `teploy status` reports outstanding debt. 7. **C01-7 — Compensation reconstructs the predecessor instead of using a receipt.** `restorePreviousRoute` (`internal/deploy/deploy.go:1097-1137`) @@ -191,7 +197,15 @@ table's, with the register item it belongs to. block/spec (F14 record, `ParseSites`/`ExtractPolicy` `internal/caddy/routes.go:89,429`) — not an inference that can compensate to the wrong block when config drifted. Register: A12/T05 - standing; the table sharpen the disposition language. + standing; the table sharpens the disposition language. **LANDED + 2026-09-22 for the deploy-side traffic-switch rollback** (see + AUDIT_OPEN's C01 record-repair-debt slice): `restorePreviousRoute` + renders from the predecessor release's F14 record (domain, replica + upstreams, recorded primary port, TLS/extra/cache/firewall/access, + health path; zero live inspect), with reconstruct-from-inspection only + as the announced legacy fallback. Still open under A12/T05: + rollback's `restoreRollbackRoute` and the exact-block + receipt/compare-and-swap restore on ParseSites/ExtractPolicy. 8. **C01-8 — Same-version `_replaced` handling is MANUAL where the table says INSPECT→compensable.** The running-`_replaced` refusal @@ -320,11 +334,13 @@ Executed against a real fixture 2026-09-21 (see AUDIT_OPEN). Implementation slices: **C01-4, C01-5, C01-10 landed 2026-09-22** (attempt-journal receipts + honest degraded log outcome; evidence in -AUDIT_OPEN's C01 implementation-slice section). Remaining findings: -C01-1/2/3 (the locking-protocol redesign — replacement-owner +AUDIT_OPEN's C01 implementation-slice section) and **C01-6, C01-7 landed +2026-09-22** (record-repair debt reconciler + receipt-driven route +compensation; evidence in AUDIT_OPEN's latest C01 slice). Remaining +findings: C01-1/2/3 (the locking-protocol redesign — replacement-owner reconciliation on acquisition, guarded pre-commit effects, fenced shared Caddy lock), C01-8 (same-version `_replaced` MANUAL — deliberate A08 -containment until F04 generation identities exist), C01-9 -(attempt-scoped candidate identities — F04/A09), and C01-6/C01-7 -(record-write convergence reconciler; receipt-driven route compensation) -which stay with their register items. +containment until F04 generation identities exist), and C01-9 +(attempt-scoped candidate identities — F04/A09). The A12/T05 remainder +of C01-7 (rollback's restoreRollbackRoute + the exact-block +compare-and-swap restore) stays with its register item. diff --git a/internal/cli/status.go b/internal/cli/status.go index 510e315..fef93f2 100644 --- a/internal/cli/status.go +++ b/internal/cli/status.go @@ -10,7 +10,10 @@ import ( "strings" "github.com/spf13/cobra" + "github.com/useteploy/teploy/internal/config" + "github.com/useteploy/teploy/internal/deploy" "github.com/useteploy/teploy/internal/docker" + "github.com/useteploy/teploy/internal/ssh" "github.com/useteploy/teploy/internal/state" ) @@ -37,10 +40,31 @@ func runStatus(flags *Flags, appName string) error { return err } defer executor.Close() + return writeStatus(ctx, flags, appCfg, executor, os.Stdout) +} + +// formatRepairDebt renders the operator-facing sentence for outstanding +// release-record repair debt (C01-6), or "" when there is none — absence +// must be silent. +func formatRepairDebt(app string, debt *deploy.RepairDebt) string { + if debt == nil { + return "" + } + return fmt.Sprintf("release record for %s@%s is missing (record write failed %d attempt(s): %s) — the next deploy rebuilds it", app, debt.Release, debt.Attempts, debt.Reason) +} +// writeStatus renders the app's server-side state. Split from runStatus so +// the state-read surface (state, containers, and now repair debt) is +// testable against a mock executor. +func writeStatus(ctx context.Context, flags *Flags, appCfg *config.AppConfig, executor ssh.Executor, out io.Writer) error { // Read deploy state. current, _ := state.Read(ctx, executor, appCfg.App) + // Outstanding release-record repair debt (C01-6): a previous deploy + // whose record write failed after the live commit. Unreadable markers + // are visible too — an unhealable debt must not be an invisible one. + debt, debtErr := deploy.ReadRepairDebt(ctx, executor, appCfg.App) + // List containers. dk := docker.NewClient(executor) containers, err := dk.ListContainers(ctx, appCfg.App) @@ -49,33 +73,39 @@ func runStatus(flags *Flags, appName string) error { } if flags.JSON { - return json.NewEncoder(os.Stdout).Encode(map[string]interface{}{ - "app": appCfg.App, - "server": executor.Host(), - "state": current, - "containers": containers, + return json.NewEncoder(out).Encode(map[string]interface{}{ + "app": appCfg.App, + "server": executor.Host(), + "state": current, + "repair_debt": debt, + "containers": containers, }) } - fmt.Printf("App: %s\n", appCfg.App) - fmt.Printf("Server: %s\n", executor.Host()) + fmt.Fprintf(out, "App: %s\n", appCfg.App) + fmt.Fprintf(out, "Server: %s\n", executor.Host()) if current != nil { - fmt.Printf("Version: %s (port %d)\n", current.CurrentHash, current.CurrentPort) + fmt.Fprintf(out, "Version: %s (port %d)\n", current.CurrentHash, current.CurrentPort) if current.PreviousHash != "" { - fmt.Printf("Previous: %s (port %d)\n", current.PreviousHash, current.PreviousPort) + fmt.Fprintf(out, "Previous: %s (port %d)\n", current.PreviousHash, current.PreviousPort) } } else { - fmt.Println("Version: not deployed") + fmt.Fprintln(out, "Version: not deployed") + } + if debtErr != nil { + fmt.Fprintf(out, "Repair debt: marker could not be read — %v\n", debtErr) + } else if line := formatRepairDebt(appCfg.App, debt); line != "" { + fmt.Fprintf(out, "Repair debt: %s\n", line) } if len(containers) == 0 { - fmt.Println("\nNo containers") + fmt.Fprintln(out, "\nNo containers") return nil } - fmt.Printf("\n%-35s %-25s %-10s %s\n", "CONTAINER", "IMAGE", "STATE", "STATUS") + fmt.Fprintf(out, "\n%-35s %-25s %-10s %s\n", "CONTAINER", "IMAGE", "STATE", "STATUS") for _, c := range containers { - fmt.Printf("%-35s %-25s %-10s %s\n", c.Name, c.Image, c.State, c.Status) + fmt.Fprintf(out, "%-35s %-25s %-10s %s\n", c.Name, c.Image, c.State, c.Status) } return nil } diff --git a/internal/cli/status_test.go b/internal/cli/status_test.go new file mode 100644 index 0000000..e4ad179 --- /dev/null +++ b/internal/cli/status_test.go @@ -0,0 +1,89 @@ +package cli + +import ( + "bytes" + "context" + "encoding/json" + "strings" + "testing" + "time" + + "github.com/useteploy/teploy/internal/config" + "github.com/useteploy/teploy/internal/ssh" +) + +// TestStatus_ShowsOutstandingRepairDebt is the C01-6 operator-surface +// regression: `teploy status` reports the app's outstanding record repair +// debt (release, attempt count, remediation), in text and JSON. +func TestStatus_ShowsOutstandingRepairDebt(t *testing.T) { + mock := ssh.NewMockExecutor("1.2.3.4", + ssh.MockCommand{Match: "docker ps", Output: ""}, + ) + debt := map[string]any{ + "schema_version": 1, + "app": "myapp", + "release": "new456", + "attempt": "new456.0123456789abcdef", + "reason": "uploading temporary file: boom", + "attempts": 2, + "first_failed_at": time.Date(2026, 9, 22, 10, 0, 0, 0, time.UTC).Format(time.RFC3339Nano), + "last_failed_at": time.Date(2026, 9, 22, 11, 0, 0, 0, time.UTC).Format(time.RFC3339Nano), + } + debtJSON, _ := json.Marshal(debt) + mock.Files["/deployments/myapp/repair-debt.json"] = debtJSON + + var out bytes.Buffer + if err := writeStatus(context.Background(), &Flags{}, &config.AppConfig{App: "myapp"}, mock, &out); err != nil { + t.Fatalf("writeStatus: %v", err) + } + text := out.String() + for _, want := range []string{"Repair debt", "new456", "2 attempt"} { + if !strings.Contains(text, want) { + t.Errorf("status must report the debt (missing %q), got:\n%s", want, text) + } + } + + // JSON surface (dash/machine readers) carries the structured marker. + var jsonOut bytes.Buffer + if err := writeStatus(context.Background(), &Flags{JSON: true}, &config.AppConfig{App: "myapp"}, mock, &jsonOut); err != nil { + t.Fatalf("writeStatus json: %v", err) + } + var decoded map[string]any + if err := json.Unmarshal(jsonOut.Bytes(), &decoded); err != nil { + t.Fatalf("invalid status JSON: %v", err) + } + debtField, ok := decoded["repair_debt"].(map[string]any) + if !ok { + t.Fatalf("status JSON must carry repair_debt, got %v", decoded["repair_debt"]) + } + if debtField["release"] != "new456" { + t.Errorf("repair_debt.release = %v, want new456", debtField["release"]) + } +} + +// TestStatus_NoDebtMarkerNoNoise pins the quiet side: an app with no repair +// debt gets no debt output (text or JSON). +func TestStatus_NoDebtMarkerNoNoise(t *testing.T) { + mock := ssh.NewMockExecutor("1.2.3.4", + ssh.MockCommand{Match: "docker ps", Output: ""}, + ) + var out bytes.Buffer + if err := writeStatus(context.Background(), &Flags{}, &config.AppConfig{App: "myapp"}, mock, &out); err != nil { + t.Fatalf("writeStatus: %v", err) + } + if strings.Contains(strings.ToLower(out.String()), "repair") { + t.Errorf("no debt means no repair output noise, got:\n%s", out.String()) + } + + var jsonOut bytes.Buffer + if err := writeStatus(context.Background(), &Flags{JSON: true}, &config.AppConfig{App: "myapp"}, mock, &jsonOut); err != nil { + t.Fatalf("writeStatus json: %v", err) + } + var decoded map[string]any + if err := json.Unmarshal(jsonOut.Bytes(), &decoded); err != nil { + t.Fatalf("invalid status JSON: %v", err) + } + if decoded["repair_debt"] != nil { + t.Errorf("repair_debt must be null without a marker, got %v", decoded["repair_debt"]) + } +} diff --git a/internal/deploy/deploy.go b/internal/deploy/deploy.go index 9e95b11..6df28aa 100644 --- a/internal/deploy/deploy.go +++ b/internal/deploy/deploy.go @@ -313,6 +313,16 @@ func (d *Deployer) DeployFenced(ctx context.Context, cfg Config, lk *state.Lock) return fmt.Errorf("refusing to deploy with unreadable state for %s: %w", cfg.App, err) } + // 1b. Converge outstanding record repair debt (C01-6): a previous + // deploy whose releasemeta record write failed after the live commit + // left a repair-debt marker. Rebuild that record from the live + // containers BEFORE this deploy's own work and clear the marker — + // under the same lock every other state mutation here holds. Never a + // deploy failure: the debt describes the previous deploy, and on its + // own failure the marker stays (with the count bumped) for the next + // one. + d.repairOutstandingRecordDebt(ctx, cfg.App, current) + // 4. Determine host ports for all web replicas. var ports []int if cfg.ingressHost() { @@ -830,8 +840,9 @@ func (d *Deployer) DeployFenced(ctx context.Context, cfg Config, lk *state.Lock) // 13b. Record the release metadata (F14). The containers are live and // the route/state are committed — a record failure is a degraded // rollback window, not a failed deploy, and it converges on the next - // deploy or backfill. Never abort into abortStateCommit from here. - d.recordRelease(ctx, cfg, newState, ports, webBindHost, webContainerName) + // deploy via the repair-debt marker (C01-6). Never abort into + // abortStateCommit from here. + d.recordRelease(ctx, cfg, assetAttempt, newState, ports, webBindHost, webContainerName) // 13c. Prune superseded attempts (F08): attempt directories (build // contexts, env files, TLS certs) are dead weight once their release @@ -1179,6 +1190,16 @@ func (d *Deployer) abortStateCommit(ctx context.Context, cfg Config, current *st return fmt.Errorf("committing authoritative applied state after route switch: %w; the previous route was restored, the old workload was left running, and the uncommitted workload was stopped", commitErr) } +// restorePreviousRoute compensates a failed deploy's traffic switch by +// putting the PREVIOUS release's route back (C01-7/A12/T05). The F14 record +// of the predecessor release is the receipt of what teploy switched away +// FROM, and it is AUTHORITATIVE: domain, replica upstream names, the +// recorded primary container port, TLS/extra/cache/firewall/access, and the +// LB health path all come from the record — never from the current config +// or a live inspect, which can disagree with the receipt precisely when +// config drifted (and compensating to a drifted block is compensating to +// the wrong route). Reconstruct-from-inspection remains only as the +// documented fallback for legacy installs without a record — and it says so. func (d *Deployer) restorePreviousRoute(ctx context.Context, cfg Config, current *state.AppState) error { if current == nil || current.CurrentHash == "" { return d.caddy.RemoveRoute(ctx, cfg.App) @@ -1187,6 +1208,87 @@ func (d *Deployer) restorePreviousRoute(ctx context.Context, cfg Config, current return d.caddy.RemoveRoute(ctx, cfg.App) } + rec, recErr := releasemeta.Read(ctx, d.exec, cfg.App, current.CurrentHash) + switch { + case recErr != nil: + fmt.Fprintf(d.out, "Warning: the release record for %s@%s could not be read (%v) — restoring the previous route from live inspection instead of the recorded receipt\n", cfg.App, current.CurrentHash, recErr) + case rec == nil: + fmt.Fprintf(d.out, "Warning: no release record for %s@%s (pre-F14 install) — restoring the previous route from live inspection instead of the recorded receipt\n", cfg.App, current.CurrentHash) + default: + if port, ok := releasemeta.PrimaryContainerPort(rec); ok { + return d.restoreRouteFromReceipt(ctx, cfg, current, rec, port) + } + // A record without a designated primary port (a backfilled record + // whose bindings identified none) cannot render the receipt's + // upstream port; that piece falls back to inspection, loudly. + fmt.Fprintf(d.out, "Warning: the release record for %s@%s names no primary container port — restoring the previous route from live inspection instead of the recorded receipt\n", cfg.App, current.CurrentHash) + } + return d.restoreRouteFromInspection(ctx, cfg, current) +} + +// restoreRouteFromReceipt renders the predecessor route from the recorded +// receipt. The upstream NAMES are deterministic per release (the same +// derivation every deploy uses), so the record's replica count plus the +// recorded primary container port reproduce the exact upstreams without a +// single live inspect. Edge-config overlays come from the record when it +// carries them; a backfilled record cannot (nothing recoverable from +// containers), and the CLI-passed config stays the fallback for it exactly +// like rollback's applyRecordToRollback. +func (d *Deployer) restoreRouteFromReceipt(ctx context.Context, cfg Config, current *state.AppState, rec *releasemeta.Record, containerPort int) error { + replicas := rec.Replicas + if replicas <= 0 { + replicas = 1 + } + names := make([]string, replicas) + upstreams := make([]caddy.Upstream, replicas) + for i := range replicas { + name := docker.ReplicaContainerName(cfg.App, "web", current.CurrentHash, i+1, replicas) + if current.CurrentHash == cfg.Version { + name += "_replaced" + } + names[i] = name + upstreams[i] = caddy.Upstream{Dial: fmt.Sprintf("%s:%d", name, containerPort)} + } + + domain := rec.Domain + if domain == "" { + domain = current.Domain + } + if domain == "" { + domain = cfg.Domain + } + + tls := caddy.TLS{Cert: cfg.TLSCert, Key: cfg.TLSKey, Internal: cfg.TLSInternal} + caddyExtra := cfg.CaddyExtra + cache := cfg.Cache + fw := cfg.Firewall + access := cfg.Access + if rec.Caddy != nil { + tls = caddy.TLS{Cert: rec.Caddy.TLSCert, Key: rec.Caddy.TLSKey, Internal: rec.Caddy.TLSInternal} + caddyExtra = rec.Caddy.CaddyExtra + cache = rec.Caddy.Cache + if rec.Caddy.Firewall != nil { + fw = *rec.Caddy.Firewall + } + if rec.Caddy.Access != nil { + access = *rec.Caddy.Access + } + } + healthPath := cfg.Health.withDefaults().Path + if rec.Health != nil && rec.Health.Path != "" { + healthPath = rec.Health.Path + } + + if replicas > 1 { + return d.caddy.SetLoadBalancerHealth(ctx, cfg.App, domain, upstreams, healthPath, tls, caddyExtra, cache, fw, access) + } + return d.caddy.SetRoute(ctx, cfg.App, domain, names[0], containerPort, tls, caddyExtra, cache, fw, access) +} + +// restoreRouteFromInspection is the legacy fallback (pre-F14 installs, or a +// record that cannot name its route): reconstruct the previous block from +// the current config plus a live inspect of the predecessor containers. +func (d *Deployer) restoreRouteFromInspection(ctx context.Context, cfg Config, current *state.AppState) error { replicas := len(current.CurrentPorts) if replicas == 0 { replicas = 1 @@ -1246,8 +1348,10 @@ func (d *Deployer) logDeploy(ctx context.Context, cfg Config, success bool, degr // binding plus every publish entry); env records the references (server-side // env-file paths + the plaintext env map), never resolved secrets. The // primary web container's full RecreateSpec is embedded from docker's own -// view of it. Every failure is a warning — see the call site. -func (d *Deployer) recordRelease(ctx context.Context, cfg Config, applied *state.AppState, ports []int, webBindHost, webContainerName string) { +// view of it. A write failure is deliberate degradation (the deploy stays +// live) made durable and convergent: the repair-debt marker it records +// (C01-6) drives the next deploy's rebuild and `status`'s reporting. +func (d *Deployer) recordRelease(ctx context.Context, cfg Config, att releasemeta.Attempt, applied *state.AppState, ports []int, webBindHost, webContainerName string) { containerPort := cfg.ContainerPort if containerPort == 0 { containerPort = 80 @@ -1321,7 +1425,8 @@ func (d *Deployer) recordRelease(ctx context.Context, cfg Config, applied *state fmt.Fprintf(d.out, "Warning: could not capture the recreate spec for %s: %v (recreate falls back to live inspect)\n", webContainerName, err) } if err := releasemeta.Write(ctx, d.exec, rec); err != nil { - fmt.Fprintf(d.out, "Warning: could not record release metadata for %s@%s: %v (rollback for this release falls back to live inspection)\n", cfg.App, cfg.Version, err) + fmt.Fprintf(d.out, "Warning: could not record release metadata for %s@%s: %v — the deploy stays live; repair debt recorded (the next deploy rebuilds the record)\n", cfg.App, cfg.Version, err) + d.recordRepairDebt(ctx, att, err) } } diff --git a/internal/deploy/repairdebt.go b/internal/deploy/repairdebt.go new file mode 100644 index 0000000..deabbdb --- /dev/null +++ b/internal/deploy/repairdebt.go @@ -0,0 +1,208 @@ +// The release-record repair-debt marker (programme workstream C01, finding +// C01-6): when a deploy's releasemeta record write fails AFTER the live +// commit, the deploy deliberately completes — a record failure must never +// roll back live traffic — but the degradation used to be invisible and +// unconverged: nothing retried the write and nothing surfaced the debt. +// +// The marker is a small JSON file in the app's state namespace +// (/deployments//repair-debt.json, atomic 0600) naming the app, the +// failed attempt, what failed, when, and how many repair attempts have +// failed since. It is the debt the crash-recovery table's transition 7 +// promises to converge: +// +// - the NEXT deploy of the app repairs it before its own work — the +// record is rebuilt from the live containers (releasemeta.Backfill, the +// same convergence rollback uses) and the marker is cleared on success; +// - a repeated failure keeps the marker with an incremented attempt +// count, so the debt can neither vanish nor reset silently; +// - `teploy status` reports outstanding debt (internal/cli/status.go). +package deploy + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "time" + + "github.com/useteploy/teploy/internal/config" + "github.com/useteploy/teploy/internal/releasemeta" + "github.com/useteploy/teploy/internal/ssh" + "github.com/useteploy/teploy/internal/state" +) + +// repairDebtSchemaVersion is the marker's schema version. +const repairDebtSchemaVersion = 1 + +// repairDebtFileName is the marker's name inside the app's state namespace. +const repairDebtFileName = "repair-debt.json" + +// RepairDebt is the durable record of a release record that failed to +// persist after its deploy went live. Attempts counts write/repair attempts +// that have failed for the SAME release (1 at the first failure; every +// failed repair increments it). +type RepairDebt struct { + SchemaVersion int `json:"schema_version"` + App string `json:"app"` + Release string `json:"release"` + Attempt string `json:"attempt,omitempty"` + Reason string `json:"reason"` + Attempts int `json:"attempts"` + FirstFailedAt time.Time `json:"first_failed_at"` + LastFailedAt time.Time `json:"last_failed_at"` +} + +// repairDebtPath is the marker's location. The app segment is grammar +// checked (A17) so it can never interpolate path metacharacters. +func repairDebtPath(app string) (string, error) { + if err := config.ValidateName(app); err != nil { + return "", fmt.Errorf("repair-debt marker requires a valid app: %w", err) + } + return "/deployments/" + app + "/" + repairDebtFileName, nil +} + +// ReadRepairDebt loads the app's repair-debt marker. Confirmed absent +// returns (nil, nil) — absence is the quiet normal case; every other +// failure (transport, malformed JSON, wrong schema, identity mismatch) is +// an error, so callers surface the unhealable debt instead of guessing +// (T56 parity with the other identity-checked records). +func ReadRepairDebt(ctx context.Context, exec ssh.Executor, app string) (*RepairDebt, error) { + path, err := repairDebtPath(app) + if err != nil { + return nil, err + } + data, present, err := state.ReadRemoteFile(ctx, exec, path) + if err != nil { + return nil, fmt.Errorf("reading repair debt for %s: %w", app, err) + } + if !present { + return nil, nil + } + var debt RepairDebt + if err := json.Unmarshal(data, &debt); err != nil { + return nil, fmt.Errorf("parsing the repair-debt marker for %s at %s: %w", app, path, err) + } + if debt.SchemaVersion != repairDebtSchemaVersion { + return nil, fmt.Errorf("unsupported repair-debt schema version %d for %s", debt.SchemaVersion, app) + } + if debt.App != app { + return nil, fmt.Errorf("repair-debt identity mismatch: requested %s, marker describes %s — refusing to use it", app, debt.App) + } + return &debt, nil +} + +// writeRepairDebt persists the marker atomically (sibling temp + rename, +// 0600 — the same discipline every state-namespace writer uses). +func (d *Deployer) writeRepairDebt(ctx context.Context, debt *RepairDebt) error { + path, err := repairDebtPath(debt.App) + if err != nil { + return err + } + data, err := json.Marshal(debt) + if err != nil { + return fmt.Errorf("marshaling the repair-debt marker: %w", err) + } + return ssh.UploadAtomic(ctx, d.exec, bytes.NewReader(data), path, "0600") +} + +// clearRepairDebt removes the marker after a successful repair. +func (d *Deployer) clearRepairDebt(ctx context.Context, app string) error { + path, err := repairDebtPath(app) + if err != nil { + return err + } + if _, err := d.exec.Run(ctx, "rm -f -- "+ssh.ShellQuote(path)); err != nil { + return fmt.Errorf("clearing the repair-debt marker for %s: %w", app, err) + } + return nil +} + +// recordRepairDebt persists the initial debt when a deploy's record write +// fails (deploy.go recordRelease). A marker for the SAME release bumps its +// attempt count and keeps FirstFailedAt — the debt's history survives +// repeated failing deploys of one release; a marker for a DIFFERENT release +// is replaced (the newest unresolved debt is the one that matters). A +// persistence failure here is warned loudly: without the marker, the next +// deploy will not know to repair anything. +func (d *Deployer) recordRepairDebt(ctx context.Context, att releasemeta.Attempt, writeErr error) { + now := time.Now().UTC() + debt := &RepairDebt{ + SchemaVersion: repairDebtSchemaVersion, + App: att.App, + Release: att.Hash, + Attempt: att.Name(), + Reason: writeErr.Error(), + Attempts: 1, + FirstFailedAt: now, + LastFailedAt: now, + } + if prev, err := ReadRepairDebt(ctx, d.exec, att.App); err == nil && prev != nil && prev.Release == att.Hash { + debt.Attempts = prev.Attempts + 1 + debt.FirstFailedAt = prev.FirstFailedAt + } + if err := d.writeRepairDebt(ctx, debt); err != nil { + path, _ := repairDebtPath(att.App) + fmt.Fprintf(d.out, "Warning: could not persist the repair-debt marker for %s@%s (%v) — the missing release record will NOT self-heal; resolve it manually at %s\n", att.App, att.Hash, err, path) + } +} + +// repairOutstandingRecordDebt is the C01-6 reconciler, run by the next +// deploy of the app BEFORE its own work: if a repair-debt marker exists, +// rebuild the failed record and clear the marker; on failure keep the +// marker with an incremented attempt count. Never a deploy failure — the +// debt is a degraded condition of the PREVIOUS deploy, and refusing to +// deploy because of it would trade a live fix for a bookkeeping gap. +func (d *Deployer) repairOutstandingRecordDebt(ctx context.Context, app string, current *state.AppState) { + debt, err := ReadRepairDebt(ctx, d.exec, app) + if err != nil { + path, _ := repairDebtPath(app) + fmt.Fprintf(d.out, "Warning: a repair-debt marker exists for %s but cannot be read (%v) — the release record it names will not self-heal; inspect or remove %s\n", app, err, path) + return + } + if debt == nil { + return + } + + // Already converged (a rollback or recreate backfilled it in the + // meantime): the debt is paid, only the marker is stale. + if rec, rerr := releasemeta.Read(ctx, d.exec, app, debt.Release); rerr == nil && rec != nil { + if cerr := d.clearRepairDebt(ctx, app); cerr != nil { + fmt.Fprintf(d.out, "Warning: the repair debt for %s@%s is already resolved, but clearing the marker failed: %v\n", app, debt.Release, cerr) + return + } + fmt.Fprintf(d.out, "Repair debt cleared: the release record for %s@%s already exists (converged by another operation)\n", app, debt.Release) + return + } + + // Re-run the record write via the live containers — the same + // convergence rollback uses (releasemeta.Backfill). + var repairErr error + switch { + case current == nil: + repairErr = fmt.Errorf("no app state to rebuild the record from") + default: + inv, ierr := d.docker.ListContainers(ctx, app) + if ierr != nil { + repairErr = fmt.Errorf("listing containers: %w", ierr) + break + } + if _, berr := releasemeta.Backfill(ctx, d.exec, d.docker, inv, app, debt.Release, current); berr != nil { + repairErr = berr + break + } + if cerr := d.clearRepairDebt(ctx, app); cerr != nil { + fmt.Fprintf(d.out, "Warning: repaired the release record for %s@%s, but clearing the repair-debt marker failed: %v\n", app, debt.Release, cerr) + return + } + fmt.Fprintf(d.out, "Repaired: release record for %s@%s rebuilt from the live containers — repair debt cleared\n", app, debt.Release) + return + } + + // The repair failed: the debt stays, visibly, with the count bumped. + debt.Attempts++ + debt.LastFailedAt = time.Now().UTC() + if werr := d.writeRepairDebt(ctx, debt); werr != nil { + fmt.Fprintf(d.out, "Warning: could not update the repair-debt marker for %s@%s: %v\n", app, debt.Release, werr) + } + fmt.Fprintf(d.out, "Warning: repair debt remains for %s@%s — repair attempt %d failed: %v; the next deploy will retry\n", app, debt.Release, debt.Attempts, repairErr) +} diff --git a/internal/deploy/repairdebt_test.go b/internal/deploy/repairdebt_test.go new file mode 100644 index 0000000..3880394 --- /dev/null +++ b/internal/deploy/repairdebt_test.go @@ -0,0 +1,284 @@ +package deploy + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "strings" + "testing" + "time" + + "github.com/useteploy/teploy/internal/ssh" +) + +// debtStubSet is the mock command set of a clean blue/green caddy deploy of +// new456 (the TestDeploy_LogsCleanSuccessWhenRetirementCompletes shape). +func debtStubSet() []ssh.MockCommand { + return []ssh.MockCommand{ + {Match: "mkdir -p /deployments/myapp", Output: ""}, + {Match: "mkdir /deployments/myapp/.lock", Output: ""}, + {Match: "if [ ! -e '/deployments/myapp/state.json' ]", Output: "absent"}, + {Match: "if [ ! -e '/deployments/myapp/state' ]", Output: "present\ncurrent_port=49152\ncurrent_hash=old123\nprevious_port=0\nprevious_hash=\n"}, + {Match: "ss -tln", Output: ssOutput}, + {Match: "docker ps --all --filter label=teploy.app='myapp'", Output: predecessorInventoryJSON}, + {Match: "docker run", Output: "newcontainer123"}, + {Match: "docker inspect", Output: "running"}, + {Match: "curl -s -o /dev/null", Output: "200"}, + {Match: "rm -f /tmp/teploy_caddy", Output: ""}, + {Match: "cat /deployments/caddy/Caddyfile", Output: "{\n\tadmin 0.0.0.0:2019\n}\n"}, + {Match: "mv /tmp/teploy_caddyfile.tmp", Output: ""}, + {Match: "mkdir /deployments/caddy/.lock", Output: ""}, + {Match: "a=$(docker exec caddy md5sum", Output: "TEPLOY_CADDY_OK"}, + {Match: "docker exec caddy caddy reload", Output: ""}, + {Match: "rmdir /deployments/caddy/.lock", Output: ""}, + {Match: "docker stop", Output: ""}, + {Match: "printf %s", Output: ""}, + {Match: "rm -rf /deployments/myapp/.lock", Output: ""}, + } +} + +// debtMarkerUnderTest finds and parses the repair-debt marker a deploy wrote. +func debtMarkerUnderTest(t *testing.T, mock *ssh.MockExecutor, app string) (path string, debt RepairDebt) { + t.Helper() + path = "/deployments/" + app + "/repair-debt.json" + raw, ok := mock.Files[path] + if !ok { + t.Fatalf("no repair-debt marker persisted at %s", path) + } + if err := json.Unmarshal(raw, &debt); err != nil { + t.Fatalf("parsing repair-debt marker: %v", err) + } + return path, debt +} + +// TestRecordWriteFailure_PersistsRepairDebtMarker is the C01-6 write-side +// regression: when the releasemeta record write fails AFTER the live commit, +// the deploy still completes (deliberate degradation — a record failure must +// never roll back live traffic) but the debt becomes DURABLE and visible: a +// repair-debt marker in the app's state namespace naming app, attempt, what +// failed, and when. +func TestRecordWriteFailure_PersistsRepairDebtMarker(t *testing.T) { + stubs := debtStubSet() + // The F14 record upload for new456 fails; everything else succeeds. + stubs = append(stubs, ssh.MockCommand{Match: "UPLOAD:/deployments/myapp/meta/new456.json", Err: errBoom}) + mock := ssh.NewMockExecutor("1.2.3.4", stubs...) + + var buf bytes.Buffer + if err := NewDeployer(mock, &buf).Deploy(context.Background(), Config{ + App: "myapp", + Domain: "myapp.com", + Image: "myapp:v2", + Version: "new456", + Health: HealthConfig{Timeout: 5 * time.Second, Interval: 10 * time.Millisecond}, + }); err != nil { + t.Fatalf("a record-write failure must not fail the deploy (deliberate degradation): %v", err) + } + + _, debt := debtMarkerUnderTest(t, mock, "myapp") + if debt.App != "myapp" || debt.Release != "new456" { + t.Errorf("marker must name the app and the release whose record failed, got app=%q release=%q", debt.App, debt.Release) + } + if !strings.HasPrefix(debt.Attempt, "new456.") || len(debt.Attempt) != len("new456.")+16 { + t.Errorf("marker must name the failed deploy attempt, got %q", debt.Attempt) + } + if !strings.Contains(debt.Reason, "boom") { + t.Errorf("marker must record what failed, got reason %q", debt.Reason) + } + if debt.Attempts != 1 { + t.Errorf("first failure records attempt count 1, got %d", debt.Attempts) + } + if debt.FirstFailedAt.IsZero() { + t.Error("marker must record when the write failed") + } + if !strings.Contains(buf.String(), "repair debt") { + t.Errorf("the deploy output must surface the debt, got:\n%s", buf.String()) + } +} + +// TestNextDeploy_RepairsRecordDebtAndClearsMarker is the C01-6 convergence +// regression: the NEXT deploy of the app, before its own work, rebuilds the +// failed record (live-container backfill — the table's transition-7 +// convergence), clears the marker, and says so in the output. +func TestNextDeploy_RepairsRecordDebtAndClearsMarker(t *testing.T) { + const nextInventory = `{"ID":"cid-new456","Names":"myapp-web-new456","Image":"myapp:v2","State":"running","Status":"Up","CreatedAt":"2026-09-20 10:00:00 +0000 UTC","Labels":{"teploy.app":"myapp","teploy.process":"web","teploy.version":"new456"}}` + "\n" + + stubs := []ssh.MockCommand{ + {Match: "mkdir -p /deployments/myapp", Output: ""}, + {Match: "mkdir /deployments/myapp/.lock", Output: ""}, + // The running predecessor is new456 (the release whose record failed). + {Match: "docker ps --all --filter label=teploy.app='myapp'", Output: nextInventory}, + // Backfill inspects the new456 web container for its recreate spec. + {Match: "docker inspect 'myapp-web-new456'", Output: myappInspectJSON("myapp-web-new456", "new456")}, + {Match: "ss -tln", Output: ssOutput}, + {Match: "docker run", Output: "newcid789"}, + {Match: "docker inspect", Output: "running"}, + {Match: "curl -s -o /dev/null", Output: "200"}, + {Match: "rm -f /tmp/teploy_caddy", Output: ""}, + {Match: "cat /deployments/caddy/Caddyfile", Output: "{\n\tadmin 0.0.0.0:2019\n}\n"}, + {Match: "mv /tmp/teploy_caddyfile.tmp", Output: ""}, + {Match: "mkdir /deployments/caddy/.lock", Output: ""}, + {Match: "a=$(docker exec caddy md5sum", Output: "TEPLOY_CADDY_OK"}, + {Match: "docker exec caddy caddy reload", Output: ""}, + {Match: "rmdir /deployments/caddy/.lock", Output: ""}, + {Match: "docker stop", Output: ""}, + {Match: "printf %s", Output: ""}, + {Match: "rm -rf /deployments/myapp/.lock", Output: ""}, + } + mock := ssh.NewMockExecutor("1.2.3.4", stubs...) + + // Seed the durable world the next deploy finds: legacy state naming + // new456 current, a repair-debt marker for new456, and NO record. + mock.Files["/deployments/myapp/state"] = []byte("current_port=49152\ncurrent_hash=new456\nprevious_port=0\nprevious_hash=\n") + seedDebt := RepairDebt{ + SchemaVersion: repairDebtSchemaVersion, + App: "myapp", + Release: "new456", + Attempt: "new456.0123456789abcdef", + Reason: "uploading temporary file: boom", + Attempts: 1, + FirstFailedAt: time.Date(2026, 9, 22, 10, 0, 0, 0, time.UTC), + LastFailedAt: time.Date(2026, 9, 22, 10, 0, 0, 0, time.UTC), + } + seedDebtJSON, _ := json.Marshal(seedDebt) + mock.Files["/deployments/myapp/repair-debt.json"] = seedDebtJSON + + var buf bytes.Buffer + if err := NewDeployer(mock, &buf).Deploy(context.Background(), Config{ + App: "myapp", + Domain: "myapp.com", + Image: "myapp:v3", + Version: "abc789", + Health: HealthConfig{Timeout: 5 * time.Second, Interval: 10 * time.Millisecond}, + }); err != nil { + t.Fatalf("Deploy: %v", err) + } + + // The record was rebuilt from the live containers. + recRaw, ok := mock.Files["/deployments/myapp/meta/new456.json"] + if !ok { + t.Fatal("the next deploy must rebuild the failed record for new456") + } + var rec struct { + App string `json:"app"` + Hash string `json:"hash"` + Backfilled bool `json:"backfilled"` + } + if err := json.Unmarshal(recRaw, &rec); err != nil { + t.Fatalf("parsing the repaired record: %v", err) + } + if rec.App != "myapp" || rec.Hash != "new456" || !rec.Backfilled { + t.Errorf("repaired record must be the backfilled record for myapp@new456, got %+v", rec) + } + + // The marker is cleared on success. + if _, still := mock.Files["/deployments/myapp/repair-debt.json"]; still { + t.Error("the repair-debt marker must be cleared after a successful repair") + } + + // The repair is reported, BEFORE the deploy's own work. + out := buf.String() + if !strings.Contains(out, "repair debt cleared") { + t.Errorf("the repair must be reported in the output, got:\n%s", out) + } + if repairIdx, deployIdx := strings.Index(out, "repair debt cleared"), strings.Index(out, "Deploying myapp"); repairIdx < 0 || deployIdx < 0 || repairIdx < deployIdx { + t.Errorf("the repair must run before the deploy's own work (repair=%d deploying=%d)", repairIdx, deployIdx) + } +} + +// TestPersistentRecordFailure_KeepsMarkerWithIncrementedAttempts pins the +// repeated-failure contract: when the repair itself fails again, the marker +// STAYS with an incremented attempt count and the output says so — the debt +// is never silently dropped or reset. +func TestPersistentRecordFailure_KeepsMarkerWithIncrementedAttempts(t *testing.T) { + const nextInventory = `{"ID":"cid-new456","Names":"myapp-web-new456","Image":"myapp:v2","State":"running","Status":"Up","CreatedAt":"2026-09-20 10:00:00 +0000 UTC","Labels":{"teploy.app":"myapp","teploy.process":"web","teploy.version":"new456"}}` + "\n" + + stubs := []ssh.MockCommand{ + {Match: "mkdir -p /deployments/myapp", Output: ""}, + {Match: "mkdir /deployments/myapp/.lock", Output: ""}, + {Match: "docker ps --all --filter label=teploy.app='myapp'", Output: nextInventory}, + {Match: "docker inspect 'myapp-web-new456'", Output: myappInspectJSON("myapp-web-new456", "new456")}, + // The record for new456 STILL cannot be written. + {Match: "UPLOAD:/deployments/myapp/meta/new456.json", Err: errBoom}, + {Match: "ss -tln", Output: ssOutput}, + {Match: "docker run", Output: "newcid789"}, + {Match: "docker inspect", Output: "running"}, + {Match: "curl -s -o /dev/null", Output: "200"}, + {Match: "rm -f /tmp/teploy_caddy", Output: ""}, + {Match: "cat /deployments/caddy/Caddyfile", Output: "{\n\tadmin 0.0.0.0:2019\n}\n"}, + {Match: "mv /tmp/teploy_caddyfile.tmp", Output: ""}, + {Match: "mkdir /deployments/caddy/.lock", Output: ""}, + {Match: "a=$(docker exec caddy md5sum", Output: "TEPLOY_CADDY_OK"}, + {Match: "docker exec caddy caddy reload", Output: ""}, + {Match: "rmdir /deployments/caddy/.lock", Output: ""}, + {Match: "docker stop", Output: ""}, + {Match: "printf %s", Output: ""}, + {Match: "rm -rf /deployments/myapp/.lock", Output: ""}, + } + mock := ssh.NewMockExecutor("1.2.3.4", stubs...) + mock.Files["/deployments/myapp/state"] = []byte("current_port=49152\ncurrent_hash=new456\nprevious_port=0\nprevious_hash=\n") + seedDebt := RepairDebt{ + SchemaVersion: repairDebtSchemaVersion, + App: "myapp", + Release: "new456", + Attempt: "new456.0123456789abcdef", + Reason: "uploading temporary file: boom", + Attempts: 1, + FirstFailedAt: time.Date(2026, 9, 22, 10, 0, 0, 0, time.UTC), + LastFailedAt: time.Date(2026, 9, 22, 10, 0, 0, 0, time.UTC), + } + seedDebtJSON, _ := json.Marshal(seedDebt) + mock.Files["/deployments/myapp/repair-debt.json"] = seedDebtJSON + + var buf bytes.Buffer + if err := NewDeployer(mock, &buf).Deploy(context.Background(), Config{ + App: "myapp", + Domain: "myapp.com", + Image: "myapp:v3", + Version: "abc789", + Health: HealthConfig{Timeout: 5 * time.Second, Interval: 10 * time.Millisecond}, + }); err != nil { + t.Fatalf("outstanding repair debt must not fail the next deploy: %v", err) + } + + _, debt := debtMarkerUnderTest(t, mock, "myapp") + if debt.Attempts != 2 { + t.Errorf("a failed repair must keep the marker with an incremented attempt count, got %d", debt.Attempts) + } + if !strings.Contains(buf.String(), "repair debt remains") { + t.Errorf("the output must say the debt remains, got:\n%s", buf.String()) + } +} + +// TestNextDeploy_NoDebtMarkerNoOutputNoise pins the quiet path: an app with +// no outstanding repair debt produces no repair output at all. +func TestNextDeploy_NoDebtMarkerNoOutputNoise(t *testing.T) { + mock := ssh.NewMockExecutor("1.2.3.4", debtStubSet()...) + var buf bytes.Buffer + if err := NewDeployer(mock, &buf).Deploy(context.Background(), Config{ + App: "myapp", + Domain: "myapp.com", + Image: "myapp:v2", + Version: "new456", + Health: HealthConfig{Timeout: 5 * time.Second, Interval: 10 * time.Millisecond}, + }); err != nil { + t.Fatalf("Deploy: %v", err) + } + if strings.Contains(strings.ToLower(buf.String()), "repair") { + t.Errorf("no debt marker means no repair output noise, got:\n%s", buf.String()) + } + if _, exists := mock.Files["/deployments/myapp/repair-debt.json"]; exists { + t.Error("no marker must be written when the record write succeeds") + } +} + +// myappInspectJSON is a minimal recreate-able container inspect for the +// myapp fixtures (the shape docker.InspectRecreate consumes). +func myappInspectJSON(name, version string) string { + return fmt.Sprintf(`[{ + "Image": "sha256:%s", + "Config": {"Image": "myapp:v2", "Cmd": ["npm", "start"], "Labels": {"teploy.app": "myapp", "teploy.process": "web", "teploy.version": %q}}, + "HostConfig": {"NetworkMode": "teploy", "PortBindings": {"3000/tcp": [{"HostIp": "0.0.0.0", "HostPort": "3000"}]}, "RestartPolicy": {"Name": "no"}}, + "NetworkSettings": {"Networks": {"teploy": {"Aliases": ["myapp"]}}} +}]`, strings.Repeat("c", 64), version) +} diff --git a/internal/deploy/route_receipt_test.go b/internal/deploy/route_receipt_test.go new file mode 100644 index 0000000..2bfc44a --- /dev/null +++ b/internal/deploy/route_receipt_test.go @@ -0,0 +1,178 @@ +package deploy + +import ( + "bytes" + "context" + "strings" + "testing" + + "github.com/useteploy/teploy/internal/ssh" + "github.com/useteploy/teploy/internal/state" +) + +// newRouteMock builds a mock whose Caddyfile machinery satisfies one +// caddy.mutate transaction (lock, read, adapt, atomic write, reload, verify, +// release). record seeds the predecessor release's F14 record ("" = none); +// extra commands (registered first, so they win) model the live docker +// world. +func newRouteMock(t *testing.T, record string, extra []ssh.MockCommand) *ssh.MockExecutor { + t.Helper() + cmds := append([]ssh.MockCommand{}, extra...) + cmds = append(cmds, + ssh.MockCommand{Match: "mkdir /deployments/caddy/.lock", Output: ""}, + ssh.MockCommand{Match: "docker exec caddy caddy reload", Output: ""}, + ssh.MockCommand{Match: "a=$(docker exec caddy md5sum", Output: "TEPLOY_CADDY_OK"}, + ssh.MockCommand{Match: "rmdir /deployments/caddy/.lock", Output: ""}, + ) + mock := ssh.NewMockExecutor("1.2.3.4", cmds...) + mock.Files["/deployments/caddy/Caddyfile"] = []byte("{\n\tadmin 0.0.0.0:2019\n}\n") + if record != "" { + mock.Files["/deployments/myapp/meta/old123.json"] = []byte(record) + } + return mock +} + +// TestRestorePreviousRoute_RendersRouteFromRecordedReceipt is the C01-7 core +// regression: compensating a traffic switch must render the previous route +// from the RECORDED receipt (the predecessor release's F14 record) — domain, +// upstream container port, and TLS all come from the record, never from the +// current config or a live inspect. The live world here disagrees on every +// axis (state's domain, cfg's domain, and an inspect that would answer a +// different port): the receipt wins. +func TestRestorePreviousRoute_RendersRouteFromRecordedReceipt(t *testing.T) { + record := `{"schema_version":1,"app":"myapp","hash":"old123","created_at":"2026-09-20T10:00:00Z",` + + `"replicas":1,"domain":"receipt.example.com",` + + `"ports":[{"host_port":49152,"container_port":3000,"primary":true}],` + + `"caddy":{"tls_cert":"/etc/caddy/tls/att/myapp/old123.deadbeefdeadbeef/myapp.crt","tls_key":"/etc/caddy/tls/att/myapp/old123.deadbeefdeadbeef/myapp.key"},` + + `"health":{"path":"/readyz"}}` + + mock := newRouteMock(t, record, nil) + d := NewDeployer(mock, new(bytes.Buffer)) + + current := &state.AppState{SchemaVersion: 2, CurrentHash: "old123", Domain: "live.example.com", CurrentPorts: []int{49152}} + cfg := Config{App: "myapp", Domain: "cfg.example.com", Version: "new456"} + if err := d.restorePreviousRoute(context.Background(), cfg, current); err != nil { + t.Fatalf("restorePreviousRoute: %v", err) + } + + caddyfile := string(mock.Files["/deployments/caddy/Caddyfile"]) + for _, want := range []string{ + "receipt.example.com", // hosts from the record, not state/cfg + "myapp-web-old123:3000", // upstream name + RECORDED container port + "/etc/caddy/tls/att/myapp/old123.deadbeefdeadbeef/myapp.crt", // recorded TLS + } { + if !strings.Contains(caddyfile, want) { + t.Errorf("route must be rendered from the recorded receipt; Caddyfile missing %q:\n%s", want, caddyfile) + } + } + for _, banned := range []string{"live.example.com", "cfg.example.com", ":8080"} { + if strings.Contains(caddyfile, banned) { + t.Errorf("route must not use non-recorded values; Caddyfile contains %q:\n%s", banned, caddyfile) + } + } + // The receipt path consults NO live inspect for the route. + for _, c := range mock.Calls { + if strings.Contains(c, ".NetworkSettings.Ports") { + t.Errorf("the recorded receipt must replace the live port inspect, got: %s", c) + } + } +} + +// TestRestorePreviousRoute_SameVersionUsesReplacedNaming keeps the +// same-version contract inside the receipt path: the predecessor containers +// were renamed aside (_replaced) by this same-version attempt, and the +// restored route must point at those names. +func TestRestorePreviousRoute_SameVersionUsesReplacedNaming(t *testing.T) { + record := `{"schema_version":1,"app":"myapp","hash":"old123","created_at":"2026-09-20T10:00:00Z",` + + `"replicas":1,"domain":"receipt.example.com",` + + `"ports":[{"host_port":49152,"container_port":3000,"primary":true}]}` + mock := newRouteMock(t, record, nil) + d := NewDeployer(mock, new(bytes.Buffer)) + + current := &state.AppState{SchemaVersion: 2, CurrentHash: "old123", CurrentPorts: []int{49152}} + cfg := Config{App: "myapp", Domain: "myapp.com", Version: "old123"} // same version + if err := d.restorePreviousRoute(context.Background(), cfg, current); err != nil { + t.Fatalf("restorePreviousRoute: %v", err) + } + caddyfile := string(mock.Files["/deployments/caddy/Caddyfile"]) + if !strings.Contains(caddyfile, "myapp-web-old123_replaced:3000") { + t.Errorf("same-version compensation must route at the renamed predecessor, got:\n%s", caddyfile) + } +} + +// TestRestorePreviousRoute_NoRecordFallsBackLoudly pins the legacy path: +// with NO record (a pre-F14 install), the route is reconstructed from live +// inspection exactly as before — but the fallback is NAMED in the output, +// never silent. +func TestRestorePreviousRoute_NoRecordFallsBackLoudly(t *testing.T) { + mock := newRouteMock(t, "", []ssh.MockCommand{ + // Live inspect answers the container's port. + ssh.MockCommand{Match: "docker inspect -f '{{range $p, $_ := .NetworkSettings.Ports}}", Output: "8080/tcp "}, + }) + var buf bytes.Buffer + d := NewDeployer(mock, &buf) + + current := &state.AppState{SchemaVersion: 2, CurrentHash: "old123", Domain: "live.example.com", CurrentPorts: []int{49152}} + cfg := Config{App: "myapp", Domain: "cfg.example.com", Version: "new456"} + if err := d.restorePreviousRoute(context.Background(), cfg, current); err != nil { + t.Fatalf("restorePreviousRoute: %v", err) + } + + caddyfile := string(mock.Files["/deployments/caddy/Caddyfile"]) + if !strings.Contains(caddyfile, "myapp-web-old123:8080") { + t.Errorf("the legacy fallback reconstructs from live inspect, got:\n%s", caddyfile) + } + if !strings.Contains(caddyfile, "live.example.com") { + t.Errorf("the legacy fallback uses the state's domain, got:\n%s", caddyfile) + } + if !strings.Contains(buf.String(), "live inspection") { + t.Errorf("the fallback must be loud — output must name the live-inspection reconstruction, got:\n%s", buf.String()) + } +} + +// TestRestorePreviousRoute_RecordDisagreesWithLiveInspect_RecordWins is the +// point of the fix: the record is AUTHORITATIVE for what teploy switched +// away FROM. A live inspect that succeeds and disagrees (8080 vs the +// recorded 3000) must never win — and the receipt path must not even consult +// it. Multi-replica: upstreams and the health path come from the record. +func TestRestorePreviousRoute_RecordDisagreesWithLiveInspect_RecordWins(t *testing.T) { + record := `{"schema_version":1,"app":"myapp","hash":"old123","created_at":"2026-09-20T10:00:00Z",` + + `"replicas":2,"domain":"receipt.example.com",` + + `"ports":[{"host_port":49152,"container_port":3000,"primary":true},{"host_port":49153,"container_port":3000}],` + + `"health":{"path":"/readyz"}}` + mock := newRouteMock(t, record, []ssh.MockCommand{ + // The inspect WOULD answer — with a port that disagrees. + ssh.MockCommand{Match: "docker inspect -f '{{range $p, $_ := .NetworkSettings.Ports}}", Output: "8080/tcp "}, + }) + var buf bytes.Buffer + d := NewDeployer(mock, &buf) + + current := &state.AppState{SchemaVersion: 2, CurrentHash: "old123", Domain: "live.example.com", CurrentPorts: []int{49152, 49153}} + cfg := Config{App: "myapp", Domain: "cfg.example.com", Version: "new456"} + if err := d.restorePreviousRoute(context.Background(), cfg, current); err != nil { + t.Fatalf("restorePreviousRoute: %v", err) + } + + caddyfile := string(mock.Files["/deployments/caddy/Caddyfile"]) + for _, want := range []string{ + "receipt.example.com", + "myapp-web-old123-1:3000", + "myapp-web-old123-2:3000", + "/readyz", // recorded LB health path + } { + if !strings.Contains(caddyfile, want) { + t.Errorf("compensation must come from the record even when live inspect disagrees; missing %q:\n%s", want, caddyfile) + } + } + if strings.Contains(caddyfile, ":8080") { + t.Errorf("the disagreeing live inspect must lose, got:\n%s", caddyfile) + } + for _, c := range mock.Calls { + if strings.Contains(c, ".NetworkSettings.Ports") { + t.Errorf("a readable record must not consult live inspect at all, got: %s", c) + } + } + if strings.Contains(buf.String(), "live inspection") { + t.Errorf("a record-driven compensation is not a fallback; the loud-fallback message must not fire, got:\n%s", buf.String()) + } +} From 6a1142dd0b2259b529efad72ff9c607328cb6553 Mon Sep 17 00:00:00 2001 From: Tyler <53561637+im-tyler@users.noreply.github.com> Date: Tue, 22 Sep 2026 21:39:48 -0700 Subject: [PATCH 4/8] feat(config,deploy): explicit readiness probe modes http|tcp|auto, surfaced before the gate (C03, closes F47/TCL-17/A22) health.mode selects the probe; empty = auto = today's compat fallback verbatim (named and surfaced in the deploy plan). Unknown modes rejected at load and validate; tcp+path rejected as a lying config. Mode forwards into the F14 record and readiness receipts; the total deadline was verified bounded and pinned by regression. Drain, liveness-vs-readiness and multi-host wave states remain C03 scope. --- AUDIT_OPEN.md | 106 +++++++- README.md | 19 ++ internal/cli/deploy.go | 3 +- internal/cli/deploy_test.go | 9 + internal/config/app.go | 46 +++- internal/config/health_mode_test.go | 101 ++++++++ internal/config/manifest.go | 6 +- internal/deploy/deploy.go | 20 +- internal/deploy/health.go | 128 +++++++-- internal/deploy/health_mode_test.go | 388 ++++++++++++++++++++++++++++ internal/deploy/journal.go | 7 +- internal/deploy/rollback.go | 9 +- internal/releasemeta/releasemeta.go | 17 +- 13 files changed, 805 insertions(+), 54 deletions(-) create mode 100644 internal/config/health_mode_test.go create mode 100644 internal/deploy/health_mode_test.go diff --git a/AUDIT_OPEN.md b/AUDIT_OPEN.md index bb08503..13f93de 100644 --- a/AUDIT_OPEN.md +++ b/AUDIT_OPEN.md @@ -173,8 +173,10 @@ defect could corrupt data today. structured route representation — LANDED 2026-09-18, see the family section; the transaction design itself remains open and can now be built on ParseSites/ExtractPolicy). -- F47 — Explicit HTTP/TCP/auto probe modes (compat fallback is - deliberate and documented). +- F47 — RESOLVED 2026-09-22 (see the C03 readiness-modes slice at the + bottom): explicit `health.mode: http | tcp | auto` with config-grammar + validation, pre-gate surfacing, record/receipt forwarding, and the + auto fallback named as documented compat. - F48 — RESOLVED 2026-09-18 (see the F16/F08/F48/F49/F57 family section at the bottom): maintenance preserves the site's TLS directive and access gate, extracted from the parsed current block; plus a pre-write @@ -344,8 +346,9 @@ into each rather than duplicated as new work items. - TCL-15 — port allocation redesign (Docker-ephemeral publish + inspect). Unblocked by F14 (the record now carries the resolved port allocation per release), design remains. -- TCL-17 — F47 tail (explicit HTTP/TCP/auto probe modes; the 404/3xx TCP - fallback is documented deliberate compat). +- TCL-17 — RESOLVED 2026-09-22 with F47 (C03 readiness-modes slice at the + bottom): the 404/3xx TCP fallback is now the NAMED `auto` compat mode, + selectable and surfaced, no longer an undocumented default. - TCL-24 — RESOLVED 2026-09-18 with F49 (family section at the bottom): adoption is parser-based; brace counting is gone. - TCL-28 — F50 (split the public static tree from /deployments). @@ -602,8 +605,9 @@ all packages ok. No push performed. - A16 — F04 external-ingress handoff (candidates reachable via the stable alias before readiness). - A20 — TCL-15 port allocation redesign. -- A22 — F47/TCL-17 explicit HTTP/TCP probe modes (the 404/3xx TCP - fallback stays documented compat). +- A22 — RESOLVED 2026-09-22 (C03 readiness-modes slice at the bottom): + F47/TCL-17 explicit probe modes landed; the 404/3xx TCP fallback stays + as `auto`, the documented compat mode. - A24 — F17 standing: Cmd remains a deliberate operator-authored shell string at the docker-run sink. - A29 — TCL-55 session-open bounding (needs a dedicated connection per @@ -1507,6 +1511,92 @@ short-lived proxy-commit lock or an owner-tagged equivalent). C01-8 same-version running `_replaced` stays MANUAL — deliberate A08 containment; the INSPECT→adopt continuation needs F04 generation identities (ADR: attempt-keyed container identities). C01-9 candidate -names are version-keyed, not attempt-keyed — two attempts of one hash -are not attributable by evidence (ADR: F08 attempt ids as the keying +names are version-keyed, not attempt-keyed — two attempts of one hash are +not attributable by evidence (ADR: F08 attempt ids as the keying surface for candidate names). C01-6 and C01-7 are LANDED (this slice). + +## Programme slice (2026-09-22, latest) — C03: explicit readiness probe modes + +Closes the F47/TCL-17/A22 standing deferral — the first bounded C03 slice +(P0: "Support HTTP, TCP, container and operator-defined readiness with +clear defaults and deadlines; retain `auto` only as an explicit +compatibility mode"). Base revision `a914631`; changes left uncommitted +for review. Drain/graceful-stop semantics, the LB health-path rendering +(5bf5594), and Caddy are untouched (next slices / explicit stay-out). + +**Design:** + +- **Grammar** — `health.mode: http | tcp | auto` in teploy.yml/TOML + (`config.AppHealthConfig.Mode`). Empty/absent = `auto`, the compat + default: HTTP GET first, exactly a 404/3xx falls back to the TCP dial — + the verbatim historical behavior (preserved in `checkHealth`, now + NAMED). `http` is status-based only (200 = ready; 404/3xx fails the + attempt, no fallback). `tcp` dials the published port and never speaks + HTTP. Unknown mode is rejected at config load AND at the shared + execution-plan validator (`deploy.Config.validate` — direct + construction via fleet/preview/autodeploy bypasses parsing, TCL-18 + parity). Field agreement: `mode: tcp` with a `path` set is REJECTED at + config load (decision: reject, not warn — a path nothing fetches is a + config that lies about what the gate does); `http`/`auto` without a + path keep the `/health` default. A mode-only destination overlay + replaces the whole health block (F57 semantics extended: Mode joined + the presence detection); the normalized manifest carries + `mode` (defaulted to auto) for drift identity. +- **Dispatch** — `probeOnce` (internal/deploy/health.go) switches on the + normalized mode per attempt; `httpStatus` (extracted from the old + monolithic attempt) returns the observed code so `checkHealth`'s + fallback condition is byte-identical to before (an interim refactor + that dialed on ANY non-200 was caught and corrected during the slice — + auto must stay exactly today's behavior). `HealthCheckPublic` / + `HealthCheckAt` (on-demand `teploy health`) keep the auto default. +- **Surfacing** — deploy (step 9) and rollback (step 3) print the gate + BEFORE it runs: `Readiness: HTTP GET /healthz (30s deadline)` / + `Readiness: TCP :3000 (30s)` / `Readiness: auto — HTTP then TCP + fallback (compat, 30s deadline)` (tcp names the first replica's port; + failures name replica + port as before). +- **Deadline verified** — `health.Timeout` was already a TOTAL deadline: + `healthCheck` wraps the context in `WithTimeout`, the retry loop + selects on ctx.Done, each HTTP attempt is curl-bounded + (--connect-timeout 2 / --max-time 5), and the remote executor cancels + the session at deadline (SIGTERM + close, RunStream). There was NO + unbounded retry loop to bound; a regression test now pins it (a + never-responding probe — an executor whose commands hang until context + death — must fail within deadline + slack, not hang). +- **Forwarding** — the F14 release record (`releasemeta.Health.Mode`) and + the C01-4 readiness receipt (`readinessProbe.Mode`) carry the effective + mode; `applyRecordToRollback` overlays it (a modeless legacy record + leaves the config's mode — compat). Rollback probes the way the target + release was actually gated. + +**Evidence** — TDD: red level 1 recorded as compile failure (Mode field +nowhere existed), red level 2 after plumbing-only (fields + passthrough, +no behavior): dispatch tests failed with mode ignored (http-mode deploy +passed via the TCP fallback; tcp-mode healthCheck timed out on an +unregistered curl; auto-explicit never dialed), unknown mode and tcp+path +were accepted, the surfaced lines were absent, the record carried no +mode. Green after the implementation. Mutation checks (in-place, +reverted, gates re-run green after each): (1) dispatch removed — always +http — fails TestHealthCheck_TCPMode* (curl issued / dial never run), +TestHealthCheck_AutoModeExplicitFallsBack (no dial), and the tcp-mode +DEPLOY test (gate times out); (2) readinessSummary collapsed to the http +line fails the tcp/auto surfaced-line tests; (3) removing the +`context.WithTimeout` total deadline hangs the never-responding-probe +test to the test-binary timeout. Gates: `go vet ./...` clean; +`go test ./... -race -count=1` all 25 packages ok; gofmt clean on every +touched file (pre-existing base strays in cli/deploy.go's fleet-rollback +region, deploy_test.go, f14_wiring_test.go, plan_a_test.go, +config/app_test.go left alone, consistent with the C02 posture); contract +probes 5/5 PASS. No push performed. + +**C03 remainder (explicit):** request drain + graceful stop (stop_timeout +wiring, SIGTERM→SIGKILL ladder — deliberately this slice's stay-out), the +liveness-vs-readiness distinction (post-switch continuous probing; today +only the container HEALTHCHECK directive approximates it), multi-host +partial-wave readiness states (canary-wave aggregate gating beyond the +existing success/fail rollback), and WebSocket/SSE/long-request drain +verification at the traffic switch. Registered interactions to decide in +those slices: `mode: tcp` × the Caddy LB active health check (the LB +block's HTTP path probe would mark a non-HTTP upstream down — LB +rendering is 5bf5594's fixed surface, untouched here), and preview's +readiness gate (internal/preview) which mirrors the auto shape and has +no mode surface of its own. diff --git a/README.md b/README.md index 2190f21..848020a 100644 --- a/README.md +++ b/README.md @@ -202,6 +202,25 @@ processes: web: "npm start" worker: "npm run worker" +# Readiness gate — what "healthy" means before traffic switches, and how +# long to wait. mode selects the probe: +# http — status-based only: GET path, 200 = ready. A 404/redirect FAILS +# (no fallback). Best when the app has a real health endpoint. +# tcp — a TCP dial against the published port; nothing is fetched. +# For apps with no HTTP surface (game servers, TCP brokers). +# Setting `path` alongside is rejected — nothing would fetch it. +# auto — compatibility default (also what an omitted mode means): HTTP +# GET first; a 404/3xx falls back to a TCP dial. The historical +# behavior, kept so existing configs deploy identically. +# timeout_seconds is the TOTAL deadline for the gate (not per-try); the +# deploy output states the mode and deadline before the gate runs, e.g. +# "Readiness: HTTP GET /healthz (30s deadline)". +health: + mode: http # http | tcp | auto (default auto/compat) + path: /healthz # default /health (http/auto only) + timeout_seconds: 30 # total gate deadline (default 30) + interval_seconds: 1 # time between attempts (default 1) + # Per-process HEALTHCHECK overrides. disable: true passes --no-healthcheck # so the container ignores the image's HEALTHCHECK — useful when a worker # shares an image with web but has no HTTP listener for the inherited probe. diff --git a/internal/cli/deploy.go b/internal/cli/deploy.go index 0974d2d..61f270a 100644 --- a/internal/cli/deploy.go +++ b/internal/cli/deploy.go @@ -1248,9 +1248,10 @@ func disabledHealthchecks(hc map[string]config.ProcessHealth) map[string]bool { // block. Zero TimeoutSeconds/IntervalSeconds map to zero time.Duration, // which HealthConfig.withDefaults() (internal/deploy/health.go) fills in // as 30s/1s — so unset fields are zero behavior change from before these -// were configurable. +// were configurable. Mode passes through: "" means auto (compat). func healthConfigFrom(h config.AppHealthConfig) deploy.HealthConfig { return deploy.HealthConfig{ + Mode: h.Mode, Path: h.Path, Timeout: time.Duration(h.TimeoutSeconds) * time.Second, Interval: time.Duration(h.IntervalSeconds) * time.Second, diff --git a/internal/cli/deploy_test.go b/internal/cli/deploy_test.go index c3b1115..adbec73 100644 --- a/internal/cli/deploy_test.go +++ b/internal/cli/deploy_test.go @@ -40,6 +40,15 @@ func TestHealthConfigFrom_UnsetFieldsStayZero(t *testing.T) { } } +func TestHealthConfigFrom_CarriesMode(t *testing.T) { + for _, mode := range []string{"http", "tcp", "auto", ""} { + got := healthConfigFrom(config.AppHealthConfig{Mode: mode}) + if got.Mode != mode { + t.Errorf("mode %q: Mode = %q, want passthrough", mode, got.Mode) + } + } +} + func pullAttempted(mock *ssh.MockExecutor) bool { for _, c := range mock.Calls { if strings.HasPrefix(c, "docker pull") { diff --git a/internal/config/app.go b/internal/config/app.go index fe8392d..8f8a80c 100644 --- a/internal/config/app.go +++ b/internal/config/app.go @@ -194,11 +194,22 @@ type ProcessHealth struct { Disable bool `yaml:"disable,omitempty" toml:"disable"` } -// AppHealthConfig configures the teploy-level deploy health check: the HTTP -// poll that gates the traffic switch after a deploy. Distinct from the -// container HEALTHCHECK directive (which is per-process, in Healthcheck map). +// AppHealthConfig configures the teploy-level deploy readiness gate. +// Distinct from the container HEALTHCHECK directive (which is per-process, +// in the Healthcheck map). type AppHealthConfig struct { - // Path is the URL path polled for a 200 response. Default: "/health". + // Mode selects the readiness probe: + // + // http — status-based only: HTTP GET path, 200 = ready. A 404/3xx + // FAILS the gate (no fallback). + // tcp — a TCP dial against the published port; nothing is fetched. + // Setting path alongside is rejected (nothing would fetch it). + // auto — compatibility (the default when unset): HTTP GET first, a + // 404/3xx falls back to the TCP dial — the exact behavior + // every deploy used before modes existed (F47/TCL-17/A22). + Mode string `yaml:"mode,omitempty" toml:"mode"` + // Path is the URL path polled for a 200 response in http/auto mode. + // Default: "/health". Not valid with mode: tcp. Path string `yaml:"path,omitempty" toml:"path"` // TimeoutSeconds is the total time to wait for a healthy response before // the deploy fails and rolls back. Default: 30. Raise this for @@ -282,6 +293,19 @@ const ( TypeStatic = "static" ) +// Health readiness-gate modes (the `health.mode` grammar). Empty and "auto" +// both mean the compatibility default: HTTP GET first, a 404/3xx answer +// falls back to a TCP dial — the behavior every deploy used before modes +// existed. "http" is status-based only (200 = ready, no fallback); "tcp" +// dials the published port and never speaks HTTP. The deploy-side probe +// dispatch mirrors these in internal/deploy/health.go (deploy cannot share +// these constants: it imports this package). +const ( + HealthModeHTTP = "http" + HealthModeTCP = "tcp" + HealthModeAuto = "auto" +) + // Ingress modes. Empty string and "caddy" both mean Teploy manages the // Caddyfile and reloads Caddy on every deploy / rollback / maintenance // toggle (current behavior). "external" means the user is fronting the @@ -979,6 +1003,18 @@ func (c *AppConfig) validate() error { if c.Health.IntervalSeconds < 0 { return fmt.Errorf("'health.interval_seconds' must be >= 0 (got %d)", c.Health.IntervalSeconds) } + // Empty mode means auto (the documented compat default, applied at + // deploy time), so the enum accepts it here. + switch c.Health.Mode { + case "", HealthModeHTTP, HealthModeTCP, HealthModeAuto: + default: + return fmt.Errorf("'health.mode' must be one of: http, tcp, auto (got %q)", c.Health.Mode) + } + // A path under tcp mode is a field nothing fetches — reject the lying + // config at load instead of deploying a gate that ignores it silently. + if c.Health.Mode == HealthModeTCP && c.Health.Path != "" { + return fmt.Errorf("'health.path' has no effect with 'health.mode: tcp' (TCP readiness dials the port; nothing is fetched) — remove the path or use mode http/auto") + } for name, dest := range c.Volumes { if !validName.MatchString(name) && !IsHostBindVolume(name) { return fmt.Errorf("volume name %q must be lowercase alphanumeric with hyphens, or an absolute host path for a bind mount", name) @@ -1396,7 +1432,7 @@ func mergeConfigs(base, overlay *AppConfig) { } // F57: the whole health object, not just Path, and the security/policy // blocks a production-only overlay previously set to silently nothing. - if overlay.Health.Path != "" || overlay.Health.TimeoutSeconds != 0 || overlay.Health.IntervalSeconds != 0 { + if overlay.Health.Path != "" || overlay.Health.Mode != "" || overlay.Health.TimeoutSeconds != 0 || overlay.Health.IntervalSeconds != 0 { base.Health = overlay.Health } if !overlay.Access.IsZero() { diff --git a/internal/config/health_mode_test.go b/internal/config/health_mode_test.go new file mode 100644 index 0000000..331d110 --- /dev/null +++ b/internal/config/health_mode_test.go @@ -0,0 +1,101 @@ +package config + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +func writeApp(t *testing.T, content string) string { + t.Helper() + dir := t.TempDir() + if err := os.WriteFile(filepath.Join(dir, "teploy.yml"), []byte(content), 0644); err != nil { + t.Fatal(err) + } + return dir +} + +func TestLoadApp_HealthModeParsed(t *testing.T) { + for _, mode := range []string{"http", "tcp", "auto"} { + dir := writeApp(t, "app: myapp\ndomain: myapp.com\nhealth:\n mode: "+mode+"\n") + cfg, err := LoadApp(dir) + if err != nil { + t.Fatalf("mode %s: LoadApp: %v", mode, err) + } + if cfg.Health.Mode != mode { + t.Errorf("mode %s: Health.Mode = %q", mode, cfg.Health.Mode) + } + } +} + +func TestLoadApp_HealthModeEmptyStaysEmpty(t *testing.T) { + // Absent mode means "" — the documented `auto` compat default, applied + // at deploy time (HealthConfig.withDefaults), so existing teploy.yml + // files are behavior-unchanged. + dir := writeApp(t, "app: myapp\ndomain: myapp.com\nhealth:\n path: /healthz\n") + cfg, err := LoadApp(dir) + if err != nil { + t.Fatalf("LoadApp: %v", err) + } + if cfg.Health.Mode != "" { + t.Errorf("Health.Mode = %q, want empty (auto compat)", cfg.Health.Mode) + } +} + +func TestLoadApp_HealthModeUnknownRejected(t *testing.T) { + dir := writeApp(t, "app: myapp\ndomain: myapp.com\nhealth:\n mode: grpc\n") + _, err := LoadApp(dir) + if err == nil { + t.Fatal("expected error for unknown health.mode") + } + if !strings.Contains(err.Error(), "health.mode") { + t.Errorf("error should mention health.mode, got: %v", err) + } + if !strings.Contains(err.Error(), "http") || !strings.Contains(err.Error(), "tcp") || !strings.Contains(err.Error(), "auto") { + t.Errorf("error should name the valid modes, got: %v", err) + } +} + +// tcp mode dials the port; a configured path is a field nothing fetches — +// reject it at load rather than deploying a config that lies about what +// the gate does. +func TestLoadApp_HealthTCPModeWithPathRejected(t *testing.T) { + dir := writeApp(t, "app: myapp\ndomain: myapp.com\nhealth:\n mode: tcp\n path: /healthz\n") + _, err := LoadApp(dir) + if err == nil { + t.Fatal("expected error for health.path set under mode tcp") + } + if !strings.Contains(err.Error(), "health.path") || !strings.Contains(err.Error(), "tcp") { + t.Errorf("error should name path and tcp mode, got: %v", err) + } +} + +// http mode without a path is fine: the /health default applies. +func TestLoadApp_HealthHTTPModeWithoutPathAccepted(t *testing.T) { + dir := writeApp(t, "app: myapp\ndomain: myapp.com\nhealth:\n mode: http\n") + if _, err := LoadApp(dir); err != nil { + t.Fatalf("http mode with no path should default, got: %v", err) + } +} + +func TestNormalizedHealth_IncludesMode(t *testing.T) { + if got := normalizedHealth(AppHealthConfig{})["mode"]; got != "auto" { + t.Errorf("normalized mode default = %v, want auto", got) + } + if got := normalizedHealth(AppHealthConfig{Mode: "tcp"})["mode"]; got != "tcp" { + t.Errorf("normalized mode = %v, want tcp", got) + } +} + +// A destination overlay that only sets health.mode must still replace the +// whole health block (F57 semantics) — a base path must not survive into a +// tcp-mode overlay. +func TestMergeConfigs_HealthModeAloneReplacesBlock(t *testing.T) { + base := &AppConfig{App: "myapp", Domain: "myapp.com", Health: AppHealthConfig{Path: "/healthz", TimeoutSeconds: 60}} + overlay := &AppConfig{Health: AppHealthConfig{Mode: "tcp"}} + mergeConfigs(base, overlay) + if base.Health.Mode != "tcp" || base.Health.Path != "" || base.Health.TimeoutSeconds != 0 { + t.Errorf("overlay health must replace the whole block, got %+v", base.Health) + } +} diff --git a/internal/config/manifest.go b/internal/config/manifest.go index 96095a4..dc19770 100644 --- a/internal/config/manifest.go +++ b/internal/config/manifest.go @@ -139,6 +139,10 @@ func sortedMapKeys[V any](values map[string]V) []string { } func normalizedHealth(health AppHealthConfig) map[string]any { + mode := health.Mode + if mode == "" { + mode = HealthModeAuto + } path := health.Path if path == "" { path = "/health" @@ -151,7 +155,7 @@ func normalizedHealth(health AppHealthConfig) map[string]any { if interval == 0 { interval = 1 } - return map[string]any{"path": path, "timeout_seconds": timeout, "interval_seconds": interval} + return map[string]any{"mode": mode, "path": path, "timeout_seconds": timeout, "interval_seconds": interval} } func normalizedAccessories(accessories map[string]AccessoryConfig) map[string]any { diff --git a/internal/deploy/deploy.go b/internal/deploy/deploy.go index 6df28aa..a692752 100644 --- a/internal/deploy/deploy.go +++ b/internal/deploy/deploy.go @@ -184,6 +184,16 @@ func (c Config) validate() error { if c.StopTimeout < 0 { return fmt.Errorf("stop timeout cannot be negative (got %ds)", c.StopTimeout) } + // Health probe mode enum (C03): config-file parsing enforces the fuller + // grammar (tcp rejects a path); the shared execution validator covers + // the enum so directly constructed Configs (fleet, preview, autodeploy) + // cannot carry an unknown mode into the gate dispatch. Empty = auto + // (documented compat). + switch c.Health.Mode { + case "", HealthModeHTTP, HealthModeTCP, HealthModeAuto: + default: + return fmt.Errorf("unknown health mode %q (expected http, tcp, or auto)", c.Health.Mode) + } return nil } @@ -683,9 +693,14 @@ func (d *Deployer) DeployFenced(ctx context.Context, cfg Config, lk *state.Lock) fmt.Fprintln(d.out, " Pre-deploy hook passed") } - // 9. Health check all web replicas. + // 9. Health check all web replicas. The gate is surfaced BEFORE it + // runs: the operator sees which probe mode and what total deadline is + // in effect (C03) — not just the verdict after the wait. fmt.Fprintln(d.out, "Running health check...") healthCfg := cfg.Health.withDefaults() + if len(ports) > 0 { + fmt.Fprintf(d.out, " Readiness: %s\n", readinessSummary(healthCfg, ports[0])) + } for i, p := range ports { if err := d.healthCheck(ctx, p, healthCfg, webBindHost); err != nil { fmt.Fprintf(d.out, " Health check failed for replica %d (port %d): %v\n", i+1, p, err) @@ -715,7 +730,7 @@ func (d *Deployer) DeployFenced(ctx context.Context, cfg Config, lk *state.Lock) cands[i] = receiptCandidate{Name: name, ID: candidateIDs[i]} } for i, p := range ports { - probes[i] = readinessProbe{Container: webContainerNames[i], Host: probeHost, Port: p, Path: healthCfg.Path} + probes[i] = readinessProbe{Container: webContainerNames[i], Host: probeHost, Port: p, Path: healthCfg.Path, Mode: healthCfg.Mode} } if err := d.persistReadinessReceipt(ctx, assetAttempt, cfg.Version, cands, probes); err != nil { fmt.Fprintf(d.out, "Warning: could not persist the readiness receipt for crash recovery: %v\n", err) @@ -1379,6 +1394,7 @@ func (d *Deployer) recordRelease(ctx context.Context, cfg Config, att releasemet StopTimeout: cfg.StopTimeout, Bind: cfg.Bind, Health: &releasemeta.Health{ + Mode: healthCfg.Mode, Path: healthCfg.Path, TimeoutSeconds: int(healthCfg.Timeout.Seconds()), IntervalSeconds: int(healthCfg.Interval.Seconds()), diff --git a/internal/deploy/health.go b/internal/deploy/health.go index d97e5f8..b661b87 100644 --- a/internal/deploy/health.go +++ b/internal/deploy/health.go @@ -13,11 +13,33 @@ import ( "github.com/useteploy/teploy/internal/ssh" ) +// Health probe modes (C03). The mode selects what the readiness gate runs; +// every mode shares the same total deadline and interval semantics. +// +// auto is the compatibility mode and the default when mode is unset: HTTP +// GET first, and a 404/3xx answer falls back to a TCP dial — the exact +// behavior every teploy deploy used before modes existed (register +// F47/TCL-17/A22). http is status-based only (200 = ready, no fallback); +// tcp dials the port and never speaks HTTP. +const ( + HealthModeHTTP = "http" + HealthModeTCP = "tcp" + HealthModeAuto = "auto" +) + // HealthConfig configures health check behavior. type HealthConfig struct { - Path string // URL path to check (default "/health") - Timeout time.Duration // total time to wait for healthy (default 30s) - Interval time.Duration // time between checks (default 1s) + // Mode selects the probe: HealthModeHTTP, HealthModeTCP, or + // HealthModeAuto. Empty means auto (documented compat default). + Mode string + // Path is the URL path checked in http/auto mode. Default: "/health". + // Irrelevant (and rejected at config load) in tcp mode. + Path string + // Timeout is the TOTAL time to wait for healthy (default 30s) — not a + // per-attempt bound: the gate fails at this deadline however many + // attempts fit inside it. + Timeout time.Duration + Interval time.Duration } // defaultHealthConfig returns a HealthConfig with all default values applied. @@ -29,6 +51,9 @@ func (h HealthConfig) withDefaults() HealthConfig { if h.Path == "" { h.Path = "/health" } + if h.Mode == "" { + h.Mode = HealthModeAuto + } if h.Timeout == 0 { h.Timeout = 30 * time.Second } @@ -58,31 +83,48 @@ func healthProbeHost(bindHost string) string { } } -// healthCheck polls the container until it responds healthy or the timeout expires. +// healthCheck polls the container until it reports ready or the timeout +// expires. cfg.Timeout is the TOTAL deadline: the loop stops there however +// many attempts fit, each HTTP attempt is additionally bounded by curl's +// --connect-timeout/--max-time, and the executor cancels the remote command +// when the deadline context dies — there is no unbounded retry. // -// Strategy: -// 1. HTTP GET to {host}:{port}{path} — 200 means healthy. -// 2. If the endpoint returns 404, fall back to a TCP port check. -// 3. Connection refused means the app hasn't started yet — retry. +// The probe itself is selected by cfg.Mode (see HealthMode* constants): +// http runs the status check only, tcp the dial only, and auto (the +// historical behavior, now named) runs the status check with the 404/3xx +// TCP fallback. func (d *Deployer) healthCheck(ctx context.Context, port int, cfg HealthConfig, bindHost string) error { + cfg = cfg.withDefaults() ctx, cancel := context.WithTimeout(ctx, cfg.Timeout) defer cancel() host := healthProbeHost(bindHost) for { - if d.checkHealth(ctx, host, port, cfg.Path) { + if d.probeOnce(ctx, host, port, cfg) { return nil } select { case <-ctx.Done(): - return fmt.Errorf("timeout after %s waiting for health check on %s:%d", cfg.Timeout, host, port) + return fmt.Errorf("timeout after %s waiting for health check (mode %s) on %s:%d", cfg.Timeout, cfg.Mode, host, port) case <-time.After(cfg.Interval): // retry } } } +// probeOnce runs ONE readiness attempt under the configured mode. +func (d *Deployer) probeOnce(ctx context.Context, host string, port int, cfg HealthConfig) bool { + switch cfg.Mode { + case HealthModeHTTP: + return d.checkHTTP(ctx, host, port, cfg.Path) + case HealthModeTCP: + return d.checkTCP(ctx, host, port) + default: + return d.checkHealth(ctx, host, port, cfg.Path) + } +} + // HealthCheckPublic runs a health check against the given port using default settings. // This is the public entry point for on-demand health checks. // @@ -104,7 +146,35 @@ func (d *Deployer) HealthCheckAt(ctx context.Context, port int, containerName st return d.healthCheck(ctx, port, defaultHealthConfig(), bindHost) } -// checkHealth performs a single health check attempt. +// checkHealth performs a single AUTO-mode attempt (the compatibility +// strategy): the HTTP status check, with exactly a 404 or a 3xx falling +// back to a TCP dial — every other answer (5xx, no response, malformed +// probe) is retried until the deadline. This is the historical behavior, +// preserved verbatim as the named compat mode. +func (d *Deployer) checkHealth(ctx context.Context, host string, port int, path string) bool { + code := d.httpStatus(ctx, host, port, path) + if code == "200" { + return true + } + // A 404 (no /health endpoint) or a 3xx redirect means the app is + // listening but the health path isn't a 200 — for example WordPress + // 301-redirects /health to its canonical HTTPS URL. Fall back to a TCP + // check rather than failing the deploy. + if code == "404" || strings.HasPrefix(code, "3") { + return d.checkTCP(ctx, host, port) + } + return false +} + +// checkHTTP performs a single HTTP-mode attempt: true only on a 200. No +// fallback — a 404/3xx fails the attempt and the gate retries or times out. +func (d *Deployer) checkHTTP(ctx context.Context, host string, port int, path string) bool { + return d.httpStatus(ctx, host, port, path) == "200" +} + +// httpStatus issues one bounded curl request and reports the HTTP status +// code it observed, or "" when the request could not be made or answered +// (unbuildable URL, transport error, empty reply). // // The URL is built with net.JoinHostPort (bracketing IPv6 literals) and // validated before it reaches the remote shell, then passed as ONE @@ -116,31 +186,20 @@ func (d *Deployer) HealthCheckAt(ctx context.Context, port int, containerName st // readiness timeout. --noproxy '*' (audit T20's contained half) makes the // host-local probe ignore ambient proxy configuration — an inherited // HTTP_PROXY made the probe ask a proxy about a loopback address. -func (d *Deployer) checkHealth(ctx context.Context, host string, port int, path string) bool { +func (d *Deployer) httpStatus(ctx context.Context, host string, port int, path string) string { url, ok := probeURL(host, port, path) if !ok { - return false + return "" } cmd := fmt.Sprintf( "curl -s -o /dev/null --noproxy '*' --globoff --connect-timeout 2 --max-time 5 -w '%%{http_code}' --url %s", ssh.ShellQuote(url), ) output, err := d.exec.Run(ctx, cmd) - if err == nil { - code := strings.TrimSpace(output) - if code == "200" { - return true - } - // A 404 (no /health endpoint) or a 3xx redirect means the app is - // listening but the health path isn't a 200 — for example WordPress - // 301-redirects /health to its canonical HTTPS URL. Fall back to a TCP - // check rather than failing the deploy. A 5xx or "000" (no response) - // falls through and is retried until the timeout. - if code == "404" || strings.HasPrefix(code, "3") { - return d.checkTCP(ctx, host, port) - } + if err != nil { + return "" } - return false + return strings.TrimSpace(output) } // probeURL renders the health-check URL and validates its inputs. The host @@ -174,6 +233,21 @@ func probeURL(host string, port int, path string) (string, bool) { return u.String(), true } +// readinessSummary renders the one-line description of the readiness gate +// surfaced in deploy/rollback output BEFORE the gate runs, so the operator +// knows what is being gated and for how long. cfg must already carry its +// defaults (withDefaults). +func readinessSummary(cfg HealthConfig, port int) string { + switch cfg.Mode { + case HealthModeHTTP: + return fmt.Sprintf("HTTP GET %s (%s deadline)", cfg.Path, cfg.Timeout) + case HealthModeTCP: + return fmt.Sprintf("TCP :%d (%s)", port, cfg.Timeout) + default: + return fmt.Sprintf("auto — HTTP then TCP fallback (compat, %s deadline)", cfg.Timeout) + } +} + // checkTCP verifies that a TCP connection can be established to the port. // The /dev/tcp redirection runs inside a single-quoted bash -c argument, so // neither the host nor the port can break out of it. diff --git a/internal/deploy/health_mode_test.go b/internal/deploy/health_mode_test.go new file mode 100644 index 0000000..78fd70b --- /dev/null +++ b/internal/deploy/health_mode_test.go @@ -0,0 +1,388 @@ +package deploy + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/json" + "fmt" + "io" + "strings" + "sync" + "testing" + "time" + + "github.com/useteploy/teploy/internal/releasemeta" + "github.com/useteploy/teploy/internal/ssh" +) + +// --- probe dispatch --- + +// http mode is status-based ONLY: a 404 must not fall back to the TCP dial +// (that fallback is auto's documented compat behavior). +func TestHealthCheck_HTTPModeHasNoTCPFallback(t *testing.T) { + mock := ssh.NewMockExecutor("h", + ssh.MockCommand{Match: "curl", Output: "404"}, + ssh.MockCommand{Match: "bash -c", Output: ""}, + ) + d := &Deployer{exec: mock, out: nopWriter{}} + + cfg := HealthConfig{Mode: "http", Timeout: 300 * time.Millisecond, Interval: 10 * time.Millisecond} + if err := d.healthCheck(context.Background(), 3456, cfg, ""); err == nil { + t.Fatal("http mode must fail on 404 (no TCP fallback)") + } + for _, c := range mock.Calls { + if strings.HasPrefix(c, "bash -c") { + t.Errorf("http mode must never dial: %s", c) + } + } +} + +func TestHealthCheck_HTTPModePassesOn200(t *testing.T) { + mock := ssh.NewMockExecutor("h", ssh.MockCommand{Match: "curl", Output: "200"}) + d := &Deployer{exec: mock, out: nopWriter{}} + + cfg := HealthConfig{Mode: "http", Timeout: 2 * time.Second, Interval: 10 * time.Millisecond} + if err := d.healthCheck(context.Background(), 3456, cfg, ""); err != nil { + t.Fatalf("healthCheck: %v", err) + } +} + +// tcp mode never issues the HTTP probe — the dial is the whole gate. +func TestHealthCheck_TCPModeDialsWithoutCurl(t *testing.T) { + mock := ssh.NewMockExecutor("h", ssh.MockCommand{Match: "bash -c", Output: ""}) + d := &Deployer{exec: mock, out: nopWriter{}} + + cfg := HealthConfig{Mode: "tcp", Timeout: 2 * time.Second, Interval: 10 * time.Millisecond} + if err := d.healthCheck(context.Background(), 3456, cfg, ""); err != nil { + t.Fatalf("healthCheck tcp: %v", err) + } + for _, c := range mock.Calls { + if strings.HasPrefix(c, "curl") { + t.Errorf("tcp mode must never run the HTTP probe: %s", c) + } + } +} + +func TestHealthCheck_TCPModeFailsWhenDialFails(t *testing.T) { + mock := ssh.NewMockExecutor("h", + ssh.MockCommand{Match: "bash -c", Err: fmt.Errorf("connection refused")}, + ssh.MockCommand{Match: "curl", Output: "200"}, + ) + d := &Deployer{exec: mock, out: nopWriter{}} + + cfg := HealthConfig{Mode: "tcp", Timeout: 300 * time.Millisecond, Interval: 10 * time.Millisecond} + if err := d.healthCheck(context.Background(), 3456, cfg, ""); err == nil { + t.Fatal("tcp mode must fail when the dial fails") + } + for _, c := range mock.Calls { + if strings.HasPrefix(c, "curl") { + t.Errorf("tcp mode must never run the HTTP probe: %s", c) + } + } +} + +// Explicit auto behaves exactly like the historical default: HTTP first, +// 404/3xx falls back to the TCP dial. +func TestHealthCheck_AutoModeExplicitFallsBack(t *testing.T) { + mock := ssh.NewMockExecutor("h", + ssh.MockCommand{Match: "curl", Output: "301"}, + ssh.MockCommand{Match: "bash -c", Output: ""}, + ) + d := &Deployer{exec: mock, out: nopWriter{}} + + cfg := HealthConfig{Mode: "auto", Timeout: 2 * time.Second, Interval: 10 * time.Millisecond} + if err := d.healthCheck(context.Background(), 3456, cfg, ""); err != nil { + t.Fatalf("healthCheck auto: %v", err) + } + var sawDial bool + for _, c := range mock.Calls { + if strings.HasPrefix(c, "bash -c") { + sawDial = true + } + } + if !sawDial { + t.Error("auto mode must fall back to the TCP dial on 3xx") + } +} + +// --- surfaced readiness line --- + +func TestReadinessSummary(t *testing.T) { + cases := []struct { + name string + cfg HealthConfig + port int + want string + }{ + {"http", HealthConfig{Mode: "http", Path: "/healthz", Timeout: 30 * time.Second, Interval: time.Second}, 3000, "HTTP GET /healthz (30s deadline)"}, + {"tcp", HealthConfig{Mode: "tcp", Timeout: 30 * time.Second}, 3000, "TCP :3000 (30s)"}, + {"auto-empty", HealthConfig{Timeout: 30 * time.Second}, 3000, "auto — HTTP then TCP fallback (compat, 30s deadline)"}, + {"auto-explicit", HealthConfig{Mode: "auto", Timeout: 45 * time.Second}, 3000, "auto — HTTP then TCP fallback (compat, 45s deadline)"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + cfg := tc.cfg.withDefaults() + cfg.Timeout = tc.cfg.Timeout + if tc.cfg.Path != "" { + cfg.Path = tc.cfg.Path + } + if got := readinessSummary(cfg, tc.port); got != tc.want { + t.Errorf("readinessSummary = %q, want %q", got, tc.want) + } + }) + } +} + +// --- total deadline --- + +// blockingExecutor models a probe endpoint that never answers: every +// command hangs until its context dies. The gate must still fail within +// the configured timeout — the timeout is a TOTAL deadline, not a +// per-attempt bound on unbounded retries. +type blockingExecutor struct { + mu sync.Mutex + calls []string +} + +func (b *blockingExecutor) Run(ctx context.Context, cmd string) (string, error) { + b.mu.Lock() + b.calls = append(b.calls, cmd) + b.mu.Unlock() + <-ctx.Done() + return "", ctx.Err() +} + +func (b *blockingExecutor) RunStream(ctx context.Context, cmd string, stdout, stderr io.Writer) error { + _, err := b.Run(ctx, cmd) + return err +} + +func (b *blockingExecutor) RunInput(ctx context.Context, cmd string, stdin io.Reader) error { + return nil +} + +func (b *blockingExecutor) Upload(ctx context.Context, content io.Reader, remotePath string, mode string) error { + return nil +} + +func (b *blockingExecutor) Close() error { return nil } +func (b *blockingExecutor) Host() string { return "h" } +func (b *blockingExecutor) User() string { return "root" } +func (b *blockingExecutor) Calls() []string { + b.mu.Lock() + defer b.mu.Unlock() + return append([]string(nil), b.calls...) +} + +func TestHealthCheck_NeverRespondingProbeFailsWithinDeadline(t *testing.T) { + exec := &blockingExecutor{} + d := &Deployer{exec: exec, out: nopWriter{}} + + const timeout = 400 * time.Millisecond + cfg := HealthConfig{Mode: "http", Timeout: timeout, Interval: 10 * time.Millisecond} + start := time.Now() + err := d.healthCheck(context.Background(), 3456, cfg, "") + elapsed := time.Since(start) + if err == nil { + t.Fatal("a never-responding probe must fail") + } + if elapsed > timeout+2*time.Second { + t.Errorf("gate must fail within deadline+slack, took %s (deadline %s)", elapsed, timeout) + } + if len(exec.Calls()) == 0 { + t.Error("probe never attempted") + } +} + +// --- shared execution-plan validator --- + +func TestConfigValidate_HealthModeEnum(t *testing.T) { + base := Config{App: "myapp", Domain: "myapp.com", Image: "myapp:latest", Version: "abc123"} + for _, mode := range []string{"", "http", "tcp", "auto"} { + c := base + c.Health = HealthConfig{Mode: mode} + if err := c.validate(); err != nil { + t.Errorf("mode %q: unexpected validate error: %v", mode, err) + } + } + c := base + c.Health = HealthConfig{Mode: "grpc"} + err := c.validate() + if err == nil { + t.Fatal("expected validate error for unknown health mode") + } + if !strings.Contains(err.Error(), "health mode") { + t.Errorf("error should name the health mode, got: %v", err) + } +} + +// --- deploy-path wiring --- + +// firstDeployMock is the TestDeploy_FirstDeploy script, parameterized on the +// probe commands the gate is expected to run. +func firstDeployMock(healthCurl, dial *ssh.MockCommand) *ssh.MockExecutor { + cmds := []ssh.MockCommand{ + {Match: "mkdir -p /deployments/myapp", Output: ""}, + {Match: "mkdir /deployments/myapp/.lock", Output: ""}, + {Match: "if [ ! -e '/deployments/myapp/state.json' ]", Output: "absent"}, + {Match: "if [ ! -e '/deployments/myapp/state' ]", Output: "absent"}, + {Match: "ss -tln", Output: ssOutput}, + {Match: "docker run", Output: "abc123def456"}, + {Match: "docker inspect -f '{{.Image}}'", Output: "sha256:" + strings.Repeat("a", 64)}, + {Match: "docker inspect", Output: "running"}, + } + if healthCurl != nil { + cmds = append(cmds, *healthCurl) + } + if dial != nil { + cmds = append(cmds, *dial) + } + cmds = append(cmds, + ssh.MockCommand{Match: "curl -sf http://localhost:2019/config/apps/http/servers/srv0", Output: `{"listen":[":80",":443"]}`}, + ssh.MockCommand{Match: "curl -sf -X PATCH", Err: fmt.Errorf("not found")}, + ssh.MockCommand{Match: "curl -sf -X POST http://localhost:2019/config/apps/http/servers/srv0/routes", Output: ""}, + ssh.MockCommand{Match: "rm -f /tmp/teploy_caddy", Output: ""}, + ssh.MockCommand{Match: "cat /deployments/caddy/Caddyfile", Output: "{\n\tadmin 0.0.0.0:2019\n}\n"}, + ssh.MockCommand{Match: "mv /tmp/teploy_caddyfile.tmp", Output: ""}, + ssh.MockCommand{Match: "mkdir /deployments/caddy/.lock", Output: ""}, + ssh.MockCommand{Match: "a=$(docker exec caddy md5sum", Output: "TEPLOY_CADDY_OK"}, + ssh.MockCommand{Match: "docker exec caddy caddy reload", Output: ""}, + ssh.MockCommand{Match: "rmdir /deployments/caddy/.lock", Output: ""}, + ssh.MockCommand{Match: "printf %s", Output: ""}, + ssh.MockCommand{Match: "rm -rf /deployments/myapp/.lock", Output: ""}, + ) + return ssh.NewMockExecutor("1.2.3.4", cmds...) +} + +func firstDeployConfig(health HealthConfig) Config { + manifest := json.RawMessage(`{"app":"myapp","env_keys":["TOKEN"]}`) + return Config{ + App: "myapp", + Domain: "myapp.com", + Image: "myapp:latest", + Version: "abc123", + Health: health, + ManifestSHA256: fmt.Sprintf("%x", sha256.Sum256(manifest)), + AppliedManifest: manifest, + } +} + +// The deploy output states which mode and deadline gate the traffic switch +// BEFORE the gate runs, and an http-mode deploy probes the configured path. +func TestDeploy_HTTPModeSurfacesReadinessLineBeforeGate(t *testing.T) { + mock := firstDeployMock(&ssh.MockCommand{Match: "curl -s -o /dev/null", Output: "200"}, nil) + var buf bytes.Buffer + d := NewDeployer(mock, &buf) + + cfg := firstDeployConfig(HealthConfig{Mode: "http", Path: "/healthz", Timeout: 5 * time.Second, Interval: 10 * time.Millisecond}) + if err := d.Deploy(context.Background(), cfg); err != nil { + t.Fatalf("Deploy: %v", err) + } + + output := buf.String() + line := "Readiness: HTTP GET /healthz (5s deadline)" + if !strings.Contains(output, line) { + t.Errorf("output must state the readiness gate up front, got: %s", output) + } + if strings.Index(output, line) > strings.Index(output, "Health check passed") { + t.Errorf("readiness line must precede the gate result, got: %s", output) + } + var probed string + for _, c := range mock.Calls { + if strings.HasPrefix(c, "curl -s -o /dev/null") { + probed = c + } + } + if !strings.Contains(probed, "/healthz") { + t.Errorf("http mode must probe the configured path, got: %s", probed) + } +} + +// A tcp-mode deploy gates on the dial alone: no HTTP probe command is ever +// issued, the surfaced line names TCP + the port, and the release record +// carries the mode for rollback. +func TestDeploy_TCPModeGatesOnDialNeverCurl(t *testing.T) { + mock := firstDeployMock(nil, &ssh.MockCommand{Match: "bash -c", Output: ""}) + var buf bytes.Buffer + d := NewDeployer(mock, &buf) + + cfg := firstDeployConfig(HealthConfig{Mode: "tcp", Timeout: 5 * time.Second, Interval: 10 * time.Millisecond}) + if err := d.Deploy(context.Background(), cfg); err != nil { + t.Fatalf("Deploy: %v", err) + } + + output := buf.String() + if !strings.Contains(output, "Readiness: TCP :49152 (5s)") { + t.Errorf("output must surface the TCP gate, got: %s", output) + } + var sawDial bool + for _, c := range mock.Calls { + if strings.HasPrefix(c, "curl -s -o /dev/null") { + t.Errorf("tcp-mode deploy must never run the HTTP probe: %s", c) + } + if strings.HasPrefix(c, "bash -c") { + sawDial = true + } + } + if !sawDial { + t.Error("tcp-mode deploy must gate on the dial") + } + var recordedMode string + for _, data := range mock.Files { + if bytes.Contains(data, []byte(`"health"`)) { + var probe struct { + Health *struct { + Mode string `json:"mode"` + } `json:"health"` + } + if json.Unmarshal(data, &probe) == nil && probe.Health != nil { + recordedMode = probe.Health.Mode + } + } + } + if recordedMode != "tcp" { + t.Errorf("release record must carry health mode tcp for rollback, got %q", recordedMode) + } +} + +// The auto default (mode empty) surfaces as the named compat mode. +func TestDeploy_AutoDefaultSurfacesCompatLine(t *testing.T) { + mock := firstDeployMock(&ssh.MockCommand{Match: "curl -s -o /dev/null", Output: "200"}, nil) + var buf bytes.Buffer + d := NewDeployer(mock, &buf) + + cfg := firstDeployConfig(HealthConfig{Timeout: 5 * time.Second, Interval: 10 * time.Millisecond}) + if err := d.Deploy(context.Background(), cfg); err != nil { + t.Fatalf("Deploy: %v", err) + } + if !strings.Contains(buf.String(), "Readiness: auto — HTTP then TCP fallback (compat, 5s deadline)") { + t.Errorf("auto default must surface the compat line, got: %s", buf.String()) + } +} + +// --- rollback record forwarding --- + +func recordWithHealthMode(mode string) *releasemeta.Record { + rec := &releasemeta.Record{App: "myapp", Hash: "v1", IngressMode: "caddy"} + if mode != "" { + rec.Health = &releasemeta.Health{Mode: mode, Path: "/rec", TimeoutSeconds: 20, IntervalSeconds: 2} + } + return rec +} + +func TestApplyRecordToRollback_ForwardsHealthMode(t *testing.T) { + cfg := &RollbackConfig{App: "myapp", Domain: "myapp.com"} + health := HealthConfig{Path: "/healthz", Timeout: 30 * time.Second, Interval: time.Second} + rec := recordWithHealthMode("tcp") + applyRecordToRollback(cfg, rec, &health) + if health.Mode != "tcp" { + t.Errorf("rollback health mode = %q, want tcp from the record", health.Mode) + } + // An old record without a mode keeps whatever the config said (compat). + health = HealthConfig{Mode: "http"} + applyRecordToRollback(cfg, recordWithHealthMode(""), &health) + if health.Mode != "http" { + t.Errorf("modeless record must not override, got %q", health.Mode) + } +} diff --git a/internal/deploy/journal.go b/internal/deploy/journal.go index c59aab6..f72ccf2 100644 --- a/internal/deploy/journal.go +++ b/internal/deploy/journal.go @@ -210,7 +210,12 @@ type readinessProbe struct { Container string `json:"container,omitempty"` Host string `json:"host,omitempty"` Port int `json:"port"` - Path string `json:"path,omitempty"` + // Path is the URL path probed in http/auto mode; empty when the record + // predates modes and none was resolvable. + Path string `json:"path,omitempty"` + // Mode is the probe mode that was in effect (http / tcp / auto) — what + // a recovery owner should re-run to reproduce the gate. + Mode string `json:"mode,omitempty"` } // readinessReceiptPath is the receipt's location in the attempt diff --git a/internal/deploy/rollback.go b/internal/deploy/rollback.go index 0894be7..9471d82 100644 --- a/internal/deploy/rollback.go +++ b/internal/deploy/rollback.go @@ -349,6 +349,11 @@ func Rollback(ctx context.Context, exec ssh.Executor, out io.Writer, cfg Rollbac healthBindHost = dk.HostBindIP(ctx, c.Name) } } + // Surface the gate before it runs (C03): which probe mode — from the + // target release's recorded spec when one exists — and what deadline. + if len(healthPorts) > 0 { + fmt.Fprintf(out, " Readiness: %s\n", readinessSummary(healthCfg, healthPorts[0])) + } for _, p := range healthPorts { if err := deployer.healthCheck(ctx, p, healthCfg, healthBindHost); err != nil { // Stop what we started and bail. @@ -364,7 +369,6 @@ func Rollback(ctx context.Context, exec ssh.Executor, out io.Writer, cfg Rollbac } } fmt.Fprintln(out, " Health check passed") - // 4. Route traffic to the target container(s). // Use the explicit container name(s) rather than the app network alias // so Docker DNS doesn't briefly round-robin to the current (about-to- @@ -607,6 +611,9 @@ func applyRecordToRollback(cfg *RollbackConfig, rec *releasemeta.Record, healthC cfg.Domain = rec.Domain } if rec.Health != nil { + if rec.Health.Mode != "" { + healthCfg.Mode = rec.Health.Mode + } if rec.Health.Path != "" { healthCfg.Path = rec.Health.Path } diff --git a/internal/releasemeta/releasemeta.go b/internal/releasemeta/releasemeta.go index e24183f..13f99fd 100644 --- a/internal/releasemeta/releasemeta.go +++ b/internal/releasemeta/releasemeta.go @@ -73,10 +73,11 @@ type Port struct { Fixed bool `json:"fixed,omitempty"` } -// Health records the deploy-time health gate so a rollback probes the path +// Health records the deploy-time health gate so a rollback probes the way // the target release was actually deployed with, not whatever the current // teploy.yml says. type Health struct { + Mode string `json:"mode,omitempty"` Path string `json:"path,omitempty"` TimeoutSeconds int `json:"timeout_seconds,omitempty"` IntervalSeconds int `json:"interval_seconds,omitempty"` @@ -86,13 +87,13 @@ type Health struct { // so rollback restores the target release's edge config instead of the // current config file's. type CaddyRoute struct { - TLSCert string `json:"tls_cert,omitempty"` - TLSKey string `json:"tls_key,omitempty"` - TLSInternal bool `json:"tls_internal,omitempty"` - CaddyExtra string `json:"caddy_extra,omitempty"` - Cache map[string]string `json:"cache,omitempty"` - Firewall *caddy.Firewall `json:"firewall,omitempty"` - Access *caddy.Access `json:"access,omitempty"` + TLSCert string `json:"tls_cert,omitempty"` + TLSKey string `json:"tls_key,omitempty"` + TLSInternal bool `json:"tls_internal,omitempty"` + CaddyExtra string `json:"caddy_extra,omitempty"` + Cache map[string]string `json:"cache,omitempty"` + Firewall *caddy.Firewall `json:"firewall,omitempty"` + Access *caddy.Access `json:"access,omitempty"` } // Static records a type:static release's serving configuration — the piece From 6faefc482890cab859b70d099611a8ca3d404043 Mon Sep 17 00:00:00 2001 From: Tyler <53561637+im-tyler@users.noreply.github.com> Date: Tue, 22 Sep 2026 22:16:04 -0700 Subject: [PATCH 5/8] feat(releasemeta,build,deploy): build provenance + plan/receipt digest equality (C04) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every attempt records provenance before execution: revision + dirty flag, context fingerprint (new build.ContextFingerprint — build had none), Dockerfile identity, platform, requested ref, resolved digest and digest-pinned flag, manifest digest; persisted in the attempt namespace and embedded in the F14 record. The plan shows digest + revision before any effect; after the live commit, record digest == plan digest is verified (pinned refs can't mismatch by construction) — a mismatch warns and files repair debt, never fails live traffic. Retry stability pinned: resolution is attempt-independent; webhook retries re-pin to the ledger commit. --- AUDIT_OPEN.md | 98 ++++++++++++ internal/build/build.go | 12 +- internal/build/fingerprint.go | 169 ++++++++++++++++++++ internal/build/fingerprint_test.go | 155 ++++++++++++++++++ internal/cli/autodeploy_serve.go | 6 +- internal/cli/deploy.go | 22 ++- internal/cli/provenance.go | 118 ++++++++++++++ internal/cli/provenance_test.go | 184 +++++++++++++++++++++ internal/cli/singledeploy.go | 7 + internal/deploy/deploy.go | 68 +++++++- internal/deploy/provenance.go | 150 ++++++++++++++++++ internal/deploy/provenance_test.go | 202 ++++++++++++++++++++++++ internal/releasemeta/provenance.go | 142 +++++++++++++++++ internal/releasemeta/provenance_test.go | 142 +++++++++++++++++ internal/releasemeta/releasemeta.go | 8 + 15 files changed, 1463 insertions(+), 20 deletions(-) create mode 100644 internal/build/fingerprint.go create mode 100644 internal/build/fingerprint_test.go create mode 100644 internal/cli/provenance.go create mode 100644 internal/cli/provenance_test.go create mode 100644 internal/deploy/provenance.go create mode 100644 internal/deploy/provenance_test.go create mode 100644 internal/releasemeta/provenance.go create mode 100644 internal/releasemeta/provenance_test.go diff --git a/AUDIT_OPEN.md b/AUDIT_OPEN.md index 13f93de..4826091 100644 --- a/AUDIT_OPEN.md +++ b/AUDIT_OPEN.md @@ -1600,3 +1600,101 @@ block's HTTP path probe would mark a non-HTTP upstream down — LB rendering is 5bf5594's fixed surface, untouched here), and preview's readiness gate (internal/preview) which mirrors the auto shape and has no mode surface of its own. + +## Programme slice (2026-09-22, latest) — C04: build provenance + plan/receipt equality + +First bounded C04 slice (base revision `6a1142d`; changes left uncommitted +for review). Contract addressed: "Resolve Git revision, build context, +Dockerfile, platform and immutable image digest BEFORE execution... image +digest and effective configuration shown in plan equal the deployed +receipt; response-loss retries do not build a different source; changed +mutable tags behave according to selected policy." Commit pinning itself +was P0-done in C02 (webhook builds reset to the authenticated commit; +this slice records what every path resolved). + +**Design:** + +- `releasemeta.Provenance` (new internal/releasemeta/provenance.go) is the + plan-time record: revision (full HEAD sha via the SourceRevision + threading both trigger paths already had), a Dirty flag, build context + path + context fingerprint, Dockerfile identity (path + content sha), + target platform, requested image ref, the immutable digest resolved + BEFORE execution, a DigestPinned-vs-mutable flag, and the + effective-config (manifest) digest. Persisted as `provenance.json` in + the F08 attempt namespace (`meta/att/./`, write-once, atomic + 0600, identity-validated on read — the journal discipline) by the shared + post-build orchestration: `deployBuiltImageFenced` (manual + ad-hoc + + autodeploy, new `sourceRoot` param: "." vs the fetched checkout) and + `singleServerDeployer.deployApp` (multi-server/scale) — all three + engine entry paths. +- Recon finding: the build package did NOT already compute a context + fingerprint — the only tree-hash machinery was the static deployer's + unexported `hashDir` (static-only semantics, symlink-rejecting). Added + `build.ContextFingerprint(dir, excludes)` with hashDir's v3 typed/ + length-prefixed record encoding (F51/TCL-38 discipline) but + symlink-INCLUSIVE (hashed by target: rsync -a preserves links into the + context, so a link is build input), excludes applied (DefaultIgnore + + .teployignore — the fingerprint describes the synced tree). Also + extracted `build.EffectiveLocalPlatform` from localBuildDockerfile's + inline rule (behavior-preserving) so the record names the platform the + local build actually targets. +- Plan/receipt equality: `Deployer.DeployFenced` prints a plan block + BEFORE any effect (image digest — not just tag, with pinned/mutable + stated; revision, flagging a dirty worktree as "building uncommitted + changes"; context fingerprint; Dockerfile identity; platform; manifest + digest). The F14 record gains `ManifestSHA256` + embedded `Provenance`, + and `recordRelease` returns it. A closing verification (step 17, after + the live commit) asserts record digest == plan digest and reports + equality explicitly; a mismatch is a loud warning + the C01-6 + repair-debt marker (next deploy reconciles) — never a failed live + deploy. `plannedImageDigest` applies ONE like-for-like rule on both + sides: a digest-pinned ref is identified by its manifest digest, + everything else by docker's resolved content ID (`ImageDigestFromRef`, + now exported) — otherwise a pinned ref's plan (manifest digest) and + record (image ID) could never agree by construction. +- Retry stability verified + pinned: the attempt machinery gives each + invocation a fresh write-once namespace, so a retry lands BESIDE the + first receipt (test); source stability is resolution purity — + `resolveDeployProvenance` is a pure function of (config, tree, image), + no time- or attempt-dependent fields (they are stamped at write) — + pinned by a DeepEqual double-resolution test; webhook retries + additionally re-pin to the ledger commit (C02, unchanged). + +**Evidence** — TDD red first: all four new test files failed to compile +against the absent machinery (undefined Provenance/WriteAttemptProvenance/ +ContextFingerprint/Config.Provenance...). New coverage: provenance +round-trip + write-time foreign-identity refusal + read-time identity +mismatch refusal + absent-is-nil-nil; distinct immutable receipts for two +attempts of one release; fingerprint determinism/sensitivity (content at +constant size, rename, empty-dir structure, symlink retarget) + +exclude-honoring; resolution field capture for build/prebuilt/mutable-tag +paths; dirty-worktree flag; retry stability; plan output (digest, +revision, "building uncommitted changes", fingerprint, manifest digest — +and printed before the first container starts); record embedding +provenance + manifest digest; mismatch → loud warning naming both digests ++ repair-debt marker + deploy still succeeds; provenance/deploy identity +mismatch refused pre-effect; no plan digest → no false alarm. Mutation +checks (in-place, all reverted): equality check disabled → mismatch test +fails; provenance fields dropped from the plan print → plan test fails on +the missing fingerprint; dirty suffix severed → "building uncommitted +changes" assertion fails; record stops embedding provenance → record test +fails; fingerprint made content-blind (digest zeroed, size kept, against +a same-size content edit) → sensitivity test fails. Gates after revert: +`go vet ./...` clean; `go test ./... -race -count=1` all 25 packages ok; +gofmt clean on every touched hunk (cli/deploy.go's pre-existing +fleet-rollback region stray left alone, consistent with the C02/C03 +posture); contract probes 5/5 PASS. No push performed. + +**C04 remainder (explicit):** registry authentication provenance (recording +WHICH credential identity pulled/built — nothing today names the docker +config/secret used); scan/attestation separation (trivy's gate currently +FAILS the deploy on scan error — the contract wants scan failures distinct +from build failures, and attestation is unmodelled); the ARM64/AMD64 +packaging matrix (cross-platform build verification on supported targets — +`platform` is now RECORDED everywhere but not matrix-tested); offline +fallback as an explicit pull POLICY (today's behavior — digest-pinned +cache reuse, mutable always-pull, warned local fallback — is now recorded +as provenance facts, not yet a selectable policy); build records for +`teploy build` outside deploys; cache diagnostics; secret-safe build-input +attestation. Changed-mutable-tag POLICY (beyond recording pinned-vs- +mutable + the mismatch warning) lands with the offline/pull-policy slice. diff --git a/internal/build/build.go b/internal/build/build.go index 6000913..936f2bd 100644 --- a/internal/build/build.go +++ b/internal/build/build.go @@ -9,7 +9,6 @@ import ( "os/exec" "path" "path/filepath" - "runtime" "strings" "github.com/useteploy/teploy/internal/ssh" @@ -237,12 +236,11 @@ func LocalBuild(ctx context.Context, cfg LocalBuildConfig, stdout io.Writer) (st func localBuildDockerfile(ctx context.Context, tag, dir, contextSub, dockerfile, platform string, stdout io.Writer) error { args := []string{"build", "-t", tag} - if platform != "" { - // Explicit platform from config. - args = append(args, "--platform", platform) - } else if runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" { - // Cross-compile for linux/amd64 when building on macOS ARM. - args = append(args, "--platform", "linux/amd64") + // Explicit config, else the shared local-build default (Apple silicon + // cross-compiles for linux/amd64) — one rule for the build and its + // C04 provenance record. + if p := EffectiveLocalPlatform(platform); p != "" { + args = append(args, "--platform", p) } // exec.Command takes an argv, so no shell quoting is needed here. diff --git a/internal/build/fingerprint.go b/internal/build/fingerprint.go new file mode 100644 index 0000000..50fc314 --- /dev/null +++ b/internal/build/fingerprint.go @@ -0,0 +1,169 @@ +package build + +import ( + "crypto/sha256" + "encoding/binary" + "encoding/hex" + "fmt" + "io" + "io/fs" + "os" + "path/filepath" + "runtime" + "sort" +) + +// ContextFingerprint returns the sha256 fingerprint of the build-context +// tree dir would transfer: every file's path and content, every directory's +// path, and every symlink's target, with the given exclude patterns +// (rsync-style: matched against each entry's base name and its +// slash-separated relative path) left out — the fingerprint describes the +// SOURCE the builder consumes, not the operator's local clutter. +// +// Encoding discipline mirrors the static deployer's v3 tree hash (audit +// F51/TCL-38): typed, length-prefixed records for EVERY entry — +// directories included — emitted in sorted path order, so no two distinct +// trees can collide through framing ambiguity. Permission bits are ignored +// (umask stability across machines). Unlike the static hash, symlinks are +// INCLUDED, hashed by target: rsync -a preserves links into the build +// context, so a link is build input whose identity is what it points at. +// This is a provenance identity, not a security boundary. +func ContextFingerprint(dir string, excludes []string) (string, error) { + if dir == "" { + dir = "." + } + type entry struct { + rel string + kind byte // 'f' file, 'd' directory, 'l' symlink + size int64 + digest [32]byte + target string + } + var entries []entry + pruned := func(rel string) bool { + if len(excludes) == 0 { + return false + } + base := filepath.Base(rel) + for _, pat := range excludes { + if pat == "" { + continue + } + if ok, _ := filepath.Match(pat, base); ok { + return true + } + if ok, _ := filepath.Match(pat, rel); ok { + return true + } + } + return false + } + + root := filepath.Clean(dir) + err := filepath.WalkDir(root, func(p string, d fs.DirEntry, walkErr error) error { + if walkErr != nil { + return walkErr + } + if p == root { + return nil + } + rel, err := filepath.Rel(root, p) + if err != nil { + return err + } + rel = filepath.ToSlash(rel) + if pruned(rel) { + if d.IsDir() { + return fs.SkipDir + } + return nil + } + info, err := d.Info() + if err != nil { + return err + } + switch { + case info.Mode()&fs.ModeSymlink != 0: + target, err := os.Readlink(p) + if err != nil { + return err + } + entries = append(entries, entry{rel: rel, kind: 'l', target: target}) + case info.IsDir(): + entries = append(entries, entry{rel: rel, kind: 'd'}) + case info.Mode().IsRegular(): + digest, size, err := hashFile(p) + if err != nil { + return err + } + entries = append(entries, entry{rel: rel, kind: 'f', size: size, digest: digest}) + default: + // Sockets, devices and FIFOs cannot be synced as build input; + // record their presence so the fingerprint still moves. + entries = append(entries, entry{rel: rel, kind: 's'}) + } + return nil + }) + if err != nil { + return "", fmt.Errorf("fingerprinting build context %s: %w", dir, err) + } + sort.Slice(entries, func(i, j int) bool { return entries[i].rel < entries[j].rel }) + + h := sha256.New() + h.Write([]byte("teploy-context-v1\x00")) + var num [8]byte + binary.BigEndian.PutUint64(num[:], uint64(len(entries))) + h.Write(num[:]) + var len8 [8]byte + writeStr := func(s string) { + binary.BigEndian.PutUint64(len8[:], uint64(len(s))) + h.Write(len8[:]) + h.Write([]byte(s)) + } + for _, e := range entries { + h.Write([]byte{e.kind}) + writeStr(e.rel) + switch e.kind { + case 'f': + binary.BigEndian.PutUint64(len8[:], uint64(e.size)) + h.Write(len8[:]) + h.Write(e.digest[:]) + case 'l': + writeStr(e.target) + } + } + return hex.EncodeToString(h.Sum(nil)), nil +} + +func hashFile(path string) ([32]byte, int64, error) { + f, err := os.Open(path) + if err != nil { + return [32]byte{}, 0, err + } + defer f.Close() + h := sha256.New() + size, err := io.Copy(h, f) + if err != nil { + return [32]byte{}, 0, err + } + var digest [32]byte + copy(digest[:], h.Sum(nil)) + return digest, size, nil +} + +// EffectiveLocalPlatform returns the platform a LOCAL Dockerfile build +// targets for the given configured platform: an explicit platform wins; +// otherwise building on Apple silicon targets linux/amd64 (the deploy +// default since the first local-build support — an arm64 macOS host +// otherwise produces images the typical amd64 server cannot run); anywhere +// else the daemon's native default applies (empty). Shared by the build +// itself and the C04 provenance record so both name the same platform. +func EffectiveLocalPlatform(platform string) string { + if platform != "" { + return platform + } + if runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" { + return "linux/amd64" + } + return "" +} diff --git a/internal/build/fingerprint_test.go b/internal/build/fingerprint_test.go new file mode 100644 index 0000000..526e4ae --- /dev/null +++ b/internal/build/fingerprint_test.go @@ -0,0 +1,155 @@ +package build + +import ( + "os" + "path/filepath" + "runtime" + "testing" +) + +func writeTree(t *testing.T, dir string, files map[string]string) { + t.Helper() + for rel, content := range files { + p := filepath.Join(dir, rel) + if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(p, []byte(content), 0o644); err != nil { + t.Fatal(err) + } + } +} + +// The context fingerprint is the provenance identity of the synced tree: +// identical trees must fingerprint identically (retry stability), and any +// change the builder could observe — content, path, structure, symlink +// target — must move it. +func TestContextFingerprint_DeterministicAndSensitive(t *testing.T) { + base := map[string]string{ + "main.go": "package main", + "cmd/app/run.go": "func Run() {}", + "Dockerfile": "FROM alpine", + "web/index.html": "", + "web/empty/.keep": "", + "docs/README.md": "# docs", + } + a, b := t.TempDir(), t.TempDir() + writeTree(t, a, base) + writeTree(t, b, base) + + fa, err := ContextFingerprint(a, DefaultIgnore) + if err != nil { + t.Fatalf("ContextFingerprint: %v", err) + } + fb, err := ContextFingerprint(b, DefaultIgnore) + if err != nil { + t.Fatalf("ContextFingerprint: %v", err) + } + if fa == "" || fa != fb { + t.Fatalf("identical trees must fingerprint identically: %q vs %q", fa, fb) + } + + // Re-resolving the SAME tree (a second attempt) is stable. + fa2, err := ContextFingerprint(a, DefaultIgnore) + if err != nil || fa2 != fa { + t.Fatalf("same tree re-fingerprinted differently: %q vs %q (%v)", fa, fa2, err) + } + + // Content change moves the fingerprint (same size, different bytes — + // isolates content from length). + if err := os.WriteFile(filepath.Join(b, "main.go"), []byte("package mian"), 0o644); err != nil { + t.Fatal(err) + } + if fb, err = ContextFingerprint(b, DefaultIgnore); err != nil || fb == fa { + t.Fatalf("a content change must move the fingerprint: %q vs %q (%v)", fa, fb, err) + } + + // Path change (rename, same bytes) moves the fingerprint. + writeTree(t, b, base) + if err := os.Rename(filepath.Join(b, "docs"), filepath.Join(b, "docz")); err != nil { + t.Fatal(err) + } + if fb, err = ContextFingerprint(b, DefaultIgnore); err != nil || fb == fa { + t.Fatalf("a rename must move the fingerprint (path is part of identity): %q vs %q (%v)", fa, fb, err) + } + + // Directory-structure change (new empty dir) moves the fingerprint. + writeTree(t, b, base) + if err := os.MkdirAll(filepath.Join(b, "brand/new/dir"), 0o755); err != nil { + t.Fatal(err) + } + if fb, err = ContextFingerprint(b, DefaultIgnore); err != nil || fb == fa { + t.Fatalf("a new empty directory must move the fingerprint (TCL-38 parity): %q vs %q (%v)", fa, fb, err) + } +} + +// The fingerprint must describe the tree that gets SYNCED: the always- +// protected patterns (and .teployignore extensions) are not build input +// and must not influence the identity. +func TestContextFingerprint_HonorsExcludePatterns(t *testing.T) { + a, b := t.TempDir(), t.TempDir() + core := map[string]string{"Dockerfile": "FROM alpine", "app.py": "print(1)"} + writeTree(t, a, core) + writeTree(t, b, core) + writeTree(t, b, map[string]string{ + "node_modules/pkg/index.js": "junk", + ".git/config": "junk", + ".env": "SECRET=1", + ".env.local": "SECRET=2", + }) + + fa, err := ContextFingerprint(a, DefaultIgnore) + if err != nil { + t.Fatal(err) + } + fb, err := ContextFingerprint(b, DefaultIgnore) + if err != nil { + t.Fatal(err) + } + if fa != fb { + t.Fatalf("excluded patterns leaked into the fingerprint: %q vs %q", fa, fb) + } +} + +func TestContextFingerprint_SymlinkTargetMovesIdentity(t *testing.T) { + a, b := t.TempDir(), t.TempDir() + for _, dir := range []*string{&a, &b} { + if err := os.WriteFile(filepath.Join(*dir, "target-a"), []byte("A"), 0o644); err != nil { + t.Fatal(err) + } + } + if err := os.Symlink("target-a", filepath.Join(a, "link")); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(b, "target-a"), []byte("A"), 0o644); err != nil { + t.Fatal(err) + } + if err := os.Symlink("elsewhere", filepath.Join(b, "link")); err != nil { + t.Fatal(err) + } + fa, err := ContextFingerprint(a, DefaultIgnore) + if err != nil { + t.Fatal(err) + } + fb, err := ContextFingerprint(b, DefaultIgnore) + if err != nil { + t.Fatal(err) + } + if fa == fb { + t.Fatal("a symlink retarget must move the fingerprint (rsync preserves links; the builder sees the target)") + } +} + +func TestEffectiveLocalPlatform(t *testing.T) { + if got := EffectiveLocalPlatform("linux/arm64"); got != "linux/arm64" { + t.Errorf("explicit platform must win: %q", got) + } + // The implicit cross-compile rule only exists on Apple silicon. + want := "" + if runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" { + want = "linux/amd64" + } + if got := EffectiveLocalPlatform(""); got != want { + t.Errorf("implicit platform: got %q want %q", got, want) + } +} diff --git a/internal/cli/autodeploy_serve.go b/internal/cli/autodeploy_serve.go index 3bd9ba5..0383259 100644 --- a/internal/cli/autodeploy_serve.go +++ b/internal/cli/autodeploy_serve.go @@ -704,9 +704,11 @@ func triggerAutoDeploy(ctx context.Context, executor ssh.Executor, app, branch, // The outer lock taken at the top of triggerAutoDeploy is still held — // route through the fenced entry point so Deploy doesn't deadlock on its // own second acquisition (audit F07), passing the fence handle (F16) and - // the attempt that keys this deploy's env/TLS artifacts (F08). + // the attempt that keys this deploy's env/TLS artifacts (F08). The + // checkout is the provenance source root (C04): revision and context + // fingerprint describe the fetched tree this deploy builds. att := releasemeta.MustAttempt(app, version) - return deployBuiltImageFenced(ctx, executor, appCfg, image, version, "localhost", false, needsBuild, lk, &att) + return deployBuiltImageFenced(ctx, executor, appCfg, image, version, "localhost", false, needsBuild, buildDir, lk, &att) } // fetchCheckout advances buildDir's origin and resets the worktree to the diff --git a/internal/cli/deploy.go b/internal/cli/deploy.go index 61f270a..b8e30ce 100644 --- a/internal/cli/deploy.go +++ b/internal/cli/deploy.go @@ -495,7 +495,7 @@ func deployAppConfig(flags *Flags, appCfg *config.AppConfig, serverName, image, } } - return deployBuiltImageFenced(ctx, executor, appCfg, image, version, host, migrateVolumes, needsBuild, lk, &att) + return deployBuiltImageFenced(ctx, executor, appCfg, image, version, host, migrateVolumes, needsBuild, ".", lk, &att) } // deployBuiltImage runs the shared post-build deploy orchestration: @@ -513,7 +513,7 @@ func deployAppConfig(flags *Flags, appCfg *config.AppConfig, serverName, image, // string for the notification payload (a hostname for the SSH path, // "localhost" for the resident-server path). func deployBuiltImage(ctx context.Context, executor ssh.Executor, appCfg *config.AppConfig, image, version, serverDisplay string, migrateVolumes, needsBuild bool) error { - return deployBuiltImageFenced(ctx, executor, appCfg, image, version, serverDisplay, migrateVolumes, needsBuild, nil, nil) + return deployBuiltImageFenced(ctx, executor, appCfg, image, version, serverDisplay, migrateVolumes, needsBuild, "", nil, nil) } // deployBuiltImageFenced is deployBuiltImage with the caller's lease and @@ -521,7 +521,10 @@ func deployBuiltImage(ctx context.Context, executor ssh.Executor, appCfg *config // lease and the resident autodeploy path — audits F07/F08) and att keys the // attempt-scoped artifacts (env file, TLS). lk == nil means Deployer.Deploy // acquires the lock itself (att must still be non-nil for the env file). -func deployBuiltImageFenced(ctx context.Context, executor ssh.Executor, appCfg *config.AppConfig, image, version, serverDisplay string, migrateVolumes, needsBuild bool, lk *state.Lock, att *releasemeta.Attempt) error { +// sourceRoot is the directory the source was synced/built from ("." for +// manual deploys, the fetched checkout for autodeploy) — it keys the +// plan-time provenance (C04); empty means no build provenance. +func deployBuiltImageFenced(ctx context.Context, executor ssh.Executor, appCfg *config.AppConfig, image, version, serverDisplay string, migrateVolumes, needsBuild bool, sourceRoot string, lk *state.Lock, att *releasemeta.Attempt) error { if att == nil { attVal := releasemeta.MustAttempt(appCfg.App, version) att = &attVal @@ -531,6 +534,18 @@ func deployBuiltImageFenced(ctx context.Context, executor ssh.Executor, appCfg * return fmt.Errorf("normalizing applied manifest: %w", err) } + // 8b. Capture and persist the plan-time provenance (C04): revision, + // worktree cleanliness, build-context fingerprint, Dockerfile + // identity, platform, the resolved image digest, and the mutability + // of the requested ref — resolved BEFORE execution and filed into the + // attempt's write-once namespace, next to the build context it + // describes. Best-effort resolution, but the receipt itself must + // land: a missing provenance file is an unwitnessed plan (warned). + prov := resolveDeployProvenance(ctx, executor, os.Stdout, appCfg, sourceRoot, image, version, manifestSHA256, needsBuild) + if err := releasemeta.WriteAttemptProvenance(ctx, executor, *att, prov); err != nil { + fmt.Printf("Warning: could not persist the deploy provenance receipt for %s@%s: %v\n", appCfg.App, version, err) + } + // 9. Ensure accessories are running. var envFile string if len(appCfg.Accessories) > 0 { @@ -647,6 +662,7 @@ func deployBuiltImageFenced(ctx context.Context, executor ssh.Executor, appCfg * // 11. Deploy. deployer := deploy.NewDeployer(executor, os.Stdout) deployCfg := deployConfigFromApp(appCfg, image, version, envFiles, volumes, tlsCert, tlsKey, tlsInternal, appliedManifest, manifestSHA256) + deployCfg.Provenance = prov // Vulnerability gate: scan the image on the server before any container // starts — fixable CRITICALs block the deploy. diff --git a/internal/cli/provenance.go b/internal/cli/provenance.go new file mode 100644 index 0000000..085f016 --- /dev/null +++ b/internal/cli/provenance.go @@ -0,0 +1,118 @@ +package cli + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "strings" + + "github.com/useteploy/teploy/internal/build" + "github.com/useteploy/teploy/internal/config" + "github.com/useteploy/teploy/internal/deploy" + "github.com/useteploy/teploy/internal/docker" + "github.com/useteploy/teploy/internal/releasemeta" + "github.com/useteploy/teploy/internal/ssh" +) + +// resolveDeployProvenance captures the plan-time provenance of a deploy +// (programme C04): the resolved source identity, build inputs, and the +// immutable image identity — all BEFORE execution. The result is a PURE +// function of (app config, source tree, image): a response-loss retry of +// the same request resolves byte-identically, which is what the retry- +// stability contract pins. Callers persist it via releasemeta. +// WriteAttemptProvenance into the attempt namespace, which stamps the +// attempt-scoped fields. +// +// sourceRoot is the directory the source syncs from (the operator's cwd +// for manual deploys, the fetched checkout for autodeploy); empty or +// needsBuild=false skips build provenance (a prebuilt-image deploy has no +// build inputs — its provenance is the requested ref plus the digest it +// resolved to). Best-effort by contract: a provenance field that cannot +// be resolved warns and stays empty — provenance must never fail a +// deploy. +func resolveDeployProvenance(ctx context.Context, exec ssh.Executor, out io.Writer, appCfg *config.AppConfig, sourceRoot, image, version, manifestSHA256 string, needsBuild bool) *releasemeta.Provenance { + prov := &releasemeta.Provenance{ + App: appCfg.App, + Release: version, + Revision: appCfg.SourceRevision, + ImageRef: image, + DigestPinned: isDigestPinned(image), + ManifestSHA256: manifestSHA256, + } + + // Like-for-like with the deployed record's digest: a digest-pinned + // reference is identified by its own digest; everything else by + // docker's resolved content ID. + if digest := deploy.ImageDigestFromRef(image); digest != "" { + prov.ImageDigest = digest + } else if resolved, err := docker.NewClient(exec).ResolveImageID(ctx, image); err == nil { + prov.ImageDigest = resolved + } else { + fmt.Fprintf(out, "Warning: could not resolve an immutable image ID for %s before execution — plan/receipt digest equality will not be verifiable (%v)\n", image, err) + } + + if !needsBuild || sourceRoot == "" { + return prov + } + + // Build provenance. Dirty first: the flag the plan surfaces as + // "building uncommitted changes". + prov.Dirty = gitDirtyIn(sourceRoot) + + prov.ContextPath = appCfg.Context + if prov.ContextPath == "" { + prov.ContextPath = "." + } + contextDir := sourceRoot + if appCfg.Context != "" && appCfg.Context != "." { + contextDir = filepath.Join(sourceRoot, appCfg.Context) + } + if excludes, err := build.LoadIgnore(sourceRoot); err != nil { + fmt.Fprintf(out, "Warning: could not load the ignore rules for the context fingerprint: %v\n", err) + } else if fp, err := build.ContextFingerprint(contextDir, excludes); err != nil { + fmt.Fprintf(out, "Warning: could not fingerprint the build context: %v\n", err) + } else { + prov.ContextFingerprint = fp + } + + // Dockerfile identity: path + content hash. A tree without one is a + // Nixpacks build — both fields stay empty. + dockerfile := appCfg.Dockerfile + if dockerfile == "" { + dockerfile = "Dockerfile" + } + if data, err := os.ReadFile(filepath.Join(contextDir, dockerfile)); err == nil { + prov.Dockerfile = dockerfile + prov.DockerfileSHA256 = sha256Hex(data) + } + + // Platform: the configured target; local Dockerfile builds get the + // shared local-build default so the record names what the build ran. + prov.Platform = appCfg.Platform + prov.Local = appCfg.BuildLocal + if prov.Platform == "" && appCfg.BuildLocal && prov.Dockerfile != "" { + prov.Platform = build.EffectiveLocalPlatform("") + } + return prov +} + +// gitDirtyIn reports whether dir's worktree has uncommitted changes. Not a +// git repository (or git unavailable) is not dirty — it is nothing this +// deploy would build over a named revision. +func gitDirtyIn(dir string) bool { + out, err := exec.Command("git", "-C", dir, "status", "--porcelain").Output() + if err != nil { + return false + } + return strings.TrimSpace(string(out)) != "" +} + +func sha256Hex(data []byte) string { + sum := sha256.Sum256(data) + return hex.EncodeToString(sum[:]) +} diff --git a/internal/cli/provenance_test.go b/internal/cli/provenance_test.go new file mode 100644 index 0000000..f9d051a --- /dev/null +++ b/internal/cli/provenance_test.go @@ -0,0 +1,184 @@ +package cli + +import ( + "bytes" + "context" + "io" + "os" + "os/exec" + "path/filepath" + "reflect" + "strings" + "testing" + + "github.com/useteploy/teploy/internal/config" + "github.com/useteploy/teploy/internal/ssh" +) + +func gitRun(t *testing.T, dir string, args ...string) string { + t.Helper() + cmd := exec.Command("git", append([]string{"-C", dir}, args...)...) + var out bytes.Buffer + cmd.Stdout = &out + cmd.Stderr = &out + if err := cmd.Run(); err != nil { + t.Fatalf("git %v: %v\n%s", args, err, out.String()) + } + return strings.TrimSpace(out.String()) +} + +// provenanceRepo builds a committed git worktree with a Dockerfile and a +// source file, returning the repo root. +func provenanceRepo(t *testing.T) string { + t.Helper() + dir := t.TempDir() + gitRun(t, dir, "init", "-q") + gitRun(t, dir, "-c", "user.email=ops@example.com", "-c", "user.name=ops", + "commit", "--allow-empty", "-qm", "root") + for rel, content := range map[string]string{ + "Dockerfile": "FROM alpine\n", + "main.go": "package main\n", + } { + if err := os.WriteFile(filepath.Join(dir, rel), []byte(content), 0o644); err != nil { + t.Fatal(err) + } + } + gitRun(t, dir, "add", "-A") + gitRun(t, dir, "-c", "user.email=ops@example.com", "-c", "user.name=ops", "commit", "-qm", "app") + return dir +} + +func provenanceMock(resolvedID string) *ssh.MockExecutor { + return ssh.NewMockExecutor("1.2.3.4", + ssh.MockCommand{Match: "docker image inspect --format '{{.Id}}'", Output: resolvedID}, + ) +} + +func TestResolveDeployProvenance_BuildPath(t *testing.T) { + dir := provenanceRepo(t) + rev, err := gitRevisionIn(dir) + if err != nil { + t.Fatal(err) + } + + appCfg := &config.AppConfig{App: "myapp", Platform: "linux/arm64"} + appCfg.SourceRevision = rev + + prov := resolveDeployProvenance(context.Background(), provenanceMock("sha256:"+strings.Repeat("b", 64)), io.Discard, + appCfg, dir, "myapp-build-v1", "v1", strings.Repeat("c", 64), true) + + if prov.App != "myapp" || prov.Release != "v1" { + t.Errorf("identity fields wrong: %+v", prov) + } + if prov.Revision != rev { + t.Errorf("revision not captured: %q want %q", prov.Revision, rev) + } + if prov.Dirty { + t.Errorf("a clean committed worktree must not be flagged dirty") + } + if prov.ImageRef != "myapp-build-v1" || prov.ImageDigest != "sha256:"+strings.Repeat("b", 64) { + t.Errorf("image identity not captured: %+v", prov) + } + if prov.DigestPinned { + t.Errorf("a built tag is not digest-pinned") + } + if prov.ContextPath != "." || prov.ContextFingerprint == "" { + t.Errorf("build context identity not captured: %+v", prov) + } + if prov.Dockerfile != "Dockerfile" || prov.DockerfileSHA256 != fileSHA256(t, filepath.Join(dir, "Dockerfile")) { + t.Errorf("dockerfile identity not captured: %+v", prov) + } + if prov.Platform != "linux/arm64" { + t.Errorf("configured platform not captured: %q", prov.Platform) + } + if prov.ManifestSHA256 != strings.Repeat("c", 64) { + t.Errorf("effective-config digest not captured: %q", prov.ManifestSHA256) + } +} + +// The dirty-worktree flag must flow into provenance: a deploy building +// uncommitted changes says so in the record that outlives it. +func TestResolveDeployProvenance_DirtyWorktreeFlagged(t *testing.T) { + dir := provenanceRepo(t) + if err := os.WriteFile(filepath.Join(dir, "main.go"), []byte("package main // WIP\n"), 0o644); err != nil { + t.Fatal(err) + } + rev, err := gitRevisionIn(dir) + if err != nil { + t.Fatal(err) + } + appCfg := &config.AppConfig{App: "myapp"} + appCfg.SourceRevision = rev + + prov := resolveDeployProvenance(context.Background(), provenanceMock(""), io.Discard, + appCfg, dir, "myapp-build-v1", "v1", "", true) + if !prov.Dirty { + t.Fatal("a worktree with uncommitted changes must be flagged dirty in provenance") + } +} + +func TestResolveDeployProvenance_PrebuiltImage(t *testing.T) { + pinned := "registry.example.com/myapp@sha256:" + strings.Repeat("7", 64) + appCfg := &config.AppConfig{App: "myapp"} + + prov := resolveDeployProvenance(context.Background(), provenanceMock("sha256:"+strings.Repeat("b", 64)), io.Discard, + appCfg, ".", pinned, "sha256-777777777777", strings.Repeat("c", 64), false) + + if !prov.DigestPinned { + t.Error("a digest-pinned reference must be recorded as pinned") + } + // Like-for-like with the deployed record: a pinned ref is identified by + // ITS manifest digest, not docker's local image ID for those bytes. + if prov.ImageDigest != "sha256:"+strings.Repeat("7", 64) { + t.Errorf("pinned ref digest not captured: %q", prov.ImageDigest) + } + if prov.ImageRef != pinned { + t.Errorf("requested ref not captured: %q", prov.ImageRef) + } + if prov.ContextPath != "" || prov.ContextFingerprint != "" || prov.Dockerfile != "" || prov.DockerfileSHA256 != "" || prov.Platform != "" { + t.Errorf("prebuilt deploy must not invent build provenance: %+v", prov) + } +} + +func TestResolveDeployProvenance_MutableTagResolvesContentID(t *testing.T) { + appCfg := &config.AppConfig{App: "myapp"} + prov := resolveDeployProvenance(context.Background(), provenanceMock("sha256:"+strings.Repeat("b", 64)), io.Discard, + appCfg, ".", "myapp:latest", "1750000000", "", false) + if prov.DigestPinned { + t.Error("a tag reference is mutable, not pinned") + } + if prov.ImageDigest != "sha256:"+strings.Repeat("b", 64) { + t.Errorf("the resolved immutable ID must be captured for a mutable tag: %q", prov.ImageDigest) + } +} + +// C04 retry stability: a response-loss retry of the SAME request reuses the +// same provenance — resolution is a pure function of the source tree and +// config, so the second attempt cannot re-resolve to a different source. +func TestResolveDeployProvenance_RetryStability(t *testing.T) { + dir := provenanceRepo(t) + rev, err := gitRevisionIn(dir) + if err != nil { + t.Fatal(err) + } + appCfg := &config.AppConfig{App: "myapp", Platform: "linux/arm64"} + appCfg.SourceRevision = rev + + first := resolveDeployProvenance(context.Background(), provenanceMock("sha256:"+strings.Repeat("b", 64)), io.Discard, + appCfg, dir, "myapp-build-v1", "v1", strings.Repeat("c", 64), true) + second := resolveDeployProvenance(context.Background(), provenanceMock("sha256:"+strings.Repeat("b", 64)), io.Discard, + appCfg, dir, "myapp-build-v1", "v1", strings.Repeat("c", 64), true) + + if !reflect.DeepEqual(first, second) { + t.Fatalf("the same request resolved differently across attempts:\nfirst: %+v\nsecond: %+v", first, second) + } +} + +func fileSHA256(t *testing.T, path string) string { + t.Helper() + data, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + return sha256Hex(data) +} diff --git a/internal/cli/singledeploy.go b/internal/cli/singledeploy.go index d6c9be1..4582b93 100644 --- a/internal/cli/singledeploy.go +++ b/internal/cli/singledeploy.go @@ -269,8 +269,15 @@ func (s *singleServerDeployer) deployApp(ctx context.Context, appCfg *config.App if err != nil { return fmt.Errorf("normalizing applied manifest: %w", err) } + // Plan-time provenance (C04), same as the single-server path: resolved + // before execution and filed into THIS server's attempt namespace. + prov := resolveDeployProvenance(ctx, s.exec, s.out, appCfg, ".", image, version, manifestSHA256, needsBuild) + if err := releasemeta.WriteAttemptProvenance(ctx, s.exec, att, prov); err != nil { + fmt.Fprintf(s.out, "Warning: could not persist the deploy provenance receipt for %s@%s: %v\n", appCfg.App, version, err) + } deployer := deploy.NewDeployer(s.exec, s.out) deployCfg := deployConfigFromApp(appCfg, image, version, envFiles, volumes, tlsCert, tlsKey, tlsInternal, appliedManifest, manifestSHA256) + deployCfg.Provenance = prov // Vulnerability gate (see deploy.go): fixable CRITICALs block before // containers start. Per-server, so every box in a multi-server deploy diff --git a/internal/deploy/deploy.go b/internal/deploy/deploy.go index a692752..0bd761e 100644 --- a/internal/deploy/deploy.go +++ b/internal/deploy/deploy.go @@ -89,6 +89,13 @@ type Config struct { ManifestSHA256 string AppliedManifest json.RawMessage SourceRevision string + // Provenance is the plan-time provenance record (C04) the CLI + // resolved before execution: revision, worktree cleanliness, build + // context fingerprint, Dockerfile identity, platform, image digest + // and mutability. Optional (direct construction without provenance + // skips the plan/receipt equality machinery); when present its + // identity must match the deploy's own. + Provenance *releasemeta.Provenance } // Deployer orchestrates zero-downtime deploys. @@ -194,6 +201,12 @@ func (c Config) validate() error { default: return fmt.Errorf("unknown health mode %q (expected http, tcp, or auto)", c.Health.Mode) } + // Provenance identity (C04): a record describing another deploy than + // the Config it rides on is a lie that would corrupt the plan/receipt + // equality surfaces — refused before any effect. + if c.Provenance != nil && (c.Provenance.App != c.App || c.Provenance.Release != c.Version) { + return fmt.Errorf("provenance identity mismatch: provenance describes %s@%s, deploy is %s@%s", c.Provenance.App, c.Provenance.Release, c.App, c.Version) + } return nil } @@ -312,6 +325,14 @@ func (d *Deployer) DeployFenced(ctx context.Context, cfg Config, lk *state.Lock) fmt.Fprintf(d.out, "Deploying %s (version %s)...\n", cfg.App, cfg.Version) } + // 0b. Surface the execution plan (C04): the immutable image identity + // this deploy will create containers from, the source revision it + // builds, and the effective-config digest — BEFORE any effect. The + // same values are verified against the deployed receipt after the + // live commit (step 17). + planDigest := plannedImageDigest(cfg.Image, runImage, cfg.Provenance) + printDeployPlan(d.out, cfg, planDigest) + // 1. Read current state. A read failure must stop the deploy — treating // an unreadable state file as "no state" loses rollback bookkeeping and // makes a replacement deploy look like a first deploy (audit F15). @@ -836,7 +857,7 @@ func (d *Deployer) DeployFenced(ctx context.Context, cfg Config, lk *state.Lock) newState.AppliedManifest = append(json.RawMessage(nil), cfg.AppliedManifest...) newState.SourceRevision = cfg.SourceRevision newState.ImageRef = cfg.Image - newState.ImageDigest = imageDigestFromRef(cfg.Image) + newState.ImageDigest = ImageDigestFromRef(cfg.Image) if newState.ImageDigest == "" { newState.ImageDigest, _ = d.docker.ContainerImageDigest(ctx, webContainerName) } @@ -856,8 +877,9 @@ func (d *Deployer) DeployFenced(ctx context.Context, cfg Config, lk *state.Lock) // the route/state are committed — a record failure is a degraded // rollback window, not a failed deploy, and it converges on the next // deploy via the repair-debt marker (C01-6). Never abort into - // abortStateCommit from here. - d.recordRelease(ctx, cfg, assetAttempt, newState, ports, webBindHost, webContainerName) + // abortStateCommit from here. The written record is returned for the + // closing plan/receipt verification (17). + rec := d.recordRelease(ctx, cfg, assetAttempt, newState, ports, webBindHost, webContainerName) // 13c. Prune superseded attempts (F08): attempt directories (build // contexts, env files, TLS certs) are dead weight once their release @@ -1029,7 +1051,16 @@ func (d *Deployer) DeployFenced(ctx context.Context, cfg Config, lk *state.Lock) } duration := time.Since(start) + + // 17. Closing verification (C04): after the live commit, assert the + // deployed record's image digest equals the digest the plan showed — + // and report the receipt's identity triple explicitly. A mismatch is + // a loud warning plus repair debt (the C01-6 marker the next deploy + // reconciles); live traffic is never failed over a bookkeeping gap. + d.verifyPlanReceiptEquality(ctx, cfg, rec, assetAttempt, planDigest) + fmt.Fprintf(d.out, "\nDeployed %s version %s in %s\n", cfg.App, cfg.Version, duration.Round(time.Millisecond)) + printDeployReceipt(d.out, cfg, rec) return nil } @@ -1363,10 +1394,14 @@ func (d *Deployer) logDeploy(ctx context.Context, cfg Config, success bool, degr // binding plus every publish entry); env records the references (server-side // env-file paths + the plaintext env map), never resolved secrets. The // primary web container's full RecreateSpec is embedded from docker's own -// view of it. A write failure is deliberate degradation (the deploy stays -// live) made durable and convergent: the repair-debt marker it records -// (C01-6) drives the next deploy's rebuild and `status`'s reporting. -func (d *Deployer) recordRelease(ctx context.Context, cfg Config, att releasemeta.Attempt, applied *state.AppState, ports []int, webBindHost, webContainerName string) { +// view of it, and the plan-time provenance + effective-config digest ride +// along (C04) so the receipt names what the plan promised. A write failure +// is deliberate degradation (the deploy stays live) made durable and +// convergent: the repair-debt marker it records (C01-6) drives the next +// deploy's rebuild and `status`'s reporting. The in-memory record is +// returned even on a write failure — the closing verification compares +// against what was attempted. +func (d *Deployer) recordRelease(ctx context.Context, cfg Config, att releasemeta.Attempt, applied *state.AppState, ports []int, webBindHost, webContainerName string) *releasemeta.Record { containerPort := cfg.ContainerPort if containerPort == 0 { containerPort = 80 @@ -1382,6 +1417,7 @@ func (d *Deployer) recordRelease(ctx context.Context, cfg Config, att releasemet Domain: cfg.Domain, ImageRef: cfg.Image, ImageDigest: applied.ImageDigest, + ManifestSHA256: cfg.ManifestSHA256, Replicas: len(ports), Processes: maps.Clone(cfg.Processes), Cmd: cfg.Cmd, @@ -1440,10 +1476,19 @@ func (d *Deployer) recordRelease(ctx context.Context, cfg Config, att releasemet } else { fmt.Fprintf(d.out, "Warning: could not capture the recreate spec for %s: %v (recreate falls back to live inspect)\n", webContainerName, err) } + // Embed the plan-time provenance (C04): the record is the deployed + // receipt — it must carry the digest, revision and config the plan + // showed, verbatim. A copy, so the caller's struct is never aliased + // into the persisted record. + if cfg.Provenance != nil { + provCopy := *cfg.Provenance + rec.Provenance = &provCopy + } if err := releasemeta.Write(ctx, d.exec, rec); err != nil { fmt.Fprintf(d.out, "Warning: could not record release metadata for %s@%s: %v — the deploy stays live; repair debt recorded (the next deploy rebuilds the record)\n", cfg.App, cfg.Version, err) d.recordRepairDebt(ctx, att, err) } + return rec } // sortedProcessNames returns process names with "web" first, then alphabetical. @@ -1480,7 +1525,14 @@ func containerPort(c Config) int { return c.ContainerPort } -func imageDigestFromRef(image string) string { +// ImageDigestFromRef extracts the digest of a digest-pinned image +// reference ("repo@sha256:<64hex>"), or "" for every other reference +// shape. Exported for the CLI's plan-time provenance, which must apply +// the SAME like-for-like rule the deployed record applies (a pinned ref +// is identified by its manifest digest, a mutable ref by docker's +// resolved content ID) or plan/receipt equality compares apples to +// oranges. +func ImageDigestFromRef(image string) string { if _, digest, ok := strings.Cut(image, "@"); ok && strings.HasPrefix(digest, "sha256:") && len(digest) == len("sha256:")+64 { return digest } diff --git a/internal/deploy/provenance.go b/internal/deploy/provenance.go new file mode 100644 index 0000000..7ad6098 --- /dev/null +++ b/internal/deploy/provenance.go @@ -0,0 +1,150 @@ +// Plan/receipt equality surfaces (programme workstream C04). +// +// The contract: "image digest and effective configuration shown in plan +// equal the deployed receipt". Three pieces live here: +// +// - plannedImageDigest — the ONE rule both sides of the equality use to +// name an image's immutable identity (a digest-pinned reference by its +// manifest digest, everything else by docker's resolved content ID), +// so plan and receipt can never disagree by construction; +// - printDeployPlan / printDeployReceipt — the pre-execution plan and +// the post-commit receipt, both showing image DIGEST (not just the +// tag), source revision, and the effective-config (manifest) digest; +// - verifyPlanReceiptEquality — the closing verification after the live +// commit: record digest == plan digest, equality reported explicitly, +// a mismatch a loud warning plus repair debt (the C01-6 marker the +// next deploy reconciles). Never a failed live deploy. +package deploy + +import ( + "context" + "fmt" + "io" + + "github.com/useteploy/teploy/internal/releasemeta" +) + +// plannedImageDigest resolves the immutable image identity the plan +// commits to. Priority: the provenance record's digest (resolved by the +// CLI BEFORE execution — the plan's own promise), then the reference's +// own digest (digest-pinned refs), then the deployer's just-resolved +// content ID. Empty when no immutable identity could be established — +// callers treat that as "nothing to verify", not as a digest. +func plannedImageDigest(ref string, resolvedID string, prov *releasemeta.Provenance) string { + if prov != nil && prov.ImageDigest != "" { + return prov.ImageDigest + } + if digest := ImageDigestFromRef(ref); digest != "" { + return digest + } + if resolvedID != "" && resolvedID != ref { + return resolvedID + } + return "" +} + +// printDeployPlan surfaces the deploy's immutable identity BEFORE any +// effect: image digest (not just the tag), source revision — flagging a +// dirty worktree as "building uncommitted changes" — build context +// fingerprint, Dockerfile identity, platform, and the effective-config +// digest. Fields the caller could not resolve print as unavailable rather +// than silently disappearing. +func printDeployPlan(out io.Writer, cfg Config, planDigest string) { + fmt.Fprintln(out, "Plan:") + digest := planDigest + if digest == "" { + digest = "(unresolved)" + } + pinned := "" + if cfg.Provenance != nil && cfg.Provenance.DigestPinned { + pinned = ", digest-pinned" + } else if cfg.Provenance != nil { + pinned = ", mutable tag" + } + fmt.Fprintf(out, " Image: %s (%s%s)\n", cfg.Image, digest, pinned) + revision := cfg.SourceRevision + if revision == "" && cfg.Provenance != nil { + revision = cfg.Provenance.Revision + } + if revision != "" { + note := "" + if cfg.Provenance != nil && cfg.Provenance.Dirty { + note = " — building uncommitted changes" + } + fmt.Fprintf(out, " Revision: %s%s\n", revision, note) + } + if cfg.Provenance != nil { + if cfg.Provenance.ContextPath != "" || cfg.Provenance.ContextFingerprint != "" { + fingerprint := cfg.Provenance.ContextFingerprint + if fingerprint == "" { + fingerprint = "(unavailable)" + } + fmt.Fprintf(out, " Context: %s (fingerprint %s)\n", cfg.Provenance.ContextPath, fingerprint) + } + if cfg.Provenance.Dockerfile != "" { + fmt.Fprintf(out, " Dockerfile: %s (sha256 %s)\n", cfg.Provenance.Dockerfile, cfg.Provenance.DockerfileSHA256) + } + if cfg.Provenance.Platform != "" { + fmt.Fprintf(out, " Platform: %s\n", cfg.Provenance.Platform) + } + } + if cfg.ManifestSHA256 != "" { + fmt.Fprintf(out, " Config: manifest %s\n", cfg.ManifestSHA256) + } +} + +// printDeployReceipt prints the post-commit receipt's identity triple — +// image digest, revision, effective-config digest — the same values the +// plan showed, now sourced from the deployed record. +func printDeployReceipt(out io.Writer, cfg Config, rec *releasemeta.Record) { + digest := "" + if rec != nil { + digest = rec.ImageDigest + } + if digest == "" && cfg.Provenance != nil { + digest = cfg.Provenance.ImageDigest + } + if digest == "" { + digest = "(unrecorded)" + } + revision := cfg.SourceRevision + if revision == "" && cfg.Provenance != nil { + revision = cfg.Provenance.Revision + } + if revision == "" { + revision = "(none)" + } + manifest := cfg.ManifestSHA256 + if manifest == "" && cfg.Provenance != nil { + manifest = cfg.Provenance.ManifestSHA256 + } + if manifest == "" { + manifest = "(none)" + } + fmt.Fprintf(out, "Receipt: image %s, revision %s, config manifest %s\n", digest, revision, manifest) +} + +// verifyPlanReceiptEquality is the closing verification (C04): after the +// live commit, the deployed record's image digest must equal the digest +// the plan showed. Equality is reported explicitly; a mismatch is a loud +// warning plus repair debt via the C01-6 marker (the next deploy's +// reconciler rebuilds the record from the live containers — the deployed +// truth) — live traffic is never failed over the gap. A plan with no +// resolvable digest verifies nothing (legacy direct construction; the +// resolution failure already warned when it happened); a receipt with no +// digest is an honest "could not verify", never a silent pass. +func (d *Deployer) verifyPlanReceiptEquality(ctx context.Context, cfg Config, rec *releasemeta.Record, att releasemeta.Attempt, planDigest string) { + if planDigest == "" { + return + } + if rec == nil || rec.ImageDigest == "" { + fmt.Fprintf(d.out, "Warning: plan/receipt equality NOT verified — the deployed record carries no image digest to compare against the plan's %s\n", planDigest) + return + } + if rec.ImageDigest != planDigest { + fmt.Fprintf(d.out, "WARNING: plan/receipt mismatch — the plan showed image %s but the deployed release records %s: the live image differs from what was planned (changed mutable tag or concurrent re-pointing). Repair debt recorded; the next deploy reconciles the record. Live traffic is untouched.\n", planDigest, rec.ImageDigest) + d.recordRepairDebt(ctx, att, fmt.Errorf("plan/receipt image digest mismatch: planned %s for %s, deployed %s", planDigest, cfg.Image, rec.ImageDigest)) + return + } + fmt.Fprintf(d.out, "Verified: the deployed image digest equals the plan (%s)\n", planDigest) +} diff --git a/internal/deploy/provenance_test.go b/internal/deploy/provenance_test.go new file mode 100644 index 0000000..4beb618 --- /dev/null +++ b/internal/deploy/provenance_test.go @@ -0,0 +1,202 @@ +package deploy + +// C04 plan/receipt equality tests: the deploy plan output (before +// execution) shows the immutable image digest, source revision and +// effective-config digest; the post-deploy receipt repeats them; a closing +// verification asserts record digest == plan digest and a mismatch is a +// loud warning plus repair debt, never a failed live deploy. + +import ( + "bytes" + "context" + "encoding/json" + "strings" + "testing" + + "github.com/useteploy/teploy/internal/releasemeta" + "github.com/useteploy/teploy/internal/ssh" +) + +func c04Provenance(digest string, dirty bool) *releasemeta.Provenance { + return &releasemeta.Provenance{ + App: "myapp", + Release: "v9", + Revision: "462d7a7b1c2d3e4f5a6b7c8d9e0f1a2b3c4d5e6f", + Dirty: dirty, + ContextPath: ".", + ContextFingerprint: strings.Repeat("f", 64), + Dockerfile: "Dockerfile", + DockerfileSHA256: strings.Repeat("d", 64), + Platform: "linux/amd64", + ImageRef: "myapp:v9", + ImageDigest: digest, + ManifestSHA256: strings.Repeat("c", 64), + } +} + +func TestDeploy_PlanOutputShowsDigestRevisionAndConfig(t *testing.T) { + imageDigest := "sha256:" + strings.Repeat("a", 64) + mock := deployRecordFixture(imageDigest, + ssh.MockCommand{Match: "docker image inspect --format '{{.Id}}'", Output: imageDigest}, + ) + + var buf bytes.Buffer + err := NewDeployer(mock, &buf).Deploy(context.Background(), Config{ + App: "myapp", + Domain: "myapp.com", + Image: "myapp:v9", + Version: "v9", + SourceRevision: "462d7a7b1c2d3e4f5a6b7c8d9e0f1a2b3c4d5e6f", + ManifestSHA256: strings.Repeat("c", 64), + Provenance: c04Provenance(imageDigest, true), + }) + if err != nil { + t.Fatalf("Deploy: %v\n%s", err, buf.String()) + } + + out := buf.String() + // The plan appears BEFORE execution: its output precedes the first + // container start, which precedes everything else the deploy mutates. + planIdx := strings.Index(out, "Plan:") + if planIdx < 0 { + t.Fatalf("no plan output:\n%s", out) + } + runLine := strings.Index(out, "Starting container myapp-web-v9") + if runLine >= 0 && planIdx > runLine { + t.Fatalf("the plan must be printed before execution begins:\n%s", out) + } + + for _, want := range []string{ + "Plan:", + imageDigest, + "462d7a7b1c2d3e4f5a6b7c8d9e0f1a2b3c4d5e6f", + "building uncommitted changes", + strings.Repeat("f", 64), + strings.Repeat("c", 64), + "Verified: the deployed image digest equals the plan (" + imageDigest + ")", + "Receipt: image " + imageDigest, + } { + if !strings.Contains(out, want) { + t.Errorf("deploy output missing %q:\n%s", want, out) + } + } + + // The F14 receipt embeds the provenance and the effective-config digest. + raw, ok := mock.Files["/deployments/myapp/meta/v9.json"] + if !ok { + t.Fatalf("release record not written\n%s", out) + } + var rec releasemeta.Record + if err := json.Unmarshal(raw, &rec); err != nil { + t.Fatalf("invalid record: %v", err) + } + if rec.ManifestSHA256 != strings.Repeat("c", 64) { + t.Errorf("record does not carry the effective-config digest: %q", rec.ManifestSHA256) + } + if rec.Provenance == nil { + t.Fatalf("record does not embed the deploy provenance") + } + if rec.Provenance.Revision != "462d7a7b1c2d3e4f5a6b7c8d9e0f1a2b3c4d5e6f" || + rec.Provenance.ContextFingerprint != strings.Repeat("f", 64) || + rec.Provenance.ImageDigest != imageDigest { + t.Errorf("record's embedded provenance lost fields: %+v", rec.Provenance) + } +} + +// A clean worktree must not claim uncommitted changes. +func TestDeploy_PlanCleanWorktreeHasNoDirtyWarning(t *testing.T) { + imageDigest := "sha256:" + strings.Repeat("a", 64) + mock := deployRecordFixture(imageDigest, + ssh.MockCommand{Match: "docker image inspect --format '{{.Id}}'", Output: imageDigest}, + ) + var buf bytes.Buffer + err := NewDeployer(mock, &buf).Deploy(context.Background(), Config{ + App: "myapp", Domain: "myapp.com", Image: "myapp:v9", Version: "v9", + SourceRevision: "462d7a7b1c2d3e4f5a6b7c8d9e0f1a2b3c4d5e6f", + Provenance: c04Provenance(imageDigest, false), + }) + if err != nil { + t.Fatalf("Deploy: %v\n%s", err, buf.String()) + } + if strings.Contains(buf.String(), "building uncommitted changes") { + t.Errorf("clean worktree flagged as dirty:\n%s", buf.String()) + } + if !strings.Contains(buf.String(), "Verified: the deployed image digest equals the plan") { + t.Errorf("equality must still be verified:\n%s", buf.String()) + } +} + +// The closing verification: a record digest that disagrees with the plan is +// a LOUD warning + repair debt (C01-6's marker), never a failed live +// deploy. +func TestDeploy_PlanReceiptMismatchWarnsAndRecordsRepairDebt(t *testing.T) { + recordDigest := "sha256:" + strings.Repeat("a", 64) + planDigest := "sha256:" + strings.Repeat("e", 64) + mock := deployRecordFixture(recordDigest, + ssh.MockCommand{Match: "docker image inspect --format '{{.Id}}'", Output: planDigest}, + ) + + var buf bytes.Buffer + err := NewDeployer(mock, &buf).Deploy(context.Background(), Config{ + App: "myapp", + Domain: "myapp.com", + Image: "myapp:v9", + Version: "v9", + Provenance: c04Provenance(planDigest, false), + }) + if err != nil { + t.Fatalf("a plan/receipt mismatch must not fail the live deploy: %v\n%s", err, buf.String()) + } + + out := buf.String() + if !strings.Contains(out, "plan/receipt mismatch") && !strings.Contains(out, "plan/receipt MISMATCH") { + t.Errorf("expected a loud mismatch warning:\n%s", out) + } + if !strings.Contains(out, planDigest) || !strings.Contains(out, recordDigest) { + t.Errorf("the warning must name both digests:\n%s", out) + } + if strings.Contains(out, "Verified: the deployed image digest equals the plan") { + t.Errorf("a mismatch must not also report equality:\n%s", out) + } + + debtRaw, ok := mock.Files["/deployments/myapp/repair-debt.json"] + if !ok { + t.Fatalf("mismatch must record repair debt via the C01-6 marker\n%s", out) + } + debt := string(debtRaw) + if !strings.Contains(debt, planDigest) || !strings.Contains(debt, recordDigest) { + t.Errorf("repair-debt marker does not describe the mismatch:\n%s", debt) + } +} + +// A provenance record describing a different deploy than the Config it +// rides on is an identity lie — refused before any effect. +func TestDeploy_ProvenanceIdentityMismatchRefused(t *testing.T) { + prov := c04Provenance("sha256:"+strings.Repeat("a", 64), false) + prov.App = "otherapp" + err := NewDeployer(deployRecordFixture("sha256:"+strings.Repeat("a", 64)), &strings.Builder{}). + Deploy(context.Background(), Config{ + App: "myapp", Domain: "myapp.com", Image: "myapp:v9", Version: "v9", + Provenance: prov, + }) + if err == nil || !strings.Contains(err.Error(), "provenance") { + t.Fatalf("expected a provenance identity refusal, got %v", err) + } +} + +// Without a resolvable plan digest (legacy direct construction, resolution +// unavailable) the equality check stays silent rather than crying wolf — +// the CLI paths that carry provenance always resolve first. +func TestDeploy_NoPlanDigestNoFalseAlarm(t *testing.T) { + mock := deployRecordFixture("sha256:" + strings.Repeat("a", 64)) + var buf bytes.Buffer + err := NewDeployer(mock, &buf).Deploy(context.Background(), Config{ + App: "myapp", Domain: "myapp.com", Image: "myapp:v9", Version: "v9", + }) + if err != nil { + t.Fatalf("Deploy: %v\n%s", err, buf.String()) + } + if strings.Contains(buf.String(), "MISMATCH") || strings.Contains(buf.String(), "mismatch") { + t.Errorf("nothing to compare must not warn:\n%s", buf.String()) + } +} diff --git a/internal/releasemeta/provenance.go b/internal/releasemeta/provenance.go new file mode 100644 index 0000000..d3a69da --- /dev/null +++ b/internal/releasemeta/provenance.go @@ -0,0 +1,142 @@ +// Deploy provenance (programme workstream C04): the plan-time record of +// WHAT a deploy attempt resolved to build and run — the git revision and +// worktree cleanliness at plan time, the build context identity, the +// Dockerfile identity, the target platform, the requested image reference +// and the immutable digest it resolved to BEFORE execution, whether that +// reference was digest-pinned or mutable, and the effective-config +// (manifest) digest. +// +// Persistence follows the attempt-journal discipline: provenance.json is +// written into the F08 attempt namespace +// (/deployments//meta/att/./provenance.json) — write-once +// per attempt, atomic 0600 — so a response-loss retry lands BESIDE the +// first attempt's receipt (never over it), and the F14 record embeds the +// same struct at commit so the deployed receipt carries what the plan +// promised. Reads are identity-validated (T56 parity): a receipt that +// describes another attempt/app is refused, never guessed from. +package releasemeta + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "time" + + "github.com/useteploy/teploy/internal/ssh" + "github.com/useteploy/teploy/internal/state" +) + +// ProvenanceSchemaVersion is the provenance record's schema version. +const ProvenanceSchemaVersion = 1 + +// provenanceFile is the receipt's name in the attempt namespace. +const provenanceFile = "provenance.json" + +// Provenance is the plan-time provenance record of one deploy attempt. +// Field semantics: +// +// - Revision: the full HEAD sha the source resolved to at plan time. +// - Dirty: the worktree had uncommitted changes — the build baked bytes +// no revision names. +// - ContextPath / ContextFingerprint: the configured build context +// (relative; "." when unset) and the sha256 fingerprint of the tree +// that was synced into the attempt's build context (build-package +// encoding, excludes applied). +// - Dockerfile / DockerfileSHA256: the Dockerfile identity — path plus +// content hash. Empty for Nixpacks and prebuilt-image deploys. +// - Platform: the target platform the build targets ("" = the daemon's +// default). +// - ImageRef: the image reference as requested (prebuilt ref, or the +// built tag). +// - ImageDigest: the immutable identity resolved BEFORE execution — the +// reference's own digest when digest-pinned, else docker's resolved +// content ID. Empty when resolution was impossible. +// - DigestPinned: the requested reference was digest-pinned (immutable) +// rather than a mutable tag — the fact the changed-mutable-tag story +// keys on. +// - ManifestSHA256: the effective-config digest (config. +// NormalizeAndDigest) — plan and receipt compare THIS too. +type Provenance struct { + SchemaVersion int `json:"schema_version"` + App string `json:"app"` + Release string `json:"release"` + Attempt string `json:"attempt,omitempty"` + WrittenAt time.Time `json:"written_at,omitempty"` + + Revision string `json:"revision,omitempty"` + Dirty bool `json:"dirty,omitempty"` + ContextPath string `json:"context_path,omitempty"` + ContextFingerprint string `json:"context_fingerprint,omitempty"` + Dockerfile string `json:"dockerfile,omitempty"` + DockerfileSHA256 string `json:"dockerfile_sha256,omitempty"` + Platform string `json:"platform,omitempty"` + Local bool `json:"local,omitempty"` + + ImageRef string `json:"image_ref,omitempty"` + ImageDigest string `json:"image_digest,omitempty"` + DigestPinned bool `json:"digest_pinned,omitempty"` + ManifestSHA256 string `json:"manifest_sha256,omitempty"` +} + +// AttemptProvenancePath is the receipt's location in the attempt namespace. +func AttemptProvenancePath(att Attempt) string { + return att.Dir() + "/" + provenanceFile +} + +// WriteAttemptProvenance persists the receipt atomically into the attempt's +// immutable namespace (0600, sibling temp + rename). The record's identity +// must describe the attempt it is written for; the attempt name and write +// time are stamped here so the caller's struct stays a pure resolution +// result (retry-stable: nothing time- or attempt-dependent enters before +// this point). +func WriteAttemptProvenance(ctx context.Context, exec ssh.Executor, att Attempt, prov *Provenance) error { + if prov == nil { + return fmt.Errorf("provenance is required") + } + if prov.App != att.App || prov.Release != att.Hash { + return fmt.Errorf("provenance identity mismatch: record describes %s@%s, attempt is %s@%s — refusing to file it under the wrong attempt", prov.App, prov.Release, att.App, att.Hash) + } + if prov.SchemaVersion == 0 { + prov.SchemaVersion = ProvenanceSchemaVersion + } + if prov.SchemaVersion != ProvenanceSchemaVersion { + return fmt.Errorf("cannot write provenance schema version %d", prov.SchemaVersion) + } + prov.Attempt = att.Name() + prov.WrittenAt = time.Now().UTC() + + data, err := json.Marshal(prov) + if err != nil { + return fmt.Errorf("marshaling provenance: %w", err) + } + if _, err := exec.Run(ctx, "mkdir -p "+att.Dir()); err != nil { + return fmt.Errorf("creating the attempt directory: %w", err) + } + return ssh.UploadAtomic(ctx, exec, bytes.NewReader(data), AttemptProvenancePath(att), "0600") +} + +// ReadAttemptProvenance loads an attempt's provenance receipt. A confirmed +// missing file returns (nil, nil); every other failure (transport, +// malformed JSON, wrong schema, identity mismatch) is an error — callers +// surface the unreadable provenance instead of guessing from it. +func ReadAttemptProvenance(ctx context.Context, exec ssh.Executor, att Attempt) (*Provenance, error) { + data, present, err := state.ReadRemoteFile(ctx, exec, AttemptProvenancePath(att)) + if err != nil { + return nil, fmt.Errorf("reading provenance for %s: %w", att.Name(), err) + } + if !present { + return nil, nil + } + var prov Provenance + if err := json.Unmarshal(data, &prov); err != nil { + return nil, fmt.Errorf("parsing provenance for %s: %w", att.Name(), err) + } + if prov.SchemaVersion != ProvenanceSchemaVersion { + return nil, fmt.Errorf("unsupported provenance schema version %d for %s", prov.SchemaVersion, att.Name()) + } + if prov.App != att.App || prov.Attempt != att.Name() { + return nil, fmt.Errorf("provenance identity mismatch: requested %s@%s, receipt describes %s@%s — refusing to use it", att.App, att.Name(), prov.App, prov.Attempt) + } + return &prov, nil +} diff --git a/internal/releasemeta/provenance_test.go b/internal/releasemeta/provenance_test.go new file mode 100644 index 0000000..6914572 --- /dev/null +++ b/internal/releasemeta/provenance_test.go @@ -0,0 +1,142 @@ +package releasemeta + +import ( + "context" + "fmt" + "strings" + "testing" + + "github.com/useteploy/teploy/internal/ssh" +) + +func provenanceFixture(app, release string) *Provenance { + return &Provenance{ + App: app, + Release: release, + Revision: "462d7a7b1c2d3e4f5a6b7c8d9e0f1a2b3c4d5e6f", + Dirty: true, + ContextPath: ".", + ContextFingerprint: strings.Repeat("f", 64), + Dockerfile: "Dockerfile", + DockerfileSHA256: strings.Repeat("d", 64), + Platform: "linux/amd64", + ImageRef: "myapp-build-v1", + ImageDigest: "sha256:" + strings.Repeat("a", 64), + DigestPinned: false, + ManifestSHA256: strings.Repeat("c", 64), + } +} + +// provenanceTestExecutor answers the commands WriteAttemptProvenance issues +// (mkdir, atomic upload) and lets framed reads resolve from recorded state. +func provenanceTestExecutor(t *testing.T, extra ...ssh.MockCommand) *ssh.MockExecutor { + t.Helper() + return ssh.NewMockExecutor("1.2.3.4", append(extra, + ssh.MockCommand{Match: "mkdir -p", Output: ""}, + ssh.MockCommand{Match: "UPLOAD:", Output: ""}, + ssh.MockCommand{Match: "mv -f --", Output: ""}, + )...) +} + +func TestWriteAttemptProvenance_PersistsIntoAttemptNamespace(t *testing.T) { + mock := provenanceTestExecutor(t) + att := MustAttempt("myapp", "v1") + + if err := WriteAttemptProvenance(context.Background(), mock, att, provenanceFixture("myapp", "v1")); err != nil { + t.Fatalf("WriteAttemptProvenance: %v", err) + } + + raw, ok := mock.Files[AttemptProvenancePath(att)] + if !ok { + t.Fatalf("provenance receipt not written to %s\nfiles: %v", AttemptProvenancePath(att), mock.Files) + } + if !strings.Contains(string(raw), `"revision":"462d7a7b1c2d3e4f5a6b7c8d9e0f1a2b3c4d5e6f"`) { + t.Errorf("revision not recorded: %s", raw) + } + if !strings.Contains(string(raw), `"dirty":true`) { + t.Errorf("dirty flag not recorded: %s", raw) + } + + got, err := ReadAttemptProvenance(context.Background(), mock, att) + if err != nil { + t.Fatalf("ReadAttemptProvenance: %v", err) + } + if got == nil { + t.Fatal("expected the provenance receipt back") + } + if got.Attempt != att.Name() { + t.Errorf("receipt does not carry the writing attempt: %q vs %q", got.Attempt, att.Name()) + } + if got.Revision != "462d7a7b1c2d3e4f5a6b7c8d9e0f1a2b3c4d5e6f" || !got.Dirty || + got.ContextFingerprint != strings.Repeat("f", 64) || + got.Dockerfile != "Dockerfile" || got.DockerfileSHA256 != strings.Repeat("d", 64) || + got.Platform != "linux/amd64" || got.ImageRef != "myapp-build-v1" || + got.ImageDigest != "sha256:"+strings.Repeat("a", 64) || + got.DigestPinned || got.ManifestSHA256 != strings.Repeat("c", 64) { + t.Errorf("provenance round-trip lost fields: %+v", got) + } +} + +func TestWriteAttemptProvenance_RejectsForeignIdentity(t *testing.T) { + mock := provenanceTestExecutor(t) + att := MustAttempt("myapp", "v1") + if err := WriteAttemptProvenance(context.Background(), mock, att, provenanceFixture("otherapp", "v1")); err == nil { + t.Fatal("a provenance record describing another app must be refused at write time") + } + if err := WriteAttemptProvenance(context.Background(), mock, att, provenanceFixture("myapp", "v2")); err == nil { + t.Fatal("a provenance record describing another release must be refused at write time") + } +} + +func TestReadAttemptProvenance_IdentityMismatchRefused(t *testing.T) { + att := MustAttempt("myapp", "v1") + foreign := fmt.Sprintf(`{"schema_version":%d,"app":"myapp","release":"v1","attempt":"v1.0000000000000000","revision":"x"}`, ProvenanceSchemaVersion) + mock := provenanceTestExecutor(t, + ssh.MockCommand{Match: "if [ ! -e '" + AttemptProvenancePath(att) + "'", Output: "present\n" + foreign}, + ) + if _, err := ReadAttemptProvenance(context.Background(), mock, att); err == nil { + t.Fatal("a receipt describing a different attempt must be refused, not accepted") + } +} + +func TestReadAttemptProvenance_AbsentIsNilNil(t *testing.T) { + att := MustAttempt("myapp", "v1") + mock := provenanceTestExecutor(t) + got, err := ReadAttemptProvenance(context.Background(), mock, att) + if err != nil || got != nil { + t.Fatalf("confirmed-missing provenance must be (nil, nil), got (%v, %v)", got, err) + } +} + +// C04 retry stability, artifact side: the F08 attempt namespace gives every +// attempt of the same release its own write-once provenance receipt — a +// response-loss retry lands beside the first attempt's receipt, never +// over it, and both name the same resolved source. +func TestAttemptProvenance_RetriesGetDistinctImmutableReceipts(t *testing.T) { + mock := provenanceTestExecutor(t) + first := MustAttempt("myapp", "v1") + second := MustAttempt("myapp", "v1") + + if err := WriteAttemptProvenance(context.Background(), mock, first, provenanceFixture("myapp", "v1")); err != nil { + t.Fatalf("first attempt write: %v", err) + } + if err := WriteAttemptProvenance(context.Background(), mock, second, provenanceFixture("myapp", "v1")); err != nil { + t.Fatalf("second attempt write: %v", err) + } + + if AttemptProvenancePath(first) == AttemptProvenancePath(second) { + t.Fatalf("two attempts of one release share a provenance path: %s", AttemptProvenancePath(first)) + } + if _, ok := mock.Files[AttemptProvenancePath(first)]; !ok { + t.Fatal("the first attempt's receipt did not survive the retry") + } + for _, att := range []Attempt{first, second} { + got, err := ReadAttemptProvenance(context.Background(), mock, att) + if err != nil || got == nil { + t.Fatalf("attempt %s receipt unreadable: (%v, %v)", att.Name(), got, err) + } + if got.Revision != "462d7a7b1c2d3e4f5a6b7c8d9e0f1a2b3c4d5e6f" { + t.Errorf("attempt %s re-resolved to a different source revision: %s", att.Name(), got.Revision) + } + } +} diff --git a/internal/releasemeta/releasemeta.go b/internal/releasemeta/releasemeta.go index 13f99fd..b31d584 100644 --- a/internal/releasemeta/releasemeta.go +++ b/internal/releasemeta/releasemeta.go @@ -126,6 +126,14 @@ type Record struct { ImageRef string `json:"image_ref,omitempty"` ImageDigest string `json:"image_digest,omitempty"` + // ManifestSHA256 is the effective-config digest (config. + // NormalizeAndDigest) the release deployed under — the plan/receipt + // equality surface (C04). + ManifestSHA256 string `json:"manifest_sha256,omitempty"` + // Provenance is the plan-time provenance the deploy resolved to + // (C04): revision, worktree cleanliness, build context fingerprint, + // Dockerfile identity, platform, image digest and mutability. + Provenance *Provenance `json:"provenance,omitempty"` Replicas int `json:"replicas,omitempty"` Processes map[string]string `json:"processes,omitempty"` From dda4911f1ee6fd5509df4564258bcfdccd5d2fb6 Mon Sep 17 00:00:00 2001 From: Tyler <53561637+im-tyler@users.noreply.github.com> Date: Wed, 23 Sep 2026 05:11:46 -0700 Subject: [PATCH 6/8] feat(cli,config): machine interface v1 + capability tokens + structured error envelope (X02-S1/S3-lite) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit version --json carries machine_interface:1 and a 15-token capability registry naming every landed contract; app list and server status --json carry the field additively (server list reshape recorded as the one non-additive S2 follow-up — dash decodes it as a bare map). Failures under --json emit the structured envelope on stderr (config-invalid + internal wired first; conflict/uncertain-outcome/degraded defined and queued) with exit codes pinned at 0/1/2. --- AUDIT_OPEN.md | 94 +++++++++++++++ internal/cli/deploy.go | 14 +-- internal/cli/drift.go | 14 ++- internal/cli/errevelope.go | 138 +++++++++++++++++++++ internal/cli/errevelope_test.go | 166 +++++++++++++++++++++++++ internal/cli/machine.go | 42 ++++--- internal/cli/machineinterface.go | 101 ++++++++++++++++ internal/cli/machineinterface_test.go | 167 ++++++++++++++++++++++++++ internal/cli/root.go | 4 +- internal/cli/version.go | 29 ++++- internal/config/app.go | 45 +++++-- 11 files changed, 771 insertions(+), 43 deletions(-) create mode 100644 internal/cli/errevelope.go create mode 100644 internal/cli/errevelope_test.go create mode 100644 internal/cli/machineinterface.go create mode 100644 internal/cli/machineinterface_test.go diff --git a/AUDIT_OPEN.md b/AUDIT_OPEN.md index 4826091..9f52d35 100644 --- a/AUDIT_OPEN.md +++ b/AUDIT_OPEN.md @@ -1698,3 +1698,97 @@ as provenance facts, not yet a selectable policy); build records for `teploy build` outside deploys; cache diagnostics; secret-safe build-input attestation. Changed-mutable-tag POLICY (beyond recording pinned-vs- mutable + the mismatch warning) lands with the offline/pull-policy slice. + +## Programme slice (2026-09-23) — X02 S1: versioned machine interface + S3-lite error envelope + +First X02 slice (ADR +`../_internal/X02_RESOURCE_CONTRACT_ADR_2026-09-22.md` §2.1-2.3, adopted +by `../_internal/DELEGATED_DECISIONS_2026-09-23.md` decisions 1/4/7/8/9 — +D8 single-integer MI, D9 capability advertisement, D10 exit codes +unchanged). Base revision `6faefc4`; changes left uncommitted for review. + +**Landed:** + +- **Machine-interface version 1** (`internal/cli/machineinterface.go`): + `MachineInterface = 1` at the root of `version --json` (new + `{"version","machine_interface","capabilities"}` envelope), `app list + --json` (appListDTO), and `server status --json` (serverStatusDTO) — + additive fields; dash's Go decoders ignore unknown fields, verified + against dash's actual decode sites. Versioning rules on the constant's + doc comment: additive changes never bump; removal/rename/type or + semantic change bumps. **Verified exclusion:** `server list --json` + emits a bare map-of-servers root (dash decodes + `map[string]{host,user}` at server.go:1641) — there is no envelope + object to carry the field additively, and injecting a + `machine_interface` KEY would materialize as a phantom server in + dash's fleet; reshaping it is a non-additive change recorded as S2 + follow-up (needs a coordinated dash decode change). +- **Capability registry** (15 stable tokens, the doc-comment block in + machineinterface.go IS the registry): `env-set-stdin`, `kv-set-stdin`, + `template-var-stdin` (cb7c0fc), `server-rename`, `server-update` + (72c57f9), `autodeploy-redeploy`, `health-modes` (C03), + `provenance-records` (C04), `readiness-receipts` (C01-4), + `preview-canonical-id`, `preview-blue-green` (C06), `repair-debt` + (C01-6), `error-envelope` (this slice), `app-list-machine`, + `server-status-machine`. Every token names a LANDED contract; + `server-list-ids` deliberately absent (S4 not landed). + TestCapabilityTokenRegistry pins the exact sorted set — rename or + removal fails it. +- **S3-lite structured error envelope** (`internal/cli/errevelope.go`): + on any command failure under `--json`, one document + `{"machine_interface","code","message","detail"}` on STDERR (stdout + stays the data channel); without `--json` the historical plain-text + stderr line is byte-identical. Code taxonomy v1 defined as the closed + registry: config-invalid, target-unreachable, unsupported, conflict, + uncertain-outcome, degraded, internal (unknown → internal, + forward-safe). **Wired classes:** config-load failures + (`config.ErrInvalidConfig` sentinel — a no-text-change wrapper so + errors.Is classifies while every message stays verbatim — wrapped at + all LoadApp/LoadAppWithDestination/Compose-propagation returns) and + deploy admission refusals (the dash-hit ad-hoc path's pre-effect + validations, `--version` grammar, no-server, tag-filter parse — + `errDeployAdmission` marker, same no-text-change discipline); both + classify config-invalid. `Execute` reports through + `reportExecutionError` before the unchanged `os.Exit(1)`; drift's exit + 2 extracted into `driftExitCode` so the 0/1/2 semantics are pinned in + code (2 only with `--exit-code` AND drift found). + +**Evidence** — TDD red level 1 recorded (all new test symbols undefined +at compile: MachineInterface, writeVersion, reportExecutionError, +ErrInvalidConfig, errDeployAdmission, driftExitCode), green after +implementation. New coverage: version JSON exact shape (3 keys, MI 1, +capabilities verbatim) + human output unchanged + cobra end-to-end; +registry completeness (golden list + per-constant membership + +uniqueness/sortedness); app list + server status envelopes carry +machine_interface; config-load envelope through a REAL failing `deploy +--json` (code, stable message, detail naming teploy.yml); admission +envelope through a real invalid `--app` ad-hoc deploy; envelope ABSENT +without --json (plain error text, no leak); unclassified → internal; exit +semantics pinned. Binary smoke: exact envelope JSON on stderr, exit 1, +human path unchanged. Mutation checks (in-place, all reverted): +suppressing the envelope under --json (`if false && jsonMode`) fails both +wired-class tests for the intended reason; renaming a token value fails +the registry test; removing a token from the advertised list fails it +(count + membership). Gates after revert: `go vet ./...` clean; +`go test ./... -race -count=1` all packages ok; gofmt clean on every +touched hunk (deploy.go's pre-existing fleet-rollback stray left alone, +consistent with the C02-C04 posture); contract probes 5/5 PASS. No push +performed. + +**S2 + error-site migration list (recorded follow-ups):** + +- `server list --json` reshape to an envelope root (coordinated dash + decode change — the one non-additive MI bump candidate). +- target-unreachable: the ssh.Connect failure returns across commands + (app list/server status "connecting to", deploy step 6). +- conflict: `preview.AmbiguousPreviewError` (typed and ready — one + errors.As), `config.ErrServerExists`/`ErrServerNotFound` (dash + currently matches message text; the envelope gives it a stable code). +- uncertain-outcome / degraded: the C01 journal outcomes (recovery + dispositions), the T57 "backends deployed but load-balancer activation + failed" class, LogEntry Degraded rendering. +- unsupported: version-skew refusals (e.g. autodeploy schedule's + server-binary-lacks-redeploy error). +- Generalizing per-command envelopes for the remaining --json verbs + (health/log/drift/stats/plan/validate/registry/template/accessory + lists) is S2's `contracts/` skeleton work, not error-site migration. diff --git a/internal/cli/deploy.go b/internal/cli/deploy.go index b8e30ce..f2474b9 100644 --- a/internal/cli/deploy.go +++ b/internal/cli/deploy.go @@ -72,7 +72,7 @@ swapping traffic.`, } tags, err := parseTagFilters(tagFilters) if err != nil { - return err + return refuseAdmission(err) } return runDeploy(flags, serverName, image, version, skipDNSCheck, parallel, destination, migrateVolumes, role, tags) }, @@ -97,23 +97,23 @@ swapping traffic.`, // and scripting. Requires --app and --image at minimum. func runAdHocDeploy(flags *Flags, serverName, appName, image, domain string, port int, version string, skipDNSCheck, migrateVolumes bool) error { if image == "" { - return fmt.Errorf("--image is required for ad-hoc deploy (no teploy.yml)") + return refuseAdmission(fmt.Errorf("--image is required for ad-hoc deploy (no teploy.yml)")) } // This path builds an AppConfig directly instead of going through // config.LoadApp, so it never reaches AppConfig.validate() — app and // domain must be validated explicitly here before either one reaches // the network (state paths, remote shell commands, Caddyfile content). if err := config.ValidateName(appName); err != nil { - return err + return refuseAdmission(err) } if err := config.ValidateDomain(domain, false); err != nil { - return err + return refuseAdmission(err) } if serverName == "" { serverName = flags.Host } if serverName == "" { - return fmt.Errorf("server is required — use 'teploy deploy --app ...' or --host") + return refuseAdmission(fmt.Errorf("server is required — use 'teploy deploy --app ...' or --host")) } if port <= 0 { port = 80 @@ -301,7 +301,7 @@ func deployAppConfig(flags *Flags, appCfg *config.AppConfig, serverName, image, } } if serverName == "" { - return fmt.Errorf("no server specified — use 'teploy deploy ' or set 'server' in teploy.yml") + return refuseAdmission(fmt.Errorf("no server specified — use 'teploy deploy ' or set 'server' in teploy.yml")) } host, user, key, err := config.ResolveServer(serverName, flags.Host, flags.User, flags.Key) @@ -347,7 +347,7 @@ func deployAppConfig(flags *Flags, appCfg *config.AppConfig, serverName, image, } } } else if err := validateVersionArg(version); err != nil { - return err + return refuseAdmission(err) } // 5. Detect build mode (when no pre-built image). Honors the optional diff --git a/internal/cli/drift.go b/internal/cli/drift.go index c5d0c48..a0ea233 100644 --- a/internal/cli/drift.go +++ b/internal/cli/drift.go @@ -152,12 +152,22 @@ func runDrift(flags *Flags, appName string, exitCode bool) error { } // --exit-code makes drift observable to CI/monitoring without treating it // as a command failure. Executor is already closed above so os.Exit is safe. - if exitCode && driftFound { - os.Exit(2) + if code := driftExitCode(exitCode, driftFound); code != 0 { + os.Exit(code) } return nil } +// driftExitCode preserves the documented exit semantics (X02 D10): 2 only +// when --exit-code is set AND drift was found — a CI signal, never a +// failure. Everything else stays 0. +func driftExitCode(exitCode, driftFound bool) int { + if exitCode && driftFound { + return 2 + } + return 0 +} + func reportDrift(ctx context.Context, flags *Flags, appCfg *config.AppConfig, executor ssh.Executor, fromState bool) (bool, error) { if appCfg.IsStatic() && !fromState { if flags.JSON { diff --git a/internal/cli/errevelope.go b/internal/cli/errevelope.go new file mode 100644 index 0000000..d3005c5 --- /dev/null +++ b/internal/cli/errevelope.go @@ -0,0 +1,138 @@ +package cli + +import ( + "encoding/json" + "errors" + "fmt" + "io" + + "github.com/spf13/cobra" + "github.com/useteploy/teploy/internal/config" +) + +// Structured error envelope for machine mode (X02 §2.3, S3-lite wiring). +// +// When a command fails under --json output mode, the error is reported as +// one JSON document on STDERR — the stream a machine reader separates +// from the data channel — while stdout keeps carrying only successful +// output. Exit codes stay 0/1/2 exactly (D10): the envelope is the +// detail channel, not a new exit signal. Without --json the historical +// plain-text stderr line is unchanged. + +// Machine error codes — closed taxonomy v1 (X02 §2.3). Consumers must +// treat an unknown code as internal (forward-safe). Codes marked +// UNMIGRATED are part of the registry but no error site classifies into +// them yet; their migration is the recorded S2 follow-up list in +// AUDIT_OPEN.md. +const ( + // teploy.yml/TOML/destination/Compose failed to load, merge, or + // validate — or a deploy request's parameters were refused before + // any effect (admission). + codeConfigInvalid = "config-invalid" + // SSH/transport failure reaching the target (UNMIGRATED: currently + // internal). + codeTargetUnreachable = "target-unreachable" + // The requested verb needs a capability this binary lacks; names the + // remedy (UNMIGRATED: currently internal). + codeUnsupported = "unsupported" + // Idempotency/ambiguity refusal — e.g. the ambiguous-preview class + // (UNMIGRATED: currently internal). + codeConflict = "conflict" + // The effect's fate is unknown pending reconciliation; never + // rendered or recorded as failure (UNMIGRATED: currently internal). + codeUncertainOutcome = "uncertain-outcome" + // Success with a flag — traffic switched but the outcome is not + // clean (UNMIGRATED: currently internal). + codeDegraded = "degraded" + // Everything else, and every failure whose site has not been + // migrated to a specific code yet. + codeInternal = "internal" +) + +// machineErrorEnvelope is the wire shape: the interface version (so a +// consumer gates decoding the same way as data envelopes), the taxonomy +// code, a stable short message, and the full error text as detail. +type machineErrorEnvelope struct { + MachineInterface int `json:"machine_interface"` + Code string `json:"code"` + Message string `json:"message"` + Detail string `json:"detail,omitempty"` +} + +// errDeployAdmission marks a deploy refusal issued before any effect: +// invalid request parameters (--app/--image/--domain/--version grammar, +// missing server), or a config the engine refused to load. The wrapped +// error's text and chain are preserved verbatim. +var errDeployAdmission = errors.New("deploy admission refused") + +type admissionError struct{ err error } + +func (e *admissionError) Error() string { return e.err.Error() } +func (e *admissionError) Unwrap() error { return e.err } +func (e *admissionError) Is(target error) bool { + return target == errDeployAdmission +} + +// refuseAdmission marks err as a pre-effect deploy refusal without +// altering its message. +func refuseAdmission(err error) error { + if err == nil { + return nil + } + return &admissionError{err: err} +} + +// classifyMachineError maps a failed command's error to a taxonomy code. +// Wired classes (this slice): config-load failures and deploy admission +// refusals → config-invalid; an absent config is a config failure for a +// machine caller the same way a malformed one is. Everything else is +// internal until its site is migrated (S2 generalization). +func classifyMachineError(err error) string { + switch { + case errors.Is(err, config.ErrInvalidConfig), + errors.Is(err, config.ErrNoConfig), + errors.Is(err, errDeployAdmission): + return codeConfigInvalid + default: + return codeInternal + } +} + +// writeMachineErrorEnvelope renders err as the machine error envelope. +func writeMachineErrorEnvelope(out io.Writer, err error) error { + code := classifyMachineError(err) + message := "command failed" + switch code { + case codeConfigInvalid: + message = "invalid teploy configuration" + if errors.Is(err, errDeployAdmission) { + message = "deploy request refused" + } + } + return json.NewEncoder(out).Encode(machineErrorEnvelope{ + MachineInterface: MachineInterface, + Code: code, + Message: message, + Detail: err.Error(), + }) +} + +// reportExecutionError renders a failed root invocation's error the way +// Execute does before exiting 1: the machine envelope under --json, the +// plain error text otherwise. A flag-parse failure never reaches parsed +// --json state and keeps the plain form. +func reportExecutionError(root *cobra.Command, err error, out io.Writer) { + if jsonMode(root) { + _ = writeMachineErrorEnvelope(out, err) + return + } + fmt.Fprintln(out, err) +} + +// jsonMode reports whether the root invocation parsed --json. Read from +// the flag value (not the bound struct) so Execute can consult it after +// the fact. +func jsonMode(root *cobra.Command) bool { + value, err := root.PersistentFlags().GetBool("json") + return err == nil && value +} diff --git a/internal/cli/errevelope_test.go b/internal/cli/errevelope_test.go new file mode 100644 index 0000000..079fde1 --- /dev/null +++ b/internal/cli/errevelope_test.go @@ -0,0 +1,166 @@ +package cli + +import ( + "bytes" + "encoding/json" + "errors" + "strings" + "testing" + + "github.com/useteploy/teploy/internal/config" +) + +// driveRoot executes a real cobra invocation and returns its error, so +// envelope tests observe the exact error Execute would report. +func driveRoot(t *testing.T, args ...string) error { + t.Helper() + root := NewRootCmd("test") + var out bytes.Buffer + root.SetOut(&out) + root.SetErr(&out) + root.SetArgs(args) + return root.Execute() +} + +const invalidModeConfig = "app: demo\ndomain: demo.example.com\nserver: prod\nhealth:\n mode: bogus\n" + +// TestMachineErrorEnvelopeConfigLoad drives a real failing deploy (an +// invalid teploy.yml) and asserts the machine error envelope: emitted on +// the error stream under --json, carrying the interface version and the +// config-invalid code. +func TestMachineErrorEnvelopeConfigLoad(t *testing.T) { + chdirTemp(t, invalidModeConfig) + + err := driveRoot(t, "deploy", "--json") + if err == nil { + t.Fatal("invalid teploy.yml must fail the deploy") + } + if !errors.Is(err, config.ErrInvalidConfig) { + t.Fatalf("config-load error is not classified: %v", err) + } + + root := NewRootCmd("test") + var out bytes.Buffer + root.SetOut(&out) + root.SetErr(&out) + root.SetArgs([]string{"deploy", "--json"}) + if err := root.Execute(); err != nil { + reportExecutionError(root, err, &out) + } + + var decoded map[string]any + if jsonErr := json.Unmarshal(out.Bytes(), &decoded); jsonErr != nil { + t.Fatalf("no machine error envelope under --json (got %q): %v", out.String(), jsonErr) + } + if decoded["machine_interface"] != float64(MachineInterface) { + t.Fatalf("envelope machine_interface = %v, want %d", decoded["machine_interface"], MachineInterface) + } + if decoded["code"] != "config-invalid" { + t.Fatalf("envelope code = %v, want config-invalid", decoded["code"]) + } + if decoded["message"] != "invalid teploy configuration" { + t.Fatalf("envelope message = %v", decoded["message"]) + } + detail, _ := decoded["detail"].(string) + if detail == "" || !strings.Contains(detail, "teploy.yml") { + t.Fatalf("envelope detail must name the config: %q", detail) + } +} + +// TestMachineErrorEnvelopeAdmission drives the ad-hoc deploy path (the +// dash-hit admission surface) with an invalid app name and asserts the +// refusal classifies as config-invalid with the admission message. +func TestMachineErrorEnvelopeAdmission(t *testing.T) { + err := driveRoot(t, "deploy", "prod", "--app", "bad app", "--image", "nginx:1", "--json") + if err == nil { + t.Fatal("invalid ad-hoc app name must be refused") + } + if !errors.Is(err, errDeployAdmission) { + t.Fatalf("admission refusal is not marked: %v", err) + } + + root := NewRootCmd("test") + var out bytes.Buffer + root.SetArgs([]string{"deploy", "prod", "--app", "bad app", "--image", "nginx:1", "--json"}) + if err := root.Execute(); err != nil { + reportExecutionError(root, err, &out) + } + + var decoded map[string]any + if jsonErr := json.Unmarshal(out.Bytes(), &decoded); jsonErr != nil { + t.Fatalf("no machine error envelope under --json (got %q): %v", out.String(), jsonErr) + } + if decoded["code"] != "config-invalid" { + t.Fatalf("envelope code = %v, want config-invalid", decoded["code"]) + } + if decoded["message"] != "deploy request refused" { + t.Fatalf("envelope message = %v, want \"deploy request refused\"", decoded["message"]) + } + detail, _ := decoded["detail"].(string) + if detail == "" || !strings.Contains(detail, "app") { + t.Fatalf("envelope detail must carry the refusal reason: %q", detail) + } +} + +// TestMachineErrorEnvelopeAbsentWithoutJSON: the same failure without +// --json keeps the historical plain-text stderr line — no envelope, no +// JSON, byte-identical message. +func TestMachineErrorEnvelopeAbsentWithoutJSON(t *testing.T) { + chdirTemp(t, invalidModeConfig) + + err := driveRoot(t, "deploy") + if err == nil { + t.Fatal("invalid teploy.yml must fail the deploy") + } + + root := NewRootCmd("test") + var out bytes.Buffer + root.SetOut(&out) + root.SetErr(&out) + root.SetArgs([]string{"deploy"}) + execErr := root.Execute() + if execErr == nil { + t.Fatal("invalid teploy.yml must fail the deploy") + } + reportExecutionError(root, execErr, &out) + if out.String() != execErr.Error()+"\n" { + t.Fatalf("human error output = %q, want the plain error %q", out.String(), execErr.Error()) + } + if strings.Contains(out.String(), "machine_interface") { + t.Fatalf("envelope leaked into non-JSON output: %q", out.String()) + } +} + +// TestMachineErrorEnvelopeUnclassifiedIsInternal: errors outside the +// wired classes classify as internal — the forward-safe default. +func TestMachineErrorEnvelopeUnclassifiedIsInternal(t *testing.T) { + var out bytes.Buffer + if err := writeMachineErrorEnvelope(&out, errors.New("dial tcp 192.0.2.10:22: i/o timeout")); err != nil { + t.Fatal(err) + } + var decoded map[string]any + if err := json.Unmarshal(out.Bytes(), &decoded); err != nil { + t.Fatalf("envelope not JSON: %q", out.String()) + } + if decoded["code"] != "internal" { + t.Fatalf("unclassified code = %v, want internal", decoded["code"]) + } + if decoded["message"] != "command failed" { + t.Fatalf("unclassified message = %v", decoded["message"]) + } +} + +// TestExitCodesPinned: 0/1/2 semantics are unchanged by the envelope — +// drift's 2 remains a signal (not a failure) gated on --exit-code, and +// every other failure exits 1 after the error is reported. +func TestExitCodesPinned(t *testing.T) { + if got := driftExitCode(true, true); got != 2 { + t.Fatalf("driftExitCode(true,true) = %d, want 2", got) + } + if got := driftExitCode(true, false); got != 0 { + t.Fatalf("driftExitCode(true,false) = %d, want 0", got) + } + if got := driftExitCode(false, true); got != 0 { + t.Fatalf("driftExitCode(false,true) = %d, want 0", got) + } +} diff --git a/internal/cli/machine.go b/internal/cli/machine.go index 131f44f..8ac1001 100644 --- a/internal/cli/machine.go +++ b/internal/cli/machine.go @@ -70,10 +70,11 @@ type appStatusDTO struct { } type appListDTO struct { - Host string `json:"host"` - Apps []appStatusDTO `json:"apps"` - ObservedAt time.Time `json:"observed_at"` - Errors []machineError `json:"errors"` + MachineInterface int `json:"machine_interface"` + Host string `json:"host"` + Apps []appStatusDTO `json:"apps"` + ObservedAt time.Time `json:"observed_at"` + Errors []machineError `json:"errors"` } func newAppListCmd(flags *Flags) *cobra.Command { @@ -124,10 +125,11 @@ func runAppList(flags *Flags, out io.Writer) error { func collectAppList(ctx context.Context, executor ssh.Executor, observedAt time.Time) appListDTO { result := appListDTO{ - Host: executor.Host(), - Apps: []appStatusDTO{}, - ObservedAt: observedAt, - Errors: []machineError{}, + MachineInterface: MachineInterface, + Host: executor.Host(), + Apps: []appStatusDTO{}, + ObservedAt: observedAt, + Errors: []machineError{}, } out, err := executor.Run(ctx, `for f in /deployments/*/state.json /deployments/*/state; do [ -f "$f" ] && basename "$(dirname "$f")"; done | sort -u`) if err != nil { @@ -303,16 +305,17 @@ type caddyObservationDTO struct { } type serverStatusDTO struct { - Server string `json:"server"` - Host string `json:"host"` - Uptime uptimeDTO `json:"uptime"` - Load loadDTO `json:"load"` - Memory memoryDTO `json:"memory"` - Disks []diskDTO `json:"disks"` - Docker dockerInventoryDTO `json:"docker"` - Caddy caddyObservationDTO `json:"caddy"` - ObservedAt time.Time `json:"observed_at"` - Errors []machineError `json:"errors"` + MachineInterface int `json:"machine_interface"` + Server string `json:"server"` + Host string `json:"host"` + Uptime uptimeDTO `json:"uptime"` + Load loadDTO `json:"load"` + Memory memoryDTO `json:"memory"` + Disks []diskDTO `json:"disks"` + Docker dockerInventoryDTO `json:"docker"` + Caddy caddyObservationDTO `json:"caddy"` + ObservedAt time.Time `json:"observed_at"` + Errors []machineError `json:"errors"` } func newServerStatusCmd(flags *Flags) *cobra.Command { @@ -356,7 +359,8 @@ func runServerStatus(flags *Flags, target string, out io.Writer) error { func collectServerStatus(ctx context.Context, executor ssh.Executor, server string, observedAt time.Time) serverStatusDTO { result := serverStatusDTO{ - Server: server, Host: executor.Host(), ObservedAt: observedAt, + MachineInterface: MachineInterface, + Server: server, Host: executor.Host(), ObservedAt: observedAt, Disks: []diskDTO{}, Errors: []machineError{}, Docker: dockerInventoryDTO{Containers: []containerDTO{}, Images: []imageDTO{}}, Caddy: caddyObservationDTO{Routes: []caddyRouteDTO{}}, diff --git a/internal/cli/machineinterface.go b/internal/cli/machineinterface.go new file mode 100644 index 0000000..6602ea1 --- /dev/null +++ b/internal/cli/machineinterface.go @@ -0,0 +1,101 @@ +package cli + +import "sort" + +// Machine-interface contract, version 1 (X02 S1 — versioned resource and +// operation contracts, _internal/X02_RESOURCE_CONTRACT_ADR_2026-09-22.md +// §2.1-2.2, adopted by DELEGATED_DECISIONS_2026-09-23 D8/D9). +// +// MachineInterface is the version of teploy's machine-readable output +// contract. It rides at the root of every --json envelope a machine +// consumer parses — `version --json`, `app list --json`, and +// `server status --json` — so a consumer (teploy-dash) can fail closed on +// an interface newer than the one it supports BEFORE submitting any +// mutation, instead of discovering the skew after the fact. +// +// Versioning rules (D8): +// - additive changes (new fields, new capability tokens) do NOT bump; +// - non-additive changes (field removal, rename, type or semantic +// change, capability-token removal or redefinition) bump it. +// +// MI 1 is assigned to the envelope shapes as implemented at v0.1.37 plus +// this field itself — assigning it is the compatibility commitment X01 +// lacked. Known exclusion: `server list --json` emits a bare +// map-of-servers root with no envelope object, so it cannot carry the +// field additively; reshaping it is recorded as S2 follow-up work +// (requires a coordinated dash decode change). +const MachineInterface = 1 + +// Capability tokens advertised by `teploy version --json` (X02 §2.2). +// THIS BLOCK IS THE REGISTRY — the single source of the token set. +// Tokens are stable identifiers consumers code against: ADDING one is +// additive; REMOVING one or changing its meaning bumps +// MachineInterface. Every token must name a contract that has LANDED in +// this binary (TestCapabilityTokenRegistry pins the exact set). +const ( + // env set KEY --stdin reads the value verbatim from stdin; secrets + // never travel in argv (cb7c0fc, dash UPSTREAM-1). + CapEnvSetStdin = "env-set-stdin" + // kv set KEY --stdin, the same stdin contract for KV values. + CapKvSetStdin = "kv-set-stdin" + // template deploy/install --var-stdin reads a JSON object that + // overrides --var (cb7c0fc). + CapTemplateVarStdin = "template-var-stdin" + // server rename moves the whole record in one atomic commit, + // preserving every field (72c57f9, dash UPSTREAM-2). + CapServerRename = "server-rename" + // server update changes only the flags passed, atomically (72c57f9). + CapServerUpdate = "server-update" + // autodeploy redeploy runs scheduled redeploys through the real + // deploy engine (fenced lock, fetch, health gate) — no side engine. + CapAutodeployRedeploy = "autodeploy-redeploy" + // health.mode http | tcp | auto — explicit readiness probe modes, + // validated at load, surfaced before the gate (C03). + CapHealthModes = "health-modes" + // attempt provenance.json: revision, context fingerprint, and the + // immutable image digest resolved BEFORE execution (C04). + CapProvenanceRecords = "provenance-records" + // attempt readiness.json: durable readiness evidence written exactly + // when the health gate passes (C01-4). + CapReadinessReceipts = "readiness-receipts" + // previews are keyed by -p- of sha256(app NUL branch) — + // branch-distinct identity with legacy adoption/ambiguity contract + // (C06). + CapPreviewCanonicalID = "preview-canonical-id" + // repair-debt.json: durable record-write failure debt, repaired by + // the next deploy before its own work (C01-6). + CapRepairDebt = "repair-debt" + // preview updates are blue/green: candidate readiness-gated before + // the route switch; predecessor retired only after (C06). + CapPreviewBlueGreen = "preview-blue-green" + // structured error envelope on stderr under --json (X02 §2.3). + CapErrorEnvelope = "error-envelope" + // `app list --json` emits the MI-1 machine envelope. + CapAppListMachine = "app-list-machine" + // `server status --json` emits the MI-1 machine envelope. + CapServerStatusMachine = "server-status-machine" +) + +// MachineCapabilities returns every capability token this build +// advertises, sorted; `version --json` emits it verbatim. +func MachineCapabilities() []string { + tokens := []string{ + CapEnvSetStdin, + CapKvSetStdin, + CapTemplateVarStdin, + CapServerRename, + CapServerUpdate, + CapAutodeployRedeploy, + CapHealthModes, + CapProvenanceRecords, + CapReadinessReceipts, + CapPreviewCanonicalID, + CapRepairDebt, + CapPreviewBlueGreen, + CapErrorEnvelope, + CapAppListMachine, + CapServerStatusMachine, + } + sort.Strings(tokens) + return tokens +} diff --git a/internal/cli/machineinterface_test.go b/internal/cli/machineinterface_test.go new file mode 100644 index 0000000..c49dfdd --- /dev/null +++ b/internal/cli/machineinterface_test.go @@ -0,0 +1,167 @@ +package cli + +import ( + "bytes" + "context" + "encoding/json" + "testing" + "time" + + "github.com/useteploy/teploy/internal/ssh" +) + +func TestWriteVersionJSONShape(t *testing.T) { + var out bytes.Buffer + if err := writeVersion(&out, "v0.1.37-test", true); err != nil { + t.Fatal(err) + } + var decoded map[string]any + if err := json.Unmarshal(out.Bytes(), &decoded); err != nil { + t.Fatalf("version --json is not valid JSON: %q: %v", out.String(), err) + } + if decoded["version"] != "v0.1.37-test" { + t.Fatalf("version = %v, want v0.1.37-test", decoded["version"]) + } + if decoded["machine_interface"] != float64(MachineInterface) { + t.Fatalf("machine_interface = %v, want %d", decoded["machine_interface"], MachineInterface) + } + caps, ok := decoded["capabilities"].([]any) + if !ok { + t.Fatalf("capabilities missing or not a list: %s", out.String()) + } + if len(caps) != len(MachineCapabilities()) { + t.Fatalf("capabilities count = %d, want %d", len(caps), len(MachineCapabilities())) + } + for i, token := range MachineCapabilities() { + if caps[i] != token { + t.Fatalf("capabilities[%d] = %v, want %q", i, caps[i], token) + } + } + // Exact key set — additive fields are allowed only with the MI rules + // documented on MachineInterface; a removal or rename is a bump. + for _, key := range []string{"version", "machine_interface", "capabilities"} { + if _, ok := decoded[key]; !ok { + t.Fatalf("version envelope missing %q: %s", key, out.String()) + } + } + if len(decoded) != 3 { + t.Fatalf("version envelope has unexpected keys: %s", out.String()) + } +} + +func TestWriteVersionHumanUnchanged(t *testing.T) { + var out bytes.Buffer + if err := writeVersion(&out, "v0.1.37-test", false); err != nil { + t.Fatal(err) + } + if out.String() != "teploy v0.1.37-test\n" { + t.Fatalf("human version output = %q", out.String()) + } +} + +func TestVersionCommandJSONEndToEnd(t *testing.T) { + var out bytes.Buffer + root := NewRootCmd("vtest") + root.SetOut(&out) + root.SetArgs([]string{"version", "--json"}) + if err := root.Execute(); err != nil { + t.Fatalf("version --json: %v", err) + } + var decoded map[string]any + if err := json.Unmarshal(out.Bytes(), &decoded); err != nil { + t.Fatalf("version --json output not JSON: %q: %v", out.String(), err) + } + if decoded["machine_interface"] != float64(1) { + t.Fatalf("machine_interface = %v, want 1", decoded["machine_interface"]) + } +} + +// TestCapabilityTokenRegistry pins the exact token set: renaming or +// removing any advertised capability fails here, and a new token cannot +// land without being added to the golden list (adding is additive per the +// MI rules; the golden list forces the addition to be deliberate). +func TestCapabilityTokenRegistry(t *testing.T) { + want := []string{ + "app-list-machine", + "autodeploy-redeploy", + "env-set-stdin", + "error-envelope", + "health-modes", + "kv-set-stdin", + "preview-blue-green", + "preview-canonical-id", + "provenance-records", + "readiness-receipts", + "repair-debt", + "server-rename", + "server-status-machine", + "server-update", + "template-var-stdin", + } + got := MachineCapabilities() + if len(got) != len(want) { + t.Fatalf("capability registry drifted: got %v, want %v", got, want) + } + for i := range want { + if got[i] != want[i] { + t.Fatalf("capability registry drifted: got %v, want %v", got, want) + } + } + // Every named constant stays a member of the advertised set — deleting + // a constant breaks compilation, redirecting one to a foreign string + // breaks this membership check. + member := map[string]bool{} + for _, token := range got { + if member[token] { + t.Fatalf("duplicate capability token: %q", token) + } + member[token] = true + } + for _, token := range []string{ + CapEnvSetStdin, CapKvSetStdin, CapTemplateVarStdin, + CapServerRename, CapServerUpdate, CapAutodeployRedeploy, + CapHealthModes, CapProvenanceRecords, CapReadinessReceipts, + CapPreviewCanonicalID, CapRepairDebt, CapPreviewBlueGreen, + CapErrorEnvelope, CapAppListMachine, CapServerStatusMachine, + } { + if !member[token] { + t.Fatalf("capability constant %q is not advertised", token) + } + } +} + +func TestMachineEnvelopesCarryInterfaceVersion(t *testing.T) { + t.Run("app list", func(t *testing.T) { + empty := ssh.NewMockExecutor("empty", ssh.MockCommand{Match: "for f in /deployments/*/state.json", Output: ""}) + got := collectAppList(context.Background(), empty, time.Now()) + raw, err := json.Marshal(got) + if err != nil { + t.Fatal(err) + } + var decoded map[string]any + if err := json.Unmarshal(raw, &decoded); err != nil { + t.Fatal(err) + } + if decoded["machine_interface"] != float64(MachineInterface) { + t.Fatalf("app list machine_interface = %v, want %d", decoded["machine_interface"], MachineInterface) + } + }) + + t.Run("server status", func(t *testing.T) { + empty := ssh.NewMockExecutor("empty", + ssh.MockCommand{Match: "cat /proc/uptime", Err: nil}, + ) + got := collectServerStatus(context.Background(), empty, "prod", time.Now()) + raw, err := json.Marshal(got) + if err != nil { + t.Fatal(err) + } + var decoded map[string]any + if err := json.Unmarshal(raw, &decoded); err != nil { + t.Fatal(err) + } + if decoded["machine_interface"] != float64(MachineInterface) { + t.Fatalf("server status machine_interface = %v, want %d", decoded["machine_interface"], MachineInterface) + } + }) +} diff --git a/internal/cli/root.go b/internal/cli/root.go index dc740f5..1c61d4e 100644 --- a/internal/cli/root.go +++ b/internal/cli/root.go @@ -90,7 +90,7 @@ func NewRootCmd(version string) *cobra.Command { root.AddCommand(newAutoDeployCmd(flags)) root.AddCommand(newMaintenanceCmd(flags)) root.AddCommand(newUpdateCmd(version)) - root.AddCommand(newVersionCmd(version)) + root.AddCommand(newVersionCmd(flags, version)) return root } @@ -98,7 +98,7 @@ func NewRootCmd(version string) *cobra.Command { func Execute(version string) { root := NewRootCmd(version) if err := root.Execute(); err != nil { - fmt.Fprintln(os.Stderr, err) + reportExecutionError(root, err, os.Stderr) os.Exit(1) } } diff --git a/internal/cli/version.go b/internal/cli/version.go index 95df3dd..c97b39b 100644 --- a/internal/cli/version.go +++ b/internal/cli/version.go @@ -1,17 +1,40 @@ package cli import ( + "encoding/json" "fmt" + "io" "github.com/spf13/cobra" ) -func newVersionCmd(version string) *cobra.Command { +// versionDTO is the `teploy version --json` envelope (X02 §2.1): the +// machine-interface version plus the capability registry, so one call +// replaces help-text scraping as the compatibility handshake. +type versionDTO struct { + Version string `json:"version"` + MachineInterface int `json:"machine_interface"` + Capabilities []string `json:"capabilities"` +} + +func newVersionCmd(flags *Flags, version string) *cobra.Command { return &cobra.Command{ Use: "version", Short: "Show teploy version", - Run: func(cmd *cobra.Command, args []string) { - fmt.Printf("teploy %s\n", version) + RunE: func(cmd *cobra.Command, args []string) error { + return writeVersion(cmd.OutOrStdout(), version, flags.JSON) }, } } + +func writeVersion(out io.Writer, version string, jsonOutput bool) error { + if !jsonOutput { + fmt.Fprintf(out, "teploy %s\n", version) + return nil + } + return json.NewEncoder(out).Encode(versionDTO{ + Version: version, + MachineInterface: MachineInterface, + Capabilities: MachineCapabilities(), + }) +} diff --git a/internal/config/app.go b/internal/config/app.go index 8f8a80c..242ed73 100644 --- a/internal/config/app.go +++ b/internal/config/app.go @@ -1162,6 +1162,31 @@ func unmarshalAppYAML(data []byte, out *AppConfig) error { // Callers match it with errors.Is to offer interactive first-run setup. var ErrNoConfig = errors.New("no teploy.yml, teploy.toml, or docker-compose file found") +// ErrInvalidConfig is the machine-facing sentinel for every failure to +// load, merge, or validate a teploy.yml/TOML/destination/Compose +// configuration (X02 §2.3's config-invalid error class). Failures wrap it +// WITHOUT altering their message text or unwrap chain: +// errors.Is(err, ErrInvalidConfig) is the classification contract used by +// the CLI's machine error envelope. +var ErrInvalidConfig = errors.New("invalid config") + +type invalidConfigError struct{ err error } + +func (e *invalidConfigError) Error() string { return e.err.Error() } +func (e *invalidConfigError) Unwrap() error { return e.err } +func (e *invalidConfigError) Is(target error) bool { + return target == ErrInvalidConfig +} + +// invalidConfig marks err as a config failure, preserving its text and +// chain verbatim. +func invalidConfig(err error) error { + if err == nil { + return nil + } + return &invalidConfigError{err: err} +} + func LoadApp(dir string) (*AppConfig, error) { for _, name := range []string{"teploy.yml", "teploy.yaml", "teploy.toml"} { path := filepath.Join(dir, name) @@ -1173,21 +1198,21 @@ func LoadApp(dir string) (*AppConfig, error) { if errors.Is(err, os.ErrNotExist) { continue } - return nil, fmt.Errorf("reading %s: %w", path, err) + return nil, invalidConfig(fmt.Errorf("reading %s: %w", path, err)) } var cfg AppConfig if strings.HasSuffix(name, ".toml") { if err := unmarshalAppTOML(data, &cfg); err != nil { - return nil, fmt.Errorf("parsing %s: %w", name, err) + return nil, invalidConfig(fmt.Errorf("parsing %s: %w", name, err)) } } else { if err := unmarshalAppYAML(data, &cfg); err != nil { - return nil, fmt.Errorf("parsing %s: %w", name, err) + return nil, invalidConfig(fmt.Errorf("parsing %s: %w", name, err)) } } if err := cfg.validate(); err != nil { - return nil, fmt.Errorf("invalid %s: %w", name, err) + return nil, invalidConfig(fmt.Errorf("invalid %s: %w", name, err)) } return &cfg, nil } @@ -1195,7 +1220,7 @@ func LoadApp(dir string) (*AppConfig, error) { // No teploy config — try docker-compose auto-detection. composeCfg, err := LoadCompose(dir) if err != nil { - return nil, err + return nil, invalidConfig(err) } if composeCfg != nil { return composeCfg, nil @@ -1235,19 +1260,19 @@ func LoadAppWithDestination(dir, dest string, opts OverlayOptions) (*AppConfig, if errors.Is(err, os.ErrNotExist) { continue } - return nil, fmt.Errorf("reading %s: %w", path, err) + return nil, invalidConfig(fmt.Errorf("reading %s: %w", path, err)) } var overlay AppConfig var present map[string]any if ext == ".toml" { if err := unmarshalAppTOML(data, &overlay); err != nil { - return nil, fmt.Errorf("parsing %s: %w", name, err) + return nil, invalidConfig(fmt.Errorf("parsing %s: %w", name, err)) } present = tomlTopLevelKeys(data) } else { if err := unmarshalAppYAML(data, &overlay); err != nil { - return nil, fmt.Errorf("parsing %s: %w", name, err) + return nil, invalidConfig(fmt.Errorf("parsing %s: %w", name, err)) } present = yamlTopLevelKeys(data) } @@ -1257,12 +1282,12 @@ func LoadAppWithDestination(dir, dest string, opts OverlayOptions) (*AppConfig, } mergeConfigs(base, &overlay) if err := base.validate(); err != nil { - return nil, fmt.Errorf("invalid config after merging %s: %w", name, err) + return nil, invalidConfig(fmt.Errorf("invalid config after merging %s: %w", name, err)) } return base, nil } - return nil, fmt.Errorf("destination %q not found — expected teploy.%s.yml or teploy.%s.toml", dest, dest, dest) + return nil, invalidConfig(fmt.Errorf("destination %q not found — expected teploy.%s.yml or teploy.%s.toml", dest, dest, dest)) } // clearExplicitEmpties implements the strict-mode half of presence-aware From 773600d8030b78f16c2b7070de28d31e0f4f6008 Mon Sep 17 00:00:00 2001 From: Tyler <53561637+im-tyler@users.noreply.github.com> Date: Wed, 23 Sep 2026 05:37:11 -0700 Subject: [PATCH 7/8] =?UTF-8?q?feat(cli):=20doctor=20=E2=80=94=20nine=20re?= =?UTF-8?q?ad-only=20diagnostics=20with=20remediations,=20MI-1=20JSON,=20n?= =?UTF-8?q?o=20deployment=20effects=20(C09)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit git/config/ssh/docker/disk/registry/caddy/compatibility/repair-debt checks, each ok|warn|fail with detail + remediation; registry auth vs unreachable distinguished; effect-free asserted against the executor call log; --json emits the machine-interface envelope; exit 0/1 only. --- AUDIT_OPEN.md | 104 ++++ README.md | 1 + internal/cli/doctor.go | 592 ++++++++++++++++++++++ internal/cli/doctor_test.go | 678 ++++++++++++++++++++++++++ internal/cli/machineinterface.go | 5 + internal/cli/machineinterface_test.go | 2 + internal/cli/root.go | 1 + 7 files changed, 1383 insertions(+) create mode 100644 internal/cli/doctor.go create mode 100644 internal/cli/doctor_test.go diff --git a/AUDIT_OPEN.md b/AUDIT_OPEN.md index 9f52d35..95928a4 100644 --- a/AUDIT_OPEN.md +++ b/AUDIT_OPEN.md @@ -1792,3 +1792,107 @@ performed. - Generalizing per-command envelopes for the remaining --json verbs (health/log/drift/stats/plan/validate/registry/template/accessory lists) is S2's `contracts/` skeleton work, not error-site migration. + +## Programme slice (2026-09-23) — C09: `teploy doctor` + +First bounded C09 slice (base revision `dda4911`; changes left +committed-free for review, per instruction). Contract addressed: +"`doctor` should diagnose local toolchain, SSH, Docker, registry, proxy, +disk and compatibility without causing deployment. Human progress goes to +the appropriate diagnostic stream; versioned JSON/events and stable exit +codes serve automation." + +**Landed** (`internal/cli/doctor.go`, `teploy doctor [--json] +[--server ]`): + +- **Nine stable checks** (fixed order, pinned by test): `git` (local + PATH probe — missing git is a WARN, not a fail: git-less boxes deploy + prebuilt images fine), `config` (the same `config.LoadApp` loader + deploy uses, so the C05 Compose field contracts, C03 health-mode + grammar, publish specs and overlay rules surface verbatim in detail; + ErrNoConfig → fail with a `teploy init` remediation), `ssh` (the + EXISTING connect path — `ssh.Connect` errors carry the key/auth hints + and the 078f610 known_hosts algorithm naming, so the doctor detail + names the presented/on-file algorithms verbatim), `docker` (daemon + reachability via `docker version --format '{{.Server.Version}}'`), + `disk` (root-filesystem headroom from `df -B1 -P /`: fail < 2 GiB, + warn < 10 GiB or ≥ 85% used), `registry` (`docker manifest inspect` of + the configured ref — a pure registry query that touches no local image + state, unlike a pull; auth class DISTINGUISHED from unreachable from + missing, each with its own remediation; build-from-source apps are + ok-skips), `caddy` (admin API probe inside the caddy container — the + same command `server status` uses; host/external ingress are ok-skips + by design), `compatibility` (local version vs the server's + `/deployments/.bin/teploy` if present — absent is ok (optional + infrastructure), skew is a warn naming both versions), and + `repair-debt` (the C01-6 marker via `deploy.ReadRepairDebt` — + outstanding debt is a warn naming release+attempts; an UNREADABLE + marker is a visible warn, never hidden). +- **No deployment effects**: every remote command is read-only + (`docker version`, `docker manifest inspect`, the caddy admin wget, + `df`, the server binary's `version`, the framed repair-debt read). + Skip semantics are two-class and deliberate: skipped-because-not- + applicable (host/external ingress, build app, no server binary, no + app identity) = ok; skipped-because-input-unavailable (SSH down, + config unreadable) = fail with the reason. Tests assert the mock's + ENTIRE call log against a read-only allowlist on both the all-healthy + and the every-remote-check-failing runs, plus that no files are ever + uploaded. +- **Output contract**: human table on stdout (check/result/detail rows, + indented `fix:` remediation lines, closing summary); `--json` emits + the MI-1 envelope `{machine_interface, checks:[{name, result, + detail, remediation}], summary:{ok, warn, fail}}` — all four check + keys ALWAYS present (no omitempty: a stable shape means consumers + never probe for optional keys), `result` closed to ok|warn|fail, + summary counts machine-checked against the checks array. Exit codes: + 0 with no fail (warnings included), 1 with any fail, and never 2 — + that stays `drift --exit-code`'s CI signal (X02 D10); documented in + the command's help text and README. The report is the successful + OUTPUT of the command — a failing diagnosis never renders the error + envelope, stdout stays the data channel. +- **Capability token**: `doctor-diagnostics` added to the MI registry + (additive, no MI bump); the golden list in + TestCapabilityTokenRegistry updated to force the addition to stay + deliberate. + +**Evidence** — TDD red level 1 recorded (all new symbols undefined at +compile), green after implementation. New coverage (doctor_test.go, 17 +test functions / 30+ subtests): all-healthy run (9 ok, stable order, +read-only call log), exact JSON shape (3 top-level keys, 4 check keys, +enum-closed results, summary cross-check), human table + remediation +lines + summary, per-check pass/fail/warn paths (config grammar from +teploy.yml AND Compose, git missing, ssh unreachable with the +known_hosts algorithm diagnostics carried through, no target, docker +daemon down, disk fail/warn/ok thresholds + parser robustness, registry +auth/missing/unreachable classes + classifier, caddy ok/fail/host/ +external skips, compat agree/skew/absent/unreadable, repair-debt +absent/present/unreadable/no-app), the no-effects assertion on a +maximally failing run, exit-code semantics, and a cobra-wired end-to- +end all-OK run. Mutation checks (in-place, all reverted, gates re-run +green after each): a failing check reported ok (docker failure branch +forced to ok) fails TestDoctorDockerCheck and the human-table summary; +doctorExitCode forced to 0 fails all three exit-code assertions; +collapsing the registry auth class into unreachable fails the +auth-distinguished remediation and the classifier. Binary smoke: real +`teploy doctor` / `doctor --json` in an empty directory — table and +envelope as specified, exit 1, no stderr envelope, token advertised by +`version --json`. + +Gates: `go vet ./...` clean; `go test ./... -race -count=1` all 25 +packages ok; gofmt clean on touched files (pre-existing strays in +deploy.go/secret_audit.go/update_test.go left alone, consistent with +the C02-C04 posture); contract probes 5/5 PASS. No push performed. + +**C09 remainder (explicit):** per-command next-recovery-action strings +(doctor's remediation field covers the diagnostic surface; every OTHER +failure path still renders free-text errors — the S2 error-envelope +migration is the machinery, this is the content), shell completion +(cobra completion for the command tree, including doctor's --server +values from servers.yml), config examples executability (README/ +docs config snippets that cannot load under the current grammar — +a docs-vs-loader drift sweep), and the doctor surface itself has +natural follow-ons recorded here rather than hidden: multi-server +fleet diagnosis (doctor currently diagnoses ONE resolved target), +DNS/health-path diagnostics, and machine-event streaming (the +"versioned JSON/events" contract's events half — doctor emits one +versioned JSON document per run, not a stream). diff --git a/README.md b/README.md index 848020a..3772a4e 100644 --- a/README.md +++ b/README.md @@ -335,6 +335,7 @@ teploy log # deploy history teploy exec # run a command on the server (SSH) teploy app exec -- # run a command in the app container (migrations, etc.) teploy validate # check config and server readiness +teploy doctor [--server ] # read-only diagnostics: toolchain, SSH, Docker, registry, Caddy, disk, compatibility, repair debt (--json for machines; exit 1 if any check fails, never 2) teploy scale # multi-server deploy + LB update teploy version / update # version info and self-update ``` diff --git a/internal/cli/doctor.go b/internal/cli/doctor.go new file mode 100644 index 0000000..eb0adbb --- /dev/null +++ b/internal/cli/doctor.go @@ -0,0 +1,592 @@ +// `teploy doctor` — the C09 diagnostic slice: diagnose local toolchain, +// SSH, Docker, registry, Caddy, disk, machine-interface compatibility and +// repair debt WITHOUT causing deployment. Every remote command is +// read-only (tests pin the executor's call log against an allowlist); a +// doctor run that fails checks never mutates server state. +// +// Human progress goes to stdout as a table; --json emits the versioned +// machine envelope {machine_interface, checks, summary}. Exit codes are +// stable: 0 when no check fails, 1 when any check fails, and never 2 — +// that code stays `drift --exit-code`'s CI signal (X02 D10). +package cli + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "os/exec" + "os/signal" + "strconv" + "strings" + + "github.com/spf13/cobra" + "github.com/useteploy/teploy/internal/config" + "github.com/useteploy/teploy/internal/deploy" + "github.com/useteploy/teploy/internal/ssh" +) + +// Check results — the closed enum carried in both output modes. +const ( + doctorOK = "ok" + doctorWarn = "warn" + doctorFail = "fail" +) + +// Disk headroom thresholds for the root filesystem (bytes / used-%). +// Below 2 GiB a deploy fails mid-flight (image layers, backups, attempt +// artifacts all land on /); below 10 GiB or above 85% used the next +// deploy is at risk. +const ( + doctorDiskFailAvailable = 2 << 30 + doctorDiskWarnAvailable = 10 << 30 + doctorDiskWarnUsedPct = 85 +) + +// serverTeployBinaryPath is where `teploy autodeploy` installs the +// server-side teploy binary (autodeploy.go's deployment target). +const serverTeployBinaryPath = "/deployments/.bin/teploy" + +// doctorCaddyAdminProbe reaches the Caddy admin API inside the caddy +// container over the SSH executor — the same probe machine.go's server +// status uses, so doctor and `server status` observe the same surface. +const doctorCaddyAdminProbe = "docker exec caddy sh -c 'wget -qO- http://localhost:2019/config/apps/http 2>/dev/null || curl -sf http://localhost:2019/config/apps/http'" + +// doctorCheck is one diagnostic result. Name is a stable identifier +// automation codes against; Remediation is the operator's next action +// ("" when ok). All four keys are ALWAYS present in --json — a stable +// shape means a consumer never probes for optional keys. +type doctorCheck struct { + Name string `json:"name"` + Result string `json:"result"` // ok | warn | fail + Detail string `json:"detail"` + Remediation string `json:"remediation"` +} + +// doctorSummary counts each result class over the whole report. +type doctorSummary struct { + OK int `json:"ok"` + Warn int `json:"warn"` + Fail int `json:"fail"` +} + +// doctorReport is the `doctor --json` envelope (MI-1 additive surface). +type doctorReport struct { + MachineInterface int `json:"machine_interface"` + Checks []doctorCheck `json:"checks"` + Summary doctorSummary `json:"summary"` +} + +// doctorDeps carries the injectable surfaces: the local teploy version +// (compatibility comparison), the SSH connect (the existing connect path, +// whose errors already carry the key/auth/known_hosts diagnostics), and +// the local git probe. +type doctorDeps struct { + localVersion string + connect func(ctx context.Context, host, user, keyPath string) (ssh.Executor, error) + gitVersion func(ctx context.Context) (string, error) +} + +func defaultDoctorDeps(version string) doctorDeps { + return doctorDeps{ + localVersion: version, + connect: func(ctx context.Context, host, user, keyPath string) (ssh.Executor, error) { + return ssh.Connect(ctx, ssh.ConnectConfig{Host: host, User: user, KeyPath: keyPath}) + }, + gitVersion: doctorGitVersion, + } +} + +func newDoctorCmd(flags *Flags, version string) *cobra.Command { + var serverName string + cmd := &cobra.Command{ + Use: "doctor", + Short: "Diagnose toolchain, SSH, Docker, registry, Caddy, disk, compatibility and repair debt (read-only)", + Long: "Runs every diagnostic read-only and never mutates server state — a doctor run\n" + + "that fails checks deploys nothing.\n\n" + + "Checks: git, config (teploy.yml or Compose grammar), SSH connectivity to the\n" + + "app's server (or --server), remote Docker, remote disk headroom, registry\n" + + "reachability for the configured image (auth failures distinguished from\n" + + "unreachable), the Caddy admin API (caddy ingress only), teploy version\n" + + "compatibility with the server's teploy binary if present, and outstanding\n" + + "release-record repair debt.\n\n" + + "Exit codes: 0 when no check fails, 1 when any check fails, 2 never (that\n" + + "code stays drift --exit-code's CI signal).", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + return runDoctor(defaultDoctorDeps(version), flags, serverName, cmd.OutOrStdout()) + }, + } + cmd.Flags().StringVar(&serverName, "server", "", "diagnose this server (name or host) instead of the app's configured one") + return cmd +} + +func runDoctor(deps doctorDeps, flags *Flags, serverName string, out io.Writer) error { + ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt) + defer cancel() + + appCfg, cfgErr := config.LoadApp(".") + report, executor := doctorRun(ctx, deps, flags, serverName, appCfg, cfgErr) + if executor != nil { + // Closed before any os.Exit below — defers would be skipped. + executor.Close() + } + if err := writeDoctorReport(out, report, flags.JSON); err != nil { + return err + } + // The report is the successful OUTPUT of this command; the exit code + // reports the diagnosis, not command failure. 1 (never 2 — drift's). + if code := doctorExitCode(report); code != 0 { + os.Exit(code) + } + return nil +} + +// doctorRun executes every check and returns the report plus the open +// executor (nil when unreachable) — the caller owns closing it. Skips are +// honest about their class: a check skipped because its subject is not in +// play (host/external ingress, build-from-source image, optional server +// binary, no app identity) is ok; a check skipped because its input is +// unavailable (SSH down, config unreadable) is fail. +func doctorRun(ctx context.Context, deps doctorDeps, flags *Flags, serverName string, appCfg *config.AppConfig, cfgErr error) (doctorReport, ssh.Executor) { + report := doctorReport{MachineInterface: MachineInterface, Checks: []doctorCheck{}} + report.Checks = append(report.Checks, doctorGitCheck(ctx, deps)) + report.Checks = append(report.Checks, doctorConfigCheck(".", appCfg, cfgErr)) + + var executor ssh.Executor + host, user, key, hasTarget, err := doctorResolveTarget(flags, serverName, appCfg) + switch { + case err != nil: + report.Checks = append(report.Checks, doctorCheck{ + Name: "ssh", Result: doctorFail, + Detail: err.Error(), Remediation: "fix the server reference (see detail)", + }) + case !hasTarget: + report.Checks = append(report.Checks, doctorCheck{ + Name: "ssh", Result: doctorFail, Detail: "no server to diagnose", + Remediation: "set 'server' in teploy.yml, or pass --server or --host", + }) + default: + ex, connectErr := deps.connect(ctx, host, user, key) + if connectErr != nil { + report.Checks = append(report.Checks, doctorCheck{ + Name: "ssh", Result: doctorFail, + // The connect path's error already carries the key/auth + // diagnostics, including the known_hosts algorithm naming. + Detail: connectErr.Error(), Remediation: doctorSSHRemediation(connectErr), + }) + } else { + executor = ex + report.Checks = append(report.Checks, doctorCheck{ + Name: "ssh", Result: doctorOK, + Detail: fmt.Sprintf("connected to %s@%s", user, host), + }) + } + } + + skipFail := func(name, what, remediation string) { + report.Checks = append(report.Checks, doctorCheck{ + Name: name, Result: doctorFail, + Detail: "skipped — " + what, + Remediation: remediation, + }) + } + + if executor == nil { + const sshDown = "restore SSH connectivity (see the ssh check above), then re-run teploy doctor" + for _, name := range []string{"docker", "disk", "registry", "caddy", "compatibility", "repair-debt"} { + skipFail(name, "SSH unreachable", sshDown) + } + } else { + report.Checks = append(report.Checks, doctorDockerCheck(ctx, executor)) + report.Checks = append(report.Checks, doctorDiskCheck(ctx, executor)) + if appCfg == nil { + const cfgBroken = "fix the config (see the config check above), then re-run teploy doctor" + skipFail("registry", "config unreadable — no image ref", cfgBroken) + skipFail("caddy", "config unreadable — ingress unknown", cfgBroken) + } else { + report.Checks = append(report.Checks, doctorRegistryCheck(ctx, executor, appCfg)) + report.Checks = append(report.Checks, doctorCaddyCheck(ctx, executor, appCfg)) + } + report.Checks = append(report.Checks, doctorCompatCheck(ctx, deps, executor)) + report.Checks = append(report.Checks, doctorRepairDebtCheck(ctx, executor, appCfg)) + } + + report.summarize() + return report, executor +} + +// doctorExitCode pins the documented semantics: 0 with no failing check +// (warnings included), 1 with any fail — and never 2, which stays +// `drift --exit-code`'s CI signal. +func doctorExitCode(report doctorReport) int { + if report.Summary.Fail > 0 { + return 1 + } + return 0 +} + +func (r *doctorReport) summarize() { + r.Summary = doctorSummary{} + for _, c := range r.Checks { + switch c.Result { + case doctorOK: + r.Summary.OK++ + case doctorWarn: + r.Summary.Warn++ + case doctorFail: + r.Summary.Fail++ + } + } +} + +func writeDoctorReport(out io.Writer, report doctorReport, jsonOutput bool) error { + if jsonOutput { + return json.NewEncoder(out).Encode(report) + } + for _, c := range report.Checks { + fmt.Fprintf(out, "%-15s %-5s %s\n", c.Name, c.Result, c.Detail) + if c.Remediation != "" { + fmt.Fprintf(out, "%-15s %-5s fix: %s\n", "", "", c.Remediation) + } + } + fmt.Fprintf(out, "\nSummary: %d ok, %d warn, %d fail\n", report.Summary.OK, report.Summary.Warn, report.Summary.Fail) + return nil +} + +// doctorResolveTarget picks the diagnosis target: --server wins, then the +// app's configured server, then --host. ok=false means no target could be +// determined (the ssh check reports it); err is a failed resolution. +func doctorResolveTarget(flags *Flags, serverName string, appCfg *config.AppConfig) (host, user, key string, ok bool, err error) { + switch { + case serverName != "": + host, user, key, err = config.ResolveServer(serverName, flags.Host, flags.User, flags.Key) + case appCfg != nil: + name := appCfg.Server + if name == "" && len(appCfg.Servers) > 0 { + name = appCfg.Servers[0] + } + if name == "" { + return "", "", "", false, nil + } + host, user, key, err = config.ResolveServer(name, flags.Host, flags.User, flags.Key) + if err == nil { + // Honor teploy.yml's user: the same way validate and deploy + // connect, so doctor diagnoses as the deploying account. + user = config.EffectiveUser(user, flags.User, appCfg.User) + } + case flags.Host != "": + host, user, key, err = config.ResolveServer(flags.Host, flags.Host, flags.User, flags.Key) + } + if err != nil || host == "" { + return "", "", "", false, err + } + return host, user, key, true, nil +} + +func doctorGitCheck(ctx context.Context, deps doctorDeps) doctorCheck { + version, err := deps.gitVersion(ctx) + if err != nil { + // Warn, not fail: git-less boxes deploy prebuilt images fine. + return doctorCheck{ + Name: "git", Result: doctorWarn, Detail: err.Error(), + Remediation: "install git — provenance records, git template installs, and autodeploy checkouts need it", + } + } + return doctorCheck{Name: "git", Result: doctorOK, Detail: version} +} + +// doctorGitVersion probes the local toolchain. exec.LookPath first so a +// missing git is a clean "not found" rather than a shell error. +func doctorGitVersion(ctx context.Context) (string, error) { + if _, err := exec.LookPath("git"); err != nil { + return "", errors.New("git not found on PATH") + } + out, err := exec.CommandContext(ctx, "git", "--version").Output() + if err != nil { + return "", fmt.Errorf("running git --version: %w", err) + } + return strings.TrimSpace(string(out)), nil +} + +// doctorConfigCheck surfaces the config grammar through the same loader +// deploy uses — the "new grammar errors" (Compose field contracts, health +// modes, publish specs, overlay rules) arrive verbatim in Detail. +func doctorConfigCheck(dir string, appCfg *config.AppConfig, cfgErr error) doctorCheck { + if cfgErr != nil { + remediation := "fix the configuration error above — the message names the file and the grammar problem" + if errors.Is(cfgErr, config.ErrNoConfig) { + remediation = "create a teploy.yml (teploy init) or a docker-compose file in this directory" + } + return doctorCheck{Name: "config", Result: doctorFail, Detail: cfgErr.Error(), Remediation: remediation} + } + return doctorCheck{Name: "config", Result: doctorOK, Detail: doctorConfigSource(dir) + " parsed and validated"} +} + +// doctorConfigSource names the file LoadApp would load from dir, for the +// config check's detail line. +func doctorConfigSource(dir string) string { + for _, name := range []string{"teploy.yml", "teploy.yaml", "teploy.toml"} { + if _, err := os.Stat(dir + string(os.PathSeparator) + name); err == nil { + return name + } + } + for _, name := range []string{"docker-compose.yml", "docker-compose.yaml", "compose.yml", "compose.yaml"} { + if _, err := os.Stat(dir + string(os.PathSeparator) + name); err == nil { + return name + " (imported)" + } + } + return "config" +} + +// doctorSSHRemediation maps the connect path's self-describing failures +// to the next action. Display-only classification; verification always +// stays with the failing connection itself. +func doctorSSHRemediation(err error) string { + msg := err.Error() + switch { + case strings.Contains(msg, "authentication failed"), strings.Contains(msg, "unable to authenticate"): + return "fix SSH authentication: --user (root SSH is disabled on most distros), --key for a specific identity, or --password" + case strings.Contains(msg, "host key mismatch"): + return "scan every host-key algorithm (ssh-keyscan without -t), or — only after verifying the host legitimately changed — re-enroll it" + case strings.Contains(msg, "no SSH keys found"): + return "provide --key, set TEPLOY_SSH_KEY, or place a key at ~/.ssh/id_ed25519" + default: + return "resolve the SSH failure above — the message names the specific key, auth, or host-key problem — then re-run teploy doctor" + } +} + +// doctorDockerCheck proves the DAEMON answers (server version via the +// docker CLI on the target), not just that the binary exists. +func doctorDockerCheck(ctx context.Context, exec ssh.Executor) doctorCheck { + out, err := exec.Run(ctx, "docker version --format '{{.Server.Version}}'") + if err != nil { + return doctorCheck{ + Name: "docker", Result: doctorFail, Detail: err.Error(), + Remediation: "install Docker on the server (teploy setup provisions it) and check docker.sock permissions for the SSH user", + } + } + version := strings.TrimSpace(out) + if version == "" { + return doctorCheck{ + Name: "docker", Result: doctorFail, + Detail: "docker answered but reported no server version", + Remediation: "check the docker service on the server (systemctl status docker)", + } + } + return doctorCheck{Name: "docker", Result: doctorOK, Detail: fmt.Sprintf("Docker server %s reachable", version)} +} + +// doctorDiskCheck reports root-filesystem headroom via df over the +// executor — read-only, POSIX -P framing, bytes (-B1) so thresholds are +// exact. +func doctorDiskCheck(ctx context.Context, exec ssh.Executor) doctorCheck { + out, err := exec.Run(ctx, "df -B1 -P /") + if err != nil { + return doctorCheck{ + Name: "disk", Result: doctorFail, Detail: err.Error(), + Remediation: "check the df output on the server — it failed outright", + } + } + avail, pct, parsed := parseDoctorDisk(out) + if !parsed { + return doctorCheck{ + Name: "disk", Result: doctorFail, + Detail: fmt.Sprintf("could not parse df output: %q", strings.TrimSpace(out)), + Remediation: "inspect `df -B1 -P /` on the server", + } + } + detail := fmt.Sprintf("%.1f GiB available on / (%d%% used)", float64(avail)/(1<<30), pct) + switch { + case avail < doctorDiskFailAvailable: + return doctorCheck{ + Name: "disk", Result: doctorFail, Detail: detail, + Remediation: "free disk space on / (docker system prune, teploy releases/preview prune, old backups) — deploys fail mid-flight below 2 GiB", + } + case avail < doctorDiskWarnAvailable || pct >= doctorDiskWarnUsedPct: + return doctorCheck{ + Name: "disk", Result: doctorWarn, Detail: detail, + Remediation: "free disk space on / before the next deploy (docker system prune, teploy releases prune)", + } + } + return doctorCheck{Name: "disk", Result: doctorOK, Detail: detail} +} + +// parseDoctorDisk reads `df -B1 -P` output: the LAST line's fields are +// fs, 1-blocks, used, available, capacity%, mounted-on (spaces in the +// mount point stay right of the numerics). +func parseDoctorDisk(raw string) (avail uint64, usedPercent int, ok bool) { + lines := strings.Split(strings.TrimSpace(raw), "\n") + if len(lines) < 2 { + return 0, 0, false + } + fields := strings.Fields(lines[len(lines)-1]) + if len(fields) < 6 { + return 0, 0, false + } + avail, err := strconv.ParseUint(fields[3], 10, 64) + if err != nil { + return 0, 0, false + } + usedPercent, err = strconv.Atoi(strings.TrimSuffix(fields[4], "%")) + if err != nil { + return 0, 0, false + } + return avail, usedPercent, true +} + +// doctorRegistryCheck asks the SERVER's docker to resolve the configured +// image ref's manifest — a pure registry query (docker manifest inspect +// touches no local image state, unlike a pull). Auth failures are +// distinguished from unreachable registries so the remediation names the +// actual fix. +func doctorRegistryCheck(ctx context.Context, exec ssh.Executor, appCfg *config.AppConfig) doctorCheck { + if appCfg.Image == "" { + return doctorCheck{ + Name: "registry", Result: doctorOK, + Detail: "no registry image ref — the image is built from source at deploy time", + } + } + if _, err := exec.Run(ctx, "docker manifest inspect "+ssh.ShellQuote(appCfg.Image)); err != nil { + switch classifyRegistryError(err) { + case "auth": + return doctorCheck{ + Name: "registry", Result: doctorFail, Detail: err.Error(), + Remediation: "store credentials on the server: teploy registry login — deploys pull as the server's docker", + } + case "missing": + return doctorCheck{ + Name: "registry", Result: doctorFail, Detail: err.Error(), + Remediation: "push the image to the registry, or correct the image ref in teploy.yml", + } + default: + return doctorCheck{ + Name: "registry", Result: doctorFail, Detail: err.Error(), + Remediation: "check the network path from the server to the registry (DNS, firewall, proxy)", + } + } + } + return doctorCheck{Name: "registry", Result: doctorOK, Detail: fmt.Sprintf("registry reachable for %s", appCfg.Image)} +} + +// classifyRegistryError buckets a manifest-inspect failure into auth / +// missing / unreachable — display classification only; the detail always +// carries the underlying error verbatim. +func classifyRegistryError(err error) string { + msg := strings.ToLower(err.Error()) + switch { + case strings.Contains(msg, "unauthorized"), strings.Contains(msg, "authentication required"), strings.Contains(msg, "denied"): + return "auth" + case strings.Contains(msg, "no such manifest"), strings.Contains(msg, "manifest unknown"), strings.Contains(msg, "not found"): + return "missing" + default: + return "unreachable" + } +} + +// doctorCaddyCheck probes the admin API inside the caddy container — +// only when teploy manages ingress. host/external ingress never routes +// through teploy's Caddy, so those are ok-skips, not failures. +func doctorCaddyCheck(ctx context.Context, exec ssh.Executor, appCfg *config.AppConfig) doctorCheck { + switch appCfg.Ingress { + case config.IngressExternal: + return doctorCheck{ + Name: "caddy", Result: doctorOK, + Detail: "ingress external — Caddy is not in the request path", + } + case config.IngressHost: + return doctorCheck{ + Name: "caddy", Result: doctorOK, + Detail: "ingress host — the app publishes directly on its bind port", + } + } + if _, err := exec.Run(ctx, doctorCaddyAdminProbe); err != nil { + return doctorCheck{ + Name: "caddy", Result: doctorFail, Detail: err.Error(), + Remediation: "check the caddy container on the server (docker ps --filter name=^caddy$) — teploy setup provisions and self-heals it", + } + } + return doctorCheck{Name: "caddy", Result: doctorOK, Detail: "Caddy admin API responding (ingress caddy)"} +} + +// doctorCompatCheck compares this binary's version against the server's +// teploy binary when one is installed (autodeploy's /deployments/.bin). +// Absence is ok — the server binary is optional infrastructure; skew is +// a warning because scheduled redeploys and webhook builds run on it. +func doctorCompatCheck(ctx context.Context, deps doctorDeps, exec ssh.Executor) doctorCheck { + out, err := exec.Run(ctx, ssh.ShellQuote(serverTeployBinaryPath)+" version") + if err != nil { + msg := err.Error() + if strings.Contains(msg, "not found") || strings.Contains(msg, "no such file") || strings.Contains(msg, "status 127") { + return doctorCheck{ + Name: "compatibility", Result: doctorOK, + Detail: fmt.Sprintf("no server-side teploy binary (optional — autodeploy installs one at %s)", serverTeployBinaryPath), + } + } + return doctorCheck{ + Name: "compatibility", Result: doctorWarn, Detail: err.Error(), + Remediation: fmt.Sprintf("inspect %s on the server — it exists but would not run", serverTeployBinaryPath), + } + } + serverVersion := doctorServerTeployVersion(out) + if serverVersion == "" { + return doctorCheck{ + Name: "compatibility", Result: doctorWarn, + Detail: "server teploy present but reported no version", + Remediation: "re-run teploy autodeploy install to refresh the server binary", + } + } + if serverVersion == deps.localVersion { + return doctorCheck{ + Name: "compatibility", Result: doctorOK, + Detail: fmt.Sprintf("local teploy %s matches the server binary", deps.localVersion), + } + } + return doctorCheck{ + Name: "compatibility", Result: doctorWarn, + Detail: fmt.Sprintf("local teploy %s, server teploy %s", deps.localVersion, serverVersion), + Remediation: "re-run teploy autodeploy install (or teploy autodeploy schedule) to refresh the server binary", + } +} + +// doctorServerTeployVersion reads the server binary's `version` output +// ("teploy v0.1.37" in the human format every release speaks). +func doctorServerTeployVersion(out string) string { + fields := strings.Fields(strings.TrimSpace(out)) + if len(fields) == 0 { + return "" + } + return fields[len(fields)-1] +} + +// doctorRepairDebtCheck reports the C01-6 marker: outstanding +// release-record debt from a deploy whose record write failed after the +// live commit. A warn, not a fail — the next deploy repairs it before its +// own work; an UNREADABLE marker is visible too (unhealable debt must not +// be invisible debt). +func doctorRepairDebtCheck(ctx context.Context, exec ssh.Executor, appCfg *config.AppConfig) doctorCheck { + if appCfg == nil { + return doctorCheck{ + Name: "repair-debt", Result: doctorOK, + Detail: "no app identity (config unreadable) — nothing to inspect", + } + } + debt, err := deploy.ReadRepairDebt(ctx, exec, appCfg.App) + if err != nil { + return doctorCheck{ + Name: "repair-debt", Result: doctorWarn, Detail: err.Error(), + Remediation: fmt.Sprintf("inspect or remove /deployments/%s/repair-debt.json on the server — the debt cannot be read", appCfg.App), + } + } + if debt == nil { + return doctorCheck{Name: "repair-debt", Result: doctorOK, Detail: "no outstanding release-record repair debt"} + } + return doctorCheck{ + Name: "repair-debt", Result: doctorWarn, + Detail: fmt.Sprintf("release record for %s@%s missing after %d failed write/repair attempt(s): %s — the next deploy rebuilds it", + debt.App, debt.Release, debt.Attempts, debt.Reason), + Remediation: "re-run teploy deploy (the reconciler repairs the record before its own work); teploy status shows the same debt", + } +} diff --git a/internal/cli/doctor_test.go b/internal/cli/doctor_test.go new file mode 100644 index 0000000..9149d5e --- /dev/null +++ b/internal/cli/doctor_test.go @@ -0,0 +1,678 @@ +package cli + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/useteploy/teploy/internal/config" + "github.com/useteploy/teploy/internal/ssh" +) + +// doctorCheckOrder pins the stable check-name sequence automation codes +// against (adding checks is additive at the end; renaming/removing is a +// machine-interface bump). +var doctorCheckOrder = []string{ + "git", "config", "ssh", "docker", "disk", "registry", "caddy", "compatibility", "repair-debt", +} + +// doctorHappyMock registers every remote read a fully healthy target +// answers, for the app config returned by doctorTestApp. +func doctorHappyMock() *ssh.MockExecutor { + return ssh.NewMockExecutor("192.0.2.10", + ssh.MockCommand{Match: "docker version", Output: "27.3.1"}, + ssh.MockCommand{Match: "df -B1 -P", Output: "Filesystem 1-blocks Used Available Capacity Mounted on\n/dev/vda1 100000000000 20000000000 80000000000 20% /"}, + ssh.MockCommand{Match: "docker manifest inspect", Output: `{"schemaVersion":2}`}, + ssh.MockCommand{Match: "docker exec caddy", Output: `{}`}, + ssh.MockCommand{Match: "'/deployments/.bin/teploy' version", Output: "teploy v0.1.37"}, + ) +} + +func doctorTestApp() *config.AppConfig { + return &config.AppConfig{App: "blog", Server: "192.0.2.10", Image: "example/blog:v1"} +} + +func doctorTestDeps(mock *ssh.MockExecutor) doctorDeps { + return doctorDeps{ + localVersion: "v0.1.37", + connect: func(ctx context.Context, host, user, key string) (ssh.Executor, error) { + return mock, nil + }, + gitVersion: func(ctx context.Context) (string, error) { + return "git version 2.39.5", nil + }, + } +} + +func doctorFindCheck(t *testing.T, report doctorReport, name string) doctorCheck { + t.Helper() + for _, c := range report.Checks { + if c.Name == name { + return c + } + } + t.Fatalf("check %q missing from report: %+v", name, report.Checks) + return doctorCheck{} +} + +func TestDoctorCommandRegistered(t *testing.T) { + root := NewRootCmd("test") + cmd, _, err := root.Find([]string{"doctor"}) + if err != nil { + t.Fatalf("finding doctor: %v", err) + } + if cmd == root || cmd.Name() != "doctor" { + t.Fatalf("doctor command not registered") + } +} + +func TestDoctorAllChecksPass(t *testing.T) { + t.Setenv("HOME", t.TempDir()) + mock := doctorHappyMock() + report, ex := doctorRun(context.Background(), doctorTestDeps(mock), &Flags{}, "", doctorTestApp(), nil) + if ex == nil { + t.Fatal("expected an executor back") + } + if report.Summary != (doctorSummary{OK: 9, Warn: 0, Fail: 0}) { + t.Fatalf("summary = %+v, want 9 ok", report.Summary) + } + if len(report.Checks) != len(doctorCheckOrder) { + t.Fatalf("got %d checks, want %d", len(report.Checks), len(doctorCheckOrder)) + } + for i, name := range doctorCheckOrder { + if report.Checks[i].Name != name { + t.Fatalf("check[%d] = %q, want %q (stable order)", i, report.Checks[i].Name, name) + } + if report.Checks[i].Result != "ok" { + t.Fatalf("check %s = %s (%s), want ok", name, report.Checks[i].Result, report.Checks[i].Detail) + } + } + sshCheck := doctorFindCheck(t, report, "ssh") + if !strings.Contains(sshCheck.Detail, "192.0.2.10") { + t.Fatalf("ssh detail must name the target: %+v", sshCheck) + } + dockerCheck := doctorFindCheck(t, report, "docker") + if !strings.Contains(dockerCheck.Detail, "27.3.1") { + t.Fatalf("docker detail must carry the version: %+v", dockerCheck) + } + assertDoctorReadOnlyCalls(t, mock.Calls) +} + +func TestDoctorJSONStableShape(t *testing.T) { + t.Setenv("HOME", t.TempDir()) + report, _ := doctorRun(context.Background(), doctorTestDeps(doctorHappyMock()), &Flags{}, "", doctorTestApp(), nil) + var out bytes.Buffer + if err := writeDoctorReport(&out, report, true); err != nil { + t.Fatal(err) + } + var doc map[string]any + if err := json.Unmarshal(out.Bytes(), &doc); err != nil { + t.Fatalf("doctor --json is not valid JSON: %q: %v", out.String(), err) + } + if len(doc) != 3 { + t.Fatalf("top-level keys = %v, want exactly machine_interface/checks/summary", doc) + } + for _, key := range []string{"machine_interface", "checks", "summary"} { + if _, ok := doc[key]; !ok { + t.Fatalf("doctor envelope missing %q: %s", key, out.String()) + } + } + if doc["machine_interface"] != float64(MachineInterface) { + t.Fatalf("machine_interface = %v, want %d", doc["machine_interface"], MachineInterface) + } + checks, ok := doc["checks"].([]any) + if !ok || len(checks) != 9 { + t.Fatalf("checks = %#v, want 9 entries", doc["checks"]) + } + for i, raw := range checks { + check, ok := raw.(map[string]any) + if !ok { + t.Fatalf("check %d is not an object: %#v", i, raw) + } + if len(check) != 4 { + t.Fatalf("check %d keys = %v, want exactly name/result/detail/remediation", i, check) + } + for _, key := range []string{"name", "result"} { + if _, ok := check[key]; !ok { + t.Fatalf("check %d missing %q: %v", i, key, check) + } + } + switch check["result"] { + case "ok", "warn", "fail": + default: + t.Fatalf("check %d result %v outside the ok/warn/fail enum", i, check["result"]) + } + } + summary, ok := doc["summary"].(map[string]any) + if !ok || len(summary) != 3 { + t.Fatalf("summary = %#v, want exactly ok/warn/fail", doc["summary"]) + } + var okCount, warnCount, failCount int + for _, raw := range checks { + check := raw.(map[string]any) + switch check["result"] { + case "ok": + okCount++ + case "warn": + warnCount++ + case "fail": + failCount++ + } + } + if summary["ok"] != float64(okCount) || summary["warn"] != float64(warnCount) || summary["fail"] != float64(failCount) { + t.Fatalf("summary %v disagrees with checks (ok=%d warn=%d fail=%d)", summary, okCount, warnCount, failCount) + } +} + +func TestDoctorHumanTable(t *testing.T) { + t.Setenv("HOME", t.TempDir()) + // One failing remote check (docker unreachable) exercises the + // remediation row rendering. + mock := ssh.NewMockExecutor("192.0.2.10", + ssh.MockCommand{Match: "df -B1 -P", Output: "Filesystem 1-blocks Used Available Capacity Mounted on\n/dev/vda1 100000000000 20000000000 80000000000 20% /"}, + ssh.MockCommand{Match: "docker manifest inspect", Output: `{}`}, + ssh.MockCommand{Match: "docker exec caddy", Output: `{}`}, + ssh.MockCommand{Match: "'/deployments/.bin/teploy' version", Output: "teploy v0.1.37"}, + ) + report, _ := doctorRun(context.Background(), doctorTestDeps(mock), &Flags{}, "", doctorTestApp(), nil) + var out bytes.Buffer + if err := writeDoctorReport(&out, report, false); err != nil { + t.Fatal(err) + } + rendered := out.String() + if !strings.Contains(rendered, "docker") || !strings.Contains(rendered, "fail") { + t.Fatalf("table missing the failing docker row:\n%s", rendered) + } + if !strings.Contains(rendered, "fix:") { + t.Fatalf("table missing the remediation line:\n%s", rendered) + } + if !strings.Contains(rendered, "Summary: 8 ok, 0 warn, 1 fail") { + t.Fatalf("table summary line wrong:\n%s", rendered) + } + assertDoctorReadOnlyCalls(t, mock.Calls) +} + +func TestDoctorConfigCheck(t *testing.T) { + t.Run("ok", func(t *testing.T) { + dir := t.TempDir() + if err := os.WriteFile(filepath.Join(dir, "teploy.yml"), []byte("app: blog\nserver: prod\ndomain: blog.example.com\n"), 0600); err != nil { + t.Fatal(err) + } + cfg, err := config.LoadApp(dir) + if err != nil { + t.Fatalf("fixture config must load: %v", err) + } + check := doctorConfigCheck(dir, cfg, nil) + if check.Result != "ok" { + t.Fatalf("config check = %+v, want ok", check) + } + if !strings.Contains(check.Detail, "teploy.yml") { + t.Fatalf("config detail must name the file: %+v", check) + } + }) + + t.Run("grammar error surfaced", func(t *testing.T) { + dir := t.TempDir() + source := "app: blog\nserver: prod\ndomain: blog.example.com\ningress: bogus-mode\n" + if err := os.WriteFile(filepath.Join(dir, "teploy.yml"), []byte(source), 0600); err != nil { + t.Fatal(err) + } + _, err := config.LoadApp(dir) + if err == nil { + t.Fatal("fixture config must fail to load") + } + check := doctorConfigCheck(dir, nil, err) + if check.Result != "fail" { + t.Fatalf("config check = %+v, want fail", check) + } + if !strings.Contains(check.Detail, "bogus-mode") { + t.Fatalf("config detail must surface the grammar error verbatim: %+v", check) + } + }) + + t.Run("compose grammar error surfaced", func(t *testing.T) { + dir := t.TempDir() + source := "services:\n web:\n image: example/web:v1\n ports: ['3000:3000']\n hostname: fixed\n" + if err := os.WriteFile(filepath.Join(dir, "compose.yml"), []byte(source), 0600); err != nil { + t.Fatal(err) + } + _, err := config.LoadApp(dir) + if err == nil { + t.Fatal("fixture compose must fail to import") + } + check := doctorConfigCheck(dir, nil, err) + if check.Result != "fail" { + t.Fatalf("compose config check = %+v, want fail", check) + } + if !strings.Contains(check.Detail, "hostname") { + t.Fatalf("compose detail must surface the refusal: %+v", check) + } + }) + + t.Run("no config", func(t *testing.T) { + check := doctorConfigCheck(t.TempDir(), nil, config.ErrNoConfig) + if check.Result != "fail" { + t.Fatalf("no-config check = %+v, want fail", check) + } + if !strings.Contains(check.Remediation, "teploy init") { + t.Fatalf("no-config remediation must point at init: %+v", check) + } + }) +} + +func TestDoctorGitCheck(t *testing.T) { + deps := doctorTestDeps(doctorHappyMock()) + deps.gitVersion = func(ctx context.Context) (string, error) { + return "git version 2.39.5", nil + } + if check := doctorGitCheck(context.Background(), deps); check.Result != "ok" { + t.Fatalf("git check = %+v, want ok", check) + } + deps.gitVersion = func(ctx context.Context) (string, error) { + return "", errors.New("git not found on PATH") + } + check := doctorGitCheck(context.Background(), deps) + if check.Result != "warn" { + t.Fatalf("missing git = %+v, want warn", check) + } + if !strings.Contains(check.Remediation, "install git") { + t.Fatalf("git remediation must say install: %+v", check) + } +} + +func TestDoctorSSHUnreachable(t *testing.T) { + t.Setenv("HOME", t.TempDir()) + // The exact enriched message shape ssh.Connect produces for a + // known_hosts algorithm-coverage mismatch (078f610) — doctor must + // carry it verbatim so the operator sees the algorithm names. + connectErr := errors.New("host key mismatch for 192.0.2.10: server presented ssh-rsa, known_hosts has no matching entry (has ssh-ed25519) — scan all algorithms (ssh-keyscan without -t), not just one: ssh: handshake failed: knownhosts: key mismatch") + deps := doctorDeps{ + localVersion: "v0.1.37", + connect: func(ctx context.Context, host, user, key string) (ssh.Executor, error) { + return nil, connectErr + }, + gitVersion: func(ctx context.Context) (string, error) { return "git version 2.39.5", nil }, + } + report, ex := doctorRun(context.Background(), deps, &Flags{}, "", doctorTestApp(), nil) + if ex != nil { + t.Fatal("no executor expected on connect failure") + } + sshCheck := doctorFindCheck(t, report, "ssh") + if sshCheck.Result != "fail" { + t.Fatalf("ssh check = %+v, want fail", sshCheck) + } + for _, frag := range []string{"ssh-rsa", "ssh-ed25519", "ssh-keyscan"} { + if !strings.Contains(sshCheck.Detail, frag) { + t.Fatalf("ssh detail must carry the known_hosts diagnostics (%s): %+v", frag, sshCheck) + } + } + // Every remote check is skipped-with-fail, naming SSH as the reason. + for _, name := range []string{"docker", "disk", "registry", "caddy", "compatibility", "repair-debt"} { + check := doctorFindCheck(t, report, name) + if check.Result != "fail" || !strings.Contains(check.Detail, "SSH unreachable") { + t.Fatalf("%s check = %+v, want fail/skipped (SSH unreachable)", name, check) + } + } + if code := doctorExitCode(report); code != 1 { + t.Fatalf("exit code = %d, want 1 with a failing check", code) + } +} + +func TestDoctorNoTargetConfigured(t *testing.T) { + t.Setenv("HOME", t.TempDir()) + deps := doctorTestDeps(doctorHappyMock()) + deps.gitVersion = func(ctx context.Context) (string, error) { return "", errors.New("git not found on PATH") } + report, ex := doctorRun(context.Background(), deps, &Flags{}, "", nil, config.ErrNoConfig) + if ex != nil { + t.Fatal("no executor expected without a target") + } + sshCheck := doctorFindCheck(t, report, "ssh") + if sshCheck.Result != "fail" || !strings.Contains(sshCheck.Remediation, "--server") { + t.Fatalf("ssh check = %+v, want fail naming --server", sshCheck) + } + if code := doctorExitCode(report); code != 1 { + t.Fatalf("exit code = %d, want 1", code) + } +} + +func TestDoctorDockerCheck(t *testing.T) { + ctx := context.Background() + mock := ssh.NewMockExecutor("h", ssh.MockCommand{Match: "docker version", Output: "27.3.1"}) + if check := doctorDockerCheck(ctx, mock); check.Result != "ok" || !strings.Contains(check.Detail, "27.3.1") { + t.Fatalf("docker check = %+v, want ok with version", check) + } + failed := ssh.NewMockExecutor("h", ssh.MockCommand{Match: "docker version", Err: errors.New("exit status 1: Cannot connect to the Docker daemon")}) + check := doctorDockerCheck(ctx, failed) + if check.Result != "fail" || !strings.Contains(check.Detail, "Cannot connect") { + t.Fatalf("docker check = %+v, want fail with the error", check) + } + if !strings.Contains(check.Remediation, "Docker") { + t.Fatalf("docker remediation must point at Docker: %+v", check) + } +} + +func TestDoctorDiskCheck(t *testing.T) { + ctx := context.Background() + const header = "Filesystem 1-blocks Used Available Capacity Mounted on\n" + + t.Run("healthy", func(t *testing.T) { + mock := ssh.NewMockExecutor("h", ssh.MockCommand{Match: "df -B1 -P", Output: header + "/dev/vda1 100000000000 20000000000 80000000000 20% /"}) + check := doctorDiskCheck(ctx, mock) + if check.Result != "ok" || !strings.Contains(check.Detail, "74.5 GiB") { + t.Fatalf("disk check = %+v, want ok with 74.5 GiB available", check) + } + }) + t.Run("low headroom warns", func(t *testing.T) { + mock := ssh.NewMockExecutor("h", ssh.MockCommand{Match: "df -B1 -P", Output: header + "/dev/vda1 100000000000 96000000000 4000000000 96% /"}) + check := doctorDiskCheck(ctx, mock) + if check.Result != "warn" { + t.Fatalf("disk check = %+v, want warn below 10 GiB", check) + } + }) + t.Run("critical fails", func(t *testing.T) { + mock := ssh.NewMockExecutor("h", ssh.MockCommand{Match: "df -B1 -P", Output: header + "/dev/vda1 100000000000 99000000000 1000000000 99% /"}) + check := doctorDiskCheck(ctx, mock) + if check.Result != "fail" { + t.Fatalf("disk check = %+v, want fail below 2 GiB", check) + } + }) + t.Run("df fails", func(t *testing.T) { + mock := ssh.NewMockExecutor("h", ssh.MockCommand{Match: "df -B1 -P", Err: errors.New("exit status 1")}) + check := doctorDiskCheck(ctx, mock) + if check.Result != "fail" { + t.Fatalf("disk check = %+v, want fail", check) + } + }) + t.Run("parser", func(t *testing.T) { + raw := header + "/dev/vda1 100 20 80 20% /" + avail, pct, ok := parseDoctorDisk(raw) + if !ok || avail != 80 || pct != 20 { + t.Fatalf("parseDoctorDisk = %d %d %v, want 80 20 true", avail, pct, ok) + } + if _, _, ok := parseDoctorDisk("garbage"); ok { + t.Fatal("parseDoctorDisk accepted garbage") + } + // Mount points with spaces keep the numeric fields aligned. + avail, pct, ok = parseDoctorDisk(header + "/dev/vda1 100 20 80 20% /mnt with space") + if !ok || avail != 80 || pct != 20 { + t.Fatalf("parseDoctorDisk with spaced mount = %d %d %v", avail, pct, ok) + } + }) +} + +func TestDoctorRegistryCheck(t *testing.T) { + ctx := context.Background() + + t.Run("reachable", func(t *testing.T) { + mock := ssh.NewMockExecutor("h", ssh.MockCommand{Match: "docker manifest inspect", Output: `{}`}) + check := doctorRegistryCheck(ctx, mock, doctorTestApp()) + if check.Result != "ok" { + t.Fatalf("registry check = %+v, want ok", check) + } + }) + t.Run("auth class distinguished", func(t *testing.T) { + mock := ssh.NewMockExecutor("h", ssh.MockCommand{Match: "docker manifest inspect", Err: errors.New("denied: requested access to the resource is denied")}) + check := doctorRegistryCheck(ctx, mock, doctorTestApp()) + if check.Result != "fail" { + t.Fatalf("registry check = %+v, want fail", check) + } + if !strings.Contains(check.Remediation, "teploy registry login") { + t.Fatalf("auth-class remediation must name registry login: %+v", check) + } + }) + t.Run("unreachable class", func(t *testing.T) { + mock := ssh.NewMockExecutor("h", ssh.MockCommand{Match: "docker manifest inspect", Err: errors.New(`Get "https://registry.example/v2/": dial tcp: lookup registry.example: no such host`)}) + check := doctorRegistryCheck(ctx, mock, doctorTestApp()) + if check.Result != "fail" { + t.Fatalf("registry check = %+v, want fail", check) + } + if strings.Contains(check.Remediation, "registry login") { + t.Fatalf("unreachable class must not ask for a login: %+v", check) + } + }) + t.Run("missing image", func(t *testing.T) { + mock := ssh.NewMockExecutor("h", ssh.MockCommand{Match: "docker manifest inspect", Err: errors.New("no such manifest for revision v1 in registry")}) + check := doctorRegistryCheck(ctx, mock, doctorTestApp()) + if check.Result != "fail" || !strings.Contains(check.Remediation, "push") { + t.Fatalf("missing-image check = %+v, want fail with push remediation", check) + } + }) + t.Run("build app has no ref to check", func(t *testing.T) { + mock := ssh.NewMockExecutor("h") + cfg := doctorTestApp() + cfg.Image = "" + check := doctorRegistryCheck(ctx, mock, cfg) + if check.Result != "ok" || !strings.Contains(check.Detail, "built from source") { + t.Fatalf("build-app check = %+v, want ok skip", check) + } + if len(mock.Calls) != 0 { + t.Fatalf("registry check must not touch the server for a build app: %v", mock.Calls) + } + }) + t.Run("classifier", func(t *testing.T) { + cases := map[string]string{ + "denied: requested access to the resource is denied": "auth", + "unauthorized: authentication required": "auth", + "no such manifest for revision v1": "missing", + "manifest unknown": "missing", + `Get "https://r.example/v2/": dial tcp: lookup r.example: no such host`: "unreachable", + "i/o timeout": "unreachable", + } + for msg, want := range cases { + if got := classifyRegistryError(errors.New(msg)); got != want { + t.Fatalf("classifyRegistryError(%q) = %q, want %q", msg, got, want) + } + } + }) +} + +func TestDoctorCaddyCheck(t *testing.T) { + ctx := context.Background() + + t.Run("caddy ingress reachable", func(t *testing.T) { + mock := ssh.NewMockExecutor("h", ssh.MockCommand{Match: "docker exec caddy", Output: `{}`}) + check := doctorCaddyCheck(ctx, mock, doctorTestApp()) + if check.Result != "ok" { + t.Fatalf("caddy check = %+v, want ok", check) + } + }) + t.Run("admin api unreachable", func(t *testing.T) { + mock := ssh.NewMockExecutor("h", ssh.MockCommand{Match: "docker exec caddy", Err: errors.New("exit status 1: Error response from daemon: No such container: caddy")}) + check := doctorCaddyCheck(ctx, mock, doctorTestApp()) + if check.Result != "fail" || !strings.Contains(check.Detail, "No such container") { + t.Fatalf("caddy check = %+v, want fail naming the error", check) + } + }) + t.Run("host ingress skips caddy", func(t *testing.T) { + mock := ssh.NewMockExecutor("h") + cfg := doctorTestApp() + cfg.Ingress = config.IngressHost + check := doctorCaddyCheck(ctx, mock, cfg) + if check.Result != "ok" || !strings.Contains(check.Detail, "host") { + t.Fatalf("host-ingress check = %+v, want ok skip", check) + } + if len(mock.Calls) != 0 { + t.Fatalf("host ingress must not probe caddy: %v", mock.Calls) + } + }) + t.Run("external ingress skips caddy", func(t *testing.T) { + mock := ssh.NewMockExecutor("h") + cfg := doctorTestApp() + cfg.Ingress = config.IngressExternal + check := doctorCaddyCheck(ctx, mock, cfg) + if check.Result != "ok" { + t.Fatalf("external-ingress check = %+v, want ok skip", check) + } + if len(mock.Calls) != 0 { + t.Fatalf("external ingress must not probe caddy: %v", mock.Calls) + } + }) +} + +func TestDoctorCompatibilityCheck(t *testing.T) { + ctx := context.Background() + deps := doctorTestDeps(nil) + + t.Run("versions agree", func(t *testing.T) { + mock := ssh.NewMockExecutor("h", ssh.MockCommand{Match: "'/deployments/.bin/teploy' version", Output: "teploy v0.1.37"}) + if check := doctorCompatCheck(ctx, deps, mock); check.Result != "ok" { + t.Fatalf("compat check = %+v, want ok", check) + } + }) + t.Run("version skew warns", func(t *testing.T) { + mock := ssh.NewMockExecutor("h", ssh.MockCommand{Match: "'/deployments/.bin/teploy' version", Output: "teploy v0.1.30"}) + check := doctorCompatCheck(ctx, deps, mock) + if check.Result != "warn" { + t.Fatalf("compat check = %+v, want warn", check) + } + if !strings.Contains(check.Detail, "v0.1.37") || !strings.Contains(check.Detail, "v0.1.30") { + t.Fatalf("compat detail must name both versions: %+v", check) + } + if !strings.Contains(check.Remediation, "autodeploy") { + t.Fatalf("compat remediation must name the refresh path: %+v", check) + } + }) + t.Run("server binary absent is not a failure", func(t *testing.T) { + mock := ssh.NewMockExecutor("h", ssh.MockCommand{Match: "'/deployments/.bin/teploy' version", Err: errors.New("exit status 127: /bin/sh: 1: /deployments/.bin/teploy: not found")}) + check := doctorCompatCheck(ctx, deps, mock) + if check.Result != "ok" { + t.Fatalf("absent server binary = %+v, want ok", check) + } + }) + t.Run("unreadable server binary warns", func(t *testing.T) { + mock := ssh.NewMockExecutor("h", ssh.MockCommand{Match: "'/deployments/.bin/teploy' version", Err: errors.New("exit status 1: permission denied")}) + if check := doctorCompatCheck(ctx, deps, mock); check.Result != "warn" { + t.Fatalf("unreadable server binary = %+v, want warn", check) + } + }) +} + +func TestDoctorRepairDebtCheck(t *testing.T) { + ctx := context.Background() + + t.Run("no debt", func(t *testing.T) { + mock := ssh.NewMockExecutor("h") + if check := doctorRepairDebtCheck(ctx, mock, doctorTestApp()); check.Result != "ok" { + t.Fatalf("repair-debt check = %+v, want ok", check) + } + }) + t.Run("outstanding debt warns", func(t *testing.T) { + mock := ssh.NewMockExecutor("h") + mock.Files["/deployments/blog/repair-debt.json"] = []byte(`{"schema_version":1,"app":"blog","release":"abc123","reason":"write failed","attempts":2,"first_failed_at":"2026-09-23T10:00:00Z","last_failed_at":"2026-09-23T10:00:00Z"}`) + check := doctorRepairDebtCheck(ctx, mock, doctorTestApp()) + if check.Result != "warn" { + t.Fatalf("repair-debt check = %+v, want warn", check) + } + if !strings.Contains(check.Detail, "abc123") { + t.Fatalf("repair-debt detail must name the release: %+v", check) + } + }) + t.Run("unreadable marker warns", func(t *testing.T) { + mock := ssh.NewMockExecutor("h", ssh.MockCommand{Match: "if [ ! -e '/deployments/blog/repair-debt.json'", Err: errors.New("permission denied")}) + if check := doctorRepairDebtCheck(ctx, mock, doctorTestApp()); check.Result != "warn" { + t.Fatalf("unreadable marker = %+v, want warn", check) + } + }) + t.Run("no app identity", func(t *testing.T) { + mock := ssh.NewMockExecutor("h") + if check := doctorRepairDebtCheck(ctx, mock, nil); check.Result != "ok" { + t.Fatalf("no-app check = %+v, want ok skip", check) + } + if len(mock.Calls) != 0 { + t.Fatalf("no-app must not touch the server: %v", mock.Calls) + } + }) +} + +// TestDoctorNoEffectsWhenFailing drives a run where every remote check +// fails and asserts the executor ONLY ever saw read-only commands — +// doctor's no-deployment-effects contract. +func TestDoctorNoEffectsWhenFailing(t *testing.T) { + t.Setenv("HOME", t.TempDir()) + mock := ssh.NewMockExecutor("192.0.2.10", + ssh.MockCommand{Match: "docker version", Err: errors.New("exit status 1: daemon down")}, + ssh.MockCommand{Match: "df -B1 -P", Err: errors.New("exit status 1")}, + ssh.MockCommand{Match: "docker manifest inspect", Err: errors.New("unauthorized: authentication required")}, + ssh.MockCommand{Match: "docker exec caddy", Err: errors.New("exit status 1: no such container")}, + ssh.MockCommand{Match: "'/deployments/.bin/teploy' version", Err: errors.New("exit status 127: not found")}, + ssh.MockCommand{Match: "if [ ! -e '/deployments/blog/repair-debt.json'", Err: errors.New("permission denied")}, + ) + report, _ := doctorRun(context.Background(), doctorTestDeps(mock), &Flags{}, "", doctorTestApp(), nil) + if report.Summary.Fail == 0 { + t.Fatal("expected failing checks") + } + if code := doctorExitCode(report); code != 1 { + t.Fatalf("exit code = %d, want 1", code) + } + assertDoctorReadOnlyCalls(t, mock.Calls) + if len(mock.Files) != 0 { + t.Fatalf("doctor must not leave uploaded files behind: %v", mock.Files) + } +} + +func TestDoctorExitCode(t *testing.T) { + ok := doctorReport{Checks: []doctorCheck{{Result: "ok"}, {Result: "warn"}}} + ok.summarize() + if code := doctorExitCode(ok); code != 0 { + t.Fatalf("warn-only exit = %d, want 0", code) + } + failing := doctorReport{Checks: []doctorCheck{{Result: "ok"}, {Result: "fail"}}} + failing.summarize() + if code := doctorExitCode(failing); code != 1 { + t.Fatalf("any-fail exit = %d, want 1", code) + } +} + +// TestDoctorEndToEndAllOK drives the cobra wiring on an all-healthy world +// (runDoctor os.Exit(1)s on any fail, so failing worlds are exercised via +// doctorRun + doctorExitCode above — drift's testing posture). +func TestDoctorEndToEndAllOK(t *testing.T) { + t.Setenv("HOME", t.TempDir()) + dir := t.TempDir() + if err := os.WriteFile(filepath.Join(dir, "teploy.yml"), []byte("app: blog\nserver: 192.0.2.10\ndomain: blog.example.com\nimage: example/blog:v1\n"), 0600); err != nil { + t.Fatal(err) + } + t.Chdir(dir) + + mock := doctorHappyMock() + + var out bytes.Buffer + if err := runDoctor(doctorTestDeps(mock), &Flags{}, "", &out); err != nil { + t.Fatalf("runDoctor: %v", err) + } + if !strings.Contains(out.String(), "Summary: 9 ok, 0 warn, 0 fail") { + t.Fatalf("human report wrong:\n%s", out.String()) + } + assertDoctorReadOnlyCalls(t, mock.Calls) +} + +// assertDoctorReadOnlyCalls fails the test when any executed command is +// outside doctor's read-only allowlist — the no-deployment-effects proof. +func assertDoctorReadOnlyCalls(t *testing.T, calls []string) { + t.Helper() + readOnly := []string{ + "docker version", + "docker manifest inspect", + "docker exec caddy", + "df ", + "'/deployments/.bin/teploy' version", + "if [ ! -e '/deployments/", + } + for _, call := range calls { + allowed := false + for _, prefix := range readOnly { + if strings.HasPrefix(call, prefix) { + allowed = true + break + } + } + if !allowed { + t.Fatalf("doctor issued a non-read-only command: %q", call) + } + } +} diff --git a/internal/cli/machineinterface.go b/internal/cli/machineinterface.go index 6602ea1..f09d098 100644 --- a/internal/cli/machineinterface.go +++ b/internal/cli/machineinterface.go @@ -74,6 +74,10 @@ const ( CapAppListMachine = "app-list-machine" // `server status --json` emits the MI-1 machine envelope. CapServerStatusMachine = "server-status-machine" + // `teploy doctor [--json] [--server]`: read-only diagnostics with + // stable check names, ok/warn/fail results, remediations, and 0/1 + // exit semantics — 2 never (that stays drift's) (C09). + CapDoctorDiagnostics = "doctor-diagnostics" ) // MachineCapabilities returns every capability token this build @@ -95,6 +99,7 @@ func MachineCapabilities() []string { CapErrorEnvelope, CapAppListMachine, CapServerStatusMachine, + CapDoctorDiagnostics, } sort.Strings(tokens) return tokens diff --git a/internal/cli/machineinterface_test.go b/internal/cli/machineinterface_test.go index c49dfdd..1cb472e 100644 --- a/internal/cli/machineinterface_test.go +++ b/internal/cli/machineinterface_test.go @@ -84,6 +84,7 @@ func TestCapabilityTokenRegistry(t *testing.T) { want := []string{ "app-list-machine", "autodeploy-redeploy", + "doctor-diagnostics", "env-set-stdin", "error-envelope", "health-modes", @@ -123,6 +124,7 @@ func TestCapabilityTokenRegistry(t *testing.T) { CapHealthModes, CapProvenanceRecords, CapReadinessReceipts, CapPreviewCanonicalID, CapRepairDebt, CapPreviewBlueGreen, CapErrorEnvelope, CapAppListMachine, CapServerStatusMachine, + CapDoctorDiagnostics, } { if !member[token] { t.Fatalf("capability constant %q is not advertised", token) diff --git a/internal/cli/root.go b/internal/cli/root.go index 1c61d4e..719f494 100644 --- a/internal/cli/root.go +++ b/internal/cli/root.go @@ -77,6 +77,7 @@ func NewRootCmd(version string) *cobra.Command { root.AddCommand(newUnlockCmd(flags)) root.AddCommand(newInitCmd()) root.AddCommand(newValidateCmd(flags)) + root.AddCommand(newDoctorCmd(flags, version)) root.AddCommand(newPlanCmd(flags)) root.AddCommand(newDriftCmd(flags)) root.AddCommand(newHealCmd(flags)) From d53c5bf8962f2916feb006fa8d64ddf34d962f98 Mon Sep 17 00:00:00 2001 From: Tyler <53561637+im-tyler@users.noreply.github.com> Date: Wed, 23 Sep 2026 10:13:41 -0700 Subject: [PATCH 8/8] feat(contracts): X02 S2 - machine-interface fixture corpus skeleton + first goldens contracts/ in teploy-cli is the corpus home (ADR section 4, D15): the producer of the envelopes owns the schema, and the CLI is the bottom of the stack with no Neutron/Nucleus dependency and a public mirror. - schema/ (draft 2020-12): version-handshake, app-list-envelope, server-status-envelope (appStatus root), error-envelope (code enum = the six-code taxonomy), release-record, attempt-name grammar pattern, preview-state eras (canonical/legacy oneOf), observation-envelope and operation-record (dash-produced; schemas shared, fixtures pending their S5/S6 encoders). - fixtures/: generated from the REAL encoders where a CLI producer exists - writeVersion drives the handshake golden verbatim; app-list pins the MI-1 envelope plus the pre-MI legacy shape (missing field = legacy producer, never MI 0); error-envelope valid + invalid-code; release-record container; attempt-name valid/invalid; preview-state canonical (ids cross-checked against the pinned TestPreviewIDGolden table), legacy slug, and the two-branches-one-slug ambiguous fixture whose rule is refuse-adoption. - MANIFEST.md: corpus revision table (rev 1 = post-v0.1.37 main), artifact/producer status, regeneration rules, the same-commit rule for contract changes, and the servers.yml downgrade hazard note. - contracts_golden_test.go regenerates with TEPLOY_UPDATE_CONTRACTS=1 and FAILS on drift otherwise - CI gets the drift check via the normal suite. Declared S2 tails (not done here): server-status fixture from a live capture, the non-additive server list --json reshape (needs a coordinated dash decode change), dash fail-closed on MI > max, and the X01 section 5.2 job-3 extension to decode the pinned corpus. Full CLI suite green (25 packages). --- contracts/MANIFEST.md | 63 +++++++ .../app-list-envelope/legacy/pre-mi.json | 6 + .../fixtures/app-list-envelope/valid/mi1.json | 41 +++++ .../attempt-name/invalid/examples.json | 7 + .../fixtures/attempt-name/valid/examples.json | 4 + .../error-envelope/invalid/unknown-code.json | 5 + .../error-envelope/valid/config-invalid.json | 6 + .../error-envelope/valid/internal.json | 6 + .../ambiguous/two-branches-one-slug.json | 10 + .../preview-state/legacy/slug-keyed.json | 5 + .../preview-state/valid/canonical.json | 6 + .../release-record/valid/container.json | 12 ++ .../fixtures/version-handshake/valid/mi1.json | 22 +++ .../schema/app-list-envelope.schema.json | 174 ++++++++++++++++++ contracts/schema/attempt-name.schema.json | 7 + contracts/schema/error-envelope.schema.json | 14 ++ .../schema/observation-envelope.schema.json | 16 ++ contracts/schema/operation-record.schema.json | 18 ++ contracts/schema/preview-state.schema.json | 20 ++ contracts/schema/release-record.schema.json | 22 +++ .../schema/server-status-envelope.schema.json | 77 ++++++++ .../schema/version-handshake.schema.json | 17 ++ internal/cli/contracts_golden_test.go | 138 ++++++++++++++ 23 files changed, 696 insertions(+) create mode 100644 contracts/MANIFEST.md create mode 100644 contracts/fixtures/app-list-envelope/legacy/pre-mi.json create mode 100644 contracts/fixtures/app-list-envelope/valid/mi1.json create mode 100644 contracts/fixtures/attempt-name/invalid/examples.json create mode 100644 contracts/fixtures/attempt-name/valid/examples.json create mode 100644 contracts/fixtures/error-envelope/invalid/unknown-code.json create mode 100644 contracts/fixtures/error-envelope/valid/config-invalid.json create mode 100644 contracts/fixtures/error-envelope/valid/internal.json create mode 100644 contracts/fixtures/preview-state/ambiguous/two-branches-one-slug.json create mode 100644 contracts/fixtures/preview-state/legacy/slug-keyed.json create mode 100644 contracts/fixtures/preview-state/valid/canonical.json create mode 100644 contracts/fixtures/release-record/valid/container.json create mode 100644 contracts/fixtures/version-handshake/valid/mi1.json create mode 100644 contracts/schema/app-list-envelope.schema.json create mode 100644 contracts/schema/attempt-name.schema.json create mode 100644 contracts/schema/error-envelope.schema.json create mode 100644 contracts/schema/observation-envelope.schema.json create mode 100644 contracts/schema/operation-record.schema.json create mode 100644 contracts/schema/preview-state.schema.json create mode 100644 contracts/schema/release-record.schema.json create mode 100644 contracts/schema/server-status-envelope.schema.json create mode 100644 contracts/schema/version-handshake.schema.json create mode 100644 internal/cli/contracts_golden_test.go diff --git a/contracts/MANIFEST.md b/contracts/MANIFEST.md new file mode 100644 index 0000000..0d53a19 --- /dev/null +++ b/contracts/MANIFEST.md @@ -0,0 +1,63 @@ +# Teploy contracts corpus — MANIFEST + +The machine-interface fixture corpus (X02 S2; ADR `_internal/ +X02_RESOURCE_CONTRACT_ADR_2026-09-22.md` §4, adopted by +DELEGATED_DECISIONS_2026-09-23 D15). teploy-cli owns the corpus because it +produces the envelopes and sits at the bottom of the stack with no +Neutron/Nucleus dependency and a public mirror. + +## Revision table + +| Corpus rev | Emitting CLI | Machine Interface | Notes | +|---|---|---|---| +| 1 | post-v0.1.37 main (S2 skeleton) | 1 | First goldens: version handshake, app-list envelope (MI + pre-MI legacy), error envelope (config-invalid, internal, invalid-code), release-record, attempt-name grammar, preview-state eras. | + +## Artifact status + +| Artifact | Schema | Fixtures | Producer | +|---|---|---|---| +| version-handshake | yes | valid (real `writeVersion` encoder) | teploy-cli | +| app-list-envelope | yes | valid (real DTO tags) + legacy pre-MI | teploy-cli | +| server-status-envelope | yes (appStatus root) | pending S2 tail (live `server status` capture) | teploy-cli | +| error-envelope | yes | valid x2 + invalid code | teploy-cli | +| release-record | yes | valid container | teploy-cli | +| attempt-name | yes (pattern) | valid + invalid examples | teploy-cli | +| preview-state | yes (canonical/legacy) | valid + legacy + ambiguous | teploy-cli | +| observation-envelope | yes | pending S6 (dash encoder) | teploy-dash | +| operation-record | yes | pending S5/S6 (dash) | teploy-dash | + +## Rules + +- Fixtures under `valid/` and `legacy/` are GENERATED from the real + encoders where a CLI producer exists (`internal/cli/ + contracts_golden_test.go`, run with `TEPLOY_UPDATE_CONTRACTS=1` to + rewrite). Hand-authored fixtures say so in this file. Never edit a + generated fixture by hand. +- `invalid/` and `ambiguous/` fixtures MUST fail schema validation / + adoption respectively — they pin refusals, not shapes. +- A corpus change lands in the SAME commit as the code that changed the + contract, with this manifest's revision table bumped. Non-additive + changes bump `machine_interface` (D8) and are coordinated with + teploy-dash's decoder first. +- Legacy fixtures are first-class forever: an id-less server, a pre-MI + envelope, a slug-keyed preview are states real deployments carry. + +## Regeneration + +``` +cd teploy-cli +TEPLOY_UPDATE_CONTRACTS=1 go test ./internal/cli/ -run TestContracts +``` + +CI runs the same test WITHOUT the env var: any drift between the corpus +and the encoders fails the build. + +## Known downgrade hazard (from the ADR §5 row 1) + +An older CLI rewriting `~/.teploy/servers.yml` silently drops unknown +fields, so an `id` minted by a newer CLI can vanish on downgrade. The +file itself cannot enforce it; the mitigation is consumer-side (dash +treats id-vanished as ambiguous-legacy requiring explicit re-binding, +never auto-re-mint). Consumers MUST NOT treat a missing +`machine_interface` field as MI 0 — it means "pre-MI producer", the +legacy decode path. diff --git a/contracts/fixtures/app-list-envelope/legacy/pre-mi.json b/contracts/fixtures/app-list-envelope/legacy/pre-mi.json new file mode 100644 index 0000000..a50e74d --- /dev/null +++ b/contracts/fixtures/app-list-envelope/legacy/pre-mi.json @@ -0,0 +1,6 @@ +{ + "apps": [], + "errors": null, + "host": "srv.example.com", + "observed_at": "2026-09-23T12:00:00Z" +} diff --git a/contracts/fixtures/app-list-envelope/valid/mi1.json b/contracts/fixtures/app-list-envelope/valid/mi1.json new file mode 100644 index 0000000..d280539 --- /dev/null +++ b/contracts/fixtures/app-list-envelope/valid/mi1.json @@ -0,0 +1,41 @@ +{ + "machine_interface": 1, + "host": "srv.example.com", + "apps": [ + { + "app": "myapp", + "domain": "myapp.example.com", + "type": "container", + "ingress": "caddy", + "current_release": { + "version": "3", + "ports": [ + 3000 + ] + }, + "previous_release": { + "version": "", + "ports": null + }, + "containers": [ + { + "id": "9f31c02", + "name": "myapp-web-3", + "image": "nginx:1.27", + "state": "running", + "status": "Up 4 minutes", + "created_at": "2026-09-23T11:55:00Z", + "process": "web", + "version": "3" + } + ], + "processes": null, + "lock": null, + "maintenance": false, + "observed_at": "2026-09-23T12:00:00Z", + "errors": null + } + ], + "observed_at": "2026-09-23T12:00:00Z", + "errors": null +} diff --git a/contracts/fixtures/attempt-name/invalid/examples.json b/contracts/fixtures/attempt-name/invalid/examples.json new file mode 100644 index 0000000..56b862b --- /dev/null +++ b/contracts/fixtures/attempt-name/invalid/examples.json @@ -0,0 +1,7 @@ +[ + "deadb17ecafef00d", + "ABC1234.deadb17ecafef00d", + "abc1234.DeadB17eCafef00d", + "abc1234.deadb17ecafef00", + "../escape.attempt0000000" +] diff --git a/contracts/fixtures/attempt-name/valid/examples.json b/contracts/fixtures/attempt-name/valid/examples.json new file mode 100644 index 0000000..2b164d2 --- /dev/null +++ b/contracts/fixtures/attempt-name/valid/examples.json @@ -0,0 +1,4 @@ +[ + "abc1234.deadb17ecafef00d", + "9f31c02.0123456789abcdef" +] diff --git a/contracts/fixtures/error-envelope/invalid/unknown-code.json b/contracts/fixtures/error-envelope/invalid/unknown-code.json new file mode 100644 index 0000000..e44fd46 --- /dev/null +++ b/contracts/fixtures/error-envelope/invalid/unknown-code.json @@ -0,0 +1,5 @@ +{ + "machine_interface": 1, + "code": "kaboom", + "message": "x" +} diff --git a/contracts/fixtures/error-envelope/valid/config-invalid.json b/contracts/fixtures/error-envelope/valid/config-invalid.json new file mode 100644 index 0000000..b71e650 --- /dev/null +++ b/contracts/fixtures/error-envelope/valid/config-invalid.json @@ -0,0 +1,6 @@ +{ + "machine_interface": 1, + "code": "config-invalid", + "message": "invalid teploy configuration", + "detail": "teploy.yml: services.0.name: required" +} diff --git a/contracts/fixtures/error-envelope/valid/internal.json b/contracts/fixtures/error-envelope/valid/internal.json new file mode 100644 index 0000000..4fb51dd --- /dev/null +++ b/contracts/fixtures/error-envelope/valid/internal.json @@ -0,0 +1,6 @@ +{ + "machine_interface": 1, + "code": "internal", + "message": "command failed", + "detail": "dial tcp: connection refused" +} diff --git a/contracts/fixtures/preview-state/ambiguous/two-branches-one-slug.json b/contracts/fixtures/preview-state/ambiguous/two-branches-one-slug.json new file mode 100644 index 0000000..247e30c --- /dev/null +++ b/contracts/fixtures/preview-state/ambiguous/two-branches-one-slug.json @@ -0,0 +1,10 @@ +{ + "id": "myapp-p-feature-login", + "era": "legacy", + "app": "myapp", + "candidates": [ + {"branch": "feature/login", "canonical_id": "myapp-p-08e81639"}, + {"branch": "feature-login", "canonical_id": "myapp-p-cb4bdf9a"} + ], + "rule": "refuse adoption; require explicit binding (C06 ambiguity contract - never auto-adopt)" +} diff --git a/contracts/fixtures/preview-state/legacy/slug-keyed.json b/contracts/fixtures/preview-state/legacy/slug-keyed.json new file mode 100644 index 0000000..12bb2ff --- /dev/null +++ b/contracts/fixtures/preview-state/legacy/slug-keyed.json @@ -0,0 +1,5 @@ +{ + "id": "myapp-p-feature-login", + "era": "legacy", + "app": "myapp" +} diff --git a/contracts/fixtures/preview-state/valid/canonical.json b/contracts/fixtures/preview-state/valid/canonical.json new file mode 100644 index 0000000..322c57a --- /dev/null +++ b/contracts/fixtures/preview-state/valid/canonical.json @@ -0,0 +1,6 @@ +{ + "id": "myapp-p-08e81639", + "era": "canonical", + "app": "myapp", + "branch": "feature/login" +} diff --git a/contracts/fixtures/release-record/valid/container.json b/contracts/fixtures/release-record/valid/container.json new file mode 100644 index 0000000..1ec418e --- /dev/null +++ b/contracts/fixtures/release-record/valid/container.json @@ -0,0 +1,12 @@ +{ + "schema_version": 1, + "app": "myapp", + "hash": "abc1234.deadb17ecafef00d", + "created_at": "2026-09-23T12:00:00Z", + "deployment_type": "container", + "ingress_mode": "caddy", + "domain": "myapp.example.com", + "image_ref": "nginx:1.27", + "image_digest": "sha256:0000", + "manifest_sha256": "sha256:beef" +} diff --git a/contracts/fixtures/version-handshake/valid/mi1.json b/contracts/fixtures/version-handshake/valid/mi1.json new file mode 100644 index 0000000..def2d5d --- /dev/null +++ b/contracts/fixtures/version-handshake/valid/mi1.json @@ -0,0 +1,22 @@ +{ + "capabilities": [ + "app-list-machine", + "autodeploy-redeploy", + "doctor-diagnostics", + "env-set-stdin", + "error-envelope", + "health-modes", + "kv-set-stdin", + "preview-blue-green", + "preview-canonical-id", + "provenance-records", + "readiness-receipts", + "repair-debt", + "server-rename", + "server-status-machine", + "server-update", + "template-var-stdin" + ], + "machine_interface": 1, + "version": "v0.0.0-contracts" +} diff --git a/contracts/schema/app-list-envelope.schema.json b/contracts/schema/app-list-envelope.schema.json new file mode 100644 index 0000000..a12014a --- /dev/null +++ b/contracts/schema/app-list-envelope.schema.json @@ -0,0 +1,174 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://teploy.github.io/contracts/schema/app-list-envelope.schema.json", + "title": "teploy app list --json (MI 1 appListDTO)", + "type": "object", + "required": [ + "machine_interface", + "host", + "apps", + "observed_at", + "errors" + ], + "properties": { + "machine_interface": { + "type": "integer", + "minimum": 1 + }, + "host": { + "type": "string" + }, + "apps": { + "type": "array", + "items": { + "$ref": "#/$defs/appStatus" + } + }, + "observed_at": { + "type": "string", + "format": "date-time" + }, + "errors": { + "type": "array", + "items": { + "$ref": "#/$defs/machineError" + } + } + }, + "$defs": { + "machineError": { + "type": "object", + "required": [ + "scope", + "message" + ], + "properties": { + "scope": { + "type": "string" + }, + "message": { + "type": "string" + } + } + }, + "container": { + "type": "object", + "required": [ + "id", + "name", + "image", + "state", + "status" + ], + "properties": { + "id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "image": { + "type": "string" + }, + "state": { + "type": "string" + }, + "status": { + "type": "string" + }, + "created_at": { + "type": "string" + }, + "process": { + "type": "string" + }, + "version": { + "type": "string" + } + } + }, + "release": { + "type": "object", + "required": [ + "version", + "ports" + ], + "properties": { + "version": { + "type": "string" + }, + "ports": { + "type": "array", + "items": { + "type": "integer" + } + } + } + }, + "appStatus": { + "type": "object", + "required": [ + "app", + "domain", + "type", + "ingress", + "current_release", + "previous_release", + "containers", + "processes", + "lock", + "maintenance", + "observed_at", + "errors" + ], + "properties": { + "app": { + "type": "string" + }, + "domain": { + "type": "string" + }, + "type": { + "type": "string" + }, + "ingress": { + "type": "string" + }, + "current_release": { + "$ref": "#/$defs/release" + }, + "previous_release": { + "$ref": "#/$defs/release" + }, + "containers": { + "type": "array", + "items": { + "$ref": "#/$defs/container" + } + }, + "processes": { + "type": "array" + }, + "lock": { + "type": [ + "object", + "null" + ] + }, + "maintenance": { + "type": "boolean" + }, + "observed_at": { + "type": "string", + "format": "date-time" + }, + "errors": { + "type": "array", + "items": { + "$ref": "#/$defs/machineError" + } + } + } + } + } +} diff --git a/contracts/schema/attempt-name.schema.json b/contracts/schema/attempt-name.schema.json new file mode 100644 index 0000000..30af49c --- /dev/null +++ b/contracts/schema/attempt-name.schema.json @@ -0,0 +1,7 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://teploy.github.io/contracts/schema/attempt-name.schema.json", + "title": "deploy attempt directory name grammar: .<16hex> (F08)", + "type": "string", + "pattern": "^[a-f0-9]{7}\\.[a-f0-9]{16}$" +} diff --git a/contracts/schema/error-envelope.schema.json b/contracts/schema/error-envelope.schema.json new file mode 100644 index 0000000..7b774d9 --- /dev/null +++ b/contracts/schema/error-envelope.schema.json @@ -0,0 +1,14 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://teploy.github.io/contracts/schema/error-envelope.schema.json", + "title": "structured error envelope on stderr under --json (X02 S3)", + "type": "object", + "required": ["machine_interface", "code", "message"], + "additionalProperties": false, + "properties": { + "machine_interface": {"type": "integer", "minimum": 1}, + "code": {"type": "string", "enum": ["config-invalid", "internal", "conflict", "uncertain-outcome", "degraded", "unsupported"]}, + "message": {"type": "string", "minLength": 1}, + "detail": {"type": "string"} + } +} diff --git a/contracts/schema/observation-envelope.schema.json b/contracts/schema/observation-envelope.schema.json new file mode 100644 index 0000000..c7ca99a --- /dev/null +++ b/contracts/schema/observation-envelope.schema.json @@ -0,0 +1,16 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://teploy.github.io/contracts/schema/observation-envelope.schema.json", + "title": "observation envelope - standard shape for any observed resource (X02 S2.4; PRODUCER = teploy-dash)", + "$comment": "Schema shared via this corpus but the emitting code lives in teploy-dash (S6). Fixtures for this artifact are dash-side until S6 lands its encoder.", + "type": "object", + "required": ["resource", "observed_at", "status"], + "properties": { + "resource": {"type": "object", "required": ["kind", "id"], + "properties": {"kind": {"type": "string"}, "id": {"type": "string"}}}, + "observed_at": {"type": "string", "format": "date-time"}, + "status": {"type": "string", "enum": ["evaluated", "unavailable", "invalid", "unknown"]}, + "last_known": {"type": ["object", "null"]}, + "reason": {"type": "string"} + } +} diff --git a/contracts/schema/operation-record.schema.json b/contracts/schema/operation-record.schema.json new file mode 100644 index 0000000..ee58e2f --- /dev/null +++ b/contracts/schema/operation-record.schema.json @@ -0,0 +1,18 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://teploy.github.io/contracts/schema/operation-record.schema.json", + "title": "Dash persistedOperation (R2/D02; PRODUCER = teploy-dash)", + "$comment": "Schema shared via this corpus; emitting code lives in teploy-dash. Fixtures arrive with S5/S6 dash slices. Pre-D02 records carry null reconciliation - the legacy fixture class.", + "type": "object", + "required": ["id", "kind", "status", "created_at"], + "properties": { + "id": {"type": "string", "minLength": 1}, + "kind": {"type": "string"}, + "status": {"type": "string"}, + "created_at": {"type": "string", "format": "date-time"}, + "principal": {"type": "string"}, + "idempotency_key": {"type": ["string", "null"]}, + "record_version": {"type": ["integer", "null"]}, + "reconciliation": {"type": ["object", "null"]} + } +} diff --git a/contracts/schema/preview-state.schema.json b/contracts/schema/preview-state.schema.json new file mode 100644 index 0000000..cf421de --- /dev/null +++ b/contracts/schema/preview-state.schema.json @@ -0,0 +1,20 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://teploy.github.io/contracts/schema/preview-state.schema.json", + "title": "preview identity eras (C06): canonical -p-, legacy slug, ambiguous", + "oneOf": [ + {"$comment": "canonical era", "type": "object", + "required": ["id", "era", "app", "branch"], + "properties": { + "id": {"type": "string", "pattern": "^[a-z0-9][a-z0-9-]*-p-[a-f0-9]{8}$"}, + "era": {"const": "canonical"}, + "app": {"type": "string"}, + "branch": {"type": "string"}}}, + {"$comment": "legacy slug-keyed era; readable, adoptable when unambiguous", "type": "object", + "required": ["id", "era", "app"], + "properties": { + "id": {"type": "string", "pattern": "^[a-z0-9][a-z0-9-]*-p-[a-z0-9][a-z0-9-]*$"}, + "era": {"const": "legacy"}, + "app": {"type": "string"}}} + ] +} diff --git a/contracts/schema/release-record.schema.json b/contracts/schema/release-record.schema.json new file mode 100644 index 0000000..1d1b686 --- /dev/null +++ b/contracts/schema/release-record.schema.json @@ -0,0 +1,22 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://teploy.github.io/contracts/schema/release-record.schema.json", + "title": "releasemeta Record (F14; releasemeta.go Record)", + "type": "object", + "required": ["schema_version", "app", "hash", "created_at", "deployment_type", "ingress_mode"], + "properties": { + "schema_version": {"type": "integer", "minimum": 1}, + "app": {"type": "string", "minLength": 1}, + "hash": {"type": "string", "pattern": "^[a-f0-9]{7}(\\.[a-f0-9]{16})?$"}, + "created_at": {"type": "string", "format": "date-time"}, + "backfilled": {"type": "boolean"}, + "generation": {"type": "integer", "minimum": 0}, + "deployment_type": {"type": "string", "enum": ["container", "static"]}, + "ingress_mode": {"type": "string"}, + "domain": {"type": "string"}, + "image_ref": {"type": "string"}, + "image_digest": {"type": "string"}, + "manifest_sha256": {"type": "string"}, + "provenance": {"type": ["object", "null"]} + } +} diff --git a/contracts/schema/server-status-envelope.schema.json b/contracts/schema/server-status-envelope.schema.json new file mode 100644 index 0000000..09a9a17 --- /dev/null +++ b/contracts/schema/server-status-envelope.schema.json @@ -0,0 +1,77 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://teploy.github.io/contracts/schema/server-status-envelope.schema.json", + "title": "server status --json envelope (MI 1, appStatusDTO root)", + "allOf": [ + { + "type": "object", + "required": [ + "app", + "domain", + "type", + "ingress", + "current_release", + "previous_release", + "containers", + "processes", + "lock", + "maintenance", + "observed_at", + "errors", + "machine_interface" + ], + "properties": { + "app": { + "type": "string" + }, + "domain": { + "type": "string" + }, + "type": { + "type": "string" + }, + "ingress": { + "type": "string" + }, + "current_release": { + "$ref": "#/$defs/release" + }, + "previous_release": { + "$ref": "#/$defs/release" + }, + "containers": { + "type": "array", + "items": { + "$ref": "#/$defs/container" + } + }, + "processes": { + "type": "array" + }, + "lock": { + "type": [ + "object", + "null" + ] + }, + "maintenance": { + "type": "boolean" + }, + "observed_at": { + "type": "string", + "format": "date-time" + }, + "errors": { + "type": "array", + "items": { + "$ref": "#/$defs/machineError" + } + }, + "machine_interface": { + "type": "integer", + "minimum": 1 + } + } + } + ] +} diff --git a/contracts/schema/version-handshake.schema.json b/contracts/schema/version-handshake.schema.json new file mode 100644 index 0000000..6703587 --- /dev/null +++ b/contracts/schema/version-handshake.schema.json @@ -0,0 +1,17 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://teploy.github.io/contracts/schema/version-handshake.schema.json", + "title": "teploy version --json handshake (X02 S1, MI 1)", + "type": "object", + "required": ["version", "machine_interface", "capabilities"], + "additionalProperties": false, + "properties": { + "version": {"type": "string", "minLength": 1}, + "machine_interface": {"type": "integer", "minimum": 1, "maximum": 1}, + "capabilities": { + "type": "array", + "items": {"type": "string", "minLength": 1}, + "uniqueItems": true + } + } +} diff --git a/internal/cli/contracts_golden_test.go b/internal/cli/contracts_golden_test.go new file mode 100644 index 0000000..d03e4ed --- /dev/null +++ b/internal/cli/contracts_golden_test.go @@ -0,0 +1,138 @@ +package cli + +// X02 S2 contracts corpus generator (contracts/ skeleton + first goldens). +// Regenerates contracts/fixtures/ from the REAL encoders and fails on any +// diff against the committed corpus - the TestPreviewIDGolden discipline +// generalized to the machine interface. Set TEPLOY_UPDATE_CONTRACTS=1 to +// rewrite the corpus after a deliberate contract change, then commit the +// diff together with the code that caused it and bump MANIFEST.md. + +import ( + "bytes" + "encoding/json" + "os" + "path/filepath" + "testing" + "time" + + "github.com/useteploy/teploy/internal/releasemeta" +) + +const contractsDir = "../../contracts" + +func writeFixture(t *testing.T, path string, v any) { + t.Helper() + var buf bytes.Buffer + enc := json.NewEncoder(&buf) + enc.SetIndent("", " ") + if err := enc.Encode(v); err != nil { + t.Fatalf("encode %s: %v", path, err) + } + full := filepath.Join(contractsDir, "fixtures", path) + if os.Getenv("TEPLOY_UPDATE_CONTRACTS") == "1" { + if err := os.MkdirAll(filepath.Dir(full), 0o755); err != nil { + t.Fatalf("mkdir %s: %v", full, err) + } + if err := os.WriteFile(full, buf.Bytes(), 0o644); err != nil { + t.Fatalf("write %s: %v", full, err) + } + return + } + want, err := os.ReadFile(full) + if err != nil { + t.Fatalf("fixture %s missing (run with TEPLOY_UPDATE_CONTRACTS=1 to seed): %v", path, err) + } + if !bytes.Equal(bytes.TrimRight(want, "\n"), bytes.TrimRight(buf.Bytes(), "\n")) { + t.Errorf("fixture %s drifted from the committed corpus - regenerate deliberately (TEPLOY_UPDATE_CONTRACTS=1), commit the diff WITH the code change, and bump contracts/MANIFEST.md", path) + } +} + +// TestContractsVersionHandshakeGolden drives the REAL writeVersion encoder. +func TestContractsVersionHandshakeGolden(t *testing.T) { + var buf bytes.Buffer + if err := writeVersion(&buf, "v0.0.0-contracts", true); err != nil { + t.Fatalf("writeVersion: %v", err) + } + var v any + if err := json.Unmarshal(buf.Bytes(), &v); err != nil { + t.Fatalf("unmarshal: %v", err) + } + writeFixture(t, "version-handshake/valid/mi1.json", v) +} + +// TestContractsAppListEnvelopeGolden emits an appListDTO with one +// representative app through the same json tags the command marshals. +// Offline stand-in: the DTO values are constructed, the ENCODER is real. +func TestContractsAppListEnvelopeGolden(t *testing.T) { + ts := time.Date(2026, 9, 23, 12, 0, 0, 0, time.UTC) + writeFixture(t, "app-list-envelope/valid/mi1.json", appListDTO{ + MachineInterface: MachineInterface, + Host: "srv.example.com", + Apps: []appStatusDTO{{ + App: "myapp", Domain: "myapp.example.com", Type: "container", + Ingress: "caddy", CurrentRelease: releaseStatusDTO{Version: "3", Ports: []int{3000}}, + Containers: []containerDTO{{ID: "9f31c02", Name: "myapp-web-3", Image: "nginx:1.27", State: "running", Status: "Up 4 minutes", CreatedAt: "2026-09-23T11:55:00Z", Process: "web", Version: "3"}}, + Lock: nil, ObservedAt: ts, Errors: nil, + }}, + ObservedAt: ts, + }) + + // Legacy: pre-MI envelope (no machine_interface field) - the shape a + // v0.1.36-or-older CLI emitted. Consumers must treat missing-MI as + // legacy, not as MI 0. + var legacy map[string]any + raw, err := json.Marshal(appListDTO{ + Host: "srv.example.com", Apps: []appStatusDTO{}, ObservedAt: ts, + }) + if err != nil { + t.Fatalf("marshal legacy: %v", err) + } + if err := json.Unmarshal(raw, &legacy); err != nil { + t.Fatalf("unmarshal legacy: %v", err) + } + delete(legacy, "machine_interface") + writeFixture(t, "app-list-envelope/legacy/pre-mi.json", legacy) +} + +// TestContractsErrorEnvelopeGolden pins the two wired error classes. +func TestContractsErrorEnvelopeGolden(t *testing.T) { + writeFixture(t, "error-envelope/valid/config-invalid.json", machineErrorEnvelope{ + MachineInterface: MachineInterface, Code: "config-invalid", + Message: "invalid teploy configuration", Detail: "teploy.yml: services.0.name: required", + }) + writeFixture(t, "error-envelope/valid/internal.json", machineErrorEnvelope{ + MachineInterface: MachineInterface, Code: "internal", + Message: "command failed", Detail: "dial tcp: connection refused", + }) + // Invalid: a code outside the taxonomy (schema enum must reject). + writeFixture(t, "error-envelope/invalid/unknown-code.json", machineErrorEnvelope{ + MachineInterface: MachineInterface, Code: "kaboom", Message: "x", + }) +} + +// TestContractsReleaseRecordGolden pins the releasemeta Record shape. +func TestContractsReleaseRecordGolden(t *testing.T) { + ts := time.Date(2026, 9, 23, 12, 0, 0, 0, time.UTC) + writeFixture(t, "release-record/valid/container.json", releasemeta.Record{ + SchemaVersion: 1, App: "myapp", Hash: "abc1234.deadb17ecafef00d", CreatedAt: ts, + DeploymentType: "container", IngressMode: "caddy", Domain: "myapp.example.com", + ImageRef: "nginx:1.27", ImageDigest: "sha256:0000", ManifestSHA256: "sha256:beef", + }) +} + +// TestContractsAttemptNameGolden pins the attempt-name grammar class via +// representative strings (the schema pattern is the contract; these are the +// examples a consumer tests against). +func TestContractsAttemptNameGolden(t *testing.T) { + writeFixture(t, "attempt-name/valid/examples.json", []string{ + "abc1234.deadb17ecafef00d", + "9f31c02.0123456789abcdef", + }) + writeFixture(t, "attempt-name/invalid/examples.json", []string{ + "deadb17ecafef00d", // missing the hash half + "ABC1234.deadb17ecafef00d", // uppercase + "abc1234.DeadB17eCafef00d", // uppercase hex half + "abc1234.deadb17ecafef00", // 15 hex chars + "../escape.attempt0000000", // path characters + }) +}