diff --git a/AUDIT_OPEN.md b/AUDIT_OPEN.md index 50dc452..13a1e13 100644 --- a/AUDIT_OPEN.md +++ b/AUDIT_OPEN.md @@ -750,7 +750,9 @@ on success. persistence-path discovery (dir/dbfilename assumptions). - T40 — NEW deferral: a versioned whole-app disaster-recovery bundle (release records, secret stores + age identity, TLS references) is a - product decision; today's archives are data-only by design. + product decision; today's archives are data-only by design. RESOLVED + 2026-09-23 (see the C07 DR-bundle slices at the bottom): the versioned + whole-app bundle landed as the `teploy dr` family. - T42 — A30: errno-aware confirmed-missing reads (test -e folds EACCES into absence); needs the structured executor result. - T43 — A30: local/remote executor output semantics (stdout/stderr split, @@ -2321,3 +2323,230 @@ pin run both clean; gofmt clean on touched files (pre-existing strays untouched). Coordinated consumer: teploy-dash branch decodes both shapes (envelope ≤ MI 2 and the legacy bare map) behind MaxSupportedMachineInterface 2; dash CI pin moves to this corpus. +||||||| 78d07d4 +## Programme slice (2026-09-23) — C07: recoverable applications (DR bundles) + +Four commits on branch c07-recovery (base `194130c`-era main): the +schema-versioned whole-app DR bundle, the isolated restore + explicit +cutover, the `teploy dr` command family, and a live-fixture hardening +pass. Closes the T40 product decision (above): `teploy backup` stays the +DATA-ONLY family; `teploy dr` is the whole-application bundle. + +- **Bundle (slice 1, `0867359`)** — manifest carries app state (verbatim + state.json), release records, the applied app manifest, secret + REFERENCES by default (encrypted material only on explicit + --include-secrets, the age key only on --include-age-key), + routing/TLS references, and per-snapshot consistency records. + Engine-aware planning is built on the EXISTING backup internals + (planAccessoryDump extracted from AccessoryBackup's inline switch, + shared restore-command builders, promoteStaged factored out of + extractToStagingThenPromote). Consistency honesty: engine dumps + record engine-consistent, redis engine-snapshot (BGSAVE-acked), + generic tars crash-consistent with an explicit note; /data engine + detection matches EXACTLY only (substring matching misclassified + /app/data as redis — caught by test). Stores: S3 (manifest uploaded + LAST as the completeness marker) or a plain server directory + (offline bundles). +- **Restore + cutover (slice 2, `a3feb8a`)** — RestoreBundleIsolated + lands in /var/tmp/teploy-dr staging (pattern-guarded wipe), ordering + pinned by tests: read-only preflight (references-mode missing keys + abort BEFORE any staging/download/docker), gzip -t integrity on every + member, decrypt proof for encrypted bundles, scratch-engine + validation (throwaway postgres/mysql/mongo/redis containers + the + recorded app image), then the receipt with MEASURED RPO (restore + start minus bundle creation) and RTO (restore+validation wall time). + CutoverBundle is the only mutating step: requires a validated staged + receipt (ErrValidationRequired), re-runs the secrets preflight, + cross-checks bundle accessories/volumes against the restore-time + teploy.yml, and under the app fence stops live containers, preserves + pre-existing engine data as named recovery copies, restores dumps + into fresh accessories, promotes volumes through promoteStaged, and + installs state/records/secrets LAST. +- **CLI (slice 3, `2fcd3ae`)** — `teploy dr create|list|show|restore| + cutover`; restore prints the receipt (JSON with --json) and exits + non-zero when validation failed; cutover refuses without a validated + staged restore. +- **Hardening (slice 4, this branch's tail)** — driven by the live + fixture, not speculation: AppRun derivation (the app check boots the + deployed image WITH the newest release record's command — without + argv most images exit instantly and prove nothing); per-volume + move-aside at cutover (env/credential files stay for the fresh + accessory) with a root-container fallback for engine-chowned data + dirs a non-root deploy user cannot rename (the reconcileDataOwnership + pattern), and an existence guard so a configured volume key with + nothing on disk never aborts the cutover; DirBundleStore creates + member SUBDIRECTORIES before copy; the volumes workspace dir exists + before tar writes into it. + +**Live evidence** — `internal/backup/dr_integration_test.go` +(`//go:build integration`, the C01/C03 fixture contract: +TEPLOY_FAULT_HOST/USER/KEY, skips cleanly when unset): against colima +(docker 29.5.2, real postgres:16-alpine, real gzip/tar, zero mocks) — +full round trip (create from a live app with real rows, isolated +restore validates data+app checks with measured RPO/RTO, /deployments +proven BYTE-IDENTICAL after the restore, cutover lands the 3 rows in +the recreated engine, replaces the drifted volume while preserving the +pre-cutover copy in a recovery dir, reinstalls state.json), corrupt +bundle refused before any engine boots, injected mid-promotion copy +failure rolls the live volume back to exactly its originals with no +leftover recovery dirs, missing references-mode key aborts before +staging with a clean tree. All four PASS (2026-09-24 run). + +**Acceptance pins** — fresh-host restore checks + RPO/RTO receipt: +TestRestoreBundleIsolated_ReceiptRPORTO (mock, injected clocks) + +TestDRIntegration_BundleRoundTrip (live). Injected copy failure +preserves originals: TestCutover_InjectedCopyFailurePreservesOriginals ++ TestDRIntegration_CutoverCopyFailurePreservesOriginals. Injected +accessory start failure aborts before promotion/state install and +restarts stopped containers: +TestCutover_InjectedStartFailurePreservesOriginals (mock-level only — +the live battery does not inject a start failure). Missing keys fail +before mutation: TestRestoreBundleIsolated_MissingSecretKeysFailBeforeMutation ++ TestCutover_MissingSecretKeysRefusedBeforeMutation + +TestDRIntegration_MissingKeysFailBeforeMutation. + +**C07 remainder (explicit):** scheduled DR bundle creation + bundle +retention/pruning policy (bundles accumulate in the store today); the +app check validates the image boots on the staged volumes with the +recorded command, not the full app env/secret surface or health gate +(cutover's NextStep is an explicit `teploy deploy`); per-engine +transactional consistency at cutover stays the A42/T35 register item, +the host-tar extractor the A43/T36 item, and restore-time writer +quiescence the A38/T34 item (the ISOLATED restore is read-only against +the source; cutover itself runs stopped-under-lock); multi-server +(fleet) apps are out of scope (single-target state model, like the +rest of the DR surface); Dash surfacing of the receipts. + +Gates: `go build ./...` clean; `go vet ./...` clean; +`go vet -tags integration ./internal/backup` clean; +`go test ./... -count=1` all 26 packages ok; gofmt clean on touched +files (pre-existing strays elsewhere left alone); integration battery +4/4 PASS against the live fixture. +||||||| 78d07d4 +## Programme slice (2026-09-23) — C08: transport, secrets and host lifecycle + +Five commits on branch `c08-transport-secrets` (base: `7e91302` → `a340bfb` +landed by the prior session, C08-5 + the join fix landed by this one). +Spec: TEPLOY_PRODUCT_EXCELLENCE_PROGRAMME_2026-09-21.md, "C08". + +**Landed (prior session):** + +- **C08-1 — structured executor result.** `internal/ssh.Result`: + bounded stdout/stderr (1 MiB/stream, Truncated flag), ExitCode 0..255 + or -1, TimedOut/Canceled as distinct flags, Err only when the command + did not complete (a clean non-zero exit is a result). Native capture + for RemoteExecutor (separate channel streams, exit-status request), + LocalExecutor, MockExecutor; generic fallback for others. Call sites + stopped parsing failure text: doctor compat absence is exit 127 (a + non-127 "not found" text no longer reads as absence — pinned), + doctor registry classifies on the tool's stderr, registry list + distinguishes confirmed file absence from a read failure (a read + failure can no longer become "No registries configured" — pinned). +- **C08-2 — bounded remote command lifetime.** ConnectConfig + .CommandTimeout bounds every command on the connection when the + caller's context carries no earlier deadline; expiry surfaces as + TimedOut. Pinned against an in-process x/crypto/ssh server (new + fixture `sshtest_test.go`): a never-answering command dies at the + deadline with the connection still usable, cancellation and timeout + stay distinguishable, and the local process-group kill (A28) is + extended to the structured path. +- **C08-3 — stdin/private-file secret transport on the remaining + argv-exposed paths.** BAO_TOKEN and kv values ride stdin + (`docker exec -i ... read -r tok; exec bao ...`, `kv put -` + with JSON on the pipe); db admin passwords and role SQL the same + (`write -`); the policy write dropped its base64 argv detour; + registry login streams the password over the session pipe; the setup + su path feeds the root password over stdin (no /tmp script artifact); + mysql/mariadb backup+restore stage MYSQL_PWD in a 0600 env-file + consumed by `docker exec --env-file`, removed by name on the + keep-workspace-on-failure path (keep the SQL, never the secret); + setup's VPN join staged the key in a 0600 file read into + TS_AUTHKEY/NB_SETUP_KEY by an rm-then-exec shell. MockExecutor + records stdin payloads (Inputs) so every pin asserts BOTH halves: + secret absent from every recorded command, secret present on the + pipe. This closes the F22 deferral register items (BAO_TOKEN / + MYSQL_PWD docker-exec argv). +- **C08-4 — concurrent-safe TOFU + rename-vs-change.** Verify+enroll + under an exclusive flock on `.teploy-lock` (flock, not + the file itself, so stock ssh(1) never interacts with our lock; + never unlinked — closes the unlinked-inode race; non-Unix builds get + a documented O_EXCL fallback with bounded stale-lock takeover); the + database is re-read FRESH under the lock (the old callback captured + it at Connect time — the actual race); enrollment is temp+fsync+ + rename (a crash can no longer leave a torn line that locks out every + future connection); an unknown host presenting a key trusted under a + DIFFERENT name is a rename (enrolled with a note), an unknown key for + a known host is an identity change (fails closed, file untouched). + Pinned: 16 concurrent first connects → exactly one enrollment and a + parseable file; concurrent distinct hosts lose nothing; rename vs + change; two full Connect()s against the in-process server with one + shared fresh $HOME. + +**Landed (this session):** + +- **C08-5 — resumable setup: preflight, stages, connection recovery.** + `setupServer` is an ordered list of named `setupStage`s, each + check-then-act idempotent, each carrying the affected-resource + description the preflight prints BEFORE anything runs (the + affected-resource list the spec requires); failures name the stage. + The password-path authorized_keys install is guarded (`grep -qF || + echo >>`) so an interrupted-and-rerun setup cannot stack duplicates. + `internal/ssh`'s `ReconnectingExecutor` redials on TRANSPORT-class + failure only (no exit status, not cancellation) with a bounded budget + (3, backoff 500ms→2s) and retries exactly the dead invocation; + ran-and-failed commands are never retried; RunStream refuses to + retry once bytes reached the caller (names the byte count instead — + a retry would duplicate output); RunInput/Upload buffer their payload + so the redialed attempt resends it whole (the first attempt consumed + the reader); a native runDetailed delegation keeps + RunInputDetailed's structured fields intact through the wrapper (the + su path's TEPLOY_SUDO_OK classification depends on stdout). runSetup + routes the whole flow (provisioning, hardening, network join, VPN + reconnection) through the wrapper. Pins: interruption at the network + stage fails naming the stage; the re-run against the same + partially-provisioned server completes skipping everything done (no + installer script, no apt install, no docker run, no Caddyfile + rewrite); a one-shot transport death mid-flow recovers and completes + (the dead command retried exactly once); recovery classification, + budget exhaustion, cancellation, stream/output/stdin fidelity, and + Close semantics each pinned separately. +- **Network-join argv exposure closed (found by this session's + inventory).** `teploy network join` (the provider Join + methods) still put the mesh credential in the detached command line — + the same class C08-3 fixed for setup's join, missed because it lives + in internal/network. The staged-file transport moved to the network + package (StageJoinCredential + EnvVarJoinShell) and both entry + points (setup's joinVPNMesh and all three provider Joins) share it; + the three tests that pinned the old `--authkey=`/`--setup-key` argv + shape now pin its absence plus the env-var/file shape. + +**Secrets-argv inventory (this session, after the fixes):** age store, +env/kv/template --var-stdin, openbao seal env-file, ShipAudit token, +generic docker exec transport — clean (as C08-3 recorded). NEWLY +REVIEWED: backup_alert's 0700 alert script carries the HMAC secret in +the file, not argv — private-file channel with documented trust +boundary, accepted. accessory.go's scheduled-backup crontab line embeds +S3 creds — root-only-readable crontab, documented same-trust-class as +~/.aws/credentials, accepted. REMAINING (recorded, not fixed here): +`S3Config.AWS` (internal/backup/backup.go) prefixes +AWS_ACCESS_KEY_ID/AWS_SECRET_ACCESS_KEY onto the aws invocation — the +session shell's argv for the life of each aws call. That file is +backup/restore internals, owned by the C07 lane; the fix (0600 env +file + `.` sourcing, or aws config-file staging, with the same +explicit cleanup semantics as the mysql credential file) belongs there. + +**C08 remainder (explicit):** the S3Config.AWS argv exposure above +(C07-owned); RunStream recovery is byte-count-guarded, not +resumption-position-aware (a long streamed output that dies mid-stream +still loses the stream — the operator re-runs; a resumable channel is +new transport surface, not a wrapper concern); CommandTimeout defaults +to 0 (caller-controlled) everywhere except where callers opt in — +sweeping the remaining unbounded long-running calls is C09's +intervention/ergonomics surface; hardening's preflight granularity is +per-function (Harden's own steps are not individually stage-listed). + +Gates: `go build ./...` && `go vet ./...` clean; `go test ./... -count=1` +26/26 packages ok; `-race` on internal/ssh, internal/cli, +internal/network (touched this session) plus internal/backup and +internal/openbao ok; gofmt clean on touched files (pre-existing strays +untouched). diff --git a/internal/backup/backup.go b/internal/backup/backup.go index c6b5f44..8ac2432 100644 --- a/internal/backup/backup.go +++ b/internal/backup/backup.go @@ -450,6 +450,28 @@ func extractToStagingThenPromote(ctx context.Context, exec ssh.Executor, archive } } + recoveryDir, err := promoteStaged(ctx, exec, stageDir, liveDir, out, stageDir, archivePath) + if err != nil { + // A recovery-incomplete failure retains its artifacts INSIDE the + // staged tree — the caller's cleanup must not delete what the + // error just described as kept (TCL-43). + if !errors.Is(err, errRecoveryIncomplete) { + cleanup() + } + return recoveryDir, err + } + cleanup() + return recoveryDir, nil +} + +// promoteStaged copies an already-extracted staged tree over liveDir's +// contents using the two-phase recovery discipline (see +// extractToStagingThenPromote): originals are moved aside first, so a +// failure at ANY point is recoverable and never destroys the only copy. +// retainedPaths name the staged tree (and archive) that a +// recovery-incomplete error must tell the operator about; the CALLER owns +// their cleanup. Shared by volume-archive restore and the DR cutover path. +func promoteStaged(ctx context.Context, exec ssh.Executor, stageDir, liveDir string, out io.Writer, retainedPaths ...string) (string, error) { fmt.Fprintf(out, "Restoring to %s...\n", liveDir) if _, err := exec.Run(ctx, "mkdir -p "+ssh.ShellQuote(liveDir)); err != nil { return "", fmt.Errorf("preparing %s: %w", liveDir, err) @@ -473,9 +495,8 @@ func extractToStagingThenPromote(ctx context.Context, exec ssh.Executor, archive ssh.ShellQuote(recoveryDir), ssh.ShellQuote(liveDir)) if _, rbErr := exec.Run(context.WithoutCancel(ctx), moveBack); rbErr != nil { return recoveryDir, fmt.Errorf("%w: moving current contents aside for %s: %v — original entries preserved split across %s and %s, staged restore kept in %s", - errRecoveryIncomplete, liveDir, err, liveDir, recoveryDir, stageDir) + errRecoveryIncomplete, liveDir, err, liveDir, recoveryDir, strings.Join(retainedPaths, ", ")) } - cleanup() exec.Run(context.WithoutCancel(ctx), "rm -rf "+ssh.ShellQuote(recoveryDir)) return "", fmt.Errorf("moving current contents aside for %s: %w — previous contents restored, live directory untouched", liveDir, err) @@ -490,15 +511,13 @@ func extractToStagingThenPromote(ctx context.Context, exec ssh.Executor, archive ssh.ShellQuote(liveDir), ssh.ShellQuote(recoveryDir), ssh.ShellQuote(liveDir)) if _, rbErr := exec.Run(context.WithoutCancel(ctx), rollbackCmd); rbErr != nil { return recoveryDir, fmt.Errorf("%w: promoting staged restore into %s: %v — rollback failed too; previous contents kept in %s, staged restore kept in %s", - errRecoveryIncomplete, liveDir, err, recoveryDir, stageDir) + errRecoveryIncomplete, liveDir, err, recoveryDir, strings.Join(retainedPaths, ", ")) } - cleanup() exec.Run(context.WithoutCancel(ctx), "rm -rf "+ssh.ShellQuote(recoveryDir)) return "", fmt.Errorf("promoting staged restore into %s: %w — previous contents restored, staged restore discarded", liveDir, err) } - cleanup() return recoveryDir, nil } @@ -541,7 +560,6 @@ func (c *Client) AccessoryBackup(ctx context.Context, app, name, image string, e return err } containerName := app + "-" + name - qContainer := ssh.ShellQuote(containerName) runOut, err := c.exec.Run(ctx, "umask 077; mktemp -d /tmp/teploy-backup.XXXXXXXX") if err != nil { @@ -552,96 +570,18 @@ func (c *Client) AccessoryBackup(ctx context.Context, app, name, image string, e c.exec.Run(context.WithoutCancel(ctx), "rm -rf "+ssh.ShellQuote(workDir)) } - dumpPath := workDir + "/dump.out.gz" - // dumpTmp is redirected into with `>`, not piped into gzip: a shell - // pipeline's exit status is its LAST command's (gzip, which "succeeds" - // compressing an empty stream even when pg_dump/mysqldump errored to - // stderr) — `| gzip > path` would silently swallow a real dump - // failure. Confirmed live: a wrong db name (see postgresDBAndUser) - // produced a 20-byte gzip of nothing while the old `| gzip` version of - // this command reported "Backup complete". Redirecting to a plain - // file with `>` preserves the dump command's own exit code, which - // c.exec.Run already surfaces (with captured stderr) as a real error. - dumpTmp := workDir + "/dump.out" - var dumpCmd string - s3Key := "" - switch { - case isDBType(image, "postgres"): - db, user := postgresDBAndUser(app, env) - s3Key = fmt.Sprintf("s3://%s/%s/accessories/%s/%s.sql.gz", s3.Bucket, app, name, timestamp) - dumpCmd = fmt.Sprintf("docker exec %s pg_dump -U %s %s > %s && gzip -c %s > %s", - qContainer, ssh.ShellQuote(user), ssh.ShellQuote(db), ssh.ShellQuote(dumpTmp), ssh.ShellQuote(dumpTmp), ssh.ShellQuote(dumpPath)) - case isDBType(image, "mysql"), isDBType(image, "mariadb"): - db := mysqlDB(app, env) - s3Key = fmt.Sprintf("s3://%s/%s/accessories/%s/%s.sql.gz", s3.Bucket, app, name, timestamp) - // Root password via MYSQL_PWD container env, never a command-line - // flag: mysqldump/mysql argv is visible in `ps` inside the - // container. Absent = current behavior (passwordless root). - execEnv := "" - if pwd := mysqlRootPassword(env); pwd != "" { - execEnv = " -e MYSQL_PWD=" + ssh.ShellQuote(pwd) - } - dumpCmd = fmt.Sprintf("docker exec%s %s mysqldump -u root %s > %s && gzip -c %s > %s", - execEnv, qContainer, ssh.ShellQuote(db), ssh.ShellQuote(dumpTmp), ssh.ShellQuote(dumpTmp), ssh.ShellQuote(dumpPath)) - case isDBType(image, "mongo"): - s3Key = fmt.Sprintf("s3://%s/%s/accessories/%s/%s.archive.gz", s3.Bucket, app, name, timestamp) - dumpCmd = fmt.Sprintf("docker exec %s mongodump --archive --gzip > %s", qContainer, ssh.ShellQuote(dumpPath)) - case isDBType(image, "redis"): - s3Key = fmt.Sprintf("s3://%s/%s/accessories/%s/%s.rdb.gz", s3.Bucket, app, name, timestamp) - // Redis: trigger bgsave, wait for an ACKNOWLEDGED new save, then - // copy dump.rdb. The old one-liner polled LASTSAVE in a loop whose - // exhaustion still exited 0 (the last `sleep` won), so docker cp - // ran with the PREVIOUS dump and uploaded it as a fresh backup - // (audit F36). This script fails closed: BGSAVE refusal (other - // than an already-running save, whose completion still moves - // LASTSAVE), a poll timeout, or a failed copy all abort before - // anything is uploaded. - redisTmp := workDir + "/dump.rdb" - dumpCmd = strings.Join([]string{ - "set -eu", - // AOF-enabled Redis persists to the append-only file; backing - // up only dump.rdb captures a stale or empty dataset. Fail - // closed rather than uploading a wrong-point-in-time artifact - // (TCL-42). - // AOF gate (A40): the old `config get appendonly | tail -n 1` - // pipeline masked a failed docker exec (empty output fell - // through as "not yes") and only refused on a substring - // match. The reply must be a proven `appendonly no` — - // anything else (auth error, empty, unexpected) refuses. - fmt.Sprintf("aof=$(docker exec %s redis-cli --raw config get appendonly)", qContainer), - `set -- $aof`, - `if [ "${1:-}" != appendonly ] || [ "${2:-}" != no ]; then echo 'cannot confirm redis appendonly=no (got: '"$aof"') — teploy backup captures dump.rdb only; refusing' >&2; exit 1; fi`, - fmt.Sprintf("ls=$(docker exec %s redis-cli lastsave)", qContainer), - fmt.Sprintf("bgs=$(docker exec %s redis-cli bgsave 2>&1) || true", qContainer), - `case "$bgs" in *ERR*) case "$bgs" in *"in progress"*) ;; *) printf 'redis BGSAVE failed: %s\n' "$bgs" >&2; exit 1;; esac;; esac`, - "saved=no; i=0", - `while [ "$i" -lt 60 ]; do`, - fmt.Sprintf("cur=$(docker exec %s redis-cli lastsave)", qContainer), - `if [ "$cur" != "$ls" ]; then saved=yes; break; fi`, - "sleep 1; i=$((i+1))", - "done", - `if [ "$saved" != yes ]; then echo 'timed out waiting for Redis BGSAVE to complete' >&2; exit 1; fi`, - fmt.Sprintf("docker cp %s:/data/dump.rdb %s", qContainer, ssh.ShellQuote(redisTmp)), - fmt.Sprintf("gzip -c %s > %s", ssh.ShellQuote(redisTmp), ssh.ShellQuote(dumpPath)), - fmt.Sprintf("rm -f %s", ssh.ShellQuote(redisTmp)), - }, "\n") - default: - // Generic: tar the volume directory. For a LIVE engine (nucleus and - // anything else that mutates its files during the read) this is a - // crash-consistent snapshot: GNU tar exits 1 with "file changed" / - // "file shrank" warnings when a WAL rotates or checkpoints mid-read. - // That shape is exactly what crash recovery is built for (torn-tail - // truncation + CRC skip), so tolerate exit 1 — real failures - // (unreadable dir, ENOSPC) exit 2. `accessory verify-backup` is the - // correctness gate: it boots the archive in a scratch container. - accDir := fmt.Sprintf("%s/%s/accessories/%s", deploymentsDir, app, name) - dumpPath = workDir + "/dump.tar.gz" - s3Key = fmt.Sprintf("s3://%s/%s/accessories/%s/%s.tar.gz", s3.Bucket, app, name, timestamp) - dumpCmd = fmt.Sprintf("tar -czf %s -C %s . || [ $? -eq 1 ]", ssh.ShellQuote(dumpPath), ssh.ShellQuote(accDir)) + // Consistency classification + the dump command come from the shared + // planner (also used by the DR bundle path) so the two snapshot paths + // can never drift apart. + plan := planAccessoryDump(app, name, image, env, workDir) + dumpPath := plan.artifactPath + s3Key := fmt.Sprintf("s3://%s/%s/accessories/%s/%s%s", s3.Bucket, app, name, timestamp, plan.ext) + if err := plan.stage(ctx, c.exec); err != nil { + cleanup() + return fmt.Errorf("staging the mysql credential file for %s (no dump was run): %w", name, err) } - fmt.Fprintf(c.out, "Backing up %s...\n", containerName) - if _, err := c.exec.Run(ctx, dumpCmd); err != nil { + if _, err := c.exec.Run(ctx, plan.cmd); err != nil { // Never leave partial backups around: they're disk-fillers at best, // restore-bait at worst. cleanup() @@ -665,9 +605,6 @@ func (c *Client) AccessoryRestore(ctx context.Context, app, name, image, date st return err } - containerName := app + "-" + name - qContainer := ssh.ShellQuote(containerName) - // All restore scratch files live under one per-invocation temp dir. The // old fixed names (/tmp/restore.sql.gz, …, and the generic branch's // /tmp/--restore-stage) were shared by every restore: two @@ -690,27 +627,30 @@ func (c *Client) AccessoryRestore(ctx context.Context, app, name, image, date st switch { case isDBType(image, "postgres"): - db, user := postgresDBAndUser(app, env) s3Key = fmt.Sprintf("s3://%s/%s/accessories/%s/%s.sql.gz", s3.Bucket, app, name, date) restorePath = tmpdir + "/restore.sql.gz" - // Decompress to a file first and feed psql via stdin redirect: a - // pipeline reports only the LAST command's status, so `gunzip | psql` - // succeeded on a corrupt archive (empty stdin) and — without - // ON_ERROR_STOP — on SQL errors too. Same convention as verify.go. - sqlPath := tmpdir + "/restore.sql" - restoreCmd = fmt.Sprintf("gunzip -c %s > %s && docker exec -i %s psql -v ON_ERROR_STOP=1 -U %s %s < %s", - ssh.ShellQuote(restorePath), ssh.ShellQuote(sqlPath), qContainer, ssh.ShellQuote(user), ssh.ShellQuote(db), ssh.ShellQuote(sqlPath)) + db, user := postgresDBAndUser(app, env) + restoreCmd = postgresRestoreCmd(app+"-"+name, user, db, restorePath, tmpdir+"/restore.sql") case isDBType(image, "mysql"), isDBType(image, "mariadb"): - db := mysqlDB(app, env) // The old branch never assigned s3Key/restorePath, so the download // and gunzip below operated on empty arguments — every MySQL/MariaDB // restore was deterministically broken (audit F31). s3Key = fmt.Sprintf("s3://%s/%s/accessories/%s/%s.sql.gz", s3.Bucket, app, name, date) restorePath = tmpdir + "/restore.sql.gz" - // Same MYSQL_PWD env injection as the backup path (see there). + qContainer := ssh.ShellQuote(app + "-" + name) + db := mysqlDB(app, env) + // Same 0600 env-file credential transport as the backup path + // (see there) — the password rides in no argv. The restore + // failure path DELIBERATELY keeps tmpdir for inspection, so the + // credential file must be removed by name there: keep the SQL, + // never the secret (C08). execEnv := "" if pwd := mysqlRootPassword(env); pwd != "" { - execEnv = " -e MYSQL_PWD=" + ssh.ShellQuote(pwd) + envFile := tmpdir + "/mysql.env" + if err := c.exec.Upload(ctx, strings.NewReader("MYSQL_PWD="+pwd+"\n"), envFile, "0600"); err != nil { + return keepTmp(fmt.Errorf("staging the mysql credential file for %s: %w", name, err)) + } + execEnv = " --env-file " + ssh.ShellQuote(envFile) } // Same pipeline-to-redirect shape as postgres (mysql itself exits // nonzero on SQL errors when reading a script, but gunzip's failure @@ -718,74 +658,28 @@ func (c *Client) AccessoryRestore(ctx context.Context, app, name, image, date st sqlPath := tmpdir + "/restore.sql" restoreCmd = fmt.Sprintf("gunzip -c %s > %s && docker exec -i%s %s mysql -u root %s < %s", ssh.ShellQuote(restorePath), ssh.ShellQuote(sqlPath), execEnv, qContainer, ssh.ShellQuote(db), ssh.ShellQuote(sqlPath)) + // The kept-on-failure scratch dir must never keep the credential. + innerKeep := keepTmp + keepTmp = func(err error) error { + c.exec.Run(context.WithoutCancel(ctx), "rm -f "+ssh.ShellQuote(tmpdir+"/mysql.env")) + return innerKeep(err) + } case isDBType(image, "mongo"): s3Key = fmt.Sprintf("s3://%s/%s/accessories/%s/%s.archive.gz", s3.Bucket, app, name, date) restorePath = tmpdir + "/restore.archive.gz" - restoreCmd = fmt.Sprintf("docker exec -i %s mongorestore --archive --gzip --drop < %s", qContainer, ssh.ShellQuote(restorePath)) + restoreCmd = mongoRestoreCmd(app+"-"+name, restorePath) case isDBType(image, "redis"): - // AccessoryBackup stores redis as .rdb.gz; without this case the - // default branch looked for a .tar.gz that doesn't exist, so redis - // restores always failed. Stop redis first so its shutdown save can't - // overwrite the snapshot we copy in, then start so it loads dump.rdb. - // - // The old `gunzip && stop && cp && start` chain could leave the - // accessory STOPPED forever when docker cp failed after a successful - // stop (audit F38). This script decompresses FIRST (no downtime while - // validating the artifact), saves the current dump, and restores + - // restarts the original on any failure after the stop. + // AccessoryBackup stores redis as .rdb.gz; without this case + // the default branch looked for a .tar.gz that doesn't exist, so + // redis restores always failed. Stop redis first so its shutdown + // save can't overwrite the snapshot we copy in, then start so it + // loads dump.rdb (full reasoning on redisRestoreScript). s3Key = fmt.Sprintf("s3://%s/%s/accessories/%s/%s.rdb.gz", s3.Bucket, app, name, date) restorePath = tmpdir + "/restore.rdb.gz" - rdbPath := tmpdir + "/restore.rdb" - oldRdb := tmpdir + "/old-dump.rdb" - // A40: the AOF gate is a Go-level preflight so a failed or - // unexpected reply (auth error, empty output, anything but a - // proven `appendonly no`) refuses BEFORE any stop or copy — the - // old `config get appendonly | tail -n 1` pipeline masked a failed - // docker exec as "not yes" and fell through to the destructive - // replacement. ("Not proven yes" is not "proven no".) - aofOut, aofErr := c.exec.Run(ctx, fmt.Sprintf("docker exec %s redis-cli --raw config get appendonly", qContainer)) - if aofErr != nil { - return keepTmp(fmt.Errorf("cannot establish the Redis persistence mode for %s: %w", containerName, aofErr)) + if err := c.redisAOFPreflight(ctx, app, name); err != nil { + return keepTmp(err) } - if aofFields := strings.Fields(aofOut); len(aofFields) != 2 || aofFields[0] != "appendonly" || aofFields[1] != "no" { - return keepTmp(fmt.Errorf("cannot confirm appendonly=no for %s (got %q) — an AOF-enabled Redis would load the append-only file on restart and teploy's dump.rdb restore would be a no-op; an explicit restore plan is required", containerName, strings.TrimSpace(aofOut))) - } - // A41 ordering + T37/T38 arming: restore_original is defined (and - // the old-dump capture attempted) AFTER the stop — a graceful redis - // shutdown writes a final RDB, so the pre-stop existence flag could - // miss data present at shutdown. The baseline copy itself - // distinguishes "no such file" (nothing to preserve) from every - // other failure, and ANY failure after the stop restarts the - // container before aborting: the old script's `set -e` exit on a - // failed docker cp left Redis stopped with no recovery attempt. - restoreCmd = strings.Join([]string{ - "set -eu", - fmt.Sprintf("gunzip -c %s > %s", ssh.ShellQuote(restorePath), ssh.ShellQuote(rdbPath)), - "had=no", - fmt.Sprintf(`restore_original() { if [ "$had" = yes ] && [ -f %s ]; then docker cp %s %s:/data/dump.rdb || true; fi; docker start %s || true; }`, - ssh.ShellQuote(oldRdb), ssh.ShellQuote(oldRdb), qContainer, qContainer), - fmt.Sprintf("docker stop %s", qContainer), - // Post-stop baseline (docker cp works on a stopped container): - // success -> had=yes; a proven not-found -> nothing to - // preserve; anything else -> restart + abort. - `cperr=$(mktemp)`, - fmt.Sprintf(`if docker cp %s:/data/dump.rdb %s 2>"$cperr"; then had=yes; elif grep -qi 'no such' "$cperr"; then had=no; else cat "$cperr" >&2; rm -f "$cperr"; restore_original; echo 'capturing the pre-restore dump failed; the container was restarted' >&2; exit 1; fi`, - qContainer, ssh.ShellQuote(oldRdb)), - `rm -f "$cperr"`, - "ok=yes", - fmt.Sprintf("docker cp %s %s:/data/dump.rdb || ok=no", ssh.ShellQuote(rdbPath), qContainer), - `if [ "$ok" != yes ]; then`, - ` restore_original`, - " echo 'redis restore failed after stopping the container; the original dump was restored when available' >&2", - " exit 1", - "fi", - fmt.Sprintf("if ! docker start %s; then", qContainer), - ` restore_original`, - " echo 'redis container failed to start after the restore; the original dump was put back — verify the accessory' >&2", - " exit 1", - "fi", - fmt.Sprintf("rm -f %s", ssh.ShellQuote(rdbPath)), - }, "\n") + restoreCmd = redisRestoreScript(app+"-"+name, restorePath, tmpdir+"/restore.rdb", tmpdir+"/old-dump.rdb") default: // Generic: extract tar to accessory directory. s3Key = fmt.Sprintf("s3://%s/%s/accessories/%s/%s.tar.gz", s3.Bucket, app, name, date) @@ -895,6 +789,246 @@ func (c *Client) ensureAWSCLI(ctx context.Context) error { // turned "registry.example:5000/postgres:16" into "registry.example" and // silently routed real databases through the generic tar backup/restore // branch. +// accessoryDumpPlan is the single source of truth for how one accessory is +// snapshotted: the engine classification, the consistency the artifact can +// honestly claim, and the exact server-side command that produces it. +// AccessoryBackup (single-accessory S3 backups) and the DR bundle path both +// build on it, so the two flows cannot drift into different dump semantics. +type accessoryDumpPlan struct { + engine string // postgres|mysql|mariadb|mongo|redis|generic + method string // pg_dump|mysqldump|mongodump|redis-bgsave|tar + consistency string // engine-consistent | crash-consistent + ext string // .sql.gz | .archive.gz | .rdb.gz | .tar.gz + // artifactPath is where the plan's command writes its artifact: + // /dump.out.gz for engine dumps, /dump.tar.gz for + // the generic tar branch. + artifactPath string + cmd string + // credential, when non-nil, is a secret the plan's command needs via a + // 0600 env-file (--env-file) rather than argv: the CALLER must upload + // it before running cmd and ensure it never survives a kept scratch + // dir (C08 transport contract). + credential *planCredential +} + +// planCredential is the staged env-file contract between the planner and +// its callers: content uploaded at path with mode 0600. +type planCredential struct { + path string + content string +} + +// stage uploads the plan's credential file (no-op when the plan carries +// none). Errors are pre-effect: no dump command has run yet. +func (p accessoryDumpPlan) stage(ctx context.Context, exec ssh.Executor) error { + if p.credential == nil { + return nil + } + return exec.Upload(ctx, strings.NewReader(p.credential.content), p.credential.path, "0600") +} + +// Consistency level labels recorded in DR bundle manifests. A raw tar of a +// live database volume is NOT a consistent backup, and these labels are how +// the code refuses to present it as one (C07). +const ( + consistencyEngineDump = "engine-consistent" // engine-native dump tooling (pg_dump, mysqldump, mongodump) + consistencySnapshot = "engine-snapshot" // engine-acknowledged point-in-time file snapshot (redis BGSAVE) + consistencyCrash = "crash-consistent" // raw file copy of a live writer; only crash recovery is guaranteed + consistencyQuiesced = "quiesced" // writer stopped before the copy (operator or teploy did it) +) + +// planAccessoryDump classifies an accessory image and builds the command that +// dumps it into workDir. The commands are byte-identical to the historical +// AccessoryBackup switch (behavior pinned by existing tests); extraction into +// a shared planner exists so the DR bundle path reuses them rather than +// forking a second backup system. +func planAccessoryDump(app, name, image string, env map[string]string, workDir string) accessoryDumpPlan { + containerName := app + "-" + name + qContainer := ssh.ShellQuote(containerName) + + // dumpTmp is redirected into with `>`, not piped into gzip: a shell + // pipeline's exit status is its LAST command's (gzip, which "succeeds" + // compressing an empty stream even when pg_dump/mysqldump errored to + // stderr) — `| gzip > path` would silently swallow a real dump + // failure. Redirecting to a plain file with `>` preserves the dump + // command's own exit code, which exec.Run already surfaces (with + // captured stderr) as a real error. + dumpPath := workDir + "/dump.out.gz" + dumpTmp := workDir + "/dump.out" + + switch { + case isDBType(image, "postgres"): + db, user := postgresDBAndUser(app, env) + return accessoryDumpPlan{ + engine: "postgres", method: "pg_dump", consistency: consistencyEngineDump, ext: ".sql.gz", + artifactPath: dumpPath, + cmd: fmt.Sprintf("docker exec %s pg_dump -U %s %s > %s && gzip -c %s > %s", + qContainer, ssh.ShellQuote(user), ssh.ShellQuote(db), ssh.ShellQuote(dumpTmp), ssh.ShellQuote(dumpTmp), ssh.ShellQuote(dumpPath)), + } + case isDBType(image, "mysql"), isDBType(image, "mariadb"): + engine := "mysql" + if isDBType(image, "mariadb") { + engine = "mariadb" + } + db := mysqlDB(app, env) + // Root password rides a 0600 env-file consumed by --env-file, never + // argv (mysqldump argv is visible in `ps` inside the container, and + // -e MYSQL_PWD puts the secret in the docker CLI's own argv on the + // host). Absent = current behavior (passwordless root). + execEnv := "" + var cred *planCredential + if pwd := mysqlRootPassword(env); pwd != "" { + envFile := workDir + "/mysql.env" + cred = &planCredential{path: envFile, content: "MYSQL_PWD=" + pwd + "\n"} + execEnv = " --env-file " + ssh.ShellQuote(envFile) + } + return accessoryDumpPlan{ + engine: engine, method: "mysqldump", consistency: consistencyEngineDump, ext: ".sql.gz", + artifactPath: dumpPath, + cmd: fmt.Sprintf("docker exec%s %s mysqldump -u root %s > %s && gzip -c %s > %s", + execEnv, qContainer, ssh.ShellQuote(db), ssh.ShellQuote(dumpTmp), ssh.ShellQuote(dumpTmp), ssh.ShellQuote(dumpPath)), + credential: cred, + } + case isDBType(image, "mongo"): + return accessoryDumpPlan{ + engine: "mongo", method: "mongodump", consistency: consistencyEngineDump, ext: ".archive.gz", + artifactPath: dumpPath, + cmd: fmt.Sprintf("docker exec %s mongodump --archive --gzip > %s", qContainer, ssh.ShellQuote(dumpPath)), + } + case isDBType(image, "redis"): + // Redis: trigger bgsave, wait for an ACKNOWLEDGED new save, then + // copy dump.rdb. The script fails closed: BGSAVE refusal (other + // than an already-running save, whose completion still moves + // LASTSAVE), a poll timeout, or a failed copy all abort before an + // artifact is produced (audit F36, TCL-42, A40 — see the history + // in git for the incidents behind each guard). + redisTmp := workDir + "/dump.rdb" + return accessoryDumpPlan{ + engine: "redis", method: "redis-bgsave", consistency: consistencySnapshot, ext: ".rdb.gz", + artifactPath: dumpPath, + cmd: strings.Join([]string{ + "set -eu", + // AOF-enabled Redis persists to the append-only file; backing + // up only dump.rdb captures a stale or empty dataset — the + // reply must be a proven `appendonly no`. + fmt.Sprintf("aof=$(docker exec %s redis-cli --raw config get appendonly)", qContainer), + `set -- $aof`, + `if [ "${1:-}" != appendonly ] || [ "${2:-}" != no ]; then echo 'cannot confirm redis appendonly=no (got: '"$aof"') — teploy backup captures dump.rdb only; refusing' >&2; exit 1; fi`, + fmt.Sprintf("ls=$(docker exec %s redis-cli lastsave)", qContainer), + fmt.Sprintf("bgs=$(docker exec %s redis-cli bgsave 2>&1) || true", qContainer), + `case "$bgs" in *ERR*) case "$bgs" in *"in progress"*) ;; *) printf 'redis BGSAVE failed: %s\n' "$bgs" >&2; exit 1;; esac;; esac`, + "saved=no; i=0", + `while [ "$i" -lt 60 ]; do`, + fmt.Sprintf("cur=$(docker exec %s redis-cli lastsave)", qContainer), + `if [ "$cur" != "$ls" ]; then saved=yes; break; fi`, + "sleep 1; i=$((i+1))", + "done", + `if [ "$saved" != yes ]; then echo 'timed out waiting for Redis BGSAVE to complete' >&2; exit 1; fi`, + fmt.Sprintf("docker cp %s:/data/dump.rdb %s", qContainer, ssh.ShellQuote(redisTmp)), + fmt.Sprintf("gzip -c %s > %s", ssh.ShellQuote(redisTmp), ssh.ShellQuote(dumpPath)), + fmt.Sprintf("rm -f %s", ssh.ShellQuote(redisTmp)), + }, "\n"), + } + default: + // Generic: tar the volume directory. For a LIVE engine (nucleus and + // anything else that mutates its files during the read) this is a + // crash-consistent snapshot: GNU tar exits 1 with "file changed" / + // "file shrank" warnings when a WAL rotates or checkpoints mid-read. + // That shape is exactly what crash recovery is built for (torn-tail + // truncation + CRC skip), so tolerate exit 1 — real failures + // (unreadable dir, ENOSPC) exit 2. The DR bundle manifest labels + // this branch crash-consistent; it is never presented as an + // engine-consistent backup (C07). + accDir := fmt.Sprintf("%s/%s/accessories/%s", deploymentsDir, app, name) + return accessoryDumpPlan{ + engine: "generic", method: "tar", consistency: consistencyCrash, ext: ".tar.gz", + artifactPath: workDir + "/dump.tar.gz", + cmd: fmt.Sprintf("tar -czf %s -C %s . || [ $? -eq 1 ]", ssh.ShellQuote(workDir+"/dump.tar.gz"), ssh.ShellQuote(accDir)), + } + } +} + +// Shared per-engine restore-command builders — the single source of truth +// for how a dump artifact lands back in an engine. AccessoryRestore +// (single-backup S3 restore) and the DR bundle paths both build on them. + +// postgresRestoreCmd decompresses a pg_dump artifact and pipes it into the +// engine container's psql with ON_ERROR_STOP. gunzip writes to a plain file +// first: a pipeline reports only the LAST command's status, so `gunzip | +// psql` "succeeds" on a corrupt archive (empty stdin). container is the +// docker container name (the live accessory or a validation scratch). +func postgresRestoreCmd(container, user, db, dumpPath, sqlPath string) string { + return fmt.Sprintf("gunzip -c %s > %s && docker exec -i %s psql -v ON_ERROR_STOP=1 -U %s %s < %s", + ssh.ShellQuote(dumpPath), ssh.ShellQuote(sqlPath), ssh.ShellQuote(container), ssh.ShellQuote(user), ssh.ShellQuote(db), ssh.ShellQuote(sqlPath)) +} + +// mysqlRestoreCmd mirrors postgresRestoreCmd for mysql/mariadb; the root +// password rides MYSQL_PWD container env, never argv (audit F22). +func mysqlRestoreCmd(container, db, pwd, dumpPath, sqlPath string) string { + execEnv := "" + if pwd != "" { + execEnv = " -e MYSQL_PWD=" + ssh.ShellQuote(pwd) + } + return fmt.Sprintf("gunzip -c %s > %s && docker exec -i%s %s mysql -u root %s < %s", + ssh.ShellQuote(dumpPath), ssh.ShellQuote(sqlPath), execEnv, ssh.ShellQuote(container), ssh.ShellQuote(db), ssh.ShellQuote(sqlPath)) +} + +// mongoRestoreCmd streams a mongodump archive into mongorestore. +func mongoRestoreCmd(container, dumpPath string) string { + return fmt.Sprintf("docker exec -i %s mongorestore --archive --gzip --drop < %s", ssh.ShellQuote(container), ssh.ShellQuote(dumpPath)) +} + +// redisRestoreScript stops the container (so its shutdown save cannot +// overwrite the snapshot), captures the current dump as a baseline, copies +// the restored one in, and restarts — restoring the original on any failure +// after the stop (audits F38/A40/A41, T37/T38; see AccessoryRestore history). +func redisRestoreScript(container, dumpPath, rdbPath, oldRdb string) string { + qContainer := ssh.ShellQuote(container) + return strings.Join([]string{ + "set -eu", + fmt.Sprintf("gunzip -c %s > %s", ssh.ShellQuote(dumpPath), ssh.ShellQuote(rdbPath)), + "had=no", + fmt.Sprintf(`restore_original() { if [ "$had" = yes ] && [ -f %s ]; then docker cp %s %s:/data/dump.rdb || true; fi; docker start %s || true; }`, + ssh.ShellQuote(oldRdb), ssh.ShellQuote(oldRdb), qContainer, qContainer), + fmt.Sprintf("docker stop %s", qContainer), + // Post-stop baseline (docker cp works on a stopped container): + // success -> had=yes; a proven not-found -> nothing to preserve; + // anything else -> restart + abort. + `cperr=$(mktemp)`, + fmt.Sprintf(`if docker cp %s:/data/dump.rdb %s 2>"$cperr"; then had=yes; elif grep -qi 'no such' "$cperr"; then had=no; else cat "$cperr" >&2; rm -f "$cperr"; restore_original; echo 'capturing the pre-restore dump failed; the container was restarted' >&2; exit 1; fi`, + qContainer, ssh.ShellQuote(oldRdb)), + `rm -f "$cperr"`, + "ok=yes", + fmt.Sprintf("docker cp %s %s:/data/dump.rdb || ok=no", ssh.ShellQuote(rdbPath), qContainer), + `if [ "$ok" != yes ]; then`, + ` restore_original`, + " echo 'redis restore failed after stopping the container; the original dump was restored when available' >&2", + " exit 1", + "fi", + fmt.Sprintf("if ! docker start %s; then", qContainer), + ` restore_original`, + " echo 'redis container failed to start after the restore; the original dump was put back — verify the accessory' >&2", + " exit 1", + "fi", + fmt.Sprintf("rm -f %s", ssh.ShellQuote(rdbPath)), + }, "\n") +} + +// redisAOFPreflight proves the accessory redis is appendonly=no before any +// stop/copy (A40): "not proven yes" is not "proven no", and an AOF-enabled +// redis would ignore a dump.rdb restore on restart. +func (c *Client) redisAOFPreflight(ctx context.Context, app, name string) error { + containerName := app + "-" + name + aofOut, aofErr := c.exec.Run(ctx, fmt.Sprintf("docker exec %s redis-cli --raw config get appendonly", ssh.ShellQuote(containerName))) + if aofErr != nil { + return fmt.Errorf("cannot establish the Redis persistence mode for %s: %w", containerName, aofErr) + } + if aofFields := strings.Fields(aofOut); len(aofFields) != 2 || aofFields[0] != "appendonly" || aofFields[1] != "no" { + return fmt.Errorf("cannot confirm appendonly=no for %s (got %q) — an AOF-enabled Redis would load the append-only file on restart and teploy's dump.rdb restore would be a no-op; an explicit restore plan is required", containerName, strings.TrimSpace(aofOut)) + } + return nil +} + func isDBType(image, dbType string) bool { return imageName(image) == dbType } @@ -951,9 +1085,10 @@ func mysqlDB(app string, env map[string]string) string { // mysqlRootPassword resolves the root password the mysql/mariadb containers // themselves honor (MYSQL_ROOT_PASSWORD, falling back to MYSQL_PASSWORD). -// Used to inject MYSQL_PWD into docker exec as container env — never as a -// command-line argument, which would expose the password in `ps` output. -// Empty means no password configured; callers keep the bare command. +// Used to stage MYSQL_PWD in a 0600 env-file consumed by `docker exec +// --env-file` — never on any argv, which would expose the password in +// the host's `ps` output. Empty means no password configured; callers +// keep the bare command. func mysqlRootPassword(env map[string]string) string { if pwd := env["MYSQL_ROOT_PASSWORD"]; pwd != "" { return pwd diff --git a/internal/backup/backup_test.go b/internal/backup/backup_test.go index 578aef4..fae7da5 100644 --- a/internal/backup/backup_test.go +++ b/internal/backup/backup_test.go @@ -296,15 +296,23 @@ func TestAccessoryBackup_MySQL_UsesRootPasswordEnv(t *testing.T) { if strings.Contains(call, "mysqldump") { dumpCmd = call } + if strings.Contains(call, "sekret") { + t.Errorf("password must appear in NO command: %s", call) + } } if dumpCmd == "" { t.Fatal("expected a mysqldump command") } - if !strings.Contains(dumpCmd, "docker exec -e MYSQL_PWD='sekret' 'myapp-mysql' mysqldump -u root 'myapp'") { - t.Errorf("password must ride as MYSQL_PWD env on docker exec, got: %s", dumpCmd) + if !strings.Contains(dumpCmd, "docker exec --env-file '/tmp/teploy-backup.abc123/mysql.env' 'myapp-mysql' mysqldump -u root 'myapp'") { + t.Errorf("password must ride via docker exec --env-file, got: %s", dumpCmd) + } + if got := string(mock.Files["/tmp/teploy-backup.abc123/mysql.env"]); got != "MYSQL_PWD=sekret\n" { + t.Errorf("credential file content = %q", got) } - if dump := dumpCmd[strings.Index(dumpCmd, "mysqldump"):]; strings.Contains(dump, "sekret") { - t.Errorf("password must not appear on the mysqldump argv: %s", dumpCmd) + for _, up := range mock.Calls { + if strings.HasPrefix(up, "UPLOAD:") && !strings.Contains(up, "mode 0600") { + t.Errorf("credential file must be uploaded 0600: %s", up) + } } } @@ -327,13 +335,19 @@ func TestAccessoryBackup_MySQL_PasswordFallbackAndAbsence(t *testing.T) { } found := false for _, call := range mock.Calls { - if strings.Contains(call, "-e MYSQL_PWD='fall'") { + if strings.Contains(call, "--env-file") { found = true } + if strings.Contains(call, "fall") { + t.Errorf("password must appear in no command: %s", call) + } } if !found { t.Errorf("MYSQL_PASSWORD must be used when MYSQL_ROOT_PASSWORD is absent, calls: %v", mock.Calls) } + if got := string(mock.Files["/tmp/teploy-backup.abc123/mysql.env"]); got != "MYSQL_PWD=fall\n" { + t.Errorf("fallback credential file content = %q", got) + } // No password configured: keep the bare command (passwordless root). mock = ssh.NewMockExecutor("1.2.3.4", @@ -356,9 +370,9 @@ func TestAccessoryBackup_MySQL_PasswordFallbackAndAbsence(t *testing.T) { } } -// The password lands inside a shell command string: it must be single-quote -// wrapped (ShellQuote), so values with spaces or quotes neither break the -// command nor escape into something executable. +// The hostile value must round-trip through the credential FILE with no +// shell exposure at all — there is no quoting surface left to get right +// because the password never enters a command string. func TestAccessoryBackup_MySQL_QuotesHostilePassword(t *testing.T) { mock := ssh.NewMockExecutor("1.2.3.4", ssh.MockCommand{Match: "which aws", Output: "/usr/bin/aws\n"}, @@ -379,14 +393,13 @@ func TestAccessoryBackup_MySQL_QuotesHostilePassword(t *testing.T) { t.Fatalf("AccessoryBackup: %v", err) } - var dumpCmd string for _, call := range mock.Calls { - if strings.Contains(call, "mysqldump") { - dumpCmd = call + if strings.Contains(call, "p@'ss") || strings.Contains(call, "word; id") { + t.Errorf("hostile password must appear in no command: %s", call) } } - if !strings.Contains(dumpCmd, "-e MYSQL_PWD='p@'\"'\"'ss word; id'") { - t.Errorf("password must be ShellQuote-wrapped, got: %s", dumpCmd) + if got := string(mock.Files["/tmp/teploy-backup.abc123/mysql.env"]); got != "MYSQL_PWD=p@'ss word; id\n" { + t.Errorf("credential file must carry the password verbatim, got %q", got) } } @@ -413,15 +426,55 @@ func TestAccessoryRestore_MySQL_UsesRootPasswordEnv(t *testing.T) { if strings.Contains(call, "mysql -u root") { restoreCmd = call } + if strings.Contains(call, "sekret") { + t.Errorf("password must appear in no command: %s", call) + } } if restoreCmd == "" { t.Fatal("expected a mysql restore command") } - if !strings.Contains(restoreCmd, "docker exec -i -e MYSQL_PWD='sekret' 'myapp-mysql' mysql -u root 'myapp'") { - t.Errorf("password must ride as MYSQL_PWD env on docker exec, got: %s", restoreCmd) + if !strings.Contains(restoreCmd, "docker exec -i --env-file '/tmp/teploy-restore.abc123/mysql.env' 'myapp-mysql' mysql -u root 'myapp'") { + t.Errorf("password must ride via docker exec --env-file, got: %s", restoreCmd) + } + if got := string(mock.Files["/tmp/teploy-restore.abc123/mysql.env"]); got != "MYSQL_PWD=sekret\n" { + t.Errorf("credential file content = %q", got) + } +} + +// TestAccessoryRestore_MySQL_FailureKeepsSQlNotSecret pins the restore +// failure semantics of the credential file: the restore workspace is +// deliberately kept for inspection on failure, but the mysql credential +// must be removed by name from what stays behind. +func TestAccessoryRestore_MySQL_FailureKeepsSQLNotSecret(t *testing.T) { + mock := ssh.NewMockExecutor("1.2.3.4", + ssh.MockCommand{Match: "which aws", Output: "/usr/bin/aws\n"}, + ssh.MockCommand{Match: "aws s3 cp", Output: "download: done\n"}, + ssh.MockCommand{Match: "mktemp -d '/tmp/teploy-restore.XXXXXX'", Output: "/tmp/teploy-restore.abc123\n"}, + // The restore command fails so keepTmp runs. + ssh.MockCommand{Match: "gunzip -c", Err: errors.New("exit status 1: gzip: invalid")}, + ssh.MockCommand{Match: "rm -f", Output: ""}, + ) + + var buf bytes.Buffer + client := NewClient(mock, &buf) + env := map[string]string{"MYSQL_ROOT_PASSWORD": "sekret"} + err := client.AccessoryRestore(context.Background(), "myapp", "mysql", "mysql:8", + "20260101-000000", env, S3Config{Bucket: "my-bucket", Region: "us-east-1"}) + if err == nil { + t.Fatal("expected the failed restore to error") + } + + removed := false + for _, call := range mock.Calls { + if strings.Contains(call, "rm -f '/tmp/teploy-restore.abc123/mysql.env'") { + removed = true + } + if strings.Contains(call, "sekret") { + t.Errorf("password must appear in no command: %s", call) + } } - if mysql := restoreCmd[strings.Index(restoreCmd, "mysql -u root"):]; strings.Contains(mysql, "sekret") { - t.Errorf("password must not appear on the mysql argv: %s", restoreCmd) + if !removed { + t.Errorf("the kept-for-inspection workspace must have its credential file removed, calls: %v", mock.Calls) } } diff --git a/internal/backup/bundle.go b/internal/backup/bundle.go new file mode 100644 index 0000000..6d69a0b --- /dev/null +++ b/internal/backup/bundle.go @@ -0,0 +1,959 @@ +package backup + +import ( + "context" + "encoding/json" + "fmt" + "sort" + "strings" + "time" + + "github.com/useteploy/teploy/internal/config" + "github.com/useteploy/teploy/internal/secret" + "github.com/useteploy/teploy/internal/ssh" + "github.com/useteploy/teploy/internal/state" +) + +// DR bundle — C07. A bundle is a COMPLETE recoverable image of one +// application: state + release records, the applied app manifest, secret +// references (or encrypted material the operator explicitly selected), the +// routing/TLS references, and engine-aware data snapshots — each snapshot +// labeled with the consistency level it actually achieved. Existing +// `teploy backup create` volume archives are DATA-ONLY backups and stay +// clearly named as such; they are not bundles and must never be presented as +// whole-app recovery. +// +// Layout (mirrored by both stores — S3 keys and local directory trees): +// +// /dr//manifest.json written LAST (completeness marker) +// /dr//volumes/.tar.gz one archive per app volume +// /dr//accessories/ engine dump or tar per accessory +// /dr//secrets/.age only with --include-secrets +// /dr//env/.env only with --include-secrets +// /dr//env/credentials/ only with --include-secrets +// /dr//age-key only with --include-age-key + +// BundleSchemaVersion is the DR bundle manifest schema this CLI writes and +// accepts. Readers refuse manifests from other versions; forward +// compatibility within a version is additive fields only (X04 discipline). +const BundleSchemaVersion = 1 + +// Bundle kinds. Data-only volume archives (the `teploy backup` family) are +// not bundles at all; Kind exists so a bundle manifest can never be confused +// with one. +const BundleKindDR = "dr" + +// BundleManifest is the schema-versioned inventory of a DR bundle: what the +// app was, what was captured, at what consistency, and what recovery +// requires. Everything needed to REBUILD the app on a fresh host — except +// the data artifacts themselves, which the manifest names. +type BundleManifest struct { + SchemaVersion int `json:"schema_version"` + Kind string `json:"kind"` + ID string `json:"id"` + App string `json:"app"` + Server string `json:"server"` + CreatedAt time.Time `json:"created_at"` + CreatedBy string `json:"created_by,omitempty"` + + // State is the app's /deployments//state.json VERBATIM (generation, + // release identity, ports, previous release). ReleaseRecords are the + // /deployments//meta/.json per-release execution records. + // AppManifest is state's applied effective-config manifest. + State json.RawMessage `json:"state,omitempty"` + ReleaseRecords []json.RawMessage `json:"release_records,omitempty"` + AppManifest json.RawMessage `json:"app_manifest,omitempty"` + // AppRun is what an isolated restore boots for the application check: + // the deployed image plus the recorded container command (from the + // newest release record). Without the command many images exit + // instantly and prove nothing. + AppRun AppRunSpec `json:"app_run"` + + Secrets SecretsRecord `json:"secrets"` + Routing RoutingRecord `json:"routing"` + Snapshots []SnapshotRecord `json:"snapshots"` + Recovery RecoveryPlan `json:"recovery"` +} + +// SecretsRecord carries what the bundle knows about the app's secrets. +// Default mode is references-only: the KEY NAMES, so a restoring operator +// knows what must exist on the target before cutover — never the material. +// Encrypted material (age ciphertexts, the resolved app .env, accessory +// credential files) is included ONLY when the operator passed +// --include-secrets, and the manifest says so explicitly. +type SecretsRecord struct { + Mode string `json:"mode"` // "references" | "encrypted" + Keys []string `json:"keys,omitempty"` + // AgeKeyIncluded reports whether the bundle carries the originating + // host's /deployments/.age-key. Without it, encrypted secrets material + // can only be decrypted with the ORIGINAL host's age key — Recovery + // states that requirement in operator language. + AgeKeyIncluded bool `json:"age_key_included"` + // Included lists the secret-material members inside the bundle + // (populated only in "encrypted" mode). + Included []string `json:"included,omitempty"` + // Recovery is the human requirement statement for getting secrets back. + Recovery string `json:"recovery,omitempty"` +} + +// RoutingRecord captures the routing/TLS IDENTITY the app had: domain, +// ingress mode, ports, and the TLS mode with cert-file REFERENCES (the cert +// files themselves live on the operator's machine; the bundle names the +// teploy.yml references so a fresh host knows what must be re-supplied). +type RoutingRecord struct { + Domain string `json:"domain,omitempty"` + IngressMode string `json:"ingress_mode,omitempty"` // caddy | external | host + Bind string `json:"bind,omitempty"` + Port int `json:"port,omitempty"` + Publish []string `json:"publish,omitempty"` + TLS *TLSRecord `json:"tls,omitempty"` +} + +type TLSRecord struct { + Mode string `json:"mode"` // acme | internal | custom-cert + Cert string `json:"cert,omitempty"` + Key string `json:"key,omitempty"` +} + +// AppRunSpec is the minimum needed to BOOT the app image for validation. +type AppRunSpec struct { + Image string `json:"image,omitempty"` + Cmd string `json:"cmd,omitempty"` +} + +// deriveAppRun reads the deployed image from state and the container +// command from the NEWEST release record (records carry created_at + cmd; +// state.json does not). Missing records leave Cmd empty — the app check +// then boots the bare image and reports skip/fail honestly. +func deriveAppRun(stateBytes []byte, records []json.RawMessage) AppRunSpec { + var s struct { + ImageRef string `json:"image_ref"` + } + _ = json.Unmarshal(stateBytes, &s) + spec := AppRunSpec{Image: s.ImageRef} + + var best struct { + CreatedAt time.Time `json:"created_at"` + Cmd string `json:"cmd"` + ImageRef string `json:"image_ref"` + } + best.CreatedAt = time.Time{} + for _, rec := range records { + var r struct { + CreatedAt time.Time `json:"created_at"` + Cmd string `json:"cmd"` + ImageRef string `json:"image_ref"` + } + if err := json.Unmarshal(rec, &r); err != nil { + continue + } + if r.CreatedAt.After(best.CreatedAt) { + best = r + } + } + if best.Cmd != "" { + spec.Cmd = best.Cmd + } + if spec.Image == "" { + spec.Image = best.ImageRef + } + return spec +} + +// SnapshotRecord describes ONE data artifact in the bundle and — honestly — +// what consistency it can claim. A tar archive is not proof of database +// consistency: the Consistency field records what the copy actually is, and +// Detection records how the engine was decided, including when that decision +// was a heuristic the operator should double-check. +type SnapshotRecord struct { + Name string `json:"name"` // volume key or accessory name + Role string `json:"role"` // "app-volume" | "accessory" + Engine string `json:"engine"` // "" for plain volumes; postgres|mysql|mariadb|mongo|redis|nucleus|generic + Method string `json:"method"` // tar | pg_dump | mysqldump | mongodump | redis-bgsave + Consistency string `json:"consistency"` // engine-consistent | engine-snapshot | crash-consistent | quiesced + Detection string `json:"detection"` // image-pattern | volume-pattern | operator-override | teploy-stop + Artifact string `json:"artifact"` // member path inside the bundle + // Image is the accessory image the snapshot was taken from (empty for + // app volumes) — what an isolated restore boots to validate the dump. + Image string `json:"image,omitempty"` + // EngineParams carries the NON-SECRET parameters a dump needs to land + // in a scratch engine (postgres/mysql database + user names). These are + // identifiers, not credentials; passwords never travel in the manifest. + EngineParams map[string]string `json:"engine_params,omitempty"` + Notes string `json:"notes,omitempty"` +} + +// RecoveryPlan is the documented recovery sequence carried in the bundle: +// what a restoring operator must do beyond running the restore command — +// accessory upgrades, credential rotation, storage-path changes, and the +// explicit cutover. Steps may be manual; the bundle carries what's needed +// and says which steps are manual. +type RecoveryPlan struct { + Steps []RecoveryStep `json:"steps"` +} + +type RecoveryStep struct { + ID string `json:"id"` + Action string `json:"action"` // restore-isolated | validate | cutover | accessory-upgrade | credential-rotation | storage-path | tls-material | deploy + Summary string `json:"summary"` + Command string `json:"command,omitempty"` + Manual bool `json:"manual"` +} + +// BundleStore reads and writes bundle members. Two implementations exist: +// S3 (the normal remote target, via the server's aws CLI) and a plain +// directory on a filesystem the server can reach (offline bundles: copy a +// directory tree off a dying host, restore from it). Both use the same +// member layout. A bundle is COMPLETE only when its manifest.json exists — +// creators upload the manifest last, and readers treat a missing manifest +// as "no bundle". +type BundleStore interface { + // Upload publishes one server-side file as a bundle member. Callers + // upload manifest.json LAST. + Upload(ctx context.Context, exec ssh.Executor, app, id, member, serverPath string) error + // Download fetches a bundle member to a server-side path. + Download(ctx context.Context, exec ssh.Executor, app, id, member, destPath string) error + // FetchManifest returns the manifest bytes. + FetchManifest(ctx context.Context, exec ssh.Executor, app, id string) ([]byte, error) + // ListIDs returns the bundle IDs present for an app. + ListIDs(ctx context.Context, exec ssh.Executor, app string) ([]string, error) +} + +// S3BundleStore stores bundles at s3:////dr//. +// (S3 has no directory atomics: a mid-upload failure can leave a prefix +// without a manifest. Such prefixes surface in ListIDs but fail loudly at +// FetchManifest with "bundle absent or incomplete" — an incomplete bundle is +// never silently usable.) +type S3BundleStore struct{ S3 S3Config } + +func (s S3BundleStore) key(app, id, member string) string { + return fmt.Sprintf("s3://%s/%s/dr/%s/%s", s.S3.Bucket, app, id, member) +} + +func (s S3BundleStore) Upload(ctx context.Context, exec ssh.Executor, app, id, member, serverPath string) error { + _, err := exec.Run(ctx, s.S3.AWS(fmt.Sprintf("s3 cp %s %s", ssh.ShellQuote(serverPath), ssh.ShellQuote(s.key(app, id, member))))) + if err != nil { + return fmt.Errorf("uploading %s: %w", s.key(app, id, member), err) + } + return nil +} + +func (s S3BundleStore) Download(ctx context.Context, exec ssh.Executor, app, id, member, destPath string) error { + _, err := exec.Run(ctx, s.S3.AWS(fmt.Sprintf("s3 cp %s %s", ssh.ShellQuote(s.key(app, id, member)), ssh.ShellQuote(destPath)))) + if err != nil { + return fmt.Errorf("downloading %s: %w", s.key(app, id, member), err) + } + return nil +} + +func (s S3BundleStore) FetchManifest(ctx context.Context, exec ssh.Executor, app, id string) ([]byte, error) { + if err := ValidateDate(id); err != nil { + return nil, err + } + out, err := exec.Run(ctx, s.S3.AWS(fmt.Sprintf("s3 cp %s -", ssh.ShellQuote(s.key(app, id, "manifest.json"))))) + if err != nil { + return nil, fmt.Errorf("fetching bundle manifest (bundle absent or incomplete?): %w", err) + } + return []byte(out), nil +} + +func (s S3BundleStore) ListIDs(ctx context.Context, exec ssh.Executor, app string) ([]string, error) { + prefix := fmt.Sprintf("s3://%s/%s/dr/", s.S3.Bucket, app) + out, err := exec.Run(ctx, s.S3.AWS(fmt.Sprintf("s3 ls %s", ssh.ShellQuote(prefix)))) + if err != nil { + return nil, fmt.Errorf("listing bundles at %s: %w", prefix, err) + } + var ids []string + for _, line := range strings.Split(strings.TrimSpace(out), "\n") { + fields := strings.Fields(line) + if len(fields) < 4 || !strings.HasSuffix(fields[len(fields)-1], "/") { + continue + } + id := strings.TrimSuffix(fields[len(fields)-1], "/") + if ValidateDate(id) == nil { + ids = append(ids, id) + } + } + return ids, nil +} + +// DirBundleStore stores bundles as a plain directory tree +// //dr// on a filesystem the server can reach — the +// offline path: an operator can rsync the tree off a dying host and restore +// from it with no S3 in play. +type DirBundleStore struct{ Root string } + +func (d DirBundleStore) dir(app, id string) string { + return fmt.Sprintf("%s/%s/dr/%s", strings.TrimRight(d.Root, "/"), app, id) +} + +func (d DirBundleStore) Upload(ctx context.Context, exec ssh.Executor, app, id, member, serverPath string) error { + dst := d.dir(app, id) + "/" + member + if _, err := exec.Run(ctx, "mkdir -p "+ssh.ShellQuote(d.dir(app, id)+"/"+dirOf(member))); err != nil { + return fmt.Errorf("preparing bundle dir: %w", err) + } + if _, err := exec.Run(ctx, fmt.Sprintf("cp -p %s %s", ssh.ShellQuote(serverPath), ssh.ShellQuote(dst))); err != nil { + return fmt.Errorf("copying bundle member %s: %w", member, err) + } + return nil +} + +func (d DirBundleStore) Download(ctx context.Context, exec ssh.Executor, app, id, member, destPath string) error { + src := d.dir(app, id) + "/" + member + if _, err := exec.Run(ctx, fmt.Sprintf("cp -p %s %s", ssh.ShellQuote(src), ssh.ShellQuote(destPath))); err != nil { + return fmt.Errorf("fetching bundle member %s: %w", member, err) + } + return nil +} + +func (d DirBundleStore) FetchManifest(ctx context.Context, exec ssh.Executor, app, id string) ([]byte, error) { + if err := ValidateDate(id); err != nil { + return nil, err + } + data, present, err := state.ReadRemoteFile(ctx, exec, d.dir(app, id)+"/manifest.json") + if err != nil { + return nil, fmt.Errorf("fetching bundle manifest: %w", err) + } + if !present { + return nil, fmt.Errorf("no manifest at %s — bundle absent or incomplete", d.dir(app, id)) + } + return data, nil +} + +func (d DirBundleStore) ListIDs(ctx context.Context, exec ssh.Executor, app string) ([]string, error) { + base := fmt.Sprintf("%s/%s/dr", strings.TrimRight(d.Root, "/"), app) + out, err := exec.Run(ctx, "find "+ssh.ShellQuote(base)+" -mindepth 2 -maxdepth 2 -name manifest.json -printf '%h\\n' 2>/dev/null | sort") + if err != nil { + return nil, fmt.Errorf("listing bundles under %s: %w", base, err) + } + var ids []string + for _, line := range strings.Split(strings.TrimSpace(out), "\n") { + line = strings.TrimSpace(line) + if line == "" { + continue + } + id := line[strings.LastIndexByte(line, '/')+1:] + if ValidateDate(id) == nil { + ids = append(ids, id) + } + } + return ids, nil +} + +// ParseBundleManifest validates and decodes a bundle manifest. Schema +// mismatches and foreign kinds are refused before any restore step runs. +func ParseBundleManifest(data []byte) (*BundleManifest, error) { + var m BundleManifest + if err := json.Unmarshal(data, &m); err != nil { + return nil, fmt.Errorf("parsing bundle manifest: %w", err) + } + if m.SchemaVersion != BundleSchemaVersion { + return nil, fmt.Errorf("unsupported bundle manifest schema version %d (this CLI understands %d)", m.SchemaVersion, BundleSchemaVersion) + } + if m.Kind != BundleKindDR { + return nil, fmt.Errorf("manifest kind %q is not a disaster-recovery bundle", m.Kind) + } + if m.ID == "" || m.App == "" { + return nil, fmt.Errorf("bundle manifest is missing its id or app") + } + if err := ValidateDate(m.ID); err != nil { + return nil, fmt.Errorf("bundle manifest id: %w", err) + } + return &m, nil +} + +// BundleOptions configures one CreateBundle run. +type BundleOptions struct { + App string + Config config.AppConfig // the local teploy.yml (volumes, accessories, tls refs) + + // IncludeSecrets opts IN encrypted secret material (age ciphertexts, + // the resolved app .env, accessory credential files). Never default. + IncludeSecrets bool + // IncludeAgeKey also carries /deployments/.age-key so a fresh host can + // decrypt the ciphertexts. Even more sensitive; never default. + IncludeAgeKey bool + + // StopApp stops the app's web containers before snapshotting volumes + // (and restarts them after): the copies are then QUIESCED rather than + // crash-consistent. Accessories keep running — their snapshots go + // through engine-native dump tooling and do not need quiescing. + StopApp bool + // VolumeQuiesced lists volume names the OPERATOR asserts are already + // quiesced (writer stopped by hand); recorded as quiesced with + // detection=operator-override. teploy cannot verify the assertion. + VolumeQuiesced []string + + // Version stamps CreatedBy (the CLI version string). + Version string + // Now overrides the clock (tests). + Now func() time.Time +} + +// engineDataPathPatterns maps container destinations that LOOK like live +// database data directories to the engine they suggest. Heuristic only — +// recorded in the manifest as volume-pattern detection with an explicit +// caveat, never as proof of consistency. Long, distinctive paths (postgres, +// mysql, mongo) match by substring; short ones (redis /data) only EXACTLY — +// "/data" as a substring also matches "/app/data", which is not redis. +var engineDataPathPatterns = []struct { + engine string + exact bool + hints []string +}{ + {"postgres", false, []string{"/var/lib/postgresql", "pgdata"}}, + {"mysql", false, []string{"/var/lib/mysql", "/var/lib/mariadb"}}, + {"mongo", false, []string{"/data/db"}}, + {"redis", true, []string{"/data"}}, +} + +// guessVolumeEngine inspects a volume's CONTAINER destination path for +// known engine data-dir patterns. Returns "" when nothing matches. +func guessVolumeEngine(containerPath string) string { + p := strings.ToLower(strings.TrimRight(containerPath, "/")) + if p == "" { + return "" + } + for _, pat := range engineDataPathPatterns { + for _, hint := range pat.hints { + if pat.exact { + if p == hint { + return pat.engine + } + continue + } + if strings.Contains(p, hint) { + return pat.engine + } + } + } + return "" +} + +// CreateBundle assembles a complete DR bundle on the server and publishes it +// through the store. The manifest is uploaded LAST so a partial upload can +// never be mistaken for a bundle. +func (c *Client) CreateBundle(ctx context.Context, opts BundleOptions, store BundleStore) (*BundleManifest, error) { + if !safeName.MatchString(opts.App) { + return nil, fmt.Errorf("invalid app name %q", opts.App) + } + now := time.Now + if opts.Now != nil { + now = opts.Now + } + id, err := newBackupID(now()) + if err != nil { + return nil, err + } + + quiesced := make(map[string]bool, len(opts.VolumeQuiesced)) + for _, v := range opts.VolumeQuiesced { + if _, ok := opts.Config.Volumes[v]; !ok { + return nil, fmt.Errorf("volume %q (--quiesced-volume) is not a volume of %s (volumes: %v)", v, opts.App, volumeNames(opts.Config)) + } + quiesced[v] = true + } + + // State first: an app with no on-server state has nothing to recover — + // data-only backup (`teploy backup create`) is the right tool, and this + // error says so instead of minting a bundle that lies about scope. + stateBytes, present, err := state.ReadRemoteFile(ctx, c.exec, fmt.Sprintf("%s/%s/state.json", deploymentsDir, opts.App)) + if err != nil { + return nil, fmt.Errorf("reading app state: %w", err) + } + if !present { + return nil, fmt.Errorf("no /deployments/%s/state.json on the server — the app was never deployed here; a DR bundle needs deployed state (for plain volume data use `teploy backup create`, a data-only backup)", opts.App) + } + var appState state.AppState + if err := json.Unmarshal(stateBytes, &appState); err != nil { + return nil, fmt.Errorf("parsing app state: %w", err) + } + if appState.SchemaVersion != state.SchemaVersionV2 { + return nil, fmt.Errorf("unsupported state schema version %d", appState.SchemaVersion) + } + + // Workspace: everything assembles on the server (where the data is), + // inside one private mktemp tree. + runOut, err := c.exec.Run(ctx, "umask 077; mktemp -d /tmp/teploy-dr.XXXXXXXX") + if err != nil { + return nil, fmt.Errorf("creating bundle workspace: %w", err) + } + ws := strings.TrimSpace(runOut) + cleanup := func() { + c.exec.Run(context.WithoutCancel(ctx), "rm -rf "+ssh.ShellQuote(ws)) + } + + m := &BundleManifest{ + SchemaVersion: BundleSchemaVersion, + Kind: BundleKindDR, + ID: id, + App: opts.App, + Server: c.exec.Host(), + CreatedAt: now().UTC(), + CreatedBy: opts.Version, + State: json.RawMessage(stateBytes), + AppManifest: appState.AppliedManifest, + } + + if m.ReleaseRecords, err = c.collectReleaseRecords(ctx, opts.App); err != nil { + cleanup() + return nil, err + } + m.AppRun = deriveAppRun(stateBytes, m.ReleaseRecords) + + // Optional quiescence: stop the app's web containers so volume copies + // are quiesced rather than crash-consistent. + var stoppedContainers []string + if opts.StopApp { + names, err := c.exec.Run(ctx, fmt.Sprintf( + "docker ps --filter label=teploy.app=%s --filter label=teploy.process=web --format '{{.Names}}'", + ssh.ShellQuote(opts.App))) + if err != nil { + cleanup() + return nil, fmt.Errorf("listing %s containers to quiesce: %w", opts.App, err) + } + for _, n := range strings.Fields(strings.TrimSpace(names)) { + if n == "" { + continue + } + fmt.Fprintf(c.out, "Stopping %s for quiesced snapshot...\n", n) + if _, err := c.exec.Run(ctx, "docker stop "+ssh.ShellQuote(n)); err != nil { + // Restart anything already stopped before aborting. + c.restartContainers(context.WithoutCancel(ctx), stoppedContainers) + cleanup() + return nil, fmt.Errorf("stopping %s for quiesced snapshot: %w", n, err) + } + stoppedContainers = append(stoppedContainers, n) + } + } + defer c.restartContainers(context.WithoutCancel(ctx), stoppedContainers) + + // Secret material — only on explicit operator selection, with the + // manifest stating exactly what traveled. + secrets := secret.NewManager(c.exec) + keys, err := secrets.List(ctx, opts.App) + if err != nil { + cleanup() + return nil, fmt.Errorf("listing secrets for %s: %w", opts.App, err) + } + m.Secrets = SecretsRecord{Mode: "references", Keys: keys} + m.Secrets.Recovery = "secret VALUES are not in this bundle — the keys above must exist on the restore target before cutover (`teploy secret set`), or re-create the bundle with --include-secrets" + if opts.IncludeSecrets { + m.Secrets.Mode = "encrypted" + m.Secrets.Recovery = "encrypted secret material IS inside this bundle (age ciphertexts + resolved .env + accessory credentials); guard it like the live secrets" + for _, k := range keys { + member := "secrets/" + k + ".age" + if err := c.wsCopy(ctx, ws, member, fmt.Sprintf("%s/%s/secrets/%s.age", deploymentsDir, opts.App, k)); err != nil { + cleanup() + return nil, err + } + m.Secrets.Included = append(m.Secrets.Included, member) + } + // The app .env and accessory credential files hold RESOLVED secrets + // (generated DB passwords, DATABASE_URL) — same opt-in gate. + included, err := c.wsCopyIfExists(ctx, ws, "env/.env", fmt.Sprintf("%s/%s/.env", deploymentsDir, opts.App)) + if err != nil { + cleanup() + return nil, err + } + if included { + m.Secrets.Included = append(m.Secrets.Included, "env/.env") + } + for _, accName := range sortedAccessoryNames(opts.Config) { + member := "env/credentials/" + accName + included, err := c.wsCopyIfExists(ctx, ws, member, fmt.Sprintf("%s/%s/accessories/%s/credentials", deploymentsDir, opts.App, accName)) + if err != nil { + cleanup() + return nil, err + } + if included { + m.Secrets.Included = append(m.Secrets.Included, member) + } + } + } + if opts.IncludeAgeKey { + if err := c.wsCopy(ctx, ws, "age-key", deploymentsDir+"/.age-key"); err != nil { + cleanup() + return nil, err + } + m.Secrets.AgeKeyIncluded = true + if !opts.IncludeSecrets { + m.Secrets.Recovery += " — the bundle carries the age key itself (--include-age-key); use it to re-key a fresh target's secret store" + } + } + + // App volume snapshots. The member directory must exist before tar + // writes into it (found live: the mock cannot catch missing paths). + if len(opts.Config.Volumes) > 0 { + if _, err := c.exec.Run(ctx, "mkdir -p "+ssh.ShellQuote(ws+"/volumes")); err != nil { + cleanup() + return nil, fmt.Errorf("preparing volumes in bundle workspace: %w", err) + } + } + for _, volName := range volumeNames(opts.Config) { + volDir := fmt.Sprintf("%s/%s/volumes/%s", deploymentsDir, opts.App, volName) + member := "volumes/" + volName + ".tar.gz" + fmt.Fprintf(c.out, "Snapshotting volume %s...\n", volName) + if _, err := c.exec.Run(ctx, fmt.Sprintf("tar -czf %s -C %s .", + ssh.ShellQuote(ws+"/"+member), ssh.ShellQuote(volDir))); err != nil { + cleanup() + return nil, fmt.Errorf("archiving volume %s: %w", volName, err) + } + rec := SnapshotRecord{ + Name: volName, + Role: "app-volume", + Method: "tar", + Artifact: member, + } + switch { + case quiesced[volName]: + rec.Consistency = consistencyQuiesced + rec.Detection = "operator-override" + rec.Notes = "operator asserted the writer was stopped for this copy; teploy did not verify" + case opts.StopApp && len(stoppedContainers) > 0: + rec.Consistency = consistencyQuiesced + rec.Detection = "teploy-stop" + rec.Notes = "app containers were stopped by teploy for this snapshot" + default: + rec.Consistency = consistencyCrash + rec.Notes = "live bind-mount copied while the app may have been writing; only crash recovery is guaranteed — use --stop-app or an engine accessory for stronger guarantees" + if eng := guessVolumeEngine(opts.Config.Volumes[volName]); eng != "" { + rec.Engine = eng + rec.Detection = "volume-pattern" + rec.Notes = fmt.Sprintf("container path %q matches the %s data-directory pattern (heuristic, unverified) — a raw tar is NOT a %s-consistent backup; this snapshot is crash-consistent at best", opts.Config.Volumes[volName], eng, eng) + } + } + m.Snapshots = append(m.Snapshots, rec) + } + + // Accessory snapshots: engine-aware, through the same planner the + // single-accessory backup uses (no second backup system). + for _, accName := range sortedAccessoryNames(opts.Config) { + accCfg := opts.Config.Accessories[accName] + image, env, err := c.accessoryImageEnv(ctx, opts.App, accName, accCfg) + if err != nil { + cleanup() + return nil, err + } + accWS := ws + "/acc/" + accName + if _, err := c.exec.Run(ctx, "mkdir -p "+ssh.ShellQuote(accWS)); err != nil { + cleanup() + return nil, fmt.Errorf("preparing accessory workspace: %w", err) + } + fmt.Fprintf(c.out, "Snapshotting accessory %s (%s)...\n", accName, image) + plan := planAccessoryDump(opts.App, accName, image, env, accWS) + if err := plan.stage(ctx, c.exec); err != nil { + return nil, fmt.Errorf("staging credential for accessory %s (no dump was run): %w", accName, err) + } + if _, err := c.exec.Run(ctx, plan.cmd); err != nil { + cleanup() + return nil, fmt.Errorf("dumping accessory %s: %w", accName, err) + } + member := "accessories/" + accName + plan.ext + if err := c.wsMove(ctx, ws, member, plan.artifactPath); err != nil { + cleanup() + return nil, err + } + rec := SnapshotRecord{ + Name: accName, + Role: "accessory", + Engine: plan.engine, + Method: plan.method, + Consistency: plan.consistency, + Detection: "image-pattern", + Artifact: member, + Image: image, + } + switch plan.engine { + case "postgres": + db, user := postgresDBAndUser(opts.App, env) + rec.EngineParams = map[string]string{"db": db, "user": user} + case "mysql", "mariadb": + rec.EngineParams = map[string]string{"db": mysqlDB(opts.App, env)} + } + if plan.engine == "generic" { + rec.Notes = "generic accessory: raw tar of the bind-mounted directory while the engine ran; engine-native dump tooling for this image is not known to teploy — the isolated restore validates it by booting a scratch engine" + } + if plan.engine == "redis" { + rec.Notes = "redis BGSAVE-acknowledged dump.rdb copy (appendonly=no proven at snapshot time)" + } + m.Snapshots = append(m.Snapshots, rec) + } + + m.Routing = RoutingRecord{ + Domain: appState.Domain, + IngressMode: appState.IngressMode, + Port: appState.CurrentPort, + Publish: opts.Config.Publish, + } + if appState.IngressMode == config.IngressHost { + m.Routing.Bind = opts.Config.Bind + } + if opts.Config.TLS != nil { + tls := &TLSRecord{Mode: "acme"} + if opts.Config.TLS.Internal { + tls.Mode = "internal" + } else if opts.Config.TLS.Cert != "" { + tls.Mode = "custom-cert" + tls.Cert = opts.Config.TLS.Cert + tls.Key = opts.Config.TLS.Key + } + m.Routing.TLS = tls + } + + m.Recovery = c.buildRecoveryPlan(opts, m) + + // Publish: members first, manifest LAST (completeness marker). + manifestJSON, err := json.MarshalIndent(m, "", " ") + if err != nil { + cleanup() + return nil, fmt.Errorf("encoding bundle manifest: %w", err) + } + if err := c.exec.Upload(ctx, strings.NewReader(string(manifestJSON)+"\n"), ws+"/manifest.json", "0600"); err != nil { + cleanup() + return nil, fmt.Errorf("staging bundle manifest: %w", err) + } + + uploadMember := func(member, serverPath string) error { + if err := store.Upload(ctx, c.exec, opts.App, id, member, serverPath); err != nil { + return err + } + return nil + } + for _, snap := range m.Snapshots { + if err := uploadMember(snap.Artifact, ws+"/"+snap.Artifact); err != nil { + cleanup() + return nil, err + } + } + for _, member := range m.Secrets.Included { + if err := uploadMember(member, ws+"/"+member); err != nil { + cleanup() + return nil, err + } + } + if opts.IncludeAgeKey { + if err := uploadMember("age-key", ws+"/age-key"); err != nil { + cleanup() + return nil, err + } + } + if err := uploadMember("manifest.json", ws+"/manifest.json"); err != nil { + cleanup() + return nil, err + } + + cleanup() + fmt.Fprintf(c.out, "DR bundle created: %s\n", id) + return m, nil +} + +func (c *Client) buildRecoveryPlan(opts BundleOptions, m *BundleManifest) RecoveryPlan { + plan := RecoveryPlan{Steps: []RecoveryStep{ + { + ID: "restore-isolated", + Action: "restore-isolated", + Summary: "restore the bundle to an ISOLATED staging target and validate it — live state is never touched", + Command: fmt.Sprintf("teploy dr restore %s", m.ID), + }, + { + ID: "cutover", + Action: "cutover", + Summary: "after validation passes, explicitly promote the staged restore over the live app (this is the mutation step)", + Command: fmt.Sprintf("teploy dr cutover %s", m.ID), + }, + { + ID: "deploy", + Action: "deploy", + Summary: "cutover restores data/state only; run a deploy to bring the app container and routing live from the restored state", + Command: "teploy deploy", + Manual: true, + }, + }} + + if len(m.Snapshots) > 0 { + var accs, vols []string + for _, s := range m.Snapshots { + if s.Role == "accessory" { + accs = append(accs, s.Name) + } else { + vols = append(vols, s.Name) + } + } + if len(accs) > 0 { + plan.Steps = append(plan.Steps, RecoveryStep{ + ID: "accessory-upgrade", + Action: "accessory-upgrade", + Summary: fmt.Sprintf("accessory data survives image upgrades on its bind mounts (%s): `teploy accessory upgrade ` recreates the container in place; consult the engine's own major-version upgrade docs before jumping versions — restore-into-scratch (teploy dr restore) is the pre-flight", strings.Join(accs, ", ")), + Manual: true, + }) + } + if len(vols) > 0 { + plan.Steps = append(plan.Steps, RecoveryStep{ + ID: "storage-path", + Action: "storage-path", + Summary: fmt.Sprintf("bundle artifacts are keyed by volume NAME (%s), not host path: moving /deployments or remapping volumes in teploy.yml re-homes storage without invalidating this bundle", strings.Join(vols, ", ")), + }) + } + } + + if m.Secrets.Mode == "encrypted" { + plan.Steps = append(plan.Steps, RecoveryStep{ + ID: "credential-rotation", + Action: "credential-rotation", + Summary: "this bundle carries secret material — after a restore, rotate the carried credentials (`teploy secret rotate KEY`, accessory passwords) if the bundle's storage was ever outside your control", + Manual: true, + }) + } else if len(m.Secrets.Keys) > 0 { + plan.Steps = append(plan.Steps, RecoveryStep{ + ID: "credential-rotation", + Action: "credential-rotation", + Summary: fmt.Sprintf("referenced secret keys (%s) must exist on the target before cutover: `teploy secret set KEY=...` — restore preflight fails without them", strings.Join(m.Secrets.Keys, ", ")), + Manual: true, + }) + } + + if m.Routing.TLS != nil && m.Routing.TLS.Mode == "custom-cert" { + plan.Steps = append(plan.Steps, RecoveryStep{ + ID: "tls-material", + Action: "tls-material", + Summary: fmt.Sprintf("routing used a custom certificate (teploy.yml tls.cert=%s, tls.key=%s) — the PEM files are NOT in the bundle; re-supply them on the fresh host before deploying", m.Routing.TLS.Cert, m.Routing.TLS.Key), + Manual: true, + }) + } + return plan +} + +// accessoryImageEnv resolves the accessory's REAL image + env (the running +// container holds the resolved credentials; teploy.yml may carry only +// `auto`/`secret:` references). Falls back to config only when the +// container is absent AND the config env carries no unresolved references. +func (c *Client) accessoryImageEnv(ctx context.Context, app, name string, cfg config.AccessoryConfig) (string, map[string]string, error) { + if image, env, err := c.InspectAccessory(ctx, app, name); err == nil { + return image, env, nil + } + for _, v := range cfg.Env { + if v == "auto" || strings.HasPrefix(v, "secret:") { + return "", nil, fmt.Errorf("accessory %s is not running and its teploy.yml env carries unresolved references — start it (teploy deploy) or snapshot with --stop-app after a deploy", name) + } + } + return cfg.Image, cfg.Env, nil +} + +func (c *Client) restartContainers(ctx context.Context, names []string) { + for _, n := range names { + if n == "" { + continue + } + fmt.Fprintf(c.out, "Restarting %s...\n", n) + if _, err := c.exec.Run(ctx, "docker start "+ssh.ShellQuote(n)); err != nil { + fmt.Fprintf(c.out, "WARNING: could not restart %s after bundling: %v\n", n, err) + } + } +} + +// wsCopy copies a server file into the bundle workspace under member, +// creating parent directories. Missing source is an error (callers decide +// which absences are fine and use wsCopyIfExists). +func (c *Client) wsCopy(ctx context.Context, ws, member, src string) error { + dst := ws + "/" + member + if _, err := c.exec.Run(ctx, "mkdir -p "+ssh.ShellQuote(ws+"/"+dirOf(member))); err != nil { + return fmt.Errorf("preparing %s in bundle workspace: %w", member, err) + } + if _, err := c.exec.Run(ctx, fmt.Sprintf("cp -p %s %s", ssh.ShellQuote(src), ssh.ShellQuote(dst))); err != nil { + return fmt.Errorf("copying %s into the bundle: %w", src, err) + } + return nil +} + +// wsCopyIfExists copies src into the workspace under member when it exists +// and reports whether it was included; a confirmed-absent source records +// nothing (the manifest's Included list is the source of truth). Transport +// failures are errors. +func (c *Client) wsCopyIfExists(ctx context.Context, ws, member, src string) (bool, error) { + exists, err := c.remoteExists(ctx, src) + if err != nil { + return false, fmt.Errorf("checking %s: %w", src, err) + } + if !exists { + return false, nil + } + if err := c.wsCopy(ctx, ws, member, src); err != nil { + return false, err + } + return true, nil +} + +func (c *Client) wsMove(ctx context.Context, ws, member, src string) error { + dst := ws + "/" + member + if _, err := c.exec.Run(ctx, "mkdir -p "+ssh.ShellQuote(ws+"/"+dirOf(member))); err != nil { + return fmt.Errorf("preparing %s in bundle workspace: %w", member, err) + } + if _, err := c.exec.Run(ctx, fmt.Sprintf("mv -f %s %s", ssh.ShellQuote(src), ssh.ShellQuote(dst))); err != nil { + return fmt.Errorf("moving %s into the bundle: %w", src, err) + } + return nil +} + +func dirOf(path string) string { + if i := strings.LastIndexByte(path, '/'); i >= 0 { + return path[:i] + } + return "." +} + +func volumeNames(cfg config.AppConfig) []string { + names := make([]string, 0, len(cfg.Volumes)) + for k := range cfg.Volumes { + names = append(names, k) + } + sort.Strings(names) + return names +} + +func sortedAccessoryNames(cfg config.AppConfig) []string { + names := make([]string, 0, len(cfg.Accessories)) + for k := range cfg.Accessories { + names = append(names, k) + } + sort.Strings(names) + return names +} + +func (c *Client) remoteExists(ctx context.Context, path string) (bool, error) { + out, err := c.exec.Run(ctx, fmt.Sprintf("if [ -e %s ]; then printf 'present\\n'; else printf 'absent\\n'; fi", ssh.ShellQuote(path))) + if err != nil { + return false, err + } + switch strings.TrimSpace(out) { + case "present": + return true, nil + case "absent": + return false, nil + } + return false, fmt.Errorf("checking %s: unrecognized output", path) +} + +func (c *Client) collectReleaseRecords(ctx context.Context, app string) ([]json.RawMessage, error) { + metaDir := fmt.Sprintf("%s/%s/meta", deploymentsDir, app) + out, err := c.exec.Run(ctx, fmt.Sprintf("find %s -maxdepth 1 -name '*.json' -type f 2>/dev/null | sort", ssh.ShellQuote(metaDir))) + if err != nil { + return nil, fmt.Errorf("listing release records: %w", err) + } + var records []json.RawMessage + for _, line := range strings.Split(strings.TrimSpace(out), "\n") { + line = strings.TrimSpace(line) + if line == "" { + continue + } + data, present, err := state.ReadRemoteFile(ctx, c.exec, line) + if err != nil { + return nil, fmt.Errorf("reading release record %s: %w", line, err) + } + if !present { + continue + } + records = append(records, json.RawMessage(data)) + } + return records, nil +} diff --git a/internal/backup/bundle_restore.go b/internal/backup/bundle_restore.go new file mode 100644 index 0000000..26afd01 --- /dev/null +++ b/internal/backup/bundle_restore.go @@ -0,0 +1,957 @@ +package backup + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "regexp" + "sort" + "strings" + "time" + + "github.com/useteploy/teploy/internal/accessories" + "github.com/useteploy/teploy/internal/config" + "github.com/useteploy/teploy/internal/secret" + "github.com/useteploy/teploy/internal/ssh" + "github.com/useteploy/teploy/internal/state" +) + +// Isolated restore + explicit cutover — C07's core safety property: live +// overwrite is NEVER the default. `RestoreBundleIsolated` lands a bundle in +// a staging area under /var/tmp, boots throwaway scratch engines/app +// containers against the STAGED data, validates them, and writes a receipt +// with measured RPO/RTO. Nothing under /deployments is touched until +// `CutoverBundle` — a separate, explicit command — promotes the staged data +// with two-phase recovery discipline. + +// DRStagingRoot is where isolated restores stage. /var/tmp (disk-backed), +// not /tmp (often tmpfs — see verify.go's 28 GB lesson). +const DRStagingRoot = "/var/tmp/teploy-dr" + +// RestoreReceiptSchemaVersion versions the restore receipt. +const RestoreReceiptSchemaVersion = 1 + +// DRStagingPath is the deterministic staging directory for one bundle of +// one app — deterministic so a later `teploy dr cutover` finds it without +// state of its own. +func DRStagingPath(app, id string) string { + return fmt.Sprintf("%s/%s/%s", DRStagingRoot, app, id) +} + +// drStagingRE pins the exact shape of staging paths: the recursive wipe on +// re-restore must only ever match this pattern, never an operator path. +var drStagingRE = regexp.MustCompile(`^/var/tmp/teploy-dr/[A-Za-z0-9][A-Za-z0-9._-]*/[0-9]{8}-[0-9]{6}(-[0-9a-f]{16})?$`) + +// DRCheckResult is one validation check in the receipt. +type DRCheckResult struct { + Name string `json:"name"` // e.g. "accessory:db (postgres)", "app" + Kind string `json:"kind"` // "data" | "app" + Status string `json:"status"` // "pass" | "fail" | "skipped" + Metric string `json:"metric,omitempty"` // e.g. "tables=12" + Detail string `json:"detail,omitempty"` // skip/fail reason +} + +// RestoreReceipt is the measured outcome of an isolated restore. RPO = +// data age at restore time (restore start − bundle creation): the window of +// writes the bundle cannot contain. RTO = measured wall time from restore +// start to validated staging (download + extract + validation) — the +// recovery-time floor before cutover. +type RestoreReceipt struct { + SchemaVersion int `json:"schema_version"` + BundleID string `json:"bundle_id"` + App string `json:"app"` + StartedAt time.Time `json:"started_at"` + ValidatedAt time.Time `json:"validated_at"` + RPOSeconds int64 `json:"rpo_seconds"` + RTOSeconds int64 `json:"rto_seconds"` + StagingPath string `json:"staging_path"` + Checks []DRCheckResult `json:"checks"` + OK bool `json:"ok"` + NextStep string `json:"next_step"` +} + +// CutoverReceipt records what the explicit cutover changed and where the +// pre-cutover originals were preserved. +type CutoverReceipt struct { + SchemaVersion int `json:"schema_version"` + BundleID string `json:"bundle_id"` + App string `json:"app"` + StartedAt time.Time `json:"started_at"` + CompletedAt time.Time `json:"completed_at"` + Promoted []string `json:"promoted"` // live paths replaced from staging + RecoveryDirs []string `json:"recovery_dirs"` // where originals were preserved + Stopped []string `json:"stopped"` // containers stopped for the cutover + NextStep string `json:"next_step"` +} + +// BundleRestoreOptions configures an isolated restore or cutover. +type BundleRestoreOptions struct { + App string + ID string + Config config.AppConfig // the restore-time local teploy.yml + // Now overrides the clock (tests). + Now func() time.Time +} + +// ErrValidationRequired is returned by cutover when no validated staging +// exists for the bundle — the explicit rejection of cutting over unproven +// data. +var ErrValidationRequired = errors.New("no validated staged restore for this bundle — run `teploy dr restore` first") + +// RestoreBundleIsolated restores a DR bundle into an isolated staging area, +// validates it against scratch containers, and writes the RPO/RTO receipt. +// Live state under /deployments is never touched. +func (c *Client) RestoreBundleIsolated(ctx context.Context, opts BundleRestoreOptions, store BundleStore) (*RestoreReceipt, error) { + started := timeNow(opts.Now) + + // ---- Preflight: everything here is read-only. Missing keys, wrong + // schema, wrong app, corrupt manifest — all fail BEFORE any mutation. + manifest, err := c.fetchAndParseManifest(ctx, store, opts.App, opts.ID) + if err != nil { + return nil, err + } + if err := c.secretsPreflight(ctx, manifest); err != nil { + return nil, err + } + + staging := DRStagingPath(opts.App, manifest.ID) + if !drStagingRE.MatchString(staging) { + return nil, fmt.Errorf("internal error: staging path %q fails the safety pattern", staging) + } + + // ---- Staging (isolated: /var/tmp/teploy-dr/..., never /deployments). + // A re-restore of the same bundle wipes only this bundle's staging tree. + if _, err := c.exec.Run(ctx, "rm -rf "+ssh.ShellQuote(staging)); err != nil { + return nil, fmt.Errorf("clearing previous staging: %w", err) + } + if _, err := c.exec.Run(ctx, "umask 077; mkdir -p "+ssh.ShellQuote(staging)); err != nil { + return nil, fmt.Errorf("creating staging dir: %w", err) + } + + // ---- Download every artifact named by the manifest (+ secret members). + fmt.Fprintf(c.out, "Downloading bundle %s into %s...\n", manifest.ID, staging) + if _, err := c.exec.Run(ctx, "mkdir -p "+ssh.ShellQuote(staging+"/volumes")+" "+ssh.ShellQuote(staging+"/accessories")); err != nil { + return nil, fmt.Errorf("preparing staging layout: %w", err) + } + for _, snap := range manifest.Snapshots { + if err := store.Download(ctx, c.exec, opts.App, manifest.ID, snap.Artifact, staging+"/"+snap.Artifact); err != nil { + return nil, err + } + } + for _, member := range manifest.Secrets.Included { + if err := store.Download(ctx, c.exec, opts.App, manifest.ID, member, staging+"/"+member); err != nil { + return nil, err + } + } + if manifest.Secrets.AgeKeyIncluded { + if err := store.Download(ctx, c.exec, opts.App, manifest.ID, "age-key", staging+"/age-key"); err != nil { + return nil, err + } + } + // Keep the manifest itself in staging: cutover re-reads it without the + // store, and the receipt records which bundle produced this tree. + if manifestBytes, err := json.MarshalIndent(manifest, "", " "); err == nil { + _ = c.exec.Upload(context.WithoutCancel(ctx), strings.NewReader(string(manifestBytes)+"\n"), staging+"/manifest.json", "0600") + } + + // ---- Integrity: every gzip artifact must prove itself BEFORE any + // engine boots off it. A corrupt bundle fails here, originals + // untouched (they were never touched at all — this is staging). + for _, snap := range manifest.Snapshots { + if strings.HasSuffix(snap.Artifact, ".gz") { + if _, err := c.exec.Run(ctx, "gzip -t "+ssh.ShellQuote(staging+"/"+snap.Artifact)); err != nil { + return nil, fmt.Errorf("bundle member %s is corrupt (gzip -t failed): %w — refusing to restore from this bundle", snap.Artifact, err) + } + } + } + + // ---- Encrypted-material decrypt proof. When the bundle carries + // ciphertexts without the age key, the TARGET's age key must decrypt + // them — proven on one member before anything else runs. + if err := c.decryptProof(ctx, manifest, staging); err != nil { + return nil, err + } + + receipt := &RestoreReceipt{ + SchemaVersion: RestoreReceiptSchemaVersion, + BundleID: manifest.ID, + App: opts.App, + StartedAt: started.UTC(), + StagingPath: staging, + NextStep: fmt.Sprintf("teploy dr cutover %s", manifest.ID), + } + + // ---- Extract app volumes + generic accessory tars into staging. + for _, snap := range manifest.Snapshots { + switch snap.Role { + case "app-volume": + dest := staging + "/volumes/" + snap.Name + if _, err := c.exec.Run(ctx, fmt.Sprintf("mkdir -p %s && tar -xzf %s -C %s", + ssh.ShellQuote(dest), ssh.ShellQuote(staging+"/"+snap.Artifact), ssh.ShellQuote(dest))); err != nil { + return nil, c.failReceipt(ctx, receipt, fmt.Errorf("extracting volume %s: %w", snap.Name, err)) + } + case "accessory": + if snap.Method == "tar" { + dest := staging + "/accessories/" + snap.Name + if _, err := c.exec.Run(ctx, fmt.Sprintf("mkdir -p %s && tar -xzf %s -C %s", + ssh.ShellQuote(dest), ssh.ShellQuote(staging+"/"+snap.Artifact), ssh.ShellQuote(dest))); err != nil { + return nil, c.failReceipt(ctx, receipt, fmt.Errorf("extracting accessory %s: %w", snap.Name, err)) + } + } + } + } + + // ---- Engine validation: scratch containers, never live ones. + for _, snap := range manifest.Snapshots { + if snap.Role != "accessory" || snap.Method == "tar" { + continue + } + check := c.validateEngineSnapshot(ctx, manifest, snap, staging) + receipt.Checks = append(receipt.Checks, check) + } + + // ---- App check: boot the recorded image on the STAGED volumes. + receipt.Checks = append(receipt.Checks, c.appCheck(ctx, manifest, opts, staging)) + + receipt.ValidatedAt = timeNow(opts.Now).UTC() + receipt.RPOSeconds = int64(receipt.StartedAt.Sub(manifest.CreatedAt).Seconds()) + receipt.RTOSeconds = int64(receipt.ValidatedAt.Sub(receipt.StartedAt).Seconds()) + receipt.OK = allPassed(receipt.Checks) + + if receipt.OK { + fmt.Fprintf(c.out, "Isolated restore validated.\n") + } else { + fmt.Fprintf(c.out, "Isolated restore FAILED validation — staging kept at %s for inspection; nothing live was touched\n", staging) + } + writeReceipt(ctx, c.exec, staging+"/receipt.json", receipt) + return receipt, nil +} + +// fetchAndParseManifest reads and validates the manifest without side +// effects. App mismatch, bad schema, foreign kind, and malformed JSON all +// die here — before any mutation anywhere. +func (c *Client) fetchAndParseManifest(ctx context.Context, store BundleStore, app, id string) (*BundleManifest, error) { + data, err := store.FetchManifest(ctx, c.exec, app, id) + if err != nil { + return nil, err + } + m, err := ParseBundleManifest(data) + if err != nil { + return nil, err + } + if m.App != app { + return nil, fmt.Errorf("bundle %s belongs to app %q, not %q — refusing to restore onto the wrong app", m.ID, m.App, app) + } + if m.ID != id { + return nil, fmt.Errorf("manifest id %q does not match the requested bundle %q", m.ID, id) + } + return m, nil +} + +// secretsPreflight enforces the missing-keys-fail-first rule: when the +// bundle carries secret REFERENCES only, every referenced key must already +// exist on the target, and the missing ones are listed BEFORE any staging, +// download, or container runs. (In references mode the restored app cannot +// boot without them, so discovering that after mutating anything would be +// the classic restore-time surprise.) +func (c *Client) secretsPreflight(ctx context.Context, manifest *BundleManifest) error { + if manifest.Secrets.Mode != "references" || len(manifest.Secrets.Keys) == 0 { + return nil + } + secrets := secret.NewManager(c.exec) + present, err := secrets.List(ctx, manifest.App) + if err != nil { + return fmt.Errorf("listing target secrets for preflight: %w", err) + } + have := make(map[string]bool, len(present)) + for _, k := range present { + have[k] = true + } + var missing []string + for _, k := range manifest.Secrets.Keys { + if !have[k] { + missing = append(missing, k) + } + } + if len(missing) > 0 { + sort.Strings(missing) + return fmt.Errorf("bundle references secret(s) missing on this target: %s — set them first (`teploy secret set KEY=...`) or restore a bundle created with --include-secrets; nothing has been touched", strings.Join(missing, ", ")) + } + return nil +} + +// decryptProof: in encrypted mode the ciphertexts must actually decrypt with +// the key that will be available at cutover — the bundled age-key when +// present, else the target's own /deployments/.age-key. Proven on one member +// before any engine work; failure aborts with staging-only footprint. +func (c *Client) decryptProof(ctx context.Context, manifest *BundleManifest, staging string) error { + if manifest.Secrets.Mode != "encrypted" || len(manifest.Secrets.Included) == 0 { + return nil + } + var member string + for _, inc := range manifest.Secrets.Included { + if strings.HasSuffix(inc, ".age") { + member = inc + break + } + } + if member == "" { + return nil // only .env/credentials included; no ciphertext to prove + } + keyPath := deploymentsDir + "/.age-key" + if manifest.Secrets.AgeKeyIncluded { + keyPath = staging + "/age-key" + } + // age -d reads the file; output goes to the shell's trash — this is a + // decryptability proof, not an extraction. + cmd := fmt.Sprintf("age -d -i %s %s >/dev/null 2>&1", ssh.ShellQuote(keyPath), ssh.ShellQuote(staging+"/"+member)) + if _, err := c.exec.Run(ctx, cmd); err != nil { + if manifest.Secrets.AgeKeyIncluded { + return fmt.Errorf("the bundled age key cannot decrypt the bundled ciphertext %s — the bundle is internally inconsistent; nothing has been touched", member) + } + return fmt.Errorf("this target's age key cannot decrypt the bundle's secret material (ciphertext %s) — the bundle was encrypted on %s; supply that host's /deployments/.age-key on this target, or restore a references-only bundle and set the secrets by hand; nothing has been touched", member, manifest.Server) + } + return nil +} + +// validateEngineSnapshot boots a scratch engine, restores the dump into it, +// and checks the restored data is real. The scratch container is always torn +// down. The LIVE accessory (if any) is never contacted. +func (c *Client) validateEngineSnapshot(ctx context.Context, manifest *BundleManifest, snap SnapshotRecord, staging string) DRCheckResult { + check := DRCheckResult{ + Name: fmt.Sprintf("accessory:%s (%s)", snap.Name, snap.Engine), + Kind: "data", + } + scratch := manifest.App + "-" + snap.Name + "-drcheck" + teardown := func() { + c.exec.Run(context.WithoutCancel(ctx), "docker rm -f "+ssh.ShellQuote(scratch)+" >/dev/null 2>&1 || true") + } + // Remove stale scratch from an interrupted earlier run. + c.exec.Run(ctx, "docker rm -f "+ssh.ShellQuote(scratch)+" >/dev/null 2>&1 || true") + defer teardown() + + dump := staging + "/" + snap.Artifact + image := snap.Image + if image == "" { + check.Status = "skipped" + check.Detail = "manifest records no image for this snapshot (older bundle?) — data check skipped; the artifact remains staged" + return check + } + + switch snap.Engine { + case "postgres": + db := snap.EngineParams["db"] + user := snap.EngineParams["user"] + if user == "" { + user = "postgres" + } + if _, err := c.exec.Run(ctx, fmt.Sprintf( + "docker run -d --name %s -e POSTGRES_DB=%s -e POSTGRES_USER=%s -e POSTGRES_HOST_AUTH_METHOD=trust %s >/dev/null", + ssh.ShellQuote(scratch), ssh.ShellQuote(db), ssh.ShellQuote(user), ssh.ShellQuote(image))); err != nil { + return checkFailed(check, fmt.Errorf("starting scratch postgres: %w", err)) + } + if err := c.waitReady(ctx, scratch, fmt.Sprintf("pg_isready -U %s", ssh.ShellQuote(user)), 30); err != nil { + return checkFailed(check, err) + } + if _, err := c.exec.Run(ctx, postgresRestoreCmd(scratch, user, db, dump, staging+"/"+snap.Name+".sql")); err != nil { + return checkFailed(check, fmt.Errorf("restoring dump into scratch: %w", err)) + } + out, err := c.exec.Run(ctx, fmt.Sprintf( + "docker exec %s psql -tA -U %s %s -c \"SELECT COUNT(*) FROM information_schema.tables WHERE table_schema='public'\"", + ssh.ShellQuote(scratch), ssh.ShellQuote(user), ssh.ShellQuote(db))) + if err != nil { + return checkFailed(check, fmt.Errorf("verify query: %w", err)) + } + n := atoiSafe(strings.TrimSpace(out)) + check.Metric = fmt.Sprintf("tables=%d", n) + if n == 0 { + check.Status = "fail" + check.Detail = "restored database has zero tables" + return check + } + check.Status = "pass" + return check + + case "mysql", "mariadb": + db := snap.EngineParams["db"] + if _, err := c.exec.Run(ctx, fmt.Sprintf( + "docker run -d --name %s -e MYSQL_DATABASE=%s -e MYSQL_ALLOW_EMPTY_PASSWORD=yes %s >/dev/null", + ssh.ShellQuote(scratch), ssh.ShellQuote(db), ssh.ShellQuote(image))); err != nil { + return checkFailed(check, fmt.Errorf("starting scratch mysql: %w", err)) + } + if err := c.waitReady(ctx, scratch, "mysqladmin ping -u root --silent", 40); err != nil { + return checkFailed(check, err) + } + if _, err := c.exec.Run(ctx, mysqlRestoreCmd(scratch, db, "", dump, staging+"/"+snap.Name+".sql")); err != nil { + return checkFailed(check, fmt.Errorf("restoring dump into scratch: %w", err)) + } + out, err := c.exec.Run(ctx, fmt.Sprintf("docker exec %s sh -c %s", ssh.ShellQuote(scratch), + ssh.ShellQuote(fmt.Sprintf("mysql -u root -N -e \"SHOW TABLES\" %s | wc -l", db)))) + if err != nil { + return checkFailed(check, fmt.Errorf("verify query: %w", err)) + } + n := atoiSafe(strings.TrimSpace(out)) + check.Metric = fmt.Sprintf("tables=%d", n) + if n == 0 { + check.Status = "fail" + check.Detail = "restored database has zero tables" + return check + } + check.Status = "pass" + return check + + case "mongo": + if _, err := c.exec.Run(ctx, fmt.Sprintf("docker run -d --name %s %s >/dev/null", + ssh.ShellQuote(scratch), ssh.ShellQuote(image))); err != nil { + return checkFailed(check, fmt.Errorf("starting scratch mongo: %w", err)) + } + probe := "sh -c 'mongosh --quiet --eval 1 || mongo --quiet --eval 1'" + if err := c.waitReady(ctx, scratch, probe, 30); err != nil { + return checkFailed(check, err) + } + if _, err := c.exec.Run(ctx, mongoRestoreCmd(scratch, dump)); err != nil { + return checkFailed(check, fmt.Errorf("restoring dump into scratch: %w", err)) + } + out, err := c.exec.Run(ctx, fmt.Sprintf( + "docker exec %s sh -c 'mongosh --quiet --eval \"db.getMongo().getDBNames().length\" || mongo --quiet --eval \"db.getMongo().getDBNames().length\"'", + ssh.ShellQuote(scratch))) + if err != nil { + return checkFailed(check, fmt.Errorf("verify query: %w", err)) + } + check.Metric = "dbs=" + strings.TrimSpace(out) + check.Status = "pass" + return check + + case "redis": + // Seed a created (not running) container with the restored rdb, then + // start it: redis boots FROM the backup, and a corrupt rdb fails the + // readiness probe (same shape as verify.go). + rdb := staging + "/" + snap.Name + ".rdb" + if _, err := c.exec.Run(ctx, fmt.Sprintf( + "gunzip -c %s > %s && docker create --name %s %s >/dev/null && docker cp %s %s:/data/dump.rdb && docker start %s >/dev/null", + ssh.ShellQuote(dump), ssh.ShellQuote(rdb), ssh.ShellQuote(scratch), ssh.ShellQuote(image), + ssh.ShellQuote(rdb), ssh.ShellQuote(scratch), ssh.ShellQuote(scratch))); err != nil { + return checkFailed(check, fmt.Errorf("seeding scratch redis: %w", err)) + } + if err := c.waitReady(ctx, scratch, "redis-cli ping", 20); err != nil { + return checkFailed(check, err) + } + out, err := c.exec.Run(ctx, fmt.Sprintf("docker exec %s redis-cli dbsize", ssh.ShellQuote(scratch))) + if err != nil { + return checkFailed(check, fmt.Errorf("verify query: %w", err)) + } + check.Metric = "keys=" + strings.TrimSpace(strings.TrimPrefix(strings.TrimSpace(out), "(integer)")) + check.Status = "pass" + return check + + default: + check.Status = "skipped" + check.Detail = fmt.Sprintf("engine %q has no scratch validator (yet) — artifact staged, cutover restores it verbatim", snap.Engine) + return check + } +} + +// appCheck boots the bundle's recorded image on the STAGED volumes with the +// bundled env (when the operator included it) and reports whether it reaches +// running state. It is isolated: no network alias, no published port, no +// Caddy involvement — it can never receive production traffic. +func (c *Client) appCheck(ctx context.Context, manifest *BundleManifest, opts BundleRestoreOptions, staging string) DRCheckResult { + check := DRCheckResult{Name: "app", Kind: "app"} + + var appState state.AppState + if err := json.Unmarshal(manifest.State, &appState); err != nil { + check.Status = "skipped" + check.Detail = "bundle state unparseable — application check skipped (data checks still ran)" + return check + } + image := manifest.AppRun.Image + if image == "" { + image = appState.ImageRef + } + if image == "" { + check.Status = "skipped" + check.Detail = "bundle carries no deployed image reference — application check skipped (data checks still ran)" + return check + } + scratch := manifest.App + "-dr-appcheck" + c.exec.Run(ctx, "docker rm -f "+ssh.ShellQuote(scratch)+" >/dev/null 2>&1 || true") + defer c.exec.Run(context.WithoutCancel(ctx), "docker rm -f "+ssh.ShellQuote(scratch)+" >/dev/null 2>&1 || true") + + args := []string{"docker", "run", "-d", "--name", ssh.ShellQuote(scratch), + "--label", "teploy.app=" + manifest.App, "--label", "teploy.role=dr-appcheck"} + if envFile := stagedEnvFile(manifest, staging); envFile != "" { + args = append(args, "--env-file", ssh.ShellQuote(envFile)) + } + for _, vol := range sortedVolumeMounts(opts.Config) { + args = append(args, "-v", ssh.ShellQuote(staging+"/volumes/"+vol.name+":"+vol.dest)) + } + args = append(args, ssh.ShellQuote(image)) + // The recorded container command, element-wise quoted (a command with + // quoted spaces cannot be represented — recorded verbatim by deploy as + // a single string; splitting on fields matches how teploy.yml's legacy + // command: is documented). + for _, w := range strings.Fields(manifest.AppRun.Cmd) { + args = append(args, ssh.ShellQuote(w)) + } + + if _, err := c.exec.Run(ctx, strings.Join(args, " ")); err != nil { + // An unavailable image is a SKIP with the reason, not a silent + // pass: the operator sees exactly what was not proven. + if strings.Contains(err.Error(), "Unable to find image") || strings.Contains(err.Error(), "pull access denied") || strings.Contains(err.Error(), "not found") { + check.Status = "skipped" + check.Detail = fmt.Sprintf("image %s unavailable on this host and could not be pulled — application check skipped", appState.ImageRef) + return check + } + return checkFailed(check, fmt.Errorf("starting scratch app container: %w", err)) + } + // Bounded wait for "running": docker ps filtering by exact name. + waitCmd := fmt.Sprintf( + "for i in $(seq 1 15); do s=$(docker inspect -f '{{.State.Status}}' %s 2>/dev/null) && [ \"$s\" = running ] && exit 0; sleep 2; done; docker logs --tail 20 %s >&2; exit 1", + ssh.ShellQuote(scratch), ssh.ShellQuote(scratch)) + if _, err := c.exec.Run(ctx, waitCmd); err != nil { + return checkFailed(check, fmt.Errorf("app container never reached running state: %w", err)) + } + check.Status = "pass" + check.Metric = fmt.Sprintf("image=%s running", appState.ImageRef) + return check +} + +func stagedEnvFile(manifest *BundleManifest, staging string) string { + for _, inc := range manifest.Secrets.Included { + if inc == "env/.env" { + return staging + "/env/.env" + } + } + return "" +} + +type volumeMount struct{ name, dest string } + +func sortedVolumeMounts(cfg config.AppConfig) []volumeMount { + out := make([]volumeMount, 0, len(cfg.Volumes)) + for name, dest := range cfg.Volumes { + out = append(out, volumeMount{name, dest}) + } + sort.Slice(out, func(i, j int) bool { return out[i].name < out[j].name }) + return out +} + +// CutoverBundle is the EXPLICIT promotion step. It requires a validated +// staged restore (its receipt), re-runs the secrets preflight, and only then +// — under the app lock, with two-phase promotion — replaces live data. +// Originals are preserved in recovery directories named in the receipt. +func (c *Client) CutoverBundle(ctx context.Context, opts BundleRestoreOptions) (*CutoverReceipt, error) { + startedAt := timeNow(opts.Now) + staging := DRStagingPath(opts.App, opts.ID) + if !drStagingRE.MatchString(staging) { + return nil, fmt.Errorf("internal error: staging path %q fails the safety pattern", staging) + } + + // The staged restore must exist AND have passed validation. + receiptData, present, err := state.ReadRemoteFile(ctx, c.exec, staging+"/receipt.json") + if err != nil { + return nil, fmt.Errorf("reading the staged restore receipt: %w", err) + } + if !present { + return nil, fmt.Errorf("%w: %s", ErrValidationRequired, staging) + } + var receipt RestoreReceipt + if err := json.Unmarshal(receiptData, &receipt); err != nil { + return nil, fmt.Errorf("parsing the staged restore receipt: %w", err) + } + if !receipt.OK { + return nil, fmt.Errorf("the staged restore for bundle %s FAILED validation — cutover refused; inspect %s or re-run teploy dr restore", opts.ID, staging) + } + + manifestData, present, err := state.ReadRemoteFile(ctx, c.exec, staging+"/manifest.json") + if err != nil || !present { + return nil, fmt.Errorf("reading the staged bundle manifest from %s: %v", staging, err) + } + manifest, err := ParseBundleManifest(manifestData) + if err != nil { + return nil, err + } + if manifest.App != opts.App { + return nil, fmt.Errorf("staged bundle belongs to app %q, not %q", manifest.App, opts.App) + } + if err := c.secretsPreflight(ctx, manifest); err != nil { + return nil, err + } + + // Cross-check: every accessory/volume the bundle captured must exist in + // the restore-time teploy.yml, or cutover would silently drop data. + for _, snap := range manifest.Snapshots { + switch snap.Role { + case "accessory": + if _, ok := opts.Config.Accessories[snap.Name]; !ok { + return nil, fmt.Errorf("bundle captured accessory %q but the restore-time teploy.yml does not define it — add it (or accept losing its data by removing it from the bundle) before cutover", snap.Name) + } + case "app-volume": + if _, ok := opts.Config.Volumes[snap.Name]; !ok { + return nil, fmt.Errorf("bundle captured volume %q but the restore-time teploy.yml does not define it — map it in teploy.yml before cutover", snap.Name) + } + } + } + + out := &CutoverReceipt{ + SchemaVersion: RestoreReceiptSchemaVersion, + BundleID: manifest.ID, + App: opts.App, + StartedAt: startedAt.UTC(), + NextStep: "teploy deploy — bring the app container and routing live from the restored state", + } + + // App lock: cutover is a mutating critical section like a deploy (C01). + lk, err := state.AcquireLockFenced(ctx, c.exec, opts.App) + if err != nil { + return nil, fmt.Errorf("acquiring the app lock for cutover: %w", err) + } + defer state.ReleaseLockFenced(c.exec, lk, opts.App) + + // Stop live containers (web + accessories). They are STOPPED, not + // removed — an aborted cutover restarts them. + liveOut, err := c.exec.Run(ctx, fmt.Sprintf( + "docker ps --filter label=teploy.app=%s --format '{{.Names}}'", ssh.ShellQuote(opts.App))) + if err != nil { + return nil, fmt.Errorf("listing live containers: %w", err) + } + var stopped []string + for _, n := range strings.Fields(strings.TrimSpace(liveOut)) { + if n == "" { + continue + } + fmt.Fprintf(c.out, "Stopping %s...\n", n) + if _, err := c.exec.Run(ctx, "docker stop "+ssh.ShellQuote(n)); err != nil { + c.restartContainers(context.WithoutCancel(ctx), stopped) + return nil, fmt.Errorf("stopping %s for cutover (it was left as-is): %w", n, err) + } + stopped = append(stopped, n) + } + out.Stopped = stopped + + fail := func(err error) (*CutoverReceipt, error) { + // Best-effort restart of the stopped containers so the pre-cutover + // state is not left dark; staged data and recovery dirs are KEPT + // and named. If a promotion already happened for some path, the + // receipt lists it and the error says exactly where originals are. + c.restartContainers(context.WithoutCancel(ctx), stopped) + writeReceipt(context.WithoutCancel(ctx), c.exec, staging+"/cutover-receipt.json", out) + return out, fmt.Errorf("%w — staged data kept at %s, pre-cutover originals kept in %v; stopped containers were restarted", err, staging, out.RecoveryDirs) + } + + // Engine-dump accessories first: their data lands through the engine, + // and a failure here leaves the volume/generic promotions untouched. + for _, snap := range manifest.Snapshots { + if snap.Role != "accessory" || snap.Method == "tar" { + continue + } + accCfg := opts.Config.Accessories[snap.Name] + accDir := fmt.Sprintf("%s/%s/accessories/%s", deploymentsDir, opts.App, snap.Name) + + // Preserve any pre-existing engine data dir: the dump must land in + // a FRESH engine (pg_dump has no DROP unless asked; restoring over + // populated tables errors mid-way). Moved aside, kept, named. + nonEmpty, err := c.exec.Run(ctx, fmt.Sprintf("if [ -z \"$(ls -A %s 2>/dev/null)\" ]; then printf 'empty\\n'; else printf 'nonempty\\n'; fi", ssh.ShellQuote(accDir))) + if err != nil { + return fail(fmt.Errorf("inspecting accessory dir %s: %w", accDir, err)) + } + if strings.TrimSpace(nonEmpty) == "nonempty" { + // Only keys with data ON DISK are preserved — a configured key + // that never materialized (volume added to teploy.yml after + // the bundle, a re-run after a partial failure) has nothing + // to move, and an accDir holding only env/credential files + // must not abort the cutover. + var toMove []string + for _, volKey := range sortedVolumeKeys(accCfg) { + existsOut, err := c.exec.Run(ctx, fmt.Sprintf("if [ -e %s ]; then printf 'present\\n'; else printf 'absent\\n'; fi", + ssh.ShellQuote(accDir+"/"+volKey))) + if err != nil { + return fail(fmt.Errorf("inspecting %s volume dir %s: %w", snap.Name, volKey, err)) + } + if strings.TrimSpace(existsOut) == "present" { + toMove = append(toMove, volKey) + } + } + if len(toMove) > 0 { + recOut, err := c.exec.Run(ctx, "mktemp -d "+ssh.ShellQuote(accDir+".pre-cutover.XXXXXX")) + if err != nil { + return fail(fmt.Errorf("creating pre-cutover copy dir for %s: %w", accDir, err)) + } + recDir := strings.TrimSpace(recOut) + // Move each VOLUME directory (the engine's data) aside; env + // files and credentials in accDir stay for the fresh accessory. + accParent := fmt.Sprintf("%s/%s/accessories", deploymentsDir, opts.App) + for _, volKey := range toMove { + move := fmt.Sprintf("mv -f %s %s", ssh.ShellQuote(accDir+"/"+volKey), ssh.ShellQuote(recDir+"/"+volKey)) + if _, err := c.exec.Run(ctx, move); err != nil { + // Engine images chown their data dir to their own uid + // with mode 700 (postgres = uid 70), and a non-root + // deploy user may not be able to rename it. Same + // reality reconcileDataOwnership solves: retry + // through a throwaway root container (the accessory + // image itself provides the shell). + if mvErr := c.rootMoveAside(ctx, accCfg.Image, accParent, snap.Name, pathBase(recDir), volKey); mvErr != nil { + return fail(fmt.Errorf("moving aside pre-cutover data for %s volume %s (plain mv: %v; root move: %v)", snap.Name, volKey, err, mvErr)) + } + } + } + out.RecoveryDirs = append(out.RecoveryDirs, recDir) + fmt.Fprintf(c.out, "Pre-cutover data for %s preserved at %s\n", snap.Name, recDir) + } + } + + // Start the accessory from config (fresh dirs), then pipe the dump. + mgr := accessories.NewManager(c.exec, c.out) + env, err := mgr.EnsureRunning(ctx, opts.App, snap.Name, accCfg) + if err != nil { + return fail(fmt.Errorf("starting accessory %s for cutover: %w", snap.Name, err)) + } + container := accessories.ContainerName(opts.App, snap.Name) + dump := staging + "/" + snap.Artifact + var restoreErr error + switch snap.Engine { + case "postgres": + db, user := postgresDBAndUser(opts.App, engineEnv(snap, env, accCfg)) + if err := c.waitReady(ctx, container, fmt.Sprintf("pg_isready -U %s", ssh.ShellQuote(user)), 30); err != nil { + restoreErr = err + } else { + _, restoreErr = c.exec.Run(ctx, postgresRestoreCmd(container, user, db, dump, staging+"/"+snap.Name+".cutover.sql")) + } + case "mysql", "mariadb": + db := mysqlDB(opts.App, engineEnv(snap, env, accCfg)) + if err := c.waitReady(ctx, container, "mysqladmin ping -u root --silent", 40); err != nil { + restoreErr = err + } else { + _, restoreErr = c.exec.Run(ctx, mysqlRestoreCmd(container, db, mysqlRootPassword(engineEnv(snap, env, accCfg)), dump, staging+"/"+snap.Name+".cutover.sql")) + } + case "mongo": + _, restoreErr = c.exec.Run(ctx, mongoRestoreCmd(container, dump)) + case "redis": + if err := c.redisAOFPreflight(ctx, opts.App, snap.Name); err != nil { + restoreErr = err + } else { + _, restoreErr = c.exec.Run(ctx, redisRestoreScript(container, dump, staging+"/"+snap.Name+".cutover.rdb", staging+"/"+snap.Name+".cutover-old.rdb")) + } + default: + restoreErr = fmt.Errorf("engine %q has no cutover restore path", snap.Engine) + } + if restoreErr != nil { + return fail(fmt.Errorf("restoring %s dump at cutover: %w (its pre-cutover data, if any, is in the recovery dirs)", snap.Name, restoreErr)) + } + fmt.Fprintf(c.out, "Restored %s dump into %s\n", snap.Name, container) + } + + // Generic accessory tars + app volumes: two-phase promotion. + for _, snap := range manifest.Snapshots { + var stageDir, liveDir string + switch snap.Role { + case "accessory": + if snap.Method != "tar" { + continue + } + stageDir = staging + "/accessories/" + snap.Name + liveDir = fmt.Sprintf("%s/%s/accessories/%s", deploymentsDir, opts.App, snap.Name) + case "app-volume": + stageDir = staging + "/volumes/" + snap.Name + liveDir = fmt.Sprintf("%s/%s/volumes/%s", deploymentsDir, opts.App, snap.Name) + default: + continue + } + recoveryDir, err := promoteStaged(ctx, c.exec, stageDir, liveDir, c.out, staging) + if err != nil { + // promoteStaged already restored the originals on rollback; the + // recovery-incomplete variant preserves them split across dirs. + return fail(fmt.Errorf("promoting %s at cutover: %w", snap.Name, err)) + } + if recoveryDir != "" { + // Deliberately KEPT (not deleted): recovery is exactly the + // moment an operator may want the pre-cutover copy back, and + // the receipt names where it is. Remove manually once confident. + out.RecoveryDirs = append(out.RecoveryDirs, recoveryDir) + } + out.Promoted = append(out.Promoted, liveDir) + } + + // State, release records, and secret material land LAST — after all + // data promotions succeeded, so a failed promotion never leaves the + // target claiming a generation it does not have. + if err := c.installBundleState(ctx, manifest, staging); err != nil { + return fail(fmt.Errorf("installing restored state: %w", err)) + } + + out.CompletedAt = timeNow(opts.Now).UTC() + writeReceipt(ctx, c.exec, staging+"/cutover-receipt.json", out) + fmt.Fprintf(c.out, "Cutover complete. Pre-cutover originals kept in %v\n", out.RecoveryDirs) + fmt.Fprintf(c.out, "Next: %s\n", out.NextStep) + return out, nil +} + +// rootMoveAside renames mountDir/srcDir/srcName to mountDir/dstDir/srcName +// through a throwaway root container of the accessory's own image — the +// reconcileDataOwnership pattern for engine data dirs a non-root deploy +// user cannot move (directory renames need write on the directory itself, +// and engines chown their data to their own uid). +func (c *Client) rootMoveAside(ctx context.Context, image, mountDir, srcDir, dstDir, name string) error { + if !safeName.MatchString(srcDir) || !safeName.MatchString(dstDir) || !safeName.MatchString(name) { + return fmt.Errorf("refusing root-container move of unsafe path segment (%q, %q, %q)", srcDir, dstDir, name) + } + inner := fmt.Sprintf("mv /w/%s/%s /w/%s/%s", srcDir, name, dstDir, name) + cmd := fmt.Sprintf("docker run --rm --user 0 -v %s:/w --entrypoint sh %s -c %s", + ssh.ShellQuote(mountDir), ssh.ShellQuote(image), ssh.ShellQuote(inner)) + if _, err := c.exec.Run(ctx, cmd); err != nil { + return err + } + return nil +} + +func sortedVolumeKeys(cfg config.AccessoryConfig) []string { + keys := make([]string, 0, len(cfg.Volumes)) + for k := range cfg.Volumes { + keys = append(keys, k) + } + sort.Strings(keys) + return keys +} + +func pathBase(p string) string { + if i := strings.LastIndexByte(p, '/'); i >= 0 { + return p[i+1:] + } + return p +} + +// engineEnv merges the manifest's non-secret engine params with the running +// accessory's resolved env so cutover restores land in the same database the +// bundle was dumped from (fresh-host edge: POSTGRES_* env from config). +func engineEnv(snap SnapshotRecord, liveEnv map[string]string, cfg config.AccessoryConfig) map[string]string { + merged := make(map[string]string) + for k, v := range cfg.Env { + merged[k] = v + } + for k, v := range liveEnv { + merged[k] = v // resolved values (auto passwords, DATABASE_URL) win + } + if p := snap.EngineParams; p != nil { + if db := p["db"]; db != "" && merged["POSTGRES_DB"] == "" && merged["MYSQL_DATABASE"] == "" { + switch snap.Engine { + case "postgres": + merged["POSTGRES_DB"] = db + case "mysql", "mariadb": + merged["MYSQL_DATABASE"] = db + } + } + } + return merged +} + +// installBundleState writes the bundle's state.json, release records and +// secret material into /deployments/ — atomically per file, 0600. +func (c *Client) installBundleState(ctx context.Context, manifest *BundleManifest, staging string) error { + appDir := fmt.Sprintf("%s/%s", deploymentsDir, manifest.App) + if _, err := c.exec.Run(ctx, "mkdir -p "+ssh.ShellQuote(appDir+"/meta")+" "+ssh.ShellQuote(appDir+"/secrets")); err != nil { + return fmt.Errorf("preparing state dirs: %w", err) + } + if err := c.exec.Upload(ctx, strings.NewReader(string(manifest.State)+"\n"), appDir+"/state.json", "0600"); err != nil { + return fmt.Errorf("writing restored state.json: %w", err) + } + for _, rec := range manifest.ReleaseRecords { + var r struct { + Hash string `json:"hash"` + } + if err := json.Unmarshal(rec, &r); err != nil || r.Hash == "" { + continue // a record without identity cannot be keyed; skip loudly below + } + if err := c.exec.Upload(ctx, strings.NewReader(string(rec)+"\n"), fmt.Sprintf("%s/meta/%s.json", appDir, r.Hash), "0600"); err != nil { + return fmt.Errorf("writing release record %s: %w", r.Hash, err) + } + } + if manifest.Secrets.Mode == "encrypted" { + for _, inc := range manifest.Secrets.Included { + switch { + case strings.HasPrefix(inc, "secrets/"): + key := strings.TrimSuffix(strings.TrimPrefix(inc, "secrets/"), ".age") + if _, err := c.exec.Run(ctx, fmt.Sprintf("cp -p %s %s", ssh.ShellQuote(staging+"/"+inc), ssh.ShellQuote(appDir+"/secrets/"+key+".age"))); err != nil { + return fmt.Errorf("installing secret ciphertext %s: %w", key, err) + } + case inc == "env/.env": + if _, err := c.exec.Run(ctx, fmt.Sprintf("cp -p %s %s", ssh.ShellQuote(staging+"/"+inc), ssh.ShellQuote(appDir+"/.env"))); err != nil { + return fmt.Errorf("installing restored .env: %w", err) + } + case strings.HasPrefix(inc, "env/credentials/"): + name := strings.TrimPrefix(inc, "env/credentials/") + accDir := fmt.Sprintf("%s/%s/accessories/%s", deploymentsDir, manifest.App, name) + if _, err := c.exec.Run(ctx, "mkdir -p "+ssh.ShellQuote(accDir)); err != nil { + return fmt.Errorf("preparing accessory dir for credentials: %w", err) + } + if _, err := c.exec.Run(ctx, fmt.Sprintf("cp -p %s %s", ssh.ShellQuote(staging+"/"+inc), ssh.ShellQuote(accDir+"/credentials"))); err != nil { + return fmt.Errorf("installing credentials for %s: %w", name, err) + } + } + } + if manifest.Secrets.AgeKeyIncluded { + // Install the bundled age key as the target's own ONLY when the + // target has none — never silently replace an existing key. + hasKey, err := c.remoteExists(ctx, deploymentsDir+"/.age-key") + if err != nil { + return fmt.Errorf("checking the target age key: %w", err) + } + if !hasKey { + if _, err := c.exec.Run(ctx, fmt.Sprintf("cp -p %s %s && chmod 600 %s", + ssh.ShellQuote(staging+"/age-key"), ssh.ShellQuote(deploymentsDir+"/.age-key"), ssh.ShellQuote(deploymentsDir+"/.age-key"))); err != nil { + return fmt.Errorf("installing the bundled age key: %w", err) + } + } + } + } + return nil +} + +func (c *Client) failReceipt(ctx context.Context, receipt *RestoreReceipt, err error) error { + receipt.Checks = append(receipt.Checks, DRCheckResult{Name: "restore", Kind: "data", Status: "fail", Detail: err.Error()}) + receipt.ValidatedAt = time.Now().UTC() + receipt.OK = false + writeReceipt(context.WithoutCancel(ctx), c.exec, receipt.StagingPath+"/receipt.json", receipt) + return err +} + +func writeReceipt(ctx context.Context, exec ssh.Executor, path string, v any) { + data, err := json.MarshalIndent(v, "", " ") + if err != nil { + return + } + _ = exec.Upload(context.WithoutCancel(ctx), strings.NewReader(string(data)+"\n"), path, "0600") +} + +func allPassed(checks []DRCheckResult) bool { + for _, ck := range checks { + if ck.Status == "fail" { + return false + } + } + return true +} + +func checkFailed(check DRCheckResult, err error) DRCheckResult { + check.Status = "fail" + check.Detail = err.Error() + return check +} + +func atoiSafe(s string) int { + n := 0 + for _, c := range s { + if c < '0' || c > '9' { + break + } + n = n*10 + int(c-'0') + } + return n +} + +func timeNow(f func() time.Time) time.Time { + if f != nil { + return f() + } + return time.Now() +} diff --git a/internal/backup/bundle_restore_test.go b/internal/backup/bundle_restore_test.go new file mode 100644 index 0000000..72c9014 --- /dev/null +++ b/internal/backup/bundle_restore_test.go @@ -0,0 +1,464 @@ +package backup + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "strings" + "testing" + "time" + + "github.com/useteploy/teploy/internal/ssh" +) + +const drTestBundleID = "20260923-101500-0123456789abcdef" + +// drTestManifest builds a manifest for myapp with one postgres accessory +// snapshot and one app-volume snapshot, references-only secrets. +func drTestManifest(keys ...string) *BundleManifest { + return &BundleManifest{ + SchemaVersion: BundleSchemaVersion, + Kind: BundleKindDR, + ID: drTestBundleID, + App: "myapp", + Server: "dr-host", + CreatedAt: time.Date(2026, 9, 23, 9, 0, 0, 0, time.UTC), + State: json.RawMessage(drTestState), + Secrets: SecretsRecord{Mode: "references", Keys: keys}, + Snapshots: []SnapshotRecord{ + {Name: "db", Role: "accessory", Engine: "postgres", Method: "pg_dump", + Consistency: consistencyEngineDump, Detection: "image-pattern", + Artifact: "accessories/db.sql.gz", Image: "postgres:16", + EngineParams: map[string]string{"db": "appdb", "user": "appuser"}}, + {Name: "data", Role: "app-volume", Method: "tar", + Consistency: consistencyCrash, Artifact: "volumes/data.tar.gz"}, + }, + } +} + +func drRestoreMock(manifest *BundleManifest, extra ...ssh.MockCommand) *ssh.MockExecutor { + staging := DRStagingPath("myapp", manifest.ID) + manifestJSON, _ := json.Marshal(manifest) + // extras are PREPENDED so injected failures outrank the base successes. + mock := ssh.NewMockExecutor("dr-host", append(extra, []ssh.MockCommand{ + {Match: "rm -rf '" + staging, Output: ""}, + {Match: "umask 077; mkdir -p '" + staging, Output: ""}, + {Match: "mkdir -p '" + staging, Output: ""}, + {Match: "cp -p", Output: ""}, + {Match: "gzip -t", Output: ""}, + {Match: "tar -xzf", Output: ""}, + {Match: "docker rm -f", Output: ""}, + {Match: "docker run -d", Output: "cid123\n"}, + {Match: "for i in $(seq", Output: ""}, + {Match: "gunzip -c", Output: ""}, + {Match: "docker exec 'myapp-db-drcheck' psql -tA", Output: "3\n"}, + }...)...) + // The store's manifest, readable through the framed auto-answer. + mock.Files["/var/drstore/myapp/dr/"+manifest.ID+"/manifest.json"] = manifestJSON + return mock +} + +// TestRestoreBundleIsolated_MissingSecretKeysFailBeforeMutation — the C07 +// acceptance: in references mode, keys missing on the target abort BEFORE +// any staging, download, or docker activity. Only read-only commands may +// have run. +func TestRestoreBundleIsolated_MissingSecretKeysFailBeforeMutation(t *testing.T) { + manifest := drTestManifest("API_KEY", "DATABASE_URL") + mock := drRestoreMock(manifest) + // Target has NO secrets: the framed auto-answer reports the secrets dir + // absent (nothing seeded). + + client := NewClient(mock, &bytes.Buffer{}) + _, err := client.RestoreBundleIsolated(context.Background(), BundleRestoreOptions{ + App: "myapp", ID: manifest.ID, Config: drTestConfig(), + }, DirBundleStore{Root: "/var/drstore"}) + if err == nil || !strings.Contains(err.Error(), "API_KEY, DATABASE_URL") { + t.Fatalf("expected missing-keys error naming the keys, got %v", err) + } + if !strings.Contains(err.Error(), "nothing has been touched") { + t.Errorf("error must state that nothing was touched: %v", err) + } + for _, call := range mock.Calls { + if strings.Contains(call, "docker ") || strings.Contains(call, "mkdir -p") || strings.HasPrefix(call, "rm -rf") || strings.Contains(call, "cp -p") { + t.Errorf("mutating command ran before preflight passed: %s", call) + } + } +} + +// TestRestoreBundleIsolated_WrongAppRefused: a bundle for another app never +// reaches staging. +func TestRestoreBundleIsolated_WrongAppRefused(t *testing.T) { + manifest := drTestManifest() + manifest.App = "otherapp" + mock := drRestoreMock(manifest) + _, err := NewClient(mock, &bytes.Buffer{}).RestoreBundleIsolated(context.Background(), + BundleRestoreOptions{App: "myapp", ID: manifest.ID, Config: drTestConfig()}, + DirBundleStore{Root: "/var/drstore"}) + if err == nil || !strings.Contains(err.Error(), "wrong app") { + t.Fatalf("expected wrong-app refusal, got %v", err) + } +} + +// TestRestoreBundleIsolated_CorruptBundleFailsBeforeEngines: a gzip member +// that fails integrity aborts before any engine boot; staging is the only +// thing that existed. +func TestRestoreBundleIsolated_CorruptBundleFailsBeforeEngines(t *testing.T) { + manifest := drTestManifest() + mock := drRestoreMock(manifest, + ssh.MockCommand{Match: "gzip -t", Err: errors.New("exit status 1: gzip: invalid compressed data")}, + ) + _, err := NewClient(mock, &bytes.Buffer{}).RestoreBundleIsolated(context.Background(), + BundleRestoreOptions{App: "myapp", ID: manifest.ID, Config: drTestConfig()}, + DirBundleStore{Root: "/var/drstore"}) + if err == nil || !strings.Contains(err.Error(), "corrupt") { + t.Fatalf("expected corrupt-bundle refusal, got %v", err) + } + for _, call := range mock.Calls { + if strings.Contains(call, "docker run") { + t.Errorf("engine booted despite corrupt bundle: %s", call) + } + } +} + +// TestRestoreBundleIsolated_ReceiptRPORTO: the happy path produces a receipt +// with passing data + app checks, RPO measured from bundle creation, RTO +// from the restore clock, scratch containers torn down, and NOTHING under +// /deployments touched. +func TestRestoreBundleIsolated_ReceiptRPORTO(t *testing.T) { + manifest := drTestManifest() + mock := drRestoreMock(manifest) + start := time.Date(2026, 9, 23, 11, 0, 0, 0, time.UTC) + validated := start.Add(90 * time.Second) + calls := 0 + now := func() time.Time { + calls++ + if calls == 1 { + return start + } + return validated + } + + client := NewClient(mock, &bytes.Buffer{}) + receipt, err := client.RestoreBundleIsolated(context.Background(), BundleRestoreOptions{ + App: "myapp", ID: manifest.ID, Config: drTestConfig(), Now: now, + }, DirBundleStore{Root: "/var/drstore"}) + if err != nil { + t.Fatalf("RestoreBundleIsolated: %v", err) + } + if !receipt.OK { + t.Fatalf("expected OK receipt, checks: %+v", receipt.Checks) + } + // RPO: restore start (11:00) - bundle creation (09:00) = 7200s. + if receipt.RPOSeconds != 7200 { + t.Errorf("RPO = %d, want 7200", receipt.RPOSeconds) + } + if receipt.RTOSeconds != 90 { + t.Errorf("RTO = %d, want 90", receipt.RTOSeconds) + } + if receipt.StagingPath != DRStagingPath("myapp", manifest.ID) { + t.Errorf("staging path: %s", receipt.StagingPath) + } + var dataCheck, appCheck bool + for _, ck := range receipt.Checks { + if ck.Kind == "data" && ck.Status == "pass" && ck.Metric == "tables=3" { + dataCheck = true + } + if ck.Kind == "app" && ck.Status == "pass" { + appCheck = true + } + } + if !dataCheck || !appCheck { + t.Errorf("expected passing data+app checks: %+v", receipt.Checks) + } + // Scratch teardown happened. + var sawTeardown bool + for _, call := range mock.Calls { + if strings.Contains(call, "docker rm -f 'myapp-db-drcheck'") { + sawTeardown = true + } + if strings.Contains(call, "/deployments/myapp") { + t.Errorf("isolated restore must not touch /deployments/myapp: %s", call) + } + } + if !sawTeardown { + t.Errorf("scratch container was not torn down") + } +} + +// TestRestoreBundleIsolated_EncryptedWithoutKeyFailsBeforeEngines: an +// encrypted bundle without the age key must prove the TARGET key decrypts +// it — failure aborts before any engine work. +func TestRestoreBundleIsolated_EncryptedWithoutKeyFailsBeforeEngines(t *testing.T) { + manifest := drTestManifest() + manifest.Secrets = SecretsRecord{ + Mode: "encrypted", + Keys: []string{"API_KEY"}, + Included: []string{"secrets/API_KEY.age"}, + } + mock := drRestoreMock(manifest, + ssh.MockCommand{Match: "age -d -i '/deployments/.age-key'", Err: errors.New("exit status 1: age: no identity matched")}, + ) + _, err := NewClient(mock, &bytes.Buffer{}).RestoreBundleIsolated(context.Background(), + BundleRestoreOptions{App: "myapp", ID: manifest.ID, Config: drTestConfig()}, + DirBundleStore{Root: "/var/drstore"}) + if err == nil || !strings.Contains(err.Error(), "age key") { + t.Fatalf("expected decrypt-proof failure naming the age key, got %v", err) + } + for _, call := range mock.Calls { + if strings.Contains(call, "docker run") { + t.Errorf("engine booted despite undecryptable bundle: %s", call) + } + } +} + +// ---- Cutover ---- + +func drCutoverMock(receipt *RestoreReceipt, manifest *BundleManifest, extra ...ssh.MockCommand) *ssh.MockExecutor { + staging := DRStagingPath("myapp", manifest.ID) + receiptJSON, _ := json.Marshal(receipt) + manifestJSON, _ := json.Marshal(manifest) + // extras are PREPENDED so injected failures outrank the base successes. + mock := ssh.NewMockExecutor("dr-host", append(extra, []ssh.MockCommand{ + {Match: "mkdir /deployments/myapp/.lock", Output: ""}, + {Match: "docker ps --filter label=teploy.app='myapp'", Output: "myapp-web\nmyapp-db\n"}, + {Match: "docker stop", Output: ""}, + {Match: "docker start", Output: ""}, + {Match: "if [ -z \"$(ls -A", Output: "empty\n"}, + {Match: "docker inspect -f '{{.State.Status}}' 'myapp-db'", Output: "exited\n"}, + {Match: "docker rm -f", Output: ""}, + {Match: "mkdir -p /deployments/myapp/accessories/db", Output: ""}, + {Match: "mkdir -p /deployments/myapp/accessories/db/pgdata", Output: ""}, + {Match: "docker image inspect", Output: "\n"}, + {Match: "docker run --detach", Output: ""}, + {Match: "for i in $(seq", Output: ""}, + {Match: "gunzip -c", Output: ""}, + {Match: "mkdir -p '/deployments/myapp/volumes/data'", Output: ""}, + {Match: "mktemp -d '/deployments", Output: "/deployments/myapp/recovery123\n"}, + {Match: "find ", Output: ""}, + {Match: "cp -a ", Output: ""}, + {Match: "mv -f ", Output: ""}, + {Match: "cp -p", Output: ""}, + {Match: "rm -rf", Output: ""}, + {Match: "mkdir -p '/deployments/myapp/meta' '/deployments/myapp/secrets'", Output: ""}, + {Match: "mkdir -p /deployments/myapp/meta", Output: ""}, + }...)...) + mock.Files[staging+"/receipt.json"] = receiptJSON + mock.Files[staging+"/manifest.json"] = manifestJSON + return mock +} + +func drOKReceipt() *RestoreReceipt { + return &RestoreReceipt{ + SchemaVersion: RestoreReceiptSchemaVersion, + BundleID: drTestBundleID, + App: "myapp", + OK: true, + StagingPath: DRStagingPath("myapp", drTestBundleID), + } +} + +// TestCutover_RequiresValidatedStaging: no staged restore, or one that +// FAILED validation, must never promote. +func TestCutover_RequiresValidatedStaging(t *testing.T) { + manifest := drTestManifest() + + noReceipt := ssh.NewMockExecutor("dr-host") + _, err := NewClient(noReceipt, &bytes.Buffer{}).CutoverBundle(context.Background(), + BundleRestoreOptions{App: "myapp", ID: manifest.ID, Config: drTestConfig()}) + if !errors.Is(err, ErrValidationRequired) { + t.Fatalf("expected ErrValidationRequired, got %v", err) + } + + bad := drOKReceipt() + bad.OK = false + mock := drCutoverMock(bad, manifest) + _, err = NewClient(mock, &bytes.Buffer{}).CutoverBundle(context.Background(), + BundleRestoreOptions{App: "myapp", ID: manifest.ID, Config: drTestConfig()}) + if err == nil || !strings.Contains(err.Error(), "FAILED validation") { + t.Fatalf("expected validation refusal, got %v", err) + } + for _, call := range mock.Calls { + if strings.Contains(call, "docker stop") { + t.Errorf("refused cutover still stopped containers: %s", call) + } + } +} + +// TestCutover_InjectedCopyFailurePreservesOriginals — the C07 acceptance: +// a promotion copy that fails mid-flight rolls the live directory back to +// its originals (move-back executed), and the error names what was kept. +func TestCutover_InjectedCopyFailurePreservesOriginals(t *testing.T) { + manifest := drTestManifest() + mock := drCutoverMock(drOKReceipt(), manifest, + ssh.MockCommand{Match: "cp -a '", Err: errors.New("exit status 1: cp: cannot create file: No space left on device")}, + ) + receipt, err := NewClient(mock, &bytes.Buffer{}).CutoverBundle(context.Background(), + BundleRestoreOptions{App: "myapp", ID: manifest.ID, Config: drTestConfig()}) + if err == nil { + t.Fatal("expected cutover failure") + } + if !strings.Contains(err.Error(), "previous contents restored") && !strings.Contains(err.Error(), "kept") { + t.Errorf("error must name the preserved originals: %v", err) + } + if receipt == nil { + t.Fatal("a cutover receipt (even failed) should be returned") + } + var sawRollback bool + for _, call := range mock.Calls { + if strings.Contains(call, "find '/deployments/myapp/volumes/data' -mindepth 1 -delete && find '/deployments/myapp/recovery123'") { + sawRollback = true + } + } + if !sawRollback { + t.Errorf("rollback (clear partial copy + move originals back) never ran") + } + // Stopped containers were restarted by the failure path. + var sawRestart bool + for _, call := range mock.Calls { + if strings.Contains(call, "docker start 'myapp-web'") { + sawRestart = true + } + } + if !sawRestart { + t.Errorf("stopped containers were not restarted after the failed cutover") + } +} + +// TestCutover_InjectedStartFailurePreservesOriginals: an accessory that +// cannot start aborts the cutover BEFORE any data promotion (no cp -a, no +// state install), with the stopped containers restarted. +func TestCutover_InjectedStartFailurePreservesOriginals(t *testing.T) { + manifest := drTestManifest() + mock := drCutoverMock(drOKReceipt(), manifest, + ssh.MockCommand{Match: "docker run --detach", Err: errors.New("exit status 125: docker: Error response from daemon: no space on device")}, + ) + _, err := NewClient(mock, &bytes.Buffer{}).CutoverBundle(context.Background(), + BundleRestoreOptions{App: "myapp", ID: manifest.ID, Config: drTestConfig()}) + if err == nil || !strings.Contains(err.Error(), "starting accessory db") { + t.Fatalf("expected accessory start failure, got %v", err) + } + for _, call := range mock.Calls { + if strings.HasPrefix(call, "cp -a ") { + t.Errorf("data promotion ran despite accessory start failure: %s", call) + } + if strings.Contains(call, "UPLOAD:/deployments/myapp/state.json") { + t.Errorf("state was installed despite accessory start failure: %s", call) + } + } + var sawRestart bool + for _, call := range mock.Calls { + if strings.Contains(call, "docker start 'myapp-web'") || strings.Contains(call, "docker start 'myapp-db'") { + sawRestart = true + } + } + if !sawRestart { + t.Errorf("stopped containers were not restarted after the aborted cutover") + } +} + +// TestCutover_HappyPath: engine dump restored into the recreated accessory, +// volume promoted two-phase, state.json + release records installed, and +// the pre-existing engine data dir preserved as a recovery copy. +func TestCutover_HappyPath(t *testing.T) { + manifest := drTestManifest() + manifest.ReleaseRecords = []json.RawMessage{json.RawMessage(`{"schema_version":1,"app":"myapp","hash":"rel-77","created_at":"2026-09-20T10:00:00Z"}`)} + // Non-empty engine dir forces the pre-cutover move-aside. + mock := drCutoverMock(drOKReceipt(), manifest, + ssh.MockCommand{Match: "if [ -z \"$(ls -A", Output: "nonempty\n"}, + ssh.MockCommand{Match: "if [ -e '/deployments/myapp/accessories/db/pgdata'", Output: "present\n"}, + ssh.MockCommand{Match: "mktemp -d '/deployments/myapp/accessories/db.pre-cutover.XXXXXX'", Output: "/deployments/myapp/accessories/db.pre-cutover.Aa1\n"}, + ) + receipt, err := NewClient(mock, &bytes.Buffer{}).CutoverBundle(context.Background(), + BundleRestoreOptions{App: "myapp", ID: manifest.ID, Config: drTestConfig()}) + if err != nil { + t.Fatalf("CutoverBundle: %v", err) + } + joined := strings.Join(mock.Calls, "\n") + if !strings.Contains(joined, "docker exec -i 'myapp-db' psql -v ON_ERROR_STOP=1 -U 'appuser' 'appdb'") { + t.Errorf("engine dump was not restored into the live accessory") + } + if !strings.Contains(joined, "mv -f '/deployments/myapp/accessories/db/pgdata' '/deployments/myapp/accessories/db.pre-cutover.Aa1/pgdata'") { + t.Errorf("pre-cutover engine data was not preserved") + } + if len(receipt.Promoted) != 1 || receipt.Promoted[0] != "/deployments/myapp/volumes/data" { + t.Errorf("promoted paths: %v", receipt.Promoted) + } + stateBytes, ok := mock.Files["/deployments/myapp/state.json"] + if !ok || !strings.Contains(string(stateBytes), `"generation":4`) { + t.Errorf("restored state.json not installed: %q", string(stateBytes)) + } + if _, ok := mock.Files["/deployments/myapp/meta/rel-77.json"]; !ok { + t.Errorf("release record not installed") + } + if len(receipt.RecoveryDirs) == 0 { + t.Errorf("recovery dirs not recorded: %+v", receipt) + } +} + +// TestCutover_NonEmptyAccessoryDirWithoutVolumeData: an accessory dir that +// is nonempty (env/credential files) but has NO on-disk volume directory — +// a volume added to teploy.yml after the bundle, or a re-run after a +// partial failure — has nothing to preserve: the cutover proceeds, moves +// nothing, and records no empty recovery dir. +func TestCutover_NonEmptyAccessoryDirWithoutVolumeData(t *testing.T) { + manifest := drTestManifest() + mock := drCutoverMock(drOKReceipt(), manifest, + ssh.MockCommand{Match: "if [ -z \"$(ls -A", Output: "nonempty\n"}, + ssh.MockCommand{Match: "if [ -e '/deployments/myapp/accessories/db/pgdata'", Output: "absent\n"}, + ) + receipt, err := NewClient(mock, &bytes.Buffer{}).CutoverBundle(context.Background(), + BundleRestoreOptions{App: "myapp", ID: manifest.ID, Config: drTestConfig()}) + if err != nil { + t.Fatalf("CutoverBundle: %v", err) + } + for _, call := range mock.Calls { + if strings.HasPrefix(call, "mv -f ") || strings.Contains(call, "docker run --rm --user 0") { + t.Errorf("nothing existed to move aside, yet a move ran: %s", call) + } + if strings.Contains(call, "pre-cutover.XXXXXX") { + t.Errorf("an empty recovery dir was created: %s", call) + } + } + for _, d := range receipt.RecoveryDirs { + if strings.Contains(d, "pre-cutover") { + t.Errorf("empty pre-cutover recovery dir recorded: %+v", receipt.RecoveryDirs) + } + } +} + +// TestCutover_MissingConfiguredAccessoryRefused: a bundle accessory absent +// from the restore-time teploy.yml aborts before anything is stopped. +func TestCutover_MissingConfiguredAccessoryRefused(t *testing.T) { + manifest := drTestManifest() + cfg := drTestConfig() + delete(cfg.Accessories, "db") + mock := drCutoverMock(drOKReceipt(), manifest) + _, err := NewClient(mock, &bytes.Buffer{}).CutoverBundle(context.Background(), + BundleRestoreOptions{App: "myapp", ID: manifest.ID, Config: cfg}) + if err == nil || !strings.Contains(err.Error(), "does not define it") { + t.Fatalf("expected config cross-check refusal, got %v", err) + } + for _, call := range mock.Calls { + if strings.Contains(call, "docker stop") { + t.Errorf("cross-check failure still stopped containers: %s", call) + } + } +} + +// TestCutover_MissingSecretKeysRefusedBeforeMutation: the preflight runs +// again at cutover — keys that vanished since restore abort before stops. +func TestCutover_MissingSecretKeysRefusedBeforeMutation(t *testing.T) { + manifest := drTestManifest("API_KEY") + mock := drCutoverMock(drOKReceipt(), manifest) + _, err := NewClient(mock, &bytes.Buffer{}).CutoverBundle(context.Background(), + BundleRestoreOptions{App: "myapp", ID: manifest.ID, Config: drTestConfig()}) + if err == nil || !strings.Contains(err.Error(), "API_KEY") { + t.Fatalf("expected missing-keys refusal, got %v", err) + } + for _, call := range mock.Calls { + if strings.Contains(call, "docker stop") { + t.Errorf("preflight failure still stopped containers: %s", call) + } + } +} diff --git a/internal/backup/bundle_test.go b/internal/backup/bundle_test.go new file mode 100644 index 0000000..48d6443 --- /dev/null +++ b/internal/backup/bundle_test.go @@ -0,0 +1,352 @@ +package backup + +import ( + "bytes" + "context" + "encoding/json" + "strings" + "testing" + "time" + + "github.com/useteploy/teploy/internal/config" + "github.com/useteploy/teploy/internal/ssh" +) + +// drTestState is a minimal valid v2 state.json for myapp. +const drTestState = `{"schema_version":2,"deployment_type":"container","ingress_mode":"caddy","domain":"myapp.example.com","updated_at":"2026-09-20T10:00:00Z","image_ref":"nginx:1.27","generation":4,"current_port":3000,"current_hash":"abc123"}` + +func drBaseMock(extra ...ssh.MockCommand) *ssh.MockExecutor { + mock := ssh.NewMockExecutor("dr-host", append([]ssh.MockCommand{ + {Match: "umask 077; mktemp -d /tmp/teploy-dr.XXXXXXXX", Output: "/tmp/drws1\n"}, + {Match: "mkdir -p '/tmp/drws1", Output: ""}, + {Match: "mkdir -p '/var/drstore/myapp/dr/", Output: ""}, + {Match: "mkdir -p /deployments/myapp/accessories/db", Output: ""}, + {Match: "find '/deployments/myapp/meta'", Output: ""}, + {Match: "tar -czf", Output: ""}, + {Match: "docker exec 'myapp-db' pg_dump", Output: ""}, + {Match: "docker inspect -f '{{.Config.Image}}' 'myapp-db'", Output: "postgres:16\n"}, + {Match: "docker inspect -f '{{range .Config.Env}}{{println .}}{{end}}' 'myapp-db'", Output: "POSTGRES_DB=appdb\nPOSTGRES_USER=appuser\n"}, + {Match: "cp -p", Output: ""}, + {Match: "mv -f", Output: ""}, + {Match: "rm -rf", Output: ""}, + {Match: "if [ -e '", Output: "present\n"}, + {Match: "find '/deployments/myapp/secrets'", Output: "API_KEY.age\n"}, + }, extra...)...) + // state.json seeds the framed-read auto-answer. + mock.Files["/deployments/myapp/state.json"] = []byte(drTestState) + return mock +} + +func drTestConfig() config.AppConfig { + return config.AppConfig{ + App: "myapp", + Volumes: map[string]string{"data": "/app/data"}, + Accessories: map[string]config.AccessoryConfig{ + "db": { + Image: "postgres:16", + Env: map[string]string{"POSTGRES_DB": "appdb", "POSTGRES_USER": "appuser"}, + Volumes: map[string]string{"pgdata": "/var/lib/postgresql/data"}, + }, + }, + } +} + +// TestCreateBundle_ManifestAndConsistency pins the bundle contract: schema +// version, DR kind, embedded state, engine-consistent accessory snapshots, +// crash-consistent volume snapshots with honest notes, references-only +// secrets, and the manifest uploaded LAST (completeness marker). +func TestCreateBundle_ManifestAndConsistency(t *testing.T) { + mock := drBaseMock() + client := NewClient(mock, &bytes.Buffer{}) + store := DirBundleStore{Root: "/var/drstore"} + + m, err := client.CreateBundle(context.Background(), BundleOptions{ + App: "myapp", + Config: drTestConfig(), + Now: func() time.Time { return time.Date(2026, 9, 23, 10, 15, 0, 0, time.UTC) }, + }, store) + if err != nil { + t.Fatalf("CreateBundle: %v", err) + } + if m.SchemaVersion != BundleSchemaVersion || m.Kind != BundleKindDR { + t.Errorf("schema/kind: %d/%q", m.SchemaVersion, m.Kind) + } + if err := ValidateDate(m.ID); err != nil { + t.Errorf("bundle id not in backup-id grammar: %v", err) + } + if !strings.Contains(string(m.State), `"generation":4`) { + t.Errorf("state not embedded verbatim: %s", m.State) + } + if m.Routing.Domain != "myapp.example.com" || m.Routing.IngressMode != "caddy" { + t.Errorf("routing not recorded: %+v", m.Routing) + } + if m.Secrets.Mode != "references" || len(m.Secrets.Included) != 0 || m.Secrets.AgeKeyIncluded { + t.Errorf("secrets must be references-only by default: %+v", m.Secrets) + } + + byName := map[string]SnapshotRecord{} + for _, s := range m.Snapshots { + byName[s.Name] = s + } + dbSnap, ok := byName["db"] + if !ok { + t.Fatalf("no db snapshot: %+v", m.Snapshots) + } + if dbSnap.Engine != "postgres" || dbSnap.Method != "pg_dump" || dbSnap.Consistency != consistencyEngineDump || dbSnap.Detection != "image-pattern" { + t.Errorf("db snapshot: %+v", dbSnap) + } + if dbSnap.EngineParams["db"] != "appdb" || dbSnap.EngineParams["user"] != "appuser" { + t.Errorf("engine params not recorded: %+v", dbSnap.EngineParams) + } + if dbSnap.Image != "postgres:16" { + t.Errorf("snapshot image not recorded: %q", dbSnap.Image) + } + volSnap, ok := byName["data"] + if !ok { + t.Fatalf("no data volume snapshot: %+v", m.Snapshots) + } + if volSnap.Consistency != consistencyCrash { + t.Errorf("live volume copy must be labeled crash-consistent: %+v", volSnap) + } + if !strings.Contains(volSnap.Notes, "crash recovery") { + t.Errorf("volume snapshot must carry the honest crash-consistency note: %q", volSnap.Notes) + } + + // Upload ordering: manifest.json LAST, after every data member. + lastData := -1 + manifestAt := -1 + for i, call := range mock.Calls { + if strings.HasPrefix(call, "cp -p '/tmp/drws1/") && strings.Contains(call, " /var/drstore/myapp/dr/") && !strings.Contains(call, "manifest.json") { + if i > lastData { + lastData = i + } + } + if strings.HasPrefix(call, "cp -p '/tmp/drws1/manifest.json'") { + manifestAt = i + } + } + if manifestAt < 0 || manifestAt < lastData { + t.Errorf("manifest must upload after all data members (manifest at %d, last data at %d)", manifestAt, lastData) + } +} + +// TestCreateBundle_NoStateRefuses: an app with no on-server state has +// nothing to recover — the error must say so and name the data-only +// command, not mint a bundle that lies about scope. +func TestCreateBundle_NoStateRefuses(t *testing.T) { + mock := ssh.NewMockExecutor("dr-host", + ssh.MockCommand{Match: "umask 077; mktemp -d /tmp/teploy-dr.XXXXXXXX", Output: "/tmp/drws1\n"}, + ) + client := NewClient(mock, &bytes.Buffer{}) + _, err := client.CreateBundle(context.Background(), BundleOptions{ + App: "myapp", Config: drTestConfig(), + }, DirBundleStore{Root: "/var/drstore"}) + if err == nil || !strings.Contains(err.Error(), "data-only") { + t.Fatalf("expected no-state refusal naming data-only backup, got %v", err) + } +} + +// TestCreateBundle_VolumePatternHeuristic: a volume whose container path +// looks like a postgres data dir is recorded with the engine hint and an +// explicit "a raw tar is NOT engine-consistent" caveat — the heuristic is +// surfaced, never trusted. +func TestCreateBundle_VolumePatternHeuristic(t *testing.T) { + cfg := drTestConfig() + cfg.Volumes["pgdata"] = "/var/lib/postgresql/data" + mock := drBaseMock() + client := NewClient(mock, &bytes.Buffer{}) + m, err := client.CreateBundle(context.Background(), BundleOptions{App: "myapp", Config: cfg}, DirBundleStore{Root: "/var/drstore"}) + if err != nil { + t.Fatalf("CreateBundle: %v", err) + } + for _, s := range m.Snapshots { + if s.Name == "pgdata" { + if s.Engine != "postgres" || s.Detection != "volume-pattern" { + t.Errorf("pattern snapshot: %+v", s) + } + if s.Consistency != consistencyCrash { + t.Errorf("pattern-matched volume must STILL be crash-consistent: %+v", s) + } + if !strings.Contains(s.Notes, "NOT a postgres-consistent backup") { + t.Errorf("note must refuse engine-consistency claim: %q", s.Notes) + } + return + } + } + t.Fatalf("pgdata snapshot missing: %+v", m.Snapshots) +} + +// TestCreateBundle_StopAppQuiescesVolumes: --stop-app stops the app's web +// containers before the volume tar and restarts them after; the volume +// snapshot is then labeled quiesced with detection=teploy-stop. +func TestCreateBundle_StopAppQuiescesVolumes(t *testing.T) { + mock := drBaseMock( + ssh.MockCommand{Match: "docker ps --filter label=teploy.app='myapp' --filter label=teploy.process=web", Output: "myapp-web\n"}, + ssh.MockCommand{Match: "docker stop", Output: ""}, + ssh.MockCommand{Match: "docker start", Output: ""}, + ) + client := NewClient(mock, &bytes.Buffer{}) + m, err := client.CreateBundle(context.Background(), BundleOptions{App: "myapp", Config: drTestConfig(), StopApp: true}, DirBundleStore{Root: "/var/drstore"}) + if err != nil { + t.Fatalf("CreateBundle: %v", err) + } + var vol SnapshotRecord + for _, s := range m.Snapshots { + if s.Name == "data" { + vol = s + } + } + if vol.Consistency != consistencyQuiesced || vol.Detection != "teploy-stop" { + t.Errorf("stopped-app volume must be quiesced/teploy-stop: %+v", vol) + } + var stopAt, tarAt, startAt int = -1, -1, -1 + for i, call := range mock.Calls { + if strings.Contains(call, "docker stop 'myapp-web'") && stopAt < 0 { + stopAt = i + } + if strings.Contains(call, "tar -czf") && strings.Contains(call, "volumes/data") && tarAt < 0 { + tarAt = i + } + if strings.Contains(call, "docker start 'myapp-web'") && startAt < 0 { + startAt = i + } + } + if stopAt < 0 || tarAt < 0 || startAt < 0 || !(stopAt < tarAt && tarAt < startAt) { + t.Errorf("expected stop -> tar -> start ordering (stop=%d tar=%d start=%d)", stopAt, tarAt, startAt) + } +} + +// TestCreateBundle_IncludeSecretsOptsIn: encrypted material travels ONLY +// with the explicit flag — default runs must not copy ciphertexts, .env, +// or credentials. +func TestCreateBundle_IncludeSecretsOptsIn(t *testing.T) { + base := func() *ssh.MockExecutor { + mock := drBaseMock() + mock.Files["/deployments/myapp/secrets"] = []byte("") + mock.Files["/deployments/myapp/secrets/API_KEY.age"] = []byte("ciphertext") + mock.Files["/deployments/myapp/.env"] = []byte("FOO=bar\n") + mock.Files["/deployments/myapp/accessories/db/credentials"] = []byte("POSTGRES_PASSWORD=p\n") + return mock + } + + defaultMock := base() + if _, err := NewClient(defaultMock, &bytes.Buffer{}).CreateBundle(context.Background(), + BundleOptions{App: "myapp", Config: drTestConfig()}, DirBundleStore{Root: "/var/drstore"}); err != nil { + t.Fatalf("CreateBundle default: %v", err) + } + for _, call := range defaultMock.Calls { + if strings.Contains(call, "secrets/API_KEY.age") || strings.Contains(call, ".env' ") && strings.Contains(call, "/tmp/drws1/env") { + t.Errorf("default run must not bundle secret material: %s", call) + } + } + + optMock := base() + m, err := NewClient(optMock, &bytes.Buffer{}).CreateBundle(context.Background(), + BundleOptions{App: "myapp", Config: drTestConfig(), IncludeSecrets: true}, DirBundleStore{Root: "/var/drstore"}) + if err != nil { + t.Fatalf("CreateBundle include-secrets: %v", err) + } + if m.Secrets.Mode != "encrypted" || len(m.Secrets.Included) != 3 { + t.Errorf("encrypted mode must list its members: %+v", m.Secrets) + } + included := strings.Join(m.Secrets.Included, ",") + for _, want := range []string{"secrets/API_KEY.age", "env/.env", "env/credentials/db"} { + if !strings.Contains(included, want) { + t.Errorf("member %s not included: %v", want, m.Secrets.Included) + } + } + if m.Secrets.AgeKeyIncluded { + t.Errorf("age key must not travel without --include-age-key") + } +} + +// TestCreateBundle_RecoveryPlanDocumentsManualSteps: accessory upgrades, +// credential rotation, storage-path keying, TLS material and the +// deploy-after-cutover step appear in the carried recovery plan. +func TestCreateBundle_RecoveryPlanDocumentsManualSteps(t *testing.T) { + cfg := drTestConfig() + cfg.TLS = &config.TLSConfig{Cert: "/etc/ssl/myapp.crt", Key: "/etc/ssl/myapp.key"} + mock := drBaseMock() + mock.Files["/deployments/myapp/secrets"] = []byte("") + mock.Files["/deployments/myapp/secrets/API_KEY.age"] = []byte("ct") + m, err := NewClient(mock, &bytes.Buffer{}).CreateBundle(context.Background(), + BundleOptions{App: "myapp", Config: cfg}, DirBundleStore{Root: "/var/drstore"}) + if err != nil { + t.Fatalf("CreateBundle: %v", err) + } + actions := map[string]RecoveryStep{} + for _, s := range m.Recovery.Steps { + actions[s.Action] = s + } + for _, want := range []string{"restore-isolated", "cutover", "accessory-upgrade", "credential-rotation", "storage-path", "tls-material", "deploy"} { + if _, ok := actions[want]; !ok { + t.Errorf("recovery plan missing %q: %+v", want, m.Recovery.Steps) + } + } + if !strings.Contains(actions["accessory-upgrade"].Summary, "teploy accessory upgrade") { + t.Errorf("accessory-upgrade step should name the command: %s", actions["accessory-upgrade"].Summary) + } + if !strings.Contains(actions["credential-rotation"].Summary, "API_KEY") { + t.Errorf("credential-rotation step should list the referenced keys: %s", actions["credential-rotation"].Summary) + } +} + +// TestParseBundleManifest_RefusesForeignSchemaAndKind: version/kind +// mismatches die at parse time, before any restore logic runs. +func TestParseBundleManifest_RefusesForeignSchemaAndKind(t *testing.T) { + good, _ := json.Marshal(&BundleManifest{SchemaVersion: BundleSchemaVersion, Kind: BundleKindDR, ID: "20260923-101500-0123456789abcdef", App: "myapp"}) + if _, err := ParseBundleManifest(good); err != nil { + t.Errorf("valid manifest rejected: %v", err) + } + future, _ := json.Marshal(&BundleManifest{SchemaVersion: BundleSchemaVersion + 1, Kind: BundleKindDR, ID: "20260923-101500-0123456789abcdef", App: "myapp"}) + if _, err := ParseBundleManifest(future); err == nil || !strings.Contains(err.Error(), "schema version") { + t.Errorf("future schema must be refused, got %v", err) + } + foreign, _ := json.Marshal(&BundleManifest{SchemaVersion: BundleSchemaVersion, Kind: "data-only", ID: "20260923-101500-0123456789abcdef", App: "myapp"}) + if _, err := ParseBundleManifest(foreign); err == nil || !strings.Contains(err.Error(), "not a disaster-recovery bundle") { + t.Errorf("foreign kind must be refused, got %v", err) + } + if _, err := ParseBundleManifest([]byte("not json")); err == nil { + t.Errorf("garbage must be refused") + } +} + +// TestDeriveAppRun_PicksNewestRecord: the app-run spec comes from state's +// image plus the NEWEST release record's command (state.json alone cannot +// boot an image that exits without argv). +func TestDeriveAppRun_PicksNewestRecord(t *testing.T) { + records := []json.RawMessage{ + json.RawMessage(`{"created_at":"2026-09-19T10:00:00Z","cmd":"old-cmd","image_ref":"nginx:old"}`), + json.RawMessage(`{"created_at":"2026-09-21T10:00:00Z","cmd":"sleep 600","image_ref":"alpine:3"}`), + } + spec := deriveAppRun([]byte(drTestState), records) + if spec.Image != "nginx:1.27" { + t.Errorf("state image must win: %+v", spec) + } + if spec.Cmd != "sleep 600" { + t.Errorf("newest record cmd must win: %+v", spec) + } + empty := deriveAppRun([]byte(drTestState), nil) + if empty.Image != "nginx:1.27" || empty.Cmd != "" { + t.Errorf("no records: %+v", empty) + } +} + +// TestDirStore_ListIDs requires manifests: a prefix without a manifest +// (partial upload) never lists as a bundle — the find in ListIDs prints +// only directories that CONTAIN manifest.json. +func TestDirStore_ListIDs(t *testing.T) { + root := "/var/drstore" + mock := ssh.NewMockExecutor("h", + ssh.MockCommand{ + Match: "find '/var/drstore/myapp/dr'", + Output: root + "/myapp/dr/20260923-101500-0123456789abcdef\n", + }) + ids, err := DirBundleStore{Root: root}.ListIDs(context.Background(), mock, "myapp") + if err != nil { + t.Fatalf("ListIDs: %v", err) + } + if len(ids) != 1 || ids[0] != "20260923-101500-0123456789abcdef" { + t.Errorf("ids: %v", ids) + } +} diff --git a/internal/backup/dr_integration_test.go b/internal/backup/dr_integration_test.go new file mode 100644 index 0000000..179f18e --- /dev/null +++ b/internal/backup/dr_integration_test.go @@ -0,0 +1,411 @@ +//go:build integration + +// REAL-fixture verification of the C07 recovery path (colima SSH+Docker +// host, real postgres:16-alpine engine, real gzip/tar, no mocks anywhere +// in the path under test). Same fixture contract as the C01/C03 lanes: +// +// TEPLOY_FAULT_HOST=127.0.0.1:50075 \ +// TEPLOY_FAULT_USER=tyler \ +// TEPLOY_FAULT_KEY=~/.colima/_lima/_config/user \ +// go test -tags integration -run TestDRIntegration -v ./internal/backup +// +// Disposable fixture only — the test creates and removes +// /deployments/drprobe, /var/tmp/teploy-dr/drprobe, /tmp/teploy-dr-fixture and +// drprobe-* containers. +package backup + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "os" + "strings" + "testing" + "time" + + "github.com/useteploy/teploy/internal/config" + "github.com/useteploy/teploy/internal/ssh" +) + +func drFixtureEnv(t *testing.T) ssh.Executor { + t.Helper() + host := os.Getenv("TEPLOY_FAULT_HOST") + user := os.Getenv("TEPLOY_FAULT_USER") + key := os.Getenv("TEPLOY_FAULT_KEY") + if host == "" || user == "" || key == "" { + t.Skip("DR fixture needs TEPLOY_FAULT_HOST, TEPLOY_FAULT_USER, TEPLOY_FAULT_KEY (see internal/backup/dr_integration_test.go)") + } + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + exec, err := ssh.Connect(ctx, ssh.ConnectConfig{Host: host, User: user, KeyPath: key}) + if err != nil { + t.Fatalf("connecting fixture: %v", err) + } + if out, err := exec.Run(ctx, "docker version --format '{{.Server.Version}}'"); err != nil { + t.Skipf("fixture host has no reachable docker daemon (%v)", err) + } else { + t.Logf("fixture docker server %s", strings.TrimSpace(out)) + } + t.Cleanup(func() { exec.Close() }) + return exec +} + +const drFixtureApp = "drprobe" + +func drFixtureCleanup(t *testing.T, exec ssh.Executor) { + t.Helper() + ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second) + defer cancel() + for _, c := range []string{drFixtureApp + "-db", drFixtureApp + "-db-drcheck", drFixtureApp + "-dr-appcheck"} { + exec.Run(ctx, "docker rm -f "+ssh.ShellQuote(c)+" >/dev/null 2>&1 || true") + } + // Engine data dirs are chowned to the image's uid with mode 700 (the + // same reality reconcileDataOwnership exists for), so removal needs + // sudo — a plain rm -rf silently leaves the data behind and the next + // run starts from a stale database. + for _, d := range []string{ + "/deployments/" + drFixtureApp, + DRStagingRoot + "/" + drFixtureApp, + "/tmp/teploy-dr-fixture", + } { + if out, err := exec.Run(ctx, "sudo rm -rf "+ssh.ShellQuote(d)); err != nil { + t.Logf("cleanup of %s failed (continuing): %v %s", d, err, out) + } + } +} + +// drFixtureState is a valid v2 state for the probe app: image alpine:3 with +// the release record carrying "sleep 600" so the app check can boot it. +const drFixtureState = `{"schema_version":2,"deployment_type":"container","ingress_mode":"host","updated_at":"2026-09-23T09:00:00Z","image_ref":"alpine:3","operation_id":"op1","generation":2,"current_port":3000,"current_hash":"h2"}` +const drFixtureRelease = `{"schema_version":1,"app":"drprobe","hash":"h2","created_at":"2026-09-22T10:00:00Z","image_ref":"alpine:3","cmd":"sleep 600"}` + +// drFixtureConfig is the restore-time teploy.yml equivalent. +func drFixtureConfig() config.AppConfig { + return config.AppConfig{ + App: drFixtureApp, + Volumes: map[string]string{"data": "/app/data"}, + Accessories: map[string]config.AccessoryConfig{ + "db": { + Image: "postgres:16-alpine", + Env: map[string]string{"POSTGRES_DB": "appdb", "POSTGRES_USER": "appuser", "POSTGRES_PASSWORD": "fixture-pw"}, + Volumes: map[string]string{"pgdata": "/var/lib/postgresql/data"}, + }, + }, + } +} + +// TestDRIntegration_BundleRoundTrip drives the full C07 acceptance against +// the real fixture: create a bundle from a live postgres accessory + app +// volume, restore it ISOLATED (proving /deployments is untouched), check +// the receipt's RPO/RTO and checks, then cut over and prove the engine data +// landed (row count), the staged volume replaced the live one (post-bundle +// file gone, preserved in a recovery dir), and state.json was reinstalled. +func TestDRIntegration_BundleRoundTrip(t *testing.T) { + exec := drFixtureEnv(t) + drFixtureCleanup(t, exec) + t.Cleanup(func() { drFixtureCleanup(t, exec) }) + + ctx, cancel := context.WithTimeout(context.Background(), 12*time.Minute) + defer cancel() + + // --- Stand up the "original host" state. + must := func(what string, err error) { + t.Helper() + if err != nil { + t.Fatalf("%s: %v", what, err) + } + } + run := func(cmd string) string { + t.Helper() + out, err := exec.Run(ctx, cmd) + if err != nil { + t.Fatalf("fixture command failed: %s\nerror: %v\noutput: %s", cmd, err, out) + } + return out + } + + run("docker network create teploy >/dev/null 2>&1 || true") + run("mkdir -p /deployments/" + drFixtureApp + "/volumes/data /deployments/" + drFixtureApp + "/meta /deployments/" + drFixtureApp + "/accessories/db/pgdata") + run("printf 'original-marker' > /deployments/" + drFixtureApp + "/volumes/data/marker.txt") + if err := exec.Upload(ctx, strings.NewReader(drFixtureState), "/deployments/"+drFixtureApp+"/state.json", "0600"); err != nil { + t.Fatalf("seeding state.json: %v", err) + } + if err := exec.Upload(ctx, strings.NewReader(drFixtureRelease), "/deployments/"+drFixtureApp+"/meta/h2.json", "0600"); err != nil { + t.Fatalf("seeding release record: %v", err) + } + + // The live accessory: a real postgres with real data. + run(fmt.Sprintf( + "docker run -d --name %s --restart no --label teploy.app=%s --label teploy.role=accessory "+ + "-e POSTGRES_DB=appdb -e POSTGRES_USER=appuser -e POSTGRES_PASSWORD=fixture-pw "+ + "-v /deployments/%s/accessories/db/pgdata:/var/lib/postgresql/data postgres:16-alpine", + drFixtureApp+"-db", drFixtureApp, drFixtureApp)) + run(fmt.Sprintf("for i in $(seq 1 30); do docker exec %s pg_isready -U appuser >/dev/null 2>&1 && break; sleep 2; done", drFixtureApp+"-db")) + run(fmt.Sprintf("docker exec %s psql -U appuser -d appdb -c 'CREATE TABLE probe(id int); INSERT INTO probe VALUES (1),(2),(3);'", drFixtureApp+"-db")) + + client := NewClient(exec, os.Stdout) + store := DirBundleStore{Root: "/tmp/teploy-dr-fixture"} + + // --- Create the bundle from the live app. + manifest, err := client.CreateBundle(ctx, BundleOptions{ + App: drFixtureApp, + Config: drFixtureConfig(), + Version: "integration-test", + }, store) + must("CreateBundle", err) + t.Logf("bundle %s created with %d snapshots", manifest.ID, len(manifest.Snapshots)) + for _, s := range manifest.Snapshots { + t.Logf(" snapshot %-6s engine=%-8s consistency=%s", s.Name, s.Engine, s.Consistency) + } + if len(manifest.Snapshots) != 2 { + t.Fatalf("expected 2 snapshots (db + data), got %+v", manifest.Snapshots) + } + if manifest.AppRun.Image != "alpine:3" || manifest.AppRun.Cmd != "sleep 600" { + t.Errorf("app run spec not derived: %+v", manifest.AppRun) + } + + // Post-bundle drift: this file exists ONLY in the live volume, so the + // cutover's replacement (and its recovery copy) is provable. + run("printf 'post-bundle' > /deployments/" + drFixtureApp + "/volumes/data/post-bundle.txt") + beforeTree := run(fmt.Sprintf("find /deployments/%s -type f | sort | xargs md5sum 2>/dev/null", drFixtureApp)) + + // --- Isolated restore + validation. + receipt, err := client.RestoreBundleIsolated(ctx, BundleRestoreOptions{ + App: drFixtureApp, ID: manifest.ID, Config: drFixtureConfig(), + }, store) + must("RestoreBundleIsolated", err) + for _, ck := range receipt.Checks { + t.Logf("check %-28s %-7s %s %s", ck.Name, ck.Status, ck.Metric, ck.Detail) + } + if !receipt.OK { + t.Fatalf("isolated restore did not validate: %+v", receipt.Checks) + } + if receipt.RPOSeconds < 0 || receipt.RTOSeconds <= 0 { + t.Errorf("RPO/RTO must be measured (RPO may be 0 for an immediate restore): %d/%d", receipt.RPOSeconds, receipt.RTOSeconds) + } + var dataOK, appOK bool + for _, ck := range receipt.Checks { + if ck.Kind == "data" && ck.Status == "pass" { + dataOK = true + } + if ck.Kind == "app" && ck.Status == "pass" { + appOK = true + } + } + if !dataOK || !appOK { + t.Fatalf("fresh-host restore must pass application AND data checks: %+v", receipt.Checks) + } + + // Isolation proof: the live tree is byte-identical after the restore. + afterTree := run(fmt.Sprintf("find /deployments/%s -type f | sort | xargs md5sum 2>/dev/null", drFixtureApp)) + if beforeTree != afterTree { + t.Errorf("isolated restore modified /deployments/%s:\nbefore:\n%s\nafter:\n%s", drFixtureApp, beforeTree, afterTree) + } + + // --- Explicit cutover. + cutover, err := client.CutoverBundle(ctx, BundleRestoreOptions{ + App: drFixtureApp, ID: manifest.ID, Config: drFixtureConfig(), + }) + must("CutoverBundle", err) + t.Logf("cutover promoted %d path(s), recovery dirs %v", len(cutover.Promoted), cutover.RecoveryDirs) + + // Engine data landed in the recreated accessory: the same 3 rows. + got := run(fmt.Sprintf("for i in $(seq 1 30); do docker exec %s pg_isready -U appuser >/dev/null 2>&1 && break; sleep 2; done; docker exec %s psql -tA -U appuser -d appdb -c 'SELECT COUNT(*) FROM probe'", + drFixtureApp+"-db", drFixtureApp+"-db")) + if strings.TrimSpace(got) != "3" { + t.Errorf("restored engine data wrong: probe count = %q, want 3", strings.TrimSpace(got)) + } + + // Volume replacement: post-bundle drift is gone from the live volume + // and preserved in a recovery dir. + liveMarker := run("cat /deployments/" + drFixtureApp + "/volumes/data/marker.txt") + if strings.TrimSpace(liveMarker) != "original-marker" { + t.Errorf("live marker after cutover = %q", liveMarker) + } + postBundleGone := run("if [ ! -f /deployments/" + drFixtureApp + "/volumes/data/post-bundle.txt ]; then printf 'gone'; else printf 'present'; fi") + if strings.TrimSpace(postBundleGone) != "gone" { + t.Errorf("cutover did not replace the live volume (post-bundle.txt still present)") + } + recRecovered := run(fmt.Sprintf("grep -rl 'post-bundle' /deployments/%s/volumes/data.restore-old.* 2>/dev/null | head -1", drFixtureApp)) + if strings.TrimSpace(recRecovered) == "" { + t.Errorf("the pre-cutover volume copy (with post-bundle.txt) was not preserved in a recovery dir") + } else { + t.Logf("pre-cutover original preserved at %s", strings.TrimSpace(recRecovered)) + } + + // State reinstalled from the bundle (parse, don't string-match: the + // staged manifest legitimately re-indents the embedded raw JSON). + var stateCheck struct { + Generation uint64 `json:"generation"` + ImageRef string `json:"image_ref"` + } + if err := json.Unmarshal([]byte(run("cat /deployments/"+drFixtureApp+"/state.json")), &stateCheck); err != nil { + t.Fatalf("restored state.json unparseable: %v", err) + } + if stateCheck.Generation != 2 || stateCheck.ImageRef != "alpine:3" { + t.Errorf("restored state.json wrong: %+v", stateCheck) + } +} + +// TestDRIntegration_CorruptBundleRefused: a truncated artifact member +// fails gzip -t on the real host and the restore refuses before any +// engine boots. +func TestDRIntegration_CorruptBundleRefused(t *testing.T) { + exec := drFixtureEnv(t) + drFixtureCleanup(t, exec) + t.Cleanup(func() { drFixtureCleanup(t, exec) }) + + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute) + defer cancel() + + run := func(cmd string) string { + t.Helper() + out, err := exec.Run(ctx, cmd) + if err != nil { + t.Fatalf("fixture command failed: %s\nerror: %v\noutput: %s", cmd, err, out) + } + return out + } + run("mkdir -p /deployments/" + drFixtureApp + "/volumes/data /tmp/teploy-dr-fixture/" + drFixtureApp + "/dr") + run("printf 'original' > /deployments/" + drFixtureApp + "/volumes/data/marker.txt") + if err := exec.Upload(ctx, strings.NewReader(drFixtureState), "/deployments/"+drFixtureApp+"/state.json", "0600"); err != nil { + t.Fatalf("seeding state.json: %v", err) + } + + client := NewClient(exec, os.Stdout) + store := DirBundleStore{Root: "/tmp/teploy-dr-fixture"} + cfg := drFixtureConfig() + delete(cfg.Accessories, "db") // volume-only: the corrupt member is the volume tar + manifest, err := client.CreateBundle(ctx, BundleOptions{App: drFixtureApp, Config: cfg}, store) + if err != nil { + t.Fatalf("CreateBundle: %v", err) + } + + // Corrupt the volume member: truncate mid-gzip. + run(fmt.Sprintf("head -c 20 %s > /tmp/teploy-dr-fixture/.corrupt && mv /tmp/teploy-dr-fixture/.corrupt %s", + ssh.ShellQuote("/tmp/teploy-dr-fixture/"+drFixtureApp+"/dr/"+manifest.ID+"/volumes/data.tar.gz"), + ssh.ShellQuote("/tmp/teploy-dr-fixture/"+drFixtureApp+"/dr/"+manifest.ID+"/volumes/data.tar.gz"))) + + _, err = client.RestoreBundleIsolated(ctx, BundleRestoreOptions{App: drFixtureApp, ID: manifest.ID, Config: drFixtureConfig()}, store) + if err == nil || !strings.Contains(err.Error(), "corrupt") { + t.Fatalf("expected corrupt-bundle refusal, got %v", err) + } + if booted := run("if docker ps -a --format '{{.Names}}' | grep -q drprobe; then printf 'booted'; else printf 'none'; fi"); strings.TrimSpace(booted) != "none" { + t.Errorf("no engine must boot from a corrupt bundle") + } +} + +// TestDRIntegration_CutoverCopyFailurePreservesOriginals: an injected +// failure of the promotion copy (staged tree made unreadable AFTER +// validation) rolls the live volume back to its originals on the real +// host — the C07 acceptance, live. +func TestDRIntegration_CutoverCopyFailurePreservesOriginals(t *testing.T) { + exec := drFixtureEnv(t) + drFixtureCleanup(t, exec) + t.Cleanup(func() { drFixtureCleanup(t, exec) }) + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) + defer cancel() + + run := func(cmd string) string { + t.Helper() + out, err := exec.Run(ctx, cmd) + if err != nil { + t.Fatalf("fixture command failed: %s\nerror: %v\noutput: %s", cmd, err, out) + } + return out + } + cfg := drFixtureConfig() + delete(cfg.Accessories, "db") // volume-only app: the promotion is the mutation under test + + run("mkdir -p /deployments/" + drFixtureApp + "/volumes/data") + run("printf 'live-original' > /deployments/" + drFixtureApp + "/volumes/data/marker.txt") + if err := exec.Upload(ctx, strings.NewReader(drFixtureState), "/deployments/"+drFixtureApp+"/state.json", "0600"); err != nil { + t.Fatalf("seeding state.json: %v", err) + } + if err := exec.Upload(ctx, strings.NewReader(drFixtureRelease), "/deployments/"+drFixtureApp+"/meta/h2.json", "0600"); err != nil { + t.Fatalf("seeding release record: %v", err) + } + + client := NewClient(exec, os.Stdout) + store := DirBundleStore{Root: "/tmp/teploy-dr-fixture"} + manifest, err := client.CreateBundle(ctx, BundleOptions{App: drFixtureApp, Config: cfg}, store) + if err != nil { + t.Fatalf("CreateBundle: %v", err) + } + receipt, err := client.RestoreBundleIsolated(ctx, BundleRestoreOptions{App: drFixtureApp, ID: manifest.ID, Config: cfg}, store) + if err != nil || !receipt.OK { + t.Fatalf("isolated restore must validate first: %v %+v", err, receipt.Checks) + } + + // Inject the copy failure: the staged tree becomes unreadable, so + // promoteStaged's `cp -a` fails AFTER the originals were moved aside — + // exactly the mid-restore partial-copy window. + run("chmod 000 " + ssh.ShellQuote(DRStagingPath(drFixtureApp, manifest.ID)+"/volumes/data")) + defer run("chmod -R 700 " + ssh.ShellQuote(DRStagingPath(drFixtureApp, manifest.ID)) + " >/dev/null 2>&1 || true") + + _, err = client.CutoverBundle(ctx, BundleRestoreOptions{App: drFixtureApp, ID: manifest.ID, Config: cfg}) + if err == nil || !strings.Contains(err.Error(), "promoting data at cutover") { + t.Fatalf("expected cutover promotion failure, got %v", err) + } + if !strings.Contains(err.Error(), "previous contents restored") && !strings.Contains(err.Error(), "kept") { + t.Errorf("error must name the preserved originals: %v", err) + } + + // The live volume is EXACTLY the original: marker intact, no partial copy. + got := run("cat /deployments/" + drFixtureApp + "/volumes/data/marker.txt") + if strings.TrimSpace(got) != "live-original" { + t.Errorf("originals not preserved after failed promotion: marker=%q", got) + } + entries := run("ls /deployments/" + drFixtureApp + "/volumes/data") + if strings.TrimSpace(entries) != "marker.txt" { + t.Errorf("live volume not rolled back cleanly: %q", entries) + } + // No leftover restore-old directory from the failed promotion. + leftover := run("ls -d /deployments/" + drFixtureApp + "/volumes/data.restore-old.* 2>/dev/null | wc -l") + if strings.TrimSpace(leftover) != "0" { + t.Errorf("failed promotion left recovery dirs behind: %s", leftover) + } +} + +// TestDRIntegration_MissingKeysFailBeforeMutation: against the real host, a +// references-mode bundle with a key the target lacks aborts before staging +// or any docker activity (only read-only commands observed). +func TestDRIntegration_MissingKeysFailBeforeMutation(t *testing.T) { + exec := drFixtureEnv(t) + drFixtureCleanup(t, exec) + t.Cleanup(func() { drFixtureCleanup(t, exec) }) + + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + + manifest := drTestManifest("API_KEY") + manifest.App = drFixtureApp + manifest.ID = "20260923-101500-0123456789abcdef" + manifestJSON, _ := json.Marshal(manifest) + + run := func(cmd string) string { + t.Helper() + out, err := exec.Run(ctx, cmd) + if err != nil { + t.Fatalf("fixture command failed: %s\nerror: %v", cmd, err) + } + return out + } + run("mkdir -p /tmp/teploy-dr-fixture/" + drFixtureApp + "/dr/" + manifest.ID) + if err := exec.Upload(ctx, bytes.NewReader(manifestJSON), "/tmp/teploy-dr-fixture/"+drFixtureApp+"/dr/"+manifest.ID+"/manifest.json", "0600"); err != nil { + t.Fatalf("seeding manifest: %v", err) + } + + client := NewClient(exec, os.Stdout) + _, err := client.RestoreBundleIsolated(ctx, BundleRestoreOptions{ + App: drFixtureApp, ID: manifest.ID, Config: drFixtureConfig(), + }, DirBundleStore{Root: "/tmp/teploy-dr-fixture"}) + if err == nil || !strings.Contains(err.Error(), "API_KEY") { + t.Fatalf("expected missing-keys failure naming API_KEY, got %v", err) + } + if staged := run("if [ -e " + DRStagingRoot + "/" + drFixtureApp + " ]; then printf 'staged'; else printf 'clean'; fi"); strings.TrimSpace(staged) != "clean" { + t.Errorf("preflight failure must not leave staging behind: %s", staged) + } +} diff --git a/internal/backup/redis_restore_test.go b/internal/backup/redis_restore_test.go index ed956f9..a53fbcc 100644 --- a/internal/backup/redis_restore_test.go +++ b/internal/backup/redis_restore_test.go @@ -98,9 +98,9 @@ func (e *redisRestoreExec) Upload(ctx context.Context, content io.Reader, remote return nil } -func (e *redisRestoreExec) Close() error { return nil } -func (e *redisRestoreExec) Host() string { return "1.2.3.4" } -func (e *redisRestoreExec) User() string { return "root" } +func (e *redisRestoreExec) Close() error { return nil } +func (e *redisRestoreExec) Host() string { return "1.2.3.4" } +func (e *redisRestoreExec) User() string { return "root" } // writeDockerStub writes the stub docker binary the restore script drives. // Scenario knobs (env): HAVE_DUMP=yes/no, FAIL_BASELINE=transport, diff --git a/internal/backup/retention_test.go b/internal/backup/retention_test.go index f6f3cfb..6875e18 100644 --- a/internal/backup/retention_test.go +++ b/internal/backup/retention_test.go @@ -23,9 +23,9 @@ func TestParseBackupEntries(t *testing.T) { ls := strings.Join([]string{ "2024-01-01 00:00:00 100 20240101-000000.tar.gz", "2024-01-02 03:04:05 100 myapp-backup-20240102-030405.tar.gz", // scheduled-shell naming - "2024-01-03 00:00:00 100 20240103-000000.sql.gz", // accessory naming - " PRE nested/", // directory prefix — skipped - "garbage line no timestamp", // skipped + "2024-01-03 00:00:00 100 20240103-000000.sql.gz", // accessory naming + " PRE nested/", // directory prefix — skipped + "garbage line no timestamp", // skipped }, "\n") entries := parseBackupEntries(ls) diff --git a/internal/cli/doctor.go b/internal/cli/doctor.go index eb0adbb..9c18b95 100644 --- a/internal/cli/doctor.go +++ b/internal/cli/doctor.go @@ -449,21 +449,25 @@ func doctorRegistryCheck(ctx context.Context, exec ssh.Executor, appCfg *config. Detail: "no registry image ref — the image is built from source at deploy time", } } - if _, err := exec.Run(ctx, "docker manifest inspect "+ssh.ShellQuote(appCfg.Image)); err != nil { - switch classifyRegistryError(err) { + res := ssh.RunDetailed(ctx, exec, "docker manifest inspect "+ssh.ShellQuote(appCfg.Image)) + if res.Failed() { + switch classifyRegistryFailure(res) { case "auth": return doctorCheck{ - Name: "registry", Result: doctorFail, Detail: err.Error(), + Name: "registry", Result: doctorFail, + Detail: registryFailureDetail(res), Remediation: "store credentials on the server: teploy registry login — deploys pull as the server's docker", } case "missing": return doctorCheck{ - Name: "registry", Result: doctorFail, Detail: err.Error(), + Name: "registry", Result: doctorFail, + Detail: registryFailureDetail(res), Remediation: "push the image to the registry, or correct the image ref in teploy.yml", } default: return doctorCheck{ - Name: "registry", Result: doctorFail, Detail: err.Error(), + Name: "registry", Result: doctorFail, + Detail: registryFailureDetail(res), Remediation: "check the network path from the server to the registry (DNS, firewall, proxy)", } } @@ -471,11 +475,25 @@ func doctorRegistryCheck(ctx context.Context, exec ssh.Executor, appCfg *config. return doctorCheck{Name: "registry", Result: doctorOK, Detail: fmt.Sprintf("registry reachable for %s", appCfg.Image)} } -// classifyRegistryError buckets a manifest-inspect failure into auth / -// missing / unreachable — display classification only; the detail always -// carries the underlying error verbatim. -func classifyRegistryError(err error) string { - msg := strings.ToLower(err.Error()) +// registryFailureDetail renders the structured failure for display: the +// command's own stderr when it produced some, the transport error +// otherwise. +func registryFailureDetail(res ssh.Result) string { + if d := res.ExitErrorText(); d != "" { + return d + } + return fmt.Sprintf("manifest inspect failed (exit status %d)", res.ExitCode) +} + +// classifyRegistryFailure buckets a manifest-inspect failure into auth / +// missing / unreachable. Docker's CLI exits 1 for every failure class, +// so the exit code cannot distinguish them — classification reads the +// tool's own stderr (via the structured Result, NOT the folded +// executor error text, which mixes in teploy's own wrapper words). +// Display classification only; the detail always carries the underlying +// output verbatim. +func classifyRegistryFailure(res ssh.Result) string { + msg := strings.ToLower(res.ExitErrorText()) switch { case strings.Contains(msg, "unauthorized"), strings.Contains(msg, "authentication required"), strings.Contains(msg, "denied"): return "auth" @@ -516,21 +534,29 @@ func doctorCaddyCheck(ctx context.Context, exec ssh.Executor, appCfg *config.App // Absence is ok — the server binary is optional infrastructure; skew is // a warning because scheduled redeploys and webhook builds run on it. func doctorCompatCheck(ctx context.Context, deps doctorDeps, exec ssh.Executor) doctorCheck { - out, err := exec.Run(ctx, ssh.ShellQuote(serverTeployBinaryPath)+" version") - if err != nil { - msg := err.Error() - if strings.Contains(msg, "not found") || strings.Contains(msg, "no such file") || strings.Contains(msg, "status 127") { + res := ssh.RunDetailed(ctx, exec, ssh.ShellQuote(serverTeployBinaryPath)+" version") + if res.ExitCode != 0 || res.Err != nil { + // Exit code 127 is the remote shell's definitive "command not + // found" — absence of the optional server binary. Structured + // status, not a guess from the failure text (which breaks the + // moment a wrapper message contains "not found" for another + // reason, or localizes). + if res.ExitCode == 127 { return doctorCheck{ Name: "compatibility", Result: doctorOK, Detail: fmt.Sprintf("no server-side teploy binary (optional — autodeploy installs one at %s)", serverTeployBinaryPath), } } + detail := res.ExitErrorText() + if detail == "" { + detail = fmt.Sprintf("exit status %d", res.ExitCode) + } return doctorCheck{ - Name: "compatibility", Result: doctorWarn, Detail: err.Error(), + Name: "compatibility", Result: doctorWarn, Detail: detail, Remediation: fmt.Sprintf("inspect %s on the server — it exists but would not run", serverTeployBinaryPath), } } - serverVersion := doctorServerTeployVersion(out) + serverVersion := doctorServerTeployVersion(res.TrimmedStdout()) if serverVersion == "" { return doctorCheck{ Name: "compatibility", Result: doctorWarn, diff --git a/internal/cli/doctor_test.go b/internal/cli/doctor_test.go index 9149d5e..59d190b 100644 --- a/internal/cli/doctor_test.go +++ b/internal/cli/doctor_test.go @@ -464,10 +464,19 @@ func TestDoctorRegistryCheck(t *testing.T) { "i/o timeout": "unreachable", } for msg, want := range cases { - if got := classifyRegistryError(errors.New(msg)); got != want { - t.Fatalf("classifyRegistryError(%q) = %q, want %q", msg, got, want) + // Docker's CLI exits 1 for every class; the classifier reads + // the structured stderr, which mocks carry via the failure + // error text. + res := ssh.Result{ExitCode: 1, Stderr: []byte(msg)} + if got := classifyRegistryFailure(res); got != want { + t.Fatalf("classifyRegistryFailure(%q) = %q, want %q", msg, got, want) } } + // A transport failure (no stderr, error set) is unreachable-class. + res := ssh.Result{ExitCode: -1, Err: errors.New("ssh: connection timed out")} + if got := classifyRegistryFailure(res); got != "unreachable" { + t.Fatalf("transport failure classified %q, want unreachable", got) + } }) } @@ -544,6 +553,16 @@ func TestDoctorCompatibilityCheck(t *testing.T) { t.Fatalf("absent server binary = %+v, want ok", check) } }) + t.Run("absence is judged by exit code, not failure text", func(t *testing.T) { + // A non-127 failure whose TEXT contains "not found" must not be + // read as absence — that is exactly the text-parse the + // structured result replaces. + mock := ssh.NewMockExecutor("h", ssh.MockCommand{Match: "'/deployments/.bin/teploy' version", Err: errors.New("exit status 1: ld.so: object not found")}) + check := doctorCompatCheck(ctx, deps, mock) + if check.Result != "warn" { + t.Fatalf("non-127 failure with not-found text = %+v, want warn", check) + } + }) t.Run("unreadable server binary warns", func(t *testing.T) { mock := ssh.NewMockExecutor("h", ssh.MockCommand{Match: "'/deployments/.bin/teploy' version", Err: errors.New("exit status 1: permission denied")}) if check := doctorCompatCheck(ctx, deps, mock); check.Result != "warn" { diff --git a/internal/cli/dr.go b/internal/cli/dr.go new file mode 100644 index 0000000..7185bb9 --- /dev/null +++ b/internal/cli/dr.go @@ -0,0 +1,352 @@ +package cli + +import ( + "context" + "encoding/json" + "fmt" + "os" + "os/signal" + + "github.com/spf13/cobra" + "github.com/useteploy/teploy/internal/backup" + "github.com/useteploy/teploy/internal/config" +) + +// The disaster-recovery command family (C07). `teploy backup` remains the +// DATA-ONLY family (volume archives, single-accessory dumps); `teploy dr` +// is the whole-application bundle: state, releases, secrets-by-reference, +// routing identity, and consistency-labeled snapshots, restored to an +// isolated target and promoted only by an explicit cutover. +func newDRCmd(flags *Flags, version string) *cobra.Command { + cmd := &cobra.Command{ + Use: "dr", + Short: "Disaster-recovery bundles: whole-app backup, isolated restore, explicit cutover", + Long: "Create and restore versioned disaster-recovery bundles.\n" + + "A bundle carries app state, release records, secret references (or\n" + + "encrypted material you explicitly opt into), routing/TLS references and\n" + + "engine-aware data snapshots — each labeled with the consistency it\n" + + "actually achieved. Restores land in an ISOLATED staging area and are\n" + + "validated before `teploy dr cutover` promotes them; live overwrite is\n" + + "never the default.\n" + + "For plain volume data without app state use `teploy backup create` — " + + "that family is data-only and stays clearly named as such.", + } + + cmd.AddCommand(newDRCreateCmd(flags, version)) + cmd.AddCommand(newDRListCmd(flags)) + cmd.AddCommand(newDRShowCmd(flags)) + cmd.AddCommand(newDRRestoreCmd(flags)) + cmd.AddCommand(newDRCutoverCmd(flags)) + return cmd +} + +// drStoreFlags registers the bundle-store selection: an S3 target +// (--bucket) or a plain directory reachable on the server (--dir) for +// offline bundles. +func drStoreFlags(cmd *cobra.Command, bucket, region, endpoint, dir *string) { + cmd.Flags().StringVar(bucket, "bucket", "", "S3 bucket for bundles") + cmd.Flags().StringVar(region, "region", "us-east-1", "AWS region") + cmd.Flags().StringVar(endpoint, "endpoint", "", "S3-compatible endpoint URL (MinIO/B2/R2); creds from TEPLOY_S3_ACCESS_KEY/SECRET_KEY or AWS_* env") + cmd.Flags().StringVar(dir, "dir", "", "store bundles in a directory on the server (offline bundles) instead of S3") +} + +// drStore builds the BundleStore from the flags; exactly one target must +// be selected. +func drStore(bucket, region, endpoint, dir string) (backup.BundleStore, error) { + switch { + case dir != "" && bucket != "": + return nil, fmt.Errorf("choose one bundle target: --bucket (S3) or --dir (server directory), not both") + case dir != "": + return backup.DirBundleStore{Root: dir}, nil + case bucket != "": + if err := backup.ValidateBucket(bucket); err != nil { + return nil, err + } + if err := backup.ValidateRegion(region); err != nil { + return nil, err + } + return backup.S3BundleStore{S3: s3Config(bucket, region, endpoint)}, nil + } + return nil, fmt.Errorf("a bundle target is required: --bucket (S3) or --dir (server directory)") +} + +func newDRCreateCmd(flags *Flags, version string) *cobra.Command { + var ( + bucket, region, endpoint, dir string + includeSecrets, includeAgeKey bool + stopApp bool + quiescedVolumes []string + ) + cmd := &cobra.Command{ + Use: "create", + Short: "Create a disaster-recovery bundle of the whole app", + Long: "Captures app state, release records, the applied manifest, secret\n" + + "references, routing/TLS references and data snapshots into one\n" + + "schema-versioned bundle. Secret MATERIAL is included only with\n" + + "--include-secrets (age ciphertexts, resolved .env, accessory\n" + + "credentials), and the age key itself only with --include-age-key.\n" + + "Each snapshot's consistency level is recorded: engine dumps are\n" + + "engine-consistent; raw volume copies are crash-consistent unless the\n" + + "app is stopped (--stop-app) or a volume is asserted quiesced\n" + + "(--quiesced-volume NAME).", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + store, err := drStore(bucket, region, endpoint, dir) + if err != nil { + return err + } + if includeAgeKey && !includeSecrets { + return fmt.Errorf("--include-age-key requires --include-secrets (the key decrypts the material it travels with)") + } + appCfg, err := config.LoadApp(".") + if err != nil { + return err + } + ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt) + defer cancel() + executor, err := connectForApp(ctx, flags, appCfg) + if err != nil { + return err + } + defer executor.Close() + client := backup.NewClient(executor, os.Stdout) + m, err := client.CreateBundle(ctx, backup.BundleOptions{ + App: appCfg.App, + Config: *appCfg, + IncludeSecrets: includeSecrets, + IncludeAgeKey: includeAgeKey, + StopApp: stopApp, + VolumeQuiesced: quiescedVolumes, + Version: version, + }, store) + if err != nil { + return err + } + if flags.JSON { + return json.NewEncoder(os.Stdout).Encode(m) + } + fmt.Printf("Bundle %s:\n", m.ID) + for _, s := range m.Snapshots { + note := "" + if s.Notes != "" { + note = " — " + s.Notes + } + fmt.Printf(" %-14s %-10s %-18s %s%s\n", s.Name, s.Engine, s.Consistency, s.Artifact, note) + } + fmt.Printf(" secrets: %s (%d keys)\n", m.Secrets.Mode, len(m.Secrets.Keys)) + return nil + }, + } + drStoreFlags(cmd, &bucket, ®ion, &endpoint, &dir) + cmd.Flags().BoolVar(&includeSecrets, "include-secrets", false, "include encrypted secret material (age ciphertexts, resolved .env, accessory credentials) — never default") + cmd.Flags().BoolVar(&includeAgeKey, "include-age-key", false, "also include /deployments/.age-key so a fresh host can decrypt (requires --include-secrets)") + cmd.Flags().BoolVar(&stopApp, "stop-app", false, "stop app containers for quiesced volume snapshots (restarted after)") + cmd.Flags().StringSliceVar(&quiescedVolumes, "quiesced-volume", nil, "volume NAME whose writer you stopped by hand (repeatable); recorded as quiesced, operator-asserted") + return cmd +} + +func newDRListCmd(flags *Flags) *cobra.Command { + var bucket, region, endpoint, dir string + cmd := &cobra.Command{ + Use: "list", + Short: "List DR bundles", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + store, err := drStore(bucket, region, endpoint, dir) + if err != nil { + return err + } + appCfg, err := config.LoadApp(".") + if err != nil { + return err + } + ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt) + defer cancel() + executor, err := connectForApp(ctx, flags, appCfg) + if err != nil { + return err + } + defer executor.Close() + ids, err := store.ListIDs(ctx, executor, appCfg.App) + if err != nil { + return err + } + if flags.JSON { + return json.NewEncoder(os.Stdout).Encode(ids) + } + if len(ids) == 0 { + fmt.Println("No DR bundles found") + return nil + } + for _, id := range ids { + fmt.Println(id) + } + return nil + }, + } + drStoreFlags(cmd, &bucket, ®ion, &endpoint, &dir) + return cmd +} + +func newDRShowCmd(flags *Flags) *cobra.Command { + var bucket, region, endpoint, dir string + cmd := &cobra.Command{ + Use: "show ", + Short: "Show a DR bundle's manifest", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + store, err := drStore(bucket, region, endpoint, dir) + if err != nil { + return err + } + appCfg, err := config.LoadApp(".") + if err != nil { + return err + } + ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt) + defer cancel() + executor, err := connectForApp(ctx, flags, appCfg) + if err != nil { + return err + } + defer executor.Close() + data, err := store.FetchManifest(ctx, executor, appCfg.App, args[0]) + if err != nil { + return err + } + m, err := backup.ParseBundleManifest(data) + if err != nil { + return err + } + if flags.JSON { + return json.NewEncoder(os.Stdout).Encode(m) + } + fmt.Printf("Bundle %s (app %s, created %s by %s)\n", m.ID, m.App, m.CreatedAt.Format("2006-01-02 15:04:05 MST"), m.CreatedBy) + for _, s := range m.Snapshots { + fmt.Printf(" %-14s %-10s %-18s %s\n", s.Name, s.Engine, s.Consistency, s.Artifact) + if s.Notes != "" { + fmt.Printf(" %s\n", s.Notes) + } + } + fmt.Printf(" secrets: %s (%d keys, age-key included: %v)\n", m.Secrets.Mode, len(m.Secrets.Keys), m.Secrets.AgeKeyIncluded) + fmt.Printf(" routing: domain=%s ingress=%s\n", m.Routing.Domain, m.Routing.IngressMode) + fmt.Println(" recovery plan:") + for _, step := range m.Recovery.Steps { + manual := "" + if step.Manual { + manual = " [manual]" + } + fmt.Printf(" %-20s %s%s\n", step.Action, step.Summary, manual) + } + return nil + }, + } + drStoreFlags(cmd, &bucket, ®ion, &endpoint, &dir) + return cmd +} + +func newDRRestoreCmd(flags *Flags) *cobra.Command { + var bucket, region, endpoint, dir string + cmd := &cobra.Command{ + Use: "restore ", + Short: "Restore a DR bundle to an ISOLATED target and validate it (live state untouched)", + Long: "Downloads the bundle into /var/tmp/teploy-dr staging, boots scratch\n" + + "engines and a scratch app container against the STAGED data, and checks\n" + + "the restored copy is actually usable. Writes a receipt with measured\n" + + "RPO/RTO. Nothing under /deployments is touched — promotion happens only\n" + + "via `teploy dr cutover`.\n" + + "Missing secret keys (references mode) and undecryptable material fail\n" + + "BEFORE any mutation.", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + store, err := drStore(bucket, region, endpoint, dir) + if err != nil { + return err + } + appCfg, err := config.LoadApp(".") + if err != nil { + return err + } + ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt) + defer cancel() + executor, err := connectForApp(ctx, flags, appCfg) + if err != nil { + return err + } + defer executor.Close() + client := backup.NewClient(executor, os.Stdout) + receipt, err := client.RestoreBundleIsolated(ctx, backup.BundleRestoreOptions{ + App: appCfg.App, + ID: args[0], + Config: *appCfg, + }, store) + if err != nil { + return err + } + if flags.JSON { + return json.NewEncoder(os.Stdout).Encode(receipt) + } + printReceipt(receipt) + if !receipt.OK { + return fmt.Errorf("restore validation failed — see the receipt; nothing live was touched") + } + return nil + }, + } + drStoreFlags(cmd, &bucket, ®ion, &endpoint, &dir) + return cmd +} + +func printReceipt(r *backup.RestoreReceipt) { + fmt.Printf("Restore receipt for bundle %s:\n", r.BundleID) + fmt.Printf(" staging: %s\n", r.StagingPath) + fmt.Printf(" RPO: %ds (data age at restore time)\n", r.RPOSeconds) + fmt.Printf(" RTO: %ds (measured restore + validation)\n", r.RTOSeconds) + for _, ck := range r.Checks { + fmt.Printf(" check %-28s %-7s %s %s\n", ck.Name, ck.Status, ck.Metric, ck.Detail) + } + fmt.Printf(" next: %s\n", r.NextStep) +} + +func newDRCutoverCmd(flags *Flags) *cobra.Command { + cmd := &cobra.Command{ + Use: "cutover ", + Short: "Explicitly promote a validated staged restore over the live app", + Long: "The mutation step — never run by accident. Requires a staged restore\n" + + "that PASSED validation (`teploy dr restore`). Stops live containers,\n" + + "restores engine dumps into fresh accessories, promotes staged volumes\n" + + "with two-phase recovery (pre-cutover originals are kept), and installs\n" + + "the bundle's state/release records/secrets. Finishes by pointing at\n" + + "`teploy deploy` to bring the app container and routing live.", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + appCfg, err := config.LoadApp(".") + if err != nil { + return err + } + ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt) + defer cancel() + executor, err := connectForApp(ctx, flags, appCfg) + if err != nil { + return err + } + defer executor.Close() + client := backup.NewClient(executor, os.Stdout) + receipt, err := client.CutoverBundle(ctx, backup.BundleRestoreOptions{ + App: appCfg.App, + ID: args[0], + Config: *appCfg, + }) + if err != nil { + return err + } + if flags.JSON { + return json.NewEncoder(os.Stdout).Encode(receipt) + } + fmt.Printf("Cutover receipt for bundle %s: %d path(s) promoted, originals kept in %v\n", + receipt.BundleID, len(receipt.Promoted), receipt.RecoveryDirs) + return nil + }, + } + return cmd +} diff --git a/internal/cli/registry.go b/internal/cli/registry.go index 5ce1b2e..93348df 100644 --- a/internal/cli/registry.go +++ b/internal/cli/registry.go @@ -87,20 +87,36 @@ func runRegistryLogin(flags *Flags, registry, serverName, username, password str } } - // Single-quote every value so the remote shell can't expand or execute it. - // `echo %q` used double quotes, under which $/backticks in the password (or - // registry/username) still expand — a shell-injection running as the SSH - // user. printf '%s' emits the password literally to docker --password-stdin. - cmd := fmt.Sprintf("printf '%%s' %s | docker login %s -u %s --password-stdin", - ssh.ShellQuote(password), ssh.ShellQuote(registry), ssh.ShellQuote(username)) - if _, err := executor.Run(ctx, cmd); err != nil { - return fmt.Errorf("docker login failed: %w", err) + // The password travels over the SSH session's stdin, never in the + // command string: the old `printf '%s' '' | docker login + // --password-stdin` put the secret in the session shell's argv — + // visible in the server's process list for the life of the login + // and in any command-bearing error output. docker reads it from + // stdin either way; only the transport channel changed (C08). + if err := registryLoginOnServer(ctx, executor, registry, username, password); err != nil { + return err } fmt.Printf("Logged in to %s on server\n", registry) return nil } +// registryLoginOnServer runs `docker login` on the server with the +// password streamed over stdin. Split from runRegistryLogin so the +// transport contract is pinnable without standing up the connect path. +func registryLoginOnServer(ctx context.Context, executor ssh.Executor, registry, username, password string) error { + cmd := fmt.Sprintf("docker login %s -u %s --password-stdin", + ssh.ShellQuote(registry), ssh.ShellQuote(username)) + res := ssh.RunInputDetailed(ctx, executor, cmd, strings.NewReader(password)) + if res.Err != nil { + return fmt.Errorf("docker login failed: %w", res.Err) + } + if res.ExitCode != 0 { + return fmt.Errorf("docker login failed: %s", res.ExitErrorText()) + } + return nil +} + func newRegistryListCmd(flags *Flags) *cobra.Command { var server string @@ -166,16 +182,16 @@ func runRegistryList(flags *Flags, serverName string) error { } defer executor.Close() - output, err := executor.Run(ctx, "cat ~/.docker/config.json 2>/dev/null || echo '{}'") + // A read failure must not become empty state (C08): the old + // `cat config 2>/dev/null || echo '{}'` turned an unreadable config + // (permissions, I/O error) into "No registries configured". The + // framed form distinguishes confirmed absence (no config file — the + // fresh-docker case) from a real read failure, which errors. + entries, err := registryListFromServer(ctx, executor) if err != nil { return err } - entries, err := parseDockerAuths(output) - if err != nil { - return fmt.Errorf("parsing ~/.docker/config.json: %w", err) - } - // --json is a documented, working global flag on every other // list-style command — this one never checked it at all, always // printing the human-readable format regardless. teploy-dash calls @@ -232,6 +248,30 @@ func runRegistryRemove(flags *Flags, registry, serverName string) error { return nil } +// registryListFromServer reads the server's ~/.docker/config.json and +// returns its registry entries. Confirmed absence of the file is the +// normal no-registries state (empty slice); a file that exists but +// cannot be read is an error — never an empty list masquerading as +// "nothing configured". +func registryListFromServer(ctx context.Context, executor ssh.Executor) ([]RegistryEntry, error) { + res := ssh.RunDetailed(ctx, executor, "if [ ! -f ~/.docker/config.json ]; then printf 'absent\\n'; else cat ~/.docker/config.json; fi") + if res.Err != nil { + return nil, res.Err + } + if res.ExitCode != 0 { + return nil, fmt.Errorf("reading ~/.docker/config.json on the server: %s", res.ExitErrorText()) + } + output := res.TrimmedStdout() + if output == "absent" { + return nil, nil + } + entries, err := parseDockerAuths(output) + if err != nil { + return nil, fmt.Errorf("parsing ~/.docker/config.json: %w", err) + } + return entries, nil +} + // connectForRegistry establishes SSH connection using server flag, app config, or flags. func connectForRegistry(ctx context.Context, flags *Flags, serverName string) (ssh.Executor, error) { if serverName == "" { diff --git a/internal/cli/registry_test.go b/internal/cli/registry_test.go index b634e1c..505688d 100644 --- a/internal/cli/registry_test.go +++ b/internal/cli/registry_test.go @@ -1,9 +1,14 @@ package cli import ( + "context" "encoding/json" + "errors" "sort" + "strings" "testing" + + "github.com/useteploy/teploy/internal/ssh" ) // TestParseDockerAuths_RealConfigStructure reproduces the shape of a real @@ -74,3 +79,49 @@ func TestParseDockerAuths_ProducesValidJSON(t *testing.T) { t.Errorf("round-tripped entries = %+v", roundTrip) } } + +// TestRegistryListFromServer_ReadFailureIsNotEmptyState pins the C08 +// acceptance for reads: a config.json that EXISTS but cannot be read is +// an error, never an empty list that reports "No registries +// configured"; confirmed absence of the file is the legitimate empty +// state. +func TestRegistryListFromServer_ReadFailureIsNotEmptyState(t *testing.T) { + ctx := context.Background() + + t.Run("absent file is the empty state", func(t *testing.T) { + mock := ssh.NewMockExecutor("h", + ssh.MockCommand{Match: "if [ ! -f ~/.docker/config.json ]", Output: "absent"}) + entries, err := registryListFromServer(ctx, mock) + if err != nil || len(entries) != 0 { + t.Fatalf("absent config = %v, %v; want empty, nil", entries, err) + } + }) + + t.Run("read failure errors", func(t *testing.T) { + mock := ssh.NewMockExecutor("h", + ssh.MockCommand{Match: "if [ ! -f ~/.docker/config.json ]", Err: errors.New("exit status 1: cat: /root/.docker/config.json: Permission denied")}) + if _, err := registryListFromServer(ctx, mock); err == nil { + t.Fatal("an unreadable config.json must error, not read as no-registries") + } + }) + + t.Run("present config parses", func(t *testing.T) { + mock := ssh.NewMockExecutor("h", + ssh.MockCommand{Match: "if [ ! -f ~/.docker/config.json ]", Output: `{"auths":{"ghcr.io":{"auth":"dTpw"}}}`}) + entries, err := registryListFromServer(ctx, mock) + if err != nil || len(entries) != 1 || entries[0].Server != "ghcr.io" { + t.Fatalf("present config = %v, %v", entries, err) + } + }) + + t.Run("framed read touches the server once", func(t *testing.T) { + mock := ssh.NewMockExecutor("h", + ssh.MockCommand{Match: "if [ ! -f ~/.docker/config.json ]", Output: "absent"}) + if _, err := registryListFromServer(ctx, mock); err != nil { + t.Fatal(err) + } + if len(mock.Calls) != 1 || !strings.HasPrefix(mock.Calls[0], "if [ ! -f ~/.docker/config.json ]") { + t.Fatalf("unexpected command shape: %v", mock.Calls) + } + }) +} diff --git a/internal/cli/root.go b/internal/cli/root.go index 055ae19..57eed3e 100644 --- a/internal/cli/root.go +++ b/internal/cli/root.go @@ -73,6 +73,7 @@ func NewRootCmd(version string) *cobra.Command { root.AddCommand(newStatsCmd(flags)) root.AddCommand(newHealthCmd(flags)) root.AddCommand(newBackupCmd(flags)) + root.AddCommand(newDRCmd(flags, version)) root.AddCommand(newLockCmd(flags)) root.AddCommand(newUnlockCmd(flags)) root.AddCommand(newInitCmd()) diff --git a/internal/cli/setup.go b/internal/cli/setup.go index 1507ec6..4f999b8 100644 --- a/internal/cli/setup.go +++ b/internal/cli/setup.go @@ -55,6 +55,13 @@ Examples: return cmd } +// setupReconnectBudget bounds how many times the setup flow redials +// after a transport failure before giving the stage-named error back to +// the operator (C08 connection recovery). Three covers a transient +// network blip and one brief outage without turning a dead box into a +// multi-minute hang. +const setupReconnectBudget = 3 + func runSetup(flags *Flags, host string, name string, noHarden bool, networkProvider string, authKey string, usePassword bool, yes bool) error { user := flags.User if user == "" { @@ -83,7 +90,17 @@ func runSetup(flags *Flags, host string, name string, noHarden bool, networkProv fmt.Printf("Connecting to %s...\n", host) - executor, err := ssh.Connect(ctx, cfg) + // The whole setup flow runs through a reconnecting executor (C08 + // connection recovery): a transport failure mid-provisioning — SSH + // channel death, connection reset — redials and retries the one + // invocation that died, instead of aborting at whatever stage it + // hit. Setup, hardening, and network join are check-then-act + // idempotent, so every retried command is safe to re-run; commands + // that RAN and failed are never retried here. + dial := func(ctx context.Context) (ssh.Executor, error) { + return ssh.Connect(ctx, cfg) + } + executor, err := ssh.NewReconnectingExecutor(ctx, dial, setupReconnectBudget) if err != nil { return err } @@ -98,13 +115,8 @@ func runSetup(flags *Flags, host string, name string, noHarden bool, networkProv if err != nil { return fmt.Errorf("deriving SSH public key: %w", err) } - pubKey := strings.TrimSpace(string(pubKeyData)) - installCmd := fmt.Sprintf( - "mkdir -p ~/.ssh && echo %s >> ~/.ssh/authorized_keys && chmod 700 ~/.ssh && chmod 600 ~/.ssh/authorized_keys", - ssh.ShellQuote(pubKey), - ) - if _, err := executor.Run(ctx, installCmd); err != nil { - return fmt.Errorf("installing SSH key: %w", err) + if err := installAuthorizedKey(ctx, executor, strings.TrimSpace(string(pubKeyData))); err != nil { + return err } fmt.Println("SSH key installed") } @@ -152,22 +164,14 @@ func runSetup(flags *Flags, host string, name string, noHarden bool, networkProv } rootExec.Close() } else { - // Root SSH denied — use expect-style su via the existing tyler connection. - // Write a helper script that uses su with the password from a file. - script := fmt.Sprintf(`#!/bin/bash -exec 2>&1 -echo '%s' | su -c 'DEBIAN_FRONTEND=noninteractive apt-get update -qq && DEBIAN_FRONTEND=noninteractive apt-get install -y -qq sudo >/dev/null 2>&1 && usermod -aG sudo %s && echo "%s ALL=(ALL) NOPASSWD:ALL" > /etc/sudoers.d/%s && chmod 440 /etc/sudoers.d/%s && echo TEPLOY_SUDO_OK' - root 2>&1 -`, strings.ReplaceAll(rootPass, "'", "'\"'\"'"), user, user, user, user) - if err := executor.Upload(ctx, strings.NewReader(script), "/tmp/teploy_install_sudo.sh", "0700"); err != nil { - return fmt.Errorf("uploading sudo installer: %w", err) - } - out, err := executor.Run(ctx, "/tmp/teploy_install_sudo.sh") - executor.Run(ctx, "rm -f /tmp/teploy_install_sudo.sh") - if err != nil || !strings.Contains(out, "TEPLOY_SUDO_OK") { - if strings.Contains(out, "Authentication failure") { - return fmt.Errorf("wrong root password") - } - return fmt.Errorf("installing sudo via su failed: %s", out) + // Root SSH denied — run su over the existing connection. + // The root password travels the session's stdin and nowhere + // else: the old path embedded it in a script uploaded to + // /tmp (an on-disk artifact) whose removal was best-effort — + // a failed run could leave the root password sitting in + // /tmp (C08). + if err := installSudoViaSu(ctx, executor, user, rootPass); err != nil { + return err } } fmt.Printf(" sudo installed, %s added to sudo group\n", user) @@ -205,7 +209,10 @@ echo '%s' | su -c 'DEBIAN_FRONTEND=noninteractive apt-get update -qq && DEBIAN_F KeyPath: flags.Key, AcceptNewHost: true, } - executor, err = ssh.Connect(ctx, reconnectCfg) + vpnDial := func(ctx context.Context) (ssh.Executor, error) { + return ssh.Connect(ctx, reconnectCfg) + } + executor, err = ssh.NewReconnectingExecutor(ctx, vpnDial, setupReconnectBudget) if err != nil { return fmt.Errorf("reconnecting via VPN IP %s: %w", vpnIP, err) } @@ -238,6 +245,45 @@ echo '%s' | su -c 'DEBIAN_FRONTEND=noninteractive apt-get update -qq && DEBIAN_F return nil } +// installSudoViaSu installs sudo + configures passwordless sudo for +// user by feeding su the root password over the session's stdin. No +// temp file, no password in any command string (C08). Success is the +// TEPLOY_SUDO_OK marker — su's own exit status alone does not prove the +// whole chain ran. +func installSudoViaSu(ctx context.Context, executor ssh.Executor, user, rootPass string) error { + inner := fmt.Sprintf( + `DEBIAN_FRONTEND=noninteractive apt-get update -qq && DEBIAN_FRONTEND=noninteractive apt-get install -y -qq sudo >/dev/null 2>&1 && usermod -aG sudo %s && printf '%%s\n' %s > /etc/sudoers.d/%s && chmod 440 /etc/sudoers.d/%s && echo TEPLOY_SUDO_OK`, + user, ssh.ShellQuote(user+" ALL=(ALL) NOPASSWD:ALL"), user, user) + cmd := "su -c " + ssh.ShellQuote(inner) + " - root" + res := ssh.RunInputDetailed(ctx, executor, cmd, strings.NewReader(rootPass+"\n")) + combined := string(res.Stdout) + "\n" + string(res.Stderr) + if res.Err == nil && strings.Contains(combined, "TEPLOY_SUDO_OK") { + return nil + } + if strings.Contains(combined, "Authentication failure") { + return fmt.Errorf("wrong root password") + } + if res.Err != nil { + return fmt.Errorf("installing sudo via su: %w", res.Err) + } + return fmt.Errorf("installing sudo via su failed: %s", strings.TrimSpace(combined)) +} + +// installAuthorizedKey appends the public key to ~/.ssh/authorized_keys +// — GUARDED, so an interrupted setup re-run cannot stack duplicate +// entries (C08 resumability): the old bare `echo >>` appended on +// every attempt. +func installAuthorizedKey(ctx context.Context, executor ssh.Executor, pubKey string) error { + cmd := fmt.Sprintf( + "mkdir -p ~/.ssh && grep -qF %s ~/.ssh/authorized_keys 2>/dev/null || echo %s >> ~/.ssh/authorized_keys; chmod 700 ~/.ssh && chmod 600 ~/.ssh/authorized_keys", + ssh.ShellQuote(pubKey), ssh.ShellQuote(pubKey), + ) + if _, err := executor.Run(ctx, cmd); err != nil { + return fmt.Errorf("installing SSH key: %w", err) + } + return nil +} + // setupNetwork installs the VPN provider, joins the mesh, and returns the VPN IP. func setupNetwork(ctx context.Context, exec ssh.Executor, w io.Writer, providerName string, authKeyFlag string) (string, error) { cfg, err := resolveNetworkConfig(providerName, authKeyFlag) @@ -290,18 +336,9 @@ func setupNetwork(ctx context.Context, exec ssh.Executor, w io.Writer, providerN // Tailscale/Headscale modifies iptables which can kill the SSH connection, // so we detach the command and poll from the local machine instead. fmt.Fprintf(w, "Joining %s mesh...\n", providerName) - var joinCmd string - // Single-quote user-provided mesh credentials (auth keys, login server) so a - // value with a shell metacharacter can't break out of the join command. - switch providerName { - case "tailscale": - joinCmd = fmt.Sprintf(sudo+"nohup tailscale up --authkey=%s --accept-routes >/dev/null 2>&1 &", ssh.ShellQuote(cfg.AuthKey)) - case "headscale": - joinCmd = fmt.Sprintf(sudo+"nohup tailscale up --login-server=%s --authkey=%s --accept-routes >/dev/null 2>&1 &", ssh.ShellQuote(cfg.Server), ssh.ShellQuote(cfg.AuthKey)) - case "netbird": - joinCmd = fmt.Sprintf(sudo+"nohup netbird up --setup-key %s >/dev/null 2>&1 &", ssh.ShellQuote(cfg.SetupKey)) + if err := joinVPNMesh(ctx, exec, sudo, providerName, cfg); err != nil { + return "", err } - exec.Run(ctx, joinCmd) // ignore error — connection may die // Poll locally for the node to appear on our tailnet. fmt.Fprintf(w, " Waiting for node to appear on tailnet...\n") @@ -333,6 +370,63 @@ func setupNetwork(ctx context.Context, exec ssh.Executor, w io.Writer, providerN return vpnIP, nil } +// vpnCredential resolves the provider's join credential and the env var +// its CLI documents for it (tailscale up reads TS_AUTHKEY, netbird up +// reads NB_SETUP_KEY). +func vpnCredential(providerName string, cfg network.Config) (envVar, value string, err error) { + switch providerName { + case "tailscale", "headscale": + return "TS_AUTHKEY", cfg.AuthKey, nil + case "netbird": + return "NB_SETUP_KEY", cfg.SetupKey, nil + default: + return "", "", fmt.Errorf("unknown network provider: %q", providerName) + } +} + +// joinVPNMesh stages the join credential in a private file and fires the +// detached provider join. The credential never enters any command +// string: the detached shell reads the file into the provider's +// documented env var, removes the file BEFORE exec'ing the provider, +// then execs (C08) — the transport lives in the network package +// (StageJoinCredential/EnvVarJoinShell) and is shared with +// `teploy network join`. Failure semantics: an upload failure aborts +// setup with the file path named (nothing was joined); a join failure +// after that surfaces through the tailnet wait below, and the key file +// is gone regardless — its only reader is the rm-ing shell itself. +func joinVPNMesh(ctx context.Context, exec ssh.Executor, sudo, providerName string, cfg network.Config) error { + _, credential, err := vpnCredential(providerName, cfg) + if err != nil { + return err + } + if credential == "" { + return fmt.Errorf("%s join credential is empty", providerName) + } + keyPath, err := network.StageJoinCredential(ctx, exec, credential) + if err != nil { + return err + } + _, _ = exec.Run(ctx, vpnJoinCommand(providerName, cfg, sudo, keyPath)) + return nil +} + +// vpnJoinCommand renders the detached join: `cat` the key file into the +// provider's env var, remove the file, then exec the provider. The +// login-server URL (headscale) is not a secret and stays a plain flag. +func vpnJoinCommand(providerName string, cfg network.Config, sudo, keyPath string) string { + envVar, _, _ := vpnCredential(providerName, cfg) + var tail string + switch providerName { + case "tailscale": + tail = "tailscale up --accept-routes" + case "headscale": + tail = "tailscale up --login-server=" + ssh.ShellQuote(cfg.Server) + " --accept-routes" + case "netbird": + tail = "netbird up" + } + return sudo + "nohup " + network.EnvVarJoinShell(envVar, keyPath, tail) + " >/dev/null 2>&1 &" +} + // runLocal executes a command on the local machine and returns its output. func runLocal(name string, args ...string) (string, error) { cmd := osexec.Command(name, args...) @@ -413,370 +507,458 @@ type dockerMount struct { RW bool } -// setupServer runs the provisioning steps on a connected server. -// Separated from runSetup for testability with MockExecutor. -// yes skips interactive confirmation for destructive upgrade steps (Caddy recreate). -func setupServer(ctx context.Context, exec ssh.Executor, w io.Writer, yes bool) error { - // Detect whether we need sudo (non-root users). - sudo := "" - if whoami, _ := exec.Run(ctx, "whoami"); strings.TrimSpace(whoami) != "root" { - sudo = "sudo " - } - - // 1. Check/install Docker - fmt.Fprintln(w, "Checking Docker...") - if _, err := exec.Run(ctx, "docker --version"); err != nil { - fmt.Fprintln(w, " Installing Docker...") - - // Try curl first, fall back to wget. - installCmd := sudo + "sh -c 'curl -fsSL https://get.docker.com | sh'" - if _, curlErr := exec.Run(ctx, "which curl"); curlErr != nil { - installCmd = sudo + "sh -c 'wget -qO- https://get.docker.com | sh'" - } - - out, err := exec.Run(ctx, installCmd) - if err != nil { - // Show output on failure for debugging. - fmt.Fprintln(w, out) - return fmt.Errorf("installing docker: %w", err) - } +// setupStage is one resumable provisioning step (C08): each stage is +// check-then-act idempotent, so an interrupted setup re-run skips what +// completed and continues — no duplicate or corrupt state. affects is +// the preflight's affected-resource description. +type setupStage struct { + name string + affects string + run func(ctx context.Context, exec ssh.Executor, w io.Writer, env *setupEnv) error +} - // Verify Docker actually installed and print version. - ver, err := exec.Run(ctx, "docker --version") - if err != nil { - return fmt.Errorf("docker install appeared to succeed but docker is not available") - } - fmt.Fprintf(w, " Docker installed (%s)\n", strings.TrimPrefix(strings.TrimSpace(ver), "Docker version ")) - - // Add current user to docker group so sudo isn't needed for docker commands. - exec.Run(ctx, sudo+"usermod -aG docker $(whoami)") - } else { - fmt.Fprintln(w, " Docker already installed") - } - - // 2. Check/install rsync — required by type:static deploys (internal/deploy - // static.go shells out to it directly). Not preinstalled on minimal - // Debian/Ubuntu cloud images, so a fresh box otherwise deploys containers - // fine but fails static deploys on the first rsync with a cryptic - // "command not found" from the remote shell. - fmt.Fprintln(w, "Checking rsync...") - if _, err := exec.Run(ctx, "rsync --version"); err != nil { - fmt.Fprintln(w, " Installing rsync...") - installCmd := sudo + "sh -c 'DEBIAN_FRONTEND=noninteractive apt-get update -qq && DEBIAN_FRONTEND=noninteractive apt-get install -y -qq rsync'" - if out, err := exec.Run(ctx, installCmd); err != nil { - fmt.Fprintln(w, out) - return fmt.Errorf("installing rsync: %w", err) - } - fmt.Fprintln(w, " rsync installed") - } else { - fmt.Fprintln(w, " rsync already installed") - } - - // 3. Check firewall - fmt.Fprintln(w, "Checking firewall...") - ufwOutput, ufwErr := exec.Run(ctx, "ufw status 2>/dev/null") - if ufwErr == nil && strings.Contains(ufwOutput, "Status: active") { - _, err1 := exec.Run(ctx, sudo+"ufw allow 80/tcp") - _, err2 := exec.Run(ctx, sudo+"ufw allow 443/tcp") - if err1 != nil || err2 != nil { - fmt.Fprintln(w, " Warning: could not configure ufw. Ensure ports 80 and 443 are open.") - } else { - fmt.Fprintln(w, " Opened ports 80 and 443 (ufw)") - } - } else if _, err := exec.Run(ctx, "systemctl is-active firewalld 2>/dev/null"); err == nil { - fmt.Fprintln(w, " Warning: firewalld detected. Ensure ports 80 and 443 are open.") - } else { - fmt.Fprintln(w, " No active firewall detected") - } - - // Determine docker prefix: use sudo if user can't access docker directly. - dockerCmd := "docker" - if _, err := exec.Run(ctx, "docker info >/dev/null 2>&1"); err != nil { - dockerCmd = sudo + "docker" - } - - // 4. Create Docker network - fmt.Fprintln(w, "Creating Docker network...") - netCmd := dockerCmd + " network inspect teploy >/dev/null 2>&1 || " + dockerCmd + " network create teploy" - if _, err := exec.Run(ctx, netCmd); err != nil { - return fmt.Errorf("creating docker network: %w", err) - } - - // 5. Create directories and upload Caddyfile - if _, err := exec.Run(ctx, sudo+"mkdir -p /deployments/caddy"); err != nil { - return fmt.Errorf("creating directories: %w", err) - } - // Ensure the deploy user owns the CONTROL-PLANE directories only — - // never the whole /deployments tree. The old `chown -R - // $(whoami):$(whoami) /deployments` reassigned every application's - // bind-mounted data (database files owned by engine UIDs, accessory - // state) to the interactive SSH user on every re-run, breaking engines - // that rely on their own numeric ownership (TCL-26). Existing app data - // ownership is an invariant, not setup's cleanup target: app - // directories created later are owned by this user anyway, and the - // error is propagated instead of ignored. - if _, err := exec.Run(ctx, sudo+"chown $(whoami):$(whoami) /deployments /deployments/caddy"); err != nil { - return fmt.Errorf("setting control-plane directory ownership: %w", err) - } - - // Caddy admin API listens on 0.0.0.0 inside container so Docker port - // forwarding can reach it. Port 2019 is only published to 127.0.0.1 - // on the host — never publicly accessible. - // Tab-indented to match `caddy fmt` output so Caddy doesn't warn. - // Only write the stub Caddyfile when none exists — on servers that - // were provisioned by other tooling (e.g., Dokploy) or hand-edited, - // the existing Caddyfile holds live production routes and must be - // preserved. - const stubCaddyfile = "{\n\tadmin 127.0.0.1:2019\n}\n" - // Only write the stub Caddyfile when the file is confirmed ABSENT — - // `test -s` also fails for an unreadable or empty-but-present file, and - // overwriting either with a stub discards live routes (TCL-27 - // containment). An existing empty file is left for the operator to - // inspect rather than silently clobbered. - present, err := exec.Run(ctx, "[ -f /deployments/caddy/Caddyfile ] && echo present || echo absent") - if err != nil { - return fmt.Errorf("checking for an existing Caddyfile: %w", err) - } - if strings.TrimSpace(present) == "absent" { - if err := exec.Upload(ctx, strings.NewReader(stubCaddyfile), "/deployments/caddy/Caddyfile", "0644"); err != nil { - return fmt.Errorf("uploading Caddyfile: %w", err) - } - } else { - fmt.Fprintln(w, " Existing Caddyfile preserved") - // Lock the admin API to the container loopback. Older setups bound it to - // 0.0.0.0:2019, reachable by any container on the teploy network. - exec.Run(ctx, "sed -i 's/admin 0.0.0.0:2019/admin 127.0.0.1:2019/' /deployments/caddy/Caddyfile") - } - - // 6. Start Caddy (idempotent — skip if container already exists). - // The on-disk Caddyfile is Teploy's single source of truth: Caddy loads it - // on every boot and `caddy reload`, so we run WITHOUT `--resume` (which - // would boot from admin-API autosave and shadow the file). The admin API - // binds the container loopback only and is never exposed off-box. - fmt.Fprintln(w, "Starting Caddy...") - caddyCheck, err := exec.Run(ctx, dockerCmd+" ps -a --filter name=^caddy$ --format '{{.Names}}'") - if err != nil { - return fmt.Errorf("checking for an existing caddy container: %w", err) - } - extraNetworks := []string{} - var extraMountFlags []string - if strings.TrimSpace(caddyCheck) != "" { - // Three legacy conditions require recreating the Caddy container. All - // three recreations are destructive (brief outage + re-attaching - // non-teploy networks/mounts), so we require explicit confirmation. - // Inventory reads must SUCCEED before any recreate decision: a failed - // inspect used to read as an empty set, which both triggered - // unnecessary migrations and silently dropped adopted networks/mounts - // during a real one (audit F67). - cmdOut, cmdErr := exec.Run(ctx, dockerCmd+" inspect -f '{{join .Config.Cmd \" \"}}' caddy") - if cmdErr != nil { - return fmt.Errorf("cannot safely inventory the existing caddy container (cmd): %w", cmdErr) - } - mountOut, mountErr := exec.Run(ctx, dockerCmd+" inspect -f '{{range .Mounts}}{{.Destination}} {{end}}' caddy") - if mountErr != nil { - return fmt.Errorf("cannot safely inventory the existing caddy container (mounts): %w", mountErr) - } +// setupEnv carries state computed by early stages for later ones. +type setupEnv struct { + sudo string + dockerCmd string + yes bool +} - // (1) --resume boots from admin-API autosave, shadowing the Caddyfile. - legacyResume := strings.Contains(cmdOut, "--resume") - // (2) A single-file Caddyfile bind mount (Destination - // /etc/caddy/Caddyfile) is pinned by Docker to the file's original - // inode. Teploy writes the Caddyfile atomically (write tmp + mv), - // which swaps the inode — so the running container never sees route - // updates and `caddy reload` reloads stale config. The directory - // mount (/etc/caddy) re-resolves the file by path on each reload and - // avoids this. Detect the legacy file mount and migrate. - legacyFileMount := strings.Contains(mountOut, "/etc/caddy/Caddyfile") - // (3) No /deployments bind mount at all means type:static deploys - // are completely non-functional on this server — Deploy writes - // releases to /deployments//current on the host, and Caddy's - // generated site block reads from {DefaultStaticMount}//current - // (also /deployments now — see internal/deploy/static.go), but - // without this mount that path doesn't exist inside the container - // at all. Confirmed live: `teploy deploy` reports success (the - // release really does land on disk) while every request 404s. - // Every server provisioned before this fix landed is missing this - // mount unconditionally — not a legacy-config edge case, a gap in - // what teploy setup has always provisioned. - missingStaticMount := !strings.Contains(mountOut, "/deployments ") - - if legacyResume || legacyFileMount || missingStaticMount { - // Capture every network the existing Caddy is attached to so - // we can reattach them after recreating. Previously these were - // silently dropped, leaving apps on other networks (e.g. - // dokploy-network) unreachable. - netOut, netErr := exec.Run(ctx, dockerCmd+" inspect -f '{{range $k,$v := .NetworkSettings.Networks}}{{$k}} {{end}}' caddy") - if netErr != nil { - return fmt.Errorf("cannot safely inventory the existing caddy container (networks): %w", netErr) - } - for _, n := range strings.Fields(netOut) { - if n != "" && n != "teploy" { - extraNetworks = append(extraNetworks, n) +// setupStages is the ordered provisioning plan. The ORDER is part of +// the contract: docker before the network that needs it, the network +// before the container that joins it, directories before the files +// that land in them. +func setupStages() []setupStage { + return []setupStage{ + { + name: "sudo detection", + affects: "nothing (read-only: whoami)", + run: func(ctx context.Context, exec ssh.Executor, w io.Writer, env *setupEnv) error { + if whoami, _ := exec.Run(ctx, "whoami"); strings.TrimSpace(whoami) != "root" { + env.sudo = "sudo " } - } - - // Capture every mount teploy itself didn't put there, so those - // are preserved too — the same principle as extraNetworks - // above, just for volumes. Without this, recreating Caddy - // blindly replaces its whole mount set with teploy's own fixed - // list (caddy_data, caddy_config, /etc/caddy, /deployments), - // silently dropping anything hand-added on a server that - // predates teploy or was adopted from other tooling — e.g. a - // legacy /srv/static bind mount serving live static sites, - // which would 404 the instant the container came back up. - // Confirmed live on a real production box before this fix: - // exactly that mount existed and would have been lost. - // This is what actually makes it safe to run `teploy setup` - // against an existing, previously-hand-managed server — not - // just a fresh one. - var extraMounts []dockerMount - // The detailed mount inventory must SUCCEED before the - // destructive recreate below: a failed inspect or unparseable - // JSON used to read as "no extra mounts", silently dropping - // adopted volumes during the migration (TCL-27). - mountJSON, mountJSONErr := exec.Run(ctx, dockerCmd+" inspect -f '{{json .Mounts}}' caddy") - if mountJSONErr != nil { - return fmt.Errorf("cannot safely inventory the existing caddy container (mount detail): %w", mountJSONErr) - } - teployDestinations := map[string]bool{ - "/data": true, "/config": true, "/etc/caddy": true, "/deployments": true, - } - var mounts []dockerMount - if err := json.Unmarshal([]byte(mountJSON), &mounts); err != nil { - return fmt.Errorf("decoding the existing caddy container's mount inventory: %w", err) - } - for _, m := range mounts { - if m.Type == "volume" && (m.Name == "caddy_data" || m.Name == "caddy_config") { - continue // teploy's own named volumes, re-added explicitly below + return nil + }, + }, + { + name: "docker", + affects: "Docker engine (installed via get.docker.com if missing), current user in the docker group", + run: func(ctx context.Context, exec ssh.Executor, w io.Writer, env *setupEnv) error { + // 1. Check/install Docker + fmt.Fprintln(w, "Checking Docker...") + if _, err := exec.Run(ctx, "docker --version"); err != nil { + fmt.Fprintln(w, " Installing Docker...") + + // Try curl first, fall back to wget. + installCmd := env.sudo + "sh -c 'curl -fsSL https://get.docker.com | sh'" + if _, curlErr := exec.Run(ctx, "which curl"); curlErr != nil { + installCmd = env.sudo + "sh -c 'wget -qO- https://get.docker.com | sh'" + } + + out, err := exec.Run(ctx, installCmd) + if err != nil { + // Show output on failure for debugging. + fmt.Fprintln(w, out) + return fmt.Errorf("installing docker: %w", err) + } + + // Verify Docker actually installed and print version. + ver, err := exec.Run(ctx, "docker --version") + if err != nil { + return fmt.Errorf("docker install appeared to succeed but docker is not available") + } + fmt.Fprintf(w, " Docker installed (%s)\n", strings.TrimPrefix(strings.TrimSpace(ver), "Docker version ")) + + // Add current user to docker group so sudo isn't needed for docker commands. + exec.Run(ctx, env.sudo+"usermod -aG docker $(whoami)") + } else { + fmt.Fprintln(w, " Docker already installed") } - if teployDestinations[m.Destination] { - continue // teploy's own bind mounts, re-added explicitly below + return nil + }, + }, + { + name: "rsync", + affects: "rsync package (apt, if missing) — required by type:static deploys", + run: func(ctx context.Context, exec ssh.Executor, w io.Writer, env *setupEnv) error { + // 2. Check/install rsync — required by type:static deploys (internal/deploy + // static.go shells out to it directly). Not preinstalled on minimal + // Debian/Ubuntu cloud images, so a fresh box otherwise deploys containers + // fine but fails static deploys on the first rsync with a cryptic + // "command not found" from the remote shell. + fmt.Fprintln(w, "Checking rsync...") + if _, err := exec.Run(ctx, "rsync --version"); err != nil { + fmt.Fprintln(w, " Installing rsync...") + installCmd := env.sudo + "sh -c 'DEBIAN_FRONTEND=noninteractive apt-get update -qq && DEBIAN_FRONTEND=noninteractive apt-get install -y -qq rsync'" + if out, err := exec.Run(ctx, installCmd); err != nil { + fmt.Fprintln(w, out) + return fmt.Errorf("installing rsync: %w", err) + } + fmt.Fprintln(w, " rsync installed") + } else { + fmt.Fprintln(w, " rsync already installed") } - extraMounts = append(extraMounts, m) - } - for _, m := range extraMounts { - src := m.Source - if m.Type == "volume" && m.Name != "" { - src = m.Name + return nil + }, + }, + { + name: "firewall", + affects: "tcp ports 80 and 443 in ufw, when ufw is active", + run: func(ctx context.Context, exec ssh.Executor, w io.Writer, env *setupEnv) error { + // 3. Check firewall + fmt.Fprintln(w, "Checking firewall...") + ufwOutput, ufwErr := exec.Run(ctx, "ufw status 2>/dev/null") + if ufwErr == nil && strings.Contains(ufwOutput, "Status: active") { + _, err1 := exec.Run(ctx, env.sudo+"ufw allow 80/tcp") + _, err2 := exec.Run(ctx, env.sudo+"ufw allow 443/tcp") + if err1 != nil || err2 != nil { + fmt.Fprintln(w, " Warning: could not configure ufw. Ensure ports 80 and 443 are open.") + } else { + fmt.Fprintln(w, " Opened ports 80 and 443 (ufw)") + } + } else if _, err := exec.Run(ctx, "systemctl is-active firewalld 2>/dev/null"); err == nil { + fmt.Fprintln(w, " Warning: firewalld detected. Ensure ports 80 and 443 are open.") + } else { + fmt.Fprintln(w, " No active firewall detected") + } + return nil + }, + }, + { + name: "docker access probe", + affects: "nothing (read-only: docker info — decides whether docker needs sudo)", + run: func(ctx context.Context, exec ssh.Executor, w io.Writer, env *setupEnv) error { + env.dockerCmd = "docker" + if _, err := exec.Run(ctx, "docker info >/dev/null 2>&1"); err != nil { + env.dockerCmd = env.sudo + "docker" } - flag := fmt.Sprintf("%s:%s", src, m.Destination) - if !m.RW { - flag += ":ro" + return nil + }, + }, + { + name: "docker network", + affects: "docker network 'teploy' (created if missing)", + run: func(ctx context.Context, exec ssh.Executor, w io.Writer, env *setupEnv) error { + // 4. Create Docker network + fmt.Fprintln(w, "Creating Docker network...") + netCmd := env.dockerCmd + " network inspect teploy >/dev/null 2>&1 || " + env.dockerCmd + " network create teploy" + if _, err := exec.Run(ctx, netCmd); err != nil { + return fmt.Errorf("creating docker network: %w", err) + } + return nil + }, + }, + { + name: "directories + Caddyfile", + affects: "/deployments and /deployments/caddy (ownership of these two control-plane dirs only), a stub Caddyfile only if none exists", + run: func(ctx context.Context, exec ssh.Executor, w io.Writer, env *setupEnv) error { + // 5. Create directories and upload Caddyfile + if _, err := exec.Run(ctx, env.sudo+"mkdir -p /deployments/caddy"); err != nil { + return fmt.Errorf("creating directories: %w", err) + } + // Ensure the deploy user owns the CONTROL-PLANE directories only — + // never the whole /deployments tree. The old `chown -R + // $(whoami):$(whoami) /deployments` reassigned every application's + // bind-mounted data (database files owned by engine UIDs, accessory + // state) to the interactive SSH user on every re-run, breaking engines + // that rely on their own numeric ownership (TCL-26). Existing app data + // ownership is an invariant, not setup's cleanup target: app + // directories created later are owned by this user anyway, and the + // error is propagated instead of ignored. + if _, err := exec.Run(ctx, env.sudo+"chown $(whoami):$(whoami) /deployments /deployments/caddy"); err != nil { + return fmt.Errorf("setting control-plane directory ownership: %w", err) } - extraMountFlags = append(extraMountFlags, flag) - } - reason := "running with --resume (legacy admin-API mode)" - switch { - case !legacyResume && legacyFileMount: - reason = "using a single-file Caddyfile mount (atomic config writes don't reach the container)" - case !legacyResume && !legacyFileMount && missingStaticMount: - reason = "missing the /deployments mount type:static deploys require to actually be served" - } - fmt.Fprintf(w, " Caddy is %s — migrating to the directory-mounted, Caddyfile-authoritative model.\n", reason) - fmt.Fprintln(w, " This recreates the container (brief outage).") - if len(extraNetworks) > 0 { - fmt.Fprintf(w, " Additional networks to reattach: %s\n", strings.Join(extraNetworks, ", ")) - } - if len(extraMountFlags) > 0 { - fmt.Fprintf(w, " Additional mounts to preserve: %s\n", strings.Join(extraMountFlags, ", ")) - } - if !yes && !confirm(w, " Recreate Caddy container now?") { - fmt.Fprintln(w, " Skipping Caddy upgrade — re-run with --yes to apply.") + // Caddy admin API listens on 0.0.0.0 inside container so Docker port + // forwarding can reach it. Port 2019 is only published to 127.0.0.1 + // on the host — never publicly accessible. + // Tab-indented to match `caddy fmt` output so Caddy doesn't warn. + // Only write the stub Caddyfile when none exists — on servers that + // were provisioned by other tooling (e.g., Dokploy) or hand-edited, + // the existing Caddyfile holds live production routes and must be + // preserved. + const stubCaddyfile = "{\n\tadmin 127.0.0.1:2019\n}\n" + // Only write the stub Caddyfile when the file is confirmed ABSENT — + // `test -s` also fails for an unreadable or empty-but-present file, and + // overwriting either with a stub discards live routes (TCL-27 + // containment). An existing empty file is left for the operator to + // inspect rather than silently clobbered. + present, err := exec.Run(ctx, "[ -f /deployments/caddy/Caddyfile ] && echo present || echo absent") + if err != nil { + return fmt.Errorf("checking for an existing Caddyfile: %w", err) + } + if strings.TrimSpace(present) == "absent" { + if err := exec.Upload(ctx, strings.NewReader(stubCaddyfile), "/deployments/caddy/Caddyfile", "0644"); err != nil { + return fmt.Errorf("uploading Caddyfile: %w", err) + } + } else { + fmt.Fprintln(w, " Existing Caddyfile preserved") + // Lock the admin API to the container loopback. Older setups bound it to + // 0.0.0.0:2019, reachable by any container on the teploy network. + exec.Run(ctx, "sed -i 's/admin 0.0.0.0:2019/admin 127.0.0.1:2019/' /deployments/caddy/Caddyfile") + } return nil - } - if _, err := exec.Run(ctx, dockerCmd+" rm -f caddy"); err != nil { - return fmt.Errorf("removing old caddy: %w", err) - } - caddyCheck = "" - } + }, + }, + { + name: "caddy container", + affects: "caddy container (started if absent/stopped; a legacy-config container is recreated after confirmation — brief outage)", + run: func(ctx context.Context, exec ssh.Executor, w io.Writer, env *setupEnv) error { + // 6. Start Caddy (idempotent — skip if container already exists). + // The on-disk Caddyfile is Teploy's single source of truth: Caddy loads it + // on every boot and `caddy reload`, so we run WITHOUT `--resume` (which + // would boot from admin-API autosave and shadow the file). The admin API + // binds the container loopback only and is never exposed off-box. + fmt.Fprintln(w, "Starting Caddy...") + caddyCheck, err := exec.Run(ctx, env.dockerCmd+" ps -a --filter name=^caddy$ --format '{{.Names}}'") + if err != nil { + return fmt.Errorf("checking for an existing caddy container: %w", err) + } + extraNetworks := []string{} + var extraMountFlags []string + if strings.TrimSpace(caddyCheck) != "" { + // Three legacy conditions require recreating the Caddy container. All + // three recreations are destructive (brief outage + re-attaching + // non-teploy networks/mounts), so we require explicit confirmation. + // Inventory reads must SUCCEED before any recreate decision: a failed + // inspect used to read as an empty set, which both triggered + // unnecessary migrations and silently dropped adopted networks/mounts + // during a real one (audit F67). + cmdOut, cmdErr := exec.Run(ctx, env.dockerCmd+" inspect -f '{{join .Config.Cmd \" \"}}' caddy") + if cmdErr != nil { + return fmt.Errorf("cannot safely inventory the existing caddy container (cmd): %w", cmdErr) + } + mountOut, mountErr := exec.Run(ctx, env.dockerCmd+" inspect -f '{{range .Mounts}}{{.Destination}} {{end}}' caddy") + if mountErr != nil { + return fmt.Errorf("cannot safely inventory the existing caddy container (mounts): %w", mountErr) + } + + // (1) --resume boots from admin-API autosave, shadowing the Caddyfile. + legacyResume := strings.Contains(cmdOut, "--resume") + // (2) A single-file Caddyfile bind mount (Destination + // /etc/caddy/Caddyfile) is pinned by Docker to the file's original + // inode. Teploy writes the Caddyfile atomically (write tmp + mv), + // which swaps the inode — so the running container never sees route + // updates and `caddy reload` reloads stale config. The directory + // mount (/etc/caddy) re-resolves the file by path on each reload and + // avoids this. Detect the legacy file mount and migrate. + legacyFileMount := strings.Contains(mountOut, "/etc/caddy/Caddyfile") + // (3) No /deployments bind mount at all means type:static deploys + // are completely non-functional on this server — Deploy writes + // releases to /deployments//current on the host, and Caddy's + // generated site block reads from {DefaultStaticMount}//current + // (also /deployments now — see internal/deploy/static.go), but + // without this mount that path doesn't exist inside the container + // at all. Confirmed live: `teploy deploy` reports success (the + // release really does land on disk) while every request 404s. + // Every server provisioned before this fix landed is missing this + // mount unconditionally — not a legacy-config edge case, a gap in + // what teploy setup has always provisioned. + missingStaticMount := !strings.Contains(mountOut, "/deployments ") + + if legacyResume || legacyFileMount || missingStaticMount { + // Capture every network the existing Caddy is attached to so + // we can reattach them after recreating. Previously these were + // silently dropped, leaving apps on other networks (e.g. + // dokploy-network) unreachable. + netOut, netErr := exec.Run(ctx, env.dockerCmd+" inspect -f '{{range $k,$v := .NetworkSettings.Networks}}{{$k}} {{end}}' caddy") + if netErr != nil { + return fmt.Errorf("cannot safely inventory the existing caddy container (networks): %w", netErr) + } + for _, n := range strings.Fields(netOut) { + if n != "" && n != "teploy" { + extraNetworks = append(extraNetworks, n) + } + } + + // Capture every mount teploy itself didn't put there, so those + // are preserved too — the same principle as extraNetworks + // above, just for volumes. Without this, recreating Caddy + // blindly replaces its whole mount set with teploy's own fixed + // list (caddy_data, caddy_config, /etc/caddy, /deployments), + // silently dropping anything hand-added on a server that + // predates teploy or was adopted from other tooling — e.g. a + // legacy /srv/static bind mount serving live static sites, + // which would 404 the instant the container came back up. + // Confirmed live on a real production box before this fix: + // exactly that mount existed and would have been lost. + // This is what actually makes it safe to run `teploy setup` + // against an existing, previously-hand-managed server — not + // just a fresh one. + var extraMounts []dockerMount + // The detailed mount inventory must SUCCEED before the + // destructive recreate below: a failed inspect or unparseable + // JSON used to read as "no extra mounts", silently dropping + // adopted volumes during the migration (TCL-27). + mountJSON, mountJSONErr := exec.Run(ctx, env.dockerCmd+" inspect -f '{{json .Mounts}}' caddy") + if mountJSONErr != nil { + return fmt.Errorf("cannot safely inventory the existing caddy container (mount detail): %w", mountJSONErr) + } + teployDestinations := map[string]bool{ + "/data": true, "/config": true, "/etc/caddy": true, "/deployments": true, + } + var mounts []dockerMount + if err := json.Unmarshal([]byte(mountJSON), &mounts); err != nil { + return fmt.Errorf("decoding the existing caddy container's mount inventory: %w", err) + } + for _, m := range mounts { + if m.Type == "volume" && (m.Name == "caddy_data" || m.Name == "caddy_config") { + continue // teploy's own named volumes, re-added explicitly below + } + if teployDestinations[m.Destination] { + continue // teploy's own bind mounts, re-added explicitly below + } + extraMounts = append(extraMounts, m) + } + for _, m := range extraMounts { + src := m.Source + if m.Type == "volume" && m.Name != "" { + src = m.Name + } + flag := fmt.Sprintf("%s:%s", src, m.Destination) + if !m.RW { + flag += ":ro" + } + extraMountFlags = append(extraMountFlags, flag) + } + + reason := "running with --resume (legacy admin-API mode)" + switch { + case !legacyResume && legacyFileMount: + reason = "using a single-file Caddyfile mount (atomic config writes don't reach the container)" + case !legacyResume && !legacyFileMount && missingStaticMount: + reason = "missing the /deployments mount type:static deploys require to actually be served" + } + fmt.Fprintf(w, " Caddy is %s — migrating to the directory-mounted, Caddyfile-authoritative model.\n", reason) + fmt.Fprintln(w, " This recreates the container (brief outage).") + if len(extraNetworks) > 0 { + fmt.Fprintf(w, " Additional networks to reattach: %s\n", strings.Join(extraNetworks, ", ")) + } + if len(extraMountFlags) > 0 { + fmt.Fprintf(w, " Additional mounts to preserve: %s\n", strings.Join(extraMountFlags, ", ")) + } + if !env.yes && !confirm(w, " Recreate Caddy container now?") { + fmt.Fprintln(w, " Skipping Caddy upgrade — re-run with --yes to apply.") + return nil + } + if _, err := exec.Run(ctx, env.dockerCmd+" rm -f caddy"); err != nil { + return fmt.Errorf("removing old caddy: %w", err) + } + caddyCheck = "" + } + } + if strings.TrimSpace(caddyCheck) == "" { + caddyRunArgs := []string{ + env.dockerCmd, "run", "-d", + "--restart", "always", + "--name", "caddy", + "--network", "teploy", + // Lets Caddy (in its own network namespace on the "teploy" + // bridge) reach services bound only to the host's loopback — + // specifically `teploy autodeploy serve`, a systemd-resident + // host process (not a container, since it needs direct Docker + // CLI access to run deploys) listening on 0.0.0.0:9876. Without + // this, host.docker.internal doesn't resolve inside the + // container on native Linux Docker (only Docker Desktop adds it + // automatically) and the webhook route can never connect — + // found live: SetupCaddyRoute's dial target was unreachable + // from inside the container regardless of what host/IP it named. + "--add-host", "host.docker.internal:host-gateway", + "-p", "80:80", + "-p", "443:443", + "-v", "caddy_data:/data", + "-v", "caddy_config:/config", + // Mount the directory, NOT the single Caddyfile. A single-file + // bind mount pins Docker to the file's inode, so teploy's atomic + // (tmp + mv) Caddyfile writes never reach the container. Mounting + // the directory lets `caddy reload` re-read the current file by + // path. It also exposes /deployments/caddy/tls/* as /etc/caddy/tls + // for apps that terminate TLS with a custom cert (e.g. a + // Cloudflare Origin Certificate). + "-v", "/deployments/caddy:/etc/caddy", + // Lets Caddy serve type:static apps. Deploy (internal/deploy/ + // static.go) writes releases to /deployments//current on + // the server's own filesystem and points each site block's + // root at the identical path — /deployments//current — + // inside the Caddy container. Read-only: Caddy the file + // server never needs to write here, and this also mounts + // every other app's /deployments// tree (state, .env, + // secrets) read-only into Caddy's filesystem — :ro caps what + // a compromised Caddy process could do with that visibility + // to read-only, even though none of it is web-exposed (each + // site block's root stays scoped to that one app's own + // current symlink). + "-v", "/deployments:/deployments:ro", + } + // Re-add any mount that was on the previous container but isn't + // one of teploy's own (see extraMountFlags above) — e.g. a legacy + // /srv/static bind mount predating this fix, or anything else + // adopted from other tooling. Without this, whatever those mounts + // served would 404 the instant the recreated container came up. + for _, m := range extraMountFlags { + caddyRunArgs = append(caddyRunArgs, "-v", ssh.ShellQuote(m)) + } + caddyRunArgs = append(caddyRunArgs, + "caddy", + "caddy", "run", "--config", "/etc/caddy/Caddyfile", "--adapter", "caddyfile", + ) + caddyRun := strings.Join(caddyRunArgs, " ") + if _, err := exec.Run(ctx, caddyRun); err != nil { + return fmt.Errorf("starting caddy: %w", err) + } + for _, n := range extraNetworks { + if _, err := exec.Run(ctx, fmt.Sprintf("%s network connect %s caddy", env.dockerCmd, n)); err != nil { + fmt.Fprintf(w, " Warning: failed to reattach %s: %v\n", n, err) + } else { + fmt.Fprintf(w, " Reattached network %s\n", n) + } + } + fmt.Fprintln(w, " Caddy started") + } else { + // Presence in `docker ps -a` proves nothing about health — an exited + // caddy also matches. Report what actually is, and start a stopped + // one instead of declaring success by name (audit F67). + running, rErr := exec.Run(ctx, env.dockerCmd+" inspect -f '{{.State.Running}}' caddy") + if rErr != nil { + return fmt.Errorf("checking the existing caddy container's state: %w", rErr) + } + switch strings.TrimSpace(running) { + case "true": + fmt.Fprintln(w, " Caddy already running") + case "false": + fmt.Fprintln(w, " Caddy container exists but is stopped — starting it") + if _, sErr := exec.Run(ctx, env.dockerCmd+" start caddy"); sErr != nil { + return fmt.Errorf("starting the stopped caddy container: %w", sErr) + } + fmt.Fprintln(w, " Caddy started") + default: + return fmt.Errorf("could not determine the caddy container's state (inspect said %q)", strings.TrimSpace(running)) + } + } + return nil + }, + }, } - if strings.TrimSpace(caddyCheck) == "" { - caddyRunArgs := []string{ - dockerCmd, "run", "-d", - "--restart", "always", - "--name", "caddy", - "--network", "teploy", - // Lets Caddy (in its own network namespace on the "teploy" - // bridge) reach services bound only to the host's loopback — - // specifically `teploy autodeploy serve`, a systemd-resident - // host process (not a container, since it needs direct Docker - // CLI access to run deploys) listening on 0.0.0.0:9876. Without - // this, host.docker.internal doesn't resolve inside the - // container on native Linux Docker (only Docker Desktop adds it - // automatically) and the webhook route can never connect — - // found live: SetupCaddyRoute's dial target was unreachable - // from inside the container regardless of what host/IP it named. - "--add-host", "host.docker.internal:host-gateway", - "-p", "80:80", - "-p", "443:443", - "-v", "caddy_data:/data", - "-v", "caddy_config:/config", - // Mount the directory, NOT the single Caddyfile. A single-file - // bind mount pins Docker to the file's inode, so teploy's atomic - // (tmp + mv) Caddyfile writes never reach the container. Mounting - // the directory lets `caddy reload` re-read the current file by - // path. It also exposes /deployments/caddy/tls/* as /etc/caddy/tls - // for apps that terminate TLS with a custom cert (e.g. a - // Cloudflare Origin Certificate). - "-v", "/deployments/caddy:/etc/caddy", - // Lets Caddy serve type:static apps. Deploy (internal/deploy/ - // static.go) writes releases to /deployments//current on - // the server's own filesystem and points each site block's - // root at the identical path — /deployments//current — - // inside the Caddy container. Read-only: Caddy the file - // server never needs to write here, and this also mounts - // every other app's /deployments// tree (state, .env, - // secrets) read-only into Caddy's filesystem — :ro caps what - // a compromised Caddy process could do with that visibility - // to read-only, even though none of it is web-exposed (each - // site block's root stays scoped to that one app's own - // current symlink). - "-v", "/deployments:/deployments:ro", - } - // Re-add any mount that was on the previous container but isn't - // one of teploy's own (see extraMountFlags above) — e.g. a legacy - // /srv/static bind mount predating this fix, or anything else - // adopted from other tooling. Without this, whatever those mounts - // served would 404 the instant the recreated container came up. - for _, m := range extraMountFlags { - caddyRunArgs = append(caddyRunArgs, "-v", ssh.ShellQuote(m)) - } - caddyRunArgs = append(caddyRunArgs, - "caddy", - "caddy", "run", "--config", "/etc/caddy/Caddyfile", "--adapter", "caddyfile", - ) - caddyRun := strings.Join(caddyRunArgs, " ") - if _, err := exec.Run(ctx, caddyRun); err != nil { - return fmt.Errorf("starting caddy: %w", err) - } - for _, n := range extraNetworks { - if _, err := exec.Run(ctx, fmt.Sprintf("%s network connect %s caddy", dockerCmd, n)); err != nil { - fmt.Fprintf(w, " Warning: failed to reattach %s: %v\n", n, err) - } else { - fmt.Fprintf(w, " Reattached network %s\n", n) - } - } - fmt.Fprintln(w, " Caddy started") - } else { - // Presence in `docker ps -a` proves nothing about health — an exited - // caddy also matches. Report what actually is, and start a stopped - // one instead of declaring success by name (audit F67). - running, rErr := exec.Run(ctx, dockerCmd+" inspect -f '{{.State.Running}}' caddy") - if rErr != nil { - return fmt.Errorf("checking the existing caddy container's state: %w", rErr) - } - switch strings.TrimSpace(running) { - case "true": - fmt.Fprintln(w, " Caddy already running") - case "false": - fmt.Fprintln(w, " Caddy container exists but is stopped — starting it") - if _, sErr := exec.Run(ctx, dockerCmd+" start caddy"); sErr != nil { - return fmt.Errorf("starting the stopped caddy container: %w", sErr) - } - fmt.Fprintln(w, " Caddy started") - default: - return fmt.Errorf("could not determine the caddy container's state (inspect said %q)", strings.TrimSpace(running)) +} + +// setupServer runs the provisioning stages on a connected server. +// Separated from runSetup for testability with MockExecutor. +// yes skips interactive confirmation for destructive upgrade steps +// (Caddy recreate). Every stage is idempotent, so an interrupted run +// (connection loss, Ctrl-C) is safely resumable by re-running setup +// (C08); the preflight states what will be touched before anything is. +func setupServer(ctx context.Context, exec ssh.Executor, w io.Writer, yes bool) error { + env := &setupEnv{yes: yes} + stages := setupStages() + + fmt.Fprintln(w, "Preflight — this run will ensure (items already in place are skipped):") + for _, s := range stages { + fmt.Fprintf(w, " - %s: %s\n", s.name, s.affects) + } + + for _, s := range stages { + if err := s.run(ctx, exec, w, env); err != nil { + return fmt.Errorf("setup stage %q: %w", s.name, err) } } diff --git a/internal/cli/setup_test.go b/internal/cli/setup_test.go index 4f00d6d..82ec947 100644 --- a/internal/cli/setup_test.go +++ b/internal/cli/setup_test.go @@ -3,10 +3,12 @@ package cli import ( "bytes" "context" + "errors" "fmt" "strings" "testing" + "github.com/useteploy/teploy/internal/network" "github.com/useteploy/teploy/internal/ssh" ) @@ -470,3 +472,355 @@ func TestSetupServer_UFWActive(t *testing.T) { t.Error("should report ports opened") } } + +// TestInstallSudoViaSu_SecretTransport pins the C08 transport for the +// root password: it travels the session stdin (recorded in Inputs), +// never a command string, and no /tmp script artifact is uploaded. +func TestInstallSudoViaSu_SecretTransport(t *testing.T) { + mock := ssh.NewMockExecutor("h", + ssh.MockCommand{Match: "su -c", Output: "TEPLOY_SUDO_OK\n"}, + ) + err := installSudoViaSu(context.Background(), mock, "tyler", "root-pw-123") + if err != nil { + t.Fatalf("installSudoViaSu: %v", err) + } + for _, call := range mock.Calls { + if strings.Contains(call, "root-pw-123") { + t.Errorf("root password in command argv: %s", call) + } + if strings.HasPrefix(call, "UPLOAD:") && strings.Contains(call, "/tmp/") { + t.Errorf("the su path must not stage a script artifact: %s", call) + } + } + if len(mock.Inputs) != 1 || mock.Inputs[0] != "root-pw-123\n" { + t.Fatalf("root password must ride stdin as one line, inputs: %v", mock.Inputs) + } + if call := mock.Calls[0]; !strings.HasPrefix(call, "su -c ") || !strings.Contains(call, "TEPLOY_SUDO_OK") { + t.Fatalf("unexpected su command shape: %s", call) + } +} + +// TestInstallSudoViaSu_WrongPassword keeps the actionable failure. +func TestInstallSudoViaSu_WrongPassword(t *testing.T) { + mock := ssh.NewMockExecutor("h", + ssh.MockCommand{Match: "su -c", Err: errors.New("exit status 1: su: Authentication failure")}, + ) + err := installSudoViaSu(context.Background(), mock, "tyler", "nope") + if err == nil || !strings.Contains(err.Error(), "wrong root password") { + t.Fatalf("wrong password = %v, want wrong-root-password error", err) + } +} + +// TestVPNJoinCommand_NoCredentialInArgv pins the join shape: the key +// file is read, removed, and fed via the provider env var — the literal +// credential appears nowhere in the command. +func TestVPNJoinCommand_NoCredentialInArgv(t *testing.T) { + cfg := network.Config{Provider: "tailscale", AuthKey: "tskey-auth-secret123"} + cmd := vpnJoinCommand("tailscale", cfg, "sudo ", "/tmp/teploy-vpn-key-abc") + if strings.Contains(cmd, "tskey-auth-secret123") { + t.Fatalf("auth key leaked into the join command: %s", cmd) + } + for _, want := range []string{ + "nohup sh -c ", + "k=$(cat /tmp/teploy-vpn-key-abc)", + "rm -f -- /tmp/teploy-vpn-key-abc", + `export TS_AUTHKEY="$k"`, + "exec tailscale up --accept-routes", + } { + if !strings.Contains(cmd, want) { + t.Errorf("join command missing %q:\n%s", want, cmd) + } + } + + headscale := network.Config{Provider: "headscale", AuthKey: "tskey-secret", Server: "https://hs.example.com"} + cmd = vpnJoinCommand("headscale", headscale, "", "/tmp/k") + if strings.Contains(cmd, "tskey-secret") { + t.Fatalf("headscale auth key leaked: %s", cmd) + } + if !strings.Contains(cmd, "--login-server=") || !strings.Contains(cmd, "https://hs.example.com") { + t.Errorf("headscale login server missing: %s", cmd) + } + + nb := network.Config{Provider: "netbird", SetupKey: "nb-secret"} + cmd = vpnJoinCommand("netbird", nb, "", "/tmp/k") + if strings.Contains(cmd, "nb-secret") { + t.Fatalf("netbird setup key leaked: %s", cmd) + } + if !strings.Contains(cmd, `export NB_SETUP_KEY="$k"`) || !strings.Contains(cmd, "exec netbird up") { + t.Errorf("netbird join shape wrong: %s", cmd) + } +} + +// TestJoinVPNMesh_StagesPrivateFileAndCleansUp pins the staging +// mechanics: the credential is uploaded 0600 exactly once, and the join +// command references that path. +func TestJoinVPNMesh_StagesPrivateFileAndCleansUp(t *testing.T) { + mock := ssh.NewMockExecutor("h", + ssh.MockCommand{Match: "nohup sh -c", Output: ""}, + ) + cfg := network.Config{Provider: "tailscale", AuthKey: "tskey-auth-x"} + if err := joinVPNMesh(context.Background(), mock, "", "tailscale", cfg); err != nil { + t.Fatalf("joinVPNMesh: %v", err) + } + uploads := 0 + for _, call := range mock.Calls { + if strings.HasPrefix(call, "UPLOAD:") { + uploads++ + if !strings.Contains(call, "mode 0600") || !strings.Contains(call, "/tmp/teploy-vpn-key-") { + t.Errorf("credential must upload 0600 to the key path: %s", call) + } + } + if strings.Contains(call, "tskey-auth-x") { + t.Errorf("credential in command argv: %s", call) + } + } + if uploads != 1 { + t.Fatalf("expected exactly one credential upload, got %d (calls: %v)", uploads, mock.Calls) + } +} + +// TestRegistryLoginOnServer_PasswordNeverInArgv pins the registry login +// transport: the password rides stdin (docker --password-stdin), and no +// command string — including the printf pipe form — carries it. +func TestRegistryLoginOnServer_PasswordNeverInArgv(t *testing.T) { + mock := ssh.NewMockExecutor("h", + ssh.MockCommand{Match: "docker login", Output: "Login Succeeded"}, + ) + if err := registryLoginOnServer(context.Background(), mock, "ghcr.io", "user", "pw-hunter2"); err != nil { + t.Fatalf("registryLoginOnServer: %v", err) + } + if len(mock.Calls) != 1 { + t.Fatalf("expected one command, got %v", mock.Calls) + } + call := mock.Calls[0] + if !strings.HasPrefix(call, "docker login 'ghcr.io' -u 'user' --password-stdin") || strings.Contains(call, "printf") { + t.Errorf("login command shape wrong: %s", call) + } + if strings.Contains(call, "pw-hunter2") { + t.Errorf("password in command argv: %s", call) + } + if len(mock.Inputs) != 1 || mock.Inputs[0] != "pw-hunter2" { + t.Fatalf("password must ride stdin verbatim, inputs: %v", mock.Inputs) + } +} + +// TestRegistryLoginOnServer_FailureSurfacesDiagnostics pins that a +// refused login names docker's own stderr (RunInput's discard-both +// contract used to hide it). +func TestRegistryLoginOnServer_FailureSurfacesDiagnostics(t *testing.T) { + mock := ssh.NewMockExecutor("h", + ssh.MockCommand{Match: "docker login", Err: errors.New("exit status 1: Error response from daemon: unauthorized")}, + ) + err := registryLoginOnServer(context.Background(), mock, "ghcr.io", "user", "bad") + if err == nil || !strings.Contains(err.Error(), "unauthorized") { + t.Fatalf("refused login = %v, want docker's stderr surfaced", err) + } +} + +// TestSetupServer_PreflightListsStagesAndAffectedResources pins the +// C08 preflight: before touching anything, setup states every stage +// and what it affects — the affected-resource list the operator reads +// before confirming a run against an existing box. +func TestSetupServer_PreflightListsStagesAndAffectedResources(t *testing.T) { + stages := setupStages() + if len(stages) < 5 { + t.Fatalf("expected the provisioning plan to have stages, got %d", len(stages)) + } + seen := map[string]bool{} + for _, s := range stages { + if s.name == "" || s.affects == "" { + t.Fatalf("every stage needs a name and an affected-resource description: %+v", s) + } + if seen[s.name] { + t.Fatalf("stage names must be unique for stage-named errors: %q", s.name) + } + seen[s.name] = true + } + + mock := ssh.NewMockExecutor("1.2.3.4", + ssh.MockCommand{Match: "whoami", Output: "root"}, + ssh.MockCommand{Match: "docker --version", Output: "Docker version 24.0.0"}, + ssh.MockCommand{Match: "rsync --version", Output: "rsync version 3.2.7"}, + ssh.MockCommand{Match: "ufw status", Err: fmt.Errorf("command not found")}, + ssh.MockCommand{Match: "systemctl is-active firewalld", Err: fmt.Errorf("inactive")}, + ssh.MockCommand{Match: "docker info", Output: ""}, + ssh.MockCommand{Match: "docker network", Output: "teploy"}, + ssh.MockCommand{Match: "mkdir", Output: ""}, + ssh.MockCommand{Match: "chown", Output: ""}, + ssh.MockCommand{Match: "[ -f /deployments/caddy/Caddyfile ]", Output: "absent"}, + ssh.MockCommand{Match: "docker ps -a --filter name=", Output: ""}, + ssh.MockCommand{Match: "docker run", Output: "caddy_id"}, + ) + var buf bytes.Buffer + if err := setupServer(context.Background(), mock, &buf, true); err != nil { + t.Fatalf("setupServer: %v", err) + } + out := buf.String() + if !strings.Contains(out, "Preflight") { + t.Error("the run must open with the preflight list") + } + for _, s := range stages { + if !strings.Contains(out, s.name+": "+s.affects) { + t.Errorf("preflight must list stage %q with its affected resources", s.name) + } + } +} + +// TestSetupServer_InterruptedRunIsResumable pins C08's interrupted-setup +// acceptance: a connection death mid-setup fails with the stage named, +// and re-running setup against the SAME partially-provisioned server +// completes by skipping what already happened — no package reinstalls, +// no second Caddyfile write, no second container start. +func TestSetupServer_InterruptedRunIsResumable(t *testing.T) { + // Phase 1: the connection dies at the docker-network stage. Every + // earlier stage's work (docker present, dirs created) has already + // happened on the server. + phase1 := ssh.NewMockExecutor("1.2.3.4", + ssh.MockCommand{Match: "whoami", Output: "root"}, + ssh.MockCommand{Match: "docker --version", Output: "Docker version 24.0.0"}, + ssh.MockCommand{Match: "rsync --version", Output: "rsync version 3.2.7"}, + ssh.MockCommand{Match: "ufw status", Err: fmt.Errorf("command not found")}, + ssh.MockCommand{Match: "systemctl is-active firewalld", Err: fmt.Errorf("inactive")}, + ssh.MockCommand{Match: "docker info", Output: ""}, + ssh.MockCommand{Match: "docker network inspect teploy", Err: errors.New("ssh: connection reset by peer")}, + ) + var out1 bytes.Buffer + err := setupServer(context.Background(), phase1, &out1, true) + if err == nil { + t.Fatal("a mid-setup connection death must fail the run") + } + if !strings.Contains(err.Error(), `setup stage "docker network"`) { + t.Fatalf("the failure must name the stage that died, got: %v", err) + } + if !strings.Contains(err.Error(), "connection reset") { + t.Fatalf("the failure must carry the transport error, got: %v", err) + } + + // Phase 2: the operator re-runs setup. The server now has docker, + // rsync, the teploy network, the directories, the Caddyfile, and a + // running modern caddy — everything the interrupted run got to. + phase2 := ssh.NewMockExecutor("1.2.3.4", + ssh.MockCommand{Match: "whoami", Output: "root"}, + ssh.MockCommand{Match: "docker --version", Output: "Docker version 24.0.0"}, + ssh.MockCommand{Match: "rsync --version", Output: "rsync version 3.2.7"}, + ssh.MockCommand{Match: "ufw status", Err: fmt.Errorf("command not found")}, + ssh.MockCommand{Match: "systemctl is-active firewalld", Err: fmt.Errorf("inactive")}, + ssh.MockCommand{Match: "docker info", Output: ""}, + ssh.MockCommand{Match: "docker network inspect teploy", Output: "teploy"}, + ssh.MockCommand{Match: "mkdir", Output: ""}, + ssh.MockCommand{Match: "chown", Output: ""}, + ssh.MockCommand{Match: "[ -f /deployments/caddy/Caddyfile ]", Output: "present"}, + ssh.MockCommand{Match: "sed -i", Output: ""}, + ssh.MockCommand{Match: "docker ps -a --filter name=", Output: "caddy"}, + ssh.MockCommand{Match: "docker inspect -f '{{join .Config.Cmd", Output: "caddy run --config /etc/caddy/Caddyfile --adapter caddyfile"}, + ssh.MockCommand{Match: "docker inspect -f '{{range .Mounts}}", Output: "/data /config /etc/caddy /deployments "}, + ssh.MockCommand{Match: "docker inspect -f '{{.State.Running}}'", Output: "true"}, + ) + var out2 bytes.Buffer + if err := setupServer(context.Background(), phase2, &out2, true); err != nil { + t.Fatalf("the re-run must complete: %v", err) + } + for _, want := range []string{ + "Docker already installed", + "rsync already installed", + "Existing Caddyfile preserved", + "Caddy already running", + "Server provisioned successfully", + } { + if !strings.Contains(out2.String(), want) { + t.Errorf("re-run output missing %q:\n%s", want, out2.String()) + } + } + // The re-run must not redo completed work: no installer script, no + // package install, no Caddyfile write, no container start. (The + // network stage's compound `inspect || create` text always names + // create; the registration pins that INSPECT succeeded, so the + // create side never fired.) + for _, call := range phase2.Calls { + for _, forbidden := range []string{"get.docker.com", "apt-get install", "docker run"} { + if strings.Contains(call, forbidden) { + t.Errorf("the resumed run must not repeat %q: %s", forbidden, call) + } + } + } + if _, uploaded := phase2.Files["/deployments/caddy/Caddyfile"]; uploaded { + t.Error("the resumed run must not rewrite an existing Caddyfile") + } +} + +// TestSetupServer_ConnectionLossRecoversMidStage pins the connection +// recovery wiring end-to-end at the flow level: setupServer running +// through a ReconnectingExecutor survives a one-shot transport death +// mid-stage and completes, the dead command retried exactly once. +// Registrations that model ran-and-failed commands carry "exit status +// N" (the mock contract) so the wrapper classifies them as completed — +// only the network-stage registration models a transport death. +func TestSetupServer_ConnectionLossRecoversMidStage(t *testing.T) { + mock := ssh.NewMockExecutor("1.2.3.4", + ssh.MockCommand{Match: "whoami", Output: "root"}, + ssh.MockCommand{Match: "docker --version", Output: "Docker version 24.0.0"}, + ssh.MockCommand{Match: "rsync --version", Output: "rsync version 3.2.7"}, + ssh.MockCommand{Match: "ufw status", Err: fmt.Errorf("exit status 1: ufw: command not found")}, + ssh.MockCommand{Match: "systemctl is-active firewalld", Err: fmt.Errorf("exit status 3: inactive")}, + ssh.MockCommand{Match: "docker info", Output: ""}, + // First attempt at the network stage dies with a transport + // error; the redialed retry finds the network (inspect OK). + ssh.MockCommand{Match: "docker network inspect teploy", Err: errors.New("ssh: connection reset by peer"), Once: true}, + ssh.MockCommand{Match: "docker network inspect teploy", Output: "teploy"}, + ssh.MockCommand{Match: "mkdir", Output: ""}, + ssh.MockCommand{Match: "chown", Output: ""}, + ssh.MockCommand{Match: "[ -f /deployments/caddy/Caddyfile ]", Output: "absent"}, + ssh.MockCommand{Match: "docker ps -a --filter name=", Output: ""}, + ssh.MockCommand{Match: "docker run", Output: "caddy_id"}, + ) + wrapper, err := ssh.NewReconnectingExecutor(context.Background(), func(ctx context.Context) (ssh.Executor, error) { + return mock, nil + }, 1) + if err != nil { + t.Fatal(err) + } + var buf bytes.Buffer + if setupErr := setupServer(context.Background(), wrapper, &buf, true); setupErr != nil { + t.Fatalf("a one-shot connection loss mid-stage must recover, got: %v", setupErr) + } + if !strings.Contains(buf.String(), "Server provisioned successfully") { + t.Errorf("recovery must complete the flow:\n%s", buf.String()) + } + attempts := 0 + for _, call := range mock.Calls { + if strings.HasPrefix(call, "docker network inspect teploy") { + attempts++ + } + } + if attempts != 2 { + t.Fatalf("the dead network command must be retried exactly once, attempts = %d", attempts) + } +} + +// TestInstallAuthorizedKey_GuardedAgainstDuplicateAppend pins the +// resumability guard for the password-path key install: the append is +// guarded by a membership check, so an interrupted setup re-run cannot +// stack duplicate authorized_keys entries (the old bare `echo >>` +// appended on every attempt). +func TestInstallAuthorizedKey_GuardedAgainstDuplicateAppend(t *testing.T) { + pubKey := "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIExample teploy-test" + mock := ssh.NewMockExecutor("h", + ssh.MockCommand{Match: "mkdir -p ~/.ssh", Output: ""}, + ) + for i := 0; i < 2; i++ { // first install, then the re-run after an interrupted setup + if err := installAuthorizedKey(context.Background(), mock, pubKey); err != nil { + t.Fatalf("installAuthorizedKey attempt %d: %v", i+1, err) + } + } + guarded := "mkdir -p ~/.ssh && grep -qF " + ssh.ShellQuote(pubKey) + + " ~/.ssh/authorized_keys 2>/dev/null || echo " + ssh.ShellQuote(pubKey) + + " >> ~/.ssh/authorized_keys; chmod 700 ~/.ssh && chmod 600 ~/.ssh/authorized_keys" + if len(mock.Calls) != 2 { + t.Fatalf("expected one command per attempt, calls: %v", mock.Calls) + } + for i, call := range mock.Calls { + if call != guarded { + t.Fatalf("attempt %d must use the guarded append:\ngot: %s\nwant: %s", i+1, call, guarded) + } + } +} diff --git a/internal/docker/docker.go b/internal/docker/docker.go index f86b395..6d2ba65 100644 --- a/internal/docker/docker.go +++ b/internal/docker/docker.go @@ -360,6 +360,25 @@ func (c *Client) ExecStream(ctx context.Context, name, command string, stdout, s return c.exec.RunStream(ctx, cmd, stdout, stderr) } +// ExecInput runs a command inside a running container with stdin streamed +// from stdin, capturing stdout — the container-exec analogue of the +// Executor.RunInput secret-transport contract (C08): secret material +// (tokens, values) must ride the stdin pipe, not the docker exec argv, +// where the host's process list would show it. The returned string on +// failure is the command's stderr (the diagnostics callers classify), +// mirroring Exec's error-text contract. +func (c *Client) ExecInput(ctx context.Context, name, command string, stdin io.Reader) (string, error) { + cmd := fmt.Sprintf("docker exec -i %s sh -c %s", ssh.ShellQuote(name), ssh.ShellQuote(command)) + res := ssh.RunInputDetailed(ctx, c.exec, cmd, stdin) + if res.Err != nil { + return res.ExitErrorText(), fmt.Errorf("exec in container %s: %w", name, res.Err) + } + if res.ExitCode != 0 { + return string(res.Stdout), fmt.Errorf("exec in container %s: exit status %d: %s", name, res.ExitCode, res.ExitErrorText()) + } + return string(res.Stdout), nil +} + // RunningContainer returns the name of a running container for the app's given // process (e.g. "web"). For a multi-replica process it returns the first // replica. Used by `app exec` to pick a target to run a one-off command in. diff --git a/internal/network/network.go b/internal/network/network.go index b907737..f875236 100644 --- a/internal/network/network.go +++ b/internal/network/network.go @@ -2,6 +2,8 @@ package network import ( "context" + "crypto/rand" + "encoding/hex" "fmt" "io" "strings" @@ -49,6 +51,35 @@ func NewProvider(cfg Config) (Provider, error) { } } +// StageJoinCredential uploads a mesh join credential (auth key / setup +// key) to a private temp file on the server and returns its generated +// path (C08): the credential must never enter a command string — the +// session shell's argv sits in the server's process list for the life +// of the join. The consuming shell (EnvVarJoinShell) reads the file +// into the provider's env var, removes the file BEFORE exec'ing the +// provider, so the on-disk window is upload-to-join. An upload failure +// aborts before anything joins. +func StageJoinCredential(ctx context.Context, exec ssh.Executor, credential string) (string, error) { + var suffix [8]byte + if _, err := rand.Read(suffix[:]); err != nil { + return "", fmt.Errorf("generating key file name: %w", err) + } + keyPath := "/tmp/teploy-vpn-key-" + hex.EncodeToString(suffix[:]) + if err := exec.Upload(ctx, strings.NewReader(credential), keyPath, "0600"); err != nil { + return "", fmt.Errorf("staging the join credential at %s: %w", keyPath, err) + } + return keyPath, nil +} + +// EnvVarJoinShell renders `sh -c 'read the key file into envVar, remove +// it, exec tail'`, quoted for one outer shell level. keyPath must be a +// path StageJoinCredential generated (never operator-supplied input) — +// it needs no inner quoting. +func EnvVarJoinShell(envVar, keyPath, tail string) string { + inner := fmt.Sprintf(`k=$(cat %s) && rm -f -- %s && export %s="$k" && exec %s`, keyPath, keyPath, envVar, tail) + return "sh -c " + ssh.ShellQuote(inner) +} + // --- Tailscale --- // TailscaleProvider manages Tailscale VPN on servers. @@ -79,7 +110,13 @@ func (t *TailscaleProvider) Join(ctx context.Context, exec ssh.Executor) error { } // Run tailscale up in the background. Tailscale modifies iptables which can kill // the SSH connection that's running the command. Using nohup + background avoids this. - cmd := t.Sudo + "nohup tailscale up --authkey=" + ssh.ShellQuote(t.AuthKey) + " --accept-routes >/dev/null 2>&1 & sleep 3 && " + t.Sudo + "tailscale status --json 2>/dev/null | grep -q Running" + // The auth key rides a private file into TS_AUTHKEY, never the argv (C08). + keyPath, err := StageJoinCredential(ctx, exec, t.AuthKey) + if err != nil { + return err + } + cmd := t.Sudo + "nohup " + EnvVarJoinShell("TS_AUTHKEY", keyPath, "tailscale up --accept-routes") + + " >/dev/null 2>&1 & sleep 3 && " + t.Sudo + "tailscale status --json 2>/dev/null | grep -q Running" if _, err := exec.Run(ctx, cmd); err != nil { // SSH disconnect during tailscale up is expected — the iptables change can kill the connection. // Check if tailscale actually joined by trying status again. @@ -138,7 +175,15 @@ func (h *HeadscaleProvider) Join(ctx context.Context, exec ssh.Executor) error { if err == nil && strings.Contains(out, `"BackendState":"Running"`) { return nil } - cmd := h.Sudo + "nohup tailscale up --login-server=" + ssh.ShellQuote(h.Server) + " --authkey=" + ssh.ShellQuote(h.AuthKey) + " --accept-routes >/dev/null 2>&1 & sleep 3 && " + h.Sudo + "tailscale status --json 2>/dev/null | grep -q Running" + // Same private-file credential transport as TailscaleProvider.Join + // (C08); the login-server URL is not a secret and stays a flag. + keyPath, err := StageJoinCredential(ctx, exec, h.AuthKey) + if err != nil { + return err + } + tail := "tailscale up --login-server=" + ssh.ShellQuote(h.Server) + " --accept-routes" + cmd := h.Sudo + "nohup " + EnvVarJoinShell("TS_AUTHKEY", keyPath, tail) + + " >/dev/null 2>&1 & sleep 3 && " + h.Sudo + "tailscale status --json 2>/dev/null | grep -q Running" if _, err := exec.Run(ctx, cmd); err != nil { out, statusErr := exec.Run(ctx, h.Sudo+"tailscale status --json 2>/dev/null") if statusErr == nil && strings.Contains(out, `"BackendState":"Running"`) { @@ -193,7 +238,13 @@ func (n *NetbirdProvider) Join(ctx context.Context, exec ssh.Executor) error { if err == nil && strings.Contains(out, "Connected") { return nil } - cmd := n.Sudo + "netbird up --setup-key " + ssh.ShellQuote(n.SetupKey) + // Same private-file credential transport as the tailscale paths + // (C08): the setup key rides NB_SETUP_KEY, never the argv. + keyPath, err := StageJoinCredential(ctx, exec, n.SetupKey) + if err != nil { + return err + } + cmd := n.Sudo + EnvVarJoinShell("NB_SETUP_KEY", keyPath, "netbird up") if _, err := exec.Run(ctx, cmd); err != nil { return fmt.Errorf("joining netbird mesh: %w", err) } diff --git a/internal/network/network_test.go b/internal/network/network_test.go index d05dcbb..112d45f 100644 --- a/internal/network/network_test.go +++ b/internal/network/network_test.go @@ -106,7 +106,7 @@ func TestTailscaleJoin_AlreadyConnected(t *testing.T) { func TestTailscaleJoin_Fresh(t *testing.T) { mock := ssh.NewMockExecutor("server1", ssh.MockCommand{Match: "tailscale status --json", Err: fmt.Errorf("not running")}, - ssh.MockCommand{Match: "nohup tailscale up", Output: ""}, + ssh.MockCommand{Match: "nohup sh -c", Output: ""}, ) p := &TailscaleProvider{AuthKey: "tskey-auth-xxx"} @@ -116,18 +116,39 @@ func TestTailscaleJoin_Fresh(t *testing.T) { var foundUp bool for _, call := range mock.Calls { - if strings.Contains(call, "nohup tailscale up") { + if strings.Contains(call, "nohup sh -c") { foundUp = true - if !strings.Contains(call, "--authkey=") || !strings.Contains(call, "tskey-auth-xxx") { - t.Errorf("tailscale up should include authkey, got: %s", call) + if strings.Contains(call, "tskey-auth-xxx") || strings.Contains(call, "--authkey=") { + t.Errorf("the auth key must never ride the argv (C08), got: %s", call) } - if !strings.Contains(call, "--accept-routes") { - t.Errorf("tailscale up should include --accept-routes, got: %s", call) + if !strings.Contains(call, `export TS_AUTHKEY="$k"`) { + t.Errorf("the key must ride TS_AUTHKEY from the staged file, got: %s", call) + } + if !strings.Contains(call, "exec tailscale up --accept-routes") { + t.Errorf("join must exec tailscale up with --accept-routes, got: %s", call) } } } if !foundUp { - t.Error("expected 'nohup tailscale up' call") + t.Error("expected the detached join call") + } + // The credential landed in a 0600 staged file, exactly once. + uploads := 0 + for _, call := range mock.Calls { + if strings.HasPrefix(call, "UPLOAD:") { + uploads++ + if !strings.Contains(call, "mode 0600") || !strings.Contains(call, "/tmp/teploy-vpn-key-") { + t.Errorf("credential must stage 0600 at the key path: %s", call) + } + } + } + if uploads != 1 { + t.Fatalf("expected exactly one credential upload, got %d (calls: %v)", uploads, mock.Calls) + } + for path, content := range mock.Files { + if strings.HasPrefix(path, "/tmp/teploy-vpn-key-") && string(content) != "tskey-auth-xxx" { + t.Errorf("staged credential content = %q", content) + } } } @@ -164,7 +185,7 @@ func TestHeadscaleInstall_AlreadyInstalled(t *testing.T) { func TestHeadscaleJoin_UsesLoginServer(t *testing.T) { mock := ssh.NewMockExecutor("server1", ssh.MockCommand{Match: "tailscale status --json", Err: fmt.Errorf("not running")}, - ssh.MockCommand{Match: "nohup tailscale up", Output: ""}, + ssh.MockCommand{Match: "nohup sh -c", Output: ""}, ) p := &HeadscaleProvider{Server: "https://headscale.example.com", AuthKey: "key123"} @@ -174,18 +195,21 @@ func TestHeadscaleJoin_UsesLoginServer(t *testing.T) { var foundUp bool for _, call := range mock.Calls { - if strings.Contains(call, "nohup tailscale up") { + if strings.Contains(call, "nohup sh -c") { foundUp = true if !strings.Contains(call, "--login-server=") || !strings.Contains(call, "headscale.example.com") { t.Errorf("should use --login-server, got: %s", call) } - if !strings.Contains(call, "--authkey=") || !strings.Contains(call, "key123") { - t.Errorf("should include authkey, got: %s", call) + if strings.Contains(call, "key123") || strings.Contains(call, "--authkey=") { + t.Errorf("the auth key must never ride the argv (C08), got: %s", call) + } + if !strings.Contains(call, `export TS_AUTHKEY="$k"`) { + t.Errorf("the key must ride TS_AUTHKEY from the staged file, got: %s", call) } } } if !foundUp { - t.Error("expected 'nohup tailscale up' call") + t.Error("expected the detached join call") } } @@ -240,7 +264,7 @@ func TestNetbirdJoin_AlreadyConnected(t *testing.T) { func TestNetbirdJoin_Fresh(t *testing.T) { mock := ssh.NewMockExecutor("server1", ssh.MockCommand{Match: "netbird status", Err: fmt.Errorf("not running")}, - ssh.MockCommand{Match: "netbird up", Output: ""}, + ssh.MockCommand{Match: "sh -c", Output: ""}, ) p := &NetbirdProvider{SetupKey: "nb-setup-xxx"} @@ -250,10 +274,13 @@ func TestNetbirdJoin_Fresh(t *testing.T) { var foundUp bool for _, call := range mock.Calls { - if strings.HasPrefix(call, "netbird up") { + if strings.HasPrefix(call, "sh -c ") { foundUp = true - if !strings.Contains(call, "--setup-key") || !strings.Contains(call, "nb-setup-xxx") { - t.Errorf("should include setup key, got: %s", call) + if strings.Contains(call, "nb-setup-xxx") || strings.Contains(call, "--setup-key") { + t.Errorf("the setup key must never ride the argv (C08), got: %s", call) + } + if !strings.Contains(call, `export NB_SETUP_KEY="$k"`) || !strings.Contains(call, "exec netbird up") { + t.Errorf("the key must ride NB_SETUP_KEY into netbird up, got: %s", call) } } } diff --git a/internal/openbao/client.go b/internal/openbao/client.go index cff05d6..d98fc42 100644 --- a/internal/openbao/client.go +++ b/internal/openbao/client.go @@ -252,19 +252,56 @@ func (c *Client) ensureContainer(ctx context.Context, opts SetupOptions, contain return nil } +// bao runs a bao CLI invocation inside the container. The auth token +// rides the session's stdin, never the docker exec argv: the old form +// embedded `BAO_TOKEN=` in the `sh -c` string, which is the +// docker exec process's command line on the HOST — the root token sat +// in the server's process list for every vault operation (C08). The +// inner shell reads one line from stdin into the env, then execs bao. +// On failure the returned string carries the command's stderr (the text +// callers scan for idempotency markers like "already in use"), matching +// the previous error-text contract. func (c *Client) bao(ctx context.Context, container, token, args string) (string, error) { - env := "BAO_ADDR=" + containerAPIAddr + inner := `IFS= read -r teploy_tok; export BAO_ADDR=` + containerAPIAddr if token != "" { - env += " BAO_TOKEN=" + token + inner += ` BAO_TOKEN="$teploy_tok"` } - // docker.Exec wraps `docker exec sh -c `; we build the inner cmd. - out, err := c.docker.Exec(ctx, container, env+" bao "+args) + inner += `; exec bao ` + args + stdin := "\n" + if token != "" { + stdin = token + "\n" + } + out, err := c.docker.ExecInput(ctx, container, inner, strings.NewReader(stdin)) if err != nil { - // The SSH executor discards stdout on a non-zero exit and folds bao's - // stderr into the error. Return that text as the "output" so callers can - // inspect one string for messages / idempotency markers ("already in - // use"). On success, out is the real stdout (e.g. JSON). - return err.Error(), err + if out == "" { + out = err.Error() + } + return out, err + } + return out, nil +} + +// baoInput runs a bao invocation whose own stdin payload follows the +// token line on the same pipe: line 1 is the auth token (consumed by +// the read in bao's inner shell), the remainder is payload — the JSON +// object `bao kv put -` reads. Both the token and the secret +// values stay off every argv (C08). +func (c *Client) baoInput(ctx context.Context, container, token, args, payload string) (string, error) { + inner := `IFS= read -r teploy_tok; export BAO_ADDR=` + containerAPIAddr + if token != "" { + inner += ` BAO_TOKEN="$teploy_tok"` + } + inner += `; exec bao ` + args + stdin := "\n" + payload + if token != "" { + stdin = token + "\n" + payload + } + out, err := c.docker.ExecInput(ctx, container, inner, strings.NewReader(stdin)) + if err != nil { + if out == "" { + out = err.Error() + } + return out, err } return out, nil } diff --git a/internal/openbao/database.go b/internal/openbao/database.go index 7de8061..dacb5e6 100644 --- a/internal/openbao/database.go +++ b/internal/openbao/database.go @@ -2,7 +2,6 @@ package openbao import ( "context" - "encoding/base64" "encoding/json" "fmt" "strings" @@ -60,20 +59,37 @@ func (c *Client) EnableDatabaseSecrets(ctx context.Context, opts DBSetupOptions) // 2. Configure the connection. The admin creds are used only to create/drop // the ephemeral roles; {{username}}/{{password}} are OpenBao's templating. + // The whole config — including the admin password — rides stdin as JSON + // (`write -`), never the docker exec argv (C08). connURL := fmt.Sprintf("postgresql://{{username}}:{{password}}@%s:5432/%s?sslmode=disable", dbHost, opts.DBName) - cfg := fmt.Sprintf("write database/config/%s plugin_name=postgresql-database-plugin allowed_roles=%s connection_url=%s username=%s password=%s", - dbConnName(opts.App), shellSingleQuote(dbRoleName(opts.App)), - shellSingleQuote(connURL), shellSingleQuote(opts.AdminUser), shellSingleQuote(opts.AdminPass)) - if out, err := c.bao(ctx, container, root, cfg); err != nil { + cfg, err := json.Marshal(map[string]string{ + "plugin_name": "postgresql-database-plugin", + "allowed_roles": dbRoleName(opts.App), + "connection_url": connURL, + "username": opts.AdminUser, + "password": opts.AdminPass, + }) + if err != nil { + return fmt.Errorf("encoding db connection config: %w", err) + } + if out, err := c.baoInput(ctx, container, root, "write database/config/"+dbConnName(opts.App)+" -", string(cfg)); err != nil { return fmt.Errorf("configuring db connection: %s", truncate(out, 200)) } // 3. Role: each cred request mints a login role granted SELECT, expiring at - // the lease end. Least privilege — read-only by default. + // the lease end. Least privilege — read-only by default. Same stdin JSON + // transport (creation_statements is arbitrary SQL — no quoting surface). creation := `CREATE ROLE "{{name}}" WITH LOGIN PASSWORD '{{password}}' VALID UNTIL '{{expiration}}'; GRANT SELECT ON ALL TABLES IN SCHEMA public TO "{{name}}";` - role := fmt.Sprintf("write database/roles/%s db_name=%s creation_statements=%s default_ttl=%s max_ttl=%s", - dbRoleName(opts.App), dbConnName(opts.App), shellSingleQuote(creation), opts.TTL, opts.MaxTTL) - if out, err := c.bao(ctx, container, root, role); err != nil { + role, err := json.Marshal(map[string]string{ + "db_name": dbConnName(opts.App), + "creation_statements": creation, + "default_ttl": opts.TTL, + "max_ttl": opts.MaxTTL, + }) + if err != nil { + return fmt.Errorf("encoding db role config: %w", err) + } + if out, err := c.baoInput(ctx, container, root, "write database/roles/"+dbRoleName(opts.App)+" -", string(role)); err != nil { return fmt.Errorf("creating db role: %s", truncate(out, 200)) } @@ -134,17 +150,32 @@ func (c *Client) EnableStaticRole(ctx context.Context, opts StaticRoleOptions) e return fmt.Errorf("enabling database engine: %s", truncate(out, 160)) } // Connection with allowed_roles "*" so dynamic + static roles both work. + // Same stdin JSON transport as the dynamic path — the admin password + // never enters an argv (C08). connURL := fmt.Sprintf("postgresql://{{username}}:{{password}}@%s:5432/%s?sslmode=disable", dbHost, opts.DBName) - cfg := fmt.Sprintf("write database/config/%s plugin_name=postgresql-database-plugin allowed_roles=* connection_url=%s username=%s password=%s", - dbConnName(opts.App), shellSingleQuote(connURL), shellSingleQuote(opts.AdminUser), shellSingleQuote(opts.AdminPass)) - if out, err := c.bao(ctx, container, root, cfg); err != nil { + cfg, err := json.Marshal(map[string]string{ + "plugin_name": "postgresql-database-plugin", + "allowed_roles": "*", + "connection_url": connURL, + "username": opts.AdminUser, + "password": opts.AdminPass, + }) + if err != nil { + return fmt.Errorf("encoding db connection config: %w", err) + } + if out, err := c.baoInput(ctx, container, root, "write database/config/"+dbConnName(opts.App)+" -", string(cfg)); err != nil { return fmt.Errorf("configuring db connection: %s", truncate(out, 200)) } // Static role: OpenBao rotates opts.Username's password every RotationPeriod. - role := fmt.Sprintf("write database/static-roles/%s db_name=%s username=%s rotation_period=%s", - staticRoleName(opts.App, opts.Username), dbConnName(opts.App), - shellSingleQuote(opts.Username), opts.RotationPeriod) - if out, err := c.bao(ctx, container, root, role); err != nil { + role, err := json.Marshal(map[string]string{ + "db_name": dbConnName(opts.App), + "username": opts.Username, + "rotation_period": opts.RotationPeriod, + }) + if err != nil { + return fmt.Errorf("encoding static role config: %w", err) + } + if out, err := c.baoInput(ctx, container, root, "write database/static-roles/"+staticRoleName(opts.App, opts.Username)+" -", string(role)); err != nil { return fmt.Errorf("creating static role: %s", truncate(out, 200)) } // Grant the app read on its rotating static creds. @@ -199,16 +230,13 @@ func (c *Client) DBCreds(ctx context.Context, app, accessory string) (map[string // writeAppPolicy (re)writes the app's read policy. When withDB is true it also // grants read on the dynamic database creds path. Single source of truth for -// the policy so EnsureAppRole and EnableDatabaseSecrets stay consistent. +// the policy so EnsureAppRole and EnableDatabaseSecrets stay consistent. The +// token (line 1) and the raw HCL policy ride stdin — the old form put the +// root token directly in the docker exec argv (C08), and the base64 dance +// existed only to survive argv quoting, which the pipe no longer needs. func (c *Client) writeAppPolicy(ctx context.Context, container, root, app string, withDB bool) error { - policy := AppReadPolicy(app, withDB) - // Pipe the policy via base64 rather than a heredoc: it avoids all quoting/ - // terminator interactions across the docker-exec + sh -c layers (a heredoc - // terminator can't share its line with the 2>&1 the bao helper appends). - b64 := base64.StdEncoding.EncodeToString([]byte(policy)) - cmd := fmt.Sprintf("echo %s | base64 -d | env BAO_ADDR=%s BAO_TOKEN=%s bao policy write %s-read -", - b64, containerAPIAddr, root, app) - if _, err := c.docker.Exec(ctx, container, cmd); err != nil { + inner := `IFS= read -r teploy_tok; export BAO_ADDR=` + containerAPIAddr + ` BAO_TOKEN="$teploy_tok"; exec bao policy write ` + app + `-read -` + if _, err := c.docker.ExecInput(ctx, container, inner, strings.NewReader(root+"\n"+AppReadPolicy(app, withDB))); err != nil { return fmt.Errorf("writing policy: %w", err) } return nil diff --git a/internal/openbao/secrets.go b/internal/openbao/secrets.go index 99f5409..07915ef 100644 --- a/internal/openbao/secrets.go +++ b/internal/openbao/secrets.go @@ -34,8 +34,11 @@ func appPath(app, name string) string { return kvMount + "/" + app + "/" + name } -// Put writes key=value pairs to secret//. Values are shell-quoted -// for the inner container shell. +// Put writes key=value pairs to secret//. The values ride +// the session's stdin as one JSON object (`bao kv put -`), never +// the docker exec argv: shell-quoted values on the command line sat in +// the server's process list for the life of every write (C08). The +// token travels the same pipe (see bao). func (c *Client) Put(ctx context.Context, app, accessory, name string, kvs []string) error { if accessory == "" { accessory = defaultAccessory @@ -45,15 +48,19 @@ func (c *Client) Put(ctx context.Context, app, accessory, name string, kvs []str return err } container := accessories.ContainerName(app, accessory) - parts := make([]string, 0, len(kvs)) + data := make(map[string]string, len(kvs)) for _, kv := range kvs { k, v, ok := strings.Cut(kv, "=") if !ok { return fmt.Errorf("invalid key=value pair %q", kv) } - parts = append(parts, k+"="+shellSingleQuote(v)) + data[k] = v } - out, err := c.bao(ctx, container, root, "kv put "+appPath(app, name)+" "+strings.Join(parts, " ")) + payload, err := json.Marshal(data) + if err != nil { + return fmt.Errorf("encoding kv payload: %w", err) + } + out, err := c.baoInput(ctx, container, root, "kv put "+appPath(app, name)+" -", string(payload)) if err != nil { return fmt.Errorf("kv put: %s", truncate(out, 160)) } @@ -183,9 +190,3 @@ func (c *Client) baoField(ctx context.Context, container, token, args, field str } return v, nil } - -// shellSingleQuote wraps a value in single quotes for the inner container shell, -// escaping embedded single quotes (mirrors the kv command's approach). -func shellSingleQuote(s string) string { - return "'" + strings.ReplaceAll(s, "'", `'\''`) + "'" -} diff --git a/internal/openbao/secrets_test.go b/internal/openbao/secrets_test.go index 13206d3..05d1f1f 100644 --- a/internal/openbao/secrets_test.go +++ b/internal/openbao/secrets_test.go @@ -1,6 +1,12 @@ package openbao -import "testing" +import ( + "context" + "strings" + "testing" + + "github.com/useteploy/teploy/internal/ssh" +) func TestAppPath(t *testing.T) { if got := appPath("myapp", "db"); got != "secret/myapp/db" { @@ -11,16 +17,135 @@ func TestAppPath(t *testing.T) { } } -func TestShellSingleQuote(t *testing.T) { - cases := map[string]string{ - "plain": "'plain'", - "a b": "'a b'", - "it's": `'it'\''s'`, - "$(rm -rf /)": `'$(rm -rf /)'`, // command substitution neutralized inside single quotes +// TestBaoTransport_SecretsNeverInArgv pins the C08 secret transport for +// every bao invocation: the auth token and the secret VALUES must travel +// the session's stdin, never the docker exec command string (where the +// server's process list would show them). Pins Put, the token-only bao +// shape, and the policy-write path. +func TestBaoTransport_SecretsNeverInArgv(t *testing.T) { + const ( + rootToken = "hvs.root-token" + apiKey = "sk-live-123" + ) + mock := ssh.NewMockExecutor("h", + ssh.MockCommand{Match: "if [ ! -e ", Output: "present"}, + ssh.MockCommand{Match: "age -d", Output: rootToken}, + ssh.MockCommand{Match: "docker exec -i 'myapp-openbao'", Output: "{}"}, + ) + c := NewClient(mock, &strings.Builder{}) + // The framed existence check reads the recorded file state; the age + // registration answers the decrypt that follows it. + mock.Files["/deployments/myapp/secrets/VAULT_ROOT_TOKEN.age"] = []byte("ciphertext") + + ctx := context.Background() + if err := c.Put(ctx, "myapp", "", "db", []string{"API_KEY=" + apiKey, "HOST=db"}); err != nil { + t.Fatalf("Put: %v", err) + } + if _, err := c.Get(ctx, "myapp", "", "db"); err != nil { + t.Fatalf("Get: %v", err) + } + + for _, call := range mock.Calls { + if strings.HasPrefix(call, "UPLOAD:") { + continue + } + if strings.Contains(call, rootToken) { + t.Errorf("root token in command argv: %s", call) + } + if strings.Contains(call, apiKey) { + t.Errorf("secret value in command argv: %s", call) + } + // The only token the argv may name is the stdin-fed variable — + // never a literal assignment. + if strings.Contains(call, "BAO_TOKEN=") && !strings.Contains(call, `BAO_TOKEN="$teploy_tok"`) { + t.Errorf("token env must be fed from stdin, not embedded: %s", call) + } + } + + // The stdin payloads carry what the argv must not: Put's payload is + // token line + JSON body; Get's is token line + newline. + putInput := "" + for _, in := range mock.Inputs { + if strings.Contains(in, "API_KEY") { + putInput = in + } + } + if putInput == "" { + t.Fatalf("Put's stdin payload not recorded: %v", mock.Inputs) + } + lines := strings.SplitN(putInput, "\n", 2) + if lines[0] != rootToken { + t.Errorf("first stdin line must be the token, got %q", lines[0]) + } + if !strings.Contains(lines[1], `"API_KEY":"`+apiKey+`"`) { + t.Errorf("JSON payload must carry the secret value: %q", lines[1]) + } +} + +// TestWriteAppPolicyTransport pins the policy write: root token on stdin, +// policy HCL on stdin, nothing secret in the command string. +func TestWriteAppPolicyTransport(t *testing.T) { + mock := ssh.NewMockExecutor("h", + ssh.MockCommand{Match: "docker exec -i 'myapp-openbao'", Output: ""}, + ) + c := NewClient(mock, &strings.Builder{}) + if err := c.writeAppPolicy(context.Background(), "myapp-openbao", "hvs.tok", "myapp", false); err != nil { + t.Fatalf("writeAppPolicy: %v", err) + } + for _, call := range mock.Calls { + if strings.Contains(call, "hvs.tok") { + t.Errorf("root token in policy-write argv: %s", call) + } + if strings.Contains(call, "base64") { + t.Errorf("policy transport no longer needs base64 argv quoting: %s", call) + } + } + found := false + for _, in := range mock.Inputs { + if strings.HasPrefix(in, "hvs.tok\n") && strings.Contains(in, "capabilities") { + found = true + } + } + if !found { + t.Errorf("policy stdin must be token line + HCL, got %v", mock.Inputs) + } +} + +// TestEnableDatabaseSecretsTransport pins the db engine config writes: +// the admin password rides the JSON stdin payload, never the command. +func TestEnableDatabaseSecretsTransport(t *testing.T) { + const adminPass = "pg-super-secret" + mock := ssh.NewMockExecutor("h", + ssh.MockCommand{Match: "if [ ! -e ", Output: "present"}, + ssh.MockCommand{Match: "age -d", Output: "hvs.root"}, + ssh.MockCommand{Match: "docker exec -i 'myapp-openbao'", Output: "{}"}, + ) + c := NewClient(mock, &strings.Builder{}) + mock.Files["/deployments/myapp/secrets/VAULT_ROOT_TOKEN.age"] = []byte("ciphertext") + + err := c.EnableDatabaseSecrets(context.Background(), DBSetupOptions{ + App: "myapp", + DBAccessory: "postgres", + AdminPass: adminPass, + }) + if err != nil { + t.Fatalf("EnableDatabaseSecrets: %v", err) } - for in, want := range cases { - if got := shellSingleQuote(in); got != want { - t.Errorf("shellSingleQuote(%q) = %q, want %q", in, got, want) + for _, call := range mock.Calls { + if strings.HasPrefix(call, "UPLOAD:") { + continue } + if strings.Contains(call, adminPass) { + t.Errorf("admin password in command argv: %s", call) + } + } + inConfig := false + for _, in := range mock.Inputs { + if strings.Contains(in, `"password":"`+adminPass+`"`) { + inConfig = true + } + } + if !inConfig { + t.Errorf("admin password must ride the JSON stdin payload, inputs: %v", mock.Inputs) } } diff --git a/internal/ssh/hostkey.go b/internal/ssh/hostkey.go new file mode 100644 index 0000000..2391320 --- /dev/null +++ b/internal/ssh/hostkey.go @@ -0,0 +1,224 @@ +package ssh + +// hostkey.go — trust-on-first-use host-key enrollment (C08). The v0.1.37 +// diagnostics (mismatchHint) are the foundation; this adds what +// concurrent first connects need: +// +// - every verify+enroll runs under an exclusive lock on a lock file +// beside known_hosts, so two simultaneous first connects serialize +// instead of racing the write; +// - the known_hosts database is re-read FRESH under the lock (the old +// callback captured the database at Connect() time, so a concurrent +// enrollment was invisible and both processes enrolled the same +// host, duplicating lines); +// - enrollment writes temp-then-rename (the old O_APPEND write could +// leave a torn line if the process died mid-write, which fails +// knownhosts parsing and locks EVERY future connection out until +// the file is hand-repaired); +// - host identity CHANGE is distinguished from RENAME: an unknown +// host presenting a key already trusted under a DIFFERENT hostname +// is the same machine, re-addressed (VPN IP rotation, DNS change) — +// the new name is enrolled with a note. An unknown key for a KNOWN +// host is an identity change and still fails closed (mismatchHint). +// +// The file format is unchanged — plain knownhosts lines, appended — so +// every older teploy and stock ssh(1) keeps reading what we write. + +import ( + "bytes" + "encoding/base64" + "errors" + "fmt" + "io/fs" + "net" + "os" + "path/filepath" + "strings" + + gossh "golang.org/x/crypto/ssh" + "golang.org/x/crypto/ssh/knownhosts" +) + +// acceptNewHostKeyCallback returns a host key callback that accepts +// unknown host keys (enrolling them in known_hosts) but rejects every +// verification failure that is NOT "host simply unknown": key +// mismatches (any algorithm), revoked keys, and an unreadable or +// malformed known_hosts database all fail closed. Trust-on-first-use +// must mean "unknown host", never "verification was inconvenient". +func acceptNewHostKeyCallback(knownHostsPath string) gossh.HostKeyCallback { + return func(hostname string, remote net.Addr, key gossh.PublicKey) error { + return enrollHostKey(knownHostsPath, hostname, remote, key) + } +} + +// enrollHostKey verifies the presented key against the CURRENT contents +// of known_hosts and enrolls unknown hosts — all under the TOFU lock. +func enrollHostKey(knownHostsPath, hostname string, remote net.Addr, key gossh.PublicKey) error { + // A fresh box may have no ~/.ssh at all; the lock file lives beside + // known_hosts, so the directory must exist before the lock is taken. + if err := os.MkdirAll(filepath.Dir(knownHostsPath), 0700); err != nil { + return fmt.Errorf("creating %s: %w", filepath.Dir(knownHostsPath), err) + } + unlock, err := lockKnownHosts(knownHostsPath) + if err != nil { + return fmt.Errorf("cannot verify host key: locking %s: %w", knownHostsPath, err) + } + defer unlock() + + // Fresh read under the lock. A missing known_hosts is the fresh-box + // case (nothing known, everything enrollable); a file that EXISTS + // but cannot be parsed fails closed. + existing, existingErr := loadKnownHosts(knownHostsPath) + if existingErr != nil { + return fmt.Errorf("cannot verify host key: reading %s failed: %w", knownHostsPath, existingErr) + } + if existing != nil { + err := existing(hostname, remote, key) + if err == nil { + return nil // known and matches + } + // Only a genuinely unknown host (empty Want list) may be + // enrolled. Any non-KeyError (revocation, database problem) and + // any mismatch against a known host (nonempty Want) is rejected. + var keyErr *knownhosts.KeyError + if !errors.As(err, &keyErr) || len(keyErr.Want) != 0 { + return mismatchHint(hostname, key, err) + } + } + + line := knownhosts.Line([]string{knownhosts.Normalize(hostname)}, key) + + // Rename, not change: the presented key is already trusted under + // another name. Possession of the host key is what TOFU established; + // re-addressing the same machine is the benign explanation and is + // enrolled (with a note) rather than presented as a first use. + if others := hostsKnownUnder(knownHostsPath, key); len(others) > 0 { + if err := appendKnownHostsLine(knownHostsPath, line); err != nil { + return err + } + fmt.Fprintf(os.Stderr, "teploy: host key for %s matches the key already trusted for %s — treating this as the same server reached by a new address, and enrolling the new name\n", + hostname, strings.Join(others, ", ")) + return nil + } + + return appendKnownHostsLine(knownHostsPath, line) +} + +// loadKnownHosts builds a verifier from the file's current contents. +// A missing file is (nil, nil) — nothing is known. A file that exists +// but cannot be read or parsed is an error: TOFU must not treat a +// broken database as an empty one. +func loadKnownHosts(knownHostsPath string) (gossh.HostKeyCallback, error) { + if _, err := os.Stat(knownHostsPath); errors.Is(err, fs.ErrNotExist) { + return nil, nil + } + cb, err := knownhosts.New(knownHostsPath) + if err != nil { + return nil, err + } + return cb, nil +} + +// appendKnownHostsLine adds one line to known_hosts atomically: +// existing bytes + the new line are written to a sibling temp file, +// synced, and renamed over the destination. A crash mid-write leaves +// the previous complete file in place — never a torn line — and the +// rename preserves every other process's entries (a plain rewrite of +// only our line would clobber them). +func appendKnownHostsLine(knownHostsPath, line string) error { + dir := filepath.Dir(knownHostsPath) + // A fresh box may have no ~/.ssh at all. + if err := os.MkdirAll(dir, 0700); err != nil { + return fmt.Errorf("creating %s: %w", dir, err) + } + + mode := os.FileMode(0644) + var contents []byte + if data, err := os.ReadFile(knownHostsPath); err == nil { + contents = data + // Preserve an existing file's permissions. + if fi, statErr := os.Stat(knownHostsPath); statErr == nil { + mode = fi.Mode().Perm() + } + } else if !errors.Is(err, fs.ErrNotExist) { + return fmt.Errorf("reading %s: %w", knownHostsPath, err) + } + if len(contents) > 0 && !bytes.HasSuffix(contents, []byte("\n")) { + contents = append(contents, '\n') + } + contents = append(contents, []byte(line+"\n")...) + + tmp, err := os.CreateTemp(dir, ".known_hosts.teploy-*") + if err != nil { + return fmt.Errorf("recording host key in %s: %w", knownHostsPath, err) + } + tmpName := tmp.Name() + defer os.Remove(tmpName) // no-op after a successful rename + if err := tmp.Chmod(mode); err != nil { + tmp.Close() + return fmt.Errorf("recording host key in %s: %w", knownHostsPath, err) + } + if _, err := tmp.Write(contents); err != nil { + tmp.Close() + return fmt.Errorf("recording host key in %s: %w", knownHostsPath, err) + } + if err := tmp.Sync(); err != nil { + tmp.Close() + return fmt.Errorf("recording host key in %s: %w", knownHostsPath, err) + } + if err := tmp.Close(); err != nil { + return fmt.Errorf("recording host key in %s: %w", knownHostsPath, err) + } + if err := os.Rename(tmpName, knownHostsPath); err != nil { + return fmt.Errorf("recording host key in %s: %w", knownHostsPath, err) + } + if d, derr := os.Open(dir); derr == nil { + _ = d.Sync() + _ = d.Close() + } + return nil +} + +// hostsKnownUnder scans known_hosts for entries whose KEY matches the +// presented key, returning the hostname fields they are trusted under. +// Hashed entries expose the key material in the line, so key matching +// works across them (the hostname field itself is reported as written). +// An unreadable file yields no matches — the caller's enroll path will +// fail on the write it then attempts. +func hostsKnownUnder(knownHostsPath string, key gossh.PublicKey) []string { + data, err := os.ReadFile(knownHostsPath) + if err != nil { + return nil + } + want := base64.StdEncoding.EncodeToString(key.Marshal()) + var hosts []string + for _, ln := range strings.Split(string(data), "\n") { + ln = strings.TrimSpace(ln) + if ln == "" || strings.HasPrefix(ln, "#") { + continue + } + fields := strings.Fields(ln) + if len(fields) < 3 { + continue + } + marker := "" + hostField := fields[0] + if strings.HasPrefix(hostField, "@") { + if len(fields) < 4 { + continue + } + marker, hostField = hostField, fields[1] + // fields[2] is the key type here; the base64 is fields[3]. + if fields[3] != want { + continue + } + } else if fields[2] != want { + continue + } + if marker == "@revoked" { + continue // revoked keys are not "trusted under" anything + } + hosts = append(hosts, hostField) + } + return hosts +} diff --git a/internal/ssh/hostkey_other.go b/internal/ssh/hostkey_other.go new file mode 100644 index 0000000..26c6434 --- /dev/null +++ b/internal/ssh/hostkey_other.go @@ -0,0 +1,42 @@ +//go:build !unix + +package ssh + +import ( + "os" + "time" +) + +// lockKnownHosts is the non-Unix fallback: no flock exists, so mutual +// exclusion is approximated with O_CREATE|O_EXCL on a lock file, with a +// stale-lock takeover after a bounded wait (a crashed holder must not +// wedge enrollment forever). Best-effort by design — the resident +// deployment mode targets Linux, where the flock implementation runs; +// this keeps the build honest about what it can guarantee. +func lockKnownHosts(knownHostsPath string) (func(), error) { + lockPath := knownHostsPath + ".teploy-lock" + deadline := time.Now().Add(5 * time.Second) + tookOver := false + for { + f, err := os.OpenFile(lockPath, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0600) + if err == nil { + return func() { + f.Close() + os.Remove(lockPath) + }, nil + } + if !os.IsExist(err) { + return nil, err + } + if time.Now().After(deadline) { + if tookOver { + return nil, os.ErrDeadlineExceeded + } + // Stale-lock takeover, once — then fail rather than spin. + os.Remove(lockPath) + tookOver = true + deadline = time.Now().Add(5 * time.Second) + } + time.Sleep(20 * time.Millisecond) + } +} diff --git a/internal/ssh/hostkey_test.go b/internal/ssh/hostkey_test.go index 6c0c673..96bb0c4 100644 --- a/internal/ssh/hostkey_test.go +++ b/internal/ssh/hostkey_test.go @@ -2,10 +2,12 @@ package ssh import ( "bytes" + "context" "crypto/ed25519" "crypto/rand" "crypto/rsa" "errors" + "fmt" "net" "os" "os/exec" @@ -13,17 +15,17 @@ import ( "strings" "testing" - "golang.org/x/crypto/ssh" + gossh "golang.org/x/crypto/ssh" "golang.org/x/crypto/ssh/knownhosts" ) -func testPublicKey(t *testing.T) ssh.PublicKey { +func testPublicKey(t *testing.T) gossh.PublicKey { t.Helper() pub, _, err := ed25519.GenerateKey(rand.Reader) if err != nil { t.Fatalf("generating test key: %v", err) } - sshPub, err := ssh.NewPublicKey(pub) + sshPub, err := gossh.NewPublicKey(pub) if err != nil { t.Fatalf("wrapping test key: %v", err) } @@ -118,7 +120,7 @@ func TestHostKeyMismatchNamesAlgorithms(t *testing.T) { if err != nil { t.Fatalf("generating rsa test key: %v", err) } - presented, err := ssh.NewPublicKey(&rsaPriv.PublicKey) + presented, err := gossh.NewPublicKey(&rsaPriv.PublicKey) if err != nil { t.Fatalf("wrapping rsa test key: %v", err) } @@ -215,3 +217,160 @@ func TestPublicKeyBytes_DerivesFromPrivateKey(t *testing.T) { t.Error("an explicit key without .pub must not select an unrelated default public key") } } + +// TestTOFU_ConcurrentFirstConnects pins C08's concurrency requirement +// at the callback level: many simultaneous first connects to the same +// unknown host must all succeed, enroll the key EXACTLY once, and leave +// a known_hosts that still parses (no torn lines, no duplicated +// entries). The old callback captured its knownhosts database at +// Connect() time and appended without a lock — every racing process saw +// "unknown", enrolled, and duplicated. +func TestTOFU_ConcurrentFirstConnects(t *testing.T) { + dir := t.TempDir() + knownHostsPath := filepath.Join(dir, ".ssh", "known_hosts") + callback := acceptNewHostKeyCallback(knownHostsPath) + key := testPublicKey(t) + + const n = 16 + errs := make(chan error, n) + for i := 0; i < n; i++ { + go func() { + errs <- callback("203.0.113.10:22", fakeAddr{}, key) + }() + } + for i := 0; i < n; i++ { + if err := <-errs; err != nil { + t.Fatalf("a concurrent first connect failed: %v", err) + } + } + + data, err := os.ReadFile(knownHostsPath) + if err != nil { + t.Fatalf("known_hosts not written: %v", err) + } + if got := strings.Count(string(data), "203.0.113.10"); got != 1 { + t.Fatalf("host enrolled %d times, want exactly 1:\n%s", got, data) + } + // The file must remain parseable — a torn line would lock out every + // future connection. + if _, err := knownhosts.New(knownHostsPath); err != nil { + t.Fatalf("known_hosts no longer parses after concurrent enrollment: %v", err) + } + // A verify against the enrolled state must now succeed (the second + // generation sees the key as known). + if err := callback("203.0.113.10:22", fakeAddr{}, key); err != nil { + t.Fatalf("post-enrollment verify failed: %v", err) + } +} + +// TestTOFU_ConcurrentDistinctHosts pins that simultaneous first +// connects to DIFFERENT hosts do not lose each other's enrollments +// (the write is read-modify-write under the lock, not a blind append). +func TestTOFU_ConcurrentDistinctHosts(t *testing.T) { + dir := t.TempDir() + knownHostsPath := filepath.Join(dir, ".ssh", "known_hosts") + + const n = 8 + errs := make(chan error, n) + for i := 0; i < n; i++ { + i := i + go func() { + cb := acceptNewHostKeyCallback(knownHostsPath) + errs <- cb(fmt.Sprintf("203.0.113.%d:22", 20+i), fakeAddr{}, testPublicKey(t)) + }() + } + for i := 0; i < n; i++ { + if err := <-errs; err != nil { + t.Fatalf("concurrent enrollment failed: %v", err) + } + } + data, err := os.ReadFile(knownHostsPath) + if err != nil { + t.Fatal(err) + } + for i := 0; i < n; i++ { + if !strings.Contains(string(data), fmt.Sprintf("203.0.113.%d", 20+i)) { + t.Fatalf("host %d lost to a concurrent enrollment:\n%s", i, data) + } + } +} + +// TestTOFU_RenameVsChange pins the distinction C08 requires: the SAME +// key presented for a NEW hostname is a rename/re-address of a machine +// already trusted — enrolled, with a note, without alarm. A DIFFERENT +// key for a KNOWN hostname is an identity change and fails closed (the +// v0.1.37 mismatch diagnostics). +func TestTOFU_RenameVsChange(t *testing.T) { + dir := t.TempDir() + knownHostsPath := filepath.Join(dir, ".ssh", "known_hosts") + callback := acceptNewHostKeyCallback(knownHostsPath) + + key := testPublicKey(t) + if err := callback("198.51.100.1:22", fakeAddr{}, key); err != nil { + t.Fatalf("first use: %v", err) + } + + // Same key, new name: rename — accepted, both names trusted. + if err := callback("198.51.100.2:22", fakeAddr{}, key); err != nil { + t.Fatalf("rename must enroll the new name, got: %v", err) + } + data, _ := os.ReadFile(knownHostsPath) + if !strings.Contains(string(data), "198.51.100.1") || !strings.Contains(string(data), "198.51.100.2") { + t.Fatalf("both names must be trusted after a rename:\n%s", data) + } + + // Different key, known name: identity change — refused, file + // unchanged by the refusal. + before, _ := os.ReadFile(knownHostsPath) + if err := callback("198.51.100.1:22", fakeAddr{}, testPublicKey(t)); err == nil { + t.Fatal("an identity change for a known host must fail closed") + } + after, _ := os.ReadFile(knownHostsPath) + if !bytes.Equal(before, after) { + t.Fatal("a rejected identity change must not modify known_hosts") + } +} + +// TestTOFU_RealConcurrentFirstConnects drives two full Connect()s +// against the in-process SSH server simultaneously, sharing one fresh +// $HOME — the end-to-end version of the callback-level race pin. +func TestTOFU_RealConcurrentFirstConnects(t *testing.T) { + if testing.Short() { + t.Skip("network + processes") + } + addr := startTestSSHServer(t, func(cmd string, ch gossh.Channel) int { return 0 }) + home := t.TempDir() + t.Setenv("HOME", home) + + const n = 2 + errs := make(chan error, n) + for i := 0; i < n; i++ { + go func() { + _, err := Connect(context.Background(), ConnectConfig{ + Host: addr, User: "root", KeyPath: writeClientKeyFile(t), AcceptNewHost: true, + }) + errs <- err + }() + } + for i := 0; i < n; i++ { + if err := <-errs; err != nil { + t.Fatalf("a simultaneous first connect failed: %v", err) + } + } + + knownHostsPath := filepath.Join(home, ".ssh", "known_hosts") + data, err := os.ReadFile(knownHostsPath) + if err != nil { + t.Fatalf("known_hosts not written: %v", err) + } + host, _, serr := SplitHostPort(addr) + if serr != nil || host == "" { + host = addr + } + if got := strings.Count(string(data), host); got != 1 { + t.Fatalf("host enrolled %d times, want exactly 1:\n%s", got, data) + } + if _, err := knownhosts.New(knownHostsPath); err != nil { + t.Fatalf("known_hosts does not parse after the race: %v", err) + } +} diff --git a/internal/ssh/hostkey_unix.go b/internal/ssh/hostkey_unix.go new file mode 100644 index 0000000..63bc6b0 --- /dev/null +++ b/internal/ssh/hostkey_unix.go @@ -0,0 +1,32 @@ +//go:build unix + +package ssh + +import ( + "os" + "syscall" +) + +// lockKnownHosts takes an exclusive advisory lock on a lock file beside +// known_hosts, serializing concurrent TOFU enrollments (C08): two +// simultaneous first connects must not race the verify+enroll window. +// flock is used rather than locking known_hosts itself so stock ssh(1) +// — which takes no locks — never interacts with ours. The lock file is +// never deleted after use: unlink-races (two processes each holding a +// lock on an unlinked inode) are exactly the corruption vector the lock +// exists to close. +func lockKnownHosts(knownHostsPath string) (func(), error) { + f, err := os.OpenFile(knownHostsPath+".teploy-lock", os.O_CREATE|os.O_RDWR, 0600) + if err != nil { + return nil, err + } + if err := syscall.Flock(int(f.Fd()), syscall.LOCK_EX); err != nil { + f.Close() + return nil, err + } + fd := int(f.Fd()) + return func() { + _ = syscall.Flock(fd, syscall.LOCK_UN) + _ = f.Close() + }, nil +} diff --git a/internal/ssh/local.go b/internal/ssh/local.go index 414eb90..3f12dcc 100644 --- a/internal/ssh/local.go +++ b/internal/ssh/local.go @@ -2,9 +2,11 @@ package ssh import ( "context" + "errors" "fmt" "io" "os" + "os/exec" "os/user" "path/filepath" "strconv" @@ -54,6 +56,70 @@ func (e *LocalExecutor) RunInput(ctx context.Context, cmd string, stdin io.Reade return c.Run() } +// runDetailed is LocalExecutor's native structured capture: separate +// bounded stdout/stderr buffers and the process exit status, with the +// process-group kill semantics localCommand already provides. See +// Result for the field contract. +func (e *LocalExecutor) runDetailed(ctx context.Context, cmd string, stdin io.Reader, limit int64) Result { + if res, done := contextFailureResult(ctx); done { + return res + } + c := localCommand(ctx, cmd) + var stdout, stderr limitedBuffer + stdout.limit, stderr.limit = limit, limit + c.Stdout = &stdout + c.Stderr = &stderr + if stdin != nil { + c.Stdin = stdin + } + done := make(chan error, 1) + if err := c.Start(); err != nil { + return Result{ExitCode: -1, Err: fmt.Errorf("starting command: %w", err)} + } + go func() { + done <- c.Wait() + }() + + select { + case err := <-done: + res := Result{ + Stdout: stdout.bytes(), + Stderr: stderr.bytes(), + ExitCode: 0, + Truncated: stdout.overflow || stderr.overflow, + } + if err == nil { + return res + } + var exitErr *exec.ExitError + if errors.As(err, &exitErr) { + res.ExitCode = exitErr.ExitCode() + return res + } + res.ExitCode = -1 + res.Err = err + return res + case <-ctx.Done(): + // exec.CommandContext's cancellation hook fires localCommand's + // Cancel (process-group SIGKILL); WaitDelay bounds the wait on + // any descendant that slipped past, so <-done returns. + if c.Cancel != nil { + _ = c.Cancel() + } + <-done + res := Result{ + Stdout: stdout.bytes(), + Stderr: stderr.bytes(), + ExitCode: -1, + Truncated: stdout.overflow || stderr.overflow, + } + res.Canceled = errors.Is(ctx.Err(), context.Canceled) + res.TimedOut = errors.Is(ctx.Err(), context.DeadlineExceeded) + res.Err = ctx.Err() + return res + } +} + // Upload writes content to a local file atomically, mirroring // RemoteExecutor.Upload's contract (audit A27): the previous version // buffered the whole input (ignoring cancellation), called os.WriteFile diff --git a/internal/ssh/mock.go b/internal/ssh/mock.go index d292e2d..70a6d2d 100644 --- a/internal/ssh/mock.go +++ b/internal/ssh/mock.go @@ -21,6 +21,10 @@ type MockExecutor struct { mu sync.Mutex Calls []string // records every command executed Files map[string][]byte // records uploaded file contents by path + // Inputs records the stdin payload of every RunInput invocation in + // call order, so tests can assert secret material traveled by stdin + // and NOT in the command string (the C08 secret-transport pins). + Inputs []string // GuardTransportFailures, when > 0, makes the next that-many GUARDED // commands (the fence-guard shape) fail with a plain transport error @@ -291,14 +295,64 @@ func (m *MockExecutor) RunStream(ctx context.Context, cmd string, stdout, stderr } func (m *MockExecutor) RunInput(ctx context.Context, cmd string, stdin io.Reader) error { - _, err := io.Copy(io.Discard, stdin) - if err != nil { - return err + if stdin != nil { + data, err := io.ReadAll(stdin) + if err != nil { + return err + } + m.mu.Lock() + m.Inputs = append(m.Inputs, string(data)) + m.mu.Unlock() } - _, err = m.Run(ctx, cmd) + _, err := m.Run(ctx, cmd) return err } +// runDetailed is MockExecutor's native structured capture: the +// registered Output/Err become Stdout/ExitCode exactly as the real +// executors report them (an error carrying "exit status N" is a command +// failure with that code; an error without one is a transport failure). +func (m *MockExecutor) runDetailed(ctx context.Context, cmd string, stdin io.Reader, limit int64) Result { + if res, done := contextFailureResult(ctx); done { + return res + } + var out string + var err error + if stdin != nil { + // Record the payload (the secret-transport pins read Inputs), + // then evaluate the command with the same output contract as the + // non-stdin path — RunInput's error-only signature would lose + // the output the structured caller exists to see. + data, readErr := io.ReadAll(stdin) + if readErr != nil { + return Result{ExitCode: -1, Err: readErr} + } + m.mu.Lock() + m.Inputs = append(m.Inputs, string(data)) + m.mu.Unlock() + out, err = m.Run(ctx, cmd) + } else { + out, err = m.Run(ctx, cmd) + } + res := Result{ExitCode: -1} + if limit > 0 && int64(len(out)) > limit { + out = out[:limit] + res.Truncated = true + } + res.Stdout = []byte(out) + if err == nil { + res.ExitCode = 0 + return res + } + if code, ok := exitCodeFromError(err); ok { + res.ExitCode = code + res.Stderr = []byte(err.Error()) + return res + } + res.Err = err + return res +} + func (m *MockExecutor) Upload(ctx context.Context, content io.Reader, remotePath string, mode string) error { data, err := io.ReadAll(content) if err != nil { diff --git a/internal/ssh/reconnect.go b/internal/ssh/reconnect.go new file mode 100644 index 0000000..0e81a50 --- /dev/null +++ b/internal/ssh/reconnect.go @@ -0,0 +1,284 @@ +package ssh + +// reconnect.go — bounded connection recovery for long multi-stage remote +// flows (C08: an interrupted setup resumes safely). A transport failure +// mid-flow (SSH channel death, connection reset) used to abort the whole +// run at whatever stage it hit; re-running was safe only because every +// stage happened to be idempotent, and the operator had to notice and +// do it. +// +// ReconnectingExecutor wraps a dial function: on a TRANSPORT-class +// failure (the command did not complete — no exit status, not a caller +// cancellation) it redials, bounded by a reconnect budget with backoff, +// and retries the single invocation that died. Commands that RAN and +// failed (any exit status) are never retried here — that decision +// belongs to the caller, who knows the command's semantics. +// +// Retrying requires every command sent through the wrapper to be safe +// to re-run; setup's commands are check-then-act idempotent (mkdir -p, +// network create-if-missing, container run-if-absent, atomic uploads), +// and hardening's are documented idempotent. Do not wrap callers whose +// commands are not. +// +// Input fidelity on retry: RunInput and Upload buffer their payload so +// a redialed attempt resends it whole (the first attempt consumed the +// reader); RunStream refuses to retry once any output has been streamed +// — a retry would duplicate it — and surfaces the byte count instead. +// This wrapper is for control-plane flows; do not route bulk transfers +// through it. + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "sync" + "time" +) + +// IsTransportError reports whether err means "the invocation did not +// complete" — connection/channel failure — as opposed to "the command +// ran and exited non-zero". Caller cancellation and deadlines are not +// transport failures: the operator asked to stop, and retrying would +// ignore them. +func IsTransportError(err error) bool { + if err == nil { + return false + } + if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { + return false + } + if _, ok := exitCodeFromError(err); ok { + return false + } + return true +} + +// ReconnectingExecutor is an Executor that survives transport failures +// by redialing through the provided dial function. +type ReconnectingExecutor struct { + dial func(ctx context.Context) (Executor, error) + + mu sync.Mutex + inner Executor + reconnectsLeft int + closed bool +} + +// NewReconnectingExecutor dials once and returns a wrapper that will +// redial at most maxReconnects further times across the wrapper's +// lifetime. +func NewReconnectingExecutor(ctx context.Context, dial func(ctx context.Context) (Executor, error), maxReconnects int) (*ReconnectingExecutor, error) { + inner, err := dial(ctx) + if err != nil { + return nil, fmt.Errorf("connecting: %w", err) + } + return &ReconnectingExecutor{dial: dial, inner: inner, reconnectsLeft: maxReconnects}, nil +} + +// recover runs invoke against the current inner executor, redialing and +// retrying once per transport failure while the reconnect budget lasts. +// mayRetry, when non-nil, is consulted before each redial: a false +// return surfaces the failure instead of retrying it (RunStream uses +// this to refuse retrying after partial output). Backoff: 500ms, 1s, +// then 2s. The mutex is NOT held across invoke or the backoff sleep — +// only around state transitions. +func (r *ReconnectingExecutor) recover(ctx context.Context, invoke func(Executor) error, mayRetry func() bool) error { + r.mu.Lock() + if r.closed { + r.mu.Unlock() + return fmt.Errorf("connection closed") + } + inner := r.inner + r.mu.Unlock() + + err := invoke(inner) + backoff := 500 * time.Millisecond + for IsTransportError(err) { + if ctx.Err() != nil { + return ctx.Err() + } + if mayRetry != nil && !mayRetry() { + return err + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(backoff): + } + backoff *= 2 + if backoff > 2*time.Second { + backoff = 2 * time.Second + } + + r.mu.Lock() + if r.closed { + r.mu.Unlock() + return fmt.Errorf("connection closed") + } + if r.reconnectsLeft <= 0 { + r.mu.Unlock() + return fmt.Errorf("connection lost and the reconnect budget is exhausted: %w", err) + } + r.reconnectsLeft-- + r.mu.Unlock() + + next, derr := r.dial(ctx) + if derr != nil { + err = fmt.Errorf("reconnecting after transport failure (%v): %w", err, derr) + continue + } + r.mu.Lock() + if r.closed { + r.mu.Unlock() + next.Close() + return fmt.Errorf("connection closed") + } + old := r.inner + r.inner = next + r.mu.Unlock() + _ = old.Close() // already dead; best effort + inner = next + err = invoke(inner) + } + return err +} + +func (r *ReconnectingExecutor) Run(ctx context.Context, cmd string) (string, error) { + var out string + err := r.recover(ctx, func(ex Executor) error { + var invokeErr error + out, invokeErr = ex.Run(ctx, cmd) + return invokeErr + }, nil) + if err != nil { + return "", err + } + return out, nil +} + +func (r *ReconnectingExecutor) RunStream(ctx context.Context, cmd string, stdout, stderr io.Writer) error { + // A retry would re-run the command from its start; if any output + // already streamed to the caller, a retry would DUPLICATE it. Only + // retry when nothing was written yet — otherwise surface the + // failure with the byte count and let the operator decide to re-run. + guard := &countingWriter{w: stdout} + err := r.recover(ctx, func(ex Executor) error { + guard.mu.Lock() + guard.n = 0 + guard.mu.Unlock() + return ex.RunStream(ctx, cmd, guard, stderr) + }, func() bool { + guard.mu.Lock() + defer guard.mu.Unlock() + return guard.n == 0 + }) + if err == nil || !IsTransportError(err) { + return err + } + guard.mu.Lock() + wrote := guard.n + guard.mu.Unlock() + if wrote == 0 { + return err // no output yet — safe for the caller to re-run + } + return fmt.Errorf("connection lost after %d bytes of streamed output — the command may have partially run; re-run when ready: %w", wrote, err) +} + +type countingWriter struct { + w io.Writer + mu sync.Mutex + n int64 +} + +func (c *countingWriter) Write(p []byte) (int, error) { + n, err := c.w.Write(p) + c.mu.Lock() + c.n += int64(n) + c.mu.Unlock() + return n, err +} + +func (r *ReconnectingExecutor) RunInput(ctx context.Context, cmd string, stdin io.Reader) error { + // Buffer the payload so a redialed attempt resends it whole: the + // first attempt consumed the reader, and a retry fed an exhausted + // one would send an empty or partial secret (C08 secret transport). + payload, err := io.ReadAll(stdin) + if err != nil { + return fmt.Errorf("reading stdin payload: %w", err) + } + return r.recover(ctx, func(ex Executor) error { + return ex.RunInput(ctx, cmd, bytes.NewReader(payload)) + }, nil) +} + +func (r *ReconnectingExecutor) Upload(ctx context.Context, content io.Reader, remotePath string, mode string) error { + // Same buffering contract as RunInput: Upload is atomic end-to-end + // (private temp + rename) on both real executors, so a retried + // upload publishes exactly one complete file — provided the retry + // still has the content. + payload, err := io.ReadAll(content) + if err != nil { + return fmt.Errorf("reading upload content: %w", err) + } + return r.recover(ctx, func(ex Executor) error { + return ex.Upload(ctx, bytes.NewReader(payload), remotePath, mode) + }, nil) +} + +// runDetailed delegates to the inner executor's structured capture and +// retries transport-class failures (res.Err set, no exit status): a +// ran-and-failed command (res.Err nil, any exit code) is a result and +// is never retried. Registered in runDetailedWithLimit's type switch so +// RunInputDetailed keeps full fidelity through the wrapper — the +// fallback path would lose the stdout markers callers classify (the su +// path's TEPLOY_SUDO_OK, C08). +func (r *ReconnectingExecutor) runDetailed(ctx context.Context, cmd string, stdin io.Reader, limit int64) Result { + var payload []byte + if stdin != nil { + var err error + payload, err = io.ReadAll(stdin) + if err != nil { + return Result{ExitCode: -1, Err: fmt.Errorf("reading stdin payload: %w", err)} + } + } + var res Result + rerr := r.recover(ctx, func(ex Executor) error { + var in io.Reader + if stdin != nil { + in = bytes.NewReader(payload) + } + res = runDetailedWithLimit(ctx, ex, cmd, in, limit) + return res.Err + }, nil) + if rerr != nil { + res.Err = rerr + } + return res +} + +func (r *ReconnectingExecutor) Close() error { + r.mu.Lock() + defer r.mu.Unlock() + if r.closed { + return nil + } + r.closed = true + return r.inner.Close() +} + +func (r *ReconnectingExecutor) Host() string { + r.mu.Lock() + defer r.mu.Unlock() + return r.inner.Host() +} + +func (r *ReconnectingExecutor) User() string { + r.mu.Lock() + defer r.mu.Unlock() + return r.inner.User() +} + +// Compile-time check: the wrapper is itself an Executor. +var _ Executor = (*ReconnectingExecutor)(nil) diff --git a/internal/ssh/reconnect_test.go b/internal/ssh/reconnect_test.go new file mode 100644 index 0000000..8f667c5 --- /dev/null +++ b/internal/ssh/reconnect_test.go @@ -0,0 +1,472 @@ +package ssh + +// reconnect_test.go — pins the C08 connection-recovery contract: +// transport-class failures redial and retry exactly the dead +// invocation; ran-and-failed commands are never retried; cancellation +// is not a transport failure; partial streamed output is never +// duplicated by a retry; stdin/upload payloads are resent whole. + +import ( + "context" + "errors" + "fmt" + "io" + "strings" + "sync" + "testing" +) + +// fakeExec is a scripted Executor for the recovery pins: every method +// pops its next scripted behavior (falling back to success) and counts +// invocations. +type fakeExec struct { + host string + + mu sync.Mutex + runCalls int + uploads []string + closed bool + + runBehaviors []func(string) (string, error) + streamBehaviors []func(string, io.Writer) error + inputBehaviors []func(string, io.Reader) error + uploadBehaviors []func(string) error +} + +func (f *fakeExec) Run(ctx context.Context, cmd string) (string, error) { + f.mu.Lock() + defer f.mu.Unlock() + f.runCalls++ + if len(f.runBehaviors) == 0 { + return "", nil + } + b := f.runBehaviors[0] + f.runBehaviors = f.runBehaviors[1:] + return b(cmd) +} + +func (f *fakeExec) RunStream(ctx context.Context, cmd string, stdout, stderr io.Writer) error { + f.mu.Lock() + f.runCalls++ + if len(f.streamBehaviors) == 0 { + f.mu.Unlock() + return nil + } + b := f.streamBehaviors[0] + f.streamBehaviors = f.streamBehaviors[1:] + f.mu.Unlock() + return b(cmd, stdout) +} + +func (f *fakeExec) RunInput(ctx context.Context, cmd string, stdin io.Reader) error { + f.mu.Lock() + f.runCalls++ + if len(f.inputBehaviors) == 0 { + f.mu.Unlock() + _, _ = io.Copy(io.Discard, stdin) + return nil + } + b := f.inputBehaviors[0] + f.inputBehaviors = f.inputBehaviors[1:] + f.mu.Unlock() + return b(cmd, stdin) +} + +func (f *fakeExec) Upload(ctx context.Context, content io.Reader, remotePath string, mode string) error { + f.mu.Lock() + f.runCalls++ + f.uploads = append(f.uploads, remotePath) + if len(f.uploadBehaviors) == 0 { + f.mu.Unlock() + _, _ = io.Copy(io.Discard, content) + return nil + } + b := f.uploadBehaviors[0] + f.uploadBehaviors = f.uploadBehaviors[1:] + f.mu.Unlock() + return b(remotePath) +} + +func (f *fakeExec) Close() error { + f.mu.Lock() + defer f.mu.Unlock() + f.closed = true + return nil +} + +func (f *fakeExec) Host() string { return f.host } +func (f *fakeExec) User() string { return "root" } + +var errConnReset = errors.New("ssh: connection reset by peer") + +// TestIsTransportError pins the classification everything else depends +// on: only "did not complete" counts; ran-and-failed (any exit-status +// shape) and caller cancellation do not. +func TestIsTransportError(t *testing.T) { + cases := []struct { + name string + err error + want bool + }{ + {"nil", nil, false}, + {"plain transport", errConnReset, true}, + {"wrapped transport", fmt.Errorf("run: %w", errConnReset), true}, + {"exit status text", errors.New("exit status 1: boom"), false}, + {"wrapped exit status", fmt.Errorf("command failed: exit status 3: denied"), false}, + {"context canceled", context.Canceled, false}, + {"context deadline", context.DeadlineExceeded, false}, + {"wrapped cancel", fmt.Errorf("session: %w", context.Canceled), false}, + } + for _, tc := range cases { + if got := IsTransportError(tc.err); got != tc.want { + t.Errorf("%s: IsTransportError(%v) = %v, want %v", tc.name, tc.err, got, tc.want) + } + } +} + +// TestReconnect_RetriesTransportFailureAndRedials pins the core +// recovery: a transport-dead command is retried once per redial, and +// the dead inner connection is closed when replaced. +func TestReconnect_RetriesTransportFailureAndRedials(t *testing.T) { + first := &fakeExec{ + runBehaviors: []func(string) (string, error){ + func(string) (string, error) { return "", errConnReset }, + }, + } + second := &fakeExec{ + runBehaviors: []func(string) (string, error){ + func(string) (string, error) { return "ok", nil }, + }, + } + dials := 0 + dial := func(ctx context.Context) (Executor, error) { + dials++ + if dials == 1 { + return first, nil + } + return second, nil + } + r, err := NewReconnectingExecutor(context.Background(), dial, 2) + if err != nil { + t.Fatal(err) + } + out, err := r.Run(context.Background(), "whoami") + if err != nil || out != "ok" { + t.Fatalf("Run = %q, %v; want ok, nil", out, err) + } + if dials != 2 { + t.Fatalf("dials = %d, want 2", dials) + } + if !first.closed { + t.Error("the dead inner connection must be closed when replaced") + } + if second.closed { + t.Error("the live inner connection must NOT be closed by a recovery") + } +} + +// TestReconnect_RanAndFailedIsNeverRetried pins that a command that +// RAN and exited non-zero is the caller's decision, not the wrapper's. +func TestReconnect_RanAndFailedIsNeverRetried(t *testing.T) { + inner := &fakeExec{ + runBehaviors: []func(string) (string, error){ + func(string) (string, error) { return "", fmt.Errorf("exit status 3: boom") }, + }, + } + dials := 0 + dial := func(ctx context.Context) (Executor, error) { + dials++ + if dials > 1 { + t.Error("a ran-and-failed command must not trigger a redial") + } + return inner, nil + } + r, err := NewReconnectingExecutor(context.Background(), dial, 5) + if err != nil { + t.Fatal(err) + } + _, runErr := r.Run(context.Background(), "false") + if runErr == nil || !strings.Contains(runErr.Error(), "exit status 3") { + t.Fatalf("ran-and-failed error must pass through verbatim, got %v", runErr) + } + if inner.runCalls != 1 { + t.Fatalf("invocations = %d, want exactly 1", inner.runCalls) + } +} + +// TestReconnect_BudgetExhaustion pins the bound: after maxReconnects +// redials the failure surfaces naming the budget, with exactly +// maxReconnects+1 invocations total. +func TestReconnect_BudgetExhaustion(t *testing.T) { + inner := &fakeExec{ + runBehaviors: []func(string) (string, error){ + func(string) (string, error) { return "", errConnReset }, + func(string) (string, error) { return "", errConnReset }, + func(string) (string, error) { return "", errConnReset }, + func(string) (string, error) { return "", errConnReset }, + }, + } + dials := 0 + dial := func(ctx context.Context) (Executor, error) { + dials++ + return inner, nil + } + r, err := NewReconnectingExecutor(context.Background(), dial, 2) + if err != nil { + t.Fatal(err) + } + _, runErr := r.Run(context.Background(), "hang") + if runErr == nil || !strings.Contains(runErr.Error(), "reconnect budget is exhausted") { + t.Fatalf("budget exhaustion must be named, got %v", runErr) + } + if !strings.Contains(runErr.Error(), "connection reset") { + t.Fatalf("the underlying transport failure must be wrapped, got %v", runErr) + } + if inner.runCalls != 3 || dials != 3 { + t.Fatalf("invocations = %d, dials = %d; want 3 and 3 (initial + 2 retries)", inner.runCalls, dials) + } +} + +// TestReconnect_CancellationIsNotRetried pins that an operator Ctrl-C +// (context cancellation surfaced through the invocation) is respected: +// no redial, exactly one invocation. +func TestReconnect_CancellationIsNotRetried(t *testing.T) { + inner := &fakeExec{ + runBehaviors: []func(string) (string, error){ + func(string) (string, error) { return "", fmt.Errorf("session: %w", context.Canceled) }, + }, + } + dials := 0 + dial := func(ctx context.Context) (Executor, error) { + dials++ + if dials > 1 { + t.Error("cancellation must not trigger a redial") + } + return inner, nil + } + r, err := NewReconnectingExecutor(context.Background(), dial, 5) + if err != nil { + t.Fatal(err) + } + _, runErr := r.Run(context.Background(), "slow") + if !errors.Is(runErr, context.Canceled) { + t.Fatalf("cancellation must surface as itself, got %v", runErr) + } + if inner.runCalls != 1 { + t.Fatalf("invocations = %d, want 1", inner.runCalls) + } +} + +// TestReconnect_RunStreamPartialOutputNotDuplicated pins the streamed +// guard: once bytes reached the caller's writer, a transport failure is +// NOT retried (a retry would duplicate the output) and the error names +// the byte count. +func TestReconnect_RunStreamPartialOutputNotDuplicated(t *testing.T) { + inner := &fakeExec{ + streamBehaviors: []func(string, io.Writer) error{ + func(_ string, stdout io.Writer) error { + fmt.Fprint(stdout, "partial") + return errConnReset + }, + func(_ string, stdout io.Writer) error { + fmt.Fprint(stdout, "SHOULD-NOT-APPEAR") + return nil + }, + }, + } + dial := func(ctx context.Context) (Executor, error) { return inner, nil } + r, err := NewReconnectingExecutor(context.Background(), dial, 5) + if err != nil { + t.Fatal(err) + } + var buf strings.Builder + streamErr := r.RunStream(context.Background(), "logs", &buf, io.Discard) + if streamErr == nil || !strings.Contains(streamErr.Error(), "7 bytes") { + t.Fatalf("partial-stream failure must name the streamed byte count, got %v", streamErr) + } + if !strings.Contains(streamErr.Error(), "re-run when ready") { + t.Fatalf("the failure must tell the operator re-running is the recovery, got %v", streamErr) + } + if buf.String() != "partial" { + t.Fatalf("streamed output = %q, want exactly the partial bytes", buf.String()) + } + if inner.runCalls != 1 { + t.Fatalf("a partial-output failure must not be retried, invocations = %d", inner.runCalls) + } +} + +// TestReconnect_RunStreamNoOutputIsRetried pins the other half: a +// transport death BEFORE any output is safely retried, streaming the +// full output exactly once. +func TestReconnect_RunStreamNoOutputIsRetried(t *testing.T) { + inner := &fakeExec{ + streamBehaviors: []func(string, io.Writer) error{ + func(string, io.Writer) error { return errConnReset }, + func(_ string, stdout io.Writer) error { + fmt.Fprint(stdout, "full output") + return nil + }, + }, + } + dial := func(ctx context.Context) (Executor, error) { return inner, nil } + r, err := NewReconnectingExecutor(context.Background(), dial, 1) + if err != nil { + t.Fatal(err) + } + var buf strings.Builder + if streamErr := r.RunStream(context.Background(), "logs", &buf, io.Discard); streamErr != nil { + t.Fatalf("zero-output transport failure must recover: %v", streamErr) + } + if buf.String() != "full output" { + t.Fatalf("streamed output = %q, want exactly one full copy", buf.String()) + } +} + +// TestReconnect_RunInputResendsPayloadWhole pins the stdin fidelity +// fix: the first attempt consumed the reader, so the redialed retry +// must still receive the complete payload — not an exhausted reader's +// empty string. +func TestReconnect_RunInputResendsPayloadWhole(t *testing.T) { + var received []string + inner := &fakeExec{ + inputBehaviors: []func(string, io.Reader) error{ + func(_ string, stdin io.Reader) error { + data, _ := io.ReadAll(stdin) + received = append(received, string(data)) + return errConnReset + }, + func(_ string, stdin io.Reader) error { + data, _ := io.ReadAll(stdin) + received = append(received, string(data)) + return nil + }, + }, + } + dial := func(ctx context.Context) (Executor, error) { return inner, nil } + r, err := NewReconnectingExecutor(context.Background(), dial, 1) + if err != nil { + t.Fatal(err) + } + if inputErr := r.RunInput(context.Background(), "read-secret", strings.NewReader("root-password\n")); inputErr != nil { + t.Fatalf("RunInput must recover after a transport death: %v", inputErr) + } + if len(received) != 2 || received[0] != "root-password\n" || received[1] != "root-password\n" { + t.Fatalf("both attempts must receive the full payload, got %q", received) + } +} + +// TestReconnect_UploadRetried pins the upload retry: a transport-dead +// upload is attempted again on the redialed connection (content +// fidelity is pinned separately below via MockExecutor's Files). +func TestReconnect_UploadRetried(t *testing.T) { + inner := &fakeExec{ + uploadBehaviors: []func(string) error{ + func(string) error { return errConnReset }, + func(string) error { return nil }, + }, + } + dial := func(ctx context.Context) (Executor, error) { return inner, nil } + r, err := NewReconnectingExecutor(context.Background(), dial, 1) + if err != nil { + t.Fatal(err) + } + if upErr := r.Upload(context.Background(), strings.NewReader("payload"), "/tmp/f", "0600"); upErr != nil { + t.Fatalf("Upload must recover after a transport death: %v", upErr) + } + if len(inner.uploads) != 2 || inner.uploads[0] != "/tmp/f" || inner.uploads[1] != "/tmp/f" { + t.Fatalf("upload paths = %v; want 2 attempts on /tmp/f", inner.uploads) + } +} + +// TestReconnect_UploadResendsContentWhole pins the content fidelity the +// path-count pin above cannot: after a transport death, the redialed +// MockExecutor receives the FULL upload content (Files records what +// actually arrived) — and a second failure plus exhausted budget +// surfaces rather than publishing a partial file. +func TestReconnect_UploadResendsContentWhole(t *testing.T) { + mock := NewMockExecutor("h", + MockCommand{Match: "UPLOAD:/deployments/caddy/Caddyfile", Err: errConnReset, Once: true}, + ) + dial := func(ctx context.Context) (Executor, error) { return mock, nil } + r, err := NewReconnectingExecutor(context.Background(), dial, 1) + if err != nil { + t.Fatal(err) + } + content := "{\n\tadmin 127.0.0.1:2019\n}\n" + if upErr := r.Upload(context.Background(), strings.NewReader(content), "/deployments/caddy/Caddyfile", "0644"); upErr != nil { + t.Fatalf("Upload must recover: %v", upErr) + } + if got := string(mock.Files["/deployments/caddy/Caddyfile"]); got != content { + t.Fatalf("the retried upload must land the FULL content, got %q", got) + } +} + +// TestReconnect_RunInputDetailedKeepsStructuredFields pins the +// structured-result delegation: RunInputDetailed through the wrapper +// returns the inner executor's native fields (exit code, stdout) after +// recovering — the generic fallback would lose the stdout markers +// callers classify (installSudoViaSu's TEPLOY_SUDO_OK). +func TestReconnect_RunInputDetailedKeepsStructuredFields(t *testing.T) { + mock := NewMockExecutor("h", + MockCommand{Match: "su -c", Err: errConnReset, Once: true}, + MockCommand{Match: "su -c", Output: "TEPLOY_SUDO_OK\n"}, + ) + dial := func(ctx context.Context) (Executor, error) { return mock, nil } + r, err := NewReconnectingExecutor(context.Background(), dial, 1) + if err != nil { + t.Fatal(err) + } + res := RunInputDetailed(context.Background(), r, "su -c 'install-sudo' - root", strings.NewReader("rootpw\n")) + if res.Err != nil { + t.Fatalf("recovered invocation must be a success result, got %+v", res) + } + if res.ExitCode != 0 || !strings.Contains(string(res.Stdout), "TEPLOY_SUDO_OK") { + t.Fatalf("structured fields must survive recovery: %+v %q", res, res.Stdout) + } + // Both attempts received the password — the retry resent it whole. + if len(mock.Inputs) != 2 || mock.Inputs[0] != "rootpw\n" || mock.Inputs[1] != "rootpw\n" { + t.Fatalf("stdin must be resent whole on the retry, inputs: %q", mock.Inputs) + } +} + +// TestReconnect_RunDetailedRanAndFailedNotRetried pins that the +// structured path shares the no-retry rule for completed commands: a +// non-zero exit is a result, not a recovery trigger. +func TestReconnect_RunDetailedRanAndFailedNotRetried(t *testing.T) { + mock := NewMockExecutor("h", + MockCommand{Match: "failing", Err: errors.New("exit status 2: nope")}, + ) + dial := func(ctx context.Context) (Executor, error) { return mock, nil } + r, err := NewReconnectingExecutor(context.Background(), dial, 5) + if err != nil { + t.Fatal(err) + } + res := RunDetailed(context.Background(), r, "failing") + if res.Err != nil || res.ExitCode != 2 { + t.Fatalf("ran-and-failed must surface as a structured result, got %+v", res) + } + if len(mock.Calls) != 1 { + t.Fatalf("a ran-and-failed command must not be retried, calls: %v", mock.Calls) + } +} + +// TestReconnect_ClosePreventsFurtherUse pins lifecycle: after Close, +// invocations fail fast without touching a dead connection, and Close +// is idempotent. +func TestReconnect_ClosePreventsFurtherUse(t *testing.T) { + inner := &fakeExec{} + dial := func(ctx context.Context) (Executor, error) { return inner, nil } + r, err := NewReconnectingExecutor(context.Background(), dial, 3) + if err != nil { + t.Fatal(err) + } + if closeErr := r.Close(); closeErr != nil { + t.Fatalf("Close: %v", closeErr) + } + if closeErr := r.Close(); closeErr != nil { + t.Fatalf("Close must be idempotent: %v", closeErr) + } + if _, runErr := r.Run(context.Background(), "whoami"); runErr == nil || !strings.Contains(runErr.Error(), "connection closed") { + t.Fatalf("post-Close Run must fail fast, got %v", runErr) + } +} diff --git a/internal/ssh/remote.go b/internal/ssh/remote.go index bfb6c88..436a28a 100644 --- a/internal/ssh/remote.go +++ b/internal/ssh/remote.go @@ -30,6 +30,9 @@ type RemoteExecutor struct { // acceptNewHost records the host-key policy this connection was created // with, so secondary channels (e.g. static-deploy rsync) can mirror it. acceptNewHost bool + // commandTimeout bounds each command when the caller's context has + // no earlier deadline (ConnectConfig.CommandTimeout, C08). + commandTimeout time.Duration } // ConnectConfig holds the parameters for establishing an SSH connection. @@ -39,6 +42,14 @@ type ConnectConfig struct { KeyPath string // Path to SSH private key (optional, tries defaults) Password string // if set, use password auth instead of/in addition to key auth AcceptNewHost bool // if true, auto-accept unknown host keys and save to known_hosts + // CommandTimeout, when > 0, bounds EVERY command run on this + // connection: a context with no deadline (or a later one) gets this + // one, so a hung remote command dies at a deadline instead of + // hanging the CLI forever (C08 bounded subprocess lifetime). A + // caller-supplied EARLIER deadline always wins. 0 keeps the historic + // caller-controlled behavior. Long-running work (deploys, log + // tails) should pass explicit deadlines rather than raise this. + CommandTimeout time.Duration } // Connect establishes an SSH connection and returns a RemoteExecutor. @@ -108,10 +119,25 @@ func Connect(ctx context.Context, cfg ConnectConfig) (*RemoteExecutor, error) { return nil, fmt.Errorf("connecting to %s: %w", cfg.Host, err) } - return &RemoteExecutor{client: client, host: cfg.Host, user: cfg.User, acceptNewHost: cfg.AcceptNewHost}, nil + return &RemoteExecutor{client: client, host: cfg.Host, user: cfg.User, acceptNewHost: cfg.AcceptNewHost, commandTimeout: cfg.CommandTimeout}, nil +} + +// boundCtx applies the connection's CommandTimeout when the caller's +// context has no deadline (or a later one). A caller-supplied earlier +// deadline always wins; CommandTimeout 0 leaves the context untouched. +func (e *RemoteExecutor) boundCtx(ctx context.Context) (context.Context, context.CancelFunc) { + if e.commandTimeout <= 0 { + return ctx, func() {} + } + if d, ok := ctx.Deadline(); ok && d.Before(time.Now().Add(e.commandTimeout)) { + return ctx, func() {} + } + return context.WithTimeout(ctx, e.commandTimeout) } func (e *RemoteExecutor) Run(ctx context.Context, cmd string) (string, error) { + ctx, cancel := e.boundCtx(ctx) + defer cancel() var stdout, stderr bytes.Buffer if err := e.RunStream(ctx, cmd, &stdout, &stderr); err != nil { if stderr.Len() > 0 { @@ -123,6 +149,8 @@ func (e *RemoteExecutor) Run(ctx context.Context, cmd string) (string, error) { } func (e *RemoteExecutor) RunStream(ctx context.Context, cmd string, stdout, stderr io.Writer) error { + ctx, cancel := e.boundCtx(ctx) + defer cancel() session, err := e.client.NewSession() if err != nil { return fmt.Errorf("creating SSH session: %w", err) @@ -152,6 +180,8 @@ func (e *RemoteExecutor) RunStream(ctx context.Context, cmd string, stdout, stde } func (e *RemoteExecutor) RunInput(ctx context.Context, cmd string, stdin io.Reader) error { + ctx, cancel := e.boundCtx(ctx) + defer cancel() session, err := e.client.NewSession() if err != nil { return fmt.Errorf("creating SSH session: %w", err) @@ -176,6 +206,73 @@ func (e *RemoteExecutor) RunInput(ctx context.Context, cmd string, stdin io.Read } } +// runDetailed is RemoteExecutor's native structured capture: separate +// bounded stdout/stderr buffers, the remote exit status from the SSH +// channel's exit-status request, and the context's cancellation vs +// deadline distinction. See Result for the field contract. +func (e *RemoteExecutor) runDetailed(ctx context.Context, cmd string, stdin io.Reader, limit int64) Result { + ctx, cancel := e.boundCtx(ctx) + defer cancel() + if res, done := contextFailureResult(ctx); done { + return res + } + session, err := e.client.NewSession() + if err != nil { + return Result{ExitCode: -1, Err: fmt.Errorf("creating SSH session: %w", err)} + } + defer session.Close() + + var stdout, stderr limitedBuffer + stdout.limit, stderr.limit = limit, limit + session.Stdout = &stdout + session.Stderr = &stderr + if stdin != nil { + session.Stdin = stdin + } + + done := make(chan error, 1) + go func() { + done <- session.Run(cmd) + }() + + select { + case err := <-done: + res := Result{ + Stdout: stdout.bytes(), + Stderr: stderr.bytes(), + ExitCode: 0, + Truncated: stdout.overflow || stderr.overflow, + } + if err == nil { + return res + } + var exitErr *gossh.ExitError + if errors.As(err, &exitErr) { + res.ExitCode = exitErr.Waitmsg.ExitStatus() + return res + } + // No exit status arrived: the command did not complete (channel + // torn down, connection lost). + res.ExitCode = -1 + res.Err = err + return res + case <-ctx.Done(): + _ = session.Signal(gossh.SIGTERM) + _ = session.Close() + <-done + res := Result{ + Stdout: stdout.bytes(), + Stderr: stderr.bytes(), + ExitCode: -1, + Truncated: stdout.overflow || stderr.overflow, + } + res.Canceled = errors.Is(ctx.Err(), context.Canceled) + res.TimedOut = errors.Is(ctx.Err(), context.DeadlineExceeded) + res.Err = ctx.Err() + return res + } +} + // Upload streams content into a securely created sibling temporary file and // atomically renames it over remotePath. // @@ -396,64 +493,8 @@ func dialWithContext(ctx context.Context, network, addr string, config *gossh.Cl return gossh.NewClient(c, chans, reqs), nil } -// acceptNewHostKeyCallback returns a host key callback that accepts unknown -// host keys (appending them to known_hosts) but rejects every verification -// failure that is NOT "host simply unknown": key mismatches (any algorithm), -// revoked keys, and an unreadable/malformed known_hosts database all fail -// closed. Trust-on-first-use must mean "unknown host", never "verification -// was inconvenient" — the previous version treated a known_hosts parse -// failure as "nothing is known" (accepting whatever key was presented) and -// let knownhosts.RevokedError fall through the unknown-host branch. -func acceptNewHostKeyCallback(knownHostsPath string) gossh.HostKeyCallback { - existing, existingErr := knownhosts.New(knownHostsPath) - if existingErr != nil && errors.Is(existingErr, fs.ErrNotExist) { - // A missing known_hosts is the fresh-box case: nothing is known, so - // every host is unknown and TOFU-enrollable. Only a file that EXISTS - // but cannot be read or parsed fails closed below. - existing, existingErr = nil, nil - } - return func(hostname string, remote net.Addr, key gossh.PublicKey) error { - if existingErr != nil { - return fmt.Errorf("cannot verify host key: reading %s failed: %w", knownHostsPath, existingErr) - } - if existing != nil { - err := existing(hostname, remote, key) - if err == nil { - return nil // known and matches - } - // Only a genuinely unknown host (empty Want list) may be enrolled. - // Any non-KeyError (revocation, database problem) and any mismatch - // against a known host (nonempty Want, same or different algorithm) - // is rejected. - var keyErr *knownhosts.KeyError - if !errors.As(err, &keyErr) || len(keyErr.Want) != 0 { - return mismatchHint(hostname, key, err) - } - } - // Append to known_hosts. Ensure the parent directory exists first (a - // fresh box may have no ~/.ssh at all) so a merely-missing directory - // doesn't get treated the same as a genuine write failure below. - if err := os.MkdirAll(filepath.Dir(knownHostsPath), 0700); err != nil { - return fmt.Errorf("creating %s: %w", filepath.Dir(knownHostsPath), err) - } - line := knownhosts.Line([]string{knownhosts.Normalize(hostname)}, key) - f, err := os.OpenFile(knownHostsPath, os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0644) - if err != nil { - // Previously returned nil here — accepting the key anyway when it - // couldn't be recorded. That silently disables TOFU protection: every - // later connection looks like another first connection, so a key - // change (MITM) is never detected. Fail the connection instead; a - // read-only home or full disk is rare enough that failing loudly - // beats a permanently-unprotected connection. - return fmt.Errorf("recording host key in %s: %w", knownHostsPath, err) - } - defer f.Close() - if _, err := f.WriteString(line + "\n"); err != nil { - return fmt.Errorf("recording host key in %s: %w", knownHostsPath, err) - } - return nil - } -} +// acceptNewHostKeyCallback, the TOFU enrollment machinery, and the +// mismatch / change-vs-rename distinction live in hostkey.go (C08). // PublicKeyBytes returns the authorized-key line for the identity the // caller will actually authenticate with (audit A32): for an explicit key diff --git a/internal/ssh/result.go b/internal/ssh/result.go new file mode 100644 index 0000000..fd6c3ea --- /dev/null +++ b/internal/ssh/result.go @@ -0,0 +1,228 @@ +package ssh + +// result.go — the structured executor result (C08). Executor.Run's +// (trimmedStdout, error) contract folds everything else into an error +// string: callers that needed the exit code, the real stderr, or the +// difference between "the command ran and failed" and "the command never +// completed" had to parse text — "not found", "status 127" — out of a +// generic wrapper, which breaks the moment a tool changes wording or a +// value legitimately contains the phrase. Result carries each fact as a +// field so call sites stop guessing. +// +// RunDetailed/RunInputDetailed are package-level helpers over any +// Executor: native structured capture for RemoteExecutor and +// LocalExecutor, a derived shape for MockExecutor (whose registered +// errors represent combined failure text), and a best-effort fallback +// (stdout captured, exit code -1) for other implementations. + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "os/exec" + "regexp" + "strings" + "sync" + + gossh "golang.org/x/crypto/ssh" +) + +// DefaultResultLimit bounds stdout and stderr capture in a Result: one +// stream of unbounded output (a runaway log dump) must not take the +// CLI's memory with it. When either stream crosses the limit the tail is +// dropped and Truncated is set — callers that need whole-stream bytes +// must use RunStream into their own writer, not RunDetailed. +const DefaultResultLimit = 1 << 20 // 1 MiB per stream + +// Result is the structured outcome of one invocation. Err is nil when +// the command RAN to completion — including a non-zero exit, which is a +// result, not a transport failure. Err is non-nil only when the command +// did not complete: session/transport failure, cancellation, or timeout. +// When Err is non-nil, ExitCode carries no information (-1 unless the +// server happened to deliver a status first). +type Result struct { + Stdout []byte + Stderr []byte + ExitCode int // 0..255 when known; -1 when unavailable + TimedOut bool // the context deadline expired before completion + Canceled bool // the context was canceled before completion + Truncated bool // Stdout or Stderr hit the capture limit + Err error +} + +// Failed reports whether the invocation ended in any non-success state: +// a non-zero exit, a transport failure, cancellation, or timeout. +func (r Result) Failed() bool { + return r.Err != nil || r.ExitCode != 0 +} + +// TrimmedStdout is Stdout with surrounding whitespace removed — the +// shape Executor.Run callers are used to, for code that reads one +// scalar value out of a command. +func (r Result) TrimmedStdout() string { + return strings.TrimSpace(string(r.Stdout)) +} + +// ExitErrorText renders the failure reason for display: stderr when the +// command produced some, the transport error otherwise. Empty for +// success and for failures that said nothing. +func (r Result) ExitErrorText() string { + if len(r.Stderr) > 0 { + return strings.TrimSpace(string(r.Stderr)) + } + if r.Err != nil { + return r.Err.Error() + } + return "" +} + +// RunDetailed executes cmd and returns its structured outcome. stdin is +// nil. See Result for the field semantics. +func RunDetailed(ctx context.Context, ex Executor, cmd string) Result { + return runDetailedWithLimit(ctx, ex, cmd, nil, DefaultResultLimit) +} + +// RunInputDetailed executes cmd with stdin streamed from stdin (the +// secret-transport contract of Executor.RunInput) while ALSO capturing +// stdout/stderr and the structured status — RunInput's discard-both +// contract hides the diagnostics a failing stdin-fed pipeline needs +// (docker login refusals, su authentication failures). +func RunInputDetailed(ctx context.Context, ex Executor, cmd string, stdin io.Reader) Result { + return runDetailedWithLimit(ctx, ex, cmd, stdin, DefaultResultLimit) +} + +func runDetailedWithLimit(ctx context.Context, ex Executor, cmd string, stdin io.Reader, limit int64) Result { + switch v := ex.(type) { + case *RemoteExecutor: + return v.runDetailed(ctx, cmd, stdin, limit) + case *LocalExecutor: + return v.runDetailed(ctx, cmd, stdin, limit) + case *MockExecutor: + return v.runDetailed(ctx, cmd, stdin, limit) + case *ReconnectingExecutor: + return v.runDetailed(ctx, cmd, stdin, limit) + default: + return fallbackDetailed(ctx, ex, cmd, stdin, limit) + } +} + +// fallbackDetailed covers executors without native structured capture +// (test doubles, other Executor implementations). Output is captured +// via RunStream; stdout and stderr are not separated by the base +// contract, so a failure's combined text lands in Stderr, with the exit +// code derived from the error when it carries one. An error with no +// derivable exit status is a transport failure and stays in Err. +func fallbackDetailed(ctx context.Context, ex Executor, cmd string, stdin io.Reader, limit int64) Result { + res := Result{ExitCode: -1} + if err := ctx.Err(); err != nil { + res.Canceled = errors.Is(err, context.Canceled) + res.TimedOut = errors.Is(err, context.DeadlineExceeded) + res.Err = err + return res + } + + var out, diag limitedBuffer + out.limit, diag.limit = limit, limit + runErr := ex.RunStream(ctx, cmd, &out, &diag) + return finishFallback(runErr, &out, &diag) +} + +func finishFallback(runErr error, out, diag *limitedBuffer) Result { + res := Result{ + Stdout: out.bytes(), + Truncated: out.overflow || diag.overflow, + ExitCode: -1, + } + if runErr == nil { + res.ExitCode = 0 + return res + } + if code, ok := exitCodeFromError(runErr); ok { + res.ExitCode = code + // The error text is the command's combined failure output — keep + // it where stderr-shaped text lives so callers can classify it. + if len(res.Stderr) == 0 { + res.Stderr = []byte(runErr.Error()) + } + return res + } + res.Stderr = diag.bytes() + res.Err = runErr + return res +} + +var exitStatusRe = regexp.MustCompile(`(?:^|: )exit status (\d+)`) + +// exitCodeFromError derives an exit code from a failure error: +// *exec.ExitError (local) and *gossh.ExitError (remote) carry it +// structurally; the string form "exit status N" (what Executor.Run's +// wrappers and MockExecutor registrations produce) is parsed as the +// documented mock shape. ok is false when the error represents no exit +// status at all (transport failure, cancellation). +func exitCodeFromError(err error) (int, bool) { + var execErr *exec.ExitError + if errors.As(err, &execErr) { + return execErr.ExitCode(), true + } + var sshErr *gossh.ExitError + if errors.As(err, &sshErr) { + return sshErr.Waitmsg.ExitStatus(), true + } + if m := exitStatusRe.FindStringSubmatch(err.Error()); m != nil { + var code int + if _, err := fmt.Sscanf(m[1], "%d", &code); err == nil { + return code, true + } + } + return -1, false +} + +// contextFailureResult classifies a context error into the +// TimedOut/Canceled flags. +func contextFailureResult(ctx context.Context) (Result, bool) { + err := ctx.Err() + if err == nil { + return Result{}, false + } + return Result{ + ExitCode: -1, + Canceled: errors.Is(err, context.Canceled), + TimedOut: errors.Is(err, context.DeadlineExceeded), + Err: err, + }, true +} + +// limitedBuffer keeps the first limit bytes written to it and records +// whether anything was dropped. +type limitedBuffer struct { + mu sync.Mutex + buf bytes.Buffer + limit int64 + overflow bool +} + +func (b *limitedBuffer) Write(p []byte) (int, error) { + b.mu.Lock() + defer b.mu.Unlock() + if b.limit <= 0 { + b.limit = DefaultResultLimit + } + room := b.limit - int64(b.buf.Len()) + if room <= 0 { + b.overflow = true + return len(p), nil // swallow, but report success to the writer + } + if int64(len(p)) > room { + b.overflow = true + p = p[:room] + } + return b.buf.Write(p) +} + +func (b *limitedBuffer) bytes() []byte { + b.mu.Lock() + defer b.mu.Unlock() + return append([]byte(nil), b.buf.Bytes()...) +} diff --git a/internal/ssh/result_test.go b/internal/ssh/result_test.go new file mode 100644 index 0000000..8b69939 --- /dev/null +++ b/internal/ssh/result_test.go @@ -0,0 +1,239 @@ +package ssh + +import ( + "bytes" + "context" + "errors" + "io" + "os" + "strings" + "testing" + "time" +) + +// TestRunDetailed_LocalExecutor pins the structured contract on the one +// executor tests can drive for real: stdout/stderr arrive SEPARATE and +// untrimmed, the exit code is the process's own, and a clean non-zero +// exit is a completed command (Err nil), not a transport failure. +func TestRunDetailed_LocalExecutor(t *testing.T) { + e := NewLocalExecutor() + res := RunDetailed(context.Background(), e, "printf out; printf err >&2; exit 42") + if res.Err != nil { + t.Fatalf("non-zero exit is not an Err: %v", res.Err) + } + if res.ExitCode != 42 { + t.Fatalf("ExitCode = %d, want 42", res.ExitCode) + } + if !bytes.Equal(res.Stdout, []byte("out")) { + t.Fatalf("Stdout = %q, want \"out\"", res.Stdout) + } + if !bytes.Equal(res.Stderr, []byte("err")) { + t.Fatalf("Stderr = %q, want \"err\"", res.Stderr) + } + if res.Truncated || res.Canceled || res.TimedOut { + t.Fatalf("no flag should be set: %+v", res) + } + if !res.Failed() { + t.Fatal("exit 42 must count as Failed") + } + if res.TrimmedStdout() != "out" { + t.Fatalf("TrimmedStdout = %q", res.TrimmedStdout()) + } +} + +func TestRunDetailed_LocalExecutor_Success(t *testing.T) { + e := NewLocalExecutor() + res := RunDetailed(context.Background(), e, "echo ok") + if res.Failed() || res.ExitCode != 0 || res.TrimmedStdout() != "ok" { + t.Fatalf("success shape: %+v %q", res, res.Stdout) + } +} + +// TestRunDetailed_LocalExecutor_TimedOutVsCanceled pins the flag +// distinction: a context deadline expiry sets TimedOut only, an explicit +// cancel sets Canceled only. Both kill the process group (A28) and +// return rather than hanging. +func TestRunDetailed_LocalExecutor_TimedOutVsCanceled(t *testing.T) { + if testing.Short() { + t.Skip("spawns real processes") + } + e := NewLocalExecutor() + + ctx, cancel := context.WithTimeout(context.Background(), 300*time.Millisecond) + res := RunDetailed(ctx, e, "sleep 5") + cancel() + if !res.TimedOut || res.Canceled { + t.Fatalf("deadline expiry: TimedOut=%v Canceled=%v, want true/false", res.TimedOut, res.Canceled) + } + if res.Err == nil || !errors.Is(res.Err, context.DeadlineExceeded) { + t.Fatalf("Err = %v, want DeadlineExceeded", res.Err) + } + if res.ExitCode != -1 { + t.Fatalf("ExitCode = %d, want -1 (no status delivered)", res.ExitCode) + } + + ctx2, cancel2 := context.WithCancel(context.Background()) + go func() { + time.Sleep(300 * time.Millisecond) + cancel2() + }() + res2 := RunDetailed(ctx2, e, "sleep 5") + cancel2() + if res2.TimedOut || !res2.Canceled { + t.Fatalf("explicit cancel: TimedOut=%v Canceled=%v, want false/true", res2.TimedOut, res2.Canceled) + } +} + +// TestRunDetailed_LocalExecutor_Truncated pins the output bound: output +// beyond the limit is dropped and flagged, never buffered. +func TestRunDetailed_LocalExecutor_Truncated(t *testing.T) { + e := NewLocalExecutor() + res := runDetailedWithLimit(context.Background(), e, "yes a", nil, 64) + if !res.Truncated { + t.Fatal("Truncated must be set when the capture limit is hit") + } + if int64(len(res.Stdout)) > 64 { + t.Fatalf("captured %d bytes, limit was 64", len(res.Stdout)) + } + if !strings.HasPrefix(string(res.Stdout), "a") { + t.Fatalf("the FIRST bytes must be kept, got %q", res.Stdout) + } +} + +// TestRunDetailed_LocalExecutor_CancellationKillsGroup extends the A28 +// pin to the structured path: canceling mid-run leaves no hidden child +// work — a descendant that would write a marker after the group kill +// must never write it. +func TestRunDetailed_LocalExecutor_CancellationKillsGroup(t *testing.T) { + if testing.Short() { + t.Skip("spawns real processes") + } + dir := t.TempDir() + marker := dir + "/alive" + e := NewLocalExecutor() + ctx, cancel := context.WithTimeout(context.Background(), 400*time.Millisecond) + defer cancel() + res := RunDetailed(ctx, e, "sh -c 'sleep 2 && touch "+marker+"' >/dev/null 2>&1; sleep 2") + if !res.Failed() { + t.Fatal("canceled run must report failure") + } + deadline := time.Now().Add(3 * time.Second) + for time.Now().Before(deadline) { + if _, err := os.Stat(marker); os.IsNotExist(err) { + return // child died with the group — pass + } + time.Sleep(50 * time.Millisecond) + } + t.Fatal("a descendant survived cancellation of the structured run") +} + +// TestRunInputDetailed_LocalExecutor pins stdin transport plus capture: +// stdin reaches the command, and the failure's stderr is available even +// on the RunInput path (whose legacy contract discards it). +func TestRunInputDetailed_LocalExecutor(t *testing.T) { + e := NewLocalExecutor() + res := RunInputDetailed(context.Background(), e, "cat; printf 'boom' >&2; exit 3", strings.NewReader("payload")) + if res.ExitCode != 3 || res.Err != nil { + t.Fatalf("status: %+v", res) + } + if string(res.Stdout) != "payload" { + t.Fatalf("stdin did not reach the command: %q", res.Stdout) + } + if string(res.Stderr) != "boom" { + t.Fatalf("stderr lost: %q", res.Stderr) + } +} + +// TestRunDetailed_MockExecutor pins the derivation rules tests rely on: +// a registered error carrying "exit status N" is a command failure with +// that code (Err nil, text in Stderr); an error without one is a +// transport failure (Err set); stdin payloads are recorded for the +// secret-transport assertions. +func TestRunDetailed_MockExecutor(t *testing.T) { + mock := NewMockExecutor("h", + MockCommand{Match: "echo fine", Output: "fine"}, + MockCommand{Match: "failing", Err: errors.New("exit status 75: TEPLOY_FENCE_LOST")}, + MockCommand{Match: "transport", Err: errors.New("ssh: connection timed out")}, + ) + + res := RunDetailed(context.Background(), mock, "echo fine") + if res.ExitCode != 0 || res.TrimmedStdout() != "fine" || res.Failed() { + t.Fatalf("success shape: %+v", res) + } + + res = RunDetailed(context.Background(), mock, "failing") + if res.ExitCode != 75 || res.Err != nil { + t.Fatalf("exit-status error shape: %+v", res) + } + if !bytes.Contains(res.Stderr, []byte("TEPLOY_FENCE_LOST")) { + t.Fatalf("failure text must land in Stderr: %q", res.Stderr) + } + + res = RunDetailed(context.Background(), mock, "transport") + if res.Err == nil || res.ExitCode != -1 { + t.Fatalf("transport shape: %+v", res) + } + + res = RunInputDetailed(context.Background(), mock, "echo fine", strings.NewReader("s3cret")) + if res.ExitCode != 0 { + t.Fatalf("input run: %+v", res) + } + if len(mock.Inputs) != 1 || mock.Inputs[0] != "s3cret" { + t.Fatalf("stdin payload not recorded: %v", mock.Inputs) + } +} + +// TestRunDetailed_FallbackExecutor pins the generic path used by test +// doubles that implement only the base Executor interface: output is +// captured, the exit code is derived when the error carries one, and an +// underivable error stays a transport failure. +func TestRunDetailed_FallbackExecutor(t *testing.T) { + fb := &fakeFallbackExecutor{output: "hello", err: nil} + res := RunDetailed(context.Background(), fb, "anything") + if res.ExitCode != 0 || res.TrimmedStdout() != "hello" { + t.Fatalf("fallback success: %+v %q", res, res.Stdout) + } + + // The string shape the base contract produces derives the code. + fb3 := &fakeFallbackExecutor{output: "x", err: errors.New("exit status 9: nope")} + res3 := RunDetailed(context.Background(), fb3, "anything") + if res3.ExitCode != 9 || res3.Err != nil { + t.Fatalf("fallback exit-status derivation: %+v", res3) + } + + fb4 := &fakeFallbackExecutor{err: errors.New("ssh: connection lost")} + res4 := RunDetailed(context.Background(), fb4, "anything") + if res4.Err == nil || res4.ExitCode != -1 { + t.Fatalf("fallback transport shape: %+v", res4) + } +} + +type fakeFallbackExecutor struct { + output string + err error + called int +} + +func (f *fakeFallbackExecutor) Run(ctx context.Context, cmd string) (string, error) { + f.called++ + return f.output, f.err +} + +func (f *fakeFallbackExecutor) RunStream(ctx context.Context, cmd string, stdout, stderr io.Writer) error { + if f.output != "" { + stdout.Write([]byte(f.output)) + } + return f.err +} + +func (f *fakeFallbackExecutor) RunInput(ctx context.Context, cmd string, stdin io.Reader) error { + return f.err +} + +func (f *fakeFallbackExecutor) Upload(ctx context.Context, content io.Reader, remotePath string, mode string) error { + return nil +} + +func (f *fakeFallbackExecutor) Close() error { return nil } +func (f *fakeFallbackExecutor) Host() string { return "fake" } +func (f *fakeFallbackExecutor) User() string { return "root" } diff --git a/internal/ssh/sshtest_test.go b/internal/ssh/sshtest_test.go new file mode 100644 index 0000000..42fe239 --- /dev/null +++ b/internal/ssh/sshtest_test.go @@ -0,0 +1,255 @@ +package ssh + +// sshtest_test.go — an in-process SSH server for pinning RemoteExecutor +// behavior without a network fixture: structured results (exit codes, +// separated streams), the bounded-command deadline, cancellation, and +// concurrent first-connect TOFU (C08). The server speaks the real +// x/crypto/ssh wire protocol over 127.0.0.1, so Connect's full path — +// dial, handshake, host-key callback, session — runs for real. + +import ( + "context" + "crypto/ed25519" + "crypto/rand" + "encoding/binary" + "encoding/pem" + "fmt" + "net" + "os" + "path/filepath" + "testing" + "time" + + gossh "golang.org/x/crypto/ssh" +) + +// testServerKey generates a fresh ed25519 SSH host key. +func testServerKey(t *testing.T) gossh.Signer { + t.Helper() + _, priv, err := ed25519.GenerateKey(rand.Reader) + if err != nil { + t.Fatalf("generating host key: %v", err) + } + signer, err := gossh.NewSignerFromKey(priv) + if err != nil { + t.Fatalf("wrapping host key: %v", err) + } + return signer +} + +// writeClientKeyFile generates a client identity and writes the private +// key PEM where resolveSigners can load it. +func writeClientKeyFile(t *testing.T) string { + t.Helper() + _, priv, err := ed25519.GenerateKey(rand.Reader) + if err != nil { + t.Fatalf("generating client key: %v", err) + } + block, err := gossh.MarshalPrivateKey(priv, "") + if err != nil { + t.Fatalf("marshaling client key: %v", err) + } + path := filepath.Join(t.TempDir(), "id_ed25519") + if err := os.WriteFile(path, pem.EncodeToMemory(block), 0600); err != nil { + t.Fatalf("writing client key: %v", err) + } + return path +} + +// sshTestHandler runs one exec request: writes to ch are the command's +// stdout; the returned int is its exit status. +type sshTestHandler func(cmd string, ch gossh.Channel) int + +// startTestSSHServer runs an SSH server on 127.0.0.1:0 that accepts any +// public key and dispatches every exec request to handler. It returns +// the dial address. +func startTestSSHServer(t *testing.T, handler sshTestHandler) string { + t.Helper() + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("listening: %v", err) + } + cfg := &gossh.ServerConfig{ + PublicKeyCallback: func(conn gossh.ConnMetadata, key gossh.PublicKey) (*gossh.Permissions, error) { + return &gossh.Permissions{}, nil + }, + } + cfg.AddHostKey(testServerKey(t)) + + serve := func() { + for { + conn, err := listener.Accept() + if err != nil { + return + } + go handleTestSSHConn(conn, cfg, handler) + } + } + go serve() + t.Cleanup(func() { listener.Close() }) + return listener.Addr().String() +} + +func handleTestSSHConn(conn net.Conn, cfg *gossh.ServerConfig, handler sshTestHandler) { + sconn, chans, reqs, err := gossh.NewServerConn(conn, cfg) + if err != nil { + return + } + defer sconn.Close() + go gossh.DiscardRequests(reqs) + for newCh := range chans { + if newCh.ChannelType() != "session" { + newCh.Reject(gossh.UnknownChannelType, "session only") + continue + } + ch, requests, err := newCh.Accept() + if err != nil { + continue + } + go handleTestSSHSession(ch, requests, handler) + } +} + +func handleTestSSHSession(ch gossh.Channel, requests <-chan *gossh.Request, handler sshTestHandler) { + defer ch.Close() + for req := range requests { + if req.Type != "exec" { + if req.WantReply { + req.Reply(false, nil) + } + continue + } + // exec payload: uint32 length + command string. + cmd := string(req.Payload[4:]) + if req.WantReply { + req.Reply(true, nil) + } + status := 0 + if handler != nil { + status = handler(cmd, ch) + } + // exit-status payload: uint32 status, big-endian. + payload := make([]byte, 4) + binary.BigEndian.PutUint32(payload, uint32(status)) + ch.SendRequest("exit-status", false, payload) + return + } +} + +// connectTestServer dials the test server with the given extra config. +// Each test gets a fresh $HOME (no prior known_hosts) and connects with +// AcceptNewHost — the real TOFU enrollment path — so the host-key +// plumbing runs exactly as a first deploy would. +func connectTestServer(t *testing.T, addr string, mutate func(*ConnectConfig)) *RemoteExecutor { + t.Helper() + t.Setenv("HOME", t.TempDir()) + keyPath := writeClientKeyFile(t) + cfg := ConnectConfig{Host: addr, User: "root", KeyPath: keyPath, AcceptNewHost: true} + if mutate != nil { + mutate(&cfg) + } + exec, err := Connect(context.Background(), cfg) + if err != nil { + t.Fatalf("connecting to test server: %v", err) + } + t.Cleanup(func() { exec.Close() }) + return exec +} + +// TestRemoteExecutor_RunDetailed_StatusPins drives the REAL session +// path: exit code arrives from the server's exit-status request, stdout +// and stderr are separated, and a non-zero exit is a completed command +// (Err nil). +func TestRemoteExecutor_RunDetailed_StatusPins(t *testing.T) { + if testing.Short() { + t.Skip("network + processes") + } + addr := startTestSSHServer(t, func(cmd string, ch gossh.Channel) int { + if cmd == "failcode" { + fmt.Fprint(ch, "partial out") + fmt.Fprint(ch.Stderr(), "the error") + return 42 + } + fmt.Fprint(ch, "hello") + return 0 + }) + exec := connectTestServer(t, addr, nil) + + res := RunDetailed(context.Background(), exec, "ok") + if res.ExitCode != 0 || res.Err != nil || string(res.Stdout) != "hello" { + t.Fatalf("success shape: %+v %q", res, res.Stdout) + } + + res = RunDetailed(context.Background(), exec, "failcode") + if res.ExitCode != 42 || res.Err != nil { + t.Fatalf("failure shape: %+v (exit code must arrive, Err must be nil)", res) + } + if string(res.Stdout) != "partial out" || string(res.Stderr) != "the error" { + t.Fatalf("streams crossed: stdout=%q stderr=%q", res.Stdout, res.Stderr) + } +} + +// TestRemoteExecutor_CommandTimeoutBoundsHungCommand pins C08's bounded +// subprocess lifetime on the remote path: with CommandTimeout set, a +// command that never finishes dies at the deadline — RunDetailed +// returns with TimedOut set instead of hanging the CLI forever. +func TestRemoteExecutor_CommandTimeoutBoundsHungCommand(t *testing.T) { + if testing.Short() { + t.Skip("network + processes") + } + addr := startTestSSHServer(t, func(cmd string, ch gossh.Channel) int { + time.Sleep(30 * time.Second) // hung past any test patience + return 0 + }) + exec := connectTestServer(t, addr, func(c *ConnectConfig) { c.CommandTimeout = 500 * time.Millisecond }) + + start := time.Now() + res := RunDetailed(context.Background(), exec, "hang") + elapsed := time.Since(start) + if !res.TimedOut { + t.Fatalf("deadline expiry must set TimedOut: %+v", res) + } + if res.Err == nil { + t.Fatal("timeout must surface an error") + } + if elapsed > 5*time.Second { + t.Fatalf("hung command outlived its deadline: %s", elapsed) + } + + // The connection remains usable after a timed-out command. + res2 := RunDetailed(context.Background(), exec, "hang") + if !res2.TimedOut { + t.Fatalf("second hung command must also be bounded: %+v", res2) + } +} + +// TestRemoteExecutor_CancelMidRun pins remote cancellation: canceling +// the context mid-command returns promptly with Canceled set and the +// session torn down (no hidden work continues on the connection's +// session; the server-side process is the server's to manage, but the +// client session is closed after SIGTERM as before). +func TestRemoteExecutor_CancelMidRun(t *testing.T) { + if testing.Short() { + t.Skip("network + processes") + } + addr := startTestSSHServer(t, func(cmd string, ch gossh.Channel) int { + time.Sleep(10 * time.Second) + return 0 + }) + exec := connectTestServer(t, addr, nil) + + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(300 * time.Millisecond) + cancel() + }() + start := time.Now() + res := RunDetailed(ctx, exec, "hang") + cancel() + if !res.Canceled || res.TimedOut { + t.Fatalf("explicit cancel: Canceled=%v TimedOut=%v, want true/false", res.Canceled, res.TimedOut) + } + if time.Since(start) > 5*time.Second { + t.Fatalf("cancellation did not return promptly: %s", time.Since(start)) + } +}