From e8409ed5ed2c517847b88464e3c12612edc7fd30 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 4 Oct 2026 03:26:44 +0000 Subject: [PATCH] S27: TeployAdapter over the existing delivery path, behind SHIP_DEPLOY_ADAPTER=teploy (default off) Implements DeploymentAdapter for Teploy (status --json, deploy, rollback --to, exec image inspect) and an optional adapter path in executeDelivery and executeDeliveryRollback. Legacy path is untouched when the flag is off. Capabilities the CLI cannot honestly provide are declared in capabilities().unsupported and the dependent conformance checks report skipped. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01VqsBNqvaWezf1DwQrAnVgX --- src/delivery.ts | 102 ++++++ src/deployment-adapter.ts | 6 + src/fixtures/fake-teploy.mjs | 139 +++++++++ src/teploy-adapter.test.ts | 586 +++++++++++++++++++++++++++++++++++ src/teploy-adapter.ts | 425 +++++++++++++++++++++++++ src/worker.ts | 9 + 6 files changed, 1267 insertions(+) create mode 100644 src/fixtures/fake-teploy.mjs create mode 100644 src/teploy-adapter.test.ts create mode 100644 src/teploy-adapter.ts diff --git a/src/delivery.ts b/src/delivery.ts index 8bd9ea3..163ee52 100644 --- a/src/delivery.ts +++ b/src/delivery.ts @@ -29,6 +29,8 @@ import type { UserStore } from "./users.js"; import { DEFAULT_WINDOW_MINUTES, readServiceHealth, telemetryAppliesTo, telemetryTargetFromEnv } from "./observe.js"; import type { TelemetryTarget } from "./observe.js"; import { repositoryKeyFold } from "./repository-reference.js"; +import { makeRecoveryPlan, runDeliveryJourney, runRecovery } from "./deployment-adapter.js"; +import type { DeploymentAdapter } from "./deployment-adapter.js"; /** What a delivery record holds. Additive only; fields grow, never rename. */ export interface DeliveryRecord { @@ -524,6 +526,48 @@ export async function recheckApprovingActor( }; } +/** + * The opt-in alternative to the inline teploy commands (S27, SHIP_DEPLOY_ADAPTER + * =teploy): a factory for the DeploymentAdapter the deploy step and the + * rollback run through. Absent (the default) leaves every line below on the + * legacy path. `unavailable` is an honest hold, e.g. no service in teploy.yml. + */ +export interface DeliveryAdapterFactory { + create(input: { deployDir: string; destination: string }): Promise<{ adapter: DeploymentAdapter; service: string } | { unavailable: string }>; +} + +/** A deploy that went through the adapter journey: `acted` means the target may have changed. */ +async function deployViaAdapter( + record: DeliveryRecord, + image: string, + destination: string, + factory: DeliveryAdapterFactory, + deployDir: string, + now: string, +): Promise<{ acted: boolean; reason: string }> { + const made = await factory.create({ deployDir, destination }); + if ("unavailable" in made) return { acted: false, reason: `deployment adapter unavailable: ${made.unavailable}` }; + const identity = { service: made.service, destination, artifact: image, revision: record.mergedSha!.slice(0, 7) }; + const journey = await runDeliveryJourney(made.adapter, { + identity, + authorisation: { + kind: "deploy", + actor: record.actor ?? "", + service: made.service, + destination, + expiresAt: new Date(Date.now() + 30 * 60_000).toISOString(), + }, + ...(record.actor !== undefined ? { holder: record.actor } : {}), + }); + const adapterName = made.adapter.capabilities().adapter; + return { + acted: journey.acted, + reason: journey.outcome === "confirmed" + ? `deployment adapter (${adapterName}) deployed and read the target back on the approved version and image; the sweep records the confirmation` + : `deployment adapter (${adapterName}) ${journey.outcome}: ${journey.reason}${journey.acted ? " — the target may have changed; reading it back before deciding" : ""}`, + }; +} + /** * Execute one approved delivery against the TRUSTED working copy. This is * the S14 execution boundary; it runs on the worker host via argv arrays @@ -542,6 +586,8 @@ export async function executeDelivery( now?: () => string; /** The live governance/account stores for the S15 actor recheck. */ authority?: ApproveAuthoritySources; + /** S27 opt-in: deploy through a DeploymentAdapter instead of the inline `teploy deploy`. Absent = legacy. */ + adapter?: DeliveryAdapterFactory; }, ): Promise { const now = options.now?.() ?? new Date().toISOString(); @@ -566,6 +612,9 @@ export async function executeDelivery( if (record.mergedSha === undefined) { return { ...record, ...patch({ reason: "the merged SHA was never proven (read-back did not confirm the merge); re-verify before delivery" }) }; } + if (options.adapter !== undefined && (record.destination === undefined || record.destination === "")) { + return { ...record, ...patch({ reason: "the deployment adapter path needs the destination recorded at approval (none was); re-approve naming the destination" }) }; + } const cwd = options.dir; const exec = async (argv: string[], timeoutMs = 120_000) => options.run(argv, { cwd, timeoutMs }); @@ -679,6 +728,13 @@ export async function executeDelivery( await exec(["git", "worktree", "remove", "--force", tree]).catch(() => undefined); return { ...record, ...patch({ reason: `the trusted build printed no image identity: ${build.stdout.slice(0, 300)}` }) }; } + if (options.adapter !== undefined) { + const via = await deployViaAdapter(record, image, record.destination!, options.adapter, tree, now); + await exec(["git", "worktree", "remove", "--force", tree]).catch(() => undefined); + // Held means "never touched the target": only a journey that did not act. + if (!via.acted) return { ...record, ...patch({ reason: via.reason }) }; + return { ...record, artifactDigest: image, ...(configIdentity !== undefined ? { configIdentity } : {}), state: "unknown", updatedAt: now, reason: via.reason }; + } const deployed = await options.run( ["teploy", "deploy", "--image", image, "--version", record.mergedSha.slice(0, 7), "--skip-dns-check"], { cwd: tree, timeoutMs: 900_000 }, @@ -1005,6 +1061,8 @@ export async function executeDeliveryRollback( now?: () => string; /** The live governance/account stores for the requester recheck. */ authority?: ApproveAuthoritySources; + /** S27 opt-in: roll back through a DeploymentAdapter instead of the inline commands. Absent = legacy. */ + adapter?: DeliveryAdapterFactory; }, ): Promise> { const now = options.now?.() ?? new Date().toISOString(); @@ -1024,6 +1082,9 @@ export async function executeDeliveryRollback( if (recheck.outcome === "hold") { return failed(`rollback requester recheck: ${recheck.reason}`); } + if (options.adapter !== undefined) { + return rollbackViaAdapter(record, options.dir, options.adapter, now, failed); + } const before = await options.run(["teploy", "status", "--json"], { cwd: options.dir, timeoutMs: 120_000 }); if (before.code !== 0) { return failed(`the target could not be read before rollback (exit ${before.code}): ${(before.stderr || before.stdout).slice(0, 300)} — refusing to act on an unread target`); @@ -1078,6 +1139,47 @@ export async function executeDeliveryRollback( } } +/** + * The adapter-path rollback: the same refusals as the inline path (an unread + * target, a target that no longer runs THIS delivery, an already-recovered + * target), decided by runRecovery over the adapter's own readback. + */ +async function rollbackViaAdapter( + record: DeliveryRecord, + dir: string, + factory: DeliveryAdapterFactory, + now: string, + failed: (evidence: string) => NonNullable, +): Promise> { + const destination = record.destination ?? ""; + if (destination === "" || record.mergedSha === undefined || record.artifactDigest === undefined) { + return failed("the deployment adapter path needs the delivery's destination, merged SHA and artifact digest, and one is not recorded"); + } + const made = await factory.create({ deployDir: dir, destination }); + if ("unavailable" in made) return failed(`deployment adapter unavailable: ${made.unavailable}`); + const recoveryVersion = record.recoveryVersion!; + const delivered = { service: made.service, destination, artifact: record.artifactDigest, revision: record.mergedSha.slice(0, 7) }; + const done = (evidence: string): NonNullable => ({ + ...(record.rollback ?? { actor: "", reason: "", requestedAt: now }), + state: "done", + finishedAt: now, + evidence: evidence.slice(0, 500), + }); + const read = await made.adapter.readback(delivered); + if (read.kind === "ok" && read.value.serving && read.value.revision === recoveryVersion) { + return done(`target read back: retained version ${recoveryVersion} serving (already — no rollback command was needed)`); + } + const plan = await makeRecoveryPlan(made.adapter, delivered, "rollback-to-version", { revision: recoveryVersion }, () => new Date(now)); + if (plan.kind !== "ok") return failed(`rollback plan: ${plan.reason}`); + const actor = record.rollback?.actor ?? ""; + const result = await runRecovery(made.adapter, { + plan: plan.value, + authorisation: { kind: "recover", actor, service: made.service, destination, expiresAt: new Date(Date.now() + 30 * 60_000).toISOString() }, + }); + if (result.outcome === "recovered") return done(`target read back: retained version ${recoveryVersion} serving (${result.reason})`); + return failed(`rollback ${result.outcome}: ${result.reason}`); +} + /** * A record stuck in `executing` past the staleness window (default 35 min — * deliberately above the 2×900 s execution ceiling) is a worker that died diff --git a/src/deployment-adapter.ts b/src/deployment-adapter.ts index 1647009..5991a1c 100644 --- a/src/deployment-adapter.ts +++ b/src/deployment-adapter.ts @@ -76,6 +76,12 @@ export interface AdapterCapabilities { fencing: readonly FencingMode[]; /** Recovery modes actually supported. Anything else must be refused, never approximated. */ recovery: readonly RecoveryMode[]; + /** + * What this adapter knows it cannot honestly provide, stated so a report can + * list it (an adapter that cannot do X must say so, never approximate X). + * Additive and optional: adapters that predate it declare nothing. + */ + unsupported?: readonly { capability: string; reason: string }[]; } // ------------------------------------------------------------------ authority diff --git a/src/fixtures/fake-teploy.mjs b/src/fixtures/fake-teploy.mjs new file mode 100644 index 0000000..20271f2 --- /dev/null +++ b/src/fixtures/fake-teploy.mjs @@ -0,0 +1,139 @@ +#!/usr/bin/env node +// A FAKE `teploy` CLI for teploy-adapter.test.ts. It emulates only the four +// commands the adapter issues (`status --json`, `deploy`, `rollback --to`, +// `exec -- docker image inspect `) over a JSON state file named +// by FAKE_TEPLOY_STATE. Nothing here talks to a network, a docker daemon or a +// real teploy binary. +// +// The status shape is the one captured live from the real CLI on 2026-09-22 +// (see delivery.test.ts): top-level `app` and `server`, snake_case +// `state.current_hash`, and containers whose fields are the Go struct's +// capitalised ID/Name/Image/State. The faults a test can arm live in the same +// state file so the test harness can flip them between calls: +// unreadable: "exit" | "garbage" status fails / prints non-JSON +// wrongApp: "" status describes another app +// hideArtifact: true containers report an image ID and `exec` +// cannot resolve it (the artifact is hidden) +// idForm: true containers report an image ID; `exec` +// resolves it through imageTags +// pending: {when, change} a rival change lands at the START of the +// next deploy ("before-deploy") or just +// after it applied ("after-deploy") +// failDeploy: "exit-before" | "exit-after-apply" +import { appendFileSync, readFileSync, writeFileSync } from "node:fs"; +import { randomBytes } from "node:crypto"; + +const path = process.env.FAKE_TEPLOY_STATE; +if (!path) { + process.stderr.write("fake teploy: FAKE_TEPLOY_STATE is not set\n"); + process.exit(2); +} +const load = () => JSON.parse(readFileSync(path, "utf8")); +const save = (s) => writeFileSync(path, JSON.stringify(s, null, 2)); +const log = (line) => appendFileSync(`${path}.calls`, `${line}\n`); + +const imageFor = (s, image) => { + if (!s.idForm && !s.hideArtifact) return image; + const id = randomBytes(6).toString("hex"); + s.imageTags = { ...(s.imageTags ?? {}), [id]: [image] }; + return id; +}; + +/** Make the target serve (hash, image) with a NEW container, as a real redeploy does. */ +const land = (s, hash, image) => { + s.current_hash = hash; + s.containers = [{ ID: randomBytes(6).toString("hex"), Name: `${s.app}-web-${hash}`, Image: imageFor(s, image), State: "running" }]; + s.releases = [...(s.releases ?? []).filter((r) => r.hash !== hash), { hash, image }]; +}; + +const applyChange = (s, change) => { + if (change.unreadable) s.unreadable = "exit"; + else land(s, change.revision, change.artifact); +}; + +const [cmd, ...args] = process.argv.slice(2); +const flag = (name) => { + const i = args.indexOf(name); + return i >= 0 ? args[i + 1] : undefined; +}; +const s = load(); +log(`${cmd} ${args.join(" ")}`); + +if (cmd === "status") { + if (s.unreadable === "exit") { + process.stderr.write("ssh: connect to host: connection refused\n"); + process.exit(1); + } + if (s.unreadable === "garbage") { + process.stdout.write("502 bad gateway"); + process.exit(0); + } + process.stdout.write(JSON.stringify({ + app: s.wrongApp ?? s.app, + server: s.server, + state: { current_hash: s.current_hash ?? "", schema_version: 2 }, + containers: s.containers ?? [], + })); + process.exit(0); +} + +if (cmd === "deploy") { + if (s.pending?.when === "before-deploy") { + applyChange(s, s.pending.change); + s.pending = null; + } + if (s.failDeploy === "exit-before") { + save(s); + process.stderr.write("deploy: build cache unavailable\n"); + process.exit(1); + } + const image = flag("--image"); + const version = flag("--version"); + if (!image || !version) { + save(s); + process.stderr.write("deploy: --image and --version are required\n"); + process.exit(2); + } + // A real deploy takes seconds; the pause is what lets a test see two + // overlapping deploys if the adapter's caller failed to serialise them. + await new Promise((r) => setTimeout(r, 25)); + land(s, version, image); + if (s.pending?.when === "after-deploy") { + applyChange(s, s.pending.change); + s.pending = null; + } + save(s); + if (s.failDeploy === "exit-after-apply") { + process.stderr.write("deploy: timed out waiting for health check\n"); + process.exit(1); + } + process.stdout.write("Deployed\n"); + process.exit(0); +} + +if (cmd === "rollback") { + const to = flag("--to"); + const release = (s.releases ?? []).find((r) => r.hash === to); + if (!release) { + process.stderr.write(`rollback: version ${to} is not retained\n`); + process.exit(1); + } + land(s, release.hash, release.image); + save(s); + process.stdout.write(`Rolled back to ${to}\n`); + process.exit(0); +} + +if (cmd === "exec") { + // exec -- docker image inspect + const name = args[args.length - 1]; + if (s.hideArtifact || s.unreadable === "exit") { + process.stderr.write("exec: ssh: connection refused\n"); + process.exit(1); + } + process.stdout.write(JSON.stringify([{ Id: `sha256:${name}`, RepoTags: (s.imageTags ?? {})[name] ?? [] }])); + process.exit(0); +} + +process.stderr.write(`fake teploy: unsupported command ${cmd}\n`); +process.exit(2); diff --git a/src/teploy-adapter.test.ts b/src/teploy-adapter.test.ts new file mode 100644 index 0000000..fd19114 --- /dev/null +++ b/src/teploy-adapter.test.ts @@ -0,0 +1,586 @@ +import assert from "node:assert/strict"; +import { execFile } from "node:child_process"; +import { mkdir, mkdtemp, readFile, rm, stat, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import { test } from "node:test"; +import { fileURLToPath } from "node:url"; + +import type { CommandRunner } from "./deploy.js"; +import { + CONFORMANCE_CHECK_NAMES, + DestinationLocks, + conformanceSuite, + makeRecoveryPlan, + runDeliveryJourney, + runRecovery, + type AdapterHarness, + type Authorisation, + type DeliveryIdentity, + type Ok, + type RecoveryPlan, + type TargetChange, + type TargetRef, + type TargetState, +} from "./deployment-adapter.js"; +import { executeDelivery, executeDeliveryRollback, readBackDelivery, type DeliveryRecord } from "./delivery.js"; +import { DEFAULT_LEASE_DIRNAME, DEPLOY_ADAPTER_ENV, TEPLOY_UNSUPPORTED, TeployAdapter, deployAdapterFlagOn, serviceFromTeployYml, teployDeliveryAdapterFactory, type TeployAdapterOptions } from "./teploy-adapter.js"; + +const FAKE = join(dirname(fileURLToPath(import.meta.url)), "..", "src", "fixtures", "fake-teploy.mjs"); + +const T: TargetRef = { service: "svc-a", destination: "prod" }; +const R0 = { revision: "rev-0", artifact: "img:0" }; +const R1 = { revision: "rev-1", artifact: "img:1" }; +const RIVAL = { revision: "rev-9", artifact: "img:9" }; +const ident = (r: { revision: string; artifact: string }): DeliveryIdentity => ({ ...T, ...r }); +const AUTH: Authorisation = { kind: "deploy", actor: "op@ship", ...T }; +const RAUTH: Authorisation = { ...AUTH, kind: "recover" }; + +interface World { + root: string; + dir: string; + stateFile: string; + calls: string[]; + stats: { deployCalls: number; maxInFlight: number; inFlight: number }; + run: CommandRunner; + adapter: TeployAdapter; + harness: AdapterHarness; + read(): Promise>; + arm(patch: Record): Promise; +} + +/** A real child process per command: the adapter's argv really reaches the fake CLI. */ +async function world(over: Partial = {}, seedState: Record = {}): Promise { + const root = await mkdtemp(join(tmpdir(), "teploy-adapter-")); + const dir = join(root, "trusted"); + await import("node:fs/promises").then((fs) => fs.mkdir(dir)); + await writeFile(join(dir, "teploy.yml"), "app: svc-a\nserver: 100.64.0.9\n"); + const stateFile = join(root, "state.json"); + await writeFile(stateFile, JSON.stringify({ app: "svc-a", server: "100.64.0.9", releases: [], containers: [], ...seedState })); + const calls: string[] = []; + const stats = { deployCalls: 0, maxInFlight: 0, inFlight: 0 }; + const run: CommandRunner = (argv, opts) => + new Promise((resolve) => { + calls.push(argv.join(" ")); + if (argv[0] !== "teploy") return resolve({ code: 127, stdout: "", stderr: `not allowed: ${argv[0]}` }); + const isDeploy = argv[1] === "deploy"; + if (isDeploy) { stats.deployCalls++; stats.inFlight++; stats.maxInFlight = Math.max(stats.maxInFlight, stats.inFlight); } + execFile(process.execPath, [FAKE, ...argv.slice(1)], { cwd: opts.cwd, timeout: opts.timeoutMs, env: { ...process.env, FAKE_TEPLOY_STATE: stateFile } }, (error, stdout, stderr) => { + if (isDeploy) stats.inFlight--; + const code = error === null ? 0 : typeof (error as { code?: unknown }).code === "number" ? (error as { code: number }).code : 1; + resolve({ code, stdout: String(stdout), stderr: String(stderr) }); + }); + }); + const adapter = new TeployAdapter({ ...T, dir, run, leaseDir: join(root, "leases"), ...over }); + const read = async () => JSON.parse(await readFile(stateFile, "utf8")) as Record; + const write = (s: Record) => writeFile(stateFile, JSON.stringify(s)); + const arm = async (patch: Record) => write({ ...(await read()), ...patch }); + const land = (s: Record, r: { revision: string; artifact: string }) => { + s.current_hash = r.revision; + let image = r.artifact; + if (s.idForm || s.hideArtifact) { + image = `${Math.random().toString(16).slice(2, 14).padEnd(12, "0")}`; + s.imageTags = { ...(s.imageTags ?? {}), [image]: [r.artifact] }; + } + s.containers = [{ ID: `c${Math.random().toString(16).slice(2, 10)}`, Name: `svc-a-web-${r.revision}`, Image: image, State: "running" }]; + s.releases = [...s.releases.filter((x: { hash: string }) => x.hash !== r.revision), { hash: r.revision, image: r.artifact }]; + }; + const harness: AdapterHarness = { + adapter, + seed: async (_t, state) => { const s = await read(); land(s, state); await write(s); }, + truth: async () => { + const s = await read(); + const running = (s.containers as Array<{ Image: string; State: string }>).filter((c) => c.State === "running"); + return running.length === 0 ? { serving: false } : { serving: true, revision: s.current_hash, artifact: running[0]!.Image }; + }, + outOfBand: async (_t, change: TargetChange) => { + const s = await read(); + if ("unreadable" in change) s.unreadable = "exit"; else land(s, change); + await write(s); + }, + interfere: async (when, change) => arm({ pending: { when, change } }), + restoreReads: async () => arm({ unreadable: null }), + stats: () => ({ deployCalls: stats.deployCalls, maxInFlightDeploys: stats.maxInFlight }), + }; + return { root, dir, stateFile, calls, stats, run, adapter, harness, read, arm }; +} + +const okOf = (r: { kind: string }): V => { assert.equal(r.kind, "ok", JSON.stringify(r)); return (r as Ok).value; }; + +// ------------------------------------------------------------------ conformance + +/** The checks this adapter cannot honestly pass, each tied to a declared gap. */ +const EXPECTED_SKIPPED: Record = { + "deploy-fenced-on-generation": "fencing compare-and-set not declared", + "journey-refuses-stale-plan": "fencing compare-and-set not declared", + "logs-attribute-service": "capability logs not declared", + "observe-attributes-service": "capability observe not declared", + "no-readback-never-confirms": "adapter declares readback", +}; + +test("the 27-check conformance suite: every check passes or is skipped for a DECLARED gap, none fails", async () => { + const report = await conformanceSuite(async () => (await world()).harness); + assert.equal(report.adapter, "teploy"); + assert.equal(report.checks.length, CONFORMANCE_CHECK_NAMES.length); + assert.equal(CONFORMANCE_CHECK_NAMES.length, 27); + assert.deepEqual(report.failed, [], report.checks.filter((c) => c.status === "fail").map((c) => `${c.name}: ${c.detail}`).join("\n")); + assert.equal(report.conformant, true); + assert.deepEqual([...report.skipped].sort(), Object.keys(EXPECTED_SKIPPED).sort()); + for (const c of report.checks.filter((c) => c.status === "skipped")) { + assert.ok(c.detail.startsWith(EXPECTED_SKIPPED[c.name]!), `${c.name} skipped for an unexpected reason: ${c.detail}`); + } + // 22 passes, 5 skips: the skip is a report, never a pass. + assert.equal(report.checks.filter((c) => c.status === "pass").length, 22); + // The skipped checks line up with declarations, not silence. + const declared = new Set(TEPLOY_UNSUPPORTED.map((u) => u.capability)); + for (const needed of ["destination-fence-vs-manual-deploy", "provider-dry-run", "target-generation-token", "recovery-artifact-digest", "logs"]) { + assert.ok(declared.has(needed), `${needed} must be declared unsupported`); + } + assert.deepEqual((await world()).adapter.capabilities().unsupported, TEPLOY_UNSUPPORTED); +}); + +test("capability declaration: no dry-run, lease-only fencing, one recovery mode; no lease directory means no fencing at all", async () => { + const w = await world(); + const caps = w.adapter.capabilities(); + assert.equal(caps.dryRun, false); + assert.equal(caps.provisioning, false); + assert.deepEqual(caps.fencing, ["lease"]); + assert.deepEqual(caps.recovery, ["rollback-to-version"]); + const bare = (await world({ leaseDir: undefined })).adapter; + assert.deepEqual(bare.capabilities().fencing, []); + assert.equal(bare.acquireLease, undefined); +}); + +// ------------------------------------------------------------------ journey on the fake CLI + +test("a faithful deploy runs the legacy argv and is confirmed only by reading status back", async () => { + const w = await world(); + await w.harness.seed(T, R0); + const r = await runDeliveryJourney(w.adapter, { identity: ident(R1), authorisation: AUTH, locks: new DestinationLocks() }); + assert.equal(r.outcome, "confirmed", r.reason); + assert.ok(w.calls.includes("teploy deploy --image img:1 --version rev-1 --skip-dns-check"), w.calls.join("\n")); + assert.equal(r.plan?.validatedByProvider, false); + assert.equal((await w.harness.truth(T)).revision, "rev-1"); +}); + +test("dry run plans without any deploy command and says the provider did not validate it", async () => { + const w = await world(); + await w.harness.seed(T, R0); + const r = await runDeliveryJourney(w.adapter, { identity: ident(R1), authorisation: AUTH, dryRun: true, locks: new DestinationLocks() }); + assert.equal(r.outcome, "planned"); + assert.match(r.reason, /without provider validation/); + assert.equal(w.stats.deployCalls, 0); +}); + +test("unreadable status: the journey holds before acting (exit failure and non-JSON), and after a deploy it is unknown, never failed", async () => { + for (const mode of ["exit", "garbage"]) { + const w = await world(); + await w.harness.seed(T, R0); + await w.arm({ unreadable: mode }); + const before = await runDeliveryJourney(w.adapter, { identity: ident(R1), authorisation: AUTH, locks: new DestinationLocks() }); + assert.equal(before.outcome, "held", mode); + assert.equal(before.acted, false); + assert.equal(w.stats.deployCalls, 0, "an unread target must not be acted on"); + } + const w = await world(); + await w.harness.seed(T, R0); + await w.harness.interfere("after-deploy", { unreadable: true }); + const after = await runDeliveryJourney(w.adapter, { identity: ident(R1), authorisation: AUTH, locks: new DestinationLocks() }); + assert.equal(after.outcome, "unknown", after.reason); + assert.equal(after.acted, true); +}); + +test("wrong service: a status that names another app is unknown for inspect AND readback, and nothing is deployed", async () => { + const w = await world(); + await w.harness.seed(T, R0); + await w.arm({ wrongApp: "someone-elses-app" }); + for (const r of [await w.adapter.inspect(T), await w.adapter.readback(T)]) { + assert.equal(r.kind, "unknown"); + assert.match((r as { reason: string }).reason, /someone-elses-app.*wrong-service attribution refused/); + } + const j = await runDeliveryJourney(w.adapter, { identity: ident(R1), authorisation: AUTH, locks: new DestinationLocks() }); + assert.equal(j.outcome, "held"); + assert.equal(w.stats.deployCalls, 0); +}); + +test("an adapter bound to one service@destination refuses to answer for another", async () => { + const w = await world(); + assert.equal((await w.adapter.inspect({ service: "svc-b", destination: "prod" })).kind, "refused"); + assert.equal((await w.adapter.readback({ service: "svc-a", destination: "staging" })).kind, "refused"); + assert.equal((await w.adapter.acquireLease!({ service: "svc-b", destination: "prod" }, "x")).kind, "refused"); +}); + +test("hidden artifact: an image ID that cannot be resolved is unknown (a lost read), never a mismatch or a confirm", async () => { + const w = await world({}, { idForm: true }); + await w.harness.seed(T, R0); + await w.arm({ hideArtifact: true }); + const read = await w.adapter.readback(ident(R0)); + assert.equal(read.kind, "unknown"); + assert.match((read as { reason: string }).reason, /could not be resolved/); + const j = await runDeliveryJourney(w.adapter, { identity: ident(R1), authorisation: AUTH, locks: new DestinationLocks() }); + assert.notEqual(j.outcome, "confirmed"); + assert.notEqual(j.outcome, "failed"); +}); + +test("ID-form images resolve to the approved tag through teploy exec, and a different tag is a readable mismatch", async () => { + const w = await world({}, { idForm: true }); + await w.harness.seed(T, R0); + const same = okOf(await w.adapter.readback(ident(R0))); + assert.equal(same.artifact, "img:0"); + assert.ok(w.calls.some((c) => c.startsWith("teploy exec 100.64.0.9 -- docker image inspect")), "must resolve through the server teploy.yml names"); + const other = okOf(await w.adapter.readback(ident(R1))); + assert.equal(other.artifact, "img:0", "reports what the target runs, not what was asked"); + const j = await runDeliveryJourney(w.adapter, { identity: ident(R1), authorisation: AUTH, locks: new DestinationLocks() }); + assert.equal(j.outcome, "confirmed", j.reason); +}); + +test("a rival landing after the deploy applied is a hold on a readable mismatch, never a confirm", async () => { + const w = await world(); + await w.harness.seed(T, R0); + await w.harness.interfere("after-deploy", RIVAL); + const r = await runDeliveryJourney(w.adapter, { identity: ident(R1), authorisation: AUTH, locks: new DestinationLocks() }); + assert.equal(r.outcome, "held", r.reason); +}); + +test("racing deploys to one destination are serialised and leave the last confirmed release serving", async () => { + const w = await world(); + await w.harness.seed(T, R0); + const locks = new DestinationLocks(); + const [a, b] = await Promise.all([ + runDeliveryJourney(w.adapter, { identity: ident(R1), authorisation: AUTH, locks }), + runDeliveryJourney(w.adapter, { identity: ident(RIVAL), authorisation: AUTH, locks }), + ]); + assert.equal(w.stats.maxInFlight, 1); + assert.deepEqual([a.outcome, b.outcome], ["confirmed", "confirmed"]); + assert.equal((await w.harness.truth(T)).revision, "rev-9"); +}); + +test("deploy exit code is not the verdict: exit 1 after the swap is indeterminate and the journey reads it back as confirmed", async () => { + const w = await world(); + await w.harness.seed(T, R0); + await w.arm({ failDeploy: "exit-after-apply" }); + const r = await runDeliveryJourney(w.adapter, { identity: ident(R1), authorisation: AUTH, locks: new DestinationLocks() }); + assert.equal(r.outcome, "confirmed", r.reason); + assert.match(r.reason, /indeterminate/); +}); + +test("deploy exit 1 before anything changed: never confirmed, held (not failed: a non-zero exit is indeterminate, not proof of no change)", async () => { + const w = await world(); + await w.harness.seed(T, R0); + await w.arm({ failDeploy: "exit-before" }); + const r = await runDeliveryJourney(w.adapter, { identity: ident(R1), authorisation: AUTH, locks: new DestinationLocks() }); + assert.equal(r.outcome, "held", r.reason); + assert.equal((await w.harness.truth(T)).revision, "rev-0"); +}); + +// ------------------------------------------------------------------ the declared gaps, pinned + +test("pre-command compare: a change between plan and deploy() is rejected stale-generation and no deploy command runs", async () => { + const w = await world(); + await w.harness.seed(T, R0); + const base = okOf(await w.adapter.inspect(T)); + const plan = okOf(await w.adapter.plan(ident(R1), base)); + await w.harness.outOfBand(T, RIVAL); + const receipt = await w.adapter.deploy({ plan, authorisation: AUTH, expectGeneration: base.generation }); + assert.equal(receipt.outcome, "rejected"); + assert.equal(receipt.rejection, "stale-generation"); + assert.equal(w.stats.deployCalls, 0); + assert.equal((await w.harness.truth(T)).revision, "rev-9"); +}); + +test("DECLARED GAP: a rival landing inside the deploy command (after the compare) is NOT fenced; the rollback-less overwrite is the destination-fence gap", async () => { + // This pins the limitation `destination-fence-vs-manual-deploy` declares. If + // teploy ever grows a real destination fence, this test must start failing + // and the declaration and the skipped checks be revisited together. + const w = await world(); + await w.harness.seed(T, R0); + await w.harness.interfere("before-deploy", RIVAL); + const r = await runDeliveryJourney(w.adapter, { identity: ident(R1), authorisation: AUTH, locks: new DestinationLocks() }); + assert.equal(r.outcome, "confirmed", "the window is real: the deploy overwrote the rival and read back as the approved release"); + assert.ok(TEPLOY_UNSUPPORTED.some((u) => u.capability === "destination-fence-vs-manual-deploy")); +}); + +test("the lease is exclusive among Ship releases and is checked at deploy time", async () => { + const w = await world(); + await w.harness.seed(T, R0); + const a = okOf(await w.adapter.acquireLease!(T, "one")); + assert.equal((await w.adapter.acquireLease!(T, "two")).kind, "refused"); + // A journey while another holder has the lease holds without acting. + const held = await runDeliveryJourney(w.adapter, { identity: ident(R1), authorisation: AUTH, locks: new DestinationLocks() }); + assert.equal(held.outcome, "held"); + assert.equal(held.acted, false); + assert.equal(w.stats.deployCalls, 0); + // A deploy carrying a lease that is not the held one is rejected. + const base = okOf(await w.adapter.inspect(T)); + const plan = okOf(await w.adapter.plan(ident(R1), base)); + const forged = await w.adapter.deploy({ plan, authorisation: AUTH, expectGeneration: base.generation, lease: { ...a, token: "forged" } }); + assert.equal(forged.outcome, "rejected"); + assert.equal(w.stats.deployCalls, 0); + await w.adapter.releaseLease!(a); + assert.equal((await w.adapter.acquireLease!(T, "two")).kind, "ok"); +}); + +test("an expired lease can be reclaimed, an unreadable lease file is never broken, and release by a stale token does not free a newer lease", async () => { + let t = Date.parse("2026-10-04T00:00:00Z"); + const w = await world({ now: () => new Date(t), leaseTtlMs: 1000 }); + const first = okOf(await w.adapter.acquireLease!(T, "one")); + t += 2000; + const second = okOf(await w.adapter.acquireLease!(T, "two")); + await w.adapter.releaseLease!(first); // stale token: must not free `second` + assert.equal((await w.adapter.acquireLease!(T, "three")).kind, "refused"); + await w.adapter.releaseLease!(second); + const path = join(w.root, "leases", `${encodeURIComponent("svc-a@prod")}.lease`); + await writeFile(path, "not json"); + assert.equal((await w.adapter.acquireLease!(T, "four")).kind, "refused"); +}); + +test("deploy and recover refuse authority that is for another kind or destination, and run no command", async () => { + const w = await world(); + await w.harness.seed(T, R0); + const base = okOf(await w.adapter.inspect(T)); + const plan = okOf(await w.adapter.plan(ident(R1), base)); + const bad = await w.adapter.deploy({ plan, authorisation: { ...AUTH, destination: "elsewhere" }, expectGeneration: base.generation }); + assert.equal(bad.rejection, "unauthorised"); + const wrongKind = await w.adapter.deploy({ plan, authorisation: RAUTH, expectGeneration: base.generation }); + assert.equal(wrongKind.rejection, "unauthorised"); + assert.equal(w.stats.deployCalls, 0); +}); + +// ------------------------------------------------------------------ recovery + +test("rollback-to-version recovers through teploy rollback --to and verifies by readback (revision only)", async () => { + const w = await world(); + await w.harness.seed(T, R0); + const j = await runDeliveryJourney(w.adapter, { identity: ident(R1), authorisation: AUTH, locks: new DestinationLocks() }); + assert.equal(j.outcome, "confirmed"); + const plan = okOf(await makeRecoveryPlan(w.adapter, ident(R1), "rollback-to-version", { revision: "rev-0" })); + const r = await runRecovery(w.adapter, { plan, authorisation: RAUTH, locks: new DestinationLocks() }); + assert.equal(r.outcome, "recovered", r.reason); + assert.deepEqual(r.verifiedAgainst, ["revision"], "the recovery artifact digest is not recorded, so only the revision is verified"); + assert.ok(w.calls.includes("teploy rollback --to rev-0")); + assert.equal((await w.harness.truth(T)).revision, "rev-0"); +}); + +test("invalid recovery is refused: unsupported modes, an explicit-less target, a target that moved, an unretained version", async () => { + const w = await world(); + await w.harness.seed(T, R0); + await runDeliveryJourney(w.adapter, { identity: ident(R1), authorisation: AUTH, locks: new DestinationLocks() }); + const base = okOf(await w.adapter.readback(ident(R1))); + for (const mode of ["redeploy-previous-artifact", "roll-forward"] as const) { + const r = await runRecovery(w.adapter, { plan: { mode, target: T, delivered: ident(R1), to: { revision: "rev-0" }, baselineGeneration: base.generation, madeAt: "2026-10-04T00:00:00Z" }, authorisation: RAUTH, locks: new DestinationLocks() }); + assert.equal(r.outcome, "held", mode); + assert.equal(r.acted, false); + } + assert.equal((await makeRecoveryPlan(w.adapter, ident(R1), "rollback-to-version", { revision: "" })).kind, "refused"); + assert.ok(!w.calls.some((c) => c.startsWith("teploy rollback")), "no rollback command for refused plans"); + // An unretained version: the command fails, the receipt is indeterminate, the outcome is held (target unchanged), never recovered. + const plan = okOf(await makeRecoveryPlan(w.adapter, ident(R1), "rollback-to-version", { revision: "rev-404" })); + const r = await runRecovery(w.adapter, { plan, authorisation: RAUTH, locks: new DestinationLocks() }); + assert.notEqual(r.outcome, "recovered"); + assert.equal((await w.harness.truth(T)).revision, "rev-1"); + // A target that moved after the plan: held without acting. + const plan2 = okOf(await makeRecoveryPlan(w.adapter, ident(R1), "rollback-to-version", { revision: "rev-0" })); + await w.harness.outOfBand(T, RIVAL); + const before = w.calls.filter((c) => c.startsWith("teploy rollback")).length; + const moved = await runRecovery(w.adapter, { plan: plan2, authorisation: RAUTH, locks: new DestinationLocks() }); + assert.equal(moved.outcome, "held"); + assert.equal(w.calls.filter((c) => c.startsWith("teploy rollback")).length, before); + assert.equal((await w.harness.truth(T)).revision, "rev-9"); +}); + +// ------------------------------------------------------------------ observe, flag, service + +test("observe is declared only when bound, and only answers for its own service", async () => { + const w = await world({ observe: { service: "svc-a", read: async () => ({ health: "healthy", reason: "0 errors" }) } }); + assert.equal(w.adapter.capabilities().observe, true); + const mine = okOf<{ service: string; health: string }>(await w.adapter.observe!(T)); + assert.equal(mine.service, "svc-a"); + assert.equal((await w.adapter.observe!({ service: "svc-b", destination: "prod" })).kind, "unknown"); + const failing = await world({ observe: { service: "svc-a", read: async () => { throw new Error("observe down"); } } }); + assert.equal((await failing.adapter.observe!(T)).kind, "unknown"); + // With observe bound the whole suite still has no failure, and its observe check now runs. + const report = await conformanceSuite(async () => (await world({ observe: { service: "svc-a", read: async () => ({ health: "healthy", reason: "ok" }) } })).harness); + assert.deepEqual(report.failed, []); + assert.ok(!report.skipped.includes("observe-attributes-service")); +}); + +test("the opt-in flag is exactly SHIP_DEPLOY_ADAPTER=teploy and defaults off", () => { + assert.equal(DEPLOY_ADAPTER_ENV, "SHIP_DEPLOY_ADAPTER"); + assert.equal(deployAdapterFlagOn({}), false); + for (const v of ["", "1", "true", "TEPLOY", "teploy ", "kubernetes"]) assert.equal(deployAdapterFlagOn({ SHIP_DEPLOY_ADAPTER: v }), false, v); + assert.equal(deployAdapterFlagOn({ SHIP_DEPLOY_ADAPTER: "teploy" }), true); + assert.equal(DEFAULT_LEASE_DIRNAME, ".teploy-ship-leases"); +}); + +test("serviceFromTeployYml reads the app line as teploy does, and is absent when unreadable", async () => { + const w = await world(); + assert.equal(await serviceFromTeployYml(w.dir), "svc-a"); + await writeFile(join(w.dir, "teploy.yml"), 'app: "quoted-app"\nserver: x\n'); + assert.equal(await serviceFromTeployYml(w.dir), "quoted-app"); + assert.equal(await serviceFromTeployYml(join(w.root, "nowhere")), undefined); +}); + +// ------------------------------------------------------------------ through delivery.ts (the real call path) + +const IMAGE = "ship-delivery-abc123"; +const dRecord = (over: Partial = {}): DeliveryRecord => ({ + id: "run-s27", + runId: "run-s27", + repo: "http://forge.example:3000/Tyler/app.git", + mergedSha: "abc123def456", + destination: "prod", + recoveryVersion: "rev-0", + actor: "token", + state: "executing", + updatedAt: "2026-10-04T00:00:00.000Z", + ...over, +}); + +/** The vcs/cat/build plumbing around the fake CLI: only `teploy status|deploy|rollback|exec` reach it. */ +function plumbed(w: World): CommandRunner { + return async (argv, opts) => { + if (argv[0] === "teploy" && argv[1] === "build") { w.calls.push(argv.join(" ")); return { code: 0, stdout: JSON.stringify({ image: IMAGE }), stderr: "" }; } + if (argv[0] === "teploy") return w.run(argv, opts); + w.calls.push(argv.join(" ")); + if (argv[0] === "cat") return readFile(argv[1]!, "utf8").then((stdout) => ({ code: 0, stdout, stderr: "" }), () => ({ code: 1, stdout: "", stderr: "no such file" })); + if (argv[0] === "git" && argv[1] === "worktree" && argv[2] === "add") { await mkdir(argv[4]!, { recursive: true }); return { code: 0, stdout: "", stderr: "" }; } + if (argv[0] === "git" && argv[1] === "worktree" && argv[2] === "remove") { await rm(argv[4]!, { recursive: true, force: true }); return { code: 0, stdout: "", stderr: "" }; } + if (argv[0] === "git" && argv[1] === "symbolic-ref") return { code: 1, stdout: "", stderr: "no default branch" }; + return { code: 0, stdout: "", stderr: "" }; + }; +} +const now = () => "2026-10-04T00:00:00.000Z"; +const deployCall = (w: World): string[] => w.calls.filter((c) => c.startsWith("teploy deploy")); + +test("flag OFF (no adapter option): the legacy path runs the inline deploy, never touches a lease, and an explicit undefined adapter is identical", async () => { + const a = await world(); + await a.harness.seed(T, R0); + const legacy = await executeDelivery(dRecord(), { dir: a.dir, run: plumbed(a), now }); + const b = await world(); + await b.harness.seed(T, R0); + const explicit = await executeDelivery(dRecord(), { dir: b.dir, run: plumbed(b), now, adapter: undefined }); + assert.deepEqual(legacy, explicit); + assert.equal(legacy.state, "unknown"); + assert.equal(legacy.artifactDigest, IMAGE); + assert.equal(legacy.reason, "deployment command completed; target state not yet read back"); + assert.deepEqual(deployCall(a), [`teploy deploy --image ${IMAGE} --version abc123d --skip-dns-check`]); + // The legacy sequence is exactly the inline one: one status read, no re-inspects, no lease directory. + assert.equal(a.calls.filter((c) => c === "teploy status --json").length, 1); + await assert.rejects(stat(join(a.root, "leases")), /ENOENT/); + await assert.rejects(stat(join(a.dir, DEFAULT_LEASE_DIRNAME)), /ENOENT/); + // The env flag is read in the worker, never in delivery.ts. + assert.equal(deployAdapterFlagOn({}), false); +}); + +test("flag ON: the adapter journey sends the SAME deploy argv, the record stays unknown for the sweep, and the legacy read-back then confirms", async () => { + const legacyWorld = await world(); + await legacyWorld.harness.seed(T, R0); + await executeDelivery(dRecord(), { dir: legacyWorld.dir, run: plumbed(legacyWorld), now }); + + const w = await world(); + await w.harness.seed(T, R0); + const out = await executeDelivery(dRecord(), { dir: w.dir, run: plumbed(w), now, adapter: teployDeliveryAdapterFactory({ dir: w.dir, run: plumbed(w), leaseDir: join(w.root, "leases") }) }); + assert.equal(out.state, "unknown"); + assert.equal(out.artifactDigest, IMAGE); + assert.match(out.reason ?? "", /deployment adapter \(teploy\) deployed and read the target back/); + assert.deepEqual(deployCall(w), deployCall(legacyWorld), "the adapter must issue the byte-identical deploy command"); + assert.ok(w.calls.filter((c) => c === "teploy status --json").length > 1, "the adapter reads the target repeatedly (inspect, re-inspect, pre-command, readback)"); + // The sweep (legacy, untouched) reconciles the same record from the same target. + const read = await readBackDelivery({ ...dRecord({ state: "unknown" }), artifactDigest: IMAGE }, { dir: w.dir, run: plumbed(w), observe: {} }); + assert.equal(read.outcome, "confirmed"); + // The lease was released. + assert.equal((await w.adapter.acquireLease!(T, "after")).kind, "ok"); +}); + +test("flag ON refusals hold WITHOUT a deploy command: unreadable status, wrong service in teploy.yml, no app line, no recorded destination", async () => { + const factoryFor = (w: World) => teployDeliveryAdapterFactory({ dir: w.dir, run: plumbed(w), leaseDir: join(w.root, "leases") }); + const unreadable = await world(); + await unreadable.harness.seed(T, R0); + await unreadable.arm({ unreadable: "garbage" }); + const a = await executeDelivery(dRecord(), { dir: unreadable.dir, run: plumbed(unreadable), now, adapter: factoryFor(unreadable) }); + assert.equal(a.state, "held"); + assert.match(a.reason ?? "", /could not be inspected/); + assert.equal(deployCall(unreadable).length, 0); + + const wrong = await world(); + await wrong.harness.seed(T, R0); + await writeFile(join(wrong.dir, "teploy.yml"), "app: some-other-app\nserver: 100.64.0.9\n"); + const b = await executeDelivery(dRecord(), { dir: wrong.dir, run: plumbed(wrong), now, adapter: factoryFor(wrong) }); + assert.equal(b.state, "held"); + assert.match(b.reason ?? "", /wrong-service attribution refused/); + assert.equal(deployCall(wrong).length, 0); + + const noApp = await world(); + await noApp.harness.seed(T, R0); + await writeFile(join(noApp.dir, "teploy.yml"), "server: 100.64.0.9\n"); + const c = await executeDelivery(dRecord(), { dir: noApp.dir, run: plumbed(noApp), now, adapter: factoryFor(noApp) }); + assert.equal(c.state, "held"); + assert.match(c.reason ?? "", /deployment adapter unavailable: no `app:` line/); + assert.equal(deployCall(noApp).length, 0); + + const noDest = await world(); + const d = await executeDelivery(dRecord({ destination: undefined }), { dir: noDest.dir, run: plumbed(noDest), now, adapter: factoryFor(noDest) }); + assert.equal(d.state, "held"); + assert.match(d.reason ?? "", /needs the destination/); + assert.equal(noDest.calls.length, 0, "nothing at all ran before the destination check"); +}); + +test("flag ON: a rival lands after the deploy applied; the record is unknown (acted), never confirmed", async () => { + const w = await world(); + await w.harness.seed(T, R0); + await w.harness.interfere("after-deploy", RIVAL); + const out = await executeDelivery(dRecord(), { dir: w.dir, run: plumbed(w), now, adapter: teployDeliveryAdapterFactory({ dir: w.dir, run: plumbed(w), leaseDir: join(w.root, "leases") }) }); + assert.equal(out.state, "unknown"); + assert.doesNotMatch(out.reason ?? "", /read the target back on the approved version/); + assert.match(out.reason ?? "", /may have changed/); + const read = await readBackDelivery({ ...dRecord({ state: "unknown" }), artifactDigest: IMAGE }, { dir: w.dir, run: plumbed(w), observe: {} }); + assert.equal(read.outcome, "mismatch"); +}); + +test("rollback flag ON vs OFF: both restore the retained version; the adapter path refuses a target that moved or is unreadable and runs no rollback command", async () => { + const rolling = (): DeliveryRecord => dRecord({ state: "confirmed", artifactDigest: IMAGE, rollback: { state: "executing", actor: "token", reason: "bad", requestedAt: now() } }); + const prepare = async (): Promise => { + const w = await world(); + await w.harness.seed(T, R0); + await executeDelivery(dRecord(), { dir: w.dir, run: plumbed(w), now }); + return w; + }; + const rollbackCalls = (w: World): string[] => w.calls.filter((c) => c.startsWith("teploy rollback")); + const factoryFor = (w: World) => teployDeliveryAdapterFactory({ dir: w.dir, run: plumbed(w), leaseDir: join(w.root, "leases") }); + + const legacy = await prepare(); + const lr = await executeDeliveryRollback(rolling(), { dir: legacy.dir, run: plumbed(legacy), now }); + assert.equal(lr.state, "done", lr.evidence); + + const on = await prepare(); + const factory = factoryFor(on); + const r = await executeDeliveryRollback(rolling(), { dir: on.dir, run: plumbed(on), now, adapter: factory }); + assert.equal(r.state, "done", r.evidence); + assert.deepEqual(rollbackCalls(on), rollbackCalls(legacy)); + assert.equal((await on.harness.truth(T)).revision, "rev-0"); + // Retried after it landed: already on the retained version, no second command. + const again = await executeDeliveryRollback(rolling(), { dir: on.dir, run: plumbed(on), now, adapter: factory }); + assert.equal(again.state, "done"); + assert.match(again.evidence ?? "", /already/); + assert.equal(rollbackCalls(on).length, 1); + + // A newer delivery replaced this one: rolling back would undo it too. + const moved = await prepare(); + await moved.harness.outOfBand(T, RIVAL); + const m = await executeDeliveryRollback(rolling(), { dir: moved.dir, run: plumbed(moved), now, adapter: factoryFor(moved) }); + assert.equal(m.state, "failed"); + assert.equal(rollbackCalls(moved).length, 0); + assert.equal((await moved.harness.truth(T)).revision, "rev-9"); + + // Unreadable target: never acted on. + const dark = await prepare(); + await dark.arm({ unreadable: "exit" }); + const u = await executeDeliveryRollback(rolling(), { dir: dark.dir, run: plumbed(dark), now, adapter: factoryFor(dark) }); + assert.equal(u.state, "failed"); + assert.equal(rollbackCalls(dark).length, 0); + + // No digest recorded: the adapter path says so rather than guessing. + const noDigest = await executeDeliveryRollback(dRecord({ state: "confirmed", rollback: { state: "executing", actor: "token", reason: "x", requestedAt: now() } }), { dir: on.dir, run: plumbed(on), now, adapter: factory }); + assert.equal(noDigest.state, "failed"); + assert.match(noDigest.evidence ?? "", /artifact digest/); +}); diff --git a/src/teploy-adapter.ts b/src/teploy-adapter.ts new file mode 100644 index 0000000..039b1d5 --- /dev/null +++ b/src/teploy-adapter.ts @@ -0,0 +1,425 @@ +/** + * TeployAdapter (S27): the native Teploy implementation of `DeploymentAdapter`, + * over the same argv runner and the same `teploy` commands delivery.ts uses + * (`status --json`, `deploy --image --version --skip-dns-check`, `rollback + * --to`, `exec -- docker image inspect`). + * + * WHY it exists. deployment-adapter.ts states the delivery invariants once, + * provider-blind; this is the first provider to run under them. delivery.ts's + * hand-written copies of those rules stay the default path. This adapter is an + * ALTERNATIVE reached only through SHIP_DEPLOY_ADAPTER=teploy (see + * `deployAdapterFlagOn`), so nothing changes until an operator opts in. + * + * WHAT IT DOES NOT CLAIM. The capability declaration lists, in + * `unsupported`, what Teploy cannot do today: a destination-level fence that a + * hand-run `teploy deploy` respects, a provider-side dry-run, a target + * generation token, a recorded recovery artifact digest, logs (the `teploy + * logs` flags are not verified here), provisioning, and the recovery modes + * other than rollback-to-version. Each is declared, not approximated, and the + * conformance checks that depend on one report `skipped` rather than pass. + * + * Attribution is a property of every answer: the adapter is bound to ONE + * service@destination (the trusted copy's teploy.yml `app:` plus the approved + * destination) and `status --json`'s `app` must name that service, otherwise + * the answer is `unknown`, never a state for somebody else's service. + * + * `revision` is the version string teploy itself reports (`current_hash`, the + * 7-character short SHA delivery.ts deploys with); `artifact` is the image + * name the deploy was given. + */ +import { createHash, randomUUID } from "node:crypto"; +import { mkdir, readFile, unlink, writeFile } from "node:fs/promises"; +import { join } from "node:path"; + +import type { CommandRunner } from "./deploy.js"; +import type { DeliveryAdapterFactory } from "./delivery.js"; +import { + authorisationCovers, + ok, + refused, + unknown, + type AdapterCapabilities, + type AdapterResult, + type DeliveryIdentity, + type DeliveryPlan, + type DeployInput, + type DeployReceipt, + type DeploymentAdapter, + type HealthReading, + type Lease, + type RecoverInput, + type Refused, + type TargetRef, + type TargetState, +} from "./deployment-adapter.js"; + +/** The operator opt-in. Anything but the exact value `teploy` leaves the legacy path in force. */ +export const DEPLOY_ADAPTER_ENV = "SHIP_DEPLOY_ADAPTER"; +export function deployAdapterFlagOn(env: NodeJS.ProcessEnv = process.env): boolean { + return env[DEPLOY_ADAPTER_ENV] === "teploy"; +} + +/** Where leases live when the operator names no directory: inside the trusted copy. */ +export const DEFAULT_LEASE_DIRNAME = ".teploy-ship-leases"; + +export const TEPLOY_UNSUPPORTED: readonly { capability: string; reason: string }[] = [ + { + capability: "destination-fence-vs-manual-deploy", + reason: "the lease is a file on the Ship host: it serialises Ship's own releases but a human running `teploy deploy` by hand (or another host not sharing the lease directory) is not fenced, and the pre-command status re-read leaves a window", + }, + { capability: "provider-dry-run", reason: "no teploy command validates a deploy without acting, so plan() is a description and validatedByProvider is always false" }, + { capability: "target-generation-token", reason: "teploy status exposes no deploy generation; `generation` is a fingerprint of status (app, current_hash, container ID/image/state) that a same-hash same-container change would not move" }, + { capability: "recovery-artifact-digest", reason: "the recovery artifact is not recorded (version + a live container is the bar), so recovery is verified against the revision only" }, + { capability: "logs", reason: "the `teploy logs` invocation is not verified against the CLI, so no logs() is offered" }, + { capability: "provisioning", reason: "teploy deploys to an existing server; provisioning has its own authority and is not part of this adapter" }, + { capability: "recovery:redeploy-previous-artifact", reason: "\"previous\" is not a stable recovery target (rollbackDeploy)" }, + { capability: "recovery:roll-forward", reason: "no teploy roll-forward command exists; a redeploy of a known-good revision is a new delivery" }, +]; + +export interface TeployAdapterOptions { + /** The service (teploy.yml `app:`) this adapter is bound to. */ + service: string; + /** The destination label the delivery was approved for. */ + destination: string; + /** The trusted working copy: status/rollback/exec run here, and teploy.yml is read from here. */ + dir: string; + run: CommandRunner; + /** Where `teploy deploy` runs (the detached build tree in delivery.ts). Defaults to `dir`. */ + deployDir?: string; + /** Lease directory. Absent means NO lease and NO fencing declared (the orchestrator then refuses to deploy). */ + leaseDir?: string; + leaseTtlMs?: number; + now?: () => Date; + /** Optional telemetry binding. Absent means observe is not declared. */ + observe?: { service: string; read: () => Promise<{ health: HealthReading["health"]; reason: string }> }; +} + +interface StatusContainer { ID?: unknown; Name?: unknown; Image?: unknown; State?: unknown } +interface StatusDoc { app?: unknown; server?: unknown; state?: { current_hash?: unknown }; containers?: StatusContainer[] } + +const STATUS_TIMEOUT = 120_000; +const DEPLOY_TIMEOUT = 900_000; +const EXEC_TIMEOUT = 60_000; + +const idForm = (name: string): boolean => /^(sha256:)?[0-9a-f]{12,64}$/i.test(name); + +export class TeployAdapter implements DeploymentAdapter { + observe?: DeploymentAdapter["observe"]; + acquireLease?: DeploymentAdapter["acquireLease"]; + releaseLease?: DeploymentAdapter["releaseLease"]; + + private readonly options: TeployAdapterOptions; + private readonly now: () => Date; + /** Artifacts this adapter was asked to plan: used only to choose WHICH observed image name to report. */ + private readonly known = new Set(); + + constructor(options: TeployAdapterOptions) { + this.options = options; + this.now = options.now ?? (() => new Date()); + if (options.observe !== undefined) { + const binding = options.observe; + this.observe = async (target) => { + if (target.service !== binding.service) return unknown(`telemetry is bound to ${binding.service}, not ${target.service}`); + try { + const read = await binding.read(); + return ok({ service: binding.service, health: read.health, reason: read.reason }); + } catch (error) { + return unknown(`telemetry read failed: ${error instanceof Error ? error.message : String(error)}`); + } + }; + } + if (options.leaseDir !== undefined) { + this.acquireLease = (target, holder) => this.lease(target, holder); + this.releaseLease = (lease) => this.unlease(lease); + } + } + + capabilities(): AdapterCapabilities { + return { + adapter: "teploy", + readback: true, + logs: false, + observe: this.observe !== undefined, + dryRun: false, + provisioning: false, + fencing: this.options.leaseDir !== undefined ? ["lease"] : [], + recovery: ["rollback-to-version"], + unsupported: TEPLOY_UNSUPPORTED, + }; + } + + private bound(target: TargetRef): Refused | undefined { + if (target.service === this.options.service && target.destination === this.options.destination) return undefined; + return refused(`this adapter is bound to ${this.options.service}@${this.options.destination}, not ${target.service}@${target.destination}`); + } + + // -------------------------------------------------------------- reading + + /** The server as teploy.yml spells it (delivery.ts: exec by display name fails DNS in the worker). */ + private async execServer(doc: StatusDoc): Promise { + let server = typeof doc.server === "string" ? doc.server : ""; + try { + const yml = await readFile(join(this.options.dir, "teploy.yml"), "utf8"); + const m = /^server:\s*(["']?)([^\s"']+)\1\s*$/m.exec(yml); + if (m !== null) server = m[2]!; + } catch { + // keep the status-derived name; exec then fails and the read is unknown + } + return server; + } + + /** Resolve an image ID to its RepoTags through the same SSH channel the deploy used. */ + private async tagsOf(name: string, server: string): Promise { + if (server === "") return undefined; + const inspect = await this.options.run(["teploy", "exec", server, "--", "docker", "image", "inspect", name], { cwd: this.options.dir, timeoutMs: EXEC_TIMEOUT }); + if (inspect.code !== 0) return undefined; + try { + const docs = JSON.parse(inspect.stdout.trim()) as unknown; + if (!Array.isArray(docs) || docs.length === 0) return undefined; + const tags = (docs[0] as { RepoTags?: unknown }).RepoTags; + return Array.isArray(tags) ? tags.filter((t): t is string => typeof t === "string") : []; + } catch { + return undefined; + } + } + + private async status(): Promise> { + const read = await this.options.run(["teploy", "status", "--json"], { cwd: this.options.dir, timeoutMs: STATUS_TIMEOUT }); + if (read.code !== 0) return unknown(`target status could not be read (exit ${read.code}): ${(read.stderr || read.stdout).slice(0, 300)}`); + let doc: StatusDoc; + try { + const parsed = JSON.parse(read.stdout.trim()) as unknown; + if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) return unknown(`target status was not an object: ${read.stdout.slice(0, 300)}`); + doc = parsed as StatusDoc; + } catch { + return unknown(`target status was not JSON: ${read.stdout.slice(0, 300)}`); + } + if (typeof doc.app !== "string" || doc.app !== this.options.service) { + return unknown(`status describes app ${typeof doc.app === "string" ? doc.app : "(none)"}, not ${this.options.service}; wrong-service attribution refused`); + } + const containers = (doc.containers ?? []).map((c) => [String(c.ID ?? ""), String(c.Image ?? ""), String(c.State ?? "")]).sort((a, b) => a.join("|").localeCompare(b.join("|"))); + const hash = typeof doc.state?.current_hash === "string" ? doc.state.current_hash : ""; + const generation = `status:${createHash("sha256").update(JSON.stringify({ app: doc.app, hash, containers })).digest("hex").slice(0, 32)}`; + return ok({ doc, generation }); + } + + private async stateOf(hint?: string): Promise> { + const got = await this.status(); + if (got.kind !== "ok") return got; + const { doc, generation } = got.value; + const running = (doc.containers ?? []).filter((c) => c.State === "running"); + const hash = typeof doc.state?.current_hash === "string" ? doc.state.current_hash : ""; + const base = { service: this.options.service, destination: this.options.destination, generation }; + if (running.length === 0) { + return ok({ ...base, serving: false, ...(hash !== "" ? { revision: hash } : {}) }); + } + const wanted = new Set([...(hint !== undefined ? [hint] : []), ...this.known]); + const server = await this.execServer(doc); + const plain: string[] = []; + const unresolved: string[] = []; + let matched: string | undefined; + let firstTag: string | undefined; + for (const c of running) { + const name = typeof c.Image === "string" ? c.Image : ""; + if (name === "") continue; + if (wanted.has(name)) { matched = name; break; } + if (!idForm(name)) { plain.push(name); continue; } + const tags = await this.tagsOf(name, server); + if (tags === undefined) { unresolved.push(name); continue; } + const hit = [...wanted].find((w) => tags.some((t) => t === w || t.startsWith(`${w}:`))); + if (hit !== undefined) { matched = hit; break; } + firstTag ??= tags[0]; + } + // A lost read of an image ID is not evidence of a different artifact + // (delivery.ts's "unresolved" answer): unknown, so the caller retries. + if (matched === undefined && unresolved.length > 0 && plain.length === 0) { + return unknown(`running image(s) [${unresolved.join(", ")}] could not be resolved to tags through teploy exec on ${server === "" ? "(no server)" : server}`); + } + const artifact = matched ?? plain[0] ?? firstTag; + return ok({ ...base, serving: true, ...(hash !== "" ? { revision: hash } : {}), ...(artifact !== undefined ? { artifact } : {}) }); + } + + async inspect(target: TargetRef): Promise> { + const unbound = this.bound(target); + if (unbound !== undefined) return unbound; + return this.stateOf((target as Partial).artifact); + } + + async readback(target: TargetRef): Promise> { + const unbound = this.bound(target); + if (unbound !== undefined) return unbound; + // Callers pass the DeliveryIdentity they are verifying; its artifact only + // chooses which OBSERVED image name to report, never what is reported. + return this.stateOf((target as Partial).artifact); + } + + // -------------------------------------------------------------- acting + + async plan(identity: DeliveryIdentity, baseline: TargetState): Promise> { + const unbound = this.bound(identity); + if (unbound !== undefined) return unbound; + if (baseline.service !== identity.service || baseline.destination !== identity.destination) { + return refused("the baseline describes a different target than the one being planned"); + } + if (identity.artifact.trim() === "" || identity.revision.trim() === "") return refused("a teploy deploy needs both an image and a version"); + this.known.add(identity.artifact); + return ok({ + planId: `teploy-${identity.revision}-${baseline.generation.slice(-8)}`, + identity, + baseline: { generation: baseline.generation, ...(baseline.revision !== undefined ? { revision: baseline.revision } : {}), ...(baseline.artifact !== undefined ? { artifact: baseline.artifact } : {}) }, + steps: [`teploy deploy --image ${identity.artifact} --version ${identity.revision} --skip-dns-check`], + validatedByProvider: false, + }); + } + + private async leaseHeld(lease: Lease): Promise { + if (this.options.leaseDir === undefined) return false; + try { + const held = JSON.parse(await readFile(this.leasePath(lease.target), "utf8")) as { token?: unknown; expiresAt?: unknown }; + return held.token === lease.token && typeof held.expiresAt === "string" && Date.parse(held.expiresAt) > this.now().getTime(); + } catch { + return false; + } + } + + async deploy(input: DeployInput): Promise { + const { plan } = input; + const identity = plan.identity; + const unbound = this.bound(identity); + if (unbound !== undefined) return { outcome: "rejected", rejection: "other", detail: unbound.reason }; + const authority = authorisationCovers(input.authorisation, "deploy", identity, this.now()); + if (!authority.ok) return { outcome: "rejected", rejection: "unauthorised", detail: authority.reason }; + if (input.lease !== undefined && !(await this.leaseHeld(input.lease))) { + return { outcome: "rejected", rejection: "other", detail: "the destination lease is not held (expired, released or taken); not acting" }; + } + // Pre-command compare: narrows, but cannot close, the window between this + // read and the command (declared in `unsupported`). + const now = await this.status(); + if (now.kind !== "ok") return { outcome: "rejected", rejection: "other", detail: `the target could not be read before deploy (${now.reason}); not acting on an unread target` }; + if (now.value.generation !== input.expectGeneration) { + return { outcome: "rejected", rejection: "stale-generation", detail: "the target's status changed since the plan was made; nothing was run" }; + } + this.known.add(identity.artifact); + const ran = await this.options.run( + ["teploy", "deploy", "--image", identity.artifact, "--version", identity.revision, "--skip-dns-check"], + { cwd: this.options.deployDir ?? this.options.dir, timeoutMs: DEPLOY_TIMEOUT }, + ); + if (ran.code !== 0) { + // A non-zero exit may follow a swap (a timeout after the switch): never "rejected". + return { outcome: "indeterminate", detail: `teploy deploy exited ${ran.code}: ${(ran.stderr || ran.stdout).slice(0, 300)}; the target may have changed` }; + } + const after = await this.stateOf(identity.artifact); + if (after.kind === "ok" && after.value.serving && after.value.revision === identity.revision && after.value.artifact === identity.artifact) { + return { outcome: "applied", detail: "teploy deploy exited 0 and status shows the approved version and image" }; + } + return { outcome: "indeterminate", detail: `teploy deploy exited 0 but ${after.kind === "ok" ? `status shows ${after.value.revision ?? "(none)"} / ${after.value.artifact ?? "(no image)"}` : `status could not be read (${after.reason})`}` }; + } + + async recover(input: RecoverInput): Promise { + const { plan } = input; + const unbound = this.bound(plan.target); + if (unbound !== undefined) return { outcome: "rejected", rejection: "other", detail: unbound.reason }; + if (plan.mode !== "rollback-to-version") { + return { outcome: "rejected", rejection: "other", detail: `teploy supports only rollback-to-version, not ${plan.mode}; nothing was run` }; + } + const authority = authorisationCovers(input.authorisation, "recover", plan.target, this.now()); + if (!authority.ok) return { outcome: "rejected", rejection: "unauthorised", detail: authority.reason }; + if (plan.to.revision.trim() === "") return { outcome: "rejected", rejection: "other", detail: "recovery needs an explicit retained version" }; + if (input.lease !== undefined && !(await this.leaseHeld(input.lease))) { + return { outcome: "rejected", rejection: "other", detail: "the destination lease is not held; not acting" }; + } + const now = await this.status(); + if (now.kind !== "ok") return { outcome: "rejected", rejection: "other", detail: `the target could not be read before rollback (${now.reason})` }; + if (now.value.generation !== input.expectGeneration) { + return { outcome: "rejected", rejection: "stale-generation", detail: "the target's status changed since the recovery was planned; nothing was run" }; + } + const ran = await this.options.run(["teploy", "rollback", "--to", plan.to.revision], { cwd: this.options.dir, timeoutMs: DEPLOY_TIMEOUT }); + if (ran.code !== 0) { + return { outcome: "indeterminate", detail: `teploy rollback exited ${ran.code}: ${(ran.stderr || ran.stdout).slice(0, 300)}; the target may have changed` }; + } + const after = await this.stateOf(plan.to.artifact); + if (after.kind === "ok" && after.value.serving && after.value.revision === plan.to.revision) { + return { outcome: "applied", detail: `teploy rollback exited 0 and status shows ${plan.to.revision} serving` }; + } + return { outcome: "indeterminate", detail: `teploy rollback exited 0 but ${after.kind === "ok" ? `status shows ${after.value.revision ?? "(none)"}` : `status could not be read (${after.reason})`}` }; + } + + // -------------------------------------------------------------- lease + + private leasePath(target: TargetRef): string { + return join(this.options.leaseDir!, `${encodeURIComponent(`${target.service}@${target.destination}`)}.lease`); + } + + private async lease(target: TargetRef, holder: string): Promise> { + const unbound = this.bound(target); + if (unbound !== undefined) return unbound; + const path = this.leasePath(target); + const ttl = this.options.leaseTtlMs ?? 20 * 60_000; + const token = randomUUID(); + const body = (): string => JSON.stringify({ token, holder, expiresAt: new Date(this.now().getTime() + ttl).toISOString() }); + try { + await mkdir(this.options.leaseDir!, { recursive: true }); + for (let attempt = 0; attempt < 2; attempt++) { + try { + await writeFile(path, body(), { flag: "wx" }); + return ok({ target: { service: target.service, destination: target.destination }, token, holder }); + } catch (error) { + if ((error as { code?: unknown }).code !== "EEXIST") throw error; + } + let held: { holder?: unknown; expiresAt?: unknown }; + try { + held = JSON.parse(await readFile(path, "utf8")) as typeof held; + } catch { + return refused("a lease file exists and cannot be read; refusing to break it"); + } + const expires = typeof held.expiresAt === "string" ? Date.parse(held.expiresAt) : NaN; + if (!Number.isFinite(expires) || expires > this.now().getTime()) { + return refused(`the destination is leased${typeof held.holder === "string" ? ` by ${held.holder}` : ""}`); + } + await unlink(path).catch(() => undefined); // an expired lease: the holder's TTL ran out + } + return refused("the destination lease was taken while reclaiming an expired one"); + } catch (error) { + return unknown(`the lease directory could not be used: ${error instanceof Error ? error.message : String(error)}`); + } + } + + private async unlease(lease: Lease): Promise { + const path = this.leasePath(lease.target); + try { + const held = JSON.parse(await readFile(path, "utf8")) as { token?: unknown }; + if (held.token === lease.token) await unlink(path); + } catch { + // gone or unreadable: the TTL is the backstop + } + } +} + +/** The service a trusted copy serves: its teploy.yml `app:` line, as teploy itself reads it. */ +export async function serviceFromTeployYml(dir: string): Promise { + try { + const yml = await readFile(join(dir, "teploy.yml"), "utf8"); + const m = /^app:\s*(["']?)([^\s"']+)\1\s*$/m.exec(yml); + return m === null ? undefined : m[2]!; + } catch { + return undefined; + } +} + +/** + * The factory delivery.ts takes under SHIP_DEPLOY_ADAPTER=teploy: one adapter + * per use, bound to the trusted copy's service (its teploy.yml `app:`) and the + * approved destination. No readable `app:` means no adapter (an honest hold), + * never a guessed service. + */ +export function teployDeliveryAdapterFactory(options: { dir: string; run: CommandRunner; leaseDir?: string; env?: NodeJS.ProcessEnv }): DeliveryAdapterFactory { + const env = options.env ?? process.env; + return { + async create({ deployDir, destination }) { + const service = await serviceFromTeployYml(options.dir); + if (service === undefined) return { unavailable: `no \`app:\` line readable in ${join(options.dir, "teploy.yml")}, so the service this trusted copy serves is unknown` }; + const leaseDir = options.leaseDir ?? (env.SHIP_DEPLOY_LEASE_DIR !== undefined && env.SHIP_DEPLOY_LEASE_DIR !== "" ? env.SHIP_DEPLOY_LEASE_DIR : join(options.dir, DEFAULT_LEASE_DIRNAME)); + return { service, adapter: new TeployAdapter({ service, destination, dir: options.dir, run: options.run, deployDir, leaseDir }) }; + }, + }; +} diff --git a/src/worker.ts b/src/worker.ts index 060cd88..100a053 100644 --- a/src/worker.ts +++ b/src/worker.ts @@ -39,6 +39,7 @@ import { reconcileStaleRollback, unknownEscalation, } from "./delivery.js"; +import { deployAdapterFlagOn, teployDeliveryAdapterFactory } from "./teploy-adapter.js"; import { intakeActor } from "./actor.js"; import type { NucleusShipRuntime } from "./runtime.js"; import type { RunMeta } from "./run-store.js"; @@ -1929,6 +1930,12 @@ export function startWorker(options: WorkerOptions): { const deliveryDir = process.env.SHIP_DELIVERY_DIR; const deliveryStaleMs = Number(process.env.SHIP_DELIVERY_STALE_MS ?? "") > 0 ? Number(process.env.SHIP_DELIVERY_STALE_MS) : 35 * 60_000; const deliveryRunner: CommandRunner = hostRunner(); + // S27 opt-in (SHIP_DEPLOY_ADAPTER=teploy, default off): the deploy step and + // rollbacks go through the TeployAdapter journey instead of the inline commands. + const deliveryAdapter = deployAdapterFlagOn() && deliveryDir !== undefined && deliveryDir !== "" + ? { adapter: teployDeliveryAdapterFactory({ dir: deliveryDir, run: deliveryRunner }) } + : {}; + if ("adapter" in deliveryAdapter) log("[worker] delivery: SHIP_DEPLOY_ADAPTER=teploy — deploys and rollbacks run through the TeployAdapter"); /** * The telemetry binding a delivery's health verdict reads through: the * worker's env target plus the PROJECT's declared Observe service when the @@ -1958,6 +1965,7 @@ export function startWorker(options: WorkerOptions): { // S15: the approving actor's authority is rechecked against the // LIVE stores at execution — an approval cannot outlive its grant. authority: { governance: options.runtime.governance, users: options.runtime.users }, + ...deliveryAdapter, }); const to = outcome.state === "held" ? "held" : "unknown"; await records @@ -1985,6 +1993,7 @@ export function startWorker(options: WorkerOptions): { run: deliveryRunner, // The requester's authority is rechecked at action time, like an approval's. authority: { governance: options.runtime.governance, users: options.runtime.users }, + ...deliveryAdapter, }); await records.finishRollback(claimed.id, outcome); log(`[worker] delivery rollback ${claimed.id} → ${outcome.state}: ${outcome.evidence ?? ""}`);