diff --git a/docs/SUPPORT.md b/docs/SUPPORT.md index 5131798c..6cd4e224 100644 --- a/docs/SUPPORT.md +++ b/docs/SUPPORT.md @@ -62,6 +62,23 @@ rather than emit a bundle that silently lacks its core. The container-name filter defaults to the `ship-` prefix (the `app:` name in `teploy.yml`); `--out` can point anywhere writable. +## Checking a fresh install: `teploy-ship doctor` + +`teploy-ship doctor [--json] [--out ]` checks prerequisites before first +use: Node version, a writable state directory with enough free space, the web +port, `SHIP_WEB_TOKEN` and a model credential (presence only, values never +printed), the `NUCLEUS_URL` shape, store connectivity, and clock sanity. + +Every check reports `pass`, `fail` or `unknown`; **unknown is not pass**. The +verdict is `ready` only when all pass, `not-ready` on any fail, `incomplete` +when nothing failed but something could not be established. Exit code is 0 only +for `ready`. Passing the `store-url` shape check does not mean the store was +reached: `store-connectivity` stays `unknown` until a live check answers (use +`teploy-ship preflight`). `--out` writes the redacted JSON, suitable to attach +to a support request. The restore side is `src/restore-readiness.ts`, which +compares backup and restored history snapshots and reports `unverified` +whenever it did not actually compare; it is not yet exposed as a command. + ## The support policy shape Hand the `.tgz` to the vendor. The bundle is safe by construction — whitelist diff --git a/src/args.ts b/src/args.ts index 0963e627..483a2086 100644 --- a/src/args.ts +++ b/src/args.ts @@ -74,6 +74,7 @@ export const COMMAND_FLAGS: Record = { }, web: { boolean: ["dev"], value: ["port", "token", "model"] }, support: { value: ["out", "log-lines", "days"] }, + doctor: { boolean: ["json"], value: ["out"] }, eval: { boolean: ["critic", "settle"], value: ["model", "suite", "repeats"] }, }; diff --git a/src/cli.ts b/src/cli.ts index d6bf5aa7..636f021e 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -83,6 +83,7 @@ import { NucleusCodeIndex } from "./code-index.js"; import type { CodeSearch } from "./code-index.js"; import { startWorker } from "./worker.js"; import { assembleSupportBundle, defaultDocker } from "./support.js"; +import { defaultProbes, formatDoctor, renderDoctor, runDoctor } from "./install-doctor.js"; import { costUSD, isPricedModel } from "./pricing.js"; import { defaultRetryPolicy, withRetry, withCallTimeout, modelTimeoutFromEnv } from "./provider.js"; import { builtinSuite } from "./tasks.js"; @@ -197,6 +198,7 @@ Usage: teploy-ship web serve the runs dashboard (browser approve/deny) [--port N] [--token ] token also via SHIP_WEB_TOKEN (required) [--dev] vite dev server instead of the built app + teploy-ship doctor [--json] [--out f] check this machine is ready (pass|fail|unknown; unknown is not pass) teploy-ship support assemble a REDACTED diagnostic bundle (see docs/SUPPORT.md) [--out DIR] [--log-lines N] what a vendor needs: versions, safe config keys, [--days N] bounded logs, run-state summaries. Credentials @@ -2375,6 +2377,23 @@ async function supportCommand(rest: string[]): Promise { else process.stderr.write(`${yellow("no tar created")} — the bundle directory above is complete; archive it yourself.\n`); } +// --------------------------------------------------------------------------- +// doctor — S19 install-readiness checks (all logic in src/install-doctor.ts) +// --------------------------------------------------------------------------- + +/** Exit 0 only for "ready"; 1 for not-ready or incomplete (unknown is not pass). */ +async function doctorCommand(rest: string[]): Promise { + const args = parseArgs(rest, COMMAND_FLAGS.doctor); + const port = Number(process.env.SHIP_WEB_PORT ?? process.env.PORT ?? 7460); + const report = await runDoctor( + defaultProbes({ env: process.env, stateDir: stateDir(), webPort: Number.isInteger(port) ? port : 7460, nodeVersion: process.version }), + ); + const rendered = renderDoctor(report); + if (typeof args.flags.out === "string") writeFileSync(args.flags.out, rendered.json); + process.stdout.write(args.flags.json === true ? rendered.json : formatDoctor(report)); + if (report.verdict !== "ready") process.exitCode = 1; +} + // --------------------------------------------------------------------------- async function main(): Promise { @@ -2430,6 +2449,8 @@ async function main(): Promise { return webCommand(rest); case "support": return supportCommand(rest); + case "doctor": + return doctorCommand(rest); case "eval": return evalCommand(rest); default: diff --git a/src/install-doctor.test.ts b/src/install-doctor.test.ts new file mode 100644 index 00000000..b38b2ef7 --- /dev/null +++ b/src/install-doctor.test.ts @@ -0,0 +1,176 @@ +import assert from "node:assert/strict"; +import { mkdtempSync, readdirSync, rmSync, writeFileSync } from "node:fs"; +import { createServer } from "node:net"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { test } from "node:test"; + +import { MAX_CLOCK_SKEW_MS, defaultProbes, formatDoctor, renderDoctor, runDoctor } from "./install-doctor.js"; +import type { DoctorProbes } from "./install-doctor.js"; + +// Fake secrets assembled at runtime so no secret-shaped literal sits in source. +const FAKE_WEB_TOKEN = "webtok" + "-" + "Zq9".repeat(8); +const FAKE_MODEL_KEY = "sk" + "-" + "c3".repeat(20); +const FAKE_DB_PASSWORD = "hunter" + "2".repeat(6) + "pw"; +const FAKE_GH = "ghp" + "_" + "d4".repeat(20); + +const NOW = Date.UTC(2026, 9, 4, 12, 0, 0); + +function allGood(over: Partial = {}): DoctorProbes { + return { + nodeVersion: "v22.1.0", + env: { SHIP_WEB_TOKEN: FAKE_WEB_TOKEN, ANTHROPIC_API_KEY: FAKE_MODEL_KEY, NUCLEUS_URL: `postgres://ship:${FAKE_DB_PASSWORD}@db.internal:5432/ship` }, + stateDir: "/var/lib/ship", + webPort: 7460, + writeProbe: async () => {}, + freeMb: async () => 50_000, + portState: async () => "free", + storeReachable: async () => true, + nowMs: NOW, + referenceMs: async () => NOW + 1000, + ...over, + }; +} + +const byId = (r: Awaited>, id: string) => r.checks.find((c) => c.id === id)!; + +test("every probe answering well is the only way to be ready", async () => { + const r = await runDoctor(allGood()); + assert.equal(r.verdict, "ready"); + assert.equal(r.counts.fail + r.counts.unknown, 0); +}); + +test("unknown is not pass: no probes at all is incomplete, never ready", async () => { + const r = await runDoctor({ nodeVersion: "v22.0.0", env: { SHIP_WEB_TOKEN: "x", ANTHROPIC_API_KEY: "y" }, stateDir: "/x", webPort: 1, nowMs: NOW }); + assert.equal(r.verdict, "incomplete"); + for (const id of ["state-dir-writable", "state-dir-space", "web-port", "store-connectivity", "clock"]) { + assert.equal(byId(r, id).status, "unknown", id); + assert.ok(byId(r, id).remedy); + } +}); + +test("a throwing free-space or port probe is unknown, not pass", async () => { + const r = await runDoctor( + allGood({ + freeMb: async () => { + throw new Error("boom"); + }, + portState: async () => { + throw new Error("boom"); + }, + referenceMs: async () => { + throw new Error("boom"); + }, + }), + ); + assert.equal(byId(r, "state-dir-space").status, "unknown"); + assert.equal(byId(r, "web-port").status, "unknown"); + assert.equal(byId(r, "clock").status, "unknown"); + assert.equal(r.verdict, "incomplete"); +}); + +test("each failure mode is reported as fail with a remedy", async () => { + const r = await runDoctor( + allGood({ + nodeVersion: "v18.19.0", + writeProbe: async () => { + throw new Error("EACCES"); + }, + freeMb: async () => 10, + portState: async () => "in-use", + env: { NUCLEUS_URL: "mysql://h/db" }, + storeReachable: async () => false, + referenceMs: async () => NOW + MAX_CLOCK_SKEW_MS + 1, + }), + ); + for (const id of ["node-version", "state-dir-writable", "state-dir-space", "web-port", "env-web-token", "env-model-credential", "store-url", "store-connectivity", "clock"]) { + assert.equal(byId(r, id).status, "fail", id); + assert.ok(byId(r, id).remedy, `${id} remedy`); + } + assert.equal(r.verdict, "not-ready"); +}); + +test("an unset clock fails regardless of any reference", async () => { + const r = await runDoctor(allGood({ nowMs: 0, referenceMs: async () => 0 })); + assert.equal(byId(r, "clock").status, "fail"); +}); + +test("store url shape passing does not imply connectivity passing", async () => { + const r = await runDoctor(allGood({ storeReachable: undefined })); + assert.equal(byId(r, "store-url").status, "pass"); + assert.equal(byId(r, "store-connectivity").status, "unknown"); + assert.equal(r.verdict, "incomplete"); +}); + +test("no NUCLEUS_URL is a valid local install but connectivity stays unknown", async () => { + const r = await runDoctor(allGood({ env: { SHIP_WEB_TOKEN: "t", AI_GATEWAY_URL: "https://gw.example" } })); + assert.equal(byId(r, "store-url").status, "pass"); + assert.equal(byId(r, "store-connectivity").status, "unknown"); +}); + +test("empty / whitespace env values count as missing", async () => { + const r = await runDoctor(allGood({ env: { SHIP_WEB_TOKEN: " ", ANTHROPIC_API_KEY: "" } })); + assert.equal(byId(r, "env-web-token").status, "fail"); + assert.equal(byId(r, "env-model-credential").status, "fail"); +}); + +test("NEGATIVE CONTROL: planted secrets never appear in JSON or text output", async () => { + const leaky = allGood({ + // an error that quotes a credential, and an unparseable URL carrying one + writeProbe: async () => { + throw new Error(`write failed using ${FAKE_GH} at postgres://ship:${FAKE_DB_PASSWORD}@h/db`); + }, + }); + const r = await runDoctor(leaky); + const json = renderDoctor(r); + const text = formatDoctor(r); + for (const secret of [FAKE_WEB_TOKEN, FAKE_MODEL_KEY, FAKE_DB_PASSWORD, FAKE_GH]) { + assert.ok(!json.json.includes(secret), `json leaked ${secret.slice(0, 6)}`); + assert.ok(!text.includes(secret), `text leaked ${secret.slice(0, 6)}`); + } + assert.ok(json.redactions >= 1, "the gate actually fired on the leaky error"); + assert.doesNotThrow(() => JSON.parse(json.json)); +}); + +test("env values are never in the report even before redaction", async () => { + const raw = JSON.stringify(await runDoctor(allGood())); + for (const secret of [FAKE_WEB_TOKEN, FAKE_MODEL_KEY, FAKE_DB_PASSWORD]) assert.ok(!raw.includes(secret)); +}); + +test("unparseable NUCLEUS_URL detail does not echo the input", async () => { + const r = await runDoctor(allGood({ env: { SHIP_WEB_TOKEN: "t", ANTHROPIC_API_KEY: "k", NUCLEUS_URL: `://${FAKE_DB_PASSWORD}` } })); + assert.equal(byId(r, "store-url").status, "fail"); + assert.ok(!JSON.stringify(r).includes(FAKE_DB_PASSWORD)); +}); + +test("real probes: writable temp dir passes, scratch file is removed, port probe sees a listener", async () => { + const dir = mkdtempSync(join(tmpdir(), "doctor-")); + const srv = createServer(); + await new Promise((res) => srv.listen(0, "127.0.0.1", res)); + const busy = (srv.address() as { port: number }).port; + try { + const base = { env: {}, stateDir: dir, webPort: busy, nodeVersion: process.version }; + const p = defaultProbes(base); + const r = await runDoctor(p); + assert.equal(byId(r, "state-dir-writable").status, "pass"); + assert.deepEqual(readdirSync(dir), []); + assert.equal(byId(r, "web-port").status, "fail"); + assert.ok(["pass", "fail"].includes(byId(r, "state-dir-space").status), "statfs gave a real answer"); + } finally { + srv.close(); + rmSync(dir, { recursive: true, force: true }); + } +}); + +test("real write probe against an unwritable path fails", async () => { + const dir = mkdtempSync(join(tmpdir(), "doctor-")); + try { + // A regular file as the parent makes mkdir fail with ENOTDIR on any platform and for root. + writeFileSync(join(dir, "afile"), "x"); + const p = defaultProbes({ env: {}, stateDir: join(dir, "afile", "ship"), webPort: 0, nodeVersion: process.version }); + const r = await runDoctor(p); + assert.equal(byId(r, "state-dir-writable").status, "fail"); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); diff --git a/src/install-doctor.ts b/src/install-doctor.ts new file mode 100644 index 00000000..406eb8b1 --- /dev/null +++ b/src/install-doctor.ts @@ -0,0 +1,261 @@ +/** + * S19: `teploy-ship doctor` — is this machine ready to run Ship? + * + * A fresh install fails in boring ways (old Node, a state directory the + * service user cannot write, the web port already taken, a missing token, a + * store URL pasted wrong, a clock that has drifted) and each one currently + * surfaces later, as a confusing runtime error. This module checks them up + * front and says exactly what it did and did not establish. + * + * Rules enforced here rather than asked of the caller: + * + * 1. **Three outcomes, and unknown is not pass.** Every check reports + * pass | fail | unknown. A probe that is absent, throws something + * unexpected, or returns nothing yields "unknown". The overall verdict is + * "ready" only when EVERY check passed; any fail is "not-ready", and + * unknowns without fails are "incomplete". A doctor that rounds unknown up + * to pass tells a fresh operator they are ready when nobody looked. + * 2. **Shape is not connectivity.** The store-url check parses NUCLEUS_URL and + * never opens a socket; the separate store-connectivity check is "unknown" + * unless a connect probe was injected and answered. Passing the first must + * not read as passing the second. + * 3. **Values never leave.** Env checks report presence only, never the value + * or its length, and the whole report passes the support RedactionGate + * before it is emitted (`renderDoctor`), so even a detail string that + * accidentally quotes an error carrying a credential is rewritten. + * + * Every probe is injected, so the logic is unit-tested against fakes; the + * real probes (`defaultProbes`) have only been exercised on this machine's + * own filesystem and loopback, never on a fresh supported host. + */ + +import { randomBytes } from "node:crypto"; +import { mkdir, rm, statfs, writeFile } from "node:fs/promises"; +import { createServer } from "node:net"; +import { join } from "node:path"; + +import { RedactionGate } from "./support.js"; + +export type CheckStatus = "pass" | "fail" | "unknown"; + +export interface DoctorCheck { + id: string; + status: CheckStatus; + /** What was observed. Never contains an env VALUE. */ + detail: string; + /** What to do next, present on fail and unknown. */ + remedy?: string; +} + +export type DoctorVerdict = "ready" | "not-ready" | "incomplete"; + +export interface DoctorReport { + version: 1; + verdict: DoctorVerdict; + counts: Record; + checks: DoctorCheck[]; +} + +export interface DoctorProbes { + nodeVersion: string; + env: Readonly>; + stateDir: string; + webPort: number; + /** Real write+remove of a scratch file in the state dir. Throws on failure. */ + writeProbe?: (dir: string) => Promise; + /** Free megabytes on the state dir's filesystem. */ + freeMb?: (dir: string) => Promise; + /** Whether something is already bound on the port. */ + portState?: (port: number) => Promise<"free" | "in-use" | undefined>; + /** Try to reach the store. Resolves true only when it answered. */ + storeReachable?: (url: string) => Promise; + /** This host's clock, ms since epoch. */ + nowMs: number; + /** An independent clock reading (e.g. the store's), ms since epoch. */ + referenceMs?: () => Promise; +} + +export const MIN_NODE_MAJOR = 20; +export const MIN_STATE_FREE_MB = 1024; +export const MAX_CLOCK_SKEW_MS = 5 * 60 * 1000; +/** A clock earlier than this is unset or reset, whatever any reference says. */ +export const CLOCK_FLOOR_MS = Date.UTC(2025, 0, 1); + +const errMessage = (e: unknown): string => (e instanceof Error ? e.message : String(e)); +const isSet = (v: string | undefined): boolean => v !== undefined && v.trim() !== ""; + +function check(id: string, status: CheckStatus, detail: string, remedy?: string): DoctorCheck { + return { id, status, detail, ...(remedy !== undefined && status !== "pass" ? { remedy } : {}) }; +} + +export function checkNode(version: string): DoctorCheck { + const m = /^v?(\d+)\.(\d+)/.exec(version); + if (m === null) return check("node-version", "unknown", "could not parse the Node version", `install Node ${MIN_NODE_MAJOR} or newer`); + const major = Number(m[1]); + return major >= MIN_NODE_MAJOR + ? check("node-version", "pass", `Node ${version}`) + : check("node-version", "fail", `Node ${version} is older than ${MIN_NODE_MAJOR}`, `install Node ${MIN_NODE_MAJOR} or newer`); +} + +export async function checkStateWritable(p: DoctorProbes): Promise { + if (p.writeProbe === undefined) return check("state-dir-writable", "unknown", "no write probe available", "check the state directory by hand"); + try { + await p.writeProbe(p.stateDir); + return check("state-dir-writable", "pass", `wrote and removed a scratch file in ${p.stateDir}`); + } catch (e) { + return check("state-dir-writable", "fail", `cannot write to ${p.stateDir}: ${errMessage(e)}`, "create the directory and give the service user write access (TEPLOY_SHIP_STATE)"); + } +} + +export async function checkStateSpace(p: DoctorProbes): Promise { + let mb: number | undefined; + try { + mb = p.freeMb === undefined ? undefined : await p.freeMb(p.stateDir); + } catch { + mb = undefined; + } + if (mb === undefined || !Number.isFinite(mb)) return check("state-dir-space", "unknown", "free space could not be read", "check free disk space by hand"); + return mb >= MIN_STATE_FREE_MB + ? check("state-dir-space", "pass", `${Math.floor(mb)} MB free (minimum ${MIN_STATE_FREE_MB})`) + : check("state-dir-space", "fail", `${Math.floor(mb)} MB free, below the ${MIN_STATE_FREE_MB} MB minimum`, "free disk space or point TEPLOY_SHIP_STATE at a larger volume"); +} + +export async function checkPort(p: DoctorProbes): Promise { + let s: "free" | "in-use" | undefined; + try { + s = p.portState === undefined ? undefined : await p.portState(p.webPort); + } catch { + s = undefined; + } + if (s === undefined) return check("web-port", "unknown", `could not tell whether port ${p.webPort} is free`, "check the port by hand"); + return s === "free" + ? check("web-port", "pass", `port ${p.webPort} is free`) + : check("web-port", "fail", `port ${p.webPort} is already in use`, "stop the other listener or set SHIP_WEB_PORT"); +} + +/** Presence only. The value, and even its length, stay out of the report. */ +export function checkEnv(env: DoctorProbes["env"]): DoctorCheck[] { + const out: DoctorCheck[] = []; + out.push( + isSet(env.SHIP_WEB_TOKEN) + ? check("env-web-token", "pass", "SHIP_WEB_TOKEN is set") + : check("env-web-token", "fail", "SHIP_WEB_TOKEN is not set", "set SHIP_WEB_TOKEN (the browser login uses it)"), + ); + const model = ["ANTHROPIC_API_KEY", "AI_GATEWAY_URL"].filter((k) => isSet(env[k])); + out.push( + model.length > 0 + ? check("env-model-credential", "pass", `model access configured via ${model.join(" and ")}`) + : check("env-model-credential", "fail", "neither ANTHROPIC_API_KEY nor AI_GATEWAY_URL is set", "set ANTHROPIC_API_KEY or AI_GATEWAY_URL"), + ); + return out; +} + +/** Parse only. Absent NUCLEUS_URL is a legitimate local-file install. */ +export function checkStoreShape(env: DoctorProbes["env"]): DoctorCheck { + const raw = env.NUCLEUS_URL; + if (!isSet(raw)) return check("store-url", "pass", "NUCLEUS_URL not set: local file store (shape only, nothing connected)"); + let u: URL; + try { + u = new URL(raw as string); + } catch { + // The parse error can quote the input, and the input may carry a password. + return check("store-url", "fail", "NUCLEUS_URL is not a parseable URL", "use postgres://host:port/db"); + } + if (u.protocol !== "postgres:" && u.protocol !== "postgresql:") { + return check("store-url", "fail", `NUCLEUS_URL scheme "${u.protocol}" is not postgres`, "use postgres://host:port/db"); + } + if (u.hostname === "") return check("store-url", "fail", "NUCLEUS_URL has no host", "use postgres://host:port/db"); + return check("store-url", "pass", `NUCLEUS_URL shape valid, host ${u.hostname}${u.port !== "" ? `:${u.port}` : ""} (shape only, nothing connected)`); +} + +export async function checkStoreConnectivity(p: DoctorProbes): Promise { + const url = p.env.NUCLEUS_URL; + if (!isSet(url)) return check("store-connectivity", "unknown", "no NUCLEUS_URL: nothing to connect to", "not applicable to a local file store; verify with a real run"); + let ok: boolean | undefined; + try { + ok = p.storeReachable === undefined ? undefined : await p.storeReachable(url as string); + } catch { + ok = false; + } + if (ok === undefined) return check("store-connectivity", "unknown", "no connect probe was run", "run `teploy-ship preflight` against the live store"); + return ok + ? check("store-connectivity", "pass", "store answered") + : check("store-connectivity", "fail", "store did not answer", "check NUCLEUS_URL, the network path and that the store is up"); +} + +export async function checkClock(p: DoctorProbes): Promise { + if (!Number.isFinite(p.nowMs) || p.nowMs < CLOCK_FLOOR_MS) { + return check("clock", "fail", "the system clock is unset or implausibly early", "set the clock (NTP) before running Ship"); + } + let ref: number | undefined; + try { + ref = p.referenceMs === undefined ? undefined : await p.referenceMs(); + } catch { + ref = undefined; + } + if (ref === undefined || !Number.isFinite(ref)) { + return check("clock", "unknown", "clock is plausible but no independent reference was available to measure skew", "compare against a trusted time source"); + } + const skew = Math.abs(p.nowMs - ref); + return skew <= MAX_CLOCK_SKEW_MS + ? check("clock", "pass", `skew ${Math.round(skew / 1000)}s against the reference`) + : check("clock", "fail", `skew ${Math.round(skew / 1000)}s exceeds ${MAX_CLOCK_SKEW_MS / 1000}s`, "enable NTP; leases, schedules and tokens depend on agreeing clocks"); +} + +export async function runDoctor(p: DoctorProbes): Promise { + const checks: DoctorCheck[] = [ + checkNode(p.nodeVersion), + await checkStateWritable(p), + await checkStateSpace(p), + await checkPort(p), + ...checkEnv(p.env), + checkStoreShape(p.env), + await checkStoreConnectivity(p), + await checkClock(p), + ]; + const counts: Record = { pass: 0, fail: 0, unknown: 0 }; + for (const c of checks) counts[c.status]++; + const verdict: DoctorVerdict = counts.fail > 0 ? "not-ready" : counts.unknown > 0 ? "incomplete" : "ready"; + return { version: 1, verdict, counts, checks }; +} + +/** The redacted support-bundle section: parseable JSON, every string through the gate. */ +export function renderDoctor(report: DoctorReport): { json: string; redactions: number } { + const gate = new RedactionGate(); + const safe = gate.redactJson(report); + return { json: JSON.stringify(safe, null, 2) + "\n", redactions: gate.total }; +} + +export function formatDoctor(report: DoctorReport): string { + const mark: Record = { pass: "PASS ", fail: "FAIL ", unknown: "UNKNOWN" }; + const gate = new RedactionGate(); + const lines = report.checks.map( + (c) => `${mark[c.status]} ${c.id}: ${gate.redact(c.detail)}${c.remedy !== undefined ? `\n -> ${gate.redact(c.remedy)}` : ""}`, + ); + lines.push(`verdict: ${report.verdict} (${report.counts.pass} pass, ${report.counts.fail} fail, ${report.counts.unknown} unknown)`); + return lines.join("\n") + "\n"; +} + +/** Real probes. Local filesystem and loopback only; no outbound network. */ +export function defaultProbes(base: Pick): DoctorProbes { + return { + ...base, + nowMs: Date.now(), + writeProbe: async (dir) => { + await mkdir(dir, { recursive: true }); + const f = join(dir, `.doctor-${randomBytes(6).toString("hex")}`); + await writeFile(f, "ok"); + await rm(f); + }, + freeMb: async (dir) => { + const s = await statfs(dir); + return (Number(s.bavail) * Number(s.bsize)) / (1024 * 1024); + }, + portState: (port) => + new Promise((resolve) => { + const srv = createServer(); + srv.once("error", (e: NodeJS.ErrnoException) => resolve(e.code === "EADDRINUSE" ? "in-use" : undefined)); + srv.listen(port, "127.0.0.1", () => srv.close(() => resolve("free"))); + }), + }; +} diff --git a/src/restore-readiness.test.ts b/src/restore-readiness.test.ts new file mode 100644 index 00000000..4d078796 --- /dev/null +++ b/src/restore-readiness.test.ts @@ -0,0 +1,67 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; + +import { compareHistory, digestRun } from "./restore-readiness.js"; +import type { HistorySnapshot, RunSnapshot } from "./restore-readiness.js"; + +const ev = (name: string, data: unknown = {}) => ({ type: "step", name, data }); + +function run(id: string, extra: Partial = {}): RunSnapshot { + const events = [ev("a"), ev("b", { n: 1 })]; + return { runId: id, events: events.length, digest: digestRun(events), fingerprint: "s1:abc", ...extra }; +} +const snap = (runs: RunSnapshot[]): HistorySnapshot => ({ takenAt: "2026-10-04T00:00:00Z", source: "t", runs }); + +test("identical history is restored-match", () => { + const r = compareHistory(snap([run("r1"), run("r2", { waiting: "approval:x" })]), snap([run("r2", { waiting: "approval:x" }), run("r1")])); + assert.equal(r.verdict, "restored-match"); + assert.equal(r.backupRuns, 2); + assert.equal(r.eventsRestored, 4); +}); + +test("unknown is not pass: missing, malformed or empty-backup comparisons are unverified", () => { + assert.equal(compareHistory(undefined, snap([run("r1")])).verdict, "unverified"); + assert.equal(compareHistory(snap([run("r1")]), undefined).verdict, "unverified"); + assert.equal(compareHistory(snap([run("r1")]), { runs: [{ runId: 1 }] }).verdict, "unverified"); + const empty = compareHistory(snap([]), snap([])); + assert.equal(empty.verdict, "unverified"); + assert.match(empty.reason ?? "", /nothing was compared/); +}); + +test("a dropped run is a mismatch and is named", () => { + const r = compareHistory(snap([run("r1"), run("r2")]), snap([run("r1")])); + assert.equal(r.verdict, "mismatch"); + assert.deepEqual(r.missing, ["r2"]); +}); + +test("same counts but swapped content is caught by the digest (counts alone would pass)", () => { + const other = [ev("a"), ev("b", { n: 2 })]; + const swapped: RunSnapshot = { runId: "r1", events: other.length, digest: digestRun(other), fingerprint: "s1:abc" }; + const r = compareHistory(snap([run("r1")]), snap([swapped])); + assert.equal(r.backupRuns, r.restoredRuns); + assert.equal(r.eventsBackup, r.eventsRestored); + assert.equal(r.verdict, "mismatch"); + assert.deepEqual(r.changed, ["r1"]); +}); + +test("event order is part of the digest", () => { + assert.notEqual(digestRun([ev("a"), ev("b")]), digestRun([ev("b"), ev("a")])); +}); + +test("a lost waiting decision, an extra run and a fingerprint change each break the match", () => { + const b = snap([run("r1", { waiting: "approval:x" }), run("r2")]); + assert.deepEqual(compareHistory(b, snap([run("r1"), run("r2")])).waitingLost, ["r1:approval:x"]); + assert.deepEqual(compareHistory(b, snap([run("r1", { waiting: "approval:x" }), run("r2"), run("r3")])).extra, ["r3"]); + const fp = compareHistory(b, snap([run("r1", { waiting: "approval:x", fingerprint: "s1:zzz" }), run("r2")])); + assert.equal(fp.verdict, "mismatch"); + assert.deepEqual(fp.fingerprintChanged, ["r1"]); +}); + +test("the report carries ids, counts and digests only (no event bodies)", () => { + const secret = "body-secret-" + "q".repeat(12); + const events = [ev("a", { token: secret })]; + const s: RunSnapshot = { runId: "r1", events: 1, digest: digestRun(events) }; + const r = compareHistory(snap([s]), snap([s])); + assert.ok(!JSON.stringify(r).includes(secret)); + assert.ok(!JSON.stringify(s).includes(secret)); +}); diff --git a/src/restore-readiness.ts b/src/restore-readiness.ts new file mode 100644 index 00000000..34a4047f --- /dev/null +++ b/src/restore-readiness.ts @@ -0,0 +1,121 @@ +/** + * S19: restore-readiness — does a restored store hold what the backup held? + * + * `scripts/ship-backup.sh rehearse` proves an archive unpacks and an engine + * boots; it does not prove the restored HISTORY matches. This module compares + * two history snapshots (taken from the source store at backup time and from + * the restored store) and refuses to say "restored" unless it actually + * compared and everything agreed. + * + * Rules: + * - Unknown is not pass. A missing snapshot, a malformed side, or a backup + * with no runs yields "unverified", never "restored-match". + * - Counts alone are weak (a restore that swaps rows keeps the count), so each + * snapshot carries a per-run digest of the run's recorded event sequence and + * the comparison is per run id: missing, extra and changed runs are named. + * - Waiting decisions (runs parked on an approval/answer) are compared as a + * set, because losing one silently strands a person's decision. + * - Snapshots hold ids, counts and digests only; no event bodies, so the + * report cannot carry credentials from run history. + * + * `digestRun` is the one place the digest is defined. Taking the snapshots + * from a real store is not implemented here (it needs live access): this is + * the comparison half. + */ + +import { createHash } from "node:crypto"; + +export interface RunSnapshot { + runId: string; + /** Number of recorded events. */ + events: number; + /** digestRun() over the ordered event sequence. */ + digest: string; + /** Open decision this run is parked on, if any (e.g. "approval:"). */ + waiting?: string; + /** Step fingerprint recorded on the run, if any. */ + fingerprint?: string; +} + +export interface HistorySnapshot { + /** When and where this was taken; informational. */ + takenAt: string; + source: string; + runs: RunSnapshot[]; +} + +export type ReadinessVerdict = "restored-match" | "mismatch" | "unverified"; + +export interface ReadinessReport { + verdict: ReadinessVerdict; + backupRuns: number; + restoredRuns: number; + eventsBackup: number; + eventsRestored: number; + missing: string[]; + extra: string[]; + changed: string[]; + waitingLost: string[]; + waitingExtra: string[]; + fingerprintChanged: string[]; + /** Why the verdict is unverified, when it is. */ + reason?: string; +} + +/** Order-sensitive digest of a run's events. Bodies are hashed, never stored. */ +export function digestRun(events: ReadonlyArray<{ type: string; name?: string; data?: unknown }>): string { + const h = createHash("sha256"); + for (const e of events) { + h.update(`${e.type}\u0000${e.name ?? ""}\u0000`); + h.update(createHash("sha256").update(JSON.stringify(e.data ?? null)).digest()); + } + return h.digest("hex"); +} + +function validSnapshot(s: unknown): s is HistorySnapshot { + if (typeof s !== "object" || s === null) return false; + const r = (s as HistorySnapshot).runs; + return Array.isArray(r) && r.every((x) => typeof x?.runId === "string" && typeof x.events === "number" && typeof x.digest === "string"); +} + +const NONE = { backupRuns: 0, restoredRuns: 0, eventsBackup: 0, eventsRestored: 0, missing: [], extra: [], changed: [], waitingLost: [], waitingExtra: [], fingerprintChanged: [] }; + +export function compareHistory(backup: unknown, restored: unknown): ReadinessReport { + if (!validSnapshot(backup)) return { ...NONE, verdict: "unverified", reason: "backup snapshot is missing or malformed" }; + if (!validSnapshot(restored)) return { ...NONE, backupRuns: backup.runs.length, verdict: "unverified", reason: "restored snapshot is missing or malformed" }; + if (backup.runs.length === 0) { + return { ...NONE, restoredRuns: restored.runs.length, verdict: "unverified", reason: "backup snapshot holds no runs: nothing was compared" }; + } + const b = new Map(backup.runs.map((r) => [r.runId, r])); + const r = new Map(restored.runs.map((x) => [x.runId, x])); + const missing = [...b.keys()].filter((k) => !r.has(k)).sort(); + const extra = [...r.keys()].filter((k) => !b.has(k)).sort(); + const changed: string[] = []; + const fingerprintChanged: string[] = []; + for (const [id, br] of b) { + const rr = r.get(id); + if (rr === undefined) continue; + if (br.events !== rr.events || br.digest !== rr.digest) changed.push(id); + if (br.fingerprint !== rr.fingerprint) fingerprintChanged.push(id); + } + const wb = new Set(backup.runs.filter((x) => x.waiting !== undefined).map((x) => `${x.runId}:${x.waiting}`)); + const wr = new Set(restored.runs.filter((x) => x.waiting !== undefined).map((x) => `${x.runId}:${x.waiting}`)); + const waitingLost = [...wb].filter((k) => !wr.has(k)).sort(); + const waitingExtra = [...wr].filter((k) => !wb.has(k)).sort(); + changed.sort(); + fingerprintChanged.sort(); + const clean = [missing, extra, changed, waitingLost, waitingExtra, fingerprintChanged].every((a) => a.length === 0); + return { + verdict: clean ? "restored-match" : "mismatch", + backupRuns: backup.runs.length, + restoredRuns: restored.runs.length, + eventsBackup: backup.runs.reduce((n, x) => n + x.events, 0), + eventsRestored: restored.runs.reduce((n, x) => n + x.events, 0), + missing, + extra, + changed, + waitingLost, + waitingExtra, + fingerprintChanged, + }; +}