diff --git a/AUDIT_OPEN.md b/AUDIT_OPEN.md index 3407b32..029640f 100644 --- a/AUDIT_OPEN.md +++ b/AUDIT_OPEN.md @@ -587,3 +587,11 @@ The real standalone API/client coordination completed through both normal merge approvals and its compatibility scan. Its client description still said the check was owed despite the complete badge. It now points to the actual check result below without asserting an obsolete pending state. + +The final-image replay additionally reproduced a startup race: Docker's port +proxy accepted the TCP probe while Nucleus was still restoring its catalog. +The first preflight query then lost its connection. TCP readiness is replaced +with a successful `nucleus shell --command "SELECT 1"` inside the proof container. +This corrects the earlier attribution: explicit data/transport is required, but +transport alone did not explain every connection failure. Original failed and +successful rehearsals remain in the release receipts. diff --git a/scripts/ship-backup.sh b/scripts/ship-backup.sh index 30d3000..6627d8c 100755 --- a/scripts/ship-backup.sh +++ b/scripts/ship-backup.sh @@ -292,14 +292,16 @@ cmd_rehearse() { endpoint="$(docker port "$name" 5432/tcp | head -1)" [ -n "$endpoint" ] || { docker logs "$name" || true; die "proof container published no port"; } timeout="${SHIP_REHEARSE_TIMEOUT_S:-300}" - # The engine refuses connections while it replays its WAL, so wait on the - # socket, not on the container state. - until (exec 3<>"/dev/tcp/${endpoint%:*}/${endpoint##*:}") 2>/dev/null; do + # Docker can accept TCP through its published-port proxy before Nucleus + # finishes recovery. Require an actual SQL response inside the container; + # an open proxy socket is not database readiness. + until docker exec "$name" nucleus shell --host 127.0.0.1 --port 5432 \ + --command "SELECT 1" --json >/dev/null 2>&1; do sleep 1 elapsed=$((elapsed + 1)) if [ "$elapsed" -ge "$timeout" ]; then docker logs "$name" || true - die "proof engine did not accept connections within ${timeout}s (see logs above)" + die "proof engine did not answer SQL within ${timeout}s (see logs above)" fi done diff --git a/scripts/ship-backup.test.mjs b/scripts/ship-backup.test.mjs index 3160292..e6afc98 100644 --- a/scripts/ship-backup.test.mjs +++ b/scripts/ship-backup.test.mjs @@ -5,6 +5,7 @@ import { tmpdir } from "node:os"; import { join, dirname } from "node:path"; import { fileURLToPath } from "node:url"; import { test } from "node:test"; +import { createServer } from "node:net"; const root = join(dirname(fileURLToPath(import.meta.url)), ".."); const script = join(root, "scripts", "ship-backup.sh"); @@ -16,8 +17,8 @@ const script = join(root, "scripts", "ship-backup.sh"); * there is no undo for. These tests pin both directions against a FAKE ship * root — they never touch /deployments, never start docker, and stub docker * where a refusal depends on it seeing a running container. The rehearsal's - * docker path needs a real engine and belongs to the operator's live pass, - * not to CI: it is only asserted to refuse cleanly when docker is absent. + * data recovery needs a real engine and belongs to the operator's live pass. + * A separate fake checks readiness ordering, without claiming a restore proof. */ function run(args, env = {}) { return spawnSync("bash", [script, ...args], { encoding: "utf8", env: { ...process.env, ...env } }); @@ -202,3 +203,44 @@ test("rehearse refuses cleanly when docker is absent (docker-dependent paths are rmSync(dir, { recursive: true, force: true }); } }); + +test("an open Docker proxy port cannot start preflight before SQL is ready", async (t) => { + const proxy = createServer((socket) => socket.destroy()); + await new Promise((resolve) => proxy.listen(0, "127.0.0.1", resolve)); + t.after(() => proxy.close()); + const { dir } = makeFakeRoot(); + t.after(() => rmSync(dir, { recursive: true, force: true })); + assert.equal(run(["backup", "--label", "proof", "--i-stopped-writers"], { SHIP_ROOT: dir }).status, 0); + const bin = join(dir, "bin"); + mkdirSync(bin); + writeFileSync(join(bin, "docker"), `#!/bin/sh +case "$1" in + ps|rm) exit 0 ;; + run) echo proof-container ;; + port) echo 127.0.0.1:${proxy.address().port} ;; + exec) + case "$*" in *"SELECT 1"*) ;; *) exit 10 ;; esac + count=0 + [ ! -f "$SQL_PROBE_COUNT" ] || count=$(cat "$SQL_PROBE_COUNT") + count=$((count + 1)) + echo "$count" > "$SQL_PROBE_COUNT" + [ "$count" -ge 2 ] ;; + *) exit 1 ;; +esac +`); + writeFileSync(join(bin, "ship-proof"), `#!/bin/sh +count=0 +[ ! -f "$SQL_PROBE_COUNT" ] || count=$(cat "$SQL_PROBE_COUNT") +[ "$count" -ge 2 ] || { echo 'preflight ran before SQL readiness' >&2; exit 42; } +echo preflight-after-sql > "$PREFLIGHT_RECEIPT" +`); + chmodSync(join(bin, "docker"), 0o755); + chmodSync(join(bin, "ship-proof"), 0o755); + const receipt = join(dir, "preflight.txt"); + const res = run(["rehearse", onlyArchive(join(dir, "_backups")), "--image", "fixture-engine"], { + SHIP_ROOT: dir, SHIP_BIN: join(bin, "ship-proof"), PATH: `${bin}:${process.env.PATH}`, + SQL_PROBE_COUNT: join(dir, "sql-probes"), PREFLIGHT_RECEIPT: receipt, SHIP_REHEARSE_TIMEOUT_S: "5", + }); + assert.equal(res.status, 0, res.stdout + res.stderr); + assert.equal(readFileSync(receipt, "utf8").trim(), "preflight-after-sql"); +});