From 8e2b4c4a2a017d9beaf066414732d8c6b502f178 Mon Sep 17 00:00:00 2001 From: tetsuo Date: Thu, 12 Nov 2026 19:29:19 +0900 Subject: [PATCH] fix(release): verify paused Cloudflare version and service bindings after rollback (#50) --- docs/operations.md | 17 ++- docs/release-preflight.md | 15 ++- docs/release-runbook.md | 62 ++++++++-- scripts/release-pause-readback.mjs | 120 +++++++++++++++++++ scripts/release-pause-readback.test.mjs | 149 ++++++++++++++++++++++++ 5 files changed, 346 insertions(+), 17 deletions(-) create mode 100644 scripts/release-pause-readback.mjs create mode 100644 scripts/release-pause-readback.test.mjs diff --git a/docs/operations.md b/docs/operations.md index 19e680f..4a3804e 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -142,15 +142,22 @@ Worker version, four SQLite DO namespaces, OAuth KV ID, actual secret names, domain, release vars and HTTPS `/health` build/profile. PR CI never receives release credentials; approval from a repository collaborator or a Slack DM alone does not bypass the protected Environment. The restricted implementation -supports the `manual` profile only. Worker deployment is **not authorized** -until all missing operator inputs and explicit approvals are provided. +supports the `manual` profile only. The first protected staging release was +separately approved and executed on 2026-11-12. **No protected production +release is authorized** without a new exact-main gate, operator evidence and +independent GitHub Environment approval. The full operator inputs, separate read/write Cloudflare tokens, private environment-scoped secrets, evidence schemas, rollback and measured hosted acceptance checklist are in [release-runbook.md](release-runbook.md). It -records **not run** for staging and production actual #50 deployments; the -previous settings-only [credential read](https://github.com/kitamura-tetsuo/agent-table/actions/runs/42969129958) -did not authorize release or verify private DO/KV ownership. +records the [protected staging deployment](https://github.com/kitamura-tetsuo/agent-table/actions/runs/43151011535), +[same-config provider rollback rehearsal](https://github.com/kitamura-tetsuo/agent-table/issues/50#issuecomment-6815095095), +[text-binding mismatch and repaired pause](https://github.com/kitamura-tetsuo/agent-table/issues/50#issuecomment-6818264043), +and [synthetic browser Close/Delete acceptance](https://github.com/kitamura-tetsuo/agent-table/issues/50#issuecomment-6820766589). +For any rollback or emergency pause, run the read-only +`scripts/release-pause-readback.mjs` verifier before claiming service settings +and active-version text bindings agree. Production release, independently +verified account backup/log retention and actual billed usage remain blocked. ## Deployment procedure after implementation diff --git a/docs/release-preflight.md b/docs/release-preflight.md index b6b972f..66e015d 100644 --- a/docs/release-preflight.md +++ b/docs/release-preflight.md @@ -63,6 +63,19 @@ The command requires the operator-supplied counterpart configuration and checks - Separate manual staging/production workflows require the exact main SHA and a selected GitHub Environment. GitHub API readback on 2026-11-11 UTC confirmed required owner reviewers and protected-branch restrictions for **both** Environments; this is historical configuration evidence, not an approval of a deployment. The [protected staging credential check #42969129958](https://github.com/kitamura-tetsuo/agent-table/actions/runs/42969129958) verified staging-only read access to settings, **not** full release resource ownership or promotion. - The protected workflow's **authenticated** `release-authorization.mjs` now emits a short-lived in-job `RELEASE_APPROVAL_VALID` only after exact-main CI provenance and live Cloudflare readback of both environments. `require-release-approval.mjs` must verify that manifest before publishing. Missing read tokens, production configuration, named secret inventory, selected domain, independently verified retention/usage disclosures, and required profile evidence remain **blocked**, rather than being replaced by fabricated metadata. See [release-runbook.md](release-runbook.md) for required Environment secret names, operator JSON schemas, current scope and the exact approval sequence. - Only the `manual` profile is implemented in the authenticated gate. New `live`, Events and autonomous host validation and **independently reconciled restore-quarantine release** remain blocked. Future DO-only outbound producers must apply the independent dispatch-time guard too. A restored copy of an older successful operator record does not clear missing post-snapshot revocations/deletions/expiry. -- Retained-state newer-schema rollback through a real staging/production deployment, production host provision and privately disclosed current account backup/log retention and pricing/usage remain **not run/not verified**. The checked-in constructor and Workerd oracle is synthetic only. Actual operator evidence is required before Issue #50 operational acceptance. +- A separately approved protected staging bootstrap and same-source, + same-config, ID-pinned rollback **did** preserve retained synthetic Durable + Object data [on 2026-11-12](https://github.com/kitamura-tetsuo/agent-table/issues/50#issuecomment-6815095095). + A subsequent temporary-intake rollback exposed divergent service settings + and active-version text bindings; a full canonical staging redeploy restored + independently verified pause [receipt](https://github.com/kitamura-tetsuo/agent-table/issues/50#issuecomment-6818264043). + The owner also completed synthetic browser Close/Delete + [acceptance](https://github.com/kitamura-tetsuo/agent-table/issues/50#issuecomment-6820766589). + Future paused-state claims must use the read-only + `scripts/release-pause-readback.mjs` verifier. **Different-schema rollback + and actual PITR restoration remain untested**; protected production + deployment and private current-account backup/log retention and billed + pricing/usage remain **not verified**. The checked-in constructor and + Workerd oracle are still synthetic and do not replace those operator gates. Local validation: `node --test scripts/release-preflight.test.mjs`. CI runs this automatically via `node --test scripts/*.test.mjs` and checks the new sources with Biome. No deploy command belongs in PR CI. diff --git a/docs/release-runbook.md b/docs/release-runbook.md index 8c6a3ea..eb2c86a 100644 --- a/docs/release-runbook.md +++ b/docs/release-runbook.md @@ -10,13 +10,16 @@ version state **before** permitting `wrangler deploy`. It reads the deployed resources and public `/health` build receipt again after publication. A local preflight success or a PR merge is never deployment authorization. -**Not completed by committing this code:** selecting or provisioning a -production HTTPS hostname, provisioning missing production resources, supplying -release Environment secrets, obtaining protected approvals, running staging or -production deployment, testing a retained-state provider rollback, and -recording measured hosted usage/retention facts. They are operator gates, -not assertions that the code has already deployed. The checked-in local -`wrangler.jsonc` is neither a staging nor a production release candidate. +**Hosted evidence as of 2026-11-12:** owner-approved protected staging bootstrap, +same-config live update and pinned code rollback, and the existing-guest +browser Close/Delete rehearsal have succeeded against synthetic retained state. +The later staging temporary-admission experiment exposed a Wrangler +service-settings/active-version text-binding mismatch; an independently +verified canonical paused-config redeploy repaired it. Production HTTPS and +identity have been read back, but a **separately approved protected production +release has not run**. Account-specific backup/log retention, billed storage +and pricing/usage acceptance are still unresolved. These are distinct gates; +the checked-in `wrangler.jsonc` is not a hosted release candidate. The authenticated gate initially supports **`manual` only**. Every production or staging release must have `RESTORE_QUARANTINE=false` only for a verified @@ -333,7 +336,39 @@ action; it is not permission to delete or reset private namespaces. `RESTORE_QUARANTINE=false`. Confirm on the *real* host that new admission and table creation stop, but existing owner browser close/delete and authorized reads/exports continue to work. Record the observed status and - current DO state, not just intended bindings. + current DO state, not just intended bindings. A successful Wrangler command + is insufficient: **both** service settings and active-version text bindings + must independently show all five switches disabled and must agree with the + reviewed canonical config. +- **Normal-pause provider receipt (read-only):** after a no-intake deploy or a + version rollback, first independently obtain the expected 100%-traffic + active version ID, then run the following script on an authorized operator + host with the environment's `CLOUDFLARE_READ_TOKEN` available in the + process environment. Both config files must be the real, reviewed private + JSON inputs; the peer config is required to verify environment isolation. + + ```sh + node scripts/release-pause-readback.mjs \ + --config /secure/operator/wrangler.staging.json \ + --peer-config /secure/operator/wrangler.production.json \ + --expected-version-id '' + ``` + + The only success receipt is `PAUSE_READBACK_VERIFIED` with environment, + Worker, active version, deployment ID and build SHA (no secrets/private + data). On any `PAUSE_READBACK_BLOCKED` code, treat the pause as **unverified** + and stop promotion; do not trust a provider version selector or proceed to + production. In particular `provider-version-config-unverified` means + Cloudflare service-level settings and the active version's bindings disagree. + This exact condition was observed after the 2026-11-12 staging + `ALLOW_NEW_TABLES` temporary-admission rollback, despite Wrangler reporting + success. It was resolved only by a **separately owner-approved, byte-for-byte + canonical paused-config redeploy** followed by independent dual-surface + verification. For a repeat incident, preserve evidence, evaluate ongoing + exposure and seek fresh authorization for the contained settings repair; + **never automatically redeploy, clear quarantine, delete namespaces or + silently declare intake stopped**. Follow a verified pause receipt with + real HTTPS health, admission-denial and existing-guest lifecycle tests. - **Security incident, unknown backup history or stale authority:** first stop external intake/delivery, then install a separately approved `RESTORE_QUARANTINE=true` candidate using the offline @@ -352,6 +387,9 @@ action; it is not permission to delete or reset private namespaces. if an older reader is unsafe, leave the data quarantined and deploy a forward-compatible repair instead. Recheck both Worker versions, domain, secrets, active DO/KV identity, public `/health` and lifecycle invariants. + Run the normal-pause dual-surface readback above; a code-version rollback + may **not** restore the old text-binding service settings. The successful + same-config staging rollback is not proof of older-schema compatibility. - **Point-in-time restoration** is not code rollback. The recovery candidate remains inaccessible until independent authority reconciliation. The platform's advertised backup window is not a substitute for measured @@ -374,9 +412,11 @@ can change, and privacy data cannot be guaranteed erased by rollback. | Structural preflight, fail-closed checks and generated-restore local oracle | Local and CI tests, PR #46 | Implemented code; not hosted approval | | Staging secret/config access | [protected config readback #43115024418](https://github.com/kitamura-tetsuo/agent-table/actions/runs/43115024418), commit `70cfe1c`, SHA-256 `22b619dd3af71bce1688a7f3ec2e451654cea0a28ac6e166f70bfa98513bb867` | Verified no-publish only; stale after new `main` commit | | Separate configured protected Environments | Human reviewer and `main` branch protection readback 2026-11-11; staging and production Cloudflare GET access [#43104683889](https://github.com/kitamura-tetsuo/agent-table/actions/runs/43104683889), [#43104688167](https://github.com/kitamura-tetsuo/agent-table/actions/runs/43104688167) | Verified read-only; each deployment still needs a separate approval | -| Both true Cloudflare Worker/DO/KV identities, origins, inventories | Read-only Worker/domain/namespace/version/secret-name checks recorded on [Issue #50](https://github.com/kitamura-tetsuo/agent-table/issues/50) | Staging active Worker is missing `RELEASE_*` bindings and needs separately reviewed bootstrap; production active Worker already has them | -| Staging protected deploy, host smoke, retained-state update/rollback | No #50 approved live run | **Not run** | -| Production origin, scoped tokens, protected deploy and host smoke | `agent-table.outliner-online.com` domain/HTTPS health verified; production no-publish config still required | **No deployment** | +| Both true Cloudflare Worker/DO/KV identities, origins, inventories | Independent authenticated metadata checks and [protected staging deploy #43151011535](https://github.com/kitamura-tetsuo/agent-table/actions/runs/43151011535); [provider receipt](https://github.com/kitamura-tetsuo/agent-table/issues/50#issuecomment-6814836152) | **Verified** for the recorded 2026-11-12 source and staging state; recheck before next release | +| Staging protected deploy, host smoke, retained-state update/rollback | [Staging bootstrap](https://github.com/kitamura-tetsuo/agent-table/issues/50#issuecomment-6814836152), [same-config update + pinned rollback](https://github.com/kitamura-tetsuo/agent-table/issues/50#issuecomment-6815095095), [binding-drift finding and canonical repair](https://github.com/kitamura-tetsuo/agent-table/issues/50#issuecomment-6818264043) | **Passed** for same-source/same-config retained synthetic data; older-schema rollback and real PITR untested | +| Normal pause and existing-guest browser Close/Delete | [7-day synthetic fixture and settings repair](https://github.com/kitamura-tetsuo/agent-table/issues/50#issuecomment-6818264043), [browser Close/Delete and independent `TABLE_NOT_FOUND` postconditions](https://github.com/kitamura-tetsuo/agent-table/issues/50#issuecomment-6820766589) | **Passed** on staging; prior browser POST refresh showed CSRF rejection and stale response label, but a fresh GET confirmed closed | +| Staging pause after text-binding change | Independent provider check rejected a Wrangler rollback with settings/version divergence; fully canonical redeploy yielded active version `f0701beb-db3a-4c6a-b224-7e988d438bc4` and matching disabled intake [receipt](https://github.com/kitamura-tetsuo/agent-table/issues/50#issuecomment-6818264043) | **Repaired and verified**; perform new readback before subsequent changes | +| Production origin, scoped tokens, protected deploy and host smoke | `agent-table.outliner-online.com` HTTPS `/health` and current Worker identity verified; recorded production active version `8e099f35-8fa7-4e16-878c-43daa89fa218` | **Protected #50 production deploy not run**; new exact-main, owner and Environment approvals required | | Measured hosted CPU/requests/DO/KV/logs plus current pricing | Cloudflare GraphQL for 2026-11-04–11 UTC: staging Worker 5,293 requests/0 errors, production Worker 271/0, temporary PR46 Worker 33/0, DO invocation group 2,090 requests; account billing usage GET returned 403 (`insufficient_permissions`), DO storage groups returned no records; Workers Logs `logpush=false` and settings `observability=null` | **Partial measurement; billed usage, retention and actual DO/KV storage unresolved** | | Post-snapshot independent authority reconciliation | [Issue #51](https://github.com/kitamura-tetsuo/agent-table/issues/51) | **Out of scope; quarantined** | diff --git a/scripts/release-pause-readback.mjs b/scripts/release-pause-readback.mjs new file mode 100644 index 0000000..387e3ca --- /dev/null +++ b/scripts/release-pause-readback.mjs @@ -0,0 +1,120 @@ +import { readFileSync } from "node:fs"; +import { pathToFileURL } from "node:url"; +import { + cloudflareReader, + ProviderVerificationError, + readCloudflareDeployment, + verifyDeployedConfiguration, +} from "./release-provider.mjs"; + +const PAUSE_SWITCHES = [ + "ALLOW_NEW_GUESTS", + "ALLOW_NEW_TABLES", + "ENABLE_MCP_SUBSCRIPTIONS", + "ENABLE_OPENAI_EVENTS", + "ALLOW_OUTBOUND_NOTIFICATIONS", +]; +const VERSION_ID = + /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i; + +function requirePause(value, code) { + if (!value) throw new ProviderVerificationError(code); +} + +/** Verify a normal intake pause using independent Cloudflare settings and active-version reads. */ +export async function verifyNormalPause({ + config, + peer, + expectedVersionId, + cloudflareRead, +}) { + const vars = config?.vars; + const environment = vars?.RELEASE_ENVIRONMENT; + requirePause( + (environment === "staging" || environment === "production") && + config.name === `agent-table-${environment}` && + peer?.name === + `agent-table-${environment === "staging" ? "production" : "staging"}` && + vars.RELEASE_PROFILE === "manual" && + /^[0-9a-f]{40}$/i.test(vars.RELEASE_BUILD_SHA) && + vars.RESTORE_QUARANTINE === "false", + "pause-release-state-unverified", + ); + requirePause( + PAUSE_SWITCHES.every((name) => vars[name] === "false"), + "pause-config-not-disabled", + ); + requirePause( + VERSION_ID.test(expectedVersionId ?? ""), + "pause-expected-version-invalid", + ); + const { targetState } = await readCloudflareDeployment( + config, + peer, + cloudflareRead, + ); + // The provider reader already compares each safety variable across service + // settings and the 100%-traffic active version. Verify the intended values too. + verifyDeployedConfiguration(targetState, config); + requirePause( + targetState.versionId === expectedVersionId, + "pause-version-id-mismatch", + ); + return { + verified: true, + environment, + workerName: targetState.workerName, + versionId: targetState.versionId, + deploymentId: targetState.deploymentId, + buildSha: vars.RELEASE_BUILD_SHA, + admission: "paused", + }; +} + +function cliOptions() { + const argv = process.argv.slice(2); + const names = new Set(["--config", "--peer-config", "--expected-version-id"]); + requirePause(argv.length === names.size * 2, "pause-arguments-invalid"); + const options = new Map(); + for (let index = 0; index < argv.length; index += 2) { + const name = argv[index]; + const value = argv[index + 1]; + requirePause( + names.has(name) && + !options.has(name) && + typeof value === "string" && + value.length > 0 && + !value.startsWith("--"), + "pause-arguments-invalid", + ); + options.set(name, value); + } + return options; +} + +async function main() { + try { + const options = cliOptions(); + const result = await verifyNormalPause({ + config: JSON.parse(readFileSync(options.get("--config"), "utf8")), + peer: JSON.parse(readFileSync(options.get("--peer-config"), "utf8")), + expectedVersionId: options.get("--expected-version-id"), + cloudflareRead: cloudflareReader(process.env.CLOUDFLARE_READ_TOKEN), + }); + process.stdout.write(`PAUSE_READBACK_VERIFIED ${JSON.stringify(result)}\n`); + } catch (error) { + const code = + error instanceof ProviderVerificationError + ? error.message + : "pause-readback-unverified"; + process.stderr.write(`PAUSE_READBACK_BLOCKED ${code}\n`); + process.exitCode = 1; + } +} + +if ( + process.argv[1] && + import.meta.url === pathToFileURL(process.argv[1]).href +) { + await main(); +} diff --git a/scripts/release-pause-readback.test.mjs b/scripts/release-pause-readback.test.mjs new file mode 100644 index 0000000..b8e569b --- /dev/null +++ b/scripts/release-pause-readback.test.mjs @@ -0,0 +1,149 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { + makeConfig, + makeProviderMock, +} from "../test/helpers/release-fixtures.mjs"; +import { verifyNormalPause } from "./release-pause-readback.mjs"; + +const expectedVersionId = "bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb"; + +function pausedFixture(environment = "staging") { + const config = makeConfig(environment); + const peer = makeConfig(environment === "staging" ? "production" : "staging"); + config.vars.ALLOW_NEW_TABLES = "false"; + peer.vars.ALLOW_NEW_TABLES = "false"; + return { config, peer, mock: makeProviderMock(config, peer) }; +} + +function targetBindings(mock, config) { + const base = `/client/v4/accounts/${config.account_id}/workers/scripts/${config.name}`; + return { + settings: mock.routes.get(`${base}/settings`).bindings, + version: mock.routes.get(`${base}/versions/${expectedVersionId}`).resources + .bindings, + }; +} + +test("normal staging and production pauses prove exact active version and disabled intake", async () => { + for (const environment of ["staging", "production"]) { + const { config, peer, mock } = pausedFixture(environment); + const result = await verifyNormalPause({ + config, + peer, + expectedVersionId, + cloudflareRead: mock.read, + }); + assert.equal(result.verified, true); + assert.equal(result.environment, environment); + assert.equal(result.versionId, expectedVersionId); + assert.equal(result.admission, "paused"); + assert.deepEqual(Object.keys(result).sort(), [ + "admission", + "buildSha", + "deploymentId", + "environment", + "verified", + "versionId", + "workerName", + ]); + } +}); + +test("candidate with admission or delivery enabled is rejected before provider access", async () => { + const { config, peer } = pausedFixture(); + config.vars.ALLOW_NEW_TABLES = "true"; + await assert.rejects( + verifyNormalPause({ + config, + peer, + expectedVersionId, + cloudflareRead: () => { + throw new Error("provider must not be contacted"); + }, + }), + /pause-config-not-disabled/, + ); + config.vars.ALLOW_NEW_TABLES = "false"; + config.vars.ENABLE_OPENAI_EVENTS = "true"; + await assert.rejects( + verifyNormalPause({ + config, + peer, + expectedVersionId, + cloudflareRead: () => {}, + }), + /pause-config-not-disabled/, + ); +}); + +test("Wrangler rollback retaining newer service settings fails closed in both directions", async () => { + for (const surface of ["settings", "version"]) { + const { config, peer, mock } = pausedFixture(); + targetBindings(mock, config)[surface].find( + (item) => item.name === "ALLOW_NEW_TABLES", + ).text = "true"; + await assert.rejects( + verifyNormalPause({ + config, + peer, + expectedVersionId, + cloudflareRead: mock.read, + }), + /provider-version-config-unverified/, + ); + } +}); + +test("matching but enabled provider settings and version cannot claim a pause", async () => { + const { config, peer, mock } = pausedFixture(); + for (const binding of Object.values(targetBindings(mock, config))) { + binding.find((item) => item.name === "ALLOW_NEW_GUESTS").text = "true"; + } + await assert.rejects( + verifyNormalPause({ + config, + peer, + expectedVersionId, + cloudflareRead: mock.read, + }), + /provider-deployed-config-mismatch/, + ); +}); + +test("a valid but unexpected deployed version denies the rollback receipt", async () => { + const { config, peer, mock } = pausedFixture(); + await assert.rejects( + verifyNormalPause({ + config, + peer, + expectedVersionId: "cccccccc-cccc-4ccc-8ccc-cccccccccccc", + cloudflareRead: mock.read, + }), + /pause-version-id-mismatch/, + ); +}); + +test("unknown release state and invalid version IDs deny before provider access", async () => { + const { config, peer, mock } = pausedFixture(); + config.vars.RESTORE_QUARANTINE = "true"; + await assert.rejects( + verifyNormalPause({ + config, + peer, + expectedVersionId, + cloudflareRead: mock.read, + }), + /pause-release-state-unverified/, + ); + config.vars.RESTORE_QUARANTINE = "false"; + await assert.rejects( + verifyNormalPause({ + config, + peer, + expectedVersionId: "not-an-uuid", + cloudflareRead: mock.read, + }), + /pause-expected-version-invalid/, + ); +});