From b7cee595811278f563f631b0985ef2e86f26cb15 Mon Sep 17 00:00:00 2001 From: Kunj Hirapara Date: Sun, 13 Sep 2026 20:00:20 +0530 Subject: [PATCH] fix(deploy): report and gate on whether the app can sign anyone in The Deploy check verifies that the VM is serving the commit that was built. It does not verify that the commit works, and for this particular rollout those are very different things. The image that switches sign-in to Auth.js can reach the VM before anyone sets the Auth.js environment variables. Nothing about that looks broken from outside: the app boots, serves every page, and /api/health returns "healthy" -- because its integration flags all predate Auth.js and check Clerk-era variables. The only symptom is that nobody can log in, and the deploy that caused it is green. So /api/health now reports an `auth` flag, and verify-rollout fails on it once the rollout itself has been confirmed. Publishing was not shipping and shipping is not working; this is the same lesson one step further along. The flag is a bare boolean. This endpoint is public -- the container healthcheck polls it with no session -- and publishing the names of the secrets a deployment is missing is a map for anyone probing it. The names go to the container log, where an operator can act on them. Five variables count as required: AUTH_SECRET, AUTH_ADAPTER_SECRET, and the three AUTH_JWT_* values. Deliberately not the OAuth client IDs -- a deployment with no Google app is degraded rather than broken, since credentials and magic-link sign-in still work, and firing this flag for something a user can work around would make it mean less when it fires for something they cannot. Two things that would have made the check useless: jq's // operator treats false as unset, so `.integrations.auth // empty` returns empty for auth:false -- precisely the case being checked -- and the gate would have passed silently. It tests for the key explicitly instead. An image built before this change has no `auth` field at all. That reads as empty and is deliberately not a failure: an old image has no opinion, and failing on it would block the very rollout that introduces the flag. --- .github/workflows/deploy.yml | 29 +++++++++++++++ src/app/api/health/route.ts | 23 ++++++++++++ src/lib/auth/readiness.test.ts | 64 ++++++++++++++++++++++++++++++++++ src/lib/auth/readiness.ts | 55 +++++++++++++++++++++++++++++ 4 files changed, 171 insertions(+) create mode 100644 src/lib/auth/readiness.test.ts create mode 100644 src/lib/auth/readiness.ts diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 204cb04..5b47653 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -143,6 +143,20 @@ jobs: fi } + # Whether the running image can actually sign anyone in. Absent on any + # image built before this check existed, which reads as empty and is + # deliberately not treated as a failure -- an old image has no opinion. + read_auth() { + if command -v jq >/dev/null 2>&1; then + # NOT `.integrations.auth // empty`. jq's // treats false as unset, + # so that form returns empty for auth:false -- the one case this + # check exists to catch -- and the gate would pass silently. + jq -r 'if (.integrations | has("auth")) then (.integrations.auth | tostring) else "" end' 2>/dev/null + else + sed -n 's/.*"auth":\(true\|false\).*/\1/p' + fi + } + # Watchtower polls every WATCHTOWER_POLL_INTERVAL seconds (300 by # default), so 12 minutes gives it at least two chances plus time to # pull an ARM image and restart. @@ -166,6 +180,21 @@ jobs: if [ -n "$live" ] && [ "$live" = "$expected" ]; then echo "::notice::Rollout confirmed after ${SECONDS}s — ${APP_URL} is serving ${expected}." + + # Shipping is not the same as working, in the same way that + # publishing was not the same as shipping. An image that reaches + # the VM before the Auth.js variables are set boots, serves pages + # and reports healthy while nobody can log in -- so a green deploy + # here would be the second lie this job exists to stop telling. + auth="$(printf '%s' "$body" | read_auth)" + + if [ "$auth" = "false" ]; then + echo "::error::${APP_URL} is serving ${expected}, but it cannot sign anyone in." + echo "::error::The Auth.js environment variables are not set on the VM. The container log names the missing ones; it does not publish them, because /api/health is public." + echo "::error::Set AUTH_SECRET, AUTH_ADAPTER_SECRET and AUTH_JWT_PRIVATE_KEY / AUTH_JWT_PUBLIC_KEY / AUTH_JWT_KID, then redeploy. AUTH_ADAPTER_SECRET must be the SAME value in the Convex environment." + exit 1 + fi + exit 0 fi diff --git a/src/app/api/health/route.ts b/src/app/api/health/route.ts index 5eca024..4f68e10 100644 --- a/src/app/api/health/route.ts +++ b/src/app/api/health/route.ts @@ -1,5 +1,6 @@ import { NextResponse } from "next/server"; import { getValidatedServerEnv } from "@/lib/env"; +import { getAuthReadiness } from "@/lib/auth/readiness"; import { getBuildVersion } from "@/lib/buildInfo"; export async function GET() { @@ -12,6 +13,27 @@ export async function GET() { try { const env = getValidatedServerEnv(); + /** + * Whether anyone can actually sign in. + * + * Not covered by the flags below, all of which predate Auth.js. An image + * that switches sign-in to Auth.js can reach a deployment before anyone + * sets the Auth.js variables, and the result is an app that boots, serves + * pages and reports healthy while nobody can log in. + * + * Reported as a bare boolean on purpose. This endpoint is public — the + * container healthcheck polls it with no session — and publishing the names + * of the secrets a deployment is missing is a map for anyone probing it. + * The names go to the server log instead, where an operator can act on them. + */ + const auth = getAuthReadiness(process.env); + + if (!auth.ready) { + console.error( + `[health] auth is not configured; sign-in will fail. Missing: ${auth.missing.join(", ")}`, + ); + } + return NextResponse.json({ status: "healthy", version, @@ -21,6 +43,7 @@ export async function GET() { convex: !!env.NEXT_PUBLIC_CONVEX_URL, stream: !!env.NEXT_PUBLIC_STREAM_API_KEY && !!env.STREAM_SECRET_KEY, webhooks: !!env.CLERK_WEBHOOK_SECRET, + auth: auth.ready, }, }); } catch (error) { diff --git a/src/lib/auth/readiness.test.ts b/src/lib/auth/readiness.test.ts new file mode 100644 index 0000000..0bba652 --- /dev/null +++ b/src/lib/auth/readiness.test.ts @@ -0,0 +1,64 @@ +import assert from "node:assert/strict"; +import { describe, it } from "node:test"; + +import { getAuthReadiness } from "./readiness.ts"; + +const complete = { + AUTH_SECRET: "s", + AUTH_ADAPTER_SECRET: "s", + AUTH_JWT_PRIVATE_KEY: "-----BEGIN PRIVATE KEY-----", + AUTH_JWT_PUBLIC_KEY: "-----BEGIN PUBLIC KEY-----", + AUTH_JWT_KID: "k1", +}; + +describe("getAuthReadiness", () => { + it("is ready when all five are set", () => { + assert.deepEqual(getAuthReadiness(complete), { ready: true, missing: [] }); + }); + + it("reports every missing name rather than stopping at the first", () => { + // An operator setting one variable per deploy is the slowest possible way + // to fix this, and it is what a first-failure-only report produces. + const result = getAuthReadiness({}); + + assert.equal(result.ready, false); + assert.deepEqual(result.missing, [ + "AUTH_SECRET", + "AUTH_ADAPTER_SECRET", + "AUTH_JWT_PRIVATE_KEY", + "AUTH_JWT_PUBLIC_KEY", + "AUTH_JWT_KID", + ]); + }); + + it("treats whitespace as unset", () => { + // An easy thing to paste by accident, and identical to unset for all five. + const result = getAuthReadiness({ ...complete, AUTH_JWT_KID: " " }); + + assert.equal(result.ready, false); + assert.deepEqual(result.missing, ["AUTH_JWT_KID"]); + }); + + it("treats an empty string as unset", () => { + const result = getAuthReadiness({ ...complete, AUTH_SECRET: "" }); + + assert.equal(result.ready, false); + assert.deepEqual(result.missing, ["AUTH_SECRET"]); + }); + + it("does not require the OAuth client IDs", () => { + // A deployment with no Google app is degraded, not broken: credentials and + // magic-link sign-in still work. Firing this flag for something a user can + // work around would make it mean less when it fires for something they + // cannot. + assert.equal(getAuthReadiness(complete).ready, true); + }); + + it("never returns a value, only a name", () => { + const secret = "super-secret-value"; + const result = getAuthReadiness({ ...complete, AUTH_SECRET: undefined, OTHER: secret }); + + assert.deepEqual(result.missing, ["AUTH_SECRET"]); + assert.ok(!JSON.stringify(result).includes(secret)); + }); +}); diff --git a/src/lib/auth/readiness.ts b/src/lib/auth/readiness.ts new file mode 100644 index 0000000..c5c3cf0 --- /dev/null +++ b/src/lib/auth/readiness.ts @@ -0,0 +1,55 @@ +/** + * Whether this deployment can actually sign anyone in. + * + * Written for a specific hazard in the Clerk migration. The image that switches + * sign-in to Auth.js can reach the VM before anyone sets the Auth.js + * environment variables, and the failure mode is total: the app boots, serves + * pages, reports healthy, and nobody can log in. `/api/health` would have said + * "healthy" throughout, because it only ever checked the Clerk-era variables. + * + * So this is a pre-flight, not a diagnostic after the fact. Check it against a + * deployment *before* letting the new image roll out to it. + * + * Names only, never values. The caller logs the missing names server-side and + * publishes a boolean, because /api/health is public — it is polled by the + * container healthcheck, which carries no session. A public endpoint listing + * exactly which secrets a deployment is missing is a map for anyone probing it. + */ + +export type AuthReadiness = { + ready: boolean; + /** Variable names, in the order an operator should set them. Never values. */ + missing: string[]; +}; + +/** + * The five that auth cannot function without. + * + * Deliberately not the OAuth client IDs. Credentials and magic-link sign-in work + * without them — a deployment with no Google app is degraded, not broken, and + * conflating the two would make this flag fire for something a user can work + * around. These five have no workaround: + * + * AUTH_SECRET signs and encrypts the session cookie + * AUTH_ADAPTER_SECRET every adapter call to Convex is rejected without it + * AUTH_JWT_PRIVATE_KEY mints the token Convex verifies + * AUTH_JWT_PUBLIC_KEY served at /.well-known/jwks.json for that verification + * AUTH_JWT_KID names the key in both halves; a mismatch rejects every token + */ +const REQUIRED = [ + "AUTH_SECRET", + "AUTH_ADAPTER_SECRET", + "AUTH_JWT_PRIVATE_KEY", + "AUTH_JWT_PUBLIC_KEY", + "AUTH_JWT_KID", +] as const; + +export const getAuthReadiness = ( + env: Record, +): AuthReadiness => { + // Trimmed, because an environment variable set to whitespace is the same as + // unset for every one of these and is an easy thing to paste by accident. + const missing = REQUIRED.filter((name) => !env[name]?.trim()); + + return { ready: missing.length === 0, missing: [...missing] }; +};