Support AGENT_SECRETS_KEY rotation with a keyring and rewrap command

apps/api/.env.example

@@ -11,6 +11,11 @@ STATE_SIGNING_SECRET=replace_me_32_byte_random_string
 # Encrypts integration secrets at rest (AWS connect external IDs, Slack/Linear
 # tokens). Required for any integration that stores a secret — without it those
 # routes 500. Must base64-decode to exactly 32 bytes: openssl rand -base64 32
+# To rotate, use a keyring instead of a single key: "2:<new key>,1:<old key>".
+# The first entry encrypts new values; later entries only decrypt. A single key
+# is version 1. After every service runs with the keyring, re-encrypt stored
+# values with `pnpm --filter @superlog/db credentials:rewrap` (try --dry-run
+# first), then drop the old entry.
 AGENT_SECRETS_KEY=
 # OAuth provider credentials for Better Auth (set per-environment).
 GOOGLE_CLIENT_ID=

apps/api/src/cloudflare.ts

@@ -17,6 +17,7 @@
 // exactly like the Slack connector's /slack/oauth/callback.
 
 import {
+  LEGACY_INTEGRATION_SECRET_KEY_VERSION,
   db,
   decryptIntegrationSecret,
   encryptIntegrationSecret,
@@ -179,7 +180,7 @@ async function ensureIngestKey(
       const ingestKey = decryptIntegrationSecret({
         ciphertext: existing.ingestKeyCiphertext,
         nonce: existing.ingestKeyNonce,
-        keyVersion: existing.ingestKeyKeyVersion ?? 1,
+        keyVersion: existing.ingestKeyKeyVersion ?? LEGACY_INTEGRATION_SECRET_KEY_VERSION,
       });
       return { ingestKey, apiKeyId: existing.apiKeyId, minted: false };
     }
@@ -235,7 +236,7 @@ function decryptRefreshToken(row: CloudflareInstallationRow): string | null {
     ? decryptIntegrationSecret({
         ciphertext: row.refreshTokenCiphertext,
         nonce: row.refreshTokenNonce,
-        keyVersion: row.refreshTokenKeyVersion ?? 1,
+        keyVersion: row.refreshTokenKeyVersion ?? LEGACY_INTEGRATION_SECRET_KEY_VERSION,
       })
     : null;
 }

apps/api/src/gcp/gcp.test.ts

@@ -47,7 +47,14 @@ test("GCP config requires the secrets key before starting OAuth", () => {
   } as NodeJS.ProcessEnv;
 
   assert.equal(gcpConfigFromEnv(env), null);
-  assert.ok(gcpConfigFromEnv({ ...env, AGENT_SECRETS_KEY: "encryption-key" }));
+  assert.equal(gcpConfigFromEnv({ ...env, AGENT_SECRETS_KEY: "encryption-key" }), null);
+  assert.ok(gcpConfigFromEnv({ ...env, AGENT_SECRETS_KEY: randomBytes(32).toString("base64") }));
+  assert.ok(
+    gcpConfigFromEnv({
+      ...env,
+      AGENT_SECRETS_KEY: `2:${randomBytes(32).toString("base64")},1:${randomBytes(32).toString("base64")}`,
+    }),
+  );
 });
 
 before(async () => {

apps/api/src/gcp/interfaces.ts

@@ -1,4 +1,4 @@
-import { db, schema } from "@superlog/db";
+import { db, isValidIntegrationSecretKeyring, schema } from "@superlog/db";
 import { eq } from "drizzle-orm";
 import type { Context, Hono } from "hono";
 import { HTTPException } from "hono/http-exception";
@@ -74,7 +74,7 @@ export function gcpConfigFromEnv(env: NodeJS.ProcessEnv = process.env): GcpConne
     !pushServiceAccountEmail ||
     !pushEndpoint ||
     !pushAudience ||
-    !env.AGENT_SECRETS_KEY
+    !isValidIntegrationSecretKeyring(env.AGENT_SECRETS_KEY)
   ) {
     return null;
   }

apps/api/src/railway.ts

@@ -20,6 +20,7 @@
 // connectors' callbacks.
 
 import {
+  LEGACY_INTEGRATION_SECRET_KEY_VERSION,
   db,
   decryptIntegrationSecret,
   encryptIntegrationSecret,
@@ -129,7 +130,7 @@ async function ensureIngestKey(
       const ingestKey = decryptIntegrationSecret({
         ciphertext: existing.ingestKeyCiphertext,
         nonce: existing.ingestKeyNonce,
-        keyVersion: existing.ingestKeyKeyVersion ?? 1,
+        keyVersion: existing.ingestKeyKeyVersion ?? LEGACY_INTEGRATION_SECRET_KEY_VERSION,
       });
       return { ingestKey, apiKeyId: existing.apiKeyId, minted: false };
     }

apps/api/src/render.ts

@@ -21,9 +21,11 @@
 // reads scoped to the workspace the user picked.
 
 import {
+  LEGACY_INTEGRATION_SECRET_KEY_VERSION,
   db,
   decryptIntegrationSecret,
   encryptIntegrationSecret,
+  integrationSecretEncryptionConfigured,
   mintApiKey,
   schema,
 } from "@superlog/db";
@@ -143,7 +145,7 @@ async function ensureIngestKey(
       const ingestKey = decryptIntegrationSecret({
         ciphertext: existing.ingestKeyCiphertext,
         nonce: existing.ingestKeyNonce,
-        keyVersion: existing.ingestKeyKeyVersion ?? 1,
+        keyVersion: existing.ingestKeyKeyVersion ?? LEGACY_INTEGRATION_SECRET_KEY_VERSION,
       });
       return { ingestKey, apiKeyId: existing.apiKeyId, minted: false };
     }
@@ -342,7 +344,7 @@ export function mountRenderAuthed(
   // Storing the pasted key requires the at-rest encryption key; without it the
   // connect flow could never complete, so the routes self-disable (and
   // system-capabilities hides the connector tile).
-  const secretsConfigured = () => !!process.env.AGENT_SECRETS_KEY;
+  const secretsConfigured = () => integrationSecretEncryptionConfigured();
 
   app.get("/api/projects/:projectId/render/installation", async (c) => {
     const ctx = await requireProjectAccess(c, c.req.param("projectId"));

apps/api/src/system-capabilities.test.ts

@@ -2,6 +2,8 @@ import { strict as assert } from "node:assert";
 import { test } from "node:test";
 import { buildSystemCapabilities } from "./system-capabilities.js";
 
+const SECRETS_KEY = Buffer.alloc(32, 7).toString("base64");
+
 test("system capabilities default to the open-core community edition", () => {
   assert.deepEqual(buildSystemCapabilities({}), {
     edition: "community",
@@ -20,7 +22,7 @@ test("system capabilities default to the open-core community edition", () => {
 
 test("renderConnect only needs AGENT_SECRETS_KEY (API-key connect, no OAuth client)", () => {
   assert.equal(buildSystemCapabilities({}).renderConnect, false);
-  assert.equal(buildSystemCapabilities({ AGENT_SECRETS_KEY: "k" }).renderConnect, true);
+  assert.equal(buildSystemCapabilities({ AGENT_SECRETS_KEY: SECRETS_KEY }).renderConnect, true);
 });
 
 test("gcpConnect requires OAuth and integration-owned Pub/Sub configuration", () => {
@@ -39,7 +41,7 @@ test("gcpConnect requires OAuth and integration-owned Pub/Sub configuration", ()
     GCP_PUBSUB_PUSH_ENDPOINT: "https://intake.example.com/gcp/pubsub",
     GCP_PUBSUB_PUSH_AUDIENCE: "https://intake.example.com/gcp/pubsub",
     STATE_SIGNING_SECRET: "state-secret",
-    AGENT_SECRETS_KEY: "encryption-key",
+    AGENT_SECRETS_KEY: SECRETS_KEY,
   };
   assert.equal(buildSystemCapabilities(complete).gcpConnect, true);
   assert.equal(
@@ -73,7 +75,7 @@ test("railwayConnect flips on only when client creds + both platform secrets are
       RAILWAY_CLIENT_ID: "id",
       RAILWAY_CLIENT_SECRET: "secret",
       STATE_SIGNING_SECRET: "s",
-      AGENT_SECRETS_KEY: "k",
+      AGENT_SECRETS_KEY: SECRETS_KEY,
     }).railwayConnect,
     true,
   );
@@ -101,7 +103,7 @@ test("cloudflareConnect flips on only when all connector vars (incl. STATE_SIGNI
       CLOUDFLARE_CLIENT_SECRET: "secret",
       CLOUDFLARE_OTLP_INTAKE_URL: "https://intake.example.com",
       STATE_SIGNING_SECRET: "state-secret",
-      AGENT_SECRETS_KEY: "k",
+      AGENT_SECRETS_KEY: SECRETS_KEY,
     }).cloudflareConnect,
     true,
   );
@@ -145,7 +147,7 @@ test("sentryConnect requires the public Sentry App and platform secrets", () =>
       SENTRY_CLIENT_SECRET: "secret",
       SENTRY_APP_SLUG: "superlog",
       STATE_SIGNING_SECRET: "state-secret",
-      AGENT_SECRETS_KEY: "encryption-key",
+      AGENT_SECRETS_KEY: SECRETS_KEY,
     }).sentryConnect,
     true,
   );
@@ -174,8 +176,40 @@ test("vercelConnect flips on only when all connector vars (incl. STATE_SIGNING_S
       VERCEL_INTEGRATION_SLUG: "superlog",
       VERCEL_OTLP_INTAKE_URL: "https://intake.example.com",
       STATE_SIGNING_SECRET: "state-secret",
-      AGENT_SECRETS_KEY: "k",
+      AGENT_SECRETS_KEY: SECRETS_KEY,
     }).vercelConnect,
     true,
   );
 });
+
+test("capabilities read the key from the env object they are given, not process.env", () => {
+  const original = process.env.AGENT_SECRETS_KEY;
+  process.env.AGENT_SECRETS_KEY = SECRETS_KEY;
+  try {
+    assert.equal(buildSystemCapabilities({}).renderConnect, false);
+  } finally {
+    if (original === undefined) Reflect.deleteProperty(process.env, "AGENT_SECRETS_KEY");
+    else process.env.AGENT_SECRETS_KEY = original;
+  }
+});
+
+test("connectors accept an AGENT_SECRETS_KEY keyring and stay hidden for a malformed key", () => {
+  const keyring = `2:${Buffer.alloc(32, 2).toString("base64")},1:${SECRETS_KEY}`;
+  assert.equal(buildSystemCapabilities({ AGENT_SECRETS_KEY: keyring }).renderConnect, true);
+  assert.equal(buildSystemCapabilities({ AGENT_SECRETS_KEY: "not-a-key" }).renderConnect, false);
+  assert.equal(
+    buildSystemCapabilities({ AGENT_SECRETS_KEY: `2:${SECRETS_KEY},2:${SECRETS_KEY}` })
+      .renderConnect,
+    false,
+  );
+  assert.equal(
+    buildSystemCapabilities({
+      SENTRY_CLIENT_ID: "id",
+      SENTRY_CLIENT_SECRET: "secret",
+      SENTRY_APP_SLUG: "superlog",
+      STATE_SIGNING_SECRET: "state-secret",
+      AGENT_SECRETS_KEY: "encryption-key",
+    }).sentryConnect,
+    false,
+  );
+});

apps/api/src/system-capabilities.ts

@@ -1,3 +1,5 @@
+import { isValidIntegrationSecretKeyring } from "@superlog/db/integration-secrets";
+
 export type SuperlogEdition = "community" | "cloud" | "private";
 export type BillingProvider = "none" | "stripe";
 
@@ -68,12 +70,14 @@ export function buildSystemCapabilities(env: CapabilityEnv = process.env): Syste
   // All three OAuth connectors also need AGENT_SECRETS_KEY — provisioning
   // encrypts the granted tokens/ingest keys at rest and throws without it, so
   // advertising the connector would offer a connect that can never complete.
+  // The key is validated (single key or keyring), not just checked for presence.
+  const secretsKeyConfigured = isValidIntegrationSecretKeyring(env.AGENT_SECRETS_KEY);
   const cloudflareConnect = !!(
     env.CLOUDFLARE_CLIENT_ID &&
     env.CLOUDFLARE_CLIENT_SECRET &&
     env.CLOUDFLARE_OTLP_INTAKE_URL &&
     env.STATE_SIGNING_SECRET &&
-    env.AGENT_SECRETS_KEY
+    secretsKeyConfigured
   );
   // Mirrors vercelConfigFromEnv's required-vars check plus the install-url
   // route's STATE_SIGNING_SECRET requirement, same as the Cloudflare gate.
@@ -83,19 +87,19 @@ export function buildSystemCapabilities(env: CapabilityEnv = process.env): Syste
     env.VERCEL_INTEGRATION_SLUG &&
     env.VERCEL_OTLP_INTAKE_URL &&
     env.STATE_SIGNING_SECRET &&
-    env.AGENT_SECRETS_KEY
+    secretsKeyConfigured
   );
   // Mirrors railwayConfigFromEnv's required-vars check plus STATE_SIGNING_SECRET.
   const railwayConnect = !!(
     env.RAILWAY_CLIENT_ID &&
     env.RAILWAY_CLIENT_SECRET &&
     env.STATE_SIGNING_SECRET &&
-    env.AGENT_SECRETS_KEY
+    secretsKeyConfigured
   );
   // Render connect is an API-key paste, not OAuth — no client id/secret and no
   // signed state. Storing the pasted key still needs the at-rest encryption
   // key, so that's the whole gate.
-  const renderConnect = !!env.AGENT_SECRETS_KEY;
+  const renderConnect = secretsKeyConfigured;
   const gcpPushAudience = env.GCP_PUBSUB_PUSH_AUDIENCE ?? env.GCP_PUBSUB_PUSH_ENDPOINT;
   const gcpConnect = !!(
     env.GCP_OAUTH_CLIENT_ID &&
@@ -107,14 +111,14 @@ export function buildSystemCapabilities(env: CapabilityEnv = process.env): Syste
     env.GCP_PUBSUB_PUSH_ENDPOINT &&
     gcpPushAudience &&
     env.STATE_SIGNING_SECRET &&
-    env.AGENT_SECRETS_KEY
+    secretsKeyConfigured
   );
   const sentryConnect = !!(
     env.SENTRY_CLIENT_ID &&
     env.SENTRY_CLIENT_SECRET &&
     env.SENTRY_APP_SLUG &&
     env.STATE_SIGNING_SECRET &&
-    env.AGENT_SECRETS_KEY
+    secretsKeyConfigured
   );
 
   return {

apps/api/src/vercel.ts

@@ -17,6 +17,7 @@
 // and Slack connectors' callbacks.
 
 import {
+  LEGACY_INTEGRATION_SECRET_KEY_VERSION,
   db,
   decryptIntegrationSecret,
   encryptIntegrationSecret,
@@ -122,7 +123,7 @@ async function ensureIngestKey(
       const ingestKey = decryptIntegrationSecret({
         ciphertext: existing.ingestKeyCiphertext,
         nonce: existing.ingestKeyNonce,
-        keyVersion: existing.ingestKeyKeyVersion ?? 1,
+        keyVersion: existing.ingestKeyKeyVersion ?? LEGACY_INTEGRATION_SECRET_KEY_VERSION,
       });
       return { ingestKey, apiKeyId: existing.apiKeyId, minted: false };
     }
@@ -317,10 +318,7 @@ async function provisionInstallation(input: {
   // revoked independently (otherwise both installs share one live key).
   const sameConfiguration =
     existing?.configurationId === input.token.installationId ? existing : null;
-  const { ingestKey, apiKeyId, minted } = await ensureIngestKey(
-    input.projectId,
-    sameConfiguration,
-  );
+  const { ingestKey, apiKeyId, minted } = await ensureIngestKey(input.projectId, sameConfiguration);
 
   // Undo everything this run created that isn't safely persisted to an install
   // row: the new remote drains (minus any id the prior same-configuration row
@@ -436,10 +434,7 @@ async function provisionInstallation(input: {
         installedByUserId: input.userId,
       })
       .onConflictDoUpdate({
-        target: [
-          schema.vercelInstallations.projectId,
-          schema.vercelInstallations.configurationId,
-        ],
+        target: [schema.vercelInstallations.projectId, schema.vercelInstallations.configurationId],
         set: {
           teamId: input.token.teamId,
           teamName,

apps/worker/src/jobs/cloudflare-reconcile.ts

@@ -17,7 +17,12 @@ import {
   reconcileWorkerWiring,
   refreshAccessToken,
 } from "@superlog/cloudflare";
-import { decryptIntegrationSecret, encryptIntegrationSecret, schema } from "@superlog/db";
+import {
+  LEGACY_INTEGRATION_SECRET_KEY_VERSION,
+  decryptIntegrationSecret,
+  encryptIntegrationSecret,
+  schema,
+} from "@superlog/db";
 import { and, eq, isNull, sql } from "drizzle-orm";
 import { lockInstallation } from "../cloudflare/lock.js";
 import {
@@ -129,7 +134,7 @@ function createStore(
             refreshToken = decryptIntegrationSecret({
               ciphertext: cur.refreshTokenCiphertext,
               nonce: cur.refreshTokenNonce,
-              keyVersion: cur.refreshTokenKeyVersion ?? 1,
+              keyVersion: cur.refreshTokenKeyVersion ?? LEGACY_INTEGRATION_SECRET_KEY_VERSION,
             });
           } catch {
             refreshToken = null;

apps/worker/src/jobs/cloudflare-refresh.ts

@@ -9,7 +9,12 @@
 // configured, since without it there's nothing to refresh with.
 
 import { cloudflareClientFromEnv } from "@superlog/cloudflare";
-import { decryptIntegrationSecret, encryptIntegrationSecret, schema } from "@superlog/db";
+import {
+  LEGACY_INTEGRATION_SECRET_KEY_VERSION,
+  decryptIntegrationSecret,
+  encryptIntegrationSecret,
+  schema,
+} from "@superlog/db";
 import { and, eq, isNull } from "drizzle-orm";
 import { lockInstallation } from "../cloudflare/lock.js";
 import {
@@ -65,7 +70,7 @@ function createStore(db: JobDeps["db"]): CloudflareRefresherStore {
               refreshToken = decryptIntegrationSecret({
                 ciphertext: cur.refreshTokenCiphertext,
                 nonce: cur.refreshTokenNonce,
-                keyVersion: cur.refreshTokenKeyVersion ?? 1,
+                keyVersion: cur.refreshTokenKeyVersion ?? LEGACY_INTEGRATION_SECRET_KEY_VERSION,
               });
             } catch (err) {
               // A row whose refresh token can't be decrypted (corrupted

apps/worker/src/jobs/gcp-metrics-pull.test.ts

@@ -21,6 +21,13 @@ test("the GCP metrics job stays disabled without the integration-secrets key", (
   assert.equal(job.create({} as JobDeps), null);
 });
 
+test("the GCP metrics job stays disabled when the integration-secrets key is malformed", () => {
+  process.env.GCP_INTEGRATION_PROJECT_ID = "superlog-observability";
+  process.env.AGENT_SECRETS_KEY = "not-a-key";
+
+  assert.equal(job.create({} as JobDeps), null);
+});
+
 test("metric intake acknowledges permanent rejections but retries transient failures", () => {
   for (const status of [200, 400, 402, 413]) {
     assert.equal(isMetricIntakeAcknowledged(status), true);