fix(responses): settle a credential hop where the replay is dispatched (#4709)

docs-site/src/content/docs/reference/management-api.md

@@ -246,6 +246,22 @@ final provider. Custom destinations and historic rows omit the field; consumers
 infer subscription usage from the current configuration, model name, or inbound API key.
 The log reports usage, not subscription invoice amounts.
 
+API-key attempts also record `accountLogLabel` as `k` followed by 32 lowercase hex digits.
+The label is the first 128 bits of SHA-256 over
+`JSON.stringify(["ocx-key-account-v1", providerName, entryId ?? null, reference])`.
+The reference is the configured key value captured for the physical request, before environment
+or keychain resolution. Raw keys, references, and pool IDs are not written to the label field.
+A consumer can derive the same label from its local configuration without resolving secrets.
+Changing a literal key or reference changes the label; replacing the secret behind an unchanged
+reference keeps the same logical account. Older unlabeled records cannot be attributed reliably.
+
+Key selection is recorded after queued requests have been rebuilt for the current selection.
+When a retry changes keys, `attempts` retains a separate record for the preceding key, including
+reported usage from failed responses. Missing usage remains unreported. Routed adapter terminals
+are observed before image/search loops or continuation guards combine their usage. Consumers
+sum the flat attempts by provider/account and do not add the parent combo total again. These
+records identify usage; provider quota percentages remain separate upstream observations.
+
 `GET /api/usage` reads `~/.opencodex/usage.jsonl` from the beginning through the current ledger
 snapshot on a cold start. It processes fixed 1 MiB chunks and retains compact aggregate state rather
 than every normalized request row. Later refreshes validate the previous line boundary and fold only

scripts/test-layout/layout.json

@@ -167,7 +167,11 @@
     }
   },
   "explicit": {
+    "key-attribution.test.ts": "usage",
     "responses-core-modules.test.ts": "responses",
+    "responses-spend-ledger-wiring.test.ts": "responses",
+    "responses-send-budget-errors.test.ts": "responses",
+    "responses-4546-incident-regression.test.ts": "responses",
     "chat-responses-control-integration.test.ts": "responses",
     "coding-agent-tool-result-images.test.ts": "adapters",
     "hub-usage.test.ts": "server",

src/adapters/command-code.ts

@@ -469,7 +469,7 @@ async function fetchCommandCode(request: AdapterRequest, ctx: AdapterFetchContex
   const timer = setTimeout(() => timeout.abort(new DOMException("Timeout elapsed", "TimeoutError")), ctx?.timeoutMs ?? 200_000);
   const callerSignal = ctx?.abortSignal ?? new AbortController().signal;
   try {
-    return await executor(request.url, {
+    return await (ctx?.executor ?? executor)(request.url, {
       method: request.method,
       headers: request.headers,
       body: request.body,

src/codex/account-label.ts

@@ -1,17 +1,19 @@
 import { createHash, randomBytes } from "node:crypto";
 import type { CodexAccount, OcxConfig } from "../types";
 import type { CodexAuthContext } from "./auth-context";
+import type { ProviderApiKeySelection } from "../types/provider";
 import { MAIN_CODEX_ACCOUNT_ID } from "./main-account";
 
 export const CODEX_ACCOUNT_LOG_LABEL_RE = /^p[a-f0-9]{6}$/;
 
 /**
- * Account log labels come in two families (#2699):
+ * Account log labels come in three families:
  *
  * - `p<hex6>` (plus the literal `main`) — a Codex pool account.
  * - `o<hex6>` — a non-Codex OAuth provider account (xai, cursor, and siblings).
+ * - `k<hex32>` — a request-owned API-key selection, scoped to provider and reference.
  *
- * Both are sha256-derived digests, never an email and never a raw provider account id. That is
+ * Labels never contain an email, raw key/reference, or raw provider account id. That is
  * a privacy requirement, not a formatting preference: these labels are written to the usage log
  * and served over the management API.
  *
@@ -20,7 +22,16 @@ export const CODEX_ACCOUNT_LOG_LABEL_RE = /^p[a-f0-9]{6}$/;
  * accepted cost of keeping the existing `p` format byte-compatible.
  */
 export const OAUTH_ACCOUNT_LOG_LABEL_RE = /^o[a-f0-9]{6}$/;
-export const ACCOUNT_LOG_LABEL_RE = /^(?:main|[po][a-f0-9]{6})$/;
+export const KEY_ACCOUNT_LOG_LABEL_RE = /^k[a-f0-9]{32}$/;
+export const ACCOUNT_LOG_LABEL_RE = /^(?:main|[po][a-f0-9]{6}|k[a-f0-9]{32})$/;
+
+/** Digest the request-owned configured selection, never serialize its key/reference. */
+export function apiKeyAccountLogLabel(provider: string, selection: ProviderApiKeySelection | undefined): `k${string}` | undefined {
+  if (!selection || typeof selection.reference !== "string" || !selection.reference.length) return undefined;
+  return `k${createHash("sha256").update(JSON.stringify([
+    "ocx-key-account-v1", provider, selection.entryId ?? null, selection.reference,
+  ])).digest("hex").slice(0, 32)}`;
+}
 
 export function oauthAccountLogLabel(accountId: string, provider = ""): string {
   return `o${createHash("sha256").update(`${provider}\0${accountId}`).digest("hex").slice(0, 6)}`;

src/lib/errors.ts

@@ -7,6 +7,15 @@ export interface OcxErrorPayload {
 export const ENCRYPTED_FUNCTION_OUTPUT_REJECTION =
   "Encrypted function output content could not be decrypted or decoded.";
 
+/**
+ * The error identity for a send this proxy declined to make (#4708).
+ *
+ * Declared here rather than only on the error class because the classifier is what decides
+ * whether the identity survives serialization, and every dispatch path has to name the same
+ * string for a client to be able to tell this apart from a provider rate limit.
+ */
+export const SEND_BUDGET_EXHAUSTED_CODE = "request_send_budget_exhausted";
+
 /** Canonical human-readable message paths used by Responses upstream failures. */
 export function upstreamErrorMessageFromPayload(payload: unknown): string | undefined {
   if (!payload || typeof payload !== "object" || Array.isArray(payload)) return undefined;
@@ -253,6 +262,14 @@ export function classifyError(status: number, type: string, message: string): Oc
   ) {
     return { message, type: "insufficient_quota", code: "insufficient_quota" };
   }
+  // A refusal this proxy made itself, kept apart from the provider rate limits below. The HTTP
+  // semantics are identical -- 429, do not send this again now -- but the code is the only thing
+  // that tells an operator reading a log whether the provider throttled the request or whether
+  // this process declined to send it. Folding it into the generic rate-limit code sent them to
+  // the provider's dashboard to explain a decision that was never made there.
+  if (type === SEND_BUDGET_EXHAUSTED_CODE) {
+    return { message, type: "rate_limit_error", code: SEND_BUDGET_EXHAUSTED_CODE };
+  }
   if (
     status === 429 ||
     text.includes("rate limit") ||

src/lib/request-execution-budget.ts

@@ -56,6 +56,7 @@ export type BudgetDenial =
   | "final-recovery-spent"
   | "alternate-target-exhausted"
   | "target-transition-exhausted"
+  | "spend-exhausted"
   | "not-replay-safe";
 
 export interface DispatchIntent {
@@ -94,12 +95,45 @@ export interface SingleUseDispatchPermit {
    * once an external send reporter already settled it.
    */
   release(): void;
+  /**
+   * Take over an externally counted booking, because the layer holding this permit is the one
+   * that physically sends.
+   *
+   * `countedExternally` promises that a retry helper will name this send through
+   * `onSendsConsumed`. An adapter that owns its own dispatch ladder -- Kiro's reset loop,
+   * Cursor's transport loop -- reserves per physical send instead, so no reporter ever arrives
+   * and the pending booking would sit there until it silently swallowed an unrelated later
+   * report. Confirming through this method settles the permit AND closes the booking, so the
+   * send stays charged exactly once (#4709). Returns false once the permit is settled, which is
+   * what keeps one permit from admitting two sends.
+   */
+  assumeCharge(): boolean;
 }
 
 export type DispatchDecision =
   | { allowed: true; permit: SingleUseDispatchPermit }
   | { allowed: false; reason: BudgetDenial };
 
+/**
+ * Notified when this request's physical-send count moves.
+ *
+ * `spent` is the only number here that counts SENDS rather than intentions: a reservation
+ * increments it, a refund decrements it, and an externally reported send settles against a
+ * booking that was already counted. Anything that books one entry per increment therefore
+ * books exactly one entry per physical send -- which is what lets the durable spend ledger
+ * have a production caller without every dispatch site in the tree remembering to call it.
+ *
+ * `charge` may refuse, and a refusal denies the dispatch. That is deliberate: the ledger is
+ * the only bound here that survives a restart, so a limit it enforces has to be able to stop a
+ * send rather than merely describe one.
+ */
+export interface RequestSendObserver {
+  /** Book one physical send. False refuses the dispatch before the budget charges it. */
+  charge(): boolean;
+  /** Give back a booking whose send never happened. */
+  refund(): void;
+}
+
 /**
  * Carried on HandleResponsesOptions so a combo child, a rebuild and an alternate-account leg
  * all decrement the same holder. `used` is the existing #4605 counter and still counts every
@@ -137,6 +171,7 @@ let logicalRequestSeq = 0;
 export function createRequestExecutionBudget(
   policy: RequestExecutionBudgetPolicy = CODEX_TEXT_GUARDED_BUDGET_POLICY,
   logicalRequestId?: string,
+  observer?: RequestSendObserver,
 ): RequestExecutionBudget {
   let spent = 0;
   // Reservations whose physical send is reported by a retry helper rather than by the permit.
@@ -160,7 +195,12 @@ export function createRequestExecutionBudget(
       }
       const settled = Math.min(delta, pendingExternalSends);
       pendingExternalSends -= settled;
-      spent += delta - settled;
+      const charged = delta - settled;
+      spent += charged;
+      // These sends have already left. The ledger records them even past a ceiling it would
+      // have refused, because refusing after the fact only hides spend that was really
+      // incurred -- the refusal has to happen at the reservation below, or not at all.
+      for (let index = 0; index < charged; index += 1) observer?.charge();
     },
     logicalRequestId: logicalRequestId ?? `lr-${Date.now().toString(36)}-${(logicalRequestSeq += 1).toString(36)}`,
     policyVersion: REQUEST_BUDGET_POLICY_VERSION,
@@ -200,6 +240,11 @@ export function createRequestExecutionBudget(
         }
       }
 
+      // Consulted last, because it is the only bound here that WRITES. A ledger entry booked
+      // for a dispatch a cheaper check above would have refused is spend this request never
+      // makes, and it would hold those tokens against the scope until retention expired.
+      if (observer && !observer.charge()) return { allowed: false, reason: "spend-exhausted" };
+
       // THE RESERVATION IS THE CHARGE. Deciding here and charging in `use()` left a window in
       // which two legs read the same remainder, both received a permit, and both dispatched:
       // one remaining send admitted two physical sends, which is the per-request multiplication
@@ -222,6 +267,17 @@ export function createRequestExecutionBudget(
             settled = "used";
             return true;
           },
+          assumeCharge(): boolean {
+            if (settled !== "open") return false;
+            settled = "used";
+            // The booking this reservation made for an external reporter is now owned by the
+            // caller. Leaving it pending is not harmless: the next `used` report of this request
+            // would settle against it and one real send would go uncharged.
+            if (intent.countedExternally === true && pendingExternalSends > 0) {
+              pendingExternalSends -= 1;
+            }
+            return true;
+          },
           release(): void {
             if (settled !== "open") return;
             settled = "released";
@@ -232,6 +288,7 @@ export function createRequestExecutionBudget(
               pendingExternalSends -= 1;
             }
             spent -= 1;
+            observer?.refund();
             if (drawsReserve) reserveSpent = false;
             if (isAlternateTarget) alternateTargetSends -= 1;
             if (changesTarget) targetTransitions -= 1;

src/lib/spend-reservation-ledger.ts

@@ -669,6 +669,24 @@ export function createSpendReservationLedger(options: {
         case "checkpoint": applyCheckpoint(record); break;
       }
     }
+    // A reservation that survived replay has no owner left. The process that made it is gone,
+    // so nothing in this one can ever settle it, and leaving it live means the send stays
+    // pending forever against a scope that can never resolve it. Deleting the entry is not the
+    // alternative either: that would hand the same send id a second reservation.
+    //
+    // Both live states resolve to UNRESOLVED, including an undispatched one. The tempting
+    // distinction -- open never reached the wire, so give its tokens back -- assumes the
+    // journal is complete up to the crash, and the torn-tail handling above says it is not: a
+    // send can dispatch and die before its dispatch record lands. Abandoning that reservation
+    // returns tokens for a send that may have been billed, and worse, it RESETS a ceiling that
+    // had already fired. An exhausted scope staying exhausted across a restart is the whole
+    // reason this store is on disk.
+    const reconciledAt = now();
+    for (const [send, reservation] of reservations) {
+      if (!isLive(reservation.status)) continue;
+      applyResolve(send, "lost", 0, reconciledAt);
+      append({ v: 1, kind: "lost", send, at: reconciledAt });
+    }
   }
 
   /**

src/providers/label.ts

@@ -1,10 +1,28 @@
-import { CODEX_ACCOUNT_LOG_LABEL_RE, oauthAccountLogLabel } from "../codex/account-label";
+import { CODEX_ACCOUNT_LOG_LABEL_RE, KEY_ACCOUNT_LOG_LABEL_RE, apiKeyAccountLogLabel, oauthAccountLogLabel } from "../codex/account-label";
 import type { OcxProviderConfig } from "../types";
 
 export function canonicalUsageProviderLabel(provider: string): string {
   return provider === "chatgpt" || provider === "openai-multi" ? "openai" : provider;
 }
 
+export function usesApiKeyAccount(provider: Pick<OcxProviderConfig, "authMode" | "_apiKeyAttempt">): boolean {
+  return provider.authMode === "key"
+    || (provider.authMode === undefined && !!provider._apiKeyAttempt?.reference);
+}
+
+/** Key identity comes from the captured selection, before env/keychain resolution. */
+export function stampApiKeyAccountLabel(
+  logCtx: { accountLogLabel?: string },
+  providerName: string,
+  provider: Pick<OcxProviderConfig, "authMode" | "_apiKeyAttempt">,
+): void {
+  if (usesApiKeyAccount(provider)) {
+    logCtx.accountLogLabel = apiKeyAccountLogLabel(providerName, provider._apiKeyAttempt);
+  } else if (KEY_ACCOUNT_LOG_LABEL_RE.test(logCtx.accountLogLabel ?? "")) {
+    delete logCtx.accountLogLabel;
+  }
+}
+
 export function baseProviderLabel(provider: string): string {
   const canonical = canonicalUsageProviderLabel(provider);
   if (canonical !== provider) return canonical;

src/routing/probe-lease.ts

@@ -349,7 +349,14 @@ export function resolveHeldAccountDispatch(input: {
     kind: "withheld",
     boundAccountId: input.boundAccountId,
     ...(input.detourAccountId !== undefined ? { detourAccountId: input.detourAccountId } : {}),
-    retryAt: nextProbeAt(input.boundAccountId, now, input.minProbeIntervalMs),
+    // Both bounds, not just the probe pacing. A request refused by the RATIO has no probe state
+    // of its own yet, so `nextProbeAt` answered `now` and the refusal told the caller to try
+    // again immediately -- a withheld dispatch that busy-loops is the same load as the dispatch
+    // it refused. The limiter is the only thing that knows when its window moves.
+    retryAt: Math.max(
+      nextProbeAt(input.boundAccountId, now, input.minProbeIntervalMs),
+      limiter.nextRecoveryAt(now),
+    ),
   };
 }
 
@@ -408,6 +415,16 @@ export interface PoolBackpressureLimiter {
   tryPermitRetryDispatch(now?: number): boolean;
   /** Admit one probe dispatch under the same shared recovery budget. */
   tryPermitProbeDispatch(now?: number): boolean;
+  /**
+   * Earliest moment this limiter could admit another recovery dispatch.
+   *
+   * A refusal has to hand back a time, or the caller has nothing to wait on and busy-loops
+   * against a pool that is already failing -- which is the load this limiter exists to remove.
+   * `now` when the allowance is not spent; otherwise the moment the oldest bucket still inside
+   * the window falls out of it, which is strictly in the future and is a real change point
+   * rather than a guess.
+   */
+  nextRecoveryAt(now?: number): number;
   state(now?: number): PoolBackpressureState;
 }
 
@@ -461,6 +478,19 @@ export function createPoolBackpressureLimiter(
     return true;
   }
 
+  function nextRecoveryAt(now: number): number {
+    const { initials, recoveries } = totals(now);
+    if (recoveries + 1 <= allowanceFor(initials)) return now;
+    // The window has to move before another recovery fits. The earliest that can happen is the
+    // moment the oldest bucket still inside it leaves, and every such bucket started after
+    // `now - windowMs`, so the answer is always strictly in the future.
+    for (const bucket of buckets) {
+      if (bucket.start <= now - policy.windowMs) continue;
+      return bucket.start + policy.windowMs;
+    }
+    return now + policy.windowMs;
+  }
+
   return {
     recordInitialSend(now = Date.now()): void {
       bucketFor(now).initials += 1;
@@ -471,6 +501,9 @@ export function createPoolBackpressureLimiter(
     tryPermitProbeDispatch(now = Date.now()): boolean {
       return tryPermit(now);
     },
+    nextRecoveryAt(now = Date.now()): number {
+      return nextRecoveryAt(now);
+    },
     state(now = Date.now()): PoolBackpressureState {
       const { initials, recoveries } = totals(now);
       return {