[WRONG BRANCH] fix(usage): attribute each attempt to the account that dispatched it (#4717)

docs-site/src/content/docs/reference/management-api.md

@@ -246,6 +246,22 @@ final provider. Custom destinations and historic rows omit the field; consumers
 infer subscription usage from the current configuration, model name, or inbound API key.
 The log reports usage, not subscription invoice amounts.
 
+API-key attempts also record `accountLogLabel` as `k` followed by 32 lowercase hex digits.
+The label is the first 128 bits of SHA-256 over
+`JSON.stringify(["ocx-key-account-v1", providerName, entryId ?? null, reference])`.
+The reference is the configured key value captured for the physical request, before environment
+or keychain resolution. Raw keys, references, and pool IDs are not written to the label field.
+A consumer can derive the same label from its local configuration without resolving secrets.
+Changing a literal key or reference changes the label; replacing the secret behind an unchanged
+reference keeps the same logical account. Older unlabeled records cannot be attributed reliably.
+
+Key selection is recorded after queued requests have been rebuilt for the current selection.
+When a retry changes keys, `attempts` retains a separate record for the preceding key, including
+reported usage from failed responses. Missing usage remains unreported. Routed adapter terminals
+are observed before image/search loops or continuation guards combine their usage. Consumers
+sum the flat attempts by provider/account and do not add the parent combo total again. These
+records identify usage; provider quota percentages remain separate upstream observations.
+
 `GET /api/usage` reads `~/.opencodex/usage.jsonl` from the beginning through the current ledger
 snapshot on a cold start. It processes fixed 1 MiB chunks and retains compact aggregate state rather
 than every normalized request row. Later refreshes validate the previous line boundary and fold only

scripts/test-layout/layout.json

@@ -167,9 +167,11 @@
     }
   },
   "explicit": {
+    "key-attribution.test.ts": "usage",
     "responses-core-modules.test.ts": "responses",
     "responses-spend-ledger-wiring.test.ts": "responses",
     "responses-send-budget-errors.test.ts": "responses",
+    "responses-4546-incident-regression.test.ts": "responses",
     "chat-responses-control-integration.test.ts": "responses",
     "coding-agent-tool-result-images.test.ts": "adapters",
     "hub-usage.test.ts": "server",

src/adapters/command-code.ts

@@ -469,7 +469,7 @@ async function fetchCommandCode(request: AdapterRequest, ctx: AdapterFetchContex
   const timer = setTimeout(() => timeout.abort(new DOMException("Timeout elapsed", "TimeoutError")), ctx?.timeoutMs ?? 200_000);
   const callerSignal = ctx?.abortSignal ?? new AbortController().signal;
   try {
-    return await executor(request.url, {
+    return await (ctx?.executor ?? executor)(request.url, {
       method: request.method,
       headers: request.headers,
       body: request.body,

src/codex/account-label.ts

@@ -1,17 +1,19 @@
 import { createHash, randomBytes } from "node:crypto";
 import type { CodexAccount, OcxConfig } from "../types";
 import type { CodexAuthContext } from "./auth-context";
+import type { ProviderApiKeySelection } from "../types/provider";
 import { MAIN_CODEX_ACCOUNT_ID } from "./main-account";
 
 export const CODEX_ACCOUNT_LOG_LABEL_RE = /^p[a-f0-9]{6}$/;
 
 /**
- * Account log labels come in two families (#2699):
+ * Account log labels come in three families:
  *
  * - `p<hex6>` (plus the literal `main`) — a Codex pool account.
  * - `o<hex6>` — a non-Codex OAuth provider account (xai, cursor, and siblings).
+ * - `k<hex32>` — a request-owned API-key selection, scoped to provider and reference.
  *
- * Both are sha256-derived digests, never an email and never a raw provider account id. That is
+ * Labels never contain an email, raw key/reference, or raw provider account id. That is
  * a privacy requirement, not a formatting preference: these labels are written to the usage log
  * and served over the management API.
  *
@@ -20,7 +22,16 @@ export const CODEX_ACCOUNT_LOG_LABEL_RE = /^p[a-f0-9]{6}$/;
  * accepted cost of keeping the existing `p` format byte-compatible.
  */
 export const OAUTH_ACCOUNT_LOG_LABEL_RE = /^o[a-f0-9]{6}$/;
-export const ACCOUNT_LOG_LABEL_RE = /^(?:main|[po][a-f0-9]{6})$/;
+export const KEY_ACCOUNT_LOG_LABEL_RE = /^k[a-f0-9]{32}$/;
+export const ACCOUNT_LOG_LABEL_RE = /^(?:main|[po][a-f0-9]{6}|k[a-f0-9]{32})$/;
+
+/** Digest the request-owned configured selection, never serialize its key/reference. */
+export function apiKeyAccountLogLabel(provider: string, selection: ProviderApiKeySelection | undefined): `k${string}` | undefined {
+  if (!selection || typeof selection.reference !== "string" || !selection.reference.length) return undefined;
+  return `k${createHash("sha256").update(JSON.stringify([
+    "ocx-key-account-v1", provider, selection.entryId ?? null, selection.reference,
+  ])).digest("hex").slice(0, 32)}`;
+}
 
 export function oauthAccountLogLabel(accountId: string, provider = ""): string {
   return `o${createHash("sha256").update(`${provider}\0${accountId}`).digest("hex").slice(0, 6)}`;

src/lib/spend-reservation-ledger.ts

@@ -670,23 +670,22 @@ export function createSpendReservationLedger(options: {
       }
     }
     // A reservation that survived replay has no owner left. The process that made it is gone,
-    // so nothing in this one can ever settle it, and leaving it live holds its tokens against
-    // the scope forever -- a ceiling that only ever tightens, which is the opposite of the
-    // bound this store exists to keep. Deleting the entry is not the alternative: that would
-    // hand the same send id a second reservation.
+    // so nothing in this one can ever settle it, and leaving it live means the send stays
+    // pending forever against a scope that can never resolve it. Deleting the entry is not the
+    // alternative either: that would hand the same send id a second reservation.
     //
-    // The distinction is the one the rest of the module already draws. An UNDISPATCHED
-    // reservation never reached the wire, so it is abandoned and its tokens come back. A
-    // DISPATCHED one may already have been billed, so it becomes unresolved spend. Both are
-    // appended, so the file agrees with memory and the next restart has nothing left to do.
+    // Both live states resolve to UNRESOLVED, including an undispatched one. The tempting
+    // distinction -- open never reached the wire, so give its tokens back -- assumes the
+    // journal is complete up to the crash, and the torn-tail handling above says it is not: a
+    // send can dispatch and die before its dispatch record lands. Abandoning that reservation
+    // returns tokens for a send that may have been billed, and worse, it RESETS a ceiling that
+    // had already fired. An exhausted scope staying exhausted across a restart is the whole
+    // reason this store is on disk.
     const reconciledAt = now();
     for (const [send, reservation] of reservations) {
       if (!isLive(reservation.status)) continue;
-      const abandoned = reservation.status === "open";
-      applyResolve(send, abandoned ? "abandoned" : "lost", 0, reconciledAt);
-      append(abandoned
-        ? { v: 1, kind: "abandon", send, at: reconciledAt }
-        : { v: 1, kind: "lost", send, at: reconciledAt });
+      applyResolve(send, "lost", 0, reconciledAt);
+      append({ v: 1, kind: "lost", send, at: reconciledAt });
     }
   }
 

src/providers/label.ts

@@ -1,10 +1,28 @@
-import { CODEX_ACCOUNT_LOG_LABEL_RE, oauthAccountLogLabel } from "../codex/account-label";
+import { CODEX_ACCOUNT_LOG_LABEL_RE, KEY_ACCOUNT_LOG_LABEL_RE, apiKeyAccountLogLabel, oauthAccountLogLabel } from "../codex/account-label";
 import type { OcxProviderConfig } from "../types";
 
 export function canonicalUsageProviderLabel(provider: string): string {
   return provider === "chatgpt" || provider === "openai-multi" ? "openai" : provider;
 }
 
+export function usesApiKeyAccount(provider: Pick<OcxProviderConfig, "authMode" | "_apiKeyAttempt">): boolean {
+  return provider.authMode === "key"
+    || (provider.authMode === undefined && !!provider._apiKeyAttempt?.reference);
+}
+
+/** Key identity comes from the captured selection, before env/keychain resolution. */
+export function stampApiKeyAccountLabel(
+  logCtx: { accountLogLabel?: string },
+  providerName: string,
+  provider: Pick<OcxProviderConfig, "authMode" | "_apiKeyAttempt">,
+): void {
+  if (usesApiKeyAccount(provider)) {
+    logCtx.accountLogLabel = apiKeyAccountLogLabel(providerName, provider._apiKeyAttempt);
+  } else if (KEY_ACCOUNT_LOG_LABEL_RE.test(logCtx.accountLogLabel ?? "")) {
+    delete logCtx.accountLogLabel;
+  }
+}
+
 export function baseProviderLabel(provider: string): string {
   const canonical = canonicalUsageProviderLabel(provider);
   if (canonical !== provider) return canonical;

src/routing/probe-lease.ts

@@ -349,7 +349,14 @@ export function resolveHeldAccountDispatch(input: {
     kind: "withheld",
     boundAccountId: input.boundAccountId,
     ...(input.detourAccountId !== undefined ? { detourAccountId: input.detourAccountId } : {}),
-    retryAt: nextProbeAt(input.boundAccountId, now, input.minProbeIntervalMs),
+    // Both bounds, not just the probe pacing. A request refused by the RATIO has no probe state
+    // of its own yet, so `nextProbeAt` answered `now` and the refusal told the caller to try
+    // again immediately -- a withheld dispatch that busy-loops is the same load as the dispatch
+    // it refused. The limiter is the only thing that knows when its window moves.
+    retryAt: Math.max(
+      nextProbeAt(input.boundAccountId, now, input.minProbeIntervalMs),
+      limiter.nextRecoveryAt(now),
+    ),
   };
 }
 
@@ -408,6 +415,16 @@ export interface PoolBackpressureLimiter {
   tryPermitRetryDispatch(now?: number): boolean;
   /** Admit one probe dispatch under the same shared recovery budget. */
   tryPermitProbeDispatch(now?: number): boolean;
+  /**
+   * Earliest moment this limiter could admit another recovery dispatch.
+   *
+   * A refusal has to hand back a time, or the caller has nothing to wait on and busy-loops
+   * against a pool that is already failing -- which is the load this limiter exists to remove.
+   * `now` when the allowance is not spent; otherwise the moment the oldest bucket still inside
+   * the window falls out of it, which is strictly in the future and is a real change point
+   * rather than a guess.
+   */
+  nextRecoveryAt(now?: number): number;
   state(now?: number): PoolBackpressureState;
 }
 
@@ -461,6 +478,19 @@ export function createPoolBackpressureLimiter(
     return true;
   }
 
+  function nextRecoveryAt(now: number): number {
+    const { initials, recoveries } = totals(now);
+    if (recoveries + 1 <= allowanceFor(initials)) return now;
+    // The window has to move before another recovery fits. The earliest that can happen is the
+    // moment the oldest bucket still inside it leaves, and every such bucket started after
+    // `now - windowMs`, so the answer is always strictly in the future.
+    for (const bucket of buckets) {
+      if (bucket.start <= now - policy.windowMs) continue;
+      return bucket.start + policy.windowMs;
+    }
+    return now + policy.windowMs;
+  }
+
   return {
     recordInitialSend(now = Date.now()): void {
       bucketFor(now).initials += 1;
@@ -471,6 +501,9 @@ export function createPoolBackpressureLimiter(
     tryPermitProbeDispatch(now = Date.now()): boolean {
       return tryPermit(now);
     },
+    nextRecoveryAt(now = Date.now()): number {
+      return nextRecoveryAt(now);
+    },
     state(now = Date.now()): PoolBackpressureState {
       const { initials, recoveries } = totals(now);
       return {

src/server/chat-native.ts

@@ -51,7 +51,9 @@ import { linkAbortSignal } from "./responses";
 import {
   addFinalRequestLog,
   beginRequestAttempt,
-  noteAttemptSend,
+  noteProviderAttemptSend,
+  recordKeyAttemptFailure,
+  recordKeyWireAttemptUsage,
   recordFirstOutput,
   recordAttemptCredentialSource,
   sealRequestAttemptIdentity,
@@ -344,10 +346,12 @@ export async function handleNativeChatCompletions(options: HandleNativeChatOptio
                 const encoding = new Headers(init.headers).get("accept-encoding");
                 if (!headers.has("accept-encoding") && encoding) headers.set("accept-encoding", encoding);
                 if (init.signal?.aborted) throw init.signal.reason;
-                noteAttemptSend(attempt, logCtx.usageLogInputTokens, transportRecovery ?? recovery);
-                return ((activeProvider as OcxProviderTransport).fetch ?? execute)(request.url, applyUpstreamRecoveryInit({
+                noteProviderAttemptSend(logCtx, route.providerName, activeProvider, logCtx.usageLogInputTokens, transportRecovery ?? recovery);
+                const dispatched = await ((activeProvider as OcxProviderTransport).fetch ?? execute)(request.url, applyUpstreamRecoveryInit({
                   ...init, method: request.method, headers, body: request.body,
                 }, transportRecovery));
+                if (!dispatched.ok) await recordKeyAttemptFailure(logCtx, dispatched, init.signal ?? upstream.signal);
+                return dispatched;
               },
             }),
           );
@@ -509,8 +513,10 @@ export async function handleNativeChatCompletions(options: HandleNativeChatOptio
       stallTimeoutSec: config.stallTimeoutSec,
       onFirstOutput: logIds ? () => recordFirstOutput(logCtx, logIds.start) : undefined,
       onUsage: usage => {
-        logCtx.usage = usage;
-        attempt.usage = usage;
+        if (!recordKeyWireAttemptUsage(logCtx, usage)) {
+          logCtx.usage = usage;
+          attempt.usage = usage;
+        }
       },
       onTerminal: (status: number, message?: string) => {
         terminalStatus = status;
@@ -600,8 +606,10 @@ export async function handleNativeChatCompletions(options: HandleNativeChatOptio
   if (!completion) return fail(502, "upstream response contained no choices", "upstream_error");
   const usage = usageFromChat(completion.usage);
   if (usage) {
-    logCtx.usage = usage;
-    attempt.usage = usage;
+    if (!recordKeyWireAttemptUsage(logCtx, usage)) {
+      logCtx.usage = usage;
+      attempt.usage = usage;
+    }
   }
   if (logIds) recordFirstOutput(logCtx, logIds.start);
   try {