Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
79 changes: 77 additions & 2 deletions src/server/responses/input-admission.ts
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,8 @@
* catches the pathological case and stays out of the way otherwise. Every uncertainty
* resolves toward admitting.
*/
import { nativeOpenAiContextWindow, nativeOpenAiMaxInputTokens, type NativeContextLimitsInput } from "../../codex/catalog/metadata";
import { nativeOpenAiContextWindow, nativeOpenAiMaxInputTokens, nativeOpenAiMaxOutputTokens, type NativeContextLimitsInput } from "../../codex/catalog/metadata";
import { getModelMetadata } from "../../generated/model-metadata";
import { estimateTokens } from "../../lib/token-estimate";
import { isCanonicalOpenAiForwardProvider, OPENAI_CODEX_PROVIDER_ID } from "../../providers/openai-tiers";
import { modelRecordValue } from "../../reasoning-effort";
Expand Down Expand Up @@ -54,6 +55,8 @@ export interface InputAdmissionResult {
estimatedTokens: number;
/** Resolved ceiling, or null when nothing could be resolved (=> always admitted). */
ceiling: number | null;
/** Output space deliberately reserved by combo preflight; absent on the loose direct gate. */
requiredOutputHeadroom?: number;
}

function positive(value: unknown): number | null {
Expand Down Expand Up @@ -166,15 +169,87 @@ export function resolveInputCeiling(
const native = canonicalNativeBare
? positive(nativeOpenAiContextWindow(modelId, nativeLimits))
: null;
// Some supported native slugs (notably Spark) are present in the generated capability
// bundle but absent from the narrower pinned-native override table. Falling through to null
// made input admission completely blind for exactly those models. The generated bundle is
// static in-tree metadata, not a live catalog read, and both OpenAI/OpenAI-Codex publish the
// same 128k Spark context window. Operator/provider caps may only narrow it.
const generatedNative = canonicalNativeBare
? positive(getModelMetadata("openai-codex", modelId)?.contextWindow)
?? positive(getModelMetadata("openai", modelId)?.contextWindow)
: null;
const nativeCap = typeof nativeContextCap === "number"
? positive(nativeContextCap)
: positive(nativeContextCap?.cap);
const generatedNarrowed = generatedNative === null
? null
: Math.min(generatedNative, configured ?? generatedNative, nativeCap ?? generatedNative);
const nativeMaxInput = canonicalNativeBare ? positive(nativeOpenAiMaxInputTokens(modelId, nativeLimits)) : null;

const window = canonicalNativeBare ? native : configured;
const window = canonicalNativeBare ? (native ?? generatedNarrowed) : configured;
// modelMaxInputTokens is an input-only cap, so it can only tighten the window.
const configuredMaxInput = positive(modelRecordValue(provider.modelMaxInputTokens, modelId));
const limits = [window, configuredMaxInput, nativeMaxInput].filter((v): v is number => v !== null);
return limits.length === 0 ? null : Math.min(...limits);
}


/**
* Resolve the largest output the concrete target can produce. This is used only to avoid
* reserving MORE than the target itself can ever emit when a combo child carries a larger
* client-side max_output_tokens. Unknown stays unknown rather than inventing a capability.
*/
export function resolveOutputCeiling(
provider: OcxProviderConfig,
providerName: string,
modelId: string,
): number | null {
const configured = positive(modelRecordValue(provider.modelMaxOutputTokens, modelId))
?? positive(provider.defaultMaxOutputTokens);
const canonicalNativeBare = providerName === OPENAI_CODEX_PROVIDER_ID
&& isCanonicalOpenAiForwardProvider(provider)
&& !modelId.includes("/");
const native = canonicalNativeBare ? positive(nativeOpenAiMaxOutputTokens(modelId)) : null;
const limits = [configured, native].filter((v): v is number => v !== null);
return limits.length === 0 ? null : Math.min(...limits);
}

/**
* Combo-only context admission. A fallback must be able to satisfy the caller's declared
* output allowance inside its OWN context window; otherwise it can return HTTP 200, emit some
* text, and terminate with finish_reason=length. At that point replaying on the next target is
* unsafe because client-visible output may already have committed.
*
* Direct/single-target requests keep the deliberately loose 2.5x pathological-input gate.
* This stricter rule applies only to synthetic combo children, where skipping one known-small
* target is safe and the ladder can continue before any upstream bytes are emitted. Unknown
* context remains fail-open. If the client omitted max_output_tokens, behavior is unchanged.
*/
export function checkComboTargetInputAdmission(
parsed: OcxParsedRequest,
provider: OcxProviderConfig,
providerName: string,
modelId: string,
nativeContextCap?: NativeContextLimitsInput,
): InputAdmissionResult {
const ceiling = resolveInputCeiling(provider, providerName, modelId, nativeContextCap);
const requestedOutput = positive(parsed.options.maxOutputTokens);
if (ceiling === null || requestedOutput === null) {
return checkInputAdmission(parsed, provider, providerName, modelId, nativeContextCap);
}
const targetOutput = resolveOutputCeiling(provider, providerName, modelId);
const requiredOutputHeadroom = targetOutput === null
? requestedOutput
: Math.min(requestedOutput, targetOutput);
const estimatedTokens = estimateInputTokens(parsed, modelId);
return {
admitted: estimatedTokens + requiredOutputHeadroom <= ceiling,
estimatedTokens,
ceiling,
requiredOutputHeadroom,
};
}

/**
* Fail-open when no ceiling is known; refuse only past `ceiling * ADMISSION_TOLERANCE`.
*
Expand Down
19 changes: 11 additions & 8 deletions src/server/responses/request-prepare.ts
Original file line number Diff line number Diff line change
Expand Up @@ -101,7 +101,7 @@ import {
isCodexReserveHelperUnsupported,
CODEX_RESERVE_HELPER_UNSUPPORTED_MESSAGE,
} from "../../codex/loopback-target";
import { checkInputAdmission } from "./input-admission";
import { checkComboTargetInputAdmission, checkInputAdmission } from "./input-admission";
import { nativeContextLimits } from "../../codex/catalog";
import { streamingContextOverflowResponse } from "./context-overflow";
import {
Expand Down Expand Up @@ -860,7 +860,9 @@ export async function prepareResponsesRequest(
// refusing the turn that shrinks the context would deadlock the client against the very
// limit this gate reports — it would be told to compact and then denied the compaction.
if (parsed._compactionRequest !== true) {
const inputAdmission = checkInputAdmission(parsed, route.provider, route.providerName, parsed.modelId, nativeContextLimits(config));
const inputAdmission = options.comboAttempt
? checkComboTargetInputAdmission(parsed, route.provider, route.providerName, parsed.modelId, nativeContextLimits(config))
: checkInputAdmission(parsed, route.provider, route.providerName, parsed.modelId, nativeContextLimits(config));
if (!inputAdmission.admitted) {
// #1524: this is a LOCAL preflight refusal, not an upstream verdict. A policy or combo
// fallback must be able to skip this candidate and try one whose context window fits,
Expand All @@ -873,13 +875,14 @@ export async function prepareResponsesRequest(
translatorBudget,
);
}
return formatErrorResponse(
413,
"input_admission_refused",
`Estimated input (~${inputAdmission.estimatedTokens} tokens) is far past the context window `
const admissionMessage = inputAdmission.requiredOutputHeadroom !== undefined
? `Estimated input (~${inputAdmission.estimatedTokens} tokens) plus ${inputAdmission.requiredOutputHeadroom} `
+ `tokens of requested output headroom cannot fit the context window of ${parsed.modelId} `
+ `(${inputAdmission.ceiling} tokens).`
: `Estimated input (~${inputAdmission.estimatedTokens} tokens) is far past the context window `
+ `of ${parsed.modelId} (${inputAdmission.ceiling} tokens). Start a new session or choose a `
+ `model with a larger context window.`,
);
+ `model with a larger context window.`;
return formatErrorResponse(413, "input_admission_refused", admissionMessage);
}
}
const preAuthHostKey = preAuthUpstreamHostCircuitKey(route, config);
Expand Down
2 changes: 2 additions & 0 deletions structure/transports/responses.md
Original file line number Diff line number Diff line change
Expand Up @@ -693,3 +693,5 @@ What must not happen is a ladder that charges and then returns through a path th
nor releases. That is not a lost send; it is a send the request never made, spending an allowance a
later recovery in the same request then cannot have. `tests/lib/execution-budget-permits.test.ts`
pins both ladder shapes against exactly that.

For combo children, `src/server/responses/request-prepare.ts` applies the input-plus-output-headroom admission in `src/server/responses/input-admission.ts` before credential or upstream dispatch. The direct-request gate and compaction exemption remain unchanged.
68 changes: 68 additions & 0 deletions tests/helpers/combo-context-headroom-cases.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,68 @@
import { expect, test } from "bun:test";
import type { OcxConfig, OcxProviderConfig } from "../../src/types";

interface ComboHarness<Server> {
serve(handler: () => Response | Promise<Response>): Server;
baseUrl(server: Server): string;
chatSuccess(text: string, model?: string): Response;
provider(adapter: string, url: string, apiKey: string, extra?: Partial<OcxProviderConfig>): OcxProviderConfig;
comboConfig(providers: OcxConfig["providers"]): OcxConfig;
post(config: OcxConfig, raw?: Record<string, unknown>): Promise<Response>;
}

/** Register under the caller's isolated homes, mock state and server cleanup hooks. */
export function registerComboContextHeadroomCases<Server>({
serve, baseUrl, chatSuccess, provider, comboConfig, post,
}: ComboHarness<Server>): void {
test("combo skips a target that cannot fit input plus requested output before any bytes commit", async () => {
let smallHits = 0;
let largeHits = 0;
const small = serve(() => {
smallHits += 1;
return chatSuccess("MUST NOT RUN", "m1");
});
const large = serve(() => {
largeHits += 1;
return chatSuccess("large context target", "m2");
});
const config = comboConfig({
a: provider("openai-chat", baseUrl(small), "key-a", {
modelContextWindows: { m1: 128_000 },
modelMaxOutputTokens: { m1: 32_000 },
}),
b: provider("openai-chat", baseUrl(large), "key-b", {
modelContextWindows: { m2: 1_000_000 },
modelMaxOutputTokens: { m2: 128_000 },
}),
});
const response = await post(config, {
input: "a".repeat(400_000), // about 100k estimated input tokens
max_output_tokens: 64_000,
});
expect(response.status).toBe(200);
expect(smallHits).toBe(0);
expect(largeHits).toBe(1);
expect(await response.text()).toContain("large context target");
});

test("provider-specific prompt-too-long 400 hops to a larger-context combo target", async () => {
let backupHits = 0;
const capped = serve(() => Response.json({ error: {
message: "Prompt 346030 > 262144 maximum context length",
type: "invalid_request_prompt_too_long",
code: "5059",
raw_status_code: 400,
} }, { status: 400 }));
const backup = serve(() => {
backupHits += 1;
return chatSuccess("larger context backup", "m2");
});
const response = await post(comboConfig({
a: provider("openai-chat", baseUrl(capped), "key-a"),
b: provider("openai-chat", baseUrl(backup), "key-b"),
}));
expect(response.status).toBe(200);
expect(backupHits).toBe(1);
expect(await response.text()).toContain("larger context backup");
});
}
51 changes: 51 additions & 0 deletions tests/server/input-admission.test.ts
Original file line number Diff line number Diff line change
@@ -1,9 +1,11 @@
import { describe, expect, test } from "bun:test";
import {
ADMISSION_TOLERANCE,
checkComboTargetInputAdmission,
checkInputAdmission,
estimateInputTokens,
resolveInputCeiling,
resolveOutputCeiling,
} from "../../src/server/responses/input-admission";
import { modelRecordValue } from "../../src/reasoning-effort";
import type { OcxMessage, OcxParsedRequest, OcxProviderConfig, OcxTool } from "../../src/types";
Expand Down Expand Up @@ -257,3 +259,52 @@ describe("checkInputAdmission", () => {
expect(calls).toBe(0);
});
});


describe("combo target input admission", () => {
const capped: OcxProviderConfig = {
adapter: "openai-chat",
baseUrl: "https://example.test/v1",
modelContextWindows: { m: 128_000 },
modelMaxOutputTokens: { m: 32_000 },
};

const withMaxOutput = (inputTokens: number, maxOutputTokens = 64_000): OcxParsedRequest => ({
...request([userText(asciiTokens(inputTokens))]),
modelId: "m",
options: { maxOutputTokens },
});

test("skips a known-small fallback before it can truncate after output", () => {
const result = checkComboTargetInputAdmission(withMaxOutput(100_000), capped, "custom", "m");
expect(result.admitted).toBe(false);
expect(result.ceiling).toBe(128_000);
expect(result.requiredOutputHeadroom).toBe(32_000);
});

test("keeps a target when input plus its own output ceiling fits", () => {
const result = checkComboTargetInputAdmission(withMaxOutput(90_000), capped, "custom", "m");
expect(result.admitted).toBe(true);
expect(result.requiredOutputHeadroom).toBe(32_000);
});

test("unknown context remains fail-open", () => {
const unknown: OcxProviderConfig = { adapter: "openai-chat", baseUrl: "https://example.test/v1" };
const result = checkComboTargetInputAdmission(withMaxOutput(2_000_000), unknown, "custom", "m");
expect(result.admitted).toBe(true);
expect(result.ceiling).toBeNull();
});

test("no explicit output allowance preserves the loose direct admission contract", () => {
const parsed = request([userText(asciiTokens(150_000))]);
parsed.modelId = "m";
const result = checkComboTargetInputAdmission(parsed, capped, "custom", "m");
expect(result.admitted).toBe(true); // 150k is still inside the existing 2.5x pathological gate.
expect(result.requiredOutputHeadroom).toBeUndefined();
});

test("canonical Spark resolves its measured 128k window and 32k output ceiling", () => {
expect(resolveInputCeiling(CANONICAL_NATIVE, "openai", "gpt-5.3-codex-spark")).toBe(128_000);
expect(resolveOutputCeiling(CANONICAL_NATIVE, "openai", "gpt-5.3-codex-spark")).toBe(32_000);
});
});
22 changes: 2 additions & 20 deletions tests/server/server-combo-failover-e2e.test.ts
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import { registerComboContextHeadroomCases } from "../helpers/combo-context-headroom-cases";
import { sessionLaneIdFromRequest } from "../../src/server/request-log-conversation";
import { afterEach, beforeEach, describe, expect, mock, setDefaultTimeout, test } from "bun:test";
import { logsFromApiBody } from "../helpers/logs-api";
Expand Down Expand Up @@ -2118,26 +2119,7 @@ describe("server combo failover 030 activation matrix", () => {
expect(await exhausted.text()).not.toContain("sk-a-should-redact");
});

test("provider-specific prompt-too-long 400 hops to a larger-context combo target", async () => {
let backupHits = 0;
const capped = serve(() => Response.json({ error: {
message: "Prompt 346030 > 262144 maximum context length",
type: "invalid_request_prompt_too_long",
code: "5059",
raw_status_code: 400,
} }, { status: 400 }));
const backup = serve(() => {
backupHits += 1;
return chatSuccess("larger context backup", "m2");
});
const response = await post(comboConfig({
a: provider("openai-chat", baseUrl(capped), "key-a"),
b: provider("openai-chat", baseUrl(backup), "key-b"),
}));
expect(response.status).toBe(200);
expect(backupHits).toBe(1);
expect(await response.text()).toContain("larger context backup");
});
registerComboContextHeadroomCases({ serve, baseUrl, chatSuccess, provider, comboConfig, post });

test("429 Retry-After 120 keeps A cooling at 60 seconds and restores it at 120", async () => {
const t0 = Date.parse("2026-07-18T00:00:00.000Z");
Expand Down
Loading