Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
122 changes: 76 additions & 46 deletions src/adapters/responses-tool-schema.ts
Original file line number Diff line number Diff line change
Expand Up @@ -107,66 +107,96 @@ function usesUnicodePropertyEscape(pattern: string): boolean {
* preserved: loosening a nested constraint can instead reject an input in those contexts.
* Unsupported patterns there remain the destination's validation responsibility.
*
* Returns `node` itself when nothing was dropped. Uses an explicit stack for caller-controlled
* nesting depth; the separate Responses-only encrypted-marker normalization is unchanged.
* Returns `node` itself when nothing was dropped. Traversal keeps only the active path and clones
* only ancestors of a removed constraint, so a broad no-op schema does not create an output tree
* or one pending closure per sibling. The explicit stack still handles caller-controlled nesting
* depth; the separate Responses-only encrypted-marker normalization is unchanged.
*/
export function stripUnicodePropertyPatterns(node: unknown, inNameBag = false): unknown {
type Assign = (value: unknown) => void;
interface Frame { node: unknown; inNameBag: boolean; assign: Assign }

let result: unknown;
let dropped = 0;
const stack: Frame[] = [{ node, inNameBag, assign: value => { result = value; } }];
interface Frame {
node: unknown[] | Record<string, unknown>;
inNameBag: boolean;
parent?: Frame;
parentKey?: string | number;
output?: unknown[] | Record<string, unknown>;
index?: number;
entries?: IterableIterator<[string, unknown]>;
}

while (stack.length > 0) {
const frame = stack.pop()!;
const current = frame.node;
function * ownEntries(value: Record<string, unknown>): IterableIterator<[string, unknown]> {
// Unlike Object.entries(), this does not materialize every key/value pair before traversal.
for (const key in value) {
if (Object.prototype.hasOwnProperty.call(value, key)) yield [key, value[key]];
}
}

if (Array.isArray(current)) {
const out: unknown[] = new Array(current.length);
frame.assign(out);
// Array items are schemas in their own right, never a name bag.
for (let i = current.length - 1; i >= 0; i--) {
stack.push({ node: current[i], inNameBag: false, assign: value => { out[i] = value; } });
}
continue;
function cloneContainer(frame: Frame): unknown[] | Record<string, unknown> {
if (frame.output) return frame.output;
if (Array.isArray(frame.node)) {
frame.output = frame.node.slice();
return frame.output;
}
if (!current || typeof current !== "object") {
frame.assign(current);
continue;
const output: Record<string, unknown> = Object.create(null) as Record<string, unknown>;
for (const key in frame.node) {
if (Object.prototype.hasOwnProperty.call(frame.node, key)) output[key] = frame.node[key];
}
frame.output = output;
return output;
}

// A schema name may be `__proto__`; a null-prototype record keeps it as data.
const out: Record<string, unknown> = Object.create(null) as Record<string, unknown>;
frame.assign(out);
function finish(frame: Frame): void {
if (!frame.output || !frame.parent) return;
const parent = cloneContainer(frame.parent);
if (Array.isArray(parent)) parent[frame.parentKey as number] = frame.output;
else parent[frame.parentKey as string] = frame.output;
}

for (const [key, value] of Object.entries(current as Record<string, unknown>)) {
if (frame.inNameBag) {
// Inside a name bag every key is a caller-chosen name, so `pattern` here is a property
// name; its value is still a schema and is walked as one.
stack.push({ node: value, inNameBag: false, assign: v => { out[key] = v; } });
continue;
}
if (PRESERVED_PATTERN_SUBTREES.has(key)) {
out[key] = value;
continue;
}
if (key === "pattern" && typeof value === "string" && usesUnicodePropertyEscape(value)) {
dropped++;
if (!node || typeof node !== "object") return node;
const root: Frame = { node: node as unknown[] | Record<string, unknown>, inNameBag };
const stack: Frame[] = [root];

while (stack.length > 0) {
const frame = stack[stack.length - 1]!;

if (Array.isArray(frame.node)) {
const index = frame.index ?? 0;
if (index >= frame.node.length) {
stack.pop();
finish(frame);
continue;
}
if (SCHEMA_LITERAL_VALUE_KEYS.has(key)) {
// Literal payloads are values, not schemas: a `pattern` key inside them is data.
out[key] = value;
continue;
frame.index = index + 1;
const child = frame.node[index];
if (child && typeof child === "object") {
stack.push({ node: child as unknown[] | Record<string, unknown>, inNameBag: false, parent: frame, parentKey: index });
}
continue;
}

frame.entries ??= ownEntries(frame.node);
const next = frame.entries.next();
if (next.done) {
stack.pop();
finish(frame);
continue;
}
const [key, value] = next.value;
if (!frame.inNameBag && key === "pattern" && typeof value === "string" && usesUnicodePropertyEscape(value)) {
delete (cloneContainer(frame) as Record<string, unknown>)[key];
continue;
}
if (!frame.inNameBag && (PRESERVED_PATTERN_SUBTREES.has(key) || SCHEMA_LITERAL_VALUE_KEYS.has(key))) {
continue;
}
if (value && typeof value === "object") {
stack.push({
node: value,
inNameBag: SCHEMA_NAME_BAG_KEYS.has(key),
assign: v => { out[key] = v; },
node: value as unknown[] | Record<string, unknown>,
inNameBag: !frame.inNameBag && SCHEMA_NAME_BAG_KEYS.has(key),
parent: frame,
parentKey: key,
});
}
}

return dropped === 0 ? node : result;
return root.output ?? node;
}
2 changes: 2 additions & 0 deletions structure/adapters/registry.md
Original file line number Diff line number Diff line change
Expand Up @@ -183,3 +183,5 @@ implement legacy call/result pairing. Modern tool-image carriers are unchanged.
raw passthrough; `tests/responses/chat-media-translation.test.ts` reaches the real HTTP
translation boundary and verifies that rejection sends no upstream request.
Canonical Responses identity sanitation and narrowly scoped pre-output combo recovery follow [request-local target compatibility](../runtime.md#request-local-target-compatibility); other adapter contracts remain unchanged.

Unicode pattern normalization uses [copy-on-write traversal](../transports/byte-accounting.md#unicode-pattern-normalization) while preserving the existing schema and wire semantics.
2 changes: 2 additions & 0 deletions structure/data-planes/inbound-compat.md
Original file line number Diff line number Diff line change
Expand Up @@ -331,3 +331,5 @@ Modern `tool` images continue through the existing following-user carrier. These
an OpenCodex conversion limit, not a provider capability claim. Final Responses-to-adapter
admission follows the [registry contract](../adapters/registry.md#untranslated-input-media).
Canonical Responses identity sanitation and narrowly scoped pre-output combo recovery follow [request-local target compatibility](../runtime.md#request-local-target-compatibility); other adapter contracts remain unchanged.

Unicode pattern normalization uses [copy-on-write traversal](../transports/byte-accounting.md#unicode-pattern-normalization) while preserving the existing schema and wire semantics.
3 changes: 3 additions & 0 deletions structure/providers/chat-compat.md
Original file line number Diff line number Diff line change
Expand Up @@ -339,3 +339,6 @@ prose the model reads beside them. Vendor tool execution stays disabled on both
adapters, and Qoder's explicit refusal of original images is unchanged.

Canonical Responses identity sanitation and narrowly scoped pre-output combo recovery follow [request-local target compatibility](../runtime.md#request-local-target-compatibility); other adapter contracts remain unchanged.


Unicode pattern normalization uses [copy-on-write traversal](../transports/byte-accounting.md#unicode-pattern-normalization) while preserving the existing schema and wire semantics.
2 changes: 2 additions & 0 deletions structure/providers/cursor.md
Original file line number Diff line number Diff line change
Expand Up @@ -130,3 +130,5 @@ Translated Chat request construction uses the [inline-image budget](../transport

Translated audio/file admission follows the [final-adapter input contract](../adapters/registry.md#untranslated-input-media); native raw passthrough remains separate.
Canonical Responses identity sanitation and narrowly scoped pre-output combo recovery follow [request-local target compatibility](../runtime.md#request-local-target-compatibility); other adapter contracts remain unchanged.

Unicode pattern normalization uses [copy-on-write traversal](../transports/byte-accounting.md#unicode-pattern-normalization) while preserving the existing schema and wire semantics.
2 changes: 2 additions & 0 deletions structure/runtime.md
Original file line number Diff line number Diff line change
Expand Up @@ -430,3 +430,5 @@ Translated audio/file admission follows the [final-adapter input contract](adapt
The combo may advance to its next eligible unattempted target before output commitment. It records no target/provider cooldown for these request-local mismatches and does not silently drop reasoning controls or raise `none` to a supported rung. Cancellation, origin/cyber-policy rejection, non-replayable post-send errors and the existing streaming commit boundary stay authoritative. Other invalid requests remain terminal.

Regression coverage: `tests/responses/responses-forward-prompt-envelope.test.ts`, `tests/routing/router-combo-failover-classification.test.ts`, and `tests/server/server-combo-failover-e2e.test.ts`.

Unicode pattern normalization uses [copy-on-write traversal](transports/byte-accounting.md#unicode-pattern-normalization) while preserving the existing schema and wire semantics.
13 changes: 13 additions & 0 deletions structure/transports/byte-accounting.md
Original file line number Diff line number Diff line change
Expand Up @@ -39,3 +39,16 @@ These optimizations do not add request queues, retry policies, or RSS-based admi

Translated audio/file admission follows the [final-adapter input contract](../adapters/registry.md#untranslated-input-media); native raw passthrough remains separate.
Canonical Responses identity sanitation and narrowly scoped pre-output combo recovery follow [request-local target compatibility](../runtime.md#request-local-target-compatibility); other adapter contracts remain unchanged.

## Unicode pattern normalization

`src/adapters/responses-tool-schema.ts` strips unsupported Unicode property patterns with an
iterative traversal and copies containers only when a descendant changes. Unchanged siblings
retain identity; a no-op returns the original input. Traversal frames follow the active path
instead of queueing an assignment closure and eagerly cloned container for each sibling.
Name bags, literal values and preserved constraint subtrees retain their existing semantics;
the separate encrypted-marker normalizer is unchanged. Inputs are not mutated.
This reduces avoidable allocations; it is not a hard heap cap or a guarantee of lower CPU cost.
Schema size still determines traversal work and the cost of copying a changed broad container.
`tests/adapters/openai/openai-chat-hardening.test.ts` covers wide, deep and mixed-array schemas;
`tests/responses/openai-responses-passthrough.test.ts` covers the existing wire contract.
2 changes: 2 additions & 0 deletions structure/transports/inventory.md
Original file line number Diff line number Diff line change
Expand Up @@ -148,3 +148,5 @@ Renamed fixed-key providers receive [missing reasoning metadata](../catalog.md#r

Translated audio/file admission follows the [final-adapter input contract](../adapters/registry.md#untranslated-input-media); native raw passthrough remains separate.
Canonical Responses identity sanitation and narrowly scoped pre-output combo recovery follow [request-local target compatibility](../runtime.md#request-local-target-compatibility); other adapter contracts remain unchanged.

Unicode pattern normalization uses [copy-on-write traversal](byte-accounting.md#unicode-pattern-normalization) while preserving the existing schema and wire semantics.
2 changes: 1 addition & 1 deletion structure/transports/responses.md
Original file line number Diff line number Diff line change
Expand Up @@ -621,7 +621,7 @@ Translated Chat request construction uses the [inline-image budget](streaming-he

The [explicit model-capability contract](../config.md#explicit-per-model-capability-declarations) preserves operator declarations through provider storage and catalog capture; it does not infer upstream capability or change this surface's routing behavior.

Provider-scoped approval reviewer settings are projected by the [catalog owner](../catalog.md#provider-scoped-approval-reviewer); this surface retains its existing routing, transport and account-selection behavior. Translated audio/file admission follows the [final-adapter input contract](../adapters/registry.md#untranslated-input-media); native raw passthrough remains separate.
Provider-scoped approval reviewer settings are projected by the [catalog owner](../catalog.md#provider-scoped-approval-reviewer); this surface retains its existing routing, transport and account-selection behavior. Translated audio/file admission follows the [final-adapter input contract](../adapters/registry.md#untranslated-input-media); native raw passthrough remains separate. Unicode pattern normalization uses [copy-on-write traversal](byte-accounting.md#unicode-pattern-normalization) while preserving the existing schema and wire semantics.

## Core module ownership

Expand Down
30 changes: 30 additions & 0 deletions tests/adapters/openai/openai-chat-hardening.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -325,6 +325,36 @@ describe("unicode property-escape pattern stripping", () => {
expect(stripped.properties.plain.pattern).toBe("^[a-z0-9_-]{1,64}$");
});

test("clones only affected paths across a broad schema", () => {
const properties: Record<string, Record<string, unknown>> = {};
for (let i = 0; i < 25_000; i++) properties[`field_${i}`] = { type: "string" };
properties.affected = { type: "string", pattern: artifactFieldPattern };
const before = { type: "object", properties };
const stripped = stripUnicodePropertyPatterns(before) as typeof before;

expect(stripped).not.toBe(before);
expect(stripped.properties).not.toBe(properties);
expect(stripped.properties.affected.pattern).toBeUndefined();
expect(properties.affected.pattern).toBe(artifactFieldPattern);
expect(stripped.properties.field_0).toBe(properties.field_0);
expect(stripped.properties.field_24999).toBe(properties.field_24999);
});

test("copies changed array paths while preserving literal and untouched siblings", () => {
const literal = { pattern: artifactFieldPattern };
const untouched = { type: "string", pattern: "^[a-z]+$" };
const changed = { type: "string", pattern: artifactFieldPattern, const: literal };
const before = { allOf: [changed, untouched, { properties: { pattern: changed } }] };
const stripped = stripUnicodePropertyPatterns(before) as typeof before;

expect(stripped.allOf).not.toBe(before.allOf);
expect(stripped.allOf[0]).toEqual({ type: "string", const: literal });
expect(stripped.allOf[0]!.const).toBe(literal);
expect(stripped.allOf[1]).toBe(untouched);
expect(stripped.allOf[2]!.properties!.pattern.pattern).toBeUndefined();
expect(changed.pattern).toBe(artifactFieldPattern);
});

test("an escaped backslash before `p{` is a literal, not a property escape", () => {
// `\\p{2}` is a literal backslash followed by a quantified `p`; Python compiles it, so a
// substring scan for `\p{` would throw away a working pattern.
Expand Down
Loading