From fbb90747fe4db31be34f830245a3bfb188a69ff0 Mon Sep 17 00:00:00 2001 From: Abhishek Sharma Date: Mon, 14 Sep 2026 15:40:26 -0700 Subject: [PATCH 1/2] feat(google): pass Gemini agentic video through instead of flattening it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Agentic video understanding could not be requested at all, because the request lost what it needed twice on the way in: 1. inputVideoBlockSchema did not declare `processing`, and z.object() strips undeclared keys, so the mode was gone before any adapter ran. 2. The Google adapter turned every non-data: video URL into a `[video: ]` text marker, so a YouTube or Files API URI never arrived as a video in the first place. `processing` now survives Chat ingress, the Responses schema, the IR and the adapter, and is emitted only when the caller sent it — no existing request gains an unknown upstream field. Fetchable URIs are an allowlist of the two forms Google documents, YouTube and the Files API, not "anything that is not a data: URL": file_data tells Gemini to dereference the URL, so a wildcard would make the proxy the reason a caller's private host got fetched by Google. Every other URL keeps the marker, which is what the existing does-not-mislabel-an-arbitrary-remote-URL test pins. Covers axis 2 of #3377; axis 1 (--text-only) already shipped. Closes #3271 --- src/adapters/google.ts | 57 ++++++++++++- src/chat/inbound.ts | 26 ++++-- src/responses/parser-content.ts | 5 +- src/responses/schema.ts | 4 + src/types/request.ts | 11 ++- tests/adapters/google/google-adapter.test.ts | 87 ++++++++++++++++++++ 6 files changed, 179 insertions(+), 11 deletions(-) diff --git a/src/adapters/google.ts b/src/adapters/google.ts index 9617c1ac10..446231146b 100644 --- a/src/adapters/google.ts +++ b/src/adapters/google.ts @@ -215,6 +215,39 @@ function toolResultImageParts(content: string | OcxContentPart[]): unknown[] { * surfaced on Claude-on-Antigravity; the guard lives here because this is where the parts are * built. Mirrors the Anthropic adapter's own empty-block guard (src/adapters/anthropic.ts). */ +/** + * A video URI Gemini fetches on its own behalf, as a `file_data` reference. + * + * Deliberately an allowlist of the two forms Google documents, not "anything + * that is not a data: URL". `file_data` tells Gemini to go and get the bytes; + * pointing it at an arbitrary host would either fail upstream or make the proxy + * the reason a caller's private URL got dereferenced by Google. Anything not + * matched here keeps the existing `[video: …]` text marker. + * + * YouTube is the case agentic video understanding is built around; the Files API + * uri is what `files.upload` hands back for a clip that was uploaded first. + */ +function geminiFetchableVideoUri(url: string): { uri: string; mimeType: string } | null { + let parsed: URL; + try { + parsed = new URL(url); + } catch { + return null; + } + if (parsed.protocol !== "https:") return null; + + const host = parsed.hostname.toLowerCase(); + const youtubeHosts = new Set(["youtube.com", "www.youtube.com", "m.youtube.com", "youtu.be"]); + if (youtubeHosts.has(host)) return { uri: url, mimeType: "video/*" }; + + // https://generativelanguage.googleapis.com/v1beta/files/ + if (host === "generativelanguage.googleapis.com" && /\/files\/[^/]+$/.test(parsed.pathname)) { + return { uri: url, mimeType: "video/*" }; + } + + return null; +} + const GEMINI_EMPTY_PLACEHOLDER = "(empty)"; const GEMINI_EMPTY_TOOL_OUTPUT_PLACEHOLDER = "(empty tool output)"; const GEMINI_MISSING_TOOL_RESULT = "[missing tool_result for this tool_use in history]"; @@ -321,10 +354,28 @@ function messagesToGeminiFormat( continue; } if (p.type === "video") { + // Gemini accepts inline video bytes in the same Part union as images. const data = parseDataUrl(p.videoUrl); - // Gemini accepts inline video bytes in the same Part union as images. Arbitrary - // remote URLs are not valid fileData references, so retain only a short marker. - parts.push(data ? { inline_data: { mime_type: data.mediaType, data: data.base64 } } : { text: `[video: ${p.videoUrl}]` }); + if (data) { + parts.push({ inline_data: { mime_type: data.mediaType, data: data.base64 } }); + continue; + } + // Two URI forms Gemini fetches itself: a YouTube watch URL and a Files API + // uri. Those ARE valid file_data references (#3271), and flattening them to + // a text marker was the whole reason agentic video could not be reached — + // the video never arrived as a video. Every other remote URL keeps the + // marker: we have no mime type for it and no evidence Gemini can fetch it. + const fileUri = geminiFetchableVideoUri(p.videoUrl); + if (fileUri) { + parts.push({ + file_data: { file_uri: fileUri.uri, mime_type: fileUri.mimeType }, + // Carried verbatim from the caller; emitted only when they asked for it, + // so no existing request gains an unknown field. + ...(p.processing ? { processing: p.processing } : {}), + }); + continue; + } + parts.push({ text: `[video: ${p.videoUrl}]` }); continue; } // Drop empty/malformed text instead of emitting `{ text: "" }` or a bare `{}` part. diff --git a/src/chat/inbound.ts b/src/chat/inbound.ts index b12761c8e4..9d5522e280 100644 --- a/src/chat/inbound.ts +++ b/src/chat/inbound.ts @@ -57,11 +57,21 @@ function contentToText(content: unknown): string { // route-eligibility predicate and this translator cannot drift apart again. const imageUrlFromPart = chatImageUrlFromPart; -function videoUrlFromPart(part: Rec): string | null { +/** + * A caller's video part, with the `processing` mode Gemini's agentic video + * understanding is requested by (#3271). The object form is the only one that + * can carry it — `video_url` as a bare string has nowhere to put it. + */ +function videoFromPart(part: Rec): { url: string; processing?: string } | null { if (part.type !== "video_url") return null; const videoUrl = part.video_url; - if (typeof videoUrl === "string" && videoUrl.length > 0) return videoUrl; - if (isRec(videoUrl) && typeof videoUrl.url === "string" && videoUrl.url.length > 0) return videoUrl.url; + if (typeof videoUrl === "string" && videoUrl.length > 0) return { url: videoUrl }; + if (isRec(videoUrl) && typeof videoUrl.url === "string" && videoUrl.url.length > 0) { + const processing = typeof videoUrl.processing === "string" && videoUrl.processing.length > 0 + ? videoUrl.processing + : undefined; + return { url: videoUrl.url, ...(processing ? { processing } : {}) }; + } return null; } @@ -91,8 +101,14 @@ function userContentToBlocks(content: unknown): Rec[] { }); continue; } - const videoUrl = videoUrlFromPart(raw); - if (videoUrl) blocks.push({ type: "input_video", video_url: videoUrl }); + const video = videoFromPart(raw); + if (video) { + blocks.push({ + type: "input_video", + video_url: video.url, + ...(video.processing ? { processing: video.processing } : {}), + }); + } } return blocks; } diff --git a/src/responses/parser-content.ts b/src/responses/parser-content.ts index 7675a42f7e..ea76652242 100644 --- a/src/responses/parser-content.ts +++ b/src/responses/parser-content.ts @@ -8,7 +8,7 @@ type InputBlock = | { type: "input_text"; text: string } | { type: "text"; text: string } | { type: "input_image"; image_url?: string; file_id?: string; detail?: string } - | { type: "input_video"; video_url?: string } + | { type: "input_video"; video_url?: string; processing?: string } // codex-rs protocol/src/models.rs sends audio as input_audio with an audio_url. | { type: "input_audio"; audio_url?: string; format?: string } | { type: "input_file"; file_id?: string; filename?: string; file_data?: string }; @@ -61,7 +61,8 @@ export function inputContentParts(blocks: unknown): string | OcxContentPart[] { // the request never carried, which is worse than dropping malformed input. } else if (block.type === "input_video") { const videoUrl = nonEmptyString(block.video_url); - if (videoUrl) parts.push({ type: "video", videoUrl }); + const processing = nonEmptyString((block as { processing?: string }).processing); + if (videoUrl) parts.push({ type: "video", videoUrl, ...(processing ? { processing } : {}) }); } else if (block.type === "input_audio") { // Upstream Codex sends input_audio with an audio_url (codex-rs // protocol/src/models.rs). The IR has no audio carrier and no adapter consumes diff --git a/src/responses/schema.ts b/src/responses/schema.ts index 5f0cf4c2d7..faa32408b9 100644 --- a/src/responses/schema.ts +++ b/src/responses/schema.ts @@ -14,6 +14,10 @@ const inputImageBlockSchema = z.object({ const inputVideoBlockSchema = z.object({ type: z.literal("input_video"), video_url: z.string().min(1), + // Gemini agentic video understanding (#3271). z.object() strips unknown keys, + // so without declaring it here the mode is dropped before any adapter sees it + // and the request silently degrades to frame-by-frame decoding. + processing: z.string().min(1).optional(), }); const inputFileBlockSchema = z.object({ type: z.literal("input_file"), diff --git a/src/types/request.ts b/src/types/request.ts index cec8294a50..58a194c0e6 100644 --- a/src/types/request.ts +++ b/src/types/request.ts @@ -196,8 +196,17 @@ export interface OcxImageContent { export interface OcxVideoContent { type: "video"; - /** A base64 `data:` URL from an OpenAI-compatible `video_url` part. */ + /** + * A base64 `data:` URL from an OpenAI-compatible `video_url` part, or a URI + * the upstream can fetch itself (a YouTube watch URL, a Files API uri). + */ videoUrl: string; + /** + * Gemini's agentic video mode, carried verbatim from the caller's + * `video_url.processing` (#3271). Absent for every request that does not ask + * for it, so no existing traffic gains a field. + */ + processing?: string; } /** A user/developer message content part: text or native media. */ diff --git a/tests/adapters/google/google-adapter.test.ts b/tests/adapters/google/google-adapter.test.ts index 1dd5a7c97a..57d736f8c2 100644 --- a/tests/adapters/google/google-adapter.test.ts +++ b/tests/adapters/google/google-adapter.test.ts @@ -139,6 +139,93 @@ describe("google adapter — Chat Completions video input", () => { }); expect(JSON.stringify(contents)).not.toContain("file_data"); }); + + test("a YouTube URL reaches Gemini as a video, carrying the agentic processing mode", async () => { + // #3271: the mode was dropped twice on the way in — z.object() strips an + // undeclared key, and the adapter then flattened the URL to a text marker, + // so agentic video understanding could not be requested at all. + const responsesBody = chatCompletionsToResponsesBody({ + model: "google-antigravity/gemini-3.7-flash", + messages: [{ + role: "user", + content: [ + { type: "text", text: "When do the arms pick up the gear?" }, + { + type: "video_url", + video_url: { url: "https://www.youtube.com/watch?v=example", processing: "agentic" }, + }, + ], + }], + }); + const parsed = parseRequest(responsesBody); + parsed.modelId = "gemini-3.7-flash"; + + const contents = await geminiContents(parsed); + + expect(contents).toContainEqual({ + role: "user", + parts: [ + { text: "When do the arms pick up the gear?" }, + { + file_data: { file_uri: "https://www.youtube.com/watch?v=example", mime_type: "video/*" }, + processing: "agentic", + }, + ], + }); + }); + + test("a Files API uri is fetchable too, and without a mode nothing is added", async () => { + const responsesBody = chatCompletionsToResponsesBody({ + model: "google-antigravity/gemini-3.7-flash", + messages: [{ + role: "user", + content: [{ + type: "video_url", + video_url: { url: "https://generativelanguage.googleapis.com/v1beta/files/abc123" }, + }], + }], + }); + const parsed = parseRequest(responsesBody); + parsed.modelId = "gemini-3.7-flash"; + + const contents = await geminiContents(parsed); + + expect(contents).toContainEqual({ + role: "user", + parts: [{ + file_data: { + file_uri: "https://generativelanguage.googleapis.com/v1beta/files/abc123", + mime_type: "video/*", + }, + }], + }); + // A caller who did not ask for a mode must not gain an unknown upstream field. + expect(JSON.stringify(contents)).not.toContain("processing"); + }); + + test("a look-alike host is not treated as fetchable", async () => { + // The allowlist matches the host, not a substring: `file_data` asks Gemini to + // dereference the URL, so a near-miss must stay a marker rather than send + // Google after an attacker-chosen host. + for (const url of [ + "https://youtube.com.evil.test/watch?v=x", + "https://notyoutube.com/watch?v=x", + "https://generativelanguage.googleapis.com.evil.test/v1beta/files/abc", + "http://www.youtube.com/watch?v=x", + ]) { + const responsesBody = chatCompletionsToResponsesBody({ + model: "google-antigravity/gemini-3.7-flash", + messages: [{ role: "user", content: [{ type: "video_url", video_url: { url, processing: "agentic" } }] }], + }); + const parsed = parseRequest(responsesBody); + parsed.modelId = "gemini-3.7-flash"; + + const contents = await geminiContents(parsed); + + expect(contents).toContainEqual({ role: "user", parts: [{ text: `[video: ${url}]` }] }); + expect(JSON.stringify(contents)).not.toContain("file_data"); + } + }); }); describe("google adapter — tool-call ids on the wire", () => { From 9182c3d46e881ecb3afa35fd3b39ccd254a5f186 Mon Sep 17 00:00:00 2001 From: Abhishek Sharma Date: Mon, 14 Sep 2026 17:06:35 -0700 Subject: [PATCH 2/2] fix(google): emit media_processing, on every video part, without a guessed mime MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three corrections from review, all confirmed against Google's video-understanding docs rather than taken on trust: 1. GenerateContent reads `media_processing` with an upper-case enum (STATIC | AGENTIC) on the part. `processing: "agentic"` is the Interactions API spelling and is ignored here, so forwarding the caller's field verbatim looked like a pass-through while agentic mode never engaged. Caught by CodeRabbit on #4663. 2. The field rides on the PART, so it applies to inline_data exactly as to file_data. Emitting it on only the fetched-uri branch dropped the mode for callers who inline their clip. 3. Dropped the invented `mime_type: "video/*"`. The documented REST example for a YouTube part carries file_uri alone, and the Files API knows the type of what it stored. Also adds music.youtube.com and youtube-nocookie.com to the allowlist — same service, and the omission was an oversight rather than a decision. Co-authored-by: Abhishek Sharma --- src/adapters/google.ts | 71 +++++++++++++++----- tests/adapters/google/google-adapter.test.ts | 63 +++++++++++++++-- 2 files changed, 109 insertions(+), 25 deletions(-) diff --git a/src/adapters/google.ts b/src/adapters/google.ts index 446231146b..1ec6e9bc45 100644 --- a/src/adapters/google.ts +++ b/src/adapters/google.ts @@ -218,16 +218,19 @@ function toolResultImageParts(content: string | OcxContentPart[]): unknown[] { /** * A video URI Gemini fetches on its own behalf, as a `file_data` reference. * - * Deliberately an allowlist of the two forms Google documents, not "anything - * that is not a data: URL". `file_data` tells Gemini to go and get the bytes; - * pointing it at an arbitrary host would either fail upstream or make the proxy - * the reason a caller's private URL got dereferenced by Google. Anything not - * matched here keeps the existing `[video: …]` text marker. + * Deliberately an allowlist of the forms Google documents, not "anything that is + * not a data: URL". `file_data` tells Gemini to go and get the bytes; pointing it + * at an arbitrary host would either fail upstream or make the proxy the reason a + * caller's private URL got dereferenced by Google. Anything not matched here + * keeps the existing `[video: …]` text marker. * - * YouTube is the case agentic video understanding is built around; the Files API - * uri is what `files.upload` hands back for a clip that was uploaded first. + * Returns the uri alone: the documented REST example for a YouTube part carries + * `file_data.file_uri` and nothing else, and the Files API knows the type of what + * it stored. An invented `mime_type` would be a guess on both paths. + * + * https://ai.google.dev/gemini-api/docs/generate-content/video-understanding */ -function geminiFetchableVideoUri(url: string): { uri: string; mimeType: string } | null { +function geminiFetchableVideoUri(url: string): string | null { let parsed: URL; try { parsed = new URL(url); @@ -237,17 +240,43 @@ function geminiFetchableVideoUri(url: string): { uri: string; mimeType: string } if (parsed.protocol !== "https:") return null; const host = parsed.hostname.toLowerCase(); - const youtubeHosts = new Set(["youtube.com", "www.youtube.com", "m.youtube.com", "youtu.be"]); - if (youtubeHosts.has(host)) return { uri: url, mimeType: "video/*" }; + const youtubeHosts = new Set([ + "youtube.com", + "www.youtube.com", + "m.youtube.com", + "music.youtube.com", + "youtu.be", + "www.youtube-nocookie.com", + "youtube-nocookie.com", + ]); + if (youtubeHosts.has(host)) return url; // https://generativelanguage.googleapis.com/v1beta/files/ if (host === "generativelanguage.googleapis.com" && /\/files\/[^/]+$/.test(parsed.pathname)) { - return { uri: url, mimeType: "video/*" }; + return url; } return null; } +/** + * The caller's requested video mode as GenerateContent spells it. + * + * `media_processing` sits on the part beside `inline_data`/`file_data` and takes + * `STATIC` (the default) or `AGENTIC`. `processing: "agentic"` — the spelling in + * the original request and in Google's Interactions API — is a different API and + * is ignored here, so forwarding it verbatim would have looked like a + * pass-through while agentic mode never actually engaged. + * + * Upper-cased and forwarded rather than checked against our own copy of the enum: + * that list is Google's to extend, and a stale allowlist here would silently + * downgrade a caller using a newer mode. An unrecognized value fails upstream + * naming the field, which is a better failure than us dropping it. + */ +function geminiMediaProcessing(processing: string | undefined): string | undefined { + return processing ? processing.toUpperCase() : undefined; +} + const GEMINI_EMPTY_PLACEHOLDER = "(empty)"; const GEMINI_EMPTY_TOOL_OUTPUT_PLACEHOLDER = "(empty tool output)"; const GEMINI_MISSING_TOOL_RESULT = "[missing tool_result for this tool_use in history]"; @@ -354,10 +383,19 @@ function messagesToGeminiFormat( continue; } if (p.type === "video") { + // `media_processing` rides on the PART, so it applies to inline bytes + // exactly as it does to a fetched uri — emitting it on only one of the + // two would silently drop the mode for data: URLs. + const mediaProcessing = geminiMediaProcessing(p.processing); + const processingPart = mediaProcessing ? { media_processing: mediaProcessing } : {}; + // Gemini accepts inline video bytes in the same Part union as images. const data = parseDataUrl(p.videoUrl); if (data) { - parts.push({ inline_data: { mime_type: data.mediaType, data: data.base64 } }); + parts.push({ + inline_data: { mime_type: data.mediaType, data: data.base64 }, + ...processingPart, + }); continue; } // Two URI forms Gemini fetches itself: a YouTube watch URL and a Files API @@ -367,12 +405,9 @@ function messagesToGeminiFormat( // marker: we have no mime type for it and no evidence Gemini can fetch it. const fileUri = geminiFetchableVideoUri(p.videoUrl); if (fileUri) { - parts.push({ - file_data: { file_uri: fileUri.uri, mime_type: fileUri.mimeType }, - // Carried verbatim from the caller; emitted only when they asked for it, - // so no existing request gains an unknown field. - ...(p.processing ? { processing: p.processing } : {}), - }); + // Emitted only when the caller asked for a mode, so no existing + // request gains a field it did not have. + parts.push({ file_data: { file_uri: fileUri }, ...processingPart }); continue; } parts.push({ text: `[video: ${p.videoUrl}]` }); diff --git a/tests/adapters/google/google-adapter.test.ts b/tests/adapters/google/google-adapter.test.ts index 57d736f8c2..4778135d73 100644 --- a/tests/adapters/google/google-adapter.test.ts +++ b/tests/adapters/google/google-adapter.test.ts @@ -167,8 +167,8 @@ describe("google adapter — Chat Completions video input", () => { parts: [ { text: "When do the arms pick up the gear?" }, { - file_data: { file_uri: "https://www.youtube.com/watch?v=example", mime_type: "video/*" }, - processing: "agentic", + file_data: { file_uri: "https://www.youtube.com/watch?v=example" }, + media_processing: "AGENTIC", }, ], }); @@ -193,14 +193,63 @@ describe("google adapter — Chat Completions video input", () => { expect(contents).toContainEqual({ role: "user", parts: [{ - file_data: { - file_uri: "https://generativelanguage.googleapis.com/v1beta/files/abc123", - mime_type: "video/*", - }, + file_data: { file_uri: "https://generativelanguage.googleapis.com/v1beta/files/abc123" }, }], }); // A caller who did not ask for a mode must not gain an unknown upstream field. - expect(JSON.stringify(contents)).not.toContain("processing"); + expect(JSON.stringify(contents)).not.toContain("media_processing"); + }); + + test("inline video bytes carry the mode too — it rides on the part, not the uri", async () => { + // `media_processing` sits beside `inline_data`/`file_data`, so a data: URL is + // just as eligible. Emitting it on only the fetched-uri branch silently + // dropped agentic mode for callers who inline their clip. + const responsesBody = chatCompletionsToResponsesBody({ + model: "google-antigravity/gemini-3.7-flash", + messages: [{ + role: "user", + content: [{ + type: "video_url", + video_url: { url: "data:video/mp4;base64,aGVsbG8=", processing: "agentic" }, + }], + }], + }); + const parsed = parseRequest(responsesBody); + parsed.modelId = "gemini-3.7-flash"; + + const contents = await geminiContents(parsed); + + expect(contents).toContainEqual({ + role: "user", + parts: [{ + inline_data: { mime_type: "video/mp4", data: "aGVsbG8=" }, + media_processing: "AGENTIC", + }], + }); + }); + + test("the mode is sent as GenerateContent spells it, not the caller's spelling", async () => { + // The caller sends `processing: "agentic"` (the Interactions API spelling, and + // what #3271 asked for). GenerateContent reads `media_processing` with an + // upper-case enum; forwarding the caller's spelling verbatim would have looked + // like a pass-through while agentic mode never engaged. + const responsesBody = chatCompletionsToResponsesBody({ + model: "google-antigravity/gemini-3.7-flash", + messages: [{ + role: "user", + content: [{ + type: "video_url", + video_url: { url: "https://youtu.be/example", processing: "agentic" }, + }], + }], + }); + const parsed = parseRequest(responsesBody); + parsed.modelId = "gemini-3.7-flash"; + + const wire = JSON.stringify(await geminiContents(parsed)); + + expect(wire).toContain('"media_processing":"AGENTIC"'); + expect(wire).not.toContain('"processing":"agentic"'); }); test("a look-alike host is not treated as fetchable", async () => {