feat(google): pass Gemini agentic video through instead of flattening it

src/adapters/google.ts

@@ -215,6 +215,68 @@ function toolResultImageParts(content: string | OcxContentPart[]): unknown[] {
  * surfaced on Claude-on-Antigravity; the guard lives here because this is where the parts are
  * built. Mirrors the Anthropic adapter's own empty-block guard (src/adapters/anthropic.ts).
  */
+/**
+ * A video URI Gemini fetches on its own behalf, as a `file_data` reference.
+ *
+ * Deliberately an allowlist of the forms Google documents, not "anything that is
+ * not a data: URL". `file_data` tells Gemini to go and get the bytes; pointing it
+ * at an arbitrary host would either fail upstream or make the proxy the reason a
+ * caller's private URL got dereferenced by Google. Anything not matched here
+ * keeps the existing `[video: …]` text marker.
+ *
+ * Returns the uri alone: the documented REST example for a YouTube part carries
+ * `file_data.file_uri` and nothing else, and the Files API knows the type of what
+ * it stored. An invented `mime_type` would be a guess on both paths.
+ *
+ * https://ai.google.dev/gemini-api/docs/generate-content/video-understanding
+ */
+function geminiFetchableVideoUri(url: string): string | null {
+  let parsed: URL;
+  try {
+    parsed = new URL(url);
+  } catch {
+    return null;
+  }
+  if (parsed.protocol !== "https:") return null;
+
+  const host = parsed.hostname.toLowerCase();
+  const youtubeHosts = new Set([
+    "youtube.com",
+    "www.youtube.com",
+    "m.youtube.com",
+    "music.youtube.com",
+    "youtu.be",
+    "www.youtube-nocookie.com",
+    "youtube-nocookie.com",
+  ]);
+  if (youtubeHosts.has(host)) return url;
+
+  // https://generativelanguage.googleapis.com/v1beta/files/<id>
+  if (host === "generativelanguage.googleapis.com" && /\/files\/[^/]+$/.test(parsed.pathname)) {
+    return url;
+  }
+
+  return null;
+}
+
+/**
+ * The caller's requested video mode as GenerateContent spells it.
+ *
+ * `media_processing` sits on the part beside `inline_data`/`file_data` and takes
+ * `STATIC` (the default) or `AGENTIC`. `processing: "agentic"` — the spelling in
+ * the original request and in Google's Interactions API — is a different API and
+ * is ignored here, so forwarding it verbatim would have looked like a
+ * pass-through while agentic mode never actually engaged.
+ *
+ * Upper-cased and forwarded rather than checked against our own copy of the enum:
+ * that list is Google's to extend, and a stale allowlist here would silently
+ * downgrade a caller using a newer mode. An unrecognized value fails upstream
+ * naming the field, which is a better failure than us dropping it.
+ */
+function geminiMediaProcessing(processing: string | undefined): string | undefined {
+  return processing ? processing.toUpperCase() : undefined;
+}
+
 const GEMINI_EMPTY_PLACEHOLDER = "(empty)";
 const GEMINI_EMPTY_TOOL_OUTPUT_PLACEHOLDER = "(empty tool output)";
 const GEMINI_MISSING_TOOL_RESULT = "[missing tool_result for this tool_use in history]";
@@ -321,10 +383,34 @@ function messagesToGeminiFormat(
               continue;
             }
             if (p.type === "video") {
+              // `media_processing` rides on the PART, so it applies to inline bytes
+              // exactly as it does to a fetched uri — emitting it on only one of the
+              // two would silently drop the mode for data: URLs.
+              const mediaProcessing = geminiMediaProcessing(p.processing);
+              const processingPart = mediaProcessing ? { media_processing: mediaProcessing } : {};
+
+              // Gemini accepts inline video bytes in the same Part union as images.
               const data = parseDataUrl(p.videoUrl);
-              // Gemini accepts inline video bytes in the same Part union as images. Arbitrary
-              // remote URLs are not valid fileData references, so retain only a short marker.
-              parts.push(data ? { inline_data: { mime_type: data.mediaType, data: data.base64 } } : { text: `[video: ${p.videoUrl}]` });
+              if (data) {
+                parts.push({
+                  inline_data: { mime_type: data.mediaType, data: data.base64 },
+                  ...processingPart,
+                });
+                continue;
+              }
+              // Two URI forms Gemini fetches itself: a YouTube watch URL and a Files API
+              // uri. Those ARE valid file_data references (#3271), and flattening them to
+              // a text marker was the whole reason agentic video could not be reached —
+              // the video never arrived as a video. Every other remote URL keeps the
+              // marker: we have no mime type for it and no evidence Gemini can fetch it.
+              const fileUri = geminiFetchableVideoUri(p.videoUrl);
+              if (fileUri) {
+                // Emitted only when the caller asked for a mode, so no existing
+                // request gains a field it did not have.
+                parts.push({ file_data: { file_uri: fileUri }, ...processingPart });
+                continue;
+              }
+              parts.push({ text: `[video: ${p.videoUrl}]` });
               continue;
             }
             // Drop empty/malformed text instead of emitting `{ text: "" }` or a bare `{}` part.

src/chat/inbound.ts

@@ -57,11 +57,21 @@ function contentToText(content: unknown): string {
 // route-eligibility predicate and this translator cannot drift apart again.
 const imageUrlFromPart = chatImageUrlFromPart;
 
-function videoUrlFromPart(part: Rec): string | null {
+/**
+ * A caller's video part, with the `processing` mode Gemini's agentic video
+ * understanding is requested by (#3271). The object form is the only one that
+ * can carry it — `video_url` as a bare string has nowhere to put it.
+ */
+function videoFromPart(part: Rec): { url: string; processing?: string } | null {
   if (part.type !== "video_url") return null;
   const videoUrl = part.video_url;
-  if (typeof videoUrl === "string" && videoUrl.length > 0) return videoUrl;
-  if (isRec(videoUrl) && typeof videoUrl.url === "string" && videoUrl.url.length > 0) return videoUrl.url;
+  if (typeof videoUrl === "string" && videoUrl.length > 0) return { url: videoUrl };
+  if (isRec(videoUrl) && typeof videoUrl.url === "string" && videoUrl.url.length > 0) {
+    const processing = typeof videoUrl.processing === "string" && videoUrl.processing.length > 0
+      ? videoUrl.processing
+      : undefined;
+    return { url: videoUrl.url, ...(processing ? { processing } : {}) };
+  }
   return null;
 }
 
@@ -91,8 +101,14 @@ function userContentToBlocks(content: unknown): Rec[] {
       });
       continue;
     }
-    const videoUrl = videoUrlFromPart(raw);
-    if (videoUrl) blocks.push({ type: "input_video", video_url: videoUrl });
+    const video = videoFromPart(raw);
+    if (video) {
+      blocks.push({
+        type: "input_video",
+        video_url: video.url,
+        ...(video.processing ? { processing: video.processing } : {}),
+      });
+    }
   }
   return blocks;
 }

src/responses/parser-content.ts

@@ -8,7 +8,7 @@ type InputBlock =
   | { type: "input_text"; text: string }
   | { type: "text"; text: string }
   | { type: "input_image"; image_url?: string; file_id?: string; detail?: string }
-  | { type: "input_video"; video_url?: string }
+  | { type: "input_video"; video_url?: string; processing?: string }
   // codex-rs protocol/src/models.rs sends audio as input_audio with an audio_url.
   | { type: "input_audio"; audio_url?: string; format?: string }
   | { type: "input_file"; file_id?: string; filename?: string; file_data?: string };
@@ -61,7 +61,8 @@ export function inputContentParts(blocks: unknown): string | OcxContentPart[] {
       // the request never carried, which is worse than dropping malformed input.
     } else if (block.type === "input_video") {
       const videoUrl = nonEmptyString(block.video_url);
-      if (videoUrl) parts.push({ type: "video", videoUrl });
+      const processing = nonEmptyString((block as { processing?: string }).processing);
+      if (videoUrl) parts.push({ type: "video", videoUrl, ...(processing ? { processing } : {}) });
     } else if (block.type === "input_audio") {
       // Upstream Codex sends input_audio with an audio_url (codex-rs
       // protocol/src/models.rs). The IR has no audio carrier and no adapter consumes

src/responses/schema.ts

@@ -14,6 +14,10 @@ const inputImageBlockSchema = z.object({
 const inputVideoBlockSchema = z.object({
   type: z.literal("input_video"),
   video_url: z.string().min(1),
+  // Gemini agentic video understanding (#3271). z.object() strips unknown keys,
+  // so without declaring it here the mode is dropped before any adapter sees it
+  // and the request silently degrades to frame-by-frame decoding.
+  processing: z.string().min(1).optional(),
 });
 const inputFileBlockSchema = z.object({
   type: z.literal("input_file"),

src/types/request.ts

@@ -196,8 +196,17 @@ export interface OcxImageContent {
 
 export interface OcxVideoContent {
   type: "video";
-  /** A base64 `data:` URL from an OpenAI-compatible `video_url` part. */
+  /**
+   * A base64 `data:` URL from an OpenAI-compatible `video_url` part, or a URI
+   * the upstream can fetch itself (a YouTube watch URL, a Files API uri).
+   */
   videoUrl: string;
+  /**
+   * Gemini's agentic video mode, carried verbatim from the caller's
+   * `video_url.processing` (#3271). Absent for every request that does not ask
+   * for it, so no existing traffic gains a field.
+   */
+  processing?: string;
 }
 
 /** A user/developer message content part: text or native media. */

tests/adapters/google/google-adapter.test.ts

@@ -139,6 +139,142 @@ describe("google adapter — Chat Completions video input", () => {
     });
     expect(JSON.stringify(contents)).not.toContain("file_data");
   });
+
+  test("a YouTube URL reaches Gemini as a video, carrying the agentic processing mode", async () => {
+    // #3271: the mode was dropped twice on the way in — z.object() strips an
+    // undeclared key, and the adapter then flattened the URL to a text marker,
+    // so agentic video understanding could not be requested at all.
+    const responsesBody = chatCompletionsToResponsesBody({
+      model: "google-antigravity/gemini-3.7-flash",
+      messages: [{
+        role: "user",
+        content: [
+          { type: "text", text: "When do the arms pick up the gear?" },
+          {
+            type: "video_url",
+            video_url: { url: "https://www.youtube.com/watch?v=example", processing: "agentic" },
+          },
+        ],
+      }],
+    });
+    const parsed = parseRequest(responsesBody);
+    parsed.modelId = "gemini-3.7-flash";
+
+    const contents = await geminiContents(parsed);
+
+    expect(contents).toContainEqual({
+      role: "user",
+      parts: [
+        { text: "When do the arms pick up the gear?" },
+        {
+          file_data: { file_uri: "https://www.youtube.com/watch?v=example" },
+          media_processing: "AGENTIC",
+        },
+      ],
+    });
+  });
+
+  test("a Files API uri is fetchable too, and without a mode nothing is added", async () => {
+    const responsesBody = chatCompletionsToResponsesBody({
+      model: "google-antigravity/gemini-3.7-flash",
+      messages: [{
+        role: "user",
+        content: [{
+          type: "video_url",
+          video_url: { url: "https://generativelanguage.googleapis.com/v1beta/files/abc123" },
+        }],
+      }],
+    });
+    const parsed = parseRequest(responsesBody);
+    parsed.modelId = "gemini-3.7-flash";
+
+    const contents = await geminiContents(parsed);
+
+    expect(contents).toContainEqual({
+      role: "user",
+      parts: [{
+        file_data: { file_uri: "https://generativelanguage.googleapis.com/v1beta/files/abc123" },
+      }],
+    });
+    // A caller who did not ask for a mode must not gain an unknown upstream field.
+    expect(JSON.stringify(contents)).not.toContain("media_processing");
+  });
+
+  test("inline video bytes carry the mode too — it rides on the part, not the uri", async () => {
+    // `media_processing` sits beside `inline_data`/`file_data`, so a data: URL is
+    // just as eligible. Emitting it on only the fetched-uri branch silently
+    // dropped agentic mode for callers who inline their clip.
+    const responsesBody = chatCompletionsToResponsesBody({
+      model: "google-antigravity/gemini-3.7-flash",
+      messages: [{
+        role: "user",
+        content: [{
+          type: "video_url",
+          video_url: { url: "data:video/mp4;base64,aGVsbG8=", processing: "agentic" },
+        }],
+      }],
+    });
+    const parsed = parseRequest(responsesBody);
+    parsed.modelId = "gemini-3.7-flash";
+
+    const contents = await geminiContents(parsed);
+
+    expect(contents).toContainEqual({
+      role: "user",
+      parts: [{
+        inline_data: { mime_type: "video/mp4", data: "aGVsbG8=" },
+        media_processing: "AGENTIC",
+      }],
+    });
+  });
+
+  test("the mode is sent as GenerateContent spells it, not the caller's spelling", async () => {
+    // The caller sends `processing: "agentic"` (the Interactions API spelling, and
+    // what #3271 asked for). GenerateContent reads `media_processing` with an
+    // upper-case enum; forwarding the caller's spelling verbatim would have looked
+    // like a pass-through while agentic mode never engaged.
+    const responsesBody = chatCompletionsToResponsesBody({
+      model: "google-antigravity/gemini-3.7-flash",
+      messages: [{
+        role: "user",
+        content: [{
+          type: "video_url",
+          video_url: { url: "https://youtu.be/example", processing: "agentic" },
+        }],
+      }],
+    });
+    const parsed = parseRequest(responsesBody);
+    parsed.modelId = "gemini-3.7-flash";
+
+    const wire = JSON.stringify(await geminiContents(parsed));
+
+    expect(wire).toContain('"media_processing":"AGENTIC"');
+    expect(wire).not.toContain('"processing":"agentic"');
+  });
+
+  test("a look-alike host is not treated as fetchable", async () => {
+    // The allowlist matches the host, not a substring: `file_data` asks Gemini to
+    // dereference the URL, so a near-miss must stay a marker rather than send
+    // Google after an attacker-chosen host.
+    for (const url of [
+      "https://youtube.com.evil.test/watch?v=x",
+      "https://notyoutube.com/watch?v=x",
+      "https://generativelanguage.googleapis.com.evil.test/v1beta/files/abc",
+      "http://www.youtube.com/watch?v=x",
+    ]) {
+      const responsesBody = chatCompletionsToResponsesBody({
+        model: "google-antigravity/gemini-3.7-flash",
+        messages: [{ role: "user", content: [{ type: "video_url", video_url: { url, processing: "agentic" } }] }],
+      });
+      const parsed = parseRequest(responsesBody);
+      parsed.modelId = "gemini-3.7-flash";
+
+      const contents = await geminiContents(parsed);
+
+      expect(contents).toContainEqual({ role: "user", parts: [{ text: `[video: ${url}]` }] });
+      expect(JSON.stringify(contents)).not.toContain("file_data");
+    }
+  });
 });
 
 describe("google adapter — tool-call ids on the wire", () => {