From 8745cc5f5bc4f82c70ecfcd7cec494768fdd82b1 Mon Sep 17 00:00:00 2001 From: Kiyeon Jeon Date: Sun, 4 Oct 2026 23:25:45 +0900 Subject: [PATCH] feat: move Seedance and Omni video onto the job contract Seedance - Runs on fal's queue API (submit, status, result, cancel) instead of the blocking `subscribe`, so a paid job keeps its request ID from the moment it is accepted and can be polled or cancelled after a crash. - A COMPLETED status can still be a failure: `error` is checked first, and a rejection raised when the result is read (the likeness 422) is a failed job with kind `likeness`, not a thrown read. - A data-URI first or last frame is uploaded to fal storage instead of being silently dropped (which turned image-to-video into text-to-video). - Tests run the real fal SDK against the fake network. Omni - Submits with `background: true` and polls the interaction, then the Files API until the video is ACTIVE. Omni accepts background mode although the docs list only text models; the completed interaction carries the video URI (probed live). - Edit and extend chain with `previous_interaction_id`, so an Omni job handle is its session handle. Cancel uses POST /interactions/{id}/cancel. - A completed interaction with no video reads as moderation. Both keep a blocking `generateVideo` that submits and waits, so CLI callers are unchanged until the CLI moves to the contract. Claude-Session: https://claude.ai/code/session_011S1BLTaFctZNQzGERpkCuA --- .agents/references/providers/fal.md | 2 + .agents/references/providers/google.md | 3 + .../ai-providers/src/fal/FalProvider.test.ts | 430 ++++++------------ packages/ai-providers/src/fal/FalProvider.ts | 259 +++++++---- .../src/gemini/gemini-omni.test.ts | 179 +++++--- .../ai-providers/src/gemini/gemini-omni.ts | 275 +++++++---- .../src/video/contract.testkit.ts | 6 +- 7 files changed, 619 insertions(+), 535 deletions(-) diff --git a/.agents/references/providers/fal.md b/.agents/references/providers/fal.md index a3a899cf..06820eba 100644 --- a/.agents/references/providers/fal.md +++ b/.agents/references/providers/fal.md @@ -52,6 +52,8 @@ Per-second figures below are fal's own 16:9 approximations. - Cancel: `PUT .../requests/{id}/cancel` returns 202 `CANCELLATION_REQUESTED` or 400 `ALREADY_COMPLETED`; in-progress jobs may still finish (docs). - `X-Fal-Request-Timeout` sets an absolute server-side deadline (docs). - The JS SDK wraps this as `fal.queue.submit/status/result/cancel`, or `fal.subscribe` which submits and polls (docs). + We use the queue calls, so a job keeps its request ID from submission; a rejected job (the likeness 422) surfaces when its result is read (probe, 2026-10-04). +- SDK 1.10 calls: `POST queue.fal.run/`, `GET queue.fal.run///requests/{id}/status?logs=0`, `GET .../requests/{id}`, `PUT .../requests/{id}/cancel`; storage uploads `POST rest.fal.ai/storage/upload/initiate?storage_type=fal-cdn-v3`, then `PUT` to the returned URL (probe). - Seedance output: `{ video: { url, content_type, file_name, file_size }, seed }`, plus `draft_id` on 2.5 (docs). - Seedance 2.0 inputs: `prompt`, `image_url` (i2v, required), `end_image_url` (i2v), `image_urls` / `video_urls` / `audio_urls` (r2v), `resolution`, `duration`, `aspect_ratio` (auto, 21:9, 16:9, 4:3, 1:1, 3:4, 9:16), `generate_audio` (default true, same price), `bitrate_mode`, `codec`, `end_user_id` (docs). - `duration` and `resolution` are string enums in the schema (`"auto"`, `"4"`..`"15"`) (docs). diff --git a/.agents/references/providers/google.md b/.agents/references/providers/google.md index a3382f75..eee4a1d4 100644 --- a/.agents/references/providers/google.md +++ b/.agents/references/providers/google.md @@ -135,6 +135,9 @@ Public YouTube URLs work, with an 8 h/day cap on the free tier (docs). - `minimal` thinking errors on 3.8 Flash, 3.7 Flash, and 3.1 Pro preview but is the default on 3.5 Flash-Lite and the image models (docs). - Omni returns the video as base64 `data` inside `steps[]` by default; a URL comes back only with `response_format.delivery: "uri"`, and only in the create response or SSE stream, not on later `GET /interactions/{id}` (docs). Use `uri` above 4 MB (docs). - Omni requests are synchronous unless `background: true`; `store: false` disables `previous_interaction_id` editing (docs). Interactions are retained 55 days on paid tier, 1 day on free (docs). +- Omni accepts `background: true` although the background-execution page lists only text models and agents: create returns `{id, status: "in_progress"}` in about 3 s, `GET /interactions/{id}` moves to `completed` after 20-70 s, and the completed interaction's `steps[]` carries the video `uri`, contrary to the note above (probe, 2026-10-04). +- A `previous_interaction_id` edit on a background interaction kept the subject and camera and applied the edit ("snowy night"); Omni chose 10 s for both clips (probe, 2026-10-04). +- Background interactions cancel with `POST /interactions/{id}/cancel` (docs, background execution). - Veo extension takes a video object from a previous generation (`instances[].video` as `uri` or `inlineData`), must be 720p, and requires `durationSeconds: 8` (docs). - Veo REST examples wrap reference images as `{ image: { inlineData }, referenceType: "asset" }` and frames as `{ inlineData }` (docs). `gcsUri` is not documented for the Gemini API. - Gemini 3.x function calling over generateContent needs thought signatures round-tripped and `FunctionResponse` `id` plus `name` matching the call; mismatches return empty responses with `STOP` rather than an error (docs). diff --git a/packages/ai-providers/src/fal/FalProvider.test.ts b/packages/ai-providers/src/fal/FalProvider.test.ts index a99903a5..b6e782b6 100644 --- a/packages/ai-providers/src/fal/FalProvider.test.ts +++ b/packages/ai-providers/src/fal/FalProvider.test.ts @@ -1,316 +1,174 @@ /** - * FalProvider unit tests. - * - * The real fal client is mocked via `vi.mock("@fal-ai/client")` so the - * tests run instantly with no network. Each case asserts the endpoint - * id we picked, the input payload we sent, or the failure path we - * surface — the behaviour the rest of the codebase relies on. + * FalProvider tests against the fake provider network: the real + * `@fal-ai/client` SDK runs, and only `fetch` is scripted, so endpoint + * choice, payloads, the queue protocol, and storage uploads are all real. */ -import { describe, it, expect, beforeEach, vi } from "vitest"; - -const mocks = vi.hoisted(() => ({ - subscribe: vi.fn(), - upload: vi.fn(), - createFalClient: vi.fn(), -})); - -vi.mock("@fal-ai/client", () => ({ - createFalClient: (...args: unknown[]) => { - mocks.createFalClient(...args); - return { subscribe: mocks.subscribe, storage: { upload: mocks.upload } }; - }, -})); +import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { describeVideoContract } from "../video/contract.testkit.js"; +import { FakeProviderNetwork, jsonResponse, type RecordedRequest } from "../testing/fake-provider-network.js"; import { FalProvider, estimateSeedanceVideoCostUsd, resolveSeedanceVariant } from "./FalProvider.js"; -describe("FalProvider", () => { - let provider: FalProvider; +const QUEUE = "queue.fal.run"; + +/** Script fal's queue: accepted, one IN_PROGRESS poll, then COMPLETED with `result` (or a failing result read). */ +function scriptQueue( + net: FakeProviderNetwork, + result: { body: unknown; status?: number } = { body: { video: { url: "https://fake.media/out.mp4" } } }, + completed: Record = {} +): void { + let polls = 0; + net + .on("POST", QUEUE, /^\/bytedance\//, () => + jsonResponse({ request_id: "req-1", status: "IN_QUEUE", queue_position: 0, response_url: "", status_url: "", cancel_url: "" }) + ) + .on("GET", QUEUE, /\/requests\/req-1\/status$/, () => + jsonResponse(++polls < 2 ? { status: "IN_PROGRESS", request_id: "req-1" } : { status: "COMPLETED", request_id: "req-1", ...completed }) + ) + .on("GET", QUEUE, /\/requests\/req-1$/, () => + jsonResponse(result.body, result.status ?? 200, { "x-fal-request-id": "req-1" }) + ) + .on("PUT", QUEUE, /\/requests\/req-1\/cancel$/, () => jsonResponse({ status: "CANCELLATION_REQUESTED" }, 202)) + // fal storage: initiate returns an upload URL and the file's public URL. + .on("POST", "rest.fal.ai", /\/storage\/upload\/initiate$/, () => + jsonResponse({ upload_url: "https://fake.media/upload-target", file_url: "https://fal.storage/uploaded.png" }) + ) + .on("PUT", "fake.media", /\/upload-target$/, () => new Response(null, { status: 200 })); +} + +async function create() { + const provider = new FalProvider(); + await provider.initialize({ apiKey: "fal_pst_test" }); + return provider; +} + +describeVideoContract("FalProvider (Seedance)", { + create, + request: { prompt: "A paper boat drifts", aspectRatio: "16:9", durationSec: 5 }, + defaultModel: "seedance-2.0", + scriptSuccess: (net) => scriptQueue(net), + scriptJobFailure: (net, reason) => + reason === "moderation" + ? scriptQueue(net, { + status: 422, + body: { detail: [{ loc: ["body", "prompt"], msg: "Output flagged by the content checker", type: "content_policy_violation" }] }, + }) + : scriptQueue(net, undefined, { error: "Internal server error", error_type: "internal_server_error" }), + scriptSubmitError: (net, status, body) => net.on("POST", QUEUE, /^\/bytedance\//, () => jsonResponse(body, status)), + cancel: true, + continuations: [], +}); + +describe("FalProvider (Seedance) requests", () => { + let net: FakeProviderNetwork; + const submitted = (): RecordedRequest => net.requests.find((r) => r.method === "POST" && r.host === QUEUE)!; + const input = () => submitted().body as Record; beforeEach(() => { - mocks.subscribe.mockReset(); - mocks.upload.mockReset(); - mocks.upload.mockResolvedValue("https://fal.storage/uploaded-reference.png"); - mocks.createFalClient.mockReset(); - provider = new FalProvider(); + net = new FakeProviderNetwork(); + net.install(); + scriptQueue(net); }); - describe("initialization", () => { - it("declares text-to-video and image-to-video capabilities", () => { - expect(provider.id).toBe("seedance"); - expect(provider.capabilities).toContain("text-to-video"); - expect(provider.capabilities).toContain("image-to-video"); - expect(provider.capabilities).toContain("reference-to-video"); - }); - - it("is unconfigured before initialize", () => { - expect(provider.isConfigured()).toBe(false); - }); + afterEach(() => net.uninstall()); - it("creates the fal client when an API key is supplied", async () => { - await provider.initialize({ apiKey: "fal_pst_test" }); - expect(mocks.createFalClient).toHaveBeenCalledWith({ credentials: "fal_pst_test" }); - expect(provider.isConfigured()).toBe(true); - }); + it("submits text-to-video to the standard endpoint with string-enum duration", async () => { + const fal = await create(); + const job = await fal.submitVideo({ prompt: "a cat surfing", aspectRatio: "16:9", durationSec: 6 }); - it("stays unconfigured when no API key is provided", async () => { - await provider.initialize({}); - expect(provider.isConfigured()).toBe(false); - }); + expect(submitted().path).toBe("/bytedance/seedance-2.0/text-to-video"); + expect(input()).toMatchObject({ prompt: "a cat surfing", aspect_ratio: "16:9", resolution: "720p", duration: "6" }); + expect(job.meta).toEqual({ endpoint: "bytedance/seedance-2.0/text-to-video" }); }); - describe("generateVideo — text-to-video", () => { - beforeEach(async () => { - await provider.initialize({ apiKey: "fal_pst_test" }); - }); - - it("hits the standard text-to-video endpoint by default", async () => { - mocks.subscribe.mockResolvedValueOnce({ - requestId: "req-abc", - data: { video: { url: "https://fal.media/output.mp4" } }, - }); - - const result = await provider.generateVideo("a cat surfing", { - prompt: "a cat surfing", - aspectRatio: "16:9", - duration: 6, - }); - - expect(mocks.subscribe).toHaveBeenCalledWith( - "bytedance/seedance-2.0/text-to-video", - expect.objectContaining({ - input: expect.objectContaining({ - prompt: "a cat surfing", - aspect_ratio: "16:9", - resolution: "720p", - duration: "6", - }), - }), - ); - expect(result).toMatchObject({ - id: "req-abc", - status: "completed", - videoUrl: "https://fal.media/output.mp4", - }); - }); - - it("routes to the fast variant when model = seedance-2.0-fast", async () => { - mocks.subscribe.mockResolvedValueOnce({ - requestId: "req-fast", - data: { video: { url: "https://fal.media/fast.mp4" } }, - }); - - await provider.generateVideo("any prompt", { - prompt: "any prompt", - model: "seedance-2.0-fast", - }); - - expect(mocks.subscribe.mock.calls[0][0]).toBe( - "bytedance/seedance-2.0/fast/text-to-video", - ); - }); - - it("clamps unreasonable durations into the 4-15 s API range, as the string enum fal documents", async () => { - mocks.subscribe.mockResolvedValueOnce({ - requestId: "req-clamp", - data: { video: { url: "https://x" } }, - }); - - await provider.generateVideo("p", { prompt: "p", duration: 99 }); - expect(mocks.subscribe.mock.calls[0][1].input.duration).toBe("15"); - - mocks.subscribe.mockResolvedValueOnce({ - requestId: "req-clamp2", - data: { video: { url: "https://x" } }, - }); - await provider.generateVideo("p", { prompt: "p", duration: 1 }); - expect(mocks.subscribe.mock.calls[1][1].input.duration).toBe("4"); - }); - - it("routes Seedance 2.5 to its endpoints, allows 30 s, and drops fields its schema lacks", async () => { - mocks.subscribe.mockResolvedValue({ requestId: "req-25", data: { video: { url: "https://x" } } }); - - await provider.generateVideo("p", { - prompt: "p", - model: "seedance-2.5", - duration: 99, - negativePrompt: "blur", - seed: 7, - }); - const [endpoint, { input }] = mocks.subscribe.mock.calls.at(-1)!; - expect(endpoint).toBe("bytedance/seedance-2.5/text-to-video"); - expect(input.duration).toBe("30"); - expect(input).not.toHaveProperty("negative_prompt"); - expect(input).not.toHaveProperty("seed"); - }); - - it("falls back to aspect=auto on unknown ratios", async () => { - mocks.subscribe.mockResolvedValueOnce({ - requestId: "req-aspect", - data: { video: { url: "https://x" } }, - }); - - await provider.generateVideo("p", { - prompt: "p", - aspectRatio: "5:3" as unknown as "16:9", - }); - - expect(mocks.subscribe.mock.calls[0][1].input.aspect_ratio).toBe("auto"); - }); - - it("returns a structured failure when subscribe rejects", async () => { - mocks.subscribe.mockRejectedValueOnce(new Error("rate limited")); - const result = await provider.generateVideo("p", { prompt: "p" }); - expect(result.status).toBe("failed"); - expect(result.error).toContain("rate limited"); - }); + it("routes the fast variant and Seedance 2.5, clamping duration to each one's range", async () => { + const fal = await create(); + await fal.submitVideo({ prompt: "p", model: "fast", durationSec: 99 }); + expect(submitted().path).toBe("/bytedance/seedance-2.0/fast/text-to-video"); + expect(input().duration).toBe("15"); + + net.requests.length = 0; + await fal.submitVideo({ prompt: "p", model: "2.5", durationSec: 99, negativePrompt: "blur", seed: 7 }); + expect(submitted().path).toBe("/bytedance/seedance-2.5/text-to-video"); + expect(input().duration).toBe("30"); + // Seedance 2.5's schema has no negative_prompt or seed. + expect(input()).not.toHaveProperty("negative_prompt"); + expect(input()).not.toHaveProperty("seed"); + + net.requests.length = 0; + await fal.submitVideo({ prompt: "p", durationSec: 1, aspectRatio: "5:3" as "16:9" }); + expect(input()).toMatchObject({ duration: "4", aspect_ratio: "auto" }); + }); - it("includes fal validation details when subscribe rejects with a body", async () => { - const error = Object.assign(new Error("Unprocessable Entity"), { - status: 422, - requestId: "req-validation", - body: { - detail: [ - { - loc: ["body", "image_urls", 0], - msg: "Invalid image URL", - type: "value_error", - }, - ], - }, - fieldErrors: [ - { - loc: ["body", "image_urls", 0], - msg: "Invalid image URL", - type: "value_error", - }, - ], - }); - mocks.subscribe.mockRejectedValueOnce(error); - - const result = await provider.generateVideo("p", { prompt: "p" }); - - expect(result.status).toBe("failed"); - expect(result.error).toContain("HTTP 422"); - expect(result.error).toContain("req-validation"); - expect(result.error).toContain("body.image_urls.0: Invalid image URL"); - }); + it("routes an HTTPS first frame to image-to-video", async () => { + const fal = await create(); + await fal.submitVideo({ prompt: "zoom in", image: "https://example.com/seed.png" }); - it("returns a structured failure when no video URL is returned", async () => { - mocks.subscribe.mockResolvedValueOnce({ - requestId: "req-empty", - data: {}, - }); - const result = await provider.generateVideo("p", { prompt: "p" }); - expect(result.status).toBe("failed"); - expect(result.error).toMatch(/video URL/); - }); + expect(submitted().path).toBe("/bytedance/seedance-2.0/image-to-video"); + expect(input().image_url).toBe("https://example.com/seed.png"); }); - describe("generateVideo — image-to-video", () => { - beforeEach(async () => { - await provider.initialize({ apiKey: "fal_pst_test" }); - }); + it("uploads a data-URI first frame instead of silently dropping it", async () => { + const fal = await create(); + await fal.submitVideo({ prompt: "zoom in", image: "data:image/png;base64,iVBORw0KGgo=" }); - it("routes to image-to-video when an HTTPS reference image is supplied", async () => { - mocks.subscribe.mockResolvedValueOnce({ - requestId: "req-i2v", - data: { video: { url: "https://fal.media/i2v.mp4" } }, - }); - - await provider.generateVideo("zoom in slowly", { - prompt: "zoom in slowly", - referenceImage: "https://example.com/seed.png", - }); - - expect(mocks.subscribe.mock.calls[0][0]).toBe( - "bytedance/seedance-2.0/image-to-video", - ); - expect(mocks.subscribe.mock.calls[0][1].input.image_url).toBe( - "https://example.com/seed.png", - ); - }); + expect(submitted().path).toBe("/bytedance/seedance-2.0/image-to-video"); + expect(input().image_url).toBe("https://fal.storage/uploaded.png"); + }); - it("ignores non-HTTPS reference images and falls back to text-to-video", async () => { - mocks.subscribe.mockResolvedValueOnce({ - requestId: "req-fallback", - data: { video: { url: "https://x" } }, - }); - - await provider.generateVideo("p", { - prompt: "p", - referenceImage: "data:image/png;base64,iVBORw0...", - }); - - expect(mocks.subscribe.mock.calls[0][0]).toBe( - "bytedance/seedance-2.0/text-to-video", - ); - expect(mocks.subscribe.mock.calls[0][1].input.image_url).toBeUndefined(); + it("routes references to reference-to-video, grouped by kind, uploading data URIs", async () => { + const fal = await create(); + await fal.submitVideo({ + prompt: "animate @Image1 with the timing of @Video1", + references: [ + { kind: "image", url: "data:image/png;base64,iVBORw0KGgo=" }, + { kind: "image", url: "https://example.com/face.png" }, + { kind: "video", url: "https://example.com/move.mp4" }, + { kind: "audio", url: "https://example.com/voice.mp3" }, + { kind: "image", url: "/tmp/local.png" }, + ], + generateAudio: false, + resolution: "1080p", + }); + + expect(submitted().path).toBe("/bytedance/seedance-2.0/reference-to-video"); + expect(input()).toMatchObject({ + image_urls: ["https://fal.storage/uploaded.png", "https://example.com/face.png"], + video_urls: ["https://example.com/move.mp4"], + audio_urls: ["https://example.com/voice.mp3"], + generate_audio: false, + resolution: "1080p", }); }); - describe("generateVideo — reference-to-video", () => { - beforeEach(async () => { - await provider.initialize({ apiKey: "fal_pst_test" }); - }); + it("reports the likeness filter, which fal raises when the result is read, as a failed job", async () => { + net.on("GET", QUEUE, /\/requests\/req-1$/, () => + jsonResponse( + { detail: [{ msg: "The images or videos provided may contain likenesses of real people", type: "content_policy_violation" }] }, + 422 + ) + ); + const fal = await create(); + const job = await fal.submitVideo({ prompt: "p", image: "https://example.com/face.png" }); + let state = await fal.getVideoJob(job); + while (state.status !== "failed" && state.status !== "completed") state = await fal.getVideoJob(job); + + expect(state).toMatchObject({ status: "failed", error: { kind: "likeness", status: 422, code: "content_policy_violation" } }); + }); - it("routes explicit references to reference-to-video with grouped inputs", async () => { - mocks.subscribe.mockResolvedValueOnce({ - requestId: "req-ref", - data: { video: { url: "https://fal.media/ref.mp4" } }, - }); - - await provider.generateVideo("animate @Image1 with the timing of @Video1", { - prompt: "animate @Image1 with the timing of @Video1", - references: [ - { kind: "image", url: "data:image/png;base64,iVBORw0" }, - { kind: "image", url: "https://example.com/face.png" }, - { kind: "video", url: "https://example.com/move.mp4" }, - { kind: "audio", url: "https://example.com/voice.mp3" }, - ], - generateAudio: false, - resolution: "1080p", - }); - - expect(mocks.subscribe.mock.calls[0][0]).toBe( - "bytedance/seedance-2.0/reference-to-video", - ); - expect(mocks.subscribe.mock.calls[0][1].input).toMatchObject({ - prompt: "animate @Image1 with the timing of @Video1", - image_urls: ["https://fal.storage/uploaded-reference.png", "https://example.com/face.png"], - video_urls: ["https://example.com/move.mp4"], - audio_urls: ["https://example.com/voice.mp3"], - generate_audio: false, - resolution: "1080p", - }); - expect(mocks.upload).toHaveBeenCalledTimes(1); - }); + it("keeps the older blocking generateVideo working on top of the queue", async () => { + const fal = await create(); + (fal as unknown as { pollingInterval: number }).pollingInterval = 1; + const result = await fal.generateVideo("p", { prompt: "p" }); - it("drops unsupported reference URL shapes before choosing endpoint", async () => { - mocks.subscribe.mockResolvedValueOnce({ - requestId: "req-text", - data: { video: { url: "https://fal.media/text.mp4" } }, - }); - - await provider.generateVideo("p", { - prompt: "p", - references: [{ kind: "image", url: "/tmp/local.png" }], - }); - - expect(mocks.subscribe.mock.calls[0][0]).toBe( - "bytedance/seedance-2.0/text-to-video", - ); - expect(mocks.subscribe.mock.calls[0][1].input.image_urls).toBeUndefined(); - }); + expect(result).toMatchObject({ id: "req-1", status: "completed", videoUrl: "https://fake.media/out.mp4" }); }); - describe("error handling without init", () => { - it("returns a clean error when generateVideo is called before initialize", async () => { - const fresh = new FalProvider(); - const result = await fresh.generateVideo("p", { prompt: "p" }); - expect(result.status).toBe("failed"); - expect(result.error).toMatch(/FAL_API_KEY/); - }); + it("fails cleanly before initialize", async () => { + const result = await new FalProvider().generateVideo("p", { prompt: "p" }); + expect(result.status).toBe("failed"); + expect(result.error).toMatch(/FAL_API_KEY/); }); }); diff --git a/packages/ai-providers/src/fal/FalProvider.ts b/packages/ai-providers/src/fal/FalProvider.ts index df01e3ff..a7aaa9e6 100644 --- a/packages/ai-providers/src/fal/FalProvider.ts +++ b/packages/ai-providers/src/fal/FalProvider.ts @@ -1,4 +1,4 @@ -import { createFalClient, type FalClient } from "@fal-ai/client"; +import { ApiError, createFalClient, type FalClient } from "@fal-ai/client"; import type { AIProvider, AICapability, @@ -8,6 +8,11 @@ import type { VideoResult, } from "../interface/types.js"; import { findModel, modelAliases } from "../catalog/catalog.js"; +import type { VideoGenerator, VideoJob, VideoJobState, VideoRequest } from "../video/contract.js"; +import { ProviderError, classifyProviderError, isProviderError } from "../shared/errors.js"; +import { providerRequest } from "../shared/http.js"; +import { resolveVideoModel } from "../video/models.js"; +import { waitForVideoJob } from "../video/wait.js"; /** * fal.ai endpoints for ByteDance Seedance 2.0. @@ -45,7 +50,6 @@ const ENDPOINT_REFERENCE_TO_VIDEO: Record = { "seedance-2.5": "bytedance/seedance-2.5/reference-to-video", }; -const DEFAULT_VARIANT: SeedanceVariant = "seedance-2.0"; /** Longest clip each variant accepts, in seconds (2.0: 4-15, 2.5: 4-30). */ const MAX_DURATION_SEC: Record = { @@ -142,7 +146,14 @@ interface SeedanceOutput { * carry an audio track. Useful when the caller is happy to let the * model compose the soundtrack. */ -export class FalProvider implements AIProvider { +/** + * ByteDance Seedance on fal.ai. + * + * Runs on fal's queue API (submit, status, result, cancel) rather than the + * blocking `subscribe`, so a job keeps its request ID from the moment it is + * accepted and survives the caller crashing mid-wait. + */ +export class FalProvider implements AIProvider, VideoGenerator { id = "seedance"; name = "fal.ai (Seedance 2.0)"; description = "fal.ai hosting ByteDance Seedance 2.0 — Artificial Analysis #2 on both text-to-video and image-to-video leaderboards"; @@ -152,6 +163,7 @@ export class FalProvider implements AIProvider { private client?: FalClient; private apiKey?: string; + private pollingInterval = 5000; async initialize(config: ProviderConfig): Promise { this.apiKey = config.apiKey; @@ -163,99 +175,192 @@ export class FalProvider implements AIProvider { return !!this.apiKey && !!this.client; } - /** - * Generate a video from a text prompt — or from a reference image when - * `options.referenceImage` is set (treated as an image-to-video - * request, routed to a different endpoint). - * - * fal.subscribe blocks until the final result is available. It also - * surfaces queue / log events through `onQueueUpdate`, which we drop - * silently for now — the CLI's existing spinner is the user-facing - * progress UI. - */ - async generateVideo( - prompt: string, - options?: GenerateOptions, - ): Promise { - if (!this.client) { - return { - id: "", - status: "failed", - error: "fal.ai API key not configured. Set FAL_API_KEY in .env.", - }; - } + // ── VideoGenerator ──────────────────────────────────────────────────── - const variant = (options?.model as SeedanceVariant | undefined) ?? DEFAULT_VARIANT; - if (!Object.hasOwn(ENDPOINT_TEXT_TO_VIDEO, variant)) { - return { - id: "", - status: "failed", - error: `Unknown Seedance variant: ${variant}. Valid: ${Object.keys(ENDPOINT_TEXT_TO_VIDEO).join(", ")}.`, - }; + async submitVideo(request: VideoRequest): Promise { + const client = this.requireClient(); + if (request.from) { + throw new ProviderError({ + kind: "unsupported", + provider: this.id, + message: `Seedance cannot ${request.from.kind} an earlier video.`, + }); } + const variant = resolveVideoModel(this.id, request.model).id as SeedanceVariant; - const references = await normaliseReferences(options?.references, this.client); + const references = await this.falCall(() => normaliseReferences(request.references, client)); const hasReferences = references.length > 0; - const referenceImage = hasReferences ? undefined : pickReferenceImageUrl(options?.referenceImage); - const isImageToVideo = !!referenceImage; - const endpointId = hasReferences + const image = !hasReferences && request.image ? await this.falCall(() => toFalUrl(client, request.image!)) : undefined; + const endpoint = hasReferences ? ENDPOINT_REFERENCE_TO_VIDEO[variant] - : isImageToVideo - ? ENDPOINT_IMAGE_TO_VIDEO[variant] - : ENDPOINT_TEXT_TO_VIDEO[variant]; - - const aspect = normaliseAspect(options?.aspectRatio); - const resolution = normaliseResolution(options?.resolution); - const duration = normaliseDuration(options?.duration, MAX_DURATION_SEC[variant]); + : image + ? ENDPOINT_IMAGE_TO_VIDEO[variant] + : ENDPOINT_TEXT_TO_VIDEO[variant]; const input: Record = { - prompt, - aspect_ratio: aspect, - resolution, - duration, + prompt: request.prompt, + aspect_ratio: normaliseAspect(request.aspectRatio), + resolution: normaliseResolution(request.resolution), + duration: normaliseDuration(request.durationSec, MAX_DURATION_SEC[variant]), }; if (hasReferences) { const grouped = groupReferences(references); if (grouped.image_urls.length > 0) input.image_urls = grouped.image_urls; if (grouped.video_urls.length > 0) input.video_urls = grouped.video_urls; if (grouped.audio_urls.length > 0) input.audio_urls = grouped.audio_urls; - } else if (referenceImage) { - input.image_url = referenceImage; + } else if (image) { + input.image_url = image; } // Seedance 2.5's schema has no negative_prompt or seed. if (variant !== "seedance-2.5") { - if (options?.negativePrompt) input.negative_prompt = options.negativePrompt; - if (typeof options?.seed === "number") input.seed = options.seed; + if (request.negativePrompt) input.negative_prompt = request.negativePrompt; + if (typeof request.seed === "number") input.seed = request.seed; + } + if (typeof request.generateAudio === "boolean") input.generate_audio = request.generateAudio; + if (typeof request.providerOptions?.endUserId === "string") input.end_user_id = request.providerOptions.endUserId; + if (!hasReferences && request.lastFrame) { + input.end_image_url = await this.falCall(() => toFalUrl(client, request.lastFrame!)); } - if (typeof options?.generateAudio === "boolean") input.generate_audio = options.generateAudio; - if (options?.endUserId) input.end_user_id = options.endUserId; - if (!hasReferences && options?.lastFrame) input.end_image_url = options.lastFrame; + const queued = await this.falCall(() => client.queue.submit(endpoint, { input })); + return { + provider: this.id, + id: queued.request_id, + model: variant, + submittedAt: new Date().toISOString(), + meta: { endpoint }, + }; + } + + async getVideoJob(job: VideoJob): Promise { + const client = this.requireClient(); + const endpoint = this.endpointOf(job); + const status = await this.falCall(() => client.queue.status(endpoint, { requestId: job.id, logs: false })); + if (status.status === "IN_QUEUE") return { status: "queued" }; + if (status.status === "IN_PROGRESS") return { status: "processing" }; + + // fal: a COMPLETED status can still be a failure; check `error` first. + const failure = status as { error?: string; error_type?: string }; + if (failure.error) { + return { status: "failed", error: classifyProviderError({ provider: this.id, message: failure.error, code: failure.error_type }).toJSON() }; + } try { - const out = await this.client.subscribe(endpointId, { input, logs: false }); - const data = (out?.data ?? {}) as SeedanceOutput; - const url = data.video?.url; + const result = await client.queue.result(endpoint, { requestId: job.id }); + const url = (result.data as SeedanceOutput | undefined)?.video?.url; if (!url) { return { - id: typeof out?.requestId === "string" ? out.requestId : "", status: "failed", - error: "fal.subscribe returned without a video URL", + error: new ProviderError({ kind: "provider", provider: this.id, message: "Seedance finished without a video URL." }).toJSON(), }; } + return { status: "completed", progress: 100, videoUrl: url }; + } catch (error) { + // A rejected job (e.g. the 422 likeness filter) surfaces when its + // result is read: the job failed, the read did not. + const classified = this.toProviderError(error); + if (classified.retryable) throw classified; + return { status: "failed", error: classified.toJSON() }; + } + } + + /** fal's cancel is best effort: an in-progress job may still finish. */ + async cancelVideoJob(job: VideoJob): Promise { + const client = this.requireClient(); + await this.falCall(() => client.queue.cancel(this.endpointOf(job), { requestId: job.id })); + } + + async downloadVideo(_job: VideoJob, state: VideoJobState): Promise { + if (!state.videoUrl) { + throw new ProviderError({ kind: "not-found", provider: this.id, message: "Seedance job has no video to download." }); + } + const response = await providerRequest(this.id, state.videoUrl); + return new Uint8Array(await response.arrayBuffer()); + } + + // ── Older interface, kept until every caller uses the contract ──────── + + /** + * Submit and wait for the finished video, as the blocking `subscribe` + * call used to. New callers should submit and poll so the job survives. + */ + async generateVideo(prompt: string, options?: GenerateOptions): Promise { + let job: VideoJob | undefined; + try { + job = await this.submitVideo({ + prompt, + model: options?.model, + durationSec: options?.duration, + aspectRatio: options?.aspectRatio, + resolution: options?.resolution, + image: typeof options?.referenceImage === "string" ? options.referenceImage : undefined, + lastFrame: options?.lastFrame, + references: options?.references, + negativePrompt: options?.negativePrompt, + seed: options?.seed, + generateAudio: options?.generateAudio, + providerOptions: options?.endUserId ? { endUserId: options.endUserId } : undefined, + }); + const state = await waitForVideoJob(this, job, { intervalMs: this.pollingInterval, timeoutMs: 30 * 60_000 }); return { - id: typeof out?.requestId === "string" ? out.requestId : "", - status: "completed", - videoUrl: url, - progress: 100, - }; - } catch (err) { - return { - id: "", - status: "failed", - error: formatFalError(err), + id: job.id, + status: state.status, + videoUrl: state.videoUrl, + progress: state.progress, + ...(state.error ? { error: state.error.message } : {}), }; + } catch (error) { + return { id: job?.id ?? "", status: "failed", error: isProviderError(error) || error instanceof Error ? error.message : String(error) }; } } + + // ── Helpers ─────────────────────────────────────────────────────────── + + private requireClient(): FalClient { + if (!this.client) { + throw new ProviderError({ kind: "auth", provider: this.id, message: "fal.ai API key not configured. Set FAL_API_KEY in .env." }); + } + return this.client; + } + + private endpointOf(job: VideoJob): string { + const endpoint = job.meta?.endpoint; + if (!endpoint) { + throw new ProviderError({ kind: "invalid-request", provider: this.id, message: `Seedance job ${job.id} has no endpoint in its handle.` }); + } + return endpoint; + } + + /** Run an SDK call, turning its errors into classified `ProviderError`s. */ + private async falCall(call: () => Promise): Promise { + try { + return await call(); + } catch (error) { + throw this.toProviderError(error); + } + } + + private toProviderError(error: unknown): ProviderError { + if (isProviderError(error)) return error; + if (error instanceof ApiError) { + return classifyProviderError({ + provider: this.id, + status: error.status, + code: falErrorType(error), + message: formatFalError(error), + }); + } + return new ProviderError({ + kind: "network", + provider: this.id, + message: error instanceof Error ? error.message : String(error), + }); + } +} + +/** fal's documented error `type` (e.g. `content_policy_violation`), the field to branch on. */ +function falErrorType(error: ApiError): string | undefined { + const body = error.body as { detail?: Array<{ type?: string }> | string } | undefined; + return Array.isArray(body?.detail) ? body.detail.find((d) => typeof d.type === "string")?.type : undefined; } function formatFalError(error: unknown): string { @@ -300,15 +405,17 @@ function safeJson(value: unknown): string { } /** - * Accept the same `referenceImage` shapes other providers do. fal.ai - * needs an HTTPS URL — base64 / Blob isn't supported here yet (callers - * that have only a local file should upload it via fal.storage first, - * which we leave as a follow-up). + * fal takes HTTPS URLs; a data URI (a local file the caller read in) is + * uploaded to fal storage first, as references already were. */ -function pickReferenceImageUrl(input: unknown): string | undefined { - if (typeof input !== "string") return undefined; - if (input.startsWith("http://") || input.startsWith("https://")) return input; - return undefined; +async function toFalUrl(client: FalClient, value: string): Promise { + if (value.startsWith("http://") || value.startsWith("https://")) return value; + if (value.startsWith("data:")) return uploadDataUri(client, value); + throw new ProviderError({ + kind: "invalid-request", + provider: "seedance", + message: "Seedance needs an image URL or a data URI.", + }); } function isFalFileInput(value: string): boolean { diff --git a/packages/ai-providers/src/gemini/gemini-omni.test.ts b/packages/ai-providers/src/gemini/gemini-omni.test.ts index 77060f73..3f23306b 100644 --- a/packages/ai-providers/src/gemini/gemini-omni.test.ts +++ b/packages/ai-providers/src/gemini/gemini-omni.test.ts @@ -1,89 +1,128 @@ -import { afterEach, describe, expect, it, vi } from "vitest"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; -import { OMNI_MODEL, OmniProvider, findOmniVideo } from "./gemini-omni.js"; +import { describeVideoContract } from "../video/contract.testkit.js"; +import { FAKE_MP4, FakeProviderNetwork, jsonResponse } from "../testing/fake-provider-network.js"; +import { OmniProvider, findOmniVideo } from "./gemini-omni.js"; -function json(body: unknown, status = 200) { - return new Response(JSON.stringify(body), { status }); +const HOST = "generativelanguage.googleapis.com"; +const VIDEO_STEP = { + type: "model_output", + content: [{ type: "video", mime_type: "video/mp4", uri: `https://${HOST}/v1beta/files/file-1:download?alt=media` }], +}; + +/** Accepted, one in-progress poll, then `final`; the file is ACTIVE unless told otherwise. */ +function scriptInteraction(net: FakeProviderNetwork, final: Record, fileStates = ["ACTIVE"]): void { + let polls = 0; + net + .on("POST", HOST, /^\/v1beta\/interactions$/, () => jsonResponse({ id: "int-1", status: "in_progress" })) + .on("GET", HOST, /^\/v1beta\/interactions\/int-1$/, () => + jsonResponse(++polls < 2 ? { id: "int-1", status: "in_progress" } : { id: "int-1", ...final }) + ) + .on("POST", HOST, /^\/v1beta\/interactions\/int-1\/cancel$/, () => jsonResponse({ id: "int-1", status: "cancelled" })) + .on("GET", HOST, /^\/v1beta\/files\/file-1$/, () => jsonResponse({ state: fileStates.shift() ?? "ACTIVE" })) + .on("GET", HOST, /^\/download\/v1beta\/files\/file-1:download$/, () => new Response(new Uint8Array(FAKE_MP4))); +} + +async function create() { + const provider = new OmniProvider(); + await provider.initialize({ apiKey: "test-google" }); + return provider; } -describe("OmniProvider", () => { - afterEach(() => { - vi.unstubAllGlobals(); +describeVideoContract("OmniProvider", { + create, + request: { prompt: "A paper boat drifts", aspectRatio: "16:9" }, + defaultModel: "gemini-omni-1.1-flash", + scriptSuccess: (net) => { + scriptInteraction(net, { status: "completed", steps: [VIDEO_STEP] }); + // The contract suite downloads from fake.media; Omni's real URL is the Files API. + net.on("GET", HOST, /^\/download\//, () => new Response(new Uint8Array(FAKE_MP4))); + }, + scriptJobFailure: (net, reason) => + scriptInteraction( + net, + reason === "moderation" + ? { status: "completed", steps: [{ type: "model_output", content: [{ type: "text", text: "I can't make that." }] }] } + : { status: "failed", error: { code: 500, status: "INTERNAL", message: "Internal error encountered." } } + ), + scriptSubmitError: (net, status, body) => + net.on("POST", HOST, /^\/v1beta\/interactions$/, () => jsonResponse(body, status)), + cancel: true, + continuations: ["edit", "extend"], + videoUrl: `https://${HOST}/download/v1beta/files/file-1:download?alt=media`, +}); + +describe("OmniProvider requests", () => { + let net: FakeProviderNetwork; + const submitted = () => + net.requests.filter((r) => r.method === "POST" && r.path === "/v1beta/interactions").at(-1)!.body as Record; + + beforeEach(() => { + net = new FakeProviderNetwork(); + net.install(); + scriptInteraction(net, { status: "completed", steps: [VIDEO_STEP] }); }); - it("sends the documented interactions request and waits for the video file", async () => { - const fetchMock = vi - .fn() - .mockResolvedValueOnce( - json({ - id: "v1_abc", - status: "completed", - steps: [ - { type: "user_input", content: [{ type: "text", text: "boat" }] }, - { - type: "model_output", - content: [{ type: "video", mime_type: "video/mp4", uri: "https://generativelanguage.googleapis.com/v1beta/files/f123" }], - }, - ], - }) - ) - .mockResolvedValueOnce(json({ state: "ACTIVE" })); - vi.stubGlobal("fetch", fetchMock); - - const omni = new OmniProvider(); - await omni.initialize({ apiKey: "key" }); - const result = await omni.generateVideo("boat", { - prompt: "boat", - referenceImage: "data:image/jpeg;base64,AAAA", - aspectRatio: "9:16", - resolution: "360p", - }); + afterEach(() => net.uninstall()); - const [url, init] = fetchMock.mock.calls[0]; - expect(url).toBe("https://generativelanguage.googleapis.com/v1beta/interactions"); - expect(init.headers["x-goog-api-key"]).toBe("key"); - const body = JSON.parse(init.body as string); - expect(body).toEqual({ - model: OMNI_MODEL, - input: [ - { type: "image", data: "AAAA", mime_type: "image/jpeg" }, - { type: "text", text: "boat" }, - ], - response_format: { type: "video", aspect_ratio: "9:16", resolution: "360p", delivery: "uri" }, - }); - expect(fetchMock.mock.calls[1][0]).toBe("https://generativelanguage.googleapis.com/v1beta/files/f123"); - expect(result).toEqual({ - id: "v1_abc", - status: "completed", - videoUrl: "https://generativelanguage.googleapis.com/download/v1beta/files/f123:download?alt=media", + it("submits text-to-video in the background with a bare text input and uri delivery", async () => { + const omni = await create(); + await omni.submitVideo({ prompt: "a paper boat", aspectRatio: "9:16", resolution: "1080p" }); + + expect(submitted()).toEqual({ + model: "gemini-omni-1.1-flash", + input: "a paper boat", + background: true, + response_format: { type: "video", aspect_ratio: "9:16", resolution: "1080p", delivery: "uri" }, }); - expect(result.videoUrl).not.toContain("key="); + expect(net.requests[0].headers["x-goog-api-key"]).toBe("test-google"); + expect(net.requests[0].url).not.toContain("key="); }); - it("sends a bare text input for text-to-video", async () => { - const fetchMock = vi.fn().mockResolvedValueOnce(json({ id: "v1", status: "failed", error: { message: "blocked" } })); - vi.stubGlobal("fetch", fetchMock); + it("sends first and last frames inline before the text", async () => { + const omni = await create(); + await omni.submitVideo({ prompt: "p", image: "data:image/jpeg;base64,AAA", lastFrame: "data:image/png;base64,BBB" }); - const omni = new OmniProvider(); - await omni.initialize({ apiKey: "key" }); - const result = await omni.generateVideo("a pond", { prompt: "a pond" }); + expect(submitted().input).toEqual([ + { type: "image", mime_type: "image/jpeg", data: "AAA" }, + { type: "image", mime_type: "image/png", data: "BBB" }, + { type: "text", text: "p" }, + ]); + }); - expect(JSON.parse(fetchMock.mock.calls[0][1].body as string).input).toBe("a pond"); - expect(result.status).toBe("failed"); - expect(result.error).toContain("blocked"); + it("refuses an image URL, since Omni only takes inline frames", async () => { + const omni = await create(); + await expect(omni.submitVideo({ prompt: "p", image: "https://fake.media/a.png" })).rejects.toMatchObject({ + kind: "invalid-request", + }); + expect(net.requests).toHaveLength(0); }); - it("surfaces HTTP errors", async () => { - vi.stubGlobal("fetch", vi.fn().mockResolvedValueOnce(json({ error: { message: "Unknown parameter 'inputs'." } }, 400))); - const omni = new OmniProvider(); - await omni.initialize({ apiKey: "key" }); - const result = await omni.generateVideo("x", { prompt: "x" }); - expect(result.status).toBe("failed"); - expect(result.error).toContain("HTTP 400"); + it.each(["edit", "extend"] as const)("chains a %s onto the earlier interaction", async (kind) => { + const omni = await create(); + const first = await omni.submitVideo({ prompt: "a paper boat" }); + await omni.submitVideo({ prompt: "make it night", from: { job: first, kind } }); + + expect(submitted()).toMatchObject({ previous_interaction_id: "int-1" }); + expect(String(submitted().input)).toContain("make it night"); + }); + + it("keeps polling while the finished video's file is still processing", async () => { + net.uninstall(); + net = new FakeProviderNetwork(); + net.install(); + scriptInteraction(net, { status: "completed", steps: [VIDEO_STEP] }, ["PROCESSING", "ACTIVE"]); + const omni = await create(); + const job = await omni.submitVideo({ prompt: "p" }); + const states: string[] = []; + for (let i = 0; i < 4; i++) states.push((await omni.getVideoJob(job)).status); + + expect(states).toEqual(["processing", "processing", "completed", "completed"]); }); it("finds the video in steps or the SDK-style output_video field", () => { - expect(findOmniVideo({ output_video: { type: "video", uri: "files/x" } })?.uri).toBe("files/x"); - expect(findOmniVideo({ steps: [{ type: "thought", content: [{ type: "video", uri: "no" }] }] })).toBeUndefined(); + expect(findOmniVideo({ steps: [VIDEO_STEP] })?.uri).toContain("file-1"); + expect(findOmniVideo({ output_video: { type: "video", uri: "x" } })?.uri).toBe("x"); + expect(findOmniVideo({ steps: [] })).toBeUndefined(); }); }); diff --git a/packages/ai-providers/src/gemini/gemini-omni.ts b/packages/ai-providers/src/gemini/gemini-omni.ts index 8624062d..7ddb6a10 100644 --- a/packages/ai-providers/src/gemini/gemini-omni.ts +++ b/packages/ai-providers/src/gemini/gemini-omni.ts @@ -2,37 +2,42 @@ * @module gemini/gemini-omni * * Google Gemini Omni 1.1 Flash (`gemini-omni-1.1-flash`) text- and - * image-to-video. GA since 2026-08-27 and Google's named replacement for the - * Veo 3.1 previews that shut down on 2026-10-22. + * image-to-video, plus multi-turn edits and extensions. GA since 2026-08-27 + * and Google's named replacement for the Veo 3.1 previews that shut down on + * 2026-10-22. * - * Omni uses the stateful `POST /v1beta/interactions` endpoint (not Veo's - * `:predictLongRunning`) with the same `GOOGLE_API_KEY`. Request shape per - * https://ai.google.dev/gemini-api/docs/omni: + * Omni runs on the Interactions API (`/v1beta/interactions`) with the same + * `GOOGLE_API_KEY`. Request shape per https://ai.google.dev/gemini-api/docs/omni: + * `input` is a string or typed items (`{type:"image", data, mime_type}` + * frames, then `{type:"text", text}`), and `response_format` is + * `{type:"video", aspect_ratio, resolution, delivery}`. * - * - `input`: a string, or typed items (`{type:"image", data, mime_type}` for - * frames, then `{type:"text", text}`). - * - `response_format`: `{type:"video", aspect_ratio, resolution, delivery}`. + * We submit with `background: true` and poll `GET /interactions/{id}`. The + * background docs only list text models, but Omni accepts it, and the + * completed interaction's `steps[]` carries the video `uri` (probe, + * 2026-10-04). The `uri` points at a Files API entry that must reach ACTIVE + * before it downloads, with the key as a header, never in the URL. * - * The video comes back in `steps[].content[]` (`type:"video"`). We ask for - * `delivery:"uri"` because inline base64 is only meant for clips under 4 MB, - * then wait for the Files API entry to become ACTIVE and hand the CLI a - * download URL that needs the key as a header, never in the URL. - * - * Omni picks the clip length itself (3-10 s); there is no duration field. + * Edits and extensions chain with `previous_interaction_id`, so an Omni job + * handle is also its session handle. Omni picks the clip length itself + * (3-10 s); there is no duration field. */ import type { GenerateOptions, VideoResult } from "../interface/types.js"; import type { ProviderConfig } from "../interface/index.js"; -import { sleep } from "../shared/http.js"; import { defaultModel } from "../catalog/catalog.js"; +import type { VideoGenerator, VideoJob, VideoJobState, VideoRequest } from "../video/contract.js"; +import { ProviderError, classifyProviderError, isProviderError } from "../shared/errors.js"; +import { providerRequest } from "../shared/http.js"; +import { resolveVideoModel } from "../video/models.js"; +import { waitForVideoJob } from "../video/wait.js"; export const OMNI_MODEL = defaultModel("omni", "video").id; const API_ROOT = "https://generativelanguage.googleapis.com"; -const FILE_POLL_INTERVAL_MS = 3000; -const FILE_POLL_TIMEOUT_MS = 5 * 60 * 1000; /** Output resolutions Omni accepts; 1080p and 4k are upscaled from 720p. */ export type OmniResolution = "360p" | "720p" | "1080p" | "4k"; +const RESOLUTIONS: readonly string[] = ["360p", "720p", "1080p", "4k"]; interface InteractionContent { type?: string; @@ -43,26 +48,19 @@ interface InteractionContent { interface InteractionResponse { id?: string; - status?: string; + status?: "in_progress" | "requires_action" | "completed" | "failed" | "cancelled" | string; steps?: Array<{ type?: string; content?: InteractionContent[] }>; output_video?: InteractionContent; - error?: { message?: string }; + error?: { code?: number | string; message?: string; status?: string }; } -/** Inline images for `input`: the first frame, then an optional last frame. */ -function frameImages(options: GenerateOptions): Array<{ base64: string; mimeType: string }> { - const frames: Array<{ base64: string; mimeType: string }> = []; - const toInline = (ref: string): { base64: string; mimeType: string } => { - const m = ref.match(/^data:(.+?);base64,(.*)$/); - return m ? { mimeType: m[1], base64: m[2] } : { mimeType: "image/png", base64: ref }; - }; - if (options.referenceImages?.[0]?.base64) frames.push(options.referenceImages[0]); - else if (typeof options.referenceImage === "string" && options.referenceImage) { - frames.push(toInline(options.referenceImage)); - } - if (frames.length > 0 && options.lastFrame) frames.push(toInline(options.lastFrame)); - return frames; -} +const STATUS: Record = { + in_progress: "processing", + requires_action: "processing", + completed: "completed", + failed: "failed", + cancelled: "cancelled", +}; /** The generated video item from a REST interaction response. */ export function findOmniVideo(response: InteractionResponse): InteractionContent | undefined { @@ -74,19 +72,22 @@ export function findOmniVideo(response: InteractionResponse): InteractionContent return response.output_video; } -/** `https://.../v1beta/files/abc` or `files/abc` → `abc`. */ +/** `https://.../v1beta/files/abc:download?alt=media` or `files/abc` → `abc`. */ function fileIdFromUri(uri: string): string | undefined { return uri.match(/files\/([^/?:]+)/)?.[1]; } -/** - * Gemini Omni video client. Mirrors the shape the CLI expects from other - * video providers (`initialize` + `generateVideo` → {@link VideoResult}). - */ -export class OmniProvider { +/** A data URI or bare base64 string as an inline image item. */ +function inlineImage(ref: string): { type: "image"; data: string; mime_type: string } { + const m = ref.match(/^data:(.+?);base64,(.*)$/s); + return m ? { type: "image", mime_type: m[1], data: m[2] } : { type: "image", mime_type: "image/png", data: ref }; +} + +export class OmniProvider implements VideoGenerator { id = "omni"; label = "Gemini Omni 1.1 Flash"; private apiKey?: string; + private pollingInterval = 5000; async initialize(config: ProviderConfig): Promise { this.apiKey = config.apiKey; @@ -96,85 +97,157 @@ export class OmniProvider { return !!this.apiKey; } - async generateVideo(prompt: string, options?: GenerateOptions): Promise { - if (!this.apiKey) { - return { id: "", status: "failed", error: "GOOGLE_API_KEY not configured for Gemini Omni" }; + // ── VideoGenerator ──────────────────────────────────────────────────── + + async submitVideo(request: VideoRequest): Promise { + const apiKey = this.requireKey(); + const model = resolveVideoModel(this.id, request.model).id; + const from = request.from; + if (from && from.job.provider !== this.id) { + throw new ProviderError({ + kind: "invalid-request", + provider: this.id, + message: `Omni can only continue an Omni video, not a ${from.job.provider} job.`, + }); } - const opts = options ?? ({ prompt } as GenerateOptions); - const frames = frameImages(opts); - const input = - frames.length === 0 - ? prompt - : [ - ...frames.map((f) => ({ type: "image", data: f.base64, mime_type: f.mimeType })), - { type: "text", text: prompt }, - ]; - const body = { - model: OMNI_MODEL, - input, + if (request.image?.startsWith("http")) { + throw new ProviderError({ + kind: "invalid-request", + provider: this.id, + message: "Omni takes frames inline; pass the image as a data URI, not a URL.", + }); + } + + const text = from?.kind === "extend" ? `Extend this video from its last frame. ${request.prompt}` : request.prompt; + const frames = request.image ? [inlineImage(request.image), ...(request.lastFrame ? [inlineImage(request.lastFrame)] : [])] : []; + const body: Record = { + model, + input: frames.length === 0 ? text : [...frames, { type: "text", text }], + background: true, response_format: { type: "video", - aspect_ratio: opts.aspectRatio === "9:16" ? "9:16" : "16:9", - resolution: (opts.resolution as OmniResolution | undefined) ?? "720p", + aspect_ratio: request.aspectRatio === "9:16" ? "9:16" : "16:9", + resolution: RESOLUTIONS.includes(request.resolution ?? "") ? request.resolution : "720p", delivery: "uri", }, }; + if (from) body.previous_interaction_id = from.job.id; - try { - const res = await fetch(`${API_ROOT}/v1beta/interactions`, { - method: "POST", - headers: { "Content-Type": "application/json", "x-goog-api-key": this.apiKey }, - body: JSON.stringify(body), - }); - const text = await res.text(); - if (!res.ok) { - return { - id: "", - status: "failed", - error: `Gemini Omni request failed: HTTP ${res.status} - ${text.slice(0, 300)}`, - }; - } - const parsed = JSON.parse(text) as InteractionResponse; - const id = parsed.id ?? ""; - const video = findOmniVideo(parsed); - if (!video?.uri) { - const reason = parsed.error?.message ?? `status ${parsed.status ?? "unknown"}`; - return { id, status: "failed", error: `Gemini Omni returned no video (${reason})` }; - } - const fileId = fileIdFromUri(video.uri); - if (!fileId) { - return { id, status: "failed", error: `Gemini Omni returned an unexpected video URI: ${video.uri}` }; - } - const ready = await this.waitForFile(fileId); - if (ready !== "ACTIVE") { - return { id, status: "failed", error: `Gemini Omni video file ended in state ${ready}` }; - } + const response = await providerRequest(this.id, `${API_ROOT}/v1beta/interactions`, { + method: "POST", + headers: { "Content-Type": "application/json", "x-goog-api-key": apiKey }, + body: JSON.stringify(body), + }); + const created = (await response.json()) as InteractionResponse; + if (!created.id) { + throw new ProviderError({ kind: "provider", provider: this.id, message: "Omni accepted the request without an interaction ID." }); + } + return { provider: this.id, id: created.id, model, submittedAt: new Date().toISOString() }; + } + + async getVideoJob(job: VideoJob): Promise { + const apiKey = this.requireKey(); + const response = await providerRequest(this.id, `${API_ROOT}/v1beta/interactions/${job.id}`, { + headers: { "x-goog-api-key": apiKey }, + }); + const interaction = (await response.json()) as InteractionResponse; + const status = STATUS[interaction.status ?? ""] ?? "processing"; + + if (status === "failed") { + const error = interaction.error; return { - id, - status: "completed", - videoUrl: `${API_ROOT}/download/v1beta/files/${fileId}:download?alt=media`, + status, + error: classifyProviderError({ + provider: this.id, + code: error?.status ?? (error?.code !== undefined ? String(error.code) : undefined), + message: error?.message ?? "Omni generation failed", + }).toJSON(), }; - } catch (err) { + } + if (status !== "completed") return { status }; + + const video = findOmniVideo(interaction); + const fileId = video?.uri ? fileIdFromUri(video.uri) : undefined; + if (!fileId) { + // Omni's safety filter can finish an interaction with no video. + return { + status: "failed", + error: new ProviderError({ + kind: "moderation", + provider: this.id, + message: "Omni finished without a video; the prompt or frame may have been filtered.", + }).toJSON(), + }; + } + + // The interaction is done, but the file may still be processing. + const file = await providerRequest(this.id, `${API_ROOT}/v1beta/files/${fileId}`, { + headers: { "x-goog-api-key": apiKey }, + }); + const fileState = ((await file.json()) as { state?: string }).state; + if (fileState === "PROCESSING") return { status: "processing", progress: 95 }; + if (fileState !== "ACTIVE") { return { - id: "", status: "failed", - error: `Gemini Omni error: ${err instanceof Error ? err.message : String(err)}`, + error: new ProviderError({ kind: "provider", provider: this.id, message: `Omni video file ended in state ${fileState}.` }).toJSON(), }; } + return { status: "completed", progress: 100, videoUrl: `${API_ROOT}/download/v1beta/files/${fileId}:download?alt=media` }; + } + + /** Background interactions cancel with `POST /interactions/{id}/cancel`. */ + async cancelVideoJob(job: VideoJob): Promise { + const apiKey = this.requireKey(); + await providerRequest(this.id, `${API_ROOT}/v1beta/interactions/${job.id}/cancel`, { + method: "POST", + headers: { "x-goog-api-key": apiKey }, + }); + } + + /** The Files API download needs the key as a header. */ + async downloadVideo(_job: VideoJob, state: VideoJobState): Promise { + if (!state.videoUrl) { + throw new ProviderError({ kind: "not-found", provider: this.id, message: "Omni job has no video to download." }); + } + const response = await providerRequest(this.id, state.videoUrl, { headers: { "x-goog-api-key": this.requireKey() } }); + return new Uint8Array(await response.arrayBuffer()); } - /** Poll the Files API until the generated video is downloadable. */ - private async waitForFile(fileId: string): Promise { - const deadline = Date.now() + FILE_POLL_TIMEOUT_MS; - for (;;) { - const res = await fetch(`${API_ROOT}/v1beta/files/${fileId}`, { - headers: { "x-goog-api-key": this.apiKey! }, + // ── Older interface, kept until every caller uses the contract ──────── + + /** Submit and wait for the finished video, as the CLI expects today. */ + async generateVideo(prompt: string, options?: GenerateOptions): Promise { + let job: VideoJob | undefined; + try { + const frame = options?.referenceImages?.[0]?.base64 + ? `data:${options.referenceImages[0].mimeType};base64,${options.referenceImages[0].base64}` + : typeof options?.referenceImage === "string" && options.referenceImage + ? options.referenceImage + : undefined; + job = await this.submitVideo({ + prompt, + aspectRatio: options?.aspectRatio, + resolution: options?.resolution, + image: frame, + lastFrame: frame ? options?.lastFrame : undefined, }); - const state = res.ok ? ((await res.json()) as { state?: string }).state : `HTTP ${res.status}`; - if (state !== "PROCESSING") return state ?? "UNKNOWN"; - if (Date.now() > deadline) return "TIMEOUT"; - await sleep(FILE_POLL_INTERVAL_MS); + const state = await waitForVideoJob(this, job, { intervalMs: this.pollingInterval, timeoutMs: 15 * 60_000 }); + return { + id: job.id, + status: state.status, + videoUrl: state.videoUrl, + ...(state.error ? { error: state.error.message } : {}), + }; + } catch (error) { + return { id: job?.id ?? "", status: "failed", error: isProviderError(error) || error instanceof Error ? error.message : String(error) }; + } + } + + private requireKey(): string { + if (!this.apiKey) { + throw new ProviderError({ kind: "auth", provider: this.id, message: "GOOGLE_API_KEY not configured for Gemini Omni" }); } + return this.apiKey; } } diff --git a/packages/ai-providers/src/video/contract.testkit.ts b/packages/ai-providers/src/video/contract.testkit.ts index d63d7352..d2d1eade 100644 --- a/packages/ai-providers/src/video/contract.testkit.ts +++ b/packages/ai-providers/src/video/contract.testkit.ts @@ -22,7 +22,7 @@ export interface VideoContractFixture { request: VideoRequest; /** The catalog model ID a request without `model` runs on. */ defaultModel: string; - /** Script a job that is accepted, runs, and completes with `https://fake.media/out.mp4`. */ + /** Script a job that is accepted, runs, and completes with `videoUrl`. */ scriptSuccess(net: FakeProviderNetwork): void; /** Script a job that is accepted and then fails for this reason. */ scriptJobFailure(net: FakeProviderNetwork, reason: "moderation" | "provider"): void; @@ -32,6 +32,8 @@ export interface VideoContractFixture { cancel: boolean; /** Continuations the provider supports. */ continuations: ReadonlyArray<"extend" | "edit">; + /** The download URL `scriptSuccess` completes with. Default `https://fake.media/out.mp4`. */ + videoUrl?: string; } async function rejection(promise: Promise): Promise { @@ -78,7 +80,7 @@ export function describeVideoContract(name: string, fixture: VideoContractFixtur const state = await waitForVideoJob(generator, job, { intervalMs: 1, onProgress: (s) => seen.push(s.status) }); expect(state.status).toBe("completed"); - expect(state.videoUrl).toBe("https://fake.media/out.mp4"); + expect(state.videoUrl).toBe(fixture.videoUrl ?? "https://fake.media/out.mp4"); expect(state.error).toBeUndefined(); expect(seen.at(-1)).toBe("completed"); expect(Buffer.from(await generator.downloadVideo(job, state))).toEqual(FAKE_MP4);