From d2a12afbc1785cf81ea10b298767cb58e2fb545a Mon Sep 17 00:00:00 2001 From: Kiyeon Jeon Date: Sun, 4 Oct 2026 23:48:59 +0900 Subject: [PATCH 1/2] feat: run CLI video executors and job records on the job contract The CLI's video executors branched on provider names (35 provider calls in ai-video.ts alone). They now open any provider through one registry and speak only the video job contract. - `createVideoGenerator(provider, key)` in ai-providers, plus a thin Veo adapter so Veo needs no branch until it is removed after 2026-10-22. - `VideoGenerator.imageInput` says whether a provider takes image URLs, data URIs, or either; the CLI uploads or inlines local frames to match. - executeVideoGenerate/Status/Cancel/Extend have no provider branches. They return the job handle and the provider error kind. - Job records store the job handle (`providerJob`), so `vibe status job` polls exactly what was submitted, and Seedance and Omni jobs are now refreshable. - Builds record a video job the moment the provider accepts it, then wait inline for every provider. A crash or a wait that runs out leaves a job `vibe status job` can still download; a timeout is "still running", not a failure. The likeness fallback checks the error kind. - `video-extend` works on Kling, Grok, Omni, and Veo by the task ID the generation returned; `video-cancel` works on Runway, Seedance, and Omni. - `providerFailure()` picks the CLI error envelope from the error kind. Fixes: - A Kling image-to-video job started without waiting was polled as text-to-video, so its status never resolved. - `generate_video` without a provider defaulted to Kling, not the configured provider its description promised; its `endImage` was ignored. - Job records and video-extend suggested `vibe generate video-status`, which no longer exists. - A cancelled Seedance job read as failed (fal reports `client_cancelled`). - The unused `_shared/video-providers.ts` retry helpers are removed. Claude-Session: https://claude.ai/code/session_011S1BLTaFctZNQzGERpkCuA --- docs/cli-reference.md | 16 +- .../ai-providers/src/fal/FalProvider.test.ts | 11 + packages/ai-providers/src/fal/FalProvider.ts | 13 +- .../ai-providers/src/gemini/gemini-omni.ts | 2 + packages/ai-providers/src/gemini/index.ts | 1 + .../ai-providers/src/gemini/veo-generator.ts | 71 ++ .../ai-providers/src/grok/GrokProvider.ts | 1 + packages/ai-providers/src/index.ts | 2 + .../ai-providers/src/kling/KlingProvider.ts | 14 +- .../ai-providers/src/runway/RunwayProvider.ts | 1 + packages/ai-providers/src/video/contract.ts | 5 + .../ai-providers/src/video/registry.test.ts | 55 ++ packages/ai-providers/src/video/registry.ts | 44 + packages/ai-providers/src/video/wait.test.ts | 1 + .../envelope-snapshots.test.ts.snap | 31 +- .../_shared/scene-build.network.test.ts | 19 +- .../cli/src/commands/_shared/scene-build.ts | 75 +- .../src/commands/_shared/status-jobs.test.ts | 53 +- .../cli/src/commands/_shared/status-jobs.ts | 43 +- .../cli/src/commands/_shared/video-jobs.ts | 113 +++ .../src/commands/_shared/video-providers.ts | 219 ----- .../cli/src/commands/ai-video.network.test.ts | 124 +++ packages/cli/src/commands/ai-video.ts | 809 ++++-------------- .../cli/src/commands/generate/video-cancel.ts | 101 +-- .../cli/src/commands/generate/video-extend.ts | 367 +++----- packages/cli/src/commands/output.ts | 17 + .../cli/src/testing/fake-provider-network.ts | 11 +- packages/cli/src/tools/manifest/generate.ts | 30 +- 28 files changed, 951 insertions(+), 1298 deletions(-) create mode 100644 packages/ai-providers/src/gemini/veo-generator.ts create mode 100644 packages/ai-providers/src/video/registry.test.ts create mode 100644 packages/ai-providers/src/video/registry.ts create mode 100644 packages/cli/src/commands/_shared/video-jobs.ts delete mode 100644 packages/cli/src/commands/_shared/video-providers.ts create mode 100644 packages/cli/src/commands/ai-video.network.test.ts diff --git a/docs/cli-reference.md b/docs/cli-reference.md index 59c0e17d..b597a7ea 100644 --- a/docs/cli-reference.md +++ b/docs/cli-reference.md @@ -651,7 +651,7 @@ Cost tier: `very-high` #### `vibe generate video-cancel` -Cancel video generation (Grok or Runway) +Cancel video generation (Runway, Seedance, Gemini Omni) Product surface: `advanced` Note: Provider lifecycle control. @@ -661,12 +661,12 @@ Cost tier: `free` **Parameters:** - `task-id` _(string)_ **required** - Task ID to cancel -- `provider` _(string)_ _(grok \| runway)_ _(default: `"grok"`)_ - Provider: grok, runway -- `apiKey` _(string)_ - API key (or set XAI_API_KEY / RUNWAY_API_SECRET env) +- `provider` _(string)_ _(runway \| seedance \| omni)_ _(default: `"runway"`)_ - Provider: runway, seedance, omni +- `apiKey` _(string)_ - API key for the provider #### `vibe generate video-extend` -Extend video duration (Kling by video ID, Veo by operation name) +Extend a generated video from its last frame (Kling, Grok, Gemini Omni, Veo) Product surface: `advanced` Note: Provider lifecycle control. @@ -675,12 +675,12 @@ Cost tier: `very-high` **Parameters:** -- `id` _(string)_ **required** - Kling video ID or Veo operation name -- `provider` _(string)_ _(kling \| veo)_ _(default: `"kling"`)_ - Provider: kling, veo -- `apiKey` _(string)_ - API key (KLING_API_KEY or GOOGLE_API_KEY) +- `task-id` _(string)_ **required** - Task ID the generation returned (Veo: its operation name) +- `provider` _(string)_ _(kling \| grok \| omni \| veo)_ _(default: `"kling"`)_ - Provider: kling, grok, omni, veo +- `apiKey` _(string)_ - API key for the provider - `output` _(string)_ - Output file path - `prompt` _(string)_ - Continuation prompt -- `duration` _(number)_ _(default: `5`)_ - Duration: 5 or 10 (Kling), 4/6/8 (Veo) +- `duration` _(number)_ _(default: `5`)_ - Seconds to add: 5 or 10 (Kling), 2-10 (Grok), 4/6/8 (Veo); Omni picks its own - `negative` _(string)_ - Negative prompt (what to avoid, Kling only) - `veoModel` _(string)_ _(default: `"3.1"`)_ - Veo model: 3.1, 3.1-fast - `noWait` _(boolean)_ - Start extension and return task ID without waiting diff --git a/packages/ai-providers/src/fal/FalProvider.test.ts b/packages/ai-providers/src/fal/FalProvider.test.ts index b6e782b6..b4885859 100644 --- a/packages/ai-providers/src/fal/FalProvider.test.ts +++ b/packages/ai-providers/src/fal/FalProvider.test.ts @@ -157,6 +157,17 @@ describe("FalProvider (Seedance) requests", () => { expect(state).toMatchObject({ status: "failed", error: { kind: "likeness", status: 422, code: "content_policy_violation" } }); }); + it("reads a cancelled job as cancelled, not failed", async () => { + net.on("GET", QUEUE, /\/requests\/req-1\/status$/, () => + jsonResponse({ status: "COMPLETED", request_id: "req-1", error: "Request cancelled by the client.", error_type: "client_cancelled" }) + ); + const fal = await create(); + const job = await fal.submitVideo({ prompt: "p" }); + await fal.cancelVideoJob(job); + + expect(await fal.getVideoJob(job)).toEqual({ status: "cancelled" }); + }); + it("keeps the older blocking generateVideo working on top of the queue", async () => { const fal = await create(); (fal as unknown as { pollingInterval: number }).pollingInterval = 1; diff --git a/packages/ai-providers/src/fal/FalProvider.ts b/packages/ai-providers/src/fal/FalProvider.ts index a7aaa9e6..52caad76 100644 --- a/packages/ai-providers/src/fal/FalProvider.ts +++ b/packages/ai-providers/src/fal/FalProvider.ts @@ -160,6 +160,8 @@ export class FalProvider implements AIProvider, VideoGenerator { capabilities: AICapability[] = ["text-to-video", "image-to-video", "reference-to-video"]; iconUrl = "/icons/fal.svg"; isAvailable = true; + /** Data URIs are uploaded to fal storage on submit. */ + readonly imageInput = "either" as const; private client?: FalClient; private apiKey?: string; @@ -241,6 +243,8 @@ export class FalProvider implements AIProvider, VideoGenerator { // fal: a COMPLETED status can still be a failure; check `error` first. const failure = status as { error?: string; error_type?: string }; + // A cancelled job also ends COMPLETED, with `error_type: "client_cancelled"` (probe). + if (failure.error_type === "client_cancelled") return { status: "cancelled" }; if (failure.error) { return { status: "failed", error: classifyProviderError({ provider: this.id, message: failure.error, code: failure.error_type }).toJSON() }; } @@ -322,10 +326,15 @@ export class FalProvider implements AIProvider, VideoGenerator { return this.client; } + /** + * The endpoint a job was submitted to. fal's status, result, and cancel + * URLs are app-level (`bytedance/seedance-2.0/requests/{id}`), so a bare + * request ID can fall back to its model's text-to-video endpoint. + */ private endpointOf(job: VideoJob): string { - const endpoint = job.meta?.endpoint; + const endpoint = job.meta?.endpoint ?? ENDPOINT_TEXT_TO_VIDEO[job.model as SeedanceVariant]; if (!endpoint) { - throw new ProviderError({ kind: "invalid-request", provider: this.id, message: `Seedance job ${job.id} has no endpoint in its handle.` }); + throw new ProviderError({ kind: "invalid-request", provider: this.id, message: `Seedance job ${job.id} names no known model.` }); } return endpoint; } diff --git a/packages/ai-providers/src/gemini/gemini-omni.ts b/packages/ai-providers/src/gemini/gemini-omni.ts index 7ddb6a10..196dcc15 100644 --- a/packages/ai-providers/src/gemini/gemini-omni.ts +++ b/packages/ai-providers/src/gemini/gemini-omni.ts @@ -85,6 +85,8 @@ function inlineImage(ref: string): { type: "image"; data: string; mime_type: str export class OmniProvider implements VideoGenerator { id = "omni"; + /** Frames go inline in the request. */ + readonly imageInput = "data-uri" as const; label = "Gemini Omni 1.1 Flash"; private apiKey?: string; private pollingInterval = 5000; diff --git a/packages/ai-providers/src/gemini/index.ts b/packages/ai-providers/src/gemini/index.ts index f597e59d..2be157f2 100644 --- a/packages/ai-providers/src/gemini/index.ts +++ b/packages/ai-providers/src/gemini/index.ts @@ -42,3 +42,4 @@ defineProvider({ resolverPriority: { video: 3 }, commandsUnlocked: ["generate video -p omni"], }); +export * from "./veo-generator.js"; diff --git a/packages/ai-providers/src/gemini/veo-generator.ts b/packages/ai-providers/src/gemini/veo-generator.ts new file mode 100644 index 00000000..4b3061cd --- /dev/null +++ b/packages/ai-providers/src/gemini/veo-generator.ts @@ -0,0 +1,71 @@ +/** + * Veo 3.1 on the video job contract, as a thin wrapper over + * `GeminiProvider`'s Veo methods. Every Veo 3.1 model shuts down on + * 2026-10-22, so this exists only so callers have no Veo-specific branch + * until then; it is deleted together with Veo. + */ + +import type { VideoGenerator, VideoJob, VideoJobState, VideoRequest } from "../video/contract.js"; +import { ProviderError, classifyProviderError } from "../shared/errors.js"; +import { providerRequest } from "../shared/http.js"; +import { resolveVideoModel } from "../video/models.js"; +import { GeminiProvider, type VeoModel } from "./GeminiProvider.js"; + +export class VeoGenerator implements VideoGenerator { + readonly id = "veo"; + readonly imageInput = "data-uri" as const; + private gemini = new GeminiProvider(); + private apiKey?: string; + + async initialize(config: { apiKey?: string }): Promise { + this.apiKey = config.apiKey; + await this.gemini.initialize({ apiKey: config.apiKey }); + } + + async submitVideo(request: VideoRequest): Promise { + if (request.from?.kind === "edit") { + throw new ProviderError({ kind: "unsupported", provider: this.id, message: "Veo cannot edit an earlier video; it can extend one." }); + } + // Extension defaults to Veo 3.1 (not Fast), as the extend command always has. + const model = resolveVideoModel(this.id, request.model ?? (request.from ? "3.1" : undefined)).id as VeoModel; + const result = request.from + ? await this.gemini.extendVideo(request.from.job.id, request.prompt, { + duration: (request.durationSec ?? 8) as 4 | 6 | 8, + model, + }) + : await this.gemini.generateVideo(request.prompt, { + prompt: request.prompt, + model, + duration: request.durationSec, + aspectRatio: request.aspectRatio, + referenceImage: request.image, + lastFrame: request.lastFrame, + negativePrompt: request.negativePrompt, + resolution: request.resolution, + }); + if (result.status === "failed" || !result.id) { + throw classifyProviderError({ provider: this.id, message: result.error ?? "Veo generation failed" }); + } + return { provider: this.id, id: result.id, model, submittedAt: new Date().toISOString() }; + } + + async getVideoJob(job: VideoJob): Promise { + const result = await this.gemini.getGenerationStatus(job.id); + return { + status: result.status, + progress: result.progress, + videoUrl: result.videoUrl, + ...(result.status === "failed" + ? { error: classifyProviderError({ provider: this.id, message: result.error ?? "Veo generation failed" }).toJSON() } + : {}), + }; + } + + async downloadVideo(_job: VideoJob, state: VideoJobState): Promise { + if (!state.videoUrl || !this.apiKey) { + throw new ProviderError({ kind: "not-found", provider: this.id, message: "Veo job has no video to download." }); + } + const response = await providerRequest(this.id, state.videoUrl, { headers: { "x-goog-api-key": this.apiKey } }); + return new Uint8Array(await response.arrayBuffer()); + } +} diff --git a/packages/ai-providers/src/grok/GrokProvider.ts b/packages/ai-providers/src/grok/GrokProvider.ts index 70481dce..65f1bb37 100644 --- a/packages/ai-providers/src/grok/GrokProvider.ts +++ b/packages/ai-providers/src/grok/GrokProvider.ts @@ -153,6 +153,7 @@ interface GrokStatusResponse { */ export class GrokProvider implements AIProvider, VideoGenerator { id = "grok"; + readonly imageInput = "either" as const; name = "xAI Grok Imagine"; description = "AI video generation with Grok Imagine (native audio, 1-15 sec)"; capabilities: AICapability[] = ["text-to-video", "image-to-video", "text-to-image", "image-editing"]; diff --git a/packages/ai-providers/src/index.ts b/packages/ai-providers/src/index.ts index f33c93d7..282906e1 100644 --- a/packages/ai-providers/src/index.ts +++ b/packages/ai-providers/src/index.ts @@ -47,6 +47,7 @@ export { geminiProvider, OmniProvider, omniProvider, + VeoGenerator, isGeminiTextModelAlias, resolveGeminiTextModel, resolveVeoModel, @@ -73,6 +74,7 @@ export { type ModelStatus, } from "./catalog/index.js"; export * from "./video/index.js"; +export { VIDEO_GENERATOR_PROVIDERS, createVideoGenerator, type VideoGeneratorProvider } from "./video/registry.js"; export { OpenAIProvider, openaiProvider } from "./openai/index.js"; export { ClaudeProvider, claudeProvider } from "./claude/index.js"; export type { MotionOptions, MotionResult, RemotionComponent, StoryboardSegment } from "./claude/index.js"; diff --git a/packages/ai-providers/src/kling/KlingProvider.ts b/packages/ai-providers/src/kling/KlingProvider.ts index 292ff27f..7a24fda3 100644 --- a/packages/ai-providers/src/kling/KlingProvider.ts +++ b/packages/ai-providers/src/kling/KlingProvider.ts @@ -165,6 +165,8 @@ export class KlingProvider implements AIProvider, VideoGenerator { capabilities: AICapability[] = ["text-to-video", "image-to-video", "video-extend"]; iconUrl = "/icons/kling.svg"; isAvailable = true; + /** Kling's legacy endpoints take image URLs only. */ + readonly imageInput = "url" as const; private apiKey?: string; private accessKey?: string; @@ -224,7 +226,17 @@ export class KlingProvider implements AIProvider, VideoGenerator { } async getVideoJob(job: VideoJob): Promise { - const type = (job.meta?.type as KlingTaskType | undefined) ?? "text2video"; + const type = job.meta?.type as KlingTaskType | undefined; + if (!type) { + // A bare task ID (typed in by a user) does not say which endpoint + // created it; Kling answers "not found" on the wrong one. + try { + return await this.getVideoJob({ ...job, meta: { ...job.meta, type: "text2video" } }); + } catch (error) { + if (!(isProviderError(error) && error.kind === "not-found")) throw error; + return this.getVideoJob({ ...job, meta: { ...job.meta, type: "image2video" } }); + } + } const response = await providerRequest(this.id, `${this.baseUrl}/videos/${type}/${job.id}`, { headers: this.headers(this.token()), }, this.requestOptions()); diff --git a/packages/ai-providers/src/runway/RunwayProvider.ts b/packages/ai-providers/src/runway/RunwayProvider.ts index da56dbca..3916a224 100644 --- a/packages/ai-providers/src/runway/RunwayProvider.ts +++ b/packages/ai-providers/src/runway/RunwayProvider.ts @@ -84,6 +84,7 @@ export class RunwayProvider implements AIProvider, VideoGenerator { capabilities: AICapability[] = ["text-to-video", "image-to-video"]; iconUrl = "/icons/runway.svg"; isAvailable = true; + readonly imageInput = "either" as const; private static readonly API_VERSION = "2024-11-06"; private apiKey?: string; diff --git a/packages/ai-providers/src/video/contract.ts b/packages/ai-providers/src/video/contract.ts index bcfad49b..99d635bb 100644 --- a/packages/ai-providers/src/video/contract.ts +++ b/packages/ai-providers/src/video/contract.ts @@ -67,9 +67,14 @@ export interface VideoJobState { error?: ProviderErrorInfo; } +/** How a provider takes an input image: a public URL, an inline data URI, or either. */ +export type ImageInput = "url" | "data-uri" | "either"; + export interface VideoGenerator { /** Provider registry id; matches `VideoJob.provider`. */ readonly id: string; + /** How `VideoRequest.image` must be given. Callers upload or inline local files to match. */ + readonly imageInput: ImageInput; /** Start a generation. Throws `ProviderError` when the provider refuses it. */ submitVideo(request: VideoRequest): Promise; /** Read a job's current state. Throws `ProviderError` only when the state cannot be read. */ diff --git a/packages/ai-providers/src/video/registry.test.ts b/packages/ai-providers/src/video/registry.test.ts new file mode 100644 index 00000000..6391ee26 --- /dev/null +++ b/packages/ai-providers/src/video/registry.test.ts @@ -0,0 +1,55 @@ +import { afterEach, beforeEach, describe, expect, it } from "vitest"; + +import { FakeProviderNetwork, jsonResponse } from "../testing/fake-provider-network.js"; +import { VIDEO_GENERATOR_PROVIDERS, createVideoGenerator } from "./registry.js"; + +describe("createVideoGenerator", () => { + it("returns an initialized generator for every video provider", async () => { + for (const provider of VIDEO_GENERATOR_PROVIDERS) { + const generator = await createVideoGenerator(provider, "test-key"); + expect(generator.id).toBe(provider); + expect(["url", "data-uri", "either"]).toContain(generator.imageInput); + } + }); + + it("accepts fal as the old name for seedance and rejects unknown providers", async () => { + expect((await createVideoGenerator("fal", "k")).id).toBe("seedance"); + await expect(createVideoGenerator("sora", "k")).rejects.toMatchObject({ kind: "invalid-request" }); + }); +}); + +describe("job lookups from a bare task ID", () => { + let net: FakeProviderNetwork; + + beforeEach(() => { + net = new FakeProviderNetwork(); + net.install(); + }); + + afterEach(() => net.uninstall()); + + it("Kling tries image-to-video when text-to-video does not know the task", async () => { + net + .on("GET", "api.klingai.com", /\/text2video\/t-9$/, () => jsonResponse({ code: 1203, message: "task not found" }, 404)) + .on("GET", "api.klingai.com", /\/image2video\/t-9$/, () => + jsonResponse({ code: 0, message: "ok", request_id: "r", data: { task_id: "t-9", task_status: "processing" } }) + ); + const kling = await createVideoGenerator("kling", "k"); + + const state = await kling.getVideoJob({ provider: "kling", id: "t-9", model: "kling-v3", submittedAt: "" }); + + expect(state.status).toBe("processing"); + expect(net.requests.map((r) => r.path)).toEqual(["/v1/videos/text2video/t-9", "/v1/videos/image2video/t-9"]); + }); + + it("Seedance polls a bare request ID on its model's app", async () => { + net.on("GET", "queue.fal.run", /\/bytedance\/seedance-2.0\/requests\/r-9\/status$/, () => + jsonResponse({ status: "IN_PROGRESS", request_id: "r-9" }) + ); + const seedance = await createVideoGenerator("seedance", "k"); + + const state = await seedance.getVideoJob({ provider: "seedance", id: "r-9", model: "seedance-2.0", submittedAt: "" }); + + expect(state.status).toBe("processing"); + }); +}); diff --git a/packages/ai-providers/src/video/registry.ts b/packages/ai-providers/src/video/registry.ts new file mode 100644 index 00000000..5d7d6701 --- /dev/null +++ b/packages/ai-providers/src/video/registry.ts @@ -0,0 +1,44 @@ +/** + * One way to get a video provider by its registry id, so callers never + * name a provider class or branch on provider names. + */ + +import { FalProvider } from "../fal/FalProvider.js"; +import { OmniProvider } from "../gemini/gemini-omni.js"; +import { VeoGenerator } from "../gemini/veo-generator.js"; +import { GrokProvider } from "../grok/GrokProvider.js"; +import { KlingProvider } from "../kling/KlingProvider.js"; +import { RunwayProvider } from "../runway/RunwayProvider.js"; +import { ProviderError } from "../shared/errors.js"; +import type { VideoGenerator } from "./contract.js"; + +export const VIDEO_GENERATOR_PROVIDERS = ["seedance", "grok", "kling", "runway", "omni", "veo"] as const; +export type VideoGeneratorProvider = (typeof VIDEO_GENERATOR_PROVIDERS)[number]; + +/** A ready video generator for `provider` (`fal` is the old name for `seedance`). */ +export async function createVideoGenerator(provider: string, apiKey: string): Promise { + const id = provider === "fal" ? "seedance" : provider; + const generator = + id === "seedance" + ? new FalProvider() + : id === "grok" + ? new GrokProvider() + : id === "kling" + ? new KlingProvider() + : id === "runway" + ? new RunwayProvider() + : id === "omni" + ? new OmniProvider() + : id === "veo" + ? new VeoGenerator() + : undefined; + if (!generator) { + throw new ProviderError({ + kind: "invalid-request", + provider: id, + message: `Unknown video provider "${provider}". Available: ${VIDEO_GENERATOR_PROVIDERS.join(", ")}.`, + }); + } + await generator.initialize({ apiKey }); + return generator; +} diff --git a/packages/ai-providers/src/video/wait.test.ts b/packages/ai-providers/src/video/wait.test.ts index a9b3853f..db9b76b3 100644 --- a/packages/ai-providers/src/video/wait.test.ts +++ b/packages/ai-providers/src/video/wait.test.ts @@ -9,6 +9,7 @@ const job: VideoJob = { provider: "test", id: "j1", model: "m", submittedAt: "20 function generator(states: Array): VideoGenerator { return { id: "test", + imageInput: "either", submitVideo: vi.fn(), downloadVideo: vi.fn(), getVideoJob: vi.fn(async () => { diff --git a/packages/cli/src/commands/__snapshots__/envelope-snapshots.test.ts.snap b/packages/cli/src/commands/__snapshots__/envelope-snapshots.test.ts.snap index 4acdf25d..b5cd6937 100644 --- a/packages/cli/src/commands/__snapshots__/envelope-snapshots.test.ts.snap +++ b/packages/cli/src/commands/__snapshots__/envelope-snapshots.test.ts.snap @@ -2225,21 +2225,22 @@ exports[`CLI --describe schemas (drift detection) > vibe generate video --descri exports[`CLI --describe schemas (drift detection) > vibe generate video-cancel --describe 1`] = ` { "cost": "free", - "description": "Cancel video generation (Grok or Runway)", + "description": "Cancel video generation (Runway, Seedance, Gemini Omni)", "name": "generate.video-cancel", "note": "Provider lifecycle control.", "parameters": { "properties": { "apiKey": { - "description": "API key (or set XAI_API_KEY / RUNWAY_API_SECRET env)", + "description": "API key for the provider", "type": "string", }, "provider": { - "default": "grok", - "description": "Provider: grok, runway", + "default": "runway", + "description": "Provider: runway, seedance, omni", "enum": [ - "grok", "runway", + "seedance", + "omni", ], "type": "string", }, @@ -2260,13 +2261,13 @@ exports[`CLI --describe schemas (drift detection) > vibe generate video-cancel - exports[`CLI --describe schemas (drift detection) > vibe generate video-extend --describe 1`] = ` { "cost": "very-high", - "description": "Extend video duration (Kling by video ID, Veo by operation name)", + "description": "Extend a generated video from its last frame (Kling, Grok, Gemini Omni, Veo)", "name": "generate.video-extend", "note": "Provider lifecycle control.", "parameters": { "properties": { "apiKey": { - "description": "API key (KLING_API_KEY or GOOGLE_API_KEY)", + "description": "API key for the provider", "type": "string", }, "dryRun": { @@ -2275,13 +2276,9 @@ exports[`CLI --describe schemas (drift detection) > vibe generate video-extend - }, "duration": { "default": 5, - "description": "Duration: 5 or 10 (Kling), 4/6/8 (Veo)", + "description": "Seconds to add: 5 or 10 (Kling), 2-10 (Grok), 4/6/8 (Veo); Omni picks its own", "type": "number", }, - "id": { - "description": "Kling video ID or Veo operation name", - "type": "string", - }, "negative": { "description": "Negative prompt (what to avoid, Kling only)", "type": "string", @@ -2300,13 +2297,19 @@ exports[`CLI --describe schemas (drift detection) > vibe generate video-extend - }, "provider": { "default": "kling", - "description": "Provider: kling, veo", + "description": "Provider: kling, grok, omni, veo", "enum": [ "kling", + "grok", + "omni", "veo", ], "type": "string", }, + "task-id": { + "description": "Task ID the generation returned (Veo: its operation name)", + "type": "string", + }, "veoModel": { "default": "3.1", "description": "Veo model: 3.1, 3.1-fast", @@ -2314,7 +2317,7 @@ exports[`CLI --describe schemas (drift detection) > vibe generate video-extend - }, }, "required": [ - "id", + "task-id", ], "type": "object", }, diff --git a/packages/cli/src/commands/_shared/scene-build.network.test.ts b/packages/cli/src/commands/_shared/scene-build.network.test.ts index 51543c85..d99f43dd 100644 --- a/packages/cli/src/commands/_shared/scene-build.network.test.ts +++ b/packages/cli/src/commands/_shared/scene-build.network.test.ts @@ -6,7 +6,7 @@ * report all run. Only local tools (ffprobe) are stubbed. */ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; @@ -108,6 +108,23 @@ music: "Soft piano"`) expect(existsSync(join(projectDir, "assets/video-pond.mp4"))).toBe(true); }); + it("records the video job with its handle before waiting, then marks it completed", async () => { + writeFileSync(join(projectDir, "STORYBOARD.md"), storyboard(`video: "Slow push-in on the boat"`)); + + await executeSceneBuild({ projectDir, stage: "assets" }); + + const jobsDir = join(projectDir, ".vibeframe", "jobs"); + const records = readdirSync(jobsDir).map((f) => JSON.parse(readFileSync(join(jobsDir, f), "utf-8"))); + expect(records).toHaveLength(1); + expect(records[0]).toMatchObject({ + jobType: "generate-video", + provider: "seedance", + status: "completed", + beatId: "pond", + providerJob: { provider: "seedance", model: "seedance-2.0", meta: { endpoint: "bytedance/seedance-2.0/text-to-video" } }, + }); + }); + it("falls back to Runway when Seedance rejects a keyframe for a real person's likeness", async () => { net.on("POST", "queue.fal.run", /image-to-video$/, () => jsonResponse( diff --git a/packages/cli/src/commands/_shared/scene-build.ts b/packages/cli/src/commands/_shared/scene-build.ts index eebefb58..7a4135ed 100644 --- a/packages/cli/src/commands/_shared/scene-build.ts +++ b/packages/cli/src/commands/_shared/scene-build.ts @@ -33,6 +33,7 @@ import { type GPTImageModel, type GeminiImageModel, type ImageOptions, + type VideoJob, } from "@vibeframe/ai-providers"; import { getAudioDuration } from "../../utils/audio.js"; @@ -102,7 +103,7 @@ import { import { augmentBackdropPrompt } from "./build-backdrop-prompt.js"; import { executeVideoGenerate } from "../ai-video.js"; import { executeMusic } from "../generate/music.js"; -import { createAndWriteJobRecord, type JobRecord } from "./status-jobs.js"; +import { createAndWriteJobRecord, writeJobRecord, type JobRecord } from "./status-jobs.js"; import { executeSceneRepair, type SceneRepairResult } from "./scene-repair.js"; import { resolveTtsProvider, TtsKeyMissingError, type TtsProviderName } from "./tts-resolve.js"; import { resolveSceneBuildMode, type SceneBuildMode } from "./scene-build-mode.js"; @@ -2313,6 +2314,37 @@ async function dispatchVideo( const keyframeImageAbs = keyframeImageRel ? join(ctx.projectDir, keyframeImageRel) : undefined; loadSceneBuildEnv(ctx.projectDir); + // The job is recorded the moment the provider accepts it, then waited on + // inline. A crash or a wait that runs out leaves a record `vibe status job` + // can still poll, download, and cache. + let job: JobRecord | undefined; + const recordJob = async (provider: BuildVideoProvider, submitted: VideoJob) => { + job = await createAndWriteJobRecord({ + jobType: "generate-video", + provider, + providerTaskId: submitted.id, + providerJob: submitted, + status: "running", + projectDir: ctx.projectDir, + workingDirectory: ctx.projectDir, + command: "build --stage assets", + prompt, + beatId: beat.id, + outputPath: abs, + cachePath: cacheAbs, + assetKind: "video", + assetCue: prompt, + assetOptions: { duration: normalizeVideoDuration(beat.duration), ratio: "16:9" }, + cacheKey: cache.key, + canonicalPath: rel, + metadataPath, + }); + }; + const settleJob = async (status: "completed" | "failed", error?: string) => { + if (!job) return; + job = { ...job, status, updatedAt: new Date().toISOString(), ...(error ? { error } : {}) }; + await writeJobRecord(job); + }; const generateWith = (provider: BuildVideoProvider, apiKey: string | undefined) => executeVideoGenerate({ prompt, @@ -2320,7 +2352,8 @@ async function dispatchVideo( duration: normalizeVideoDuration(beat.duration), ratio: "16:9", output: abs, - wait: false, + wait: true, + onSubmitted: (submitted) => recordJob(provider, submitted), // Keyframe mode → single init frame (image-to-video); otherwise character // reference-to-video. image: keyframeImageAbs, @@ -2342,11 +2375,13 @@ async function dispatchVideo( ); if (!result.success || !result.taskId) { const error = result.error ?? "video generation did not return a task id"; + await settleJob("failed", error); // ByteDance rejects face-visible i2v inputs with a deterministic likeness // 422 — retrying the same provider is useless, but Runway accepts the same // keyframe. Fall back automatically when a Runway key is configured. + const likeness = result.errorKind === "likeness" || isLikenessRejection(error); const runwayKey = - isLikenessRejection(error) && activeProvider !== "runway" + likeness && activeProvider !== "runway" ? await apiKeyForVideoProvider("runway", ctx.projectDir) : undefined; if (!runwayKey) { @@ -2364,12 +2399,14 @@ async function dispatchVideo( result = await generateWith(activeProvider, runwayKey); if (!result.success || !result.taskId) { const fallbackError = result.error ?? "video generation did not return a task id"; + await settleJob("failed", fallbackError); ctx.onProgress({ type: "video-failed", beatId: beat.id, error: fallbackError }); return { status: "failed", error: fallbackError }; } } if (result.status === "completed" && existsSync(abs)) { + await settleJob("completed"); await mkdir(dirname(cacheAbs), { recursive: true }); await copyFile(abs, cacheAbs); await writeAssetMetadata({ @@ -2400,33 +2437,17 @@ async function dispatchVideo( }; } - const job = await createAndWriteJobRecord({ - jobType: "generate-video", - provider: result.provider ?? activeProvider, - providerTaskId: result.taskId, - providerTaskType: keyframeImageAbs ? "image2video" : "text2video", - status: "running", - projectDir: ctx.projectDir, - workingDirectory: ctx.projectDir, - command: "build --stage assets", - prompt, - resultUrl: result.videoUrl, - beatId: beat.id, - outputPath: abs, - cachePath: cacheAbs, - assetKind: "video", - assetCue: prompt, - assetOptions: { duration: normalizeVideoDuration(beat.duration), ratio: "16:9" }, - cacheKey: cache.key, - canonicalPath: rel, - metadataPath, - }); - ctx.onProgress({ type: "video-pending", beatId: beat.id, jobId: job.id, provider: job.provider }); + // Still running after the wait: hand back the recorded job to poll later. + if (!job) { + await recordJob(activeProvider, result.job ?? { provider: activeProvider, id: result.taskId, model: "", submittedAt: "" }); + } + const pendingJob = job!; + ctx.onProgress({ type: "video-pending", beatId: beat.id, jobId: pendingJob.id, provider: pendingJob.provider }); return { status: "pending", path: rel, - job, - provider: job.provider, + job: pendingJob, + provider: pendingJob.provider, cachePath: cache.path, cacheKey: cache.key, metadataPath, diff --git a/packages/cli/src/commands/_shared/status-jobs.test.ts b/packages/cli/src/commands/_shared/status-jobs.test.ts index 36bf94b8..f013c270 100644 --- a/packages/cli/src/commands/_shared/status-jobs.test.ts +++ b/packages/cli/src/commands/_shared/status-jobs.test.ts @@ -49,8 +49,8 @@ describe("status job records", () => { }); expect(record.id).toBe("job_test"); - expect(record.retryWith).toContain(`vibe status job job_test --project ${resolve(dir)} --json`); - expect(record.retryWith).toContain("vibe generate video-status task_123 -p runway --json"); + // `vibe status job` is the one way to poll; the old `generate video-status` is gone. + expect(record.retryWith).toEqual([`vibe status job job_test --project ${resolve(dir)} --json`]); const raw = await readFile(join(dir, ".vibeframe", "jobs", "job_test.json"), "utf-8"); expect(JSON.parse(raw)).toMatchObject({ id: "job_test", providerTaskId: "task_123" }); @@ -59,31 +59,49 @@ describe("status job records", () => { expect(read?.provider).toBe("runway"); }); - it("does not suggest legacy video-status for video providers without live status support", async () => { + it("reports live status as unsupported for a provider outside the video contract", async () => { const dir = await tempProject(); const record = await createAndWriteJobRecord({ - id: "job_seedance", + id: "job_sora", now: new Date("2026-01-01T00:00:00.000Z"), jobType: "generate-video", - provider: "seedance", - providerTaskId: "task_seedance", + provider: "sora", + providerTaskId: "task_sora", projectDir: dir, command: "generate video --no-wait", }); - expect(record.retryWith).toEqual([ - `vibe status job job_seedance --project ${resolve(dir)} --json`, - ]); - const status = await refreshJobRecord(record, { write: false }); expect(status.refreshed).toBe(false); expect(status.live.supported).toBe(false); - expect(status.warnings).toContain( - "Live status is not supported for seedance generate-video jobs yet." - ); - expect(status.retryWith).toEqual([ - `vibe status job job_seedance --project ${resolve(dir)} --json`, - ]); + expect(status.warnings).toContain("Live status is not supported for sora generate-video jobs yet."); + expect(status.retryWith).toEqual([`vibe status job job_sora --project ${resolve(dir)} --json`]); + }); + + it("refreshes Seedance jobs with the stored job handle", async () => { + const dir = await tempProject(); + const providerJob = { + provider: "seedance", + id: "req_1", + model: "seedance-2.0", + submittedAt: "2026-10-04T00:00:00.000Z", + meta: { endpoint: "bytedance/seedance-2.0/image-to-video" }, + }; + const record = await createAndWriteJobRecord({ + id: "job_seedance", + jobType: "generate-video", + provider: "seedance", + providerTaskId: "req_1", + providerJob, + projectDir: dir, + command: "build --stage assets", + }); + vi.mocked(executeVideoStatus).mockResolvedValueOnce({ success: true, taskId: "req_1", status: "processing" }); + + const result = await refreshJobRecord(record, { write: false }); + + expect(result.live.supported).toBe(true); + expect(executeVideoStatus).toHaveBeenCalledWith(expect.objectContaining({ job: providerJob })); }); it("summarizes build, review, and job state for a project", async () => { @@ -377,7 +395,6 @@ describe("status job records", () => { projectDir: dir, command: "generate video --no-wait", }); - expect(record.retryWith).toContain("vibe generate video-status grok_1 -p grok --json"); vi.mocked(executeVideoStatus).mockResolvedValueOnce({ success: true, @@ -396,6 +413,7 @@ describe("status job records", () => { expect(executeVideoStatus).toHaveBeenCalledWith({ taskId: "grok_1", provider: "grok", + job: undefined, taskType: undefined, wait: undefined, output: undefined, @@ -412,7 +430,6 @@ describe("status job records", () => { projectDir: dir, command: "generate video --no-wait", }); - expect(record.retryWith).toContain("vibe generate video-status operations/veo_1 -p veo --json"); vi.mocked(executeVideoStatus).mockResolvedValueOnce({ success: true, diff --git a/packages/cli/src/commands/_shared/status-jobs.ts b/packages/cli/src/commands/_shared/status-jobs.ts index d1a0f8f8..979e00c8 100644 --- a/packages/cli/src/commands/_shared/status-jobs.ts +++ b/packages/cli/src/commands/_shared/status-jobs.ts @@ -3,6 +3,7 @@ import { existsSync, statSync } from "node:fs"; import { copyFile, mkdir, readFile, readdir, rename, stat, writeFile } from "node:fs/promises"; import { dirname, join, parse, relative, resolve } from "node:path"; +import { VIDEO_GENERATOR_PROVIDERS, type VideoJob } from "@vibeframe/ai-providers"; import { executeVideoStatus } from "../ai-video.js"; import { executeMusicStatus } from "../generate/music-status.js"; import type { BuildAssetKind } from "./build-cache.js"; @@ -56,7 +57,10 @@ export interface JobRecord { command: string; provider: string; providerTaskId: string; + /** Kling task type; superseded by `providerJob` and kept for older records. */ providerTaskType?: "text2video" | "image2video"; + /** The provider's job handle, so a refresh polls exactly the job that was submitted. */ + providerJob?: VideoJob; progress?: number; resultUrl?: string; beatId?: string; @@ -91,6 +95,7 @@ export interface CreateJobRecordOptions { provider: string; providerTaskId: string; providerTaskType?: "text2video" | "image2video"; + providerJob?: VideoJob; projectDir?: string; workingDirectory?: string; command: string; @@ -300,6 +305,7 @@ export function createJobRecord(opts: CreateJobRecordOptions): JobRecord { provider: opts.provider, providerTaskId: opts.providerTaskId, providerTaskType: opts.providerTaskType, + providerJob: opts.providerJob, progress: opts.progress, resultUrl: opts.resultUrl, beatId: opts.beatId, @@ -396,8 +402,9 @@ export async function refreshJobRecord( if (output) await mkdir(dirname(output), { recursive: true }); const result = await executeVideoStatus({ taskId: record.providerTaskId, - provider: record.provider as "grok" | "runway" | "kling" | "veo", - taskType: record.providerTaskType as "text2video" | "image2video" | undefined, + provider: record.provider, + job: record.providerJob, + taskType: record.providerTaskType, wait: opts.wait, output, }); @@ -586,10 +593,7 @@ export function summarizeJob(record: JobRecord): JobSummary { } export function retryWithForJob(record: JobRecord): string[] { - return unique([ - `vibe status job ${record.id} --project ${record.projectDir} --json`, - providerStatusCommand(record), - ]); + return [`vibe status job ${record.id} --project ${record.projectDir} --json`]; } export function makeJobStatusResult( @@ -740,10 +744,7 @@ function resolveRefreshOutput(record: JobRecord, opts: RefreshJobOptions): strin function liveSupport(record: JobRecord): { supported: boolean; error?: string } { if ( record.jobType === "generate-video" && - (record.provider === "grok" || - record.provider === "runway" || - record.provider === "kling" || - record.provider === "veo") + (VIDEO_GENERATOR_PROVIDERS as readonly string[]).includes(record.provider === "fal" ? "seedance" : record.provider) ) { return { supported: true }; } @@ -1329,28 +1330,6 @@ function isActiveStatus(status: JobStatus): boolean { return status === "queued" || status === "running"; } -function providerStatusCommand(record: JobRecord): string | undefined { - if (record.jobType === "generate-video") { - if ( - record.provider !== "grok" && - record.provider !== "runway" && - record.provider !== "kling" && - record.provider !== "veo" - ) { - return undefined; - } - const type = - record.provider === "kling" && record.providerTaskType - ? ` --type ${record.providerTaskType}` - : ""; - return `vibe generate video-status ${record.providerTaskId} -p ${record.provider}${type} --json`; - } - if (record.jobType === "generate-music" && record.provider === "replicate") { - return `vibe generate music-status ${record.providerTaskId} --json`; - } - return undefined; -} - function previewPrompt(prompt: string | undefined): string | undefined { if (!prompt) return undefined; const compact = prompt.replace(/\s+/g, " ").trim(); diff --git a/packages/cli/src/commands/_shared/video-jobs.ts b/packages/cli/src/commands/_shared/video-jobs.ts new file mode 100644 index 00000000..286e3e57 --- /dev/null +++ b/packages/cli/src/commands/_shared/video-jobs.ts @@ -0,0 +1,113 @@ +/** + * The CLI side of the video job contract: open a provider by name with the + * user's key, and turn local files into the image form it takes. Video + * executors, job records, and builds go through here instead of branching + * on provider names. + */ + +import { readFile } from "node:fs/promises"; +import { resolve } from "node:path"; +import { + ProviderError, + createVideoGenerator, + defaultModel, + type VideoGenerator, + type VideoJob, +} from "@vibeframe/ai-providers"; +import { getConfiguredApiKey } from "../../utils/api-key.js"; +import { resolveUploadHost } from "../../utils/upload-host.js"; + +/** Env var holding each video provider's key. */ +export const VIDEO_PROVIDER_ENV: Readonly> = { + seedance: "FAL_API_KEY", + fal: "FAL_API_KEY", + grok: "XAI_API_KEY", + kling: "KLING_API_KEY", + runway: "RUNWAY_API_SECRET", + omni: "GOOGLE_API_KEY", + veo: "GOOGLE_API_KEY", +}; + +/** `fal` is the old name for `seedance`. */ +export function canonicalVideoProvider(provider: string): string { + const id = provider.toLowerCase(); + return id === "fal" ? "seedance" : id; +} + +/** A video generator for `provider`, with the given key or the configured one. */ +export async function openVideoGenerator(provider: string, apiKey?: string): Promise { + const id = canonicalVideoProvider(provider); + const envVar = VIDEO_PROVIDER_ENV[id]; + if (!envVar) { + throw new ProviderError({ kind: "invalid-request", provider: id, message: `Unknown video provider "${provider}".` }); + } + const key = await getConfiguredApiKey(envVar, apiKey); + if (!key) { + throw new ProviderError({ kind: "auth", provider: id, message: `${envVar} required for ${id}.` }); + } + return createVideoGenerator(id, key); +} + +/** + * A job handle for a bare task ID (a user typed it, or an old job record + * predates stored handles). Providers that need more than the ID recover + * it themselves: Kling tries both task types, Seedance uses its model's app. + */ +export function jobFromTaskId(provider: string, taskId: string, opts: { model?: string; taskType?: string } = {}): VideoJob { + const id = canonicalVideoProvider(provider); + return { + provider: id, + id: taskId, + model: opts.model ?? defaultModel(id, "video").id, + submittedAt: "", + ...(opts.taskType ? { meta: { type: opts.taskType } } : {}), + }; +} + +const MIME_TYPES: Readonly> = { + jpg: "image/jpeg", + jpeg: "image/jpeg", + png: "image/png", + gif: "image/gif", + webp: "image/webp", + mp4: "video/mp4", + mov: "video/quicktime", + mp3: "audio/mpeg", + wav: "audio/wav", +}; + +export function mimeTypeForPath(path: string, fallback: string): string { + return MIME_TYPES[path.toLowerCase().split(".").pop() ?? ""] ?? fallback; +} + +/** A local path, URL, or data URI as a URL or data URI (local files are inlined). */ +export async function fileToUrlOrDataUri(input: string, fallbackMimeType: string): Promise { + if (/^(https?:|data:)/.test(input)) return input; + const buffer = await readFile(resolve(process.cwd(), input)); + return `data:${mimeTypeForPath(input, fallbackMimeType)};base64,${buffer.toString("base64")}`; +} + +/** + * An input image in the form `generator` takes: uploaded to the configured + * upload host for URL-only providers, inlined as a data URI otherwise. + */ +export async function videoImageInput(generator: VideoGenerator, input: string): Promise { + if (generator.imageInput !== "url" || /^https?:/.test(input)) { + if (generator.imageInput === "data-uri" && /^https?:/.test(input)) { + const response = await fetch(input); + if (!response.ok) throw new Error(`Could not fetch image ${input}: HTTP ${response.status}`); + const type = response.headers.get("content-type") ?? mimeTypeForPath(input, "image/png"); + return `data:${type};base64,${Buffer.from(await response.arrayBuffer()).toString("base64")}`; + } + return fileToUrlOrDataUri(input, "image/png"); + } + const buffer = input.startsWith("data:") + ? Buffer.from(input.slice(input.indexOf(",") + 1), "base64") + : await readFile(resolve(process.cwd(), input)); + const host = await resolveUploadHost(); + const upload = await host.uploadImage(buffer, { + filename: input.startsWith("data:") ? "frame.png" : input, + mimeType: input.startsWith("data:") ? input.slice(5, input.indexOf(";")) : mimeTypeForPath(input, "image/png"), + }); + return upload.url; +} diff --git a/packages/cli/src/commands/_shared/video-providers.ts b/packages/cli/src/commands/_shared/video-providers.ts deleted file mode 100644 index 5d00b6bc..00000000 --- a/packages/cli/src/commands/_shared/video-providers.ts +++ /dev/null @@ -1,219 +0,0 @@ -/** - * @module _shared/video-providers - * @description Per-provider `generateVideoWithRetry*` helpers used by the - * script-to-video pipeline. Each wraps a single AI provider's video - * generation API in a retry loop with onProgress callbacks. Split out of - * `ai-script-pipeline.ts` in v0.69 (Plan G Phase 4). - */ - -import chalk from "chalk"; -import type { - GeminiProvider, - GrokProvider, - KlingProvider, - RunwayProvider, -} from "@vibeframe/ai-providers"; -import { sleep, RETRY_DELAY_MS, type StoryboardSegment } from "./video-utils.js"; - -/** Generate video with retry logic for Grok provider. */ -export async function generateVideoWithRetryGrok( - grok: GrokProvider, - segment: StoryboardSegment, - options: { - duration: number; - aspectRatio: "16:9" | "9:16" | "1:1"; - referenceImage?: string; - }, - maxRetries: number, - onProgress?: (message: string) => void, -): Promise<{ requestId: string } | null> { - const prompt = segment.visualStyle - ? `${segment.visuals}. Style: ${segment.visualStyle}` - : segment.visuals; - - for (let attempt = 0; attempt <= maxRetries; attempt++) { - try { - const result = await grok.generateVideo(prompt, { - prompt, - duration: options.duration, - aspectRatio: options.aspectRatio, - referenceImage: options.referenceImage, - }); - - if (result.status !== "failed" && result.id) { - return { requestId: result.id }; - } - - const providerErr = result.error || "Grok returned failed status"; - if (attempt < maxRetries) { - onProgress?.( - `⚠ ${providerErr.slice(0, 50)}... retry ${attempt + 1}/${maxRetries}`, - ); - await sleep(RETRY_DELAY_MS); - } else { - console.error(chalk.dim(`\n [Grok error: ${providerErr}]`)); - } - } catch (err) { - const errMsg = err instanceof Error ? err.message : String(err); - if (attempt < maxRetries) { - onProgress?.(`⚠ Error: ${errMsg.slice(0, 50)}... retry ${attempt + 1}/${maxRetries}`); - await sleep(RETRY_DELAY_MS); - } else { - console.error(chalk.dim(`\n [Grok error: ${errMsg}]`)); - } - } - } - return null; -} - -/** - * Generate video with retry logic for Kling provider. - * Supports image-to-video with URL (v2.5/v2.6 models). - */ -export async function generateVideoWithRetryKling( - kling: KlingProvider, - segment: StoryboardSegment, - options: { - duration: 5 | 10; - aspectRatio: "16:9" | "9:16" | "1:1"; - referenceImage?: string; // Optional: base64 or URL for image2video - }, - maxRetries: number, - onProgress?: (message: string) => void, -): Promise<{ taskId: string; type: "text2video" | "image2video" } | null> { - const prompt = segment.visualStyle - ? `${segment.visuals}. Style: ${segment.visualStyle}` - : segment.visuals; - - for (let attempt = 0; attempt <= maxRetries; attempt++) { - try { - const result = await kling.generateVideo(prompt, { - prompt, - // Pass reference image (base64 or URL) - KlingProvider handles v1.5 fallback for base64 - referenceImage: options.referenceImage, - duration: options.duration, - aspectRatio: options.aspectRatio, - mode: "std", // std mode for faster generation - }); - - if (result.status !== "failed" && result.id) { - return { - taskId: result.id, - type: options.referenceImage ? "image2video" : "text2video", - }; - } - - const providerErr = result.error || "Kling returned failed status"; - if (attempt < maxRetries) { - onProgress?.(`⚠ ${providerErr.slice(0, 50)}... retry ${attempt + 1}/${maxRetries}`); - await sleep(RETRY_DELAY_MS); - } else { - console.error(chalk.dim(`\n [Kling error: ${providerErr}]`)); - } - } catch (err) { - const errMsg = err instanceof Error ? err.message : String(err); - if (attempt < maxRetries) { - onProgress?.(`⚠ Error: ${errMsg.slice(0, 50)}... retry ${attempt + 1}/${maxRetries}`); - await sleep(RETRY_DELAY_MS); - } else { - console.error(chalk.dim(`\n [Kling error: ${errMsg}]`)); - } - } - } - return null; -} - -/** Generate video with retry logic for Runway provider. */ -export async function generateVideoWithRetryRunway( - runway: RunwayProvider, - segment: StoryboardSegment, - referenceImage: string, - options: { - duration: 5 | 10; - aspectRatio: "16:9" | "9:16"; - }, - maxRetries: number, - onProgress?: (message: string) => void, -): Promise<{ taskId: string } | null> { - for (let attempt = 0; attempt <= maxRetries; attempt++) { - try { - const result = await runway.generateVideo(segment.visuals, { - prompt: segment.visuals, - referenceImage, - duration: options.duration, - aspectRatio: options.aspectRatio, - }); - - if (result.status !== "failed" && result.id) { - return { taskId: result.id }; - } - - const providerErr = result.error || "Runway returned failed status"; - if (attempt < maxRetries) { - onProgress?.(`⚠ ${providerErr.slice(0, 50)}... retry ${attempt + 1}/${maxRetries}`); - await sleep(RETRY_DELAY_MS); - } else { - console.error(chalk.dim(`\n [Runway error: ${providerErr}]`)); - } - } catch (err) { - const errMsg = err instanceof Error ? err.message : String(err); - if (attempt < maxRetries) { - onProgress?.(`⚠ Error: ${errMsg.slice(0, 50)}... retry ${attempt + 1}/${maxRetries}`); - await sleep(RETRY_DELAY_MS); - } else { - console.error(chalk.dim(`\n [Runway error: ${errMsg}]`)); - } - } - } - return null; -} - -/** Generate video with retry logic for Veo (Gemini) provider. */ -export async function generateVideoWithRetryVeo( - gemini: GeminiProvider, - segment: StoryboardSegment, - options: { - duration: 4 | 6 | 8; - aspectRatio: "16:9" | "9:16" | "1:1"; - referenceImage?: string; - }, - maxRetries: number, - onProgress?: (message: string) => void, -): Promise<{ operationName: string } | null> { - const prompt = segment.visualStyle - ? `${segment.visuals}. Style: ${segment.visualStyle}` - : segment.visuals; - - for (let attempt = 0; attempt <= maxRetries; attempt++) { - try { - const result = await gemini.generateVideo(prompt, { - prompt, - referenceImage: options.referenceImage, - duration: options.duration, - aspectRatio: options.aspectRatio, - model: "veo-3.1-fast-generate-preview", - }); - - if (result.status !== "failed" && result.id) { - return { operationName: result.id }; - } - - const providerErr = result.error || "Veo returned failed status"; - if (attempt < maxRetries) { - onProgress?.(`⚠ ${providerErr.slice(0, 50)}... retry ${attempt + 1}/${maxRetries}`); - await sleep(RETRY_DELAY_MS); - } else { - console.error(chalk.dim(`\n [Veo error: ${providerErr}]`)); - } - } catch (err) { - const errMsg = err instanceof Error ? err.message : String(err); - if (attempt < maxRetries) { - onProgress?.(`⚠ Error: ${errMsg.slice(0, 50)}... retry ${attempt + 1}/${maxRetries}`); - await sleep(RETRY_DELAY_MS); - } else { - console.error(chalk.dim(`\n [Veo error: ${errMsg}]`)); - } - } - } - return null; -} diff --git a/packages/cli/src/commands/ai-video.network.test.ts b/packages/cli/src/commands/ai-video.network.test.ts new file mode 100644 index 00000000..ba1f8c49 --- /dev/null +++ b/packages/cli/src/commands/ai-video.network.test.ts @@ -0,0 +1,124 @@ +/** + * Video executors against the fake provider network: real providers, real + * job handles, no provider-name branches in between. + */ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import { FAKE_PNG, createFakeProviderNetwork, jsonResponse, type FakeProviderNetwork } from "../testing/fake-provider-network.js"; +import { executeVideoCancel, executeVideoExtend, executeVideoGenerate, executeVideoStatus } from "./ai-video.js"; + +const KLING = "api.klingai.com"; +const kling = (data: Record) => jsonResponse({ code: 0, message: "ok", request_id: "r", data }); + +let net: FakeProviderNetwork; +let dir: string; +const savedEnv = { ...process.env }; + +beforeEach(() => { + Object.assign(process.env, { + FAL_API_KEY: "test-fal", + KLING_API_KEY: "test-kling", + RUNWAY_API_SECRET: "test-runway", + IMGBB_API_KEY: "test-imgbb", + VIBE_UPLOAD_PROVIDER: "imgbb", + }); + dir = mkdtempSync(join(tmpdir(), "ai-video-network-")); + net = createFakeProviderNetwork(); + net.install(); +}); + +afterEach(() => { + net.uninstall(); + process.env = { ...savedEnv }; + rmSync(dir, { recursive: true, force: true }); + vi.useRealTimers(); +}); + +describe("video executors over the provider network", () => { + it("defaults to the configured video provider, not Kling", async () => { + const result = await executeVideoGenerate({ prompt: "a paper boat", wait: false }); + + expect(result).toMatchObject({ success: true, provider: "seedance", status: "processing" }); + expect(result.job).toMatchObject({ provider: "seedance", model: "seedance-2.0" }); + }); + + it("uploads a local first frame for Kling, which only takes URLs, and keeps the task type in the handle", async () => { + const frame = join(dir, "frame.png"); + writeFileSync(frame, FAKE_PNG); + net.on("POST", KLING, /\/image2video$/, () => kling({ task_id: "k-1", task_status: "submitted" })); + + const result = await executeVideoGenerate({ prompt: "p", provider: "kling", image: frame, wait: false }); + + expect(result.job).toMatchObject({ provider: "kling", id: "k-1", meta: { type: "image2video" } }); + const submit = net.to(KLING).find((r) => r.method === "POST"); + expect(submit?.body).toMatchObject({ image: "https://fake.media/upload.png" }); + expect(net.to("api.imgbb.com")).toHaveLength(1); + }); + + it("polls a stored handle on the endpoint it was submitted to", async () => { + net.on("GET", KLING, /\/image2video\/k-1$/, () => + kling({ task_id: "k-1", task_status: "succeed", task_result: { videos: [{ id: "v-1", url: "https://fake.media/k.mp4", duration: "5" }] } }) + ); + const out = join(dir, "out.mp4"); + + const result = await executeVideoStatus({ + taskId: "k-1", + job: { provider: "kling", id: "k-1", model: "kling-v3", submittedAt: "", meta: { type: "image2video" } }, + output: out, + }); + + expect(result).toMatchObject({ success: true, status: "completed", videoUrl: "https://fake.media/k.mp4" }); + expect(readFileSync(out).length).toBeGreaterThan(0); + }); + + it("refuses to cancel on a provider with no cancel API, and cancels Runway", async () => { + await expect(executeVideoCancel({ taskId: "k-1", provider: "kling" })).resolves.toMatchObject({ + success: false, + errorKind: "unsupported", + }); + net.on("DELETE", "api.dev.runwayml.com", /\/v1\/tasks\/rw-1$/, () => new Response(null, { status: 204 })); + await expect(executeVideoCancel({ taskId: "rw-1", provider: "runway" })).resolves.toEqual({ success: true }); + }); + + it("extends a Kling video by the task ID the generation returned", async () => { + net + .on("GET", KLING, /\/text2video\/k-2$/, () => + kling({ task_id: "k-2", task_status: "succeed", task_result: { videos: [{ id: "v-2", url: "https://fake.media/k.mp4", duration: "5" }] } }) + ) + .on("POST", KLING, /\/video-extend$/, () => kling({ task_id: "k-3", task_status: "submitted" })); + + const result = await executeVideoExtend({ videoId: "k-2", provider: "kling", prompt: "keep drifting", wait: false }); + + expect(result).toMatchObject({ success: true, taskId: "k-3", status: "processing", job: { meta: { type: "video-extend" } } }); + expect(net.to(KLING).find((r) => r.method === "POST")?.body).toMatchObject({ video_id: "v-2" }); + }); + + it("hands back a still-running job, not a failure, when the wait runs out", async () => { + vi.useFakeTimers({ toFake: ["Date"] }); + const started = Date.now(); + let polls = 0; + net.on("GET", "queue.fal.run", /\/requests\/[^/]+\/status$/, () => { + // Jump the clock past the 15-minute wait on the second poll. + if (++polls === 2) vi.setSystemTime(started + 16 * 60_000); + return jsonResponse({ status: "IN_PROGRESS" }); + }); + + const result = await executeVideoGenerate({ prompt: "p", provider: "seedance" }); + + expect(result).toMatchObject({ success: true, status: "processing", provider: "seedance" }); + expect(result.job?.id).toBeTruthy(); + }, 20_000); + + it("reports a classified failure with the task ID kept", async () => { + net.on("GET", "queue.fal.run", /\/requests\/[^/]+$/, () => + jsonResponse({ detail: [{ msg: "likenesses of real people", type: "content_policy_violation" }] }, 422) + ); + + const result = await executeVideoGenerate({ prompt: "p", provider: "seedance" }); + + expect(result).toMatchObject({ success: false, errorKind: "likeness", taskId: "fal-1" }); + }); +}); diff --git a/packages/cli/src/commands/ai-video.ts b/packages/cli/src/commands/ai-video.ts index 962ec199..487569ef 100644 --- a/packages/cli/src/commands/ai-video.ts +++ b/packages/cli/src/commands/ai-video.ts @@ -1,53 +1,80 @@ /** * @module ai-video - * @description Library functions for video generation, status, cancel, and - * extension. Powers the manifest tools `generate_video`, - * `generate_video_status`, `generate_video_cancel`, `generate_video_extend` - * (the user reaches these via `vibe generate video[-...]`). - * - * The legacy `vibe ai video / video-status / video-cancel / kling / kling-status / - * video-extend` Commander registrations were removed alongside the dead - * `commands/ai.ts` orchestrator (the `vibe ai *` namespace was never - * `addCommand`'d to `program`). + * @description Video generation, status, cancel, and extension on the video + * job contract. Powers the manifest tools `generate_video`, + * `generate_video_cancel`, `generate_video_extend`, and job refreshes in + * `vibe status`; every provider goes through `VideoGenerator`, so nothing + * here branches on provider names. * * @see MODELS.md for AI model configuration */ -import { mkdir, readFile, writeFile } from "node:fs/promises"; +import { mkdir, writeFile } from "node:fs/promises"; import { dirname, resolve } from "node:path"; import { - FalProvider, - GeminiProvider, - GrokProvider, - KlingProvider, - OmniProvider, - RunwayProvider, - resolveGrokVideoModel, - resolveKlingModel, - resolveSeedanceVariant, - resolveVeoModel, + isProviderError, + waitForVideoJob, type MediaReference, + type ProviderErrorKind, + type VideoGenerator, + type VideoJob, + type VideoJobState, + type VideoRequest, } from "@vibeframe/ai-providers"; -import { resolveUploadHost } from "../utils/upload-host.js"; -import { downloadVideo } from "./ai-helpers.js"; -import { getConfiguredApiKey } from "../utils/api-key.js"; +import { resolveProvider } from "../utils/provider-resolver.js"; +import { videoModelSpec } from "../utils/model-lifecycle.js"; +import { + canonicalVideoProvider, + fileToUrlOrDataUri, + jobFromTaskId, + openVideoGenerator, + videoImageInput, +} from "./_shared/video-jobs.js"; + +/** How long a waiting call polls before handing back the job to check later. */ +const WAIT_TIMEOUT_MS = 15 * 60_000; /** Write a downloaded video to `output` (relative to cwd), creating its directory. */ -async function saveOutput(output: string, buffer: Buffer): Promise { +async function saveOutput(output: string, bytes: Uint8Array): Promise { const outputPath = resolve(process.cwd(), output); await mkdir(dirname(outputPath), { recursive: true }); - await writeFile(outputPath, buffer); + await writeFile(outputPath, bytes); return outputPath; } +/** A failed result from a thrown error, keeping the provider error kind. */ +function failure(error: unknown, prefix: string): { success: false; error: string; errorKind?: ProviderErrorKind } { + if (isProviderError(error)) return { success: false, error: error.message, errorKind: error.kind }; + return { success: false, error: `${prefix}: ${error instanceof Error ? error.message : String(error)}` }; +} + +/** Wait for a job (or read it once), then download the video when asked. */ +async function settle( + generator: VideoGenerator, + job: VideoJob, + opts: { wait: boolean; output?: string } +): Promise<{ state: VideoJobState; outputPath?: string }> { + const state = opts.wait + ? await waitForVideoJob(generator, job, { timeoutMs: WAIT_TIMEOUT_MS }) + : await generator.getVideoJob(job); + const outputPath = + opts.output && state.status === "completed" + ? await saveOutput(opts.output, await generator.downloadVideo(job, state)) + : undefined; + return { state, outputPath }; +} + // ============================================================================ // Video Generation // ============================================================================ export interface VideoGenerateOptions { prompt: string; + /** Default: the configured or first available video provider. */ provider?: "grok" | "runway" | "kling" | "veo" | "seedance" | "fal" | "omni"; image?: string; + /** Ending frame for providers that interpolate between two frames. */ + endImage?: string; refImages?: string[]; refVideos?: string[]; refAudio?: string[]; @@ -66,393 +93,109 @@ export interface VideoGenerateOptions { output?: string; wait?: boolean; apiKey?: string; + /** + * Called as soon as the provider accepts the job, before waiting, so the + * caller can record it: a crash or timeout mid-wait then leaves a job that + * can still be polled and downloaded. + */ + onSubmitted?: (job: VideoJob) => void | Promise; } export interface VideoGenerateResult { success: boolean; taskId?: string; + /** The provider job handle; store it to poll, cancel, or extend later. */ + job?: VideoJob; status?: string; videoUrl?: string; duration?: number; outputPath?: string; provider?: string; error?: string; + errorKind?: ProviderErrorKind; } -export async function executeVideoGenerate( - options: VideoGenerateOptions -): Promise { - const { - prompt, - provider = "kling", - image, - refImages, - refVideos, - refAudio, - duration = 5, - ratio = "16:9", - seed, - mode = "std", - negative, - resolution, - veoModel = "3.1-fast", - runwayModel, - seedanceModel = "quality", - grokModel, - klingModel, - generateAudio, - output, - wait = true, - apiKey, - } = options; - +export async function executeVideoGenerate(options: VideoGenerateOptions): Promise { + const provider = canonicalVideoProvider(options.provider ?? resolveProvider("video")?.name ?? "seedance"); try { - const envKeyMap: Record = { - grok: "XAI_API_KEY", - runway: "RUNWAY_API_SECRET", - kling: "KLING_API_KEY", - veo: "GOOGLE_API_KEY", - omni: "GOOGLE_API_KEY", - seedance: "FAL_API_KEY", - fal: "FAL_API_KEY", + const generator = await openVideoGenerator(provider, options.apiKey); + const references: MediaReference[] = []; + for (const [kind, paths, mime] of [ + ["image", options.refImages, "image/png"], + ["video", options.refVideos, "video/mp4"], + ["audio", options.refAudio, "audio/mpeg"], + ] as const) { + for (const sourcePath of paths ?? []) { + references.push({ kind, url: await fileToUrlOrDataUri(sourcePath, mime), sourcePath }); + } + } + const request: VideoRequest = { + prompt: options.prompt, + model: videoModelSpec(provider, options)?.id, + durationSec: options.duration ?? 5, + aspectRatio: (options.ratio ?? "16:9") as VideoRequest["aspectRatio"], + resolution: options.resolution, + image: options.image ? await videoImageInput(generator, options.image) : undefined, + lastFrame: options.endImage ? await videoImageInput(generator, options.endImage) : undefined, + references: references.length > 0 ? references : undefined, + negativePrompt: options.negative, + seed: options.seed, + generateAudio: options.generateAudio, + providerOptions: options.mode ? { mode: options.mode } : undefined, }; - const envKey = envKeyMap[provider] || ""; - const key = await getConfiguredApiKey(envKey, apiKey); - if (!key) return { success: false, error: `${envKeyMap[provider]} required for ${provider}` }; - let referenceImage: string | undefined; - let referenceImageBuffer: Buffer | undefined; - let referenceImageMimeType: string | undefined; - if (image) { - const imagePath = resolve(process.cwd(), image); - const imageBuffer = await readFile(imagePath); - const ext = image.toLowerCase().split(".").pop(); - const mimeTypes: Record = { - jpg: "image/jpeg", - jpeg: "image/jpeg", - png: "image/png", - gif: "image/gif", - webp: "image/webp", - }; - const mimeType = mimeTypes[ext || "png"] || "image/png"; - referenceImageBuffer = imageBuffer; - referenceImageMimeType = mimeType; - referenceImage = `data:${mimeType};base64,${imageBuffer.toString("base64")}`; + const job = await generator.submitVideo(request); + await options.onSubmitted?.(job); + if (options.wait === false) { + return { success: true, taskId: job.id, job, status: "processing", provider }; } - - if (provider === "seedance" || provider === "fal") { - const fal = new FalProvider(); - await fal.initialize({ apiKey: key }); - - const references = await prepareSeedanceReferences({ - refImages, - refVideos, - refAudio, - }); - - let falImage = referenceImage; - if (falImage && falImage.startsWith("data:") && references.length === 0) { - const uploadHost = await resolveUploadHost(); - const upload = await uploadHost.uploadImage(referenceImageBuffer!, { - filename: image, - mimeType: referenceImageMimeType, - }); - falImage = upload.url; - } - - const model = resolveSeedanceVariant(seedanceModel); - const result = await fal.generateVideo(prompt, { - prompt, - referenceImage: references.length > 0 ? undefined : falImage, - references: references.length > 0 ? references : undefined, - duration, - aspectRatio: ratio as "16:9" | "9:16" | "1:1" | "4:5", - negativePrompt: negative, - model, - resolution, - generateAudio, - }); - - if (result.status === "failed") - return { success: false, error: result.error || "Seedance generation failed" }; - - let outputPath: string | undefined; - if (output && result.videoUrl) { - const buffer = await downloadVideo(result.videoUrl, key); - outputPath = await saveOutput(output, buffer); - } - - return { - success: true, - taskId: result.id, - status: "completed", - videoUrl: result.videoUrl, - outputPath, - provider: "seedance", - }; - } else if (provider === "runway") { - const runway = new RunwayProvider(); - await runway.initialize({ apiKey: key }); - - const result = await runway.generateVideo(prompt, { - prompt, - model: runwayModel, - referenceImage, - duration, - aspectRatio: ratio as "16:9" | "9:16", - seed, - }); - - if (result.status === "failed") - return { success: false, error: result.error || "Runway generation failed" }; - if (!wait) - return { success: true, taskId: result.id, status: "processing", provider: "runway" }; - - const finalResult = await runway.waitForCompletion(result.id, () => {}, 300000); - if (finalResult.status !== "completed") - return { success: false, error: finalResult.error || "Runway generation failed" }; - - let outputPath: string | undefined; - if (output && finalResult.videoUrl) { - const buffer = await downloadVideo(finalResult.videoUrl, key); - outputPath = await saveOutput(output, buffer); - } - - return { - success: true, - taskId: result.id, - status: "completed", - videoUrl: finalResult.videoUrl, - duration: finalResult.duration, - outputPath, - provider: "runway", - }; - } else if (provider === "kling") { - const kling = new KlingProvider(); - await kling.initialize({ apiKey: key }); - if (!kling.isConfigured()) return { success: false, error: "Invalid Kling API key format" }; - - let klingImage = referenceImage; - if (klingImage && klingImage.startsWith("data:")) { - const uploadHost = await resolveUploadHost(); - const upload = await uploadHost.uploadImage(referenceImageBuffer!, { - filename: image, - mimeType: referenceImageMimeType, - }); - klingImage = upload.url; - } - - const result = await kling.generateVideo(prompt, { - prompt, - model: resolveKlingModel(klingModel), - referenceImage: klingImage, - duration, - aspectRatio: ratio as "16:9" | "9:16" | "1:1", - negativePrompt: negative, - mode: mode as "std" | "pro", - }); - - if (result.status === "failed") - return { success: false, error: result.error || "Kling generation failed" }; - const taskType = referenceImage ? "image2video" : "text2video"; - if (!wait) - return { success: true, taskId: result.id, status: "processing", provider: "kling" }; - - const finalResult = await kling.waitForCompletion(result.id, taskType, () => {}, 600000); - if (finalResult.status !== "completed") - return { success: false, error: finalResult.error || "Kling generation failed" }; - - let outputPath: string | undefined; - if (output && finalResult.videoUrl) { - const buffer = await downloadVideo(finalResult.videoUrl, key); - outputPath = await saveOutput(output, buffer); - } - - return { - success: true, - taskId: result.id, - status: "completed", - videoUrl: finalResult.videoUrl, - duration: finalResult.duration, - outputPath, - provider: "kling", - }; - } else if (provider === "veo") { - const gemini = new GeminiProvider(); - await gemini.initialize({ apiKey: key }); - - const model = resolveVeoModel(veoModel); - const veoDuration = duration <= 6 ? 6 : 8; - - const result = await gemini.generateVideo(prompt, { - prompt, - referenceImage, - duration: veoDuration, - aspectRatio: ratio as "16:9" | "9:16" | "1:1", - model: model, - negativePrompt: negative, - resolution: resolution as "720p" | "1080p" | "4k" | undefined, - }); - - if (result.status === "failed") - return { success: false, error: result.error || "Veo generation failed" }; - if (!wait) return { success: true, taskId: result.id, status: "processing", provider: "veo" }; - - const finalResult = await gemini.waitForVideoCompletion(result.id, () => {}, 300000); - if (finalResult.status !== "completed") - return { success: false, error: finalResult.error || "Veo generation failed" }; - - let outputPath: string | undefined; - if (output && finalResult.videoUrl) { - const buffer = await downloadVideo(finalResult.videoUrl, key); - outputPath = await saveOutput(output, buffer); - } - - return { - success: true, - taskId: result.id, - status: "completed", - videoUrl: finalResult.videoUrl, - outputPath, - provider: "veo", - }; - } else if (provider === "grok") { - const grok = new GrokProvider(); - await grok.initialize({ apiKey: key }); - - const result = await grok.generateVideo(prompt, { - prompt, - model: resolveGrokVideoModel(grokModel), - referenceImage, - duration, - aspectRatio: ratio as "16:9" | "9:16" | "1:1", - resolution, - generateAudio, - }); - - if (result.status === "failed") - return { success: false, error: result.error || "Grok generation failed" }; - if (!wait) - return { success: true, taskId: result.id, status: "processing", provider: "grok" }; - - const finalResult = await grok.waitForCompletion(result.id, () => {}, 300000); - if (finalResult.status !== "completed") - return { success: false, error: finalResult.error || "Grok generation failed" }; - - let outputPath: string | undefined; - if (output && finalResult.videoUrl) { - const buffer = await downloadVideo(finalResult.videoUrl, key); - outputPath = await saveOutput(output, buffer); - } - - return { - success: true, - taskId: result.id, - status: "completed", - videoUrl: finalResult.videoUrl, - duration: finalResult.duration, - outputPath, - provider: "grok", - }; - } else if (provider === "omni") { - // Gemini Omni 1.1 Flash (interactions endpoint), same GOOGLE_API_KEY. - const omni = new OmniProvider(); - await omni.initialize({ apiKey: key }); - const result = await omni.generateVideo(prompt, { - prompt, - referenceImage, - aspectRatio: ratio as "16:9" | "9:16" | "1:1", - resolution, - }); - if (result.status !== "completed" || !result.videoUrl) { - return { success: false, error: result.error || "Gemini Omni generation failed" }; - } - let outputPath: string | undefined; - if (output) { - const buffer = await downloadVideo(result.videoUrl, key); - outputPath = await saveOutput(output, buffer); + let settled: Awaited>; + try { + settled = await settle(generator, job, { wait: true, output: options.output }); + } catch (error) { + // Running out of wait time is not a failure: the job may still finish. + if (isProviderError(error) && error.kind === "timeout") { + return { success: true, taskId: job.id, job, status: "processing", provider }; } + throw error; + } + const { state, outputPath } = settled; + if (state.status !== "completed") { return { - success: true, - taskId: result.id, - status: "completed", - videoUrl: result.videoUrl, - outputPath, - provider: "omni", + success: false, + taskId: job.id, + job, + provider, + status: state.status, + error: state.error?.message ?? `${provider} generation ${state.status}`, + errorKind: state.error?.kind, }; } - - return { success: false, error: `Unsupported provider: ${provider}` }; - } catch (error) { return { - success: false, - error: `Video generation failed: ${error instanceof Error ? error.message : String(error)}`, + success: true, + taskId: job.id, + job, + status: "completed", + videoUrl: state.videoUrl, + duration: state.durationSec, + outputPath, + provider, }; + } catch (error) { + return { ...failure(error, "Video generation failed"), provider }; } } -async function prepareSeedanceReferences(opts: { - refImages?: string[]; - refVideos?: string[]; - refAudio?: string[]; -}): Promise { - const references: MediaReference[] = []; - for (const sourcePath of opts.refImages ?? []) { - references.push({ - kind: "image", - url: await fileInputToUrlOrDataUri(sourcePath, "image/png"), - sourcePath, - }); - } - for (const sourcePath of opts.refVideos ?? []) { - references.push({ - kind: "video", - url: await fileInputToUrlOrDataUri(sourcePath, "video/mp4"), - sourcePath, - }); - } - for (const sourcePath of opts.refAudio ?? []) { - references.push({ - kind: "audio", - url: await fileInputToUrlOrDataUri(sourcePath, "audio/mpeg"), - sourcePath, - }); - } - return references; -} - -async function fileInputToUrlOrDataUri(input: string, fallbackMimeType: string): Promise { - if (input.startsWith("http://") || input.startsWith("https://") || input.startsWith("data:")) { - return input; - } - const absPath = resolve(process.cwd(), input); - const buffer = await readFile(absPath); - return `data:${mimeTypeForPath(input, fallbackMimeType)};base64,${buffer.toString("base64")}`; -} - -function mimeTypeForPath(path: string, fallback: string): string { - const ext = path.toLowerCase().split(".").pop(); - const mimeTypes: Record = { - jpg: "image/jpeg", - jpeg: "image/jpeg", - png: "image/png", - gif: "image/gif", - webp: "image/webp", - mp4: "video/mp4", - mov: "video/quicktime", - mp3: "audio/mpeg", - wav: "audio/wav", - }; - return mimeTypes[ext || ""] || fallback; -} - // ============================================================================ -// Video Status (Runway / Kling) +// Video Status // ============================================================================ export interface VideoStatusOptions { taskId: string; - provider?: "grok" | "runway" | "kling" | "veo"; + provider?: string; + /** The stored job handle; preferred over `taskId` + `taskType`. */ + job?: VideoJob; + /** Kling task type, for a bare task ID. */ taskType?: "text2video" | "image2video"; wait?: boolean; output?: string; @@ -468,194 +211,73 @@ export interface VideoStatusResult { duration?: number; outputPath?: string; error?: string; + errorKind?: ProviderErrorKind; } export async function executeVideoStatus(options: VideoStatusOptions): Promise { - const { - taskId, - provider = "runway", - taskType = "text2video", - wait = false, - output, - apiKey, - } = options; - + const job = + options.job ?? jobFromTaskId(options.provider ?? "runway", options.taskId, { taskType: options.taskType }); try { - const envKeyMap: Record = { - grok: "XAI_API_KEY", - runway: "RUNWAY_API_SECRET", - kling: "KLING_API_KEY", - veo: "GOOGLE_API_KEY", - }; - const key = apiKey || process.env[envKeyMap[provider] || ""]; - if (!key) return { success: false, error: `${envKeyMap[provider]} required` }; - - if (provider === "grok") { - const grok = new GrokProvider(); - await grok.initialize({ apiKey: key }); - - let result = await grok.getGenerationStatus(taskId); - - if (wait && result.status !== "completed" && result.status !== "failed") { - result = await grok.waitForCompletion(taskId, () => {}); - } - - let outputPath: string | undefined; - if (output && result.videoUrl) { - const buffer = await downloadVideo(result.videoUrl); - outputPath = await saveOutput(output, buffer); - } - - return { - success: true, - taskId, - status: result.status, - progress: result.progress, - videoUrl: result.videoUrl, - outputPath, - error: result.error, - }; - } else if (provider === "runway") { - const runway = new RunwayProvider(); - await runway.initialize({ apiKey: key }); - - let result = await runway.getGenerationStatus(taskId); - - if ( - wait && - result.status !== "completed" && - result.status !== "failed" && - result.status !== "cancelled" - ) { - result = await runway.waitForCompletion(taskId, () => {}); - } - - let outputPath: string | undefined; - if (output && result.videoUrl) { - const buffer = await downloadVideo(result.videoUrl, key); - outputPath = await saveOutput(output, buffer); - } - - return { - success: true, - taskId, - status: result.status, - progress: result.progress, - videoUrl: result.videoUrl, - outputPath, - }; - } else if (provider === "kling") { - const kling = new KlingProvider(); - await kling.initialize({ apiKey: key }); - - let result = await kling.getGenerationStatus(taskId, taskType); - - if ( - wait && - result.status !== "completed" && - result.status !== "failed" && - result.status !== "cancelled" - ) { - result = await kling.waitForCompletion(taskId, taskType, () => {}); - } - - let outputPath: string | undefined; - if (output && result.videoUrl) { - const buffer = await downloadVideo(result.videoUrl, key); - outputPath = await saveOutput(output, buffer); - } - - return { - success: true, - taskId, - status: result.status, - videoUrl: result.videoUrl, - duration: result.duration, - outputPath, - error: result.error, - }; - } else if (provider === "veo") { - const gemini = new GeminiProvider(); - await gemini.initialize({ apiKey: key }); - - let result = await gemini.getGenerationStatus(taskId); - - if ( - wait && - result.status !== "completed" && - result.status !== "failed" && - result.status !== "cancelled" - ) { - result = await gemini.waitForVideoCompletion(taskId, () => {}); - } - - let outputPath: string | undefined; - if (output && result.videoUrl) { - const buffer = await downloadVideo(result.videoUrl, key); - outputPath = await saveOutput(output, buffer); - } - - return { - success: true, - taskId, - status: result.status, - progress: result.progress, - videoUrl: result.videoUrl, - outputPath, - error: result.error, - }; - } - - return { success: false, error: `Unsupported provider: ${provider}` }; - } catch (error) { + const generator = await openVideoGenerator(job.provider, options.apiKey); + const { state, outputPath } = await settle(generator, job, { wait: options.wait ?? false, output: options.output }); return { - success: false, - error: `Status check failed: ${error instanceof Error ? error.message : String(error)}`, + success: true, + taskId: job.id, + status: state.status, + progress: state.progress, + videoUrl: state.videoUrl, + duration: state.durationSec, + outputPath, + error: state.error?.message, + errorKind: state.error?.kind, }; + } catch (error) { + return failure(error, "Status check failed"); } } // ============================================================================ -// Video Cancel (Runway) +// Video Cancel // ============================================================================ export interface VideoCancelOptions { taskId: string; + /** Default: runway. */ + provider?: string; + job?: VideoJob; apiKey?: string; } export interface VideoCancelResult { success: boolean; error?: string; + errorKind?: ProviderErrorKind; } export async function executeVideoCancel(options: VideoCancelOptions): Promise { - const { taskId, apiKey } = options; - + const job = options.job ?? jobFromTaskId(options.provider ?? "runway", options.taskId); try { - const key = apiKey || process.env.RUNWAY_API_SECRET; - if (!key) return { success: false, error: "RUNWAY_API_SECRET required" }; - - const runway = new RunwayProvider(); - await runway.initialize({ apiKey: key }); - - const success = await runway.cancelGeneration(taskId); - return { success }; + const generator = await openVideoGenerator(job.provider, options.apiKey); + if (!generator.cancelVideoJob) { + return { success: false, error: `${job.provider} has no cancel API.`, errorKind: "unsupported" }; + } + await generator.cancelVideoJob(job); + return { success: true }; } catch (error) { - return { - success: false, - error: `Cancel failed: ${error instanceof Error ? error.message : String(error)}`, - }; + return failure(error, "Cancel failed"); } } // ============================================================================ -// Video Extend (Kling / Veo) +// Video Extend // ============================================================================ export interface VideoExtendOptions { + /** The task ID of the video to extend (Veo: its operation name). */ videoId: string; - provider?: "kling" | "veo"; + /** Default: kling. */ + provider?: string; + job?: VideoJob; prompt?: string; duration?: number; negative?: string; @@ -668,105 +290,48 @@ export interface VideoExtendOptions { export interface VideoExtendResult { success: boolean; taskId?: string; + job?: VideoJob; status?: string; videoUrl?: string; duration?: number; outputPath?: string; error?: string; + errorKind?: ProviderErrorKind; } export async function executeVideoExtend(options: VideoExtendOptions): Promise { - const { - videoId, - provider = "kling", - prompt, - duration = 5, - negative, - veoModel = "3.1", - output, - wait = true, - apiKey, - } = options; - + const source = options.job ?? jobFromTaskId(options.provider ?? "kling", options.videoId); try { - if (provider === "kling") { - const key = apiKey || process.env.KLING_API_KEY; - if (!key) return { success: false, error: "KLING_API_KEY required" }; - - const kling = new KlingProvider(); - await kling.initialize({ apiKey: key }); - if (!kling.isConfigured()) return { success: false, error: "Invalid Kling API key format" }; - - const result = await kling.extendVideo(videoId, { - prompt, - negativePrompt: negative, - duration: String(duration) as "5" | "10", - }); - - if (result.status === "failed") - return { success: false, error: result.error || "Kling extension failed" }; - if (!wait) return { success: true, taskId: result.id, status: "processing" }; - - const finalResult = await kling.waitForExtendCompletion(result.id, () => {}, 600000); - if (finalResult.status !== "completed") - return { success: false, error: finalResult.error || "Kling extension failed" }; - - let outputPath: string | undefined; - if (output && finalResult.videoUrl) { - const buffer = await downloadVideo(finalResult.videoUrl, key); - outputPath = await saveOutput(output, buffer); - } - - return { - success: true, - taskId: result.id, - status: "completed", - videoUrl: finalResult.videoUrl, - duration: finalResult.duration, - outputPath, - }; - } else if (provider === "veo") { - const key = apiKey || process.env.GOOGLE_API_KEY; - if (!key) return { success: false, error: "GOOGLE_API_KEY required" }; - - const gemini = new GeminiProvider(); - await gemini.initialize({ apiKey: key }); - - const model = resolveVeoModel(veoModel); - - const result = await gemini.extendVideo(videoId, prompt, { - duration: duration as 4 | 6 | 8, - model: model, - }); - - if (result.status === "failed") - return { success: false, error: result.error || "Veo extension failed" }; - if (!wait) return { success: true, taskId: result.id, status: "processing" }; - - const finalResult = await gemini.waitForVideoCompletion(result.id, () => {}, 300000); - if (finalResult.status !== "completed") - return { success: false, error: finalResult.error || "Veo extension failed" }; - - let outputPath: string | undefined; - if (output && finalResult.videoUrl) { - const buffer = await downloadVideo(finalResult.videoUrl, key); - outputPath = await saveOutput(output, buffer); - } - + const generator = await openVideoGenerator(source.provider, options.apiKey); + const job = await generator.submitVideo({ + prompt: options.prompt ?? "", + model: source.provider === "veo" ? videoModelSpec("veo", { veoModel: options.veoModel ?? "3.1" })?.id : undefined, + durationSec: options.duration, + negativePrompt: options.negative, + from: { job: source, kind: "extend" }, + }); + if (options.wait === false) return { success: true, taskId: job.id, job, status: "processing" }; + const { state, outputPath } = await settle(generator, job, { wait: true, output: options.output }); + if (state.status !== "completed") { return { - success: true, - taskId: result.id, - status: "completed", - videoUrl: finalResult.videoUrl, - outputPath, + success: false, + taskId: job.id, + job, + status: state.status, + error: state.error?.message ?? `${source.provider} extension ${state.status}`, + errorKind: state.error?.kind, }; } - - return { success: false, error: `Unsupported provider: ${provider}` }; - } catch (error) { return { - success: false, - error: `Video extension failed: ${error instanceof Error ? error.message : String(error)}`, + success: true, + taskId: job.id, + job, + status: "completed", + videoUrl: state.videoUrl, + duration: state.durationSec, + outputPath, }; + } catch (error) { + return failure(error, "Video extension failed"); } } diff --git a/packages/cli/src/commands/generate/video-cancel.ts b/packages/cli/src/commands/generate/video-cancel.ts index 036c4cb2..0420f7e2 100644 --- a/packages/cli/src/commands/generate/video-cancel.ts +++ b/packages/cli/src/commands/generate/video-cancel.ts @@ -1,91 +1,44 @@ /** * @module generate/video-cancel - * @description `vibe generate video-cancel` (hidden) — cancel in-flight Grok - * or Runway video generation. Split out of `generate.ts` in v0.69 (Plan G - * Phase 2). + * @description `vibe generate video-cancel` (hidden) — cancel an in-flight + * video job on a provider with a cancel API (Runway, Seedance, Gemini Omni). */ import type { Command } from "commander"; import chalk from "chalk"; import ora from "ora"; -import { GrokProvider, RunwayProvider } from "@vibeframe/ai-providers"; -import { requireApiKey } from "../../utils/api-key.js"; -import { - isJsonMode, - outputSuccess, - exitWithError, - apiError, - usageError, -} from "../output.js"; +import { executeVideoCancel } from "../ai-video.js"; +import { isJsonMode, outputSuccess, exitWithError, providerFailure, usageError } from "../output.js"; + +const CANCEL_PROVIDERS = ["runway", "seedance", "omni"]; export function registerVideoCancelCommand(parent: Command): void { parent .command("video-cancel", { hidden: true }) - .description("Cancel video generation (Grok or Runway)") + .description("Cancel video generation (Runway, Seedance, Gemini Omni)") .argument("", "Task ID to cancel") - .option("-p, --provider ", "Provider: grok, runway", "grok") - .option("-k, --api-key ", "API key (or set XAI_API_KEY / RUNWAY_API_SECRET env)") + .option("-p, --provider ", `Provider: ${CANCEL_PROVIDERS.join(", ")}`, "runway") + .option("-k, --api-key ", "API key for the provider") .action(async (taskId: string, options) => { const startedAt = Date.now(); - try { - const provider = (options.provider || "grok").toLowerCase(); - - let success = false; - - if (provider === "grok") { - const apiKey = await requireApiKey("XAI_API_KEY", "xAI", options.apiKey); - - const spinner = ora("Cancelling generation...").start(); - const grok = new GrokProvider(); - await grok.initialize({ apiKey }); - success = await grok.cancelGeneration(taskId); - - if (success) { - spinner.succeed(chalk.green("Generation cancelled")); - if (isJsonMode()) { - outputSuccess({ - command: "generate video-cancel", - startedAt, - data: { taskId, provider: "grok", cancelled: true }, - }); - return; - } - } else { - spinner.fail("Failed to cancel generation"); - exitWithError(apiError("Failed to cancel generation", true)); - } - } else if (provider === "runway") { - const apiKey = await requireApiKey( - "RUNWAY_API_SECRET", - "Runway", - options.apiKey, - ); - - const spinner = ora("Cancelling generation...").start(); - const runway = new RunwayProvider(); - await runway.initialize({ apiKey }); - success = await runway.cancelGeneration(taskId); - - if (success) { - spinner.succeed(chalk.green("Generation cancelled")); - if (isJsonMode()) { - outputSuccess({ - command: "generate video-cancel", - startedAt, - data: { taskId, provider: "runway", cancelled: true }, - }); - return; - } - } else { - spinner.fail("Failed to cancel generation"); - exitWithError(apiError("Failed to cancel generation", true)); - } - } else { - exitWithError(usageError(`Invalid provider: ${provider}. Use grok or runway.`)); - } - } catch (error) { - const msg = error instanceof Error ? error.message : String(error); - exitWithError(apiError(`Failed to cancel: ${msg}`, true)); + const provider = String(options.provider || "runway").toLowerCase(); + if (!CANCEL_PROVIDERS.includes(provider)) { + exitWithError( + usageError( + `${provider} has no cancel API. Video cancel supports: ${CANCEL_PROVIDERS.join(", ")}.`, + "Kling and Grok jobs run to completion once accepted." + ) + ); + } + const spinner = isJsonMode() ? null : ora("Cancelling generation...").start(); + const result = await executeVideoCancel({ taskId, provider, apiKey: options.apiKey }); + if (!result.success) { + spinner?.fail("Failed to cancel generation"); + exitWithError(providerFailure(result.error ?? "Cancel failed", result.errorKind)); + } + spinner?.succeed(chalk.green("Generation cancelled")); + if (isJsonMode()) { + outputSuccess({ command: "generate video-cancel", startedAt, data: { taskId, provider, cancelled: true } }); } }); } diff --git a/packages/cli/src/commands/generate/video-extend.ts b/packages/cli/src/commands/generate/video-extend.ts index 7284decf..d0b47044 100644 --- a/packages/cli/src/commands/generate/video-extend.ts +++ b/packages/cli/src/commands/generate/video-extend.ts @@ -1,295 +1,126 @@ /** * @module generate/video-extend - * @description `vibe generate video-extend` (hidden) — extend video duration - * via Kling or Veo (Gemini). Split out of `generate.ts` in v0.69 (Plan G - * Phase 2). + * @description `vibe generate video-extend` (hidden) — extend a generated + * video from its last frame, on any provider that can (Kling, Grok classic, + * Gemini Omni, and Veo until 2026-10-22). */ import type { Command } from "commander"; -import { resolve } from "node:path"; -import { writeFile } from "node:fs/promises"; import chalk from "chalk"; import ora from "ora"; -import { - GeminiProvider, - KlingProvider, - findModel, - resolveVeoModel, -} from "@vibeframe/ai-providers"; -import { requireApiKey } from "../../utils/api-key.js"; +import { findModel } from "@vibeframe/ai-providers"; import { checkModelLifecycle } from "../../utils/model-lifecycle.js"; -import { - isJsonMode, - outputSuccess, - printWarnings, - exitWithError, - apiError, - authError, - usageError, -} from "../output.js"; +import { executeVideoExtend } from "../ai-video.js"; +import { createAndWriteJobRecord } from "../_shared/status-jobs.js"; +import { isJsonMode, outputSuccess, printWarnings, exitWithError, providerFailure, usageError } from "../output.js"; import { validateOutputPath } from "../validate.js"; -import { downloadVideo } from "../ai-helpers.js"; + +const EXTEND_PROVIDERS = ["kling", "grok", "omni", "veo"]; export function registerVideoExtendCommand(parent: Command): void { parent .command("video-extend", { hidden: true }) - .description("Extend video duration (Kling by video ID, Veo by operation name)") - .argument("", "Kling video ID or Veo operation name") - .option("-p, --provider ", "Provider: kling, veo", "kling") - .option("-k, --api-key ", "API key (KLING_API_KEY or GOOGLE_API_KEY)") + .description("Extend a generated video from its last frame (Kling, Grok, Gemini Omni, Veo)") + .argument("", "Task ID the generation returned (Veo: its operation name)") + .option("-p, --provider ", `Provider: ${EXTEND_PROVIDERS.join(", ")}`, "kling") + .option("-k, --api-key ", "API key for the provider") .option("-o, --output ", "Output file path") .option("--prompt ", "Continuation prompt") - .option("-d, --duration ", "Duration: 5 or 10 (Kling), 4/6/8 (Veo)", "5") + .option("-d, --duration ", "Seconds to add: 5 or 10 (Kling), 2-10 (Grok), 4/6/8 (Veo); Omni picks its own", "5") .option("--negative ", "Negative prompt (what to avoid, Kling only)") .option("--veo-model ", "Veo model: 3.1, 3.1-fast", "3.1") .option("--no-wait", "Start extension and return task ID without waiting") .option("--dry-run", "Preview parameters without executing") .action(async (id: string, options) => { const startedAt = Date.now(); - try { - const provider = (options.provider || "kling").toLowerCase(); - if (options.output) { - validateOutputPath(options.output); - } - - let lifecycleWarnings: string[] = []; - if (provider === "veo") { - try { - resolveVeoModel(options.veoModel); - } catch (err) { - exitWithError(usageError(err instanceof Error ? err.message : String(err))); - } - lifecycleWarnings = checkModelLifecycle(findModel("veo", "video", options.veoModel)); - if (!options.dryRun) printWarnings(lifecycleWarnings); - } - - if (options.dryRun) { - outputSuccess({ - command: "generate video-extend", - startedAt, - dryRun: true, - warnings: lifecycleWarnings, - data: { - params: { - id, - provider, - prompt: options.prompt, - duration: options.duration, - negative: options.negative, - veoModel: options.veoModel, - }, - }, - }); - return; - } - - if (provider === "kling") { - const apiKey = await requireApiKey("KLING_API_KEY", "Kling", options.apiKey); - - const spinner = ora("Initializing Kling AI...").start(); - - const kling = new KlingProvider(); - await kling.initialize({ apiKey }); - - if (!kling.isConfigured()) { - spinner.fail("Invalid API key format"); - exitWithError(authError("KLING_API_KEY", "Kling")); - } - - spinner.text = "Starting video extension..."; - - const result = await kling.extendVideo(id, { - prompt: options.prompt, - negativePrompt: options.negative, - duration: options.duration as "5" | "10", - }); - - if (result.status === "failed") { - spinner.fail(result.error || "Failed to start extension"); - exitWithError(apiError(result.error || "Failed to start extension", true)); - } - - console.log(); - console.log(chalk.bold.cyan("Video Extension Started")); - console.log(chalk.dim("─".repeat(60))); - console.log(`Provider: Kling`); - console.log(`Task ID: ${chalk.bold(result.id)}`); - - if (!options.wait) { - spinner.succeed(chalk.green("Extension started")); - console.log(); - console.log(chalk.dim("Check status with:")); - console.log(chalk.dim(` vibe generate video-status ${result.id} -p kling`)); - console.log(); - return; - } - - spinner.text = "Extending video (this may take 2-5 minutes)..."; - - const finalResult = await kling.waitForExtendCompletion( - result.id, - (status) => { - spinner.text = `Extending video... ${status.status}`; - }, - 600000, - ); - - if (finalResult.status !== "completed") { - spinner.fail(finalResult.error || "Extension failed"); - exitWithError(apiError(finalResult.error || "Extension failed", true)); - } - - spinner.succeed(chalk.green("Video extended")); - - if (isJsonMode()) { - let outputPath: string | undefined; - if (options.output && finalResult.videoUrl) { - const buffer = await downloadVideo(finalResult.videoUrl, apiKey); - outputPath = resolve(process.cwd(), options.output); - await writeFile(outputPath, buffer); - } - outputSuccess({ - command: "generate video-extend", - startedAt, - data: { - provider: "kling", - taskId: result.id, - videoUrl: finalResult.videoUrl, - duration: finalResult.duration, - outputPath, - }, - }); - return; - } - - console.log(); - if (finalResult.videoUrl) { - console.log(`Video URL: ${finalResult.videoUrl}`); - } - if (finalResult.duration) { - console.log(`Duration: ${finalResult.duration}s`); - } - console.log(); - - if (options.output && finalResult.videoUrl) { - const downloadSpinner = ora("Downloading video...").start(); - try { - const buffer = await downloadVideo(finalResult.videoUrl, apiKey); - const outputPath = resolve(process.cwd(), options.output); - await writeFile(outputPath, buffer); - downloadSpinner.succeed(chalk.green(`Saved to: ${outputPath}`)); - } catch (err) { - downloadSpinner.fail( - chalk.red( - `Failed to download video: ${err instanceof Error ? err.message : err}`, - ), - ); - } - } - } else if (provider === "veo") { - const apiKey = await requireApiKey("GOOGLE_API_KEY", "Google", options.apiKey); - - const spinner = ora("Initializing Veo...").start(); - - const gemini = new GeminiProvider(); - await gemini.initialize({ apiKey }); - - const veoModel = resolveVeoModel(options.veoModel); - - spinner.text = "Starting video extension..."; - - const result = await gemini.extendVideo(id, options.prompt, { - duration: parseInt(options.duration) as 4 | 6 | 8, - model: veoModel, - }); - - if (result.status === "failed") { - spinner.fail(result.error || "Failed to start extension"); - exitWithError(apiError(result.error || "Failed to start extension", true)); - } - - console.log(); - console.log(chalk.bold.cyan("Veo Video Extension Started")); - console.log(chalk.dim("─".repeat(60))); - console.log(`Provider: Veo`); - console.log(`Operation: ${chalk.bold(result.id)}`); - - if (!options.wait) { - spinner.succeed(chalk.green("Extension started")); - console.log(); - console.log(chalk.dim("Check status or wait with:")); - console.log(chalk.dim(` vibe generate video-extend ${result.id} -p veo`)); - console.log(); - return; - } - - spinner.text = "Extending video (this may take 1-3 minutes)..."; - const finalResult = await gemini.waitForVideoCompletion( - result.id, - (status) => { - spinner.text = `Extending video... ${status.status}`; - }, - 300000, - ); - - if (finalResult.status !== "completed") { - spinner.fail(finalResult.error || "Extension failed"); - exitWithError(apiError(finalResult.error || "Extension failed", true)); - } - - spinner.succeed(chalk.green("Video extended")); + const provider = String(options.provider || "kling").toLowerCase(); + if (!EXTEND_PROVIDERS.includes(provider)) { + exitWithError(usageError(`Invalid provider: ${provider}. Video extend supports: ${EXTEND_PROVIDERS.join(", ")}`)); + } + if (options.output) validateOutputPath(options.output); - if (isJsonMode()) { - let outputPath: string | undefined; - if (options.output && finalResult.videoUrl) { - const buffer = await downloadVideo(finalResult.videoUrl, apiKey); - outputPath = resolve(process.cwd(), options.output); - await writeFile(outputPath, buffer); - } - outputSuccess({ - command: "generate video-extend", - startedAt, - warnings: lifecycleWarnings, - data: { - provider: "veo", - taskId: result.id, - videoUrl: finalResult.videoUrl, - duration: finalResult.duration, - outputPath, - }, - }); - return; - } + const veoSpec = provider === "veo" ? findModel("veo", "video", options.veoModel) : undefined; + if (provider === "veo" && !veoSpec) { + exitWithError(usageError(`Unknown Veo model "${options.veoModel}". Valid: 3.1, 3.1-fast.`)); + } + const lifecycleWarnings = checkModelLifecycle(veoSpec); + const params = { + id, + provider, + prompt: options.prompt, + duration: options.duration, + negative: options.negative, + veoModel: provider === "veo" ? options.veoModel : undefined, + }; + + if (options.dryRun) { + outputSuccess({ command: "generate video-extend", startedAt, dryRun: true, warnings: lifecycleWarnings, data: { params } }); + return; + } + printWarnings(lifecycleWarnings); + + const spinner = isJsonMode() ? null : ora(`Extending ${provider} video (this may take a few minutes)...`).start(); + const result = await executeVideoExtend({ + videoId: id, + provider, + prompt: options.prompt, + duration: Number(options.duration), + negative: options.negative, + veoModel: options.veoModel, + output: options.output, + wait: options.wait, + apiKey: options.apiKey, + }); + + if (!result.success) { + spinner?.fail(result.error ?? "Extension failed"); + const task = result.taskId ? ` (${provider} task ${result.taskId})` : ""; + exitWithError(providerFailure(`${result.error ?? "Video extension failed"}${task}`, result.errorKind)); + } - console.log(); - if (finalResult.videoUrl) { - console.log(`Video URL: ${finalResult.videoUrl}`); - } - console.log(); + // A job left running gets a record, so `vibe status job` can poll, + // download, and cache it later. + const record = + result.status !== "completed" && result.taskId + ? await createAndWriteJobRecord({ + jobType: "generate-video", + provider, + providerTaskId: result.taskId, + providerJob: result.job, + status: "running", + command: "vibe generate video-extend", + prompt: options.prompt, + outputPath: options.output, + }) + : undefined; + + if (isJsonMode()) { + outputSuccess({ + command: "generate video-extend", + startedAt, + warnings: lifecycleWarnings, + data: { + provider, + jobId: record?.id, + statusCommand: record?.retryWith[0], + taskId: result.taskId, + status: result.status, + videoUrl: result.videoUrl, + duration: result.duration, + outputPath: result.outputPath, + }, + }); + return; + } - if (options.output && finalResult.videoUrl) { - const downloadSpinner = ora("Downloading video...").start(); - try { - const buffer = await downloadVideo(finalResult.videoUrl, apiKey); - const outputPath = resolve(process.cwd(), options.output); - await writeFile(outputPath, buffer); - downloadSpinner.succeed(chalk.green(`Saved to: ${outputPath}`)); - } catch (err) { - downloadSpinner.fail( - chalk.red( - `Failed to download video: ${err instanceof Error ? err.message : err}`, - ), - ); - } - } - } else { - exitWithError( - usageError(`Invalid provider: ${provider}. Video extend supports: kling, veo`), - ); - } - } catch (error) { - const msg = error instanceof Error ? error.message : String(error); - exitWithError(apiError(`Video extension failed: ${msg}`, true)); + if (record) { + spinner?.succeed(chalk.green(`Extension started: ${provider} task ${result.taskId}`)); + console.log(chalk.dim(` Check it with: ${record.retryWith[0]}`)); + return; } + spinner?.succeed(chalk.green("Video extended")); + if (result.videoUrl) console.log(`Video URL: ${result.videoUrl}`); + if (result.duration) console.log(`Duration: ${result.duration}s`); + if (result.outputPath) console.log(chalk.green(`Saved to: ${result.outputPath}`)); }); } diff --git a/packages/cli/src/commands/output.ts b/packages/cli/src/commands/output.ts index d85a574f..bc839536 100644 --- a/packages/cli/src/commands/output.ts +++ b/packages/cli/src/commands/output.ts @@ -30,6 +30,8 @@ export interface StructuredError { retryWith?: string[]; recoverable?: boolean; retryable: boolean; + /** The provider error kind (`auth`, `quota`, `likeness`, ...) when a provider call failed. */ + errorKind?: string; /** * Structured context carried alongside the failure, so a refusal can hand * back the evidence it refused on instead of forcing a second call. Used by @@ -97,6 +99,21 @@ export function apiError(msg: string, retryable = false): StructuredError { return { success: false, error: msg, code: "API_ERROR", exitCode: ExitCode.API_ERROR, suggestion: retryable ? "Retry the command." : undefined, retryable }; } +/** + * The structured error for a failed provider call, chosen by its error kind + * (`ProviderError.kind`) instead of by matching provider wording. + */ +export function providerFailure(message: string, kind?: string): StructuredError { + if (kind === "auth") { + return { success: false, error: message, code: "AUTH_ERROR", exitCode: ExitCode.AUTH, retryable: false, errorKind: kind }; + } + if (kind === "invalid-request" || kind === "unsupported" || kind === "model-retired") { + return { ...usageError(message), errorKind: kind }; + } + const retryable = kind === "rate-limit" || kind === "provider" || kind === "network" || kind === "timeout"; + return { ...apiError(message, retryable), ...(kind ? { errorKind: kind } : {}) }; +} + export function notFoundError(path: string): StructuredError { return { success: false, error: `File not found: ${path}`, code: "NOT_FOUND", exitCode: ExitCode.NOT_FOUND, retryable: false }; } diff --git a/packages/cli/src/testing/fake-provider-network.ts b/packages/cli/src/testing/fake-provider-network.ts index 343f3447..fd836f74 100644 --- a/packages/cli/src/testing/fake-provider-network.ts +++ b/packages/cli/src/testing/fake-provider-network.ts @@ -18,7 +18,8 @@ export * from "@vibeframe/ai-providers/testing"; /** * Default happy-path routes for the providers a project build uses: * ElevenLabs (narration, music), OpenAI images, ImgBB uploads, fal - * (Seedance queue protocol), Runway, and a CDN for media downloads. + * (Seedance queue protocol and storage uploads), Runway (submit and + * polling), and a CDN for media downloads. */ export function createFakeProviderNetwork(): FakeProviderNetwork { const net = new FakeProviderNetwork(); @@ -44,7 +45,15 @@ export function createFakeProviderNetwork(): FakeProviderNetwork { "x-fal-request-id": req.path.split("/").at(-1) ?? "", }) ) + // fal storage: local frames are uploaded before a Seedance submit. + .on("POST", "rest.fal.ai", /^\/storage\/upload\/initiate$/, () => + jsonResponse({ upload_url: "https://fake.media/fal-upload", file_url: "https://fake.media/fal-frame.png" }) + ) + .on("PUT", "fake.media", /^\/fal-upload$/, () => new Response(null, { status: 200 })) .on("POST", "api.dev.runwayml.com", /^\/v1\/(image|text)_to_video$/, () => jsonResponse({ id: `runway-${++runwayTasks}` }) + ) + .on("GET", "api.dev.runwayml.com", /^\/v1\/tasks\/[^/]+$/, (req) => + jsonResponse({ id: req.path.split("/").at(-1), status: "SUCCEEDED", output: ["https://fake.media/runway.mp4"] }) ); } diff --git a/packages/cli/src/tools/manifest/generate.ts b/packages/cli/src/tools/manifest/generate.ts index 208eb319..a84c4c11 100644 --- a/packages/cli/src/tools/manifest/generate.ts +++ b/packages/cli/src/tools/manifest/generate.ts @@ -331,19 +331,19 @@ export const generateVideoTool = defineTool({ }), async execute(args, ctx) { const result = await executeVideoGenerate(args); - if (!result.success) return { success: false, error: result.error ?? "Video gen failed" }; + if (!result.success) { + // Keep the job ID: a job that failed or timed out after submission may + // still have been billed, or may still finish. + const taskNote = result.taskId ? ` (${result.provider} task ${result.taskId})` : ""; + return { success: false, error: `${result.error ?? "Video gen failed"}${taskNote}` }; + } let job: Awaited> | undefined; if (args.wait === false && result.taskId && result.status !== "completed") { job = await createAndWriteJobRecord({ jobType: "generate-video", provider: result.provider ?? args.provider ?? "unknown", providerTaskId: result.taskId, - providerTaskType: - result.provider === "kling" && args.image - ? "image2video" - : result.provider === "kling" - ? "text2video" - : undefined, + providerJob: result.job, status: "running", workingDirectory: ctx.workingDirectory, command: "generate_video wait=false", @@ -379,9 +379,14 @@ export const generateVideoCancelTool = defineTool({ cost: "free", title: "Cancel Video Generation", annotations: { readOnly: false, openWorld: true }, - description: "Cancel a Runway video generation task.", + description: + "Cancel a video generation task. Runway, Seedance, and Gemini Omni have cancel APIs; Kling and Grok do not.", schema: z.object({ taskId: z.string().describe("Task ID to cancel"), + provider: z + .enum(["runway", "seedance", "omni"]) + .optional() + .describe("Provider that owns the task (default: runway)"), }), async execute(args) { const result = await executeVideoCancel(args); @@ -403,10 +408,13 @@ export const generateVideoExtendTool = defineTool({ title: "Extend Generated Video", annotations: { readOnly: false, openWorld: true }, description: - "Extend video duration using Kling or Veo. Requires the video/operation ID from a previous generation.", + "Extend a generated video from its last frame. Kling, Grok (classic model), Gemini Omni, and Veo (shuts down 2026-10-22) can extend; pass the task ID the generation returned.", schema: z.object({ - videoId: z.string().describe("Video ID (Kling) or operation name (Veo)"), - provider: z.enum(["kling", "veo"]).optional().describe("Provider (default: kling)"), + videoId: z.string().describe("Task ID of the video to extend (Veo: its operation name)"), + provider: z + .enum(["kling", "grok", "omni", "veo"]) + .optional() + .describe("Provider that generated the video (default: kling)"), prompt: z.string().optional().describe("Continuation prompt"), duration: z.number().optional().describe("Duration in seconds"), negative: z.string().optional().describe("Negative prompt (Kling)"), From 1c20c81134275490095c7587d6481bc0a0e9f7d9 Mon Sep 17 00:00:00 2001 From: Kiyeon Jeon Date: Sun, 4 Oct 2026 23:58:50 +0900 Subject: [PATCH 2/2] fix: never fetch remote image URLs for inline-only video providers `videoImageInput` fetched any http(s) URL to inline it for Omni and Veo. Through the MCP tool, that let a caller make the CLI read an internal URL (a cloud metadata endpoint, say) and forward its contents to a provider. Inline-only providers now take local files or data URIs and refuse URLs with a usage error, as the old Omni path did. Also updates the video-extend help assertion in the integration suite. Claude-Session: https://claude.ai/code/session_011S1BLTaFctZNQzGERpkCuA --- .../cli/src/commands/_shared/video-jobs.ts | 20 ++++++++++++------- .../cli/src/commands/ai-video.network.test.ts | 14 +++++++++++++ packages/cli/src/commands/ai.test.ts | 2 +- 3 files changed, 28 insertions(+), 8 deletions(-) diff --git a/packages/cli/src/commands/_shared/video-jobs.ts b/packages/cli/src/commands/_shared/video-jobs.ts index 286e3e57..7d39993a 100644 --- a/packages/cli/src/commands/_shared/video-jobs.ts +++ b/packages/cli/src/commands/_shared/video-jobs.ts @@ -90,15 +90,21 @@ export async function fileToUrlOrDataUri(input: string, fallbackMimeType: string /** * An input image in the form `generator` takes: uploaded to the configured * upload host for URL-only providers, inlined as a data URI otherwise. + * + * Remote URLs are never fetched here: an inline-only provider gets a usage + * error instead, so a tool call cannot make the CLI read an internal URL + * and forward its contents to a provider. */ export async function videoImageInput(generator: VideoGenerator, input: string): Promise { - if (generator.imageInput !== "url" || /^https?:/.test(input)) { - if (generator.imageInput === "data-uri" && /^https?:/.test(input)) { - const response = await fetch(input); - if (!response.ok) throw new Error(`Could not fetch image ${input}: HTTP ${response.status}`); - const type = response.headers.get("content-type") ?? mimeTypeForPath(input, "image/png"); - return `data:${type};base64,${Buffer.from(await response.arrayBuffer()).toString("base64")}`; - } + const isUrl = /^https?:/i.test(input); + if (generator.imageInput === "data-uri" && isUrl) { + throw new ProviderError({ + kind: "invalid-request", + provider: generator.id, + message: `${generator.id} takes images inline; pass a local file path or a data URI, not a URL.`, + }); + } + if (generator.imageInput !== "url" || isUrl) { return fileToUrlOrDataUri(input, "image/png"); } const buffer = input.startsWith("data:") diff --git a/packages/cli/src/commands/ai-video.network.test.ts b/packages/cli/src/commands/ai-video.network.test.ts index ba1f8c49..c50e0937 100644 --- a/packages/cli/src/commands/ai-video.network.test.ts +++ b/packages/cli/src/commands/ai-video.network.test.ts @@ -58,6 +58,20 @@ describe("video executors over the provider network", () => { expect(net.to("api.imgbb.com")).toHaveLength(1); }); + it("never fetches an image URL for an inline-only provider", async () => { + process.env.GOOGLE_API_KEY = "test-google"; + + const result = await executeVideoGenerate({ + prompt: "p", + provider: "omni", + image: "http://169.254.169.254/latest/meta-data/", + wait: false, + }); + + expect(result).toMatchObject({ success: false, errorKind: "invalid-request" }); + expect(net.requests).toHaveLength(0); + }); + it("polls a stored handle on the endpoint it was submitted to", async () => { net.on("GET", KLING, /\/image2video\/k-1$/, () => kling({ task_id: "k-1", task_status: "succeed", task_result: { videos: [{ id: "v-1", url: "https://fake.media/k.mp4", duration: "5" }] } }) diff --git a/packages/cli/src/commands/ai.test.ts b/packages/cli/src/commands/ai.test.ts index 1209c9a5..a43d3076 100644 --- a/packages/cli/src/commands/ai.test.ts +++ b/packages/cli/src/commands/ai.test.ts @@ -95,7 +95,7 @@ describe("CLI command groups", () => { encoding: "utf-8", }); - expect(output).toContain("Extend video duration"); + expect(output).toContain("Extend a generated video from its last frame"); expect(output).toContain("--output"); expect(output).toContain("--prompt"); expect(output).toContain("--duration");