From e497236e4cf709258b14e02c94e56109a85e79da Mon Sep 17 00:00:00 2001 From: Kiyeon Jeon Date: Mon, 5 Oct 2026 16:09:23 +0900 Subject: [PATCH] feat: put image providers on one contract and drop CLI provider branches Image generation and editing were duplicated per provider in five CLI places (generate image, the image executor, edit image, scene visuals, and two build helpers), each with its own fetch, error parsing, model mapping, and URL-vs-base64 handling. - `ImageGenerator` (ai-providers/src/image): `createImage(request)` returns image bytes; a request with input images is an edit. `maxEditImages` declares each provider's edit limit (OpenAI 16, Gemini 3 / Pro 14, Grok 1). `createImageGenerator(provider, key)` opens one by name. - OpenAI, Gemini, and Grok implement it over `providerRequest` with `ProviderError` kinds; unknown models are refused before any request, and the old `generateImage` / `editImage` methods wrap it. - Gemini sends its key as a header instead of `?key=` in the URL, and a safety block reads as moderation. Grok asks for base64, so nothing downloads expiring URLs. - The CLI's generate image, edit image, scene visuals, and build backdrops/keyframes all go through the contract (`executeImageGenerate`, `executeImageEdit`). `resolveCatalogModel` generalizes the video model resolver. Fixes: - OpenAI generation ignored the model and the aspect ratio; OpenAI edits ignored the model (and the spinner always said GPT Image 1.5). - `edit image -m` defaulted to the Gemini alias `flash` for every provider. - Grok backdrops ignored the build's image model. - `--count N` saved only the first image; all are saved now (out-2.png...). - Scene visuals wrote Gemini JPEG bytes into .png files. - `generate image -p runway` ran a Python script that no longer exists. - A missing image key surfaced as API_ERROR instead of API_KEY_MISSING. - `edit image` used a hardcoded Gemini model-ID map instead of the catalog. - The MCP build tool allowed only OpenAI images though builds support Gemini and Grok; DALL-E's `style` option is gone from the pipeline. Claude-Session: https://claude.ai/code/session_011S1BLTaFctZNQzGERpkCuA --- AGENTS.md | 10 +- docs/cli-reference.md | 22 +- .../ai-providers/src/gemini/GeminiProvider.ts | 419 +++-------- .../src/grok/GrokProvider.test.ts | 4 +- .../ai-providers/src/grok/GrokProvider.ts | 239 +++--- .../ai-providers/src/image-contracts.test.ts | 10 +- packages/ai-providers/src/image/contract.ts | 67 ++ .../src/image/image-contract.test.ts | 130 ++++ packages/ai-providers/src/image/index.ts | 2 + packages/ai-providers/src/image/registry.ts | 31 + packages/ai-providers/src/index.ts | 1 + .../src/openai-image/OpenAIImageProvider.ts | 257 +++---- packages/ai-providers/src/video/contract.ts | 4 +- packages/ai-providers/src/video/models.ts | 22 +- .../envelope-snapshots.test.ts.snap | 46 +- .../error-envelopes.test.ts.snap | 16 + .../cli/src/commands/_shared/image-jobs.ts | 72 ++ .../cli/src/commands/_shared/openai-image.ts | 68 +- .../src/commands/_shared/scene-build.test.ts | 63 +- .../cli/src/commands/_shared/scene-build.ts | 128 +--- packages/cli/src/commands/ai-image.ts | 298 +++----- packages/cli/src/commands/edit-cmd.ts | 219 ++---- .../cli/src/commands/error-envelopes.test.ts | 7 +- packages/cli/src/commands/generate/image.ts | 683 ++++-------------- packages/cli/src/commands/scene.ts | 83 +-- packages/cli/src/pipeline/executor.ts | 1 - packages/cli/src/tools/manifest/edit.ts | 11 +- packages/cli/src/tools/manifest/generate.ts | 9 +- packages/cli/src/tools/manifest/scene.ts | 6 +- 29 files changed, 1060 insertions(+), 1868 deletions(-) create mode 100644 packages/ai-providers/src/image/contract.ts create mode 100644 packages/ai-providers/src/image/image-contract.test.ts create mode 100644 packages/ai-providers/src/image/index.ts create mode 100644 packages/ai-providers/src/image/registry.ts create mode 100644 packages/cli/src/commands/_shared/image-jobs.ts diff --git a/AGENTS.md b/AGENTS.md index 9a2c1a33..67914c62 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -129,10 +129,12 @@ General expectation: After editing the catalog, run `pnpm gen:models` (MODELS.md is partly generated). - Video providers implement `VideoGenerator` - (`packages/ai-providers/src/video/`): HTTP goes through `providerRequest`, - failures are `ProviderError`s with a `kind`, and each provider passes - `describeVideoContract`. Callers branch on `error.kind`, never on provider - wording. + (`packages/ai-providers/src/video/`) and image providers implement + `ImageGenerator` (`src/image/`): HTTP goes through `providerRequest`, + failures are `ProviderError`s with a `kind`, and each provider passes its + contract suite. The CLI opens providers through `createVideoGenerator` / + `createImageGenerator` and branches on `error.kind`, never on provider + names or wording. ## Conventions diff --git a/docs/cli-reference.md b/docs/cli-reference.md index c5816f39..1544d2de 100644 --- a/docs/cli-reference.md +++ b/docs/cli-reference.md @@ -491,7 +491,7 @@ Cost tier: _not tagged_ #### `vibe generate image` -Generate image using AI (Gemini, OpenAI gpt-image, Grok, or Runway) +Generate image using AI (OpenAI GPT Image, Gemini, or Grok) Product surface: `public` @@ -500,15 +500,15 @@ Cost tier: `high` **Parameters:** - `prompt` _(string)_ - Image description prompt (interactive if omitted) -- `provider` _(string)_ _(openai \| gemini \| grok \| runway)_ - Provider: openai (default when OPENAI_API_KEY set), gemini, grok, runway -- `apiKey` _(string)_ - API key (or set env: OPENAI_API_KEY, GOOGLE_API_KEY) -- `output` _(string)_ - Output file path (downloads image) -- `size` _(string)_ _(default: `"1024x1024"`)_ - Image size (openai: 1024x1024, 1536x1024, 1024x1536) -- `ratio` _(string)_ _(default: `"1:1"`)_ - Aspect ratio (gemini: 1:1, 1:4, 1:8, 4:1, 8:1, 16:9, 9:16, 3:4, 4:3, etc.) -- `quality` _(string)_ _(standard \| hd)_ _(default: `"standard"`)_ - Quality: standard, hd (openai only) -- `style` _(string)_ _(vivid \| natural)_ _(default: `"vivid"`)_ - Style: vivid, natural (openai only) +- `provider` _(string)_ _(openai \| gemini \| grok)_ - Provider: openai (default when OPENAI_API_KEY set), gemini, grok +- `apiKey` _(string)_ - API key (or set env: OPENAI_API_KEY, GOOGLE_API_KEY, XAI_API_KEY) +- `output` _(string)_ - Output file path +- `ratio` _(string)_ _(default: `"1:1"`)_ - Aspect ratio: 1:1, 16:9, 9:16, 4:3, 3:4, ... (OpenAI maps it to its nearest size) +- `size` _(string)_ - Explicit OpenAI size: 1024x1024, 1536x1024, 1024x1536 (overrides --ratio) +- `quality` _(string)_ _(low \| medium \| high)_ - Quality: low, medium, high (OpenAI and Grok; standard/hd still accepted) +- `resolution` _(string)_ - Gemini 1K/2K/4K (2K and 4K need Pro) or Grok 1k/2k - `count` _(number)_ _(default: `1`)_ - Number of images to generate -- `model` _(string)_ - Model. Gemini: flash (default), lite, pro. OpenAI: 2.5 (default), flare, 2, 1.5. Grok: pro +- `model` _(string)_ - Model. OpenAI: 2.5 (default), flare, 2. Gemini: flash (default), lite, pro. Grok: pro - `dryRun` _(boolean)_ - Preview parameters without executing #### `vibe generate motion` @@ -777,7 +777,7 @@ Cost tier: `high` - `provider` _(string)_ _(gemini \| openai \| grok)_ _(default: `"gemini"`)_ - Provider: gemini (default), openai, grok - `apiKey` _(string)_ - API key (or set env variable) - `output` _(string)_ _(default: `"edited.png"`)_ - Output file path -- `model` _(string)_ _(default: `"flash"`)_ - Model: flash, lite, pro (Gemini); pro (Grok) +- `model` _(string)_ - Model: flash (default), lite, pro (Gemini); 2.5, flare, 2 (OpenAI); pro (Grok) - `ratio` _(string)_ - Output aspect ratio - `size` _(string)_ - Resolution: 1K, 2K, 4K (Gemini Pro only) - `dryRun` _(boolean)_ - Preview parameters without executing @@ -1256,7 +1256,7 @@ Cost tier: `free` - `insertInto` _(string)_ _(default: `"index.html"`)_ - Root composition file to update - `project` _(string)_ _(default: `"."`)_ - Project directory - `noStoryboard` _(boolean)_ - Do not sync STORYBOARD.md; insert this scene directly into the root composition only -- `imageProvider` _(string)_ _(gemini \| openai)_ _(default: `"gemini"`)_ - Image provider: gemini, openai +- `imageProvider` _(string)_ _(gemini \| openai \| grok)_ _(default: `"gemini"`)_ - Image provider: gemini, openai, grok - `tts` _(string)_ _(auto \| elevenlabs \| openai \| kokoro)_ _(default: `"auto"`)_ - TTS provider: auto, elevenlabs, openai, kokoro (default auto - ElevenLabs key > OpenAI key > Kokoro local) - `voice` _(string)_ - Voice id (ElevenLabs name/id, OpenAI voice like marin, or Kokoro id like af_heart) - `noAudio` _(boolean)_ - Skip TTS even when --narration is provided (useful for tests/agent dry runs) diff --git a/packages/ai-providers/src/gemini/GeminiProvider.ts b/packages/ai-providers/src/gemini/GeminiProvider.ts index a1a5dbb2..004ac3c8 100644 --- a/packages/ai-providers/src/gemini/GeminiProvider.ts +++ b/packages/ai-providers/src/gemini/GeminiProvider.ts @@ -20,6 +20,10 @@ import type { StoryboardSegment } from "../claude/ClaudeProvider.js"; import { analyzeContent as analyzeContentImpl } from "./gemini-storyboard.js"; import { errorMessage, fetchJson, sleep } from "../shared/http.js"; import { assertModelServed, defaultModel, findModel, listModels } from "../catalog/catalog.js"; +import { fromBase64, type ImageGenerator, type ImageRequest, type ImageResultSet } from "../image/contract.js"; +import { ProviderError } from "../shared/errors.js"; +import { providerRequest } from "../shared/http.js"; +import { resolveCatalogModel } from "../video/models.js"; /** * Gemini model types for image generation @@ -210,13 +214,14 @@ export interface GeminiImageAnalysisResult { * - Video: Veo 3.1 Fast / Veo 3.1 (text-to-video, image-to-video) * - Image: Nano Banana 2 (gemini-3.1-flash-image) / Nano Banana Pro (gemini-3-pro-image) */ -export class GeminiProvider implements AIProvider { +export class GeminiProvider implements AIProvider, ImageGenerator { id = "gemini"; name = "Google Gemini"; description = "AI video (Veo 3.1) and image (Nano Banana) generation"; capabilities: AICapability[] = ["text-to-video", "image-to-video", "text-to-image", "auto-edit", "vision"]; iconUrl = "/icons/gemini.svg"; isAvailable = true; + readonly imageProvider = "gemini"; private apiKey?: string; private baseUrl = "https://generativelanguage.googleapis.com/v1beta"; @@ -585,335 +590,135 @@ export class GeminiProvider implements AIProvider { return false; } - /** - * Resolve model alias to full model ID - */ - private resolveModel(model?: GeminiImageModel): string { - // Aliases and full IDs come from the model catalog; unknown values fall - // back to the default Nano Banana model. - return (findModel("gemini", "image", model) ?? defaultModel("gemini", "image")).id; - } + // ── ImageGenerator ──────────────────────────────────────────────────── - /** - * Check if model is Pro - */ - private isProModel(modelId: string): boolean { - return modelId.includes("pro"); + /** Nano Banana edits take up to 3 input images, Pro up to 14. */ + maxEditImages(model?: string): number { + return resolveCatalogModel("gemini", "image", model).id.includes("pro") ? 14 : 3; } - /** - * Generate images using Gemini (Nano Banana) - * Uses generateContent with responseModalities: ["TEXT", "IMAGE"] - */ - async generateImage( - prompt: string, - options: GeminiImageOptions = {} - ): Promise { + async createImage(request: ImageRequest): Promise { if (!this.apiKey) { - return { - success: false, - error: "Google API key not configured", - }; + throw new ProviderError({ kind: "auth", provider: this.imageProvider, message: "Google API key not configured" }); + } + const modelId = resolveCatalogModel(this.imageProvider, "image", request.model).id; + const isPro = modelId.includes("pro"); + const inputs = request.images ?? []; + if (inputs.length > this.maxEditImages(modelId)) { + throw new ProviderError({ + kind: "invalid-request", + provider: this.imageProvider, + message: `${modelId} edits take up to ${this.maxEditImages(modelId)} images, not ${inputs.length}.`, + }); } - try { - const modelId = this.resolveModel(options.model); - const isPro = this.isProModel(modelId); - - // Build image config - const imageConfig: Record = {}; - if (options.aspectRatio) { - imageConfig.aspectRatio = options.aspectRatio; - } else { - imageConfig.aspectRatio = "1:1"; - } - - // Resolution: 512px available for all models, 2K/4K for Pro only - if (options.resolution) { - if (options.resolution === "512px" || options.resolution === "1K" || isPro) { - imageConfig.imageSize = options.resolution; - } - } - - // Build generation config - const generationConfig: Record = { - responseModalities: ["TEXT", "IMAGE"], - imageConfig, - }; - - // Add thinking config if specified - if (options.thinkingConfig) { - generationConfig.thinkingConfig = options.thinkingConfig; - } - - // Build payload - const payload: Record = { - contents: [{ parts: [{ text: prompt }] }], - generationConfig, - }; - - // Add Google Search grounding - const is31Flash = modelId.startsWith("gemini-3.1-flash-image"); - if (options.imageSearchGrounding && is31Flash) { - // 3.1 Flash supports Image Search grounding - payload.tools = [{ googleSearch: { searchTypes: { webSearch: {}, imageSearch: {} } } }]; - } else if (options.grounding && isPro) { - payload.tools = [{ googleSearch: {} }]; - } + const imageConfig: Record = {}; + if (request.aspectRatio) imageConfig.aspectRatio = request.aspectRatio; + else if (inputs.length === 0) imageConfig.aspectRatio = "1:1"; + // 512px and 1K work on every model; 2K and 4K need Pro. + if (request.resolution && (request.resolution === "512px" || request.resolution === "1K" || isPro)) { + imageConfig.imageSize = request.resolution; + } + const generationConfig: Record = { responseModalities: ["TEXT", "IMAGE"] }; + if (Object.keys(imageConfig).length > 0) generationConfig.imageConfig = imageConfig; + if (request.providerOptions?.thinkingConfig) generationConfig.thinkingConfig = request.providerOptions.thinkingConfig; - const response = await fetch( - `${this.baseUrl}/models/${modelId}:generateContent?key=${this.apiKey}`, + const payload: Record = { + contents: [ { - method: "POST", - headers: { - "Content-Type": "application/json", - }, - body: JSON.stringify(payload), - } - ); - - if (!response.ok) { - const errorText = await response.text(); - let errorMessage: string; - try { - const errorData = JSON.parse(errorText); - errorMessage = errorData.error?.message || errorText; - } catch { - errorMessage = errorText; - } - return { - success: false, - error: `API error (${response.status}): ${errorMessage}`, - }; - } + parts: [ + { text: request.prompt }, + ...inputs.map((img) => ({ inlineData: { mimeType: img.mimeType, data: Buffer.from(img.bytes).toString("base64") } })), + ], + }, + ], + generationConfig, + }; + if (request.providerOptions?.imageSearchGrounding && modelId.startsWith("gemini-3.1-flash-image")) { + payload.tools = [{ googleSearch: { searchTypes: { webSearch: {}, imageSearch: {} } } }]; + } else if (request.providerOptions?.grounding && isPro) { + payload.tools = [{ googleSearch: {} }]; + } - const data = (await response.json()) as { - candidates?: Array<{ - content?: { - parts?: Array<{ - text?: string; - thought?: boolean; - inlineData?: { - mimeType: string; - data: string; - }; - }>; - }; - finishReason?: string; - }>; - promptFeedback?: { blockReason?: string }; - }; + // The key goes in a header so it never lands in a URL, log, or error message. + const response = await providerRequest(this.imageProvider, `${this.baseUrl}/models/${modelId}:generateContent`, { + method: "POST", + headers: { "Content-Type": "application/json", "x-goog-api-key": this.apiKey }, + body: JSON.stringify(payload), + }); + const data = (await response.json()) as { + candidates?: Array<{ + content?: { parts?: Array<{ text?: string; thought?: boolean; inlineData?: { mimeType: string; data: string } }> }; + finishReason?: string; + }>; + promptFeedback?: { blockReason?: string }; + }; - const parts = data.candidates?.[0]?.content?.parts; - if (!parts || parts.length === 0) { - const finishReason = data.candidates?.[0]?.finishReason; - const blockReason = data.promptFeedback?.blockReason; - const detail = blockReason + const parts = (data.candidates?.[0]?.content?.parts ?? []).filter((part) => !part.thought); + const images = parts + .filter((part) => part.inlineData) + .map((part) => ({ bytes: fromBase64(part.inlineData!.data), mimeType: part.inlineData!.mimeType })); + if (images.length === 0) { + const blockReason = data.promptFeedback?.blockReason; + const finishReason = data.candidates?.[0]?.finishReason; + throw new ProviderError({ + kind: blockReason || finishReason === "SAFETY" || finishReason === "IMAGE_SAFETY" ? "moderation" : "provider", + provider: this.imageProvider, + message: blockReason ? `Blocked by safety filter: ${blockReason}` - : finishReason - ? `Model returned no image (finishReason: ${finishReason})` - : "No content in response"; - return { - success: false, - error: detail, - }; - } - - // Extract images from parts (skip thought images from Pro model) - const images: Array<{ base64: string; mimeType: string }> = []; - let description: string | undefined; - - for (const part of parts) { - // Skip thought images (Pro model thinking process) - if (part.thought) continue; - - if (part.inlineData) { - images.push({ - base64: part.inlineData.data, - mimeType: part.inlineData.mimeType, - }); - } else if (part.text) { - description = part.text; - } - } - - if (images.length === 0) { - return { - success: false, - error: "No images in response", - }; - } - - return { - success: true, - images, - description, - model: modelId, - }; - } catch (error) { - return { - success: false, - error: error instanceof Error ? error.message : "Unknown error", - }; + : `Model returned no image${finishReason ? ` (finishReason: ${finishReason})` : ""}`, + }); } + return { images, model: modelId, text: parts.find((part) => part.text)?.text }; } + // ── Older interface, kept until every caller uses the contract ──────── + /** - * Edit image(s) using Gemini (Nano Banana) - * Provide input image(s) with a text prompt to edit/transform/compose + * Generate images using Gemini (Nano Banana). */ - async editImage( - imageBuffers: Buffer[], - prompt: string, - options: GeminiEditOptions = {} - ): Promise { - if (!this.apiKey) { - return { - success: false, - error: "Google API key not configured", - }; - } - - const modelId = this.resolveModel(options.model); - const isPro = this.isProModel(modelId); + async generateImage(prompt: string, options: GeminiImageOptions = {}): Promise { + return this.legacyImage(() => + this.createImage({ + prompt, + model: options.model, + aspectRatio: options.aspectRatio, + resolution: options.resolution, + providerOptions: { + grounding: options.grounding, + imageSearchGrounding: options.imageSearchGrounding, + thinkingConfig: options.thinkingConfig, + }, + }) + ); + } - // Validate image count - const maxImages = isPro ? 14 : 3; - if (imageBuffers.length > maxImages) { - return { - success: false, - error: `Too many input images. ${modelId} supports up to ${maxImages} images.`, - }; - } + /** + * Edit image(s) using Gemini (Nano Banana). + */ + async editImage(imageBuffers: Buffer[], prompt: string, options: GeminiEditOptions = {}): Promise { + return this.legacyImage(() => + this.createImage({ + prompt, + model: options.model, + aspectRatio: options.aspectRatio, + resolution: options.resolution, + images: imageBuffers.map((b) => ({ bytes: new Uint8Array(b), mimeType: "image/png" })), + }) + ); + } + private async legacyImage(run: () => Promise): Promise { try { - // Build parts: text prompt first, then images - const parts: Array> = [{ text: prompt }]; - - for (const buffer of imageBuffers) { - parts.push({ - inlineData: { - mimeType: "image/png", - data: buffer.toString("base64"), - }, - }); - } - - // Build image config - const imageConfig: Record = {}; - if (options.aspectRatio) { - imageConfig.aspectRatio = options.aspectRatio; - } - if (options.resolution && isPro) { - imageConfig.imageSize = options.resolution; - } - - // Build generation config - const generationConfig: Record = { - responseModalities: ["TEXT", "IMAGE"], - }; - if (Object.keys(imageConfig).length > 0) { - generationConfig.imageConfig = imageConfig; - } - - const payload = { - contents: [{ parts }], - generationConfig, - }; - - const response = await fetch( - `${this.baseUrl}/models/${modelId}:generateContent?key=${this.apiKey}`, - { - method: "POST", - headers: { - "Content-Type": "application/json", - }, - body: JSON.stringify(payload), - } - ); - - if (!response.ok) { - const errorText = await response.text(); - let errorMessage: string; - try { - const errorData = JSON.parse(errorText); - errorMessage = errorData.error?.message || errorText; - } catch { - errorMessage = errorText; - } - return { - success: false, - error: `API error (${response.status}): ${errorMessage}`, - }; - } - - const data = (await response.json()) as { - candidates?: Array<{ - content?: { - parts?: Array<{ - text?: string; - thought?: boolean; - inlineData?: { - mimeType: string; - data: string; - }; - }>; - }; - finishReason?: string; - }>; - promptFeedback?: { blockReason?: string }; - }; - - const responseParts = data.candidates?.[0]?.content?.parts; - if (!responseParts || responseParts.length === 0) { - const finishReason = data.candidates?.[0]?.finishReason; - const blockReason = data.promptFeedback?.blockReason; - const detail = blockReason - ? `Blocked by safety filter: ${blockReason}` - : finishReason - ? `Model returned no image (finishReason: ${finishReason})` - : "No content in response"; - return { - success: false, - error: detail, - }; - } - - // Extract images (skip thought images) - const images: Array<{ base64: string; mimeType: string }> = []; - let description: string | undefined; - - for (const part of responseParts) { - if (part.thought) continue; - - if (part.inlineData) { - images.push({ - base64: part.inlineData.data, - mimeType: part.inlineData.mimeType, - }); - } else if (part.text) { - description = part.text; - } - } - - if (images.length === 0) { - return { - success: false, - error: "No images in response", - }; - } - + const result = await run(); return { success: true, - images, - description, - model: modelId, + images: result.images.map((img) => ({ base64: Buffer.from(img.bytes).toString("base64"), mimeType: img.mimeType })), + description: result.text, + model: result.model, }; } catch (error) { - return { - success: false, - error: error instanceof Error ? error.message : "Unknown error", - }; + return { success: false, error: errorMessage(error) }; } } diff --git a/packages/ai-providers/src/grok/GrokProvider.test.ts b/packages/ai-providers/src/grok/GrokProvider.test.ts index f6283fcc..b8bd5491 100644 --- a/packages/ai-providers/src/grok/GrokProvider.test.ts +++ b/packages/ai-providers/src/grok/GrokProvider.test.ts @@ -50,7 +50,7 @@ describe("GrokProvider", () => { mockFetch.mockResolvedValueOnce({ ok: true, json: async () => ({ - data: [{ url: "https://imgen.x.ai/test.jpg" }], + data: [{ b64_json: "aW1n" }], }), }); @@ -69,6 +69,8 @@ describe("GrokProvider", () => { const body = JSON.parse(options.body); expect(body.prompt).toBe("a blue cat"); expect(body.model).toBe("grok-imagine-image"); + // Base64 back, so callers never download an expiring URL. + expect(body.response_format).toBe("b64_json"); }); it("should handle API error gracefully", async () => { diff --git a/packages/ai-providers/src/grok/GrokProvider.ts b/packages/ai-providers/src/grok/GrokProvider.ts index 65f1bb37..b7e3467c 100644 --- a/packages/ai-providers/src/grok/GrokProvider.ts +++ b/packages/ai-providers/src/grok/GrokProvider.ts @@ -8,6 +8,8 @@ import type { import type { ImageResult } from "../openai-image/OpenAIImageProvider.js"; import { defaultModel, findModel, modelAliases } from "../catalog/catalog.js"; import type { VideoGenerator, VideoJob, VideoJobState, VideoRequest } from "../video/contract.js"; +import { fromBase64, type ImageGenerator, type ImageRequest, type ImageResultSet } from "../image/contract.js"; +import { resolveCatalogModel } from "../video/models.js"; import { ProviderError, classifyProviderError, isProviderError } from "../shared/errors.js"; import { providerRequest } from "../shared/http.js"; import { resolveVideoModel } from "../video/models.js"; @@ -151,8 +153,9 @@ interface GrokStatusResponse { * xAI Grok Imagine provider for video generation * Supports text-to-video and image-to-video with native audio */ -export class GrokProvider implements AIProvider, VideoGenerator { +export class GrokProvider implements AIProvider, VideoGenerator, ImageGenerator { id = "grok"; + readonly imageProvider = "grok"; readonly imageInput = "either" as const; name = "xAI Grok Imagine"; description = "AI video generation with Grok Imagine (native audio, 1-15 sec)"; @@ -175,171 +178,99 @@ export class GrokProvider implements AIProvider, VideoGenerator { return !!this.apiKey; } - /** - * Generate image using Grok Imagine - */ - async generateImage( - prompt: string, - options: GrokImageOptions = {} - ): Promise { - if (!this.apiKey) { - return { - success: false, - error: "xAI API key not configured. Set XAI_API_KEY environment variable.", - }; - } + // ── ImageGenerator ──────────────────────────────────────────────────── - try { - const body: Record = { - model: options.model || "grok-imagine-image", - ...(options.quality ? { quality: options.quality } : {}), - prompt, - n: options.n || 1, - response_format: options.responseFormat || "url", - }; - - if (options.aspectRatio) { - body.aspect_ratio = options.aspectRatio; - } - - if (options.resolution) { - body.resolution = options.resolution; - } + /** Grok Imagine edits one image at a time. */ + maxEditImages(): number { + return 1; + } - const response = await fetch(`${this.baseUrl}/images/generations`, { - method: "POST", - headers: { - "Content-Type": "application/json", - Authorization: `Bearer ${this.apiKey}`, - }, - body: JSON.stringify(body), + async createImage(request: ImageRequest): Promise { + const apiKey = this.requireKey(); + const model = resolveCatalogModel(this.imageProvider, "image", request.model).id as GrokImageModel; + const inputs = request.images ?? []; + if (inputs.length > 1) { + throw new ProviderError({ + kind: "invalid-request", + provider: this.imageProvider, + message: `Grok Imagine edits one image at a time, not ${inputs.length}.`, }); - - if (!response.ok) { - const errorText = await response.text(); - let errorMessage = `API error: ${response.status}`; - try { - const errorJson = JSON.parse(errorText); - if (errorJson.error?.message) { - errorMessage = errorJson.error.message; - } - } catch { - if (errorText) { - errorMessage = errorText.substring(0, 200); - } - } - return { - success: false, - error: errorMessage, - }; - } - - const data = (await response.json()) as { - data: Array<{ - url?: string; - b64_json?: string; - }>; - }; - - return { - success: true, - images: data.data.map((img) => ({ - url: img.url, - base64: img.b64_json, - })), - }; - } catch (error) { - return { - success: false, - error: error instanceof Error ? error.message : "Unknown error", - }; } + // Grok takes quality low/medium/auto; the 2.0 model defaults to medium. + const quality = + request.quality === "low" || request.quality === "medium" + ? request.quality + : model === "grok-imagine-image-2.0" + ? "medium" + : undefined; + const body: Record = { + model, + prompt: request.prompt, + n: inputs.length ? 1 : (request.count ?? 1), + response_format: "b64_json", + ...(quality ? { quality } : {}), + ...(request.aspectRatio ? { aspect_ratio: request.aspectRatio } : {}), + }; + if (inputs.length) { + const [img] = inputs; + body.image = { url: `data:${img.mimeType};base64,${Buffer.from(img.bytes).toString("base64")}`, type: "image_url" }; + } else if (request.resolution) { + body.resolution = request.resolution.toLowerCase(); + } + const response = await providerRequest(this.imageProvider, `${this.baseUrl}/images/${inputs.length ? "edits" : "generations"}`, { + method: "POST", + headers: { "Content-Type": "application/json", Authorization: `Bearer ${apiKey}` }, + body: JSON.stringify(body), + }); + const data = (await response.json()) as { data?: Array<{ b64_json?: string }> }; + const images = (data.data ?? []) + .filter((img) => img.b64_json) + .map((img) => ({ bytes: fromBase64(img.b64_json!), mimeType: "image/jpeg" })); + if (images.length === 0) { + throw new ProviderError({ kind: "provider", provider: this.imageProvider, message: "Grok returned no image." }); + } + return { images, model }; } + // ── Older interface, kept until every caller uses the contract ──────── + /** - * Edit image using Grok Imagine - * Supports single image input with text instruction-based editing + * Generate image using Grok Imagine. */ - async editImage( - imageBuffer: Buffer, - prompt: string, - options: GrokEditOptions = {} - ): Promise { - if (!this.apiKey) { - return { - success: false, - error: "xAI API key not configured. Set XAI_API_KEY environment variable.", - }; - } - - try { - // Convert buffer to base64 data URI - const base64 = imageBuffer.toString("base64"); - const dataUri = `data:image/png;base64,${base64}`; - - const body: Record = { - model: options.model || "grok-imagine-image", - ...(options.quality ? { quality: options.quality } : {}), + async generateImage(prompt: string, options: GrokImageOptions = {}): Promise { + return this.legacyImage(() => + this.createImage({ prompt, - image: { - url: dataUri, - type: "image_url", - }, - n: 1, - response_format: options.responseFormat || "url", - }; - - if (options.aspectRatio) { - body.aspect_ratio = options.aspectRatio; - } - - const response = await fetch(`${this.baseUrl}/images/edits`, { - method: "POST", - headers: { - "Content-Type": "application/json", - Authorization: `Bearer ${this.apiKey}`, - }, - body: JSON.stringify(body), - }); + model: options.model, + count: options.n, + aspectRatio: options.aspectRatio, + resolution: options.resolution, + quality: options.quality === "auto" ? undefined : options.quality, + }) + ); + } - if (!response.ok) { - const errorText = await response.text(); - let errorMessage = `API error: ${response.status}`; - try { - const errorJson = JSON.parse(errorText); - if (errorJson.error?.message) { - errorMessage = errorJson.error.message; - } - } catch { - if (errorText) { - errorMessage = errorText.substring(0, 200); - } - } - return { - success: false, - error: errorMessage, - }; - } + /** + * Edit one image using Grok Imagine. + */ + async editImage(imageBuffer: Buffer, prompt: string, options: GrokEditOptions = {}): Promise { + return this.legacyImage(() => + this.createImage({ + prompt, + model: options.model, + aspectRatio: options.aspectRatio, + quality: options.quality === "auto" ? undefined : options.quality, + images: [{ bytes: new Uint8Array(imageBuffer), mimeType: "image/png" }], + }) + ); + } - const data = (await response.json()) as { - data: Array<{ - url?: string; - b64_json?: string; - }>; - }; - - return { - success: true, - images: data.data.map((img) => ({ - url: img.url, - base64: img.b64_json, - })), - }; + private async legacyImage(run: () => Promise): Promise { + try { + const result = await run(); + return { success: true, images: result.images.map((img) => ({ base64: Buffer.from(img.bytes).toString("base64") })) }; } catch (error) { - return { - success: false, - error: error instanceof Error ? error.message : "Unknown error", - }; + return { success: false, error: this.legacyMessage(error) }; } } diff --git a/packages/ai-providers/src/image-contracts.test.ts b/packages/ai-providers/src/image-contracts.test.ts index b2b0a864..3b76d0ac 100644 --- a/packages/ai-providers/src/image-contracts.test.ts +++ b/packages/ai-providers/src/image-contracts.test.ts @@ -25,7 +25,7 @@ describe("Gemini image contract", () => { content: { parts: [ { thought: true, inlineData: { mimeType: "image/png", data: "THOUGHT" } }, - { inlineData: { mimeType: "image/jpeg", data: "IMAGE" } }, + { inlineData: { mimeType: "image/jpeg", data: "aW1hZ2U=" } }, ], }, }, @@ -50,13 +50,13 @@ describe("Gemini image contract", () => { expect(body.generationConfig.imageConfig.aspectRatio).toBe("16:9"); expect(body.generationConfig).not.toHaveProperty("temperature"); // Thought images are skipped; the real image keeps its JPEG mime type. - expect(result.images).toEqual([{ base64: "IMAGE", mimeType: "image/jpeg" }]); + expect(result.images).toEqual([{ base64: "aW1hZ2U=", mimeType: "image/jpeg" }]); }); }); describe("OpenAI image contract", () => { it("defaults to gpt-image-2.5-sunburst with medium quality and no response_format", async () => { - const fetchMock = stubFetch({ data: [{ b64_json: "IMG", revised_prompt: "an apple" }] }); + const fetchMock = stubFetch({ data: [{ b64_json: "aW1n", revised_prompt: "an apple" }] }); const openai = new OpenAIImageProvider(); await openai.initialize({ apiKey: "key" }); @@ -67,11 +67,11 @@ describe("OpenAI image contract", () => { expect(body).toMatchObject({ model: "gpt-image-2.5-sunburst", quality: "medium", size: "1024x1024" }); expect(body).not.toHaveProperty("response_format"); expect(body).not.toHaveProperty("style"); - expect(result.images?.[0].base64).toBe("IMG"); + expect(result.images?.[0].base64).toBe("aW1n"); }); it("maps the legacy hd quality name to high", async () => { - const fetchMock = stubFetch({ data: [{ b64_json: "IMG" }] }); + const fetchMock = stubFetch({ data: [{ b64_json: "aW1n" }] }); const openai = new OpenAIImageProvider(); await openai.initialize({ apiKey: "key" }); diff --git a/packages/ai-providers/src/image/contract.ts b/packages/ai-providers/src/image/contract.ts new file mode 100644 index 00000000..cab4235b --- /dev/null +++ b/packages/ai-providers/src/image/contract.ts @@ -0,0 +1,67 @@ +/** + * The image contract every image provider implements. + * + * Image APIs answer in one call, so there is no job: a request goes in and + * images come back as bytes (providers that return URLs are asked for + * base64 instead), or a `ProviderError` is thrown. A request with input + * images is an edit. + */ + +export interface ImageInput { + bytes: Uint8Array; + mimeType: string; +} + +export interface ImageRequest { + prompt: string; + /** Catalog model ID or alias; the provider's default when omitted. */ + model?: string; + /** "16:9", "9:16", "1:1", ... Providers map it to their own size field. */ + aspectRatio?: string; + /** Output size class where the provider has one (Gemini "1K"/"2K"/"4K", Grok "1k"/"2k"). */ + resolution?: string; + quality?: "low" | "medium" | "high"; + /** Images to produce. Default 1. */ + count?: number; + /** Input images: present means edit or compose. */ + images?: ImageInput[]; + /** Edit mask (OpenAI only). */ + mask?: Uint8Array; + /** Provider-specific settings with no shared meaning (Gemini grounding). */ + providerOptions?: Record; +} + +export interface GeneratedImage extends ImageInput { + /** The prompt the provider actually used, when it rewrites prompts. */ + revisedPrompt?: string; +} + +export interface ImageResultSet { + images: GeneratedImage[]; + /** The catalog model ID that produced them. */ + model: string; + /** Text the model returned alongside the images (Gemini). */ + text?: string; +} + +export interface ImageGenerator { + /** Catalog provider id ("openai", "gemini", "grok"). */ + readonly imageProvider: string; + /** Most input images one edit accepts for `model`; 0 means the provider cannot edit. */ + maxEditImages(model?: string): number; + /** Generate, or edit when `request.images` is set. Throws `ProviderError`. */ + createImage(request: ImageRequest): Promise; +} + +/** Base64 to bytes. */ +export function fromBase64(data: string): Uint8Array { + return new Uint8Array(Buffer.from(data, "base64")); +} + +/** Map an aspect ratio to OpenAI's three GPT Image sizes. */ +export function openAiImageSize(aspectRatio?: string): "1024x1024" | "1536x1024" | "1024x1536" | undefined { + if (!aspectRatio) return undefined; + const [w, h] = aspectRatio.split(":").map(Number); + if (!(w > 0 && h > 0)) return undefined; + return w > h ? "1536x1024" : w < h ? "1024x1536" : "1024x1024"; +} diff --git a/packages/ai-providers/src/image/image-contract.test.ts b/packages/ai-providers/src/image/image-contract.test.ts new file mode 100644 index 00000000..9e8ee665 --- /dev/null +++ b/packages/ai-providers/src/image/image-contract.test.ts @@ -0,0 +1,130 @@ +/** + * The image contract for every image provider, over the fake provider + * network: bytes back, model validation and edit limits before any request, + * classified errors, and keys kept out of URLs. + */ +import { afterEach, beforeEach, describe, expect, it } from "vitest"; + +import { FAKE_PNG, FakeProviderNetwork, jsonResponse } from "../testing/fake-provider-network.js"; +import { ProviderError } from "../shared/errors.js"; +import { IMAGE_GENERATOR_PROVIDERS, createImageGenerator } from "./registry.js"; + +const B64 = FAKE_PNG.toString("base64"); +const ROUTES: Record Response; defaultModel: string; maxEdit: number }> = { + openai: { + host: "api.openai.com", + path: /^\/v1\/images\/(generations|edits)$/, + ok: () => jsonResponse({ data: [{ b64_json: B64 }] }), + defaultModel: "gpt-image-2.5-sunburst", + maxEdit: 16, + }, + gemini: { + host: "generativelanguage.googleapis.com", + path: /:generateContent$/, + ok: () => jsonResponse({ candidates: [{ content: { parts: [{ text: "here" }, { inlineData: { mimeType: "image/png", data: B64 } }] } }] }), + defaultModel: "gemini-3.1-flash-image", + maxEdit: 3, + }, + grok: { + host: "api.x.ai", + path: /^\/v1\/images\/(generations|edits)$/, + ok: () => jsonResponse({ data: [{ b64_json: B64 }] }), + defaultModel: "grok-imagine-image", + maxEdit: 1, + }, +}; + +let net: FakeProviderNetwork; +beforeEach(() => { + net = new FakeProviderNetwork(); + net.install(); +}); +afterEach(() => net.uninstall()); + +async function rejection(promise: Promise): Promise { + try { + await promise; + } catch (error) { + expect(error).toBeInstanceOf(ProviderError); + return error as ProviderError; + } + throw new Error("expected a ProviderError"); +} + +describe.each(IMAGE_GENERATOR_PROVIDERS)("%s meets the image contract", (provider) => { + const route = ROUTES[provider]; + + it("generates image bytes on its default model", async () => { + net.on("POST", route.host, route.path, route.ok); + const generator = await createImageGenerator(provider, "test-key"); + const result = await generator.createImage({ prompt: "a paper boat", aspectRatio: "16:9" }); + + expect(result.model).toBe(route.defaultModel); + expect(Buffer.from(result.images[0].bytes)).toEqual(FAKE_PNG); + expect(net.requests[0].url).not.toContain("key="); + }); + + it("edits with input images, up to its limit, refusing more before sending", async () => { + net.on("POST", route.host, route.path, route.ok); + const generator = await createImageGenerator(provider, "test-key"); + const image = { bytes: new Uint8Array(FAKE_PNG), mimeType: "image/png" }; + + await generator.createImage({ prompt: "make it night", images: [image] }); + expect(net.requests).toHaveLength(1); + expect(generator.maxEditImages()).toBe(route.maxEdit); + + const error = await rejection( + generator.createImage({ prompt: "x", images: Array(route.maxEdit + 1).fill(image) }) + ); + expect(error.kind).toBe("invalid-request"); + expect(net.requests).toHaveLength(1); + }); + + it("rejects an unknown model before sending", async () => { + const generator = await createImageGenerator(provider, "test-key"); + const error = await rejection(generator.createImage({ prompt: "x", model: "no-such-model" })); + expect(error.kind).toBe("invalid-request"); + expect(net.requests).toHaveLength(0); + }); + + it("classifies a bad key", async () => { + net.on("POST", route.host, route.path, () => jsonResponse({ error: { message: "Incorrect API key provided" } }, 401)); + const generator = await createImageGenerator(provider, "test-key"); + const error = await rejection(generator.createImage({ prompt: "x" })); + expect(error).toMatchObject({ kind: "auth", status: 401 }); + }); +}); + +describe("image provider specifics", () => { + it("OpenAI maps the aspect ratio to a GPT Image size", async () => { + net.on("POST", ROUTES.openai.host, ROUTES.openai.path, ROUTES.openai.ok); + const openai = await createImageGenerator("openai", "k"); + await openai.createImage({ prompt: "x", aspectRatio: "9:16", quality: "high" }); + expect(net.requests[0].body).toMatchObject({ size: "1024x1536", quality: "high", model: "gpt-image-2.5-sunburst" }); + }); + + it("Gemini sends the key as a header and reports a safety block as moderation", async () => { + net.on("POST", ROUTES.gemini.host, ROUTES.gemini.path, () => jsonResponse({ promptFeedback: { blockReason: "SAFETY" } })); + const gemini = await createImageGenerator("gemini", "k"); + const error = await rejection(gemini.createImage({ prompt: "x" })); + + expect(error.kind).toBe("moderation"); + expect(net.requests[0].headers["x-goog-api-key"]).toBe("k"); + }); + + it("Gemini Pro edits up to 14 images", async () => { + const gemini = await createImageGenerator("gemini", "k"); + expect(gemini.maxEditImages("pro")).toBe(14); + }); + + it("Grok asks for base64 and sends quality medium for the 2.0 model", async () => { + net.on("POST", ROUTES.grok.host, ROUTES.grok.path, ROUTES.grok.ok); + const grok = await createImageGenerator("grok", "k"); + await grok.createImage({ prompt: "x", model: "2.0", resolution: "2K" }); + expect(net.requests[0].body).toMatchObject({ model: "grok-imagine-image-2.0", quality: "medium", response_format: "b64_json", resolution: "2k" }); + }); + + it("refuses an unknown provider", async () => { + await expect(createImageGenerator("midjourney", "k")).rejects.toMatchObject({ kind: "invalid-request" }); + }); +}); diff --git a/packages/ai-providers/src/image/index.ts b/packages/ai-providers/src/image/index.ts new file mode 100644 index 00000000..fa4577bd --- /dev/null +++ b/packages/ai-providers/src/image/index.ts @@ -0,0 +1,2 @@ +export * from "./contract.js"; +export * from "./registry.js"; diff --git a/packages/ai-providers/src/image/registry.ts b/packages/ai-providers/src/image/registry.ts new file mode 100644 index 00000000..09f19f99 --- /dev/null +++ b/packages/ai-providers/src/image/registry.ts @@ -0,0 +1,31 @@ +/** One way to get an image provider by its catalog id. */ + +import { GeminiProvider } from "../gemini/GeminiProvider.js"; +import { GrokProvider } from "../grok/GrokProvider.js"; +import { OpenAIImageProvider } from "../openai-image/OpenAIImageProvider.js"; +import { ProviderError } from "../shared/errors.js"; +import type { ImageGenerator } from "./contract.js"; + +export const IMAGE_GENERATOR_PROVIDERS = ["openai", "gemini", "grok"] as const; +export type ImageGeneratorProvider = (typeof IMAGE_GENERATOR_PROVIDERS)[number]; + +/** A ready image generator for `provider`. */ +export async function createImageGenerator(provider: string, apiKey: string): Promise { + const generator = + provider === "openai" + ? new OpenAIImageProvider() + : provider === "gemini" + ? new GeminiProvider() + : provider === "grok" + ? new GrokProvider() + : undefined; + if (!generator) { + throw new ProviderError({ + kind: "invalid-request", + provider, + message: `Unknown image provider "${provider}". Available: ${IMAGE_GENERATOR_PROVIDERS.join(", ")}.`, + }); + } + await generator.initialize({ apiKey }); + return generator; +} diff --git a/packages/ai-providers/src/index.ts b/packages/ai-providers/src/index.ts index 282906e1..f338efcf 100644 --- a/packages/ai-providers/src/index.ts +++ b/packages/ai-providers/src/index.ts @@ -74,6 +74,7 @@ export { type ModelStatus, } from "./catalog/index.js"; export * from "./video/index.js"; +export * from "./image/index.js"; export { VIDEO_GENERATOR_PROVIDERS, createVideoGenerator, type VideoGeneratorProvider } from "./video/registry.js"; export { OpenAIProvider, openaiProvider } from "./openai/index.js"; export { ClaudeProvider, claudeProvider } from "./claude/index.js"; diff --git a/packages/ai-providers/src/openai-image/OpenAIImageProvider.ts b/packages/ai-providers/src/openai-image/OpenAIImageProvider.ts index a100c0f0..84ffa25f 100644 --- a/packages/ai-providers/src/openai-image/OpenAIImageProvider.ts +++ b/packages/ai-providers/src/openai-image/OpenAIImageProvider.ts @@ -4,6 +4,10 @@ import type { ProviderConfig, } from "../interface/types.js"; import { defaultModel } from "../catalog/catalog.js"; +import { fromBase64, openAiImageSize, type ImageGenerator, type ImageRequest, type ImageResultSet } from "../image/contract.js"; +import { ProviderError, isProviderError } from "../shared/errors.js"; +import { providerRequest } from "../shared/http.js"; +import { resolveCatalogModel } from "../video/models.js"; /** * GPT Image model types @@ -75,18 +79,18 @@ export interface ImageEditOptions { /** Default text-to-image model. */ export const OPENAI_IMAGE_DEFAULT_MODEL = defaultModel("openai", "image").id as GPTImageModel; -const DEFAULT_MODEL = OPENAI_IMAGE_DEFAULT_MODEL; /** * OpenAI Image provider (GPT Image 1.5 / GPT Image 2 / DALL-E) */ -export class OpenAIImageProvider implements AIProvider { +export class OpenAIImageProvider implements AIProvider, ImageGenerator { id = "openai-image"; name = "OpenAI GPT Image"; description = "AI image generation with GPT Image 1.5 (default) and GPT Image 2 (opt-in)"; capabilities: AICapability[] = ["text-to-image", "background-removal", "image-editing"]; iconUrl = "/icons/openai.svg"; isAvailable = true; + readonly imageProvider = "openai"; private apiKey?: string; private baseUrl = "https://api.openai.com/v1"; @@ -106,94 +110,81 @@ export class OpenAIImageProvider implements AIProvider { * Generate images from text prompt * Uses GPT Image 2.5 Sunburst by default */ - async generateImage( - prompt: string, - options: ImageOptions = {} - ): Promise { - if (!this.apiKey) { - return { - success: false, - error: "OpenAI API key not configured", - }; - } + // ── ImageGenerator ──────────────────────────────────────────────────── - const model = options.model || DEFAULT_MODEL; + /** GPT Image edits take up to 16 input images. */ + maxEditImages(): number { + return 16; + } - try { - // Build request body based on model + async createImage(request: ImageRequest): Promise { + if (!this.apiKey) { + throw new ProviderError({ kind: "auth", provider: this.imageProvider, message: "OpenAI API key not configured" }); + } + const model = resolveCatalogModel(this.imageProvider, "image", request.model).id; + const size = openAiSize(request); + let response: Response; + if (request.images?.length) { + if (request.images.length > this.maxEditImages()) { + throw new ProviderError({ + kind: "invalid-request", + provider: this.imageProvider, + message: `GPT Image edits take up to ${this.maxEditImages()} images, not ${request.images.length}.`, + }); + } + const form = new FormData(); + form.append("model", model); + form.append("prompt", request.prompt); + for (const image of request.images) { + form.append("image[]", new Blob([new Uint8Array(image.bytes)], { type: image.mimeType }), "image.png"); + } + if (request.mask) form.append("mask", new Blob([new Uint8Array(request.mask)], { type: "image/png" }), "mask.png"); + if (request.quality) form.append("quality", request.quality); + if (size) form.append("size", size); + if (request.count) form.append("n", String(request.count)); + response = await providerRequest(this.imageProvider, `${this.baseUrl}/images/edits`, { + method: "POST", + headers: { Authorization: `Bearer ${this.apiKey}` }, + body: form, + }); + } else { const body: Record = { model, - prompt, - n: options.n || 1, - }; - - // GPT Image models do NOT support response_format. - // Quality values: low, medium, high, auto. - const qualityMap: Record = { - standard: "medium", - hd: "high", + prompt: request.prompt, + n: request.count ?? 1, + // GPT Image models take quality low/medium/high and no response_format. + quality: request.quality ?? "medium", }; - const quality = options.quality || "medium"; - body.quality = qualityMap[quality] || quality; - if (options.size && options.size !== "auto") { - body.size = options.size; - } - - const response = await fetch(`${this.baseUrl}/images/generations`, { + if (size) body.size = size; + response = await providerRequest(this.imageProvider, `${this.baseUrl}/images/generations`, { method: "POST", - headers: { - "Content-Type": "application/json", - Authorization: `Bearer ${this.apiKey}`, - }, + headers: { "Content-Type": "application/json", Authorization: `Bearer ${this.apiKey}` }, body: JSON.stringify(body), }); + } + const data = (await response.json()) as { data?: Array<{ b64_json?: string; revised_prompt?: string }> }; + const images = (data.data ?? []) + .filter((img) => img.b64_json) + .map((img) => ({ bytes: fromBase64(img.b64_json!), mimeType: "image/png", revisedPrompt: img.revised_prompt })); + if (images.length === 0) { + throw new ProviderError({ kind: "provider", provider: this.imageProvider, message: "OpenAI returned no image." }); + } + return { images, model }; + } - if (!response.ok) { - const errorText = await response.text(); - console.error("OpenAI Image API error:", errorText); - - // Parse error to get detailed message - let errorMessage = `API error: ${response.status}`; - try { - const errorJson = JSON.parse(errorText); - if (errorJson.error?.message) { - errorMessage = errorJson.error.message; - } - } catch { - // If not JSON, use the raw text - if (errorText) { - errorMessage = errorText.substring(0, 200); - } - } - - return { - success: false, - error: errorMessage, - }; - } - - const data = (await response.json()) as { - data: Array<{ - url?: string; - b64_json?: string; - revised_prompt?: string; - }>; - }; + // ── Older interface, kept until every caller uses the contract ──────── - return { - success: true, - images: data.data.map((img) => ({ - url: img.url, - base64: img.b64_json, - revisedPrompt: img.revised_prompt, - })), - }; - } catch (error) { - return { - success: false, - error: error instanceof Error ? error.message : "Unknown error", - }; - } + async generateImage(prompt: string, options: ImageOptions = {}): Promise { + const quality = options.quality === "standard" ? "medium" : options.quality === "hd" ? "high" : options.quality; + return this.legacy(() => + this.createImage({ + prompt, + model: options.model, + count: options.n, + quality, + providerOptions: options.size && options.size !== "auto" ? { size: options.size } : undefined, + }) + ); } /** @@ -248,96 +239,44 @@ export class OpenAIImageProvider implements AIProvider { } /** - * Edit images using GPT Image 1.5 - * Supports up to 16 input images with text instruction-based editing + * Edit images (up to 16 inputs) with a text instruction. */ - async editImage( - imageBuffers: Buffer[], - prompt: string, - options: ImageEditOptions = {} - ): Promise { - if (!this.apiKey) { - return { - success: false, - error: "OpenAI API key not configured", - }; - } + async editImage(imageBuffers: Buffer[], prompt: string, options: ImageEditOptions = {}): Promise { + return this.legacy(() => + this.createImage({ + prompt, + model: options.model, + quality: options.quality, + count: options.n, + mask: options.mask, + images: imageBuffers.map((b) => ({ bytes: new Uint8Array(b), mimeType: "image/png" })), + providerOptions: options.size ? { size: options.size } : undefined, + }) + ); + } + private async legacy(run: () => Promise): Promise { try { - const formData = new FormData(); - formData.append("model", options.model || DEFAULT_MODEL); - formData.append("prompt", prompt); - - // Add images (up to 16) - for (const buf of imageBuffers) { - const uint8Array = new Uint8Array(buf); - formData.append("image[]", new Blob([uint8Array], { type: "image/png" }), "image.png"); - } - - if (options.mask) { - const maskUint8 = new Uint8Array(options.mask); - formData.append("mask", new Blob([maskUint8], { type: "image/png" }), "mask.png"); - } - - if (options.quality) { - formData.append("quality", options.quality); - } - - if (options.size) { - formData.append("size", options.size); - } - - const response = await fetch(`${this.baseUrl}/images/edits`, { - method: "POST", - headers: { - Authorization: `Bearer ${this.apiKey}`, - }, - body: formData, - }); - - if (!response.ok) { - const errorText = await response.text(); - let errorMessage = `API error: ${response.status}`; - try { - const errorJson = JSON.parse(errorText); - if (errorJson.error?.message) { - errorMessage = errorJson.error.message; - } - } catch { - if (errorText) { - errorMessage = errorText.substring(0, 200); - } - } - return { - success: false, - error: errorMessage, - }; - } - - const data = (await response.json()) as { - data: Array<{ - b64_json?: string; - url?: string; - revised_prompt?: string; - }>; - }; - + const result = await run(); return { success: true, - images: data.data.map((img) => ({ - base64: img.b64_json, - url: img.url, - revisedPrompt: img.revised_prompt, + images: result.images.map((img) => ({ + base64: Buffer.from(img.bytes).toString("base64"), + revisedPrompt: img.revisedPrompt, })), }; } catch (error) { - return { - success: false, - error: error instanceof Error ? error.message : "Unknown error", - }; + return { success: false, error: isProviderError(error) || error instanceof Error ? error.message : "Unknown error" }; } } } export const openaiImageProvider = new OpenAIImageProvider(); + +/** An explicit `size` wins; otherwise the aspect ratio picks one of the three GPT Image sizes. */ +function openAiSize(request: ImageRequest): string | undefined { + const explicit = request.providerOptions?.size; + if (typeof explicit === "string" && explicit !== "auto") return explicit; + return openAiImageSize(request.aspectRatio); +} diff --git a/packages/ai-providers/src/video/contract.ts b/packages/ai-providers/src/video/contract.ts index 99d635bb..81bf9c58 100644 --- a/packages/ai-providers/src/video/contract.ts +++ b/packages/ai-providers/src/video/contract.ts @@ -68,13 +68,13 @@ export interface VideoJobState { } /** How a provider takes an input image: a public URL, an inline data URI, or either. */ -export type ImageInput = "url" | "data-uri" | "either"; +export type VideoImageInput = "url" | "data-uri" | "either"; export interface VideoGenerator { /** Provider registry id; matches `VideoJob.provider`. */ readonly id: string; /** How `VideoRequest.image` must be given. Callers upload or inline local files to match. */ - readonly imageInput: ImageInput; + readonly imageInput: VideoImageInput; /** Start a generation. Throws `ProviderError` when the provider refuses it. */ submitVideo(request: VideoRequest): Promise; /** Read a job's current state. Throws `ProviderError` only when the state cannot be read. */ diff --git a/packages/ai-providers/src/video/models.ts b/packages/ai-providers/src/video/models.ts index 7b051f5d..da69fbc8 100644 --- a/packages/ai-providers/src/video/models.ts +++ b/packages/ai-providers/src/video/models.ts @@ -1,23 +1,29 @@ -import { assertModelServed, findModel, listModels, type ModelSpec } from "../catalog/catalog.js"; +import { assertModelServed, findModel, listModels, type ModelKind, type ModelSpec } from "../catalog/catalog.js"; import { ProviderError } from "../shared/errors.js"; /** - * The catalog video model a request names (alias or ID; the provider's - * default when omitted). Throws `invalid-request` for an unknown model and + * The catalog model a request names (alias or ID; the provider's default + * when omitted). Throws `invalid-request` for an unknown model and * `model-retired` for one past its shutdown date, before any request. */ -export function resolveVideoModel(provider: string, model?: string): ModelSpec { - const spec = findModel(provider, "video", model); +export function resolveCatalogModel(provider: string, kind: ModelKind, model?: string): ModelSpec { + const spec = findModel(provider, kind, model); if (!spec) { - const valid = listModels({ provider, kind: "video" }) - .flatMap((m) => [m.id, ...(m.aliases ?? [])]) + const valid = listModels({ provider, kind }) + .filter((m) => m.status !== "deprecated" || !m.shutdown || Date.parse(`${m.shutdown}T00:00:00Z`) > Date.now()) + .flatMap((m) => [...(m.aliases ?? []), m.id]) .join(", "); throw new ProviderError({ kind: "invalid-request", provider, - message: `Unknown ${provider} video model "${model}". Valid: ${valid}.`, + message: `Unknown ${provider} ${kind} model "${model}". Valid: ${valid}.`, }); } assertModelServed(spec); return spec; } + +/** `resolveCatalogModel` for video. */ +export function resolveVideoModel(provider: string, model?: string): ModelSpec { + return resolveCatalogModel(provider, "video", model); +} diff --git a/packages/cli/src/commands/__snapshots__/envelope-snapshots.test.ts.snap b/packages/cli/src/commands/__snapshots__/envelope-snapshots.test.ts.snap index b5cd6937..9f5dd1e6 100644 --- a/packages/cli/src/commands/__snapshots__/envelope-snapshots.test.ts.snap +++ b/packages/cli/src/commands/__snapshots__/envelope-snapshots.test.ts.snap @@ -1092,8 +1092,7 @@ exports[`CLI --describe schemas (drift detection) > vibe edit image --describe 1 "type": "array", }, "model": { - "default": "flash", - "description": "Model: flash, lite, pro (Gemini); pro (Grok)", + "description": "Model: flash (default), lite, pro (Gemini); 2.5, flare, 2 (OpenAI); pro (Grok)", "type": "string", }, "output": { @@ -1727,12 +1726,12 @@ exports[`CLI --describe schemas (drift detection) > vibe edit upscale --describe exports[`CLI --describe schemas (drift detection) > vibe generate image --describe 1`] = ` { "cost": "high", - "description": "Generate image using AI (Gemini, OpenAI gpt-image, Grok, or Runway)", + "description": "Generate image using AI (OpenAI GPT Image, Gemini, or Grok)", "name": "generate.image", "parameters": { "properties": { "apiKey": { - "description": "API key (or set env: OPENAI_API_KEY, GOOGLE_API_KEY)", + "description": "API key (or set env: OPENAI_API_KEY, GOOGLE_API_KEY, XAI_API_KEY)", "type": "string", }, "count": { @@ -1745,11 +1744,11 @@ exports[`CLI --describe schemas (drift detection) > vibe generate image --descri "type": "boolean", }, "model": { - "description": "Model. Gemini: flash (default), lite, pro. OpenAI: 2.5 (default), flare, 2, 1.5. Grok: pro", + "description": "Model. OpenAI: 2.5 (default), flare, 2. Gemini: flash (default), lite, pro. Grok: pro", "type": "string", }, "output": { - "description": "Output file path (downloads image)", + "description": "Output file path", "type": "string", }, "prompt": { @@ -1757,41 +1756,34 @@ exports[`CLI --describe schemas (drift detection) > vibe generate image --descri "type": "string", }, "provider": { - "description": "Provider: openai (default when OPENAI_API_KEY set), gemini, grok, runway", + "description": "Provider: openai (default when OPENAI_API_KEY set), gemini, grok", "enum": [ "openai", "gemini", "grok", - "runway", ], "type": "string", }, "quality": { - "default": "standard", - "description": "Quality: standard, hd (openai only)", + "description": "Quality: low, medium, high (OpenAI and Grok; standard/hd still accepted)", "enum": [ - "standard", - "hd", + "low", + "medium", + "high", ], "type": "string", }, "ratio": { "default": "1:1", - "description": "Aspect ratio (gemini: 1:1, 1:4, 1:8, 4:1, 8:1, 16:9, 9:16, 3:4, 4:3, etc.)", + "description": "Aspect ratio: 1:1, 16:9, 9:16, 4:3, 3:4, ... (OpenAI maps it to its nearest size)", "type": "string", }, - "size": { - "default": "1024x1024", - "description": "Image size (openai: 1024x1024, 1536x1024, 1024x1536)", + "resolution": { + "description": "Gemini 1K/2K/4K (2K and 4K need Pro) or Grok 1k/2k", "type": "string", }, - "style": { - "default": "vivid", - "description": "Style: vivid, natural (openai only)", - "enum": [ - "vivid", - "natural", - ], + "size": { + "description": "Explicit OpenAI size: 1024x1024, 1536x1024, 1024x1536 (overrides --ratio)", "type": "string", }, }, @@ -3241,10 +3233,11 @@ exports[`CLI --describe schemas (drift detection) > vibe scene add --describe 1` }, "imageProvider": { "default": "gemini", - "description": "Image provider: gemini, openai", + "description": "Image provider: gemini, openai, grok", "enum": [ "gemini", "openai", + "grok", ], "type": "string", }, @@ -4368,12 +4361,11 @@ exports[`envelope shape (--dry-run --json) > generate image (low cost, OpenAI de "costUsd": 0.211, "data": { "params": { - "count": "1", + "count": 1, + "model": "gpt-image-2.5-sunburst", "prompt": "test prompt", "provider": "openai", - "quality": "standard", "ratio": "1:1", - "size": "1024x1024", }, }, "dryRun": true, diff --git a/packages/cli/src/commands/__snapshots__/error-envelopes.test.ts.snap b/packages/cli/src/commands/__snapshots__/error-envelopes.test.ts.snap index 848cdb9e..eae98ffb 100644 --- a/packages/cli/src/commands/__snapshots__/error-envelopes.test.ts.snap +++ b/packages/cli/src/commands/__snapshots__/error-envelopes.test.ts.snap @@ -58,3 +58,19 @@ exports[`structured error envelope (exitWithError → stderr) > generate image "success": false, } `; + +exports[`structured error envelope (exitWithError → stderr) > generate image — no API key, hermetic (API_KEY_MISSING) 1`] = ` +{ + "code": "API_KEY_MISSING", + "error": "OpenAI API key required.", + "exitCode": 4, + "message": "OpenAI API key required.", + "recoverable": false, + "retryWith": [ + "Set OPENAI_API_KEY in .env, or run: vibe setup. Check available features: vibe doctor --json. Get key: https://platform.openai.com/api-keys", + ], + "retryable": false, + "success": false, + "suggestion": "Set OPENAI_API_KEY in .env, or run: vibe setup. Check available features: vibe doctor --json. Get key: https://platform.openai.com/api-keys", +} +`; diff --git a/packages/cli/src/commands/_shared/image-jobs.ts b/packages/cli/src/commands/_shared/image-jobs.ts new file mode 100644 index 00000000..f0dc5171 --- /dev/null +++ b/packages/cli/src/commands/_shared/image-jobs.ts @@ -0,0 +1,72 @@ +/** + * The CLI side of the image contract: open an image provider by name with + * the user's key, read input images, and save results. Image commands, + * tools, scenes, and builds go through here instead of branching on + * provider names. + */ + +import { existsSync } from "node:fs"; +import { mkdir, readFile } from "node:fs/promises"; +import { dirname, resolve } from "node:path"; +import { ProviderError, createImageGenerator, findModel, type ImageGenerator, type ImageInput } from "@vibeframe/ai-providers"; +import { getConfiguredApiKey } from "../../utils/api-key.js"; +import { writeImageFile } from "../../utils/image-file.js"; +import { mimeTypeForPath } from "./video-jobs.js"; + +/** Env var holding each image provider's key. */ +export const IMAGE_PROVIDER_ENV: Readonly> = { + openai: "OPENAI_API_KEY", + gemini: "GOOGLE_API_KEY", + grok: "XAI_API_KEY", +}; + +/** Display names for spinners and messages. */ +export const IMAGE_PROVIDER_LABELS: Readonly> = { + openai: "OpenAI", + gemini: "Google Gemini", + grok: "xAI Grok", +}; + +/** An image generator for `provider`, with the given key or the configured one. */ +export async function openImageGenerator(provider: string, apiKey?: string): Promise { + const envVar = IMAGE_PROVIDER_ENV[provider]; + if (!envVar) { + throw new ProviderError({ + kind: "invalid-request", + provider, + message: `Unknown image provider "${provider}". Available: ${Object.keys(IMAGE_PROVIDER_ENV).join(", ")}.`, + }); + } + const key = await getConfiguredApiKey(envVar, apiKey); + if (!key) throw new ProviderError({ kind: "auth", provider, message: `${envVar} required for ${provider}.` }); + return createImageGenerator(provider, key); +} + +/** A local image file as contract input. */ +export async function readImageInput(path: string): Promise { + const absPath = resolve(process.cwd(), path); + if (!existsSync(absPath)) { + throw new ProviderError({ kind: "invalid-request", provider: "local", message: `Image not found: ${absPath}` }); + } + return { bytes: new Uint8Array(await readFile(absPath)), mimeType: mimeTypeForPath(path, "image/png") }; +} + +/** Save image bytes to `output` (relative to cwd), converting to the extension's format. */ +export async function saveImage(output: string, bytes: Uint8Array): Promise { + const outputPath = resolve(process.cwd(), output); + await mkdir(dirname(outputPath), { recursive: true }); + await writeImageFile(outputPath, Buffer.from(bytes)); + return outputPath; +} + +/** The catalog label for a model ID, for human output. */ +export function imageModelLabel(provider: string, modelId: string): string { + return findModel(provider, "image", modelId)?.label ?? modelId; +} + +/** CLI quality words to the contract's: `standard`/`hd` are the old OpenAI names. */ +export function contractQuality(quality?: string): "low" | "medium" | "high" | undefined { + if (quality === "standard") return "medium"; + if (quality === "hd") return "high"; + return quality === "low" || quality === "medium" || quality === "high" ? quality : undefined; +} diff --git a/packages/cli/src/commands/_shared/openai-image.ts b/packages/cli/src/commands/_shared/openai-image.ts index 30104288..74cb8182 100644 --- a/packages/cli/src/commands/_shared/openai-image.ts +++ b/packages/cli/src/commands/_shared/openai-image.ts @@ -1,40 +1,12 @@ /** * @module _shared/openai-image * - * Shared OpenAI `generateImage` helper for `vibe ai image` and - * `vibe generate image`. Both commands had their own near-identical - * provider initialisation + model-alias parsing + result construction. - * v0.52.0 added gpt-image-2 wiring to `ai-image.ts` only and - * `vibe generate image -m 2` silently fell back to gpt-image-1.5 — the - * duplication guarantees a recurrence next time the OpenAI image flow - * is touched. - * - * Each top-level command keeps its own CLI wiring and output formatting; - * this helper owns just the API call + label derivation. - * - * See issue #58. + * OpenAI image model alias resolution, used by image cost estimates. + * Generation itself goes through the image contract (`executeImageGenerate`). */ -import { defaultModel, findModel, OpenAIImageProvider } from "@vibeframe/ai-providers"; -import type { GPTImageModel, ImageOptions, ImageResult } from "@vibeframe/ai-providers"; - -/** Subset of CLI options consumed by the helper. */ -export interface OpenAIImageHelperOptions { - model?: string; - size?: string; - quality?: string; - style?: string; - /** Stringified count from commander; parsed inside the helper. */ - count?: string; -} - -export interface OpenAIImageHelperResult { - result: ImageResult; - /** Resolved model id passed to the API. */ - openaiModel: GPTImageModel; - /** Human-friendly label for spinner / success output. */ - modelLabel: string; -} +import { defaultModel, findModel } from "@vibeframe/ai-providers"; +import type { GPTImageModel } from "@vibeframe/ai-providers"; /** * Resolve the user-supplied model alias to the API id + display label, from @@ -49,35 +21,3 @@ export function resolveOpenAIImageModel(modelAlias?: string): { const model = findModel("openai", "image", modelAlias) ?? defaultModel("openai", "image"); return { openaiModel: model.id as GPTImageModel, modelLabel: model.label }; } - -/** - * Run an OpenAI image generation. Caller owns: api-key acquisition, - * spinner lifecycle, output formatting (JSON vs human), and file save. - */ -export async function executeOpenAIImageGenerate( - prompt: string, - options: OpenAIImageHelperOptions, - ctx: { apiKey: string }, -): Promise { - const provider = new OpenAIImageProvider(); - await provider.initialize({ apiKey: ctx.apiKey }); - - const { openaiModel, modelLabel } = resolveOpenAIImageModel(options.model); - - // commander hands back unconstrained strings for `--size` / `--quality` - // / `--style`. The provider's `ImageOptions` narrows these to specific - // unions ("1024x1024" | "1536x1024" | …, etc.). The original duplicated - // handlers were `any`-typed via commander's loose `OptionValues`, which - // is why this never tripped before the dedup; the helper makes the - // shape explicit, so we cast at the single call boundary instead of - // ANY-laundering through the call sites. - const result = await provider.generateImage(prompt, { - model: openaiModel, - size: options.size as ImageOptions["size"], - quality: options.quality as ImageOptions["quality"], - style: options.style as ImageOptions["style"], - n: options.count !== undefined ? parseInt(options.count, 10) : undefined, - }); - - return { result, openaiModel, modelLabel }; -} diff --git a/packages/cli/src/commands/_shared/scene-build.test.ts b/packages/cli/src/commands/_shared/scene-build.test.ts index cf1d2b11..f0fbbf2a 100644 --- a/packages/cli/src/commands/_shared/scene-build.test.ts +++ b/packages/cli/src/commands/_shared/scene-build.test.ts @@ -22,9 +22,7 @@ vi.mock("./tts-resolve.js", () => ({ vi.mock("@vibeframe/ai-providers", async (importOriginal) => ({ ...(await importOriginal()), - GeminiProvider: vi.fn(), - GrokProvider: vi.fn(), - OpenAIImageProvider: vi.fn(), + createImageGenerator: vi.fn(), })); // Stub only the Whisper network call; keep the pure path/read helpers real so @@ -51,7 +49,7 @@ vi.mock("../generate/music.js", () => ({ })); import { resolveTtsProvider } from "./tts-resolve.js"; -import { GeminiProvider, GrokProvider, OpenAIImageProvider } from "@vibeframe/ai-providers"; +import { createImageGenerator, type ImageGenerator } from "@vibeframe/ai-providers"; import { transcribeNarrationWords } from "./transcribe-narration.js"; import { executeSceneRender } from "./scene-render.js"; import { executeFootageAssemble } from "./footage-assemble.js"; @@ -126,35 +124,18 @@ beforeEach(() => { }), }); - vi.mocked(OpenAIImageProvider).mockImplementation( - () => + // Each image provider returns distinct bytes, so tests can tell which one ran. + const imageBytes: Record = { openai: [5, 6, 7, 8], gemini: [9, 10, 11, 12], grok: [13, 14, 15, 16] }; + vi.mocked(createImageGenerator).mockImplementation( + async (provider: string) => ({ - initialize: vi.fn().mockResolvedValue(undefined), - generateImage: vi.fn().mockResolvedValue({ - success: true, - images: [{ base64: Buffer.from([5, 6, 7, 8]).toString("base64") }], + imageProvider: provider, + maxEditImages: () => 3, + createImage: vi.fn().mockResolvedValue({ + images: [{ bytes: new Uint8Array(imageBytes[provider]), mimeType: "image/png" }], + model: `${provider}-model`, }), - }) as unknown as InstanceType - ); - vi.mocked(GeminiProvider).mockImplementation( - () => - ({ - initialize: vi.fn().mockResolvedValue(undefined), - generateImage: vi.fn().mockResolvedValue({ - success: true, - images: [{ base64: Buffer.from([9, 10, 11, 12]).toString("base64") }], - }), - }) as unknown as InstanceType - ); - vi.mocked(GrokProvider).mockImplementation( - () => - ({ - initialize: vi.fn().mockResolvedValue(undefined), - generateImage: vi.fn().mockResolvedValue({ - success: true, - images: [{ base64: Buffer.from([13, 14, 15, 16]).toString("base64") }], - }), - }) as unknown as InstanceType + }) as ImageGenerator ); vi.mocked(executeSceneRender).mockResolvedValue({ @@ -392,19 +373,19 @@ describe("executeSceneBuild", () => { expect(r.success).toBe(true); expect(r.beats[0].backdropStatus).toBe("generated"); - expect(OpenAIImageProvider).toHaveBeenCalled(); + expect(createImageGenerator).toHaveBeenCalledWith("openai", "test-key-from-dotenv"); }); it.each([ - ["gemini", GeminiProvider, [9, 10, 11, 12]], - ["grok", GrokProvider, [13, 14, 15, 16]], - ] as const)("dispatches backdrop generation with %s", async (imageProvider, Provider, bytes) => { + ["gemini", [9, 10, 11, 12]], + ["grok", [13, 14, 15, 16]], + ] as const)("dispatches backdrop generation with %s", async (imageProvider, bytes) => { const r = await executeSceneBuild({ projectDir, stage: "assets", imageProvider }); expect(r.success).toBe(true); expect(r.beats[0].backdropStatus).toBe("generated"); - expect(Provider).toHaveBeenCalled(); - expect(OpenAIImageProvider).not.toHaveBeenCalled(); + expect(createImageGenerator).toHaveBeenCalledWith(imageProvider, expect.any(String)); + expect(createImageGenerator).not.toHaveBeenCalledWith("openai", expect.anything()); expect(Array.from(readFileSync(join(projectDir, "assets", "backdrop-hook.png")))).toEqual( bytes ); @@ -456,7 +437,7 @@ describe("executeSceneBuild", () => { expect(r.beats[0].narrationStatus).toBe("skipped"); expect(r.beats[0].backdropStatus).toBe("skipped"); expect(resolveTtsProvider).not.toHaveBeenCalled(); - expect(OpenAIImageProvider).not.toHaveBeenCalled(); + expect(createImageGenerator).not.toHaveBeenCalled(); }); it("runs `--stage transcript` standalone over on-disk narration and reports `transcript-only`", async () => { @@ -546,7 +527,7 @@ asset: "assets/frame.png" expect(r.beats[0].backdropPath).toBe("assets/frame.png"); expect(r.beats[0].backdropSourcePath).toBe("assets/frame.png"); expect(resolveTtsProvider).not.toHaveBeenCalled(); - expect(OpenAIImageProvider).not.toHaveBeenCalled(); + expect(createImageGenerator).not.toHaveBeenCalled(); const report = readBuildReport(); expectBuildReportContract(report, { @@ -601,7 +582,7 @@ backdrop: "../outside.png" expect(r.beats[0].backdropError).toBe( 'Asset reference "../outside.png" must stay inside the project directory.' ); - expect(OpenAIImageProvider).not.toHaveBeenCalled(); + expect(createImageGenerator).not.toHaveBeenCalled(); const report = readBuildReport(); expectBuildReportContract(report, { @@ -830,7 +811,7 @@ backdrop: "This should not dispatch." `vibe storyboard revise ${projectDir} --from "" --dry-run --json`, ]); expect(resolveTtsProvider).not.toHaveBeenCalled(); - expect(OpenAIImageProvider).not.toHaveBeenCalled(); + expect(createImageGenerator).not.toHaveBeenCalled(); expect(executeSceneRender).not.toHaveBeenCalled(); const report = readBuildReport(); diff --git a/packages/cli/src/commands/_shared/scene-build.ts b/packages/cli/src/commands/_shared/scene-build.ts index 7a4135ed..797fde48 100644 --- a/packages/cli/src/commands/_shared/scene-build.ts +++ b/packages/cli/src/commands/_shared/scene-build.ts @@ -25,13 +25,8 @@ import { dirname, join, resolve } from "node:path"; import { config as loadDotenv } from "dotenv"; import { - GeminiProvider, - GrokProvider, - OPENAI_IMAGE_DEFAULT_MODEL, - OpenAIImageProvider, + createImageGenerator, estimateSeedanceVideoCostUsd, - type GPTImageModel, - type GeminiImageModel, type ImageOptions, type VideoJob, } from "@vibeframe/ai-providers"; @@ -104,6 +99,7 @@ import { augmentBackdropPrompt } from "./build-backdrop-prompt.js"; import { executeVideoGenerate } from "../ai-video.js"; import { executeMusic } from "../generate/music.js"; import { createAndWriteJobRecord, writeJobRecord, type JobRecord } from "./status-jobs.js"; +import { contractQuality } from "./image-jobs.js"; import { executeSceneRepair, type SceneRepairResult } from "./scene-repair.js"; import { resolveTtsProvider, TtsKeyMissingError, type TtsProviderName } from "./tts-resolve.js"; import { resolveSceneBuildMode, type SceneBuildMode } from "./scene-build-mode.js"; @@ -1930,11 +1926,6 @@ async function dispatchBackdrop(beat: Beat, ctx: BeatDispatchContext): Promise

{ - loadSceneBuildEnv(ctx.projectDir); - const keyInfo = imageProviderKeyInfo(ctx.imageProvider); - const apiKey = - (await getApiKeyFromConfig(keyInfo.configKey, { cwd: ctx.projectDir })) ?? - process.env[keyInfo.envVar] ?? - ""; - if (!apiKey) { - return { - success: false, - error: `${keyInfo.envVar} not set — cannot dispatch backdrop with ${ctx.imageProvider}`, - }; - } - - if (ctx.imageProvider === "openai") { - const provider = new OpenAIImageProvider(); - await provider.initialize({ apiKey }); - const result = await provider.generateImage(prompt, { - // CLI input is free-form; an alias the provider doesn't know fails the - // API call and surfaces as a beat-scoped backdrop/keyframe error. - model: (ctx.imageModel as GPTImageModel | undefined) ?? OPENAI_IMAGE_DEFAULT_MODEL, - size: ctx.imageSize, - quality: ctx.imageQuality, - }); - return imageBufferFromResult(result); - } - - if (ctx.imageProvider === "gemini") { - const provider = new GeminiProvider(); - await provider.initialize({ apiKey }); - const result = await provider.generateImage(prompt, { - model: (ctx.imageModel as GeminiImageModel | undefined) ?? "flash", - aspectRatio: ratio as "1:1" | "2:3" | "3:2" | "16:9", - }); - return imageBufferFromResult(result); - } - - const provider = new GrokProvider(); - await provider.initialize({ apiKey }); - const result = await provider.generateImage(prompt, { - aspectRatio: ratio, - n: 1, - }); - return imageBufferFromResult(result); + return buildImage(prompt, ctx, ratio); } /** * Generate a keyframe still by editing the character sheet(s) with the - * keyframe prompt (so the character stays consistent). Mirrors - * `generateBackdropImage` but uses each provider's `editImage`. + * keyframe prompt (so the character stays consistent). Providers that take + * fewer input images than there are sheets get the first ones they accept + * (Grok edits one image). */ async function editKeyframeImage( prompt: string, sheetBuffers: Buffer[], ctx: ImageGenContext, ratio: string +): Promise { + return buildImage(prompt, ctx, ratio, sheetBuffers); +} + +/** One image through the image contract, with the build's project-scoped key. */ +async function buildImage( + prompt: string, + ctx: ImageGenContext, + ratio: string, + sheetBuffers?: Buffer[] ): Promise { loadSceneBuildEnv(ctx.projectDir); const keyInfo = imageProviderKeyInfo(ctx.imageProvider); @@ -2018,32 +1978,24 @@ async function editKeyframeImage( if (!apiKey) { return { success: false, - error: `${keyInfo.envVar} not set — cannot generate keyframe with ${ctx.imageProvider}`, + error: `${keyInfo.envVar} not set — cannot generate ${sheetBuffers ? "keyframe" : "backdrop"} with ${ctx.imageProvider}`, }; } - - if (ctx.imageProvider === "openai") { - const provider = new OpenAIImageProvider(); - await provider.initialize({ apiKey }); - const result = await provider.editImage(sheetBuffers, prompt); - return imageBufferFromResult(result); - } - - if (ctx.imageProvider === "gemini") { - const provider = new GeminiProvider(); - await provider.initialize({ apiKey }); - const result = await provider.editImage(sheetBuffers, prompt, { - model: (ctx.imageModel as GeminiImageModel | undefined) ?? "flash", - aspectRatio: ratio as "1:1" | "2:3" | "3:2" | "16:9", + try { + const generator = await createImageGenerator(ctx.imageProvider, apiKey); + const maxInputs = sheetBuffers ? generator.maxEditImages(ctx.imageModel) : 0; + const result = await generator.createImage({ + prompt, + model: ctx.imageModel, + aspectRatio: ratio, + quality: contractQuality(ctx.imageQuality), + providerOptions: ctx.imageSize ? { size: ctx.imageSize } : undefined, + images: sheetBuffers?.slice(0, maxInputs).map((b) => ({ bytes: new Uint8Array(b), mimeType: "image/png" })), }); - return imageBufferFromResult(result); + return { success: true, buffer: Buffer.from(result.images[0].bytes) }; + } catch (error) { + return { success: false, error: error instanceof Error ? error.message : String(error) }; } - - const provider = new GrokProvider(); - await provider.initialize({ apiKey }); - // Grok edits a single input image. - const result = await provider.editImage(sheetBuffers[0], prompt, { aspectRatio: ratio }); - return imageBufferFromResult(result); } /** @@ -2149,26 +2101,6 @@ async function ensureKeyframe( return { status: "generated", rel }; } -async function imageBufferFromResult(result: { - success: boolean; - error?: string; - images?: GeneratedImageData[]; -}): Promise { - const image = result.images?.[0]; - if (!result.success || !image) { - return { success: false, error: result.error ?? "no image data returned" }; - } - if (image.base64) return { success: true, buffer: Buffer.from(image.base64, "base64") }; - if (image.url) { - const response = await fetch(image.url); - if (!response.ok) { - return { success: false, error: `failed to download image: HTTP ${response.status}` }; - } - return { success: true, buffer: Buffer.from(await response.arrayBuffer()) }; - } - return { success: false, error: "no image data returned" }; -} - function imageProviderKeyInfo(provider: BuildImageProvider): { configKey: string; envVar: string } { if (provider === "gemini") return { configKey: "google", envVar: "GOOGLE_API_KEY" }; if (provider === "grok") return { configKey: "xai", envVar: "XAI_API_KEY" }; diff --git a/packages/cli/src/commands/ai-image.ts b/packages/cli/src/commands/ai-image.ts index 7e306a07..8adb3a0c 100644 --- a/packages/cli/src/commands/ai-image.ts +++ b/packages/cli/src/commands/ai-image.ts @@ -13,18 +13,17 @@ * @see MODELS.md for AI model configuration */ -import { resolve, dirname } from "node:path"; -import { readFile, mkdir } from "node:fs/promises"; +import { readFile } from "node:fs/promises"; import { existsSync } from "node:fs"; import { GeminiProvider, - GrokProvider, - OpenAIImageProvider, + isProviderError, resolveGeminiTextModel, - resolveGrokImageModel, + type ProviderErrorKind, } from "@vibeframe/ai-providers"; +import { resolveProvider } from "../utils/provider-resolver.js"; +import { contractQuality, openImageGenerator, readImageInput, saveImage } from "./_shared/image-jobs.js"; import { execSafe, commandExists } from "../utils/exec-safe.js"; -import { writeImageFile } from "../utils/image-file.js"; // ============================================================================ // Image Generate @@ -32,12 +31,15 @@ import { writeImageFile } from "../utils/image-file.js"; export interface ImageGenerateOptions { prompt: string; + /** Default: the configured or first available image provider. */ provider?: string; output?: string; + /** Explicit OpenAI size; otherwise the ratio picks one. */ size?: string; ratio?: string; quality?: string; - style?: string; + /** Gemini "1K"/"2K"/"4K", Grok "1k"/"2k". */ + resolution?: string; count?: number; model?: string; apiKey?: string; @@ -45,250 +47,120 @@ export interface ImageGenerateOptions { export interface ImageGenerateResult { success: boolean; + /** Where the first image was saved. */ outputPath?: string; - images?: Array<{ url?: string; base64?: string; mimeType?: string; revisedPrompt?: string }>; + /** Every saved image: `out.png`, then `out-2.png`, `out-3.png`, ... */ + outputPaths?: string[]; + images?: Array<{ base64: string; mimeType: string; revisedPrompt?: string }>; provider?: string; model?: string; error?: string; + errorKind?: ProviderErrorKind; } -export async function executeImageGenerate(options: ImageGenerateOptions): Promise { - const { - prompt, - provider = "gemini", - output, - size = "1024x1024", - ratio = "1:1", - quality = "standard", - style = "vivid", - count = 1, - model, - apiKey, - } = options; - - try { - if (provider === "openai") { - const key = apiKey || process.env.OPENAI_API_KEY; - if (!key) return { success: false, error: "OPENAI_API_KEY required" }; - - const openaiImage = new OpenAIImageProvider(); - await openaiImage.initialize({ apiKey: key }); - - const result = await openaiImage.generateImage(prompt, { - size: size as "1024x1024" | "1536x1024" | "1024x1536" | "auto" | undefined, - quality: quality as "standard" | "hd" | undefined, - style: style as "vivid" | "natural" | undefined, - n: count, - }); - - if (!result.success || !result.images) { - return { success: false, error: result.error || "Image generation failed" }; - } - - let outputPath: string | undefined; - if (output && result.images.length > 0) { - const img = result.images[0]; - let buffer: Buffer; - if (img.url) { - const response = await fetch(img.url); - buffer = Buffer.from(await response.arrayBuffer()); - } else if (img.base64) { - buffer = Buffer.from(img.base64, "base64"); - } else { - return { success: false, error: "No image data available" }; - } - outputPath = resolve(process.cwd(), output); - await mkdir(dirname(outputPath), { recursive: true }); - await writeImageFile(outputPath, buffer); - } - - return { - success: true, - outputPath, - images: result.images.map(img => ({ url: img.url, base64: img.base64, revisedPrompt: img.revisedPrompt })), - provider: "openai", - }; - } else if (provider === "gemini") { - const key = apiKey || process.env.GOOGLE_API_KEY; - if (!key) return { success: false, error: "GOOGLE_API_KEY required" }; - - const gemini = new GeminiProvider(); - await gemini.initialize({ apiKey: key }); - - const modelMap: Record = { latest: "3.1-flash" }; - const mappedModel = model ? (modelMap[model] || model) : undefined; - - let result = await gemini.generateImage(prompt, { - model: mappedModel as "flash" | "3.1-flash" | "pro" | undefined, - aspectRatio: ratio as "1:1" | "16:9" | "9:16" | "4:3" | "3:4", - }); - - const fallbackModels = ["3.1-flash"]; - if (!result.success && mappedModel && fallbackModels.includes(mappedModel)) { - result = await gemini.generateImage(prompt, { - model: "flash", - aspectRatio: ratio as "1:1" | "16:9" | "9:16" | "4:3" | "3:4", - }); - } - - if (!result.success || !result.images) { - return { success: false, error: result.error || "Image generation failed" }; - } - - let outputPath: string | undefined; - if (output && result.images.length > 0) { - const img = result.images[0]; - if (img.base64) { - outputPath = resolve(process.cwd(), output); - await mkdir(dirname(outputPath), { recursive: true }); - await writeImageFile(outputPath, Buffer.from(img.base64, "base64")); - } - } - - return { - success: true, - outputPath, - images: result.images.map(img => ({ base64: img.base64, mimeType: img.mimeType })), - provider: "gemini", - model: result.model, - }; - } else if (provider === "grok") { - const key = apiKey || process.env.XAI_API_KEY; - if (!key) return { success: false, error: "XAI_API_KEY required" }; - - const grok = new GrokProvider(); - await grok.initialize({ apiKey: key }); - - const result = await grok.generateImage(prompt, { - ...resolveGrokImageModel(model), - aspectRatio: ratio, - n: count, - }); +/** `out.png` → `out-2.png`. */ +function numberedPath(path: string, n: number): string { + const dot = path.lastIndexOf("."); + return dot > path.lastIndexOf("/") ? `${path.slice(0, dot)}-${n}${path.slice(dot)}` : `${path}-${n}`; +} - if (!result.success || !result.images) { - return { success: false, error: result.error || "Image generation failed" }; - } +function imageFailure(error: unknown, prefix: string, provider?: string): ImageGenerateResult { + if (isProviderError(error)) return { success: false, error: error.message, errorKind: error.kind, provider }; + return { success: false, error: `${prefix}: ${error instanceof Error ? error.message : String(error)}`, provider }; +} - let outputPath: string | undefined; - if (output && result.images.length > 0) { - const img = result.images[0]; - let buffer: Buffer; - if (img.url) { - const response = await fetch(img.url); - buffer = Buffer.from(await response.arrayBuffer()); - } else if (img.base64) { - buffer = Buffer.from(img.base64, "base64"); - } else { - return { success: false, error: "No image data available" }; - } - outputPath = resolve(process.cwd(), output); - await mkdir(dirname(outputPath), { recursive: true }); - await writeImageFile(outputPath, buffer); +export async function executeImageGenerate(options: ImageGenerateOptions): Promise { + const provider = options.provider ?? resolveProvider("image")?.name ?? "gemini"; + try { + const generator = await openImageGenerator(provider, options.apiKey); + const result = await generator.createImage({ + prompt: options.prompt, + model: options.model, + aspectRatio: options.ratio, + resolution: options.resolution, + quality: contractQuality(options.quality), + count: options.count, + providerOptions: options.size ? { size: options.size } : undefined, + }); + const outputPaths: string[] = []; + if (options.output) { + for (const [i, img] of result.images.entries()) { + outputPaths.push(await saveImage(i === 0 ? options.output : numberedPath(options.output, i + 1), img.bytes)); } - - return { - success: true, - outputPath, - images: result.images.map(img => ({ url: img.url, base64: img.base64, revisedPrompt: img.revisedPrompt })), - provider: "grok", - }; } - - return { success: false, error: `Unsupported provider: ${provider}` }; + return { + success: true, + outputPath: outputPaths[0], + outputPaths: outputPaths.length > 0 ? outputPaths : undefined, + images: result.images.map((img) => ({ + base64: Buffer.from(img.bytes).toString("base64"), + mimeType: img.mimeType, + revisedPrompt: img.revisedPrompt, + })), + provider, + model: result.model, + }; } catch (error) { - return { success: false, error: `Image generation failed: ${error instanceof Error ? error.message : String(error)}` }; + return imageFailure(error, "Image generation failed", provider); } } // ============================================================================ -// Gemini Image Edit +// Image Edit // ============================================================================ -export interface GeminiEditOptions { +export interface ImageEditOptions { imagePaths: string[]; prompt: string; + /** Default: gemini. */ + provider?: string; output?: string; model?: string; ratio?: string; resolution?: string; + quality?: string; apiKey?: string; } -export interface GeminiEditResult { +export interface ImageEditResult { success: boolean; outputPath?: string; + provider?: string; model?: string; error?: string; + errorKind?: ProviderErrorKind; } -export async function executeGeminiEdit(options: GeminiEditOptions): Promise { - const { - imagePaths, - prompt, - output = "edited.png", - model = "flash", - ratio, - resolution, - apiKey, - } = options; - +export async function executeImageEdit(options: ImageEditOptions): Promise { + const provider = options.provider ?? "gemini"; try { - const key = apiKey || process.env.GOOGLE_API_KEY; - if (!key) return { success: false, error: "GOOGLE_API_KEY required" }; - - const imageBuffers: Buffer[] = []; - for (const imagePath of imagePaths) { - const absPath = resolve(process.cwd(), imagePath); - if (!existsSync(absPath)) { - return { success: false, error: `Image not found: ${absPath}` }; - } - const buffer = await readFile(absPath); - imageBuffers.push(buffer); - } - - const gemini = new GeminiProvider(); - await gemini.initialize({ apiKey: key }); - - type GeminiAspectRatio = "1:1" | "1:4" | "1:8" | "2:3" | "3:2" | "3:4" | "4:1" | "4:3" | "4:5" | "5:4" | "8:1" | "9:16" | "16:9" | "21:9"; - type GeminiRes = "512px" | "1K" | "2K" | "4K"; - - let result = await gemini.editImage(imageBuffers, prompt, { - model: model as "flash" | "3.1-flash" | "pro" | undefined, - aspectRatio: ratio as GeminiAspectRatio | undefined, - resolution: resolution as GeminiRes | undefined, + const generator = await openImageGenerator(provider, options.apiKey); + const images = await Promise.all(options.imagePaths.map((path) => readImageInput(path))); + const result = await generator.createImage({ + prompt: options.prompt, + model: options.model, + aspectRatio: options.ratio, + resolution: options.resolution, + quality: contractQuality(options.quality), + images, }); - - const fallbackModels = ["latest", "3.1-flash"]; - if (!result.success && fallbackModels.includes(model)) { - result = await gemini.editImage(imageBuffers, prompt, { - model: "flash", - aspectRatio: ratio as GeminiAspectRatio | undefined, - resolution: resolution as GeminiRes | undefined, - }); - } - - if (!result.success || !result.images || result.images.length === 0) { - return { success: false, error: result.error || "Image editing failed" }; - } - - const img = result.images[0]; - let outputPath: string | undefined; - if (img.base64) { - outputPath = resolve(process.cwd(), output); - await mkdir(dirname(outputPath), { recursive: true }); - await writeImageFile(outputPath, Buffer.from(img.base64, "base64")); - } - - return { - success: true, - outputPath, - model: result.model, - }; + const outputPath = await saveImage(options.output ?? "edited.png", result.images[0].bytes); + return { success: true, outputPath, provider, model: result.model }; } catch (error) { - return { success: false, error: `Image editing failed: ${error instanceof Error ? error.message : String(error)}` }; + const failed = imageFailure(error, "Image editing failed", provider); + return { success: false, error: failed.error, errorKind: failed.errorKind, provider }; } } +/** @deprecated Use `executeImageEdit`; kept for callers that predate other edit providers. */ +export type GeminiEditOptions = Omit; +/** @deprecated Use `executeImageEdit`. */ +export type GeminiEditResult = ImageEditResult; +/** @deprecated Use `executeImageEdit`. */ +export function executeGeminiEdit(options: GeminiEditOptions): Promise { + return executeImageEdit({ ...options, provider: "gemini" }); +} + // ============================================================================ // Thumbnail Best Frame // ============================================================================ diff --git a/packages/cli/src/commands/edit-cmd.ts b/packages/cli/src/commands/edit-cmd.ts index 2e0dbee4..39384a67 100644 --- a/packages/cli/src/commands/edit-cmd.ts +++ b/packages/cli/src/commands/edit-cmd.ts @@ -24,18 +24,11 @@ */ import { Command } from "commander"; -import { resolve, dirname } from "node:path"; -import { readFile, writeFile, mkdir } from "node:fs/promises"; +import { resolve } from "node:path"; +import { readFile, writeFile } from "node:fs/promises"; import chalk from "chalk"; import ora from "ora"; -import { - WhisperProvider, - ClaudeProvider, - GeminiProvider, - OpenAIImageProvider, - GrokProvider, - resolveGrokImageModel, -} from "@vibeframe/ai-providers"; +import { WhisperProvider, ClaudeProvider, findModel, modelAliases } from "@vibeframe/ai-providers"; import { requireApiKey } from "../utils/api-key.js"; import { execSafe, commandExists } from "../utils/exec-safe.js"; import { formatTime } from "./ai-helpers.js"; @@ -43,10 +36,11 @@ import { applyTextOverlays, type TextOverlayStyle } from "./ai-edit.js"; import { registerEditCommands } from "./ai-edit-cli.js"; import { registerFillGapsCommand } from "./ai-fill-gaps.js"; import { registerMotionOverlayCommand } from "./edit/motion-overlay.js"; -import { isJsonMode, outputSuccess, exitWithError, usageError, notFoundError, apiError, generalError } from "./output.js"; +import { isJsonMode, outputSuccess, exitWithError, usageError, notFoundError, apiError, generalError, providerFailure } from "./output.js"; import { rejectControlChars, validateOutputPath } from "./validate.js"; import { applyTiers } from "./_shared/cost-tier.js"; -import { writeImageFile } from "../utils/image-file.js"; +import { executeImageEdit } from "./ai-image.js"; +import { IMAGE_PROVIDER_ENV, IMAGE_PROVIDER_LABELS } from "./_shared/image-jobs.js"; export const editCommand = new Command("edit") .alias("ed") @@ -689,7 +683,7 @@ editCommand } }); -// ── edit image (Gemini multi-image editing) ───────────────────────────── +// ── edit image (multi-image editing on any image provider) ────────────── editCommand .command("image") @@ -698,157 +692,76 @@ editCommand .option("-p, --provider ", "Provider: gemini (default), openai, grok", "gemini") .option("-k, --api-key ", "API key (or set env variable)") .option("-o, --output ", "Output file path", "edited.png") - .option("-m, --model ", "Model: flash, lite, pro (Gemini); pro (Grok)", "flash") + .option("-m, --model ", "Model: flash (default), lite, pro (Gemini); 2.5, flare, 2 (OpenAI); pro (Grok)") .option("-r, --ratio ", "Output aspect ratio") .option("--size ", "Resolution: 1K, 2K, 4K (Gemini Pro only)") .option("--dry-run", "Preview parameters without executing") .action(async (args: string[], options) => { const startedAt = Date.now(); - try { - // Last argument is the prompt, rest are image paths - if (args.length < 2) { - exitWithError(usageError("Need at least one image and a prompt")); - } - - const prompt = args[args.length - 1]; - rejectControlChars(prompt); - if (options.output) { - validateOutputPath(options.output); - } - const imagePaths = args.slice(0, -1); - const provider = options.provider as string; - - // Grok only supports 1 image - if (provider === "grok" && imagePaths.length > 1) { - exitWithError(usageError("Grok supports only 1 input image for editing.", "Use -p gemini (up to 14 images) or -p openai (up to 16 images) for multi-image editing.")); - } - - if (options.dryRun) { - outputSuccess({ - command: "edit image", - startedAt, - dryRun: true, - data: { - params: { - imagePaths: imagePaths.map((p: string) => resolve(process.cwd(), p)), - prompt, - provider, - model: options.model, - ratio: options.ratio, - size: options.size, - }, - }, - }); - return; - } - - // Provider-specific API key resolution - const apiKeyMap: Record = { - gemini: { envVar: "GOOGLE_API_KEY", label: "Google" }, - openai: { envVar: "OPENAI_API_KEY", label: "OpenAI" }, - grok: { envVar: "XAI_API_KEY", label: "xAI" }, - }; - const keyInfo = apiKeyMap[provider] || apiKeyMap.gemini; - const apiKey = await requireApiKey(keyInfo.envVar, keyInfo.label, options.apiKey); - - const spinner = ora(`Reading ${imagePaths.length} image(s)...`).start(); - - // Load all images - const imageBuffers: Buffer[] = []; - for (const imagePath of imagePaths) { - const absPath = resolve(process.cwd(), imagePath); - const buffer = await readFile(absPath); - imageBuffers.push(buffer); - } - - let result: import("@vibeframe/ai-providers").ImageResult; - - if (provider === "openai") { - spinner.text = "Editing with GPT Image 1.5..."; - const openaiImage = new OpenAIImageProvider(); - await openaiImage.initialize({ apiKey }); - result = await openaiImage.editImage(imageBuffers, prompt); - } else if (provider === "grok") { - spinner.text = "Editing with Grok Imagine..."; - const grok = new GrokProvider(); - await grok.initialize({ apiKey }); - result = await grok.editImage(imageBuffers[0], prompt, { - ...resolveGrokImageModel(options.model), - aspectRatio: options.ratio, - }); - } else { - // Gemini (default) - const editModelNames: Record = { - flash: "gemini-3.1-flash-image", - "3.1-flash": "gemini-3.1-flash-image", - latest: "gemini-3.1-flash-image", - lite: "gemini-3.1-flash-lite-image", - pro: "gemini-3-pro-image", - }; - const editModelName = editModelNames[options.model] || editModelNames.flash; - spinner.text = `Editing with ${editModelName}...`; - - const gemini = new GeminiProvider(); - await gemini.initialize({ apiKey }); - - result = await gemini.editImage(imageBuffers, prompt, { - model: options.model, - aspectRatio: options.ratio, - resolution: options.size, - }); - - } - - if (!result.success || !result.images || result.images.length === 0) { - spinner.fail(result.error || "Image editing failed"); - exitWithError(apiError(result.error || "Image editing failed", true)); - } - - spinner.succeed(chalk.green("Image edited")); - - // Save image — handle both base64 and URL responses - const img = result.images[0]; - const outputPath = resolve(process.cwd(), options.output); - - const saveImage = async () => { - await mkdir(dirname(outputPath), { recursive: true }); - if (img.base64) { - const buffer = Buffer.from(img.base64, "base64"); - await writeImageFile(outputPath, buffer); - } else if (img.url) { - const resp = await fetch(img.url); - const arrayBuf = await resp.arrayBuffer(); - await writeImageFile(outputPath, Buffer.from(arrayBuf)); - } - }; - - // Gemini results may include a `model` field - const resultModel = (result as { model?: string }).model; + // Last argument is the prompt, the rest are image paths + if (args.length < 2) { + exitWithError(usageError("Need at least one image and a prompt")); + } + const prompt = args[args.length - 1]; + rejectControlChars(prompt); + if (options.output) validateOutputPath(options.output); + const imagePaths = args.slice(0, -1); + const provider = String(options.provider || "gemini").toLowerCase(); + if (!IMAGE_PROVIDER_ENV[provider]) { + exitWithError(usageError(`Invalid provider: ${provider}`, `Available providers: ${Object.keys(IMAGE_PROVIDER_ENV).join(", ")}`)); + } + const modelSpec = findModel(provider, "image", options.model); + if (!modelSpec) { + exitWithError(usageError(`Unknown ${provider} image model "${options.model}". Valid: ${modelAliases(provider, "image").join(", ")}.`)); + } - if (isJsonMode()) { - outputSuccess({ - command: "edit image", - startedAt, - data: { + if (options.dryRun) { + outputSuccess({ + command: "edit image", + startedAt, + dryRun: true, + data: { + params: { + imagePaths: imagePaths.map((p: string) => resolve(process.cwd(), p)), + prompt, provider, - model: resultModel || options.model, - outputPath, + model: modelSpec.id, + ratio: options.ratio, + size: options.size, }, - }); - await saveImage(); - return; - } + }, + }); + return; + } - if (resultModel) { - console.log(chalk.dim(`Model: ${resultModel}`)); - } + const label = IMAGE_PROVIDER_LABELS[provider]; + const apiKey = await requireApiKey(IMAGE_PROVIDER_ENV[provider], label, options.apiKey); + const spinner = isJsonMode() ? null : ora(`Editing ${imagePaths.length} image(s) with ${label} ${modelSpec.label}...`).start(); + const result = await executeImageEdit({ + imagePaths, + prompt, + provider, + model: options.model, + ratio: options.ratio, + resolution: options.size, + output: options.output, + apiKey, + }); + if (!result.success) { + spinner?.fail(result.error ?? "Image editing failed"); + exitWithError(providerFailure(result.error ?? "Image editing failed", result.errorKind)); + } - await saveImage(); - console.log(chalk.green(`Saved to: ${outputPath}`)); - } catch (error) { - const msg = error instanceof Error ? error.message : String(error); - exitWithError(apiError(`Image editing failed: ${msg}`, true)); + if (isJsonMode()) { + outputSuccess({ + command: "edit image", + startedAt, + data: { provider, model: result.model, outputPath: result.outputPath }, + }); + return; } + spinner?.succeed(chalk.green("Image edited")); + console.log(chalk.green(`Saved to: ${result.outputPath}`)); }); // ── edit interpolate (frame interpolation / slow motion) ──────────────── diff --git a/packages/cli/src/commands/error-envelopes.test.ts b/packages/cli/src/commands/error-envelopes.test.ts index f64e3d37..2ae0752b 100644 --- a/packages/cli/src/commands/error-envelopes.test.ts +++ b/packages/cli/src/commands/error-envelopes.test.ts @@ -101,10 +101,11 @@ const cases: ErrorCase[] = [ exitCode: 2, }, { - name: "generate image — no API key, hermetic (API_ERROR)", + // A missing key is an auth error with setup hints, not a generic API failure. + name: "generate image — no API key, hermetic (API_KEY_MISSING)", cmd: 'generate image "a test prompt" -p openai --json', - code: "API_ERROR", - exitCode: 5, + code: "API_KEY_MISSING", + exitCode: 4, }, ]; diff --git a/packages/cli/src/commands/generate/image.ts b/packages/cli/src/commands/generate/image.ts index 2d32827f..a9a05dd2 100644 --- a/packages/cli/src/commands/generate/image.ts +++ b/packages/cli/src/commands/generate/image.ts @@ -1,58 +1,50 @@ /** * @module generate/image - * @description `vibe generate image` (alias `img`) — multi-provider image - * generation. OpenAI gpt-image-2.5, Gemini Nano Banana, Grok, Runway. Split - * out of `generate.ts` in v0.69 (Plan G Phase 2). + * @description `vibe generate image` (alias `img`) — image generation on + * OpenAI GPT Image, Gemini Nano Banana, or Grok Imagine. Validation, + * provider choice, and the dry run live here; generation is + * `executeImageGenerate`, the same executor the MCP tool and pipelines use. */ import type { Command } from "commander"; -import { resolve, dirname } from "node:path"; -import { fileURLToPath } from "node:url"; -import { mkdir } from "node:fs/promises"; import chalk from "chalk"; import ora from "ora"; -import { - GeminiProvider, - GrokProvider, - getProvidersFor, - resolveGrokImageModel, -} from "@vibeframe/ai-providers"; +import { findModel, getProvidersFor, modelAliases } from "@vibeframe/ai-providers"; import { requireApiKey, hasConfiguredApiKey } from "../../utils/api-key.js"; import { hasTTY, prompt as promptText } from "../../utils/tty.js"; -import { isJsonMode, outputSuccess, log, exitWithError, apiError, usageError } from "../output.js"; +import { checkModelLifecycle } from "../../utils/model-lifecycle.js"; +import { + isJsonMode, + outputSuccess, + printWarnings, + log, + exitWithError, + providerFailure, + usageError, +} from "../output.js"; import { rejectControlChars, validateOutputPath } from "../validate.js"; import { loadProviderDefaults, resolveProvider } from "../../utils/provider-resolver.js"; -import { executeOpenAIImageGenerate } from "../_shared/openai-image.js"; import { estimateImageCostUsd } from "../_shared/image-cost.js"; -import { writeImageFile } from "../../utils/image-file.js"; +import { executeImageGenerate } from "../ai-image.js"; +import { IMAGE_PROVIDER_ENV, IMAGE_PROVIDER_LABELS, imageModelLabel } from "../_shared/image-jobs.js"; export function registerImageCommand(parent: Command): void { parent .command("image") .alias("img") - .description("Generate image using AI (Gemini, OpenAI gpt-image, Grok, or Runway)") + .description("Generate image using AI (OpenAI GPT Image, Gemini, or Grok)") .argument("[prompt]", "Image description prompt (interactive if omitted)") - .option( - "-p, --provider ", - "Provider: openai (default when OPENAI_API_KEY set), gemini, grok, runway" - ) - .option("-k, --api-key ", "API key (or set env: OPENAI_API_KEY, GOOGLE_API_KEY)") - .option("-o, --output ", "Output file path (downloads image)") - .option("--size ", "Image size (openai: 1024x1024, 1536x1024, 1024x1536)", "1024x1024") - .option( - "-r, --ratio ", - "Aspect ratio (gemini: 1:1, 1:4, 1:8, 4:1, 8:1, 16:9, 9:16, 3:4, 4:3, etc.)", - "1:1" - ) + .option("-p, --provider ", "Provider: openai (default when OPENAI_API_KEY set), gemini, grok") + .option("-k, --api-key ", "API key (or set env: OPENAI_API_KEY, GOOGLE_API_KEY, XAI_API_KEY)") + .option("-o, --output ", "Output file path") + .option("-r, --ratio ", "Aspect ratio: 1:1, 16:9, 9:16, 4:3, 3:4, ... (OpenAI maps it to its nearest size)", "1:1") + .option("--size ", "Explicit OpenAI size: 1024x1024, 1536x1024, 1024x1536 (overrides --ratio)") // `-q` shorthand intentionally omitted: collides with global `vibe -q,--quiet`, // which previously ate the value silently and dropped the prompt positional. - .option("--quality ", "Quality: standard, hd (openai only)", "standard") - .option("--style