diff --git a/AGENTS.md b/AGENTS.md index 9a2c1a33..67914c62 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -129,10 +129,12 @@ General expectation: After editing the catalog, run `pnpm gen:models` (MODELS.md is partly generated). - Video providers implement `VideoGenerator` - (`packages/ai-providers/src/video/`): HTTP goes through `providerRequest`, - failures are `ProviderError`s with a `kind`, and each provider passes - `describeVideoContract`. Callers branch on `error.kind`, never on provider - wording. + (`packages/ai-providers/src/video/`) and image providers implement + `ImageGenerator` (`src/image/`): HTTP goes through `providerRequest`, + failures are `ProviderError`s with a `kind`, and each provider passes its + contract suite. The CLI opens providers through `createVideoGenerator` / + `createImageGenerator` and branches on `error.kind`, never on provider + names or wording. ## Conventions diff --git a/docs/cli-reference.md b/docs/cli-reference.md index c5816f39..1544d2de 100644 --- a/docs/cli-reference.md +++ b/docs/cli-reference.md @@ -491,7 +491,7 @@ Cost tier: _not tagged_ #### `vibe generate image` -Generate image using AI (Gemini, OpenAI gpt-image, Grok, or Runway) +Generate image using AI (OpenAI GPT Image, Gemini, or Grok) Product surface: `public` @@ -500,15 +500,15 @@ Cost tier: `high` **Parameters:** - `prompt` _(string)_ - Image description prompt (interactive if omitted) -- `provider` _(string)_ _(openai \| gemini \| grok \| runway)_ - Provider: openai (default when OPENAI_API_KEY set), gemini, grok, runway -- `apiKey` _(string)_ - API key (or set env: OPENAI_API_KEY, GOOGLE_API_KEY) -- `output` _(string)_ - Output file path (downloads image) -- `size` _(string)_ _(default: `"1024x1024"`)_ - Image size (openai: 1024x1024, 1536x1024, 1024x1536) -- `ratio` _(string)_ _(default: `"1:1"`)_ - Aspect ratio (gemini: 1:1, 1:4, 1:8, 4:1, 8:1, 16:9, 9:16, 3:4, 4:3, etc.) -- `quality` _(string)_ _(standard \| hd)_ _(default: `"standard"`)_ - Quality: standard, hd (openai only) -- `style` _(string)_ _(vivid \| natural)_ _(default: `"vivid"`)_ - Style: vivid, natural (openai only) +- `provider` _(string)_ _(openai \| gemini \| grok)_ - Provider: openai (default when OPENAI_API_KEY set), gemini, grok +- `apiKey` _(string)_ - API key (or set env: OPENAI_API_KEY, GOOGLE_API_KEY, XAI_API_KEY) +- `output` _(string)_ - Output file path +- `ratio` _(string)_ _(default: `"1:1"`)_ - Aspect ratio: 1:1, 16:9, 9:16, 4:3, 3:4, ... (OpenAI maps it to its nearest size) +- `size` _(string)_ - Explicit OpenAI size: 1024x1024, 1536x1024, 1024x1536 (overrides --ratio) +- `quality` _(string)_ _(low \| medium \| high)_ - Quality: low, medium, high (OpenAI and Grok; standard/hd still accepted) +- `resolution` _(string)_ - Gemini 1K/2K/4K (2K and 4K need Pro) or Grok 1k/2k - `count` _(number)_ _(default: `1`)_ - Number of images to generate -- `model` _(string)_ - Model. Gemini: flash (default), lite, pro. OpenAI: 2.5 (default), flare, 2, 1.5. Grok: pro +- `model` _(string)_ - Model. OpenAI: 2.5 (default), flare, 2. Gemini: flash (default), lite, pro. Grok: pro - `dryRun` _(boolean)_ - Preview parameters without executing #### `vibe generate motion` @@ -777,7 +777,7 @@ Cost tier: `high` - `provider` _(string)_ _(gemini \| openai \| grok)_ _(default: `"gemini"`)_ - Provider: gemini (default), openai, grok - `apiKey` _(string)_ - API key (or set env variable) - `output` _(string)_ _(default: `"edited.png"`)_ - Output file path -- `model` _(string)_ _(default: `"flash"`)_ - Model: flash, lite, pro (Gemini); pro (Grok) +- `model` _(string)_ - Model: flash (default), lite, pro (Gemini); 2.5, flare, 2 (OpenAI); pro (Grok) - `ratio` _(string)_ - Output aspect ratio - `size` _(string)_ - Resolution: 1K, 2K, 4K (Gemini Pro only) - `dryRun` _(boolean)_ - Preview parameters without executing @@ -1256,7 +1256,7 @@ Cost tier: `free` - `insertInto` _(string)_ _(default: `"index.html"`)_ - Root composition file to update - `project` _(string)_ _(default: `"."`)_ - Project directory - `noStoryboard` _(boolean)_ - Do not sync STORYBOARD.md; insert this scene directly into the root composition only -- `imageProvider` _(string)_ _(gemini \| openai)_ _(default: `"gemini"`)_ - Image provider: gemini, openai +- `imageProvider` _(string)_ _(gemini \| openai \| grok)_ _(default: `"gemini"`)_ - Image provider: gemini, openai, grok - `tts` _(string)_ _(auto \| elevenlabs \| openai \| kokoro)_ _(default: `"auto"`)_ - TTS provider: auto, elevenlabs, openai, kokoro (default auto - ElevenLabs key > OpenAI key > Kokoro local) - `voice` _(string)_ - Voice id (ElevenLabs name/id, OpenAI voice like marin, or Kokoro id like af_heart) - `noAudio` _(boolean)_ - Skip TTS even when --narration is provided (useful for tests/agent dry runs) diff --git a/packages/ai-providers/src/gemini/GeminiProvider.ts b/packages/ai-providers/src/gemini/GeminiProvider.ts index a1a5dbb2..004ac3c8 100644 --- a/packages/ai-providers/src/gemini/GeminiProvider.ts +++ b/packages/ai-providers/src/gemini/GeminiProvider.ts @@ -20,6 +20,10 @@ import type { StoryboardSegment } from "../claude/ClaudeProvider.js"; import { analyzeContent as analyzeContentImpl } from "./gemini-storyboard.js"; import { errorMessage, fetchJson, sleep } from "../shared/http.js"; import { assertModelServed, defaultModel, findModel, listModels } from "../catalog/catalog.js"; +import { fromBase64, type ImageGenerator, type ImageRequest, type ImageResultSet } from "../image/contract.js"; +import { ProviderError } from "../shared/errors.js"; +import { providerRequest } from "../shared/http.js"; +import { resolveCatalogModel } from "../video/models.js"; /** * Gemini model types for image generation @@ -210,13 +214,14 @@ export interface GeminiImageAnalysisResult { * - Video: Veo 3.1 Fast / Veo 3.1 (text-to-video, image-to-video) * - Image: Nano Banana 2 (gemini-3.1-flash-image) / Nano Banana Pro (gemini-3-pro-image) */ -export class GeminiProvider implements AIProvider { +export class GeminiProvider implements AIProvider, ImageGenerator { id = "gemini"; name = "Google Gemini"; description = "AI video (Veo 3.1) and image (Nano Banana) generation"; capabilities: AICapability[] = ["text-to-video", "image-to-video", "text-to-image", "auto-edit", "vision"]; iconUrl = "/icons/gemini.svg"; isAvailable = true; + readonly imageProvider = "gemini"; private apiKey?: string; private baseUrl = "https://generativelanguage.googleapis.com/v1beta"; @@ -585,335 +590,135 @@ export class GeminiProvider implements AIProvider { return false; } - /** - * Resolve model alias to full model ID - */ - private resolveModel(model?: GeminiImageModel): string { - // Aliases and full IDs come from the model catalog; unknown values fall - // back to the default Nano Banana model. - return (findModel("gemini", "image", model) ?? defaultModel("gemini", "image")).id; - } + // ── ImageGenerator ──────────────────────────────────────────────────── - /** - * Check if model is Pro - */ - private isProModel(modelId: string): boolean { - return modelId.includes("pro"); + /** Nano Banana edits take up to 3 input images, Pro up to 14. */ + maxEditImages(model?: string): number { + return resolveCatalogModel("gemini", "image", model).id.includes("pro") ? 14 : 3; } - /** - * Generate images using Gemini (Nano Banana) - * Uses generateContent with responseModalities: ["TEXT", "IMAGE"] - */ - async generateImage( - prompt: string, - options: GeminiImageOptions = {} - ): Promise { + async createImage(request: ImageRequest): Promise { if (!this.apiKey) { - return { - success: false, - error: "Google API key not configured", - }; + throw new ProviderError({ kind: "auth", provider: this.imageProvider, message: "Google API key not configured" }); + } + const modelId = resolveCatalogModel(this.imageProvider, "image", request.model).id; + const isPro = modelId.includes("pro"); + const inputs = request.images ?? []; + if (inputs.length > this.maxEditImages(modelId)) { + throw new ProviderError({ + kind: "invalid-request", + provider: this.imageProvider, + message: `${modelId} edits take up to ${this.maxEditImages(modelId)} images, not ${inputs.length}.`, + }); } - try { - const modelId = this.resolveModel(options.model); - const isPro = this.isProModel(modelId); - - // Build image config - const imageConfig: Record = {}; - if (options.aspectRatio) { - imageConfig.aspectRatio = options.aspectRatio; - } else { - imageConfig.aspectRatio = "1:1"; - } - - // Resolution: 512px available for all models, 2K/4K for Pro only - if (options.resolution) { - if (options.resolution === "512px" || options.resolution === "1K" || isPro) { - imageConfig.imageSize = options.resolution; - } - } - - // Build generation config - const generationConfig: Record = { - responseModalities: ["TEXT", "IMAGE"], - imageConfig, - }; - - // Add thinking config if specified - if (options.thinkingConfig) { - generationConfig.thinkingConfig = options.thinkingConfig; - } - - // Build payload - const payload: Record = { - contents: [{ parts: [{ text: prompt }] }], - generationConfig, - }; - - // Add Google Search grounding - const is31Flash = modelId.startsWith("gemini-3.1-flash-image"); - if (options.imageSearchGrounding && is31Flash) { - // 3.1 Flash supports Image Search grounding - payload.tools = [{ googleSearch: { searchTypes: { webSearch: {}, imageSearch: {} } } }]; - } else if (options.grounding && isPro) { - payload.tools = [{ googleSearch: {} }]; - } + const imageConfig: Record = {}; + if (request.aspectRatio) imageConfig.aspectRatio = request.aspectRatio; + else if (inputs.length === 0) imageConfig.aspectRatio = "1:1"; + // 512px and 1K work on every model; 2K and 4K need Pro. + if (request.resolution && (request.resolution === "512px" || request.resolution === "1K" || isPro)) { + imageConfig.imageSize = request.resolution; + } + const generationConfig: Record = { responseModalities: ["TEXT", "IMAGE"] }; + if (Object.keys(imageConfig).length > 0) generationConfig.imageConfig = imageConfig; + if (request.providerOptions?.thinkingConfig) generationConfig.thinkingConfig = request.providerOptions.thinkingConfig; - const response = await fetch( - `${this.baseUrl}/models/${modelId}:generateContent?key=${this.apiKey}`, + const payload: Record = { + contents: [ { - method: "POST", - headers: { - "Content-Type": "application/json", - }, - body: JSON.stringify(payload), - } - ); - - if (!response.ok) { - const errorText = await response.text(); - let errorMessage: string; - try { - const errorData = JSON.parse(errorText); - errorMessage = errorData.error?.message || errorText; - } catch { - errorMessage = errorText; - } - return { - success: false, - error: `API error (${response.status}): ${errorMessage}`, - }; - } + parts: [ + { text: request.prompt }, + ...inputs.map((img) => ({ inlineData: { mimeType: img.mimeType, data: Buffer.from(img.bytes).toString("base64") } })), + ], + }, + ], + generationConfig, + }; + if (request.providerOptions?.imageSearchGrounding && modelId.startsWith("gemini-3.1-flash-image")) { + payload.tools = [{ googleSearch: { searchTypes: { webSearch: {}, imageSearch: {} } } }]; + } else if (request.providerOptions?.grounding && isPro) { + payload.tools = [{ googleSearch: {} }]; + } - const data = (await response.json()) as { - candidates?: Array<{ - content?: { - parts?: Array<{ - text?: string; - thought?: boolean; - inlineData?: { - mimeType: string; - data: string; - }; - }>; - }; - finishReason?: string; - }>; - promptFeedback?: { blockReason?: string }; - }; + // The key goes in a header so it never lands in a URL, log, or error message. + const response = await providerRequest(this.imageProvider, `${this.baseUrl}/models/${modelId}:generateContent`, { + method: "POST", + headers: { "Content-Type": "application/json", "x-goog-api-key": this.apiKey }, + body: JSON.stringify(payload), + }); + const data = (await response.json()) as { + candidates?: Array<{ + content?: { parts?: Array<{ text?: string; thought?: boolean; inlineData?: { mimeType: string; data: string } }> }; + finishReason?: string; + }>; + promptFeedback?: { blockReason?: string }; + }; - const parts = data.candidates?.[0]?.content?.parts; - if (!parts || parts.length === 0) { - const finishReason = data.candidates?.[0]?.finishReason; - const blockReason = data.promptFeedback?.blockReason; - const detail = blockReason + const parts = (data.candidates?.[0]?.content?.parts ?? []).filter((part) => !part.thought); + const images = parts + .filter((part) => part.inlineData) + .map((part) => ({ bytes: fromBase64(part.inlineData!.data), mimeType: part.inlineData!.mimeType })); + if (images.length === 0) { + const blockReason = data.promptFeedback?.blockReason; + const finishReason = data.candidates?.[0]?.finishReason; + throw new ProviderError({ + kind: blockReason || finishReason === "SAFETY" || finishReason === "IMAGE_SAFETY" ? "moderation" : "provider", + provider: this.imageProvider, + message: blockReason ? `Blocked by safety filter: ${blockReason}` - : finishReason - ? `Model returned no image (finishReason: ${finishReason})` - : "No content in response"; - return { - success: false, - error: detail, - }; - } - - // Extract images from parts (skip thought images from Pro model) - const images: Array<{ base64: string; mimeType: string }> = []; - let description: string | undefined; - - for (const part of parts) { - // Skip thought images (Pro model thinking process) - if (part.thought) continue; - - if (part.inlineData) { - images.push({ - base64: part.inlineData.data, - mimeType: part.inlineData.mimeType, - }); - } else if (part.text) { - description = part.text; - } - } - - if (images.length === 0) { - return { - success: false, - error: "No images in response", - }; - } - - return { - success: true, - images, - description, - model: modelId, - }; - } catch (error) { - return { - success: false, - error: error instanceof Error ? error.message : "Unknown error", - }; + : `Model returned no image${finishReason ? ` (finishReason: ${finishReason})` : ""}`, + }); } + return { images, model: modelId, text: parts.find((part) => part.text)?.text }; } + // ── Older interface, kept until every caller uses the contract ──────── + /** - * Edit image(s) using Gemini (Nano Banana) - * Provide input image(s) with a text prompt to edit/transform/compose + * Generate images using Gemini (Nano Banana). */ - async editImage( - imageBuffers: Buffer[], - prompt: string, - options: GeminiEditOptions = {} - ): Promise { - if (!this.apiKey) { - return { - success: false, - error: "Google API key not configured", - }; - } - - const modelId = this.resolveModel(options.model); - const isPro = this.isProModel(modelId); + async generateImage(prompt: string, options: GeminiImageOptions = {}): Promise { + return this.legacyImage(() => + this.createImage({ + prompt, + model: options.model, + aspectRatio: options.aspectRatio, + resolution: options.resolution, + providerOptions: { + grounding: options.grounding, + imageSearchGrounding: options.imageSearchGrounding, + thinkingConfig: options.thinkingConfig, + }, + }) + ); + } - // Validate image count - const maxImages = isPro ? 14 : 3; - if (imageBuffers.length > maxImages) { - return { - success: false, - error: `Too many input images. ${modelId} supports up to ${maxImages} images.`, - }; - } + /** + * Edit image(s) using Gemini (Nano Banana). + */ + async editImage(imageBuffers: Buffer[], prompt: string, options: GeminiEditOptions = {}): Promise { + return this.legacyImage(() => + this.createImage({ + prompt, + model: options.model, + aspectRatio: options.aspectRatio, + resolution: options.resolution, + images: imageBuffers.map((b) => ({ bytes: new Uint8Array(b), mimeType: "image/png" })), + }) + ); + } + private async legacyImage(run: () => Promise): Promise { try { - // Build parts: text prompt first, then images - const parts: Array> = [{ text: prompt }]; - - for (const buffer of imageBuffers) { - parts.push({ - inlineData: { - mimeType: "image/png", - data: buffer.toString("base64"), - }, - }); - } - - // Build image config - const imageConfig: Record = {}; - if (options.aspectRatio) { - imageConfig.aspectRatio = options.aspectRatio; - } - if (options.resolution && isPro) { - imageConfig.imageSize = options.resolution; - } - - // Build generation config - const generationConfig: Record = { - responseModalities: ["TEXT", "IMAGE"], - }; - if (Object.keys(imageConfig).length > 0) { - generationConfig.imageConfig = imageConfig; - } - - const payload = { - contents: [{ parts }], - generationConfig, - }; - - const response = await fetch( - `${this.baseUrl}/models/${modelId}:generateContent?key=${this.apiKey}`, - { - method: "POST", - headers: { - "Content-Type": "application/json", - }, - body: JSON.stringify(payload), - } - ); - - if (!response.ok) { - const errorText = await response.text(); - let errorMessage: string; - try { - const errorData = JSON.parse(errorText); - errorMessage = errorData.error?.message || errorText; - } catch { - errorMessage = errorText; - } - return { - success: false, - error: `API error (${response.status}): ${errorMessage}`, - }; - } - - const data = (await response.json()) as { - candidates?: Array<{ - content?: { - parts?: Array<{ - text?: string; - thought?: boolean; - inlineData?: { - mimeType: string; - data: string; - }; - }>; - }; - finishReason?: string; - }>; - promptFeedback?: { blockReason?: string }; - }; - - const responseParts = data.candidates?.[0]?.content?.parts; - if (!responseParts || responseParts.length === 0) { - const finishReason = data.candidates?.[0]?.finishReason; - const blockReason = data.promptFeedback?.blockReason; - const detail = blockReason - ? `Blocked by safety filter: ${blockReason}` - : finishReason - ? `Model returned no image (finishReason: ${finishReason})` - : "No content in response"; - return { - success: false, - error: detail, - }; - } - - // Extract images (skip thought images) - const images: Array<{ base64: string; mimeType: string }> = []; - let description: string | undefined; - - for (const part of responseParts) { - if (part.thought) continue; - - if (part.inlineData) { - images.push({ - base64: part.inlineData.data, - mimeType: part.inlineData.mimeType, - }); - } else if (part.text) { - description = part.text; - } - } - - if (images.length === 0) { - return { - success: false, - error: "No images in response", - }; - } - + const result = await run(); return { success: true, - images, - description, - model: modelId, + images: result.images.map((img) => ({ base64: Buffer.from(img.bytes).toString("base64"), mimeType: img.mimeType })), + description: result.text, + model: result.model, }; } catch (error) { - return { - success: false, - error: error instanceof Error ? error.message : "Unknown error", - }; + return { success: false, error: errorMessage(error) }; } } diff --git a/packages/ai-providers/src/grok/GrokProvider.test.ts b/packages/ai-providers/src/grok/GrokProvider.test.ts index f6283fcc..b8bd5491 100644 --- a/packages/ai-providers/src/grok/GrokProvider.test.ts +++ b/packages/ai-providers/src/grok/GrokProvider.test.ts @@ -50,7 +50,7 @@ describe("GrokProvider", () => { mockFetch.mockResolvedValueOnce({ ok: true, json: async () => ({ - data: [{ url: "https://imgen.x.ai/test.jpg" }], + data: [{ b64_json: "aW1n" }], }), }); @@ -69,6 +69,8 @@ describe("GrokProvider", () => { const body = JSON.parse(options.body); expect(body.prompt).toBe("a blue cat"); expect(body.model).toBe("grok-imagine-image"); + // Base64 back, so callers never download an expiring URL. + expect(body.response_format).toBe("b64_json"); }); it("should handle API error gracefully", async () => { diff --git a/packages/ai-providers/src/grok/GrokProvider.ts b/packages/ai-providers/src/grok/GrokProvider.ts index 65f1bb37..b7e3467c 100644 --- a/packages/ai-providers/src/grok/GrokProvider.ts +++ b/packages/ai-providers/src/grok/GrokProvider.ts @@ -8,6 +8,8 @@ import type { import type { ImageResult } from "../openai-image/OpenAIImageProvider.js"; import { defaultModel, findModel, modelAliases } from "../catalog/catalog.js"; import type { VideoGenerator, VideoJob, VideoJobState, VideoRequest } from "../video/contract.js"; +import { fromBase64, type ImageGenerator, type ImageRequest, type ImageResultSet } from "../image/contract.js"; +import { resolveCatalogModel } from "../video/models.js"; import { ProviderError, classifyProviderError, isProviderError } from "../shared/errors.js"; import { providerRequest } from "../shared/http.js"; import { resolveVideoModel } from "../video/models.js"; @@ -151,8 +153,9 @@ interface GrokStatusResponse { * xAI Grok Imagine provider for video generation * Supports text-to-video and image-to-video with native audio */ -export class GrokProvider implements AIProvider, VideoGenerator { +export class GrokProvider implements AIProvider, VideoGenerator, ImageGenerator { id = "grok"; + readonly imageProvider = "grok"; readonly imageInput = "either" as const; name = "xAI Grok Imagine"; description = "AI video generation with Grok Imagine (native audio, 1-15 sec)"; @@ -175,171 +178,99 @@ export class GrokProvider implements AIProvider, VideoGenerator { return !!this.apiKey; } - /** - * Generate image using Grok Imagine - */ - async generateImage( - prompt: string, - options: GrokImageOptions = {} - ): Promise { - if (!this.apiKey) { - return { - success: false, - error: "xAI API key not configured. Set XAI_API_KEY environment variable.", - }; - } + // ── ImageGenerator ──────────────────────────────────────────────────── - try { - const body: Record = { - model: options.model || "grok-imagine-image", - ...(options.quality ? { quality: options.quality } : {}), - prompt, - n: options.n || 1, - response_format: options.responseFormat || "url", - }; - - if (options.aspectRatio) { - body.aspect_ratio = options.aspectRatio; - } - - if (options.resolution) { - body.resolution = options.resolution; - } + /** Grok Imagine edits one image at a time. */ + maxEditImages(): number { + return 1; + } - const response = await fetch(`${this.baseUrl}/images/generations`, { - method: "POST", - headers: { - "Content-Type": "application/json", - Authorization: `Bearer ${this.apiKey}`, - }, - body: JSON.stringify(body), + async createImage(request: ImageRequest): Promise { + const apiKey = this.requireKey(); + const model = resolveCatalogModel(this.imageProvider, "image", request.model).id as GrokImageModel; + const inputs = request.images ?? []; + if (inputs.length > 1) { + throw new ProviderError({ + kind: "invalid-request", + provider: this.imageProvider, + message: `Grok Imagine edits one image at a time, not ${inputs.length}.`, }); - - if (!response.ok) { - const errorText = await response.text(); - let errorMessage = `API error: ${response.status}`; - try { - const errorJson = JSON.parse(errorText); - if (errorJson.error?.message) { - errorMessage = errorJson.error.message; - } - } catch { - if (errorText) { - errorMessage = errorText.substring(0, 200); - } - } - return { - success: false, - error: errorMessage, - }; - } - - const data = (await response.json()) as { - data: Array<{ - url?: string; - b64_json?: string; - }>; - }; - - return { - success: true, - images: data.data.map((img) => ({ - url: img.url, - base64: img.b64_json, - })), - }; - } catch (error) { - return { - success: false, - error: error instanceof Error ? error.message : "Unknown error", - }; } + // Grok takes quality low/medium/auto; the 2.0 model defaults to medium. + const quality = + request.quality === "low" || request.quality === "medium" + ? request.quality + : model === "grok-imagine-image-2.0" + ? "medium" + : undefined; + const body: Record = { + model, + prompt: request.prompt, + n: inputs.length ? 1 : (request.count ?? 1), + response_format: "b64_json", + ...(quality ? { quality } : {}), + ...(request.aspectRatio ? { aspect_ratio: request.aspectRatio } : {}), + }; + if (inputs.length) { + const [img] = inputs; + body.image = { url: `data:${img.mimeType};base64,${Buffer.from(img.bytes).toString("base64")}`, type: "image_url" }; + } else if (request.resolution) { + body.resolution = request.resolution.toLowerCase(); + } + const response = await providerRequest(this.imageProvider, `${this.baseUrl}/images/${inputs.length ? "edits" : "generations"}`, { + method: "POST", + headers: { "Content-Type": "application/json", Authorization: `Bearer ${apiKey}` }, + body: JSON.stringify(body), + }); + const data = (await response.json()) as { data?: Array<{ b64_json?: string }> }; + const images = (data.data ?? []) + .filter((img) => img.b64_json) + .map((img) => ({ bytes: fromBase64(img.b64_json!), mimeType: "image/jpeg" })); + if (images.length === 0) { + throw new ProviderError({ kind: "provider", provider: this.imageProvider, message: "Grok returned no image." }); + } + return { images, model }; } + // ── Older interface, kept until every caller uses the contract ──────── + /** - * Edit image using Grok Imagine - * Supports single image input with text instruction-based editing + * Generate image using Grok Imagine. */ - async editImage( - imageBuffer: Buffer, - prompt: string, - options: GrokEditOptions = {} - ): Promise { - if (!this.apiKey) { - return { - success: false, - error: "xAI API key not configured. Set XAI_API_KEY environment variable.", - }; - } - - try { - // Convert buffer to base64 data URI - const base64 = imageBuffer.toString("base64"); - const dataUri = `data:image/png;base64,${base64}`; - - const body: Record = { - model: options.model || "grok-imagine-image", - ...(options.quality ? { quality: options.quality } : {}), + async generateImage(prompt: string, options: GrokImageOptions = {}): Promise { + return this.legacyImage(() => + this.createImage({ prompt, - image: { - url: dataUri, - type: "image_url", - }, - n: 1, - response_format: options.responseFormat || "url", - }; - - if (options.aspectRatio) { - body.aspect_ratio = options.aspectRatio; - } - - const response = await fetch(`${this.baseUrl}/images/edits`, { - method: "POST", - headers: { - "Content-Type": "application/json", - Authorization: `Bearer ${this.apiKey}`, - }, - body: JSON.stringify(body), - }); + model: options.model, + count: options.n, + aspectRatio: options.aspectRatio, + resolution: options.resolution, + quality: options.quality === "auto" ? undefined : options.quality, + }) + ); + } - if (!response.ok) { - const errorText = await response.text(); - let errorMessage = `API error: ${response.status}`; - try { - const errorJson = JSON.parse(errorText); - if (errorJson.error?.message) { - errorMessage = errorJson.error.message; - } - } catch { - if (errorText) { - errorMessage = errorText.substring(0, 200); - } - } - return { - success: false, - error: errorMessage, - }; - } + /** + * Edit one image using Grok Imagine. + */ + async editImage(imageBuffer: Buffer, prompt: string, options: GrokEditOptions = {}): Promise { + return this.legacyImage(() => + this.createImage({ + prompt, + model: options.model, + aspectRatio: options.aspectRatio, + quality: options.quality === "auto" ? undefined : options.quality, + images: [{ bytes: new Uint8Array(imageBuffer), mimeType: "image/png" }], + }) + ); + } - const data = (await response.json()) as { - data: Array<{ - url?: string; - b64_json?: string; - }>; - }; - - return { - success: true, - images: data.data.map((img) => ({ - url: img.url, - base64: img.b64_json, - })), - }; + private async legacyImage(run: () => Promise): Promise { + try { + const result = await run(); + return { success: true, images: result.images.map((img) => ({ base64: Buffer.from(img.bytes).toString("base64") })) }; } catch (error) { - return { - success: false, - error: error instanceof Error ? error.message : "Unknown error", - }; + return { success: false, error: this.legacyMessage(error) }; } } diff --git a/packages/ai-providers/src/image-contracts.test.ts b/packages/ai-providers/src/image-contracts.test.ts index b2b0a864..3b76d0ac 100644 --- a/packages/ai-providers/src/image-contracts.test.ts +++ b/packages/ai-providers/src/image-contracts.test.ts @@ -25,7 +25,7 @@ describe("Gemini image contract", () => { content: { parts: [ { thought: true, inlineData: { mimeType: "image/png", data: "THOUGHT" } }, - { inlineData: { mimeType: "image/jpeg", data: "IMAGE" } }, + { inlineData: { mimeType: "image/jpeg", data: "aW1hZ2U=" } }, ], }, }, @@ -50,13 +50,13 @@ describe("Gemini image contract", () => { expect(body.generationConfig.imageConfig.aspectRatio).toBe("16:9"); expect(body.generationConfig).not.toHaveProperty("temperature"); // Thought images are skipped; the real image keeps its JPEG mime type. - expect(result.images).toEqual([{ base64: "IMAGE", mimeType: "image/jpeg" }]); + expect(result.images).toEqual([{ base64: "aW1hZ2U=", mimeType: "image/jpeg" }]); }); }); describe("OpenAI image contract", () => { it("defaults to gpt-image-2.5-sunburst with medium quality and no response_format", async () => { - const fetchMock = stubFetch({ data: [{ b64_json: "IMG", revised_prompt: "an apple" }] }); + const fetchMock = stubFetch({ data: [{ b64_json: "aW1n", revised_prompt: "an apple" }] }); const openai = new OpenAIImageProvider(); await openai.initialize({ apiKey: "key" }); @@ -67,11 +67,11 @@ describe("OpenAI image contract", () => { expect(body).toMatchObject({ model: "gpt-image-2.5-sunburst", quality: "medium", size: "1024x1024" }); expect(body).not.toHaveProperty("response_format"); expect(body).not.toHaveProperty("style"); - expect(result.images?.[0].base64).toBe("IMG"); + expect(result.images?.[0].base64).toBe("aW1n"); }); it("maps the legacy hd quality name to high", async () => { - const fetchMock = stubFetch({ data: [{ b64_json: "IMG" }] }); + const fetchMock = stubFetch({ data: [{ b64_json: "aW1n" }] }); const openai = new OpenAIImageProvider(); await openai.initialize({ apiKey: "key" }); diff --git a/packages/ai-providers/src/image/contract.ts b/packages/ai-providers/src/image/contract.ts new file mode 100644 index 00000000..cab4235b --- /dev/null +++ b/packages/ai-providers/src/image/contract.ts @@ -0,0 +1,67 @@ +/** + * The image contract every image provider implements. + * + * Image APIs answer in one call, so there is no job: a request goes in and + * images come back as bytes (providers that return URLs are asked for + * base64 instead), or a `ProviderError` is thrown. A request with input + * images is an edit. + */ + +export interface ImageInput { + bytes: Uint8Array; + mimeType: string; +} + +export interface ImageRequest { + prompt: string; + /** Catalog model ID or alias; the provider's default when omitted. */ + model?: string; + /** "16:9", "9:16", "1:1", ... Providers map it to their own size field. */ + aspectRatio?: string; + /** Output size class where the provider has one (Gemini "1K"/"2K"/"4K", Grok "1k"/"2k"). */ + resolution?: string; + quality?: "low" | "medium" | "high"; + /** Images to produce. Default 1. */ + count?: number; + /** Input images: present means edit or compose. */ + images?: ImageInput[]; + /** Edit mask (OpenAI only). */ + mask?: Uint8Array; + /** Provider-specific settings with no shared meaning (Gemini grounding). */ + providerOptions?: Record; +} + +export interface GeneratedImage extends ImageInput { + /** The prompt the provider actually used, when it rewrites prompts. */ + revisedPrompt?: string; +} + +export interface ImageResultSet { + images: GeneratedImage[]; + /** The catalog model ID that produced them. */ + model: string; + /** Text the model returned alongside the images (Gemini). */ + text?: string; +} + +export interface ImageGenerator { + /** Catalog provider id ("openai", "gemini", "grok"). */ + readonly imageProvider: string; + /** Most input images one edit accepts for `model`; 0 means the provider cannot edit. */ + maxEditImages(model?: string): number; + /** Generate, or edit when `request.images` is set. Throws `ProviderError`. */ + createImage(request: ImageRequest): Promise; +} + +/** Base64 to bytes. */ +export function fromBase64(data: string): Uint8Array { + return new Uint8Array(Buffer.from(data, "base64")); +} + +/** Map an aspect ratio to OpenAI's three GPT Image sizes. */ +export function openAiImageSize(aspectRatio?: string): "1024x1024" | "1536x1024" | "1024x1536" | undefined { + if (!aspectRatio) return undefined; + const [w, h] = aspectRatio.split(":").map(Number); + if (!(w > 0 && h > 0)) return undefined; + return w > h ? "1536x1024" : w < h ? "1024x1536" : "1024x1024"; +} diff --git a/packages/ai-providers/src/image/image-contract.test.ts b/packages/ai-providers/src/image/image-contract.test.ts new file mode 100644 index 00000000..9e8ee665 --- /dev/null +++ b/packages/ai-providers/src/image/image-contract.test.ts @@ -0,0 +1,130 @@ +/** + * The image contract for every image provider, over the fake provider + * network: bytes back, model validation and edit limits before any request, + * classified errors, and keys kept out of URLs. + */ +import { afterEach, beforeEach, describe, expect, it } from "vitest"; + +import { FAKE_PNG, FakeProviderNetwork, jsonResponse } from "../testing/fake-provider-network.js"; +import { ProviderError } from "../shared/errors.js"; +import { IMAGE_GENERATOR_PROVIDERS, createImageGenerator } from "./registry.js"; + +const B64 = FAKE_PNG.toString("base64"); +const ROUTES: Record Response; defaultModel: string; maxEdit: number }> = { + openai: { + host: "api.openai.com", + path: /^\/v1\/images\/(generations|edits)$/, + ok: () => jsonResponse({ data: [{ b64_json: B64 }] }), + defaultModel: "gpt-image-2.5-sunburst", + maxEdit: 16, + }, + gemini: { + host: "generativelanguage.googleapis.com", + path: /:generateContent$/, + ok: () => jsonResponse({ candidates: [{ content: { parts: [{ text: "here" }, { inlineData: { mimeType: "image/png", data: B64 } }] } }] }), + defaultModel: "gemini-3.1-flash-image", + maxEdit: 3, + }, + grok: { + host: "api.x.ai", + path: /^\/v1\/images\/(generations|edits)$/, + ok: () => jsonResponse({ data: [{ b64_json: B64 }] }), + defaultModel: "grok-imagine-image", + maxEdit: 1, + }, +}; + +let net: FakeProviderNetwork; +beforeEach(() => { + net = new FakeProviderNetwork(); + net.install(); +}); +afterEach(() => net.uninstall()); + +async function rejection(promise: Promise): Promise { + try { + await promise; + } catch (error) { + expect(error).toBeInstanceOf(ProviderError); + return error as ProviderError; + } + throw new Error("expected a ProviderError"); +} + +describe.each(IMAGE_GENERATOR_PROVIDERS)("%s meets the image contract", (provider) => { + const route = ROUTES[provider]; + + it("generates image bytes on its default model", async () => { + net.on("POST", route.host, route.path, route.ok); + const generator = await createImageGenerator(provider, "test-key"); + const result = await generator.createImage({ prompt: "a paper boat", aspectRatio: "16:9" }); + + expect(result.model).toBe(route.defaultModel); + expect(Buffer.from(result.images[0].bytes)).toEqual(FAKE_PNG); + expect(net.requests[0].url).not.toContain("key="); + }); + + it("edits with input images, up to its limit, refusing more before sending", async () => { + net.on("POST", route.host, route.path, route.ok); + const generator = await createImageGenerator(provider, "test-key"); + const image = { bytes: new Uint8Array(FAKE_PNG), mimeType: "image/png" }; + + await generator.createImage({ prompt: "make it night", images: [image] }); + expect(net.requests).toHaveLength(1); + expect(generator.maxEditImages()).toBe(route.maxEdit); + + const error = await rejection( + generator.createImage({ prompt: "x", images: Array(route.maxEdit + 1).fill(image) }) + ); + expect(error.kind).toBe("invalid-request"); + expect(net.requests).toHaveLength(1); + }); + + it("rejects an unknown model before sending", async () => { + const generator = await createImageGenerator(provider, "test-key"); + const error = await rejection(generator.createImage({ prompt: "x", model: "no-such-model" })); + expect(error.kind).toBe("invalid-request"); + expect(net.requests).toHaveLength(0); + }); + + it("classifies a bad key", async () => { + net.on("POST", route.host, route.path, () => jsonResponse({ error: { message: "Incorrect API key provided" } }, 401)); + const generator = await createImageGenerator(provider, "test-key"); + const error = await rejection(generator.createImage({ prompt: "x" })); + expect(error).toMatchObject({ kind: "auth", status: 401 }); + }); +}); + +describe("image provider specifics", () => { + it("OpenAI maps the aspect ratio to a GPT Image size", async () => { + net.on("POST", ROUTES.openai.host, ROUTES.openai.path, ROUTES.openai.ok); + const openai = await createImageGenerator("openai", "k"); + await openai.createImage({ prompt: "x", aspectRatio: "9:16", quality: "high" }); + expect(net.requests[0].body).toMatchObject({ size: "1024x1536", quality: "high", model: "gpt-image-2.5-sunburst" }); + }); + + it("Gemini sends the key as a header and reports a safety block as moderation", async () => { + net.on("POST", ROUTES.gemini.host, ROUTES.gemini.path, () => jsonResponse({ promptFeedback: { blockReason: "SAFETY" } })); + const gemini = await createImageGenerator("gemini", "k"); + const error = await rejection(gemini.createImage({ prompt: "x" })); + + expect(error.kind).toBe("moderation"); + expect(net.requests[0].headers["x-goog-api-key"]).toBe("k"); + }); + + it("Gemini Pro edits up to 14 images", async () => { + const gemini = await createImageGenerator("gemini", "k"); + expect(gemini.maxEditImages("pro")).toBe(14); + }); + + it("Grok asks for base64 and sends quality medium for the 2.0 model", async () => { + net.on("POST", ROUTES.grok.host, ROUTES.grok.path, ROUTES.grok.ok); + const grok = await createImageGenerator("grok", "k"); + await grok.createImage({ prompt: "x", model: "2.0", resolution: "2K" }); + expect(net.requests[0].body).toMatchObject({ model: "grok-imagine-image-2.0", quality: "medium", response_format: "b64_json", resolution: "2k" }); + }); + + it("refuses an unknown provider", async () => { + await expect(createImageGenerator("midjourney", "k")).rejects.toMatchObject({ kind: "invalid-request" }); + }); +}); diff --git a/packages/ai-providers/src/image/index.ts b/packages/ai-providers/src/image/index.ts new file mode 100644 index 00000000..fa4577bd --- /dev/null +++ b/packages/ai-providers/src/image/index.ts @@ -0,0 +1,2 @@ +export * from "./contract.js"; +export * from "./registry.js"; diff --git a/packages/ai-providers/src/image/registry.ts b/packages/ai-providers/src/image/registry.ts new file mode 100644 index 00000000..09f19f99 --- /dev/null +++ b/packages/ai-providers/src/image/registry.ts @@ -0,0 +1,31 @@ +/** One way to get an image provider by its catalog id. */ + +import { GeminiProvider } from "../gemini/GeminiProvider.js"; +import { GrokProvider } from "../grok/GrokProvider.js"; +import { OpenAIImageProvider } from "../openai-image/OpenAIImageProvider.js"; +import { ProviderError } from "../shared/errors.js"; +import type { ImageGenerator } from "./contract.js"; + +export const IMAGE_GENERATOR_PROVIDERS = ["openai", "gemini", "grok"] as const; +export type ImageGeneratorProvider = (typeof IMAGE_GENERATOR_PROVIDERS)[number]; + +/** A ready image generator for `provider`. */ +export async function createImageGenerator(provider: string, apiKey: string): Promise { + const generator = + provider === "openai" + ? new OpenAIImageProvider() + : provider === "gemini" + ? new GeminiProvider() + : provider === "grok" + ? new GrokProvider() + : undefined; + if (!generator) { + throw new ProviderError({ + kind: "invalid-request", + provider, + message: `Unknown image provider "${provider}". Available: ${IMAGE_GENERATOR_PROVIDERS.join(", ")}.`, + }); + } + await generator.initialize({ apiKey }); + return generator; +} diff --git a/packages/ai-providers/src/index.ts b/packages/ai-providers/src/index.ts index 282906e1..f338efcf 100644 --- a/packages/ai-providers/src/index.ts +++ b/packages/ai-providers/src/index.ts @@ -74,6 +74,7 @@ export { type ModelStatus, } from "./catalog/index.js"; export * from "./video/index.js"; +export * from "./image/index.js"; export { VIDEO_GENERATOR_PROVIDERS, createVideoGenerator, type VideoGeneratorProvider } from "./video/registry.js"; export { OpenAIProvider, openaiProvider } from "./openai/index.js"; export { ClaudeProvider, claudeProvider } from "./claude/index.js"; diff --git a/packages/ai-providers/src/openai-image/OpenAIImageProvider.ts b/packages/ai-providers/src/openai-image/OpenAIImageProvider.ts index a100c0f0..84ffa25f 100644 --- a/packages/ai-providers/src/openai-image/OpenAIImageProvider.ts +++ b/packages/ai-providers/src/openai-image/OpenAIImageProvider.ts @@ -4,6 +4,10 @@ import type { ProviderConfig, } from "../interface/types.js"; import { defaultModel } from "../catalog/catalog.js"; +import { fromBase64, openAiImageSize, type ImageGenerator, type ImageRequest, type ImageResultSet } from "../image/contract.js"; +import { ProviderError, isProviderError } from "../shared/errors.js"; +import { providerRequest } from "../shared/http.js"; +import { resolveCatalogModel } from "../video/models.js"; /** * GPT Image model types @@ -75,18 +79,18 @@ export interface ImageEditOptions { /** Default text-to-image model. */ export const OPENAI_IMAGE_DEFAULT_MODEL = defaultModel("openai", "image").id as GPTImageModel; -const DEFAULT_MODEL = OPENAI_IMAGE_DEFAULT_MODEL; /** * OpenAI Image provider (GPT Image 1.5 / GPT Image 2 / DALL-E) */ -export class OpenAIImageProvider implements AIProvider { +export class OpenAIImageProvider implements AIProvider, ImageGenerator { id = "openai-image"; name = "OpenAI GPT Image"; description = "AI image generation with GPT Image 1.5 (default) and GPT Image 2 (opt-in)"; capabilities: AICapability[] = ["text-to-image", "background-removal", "image-editing"]; iconUrl = "/icons/openai.svg"; isAvailable = true; + readonly imageProvider = "openai"; private apiKey?: string; private baseUrl = "https://api.openai.com/v1"; @@ -106,94 +110,81 @@ export class OpenAIImageProvider implements AIProvider { * Generate images from text prompt * Uses GPT Image 2.5 Sunburst by default */ - async generateImage( - prompt: string, - options: ImageOptions = {} - ): Promise { - if (!this.apiKey) { - return { - success: false, - error: "OpenAI API key not configured", - }; - } + // ── ImageGenerator ──────────────────────────────────────────────────── - const model = options.model || DEFAULT_MODEL; + /** GPT Image edits take up to 16 input images. */ + maxEditImages(): number { + return 16; + } - try { - // Build request body based on model + async createImage(request: ImageRequest): Promise { + if (!this.apiKey) { + throw new ProviderError({ kind: "auth", provider: this.imageProvider, message: "OpenAI API key not configured" }); + } + const model = resolveCatalogModel(this.imageProvider, "image", request.model).id; + const size = openAiSize(request); + let response: Response; + if (request.images?.length) { + if (request.images.length > this.maxEditImages()) { + throw new ProviderError({ + kind: "invalid-request", + provider: this.imageProvider, + message: `GPT Image edits take up to ${this.maxEditImages()} images, not ${request.images.length}.`, + }); + } + const form = new FormData(); + form.append("model", model); + form.append("prompt", request.prompt); + for (const image of request.images) { + form.append("image[]", new Blob([new Uint8Array(image.bytes)], { type: image.mimeType }), "image.png"); + } + if (request.mask) form.append("mask", new Blob([new Uint8Array(request.mask)], { type: "image/png" }), "mask.png"); + if (request.quality) form.append("quality", request.quality); + if (size) form.append("size", size); + if (request.count) form.append("n", String(request.count)); + response = await providerRequest(this.imageProvider, `${this.baseUrl}/images/edits`, { + method: "POST", + headers: { Authorization: `Bearer ${this.apiKey}` }, + body: form, + }); + } else { const body: Record = { model, - prompt, - n: options.n || 1, - }; - - // GPT Image models do NOT support response_format. - // Quality values: low, medium, high, auto. - const qualityMap: Record = { - standard: "medium", - hd: "high", + prompt: request.prompt, + n: request.count ?? 1, + // GPT Image models take quality low/medium/high and no response_format. + quality: request.quality ?? "medium", }; - const quality = options.quality || "medium"; - body.quality = qualityMap[quality] || quality; - if (options.size && options.size !== "auto") { - body.size = options.size; - } - - const response = await fetch(`${this.baseUrl}/images/generations`, { + if (size) body.size = size; + response = await providerRequest(this.imageProvider, `${this.baseUrl}/images/generations`, { method: "POST", - headers: { - "Content-Type": "application/json", - Authorization: `Bearer ${this.apiKey}`, - }, + headers: { "Content-Type": "application/json", Authorization: `Bearer ${this.apiKey}` }, body: JSON.stringify(body), }); + } + const data = (await response.json()) as { data?: Array<{ b64_json?: string; revised_prompt?: string }> }; + const images = (data.data ?? []) + .filter((img) => img.b64_json) + .map((img) => ({ bytes: fromBase64(img.b64_json!), mimeType: "image/png", revisedPrompt: img.revised_prompt })); + if (images.length === 0) { + throw new ProviderError({ kind: "provider", provider: this.imageProvider, message: "OpenAI returned no image." }); + } + return { images, model }; + } - if (!response.ok) { - const errorText = await response.text(); - console.error("OpenAI Image API error:", errorText); - - // Parse error to get detailed message - let errorMessage = `API error: ${response.status}`; - try { - const errorJson = JSON.parse(errorText); - if (errorJson.error?.message) { - errorMessage = errorJson.error.message; - } - } catch { - // If not JSON, use the raw text - if (errorText) { - errorMessage = errorText.substring(0, 200); - } - } - - return { - success: false, - error: errorMessage, - }; - } - - const data = (await response.json()) as { - data: Array<{ - url?: string; - b64_json?: string; - revised_prompt?: string; - }>; - }; + // ── Older interface, kept until every caller uses the contract ──────── - return { - success: true, - images: data.data.map((img) => ({ - url: img.url, - base64: img.b64_json, - revisedPrompt: img.revised_prompt, - })), - }; - } catch (error) { - return { - success: false, - error: error instanceof Error ? error.message : "Unknown error", - }; - } + async generateImage(prompt: string, options: ImageOptions = {}): Promise { + const quality = options.quality === "standard" ? "medium" : options.quality === "hd" ? "high" : options.quality; + return this.legacy(() => + this.createImage({ + prompt, + model: options.model, + count: options.n, + quality, + providerOptions: options.size && options.size !== "auto" ? { size: options.size } : undefined, + }) + ); } /** @@ -248,96 +239,44 @@ export class OpenAIImageProvider implements AIProvider { } /** - * Edit images using GPT Image 1.5 - * Supports up to 16 input images with text instruction-based editing + * Edit images (up to 16 inputs) with a text instruction. */ - async editImage( - imageBuffers: Buffer[], - prompt: string, - options: ImageEditOptions = {} - ): Promise { - if (!this.apiKey) { - return { - success: false, - error: "OpenAI API key not configured", - }; - } + async editImage(imageBuffers: Buffer[], prompt: string, options: ImageEditOptions = {}): Promise { + return this.legacy(() => + this.createImage({ + prompt, + model: options.model, + quality: options.quality, + count: options.n, + mask: options.mask, + images: imageBuffers.map((b) => ({ bytes: new Uint8Array(b), mimeType: "image/png" })), + providerOptions: options.size ? { size: options.size } : undefined, + }) + ); + } + private async legacy(run: () => Promise): Promise { try { - const formData = new FormData(); - formData.append("model", options.model || DEFAULT_MODEL); - formData.append("prompt", prompt); - - // Add images (up to 16) - for (const buf of imageBuffers) { - const uint8Array = new Uint8Array(buf); - formData.append("image[]", new Blob([uint8Array], { type: "image/png" }), "image.png"); - } - - if (options.mask) { - const maskUint8 = new Uint8Array(options.mask); - formData.append("mask", new Blob([maskUint8], { type: "image/png" }), "mask.png"); - } - - if (options.quality) { - formData.append("quality", options.quality); - } - - if (options.size) { - formData.append("size", options.size); - } - - const response = await fetch(`${this.baseUrl}/images/edits`, { - method: "POST", - headers: { - Authorization: `Bearer ${this.apiKey}`, - }, - body: formData, - }); - - if (!response.ok) { - const errorText = await response.text(); - let errorMessage = `API error: ${response.status}`; - try { - const errorJson = JSON.parse(errorText); - if (errorJson.error?.message) { - errorMessage = errorJson.error.message; - } - } catch { - if (errorText) { - errorMessage = errorText.substring(0, 200); - } - } - return { - success: false, - error: errorMessage, - }; - } - - const data = (await response.json()) as { - data: Array<{ - b64_json?: string; - url?: string; - revised_prompt?: string; - }>; - }; - + const result = await run(); return { success: true, - images: data.data.map((img) => ({ - base64: img.b64_json, - url: img.url, - revisedPrompt: img.revised_prompt, + images: result.images.map((img) => ({ + base64: Buffer.from(img.bytes).toString("base64"), + revisedPrompt: img.revisedPrompt, })), }; } catch (error) { - return { - success: false, - error: error instanceof Error ? error.message : "Unknown error", - }; + return { success: false, error: isProviderError(error) || error instanceof Error ? error.message : "Unknown error" }; } } } export const openaiImageProvider = new OpenAIImageProvider(); + +/** An explicit `size` wins; otherwise the aspect ratio picks one of the three GPT Image sizes. */ +function openAiSize(request: ImageRequest): string | undefined { + const explicit = request.providerOptions?.size; + if (typeof explicit === "string" && explicit !== "auto") return explicit; + return openAiImageSize(request.aspectRatio); +} diff --git a/packages/ai-providers/src/video/contract.ts b/packages/ai-providers/src/video/contract.ts index 99d635bb..81bf9c58 100644 --- a/packages/ai-providers/src/video/contract.ts +++ b/packages/ai-providers/src/video/contract.ts @@ -68,13 +68,13 @@ export interface VideoJobState { } /** How a provider takes an input image: a public URL, an inline data URI, or either. */ -export type ImageInput = "url" | "data-uri" | "either"; +export type VideoImageInput = "url" | "data-uri" | "either"; export interface VideoGenerator { /** Provider registry id; matches `VideoJob.provider`. */ readonly id: string; /** How `VideoRequest.image` must be given. Callers upload or inline local files to match. */ - readonly imageInput: ImageInput; + readonly imageInput: VideoImageInput; /** Start a generation. Throws `ProviderError` when the provider refuses it. */ submitVideo(request: VideoRequest): Promise; /** Read a job's current state. Throws `ProviderError` only when the state cannot be read. */ diff --git a/packages/ai-providers/src/video/models.ts b/packages/ai-providers/src/video/models.ts index 7b051f5d..da69fbc8 100644 --- a/packages/ai-providers/src/video/models.ts +++ b/packages/ai-providers/src/video/models.ts @@ -1,23 +1,29 @@ -import { assertModelServed, findModel, listModels, type ModelSpec } from "../catalog/catalog.js"; +import { assertModelServed, findModel, listModels, type ModelKind, type ModelSpec } from "../catalog/catalog.js"; import { ProviderError } from "../shared/errors.js"; /** - * The catalog video model a request names (alias or ID; the provider's - * default when omitted). Throws `invalid-request` for an unknown model and + * The catalog model a request names (alias or ID; the provider's default + * when omitted). Throws `invalid-request` for an unknown model and * `model-retired` for one past its shutdown date, before any request. */ -export function resolveVideoModel(provider: string, model?: string): ModelSpec { - const spec = findModel(provider, "video", model); +export function resolveCatalogModel(provider: string, kind: ModelKind, model?: string): ModelSpec { + const spec = findModel(provider, kind, model); if (!spec) { - const valid = listModels({ provider, kind: "video" }) - .flatMap((m) => [m.id, ...(m.aliases ?? [])]) + const valid = listModels({ provider, kind }) + .filter((m) => m.status !== "deprecated" || !m.shutdown || Date.parse(`${m.shutdown}T00:00:00Z`) > Date.now()) + .flatMap((m) => [...(m.aliases ?? []), m.id]) .join(", "); throw new ProviderError({ kind: "invalid-request", provider, - message: `Unknown ${provider} video model "${model}". Valid: ${valid}.`, + message: `Unknown ${provider} ${kind} model "${model}". Valid: ${valid}.`, }); } assertModelServed(spec); return spec; } + +/** `resolveCatalogModel` for video. */ +export function resolveVideoModel(provider: string, model?: string): ModelSpec { + return resolveCatalogModel(provider, "video", model); +} diff --git a/packages/cli/src/commands/__snapshots__/envelope-snapshots.test.ts.snap b/packages/cli/src/commands/__snapshots__/envelope-snapshots.test.ts.snap index b5cd6937..9f5dd1e6 100644 --- a/packages/cli/src/commands/__snapshots__/envelope-snapshots.test.ts.snap +++ b/packages/cli/src/commands/__snapshots__/envelope-snapshots.test.ts.snap @@ -1092,8 +1092,7 @@ exports[`CLI --describe schemas (drift detection) > vibe edit image --describe 1 "type": "array", }, "model": { - "default": "flash", - "description": "Model: flash, lite, pro (Gemini); pro (Grok)", + "description": "Model: flash (default), lite, pro (Gemini); 2.5, flare, 2 (OpenAI); pro (Grok)", "type": "string", }, "output": { @@ -1727,12 +1726,12 @@ exports[`CLI --describe schemas (drift detection) > vibe edit upscale --describe exports[`CLI --describe schemas (drift detection) > vibe generate image --describe 1`] = ` { "cost": "high", - "description": "Generate image using AI (Gemini, OpenAI gpt-image, Grok, or Runway)", + "description": "Generate image using AI (OpenAI GPT Image, Gemini, or Grok)", "name": "generate.image", "parameters": { "properties": { "apiKey": { - "description": "API key (or set env: OPENAI_API_KEY, GOOGLE_API_KEY)", + "description": "API key (or set env: OPENAI_API_KEY, GOOGLE_API_KEY, XAI_API_KEY)", "type": "string", }, "count": { @@ -1745,11 +1744,11 @@ exports[`CLI --describe schemas (drift detection) > vibe generate image --descri "type": "boolean", }, "model": { - "description": "Model. Gemini: flash (default), lite, pro. OpenAI: 2.5 (default), flare, 2, 1.5. Grok: pro", + "description": "Model. OpenAI: 2.5 (default), flare, 2. Gemini: flash (default), lite, pro. Grok: pro", "type": "string", }, "output": { - "description": "Output file path (downloads image)", + "description": "Output file path", "type": "string", }, "prompt": { @@ -1757,41 +1756,34 @@ exports[`CLI --describe schemas (drift detection) > vibe generate image --descri "type": "string", }, "provider": { - "description": "Provider: openai (default when OPENAI_API_KEY set), gemini, grok, runway", + "description": "Provider: openai (default when OPENAI_API_KEY set), gemini, grok", "enum": [ "openai", "gemini", "grok", - "runway", ], "type": "string", }, "quality": { - "default": "standard", - "description": "Quality: standard, hd (openai only)", + "description": "Quality: low, medium, high (OpenAI and Grok; standard/hd still accepted)", "enum": [ - "standard", - "hd", + "low", + "medium", + "high", ], "type": "string", }, "ratio": { "default": "1:1", - "description": "Aspect ratio (gemini: 1:1, 1:4, 1:8, 4:1, 8:1, 16:9, 9:16, 3:4, 4:3, etc.)", + "description": "Aspect ratio: 1:1, 16:9, 9:16, 4:3, 3:4, ... (OpenAI maps it to its nearest size)", "type": "string", }, - "size": { - "default": "1024x1024", - "description": "Image size (openai: 1024x1024, 1536x1024, 1024x1536)", + "resolution": { + "description": "Gemini 1K/2K/4K (2K and 4K need Pro) or Grok 1k/2k", "type": "string", }, - "style": { - "default": "vivid", - "description": "Style: vivid, natural (openai only)", - "enum": [ - "vivid", - "natural", - ], + "size": { + "description": "Explicit OpenAI size: 1024x1024, 1536x1024, 1024x1536 (overrides --ratio)", "type": "string", }, }, @@ -3241,10 +3233,11 @@ exports[`CLI --describe schemas (drift detection) > vibe scene add --describe 1` }, "imageProvider": { "default": "gemini", - "description": "Image provider: gemini, openai", + "description": "Image provider: gemini, openai, grok", "enum": [ "gemini", "openai", + "grok", ], "type": "string", }, @@ -4368,12 +4361,11 @@ exports[`envelope shape (--dry-run --json) > generate image (low cost, OpenAI de "costUsd": 0.211, "data": { "params": { - "count": "1", + "count": 1, + "model": "gpt-image-2.5-sunburst", "prompt": "test prompt", "provider": "openai", - "quality": "standard", "ratio": "1:1", - "size": "1024x1024", }, }, "dryRun": true, diff --git a/packages/cli/src/commands/__snapshots__/error-envelopes.test.ts.snap b/packages/cli/src/commands/__snapshots__/error-envelopes.test.ts.snap index 848cdb9e..eae98ffb 100644 --- a/packages/cli/src/commands/__snapshots__/error-envelopes.test.ts.snap +++ b/packages/cli/src/commands/__snapshots__/error-envelopes.test.ts.snap @@ -58,3 +58,19 @@ exports[`structured error envelope (exitWithError → stderr) > generate image "success": false, } `; + +exports[`structured error envelope (exitWithError → stderr) > generate image — no API key, hermetic (API_KEY_MISSING) 1`] = ` +{ + "code": "API_KEY_MISSING", + "error": "OpenAI API key required.", + "exitCode": 4, + "message": "OpenAI API key required.", + "recoverable": false, + "retryWith": [ + "Set OPENAI_API_KEY in .env, or run: vibe setup. Check available features: vibe doctor --json. Get key: https://platform.openai.com/api-keys", + ], + "retryable": false, + "success": false, + "suggestion": "Set OPENAI_API_KEY in .env, or run: vibe setup. Check available features: vibe doctor --json. Get key: https://platform.openai.com/api-keys", +} +`; diff --git a/packages/cli/src/commands/_shared/image-jobs.ts b/packages/cli/src/commands/_shared/image-jobs.ts new file mode 100644 index 00000000..f0dc5171 --- /dev/null +++ b/packages/cli/src/commands/_shared/image-jobs.ts @@ -0,0 +1,72 @@ +/** + * The CLI side of the image contract: open an image provider by name with + * the user's key, read input images, and save results. Image commands, + * tools, scenes, and builds go through here instead of branching on + * provider names. + */ + +import { existsSync } from "node:fs"; +import { mkdir, readFile } from "node:fs/promises"; +import { dirname, resolve } from "node:path"; +import { ProviderError, createImageGenerator, findModel, type ImageGenerator, type ImageInput } from "@vibeframe/ai-providers"; +import { getConfiguredApiKey } from "../../utils/api-key.js"; +import { writeImageFile } from "../../utils/image-file.js"; +import { mimeTypeForPath } from "./video-jobs.js"; + +/** Env var holding each image provider's key. */ +export const IMAGE_PROVIDER_ENV: Readonly> = { + openai: "OPENAI_API_KEY", + gemini: "GOOGLE_API_KEY", + grok: "XAI_API_KEY", +}; + +/** Display names for spinners and messages. */ +export const IMAGE_PROVIDER_LABELS: Readonly> = { + openai: "OpenAI", + gemini: "Google Gemini", + grok: "xAI Grok", +}; + +/** An image generator for `provider`, with the given key or the configured one. */ +export async function openImageGenerator(provider: string, apiKey?: string): Promise { + const envVar = IMAGE_PROVIDER_ENV[provider]; + if (!envVar) { + throw new ProviderError({ + kind: "invalid-request", + provider, + message: `Unknown image provider "${provider}". Available: ${Object.keys(IMAGE_PROVIDER_ENV).join(", ")}.`, + }); + } + const key = await getConfiguredApiKey(envVar, apiKey); + if (!key) throw new ProviderError({ kind: "auth", provider, message: `${envVar} required for ${provider}.` }); + return createImageGenerator(provider, key); +} + +/** A local image file as contract input. */ +export async function readImageInput(path: string): Promise { + const absPath = resolve(process.cwd(), path); + if (!existsSync(absPath)) { + throw new ProviderError({ kind: "invalid-request", provider: "local", message: `Image not found: ${absPath}` }); + } + return { bytes: new Uint8Array(await readFile(absPath)), mimeType: mimeTypeForPath(path, "image/png") }; +} + +/** Save image bytes to `output` (relative to cwd), converting to the extension's format. */ +export async function saveImage(output: string, bytes: Uint8Array): Promise { + const outputPath = resolve(process.cwd(), output); + await mkdir(dirname(outputPath), { recursive: true }); + await writeImageFile(outputPath, Buffer.from(bytes)); + return outputPath; +} + +/** The catalog label for a model ID, for human output. */ +export function imageModelLabel(provider: string, modelId: string): string { + return findModel(provider, "image", modelId)?.label ?? modelId; +} + +/** CLI quality words to the contract's: `standard`/`hd` are the old OpenAI names. */ +export function contractQuality(quality?: string): "low" | "medium" | "high" | undefined { + if (quality === "standard") return "medium"; + if (quality === "hd") return "high"; + return quality === "low" || quality === "medium" || quality === "high" ? quality : undefined; +} diff --git a/packages/cli/src/commands/_shared/openai-image.ts b/packages/cli/src/commands/_shared/openai-image.ts index 30104288..74cb8182 100644 --- a/packages/cli/src/commands/_shared/openai-image.ts +++ b/packages/cli/src/commands/_shared/openai-image.ts @@ -1,40 +1,12 @@ /** * @module _shared/openai-image * - * Shared OpenAI `generateImage` helper for `vibe ai image` and - * `vibe generate image`. Both commands had their own near-identical - * provider initialisation + model-alias parsing + result construction. - * v0.52.0 added gpt-image-2 wiring to `ai-image.ts` only and - * `vibe generate image -m 2` silently fell back to gpt-image-1.5 — the - * duplication guarantees a recurrence next time the OpenAI image flow - * is touched. - * - * Each top-level command keeps its own CLI wiring and output formatting; - * this helper owns just the API call + label derivation. - * - * See issue #58. + * OpenAI image model alias resolution, used by image cost estimates. + * Generation itself goes through the image contract (`executeImageGenerate`). */ -import { defaultModel, findModel, OpenAIImageProvider } from "@vibeframe/ai-providers"; -import type { GPTImageModel, ImageOptions, ImageResult } from "@vibeframe/ai-providers"; - -/** Subset of CLI options consumed by the helper. */ -export interface OpenAIImageHelperOptions { - model?: string; - size?: string; - quality?: string; - style?: string; - /** Stringified count from commander; parsed inside the helper. */ - count?: string; -} - -export interface OpenAIImageHelperResult { - result: ImageResult; - /** Resolved model id passed to the API. */ - openaiModel: GPTImageModel; - /** Human-friendly label for spinner / success output. */ - modelLabel: string; -} +import { defaultModel, findModel } from "@vibeframe/ai-providers"; +import type { GPTImageModel } from "@vibeframe/ai-providers"; /** * Resolve the user-supplied model alias to the API id + display label, from @@ -49,35 +21,3 @@ export function resolveOpenAIImageModel(modelAlias?: string): { const model = findModel("openai", "image", modelAlias) ?? defaultModel("openai", "image"); return { openaiModel: model.id as GPTImageModel, modelLabel: model.label }; } - -/** - * Run an OpenAI image generation. Caller owns: api-key acquisition, - * spinner lifecycle, output formatting (JSON vs human), and file save. - */ -export async function executeOpenAIImageGenerate( - prompt: string, - options: OpenAIImageHelperOptions, - ctx: { apiKey: string }, -): Promise { - const provider = new OpenAIImageProvider(); - await provider.initialize({ apiKey: ctx.apiKey }); - - const { openaiModel, modelLabel } = resolveOpenAIImageModel(options.model); - - // commander hands back unconstrained strings for `--size` / `--quality` - // / `--style`. The provider's `ImageOptions` narrows these to specific - // unions ("1024x1024" | "1536x1024" | …, etc.). The original duplicated - // handlers were `any`-typed via commander's loose `OptionValues`, which - // is why this never tripped before the dedup; the helper makes the - // shape explicit, so we cast at the single call boundary instead of - // ANY-laundering through the call sites. - const result = await provider.generateImage(prompt, { - model: openaiModel, - size: options.size as ImageOptions["size"], - quality: options.quality as ImageOptions["quality"], - style: options.style as ImageOptions["style"], - n: options.count !== undefined ? parseInt(options.count, 10) : undefined, - }); - - return { result, openaiModel, modelLabel }; -} diff --git a/packages/cli/src/commands/_shared/scene-build.test.ts b/packages/cli/src/commands/_shared/scene-build.test.ts index cf1d2b11..f0fbbf2a 100644 --- a/packages/cli/src/commands/_shared/scene-build.test.ts +++ b/packages/cli/src/commands/_shared/scene-build.test.ts @@ -22,9 +22,7 @@ vi.mock("./tts-resolve.js", () => ({ vi.mock("@vibeframe/ai-providers", async (importOriginal) => ({ ...(await importOriginal()), - GeminiProvider: vi.fn(), - GrokProvider: vi.fn(), - OpenAIImageProvider: vi.fn(), + createImageGenerator: vi.fn(), })); // Stub only the Whisper network call; keep the pure path/read helpers real so @@ -51,7 +49,7 @@ vi.mock("../generate/music.js", () => ({ })); import { resolveTtsProvider } from "./tts-resolve.js"; -import { GeminiProvider, GrokProvider, OpenAIImageProvider } from "@vibeframe/ai-providers"; +import { createImageGenerator, type ImageGenerator } from "@vibeframe/ai-providers"; import { transcribeNarrationWords } from "./transcribe-narration.js"; import { executeSceneRender } from "./scene-render.js"; import { executeFootageAssemble } from "./footage-assemble.js"; @@ -126,35 +124,18 @@ beforeEach(() => { }), }); - vi.mocked(OpenAIImageProvider).mockImplementation( - () => + // Each image provider returns distinct bytes, so tests can tell which one ran. + const imageBytes: Record = { openai: [5, 6, 7, 8], gemini: [9, 10, 11, 12], grok: [13, 14, 15, 16] }; + vi.mocked(createImageGenerator).mockImplementation( + async (provider: string) => ({ - initialize: vi.fn().mockResolvedValue(undefined), - generateImage: vi.fn().mockResolvedValue({ - success: true, - images: [{ base64: Buffer.from([5, 6, 7, 8]).toString("base64") }], + imageProvider: provider, + maxEditImages: () => 3, + createImage: vi.fn().mockResolvedValue({ + images: [{ bytes: new Uint8Array(imageBytes[provider]), mimeType: "image/png" }], + model: `${provider}-model`, }), - }) as unknown as InstanceType - ); - vi.mocked(GeminiProvider).mockImplementation( - () => - ({ - initialize: vi.fn().mockResolvedValue(undefined), - generateImage: vi.fn().mockResolvedValue({ - success: true, - images: [{ base64: Buffer.from([9, 10, 11, 12]).toString("base64") }], - }), - }) as unknown as InstanceType - ); - vi.mocked(GrokProvider).mockImplementation( - () => - ({ - initialize: vi.fn().mockResolvedValue(undefined), - generateImage: vi.fn().mockResolvedValue({ - success: true, - images: [{ base64: Buffer.from([13, 14, 15, 16]).toString("base64") }], - }), - }) as unknown as InstanceType + }) as ImageGenerator ); vi.mocked(executeSceneRender).mockResolvedValue({ @@ -392,19 +373,19 @@ describe("executeSceneBuild", () => { expect(r.success).toBe(true); expect(r.beats[0].backdropStatus).toBe("generated"); - expect(OpenAIImageProvider).toHaveBeenCalled(); + expect(createImageGenerator).toHaveBeenCalledWith("openai", "test-key-from-dotenv"); }); it.each([ - ["gemini", GeminiProvider, [9, 10, 11, 12]], - ["grok", GrokProvider, [13, 14, 15, 16]], - ] as const)("dispatches backdrop generation with %s", async (imageProvider, Provider, bytes) => { + ["gemini", [9, 10, 11, 12]], + ["grok", [13, 14, 15, 16]], + ] as const)("dispatches backdrop generation with %s", async (imageProvider, bytes) => { const r = await executeSceneBuild({ projectDir, stage: "assets", imageProvider }); expect(r.success).toBe(true); expect(r.beats[0].backdropStatus).toBe("generated"); - expect(Provider).toHaveBeenCalled(); - expect(OpenAIImageProvider).not.toHaveBeenCalled(); + expect(createImageGenerator).toHaveBeenCalledWith(imageProvider, expect.any(String)); + expect(createImageGenerator).not.toHaveBeenCalledWith("openai", expect.anything()); expect(Array.from(readFileSync(join(projectDir, "assets", "backdrop-hook.png")))).toEqual( bytes ); @@ -456,7 +437,7 @@ describe("executeSceneBuild", () => { expect(r.beats[0].narrationStatus).toBe("skipped"); expect(r.beats[0].backdropStatus).toBe("skipped"); expect(resolveTtsProvider).not.toHaveBeenCalled(); - expect(OpenAIImageProvider).not.toHaveBeenCalled(); + expect(createImageGenerator).not.toHaveBeenCalled(); }); it("runs `--stage transcript` standalone over on-disk narration and reports `transcript-only`", async () => { @@ -546,7 +527,7 @@ asset: "assets/frame.png" expect(r.beats[0].backdropPath).toBe("assets/frame.png"); expect(r.beats[0].backdropSourcePath).toBe("assets/frame.png"); expect(resolveTtsProvider).not.toHaveBeenCalled(); - expect(OpenAIImageProvider).not.toHaveBeenCalled(); + expect(createImageGenerator).not.toHaveBeenCalled(); const report = readBuildReport(); expectBuildReportContract(report, { @@ -601,7 +582,7 @@ backdrop: "../outside.png" expect(r.beats[0].backdropError).toBe( 'Asset reference "../outside.png" must stay inside the project directory.' ); - expect(OpenAIImageProvider).not.toHaveBeenCalled(); + expect(createImageGenerator).not.toHaveBeenCalled(); const report = readBuildReport(); expectBuildReportContract(report, { @@ -830,7 +811,7 @@ backdrop: "This should not dispatch." `vibe storyboard revise ${projectDir} --from "" --dry-run --json`, ]); expect(resolveTtsProvider).not.toHaveBeenCalled(); - expect(OpenAIImageProvider).not.toHaveBeenCalled(); + expect(createImageGenerator).not.toHaveBeenCalled(); expect(executeSceneRender).not.toHaveBeenCalled(); const report = readBuildReport(); diff --git a/packages/cli/src/commands/_shared/scene-build.ts b/packages/cli/src/commands/_shared/scene-build.ts index 7a4135ed..797fde48 100644 --- a/packages/cli/src/commands/_shared/scene-build.ts +++ b/packages/cli/src/commands/_shared/scene-build.ts @@ -25,13 +25,8 @@ import { dirname, join, resolve } from "node:path"; import { config as loadDotenv } from "dotenv"; import { - GeminiProvider, - GrokProvider, - OPENAI_IMAGE_DEFAULT_MODEL, - OpenAIImageProvider, + createImageGenerator, estimateSeedanceVideoCostUsd, - type GPTImageModel, - type GeminiImageModel, type ImageOptions, type VideoJob, } from "@vibeframe/ai-providers"; @@ -104,6 +99,7 @@ import { augmentBackdropPrompt } from "./build-backdrop-prompt.js"; import { executeVideoGenerate } from "../ai-video.js"; import { executeMusic } from "../generate/music.js"; import { createAndWriteJobRecord, writeJobRecord, type JobRecord } from "./status-jobs.js"; +import { contractQuality } from "./image-jobs.js"; import { executeSceneRepair, type SceneRepairResult } from "./scene-repair.js"; import { resolveTtsProvider, TtsKeyMissingError, type TtsProviderName } from "./tts-resolve.js"; import { resolveSceneBuildMode, type SceneBuildMode } from "./scene-build-mode.js"; @@ -1930,11 +1926,6 @@ async function dispatchBackdrop(beat: Beat, ctx: BeatDispatchContext): Promise

{ - loadSceneBuildEnv(ctx.projectDir); - const keyInfo = imageProviderKeyInfo(ctx.imageProvider); - const apiKey = - (await getApiKeyFromConfig(keyInfo.configKey, { cwd: ctx.projectDir })) ?? - process.env[keyInfo.envVar] ?? - ""; - if (!apiKey) { - return { - success: false, - error: `${keyInfo.envVar} not set — cannot dispatch backdrop with ${ctx.imageProvider}`, - }; - } - - if (ctx.imageProvider === "openai") { - const provider = new OpenAIImageProvider(); - await provider.initialize({ apiKey }); - const result = await provider.generateImage(prompt, { - // CLI input is free-form; an alias the provider doesn't know fails the - // API call and surfaces as a beat-scoped backdrop/keyframe error. - model: (ctx.imageModel as GPTImageModel | undefined) ?? OPENAI_IMAGE_DEFAULT_MODEL, - size: ctx.imageSize, - quality: ctx.imageQuality, - }); - return imageBufferFromResult(result); - } - - if (ctx.imageProvider === "gemini") { - const provider = new GeminiProvider(); - await provider.initialize({ apiKey }); - const result = await provider.generateImage(prompt, { - model: (ctx.imageModel as GeminiImageModel | undefined) ?? "flash", - aspectRatio: ratio as "1:1" | "2:3" | "3:2" | "16:9", - }); - return imageBufferFromResult(result); - } - - const provider = new GrokProvider(); - await provider.initialize({ apiKey }); - const result = await provider.generateImage(prompt, { - aspectRatio: ratio, - n: 1, - }); - return imageBufferFromResult(result); + return buildImage(prompt, ctx, ratio); } /** * Generate a keyframe still by editing the character sheet(s) with the - * keyframe prompt (so the character stays consistent). Mirrors - * `generateBackdropImage` but uses each provider's `editImage`. + * keyframe prompt (so the character stays consistent). Providers that take + * fewer input images than there are sheets get the first ones they accept + * (Grok edits one image). */ async function editKeyframeImage( prompt: string, sheetBuffers: Buffer[], ctx: ImageGenContext, ratio: string +): Promise { + return buildImage(prompt, ctx, ratio, sheetBuffers); +} + +/** One image through the image contract, with the build's project-scoped key. */ +async function buildImage( + prompt: string, + ctx: ImageGenContext, + ratio: string, + sheetBuffers?: Buffer[] ): Promise { loadSceneBuildEnv(ctx.projectDir); const keyInfo = imageProviderKeyInfo(ctx.imageProvider); @@ -2018,32 +1978,24 @@ async function editKeyframeImage( if (!apiKey) { return { success: false, - error: `${keyInfo.envVar} not set — cannot generate keyframe with ${ctx.imageProvider}`, + error: `${keyInfo.envVar} not set — cannot generate ${sheetBuffers ? "keyframe" : "backdrop"} with ${ctx.imageProvider}`, }; } - - if (ctx.imageProvider === "openai") { - const provider = new OpenAIImageProvider(); - await provider.initialize({ apiKey }); - const result = await provider.editImage(sheetBuffers, prompt); - return imageBufferFromResult(result); - } - - if (ctx.imageProvider === "gemini") { - const provider = new GeminiProvider(); - await provider.initialize({ apiKey }); - const result = await provider.editImage(sheetBuffers, prompt, { - model: (ctx.imageModel as GeminiImageModel | undefined) ?? "flash", - aspectRatio: ratio as "1:1" | "2:3" | "3:2" | "16:9", + try { + const generator = await createImageGenerator(ctx.imageProvider, apiKey); + const maxInputs = sheetBuffers ? generator.maxEditImages(ctx.imageModel) : 0; + const result = await generator.createImage({ + prompt, + model: ctx.imageModel, + aspectRatio: ratio, + quality: contractQuality(ctx.imageQuality), + providerOptions: ctx.imageSize ? { size: ctx.imageSize } : undefined, + images: sheetBuffers?.slice(0, maxInputs).map((b) => ({ bytes: new Uint8Array(b), mimeType: "image/png" })), }); - return imageBufferFromResult(result); + return { success: true, buffer: Buffer.from(result.images[0].bytes) }; + } catch (error) { + return { success: false, error: error instanceof Error ? error.message : String(error) }; } - - const provider = new GrokProvider(); - await provider.initialize({ apiKey }); - // Grok edits a single input image. - const result = await provider.editImage(sheetBuffers[0], prompt, { aspectRatio: ratio }); - return imageBufferFromResult(result); } /** @@ -2149,26 +2101,6 @@ async function ensureKeyframe( return { status: "generated", rel }; } -async function imageBufferFromResult(result: { - success: boolean; - error?: string; - images?: GeneratedImageData[]; -}): Promise { - const image = result.images?.[0]; - if (!result.success || !image) { - return { success: false, error: result.error ?? "no image data returned" }; - } - if (image.base64) return { success: true, buffer: Buffer.from(image.base64, "base64") }; - if (image.url) { - const response = await fetch(image.url); - if (!response.ok) { - return { success: false, error: `failed to download image: HTTP ${response.status}` }; - } - return { success: true, buffer: Buffer.from(await response.arrayBuffer()) }; - } - return { success: false, error: "no image data returned" }; -} - function imageProviderKeyInfo(provider: BuildImageProvider): { configKey: string; envVar: string } { if (provider === "gemini") return { configKey: "google", envVar: "GOOGLE_API_KEY" }; if (provider === "grok") return { configKey: "xai", envVar: "XAI_API_KEY" }; diff --git a/packages/cli/src/commands/ai-image.ts b/packages/cli/src/commands/ai-image.ts index 7e306a07..8adb3a0c 100644 --- a/packages/cli/src/commands/ai-image.ts +++ b/packages/cli/src/commands/ai-image.ts @@ -13,18 +13,17 @@ * @see MODELS.md for AI model configuration */ -import { resolve, dirname } from "node:path"; -import { readFile, mkdir } from "node:fs/promises"; +import { readFile } from "node:fs/promises"; import { existsSync } from "node:fs"; import { GeminiProvider, - GrokProvider, - OpenAIImageProvider, + isProviderError, resolveGeminiTextModel, - resolveGrokImageModel, + type ProviderErrorKind, } from "@vibeframe/ai-providers"; +import { resolveProvider } from "../utils/provider-resolver.js"; +import { contractQuality, openImageGenerator, readImageInput, saveImage } from "./_shared/image-jobs.js"; import { execSafe, commandExists } from "../utils/exec-safe.js"; -import { writeImageFile } from "../utils/image-file.js"; // ============================================================================ // Image Generate @@ -32,12 +31,15 @@ import { writeImageFile } from "../utils/image-file.js"; export interface ImageGenerateOptions { prompt: string; + /** Default: the configured or first available image provider. */ provider?: string; output?: string; + /** Explicit OpenAI size; otherwise the ratio picks one. */ size?: string; ratio?: string; quality?: string; - style?: string; + /** Gemini "1K"/"2K"/"4K", Grok "1k"/"2k". */ + resolution?: string; count?: number; model?: string; apiKey?: string; @@ -45,250 +47,120 @@ export interface ImageGenerateOptions { export interface ImageGenerateResult { success: boolean; + /** Where the first image was saved. */ outputPath?: string; - images?: Array<{ url?: string; base64?: string; mimeType?: string; revisedPrompt?: string }>; + /** Every saved image: `out.png`, then `out-2.png`, `out-3.png`, ... */ + outputPaths?: string[]; + images?: Array<{ base64: string; mimeType: string; revisedPrompt?: string }>; provider?: string; model?: string; error?: string; + errorKind?: ProviderErrorKind; } -export async function executeImageGenerate(options: ImageGenerateOptions): Promise { - const { - prompt, - provider = "gemini", - output, - size = "1024x1024", - ratio = "1:1", - quality = "standard", - style = "vivid", - count = 1, - model, - apiKey, - } = options; - - try { - if (provider === "openai") { - const key = apiKey || process.env.OPENAI_API_KEY; - if (!key) return { success: false, error: "OPENAI_API_KEY required" }; - - const openaiImage = new OpenAIImageProvider(); - await openaiImage.initialize({ apiKey: key }); - - const result = await openaiImage.generateImage(prompt, { - size: size as "1024x1024" | "1536x1024" | "1024x1536" | "auto" | undefined, - quality: quality as "standard" | "hd" | undefined, - style: style as "vivid" | "natural" | undefined, - n: count, - }); - - if (!result.success || !result.images) { - return { success: false, error: result.error || "Image generation failed" }; - } - - let outputPath: string | undefined; - if (output && result.images.length > 0) { - const img = result.images[0]; - let buffer: Buffer; - if (img.url) { - const response = await fetch(img.url); - buffer = Buffer.from(await response.arrayBuffer()); - } else if (img.base64) { - buffer = Buffer.from(img.base64, "base64"); - } else { - return { success: false, error: "No image data available" }; - } - outputPath = resolve(process.cwd(), output); - await mkdir(dirname(outputPath), { recursive: true }); - await writeImageFile(outputPath, buffer); - } - - return { - success: true, - outputPath, - images: result.images.map(img => ({ url: img.url, base64: img.base64, revisedPrompt: img.revisedPrompt })), - provider: "openai", - }; - } else if (provider === "gemini") { - const key = apiKey || process.env.GOOGLE_API_KEY; - if (!key) return { success: false, error: "GOOGLE_API_KEY required" }; - - const gemini = new GeminiProvider(); - await gemini.initialize({ apiKey: key }); - - const modelMap: Record = { latest: "3.1-flash" }; - const mappedModel = model ? (modelMap[model] || model) : undefined; - - let result = await gemini.generateImage(prompt, { - model: mappedModel as "flash" | "3.1-flash" | "pro" | undefined, - aspectRatio: ratio as "1:1" | "16:9" | "9:16" | "4:3" | "3:4", - }); - - const fallbackModels = ["3.1-flash"]; - if (!result.success && mappedModel && fallbackModels.includes(mappedModel)) { - result = await gemini.generateImage(prompt, { - model: "flash", - aspectRatio: ratio as "1:1" | "16:9" | "9:16" | "4:3" | "3:4", - }); - } - - if (!result.success || !result.images) { - return { success: false, error: result.error || "Image generation failed" }; - } - - let outputPath: string | undefined; - if (output && result.images.length > 0) { - const img = result.images[0]; - if (img.base64) { - outputPath = resolve(process.cwd(), output); - await mkdir(dirname(outputPath), { recursive: true }); - await writeImageFile(outputPath, Buffer.from(img.base64, "base64")); - } - } - - return { - success: true, - outputPath, - images: result.images.map(img => ({ base64: img.base64, mimeType: img.mimeType })), - provider: "gemini", - model: result.model, - }; - } else if (provider === "grok") { - const key = apiKey || process.env.XAI_API_KEY; - if (!key) return { success: false, error: "XAI_API_KEY required" }; - - const grok = new GrokProvider(); - await grok.initialize({ apiKey: key }); - - const result = await grok.generateImage(prompt, { - ...resolveGrokImageModel(model), - aspectRatio: ratio, - n: count, - }); +/** `out.png` → `out-2.png`. */ +function numberedPath(path: string, n: number): string { + const dot = path.lastIndexOf("."); + return dot > path.lastIndexOf("/") ? `${path.slice(0, dot)}-${n}${path.slice(dot)}` : `${path}-${n}`; +} - if (!result.success || !result.images) { - return { success: false, error: result.error || "Image generation failed" }; - } +function imageFailure(error: unknown, prefix: string, provider?: string): ImageGenerateResult { + if (isProviderError(error)) return { success: false, error: error.message, errorKind: error.kind, provider }; + return { success: false, error: `${prefix}: ${error instanceof Error ? error.message : String(error)}`, provider }; +} - let outputPath: string | undefined; - if (output && result.images.length > 0) { - const img = result.images[0]; - let buffer: Buffer; - if (img.url) { - const response = await fetch(img.url); - buffer = Buffer.from(await response.arrayBuffer()); - } else if (img.base64) { - buffer = Buffer.from(img.base64, "base64"); - } else { - return { success: false, error: "No image data available" }; - } - outputPath = resolve(process.cwd(), output); - await mkdir(dirname(outputPath), { recursive: true }); - await writeImageFile(outputPath, buffer); +export async function executeImageGenerate(options: ImageGenerateOptions): Promise { + const provider = options.provider ?? resolveProvider("image")?.name ?? "gemini"; + try { + const generator = await openImageGenerator(provider, options.apiKey); + const result = await generator.createImage({ + prompt: options.prompt, + model: options.model, + aspectRatio: options.ratio, + resolution: options.resolution, + quality: contractQuality(options.quality), + count: options.count, + providerOptions: options.size ? { size: options.size } : undefined, + }); + const outputPaths: string[] = []; + if (options.output) { + for (const [i, img] of result.images.entries()) { + outputPaths.push(await saveImage(i === 0 ? options.output : numberedPath(options.output, i + 1), img.bytes)); } - - return { - success: true, - outputPath, - images: result.images.map(img => ({ url: img.url, base64: img.base64, revisedPrompt: img.revisedPrompt })), - provider: "grok", - }; } - - return { success: false, error: `Unsupported provider: ${provider}` }; + return { + success: true, + outputPath: outputPaths[0], + outputPaths: outputPaths.length > 0 ? outputPaths : undefined, + images: result.images.map((img) => ({ + base64: Buffer.from(img.bytes).toString("base64"), + mimeType: img.mimeType, + revisedPrompt: img.revisedPrompt, + })), + provider, + model: result.model, + }; } catch (error) { - return { success: false, error: `Image generation failed: ${error instanceof Error ? error.message : String(error)}` }; + return imageFailure(error, "Image generation failed", provider); } } // ============================================================================ -// Gemini Image Edit +// Image Edit // ============================================================================ -export interface GeminiEditOptions { +export interface ImageEditOptions { imagePaths: string[]; prompt: string; + /** Default: gemini. */ + provider?: string; output?: string; model?: string; ratio?: string; resolution?: string; + quality?: string; apiKey?: string; } -export interface GeminiEditResult { +export interface ImageEditResult { success: boolean; outputPath?: string; + provider?: string; model?: string; error?: string; + errorKind?: ProviderErrorKind; } -export async function executeGeminiEdit(options: GeminiEditOptions): Promise { - const { - imagePaths, - prompt, - output = "edited.png", - model = "flash", - ratio, - resolution, - apiKey, - } = options; - +export async function executeImageEdit(options: ImageEditOptions): Promise { + const provider = options.provider ?? "gemini"; try { - const key = apiKey || process.env.GOOGLE_API_KEY; - if (!key) return { success: false, error: "GOOGLE_API_KEY required" }; - - const imageBuffers: Buffer[] = []; - for (const imagePath of imagePaths) { - const absPath = resolve(process.cwd(), imagePath); - if (!existsSync(absPath)) { - return { success: false, error: `Image not found: ${absPath}` }; - } - const buffer = await readFile(absPath); - imageBuffers.push(buffer); - } - - const gemini = new GeminiProvider(); - await gemini.initialize({ apiKey: key }); - - type GeminiAspectRatio = "1:1" | "1:4" | "1:8" | "2:3" | "3:2" | "3:4" | "4:1" | "4:3" | "4:5" | "5:4" | "8:1" | "9:16" | "16:9" | "21:9"; - type GeminiRes = "512px" | "1K" | "2K" | "4K"; - - let result = await gemini.editImage(imageBuffers, prompt, { - model: model as "flash" | "3.1-flash" | "pro" | undefined, - aspectRatio: ratio as GeminiAspectRatio | undefined, - resolution: resolution as GeminiRes | undefined, + const generator = await openImageGenerator(provider, options.apiKey); + const images = await Promise.all(options.imagePaths.map((path) => readImageInput(path))); + const result = await generator.createImage({ + prompt: options.prompt, + model: options.model, + aspectRatio: options.ratio, + resolution: options.resolution, + quality: contractQuality(options.quality), + images, }); - - const fallbackModels = ["latest", "3.1-flash"]; - if (!result.success && fallbackModels.includes(model)) { - result = await gemini.editImage(imageBuffers, prompt, { - model: "flash", - aspectRatio: ratio as GeminiAspectRatio | undefined, - resolution: resolution as GeminiRes | undefined, - }); - } - - if (!result.success || !result.images || result.images.length === 0) { - return { success: false, error: result.error || "Image editing failed" }; - } - - const img = result.images[0]; - let outputPath: string | undefined; - if (img.base64) { - outputPath = resolve(process.cwd(), output); - await mkdir(dirname(outputPath), { recursive: true }); - await writeImageFile(outputPath, Buffer.from(img.base64, "base64")); - } - - return { - success: true, - outputPath, - model: result.model, - }; + const outputPath = await saveImage(options.output ?? "edited.png", result.images[0].bytes); + return { success: true, outputPath, provider, model: result.model }; } catch (error) { - return { success: false, error: `Image editing failed: ${error instanceof Error ? error.message : String(error)}` }; + const failed = imageFailure(error, "Image editing failed", provider); + return { success: false, error: failed.error, errorKind: failed.errorKind, provider }; } } +/** @deprecated Use `executeImageEdit`; kept for callers that predate other edit providers. */ +export type GeminiEditOptions = Omit; +/** @deprecated Use `executeImageEdit`. */ +export type GeminiEditResult = ImageEditResult; +/** @deprecated Use `executeImageEdit`. */ +export function executeGeminiEdit(options: GeminiEditOptions): Promise { + return executeImageEdit({ ...options, provider: "gemini" }); +} + // ============================================================================ // Thumbnail Best Frame // ============================================================================ diff --git a/packages/cli/src/commands/edit-cmd.ts b/packages/cli/src/commands/edit-cmd.ts index 2e0dbee4..39384a67 100644 --- a/packages/cli/src/commands/edit-cmd.ts +++ b/packages/cli/src/commands/edit-cmd.ts @@ -24,18 +24,11 @@ */ import { Command } from "commander"; -import { resolve, dirname } from "node:path"; -import { readFile, writeFile, mkdir } from "node:fs/promises"; +import { resolve } from "node:path"; +import { readFile, writeFile } from "node:fs/promises"; import chalk from "chalk"; import ora from "ora"; -import { - WhisperProvider, - ClaudeProvider, - GeminiProvider, - OpenAIImageProvider, - GrokProvider, - resolveGrokImageModel, -} from "@vibeframe/ai-providers"; +import { WhisperProvider, ClaudeProvider, findModel, modelAliases } from "@vibeframe/ai-providers"; import { requireApiKey } from "../utils/api-key.js"; import { execSafe, commandExists } from "../utils/exec-safe.js"; import { formatTime } from "./ai-helpers.js"; @@ -43,10 +36,11 @@ import { applyTextOverlays, type TextOverlayStyle } from "./ai-edit.js"; import { registerEditCommands } from "./ai-edit-cli.js"; import { registerFillGapsCommand } from "./ai-fill-gaps.js"; import { registerMotionOverlayCommand } from "./edit/motion-overlay.js"; -import { isJsonMode, outputSuccess, exitWithError, usageError, notFoundError, apiError, generalError } from "./output.js"; +import { isJsonMode, outputSuccess, exitWithError, usageError, notFoundError, apiError, generalError, providerFailure } from "./output.js"; import { rejectControlChars, validateOutputPath } from "./validate.js"; import { applyTiers } from "./_shared/cost-tier.js"; -import { writeImageFile } from "../utils/image-file.js"; +import { executeImageEdit } from "./ai-image.js"; +import { IMAGE_PROVIDER_ENV, IMAGE_PROVIDER_LABELS } from "./_shared/image-jobs.js"; export const editCommand = new Command("edit") .alias("ed") @@ -689,7 +683,7 @@ editCommand } }); -// ── edit image (Gemini multi-image editing) ───────────────────────────── +// ── edit image (multi-image editing on any image provider) ────────────── editCommand .command("image") @@ -698,157 +692,76 @@ editCommand .option("-p, --provider ", "Provider: gemini (default), openai, grok", "gemini") .option("-k, --api-key ", "API key (or set env variable)") .option("-o, --output ", "Output file path", "edited.png") - .option("-m, --model ", "Model: flash, lite, pro (Gemini); pro (Grok)", "flash") + .option("-m, --model ", "Model: flash (default), lite, pro (Gemini); 2.5, flare, 2 (OpenAI); pro (Grok)") .option("-r, --ratio ", "Output aspect ratio") .option("--size ", "Resolution: 1K, 2K, 4K (Gemini Pro only)") .option("--dry-run", "Preview parameters without executing") .action(async (args: string[], options) => { const startedAt = Date.now(); - try { - // Last argument is the prompt, rest are image paths - if (args.length < 2) { - exitWithError(usageError("Need at least one image and a prompt")); - } - - const prompt = args[args.length - 1]; - rejectControlChars(prompt); - if (options.output) { - validateOutputPath(options.output); - } - const imagePaths = args.slice(0, -1); - const provider = options.provider as string; - - // Grok only supports 1 image - if (provider === "grok" && imagePaths.length > 1) { - exitWithError(usageError("Grok supports only 1 input image for editing.", "Use -p gemini (up to 14 images) or -p openai (up to 16 images) for multi-image editing.")); - } - - if (options.dryRun) { - outputSuccess({ - command: "edit image", - startedAt, - dryRun: true, - data: { - params: { - imagePaths: imagePaths.map((p: string) => resolve(process.cwd(), p)), - prompt, - provider, - model: options.model, - ratio: options.ratio, - size: options.size, - }, - }, - }); - return; - } - - // Provider-specific API key resolution - const apiKeyMap: Record = { - gemini: { envVar: "GOOGLE_API_KEY", label: "Google" }, - openai: { envVar: "OPENAI_API_KEY", label: "OpenAI" }, - grok: { envVar: "XAI_API_KEY", label: "xAI" }, - }; - const keyInfo = apiKeyMap[provider] || apiKeyMap.gemini; - const apiKey = await requireApiKey(keyInfo.envVar, keyInfo.label, options.apiKey); - - const spinner = ora(`Reading ${imagePaths.length} image(s)...`).start(); - - // Load all images - const imageBuffers: Buffer[] = []; - for (const imagePath of imagePaths) { - const absPath = resolve(process.cwd(), imagePath); - const buffer = await readFile(absPath); - imageBuffers.push(buffer); - } - - let result: import("@vibeframe/ai-providers").ImageResult; - - if (provider === "openai") { - spinner.text = "Editing with GPT Image 1.5..."; - const openaiImage = new OpenAIImageProvider(); - await openaiImage.initialize({ apiKey }); - result = await openaiImage.editImage(imageBuffers, prompt); - } else if (provider === "grok") { - spinner.text = "Editing with Grok Imagine..."; - const grok = new GrokProvider(); - await grok.initialize({ apiKey }); - result = await grok.editImage(imageBuffers[0], prompt, { - ...resolveGrokImageModel(options.model), - aspectRatio: options.ratio, - }); - } else { - // Gemini (default) - const editModelNames: Record = { - flash: "gemini-3.1-flash-image", - "3.1-flash": "gemini-3.1-flash-image", - latest: "gemini-3.1-flash-image", - lite: "gemini-3.1-flash-lite-image", - pro: "gemini-3-pro-image", - }; - const editModelName = editModelNames[options.model] || editModelNames.flash; - spinner.text = `Editing with ${editModelName}...`; - - const gemini = new GeminiProvider(); - await gemini.initialize({ apiKey }); - - result = await gemini.editImage(imageBuffers, prompt, { - model: options.model, - aspectRatio: options.ratio, - resolution: options.size, - }); - - } - - if (!result.success || !result.images || result.images.length === 0) { - spinner.fail(result.error || "Image editing failed"); - exitWithError(apiError(result.error || "Image editing failed", true)); - } - - spinner.succeed(chalk.green("Image edited")); - - // Save image — handle both base64 and URL responses - const img = result.images[0]; - const outputPath = resolve(process.cwd(), options.output); - - const saveImage = async () => { - await mkdir(dirname(outputPath), { recursive: true }); - if (img.base64) { - const buffer = Buffer.from(img.base64, "base64"); - await writeImageFile(outputPath, buffer); - } else if (img.url) { - const resp = await fetch(img.url); - const arrayBuf = await resp.arrayBuffer(); - await writeImageFile(outputPath, Buffer.from(arrayBuf)); - } - }; - - // Gemini results may include a `model` field - const resultModel = (result as { model?: string }).model; + // Last argument is the prompt, the rest are image paths + if (args.length < 2) { + exitWithError(usageError("Need at least one image and a prompt")); + } + const prompt = args[args.length - 1]; + rejectControlChars(prompt); + if (options.output) validateOutputPath(options.output); + const imagePaths = args.slice(0, -1); + const provider = String(options.provider || "gemini").toLowerCase(); + if (!IMAGE_PROVIDER_ENV[provider]) { + exitWithError(usageError(`Invalid provider: ${provider}`, `Available providers: ${Object.keys(IMAGE_PROVIDER_ENV).join(", ")}`)); + } + const modelSpec = findModel(provider, "image", options.model); + if (!modelSpec) { + exitWithError(usageError(`Unknown ${provider} image model "${options.model}". Valid: ${modelAliases(provider, "image").join(", ")}.`)); + } - if (isJsonMode()) { - outputSuccess({ - command: "edit image", - startedAt, - data: { + if (options.dryRun) { + outputSuccess({ + command: "edit image", + startedAt, + dryRun: true, + data: { + params: { + imagePaths: imagePaths.map((p: string) => resolve(process.cwd(), p)), + prompt, provider, - model: resultModel || options.model, - outputPath, + model: modelSpec.id, + ratio: options.ratio, + size: options.size, }, - }); - await saveImage(); - return; - } + }, + }); + return; + } - if (resultModel) { - console.log(chalk.dim(`Model: ${resultModel}`)); - } + const label = IMAGE_PROVIDER_LABELS[provider]; + const apiKey = await requireApiKey(IMAGE_PROVIDER_ENV[provider], label, options.apiKey); + const spinner = isJsonMode() ? null : ora(`Editing ${imagePaths.length} image(s) with ${label} ${modelSpec.label}...`).start(); + const result = await executeImageEdit({ + imagePaths, + prompt, + provider, + model: options.model, + ratio: options.ratio, + resolution: options.size, + output: options.output, + apiKey, + }); + if (!result.success) { + spinner?.fail(result.error ?? "Image editing failed"); + exitWithError(providerFailure(result.error ?? "Image editing failed", result.errorKind)); + } - await saveImage(); - console.log(chalk.green(`Saved to: ${outputPath}`)); - } catch (error) { - const msg = error instanceof Error ? error.message : String(error); - exitWithError(apiError(`Image editing failed: ${msg}`, true)); + if (isJsonMode()) { + outputSuccess({ + command: "edit image", + startedAt, + data: { provider, model: result.model, outputPath: result.outputPath }, + }); + return; } + spinner?.succeed(chalk.green("Image edited")); + console.log(chalk.green(`Saved to: ${result.outputPath}`)); }); // ── edit interpolate (frame interpolation / slow motion) ──────────────── diff --git a/packages/cli/src/commands/error-envelopes.test.ts b/packages/cli/src/commands/error-envelopes.test.ts index f64e3d37..2ae0752b 100644 --- a/packages/cli/src/commands/error-envelopes.test.ts +++ b/packages/cli/src/commands/error-envelopes.test.ts @@ -101,10 +101,11 @@ const cases: ErrorCase[] = [ exitCode: 2, }, { - name: "generate image — no API key, hermetic (API_ERROR)", + // A missing key is an auth error with setup hints, not a generic API failure. + name: "generate image — no API key, hermetic (API_KEY_MISSING)", cmd: 'generate image "a test prompt" -p openai --json', - code: "API_ERROR", - exitCode: 5, + code: "API_KEY_MISSING", + exitCode: 4, }, ]; diff --git a/packages/cli/src/commands/generate/image.ts b/packages/cli/src/commands/generate/image.ts index 2d32827f..a9a05dd2 100644 --- a/packages/cli/src/commands/generate/image.ts +++ b/packages/cli/src/commands/generate/image.ts @@ -1,58 +1,50 @@ /** * @module generate/image - * @description `vibe generate image` (alias `img`) — multi-provider image - * generation. OpenAI gpt-image-2.5, Gemini Nano Banana, Grok, Runway. Split - * out of `generate.ts` in v0.69 (Plan G Phase 2). + * @description `vibe generate image` (alias `img`) — image generation on + * OpenAI GPT Image, Gemini Nano Banana, or Grok Imagine. Validation, + * provider choice, and the dry run live here; generation is + * `executeImageGenerate`, the same executor the MCP tool and pipelines use. */ import type { Command } from "commander"; -import { resolve, dirname } from "node:path"; -import { fileURLToPath } from "node:url"; -import { mkdir } from "node:fs/promises"; import chalk from "chalk"; import ora from "ora"; -import { - GeminiProvider, - GrokProvider, - getProvidersFor, - resolveGrokImageModel, -} from "@vibeframe/ai-providers"; +import { findModel, getProvidersFor, modelAliases } from "@vibeframe/ai-providers"; import { requireApiKey, hasConfiguredApiKey } from "../../utils/api-key.js"; import { hasTTY, prompt as promptText } from "../../utils/tty.js"; -import { isJsonMode, outputSuccess, log, exitWithError, apiError, usageError } from "../output.js"; +import { checkModelLifecycle } from "../../utils/model-lifecycle.js"; +import { + isJsonMode, + outputSuccess, + printWarnings, + log, + exitWithError, + providerFailure, + usageError, +} from "../output.js"; import { rejectControlChars, validateOutputPath } from "../validate.js"; import { loadProviderDefaults, resolveProvider } from "../../utils/provider-resolver.js"; -import { executeOpenAIImageGenerate } from "../_shared/openai-image.js"; import { estimateImageCostUsd } from "../_shared/image-cost.js"; -import { writeImageFile } from "../../utils/image-file.js"; +import { executeImageGenerate } from "../ai-image.js"; +import { IMAGE_PROVIDER_ENV, IMAGE_PROVIDER_LABELS, imageModelLabel } from "../_shared/image-jobs.js"; export function registerImageCommand(parent: Command): void { parent .command("image") .alias("img") - .description("Generate image using AI (Gemini, OpenAI gpt-image, Grok, or Runway)") + .description("Generate image using AI (OpenAI GPT Image, Gemini, or Grok)") .argument("[prompt]", "Image description prompt (interactive if omitted)") - .option( - "-p, --provider ", - "Provider: openai (default when OPENAI_API_KEY set), gemini, grok, runway" - ) - .option("-k, --api-key ", "API key (or set env: OPENAI_API_KEY, GOOGLE_API_KEY)") - .option("-o, --output ", "Output file path (downloads image)") - .option("--size ", "Image size (openai: 1024x1024, 1536x1024, 1024x1536)", "1024x1024") - .option( - "-r, --ratio ", - "Aspect ratio (gemini: 1:1, 1:4, 1:8, 4:1, 8:1, 16:9, 9:16, 3:4, 4:3, etc.)", - "1:1" - ) + .option("-p, --provider ", "Provider: openai (default when OPENAI_API_KEY set), gemini, grok") + .option("-k, --api-key ", "API key (or set env: OPENAI_API_KEY, GOOGLE_API_KEY, XAI_API_KEY)") + .option("-o, --output ", "Output file path") + .option("-r, --ratio ", "Aspect ratio: 1:1, 16:9, 9:16, 4:3, 3:4, ... (OpenAI maps it to its nearest size)", "1:1") + .option("--size ", "Explicit OpenAI size: 1024x1024, 1536x1024, 1024x1536 (overrides --ratio)") // `-q` shorthand intentionally omitted: collides with global `vibe -q,--quiet`, // which previously ate the value silently and dropped the prompt positional. - .option("--quality ", "Quality: standard, hd (openai only)", "standard") - .option("--style