diff --git a/packages/coding-agent/docs/skills.md b/packages/coding-agent/docs/skills.md index 14c8f38fc..6075ddd55 100644 --- a/packages/coding-agent/docs/skills.md +++ b/packages/coding-agent/docs/skills.md @@ -270,7 +270,7 @@ cd /path/to/brave-search && bun install Senpi contributes built-in skills conditionally based on available credentials and capabilities. -**gpt-image-gen** is contributed by the `imagegen` builtin extension when image-generation credentials exist (stored OpenAI key, `OPENAI_API_KEY`, or an OpenAI-compatible gateway). It is the GPT Image 2.5 prompting guide: tool routing (native `image_generation` server tool vs. the client `generate_image` tool), model selection, a specificity policy (normalize specific requests, add concreteness only to generic ones), verbatim quoted text, reference roles and end-state edits, transparent assets, output formats, multi-turn refinement, and a result checklist. For `generate_image`, `model` selects `gpt-image-2.5-sunburst` (default, most capable), `gpt-image-2.5-flare` (speed), or `gpt-image-2`; `quality` accepts `auto`, `low`, `medium`, `high`, `xhigh`, or `max`; `size` accepts `auto`, presets, or validated custom dimensions; `background`, `output_format` (`png`/`jpeg`/`webp`), `output_compression` (jpeg/webp), and `moderation` shape the output, and the saved extension follows the delivered bytes. `reference_image_paths` supplies 1-5 local images; `mask_image_path` adds an alpha mask for local repaints. Only the native server tool returns a `revised_prompt`. +**gpt-image-gen** is contributed by the `imagegen` builtin extension when native image generation is enabled for the current model, or when image-generation credentials exist (stored OpenAI key, `OPENAI_API_KEY`, or an OpenAI-compatible gateway). Supported native OpenAI OAuth sessions do not require a separate image API key. It is the GPT Image 2.5 prompting guide: tool routing (native `image_generation` server tool vs. the client `generate_image` tool), model selection, a specificity policy (normalize specific requests, add concreteness only to generic ones), verbatim quoted text, reference roles and end-state edits, transparent assets, output formats, multi-turn refinement, and a result checklist. For `generate_image`, `model` selects `gpt-image-2.5-sunburst` (default, most capable), `gpt-image-2.5-flare` (speed), or `gpt-image-2`; `quality` accepts `auto`, `low`, `medium`, `high`, `xhigh`, or `max`; `size` accepts `auto`, presets, or validated custom dimensions; `background`, `output_format` (`png`/`jpeg`/`webp`), `output_compression` (jpeg/webp), and `moderation` shape the output, and the saved extension follows the delivered bytes. `reference_image_paths` supplies 1-5 local images; `mask_image_path` adds an alpha mask for local repaints. Only the native server tool returns a `revised_prompt`. Skill visibility refreshes at startup and on `/reload`. Mid-session credential changes (login, environment variable updates) take effect on the tool and injector immediately but are reflected in the skill list only after the next reload. diff --git a/packages/coding-agent/src/core/extensions/builtin/imagegen/changes.md b/packages/coding-agent/src/core/extensions/builtin/imagegen/changes.md index 7abcb0aa0..dc4926160 100644 --- a/packages/coding-agent/src/core/extensions/builtin/imagegen/changes.md +++ b/packages/coding-agent/src/core/extensions/builtin/imagegen/changes.md @@ -1,3 +1,22 @@ +## 2026-09-11 - Guidance follows native capability or client credentials + +### What changed + +- `packages/coding-agent/src/core/extensions/builtin/imagegen/index.ts` contributes the bundled skill and shared guidance when the native image-generation gate and enable predicate are active, falling back to the unchanged client credential resolver otherwise. +- Focused harness regressions load the skill for OAuth-only sessions without an image key and verify switching away or disabling native generation. The arbitration matrix's two native-without-client-credentials rows now expect guidance and skill availability; other surface contracts remain unchanged. + +### Why + +- `packages/coding-agent/src/core/extensions/builtin/imagegen/index.ts` withheld guidance from native-capable sessions that did not have separate image API credentials. + +### Why an extension could not handle it + +- `packages/coding-agent/src/core/extensions/builtin/imagegen/index.ts` owns the bundled resource contribution and reuses the native builtin's existing gate rather than duplicating transport or credential policy. + +### Expected merge conflict zones + +- LOW: imports and the shared availability predicate in `packages/coding-agent/src/core/extensions/builtin/imagegen/index.ts`. + ## 2026-09-10 - Sunburst stays the default model ### What changed diff --git a/packages/coding-agent/src/core/extensions/builtin/imagegen/index.ts b/packages/coding-agent/src/core/extensions/builtin/imagegen/index.ts index f2c4da888..36b0c2fdd 100644 --- a/packages/coding-agent/src/core/extensions/builtin/imagegen/index.ts +++ b/packages/coding-agent/src/core/extensions/builtin/imagegen/index.ts @@ -2,6 +2,7 @@ import { existsSync } from "node:fs"; import { dirname, join } from "node:path"; import { fileURLToPath } from "node:url"; import type { ExtensionAPI, ExtensionContext } from "../../types.ts"; +import { isOpenAiImageGenEnabled, supportsNativeOpenAiImageGeneration } from "../openai-image-gen/gate.ts"; import { resolveImageGenAuth } from "./auth.ts"; import { imageGenRegistryOverride } from "./state.ts"; import { generateImageTool } from "./tool.ts"; @@ -21,6 +22,7 @@ Use the image generation tool currently available in this session. `; async function isImageGenActive(ctx: ExtensionContext): Promise { + if (isOpenAiImageGenEnabled() && supportsNativeOpenAiImageGeneration(ctx.model)) return true; const auth = await resolveImageGenAuth({ modelRegistry: imageGenRegistryOverride() ?? ctx.modelRegistry }); return auth.kind !== "none"; } diff --git a/packages/coding-agent/src/core/extensions/builtin/openai-image-gen/changes.md b/packages/coding-agent/src/core/extensions/builtin/openai-image-gen/changes.md index 0d232f43b..21260b0f4 100644 --- a/packages/coding-agent/src/core/extensions/builtin/openai-image-gen/changes.md +++ b/packages/coding-agent/src/core/extensions/builtin/openai-image-gen/changes.md @@ -1,3 +1,22 @@ +## 2026-09-11 - Native image generation on Codex OAuth + +### What changed + +- `packages/coding-agent/src/core/extensions/builtin/openai-image-gen/gate.ts` recognizes `openai-codex-responses` on the exact `chatgpt.com` host without rewriting its API or base URL. Official `api.openai.com` Responses behavior, explicit compatibility overrides, global disable, and unrelated API exclusions remain unchanged. Codex proxies, lookalikes, malformed URLs, and an empty base URL do not default to native generation. +- Focused suite tests cover the gate and harness lifecycle, including pinned Sunburst payload descriptors, native/client exclusivity, bypass, and model switches. + +### Why + +- `packages/coding-agent/src/core/extensions/builtin/openai-image-gen/gate.ts` previously rejected the Codex OAuth transport even though its official endpoint supports the existing native server tool. + +### Why an extension could not handle it + +- `packages/coding-agent/src/core/extensions/builtin/openai-image-gen/gate.ts` is already the owning builtin's capability gate; another injector would conflict with its arbitration pass. + +### Expected merge conflict zones + +- LOW: API discrimination and official-host detection in `packages/coding-agent/src/core/extensions/builtin/openai-image-gen/gate.ts`. + ## 2026-09-10 - Native tool follows the Sunburst default ### What changed diff --git a/packages/coding-agent/src/core/extensions/builtin/openai-image-gen/gate.ts b/packages/coding-agent/src/core/extensions/builtin/openai-image-gen/gate.ts index 19487f724..45c638d11 100644 --- a/packages/coding-agent/src/core/extensions/builtin/openai-image-gen/gate.ts +++ b/packages/coding-agent/src/core/extensions/builtin/openai-image-gen/gate.ts @@ -7,6 +7,7 @@ export type NativeImageGenTarget = NativeImageGenModel | undefined; const ENABLE_ENV = "PI_OPENAI_IMAGE_GEN"; const OFFICIAL_OPENAI_HOST = "api.openai.com"; const OFFICIAL_OPENAI_BASE_URL = "https://api.openai.com/v1"; +const OFFICIAL_CODEX_HOST = "chatgpt.com"; function parseEnableEnv(envVar: string): boolean { const envValue = process.env[envVar]; @@ -41,9 +42,12 @@ function compatImageGenerationOverride(compat: unknown): boolean | undefined { return typeof value === "boolean" ? value : undefined; } -function isOfficialOpenAiEndpoint(baseUrl: string): boolean { +function isOfficialOpenAiEndpoint(target: NativeImageGenModel): boolean { try { - return new URL(baseUrl || OFFICIAL_OPENAI_BASE_URL).hostname === OFFICIAL_OPENAI_HOST; + if (target.api === "openai-codex-responses") { + return new URL(target.baseUrl).hostname === OFFICIAL_CODEX_HOST; + } + return new URL(target.baseUrl || OFFICIAL_OPENAI_BASE_URL).hostname === OFFICIAL_OPENAI_HOST; } catch { return false; } @@ -53,20 +57,21 @@ function isOfficialOpenAiEndpoint(baseUrl: string): boolean { * Whether the model's endpoint serves the OpenAI Responses `image_generation` * server tool. * - * Unlike the web-search gate, `azure-openai-responses` defaults to FALSE: Azure + * Official Responses and Codex OAuth endpoints default to TRUE without changing + * the model's transport. Unlike the web-search gate, Azure is excluded: Azure * deployments expose image generation as a separate deployment rather than as a - * Responses server tool, so azure opts in only through - * `compat.supportsImageGeneration`. Proxied `openai-responses` endpoints default + * Responses server tool. Proxied Responses and Codex endpoints default * to the client tool for the same reason they do for web search: a translating - * gateway rejects the tool type it never implemented. + * gateway rejects the tool type it never implemented. They can opt in through + * `compat.supportsImageGeneration`. */ export function supportsNativeOpenAiImageGeneration(target: NativeImageGenTarget): boolean { - if (target === undefined || target.api !== "openai-responses") { + if (target === undefined || (target.api !== "openai-responses" && target.api !== "openai-codex-responses")) { return false; } const override = compatImageGenerationOverride(target.compat); - return override ?? isOfficialOpenAiEndpoint(target.baseUrl); + return override ?? isOfficialOpenAiEndpoint(target); } /** Identity of the model an arbitration decision was made for. */ diff --git a/packages/coding-agent/test/suite/codex-image-generation-gate.test.ts b/packages/coding-agent/test/suite/codex-image-generation-gate.test.ts new file mode 100644 index 000000000..be43d5c31 --- /dev/null +++ b/packages/coding-agent/test/suite/codex-image-generation-gate.test.ts @@ -0,0 +1,46 @@ +import { describe, expect, it } from "vitest"; +import { + type NativeImageGenModel, + supportsNativeOpenAiImageGeneration, +} from "../../src/core/extensions/builtin/openai-image-gen/gate.ts"; + +const codex: NativeImageGenModel = { + id: "gpt-5.5", + provider: "openai-codex", + api: "openai-codex-responses", + baseUrl: "https://chatgpt.com/backend-api", +}; + +describe("Codex native image generation capability", () => { + it.each([ + ["official Codex", {}, true], + ["official opt-out", { compat: { supportsImageGeneration: false } }, false], + ["proxy", { baseUrl: "https://codex-proxy.example.test/backend-api" }, false], + [ + "proxy opt-in", + { baseUrl: "https://codex-proxy.example.test/backend-api", compat: { supportsImageGeneration: true } }, + true, + ], + ["lookalike", { baseUrl: "https://chatgpt.com.example.test/backend-api" }, false], + ["userinfo lookalike", { baseUrl: "https://chatgpt.com@proxy.example.test/backend-api" }, false], + ["malformed", { baseUrl: "https://[chatgpt.com" }, false], + ["relative", { baseUrl: "/backend-api" }, false], + ["empty Codex URL", { baseUrl: "" }, false], + ["Responses host on Codex", { baseUrl: "https://api.openai.com/v1" }, false], + ["Codex host on Responses", { api: "openai-responses" }, false], + ["official Responses", { api: "openai-responses", baseUrl: "https://api.openai.com/v1" }, true], + ["default Responses URL", { api: "openai-responses", baseUrl: "" }, true], + ["completions opt-in", { api: "openai-completions", compat: { supportsImageGeneration: true } }, false], + ["Azure opt-in", { api: "azure-openai-responses", compat: { supportsImageGeneration: true } }, false], + ["unrelated API", { api: "anthropic-messages" }, false], + ] satisfies [string, Partial, boolean][])("%s", (_label, fields, expected) => { + const model = { ...codex, ...fields }; + const original = structuredClone(model); + expect(supportsNativeOpenAiImageGeneration(model)).toBe(expected); + expect(model).toEqual(original); + }); + + it("does not infer capability without a model", () => { + expect(supportsNativeOpenAiImageGeneration(undefined)).toBe(false); + }); +}); diff --git a/packages/coding-agent/test/suite/codex-image-generation.test.ts b/packages/coding-agent/test/suite/codex-image-generation.test.ts new file mode 100644 index 000000000..40d7acb24 --- /dev/null +++ b/packages/coding-agent/test/suite/codex-image-generation.test.ts @@ -0,0 +1,162 @@ +import { join } from "node:path"; +import type { Api, Model } from "@earendil-works/pi-ai"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { type ImageGenAuthRegistry, resolveImageGenAuth } from "../../src/core/extensions/builtin/imagegen/auth.ts"; +import imageGenExtension, { IMAGE_GEN_SECTION } from "../../src/core/extensions/builtin/imagegen/index.ts"; +import { setImageGenRegistry, setNativeBypass } from "../../src/core/extensions/builtin/imagegen/state.ts"; +import type { GenerateImageDetails } from "../../src/core/extensions/builtin/imagegen/tool.ts"; +import openaiImageGenExtension, { + OPENAI_IMAGE_GEN_SECTION, +} from "../../src/core/extensions/builtin/openai-image-gen/index.ts"; +import { loadSkills } from "../../src/core/skills.ts"; +import { createHarness, type Harness } from "./harness.ts"; + +const codex: Model<"openai-codex-responses"> = { + id: "gpt-5.5", + name: "Codex OAuth image fixture", + api: "openai-codex-responses", + provider: "openai-codex", + baseUrl: "https://chatgpt.com/backend-api", + reasoning: false, + input: ["text"], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 128_000, + maxTokens: 16_384, +}; +const proxy: Model<"openai-codex-responses"> = { + ...codex, + id: "codex-proxy-fixture", + baseUrl: "https://codex-proxy.example.test/backend-api", +}; +const optedOut: Model<"openai-codex-responses"> = { + ...codex, + id: "codex-opt-out-fixture", + compat: { supportsImageGeneration: false }, +}; +const oauthOnly: ImageGenAuthRegistry = { + authStorage: { get: (provider) => (provider === "openai-codex" ? { type: "oauth" } : undefined) }, + getAll: () => [codex], + getApiKeyAndHeaders: async () => ({ ok: false, error: "no image API key" }), + getProviderAuth: async () => undefined, +}; +const readTool = { type: "function", name: "read", parameters: { type: "object" } }; +const clientTool = { type: "function", name: "generate_image", parameters: { type: "object" } }; +const nativeTool = { type: "image_generation", model: "gpt-image-2.5-sunburst" }; +const harnesses: Harness[] = []; + +async function start(model: Model = codex, native = true): Promise { + const harness = await createHarness({ + extensionFactories: native ? [imageGenExtension, openaiImageGenExtension] : [imageGenExtension], + }); + harnesses.push(harness); + harness.modelRegistry.registerProvider(codex.provider, { + api: codex.api, + baseUrl: codex.baseUrl, + apiKey: "offline-session-switch-fixture", + models: [codex, proxy, optedOut], + }); + harness.agent.state.model = model; + await harness.session.bindExtensions({}); + return harness; +} + +async function payload(harness: Harness): Promise { + return harness.getExtensionRunner().emitBeforeProviderRequest({ + model: harness.session.model?.id, + tools: [clientTool, readTool, { type: "image_generation", model: "gpt-image-1" }], + }); +} + +async function skillNames(harness: Harness): Promise { + const resources = await harness.getExtensionRunner().emitResourcesDiscover(harness.tempDir, "reload"); + const loaded = loadSkills({ + cwd: harness.tempDir, + agentDir: join(harness.tempDir, "agent"), + skillPaths: resources.skillPaths.map((entry) => entry.path), + includeDefaults: false, + }); + expect(loaded.diagnostics).toEqual([]); + return loaded.skills.map((skill) => skill.name); +} + +async function prompt(harness: Harness) { + return harness.getExtensionRunner().emitBeforeAgentStart("draw a fox", undefined, "base", { cwd: harness.tempDir }); +} + +beforeEach(() => { + vi.stubEnv("OPENAI_API_KEY", ""); + vi.stubEnv("PI_IMAGE_GEN_PROVIDER", ""); + vi.stubEnv("PI_OPENAI_IMAGE_GEN", ""); + setImageGenRegistry(oauthOnly); + setNativeBypass(false); +}); + +afterEach(() => { + while (harnesses.length > 0) harnesses.pop()?.cleanup(); + setImageGenRegistry(undefined); + setNativeBypass(false); + vi.unstubAllEnvs(); +}); + +describe("Codex OAuth image generation lifecycle", () => { + it("automatically injects one pinned native tool and bypasses the client without changing transport", async () => { + const harness = await start(); + expect((await resolveImageGenAuth({ modelRegistry: oauthOnly })).kind).toBe("none"); + expect(await payload(harness)).toEqual({ model: codex.id, tools: [readTool, nativeTool] }); + const result = await harness.session.executeTool("generate_image", { prompt: "a fox" }); + expect(result.details.reason).toBe("provider_native_bypass"); + expect(harness.session.model).toEqual(codex); + expect(await skillNames(harness)).toEqual(["gpt-image-gen"]); + expect((await prompt(harness))?.systemPrompt).toBe(`base\n${IMAGE_GEN_SECTION}\n${OPENAI_IMAGE_GEN_SECTION}`); + }); + + it.each([ + ["Codex OAuth", codex], + ["OpenAI Responses", { ...codex, api: "openai-responses", baseUrl: "https://api.openai.com/v1" }], + ] as const)( + "contributes the bundled skill and shipped guidance with only native capability: %s", + async (_label, model) => { + const harness = await start(model, false); + expect(await skillNames(harness)).toEqual(["gpt-image-gen"]); + expect((await prompt(harness))?.systemPrompt).toBe(`base\n${IMAGE_GEN_SECTION}`); + }, + ); + + it.each([ + ["proxy", proxy], + ["explicit opt-out", optedOut], + ] as const)( + "removes native tooling and guidance when switching to %s without image credentials", + async (_label, model) => { + const harness = await start(); + expect(await payload(harness)).toEqual({ model: codex.id, tools: [readTool, nativeTool] }); + await harness.session.setModel(model); + expect(await payload(harness)).toEqual({ model: model.id, tools: [readTool] }); + expect(await skillNames(harness)).toEqual([]); + expect(await prompt(harness)).toBeUndefined(); + const result = await harness.session.executeTool("generate_image", { prompt: "a fox" }); + expect(result.details.reason).toBe("missing_config"); + }, + ); + + it("honors the global disable even with proxy opt-in", async () => { + vi.stubEnv("PI_OPENAI_IMAGE_GEN", "off"); + const harness = await start({ ...proxy, compat: { supportsImageGeneration: true } }); + expect(await payload(harness)).toEqual({ model: proxy.id, tools: [readTool] }); + expect(await skillNames(harness)).toEqual([]); + expect(await prompt(harness)).toBeUndefined(); + }); + + it("keeps the client descriptor and skill when disabled native generation has gateway credentials", async () => { + vi.stubEnv("PI_OPENAI_IMAGE_GEN", "0"); + setImageGenRegistry({ + ...oauthOnly, + getAll: () => [{ ...proxy, provider: "image-gateway-fixture", api: "openai-responses" }], + getApiKeyAndHeaders: async () => ({ ok: true, apiKey: "offline-image-gateway-fixture" }), + }); + const harness = await start(); + expect(await payload(harness)).toEqual({ model: codex.id, tools: [clientTool, readTool] }); + expect(await skillNames(harness)).toEqual(["gpt-image-gen"]); + expect((await prompt(harness))?.systemPrompt).toBe(`base\n${IMAGE_GEN_SECTION}`); + }); +}); diff --git a/packages/coding-agent/test/suite/regressions/imagegen-arbitration.test.ts b/packages/coding-agent/test/suite/regressions/imagegen-arbitration.test.ts index 591e213a3..f307ec6b0 100644 --- a/packages/coding-agent/test/suite/regressions/imagegen-arbitration.test.ts +++ b/packages/coding-agent/test/suite/regressions/imagegen-arbitration.test.ts @@ -198,7 +198,7 @@ type EnvDirection = "on" | "off"; interface Expected { /** Skill paths emitted by resources_discover. */ skillPresent: boolean; - /** IMAGE_GEN_SECTION (client) from imagegen before_agent_start. */ + /** Shared IMAGE_GEN_SECTION guidance from imagegen before_agent_start. */ clientSection: boolean; /** OPENAI_IMAGE_GEN_SECTION (native) from openai-image-gen before_agent_start. */ nativeSection: boolean; @@ -243,13 +243,13 @@ function registryFor(creds: CredDirection): ImageGenAuthRegistry { // // Native injection depends on: model (api=responses + official-or-compat) AND env=on. // Client tool behavior depends on: nativeBypass (set by injector) or resolveImageGenAuth. -// Skill contribution depends on: resolveImageGenAuth != none. +// Skill contribution depends on: native injection OR resolveImageGenAuth != none. // // Coherence rules: // nativeInjection > 0 → toolBehavior = bypass AND nativeSection = true // nativeInjection = 0 → nativeSection = false -// creds != none → skillPresent = true AND clientSection = true (when not native) -// creds = none → skillPresent = false AND clientSection = false +// creds != none OR nativeInjection > 0 → skillPresent = true AND clientSection = true +// creds = none + nativeInjection = 0 → skillPresent = false AND clientSection = false // creds = none + nativeInjection = 0 → toolBehavior = missing_config // creds != none + nativeInjection = 0 → toolBehavior = live @@ -261,8 +261,8 @@ const TRUTH_TABLE: TruthRow[] = [ env: "on", label: "no creds, official endpoint, env on → native injection without client creds", expected: { - skillPresent: false, - clientSection: false, + skillPresent: true, + clientSection: true, nativeSection: true, toolBehavior: "provider_native_bypass", nativeInjection: 1, @@ -527,8 +527,8 @@ const TRUTH_TABLE: TruthRow[] = [ env: "on", label: "no creds, proxied + compat true, env on → native injection without client creds", expected: { - skillPresent: false, - clientSection: false, + skillPresent: true, + clientSection: true, nativeSection: true, toolBehavior: "provider_native_bypass", nativeInjection: 1, @@ -677,7 +677,7 @@ describe("imagegen arbitration truth table", () => { expect(exp.nativeSection, "native section implies native injection").toBe(hasNative); expect(exp.toolBehavior === "provider_native_bypass", "bypass implies native injection").toBe(hasNative); - // (client and native sections CAN coexist when creds exist and native mode is active; + // (shared and native sections coexist whenever native mode is active; // the imagegen section is conditional-safe text that applies regardless of which surface owns the request) }); } @@ -693,15 +693,19 @@ describe("imagegen arbitration truth table", () => { hasNative, ); expect(exp.nativeSection, `${row.label}: native section ⟺ native injection`).toBe(hasNative); - expect(exp.clientSection, `${row.label}: client section ⟺ creds exist`).toBe(row.creds !== "none"); + expect(exp.clientSection, `${row.label}: shared guidance ⟺ native or creds`).toBe( + hasNative || row.creds !== "none", + ); } }); - it("every truth-table row satisfies: skill present ⟺ creds != none", () => { + it("every truth-table row satisfies: skill present ⟺ native injection or creds != none", () => { for (const row of TRUTH_TABLE) { const exp = row.expected; const hasCreds = row.creds !== "none"; - expect(exp.skillPresent, `${row.label}: skill presence ⟺ creds exist`).toBe(hasCreds); + expect(exp.skillPresent, `${row.label}: skill presence ⟺ native or creds`).toBe( + exp.nativeInjection > 0 || hasCreds, + ); } });