diff --git a/src/cli/effort.ts b/src/cli/effort.ts index cd4a2c4..859b022 100644 --- a/src/cli/effort.ts +++ b/src/cli/effort.ts @@ -3,9 +3,13 @@ export { EffortCommand }; import * as p from "@clack/prompts"; import { Future } from "@/libs/future"; -import { type Config } from "@/domain/config/config"; +import { type Config, type ProviderConfig } from "@/domain/config/config"; import { loadConfig, saveConfig } from "@/infra/storage/config"; import { selectEffortForProvider } from "@/domain/llm/effort"; +import { resolveProvider } from "@/domain/llm/auth-resolver"; +import { fetchModels } from "@/domain/commit/models"; +import { loading } from "@/infra/ui/spinner"; +import { fromOptional } from "@/libs/maybe"; import color from "picocolors"; @@ -21,7 +25,7 @@ class EffortCommand { run(): Future { p.intro(color.bgCyan(color.black(" Change Effort "))); - return selectEffortForProvider(this.config.ai) + return this.selectEffort() .chain((ai) => saveConfig({ ...this.config, ai })) .map(() => p.outro(color.green("Effort updated successfully!"))) .mapRej((e) => { @@ -29,4 +33,20 @@ class EffortCommand { return e; }); } + + private selectEffort(): Future { + const { ai } = this.config; + + if (ai.provider !== "openai" || ai.auth_method.type !== "openai_oauth") { + return selectEffortForProvider(ai); + } + + return resolveProvider(this.config).chain((current) => + loading("Fetching model capabilities...", "Model capabilities fetched!", fetchModels(current.provider, current.auth_method)).chain((models) => { + const capabilities = fromOptional(models.find((model) => model.id === current.model)).chain((model) => model.openaiEffort); + + return selectEffortForProvider(current, capabilities); + }) + ); + } } diff --git a/src/cli/model.ts b/src/cli/model.ts index b98fcf5..ae55780 100644 --- a/src/cli/model.ts +++ b/src/cli/model.ts @@ -30,7 +30,7 @@ class ModelCommand { return loading("Fetching available models...", "Models fetched!", fetchModels(this.providerConfig.provider, this.providerConfig.auth_method)) .chain((models) => selectModelInteractively(models)) - .chain((modelId) => selectEffortForProvider(withModel(this.config.ai, modelId))) + .chain((model) => selectEffortForProvider(withModel(this.providerConfig, model.id), model.openaiEffort)) .chain((ai) => saveConfig({ ...this.config, ai })) .map(() => p.outro(color.green("Model updated successfully!"))) .mapRej((e) => { diff --git a/src/cli/setup.ts b/src/cli/setup.ts index c67d974..4ca6a62 100644 --- a/src/cli/setup.ts +++ b/src/cli/setup.ts @@ -172,7 +172,7 @@ class Setup { private finalizeAfterModels(authMethod: ProviderConfig["auth_method"], models: Model[]): Future { return selectModelInteractively(models) - .chain((modelId) => selectEffortForProvider(seedProviderConfig(this.preferences.provider, modelId, authMethod))) + .chain((model) => selectEffortForProvider(seedProviderConfig(this.preferences.provider, model.id, authMethod), model.openaiEffort)) .chain((ai) => saveConfig(this.buildConfig(ai))) .map(() => { p.outro(color.green("Setup complete!")); diff --git a/src/domain/commit/models.ts b/src/domain/commit/models.ts index 74673bc..245e17f 100644 --- a/src/domain/commit/models.ts +++ b/src/domain/commit/models.ts @@ -1,9 +1,10 @@ export { fetchModels }; import { Future } from "@/libs/future"; -import { Model, type ProviderConfig } from "@/domain/config/config"; +import { OPENAI_EFFORTS, type Model, type OpenAIEffort, type OpenAIModelEffort, type ProviderConfig } from "@/domain/config/config"; import { getOpenAIAccessToken } from "@/infra/auth/openai"; import { anthropicOAuthHeaders } from "@/infra/auth/anthropic"; +import { Just, Nothing, type Maybe } from "@/libs/maybe"; import OpenAI from "openai"; @@ -11,6 +12,19 @@ type CodexModel = { readonly slug: string; readonly display_name: string; readonly description: string; + readonly default_reasoning_level?: string; + readonly supported_reasoning_levels?: readonly { readonly effort: string }[]; +}; + +const isOpenAIEffort = (value: string): value is OpenAIEffort => OPENAI_EFFORTS.some((effort) => effort === value); + +const openAIEffortFor = (model: CodexModel): Maybe => { + const [first, ...rest] = (model.supported_reasoning_levels ?? []).map(({ effort }) => effort).filter(isOpenAIEffort); + if (first === undefined) return Nothing(); + + const options: [OpenAIEffort, ...OpenAIEffort[]] = [first, ...rest]; + const defaultValue = options.find((effort) => effort === model.default_reasoning_level) ?? options.find((effort) => effort === "medium") ?? first; + return Just({ options, defaultValue }); }; const fetchOpenAIModelsWithApiKey = (apiKey: string): Future => @@ -24,7 +38,7 @@ const fetchOpenAIModelsWithApiKey = (apiKey: string): Future => return models .filter((m) => m.id.startsWith("gpt-") || m.id.startsWith("o")) .sort((a, b) => a.id.localeCompare(b.id)) - .map((m) => ({ id: m.id, description: "" })); + .map((m) => ({ id: m.id, description: "", openaiEffort: Nothing() })); }); const fetchOpenAIModelsWithOAuth = (tokens: ProviderConfig["auth_method"]["content"]): Future => @@ -41,7 +55,9 @@ const fetchOpenAIModelsWithOAuth = (tokens: ProviderConfig["auth_method"]["conte } const data = (await response.json()) as { models: CodexModel[] }; - return data.models.sort((a, b) => a.slug.localeCompare(b.slug)).map((m) => ({ id: m.slug, description: m.description })); + return data.models + .sort((a, b) => a.slug.localeCompare(b.slug)) + .map((m) => ({ id: m.slug, description: m.description, openaiEffort: openAIEffortFor(m) })); }) ); @@ -76,7 +92,8 @@ const fetchGeminiModels = (authMethod: ProviderConfig["auth_method"]): Future ({ id: m.name.replace("models/", ""), - description: m.description || "" + description: m.description || "", + openaiEffort: Nothing() })); }); @@ -110,7 +127,9 @@ const fetchAnthropicModels = (authMethod: ProviderConfig["auth_method"]): Future data?: Array<{ id: string; display_name?: string }>; }; - return (data.data ?? []).sort((a, b) => a.id.localeCompare(b.id)).map((m) => ({ id: m.id, description: m.display_name ?? "" })); + return (data.data ?? []) + .sort((a, b) => a.id.localeCompare(b.id)) + .map((m) => ({ id: m.id, description: m.display_name ?? "", openaiEffort: Nothing() })); }); const fetchModels = (provider: ProviderConfig["provider"], authMethod: ProviderConfig["auth_method"]): Future => { diff --git a/src/domain/config/config.ts b/src/domain/config/config.ts index 986001c..31386f3 100644 --- a/src/domain/config/config.ts +++ b/src/domain/config/config.ts @@ -6,6 +6,7 @@ export { type AuthMethod, type ProviderConfig, type OpenAIEffort, + type OpenAIModelEffort, type AnthropicEffort, type GeminiEffort, type Model, @@ -25,6 +26,7 @@ export { import * as s from "@/libs/json/schema"; import { absurd } from "@/libs/types"; +import { type Maybe } from "@/libs/maybe"; import { ThinkingLevel } from "@google/genai"; import type OpenAIPkg from "openai"; @@ -78,6 +80,10 @@ const ANTHROPIC_EFFORTS = ["low", "medium", "high", "xhigh", "max"] as const sat const GEMINI_EFFORTS = [ThinkingLevel.MINIMAL, ThinkingLevel.LOW, ThinkingLevel.MEDIUM, ThinkingLevel.HIGH] as const satisfies readonly ThinkingLevel[]; type OpenAIEffort = (typeof OPENAI_EFFORTS)[number]; +type OpenAIModelEffort = { + readonly options: readonly [OpenAIEffort, ...OpenAIEffort[]]; + readonly defaultValue: OpenAIEffort; +}; type AnthropicEffort = (typeof ANTHROPIC_EFFORTS)[number]; type GeminiEffort = (typeof GEMINI_EFFORTS)[number]; @@ -123,8 +129,8 @@ const Config = s.object({ }); type Config = s.Infer; -const Model = s.object({ - id: s.string, - description: s.string -}); -type Model = s.Infer; +type Model = { + readonly id: string; + readonly description: string; + readonly openaiEffort: Maybe; +}; diff --git a/src/domain/llm/effort.ts b/src/domain/llm/effort.ts index 6db50ae..abe86b4 100644 --- a/src/domain/llm/effort.ts +++ b/src/domain/llm/effort.ts @@ -1,8 +1,8 @@ export { seedProviderConfig, withModel, selectEffortForProvider }; import { type Future } from "@/libs/future"; -import { type ProviderConfig, type OpenAIEffort, type AnthropicEffort, type GeminiEffort } from "@/domain/config/config"; -import { Nothing } from "@/libs/maybe"; +import { type ProviderConfig, type OpenAIEffort, type OpenAIModelEffort, type AnthropicEffort, type GeminiEffort } from "@/domain/config/config"; +import { Nothing, type Maybe } from "@/libs/maybe"; import { selectOpenAIEffort, selectAnthropicEffort, selectGeminiEffort } from "@/infra/ui/effort-picker"; import { absurd } from "@/libs/types"; @@ -32,10 +32,10 @@ const withModel = (ai: ProviderConfig, model: string): ProviderConfig => { } }; -const selectEffortForProvider = (current: ProviderConfig): Future => { +const selectEffortForProvider = (current: ProviderConfig, modelEffort: Maybe = Nothing()): Future => { switch (current.provider) { case "openai": - return selectOpenAIEffort(current.model, current.effort).map( + return selectOpenAIEffort(current.model, current.effort, modelEffort).map( (effort): ProviderConfig => ({ provider: "openai", model: current.model, diff --git a/src/domain/llm/router.ts b/src/domain/llm/router.ts index eb0a451..6074b56 100644 --- a/src/domain/llm/router.ts +++ b/src/domain/llm/router.ts @@ -59,25 +59,34 @@ type BranchNameSuggestions = { type ProviderGeneratedContent = { readonly text: string; readonly tokens: Maybe; + readonly effectiveEffort: Maybe; }; -const modelRequestMetadata = (config: ProviderConfig): ModelRequestMetadata => { +const modelRequestMetadata = (config: ProviderConfig, effectiveEffort: Maybe): ModelRequestMetadata => { switch (config.provider) { case "openai": - return { provider: config.provider, model: config.model, effort: config.effort.maybe("provider default", (effort) => effort) }; + return { + provider: config.provider, + model: config.model, + effort: effectiveEffort.withDefault(config.effort.maybe("provider default", (effort) => effort)) + }; case "gemini": case "anthropic": - return { provider: config.provider, model: config.model, effort: config.effort.maybe("medium", (effort) => effort) }; + return { + provider: config.provider, + model: config.model, + effort: effectiveEffort.withDefault(config.effort.maybe("medium", (effort) => effort)) + }; } }; const withRequestMetadata = (config: ProviderConfig, f: Future): Future => { const startedAt = Date.now(); - return f.map(({ text, tokens }) => ({ + return f.map(({ text, tokens, effectiveEffort }) => ({ text, metadata: { durationMs: Date.now() - startedAt, - model: modelRequestMetadata(config), + model: modelRequestMetadata(config, effectiveEffort), tokens } })); diff --git a/src/infra/llm/anthropic.ts b/src/infra/llm/anthropic.ts index c391218..698b5aa 100644 --- a/src/infra/llm/anthropic.ts +++ b/src/infra/llm/anthropic.ts @@ -9,7 +9,7 @@ import { anthropicOAuthHeaders, CLAUDE_CODE_SYSTEM_PROMPT } from "@/infra/auth/a import { absurd } from "@/libs/types"; import { extractResponse } from "@/domain/llm/response-parser"; import { unsupportedAuth } from "@/domain/llm/auth-error"; -import { Just, fromOptional, type Maybe } from "@/libs/maybe"; +import { Just, Nothing, fromOptional, type Maybe } from "@/libs/maybe"; type AnthropicConfig = Extract; type SystemParam = NonNullable; @@ -66,7 +66,8 @@ const callAnthropicWithApiKey = ( .chain((message) => extractResponse({ text: Just(extractAnthropicText(message.content)) }).map((text) => ({ text, - tokens: Just(toTokenUsage(message.usage)) + tokens: Just(toTokenUsage(message.usage)), + effectiveEffort: Nothing() })) ); @@ -92,7 +93,8 @@ const callAnthropicWithSetupToken = ( .chain((message) => extractResponse({ text: Just(extractAnthropicText(message.content)) }).map((text) => ({ text, - tokens: Just(toTokenUsage(message.usage)) + tokens: Just(toTokenUsage(message.usage)), + effectiveEffort: Nothing() })) ); diff --git a/src/infra/llm/gemini.ts b/src/infra/llm/gemini.ts index ffa750c..4ac565b 100644 --- a/src/infra/llm/gemini.ts +++ b/src/infra/llm/gemini.ts @@ -26,7 +26,8 @@ const extractGeminiText = (response: GenerateContentResponse): string => const toGeneratedContent = (response: GenerateContentResponse): ProviderGeneratedContent => ({ text: extractGeminiText(response), - tokens: toTokenUsage(response.usageMetadata) + tokens: toTokenUsage(response.usageMetadata), + effectiveEffort: Nothing() }); const getAuthCredentials = (config: Config): Maybe => { diff --git a/src/infra/llm/openai.ts b/src/infra/llm/openai.ts index e8bbf9b..132821a 100644 --- a/src/infra/llm/openai.ts +++ b/src/infra/llm/openai.ts @@ -9,13 +9,14 @@ import { getOpenAIAccessToken } from "@/infra/auth/openai"; import { extractResponse } from "@/domain/llm/response-parser"; import { unsupportedAuth } from "@/domain/llm/auth-error"; import { absurd } from "@/libs/types"; -import { Just, fromOptional, type Maybe } from "@/libs/maybe"; +import { Just, Nothing, fromOptional, type Maybe } from "@/libs/maybe"; type OpenAIConfig = Extract; type StreamBundle = { response: OpenAI.Responses.Response; doneEventText: string; deltaSnapshotText: string; + attemptedEffort: Maybe; }; const extractStreamText = (bundle: StreamBundle): Maybe => { @@ -47,6 +48,40 @@ const buildStreamParams = (model: string, effort: Maybe, params: G return openaiReasoning(effort).maybe(core, (r) => ({ ...core, reasoning: r })); }; +const isUnsupportedEffort = (error: unknown, effort: OpenAIEffort): boolean => { + if (!(error instanceof OpenAI.BadRequestError)) return false; + + const typedError = error.param === "reasoning.effort" && error.code === "unsupported_value"; + const quotedEfforts = [`'${effort}'`, `"${effort}"`, `\`${effort}\``]; + const codexError = + quotedEfforts.some((quotedEffort) => error.message.includes(`Unsupported value: ${quotedEffort} is not supported`)) && + error.message.includes("Supported values are:"); + return typedError || codexError; +}; + +const readOpenAIStream = async (client: OpenAI, model: string, effort: Maybe, params: GenerateContentParams): Promise => { + const stream = client.responses.stream(buildStreamParams(model, effort, params)); + + let deltaSnapshotText = ""; + let doneEventText = ""; + + stream.on("response.output_text.delta", (event) => { + deltaSnapshotText = event.snapshot; + }); + + stream.on("response.output_text.done", (event) => { + doneEventText = event.text; + }); + + const response = await stream.finalResponse(); + return { + response, + doneEventText, + deltaSnapshotText, + attemptedEffort: effort + }; +}; + const callOpenAIStream = ( client: OpenAI, model: string, @@ -54,27 +89,20 @@ const callOpenAIStream = ( params: GenerateContentParams ): Future => Future.attemptP(async () => { - const stream = client.responses.stream(buildStreamParams(model, effort, params)); - - let deltaSnapshotText = ""; - let doneEventText = ""; - - stream.on("response.output_text.delta", (event) => { - deltaSnapshotText = event.snapshot; - }); - - stream.on("response.output_text.done", (event) => { - doneEventText = event.text; - }); - - const response = await stream.finalResponse(); - return { response, doneEventText, deltaSnapshotText }; + try { + return await readOpenAIStream(client, model, effort, params); + } catch (error) { + const selectedEffort = effort.asNullable(); + if (selectedEffort === null || !isUnsupportedEffort(error, selectedEffort)) throw error; + return await readOpenAIStream(client, model, Nothing(), params); + } }) .mapRej((error) => new Error(`Failed to create OpenAI response: ${error instanceof Error ? error.message : String(error)}`)) .chain((bundle) => extractResponse({ text: extractStreamText(bundle) }).map((text) => ({ text, - tokens: fromOptional(bundle.response.usage).map(toTokenUsage) + tokens: fromOptional(bundle.response.usage).map(toTokenUsage), + effectiveEffort: Just(bundle.attemptedEffort.maybe("provider default", (value) => value)) })) ); diff --git a/src/infra/ui/effort-picker.ts b/src/infra/ui/effort-picker.ts index 37a8287..34b8d63 100644 --- a/src/infra/ui/effort-picker.ts +++ b/src/infra/ui/effort-picker.ts @@ -4,15 +4,26 @@ import { ThinkingLevel } from "@google/genai"; import { Future } from "@/libs/future"; import { Just, type Maybe } from "@/libs/maybe"; -import { OPENAI_EFFORTS, ANTHROPIC_EFFORTS, GEMINI_EFFORTS, type OpenAIEffort, type AnthropicEffort, type GeminiEffort } from "@/domain/config/config"; +import { + OPENAI_EFFORTS, + ANTHROPIC_EFFORTS, + GEMINI_EFFORTS, + type OpenAIEffort, + type OpenAIModelEffort, + type AnthropicEffort, + type GeminiEffort +} from "@/domain/config/config"; type EffortSliderModule = typeof import("@/infra/ui/effort-slider"); -const selectEffort = (options: readonly V[], modelId: string, currentEffort: Maybe, defaultValue: V): Future> => { - const initialIndex = currentEffort.maybe(Math.max(0, options.indexOf(defaultValue)), (v) => { - const idx = options.indexOf(v); - return idx >= 0 ? idx : Math.max(0, options.indexOf(defaultValue)); - }); +const selectEffort = ( + options: readonly [V, ...V[]], + modelId: string, + currentEffort: Maybe, + defaultValue: V +): Future> => { + const normalizedCurrent = currentEffort.map((value) => (options.includes(value) ? value : defaultValue)); + const initialIndex = normalizedCurrent.maybe(options.indexOf(defaultValue), (value) => options.indexOf(value)); return Future.attemptP(async () => { // Lazy-load Ink/React so non-interactive CLI paths don't pay their startup cost. @@ -39,7 +50,7 @@ const selectEffort = (options: readonly V[], modelId: string, }, onCancel: () => { unmount(); - resolve(currentEffort); + resolve(normalizedCurrent); } }) ); @@ -48,8 +59,10 @@ const selectEffort = (options: readonly V[], modelId: string, ); }; -const selectOpenAIEffort = (modelId: string, current: Maybe): Future> => - selectEffort(OPENAI_EFFORTS, modelId, current, "medium"); +const selectOpenAIEffort = (modelId: string, current: Maybe, modelEffort: Maybe): Future> => { + const { options, defaultValue } = modelEffort.withDefault({ options: OPENAI_EFFORTS, defaultValue: "medium" }); + return selectEffort(options, modelId, current, defaultValue); +}; const selectAnthropicEffort = (modelId: string, current: Maybe): Future> => selectEffort(ANTHROPIC_EFFORTS, modelId, current, "medium"); diff --git a/src/infra/ui/model-picker.ts b/src/infra/ui/model-picker.ts index 60440d0..85f7017 100644 --- a/src/infra/ui/model-picker.ts +++ b/src/infra/ui/model-picker.ts @@ -1,9 +1,9 @@ export { selectModelInteractively }; import { Future } from "@/libs/future"; -import { Model } from "@/domain/config/config"; +import { type Model } from "@/domain/config/config"; -const selectModelInteractively = (models: Model[]): Future => +const selectModelInteractively = (models: Model[]): Future => Future.attemptP(async () => { // Lazy-load Ink/React so non-interactive CLI paths don't pay their startup cost. // Ink pulls in React, Yoga layout, and a render loop — non-trivial to initialize @@ -17,13 +17,13 @@ const selectModelInteractively = (models: Model[]): Future => const { ModelSelector } = await import("@/infra/ui/model-selector"); return { render, React, ModelSelector }; }).chain(({ render, React, ModelSelector }) => - Future.create((reject, resolve) => { + Future.create((reject, resolve) => { const { unmount } = render( React.createElement(ModelSelector, { models, - onSelect: (modelId: string) => { + onSelect: (model: Model) => { unmount(); - resolve(modelId); + resolve(model); }, onCancel: () => { unmount(); diff --git a/src/infra/ui/model-selector.tsx b/src/infra/ui/model-selector.tsx index f8f44f0..d01739b 100644 --- a/src/infra/ui/model-selector.tsx +++ b/src/infra/ui/model-selector.tsx @@ -2,20 +2,17 @@ export { ModelSelector, type Model, type ModelSelectorProps }; import * as React from "react"; +import type { Model } from "@/domain/config/config"; + import { Box, Text, useInput, useApp, type Key } from "ink"; import chalk from "chalk"; import { search } from "@/libs/fuzzy"; -type Model = { - id: string; - description: string; -}; - type ModelSelectorProps = { models: Model[]; - onSelect: (modelId: string) => void; + onSelect: (model: Model) => void; onCancel: () => void; }; @@ -57,7 +54,7 @@ const ModelSelector = ({ models, onSelect, onCancel }: ModelSelectorProps) => { if (key.return) { const selectedModel = filteredModels[selectedIndex]; if (selectedModel) { - onSelect(selectedModel.id); + onSelect(selectedModel); } else { onCancel(); } diff --git a/test/cli/effort.test.ts b/test/cli/effort.test.ts index b9b668e..9ffee0d 100644 --- a/test/cli/effort.test.ts +++ b/test/cli/effort.test.ts @@ -15,6 +15,15 @@ vi.mock("@/infra/storage/config", () => ({ vi.mock("@/infra/ui/effort-picker", () => ({ selectOpenAIEffort: vi.fn(() => Future.resolve(Just("low"))) })); +vi.mock("@/domain/llm/auth-resolver", () => ({ + resolveProvider: vi.fn() +})); +vi.mock("@/domain/commit/models", () => ({ + fetchModels: vi.fn() +})); +vi.mock("@/infra/ui/spinner", () => ({ + loading: vi.fn((_a: string, _b: string, f: Future) => f as Future) +})); vi.mock("@clack/prompts", () => ({ intro: vi.fn(), outro: vi.fn(), @@ -23,6 +32,7 @@ vi.mock("@clack/prompts", () => ({ describe("EffortCommand", () => { beforeEach(async () => { + vi.clearAllMocks(); const storage = await import("@/infra/storage/config"); vi.mocked(storage.loadConfig).mockReturnValue( Future.resolve({ @@ -38,4 +48,36 @@ describe("EffortCommand", () => { await runFuture(EffortCommand.create().chain((e) => e.run())); expect(saveConfig).toHaveBeenCalled(); }); + + it("uses live OAuth model capabilities", async () => { + const config: ConfigValue = { + commit_convention: "conventional", + custom_template: Nothing(), + ai: { + provider: "openai", + model: "gpt-5.6-sol", + effort: Nothing(), + auth_method: { type: "openai_oauth", content: { access_token: "access", refresh_token: "refresh", expiry_date: 0 } } + } + }; + const model = { + id: "gpt-5.6-sol", + description: "", + openaiEffort: Just({ options: ["low", "medium", "high", "xhigh"] as const, defaultValue: "low" as const }) + }; + const storage = await import("@/infra/storage/config"); + const resolver = await import("@/domain/llm/auth-resolver"); + const models = await import("@/domain/commit/models"); + const picker = await import("@/infra/ui/effort-picker"); + + vi.mocked(storage.loadConfig).mockReturnValue(Future.resolve(config)); + vi.mocked(resolver.resolveProvider).mockReturnValue(Future.resolve(config.ai)); + vi.mocked(models.fetchModels).mockReturnValue(Future.resolve([model])); + + await runFuture(EffortCommand.create().chain((command) => command.run())); + + expect(models.fetchModels).toHaveBeenCalledWith("openai", config.ai.auth_method); + const [, , capabilities] = vi.mocked(picker.selectOpenAIEffort).mock.calls[0] ?? []; + expect(capabilities).toEqual(model.openaiEffort); + }); }); diff --git a/test/cli/model.test.ts b/test/cli/model.test.ts index 8e948b9..77ee9b0 100644 --- a/test/cli/model.test.ts +++ b/test/cli/model.test.ts @@ -22,10 +22,24 @@ vi.mock("@/domain/llm/auth-resolver", () => ({ resolveProvider: vi.fn((c: ConfigValue) => Future.resolve(c.ai)) })); vi.mock("@/domain/commit/models", () => ({ - fetchModels: vi.fn(() => Future.resolve([{ id: "gpt-4.1-mini", description: "fast" }])) + fetchModels: vi.fn(() => + Future.resolve([ + { + id: "gpt-4.1-mini", + description: "fast", + openaiEffort: Just({ options: ["low", "medium", "high", "xhigh"] as const, defaultValue: "medium" as const }) + } + ]) + ) })); vi.mock("@/infra/ui/model-picker", () => ({ - selectModelInteractively: vi.fn(() => Future.resolve("gpt-4.1-mini")) + selectModelInteractively: vi.fn(() => + Future.resolve({ + id: "gpt-4.1-mini", + description: "fast", + openaiEffort: Just({ options: ["low", "medium", "high", "xhigh"] as const, defaultValue: "medium" as const }) + }) + ) })); vi.mock("@/infra/ui/effort-picker", () => ({ selectOpenAIEffort: vi.fn(() => Future.resolve(Just("high"))) diff --git a/test/cli/setup.test.ts b/test/cli/setup.test.ts index 02c7cf9..49c5057 100644 --- a/test/cli/setup.test.ts +++ b/test/cli/setup.test.ts @@ -27,10 +27,24 @@ vi.mock("@clack/prompts", () => ({ log: { info: vi.fn(), warn: vi.fn(), error: vi.fn() } })); vi.mock("@/domain/commit/models", () => ({ - fetchModels: vi.fn(() => Future.resolve([{ id: "gpt-4.1-mini", description: "fast" }])) + fetchModels: vi.fn(() => + Future.resolve([ + { + id: "gpt-4.1-mini", + description: "fast", + openaiEffort: Just({ options: ["low", "medium", "high", "xhigh"] as const, defaultValue: "medium" as const }) + } + ]) + ) })); vi.mock("@/infra/ui/model-picker", () => ({ - selectModelInteractively: vi.fn(() => Future.resolve("gpt-4.1-mini")) + selectModelInteractively: vi.fn(() => + Future.resolve({ + id: "gpt-4.1-mini", + description: "fast", + openaiEffort: Just({ options: ["low", "medium", "high", "xhigh"] as const, defaultValue: "medium" as const }) + }) + ) })); vi.mock("@/infra/ui/effort-picker", () => ({ selectOpenAIEffort: vi.fn(() => Future.resolve(Just("medium" as const))) diff --git a/test/domain/commit/models.test.ts b/test/domain/commit/models.test.ts new file mode 100644 index 0000000..3b772cc --- /dev/null +++ b/test/domain/commit/models.test.ts @@ -0,0 +1,51 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; + +import { fetchModels } from "@/domain/commit/models"; +import { runFuture } from "@test/helpers/run-future"; + +const openAIOAuth = { + type: "openai_oauth" as const, + content: { access_token: "access", refresh_token: "refresh", expiry_date: 0 } +}; + +afterEach(() => vi.unstubAllGlobals()); + +describe("fetchModels", () => { + it("preserves supported OpenAI efforts from the Codex catalog", async () => { + vi.stubGlobal( + "fetch", + vi.fn( + async () => + new Response( + JSON.stringify({ + models: [ + { + slug: "gpt-5.6-sol", + display_name: "GPT-5.6 Sol", + description: "", + default_reasoning_level: "low", + supported_reasoning_levels: [ + { effort: "low" }, + { effort: "medium" }, + { effort: "high" }, + { effort: "xhigh" }, + { effort: "max" }, + { effort: "ultra" } + ] + } + ] + }), + { status: 200 } + ) + ) + ); + + const [model] = await runFuture(fetchModels("openai", openAIOAuth)); + if (model === undefined) throw new Error("Expected a model"); + + expect(model.openaiEffort.expect("Expected OpenAI effort capabilities")).toEqual({ + options: ["low", "medium", "high", "xhigh"], + defaultValue: "low" + }); + }); +}); diff --git a/test/domain/config/config.test.ts b/test/domain/config/config.test.ts index ef0640a..bd45762 100644 --- a/test/domain/config/config.test.ts +++ b/test/domain/config/config.test.ts @@ -45,4 +45,15 @@ describe("Config schema", () => { expect(result.isSuccess()).toBe(true); if (result instanceof Success) expect(result.value.custom_template).toBeInstanceOf(Just); }); + + it("continues decoding legacy OpenAI configs containing minimal effort", () => { + const config = sampleConfig(); + if (config.ai.provider !== "openai") throw new Error("Expected OpenAI config"); + const encoded = s.encode(Config, { + ...config, + ai: { ...config.ai, effort: Just("minimal" as const) } + }); + + expect(s.decode(Config, encoded).isSuccess()).toBe(true); + }); }); diff --git a/test/domain/llm/router.test.ts b/test/domain/llm/router.test.ts index 3af16c8..0028d44 100644 --- a/test/domain/llm/router.test.ts +++ b/test/domain/llm/router.test.ts @@ -1,7 +1,7 @@ import { describe, expect, it, vi, beforeEach } from "vitest"; import { generateCommitMessage, refineCommitMessage } from "@/domain/llm/router"; import { Future } from "@/libs/future"; -import { Nothing } from "@/libs/maybe"; +import { Just, Nothing } from "@/libs/maybe"; import { runFuture } from "@test/helpers/run-future"; import type { ProviderConfig } from "@/domain/config/config"; @@ -14,13 +14,13 @@ const mockProvider = (provider: ProviderConfig["provider"]): ProviderConfig => }) as ProviderConfig; vi.mock("@/infra/llm/gemini", () => ({ - generateContentWithGemini: vi.fn(() => Future.resolve({ text: "feat: test", tokens: Nothing() })) + generateContentWithGemini: vi.fn(() => Future.resolve({ text: "feat: test", tokens: Nothing(), effectiveEffort: Nothing() })) })); vi.mock("@/infra/llm/openai", () => ({ - generateContentWithOpenAI: vi.fn(() => Future.resolve({ text: "feat: test", tokens: Nothing() })) + generateContentWithOpenAI: vi.fn(() => Future.resolve({ text: "feat: test", tokens: Nothing(), effectiveEffort: Nothing() })) })); vi.mock("@/infra/llm/anthropic", () => ({ - generateContentWithAnthropic: vi.fn(() => Future.resolve({ text: "feat: test", tokens: Nothing() })) + generateContentWithAnthropic: vi.fn(() => Future.resolve({ text: "feat: test", tokens: Nothing(), effectiveEffort: Nothing() })) })); describe("generateCommitMessage", () => { @@ -40,4 +40,14 @@ describe("refineCommitMessage", () => { await runFuture(refineCommitMessage(mockProvider("openai"), "feat: x", "shorter", "diff")); expect(generateContentWithOpenAI).toHaveBeenCalled(); }); + + it("reports the effort used by the provider", async () => { + const { generateContentWithOpenAI } = await import("@/infra/llm/openai"); + vi.mocked(generateContentWithOpenAI).mockReturnValue( + Future.resolve({ text: "feat: test", tokens: Nothing(), effectiveEffort: Just("provider default") }) + ); + + const result = await runFuture(refineCommitMessage(mockProvider("openai"), "feat: x", "shorter", "diff")); + expect(result.metadata.model.effort).toBe("provider default"); + }); }); diff --git a/test/infra/llm/openai.test.ts b/test/infra/llm/openai.test.ts new file mode 100644 index 0000000..2b43665 --- /dev/null +++ b/test/infra/llm/openai.test.ts @@ -0,0 +1,96 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +import OpenAI from "openai"; + +import { type Config, type OpenAIEffort } from "@/domain/config/config"; +import { generateContentWithOpenAI } from "@/infra/llm/openai"; +import { Just } from "@/libs/maybe"; +import { runFuture } from "@test/helpers/run-future"; + +const stream = vi.hoisted(() => vi.fn()); + +vi.mock("openai", async (importOriginal) => { + const actual = await importOriginal(); + + class MockOpenAI { + static BadRequestError = actual.default.BadRequestError; + readonly responses = { stream }; + } + + return { default: MockOpenAI }; +}); + +type OpenAIConfig = Extract; + +const configWith = (effort: OpenAIEffort): OpenAIConfig => ({ + provider: "openai", + model: "gpt-5.6-sol", + effort: Just(effort), + auth_method: { type: "api_key", content: "sk-test" } +}); + +const response = { + output: [{ type: "message", content: [{ type: "output_text", text: "feat: test" }] }], + output_text: "feat: test", + usage: { input_tokens: 1, output_tokens: 2, total_tokens: 3 } +}; + +const successfulStream = () => ({ + on: vi.fn(), + finalResponse: vi.fn().mockResolvedValue(response) +}); + +describe("generateContentWithOpenAI", () => { + beforeEach(() => stream.mockReset()); + + it("sends a supported effort once", async () => { + stream.mockReturnValue(successfulStream()); + + const result = await runFuture(generateContentWithOpenAI(configWith("low"), { prompt: "diff" })); + + expect(stream).toHaveBeenCalledTimes(1); + expect(stream.mock.calls[0]?.[0]).toMatchObject({ reasoning: { effort: "low" } }); + expect(result.effectiveEffort.expect("Expected effective effort")).toBe("low"); + }); + + it("retries an effort-specific unsupported value once without reasoning", async () => { + const error = new OpenAI.BadRequestError( + 400, + { code: "unsupported_value", param: "reasoning.effort", message: "Unsupported value: 'minimal' is not supported with the 'gpt-5.6-sol' model." }, + undefined, + new Headers() + ); + stream.mockReturnValueOnce({ on: vi.fn(), finalResponse: vi.fn().mockRejectedValue(error) }).mockReturnValueOnce(successfulStream()); + + const result = await runFuture(generateContentWithOpenAI(configWith("minimal"), { prompt: "diff" })); + + expect(stream).toHaveBeenCalledTimes(2); + expect(stream.mock.calls[0]?.[0]).toMatchObject({ reasoning: { effort: "minimal" } }); + expect(stream.mock.calls[1]?.[0]).not.toHaveProperty("reasoning"); + expect(result.effectiveEffort.expect("Expected effective effort")).toBe("provider default"); + }); + + it("retries a message-only unsupported value with backtick quotes", async () => { + const error = new OpenAI.BadRequestError( + 400, + undefined, + "Unsupported value: `minimal` is not supported with the `gpt-5.6-sol` model. Supported values are: `none`, `low`, `medium`, `high`, and `xhigh`.", + new Headers() + ); + stream.mockReturnValueOnce({ on: vi.fn(), finalResponse: vi.fn().mockRejectedValue(error) }).mockReturnValueOnce(successfulStream()); + + const result = await runFuture(generateContentWithOpenAI(configWith("minimal"), { prompt: "diff" })); + + expect(stream).toHaveBeenCalledTimes(2); + expect(stream.mock.calls[1]?.[0]).not.toHaveProperty("reasoning"); + expect(result.effectiveEffort.expect("Expected effective effort")).toBe("provider default"); + }); + + it("does not retry unrelated bad requests", async () => { + const error = new OpenAI.BadRequestError(400, { code: "invalid_parameter", param: "input", message: "Invalid input." }, undefined, new Headers()); + stream.mockReturnValue({ on: vi.fn(), finalResponse: vi.fn().mockRejectedValue(error) }); + + await expect(runFuture(generateContentWithOpenAI(configWith("minimal"), { prompt: "diff" }))).rejects.toThrow("Failed to create OpenAI response"); + expect(stream).toHaveBeenCalledTimes(1); + }); +}); diff --git a/test/infra/ui/effort-picker.test.ts b/test/infra/ui/effort-picker.test.ts new file mode 100644 index 0000000..1e24eef --- /dev/null +++ b/test/infra/ui/effort-picker.test.ts @@ -0,0 +1,45 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +import { selectOpenAIEffort } from "@/infra/ui/effort-picker"; +import { Just } from "@/libs/maybe"; +import { runFuture } from "@test/helpers/run-future"; + +const render = vi.hoisted(() => vi.fn()); + +vi.mock("ink", () => ({ render })); +vi.mock("@/infra/ui/effort-slider", () => ({ EffortSlider: () => null })); + +type SliderProps = { + readonly options: readonly string[]; + readonly onSubmit: (value: "low" | "medium" | "high" | "xhigh") => void; + readonly onCancel: () => void; +}; + +const capabilities = Just({ options: ["low", "medium", "high", "xhigh"] as const, defaultValue: "low" as const }); + +describe("selectOpenAIEffort", () => { + beforeEach(() => vi.clearAllMocks()); + + it("passes catalog capabilities to the effort slider", async () => { + render.mockImplementation((element: { props: SliderProps }) => { + queueMicrotask(() => element.props.onSubmit("high")); + return { unmount: vi.fn() }; + }); + + const result = await runFuture(selectOpenAIEffort("gpt-5.6-sol", Just("minimal"), capabilities)); + + expect(render.mock.calls[0]?.[0].props.options).toEqual(["low", "medium", "high", "xhigh"]); + expect(result.expect("Expected selected effort")).toBe("high"); + }); + + it("normalizes a stale effort to the model default on cancel", async () => { + render.mockImplementation((element: { props: SliderProps }) => { + queueMicrotask(() => element.props.onCancel()); + return { unmount: vi.fn() }; + }); + + const result = await runFuture(selectOpenAIEffort("gpt-5.6-sol", Just("minimal"), capabilities)); + + expect(result.expect("Expected normalized effort")).toBe("low"); + }); +});