diff --git a/README.md b/README.md index d431df9587..a4e7cf0538 100644 --- a/README.md +++ b/README.md @@ -55,6 +55,61 @@ verboo /login `WebFetch` works via basic HTTP plus HTML-to-markdown conversion. It may fail on JavaScript-rendered sites or sites that block plain HTTP requests. +## Agent Model Routing + +Verboo resolves subagent models against the authenticated `/models` catalog and +preserves the complete server-provided ID, including plan prefixes such as +`max/qwen3.6-27b`. Router-provided `agent_model_roles` are authoritative; the +portable profiles below are compatibility fallbacks and can also be selected by +user-defined workers. + +Configure agents in `~/.verboo/settings.json`: + +```json +{ + "agentRouting": { + "Explore": "fast", + "worker-review": { "profile": "review" }, + "worker-backend": { "profile": "coding" }, + "worker-tests": { "profile": "testing" }, + "default": { "profile": "balanced" } + } +} +``` + +Available profiles are `fast`, `review`, `coding`, `testing`, and `balanced`. +Each chooses a preferred model that is present in the logged-in account. If no +candidate is available, the worker inherits the parent model. A profile can opt +into the first catalog entry as a last resort with +`"fallback": "first-available"`. + +Forked and inline skills support the same profiles: + +```yaml +--- +name: worker-review +model: profile:review +context: fork +--- +``` + +An exact Verboo model can be requested without adding another API key. It is +used only when it exists in the authenticated catalog: + +```json +{ + "agentRouting": { + "worker-review": { + "model": "deepseek-v4-pro", + "provider": "inherit" + } + } +} +``` + +The legacy `agentModels` plus string `agentRouting` format for external +OpenAI-compatible providers remains supported. + --- ## Headless gRPC Server diff --git a/src/commands/model/model.test.tsx b/src/commands/model/model.test.tsx index e522eb647c..8953e8d699 100644 --- a/src/commands/model/model.test.tsx +++ b/src/commands/model/model.test.tsx @@ -75,6 +75,7 @@ mock.module('../../services/api/verbooModels.js', () => ({ clearVerbooModelsCache: () => {}, fetchVerbooModels: mock(async () => verbooModels), getCachedVerbooModels: () => verbooModels, + getVerbooAgentModelForRole: () => undefined, getVerbooModelMeta: (modelId: string) => verbooModels.find(model => model.id === modelId), getVerbooModelReasoning: () => undefined, diff --git a/src/components/StartupScreen.test.ts b/src/components/StartupScreen.test.ts index 991f5a6870..af00c2ace3 100644 --- a/src/components/StartupScreen.test.ts +++ b/src/components/StartupScreen.test.ts @@ -53,6 +53,7 @@ async function importStartupScreenWithModels( })) mock.module('../services/api/verbooModels.js', () => ({ getCachedVerbooModels: () => models, + getVerbooAgentModelForRole: () => undefined, getVerbooModelMeta: (modelId: string) => models.find(model => model.id === modelId), })) diff --git a/src/components/tasks/AsyncAgentDetailDialog.tsx b/src/components/tasks/AsyncAgentDetailDialog.tsx index aa6f3234e5..87fb34ef98 100644 --- a/src/components/tasks/AsyncAgentDetailDialog.tsx +++ b/src/components/tasks/AsyncAgentDetailDialog.tsx @@ -135,10 +135,11 @@ export function AsyncAgentDetailDialog(t0) { } else { t11 = $[19]; } + const elapsedAndModel = agent.model ? `${elapsedTime} · ${agent.model}` : elapsedTime; let t12; - if ($[20] !== elapsedTime || $[21] !== t10 || $[22] !== t11) { - t12 = {elapsedTime}{t10}{t11}; - $[20] = elapsedTime; + if ($[20] !== elapsedAndModel || $[21] !== t10 || $[22] !== t11) { + t12 = {elapsedAndModel}{t10}{t11}; + $[20] = elapsedAndModel; $[21] = t10; $[22] = t11; $[23] = t12; diff --git a/src/query.ts b/src/query.ts index 905c0ac283..9248c7d85f 100644 --- a/src/query.ts +++ b/src/query.ts @@ -48,6 +48,7 @@ import { maybeLogMemoryHighWatermark, } from './utils/memoryDiagnostics.js' import { + createAssistantMessage, createUserMessage, createUserInterruptionMessage, normalizeMessagesForAPI, @@ -111,6 +112,14 @@ import { } from './bootstrap/state.js' import { createBudgetTracker, checkTokenBudget } from './query/tokenBudget.js' import { count } from './utils/array.js' +import { + createBudgetedCanUseTool, + isAgentBudgetTimeout, + markAgentBudgetCompletion, + refreshAgentBudgetDeadline, + shouldFinalizeAgentBudget, + type AgentExecutionBudgetState, +} from './query/agentExecutionBudget.js' /* eslint-disable @typescript-eslint/no-require-imports */ const snipModule = feature('HISTORY_SNIP') ? (require('./services/compact/snipCompact.js') as typeof import('./services/compact/snipCompact.js')) @@ -123,7 +132,18 @@ const taskSummaryModule = feature('BG_SESSIONS') function* yieldMissingToolResultBlocks( assistantMessages: AssistantMessage[], errorMessage: string, + existingToolResults: Array = [], ) { + const completedToolUseIds = new Set( + existingToolResults.flatMap(message => { + if (message.type !== 'user' || !Array.isArray(message.message.content)) { + return [] + } + return message.message.content.flatMap(content => + content.type === 'tool_result' ? [content.tool_use_id] : [], + ) + }), + ) for (const assistantMessage of assistantMessages) { // Extract all tool use blocks from this assistant message const toolUseBlocks = assistantMessage.message.content.filter( @@ -132,6 +152,7 @@ function* yieldMissingToolResultBlocks( // Emit an interruption message for each tool use for (const toolUse of toolUseBlocks) { + if (completedToolUseIds.has(toolUse.id)) continue yield createUserMessage({ content: [ { @@ -190,6 +211,8 @@ export type QueryParams = { querySource: QuerySource maxOutputTokensOverride?: number maxTurns?: number + /** Optional mutable state shared across an agent's foreground/background lifecycle. */ + executionBudgetState?: AgentExecutionBudgetState skipCacheWrite?: boolean // API task_budget (output_config.task_budget, beta task-budgets-2026-03-13). // Distinct from the tokenBudget +500k auto-continue feature. `total` is the @@ -283,13 +306,20 @@ async function* queryLoop( systemPrompt, userContext, systemContext, - canUseTool, + canUseTool: baseCanUseTool, fallbackModel, querySource, maxTurns, skipCacheWrite, } = params const deps = params.deps ?? productionDeps() + const executionBudgetState = params.executionBudgetState + const canUseTool = executionBudgetState + ? createBudgetedCanUseTool(baseCanUseTool, executionBudgetState) + : baseCanUseTool + const budgetTimeoutMessage = executionBudgetState + ? `Explore reached its ${Math.round(executionBudgetState.config.hardTimeoutMs / 1000)}-second time budget. Returning the partial findings collected before the deadline.` + : 'Explore reached its time budget. Returning partial findings.' // Mutable cross-iteration state. The loop body destructures this at the top // of each iteration so reads stay bare-name (`messages`, `toolUseContext`). @@ -335,6 +365,54 @@ async function* queryLoop( // eslint-disable-next-line no-constant-condition while (true) { + if (executionBudgetState) { + refreshAgentBudgetDeadline(executionBudgetState) + if (maxTurns && executionBudgetState.apiCalls >= maxTurns) { + markAgentBudgetCompletion(executionBudgetState, 'max_turns') + yield createAttachmentMessage({ + type: 'max_turns_reached', + maxTurns, + turnCount: executionBudgetState.apiCalls, + }) + return { + reason: 'max_turns', + turnCount: executionBudgetState.apiCalls, + } + } + if ( + shouldFinalizeAgentBudget(executionBudgetState, maxTurns) + ) { + if ( + maxTurns && + executionBudgetState.apiCalls >= maxTurns - 1 && + executionBudgetState.completionReason === undefined + ) { + markAgentBudgetCompletion(executionBudgetState, 'max_turns') + } + executionBudgetState.finalizing = true + state = { + ...state, + messages: [ + ...state.messages, + createUserMessage({ + content: + 'The Explore time budget is nearly exhausted. Do not call tools. Summarize the useful findings and explicitly identify any uncertainty.', + isMeta: true, + }), + ], + toolUseContext: { + ...state.toolUseContext, + options: { + ...state.toolUseContext.options, + tools: [], + refreshTools: undefined, + }, + }, + pendingToolUseSummary: undefined, + } + } + } + // Destructure state at the top of each iteration. toolUseContext alone // is reassigned within an iteration (queryTracking, messages updates); // the rest are read-only between continue sites. @@ -745,6 +823,21 @@ async function* queryLoop( while (attemptWithFallback) { attemptWithFallback = false try { + if (executionBudgetState) { + if (maxTurns && executionBudgetState.apiCalls >= maxTurns) { + markAgentBudgetCompletion(executionBudgetState, 'max_turns') + yield createAttachmentMessage({ + type: 'max_turns_reached', + maxTurns, + turnCount: executionBudgetState.apiCalls, + }) + return { + reason: 'max_turns', + turnCount: executionBudgetState.apiCalls, + } + } + executionBudgetState.apiCalls++ + } let streamingFallbackOccured = false queryCheckpoint('query_api_streaming_start') for await (const message of deps.callModel({ @@ -1042,6 +1135,24 @@ async function* queryLoop( } } } catch (error) { + if (isAgentBudgetTimeout(toolUseContext.abortController.signal)) { + if (streamingToolExecutor) { + for await (const update of streamingToolExecutor.getRemainingResults()) { + if (update.message) yield update.message + } + } else { + yield* yieldMissingToolResultBlocks( + assistantMessages, + 'Explore time budget reached; tool execution was cancelled.', + toolResults, + ) + } + yield createAssistantMessage({ + content: budgetTimeoutMessage, + isVirtual: true, + }) + return { reason: 'budget_timeout' } + } logError(error) const errorMessage = error instanceof Error ? error.message : String(error) @@ -1116,6 +1227,13 @@ async function* queryLoop( 'Interrupted by user', ) } + if (isAgentBudgetTimeout(toolUseContext.abortController.signal)) { + yield createAssistantMessage({ + content: budgetTimeoutMessage, + isVirtual: true, + }) + return { reason: 'budget_timeout' } + } // chicago MCP: auto-unhide + lock release on interrupt. Same cleanup // as the natural turn-end path in stopHooks.ts. Main thread only — // see stopHooks.ts for the subagent-releasing-main's-lock rationale. @@ -1743,6 +1861,13 @@ async function* queryLoop( // We were aborted during tool calls if (toolUseContext.abortController.signal.aborted) { + if (isAgentBudgetTimeout(toolUseContext.abortController.signal)) { + yield createAssistantMessage({ + content: budgetTimeoutMessage, + isVirtual: true, + }) + return { reason: 'budget_timeout' } + } // chicago MCP: auto-unhide + lock release when aborted mid-tool-call. // This is the most likely Ctrl+C path for CU (e.g. slow screenshot). // Main thread only — see stopHooks.ts for the subagent rationale. @@ -1961,8 +2086,63 @@ async function* queryLoop( } } + if (executionBudgetState) { + refreshAgentBudgetDeadline(executionBudgetState) + const shouldReserveFinalTurn = + executionBudgetState.config.reserveFinalTurn && + maxTurns !== undefined && + executionBudgetState.apiCalls >= maxTurns - 1 + const shouldFinalize = shouldFinalizeAgentBudget( + executionBudgetState, + maxTurns, + ) + + if (shouldFinalize) { + if ( + shouldReserveFinalTurn && + executionBudgetState.completionReason === undefined + ) { + markAgentBudgetCompletion(executionBudgetState, 'max_turns') + } + executionBudgetState.finalizing = true + state = { + messages: [ + ...messagesForQuery, + ...assistantMessages, + ...toolResults, + createUserMessage({ + content: + 'The Explore execution budget has been reached. Do not call tools. Summarize the useful findings collected so far and explicitly identify any uncertainty.', + isMeta: true, + }), + ], + toolUseContext: { + ...toolUseContextWithQueryTracking, + options: { + ...toolUseContextWithQueryTracking.options, + tools: [], + refreshTools: undefined, + }, + }, + autoCompactTracking: tracking, + turnCount: nextTurnCount, + maxOutputTokensRecoveryCount: 0, + hasAttemptedReactiveCompact: false, + continuationNudgeCount: 0, + pendingToolUseSummary: undefined, + maxOutputTokensOverride: undefined, + stopHookActive, + transition: { reason: 'next_turn' }, + } + continue + } + } + // Check if we've reached the max turns limit if (maxTurns && nextTurnCount > maxTurns) { + if (executionBudgetState) { + markAgentBudgetCompletion(executionBudgetState, 'max_turns') + } yield createAttachmentMessage({ type: 'max_turns_reached', maxTurns, diff --git a/src/query/agentExecutionBudget.test.ts b/src/query/agentExecutionBudget.test.ts new file mode 100644 index 0000000000..b235b285a5 --- /dev/null +++ b/src/query/agentExecutionBudget.test.ts @@ -0,0 +1,96 @@ +import { expect, mock, test } from 'bun:test' +import type { CanUseToolFn } from '../hooks/useCanUseTool.js' +import { + AGENT_BUDGET_TIMEOUT_REASON, + createAgentExecutionBudgetState, + createBudgetedCanUseTool, + refreshAgentBudgetDeadline, + shouldFinalizeAgentBudget, + startAgentBudgetTimers, +} from './agentExecutionBudget.js' + +function callBudgeted(canUseTool: CanUseToolFn, toolUseID: string) { + return canUseTool( + {} as never, + {}, + {} as never, + {} as never, + toolUseID, + ) +} + +test('admits each tool ID once and rejects calls beyond the cap', async () => { + const base = mock(async (_tool, input) => ({ + behavior: 'allow' as const, + updatedInput: input, + })) as CanUseToolFn + const state = createAgentExecutionBudgetState({ + maxToolCalls: 2, + softTimeoutMs: 1_000, + hardTimeoutMs: 2_000, + reserveFinalTurn: true, + }) + const budgeted = createBudgetedCanUseTool(base, state) + + const first = await callBudgeted(budgeted, 'call-1') + const duplicate = await callBudgeted(budgeted, 'call-1') + const second = await callBudgeted(budgeted, 'call-2') + const overflow = await callBudgeted(budgeted, 'call-3') + + expect(first.behavior).toBe('allow') + expect(duplicate.behavior).toBe('allow') + expect(second.behavior).toBe('allow') + expect(overflow).toMatchObject({ + behavior: 'deny', + toolUseID: 'call-3', + }) + expect(state.toolCalls).toBe(2) + expect(state.completionReason).toBe('max_tool_calls') + expect(base).toHaveBeenCalledTimes(2) +}) + +test('soft and hard deadlines are distinguished and hard timeout aborts only its controller', async () => { + const state = createAgentExecutionBudgetState( + { + maxToolCalls: 40, + softTimeoutMs: 10, + hardTimeoutMs: 20, + reserveFinalTurn: true, + }, + 1_000, + ) + refreshAgentBudgetDeadline(state, 1_010) + expect(state.softDeadlineReached).toBe(true) + expect(state.hardDeadlineReached).toBe(false) + + const timedState = createAgentExecutionBudgetState({ + maxToolCalls: 40, + softTimeoutMs: 5, + hardTimeoutMs: 10, + reserveFinalTurn: true, + }) + const controller = new AbortController() + const stop = startAgentBudgetTimers(timedState, controller) + await Bun.sleep(25) + stop() + + expect(timedState.hardDeadlineReached).toBe(true) + expect(timedState.completionReason).toBe('timeout') + expect(controller.signal.reason).toBe(AGENT_BUDGET_TIMEOUT_REASON) +}) + +test('reserves the fourth API turn for a tool-free final response', () => { + const state = createAgentExecutionBudgetState({ + maxToolCalls: 40, + softTimeoutMs: 150_000, + hardTimeoutMs: 180_000, + reserveFinalTurn: true, + }) + + state.apiCalls = 2 + expect(shouldFinalizeAgentBudget(state, 4)).toBe(false) + state.apiCalls = 3 + expect(shouldFinalizeAgentBudget(state, 4)).toBe(true) + state.finalizing = true + expect(shouldFinalizeAgentBudget(state, 4)).toBe(false) +}) diff --git a/src/query/agentExecutionBudget.ts b/src/query/agentExecutionBudget.ts new file mode 100644 index 0000000000..d27350c2d3 --- /dev/null +++ b/src/query/agentExecutionBudget.ts @@ -0,0 +1,203 @@ +import type { CanUseToolFn } from '../hooks/useCanUseTool.js' +import type { PermissionDecision } from '../utils/permissions/PermissionResult.js' + +export const AGENT_BUDGET_TIMEOUT_REASON = 'agent_execution_budget_timeout' + +export type AgentCompletionReason = + | 'completed' + | 'max_turns' + | 'max_tool_calls' + | 'timeout' + +export type AgentExecutionBudgetConfig = { + maxToolCalls: number + softTimeoutMs: number + hardTimeoutMs: number + reserveFinalTurn: boolean +} + +export type AgentExecutionBudgetState = { + config: AgentExecutionBudgetConfig + startedAt: number + apiCalls: number + toolCalls: number + admittedToolUseIds: Set + decisions: Map> + completionReason?: Exclude + softDeadlineReached: boolean + hardDeadlineReached: boolean + finalizing: boolean +} + +export function createAgentExecutionBudgetState( + config: AgentExecutionBudgetConfig, + startedAt: number = Date.now(), +): AgentExecutionBudgetState { + return { + config, + startedAt, + apiCalls: 0, + toolCalls: 0, + admittedToolUseIds: new Set(), + decisions: new Map(), + softDeadlineReached: false, + hardDeadlineReached: false, + finalizing: false, + } +} + +export function markAgentBudgetCompletion( + state: AgentExecutionBudgetState, + reason: Exclude, +): void { + if (reason === 'timeout' || state.completionReason === undefined) { + state.completionReason = reason + } +} + +export function refreshAgentBudgetDeadline( + state: AgentExecutionBudgetState, + now: number = Date.now(), +): void { + const elapsed = now - state.startedAt + if (elapsed >= state.config.softTimeoutMs) { + state.softDeadlineReached = true + markAgentBudgetCompletion(state, 'timeout') + } + if (elapsed >= state.config.hardTimeoutMs) { + state.hardDeadlineReached = true + markAgentBudgetCompletion(state, 'timeout') + } +} + +export function startAgentBudgetTimers( + state: AgentExecutionBudgetState, + abortController: AbortController, +): () => void { + refreshAgentBudgetDeadline(state) + const elapsed = Date.now() - state.startedAt + const softTimer = state.softDeadlineReached + ? undefined + : setTimeout(() => { + state.softDeadlineReached = true + markAgentBudgetCompletion(state, 'timeout') + }, Math.max(0, state.config.softTimeoutMs - elapsed)) + const hardTimer = state.hardDeadlineReached + ? undefined + : setTimeout(() => { + state.hardDeadlineReached = true + markAgentBudgetCompletion(state, 'timeout') + abortController.abort(AGENT_BUDGET_TIMEOUT_REASON) + }, Math.max(0, state.config.hardTimeoutMs - elapsed)) + + if (state.hardDeadlineReached && !abortController.signal.aborted) { + abortController.abort(AGENT_BUDGET_TIMEOUT_REASON) + } + + return () => { + if (softTimer) clearTimeout(softTimer) + if (hardTimer) clearTimeout(hardTimer) + } +} + +export function isAgentBudgetTimeout(signal: AbortSignal): boolean { + return signal.aborted && signal.reason === AGENT_BUDGET_TIMEOUT_REASON +} + +/** + * Applies one admission policy to both streaming and non-streaming execution. + * The existing permission/tool pipeline still creates the terminal tool_result. + */ +export function createBudgetedCanUseTool( + canUseTool: CanUseToolFn, + state: AgentExecutionBudgetState, +): CanUseToolFn { + return async ( + tool, + input, + toolUseContext, + assistantMessage, + toolUseID, + forceDecision, + ) => { + const previous = state.decisions.get(toolUseID) + if (previous) return previous + + refreshAgentBudgetDeadline(state) + const decision = (async (): Promise => { + if (state.softDeadlineReached || state.hardDeadlineReached) { + markAgentBudgetCompletion(state, 'timeout') + return { + behavior: 'deny', + message: + 'Explore time budget reached. This tool call was not executed; summarize the findings collected so far.', + decisionReason: { + type: 'asyncAgent', + reason: 'execution_time_budget', + }, + toolUseID, + } + } + + if (state.toolCalls >= state.config.maxToolCalls) { + markAgentBudgetCompletion(state, 'max_tool_calls') + return { + behavior: 'deny', + message: + 'Explore tool budget reached. This tool call was not executed; summarize the findings collected so far.', + decisionReason: { + type: 'asyncAgent', + reason: 'execution_tool_budget', + }, + toolUseID, + } + } + + state.toolCalls++ + state.admittedToolUseIds.add(toolUseID) + return canUseTool( + tool, + input, + toolUseContext, + assistantMessage, + toolUseID, + forceDecision, + ) + })() + + state.decisions.set(toolUseID, decision) + return decision + } +} + +export function getAgentBudgetUsage(state: AgentExecutionBudgetState): { + apiCalls: number + toolCalls: number + elapsedMs: number + maxToolCalls: number + hardTimeoutMs: number +} { + return { + apiCalls: state.apiCalls, + toolCalls: state.toolCalls, + elapsedMs: Date.now() - state.startedAt, + maxToolCalls: state.config.maxToolCalls, + hardTimeoutMs: state.config.hardTimeoutMs, + } +} + +export function shouldFinalizeAgentBudget( + state: AgentExecutionBudgetState, + maxTurns: number | undefined, +): boolean { + if (state.finalizing) return false + const shouldReserveFinalTurn = + state.config.reserveFinalTurn && + maxTurns !== undefined && + state.apiCalls >= maxTurns - 1 + return ( + state.completionReason === 'max_tool_calls' || + state.softDeadlineReached || + shouldReserveFinalTurn + ) +} diff --git a/src/query/model.test.ts b/src/query/model.test.ts index bd86116ac2..fac061357a 100644 --- a/src/query/model.test.ts +++ b/src/query/model.test.ts @@ -1,5 +1,17 @@ -import { describe, expect, test } from 'bun:test' +import { afterEach, describe, expect, mock, test } from 'bun:test' +import axios from 'axios' import { resolveQueryTurnModel } from './model.js' +import { + clearVerbooModelsCache, + fetchVerbooModels, +} from '../services/api/verbooModels.js' + +const originalGet = axios.get + +afterEach(() => { + axios.get = originalGet + clearVerbooModelsCache() +}) describe('resolveQueryTurnModel', () => { test('prefers a skill model override to the persisted session model', () => { @@ -30,4 +42,31 @@ describe('resolveQueryTurnModel', () => { }), ).toBe('max/deepseek-v4-pro') }) + + test('resolves an inline skill profile and preserves its qualified catalog ID', async () => { + axios.get = mock(async () => ({ + data: { + data: [{ id: 'max/deepseek-v4-pro' }, { id: 'gpt-5.6-sol' }], + }, + })) as typeof axios.get + await fetchVerbooModels('token', { force: true }) + + expect( + resolveQueryTurnModel({ + permissionMode: 'default', + turnModel: 'profile:review', + sessionModel: 'gpt-5.6-sol', + }), + ).toBe('max/deepseek-v4-pro') + }) + + test('inherits the session model when an inline profile is unavailable', () => { + expect( + resolveQueryTurnModel({ + permissionMode: 'default', + turnModel: 'profile:review', + sessionModel: 'gpt-5.6-sol', + }), + ).toBe('gpt-5.6-sol') + }) }) diff --git a/src/query/model.ts b/src/query/model.ts index ef6b17d568..e8f3a5fe35 100644 --- a/src/query/model.ts +++ b/src/query/model.ts @@ -1,4 +1,9 @@ import type { PermissionMode } from '../utils/permissions/PermissionMode.js' +import { + parseAgentModelProfileReference, + resolveAgentProfileModel, +} from '../services/api/agentModelProfiles.js' +import { getCachedVerbooModels } from '../services/api/verbooModels.js' import { getDefaultMainLoopModelSetting, getRuntimeMainLoopModel, @@ -26,8 +31,12 @@ export function resolveQueryTurnModel({ sessionModel, exceeds200kTokens = false, }: ResolveQueryTurnModelParams): string { + const turnProfile = parseAgentModelProfileReference(turnModel) + const profileModel = turnProfile + ? resolveAgentProfileModel(turnProfile, getCachedVerbooModels() ?? []) + : null const requestedModel = - turnModel ?? + (turnProfile ? profileModel : turnModel) ?? parseUserSpecifiedModel( sessionModel ?? getDefaultMainLoopModelSetting(), ) diff --git a/src/services/api/agentModelProfiles.ts b/src/services/api/agentModelProfiles.ts new file mode 100644 index 0000000000..108bc3defa --- /dev/null +++ b/src/services/api/agentModelProfiles.ts @@ -0,0 +1,118 @@ +import type { VerbooModel } from './verbooModels.js' + +export const AGENT_MODEL_PROFILES = { + fast: [ + 'deepseek-v4-flash', + 'glm-4.7-flash', + 'mimo-v2.5', + 'qwen3.6-27b', + 'minimax-m3', + 'mimo-v2.5-pro', + 'deepseek-v4-pro', + 'kimi-k2.7', + 'glm-5.2', + ], + review: [ + 'deepseek-v4-pro', + 'mimo-v2.5-pro', + 'glm-5.2', + 'kimi-k2.7', + 'qwen3.6-27b', + 'minimax-m3', + 'mimo-v2.5', + 'deepseek-v4-flash', + 'glm-4.7-flash', + ], + coding: [ + 'mimo-v2.5-pro', + 'deepseek-v4-pro', + 'glm-5.2', + 'kimi-k2.7', + 'qwen3.6-27b', + 'minimax-m3', + 'mimo-v2.5', + 'deepseek-v4-flash', + 'glm-4.7-flash', + ], + testing: [ + 'qwen3.6-27b', + 'deepseek-v4-pro', + 'mimo-v2.5-pro', + 'mimo-v2.5', + 'minimax-m3', + 'deepseek-v4-flash', + 'glm-5.2', + 'kimi-k2.7', + 'glm-4.7-flash', + ], + balanced: [ + 'mimo-v2.5', + 'qwen3.6-27b', + 'minimax-m3', + 'deepseek-v4-flash', + 'glm-4.7-flash', + 'mimo-v2.5-pro', + 'deepseek-v4-pro', + 'kimi-k2.7', + 'glm-5.2', + ], +} as const + +export type AgentModelProfile = keyof typeof AGENT_MODEL_PROFILES + +function canonicalModelName(model: string): string { + const normalized = model.trim().toLowerCase().replace(/\[1m\]$/i, '') + return normalized.split('/').at(-1) ?? normalized +} + +export function findAvailableAgentModel( + requestedModel: string, + availableModels: readonly VerbooModel[], +): VerbooModel | undefined { + const exact = availableModels.find( + model => model.id.toLowerCase() === requestedModel.toLowerCase(), + ) + if (exact) return exact + + const canonicalRequested = canonicalModelName(requestedModel) + return availableModels.find( + model => canonicalModelName(model.id) === canonicalRequested, + ) +} + +function isAgentModelProfile(value: string): value is AgentModelProfile { + return Object.hasOwn(AGENT_MODEL_PROFILES, value) +} + +export function parseAgentModelProfileReference( + value: string | undefined, +): AgentModelProfile | null { + if (!value) return null + const normalized = value.trim().toLowerCase() + if (!normalized.startsWith('profile:')) return null + const candidate = normalized.slice('profile:'.length) + return isAgentModelProfile(candidate) ? candidate : null +} + +/** + * The router-provided role is authoritative when available. Static preference + * lists are a compatibility fallback for routers that do not advertise roles. + */ +export function resolveAgentProfileModel( + profile: AgentModelProfile, + availableModels: readonly VerbooModel[], + routerRoleModel?: string, +): string | null { + if (routerRoleModel) { + const entitledRoleModel = findAvailableAgentModel( + routerRoleModel, + availableModels, + ) + if (entitledRoleModel) return entitledRoleModel.id + } + + const selected = AGENT_MODEL_PROFILES[profile] + .map(candidate => findAvailableAgentModel(candidate, availableModels)) + .find((model): model is VerbooModel => model !== undefined) + return selected?.id ?? null +} diff --git a/src/services/api/agentRouting.test.ts b/src/services/api/agentRouting.test.ts index 1522b54548..d88e1ad37f 100644 --- a/src/services/api/agentRouting.test.ts +++ b/src/services/api/agentRouting.test.ts @@ -1,6 +1,22 @@ -import { describe, expect, test } from 'bun:test' -import { resolveAgentProvider } from './agentRouting.js' -import type { SettingsJson } from '../../utils/settings/types.js' +import { afterEach, describe, expect, mock, test } from 'bun:test' +import axios from 'axios' +import { + resolveAgentExecutionModel, + resolveAgentProvider, + resolveAgentRoute, +} from './agentRouting.js' +import { + clearVerbooModelsCache, + fetchVerbooModels, +} from './verbooModels.js' +import { + SettingsSchema, + type SettingsJson, +} from '../../utils/settings/types.js' +import type { VerbooModel } from './verbooModels.js' + +const models = (...ids: string[]): VerbooModel[] => + ids.map(id => ({ id, raw: { id } })) const baseSettings = { agentModels: { @@ -15,6 +31,19 @@ const baseSettings = { }, } as unknown as SettingsJson +const originalGet = axios.get +const originalSubagentModel = process.env.CLAUDE_CODE_SUBAGENT_MODEL + +afterEach(() => { + axios.get = originalGet + clearVerbooModelsCache() + if (originalSubagentModel === undefined) { + delete process.env.CLAUDE_CODE_SUBAGENT_MODEL + } else { + process.env.CLAUDE_CODE_SUBAGENT_MODEL = originalSubagentModel + } +}) + describe('resolveAgentProvider', () => { // ── Priority chain ────────────────────────────────────────── @@ -123,3 +152,253 @@ describe('resolveAgentProvider', () => { expect(result?.model).toBe('deepseek-chat') }) }) + +describe('resolveAgentExecutionModel', () => { + test('maps Explore haiku to the authenticated catalog role', async () => { + delete process.env.CLAUDE_CODE_SUBAGENT_MODEL + axios.get = mock(async () => ({ + data: { + data: [{ id: 'xiaomi/mimo-v2-flash' }, { id: 'gpt-5.6-sol' }], + agent_model_roles: { explore: 'xiaomi/mimo-v2-flash' }, + }, + })) as typeof axios.get + await fetchVerbooModels('token', { force: true }) + + expect( + resolveAgentExecutionModel({ + agentModel: 'haiku', + agentModelRole: 'explore', + parentModel: 'gpt-5.6-sol', + toolSpecifiedModel: 'haiku', + permissionMode: 'default', + agentType: 'Explore', + settings: null, + }), + ).toEqual({ + effectiveModel: 'xiaomi/mimo-v2-flash', + requestedModel: 'haiku', + source: 'catalog_role', + providerOverride: null, + catalogRole: 'explore', + }) + }) + + test('inherits the parent with an explicit reason when the role is missing', () => { + delete process.env.CLAUDE_CODE_SUBAGENT_MODEL + expect( + resolveAgentExecutionModel({ + agentModel: 'haiku', + agentModelRole: 'explore', + parentModel: 'gpt-5.6-sol', + permissionMode: 'default', + agentType: 'Explore', + settings: null, + }), + ).toEqual({ + effectiveModel: 'gpt-5.6-sol', + requestedModel: 'haiku', + source: 'parent_fallback', + providerOverride: null, + catalogRole: 'explore', + fallbackReason: 'missing_catalog_role', + }) + }) + + test('preserves legacy external provider routing precedence', () => { + delete process.env.CLAUDE_CODE_SUBAGENT_MODEL + const result = resolveAgentExecutionModel({ + agentModel: 'haiku', + agentModelRole: 'explore', + parentModel: 'gpt-5.6-sol', + permissionMode: 'default', + agentType: 'Explore', + settings: baseSettings, + }) + expect(result.source).toBe('external_route') + expect(result.effectiveModel).toBe('deepseek-chat') + expect(result.providerOverride?.baseURL).toBe('https://api.deepseek.com/v1') + }) + + test('does not catalog-remap agents that did not opt into a role', () => { + delete process.env.CLAUDE_CODE_SUBAGENT_MODEL + expect( + resolveAgentExecutionModel({ + agentModel: 'custom-worker-model', + parentModel: 'gpt-5.6-sol', + permissionMode: 'default', + agentType: 'claude-code-guide', + settings: null, + }), + ).toEqual({ + effectiveModel: 'custom-worker-model', + requestedModel: 'custom-worker-model', + source: 'agent_definition', + providerOverride: null, + }) + }) +}) + +describe('catalog-aware profiles from PR 18', () => { + test('prefers a router-advertised profile role over client compatibility candidates', async () => { + axios.get = mock(async () => ({ + data: { + data: [{ id: 'router/reviewer' }, { id: 'max/deepseek-v4-pro' }], + agent_model_roles: { review: 'router/reviewer' }, + }, + })) as typeof axios.get + const catalog = await fetchVerbooModels('token', { force: true }) + + expect( + resolveAgentRoute( + undefined, + 'worker-review', + { + agentRouting: { 'worker-review': { profile: 'review' } }, + } as SettingsJson, + catalog, + ), + ).toEqual({ + model: 'router/reviewer', + source: 'profile', + profile: 'review', + }) + }) + + test('preserves a plan-qualified canonical ID selected by a profile', () => { + const settings = { + agentRouting: { 'worker-review': { profile: 'review' } }, + } as SettingsJson + + expect( + resolveAgentRoute( + undefined, + 'worker-review', + settings, + models('max/qwen3.6-27b', 'max/deepseek-v4-pro'), + ), + ).toEqual({ + model: 'max/deepseek-v4-pro', + source: 'profile', + profile: 'review', + }) + }) + + test('matches an unqualified exact model to its authenticated canonical ID', () => { + const settings = { + agentRouting: { + 'worker-tests': { + model: 'qwen3.6-27b', + provider: 'inherit', + }, + }, + } as SettingsJson + + expect( + resolveAgentRoute( + undefined, + 'worker-tests', + settings, + models('max/qwen3.6-27b'), + ), + ).toEqual({ + model: 'max/qwen3.6-27b', + source: 'verboo-model', + }) + }) + + test('uses a fast catalog profile when the router has not advertised Explore role metadata yet', async () => { + delete process.env.CLAUDE_CODE_SUBAGENT_MODEL + axios.get = mock(async () => ({ + data: { + data: [{ id: 'max/deepseek-v4-flash' }, { id: 'gpt-5.6-sol' }], + }, + })) as typeof axios.get + await fetchVerbooModels('token', { force: true }) + + expect( + resolveAgentExecutionModel({ + agentModel: 'haiku', + agentModelRole: 'explore', + parentModel: 'gpt-5.6-sol', + permissionMode: 'default', + agentType: 'Explore', + settings: null, + }), + ).toEqual({ + effectiveModel: 'max/deepseek-v4-flash', + requestedModel: 'haiku', + source: 'catalog_profile', + providerOverride: null, + catalogRole: 'explore', + profile: 'fast', + }) + }) + + test('resolves profile references used by forked skills', async () => { + delete process.env.CLAUDE_CODE_SUBAGENT_MODEL + axios.get = mock(async () => ({ + data: { + data: [{ id: 'max/qwen3.6-27b' }, { id: 'gpt-5.6-sol' }], + }, + })) as typeof axios.get + await fetchVerbooModels('token', { force: true }) + + expect( + resolveAgentExecutionModel({ + agentModel: 'profile:testing', + parentModel: 'gpt-5.6-sol', + permissionMode: 'default', + agentType: 'worker-tests', + settings: null, + }), + ).toEqual({ + effectiveModel: 'max/qwen3.6-27b', + requestedModel: 'profile:testing', + source: 'catalog_profile', + providerOverride: null, + profile: 'testing', + }) + }) + + test('settings schema accepts profiles and inherited exact models', () => { + expect( + SettingsSchema().safeParse({ + agentRouting: { + Explore: 'fast', + review: { profile: 'review' }, + tests: { model: 'max/qwen3.6-27b', provider: 'inherit' }, + }, + }).success, + ).toBe(true) + + expect( + SettingsSchema().safeParse({ + agentRouting: { review: { profile: 'unknown' } }, + }).success, + ).toBe(false) + }) + + test('new profile routes inherit safely when the catalog has no candidate', () => { + delete process.env.CLAUDE_CODE_SUBAGENT_MODEL + expect( + resolveAgentExecutionModel({ + agentModel: 'haiku', + parentModel: 'gpt-5.6-sol', + permissionMode: 'default', + agentType: 'worker-review', + settings: { + agentRouting: { + 'worker-review': { profile: 'review' }, + }, + } as SettingsJson, + }), + ).toEqual({ + effectiveModel: 'gpt-5.6-sol', + requestedModel: 'profile:review', + source: 'parent_fallback', + providerOverride: null, + profile: 'review', + fallbackReason: 'missing_catalog_profile', + }) + }) +}) diff --git a/src/services/api/agentRouting.ts b/src/services/api/agentRouting.ts index 1afdf7959c..b5b696d799 100644 --- a/src/services/api/agentRouting.ts +++ b/src/services/api/agentRouting.ts @@ -1,4 +1,23 @@ import type { SettingsJson } from '../../utils/settings/types.js' +import { isVerbooMode } from '../../constants/oauth.js' +import type { PermissionMode } from '../../utils/permissions/PermissionMode.js' +import { + checkIsClaudeNativeProvider, + getAgentModel, +} from '../../utils/model/agent.js' +import { getRuntimeMainLoopModel } from '../../utils/model/model.js' +import { + getCachedVerbooModels, + getVerbooAgentModelForRole, + type VerbooModel, + type VerbooAgentModelRole, +} from './verbooModels.js' +import { + findAvailableAgentModel, + parseAgentModelProfileReference, + resolveAgentProfileModel, + type AgentModelProfile, +} from './agentModelProfiles.js' /** * Provider override resolved from agent routing config. @@ -13,6 +32,37 @@ export interface ProviderOverride { apiKey: string } +export type AgentModelResolutionSource = + | 'external_route' + | 'environment' + | 'tool_override' + | 'catalog_role' + | 'catalog_profile' + | 'catalog_model' + | 'agent_definition' + | 'parent_fallback' + +export type AgentModelResolution = { + effectiveModel: string + requestedModel?: string + source: AgentModelResolutionSource + providerOverride: ProviderOverride | null + catalogRole?: VerbooAgentModelRole + profile?: AgentModelProfile + fallbackReason?: + | 'missing_catalog_role' + | 'missing_catalog_profile' + | 'missing_catalog_model' + | 'unsupported_provider_alias' +} + +export interface ResolvedAgentRoute { + model: string + source: 'external-provider' | 'profile' | 'verboo-model' + profile?: AgentModelProfile + providerOverride?: ProviderOverride +} + /** * Normalize an agent identifier for case-insensitive, hyphen/underscore-agnostic matching. */ @@ -20,6 +70,112 @@ function normalize(key: string): string { return key.toLowerCase().replace(/[-_]/g, '') } +function resolveProfileModel( + profile: AgentModelProfile, + availableModels: readonly VerbooModel[], +): string | null { + return resolveAgentProfileModel( + profile, + availableModels, + getVerbooAgentModelForRole(profile), + ) +} + +function findRoutingValue( + name: string | undefined, + subagentType: string | undefined, + settings: SettingsJson | null, +): NonNullable[string] | undefined { + const routing = settings?.agentRouting + if (!routing) return undefined + + const normalizedRouting = new Map< + string, + NonNullable[string] + >() + for (const [key, value] of Object.entries(routing)) { + const normalizedKey = normalize(key) + if (normalizedRouting.has(normalizedKey)) { + console.error( + `[agentRouting] Warning: routing key "${key}" collides with an existing key after normalization (both map to "${normalizedKey}"). First entry wins.`, + ) + } + if (!normalizedRouting.has(normalizedKey)) { + normalizedRouting.set(normalizedKey, value) + } + } + + for (const candidate of [name, subagentType, 'default'].filter( + Boolean, + ) as string[]) { + const match = normalizedRouting.get(normalize(candidate)) + if (match !== undefined) return match + } + return undefined +} + +/** Resolve settings routing against external providers or the Verboo catalog. */ +export function resolveAgentRoute( + name: string | undefined, + subagentType: string | undefined, + settings: SettingsJson | null, + availableModels: readonly VerbooModel[] = getCachedVerbooModels() ?? [], +): ResolvedAgentRoute | null { + if (!settings) return null + const routingValue = findRoutingValue(name, subagentType, settings) + if (routingValue === undefined) return null + + if (typeof routingValue === 'string') { + const externalModel = settings.agentModels?.[routingValue] + if (externalModel) { + const providerOverride = { + model: routingValue, + baseURL: externalModel.base_url, + apiKey: externalModel.api_key, + } + return { + model: routingValue, + source: 'external-provider', + providerOverride, + } + } + + const profile = parseAgentModelProfileReference(`profile:${routingValue}`) + if (profile) { + const model = resolveProfileModel(profile, availableModels) + return model ? { model, source: 'profile', profile } : null + } + + const model = findAvailableAgentModel(routingValue, availableModels) + return model ? { model: model.id, source: 'verboo-model' } : null + } + + if ('model' in routingValue) { + const model = findAvailableAgentModel(routingValue.model, availableModels) + return model ? { model: model.id, source: 'verboo-model' } : null + } + + const model = resolveProfileModel( + routingValue.profile, + availableModels, + ) + if (model) { + return { + model, + source: 'profile', + profile: routingValue.profile, + } + } + if (routingValue.fallback === 'first-available' && availableModels[0]) { + return { + model: availableModels[0].id, + source: 'profile', + profile: routingValue.profile, + } + } + return null +} + /** * Look up agent.routing by name or subagent_type, then resolve via agent.models. * @@ -30,46 +186,228 @@ export function resolveAgentProvider( subagentType: string | undefined, settings: SettingsJson | null, ): ProviderOverride | null { - if (!settings) return null + return resolveAgentRoute(name, subagentType, settings)?.providerOverride ?? null +} + +const VERBOO_ALIAS_ROLES: Partial> = { + haiku: 'fast', + sonnet: 'balanced', + opus: 'powerful', +} - const routing = settings.agentRouting - const models = settings.agentModels - if (!routing || !models) return null +function resolveParentModel( + parentModel: string, + permissionMode: PermissionMode | undefined, +): string { + return getRuntimeMainLoopModel({ + permissionMode: permissionMode ?? 'default', + mainLoopModel: parentModel, + exceeds200kTokens: false, + }) +} - // Build normalized lookup from routing config. - // Warn on duplicate normalized keys (e.g. "explore-agent" and "explore_agent" - // both normalize to "exploreagent") to prevent silent shadowing. - const normalizedRouting = new Map() - for (const [key, value] of Object.entries(routing)) { - const nk = normalize(key) - if (normalizedRouting.has(nk)) { - console.error(`[agentRouting] Warning: routing key "${key}" collides with an existing key after normalization (both map to "${nk}"). First entry wins.`) +/** + * Resolves the complete agent route once so the prompt, API request, UI and + * analytics all observe the same effective model. + */ +export function resolveAgentExecutionModel({ + agentModel, + agentModelRole, + parentModel, + toolSpecifiedModel, + permissionMode, + agentName, + agentType, + settings, +}: { + agentModel: string | undefined + agentModelRole?: VerbooAgentModelRole + parentModel: string + toolSpecifiedModel?: string + permissionMode?: PermissionMode + agentName?: string + agentType?: string + settings: SettingsJson | null +}): AgentModelResolution { + const configuredRoutingValue = findRoutingValue( + agentName, + agentType, + settings, + ) + const configuredRoute = resolveAgentRoute(agentName, agentType, settings) + if (configuredRoute) { + return { + effectiveModel: configuredRoute.model, + requestedModel: configuredRoute.model, + source: + configuredRoute.source === 'external-provider' + ? 'external_route' + : configuredRoute.source === 'profile' + ? 'catalog_profile' + : 'catalog_model', + providerOverride: configuredRoute.providerOverride ?? null, + profile: configuredRoute.profile, } - if (!normalizedRouting.has(nk)) { - normalizedRouting.set(nk, value) + } + + const configuredProfile = + typeof configuredRoutingValue === 'string' + ? parseAgentModelProfileReference(`profile:${configuredRoutingValue}`) + : configuredRoutingValue && 'profile' in configuredRoutingValue + ? configuredRoutingValue.profile + : null + const configuredModel = + configuredRoutingValue && + typeof configuredRoutingValue !== 'string' && + 'model' in configuredRoutingValue + ? configuredRoutingValue.model + : undefined + if (configuredProfile || configuredModel) { + return { + effectiveModel: resolveParentModel(parentModel, permissionMode), + requestedModel: + configuredModel ?? + (configuredProfile ? `profile:${configuredProfile}` : undefined), + source: 'parent_fallback', + providerOverride: null, + ...(configuredProfile && { profile: configuredProfile }), + fallbackReason: configuredProfile + ? 'missing_catalog_profile' + : 'missing_catalog_model', } } - // Try name first, then subagentType, then "default" - const candidates = [name, subagentType, 'default'].filter(Boolean) as string[] - let modelName: string | undefined + const environmentModel = process.env.CLAUDE_CODE_SUBAGENT_MODEL?.trim() + const requestedModel = environmentModel || toolSpecifiedModel || agentModel + const requestedSource: AgentModelResolutionSource = environmentModel + ? 'environment' + : toolSpecifiedModel + ? 'tool_override' + : 'agent_definition' - for (const candidate of candidates) { - const match = normalizedRouting.get(normalize(candidate)) - if (match) { - modelName = match - break + if (isVerbooMode()) { + const normalizedRequested = requestedModel?.trim().toLowerCase() + const availableModels = getCachedVerbooModels() ?? [] + const requestedProfile = parseAgentModelProfileReference(requestedModel) + if (requestedProfile) { + const profileModel = resolveProfileModel( + requestedProfile, + availableModels, + ) + if (profileModel) { + return { + effectiveModel: profileModel, + requestedModel, + source: 'catalog_profile', + providerOverride: null, + profile: requestedProfile, + } + } + return { + effectiveModel: resolveParentModel(parentModel, permissionMode), + requestedModel, + source: 'parent_fallback', + providerOverride: null, + profile: requestedProfile, + fallbackReason: 'missing_catalog_profile', + } } - } - if (!modelName) return null + const catalogRole = agentModelRole + ? agentType?.toLowerCase() === 'explore' && + (!normalizedRequested || normalizedRequested === 'haiku') + ? agentModelRole + : normalizedRequested + ? VERBOO_ALIAS_ROLES[normalizedRequested] + : agentModelRole + : undefined + + if (catalogRole) { + const catalogModel = getVerbooAgentModelForRole(catalogRole) + if (catalogModel) { + return { + effectiveModel: catalogModel, + requestedModel, + source: 'catalog_role', + providerOverride: null, + catalogRole, + } + } + if (catalogRole === 'explore') { + const compatibleFastModel = resolveProfileModel( + 'fast', + availableModels, + ) + if (compatibleFastModel) { + return { + effectiveModel: compatibleFastModel, + requestedModel, + source: 'catalog_profile', + providerOverride: null, + catalogRole, + profile: 'fast', + } + } + } + return { + effectiveModel: resolveParentModel(parentModel, permissionMode), + requestedModel, + source: 'parent_fallback', + providerOverride: null, + catalogRole, + fallbackReason: 'missing_catalog_role', + } + } + + if ( + normalizedRequested && + normalizedRequested !== 'inherit' && + VERBOO_ALIAS_ROLES[normalizedRequested] === undefined + ) { + const catalogModel = findAvailableAgentModel( + requestedModel!, + availableModels, + ) + if (catalogModel) { + return { + effectiveModel: catalogModel.id, + requestedModel, + source: 'catalog_model', + providerOverride: null, + } + } + if (availableModels.length > 0) { + return { + effectiveModel: resolveParentModel(parentModel, permissionMode), + requestedModel, + source: 'parent_fallback', + providerOverride: null, + fallbackReason: 'missing_catalog_model', + } + } + } + } - const modelConfig = models[modelName] - if (!modelConfig) return null + const effectiveModel = getAgentModel( + agentModel, + parentModel, + toolSpecifiedModel, + permissionMode, + ) + const normalizedRequested = requestedModel?.trim().toLowerCase() + const unsupportedProviderAlias = + normalizedRequested !== undefined && + VERBOO_ALIAS_ROLES[normalizedRequested] !== undefined && + !checkIsClaudeNativeProvider() && + effectiveModel === resolveParentModel(parentModel, permissionMode) return { - model: modelName, - baseURL: modelConfig.base_url, - apiKey: modelConfig.api_key, + effectiveModel, + requestedModel, + source: unsupportedProviderAlias ? 'parent_fallback' : requestedSource, + providerOverride: null, + ...(unsupportedProviderAlias && { + fallbackReason: 'unsupported_provider_alias' as const, + }), } } diff --git a/src/services/api/verbooModels.test.ts b/src/services/api/verbooModels.test.ts index e0874a8d3c..bede359b95 100644 --- a/src/services/api/verbooModels.test.ts +++ b/src/services/api/verbooModels.test.ts @@ -4,6 +4,8 @@ import axios from 'axios' import { clearVerbooModelsCache, fetchVerbooModels, + getCachedVerbooAgentModelRoles, + getVerbooAgentModelForRole, getVerbooModelReasoning, getVerbooReasoningEffort, } from './verbooModels.js' @@ -89,6 +91,33 @@ test('does not advertise reasoning when the router response is incomplete', asyn expect(getVerbooModelReasoning('verboo/no-reasoning')).toBeUndefined() }) +test('accepts only entitled model IDs for agent catalog roles', async () => { + axios.get = mock(async () => ({ + data: { + data: [{ id: 'xiaomi/mimo-v2-flash' }, { id: 'gpt-5.6-sol' }], + agent_model_roles: { + explore: 'xiaomi/mimo-v2-flash', + balanced: 'gpt-5.6-sol', + review: 'gpt-5.6-sol', + powerful: 'not-entitled', + future_role: 'ignored', + fast: { model: 'future-contract-shape' }, + }, + }, + })) as typeof axios.get + + await fetchVerbooModels('access-token', { force: true }) + + expect(getCachedVerbooAgentModelRoles()).toEqual({ + explore: 'xiaomi/mimo-v2-flash', + balanced: 'gpt-5.6-sol', + review: 'gpt-5.6-sol', + }) + expect(getVerbooAgentModelForRole('explore')).toBe('xiaomi/mimo-v2-flash') + expect(getVerbooAgentModelForRole('powerful')).toBeUndefined() + expect(getVerbooAgentModelForRole('fast')).toBeUndefined() +}) + test('surfaces a model lookup failure instead of treating it as no models', async () => { axios.get = mock(async () => { throw new Error('timeout of 10000ms exceeded') diff --git a/src/services/api/verbooModels.ts b/src/services/api/verbooModels.ts index 0e9b763403..e3094afb6d 100644 --- a/src/services/api/verbooModels.ts +++ b/src/services/api/verbooModels.ts @@ -22,13 +22,35 @@ export type VerbooModelReasoning = { defaultEffort: string } +export const VERBOO_AGENT_MODEL_ROLES = [ + 'explore', + 'fast', + 'balanced', + 'powerful', + 'review', + 'coding', + 'testing', +] as const + +export type VerbooAgentModelRole = (typeof VERBOO_AGENT_MODEL_ROLES)[number] +export type VerbooAgentModelRoles = Partial< + Record +> + const modelsResponseSchema = z - .object({ data: z.array(z.record(z.unknown())) }) + .object({ + data: z.array(z.record(z.unknown())), + agent_model_roles: z.record(z.unknown()).optional(), + }) .passthrough() const CACHE_TTL_MS = 5 * 60 * 1000 -let cache: { fetchedAt: number; models: VerbooModel[] } | null = null +let cache: { + fetchedAt: number + models: VerbooModel[] + agentModelRoles: VerbooAgentModelRoles +} | null = null let inflight: Promise | null = null function pickNumber( @@ -114,6 +136,31 @@ function normalizeModel(raw: Record): VerbooModel | null { } } +function normalizeAgentModelRoles( + rawRoles: Record | undefined, + models: VerbooModel[], +): VerbooAgentModelRoles { + if (!rawRoles) return {} + + const entitledModelIds = new Set(models.map(model => model.id)) + const roles: VerbooAgentModelRoles = {} + for (const role of VERBOO_AGENT_MODEL_ROLES) { + const rawModelId = rawRoles[role] + const modelId = + typeof rawModelId === 'string' ? rawModelId.trim() : undefined + if (!modelId) continue + if (!entitledModelIds.has(modelId)) { + logForDebugging( + `[VerbooModels] Ignoring agent model role "${role}" because "${modelId}" is not in the authenticated model catalog`, + { level: 'warn' }, + ) + continue + } + roles[role] = modelId + } + return roles +} + export function clearVerbooModelsCache(): void { cache = null inflight = null @@ -152,7 +199,11 @@ export async function fetchVerbooModels( const models = data .map(normalizeModel) .filter((m): m is VerbooModel => m !== null) - cache = { fetchedAt: Date.now(), models } + const agentModelRoles = normalizeAgentModelRoles( + parsed.data.agent_model_roles, + models, + ) + cache = { fetchedAt: Date.now(), models, agentModelRoles } logForDebugging( `[VerbooModels] Fetched ${models.length} models from ${endpoint}`, ) @@ -185,6 +236,16 @@ export function getCachedVerbooModels(): VerbooModel[] | null { return cache?.models ?? null } +export function getCachedVerbooAgentModelRoles(): VerbooAgentModelRoles | null { + return cache?.agentModelRoles ?? null +} + +export function getVerbooAgentModelForRole( + role: VerbooAgentModelRole, +): string | undefined { + return cache?.agentModelRoles[role] +} + export function getVerbooModelMeta(modelId: string): VerbooModel | undefined { if (!cache) return undefined return cache.models.find((m) => m.id === modelId) diff --git a/src/services/tools/StreamingToolExecutor.toolPairing.test.ts b/src/services/tools/StreamingToolExecutor.toolPairing.test.ts index cdb60beda7..ac1f8d4110 100644 --- a/src/services/tools/StreamingToolExecutor.toolPairing.test.ts +++ b/src/services/tools/StreamingToolExecutor.toolPairing.test.ts @@ -9,14 +9,24 @@ import { } from '../../Tool.js' import type { AssistantMessage, Message } from '../../types/message.js' import { createAssistantMessage } from '../../utils/messages.js' +import { + createAgentExecutionBudgetState, + createBudgetedCanUseTool, +} from '../../query/agentExecutionBudget.js' import { StreamingToolExecutor } from './StreamingToolExecutor.js' +import { runTools } from './toolOrchestration.js' -function createTool(name: string, outcome: 'success' | 'error'): Tool { +function createTool( + name: string, + outcome: 'success' | 'error', + onCall: () => void = () => {}, +): Tool { return { name, inputSchema: z.object({}), maxResultSizeChars: 0, async call() { + onCall() if (outcome === 'error') { throw new Error(`${name} failed`) } @@ -148,3 +158,85 @@ test('parallel tools produce one terminal result each when one tool fails', asyn ]), ) }) + +function createBudgetedPermission(maxToolCalls: number) { + const state = createAgentExecutionBudgetState({ + maxToolCalls, + softTimeoutMs: 1_000, + hardTimeoutMs: 2_000, + reserveFinalTurn: true, + }) + const allow = (async (_tool, input) => ({ + behavior: 'allow', + updatedInput: input, + })) as CanUseToolFn + return { state, canUseTool: createBudgetedCanUseTool(allow, state) } +} + +test('streaming budget rejection keeps one terminal result per tool call', async () => { + let executed = 0 + const tool = createTool('BudgetedRead', 'success', () => executed++) + const context = createContext([tool]) + const { state, canUseTool } = createBudgetedPermission(1) + const executor = new StreamingToolExecutor([tool], canUseTool, context) + const blocks = ['call-1', 'call-2'].map( + id => + ({ type: 'tool_use', id, name: tool.name, input: {} }) as ToolUseBlock, + ) + const assistant = createAssistantMessage({ + content: blocks as never, + }) as AssistantMessage + + for (const block of blocks) executor.addTool(block, assistant) + + const messages: Message[] = [] + for await (const update of executor.getRemainingResults()) { + if (update.message) messages.push(update.message) + } + + const results = collectToolResults(messages) + expect(results).toHaveLength(2) + expect(results).toEqual( + expect.arrayContaining([ + { id: 'call-1', isError: false }, + { id: 'call-2', isError: true }, + ]), + ) + expect(executed).toBe(1) + expect(state.completionReason).toBe('max_tool_calls') +}) + +test('non-streaming budget rejection keeps one terminal result per tool call', async () => { + let executed = 0 + const tool = createTool('BudgetedRead', 'success', () => executed++) + const context = createContext([tool]) + const { state, canUseTool } = createBudgetedPermission(1) + const blocks = ['call-1', 'call-2'].map( + id => + ({ type: 'tool_use', id, name: tool.name, input: {} }) as ToolUseBlock, + ) + const assistant = createAssistantMessage({ + content: blocks as never, + }) as AssistantMessage + + const messages: Message[] = [] + for await (const update of runTools( + blocks, + [assistant], + canUseTool, + context, + )) { + if (update.message) messages.push(update.message) + } + + const results = collectToolResults(messages) + expect(results).toHaveLength(2) + expect(results).toEqual( + expect.arrayContaining([ + { id: 'call-1', isError: false }, + { id: 'call-2', isError: true }, + ]), + ) + expect(executed).toBe(1) + expect(state.completionReason).toBe('max_tool_calls') +}) diff --git a/src/tasks/LocalAgentTask/LocalAgentTask.tsx b/src/tasks/LocalAgentTask/LocalAgentTask.tsx index 7021e940b9..541e95fe3b 100644 --- a/src/tasks/LocalAgentTask/LocalAgentTask.tsx +++ b/src/tasks/LocalAgentTask/LocalAgentTask.tsx @@ -468,6 +468,7 @@ export function registerAsyncAgent({ description, prompt, selectedAgent, + model, setAppState, parentAbortController, toolUseId @@ -476,6 +477,7 @@ export function registerAsyncAgent({ description: string; prompt: string; selectedAgent: AgentDefinition; + model?: string; setAppState: SetAppState; parentAbortController?: AbortController; toolUseId?: string; @@ -492,6 +494,7 @@ export function registerAsyncAgent({ prompt, selectedAgent, agentType: selectedAgent.agentType ?? 'general-purpose', + model, abortController, retrieved: false, lastReportedToolCount: 0, @@ -528,6 +531,7 @@ export function registerAgentForeground({ description, prompt, selectedAgent, + model, setAppState, autoBackgroundMs, toolUseId @@ -536,6 +540,7 @@ export function registerAgentForeground({ description: string; prompt: string; selectedAgent: AgentDefinition; + model?: string; setAppState: SetAppState; autoBackgroundMs?: number; toolUseId?: string; @@ -558,6 +563,7 @@ export function registerAgentForeground({ prompt, selectedAgent, agentType: selectedAgent.agentType ?? 'general-purpose', + model, abortController, unregisterCleanup, retrieved: false, diff --git a/src/tools/AgentTool/AgentTool.tsx b/src/tools/AgentTool/AgentTool.tsx index 376bd91a0e..70ab0732d8 100644 --- a/src/tools/AgentTool/AgentTool.tsx +++ b/src/tools/AgentTool/AgentTool.tsx @@ -25,7 +25,9 @@ import { AbortError, errorMessage, toError } from '../../utils/errors.js'; import type { CacheSafeParams } from '../../utils/forkedAgent.js'; import { lazySchema } from '../../utils/lazySchema.js'; import { createUserMessage, extractTextContent, isSyntheticMessage, normalizeMessages } from '../../utils/messages.js'; -import { getAgentModel } from '../../utils/model/agent.js'; +import { resolveAgentExecutionModel } from '../../services/api/agentRouting.js'; +import { getInitialSettings } from '../../utils/settings/settings.js'; +import { createAgentExecutionBudgetState } from '../../query/agentExecutionBudget.js'; import { permissionModeSchema } from '../../utils/permissions/PermissionMode.js'; import type { PermissionResult } from '../../utils/permissions/PermissionResult.js'; import { filterDeniedAgents, getDenyRuleForAgent } from '../../utils/permissions/permissions.js'; @@ -415,11 +417,28 @@ export const AgentTool = buildTool({ setAgentColor(selectedAgent.agentType, selectedAgent.color); } - // Resolve agent params for logging (these are already resolved in runAgent) - const resolvedAgentModel = getAgentModel(selectedAgent.model, toolUseContext.options.mainLoopModel, isForkPath ? undefined : model, permissionMode); + // Resolve once so logging, prompts, task UI and the API use the same model. + const modelResolution = resolveAgentExecutionModel({ + agentModel: selectedAgent.model, + agentModelRole: selectedAgent.modelRole, + parentModel: toolUseContext.options.mainLoopModel, + toolSpecifiedModel: isForkPath ? undefined : model, + permissionMode, + agentName: name, + agentType: selectedAgent.agentType, + settings: getInitialSettings() + }); + const resolvedAgentModel = modelResolution.effectiveModel; logEvent('tengu_agent_tool_selected', { agent_type: selectedAgent.agentType as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS, model: resolvedAgentModel as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS, + model_source: modelResolution.source as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS, + ...(modelResolution.profile && { + model_profile: modelResolution.profile as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS + }), + ...(modelResolution.fallbackReason && { + fallback_reason: modelResolution.fallbackReason as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS + }), source: selectedAgent.source as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS, color: selectedAgent.color as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS, is_built_in_agent: isBuiltInAgent(selectedAgent), @@ -540,13 +559,15 @@ export const AgentTool = buildTool({ content: prompt })]; } + const executionBudgetState = selectedAgent.executionBudget ? createAgentExecutionBudgetState(selectedAgent.executionBudget, startTime) : undefined; const metadata = { prompt, resolvedAgentModel, isBuiltInAgent: isBuiltInAgent(selectedAgent), startTime, agentType: selectedAgent.agentType, - isAsync: (run_in_background === true || selectedAgent.background === true) && !isBackgroundTasksDisabled + isAsync: (run_in_background === true || selectedAgent.background === true) && !isBackgroundTasksDisabled, + executionBudgetState }; // Use inline env check instead of coordinatorModule to avoid circular @@ -647,6 +668,8 @@ export const AgentTool = buildTool({ worktreePath: worktreeInfo?.worktreePath, description, agentName: name, + modelResolution, + executionBudgetState, }; // Helper to wrap execution with a cwd override: explicit cwd arg (KAIROS) @@ -704,6 +727,7 @@ export const AgentTool = buildTool({ description, prompt, selectedAgent, + model: resolvedAgentModel, setAppState: rootSetAppState, // Don't link to parent's abort controller -- background agents should // survive when the user presses ESC to cancel the main thread. @@ -836,6 +860,7 @@ export const AgentTool = buildTool({ description, prompt, selectedAgent, + model: resolvedAgentModel, setAppState: rootSetAppState, toolUseId: toolUseContext.toolUseId, autoBackgroundMs: getAutoBackgroundMs() || undefined diff --git a/src/tools/AgentTool/agentToolUtils.budget.test.ts b/src/tools/AgentTool/agentToolUtils.budget.test.ts new file mode 100644 index 0000000000..afe3edea42 --- /dev/null +++ b/src/tools/AgentTool/agentToolUtils.budget.test.ts @@ -0,0 +1,52 @@ +import { expect, test } from 'bun:test' + +import { createAssistantMessage } from '../../utils/messages.js' +import { createAgentExecutionBudgetState } from '../../query/agentExecutionBudget.js' +import { finalizeAgentTool } from './agentToolUtils.js' + +test('returns prior findings together with the hard-timeout notice', () => { + const budget = createAgentExecutionBudgetState( + { + maxToolCalls: 40, + softTimeoutMs: 150_000, + hardTimeoutMs: 180_000, + reserveFinalTurn: true, + }, + Date.now() - 1_000, + ) + budget.apiCalls = 2 + budget.toolCalls = 3 + budget.completionReason = 'timeout' + + const result = finalizeAgentTool( + [ + createAssistantMessage({ content: 'Useful partial finding.' }), + createAssistantMessage({ + content: 'Explore reached its time budget.', + isVirtual: true, + }), + ], + 'agent-test', + { + prompt: 'Inspect the repository.', + resolvedAgentModel: 'gpt-5.6-sol', + isBuiltInAgent: true, + startTime: Date.now() - 1_000, + agentType: 'Explore', + isAsync: false, + executionBudgetState: budget, + }, + ) + + expect(result.content.map(block => block.text)).toEqual([ + 'Useful partial finding.', + 'Explore reached its time budget.', + ]) + expect(result.completionReason).toBe('timeout') + expect(result.budgetUsage).toMatchObject({ + apiCalls: 2, + toolCalls: 3, + maxToolCalls: 40, + hardTimeoutMs: 180_000, + }) +}) diff --git a/src/tools/AgentTool/agentToolUtils.ts b/src/tools/AgentTool/agentToolUtils.ts index a566ac3179..5f2ec7c456 100644 --- a/src/tools/AgentTool/agentToolUtils.ts +++ b/src/tools/AgentTool/agentToolUtils.ts @@ -43,6 +43,11 @@ import { isInProtectedNamespace } from '../../utils/envUtils.js' import { AbortError, errorMessage } from '../../utils/errors.js' import type { CacheSafeParams } from '../../utils/forkedAgent.js' import { lazySchema } from '../../utils/lazySchema.js' +import { + getAgentBudgetUsage, + type AgentCompletionReason, + type AgentExecutionBudgetState, +} from '../../query/agentExecutionBudget.js' import { extractTextContent, getLastAssistantMessage, @@ -254,6 +259,18 @@ export const agentToolResultSchema = lazySchema(() => }) .nullable(), }), + completionReason: z + .enum(['completed', 'max_turns', 'max_tool_calls', 'timeout']) + .optional(), + budgetUsage: z + .object({ + apiCalls: z.number(), + toolCalls: z.number(), + elapsedMs: z.number(), + maxToolCalls: z.number(), + hardTimeoutMs: z.number(), + }) + .optional(), }), ) @@ -283,6 +300,7 @@ export function finalizeAgentTool( startTime: number agentType: string isAsync: boolean + executionBudgetState?: AgentExecutionBudgetState }, ): AgentToolResult { const { @@ -292,6 +310,7 @@ export function finalizeAgentTool( startTime, agentType, isAsync, + executionBudgetState, } = metadata const lastAssistantMessage = getLastAssistantMessage(agentMessages) @@ -315,9 +334,22 @@ export function finalizeAgentTool( } } } + if (executionBudgetState?.completionReason && lastAssistantMessage.isVirtual) { + for (let i = agentMessages.length - 2; i >= 0; i--) { + const message = agentMessages[i]! + if (message.type !== 'assistant' || message.isVirtual) continue + const previousText = message.message.content.filter(_ => _.type === 'text') + if (previousText.length > 0) { + content = [...previousText, ...content] + break + } + } + } const totalTokens = getTokenCountFromUsage(lastAssistantMessage.message.usage) const totalToolUseCount = countToolUses(agentMessages) + const completionReason: AgentCompletionReason = + executionBudgetState?.completionReason ?? 'completed' logEvent('tengu_agent_tool_completed', { agent_type: @@ -332,6 +364,13 @@ export function finalizeAgentTool( total_tokens: totalTokens, is_built_in_agent: isBuiltInAgent, is_async: isAsync, + completion_reason: + completionReason as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS, + ...(executionBudgetState && { + budget_api_calls: executionBudgetState.apiCalls, + budget_tool_calls: executionBudgetState.toolCalls, + budget_elapsed_ms: Date.now() - executionBudgetState.startedAt, + }), }) // Signal to inference that this subagent's cache chain can be evicted. @@ -353,6 +392,10 @@ export function finalizeAgentTool( totalTokens, totalToolUseCount, usage: lastAssistantMessage.message.usage, + ...(executionBudgetState && { + completionReason, + budgetUsage: getAgentBudgetUsage(executionBudgetState), + }), } } diff --git a/src/tools/AgentTool/built-in/exploreAgent.ts b/src/tools/AgentTool/built-in/exploreAgent.ts index 021120a71e..9e771d3236 100644 --- a/src/tools/AgentTool/built-in/exploreAgent.ts +++ b/src/tools/AgentTool/built-in/exploreAgent.ts @@ -75,6 +75,15 @@ export const EXPLORE_AGENT: BuiltInAgentDefinition = { baseDir: 'built-in', // Use haiku for speed — explore is a fast read-only search agent model: 'haiku', + // Verboo resolves this role against the authenticated router catalog. + modelRole: 'explore', + maxTurns: 4, + executionBudget: { + maxToolCalls: 40, + softTimeoutMs: 150_000, + hardTimeoutMs: 180_000, + reserveFinalTurn: true, + }, // Explore is a fast read-only search agent — it doesn't need commit/PR/lint // rules from CLAUDE.md. The main agent has full context and interprets results. omitClaudeMd: true, diff --git a/src/tools/AgentTool/loadAgentsDir.ts b/src/tools/AgentTool/loadAgentsDir.ts index fdfa9f4bd3..aa5642edd0 100644 --- a/src/tools/AgentTool/loadAgentsDir.ts +++ b/src/tools/AgentTool/loadAgentsDir.ts @@ -13,6 +13,8 @@ import { McpServerConfigSchema, } from '../../services/mcp/types.js' import type { ToolUseContext } from '../../Tool.js' +import type { VerbooAgentModelRole } from '../../services/api/verbooModels.js' +import type { AgentExecutionBudgetConfig } from '../../query/agentExecutionBudget.js' import { logForDebugging } from '../../utils/debug.js' import { EFFORT_LEVELS, @@ -112,9 +114,13 @@ export type BaseAgentDefinition = { hooks?: HooksSettings // Session-scoped hooks registered when agent starts color?: AgentColorName model?: string + /** Provider catalog role used to resolve a model without hard-coding an ID. */ + modelRole?: VerbooAgentModelRole effort?: EffortValue permissionMode?: PermissionMode maxTurns?: number // Maximum number of agentic turns before stopping + /** Runtime-only budget for built-in agents. Custom agent files cannot set it. */ + executionBudget?: AgentExecutionBudgetConfig filename?: string // Original filename without .md extension (for user/project/managed agents) baseDir?: string criticalSystemReminder_EXPERIMENTAL?: string // Short message re-injected at every user turn diff --git a/src/tools/AgentTool/resumeAgent.ts b/src/tools/AgentTool/resumeAgent.ts index a688bca0c1..031034e982 100644 --- a/src/tools/AgentTool/resumeAgent.ts +++ b/src/tools/AgentTool/resumeAgent.ts @@ -16,7 +16,9 @@ import { filterUnresolvedToolUses, filterWhitespaceOnlyAssistantMessages, } from '../../utils/messages.js' -import { getAgentModel } from '../../utils/model/agent.js' +import { resolveAgentExecutionModel } from '../../services/api/agentRouting.js' +import { getInitialSettings } from '../../utils/settings/settings.js' +import { createAgentExecutionBudgetState } from '../../query/agentExecutionBudget.js' import { getQuerySourceForAgent } from '../../utils/promptCategory.js' import { getAgentTranscript, @@ -147,13 +149,18 @@ export async function resumeAgentBackground({ } } - // Resolve model for analytics metadata (runAgent resolves its own internally) - const resolvedAgentModel = getAgentModel( - selectedAgent.model, - toolUseContext.options.mainLoopModel, - undefined, + const modelResolution = resolveAgentExecutionModel({ + agentModel: selectedAgent.model, + agentModelRole: selectedAgent.modelRole, + parentModel: toolUseContext.options.mainLoopModel, permissionMode, - ) + agentType: selectedAgent.agentType, + settings: getInitialSettings(), + }) + const resolvedAgentModel = modelResolution.effectiveModel + const executionBudgetState = selectedAgent.executionBudget + ? createAgentExecutionBudgetState(selectedAgent.executionBudget, startTime) + : undefined const workerPermissionContext = { ...appState.toolPermissionContext, @@ -192,6 +199,8 @@ export async function resumeAgentBackground({ worktreePath: resumedWorktreePath, description: meta?.description, contentReplacementState: resumedReplacementState, + modelResolution, + executionBudgetState, } // Skip name-registry write — original entry persists from the initial spawn @@ -200,6 +209,7 @@ export async function resumeAgentBackground({ description: uiDescription, prompt, selectedAgent, + model: resolvedAgentModel, setAppState: rootSetAppState, toolUseId: toolUseContext.toolUseId, }) @@ -211,6 +221,7 @@ export async function resumeAgentBackground({ startTime, agentType: selectedAgent.agentType, isAsync: true, + executionBudgetState, } const asyncAgentContext = { diff --git a/src/tools/AgentTool/runAgent.ts b/src/tools/AgentTool/runAgent.ts index b249131a02..2ae7eb92f9 100644 --- a/src/tools/AgentTool/runAgent.ts +++ b/src/tools/AgentTool/runAgent.ts @@ -14,6 +14,10 @@ import { getSystemContext, getUserContext } from '../../context.js' import type { CanUseToolFn } from '../../hooks/useCanUseTool.js' import { query } from '../../query.js' import { getFeatureValue_CACHED_MAY_BE_STALE } from '../../services/analytics/growthbook.js' +import { + logEvent, + type AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS, +} from '../../services/analytics/index.js' import { getDumpPromptsPath } from '../../services/api/dumpPrompts.js' import { cleanupAgentTracking } from '../../services/api/promptCacheBreakDetection.js' import { @@ -55,11 +59,19 @@ import { import { registerFrontmatterHooks } from '../../utils/hooks/registerFrontmatterHooks.js' import { clearSessionHooks } from '../../utils/hooks/sessionHooks.js' import { executeSubagentStartHooks } from '../../utils/hooks.js' -import { createUserMessage } from '../../utils/messages.js' -import { getAgentModel } from '../../utils/model/agent.js' -import { resolveAgentProvider } from '../../services/api/agentRouting.js' +import { createSystemMessage, createUserMessage } from '../../utils/messages.js' +import { + resolveAgentExecutionModel, + type AgentModelResolution, +} from '../../services/api/agentRouting.js' import { getInitialSettings } from '../../utils/settings/settings.js' -import type { ModelAlias } from '../../utils/model/aliases.js' +import { createChildAbortController } from '../../utils/abortController.js' +import { + createAgentExecutionBudgetState, + isAgentBudgetTimeout, + startAgentBudgetTimers, + type AgentExecutionBudgetState, +} from '../../query/agentExecutionBudget.js' import { clearAgentTranscriptSubdir, recordSidechainTranscript, @@ -265,6 +277,8 @@ export async function* runAgent({ transcriptSubdir, onQueryProgress, agentName, + modelResolution, + executionBudgetState: providedExecutionBudgetState, }: { agentDefinition: AgentDefinition promptMessages: Message[] @@ -283,7 +297,7 @@ export async function* runAgent({ abortController?: AbortController agentId?: AgentId } - model?: ModelAlias + model?: string maxTurns?: number /** Preserve toolUseResult on messages for subagents with viewable transcripts */ preserveToolUseResults?: boolean @@ -326,6 +340,10 @@ export async function* runAgent({ onQueryProgress?: () => void /** Agent name (team member name) for routing resolution */ agentName?: string + /** Pre-resolved by AgentTool so prompt/logging/API use the exact same route. */ + modelResolution?: AgentModelResolution + /** Shared across foreground/background transitions to avoid resetting limits. */ + executionBudgetState?: AgentExecutionBudgetState }): AsyncGenerator { // Track subagent usage for feature discovery @@ -337,20 +355,43 @@ export async function* runAgent({ const rootSetAppState = toolUseContext.setAppStateForTasks ?? toolUseContext.setAppState - const resolvedAgentModel = getAgentModel( - agentDefinition.model, - toolUseContext.options.mainLoopModel, - model, - permissionMode, - ) - - // Resolve per-agent provider routing from settings - const providerOverride = resolveAgentProvider( - agentName, - agentDefinition.agentType, - getInitialSettings(), - ) - const effectiveModel = providerOverride ? providerOverride.model : resolvedAgentModel + const resolvedModel = + modelResolution ?? + resolveAgentExecutionModel({ + agentModel: agentDefinition.model, + agentModelRole: agentDefinition.modelRole, + parentModel: toolUseContext.options.mainLoopModel, + toolSpecifiedModel: model, + permissionMode, + agentName, + agentType: agentDefinition.agentType, + settings: getInitialSettings(), + }) + const resolvedAgentModel = resolvedModel.effectiveModel + const effectiveModel = resolvedModel.effectiveModel + const providerOverride = resolvedModel.providerOverride + + logEvent('tengu_agent_model_resolved', { + agent_type: + agentDefinition.agentType as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS, + model: + effectiveModel as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS, + source: + resolvedModel.source as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS, + ...(resolvedModel.catalogRole && { + catalog_role: + resolvedModel.catalogRole as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS, + }), + ...(resolvedModel.profile && { + model_profile: + resolvedModel.profile as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS, + }), + ...(resolvedModel.fallbackReason && { + fallback_reason: + resolvedModel.fallbackReason as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS, + }), + is_async: isAsync, + }) const agentId = override?.agentId ? override.agentId : createAgentId() @@ -524,11 +565,19 @@ export async function* runAgent({ // - Override takes precedence // - Async agents get a new unlinked controller (runs independently) // - Sync agents share parent's controller - const agentAbortController = override?.abortController + const baseAgentAbortController = override?.abortController ? override.abortController : isAsync ? new AbortController() : toolUseContext.abortController + const executionBudgetState = + providedExecutionBudgetState ?? + (agentDefinition.executionBudget + ? createAgentExecutionBudgetState(agentDefinition.executionBudget) + : undefined) + const agentAbortController = executionBudgetState + ? createChildAbortController(baseAgentAbortController) + : baseAgentAbortController // Execute SubagentStart hooks and collect additional context const additionalContexts: string[] = [] @@ -714,6 +763,7 @@ export async function* runAgent({ shareSetResponseLength: true, // Both sync and async contribute to response metrics criticalSystemReminder_EXPERIMENTAL: agentDefinition.criticalSystemReminder_EXPERIMENTAL, + requireCanUseTool: executionBudgetState !== undefined, contentReplacementState, }) @@ -747,8 +797,26 @@ export async function* runAgent({ // Track the last recorded message UUID for parent chain continuity let lastRecordedUuid: UUID | null = initialMessages.at(-1)?.uuid ?? null + const stopBudgetTimers = executionBudgetState + ? startAgentBudgetTimers(executionBudgetState, agentAbortController) + : undefined try { + if (resolvedModel.fallbackReason) { + const unavailableRoute = resolvedModel.profile + ? `${resolvedModel.profile} profile` + : resolvedModel.catalogRole + ? `${resolvedModel.catalogRole} role` + : `requested model (${resolvedModel.requestedModel ?? 'unknown'})` + const warning = + resolvedModel.fallbackReason === 'unsupported_provider_alias' + ? `The Claude model alias (${resolvedModel.requestedModel ?? 'unknown'}) is unavailable on the active provider. Using the parent model (${effectiveModel}).` + : `No eligible model was advertised for the ${unavailableRoute}. Using the parent model (${effectiveModel})${executionBudgetState ? ' with the agent execution limits enabled' : ''}.` + yield createSystemMessage( + warning, + 'warning', + ) + } for await (const message of query({ messages: initialMessages, systemPrompt: agentSystemPrompt, @@ -758,6 +826,7 @@ export async function* runAgent({ toolUseContext: agentToolUseContext, querySource, maxTurns: maxTurns ?? agentDefinition.maxTurns, + executionBudgetState, })) { onQueryProgress?.() // Forward subagent API request starts to parent's metrics display @@ -809,7 +878,10 @@ export async function* runAgent({ } } - if (agentAbortController.signal.aborted) { + if ( + agentAbortController.signal.aborted && + !isAgentBudgetTimeout(agentAbortController.signal) + ) { throw new AbortError() } @@ -818,6 +890,7 @@ export async function* runAgent({ agentDefinition.callback() } } finally { + stopBudgetTimers?.() // Clean up agent-specific MCP servers (runs on normal completion, abort, or error) await mcpCleanup() // Clean up agent's session hooks diff --git a/src/tools/SkillTool/SkillTool.ts b/src/tools/SkillTool/SkillTool.ts index 519e957ff4..3721cc54c9 100644 --- a/src/tools/SkillTool/SkillTool.ts +++ b/src/tools/SkillTool/SkillTool.ts @@ -55,7 +55,6 @@ import { import { parseFrontmatter } from '../../utils/frontmatterParser.js' import { lazySchema } from '../../utils/lazySchema.js' import { createUserMessage, normalizeMessages } from '../../utils/messages.js' -import type { ModelAlias } from '../../utils/model/aliases.js' import { resolveSkillModelOverride } from '../../utils/model/model.js' import { recordSkillUsage } from '../../utils/suggestions/skillUsageTracking.js' import { createAgentId } from '../../utils/uuid.js' @@ -230,7 +229,7 @@ async function executeForkedSkill( canUseTool, isAsync: false, querySource: 'agent:custom', - model: command.model as ModelAlias | undefined, + model: command.model, availableTools: context.options.tools, override: { agentId }, })) { diff --git a/src/utils/model/agent.test.ts b/src/utils/model/agent.test.ts index 9b979ab0cc..1375e16358 100644 --- a/src/utils/model/agent.test.ts +++ b/src/utils/model/agent.test.ts @@ -64,6 +64,33 @@ describe('getAgentModel provider-aware fallback', () => { }) describe('Non-Claude-native providers', () => { + test.each(['haiku', 'sonnet', 'opus'])( + 'tool-specified %s alias inherits the Sol parent', + async alias => { + mock.module('./providers.js', () => ({ + getAPIProvider: () => 'firstParty', + isFirstPartyAnthropicBaseUrl: () => false, + })) + + const { getAgentModel } = await import('./agent.js') + expect( + getAgentModel('inherit', 'gpt-5.6-sol', alias, 'plan'), + ).toBe('gpt-5.6-sol') + }, + ) + + test('agent-defined opus alias inherits the parent on a non-Claude provider', async () => { + mock.module('./providers.js', () => ({ + getAPIProvider: () => 'firstParty', + isFirstPartyAnthropicBaseUrl: () => false, + })) + + const { getAgentModel } = await import('./agent.js') + expect( + getAgentModel('opus', 'gpt-5.6-sol', undefined, 'plan'), + ).toBe('gpt-5.6-sol') + }) + test('haiku alias inherits parent model for OpenAI provider', async () => { mock.module('./providers.js', () => ({ getAPIProvider: () => 'openai', @@ -258,4 +285,4 @@ describe('getAgentModel provider-aware fallback', () => { expect(checkIsClaudeNativeProvider()).toBe(false) }) }) -}) \ No newline at end of file +}) diff --git a/src/utils/model/agent.ts b/src/utils/model/agent.ts index 904873e534..ea7685114c 100644 --- a/src/utils/model/agent.ts +++ b/src/utils/model/agent.ts @@ -1,6 +1,6 @@ import type { PermissionMode } from '../permissions/PermissionMode.js' import { capitalize } from '../stringUtils.js' -import { MODEL_ALIASES, type ModelAlias } from './aliases.js' +import { MODEL_ALIASES } from './aliases.js' import { applyBedrockRegionPrefix, getBedrockRegionPrefix } from './bedrock.js' import { getCanonicalName, @@ -37,7 +37,7 @@ export function getDefaultSubagentModel(): string { export function getAgentModel( agentModel: string | undefined, parentModel: string, - toolSpecifiedModel?: ModelAlias, + toolSpecifiedModel?: string, permissionMode?: PermissionMode, ): string { if (process.env.CLAUDE_CODE_SUBAGENT_MODEL) { @@ -71,6 +71,16 @@ export function getAgentModel( if (aliasMatchesParentTier(toolSpecifiedModel, parentModel)) { return parentModel } + if ( + isClaudeFamilyAlias(toolSpecifiedModel) && + !checkIsClaudeNativeProvider() + ) { + return getRuntimeMainLoopModel({ + permissionMode: permissionMode ?? 'default', + mainLoopModel: parentModel, + exceeds200kTokens: false, + }) + } const model = parseUserSpecifiedModel(toolSpecifiedModel) return applyParentRegionPrefix(model, toolSpecifiedModel) } @@ -82,11 +92,10 @@ export function getAgentModel( // have guaranteed haiku/sonnet model availability. Custom Anthropic-compatible // endpoints, OpenAI-shim, Gemini, Mistral, and other providers may not have // equivalent models, causing "model not found" errors when resolving aliases. - // For haiku/sonnet aliases on non-Claude-native providers, inherit parent model. - // Note: 'opus' is NOT included here because it's handled separately by - // aliasMatchesParentTier() which checks if parent's tier matches the alias. + // Claude family aliases cannot be resolved by non-Claude-native providers. + // Inherit the parent instead of sending an unavailable literal alias. if ( - (agentModelWithExp === 'haiku' || agentModelWithExp === 'sonnet') && + isClaudeFamilyAlias(agentModelWithExp) && !checkIsClaudeNativeProvider() ) { // Non-Claude-native provider → inherit parent model @@ -114,6 +123,15 @@ export function getAgentModel( return applyParentRegionPrefix(model, agentModelWithExp) } +function isClaudeFamilyAlias(model: string): boolean { + const normalized = model.trim().toLowerCase() + return ( + normalized === 'haiku' || + normalized === 'sonnet' || + normalized === 'opus' + ) +} + /** * Check if a bare family alias (opus/sonnet/haiku) matches the parent model's * tier. When it does, the subagent inherits the parent's exact model string diff --git a/src/utils/model/model.verboo.test.ts b/src/utils/model/model.verboo.test.ts index ed49b3f394..975d30f4e3 100644 --- a/src/utils/model/model.verboo.test.ts +++ b/src/utils/model/model.verboo.test.ts @@ -21,6 +21,7 @@ async function importFreshModelModule( })) mock.module('../../services/api/verbooModels.js', () => ({ getCachedVerbooModels: () => models, + getVerbooAgentModelForRole: () => undefined, getVerbooModelMeta: (modelId: string) => models.find(model => model.id === modelId), })) diff --git a/src/utils/processUserInput/processSlashCommand.tsx b/src/utils/processUserInput/processSlashCommand.tsx index e03c71b876..a325213cc3 100644 --- a/src/utils/processUserInput/processSlashCommand.tsx +++ b/src/utils/processUserInput/processSlashCommand.tsx @@ -32,7 +32,6 @@ import { registerSkillHooks } from '../hooks/registerSkillHooks.js'; import { logError } from '../log.js'; import { enqueuePendingNotification } from '../messageQueueManager.js'; import { createCommandInputMessage, createSyntheticUserCaveatMessage, createSystemMessage, createUserInterruptionMessage, createUserMessage, formatCommandInputTags, isCompactBoundaryMessage, isSystemLocalCommandMessage, normalizeMessages, prepareUserContent } from '../messages.js'; -import type { ModelAlias } from '../model/aliases.js'; import { parseToolListFromCLI } from '../permissions/permissionSetup.js'; import { hasPermissionsToUseTool } from '../permissions/permissions.js'; import { isOfficialMarketplaceName, parsePluginIdentifier } from '../plugins/pluginIdentifier.js'; @@ -157,7 +156,7 @@ async function executeForkedSlashCommand(command: CommandBase & PromptCommand, a canUseTool, isAsync: true, querySource: 'agent:custom', - model: command.model as ModelAlias | undefined, + model: command.model, availableTools: freshTools, override: { agentId @@ -236,7 +235,7 @@ async function executeForkedSlashCommand(command: CommandBase & PromptCommand, a canUseTool, isAsync: false, querySource: 'agent:custom', - model: command.model as ModelAlias | undefined, + model: command.model, availableTools: context.options.tools })) { agentMessages.push(message); diff --git a/src/utils/settings/types.ts b/src/utils/settings/types.ts index e739614a11..3df820bb26 100644 --- a/src/utils/settings/types.ts +++ b/src/utils/settings/types.ts @@ -741,12 +741,39 @@ spinnerVerbs: z 'Example: { "deepseek-chat": { "base_url": "https://api.deepseek.com/v1", "api_key": "sk-xxx" } }', ), agentRouting: z - .record(z.string(), z.string()) + .record( + z.string(), + z.union([ + // Backwards compatible: external provider name, profile name, or + // exact model advertised by the authenticated Verboo catalog. + z.string().trim().min(1), + z + .object({ + profile: z.enum([ + 'fast', + 'review', + 'coding', + 'testing', + 'balanced', + ]), + fallback: z + .enum(['inherit', 'first-available']) + .optional(), + }) + .strict(), + z + .object({ + model: z.string().trim().min(1), + provider: z.literal('inherit').optional(), + }) + .strict(), + ]), + ) .optional() .describe( - 'Map of agent identifier (subagent_type or team member name) to model name. ' + - 'Use "default" key as fallback. Model name must exist in agentModels. ' + - 'Example: { "Explore": "deepseek-chat", "general-purpose": "gpt-4o", "default": "gpt-4o" }', + 'Map of agent identifier to an external provider model, semantic profile, ' + + 'or authenticated Verboo model. Use "default" as fallback. ' + + 'Example: { "Explore": "fast", "worker-review": { "profile": "review" } }', ), fastMode: z .boolean()