From 849c595b93892aae90c32e671a0afb39ad58c858 Mon Sep 17 00:00:00 2001 From: Christopher Tso Date: Tue, 23 Jun 2026 08:54:19 +0200 Subject: [PATCH] feat(core): add experimental ai-sdk-agent provider --- apps/cli/tsup.config.ts | 2 + .../docs/docs/targets/coding-agents.mdx | 40 + .../docs/docs/targets/configuration.mdx | 1 + .../src/evaluation/providers/ai-sdk-agent.ts | 948 ++++++++++++++++++ .../core/src/evaluation/providers/index.ts | 3 + .../providers/normalize-tool-call.ts | 6 + .../core/src/evaluation/providers/targets.ts | 92 ++ .../core/src/evaluation/providers/types.ts | 7 +- .../validation/targets-validator.ts | 18 + packages/core/src/types/ai-sdk-agent.d.ts | 24 + .../evaluation/providers/ai-sdk-agent.test.ts | 244 +++++ .../test/evaluation/providers/targets.test.ts | 69 ++ packages/core/tsup.config.ts | 2 + 13 files changed, 1455 insertions(+), 1 deletion(-) create mode 100644 packages/core/src/evaluation/providers/ai-sdk-agent.ts create mode 100644 packages/core/src/types/ai-sdk-agent.d.ts create mode 100644 packages/core/test/evaluation/providers/ai-sdk-agent.test.ts diff --git a/apps/cli/tsup.config.ts b/apps/cli/tsup.config.ts index cf84e118f..9f231b351 100644 --- a/apps/cli/tsup.config.ts +++ b/apps/cli/tsup.config.ts @@ -21,6 +21,8 @@ export default defineConfig({ '@github/copilot-sdk', '@openai/codex-sdk', '@anthropic-ai/claude-agent-sdk', + 'ai', + '@ai-sdk/openai', ], // Provide a real require() for bundled CJS modules (e.g. debug) that need Node.js builtins like tty banner: { diff --git a/apps/web/src/content/docs/docs/targets/coding-agents.mdx b/apps/web/src/content/docs/docs/targets/coding-agents.mdx index 8f3064bd9..f08c4ad49 100644 --- a/apps/web/src/content/docs/docs/targets/coding-agents.mdx +++ b/apps/web/src/content/docs/docs/targets/coding-agents.mdx @@ -227,6 +227,46 @@ targets: grader_target: azure-base ``` +## AI SDK Agent + +`provider: ai-sdk-agent` is experimental. It uses Vercel AI SDK model and +tool-calling primitives with AgentV-owned coding tools and a bounded tool loop. +It is not a one-shot LLM target. + +```yaml +targets: + - name: ai-sdk-agent + provider: ai-sdk-agent + base_url: ${{ AGENTV_OPENAI_BASE_URL }} + api_key: ${{ AGENTV_OPENAI_API_KEY }} + model: ${{ AGENTV_CODEX_MODEL }} + max_steps: 20 + tools: read,bash,edit,write + grader_target: azure-base +``` + +| Field | Required | Description | +|-------|----------|-------------| +| `base_url` | No | OpenAI-compatible base URL. Defaults to `https://api.openai.com/v1`. | +| `api_key` | Yes | OpenAI-compatible API key. Prefer `${{ ENV_VAR }}` references. | +| `model` | Yes | Chat-completions model id. | +| `temperature` | No | Model temperature. | +| `max_steps` | No | Agent loop step limit. Defaults to `20`. | +| `tools` | No | Comma-separated allowlist from `read,bash,edit,write`. Defaults to all four. | +| `system_prompt` | No | Extra system instructions appended to AgentV's coding-agent prompt. | +| `cwd` | No | Working directory override. | +| `timeout_seconds` | No | Per-case model-call timeout. | +| `grader_target` | Yes | LLM target for evaluation. | + +The first slice includes minimal repo-owned tools: `read` and `write` operate on +UTF-8 files relative to the eval workspace, `edit` performs exact string +replacement, and `bash` runs shell commands in the workspace with timeout and +bounded stdout/stderr capture. + +AgentV loads `ai` and `@ai-sdk/openai` only when this provider is invoked. If +they are not installed locally, AgentV uses a managed dependency directory under +`~/.agentv/deps/ai-sdk-agent`. + ## VS Code ```yaml diff --git a/apps/web/src/content/docs/docs/targets/configuration.mdx b/apps/web/src/content/docs/docs/targets/configuration.mdx index b2a2e0317..dd41ed5ad 100644 --- a/apps/web/src/content/docs/docs/targets/configuration.mdx +++ b/apps/web/src/content/docs/docs/targets/configuration.mdx @@ -53,6 +53,7 @@ already-exported secrets into `.env`. | `gemini` | LLM | Google Gemini | | `claude` | Agent | Claude Agent SDK | | `codex` | Agent | Codex CLI | +| `ai-sdk-agent` | Agent | Experimental Vercel AI SDK coding-agent harness | | `pi-coding-agent` | Agent | Pi Coding Agent | | `vscode` | Agent | VS Code with Copilot | | `vscode-insiders` | Agent | VS Code Insiders | diff --git a/packages/core/src/evaluation/providers/ai-sdk-agent.ts b/packages/core/src/evaluation/providers/ai-sdk-agent.ts new file mode 100644 index 000000000..d7e33ee02 --- /dev/null +++ b/packages/core/src/evaluation/providers/ai-sdk-agent.ts @@ -0,0 +1,948 @@ +import { execFileSync, spawn } from 'node:child_process'; +import { randomUUID } from 'node:crypto'; +import { accessSync, existsSync, mkdirSync, writeFileSync } from 'node:fs'; +import { mkdir, open, readFile, stat, writeFile } from 'node:fs/promises'; +import path from 'node:path'; +import { pathToFileURL } from 'node:url'; + +import { getAgentvDataDir } from '../../paths.js'; +import type { JsonObject } from '../types.js'; +import { normalizeToolCall } from './normalize-tool-call.js'; +import { buildPromptDocument, normalizeInputFiles } from './preread.js'; +import type { AiSdkAgentResolvedConfig } from './targets.js'; +import type { + Message, + Provider, + ProviderRequest, + ProviderResponse, + ProviderStreamCallbacks, + ProviderTokenUsage, + ProviderTool, + ToolCall, +} from './types.js'; + +const DEFAULT_SYSTEM_PROMPT = [ + 'You are an experimental coding agent running inside an AgentV eval workspace.', + 'Use the provided tools to inspect and modify files. All file paths are relative to the workspace root.', + 'Prefer exact edits over full rewrites when changing existing files. Use bash only when it helps verify the work.', + 'When the task is complete, respond with a concise summary of the files changed and checks run.', +].join(' '); + +const DEFAULT_TOOLS = ['read', 'bash', 'edit', 'write'] as const; +const SUPPORTED_TOOLS = new Set(DEFAULT_TOOLS); +const DEFAULT_BASH_TIMEOUT_MS = 60_000; +const MAX_BASH_TIMEOUT_MS = 120_000; +const MAX_FILE_BYTES = 200_000; +const MAX_BASH_OUTPUT_BYTES = 200_000; +const MANAGED_DEPENDENCIES = ['ai@^6.0.0', '@ai-sdk/openai@^3.0.0'] as const; + +type AiSdkToolName = (typeof DEFAULT_TOOLS)[number]; + +interface AiSdkCoreModule { + generateText(options: Record): Promise; + stepCountIs(stepCount: number): unknown; + tool(definition: Record): unknown; + jsonSchema(schema: unknown): unknown; +} + +interface AiOpenAIProvider { + (modelId: string): unknown; + chat?: (modelId: string) => unknown; +} + +interface AiSdkOpenAIModule { + createOpenAI(options?: { + baseURL?: string; + apiKey?: string; + headers?: Record; + name?: string; + }): AiOpenAIProvider; +} + +interface AiSdkModules { + readonly ai: AiSdkCoreModule; + readonly openai: AiSdkOpenAIModule; +} + +interface AiGenerateTextResult { + readonly text?: string; + readonly content?: unknown; + readonly toolCalls?: readonly AiToolCall[]; + readonly toolResults?: readonly AiToolResult[]; + readonly finishReason?: string; + readonly rawFinishReason?: string; + readonly usage?: AiUsage; + readonly totalUsage?: AiUsage; + readonly warnings?: unknown; + readonly request?: unknown; + readonly response?: unknown; + readonly providerMetadata?: unknown; + readonly steps?: readonly AiStepResult[]; +} + +interface AiStepResult { + readonly text?: string; + readonly toolCalls?: readonly AiToolCall[]; + readonly toolResults?: readonly AiToolResult[]; + readonly usage?: AiUsage; + readonly finishReason?: string; + readonly rawFinishReason?: string; + readonly response?: unknown; +} + +interface AiToolCall { + readonly toolCallId?: string; + readonly toolName?: string; + readonly input?: unknown; +} + +interface AiToolResult { + readonly toolCallId?: string; + readonly toolName?: string; + readonly input?: unknown; + readonly output?: unknown; +} + +interface AiUsage { + readonly inputTokens?: number; + readonly outputTokens?: number; + readonly totalTokens?: number; + readonly reasoningTokens?: number; + readonly cachedInputTokens?: number; + readonly inputTokenDetails?: { + readonly cacheReadTokens?: number; + readonly cacheWriteTokens?: number; + }; + readonly outputTokenDetails?: { + readonly reasoningTokens?: number; + }; + readonly raw?: unknown; +} + +interface BashResult { + readonly command: string; + readonly exit_code: number | null; + readonly stdout: string; + readonly stderr: string; + readonly duration_ms: number; + readonly timed_out: boolean; + readonly truncated: boolean; +} + +let sdkModules: AiSdkModules | null = null; +let loadingPromise: Promise | null = null; +let loaderOverride: (() => Promise) | undefined; + +function findManagedSdkInstallRoot(): string { + return path.join(getAgentvDataDir(), 'deps', 'ai-sdk-agent'); +} + +function findAccessiblePath(paths: readonly string[]): string | undefined { + for (const candidate of paths) { + try { + accessSync(candidate); + return candidate; + } catch {} + } + return undefined; +} + +function coerceModules(aiModule: unknown, openaiModule: unknown): AiSdkModules | undefined { + const ai = aiModule as Partial; + const openai = openaiModule as Partial; + if ( + typeof ai.generateText !== 'function' || + typeof ai.stepCountIs !== 'function' || + typeof ai.tool !== 'function' || + typeof ai.jsonSchema !== 'function' || + typeof openai.createOpenAI !== 'function' + ) { + return undefined; + } + return { + ai: ai as AiSdkCoreModule, + openai: openai as AiSdkOpenAIModule, + }; +} + +async function tryImportLocalSdkModules(): Promise { + try { + const [aiModule, openaiModule] = await Promise.all([import('ai'), import('@ai-sdk/openai')]); + const modules = coerceModules(aiModule, openaiModule); + if (!modules) return false; + sdkModules = modules; + return true; + } catch { + return false; + } +} + +async function tryImportManagedSdkModules(): Promise { + const managedRoot = findManagedSdkInstallRoot(); + const aiEntry = findAccessiblePath([ + path.join(managedRoot, 'node_modules', 'ai', 'dist', 'index.mjs'), + path.join(managedRoot, 'node_modules', 'ai', 'dist', 'index.js'), + ]); + const openaiEntry = findAccessiblePath([ + path.join(managedRoot, 'node_modules', '@ai-sdk', 'openai', 'dist', 'index.mjs'), + path.join(managedRoot, 'node_modules', '@ai-sdk', 'openai', 'dist', 'index.js'), + ]); + if (!aiEntry || !openaiEntry) return false; + + try { + const [aiModule, openaiModule] = await Promise.all([ + import(pathToFileURL(aiEntry).href), + import(pathToFileURL(openaiEntry).href), + ]); + const modules = coerceModules(aiModule, openaiModule); + if (!modules) return false; + sdkModules = modules; + return true; + } catch { + return false; + } +} + +function installManagedSdkModules(installDir: string): void { + mkdirSync(installDir, { recursive: true }); + const packageJsonPath = path.join(installDir, 'package.json'); + if (!existsSync(packageJsonPath)) { + writeFileSync( + packageJsonPath, + `${JSON.stringify({ private: true, type: 'module' }, null, 2)}\n`, + 'utf8', + ); + } + + const bunExecutable = process.versions.bun ? process.execPath : 'bun'; + console.error( + `Installing ai-sdk-agent dependencies into ${installDir} with bun: ${MANAGED_DEPENDENCIES.join( + ' ', + )}`, + ); + execFileSync(bunExecutable, ['add', ...MANAGED_DEPENDENCIES], { + cwd: installDir, + stdio: 'inherit', + }); +} + +function formatDependencyLoadError(error: unknown): string { + const managedRoot = findManagedSdkInstallRoot(); + const message = error instanceof Error ? error.message : String(error); + return [ + 'ai-sdk-agent requires the Vercel AI SDK packages `ai` and `@ai-sdk/openai`.', + `AgentV tried local imports first, then the managed install directory: ${managedRoot}`, + 'Repair with:', + ` mkdir -p ${managedRoot}`, + ` cd ${managedRoot}`, + ` bun add ${MANAGED_DEPENDENCIES.join(' ')}`, + `Original error: ${message}`, + ].join('\n'); +} + +async function doLoadSdkModules(): Promise { + if (loaderOverride) { + sdkModules = await loaderOverride(); + return; + } + if ((await tryImportLocalSdkModules()) || (await tryImportManagedSdkModules())) { + return; + } + + const installDir = findManagedSdkInstallRoot(); + try { + installManagedSdkModules(installDir); + if (await tryImportManagedSdkModules()) { + return; + } + throw new Error('managed install completed but modules still could not be imported'); + } catch (error) { + throw new Error(formatDependencyLoadError(error)); + } +} + +async function loadSdkModules(): Promise { + if (!sdkModules) { + if (!loadingPromise) { + loadingPromise = doLoadSdkModules().catch((error) => { + loadingPromise = null; + throw error; + }); + } + await loadingPromise; + } + return sdkModules as AiSdkModules; +} + +export class AiSdkAgentProvider implements Provider { + readonly id: string; + readonly kind = 'ai-sdk-agent' as const; + readonly targetName: string; + readonly supportsBatch = false; + + private readonly config: AiSdkAgentResolvedConfig; + private readonly toolAllowlist: readonly AiSdkToolName[]; + + constructor(targetName: string, config: AiSdkAgentResolvedConfig) { + this.id = `ai-sdk-agent:${targetName}`; + this.targetName = targetName; + this.config = config; + this.toolAllowlist = parseToolAllowlist(config.tools); + } + + async invoke(request: ProviderRequest): Promise { + if (request.signal?.aborted) { + throw new Error('ai-sdk-agent request was aborted before execution'); + } + + const sdk = await loadSdkModules(); + const cwd = this.resolveCwd(request.cwd); + const startTime = new Date().toISOString(); + const startMs = Date.now(); + + const toolCalls: ToolCall[] = []; + const providerTools = createCodingTools(cwd, this.toolAllowlist); + const tools = toAiSdkToolSet(sdk, providerTools, { + onToolCall: (toolCall) => { + toolCalls.push(toolCall); + }, + callbacks: request.streamCallbacks, + }); + + const openai = sdk.openai.createOpenAI({ + baseURL: this.config.baseURL, + apiKey: this.config.apiKey, + name: 'agentv-ai-sdk-agent', + }); + const model = + typeof openai.chat === 'function' + ? openai.chat(this.config.model) + : openai(this.config.model); + + const inputFiles = normalizeInputFiles(request.inputFiles); + const prompt = buildPromptDocument(request, inputFiles); + const system = buildSystemPrompt(this.config.systemPrompt, request.systemPrompt); + const temperature = request.temperature ?? this.config.temperature; + const maxOutputTokens = request.maxOutputTokens; + + const result = await sdk.ai.generateText({ + model, + system, + prompt, + tools, + stopWhen: sdk.ai.stepCountIs(this.config.maxSteps), + ...(temperature !== undefined ? { temperature } : {}), + ...(maxOutputTokens !== undefined ? { maxOutputTokens } : {}), + ...(this.config.timeoutMs !== undefined ? { timeout: this.config.timeoutMs } : {}), + ...(request.signal ? { abortSignal: request.signal } : {}), + }); + + const endTime = new Date().toISOString(); + const durationMs = Date.now() - startMs; + + return mapAiSdkResponse(result, { + model: this.config.model, + baseURL: this.config.baseURL, + maxSteps: this.config.maxSteps, + toolAllowlist: this.toolAllowlist, + observedToolCalls: toolCalls, + durationMs, + startTime, + endTime, + }); + } + + private resolveCwd(cwdOverride?: string): string { + if (cwdOverride) { + return path.resolve(cwdOverride); + } + if (this.config.cwd) { + return path.resolve(this.config.cwd); + } + return process.cwd(); + } +} + +function buildSystemPrompt(configSystemPrompt?: string, requestSystemPrompt?: string): string { + return [DEFAULT_SYSTEM_PROMPT, configSystemPrompt, requestSystemPrompt] + .filter((part): part is string => typeof part === 'string' && part.trim().length > 0) + .map((part) => part.trim()) + .join('\n\n'); +} + +function parseToolAllowlist(value: string | undefined): readonly AiSdkToolName[] { + if (!value || value.trim().length === 0) { + return DEFAULT_TOOLS; + } + + const selected: AiSdkToolName[] = []; + for (const raw of value.split(',')) { + const name = raw.trim().toLowerCase(); + if (name.length === 0) continue; + if (!SUPPORTED_TOOLS.has(name)) { + throw new Error( + `ai-sdk-agent tools includes unsupported tool '${name}'. Supported tools: ${DEFAULT_TOOLS.join( + ',', + )}`, + ); + } + if (!selected.includes(name as AiSdkToolName)) { + selected.push(name as AiSdkToolName); + } + } + + if (selected.length === 0) { + throw new Error('ai-sdk-agent tools allowlist cannot be empty'); + } + return selected; +} + +function toAiSdkToolSet( + sdk: AiSdkModules, + providerTools: readonly ProviderTool[], + options: { + readonly onToolCall: (toolCall: ToolCall) => void; + readonly callbacks?: ProviderStreamCallbacks; + }, +): Record { + const toolSet: Record = {}; + for (const providerTool of providerTools) { + toolSet[providerTool.name] = sdk.ai.tool({ + description: providerTool.description, + inputSchema: sdk.ai.jsonSchema(providerTool.parameters), + execute: async (input: unknown, executionOptions?: { toolCallId?: string }) => { + const toolCallId = executionOptions?.toolCallId ?? randomUUID(); + const startTime = new Date().toISOString(); + const startMs = Date.now(); + options.callbacks?.onToolCallStart?.(providerTool.name, toolCallId); + + let output: unknown; + let status: ToolCall['status'] = 'ok'; + try { + output = await providerTool.execute(input); + if (isToolErrorResult(output)) { + status = output.status ?? 'error'; + } + } catch (error) { + output = { error: error instanceof Error ? error.message : String(error) }; + status = 'error'; + } + + const endMs = Date.now(); + const toolCall = normalizeToolCall('ai-sdk-agent', { + tool: providerTool.name, + input, + output, + id: toolCallId, + startTime, + endTime: new Date().toISOString(), + durationMs: endMs - startMs, + status, + }); + options.onToolCall(toolCall); + options.callbacks?.onToolCallEnd?.( + providerTool.name, + input, + output, + toolCall.durationMs ?? 0, + toolCallId, + ); + return output; + }, + }); + } + return toolSet; +} + +function isToolErrorResult( + value: unknown, +): value is { error: unknown; status?: ToolCall['status'] } { + return typeof value === 'object' && value !== null && 'error' in value; +} + +function createCodingTools( + workspacePath: string, + allowlist: readonly AiSdkToolName[] = DEFAULT_TOOLS, +): ProviderTool[] { + const tools: Record = { + read: { + name: 'read', + description: + 'Read a UTF-8 file relative to the workspace. Returns content, byte size, and whether output was truncated.', + parameters: { + type: 'object', + properties: { + path: { type: 'string', description: 'File path relative to the workspace root.' }, + file_path: { type: 'string', description: 'Alias for path.' }, + }, + anyOf: [{ required: ['path'] }, { required: ['file_path'] }], + }, + execute: async (input) => { + const relPath = requiredPath(input); + const resolved = resolveWorkspacePath(workspacePath, relPath); + const fileStat = await stat(resolved); + if (fileStat.isDirectory()) { + return { error: `'${relPath}' is a directory, not a file`, status: 'error' }; + } + const fd = await open(resolved, 'r'); + try { + const bytesToRead = Math.min(fileStat.size, MAX_FILE_BYTES); + const buffer = Buffer.alloc(bytesToRead); + await fd.read(buffer, 0, bytesToRead, 0); + return { + path: relPath, + content: buffer.toString('utf8'), + bytes: fileStat.size, + truncated: fileStat.size > MAX_FILE_BYTES, + }; + } finally { + await fd.close(); + } + }, + }, + write: { + name: 'write', + description: 'Write UTF-8 content to a file relative to the workspace, creating parent dirs.', + parameters: { + type: 'object', + properties: { + path: { type: 'string', description: 'File path relative to the workspace root.' }, + file_path: { type: 'string', description: 'Alias for path.' }, + content: { type: 'string', description: 'Full file contents to write.' }, + }, + required: ['content'], + anyOf: [{ required: ['path'] }, { required: ['file_path'] }], + }, + execute: async (input) => { + const relPath = requiredPath(input); + const content = requiredString(input, 'content'); + const resolved = resolveWorkspacePath(workspacePath, relPath); + await mkdir(path.dirname(resolved), { recursive: true }); + await writeFile(resolved, content, 'utf8'); + return { + path: relPath, + bytes_written: Buffer.byteLength(content, 'utf8'), + }; + }, + }, + edit: { + name: 'edit', + description: + 'Replace an exact string in a UTF-8 file relative to the workspace. By default the old string must appear exactly once; set replace_all true to replace every occurrence.', + parameters: { + type: 'object', + properties: { + path: { type: 'string', description: 'File path relative to the workspace root.' }, + file_path: { type: 'string', description: 'Alias for path.' }, + old_string: { type: 'string', description: 'Exact text to replace.' }, + new_string: { type: 'string', description: 'Replacement text.' }, + replace_all: { + type: 'boolean', + description: 'Replace all occurrences instead of requiring exactly one.', + }, + }, + required: ['old_string', 'new_string'], + anyOf: [{ required: ['path'] }, { required: ['file_path'] }], + }, + execute: async (input) => { + const relPath = requiredPath(input); + const oldString = requiredString(input, 'old_string'); + const newString = requiredString(input, 'new_string'); + const replaceAll = optionalBoolean(input, 'replace_all') ?? false; + if (oldString.length === 0) { + return { error: 'old_string must not be empty', status: 'error' }; + } + + const resolved = resolveWorkspacePath(workspacePath, relPath); + const original = await readFile(resolved, 'utf8'); + const occurrences = countOccurrences(original, oldString); + if (occurrences === 0) { + return { error: 'old_string was not found', status: 'error' }; + } + if (!replaceAll && occurrences !== 1) { + return { + error: `old_string matched ${occurrences} times; provide a more specific string or set replace_all true`, + status: 'error', + }; + } + + const updated = replaceAll + ? original.split(oldString).join(newString) + : original.replace(oldString, newString); + await writeFile(resolved, updated, 'utf8'); + return { path: relPath, replacements: replaceAll ? occurrences : 1 }; + }, + }, + bash: { + name: 'bash', + description: + 'Run a shell command in the workspace with timeout and bounded stdout/stderr capture.', + parameters: { + type: 'object', + properties: { + command: { type: 'string', description: 'Shell command to run in the workspace.' }, + timeout_seconds: { + type: 'number', + description: `Optional timeout, capped at ${MAX_BASH_TIMEOUT_MS / 1000} seconds.`, + }, + }, + required: ['command'], + }, + execute: async (input) => { + const command = requiredString(input, 'command'); + const timeoutSeconds = optionalNumber(input, 'timeout_seconds'); + const timeoutMs = + timeoutSeconds === undefined + ? DEFAULT_BASH_TIMEOUT_MS + : Math.min(Math.max(Math.floor(timeoutSeconds * 1000), 1), MAX_BASH_TIMEOUT_MS); + return runBashCommand(command, { cwd: workspacePath, timeoutMs }); + }, + }, + }; + + return allowlist.map((name) => tools[name]); +} + +function requiredPath(input: unknown): string { + const args = asRecord(input); + const value = args.path ?? args.file_path ?? args.filePath; + if (typeof value !== 'string' || value.trim().length === 0) { + throw new Error('path is required'); + } + return value.trim(); +} + +function requiredString(input: unknown, key: string): string { + const value = asRecord(input)[key]; + if (typeof value !== 'string') { + throw new Error(`${key} is required`); + } + return value; +} + +function optionalNumber(input: unknown, key: string): number | undefined { + const value = asRecord(input)[key]; + if (value === undefined || value === null || value === '') return undefined; + const numeric = typeof value === 'number' ? value : Number(value); + if (!Number.isFinite(numeric)) { + throw new Error(`${key} must be a number`); + } + return numeric; +} + +function optionalBoolean(input: unknown, key: string): boolean | undefined { + const value = asRecord(input)[key]; + if (value === undefined || value === null || value === '') return undefined; + if (typeof value === 'boolean') return value; + if (typeof value === 'string') { + const normalized = value.trim().toLowerCase(); + if (normalized === 'true') return true; + if (normalized === 'false') return false; + } + throw new Error(`${key} must be a boolean`); +} + +function asRecord(input: unknown): Record { + return typeof input === 'object' && input !== null ? (input as Record) : {}; +} + +function resolveWorkspacePath(workspacePath: string, relativePath: string): string { + if (path.isAbsolute(relativePath)) { + throw new Error(`Path '${relativePath}' must be relative to the workspace`); + } + const root = path.resolve(workspacePath); + const resolved = path.resolve(root, relativePath); + const relative = path.relative(root, resolved); + if (relative.startsWith('..') || path.isAbsolute(relative)) { + throw new Error(`Path '${relativePath}' is outside the workspace`); + } + return resolved; +} + +function countOccurrences(value: string, needle: string): number { + let count = 0; + let index = 0; + while (true) { + index = value.indexOf(needle, index); + if (index === -1) return count; + count += 1; + index += needle.length; + } +} + +function isBenignPipeError(error: unknown): boolean { + if (!error || typeof error !== 'object') return false; + const code = (error as { code?: unknown }).code; + return code === 'EPIPE' || code === 'ERR_STREAM_DESTROYED'; +} + +function runBashCommand( + command: string, + options: { readonly cwd: string; readonly timeoutMs: number; readonly signal?: AbortSignal }, +): Promise { + const startedAt = Date.now(); + const detached = process.platform !== 'win32'; + + return new Promise((resolve, reject) => { + const child = spawn(command, { + shell: true, + cwd: options.cwd, + stdio: ['ignore', 'pipe', 'pipe'], + env: process.env, + detached, + }); + + const stdout = new BoundedBuffer(MAX_BASH_OUTPUT_BYTES); + const stderr = new BoundedBuffer(MAX_BASH_OUTPUT_BYTES); + let timedOut = false; + let childError: Error | undefined; + let settled = false; + + const killChild = () => { + try { + if (detached && child.pid) { + process.kill(-child.pid, 'SIGKILL'); + } else { + child.kill('SIGKILL'); + } + } catch { + try { + child.kill('SIGKILL'); + } catch {} + } + }; + + const timeout = setTimeout(() => { + timedOut = true; + killChild(); + }, options.timeoutMs); + timeout.unref?.(); + + const abortHandler = () => { + timedOut = true; + killChild(); + }; + options.signal?.addEventListener('abort', abortHandler, { once: true }); + + child.stdout?.on('data', (chunk: Buffer) => stdout.append(chunk)); + child.stderr?.on('data', (chunk: Buffer) => stderr.append(chunk)); + child.stdout?.on('error', (error) => { + if (!isBenignPipeError(error)) { + stderr.append(Buffer.from(`stdout error: ${(error as Error).message}\n`)); + } + }); + child.stderr?.on('error', (error) => { + if (!isBenignPipeError(error)) { + stderr.append(Buffer.from(`stderr error: ${(error as Error).message}\n`)); + } + }); + + child.on('error', (error) => { + if (isBenignPipeError(error)) return; + childError = error; + }); + + child.on('close', (code) => { + if (settled) return; + settled = true; + clearTimeout(timeout); + options.signal?.removeEventListener('abort', abortHandler); + if (childError) { + reject(childError); + return; + } + resolve({ + command, + exit_code: code, + stdout: stdout.toString(), + stderr: stderr.toString(), + duration_ms: Date.now() - startedAt, + timed_out: timedOut, + truncated: stdout.truncated || stderr.truncated, + }); + }); + }); +} + +class BoundedBuffer { + private readonly chunks: Buffer[] = []; + private bytes = 0; + truncated = false; + + constructor(private readonly maxBytes: number) {} + + append(chunk: Buffer): void { + if (this.bytes >= this.maxBytes) { + this.truncated = true; + return; + } + const remaining = this.maxBytes - this.bytes; + if (chunk.byteLength <= remaining) { + this.chunks.push(chunk); + this.bytes += chunk.byteLength; + return; + } + this.chunks.push(chunk.subarray(0, remaining)); + this.bytes = this.maxBytes; + this.truncated = true; + } + + toString(): string { + return Buffer.concat(this.chunks).toString('utf8').replace(/\r\n/g, '\n'); + } +} + +function mapAiSdkResponse( + result: AiGenerateTextResult, + context: { + readonly model: string; + readonly baseURL: string; + readonly maxSteps: number; + readonly toolAllowlist: readonly AiSdkToolName[]; + readonly observedToolCalls: readonly ToolCall[]; + readonly durationMs: number; + readonly startTime: string; + readonly endTime: string; + }, +): ProviderResponse { + const resultToolCalls = toolCallsFromResult(result); + const toolCalls = + context.observedToolCalls.length > 0 ? context.observedToolCalls : resultToolCalls; + const output: Message[] = [ + { + role: 'assistant', + content: typeof result.text === 'string' ? result.text : '', + ...(toolCalls.length > 0 ? { toolCalls } : {}), + startTime: context.startTime, + endTime: context.endTime, + durationMs: context.durationMs, + metadata: { + finishReason: result.finishReason, + rawFinishReason: result.rawFinishReason, + }, + }, + ]; + + const tokenUsage = toProviderTokenUsage(result.totalUsage ?? result.usage); + const stepCount = + Array.isArray(result.steps) && result.steps.length > 0 ? result.steps.length : 1; + + return { + raw: { + provider: 'ai-sdk-agent', + model: context.model, + baseURL: context.baseURL, + maxSteps: context.maxSteps, + tools: context.toolAllowlist, + finishReason: result.finishReason, + rawFinishReason: result.rawFinishReason, + warnings: result.warnings, + request: result.request, + response: result.response, + providerMetadata: result.providerMetadata, + steps: result.steps, + }, + output, + ...(tokenUsage ? { tokenUsage } : {}), + usage: toJsonObject(result.totalUsage ?? result.usage), + durationMs: context.durationMs, + startTime: context.startTime, + endTime: context.endTime, + steps: { + count: stepCount, + toolCallCount: toolCalls.length, + }, + }; +} + +function toolCallsFromResult(result: AiGenerateTextResult): ToolCall[] { + const outputById = new Map(); + for (const toolResult of result.toolResults ?? []) { + if (toolResult.toolCallId) { + outputById.set(toolResult.toolCallId, toolResult); + } + } + for (const step of result.steps ?? []) { + for (const toolResult of step.toolResults ?? []) { + if (toolResult.toolCallId) { + outputById.set(toolResult.toolCallId, toolResult); + } + } + } + + const calls: ToolCall[] = []; + const seen = new Set(); + const append = (toolCall: AiToolCall) => { + const id = toolCall.toolCallId; + const name = toolCall.toolName; + if (!id || !name || seen.has(id)) return; + seen.add(id); + const resultForCall = outputById.get(id); + calls.push( + normalizeToolCall('ai-sdk-agent', { + tool: name, + input: toolCall.input ?? resultForCall?.input, + output: resultForCall?.output, + id, + }), + ); + }; + + for (const step of result.steps ?? []) { + for (const toolCall of step.toolCalls ?? []) append(toolCall); + } + for (const toolCall of result.toolCalls ?? []) append(toolCall); + return calls; +} + +function toProviderTokenUsage(usage: AiUsage | undefined): ProviderTokenUsage | undefined { + if (!usage) return undefined; + const input = finiteOrZero(usage.inputTokens); + const output = finiteOrZero(usage.outputTokens); + const cached = finiteOrUndefined( + usage.inputTokenDetails?.cacheReadTokens ?? usage.cachedInputTokens, + ); + const reasoning = finiteOrUndefined( + usage.outputTokenDetails?.reasoningTokens ?? usage.reasoningTokens, + ); + return { + input, + output, + ...(cached !== undefined ? { cached } : {}), + ...(reasoning !== undefined ? { reasoning } : {}), + }; +} + +function finiteOrZero(value: unknown): number { + return typeof value === 'number' && Number.isFinite(value) ? value : 0; +} + +function finiteOrUndefined(value: unknown): number | undefined { + return typeof value === 'number' && Number.isFinite(value) ? value : undefined; +} + +function toJsonObject(value: unknown): JsonObject | undefined { + if (!value || typeof value !== 'object') { + return undefined; + } + try { + return JSON.parse(JSON.stringify(value)) as JsonObject; + } catch { + return undefined; + } +} + +export const _internal = { + createCodingTools, + findManagedSdkInstallRoot, + formatDependencyLoadError, + isBenignPipeError, + parseToolAllowlist, + resolveWorkspacePath, + runBashCommand, + setLoaderForTesting(loader: (() => Promise) | undefined): void { + loaderOverride = loader; + sdkModules = null; + loadingPromise = null; + }, +}; diff --git a/packages/core/src/evaluation/providers/index.ts b/packages/core/src/evaluation/providers/index.ts index 89de1de05..b5122e11c 100644 --- a/packages/core/src/evaluation/providers/index.ts +++ b/packages/core/src/evaluation/providers/index.ts @@ -1,4 +1,5 @@ import { AgentvProvider } from './agentv-provider.js'; +import { AiSdkAgentProvider } from './ai-sdk-agent.js'; import { ClaudeCliProvider } from './claude-cli.js'; import { ClaudeSdkProvider } from './claude-sdk.js'; import { ClaudeProvider } from './claude.js'; @@ -46,6 +47,7 @@ export { extractLastAssistantContent } from './types.js'; export type { AgentVResolvedConfig, + AiSdkAgentResolvedConfig, AnthropicResolvedConfig, ApiFormat, AzureResolvedConfig, @@ -106,6 +108,7 @@ export function createBuiltinProviderRegistry(): ProviderRegistry { registry .register('openai', (t) => new OpenAIProvider(t.name, t.config as never)) + .register('ai-sdk-agent', (t) => new AiSdkAgentProvider(t.name, t.config as never)) .register('openrouter', (t) => new OpenRouterProvider(t.name, t.config as never)) .register('azure', (t) => new AzureProvider(t.name, t.config as never)) .register('anthropic', (t) => new AnthropicProvider(t.name, t.config as never)) diff --git a/packages/core/src/evaluation/providers/normalize-tool-call.ts b/packages/core/src/evaluation/providers/normalize-tool-call.ts index 53b171376..1235134f9 100644 --- a/packages/core/src/evaluation/providers/normalize-tool-call.ts +++ b/packages/core/src/evaluation/providers/normalize-tool-call.ts @@ -125,6 +125,12 @@ const TOOL_NAME_MAP = new Map([ ['codex::command_execution', 'Bash'], ['codex::file_change', 'Edit'], + // --- AI SDK Agent --- + ['ai-sdk-agent::read', 'Read'], + ['ai-sdk-agent::write', 'Write'], + ['ai-sdk-agent::edit', 'Edit'], + ['ai-sdk-agent::bash', 'Bash'], + // --- Pi --- ['pi-coding-agent::read', 'Read'], ['pi-coding-agent::bash', 'Bash'], diff --git a/packages/core/src/evaluation/providers/targets.ts b/packages/core/src/evaluation/providers/targets.ts index 9996b01fe..51851a765 100644 --- a/packages/core/src/evaluation/providers/targets.ts +++ b/packages/core/src/evaluation/providers/targets.ts @@ -388,6 +388,18 @@ export interface OpenAIResolvedConfig { readonly retry?: RetryConfig; } +export interface AiSdkAgentResolvedConfig { + readonly baseURL: string; + readonly apiKey: string; + readonly model: string; + readonly temperature?: number; + readonly maxSteps: number; + readonly tools?: string; + readonly systemPrompt?: string; + readonly cwd?: string; + readonly timeoutMs?: number; +} + /** * OpenRouter settings used by the Vercel AI SDK provider. */ @@ -593,6 +605,7 @@ const DEPRECATED_TARGET_CAMEL_CASE_FIELDS = new Map([ ['deploymentName', 'model'], ['thinkingBudget', 'thinking_budget'], ['maxTokens', 'max_output_tokens'], + ['maxSteps', 'max_steps'], ['apiFormat', 'api_format'], ['timeoutSeconds', 'timeout_seconds'], ['logDir', 'log_dir'], @@ -751,6 +764,10 @@ interface ResolvedTargetBase { export type ResolvedTarget = | (ResolvedTargetBase & { readonly kind: 'openai'; readonly config: OpenAIResolvedConfig }) + | (ResolvedTargetBase & { + readonly kind: 'ai-sdk-agent'; + readonly config: AiSdkAgentResolvedConfig; + }) | (ResolvedTargetBase & { readonly kind: 'openrouter'; readonly config: OpenRouterResolvedConfig; @@ -979,6 +996,12 @@ export function resolveTargetDefinition( ...base, config: resolveOpenAIConfig(parsed, env), }; + case 'ai-sdk-agent': + return { + kind: 'ai-sdk-agent', + ...base, + config: resolveAiSdkAgentConfig(parsed, env, evalFilePath), + }; case 'openrouter': return { kind: 'openrouter', @@ -1243,6 +1266,75 @@ function resolveOpenAIConfig( }; } +function resolveAiSdkAgentConfig( + target: z.infer, + env: EnvLookup, + _evalFilePath?: string, +): AiSdkAgentResolvedConfig { + const baseUrlSource = target.base_url ?? target.endpoint; + const apiKeySource = target.api_key; + const modelSource = target.model ?? target.deployment ?? target.variant; + const temperatureSource = target.temperature; + const maxStepsSource = target.max_steps; + const toolsSource = target.tools; + const systemPromptSource = target.system_prompt; + const cwdSource = target.cwd; + const timeoutSource = target.timeout_seconds; + + const baseURL = normalizeOpenAIBaseUrl( + resolveOptionalString(baseUrlSource, env, `${target.name} ai-sdk-agent base URL`, { + allowLiteral: true, + optionalEnv: true, + }), + ); + const apiKey = resolveString(apiKeySource, env, `${target.name} ai-sdk-agent API key`); + const model = resolveOptionalString(modelSource, env, `${target.name} ai-sdk-agent model`, { + allowLiteral: true, + optionalEnv: true, + }); + if (!model) { + throw new Error(`${target.name} ai-sdk-agent model is required`); + } + + const maxStepsRaw = resolveOptionalNumber( + maxStepsSource, + `${target.name} ai-sdk-agent max steps`, + ); + const maxSteps = maxStepsRaw === undefined ? 20 : maxStepsRaw; + if (!Number.isInteger(maxSteps) || maxSteps < 1 || maxSteps > 100) { + throw new Error(`${target.name} ai-sdk-agent max_steps must be an integer from 1 to 100`); + } + + const tools = resolveOptionalString(toolsSource, env, `${target.name} ai-sdk-agent tools`, { + allowLiteral: true, + optionalEnv: true, + }); + + const systemPrompt = + typeof systemPromptSource === 'string' && systemPromptSource.trim().length > 0 + ? systemPromptSource.trim() + : undefined; + + const cwd = resolveOptionalString(cwdSource, env, `${target.name} ai-sdk-agent cwd`, { + allowLiteral: true, + optionalEnv: true, + }); + + const timeoutMs = resolveTimeoutMs(timeoutSource, `${target.name} ai-sdk-agent timeout`); + + return { + baseURL, + apiKey, + model, + temperature: resolveOptionalNumber(temperatureSource, `${target.name} temperature`), + maxSteps, + tools, + systemPrompt, + cwd, + timeoutMs, + }; +} + function resolveOpenRouterConfig( target: z.infer, env: EnvLookup, diff --git a/packages/core/src/evaluation/providers/types.ts b/packages/core/src/evaluation/providers/types.ts index d265d5a01..c75da6dcb 100644 --- a/packages/core/src/evaluation/providers/types.ts +++ b/packages/core/src/evaluation/providers/types.ts @@ -19,6 +19,7 @@ export type ProviderKind = | 'anthropic' | 'gemini' | 'codex' + | 'ai-sdk-agent' | 'copilot-sdk' | 'copilot-cli' | 'copilot-log' @@ -45,6 +46,7 @@ export type ProviderKind = */ export const AGENT_PROVIDER_KINDS: readonly ProviderKind[] = [ 'codex', + 'ai-sdk-agent', 'copilot-sdk', 'copilot-cli', 'pi-coding-agent', @@ -85,6 +87,7 @@ export const KNOWN_PROVIDERS: readonly ProviderKind[] = [ 'anthropic', 'gemini', 'codex', + 'ai-sdk-agent', 'copilot-sdk', 'copilot-cli', 'copilot-log', @@ -394,6 +397,8 @@ export interface TargetDefinition { // Common fields readonly temperature?: number | unknown | undefined; readonly max_output_tokens?: number | unknown | undefined; + readonly max_steps?: number | unknown | undefined; + readonly tools?: string | unknown | undefined; // Codex fields readonly executable?: string | unknown | undefined; readonly command?: string | unknown | undefined; @@ -412,7 +417,7 @@ export interface TargetDefinition { readonly log_output_format?: string | unknown | undefined; /** New stream_log field — replaces log_format. false=no stream log, 'raw'=per-event, 'summary'=consolidated. */ readonly stream_log?: string | boolean | unknown | undefined; - // System prompt (codex, copilot, claude, pi-coding-agent) + // System prompt (codex, copilot, claude, pi-coding-agent, ai-sdk-agent) readonly system_prompt?: string | unknown | undefined; // Claude Agent SDK fields readonly max_turns?: number | unknown | undefined; diff --git a/packages/core/src/evaluation/validation/targets-validator.ts b/packages/core/src/evaluation/validation/targets-validator.ts index 4d30a3491..22b22a037 100644 --- a/packages/core/src/evaluation/validation/targets-validator.ts +++ b/packages/core/src/evaluation/validation/targets-validator.ts @@ -63,6 +63,22 @@ const OPENAI_SETTINGS = new Set([ 'max_output_tokens', ]); +const AI_SDK_AGENT_SETTINGS = new Set([ + ...COMMON_SETTINGS, + 'endpoint', + 'base_url', + 'api_key', + 'model', + 'deployment', + 'variant', + 'temperature', + 'max_steps', + 'tools', + 'system_prompt', + 'cwd', + 'timeout_seconds', +]); + const OPENROUTER_SETTINGS = new Set([ ...COMMON_SETTINGS, ...RETRY_SETTINGS, @@ -222,6 +238,8 @@ function getKnownSettings(provider: string): Set | null { switch (normalizedProvider) { case 'openai': return OPENAI_SETTINGS; + case 'ai-sdk-agent': + return AI_SDK_AGENT_SETTINGS; case 'openrouter': return OPENROUTER_SETTINGS; case 'azure': diff --git a/packages/core/src/types/ai-sdk-agent.d.ts b/packages/core/src/types/ai-sdk-agent.d.ts new file mode 100644 index 000000000..cd31db9dd --- /dev/null +++ b/packages/core/src/types/ai-sdk-agent.d.ts @@ -0,0 +1,24 @@ +// The ai-sdk-agent provider loads Vercel AI SDK packages lazily only when the +// target is used. They are intentionally not normal AgentV startup +// dependencies, so keep a small type stub for dynamic imports. + +declare module 'ai' { + export function generateText(options: Record): Promise>; + export function stepCountIs(stepCount: number): unknown; + export function tool>(definition: T): T; + export function jsonSchema(schema: unknown): unknown; +} + +declare module '@ai-sdk/openai' { + interface OpenAIProviderStub { + (modelId: string): unknown; + chat(modelId: string): unknown; + } + + export function createOpenAI(options?: { + baseURL?: string; + apiKey?: string; + headers?: Record; + name?: string; + }): OpenAIProviderStub; +} diff --git a/packages/core/test/evaluation/providers/ai-sdk-agent.test.ts b/packages/core/test/evaluation/providers/ai-sdk-agent.test.ts new file mode 100644 index 000000000..53cd74890 --- /dev/null +++ b/packages/core/test/evaluation/providers/ai-sdk-agent.test.ts @@ -0,0 +1,244 @@ +import { afterEach, describe, expect, it, mock } from 'bun:test'; +import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; + +import { AiSdkAgentProvider, _internal } from '../../../src/evaluation/providers/ai-sdk-agent.js'; + +function mockAiSdk(generateText: ReturnType) { + return { + ai: { + generateText, + stepCountIs: (stepCount: number) => ({ stepCount }), + tool: (definition: Record) => definition, + jsonSchema: (schema: unknown) => schema, + }, + openai: { + createOpenAI: mock((options?: Record) => { + const provider = Object.assign( + (modelId: string) => ({ modelId, options, api: 'responses' }), + { + chat: (modelId: string) => ({ modelId, options, api: 'chat' }), + }, + ); + return provider; + }), + }, + }; +} + +describe('AiSdkAgentProvider', () => { + afterEach(() => { + _internal.setLoaderForTesting(undefined); + }); + + it('has the correct kind and id', () => { + const provider = new AiSdkAgentProvider('test-target', { + baseURL: 'http://127.0.0.1:10531/v1', + apiKey: 'dummy', + model: 'gpt-test', + maxSteps: 20, + }); + expect(provider.kind).toBe('ai-sdk-agent'); + expect(provider.id).toBe('ai-sdk-agent:test-target'); + expect(provider.targetName).toBe('test-target'); + expect(provider.supportsBatch).toBe(false); + }); + + it('rejects when signal is already aborted', async () => { + const provider = new AiSdkAgentProvider('test-target', { + baseURL: 'http://127.0.0.1:10531/v1', + apiKey: 'dummy', + model: 'gpt-test', + maxSteps: 20, + }); + const controller = new AbortController(); + controller.abort(); + + await expect(provider.invoke({ question: 'Hello', signal: controller.signal })).rejects.toThrow( + 'aborted before execution', + ); + }); + + it('maps tool allowlists to repo-owned coding tools', async () => { + const workspace = await mkdtemp(path.join(tmpdir(), 'agentv-ai-sdk-tools-')); + try { + await writeFile(path.join(workspace, 'input.txt'), 'hello world\n', 'utf8'); + + const tools = _internal.createCodingTools( + workspace, + _internal.parseToolAllowlist('read,edit'), + ); + expect(tools.map((tool) => tool.name)).toEqual(['read', 'edit']); + + const read = tools.find((tool) => tool.name === 'read'); + const edit = tools.find((tool) => tool.name === 'edit'); + expect(await read?.execute({ path: 'input.txt' })).toMatchObject({ + path: 'input.txt', + content: 'hello world\n', + }); + expect( + await edit?.execute({ + path: 'input.txt', + old_string: 'hello', + new_string: 'hi', + }), + ).toEqual({ path: 'input.txt', replacements: 1 }); + expect(await readFile(path.join(workspace, 'input.txt'), 'utf8')).toBe('hi world\n'); + } finally { + await rm(workspace, { recursive: true, force: true }); + } + }); + + it('rejects unsupported tool names', () => { + expect(() => _internal.parseToolAllowlist('read,grep')).toThrow("unsupported tool 'grep'"); + }); + + it('keeps tool paths inside the eval workspace', async () => { + const workspace = await mkdtemp(path.join(tmpdir(), 'agentv-ai-sdk-workspace-')); + try { + expect(() => _internal.resolveWorkspacePath(workspace, '../outside.txt')).toThrow( + 'outside the workspace', + ); + expect(() => _internal.resolveWorkspacePath(workspace, '/tmp/outside.txt')).toThrow( + 'must be relative', + ); + } finally { + await rm(workspace, { recursive: true, force: true }); + } + }); + + it('runs a mocked AI SDK tool loop and maps AgentV ProviderResponse fields', async () => { + const workspace = await mkdtemp(path.join(tmpdir(), 'agentv-ai-sdk-loop-')); + try { + await writeFile(path.join(workspace, 'input.txt'), 'hello world\n', 'utf8'); + + const generateText = mock(async (options: Record) => { + const tools = options.tools as Record< + string, + { + execute(input: unknown, options?: { toolCallId?: string }): Promise; + } + >; + + const readOutput = await tools.read.execute( + { path: 'input.txt' }, + { toolCallId: 'read-1' }, + ); + const editOutput = await tools.edit.execute( + { path: 'input.txt', old_string: 'hello', new_string: 'hi' }, + { toolCallId: 'edit-1' }, + ); + const bashOutput = await tools.bash.execute( + { command: 'printf done' }, + { toolCallId: 'bash-1' }, + ); + + return { + text: 'Completed.', + finishReason: 'stop', + totalUsage: { + inputTokens: 11, + outputTokens: 7, + inputTokenDetails: { cacheReadTokens: 3 }, + outputTokenDetails: { reasoningTokens: 2 }, + }, + steps: [ + { + toolCalls: [ + { toolCallId: 'read-1', toolName: 'read', input: { path: 'input.txt' } }, + { + toolCallId: 'edit-1', + toolName: 'edit', + input: { path: 'input.txt', old_string: 'hello', new_string: 'hi' }, + }, + { toolCallId: 'bash-1', toolName: 'bash', input: { command: 'printf done' } }, + ], + toolResults: [ + { toolCallId: 'read-1', toolName: 'read', output: readOutput }, + { toolCallId: 'edit-1', toolName: 'edit', output: editOutput }, + { toolCallId: 'bash-1', toolName: 'bash', output: bashOutput }, + ], + }, + ], + }; + }); + _internal.setLoaderForTesting(async () => mockAiSdk(generateText)); + + const provider = new AiSdkAgentProvider('test-target', { + baseURL: 'http://127.0.0.1:10531/v1', + apiKey: 'dummy', + model: 'gpt-test', + temperature: 0.2, + maxSteps: 12, + tools: 'read,bash,edit', + systemPrompt: 'Project-specific instructions.', + }); + + const response = await provider.invoke({ + question: 'Update greeting', + cwd: workspace, + maxOutputTokens: 123, + }); + + expect(generateText).toHaveBeenCalledTimes(1); + const callOptions = generateText.mock.calls[0]?.[0] as Record; + expect(callOptions.prompt).toContain('Update greeting'); + expect(callOptions.system).toContain('Project-specific instructions.'); + expect(callOptions.stopWhen).toEqual({ stepCount: 12 }); + expect(callOptions.temperature).toBe(0.2); + expect(callOptions.maxOutputTokens).toBe(123); + expect(callOptions.model).toMatchObject({ + modelId: 'gpt-test', + api: 'chat', + }); + + expect(await readFile(path.join(workspace, 'input.txt'), 'utf8')).toBe('hi world\n'); + expect(response.output?.[0]?.content).toBe('Completed.'); + expect(response.output?.[0]?.toolCalls?.map((toolCall) => toolCall.tool)).toEqual([ + 'Read', + 'Edit', + 'Bash', + ]); + expect(response.tokenUsage).toEqual({ + input: 11, + output: 7, + cached: 3, + reasoning: 2, + }); + expect(response.steps).toEqual({ count: 1, toolCallCount: 3 }); + expect(response.raw).toMatchObject({ + provider: 'ai-sdk-agent', + model: 'gpt-test', + baseURL: 'http://127.0.0.1:10531/v1', + maxSteps: 12, + tools: ['read', 'bash', 'edit'], + }); + } finally { + await rm(workspace, { recursive: true, force: true }); + } + }); + + it('formats dependency load failures with the managed install path', () => { + const message = _internal.formatDependencyLoadError(new Error('boom')); + expect(message).toContain('deps/ai-sdk-agent'); + expect(message).toContain('bun add ai@^6.0.0 @ai-sdk/openai@^3.0.0'); + expect(message).toContain('Original error: boom'); + }); + + it('treats closed-pipe errors as benign and captures bounded bash output', async () => { + expect(_internal.isBenignPipeError({ code: 'EPIPE' })).toBe(true); + + const workspace = await mkdtemp(path.join(tmpdir(), 'agentv-ai-sdk-bash-')); + try { + const result = await _internal.runBashCommand('yes | head -n 1', { + cwd: workspace, + timeoutMs: 1000, + }); + expect(result.stdout.trim()).toBe('y'); + expect(result.timed_out).toBe(false); + } finally { + await rm(workspace, { recursive: true, force: true }); + } + }); +}); diff --git a/packages/core/test/evaluation/providers/targets.test.ts b/packages/core/test/evaluation/providers/targets.test.ts index e2acc1cb2..d26f5ab30 100644 --- a/packages/core/test/evaluation/providers/targets.test.ts +++ b/packages/core/test/evaluation/providers/targets.test.ts @@ -1332,6 +1332,75 @@ describe('createProvider', () => { }); }); + it('resolves ai-sdk-agent OpenAI-compatible target config', () => { + const env = { + AGENTV_OPENAI_BASE_URL: 'http://127.0.0.1:10531/v1', + AGENTV_OPENAI_API_KEY: 'dummy', + AGENTV_CODEX_MODEL: 'gpt-5.3-codex-spark', + } satisfies Record; + + const resolved = resolveTargetDefinition( + { + name: 'ai-sdk-coding-agent', + provider: 'ai-sdk-agent', + base_url: '${{ AGENTV_OPENAI_BASE_URL }}', + api_key: '${{ AGENTV_OPENAI_API_KEY }}', + model: '${{ AGENTV_CODEX_MODEL }}', + temperature: 0.1, + max_steps: 12, + tools: 'read,bash,edit,write', + system_prompt: 'Use the repo instructions.', + grader_target: 'grader', + }, + env, + ); + + expect(resolved.kind).toBe('ai-sdk-agent'); + if (resolved.kind !== 'ai-sdk-agent') throw new Error('expected ai-sdk-agent'); + expect(resolved.graderTarget).toBe('grader'); + expect(resolved.config).toEqual({ + baseURL: 'http://127.0.0.1:10531/v1', + apiKey: 'dummy', + model: 'gpt-5.3-codex-spark', + temperature: 0.1, + maxSteps: 12, + tools: 'read,bash,edit,write', + systemPrompt: 'Use the repo instructions.', + cwd: undefined, + timeoutMs: undefined, + }); + }); + + it('validates ai-sdk-agent max_steps bounds', () => { + expect(() => + resolveTargetDefinition( + { + name: 'ai-sdk-coding-agent', + provider: 'ai-sdk-agent', + api_key: '${{ AGENTV_OPENAI_API_KEY }}', + model: 'gpt-test', + max_steps: 0, + }, + { AGENTV_OPENAI_API_KEY: 'dummy' }, + ), + ).toThrow('max_steps must be an integer from 1 to 100'); + }); + + it('creates an ai-sdk-agent provider from resolved target config', () => { + const resolved = resolveTargetDefinition( + { + name: 'ai-sdk-coding-agent', + provider: 'ai-sdk-agent', + api_key: '${{ AGENTV_OPENAI_API_KEY }}', + model: 'gpt-test', + }, + { AGENTV_OPENAI_API_KEY: 'dummy' }, + ); + const provider = createProvider(resolved); + expect(provider.kind).toBe('ai-sdk-agent'); + expect(provider.targetName).toBe('ai-sdk-coding-agent'); + }); + it('resolves pi-coding-agent with azure subprovider and base_url', () => { const env = { AZURE_OPENAI_ENDPOINT: 'https://my-resource.openai.azure.com', diff --git a/packages/core/tsup.config.ts b/packages/core/tsup.config.ts index 2dcba83d7..c121a69e5 100644 --- a/packages/core/tsup.config.ts +++ b/packages/core/tsup.config.ts @@ -22,6 +22,8 @@ export default defineConfig({ '@opentelemetry/semantic-conventions', '@earendil-works/pi-coding-agent', '@earendil-works/pi-ai', + 'ai', + '@ai-sdk/openai', ], outExtension({ format }) { return {