diff --git a/.changeset/decision-executor.md b/.changeset/decision-executor.md index 9c7941594..ee24acf2d 100644 --- a/.changeset/decision-executor.md +++ b/.changeset/decision-executor.md @@ -3,9 +3,11 @@ --- Add `@e2e-dev/decision`, an executor that runs `agent.act` and `agent.assert` -through a decision model instead of an LLM. It takes any AI SDK evaluation -model with choice distributions and decides through `experimental_evaluate`, -one call per action with the operation and target questions fanned out; a -small language model writes field values when the choice is `type`. Tests -stay plain natural language with no params. Probability and confidence -gates are off by default. +through a decision model instead of an LLM. It takes any AI SDK decision +model with choice distributions (`evaluationModel()` instances too) and +decides through `experimental_decide`, so it needs `ai` 7.0.128 or later. One +call per action fans out the operation and target questions; a small +language model writes field values when the choice is `type`. Tests stay +plain natural language with no params. Probability and confidence gates are +off by default. A model id string or a language model fails config load with +`INVALID_CONFIG`. diff --git a/AGENTS.md b/AGENTS.md index 481b8eca9..2fccc84c3 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -87,7 +87,7 @@ suites that consume the built packages the way a user would. Expo's GraphQL API with `fetch`, and `@e2e-dev/mobile` is its only peer. - `packages/decision` — the published `@e2e-dev/decision` package: a `StepExecutor` (`decisionExecutor()`) that drives `agent.act` and - `agent.assert` through an AI SDK *evaluation* model answering `choice` + `agent.assert` through an AI SDK *decision* model answering `choice` questions with probability distributions, plus an optional small language model that writes field values when the decision model picks `type`. `minProbability` and `minConfidence` gate a chosen operation, target, diff --git a/docs/decision-models.mdx b/docs/decision-models.mdx index 93e3f6c58..4a10b4cf8 100644 --- a/docs/decision-models.mdx +++ b/docs/decision-models.mdx @@ -10,7 +10,7 @@ request: which operation, and which element for each operation that has more than one, chosen from what the executor builds out of the latest semantic tree. It never generates free text: when the choice is `type`, a small language model writes the field value. Both models are -plain AI SDK models, so any evaluation provider that answers `choice` +plain AI SDK models, so any decision model provider that answers `choice` questions with probability distributions (Jev) and any language-model provider works; the runner still authorizes every dispatched action. @@ -32,7 +32,7 @@ bun add -d @e2e-dev/decision The decision model answers `choice` questions with probability distributions; the text model writes field values. Each installs as its -own package: +own package. Decision models need `ai` 7, version 7.0.128 or later: ```bash npm @@ -60,7 +60,7 @@ export default { agents: { default: { executor: decisionExecutor({ - model: typeSafeAi.evaluationModel('jev-latest'), + model: typeSafeAi.decisionModel('jev-latest'), textModel: openrouter('inception/mercury-2.5'), }), }, @@ -68,6 +68,9 @@ export default { } satisfies E2EConfig; ``` +`evaluationModel()` instances still work. A model id string or a language +model in `model` fails config load with `INVALID_CONFIG`. + Tests stay plain natural language, with no params needed: ```ts title="tests/todos.e2e.ts" diff --git a/docs/examples/models/e2e.decision.config.ts b/docs/examples/models/e2e.decision.config.ts index 52c869c8b..3b78195cd 100644 --- a/docs/examples/models/e2e.decision.config.ts +++ b/docs/examples/models/e2e.decision.config.ts @@ -9,7 +9,7 @@ export default { agents: { default: { executor: decisionExecutor({ - model: typeSafeAi.evaluationModel('jev-latest'), + model: typeSafeAi.decisionModel('jev-latest'), textModel: openrouter('inception/mercury-2.5'), }), }, diff --git a/docs/package.json b/docs/package.json index c9356054f..36d3c50ee 100644 --- a/docs/package.json +++ b/docs/package.json @@ -9,14 +9,14 @@ "typecheck": "tsc --noEmit --project examples/tsconfig.json" }, "devDependencies": { - "@ai-sdk/typesafe-ai": "3.0.12", + "@ai-sdk/typesafe-ai": "3.0.13", "@e2e-dev/decision": "workspace:*", "@e2e-dev/mobile": "workspace:*", "@e2e-dev/web": "workspace:*", "@openrouter/ai-sdk-provider": "3.1.0", "@types/node": "26.6.2", "agent-device": "0.21.21", - "ai": "7.0.107", + "ai": "7.0.128", "e2e": "workspace:*", "mint": "4.2.909", "typescript": "7.0.2", diff --git a/docs/reference/errors.mdx b/docs/reference/errors.mdx index b6fe86145..d8ffed841 100644 --- a/docs/reference/errors.mdx +++ b/docs/reference/errors.mdx @@ -84,8 +84,8 @@ These appear in terminal output and in the report as plain string codes. | --- | --- | --- | ---: | | `CONFIG_NOT_FOUND` | `--config ` names a file that does not exist, or no `e2e.config.ts` or `e2e.config.mts` exists from the working directory up to the repository root | configuration | 2 | | `CONFIG_AMBIGUOUS` | Both `e2e.config.ts` and `e2e.config.mts` in one directory | configuration | 2 | -| `CONFIG_LOAD_FAILED` | The config module failed to import: a package that is declared but not installed, an undeclared package, a wrong subpath, a removed export, or any other throw while it evaluates. The message names the fix. A config error raised while it evaluates keeps its own code: an option a factory the config calls (`web()`, `mobile()`, `defineEngine()`) refuses, or a `secrets.get()` handle turned into a string, is `INVALID_CONFIG` | configuration | 2 | -| `INVALID_CONFIG` | A factory the config module calls (`web()`, `mobile()`, `defineEngine()`) refused its options, with the factory's own message; or an unknown key, out-of-range value, invalid enum, missing `targets`, a target key outside `{ name, platform, engine, app, trace, video }`, a target `app` key that is unknown or malformed, `app` on a target without an engine, a `web()` target without `app.url` or with a device field, a `mobile()` target without `app.bundleId` or `app.appPath`, with `app.url` (not supported yet), a link in `bundleId`, or an unknown permission name or state (the message names the nearest name), an unknown key in a `web()` or `mobile()` option, including `web({ connect })` and `web({ basicAuth })` (the message names the nearest one), a `trace` or `video` that is not a recording mode, the removed artifact kinds list (`artifacts: [...]` or `artifacts.kinds`) and `artifacts.trace` block (the message names the `trace` mode they meant), the removed `artifacts.video` block or `video` artifact kind (the message names the `video` option), the removed `specVersion` key, a `secrets.get()` in an engine option that names no `secrets` entry (a credential's password is not one), a `secrets.get()` reference turned into a string while the config evaluates (a template literal, `String()`) or given where a string is needed (an `app.command` `args` entry or `env` value; only an engine option that declares secrets takes one), an `app.command` `args` entry or `env` value that is not a string (an `undefined` `env` value is dropped instead), an unknown key in `app.command` (the message names the nearest one), an `engine` that is not a `defineEngine` handle, `json` reporter combined with `list`, duplicate or malformed target name, a secret or credential password shorter than 6 characters (code points), two credentials or two secrets whose names map to one `E2E_USER_*` or `E2E_SECRET_*` override variable (`api-key` and `api_key`), oversized `agents..context`, an `expect.poll` `timeout` or `interval` that is not a finite number of milliseconds, a `tests` value that is not a glob or a list of globs, a `tests` list of only `!` exclusions, the removed top-level `limits` key, an agents entry that is a bare `StepExecutor` (the message names `{ executor }`) or a model instance (the message names `{ model }`), an unknown agent key (the message names the nearest one), the removed agent keys `timeout` and `maxTurns` (the message names `judgmentTimeout` and `maxModelCalls`), `system` or `tools` beside `executor`, a `tools` entry `defineTool` did not build or that takes a built-in tool name (a grammar verb, `complete_step`, `locate`, `start_recording`, `stop_recording`, or `report_finding`), `createToolLoopExecutor({ maxTurns })`, `reuseExisting` on an app whose `app.url` asks for a free port, or a port-0 `app.url` with no `app.command` to serve it | configuration | 2 | +| `CONFIG_LOAD_FAILED` | The config module failed to import: a package that is declared but not installed, an undeclared package, a wrong subpath, a removed export, or any other throw while it evaluates. The message names the fix. A config error raised while it evaluates keeps its own code: an option a factory the config calls (`web()`, `mobile()`, `defineEngine()`, `decisionExecutor()`) refuses, or a `secrets.get()` handle turned into a string, is `INVALID_CONFIG` | configuration | 2 | +| `INVALID_CONFIG` | A factory the config module calls (`web()`, `mobile()`, `defineEngine()`, `decisionExecutor()`) refused its options, with the factory's own message; or an unknown key, out-of-range value, invalid enum, missing `targets`, a target key outside `{ name, platform, engine, app, trace, video }`, a target `app` key that is unknown or malformed, `app` on a target without an engine, a `web()` target without `app.url` or with a device field, a `mobile()` target without `app.bundleId` or `app.appPath`, with `app.url` (not supported yet), a link in `bundleId`, or an unknown permission name or state (the message names the nearest name), an unknown key in a `web()` or `mobile()` option, including `web({ connect })` and `web({ basicAuth })` (the message names the nearest one), a `trace` or `video` that is not a recording mode, the removed artifact kinds list (`artifacts: [...]` or `artifacts.kinds`) and `artifacts.trace` block (the message names the `trace` mode they meant), the removed `artifacts.video` block or `video` artifact kind (the message names the `video` option), the removed `specVersion` key, a `secrets.get()` in an engine option that names no `secrets` entry (a credential's password is not one), a `secrets.get()` reference turned into a string while the config evaluates (a template literal, `String()`) or given where a string is needed (an `app.command` `args` entry or `env` value; only an engine option that declares secrets takes one), an `app.command` `args` entry or `env` value that is not a string (an `undefined` `env` value is dropped instead), an unknown key in `app.command` (the message names the nearest one), an `engine` that is not a `defineEngine` handle, `json` reporter combined with `list`, duplicate or malformed target name, a secret or credential password shorter than 6 characters (code points), two credentials or two secrets whose names map to one `E2E_USER_*` or `E2E_SECRET_*` override variable (`api-key` and `api_key`), oversized `agents..context`, an `expect.poll` `timeout` or `interval` that is not a finite number of milliseconds, a `tests` value that is not a glob or a list of globs, a `tests` list of only `!` exclusions, the removed top-level `limits` key, an agents entry that is a bare `StepExecutor` (the message names `{ executor }`) or a model instance (the message names `{ model }`), an unknown agent key (the message names the nearest one), the removed agent keys `timeout` and `maxTurns` (the message names `judgmentTimeout` and `maxModelCalls`), `system` or `tools` beside `executor`, a `tools` entry `defineTool` did not build or that takes a built-in tool name (a grammar verb, `complete_step`, `locate`, `start_recording`, `stop_recording`, or `report_finding`), `createToolLoopExecutor({ maxTurns })`, `reuseExisting` on an app whose `app.url` asks for a free port, or a port-0 `app.url` with no `app.command` to serve it | configuration | 2 | | `CONFIG_NOT_DETERMINISTIC` | A worker re-resolved the config module and got a different digest than the runner | configuration | 2 | | `APP_URL_REQUIRED` | A navigation or relative URL on a target that declares no `app.url`, or an `app.command` with no `app.url` or `app.readyUrl` to poll | configuration | 2 | | `INVALID_APP_URL` | Base URL with userinfo, a query, a fragment, a non-HTTP scheme, plain HTTP on a non-loopback host, port 0 on a host other than a literal loopback address (`127.0.0.1`, `[::1]`), a loopback address the run could not bind a free port on, or unparseable | configuration | 2 | diff --git a/packages/decision/README.md b/packages/decision/README.md index f2f429ad0..bdd310f3c 100644 --- a/packages/decision/README.md +++ b/packages/decision/README.md @@ -1,9 +1,10 @@ # @e2e-dev/decision Run `agent.act` and `agent.assert` through a decision model instead of an LLM: -any AI SDK evaluation model that answers `choice` questions with probability +any AI SDK decision model that answers `choice` questions with probability distributions (e.g. TypeSafe Jev), plus a small language model that writes -field values when the decision model picks `type`. +field values when the decision model picks `type`. Needs `ai` 7, version 7.0.128 or +later. ```ts import type { E2EConfig } from 'e2e'; @@ -15,7 +16,7 @@ export default { agents: { default: { executor: decisionExecutor({ - model: typeSafeAi.evaluationModel('jev-latest'), + model: typeSafeAi.decisionModel('jev-latest'), textModel: openrouter('inception/mercury-2.5'), }), }, @@ -24,7 +25,7 @@ export default { ``` Set `agents.default.executor` in the e2e config. Tests stay plain natural -language, with no params needed. One `experimental_evaluate` call per action +language, with no params needed. One `experimental_decide` call per action asks the operation plus one target question per operation; operations with a single target dispatch without a question. The text model is also the agent's judgment tier (`waitFor`, `extract`); without a text model or a configured agent diff --git a/packages/decision/package.json b/packages/decision/package.json index c6d8d60c1..a21370724 100644 --- a/packages/decision/package.json +++ b/packages/decision/package.json @@ -31,17 +31,17 @@ "test": "vitest run" }, "dependencies": { - "@ai-sdk/provider": "4.0.21", "zod": "4.6.1" }, "peerDependencies": { - "ai": "^7.0.0", + "ai": "^7.0.128", "e2e": ">=0.15.2 <1" }, "devDependencies": { + "@ai-sdk/typesafe-ai": "3.0.13", "@e2e-dev/web": "workspace:*", "@types/node": "26.6.2", - "ai": "7.0.107", + "ai": "7.0.128", "ajv": "8.20.0", "ajv-formats": "3.0.1", "e2e": "workspace:*", diff --git a/packages/decision/src/api-key.ts b/packages/decision/src/api-key.ts new file mode 100644 index 000000000..5cf567608 --- /dev/null +++ b/packages/decision/src/api-key.ts @@ -0,0 +1,13 @@ +import { AgentError } from 'e2e/agent'; +/** + * MODEL_UNAVAILABLE for a missing API key, naming the environment variable + * from the SDK's own message when it names one. Only that identifier is + * kept, never the rest of the text. + */ +export function missingKey(role: 'decision' | 'field-text', error: Error): AgentError { + const variable = /\bthe ([A-Z][A-Z0-9_]*) environment variable\b/.exec(error.message)?.[1]; + const message = variable === undefined + ? `Set the ${role} model API key.` + : `Set ${variable} to the ${role} model API key.`; + return new AgentError('MODEL_UNAVAILABLE', message); +} diff --git a/packages/decision/src/evaluate.ts b/packages/decision/src/decide.ts similarity index 76% rename from packages/decision/src/evaluate.ts rename to packages/decision/src/decide.ts index 02c369ce4..9e23cb268 100644 --- a/packages/decision/src/evaluate.ts +++ b/packages/decision/src/decide.ts @@ -1,15 +1,29 @@ -import type { Experimental_EvaluationModelV4 } from '@ai-sdk/provider'; import type { StepExecutorContext } from 'e2e'; import { AgentError, isAgentError } from 'e2e/agent'; +import { ConfigurationError } from 'e2e/engine'; +import * as ai from 'ai'; import { - experimental_evaluate, InvalidArgumentError, InvalidResponseDataError, JSONParseError, LoadAPIKeyError, TypeValidationError, } from 'ai'; +import { missingKey } from './api-key.ts'; import type { DecisionRequest } from './questions.ts'; +import type { DecisionExecutorOptions } from './types.ts'; +/** + * The SDK's decide call, or INVALID_CONFIG when the installed `ai` predates + * it. Read off the namespace so an older `ai` still loads this module and + * the user gets the version to install instead of a link error. + */ +export function requireDecide(): typeof ai.experimental_decide { + const found = (ai as Partial).experimental_decide; + if (typeof found !== 'function') { + throw new ConfigurationError('INVALID_CONFIG', 'decisionExecutor() needs ai 7.0.128 or later; update the ai package'); + } + return found; +} /** One gated answer: the chosen option, its probability, and the provider confidence. */ export interface Decision { readonly choice: string; @@ -23,13 +37,13 @@ interface RawAnswer { readonly probabilities?: Record; } /** - * Makes one evaluate call under the step budget and returns the answers. - * This is the only file that touches the SDK evaluate API, so a rename + * Makes one decide call under the step budget and returns the answers. + * This is the only file that touches the SDK decide API, so a rename * there stays a one-file change. */ -export async function evaluate( +export async function decide( ctx: StepExecutorContext, - model: Experimental_EvaluationModelV4, + model: DecisionExecutorOptions['model'], request: DecisionRequest, ): Promise> { ctx.signal.throwIfAborted(); @@ -42,8 +56,8 @@ export async function evaluate( let answers: Record; try { const call = { model, state: request.state, questions: request.questions }; - const result = await experimental_evaluate({ - ...(call as unknown as Parameters[0]), + const result = await requireDecide()({ + ...(call as unknown as Parameters[0]), maxRetries: 0, abortSignal: ctx.signal, }); @@ -54,7 +68,7 @@ export async function evaluate( providerMetadata = result.providerMetadata as Record> | undefined; answers = result.answers as Record; } catch (error) { - throw evaluateError(error, ctx.signal); + throw decideError(error, ctx.signal); } finally { ctx.budgets.recordModelCall({ provider: model.provider, @@ -94,10 +108,10 @@ function reportedConfidence(metadata: Record> | return 0; } /** Maps SDK failures onto the runner error codes without echoing provider text. */ -function evaluateError(error: unknown, signal: AbortSignal): unknown { +function decideError(error: unknown, signal: AbortSignal): unknown { signal.throwIfAborted(); if (isAgentError(error)) return error; - if (LoadAPIKeyError.isInstance(error)) return new AgentError('MODEL_UNAVAILABLE', 'Set the evaluation model API key.'); + if (LoadAPIKeyError.isInstance(error)) return missingKey('decision', error); if (InvalidArgumentError.isInstance(error)) return error; if (InvalidResponseDataError.isInstance(error) || TypeValidationError.isInstance(error) || JSONParseError.isInstance(error)) { return new AgentError('MODEL_OUTPUT_INVALID', 'The decision model returned an invalid answer.'); diff --git a/packages/decision/src/executor.ts b/packages/decision/src/executor.ts index ea9ffed65..10121306e 100644 --- a/packages/decision/src/executor.ts +++ b/packages/decision/src/executor.ts @@ -1,8 +1,9 @@ import type { LanguageModel } from 'ai'; import type { ExecutorObservation, StepExecutor, StepExecutorContext, StepTurn, StepVerdict } from 'e2e'; import { AgentError, isAgentError } from 'e2e/agent'; +import { ConfigurationError } from 'e2e/engine'; +import { decide, requireDecide, type Decision } from './decide.ts'; import { actionSpace, type ActionSpace, type Control, type Operation, type Target } from './elements.ts'; -import { evaluate, type Decision } from './evaluate.ts'; import { assertionRequest, completionRequest, @@ -21,13 +22,15 @@ const RUNTIME_CODES = new Set(['STEP_BUDGET_EXHAUSTED', 'STEP_TIMEOUT', 'CANCELL const SECRET_FILL = /^fill secret "(?:[^"\\]|\\.)*"/; /** Builds a step executor that acts through a decision model and an optional text model. */ export function decisionExecutor(options: DecisionExecutorOptions): StepExecutor { + if (typeof options !== 'object' || options === null) { + throw new ConfigurationError('INVALID_CONFIG', `decisionExecutor() takes an options object; ${EXAMPLE}`); + } + requireDecide(); + checkModel(options.model); const minProbability = options.minProbability ?? 0; const minConfidence = options.minConfidence ?? 0; if (!inUnit(minProbability) || !inUnit(minConfidence)) { - throw new Error('minProbability and minConfidence must be between 0 and 1.'); - } - if (!options.model.supportedQuestionTypes.includes('choice')) { - throw new Error('The decision model must answer choice questions.'); + throw new ConfigurationError('INVALID_CONFIG', 'decisionExecutor({ minProbability, minConfidence }) must be between 0 and 1'); } const textModel = options.textModel; return { @@ -40,6 +43,37 @@ export function decisionExecutor(options: DecisionExecutorOptions): StepExecutor }, }; } +/** How a valid model reads, for every model error. */ +const EXAMPLE = "pass an AI SDK decision model, e.g. decisionExecutor({ model: typeSafeAi.decisionModel('jev-latest') })"; +/** + * Rejects anything the AI SDK cannot decide with, at config load: a model id + * string, a language model, or a decision model that cannot answer `choice`. + */ +function checkModel(model: unknown): void { + if (typeof model === 'function') { + throw new ConfigurationError('INVALID_CONFIG', `decisionExecutor({ model }) got a function, not a model; call it with a model id, ${EXAMPLE}`); + } + if (typeof model !== 'object' || model === null) { + const got = typeof model === 'string' ? `the string ${JSON.stringify(model)}` : String(model); + throw new ConfigurationError('INVALID_CONFIG', `decisionExecutor({ model }) got ${got}; ${EXAMPLE}`); + } + const fields = model as Record; + const name = `${String(fields['provider'])}/${String(fields['modelId'])}`; + if (typeof fields['doDecide'] !== 'function' && typeof fields['doEvaluate'] !== 'function') { + const kind = typeof fields['doGenerate'] === 'function' ? `the language model ${name}` : 'an object that is not a decision model'; + throw new ConfigurationError('INVALID_CONFIG', `decisionExecutor({ model }) got ${kind}; ${EXAMPLE}`); + } + if (fields['specificationVersion'] !== 'v4') { + throw new ConfigurationError( + 'INVALID_CONFIG', + `decisionExecutor({ model }) got ${name}, a ${String(fields['specificationVersion'])} model; the AI SDK decides with v4 decision models`, + ); + } + const supported = fields['supportedQuestionTypes']; + if (!Array.isArray(supported) || !supported.includes('choice')) { + throw new ConfigurationError('INVALID_CONFIG', `decisionExecutor({ model }) got ${name}, which does not answer choice questions`); + } +} function inUnit(value: number): boolean { return Number.isFinite(value) && value >= 0 && value <= 1; } @@ -90,7 +124,7 @@ async function run( if (calls >= ctx.budgets.maxModelCalls) return undefined; transcript.push(JSON.stringify({ state: request.state, questions: request.questions })); calls += 1; - const answers = await evaluate(ctx, model, request); + const answers = await decide(ctx, model, request); transcript.push(JSON.stringify(answers)); return answers; }; diff --git a/packages/decision/src/questions.ts b/packages/decision/src/questions.ts index aa94dbcce..a23a1c453 100644 --- a/packages/decision/src/questions.ts +++ b/packages/decision/src/questions.ts @@ -52,7 +52,7 @@ export interface DecisionRequest { readonly questions: Record; } /** - * Builds one evaluate request: the operation question plus one target + * Builds one decide request: the operation question plus one target * question per operation with two or more targets, and a secret question * when two or more secrets are declared. Operations with no targets are * left out; a lone target dispatches with no question. diff --git a/packages/decision/src/text.ts b/packages/decision/src/text.ts index ed534614a..a17e318e7 100644 --- a/packages/decision/src/text.ts +++ b/packages/decision/src/text.ts @@ -9,10 +9,10 @@ import { TypeValidationError, } from 'ai'; import type { LanguageModel } from 'ai'; -import type { SharedV4ProviderOptions } from '@ai-sdk/provider'; import { AgentError, isAgentError } from 'e2e/agent'; import type { JsonValue, StepExecutorContext } from 'e2e'; import { z } from 'zod'; +import { missingKey } from './api-key.ts'; import type { HistoryEntry } from './questions.ts'; /** What the text helper needs: the goal, the field, and what the page shows. */ export interface FieldInput { @@ -33,6 +33,8 @@ const TEXT_SYSTEM = [ // code fence, and fail validation. 'Answer with only a JSON object, no code fence, with exactly one key, "text", set to that string or null.', ].join('\n'); +/** The provider options `generateText` takes. */ +type ProviderOptions = NonNullable[0]['providerOptions']>; const textSchema = z.object({ text: z.string().max(2000).nullable() }); /** * Asks the language model for one field value; null means the goal supplies @@ -65,7 +67,7 @@ export async function fieldText( output: Output.object({ schema: textSchema }), maxRetries: 0, abortSignal: ctx.signal, - ...(ctx.providerOptions === undefined ? {} : { providerOptions: ctx.providerOptions as SharedV4ProviderOptions }), + ...(ctx.providerOptions === undefined ? {} : { providerOptions: ctx.providerOptions as ProviderOptions }), }); ctx.signal.throwIfAborted(); inputTokens = result.usage.inputTokens; @@ -90,7 +92,7 @@ export async function fieldText( function textError(error: unknown, signal: AbortSignal): unknown { signal.throwIfAborted(); if (isAgentError(error)) return error; - if (LoadAPIKeyError.isInstance(error)) return new AgentError('MODEL_UNAVAILABLE', 'Set the field-text model API key.'); + if (LoadAPIKeyError.isInstance(error)) return missingKey('field-text', error); if (InvalidArgumentError.isInstance(error)) return error; const invalid = InvalidResponseDataError.isInstance(error) || TypeValidationError.isInstance(error) || JSONParseError.isInstance(error) || NoObjectGeneratedError.isInstance(error); diff --git a/packages/decision/src/types.ts b/packages/decision/src/types.ts index 624286549..ff14e5836 100644 --- a/packages/decision/src/types.ts +++ b/packages/decision/src/types.ts @@ -1,15 +1,14 @@ -import type { Experimental_EvaluationModelV4 } from '@ai-sdk/provider'; -import type { LanguageModel } from 'ai'; +import type { Experimental_DecisionModel, LanguageModel } from 'ai'; /** Options for an executor that acts through a decision model and writes field text with a language model. */ export interface DecisionExecutorOptions { /** - * An AI SDK evaluation model that answers `choice` questions with a - * probability distribution, e.g. `typeSafeAi.evaluationModel('jev-latest')`. - * Models that answer without `probabilities` fail the step with - * MODEL_OUTPUT_INVALID. + * An AI SDK decision model that answers `choice` questions with a + * probability distribution, e.g. `typeSafeAi.decisionModel('jev-latest')`. + * Evaluation models (`evaluationModel()`) work too. Models that answer + * without `probabilities` fail the step with MODEL_OUTPUT_INVALID. */ - readonly model: Experimental_EvaluationModelV4; + readonly model: Exclude; /** * A small AI SDK language model that writes field values when the * decision model picks `type`. It is exposed as the executor's `model`, so diff --git a/packages/decision/tests/ai-version.test.ts b/packages/decision/tests/ai-version.test.ts new file mode 100644 index 000000000..495c15947 --- /dev/null +++ b/packages/decision/tests/ai-version.test.ts @@ -0,0 +1,15 @@ +import { describe, expect, it, vi } from 'vitest'; +import { decisionExecutor } from '../src/index.ts'; +import { scriptedDecision } from './helpers.ts'; + +vi.mock('ai', async (original) => ({ ...(await original()), experimental_decide: undefined })); + +describe('an ai release without experimental_decide', () => { + it('loads the package and fails config load naming the version to install', () => { + const { model } = scriptedDecision(() => ({ choice: 'done' })); + expect(() => decisionExecutor({ model })).toThrow(expect.objectContaining({ + code: 'INVALID_CONFIG', + message: 'decisionExecutor() needs ai 7.0.128 or later; update the ai package', + })); + }); +}); diff --git a/packages/decision/tests/cli.test.ts b/packages/decision/tests/cli.test.ts index 7c61153d7..ef74b0ecd 100644 --- a/packages/decision/tests/cli.test.ts +++ b/packages/decision/tests/cli.test.ts @@ -63,7 +63,7 @@ describe('decision executor through the built CLI and real Chromium', () => { const requestsFile = join(directory, 'requests.jsonl'); await writeFile(join(directory, 'models.ts'), [ 'import { appendFileSync } from "node:fs";', - 'import type { Experimental_EvaluationModelV4 } from "@ai-sdk/provider";', + 'import type { DecisionExecutorOptions } from "@e2e-dev/decision";', 'import type { LanguageModel } from "ai";', 'const log = process.env.E2E_DECISION_REQUESTS === undefined ? "" : process.env.E2E_DECISION_REQUESTS;', 'function record(entry: unknown): void { if (log !== "") appendFileSync(log, JSON.stringify(entry) + "\\n"); }', @@ -96,9 +96,9 @@ describe('decision executor through the built CLI and real Chromium', () => { ' const hit = entries.find((entry) => { const record = entry[1]; return typeof record !== "string" && record.role === role; });', ' return (hit ?? entries[0] as [string, unknown])[0];', '}', - 'export const evaluationModel: Experimental_EvaluationModelV4 = {', + 'export const decisionModel: DecisionExecutorOptions["model"] = {', ' specificationVersion: "v4", provider: "scripted", modelId: "scripted-1", supportedQuestionTypes: ["choice"],', - ' async doEvaluate({ state, questions }) {', + ' async doDecide({ state, questions }) {', ' record({ state, questions });', ' const view = state as { goal?: unknown; recentActions?: { action?: unknown }[]; page?: { text?: unknown } };', ' const goal = typeof view.goal === "string" ? view.goal : "";', @@ -135,12 +135,12 @@ describe('decision executor through the built CLI and real Chromium', () => { await writeFile(join(directory, 'e2e.config.ts'), [ 'import { web } from "@e2e-dev/web";', 'import { decisionExecutor } from "@e2e-dev/decision";', - 'import { evaluationModel, textModel } from "./models.ts";', + 'import { decisionModel, textModel } from "./models.ts";', 'export default {', ' tests: "decision.e2e.ts",', ` targets: [{ engine: web(), app: { url: ${JSON.stringify(appUrl)} } }],`, ` credentials: { admin: { username: "Ada", password: ${JSON.stringify(password)} } },`, - ' agents: { default: { executor: decisionExecutor({ model: evaluationModel, textModel }) } },', + ' agents: { default: { executor: decisionExecutor({ model: decisionModel, textModel }) } },', '};', ].join('\n')); await writeFile(join(directory, 'decision.e2e.ts'), [ diff --git a/packages/decision/tests/executor.test.ts b/packages/decision/tests/executor.test.ts index 48e51d98b..5e79d9c1d 100644 --- a/packages/decision/tests/executor.test.ts +++ b/packages/decision/tests/executor.test.ts @@ -1,10 +1,14 @@ import { describe, expect, it } from 'vitest'; -import { InvalidArgumentError } from 'ai'; +import { InvalidArgumentError, LoadAPIKeyError } from 'ai'; import { AgentError } from 'e2e/agent'; import type { ExecutorNode } from 'e2e'; import { decisionExecutor } from '../src/index.ts'; -import { context, scriptedEvaluation, scriptedText } from './helpers.ts'; +import { context, scriptedDecision, scriptedText } from './helpers.ts'; +/** Matches the ConfigurationError a factory throws at config load. */ +function invalidConfig(message: string): object { + return expect.objectContaining({ name: 'ConfigurationError', code: 'INVALID_CONFIG', message: expect.stringContaining(message) }); +} const BUTTONS: ExecutorNode = { id: 'root', children: [ { id: 'save', role: 'button', name: 'Save' }, { id: 'cancel', role: 'button', name: 'Cancel' }, @@ -17,18 +21,35 @@ const FIELD: ExecutorNode = { id: 'root', children: [ /** Construction-time validation: gates and model support. */ describe('construction', () => { it('rejects gates outside [0, 1]', () => { - const { model } = scriptedEvaluation(() => ({ choice: 'done' })); + const { model } = scriptedDecision(() => ({ choice: 'done' })); for (const bad of [-1, 2, NaN]) { - expect(() => decisionExecutor({ model, minProbability: bad })).toThrow(); - expect(() => decisionExecutor({ model, minConfidence: bad })).toThrow(); + expect(() => decisionExecutor({ model, minProbability: bad })).toThrow(invalidConfig('between 0 and 1')); + expect(() => decisionExecutor({ model, minConfidence: bad })).toThrow(invalidConfig('between 0 and 1')); } }); it('rejects a model without choice support', () => { - const { model } = scriptedEvaluation(() => ({ choice: 'done' }), { supported: [] }); - expect(() => decisionExecutor({ model })).toThrow('choice'); + const { model } = scriptedDecision(() => ({ choice: 'done' }), { supported: [] }); + expect(() => decisionExecutor({ model })).toThrow(invalidConfig('got scripted/scripted-1, which does not answer choice questions')); + }); + it.each([ + ['no options', undefined, 'decisionExecutor() takes an options object'], + ['no model', {}, 'decisionExecutor({ model }) got undefined;'], + ['a model id', { model: 'openai/gpt-5' }, 'decisionExecutor({ model }) got the string "openai/gpt-5";'], + ['an uncalled provider factory', { model: (id: string) => id }, 'decisionExecutor({ model }) got a function, not a model; call it'], + ['a language model', { model: scriptedText([]).model }, 'got the language model scripted-text/scripted-text-1;'], + ['a plain object', { model: { modelId: 'x' } }, 'got an object that is not a decision model;'], + ['a v3 model', { model: { specificationVersion: 'v3', provider: 'p', modelId: 'm', doDecide: () => undefined } }, 'got p/m, a v3 model'], + ])('rejects %s as INVALID_CONFIG naming the fix', (_name, options, message) => { + expect(() => decisionExecutor(options as never)).toThrow(invalidConfig(message)); + }); + it('accepts a decision model and a deprecated evaluation model', () => { + for (const legacy of [undefined, true] as const) { + const { model } = scriptedDecision(() => ({ choice: 'done' }), legacy === undefined ? {} : { legacy }); + expect(decisionExecutor({ model }).name).toBe('decision'); + } }); it('exposes the text model and replays cache', () => { - const { model } = scriptedEvaluation(() => ({ choice: 'done' })); + const { model } = scriptedDecision(() => ({ choice: 'done' })); const text = scriptedText([]); const executor = decisionExecutor({ model, textModel: text.model }); expect(executor.cache).toBe('inherit'); @@ -40,7 +61,7 @@ describe('construction', () => { describe('act loop', () => { it('taps a button and passes on a held completion check', async () => { - const { model, requests } = scriptedEvaluation((id, keys, call) => { + const { model, requests } = scriptedDecision((id, keys, call) => { if (id === 'operation') return { choice: call === 0 ? 'tap' : 'done' }; if (id === 'verdict') return { choice: 'holds' }; return { choice: keys[0] ?? '' }; @@ -56,8 +77,19 @@ describe('act loop', () => { expect(fixture.turns.length).toBeGreaterThan(0); expect(fixture.transcripts).toHaveLength(1); }); + it('runs a deprecated evaluation model through the same loop', async () => { + const { model, requests } = scriptedDecision((id, keys, call) => { + if (id === 'operation') return { choice: call === 0 ? 'tap' : 'done' }; + if (id === 'verdict') return { choice: 'holds' }; + return { choice: keys[0] ?? '' }; + }, { legacy: true }); + const fixture = context({ tree: BUTTONS }); + expect(await decisionExecutor({ model }).runStep(fixture.ctx)).toMatchObject({ status: 'passed' }); + expect(fixture.actions.tap).toHaveBeenCalledTimes(1); + expect(requests).toHaveLength(3); + }); it('dispatches a lone target with no target question', async () => { - const { model, requests } = scriptedEvaluation((id, keys, call) => ({ + const { model, requests } = scriptedDecision((id, keys, call) => ({ choice: id === 'operation' ? (call === 0 ? 'tap' : 'done') : id === 'verdict' ? 'holds' : (keys[0] ?? ''), })); const fixture = context({ tree: { id: 'root', children: [{ id: 'only', role: 'button', name: 'Only' }] } }); @@ -66,7 +98,7 @@ describe('act loop', () => { expect(Object.keys(requests[0]?.questions ?? {})).toEqual(['operation']); }); it('stops on blocked without acting', async () => { - const { model, requests } = scriptedEvaluation((id, keys) => ({ choice: id === 'operation' ? 'blocked' : (keys[0] ?? '') })); + const { model, requests } = scriptedDecision((id, keys) => ({ choice: id === 'operation' ? 'blocked' : (keys[0] ?? '') })); const fixture = context({ tree: BUTTONS }); const verdict = await decisionExecutor({ model }).runStep(fixture.ctx); expect(verdict).toMatchObject({ status: 'blocked', errorCode: 'AUTOMATION_UNSUPPORTED' }); @@ -78,7 +110,7 @@ describe('act loop', () => { describe('secrets', () => { const LOGIN: ExecutorNode = { id: 'root', children: [{ id: 'pw', role: 'textbox', name: 'Password', inputPurpose: 'password' }] }; it('asks the secret question only with two or more secrets', async () => { - const scripted = scriptedEvaluation((id, keys, call) => { + const scripted = scriptedDecision((id, keys, call) => { if (id === 'operation') return { choice: call === 0 ? 'typeSecret' : 'done' }; if (id === 'secret') return { choice: 'password' }; if (id === 'verdict') return { choice: 'holds' }; @@ -92,7 +124,7 @@ describe('secrets', () => { expect(fixture.actions.typeSecret).toHaveBeenCalledWith({ id: 'pw' }, 'password'); }); it('fills a lone secret with no secret question', async () => { - const scripted = scriptedEvaluation((id, keys, call) => ({ + const scripted = scriptedDecision((id, keys, call) => ({ choice: id === 'operation' ? (call === 0 ? 'typeSecret' : 'done') : id === 'verdict' ? 'holds' : (keys[0] ?? ''), })); const fixture = context({ tree: LOGIN }); @@ -104,7 +136,7 @@ describe('secrets', () => { describe('field text', () => { it('types the text-model value into the chosen field', async () => { - const scripted = scriptedEvaluation((id, keys, call) => ({ + const scripted = scriptedDecision((id, keys, call) => ({ choice: id === 'operation' ? (call === 0 ? 'type' : 'done') : id === 'verdict' ? 'holds' : (keys[0] ?? ''), })); const text = scriptedText(['Buy milk']); @@ -117,7 +149,7 @@ describe('field text', () => { expect(fixture.turns[0]?.calls?.[0]).toContain('Buy milk'); }); it('keeps secret values out of the text prompt', async () => { - const scripted = scriptedEvaluation((id, keys, call) => ({ + const scripted = scriptedDecision((id, keys, call) => ({ choice: id === 'operation' ? (call === 0 ? 'type' : 'done') : id === 'verdict' ? 'holds' : (keys[0] ?? ''), })); const text = scriptedText(['irrelevant']); @@ -131,7 +163,7 @@ describe('field text', () => { expect(JSON.stringify(text.prompts)).not.toContain('admin.password'); }); it('continues with history when the goal supplies no value', async () => { - const scripted = scriptedEvaluation((id, keys, call) => ({ + const scripted = scriptedDecision((id, keys, call) => ({ choice: id === 'operation' ? (call === 0 ? 'type' : call === 1 ? 'tap' : 'done') : id === 'verdict' ? 'holds' : (keys[0] ?? ''), })); const text = scriptedText([null]); @@ -143,7 +175,7 @@ describe('field text', () => { }); it('never offers type without a text model', async () => { const seen: string[][] = []; - const scripted = scriptedEvaluation((id, keys, call) => { + const scripted = scriptedDecision((id, keys, call) => { seen.push([id, ...keys]); return { choice: id === 'operation' ? (call === 0 ? 'tap' : 'done') : id === 'verdict' ? 'holds' : (keys[0] ?? '') }; }); @@ -152,7 +184,7 @@ describe('field text', () => { expect(JSON.stringify(seen)).not.toContain('"type"'); }); it('counts the text call toward the model-call budget', async () => { - const scripted = scriptedEvaluation((id, keys) => ({ + const scripted = scriptedDecision((id, keys) => ({ choice: id === 'operation' ? 'type' : id === 'verdict' ? 'holds' : (keys[0] ?? ''), })); const text = scriptedText(['Buy milk']); @@ -166,7 +198,7 @@ describe('field text', () => { describe('terminal checks', () => { it('passes done on holds', async () => { - const { model, requests } = scriptedEvaluation((id, keys, call) => ({ + const { model, requests } = scriptedDecision((id, keys, call) => ({ choice: id === 'operation' ? (call === 0 ? 'done' : 'blocked') : id === 'verdict' ? 'holds' : (keys[0] ?? ''), })); const fixture = context({ tree: BUTTONS }); @@ -175,7 +207,7 @@ describe('terminal checks', () => { expect(requests).toHaveLength(2); }); it('fails failed on fails with ACTION_FAILED', async () => { - const { model } = scriptedEvaluation((id, keys, call) => ({ + const { model } = scriptedDecision((id, keys, call) => ({ choice: id === 'operation' ? (call === 0 ? 'failed' : 'blocked') : id === 'verdict' ? 'fails' : (keys[0] ?? ''), })); const fixture = context({ tree: BUTTONS }); @@ -183,7 +215,7 @@ describe('terminal checks', () => { expect(verdict).toMatchObject({ status: 'failed', errorCode: 'ACTION_FAILED' }); }); it('continues after one rejected claim with history', async () => { - const { model } = scriptedEvaluation((id, keys, call) => { + const { model } = scriptedDecision((id, keys, call) => { if (id === 'operation') return { choice: call === 0 ? 'done' : call <= 2 ? 'tap' : 'done' }; if (id === 'verdict') return { choice: call <= 1 ? 'fails' : 'holds' }; return { choice: keys[0] ?? '' }; @@ -201,7 +233,7 @@ describe('terminal checks', () => { { claim: 'failed', check: 'inconclusive', expected: { status: 'blocked', errorCode: 'AUTOMATION_UNSUPPORTED' } }, ] as const; for (const { claim, check, expected } of cases) { - const { model, requests } = scriptedEvaluation((id, keys) => ({ + const { model, requests } = scriptedDecision((id, keys) => ({ choice: id === 'operation' ? claim : id === 'verdict' ? check : (keys[0] ?? ''), })); const fixture = context({ tree: BUTTONS }); @@ -211,7 +243,7 @@ describe('terminal checks', () => { } }); it('counts a completion check under the gate as a rejection', async () => { - const { model } = scriptedEvaluation((id, keys) => { + const { model } = scriptedDecision((id, keys) => { if (id === 'operation') return { choice: 'done' }; if (id === 'verdict') return { choice: 'holds', probabilities: { holds: 0.4, fails: 0.3, inconclusive: 0.3 }, confidence: 0.95 }; return { choice: keys[0] ?? '' }; @@ -221,7 +253,7 @@ describe('terminal checks', () => { expect(verdict).toMatchObject({ status: 'blocked', errorCode: 'AUTOMATION_UNSUPPORTED' }); }); it('sends action descriptions but no model output to the check', async () => { - const { model, requests } = scriptedEvaluation((id, keys, call) => ({ + const { model, requests } = scriptedDecision((id, keys, call) => ({ choice: id === 'operation' ? (call === 0 ? 'tap' : 'done') : id === 'verdict' ? 'holds' : (keys[0] ?? ''), })); const fixture = context({ tree: BUTTONS }); @@ -231,7 +263,7 @@ describe('terminal checks', () => { expect(JSON.stringify(check?.state)).not.toContain('recentActions'); }); it('retries the claim when the check screen is incomplete', async () => { - const { model, requests } = scriptedEvaluation((id, keys, call) => ({ + const { model, requests } = scriptedDecision((id, keys, call) => ({ choice: id === 'operation' ? (call === 0 ? 'done' : 'done') : id === 'verdict' ? 'holds' : (keys[0] ?? ''), })); const fixture = context({ tree: BUTTONS }); @@ -254,7 +286,7 @@ describe('terminal checks', () => { ] }, { id: 'save', role: 'button', name: 'Save' }, ] }; - const { model, requests } = scriptedEvaluation((id, keys, call) => ({ + const { model, requests } = scriptedDecision((id, keys, call) => ({ choice: id === 'operation' ? (call === 0 ? 'select' : 'done') : id === 'verdict' ? 'holds' : (keys[0] ?? ''), })); const fixture = context({ tree }); @@ -269,7 +301,7 @@ describe('terminal checks', () => { describe('guard rails', () => { it('blocks three actions with no page change', async () => { - const { model } = scriptedEvaluation((id, keys) => ({ + const { model } = scriptedDecision((id, keys) => ({ choice: id === 'operation' ? 'tap' : id === 'verdict' ? 'holds' : (keys[0] ?? ''), })); const fixture = context({ tree: BUTTONS }); @@ -278,7 +310,7 @@ describe('guard rails', () => { expect(fixture.actions.tap).toHaveBeenCalledTimes(3); }); it('counts a scroll that brings other nodes into view as progress', async () => { - const { model } = scriptedEvaluation((id, keys, call) => ({ + const { model } = scriptedDecision((id, keys, call) => ({ choice: id === 'operation' ? (call < 4 ? 'scroll_down' : 'done') : id === 'verdict' ? 'holds' : (keys[0] ?? ''), })); const fixture = context({ tree: BUTTONS }); @@ -297,7 +329,7 @@ describe('guard rails', () => { expect(fixture.actions.scroll).toHaveBeenCalledTimes(4); }); it('blocks three scrolls that move nothing, as at the bottom of the page', async () => { - const { model } = scriptedEvaluation((id, keys) => ({ + const { model } = scriptedDecision((id, keys) => ({ choice: id === 'operation' ? 'scroll_down' : id === 'verdict' ? 'holds' : (keys[0] ?? ''), })); const tree: ExecutorNode = { id: 'root', children: [ @@ -309,7 +341,7 @@ describe('guard rails', () => { expect(fixture.actions.scroll).toHaveBeenCalledTimes(3); }); it('never trips the guard when pages keep changing', async () => { - const { model } = scriptedEvaluation((id, keys, call) => ({ + const { model } = scriptedDecision((id, keys, call) => ({ choice: id === 'operation' ? (call < 4 ? 'tap' : 'done') : id === 'verdict' ? 'holds' : (keys[0] ?? ''), })); const fixture = context({ tree: BUTTONS }); @@ -326,7 +358,7 @@ describe('guard rails', () => { expect(fixture.actions.tap.mock.calls.length).toBeGreaterThan(3); }); it('sends failing actions back as history and rethrows hard stops', async () => { - const { model } = scriptedEvaluation((id, keys, call) => ({ + const { model } = scriptedDecision((id, keys, call) => ({ choice: id === 'operation' ? (call === 0 ? 'tap' : call === 1 ? 'tap' : 'done') : id === 'verdict' ? 'holds' : (keys[0] ?? ''), })); const fixture = context({ tree: BUTTONS }); @@ -337,14 +369,14 @@ describe('guard rails', () => { }); it('rethrows STEP_TIMEOUT and CANCELLED', async () => { for (const code of ['STEP_TIMEOUT', 'CANCELLED'] as const) { - const { model } = scriptedEvaluation((id, keys) => ({ choice: id === 'operation' ? 'tap' : (keys[0] ?? '') })); + const { model } = scriptedDecision((id, keys) => ({ choice: id === 'operation' ? 'tap' : (keys[0] ?? '') })); const fixture = context({ tree: BUTTONS }); fixture.actions.tap.mockRejectedValueOnce(new AgentError(code, 'stop')); await expect(decisionExecutor({ model }).runStep(fixture.ctx)).rejects.toMatchObject({ code }); } }); it('never makes the call that would exceed the budget', async () => { - const { model, requests } = scriptedEvaluation((id, keys) => ({ choice: id === 'operation' ? 'tap' : (keys[0] ?? '') })); + const { model, requests } = scriptedDecision((id, keys) => ({ choice: id === 'operation' ? 'tap' : (keys[0] ?? '') })); const fixture = context({ tree: BUTTONS, maxModelCalls: 1 }); const verdict = await decisionExecutor({ model }).runStep(fixture.ctx); expect(verdict).toMatchObject({ status: 'blocked', errorCode: 'AUTOMATION_UNSUPPORTED' }); @@ -352,7 +384,7 @@ describe('guard rails', () => { expect(fixture.usage).toHaveLength(1); }); it('records failed calls too', async () => { - const { model } = scriptedEvaluation(() => ({ choice: 'tap' }), { throws: new Error('boom') }); + const { model } = scriptedDecision(() => ({ choice: 'tap' }), { throws: new Error('boom') }); const fixture = context({ tree: BUTTONS }); await expect(decisionExecutor({ model }).runStep(fixture.ctx)).rejects.toMatchObject({ code: 'MODEL_PROVIDER_FAILED' }); expect(fixture.usage).toHaveLength(1); @@ -365,7 +397,7 @@ describe('gates', () => { return Object.fromEntries(keys.map((key) => [key, key === choice ? mass : rest])); }; it('stays off by default', async () => { - const { model } = scriptedEvaluation((id, keys, call) => { + const { model } = scriptedDecision((id, keys, call) => { if (id === 'operation') return call === 0 ? { choice: 'tap', probabilities: spread(keys, 'tap', 0.2), confidence: 0.9 } : { choice: 'done' }; if (id === 'verdict') return { choice: 'holds' }; return { choice: keys[0] ?? '' }; @@ -375,7 +407,7 @@ describe('gates', () => { expect(verdict).toMatchObject({ status: 'passed' }); }); it('blocks a low-probability operation under a gate', async () => { - const { model } = scriptedEvaluation((id, keys) => ({ + const { model } = scriptedDecision((id, keys) => ({ choice: id === 'operation' ? 'tap' : (keys[0] ?? ''), ...(id === 'operation' ? { probabilities: spread(keys, 'tap', 0.2), confidence: 0.9 } : {}), })); @@ -385,7 +417,7 @@ describe('gates', () => { expect(fixture.actions.tap).not.toHaveBeenCalled(); }); it('blocks a low-probability target under a gate', async () => { - const { model } = scriptedEvaluation((id, keys) => { + const { model } = scriptedDecision((id, keys) => { if (id === 'operation') return { choice: 'tap' }; return { choice: keys[0] ?? '', probabilities: Object.fromEntries(keys.map((key) => [key, 0.5])), confidence: 0.9 }; }); @@ -399,7 +431,7 @@ describe('gates', () => { describe('assert', () => { it('passes holds, fails fails, and calls the rest inconclusive', async () => { for (const [answer, status, code] of [['holds', 'passed', undefined], ['fails', 'failed', 'ASSERTION_FAILED'], ['inconclusive', 'failed', 'ASSERTION_INCONCLUSIVE']] as const) { - const { model, requests } = scriptedEvaluation(() => ({ choice: answer })); + const { model, requests } = scriptedDecision(() => ({ choice: answer })); const fixture = context({ kind: 'assert', tree: BUTTONS }); const verdict = await decisionExecutor({ model }).runStep(fixture.ctx); expect(verdict.status).toBe(status); @@ -408,7 +440,7 @@ describe('assert', () => { } }); it('treats a gated-out verdict as inconclusive', async () => { - const { model } = scriptedEvaluation(() => ({ choice: 'holds', probabilities: { holds: 0.4, fails: 0.3, inconclusive: 0.3 }, confidence: 0.9 })); + const { model } = scriptedDecision(() => ({ choice: 'holds', probabilities: { holds: 0.4, fails: 0.3, inconclusive: 0.3 }, confidence: 0.9 })); const fixture = context({ kind: 'assert', tree: BUTTONS }); const verdict = await decisionExecutor({ model, minProbability: 0.5 }).runStep(fixture.ctx); expect(verdict).toMatchObject({ status: 'failed', errorCode: 'ASSERTION_INCONCLUSIVE' }); @@ -417,12 +449,12 @@ describe('assert', () => { describe('model failures', () => { it('fails without a choice distribution', async () => { - const { model } = scriptedEvaluation(() => ({ choice: 'tap', bare: true })); + const { model } = scriptedDecision(() => ({ choice: 'tap', bare: true })); const fixture = context({ tree: BUTTONS }); await expect(decisionExecutor({ model }).runStep(fixture.ctx)).rejects.toMatchObject({ code: 'MODEL_OUTPUT_INVALID' }); }); it('fails a probability outside [0, 1]', async () => { - const { model } = scriptedEvaluation((id, keys) => { + const { model } = scriptedDecision((id, keys) => { if (id !== 'operation') return { choice: keys[0] ?? '' }; return { choice: 'tap', probabilities: Object.fromEntries(keys.map((key) => [key, key === 'tap' ? 1.5 : key === 'done' ? -0.5 : 0])) }; }); @@ -430,21 +462,32 @@ describe('model failures', () => { await expect(decisionExecutor({ model }).runStep(fixture.ctx)).rejects.toMatchObject({ code: 'MODEL_OUTPUT_INVALID' }); }); it('maps provider failures without provider text', async () => { - const { model } = scriptedEvaluation(() => ({ choice: 'tap' }), { throws: new Error('upstream detail') }); + const { model } = scriptedDecision(() => ({ choice: 'tap' }), { throws: new Error('upstream detail') }); const fixture = context({ tree: BUTTONS }); await expect(decisionExecutor({ model }).runStep(fixture.ctx)).rejects.toMatchObject({ code: 'MODEL_PROVIDER_FAILED' }); await expect(decisionExecutor({ model }).runStep(fixture.ctx)).rejects.not.toMatchObject({ message: expect.stringContaining('upstream') }); }); + it('names the API key variable the provider reads', async () => { + const missing = new LoadAPIKeyError({ message: "TypeSafe AI API key is missing. Pass it using the 'apiKey' parameter or the TYPESAFE_AI_API_KEY environment variable." }); + const { model } = scriptedDecision(() => ({ choice: 'tap' }), { throws: missing }); + await expect(decisionExecutor({ model }).runStep(context({ tree: BUTTONS }).ctx)).rejects.toMatchObject({ + code: 'MODEL_UNAVAILABLE', message: 'Set TYPESAFE_AI_API_KEY to the decision model API key.', + }); + const { model: unnamed } = scriptedDecision(() => ({ choice: 'tap' }), { throws: new LoadAPIKeyError({ message: 'no key' }) }); + await expect(decisionExecutor({ model: unnamed }).runStep(context({ tree: BUTTONS }).ctx)).rejects.toMatchObject({ + code: 'MODEL_UNAVAILABLE', message: 'Set the decision model API key.', + }); + }); it('rethrows InvalidArgumentError as our own bug', async () => { const failure = new InvalidArgumentError({ parameter: 'questions', value: 1, message: 'bad' }); - const { model } = scriptedEvaluation(() => ({ choice: 'tap' }), { throws: failure }); + const { model } = scriptedDecision(() => ({ choice: 'tap' }), { throws: failure }); const fixture = context({ tree: BUTTONS }); await expect(decisionExecutor({ model }).runStep(fixture.ctx)).rejects.toBe(failure); }); it('honors abort', async () => { const controller = new AbortController(); controller.abort(); - const { model } = scriptedEvaluation(() => ({ choice: 'tap' })); + const { model } = scriptedDecision(() => ({ choice: 'tap' })); const fixture = context({ tree: BUTTONS, signal: controller.signal }); await expect(decisionExecutor({ model }).runStep(fixture.ctx)).rejects.toThrow(); }); @@ -452,7 +495,7 @@ describe('model failures', () => { describe('observations', () => { it('acts on a truncated observation', async () => { - const { model } = scriptedEvaluation((id, keys, call) => ({ + const { model } = scriptedDecision((id, keys, call) => ({ choice: id === 'operation' ? (call === 0 ? 'tap' : 'done') : id === 'verdict' ? 'holds' : (keys[0] ?? ''), })); const fixture = context({ tree: BUTTONS, observation: { truncated: true } }); @@ -460,14 +503,14 @@ describe('observations', () => { expect(verdict).toMatchObject({ status: 'passed' }); }); it('blocks act and judges inconclusive without a tree', async () => { - const { model } = scriptedEvaluation(() => ({ choice: 'holds' })); + const { model } = scriptedDecision(() => ({ choice: 'holds' })); const act = context({ tree: BUTTONS, observation: { treeUnavailable: true } }); expect(await decisionExecutor({ model }).runStep(act.ctx)).toMatchObject({ status: 'blocked', errorCode: 'AUTOMATION_UNSUPPORTED' }); const assert = context({ kind: 'assert', tree: BUTTONS, observation: { treeUnavailable: true } }); expect(await decisionExecutor({ model }).runStep(assert.ctx)).toMatchObject({ status: 'failed', errorCode: 'ASSERTION_INCONCLUSIVE' }); }); it('judges a truncated assertion on what is there', async () => { - const { model } = scriptedEvaluation(() => ({ choice: 'holds' })); + const { model } = scriptedDecision(() => ({ choice: 'holds' })); const fixture = context({ kind: 'assert', tree: BUTTONS, observation: { truncated: true } }); const verdict = await decisionExecutor({ model }).runStep(fixture.ctx); expect(verdict).toMatchObject({ status: 'passed' }); @@ -476,7 +519,7 @@ describe('observations', () => { describe('context', () => { it('sends the ledger and non-secret params, never secret handles', async () => { - const { model, requests } = scriptedEvaluation((id, keys, call) => ({ + const { model, requests } = scriptedDecision((id, keys, call) => ({ choice: id === 'operation' ? (call === 0 ? 'tap' : 'done') : id === 'verdict' ? 'holds' : (keys[0] ?? ''), })); const fixture = context({ @@ -495,7 +538,7 @@ describe('context', () => { expect(JSON.stringify(state)).not.toContain('nested.token'); }); it('seeds history from a replayed prefix and flags the uncertain action', async () => { - const { model, requests } = scriptedEvaluation((id, keys, call) => ({ + const { model, requests } = scriptedDecision((id, keys, call) => ({ choice: id === 'operation' ? (call === 0 ? 'tap' : 'done') : id === 'verdict' ? 'holds' : (keys[0] ?? ''), })); const fixture = context({ @@ -515,7 +558,7 @@ describe('context', () => { expect(history[1]).toMatchObject({ action: 'type Name', uncertain: true }); }); it('keeps secret handles out of replayed summaries', async () => { - const { model, requests } = scriptedEvaluation((id, keys) => ({ + const { model, requests } = scriptedDecision((id, keys) => ({ choice: id === 'operation' ? 'done' : id === 'verdict' ? 'holds' : (keys[0] ?? ''), })); const fixture = context({ @@ -544,7 +587,7 @@ describe('context', () => { expect(JSON.stringify(requests.map((request) => request.state))).not.toContain('secret \\"password\\"'); }); it('sends only the last 10 actions', async () => { - const { model, requests } = scriptedEvaluation((id, keys, call) => ({ + const { model, requests } = scriptedDecision((id, keys, call) => ({ choice: id === 'operation' ? (call < 12 ? 'tap' : 'done') : id === 'verdict' ? 'holds' : (keys[0] ?? ''), })); const fixture = context({ tree: BUTTONS }); @@ -561,7 +604,7 @@ describe('context', () => { expect(actions).toEqual(Array.from({ length: 10 }, (_, index) => `tap Page ${index + 3} [a]`)); }); it('gives the completion check the inputs, typed values, and actions, never claims or secrets', async () => { - const { model, requests } = scriptedEvaluation((id, keys, call) => { + const { model, requests } = scriptedDecision((id, keys, call) => { if (id === 'operation') return { choice: call === 0 ? 'type' : 'done' }; if (id === 'verdict') return { choice: call === 2 ? 'inconclusive' : 'holds' }; return { choice: keys[0] ?? '' }; @@ -587,7 +630,7 @@ describe('context', () => { { id: 'greeting', role: 'status', name: 'Greeting', text: 'Welcome back, admin!' }, { id: 'out', role: 'button', name: 'Sign out' }, ] }; - const { model, requests } = scriptedEvaluation((id, keys) => ({ + const { model, requests } = scriptedDecision((id, keys) => ({ choice: id === 'operation' ? 'done' : id === 'verdict' ? 'holds' : (keys[0] ?? ''), })); const fixture = context({ tree }); @@ -601,7 +644,7 @@ describe('context', () => { { id: 'name', role: 'textbox', name: 'Name', value: 'Ada' }, { id: 'save', role: 'button', name: 'Save' }, ] }; - const { model, requests } = scriptedEvaluation((id, keys) => ({ + const { model, requests } = scriptedDecision((id, keys) => ({ choice: id === 'operation' ? 'done' : id === 'verdict' ? 'holds' : (keys[0] ?? ''), })); // No text model and no press verb: in the act loop the field offers no operation. @@ -618,7 +661,7 @@ describe('context', () => { { id: 'milk', role: 'checkbox', name: 'Buy milk', states: { checked: true } }, { id: 'dog', role: 'checkbox', name: 'Walk the dog', states: { checked: false } }, ] }; - const { model, requests } = scriptedEvaluation((id, keys, call) => ({ + const { model, requests } = scriptedDecision((id, keys, call) => ({ choice: id === 'operation' ? (call === 0 ? 'check' : 'done') : id === 'verdict' ? 'holds' : (keys[1] ?? ''), })); const fixture = context({ tree }); @@ -632,7 +675,7 @@ describe('context', () => { }); it('rejects an undeclared secret name', async () => { const login: ExecutorNode = { id: 'root', children: [{ id: 'pw', role: 'textbox', name: 'Password', inputPurpose: 'password' }] }; - const { model } = scriptedEvaluation((id, keys) => { + const { model } = scriptedDecision((id, keys) => { if (id === 'operation') return { choice: 'typeSecret' }; if (id === 'secret') return { choice: 'intruder' }; return { choice: keys[0] ?? '' }; diff --git a/packages/decision/tests/helpers.ts b/packages/decision/tests/helpers.ts index 7d6045e17..3e0ac48b9 100644 --- a/packages/decision/tests/helpers.ts +++ b/packages/decision/tests/helpers.ts @@ -1,5 +1,5 @@ -import type { Experimental_EvaluationModelV4 } from '@ai-sdk/provider'; import type { LanguageModel } from 'ai'; +import type { DecisionExecutorOptions } from '../src/index.ts'; import type { ExecutorActions, ExecutorModelCall, ExecutorNode, ExecutorObservation, JsonValue, StepExecutorContext, StepTurn } from 'e2e'; import { vi } from 'vitest'; /** One scripted answer: the choice, its distribution, and the reported confidence. */ @@ -10,46 +10,52 @@ export interface ScriptedAnswer { /** Omit `probabilities` from the answer entirely. */ readonly bare?: true; } -/** A recorded evaluate call: the state and the question map the executor sent. */ +/** A recorded decide call: the state and the question map the executor sent. */ export interface EvalRequest { readonly state: unknown; readonly questions: Record; } +/** The model shapes the executor accepts: a decision model, or a deprecated evaluation model. */ +type AcceptedModel = DecisionExecutorOptions['model']; +type DecideCall = Parameters['doDecide']>[0]; +type DecideResult = Awaited['doDecide']>>; /** - * A scripted evaluation model: records requests and answers every choice + * A scripted decision model: records requests and answers every choice * question through one resolver. Unspecified probabilities become a * unanimous distribution for the choice, which core validation accepts. + * `legacy` builds the deprecated evaluation model shape (`doEvaluate`). */ -export function scriptedEvaluation(resolve: (id: string, keys: string[], call: number) => ScriptedAnswer, options?: { supported?: readonly ('choice' | 'score' | 'boolean')[]; throws?: unknown }): { model: Experimental_EvaluationModelV4; requests: EvalRequest[] } { +export function scriptedDecision(resolve: (id: string, keys: string[], call: number) => ScriptedAnswer, options?: { supported?: readonly ('choice' | 'score' | 'boolean')[]; throws?: unknown; legacy?: true }): { model: AcceptedModel; requests: EvalRequest[] } { const requests: EvalRequest[] = []; - const model: Experimental_EvaluationModelV4 = { - specificationVersion: 'v4', + const respond = async (call: DecideCall): Promise => { + if (options?.throws !== undefined) throw options.throws; + const index = requests.length; + const questions: EvalRequest['questions'] = {}; + const answers: Record }> = {}; + const confidence: Record = {}; + for (const [id, question] of Object.entries(call.questions)) { + if (question.type !== 'choice') throw new Error('scripted model answers choice questions only'); + const criteria = question.criteria as Record; + questions[id] = { type: question.type, criteria }; + const keys = Object.keys(criteria); + const answer = resolve(id, keys, index); + if (answer.bare === true) { + answers[id] = { type: 'choice', choice: answer.choice } as (typeof answers)[string]; + } else { + answers[id] = { type: 'choice', choice: answer.choice, probabilities: answer.probabilities ?? Object.fromEntries(keys.map((key) => [key, key === answer.choice ? 1 : 0])) }; + } + if (answer.confidence !== undefined) confidence[id] = answer.confidence; + } + requests.push({ state: call.state, questions }); + return { answers, warnings: [], usage: { inputTokens: 10, outputTokens: 0 }, providerMetadata: { scripted: { confidence } }, response: { modelId: 'scripted-1' } }; + }; + const base = { + specificationVersion: 'v4' as const, provider: 'scripted', modelId: 'scripted-1', - supportedQuestionTypes: [...(options?.supported ?? ['choice'])], - doEvaluate: async (call) => { - if (options?.throws !== undefined) throw options.throws; - const index = requests.length; - const questions: EvalRequest['questions'] = {}; - const answers: Record }> = {}; - const confidence: Record = {}; - for (const [id, question] of Object.entries(call.questions)) { - if (question.type !== 'choice') throw new Error('scripted model answers choice questions only'); - const criteria = question.criteria as Record; - questions[id] = { type: question.type, criteria }; - const keys = Object.keys(criteria); - const answer = resolve(id, keys, index); - if (answer.bare === true) { - answers[id] = { type: 'choice', choice: answer.choice } as (typeof answers)[string]; - } else { - answers[id] = { type: 'choice', choice: answer.choice, probabilities: answer.probabilities ?? Object.fromEntries(keys.map((key) => [key, key === answer.choice ? 1 : 0])) }; - } - if (answer.confidence !== undefined) confidence[id] = answer.confidence; - } - requests.push({ state: call.state, questions }); - return { answers, warnings: [], usage: { inputTokens: 10, outputTokens: 0 }, providerMetadata: { scripted: { confidence } }, response: { modelId: 'scripted-1' } }; - }, + supportedQuestionTypes: [...(options?.supported ?? ['choice' as const])], }; + const model: AcceptedModel = options?.legacy === true ? { ...base, doEvaluate: respond } : { ...base, doDecide: respond }; return { model, requests }; } /** A scripted language model: returns queued texts as JSON through real generateText. */ diff --git a/packages/decision/tests/text.test.ts b/packages/decision/tests/text.test.ts index 2161a8a98..9954aa238 100644 --- a/packages/decision/tests/text.test.ts +++ b/packages/decision/tests/text.test.ts @@ -44,9 +44,9 @@ describe('fieldText', () => { }); it('maps a missing key to unavailable and bad output to invalid', async () => { const text = scriptedText(['Ada']); - const missing: Exclude = { ...text.model, doGenerate: async () => { throw new LoadAPIKeyError({ message: 'no key' }); } }; + const missing: Exclude = { ...text.model, doGenerate: async () => { throw new LoadAPIKeyError({ message: "OpenRouter API key is missing. Pass it using the 'apiKey' parameter or the OPENROUTER_API_KEY environment variable." }); } }; const malformed: Exclude = { ...text.model, doGenerate: async () => { throw new TypeValidationError({ value: 'nope', cause: new Error('bad') }); } }; - await expect(fieldText(context({ model: missing }).ctx, missing, input())).rejects.toMatchObject({ code: 'MODEL_UNAVAILABLE' }); + await expect(fieldText(context({ model: missing }).ctx, missing, input())).rejects.toMatchObject({ code: 'MODEL_UNAVAILABLE', message: 'Set OPENROUTER_API_KEY to the field-text model API key.' }); await expect(fieldText(context({ model: malformed }).ctx, malformed, input())).rejects.toMatchObject({ code: 'MODEL_OUTPUT_INVALID' }); }); it('honors abort', async () => { diff --git a/packages/decision/tests/types/options.ts b/packages/decision/tests/types/options.ts index 759c7e31c..2def7842e 100644 --- a/packages/decision/tests/types/options.ts +++ b/packages/decision/tests/types/options.ts @@ -1,19 +1,35 @@ -import type { Experimental_EvaluationModelV4 } from '@ai-sdk/provider'; +import { typeSafeAi } from '@ai-sdk/typesafe-ai'; import type { E2EConfig } from 'e2e'; import { decisionExecutor } from '@e2e-dev/decision'; -const model: Experimental_EvaluationModelV4 = { - specificationVersion: 'v4', - provider: 'test', - modelId: 'test-1', - supportedQuestionTypes: ['choice'], - doEvaluate: () => Promise.resolve({ answers: {}, warnings: [] }), -}; -const executor = decisionExecutor({ model }); +const executor = decisionExecutor({ model: typeSafeAi.decisionModel('jev-latest') }); const agents: E2EConfig['agents'] = { default: { executor } }; void agents; -// @ts-expect-error A decision executor requires an evaluation model, not a model id. +// The deprecated evaluation model shape stays accepted. +decisionExecutor({ model: typeSafeAi.evaluationModel('jev-latest') }); +decisionExecutor({ + model: { + specificationVersion: 'v4', + provider: 'test', + modelId: 'test-1', + supportedQuestionTypes: ['choice'], + doEvaluate: () => Promise.resolve({ answers: {}, warnings: [] }), + }, +}); +decisionExecutor({ + model: { + specificationVersion: 'v4', + provider: 'test', + modelId: 'test-1', + supportedQuestionTypes: ['choice'], + doDecide: () => Promise.resolve({ answers: {}, warnings: [] }), + }, +}); + +// @ts-expect-error A decision executor requires a decision model, not a model id. decisionExecutor({ model: 'jev-latest' }); // @ts-expect-error The text model is an instance, never an id string. -decisionExecutor({ model, textModel: 'openai/gpt-4o' }); +decisionExecutor({ model: typeSafeAi.decisionModel('jev-latest'), textModel: 'openai/gpt-4o' }); +// @ts-expect-error A model with neither doDecide nor doEvaluate is not a decision model. +decisionExecutor({ model: { specificationVersion: 'v4', provider: 'test', modelId: 'test-1', supportedQuestionTypes: ['choice'] } }); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 938fff167..cc9c4ebd0 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -282,8 +282,8 @@ importers: docs: devDependencies: '@ai-sdk/typesafe-ai': - specifier: 3.0.12 - version: 3.0.12(zod@4.6.1) + specifier: 3.0.13 + version: 3.0.13(zod@4.6.1) '@e2e-dev/decision': specifier: workspace:* version: link:../packages/decision @@ -295,16 +295,16 @@ importers: version: link:../packages/web '@openrouter/ai-sdk-provider': specifier: 3.1.0 - version: 3.1.0(ai@7.0.107(zod@4.6.1))(zod@4.6.1) + version: 3.1.0(ai@7.0.128(zod@4.6.1))(zod@4.6.1) '@types/node': specifier: 26.6.2 version: 26.6.2 agent-device: specifier: 0.21.21 - version: 0.21.21(ai@7.0.107(zod@4.6.1)) + version: 0.21.21(ai@7.0.128(zod@4.6.1)) ai: - specifier: 7.0.107 - version: 7.0.107(zod@4.6.1) + specifier: 7.0.128 + version: 7.0.128(zod@4.6.1) e2e: specifier: workspace:* version: link:../packages/e2e @@ -320,13 +320,13 @@ importers: packages/decision: dependencies: - '@ai-sdk/provider': - specifier: 4.0.21 - version: 4.0.21 zod: specifier: 4.6.1 version: 4.6.1 devDependencies: + '@ai-sdk/typesafe-ai': + specifier: 3.0.13 + version: 3.0.13(zod@4.6.1) '@e2e-dev/web': specifier: workspace:* version: link:../web @@ -334,8 +334,8 @@ importers: specifier: 26.6.2 version: 26.6.2 ai: - specifier: 7.0.107 - version: 7.0.107(zod@4.6.1) + specifier: 7.0.128 + version: 7.0.128(zod@4.6.1) ajv: specifier: 8.20.0 version: 8.20.0 @@ -563,6 +563,12 @@ packages: engines: {node: '>=22'} hasBin: true + '@ai-sdk/gateway@4.0.104': + resolution: {integrity: sha512-4lTyt8a4BMfVz4QGo+VO5odakmYMX5kq+xiRgdQGGY7o/jP2WSasp6LvIvYvt/lHzz1MfE+xKTOJhz7mMwjynA==} + engines: {node: '>=22'} + peerDependencies: + zod: ^3.25.76 || ^4.1.8 + '@ai-sdk/gateway@4.0.87': resolution: {integrity: sha512-6wJmX/D5OO7tvDJbtFdXdBuRt8AkCnW+FLK+W35UHe/tHgolAa4hclXtJcMSYnlTDy/t5DNRgdesVaoMXtwzow==} engines: {node: '>=22'} @@ -593,8 +599,8 @@ packages: peerDependencies: zod: ^3.25.76 || ^4.1.8 - '@ai-sdk/provider-utils@5.0.53': - resolution: {integrity: sha512-VVe6UDd0y0/B4TfauuKKzcppqvhTmSluwwUVT4WG2/8h7gFbyFm7PM4UGM+ZXe3/RI6bv+Vl1cw/7aiBoQqdxg==} + '@ai-sdk/provider-utils@5.0.54': + resolution: {integrity: sha512-amqDxFnw9+dpVrDACRDGmExUUNOH9C1D2bhg/DqWJnamEOieDTXBBXLHujSby/0MqHdAFksnpE8Pzfqx6ABMBw==} engines: {node: '>=22'} peerDependencies: zod: ^3.25.76 || ^4.1.8 @@ -607,8 +613,12 @@ packages: resolution: {integrity: sha512-UpbC9C1oht8dhfKPbVXSLRZS3DI8uk8n5v2uMjBPNIYGsC2kL045ywq7D8Hh9KgnH9rP5W/GxQdYtWScRouqlA==} engines: {node: '>=22'} - '@ai-sdk/typesafe-ai@3.0.12': - resolution: {integrity: sha512-XbwOzWckIDSBPJTFLp2JnmGhdkxeQ1s4fwRGMgo0aubdp66AhnD9/I0CAnC8nESJlns5oGfXICC4m/NTGO6/uA==} + '@ai-sdk/provider@4.0.22': + resolution: {integrity: sha512-1Jtmn36VNMOBo9CMUQrPqsR744qtqRloEiXpWCYS8ffPlhS2CrSNmoho5lPjp3R7EnkzBsGI6p51TA7fQJQZdA==} + engines: {node: '>=22'} + + '@ai-sdk/typesafe-ai@3.0.13': + resolution: {integrity: sha512-yCy1Qc3Bpnzir9HkXs1o7mPeWfMmo5BU5L7w2CeL3ghr6qC8/k7b4Y/C5tGXn3z6S5qoSfv7UiXntLkxLFVm+w==} engines: {node: '>=22'} peerDependencies: zod: ^3.25.76 || ^4.1.8 @@ -3329,6 +3339,12 @@ packages: peerDependencies: zod: ^3.25.76 || ^4.1.8 + ai@7.0.128: + resolution: {integrity: sha512-WMKN83DodYtH52ETPHq0jrSwj3wU0aHkhLHXMrwMRabC/aMpIqeWsXJgjkUwrOqkaKHWONdupk/2ttPRvAyFcg==} + engines: {node: '>=22'} + peerDependencies: + zod: ^3.25.76 || ^4.1.8 + ajv-draft-04@1.0.0: resolution: {integrity: sha512-mv00Te6nmYbRp5DCwclxtt7yV/joXJPGS7nM+97GdxvuttCOfgI3K4U25zboyeX0O+myI8ERluxQe5wljMmVIw==} peerDependencies: @@ -7624,6 +7640,13 @@ snapshots: '@hono/node-server': 1.19.17(hono@4.13.5) hono: 4.13.5 + '@ai-sdk/gateway@4.0.104(zod@4.6.1)': + dependencies: + '@ai-sdk/provider': 4.0.22 + '@ai-sdk/provider-utils': 5.0.54(zod@4.6.1) + '@vercel/oidc': 3.2.0 + zod: 4.6.1 + '@ai-sdk/gateway@4.0.87(zod@4.6.1)': dependencies: '@ai-sdk/provider': 4.0.17 @@ -7658,9 +7681,9 @@ snapshots: undici: 7.29.1 zod: 4.6.1 - '@ai-sdk/provider-utils@5.0.53(zod@4.6.1)': + '@ai-sdk/provider-utils@5.0.54(zod@4.6.1)': dependencies: - '@ai-sdk/provider': 4.0.21 + '@ai-sdk/provider': 4.0.22 '@standard-schema/spec': 1.1.0 '@workflow/serde': 4.1.0 eventsource-parser: 3.1.1 @@ -7675,10 +7698,14 @@ snapshots: dependencies: json-schema: 0.4.0 - '@ai-sdk/typesafe-ai@3.0.12(zod@4.6.1)': + '@ai-sdk/provider@4.0.22': + dependencies: + json-schema: 0.4.0 + + '@ai-sdk/typesafe-ai@3.0.13(zod@4.6.1)': dependencies: - '@ai-sdk/provider': 4.0.21 - '@ai-sdk/provider-utils': 5.0.53(zod@4.6.1) + '@ai-sdk/provider': 4.0.22 + '@ai-sdk/provider-utils': 5.0.54(zod@4.6.1) zod: 4.6.1 '@ai-sdk/xai@5.0.4(zod@4.6.1)': @@ -9752,6 +9779,11 @@ snapshots: ai: 7.0.107(zod@4.6.1) zod: 4.6.1 + '@openrouter/ai-sdk-provider@3.1.0(ai@7.0.128(zod@4.6.1))(zod@4.6.1)': + dependencies: + ai: 7.0.128(zod@4.6.1) + zod: 4.6.1 + '@oxc-parser/binding-android-arm-eabi@0.152.0': optional: true @@ -10679,6 +10711,10 @@ snapshots: optionalDependencies: ai: 7.0.107(zod@4.6.1) + agent-device@0.21.21(ai@7.0.128(zod@4.6.1)): + optionalDependencies: + ai: 7.0.128(zod@4.6.1) + aggregate-error@4.0.1: dependencies: clean-stack: 4.2.0 @@ -10691,6 +10727,13 @@ snapshots: '@ai-sdk/provider-utils': 5.0.45(zod@4.6.1) zod: 4.6.1 + ai@7.0.128(zod@4.6.1): + dependencies: + '@ai-sdk/gateway': 4.0.104(zod@4.6.1) + '@ai-sdk/provider': 4.0.22 + '@ai-sdk/provider-utils': 5.0.54(zod@4.6.1) + zod: 4.6.1 + ajv-draft-04@1.0.0(ajv@8.20.0): optionalDependencies: ajv: 8.20.0 diff --git a/skills/e2e/references/agent.md b/skills/e2e/references/agent.md index 4cdc535e0..082a87217 100644 --- a/skills/e2e/references/agent.md +++ b/skills/e2e/references/agent.md @@ -31,7 +31,7 @@ any provider. Decision models pick actions from choices the executor builds: install `@e2e-dev/decision` and put `decisionExecutor({ model: -typeSafeAi.evaluationModel('jev-latest'), textModel: openrouter('inception/mercury-2.5') })` +typeSafeAi.decisionModel('jev-latest'), textModel: openrouter('inception/mercury-2.5') })` under `executor`. Tests stay plain language with no params; the text model writes field values. See the shipped `docs/decision-models.mdx` or [the online guide](https://e2e.tester.army/docs/decision-models) for setup,