fix(decision): decide through experimental_decide and reject unsupported models at config load

.changeset/decision-executor.md

@@ -3,9 +3,11 @@
 ---
 
 Add `@e2e-dev/decision`, an executor that runs `agent.act` and `agent.assert`
-through a decision model instead of an LLM. It takes any AI SDK evaluation
-model with choice distributions and decides through `experimental_evaluate`,
-one call per action with the operation and target questions fanned out; a
-small language model writes field values when the choice is `type`. Tests
-stay plain natural language with no params. Probability and confidence
-gates are off by default.
+through a decision model instead of an LLM. It takes any AI SDK decision
+model with choice distributions (`evaluationModel()` instances too) and
+decides through `experimental_decide`, so it needs `ai` 7.0.128 or later. One
+call per action fans out the operation and target questions; a small
+language model writes field values when the choice is `type`. Tests stay
+plain natural language with no params. Probability and confidence gates are
+off by default. A model id string or a language model fails config load with
+`INVALID_CONFIG`.

AGENTS.md

@@ -87,7 +87,7 @@ suites that consume the built packages the way a user would.
   Expo's GraphQL API with `fetch`, and `@e2e-dev/mobile` is its only peer.
 - `packages/decision` — the published `@e2e-dev/decision` package: a
   `StepExecutor` (`decisionExecutor()`) that drives `agent.act` and
-  `agent.assert` through an AI SDK *evaluation* model answering `choice`
+  `agent.assert` through an AI SDK *decision* model answering `choice`
   questions with probability distributions, plus an optional small language
   model that writes field values when the decision model picks `type`.
   `minProbability` and `minConfidence` gate a chosen operation, target,

docs/decision-models.mdx

@@ -10,7 +10,7 @@ request: which operation, and which element for each operation that has
 more than one, chosen from what the executor builds out of the latest
 semantic tree. It never generates free text: when the choice is `type`, a
 small language model writes the field value. Both models are
-plain AI SDK models, so any evaluation provider that answers `choice`
+plain AI SDK models, so any decision model provider that answers `choice`
 questions with probability distributions (Jev) and any language-model
 provider works; the runner still authorizes every dispatched action.
 
@@ -32,7 +32,7 @@ bun add -d @e2e-dev/decision
 
 The decision model answers `choice` questions with probability
 distributions; the text model writes field values. Each installs as its
-own package:
+own package. Decision models need `ai` 7, version 7.0.128 or later:
 
 <CodeGroup>
 ```bash npm
@@ -60,14 +60,17 @@ export default {
   agents: {
     default: {
       executor: decisionExecutor({
-        model: typeSafeAi.evaluationModel('jev-latest'),
+        model: typeSafeAi.decisionModel('jev-latest'),
         textModel: openrouter('inception/mercury-2.5'),
       }),
     },
   },
 } satisfies E2EConfig;
 ```
 
+`evaluationModel()` instances still work. A model id string or a language
+model in `model` fails config load with `INVALID_CONFIG`.
+
 Tests stay plain natural language, with no params needed:
 
 ```ts title="tests/todos.e2e.ts"

docs/examples/models/e2e.decision.config.ts

@@ -9,7 +9,7 @@ export default {
   agents: {
     default: {
       executor: decisionExecutor({
-        model: typeSafeAi.evaluationModel('jev-latest'),
+        model: typeSafeAi.decisionModel('jev-latest'),
         textModel: openrouter('inception/mercury-2.5'),
       }),
     },

docs/package.json

@@ -9,14 +9,14 @@
     "typecheck": "tsc --noEmit --project examples/tsconfig.json"
   },
   "devDependencies": {
-    "@ai-sdk/typesafe-ai": "3.0.12",
+    "@ai-sdk/typesafe-ai": "3.0.13",
     "@e2e-dev/decision": "workspace:*",
     "@e2e-dev/mobile": "workspace:*",
     "@e2e-dev/web": "workspace:*",
     "@openrouter/ai-sdk-provider": "3.1.0",
     "@types/node": "26.6.2",
     "agent-device": "0.21.21",
-    "ai": "7.0.107",
+    "ai": "7.0.128",
     "e2e": "workspace:*",
     "mint": "4.2.909",
     "typescript": "7.0.2",

docs/reference/errors.mdx

@@ -84,8 +84,8 @@ These appear in terminal output and in the report as plain string codes.
 | --- | --- | --- | ---: |
 | `CONFIG_NOT_FOUND` | `--config <path>` names a file that does not exist, or no `e2e.config.ts` or `e2e.config.mts` exists from the working directory up to the repository root | configuration | 2 |
 | `CONFIG_AMBIGUOUS` | Both `e2e.config.ts` and `e2e.config.mts` in one directory | configuration | 2 |
-| `CONFIG_LOAD_FAILED` | The config module failed to import: a package that is declared but not installed, an undeclared package, a wrong subpath, a removed export, or any other throw while it evaluates. The message names the fix. A config error raised while it evaluates keeps its own code: an option a factory the config calls (`web()`, `mobile()`, `defineEngine()`) refuses, or a `secrets.get()` handle turned into a string, is `INVALID_CONFIG` | configuration | 2 |
-| `INVALID_CONFIG` | A factory the config module calls (`web()`, `mobile()`, `defineEngine()`) refused its options, with the factory's own message; or an unknown key, out-of-range value, invalid enum, missing `targets`, a target key outside `{ name, platform, engine, app, trace, video }`, a target `app` key that is unknown or malformed, `app` on a target without an engine, a `web()` target without `app.url` or with a device field, a `mobile()` target without `app.bundleId` or `app.appPath`, with `app.url` (not supported yet), a link in `bundleId`, or an unknown permission name or state (the message names the nearest name), an unknown key in a `web()` or `mobile()` option, including `web({ connect })` and `web({ basicAuth })` (the message names the nearest one), a `trace` or `video` that is not a recording mode, the removed artifact kinds list (`artifacts: [...]` or `artifacts.kinds`) and `artifacts.trace` block (the message names the `trace` mode they meant), the removed `artifacts.video` block or `video` artifact kind (the message names the `video` option), the removed `specVersion` key, a `secrets.get()` in an engine option that names no `secrets` entry (a credential's password is not one), a `secrets.get()` reference turned into a string while the config evaluates (a template literal, `String()`) or given where a string is needed (an `app.command` `args` entry or `env` value; only an engine option that declares secrets takes one), an `app.command` `args` entry or `env` value that is not a string (an `undefined` `env` value is dropped instead), an unknown key in `app.command` (the message names the nearest one), an `engine` that is not a `defineEngine` handle, `json` reporter combined with `list`, duplicate or malformed target name, a secret or credential password shorter than 6 characters (code points), two credentials or two secrets whose names map to one `E2E_USER_*` or `E2E_SECRET_*` override variable (`api-key` and `api_key`), oversized `agents.<name>.context`, an `expect.poll` `timeout` or `interval` that is not a finite number of milliseconds, a `tests` value that is not a glob or a list of globs, a `tests` list of only `!` exclusions, the removed top-level `limits` key, an agents entry that is a bare `StepExecutor` (the message names `{ executor }`) or a model instance (the message names `{ model }`), an unknown agent key (the message names the nearest one), the removed agent keys `timeout` and `maxTurns` (the message names `judgmentTimeout` and `maxModelCalls`), `system` or `tools` beside `executor`, a `tools` entry `defineTool` did not build or that takes a built-in tool name (a grammar verb, `complete_step`, `locate`, `start_recording`, `stop_recording`, or `report_finding`), `createToolLoopExecutor({ maxTurns })`, `reuseExisting` on an app whose `app.url` asks for a free port, or a port-0 `app.url` with no `app.command` to serve it | configuration | 2 |
+| `CONFIG_LOAD_FAILED` | The config module failed to import: a package that is declared but not installed, an undeclared package, a wrong subpath, a removed export, or any other throw while it evaluates. The message names the fix. A config error raised while it evaluates keeps its own code: an option a factory the config calls (`web()`, `mobile()`, `defineEngine()`, `decisionExecutor()`) refuses, or a `secrets.get()` handle turned into a string, is `INVALID_CONFIG` | configuration | 2 |
+| `INVALID_CONFIG` | A factory the config module calls (`web()`, `mobile()`, `defineEngine()`, `decisionExecutor()`) refused its options, with the factory's own message; or an unknown key, out-of-range value, invalid enum, missing `targets`, a target key outside `{ name, platform, engine, app, trace, video }`, a target `app` key that is unknown or malformed, `app` on a target without an engine, a `web()` target without `app.url` or with a device field, a `mobile()` target without `app.bundleId` or `app.appPath`, with `app.url` (not supported yet), a link in `bundleId`, or an unknown permission name or state (the message names the nearest name), an unknown key in a `web()` or `mobile()` option, including `web({ connect })` and `web({ basicAuth })` (the message names the nearest one), a `trace` or `video` that is not a recording mode, the removed artifact kinds list (`artifacts: [...]` or `artifacts.kinds`) and `artifacts.trace` block (the message names the `trace` mode they meant), the removed `artifacts.video` block or `video` artifact kind (the message names the `video` option), the removed `specVersion` key, a `secrets.get()` in an engine option that names no `secrets` entry (a credential's password is not one), a `secrets.get()` reference turned into a string while the config evaluates (a template literal, `String()`) or given where a string is needed (an `app.command` `args` entry or `env` value; only an engine option that declares secrets takes one), an `app.command` `args` entry or `env` value that is not a string (an `undefined` `env` value is dropped instead), an unknown key in `app.command` (the message names the nearest one), an `engine` that is not a `defineEngine` handle, `json` reporter combined with `list`, duplicate or malformed target name, a secret or credential password shorter than 6 characters (code points), two credentials or two secrets whose names map to one `E2E_USER_*` or `E2E_SECRET_*` override variable (`api-key` and `api_key`), oversized `agents.<name>.context`, an `expect.poll` `timeout` or `interval` that is not a finite number of milliseconds, a `tests` value that is not a glob or a list of globs, a `tests` list of only `!` exclusions, the removed top-level `limits` key, an agents entry that is a bare `StepExecutor` (the message names `{ executor }`) or a model instance (the message names `{ model }`), an unknown agent key (the message names the nearest one), the removed agent keys `timeout` and `maxTurns` (the message names `judgmentTimeout` and `maxModelCalls`), `system` or `tools` beside `executor`, a `tools` entry `defineTool` did not build or that takes a built-in tool name (a grammar verb, `complete_step`, `locate`, `start_recording`, `stop_recording`, or `report_finding`), `createToolLoopExecutor({ maxTurns })`, `reuseExisting` on an app whose `app.url` asks for a free port, or a port-0 `app.url` with no `app.command` to serve it | configuration | 2 |
 | `CONFIG_NOT_DETERMINISTIC` | A worker re-resolved the config module and got a different digest than the runner | configuration | 2 |
 | `APP_URL_REQUIRED` | A navigation or relative URL on a target that declares no `app.url`, or an `app.command` with no `app.url` or `app.readyUrl` to poll | configuration | 2 |
 | `INVALID_APP_URL` | Base URL with userinfo, a query, a fragment, a non-HTTP scheme, plain HTTP on a non-loopback host, port 0 on a host other than a literal loopback address (`127.0.0.1`, `[::1]`), a loopback address the run could not bind a free port on, or unparseable | configuration | 2 |

packages/decision/README.md

@@ -1,9 +1,10 @@
 # @e2e-dev/decision
 
 Run `agent.act` and `agent.assert` through a decision model instead of an LLM:
-any AI SDK evaluation model that answers `choice` questions with probability
+any AI SDK decision model that answers `choice` questions with probability
 distributions (e.g. TypeSafe Jev), plus a small language model that writes
-field values when the decision model picks `type`.
+field values when the decision model picks `type`. Needs `ai` 7, version 7.0.128 or
+later.
 
 ```ts
 import type { E2EConfig } from 'e2e';
@@ -15,7 +16,7 @@ export default {
   agents: {
     default: {
       executor: decisionExecutor({
-        model: typeSafeAi.evaluationModel('jev-latest'),
+        model: typeSafeAi.decisionModel('jev-latest'),
         textModel: openrouter('inception/mercury-2.5'),
       }),
     },
@@ -24,7 +25,7 @@ export default {
 ```
 
 Set `agents.default.executor` in the e2e config. Tests stay plain natural
-language, with no params needed. One `experimental_evaluate` call per action
+language, with no params needed. One `experimental_decide` call per action
 asks the operation plus one target question per operation; operations with a
 single target dispatch without a question. The text model is also the agent's
 judgment tier (`waitFor`, `extract`); without a text model or a configured agent

packages/decision/package.json

@@ -31,17 +31,17 @@
     "test": "vitest run"
   },
   "dependencies": {
-    "@ai-sdk/provider": "4.0.21",
     "zod": "4.6.1"
   },
   "peerDependencies": {
-    "ai": "^7.0.0",
+    "ai": "^7.0.128",
     "e2e": ">=0.15.2 <1"
   },
   "devDependencies": {
+    "@ai-sdk/typesafe-ai": "3.0.13",
     "@e2e-dev/web": "workspace:*",
     "@types/node": "26.6.2",
-    "ai": "7.0.107",
+    "ai": "7.0.128",
     "ajv": "8.20.0",
     "ajv-formats": "3.0.1",
     "e2e": "workspace:*",

packages/decision/src/api-key.ts

@@ -0,0 +1,13 @@
+import { AgentError } from 'e2e/agent';
+/**
+ * MODEL_UNAVAILABLE for a missing API key, naming the environment variable
+ * from the SDK's own message when it names one. Only that identifier is
+ * kept, never the rest of the text.
+ */
+export function missingKey(role: 'decision' | 'field-text', error: Error): AgentError {
+  const variable = /\bthe ([A-Z][A-Z0-9_]*) environment variable\b/.exec(error.message)?.[1];
+  const message = variable === undefined
+    ? `Set the ${role} model API key.`
+    : `Set ${variable} to the ${role} model API key.`;
+  return new AgentError('MODEL_UNAVAILABLE', message);
+}

packages/decision/src/evaluate.ts → packages/decision/src/decide.ts

@@ -1,15 +1,29 @@
-import type { Experimental_EvaluationModelV4 } from '@ai-sdk/provider';
 import type { StepExecutorContext } from 'e2e';
 import { AgentError, isAgentError } from 'e2e/agent';
+import { ConfigurationError } from 'e2e/engine';
+import * as ai from 'ai';
 import {
-  experimental_evaluate,
   InvalidArgumentError,
   InvalidResponseDataError,
   JSONParseError,
   LoadAPIKeyError,
   TypeValidationError,
 } from 'ai';
+import { missingKey } from './api-key.ts';
 import type { DecisionRequest } from './questions.ts';
+import type { DecisionExecutorOptions } from './types.ts';
+/**
+ * The SDK's decide call, or INVALID_CONFIG when the installed `ai` predates
+ * it. Read off the namespace so an older `ai` still loads this module and
+ * the user gets the version to install instead of a link error.
+ */
+export function requireDecide(): typeof ai.experimental_decide {
+  const found = (ai as Partial<typeof ai>).experimental_decide;
+  if (typeof found !== 'function') {
+    throw new ConfigurationError('INVALID_CONFIG', 'decisionExecutor() needs ai 7.0.128 or later; update the ai package');
+  }
+  return found;
+}
 /** One gated answer: the chosen option, its probability, and the provider confidence. */
 export interface Decision {
   readonly choice: string;
@@ -23,13 +37,13 @@ interface RawAnswer {
   readonly probabilities?: Record<string, number>;
 }
 /**
- * Makes one evaluate call under the step budget and returns the answers.
- * This is the only file that touches the SDK evaluate API, so a rename
+ * Makes one decide call under the step budget and returns the answers.
+ * This is the only file that touches the SDK decide API, so a rename
  * there stays a one-file change.
  */
-export async function evaluate(
+export async function decide(
   ctx: StepExecutorContext,
-  model: Experimental_EvaluationModelV4,
+  model: DecisionExecutorOptions['model'],
   request: DecisionRequest,
 ): Promise<Record<string, Decision>> {
   ctx.signal.throwIfAborted();
@@ -42,8 +56,8 @@ export async function evaluate(
   let answers: Record<string, RawAnswer>;
   try {
     const call = { model, state: request.state, questions: request.questions };
-    const result = await experimental_evaluate({
-      ...(call as unknown as Parameters<typeof experimental_evaluate>[0]),
+    const result = await requireDecide()({
+      ...(call as unknown as Parameters<typeof ai.experimental_decide>[0]),
       maxRetries: 0,
       abortSignal: ctx.signal,
     });
@@ -54,7 +68,7 @@ export async function evaluate(
     providerMetadata = result.providerMetadata as Record<string, Record<string, unknown>> | undefined;
     answers = result.answers as Record<string, RawAnswer>;
   } catch (error) {
-    throw evaluateError(error, ctx.signal);
+    throw decideError(error, ctx.signal);
   } finally {
     ctx.budgets.recordModelCall({
       provider: model.provider,
@@ -94,10 +108,10 @@ function reportedConfidence(metadata: Record<string, Record<string, unknown>> |
   return 0;
 }
 /** Maps SDK failures onto the runner error codes without echoing provider text. */
-function evaluateError(error: unknown, signal: AbortSignal): unknown {
+function decideError(error: unknown, signal: AbortSignal): unknown {
   signal.throwIfAborted();
   if (isAgentError(error)) return error;
-  if (LoadAPIKeyError.isInstance(error)) return new AgentError('MODEL_UNAVAILABLE', 'Set the evaluation model API key.');
+  if (LoadAPIKeyError.isInstance(error)) return missingKey('decision', error);
   if (InvalidArgumentError.isInstance(error)) return error;
   if (InvalidResponseDataError.isInstance(error) || TypeValidationError.isInstance(error) || JSONParseError.isInstance(error)) {
     return new AgentError('MODEL_OUTPUT_INVALID', 'The decision model returned an invalid answer.');