diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0b066b6..93151b1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -21,10 +21,14 @@ jobs: - run: npm test # Live end-to-end navigation only when a TYPESAFE_API_KEY secret is set. - name: Install Chromium for e2e + id: install-chromium if: env.TYPESAFE_API_KEY != '' env: TYPESAFE_API_KEY: ${{ secrets.TYPESAFE_API_KEY }} run: npx playwright install --with-deps chromium + - name: Transport integration (optional) + if: steps.install-chromium.outcome == 'success' + run: node --test test/transports.test.mjs - name: End-to-end (optional) if: env.TYPESAFE_API_KEY != '' env: diff --git a/CHANGELOG.md b/CHANGELOG.md index f9e868f..e347cb1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,14 @@ # Changelog +## Unreleased + +- Transport drivers: the four judgment transports (TypeSafe, OpenRouter, Cloudflare, Vercel) are now run-bound drivers behind one registry. Typing providers are unchanged. +- An unknown `JEV_PROVIDER` now errors instead of silently falling through to auto-detection; the no-provider diagnostic names all four credential sets. +- Every answer is validated at a shared boundary before tokens are credited or an action executes: missing or malformed answers error the run, and transport errors from built-in providers no longer include raw response bodies. +- Removed the select-option fallback to the first option; a malformed option judgment errors the run without selecting anything. +- Library callers can inject a transport with `NavigateOptions.transport`; results report its name and effective model. `est_cost_usd` stays a Jev-token estimate. + + ## 0.5.0 - Cloudflare challenges and hard blocks are now detected and named: the run stops with status `blocked` plus `bot_protection` evidence and guidance, instead of burning steps against a wall. Challenges get a short window to clear first. diff --git a/README.md b/README.md index 67aeb04..07c329d 100644 --- a/README.md +++ b/README.md @@ -150,6 +150,30 @@ console.log(result.status, result.final_url); console.log(result.page.content); ``` +### Judgment transports in the library + +The built-in Jev transports are TypeSafe, OpenRouter, Cloudflare, and Vercel, selected in that order from configured credentials. `JEV_PROVIDER` forces one of them; an unknown name or missing credential is an error. This is separate from the typing model. Library callers can provide a transport instead, bypassing judgment provider detection without changing typing configuration: + +```ts +import { navigate, type JevTransport } from "@jkudish/jev-browser"; + +const transport: JevTransport = { + name: "my-gateway", // reported as jev_provider + async ask({ state, questions, model, signal }) { + const response = await myGateway.decide({ state, questions, model, signal }); + return { + answers: response.answers, + usage: { input_tokens: response.inputTokens, output_tokens: response.outputTokens }, + model: response.effectiveModel, + }; + }, +}; + +const result = await navigate({ task: "Find the price", startUrl: "https://example.com", transport }); +``` + +`ask` receives the page state, named questions, requested model, and run abort signal. Return an answer for every requested ID in the matching Jev shape, plus nonnegative integer token counts and the effective model. Choice distributions must cover exactly the offered criteria, sum to about 1, and select a maximum; Noul values must be in [0,1]. Invalid answers stop the run before the action executes. `est_cost_usd` stays a Jev-token estimate, not verified billing for injected carriers. + ## Password fill (logins) The agent can fill native password fields without the password ever reaching a model. The value arrives through one of three channels, lives in memory for a single run, and is scrubbed from every state, trace, error, URL, and payload the run produces. Video recording is refused on credential runs and the final screenshot is suppressed once a fill is attempted (on injected pages, which may already show the value, from the start of the run). A fill never submits: no Enter, no click. @@ -409,6 +433,7 @@ With no provider at all, or when the typing model fails or returns empty text, t | `TYPESAFE_API_KEY` | none | TypeSafe direct. Default provider when set. | | `OPENROUTER_API_KEY` | none | Powers both the Jev judgments (when `TYPESAFE_API_KEY` is absent) and, optionally, the typing model. One key runs everything. | | `CLOUDFLARE_API_TOKEN` + `CLOUDFLARE_ACCOUNT_ID` | none | Cloudflare Workers AI for the Jev judgments; used when no other provider key is present. | +| `AI_GATEWAY_API_KEY` | none | Vercel AI Gateway for Jev judgments after the other providers. | | `JEV_PROVIDER` | `auto` | Force `typesafe`, `openrouter`, `cloudflare`, or `vercel` for the Jev calls instead of auto-detection. Judgment transport only; typing is configured separately with `JEV_BROWSER_TYPE_*`. | | `JEV_BROWSER_MODEL` | `jev-latest` | Pin a Jev version, or `typesafe/jev-1.13` on OpenRouter. | | `JEV_BROWSER_TYPE_*` | see above | Typing provider, model, and endpoint. | diff --git a/package.json b/package.json index 50be0a6..d5284d9 100644 --- a/package.json +++ b/package.json @@ -39,7 +39,7 @@ "build": "tsc", "prepare": "npm run build", "typecheck": "tsc --noEmit", - "test": "node --test test/unit.test.mjs", + "test": "node --test test/unit.test.mjs test/transports.test.mjs", "test:e2e": "node --test test/e2e.test.mjs", "postinstall": "node scripts/ensure-chromium.mjs" }, diff --git a/src/library.ts b/src/library.ts index e6a6607..070098e 100644 --- a/src/library.ts +++ b/src/library.ts @@ -4,3 +4,4 @@ export { navigate } from "./navigate.js"; export type { NavigateOptions, StepRecord, ConsoleEvent, JevUsage } from "./navigate.js"; export type { TypingGenerator, TypingTextResult } from "./navigate.js"; export type { TypingWarning, TypingWarningCode, TypingSelection } from "./lib.js"; +export type { JevTransport, JevTransportInput, JevTransportReply, AskResult } from "./provider.js"; diff --git a/src/navigate.ts b/src/navigate.ts index 5e50181..3744642 100644 --- a/src/navigate.ts +++ b/src/navigate.ts @@ -33,6 +33,7 @@ import { } from "./lib.js"; import { selectOptionQuestion, stepQuestions } from "./questions.js"; import { assertNoPlaywrightDebug, makeRedactor, parseTrustedOrigin, validateSecretBuffer, type Redactor } from "./password.js"; +import { askJev as askProvider, InvalidJevAnswer, resolveTransport, type JevTransport, type JevAnswer } from "./provider.js"; const MAX_CONSOLE_EVENTS = 200; const STATE_EXCERPT_CHARS = 1_500; @@ -46,6 +47,8 @@ export interface NavigateOptions { startUrl?: string; /** Reuse an existing Playwright page instead of launching a new browser. */ page?: Page; + /** Override judgment transport for this run, independent of JEV_PROVIDER and credentials. */ + transport?: JevTransport; maxSteps?: number; maxSeconds?: number; allowTyping?: boolean; @@ -108,10 +111,9 @@ const DEFAULT_CAPS: Record = { const turndown = new TurndownService({ headingStyle: "atx", codeBlockStyle: "fenced" }); turndown.use(gfm.gfm); -import { askJev as askProvider, type JevProvider } from "./provider.js"; - interface RunBudget { usage: JevUsage; + transport: JevTransport; signal: AbortSignal; deadlineAt: number; // performance.now() milliseconds // Per-run model/provider state: resolved inside navigate() and mutated only @@ -119,11 +121,11 @@ interface RunBudget { // provider and a failed run cannot inherit values from a previous one. requestedModel: string; model: string; // model reported by the most recent Jev call - provider: JevProvider | null; + provider: string | null; } async function askJev(budget: RunBudget, state: unknown, questions: Record) { - const result = await askProvider(state, questions, budget.requestedModel, budget.signal); + const result = await askProvider(budget.transport, { state, questions, model: budget.requestedModel, signal: budget.signal }); budget.provider = result.provider; budget.model = result.model; budget.usage.jev_calls += 1; @@ -496,6 +498,7 @@ export async function navigate(options: NavigateOptions, externalSignal?: AbortS // another provider. Runs with typing disabled ignore typing config at all, // so a broken config can always be worked around with allowTyping: false. const typingGenerator = allowTyping ? createTypingGenerator() : null; + const transport = options.transport ?? resolveTransport(); // One abort source per run: the wall-clock deadline, optionally composed // with caller cancellation (the MCP layer forwards its signal). @@ -510,6 +513,7 @@ export async function navigate(options: NavigateOptions, externalSignal?: AbortS const requestedModel = process.env.JEV_BROWSER_MODEL ?? "jev-latest"; const budget: RunBudget = { usage: { jev_calls: 0, input_tokens: 0, output_tokens: 0, est_cost_usd: 0 }, + transport, signal: controller.signal, deadlineAt, requestedModel, @@ -759,7 +763,7 @@ export async function navigate(options: NavigateOptions, externalSignal?: AbortS history, }; const answers = await askJev(budget, state, stepQuestions(buildCriteria(elements))); - const actionAnswer = answers.action; + const actionAnswer = answers.action as Extract; const proposed: string = actionAnswer.choice; const probabilities: Record = actionAnswer.probabilities ?? {}; const base = { @@ -768,8 +772,8 @@ export async function navigate(options: NavigateOptions, externalSignal?: AbortS proposed_action: proposed, confidence: actionAnswer.confidence ?? null, top_probability: probabilities[proposed] ?? null, - goal_done: answers.goal_done.noul, - stuck: answers.stuck.noul, + goal_done: (answers.goal_done as Extract).noul, + stuck: (answers.stuck as Extract).noul, }; // Stop gates run BEFORE execution: a watcher that fires on the current @@ -779,12 +783,12 @@ export async function navigate(options: NavigateOptions, externalSignal?: AbortS status = "done"; break; } - if (answers.goal_done.noul > 0.85) { + if ((answers.goal_done as Extract).noul > 0.85) { steps.push({ ...base, executed_action: null, detail: "goal watcher fired; proposed action not executed", outcome: "goal watcher fired before acting" }); status = "goal_achieved"; break; } - if (answers.stuck.noul > 0.85 && step > 2) { + if ((answers.stuck as Extract).noul > 0.85 && step > 2) { steps.push({ ...base, executed_action: null, detail: "stuck watcher fired; proposed action not executed", outcome: "stuck watcher fired before acting" }); status = "stuck"; break; @@ -904,9 +908,13 @@ export async function navigate(options: NavigateOptions, externalSignal?: AbortS budget, { task: safeTask, page: { url: R(observables.url), title: R(observables.title) }, dropdown: element.description, options: opts.map((o) => o.label) }, { option: selectOptionQuestion(element.description, opts.map((o) => o.label)) }, - ); - const pickedIndex = Number((optionAnswer.option.choice as string).slice(1)); - const opt = opts[pickedIndex] ?? opts[0]; + ).catch((error) => { + if (budget.signal.aborted) throw budget.signal.reason; + if (error instanceof InvalidJevAnswer) throw error; + throw new InvalidJevAnswer(`Jev provider ${budget.transport.name} question option: request failed`); + }); + const pickedIndex = Number((optionAnswer.option as Extract).choice.slice(1)); + const opt = opts[pickedIndex]; await page.selectOption(selectorFor(element), { index: opt.i }, { timeout: bounded(4_000) }); detail = `selected "${opt.label}"`; } @@ -958,6 +966,7 @@ export async function navigate(options: NavigateOptions, externalSignal?: AbortS } } catch (error) { if (controller.signal.aborted) throw error; // deadline/cancellation propagates + if (error instanceof InvalidJevAnswer) throw error; // malformed second-stage answer is a run error, never an action fallback actionError = R((error as Error).message).slice(0, 160); } diff --git a/src/provider.ts b/src/provider.ts index 328d4be..fe11c37 100644 --- a/src/provider.ts +++ b/src/provider.ts @@ -1,184 +1,135 @@ -// Jev transport: TypeSafe direct (default), OpenRouter Decisions, or -// Cloudflare Workers AI. All speak the {state, questions} / answers contract; -// URL, auth, and model slugs differ. Proxies add hops, so direct TypeSafe -// remains the recommended default. +import { typesafe } from "./transports/typesafe.js"; +import { openrouter } from "./transports/openrouter.js"; +import { cloudflare } from "./transports/cloudflare.js"; +import { vercel } from "./transports/vercel.js"; -import { experimental_evaluate } from "ai"; -import { TypeSafeClient } from "@typesafe-ai/sdk"; - -export type JevProvider = "typesafe" | "openrouter" | "cloudflare" | "vercel"; +export interface JevTransportInput { + state: unknown; + questions: Record; + model: string; + signal: AbortSignal; +} -export interface AskResult { - answers: Record; +export interface JevTransportReply { + answers: unknown; usage: { input_tokens: number; output_tokens: number }; - provider: JevProvider; model: string; } -const X_TITLE = "jev-browser"; -const REFERER = "https://github.com/jkudish/jev-browser"; +export interface JevTransport { + readonly name: string; + ask(input: JevTransportInput): Promise; +} -let typesafeClient: TypeSafeClient | null = null; +export interface BuiltinDriver { + readonly name: "typesafe" | "openrouter" | "cloudflare" | "vercel"; + isConfigured(env: NodeJS.ProcessEnv): boolean; + assertConfigured(env: NodeJS.ProcessEnv): void; + create(env: NodeJS.ProcessEnv): JevTransport; +} -function resolve(env: NodeJS.ProcessEnv): JevProvider { - const explicit = (env.JEV_PROVIDER ?? "auto").toLowerCase(); - const hasTypesafe = Boolean(env.TYPESAFE_API_KEY); - const hasOpenRouter = /^sk-or-/.test(env.OPENROUTER_API_KEY ?? ""); - const cfToken = env.JEV_CLOUDFLARE_API_TOKEN || env.CLOUDFLARE_API_TOKEN; - const hasCloudflare = Boolean(cfToken && env.CLOUDFLARE_ACCOUNT_ID); +export type JevAnswer = + | { type: "noul"; noul: number } + | { type: "choice"; choice: string; probabilities: Record; confidence: number | null } + | { type: "score"; score: number; probabilities: Record; confidence: number | null }; - if (explicit === "typesafe") { - if (!hasTypesafe) throw new Error("JEV_PROVIDER=typesafe but TYPESAFE_API_KEY is not set."); - return "typesafe"; - } - if (explicit === "openrouter") { - if (!hasOpenRouter) throw new Error("JEV_PROVIDER=openrouter but OPENROUTER_API_KEY is not set or not an sk-or- key."); - return "openrouter"; - } - if (explicit === "vercel") { - if (!env.AI_GATEWAY_API_KEY) throw new Error("JEV_PROVIDER=vercel but AI_GATEWAY_API_KEY is not set."); - return "vercel"; - } - if (explicit === "cloudflare") { - if (!hasCloudflare) throw new Error("JEV_PROVIDER=cloudflare but a Cloudflare API token (CLOUDFLARE_API_TOKEN or JEV_CLOUDFLARE_API_TOKEN) and CLOUDFLARE_ACCOUNT_ID are not both set."); - return "cloudflare"; - } - if (hasTypesafe) return "typesafe"; - if (hasOpenRouter) return "openrouter"; - if (hasCloudflare) return "cloudflare"; - if (env.AI_GATEWAY_API_KEY) return "vercel"; - throw new Error( - "No TYPESAFE_API_KEY, OPENROUTER_API_KEY (sk-or-), or Cloudflare token + CLOUDFLARE_ACCOUNT_ID found. Set one, or JEV_PROVIDER to choose explicitly.", - ); +export interface AskResult { + answers: Record; + usage: { input_tokens: number; output_tokens: number }; + provider: string; + model: string; } -export async function askJev( - state: unknown, - questions: Record, - model: string, - signal?: AbortSignal, -): Promise { - const provider = resolve(process.env); +/** Internal marker so a failed judgment cannot be mistaken for a page action error. */ +export class InvalidJevAnswer extends Error {} - if (provider === "typesafe") { - typesafeClient ??= new TypeSafeClient( - process.env.TYPESAFE_BASE_URL ? { baseURL: process.env.TYPESAFE_BASE_URL } : undefined, - ); - const response = await ( - typesafeClient.systemOne as unknown as ( - payload: { state: unknown; questions: Record; model?: string }, - options?: { signal?: AbortSignal }, - ) => Promise - )({ state, questions, model }, { signal }); - return { - answers: response.answers, - usage: { input_tokens: response.usage?.input_tokens ?? 0, output_tokens: response.usage?.output_tokens ?? 0 }, - provider, - model, - }; - } +const drivers: readonly BuiltinDriver[] = [typesafe, openrouter, cloudflare, vercel]; - if (provider === "openrouter") { - // OpenRouter has no redirecting "latest" slug; map it to the current - // release. Pin exact versions with the model env var when that matters. - const OPENROUTER_LATEST = "jev-1.13"; - const effective = model === "jev-latest" ? OPENROUTER_LATEST : model; - const slug = effective.startsWith("typesafe/") ? effective : `typesafe/${effective}`; - const response = await fetch("https://openrouter.ai/api/alpha/decisions", { - method: "POST", - headers: { - Authorization: `Bearer ${process.env.OPENROUTER_API_KEY}`, - "Content-Type": "application/json", - "HTTP-Referer": REFERER, - "X-Title": X_TITLE, - "X-OpenRouter-Title": X_TITLE, - }, - body: JSON.stringify({ model: slug, state, questions }), - signal, - }); - if (!response.ok) { - const body = await response.text().catch(() => ""); - throw new Error(`OpenRouter decisions API ${response.status}: ${body.slice(0, 200)}`); +export function resolveTransport(env: NodeJS.ProcessEnv = process.env): JevTransport { + const explicit = (env.JEV_PROVIDER ?? "auto").toLowerCase(); + if (explicit !== "auto") { + const driver = drivers.find((candidate) => candidate.name === explicit); + if (!driver) throw new Error("Unknown JEV_PROVIDER; choose typesafe, openrouter, cloudflare, vercel, or auto."); + try { + driver.assertConfigured(env); + } catch (error) { + throw new Error(`JEV_PROVIDER=${driver.name} but ${(error as Error).message}`); } - const body = await response.json(); - return { - answers: body.answers ?? {}, - // The decisions endpoint does not document a usage block; tolerate absence. - usage: { input_tokens: body.usage?.input_tokens ?? 0, output_tokens: body.usage?.output_tokens ?? 0 }, - provider, - model: slug, - }; + return driver.create(env); + } + const driver = drivers.find((candidate) => candidate.isConfigured(env)); + if (!driver) { + throw new Error("No TYPESAFE_API_KEY, OPENROUTER_API_KEY (sk-or-), Cloudflare token (CLOUDFLARE_API_TOKEN or JEV_CLOUDFLARE_API_TOKEN) + CLOUDFLARE_ACCOUNT_ID, or AI_GATEWAY_API_KEY found. Set one, or JEV_PROVIDER to choose explicitly."); } + return driver.create(env); +} - if (provider === "vercel") { - // Vercel AI Gateway exposes Jev through the AI SDK's experimental evaluate - // API: "noul" questions become "boolean", answers return as probabilities, - // and Choice/Score confidence lives in providerMetadata.typesafe. - const vercelQuestions: Record = {}; - for (const [id, question] of Object.entries(questions)) { - const q = question as { type: string; instructions?: unknown; criteria?: unknown }; - vercelQuestions[id] = { - type: q.type === "noul" ? "boolean" : q.type, - instructions: q.instructions, - criteria: q.criteria, - }; +function record(value: unknown): value is Record { + return value !== null && typeof value === "object" && !Array.isArray(value); +} + +function distribution(value: unknown, keys: string[], fail: (reason: string) => never): Record { + if (!record(value) || Object.keys(value).length !== keys.length || keys.some((key) => !Object.hasOwn(value, key))) { + fail("distribution must contain exactly the criteria keys"); } - const result = await experimental_evaluate({ - model: model.startsWith("typesafe-ai/") ? model : "typesafe-ai/jev", - state: state as any, - questions: vercelQuestions as any, - abortSignal: signal, - }); - const confidence = ((result as any).providerMetadata?.typesafe?.confidence ?? {}) as Record; - const adapted: Record = {}; - for (const [id, answer] of Object.entries(result.answers as Record)) { - if (answer?.type === "boolean") { - adapted[id] = { type: "noul", noul: answer.probability }; - } else if (answer?.type === "choice") { - adapted[id] = { type: "choice", choice: answer.choice, probabilities: answer.probabilities ?? {}, confidence: confidence[id] ?? null }; - } else if (answer?.type === "score") { - adapted[id] = { type: "score", score: answer.score, probabilities: answer.probabilities ?? {}, confidence: confidence[id] ?? null }; - } else { - adapted[id] = answer; + let sum = 0; + const probabilities: Record = {}; + for (const key of keys) { + const probability = value[key]; + if (typeof probability !== "number" || !Number.isFinite(probability) || probability < 0 || probability > 1) { + fail("distribution must contain finite probabilities in [0,1]"); } + probabilities[key] = probability as number; + sum += probability as number; } - return { - answers: adapted, - usage: { input_tokens: result.usage?.inputTokens ?? 0, output_tokens: result.usage?.outputTokens ?? 0 }, - provider, - model: "typesafe-ai/jev", - }; + // Upstream distributions are rounded; permit a one-percent sum drift and + // a 0.001 selection tie, but not a different winner. + if (Math.abs(sum - 1) > 0.01) fail("distribution must sum to approximately 1"); + return probabilities; } - // Cloudflare Workers AI wraps the same contract in {model, input} and the - // v4 {result, success} envelope. Single alias; no version pinning. - const cfSlug = model.startsWith("typesafe/") ? model : `typesafe/${model === "jev-latest" ? "jev" : model}`; - const cfResponse = await fetch( - `https://api.cloudflare.com/client/v4/accounts/${process.env.CLOUDFLARE_ACCOUNT_ID}/ai/run`, - { - method: "POST", - headers: { - Authorization: `Bearer ${process.env.JEV_CLOUDFLARE_API_TOKEN || process.env.CLOUDFLARE_API_TOKEN}`, - "Content-Type": "application/json", - }, - body: JSON.stringify({ model: cfSlug, input: { state, questions } }), - signal, - }, - ); - const cfBody = await cfResponse.json().catch(() => ({})); - if (!cfResponse.ok || cfBody.success === false) { - throw new Error(`Cloudflare AI run ${cfResponse.status}: ${JSON.stringify(cfBody.errors ?? cfBody).slice(0, 200)}`); +export async function askJev(transport: JevTransport, input: JevTransportInput): Promise { + // The label is caller-supplied for injected transports, not a wire field. + const provider = typeof transport.name === "string" && transport.name.trim() ? transport.name : "unknown"; + const fail = (id: string, reason: string): never => { + throw new InvalidJevAnswer(`Jev provider ${provider} question ${id}: ${reason}`); + }; + const reply = await transport.ask(input); + if (!record(reply) || !record(reply.answers)) fail("", "answers must be an object"); + const answers = reply.answers as Record; + const ids = Object.keys(input.questions); + for (const id of ids) if (!Object.hasOwn(answers, id)) fail(id, "missing answer"); + if (Object.keys(answers).length !== ids.length) fail("", "unexpected answer ID"); + const validated: Record = {}; + for (const id of ids) { + const question = input.questions[id]; + const answer = answers[id]; + const invalid = (reason: string): never => fail(id, reason); + if (!record(question) || !record(answer) || answer.type !== question.type) throw new InvalidJevAnswer(`Jev provider ${provider} question ${id}: missing answer or wrong type`); + if (question.type === "noul") { + if (typeof answer.noul !== "number" || !Number.isFinite(answer.noul) || answer.noul < 0 || answer.noul > 1) invalid("noul must be finite in [0,1]"); + validated[id] = { type: "noul", noul: answer.noul as number }; + continue; + } + const keys = question.type === "score" && Array.isArray(question.criteria) + ? question.criteria.map((_, index) => String(index)) + : question.type === "choice" && record(question.criteria) ? Object.keys(question.criteria) : null; + if (!keys?.length) invalid("invalid question criteria"); + const probabilities = distribution(answer.probabilities, keys!, invalid); + const confidence = answer.confidence === undefined ? null : answer.confidence; + if (confidence !== null && (typeof confidence !== "number" || !Number.isFinite(confidence))) invalid("confidence must be finite or null"); + if (question.type === "choice") { + if (typeof answer.choice !== "string" || !keys!.includes(answer.choice)) invalid("choice is outside criteria"); + if (probabilities[answer.choice as string] + 0.001 < Math.max(...Object.values(probabilities))) invalid("choice is not a distribution maximum"); + validated[id] = { type: "choice", choice: answer.choice as string, probabilities, confidence: confidence as number | null }; + } else if (question.type === "score") { + if (typeof answer.score !== "number" || !Number.isInteger(answer.score) || !keys!.includes(String(answer.score))) invalid("score is outside criteria levels"); + validated[id] = { type: "score", score: answer.score as number, probabilities, confidence: confidence as number | null }; + } else invalid("unsupported question type"); } - // The v4 envelope double-nests: body.result.result holds the model output. - const cfOuter = cfBody.result; - if (cfOuter && typeof cfOuter.state === "string" && cfOuter.state !== "Completed") { - throw new Error(`Cloudflare AI run state ${cfOuter.state}: ${JSON.stringify(cfBody.errors ?? []).slice(0, 200)}`); + if (!record(reply.usage) || !Number.isSafeInteger(reply.usage.input_tokens) || (reply.usage.input_tokens as number) < 0 || !Number.isSafeInteger(reply.usage.output_tokens) || (reply.usage.output_tokens as number) < 0) { + fail("", "usage counters must be non-negative safe integers"); } - const cfPayload = cfOuter?.result ?? cfOuter ?? cfBody; - return { - answers: cfPayload.answers ?? {}, - usage: { input_tokens: cfPayload.usage?.input_tokens ?? 0, output_tokens: cfPayload.usage?.output_tokens ?? 0 }, - provider, - model: cfPayload.model ?? cfSlug, - }; + if (typeof reply.model !== "string" || !reply.model.trim()) fail("", "effective model must be nonempty"); + return { answers: validated, usage: reply.usage as AskResult["usage"], provider, model: reply.model as string }; } diff --git a/src/transports/cloudflare.ts b/src/transports/cloudflare.ts new file mode 100644 index 0000000..b160b56 --- /dev/null +++ b/src/transports/cloudflare.ts @@ -0,0 +1,61 @@ +import type { BuiltinDriver } from "../provider.js"; + +export const cloudflare: BuiltinDriver = { + name: "cloudflare", + isConfigured: (env) => Boolean((env.JEV_CLOUDFLARE_API_TOKEN || env.CLOUDFLARE_API_TOKEN) && env.CLOUDFLARE_ACCOUNT_ID), + assertConfigured(env) { + if (!this.isConfigured(env)) throw new Error("a Cloudflare API token (CLOUDFLARE_API_TOKEN or JEV_CLOUDFLARE_API_TOKEN) and CLOUDFLARE_ACCOUNT_ID are not both set."); + }, + create(env) { + this.assertConfigured(env); + const token = env.JEV_CLOUDFLARE_API_TOKEN || env.CLOUDFLARE_API_TOKEN; + const account = env.CLOUDFLARE_ACCOUNT_ID!; + return { + name: this.name, + async ask({ state, questions, model, signal }) { + const slug = model.startsWith("typesafe/") ? model : `typesafe/${model === "jev-latest" ? "jev" : model}`; + const response = await fetch(`https://api.cloudflare.com/client/v4/accounts/${account}/ai/run`, { + method: "POST", + headers: { Authorization: `Bearer ${token}`, "Content-Type": "application/json" }, + body: JSON.stringify({ model: slug, input: { state, questions } }), + signal, + }).catch(() => { + if (signal.aborted) throw signal.reason; + throw new Error("Cloudflare AI run HTTP unavailable (network error; 0 response bytes)"); + }); + const raw = await response.text().catch(() => { + if (signal.aborted) throw signal.reason; + throw new Error(`Cloudflare AI run HTTP ${response.status} (body read error; 0 response bytes)`); + }); + const bytes = Buffer.byteLength(raw); + if (!response.ok) throw new Error(`Cloudflare AI run HTTP ${response.status} (request failed; ${bytes} response bytes)`); + let body: any; + try { + body = JSON.parse(raw); + } catch { + throw new Error(`Cloudflare AI run HTTP ${response.status} (invalid JSON; ${bytes} response bytes)`); + } + if (!body || typeof body !== "object" || Array.isArray(body)) throw new Error(`Cloudflare AI run HTTP ${response.status} (invalid envelope; ${bytes} response bytes)`); + if (body.success === false) throw new Error(`Cloudflare AI run HTTP ${response.status} (API unsuccessful; ${bytes} response bytes)`); + // The v4 envelope double-nests the model output under result.result. + const outer = body.result; + if (outer && typeof outer.state === "string" && outer.state !== "Completed") { + throw new Error(`Cloudflare AI run HTTP ${response.status} (non-Completed state; ${bytes} response bytes)`); + } + const payload = outer?.result ?? outer ?? body; + const usage = payload?.usage; + if (usage !== undefined && (typeof usage !== "object" || usage === null || Array.isArray(usage))) { + throw new Error(`Cloudflare AI run HTTP ${response.status} (invalid usage; ${bytes} response bytes)`); + } + return { + answers: payload?.answers, + usage: { + input_tokens: usage && Object.hasOwn(usage, "input_tokens") ? usage.input_tokens : 0, + output_tokens: usage && Object.hasOwn(usage, "output_tokens") ? usage.output_tokens : 0, + }, + model: payload?.model ?? slug, + }; + }, + }; + }, +}; diff --git a/src/transports/openrouter.ts b/src/transports/openrouter.ts new file mode 100644 index 0000000..0d20289 --- /dev/null +++ b/src/transports/openrouter.ts @@ -0,0 +1,64 @@ +import type { BuiltinDriver } from "../provider.js"; + +const TITLE = "jev-browser"; +const REFERER = "https://github.com/jkudish/jev-browser"; + +export const openrouter: BuiltinDriver = { + name: "openrouter", + isConfigured: (env) => /^sk-or-/.test(env.OPENROUTER_API_KEY ?? ""), + assertConfigured(env) { + if (!this.isConfigured(env)) throw new Error("OPENROUTER_API_KEY is not set or not an sk-or- key."); + }, + create(env) { + this.assertConfigured(env); + const key = env.OPENROUTER_API_KEY!; + return { + name: this.name, + async ask({ state, questions, model, signal }) { + const effective = model === "jev-latest" ? "jev-1.13" : model; + const slug = effective.startsWith("typesafe/") ? effective : `typesafe/${effective}`; + const response = await fetch("https://openrouter.ai/api/alpha/decisions", { + method: "POST", + headers: { + Authorization: `Bearer ${key}`, + "Content-Type": "application/json", + "HTTP-Referer": REFERER, + "X-Title": TITLE, + "X-OpenRouter-Title": TITLE, + }, + body: JSON.stringify({ model: slug, state, questions }), + signal, + }).catch(() => { + if (signal.aborted) throw signal.reason; + throw new Error("OpenRouter decisions API HTTP unavailable (network error; 0 response bytes)"); + }); + const raw = await response.text().catch(() => { + if (signal.aborted) throw signal.reason; + throw new Error(`OpenRouter decisions API HTTP ${response.status} (body read error; 0 response bytes)`); + }); + const bytes = Buffer.byteLength(raw); + if (!response.ok) throw new Error(`OpenRouter decisions API HTTP ${response.status} (request failed; ${bytes} response bytes)`); + let body: any; + try { + body = JSON.parse(raw); + } catch { + throw new Error(`OpenRouter decisions API HTTP ${response.status} (invalid JSON; ${bytes} response bytes)`); + } + if (!body || typeof body !== "object" || Array.isArray(body)) throw new Error(`OpenRouter decisions API HTTP ${response.status} (invalid envelope; ${bytes} response bytes)`); + const usage = body.usage; + if (usage !== undefined && (typeof usage !== "object" || usage === null || Array.isArray(usage))) { + throw new Error(`OpenRouter decisions API HTTP ${response.status} (invalid usage; ${bytes} response bytes)`); + } + return { + answers: body.answers, + // The decisions endpoint does not document usage; absence means zero. + usage: { + input_tokens: usage && Object.hasOwn(usage, "input_tokens") ? usage.input_tokens : 0, + output_tokens: usage && Object.hasOwn(usage, "output_tokens") ? usage.output_tokens : 0, + }, + model: slug, + }; + }, + }; + }, +}; diff --git a/src/transports/typesafe.ts b/src/transports/typesafe.ts new file mode 100644 index 0000000..6919c0a --- /dev/null +++ b/src/transports/typesafe.ts @@ -0,0 +1,47 @@ +import { TypeSafeClient } from "@typesafe-ai/sdk"; +import type { BuiltinDriver } from "../provider.js"; + +export const typesafe: BuiltinDriver = { + name: "typesafe", + isConfigured: (env) => Boolean(env.TYPESAFE_API_KEY), + assertConfigured(env) { + if (!this.isConfigured(env)) throw new Error("TYPESAFE_API_KEY is not set."); + }, + create(env) { + this.assertConfigured(env); + const client = new TypeSafeClient({ + apiKey: env.TYPESAFE_API_KEY!, + baseURL: env.TYPESAFE_BASE_URL || "https://api.typesafe.ai", + }); + return { + name: this.name, + async ask({ state, questions, model, signal }) { + let response: { answers: unknown; usage?: { input_tokens?: number; output_tokens?: number } }; + try { + response = await ( + client.systemOne as unknown as ( + payload: { state: unknown; questions: Record; model: string }, + options: { signal: AbortSignal }, + ) => Promise + )({ state, questions, model }, { signal }); + } catch (error) { + if (signal.aborted) throw signal.reason; + const status = (error as { status?: unknown }).status; + throw new Error(`TypeSafe API ${typeof status === "number" ? `HTTP ${status}` : "request failed"} (response omitted)`); + } + const usage = response.usage; + if (usage !== undefined && (typeof usage !== "object" || usage === null || Array.isArray(usage))) { + throw new Error("TypeSafe API invalid usage (response omitted)"); + } + return { + answers: response.answers, + usage: { + input_tokens: usage && Object.hasOwn(usage, "input_tokens") ? usage.input_tokens as number : 0, + output_tokens: usage && Object.hasOwn(usage, "output_tokens") ? usage.output_tokens as number : 0, + }, + model, + }; + }, + }; + }, +}; diff --git a/src/transports/vercel.ts b/src/transports/vercel.ts new file mode 100644 index 0000000..edc4569 --- /dev/null +++ b/src/transports/vercel.ts @@ -0,0 +1,64 @@ +import { createGateway, experimental_evaluate } from "ai"; +import type { BuiltinDriver } from "../provider.js"; + +// Internal factory seam: tests provide evaluate without replacing ESM exports. +export function createVercelDriver(evaluate: typeof experimental_evaluate = experimental_evaluate): BuiltinDriver { + return { + name: "vercel", + isConfigured: (env) => Boolean(env.AI_GATEWAY_API_KEY), + assertConfigured(env) { + if (!this.isConfigured(env)) throw new Error("AI_GATEWAY_API_KEY is not set."); + }, + create(env) { + this.assertConfigured(env); + const gateway = createGateway({ apiKey: env.AI_GATEWAY_API_KEY! }); + return { + name: this.name, + async ask({ state, questions, model, signal }) { + const adaptedQuestions: Record = {}; + for (const [id, question] of Object.entries(questions)) { + const q = question as { type: string; instructions?: unknown; criteria?: unknown }; + adaptedQuestions[id] = { type: q.type === "noul" ? "boolean" : q.type, instructions: q.instructions, criteria: q.criteria }; + } + const effective = model.startsWith("typesafe-ai/") ? model : "typesafe-ai/jev"; + let result: Awaited>; + try { + result = await evaluate({ model: gateway.evaluation(effective), state: state as any, questions: adaptedQuestions as any, abortSignal: signal }); + } catch (error) { + if (signal.aborted) throw signal.reason; + const status = (error as { statusCode?: unknown }).statusCode; + throw new Error(`Vercel AI Gateway ${typeof status === "number" ? `HTTP ${status}` : "request failed"} (response omitted)`); + } + const usage = result.usage; + if (usage !== undefined && (typeof usage !== "object" || usage === null || Array.isArray(usage))) { + throw new Error("Vercel AI Gateway invalid usage (response omitted)"); + } + return { + answers: adaptVercelAnswers(result.answers, result.providerMetadata), + usage: { + input_tokens: usage && Object.hasOwn(usage, "inputTokens") ? usage.inputTokens as number : 0, + output_tokens: usage && Object.hasOwn(usage, "outputTokens") ? usage.outputTokens as number : 0, + }, + model: effective, + }; + }, + }; + }, + }; +} + +export function adaptVercelAnswers(answers: unknown, metadata: unknown): unknown { + if (!answers || typeof answers !== "object" || Array.isArray(answers)) return answers; + const confidence = (metadata as any)?.typesafe?.confidence ?? {}; + const adapted: Record = {}; + for (const [id, answer] of Object.entries(answers)) { + const value = answer as any; + if (value?.type === "boolean") adapted[id] = { type: "noul", noul: value.probability }; + else if (value?.type === "choice") adapted[id] = { type: "choice", choice: value.choice, probabilities: value.probabilities, confidence: confidence[id] ?? null }; + else if (value?.type === "score") adapted[id] = { type: "score", score: value.score, probabilities: value.probabilities, confidence: confidence[id] ?? null }; + else adapted[id] = answer; + } + return adapted; +} + +export const vercel = createVercelDriver(); diff --git a/test/e2e.test.mjs b/test/e2e.test.mjs index 4b01c4f..f749cc2 100644 --- a/test/e2e.test.mjs +++ b/test/e2e.test.mjs @@ -239,7 +239,7 @@ test("typed search: find the Ristretto article", { skip: !hasKey }, async () => { timeout: 240_000 }, ); const body = payload(result); - assert.ok(["done", "goal_achieved"].includes(body.status), `status was ${body.status}: ${JSON.stringify(body.steps)}`); + assert.ok(["done", "goal_achieved"].includes(body.status), `status was ${body.status}: ${JSON.stringify(body.error)} steps: ${JSON.stringify(body.steps)}`); assert.match(body.final_url, /Ristretto/); }); }); diff --git a/test/transports.test.mjs b/test/transports.test.mjs new file mode 100644 index 0000000..5004d42 --- /dev/null +++ b/test/transports.test.mjs @@ -0,0 +1,279 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { chromium } from "playwright"; +import { askJev, resolveTransport } from "../dist/provider.js"; +import { typesafe } from "../dist/transports/typesafe.js"; +import { openrouter } from "../dist/transports/openrouter.js"; +import { cloudflare } from "../dist/transports/cloudflare.js"; +import { createVercelDriver, adaptVercelAnswers } from "../dist/transports/vercel.js"; +import { navigate } from "../dist/library.js"; + +const signal = new AbortController().signal; +const questions = { item: { type: "choice", criteria: { alpha: "A", beta: "B" } }, yes: { type: "noul" } }; +const answers = { item: { type: "choice", choice: "beta", probabilities: { alpha: 0.2, beta: 0.8 } }, yes: { type: "noul", noul: 0.7 } }; +const input = { state: { title: "Example" }, questions, model: "jev-latest", signal }; +const usage = { input_tokens: 13, output_tokens: 3 }; +const carrier = (override = {}) => ({ name: "fixture", ask: async () => ({ answers, usage, model: "effective", ...override }) }); + +async function withFetch(fn, run) { + const original = globalThis.fetch; + globalThis.fetch = fn; + try { return await run(); } finally { globalThis.fetch = original; } +} + +test("registry auto-detects in precedence order and explicit names select only themselves", () => { + const all = { TYPESAFE_API_KEY: "ts-secret", OPENROUTER_API_KEY: "sk-or-secret", CLOUDFLARE_API_TOKEN: "cf-secret", CLOUDFLARE_ACCOUNT_ID: "account", AI_GATEWAY_API_KEY: "ai-secret" }; + for (const [removed, expected] of [ + [[], "typesafe"], + [["TYPESAFE_API_KEY"], "openrouter"], + [["TYPESAFE_API_KEY", "OPENROUTER_API_KEY"], "cloudflare"], + [["TYPESAFE_API_KEY", "OPENROUTER_API_KEY", "CLOUDFLARE_API_TOKEN"], "vercel"], + ]) { + const env = { ...all }; + for (const key of removed) delete env[key]; + assert.equal(resolveTransport(env).name, expected); + } + for (const name of ["typesafe", "openrouter", "cloudflare", "vercel"]) { + assert.equal(resolveTransport({ ...all, JEV_PROVIDER: name.toUpperCase() }).name, name); + } + assert.equal(resolveTransport({ JEV_CLOUDFLARE_API_TOKEN: "pref", CLOUDFLARE_ACCOUNT_ID: "account" }).name, "cloudflare"); + assert.throws(() => resolveTransport({ ...all, JEV_PROVIDER: "typo" }), /Unknown JEV_PROVIDER/); + assert.throws(() => resolveTransport({}), (error) => ["TYPESAFE_API_KEY", "OPENROUTER_API_KEY", "CLOUDFLARE_API_TOKEN", "JEV_CLOUDFLARE_API_TOKEN", "CLOUDFLARE_ACCOUNT_ID", "AI_GATEWAY_API_KEY"].every((name) => error.message.includes(name))); +}); + +test("forced credentials reject every missing or invalid variant without leaking values", () => { + const cases = [ + ["typesafe", {}, /TYPESAFE_API_KEY/], + ["openrouter", {}, /OPENROUTER_API_KEY/], + ["openrouter", { OPENROUTER_API_KEY: "wrong-secret" }, /sk-or-/], + ["cloudflare", {}, /CLOUDFLARE_ACCOUNT_ID/], + ["cloudflare", { CLOUDFLARE_API_TOKEN: "cf-secret" }, /CLOUDFLARE_ACCOUNT_ID/], + ["cloudflare", { CLOUDFLARE_ACCOUNT_ID: "account-secret" }, /CLOUDFLARE_API_TOKEN/], + ["cloudflare", { JEV_CLOUDFLARE_API_TOKEN: "cf-secret" }, /CLOUDFLARE_ACCOUNT_ID/], + ["vercel", {}, /AI_GATEWAY_API_KEY/], + ]; + for (const [name, env, pattern] of cases) { + assert.throws(() => resolveTransport({ ...env, JEV_PROVIDER: name }), (error) => { + assert.match(error.message, pattern); + assert.ok(error.message.startsWith(`JEV_PROVIDER=${name}`)); + assert.ok(!error.message.includes("secret")); + return true; + }); + } + assert.equal(resolveTransport({ OPENROUTER_API_KEY: "wrong-secret", AI_GATEWAY_API_KEY: "ai-secret" }).name, "vercel"); + assert.equal(resolveTransport({ CLOUDFLARE_ACCOUNT_ID: "account-secret", AI_GATEWAY_API_KEY: "ai-secret" }).name, "vercel"); +}); + +test("facade validates the complete answer contract before usage can be credited", async () => { + const good = await askJev(carrier(), input); + assert.equal(good.provider, "fixture"); + assert.equal(good.answers.item.confidence, null); + assert.equal(good.model, "effective"); + for (const [mutated, id, reason] of [ + [{ item: answers.item }, "yes", /missing answer/], + [{ ...answers, item: { ...answers.item, type: "noul" } }, "item", /wrong type/], + [{ ...answers, item: { ...answers.item, choice: "gamma" } }, "item", /outside criteria/], + [{ ...answers, item: { ...answers.item, choice: "alpha" } }, "item", /not a distribution maximum/], + [{ ...answers, item: { ...answers.item, probabilities: { alpha: NaN, beta: 0.8 } } }, "item", /finite probabilities/], + [{ ...answers, item: { ...answers.item, probabilities: { alpha: 0.1, beta: 0.7 } } }, "item", /approximately 1/], + [{ ...answers, item: { ...answers.item, probabilities: { alpha: 0.2, beta: 0.7, extra: 0.1 } } }, "item", /exactly the criteria/], + [{ ...answers, item: { ...answers.item, confidence: Infinity } }, "item", /confidence/], + [{ ...answers, yes: { type: "noul", noul: 1.1 } }, "yes", /noul/], + ]) { + await assert.rejects(() => askJev(carrier({ answers: mutated }), input), (error) => error.message.includes(`provider fixture question ${id}`) && reason.test(error.message)); + } + await assert.rejects(() => askJev(carrier({ usage: { input_tokens: -1, output_tokens: 0 } }), input), /provider fixture question .*usage/); + await assert.rejects(() => askJev(carrier({ usage: { input_tokens: undefined, output_tokens: 0 } }), input), /provider fixture question .*usage/); + await assert.rejects(() => askJev(carrier({ model: " " }), input), /provider fixture question .*model/); + const tied = { ...answers, item: { ...answers.item, choice: "alpha", probabilities: { alpha: 0.4995, beta: 0.5005 }, confidence: 0 } }; + assert.equal((await askJev(carrier({ answers: tied }), input)).answers.item.choice, "alpha"); + const score = { state: null, model: "jev", signal, questions: { rank: { type: "score", criteria: ["poor", "good", "great"] } } }; + const scoreReply = { rank: { type: "score", score: 2, probabilities: { "0": 0.1, "1": 0.2, "2": 0.7 }, confidence: null } }; + assert.equal((await askJev(carrier({ answers: scoreReply }), score)).answers.rank.score, 2); + await assert.rejects(() => askJev(carrier({ answers: { rank: { ...scoreReply.rank, score: 3 } } }), score), /question rank.*score is outside/); +}); + +test("TypeSafe client binds key and base URL at creation, forwards request and cancellation", async () => { + const calls = []; + await withFetch(async (url, init) => { + calls.push({ url, init }); + return Response.json({ answers, usage }); + }, async () => { + const transport = typesafe.create({ TYPESAFE_API_KEY: "ts-secret", TYPESAFE_BASE_URL: "https://local.typesafe.test" }); + const reply = await askJev(transport, input); + assert.deepEqual(reply.usage, usage); + assert.equal(reply.model, "jev-latest"); + assert.equal(calls.length, 1); + assert.equal(calls[0].url, "https://local.typesafe.test/v1/systemone"); + assert.equal(calls[0].init.headers.Authorization, "Bearer ts-secret"); + assert.deepEqual(JSON.parse(calls[0].init.body), { state: input.state, questions, model: "jev-latest" }); + assert.ok(calls[0].init.signal instanceof AbortSignal); + assert.equal(calls[0].init.signal.aborted, false); + }); + await withFetch(async () => Response.json({ error: "body-secret" }, { status: 400 }), async () => { + await assert.rejects(() => typesafe.create({ TYPESAFE_API_KEY: "ts-secret" }).ask(input), (error) => !/body-secret|ts-secret/.test(error.message) && /TypeSafe API HTTP 400/.test(error.message)); + }); +}); + +test("OpenRouter maps latest and pinned slugs, sends exact envelope and redacts HTTP errors", async () => { + const calls = []; + await withFetch(async (url, init) => { + calls.push({ url, init }); + return Response.json({ answers, usage }); + }, async () => { + const transport = openrouter.create({ OPENROUTER_API_KEY: "sk-or-secret" }); + assert.equal((await askJev(transport, input)).model, "typesafe/jev-1.13"); + assert.equal((await askJev(transport, { ...input, model: "typesafe/jev-1.12" })).model, "typesafe/jev-1.12"); + assert.equal(calls[0].url, "https://openrouter.ai/api/alpha/decisions"); + assert.equal(calls[0].init.method, "POST"); + assert.deepEqual(calls[0].init.headers, { Authorization: "Bearer sk-or-secret", "Content-Type": "application/json", "HTTP-Referer": "https://github.com/jkudish/jev-browser", "X-Title": "jev-browser", "X-OpenRouter-Title": "jev-browser" }); + assert.equal(calls[0].init.signal, signal); + assert.deepEqual(JSON.parse(calls[0].init.body), { model: "typesafe/jev-1.13", state: input.state, questions }); + }); + for (const response of [new Response("body-secret", { status: 403 }), new Response("body-secret", { status: 200 })]) { + await withFetch(async () => response, async () => { + await assert.rejects(() => openrouter.create({ OPENROUTER_API_KEY: "sk-or-secret" }).ask(input), (error) => /OpenRouter decisions API HTTP/.test(error.message) && !/body-secret|sk-or-secret/.test(error.message)); + }); + } + await withFetch(async () => Response.json({ usage }), async () => { + await assert.rejects(() => askJev(openrouter.create({ OPENROUTER_API_KEY: "sk-or-secret" }), input), /question .*answers/); + }); + await withFetch(async () => { throw new Error("body-secret sk-or-secret"); }, async () => { + await assert.rejects(() => openrouter.create({ OPENROUTER_API_KEY: "sk-or-secret" }).ask(input), (error) => /HTTP unavailable.*network error/.test(error.message) && !/body-secret|sk-or-secret/.test(error.message)); + }); +}); + +test("Cloudflare token priority, double envelope, state, usage, and safe errors", async () => { + const calls = []; + const env = { CLOUDFLARE_API_TOKEN: "low-secret", JEV_CLOUDFLARE_API_TOKEN: "high-secret", CLOUDFLARE_ACCOUNT_ID: "account" }; + await withFetch(async (url, init) => { + calls.push({ url, init }); + return Response.json({ success: true, result: { state: "Completed", result: { answers, usage, model: "typesafe/jev" } } }); + }, async () => { + const transport = cloudflare.create(env); + assert.deepEqual((await askJev(transport, input)).usage, usage); + assert.equal(calls[0].url, "https://api.cloudflare.com/client/v4/accounts/account/ai/run"); + assert.deepEqual(calls[0].init.headers, { Authorization: "Bearer high-secret", "Content-Type": "application/json" }); + assert.equal(calls[0].init.signal, signal); + assert.deepEqual(JSON.parse(calls[0].init.body), { model: "typesafe/jev", input: { state: input.state, questions } }); + assert.equal((await transport.ask({ ...input, model: "typesafe/jev-1.2" })).model, "typesafe/jev"); + assert.equal(JSON.parse(calls[1].init.body).model, "typesafe/jev-1.2"); + }); + for (const response of [new Response("body-secret", { status: 401 }), Response.json({ success: false, errors: ["body-secret"] }), Response.json({ result: { state: "Failed body-secret" } }), new Response("body-secret", { status: 200 })]) { + await withFetch(async () => response, async () => { + await assert.rejects(() => cloudflare.create(env).ask(input), (error) => /Cloudflare AI run HTTP/.test(error.message) && !/body-secret|high-secret|low-secret/.test(error.message)); + }); + } + await withFetch(async () => { throw new Error("body-secret high-secret"); }, async () => { + await assert.rejects(() => cloudflare.create(env).ask(input), (error) => /HTTP unavailable.*network error/.test(error.message) && !/body-secret|high-secret/.test(error.message)); + }); +}); + +test("Vercel factory forwards evaluate request and pure adaptation preserves confidence", async () => { + let call; + const raw = { item: { type: "choice", choice: "beta", probabilities: { alpha: 0.2, beta: 0.8 } }, yes: { type: "boolean", probability: 0.7 } }; + const result = { answers: raw, usage: { inputTokens: 13, outputTokens: 3 }, providerMetadata: { typesafe: { confidence: { item: 0.92 } } } }; + const factory = createVercelDriver(async (args) => { call = args; return result; }); + const reply = await askJev(factory.create({ AI_GATEWAY_API_KEY: "ai-secret" }), input); + assert.equal(reply.model, "typesafe-ai/jev"); + assert.deepEqual(reply.usage, usage); + assert.equal(call.abortSignal, signal); + assert.equal(call.model.modelId, "typesafe-ai/jev"); + assert.deepEqual(call.questions, { item: { type: "choice", instructions: undefined, criteria: questions.item.criteria }, yes: { type: "boolean", instructions: undefined, criteria: undefined } }); + assert.deepEqual(reply.answers, { item: { ...answers.item, confidence: 0.92 }, yes: answers.yes }); + assert.deepEqual(adaptVercelAnswers({ yes: raw.yes, rank: { type: "score", score: 1, probabilities: { "0": 0.2, "1": 0.8 } } }, {}), { yes: answers.yes, rank: { type: "score", score: 1, probabilities: { "0": 0.2, "1": 0.8 }, confidence: null } }); + const failing = createVercelDriver(async () => { throw Object.assign(new Error("body-secret"), { statusCode: 403 }); }); + await assert.rejects(() => failing.create({ AI_GATEWAY_API_KEY: "ai-secret" }).ask(input), (error) => /Vercel AI Gateway HTTP 403/.test(error.message) && !/body-secret|ai-secret/.test(error.message)); + const malformed = createVercelDriver(async () => ({ ...result, answers: { item: raw.item } })); + await assert.rejects(() => askJev(malformed.create({ AI_GATEWAY_API_KEY: "ai-secret" }), input), /provider vercel question yes.*missing answer/); +}); + +test("all adapters distinguish absent usage from malformed containers and present null counters", async () => { + const vercelAnswers = { item: answers.item, yes: { type: "boolean", probability: answers.yes.noul } }; + for (const [name, field, run] of [ + ["typesafe", "input_tokens", (wire) => withFetch(async () => Response.json({ answers, ...wire }), () => askJev(typesafe.create({ TYPESAFE_API_KEY: "ts-secret" }), input))], + ["openrouter", "input_tokens", (wire) => withFetch(async () => Response.json({ answers, ...wire }), () => askJev(openrouter.create({ OPENROUTER_API_KEY: "sk-or-secret" }), input))], + ["cloudflare", "input_tokens", (wire) => withFetch(async () => Response.json({ result: { state: "Completed", result: { answers, ...wire } } }), () => askJev(cloudflare.create({ CLOUDFLARE_API_TOKEN: "cf-secret", CLOUDFLARE_ACCOUNT_ID: "account" }), input))], + ["vercel", "inputTokens", (wire) => askJev(createVercelDriver(async () => ({ answers: vercelAnswers, ...wire })).create({ AI_GATEWAY_API_KEY: "ai-secret" }), input)], + ]) { + assert.deepEqual((await run({})).usage, { input_tokens: 0, output_tokens: 0 }, name); + assert.deepEqual((await run({ usage: {} })).usage, { input_tokens: 0, output_tokens: 0 }, name); + const invalidCases = [{ usage: "body-secret" }, { usage: null }, { usage: { [field]: null } }]; + if (name === "vercel") invalidCases.push({ usage: { [field]: undefined } }); // JSON drops undefined properties on the HTTP drivers. + for (const invalid of invalidCases) { + await assert.rejects(() => run(invalid), (error) => /usage/.test(error.message) && !error.message.includes("body-secret"), `${name}: ${JSON.stringify(invalid)}`); + } + } +}); + +test("injected transport drives both call sites and malformed second-stage answer executes nothing", async (t) => { + let browser; + try { + browser = await chromium.launch(); + } catch (error) { + if (String(error).includes("Executable doesn't exist")) { + t.skip("Playwright browser binary is not installed"); + return; + } + throw error; + } + try { + const page = await browser.newPage(); + await page.setContent(''); + for (const broken of [false, true]) { + const calls = []; + const transport = { + name: "custom-carrier", + async ask(request) { + calls.push(request); + const reply = request.questions.option + ? { option: broken ? { type: "choice", choice: "missing", probabilities: { o0: 0.1, o1: 0.9 } } : { type: "choice", choice: "o1", probabilities: { o0: 0.1, o1: 0.9 } } } + : { action: { type: "choice", choice: "select_e1", probabilities: Object.fromEntries(Object.keys(request.questions.action.criteria).map((key) => [key, key === "select_e1" ? 1 : 0])) }, goal_done: { type: "noul", noul: 0 }, stuck: { type: "noul", noul: 0 } }; + return { answers: reply, usage, model: "custom-model" }; + }, + }; + const previous = process.env.JEV_PROVIDER; + process.env.JEV_PROVIDER = "invalid-forced-name"; + let result; + try { result = await navigate({ task: "Choose Premium", page, transport, allowTyping: false, maxSteps: 1, screenshot: "none" }); } + finally { if (previous === undefined) delete process.env.JEV_PROVIDER; else process.env.JEV_PROVIDER = previous; } + assert.equal(calls.length, 2); + assert.equal(result.jev_provider, "custom-carrier"); + assert.equal(result.model, "custom-model"); + if (broken) { + assert.equal(result.status, "error"); + assert.match(result.error, /question option.*outside criteria/); + assert.equal(result.usage.jev_calls, 1); + assert.equal(result.steps.length, 0); + assert.equal(await page.locator("select").inputValue(), "a"); + } else { + assert.equal(result.status, "max_steps"); + assert.equal(result.usage.jev_calls, 2); + assert.equal(result.steps[0].executed_action, "select_e1"); + assert.equal(await page.locator("select").inputValue(), "b"); + await page.locator("select").selectOption("a"); + } + } + const refused = await navigate({ + task: "Choose Premium", page, allowTyping: false, maxSteps: 1, screenshot: "none", + transport: { name: "custom-carrier", async ask(request) { + if (request.questions.option) throw new Error("body-secret"); + return { answers: { action: { type: "choice", choice: "select_e1", probabilities: Object.fromEntries(Object.keys(request.questions.action.criteria).map((key) => [key, key === "select_e1" ? 1 : 0])) }, goal_done: { type: "noul", noul: 0 }, stuck: { type: "noul", noul: 0 } }, usage, model: "custom-model" }; + } }, + }); + assert.equal(refused.status, "error"); + assert.match(refused.error, /question option: request failed/); + assert.ok(!refused.error.includes("body-secret")); + assert.equal(refused.steps.length, 0); + assert.equal(await page.locator("select").inputValue(), "a"); + const primary = await navigate({ + task: "Choose Premium", page, allowTyping: false, maxSteps: 1, screenshot: "none", + transport: { name: "custom-carrier", ask: async () => ({ answers: { action: { type: "choice", choice: "select_e1", probabilities: {} } }, usage, model: "custom-model" }) }, + }); + assert.equal(primary.status, "error"); + assert.match(primary.error, /question goal_done.*missing answer/); + assert.equal(primary.steps.length, 0); + assert.equal(primary.usage.jev_calls, 0); + assert.equal(await page.locator("select").inputValue(), "a"); + } finally { await browser.close(); } +});