diff --git a/CHANGELOG.md b/CHANGELOG.md index e347cb1..65f82a8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ## Unreleased +- Judgment transport now uses `@jkudish/jev-agent-tools` for provider selection and answer validation; browser run errors remain fail-loud. - Transport drivers: the four judgment transports (TypeSafe, OpenRouter, Cloudflare, Vercel) are now run-bound drivers behind one registry. Typing providers are unchanged. - An unknown `JEV_PROVIDER` now errors instead of silently falling through to auto-detection; the no-provider diagnostic names all four credential sets. - Every answer is validated at a shared boundary before tokens are credited or an action executes: missing or malformed answers error the run, and transport errors from built-in providers no longer include raw response bodies. diff --git a/package-lock.json b/package-lock.json index 5624008..cb27858 100644 --- a/package-lock.json +++ b/package-lock.json @@ -14,6 +14,7 @@ "@ai-sdk/google": "2.0.9", "@ai-sdk/openai": "^2.0.0", "@ai-sdk/openai-compatible": "^1.0.0", + "@jkudish/jev-agent-tools": "^0.1.0", "@modelcontextprotocol/sdk": "^1.17.0", "@typesafe-ai/sdk": "^0.6.0", "ai": "^7.0.105", @@ -235,6 +236,15 @@ "hono": "^4" } }, + "node_modules/@jkudish/jev-agent-tools": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/@jkudish/jev-agent-tools/-/jev-agent-tools-0.1.0.tgz", + "integrity": "sha512-Sjiaj/U4J4XqrXQMzI14DWlPAb3h1b+kgVwhnMSlvvtp13c+T7NjZCqYLa4LipGDK0iE6bh1AenZIbj+9qhj4g==", + "license": "MIT", + "engines": { + "node": ">=22" + } + }, "node_modules/@mixmark-io/domino": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/@mixmark-io/domino/-/domino-2.2.0.tgz", diff --git a/package.json b/package.json index d5284d9..b414616 100644 --- a/package.json +++ b/package.json @@ -58,6 +58,7 @@ "@ai-sdk/google": "2.0.9", "@ai-sdk/openai": "^2.0.0", "@ai-sdk/openai-compatible": "^1.0.0", + "@jkudish/jev-agent-tools": "^0.1.0", "@modelcontextprotocol/sdk": "^1.17.0", "@typesafe-ai/sdk": "^0.6.0", "ai": "^7.0.105", diff --git a/src/provider.ts b/src/provider.ts index fe11c37..2fedc53 100644 --- a/src/provider.ts +++ b/src/provider.ts @@ -1,41 +1,12 @@ -import { typesafe } from "./transports/typesafe.js"; -import { openrouter } from "./transports/openrouter.js"; -import { cloudflare } from "./transports/cloudflare.js"; -import { vercel } from "./transports/vercel.js"; +import { ask, resolveTransport } from "@jkudish/jev-agent-tools"; +import type { JevAnswer, JevTransport, JevTransportInput, JevTransportReply } from "@jkudish/jev-agent-tools"; -export interface JevTransportInput { - state: unknown; - questions: Record; - model: string; - signal: AbortSignal; -} - -export interface JevTransportReply { - answers: unknown; - usage: { input_tokens: number; output_tokens: number }; - model: string; -} - -export interface JevTransport { - readonly name: string; - ask(input: JevTransportInput): Promise; -} - -export interface BuiltinDriver { - readonly name: "typesafe" | "openrouter" | "cloudflare" | "vercel"; - isConfigured(env: NodeJS.ProcessEnv): boolean; - assertConfigured(env: NodeJS.ProcessEnv): void; - create(env: NodeJS.ProcessEnv): JevTransport; -} - -export type JevAnswer = - | { type: "noul"; noul: number } - | { type: "choice"; choice: string; probabilities: Record; confidence: number | null } - | { type: "score"; score: number; probabilities: Record; confidence: number | null }; +export { resolveTransport }; +export type { JevAnswer, JevTransport, JevTransportInput, JevTransportReply }; export interface AskResult { answers: Record; - usage: { input_tokens: number; output_tokens: number }; + usage: JevTransportReply["usage"]; provider: string; model: string; } @@ -43,93 +14,15 @@ export interface AskResult { /** Internal marker so a failed judgment cannot be mistaken for a page action error. */ export class InvalidJevAnswer extends Error {} -const drivers: readonly BuiltinDriver[] = [typesafe, openrouter, cloudflare, vercel]; - -export function resolveTransport(env: NodeJS.ProcessEnv = process.env): JevTransport { - const explicit = (env.JEV_PROVIDER ?? "auto").toLowerCase(); - if (explicit !== "auto") { - const driver = drivers.find((candidate) => candidate.name === explicit); - if (!driver) throw new Error("Unknown JEV_PROVIDER; choose typesafe, openrouter, cloudflare, vercel, or auto."); - try { - driver.assertConfigured(env); - } catch (error) { - throw new Error(`JEV_PROVIDER=${driver.name} but ${(error as Error).message}`); - } - return driver.create(env); - } - const driver = drivers.find((candidate) => candidate.isConfigured(env)); - if (!driver) { - throw new Error("No TYPESAFE_API_KEY, OPENROUTER_API_KEY (sk-or-), Cloudflare token (CLOUDFLARE_API_TOKEN or JEV_CLOUDFLARE_API_TOKEN) + CLOUDFLARE_ACCOUNT_ID, or AI_GATEWAY_API_KEY found. Set one, or JEV_PROVIDER to choose explicitly."); - } - return driver.create(env); -} - -function record(value: unknown): value is Record { - return value !== null && typeof value === "object" && !Array.isArray(value); -} - -function distribution(value: unknown, keys: string[], fail: (reason: string) => never): Record { - if (!record(value) || Object.keys(value).length !== keys.length || keys.some((key) => !Object.hasOwn(value, key))) { - fail("distribution must contain exactly the criteria keys"); - } - let sum = 0; - const probabilities: Record = {}; - for (const key of keys) { - const probability = value[key]; - if (typeof probability !== "number" || !Number.isFinite(probability) || probability < 0 || probability > 1) { - fail("distribution must contain finite probabilities in [0,1]"); - } - probabilities[key] = probability as number; - sum += probability as number; - } - // Upstream distributions are rounded; permit a one-percent sum drift and - // a 0.001 selection tie, but not a different winner. - if (Math.abs(sum - 1) > 0.01) fail("distribution must sum to approximately 1"); - return probabilities; -} - export async function askJev(transport: JevTransport, input: JevTransportInput): Promise { - // The label is caller-supplied for injected transports, not a wire field. - const provider = typeof transport.name === "string" && transport.name.trim() ? transport.name : "unknown"; - const fail = (id: string, reason: string): never => { - throw new InvalidJevAnswer(`Jev provider ${provider} question ${id}: ${reason}`); - }; - const reply = await transport.ask(input); - if (!record(reply) || !record(reply.answers)) fail("", "answers must be an object"); - const answers = reply.answers as Record; - const ids = Object.keys(input.questions); - for (const id of ids) if (!Object.hasOwn(answers, id)) fail(id, "missing answer"); - if (Object.keys(answers).length !== ids.length) fail("", "unexpected answer ID"); - const validated: Record = {}; - for (const id of ids) { - const question = input.questions[id]; - const answer = answers[id]; - const invalid = (reason: string): never => fail(id, reason); - if (!record(question) || !record(answer) || answer.type !== question.type) throw new InvalidJevAnswer(`Jev provider ${provider} question ${id}: missing answer or wrong type`); - if (question.type === "noul") { - if (typeof answer.noul !== "number" || !Number.isFinite(answer.noul) || answer.noul < 0 || answer.noul > 1) invalid("noul must be finite in [0,1]"); - validated[id] = { type: "noul", noul: answer.noul as number }; - continue; - } - const keys = question.type === "score" && Array.isArray(question.criteria) - ? question.criteria.map((_, index) => String(index)) - : question.type === "choice" && record(question.criteria) ? Object.keys(question.criteria) : null; - if (!keys?.length) invalid("invalid question criteria"); - const probabilities = distribution(answer.probabilities, keys!, invalid); - const confidence = answer.confidence === undefined ? null : answer.confidence; - if (confidence !== null && (typeof confidence !== "number" || !Number.isFinite(confidence))) invalid("confidence must be finite or null"); - if (question.type === "choice") { - if (typeof answer.choice !== "string" || !keys!.includes(answer.choice)) invalid("choice is outside criteria"); - if (probabilities[answer.choice as string] + 0.001 < Math.max(...Object.values(probabilities))) invalid("choice is not a distribution maximum"); - validated[id] = { type: "choice", choice: answer.choice as string, probabilities, confidence: confidence as number | null }; - } else if (question.type === "score") { - if (typeof answer.score !== "number" || !Number.isInteger(answer.score) || !keys!.includes(String(answer.score))) invalid("score is outside criteria levels"); - validated[id] = { type: "score", score: answer.score as number, probabilities, confidence: confidence as number | null }; - } else invalid("unsupported question type"); - } - if (!record(reply.usage) || !Number.isSafeInteger(reply.usage.input_tokens) || (reply.usage.input_tokens as number) < 0 || !Number.isSafeInteger(reply.usage.output_tokens) || (reply.usage.output_tokens as number) < 0) { - fail("", "usage counters must be non-negative safe integers"); + const result = await ask(input, { transport }); + if (!result.ok) { + if (input.signal.aborted) throw input.signal.reason; + if (result.code === "request_failed") throw new Error(result.message); + throw new InvalidJevAnswer(result.message); } - if (typeof reply.model !== "string" || !reply.model.trim()) fail("", "effective model must be nonempty"); - return { answers: validated, usage: reply.usage as AskResult["usage"], provider, model: reply.model as string }; + // The package redacts unrecognized names; successful injected transports + // still report the caller's name, as the public navigate() API promises. + const provider = typeof transport.name === "string" && transport.name.trim() ? transport.name : "unknown"; + return { answers: result.answer, usage: result.usage, provider, model: result.model }; } diff --git a/src/transports/cloudflare.ts b/src/transports/cloudflare.ts deleted file mode 100644 index b160b56..0000000 --- a/src/transports/cloudflare.ts +++ /dev/null @@ -1,61 +0,0 @@ -import type { BuiltinDriver } from "../provider.js"; - -export const cloudflare: BuiltinDriver = { - name: "cloudflare", - isConfigured: (env) => Boolean((env.JEV_CLOUDFLARE_API_TOKEN || env.CLOUDFLARE_API_TOKEN) && env.CLOUDFLARE_ACCOUNT_ID), - assertConfigured(env) { - if (!this.isConfigured(env)) throw new Error("a Cloudflare API token (CLOUDFLARE_API_TOKEN or JEV_CLOUDFLARE_API_TOKEN) and CLOUDFLARE_ACCOUNT_ID are not both set."); - }, - create(env) { - this.assertConfigured(env); - const token = env.JEV_CLOUDFLARE_API_TOKEN || env.CLOUDFLARE_API_TOKEN; - const account = env.CLOUDFLARE_ACCOUNT_ID!; - return { - name: this.name, - async ask({ state, questions, model, signal }) { - const slug = model.startsWith("typesafe/") ? model : `typesafe/${model === "jev-latest" ? "jev" : model}`; - const response = await fetch(`https://api.cloudflare.com/client/v4/accounts/${account}/ai/run`, { - method: "POST", - headers: { Authorization: `Bearer ${token}`, "Content-Type": "application/json" }, - body: JSON.stringify({ model: slug, input: { state, questions } }), - signal, - }).catch(() => { - if (signal.aborted) throw signal.reason; - throw new Error("Cloudflare AI run HTTP unavailable (network error; 0 response bytes)"); - }); - const raw = await response.text().catch(() => { - if (signal.aborted) throw signal.reason; - throw new Error(`Cloudflare AI run HTTP ${response.status} (body read error; 0 response bytes)`); - }); - const bytes = Buffer.byteLength(raw); - if (!response.ok) throw new Error(`Cloudflare AI run HTTP ${response.status} (request failed; ${bytes} response bytes)`); - let body: any; - try { - body = JSON.parse(raw); - } catch { - throw new Error(`Cloudflare AI run HTTP ${response.status} (invalid JSON; ${bytes} response bytes)`); - } - if (!body || typeof body !== "object" || Array.isArray(body)) throw new Error(`Cloudflare AI run HTTP ${response.status} (invalid envelope; ${bytes} response bytes)`); - if (body.success === false) throw new Error(`Cloudflare AI run HTTP ${response.status} (API unsuccessful; ${bytes} response bytes)`); - // The v4 envelope double-nests the model output under result.result. - const outer = body.result; - if (outer && typeof outer.state === "string" && outer.state !== "Completed") { - throw new Error(`Cloudflare AI run HTTP ${response.status} (non-Completed state; ${bytes} response bytes)`); - } - const payload = outer?.result ?? outer ?? body; - const usage = payload?.usage; - if (usage !== undefined && (typeof usage !== "object" || usage === null || Array.isArray(usage))) { - throw new Error(`Cloudflare AI run HTTP ${response.status} (invalid usage; ${bytes} response bytes)`); - } - return { - answers: payload?.answers, - usage: { - input_tokens: usage && Object.hasOwn(usage, "input_tokens") ? usage.input_tokens : 0, - output_tokens: usage && Object.hasOwn(usage, "output_tokens") ? usage.output_tokens : 0, - }, - model: payload?.model ?? slug, - }; - }, - }; - }, -}; diff --git a/src/transports/openrouter.ts b/src/transports/openrouter.ts deleted file mode 100644 index 0d20289..0000000 --- a/src/transports/openrouter.ts +++ /dev/null @@ -1,64 +0,0 @@ -import type { BuiltinDriver } from "../provider.js"; - -const TITLE = "jev-browser"; -const REFERER = "https://github.com/jkudish/jev-browser"; - -export const openrouter: BuiltinDriver = { - name: "openrouter", - isConfigured: (env) => /^sk-or-/.test(env.OPENROUTER_API_KEY ?? ""), - assertConfigured(env) { - if (!this.isConfigured(env)) throw new Error("OPENROUTER_API_KEY is not set or not an sk-or- key."); - }, - create(env) { - this.assertConfigured(env); - const key = env.OPENROUTER_API_KEY!; - return { - name: this.name, - async ask({ state, questions, model, signal }) { - const effective = model === "jev-latest" ? "jev-1.13" : model; - const slug = effective.startsWith("typesafe/") ? effective : `typesafe/${effective}`; - const response = await fetch("https://openrouter.ai/api/alpha/decisions", { - method: "POST", - headers: { - Authorization: `Bearer ${key}`, - "Content-Type": "application/json", - "HTTP-Referer": REFERER, - "X-Title": TITLE, - "X-OpenRouter-Title": TITLE, - }, - body: JSON.stringify({ model: slug, state, questions }), - signal, - }).catch(() => { - if (signal.aborted) throw signal.reason; - throw new Error("OpenRouter decisions API HTTP unavailable (network error; 0 response bytes)"); - }); - const raw = await response.text().catch(() => { - if (signal.aborted) throw signal.reason; - throw new Error(`OpenRouter decisions API HTTP ${response.status} (body read error; 0 response bytes)`); - }); - const bytes = Buffer.byteLength(raw); - if (!response.ok) throw new Error(`OpenRouter decisions API HTTP ${response.status} (request failed; ${bytes} response bytes)`); - let body: any; - try { - body = JSON.parse(raw); - } catch { - throw new Error(`OpenRouter decisions API HTTP ${response.status} (invalid JSON; ${bytes} response bytes)`); - } - if (!body || typeof body !== "object" || Array.isArray(body)) throw new Error(`OpenRouter decisions API HTTP ${response.status} (invalid envelope; ${bytes} response bytes)`); - const usage = body.usage; - if (usage !== undefined && (typeof usage !== "object" || usage === null || Array.isArray(usage))) { - throw new Error(`OpenRouter decisions API HTTP ${response.status} (invalid usage; ${bytes} response bytes)`); - } - return { - answers: body.answers, - // The decisions endpoint does not document usage; absence means zero. - usage: { - input_tokens: usage && Object.hasOwn(usage, "input_tokens") ? usage.input_tokens : 0, - output_tokens: usage && Object.hasOwn(usage, "output_tokens") ? usage.output_tokens : 0, - }, - model: slug, - }; - }, - }; - }, -}; diff --git a/src/transports/typesafe.ts b/src/transports/typesafe.ts deleted file mode 100644 index 6919c0a..0000000 --- a/src/transports/typesafe.ts +++ /dev/null @@ -1,47 +0,0 @@ -import { TypeSafeClient } from "@typesafe-ai/sdk"; -import type { BuiltinDriver } from "../provider.js"; - -export const typesafe: BuiltinDriver = { - name: "typesafe", - isConfigured: (env) => Boolean(env.TYPESAFE_API_KEY), - assertConfigured(env) { - if (!this.isConfigured(env)) throw new Error("TYPESAFE_API_KEY is not set."); - }, - create(env) { - this.assertConfigured(env); - const client = new TypeSafeClient({ - apiKey: env.TYPESAFE_API_KEY!, - baseURL: env.TYPESAFE_BASE_URL || "https://api.typesafe.ai", - }); - return { - name: this.name, - async ask({ state, questions, model, signal }) { - let response: { answers: unknown; usage?: { input_tokens?: number; output_tokens?: number } }; - try { - response = await ( - client.systemOne as unknown as ( - payload: { state: unknown; questions: Record; model: string }, - options: { signal: AbortSignal }, - ) => Promise - )({ state, questions, model }, { signal }); - } catch (error) { - if (signal.aborted) throw signal.reason; - const status = (error as { status?: unknown }).status; - throw new Error(`TypeSafe API ${typeof status === "number" ? `HTTP ${status}` : "request failed"} (response omitted)`); - } - const usage = response.usage; - if (usage !== undefined && (typeof usage !== "object" || usage === null || Array.isArray(usage))) { - throw new Error("TypeSafe API invalid usage (response omitted)"); - } - return { - answers: response.answers, - usage: { - input_tokens: usage && Object.hasOwn(usage, "input_tokens") ? usage.input_tokens as number : 0, - output_tokens: usage && Object.hasOwn(usage, "output_tokens") ? usage.output_tokens as number : 0, - }, - model, - }; - }, - }; - }, -}; diff --git a/src/transports/vercel.ts b/src/transports/vercel.ts deleted file mode 100644 index edc4569..0000000 --- a/src/transports/vercel.ts +++ /dev/null @@ -1,64 +0,0 @@ -import { createGateway, experimental_evaluate } from "ai"; -import type { BuiltinDriver } from "../provider.js"; - -// Internal factory seam: tests provide evaluate without replacing ESM exports. -export function createVercelDriver(evaluate: typeof experimental_evaluate = experimental_evaluate): BuiltinDriver { - return { - name: "vercel", - isConfigured: (env) => Boolean(env.AI_GATEWAY_API_KEY), - assertConfigured(env) { - if (!this.isConfigured(env)) throw new Error("AI_GATEWAY_API_KEY is not set."); - }, - create(env) { - this.assertConfigured(env); - const gateway = createGateway({ apiKey: env.AI_GATEWAY_API_KEY! }); - return { - name: this.name, - async ask({ state, questions, model, signal }) { - const adaptedQuestions: Record = {}; - for (const [id, question] of Object.entries(questions)) { - const q = question as { type: string; instructions?: unknown; criteria?: unknown }; - adaptedQuestions[id] = { type: q.type === "noul" ? "boolean" : q.type, instructions: q.instructions, criteria: q.criteria }; - } - const effective = model.startsWith("typesafe-ai/") ? model : "typesafe-ai/jev"; - let result: Awaited>; - try { - result = await evaluate({ model: gateway.evaluation(effective), state: state as any, questions: adaptedQuestions as any, abortSignal: signal }); - } catch (error) { - if (signal.aborted) throw signal.reason; - const status = (error as { statusCode?: unknown }).statusCode; - throw new Error(`Vercel AI Gateway ${typeof status === "number" ? `HTTP ${status}` : "request failed"} (response omitted)`); - } - const usage = result.usage; - if (usage !== undefined && (typeof usage !== "object" || usage === null || Array.isArray(usage))) { - throw new Error("Vercel AI Gateway invalid usage (response omitted)"); - } - return { - answers: adaptVercelAnswers(result.answers, result.providerMetadata), - usage: { - input_tokens: usage && Object.hasOwn(usage, "inputTokens") ? usage.inputTokens as number : 0, - output_tokens: usage && Object.hasOwn(usage, "outputTokens") ? usage.outputTokens as number : 0, - }, - model: effective, - }; - }, - }; - }, - }; -} - -export function adaptVercelAnswers(answers: unknown, metadata: unknown): unknown { - if (!answers || typeof answers !== "object" || Array.isArray(answers)) return answers; - const confidence = (metadata as any)?.typesafe?.confidence ?? {}; - const adapted: Record = {}; - for (const [id, answer] of Object.entries(answers)) { - const value = answer as any; - if (value?.type === "boolean") adapted[id] = { type: "noul", noul: value.probability }; - else if (value?.type === "choice") adapted[id] = { type: "choice", choice: value.choice, probabilities: value.probabilities, confidence: confidence[id] ?? null }; - else if (value?.type === "score") adapted[id] = { type: "score", score: value.score, probabilities: value.probabilities, confidence: confidence[id] ?? null }; - else adapted[id] = answer; - } - return adapted; -} - -export const vercel = createVercelDriver(); diff --git a/test/transports.test.mjs b/test/transports.test.mjs index 5004d42..b5f1eaa 100644 --- a/test/transports.test.mjs +++ b/test/transports.test.mjs @@ -1,11 +1,7 @@ import assert from "node:assert/strict"; import { test } from "node:test"; import { chromium } from "playwright"; -import { askJev, resolveTransport } from "../dist/provider.js"; -import { typesafe } from "../dist/transports/typesafe.js"; -import { openrouter } from "../dist/transports/openrouter.js"; -import { cloudflare } from "../dist/transports/cloudflare.js"; -import { createVercelDriver, adaptVercelAnswers } from "../dist/transports/vercel.js"; +import { askJev, InvalidJevAnswer, resolveTransport } from "../dist/provider.js"; import { navigate } from "../dist/library.js"; const signal = new AbortController().signal; @@ -14,6 +10,7 @@ const answers = { item: { type: "choice", choice: "beta", probabilities: { alpha const input = { state: { title: "Example" }, questions, model: "jev-latest", signal }; const usage = { input_tokens: 13, output_tokens: 3 }; const carrier = (override = {}) => ({ name: "fixture", ask: async () => ({ answers, usage, model: "effective", ...override }) }); +const builtin = (name, env) => resolveTransport({ ...env, JEV_PROVIDER: name }); async function withFetch(fn, run) { const original = globalThis.fetch; @@ -80,7 +77,7 @@ test("facade validates the complete answer contract before usage can be credited [{ ...answers, item: { ...answers.item, confidence: Infinity } }, "item", /confidence/], [{ ...answers, yes: { type: "noul", noul: 1.1 } }, "yes", /noul/], ]) { - await assert.rejects(() => askJev(carrier({ answers: mutated }), input), (error) => error.message.includes(`provider fixture question ${id}`) && reason.test(error.message)); + await assert.rejects(() => askJev(carrier({ answers: mutated }), input), (error) => error instanceof InvalidJevAnswer && error.message.includes(`provider fixture question ${id}`) && reason.test(error.message)); } await assert.rejects(() => askJev(carrier({ usage: { input_tokens: -1, output_tokens: 0 } }), input), /provider fixture question .*usage/); await assert.rejects(() => askJev(carrier({ usage: { input_tokens: undefined, output_tokens: 0 } }), input), /provider fixture question .*usage/); @@ -93,13 +90,23 @@ test("facade validates the complete answer contract before usage can be credited await assert.rejects(() => askJev(carrier({ answers: { rank: { ...scoreReply.rank, score: 3 } } }), score), /question rank.*score is outside/); }); +test("transport failure stays distinct from invalid answers and cancellation keeps its reason", async () => { + const failed = { name: "fixture", ask: async () => { throw new Error("body-secret"); } }; + await assert.rejects(() => askJev(failed, input), (error) => !(error instanceof InvalidJevAnswer) && /request failed/.test(error.message) && !/body-secret/.test(error.message)); + + const controller = new AbortController(); + const reason = new Error("deadline-exceeded"); + const cancelled = { name: "fixture", ask: async () => { controller.abort(reason); throw reason; } }; + await assert.rejects(() => askJev(cancelled, { ...input, signal: controller.signal }), (error) => error === reason); +}); + test("TypeSafe client binds key and base URL at creation, forwards request and cancellation", async () => { const calls = []; await withFetch(async (url, init) => { calls.push({ url, init }); return Response.json({ answers, usage }); }, async () => { - const transport = typesafe.create({ TYPESAFE_API_KEY: "ts-secret", TYPESAFE_BASE_URL: "https://local.typesafe.test" }); + const transport = builtin("typesafe", { TYPESAFE_API_KEY: "ts-secret", TYPESAFE_BASE_URL: "https://local.typesafe.test" }); const reply = await askJev(transport, input); assert.deepEqual(reply.usage, usage); assert.equal(reply.model, "jev-latest"); @@ -111,7 +118,7 @@ test("TypeSafe client binds key and base URL at creation, forwards request and c assert.equal(calls[0].init.signal.aborted, false); }); await withFetch(async () => Response.json({ error: "body-secret" }, { status: 400 }), async () => { - await assert.rejects(() => typesafe.create({ TYPESAFE_API_KEY: "ts-secret" }).ask(input), (error) => !/body-secret|ts-secret/.test(error.message) && /TypeSafe API HTTP 400/.test(error.message)); + await assert.rejects(() => askJev(builtin("typesafe", { TYPESAFE_API_KEY: "ts-secret" }), input), (error) => !/body-secret|ts-secret/.test(error.message) && /request failed/.test(error.message) && !(error instanceof InvalidJevAnswer)); }); }); @@ -121,7 +128,7 @@ test("OpenRouter maps latest and pinned slugs, sends exact envelope and redacts calls.push({ url, init }); return Response.json({ answers, usage }); }, async () => { - const transport = openrouter.create({ OPENROUTER_API_KEY: "sk-or-secret" }); + const transport = builtin("openrouter", { OPENROUTER_API_KEY: "sk-or-secret" }); assert.equal((await askJev(transport, input)).model, "typesafe/jev-1.13"); assert.equal((await askJev(transport, { ...input, model: "typesafe/jev-1.12" })).model, "typesafe/jev-1.12"); assert.equal(calls[0].url, "https://openrouter.ai/api/alpha/decisions"); @@ -132,14 +139,14 @@ test("OpenRouter maps latest and pinned slugs, sends exact envelope and redacts }); for (const response of [new Response("body-secret", { status: 403 }), new Response("body-secret", { status: 200 })]) { await withFetch(async () => response, async () => { - await assert.rejects(() => openrouter.create({ OPENROUTER_API_KEY: "sk-or-secret" }).ask(input), (error) => /OpenRouter decisions API HTTP/.test(error.message) && !/body-secret|sk-or-secret/.test(error.message)); + await assert.rejects(() => askJev(builtin("openrouter", { OPENROUTER_API_KEY: "sk-or-secret" }), input), (error) => /request failed/.test(error.message) && !/body-secret|sk-or-secret/.test(error.message)); }); } await withFetch(async () => Response.json({ usage }), async () => { - await assert.rejects(() => askJev(openrouter.create({ OPENROUTER_API_KEY: "sk-or-secret" }), input), /question .*answers/); + await assert.rejects(() => askJev(builtin("openrouter", { OPENROUTER_API_KEY: "sk-or-secret" }), input), /question .*answers/); }); await withFetch(async () => { throw new Error("body-secret sk-or-secret"); }, async () => { - await assert.rejects(() => openrouter.create({ OPENROUTER_API_KEY: "sk-or-secret" }).ask(input), (error) => /HTTP unavailable.*network error/.test(error.message) && !/body-secret|sk-or-secret/.test(error.message)); + await assert.rejects(() => askJev(builtin("openrouter", { OPENROUTER_API_KEY: "sk-or-secret" }), input), (error) => /request failed/.test(error.message) && !/body-secret|sk-or-secret/.test(error.message) && !(error instanceof InvalidJevAnswer)); }); }); @@ -150,7 +157,7 @@ test("Cloudflare token priority, double envelope, state, usage, and safe errors" calls.push({ url, init }); return Response.json({ success: true, result: { state: "Completed", result: { answers, usage, model: "typesafe/jev" } } }); }, async () => { - const transport = cloudflare.create(env); + const transport = builtin("cloudflare", env); assert.deepEqual((await askJev(transport, input)).usage, usage); assert.equal(calls[0].url, "https://api.cloudflare.com/client/v4/accounts/account/ai/run"); assert.deepEqual(calls[0].init.headers, { Authorization: "Bearer high-secret", "Content-Type": "application/json" }); @@ -161,47 +168,51 @@ test("Cloudflare token priority, double envelope, state, usage, and safe errors" }); for (const response of [new Response("body-secret", { status: 401 }), Response.json({ success: false, errors: ["body-secret"] }), Response.json({ result: { state: "Failed body-secret" } }), new Response("body-secret", { status: 200 })]) { await withFetch(async () => response, async () => { - await assert.rejects(() => cloudflare.create(env).ask(input), (error) => /Cloudflare AI run HTTP/.test(error.message) && !/body-secret|high-secret|low-secret/.test(error.message)); + await assert.rejects(() => askJev(builtin("cloudflare", env), input), (error) => /request failed/.test(error.message) && !/body-secret|high-secret|low-secret/.test(error.message)); }); } await withFetch(async () => { throw new Error("body-secret high-secret"); }, async () => { - await assert.rejects(() => cloudflare.create(env).ask(input), (error) => /HTTP unavailable.*network error/.test(error.message) && !/body-secret|high-secret/.test(error.message)); + await assert.rejects(() => askJev(builtin("cloudflare", env), input), (error) => /request failed/.test(error.message) && !/body-secret|high-secret/.test(error.message) && !(error instanceof InvalidJevAnswer)); }); }); -test("Vercel factory forwards evaluate request and pure adaptation preserves confidence", async () => { +test("Vercel forwards evaluation request and adapts boolean and confidence", async () => { let call; const raw = { item: { type: "choice", choice: "beta", probabilities: { alpha: 0.2, beta: 0.8 } }, yes: { type: "boolean", probability: 0.7 } }; const result = { answers: raw, usage: { inputTokens: 13, outputTokens: 3 }, providerMetadata: { typesafe: { confidence: { item: 0.92 } } } }; - const factory = createVercelDriver(async (args) => { call = args; return result; }); - const reply = await askJev(factory.create({ AI_GATEWAY_API_KEY: "ai-secret" }), input); - assert.equal(reply.model, "typesafe-ai/jev"); - assert.deepEqual(reply.usage, usage); - assert.equal(call.abortSignal, signal); - assert.equal(call.model.modelId, "typesafe-ai/jev"); - assert.deepEqual(call.questions, { item: { type: "choice", instructions: undefined, criteria: questions.item.criteria }, yes: { type: "boolean", instructions: undefined, criteria: undefined } }); - assert.deepEqual(reply.answers, { item: { ...answers.item, confidence: 0.92 }, yes: answers.yes }); - assert.deepEqual(adaptVercelAnswers({ yes: raw.yes, rank: { type: "score", score: 1, probabilities: { "0": 0.2, "1": 0.8 } } }, {}), { yes: answers.yes, rank: { type: "score", score: 1, probabilities: { "0": 0.2, "1": 0.8 }, confidence: null } }); - const failing = createVercelDriver(async () => { throw Object.assign(new Error("body-secret"), { statusCode: 403 }); }); - await assert.rejects(() => failing.create({ AI_GATEWAY_API_KEY: "ai-secret" }).ask(input), (error) => /Vercel AI Gateway HTTP 403/.test(error.message) && !/body-secret|ai-secret/.test(error.message)); - const malformed = createVercelDriver(async () => ({ ...result, answers: { item: raw.item } })); - await assert.rejects(() => askJev(malformed.create({ AI_GATEWAY_API_KEY: "ai-secret" }), input), /provider vercel question yes.*missing answer/); + const transport = builtin("vercel", { AI_GATEWAY_API_KEY: "ai-secret" }); + await withFetch(async (url, init) => { call = { url, init }; return Response.json(result); }, async () => { + const reply = await askJev(transport, input); + assert.equal(reply.model, "typesafe-ai/jev"); + assert.deepEqual(reply.usage, usage); + assert.equal(call.url, "https://ai-gateway.vercel.sh/v4/ai/evaluation-model"); + assert.equal(call.init.signal, signal); + assert.equal(call.init.headers.Authorization, "Bearer ai-secret"); + assert.equal(call.init.headers["ai-model-id"], "typesafe-ai/jev"); + assert.deepEqual(JSON.parse(call.init.body), { state: input.state, questions: { item: { type: "choice", criteria: questions.item.criteria }, yes: { type: "boolean" } } }); + assert.deepEqual(reply.answers, { item: { ...answers.item, confidence: 0.92 }, yes: answers.yes }); + }); + await withFetch(async () => Response.json({ ...result, answers: { item: raw.item } }), async () => { + await assert.rejects(() => askJev(transport, input), (error) => error instanceof InvalidJevAnswer && /question yes.*missing answer/.test(error.message)); + }); + await withFetch(async () => new Response("body-secret", { status: 403 }), async () => { + await assert.rejects(() => askJev(transport, input), (error) => !(error instanceof InvalidJevAnswer) && /request failed/.test(error.message) && !/body-secret|ai-secret/.test(error.message)); + }); }); test("all adapters distinguish absent usage from malformed containers and present null counters", async () => { const vercelAnswers = { item: answers.item, yes: { type: "boolean", probability: answers.yes.noul } }; for (const [name, field, run] of [ - ["typesafe", "input_tokens", (wire) => withFetch(async () => Response.json({ answers, ...wire }), () => askJev(typesafe.create({ TYPESAFE_API_KEY: "ts-secret" }), input))], - ["openrouter", "input_tokens", (wire) => withFetch(async () => Response.json({ answers, ...wire }), () => askJev(openrouter.create({ OPENROUTER_API_KEY: "sk-or-secret" }), input))], - ["cloudflare", "input_tokens", (wire) => withFetch(async () => Response.json({ result: { state: "Completed", result: { answers, ...wire } } }), () => askJev(cloudflare.create({ CLOUDFLARE_API_TOKEN: "cf-secret", CLOUDFLARE_ACCOUNT_ID: "account" }), input))], - ["vercel", "inputTokens", (wire) => askJev(createVercelDriver(async () => ({ answers: vercelAnswers, ...wire })).create({ AI_GATEWAY_API_KEY: "ai-secret" }), input)], + ["typesafe", "input_tokens", (wire) => withFetch(async () => Response.json({ answers, ...wire }), () => askJev(builtin("typesafe", { TYPESAFE_API_KEY: "ts-secret" }), input))], + ["openrouter", "input_tokens", (wire) => withFetch(async () => Response.json({ answers, ...wire }), () => askJev(builtin("openrouter", { OPENROUTER_API_KEY: "sk-or-secret" }), input))], + ["cloudflare", "input_tokens", (wire) => withFetch(async () => Response.json({ result: { state: "Completed", result: { answers, ...wire } } }), () => askJev(builtin("cloudflare", { CLOUDFLARE_API_TOKEN: "cf-secret", CLOUDFLARE_ACCOUNT_ID: "account" }), input))], + ["vercel", "inputTokens", (wire) => withFetch(async () => Response.json({ answers: vercelAnswers, ...wire }), () => askJev(builtin("vercel", { AI_GATEWAY_API_KEY: "ai-secret" }), input))], ]) { assert.deepEqual((await run({})).usage, { input_tokens: 0, output_tokens: 0 }, name); assert.deepEqual((await run({ usage: {} })).usage, { input_tokens: 0, output_tokens: 0 }, name); const invalidCases = [{ usage: "body-secret" }, { usage: null }, { usage: { [field]: null } }]; - if (name === "vercel") invalidCases.push({ usage: { [field]: undefined } }); // JSON drops undefined properties on the HTTP drivers. for (const invalid of invalidCases) { - await assert.rejects(() => run(invalid), (error) => /usage/.test(error.message) && !error.message.includes("body-secret"), `${name}: ${JSON.stringify(invalid)}`); + await assert.rejects(() => run(invalid), (error) => /usage|request failed/.test(error.message) && !error.message.includes("body-secret"), `${name}: ${JSON.stringify(invalid)}`); } } });