diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..3955af7 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,24 @@ +name: CI + +on: + push: + branches: [main] + pull_request: + +jobs: + test: + runs-on: ubuntu-latest + env: + TYPESAFE_API_KEY: ${{ secrets.TYPESAFE_API_KEY }} + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: 22 + - run: npm ci + - run: npm run typecheck + - run: npm run build + - run: npm test + - name: TypeSafe live smoke (optional) + if: env.TYPESAFE_API_KEY != '' + run: npm run test:live diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..b947077 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +node_modules/ +dist/ diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..ffd332a --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,6 @@ +# Changelog + +## 0.1.0 + +- Extract four run-bound Jev judgment carriers and strict provider selection from jev-browser. +- Return validated answers or typed rejections through a non-throwing Result API, with hermetic tests and optional live TypeSafe smoke. diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..e02c434 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 Joey Kudish + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md index d5dad5d..42c88cd 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,29 @@ -# jev-agent-tools -Shared Jev wire layer: judgment carriers and fail-closed validation used by jev-browser and jev-mcp +# @jkudish/jev-agent-tools + +Shared Jev judgment wire layer for [jev-browser](https://github.com/jkudish/jev-browser) and [jev-mcp](https://github.com/jkudish/jev-mcp). This package only selects a carrier, sends a judgment, and validates its answer. It has no runtime dependencies, Playwright, MCP SDK, or AI SDK. + +## Result contract + +`ask(input, config?)` returns `Promise<{ ok: true, answer, usage, model, provider } | { ok: false, code, message }>` and never throws a verdict. Transport failures return `request_failed`; invalid responses return specific codes such as `answer_id_mismatch`, `invalid_distribution`, `invalid_noul`, `invalid_usage`, and `invalid_model`. Configuration errors use `configuration_error`. Messages do not include response bodies or credentials. The two consumers need different error behavior: jev-browser can map `!ok` to its own exception, while jev-mcp can map `!ok` to `invalid_response`. + +```js +import { ask } from "@jkudish/jev-agent-tools"; + +const result = await ask({ + state: "A customer asks for a refund of a duplicate charge.", + questions: { refund: { type: "noul", instructions: "Is a refund requested?" } }, + model: "jev-latest", + signal: new AbortController().signal, +}); + +if (!result.ok) console.error(result.code, result.message); +else console.log(result.answer.refund.noul, result.usage, result.model); +``` + +## Providers + +Auto-selection precedence: TypeSafe (`TYPESAFE_API_KEY`, optional `TYPESAFE_BASE_URL`), OpenRouter (`OPENROUTER_API_KEY` beginning `sk-or-`), Cloudflare (`JEV_CLOUDFLARE_API_TOKEN` preferred over `CLOUDFLARE_API_TOKEN`, plus `CLOUDFLARE_ACCOUNT_ID`), then Vercel AI Gateway (`AI_GATEWAY_API_KEY`). Set `JEV_PROVIDER` to `typesafe`, `openrouter`, `cloudflare`, `vercel`, or `auto` to select strictly. Unknown names and missing credentials for forced providers fail rather than falling through. Without a provider, the diagnostic names every supported credential variable. `config.env` accepts an injectable environment record, and `config.transport` accepts a run-bound transport for callers that own one. + +Models retain the browser driver's mappings: OpenRouter maps `jev-latest` to `typesafe/jev-1.13`; Cloudflare maps it to `typesafe/jev`; Vercel selects `typesafe-ai/jev` unless given a `typesafe-ai/` model. The direct TypeSafe driver sends the supplied model (usually `jev-latest`). + +Validation requires exactly the requested answer IDs and criterion IDs, finite probabilities in [0,1] summing to within 0.01 of 1, and a selected Choice maximum within a 0.001 tie tolerance. Noul values must be in [0,1], confidence finite or null, usage counters non-negative safe integers, and the effective model nonempty. Invalid responses yield `ok: false` before any result usage is credited. diff --git a/package-lock.json b/package-lock.json new file mode 100644 index 0000000..2289ff0 --- /dev/null +++ b/package-lock.json @@ -0,0 +1,51 @@ +{ + "name": "@jkudish/jev-agent-tools", + "version": "0.1.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "@jkudish/jev-agent-tools", + "version": "0.1.0", + "license": "MIT", + "devDependencies": { + "@types/node": "^22.0.0", + "typescript": "^5.6.0" + }, + "engines": { + "node": ">=22" + } + }, + "node_modules/@types/node": { + "version": "22.20.4", + "resolved": "https://registry.npmjs.org/@types/node/-/node-22.20.4.tgz", + "integrity": "sha512-zJRE40jpHtKqE/C4fgHrAKQLJuSpzEnP9ff9Y7YtoR3Wd2pwqzlekDeEuUQXjRd+QCYnVnNwuJYmhdk9XV8gvA==", + "dev": true, + "license": "MIT", + "dependencies": { + "undici-types": "~6.21.0" + } + }, + "node_modules/typescript": { + "version": "5.9.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", + "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, + "node_modules/undici-types": { + "version": "6.21.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz", + "integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==", + "dev": true, + "license": "MIT" + } + } +} diff --git a/package.json b/package.json new file mode 100644 index 0000000..65899f2 --- /dev/null +++ b/package.json @@ -0,0 +1,22 @@ +{ + "name": "@jkudish/jev-agent-tools", + "version": "0.1.0", + "description": "Shared Jev judgment transports and fail-closed Result validation", + "type": "module", + "license": "MIT", + "author": "Joey Kudish", + "repository": { "type": "git", "url": "git+https://github.com/jkudish/jev-agent-tools.git" }, + "main": "./dist/index.js", + "types": "./dist/index.d.ts", + "exports": { ".": { "types": "./dist/index.d.ts", "default": "./dist/index.js" } }, + "files": ["dist", "README.md", "LICENSE", "CHANGELOG.md"], + "engines": { "node": ">=22" }, + "scripts": { + "typecheck": "tsc --noEmit", + "build": "tsc", + "test": "node --test test/transports.test.mjs", + "test:live": "node --test test/live.test.mjs" + }, + "devDependencies": { "typescript": "^5.6.0", "@types/node": "^22.0.0" }, + "publishConfig": { "access": "public" } +} diff --git a/src/index.ts b/src/index.ts new file mode 100644 index 0000000..aac669e --- /dev/null +++ b/src/index.ts @@ -0,0 +1,2 @@ +export { ask, resolveTransport } from "./provider.js"; +export type { AskConfig, AskResult, Env, JevAnswer, JevTransport, JevTransportInput, JevTransportReply, RejectionCode } from "./provider.js"; diff --git a/src/provider.ts b/src/provider.ts new file mode 100644 index 0000000..81292c4 --- /dev/null +++ b/src/provider.ts @@ -0,0 +1,149 @@ +import { typesafe } from "./transports/typesafe.js"; +import { openrouter } from "./transports/openrouter.js"; +import { cloudflare } from "./transports/cloudflare.js"; +import { vercel } from "./transports/vercel.js"; + +export interface JevTransportInput { + state: unknown; + questions: Record; + model: string; + signal: AbortSignal; +} + +export interface JevTransportReply { + answers: unknown; + usage: { input_tokens: number; output_tokens: number }; + model: string; +} + +export interface JevTransport { + readonly name: string; + ask(input: JevTransportInput): Promise; +} + +export interface BuiltinDriver { + readonly name: "typesafe" | "openrouter" | "cloudflare" | "vercel"; + isConfigured(env: Env): boolean; + assertConfigured(env: Env): void; + create(env: Env): JevTransport; +} + +export type Env = Record; + +export type JevAnswer = + | { type: "noul"; noul: number } + | { type: "choice"; choice: string; probabilities: Record; confidence: number | null } + | { type: "score"; score: number; probabilities: Record; confidence: number | null }; + +export type RejectionCode = "request_failed" | "configuration_error" | "malformed_answer" | "answer_id_mismatch" | "invalid_criteria" | "invalid_distribution" | "invalid_choice" | "invalid_noul" | "invalid_confidence" | "invalid_usage" | "invalid_model"; + +export type AskResult = + | { ok: true; answer: Record; usage: JevTransportReply["usage"]; model: string; provider: string } + | { ok: false; code: RejectionCode; message: string }; + +export interface AskConfig { env?: Env; transport?: JevTransport } + +const drivers: readonly BuiltinDriver[] = [typesafe, openrouter, cloudflare, vercel]; + +export function resolveTransport(env: Env = process.env): JevTransport { + const explicit = (env.JEV_PROVIDER ?? "auto").toLowerCase(); + if (explicit !== "auto") { + const driver = drivers.find((candidate) => candidate.name === explicit); + if (!driver) throw new Error("Unknown JEV_PROVIDER; choose typesafe, openrouter, cloudflare, vercel, or auto."); + try { + driver.assertConfigured(env); + } catch (error) { + throw new Error(`JEV_PROVIDER=${driver.name} but ${(error as Error).message}`); + } + return driver.create(env); + } + const driver = drivers.find((candidate) => candidate.isConfigured(env)); + if (!driver) { + throw new Error("No TYPESAFE_API_KEY, OPENROUTER_API_KEY (sk-or-), Cloudflare token (CLOUDFLARE_API_TOKEN or JEV_CLOUDFLARE_API_TOKEN) + CLOUDFLARE_ACCOUNT_ID, or AI_GATEWAY_API_KEY found. Set one, or JEV_PROVIDER to choose explicitly."); + } + return driver.create(env); +} + +function record(value: unknown): value is Record { + return value !== null && typeof value === "object" && !Array.isArray(value); +} + +function distribution(value: unknown, keys: string[]): { ok: true; probabilities: Record } | { ok: false; reason: string } { + if (!record(value) || Object.keys(value).length !== keys.length || keys.some((key) => !Object.hasOwn(value, key))) { + return { ok: false, reason: "distribution must contain exactly the criteria keys" }; + } + let sum = 0; + const entries: [string, number][] = []; + for (const key of keys) { + const probability = value[key]; + if (typeof probability !== "number" || !Number.isFinite(probability) || probability < 0 || probability > 1) { + return { ok: false, reason: "distribution must contain finite probabilities in [0,1]" }; + } + entries.push([key, probability]); + sum += probability as number; + } + // Upstream distributions are rounded; permit a one-percent sum drift and + // a 0.001 selection tie, but not a different winner. + if (Math.abs(sum - 1) > 0.01) return { ok: false, reason: "distribution must sum to approximately 1" }; + return { ok: true, probabilities: Object.fromEntries(entries) }; +} + +export async function ask(input: JevTransportInput, config: AskConfig = {}): Promise { + let transport: JevTransport; + try { + transport = config.transport ?? resolveTransport(config.env); + } catch (error) { + // Registry errors are fixed strings; never echo arbitrary driver exceptions. + const message = error instanceof Error && /^(Unknown JEV_PROVIDER|No TYPESAFE_API_KEY|JEV_PROVIDER=)/.test(error.message) + ? error.message : "Jev provider configuration failed"; + return { ok: false, code: "configuration_error", message }; + } + // Injected transport names are untrusted and never included in error text. + const provider = typeof transport.name === "string" && /^(typesafe|openrouter|cloudflare|vercel|fixture)$/.test(transport.name) ? transport.name : "unknown"; + let reply: JevTransportReply; + try { + reply = await transport.ask(input); + } catch { + return { ok: false, code: "request_failed", message: `Jev provider ${provider}: request failed` }; + } + const fail = (id: string, code: RejectionCode, reason: string): AskResult => ({ ok: false, code, message: `Jev provider ${provider} question ${id}: ${reason}` }); + if (!record(input.questions)) return fail("", "invalid_criteria", "questions must be an object"); + if (!record(reply) || !record(reply.answers)) return fail("", "malformed_answer", "answers must be an object"); + const answers = reply.answers as Record; + const ids = Object.keys(input.questions); + for (const id of ids) if (!Object.hasOwn(answers, id)) return fail(id, "answer_id_mismatch", "missing answer"); + if (Object.keys(answers).length !== ids.length) return fail("", "answer_id_mismatch", "unexpected answer ID"); + const validated: [string, JevAnswer][] = []; + for (const id of ids) { + const question = input.questions[id]; + const answer = answers[id]; + if (!record(question) || !record(answer) || answer.type !== question.type) return fail(id, "malformed_answer", "missing answer or wrong type"); + if (question.type === "noul") { + if (typeof answer.noul !== "number" || !Number.isFinite(answer.noul) || answer.noul < 0 || answer.noul > 1) return fail(id, "invalid_noul", "noul must be finite in [0,1]"); + validated.push([id, { type: "noul", noul: answer.noul as number }]); + continue; + } + const keys = question.type === "score" && Array.isArray(question.criteria) + ? question.criteria.map((_, index) => String(index)) + : question.type === "choice" && record(question.criteria) ? Object.keys(question.criteria) : null; + if (!keys?.length) return fail(id, "invalid_criteria", "invalid question criteria"); + const checked = distribution(answer.probabilities, keys); + if (!checked.ok) return fail(id, "invalid_distribution", checked.reason); + const probabilities = checked.probabilities; + const confidence = answer.confidence === undefined ? null : answer.confidence; + if (confidence !== null && (typeof confidence !== "number" || !Number.isFinite(confidence))) return fail(id, "invalid_confidence", "confidence must be finite or null"); + if (question.type === "choice") { + if (typeof answer.choice !== "string" || !keys.includes(answer.choice)) return fail(id, "invalid_choice", "choice is outside criteria"); + if (probabilities[answer.choice as string] + 0.001 < Math.max(...Object.values(probabilities))) return fail(id, "invalid_choice", "choice is not a distribution maximum"); + validated.push([id, { type: "choice", choice: answer.choice as string, probabilities, confidence: confidence as number | null }]); + } else if (question.type === "score") { + if (typeof answer.score !== "number" || !Number.isInteger(answer.score) || !keys.includes(String(answer.score))) return fail(id, "invalid_choice", "score is outside criteria levels"); + validated.push([id, { type: "score", score: answer.score as number, probabilities, confidence: confidence as number | null }]); + } else return fail(id, "malformed_answer", "unsupported question type"); + } + if (!record(reply.usage) || !Number.isSafeInteger(reply.usage.input_tokens) || (reply.usage.input_tokens as number) < 0 || !Number.isSafeInteger(reply.usage.output_tokens) || (reply.usage.output_tokens as number) < 0) { + return fail("", "invalid_usage", "usage counters must be non-negative safe integers"); + } + if (typeof reply.model !== "string" || !reply.model.trim()) return fail("", "invalid_model", "effective model must be nonempty"); + return { ok: true, answer: Object.fromEntries(validated), usage: reply.usage as JevTransportReply["usage"], provider, model: reply.model as string }; +} diff --git a/src/transports/cloudflare.ts b/src/transports/cloudflare.ts new file mode 100644 index 0000000..b160b56 --- /dev/null +++ b/src/transports/cloudflare.ts @@ -0,0 +1,61 @@ +import type { BuiltinDriver } from "../provider.js"; + +export const cloudflare: BuiltinDriver = { + name: "cloudflare", + isConfigured: (env) => Boolean((env.JEV_CLOUDFLARE_API_TOKEN || env.CLOUDFLARE_API_TOKEN) && env.CLOUDFLARE_ACCOUNT_ID), + assertConfigured(env) { + if (!this.isConfigured(env)) throw new Error("a Cloudflare API token (CLOUDFLARE_API_TOKEN or JEV_CLOUDFLARE_API_TOKEN) and CLOUDFLARE_ACCOUNT_ID are not both set."); + }, + create(env) { + this.assertConfigured(env); + const token = env.JEV_CLOUDFLARE_API_TOKEN || env.CLOUDFLARE_API_TOKEN; + const account = env.CLOUDFLARE_ACCOUNT_ID!; + return { + name: this.name, + async ask({ state, questions, model, signal }) { + const slug = model.startsWith("typesafe/") ? model : `typesafe/${model === "jev-latest" ? "jev" : model}`; + const response = await fetch(`https://api.cloudflare.com/client/v4/accounts/${account}/ai/run`, { + method: "POST", + headers: { Authorization: `Bearer ${token}`, "Content-Type": "application/json" }, + body: JSON.stringify({ model: slug, input: { state, questions } }), + signal, + }).catch(() => { + if (signal.aborted) throw signal.reason; + throw new Error("Cloudflare AI run HTTP unavailable (network error; 0 response bytes)"); + }); + const raw = await response.text().catch(() => { + if (signal.aborted) throw signal.reason; + throw new Error(`Cloudflare AI run HTTP ${response.status} (body read error; 0 response bytes)`); + }); + const bytes = Buffer.byteLength(raw); + if (!response.ok) throw new Error(`Cloudflare AI run HTTP ${response.status} (request failed; ${bytes} response bytes)`); + let body: any; + try { + body = JSON.parse(raw); + } catch { + throw new Error(`Cloudflare AI run HTTP ${response.status} (invalid JSON; ${bytes} response bytes)`); + } + if (!body || typeof body !== "object" || Array.isArray(body)) throw new Error(`Cloudflare AI run HTTP ${response.status} (invalid envelope; ${bytes} response bytes)`); + if (body.success === false) throw new Error(`Cloudflare AI run HTTP ${response.status} (API unsuccessful; ${bytes} response bytes)`); + // The v4 envelope double-nests the model output under result.result. + const outer = body.result; + if (outer && typeof outer.state === "string" && outer.state !== "Completed") { + throw new Error(`Cloudflare AI run HTTP ${response.status} (non-Completed state; ${bytes} response bytes)`); + } + const payload = outer?.result ?? outer ?? body; + const usage = payload?.usage; + if (usage !== undefined && (typeof usage !== "object" || usage === null || Array.isArray(usage))) { + throw new Error(`Cloudflare AI run HTTP ${response.status} (invalid usage; ${bytes} response bytes)`); + } + return { + answers: payload?.answers, + usage: { + input_tokens: usage && Object.hasOwn(usage, "input_tokens") ? usage.input_tokens : 0, + output_tokens: usage && Object.hasOwn(usage, "output_tokens") ? usage.output_tokens : 0, + }, + model: payload?.model ?? slug, + }; + }, + }; + }, +}; diff --git a/src/transports/openrouter.ts b/src/transports/openrouter.ts new file mode 100644 index 0000000..0d20289 --- /dev/null +++ b/src/transports/openrouter.ts @@ -0,0 +1,64 @@ +import type { BuiltinDriver } from "../provider.js"; + +const TITLE = "jev-browser"; +const REFERER = "https://github.com/jkudish/jev-browser"; + +export const openrouter: BuiltinDriver = { + name: "openrouter", + isConfigured: (env) => /^sk-or-/.test(env.OPENROUTER_API_KEY ?? ""), + assertConfigured(env) { + if (!this.isConfigured(env)) throw new Error("OPENROUTER_API_KEY is not set or not an sk-or- key."); + }, + create(env) { + this.assertConfigured(env); + const key = env.OPENROUTER_API_KEY!; + return { + name: this.name, + async ask({ state, questions, model, signal }) { + const effective = model === "jev-latest" ? "jev-1.13" : model; + const slug = effective.startsWith("typesafe/") ? effective : `typesafe/${effective}`; + const response = await fetch("https://openrouter.ai/api/alpha/decisions", { + method: "POST", + headers: { + Authorization: `Bearer ${key}`, + "Content-Type": "application/json", + "HTTP-Referer": REFERER, + "X-Title": TITLE, + "X-OpenRouter-Title": TITLE, + }, + body: JSON.stringify({ model: slug, state, questions }), + signal, + }).catch(() => { + if (signal.aborted) throw signal.reason; + throw new Error("OpenRouter decisions API HTTP unavailable (network error; 0 response bytes)"); + }); + const raw = await response.text().catch(() => { + if (signal.aborted) throw signal.reason; + throw new Error(`OpenRouter decisions API HTTP ${response.status} (body read error; 0 response bytes)`); + }); + const bytes = Buffer.byteLength(raw); + if (!response.ok) throw new Error(`OpenRouter decisions API HTTP ${response.status} (request failed; ${bytes} response bytes)`); + let body: any; + try { + body = JSON.parse(raw); + } catch { + throw new Error(`OpenRouter decisions API HTTP ${response.status} (invalid JSON; ${bytes} response bytes)`); + } + if (!body || typeof body !== "object" || Array.isArray(body)) throw new Error(`OpenRouter decisions API HTTP ${response.status} (invalid envelope; ${bytes} response bytes)`); + const usage = body.usage; + if (usage !== undefined && (typeof usage !== "object" || usage === null || Array.isArray(usage))) { + throw new Error(`OpenRouter decisions API HTTP ${response.status} (invalid usage; ${bytes} response bytes)`); + } + return { + answers: body.answers, + // The decisions endpoint does not document usage; absence means zero. + usage: { + input_tokens: usage && Object.hasOwn(usage, "input_tokens") ? usage.input_tokens : 0, + output_tokens: usage && Object.hasOwn(usage, "output_tokens") ? usage.output_tokens : 0, + }, + model: slug, + }; + }, + }; + }, +}; diff --git a/src/transports/typesafe.ts b/src/transports/typesafe.ts new file mode 100644 index 0000000..9df15dd --- /dev/null +++ b/src/transports/typesafe.ts @@ -0,0 +1,46 @@ +import type { BuiltinDriver } from "../provider.js"; + +export const typesafe: BuiltinDriver = { + name: "typesafe", + isConfigured: (env) => Boolean(env.TYPESAFE_API_KEY), + assertConfigured(env) { + if (!this.isConfigured(env)) throw new Error("TYPESAFE_API_KEY is not set."); + }, + create(env) { + this.assertConfigured(env); + const key = env.TYPESAFE_API_KEY!; + const baseURL = env.TYPESAFE_BASE_URL || "https://api.typesafe.ai"; + return { + name: this.name, + async ask({ state, questions, model, signal }) { + let response: { answers: unknown; usage?: { input_tokens?: number; output_tokens?: number } }; + try { + const wire = await fetch(`${baseURL.replace(/\/$/, "")}/v1/systemone`, { + method: "POST", + headers: { Authorization: `Bearer ${key}`, Accept: "application/json", "Content-Type": "application/json" }, + body: JSON.stringify({ state, questions, model }), + signal, + }); + if (!wire.ok) throw { status: wire.status }; + response = await wire.json(); + } catch (error) { + if (signal.aborted) throw signal.reason; + const status = (error as { status?: unknown }).status; + throw new Error(`TypeSafe API ${typeof status === "number" ? `HTTP ${status}` : "request failed"} (response omitted)`); + } + const usage = response?.usage; + if (usage !== undefined && (typeof usage !== "object" || usage === null || Array.isArray(usage))) { + throw new Error("TypeSafe API invalid usage (response omitted)"); + } + return { + answers: response?.answers, + usage: { + input_tokens: usage && Object.hasOwn(usage, "input_tokens") ? usage.input_tokens as number : 0, + output_tokens: usage && Object.hasOwn(usage, "output_tokens") ? usage.output_tokens as number : 0, + }, + model, + }; + }, + }; + }, +}; diff --git a/src/transports/vercel.ts b/src/transports/vercel.ts new file mode 100644 index 0000000..191a93e --- /dev/null +++ b/src/transports/vercel.ts @@ -0,0 +1,84 @@ +import type { BuiltinDriver } from "../provider.js"; + +type Evaluation = (args: { apiKey: string; model: string; state: unknown; questions: Record; signal: AbortSignal }) => Promise; + +async function evaluate({ apiKey, model, state, questions, signal }: Parameters[0]): Promise { + const response = await fetch("https://ai-gateway.vercel.sh/v4/ai/evaluation-model", { + method: "POST", + headers: { + Authorization: `Bearer ${apiKey}`, + "Content-Type": "application/json", + "ai-gateway-protocol-version": "0.0.1", + "ai-gateway-auth-method": "api-key", + "ai-evaluation-model-specification-version": "4", + "ai-model-id": model, + }, + body: JSON.stringify({ state, questions }), + signal, + }); + if (!response.ok) throw { statusCode: response.status }; + return response.json(); +} + +// Internal factory seam: tests provide evaluate without replacing ESM exports. +export function createVercelDriver(evaluateRequest: Evaluation = evaluate): BuiltinDriver { + return { + name: "vercel", + isConfigured: (env) => Boolean(env.AI_GATEWAY_API_KEY), + assertConfigured(env) { + if (!this.isConfigured(env)) throw new Error("AI_GATEWAY_API_KEY is not set."); + }, + create(env) { + this.assertConfigured(env); + const key = env.AI_GATEWAY_API_KEY!; + return { + name: this.name, + async ask({ state, questions, model, signal }) { + const adaptedQuestions: [string, unknown][] = []; + for (const [id, question] of Object.entries(questions)) { + const q = question as { type: string; instructions?: unknown; criteria?: unknown }; + adaptedQuestions.push([id, { type: q.type === "noul" ? "boolean" : q.type, instructions: q.instructions, criteria: q.criteria }]); + } + const effective = model.startsWith("typesafe-ai/") ? model : "typesafe-ai/jev"; + let result: Awaited>; + try { + result = await evaluateRequest({ apiKey: key, model: effective, state, questions: Object.fromEntries(adaptedQuestions), signal }); + } catch (error) { + if (signal.aborted) throw signal.reason; + const status = (error as { statusCode?: unknown }).statusCode; + throw new Error(`Vercel AI Gateway ${typeof status === "number" ? `HTTP ${status}` : "request failed"} (response omitted)`); + } + const usage = result.usage; + if (usage !== undefined && (typeof usage !== "object" || usage === null || Array.isArray(usage))) { + throw new Error("Vercel AI Gateway invalid usage (response omitted)"); + } + return { + answers: adaptVercelAnswers(result.answers, result.providerMetadata), + usage: { + input_tokens: usage && Object.hasOwn(usage, "inputTokens") ? usage.inputTokens as number : 0, + output_tokens: usage && Object.hasOwn(usage, "outputTokens") ? usage.outputTokens as number : 0, + }, + model: effective, + }; + }, + }; + }, + }; +} + +export function adaptVercelAnswers(answers: unknown, metadata: unknown): unknown { + if (!answers || typeof answers !== "object" || Array.isArray(answers)) return answers; + const confidence = (metadata as any)?.typesafe?.confidence ?? {}; + const adapted: [string, unknown][] = []; + for (const [id, answer] of Object.entries(answers)) { + const value = answer as any; + const answerConfidence = Object.hasOwn(confidence, id) ? confidence[id] : null; + if (value?.type === "boolean") adapted.push([id, { type: "noul", noul: value.probability }]); + else if (value?.type === "choice") adapted.push([id, { type: "choice", choice: value.choice, probabilities: value.probabilities, confidence: answerConfidence ?? null }]); + else if (value?.type === "score") adapted.push([id, { type: "score", score: value.score, probabilities: value.probabilities, confidence: answerConfidence ?? null }]); + else adapted.push([id, answer]); + } + return Object.fromEntries(adapted); +} + +export const vercel = createVercelDriver(); diff --git a/test/live.test.mjs b/test/live.test.mjs new file mode 100644 index 0000000..7482f6d --- /dev/null +++ b/test/live.test.mjs @@ -0,0 +1,18 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { ask } from "../dist/index.js"; + +test("one real TypeSafe judgment", { skip: !process.env.TYPESAFE_API_KEY }, async () => { + const result = await ask({ + state: "A customer requests a refund for a duplicate charge.", + questions: { refund: { type: "noul", instructions: "Does the customer ask for a refund?" } }, + model: "jev-latest", + signal: AbortSignal.timeout(30_000), + }, { env: { TYPESAFE_API_KEY: process.env.TYPESAFE_API_KEY } }); + assert.equal(result.ok, true, result.ok ? "" : `${result.code}: ${result.message}`); + assert.equal(result.answer.refund.type, "noul"); + assert.ok(result.answer.refund.noul >= 0 && result.answer.refund.noul <= 1); + assert.ok(Number.isSafeInteger(result.usage.input_tokens) && result.usage.input_tokens > 0); + assert.ok(Number.isSafeInteger(result.usage.output_tokens) && result.usage.output_tokens >= 0); + assert.ok(result.model.trim()); +}); diff --git a/test/transports.test.mjs b/test/transports.test.mjs new file mode 100644 index 0000000..22f1d9e --- /dev/null +++ b/test/transports.test.mjs @@ -0,0 +1,299 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { ask, resolveTransport } from "../dist/index.js"; +import { typesafe } from "../dist/transports/typesafe.js"; +import { openrouter } from "../dist/transports/openrouter.js"; +import { cloudflare } from "../dist/transports/cloudflare.js"; +import { createVercelDriver, adaptVercelAnswers } from "../dist/transports/vercel.js"; + +const askJev = (transport, input) => ask(input, { transport }); +const rejected = (result, code, pattern) => { + assert.equal(result.ok, false); + assert.equal(result.code, code); + assert.match(result.message, pattern); + assert.doesNotMatch(result.message, /body-secret|ts-secret|sk-or-secret|high-secret|low-secret|ai-secret/); +}; + +const signal = new AbortController().signal; +const questions = { item: { type: "choice", criteria: { alpha: "A", beta: "B" } }, yes: { type: "noul" } }; +const answers = { item: { type: "choice", choice: "beta", probabilities: { alpha: 0.2, beta: 0.8 } }, yes: { type: "noul", noul: 0.7 } }; +const input = { state: { title: "Example" }, questions, model: "jev-latest", signal }; +const usage = { input_tokens: 13, output_tokens: 3 }; +const carrier = (override = {}) => ({ name: "fixture", ask: async () => ({ answers, usage, model: "effective", ...override }) }); + +async function withFetch(fn, run) { + const original = globalThis.fetch; + globalThis.fetch = fn; + try { return await run(); } finally { globalThis.fetch = original; } +} + +test("registry auto-detects in precedence order and explicit names select only themselves", () => { + const all = { TYPESAFE_API_KEY: "ts-secret", OPENROUTER_API_KEY: "sk-or-secret", CLOUDFLARE_API_TOKEN: "cf-secret", CLOUDFLARE_ACCOUNT_ID: "account", AI_GATEWAY_API_KEY: "ai-secret" }; + for (const [removed, expected] of [ + [[], "typesafe"], + [["TYPESAFE_API_KEY"], "openrouter"], + [["TYPESAFE_API_KEY", "OPENROUTER_API_KEY"], "cloudflare"], + [["TYPESAFE_API_KEY", "OPENROUTER_API_KEY", "CLOUDFLARE_API_TOKEN"], "vercel"], + ]) { + const env = { ...all }; + for (const key of removed) delete env[key]; + assert.equal(resolveTransport(env).name, expected); + } + for (const name of ["typesafe", "openrouter", "cloudflare", "vercel"]) { + assert.equal(resolveTransport({ ...all, JEV_PROVIDER: name.toUpperCase() }).name, name); + } + assert.equal(resolveTransport({ JEV_CLOUDFLARE_API_TOKEN: "pref", CLOUDFLARE_ACCOUNT_ID: "account" }).name, "cloudflare"); + assert.throws(() => resolveTransport({ ...all, JEV_PROVIDER: "typo" }), /Unknown JEV_PROVIDER/); + assert.throws(() => resolveTransport({}), (error) => ["TYPESAFE_API_KEY", "OPENROUTER_API_KEY", "CLOUDFLARE_API_TOKEN", "JEV_CLOUDFLARE_API_TOKEN", "CLOUDFLARE_ACCOUNT_ID", "AI_GATEWAY_API_KEY"].every((name) => error.message.includes(name))); +}); + +test("public ask returns configuration errors and redacts thrown transport details", async () => { + rejected(await ask(input, { env: { JEV_PROVIDER: "typo" } }), "configuration_error", /Unknown JEV_PROVIDER/); + const absent = await ask(input, { env: {} }); + rejected(absent, "configuration_error", /No TYPESAFE_API_KEY/); + for (const name of ["TYPESAFE_API_KEY", "OPENROUTER_API_KEY", "CLOUDFLARE_API_TOKEN", "JEV_CLOUDFLARE_API_TOKEN", "CLOUDFLARE_ACCOUNT_ID", "AI_GATEWAY_API_KEY"]) { + assert.ok(absent.message.includes(name)); + } + rejected(await ask(input, { transport: { name: "body-secret", ask: async () => { throw Error("body-secret ts-secret"); } } }), "request_failed", /provider unknown: request failed/); +}); + +test("forced credentials reject every missing or invalid variant without leaking values", () => { + const cases = [ + ["typesafe", {}, /TYPESAFE_API_KEY/], + ["openrouter", {}, /OPENROUTER_API_KEY/], + ["openrouter", { OPENROUTER_API_KEY: "wrong-secret" }, /sk-or-/], + ["cloudflare", {}, /CLOUDFLARE_ACCOUNT_ID/], + ["cloudflare", { CLOUDFLARE_API_TOKEN: "cf-secret" }, /CLOUDFLARE_ACCOUNT_ID/], + ["cloudflare", { CLOUDFLARE_ACCOUNT_ID: "account-secret" }, /CLOUDFLARE_API_TOKEN/], + ["cloudflare", { JEV_CLOUDFLARE_API_TOKEN: "cf-secret" }, /CLOUDFLARE_ACCOUNT_ID/], + ["vercel", {}, /AI_GATEWAY_API_KEY/], + ]; + for (const [name, env, pattern] of cases) { + assert.throws(() => resolveTransport({ ...env, JEV_PROVIDER: name }), (error) => { + assert.match(error.message, pattern); + assert.ok(error.message.startsWith(`JEV_PROVIDER=${name}`)); + assert.ok(!error.message.includes("secret")); + return true; + }); + } + assert.equal(resolveTransport({ OPENROUTER_API_KEY: "wrong-secret", AI_GATEWAY_API_KEY: "ai-secret" }).name, "vercel"); + assert.equal(resolveTransport({ CLOUDFLARE_ACCOUNT_ID: "account-secret", AI_GATEWAY_API_KEY: "ai-secret" }).name, "vercel"); +}); + +test("facade validates the complete answer contract before usage can be credited", async () => { + const good = await askJev(carrier(), input); + assert.equal(good.ok, true); + assert.equal(good.provider, "fixture"); + assert.equal(good.answer.item.confidence, null); + assert.equal(good.model, "effective"); + for (const [mutated, id, code, reason] of [ + [{ item: answers.item }, "yes", "answer_id_mismatch", /missing answer/], + [{ ...answers, item: { ...answers.item, type: "noul" } }, "item", "malformed_answer", /wrong type/], + [{ ...answers, item: { ...answers.item, choice: "gamma" } }, "item", "invalid_choice", /outside criteria/], + [{ ...answers, item: { ...answers.item, choice: "alpha" } }, "item", "invalid_choice", /not a distribution maximum/], + [{ ...answers, item: { ...answers.item, probabilities: { alpha: NaN, beta: 0.8 } } }, "item", "invalid_distribution", /finite probabilities/], + [{ ...answers, item: { ...answers.item, probabilities: { alpha: 0.1, beta: 0.7 } } }, "item", "invalid_distribution", /approximately 1/], + [{ ...answers, item: { ...answers.item, probabilities: { alpha: 0.2, beta: 0.7, extra: 0.1 } } }, "item", "invalid_distribution", /exactly the criteria/], + [{ ...answers, item: { ...answers.item, confidence: Infinity } }, "item", "invalid_confidence", /confidence/], + [{ ...answers, yes: { type: "noul", noul: 1.1 } }, "yes", "invalid_noul", /noul/], + ]) { + rejected(await askJev(carrier({ answers: mutated }), input), code, new RegExp(`provider fixture question ${id}.*${reason.source}`)); + } + rejected(await askJev(carrier({ usage: { input_tokens: -1, output_tokens: 0 } }), input), "invalid_usage", /question .*usage/); + rejected(await askJev(carrier({ usage: { input_tokens: undefined, output_tokens: 0 } }), input), "invalid_usage", /question .*usage/); + rejected(await askJev(carrier({ model: " " }), input), "invalid_model", /question .*model/); + rejected(await askJev(carrier({ answers: { ...answers, extra: answers.yes } }), input), "answer_id_mismatch", /unexpected answer ID/); + rejected(await askJev(carrier({ usage: { input_tokens: Number.MAX_SAFE_INTEGER + 1, output_tokens: 0 } }), input), "invalid_usage", /safe integers/); + rejected(await askJev(carrier({ answers: { ...answers, yes: { type: "noul", noul: -0.1 } } }), input), "invalid_noul", /noul/); + const boundary = { ...answers, item: { ...answers.item, probabilities: { alpha: 0.2, beta: 0.809 } } }; + assert.equal((await askJev(carrier({ answers: boundary }), input)).ok, true); + rejected(await askJev(carrier({ answers: { ...answers, item: { ...boundary.item, probabilities: { alpha: 0.2, beta: 0.811 } } } }), input), "invalid_distribution", /approximately 1/); + const tied = { ...answers, item: { ...answers.item, choice: "alpha", probabilities: { alpha: 0.4995, beta: 0.5005 }, confidence: 0 } }; + assert.equal((await askJev(carrier({ answers: tied }), input)).answer.item.choice, "alpha"); + const score = { state: null, model: "jev", signal, questions: { rank: { type: "score", criteria: ["poor", "good", "great"] } } }; + const scoreReply = { rank: { type: "score", score: 2, probabilities: { "0": 0.1, "1": 0.2, "2": 0.7 }, confidence: null } }; + assert.equal((await askJev(carrier({ answers: scoreReply }), score)).answer.rank.score, 2); + rejected(await askJev(carrier({ answers: { rank: { ...scoreReply.rank, score: 3 } } }), score), "invalid_choice", /question rank.*score is outside/); +}); + +test("reserved object keys remain owned criteria and answer IDs", async () => { + const criteria = JSON.parse('{"__proto__":"First","other":"Second"}'); + const special = { ...input, questions: { item: { type: "choice", criteria } } }; + const makeReply = (first, second, choice) => ({ item: { type: "choice", choice, probabilities: JSON.parse(`{"__proto__":${first},"other":${second}}`) } }); + rejected(await askJev(carrier({ answers: makeReply(0.1, 0.9, "__proto__") }), special), "invalid_choice", /not a distribution maximum/); + const winner = await askJev(carrier({ answers: makeReply(0.9, 0.1, "__proto__") }), special); + assert.equal(winner.ok, true); + assert.ok(Object.hasOwn(winner.answer.item.probabilities, "__proto__")); + assert.equal(winner.answer.item.probabilities.__proto__, 0.9); + + const namedQuestion = { ...input, questions: JSON.parse('{"__proto__":{"type":"noul"}}') }; + const namedReply = JSON.parse('{"__proto__":{"type":"noul","noul":0.7}}'); + const direct = await askJev(carrier({ answers: namedReply }), namedQuestion); + assert.equal(direct.ok, true); + assert.ok(Object.hasOwn(direct.answer, "__proto__")); + assert.equal(JSON.parse(JSON.stringify(direct.answer)).__proto__.noul, 0.7); + + let submitted; + await withFetch(async (_url, init) => { + submitted = JSON.parse(init.body); + return Response.json({ answers: JSON.parse('{"__proto__":{"type":"boolean","probability":0.7}}'), usage: { inputTokens: 13, outputTokens: 3 } }); + }, async () => { + const result = await ask(namedQuestion, { env: { AI_GATEWAY_API_KEY: "ai-secret" } }); + assert.equal(result.ok, true); + assert.ok(Object.hasOwn(result.answer, "__proto__")); + }); + assert.ok(Object.hasOwn(submitted.questions, "__proto__")); + + for (const [type, criterion, reply] of [ + ["choice", { alpha: "A", beta: "B" }, { type: "choice", choice: "beta", probabilities: { alpha: 0.2, beta: 0.8 } }], + ["score", ["low", "high"], { type: "score", score: 1, probabilities: { "0": 0.2, "1": 0.8 } }], + ]) { + const named = { ...input, questions: Object.fromEntries([["__proto__", { type, criteria: criterion }]]) }; + const wire = { answers: Object.fromEntries([["__proto__", reply]]), usage: { inputTokens: 13, outputTokens: 3 } }; + for (const metadata of [undefined, { typesafe: { confidence: {} } }]) { + const result = await askJev(createVercelDriver(async () => ({ ...wire, providerMetadata: metadata })).create({ AI_GATEWAY_API_KEY: "ai-secret" }), named); + assert.equal(result.ok, true); + assert.ok(Object.hasOwn(result.answer, "__proto__")); + assert.equal(result.answer.__proto__.confidence, null); + } + const owned = await askJev(createVercelDriver(async () => ({ ...wire, providerMetadata: { typesafe: { confidence: Object.fromEntries([["__proto__", 0.74]]) } } })).create({ AI_GATEWAY_API_KEY: "ai-secret" }), named); + assert.equal(owned.ok, true); + assert.equal(owned.answer.__proto__.confidence, 0.74); + } +}); + +test("TypeSafe client binds key and base URL at creation, forwards request and cancellation", async () => { + const calls = []; + await withFetch(async (url, init) => { + calls.push({ url, init }); + return Response.json({ answers, usage }); + }, async () => { + const transport = typesafe.create({ TYPESAFE_API_KEY: "ts-secret", TYPESAFE_BASE_URL: "https://local.typesafe.test" }); + const reply = await askJev(transport, input); + assert.deepEqual(reply.usage, usage); + assert.equal(reply.model, "jev-latest"); + assert.equal(calls.length, 1); + assert.equal(calls[0].url, "https://local.typesafe.test/v1/systemone"); + assert.equal(calls[0].init.headers.Authorization, "Bearer ts-secret"); + assert.deepEqual(JSON.parse(calls[0].init.body), { state: input.state, questions, model: "jev-latest" }); + assert.ok(calls[0].init.signal instanceof AbortSignal); + assert.equal(calls[0].init.signal.aborted, false); + }); + await withFetch(async () => Response.json({ error: "body-secret" }, { status: 400 }), async () => { + rejected(await askJev(typesafe.create({ TYPESAFE_API_KEY: "ts-secret" }), input), "request_failed", /request failed/); + }); +}); + +test("OpenRouter maps latest and pinned slugs, sends exact envelope and redacts HTTP errors", async () => { + const calls = []; + await withFetch(async (url, init) => { + calls.push({ url, init }); + return Response.json({ answers, usage }); + }, async () => { + const transport = openrouter.create({ OPENROUTER_API_KEY: "sk-or-secret" }); + assert.equal((await askJev(transport, input)).model, "typesafe/jev-1.13"); + assert.equal((await askJev(transport, { ...input, model: "typesafe/jev-1.12" })).model, "typesafe/jev-1.12"); + assert.equal(calls[0].url, "https://openrouter.ai/api/alpha/decisions"); + assert.equal(calls[0].init.method, "POST"); + assert.deepEqual(calls[0].init.headers, { Authorization: "Bearer sk-or-secret", "Content-Type": "application/json", "HTTP-Referer": "https://github.com/jkudish/jev-browser", "X-Title": "jev-browser", "X-OpenRouter-Title": "jev-browser" }); + assert.equal(calls[0].init.signal, signal); + assert.deepEqual(JSON.parse(calls[0].init.body), { model: "typesafe/jev-1.13", state: input.state, questions }); + }); + for (const response of [new Response("body-secret", { status: 403 }), new Response("body-secret", { status: 200 })]) { + await withFetch(async () => response, async () => { + rejected(await askJev(openrouter.create({ OPENROUTER_API_KEY: "sk-or-secret" }), input), "request_failed", /request failed/); + }); + } + await withFetch(async () => Response.json({ usage }), async () => { + rejected(await askJev(openrouter.create({ OPENROUTER_API_KEY: "sk-or-secret" }), input), "malformed_answer", /question .*answers/); + }); + await withFetch(async () => { throw new Error("body-secret sk-or-secret"); }, async () => { + rejected(await askJev(openrouter.create({ OPENROUTER_API_KEY: "sk-or-secret" }), input), "request_failed", /request failed/); + }); +}); + +test("Cloudflare token priority, double envelope, state, usage, and safe errors", async () => { + const calls = []; + const env = { CLOUDFLARE_API_TOKEN: "low-secret", JEV_CLOUDFLARE_API_TOKEN: "high-secret", CLOUDFLARE_ACCOUNT_ID: "account" }; + await withFetch(async (url, init) => { + calls.push({ url, init }); + return Response.json({ success: true, result: { state: "Completed", result: { answers, usage, model: "typesafe/jev" } } }); + }, async () => { + const transport = cloudflare.create(env); + assert.deepEqual((await askJev(transport, input)).usage, usage); + assert.equal(calls[0].url, "https://api.cloudflare.com/client/v4/accounts/account/ai/run"); + assert.deepEqual(calls[0].init.headers, { Authorization: "Bearer high-secret", "Content-Type": "application/json" }); + assert.equal(calls[0].init.signal, signal); + assert.deepEqual(JSON.parse(calls[0].init.body), { model: "typesafe/jev", input: { state: input.state, questions } }); + assert.equal((await transport.ask({ ...input, model: "typesafe/jev-1.2" })).model, "typesafe/jev"); + assert.equal(JSON.parse(calls[1].init.body).model, "typesafe/jev-1.2"); + }); + for (const response of [new Response("body-secret", { status: 401 }), Response.json({ success: false, errors: ["body-secret"] }), Response.json({ result: { state: "Failed body-secret" } }), new Response("body-secret", { status: 200 })]) { + await withFetch(async () => response, async () => { + rejected(await askJev(cloudflare.create(env), input), "request_failed", /request failed/); + }); + } + await withFetch(async () => { throw new Error("body-secret high-secret"); }, async () => { + rejected(await askJev(cloudflare.create(env), input), "request_failed", /request failed/); + }); +}); + +test("Vercel factory forwards evaluate request and pure adaptation preserves confidence", async () => { + let call; + const raw = { item: { type: "choice", choice: "beta", probabilities: { alpha: 0.2, beta: 0.8 } }, yes: { type: "boolean", probability: 0.7 } }; + const result = { answers: raw, usage: { inputTokens: 13, outputTokens: 3 }, providerMetadata: { typesafe: { confidence: { item: 0.92 } } } }; + const factory = createVercelDriver(async (args) => { call = args; return result; }); + const reply = await askJev(factory.create({ AI_GATEWAY_API_KEY: "ai-secret" }), input); + assert.equal(reply.model, "typesafe-ai/jev"); + assert.deepEqual(reply.usage, usage); + assert.equal(call.signal, signal); + assert.equal(call.model, "typesafe-ai/jev"); + assert.deepEqual(call.questions, { item: { type: "choice", instructions: undefined, criteria: questions.item.criteria }, yes: { type: "boolean", instructions: undefined, criteria: undefined } }); + assert.deepEqual(reply.answer, { item: { ...answers.item, confidence: 0.92 }, yes: answers.yes }); + assert.deepEqual(adaptVercelAnswers({ yes: raw.yes, rank: { type: "score", score: 1, probabilities: { "0": 0.2, "1": 0.8 } } }, {}), { yes: answers.yes, rank: { type: "score", score: 1, probabilities: { "0": 0.2, "1": 0.8 }, confidence: null } }); + const failing = createVercelDriver(async () => { throw Object.assign(new Error("body-secret"), { statusCode: 403 }); }); + rejected(await askJev(failing.create({ AI_GATEWAY_API_KEY: "ai-secret" }), input), "request_failed", /request failed/); + const malformed = createVercelDriver(async () => ({ ...result, answers: { item: raw.item } })); + rejected(await askJev(malformed.create({ AI_GATEWAY_API_KEY: "ai-secret" }), input), "answer_id_mismatch", /provider vercel question yes.*missing answer/); +}); + +test("Vercel default fetch sends the evaluation protocol and never echoes HTTP bodies", async () => { + let request; + await withFetch(async (url, init) => { + request = { url, init }; + return Response.json({ answers: { item: answers.item, yes: { type: "boolean", probability: 0.7 } }, usage: { inputTokens: 13, outputTokens: 3 } }); + }, async () => { + const result = await ask(input, { env: { AI_GATEWAY_API_KEY: "ai-secret" } }); + assert.equal(result.ok, true); + assert.deepEqual(result.usage, usage); + assert.deepEqual(result.answer.yes, answers.yes); + }); + assert.equal(request.url, "https://ai-gateway.vercel.sh/v4/ai/evaluation-model"); + assert.equal(request.init.headers["ai-model-id"], "typesafe-ai/jev"); + assert.equal(request.init.headers["ai-evaluation-model-specification-version"], "4"); + assert.equal(request.init.headers.Authorization, "Bearer ai-secret"); + assert.equal(request.init.signal, signal); + assert.deepEqual(JSON.parse(request.init.body), { state: input.state, questions: { item: { type: "choice", criteria: questions.item.criteria }, yes: { type: "boolean" } } }); + await withFetch(async () => Response.json({ error: "body-secret" }, { status: 401 }), async () => { + rejected(await ask(input, { env: { AI_GATEWAY_API_KEY: "ai-secret" } }), "request_failed", /request failed/); + }); +}); + +test("all adapters distinguish absent usage from malformed containers and present null counters", async () => { + const vercelAnswers = { item: answers.item, yes: { type: "boolean", probability: answers.yes.noul } }; + for (const [name, field, run] of [ + ["typesafe", "input_tokens", (wire) => withFetch(async () => Response.json({ answers, ...wire }), () => askJev(typesafe.create({ TYPESAFE_API_KEY: "ts-secret" }), input))], + ["openrouter", "input_tokens", (wire) => withFetch(async () => Response.json({ answers, ...wire }), () => askJev(openrouter.create({ OPENROUTER_API_KEY: "sk-or-secret" }), input))], + ["cloudflare", "input_tokens", (wire) => withFetch(async () => Response.json({ result: { state: "Completed", result: { answers, ...wire } } }), () => askJev(cloudflare.create({ CLOUDFLARE_API_TOKEN: "cf-secret", CLOUDFLARE_ACCOUNT_ID: "account" }), input))], + ["vercel", "inputTokens", (wire) => askJev(createVercelDriver(async () => ({ answers: vercelAnswers, ...wire })).create({ AI_GATEWAY_API_KEY: "ai-secret" }), input)], + ]) { + assert.deepEqual((await run({})).usage, { input_tokens: 0, output_tokens: 0 }, name); + assert.deepEqual((await run({ usage: {} })).usage, { input_tokens: 0, output_tokens: 0 }, name); + const invalidCases = [{ usage: "body-secret" }, { usage: null }, { usage: { [field]: null } }]; + if (name === "vercel") invalidCases.push({ usage: { [field]: undefined } }); // JSON drops undefined properties on the HTTP drivers. + for (const invalid of invalidCases) { + const result = await run(invalid); + rejected(result, invalid.usage === "body-secret" || invalid.usage === null ? "request_failed" : "invalid_usage", /usage|request failed/); + } + } +}); diff --git a/tsconfig.json b/tsconfig.json new file mode 100644 index 0000000..34f8c08 --- /dev/null +++ b/tsconfig.json @@ -0,0 +1,16 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "NodeNext", + "moduleResolution": "NodeNext", + "outDir": "dist", + "rootDir": "src", + "strict": true, + "declaration": true, + "sourceMap": true, + "esModuleInterop": true, + "skipLibCheck": true, + "forceConsistentCasingInFileNames": true + }, + "include": ["src"] +}