From 2eec1275275f10930259e2837d8799eb4087d38e Mon Sep 17 00:00:00 2001 From: Ryan Gapac Date: Thu, 24 Sep 2026 08:20:41 +0800 Subject: [PATCH 1/2] fix: send bare Jev model ids in OpenRouter's own form A caller who writes the TypeSafe default model: "jev-latest" on the OpenRouter backend got a 400, because OpenRouter routes that id as its alias ~typesafe/jev-latest and a bare version as typesafe/jev-. . backendModelId() maps a bare Jev id for the backend that needs it, for the client default and a per-request model alike; ids that already carry an author, and every model on TypeSafe direct, pass unchanged. The validation limits still apply to the caller's own id. /typesafe status now derives its Model line from the same rule instead of naming jev-latest unconditionally. --- CHANGELOG.md | 2 +- docs/api.md | 2 ++ src/backends.ts | 26 ++++++++++++++++ src/client.ts | 14 +++++---- src/extension.ts | 3 +- tests/client.test.ts | 66 ++++++++++++++++++++++++++++++++--------- tests/extension.test.ts | 5 ++++ 7 files changed, 97 insertions(+), 21 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a236e1f..26eb371 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ ## Unreleased - +- A bare Jev model id is mapped to the backend's own form before it is sent — on OpenRouter `jev-latest` goes as `~typesafe/jev-latest` and `jev-1.13` as `typesafe/jev-1.13`, for the client default and a per-request `model` alike — so OpenRouter no longer answers 400, and `/typesafe status` reports the model the configured backend actually sends. ## 0.7.3 diff --git a/docs/api.md b/docs/api.md index 814f55d..79444f1 100644 --- a/docs/api.md +++ b/docs/api.md @@ -34,6 +34,8 @@ result.answers.severity.score; // 0..2, may be fractional | `ledger` | the store next to the key | Inject a ledger in tests | | `fetch` | global fetch | Inject a transport for offline tests | +A `model` is mapped to the backend's own id form before it is sent: on OpenRouter a bare `jev-latest` goes as `~typesafe/jev-latest` and a bare `jev-1.13` (or `jev-1.13.0`) as `typesafe/jev-1.13`, while an id that already carries an author, such as `vendor/other`, passes unchanged, and TypeSafe sends ids as written. The same mapping applies to a per-request `model` inside `evaluate()`; the limits of 1–100 characters apply to your own id, before mapping. + `DECISIONS_BACKENDS` is the registry behind `backend`: each entry carries `label`, `host`, `keyEnv`, and, when the service does not serve the SDK's own paths, `path` for the judgment request plus `modelsPath`, `modelsField`, and `modelsIdField` for the model list — OpenRouter's list arrives under `data` and is renamed to the `models` the SDK reads, with each entry's `id` promoted to the `name` that `listModels()` returns. `modelsVerifyKey: false` marks a backend whose model list is public, and therefore proves nothing about the key. `DEFAULT_BACKEND` is `"typesafe"`. The TypeSafe backend takes its key from `TYPESAFE_API_KEY`, then the `/typesafe login` store. Every other backend reads only its own environment variable (`OPENROUTER_API_KEY` for OpenRouter): the store holds a TypeSafe key, and a login verifies against api.typesafe.ai, so neither applies elsewhere. Pass the same `backend` to `authState`, `keySituation`, and `ensureApiKey` so what you report matches what you send. `evaluate(request, { signal })` validates before sending and rejects with `TypeSafeIntegrationError`. `code` is one of `configuration`, `validation`, `budget`, `aborted`, `timeout`, `http`, `connection`, `response`; messages never contain upstream bodies, keys, or your submitted state, and no header value except a numeric `Retry-After` count in seconds (quoted by the 429 advice as `Retry after seconds.`). The advice is backend-aware: a 401 says `Check TYPESAFE_API_KEY.` or `Check OPENROUTER_API_KEY.`, and a 402 says `Check your account balance.` except on OpenRouter, which says `Insufficient credits. Add credits at https://openrouter.ai/credits.` `listModels()` verifies the key without counting toward `maxRequests`, except on a backend whose model list is public (`modelsVerifyKey: false`), which accepts any key and leaves the auth state unverified. diff --git a/src/backends.ts b/src/backends.ts index 9b9f9e2..0e8826a 100644 --- a/src/backends.ts +++ b/src/backends.ts @@ -48,6 +48,32 @@ export function backendConfig(name: TypeSafeBackend): BackendConfig { return backend; } +/** Each backend's default model id as the caller writes it, before mapping: OpenRouter pins a version, TypeSafe follows latest. */ +const DEFAULT_MODEL: Record = { + typesafe: "jev-latest", + openrouter: "typesafe/jev-1.13", +}; + +/** + * The model id to send for a caller's `model` on this backend. OpenRouter routes a bare Jev id under the `typesafe` + * author: `jev-latest` becomes its alias form `~typesafe/jev-latest`, and a bare `jev-.` — with or + * without TypeSafe direct's optional `.` segment — becomes `typesafe/jev-.`. An id that already + * carries an author (`vendor/model`), a bare id this rule does not know, and every model on a backend without a + * mapping go through unchanged. Every mapped id contains `/`, so mapping an already-mapped id changes nothing. + */ +export function backendModelId(backend: TypeSafeBackend, model: string): string { + if (model.includes("/")) return model; + if (backend !== "openrouter") return model; + if (model === "jev-latest") return "~typesafe/jev-latest"; + const version = /^jev-(\d+)\.(\d+)(?:\.\d+)?$/.exec(model); + return version === null ? model : `typesafe/jev-${version[1]}.${version[2]}`; +} + +/** The model a client sends when the caller names none: the backend's own default, in the form that backend accepts. */ +export function defaultModelId(backend: TypeSafeBackend): string { + return backendModelId(backend, DEFAULT_MODEL[backend]); +} + /** * Whether a backend's key comes from the TypeSafe resolution (`TYPESAFE_API_KEY`, then the login store) or only from * its own environment variable. Only the TypeSafe backend has a login store; every other backend is environment-only. diff --git a/src/client.ts b/src/client.ts index eb9118b..56ee82e 100644 --- a/src/client.ts +++ b/src/client.ts @@ -1,7 +1,7 @@ import { TypeSafeClient } from "@typesafe-ai/sdk"; import type { Fetch, Questions, SystemOneRequest, SystemOneResult } from "@typesafe-ai/sdk"; import { recordAuthFailure, recordAuthVerified } from "./auth.js"; -import { DEFAULT_BACKEND, TYPESAFE_KEY_ENV, backendConfig, usesTypesafeKey } from "./backends.js"; +import { DEFAULT_BACKEND, TYPESAFE_KEY_ENV, backendConfig, backendModelId, defaultModelId, usesTypesafeKey } from "./backends.js"; import type { BackendConfig, TypeSafeBackend } from "./backends.js"; import type { BatchEvaluation, BatchOptions } from "./batch.js"; import { evaluateAll, evaluateMany } from "./batch.js"; @@ -63,7 +63,7 @@ export interface TypeSafeOptions { apiKey?: string; /** Judgment backend. When omitted, routes to the default TypeSafe host. */ backend?: TypeSafeBackend; - /** Defaults to jev-latest. No model is inferred from submitted content. */ + /** Defaults to the backend's own default (`jev-latest`, `typesafe/jev-1.13` on OpenRouter); a bare Jev id is mapped to the backend's id form before sending. No model is inferred from submitted content. */ model?: string; /** Per request. Default: 15 seconds. No automatic retries. */ timeoutMs?: number; @@ -207,8 +207,11 @@ export function createTypeSafe(options: TypeSafeOptions = {}): TypeSafe { }, capsFromEnvironment()); // A backend that serves its own paths gets a transport that rewrites them; the default backend keeps the caller's. const transport = backend.path !== undefined || backend.modelsPath !== undefined ? backendFetch(backend, options.fetch) : options.fetch; - const model = options.model ?? (backendName === "openrouter" ? "typesafe/jev-1.13" : "jev-latest"); - if (typeof model !== "string" || !model.trim() || model.length > 100) throw new TypeSafeIntegrationError("configuration", "model must be a nonempty string of at most 100 characters."); + // The caller's input is validated as written, then mapped to the backend's id form; omitting it sends the backend's + // own default, which the mapping leaves unchanged. + const requested = options.model ?? defaultModelId(backendName); + if (typeof requested !== "string" || !requested.trim() || requested.length > 100) throw new TypeSafeIntegrationError("configuration", "model must be a nonempty string of at most 100 characters."); + const model = backendModelId(backendName, requested); // Do not inherit SDK debug logging or alternate destinations from the environment. const client = new TypeSafeClient({ apiKey, @@ -256,7 +259,8 @@ export function createTypeSafe(options: TypeSafeOptions = {}): TypeSafe { }, async evaluate(input: SystemOneRequest, callOptions: EvaluationOptions = {}): Promise> { const validated = prepareEvaluationRequest(input, { maxInputBytes }); - const body = JSON.stringify({ ...validated, model: validated.model ?? model }); + // A per-request model meets the same mapping as the client default; the schema already limited the caller's own id. + const body = JSON.stringify({ ...validated, model: validated.model === undefined ? model : backendModelId(backendName, validated.model) }); assertWithinByteLimit(body, maxInputBytes); if (callOptions.signal?.aborted) throw new TypeSafeIntegrationError("aborted", "TypeSafe request cancelled before submission."); if (usage.requestsStarted >= maxRequests) { diff --git a/src/extension.ts b/src/extension.ts index b0f0a8e..0e44466 100644 --- a/src/extension.ts +++ b/src/extension.ts @@ -2,6 +2,7 @@ import type { AgentToolResult, ExtensionAPI, ExtensionContext } from "@earendil- import type { Static } from "typebox"; import { Text } from "@earendil-works/pi-tui"; import type { Questions } from "@typesafe-ai/sdk"; +import { DEFAULT_BACKEND, defaultModelId } from "./backends.js"; import { createTypeSafe, DEFAULT_MAX_REQUESTS } from "./client.js"; import type { Evaluation, TypeSafe } from "./client.js"; import { authState, clearAuthState, describeAuth } from "./auth.js"; @@ -132,7 +133,7 @@ export default function typesafeExtension(pi: ExtensionAPI): void { ? `Today ${spend.today.requestsStarted} requests (${spend.today.requestsSucceeded} ok, ${spend.today.requestsFailed} failed), ${spend.today.inputTokens} input tokens, ~$${spend.today.estimatedUsd.toFixed(4)}.` : ""; const blocked = spend?.blocked ? ` Cap reached: ${spend.blocked.cap} ${spend.blocked.used}/${spend.blocked.limit} on ${spend.blocked.day}; no request will be submitted until the local day rolls over.` : ""; - report(`TypeSafe: ${enabled ? "enabled" : "disabled"}. ${auth.text} ${session} ${today}${blocked} Model: jev-latest. Session limits reset on session start/reload; daily counters persist and caps come from client options or PI_TYPESAFE_MAX_* environment variables. ${disclosure}`, auth.level === "error" && enabled ? "warning" : "info"); + report(`TypeSafe: ${enabled ? "enabled" : "disabled"}. ${auth.text} ${session} ${today}${blocked} Model: ${defaultModelId(DEFAULT_BACKEND)}. Session limits reset on session start/reload; daily counters persist and caps come from client options or PI_TYPESAFE_MAX_* environment variables. ${disclosure}`, auth.level === "error" && enabled ? "warning" : "info"); return; } if (action === "logout") { diff --git a/tests/client.test.ts b/tests/client.test.ts index 3801fd9..25bb867 100644 --- a/tests/client.test.ts +++ b/tests/client.test.ts @@ -281,7 +281,7 @@ test("client ignores SDK endpoint and logging environment overrides", async () = test("configuration errors are early and usage snapshots are detached", async () => { assert.throws(() => createTypeSafe({ apiKey: " " }), hasCode("configuration")); - for (const options of [{ timeoutMs: 0 }, { maxRequests: -1 }, { maxInputBytes: NaN }, { model: "" }]) { + for (const options of [{ timeoutMs: 0 }, { maxRequests: -1 }, { maxInputBytes: NaN }, { model: "" }, { model: "x".repeat(101) }]) { assert.throws(() => createTypeSafe({ apiKey: "test-key", ...options }), hasCode("configuration")); } const client = createTypeSafe({ apiKey: "test-key", fetch: async () => responseFor(sample().questions) }); @@ -400,19 +400,57 @@ test("a public model list leaves the auth state unverified", async () => { assert.equal(authState({ backend: "openrouter" }).verified, false); }); -test("openrouter backend uses default model typesafe/jev-1.13", async () => { - let sentModel: string | undefined; - const client = createTypeSafe({ - apiKey: "test-key", - backend: "openrouter", - fetch: async (_url, init) => { - const body = JSON.parse(String(init?.body)) as { model?: string }; - sentModel = body.model; - return responseFor(sample().questions); - }, - }); - await client.evaluate(sample()); - assert.equal(sentModel, "typesafe/jev-1.13"); +test("the model in the request body is the backend's own id form", async () => { + const cases = [ + ["openrouter", undefined, "typesafe/jev-1.13"], + ["openrouter", "jev-latest", "~typesafe/jev-latest"], + ["openrouter", "jev-1.13", "typesafe/jev-1.13"], + ["openrouter", "jev-1.13.0", "typesafe/jev-1.13"], + ["openrouter", "vendor/other", "vendor/other"], + ["typesafe", "jev-latest", "jev-latest"], + ] as const; + for (const [backend, requested, expected] of cases) { + let sentUrl = ""; + let sentModel: string | undefined; + const client = createTypeSafe({ + apiKey: "test-key", + backend, + ...(requested === undefined ? {} : { model: requested }), + fetch: async (url, init) => { + sentUrl = String(url); + const body = JSON.parse(String(init?.body)) as { model?: string }; + sentModel = body.model; + return responseFor(sample().questions); + }, + }); + await client.evaluate(sample()); + assert.equal(sentModel, expected); + // OpenRouter judgments travel to its own decisions path, so the mapped id is proven on the wire that uses it. + assert.equal(sentUrl, backend === "openrouter" ? "https://openrouter.ai/api/alpha/decisions" : "https://api.typesafe.ai/v1/systemone"); + } +}); + +test("a per-request model gets the same mapping as the client default", async () => { + const cases = [ + ["openrouter", "jev-latest", "~typesafe/jev-latest"], + ["openrouter", "jev-1.13", "typesafe/jev-1.13"], + ["openrouter", "vendor/other", "vendor/other"], + ["typesafe", "jev-latest", "jev-latest"], + ] as const; + for (const [backend, requested, expected] of cases) { + let sentModel: string | undefined; + const client = createTypeSafe({ + apiKey: "test-key", + backend, + fetch: async (_url, init) => { + const body = JSON.parse(String(init?.body)) as { model?: string }; + sentModel = body.model; + return responseFor(sample().questions); + }, + }); + await client.evaluate({ ...sample(), model: requested }); + assert.equal(sentModel, expected); + } }); test("a TypeSafe key is never sent to another backend", () => { diff --git a/tests/extension.test.ts b/tests/extension.test.ts index a233642..0d20547 100644 --- a/tests/extension.test.ts +++ b/tests/extension.test.ts @@ -107,6 +107,11 @@ test("setup and status never display the API key", async () => { assert.equal(notices.some(text => text.includes("offline-test-key")), false); }); +test("status names the model the configured backend actually sends", async () => { + await runCommand("status"); + assert.ok(notices.at(-1)?.includes("Model: jev-latest.")); +}); + test("login refuses to shadow an environment key", async () => { await runCommand("login"); assert.ok(notices.at(-1)?.includes("takes precedence")); From 7a85c787791033fa203e745efb61f6d2419df06e Mon Sep 17 00:00:00 2001 From: Ryan Gapac Date: Thu, 24 Sep 2026 08:25:10 +0800 Subject: [PATCH 2/2] Release 0.7.4 --- CHANGELOG.md | 6 ++++++ package-lock.json | 4 ++-- package.json | 2 +- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 26eb371..e078b7d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,12 @@ ## Unreleased + + +## 0.7.4 + +### Fixed + - A bare Jev model id is mapped to the backend's own form before it is sent — on OpenRouter `jev-latest` goes as `~typesafe/jev-latest` and `jev-1.13` as `typesafe/jev-1.13`, for the client default and a per-request `model` alike — so OpenRouter no longer answers 400, and `/typesafe status` reports the model the configured backend actually sends. ## 0.7.3 diff --git a/package-lock.json b/package-lock.json index 509951e..07933aa 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "pi-typesafe", - "version": "0.7.3", + "version": "0.7.4", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "pi-typesafe", - "version": "0.7.3", + "version": "0.7.4", "license": "MIT", "dependencies": { "@typesafe-ai/sdk": "^0.6.0", diff --git a/package.json b/package.json index c996ab5..00d0377 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "pi-typesafe", - "version": "0.7.3", + "version": "0.7.4", "description": "TypeSafe AI (Jev) decisions for Pi: batched Choice/Score/Noul evaluation tool, terminal playground, and a typed API other extensions build on.", "type": "module", "license": "MIT",