Send bare Jev model ids in OpenRouter's own form; release 0.7.4

CHANGELOG.md

@@ -4,6 +4,12 @@
 
 <!-- Empty. Next release starts here. -->
 
+## 0.7.4
+
+### Fixed
+
+- A bare Jev model id is mapped to the backend's own form before it is sent — on OpenRouter `jev-latest` goes as `~typesafe/jev-latest` and `jev-1.13` as `typesafe/jev-1.13`, for the client default and a per-request `model` alike — so OpenRouter no longer answers 400, and `/typesafe status` reports the model the configured backend actually sends.
+
 ## 0.7.3
 
 ### Changed

docs/api.md

@@ -34,6 +34,8 @@ result.answers.severity.score;   // 0..2, may be fractional
 | `ledger` | the store next to the key | Inject a ledger in tests |
 | `fetch` | global fetch | Inject a transport for offline tests |
 
+A `model` is mapped to the backend's own id form before it is sent: on OpenRouter a bare `jev-latest` goes as `~typesafe/jev-latest` and a bare `jev-1.13` (or `jev-1.13.0`) as `typesafe/jev-1.13`, while an id that already carries an author, such as `vendor/other`, passes unchanged, and TypeSafe sends ids as written. The same mapping applies to a per-request `model` inside `evaluate()`; the limits of 1–100 characters apply to your own id, before mapping.
+
 `DECISIONS_BACKENDS` is the registry behind `backend`: each entry carries `label`, `host`, `keyEnv`, and, when the service does not serve the SDK's own paths, `path` for the judgment request plus `modelsPath`, `modelsField`, and `modelsIdField` for the model list — OpenRouter's list arrives under `data` and is renamed to the `models` the SDK reads, with each entry's `id` promoted to the `name` that `listModels()` returns. `modelsVerifyKey: false` marks a backend whose model list is public, and therefore proves nothing about the key. `DEFAULT_BACKEND` is `"typesafe"`. The TypeSafe backend takes its key from `TYPESAFE_API_KEY`, then the `/typesafe login` store. Every other backend reads only its own environment variable (`OPENROUTER_API_KEY` for OpenRouter): the store holds a TypeSafe key, and a login verifies against api.typesafe.ai, so neither applies elsewhere. Pass the same `backend` to `authState`, `keySituation`, and `ensureApiKey` so what you report matches what you send.
 
 `evaluate(request, { signal })` validates before sending and rejects with `TypeSafeIntegrationError`. `code` is one of `configuration`, `validation`, `budget`, `aborted`, `timeout`, `http`, `connection`, `response`; messages never contain upstream bodies, keys, or your submitted state, and no header value except a numeric `Retry-After` count in seconds (quoted by the 429 advice as `Retry after <n> seconds.`). The advice is backend-aware: a 401 says `Check TYPESAFE_API_KEY.` or `Check OPENROUTER_API_KEY.`, and a 402 says `Check your account balance.` except on OpenRouter, which says `Insufficient credits. Add credits at https://openrouter.ai/credits.` `listModels()` verifies the key without counting toward `maxRequests`, except on a backend whose model list is public (`modelsVerifyKey: false`), which accepts any key and leaves the auth state unverified.

package.json

@@ -1,6 +1,6 @@
 {
   "name": "pi-typesafe",
-  "version": "0.7.3",
+  "version": "0.7.4",
   "description": "TypeSafe AI (Jev) decisions for Pi: batched Choice/Score/Noul evaluation tool, terminal playground, and a typed API other extensions build on.",
   "type": "module",
   "license": "MIT",

src/backends.ts

@@ -48,6 +48,32 @@ export function backendConfig(name: TypeSafeBackend): BackendConfig {
   return backend;
 }
 
+/** Each backend's default model id as the caller writes it, before mapping: OpenRouter pins a version, TypeSafe follows latest. */
+const DEFAULT_MODEL: Record<TypeSafeBackend, string> = {
+  typesafe: "jev-latest",
+  openrouter: "typesafe/jev-1.13",
+};
+
+/**
+ * The model id to send for a caller's `model` on this backend. OpenRouter routes a bare Jev id under the `typesafe`
+ * author: `jev-latest` becomes its alias form `~typesafe/jev-latest`, and a bare `jev-<major>.<minor>` — with or
+ * without TypeSafe direct's optional `.<patch>` segment — becomes `typesafe/jev-<major>.<minor>`. An id that already
+ * carries an author (`vendor/model`), a bare id this rule does not know, and every model on a backend without a
+ * mapping go through unchanged. Every mapped id contains `/`, so mapping an already-mapped id changes nothing.
+ */
+export function backendModelId(backend: TypeSafeBackend, model: string): string {
+  if (model.includes("/")) return model;
+  if (backend !== "openrouter") return model;
+  if (model === "jev-latest") return "~typesafe/jev-latest";
+  const version = /^jev-(\d+)\.(\d+)(?:\.\d+)?$/.exec(model);
+  return version === null ? model : `typesafe/jev-${version[1]}.${version[2]}`;
+}
+
+/** The model a client sends when the caller names none: the backend's own default, in the form that backend accepts. */
+export function defaultModelId(backend: TypeSafeBackend): string {
+  return backendModelId(backend, DEFAULT_MODEL[backend]);
+}
+
 /**
  * Whether a backend's key comes from the TypeSafe resolution (`TYPESAFE_API_KEY`, then the login store) or only from
  * its own environment variable. Only the TypeSafe backend has a login store; every other backend is environment-only.

src/client.ts

@@ -1,7 +1,7 @@
 import { TypeSafeClient } from "@typesafe-ai/sdk";
 import type { Fetch, Questions, SystemOneRequest, SystemOneResult } from "@typesafe-ai/sdk";
 import { recordAuthFailure, recordAuthVerified } from "./auth.js";
-import { DEFAULT_BACKEND, TYPESAFE_KEY_ENV, backendConfig, usesTypesafeKey } from "./backends.js";
+import { DEFAULT_BACKEND, TYPESAFE_KEY_ENV, backendConfig, backendModelId, defaultModelId, usesTypesafeKey } from "./backends.js";
 import type { BackendConfig, TypeSafeBackend } from "./backends.js";
 import type { BatchEvaluation, BatchOptions } from "./batch.js";
 import { evaluateAll, evaluateMany } from "./batch.js";
@@ -63,7 +63,7 @@ export interface TypeSafeOptions {
   apiKey?: string;
   /** Judgment backend. When omitted, routes to the default TypeSafe host. */
   backend?: TypeSafeBackend;
-  /** Defaults to jev-latest. No model is inferred from submitted content. */
+  /** Defaults to the backend's own default (`jev-latest`, `typesafe/jev-1.13` on OpenRouter); a bare Jev id is mapped to the backend's id form before sending. No model is inferred from submitted content. */
   model?: string;
   /** Per request. Default: 15 seconds. No automatic retries. */
   timeoutMs?: number;
@@ -207,8 +207,11 @@ export function createTypeSafe(options: TypeSafeOptions = {}): TypeSafe {
   }, capsFromEnvironment());
   // A backend that serves its own paths gets a transport that rewrites them; the default backend keeps the caller's.
   const transport = backend.path !== undefined || backend.modelsPath !== undefined ? backendFetch(backend, options.fetch) : options.fetch;
-  const model = options.model ?? (backendName === "openrouter" ? "typesafe/jev-1.13" : "jev-latest");
-  if (typeof model !== "string" || !model.trim() || model.length > 100) throw new TypeSafeIntegrationError("configuration", "model must be a nonempty string of at most 100 characters.");
+  // The caller's input is validated as written, then mapped to the backend's id form; omitting it sends the backend's
+  // own default, which the mapping leaves unchanged.
+  const requested = options.model ?? defaultModelId(backendName);
+  if (typeof requested !== "string" || !requested.trim() || requested.length > 100) throw new TypeSafeIntegrationError("configuration", "model must be a nonempty string of at most 100 characters.");
+  const model = backendModelId(backendName, requested);
   // Do not inherit SDK debug logging or alternate destinations from the environment.
   const client = new TypeSafeClient({
     apiKey,
@@ -256,7 +259,8 @@ export function createTypeSafe(options: TypeSafeOptions = {}): TypeSafe {
     },
     async evaluate<Q extends Questions>(input: SystemOneRequest<Q>, callOptions: EvaluationOptions = {}): Promise<Evaluation<Q>> {
       const validated = prepareEvaluationRequest(input, { maxInputBytes });
-      const body = JSON.stringify({ ...validated, model: validated.model ?? model });
+      // A per-request model meets the same mapping as the client default; the schema already limited the caller's own id.
+      const body = JSON.stringify({ ...validated, model: validated.model === undefined ? model : backendModelId(backendName, validated.model) });
       assertWithinByteLimit(body, maxInputBytes);
       if (callOptions.signal?.aborted) throw new TypeSafeIntegrationError("aborted", "TypeSafe request cancelled before submission.");
       if (usage.requestsStarted >= maxRequests) {

src/extension.ts

@@ -2,6 +2,7 @@ import type { AgentToolResult, ExtensionAPI, ExtensionContext } from "@earendil-
 import type { Static } from "typebox";
 import { Text } from "@earendil-works/pi-tui";
 import type { Questions } from "@typesafe-ai/sdk";
+import { DEFAULT_BACKEND, defaultModelId } from "./backends.js";
 import { createTypeSafe, DEFAULT_MAX_REQUESTS } from "./client.js";
 import type { Evaluation, TypeSafe } from "./client.js";
 import { authState, clearAuthState, describeAuth } from "./auth.js";
@@ -132,7 +133,7 @@ export default function typesafeExtension(pi: ExtensionAPI): void {
             ? `Today ${spend.today.requestsStarted} requests (${spend.today.requestsSucceeded} ok, ${spend.today.requestsFailed} failed), ${spend.today.inputTokens} input tokens, ~$${spend.today.estimatedUsd.toFixed(4)}.`
             : "";
           const blocked = spend?.blocked ? ` Cap reached: ${spend.blocked.cap} ${spend.blocked.used}/${spend.blocked.limit} on ${spend.blocked.day}; no request will be submitted until the local day rolls over.` : "";
-          report(`TypeSafe: ${enabled ? "enabled" : "disabled"}. ${auth.text} ${session} ${today}${blocked} Model: jev-latest. Session limits reset on session start/reload; daily counters persist and caps come from client options or PI_TYPESAFE_MAX_* environment variables. ${disclosure}`, auth.level === "error" && enabled ? "warning" : "info");
+          report(`TypeSafe: ${enabled ? "enabled" : "disabled"}. ${auth.text} ${session} ${today}${blocked} Model: ${defaultModelId(DEFAULT_BACKEND)}. Session limits reset on session start/reload; daily counters persist and caps come from client options or PI_TYPESAFE_MAX_* environment variables. ${disclosure}`, auth.level === "error" && enabled ? "warning" : "info");
           return;
         }
         if (action === "logout") {

tests/client.test.ts

@@ -281,7 +281,7 @@ test("client ignores SDK endpoint and logging environment overrides", async () =
 
 test("configuration errors are early and usage snapshots are detached", async () => {
   assert.throws(() => createTypeSafe({ apiKey: " " }), hasCode("configuration"));
-  for (const options of [{ timeoutMs: 0 }, { maxRequests: -1 }, { maxInputBytes: NaN }, { model: "" }]) {
+  for (const options of [{ timeoutMs: 0 }, { maxRequests: -1 }, { maxInputBytes: NaN }, { model: "" }, { model: "x".repeat(101) }]) {
     assert.throws(() => createTypeSafe({ apiKey: "test-key", ...options }), hasCode("configuration"));
   }
   const client = createTypeSafe({ apiKey: "test-key", fetch: async () => responseFor(sample().questions) });
@@ -400,19 +400,57 @@ test("a public model list leaves the auth state unverified", async () => {
   assert.equal(authState({ backend: "openrouter" }).verified, false);
 });
 
-test("openrouter backend uses default model typesafe/jev-1.13", async () => {
-  let sentModel: string | undefined;
-  const client = createTypeSafe({
-    apiKey: "test-key",
-    backend: "openrouter",
-    fetch: async (_url, init) => {
-      const body = JSON.parse(String(init?.body)) as { model?: string };
-      sentModel = body.model;
-      return responseFor(sample().questions);
-    },
-  });
-  await client.evaluate(sample());
-  assert.equal(sentModel, "typesafe/jev-1.13");
+test("the model in the request body is the backend's own id form", async () => {
+  const cases = [
+    ["openrouter", undefined, "typesafe/jev-1.13"],
+    ["openrouter", "jev-latest", "~typesafe/jev-latest"],
+    ["openrouter", "jev-1.13", "typesafe/jev-1.13"],
+    ["openrouter", "jev-1.13.0", "typesafe/jev-1.13"],
+    ["openrouter", "vendor/other", "vendor/other"],
+    ["typesafe", "jev-latest", "jev-latest"],
+  ] as const;
+  for (const [backend, requested, expected] of cases) {
+    let sentUrl = "";
+    let sentModel: string | undefined;
+    const client = createTypeSafe({
+      apiKey: "test-key",
+      backend,
+      ...(requested === undefined ? {} : { model: requested }),
+      fetch: async (url, init) => {
+        sentUrl = String(url);
+        const body = JSON.parse(String(init?.body)) as { model?: string };
+        sentModel = body.model;
+        return responseFor(sample().questions);
+      },
+    });
+    await client.evaluate(sample());
+    assert.equal(sentModel, expected);
+    // OpenRouter judgments travel to its own decisions path, so the mapped id is proven on the wire that uses it.
+    assert.equal(sentUrl, backend === "openrouter" ? "https://openrouter.ai/api/alpha/decisions" : "https://api.typesafe.ai/v1/systemone");
+  }
+});
+
+test("a per-request model gets the same mapping as the client default", async () => {
+  const cases = [
+    ["openrouter", "jev-latest", "~typesafe/jev-latest"],
+    ["openrouter", "jev-1.13", "typesafe/jev-1.13"],
+    ["openrouter", "vendor/other", "vendor/other"],
+    ["typesafe", "jev-latest", "jev-latest"],
+  ] as const;
+  for (const [backend, requested, expected] of cases) {
+    let sentModel: string | undefined;
+    const client = createTypeSafe({
+      apiKey: "test-key",
+      backend,
+      fetch: async (_url, init) => {
+        const body = JSON.parse(String(init?.body)) as { model?: string };
+        sentModel = body.model;
+        return responseFor(sample().questions);
+      },
+    });
+    await client.evaluate({ ...sample(), model: requested });
+    assert.equal(sentModel, expected);
+  }
 });
 
 test("a TypeSafe key is never sent to another backend", () => {

tests/extension.test.ts

@@ -107,6 +107,11 @@ test("setup and status never display the API key", async () => {
   assert.equal(notices.some(text => text.includes("offline-test-key")), false);
 });
 
+test("status names the model the configured backend actually sends", async () => {
+  await runCommand("status");
+  assert.ok(notices.at(-1)?.includes("Model: jev-latest."));
+});
+
 test("login refuses to shadow an environment key", async () => {
   await runCommand("login");
   assert.ok(notices.at(-1)?.includes("takes precedence"));