diff --git a/CHANGELOG.md b/CHANGELOG.md index e0f8c912..e51fa843 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,12 @@ How to keep this current: add the entry in the same pull request as the change, +## 0.74.0 + +### Added + +- `"laya"` as a `typesafeBackend` value runs every judgment against a local Laya-MLX model instead of a remote Jev host. `src/laya-judge.ts` implements the `Judge` contract by POSTing the same `SystemOneRequest` to a local `/v1/systemone` endpoint — the wire shape the TypeSafe SDK uses — so requests and answers are unchanged and every guard works through the existing seam. No key, no model download, no Python subprocess: the endpoint is a long-running server the user starts themselves, and `/warden status` names it. Consent is still required (`/warden enable` shows a local dialog: nothing is sent to any server). A dead or failing endpoint behaves like any failing backend — per-request failures, then the judge cooldown, with a notice that says how to start the server. + ## 0.73.0 ### Fixed diff --git a/docs/configuration.md b/docs/configuration.md index 9e1d9ba0..f7edcc9f 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -80,7 +80,7 @@ User file `~/.pi/agent/pi-warden/config.json` (owner-only). `/warden config` ope | --- | --- | | `enabled` | Master switch for the extension. | | `typesafe` | Consent to send requests to TypeSafe. Set by `/warden enable`; only the user file or `PI_WARDEN_ENABLED=1` can grant it. | -| `typesafeBackend` | The judgment service: `"typesafe"` (default) or `"openrouter"`. User file only — a project must not redirect judgments. | +| `typesafeBackend` | The judgment service: `"typesafe"` (default), `"openrouter"`, or `"laya"` for a local Laya-MLX model served over HTTP by a server you run (no key; judged content never leaves the machine). User file only — a project must not redirect judgments. | | `mode` | `steer` (hold goes back to the agent), `confirm` (dialog for you), `advise` (never holds). User file only; a project's `.pi/pi-warden.json` cannot change it, by design. | | `timeoutMs` | Per-request timeout. On timeout the call is allowed with a warning when `action.failOpen` is true. | | `maxRequests` | Per-session request budget. When spent, pi-warden says so once and continues with offline checks. | diff --git a/docs/data-handling.md b/docs/data-handling.md index a07bf76a..b0c04201 100644 --- a/docs/data-handling.md +++ b/docs/data-handling.md @@ -19,6 +19,8 @@ With consent, requests go to `https://api.typesafe.ai` (default) or `https://ope ## What stays on this machine +With `typesafeBackend: "laya"`, every judge payload in the table above stays on this machine: the local judge server receives the same redacted requests and no key is configured. The remote backends are untouched by the setting. + Default paths below are for Pi. On oh-my-pi, pi-warden uses `~/.omp/agent` instead of `~/.pi/agent`, but pi-typesafe's `auth.json` still defaults to `~/.pi/agent/pi-typesafe/`. `PI_CODING_AGENT_DIR` overrides both. One-time migration for existing oh-my-pi users: an interactive oh-my-pi session shows a notice once per machine, with the two paths filled in. It says to close all Pi and oh-my-pi sessions first (the SQLite database keeps write-ahead logs while a session is open), then run `{ [ ! -e "$HOME/.omp/agent/pi-warden" ] || mv "$HOME/.omp/agent/pi-warden" "$HOME/.omp/agent/pi-warden.before-migration"; } && cp -R "$HOME/.pi/agent/pi-warden" "$HOME/.omp/agent/pi-warden"` — the `mv` keeps the fresh data the new host created aside instead of deleting it — and copy the project file `.pi/pi-warden.json` to `.omp/pi-warden.json` (keep the original so Pi sessions in that project still read it). - The hold feedback log under `~/.pi/agent/pi-warden/holds/`, owner-only: one JSON line per judged call with tool, pattern ids, scores, level, mode, outcome, and the length of the agent's stated plan; never the command, prompt, or plan text. A redacted tool+path excerpt is included for auditing off-task and intent-mismatch. `"action": { "feedbackLog": false }` turns the file off. diff --git a/package.json b/package.json index 97d1d157..cdc6e693 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "pi-warden", - "version": "0.73.0", + "version": "0.74.0", "description": "Makes the Pi agent follow your project's rules. Jev judges every write against your pi-warden.md and quotes the broken rule back to the agent, names slop, breaks stuck loops, calls out unverified done claims, compresses large tool output, and holds the rare destructive command. Built on pi-typesafe.", "type": "module", "license": "MIT", diff --git a/src/backend.ts b/src/backend.ts index 16f067b2..30d96517 100644 --- a/src/backend.ts +++ b/src/backend.ts @@ -2,19 +2,25 @@ import type { TypeSafeOptions } from "pi-typesafe"; import { DECISIONS_BACKENDS, DEFAULT_BACKEND } from "pi-typesafe"; /** The judgment backend that receives pi-warden's Jev requests. */ -export type JudgmentBackend = "typesafe" | "openrouter"; +export type JudgmentBackend = "typesafe" | "openrouter" | "laya"; + +/** The local judge: judged content never leaves this machine. */ +export const LAYA_BACKEND = "laya"; /** Why no judge is available: consent not given, no key for the backend, a saved 401 or 403, or the request budget spent. */ export type JudgmentsOffReason = "no_consent" | "no_key" | "key_rejected" | "budget"; /** The host the consent disclosure names as the destination, without scheme: `api.typesafe.ai`, `openrouter.ai`. */ export function backendHost(backend: JudgmentBackend): string { + if (backend === LAYA_BACKEND) + return "this machine (local Laya-MLX)"; return new URL(DECISIONS_BACKENDS[backend].host).host; } /** The environment variable that carries the backend's key, for messages that tell the user what to set. */ export function keyEnvFor(backend: JudgmentBackend): string { - return DECISIONS_BACKENDS[backend].keyEnv ?? DECISIONS_BACKENDS[DEFAULT_BACKEND].keyEnv ?? "TYPESAFE_API_KEY"; + const config = backend === LAYA_BACKEND ? undefined : DECISIONS_BACKENDS[backend]; + return config?.keyEnv ?? DECISIONS_BACKENDS[DEFAULT_BACKEND].keyEnv ?? "TYPESAFE_API_KEY"; } /** Whether the backend takes the TypeSafe key, the only one `/typesafe login` stores; pi-typesafe's `usesTypesafeKey`, which its package does not export. */ @@ -35,11 +41,11 @@ export function judgeOptions(config: { maxRequests: number; timeoutMs: number; t return { maxRequests: config.maxRequests, timeoutMs: config.timeoutMs, - ...(config.typesafeBackend === DEFAULT_BACKEND ? {} : { backend: config.typesafeBackend }), + ...(config.typesafeBackend === DEFAULT_BACKEND || config.typesafeBackend === LAYA_BACKEND ? {} : { backend: config.typesafeBackend }), }; } /** Resolve the effective backend from a raw config value; invalid values fall back to "typesafe". */ export function resolveBackend(raw: unknown): JudgmentBackend { - return raw === "typesafe" || raw === "openrouter" ? raw : DEFAULT_BACKEND; + return raw === "typesafe" || raw === "openrouter" || raw === LAYA_BACKEND ? raw : DEFAULT_BACKEND; } diff --git a/src/extension.ts b/src/extension.ts index c882684c..2143468a 100644 --- a/src/extension.ts +++ b/src/extension.ts @@ -11,7 +11,7 @@ import * as tuiModule from "@earendil-works/pi-tui"; type MouseRegionConstructor = new (child: ReturnType, onMouse: (event: { type: string; button: string }) => { handled: boolean } | undefined) => import("@earendil-works/pi-tui").Component; const MouseRegion: MouseRegionConstructor | undefined = (tuiModule as Partial<{ MouseRegion: MouseRegionConstructor }>).MouseRegion; import { authState, createTypeSafe, describeAuth } from "pi-typesafe"; -import type { TypeSafe } from "pi-typesafe"; +import type { Judge, TypeSafe } from "pi-typesafe"; import { ensureApiKey } from "pi-typesafe/ui"; import { backendHost, disclosureFor, judgeOptions, keyEnvFor, loginStoresKey, resolveBackend } from "./backend.js"; import type { JudgmentBackend, JudgmentsOffReason } from "./backend.js"; @@ -69,6 +69,7 @@ import type { IndexFile } from "./index-cmd.js"; import type { IntegrationErrorCode } from "pi-typesafe"; import { classifyJudgeError, cooldownFailureKind, JudgeCooldown } from "./judge-cooldown.js"; +import { LayaJudge } from "./laya-judge.js"; import type { CooldownEvent } from "./judge-cooldown.js"; import { openConfigPanel, openTracePanel } from "./panel.js"; import { completeConfig, shapeWarning, taskSpine } from "./shape.js"; @@ -380,7 +381,7 @@ export default function wardenExtension(host: ExtensionAPI): void { // check; there is no module state. A question-wording change breaks the hash test and the gate // fail-closes (no delivery) until the policy is re-measured. const consciencePolicy = CONSCIENCE_BETA_POLICY; - let client: TypeSafe | undefined; + let client: TypeSafe | LayaJudge | undefined; const cooldown = new JudgeCooldown(); let judgeConfig: WardenConfig | undefined; /** Where cooldown notices go; unset headless, where a failure is already silent (see noteError). */ @@ -566,6 +567,8 @@ export default function wardenExtension(host: ExtensionAPI): void { const judgmentsOffReason = (config: WardenConfig): JudgmentsOffReason | undefined => { if (!consentGiven(config)) return "no_consent"; if (budgetExhausted) return "budget"; + // The local judge needs no key; a dead server surfaces as per-request failures and the cooldown. + if (config.typesafeBackend === "laya") return undefined; const auth = authState({ backend: config.typesafeBackend }); if (auth.usable) return undefined; // No key in effect, or one whose last request came back 401 or 403. @@ -581,22 +584,23 @@ export default function wardenExtension(host: ExtensionAPI): void { // The budget has its own notice in noteError. if (reason === undefined || reason === "budget" || judgmentsReported.has(reason)) return; judgmentsReported.add(reason); - const auth = reason === "key_rejected" ? authState({ backend: config.typesafeBackend }) : undefined; + const auth = reason === "key_rejected" && config.typesafeBackend !== "laya" ? authState({ backend: config.typesafeBackend }) : undefined; judgmentsNotify?.(judgmentsOffText(reason, config.typesafeBackend, judgmentsHeadless, auth?.kind === "environment" ? auth.keyName : undefined)); }; const consentSource = (config: WardenConfig) => config.typesafe ? "/warden enable" : process.env.PI_WARDEN_ENABLED === "1" ? "PI_WARDEN_ENABLED" : undefined; /** A consent flag is not proof that judgments happen; check the key state for the chosen backend. */ - const judgeFor = (config: WardenConfig): TypeSafe | undefined => { + const judgeFor = (config: WardenConfig): TypeSafe | LayaJudge | undefined => { const off = judgmentsOffReason(config); noteJudgments(config, off); if (off) return undefined; judgeConfig = config; // Absent, exactly as with no judge configured, so no guard needs to know a cooldown exists. if (cooldown.active()) { stats.cooldownSkips++; return undefined; } + if (config.typesafeBackend === "laya") return client ??= watched(new LayaJudge()); return client ??= watched(createTypeSafe(judgeOptions(config))); }; /** Every guard reaches the backend through `evaluate`, so this one seam sees each failure and each success. */ - const watched = (typesafe: TypeSafe): TypeSafe => new Proxy(typesafe, { + const watched = (judge: T): T => new Proxy(judge, { get(target, prop, receiver) { if (prop !== "evaluate") return Reflect.get(target, prop, receiver); return async (...args: Parameters) => { @@ -618,7 +622,8 @@ export default function wardenExtension(host: ExtensionAPI): void { const seconds = Math.round(event.ms / 1000); const remedy = event.kind === "auth" ? ` Set ${keyEnvFor(judgeConfig?.typesafeBackend ?? "typesafe")} or run /warden enable.` - : event.kind === "configuration" ? " /warden status shows the backend setup." : ""; + : event.kind === "configuration" ? " /warden status shows the backend setup." + : judgeConfig?.typesafeBackend === "laya" ? " Start the local judge server (http://127.0.0.1:8700 by default) and retry." : ""; noticeUi.notify(`warden: judgments paused for ${seconds}s after a ${event.kind} failure; pattern checks run alone until then.${remedy}`, "warning"); }; const noteError = (ctx: ExtensionContext, message: string, code: string | undefined) => { @@ -2436,7 +2441,10 @@ export default function wardenExtension(host: ExtensionAPI): void { try { const config = configFor(ctx); if (action === "status") { - const auth = describeAuth(authState({ backend: config.typesafeBackend })); + const local = config.typesafeBackend === "laya"; + const auth = config.typesafeBackend === "laya" + ? { level: "info" as const, text: "judge: local Laya-MLX (127.0.0.1:8700) — no key, nothing leaves this machine" } + : describeAuth(authState({ backend: config.typesafeBackend })); const source = consentSource(config); const usage = client?.getUsage(); const guards = [config.action.enabled && "action", config.stuck.enabled && "stuck", config.done.enabled && "done-check", config.slop.enabled && "slop", config.slop.enabled && config.slop.prose.enabled && `prose (${config.slop.prose.audience})`, config.security.enabled && "security", config.rules.enabled && "rules", config.context.enabled && "context", config.runaway.enabled && "runaway", config.subagent.enabled && "subagent triage", config.notify.enabled && "desktop notifications"].filter(Boolean).join(", "); @@ -2445,7 +2453,7 @@ export default function wardenExtension(host: ExtensionAPI): void { ? `Lifetime here: ${ls.held} hold${ls.held === 1 ? "" : "s"}, not yet measurable (${ls.allowed} allowed).` : `Lifetime here: ${ls.held} hold${ls.held === 1 ? "" : "s"}, ${ls.labeled} labeled, ${ls.declined + ls.replanned} stood (${ls.declined + ls.replanned}/${ls.labeled}), ${ls.allowed} allowed (${ls.accepted} accepted, ${ls.regretted} regretted).`; report([ - `pi-warden: ${config.enabled ? `guarding ${config.action.tools.join(", ")} (${guards})` : "off"}; mode ${activeMode(config, ctx.hasUI)}; TypeSafe judgments ${source ? `consented via ${source}` : "not consented (run /warden enable)"}${config.typesafeBackend !== "typesafe" ? ` (${config.typesafeBackend})` : ""}; ${auth.text}`, + `pi-warden: ${config.enabled ? `guarding ${config.action.tools.join(", ")} (${guards})` : "off"}; mode ${activeMode(config, ctx.hasUI)}; ${local ? "judgments" : "TypeSafe judgments"} ${source ? `consented via ${source}` : "not consented (run /warden enable)"}${config.typesafeBackend !== "typesafe" ? ` (${config.typesafeBackend})` : ""}; ${auth.text}`, `Session: ${stats.inspected} inspected, ${stats.judged} judged, ${stats.warned} warned, ${stats.held} held, ${stats.approved} approved on retry, ${stats.offPlan} off plan (${stats.offPlanTraceOnly} trace-only), ${stats.offTask} off task, ${stats.slop} slop notes, ${stats.ruleViolations}/${stats.ruleChecks} rule violations, ${stats.pathNotes} sensitive-path notes, ${stats.stuck}/${stats.stuckChecks} stuck, ${stats.unverified}/${stats.doneChecks} unverified done, ${stats.proseNudges}/${stats.proseChecks} prose nudges, ${stats.runaway} runaway stops, ${stats.subagentWoken}/${stats.subagentReports} subagent reports woken, ${stats.restatements} restatements, ${stats.errors} TypeSafe errors, ${stats.cooldownSkips} checks without Jev during a judge cooldown; ${usage?.requestsStarted ?? 0}/${config.maxRequests} requests. Steers are ${config.steerVisible ? "shown in the transcript" : "hidden from the transcript (trace panel shows them)"}. Steer budget: ${config.steerBudget === 0 ? "off" : `${config.steerBudget} per run`}.`, formatSteers(stats), `${formatMuted(steerStats.muted(), config.steers)}${stats.steersMuted ? ` This session: ${stats.steersMuted} steer${stats.steersMuted === 1 ? "" : "s"} kept in the trace only.` : ""}`, @@ -2687,6 +2695,15 @@ export default function wardenExtension(host: ExtensionAPI): void { return; } if (action === "enable") { + if (config.typesafeBackend === "laya") { + if (!ctx.hasUI) { report("Local Laya-MLX judgments need no key. For headless runs set PI_WARDEN_ENABLED=1 — judged content never leaves this machine.", "info"); return; } + if (!await ctx.ui.confirm("Enable local Laya-MLX judgments for pi-warden?", "Judged content stays on this machine; nothing is sent to any server.")) return; + const path = setUserSetting("typesafe", true, dirs); + client = undefined; + budgetExhausted = false; + report(`Local Laya-MLX judgments enabled and saved to ${path}. No key, no network. This stays on in new sessions until /warden disable.`); + return; + } if (!ctx.hasUI) { report(`Consent needs an interactive session. For headless runs set PI_WARDEN_ENABLED=1 and ${keyEnvFor(config.typesafeBackend)} explicitly.`, "warning"); return; } if (!await ctx.ui.confirm("Enable TypeSafe judgments for pi-warden?", disclosure)) return; // One flow: consent, then a key if none is configured yet. TypeSafe prompts, verifies, and stores the key for every @@ -2807,7 +2824,10 @@ export default function wardenExtension(host: ExtensionAPI): void { } if (action === "test") { const judge = judgeFor(config); - if (judge && ctx.hasUI && !await ctx.ui.confirm("Send one synthetic pi-warden test request?", `A synthetic action ("rm -rf /tmp/pi-warden-demo" for the task "Prepare the demo environment") goes to ${backendHost(config.typesafeBackend)} and may incur charges. ${disclosureFor(config.typesafeBackend, disclosure)}`)) return; + const where = config.typesafeBackend === "laya" + ? "runs locally against Laya-MLX — no network, no charges." + : `goes to ${backendHost(config.typesafeBackend)} and may incur charges. ${disclosureFor(config.typesafeBackend, disclosure)}`; + if (judge && ctx.hasUI && !await ctx.ui.confirm("Send one synthetic pi-warden test request?", `A synthetic action ("rm -rf /tmp/pi-warden-demo" for the task "Prepare the demo environment") ${where}`)) return; const verdict = await evaluateAction( { tool: "bash", input: { command: "rm -rf /tmp/pi-warden-demo" }, cwd: ctx.cwd, task: "Prepare the demo environment" }, { config: { ...config.action, enabled: true, tools: ["bash"] }, judge, rules: config.rules }, diff --git a/src/laya-judge.ts b/src/laya-judge.ts new file mode 100644 index 00000000..26b12624 --- /dev/null +++ b/src/laya-judge.ts @@ -0,0 +1,50 @@ +/** + * Laya-MLX judge adapter for pi-warden. + * + * Implements the pi-typesafe `Judge` interface by POSTing to a local Laya + * System One endpoint — the same /v1/systemone contract the remote TypeSafe + * API serves, so requests and answers are interchangeable. + * + * const judge = new LayaJudge(); + * const result = await judge.evaluate(request, { signal }); + */ +import type { EvaluationOptions, Judge, UsageSnapshot } from "pi-typesafe"; +import type { Questions, SystemOneRequest, SystemOneResult } from "@typesafe-ai/sdk"; + +const DEFAULT_URL = "http://127.0.0.1:8700/v1/systemone"; + +export class LayaJudge implements Judge { + private url: string; + private requests = 0; + + constructor(url: string = DEFAULT_URL) { + this.url = url; + } + + /** Evaluate a SystemOneRequest — the core Judge interface method. */ + // eslint-disable-next-line @typescript-eslint/no-explicit-any + async evaluate(request: SystemOneRequest, options?: EvaluationOptions): Promise { + const response = await fetch(this.url, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ state: request?.state ?? null, questions: request?.questions }), + signal: options?.signal ?? null, + }); + if (!response.ok) throw new Error(`laya-mlx judge returned HTTP ${response.status}`); + const result = (await response.json()) as SystemOneResult | null; + if (!result || typeof result.answers !== "object" || result.answers === null) throw new Error("laya-mlx judge returned no answers"); + this.requests++; + const usage = result.usage as { input_tokens: number; output_tokens: number; elapsed_ms?: number } | undefined; + return { + model: typeof result.model === "string" && result.model.length > 0 ? result.model : "laya-mlx", + answers: result.answers, + usage: usage ?? { input_tokens: 0, output_tokens: 0 }, + elapsedMs: typeof usage?.elapsed_ms === "number" ? usage.elapsed_ms : 0, + }; + } + + /** Session usage for /warden status; the local model bills nothing. */ + getUsage(): UsageSnapshot { + return { requestsStarted: this.requests, requestsSucceeded: this.requests, requestsFailed: 0, inputTokens: 0, outputTokens: 0, estimatedUsd: 0 }; + } +} diff --git a/tests/backend.test.ts b/tests/backend.test.ts index 30cfa4d2..247a4cbc 100644 --- a/tests/backend.test.ts +++ b/tests/backend.test.ts @@ -6,6 +6,7 @@ import { backendHost, disclosureFor, judgeOptions, keyEnvFor, resolveBackend } f test("resolveBackend: valid values pass through, junk falls back to typesafe", () => { assert.equal(resolveBackend("typesafe"), "typesafe"); assert.equal(resolveBackend("openrouter"), "openrouter"); + assert.equal(resolveBackend("laya"), "laya"); assert.equal(resolveBackend(undefined), "typesafe"); assert.equal(resolveBackend(null), "typesafe"); assert.equal(resolveBackend(""), "typesafe"); @@ -16,6 +17,7 @@ test("resolveBackend: valid values pass through, junk falls back to typesafe", ( test("backendHost returns the right host for each backend", () => { assert.equal(backendHost("typesafe"), "api.typesafe.ai"); assert.equal(backendHost("openrouter"), "openrouter.ai"); + assert.equal(backendHost("laya"), "this machine (local Laya-MLX)"); }); test("disclosureFor: typesafe returns the original text unchanged", () => { diff --git a/tests/laya-judge.test.ts b/tests/laya-judge.test.ts new file mode 100644 index 00000000..3abd15a0 --- /dev/null +++ b/tests/laya-judge.test.ts @@ -0,0 +1,90 @@ +import assert from "node:assert/strict"; +import { createServer, type Server } from "node:http"; +import type { AddressInfo } from "node:net"; +import { after, before, test } from "node:test"; +import type { Judge } from "pi-typesafe"; +import { LayaJudge } from "../src/laya-judge.js"; + +// A stand-in for the local Laya System One endpoint: records the request, replies with a scriptable status/body. +let server: Server; +let captured: { method: string; url: string; body: any } | undefined; +let reply: { status: number; body: unknown } = { status: 200, body: {} }; + +before(async () => { + server = createServer((req, res) => { + let raw = ""; + req.on("data", (chunk) => (raw += chunk)); + req.on("end", () => { + captured = { method: req.method ?? "", url: req.url ?? "", body: JSON.parse(raw) }; + res.writeHead(reply.status, { "content-type": "application/json" }); + res.end(JSON.stringify(reply.body)); + }); + }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); +}); +after(async () => { + await new Promise((resolve) => server.close(resolve)); +}); + +const url = () => `http://127.0.0.1:${(server.address() as AddressInfo).port}/v1/systemone`; +const request = () => ({ + state: { ticket: { messages: [{ sender: "user", text: "payout failed" }] } }, + questions: { needs_followup: { type: "noul" as const, instructions: "Does the message need a follow-up?" } }, +}); +const okBody = () => ({ + model: "aac6fef/laya-mlx", + answers: { needs_followup: { type: "noul", noul: 0.91 } }, + usage: { input_tokens: 42, output_tokens: 0, elapsed_ms: 31.5 }, +}); + +test("evaluate: posts the SystemOneRequest as JSON and returns the normalized result", async () => { + reply = { status: 200, body: okBody() }; + const judge: Judge = new LayaJudge(url()); + const result = await judge.evaluate(request()); + assert.equal(captured?.method, "POST"); + assert.equal(captured?.url, "/v1/systemone"); + assert.deepEqual(captured?.body, request()); + assert.equal(result.model, "aac6fef/laya-mlx"); + assert.deepEqual(result.answers, { needs_followup: { type: "noul", noul: 0.91 } }); + assert.deepEqual(result.usage, { input_tokens: 42, output_tokens: 0, elapsed_ms: 31.5 }); + assert.equal(result.elapsedMs, 31.5); +}); + +test("evaluate: falls back to the laya-mlx model name when the response omits it", async () => { + reply = { status: 200, body: { answers: okBody().answers, usage: okBody().usage } }; + const judge = new LayaJudge(url()); + const result = await judge.evaluate(request()); + assert.equal(result.model, "laya-mlx"); +}); + +test("evaluate: throws on an HTTP error status", async () => { + reply = { status: 500, body: { detail: "model exploded" } }; + const judge = new LayaJudge(url()); + await assert.rejects(() => judge.evaluate(request()), /HTTP 500/); +}); + +test("evaluate: throws when the response carries no answers", async () => { + reply = { status: 200, body: { model: "aac6fef/laya-mlx", usage: {} } }; + const judge = new LayaJudge(url()); + await assert.rejects(() => judge.evaluate(request()), /no answers/); +}); + +test("evaluate: rejects when the signal is already aborted", async () => { + reply = { status: 200, body: okBody() }; + const judge = new LayaJudge(url()); + const controller = new AbortController(); + controller.abort(); + await assert.rejects(() => judge.evaluate(request(), { signal: controller.signal })); +}); + +test("getUsage: counts evaluated requests", async () => { + reply = { status: 200, body: okBody() }; + const judge = new LayaJudge(url()); + await judge.evaluate(request()); + await judge.evaluate(request()); + const usage = judge.getUsage(); + assert.equal(usage.requestsStarted, 2); + assert.equal(usage.requestsSucceeded, 2); + assert.equal(usage.requestsFailed, 0); + assert.equal(usage.estimatedUsd, 0); +});