From d5b44fdda47a4db2d92a112aada59a5c47232a7e Mon Sep 17 00:00:00 2001 From: Ryan Gapac Date: Mon, 28 Sep 2026 21:00:31 +0800 Subject: [PATCH 1/2] feat: typesafeBackend takes commandcode and a caller-supplied endpoint pi-typesafe 0.8.0 resolves and validates the backend: the names "typesafe", "openrouter", and "commandcode", or an endpoint object with its own label, host, path, keyEnv, and defaultModel. The spec goes to createTypeSafe, authState, and ensureApiKey unchanged, so the status line describes the key the requests use. A non-TypeSafe backend names its label, host, and model in /warden status, the /warden enable dialog, and the /warden test confirmation, and the consent disclosure names the real destination host. The TypeSafe key and the login store never go to an endpoint object. An unknown typesafeBackend name, or an object the judge refuses, no longer silently falls back to typesafe: judgments turn off with the refusal message shown once and in /warden status, and nothing is sent to api.typesafe.ai because a value was mistyped. --- CHANGELOG.md | 9 ++++- docs/configuration.md | 5 ++- docs/data-handling.md | 2 +- package-lock.json | 8 ++-- package.json | 6 +-- src/backend.ts | 68 +++++++++++++++++++++---------- src/config.ts | 13 ++++-- src/conscience.ts | 1 + src/extension.ts | 73 +++++++++++++++++++++++---------- src/shape.ts | 10 ++++- tests/backend.test.ts | 81 ++++++++++++++++++++++++++----------- tests/config.test.ts | 20 ++++++--- tests/extension.test.ts | 89 +++++++++++++++++++++++++++++++++++++++-- 13 files changed, 293 insertions(+), 92 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4813855..8081cb5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,14 @@ How to keep this current: add the entry in the same pull request as the change, ## Unreleased - +### Added + +- `typesafeBackend` accepts `"commandcode"`: judgments go to api.commandcode.ai under `/provider/v1/systemone`, with the key from `COMMANDCODE_API_KEY` and the model `typesafe/jev`. +- `typesafeBackend` accepts a caller-supplied endpoint object (`{ "label", "host", "path", "keyEnv", "defaultModel" }`, plus optional model-list fields) for a gateway that serves the same decisions protocol (pi-typesafe 0.8.0). The object is passed to the judge as written and validated on every call. It reads only its own `keyEnv` variable: the TypeSafe key and the `/typesafe login` store are never sent to it. `/warden status`, the `/warden enable` dialog, and the `/warden test` confirmation name the label, host, and model sent for any non-TypeSafe backend, and the consent disclosure names the real destination host. + +### Changed + +- An unknown `typesafeBackend` name, or an endpoint object the judge refuses, no longer silently falls back to `"typesafe"`: judgments turn off, the refusal message is shown once and in `/warden status`, and nothing is sent to api.typesafe.ai because a value was mistyped. ## 0.73.1 diff --git a/docs/configuration.md b/docs/configuration.md index 9e1d9ba..b86d2ea 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -80,7 +80,7 @@ User file `~/.pi/agent/pi-warden/config.json` (owner-only). `/warden config` ope | --- | --- | | `enabled` | Master switch for the extension. | | `typesafe` | Consent to send requests to TypeSafe. Set by `/warden enable`; only the user file or `PI_WARDEN_ENABLED=1` can grant it. | -| `typesafeBackend` | The judgment service: `"typesafe"` (default) or `"openrouter"`. User file only — a project must not redirect judgments. | +| `typesafeBackend` | Where the judgments go: `"typesafe"` (default, api.typesafe.ai), `"openrouter"` (openrouter.ai), `"commandcode"` (api.commandcode.ai), or an endpoint object for a gateway that serves the same decisions protocol: `{ "label": "Corp judge gateway", "host": "https://gw.example.com", "path": "/judge/systemone", "keyEnv": "GW_JUDGE_KEY", "defaultModel": "jev-1.13" }`. `keyEnv` is required, must not be `TYPESAFE_API_KEY`, and names the environment variable that carries the endpoint's key: the TypeSafe key and the `/typesafe login` store are never sent to it. `path`, `defaultModel`, and the model-list fields `modelsPath`, `modelsField`, `modelsIdField`, `modelsVerifyKey` are optional; the host must be `https:` (`http:` only for a loopback host). User file only — a project must not redirect judgments. Nothing silently falls back: an unknown name or an object the judge refuses turns judgments off, and the refusal message is shown once and in `/warden status`. | | `mode` | `steer` (hold goes back to the agent), `confirm` (dialog for you), `advise` (never holds). User file only; a project's `.pi/pi-warden.json` cannot change it, by design. | | `timeoutMs` | Per-request timeout. On timeout the call is allowed with a warning when `action.failOpen` is true. | | `maxRequests` | Per-session request budget. When spent, pi-warden says so once and continues with offline checks. | @@ -263,12 +263,13 @@ It is a backstop for sanctioned work, not a boundary anyone hostile respects: ke | --- | --- | | `TYPESAFE_API_KEY` | Takes precedence over the key stored by `/warden enable` or `/typesafe login`. | | `OPENROUTER_API_KEY` | API key for the OpenRouter backend. Required when `typesafeBackend` is `"openrouter"`. | +| `COMMANDCODE_API_KEY` | API key for the Command Code backend. Required when `typesafeBackend` is `"commandcode"`. | | `PI_WARDEN_ENABLED=1` | Grants consent for headless runs (same as `"typesafe": true`). | | `PI_WARDEN_MODE=steer\|confirm\|advise` | Overrides `mode`. | | `PI_WARDEN_TRACE_DIR=` | Appends the trace to `/.jsonl`, one file per Pi session. For a host that runs Pi in RPC mode, where the status line and the sidebar never show. An empty or relative path turns it off. See [Trace file](#trace-file). | | `PI_WARDEN_HOST_PATHS=:` | Directories outside the project where the host lets its agent write, `:`-separated. A `write` or `edit` in one of them, after `..` and symlinks are resolved, is not held or warned by the outside-project rule. Every other check still applies: command rules, path rules, deny rules, sensitive paths, secrets, and Jev; the action summary still shows the path as outside the project, and the rules guard still does not judge these files against the project rules. Relative entries, empty entries, and `/` are ignored. Read once per session. Environment only: no config file can set or extend it. pi-warden's index directory (`pi-warden/index/` under Pi's agent directory), where `/warden index` asks the agent to write, is always treated as a host path; the rest of the agent directory is not. | -When a guard would ask Jev but cannot, Warden tells you once per session and reason why, and what to do: no consent (`/warden enable`, or `PI_WARDEN_ENABLED=1` headless), no key for the backend (set its key variable, or run `/typesafe login` for the TypeSafe backend), or a rejected key (named by where it comes from: run `/typesafe login` again for the key it saved, or check the key in the environment variable and run `/warden status`); a headless session gets a status message instead of a notice, and a spent request budget keeps its own warning. +When a guard would ask Jev but cannot, Warden tells you once per session and reason why, and what to do: no consent (`/warden enable`, or `PI_WARDEN_ENABLED=1` headless), a refused `typesafeBackend` (the refusal message, quoted; fix the value in the user config), no key for the backend (set its key variable, or run `/typesafe login` for the TypeSafe backend), or a rejected key (named by where it comes from: run `/typesafe login` again for the key it saved, or check the key in the environment variable and run `/warden status`); a headless session gets a status message instead of a notice, and a spent request budget keeps its own warning. ### Trace file diff --git a/docs/data-handling.md b/docs/data-handling.md index a07bf76..72b09fe 100644 --- a/docs/data-handling.md +++ b/docs/data-handling.md @@ -4,7 +4,7 @@ What pi-warden sends to TypeSafe, what it keeps on this machine, and what it nev ## What is sent, per guard -With consent, requests go to `https://api.typesafe.ai` (default) or `https://openrouter.ai` when `typesafeBackend` is set to `"openrouter"` in the user config. +With consent, requests go to `https://api.typesafe.ai` (default), or to the host `typesafeBackend` names in the user config: `https://openrouter.ai` for `"openrouter"`, `https://api.commandcode.ai` for `"commandcode"`, or the `host` of a caller-supplied endpoint object. `/warden status`, the `/warden enable` dialog, and the `/warden test` confirmation name the destination host — and for a non-TypeSafe backend the label, host, and model sent — so who answers is always visible before anything leaves. An endpoint object never receives the TypeSafe key or the `/typesafe login` store; it gets only the key from the environment variable its `keyEnv` names. | Guard | Sent | | --- | --- | diff --git a/package-lock.json b/package-lock.json index 9029ca8..8563036 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,7 +9,7 @@ "version": "0.65.1", "license": "MIT", "dependencies": { - "pi-typesafe": "^0.7.0" + "pi-typesafe": "^0.8.0" }, "devDependencies": { "@earendil-works/pi-coding-agent": "0.87.0", @@ -2615,9 +2615,9 @@ } }, "node_modules/pi-typesafe": { - "version": "0.7.0", - "resolved": "https://registry.npmjs.org/pi-typesafe/-/pi-typesafe-0.7.0.tgz", - "integrity": "sha512-423b/fuOywhEP2kqa900JAsFYEvDpbvSI6QYKYDl3/GKqXpAEEk6nrWzEQOpJoCydMj/zfSnfopIE5nBlypyEQ==", + "version": "0.8.0", + "resolved": "https://registry.npmjs.org/pi-typesafe/-/pi-typesafe-0.8.0.tgz", + "integrity": "sha512-8MYyd2i0tR7bhuayzxZQJwEP2ygMRoxUnR33PWwv/0aXO3ts1kgmdozPvR1TklYolqOpafMKbjTmnZb8ggPx2w==", "license": "MIT", "dependencies": { "@typesafe-ai/sdk": "^0.6.0", diff --git a/package.json b/package.json index 8dd9032..b1051b7 100644 --- a/package.json +++ b/package.json @@ -63,8 +63,8 @@ "eval:judge": "node scripts/judge-bench.mjs", "eval:replay": "node scripts/rules-replay.mjs", "eval:rules": "node scripts/rules-bench.mjs", - "eval:replay": "node scripts/rules-replay.mjs", - "eval:rules-turn": "node scripts/rules-turn-bench.mjs" }, + "eval:rules-turn": "node scripts/rules-turn-bench.mjs" + }, "peerDependencies": { "@earendil-works/pi-coding-agent": ">=0.85.1 <1", "@earendil-works/pi-tui": ">=0.85.1 <1" @@ -85,6 +85,6 @@ "typescript": "^6.0.0" }, "dependencies": { - "pi-typesafe": "^0.7.0" + "pi-typesafe": "^0.8.0" } } diff --git a/src/backend.ts b/src/backend.ts index 16f067b..486b818 100644 --- a/src/backend.ts +++ b/src/backend.ts @@ -1,30 +1,59 @@ -import type { TypeSafeOptions } from "pi-typesafe"; -import { DECISIONS_BACKENDS, DEFAULT_BACKEND } from "pi-typesafe"; +import type { BackendSpec, TypeSafeOptions } from "pi-typesafe"; +import { DEFAULT_BACKEND, backendHost, resolveBackend } from "pi-typesafe"; -/** The judgment backend that receives pi-warden's Jev requests. */ -export type JudgmentBackend = "typesafe" | "openrouter"; +/** The judgment backend: a registry name ("typesafe", "openrouter", "commandcode") or a caller-supplied endpoint object. */ +export type JudgmentBackend = BackendSpec; -/** Why no judge is available: consent not given, no key for the backend, a saved 401 or 403, or the request budget spent. */ -export type JudgmentsOffReason = "no_consent" | "no_key" | "key_rejected" | "budget"; +/** The judgment destination as the config carries it, plus the refusal that turns judgments off when the configured value was refused. */ +export interface BackendSetting { + /** The spec judgments go to; undefined only when the configured value was refused. */ + readonly typesafeBackend: JudgmentBackend | undefined; + /** Why the configured value was refused: the once-per-session notice and `/warden status` quote it. */ + readonly backendRefusal: string | undefined; +} + +/** What a raw `typesafeBackend` value resolves to: the spec judgments go to, or the refusal that turns them off. Exactly one is set. */ +export type BackendResolution = + | { readonly typesafeBackend: JudgmentBackend; readonly backendRefusal?: undefined } + | { readonly typesafeBackend?: undefined; readonly backendRefusal: string }; + +/** Why no judge is available: consent not given, the backend refused, no key for the backend, a saved 401 or 403, or the request budget spent. */ +export type JudgmentsOffReason = "no_consent" | "bad_backend" | "no_key" | "key_rejected" | "budget"; -/** The host the consent disclosure names as the destination, without scheme: `api.typesafe.ai`, `openrouter.ai`. */ -export function backendHost(backend: JudgmentBackend): string { - return new URL(DECISIONS_BACKENDS[backend].host).host; +/** + * Resolve a raw config value to the spec judgments go to, kept as written, or to the refusal that turns judgments off. + * pi-typesafe validates the spec again on every call. An unknown name or an object it refuses is never silently mapped + * to the default backend: a mistyped value must not send judgments to api.typesafe.ai. + */ +export function resolveJudgmentBackend(raw: unknown): BackendResolution { + if (raw === undefined || raw === null) return { typesafeBackend: DEFAULT_BACKEND }; + try { + resolveBackend(raw as BackendSpec); + } catch (error) { + return { backendRefusal: error instanceof Error ? error.message : String(error) }; + } + return { typesafeBackend: raw as JudgmentBackend }; } -/** The environment variable that carries the backend's key, for messages that tell the user what to set. */ -export function keyEnvFor(backend: JudgmentBackend): string { - return DECISIONS_BACKENDS[backend].keyEnv ?? DECISIONS_BACKENDS[DEFAULT_BACKEND].keyEnv ?? "TYPESAFE_API_KEY"; +/** The name messages show for a backend: the registry name, or an endpoint's validated label. */ +export function backendName(backend: JudgmentBackend | undefined): string { + return backend === undefined ? DEFAULT_BACKEND : typeof backend === "string" ? backend : resolveBackend(backend).label; } -/** Whether the backend takes the TypeSafe key, the only one `/typesafe login` stores; pi-typesafe's `usesTypesafeKey`, which its package does not export. */ -export function loginStoresKey(backend: JudgmentBackend): boolean { - return keyEnvFor(backend) === keyEnvFor(DEFAULT_BACKEND); +/** Whether the backend takes the TypeSafe key, the only one `/typesafe login` stores; true only for "typesafe". */ +export function loginStoresKey(backend: JudgmentBackend | undefined): boolean { + return backend === DEFAULT_BACKEND; +} + +/** One phrase naming where judgments go — the label, the host, and the model sent — for consent text and status lines. */ +export function describeBackend(backend: JudgmentBackend | undefined): string { + const resolved = resolveBackend(backend); + return `${resolved.label} at ${backendHost(backend)}, model ${resolved.defaultModel ?? "none configured"}`; } /** Adapt the consent text to the active backend by substituting the destination host. */ -export function disclosureFor(backend: JudgmentBackend, disclosure: string): string { - return backend === DEFAULT_BACKEND ? disclosure : disclosure.replace(backendHost(DEFAULT_BACKEND), backendHost(backend)); +export function disclosureFor(backend: JudgmentBackend | undefined, disclosure: string): string { + return backend === undefined || backend === DEFAULT_BACKEND ? disclosure : disclosure.replace(backendHost(DEFAULT_BACKEND), backendHost(backend)); } /** @@ -38,8 +67,3 @@ export function judgeOptions(config: { maxRequests: number; timeoutMs: number; t ...(config.typesafeBackend === DEFAULT_BACKEND ? {} : { backend: config.typesafeBackend }), }; } - -/** Resolve the effective backend from a raw config value; invalid values fall back to "typesafe". */ -export function resolveBackend(raw: unknown): JudgmentBackend { - return raw === "typesafe" || raw === "openrouter" ? raw : DEFAULT_BACKEND; -} diff --git a/src/config.ts b/src/config.ts index 43b7888..cd616f7 100644 --- a/src/config.ts +++ b/src/config.ts @@ -2,7 +2,7 @@ import { readFileSync } from "node:fs"; import { dirname, join } from "node:path"; import { writeFileAtomicSync } from "./atomic.js"; import type { JudgmentBackend } from "./backend.js"; -import { resolveBackend } from "./backend.js"; +import { resolveJudgmentBackend } from "./backend.js"; import { defaultHostDirs } from "./host-dirs.js"; import type { HostDirs } from "./host-dirs.js"; import { COMMAND_TOOLS } from "./tools.js"; @@ -420,8 +420,10 @@ export interface WardenConfig { enabled: boolean; /** Consent to send task and action summaries to api.typesafe.ai. Set by /warden enable; never by a project file. */ typesafe: boolean; - /** The decisions service the judgments go to. User file only: a project must not redirect judgments to another vendor. */ - typesafeBackend: JudgmentBackend; + /** The decisions service the judgments go to: a name ("typesafe", "openrouter", "commandcode") or a caller-supplied endpoint object. User file only: a project must not redirect judgments to another vendor. Undefined when the configured value was refused. */ + typesafeBackend: JudgmentBackend | undefined; + /** Why the configured typesafeBackend was refused: judgments stay off, and the once-per-session notice and /warden status quote this. */ + backendRefusal: string | undefined; /** * steer (default): a confirm-level call is held and the agent receives the judgment as its tool result, so it re-plans or asks * the user in chat. confirm: open a dialog and let the user decide (falls back to steer without a UI). advise: never hold; report only. @@ -476,6 +478,7 @@ export function defaultConfig(): WardenConfig { enabled: true, typesafe: false, typesafeBackend: "typesafe", + backendRefusal: undefined, mode: "steer", timeoutMs: 5000, maxRequests: 500, @@ -930,10 +933,12 @@ function applyGuards(base: WardenConfig, raw: Json, timeoutMs: number, source: " export function applyUserOverrides(base: WardenConfig, raw: unknown): WardenConfig { if (!isObject(raw)) return base; const shared = applyShared(base, raw); + const backend = resolveJudgmentBackend(raw.typesafeBackend); return { enabled: boolean(raw.enabled, base.enabled), typesafe: boolean(raw.typesafe, base.typesafe), - typesafeBackend: resolveBackend(raw.typesafeBackend), + typesafeBackend: backend.typesafeBackend, + backendRefusal: backend.backendRefusal, mode: isMode(raw.mode) ? raw.mode : base.mode, ...shared, ...applyGuards(base, raw, shared.timeoutMs, "user"), diff --git a/src/conscience.ts b/src/conscience.ts index 70005ca..21dd22e 100644 --- a/src/conscience.ts +++ b/src/conscience.ts @@ -82,6 +82,7 @@ export interface AssessmentResult { export type SkipReason = | "disabled" | "no_consent" + | "bad_backend" | "no_key" | "key_rejected" | "no_match" diff --git a/src/extension.ts b/src/extension.ts index c882684..4e470d8 100644 --- a/src/extension.ts +++ b/src/extension.ts @@ -10,11 +10,11 @@ import type { KeyId } from "@earendil-works/pi-tui"; import * as tuiModule from "@earendil-works/pi-tui"; type MouseRegionConstructor = new (child: ReturnType, onMouse: (event: { type: string; button: string }) => { handled: boolean } | undefined) => import("@earendil-works/pi-tui").Component; const MouseRegion: MouseRegionConstructor | undefined = (tuiModule as Partial<{ MouseRegion: MouseRegionConstructor }>).MouseRegion; -import { authState, createTypeSafe, describeAuth } from "pi-typesafe"; +import { authState, backendHost, createTypeSafe, describeAuth, resolveBackend } from "pi-typesafe"; import type { TypeSafe } from "pi-typesafe"; import { ensureApiKey } from "pi-typesafe/ui"; -import { backendHost, disclosureFor, judgeOptions, keyEnvFor, loginStoresKey, resolveBackend } from "./backend.js"; -import type { JudgmentBackend, JudgmentsOffReason } from "./backend.js"; +import { backendName, describeBackend, disclosureFor, judgeOptions, loginStoresKey } from "./backend.js"; +import type { BackendSetting, JudgmentsOffReason } from "./backend.js"; import { ActionGuard } from "./action-guard.js"; import { adaptHost } from "./host-compat.js"; import { assistantView, formatMuted, NEVER_MUTED, STEER_KINDS, SteerStats, SteerWatch, steerStatsPath } from "./adaptive.js"; @@ -221,10 +221,11 @@ function planForCall(entries: ReturnType, backend: JudgmentBackend, headless: boolean, rejectedEnv?: string): string { +export function judgmentsOffText(reason: Exclude, setting: BackendSetting, headless: boolean, rejectedEnv?: string): string { switch (reason) { case "no_consent": return `warden: Jev judgments are off (no consent). ${headless ? "Set PI_WARDEN_ENABLED=1." : "Run /warden enable."}`; - case "no_key": return `warden: Jev judgments are off (no key for ${backend}). Set ${keyEnvFor(backend)}${loginStoresKey(backend) ? " or run /typesafe login" : ""}.`; + case "bad_backend": return `warden: Jev judgments are off (typesafeBackend refused: ${setting.backendRefusal ?? "the configured value is not usable"})`; + case "no_key": return `warden: Jev judgments are off (no key for ${backendName(setting.typesafeBackend)}). Set ${resolveBackend(setting.typesafeBackend).keyEnv}${loginStoresKey(setting.typesafeBackend) ? " or run /typesafe login" : ""}.`; case "key_rejected": return rejectedEnv ? `warden: Jev judgments are off (the key in ${rejectedEnv} was rejected). Check the key, then run /warden status.` : "warden: Jev judgments are off (the key saved by /typesafe login was rejected). Run /typesafe login."; @@ -564,9 +565,12 @@ export default function wardenExtension(host: ExtensionAPI): void { const consentGiven = (config: WardenConfig) => config.typesafe || process.env.PI_WARDEN_ENABLED === "1"; /** Why no judge is available, or undefined when one is. The notice, the trace file, and the conscience all use it. */ const judgmentsOffReason = (config: WardenConfig): JudgmentsOffReason | undefined => { + // A refused backend is a config error the user must see even before consent or keys matter. + const backend = config.typesafeBackend; + if (config.backendRefusal !== undefined || backend === undefined) return "bad_backend"; if (!consentGiven(config)) return "no_consent"; if (budgetExhausted) return "budget"; - const auth = authState({ backend: config.typesafeBackend }); + const auth = authState({ backend }); if (auth.usable) return undefined; // No key in effect, or one whose last request came back 401 or 403. return auth.source === undefined ? "no_key" : "key_rejected"; @@ -581,8 +585,8 @@ export default function wardenExtension(host: ExtensionAPI): void { // The budget has its own notice in noteError. if (reason === undefined || reason === "budget" || judgmentsReported.has(reason)) return; judgmentsReported.add(reason); - const auth = reason === "key_rejected" ? authState({ backend: config.typesafeBackend }) : undefined; - judgmentsNotify?.(judgmentsOffText(reason, config.typesafeBackend, judgmentsHeadless, auth?.kind === "environment" ? auth.keyName : undefined)); + const auth = reason === "key_rejected" && config.typesafeBackend !== undefined ? authState({ backend: config.typesafeBackend }) : undefined; + judgmentsNotify?.(judgmentsOffText(reason, config, judgmentsHeadless, auth?.kind === "environment" ? auth.keyName : undefined)); }; const consentSource = (config: WardenConfig) => config.typesafe ? "/warden enable" : process.env.PI_WARDEN_ENABLED === "1" ? "PI_WARDEN_ENABLED" : undefined; /** A consent flag is not proof that judgments happen; check the key state for the chosen backend. */ @@ -593,7 +597,23 @@ export default function wardenExtension(host: ExtensionAPI): void { judgeConfig = config; // Absent, exactly as with no judge configured, so no guard needs to know a cooldown exists. if (cooldown.active()) { stats.cooldownSkips++; return undefined; } - return client ??= watched(createTypeSafe(judgeOptions(config))); + if (client) return client; + const backend = config.typesafeBackend; + // Reached only if a config ever sets no backend and no refusal at once; treating it as off keeps the invariant. + if (backend === undefined) return undefined; + try { + client = watched(createTypeSafe(judgeOptions({ maxRequests: config.maxRequests, timeoutMs: config.timeoutMs, typesafeBackend: backend }))); + } catch (error) { + // An endpoint the config accepts can still be unusable (one that names no defaultModel); a hook must never crash. + const message = error instanceof Error ? error.message : String(error); + if (!judgmentsReported.has("bad_backend")) { + judgmentsReported.add("bad_backend"); + traceFile?.judgments(judgmentsState("bad_backend")); + judgmentsNotify?.(`warden: Jev judgments are off (typesafeBackend refused: ${message})`); + } + return undefined; + } + return client; }; /** Every guard reaches the backend through `evaluate`, so this one seam sees each failure and each success. */ const watched = (typesafe: TypeSafe): TypeSafe => new Proxy(typesafe, { @@ -617,7 +637,7 @@ export default function wardenExtension(host: ExtensionAPI): void { if (event.type === "recovered") { noticeUi.notify("warden: judgments resumed.", "info"); return; } const seconds = Math.round(event.ms / 1000); const remedy = event.kind === "auth" - ? ` Set ${keyEnvFor(judgeConfig?.typesafeBackend ?? "typesafe")} or run /warden enable.` + ? ` Set ${resolveBackend(judgeConfig?.typesafeBackend).keyEnv} or run /warden enable.` : event.kind === "configuration" ? " /warden status shows the backend setup." : ""; noticeUi.notify(`warden: judgments paused for ${seconds}s after a ${event.kind} failure; pattern checks run alone until then.${remedy}`, "warning"); }; @@ -2436,7 +2456,9 @@ export default function wardenExtension(host: ExtensionAPI): void { try { const config = configFor(ctx); if (action === "status") { - const auth = describeAuth(authState({ backend: config.typesafeBackend })); + // A refused backend has no key to describe; the refusal itself is the status line. + const backend = config.typesafeBackend; + const auth = config.backendRefusal !== undefined || backend === undefined ? undefined : describeAuth(authState({ backend })); const source = consentSource(config); const usage = client?.getUsage(); const guards = [config.action.enabled && "action", config.stuck.enabled && "stuck", config.done.enabled && "done-check", config.slop.enabled && "slop", config.slop.enabled && config.slop.prose.enabled && `prose (${config.slop.prose.audience})`, config.security.enabled && "security", config.rules.enabled && "rules", config.context.enabled && "context", config.runaway.enabled && "runaway", config.subagent.enabled && "subagent triage", config.notify.enabled && "desktop notifications"].filter(Boolean).join(", "); @@ -2445,7 +2467,7 @@ export default function wardenExtension(host: ExtensionAPI): void { ? `Lifetime here: ${ls.held} hold${ls.held === 1 ? "" : "s"}, not yet measurable (${ls.allowed} allowed).` : `Lifetime here: ${ls.held} hold${ls.held === 1 ? "" : "s"}, ${ls.labeled} labeled, ${ls.declined + ls.replanned} stood (${ls.declined + ls.replanned}/${ls.labeled}), ${ls.allowed} allowed (${ls.accepted} accepted, ${ls.regretted} regretted).`; report([ - `pi-warden: ${config.enabled ? `guarding ${config.action.tools.join(", ")} (${guards})` : "off"}; mode ${activeMode(config, ctx.hasUI)}; TypeSafe judgments ${source ? `consented via ${source}` : "not consented (run /warden enable)"}${config.typesafeBackend !== "typesafe" ? ` (${config.typesafeBackend})` : ""}; ${auth.text}`, + `pi-warden: ${config.enabled ? `guarding ${config.action.tools.join(", ")} (${guards})` : "off"}; mode ${activeMode(config, ctx.hasUI)}; TypeSafe judgments ${source ? `consented via ${source}` : "not consented (run /warden enable)"}${config.backendRefusal !== undefined || backend === undefined || backend === "typesafe" ? "" : ` (${describeBackend(backend)})`}; ${config.backendRefusal !== undefined ? `judgments are off: typesafeBackend refused: ${config.backendRefusal}` : auth?.text ?? ""}`, `Session: ${stats.inspected} inspected, ${stats.judged} judged, ${stats.warned} warned, ${stats.held} held, ${stats.approved} approved on retry, ${stats.offPlan} off plan (${stats.offPlanTraceOnly} trace-only), ${stats.offTask} off task, ${stats.slop} slop notes, ${stats.ruleViolations}/${stats.ruleChecks} rule violations, ${stats.pathNotes} sensitive-path notes, ${stats.stuck}/${stats.stuckChecks} stuck, ${stats.unverified}/${stats.doneChecks} unverified done, ${stats.proseNudges}/${stats.proseChecks} prose nudges, ${stats.runaway} runaway stops, ${stats.subagentWoken}/${stats.subagentReports} subagent reports woken, ${stats.restatements} restatements, ${stats.errors} TypeSafe errors, ${stats.cooldownSkips} checks without Jev during a judge cooldown; ${usage?.requestsStarted ?? 0}/${config.maxRequests} requests. Steers are ${config.steerVisible ? "shown in the transcript" : "hidden from the transcript (trace panel shows them)"}. Steer budget: ${config.steerBudget === 0 ? "off" : `${config.steerBudget} per run`}.`, formatSteers(stats), `${formatMuted(steerStats.muted(), config.steers)}${stats.steersMuted ? ` This session: ${stats.steersMuted} steer${stats.steersMuted === 1 ? "" : "s"} kept in the trace only.` : ""}`, @@ -2477,7 +2499,7 @@ export default function wardenExtension(host: ExtensionAPI): void { switch (result.status) { // No key, no consent, or a spent budget: say why and send nothing. case "no-judge": - report(`Rules audit sent nothing: Jev judgments are off${off === "budget" ? " (the request budget is used up)" : ""}.${off && off !== "budget" ? ` ${judgmentsOffText(off, config.typesafeBackend, !ctx.hasUI)}` : ""}`); + report(`Rules audit sent nothing: Jev judgments are off${off === "budget" ? " (the request budget is used up)" : ""}.${off && off !== "budget" ? ` ${judgmentsOffText(off, config, !ctx.hasUI)}` : ""}`); return; case "no-files": report(`Rules audit sent nothing: ${result.reason}.`); return; case "needs-yes": report(`Rules audit sent nothing: a headless run needs --yes before ${result.files} file samples go to ${backendHost(config.typesafeBackend)} (${result.leftOut} more files left out at the --max ${parsed.max} cap).`, "warning"); return; @@ -2495,7 +2517,7 @@ export default function wardenExtension(host: ExtensionAPI): void { switch (result.status) { // No key, no consent, or a spent budget: say why and send nothing. case "no-judge": - report(`Bench sent nothing: Jev judgments are off${off === "budget" ? " (the request budget is used up)" : ""}.${off && off !== "budget" ? ` ${judgmentsOffText(off, config.typesafeBackend, !ctx.hasUI)}` : ""}`); + report(`Bench sent nothing: Jev judgments are off${off === "budget" ? " (the request budget is used up)" : ""}.${off && off !== "budget" ? ` ${judgmentsOffText(off, config, !ctx.hasUI)}` : ""}`); return; case "skipped": report(`Bench sent nothing: ${result.reason}.`); return; case "done": report(formatBench(result)); return; @@ -2508,7 +2530,7 @@ export default function wardenExtension(host: ExtensionAPI): void { const result = await checkRules({ set: rulesGuard.store.load(ctx.cwd, config.rules), judge: judgeFor(config), timeoutMs: config.timeoutMs, signal: ctx.signal }); if (result.source === "skipped") { // No key, no consent, or a spent budget: say why and send nothing. - report(`Rules check sent nothing: Jev judgments are off${off === "budget" ? " (the request budget is used up)" : ""}.${off && off !== "budget" ? ` ${judgmentsOffText(off, config.typesafeBackend, !ctx.hasUI)}` : ""} /warden rules shows the rule set locally.`); + report(`Rules check sent nothing: Jev judgments are off${off === "budget" ? " (the request budget is used up)" : ""}.${off && off !== "budget" ? ` ${judgmentsOffText(off, config, !ctx.hasUI)}` : ""} /warden rules shows the rule set locally.`); return; } if (result.source === "typesafe") lastCheck = result; @@ -2530,7 +2552,7 @@ export default function wardenExtension(host: ExtensionAPI): void { const off = judgmentsOffReason(config); const judge = judgeFor(config); if (!judge) { - report(`Rules calibrate sent nothing: Jev judgments are off${off === "budget" ? " (the request budget is used up)" : ""}.${off && off !== "budget" ? ` ${judgmentsOffText(off, config.typesafeBackend, !ctx.hasUI)}` : ""} /warden rules shows the rule set locally.`); + report(`Rules calibrate sent nothing: Jev judgments are off${off === "budget" ? " (the request budget is used up)" : ""}.${off && off !== "budget" ? ` ${judgmentsOffText(off, config, !ctx.hasUI)}` : ""} /warden rules shows the rule set locally.`); return; } let history: HistoryCommit[]; @@ -2687,16 +2709,22 @@ export default function wardenExtension(host: ExtensionAPI): void { return; } if (action === "enable") { - if (!ctx.hasUI) { report(`Consent needs an interactive session. For headless runs set PI_WARDEN_ENABLED=1 and ${keyEnvFor(config.typesafeBackend)} explicitly.`, "warning"); return; } - if (!await ctx.ui.confirm("Enable TypeSafe judgments for pi-warden?", disclosure)) return; + if (config.backendRefusal !== undefined) { report(`Cannot enable judgments: typesafeBackend refused: ${config.backendRefusal} Fix the value in ${userConfigPath(dirs)}, then run /warden enable again.`, "warning"); return; } + const backend = config.typesafeBackend; + if (!ctx.hasUI) { report(`Consent needs an interactive session. For headless runs set PI_WARDEN_ENABLED=1 and ${resolveBackend(backend).keyEnv} explicitly.`, "warning"); return; } + // For a non-TypeSafe backend the dialog names who answers: the label, host, and model sent. + const consentText = backend === undefined || backend === "typesafe" + ? disclosure + : `Judgments go to ${describeBackend(backend)}. ${disclosureFor(backend, disclosure)}`; + if (!await ctx.ui.confirm("Enable TypeSafe judgments for pi-warden?", consentText)) return; // One flow: consent, then a key if none is configured yet. TypeSafe prompts, verifies, and stores the key for every // pi-typesafe consumer; any other backend has no login, so a missing key is reported with the variable to set. - const key = await ensureApiKey(ctx, { backend: config.typesafeBackend }); + const key = await ensureApiKey(ctx, backend === undefined ? {} : { backend }); if (!key) { report("No key entered; pi-warden stays on pattern checks only. Run /warden enable again when you have a key from console.typesafe.ai.", "warning"); return; } const path = setUserSetting("typesafe", true, dirs); client = undefined; budgetExhausted = false; - report(`TypeSafe judgments enabled and saved to ${path}${key.login ? `; key verified (${key.login.models} model${key.login.models === 1 ? "" : "s"}) and stored at ${key.login.path}` : ` using the ${key.source === "stored" ? "stored key" : `key from ${keyEnvFor(config.typesafeBackend)}`}`}. This stays on in new sessions until /warden disable.`); + report(`TypeSafe judgments enabled and saved to ${path}${key.login ? `; key verified (${key.login.models} model${key.login.models === 1 ? "" : "s"}) and stored at ${key.login.path}` : ` using the ${key.source === "stored" ? "stored key" : `key from ${resolveBackend(config.typesafeBackend).keyEnv}`}`}. This stays on in new sessions until /warden disable.`); return; } if (action === "disable") { @@ -2807,7 +2835,10 @@ export default function wardenExtension(host: ExtensionAPI): void { } if (action === "test") { const judge = judgeFor(config); - if (judge && ctx.hasUI && !await ctx.ui.confirm("Send one synthetic pi-warden test request?", `A synthetic action ("rm -rf /tmp/pi-warden-demo" for the task "Prepare the demo environment") goes to ${backendHost(config.typesafeBackend)} and may incur charges. ${disclosureFor(config.typesafeBackend, disclosure)}`)) return; + // For a non-TypeSafe backend the confirmation names who answers: the label, host, and model sent. + const backend = config.typesafeBackend; + const destination = backend === undefined || backend === "typesafe" ? backendHost(backend) : describeBackend(backend); + if (judge && ctx.hasUI && !await ctx.ui.confirm("Send one synthetic pi-warden test request?", `A synthetic action ("rm -rf /tmp/pi-warden-demo" for the task "Prepare the demo environment") goes to ${destination} and may incur charges. ${disclosureFor(config.typesafeBackend, disclosure)}`)) return; const verdict = await evaluateAction( { tool: "bash", input: { command: "rm -rf /tmp/pi-warden-demo" }, cwd: ctx.cwd, task: "Prepare the demo environment" }, { config: { ...config.action, enabled: true, tools: ["bash"] }, judge, rules: config.rules }, diff --git a/src/shape.ts b/src/shape.ts index eeec2d4..4158f21 100644 --- a/src/shape.ts +++ b/src/shape.ts @@ -1,6 +1,6 @@ import { isMode } from "./config.js"; import type { WardenConfig } from "./config.js"; -import { resolveBackend } from "./backend.js"; +import { resolveJudgmentBackend } from "./backend.js"; import { redact } from "./redact.js"; import { DEFAULT_TEMPLATES } from "./widget.js"; @@ -17,6 +17,10 @@ const off = { enabled: false }; // Duplicated from defaultConfig(): this file must work when the config module is stale and lacks the key. const coreTools = () => ["read", "bash", "edit", "write", "grep", "find", "ls"]; const proseOff = () => ({ ...off, audience: "technical", threshold: 1, trend: 3, minChars: 1 }); +// A stale config module hands back either its own resolution or a raw value; both end up resolved here. +const resolveSetting = (source: Partial) => source.backendRefusal !== undefined + ? { typesafeBackend: source.typesafeBackend, backendRefusal: source.backendRefusal } + : resolveJudgmentBackend(source.typesafeBackend); /** * `loadConfig()` always returns a complete object, yet live sessions crashed at `config.slop.prose.enabled` after a package @@ -32,10 +36,12 @@ export function completeConfig(loaded: Partial | undefined): Shape missing.push(key); return fallback; }; + const backend = resolveSetting(source); const config: WardenConfig = { enabled: source.enabled ?? true, typesafe: source.typesafe ?? false, - typesafeBackend: resolveBackend(source.typesafeBackend), + typesafeBackend: backend.typesafeBackend, + backendRefusal: backend.backendRefusal, mode: isMode(source.mode) ? source.mode : "steer", timeoutMs: source.timeoutMs ?? 5000, maxRequests: source.maxRequests ?? 500, diff --git a/tests/backend.test.ts b/tests/backend.test.ts index 30cfa4d..98d3f45 100644 --- a/tests/backend.test.ts +++ b/tests/backend.test.ts @@ -1,21 +1,48 @@ import assert from "node:assert/strict"; import { test } from "node:test"; import { DECISIONS_BACKENDS } from "pi-typesafe"; -import { backendHost, disclosureFor, judgeOptions, keyEnvFor, resolveBackend } from "../src/backend.js"; +import { backendName, describeBackend, disclosureFor, judgeOptions, loginStoresKey, resolveJudgmentBackend } from "../src/backend.js"; -test("resolveBackend: valid values pass through, junk falls back to typesafe", () => { - assert.equal(resolveBackend("typesafe"), "typesafe"); - assert.equal(resolveBackend("openrouter"), "openrouter"); - assert.equal(resolveBackend(undefined), "typesafe"); - assert.equal(resolveBackend(null), "typesafe"); - assert.equal(resolveBackend(""), "typesafe"); - assert.equal(resolveBackend("azure"), "typesafe"); - assert.equal(resolveBackend(42), "typesafe"); +const gateway = { label: "Acme judge gateway", host: "https://gw.acme.example", path: "/judge/v1/decide", keyEnv: "ACME_JUDGE_KEY", defaultModel: "jev-1.13" }; + +test("resolveJudgmentBackend: names pass through, junk is refused instead of silently mapped to typesafe", () => { + assert.deepEqual(resolveJudgmentBackend("typesafe"), { typesafeBackend: "typesafe" }); + assert.deepEqual(resolveJudgmentBackend("openrouter"), { typesafeBackend: "openrouter" }); + assert.deepEqual(resolveJudgmentBackend("commandcode"), { typesafeBackend: "commandcode" }); + assert.deepEqual(resolveJudgmentBackend(undefined), { typesafeBackend: "typesafe" }); + assert.deepEqual(resolveJudgmentBackend(null), { typesafeBackend: "typesafe" }); + for (const junk of ["", "azure", 42]) { + const resolution = resolveJudgmentBackend(junk); + assert.equal(resolution.typesafeBackend, undefined, `${JSON.stringify(junk)} must not fall back to typesafe`); + assert.ok(resolution.backendRefusal, "the refusal message is kept for the notice and the status line"); + } + assert.match(resolveJudgmentBackend("azure").backendRefusal!, /Unknown judgment backend "azure"/); }); -test("backendHost returns the right host for each backend", () => { - assert.equal(backendHost("typesafe"), "api.typesafe.ai"); - assert.equal(backendHost("openrouter"), "openrouter.ai"); +test("resolveJudgmentBackend: an endpoint object is kept as written, and one pi-typesafe refuses is refused here too", () => { + assert.equal(resolveJudgmentBackend(gateway).typesafeBackend, gateway, "the object passes through unchanged; pi-typesafe validates it on every call"); + const bad = resolveJudgmentBackend({ label: "Acme judge gateway", host: "http://gw.acme.example", keyEnv: "ACME_JUDGE_KEY" }); + assert.equal(bad.typesafeBackend, undefined); + assert.match(bad.backendRefusal!, /absolute https/); +}); + +test("loginStoresKey is true only for typesafe", () => { + assert.equal(loginStoresKey("typesafe"), true); + assert.equal(loginStoresKey("openrouter"), false); + assert.equal(loginStoresKey("commandcode"), false); + assert.equal(loginStoresKey(gateway), false, "an endpoint object never gets the TypeSafe login store"); +}); + +test("backendName shows the registry name or the endpoint label", () => { + assert.equal(backendName("typesafe"), "typesafe"); + assert.equal(backendName("openrouter"), "openrouter"); + assert.equal(backendName(gateway), "Acme judge gateway"); +}); + +test("describeBackend names the label, host, and model sent", () => { + assert.equal(describeBackend("typesafe"), "TypeSafe at api.typesafe.ai, model jev-latest"); + assert.equal(describeBackend("commandcode"), "Command Code at api.commandcode.ai, model typesafe/jev"); + assert.equal(describeBackend(gateway), "Acme judge gateway at gw.acme.example, model jev-1.13"); }); test("disclosureFor: typesafe returns the original text unchanged", () => { @@ -31,13 +58,9 @@ test("disclosureFor: openrouter replaces the host in the disclosure", () => { assert.equal(result, "pi-warden sends to openrouter.ai: your latest request"); }); -test("backendHost and keyEnvFor come from pi-typesafe's registry, not a local copy", () => { - for (const backend of ["typesafe", "openrouter"] as const) { - assert.equal(backendHost(backend), new URL(DECISIONS_BACKENDS[backend].host).host); - assert.equal(keyEnvFor(backend), DECISIONS_BACKENDS[backend].keyEnv); - } - assert.equal(keyEnvFor("typesafe"), "TYPESAFE_API_KEY"); - assert.equal(keyEnvFor("openrouter"), "OPENROUTER_API_KEY"); +test("disclosureFor: an endpoint object replaces the host with its own", () => { + const text = "pi-warden sends to api.typesafe.ai: your latest request"; + assert.equal(disclosureFor(gateway, text), "pi-warden sends to gw.acme.example: your latest request"); }); test("judgeOptions: typesafe backend omits the backend field", () => { @@ -47,9 +70,19 @@ test("judgeOptions: typesafe backend omits the backend field", () => { assert.equal("backend" in opts, false, "default backend should not be forwarded"); }); -test("judgeOptions: openrouter includes the backend field", () => { - const opts = judgeOptions({ maxRequests: 10, timeoutMs: 3000, typesafeBackend: "openrouter" }); - assert.equal(opts.maxRequests, 10); - assert.equal(opts.timeoutMs, 3000); - assert.equal((opts as Record).backend, "openrouter"); +test("judgeOptions: commandcode and openrouter are forwarded as names", () => { + assert.equal((judgeOptions({ maxRequests: 10, timeoutMs: 3000, typesafeBackend: "commandcode" }) as Record).backend, "commandcode"); + assert.equal((judgeOptions({ maxRequests: 10, timeoutMs: 3000, typesafeBackend: "openrouter" }) as Record).backend, "openrouter"); +}); + +test("judgeOptions: an endpoint object reaches createTypeSafe unchanged", () => { + const opts = judgeOptions({ maxRequests: 10, timeoutMs: 3000, typesafeBackend: gateway }); + assert.equal(opts.backend, gateway); +}); + +test("backend hosts come from pi-typesafe's registry, not a local copy", () => { + for (const backend of ["typesafe", "openrouter", "commandcode"] as const) { + const host = new URL(DECISIONS_BACKENDS[backend].host).host; + assert.match(describeBackend(backend), new RegExp(host.replace(/[.]/g, "\\."))); + } }); diff --git a/tests/config.test.ts b/tests/config.test.ts index 1b943cc..eb8fae7 100644 --- a/tests/config.test.ts +++ b/tests/config.test.ts @@ -297,19 +297,29 @@ test("arming rules parse duration strings and numbers", () => { test("defaults: typesafeBackend is typesafe", () => { assert.equal(defaultConfig().typesafeBackend, "typesafe"); + assert.equal(defaultConfig().backendRefusal, undefined); }); -test("user overrides: typesafeBackend accepts valid values and ignores junk", () => { +test("user overrides: typesafeBackend accepts names and endpoint objects and refuses junk", () => { assert.equal(applyUserOverrides(defaultConfig(), { typesafeBackend: "openrouter" }).typesafeBackend, "openrouter"); + assert.equal(applyUserOverrides(defaultConfig(), { typesafeBackend: "commandcode" }).typesafeBackend, "commandcode"); assert.equal(applyUserOverrides(defaultConfig(), { typesafeBackend: "typesafe" }).typesafeBackend, "typesafe"); - assert.equal(applyUserOverrides(defaultConfig(), { typesafeBackend: "azure" }).typesafeBackend, "typesafe", "invalid backend falls back"); + const gateway = { label: "Acme judge gateway", host: "https://gw.acme.example", path: "/judge/v1/decide", keyEnv: "ACME_JUDGE_KEY", defaultModel: "jev-1.13" }; + assert.deepEqual(applyUserOverrides(defaultConfig(), { typesafeBackend: gateway }).typesafeBackend, gateway, "an endpoint object is kept as written"); + const refusedName = applyUserOverrides(defaultConfig(), { typesafeBackend: "azure" }); + assert.equal(refusedName.typesafeBackend, undefined, "an unknown name no longer falls back to typesafe"); + assert.match(refusedName.backendRefusal!, /Unknown judgment backend "azure"/); + const refusedObject = applyUserOverrides(defaultConfig(), { typesafeBackend: { label: "Acme judge gateway", host: "http://gw.acme.example", keyEnv: "ACME_JUDGE_KEY" } }); + assert.equal(refusedObject.typesafeBackend, undefined, "an object pi-typesafe refuses turns judgments off"); + assert.match(refusedObject.backendRefusal!, /absolute https/); assert.equal(applyUserOverrides(defaultConfig(), { typesafeBackend: null }).typesafeBackend, "typesafe", "null falls back"); assert.equal(applyUserOverrides(defaultConfig(), {}).typesafeBackend, "typesafe", "missing falls back"); }); -test("project overrides cannot set typesafeBackend", () => { - const config = applyProjectOverrides(defaultConfig(), { typesafeBackend: "openrouter" }); - assert.equal(config.typesafeBackend, "typesafe", "project file cannot redirect judgments"); +test("project overrides cannot set typesafeBackend in either form", () => { + const gateway = { label: "Evil gateway", host: "https://evil.example", keyEnv: "EVIL_KEY", defaultModel: "jev-1.13" }; + assert.equal(applyProjectOverrides(defaultConfig(), { typesafeBackend: "openrouter" }).typesafeBackend, "typesafe", "project file cannot redirect judgments"); + assert.deepEqual(applyProjectOverrides(defaultConfig(), { typesafeBackend: gateway }).typesafeBackend, "typesafe", "project file cannot redirect judgments with an endpoint object"); }); test("floor: user 'level' persists through project override without floor key", () => { diff --git a/tests/extension.test.ts b/tests/extension.test.ts index ce55ac5..0669454 100644 --- a/tests/extension.test.ts +++ b/tests/extension.test.ts @@ -41,7 +41,10 @@ let hangNetwork = false; let failStatus: number | undefined; const sentMessages: Array<{ message: { customType: string; content: string }; options?: Record }> = []; const sentUserMessages: Array = []; -const requests: Array<{ state: Record; questions: Record }> = []; +const requests: Array<{ model?: string; state: Record; questions: Record }> = []; +/** Where each judgment request went, and with what Authorization header, so a backend test can see the wire. */ +const requestUrls: string[] = []; +const requestAuth: Array = []; let prompt: string | undefined = "Run the test suite"; const ui = { @@ -149,8 +152,10 @@ before(async () => { signal.addEventListener("abort", () => reject(signal.reason), { once: true }); }); if (failStatus !== undefined) return new Response("upstream body must not leak", { status: failStatus }); - const body = JSON.parse(String(init?.body)) as { state: Record; questions: Record }; + const body = JSON.parse(String(init?.body)) as { model?: string; state: Record; questions: Record }; requests.push(body); + requestUrls.push(String(input)); + requestAuth.push(new Headers(init?.headers).get("authorization")); // Answer every asked question from nextAnswers so slop, approval, stuck, and done requests all work with one mock. const answers: Record = {}; for (const [id, question] of Object.entries(body.questions)) { @@ -193,7 +198,7 @@ before(async () => { beforeEach(async () => { notices.length = 0; widgets.length = 0; confirms.length = 0; confirmResult = true; editorText = undefined; networkCalls = 0; failNetwork = false; hangNetwork = false; failStatus = undefined; prompt = "Run the test suite"; - keyPrompts = 0; keyInput = undefined; modelListCalls = 0; sentMessages.length = 0; requests.length = 0; + keyPrompts = 0; keyInput = undefined; modelListCalls = 0; sentMessages.length = 0; requests.length = 0; requestUrls.length = 0; requestAuth.length = 0; widgetComponent = undefined; widgetPlacement = undefined; customCalls.length = 0; openPanels.length = 0; panelClosed.length = 0; renders = 0; await rm(join(temporary, "agent", "pi-typesafe"), { recursive: true, force: true }); nextAnswers = { irreversible: 0.1, off_task: 0.1, scope: "expected_step" }; @@ -4179,6 +4184,84 @@ test("judgments off: no key on OpenRouter names only its variable, since /typesa } }); +const gateway = { label: "Acme judge gateway", host: "https://gw.acme.example", path: "/judge/v1/decide", keyEnv: "ACME_JUDGE_KEY", defaultModel: "jev-1.13" }; + +test("the commandcode backend sends judgments to its own host, path, and model", async () => { + await writeFile(configPath(), JSON.stringify({ typesafe: true, typesafeBackend: "commandcode", rules: { enabled: false }, ...STACK_BAR })); + const saved = process.env.COMMANDCODE_API_KEY; + process.env.COMMANDCODE_API_KEY = "cc-fake-test-key-000"; + try { + await toolCall("bash", { command: "npm test" }); + assert.equal(networkCalls, 1); + assert.equal(requestUrls[0], "https://api.commandcode.ai/provider/v1/systemone"); + assert.equal(requests[0]!.model, "typesafe/jev"); + assert.equal(requestAuth[0], "Bearer cc-fake-test-key-000", "the key comes from COMMANDCODE_API_KEY"); + } finally { + if (saved === undefined) delete process.env.COMMANDCODE_API_KEY; else process.env.COMMANDCODE_API_KEY = saved; + } +}); + +test("a caller-supplied endpoint object reaches createTypeSafe unchanged and answers from its own host, model, and key", async () => { + await writeFile(configPath(), JSON.stringify({ typesafe: true, typesafeBackend: gateway, rules: { enabled: false }, ...STACK_BAR })); + process.env.ACME_JUDGE_KEY = "acme-fake-test-key-000"; + try { + await toolCall("bash", { command: "npm test" }); + assert.equal(networkCalls, 1); + assert.equal(requestUrls[0], "https://gw.acme.example/judge/v1/decide", "the object's host and path are used as written"); + assert.equal(requests[0]!.model, "jev-1.13", "the model goes out unmapped, as the object wrote it"); + assert.equal(requestAuth[0], "Bearer acme-fake-test-key-000", "the endpoint's own keyEnv carries the key"); + assert.notEqual(requestAuth[0], `Bearer ${process.env.TYPESAFE_API_KEY}`, "the TypeSafe key never goes to a caller-supplied endpoint"); + } finally { + delete process.env.ACME_JUDGE_KEY; + } +}); + +test("judgments off: an unknown typesafeBackend name is refused with the message, in the notice and in /warden status", async () => { + await writeFile(configPath(), JSON.stringify({ typesafe: true, typesafeBackend: "azure", rules: { enabled: false }, ...STACK_BAR })); + await toolCall("bash", { command: "npm test" }); + await toolCall("bash", { command: "npm run lint" }); + assert.deepEqual(judgmentsOff(), ["warden: Jev judgments are off (typesafeBackend refused: Unknown judgment backend \"azure\". Valid backends: typesafe, openrouter, commandcode.)"], "said once, carrying pi-typesafe's refusal message"); + assert.equal(networkCalls, 0, "no client is created and nothing is sent"); + assert.equal(requests.length, 0); + await runCommand("status"); + assert.match(notices.at(-1)!.text, /typesafeBackend refused: Unknown judgment backend "azure"/); +}); + +test("judgments off: an endpoint object pi-typesafe refuses is refused with the message and never creates a client", async () => { + await writeFile(configPath(), JSON.stringify({ typesafe: true, typesafeBackend: { label: "Acme judge gateway", host: "http://gw.acme.example", keyEnv: "ACME_JUDGE_KEY" }, rules: { enabled: false }, ...STACK_BAR })); + await toolCall("bash", { command: "npm test" }); + assert.equal(judgmentsOff().length, 1); + assert.match(judgmentsOff()[0]!, /^warden: Jev judgments are off \(typesafeBackend refused: Backend host must be an absolute https/); + assert.equal(networkCalls, 0, "no client is created and nothing is sent"); + assert.equal(requests.length, 0); +}); + +test("a custom backend's label, host, and model are named in /warden status, the /warden enable dialog, and the /warden test confirmation", async () => { + await writeFile(configPath(), JSON.stringify({ typesafe: true, typesafeBackend: gateway, rules: { enabled: false }, ...STACK_BAR })); + process.env.ACME_JUDGE_KEY = "acme-fake-test-key-000"; + try { + await runCommand("status"); + assert.match(notices.at(-1)!.text, /Acme judge gateway at gw\.acme\.example, model jev-1\.13/); + confirmResult = false; + await runCommand("enable"); + assert.match(confirms.at(-1)!.message, /Judgments go to Acme judge gateway at gw\.acme\.example, model jev-1\.13/); + assert.doesNotMatch(confirms.at(-1)!.message, /api\.typesafe\.ai/, "the disclosure names the real destination"); + await runCommand("test"); + assert.match(confirms.at(-1)!.message, /goes to Acme judge gateway at gw\.acme\.example, model jev-1\.13/); + assert.doesNotMatch(confirms.at(-1)!.message, /api\.typesafe\.ai/); + } finally { + delete process.env.ACME_JUDGE_KEY; + } +}); + +test("judgments off: no key on a custom endpoint names its own key variable and no login", async () => { + await writeFile(configPath(), JSON.stringify({ typesafe: true, typesafeBackend: gateway, rules: { enabled: false }, ...STACK_BAR })); + delete process.env.ACME_JUDGE_KEY; + await toolCall("bash", { command: "npm test" }); + assert.deepEqual(judgmentsOff(), ["warden: Jev judgments are off (no key for Acme judge gateway). Set ACME_JUDGE_KEY."]); + assert.equal(networkCalls, 0); +}); + test("judgments off: a rejected key saved by /typesafe login is named as that key, and the notice never shows it", async () => { await grantConsent(); const savedTestKey = process.env.TYPESAFE_API_KEY; From ac88f61856afd4b608f7b5c1d19cd1e7ac4ef221 Mon Sep 17 00:00:00 2001 From: Ryan Gapac Date: Mon, 28 Sep 2026 21:00:52 +0800 Subject: [PATCH 2/2] chore: version bump to 0.74.0, finalize CHANGELOG --- CHANGELOG.md | 4 ++++ package.json | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8081cb5..5103a72 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,10 @@ How to keep this current: add the entry in the same pull request as the change, ## Unreleased + + +## 0.74.0 + ### Added - `typesafeBackend` accepts `"commandcode"`: judgments go to api.commandcode.ai under `/provider/v1/systemone`, with the key from `COMMANDCODE_API_KEY` and the model `typesafe/jev`. diff --git a/package.json b/package.json index b1051b7..49ae808 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "pi-warden", - "version": "0.73.1", + "version": "0.74.0", "description": "Makes the Pi agent follow your project's rules. Jev judges every write against your pi-warden.md and quotes the broken rule back to the agent, names slop, breaks stuck loops, calls out unverified done claims, compresses large tool output, and holds the rare destructive command. Built on pi-typesafe.", "type": "module", "license": "MIT",