diff --git a/CHANGELOG.md b/CHANGELOG.md index eb59127..d9f6e1c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,12 @@ +## 0.6.2 + +### Fixed + +- Describe every field the agent authors in the `typesafe_evaluate` schema (`state`, `questions`, `type`, `instructions`, `criteria`, `model`) and show one request payload in the prompt guidelines, so the first call no longer has to fail to learn the shape (#6, #7). + ## 0.6.1 ### Fixed diff --git a/package-lock.json b/package-lock.json index 8ece339..33c1d1b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "pi-typesafe", - "version": "0.6.1", + "version": "0.6.2", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "pi-typesafe", - "version": "0.6.1", + "version": "0.6.2", "license": "MIT", "dependencies": { "@typesafe-ai/sdk": "^0.6.0", diff --git a/package.json b/package.json index 225be68..9292521 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "pi-typesafe", - "version": "0.6.1", + "version": "0.6.2", "description": "TypeSafe AI (Jev) decisions for Pi: batched Choice/Score/Noul evaluation tool, terminal playground, and a typed API other extensions build on.", "type": "module", "license": "MIT", diff --git a/src/extension.ts b/src/extension.ts index b3a8e50..b0f0a8e 100644 --- a/src/extension.ts +++ b/src/extension.ts @@ -16,7 +16,7 @@ const sample = { questions: { category: { type: "choice", instructions: "Which team should handle this message?", criteria: { billing: "Charges and payments", technical: "Software failures", other: "None of these" } }, urgent: { type: "noul", instructions: "Does the sender request help today?" }, - frustration: { type: "score", instructions: "How frustrated does the sender sound?", criteria: ["A neutral request without expressed frustration", "Expressed frustration while remaining civil", "Explicit anger or threats"] }, + frustration: { type: "score", instructions: "How frustrated does the sender sound?", criteria: ["Neutral request", "Frustrated but civil", "Angry or threatening"] }, }, }; @@ -70,6 +70,9 @@ export default function typesafeExtension(pi: ExtensionAPI): void { description: `Evaluate supplied state with independent Choice, Score, and Noul questions in one TypeSafe request. Each question judges the whole state, so when several items are involved, put each item in a named state field (e.g. \`reports.r1\`) and ask one question per item per dimension (e.g. \`r1_owner\`, \`r2_owner\`), naming the field in the instructions; never aggregate several items into one question. ${disclosure} Requires operator opt-in via /typesafe enable or PI_TYPESAFE_ENABLED=1. Limit: 32 questions, ${DEFAULT_MAX_INPUT_BYTES / 1024} KiB JSON, ${DEFAULT_MAX_REQUESTS} attempts per session; no retries.`, promptSnippet: "Ask batched structured questions with TypeSafe (external service; operator opt-in required)", promptGuidelines: [ + // The payload shape is what models get wrong on the first call; the same sample the playground edits is the cheapest way to show it. + // Every session pays for this line on every tool listing, so the sample stays short. + `Request shape, all three question kinds in one call: ${JSON.stringify(sample)}`, "Use typesafe_evaluate only for requested semantic judgments, not calculations or exact lookups; send only the relevant permitted data.", "Batch independent typesafe_evaluate questions over the same state; use code or explicit permission rules for actions, never confidence as authorization.", "When typesafe_evaluate judges several items, give each item a named state field and ask one question per item per dimension, naming the field in the instructions; one question over many items returns an unusable blend.", diff --git a/src/schema.ts b/src/schema.ts index 22bc81e..ba69703 100644 --- a/src/schema.ts +++ b/src/schema.ts @@ -9,43 +9,49 @@ export const DEFAULT_MAX_INPUT_BYTES = 65_536; /** Questions one request may ask. More than this needs `chunkEvaluationRequest`, which splits and fans out. */ export const DEFAULT_MAX_QUESTIONS = 32; -// The API accepts structured descriptions, not only strings. -const entry = Type.Union([ +// The API accepts structured descriptions, not only strings. The schema is the only shape guidance the model gets +// before its first call, so every field the agent authors says what it means. +const entry = (description?: string) => Type.Union([ Type.String(), Type.Null(), Type.Array(Type.Unknown()), Type.Record(Type.String(), Type.Unknown()), -]); -const instructions = Type.Optional(entry); +], description === undefined ? {} : { description }); +const instructions = Type.Optional(entry("One judgment about the whole state, phrased as a question or a statement.")); const question = Type.Union([ Type.Object({ - type: Type.Literal("noul"), + type: Type.Literal("noul", { description: "Yes or no: the probability the instructions hold." }), instructions, criteria: Type.Optional(Type.Union([ Type.Null(), - Type.Object({ true: Type.Optional(entry), false: Type.Optional(entry) }, { additionalProperties: false }), - ])), + Type.Object({ true: Type.Optional(entry()), false: Type.Optional(entry()) }, { additionalProperties: false }), + ], { description: "Optional: what counts as yes and what counts as no, { true, false }." })), }, { additionalProperties: false }), Type.Object({ - type: Type.Literal("choice"), + type: Type.Literal("choice", { description: "Pick one criteria label." }), instructions, - criteria: Type.Record(Type.String({ minLength: 1, maxLength: 200 }), entry, { minProperties: 1, maxProperties: 64 }), + criteria: Type.Record(Type.String({ minLength: 1, maxLength: 200 }), entry(), { + minProperties: 1, + maxProperties: 64, + description: "The options as a map from label to when it applies: { billing: \"Charges and payments\", other: null }. 1–64 entries.", + }), }, { additionalProperties: false }), Type.Object({ - type: Type.Literal("score"), + type: Type.Literal("score", { description: "Rate against the ordered criteria levels." }), instructions, - criteria: Type.Array(entry, { minItems: 2, maxItems: 32 }), + criteria: Type.Array(entry(), { minItems: 2, maxItems: 32, description: "Ordered rubric levels, lowest first: [\"neutral\", \"angry\"]. 2–32 levels." }), }, { additionalProperties: false }), ]); /** The JSON schema used by both the Pi tool and the programmatic interface. */ export const evaluationSchema = Type.Object({ - state: entry, + state: entry("What to judge: text, or an object whose fields the questions name."), questions: Type.Record(Type.String({ minLength: 1, maxLength: 100 }), question, { minProperties: 1, maxProperties: DEFAULT_MAX_QUESTIONS, + description: "Questions keyed by a short id, as an object map, not an array: { \"urgent\": { type: \"noul\", instructions: ... } }.", }), - model: Type.Optional(Type.String({ minLength: 1, maxLength: 100 })), + model: Type.Optional(Type.String({ minLength: 1, maxLength: 100, description: "Jev model id, e.g. jev-latest. Omit for the default." })), }, { additionalProperties: false }); const usage = `Expected { state, questions: { : { type: "choice", instructions, criteria: { label: description|null } } | { type: "score", instructions, criteria: [level0, level1, ...] } | { type: "noul", instructions } } }; 1–${DEFAULT_MAX_QUESTIONS} questions, Choice 1–64 options, Score 2–32 levels.`; diff --git a/tests/extension.test.ts b/tests/extension.test.ts index 516b92a..a233642 100644 --- a/tests/extension.test.ts +++ b/tests/extension.test.ts @@ -7,6 +7,7 @@ import { after, before, test } from "node:test"; import { DefaultResourceLoader, SettingsManager } from "@earendil-works/pi-coding-agent"; import type { Extension, RegisteredCommand, RegisteredTool } from "@earendil-works/pi-coding-agent"; import { visibleWidth } from "@earendil-works/pi-tui"; +import { parseEvaluationRequest } from "../src/index.js"; let temporary: string; let extension: Extension; @@ -84,6 +85,13 @@ test("Pi loads a tool, a slash command, and a result renderer without network ca const completions = await command.getArgumentCompletions?.("pla"); assert.ok(completions?.some(item => item.value === "playground")); assert.ok(tool.definition.promptGuidelines?.some(text => /one question per item per dimension/.test(text))); + // Models that never saw a payload author questions as an array; the guidelines must show one that actually validates. + const example = tool.definition.promptGuidelines?.find(text => /"state":/.test(text)); + assert.ok(example, "one guideline shows a request payload"); + assert.doesNotMatch(example, /\n/, "the example stays on one line"); + assert.ok(example.length < 1024, "the example is paid for on every tool listing, so it stays short"); + const payload = parseEvaluationRequest(JSON.parse(example.slice(example.indexOf("{")))); + assert.deepEqual(Object.values(payload.questions).map(question => question.type).sort(), ["choice", "noul", "score"]); assert.ok(/named state field/.test(tool.definition.description)); }); diff --git a/tests/schema.test.ts b/tests/schema.test.ts index 0629050..fa64c26 100644 --- a/tests/schema.test.ts +++ b/tests/schema.test.ts @@ -1,7 +1,7 @@ import assert from "node:assert/strict"; import { test } from "node:test"; import { - DEFAULT_MAX_INPUT_BYTES, normalizeEvaluationRequest, parseEvaluationRequest, prepareEvaluationRequest, TypeSafeIntegrationError, + DEFAULT_MAX_INPUT_BYTES, evaluationSchema, normalizeEvaluationRequest, parseEvaluationRequest, prepareEvaluationRequest, TypeSafeIntegrationError, } from "../src/index.js"; const hasCode = (code: string) => (error: unknown) => error instanceof TypeSafeIntegrationError && error.code === code; @@ -60,3 +60,26 @@ test("admission still rejects non-JSON state", () => { assert.throws(() => prepareEvaluationRequest({ state: cycle, questions: { yes: { type: "noul", instructions: "?" } } }), hasCode("validation")); assert.throws(() => prepareEvaluationRequest({ state: { n: NaN }, questions: { yes: { type: "noul", instructions: "?" } } }), hasCode("validation")); }); + +test("every field the agent authors carries a description, so a bare union is not its only guidance", () => { + // Walk the serialized schema rather than pin TypeBox's layout: collect every `properties` entry by name. + const authored = new Map(); + const undescribed: string[] = []; + const walk = (node: unknown): void => { + if (node === null || typeof node !== "object") return; + if (Array.isArray(node)) { node.forEach(walk); return; } + const record = node as Record; + if (record.properties && typeof record.properties === "object") { + for (const [name, field] of Object.entries(record.properties as Record>)) { + authored.set(name, (authored.get(name) ?? 0) + 1); + if (typeof field.description !== "string" || field.description.length === 0) undescribed.push(name); + } + } + Object.values(record).forEach(walk); + }; + walk(JSON.parse(JSON.stringify(evaluationSchema))); + assert.equal(authored.get("type"), 3, "the schema must still offer noul, choice, and score"); + for (const name of ["state", "questions", "model", "instructions", "criteria"]) assert.ok(authored.has(name), `${name} is still authored`); + // `true` and `false` inside noul criteria are the only fields whose parent already explains them. + assert.deepEqual(undescribed.filter(name => name !== "true" && name !== "false"), [], "every authored field must say what it means"); +});