From 896ca157bb9767d642f9e49ef217758fcfd74148 Mon Sep 17 00:00:00 2001 From: yqf Date: Tue, 22 Sep 2026 11:42:28 +0800 Subject: [PATCH 1/2] fix: describe tool schema fields and show one request payload MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Models read `parameters` and `promptGuidelines` before their first call, and neither said what the payload must look like: the emitted schema carried no `description` on `state`, `questions`, `type`, `instructions`, or `criteria`, and the canonical sample only reached `/typesafe test` and the playground. The syntax guide lived in the validation error, so a first-call failure was the only way to learn the shape — `questions` authored as an array failed every time. Descriptions on the fields the agent authors, plus the existing sample inlined as a single-line guideline. Accepted input is unchanged. Refs #6 --- src/extension.ts | 2 ++ src/schema.ts | 32 +++++++++++++++++++------------- tests/extension.test.ts | 2 ++ tests/schema.test.ts | 23 ++++++++++++++++++++++- 4 files changed, 45 insertions(+), 14 deletions(-) diff --git a/src/extension.ts b/src/extension.ts index b3a8e50..2fafac9 100644 --- a/src/extension.ts +++ b/src/extension.ts @@ -70,6 +70,8 @@ export default function typesafeExtension(pi: ExtensionAPI): void { description: `Evaluate supplied state with independent Choice, Score, and Noul questions in one TypeSafe request. Each question judges the whole state, so when several items are involved, put each item in a named state field (e.g. \`reports.r1\`) and ask one question per item per dimension (e.g. \`r1_owner\`, \`r2_owner\`), naming the field in the instructions; never aggregate several items into one question. ${disclosure} Requires operator opt-in via /typesafe enable or PI_TYPESAFE_ENABLED=1. Limit: 32 questions, ${DEFAULT_MAX_INPUT_BYTES / 1024} KiB JSON, ${DEFAULT_MAX_REQUESTS} attempts per session; no retries.`, promptSnippet: "Ask batched structured questions with TypeSafe (external service; operator opt-in required)", promptGuidelines: [ + // The payload shape is what models get wrong on the first call; the same sample the playground edits is the cheapest way to show it. + `Request shape, all three question kinds in one call: ${JSON.stringify(sample)}`, "Use typesafe_evaluate only for requested semantic judgments, not calculations or exact lookups; send only the relevant permitted data.", "Batch independent typesafe_evaluate questions over the same state; use code or explicit permission rules for actions, never confidence as authorization.", "When typesafe_evaluate judges several items, give each item a named state field and ask one question per item per dimension, naming the field in the instructions; one question over many items returns an unusable blend.", diff --git a/src/schema.ts b/src/schema.ts index 22bc81e..aa4c3e7 100644 --- a/src/schema.ts +++ b/src/schema.ts @@ -9,43 +9,49 @@ export const DEFAULT_MAX_INPUT_BYTES = 65_536; /** Questions one request may ask. More than this needs `chunkEvaluationRequest`, which splits and fans out. */ export const DEFAULT_MAX_QUESTIONS = 32; -// The API accepts structured descriptions, not only strings. -const entry = Type.Union([ +// The API accepts structured descriptions, not only strings. The schema is the only shape guidance the model gets +// before its first call, so every field the agent authors says what it means. +const entry = (options: { description?: string } = {}) => Type.Union([ Type.String(), Type.Null(), Type.Array(Type.Unknown()), Type.Record(Type.String(), Type.Unknown()), -]); -const instructions = Type.Optional(entry); +], options); +const instructions = Type.Optional(entry({ description: "One judgment about the whole state, phrased as a question or a statement." })); const question = Type.Union([ Type.Object({ - type: Type.Literal("noul"), + type: Type.Literal("noul", { description: "Yes or no: the probability the instructions hold." }), instructions, criteria: Type.Optional(Type.Union([ Type.Null(), - Type.Object({ true: Type.Optional(entry), false: Type.Optional(entry) }, { additionalProperties: false }), - ])), + Type.Object({ true: Type.Optional(entry()), false: Type.Optional(entry()) }, { additionalProperties: false }), + ], { description: "Optional: what counts as yes and what counts as no, { true, false }." })), }, { additionalProperties: false }), Type.Object({ - type: Type.Literal("choice"), + type: Type.Literal("choice", { description: "Pick one criteria label." }), instructions, - criteria: Type.Record(Type.String({ minLength: 1, maxLength: 200 }), entry, { minProperties: 1, maxProperties: 64 }), + criteria: Type.Record(Type.String({ minLength: 1, maxLength: 200 }), entry(), { + minProperties: 1, + maxProperties: 64, + description: "The options as a map from label to when it applies: { billing: \"Charges and payments\", other: null }. 1–64 entries.", + }), }, { additionalProperties: false }), Type.Object({ - type: Type.Literal("score"), + type: Type.Literal("score", { description: "Rate against the ordered criteria levels." }), instructions, - criteria: Type.Array(entry, { minItems: 2, maxItems: 32 }), + criteria: Type.Array(entry(), { minItems: 2, maxItems: 32, description: "Ordered rubric levels, lowest first: [\"neutral\", \"angry\"]. 2–32 levels." }), }, { additionalProperties: false }), ]); /** The JSON schema used by both the Pi tool and the programmatic interface. */ export const evaluationSchema = Type.Object({ - state: entry, + state: entry({ description: "What to judge: text, or an object whose fields the questions name." }), questions: Type.Record(Type.String({ minLength: 1, maxLength: 100 }), question, { minProperties: 1, maxProperties: DEFAULT_MAX_QUESTIONS, + description: "Questions keyed by a short id — an object map, not an array: { \"urgent\": { type: \"noul\", instructions: ... } }.", }), - model: Type.Optional(Type.String({ minLength: 1, maxLength: 100 })), + model: Type.Optional(Type.String({ minLength: 1, maxLength: 100, description: "Jev model id, e.g. jev-latest. Omit for the default." })), }, { additionalProperties: false }); const usage = `Expected { state, questions: { : { type: "choice", instructions, criteria: { label: description|null } } | { type: "score", instructions, criteria: [level0, level1, ...] } | { type: "noul", instructions } } }; 1–${DEFAULT_MAX_QUESTIONS} questions, Choice 1–64 options, Score 2–32 levels.`; diff --git a/tests/extension.test.ts b/tests/extension.test.ts index 516b92a..d429c11 100644 --- a/tests/extension.test.ts +++ b/tests/extension.test.ts @@ -84,6 +84,8 @@ test("Pi loads a tool, a slash command, and a result renderer without network ca const completions = await command.getArgumentCompletions?.("pla"); assert.ok(completions?.some(item => item.value === "playground")); assert.ok(tool.definition.promptGuidelines?.some(text => /one question per item per dimension/.test(text))); + // Models that never saw a payload author questions as an array; the guidelines must show one. + assert.ok(tool.definition.promptGuidelines?.some(text => /"state":/.test(text) && !/\n/.test(text))); assert.ok(/named state field/.test(tool.definition.description)); }); diff --git a/tests/schema.test.ts b/tests/schema.test.ts index 0629050..d21947d 100644 --- a/tests/schema.test.ts +++ b/tests/schema.test.ts @@ -1,7 +1,7 @@ import assert from "node:assert/strict"; import { test } from "node:test"; import { - DEFAULT_MAX_INPUT_BYTES, normalizeEvaluationRequest, parseEvaluationRequest, prepareEvaluationRequest, TypeSafeIntegrationError, + DEFAULT_MAX_INPUT_BYTES, evaluationSchema, normalizeEvaluationRequest, parseEvaluationRequest, prepareEvaluationRequest, TypeSafeIntegrationError, } from "../src/index.js"; const hasCode = (code: string) => (error: unknown) => error instanceof TypeSafeIntegrationError && error.code === code; @@ -60,3 +60,24 @@ test("admission still rejects non-JSON state", () => { assert.throws(() => prepareEvaluationRequest({ state: cycle, questions: { yes: { type: "noul", instructions: "?" } } }), hasCode("validation")); assert.throws(() => prepareEvaluationRequest({ state: { n: NaN }, questions: { yes: { type: "noul", instructions: "?" } } }), hasCode("validation")); }); + +type Field = { description?: string }; +type Variant = { properties: { type: Field; instructions: Field; criteria: Field } }; + +test("every field the agent authors carries a description, so a bare union is not its only guidance", () => { + const schema = evaluationSchema as unknown as { + properties: { + state: Field; + model: Field; + questions: Field & { patternProperties: { "^.*$": { anyOf: Variant[] } } }; + }; + }; + const { state, model, questions } = schema.properties; + const variants = questions.patternProperties["^.*$"].anyOf; + assert.equal(variants.length, 3, "the schema must still offer noul, choice, and score"); + const authored: Field[] = [ + state, model, questions, + ...variants.flatMap(variant => [variant.properties.type, variant.properties.instructions, variant.properties.criteria]), + ]; + for (const field of authored) assert.ok(field.description, "every authored field must say what it means"); +}); From 1cdcd7ee8535e28d05f5ff5a8d1887411d2d6f88 Mon Sep 17 00:00:00 2001 From: Ryan Gapac Date: Tue, 22 Sep 2026 13:48:50 +0800 Subject: [PATCH 2/2] Prove the example payload validates, shorten it, and release 0.6.2 The guideline test only checked that some line contained `"state":`; it now parses the payload out of the guideline and runs it through parseEvaluationRequest, so the example the model copies is one the schema accepts. The sample's score rubric said in sentences what three words say, and every session pays for that line on every tool listing, so it is short now and the test caps its length. The schema test walks the serialized schema instead of pinning TypeBox's patternProperties/anyOf layout. `entry` takes the description directly, and the questions description drops the em-dash the rest of the file avoids. Version 0.6.2 with a changelog line. Refs #6, #7. --- CHANGELOG.md | 6 ++++++ package-lock.json | 4 ++-- package.json | 2 +- src/extension.ts | 3 ++- src/schema.ts | 10 +++++----- tests/extension.test.ts | 10 ++++++++-- tests/schema.test.ts | 36 +++++++++++++++++++----------------- 7 files changed, 43 insertions(+), 28 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index eb59127..d9f6e1c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,12 @@ +## 0.6.2 + +### Fixed + +- Describe every field the agent authors in the `typesafe_evaluate` schema (`state`, `questions`, `type`, `instructions`, `criteria`, `model`) and show one request payload in the prompt guidelines, so the first call no longer has to fail to learn the shape (#6, #7). + ## 0.6.1 ### Fixed diff --git a/package-lock.json b/package-lock.json index 8ece339..33c1d1b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "pi-typesafe", - "version": "0.6.1", + "version": "0.6.2", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "pi-typesafe", - "version": "0.6.1", + "version": "0.6.2", "license": "MIT", "dependencies": { "@typesafe-ai/sdk": "^0.6.0", diff --git a/package.json b/package.json index 225be68..9292521 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "pi-typesafe", - "version": "0.6.1", + "version": "0.6.2", "description": "TypeSafe AI (Jev) decisions for Pi: batched Choice/Score/Noul evaluation tool, terminal playground, and a typed API other extensions build on.", "type": "module", "license": "MIT", diff --git a/src/extension.ts b/src/extension.ts index 2fafac9..b0f0a8e 100644 --- a/src/extension.ts +++ b/src/extension.ts @@ -16,7 +16,7 @@ const sample = { questions: { category: { type: "choice", instructions: "Which team should handle this message?", criteria: { billing: "Charges and payments", technical: "Software failures", other: "None of these" } }, urgent: { type: "noul", instructions: "Does the sender request help today?" }, - frustration: { type: "score", instructions: "How frustrated does the sender sound?", criteria: ["A neutral request without expressed frustration", "Expressed frustration while remaining civil", "Explicit anger or threats"] }, + frustration: { type: "score", instructions: "How frustrated does the sender sound?", criteria: ["Neutral request", "Frustrated but civil", "Angry or threatening"] }, }, }; @@ -71,6 +71,7 @@ export default function typesafeExtension(pi: ExtensionAPI): void { promptSnippet: "Ask batched structured questions with TypeSafe (external service; operator opt-in required)", promptGuidelines: [ // The payload shape is what models get wrong on the first call; the same sample the playground edits is the cheapest way to show it. + // Every session pays for this line on every tool listing, so the sample stays short. `Request shape, all three question kinds in one call: ${JSON.stringify(sample)}`, "Use typesafe_evaluate only for requested semantic judgments, not calculations or exact lookups; send only the relevant permitted data.", "Batch independent typesafe_evaluate questions over the same state; use code or explicit permission rules for actions, never confidence as authorization.", diff --git a/src/schema.ts b/src/schema.ts index aa4c3e7..ba69703 100644 --- a/src/schema.ts +++ b/src/schema.ts @@ -11,13 +11,13 @@ export const DEFAULT_MAX_QUESTIONS = 32; // The API accepts structured descriptions, not only strings. The schema is the only shape guidance the model gets // before its first call, so every field the agent authors says what it means. -const entry = (options: { description?: string } = {}) => Type.Union([ +const entry = (description?: string) => Type.Union([ Type.String(), Type.Null(), Type.Array(Type.Unknown()), Type.Record(Type.String(), Type.Unknown()), -], options); -const instructions = Type.Optional(entry({ description: "One judgment about the whole state, phrased as a question or a statement." })); +], description === undefined ? {} : { description }); +const instructions = Type.Optional(entry("One judgment about the whole state, phrased as a question or a statement.")); const question = Type.Union([ Type.Object({ type: Type.Literal("noul", { description: "Yes or no: the probability the instructions hold." }), @@ -45,11 +45,11 @@ const question = Type.Union([ /** The JSON schema used by both the Pi tool and the programmatic interface. */ export const evaluationSchema = Type.Object({ - state: entry({ description: "What to judge: text, or an object whose fields the questions name." }), + state: entry("What to judge: text, or an object whose fields the questions name."), questions: Type.Record(Type.String({ minLength: 1, maxLength: 100 }), question, { minProperties: 1, maxProperties: DEFAULT_MAX_QUESTIONS, - description: "Questions keyed by a short id — an object map, not an array: { \"urgent\": { type: \"noul\", instructions: ... } }.", + description: "Questions keyed by a short id, as an object map, not an array: { \"urgent\": { type: \"noul\", instructions: ... } }.", }), model: Type.Optional(Type.String({ minLength: 1, maxLength: 100, description: "Jev model id, e.g. jev-latest. Omit for the default." })), }, { additionalProperties: false }); diff --git a/tests/extension.test.ts b/tests/extension.test.ts index d429c11..a233642 100644 --- a/tests/extension.test.ts +++ b/tests/extension.test.ts @@ -7,6 +7,7 @@ import { after, before, test } from "node:test"; import { DefaultResourceLoader, SettingsManager } from "@earendil-works/pi-coding-agent"; import type { Extension, RegisteredCommand, RegisteredTool } from "@earendil-works/pi-coding-agent"; import { visibleWidth } from "@earendil-works/pi-tui"; +import { parseEvaluationRequest } from "../src/index.js"; let temporary: string; let extension: Extension; @@ -84,8 +85,13 @@ test("Pi loads a tool, a slash command, and a result renderer without network ca const completions = await command.getArgumentCompletions?.("pla"); assert.ok(completions?.some(item => item.value === "playground")); assert.ok(tool.definition.promptGuidelines?.some(text => /one question per item per dimension/.test(text))); - // Models that never saw a payload author questions as an array; the guidelines must show one. - assert.ok(tool.definition.promptGuidelines?.some(text => /"state":/.test(text) && !/\n/.test(text))); + // Models that never saw a payload author questions as an array; the guidelines must show one that actually validates. + const example = tool.definition.promptGuidelines?.find(text => /"state":/.test(text)); + assert.ok(example, "one guideline shows a request payload"); + assert.doesNotMatch(example, /\n/, "the example stays on one line"); + assert.ok(example.length < 1024, "the example is paid for on every tool listing, so it stays short"); + const payload = parseEvaluationRequest(JSON.parse(example.slice(example.indexOf("{")))); + assert.deepEqual(Object.values(payload.questions).map(question => question.type).sort(), ["choice", "noul", "score"]); assert.ok(/named state field/.test(tool.definition.description)); }); diff --git a/tests/schema.test.ts b/tests/schema.test.ts index d21947d..fa64c26 100644 --- a/tests/schema.test.ts +++ b/tests/schema.test.ts @@ -61,23 +61,25 @@ test("admission still rejects non-JSON state", () => { assert.throws(() => prepareEvaluationRequest({ state: { n: NaN }, questions: { yes: { type: "noul", instructions: "?" } } }), hasCode("validation")); }); -type Field = { description?: string }; -type Variant = { properties: { type: Field; instructions: Field; criteria: Field } }; - test("every field the agent authors carries a description, so a bare union is not its only guidance", () => { - const schema = evaluationSchema as unknown as { - properties: { - state: Field; - model: Field; - questions: Field & { patternProperties: { "^.*$": { anyOf: Variant[] } } }; - }; + // Walk the serialized schema rather than pin TypeBox's layout: collect every `properties` entry by name. + const authored = new Map(); + const undescribed: string[] = []; + const walk = (node: unknown): void => { + if (node === null || typeof node !== "object") return; + if (Array.isArray(node)) { node.forEach(walk); return; } + const record = node as Record; + if (record.properties && typeof record.properties === "object") { + for (const [name, field] of Object.entries(record.properties as Record>)) { + authored.set(name, (authored.get(name) ?? 0) + 1); + if (typeof field.description !== "string" || field.description.length === 0) undescribed.push(name); + } + } + Object.values(record).forEach(walk); }; - const { state, model, questions } = schema.properties; - const variants = questions.patternProperties["^.*$"].anyOf; - assert.equal(variants.length, 3, "the schema must still offer noul, choice, and score"); - const authored: Field[] = [ - state, model, questions, - ...variants.flatMap(variant => [variant.properties.type, variant.properties.instructions, variant.properties.criteria]), - ]; - for (const field of authored) assert.ok(field.description, "every authored field must say what it means"); + walk(JSON.parse(JSON.stringify(evaluationSchema))); + assert.equal(authored.get("type"), 3, "the schema must still offer noul, choice, and score"); + for (const name of ["state", "questions", "model", "instructions", "criteria"]) assert.ok(authored.has(name), `${name} is still authored`); + // `true` and `false` inside noul criteria are the only fields whose parent already explains them. + assert.deepEqual(undescribed.filter(name => name !== "true" && name !== "false"), [], "every authored field must say what it means"); });