fix: describe tool schema fields and show one request payload

CHANGELOG.md

@@ -4,6 +4,12 @@
 
 <!-- Empty. Next release starts here. -->
 
+## 0.6.2
+
+### Fixed
+
+- Describe every field the agent authors in the `typesafe_evaluate` schema (`state`, `questions`, `type`, `instructions`, `criteria`, `model`) and show one request payload in the prompt guidelines, so the first call no longer has to fail to learn the shape (#6, #7).
+
 ## 0.6.1
 
 ### Fixed

package.json

@@ -1,6 +1,6 @@
 {
   "name": "pi-typesafe",
-  "version": "0.6.1",
+  "version": "0.6.2",
   "description": "TypeSafe AI (Jev) decisions for Pi: batched Choice/Score/Noul evaluation tool, terminal playground, and a typed API other extensions build on.",
   "type": "module",
   "license": "MIT",

src/extension.ts

@@ -16,7 +16,7 @@ const sample = {
   questions: {
     category: { type: "choice", instructions: "Which team should handle this message?", criteria: { billing: "Charges and payments", technical: "Software failures", other: "None of these" } },
     urgent: { type: "noul", instructions: "Does the sender request help today?" },
-    frustration: { type: "score", instructions: "How frustrated does the sender sound?", criteria: ["A neutral request without expressed frustration", "Expressed frustration while remaining civil", "Explicit anger or threats"] },
+    frustration: { type: "score", instructions: "How frustrated does the sender sound?", criteria: ["Neutral request", "Frustrated but civil", "Angry or threatening"] },
   },
 };
 
@@ -70,6 +70,9 @@ export default function typesafeExtension(pi: ExtensionAPI): void {
     description: `Evaluate supplied state with independent Choice, Score, and Noul questions in one TypeSafe request. Each question judges the whole state, so when several items are involved, put each item in a named state field (e.g. \`reports.r1\`) and ask one question per item per dimension (e.g. \`r1_owner\`, \`r2_owner\`), naming the field in the instructions; never aggregate several items into one question. ${disclosure} Requires operator opt-in via /typesafe enable or PI_TYPESAFE_ENABLED=1. Limit: 32 questions, ${DEFAULT_MAX_INPUT_BYTES / 1024} KiB JSON, ${DEFAULT_MAX_REQUESTS} attempts per session; no retries.`,
     promptSnippet: "Ask batched structured questions with TypeSafe (external service; operator opt-in required)",
     promptGuidelines: [
+      // The payload shape is what models get wrong on the first call; the same sample the playground edits is the cheapest way to show it.
+      // Every session pays for this line on every tool listing, so the sample stays short.
+      `Request shape, all three question kinds in one call: ${JSON.stringify(sample)}`,
       "Use typesafe_evaluate only for requested semantic judgments, not calculations or exact lookups; send only the relevant permitted data.",
       "Batch independent typesafe_evaluate questions over the same state; use code or explicit permission rules for actions, never confidence as authorization.",
       "When typesafe_evaluate judges several items, give each item a named state field and ask one question per item per dimension, naming the field in the instructions; one question over many items returns an unusable blend.",

src/schema.ts

@@ -9,43 +9,49 @@ export const DEFAULT_MAX_INPUT_BYTES = 65_536;
 /** Questions one request may ask. More than this needs `chunkEvaluationRequest`, which splits and fans out. */
 export const DEFAULT_MAX_QUESTIONS = 32;
 
-// The API accepts structured descriptions, not only strings.
-const entry = Type.Union([
+// The API accepts structured descriptions, not only strings. The schema is the only shape guidance the model gets
+// before its first call, so every field the agent authors says what it means.
+const entry = (description?: string) => Type.Union([
   Type.String(),
   Type.Null(),
   Type.Array(Type.Unknown()),
   Type.Record(Type.String(), Type.Unknown()),
-]);
-const instructions = Type.Optional(entry);
+], description === undefined ? {} : { description });
+const instructions = Type.Optional(entry("One judgment about the whole state, phrased as a question or a statement."));
 const question = Type.Union([
   Type.Object({
-    type: Type.Literal("noul"),
+    type: Type.Literal("noul", { description: "Yes or no: the probability the instructions hold." }),
     instructions,
     criteria: Type.Optional(Type.Union([
       Type.Null(),
-      Type.Object({ true: Type.Optional(entry), false: Type.Optional(entry) }, { additionalProperties: false }),
-    ])),
+      Type.Object({ true: Type.Optional(entry()), false: Type.Optional(entry()) }, { additionalProperties: false }),
+    ], { description: "Optional: what counts as yes and what counts as no, { true, false }." })),
   }, { additionalProperties: false }),
   Type.Object({
-    type: Type.Literal("choice"),
+    type: Type.Literal("choice", { description: "Pick one criteria label." }),
     instructions,
-    criteria: Type.Record(Type.String({ minLength: 1, maxLength: 200 }), entry, { minProperties: 1, maxProperties: 64 }),
+    criteria: Type.Record(Type.String({ minLength: 1, maxLength: 200 }), entry(), {
+      minProperties: 1,
+      maxProperties: 64,
+      description: "The options as a map from label to when it applies: { billing: \"Charges and payments\", other: null }. 1–64 entries.",
+    }),
   }, { additionalProperties: false }),
   Type.Object({
-    type: Type.Literal("score"),
+    type: Type.Literal("score", { description: "Rate against the ordered criteria levels." }),
     instructions,
-    criteria: Type.Array(entry, { minItems: 2, maxItems: 32 }),
+    criteria: Type.Array(entry(), { minItems: 2, maxItems: 32, description: "Ordered rubric levels, lowest first: [\"neutral\", \"angry\"]. 2–32 levels." }),
   }, { additionalProperties: false }),
 ]);
 
 /** The JSON schema used by both the Pi tool and the programmatic interface. */
 export const evaluationSchema = Type.Object({
-  state: entry,
+  state: entry("What to judge: text, or an object whose fields the questions name."),
   questions: Type.Record(Type.String({ minLength: 1, maxLength: 100 }), question, {
     minProperties: 1,
     maxProperties: DEFAULT_MAX_QUESTIONS,
+    description: "Questions keyed by a short id, as an object map, not an array: { \"urgent\": { type: \"noul\", instructions: ... } }.",
   }),
-  model: Type.Optional(Type.String({ minLength: 1, maxLength: 100 })),
+  model: Type.Optional(Type.String({ minLength: 1, maxLength: 100, description: "Jev model id, e.g. jev-latest. Omit for the default." })),
 }, { additionalProperties: false });
 
 const usage = `Expected { state, questions: { <id>: { type: "choice", instructions, criteria: { label: description|null } } | { type: "score", instructions, criteria: [level0, level1, ...] } | { type: "noul", instructions } } }; 1–${DEFAULT_MAX_QUESTIONS} questions, Choice 1–64 options, Score 2–32 levels.`;

tests/extension.test.ts

@@ -7,6 +7,7 @@ import { after, before, test } from "node:test";
 import { DefaultResourceLoader, SettingsManager } from "@earendil-works/pi-coding-agent";
 import type { Extension, RegisteredCommand, RegisteredTool } from "@earendil-works/pi-coding-agent";
 import { visibleWidth } from "@earendil-works/pi-tui";
+import { parseEvaluationRequest } from "../src/index.js";
 
 let temporary: string;
 let extension: Extension;
@@ -84,6 +85,13 @@ test("Pi loads a tool, a slash command, and a result renderer without network ca
   const completions = await command.getArgumentCompletions?.("pla");
   assert.ok(completions?.some(item => item.value === "playground"));
   assert.ok(tool.definition.promptGuidelines?.some(text => /one question per item per dimension/.test(text)));
+  // Models that never saw a payload author questions as an array; the guidelines must show one that actually validates.
+  const example = tool.definition.promptGuidelines?.find(text => /"state":/.test(text));
+  assert.ok(example, "one guideline shows a request payload");
+  assert.doesNotMatch(example, /\n/, "the example stays on one line");
+  assert.ok(example.length < 1024, "the example is paid for on every tool listing, so it stays short");
+  const payload = parseEvaluationRequest(JSON.parse(example.slice(example.indexOf("{"))));
+  assert.deepEqual(Object.values(payload.questions).map(question => question.type).sort(), ["choice", "noul", "score"]);
   assert.ok(/named state field/.test(tool.definition.description));
 });
 

tests/schema.test.ts

@@ -1,7 +1,7 @@
 import assert from "node:assert/strict";
 import { test } from "node:test";
 import {
-  DEFAULT_MAX_INPUT_BYTES, normalizeEvaluationRequest, parseEvaluationRequest, prepareEvaluationRequest, TypeSafeIntegrationError,
+  DEFAULT_MAX_INPUT_BYTES, evaluationSchema, normalizeEvaluationRequest, parseEvaluationRequest, prepareEvaluationRequest, TypeSafeIntegrationError,
 } from "../src/index.js";
 
 const hasCode = (code: string) => (error: unknown) => error instanceof TypeSafeIntegrationError && error.code === code;
@@ -60,3 +60,26 @@ test("admission still rejects non-JSON state", () => {
   assert.throws(() => prepareEvaluationRequest({ state: cycle, questions: { yes: { type: "noul", instructions: "?" } } }), hasCode("validation"));
   assert.throws(() => prepareEvaluationRequest({ state: { n: NaN }, questions: { yes: { type: "noul", instructions: "?" } } }), hasCode("validation"));
 });
+
+test("every field the agent authors carries a description, so a bare union is not its only guidance", () => {
+  // Walk the serialized schema rather than pin TypeBox's layout: collect every `properties` entry by name.
+  const authored = new Map<string, number>();
+  const undescribed: string[] = [];
+  const walk = (node: unknown): void => {
+    if (node === null || typeof node !== "object") return;
+    if (Array.isArray(node)) { node.forEach(walk); return; }
+    const record = node as Record<string, unknown>;
+    if (record.properties && typeof record.properties === "object") {
+      for (const [name, field] of Object.entries(record.properties as Record<string, Record<string, unknown>>)) {
+        authored.set(name, (authored.get(name) ?? 0) + 1);
+        if (typeof field.description !== "string" || field.description.length === 0) undescribed.push(name);
+      }
+    }
+    Object.values(record).forEach(walk);
+  };
+  walk(JSON.parse(JSON.stringify(evaluationSchema)));
+  assert.equal(authored.get("type"), 3, "the schema must still offer noul, choice, and score");
+  for (const name of ["state", "questions", "model", "instructions", "criteria"]) assert.ok(authored.has(name), `${name} is still authored`);
+  // `true` and `false` inside noul criteria are the only fields whose parent already explains them.
+  assert.deepEqual(undescribed.filter(name => name !== "true" && name !== "false"), [], "every authored field must say what it means");
+});