feat(contract): add code-grader stdin payload contract eval

examples/contract/evals/code-grader-contract.eval.yaml

@@ -0,0 +1,19 @@
+name: code-grader-contract
+description: Release gate verifying the code-grader stdin payload contract.
+
+execution:
+  target: github-models-contract
+  workers: 1
+
+tests:
+  - id: code-grader-stdin-payload
+    criteria: Response contains the word CONTRACT.
+    input:
+      - role: system
+        content: Reply with exactly the uppercase word CONTRACT and no other text.
+      - role: user
+        content: "Reply with the single word: CONTRACT"
+    assertions:
+      - name: code-grader-payload-contract
+        type: code-grader
+        command: ["bun", "../scripts/check-code-grader-payload.ts"]

examples/contract/scripts/check-code-grader-payload.ts

@@ -0,0 +1,143 @@
+#!/usr/bin/env bun
+/**
+ * Deterministic code grader for the code-grader stdin payload contract.
+ *
+ * It verifies that AgentV sends the canonical wire payload to code graders:
+ * final answer text in `output`, full transcript in `messages`, test context in
+ * `input`/`criteria`, and no deprecated `answer` alias.
+ */
+
+import { readFileSync } from 'node:fs';
+
+interface Assertion {
+  readonly text: string;
+  readonly passed: boolean;
+  readonly evidence?: string;
+}
+
+const assertions: Assertion[] = [];
+
+function push(text: string, passed: boolean, evidence?: string): void {
+  assertions.push({ text, passed, ...(evidence ? { evidence } : {}) });
+}
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return value !== null && typeof value === 'object' && !Array.isArray(value);
+}
+
+function hasOwn(record: Record<string, unknown>, key: string): boolean {
+  return Object.prototype.hasOwnProperty.call(record, key);
+}
+
+function collectStrings(value: unknown, strings: string[]): void {
+  if (typeof value === 'string') {
+    strings.push(value);
+    return;
+  }
+  if (Array.isArray(value)) {
+    for (const item of value) collectStrings(item, strings);
+    return;
+  }
+  if (isRecord(value)) {
+    for (const item of Object.values(value)) collectStrings(item, strings);
+  }
+}
+
+let payload: Record<string, unknown>;
+
+try {
+  const parsed = JSON.parse(readFileSync('/dev/stdin', 'utf8')) as unknown;
+  if (!isRecord(parsed)) {
+    push('stdin payload is a JSON object', false, `Received ${typeof parsed}`);
+    console.log(JSON.stringify({ assertions }));
+    process.exit(0);
+  }
+  payload = parsed;
+} catch (error) {
+  const message = error instanceof Error ? error.message : String(error);
+  push('stdin payload is valid JSON', false, message);
+  console.log(JSON.stringify({ assertions }));
+  process.exit(0);
+}
+
+const output = payload.output;
+const outputIsString = typeof output === 'string';
+push(
+  'output is the final answer string',
+  outputIsString,
+  outputIsString
+    ? undefined
+    : `Expected string, got ${Array.isArray(output) ? 'array' : typeof output}`,
+);
+
+const outputText = outputIsString ? output : '';
+push(
+  'output contains the agent answer',
+  outputText.trim().length > 0 && outputText.includes('CONTRACT'),
+  outputText.trim().length > 0
+    ? `output=${JSON.stringify(outputText.slice(0, 80))}`
+    : 'output was empty',
+);
+
+const messages = payload.messages;
+const messagesIsArray = Array.isArray(messages);
+push(
+  'messages is the full transcript array',
+  messagesIsArray,
+  messagesIsArray ? undefined : `Expected array, got ${typeof messages}`,
+);
+
+const hasAssistantMessage =
+  messagesIsArray && messages.some((message) => isRecord(message) && message.role === 'assistant');
+push(
+  'messages contains an assistant message',
+  hasAssistantMessage,
+  hasAssistantMessage ? undefined : 'No assistant role message found in messages',
+);
+
+const inputStrings: string[] = [];
+collectStrings(payload.input, inputStrings);
+const inputIncludesOriginalRequest = inputStrings.some((text) => text.includes('CONTRACT'));
+push(
+  'input contains the original user request',
+  inputIncludesOriginalRequest,
+  inputIncludesOriginalRequest ? undefined : 'CONTRACT marker not found in input',
+);
+
+const criteria = payload.criteria;
+push(
+  'criteria contains the test criteria',
+  typeof criteria === 'string' && criteria.includes('CONTRACT'),
+  typeof criteria === 'string' ? `criteria=${JSON.stringify(criteria)}` : 'criteria was missing',
+);
+
+const camelCaseKeys = [
+  'expectedOutput',
+  'inputFiles',
+  'outputPath',
+  'traceSummary',
+  'workspacePath',
+  'costUsd',
+  'durationMs',
+  'startTime',
+  'endTime',
+  'fileChanges',
+];
+const leakedCamelCaseKeys = camelCaseKeys.filter((key) => hasOwn(payload, key));
+push(
+  'grader payload uses snake_case wire keys',
+  Array.isArray(payload.input_files) && leakedCamelCaseKeys.length === 0,
+  leakedCamelCaseKeys.length > 0
+    ? `Unexpected camelCase keys: ${leakedCamelCaseKeys.join(', ')}`
+    : Array.isArray(payload.input_files)
+      ? undefined
+      : 'input_files was missing or not an array',
+);
+
+push(
+  'deprecated answer alias is absent',
+  !hasOwn(payload, 'answer'),
+  hasOwn(payload, 'answer') ? 'Unexpected answer field found in payload' : undefined,
+);
+
+console.log(JSON.stringify({ assertions }));

examples/features/prompt-template-sdk/README.md

@@ -25,7 +25,7 @@ function textFromMessages(messages) {
 
 export default definePromptTemplate((ctx) => `
   Question: ${textFromMessages(ctx.input.filter((message) => message.role === 'user'))}
-  Answer: ${ctx.output ?? ctx.answer ?? ''}
+  Answer: ${ctx.output ?? ''}
 
   ${ctx.expectedOutput.length > 0 ? `Reference: ${textFromMessages(ctx.expectedOutput)}` : ''}
 `);
@@ -37,7 +37,6 @@ The template receives evaluation context via stdin (JSON) and outputs the prompt
 
 - `input` - Input messages to agent
 - `output` - The agent's final answer being evaluated
-- `answer` - Deprecated alias for `output`
 - `expectedOutput` - Optional expected output messages
 - `criteria` - Optional criteria / expected outcome
 - `messages` - Transcript messages from the target execution

examples/features/prompt-template-sdk/prompts/custom-evaluator.ts

@@ -28,7 +28,7 @@ function getMessageText(
 
 export default definePromptTemplate((ctx) => {
   const inputText = getMessageText(ctx.input, 'user');
-  const outputText = ctx.output ?? ctx.answer ?? '';
+  const outputText = ctx.output ?? '';
   const expectedOutputText = getMessageText(ctx.expectedOutput);
 
   // Access typed config from YAML

packages/core/src/evaluation/graders/code-grader.ts

@@ -171,7 +171,6 @@ export class CodeGrader implements Grader {
         getImageDir,
       ),
       output: outputForPayload,
-      answer: context.candidate,
       messages: materializedMessages ?? [],
       outputPath,
       inputFiles: context.evalCase.file_paths,

packages/core/test/evaluation/code-grader-multimodal.test.ts

@@ -270,6 +270,7 @@ describe('CodeGrader multimodal integration', () => {
     const details = result.details as Record<string, unknown>;
     const payload = details.payload as Record<string, unknown>;
     expect(payload.output).toBe('answer');
+    expect(payload).not.toHaveProperty('answer');
     const messages = payload.messages as Record<string, unknown>[];
     expect(messages[0].content).toBe('Hello world');
   });

packages/core/test/evaluation/graders.test.ts

@@ -1010,8 +1010,8 @@ describe('CodeGrader', () => {
     expect(result.verdict).toBe('pass');
     const passedTexts = result.assertions.filter((a) => a.passed).map((a) => a.text);
     expect(passedTexts).toContain('expected_output present');
-    expect(passedTexts).toContain('answer present');
-    expect(passedTexts).toContain('answer parses');
+    expect(passedTexts).toContain('output present');
+    expect(passedTexts).toContain('output parses');
   });
 
   it('surfaces stderr and exit code on failure', async () => {

packages/core/test/evaluation/graders/prompt-resolution.test.ts

@@ -105,9 +105,6 @@ definePromptTemplate((ctx) => {
   if (ctx.output !== 'Final answer') {
     throw new Error('unexpected final answer: ' + ctx.output);
   }
-  if (ctx.answer !== ctx.output) {
-    throw new Error('answer should mirror output');
-  }
   if (!Array.isArray(ctx.messages) || ctx.messages.length < 2) {
     throw new Error('expected transcript messages');
   }

packages/core/test/fixtures/test-grader-with-details.cjs

@@ -7,16 +7,14 @@ const fs = require('node:fs');
 const input = JSON.parse(fs.readFileSync(0, 'utf8'));
 
 const hasExpected = Array.isArray(input.expected_output);
-// `output` is the final answer/scored result. Keep a tiny legacy fallback so
-// this fixture can still explain failures if an old message-array payload leaks.
+// `output` is the final answer/scored result. Keep a tiny fallback so this
+// fixture can still explain failures if an old message-array payload leaks.
 const candidateText =
   typeof input.output === 'string'
     ? input.output
-    : typeof input.answer === 'string'
-      ? input.answer
-      : Array.isArray(input.output)
-        ? input.output.map((m) => String(m.content ?? '')).join('')
-        : '';
+    : Array.isArray(input.output)
+      ? input.output.map((m) => String(m.content ?? '')).join('')
+      : '';
 const hasCandidate = candidateText.length > 0;
 
 // Emit details with structured metrics
@@ -25,7 +23,7 @@ console.log(
     score: hasExpected && hasCandidate ? 0.75 : 0,
     assertions: [
       ...(hasExpected ? [{ text: 'expected_output present', passed: true }] : []),
-      ...(hasCandidate ? [] : [{ text: 'answer missing', passed: false }]),
+      ...(hasCandidate ? [] : [{ text: 'output missing', passed: false }]),
     ],
     details: {
       metrics: {

packages/core/test/fixtures/test-grader.cjs

@@ -4,18 +4,16 @@ const fs = require('node:fs');
 const input = JSON.parse(fs.readFileSync(0, 'utf8'));
 
 const hasExpected = Array.isArray(input.expected_output);
-// `output` is the final answer/scored result. Keep a tiny legacy fallback so
-// this fixture can still explain failures if an old message-array payload leaks.
+// `output` is the final answer/scored result. Keep a tiny fallback so this
+// fixture can still explain failures if an old message-array payload leaks.
 const candidateText =
   typeof input.output === 'string'
     ? input.output
-    : typeof input.answer === 'string'
-      ? input.answer
-      : Array.isArray(input.output)
-        ? input.output
-            .map((m) => (typeof m.content === 'string' ? m.content : JSON.stringify(m.content)))
-            .join('')
-        : '';
+    : Array.isArray(input.output)
+      ? input.output
+          .map((m) => (typeof m.content === 'string' ? m.content : JSON.stringify(m.content)))
+          .join('')
+      : '';
 const hasCandidate = candidateText.length > 0;
 let candidateDecisionOk = false;
 
@@ -31,8 +29,8 @@ console.log(
     score: ok ? 1 : 0,
     assertions: [
       { text: 'expected_output present', passed: hasExpected },
-      { text: 'answer present', passed: hasCandidate },
-      { text: 'answer parses', passed: candidateDecisionOk },
+      { text: 'output present', passed: hasCandidate },
+      { text: 'output parses', passed: candidateDecisionOk },
     ].filter((a) => a.passed !== undefined),
   }),
 );

packages/eval/src/index.ts

@@ -220,7 +220,7 @@ export function defineCodeGrader(handler: CodeGraderHandler): void {
  *     .filter((message) => message.role === 'user')
  *     .map((message) => typeof message.content === 'string' ? message.content : '')
  *     .join('\n');
- *   const answer = ctx.output ?? ctx.answer ?? '';
+ *   const answer = ctx.output ?? '';
  *   return `Question: ${question}\nAnswer: ${answer}`;
  * });
  * ```

packages/eval/src/prompt-template.ts

@@ -75,7 +75,7 @@ export async function runPromptTemplate(handler: PromptTemplateHandler): Promise
  *     .filter((message) => message.role === 'user')
  *     .map((message) => typeof message.content === 'string' ? message.content : '')
  *     .join('\n');
- *   const answer = ctx.output ?? ctx.answer ?? '';
+ *   const answer = ctx.output ?? '';
  *   return `Question: ${question}\nAnswer: ${answer}`;
  * });
  * ```

packages/eval/src/schemas.ts

@@ -319,8 +319,6 @@ export const CodeGraderInputSchema = z.object({
   criteria: z.string(),
   expectedOutput: z.array(MessageSchema),
   output: z.string().nullable().optional(),
-  /** Deprecated migration alias; same value as output for text agents. */
-  answer: z.string().optional(),
   messages: z.array(MessageSchema).optional().default([]),
   /** Path to a temp file containing the output JSON (used for large payloads). */
   outputPath: z.string().optional(),

scripts/run-contract-eval.ts

@@ -15,6 +15,7 @@ process.env.CONTRACT_EVAL_MODEL ||= 'openai/gpt-4.1-mini';
 const evalFiles = [
   'examples/contract/evals/release-gate.eval.yaml',
   'examples/contract/evals/repo-materialization.eval.yaml',
+  'examples/contract/evals/code-grader-contract.eval.yaml',
 ];
 
 for (const evalFile of evalFiles) {