From e94df5441ef62e8036f97ca5acd1b3dbf6c9b6cf Mon Sep 17 00:00:00 2001 From: Christopher Date: Thu, 18 Jun 2026 11:12:27 +1000 Subject: [PATCH] feat(contract): add code-grader stdin payload contract eval --- .../evals/code-grader-contract.eval.yaml | 19 +++ .../scripts/check-code-grader-payload.ts | 143 ++++++++++++++++++ .../features/prompt-template-sdk/README.md | 3 +- .../prompts/custom-evaluator.ts | 2 +- .../src/evaluation/graders/code-grader.ts | 1 - .../evaluation/code-grader-multimodal.test.ts | 1 + packages/core/test/evaluation/graders.test.ts | 4 +- .../graders/prompt-resolution.test.ts | 3 - .../fixtures/test-grader-with-details.cjs | 14 +- packages/core/test/fixtures/test-grader.cjs | 20 ++- packages/eval/src/index.ts | 2 +- packages/eval/src/prompt-template.ts | 2 +- packages/eval/src/schemas.ts | 2 - scripts/run-contract-eval.ts | 1 + 14 files changed, 185 insertions(+), 32 deletions(-) create mode 100644 examples/contract/evals/code-grader-contract.eval.yaml create mode 100644 examples/contract/scripts/check-code-grader-payload.ts diff --git a/examples/contract/evals/code-grader-contract.eval.yaml b/examples/contract/evals/code-grader-contract.eval.yaml new file mode 100644 index 000000000..8745da373 --- /dev/null +++ b/examples/contract/evals/code-grader-contract.eval.yaml @@ -0,0 +1,19 @@ +name: code-grader-contract +description: Release gate verifying the code-grader stdin payload contract. + +execution: + target: github-models-contract + workers: 1 + +tests: + - id: code-grader-stdin-payload + criteria: Response contains the word CONTRACT. + input: + - role: system + content: Reply with exactly the uppercase word CONTRACT and no other text. + - role: user + content: "Reply with the single word: CONTRACT" + assertions: + - name: code-grader-payload-contract + type: code-grader + command: ["bun", "../scripts/check-code-grader-payload.ts"] diff --git a/examples/contract/scripts/check-code-grader-payload.ts b/examples/contract/scripts/check-code-grader-payload.ts new file mode 100644 index 000000000..8e47bbdac --- /dev/null +++ b/examples/contract/scripts/check-code-grader-payload.ts @@ -0,0 +1,143 @@ +#!/usr/bin/env bun +/** + * Deterministic code grader for the code-grader stdin payload contract. + * + * It verifies that AgentV sends the canonical wire payload to code graders: + * final answer text in `output`, full transcript in `messages`, test context in + * `input`/`criteria`, and no deprecated `answer` alias. + */ + +import { readFileSync } from 'node:fs'; + +interface Assertion { + readonly text: string; + readonly passed: boolean; + readonly evidence?: string; +} + +const assertions: Assertion[] = []; + +function push(text: string, passed: boolean, evidence?: string): void { + assertions.push({ text, passed, ...(evidence ? { evidence } : {}) }); +} + +function isRecord(value: unknown): value is Record { + return value !== null && typeof value === 'object' && !Array.isArray(value); +} + +function hasOwn(record: Record, key: string): boolean { + return Object.prototype.hasOwnProperty.call(record, key); +} + +function collectStrings(value: unknown, strings: string[]): void { + if (typeof value === 'string') { + strings.push(value); + return; + } + if (Array.isArray(value)) { + for (const item of value) collectStrings(item, strings); + return; + } + if (isRecord(value)) { + for (const item of Object.values(value)) collectStrings(item, strings); + } +} + +let payload: Record; + +try { + const parsed = JSON.parse(readFileSync('/dev/stdin', 'utf8')) as unknown; + if (!isRecord(parsed)) { + push('stdin payload is a JSON object', false, `Received ${typeof parsed}`); + console.log(JSON.stringify({ assertions })); + process.exit(0); + } + payload = parsed; +} catch (error) { + const message = error instanceof Error ? error.message : String(error); + push('stdin payload is valid JSON', false, message); + console.log(JSON.stringify({ assertions })); + process.exit(0); +} + +const output = payload.output; +const outputIsString = typeof output === 'string'; +push( + 'output is the final answer string', + outputIsString, + outputIsString + ? undefined + : `Expected string, got ${Array.isArray(output) ? 'array' : typeof output}`, +); + +const outputText = outputIsString ? output : ''; +push( + 'output contains the agent answer', + outputText.trim().length > 0 && outputText.includes('CONTRACT'), + outputText.trim().length > 0 + ? `output=${JSON.stringify(outputText.slice(0, 80))}` + : 'output was empty', +); + +const messages = payload.messages; +const messagesIsArray = Array.isArray(messages); +push( + 'messages is the full transcript array', + messagesIsArray, + messagesIsArray ? undefined : `Expected array, got ${typeof messages}`, +); + +const hasAssistantMessage = + messagesIsArray && messages.some((message) => isRecord(message) && message.role === 'assistant'); +push( + 'messages contains an assistant message', + hasAssistantMessage, + hasAssistantMessage ? undefined : 'No assistant role message found in messages', +); + +const inputStrings: string[] = []; +collectStrings(payload.input, inputStrings); +const inputIncludesOriginalRequest = inputStrings.some((text) => text.includes('CONTRACT')); +push( + 'input contains the original user request', + inputIncludesOriginalRequest, + inputIncludesOriginalRequest ? undefined : 'CONTRACT marker not found in input', +); + +const criteria = payload.criteria; +push( + 'criteria contains the test criteria', + typeof criteria === 'string' && criteria.includes('CONTRACT'), + typeof criteria === 'string' ? `criteria=${JSON.stringify(criteria)}` : 'criteria was missing', +); + +const camelCaseKeys = [ + 'expectedOutput', + 'inputFiles', + 'outputPath', + 'traceSummary', + 'workspacePath', + 'costUsd', + 'durationMs', + 'startTime', + 'endTime', + 'fileChanges', +]; +const leakedCamelCaseKeys = camelCaseKeys.filter((key) => hasOwn(payload, key)); +push( + 'grader payload uses snake_case wire keys', + Array.isArray(payload.input_files) && leakedCamelCaseKeys.length === 0, + leakedCamelCaseKeys.length > 0 + ? `Unexpected camelCase keys: ${leakedCamelCaseKeys.join(', ')}` + : Array.isArray(payload.input_files) + ? undefined + : 'input_files was missing or not an array', +); + +push( + 'deprecated answer alias is absent', + !hasOwn(payload, 'answer'), + hasOwn(payload, 'answer') ? 'Unexpected answer field found in payload' : undefined, +); + +console.log(JSON.stringify({ assertions })); diff --git a/examples/features/prompt-template-sdk/README.md b/examples/features/prompt-template-sdk/README.md index 5c221d7c4..ad026be34 100644 --- a/examples/features/prompt-template-sdk/README.md +++ b/examples/features/prompt-template-sdk/README.md @@ -25,7 +25,7 @@ function textFromMessages(messages) { export default definePromptTemplate((ctx) => ` Question: ${textFromMessages(ctx.input.filter((message) => message.role === 'user'))} - Answer: ${ctx.output ?? ctx.answer ?? ''} + Answer: ${ctx.output ?? ''} ${ctx.expectedOutput.length > 0 ? `Reference: ${textFromMessages(ctx.expectedOutput)}` : ''} `); @@ -37,7 +37,6 @@ The template receives evaluation context via stdin (JSON) and outputs the prompt - `input` - Input messages to agent - `output` - The agent's final answer being evaluated -- `answer` - Deprecated alias for `output` - `expectedOutput` - Optional expected output messages - `criteria` - Optional criteria / expected outcome - `messages` - Transcript messages from the target execution diff --git a/examples/features/prompt-template-sdk/prompts/custom-evaluator.ts b/examples/features/prompt-template-sdk/prompts/custom-evaluator.ts index 615064a73..f472fef0a 100644 --- a/examples/features/prompt-template-sdk/prompts/custom-evaluator.ts +++ b/examples/features/prompt-template-sdk/prompts/custom-evaluator.ts @@ -28,7 +28,7 @@ function getMessageText( export default definePromptTemplate((ctx) => { const inputText = getMessageText(ctx.input, 'user'); - const outputText = ctx.output ?? ctx.answer ?? ''; + const outputText = ctx.output ?? ''; const expectedOutputText = getMessageText(ctx.expectedOutput); // Access typed config from YAML diff --git a/packages/core/src/evaluation/graders/code-grader.ts b/packages/core/src/evaluation/graders/code-grader.ts index e58c1b438..330830c86 100644 --- a/packages/core/src/evaluation/graders/code-grader.ts +++ b/packages/core/src/evaluation/graders/code-grader.ts @@ -171,7 +171,6 @@ export class CodeGrader implements Grader { getImageDir, ), output: outputForPayload, - answer: context.candidate, messages: materializedMessages ?? [], outputPath, inputFiles: context.evalCase.file_paths, diff --git a/packages/core/test/evaluation/code-grader-multimodal.test.ts b/packages/core/test/evaluation/code-grader-multimodal.test.ts index 130eb5e0e..47a951563 100644 --- a/packages/core/test/evaluation/code-grader-multimodal.test.ts +++ b/packages/core/test/evaluation/code-grader-multimodal.test.ts @@ -270,6 +270,7 @@ describe('CodeGrader multimodal integration', () => { const details = result.details as Record; const payload = details.payload as Record; expect(payload.output).toBe('answer'); + expect(payload).not.toHaveProperty('answer'); const messages = payload.messages as Record[]; expect(messages[0].content).toBe('Hello world'); }); diff --git a/packages/core/test/evaluation/graders.test.ts b/packages/core/test/evaluation/graders.test.ts index 414dd001d..1db1b9c3f 100644 --- a/packages/core/test/evaluation/graders.test.ts +++ b/packages/core/test/evaluation/graders.test.ts @@ -1010,8 +1010,8 @@ describe('CodeGrader', () => { expect(result.verdict).toBe('pass'); const passedTexts = result.assertions.filter((a) => a.passed).map((a) => a.text); expect(passedTexts).toContain('expected_output present'); - expect(passedTexts).toContain('answer present'); - expect(passedTexts).toContain('answer parses'); + expect(passedTexts).toContain('output present'); + expect(passedTexts).toContain('output parses'); }); it('surfaces stderr and exit code on failure', async () => { diff --git a/packages/core/test/evaluation/graders/prompt-resolution.test.ts b/packages/core/test/evaluation/graders/prompt-resolution.test.ts index 53a42e801..8b76fb618 100644 --- a/packages/core/test/evaluation/graders/prompt-resolution.test.ts +++ b/packages/core/test/evaluation/graders/prompt-resolution.test.ts @@ -105,9 +105,6 @@ definePromptTemplate((ctx) => { if (ctx.output !== 'Final answer') { throw new Error('unexpected final answer: ' + ctx.output); } - if (ctx.answer !== ctx.output) { - throw new Error('answer should mirror output'); - } if (!Array.isArray(ctx.messages) || ctx.messages.length < 2) { throw new Error('expected transcript messages'); } diff --git a/packages/core/test/fixtures/test-grader-with-details.cjs b/packages/core/test/fixtures/test-grader-with-details.cjs index 54e40eed2..ae8604d17 100644 --- a/packages/core/test/fixtures/test-grader-with-details.cjs +++ b/packages/core/test/fixtures/test-grader-with-details.cjs @@ -7,16 +7,14 @@ const fs = require('node:fs'); const input = JSON.parse(fs.readFileSync(0, 'utf8')); const hasExpected = Array.isArray(input.expected_output); -// `output` is the final answer/scored result. Keep a tiny legacy fallback so -// this fixture can still explain failures if an old message-array payload leaks. +// `output` is the final answer/scored result. Keep a tiny fallback so this +// fixture can still explain failures if an old message-array payload leaks. const candidateText = typeof input.output === 'string' ? input.output - : typeof input.answer === 'string' - ? input.answer - : Array.isArray(input.output) - ? input.output.map((m) => String(m.content ?? '')).join('') - : ''; + : Array.isArray(input.output) + ? input.output.map((m) => String(m.content ?? '')).join('') + : ''; const hasCandidate = candidateText.length > 0; // Emit details with structured metrics @@ -25,7 +23,7 @@ console.log( score: hasExpected && hasCandidate ? 0.75 : 0, assertions: [ ...(hasExpected ? [{ text: 'expected_output present', passed: true }] : []), - ...(hasCandidate ? [] : [{ text: 'answer missing', passed: false }]), + ...(hasCandidate ? [] : [{ text: 'output missing', passed: false }]), ], details: { metrics: { diff --git a/packages/core/test/fixtures/test-grader.cjs b/packages/core/test/fixtures/test-grader.cjs index 5e042b807..74ebf0312 100644 --- a/packages/core/test/fixtures/test-grader.cjs +++ b/packages/core/test/fixtures/test-grader.cjs @@ -4,18 +4,16 @@ const fs = require('node:fs'); const input = JSON.parse(fs.readFileSync(0, 'utf8')); const hasExpected = Array.isArray(input.expected_output); -// `output` is the final answer/scored result. Keep a tiny legacy fallback so -// this fixture can still explain failures if an old message-array payload leaks. +// `output` is the final answer/scored result. Keep a tiny fallback so this +// fixture can still explain failures if an old message-array payload leaks. const candidateText = typeof input.output === 'string' ? input.output - : typeof input.answer === 'string' - ? input.answer - : Array.isArray(input.output) - ? input.output - .map((m) => (typeof m.content === 'string' ? m.content : JSON.stringify(m.content))) - .join('') - : ''; + : Array.isArray(input.output) + ? input.output + .map((m) => (typeof m.content === 'string' ? m.content : JSON.stringify(m.content))) + .join('') + : ''; const hasCandidate = candidateText.length > 0; let candidateDecisionOk = false; @@ -31,8 +29,8 @@ console.log( score: ok ? 1 : 0, assertions: [ { text: 'expected_output present', passed: hasExpected }, - { text: 'answer present', passed: hasCandidate }, - { text: 'answer parses', passed: candidateDecisionOk }, + { text: 'output present', passed: hasCandidate }, + { text: 'output parses', passed: candidateDecisionOk }, ].filter((a) => a.passed !== undefined), }), ); diff --git a/packages/eval/src/index.ts b/packages/eval/src/index.ts index 38895ff61..e1ff4eef9 100644 --- a/packages/eval/src/index.ts +++ b/packages/eval/src/index.ts @@ -220,7 +220,7 @@ export function defineCodeGrader(handler: CodeGraderHandler): void { * .filter((message) => message.role === 'user') * .map((message) => typeof message.content === 'string' ? message.content : '') * .join('\n'); - * const answer = ctx.output ?? ctx.answer ?? ''; + * const answer = ctx.output ?? ''; * return `Question: ${question}\nAnswer: ${answer}`; * }); * ``` diff --git a/packages/eval/src/prompt-template.ts b/packages/eval/src/prompt-template.ts index 8db66c24c..e0936876c 100644 --- a/packages/eval/src/prompt-template.ts +++ b/packages/eval/src/prompt-template.ts @@ -75,7 +75,7 @@ export async function runPromptTemplate(handler: PromptTemplateHandler): Promise * .filter((message) => message.role === 'user') * .map((message) => typeof message.content === 'string' ? message.content : '') * .join('\n'); - * const answer = ctx.output ?? ctx.answer ?? ''; + * const answer = ctx.output ?? ''; * return `Question: ${question}\nAnswer: ${answer}`; * }); * ``` diff --git a/packages/eval/src/schemas.ts b/packages/eval/src/schemas.ts index 1c42f6666..a3a924a56 100644 --- a/packages/eval/src/schemas.ts +++ b/packages/eval/src/schemas.ts @@ -319,8 +319,6 @@ export const CodeGraderInputSchema = z.object({ criteria: z.string(), expectedOutput: z.array(MessageSchema), output: z.string().nullable().optional(), - /** Deprecated migration alias; same value as output for text agents. */ - answer: z.string().optional(), messages: z.array(MessageSchema).optional().default([]), /** Path to a temp file containing the output JSON (used for large payloads). */ outputPath: z.string().optional(), diff --git a/scripts/run-contract-eval.ts b/scripts/run-contract-eval.ts index fc81148fa..19d78b2dd 100644 --- a/scripts/run-contract-eval.ts +++ b/scripts/run-contract-eval.ts @@ -15,6 +15,7 @@ process.env.CONTRACT_EVAL_MODEL ||= 'openai/gpt-4.1-mini'; const evalFiles = [ 'examples/contract/evals/release-gate.eval.yaml', 'examples/contract/evals/repo-materialization.eval.yaml', + 'examples/contract/evals/code-grader-contract.eval.yaml', ]; for (const evalFile of evalFiles) {