From daf59c9016d2ee7e2c127428491fb60b28047638 Mon Sep 17 00:00:00 2001 From: Christopher Tso Date: Thu, 18 Jun 2026 01:26:33 +0200 Subject: [PATCH 1/3] chore(sdk): remove deprecated grader aliases --- apps/cli/src/commands/eval/commands/assert.ts | 6 --- apps/cli/src/commands/pipeline/grade.ts | 4 -- apps/cli/test/commands/eval/assert.test.ts | 45 ++++++++++++++++++- .../docs/docs/evaluation/batch-cli.mdx | 6 +-- .../docs/docs/graders/code-graders.mdx | 2 +- .../content/docs/docs/graders/llm-graders.mdx | 24 ++++------ .../integrations/autoevals-integration.mdx | 20 ++++----- .../basic/evals/check_python_keywords.py | 2 +- .../graders/fuzzy_match.ts | 12 ++--- .../graders/header_confusion_metrics.ts | 4 +- .../graders/line_item_matching.ts | 4 +- .../graders/multi_field_fuzzy.ts | 8 ++-- .../features/rubric/evals/check_syntax.py | 2 +- .../evals/validate_output.py | 2 +- .../grader-conformance/conformance-check.ts | 8 ++-- .../psychotherapy/evals/validate_output.py | 2 +- .../tool-evaluation-plugins/README.md | 2 +- .../tool-eval-demo.eval.yaml | 4 +- .../evaluation/graders/llm-grader-prompt.ts | 3 -- .../core/src/evaluation/graders/llm-grader.ts | 40 +---------------- .../core/src/evaluation/template-variables.ts | 20 --------- .../evaluation/validation/prompt-validator.ts | 35 +++------------ .../evaluation/evaluators_variables.test.ts | 39 ---------------- .../graders/prompt-resolution.test.ts | 9 ++-- .../validation/prompt-validator.test.ts | 10 +++-- skills-data/agentv-eval-writer/SKILL.md | 2 +- 26 files changed, 107 insertions(+), 208 deletions(-) diff --git a/apps/cli/src/commands/eval/commands/assert.ts b/apps/cli/src/commands/eval/commands/assert.ts index c50e20e5c..4e7220edc 100644 --- a/apps/cli/src/commands/eval/commands/assert.ts +++ b/apps/cli/src/commands/eval/commands/assert.ts @@ -80,8 +80,6 @@ export const evalAssertCommand = command({ question: resolvedInput, criteria: '', expected_output: [], - reference_answer: '', - input_files: [], trace, token_usage: null, @@ -93,10 +91,6 @@ export const evalAssertCommand = command({ workspace_path: null, config: null, metadata: {}, - // Text convenience accessors (new names) - input_text: resolvedInput, - output_text: resolvedOutput, - expected_output_text: '', }, null, 2, diff --git a/apps/cli/src/commands/pipeline/grade.ts b/apps/cli/src/commands/pipeline/grade.ts index 906987245..738924275 100644 --- a/apps/cli/src/commands/pipeline/grade.ts +++ b/apps/cli/src/commands/pipeline/grade.ts @@ -107,7 +107,6 @@ export async function runCodeGraders( const executeCodeGrader = async (graderConfig: Record, task: GraderTask) => { const { testId, resultsDir, responseText, inputData } = task; const graderName = graderConfig.name as string; - const inputText = extractInputText(inputData.input); const messages = [{ role: 'assistant' as const, content: responseText }]; const trace = buildTraceFromMessages({ input: inputData.input, @@ -133,9 +132,6 @@ export async function runCodeGraders( workspace_path: null, config: graderConfig.config ?? null, metadata: inputData.metadata ?? {}, - input_text: inputText, - output_text: responseText, - expected_output_text: '', }); try { diff --git a/apps/cli/test/commands/eval/assert.test.ts b/apps/cli/test/commands/eval/assert.test.ts index 0a661d7f9..0302a958a 100644 --- a/apps/cli/test/commands/eval/assert.test.ts +++ b/apps/cli/test/commands/eval/assert.test.ts @@ -27,12 +27,29 @@ console.log(JSON.stringify({ score: 1.0, assertions: [{ text: "always passes", p path.join(gradersDir, 'check-contains.ts'), `const input = await Bun.stdin.text(); const payload = JSON.parse(input); -const output = payload.output_text ?? payload.output ?? ''; +const output = payload.output ?? ''; const score = typeof output === 'string' && output.includes('hello') ? 1.0 : 0.0; console.log(JSON.stringify({ score, assertions: [{ text: score ? "contains hello" : "missing hello", passed: !!score }] }));`, 'utf8', ); + await writeFile( + path.join(gradersDir, 'reject-legacy-fields.ts'), + `const input = await Bun.stdin.text(); +const payload = JSON.parse(input); +const hasLegacyFields = [ + "input_text", + "output_text", + "expected_output_text", + "reference_answer", +].some((key) => Object.prototype.hasOwnProperty.call(payload, key)); +console.log(JSON.stringify({ + score: hasLegacyFields ? 0 : 1, + assertions: [{ text: hasLegacyFields ? "legacy fields present" : "canonical fields only", passed: !hasLegacyFields }], +}));`, + 'utf8', + ); + return { baseDir }; } @@ -138,4 +155,30 @@ describe('agentv eval assert', () => { await rm(baseDir, { recursive: true, force: true }); } }, 30_000); + + it('sends only canonical wire fields to code graders', async () => { + const { baseDir } = await createGraderFixture(); + try { + const result = await execa( + 'bun', + [ + '--no-env-file', + CLI_ENTRY, + 'eval', + 'assert', + 'reject-legacy-fields', + '--agent-output', + 'hello world', + '--agent-input', + 'test', + ], + { cwd: baseDir, reject: false }, + ); + expect(result.exitCode).toBe(0); + const parsed = JSON.parse(result.stdout); + expect(parsed.score).toBe(1.0); + } finally { + await rm(baseDir, { recursive: true, force: true }); + } + }, 30_000); }); diff --git a/apps/web/src/content/docs/docs/evaluation/batch-cli.mdx b/apps/web/src/content/docs/docs/evaluation/batch-cli.mdx index ed01ab3e5..4ca45caa7 100644 --- a/apps/web/src/content/docs/docs/evaluation/batch-cli.mdx +++ b/apps/web/src/content/docs/docs/evaluation/batch-cli.mdx @@ -147,7 +147,7 @@ Each test has its own grader that validates the batch runner output. The grader **Input (stdin):** ```json { - "output_text": "{\"id\":\"case-001\",\"decision\":\"CLEAR\",...}", + "output": "{\"id\":\"case-001\",\"decision\":\"CLEAR\",...}", "expected_output": [{"role": "assistant", "content": {"decision": "CLEAR"}}], "input": [...] } @@ -170,7 +170,7 @@ Each test has its own grader that validates the batch runner output. The grader import fs from 'node:fs'; type EvalInput = { - output_text?: string; + output?: string; expected_output?: Array<{ role: string; content: unknown }>; }; @@ -182,7 +182,7 @@ function main() { let candidateDecision: string | undefined; try { - const parsed = JSON.parse(input.output_text ?? ''); + const parsed = JSON.parse(input.output ?? ''); candidateDecision = parsed.decision; } catch { candidateDecision = undefined; diff --git a/apps/web/src/content/docs/docs/graders/code-graders.mdx b/apps/web/src/content/docs/docs/graders/code-graders.mdx index 49550ee39..971d49206 100644 --- a/apps/web/src/content/docs/docs/graders/code-graders.mdx +++ b/apps/web/src/content/docs/docs/graders/code-graders.mdx @@ -35,7 +35,7 @@ Raw grader stdin is a process-boundary wire format, so keys are `snake_case`. Ty | Raw stdin key | SDK field | Meaning | |---------------|-----------|---------| | `output` | `output` | Final answer / scored result as a string | -| `answer` | `answer` | Deprecated alias for the same final answer string | +| `answer` | `answer` | Same final answer string, exposed for ergonomic handler code | | `messages` | `messages` | Transcript messages for transcript-aware graders | | `expected_output` | `expectedOutput` | Reference answer messages | | `output_path` | `outputPath` | Temp file containing large final answer JSON, when used | diff --git a/apps/web/src/content/docs/docs/graders/llm-graders.mdx b/apps/web/src/content/docs/docs/graders/llm-graders.mdx index e46d571dd..f6663759a 100644 --- a/apps/web/src/content/docs/docs/graders/llm-graders.mdx +++ b/apps/web/src/content/docs/docs/graders/llm-graders.mdx @@ -48,10 +48,10 @@ Write evaluation instructions as markdown. Template variables are interpolated: Evaluate the candidate's response to the following question: -**Question:** {{input_text}} +**Question:** {{input}} **Criteria:** {{criteria}} -**Reference Answer:** {{expected_output_text}} -**Candidate Answer:** {{output_text}} +**Reference Answer:** {{expected_output}} +**Candidate Answer:** {{output}} ## Scoring @@ -65,13 +65,10 @@ Score the response from 0.0 to 1.0 based on: | Variable | Source | |----------|--------| -| `input_text` | First user message content | -| `output_text` | Candidate final answer text | -| `expected_output_text` | Last expected message content | -| `criteria` | Test `criteria` field | | `input` | Resolved input text | | `expected_output` | Reference answer text | | `output` | Candidate answer text | +| `criteria` | Test `criteria` field | | `metadata` | Test metadata as formatted JSON | | `metadata_json` | Test metadata as compact JSON | | `rubrics` | LLM-grader rubric items as formatted JSON | @@ -230,7 +227,7 @@ TypeScript templates receive a context object with these fields: |-------|------|-------------| | `input` | `Message[]` | Full resolved input messages | | `output` | `string \| null` | Candidate final answer / scored result | -| `answer` | `string` | Deprecated alias for `output` | +| `answer` | `string` | Same final answer string, exposed for ergonomic handler code | | `messages` | `Message[]` | Transcript messages from the target execution | | `criteria` | `string` | Test `criteria` field | | `expectedOutput` | `Message[]` | Full resolved expected output | @@ -267,11 +264,8 @@ Derived strings injected into grader prompts: | Variable | Derivation | |----------|------------| -| `input_text` | Content of the first `user` role entry in `input` | -| `criteria` | Passed through from the test field | -| `expected_output_text` | Content of the last entry in `expected_output` | -| `output_text` | Candidate final answer text | | `input` | Resolved input text | +| `criteria` | Passed through from the test field | | `expected_output` | Reference answer text | | `output` | Candidate answer text | | `metadata_json` | Test metadata, compact JSON | @@ -292,7 +286,7 @@ input: [{ role: "user", content: "What is 2+2?" }] expected_output: [{ role: "assistant", content: "The answer is 4" }] # Derived template variables: -input_text: "What is 2+2?" -expected_output_text: "The answer is 4" -output_text: (extracted from provider output at runtime) +input: "What is 2+2?" +expected_output: "The answer is 4" +output: (extracted from provider output at runtime) ``` diff --git a/apps/web/src/content/docs/docs/integrations/autoevals-integration.mdx b/apps/web/src/content/docs/docs/integrations/autoevals-integration.mdx index 12c700023..f4aeee750 100644 --- a/apps/web/src/content/docs/docs/integrations/autoevals-integration.mdx +++ b/apps/web/src/content/docs/docs/integrations/autoevals-integration.mdx @@ -77,9 +77,9 @@ import { Factuality } from "autoevals"; const input = JSON.parse(readFileSync("/dev/stdin", "utf-8")); const result = await Factuality({ - input: input.input_text, - output: input.output_text, - expected: input.reference_answer, + input: input.input, + output: input.output ?? input.answer, + expected: input.expected_output, }); const score = result.score ?? 0; @@ -94,7 +94,7 @@ console.log( ); ``` -The code grader reads the AgentV input from stdin (`input_text`, `output_text`, `expected_output_text`), maps the fields to autoevals parameters (`input`, `output`, `expected`), runs the scorer, and writes the AgentV result format (with `assertions` array) to stdout. +The code grader reads the AgentV input from stdin (`input`, `output`, `expected_output`), maps the fields to autoevals parameters (`input`, `output`, `expected`), runs the scorer, and writes the AgentV result format (with `assertions` array) to stdout. ## Python Example @@ -127,9 +127,9 @@ data = json.load(sys.stdin) grader = Faithfulness() result = grader( - input=data.get("input_text", ""), - output=data.get("output_text", ""), - expected=data.get("reference_answer", ""), + input=data.get("input", ""), + output=data.get("output") or data.get("answer", ""), + expected=data.get("expected_output", ""), ) score = result.score or 0 @@ -218,9 +218,9 @@ import { const input = JSON.parse(readFileSync("/dev/stdin", "utf-8")); const scorerArgs = { - input: input.input_text, - output: input.output_text, - expected: input.reference_answer, + input: input.input, + output: input.output ?? input.answer, + expected: input.expected_output, }; // Run all scorers in parallel diff --git a/examples/features/basic/evals/check_python_keywords.py b/examples/features/basic/evals/check_python_keywords.py index 10a5658d2..1e1405642 100644 --- a/examples/features/basic/evals/check_python_keywords.py +++ b/examples/features/basic/evals/check_python_keywords.py @@ -80,7 +80,7 @@ def main(): input_data = json.loads(sys.stdin.read()) # Extract the generated output - output = input_data.get("output_text", "") + output = input_data.get("output") or input_data.get("answer", "") # Extract code from markdown if present code = extract_code_from_markdown(output) diff --git a/examples/features/document-extraction/graders/fuzzy_match.ts b/examples/features/document-extraction/graders/fuzzy_match.ts index eff4d3074..c2ea71474 100644 --- a/examples/features/document-extraction/graders/fuzzy_match.ts +++ b/examples/features/document-extraction/graders/fuzzy_match.ts @@ -20,10 +20,10 @@ import { jaroWinklerSimilarity, levenshteinSimilarity } from '../lib/fuzzy_utils'; interface EvalInput { - output_text: string; - expected_output_text: string; + output: string; + expected_output: string; criteria: string; - input_text: string; + input: string; } interface AssertionEntry { @@ -51,10 +51,10 @@ async function main(): Promise { const input: EvalInput = JSON.parse(inputText); // Extract and normalize strings for comparison - const candidate = String(input.output_text || '') + const candidate = String(input.output || '') .trim() .toLowerCase(); - const expected = String(input.expected_output_text || '') + const expected = String(input.expected_output || '') .trim() .toLowerCase(); @@ -77,7 +77,7 @@ async function main(): Promise { ? `Similarity: ${(similarity * 100).toFixed(1)}% (threshold: ${SIMILARITY_THRESHOLD * 100}%)` : `Similarity: ${(similarity * 100).toFixed(1)}% < ${SIMILARITY_THRESHOLD * 100}% threshold`, passed, - evidence: `${ALGORITHM} similarity between "${input.output_text}" and "${input.expected_output_text}": ${(similarity * 100).toFixed(1)}%`, + evidence: `${ALGORITHM} similarity between "${input.output}" and "${input.expected_output}": ${(similarity * 100).toFixed(1)}%`, }, ], }; diff --git a/examples/features/document-extraction/graders/header_confusion_metrics.ts b/examples/features/document-extraction/graders/header_confusion_metrics.ts index eb9878a6a..70dfa5b79 100644 --- a/examples/features/document-extraction/graders/header_confusion_metrics.ts +++ b/examples/features/document-extraction/graders/header_confusion_metrics.ts @@ -34,7 +34,7 @@ interface EvalConfig { } interface EvalInput { - output_text: string; + output: string; expected_output: Array<{ role: string; content: unknown }>; config: EvalConfig | null; } @@ -146,7 +146,7 @@ async function main(): Promise { // Parse candidate answer let candidateObj: unknown; try { - candidateObj = JSON.parse(input.output_text); + candidateObj = JSON.parse(input.output); } catch { console.log( JSON.stringify({ diff --git a/examples/features/document-extraction/graders/line_item_matching.ts b/examples/features/document-extraction/graders/line_item_matching.ts index 751eef55e..d25084bb8 100644 --- a/examples/features/document-extraction/graders/line_item_matching.ts +++ b/examples/features/document-extraction/graders/line_item_matching.ts @@ -34,7 +34,7 @@ interface EvalConfig { } interface EvalInput { - output_text: string; + output: string; expected_output: Array<{ role: string; content: unknown }>; config: EvalConfig | null; } @@ -224,7 +224,7 @@ async function main(): Promise { // Parse candidate answer let candidateObj: unknown; try { - candidateObj = JSON.parse(input.output_text); + candidateObj = JSON.parse(input.output); } catch { console.log( JSON.stringify({ diff --git a/examples/features/document-extraction/graders/multi_field_fuzzy.ts b/examples/features/document-extraction/graders/multi_field_fuzzy.ts index e6e788c2d..8630508a7 100644 --- a/examples/features/document-extraction/graders/multi_field_fuzzy.ts +++ b/examples/features/document-extraction/graders/multi_field_fuzzy.ts @@ -34,8 +34,8 @@ interface EvalConfig { } interface EvalInput { - output_text: string; - expected_output_text: string; + output: string; + expected_output: string; config: EvalConfig | null; } @@ -86,8 +86,8 @@ async function main(): Promise { let candidateObj: unknown; let referenceObj: unknown; try { - candidateObj = JSON.parse(input.output_text); - referenceObj = JSON.parse(input.expected_output_text); + candidateObj = JSON.parse(input.output); + referenceObj = JSON.parse(input.expected_output); } catch { console.log( JSON.stringify({ diff --git a/examples/features/rubric/evals/check_syntax.py b/examples/features/rubric/evals/check_syntax.py index c213c197e..7b9a5d84c 100644 --- a/examples/features/rubric/evals/check_syntax.py +++ b/examples/features/rubric/evals/check_syntax.py @@ -13,7 +13,7 @@ def main(): # Fallback if no input or invalid JSON input_data = {} - candidate_answer = input_data.get("output_text", "") + candidate_answer = input_data.get("output") or input_data.get("answer", "") # Extract code block if present (simple heuristic) if "```python" in candidate_answer: diff --git a/examples/showcase/cw-incident-triage/evals/validate_output.py b/examples/showcase/cw-incident-triage/evals/validate_output.py index 886a3720a..bb9fbcb3e 100644 --- a/examples/showcase/cw-incident-triage/evals/validate_output.py +++ b/examples/showcase/cw-incident-triage/evals/validate_output.py @@ -81,7 +81,7 @@ def main(): sys.exit(1) # Extract candidate answer - candidate_answer = eval_data.get("output_text", "") + candidate_answer = eval_data.get("output") or eval_data.get("answer", "") # Required keys for CargoWise criticality rating required_keys = ["criticalityRating", "reasoning"] diff --git a/examples/showcase/grader-conformance/conformance-check.ts b/examples/showcase/grader-conformance/conformance-check.ts index 613f88aeb..b81b06c48 100644 --- a/examples/showcase/grader-conformance/conformance-check.ts +++ b/examples/showcase/grader-conformance/conformance-check.ts @@ -101,13 +101,11 @@ const maxFlipRate = Number.parseFloat(values['max-flip-rate'] ?? '0'); function buildCodeGraderInput(fixture: Fixture): string { // Build a minimal CodeGraderInput in the snake_case wire format return JSON.stringify({ - input_text: fixture.question, criteria: fixture.criteria, - output_text: fixture.answer, - expected_output_text: fixture.expected_output, - expected_output: [], + output: fixture.answer, + answer: fixture.answer, + expected_output: [{ role: 'assistant', content: fixture.expected_output }], input: [{ role: 'user', content: fixture.question }], - output: [{ role: 'assistant', content: fixture.answer }], input_files: [], }); } diff --git a/examples/showcase/psychotherapy/evals/validate_output.py b/examples/showcase/psychotherapy/evals/validate_output.py index 187d64cab..861107258 100644 --- a/examples/showcase/psychotherapy/evals/validate_output.py +++ b/examples/showcase/psychotherapy/evals/validate_output.py @@ -229,7 +229,7 @@ def main(): sys.exit(1) # Extract candidate answer - we only validate the candidate's structure - candidate_answer = eval_data.get("output_text", "") + candidate_answer = eval_data.get("output") or eval_data.get("answer", "") # Auto-detect required keys from candidate answer required_keys = detect_framework_keys(candidate_answer) diff --git a/examples/showcase/tool-evaluation-plugins/README.md b/examples/showcase/tool-evaluation-plugins/README.md index 1ee18e7ec..5776985b4 100644 --- a/examples/showcase/tool-evaluation-plugins/README.md +++ b/examples/showcase/tool-evaluation-plugins/README.md @@ -67,7 +67,7 @@ All code graders receive a JSON object on stdin with: { "question": "User's question/task", "criteria": "Expected behavior description", - "reference_answer": "Gold standard answer (from expected_output)", + "expected_output": "Gold standard answer", "answer": "Agent's final response", "output": [ { diff --git a/examples/showcase/tool-evaluation-plugins/tool-eval-demo.eval.yaml b/examples/showcase/tool-evaluation-plugins/tool-eval-demo.eval.yaml index 18bc492e9..31097c865 100644 --- a/examples/showcase/tool-evaluation-plugins/tool-eval-demo.eval.yaml +++ b/examples/showcase/tool-evaluation-plugins/tool-eval-demo.eval.yaml @@ -104,7 +104,7 @@ tests: # ========================================== # Example 4: Pairwise Comparison # Use case: Compare candidate against baseline response - # Requires reference_answer field + # Requires expected_output to supply a reference answer # ========================================== - id: pairwise-demo @@ -116,7 +116,7 @@ tests: content: Summarize the main points of the user manual. # Reference answer for comparison (from a baseline agent) - # Note: reference_answer is constructed from expected_output content + # Note: the reference answer is derived from expected_output content expected_output: - role: assistant content: |- diff --git a/packages/core/src/evaluation/graders/llm-grader-prompt.ts b/packages/core/src/evaluation/graders/llm-grader-prompt.ts index fe79c525e..234d99613 100644 --- a/packages/core/src/evaluation/graders/llm-grader-prompt.ts +++ b/packages/core/src/evaluation/graders/llm-grader-prompt.ts @@ -50,9 +50,6 @@ function buildTemplateVariables(input: { [TEMPLATE_VARIABLES.RUBRICS_JSON]: stringifyCompact(input.rubrics), [TEMPLATE_VARIABLES.FILE_CHANGES]: input.fileChanges ?? '', [TEMPLATE_VARIABLES.TOOL_CALLS]: input.toolCalls ?? '', - [TEMPLATE_VARIABLES.INPUT_TEXT]: formattedQuestion.trim(), - [TEMPLATE_VARIABLES.OUTPUT_TEXT]: input.candidate.trim(), - [TEMPLATE_VARIABLES.EXPECTED_OUTPUT_TEXT]: (input.evalCase.reference_answer ?? '').trim(), }; } diff --git a/packages/core/src/evaluation/graders/llm-grader.ts b/packages/core/src/evaluation/graders/llm-grader.ts index acdf85248..f5593801a 100644 --- a/packages/core/src/evaluation/graders/llm-grader.ts +++ b/packages/core/src/evaluation/graders/llm-grader.ts @@ -11,7 +11,7 @@ import type { ContentImage } from '../content.js'; import { isContentArray } from '../content.js'; import type { Message, Provider, ProviderResponse, ProviderTool } from '../providers/types.js'; import { extractLastAssistantContent, isAgentProvider } from '../providers/types.js'; -import { DEPRECATED_TEMPLATE_VARIABLES, TEMPLATE_VARIABLES } from '../template-variables.js'; +import { TEMPLATE_VARIABLES } from '../template-variables.js'; import type { TokenUsage } from '../trace.js'; import type { AssertionEntry, JsonObject, RubricItem } from '../types.js'; import { formatRubricOperatorGuidance, formatRubricOperatorLabel } from './rubric-operators.js'; @@ -181,10 +181,6 @@ function buildTemplateVariables(context: EvaluationContext): Record(); - -/** - * Emit a one-time stderr warning when a template uses deprecated _text variable names. - * Skips the default template (which uses the new names and should never trigger warnings). - */ -export function warnDeprecatedTemplateVars(template: string): void { - if (warnedTemplateStrings.has(template)) return; - - const used: string[] = []; - for (const [deprecated, replacement] of DEPRECATED_TEMPLATE_VARIABLES) { - if (new RegExp(`\\{\\{\\s*${deprecated}\\s*\\}\\}`).test(template)) { - used.push(`{{ ${deprecated} }} → {{ ${replacement} }}`); - } - } - - if (used.length > 0) { - warnedTemplateStrings.add(template); - console.warn( - `${ANSI_YELLOW}⚠ Deprecated template variables detected (they still work but will be removed in a future version):\n ${used.join('\n ')}\n Update your custom grader template to use the new names.${ANSI_RESET}`, - ); - } -} - export function calculateRubricScore( result: z.infer, rubrics: readonly RubricItem[], diff --git a/packages/core/src/evaluation/template-variables.ts b/packages/core/src/evaluation/template-variables.ts index 81e13c36a..e14484a39 100644 --- a/packages/core/src/evaluation/template-variables.ts +++ b/packages/core/src/evaluation/template-variables.ts @@ -14,10 +14,6 @@ * - {{ file_changes }} — file diff (if available) * - {{ tool_calls }} — formatted summary of tool calls from agent execution * - * Deprecated aliases (emit a warning when used in custom templates): - * - {{ input_text }} → use {{ input }} - * - {{ output_text }} → use {{ output }} - * - {{ expected_output_text }} → use {{ expected_output }} */ export const TEMPLATE_VARIABLES = { EXPECTED_OUTPUT: 'expected_output', @@ -30,12 +26,6 @@ export const TEMPLATE_VARIABLES = { OUTPUT: 'output', FILE_CHANGES: 'file_changes', TOOL_CALLS: 'tool_calls', - /** @deprecated Use INPUT instead — resolves to the same text value. */ - INPUT_TEXT: 'input_text', - /** @deprecated Use OUTPUT instead — resolves to the same text value. */ - OUTPUT_TEXT: 'output_text', - /** @deprecated Use EXPECTED_OUTPUT instead — resolves to the same text value. */ - EXPECTED_OUTPUT_TEXT: 'expected_output_text', } as const; /** @@ -56,13 +46,3 @@ export const REQUIRED_TEMPLATE_VARIABLES = new Set([ TEMPLATE_VARIABLES.OUTPUT, TEMPLATE_VARIABLES.EXPECTED_OUTPUT, ]); - -/** - * Deprecated template variable names that still work but trigger a warning. - * Maps deprecated name → replacement name. - */ -export const DEPRECATED_TEMPLATE_VARIABLES: ReadonlyMap = new Map([ - [TEMPLATE_VARIABLES.INPUT_TEXT, TEMPLATE_VARIABLES.INPUT], - [TEMPLATE_VARIABLES.OUTPUT_TEXT, TEMPLATE_VARIABLES.OUTPUT], - [TEMPLATE_VARIABLES.EXPECTED_OUTPUT_TEXT, TEMPLATE_VARIABLES.EXPECTED_OUTPUT], -]); diff --git a/packages/core/src/evaluation/validation/prompt-validator.ts b/packages/core/src/evaluation/validation/prompt-validator.ts index 14ddf363e..5122cef8f 100644 --- a/packages/core/src/evaluation/validation/prompt-validator.ts +++ b/packages/core/src/evaluation/validation/prompt-validator.ts @@ -1,13 +1,6 @@ import { readFile } from 'node:fs/promises'; -import { - DEPRECATED_TEMPLATE_VARIABLES, - TEMPLATE_VARIABLES, - VALID_TEMPLATE_VARIABLES, -} from '../template-variables.js'; - -const ANSI_YELLOW = '\u001b[33m'; -const ANSI_RESET = '\u001b[0m'; +import { TEMPLATE_VARIABLES, VALID_TEMPLATE_VARIABLES } from '../template-variables.js'; /** * Validate custom prompt template content from a file. @@ -41,13 +34,8 @@ export function validateTemplateVariables(content: string, source: string): void } // Check if template contains required variables for evaluation. - // Accept both new names (output, expected_output) and deprecated aliases (output_text, expected_output_text). - const hasCandidateAnswer = - foundVariables.has(TEMPLATE_VARIABLES.OUTPUT) || - foundVariables.has(TEMPLATE_VARIABLES.OUTPUT_TEXT); - const hasExpectedOutput = - foundVariables.has(TEMPLATE_VARIABLES.EXPECTED_OUTPUT) || - foundVariables.has(TEMPLATE_VARIABLES.EXPECTED_OUTPUT_TEXT); + const hasCandidateAnswer = foundVariables.has(TEMPLATE_VARIABLES.OUTPUT); + const hasExpectedOutput = foundVariables.has(TEMPLATE_VARIABLES.EXPECTED_OUTPUT); const hasRequiredFields = hasCandidateAnswer || hasExpectedOutput; // ERROR: Missing required fields - throw error to skip this evaluator/eval case @@ -57,26 +45,13 @@ export function validateTemplateVariables(content: string, source: string): void ); } - // WARNING: Deprecated variables - show warning but continue - const deprecatedUsed: string[] = []; - for (const [deprecated, replacement] of DEPRECATED_TEMPLATE_VARIABLES) { - if (foundVariables.has(deprecated)) { - deprecatedUsed.push(`{{ ${deprecated} }} → {{ ${replacement} }}`); - } - } - if (deprecatedUsed.length > 0) { - console.warn( - `${ANSI_YELLOW}Warning: Template at ${source} uses deprecated variable names:\n ${deprecatedUsed.join('\n ')}\n These still work but will be removed in a future version.${ANSI_RESET}`, - ); - } - // WARNING: Invalid variables - show warning but continue if (invalidVariables.length > 0) { - const warningMessage = `${ANSI_YELLOW}Warning: Custom grader template at ${source} + const warningMessage = `Warning: Custom grader template at ${source} Contains invalid variables: ${invalidVariables.map((v) => `{{ ${v} }}`).join(', ')} Valid variables: ${Array.from(VALID_TEMPLATE_VARIABLES) .map((v) => `{{ ${v} }}`) - .join(', ')}${ANSI_RESET}`; + .join(', ')}`; console.warn(warningMessage); } diff --git a/packages/core/test/evaluation/evaluators_variables.test.ts b/packages/core/test/evaluation/evaluators_variables.test.ts index 835084494..929f52397 100644 --- a/packages/core/test/evaluation/evaluators_variables.test.ts +++ b/packages/core/test/evaluation/evaluators_variables.test.ts @@ -155,45 +155,6 @@ Candidate: {{output}} expect(prompt).toContain('Candidate: Apple revenue increased.'); }); - it('deprecated _text aliases still resolve correctly', async () => { - const formattedQuestion = 'What is 2+2?'; - const customPrompt = ` -Question: {{input_text}} -Reference: {{expected_output_text}} -Candidate: {{output_text}} -`; - - const graderProvider = new CapturingProvider({ - text: JSON.stringify({ - score: 0.9, - assertions: [{ text: 'OK', passed: true }], - }), - }); - - const evaluator = new LlmGrader({ - resolveGraderProvider: async () => graderProvider, - graderTemplate: customPrompt, - }); - - await evaluator.evaluate({ - evalCase: { ...baseTestCase, evaluator: 'llm-grader' }, - candidate: 'Four', - target: baseTarget, - provider: graderProvider, - attempt: 0, - promptInputs: { question: formattedQuestion }, - now: new Date(), - }); - - const request = graderProvider.lastRequest; - expect(request).toBeDefined(); - - // Deprecated aliases resolve to the same text values as the primary variables - expect(request?.question).toContain(`Question: ${formattedQuestion}`); - expect(request?.question).toContain('Reference: Reference Answer Text'); - expect(request?.question).toContain('Candidate: Four'); - }); - it('does not substitute if no variables are present', async () => { const customPrompt = 'Fixed prompt without variables'; const promptQuestion = 'Summarize the latest logs without markers.'; diff --git a/packages/core/test/evaluation/graders/prompt-resolution.test.ts b/packages/core/test/evaluation/graders/prompt-resolution.test.ts index 731a28dde..53a42e801 100644 --- a/packages/core/test/evaluation/graders/prompt-resolution.test.ts +++ b/packages/core/test/evaluation/graders/prompt-resolution.test.ts @@ -41,12 +41,9 @@ describe('containsTemplateVariables', () => { ).toBe(true); }); - it('returns true for deprecated {{output_text}} variable', () => { - expect(containsTemplateVariables('Grade the {{output_text}}')).toBe(true); - }); - - it('returns true for deprecated {{input_text}} variable', () => { - expect(containsTemplateVariables('Evaluate {{input_text}}')).toBe(true); + it('returns false for removed deprecated _text variables', () => { + expect(containsTemplateVariables('Grade the {{output_text}}')).toBe(false); + expect(containsTemplateVariables('Evaluate {{input_text}}')).toBe(false); }); it('returns true with whitespace in braces', () => { diff --git a/packages/core/test/evaluation/validation/prompt-validator.test.ts b/packages/core/test/evaluation/validation/prompt-validator.test.ts index 9a189c953..30f58ae16 100644 --- a/packages/core/test/evaluation/validation/prompt-validator.test.ts +++ b/packages/core/test/evaluation/validation/prompt-validator.test.ts @@ -13,14 +13,16 @@ describe('validateTemplateVariables', () => { ).not.toThrow(); }); - it('passes when template contains deprecated {{ output_text }}', () => { - expect(() => validateTemplateVariables('Score: {{ output_text }}', 'test.txt')).not.toThrow(); + it('rejects removed deprecated {{ output_text }}', () => { + expect(() => validateTemplateVariables('Score: {{ output_text }}', 'test.txt')).toThrow( + 'Missing required fields', + ); }); - it('passes when template contains deprecated {{ expected_output_text }}', () => { + it('rejects removed deprecated {{ expected_output_text }}', () => { expect(() => validateTemplateVariables('Reference: {{ expected_output_text }}', 'test.txt'), - ).not.toThrow(); + ).toThrow('Missing required fields'); }); it('throws when no required or deprecated variables are present', () => { diff --git a/skills-data/agentv-eval-writer/SKILL.md b/skills-data/agentv-eval-writer/SKILL.md index f4172489e..f33fcff7b 100644 --- a/skills-data/agentv-eval-writer/SKILL.md +++ b/skills-data/agentv-eval-writer/SKILL.md @@ -372,7 +372,7 @@ See docs at https://agentv.dev/graders/code-graders/ config: # passed to prompt templates as context.config strictness: high ``` -Variables: `{{question}}`, `{{criteria}}`, `{{answer}}`, `{{reference_answer}}`, `{{input}}`, `{{expected_output}}`, `{{output}}`, `{{file_changes}}` +Variables: `{{criteria}}`, `{{answer}}`, `{{input}}`, `{{expected_output}}`, `{{output}}`, `{{file_changes}}` - Markdown templates: use `{{variable}}` syntax - TypeScript templates: use `definePromptTemplate(fn)` from `@agentv/eval`, receives context object with all variables + `config` - Use `target:` to run different `llm-grader` graders against different named LLM targets in the same eval (useful for grader panels / ensembles) From 938ec67513a0217e954af7bb77568d6c81f4fc09 Mon Sep 17 00:00:00 2001 From: Christopher Tso Date: Thu, 18 Jun 2026 01:40:01 +0200 Subject: [PATCH 2/3] chore(sdk): align assertion scaffolds with canonical output --- apps/cli/src/commands/create/commands.ts | 34 +++--------------- .../test/commands/create/assertion.test.ts | 35 +++++++++++++++++++ .../src/content/docs/docs/evaluation/sdk.mdx | 16 +++++---- .../docs/docs/graders/code-graders.mdx | 9 +++-- .../docs/docs/graders/custom-assertions.mdx | 24 ++++++++----- examples/features/eval-assert-demo/README.md | 2 +- packages/eval/README.md | 11 +++--- skills-data/agentv-eval-writer/SKILL.md | 20 ++++++----- .../references/custom-evaluators.md | 11 +++--- 9 files changed, 93 insertions(+), 69 deletions(-) create mode 100644 apps/cli/test/commands/create/assertion.test.ts diff --git a/apps/cli/src/commands/create/commands.ts b/apps/cli/src/commands/create/commands.ts index 762150601..82488e29e 100644 --- a/apps/cli/src/commands/create/commands.ts +++ b/apps/cli/src/commands/create/commands.ts @@ -6,53 +6,27 @@ const ASSERTION_TEMPLATES: Record = { default: `#!/usr/bin/env bun import { defineAssertion } from '@agentv/eval'; -/** Extract text from the last message with the given role. */ -function getMessageText(messages: Array<{ role: string; content?: unknown }>, role = 'assistant'): string { - for (let i = messages.length - 1; i >= 0; i--) { - const msg = messages[i]; - if (msg.role !== role) continue; - if (typeof msg.content === 'string') return msg.content; - if (Array.isArray(msg.content)) { - return msg.content.filter((b: any) => b.type === 'text').map((b: any) => b.text).join('\\n'); - } - } - return ''; -} - export default defineAssertion(({ output }) => { // TODO: Implement your assertion logic - const text = getMessageText(output ?? []); + const text = output ?? ''; const pass = text.length > 0; return { pass, - reasoning: pass ? 'Output has content' : 'Output is empty', + assertions: [{ text: pass ? 'Output has content' : 'Output is empty', passed: pass }], }; }); `, score: `#!/usr/bin/env bun import { defineAssertion } from '@agentv/eval'; -/** Extract text from the last message with the given role. */ -function getMessageText(messages: Array<{ role: string; content?: unknown }>, role = 'assistant'): string { - for (let i = messages.length - 1; i >= 0; i--) { - const msg = messages[i]; - if (msg.role !== role) continue; - if (typeof msg.content === 'string') return msg.content; - if (Array.isArray(msg.content)) { - return msg.content.filter((b: any) => b.type === 'text').map((b: any) => b.text).join('\\n'); - } - } - return ''; -} - export default defineAssertion(({ output }) => { // TODO: Implement your scoring logic (0.0 to 1.0) - const text = getMessageText(output ?? []); + const text = output ?? ''; const score = text.length > 0 ? 1.0 : 0.0; return { pass: score >= 0.5, score, - reasoning: \`Score: \${score}\`, + assertions: [{ text: 'Output has content', passed: score === 1.0 }], }; }); `, diff --git a/apps/cli/test/commands/create/assertion.test.ts b/apps/cli/test/commands/create/assertion.test.ts new file mode 100644 index 000000000..abe2bc790 --- /dev/null +++ b/apps/cli/test/commands/create/assertion.test.ts @@ -0,0 +1,35 @@ +import { describe, expect, it } from 'bun:test'; +import { mkdtemp, readFile, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { execa } from 'execa'; + +const __filename = fileURLToPath(import.meta.url); +const __dirname = path.dirname(__filename); +const projectRoot = path.resolve(__dirname, '../../../../..'); +const CLI_ENTRY = path.join(projectRoot, 'apps/cli/src/cli.ts'); + +describe('agentv create assertion', () => { + it('scaffolds canonical assertion output without deprecated reasoning examples', async () => { + const cwd = await mkdtemp(path.join(tmpdir(), 'agentv-create-assertion-')); + + try { + const result = await execa( + 'bun', + ['--no-env-file', CLI_ENTRY, 'create', 'assertion', 'word-count'], + { cwd, reject: false }, + ); + + expect(result.exitCode).toBe(0); + + const content = await readFile(path.join(cwd, '.agentv', 'assertions', 'word-count.ts'), 'utf8'); + expect(content).toContain("const text = output ?? '';"); + expect(content).toContain('assertions: [{ text: pass ? \'Output has content\' : \'Output is empty\', passed: pass }]'); + expect(content).not.toContain('reasoning:'); + expect(content).not.toContain('getMessageText'); + } finally { + await rm(cwd, { recursive: true, force: true }); + } + }, 30_000); +}); diff --git a/apps/web/src/content/docs/docs/evaluation/sdk.mdx b/apps/web/src/content/docs/docs/evaluation/sdk.mdx index 5b0e7e409..ccaed5dfd 100644 --- a/apps/web/src/content/docs/docs/evaluation/sdk.mdx +++ b/apps/web/src/content/docs/docs/evaluation/sdk.mdx @@ -32,9 +32,10 @@ import { defineAssertion } from '@agentv/eval'; export default defineAssertion(({ output }) => { const wordCount = (output ?? '').trim().split(/\s+/).filter(Boolean).length; + const pass = wordCount >= 3; return { - pass: wordCount >= 3, - reasoning: `Output has ${wordCount} words`, + pass, + assertions: [{ text: `Output has ${wordCount} words`, passed: pass }], }; }); ``` @@ -47,10 +48,13 @@ Return a `score` (0–1) instead of `pass` for graded evaluation: // .agentv/assertions/efficiency.ts import { defineAssertion } from '@agentv/eval'; -export default defineAssertion(({ output, traceSummary }) => ({ - pass: (output ?? '').length > 0 && (traceSummary?.eventCount ?? 0) <= 10, - reasoning: 'Checks content exists and is efficient', -})); +export default defineAssertion(({ output, traceSummary }) => { + const pass = (output ?? '').length > 0 && (traceSummary?.eventCount ?? 0) <= 10; + return { + pass, + assertions: [{ text: 'Checks content exists and is efficient', passed: pass }], + }; +}); ``` If only `pass` is given, score is `1` (pass) or `0` (fail). diff --git a/apps/web/src/content/docs/docs/graders/code-graders.mdx b/apps/web/src/content/docs/docs/graders/code-graders.mdx index 971d49206..821e905cf 100644 --- a/apps/web/src/content/docs/docs/graders/code-graders.mdx +++ b/apps/web/src/content/docs/docs/graders/code-graders.mdx @@ -108,11 +108,11 @@ Scripts that write to stderr and exit non-zero surface as execution errors rathe # validators/check_answer.py import json, sys data = json.load(sys.stdin) -output_text = data.get("output") or data.get("answer") or "" +output = data.get("output") or data.get("answer") or "" assertions = [] -if "42" in output_text: +if "42" in output: assertions.append({"text": "Output contains correct value (42)", "passed": True}) else: assertions.append({"text": "Output does not contain expected value (42)", "passed": False}) @@ -133,11 +133,11 @@ print(json.dumps({ import { readFileSync } from "fs"; const data = JSON.parse(readFileSync("/dev/stdin", "utf-8")); -const outputText: string = data.output ?? data.answer ?? ""; +const output: string = data.output ?? data.answer ?? ""; const assertions: Array<{ text: string; passed: boolean }> = []; -if (outputText.includes("42")) { +if (output.includes("42")) { assertions.push({ text: "Output contains correct value (42)", passed: true }); } else { assertions.push({ text: "Output does not contain expected value (42)", passed: false }); @@ -148,7 +148,6 @@ const passed = assertions.filter(a => a.passed).length; console.log(JSON.stringify({ score: passed > 0 ? 1.0 : 0.0, assertions, - reasoning: `Passed ${passed} check(s)`, })); ``` diff --git a/apps/web/src/content/docs/docs/graders/custom-assertions.mdx b/apps/web/src/content/docs/docs/graders/custom-assertions.mdx index 5caa1d9ef..cc83f5a5d 100644 --- a/apps/web/src/content/docs/docs/graders/custom-assertions.mdx +++ b/apps/web/src/content/docs/docs/graders/custom-assertions.mdx @@ -57,7 +57,7 @@ assertions: ## Pass/Fail Pattern -The simplest pattern returns `pass` (boolean) and `reasoning` (string): +The simplest pattern returns `pass` (boolean) and an optional `assertions` array: ```typescript // .agentv/assertions/word-count.ts @@ -65,9 +65,10 @@ import { defineAssertion } from '@agentv/eval'; export default defineAssertion(({ output }) => { const wordCount = (output ?? '').trim().split(/\s+/).filter(Boolean).length; + const pass = wordCount >= 3; return { - pass: wordCount >= 3, - reasoning: `Output has ${wordCount} words`, + pass, + assertions: [{ text: `Output has ${wordCount} words`, passed: pass }], }; }); ``` @@ -106,7 +107,6 @@ The handler must return an `AssertionScore` object: | `pass` | `boolean` | Explicit pass/fail. If omitted, derived from `score` (>= 0.5 = pass). | | `score` | `number` | Numeric score between 0 and 1. Defaults to 1 if `pass=true`, 0 if `pass=false`. | | `assertions` | `Array<{ text: string, passed: boolean, evidence?: string }>` | Per-aspect results. Each entry describes one check with its verdict and optional evidence. | -| `reasoning` | `string` | Human-readable explanation. | | `details` | `Record` | Optional structured data for domain-specific metrics. | ## Context Available to Assertions @@ -140,8 +140,9 @@ Expected output: ```json { "score": 1, - "assertions": [], - "reasoning": "Output has 6 words (>= 3 required)" + "assertions": [ + { "text": "Output has 6 words", "passed": true } + ] } ``` @@ -201,9 +202,14 @@ export default defineAssertion(({ output }) => { return { pass, score: pass ? 1.0 : Math.min(wordCount / minWords, 0.9), - reasoning: pass - ? `Output has ${wordCount} words (>= ${minWords} required)` - : `Output has only ${wordCount} words (need >= ${minWords})`, + assertions: [ + { + text: pass + ? `Output has ${wordCount} words (>= ${minWords} required)` + : `Output has only ${wordCount} words (need >= ${minWords})`, + passed: pass, + }, + ], }; }); ``` diff --git a/examples/features/eval-assert-demo/README.md b/examples/features/eval-assert-demo/README.md index d93e2dab2..d1478a1a8 100644 --- a/examples/features/eval-assert-demo/README.md +++ b/examples/features/eval-assert-demo/README.md @@ -67,7 +67,7 @@ When running the eval, the transpiler emits natural-language instructions for ea ``` Run `agentv eval assert keyword-check --agent-output --agent-input ` and check the result. This grader: Checks that the answer mentions Paris and France. -The command returns JSON: {"score": 0-1, "reasoning": "..."}. +The command returns JSON: {"score": 0-1, "assertions": [{"text": "...", "passed": true|false}]}. A score >= 0.5 means pass (exit 0); below 0.5 means fail (exit 1). ``` diff --git a/packages/eval/README.md b/packages/eval/README.md index 103fbbec2..2ca5ade84 100644 --- a/packages/eval/README.md +++ b/packages/eval/README.md @@ -16,10 +16,13 @@ npm install @agentv/eval #!/usr/bin/env bun import { defineAssertion } from '@agentv/eval'; -export default defineAssertion(({ answer }) => ({ - pass: answer.toLowerCase().includes('hello'), - reasoning: 'Checks for greeting', -})); +export default defineAssertion(({ output }) => { + const hasGreeting = (output ?? '').toLowerCase().includes('hello'); + return { + pass: hasGreeting, + assertions: [{ text: 'Checks for greeting', passed: hasGreeting }], + }; +}); ``` Assertions support `pass: boolean` for simple checks and `score: number` (0-1) for granular scoring. diff --git a/skills-data/agentv-eval-writer/SKILL.md b/skills-data/agentv-eval-writer/SKILL.md index f33fcff7b..f42aa9089 100644 --- a/skills-data/agentv-eval-writer/SKILL.md +++ b/skills-data/agentv-eval-writer/SKILL.md @@ -356,7 +356,7 @@ Configure via `assertions` array. Multiple graders produce a weighted average sc cwd: ./scripts # optional working directory target: {} # optional: enable LLM target proxy (max_calls: 50) ``` -Contract: stdin JSON -> stdout JSON `{score, assertions: [{text, passed, evidence?}], reasoning}` +Contract: stdin JSON -> stdout JSON `{score, assertions: [{text, passed, evidence?}], details?}` Raw stdin uses snake_case and includes: `criteria`, `input`, `expected_output`, `output` (final answer string), `answer` (deprecated alias), `messages`, `trace`, `trace_summary`, `token_usage`, `cost_usd`, `duration_ms`, `start_time`, `end_time`, `file_changes`, `workspace_path`, `config` SDK handlers receive the same payload in camelCase: `expectedOutput`, `traceSummary`, `tokenUsage`, `costUsd`, `durationMs`, `startTime`, `endTime`, `fileChanges`, `workspacePath`. When a workspace is configured, `workspace_path` is the absolute path to the workspace dir (also available as `AGENTV_WORKSPACE_PATH` env var). Use this for functional grading (e.g., running `npm test` in the workspace). @@ -575,10 +575,13 @@ Use `@agentv/eval` to build custom graders in TypeScript/JavaScript: #!/usr/bin/env bun import { defineAssertion } from '@agentv/eval'; -export default defineAssertion(({ answer, trace }) => ({ - pass: answer.length > 0 && (trace?.eventCount ?? 0) <= 10, - reasoning: 'Checks content exists and is efficient', -})); +export default defineAssertion(({ output, traceSummary }) => { + const pass = (output ?? '').length > 0 && (traceSummary?.eventCount ?? 0) <= 10; + return { + pass, + assertions: [{ text: 'Checks content exists and is efficient', passed: pass }], + }; +}); ``` Assertions support both `pass: boolean` and `score: number` (0-1). If only `pass` is given, score is 1 (pass) or 0 (fail). @@ -588,10 +591,11 @@ Assertions support both `pass: boolean` and `score: number` (0-1). If only `pass #!/usr/bin/env bun import { defineCodeGrader } from '@agentv/eval'; -export default defineCodeGrader(({ trace, answer }) => ({ - score: trace?.eventCount <= 5 ? 1.0 : 0.5, +export default defineCodeGrader(({ output, traceSummary }) => ({ + score: (output ?? '').length > 0 && (traceSummary?.eventCount ?? 0) <= 5 ? 1.0 : 0.5, assertions: [ - { text: 'Efficient tool usage', passed: (trace?.eventCount ?? 0) <= 5 }, + { text: 'Output is not empty', passed: (output ?? '').length > 0 }, + { text: 'Efficient tool usage', passed: (traceSummary?.eventCount ?? 0) <= 5 }, ], })); ``` diff --git a/skills-data/agentv-eval-writer/references/custom-evaluators.md b/skills-data/agentv-eval-writer/references/custom-evaluators.md index 1d5bfda18..f437eb209 100644 --- a/skills-data/agentv-eval-writer/references/custom-evaluators.md +++ b/skills-data/agentv-eval-writer/references/custom-evaluators.md @@ -44,12 +44,11 @@ "assertions": [ { "text": "passed check", "passed": true }, { "text": "failed check", "passed": false } - ], - "reasoning": "explanation" + ] } ``` -`score` (0.0-1.0) required. `assertions`, `reasoning` optional. +`score` (0.0-1.0) required. `assertions` and `details` optional. ## SDK Functions @@ -72,7 +71,7 @@ import { defineCodeGrader, createTargetClient, definePromptTemplate } from '@age import json, sys def evaluate(data: dict) -> dict: - candidate = data.get("answer", "") + candidate = data.get("output") or data.get("answer") or "" assertions = [] for kw in ["async", "await"]: assertions.append({"text": f"Keyword '{kw}'", "passed": kw in candidate}) @@ -97,9 +96,9 @@ if __name__ == "__main__": import { defineCodeGrader } from '@agentv/eval'; export default defineCodeGrader(({ output, criteria }) => { - const answer = output ?? ''; + const candidate = output ?? ''; const assertions: Array<{ text: string; passed: boolean }> = []; - if (answer.includes(criteria)) { + if (candidate.includes(criteria)) { assertions.push({ text: 'Matches expected outcome', passed: true }); } else { assertions.push({ text: 'Does not match expected outcome', passed: false }); From b9eebd0851275526d2010efd7d764be85d0d9d83 Mon Sep 17 00:00:00 2001 From: Christopher Tso Date: Thu, 18 Jun 2026 02:35:19 +0200 Subject: [PATCH 3/3] fix(cli): format assertion scaffold test --- apps/cli/test/commands/create/assertion.test.ts | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/apps/cli/test/commands/create/assertion.test.ts b/apps/cli/test/commands/create/assertion.test.ts index abe2bc790..4a1f6ebe6 100644 --- a/apps/cli/test/commands/create/assertion.test.ts +++ b/apps/cli/test/commands/create/assertion.test.ts @@ -23,9 +23,14 @@ describe('agentv create assertion', () => { expect(result.exitCode).toBe(0); - const content = await readFile(path.join(cwd, '.agentv', 'assertions', 'word-count.ts'), 'utf8'); + const content = await readFile( + path.join(cwd, '.agentv', 'assertions', 'word-count.ts'), + 'utf8', + ); expect(content).toContain("const text = output ?? '';"); - expect(content).toContain('assertions: [{ text: pass ? \'Output has content\' : \'Output is empty\', passed: pass }]'); + expect(content).toContain( + "assertions: [{ text: pass ? 'Output has content' : 'Output is empty', passed: pass }]", + ); expect(content).not.toContain('reasoning:'); expect(content).not.toContain('getMessageText'); } finally {