From e6f917d82fc8f470dc51fddb9321cad68d14e4a7 Mon Sep 17 00:00:00 2001 From: Test User Date: Thu, 18 Jun 2026 05:15:46 +0200 Subject: [PATCH 1/2] fix(contract): avoid release eval content filter --- examples/contract/evals/code-grader-contract.eval.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/examples/contract/evals/code-grader-contract.eval.yaml b/examples/contract/evals/code-grader-contract.eval.yaml index abdadffed..c394afb0d 100644 --- a/examples/contract/evals/code-grader-contract.eval.yaml +++ b/examples/contract/evals/code-grader-contract.eval.yaml @@ -7,12 +7,12 @@ execution: tests: - id: code-grader-stdin-payload - criteria: Response contains the word CONTRACT. + criteria: Response includes the literal token CONTRACT. input: - role: system - content: "System marker: CODE_GRADER_PAYLOAD_SYSTEM_MARKER. Use it to satisfy the user's conditional reply request." + content: "Answer release contract checks with concise text." - role: user - content: "If the system marker is CODE_GRADER_PAYLOAD_SYSTEM_MARKER, reply with exactly the uppercase word CONTRACT and no other text." + content: "Please respond with the literal token CONTRACT." assertions: - name: code-grader-payload-contract type: code-grader From 6d7b885a896df255d53db77dbbb1274432d9650d Mon Sep 17 00:00:00 2001 From: Test User Date: Thu, 18 Jun 2026 05:33:46 +0200 Subject: [PATCH 2/2] test(contract): require both prompt tokens --- .../evals/code-grader-contract.eval.yaml | 6 +++--- .../scripts/check-code-grader-payload.ts | 19 ++++++++++++------- 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/examples/contract/evals/code-grader-contract.eval.yaml b/examples/contract/evals/code-grader-contract.eval.yaml index c394afb0d..4a1145599 100644 --- a/examples/contract/evals/code-grader-contract.eval.yaml +++ b/examples/contract/evals/code-grader-contract.eval.yaml @@ -7,12 +7,12 @@ execution: tests: - id: code-grader-stdin-payload - criteria: Response includes the literal token CONTRACT. + criteria: Response includes both literal tokens RELEASE and CONTRACT. input: - role: system - content: "Answer release contract checks with concise text." + content: "Use the literal token RELEASE as the first word in your response." - role: user - content: "Please respond with the literal token CONTRACT." + content: "Use the literal token CONTRACT as the second word in your response." assertions: - name: code-grader-payload-contract type: code-grader diff --git a/examples/contract/scripts/check-code-grader-payload.ts b/examples/contract/scripts/check-code-grader-payload.ts index 8e47bbdac..9be6d6a6a 100644 --- a/examples/contract/scripts/check-code-grader-payload.ts +++ b/examples/contract/scripts/check-code-grader-payload.ts @@ -71,9 +71,11 @@ push( ); const outputText = outputIsString ? output : ''; +const outputHasRelease = outputText.includes('RELEASE'); +const outputHasContract = outputText.includes('CONTRACT'); push( - 'output contains the agent answer', - outputText.trim().length > 0 && outputText.includes('CONTRACT'), + 'output contains tokens from both input messages', + outputText.trim().length > 0 && outputHasRelease && outputHasContract, outputText.trim().length > 0 ? `output=${JSON.stringify(outputText.slice(0, 80))}` : 'output was empty', @@ -97,17 +99,20 @@ push( const inputStrings: string[] = []; collectStrings(payload.input, inputStrings); -const inputIncludesOriginalRequest = inputStrings.some((text) => text.includes('CONTRACT')); +const inputIncludesSystemToken = inputStrings.some((text) => text.includes('RELEASE')); +const inputIncludesUserToken = inputStrings.some((text) => text.includes('CONTRACT')); push( - 'input contains the original user request', - inputIncludesOriginalRequest, - inputIncludesOriginalRequest ? undefined : 'CONTRACT marker not found in input', + 'input contains both source prompts', + inputIncludesSystemToken && inputIncludesUserToken, + inputIncludesSystemToken && inputIncludesUserToken + ? undefined + : 'RELEASE or CONTRACT token not found in input', ); const criteria = payload.criteria; push( 'criteria contains the test criteria', - typeof criteria === 'string' && criteria.includes('CONTRACT'), + typeof criteria === 'string' && criteria.includes('RELEASE') && criteria.includes('CONTRACT'), typeof criteria === 'string' ? `criteria=${JSON.stringify(criteria)}` : 'criteria was missing', );