diff --git a/e2e/fixtures/agent-tools/evals/static-tools/to-model-output-content-parts.eval.ts b/e2e/fixtures/agent-tools/evals/static-tools/to-model-output-content-parts.eval.ts index dcf53e558..77f54483c 100644 --- a/e2e/fixtures/agent-tools/evals/static-tools/to-model-output-content-parts.eval.ts +++ b/e2e/fixtures/agent-tools/evals/static-tools/to-model-output-content-parts.eval.ts @@ -6,7 +6,8 @@ const TOOL_NAME = "render-stripes"; // The stripe colors are randomized per run, so a blind model cannot pass by // guessing; the eval stays self-contained by validating the reply against // the answer key the tool records on action.result. The pixels reach the -// model exclusively through `toModelOutput` content parts. +// model exclusively through `toModelOutput` content parts. Color recognition +// remains tracked rather than gated because live vision quality varies. export default defineEval({ description: "Static tools smoke: toModelOutput content parts deliver an image to the model.", async test(t) { @@ -21,7 +22,7 @@ export default defineEval({ t.eventsSatisfy("a reply names the rendered colors in order", (events) => { const answer = assistantAnswers(events)[0]; return answer !== undefined && namesColorsInOrder(events, answer); - }); + }).soft(); // The content part is baked into persisted history, so a follow-up turn // must answer from replay without re-running the tool. @@ -34,7 +35,7 @@ export default defineEval({ t.eventsSatisfy("the replayed image still answers the follow-up", (events) => { const answer = assistantAnswers(events).at(-1); return answer !== undefined && namesColorsInOrder(events, answer); - }); + }).soft(); }, });