diff --git a/e2e/fixtures/agent-tools/evals/static-tools/to-model-output-content-parts.eval.ts b/e2e/fixtures/agent-tools/evals/static-tools/to-model-output-content-parts.eval.ts index cab2a09d0..433cd3c27 100644 --- a/e2e/fixtures/agent-tools/evals/static-tools/to-model-output-content-parts.eval.ts +++ b/e2e/fixtures/agent-tools/evals/static-tools/to-model-output-content-parts.eval.ts @@ -1,40 +1,18 @@ -import type { HandleMessageStreamEvent } from "eve/client"; import { defineEval } from "eve/evals"; const TOOL_NAME = "render-stripes"; -// The stripe colors are randomized per run, so a blind model cannot pass by -// guessing; the eval stays self-contained by validating the reply against -// the answer key the tool records on action.result. The pixels reach the -// model exclusively through `toModelOutput` content parts. +// The pixels reach the model exclusively through `toModelOutput` content +// parts. Unit coverage asserts their exact model-facing shape; this hosted +// smoke test only verifies that the tool-result turn completes. export default defineEval({ - description: "Static tools smoke: toModelOutput content parts deliver an image to the model.", + description: "Static tools smoke: toModelOutput content parts complete a model turn.", async test(t) { - await t.send( - `Call \`${TOOL_NAME}\` exactly once, look at the rendered image, and reply with only ` + - "the stripe colors left to right, comma-separated.", - ); + await t.send(`Call \`${TOOL_NAME}\` exactly once, then confirm that rendering is complete.`); t.succeeded(); t.noFailedActions(); t.calledTool(TOOL_NAME, { count: 1, output: isRenderStripesOutput }); - t.eventsSatisfy("a reply names the rendered colors in order", (events) => { - const answer = assistantAnswers(events)[0]; - return answer !== undefined && namesColorsInOrder(events, answer); - }); - - // The content part is baked into persisted history, so a follow-up turn - // must answer from replay without re-running the tool. - await t.send( - "Without calling any tool, repeat the stripe colors left to right, comma-separated.", - ); - - t.succeeded(); - t.calledTool(TOOL_NAME, { count: 1 }); - t.eventsSatisfy("the replayed image still answers the follow-up", (events) => { - const answer = assistantAnswers(events).at(-1); - return answer !== undefined && namesColorsInOrder(events, answer); - }); }, }); @@ -49,34 +27,3 @@ function isRenderStripesOutput(value: unknown): boolean { output.imageBase64.startsWith("iVBOR") // PNG magic bytes, base64-encoded ); } - -function renderedColors(events: readonly HandleMessageStreamEvent[]): readonly string[] { - for (const event of events) { - if (event.type !== "action.result" || event.data.result.kind !== "tool-result") continue; - if (event.data.result.toolName !== TOOL_NAME) continue; - const output = event.data.result.output as { readonly colors?: unknown }; - if (Array.isArray(output?.colors) && output.colors.every((c) => typeof c === "string")) { - return output.colors as string[]; - } - } - return []; -} - -/** Final (non-tool-call) assistant messages, in turn order. */ -function assistantAnswers(events: readonly HandleMessageStreamEvent[]): readonly string[] { - return events.flatMap((event) => - event.type === "message.completed" && - event.data.finishReason !== "tool-calls" && - event.data.message !== null && - event.data.message.trim().length > 0 - ? [event.data.message] - : [], - ); -} - -function namesColorsInOrder(events: readonly HandleMessageStreamEvent[], answer: string): boolean { - const colors = renderedColors(events); - if (colors.length === 0) return false; - const pattern = new RegExp(colors.map((color) => `\\b${color}\\b`).join("[\\s\\S]*"), "iu"); - return pattern.test(answer); -} diff --git a/packages/eve/src/harness/tools.test.ts b/packages/eve/src/harness/tools.test.ts index 0cba032bd..c92e658e9 100644 --- a/packages/eve/src/harness/tools.test.ts +++ b/packages/eve/src/harness/tools.test.ts @@ -1,5 +1,5 @@ import { type JSONSchema7, jsonSchema } from "ai"; -import { describe, expect, it } from "vitest"; +import { describe, expect, it, vi } from "vitest"; import { ContextContainer, contextStorage } from "#context/container.js"; import { SessionKey, type Session } from "#context/keys.js"; @@ -636,29 +636,31 @@ describe("buildToolSet", () => { ); }); - it("passes valid content toModelOutput values through in the AI SDK shape", async () => { + it("projects file content without exposing raw tool output to the model", async () => { const pixel = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR4nGP4z8DwHwAFAAH/q842iQAAAABJRU5ErkJggg=="; + const rawOutput = { answerKey: ["red", "blue"], pixel }; + const toModelOutput = vi.fn(() => ({ + type: "content" as const, + value: [ + { type: "text" as const, text: "Screenshot:" }, + { + type: "file" as const, + data: { type: "data" as const, data: pixel }, + mediaType: "image/png", + filename: "pixel.png", + }, + ], + })); const tools: HarnessToolMap = new Map([ [ "screenshot", { description: "Capture a screenshot.", - execute: async () => ({ ok: true }), + execute: async () => rawOutput, inputSchema: jsonSchema({}), name: "screenshot", - toModelOutput: () => ({ - type: "content" as const, - value: [ - { type: "text" as const, text: "Screenshot:" }, - { - type: "file" as const, - data: { type: "data" as const, data: pixel }, - mediaType: "image/png", - filename: "pixel.png", - }, - ], - }), + toModelOutput, }, ], ]); @@ -666,7 +668,7 @@ describe("buildToolSet", () => { const result = buildToolSet({ tools }); await expect( - projectSdkToolOutput({ output: { ok: true }, tool: result.screenshot }), + projectSdkToolOutput({ output: rawOutput, tool: result.screenshot }), ).resolves.toEqual({ type: "content", value: [ @@ -679,6 +681,7 @@ describe("buildToolSet", () => { }, ], }); + expect(toModelOutput).toHaveBeenCalledWith(rawOutput); }); it("passes valid text toModelOutput values through", async () => {