diff --git a/.changeset/braintrust-empty-output.md b/.changeset/braintrust-empty-output.md new file mode 100644 index 000000000..9006e1f2b --- /dev/null +++ b/.changeset/braintrust-empty-output.md @@ -0,0 +1,5 @@ +--- +"eve": patch +--- + +Prevent Braintrust reporting from stopping an eval run when a case produces no agent output. diff --git a/packages/eve/src/evals/runner/reporters/braintrust.test.ts b/packages/eve/src/evals/runner/reporters/braintrust.test.ts index a70421659..0dcf6ba9a 100644 --- a/packages/eve/src/evals/runner/reporters/braintrust.test.ts +++ b/packages/eve/src/evals/runner/reporters/braintrust.test.ts @@ -199,4 +199,32 @@ describe("Braintrust", () => { }), ); }); + + it("logs evals that produce no agent output", async () => { + const reporter = Braintrust(makeConfig()); + await reporter.onRunStart([makeEval()], makeTarget()); + + braintrustMocks.log.mockImplementationOnce((event: { output?: unknown }) => { + if (event.output === null || event.output === undefined) { + throw new Error("output must be specified"); + } + }); + + const result = makeEvalResult(); + const resultWithoutOutput: EveEvalResult = { + ...result, + result: { + ...result.result, + output: null, + finalMessage: null, + }, + }; + + expect(() => reporter.onEvalComplete(resultWithoutOutput)).not.toThrow(); + expect(braintrustMocks.log).toHaveBeenCalledWith( + expect.objectContaining({ + output: "", + }), + ); + }); }); diff --git a/packages/eve/src/evals/runner/reporters/braintrust.ts b/packages/eve/src/evals/runner/reporters/braintrust.ts index 2b611cde6..9b46ab6bd 100644 --- a/packages/eve/src/evals/runner/reporters/braintrust.ts +++ b/packages/eve/src/evals/runner/reporters/braintrust.ts @@ -175,7 +175,7 @@ class BraintrustReporter implements EvalReporter { this.#experiment.log({ id: result.id, input: evaluation?.description ?? "", - output: result.result.output, + output: result.result.output ?? "", error: result.error ?? undefined, scores, metadata,