From ce73d8cde614c1c85c5c023719953ce76f3efd8d Mon Sep 17 00:00:00 2001 From: jsonbailey Date: Thu, 14 May 2026 13:12:48 -0500 Subject: [PATCH] fix: Make judge runners non-multi-turn Add a multiTurn parameter (default true) on provider model runners and RunnerFactory.createModel. When false, the runner does not persist the user prompt and assistant reply back into its conversation history, so each run() call starts fresh from the seeded config messages. Judges now construct their underlying runner with multiTurn=false so successive evaluate() calls on a shared Judge instance do not see each other's prompts and responses. Without this, every evaluation after the first contaminated the judge model's input with prior conversations and concurrent evaluations raced on the mutable history. Also fix Judge.evaluateMessages to render messages as ": " joined by newlines, preserving speaker identity in the message history section the judge model receives. Co-Authored-By: Claude Opus 4.7 --- .../__tests__/LangChainModelRunner.test.ts | 64 +++++++++++++ .../src/LangChainModelRunner.ts | 27 ++++-- .../src/LangChainRunnerFactory.ts | 12 ++- .../__tests__/OpenAIModelRunner.test.ts | 85 +++++++++++++++++ .../server-ai-openai/src/OpenAIModelRunner.ts | 27 ++++-- .../src/OpenAIRunnerFactory.ts | 12 ++- .../__tests__/VercelModelRunner.test.ts | 92 +++++++++++++++++++ .../server-ai-vercel/src/VercelModelRunner.ts | 20 ++-- .../src/VercelRunnerFactory.ts | 12 ++- .../sdk/server-ai/__tests__/Judge.test.ts | 72 ++++++++++++++- .../__tests__/LDAIClientImpl.test.ts | 14 ++- .../server-ai/__tests__/RunnerFactory.test.ts | 37 +++++++- packages/sdk/server-ai/src/LDAIClientImpl.ts | 7 +- packages/sdk/server-ai/src/api/judge/Judge.ts | 9 +- .../server-ai/src/api/providers/AIProvider.ts | 9 +- .../src/api/providers/RunnerFactory.ts | 7 +- 16 files changed, 469 insertions(+), 37 deletions(-) diff --git a/packages/ai-providers/server-ai-langchain/__tests__/LangChainModelRunner.test.ts b/packages/ai-providers/server-ai-langchain/__tests__/LangChainModelRunner.test.ts index b48df2046f..2ae5a3e018 100644 --- a/packages/ai-providers/server-ai-langchain/__tests__/LangChainModelRunner.test.ts +++ b/packages/ai-providers/server-ai-langchain/__tests__/LangChainModelRunner.test.ts @@ -180,4 +180,68 @@ describe('LangChainModelRunner', () => { expect(secondCallMessages[3].content).toBe('Q2'); }); }); + + describe('multiTurn=false (stateless)', () => { + const configWithMessages: LDAICompletionConfig = { + ...baseConfig, + messages: [{ role: 'system', content: 'You are a judge.' }], + }; + + it('does not accumulate history across successful calls', async () => { + const statelessRunner = new LangChainModelRunner( + mockLLM, + configWithMessages, + mockLogger, + false, + ); + + mockLLM.invoke + .mockResolvedValueOnce(new AIMessage('First response')) + .mockResolvedValueOnce(new AIMessage('Second response')); + + await statelessRunner.run('First question'); + await statelessRunner.run('Second question'); + + const firstCallMessages = mockLLM.invoke.mock.calls[0][0]; + const secondCallMessages = mockLLM.invoke.mock.calls[1][0]; + expect(firstCallMessages).toHaveLength(2); + expect(firstCallMessages[0].content).toBe('You are a judge.'); + expect(firstCallMessages[1].content).toBe('First question'); + expect(secondCallMessages).toHaveLength(2); + expect(secondCallMessages[0].content).toBe('You are a judge.'); + expect(secondCallMessages[1].content).toBe('Second question'); + }); + + it('keeps the internal chat history length pinned to the seeded config messages', async () => { + const statelessRunner = new LangChainModelRunner( + mockLLM, + configWithMessages, + mockLogger, + false, + ); + + mockLLM.invoke.mockResolvedValue(new AIMessage('response')); + + await statelessRunner.run('Q1'); + await statelessRunner.run('Q2'); + + // eslint-disable-next-line no-underscore-dangle + const messages = await (statelessRunner as any)._chatHistory.getMessages(); + expect(messages).toHaveLength(1); + }); + + it('defaults to multiTurn=true when the parameter is omitted', async () => { + const defaultRunner = new LangChainModelRunner(mockLLM, configWithMessages, mockLogger); + + mockLLM.invoke + .mockResolvedValueOnce(new AIMessage('Answer 1')) + .mockResolvedValueOnce(new AIMessage('Answer 2')); + + await defaultRunner.run('Q1'); + await defaultRunner.run('Q2'); + + const secondCallMessages = mockLLM.invoke.mock.calls[1][0]; + expect(secondCallMessages).toHaveLength(4); + }); + }); }); diff --git a/packages/ai-providers/server-ai-langchain/src/LangChainModelRunner.ts b/packages/ai-providers/server-ai-langchain/src/LangChainModelRunner.ts index 57f2e0d4f2..5f86191f81 100644 --- a/packages/ai-providers/server-ai-langchain/src/LangChainModelRunner.ts +++ b/packages/ai-providers/server-ai-langchain/src/LangChainModelRunner.ts @@ -20,13 +20,20 @@ import { convertMessagesToLangChain, getAIMetricsFromResponse } from './LangChai export class LangChainModelRunner implements Runner { private _llm: BaseChatModel; private _chatHistory: InMemoryChatMessageHistory; + private _multiTurn: boolean; private _logger?: LDLogger; - constructor(llm: BaseChatModel, config: LDAICompletionConfig, logger?: LDLogger) { + constructor( + llm: BaseChatModel, + config: LDAICompletionConfig, + logger?: LDLogger, + multiTurn: boolean = true, + ) { this._llm = llm; this._chatHistory = new InMemoryChatMessageHistory( convertMessagesToLangChain(config.messages ?? []), ); + this._multiTurn = multiTurn; this._logger = logger; } @@ -34,12 +41,16 @@ export class LangChainModelRunner implements Runner { * Run the LangChain model with the given user prompt. * * The runner maintains a LangChain `InMemoryChatMessageHistory` that is - * initialized from any messages on the AI config (system prompt, etc.) and - * grows with each successful call. On every invocation the user prompt is - * appended to the existing history before being sent to the model. When the - * call succeeds and produces non-empty content, the user prompt and the - * assistant's reply are persisted to the history; failed calls leave the - * history unchanged so the next call can retry cleanly. + * initialized from any messages on the AI config (system prompt, etc.). On + * every invocation the user prompt is appended to the existing history + * before being sent to the model. When `multiTurn` is `true` (the default) + * and the call succeeds with non-empty content, the user prompt and the + * assistant's reply are persisted to the history so subsequent calls + * continue the conversation. When `multiTurn` is `false`, history is + * treated as read-only — useful for stateless runners (e.g. judges) where + * every call should see only the initial config messages plus the current + * input. Failed calls leave the history unchanged so the next call can + * retry cleanly. * * @param input The user prompt string. * @param outputType Optional JSON schema for structured output. When provided, @@ -56,7 +67,7 @@ export class LangChainModelRunner implements Runner { ? await this._runStructured(langchainMessages, outputType) : await this._runCompletion(langchainMessages); - if (result.metrics.success && result.content) { + if (result.metrics.success && result.content && this._multiTurn) { await this._chatHistory.addUserMessage(input); await this._chatHistory.addAIMessage(result.content); } diff --git a/packages/ai-providers/server-ai-langchain/src/LangChainRunnerFactory.ts b/packages/ai-providers/server-ai-langchain/src/LangChainRunnerFactory.ts index f00cfb0e14..69a0144e2b 100644 --- a/packages/ai-providers/server-ai-langchain/src/LangChainRunnerFactory.ts +++ b/packages/ai-providers/server-ai-langchain/src/LangChainRunnerFactory.ts @@ -26,10 +26,18 @@ export class LangChainRunnerFactory extends AIProvider { /** * Create a model runner from a completion AI configuration. + * + * @param config The completion (or judge) AI configuration. + * @param multiTurn Whether the runner should accumulate conversation history + * across successive `run()` calls. Defaults to `true` (chat semantics). + * Pass `false` for stateless runners such as judges. */ - async createModel(config: LDAICompletionConfig): Promise { + async createModel( + config: LDAICompletionConfig, + multiTurn: boolean = true, + ): Promise { const llm = await createLangChainModel(config); - return new LangChainModelRunner(llm, config, this._logger); + return new LangChainModelRunner(llm, config, this._logger, multiTurn); } /** diff --git a/packages/ai-providers/server-ai-openai/__tests__/OpenAIModelRunner.test.ts b/packages/ai-providers/server-ai-openai/__tests__/OpenAIModelRunner.test.ts index b4b44c9cb1..f0b20be095 100644 --- a/packages/ai-providers/server-ai-openai/__tests__/OpenAIModelRunner.test.ts +++ b/packages/ai-providers/server-ai-openai/__tests__/OpenAIModelRunner.test.ts @@ -231,4 +231,89 @@ describe('OpenAIModelRunner', () => { ]); }); }); + + describe('multiTurn=false (stateless)', () => { + const configWithMessages: LDAICompletionConfig = { + ...baseConfig, + messages: [{ role: 'system', content: 'You are a judge.' }], + }; + + it('does not accumulate history across successful calls', async () => { + const statelessRunner = new OpenAIModelRunner( + mockOpenAI, + configWithMessages, + undefined, + false, + ); + + (mockOpenAI.chat.completions.create as jest.Mock) + .mockResolvedValueOnce({ + choices: [{ message: { content: 'First response' } }], + usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 }, + } as any) + .mockResolvedValueOnce({ + choices: [{ message: { content: 'Second response' } }], + usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 }, + } as any); + + await statelessRunner.run('First question'); + await statelessRunner.run('Second question'); + + const firstCallArgs = (mockOpenAI.chat.completions.create as jest.Mock).mock.calls[0][0]; + const secondCallArgs = (mockOpenAI.chat.completions.create as jest.Mock).mock.calls[1][0]; + expect(firstCallArgs.messages).toEqual([ + { role: 'system', content: 'You are a judge.' }, + { role: 'user', content: 'First question' }, + ]); + expect(secondCallArgs.messages).toEqual([ + { role: 'system', content: 'You are a judge.' }, + { role: 'user', content: 'Second question' }, + ]); + }); + + it('keeps the internal history length pinned to the seeded config messages', async () => { + const statelessRunner = new OpenAIModelRunner( + mockOpenAI, + configWithMessages, + undefined, + false, + ); + + (mockOpenAI.chat.completions.create as jest.Mock).mockResolvedValue({ + choices: [{ message: { content: 'response' } }], + usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 }, + } as any); + + await statelessRunner.run('Q1'); + await statelessRunner.run('Q2'); + + // eslint-disable-next-line no-underscore-dangle + expect((statelessRunner as any)._history).toHaveLength(1); + }); + + it('defaults to multiTurn=true when the parameter is omitted', async () => { + const defaultRunner = new OpenAIModelRunner(mockOpenAI, configWithMessages); + + (mockOpenAI.chat.completions.create as jest.Mock) + .mockResolvedValueOnce({ + choices: [{ message: { content: 'Answer 1' } }], + usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 }, + } as any) + .mockResolvedValueOnce({ + choices: [{ message: { content: 'Answer 2' } }], + usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 }, + } as any); + + await defaultRunner.run('Q1'); + await defaultRunner.run('Q2'); + + const secondCallArgs = (mockOpenAI.chat.completions.create as jest.Mock).mock.calls[1][0]; + expect(secondCallArgs.messages).toEqual([ + { role: 'system', content: 'You are a judge.' }, + { role: 'user', content: 'Q1' }, + { role: 'assistant', content: 'Answer 1' }, + { role: 'user', content: 'Q2' }, + ]); + }); + }); }); diff --git a/packages/ai-providers/server-ai-openai/src/OpenAIModelRunner.ts b/packages/ai-providers/server-ai-openai/src/OpenAIModelRunner.ts index 62882455d5..e87b83d52e 100644 --- a/packages/ai-providers/server-ai-openai/src/OpenAIModelRunner.ts +++ b/packages/ai-providers/server-ai-openai/src/OpenAIModelRunner.ts @@ -21,13 +21,20 @@ export class OpenAIModelRunner implements Runner { private _modelName: string; private _parameters: Record; private _history: LDMessage[]; + private _multiTurn: boolean; private _logger?: LDLogger; - constructor(client: OpenAI, config: LDAICompletionConfig, logger?: LDLogger) { + constructor( + client: OpenAI, + config: LDAICompletionConfig, + logger?: LDLogger, + multiTurn: boolean = true, + ) { this._client = client; this._modelName = config.model?.name ?? ''; this._parameters = { ...(config.model?.parameters ?? {}) }; this._history = [...(config.messages ?? [])]; + this._multiTurn = multiTurn; this._logger = logger; } @@ -35,12 +42,16 @@ export class OpenAIModelRunner implements Runner { * Run the OpenAI model with the given user prompt. * * The runner maintains a conversation history that is initialized from any - * messages on the AI config (system prompt, instructions, etc.) and grows - * with each successful call. On every invocation the user prompt is appended - * to the existing history before being sent to the model. When the call - * succeeds and produces non-empty content, the user prompt and the - * assistant's reply are persisted to the history; failed calls leave the - * history unchanged so the next call can retry cleanly. + * messages on the AI config (system prompt, instructions, etc.). On every + * invocation the user prompt is appended to the existing history before + * being sent to the model. When `multiTurn` is `true` (the default) and the + * call succeeds with non-empty content, the user prompt and the assistant's + * reply are persisted to the history so subsequent calls continue the + * conversation. When `multiTurn` is `false`, history is treated as + * read-only — useful for stateless runners (e.g. judges) where every call + * should see only the initial config messages plus the current input. + * Failed calls leave the history unchanged so the next call can retry + * cleanly. * * @param input The user prompt string. * @param outputType Optional JSON schema for structured output. When provided, @@ -55,7 +66,7 @@ export class OpenAIModelRunner implements Runner { ? await this._runStructured(messages, outputType) : await this._runCompletion(messages); - if (result.metrics.success && result.content) { + if (result.metrics.success && result.content && this._multiTurn) { this._history.push(userMessage); this._history.push({ role: 'assistant', content: result.content }); } diff --git a/packages/ai-providers/server-ai-openai/src/OpenAIRunnerFactory.ts b/packages/ai-providers/server-ai-openai/src/OpenAIRunnerFactory.ts index 9f99f7d798..13511653a0 100644 --- a/packages/ai-providers/server-ai-openai/src/OpenAIRunnerFactory.ts +++ b/packages/ai-providers/server-ai-openai/src/OpenAIRunnerFactory.ts @@ -28,9 +28,17 @@ export class OpenAIRunnerFactory extends AIProvider { /** * Create a model runner from a completion AI configuration. + * + * @param config The completion (or judge) AI configuration. + * @param multiTurn Whether the runner should accumulate conversation history + * across successive `run()` calls. Defaults to `true` (chat semantics). + * Pass `false` for stateless runners such as judges. */ - async createModel(config: LDAICompletionConfig): Promise { - return new OpenAIModelRunner(this._client, config, this._logger); + async createModel( + config: LDAICompletionConfig, + multiTurn: boolean = true, + ): Promise { + return new OpenAIModelRunner(this._client, config, this._logger, multiTurn); } /** diff --git a/packages/ai-providers/server-ai-vercel/__tests__/VercelModelRunner.test.ts b/packages/ai-providers/server-ai-vercel/__tests__/VercelModelRunner.test.ts index f781ec9eff..d78d28edc6 100644 --- a/packages/ai-providers/server-ai-vercel/__tests__/VercelModelRunner.test.ts +++ b/packages/ai-providers/server-ai-vercel/__tests__/VercelModelRunner.test.ts @@ -231,4 +231,96 @@ describe('VercelModelRunner', () => { ]); }); }); + + describe('multiTurn=false (stateless)', () => { + const configWithMessages: LDAICompletionConfig = { + ...baseConfig, + messages: [{ role: 'system', content: 'You are a judge.' }], + }; + + it('does not accumulate history across successful calls', async () => { + const statelessRunner = new VercelModelRunner( + fakeModel as any, + configWithMessages, + {}, + mockLogger, + false, + ); + + (generateText as jest.Mock) + .mockResolvedValueOnce({ + text: 'First response', + usage: { totalTokens: 2, promptTokens: 1, completionTokens: 1 }, + }) + .mockResolvedValueOnce({ + text: 'Second response', + usage: { totalTokens: 2, promptTokens: 1, completionTokens: 1 }, + }); + + await statelessRunner.run('First question'); + await statelessRunner.run('Second question'); + + const firstCallArgs = (generateText as jest.Mock).mock.calls[0][0]; + const secondCallArgs = (generateText as jest.Mock).mock.calls[1][0]; + expect(firstCallArgs.messages).toEqual([ + { role: 'system', content: 'You are a judge.' }, + { role: 'user', content: 'First question' }, + ]); + expect(secondCallArgs.messages).toEqual([ + { role: 'system', content: 'You are a judge.' }, + { role: 'user', content: 'Second question' }, + ]); + }); + + it('keeps the internal history length pinned to the seeded config messages', async () => { + const statelessRunner = new VercelModelRunner( + fakeModel as any, + configWithMessages, + {}, + mockLogger, + false, + ); + + (generateText as jest.Mock).mockResolvedValue({ + text: 'response', + usage: { totalTokens: 2, promptTokens: 1, completionTokens: 1 }, + }); + + await statelessRunner.run('Q1'); + await statelessRunner.run('Q2'); + + // eslint-disable-next-line no-underscore-dangle + expect((statelessRunner as any)._history).toHaveLength(1); + }); + + it('defaults to multiTurn=true when the parameter is omitted', async () => { + const defaultRunner = new VercelModelRunner( + fakeModel as any, + configWithMessages, + {}, + mockLogger, + ); + + (generateText as jest.Mock) + .mockResolvedValueOnce({ + text: 'Answer 1', + usage: { totalTokens: 2, promptTokens: 1, completionTokens: 1 }, + }) + .mockResolvedValueOnce({ + text: 'Answer 2', + usage: { totalTokens: 2, promptTokens: 1, completionTokens: 1 }, + }); + + await defaultRunner.run('Q1'); + await defaultRunner.run('Q2'); + + const secondCallArgs = (generateText as jest.Mock).mock.calls[1][0]; + expect(secondCallArgs.messages).toEqual([ + { role: 'system', content: 'You are a judge.' }, + { role: 'user', content: 'Q1' }, + { role: 'assistant', content: 'Answer 1' }, + { role: 'user', content: 'Q2' }, + ]); + }); + }); }); diff --git a/packages/ai-providers/server-ai-vercel/src/VercelModelRunner.ts b/packages/ai-providers/server-ai-vercel/src/VercelModelRunner.ts index 2c842f324b..ccadc3d6ba 100644 --- a/packages/ai-providers/server-ai-vercel/src/VercelModelRunner.ts +++ b/packages/ai-providers/server-ai-vercel/src/VercelModelRunner.ts @@ -20,6 +20,7 @@ export class VercelModelRunner implements Runner { private _model: LanguageModel; private _parameters: VercelAIModelParameters; private _history: ModelMessage[]; + private _multiTurn: boolean; private _logger?: LDLogger; constructor( @@ -27,10 +28,12 @@ export class VercelModelRunner implements Runner { config: LDAICompletionConfig, parameters: VercelAIModelParameters, logger?: LDLogger, + multiTurn: boolean = true, ) { this._model = model; this._parameters = parameters; this._history = convertMessagesToVercel(config.messages ?? []) as ModelMessage[]; + this._multiTurn = multiTurn; this._logger = logger; } @@ -39,12 +42,15 @@ export class VercelModelRunner implements Runner { * * The runner maintains a conversation history (as Vercel AI SDK * `ModelMessage`s) that is initialized from any messages on the AI config - * (system prompt, etc.) and grows with each successful call. On every - * invocation the user prompt is appended to the existing history before - * being sent to the model. When the call succeeds and produces non-empty - * content, the user prompt and the assistant's reply are persisted to the - * history; failed calls leave the history unchanged so the next call can - * retry cleanly. + * (system prompt, etc.). On every invocation the user prompt is appended to + * the existing history before being sent to the model. When `multiTurn` is + * `true` (the default) and the call succeeds with non-empty content, the + * user prompt and the assistant's reply are persisted to the history so + * subsequent calls continue the conversation. When `multiTurn` is `false`, + * history is treated as read-only — useful for stateless runners (e.g. + * judges) where every call should see only the initial config messages + * plus the current input. Failed calls leave the history unchanged so the + * next call can retry cleanly. * * @param input The user prompt string. * @param outputType Optional JSON schema for structured output. When provided, @@ -59,7 +65,7 @@ export class VercelModelRunner implements Runner { ? await this._runStructured(messages, outputType) : await this._runCompletion(messages); - if (result.metrics.success && result.content) { + if (result.metrics.success && result.content && this._multiTurn) { this._history.push(userMessage); this._history.push({ role: 'assistant', content: result.content }); } diff --git a/packages/ai-providers/server-ai-vercel/src/VercelRunnerFactory.ts b/packages/ai-providers/server-ai-vercel/src/VercelRunnerFactory.ts index 9869301314..083d164387 100644 --- a/packages/ai-providers/server-ai-vercel/src/VercelRunnerFactory.ts +++ b/packages/ai-providers/server-ai-vercel/src/VercelRunnerFactory.ts @@ -21,11 +21,19 @@ export class VercelRunnerFactory extends AIProvider { /** * Create a model runner from a completion AI configuration. + * + * @param config The completion (or judge) AI configuration. + * @param multiTurn Whether the runner should accumulate conversation history + * across successive `run()` calls. Defaults to `true` (chat semantics). + * Pass `false` for stateless runners such as judges. */ - async createModel(config: LDAICompletionConfig): Promise { + async createModel( + config: LDAICompletionConfig, + multiTurn: boolean = true, + ): Promise { const model = await VercelRunnerFactory.createVercelModel(config); const parameters = VercelRunnerFactory.mapParameters(config.model?.parameters); - return new VercelModelRunner(model, config, parameters, this._logger); + return new VercelModelRunner(model, config, parameters, this._logger, multiTurn); } /** diff --git a/packages/sdk/server-ai/__tests__/Judge.test.ts b/packages/sdk/server-ai/__tests__/Judge.test.ts index 1b05737f05..75a1d33a89 100644 --- a/packages/sdk/server-ai/__tests__/Judge.test.ts +++ b/packages/sdk/server-ai/__tests__/Judge.test.ts @@ -553,11 +553,81 @@ describe('Judge', () => { }); expect(mockRunner.run).toHaveBeenCalledWith( - 'MESSAGE HISTORY:\nWhat is the capital of France?\r\nParis is the capital of France.\n\nRESPONSE TO EVALUATE:\nParis is the capital of France.', + 'MESSAGE HISTORY:\nuser: What is the capital of France?\nassistant: Paris is the capital of France.\n\nRESPONSE TO EVALUATE:\nParis is the capital of France.', expect.any(Object), // evaluation schema ); }); + it('renders each message as ": " joined by newlines', async () => { + const messages: LDMessage[] = [ + { role: 'user', content: 'hi' }, + { role: 'assistant', content: 'hello' }, + ]; + const response: RunnerResult = { + content: 'hello', + metrics: { success: true }, + }; + + const mockRunnerResult: RunnerResult = { + content: '', + parsed: { score: 0.5, reasoning: 'ok' }, + metrics: { success: true }, + }; + mockTracker.trackMetricsOf.mockImplementation(async (extractor, func) => func()); + mockRunner.run.mockResolvedValue(mockRunnerResult); + + await judge.evaluateMessages(messages, response); + + const inputArg = mockRunner.run.mock.calls[0][0] as string; + expect(inputArg).toContain('MESSAGE HISTORY:\nuser: hi\nassistant: hello'); + }); + + it('produces an empty MESSAGE HISTORY section when messages is empty', async () => { + const response: RunnerResult = { + content: 'output', + metrics: { success: true }, + }; + + const mockRunnerResult: RunnerResult = { + content: '', + parsed: { score: 0.5, reasoning: 'ok' }, + metrics: { success: true }, + }; + mockTracker.trackMetricsOf.mockImplementation(async (extractor, func) => func()); + mockRunner.run.mockResolvedValue(mockRunnerResult); + + await judge.evaluateMessages([], response); + + const inputArg = mockRunner.run.mock.calls[0][0] as string; + expect(inputArg).toBe('MESSAGE HISTORY:\n\n\nRESPONSE TO EVALUATE:\noutput'); + }); + + it('does not contaminate the runner input across successive evaluations', async () => { + const mockRunnerResult: RunnerResult = { + content: '', + parsed: { score: 0.5, reasoning: 'ok' }, + metrics: { success: true }, + }; + mockTracker.trackMetricsOf.mockImplementation(async (extractor, func) => func()); + mockRunner.run.mockResolvedValue(mockRunnerResult); + + await judge.evaluate('Q1', 'A1'); + await judge.evaluate('Q2', 'A2'); + + const firstInput = mockRunner.run.mock.calls[0][0] as string; + const secondInput = mockRunner.run.mock.calls[1][0] as string; + + expect(firstInput).toBe( + 'MESSAGE HISTORY:\nQ1\n\nRESPONSE TO EVALUATE:\nA1', + ); + expect(secondInput).toBe( + 'MESSAGE HISTORY:\nQ2\n\nRESPONSE TO EVALUATE:\nA2', + ); + // The second call's input must not reference the first call. + expect(secondInput).not.toContain('Q1'); + expect(secondInput).not.toContain('A1'); + }); + it('handles sampling rate correctly', async () => { const messages: LDMessage[] = [{ role: 'user', content: 'test' }]; const response: RunnerResult = { diff --git a/packages/sdk/server-ai/__tests__/LDAIClientImpl.test.ts b/packages/sdk/server-ai/__tests__/LDAIClientImpl.test.ts index 71a1fd0ceb..d7db880263 100644 --- a/packages/sdk/server-ai/__tests__/LDAIClientImpl.test.ts +++ b/packages/sdk/server-ai/__tests__/LDAIClientImpl.test.ts @@ -738,7 +738,12 @@ describe('createJudge method', () => { 1, ); expect(judgeConfigSpy).toHaveBeenCalledWith(key, testContext, defaultValue, undefined); - expect(RunnerFactory.createModel).toHaveBeenCalledWith(mockJudgeConfig, undefined, undefined); + expect(RunnerFactory.createModel).toHaveBeenCalledWith( + mockJudgeConfig, + undefined, + undefined, + false, + ); expect(Judge).toHaveBeenCalledWith(mockJudgeConfig, mockProvider, 1.0, undefined); expect(result).toBe(mockJudge); judgeConfigSpy.mockRestore(); @@ -793,7 +798,12 @@ describe('createJudge method', () => { const result = await client.createJudge(key, testContext, defaultValue); expect(result).toBeUndefined(); - expect(RunnerFactory.createModel).toHaveBeenCalledWith(mockJudgeConfig, undefined, undefined); + expect(RunnerFactory.createModel).toHaveBeenCalledWith( + mockJudgeConfig, + undefined, + undefined, + false, + ); expect(Judge).not.toHaveBeenCalled(); judgeConfigSpy.mockRestore(); }); diff --git a/packages/sdk/server-ai/__tests__/RunnerFactory.test.ts b/packages/sdk/server-ai/__tests__/RunnerFactory.test.ts index b7aa331fa7..190ea2c13d 100644 --- a/packages/sdk/server-ai/__tests__/RunnerFactory.test.ts +++ b/packages/sdk/server-ai/__tests__/RunnerFactory.test.ts @@ -67,7 +67,10 @@ describe('RunnerFactory.createModel', () => { const result = await RunnerFactory.createModel(makeConfig('openai')); expect(result).toBe(runner); - expect(mockFactory.createModel).toHaveBeenCalledWith(expect.objectContaining({ enabled: true })); + expect(mockFactory.createModel).toHaveBeenCalledWith( + expect.objectContaining({ enabled: true }), + true, + ); }); it('uses only the defaultAiProvider when one is specified', async () => { @@ -87,6 +90,38 @@ describe('RunnerFactory.createModel', () => { expect(getProviderSpy).toHaveBeenCalledWith('openai', undefined); }); + it('defaults multiTurn to true when forwarding to the provider factory', async () => { + const runner = makeRunner(); + const mockFactory: AIProvider = { + createModel: jest.fn().mockResolvedValue(runner), + } as unknown as AIProvider; + + jest.spyOn(RunnerFactory as any, '_getProviderFactory').mockResolvedValue(mockFactory); + + await RunnerFactory.createModel(makeConfig('openai')); + + expect(mockFactory.createModel).toHaveBeenCalledWith( + expect.objectContaining({ enabled: true }), + true, + ); + }); + + it('forwards multiTurn=false to the provider factory when specified', async () => { + const runner = makeRunner(); + const mockFactory: AIProvider = { + createModel: jest.fn().mockResolvedValue(runner), + } as unknown as AIProvider; + + jest.spyOn(RunnerFactory as any, '_getProviderFactory').mockResolvedValue(mockFactory); + + await RunnerFactory.createModel(makeConfig('openai'), undefined, undefined, false); + + expect(mockFactory.createModel).toHaveBeenCalledWith( + expect.objectContaining({ enabled: true }), + false, + ); + }); + it('falls through to multi-provider packages when specific provider returns undefined', async () => { const runner = makeRunner(); diff --git a/packages/sdk/server-ai/src/LDAIClientImpl.ts b/packages/sdk/server-ai/src/LDAIClientImpl.ts index 0ed7923a3f..075f950789 100644 --- a/packages/sdk/server-ai/src/LDAIClientImpl.ts +++ b/packages/sdk/server-ai/src/LDAIClientImpl.ts @@ -381,7 +381,12 @@ export class LDAIClientImpl implements LDAIClient { return undefined; } - const runner = await RunnerFactory.createModel(judgeConfig, this._logger, defaultAiProvider); + const runner = await RunnerFactory.createModel( + judgeConfig, + this._logger, + defaultAiProvider, + false, + ); if (!runner) { return undefined; } diff --git a/packages/sdk/server-ai/src/api/judge/Judge.ts b/packages/sdk/server-ai/src/api/judge/Judge.ts index bf64f9198f..129eca7472 100644 --- a/packages/sdk/server-ai/src/api/judge/Judge.ts +++ b/packages/sdk/server-ai/src/api/judge/Judge.ts @@ -158,6 +158,10 @@ export class Judge { /** * Evaluates an AI response from chat messages and a runner result. * + * Each message is rendered as `: ` so the judge model can + * distinguish speakers in the message history. Messages are joined with a + * single newline. + * * @param messages Array of messages representing the conversation history * @param response The runner result containing the AI-generated content to evaluate * @param samplingRatio Sampling ratio (0-1). When omitted, the Judge's @@ -169,7 +173,10 @@ export class Judge { response: RunnerResult, samplingRatio?: number, ): Promise { - const input = messages.length === 0 ? '' : messages.map((msg) => msg.content).join('\r\n'); + const input = + messages.length === 0 + ? '' + : messages.map((msg) => `${msg.role}: ${msg.content}`).join('\n'); const output = response.content; return this.evaluate(input, output, samplingRatio); diff --git a/packages/sdk/server-ai/src/api/providers/AIProvider.ts b/packages/sdk/server-ai/src/api/providers/AIProvider.ts index c795f36d35..082a303502 100644 --- a/packages/sdk/server-ai/src/api/providers/AIProvider.ts +++ b/packages/sdk/server-ai/src/api/providers/AIProvider.ts @@ -38,10 +38,17 @@ export abstract class AIProvider { * Default implementation returns `undefined`. * * @param config The completion or judge AI configuration. + * @param multiTurn Whether the runner should accumulate conversation history + * across successive `run()` calls. Defaults to `true` (chat semantics). + * Pass `false` for stateless runners such as judges where each call must + * start from the initial config messages. * @returns Promise resolving to a {@link Runner}, or `undefined` if this * provider does not support model creation. */ - async createModel(_config: LDAICompletionConfig | LDAIJudgeConfig): Promise { + async createModel( + _config: LDAICompletionConfig | LDAIJudgeConfig, + _multiTurn: boolean = true, + ): Promise { return undefined; } diff --git a/packages/sdk/server-ai/src/api/providers/RunnerFactory.ts b/packages/sdk/server-ai/src/api/providers/RunnerFactory.ts index 0451118495..40b4f34005 100644 --- a/packages/sdk/server-ai/src/api/providers/RunnerFactory.ts +++ b/packages/sdk/server-ai/src/api/providers/RunnerFactory.ts @@ -158,6 +158,10 @@ export class RunnerFactory { * ('openai', 'langchain', 'vercel', …). When set, only that provider is * tried. When omitted, providers are tried in priority order based on the * provider name in the config. + * @param multiTurn Whether the runner should accumulate conversation history + * across successive `run()` calls. Defaults to `true` (chat semantics). + * Judges pass `false` so each evaluation starts from the initial config + * messages. * @returns A configured {@link Runner} ready to invoke the model, or * `undefined` if no suitable provider could be loaded. */ @@ -165,6 +169,7 @@ export class RunnerFactory { config: LDAICompletionConfig | LDAIJudgeConfig, logger?: LDLogger, defaultAiProvider?: SupportedAIProvider, + multiTurn: boolean = true, ): Promise { const providerName = config.provider?.name?.toLowerCase(); // eslint-disable-next-line no-underscore-dangle @@ -173,7 +178,7 @@ export class RunnerFactory { // eslint-disable-next-line no-underscore-dangle const runner = await RunnerFactory._withFallback( providers, - (factory) => factory.createModel(config), + (factory) => factory.createModel(config, multiTurn), logger, );