diff --git a/packages/ai-providers/server-ai-langchain/__tests__/LangChainModelRunner.test.ts b/packages/ai-providers/server-ai-langchain/__tests__/LangChainModelRunner.test.ts index b48df2046f..2ae5a3e018 100644 --- a/packages/ai-providers/server-ai-langchain/__tests__/LangChainModelRunner.test.ts +++ b/packages/ai-providers/server-ai-langchain/__tests__/LangChainModelRunner.test.ts @@ -180,4 +180,68 @@ describe('LangChainModelRunner', () => { expect(secondCallMessages[3].content).toBe('Q2'); }); }); + + describe('multiTurn=false (stateless)', () => { + const configWithMessages: LDAICompletionConfig = { + ...baseConfig, + messages: [{ role: 'system', content: 'You are a judge.' }], + }; + + it('does not accumulate history across successful calls', async () => { + const statelessRunner = new LangChainModelRunner( + mockLLM, + configWithMessages, + mockLogger, + false, + ); + + mockLLM.invoke + .mockResolvedValueOnce(new AIMessage('First response')) + .mockResolvedValueOnce(new AIMessage('Second response')); + + await statelessRunner.run('First question'); + await statelessRunner.run('Second question'); + + const firstCallMessages = mockLLM.invoke.mock.calls[0][0]; + const secondCallMessages = mockLLM.invoke.mock.calls[1][0]; + expect(firstCallMessages).toHaveLength(2); + expect(firstCallMessages[0].content).toBe('You are a judge.'); + expect(firstCallMessages[1].content).toBe('First question'); + expect(secondCallMessages).toHaveLength(2); + expect(secondCallMessages[0].content).toBe('You are a judge.'); + expect(secondCallMessages[1].content).toBe('Second question'); + }); + + it('keeps the internal chat history length pinned to the seeded config messages', async () => { + const statelessRunner = new LangChainModelRunner( + mockLLM, + configWithMessages, + mockLogger, + false, + ); + + mockLLM.invoke.mockResolvedValue(new AIMessage('response')); + + await statelessRunner.run('Q1'); + await statelessRunner.run('Q2'); + + // eslint-disable-next-line no-underscore-dangle + const messages = await (statelessRunner as any)._chatHistory.getMessages(); + expect(messages).toHaveLength(1); + }); + + it('defaults to multiTurn=true when the parameter is omitted', async () => { + const defaultRunner = new LangChainModelRunner(mockLLM, configWithMessages, mockLogger); + + mockLLM.invoke + .mockResolvedValueOnce(new AIMessage('Answer 1')) + .mockResolvedValueOnce(new AIMessage('Answer 2')); + + await defaultRunner.run('Q1'); + await defaultRunner.run('Q2'); + + const secondCallMessages = mockLLM.invoke.mock.calls[1][0]; + expect(secondCallMessages).toHaveLength(4); + }); + }); }); diff --git a/packages/ai-providers/server-ai-langchain/src/LangChainModelRunner.ts b/packages/ai-providers/server-ai-langchain/src/LangChainModelRunner.ts index 57f2e0d4f2..5f86191f81 100644 --- a/packages/ai-providers/server-ai-langchain/src/LangChainModelRunner.ts +++ b/packages/ai-providers/server-ai-langchain/src/LangChainModelRunner.ts @@ -20,13 +20,20 @@ import { convertMessagesToLangChain, getAIMetricsFromResponse } from './LangChai export class LangChainModelRunner implements Runner { private _llm: BaseChatModel; private _chatHistory: InMemoryChatMessageHistory; + private _multiTurn: boolean; private _logger?: LDLogger; - constructor(llm: BaseChatModel, config: LDAICompletionConfig, logger?: LDLogger) { + constructor( + llm: BaseChatModel, + config: LDAICompletionConfig, + logger?: LDLogger, + multiTurn: boolean = true, + ) { this._llm = llm; this._chatHistory = new InMemoryChatMessageHistory( convertMessagesToLangChain(config.messages ?? []), ); + this._multiTurn = multiTurn; this._logger = logger; } @@ -34,12 +41,16 @@ export class LangChainModelRunner implements Runner { * Run the LangChain model with the given user prompt. * * The runner maintains a LangChain `InMemoryChatMessageHistory` that is - * initialized from any messages on the AI config (system prompt, etc.) and - * grows with each successful call. On every invocation the user prompt is - * appended to the existing history before being sent to the model. When the - * call succeeds and produces non-empty content, the user prompt and the - * assistant's reply are persisted to the history; failed calls leave the - * history unchanged so the next call can retry cleanly. + * initialized from any messages on the AI config (system prompt, etc.). On + * every invocation the user prompt is appended to the existing history + * before being sent to the model. When `multiTurn` is `true` (the default) + * and the call succeeds with non-empty content, the user prompt and the + * assistant's reply are persisted to the history so subsequent calls + * continue the conversation. When `multiTurn` is `false`, history is + * treated as read-only — useful for stateless runners (e.g. judges) where + * every call should see only the initial config messages plus the current + * input. Failed calls leave the history unchanged so the next call can + * retry cleanly. * * @param input The user prompt string. * @param outputType Optional JSON schema for structured output. When provided, @@ -56,7 +67,7 @@ export class LangChainModelRunner implements Runner { ? await this._runStructured(langchainMessages, outputType) : await this._runCompletion(langchainMessages); - if (result.metrics.success && result.content) { + if (result.metrics.success && result.content && this._multiTurn) { await this._chatHistory.addUserMessage(input); await this._chatHistory.addAIMessage(result.content); } diff --git a/packages/ai-providers/server-ai-langchain/src/LangChainRunnerFactory.ts b/packages/ai-providers/server-ai-langchain/src/LangChainRunnerFactory.ts index f00cfb0e14..69a0144e2b 100644 --- a/packages/ai-providers/server-ai-langchain/src/LangChainRunnerFactory.ts +++ b/packages/ai-providers/server-ai-langchain/src/LangChainRunnerFactory.ts @@ -26,10 +26,18 @@ export class LangChainRunnerFactory extends AIProvider { /** * Create a model runner from a completion AI configuration. + * + * @param config The completion (or judge) AI configuration. + * @param multiTurn Whether the runner should accumulate conversation history + * across successive `run()` calls. Defaults to `true` (chat semantics). + * Pass `false` for stateless runners such as judges. */ - async createModel(config: LDAICompletionConfig): Promise { + async createModel( + config: LDAICompletionConfig, + multiTurn: boolean = true, + ): Promise { const llm = await createLangChainModel(config); - return new LangChainModelRunner(llm, config, this._logger); + return new LangChainModelRunner(llm, config, this._logger, multiTurn); } /** diff --git a/packages/ai-providers/server-ai-openai/__tests__/OpenAIModelRunner.test.ts b/packages/ai-providers/server-ai-openai/__tests__/OpenAIModelRunner.test.ts index b4b44c9cb1..f0b20be095 100644 --- a/packages/ai-providers/server-ai-openai/__tests__/OpenAIModelRunner.test.ts +++ b/packages/ai-providers/server-ai-openai/__tests__/OpenAIModelRunner.test.ts @@ -231,4 +231,89 @@ describe('OpenAIModelRunner', () => { ]); }); }); + + describe('multiTurn=false (stateless)', () => { + const configWithMessages: LDAICompletionConfig = { + ...baseConfig, + messages: [{ role: 'system', content: 'You are a judge.' }], + }; + + it('does not accumulate history across successful calls', async () => { + const statelessRunner = new OpenAIModelRunner( + mockOpenAI, + configWithMessages, + undefined, + false, + ); + + (mockOpenAI.chat.completions.create as jest.Mock) + .mockResolvedValueOnce({ + choices: [{ message: { content: 'First response' } }], + usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 }, + } as any) + .mockResolvedValueOnce({ + choices: [{ message: { content: 'Second response' } }], + usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 }, + } as any); + + await statelessRunner.run('First question'); + await statelessRunner.run('Second question'); + + const firstCallArgs = (mockOpenAI.chat.completions.create as jest.Mock).mock.calls[0][0]; + const secondCallArgs = (mockOpenAI.chat.completions.create as jest.Mock).mock.calls[1][0]; + expect(firstCallArgs.messages).toEqual([ + { role: 'system', content: 'You are a judge.' }, + { role: 'user', content: 'First question' }, + ]); + expect(secondCallArgs.messages).toEqual([ + { role: 'system', content: 'You are a judge.' }, + { role: 'user', content: 'Second question' }, + ]); + }); + + it('keeps the internal history length pinned to the seeded config messages', async () => { + const statelessRunner = new OpenAIModelRunner( + mockOpenAI, + configWithMessages, + undefined, + false, + ); + + (mockOpenAI.chat.completions.create as jest.Mock).mockResolvedValue({ + choices: [{ message: { content: 'response' } }], + usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 }, + } as any); + + await statelessRunner.run('Q1'); + await statelessRunner.run('Q2'); + + // eslint-disable-next-line no-underscore-dangle + expect((statelessRunner as any)._history).toHaveLength(1); + }); + + it('defaults to multiTurn=true when the parameter is omitted', async () => { + const defaultRunner = new OpenAIModelRunner(mockOpenAI, configWithMessages); + + (mockOpenAI.chat.completions.create as jest.Mock) + .mockResolvedValueOnce({ + choices: [{ message: { content: 'Answer 1' } }], + usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 }, + } as any) + .mockResolvedValueOnce({ + choices: [{ message: { content: 'Answer 2' } }], + usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 }, + } as any); + + await defaultRunner.run('Q1'); + await defaultRunner.run('Q2'); + + const secondCallArgs = (mockOpenAI.chat.completions.create as jest.Mock).mock.calls[1][0]; + expect(secondCallArgs.messages).toEqual([ + { role: 'system', content: 'You are a judge.' }, + { role: 'user', content: 'Q1' }, + { role: 'assistant', content: 'Answer 1' }, + { role: 'user', content: 'Q2' }, + ]); + }); + }); }); diff --git a/packages/ai-providers/server-ai-openai/src/OpenAIModelRunner.ts b/packages/ai-providers/server-ai-openai/src/OpenAIModelRunner.ts index 62882455d5..e87b83d52e 100644 --- a/packages/ai-providers/server-ai-openai/src/OpenAIModelRunner.ts +++ b/packages/ai-providers/server-ai-openai/src/OpenAIModelRunner.ts @@ -21,13 +21,20 @@ export class OpenAIModelRunner implements Runner { private _modelName: string; private _parameters: Record; private _history: LDMessage[]; + private _multiTurn: boolean; private _logger?: LDLogger; - constructor(client: OpenAI, config: LDAICompletionConfig, logger?: LDLogger) { + constructor( + client: OpenAI, + config: LDAICompletionConfig, + logger?: LDLogger, + multiTurn: boolean = true, + ) { this._client = client; this._modelName = config.model?.name ?? ''; this._parameters = { ...(config.model?.parameters ?? {}) }; this._history = [...(config.messages ?? [])]; + this._multiTurn = multiTurn; this._logger = logger; } @@ -35,12 +42,16 @@ export class OpenAIModelRunner implements Runner { * Run the OpenAI model with the given user prompt. * * The runner maintains a conversation history that is initialized from any - * messages on the AI config (system prompt, instructions, etc.) and grows - * with each successful call. On every invocation the user prompt is appended - * to the existing history before being sent to the model. When the call - * succeeds and produces non-empty content, the user prompt and the - * assistant's reply are persisted to the history; failed calls leave the - * history unchanged so the next call can retry cleanly. + * messages on the AI config (system prompt, instructions, etc.). On every + * invocation the user prompt is appended to the existing history before + * being sent to the model. When `multiTurn` is `true` (the default) and the + * call succeeds with non-empty content, the user prompt and the assistant's + * reply are persisted to the history so subsequent calls continue the + * conversation. When `multiTurn` is `false`, history is treated as + * read-only — useful for stateless runners (e.g. judges) where every call + * should see only the initial config messages plus the current input. + * Failed calls leave the history unchanged so the next call can retry + * cleanly. * * @param input The user prompt string. * @param outputType Optional JSON schema for structured output. When provided, @@ -55,7 +66,7 @@ export class OpenAIModelRunner implements Runner { ? await this._runStructured(messages, outputType) : await this._runCompletion(messages); - if (result.metrics.success && result.content) { + if (result.metrics.success && result.content && this._multiTurn) { this._history.push(userMessage); this._history.push({ role: 'assistant', content: result.content }); } diff --git a/packages/ai-providers/server-ai-openai/src/OpenAIRunnerFactory.ts b/packages/ai-providers/server-ai-openai/src/OpenAIRunnerFactory.ts index 9f99f7d798..13511653a0 100644 --- a/packages/ai-providers/server-ai-openai/src/OpenAIRunnerFactory.ts +++ b/packages/ai-providers/server-ai-openai/src/OpenAIRunnerFactory.ts @@ -28,9 +28,17 @@ export class OpenAIRunnerFactory extends AIProvider { /** * Create a model runner from a completion AI configuration. + * + * @param config The completion (or judge) AI configuration. + * @param multiTurn Whether the runner should accumulate conversation history + * across successive `run()` calls. Defaults to `true` (chat semantics). + * Pass `false` for stateless runners such as judges. */ - async createModel(config: LDAICompletionConfig): Promise { - return new OpenAIModelRunner(this._client, config, this._logger); + async createModel( + config: LDAICompletionConfig, + multiTurn: boolean = true, + ): Promise { + return new OpenAIModelRunner(this._client, config, this._logger, multiTurn); } /** diff --git a/packages/ai-providers/server-ai-vercel/__tests__/VercelModelRunner.test.ts b/packages/ai-providers/server-ai-vercel/__tests__/VercelModelRunner.test.ts index f781ec9eff..d78d28edc6 100644 --- a/packages/ai-providers/server-ai-vercel/__tests__/VercelModelRunner.test.ts +++ b/packages/ai-providers/server-ai-vercel/__tests__/VercelModelRunner.test.ts @@ -231,4 +231,96 @@ describe('VercelModelRunner', () => { ]); }); }); + + describe('multiTurn=false (stateless)', () => { + const configWithMessages: LDAICompletionConfig = { + ...baseConfig, + messages: [{ role: 'system', content: 'You are a judge.' }], + }; + + it('does not accumulate history across successful calls', async () => { + const statelessRunner = new VercelModelRunner( + fakeModel as any, + configWithMessages, + {}, + mockLogger, + false, + ); + + (generateText as jest.Mock) + .mockResolvedValueOnce({ + text: 'First response', + usage: { totalTokens: 2, promptTokens: 1, completionTokens: 1 }, + }) + .mockResolvedValueOnce({ + text: 'Second response', + usage: { totalTokens: 2, promptTokens: 1, completionTokens: 1 }, + }); + + await statelessRunner.run('First question'); + await statelessRunner.run('Second question'); + + const firstCallArgs = (generateText as jest.Mock).mock.calls[0][0]; + const secondCallArgs = (generateText as jest.Mock).mock.calls[1][0]; + expect(firstCallArgs.messages).toEqual([ + { role: 'system', content: 'You are a judge.' }, + { role: 'user', content: 'First question' }, + ]); + expect(secondCallArgs.messages).toEqual([ + { role: 'system', content: 'You are a judge.' }, + { role: 'user', content: 'Second question' }, + ]); + }); + + it('keeps the internal history length pinned to the seeded config messages', async () => { + const statelessRunner = new VercelModelRunner( + fakeModel as any, + configWithMessages, + {}, + mockLogger, + false, + ); + + (generateText as jest.Mock).mockResolvedValue({ + text: 'response', + usage: { totalTokens: 2, promptTokens: 1, completionTokens: 1 }, + }); + + await statelessRunner.run('Q1'); + await statelessRunner.run('Q2'); + + // eslint-disable-next-line no-underscore-dangle + expect((statelessRunner as any)._history).toHaveLength(1); + }); + + it('defaults to multiTurn=true when the parameter is omitted', async () => { + const defaultRunner = new VercelModelRunner( + fakeModel as any, + configWithMessages, + {}, + mockLogger, + ); + + (generateText as jest.Mock) + .mockResolvedValueOnce({ + text: 'Answer 1', + usage: { totalTokens: 2, promptTokens: 1, completionTokens: 1 }, + }) + .mockResolvedValueOnce({ + text: 'Answer 2', + usage: { totalTokens: 2, promptTokens: 1, completionTokens: 1 }, + }); + + await defaultRunner.run('Q1'); + await defaultRunner.run('Q2'); + + const secondCallArgs = (generateText as jest.Mock).mock.calls[1][0]; + expect(secondCallArgs.messages).toEqual([ + { role: 'system', content: 'You are a judge.' }, + { role: 'user', content: 'Q1' }, + { role: 'assistant', content: 'Answer 1' }, + { role: 'user', content: 'Q2' }, + ]); + }); + }); }); diff --git a/packages/ai-providers/server-ai-vercel/src/VercelModelRunner.ts b/packages/ai-providers/server-ai-vercel/src/VercelModelRunner.ts index 2c842f324b..ccadc3d6ba 100644 --- a/packages/ai-providers/server-ai-vercel/src/VercelModelRunner.ts +++ b/packages/ai-providers/server-ai-vercel/src/VercelModelRunner.ts @@ -20,6 +20,7 @@ export class VercelModelRunner implements Runner { private _model: LanguageModel; private _parameters: VercelAIModelParameters; private _history: ModelMessage[]; + private _multiTurn: boolean; private _logger?: LDLogger; constructor( @@ -27,10 +28,12 @@ export class VercelModelRunner implements Runner { config: LDAICompletionConfig, parameters: VercelAIModelParameters, logger?: LDLogger, + multiTurn: boolean = true, ) { this._model = model; this._parameters = parameters; this._history = convertMessagesToVercel(config.messages ?? []) as ModelMessage[]; + this._multiTurn = multiTurn; this._logger = logger; } @@ -39,12 +42,15 @@ export class VercelModelRunner implements Runner { * * The runner maintains a conversation history (as Vercel AI SDK * `ModelMessage`s) that is initialized from any messages on the AI config - * (system prompt, etc.) and grows with each successful call. On every - * invocation the user prompt is appended to the existing history before - * being sent to the model. When the call succeeds and produces non-empty - * content, the user prompt and the assistant's reply are persisted to the - * history; failed calls leave the history unchanged so the next call can - * retry cleanly. + * (system prompt, etc.). On every invocation the user prompt is appended to + * the existing history before being sent to the model. When `multiTurn` is + * `true` (the default) and the call succeeds with non-empty content, the + * user prompt and the assistant's reply are persisted to the history so + * subsequent calls continue the conversation. When `multiTurn` is `false`, + * history is treated as read-only — useful for stateless runners (e.g. + * judges) where every call should see only the initial config messages + * plus the current input. Failed calls leave the history unchanged so the + * next call can retry cleanly. * * @param input The user prompt string. * @param outputType Optional JSON schema for structured output. When provided, @@ -59,7 +65,7 @@ export class VercelModelRunner implements Runner { ? await this._runStructured(messages, outputType) : await this._runCompletion(messages); - if (result.metrics.success && result.content) { + if (result.metrics.success && result.content && this._multiTurn) { this._history.push(userMessage); this._history.push({ role: 'assistant', content: result.content }); } diff --git a/packages/ai-providers/server-ai-vercel/src/VercelRunnerFactory.ts b/packages/ai-providers/server-ai-vercel/src/VercelRunnerFactory.ts index 9869301314..083d164387 100644 --- a/packages/ai-providers/server-ai-vercel/src/VercelRunnerFactory.ts +++ b/packages/ai-providers/server-ai-vercel/src/VercelRunnerFactory.ts @@ -21,11 +21,19 @@ export class VercelRunnerFactory extends AIProvider { /** * Create a model runner from a completion AI configuration. + * + * @param config The completion (or judge) AI configuration. + * @param multiTurn Whether the runner should accumulate conversation history + * across successive `run()` calls. Defaults to `true` (chat semantics). + * Pass `false` for stateless runners such as judges. */ - async createModel(config: LDAICompletionConfig): Promise { + async createModel( + config: LDAICompletionConfig, + multiTurn: boolean = true, + ): Promise { const model = await VercelRunnerFactory.createVercelModel(config); const parameters = VercelRunnerFactory.mapParameters(config.model?.parameters); - return new VercelModelRunner(model, config, parameters, this._logger); + return new VercelModelRunner(model, config, parameters, this._logger, multiTurn); } /** diff --git a/packages/sdk/server-ai/__tests__/Judge.test.ts b/packages/sdk/server-ai/__tests__/Judge.test.ts index 1b05737f05..75a1d33a89 100644 --- a/packages/sdk/server-ai/__tests__/Judge.test.ts +++ b/packages/sdk/server-ai/__tests__/Judge.test.ts @@ -553,11 +553,81 @@ describe('Judge', () => { }); expect(mockRunner.run).toHaveBeenCalledWith( - 'MESSAGE HISTORY:\nWhat is the capital of France?\r\nParis is the capital of France.\n\nRESPONSE TO EVALUATE:\nParis is the capital of France.', + 'MESSAGE HISTORY:\nuser: What is the capital of France?\nassistant: Paris is the capital of France.\n\nRESPONSE TO EVALUATE:\nParis is the capital of France.', expect.any(Object), // evaluation schema ); }); + it('renders each message as ": " joined by newlines', async () => { + const messages: LDMessage[] = [ + { role: 'user', content: 'hi' }, + { role: 'assistant', content: 'hello' }, + ]; + const response: RunnerResult = { + content: 'hello', + metrics: { success: true }, + }; + + const mockRunnerResult: RunnerResult = { + content: '', + parsed: { score: 0.5, reasoning: 'ok' }, + metrics: { success: true }, + }; + mockTracker.trackMetricsOf.mockImplementation(async (extractor, func) => func()); + mockRunner.run.mockResolvedValue(mockRunnerResult); + + await judge.evaluateMessages(messages, response); + + const inputArg = mockRunner.run.mock.calls[0][0] as string; + expect(inputArg).toContain('MESSAGE HISTORY:\nuser: hi\nassistant: hello'); + }); + + it('produces an empty MESSAGE HISTORY section when messages is empty', async () => { + const response: RunnerResult = { + content: 'output', + metrics: { success: true }, + }; + + const mockRunnerResult: RunnerResult = { + content: '', + parsed: { score: 0.5, reasoning: 'ok' }, + metrics: { success: true }, + }; + mockTracker.trackMetricsOf.mockImplementation(async (extractor, func) => func()); + mockRunner.run.mockResolvedValue(mockRunnerResult); + + await judge.evaluateMessages([], response); + + const inputArg = mockRunner.run.mock.calls[0][0] as string; + expect(inputArg).toBe('MESSAGE HISTORY:\n\n\nRESPONSE TO EVALUATE:\noutput'); + }); + + it('does not contaminate the runner input across successive evaluations', async () => { + const mockRunnerResult: RunnerResult = { + content: '', + parsed: { score: 0.5, reasoning: 'ok' }, + metrics: { success: true }, + }; + mockTracker.trackMetricsOf.mockImplementation(async (extractor, func) => func()); + mockRunner.run.mockResolvedValue(mockRunnerResult); + + await judge.evaluate('Q1', 'A1'); + await judge.evaluate('Q2', 'A2'); + + const firstInput = mockRunner.run.mock.calls[0][0] as string; + const secondInput = mockRunner.run.mock.calls[1][0] as string; + + expect(firstInput).toBe( + 'MESSAGE HISTORY:\nQ1\n\nRESPONSE TO EVALUATE:\nA1', + ); + expect(secondInput).toBe( + 'MESSAGE HISTORY:\nQ2\n\nRESPONSE TO EVALUATE:\nA2', + ); + // The second call's input must not reference the first call. + expect(secondInput).not.toContain('Q1'); + expect(secondInput).not.toContain('A1'); + }); + it('handles sampling rate correctly', async () => { const messages: LDMessage[] = [{ role: 'user', content: 'test' }]; const response: RunnerResult = { diff --git a/packages/sdk/server-ai/__tests__/LDAIClientImpl.test.ts b/packages/sdk/server-ai/__tests__/LDAIClientImpl.test.ts index 71a1fd0ceb..d7db880263 100644 --- a/packages/sdk/server-ai/__tests__/LDAIClientImpl.test.ts +++ b/packages/sdk/server-ai/__tests__/LDAIClientImpl.test.ts @@ -738,7 +738,12 @@ describe('createJudge method', () => { 1, ); expect(judgeConfigSpy).toHaveBeenCalledWith(key, testContext, defaultValue, undefined); - expect(RunnerFactory.createModel).toHaveBeenCalledWith(mockJudgeConfig, undefined, undefined); + expect(RunnerFactory.createModel).toHaveBeenCalledWith( + mockJudgeConfig, + undefined, + undefined, + false, + ); expect(Judge).toHaveBeenCalledWith(mockJudgeConfig, mockProvider, 1.0, undefined); expect(result).toBe(mockJudge); judgeConfigSpy.mockRestore(); @@ -793,7 +798,12 @@ describe('createJudge method', () => { const result = await client.createJudge(key, testContext, defaultValue); expect(result).toBeUndefined(); - expect(RunnerFactory.createModel).toHaveBeenCalledWith(mockJudgeConfig, undefined, undefined); + expect(RunnerFactory.createModel).toHaveBeenCalledWith( + mockJudgeConfig, + undefined, + undefined, + false, + ); expect(Judge).not.toHaveBeenCalled(); judgeConfigSpy.mockRestore(); }); diff --git a/packages/sdk/server-ai/__tests__/RunnerFactory.test.ts b/packages/sdk/server-ai/__tests__/RunnerFactory.test.ts index b7aa331fa7..190ea2c13d 100644 --- a/packages/sdk/server-ai/__tests__/RunnerFactory.test.ts +++ b/packages/sdk/server-ai/__tests__/RunnerFactory.test.ts @@ -67,7 +67,10 @@ describe('RunnerFactory.createModel', () => { const result = await RunnerFactory.createModel(makeConfig('openai')); expect(result).toBe(runner); - expect(mockFactory.createModel).toHaveBeenCalledWith(expect.objectContaining({ enabled: true })); + expect(mockFactory.createModel).toHaveBeenCalledWith( + expect.objectContaining({ enabled: true }), + true, + ); }); it('uses only the defaultAiProvider when one is specified', async () => { @@ -87,6 +90,38 @@ describe('RunnerFactory.createModel', () => { expect(getProviderSpy).toHaveBeenCalledWith('openai', undefined); }); + it('defaults multiTurn to true when forwarding to the provider factory', async () => { + const runner = makeRunner(); + const mockFactory: AIProvider = { + createModel: jest.fn().mockResolvedValue(runner), + } as unknown as AIProvider; + + jest.spyOn(RunnerFactory as any, '_getProviderFactory').mockResolvedValue(mockFactory); + + await RunnerFactory.createModel(makeConfig('openai')); + + expect(mockFactory.createModel).toHaveBeenCalledWith( + expect.objectContaining({ enabled: true }), + true, + ); + }); + + it('forwards multiTurn=false to the provider factory when specified', async () => { + const runner = makeRunner(); + const mockFactory: AIProvider = { + createModel: jest.fn().mockResolvedValue(runner), + } as unknown as AIProvider; + + jest.spyOn(RunnerFactory as any, '_getProviderFactory').mockResolvedValue(mockFactory); + + await RunnerFactory.createModel(makeConfig('openai'), undefined, undefined, false); + + expect(mockFactory.createModel).toHaveBeenCalledWith( + expect.objectContaining({ enabled: true }), + false, + ); + }); + it('falls through to multi-provider packages when specific provider returns undefined', async () => { const runner = makeRunner(); diff --git a/packages/sdk/server-ai/src/LDAIClientImpl.ts b/packages/sdk/server-ai/src/LDAIClientImpl.ts index 0ed7923a3f..075f950789 100644 --- a/packages/sdk/server-ai/src/LDAIClientImpl.ts +++ b/packages/sdk/server-ai/src/LDAIClientImpl.ts @@ -381,7 +381,12 @@ export class LDAIClientImpl implements LDAIClient { return undefined; } - const runner = await RunnerFactory.createModel(judgeConfig, this._logger, defaultAiProvider); + const runner = await RunnerFactory.createModel( + judgeConfig, + this._logger, + defaultAiProvider, + false, + ); if (!runner) { return undefined; } diff --git a/packages/sdk/server-ai/src/api/judge/Judge.ts b/packages/sdk/server-ai/src/api/judge/Judge.ts index bf64f9198f..129eca7472 100644 --- a/packages/sdk/server-ai/src/api/judge/Judge.ts +++ b/packages/sdk/server-ai/src/api/judge/Judge.ts @@ -158,6 +158,10 @@ export class Judge { /** * Evaluates an AI response from chat messages and a runner result. * + * Each message is rendered as `: ` so the judge model can + * distinguish speakers in the message history. Messages are joined with a + * single newline. + * * @param messages Array of messages representing the conversation history * @param response The runner result containing the AI-generated content to evaluate * @param samplingRatio Sampling ratio (0-1). When omitted, the Judge's @@ -169,7 +173,10 @@ export class Judge { response: RunnerResult, samplingRatio?: number, ): Promise { - const input = messages.length === 0 ? '' : messages.map((msg) => msg.content).join('\r\n'); + const input = + messages.length === 0 + ? '' + : messages.map((msg) => `${msg.role}: ${msg.content}`).join('\n'); const output = response.content; return this.evaluate(input, output, samplingRatio); diff --git a/packages/sdk/server-ai/src/api/providers/AIProvider.ts b/packages/sdk/server-ai/src/api/providers/AIProvider.ts index c795f36d35..082a303502 100644 --- a/packages/sdk/server-ai/src/api/providers/AIProvider.ts +++ b/packages/sdk/server-ai/src/api/providers/AIProvider.ts @@ -38,10 +38,17 @@ export abstract class AIProvider { * Default implementation returns `undefined`. * * @param config The completion or judge AI configuration. + * @param multiTurn Whether the runner should accumulate conversation history + * across successive `run()` calls. Defaults to `true` (chat semantics). + * Pass `false` for stateless runners such as judges where each call must + * start from the initial config messages. * @returns Promise resolving to a {@link Runner}, or `undefined` if this * provider does not support model creation. */ - async createModel(_config: LDAICompletionConfig | LDAIJudgeConfig): Promise { + async createModel( + _config: LDAICompletionConfig | LDAIJudgeConfig, + _multiTurn: boolean = true, + ): Promise { return undefined; } diff --git a/packages/sdk/server-ai/src/api/providers/RunnerFactory.ts b/packages/sdk/server-ai/src/api/providers/RunnerFactory.ts index 0451118495..40b4f34005 100644 --- a/packages/sdk/server-ai/src/api/providers/RunnerFactory.ts +++ b/packages/sdk/server-ai/src/api/providers/RunnerFactory.ts @@ -158,6 +158,10 @@ export class RunnerFactory { * ('openai', 'langchain', 'vercel', …). When set, only that provider is * tried. When omitted, providers are tried in priority order based on the * provider name in the config. + * @param multiTurn Whether the runner should accumulate conversation history + * across successive `run()` calls. Defaults to `true` (chat semantics). + * Judges pass `false` so each evaluation starts from the initial config + * messages. * @returns A configured {@link Runner} ready to invoke the model, or * `undefined` if no suitable provider could be loaded. */ @@ -165,6 +169,7 @@ export class RunnerFactory { config: LDAICompletionConfig | LDAIJudgeConfig, logger?: LDLogger, defaultAiProvider?: SupportedAIProvider, + multiTurn: boolean = true, ): Promise { const providerName = config.provider?.name?.toLowerCase(); // eslint-disable-next-line no-underscore-dangle @@ -173,7 +178,7 @@ export class RunnerFactory { // eslint-disable-next-line no-underscore-dangle const runner = await RunnerFactory._withFallback( providers, - (factory) => factory.createModel(config), + (factory) => factory.createModel(config, multiTurn), logger, );