fix: Make judge runners non-multi-turn

packages/ai-providers/server-ai-langchain/__tests__/LangChainModelRunner.test.ts

@@ -180,4 +180,68 @@ describe('LangChainModelRunner', () => {
       expect(secondCallMessages[3].content).toBe('Q2');
     });
   });
+
+  describe('multiTurn=false (stateless)', () => {
+    const configWithMessages: LDAICompletionConfig = {
+      ...baseConfig,
+      messages: [{ role: 'system', content: 'You are a judge.' }],
+    };
+
+    it('does not accumulate history across successful calls', async () => {
+      const statelessRunner = new LangChainModelRunner(
+        mockLLM,
+        configWithMessages,
+        mockLogger,
+        false,
+      );
+
+      mockLLM.invoke
+        .mockResolvedValueOnce(new AIMessage('First response'))
+        .mockResolvedValueOnce(new AIMessage('Second response'));
+
+      await statelessRunner.run('First question');
+      await statelessRunner.run('Second question');
+
+      const firstCallMessages = mockLLM.invoke.mock.calls[0][0];
+      const secondCallMessages = mockLLM.invoke.mock.calls[1][0];
+      expect(firstCallMessages).toHaveLength(2);
+      expect(firstCallMessages[0].content).toBe('You are a judge.');
+      expect(firstCallMessages[1].content).toBe('First question');
+      expect(secondCallMessages).toHaveLength(2);
+      expect(secondCallMessages[0].content).toBe('You are a judge.');
+      expect(secondCallMessages[1].content).toBe('Second question');
+    });
+
+    it('keeps the internal chat history length pinned to the seeded config messages', async () => {
+      const statelessRunner = new LangChainModelRunner(
+        mockLLM,
+        configWithMessages,
+        mockLogger,
+        false,
+      );
+
+      mockLLM.invoke.mockResolvedValue(new AIMessage('response'));
+
+      await statelessRunner.run('Q1');
+      await statelessRunner.run('Q2');
+
+      // eslint-disable-next-line no-underscore-dangle
+      const messages = await (statelessRunner as any)._chatHistory.getMessages();
+      expect(messages).toHaveLength(1);
+    });
+
+    it('defaults to multiTurn=true when the parameter is omitted', async () => {
+      const defaultRunner = new LangChainModelRunner(mockLLM, configWithMessages, mockLogger);
+
+      mockLLM.invoke
+        .mockResolvedValueOnce(new AIMessage('Answer 1'))
+        .mockResolvedValueOnce(new AIMessage('Answer 2'));
+
+      await defaultRunner.run('Q1');
+      await defaultRunner.run('Q2');
+
+      const secondCallMessages = mockLLM.invoke.mock.calls[1][0];
+      expect(secondCallMessages).toHaveLength(4);
+    });
+  });
 });

packages/ai-providers/server-ai-langchain/src/LangChainModelRunner.ts

@@ -20,26 +20,37 @@ import { convertMessagesToLangChain, getAIMetricsFromResponse } from './LangChai
 export class LangChainModelRunner implements Runner {
   private _llm: BaseChatModel;
   private _chatHistory: InMemoryChatMessageHistory;
+  private _multiTurn: boolean;
   private _logger?: LDLogger;
 
-  constructor(llm: BaseChatModel, config: LDAICompletionConfig, logger?: LDLogger) {
+  constructor(
+    llm: BaseChatModel,
+    config: LDAICompletionConfig,
+    logger?: LDLogger,
+    multiTurn: boolean = true,
+  ) {
     this._llm = llm;
     this._chatHistory = new InMemoryChatMessageHistory(
       convertMessagesToLangChain(config.messages ?? []),
     );
+    this._multiTurn = multiTurn;
     this._logger = logger;
   }
 
   /**
    * Run the LangChain model with the given user prompt.
    *
    * The runner maintains a LangChain `InMemoryChatMessageHistory` that is
-   * initialized from any messages on the AI config (system prompt, etc.) and
-   * grows with each successful call. On every invocation the user prompt is
-   * appended to the existing history before being sent to the model. When the
-   * call succeeds and produces non-empty content, the user prompt and the
-   * assistant's reply are persisted to the history; failed calls leave the
-   * history unchanged so the next call can retry cleanly.
+   * initialized from any messages on the AI config (system prompt, etc.). On
+   * every invocation the user prompt is appended to the existing history
+   * before being sent to the model. When `multiTurn` is `true` (the default)
+   * and the call succeeds with non-empty content, the user prompt and the
+   * assistant's reply are persisted to the history so subsequent calls
+   * continue the conversation. When `multiTurn` is `false`, history is
+   * treated as read-only — useful for stateless runners (e.g. judges) where
+   * every call should see only the initial config messages plus the current
+   * input. Failed calls leave the history unchanged so the next call can
+   * retry cleanly.
    *
    * @param input The user prompt string.
    * @param outputType Optional JSON schema for structured output. When provided,
@@ -56,7 +67,7 @@ export class LangChainModelRunner implements Runner {
         ? await this._runStructured(langchainMessages, outputType)
         : await this._runCompletion(langchainMessages);
 
-    if (result.metrics.success && result.content) {
+    if (result.metrics.success && result.content && this._multiTurn) {
       await this._chatHistory.addUserMessage(input);
       await this._chatHistory.addAIMessage(result.content);
     }

packages/ai-providers/server-ai-langchain/src/LangChainRunnerFactory.ts

@@ -26,10 +26,18 @@ export class LangChainRunnerFactory extends AIProvider {
 
   /**
    * Create a model runner from a completion AI configuration.
+   *
+   * @param config The completion (or judge) AI configuration.
+   * @param multiTurn Whether the runner should accumulate conversation history
+   *   across successive `run()` calls. Defaults to `true` (chat semantics).
+   *   Pass `false` for stateless runners such as judges.
    */
-  async createModel(config: LDAICompletionConfig): Promise<LangChainModelRunner> {
+  async createModel(
+    config: LDAICompletionConfig,
+    multiTurn: boolean = true,
+  ): Promise<LangChainModelRunner> {
     const llm = await createLangChainModel(config);
-    return new LangChainModelRunner(llm, config, this._logger);
+    return new LangChainModelRunner(llm, config, this._logger, multiTurn);
   }
 
   /**

packages/ai-providers/server-ai-openai/__tests__/OpenAIModelRunner.test.ts

@@ -231,4 +231,89 @@ describe('OpenAIModelRunner', () => {
       ]);
     });
   });
+
+  describe('multiTurn=false (stateless)', () => {
+    const configWithMessages: LDAICompletionConfig = {
+      ...baseConfig,
+      messages: [{ role: 'system', content: 'You are a judge.' }],
+    };
+
+    it('does not accumulate history across successful calls', async () => {
+      const statelessRunner = new OpenAIModelRunner(
+        mockOpenAI,
+        configWithMessages,
+        undefined,
+        false,
+      );
+
+      (mockOpenAI.chat.completions.create as jest.Mock)
+        .mockResolvedValueOnce({
+          choices: [{ message: { content: 'First response' } }],
+          usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 },
+        } as any)
+        .mockResolvedValueOnce({
+          choices: [{ message: { content: 'Second response' } }],
+          usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 },
+        } as any);
+
+      await statelessRunner.run('First question');
+      await statelessRunner.run('Second question');
+
+      const firstCallArgs = (mockOpenAI.chat.completions.create as jest.Mock).mock.calls[0][0];
+      const secondCallArgs = (mockOpenAI.chat.completions.create as jest.Mock).mock.calls[1][0];
+      expect(firstCallArgs.messages).toEqual([
+        { role: 'system', content: 'You are a judge.' },
+        { role: 'user', content: 'First question' },
+      ]);
+      expect(secondCallArgs.messages).toEqual([
+        { role: 'system', content: 'You are a judge.' },
+        { role: 'user', content: 'Second question' },
+      ]);
+    });
+
+    it('keeps the internal history length pinned to the seeded config messages', async () => {
+      const statelessRunner = new OpenAIModelRunner(
+        mockOpenAI,
+        configWithMessages,
+        undefined,
+        false,
+      );
+
+      (mockOpenAI.chat.completions.create as jest.Mock).mockResolvedValue({
+        choices: [{ message: { content: 'response' } }],
+        usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 },
+      } as any);
+
+      await statelessRunner.run('Q1');
+      await statelessRunner.run('Q2');
+
+      // eslint-disable-next-line no-underscore-dangle
+      expect((statelessRunner as any)._history).toHaveLength(1);
+    });
+
+    it('defaults to multiTurn=true when the parameter is omitted', async () => {
+      const defaultRunner = new OpenAIModelRunner(mockOpenAI, configWithMessages);
+
+      (mockOpenAI.chat.completions.create as jest.Mock)
+        .mockResolvedValueOnce({
+          choices: [{ message: { content: 'Answer 1' } }],
+          usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 },
+        } as any)
+        .mockResolvedValueOnce({
+          choices: [{ message: { content: 'Answer 2' } }],
+          usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 },
+        } as any);
+
+      await defaultRunner.run('Q1');
+      await defaultRunner.run('Q2');
+
+      const secondCallArgs = (mockOpenAI.chat.completions.create as jest.Mock).mock.calls[1][0];
+      expect(secondCallArgs.messages).toEqual([
+        { role: 'system', content: 'You are a judge.' },
+        { role: 'user', content: 'Q1' },
+        { role: 'assistant', content: 'Answer 1' },
+        { role: 'user', content: 'Q2' },
+      ]);
+    });
+  });
 });

packages/ai-providers/server-ai-openai/src/OpenAIModelRunner.ts

@@ -21,26 +21,37 @@ export class OpenAIModelRunner implements Runner {
   private _modelName: string;
   private _parameters: Record<string, unknown>;
   private _history: LDMessage[];
+  private _multiTurn: boolean;
   private _logger?: LDLogger;
 
-  constructor(client: OpenAI, config: LDAICompletionConfig, logger?: LDLogger) {
+  constructor(
+    client: OpenAI,
+    config: LDAICompletionConfig,
+    logger?: LDLogger,
+    multiTurn: boolean = true,
+  ) {
     this._client = client;
     this._modelName = config.model?.name ?? '';
     this._parameters = { ...(config.model?.parameters ?? {}) };
     this._history = [...(config.messages ?? [])];
+    this._multiTurn = multiTurn;
     this._logger = logger;
   }
 
   /**
    * Run the OpenAI model with the given user prompt.
    *
    * The runner maintains a conversation history that is initialized from any
-   * messages on the AI config (system prompt, instructions, etc.) and grows
-   * with each successful call. On every invocation the user prompt is appended
-   * to the existing history before being sent to the model. When the call
-   * succeeds and produces non-empty content, the user prompt and the
-   * assistant's reply are persisted to the history; failed calls leave the
-   * history unchanged so the next call can retry cleanly.
+   * messages on the AI config (system prompt, instructions, etc.). On every
+   * invocation the user prompt is appended to the existing history before
+   * being sent to the model. When `multiTurn` is `true` (the default) and the
+   * call succeeds with non-empty content, the user prompt and the assistant's
+   * reply are persisted to the history so subsequent calls continue the
+   * conversation. When `multiTurn` is `false`, history is treated as
+   * read-only — useful for stateless runners (e.g. judges) where every call
+   * should see only the initial config messages plus the current input.
+   * Failed calls leave the history unchanged so the next call can retry
+   * cleanly.
    *
    * @param input The user prompt string.
    * @param outputType Optional JSON schema for structured output. When provided,
@@ -55,7 +66,7 @@ export class OpenAIModelRunner implements Runner {
         ? await this._runStructured(messages, outputType)
         : await this._runCompletion(messages);
 
-    if (result.metrics.success && result.content) {
+    if (result.metrics.success && result.content && this._multiTurn) {
       this._history.push(userMessage);
       this._history.push({ role: 'assistant', content: result.content });
     }

packages/ai-providers/server-ai-openai/src/OpenAIRunnerFactory.ts

@@ -28,9 +28,17 @@ export class OpenAIRunnerFactory extends AIProvider {
 
   /**
    * Create a model runner from a completion AI configuration.
+   *
+   * @param config The completion (or judge) AI configuration.
+   * @param multiTurn Whether the runner should accumulate conversation history
+   *   across successive `run()` calls. Defaults to `true` (chat semantics).
+   *   Pass `false` for stateless runners such as judges.
    */
-  async createModel(config: LDAICompletionConfig): Promise<OpenAIModelRunner> {
-    return new OpenAIModelRunner(this._client, config, this._logger);
+  async createModel(
+    config: LDAICompletionConfig,
+    multiTurn: boolean = true,
+  ): Promise<OpenAIModelRunner> {
+    return new OpenAIModelRunner(this._client, config, this._logger, multiTurn);
   }
 
   /**