From b74c8512b80d2065a4c86573a741dd2908e4b1bc Mon Sep 17 00:00:00 2001 From: jsonbailey Date: Thu, 7 May 2026 17:16:13 -0500 Subject: [PATCH] feat: support conversation history directly in AI Provider model runners Each provider model runner now keeps an internal conversation history that is seeded from the AI config's messages and grows with each successful call. The user prompt and the assistant's reply are appended to history only when the call succeeds and produces non-empty content, so failed calls leave history unchanged for retries. Mirrors python-server-sdk-ai #166. - OpenAI: private LDMessage[] history, mirrored on the Python OpenAI runner. - LangChain: uses InMemoryChatMessageHistory from @langchain/core/chat_history to mirror the Python LangChain runner. - Vercel: private ModelMessage[] history (Vercel-native types). --- .../__tests__/LangChainModelRunner.test.ts | 65 ++++++++++++++ .../src/LangChainModelRunner.ts | 48 +++++++---- .../__tests__/OpenAIModelRunner.test.ts | 83 ++++++++++++++++++ .../server-ai-openai/src/OpenAIModelRunner.ts | 32 ++++--- .../__tests__/VercelModelRunner.test.ts | 85 +++++++++++++++++++ .../server-ai-vercel/src/VercelModelRunner.ts | 44 ++++++---- 6 files changed, 311 insertions(+), 46 deletions(-) diff --git a/packages/ai-providers/server-ai-langchain/__tests__/LangChainModelRunner.test.ts b/packages/ai-providers/server-ai-langchain/__tests__/LangChainModelRunner.test.ts index 73dbd940c6..b48df2046f 100644 --- a/packages/ai-providers/server-ai-langchain/__tests__/LangChainModelRunner.test.ts +++ b/packages/ai-providers/server-ai-langchain/__tests__/LangChainModelRunner.test.ts @@ -115,4 +115,69 @@ describe('LangChainModelRunner', () => { it('returns the underlying chat model', () => { expect(runner.getChatModel()).toBe(mockLLM); }); + + describe('conversation history', () => { + it('accumulates history across successful calls', async () => { + mockLLM.invoke + .mockResolvedValueOnce(new AIMessage('First response')) + .mockResolvedValueOnce(new AIMessage('Second response')); + + await runner.run('First question'); + await runner.run('Second question'); + + const secondCallMessages = mockLLM.invoke.mock.calls[1][0]; + const roles = secondCallMessages.map((m: any) => m.constructor.name); + expect(roles).toEqual(['HumanMessage', 'AIMessage', 'HumanMessage']); + expect(secondCallMessages[0].content).toBe('First question'); + expect(secondCallMessages[1].content).toBe('First response'); + expect(secondCallMessages[2].content).toBe('Second question'); + }); + + it('does not accumulate history when the call throws', async () => { + mockLLM.invoke.mockRejectedValueOnce(new Error('Model error')); + await runner.run('Hello'); + + mockLLM.invoke.mockResolvedValueOnce(new AIMessage('Recovery')); + await runner.run('Try again'); + + const secondCallMessages = mockLLM.invoke.mock.calls[1][0]; + expect(secondCallMessages).toHaveLength(1); + expect(secondCallMessages[0].content).toBe('Try again'); + }); + + it('does not accumulate history when content is empty (multimodal)', async () => { + mockLLM.invoke.mockResolvedValueOnce(new AIMessage([{ type: 'image' }] as any)); + await runner.run('Hello'); + + mockLLM.invoke.mockResolvedValueOnce(new AIMessage('Recovery')); + await runner.run('Try again'); + + const secondCallMessages = mockLLM.invoke.mock.calls[1][0]; + expect(secondCallMessages).toHaveLength(1); + expect(secondCallMessages[0].content).toBe('Try again'); + }); + + it('keeps config messages prepended ahead of accumulated history on every call', async () => { + const configWithMessages: LDAICompletionConfig = { + ...baseConfig, + messages: [{ role: 'system', content: 'You are helpful.' }], + }; + const r = new LangChainModelRunner(mockLLM, configWithMessages, mockLogger); + + mockLLM.invoke + .mockResolvedValueOnce(new AIMessage('Answer 1')) + .mockResolvedValueOnce(new AIMessage('Answer 2')); + + await r.run('Q1'); + await r.run('Q2'); + + const secondCallMessages = mockLLM.invoke.mock.calls[1][0]; + expect(secondCallMessages).toHaveLength(4); + expect(secondCallMessages[0].constructor.name).toBe('SystemMessage'); + expect(secondCallMessages[0].content).toBe('You are helpful.'); + expect(secondCallMessages[1].content).toBe('Q1'); + expect(secondCallMessages[2].content).toBe('Answer 1'); + expect(secondCallMessages[3].content).toBe('Q2'); + }); + }); }); diff --git a/packages/ai-providers/server-ai-langchain/src/LangChainModelRunner.ts b/packages/ai-providers/server-ai-langchain/src/LangChainModelRunner.ts index a53c4255d1..57f2e0d4f2 100644 --- a/packages/ai-providers/server-ai-langchain/src/LangChainModelRunner.ts +++ b/packages/ai-providers/server-ai-langchain/src/LangChainModelRunner.ts @@ -1,10 +1,10 @@ +import { InMemoryChatMessageHistory } from '@langchain/core/chat_history'; import { BaseChatModel } from '@langchain/core/language_models/chat_models'; -import { AIMessage } from '@langchain/core/messages'; +import { AIMessage, BaseMessage, HumanMessage } from '@langchain/core/messages'; import type { LDAICompletionConfig, LDLogger, - LDMessage, Runner, RunnerResult, } from '@launchdarkly/server-sdk-ai'; @@ -19,35 +19,49 @@ import { convertMessagesToLangChain, getAIMetricsFromResponse } from './LangChai */ export class LangChainModelRunner implements Runner { private _llm: BaseChatModel; - private _config: LDAICompletionConfig; + private _chatHistory: InMemoryChatMessageHistory; private _logger?: LDLogger; constructor(llm: BaseChatModel, config: LDAICompletionConfig, logger?: LDLogger) { this._llm = llm; - this._config = config; + this._chatHistory = new InMemoryChatMessageHistory( + convertMessagesToLangChain(config.messages ?? []), + ); this._logger = logger; } /** * Run the LangChain model with the given user prompt. * - * Prepends any messages defined in the AI config (system prompt, etc.) before - * the user prompt. + * The runner maintains a LangChain `InMemoryChatMessageHistory` that is + * initialized from any messages on the AI config (system prompt, etc.) and + * grows with each successful call. On every invocation the user prompt is + * appended to the existing history before being sent to the model. When the + * call succeeds and produces non-empty content, the user prompt and the + * assistant's reply are persisted to the history; failed calls leave the + * history unchanged so the next call can retry cleanly. * * @param input The user prompt string. * @param outputType Optional JSON schema for structured output. When provided, * the parsed result is exposed via {@link RunnerResult.parsed}. */ async run(input: string, outputType?: Record): Promise { - const messages: LDMessage[] = [ - ...(this._config.messages ?? []), - { role: 'user', content: input }, + const langchainMessages: BaseMessage[] = [ + ...(await this._chatHistory.getMessages()), + new HumanMessage(input), ]; - if (outputType !== undefined) { - return this._runStructured(messages, outputType); + const result = + outputType !== undefined + ? await this._runStructured(langchainMessages, outputType) + : await this._runCompletion(langchainMessages); + + if (result.metrics.success && result.content) { + await this._chatHistory.addUserMessage(input); + await this._chatHistory.addAIMessage(result.content); } - return this._runCompletion(messages); + + return result; } /** @@ -57,10 +71,9 @@ export class LangChainModelRunner implements Runner { return this._llm; } - private async _runCompletion(messages: LDMessage[]): Promise { + private async _runCompletion(messages: BaseMessage[]): Promise { try { - const langchainMessages = convertMessagesToLangChain(messages); - const response: AIMessage = await this._llm.invoke(langchainMessages); + const response: AIMessage = await this._llm.invoke(messages); const metrics = getAIMetricsFromResponse(response); let content: string = ''; @@ -85,14 +98,13 @@ export class LangChainModelRunner implements Runner { } private async _runStructured( - messages: LDMessage[], + messages: BaseMessage[], outputType: Record, ): Promise { try { - const langchainMessages = convertMessagesToLangChain(messages); const response = (await this._llm .withStructuredOutput(outputType) - .invoke(langchainMessages)) as Record; + .invoke(messages)) as Record; const metrics = { success: true, diff --git a/packages/ai-providers/server-ai-openai/__tests__/OpenAIModelRunner.test.ts b/packages/ai-providers/server-ai-openai/__tests__/OpenAIModelRunner.test.ts index 93e4aebff9..b4b44c9cb1 100644 --- a/packages/ai-providers/server-ai-openai/__tests__/OpenAIModelRunner.test.ts +++ b/packages/ai-providers/server-ai-openai/__tests__/OpenAIModelRunner.test.ts @@ -148,4 +148,87 @@ describe('OpenAIModelRunner', () => { expect(runner.getClient()).toBe(mockOpenAI); }); }); + + describe('conversation history', () => { + it('accumulates history across successful calls', async () => { + (mockOpenAI.chat.completions.create as jest.Mock) + .mockResolvedValueOnce({ + choices: [{ message: { content: 'First response' } }], + usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 }, + } as any) + .mockResolvedValueOnce({ + choices: [{ message: { content: 'Second response' } }], + usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 }, + } as any); + + await runner.run('First question'); + await runner.run('Second question'); + + const secondCallArgs = (mockOpenAI.chat.completions.create as jest.Mock).mock.calls[1][0]; + expect(secondCallArgs.messages).toEqual([ + { role: 'user', content: 'First question' }, + { role: 'assistant', content: 'First response' }, + { role: 'user', content: 'Second question' }, + ]); + }); + + it('does not accumulate history when the call throws', async () => { + (mockOpenAI.chat.completions.create as jest.Mock).mockRejectedValueOnce(new Error('boom')); + await runner.run('Hello!'); + + (mockOpenAI.chat.completions.create as jest.Mock).mockResolvedValueOnce({ + choices: [{ message: { content: 'Recovery' } }], + usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 }, + } as any); + await runner.run('Try again'); + + const secondCallArgs = (mockOpenAI.chat.completions.create as jest.Mock).mock.calls[1][0]; + expect(secondCallArgs.messages).toEqual([{ role: 'user', content: 'Try again' }]); + }); + + it('does not accumulate history when content is empty', async () => { + (mockOpenAI.chat.completions.create as jest.Mock).mockResolvedValueOnce({ + choices: [{ message: {} }], + } as any); + await runner.run('Hello!'); + + (mockOpenAI.chat.completions.create as jest.Mock).mockResolvedValueOnce({ + choices: [{ message: { content: 'Recovery' } }], + usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 }, + } as any); + await runner.run('Try again'); + + const secondCallArgs = (mockOpenAI.chat.completions.create as jest.Mock).mock.calls[1][0]; + expect(secondCallArgs.messages).toEqual([{ role: 'user', content: 'Try again' }]); + }); + + it('keeps config messages prepended ahead of accumulated history on every call', async () => { + const configWithMessages: LDAICompletionConfig = { + ...baseConfig, + messages: [{ role: 'system', content: 'You are helpful.' }], + }; + const r = new OpenAIModelRunner(mockOpenAI, configWithMessages); + + (mockOpenAI.chat.completions.create as jest.Mock) + .mockResolvedValueOnce({ + choices: [{ message: { content: 'Answer 1' } }], + usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 }, + } as any) + .mockResolvedValueOnce({ + choices: [{ message: { content: 'Answer 2' } }], + usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 }, + } as any); + + await r.run('Q1'); + await r.run('Q2'); + + const secondCallArgs = (mockOpenAI.chat.completions.create as jest.Mock).mock.calls[1][0]; + expect(secondCallArgs.messages).toEqual([ + { role: 'system', content: 'You are helpful.' }, + { role: 'user', content: 'Q1' }, + { role: 'assistant', content: 'Answer 1' }, + { role: 'user', content: 'Q2' }, + ]); + }); + }); }); diff --git a/packages/ai-providers/server-ai-openai/src/OpenAIModelRunner.ts b/packages/ai-providers/server-ai-openai/src/OpenAIModelRunner.ts index 7f8b874e3b..62882455d5 100644 --- a/packages/ai-providers/server-ai-openai/src/OpenAIModelRunner.ts +++ b/packages/ai-providers/server-ai-openai/src/OpenAIModelRunner.ts @@ -18,39 +18,49 @@ import { convertMessagesToOpenAI, getAIMetricsFromResponse } from './OpenAIHelpe */ export class OpenAIModelRunner implements Runner { private _client: OpenAI; - private _config: LDAICompletionConfig; private _modelName: string; private _parameters: Record; + private _history: LDMessage[]; private _logger?: LDLogger; constructor(client: OpenAI, config: LDAICompletionConfig, logger?: LDLogger) { this._client = client; - this._config = config; this._modelName = config.model?.name ?? ''; this._parameters = { ...(config.model?.parameters ?? {}) }; + this._history = [...(config.messages ?? [])]; this._logger = logger; } /** * Run the OpenAI model with the given user prompt. * - * Prepends any messages defined in the AI config (system prompt, - * instructions, etc.) before the user prompt. + * The runner maintains a conversation history that is initialized from any + * messages on the AI config (system prompt, instructions, etc.) and grows + * with each successful call. On every invocation the user prompt is appended + * to the existing history before being sent to the model. When the call + * succeeds and produces non-empty content, the user prompt and the + * assistant's reply are persisted to the history; failed calls leave the + * history unchanged so the next call can retry cleanly. * * @param input The user prompt string. * @param outputType Optional JSON schema for structured output. When provided, * the response is parsed and exposed via {@link RunnerResult.parsed}. */ async run(input: string, outputType?: Record): Promise { - const messages: LDMessage[] = [ - ...(this._config.messages ?? []), - { role: 'user', content: input }, - ]; + const userMessage: LDMessage = { role: 'user', content: input }; + const messages: LDMessage[] = [...this._history, userMessage]; - if (outputType !== undefined) { - return this._runStructured(messages, outputType); + const result = + outputType !== undefined + ? await this._runStructured(messages, outputType) + : await this._runCompletion(messages); + + if (result.metrics.success && result.content) { + this._history.push(userMessage); + this._history.push({ role: 'assistant', content: result.content }); } - return this._runCompletion(messages); + + return result; } /** diff --git a/packages/ai-providers/server-ai-vercel/__tests__/VercelModelRunner.test.ts b/packages/ai-providers/server-ai-vercel/__tests__/VercelModelRunner.test.ts index f7beb6f4d4..f781ec9eff 100644 --- a/packages/ai-providers/server-ai-vercel/__tests__/VercelModelRunner.test.ts +++ b/packages/ai-providers/server-ai-vercel/__tests__/VercelModelRunner.test.ts @@ -146,4 +146,89 @@ describe('VercelModelRunner', () => { expect(runner.getModel()).toBe(fakeModel); }); }); + + describe('conversation history', () => { + it('accumulates history across successful calls', async () => { + (generateText as jest.Mock) + .mockResolvedValueOnce({ + text: 'First response', + usage: { totalTokens: 2, promptTokens: 1, completionTokens: 1 }, + }) + .mockResolvedValueOnce({ + text: 'Second response', + usage: { totalTokens: 2, promptTokens: 1, completionTokens: 1 }, + }); + + await runner.run('First question'); + await runner.run('Second question'); + + const secondCallArgs = (generateText as jest.Mock).mock.calls[1][0]; + expect(secondCallArgs.messages).toEqual([ + { role: 'user', content: 'First question' }, + { role: 'assistant', content: 'First response' }, + { role: 'user', content: 'Second question' }, + ]); + }); + + it('does not accumulate history when the call throws', async () => { + (generateText as jest.Mock).mockRejectedValueOnce(new Error('boom')); + await runner.run('Hello'); + + (generateText as jest.Mock).mockResolvedValueOnce({ + text: 'Recovery', + usage: { totalTokens: 2, promptTokens: 1, completionTokens: 1 }, + }); + await runner.run('Try again'); + + const secondCallArgs = (generateText as jest.Mock).mock.calls[1][0]; + expect(secondCallArgs.messages).toEqual([{ role: 'user', content: 'Try again' }]); + }); + + it('does not accumulate history when content is empty', async () => { + (generateText as jest.Mock).mockResolvedValueOnce({ + text: '', + finishReason: 'error', + usage: { totalTokens: 0, promptTokens: 0, completionTokens: 0 }, + }); + await runner.run('Hello'); + + (generateText as jest.Mock).mockResolvedValueOnce({ + text: 'Recovery', + usage: { totalTokens: 2, promptTokens: 1, completionTokens: 1 }, + }); + await runner.run('Try again'); + + const secondCallArgs = (generateText as jest.Mock).mock.calls[1][0]; + expect(secondCallArgs.messages).toEqual([{ role: 'user', content: 'Try again' }]); + }); + + it('keeps config messages prepended ahead of accumulated history on every call', async () => { + const configWithMessages: LDAICompletionConfig = { + ...baseConfig, + messages: [{ role: 'system', content: 'You are helpful.' }], + }; + const r = new VercelModelRunner(fakeModel as any, configWithMessages, {}, mockLogger); + + (generateText as jest.Mock) + .mockResolvedValueOnce({ + text: 'Answer 1', + usage: { totalTokens: 2, promptTokens: 1, completionTokens: 1 }, + }) + .mockResolvedValueOnce({ + text: 'Answer 2', + usage: { totalTokens: 2, promptTokens: 1, completionTokens: 1 }, + }); + + await r.run('Q1'); + await r.run('Q2'); + + const secondCallArgs = (generateText as jest.Mock).mock.calls[1][0]; + expect(secondCallArgs.messages).toEqual([ + { role: 'system', content: 'You are helpful.' }, + { role: 'user', content: 'Q1' }, + { role: 'assistant', content: 'Answer 1' }, + { role: 'user', content: 'Q2' }, + ]); + }); + }); }); diff --git a/packages/ai-providers/server-ai-vercel/src/VercelModelRunner.ts b/packages/ai-providers/server-ai-vercel/src/VercelModelRunner.ts index 54e7db3472..2c842f324b 100644 --- a/packages/ai-providers/server-ai-vercel/src/VercelModelRunner.ts +++ b/packages/ai-providers/server-ai-vercel/src/VercelModelRunner.ts @@ -1,9 +1,8 @@ -import { generateObject, generateText, jsonSchema, LanguageModel } from 'ai'; +import { generateObject, generateText, jsonSchema, LanguageModel, ModelMessage } from 'ai'; import type { LDAICompletionConfig, LDLogger, - LDMessage, Runner, RunnerResult, } from '@launchdarkly/server-sdk-ai'; @@ -19,8 +18,8 @@ import { convertMessagesToVercel, getAIMetricsFromResponse } from './VercelHelpe */ export class VercelModelRunner implements Runner { private _model: LanguageModel; - private _config: LDAICompletionConfig; private _parameters: VercelAIModelParameters; + private _history: ModelMessage[]; private _logger?: LDLogger; constructor( @@ -30,31 +29,42 @@ export class VercelModelRunner implements Runner { logger?: LDLogger, ) { this._model = model; - this._config = config; this._parameters = parameters; + this._history = convertMessagesToVercel(config.messages ?? []) as ModelMessage[]; this._logger = logger; } /** * Run the Vercel AI model with the given user prompt. * - * Prepends any messages defined in the AI config (system prompt, etc.) before - * the user prompt. + * The runner maintains a conversation history (as Vercel AI SDK + * `ModelMessage`s) that is initialized from any messages on the AI config + * (system prompt, etc.) and grows with each successful call. On every + * invocation the user prompt is appended to the existing history before + * being sent to the model. When the call succeeds and produces non-empty + * content, the user prompt and the assistant's reply are persisted to the + * history; failed calls leave the history unchanged so the next call can + * retry cleanly. * * @param input The user prompt string. * @param outputType Optional JSON schema for structured output. When provided, * the parsed object is exposed via {@link RunnerResult.parsed}. */ async run(input: string, outputType?: Record): Promise { - const messages: LDMessage[] = [ - ...(this._config.messages ?? []), - { role: 'user', content: input }, - ]; + const userMessage: ModelMessage = { role: 'user', content: input }; + const messages: ModelMessage[] = [...this._history, userMessage]; - if (outputType !== undefined) { - return this._runStructured(messages, outputType); + const result = + outputType !== undefined + ? await this._runStructured(messages, outputType) + : await this._runCompletion(messages); + + if (result.metrics.success && result.content) { + this._history.push(userMessage); + this._history.push({ role: 'assistant', content: result.content }); } - return this._runCompletion(messages); + + return result; } /** @@ -64,12 +74,12 @@ export class VercelModelRunner implements Runner { return this._model; } - private async _runCompletion(messages: LDMessage[]): Promise { + private async _runCompletion(messages: ModelMessage[]): Promise { try { const result = await generateText({ ...this._parameters, model: this._model, - messages: convertMessagesToVercel(messages), + messages, experimental_telemetry: { isEnabled: true }, }); @@ -85,14 +95,14 @@ export class VercelModelRunner implements Runner { } private async _runStructured( - messages: LDMessage[], + messages: ModelMessage[], outputType: Record, ): Promise { try { const result = await generateObject({ ...this._parameters, model: this._model, - messages: convertMessagesToVercel(messages), + messages, schema: jsonSchema(outputType), experimental_telemetry: { isEnabled: true }, });