diff --git a/packages/ai-providers/server-ai-langchain/__tests__/LangChainModelRunner.test.ts b/packages/ai-providers/server-ai-langchain/__tests__/LangChainModelRunner.test.ts index 73dbd940c6..b48df2046f 100644 --- a/packages/ai-providers/server-ai-langchain/__tests__/LangChainModelRunner.test.ts +++ b/packages/ai-providers/server-ai-langchain/__tests__/LangChainModelRunner.test.ts @@ -115,4 +115,69 @@ describe('LangChainModelRunner', () => { it('returns the underlying chat model', () => { expect(runner.getChatModel()).toBe(mockLLM); }); + + describe('conversation history', () => { + it('accumulates history across successful calls', async () => { + mockLLM.invoke + .mockResolvedValueOnce(new AIMessage('First response')) + .mockResolvedValueOnce(new AIMessage('Second response')); + + await runner.run('First question'); + await runner.run('Second question'); + + const secondCallMessages = mockLLM.invoke.mock.calls[1][0]; + const roles = secondCallMessages.map((m: any) => m.constructor.name); + expect(roles).toEqual(['HumanMessage', 'AIMessage', 'HumanMessage']); + expect(secondCallMessages[0].content).toBe('First question'); + expect(secondCallMessages[1].content).toBe('First response'); + expect(secondCallMessages[2].content).toBe('Second question'); + }); + + it('does not accumulate history when the call throws', async () => { + mockLLM.invoke.mockRejectedValueOnce(new Error('Model error')); + await runner.run('Hello'); + + mockLLM.invoke.mockResolvedValueOnce(new AIMessage('Recovery')); + await runner.run('Try again'); + + const secondCallMessages = mockLLM.invoke.mock.calls[1][0]; + expect(secondCallMessages).toHaveLength(1); + expect(secondCallMessages[0].content).toBe('Try again'); + }); + + it('does not accumulate history when content is empty (multimodal)', async () => { + mockLLM.invoke.mockResolvedValueOnce(new AIMessage([{ type: 'image' }] as any)); + await runner.run('Hello'); + + mockLLM.invoke.mockResolvedValueOnce(new AIMessage('Recovery')); + await runner.run('Try again'); + + const secondCallMessages = mockLLM.invoke.mock.calls[1][0]; + expect(secondCallMessages).toHaveLength(1); + expect(secondCallMessages[0].content).toBe('Try again'); + }); + + it('keeps config messages prepended ahead of accumulated history on every call', async () => { + const configWithMessages: LDAICompletionConfig = { + ...baseConfig, + messages: [{ role: 'system', content: 'You are helpful.' }], + }; + const r = new LangChainModelRunner(mockLLM, configWithMessages, mockLogger); + + mockLLM.invoke + .mockResolvedValueOnce(new AIMessage('Answer 1')) + .mockResolvedValueOnce(new AIMessage('Answer 2')); + + await r.run('Q1'); + await r.run('Q2'); + + const secondCallMessages = mockLLM.invoke.mock.calls[1][0]; + expect(secondCallMessages).toHaveLength(4); + expect(secondCallMessages[0].constructor.name).toBe('SystemMessage'); + expect(secondCallMessages[0].content).toBe('You are helpful.'); + expect(secondCallMessages[1].content).toBe('Q1'); + expect(secondCallMessages[2].content).toBe('Answer 1'); + expect(secondCallMessages[3].content).toBe('Q2'); + }); + }); }); diff --git a/packages/ai-providers/server-ai-langchain/src/LangChainModelRunner.ts b/packages/ai-providers/server-ai-langchain/src/LangChainModelRunner.ts index a53c4255d1..57f2e0d4f2 100644 --- a/packages/ai-providers/server-ai-langchain/src/LangChainModelRunner.ts +++ b/packages/ai-providers/server-ai-langchain/src/LangChainModelRunner.ts @@ -1,10 +1,10 @@ +import { InMemoryChatMessageHistory } from '@langchain/core/chat_history'; import { BaseChatModel } from '@langchain/core/language_models/chat_models'; -import { AIMessage } from '@langchain/core/messages'; +import { AIMessage, BaseMessage, HumanMessage } from '@langchain/core/messages'; import type { LDAICompletionConfig, LDLogger, - LDMessage, Runner, RunnerResult, } from '@launchdarkly/server-sdk-ai'; @@ -19,35 +19,49 @@ import { convertMessagesToLangChain, getAIMetricsFromResponse } from './LangChai */ export class LangChainModelRunner implements Runner { private _llm: BaseChatModel; - private _config: LDAICompletionConfig; + private _chatHistory: InMemoryChatMessageHistory; private _logger?: LDLogger; constructor(llm: BaseChatModel, config: LDAICompletionConfig, logger?: LDLogger) { this._llm = llm; - this._config = config; + this._chatHistory = new InMemoryChatMessageHistory( + convertMessagesToLangChain(config.messages ?? []), + ); this._logger = logger; } /** * Run the LangChain model with the given user prompt. * - * Prepends any messages defined in the AI config (system prompt, etc.) before - * the user prompt. + * The runner maintains a LangChain `InMemoryChatMessageHistory` that is + * initialized from any messages on the AI config (system prompt, etc.) and + * grows with each successful call. On every invocation the user prompt is + * appended to the existing history before being sent to the model. When the + * call succeeds and produces non-empty content, the user prompt and the + * assistant's reply are persisted to the history; failed calls leave the + * history unchanged so the next call can retry cleanly. * * @param input The user prompt string. * @param outputType Optional JSON schema for structured output. When provided, * the parsed result is exposed via {@link RunnerResult.parsed}. */ async run(input: string, outputType?: Record): Promise { - const messages: LDMessage[] = [ - ...(this._config.messages ?? []), - { role: 'user', content: input }, + const langchainMessages: BaseMessage[] = [ + ...(await this._chatHistory.getMessages()), + new HumanMessage(input), ]; - if (outputType !== undefined) { - return this._runStructured(messages, outputType); + const result = + outputType !== undefined + ? await this._runStructured(langchainMessages, outputType) + : await this._runCompletion(langchainMessages); + + if (result.metrics.success && result.content) { + await this._chatHistory.addUserMessage(input); + await this._chatHistory.addAIMessage(result.content); } - return this._runCompletion(messages); + + return result; } /** @@ -57,10 +71,9 @@ export class LangChainModelRunner implements Runner { return this._llm; } - private async _runCompletion(messages: LDMessage[]): Promise { + private async _runCompletion(messages: BaseMessage[]): Promise { try { - const langchainMessages = convertMessagesToLangChain(messages); - const response: AIMessage = await this._llm.invoke(langchainMessages); + const response: AIMessage = await this._llm.invoke(messages); const metrics = getAIMetricsFromResponse(response); let content: string = ''; @@ -85,14 +98,13 @@ export class LangChainModelRunner implements Runner { } private async _runStructured( - messages: LDMessage[], + messages: BaseMessage[], outputType: Record, ): Promise { try { - const langchainMessages = convertMessagesToLangChain(messages); const response = (await this._llm .withStructuredOutput(outputType) - .invoke(langchainMessages)) as Record; + .invoke(messages)) as Record; const metrics = { success: true, diff --git a/packages/ai-providers/server-ai-openai/__tests__/OpenAIModelRunner.test.ts b/packages/ai-providers/server-ai-openai/__tests__/OpenAIModelRunner.test.ts index 93e4aebff9..b4b44c9cb1 100644 --- a/packages/ai-providers/server-ai-openai/__tests__/OpenAIModelRunner.test.ts +++ b/packages/ai-providers/server-ai-openai/__tests__/OpenAIModelRunner.test.ts @@ -148,4 +148,87 @@ describe('OpenAIModelRunner', () => { expect(runner.getClient()).toBe(mockOpenAI); }); }); + + describe('conversation history', () => { + it('accumulates history across successful calls', async () => { + (mockOpenAI.chat.completions.create as jest.Mock) + .mockResolvedValueOnce({ + choices: [{ message: { content: 'First response' } }], + usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 }, + } as any) + .mockResolvedValueOnce({ + choices: [{ message: { content: 'Second response' } }], + usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 }, + } as any); + + await runner.run('First question'); + await runner.run('Second question'); + + const secondCallArgs = (mockOpenAI.chat.completions.create as jest.Mock).mock.calls[1][0]; + expect(secondCallArgs.messages).toEqual([ + { role: 'user', content: 'First question' }, + { role: 'assistant', content: 'First response' }, + { role: 'user', content: 'Second question' }, + ]); + }); + + it('does not accumulate history when the call throws', async () => { + (mockOpenAI.chat.completions.create as jest.Mock).mockRejectedValueOnce(new Error('boom')); + await runner.run('Hello!'); + + (mockOpenAI.chat.completions.create as jest.Mock).mockResolvedValueOnce({ + choices: [{ message: { content: 'Recovery' } }], + usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 }, + } as any); + await runner.run('Try again'); + + const secondCallArgs = (mockOpenAI.chat.completions.create as jest.Mock).mock.calls[1][0]; + expect(secondCallArgs.messages).toEqual([{ role: 'user', content: 'Try again' }]); + }); + + it('does not accumulate history when content is empty', async () => { + (mockOpenAI.chat.completions.create as jest.Mock).mockResolvedValueOnce({ + choices: [{ message: {} }], + } as any); + await runner.run('Hello!'); + + (mockOpenAI.chat.completions.create as jest.Mock).mockResolvedValueOnce({ + choices: [{ message: { content: 'Recovery' } }], + usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 }, + } as any); + await runner.run('Try again'); + + const secondCallArgs = (mockOpenAI.chat.completions.create as jest.Mock).mock.calls[1][0]; + expect(secondCallArgs.messages).toEqual([{ role: 'user', content: 'Try again' }]); + }); + + it('keeps config messages prepended ahead of accumulated history on every call', async () => { + const configWithMessages: LDAICompletionConfig = { + ...baseConfig, + messages: [{ role: 'system', content: 'You are helpful.' }], + }; + const r = new OpenAIModelRunner(mockOpenAI, configWithMessages); + + (mockOpenAI.chat.completions.create as jest.Mock) + .mockResolvedValueOnce({ + choices: [{ message: { content: 'Answer 1' } }], + usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 }, + } as any) + .mockResolvedValueOnce({ + choices: [{ message: { content: 'Answer 2' } }], + usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 }, + } as any); + + await r.run('Q1'); + await r.run('Q2'); + + const secondCallArgs = (mockOpenAI.chat.completions.create as jest.Mock).mock.calls[1][0]; + expect(secondCallArgs.messages).toEqual([ + { role: 'system', content: 'You are helpful.' }, + { role: 'user', content: 'Q1' }, + { role: 'assistant', content: 'Answer 1' }, + { role: 'user', content: 'Q2' }, + ]); + }); + }); }); diff --git a/packages/ai-providers/server-ai-openai/src/OpenAIModelRunner.ts b/packages/ai-providers/server-ai-openai/src/OpenAIModelRunner.ts index 7f8b874e3b..62882455d5 100644 --- a/packages/ai-providers/server-ai-openai/src/OpenAIModelRunner.ts +++ b/packages/ai-providers/server-ai-openai/src/OpenAIModelRunner.ts @@ -18,39 +18,49 @@ import { convertMessagesToOpenAI, getAIMetricsFromResponse } from './OpenAIHelpe */ export class OpenAIModelRunner implements Runner { private _client: OpenAI; - private _config: LDAICompletionConfig; private _modelName: string; private _parameters: Record; + private _history: LDMessage[]; private _logger?: LDLogger; constructor(client: OpenAI, config: LDAICompletionConfig, logger?: LDLogger) { this._client = client; - this._config = config; this._modelName = config.model?.name ?? ''; this._parameters = { ...(config.model?.parameters ?? {}) }; + this._history = [...(config.messages ?? [])]; this._logger = logger; } /** * Run the OpenAI model with the given user prompt. * - * Prepends any messages defined in the AI config (system prompt, - * instructions, etc.) before the user prompt. + * The runner maintains a conversation history that is initialized from any + * messages on the AI config (system prompt, instructions, etc.) and grows + * with each successful call. On every invocation the user prompt is appended + * to the existing history before being sent to the model. When the call + * succeeds and produces non-empty content, the user prompt and the + * assistant's reply are persisted to the history; failed calls leave the + * history unchanged so the next call can retry cleanly. * * @param input The user prompt string. * @param outputType Optional JSON schema for structured output. When provided, * the response is parsed and exposed via {@link RunnerResult.parsed}. */ async run(input: string, outputType?: Record): Promise { - const messages: LDMessage[] = [ - ...(this._config.messages ?? []), - { role: 'user', content: input }, - ]; + const userMessage: LDMessage = { role: 'user', content: input }; + const messages: LDMessage[] = [...this._history, userMessage]; - if (outputType !== undefined) { - return this._runStructured(messages, outputType); + const result = + outputType !== undefined + ? await this._runStructured(messages, outputType) + : await this._runCompletion(messages); + + if (result.metrics.success && result.content) { + this._history.push(userMessage); + this._history.push({ role: 'assistant', content: result.content }); } - return this._runCompletion(messages); + + return result; } /** diff --git a/packages/ai-providers/server-ai-vercel/__tests__/VercelModelRunner.test.ts b/packages/ai-providers/server-ai-vercel/__tests__/VercelModelRunner.test.ts index f7beb6f4d4..f781ec9eff 100644 --- a/packages/ai-providers/server-ai-vercel/__tests__/VercelModelRunner.test.ts +++ b/packages/ai-providers/server-ai-vercel/__tests__/VercelModelRunner.test.ts @@ -146,4 +146,89 @@ describe('VercelModelRunner', () => { expect(runner.getModel()).toBe(fakeModel); }); }); + + describe('conversation history', () => { + it('accumulates history across successful calls', async () => { + (generateText as jest.Mock) + .mockResolvedValueOnce({ + text: 'First response', + usage: { totalTokens: 2, promptTokens: 1, completionTokens: 1 }, + }) + .mockResolvedValueOnce({ + text: 'Second response', + usage: { totalTokens: 2, promptTokens: 1, completionTokens: 1 }, + }); + + await runner.run('First question'); + await runner.run('Second question'); + + const secondCallArgs = (generateText as jest.Mock).mock.calls[1][0]; + expect(secondCallArgs.messages).toEqual([ + { role: 'user', content: 'First question' }, + { role: 'assistant', content: 'First response' }, + { role: 'user', content: 'Second question' }, + ]); + }); + + it('does not accumulate history when the call throws', async () => { + (generateText as jest.Mock).mockRejectedValueOnce(new Error('boom')); + await runner.run('Hello'); + + (generateText as jest.Mock).mockResolvedValueOnce({ + text: 'Recovery', + usage: { totalTokens: 2, promptTokens: 1, completionTokens: 1 }, + }); + await runner.run('Try again'); + + const secondCallArgs = (generateText as jest.Mock).mock.calls[1][0]; + expect(secondCallArgs.messages).toEqual([{ role: 'user', content: 'Try again' }]); + }); + + it('does not accumulate history when content is empty', async () => { + (generateText as jest.Mock).mockResolvedValueOnce({ + text: '', + finishReason: 'error', + usage: { totalTokens: 0, promptTokens: 0, completionTokens: 0 }, + }); + await runner.run('Hello'); + + (generateText as jest.Mock).mockResolvedValueOnce({ + text: 'Recovery', + usage: { totalTokens: 2, promptTokens: 1, completionTokens: 1 }, + }); + await runner.run('Try again'); + + const secondCallArgs = (generateText as jest.Mock).mock.calls[1][0]; + expect(secondCallArgs.messages).toEqual([{ role: 'user', content: 'Try again' }]); + }); + + it('keeps config messages prepended ahead of accumulated history on every call', async () => { + const configWithMessages: LDAICompletionConfig = { + ...baseConfig, + messages: [{ role: 'system', content: 'You are helpful.' }], + }; + const r = new VercelModelRunner(fakeModel as any, configWithMessages, {}, mockLogger); + + (generateText as jest.Mock) + .mockResolvedValueOnce({ + text: 'Answer 1', + usage: { totalTokens: 2, promptTokens: 1, completionTokens: 1 }, + }) + .mockResolvedValueOnce({ + text: 'Answer 2', + usage: { totalTokens: 2, promptTokens: 1, completionTokens: 1 }, + }); + + await r.run('Q1'); + await r.run('Q2'); + + const secondCallArgs = (generateText as jest.Mock).mock.calls[1][0]; + expect(secondCallArgs.messages).toEqual([ + { role: 'system', content: 'You are helpful.' }, + { role: 'user', content: 'Q1' }, + { role: 'assistant', content: 'Answer 1' }, + { role: 'user', content: 'Q2' }, + ]); + }); + }); }); diff --git a/packages/ai-providers/server-ai-vercel/src/VercelModelRunner.ts b/packages/ai-providers/server-ai-vercel/src/VercelModelRunner.ts index 54e7db3472..2c842f324b 100644 --- a/packages/ai-providers/server-ai-vercel/src/VercelModelRunner.ts +++ b/packages/ai-providers/server-ai-vercel/src/VercelModelRunner.ts @@ -1,9 +1,8 @@ -import { generateObject, generateText, jsonSchema, LanguageModel } from 'ai'; +import { generateObject, generateText, jsonSchema, LanguageModel, ModelMessage } from 'ai'; import type { LDAICompletionConfig, LDLogger, - LDMessage, Runner, RunnerResult, } from '@launchdarkly/server-sdk-ai'; @@ -19,8 +18,8 @@ import { convertMessagesToVercel, getAIMetricsFromResponse } from './VercelHelpe */ export class VercelModelRunner implements Runner { private _model: LanguageModel; - private _config: LDAICompletionConfig; private _parameters: VercelAIModelParameters; + private _history: ModelMessage[]; private _logger?: LDLogger; constructor( @@ -30,31 +29,42 @@ export class VercelModelRunner implements Runner { logger?: LDLogger, ) { this._model = model; - this._config = config; this._parameters = parameters; + this._history = convertMessagesToVercel(config.messages ?? []) as ModelMessage[]; this._logger = logger; } /** * Run the Vercel AI model with the given user prompt. * - * Prepends any messages defined in the AI config (system prompt, etc.) before - * the user prompt. + * The runner maintains a conversation history (as Vercel AI SDK + * `ModelMessage`s) that is initialized from any messages on the AI config + * (system prompt, etc.) and grows with each successful call. On every + * invocation the user prompt is appended to the existing history before + * being sent to the model. When the call succeeds and produces non-empty + * content, the user prompt and the assistant's reply are persisted to the + * history; failed calls leave the history unchanged so the next call can + * retry cleanly. * * @param input The user prompt string. * @param outputType Optional JSON schema for structured output. When provided, * the parsed object is exposed via {@link RunnerResult.parsed}. */ async run(input: string, outputType?: Record): Promise { - const messages: LDMessage[] = [ - ...(this._config.messages ?? []), - { role: 'user', content: input }, - ]; + const userMessage: ModelMessage = { role: 'user', content: input }; + const messages: ModelMessage[] = [...this._history, userMessage]; - if (outputType !== undefined) { - return this._runStructured(messages, outputType); + const result = + outputType !== undefined + ? await this._runStructured(messages, outputType) + : await this._runCompletion(messages); + + if (result.metrics.success && result.content) { + this._history.push(userMessage); + this._history.push({ role: 'assistant', content: result.content }); } - return this._runCompletion(messages); + + return result; } /** @@ -64,12 +74,12 @@ export class VercelModelRunner implements Runner { return this._model; } - private async _runCompletion(messages: LDMessage[]): Promise { + private async _runCompletion(messages: ModelMessage[]): Promise { try { const result = await generateText({ ...this._parameters, model: this._model, - messages: convertMessagesToVercel(messages), + messages, experimental_telemetry: { isEnabled: true }, }); @@ -85,14 +95,14 @@ export class VercelModelRunner implements Runner { } private async _runStructured( - messages: LDMessage[], + messages: ModelMessage[], outputType: Record, ): Promise { try { const result = await generateObject({ ...this._parameters, model: this._model, - messages: convertMessagesToVercel(messages), + messages, schema: jsonSchema(outputType), experimental_telemetry: { isEnabled: true }, });