diff --git a/api/package.json b/api/package.json index 27b076a7eb3..bd53c5a4837 100644 --- a/api/package.json +++ b/api/package.json @@ -39,7 +39,7 @@ "@azure/storage-blob": "^12.30.0", "@google/genai": "^2.8.0", "@keyv/redis": "5.1.6", - "@librechat/agents": "^4.0.4", + "@librechat/agents": "^4.0.5", "@librechat/api": "*", "@librechat/data-schemas": "*", "@microsoft/microsoft-graph-client": "^3.0.7", diff --git a/package-lock.json b/package-lock.json index 4d0be82cc56..2dcc7f89751 100644 --- a/package-lock.json +++ b/package-lock.json @@ -65,7 +65,7 @@ "@azure/storage-blob": "^12.30.0", "@google/genai": "^2.8.0", "@keyv/redis": "5.1.6", - "@librechat/agents": "^4.0.4", + "@librechat/agents": "^4.0.5", "@librechat/api": "*", "@librechat/data-schemas": "*", "@microsoft/microsoft-graph-client": "^3.0.7", @@ -9216,9 +9216,9 @@ } }, "node_modules/@librechat/agents": { - "version": "4.0.4", - "resolved": "https://registry.npmjs.org/@librechat/agents/-/agents-4.0.4.tgz", - "integrity": "sha512-gVBBlqCmEimwgBk+pfHogTTksl76vjv9YRI4V1VnSIumSk8dCsofrjHW/4Fz4zZHeqHa10eJbKtJatukQXwFhA==", + "version": "4.0.5", + "resolved": "https://registry.npmjs.org/@librechat/agents/-/agents-4.0.5.tgz", + "integrity": "sha512-huoNIKPTiYviihqkXJB9/Ht4aG5YkNBSMrkGqOvtuspyZgSlqQNTJIU6qGzOGS6phNohP1dANPkH+qkRpWGPyg==", "license": "MIT", "dependencies": { "@anthropic-ai/sdk": "^0.115.0", @@ -40791,7 +40791,7 @@ "@azure/storage-blob": "^12.30.0", "@google/genai": "^2.8.0", "@keyv/redis": "5.1.6", - "@librechat/agents": "^4.0.4", + "@librechat/agents": "^4.0.5", "@librechat/data-schemas": "*", "@modelcontextprotocol/sdk": "^1.32.1", "@opentelemetry/api": "^1.9.0", diff --git a/packages/api/package.json b/packages/api/package.json index 28c6d9c5867..c193af9eab2 100644 --- a/packages/api/package.json +++ b/packages/api/package.json @@ -120,7 +120,7 @@ "@azure/storage-blob": "^12.30.0", "@google/genai": "^2.8.0", "@keyv/redis": "5.1.6", - "@librechat/agents": "^4.0.4", + "@librechat/agents": "^4.0.5", "@librechat/data-schemas": "*", "@modelcontextprotocol/sdk": "^1.32.1", "@opentelemetry/api": "^1.9.0", diff --git a/packages/api/src/endpoints/bedrock/harmony.spec.ts b/packages/api/src/endpoints/bedrock/harmony.spec.ts new file mode 100644 index 00000000000..ad33e6de4dc --- /dev/null +++ b/packages/api/src/endpoints/bedrock/harmony.spec.ts @@ -0,0 +1,101 @@ +import { FileSources } from 'librechat-data-provider'; +import { BedrockRuntimeClient } from '@aws-sdk/client-bedrock-runtime'; +import { Providers, getChatModelClass, formatAgentMessages } from '@librechat/agents'; +import type { ConverseStreamCommand, ConverseStreamOutput } from '@aws-sdk/client-bedrock-runtime'; +import type { IMongoFile } from '@librechat/data-schemas'; +import type { ServerRequest } from '~/types'; +import { prependFileContext } from '~/agents/client'; +import { extractFileContext } from '~/files/context'; + +/** + * An OpenAI model on Bedrock answers a request whose text carries a harmony header with a + * `200 OK` event stream that closes before any event, which crashed the turn. LibreChat sends a + * `text/plain` attachment inline as file context, so the header reaches every later turn. + */ +const HEADER = '<|start|>assistant<|channel|>analysis<|message|>thinking here<|end|>'; +const NEUTRALIZED = '<|start|>assistant<|channel|>analysis<|message|>thinking here<|end|>'; + +const RECORDED_EVENTS: ConverseStreamOutput[] = [ + { messageStart: { role: 'assistant' } }, + { contentBlockDelta: { delta: { text: 'analysis' }, contentBlockIndex: 0 } }, + { contentBlockStop: { contentBlockIndex: 0 } }, + { messageStop: { stopReason: 'end_turn' } }, + { + metadata: { + usage: { inputTokens: 96, outputTokens: 5, totalTokens: 101 }, + metrics: { latencyMs: 0 }, + }, + }, +]; + +async function* streamOf(events: ConverseStreamOutput[]): AsyncGenerator { + for (const event of events) { + yield event; + } +} + +async function formatPastedTextTurn(): Promise['messages']> { + const attachment = { + filename: 'pasted-text.txt', + type: 'text/plain', + source: FileSources.local, + llmDeliveryPath: 'text', + text: `Notes on the format:\n${HEADER}`, + } as IMongoFile; + const fileContext = await extractFileContext({ + attachments: [attachment], + req: { body: { fileTokenLimit: 10000 }, config: { fileConfig: {} } } as ServerRequest, + tokenCountFn: (text: string) => text.length, + }); + const formattedMessage = { + role: 'user', + content: [{ type: 'text' as const, text: 'Which channel does the example use?' }], + }; + prependFileContext(formattedMessage, fileContext); + return formatAgentMessages([formattedMessage]).messages; +} + +function createModel(model: string): { + chat: InstanceType>; + send: jest.SpyInstance; +} { + const client = new BedrockRuntimeClient({ + region: 'us-west-2', + credentials: { accessKeyId: 'test', secretAccessKey: 'test' }, + }); + const send = jest + .spyOn(client, 'send') + .mockImplementation(async () => ({ stream: streamOf(RECORDED_EVENTS), $metadata: {} })); + const ChatBedrock = getChatModelClass(Providers.BEDROCK); + const chat = new ChatBedrock({ model, region: 'us-west-2', client }); + return { chat, send }; +} + +async function sentUserText(model: string): Promise { + const { chat, send } = createModel(model); + const stream = await chat.stream(await formatPastedTextTurn()); + for await (const chunk of stream) { + expect(chunk).toBeDefined(); + } + const command = send.mock.calls[0][0] as ConverseStreamCommand; + return command.input.messages?.[0].content?.[0].text; +} + +describe('pasted text attachment sent to Bedrock', () => { + afterEach(() => { + jest.restoreAllMocks(); + }); + + it('neutralizes harmony headers for an OpenAI model', async () => { + const text = await sentUserText('global.openai.gpt-6.1-sol'); + + expect(text).toContain(`# "pasted-text.txt"\nNotes on the format:\n${NEUTRALIZED}`); + expect(text).not.toContain('<|'); + }); + + it('sends the attachment text unchanged to other model families', async () => { + const text = await sentUserText('global.anthropic.claude-sonnet-5-5'); + + expect(text).toContain(`# "pasted-text.txt"\nNotes on the format:\n${HEADER}`); + }); +});