diff --git a/api/server/controllers/agents/v1.js b/api/server/controllers/agents/v1.js index 5aad55ad35a..5d49b6bca3c 100644 --- a/api/server/controllers/agents/v1.js +++ b/api/server/controllers/agents/v1.js @@ -19,7 +19,7 @@ const { getAgentListAccess, getAgentListAvatarRefreshKey, mergeAgentOcrConversion, - sanitizeModelParameters, + applyAgentInferenceInput, collectToolResourceFileIds, convertOcrToContextInPlace, normalizeToolResourceFiles, @@ -755,12 +755,7 @@ const createAgentHandler = async (req, res) => { return; } - if (agentData.model_parameters && typeof agentData.model_parameters === 'object') { - agentData.model_parameters = removeNullishValues( - sanitizeModelParameters(agentData.model_parameters), - true, - ); - } + applyAgentInferenceInput(agentData, undefined, req.config); const { id: userId, role: userRole } = req.user; agentData.id = `agent_${nanoid()}`; agentData.edges = replaceEdgeSourceId(agentData.edges, '', agentData.id); @@ -1179,12 +1174,6 @@ const updateAgentHandler = async (req, res) => { } } - if (updateData.model_parameters && typeof updateData.model_parameters === 'object') { - updateData.model_parameters = removeNullishValues( - sanitizeModelParameters(updateData.model_parameters), - true, - ); - } if (avatarField === null) { updateData.avatar = avatarField; } @@ -1237,6 +1226,8 @@ const updateAgentHandler = async (req, res) => { return res.status(404).json({ error: 'Agent not found' }); } + applyAgentInferenceInput(updateData, existingAgent, req.config); + const instructionsPromptError = await checkInstructionsPromptWrite({ access: instructionsPromptAccess, operation: 'update', diff --git a/api/server/controllers/agents/v1.spec.js b/api/server/controllers/agents/v1.spec.js index 68824ce624a..df3e6e36851 100644 --- a/api/server/controllers/agents/v1.spec.js +++ b/api/server/controllers/agents/v1.spec.js @@ -220,6 +220,57 @@ describe('Agent Controllers - Mass Assignment Protection', () => { }; }); + describe('inference input boundary', () => { + test.each(['modelKwargs', 'configuration', 'fallbacks', 'apiKey', 'callbacks'])( + 'rejects %s on creation without persisting or disclosing its value', + async (key) => { + mockReq.body = { + name: 'Rejected settings', + provider: 'openAI', + model: 'gpt-4o-mini', + model_parameters: { temperature: 0.2, [key]: { value: 'submitted-secret' } }, + }; + await createAgentHandler(mockReq, mockRes); + expect(mockRes.status).toHaveBeenCalledWith(400); + expect(await Agent.countDocuments()).toBe(0); + expect(JSON.stringify(mockRes.json.mock.calls)).not.toContain('submitted-secret'); + }, + ); + + test('preserves configured Bedrock region and reasoning controls on creation', async () => { + mockReq.config = { endpoints: { bedrock: { availableRegions: ['us-west-2'] } } }; + mockReq.body = { + name: 'Bedrock inference', + provider: 'bedrock', + model: 'moonshot.kimi', + model_parameters: { region: 'us-west-2', reasoning_effort: 'low' }, + }; + await createAgentHandler(mockReq, mockRes); + expect(mockRes.status).toHaveBeenCalledWith(201); + const stored = await Agent.findOne({ name: 'Bedrock inference' }).lean(); + expect(stored.model_parameters).toEqual({ region: 'us-west-2', reasoning_effort: 'low' }); + }); + + test('rejects an unsafe partial update using the existing provider and retains stored settings', async () => { + const existing = await Agent.create({ + id: 'agent_inference_boundary', + name: 'Existing', + author: mockReq.user.id, + provider: 'openAI', + model: 'gpt-4o-mini', + model_parameters: { temperature: 0.2 }, + }); + mockReq.params.id = existing.id; + mockReq.body = { + model_parameters: { configuration: { baseURL: 'https://unchecked.invalid' } }, + }; + await updateAgentHandler(mockReq, mockRes); + expect(mockRes.status).toHaveBeenCalledWith(400); + const after = await Agent.findOne({ id: existing.id }).lean(); + expect(after.model_parameters).toEqual({ temperature: 0.2 }); + }); + }); + describe('agent machine allowlist authorization', () => { test.each(['create', 'update', 'duplicate', 'restore'])( '%s refuses machine references outside the principal-scoped configuration', @@ -1294,7 +1345,7 @@ describe('Agent Controllers - Mass Assignment Protection', () => { expect(createdAgent.model_parameters).toBeDefined(); // Valid numbers should be preserved expect(createdAgent.model_parameters.temperature).toBe(0.7); - expect(createdAgent.model_parameters.topP).toBe(0); + expect(createdAgent.model_parameters.top_p).toBe(0); // Empty strings should be removed expect(createdAgent.model_parameters.max_tokens).toBeUndefined(); expect(createdAgent.model_parameters.maxContextTokens).toBeUndefined(); @@ -1303,12 +1354,12 @@ describe('Agent Controllers - Mass Assignment Protection', () => { // Verify in database const agentInDb = await Agent.findOne({ id: createdAgent.id }); expect(agentInDb.model_parameters.temperature).toBe(0.7); - expect(agentInDb.model_parameters.topP).toBe(0); + expect(agentInDb.model_parameters.top_p).toBe(0); expect(agentInDb.model_parameters.max_tokens).toBeUndefined(); expect(agentInDb.model_parameters.maxContextTokens).toBeUndefined(); }); - test('should drop non-numeric strings and coerce numeric strings in model_parameters', async () => { + test('should reject corrupt numeric model_parameters without persisting them', async () => { // Regression test for #12920: a stray placeholder string ("System") persisted // into max_tokens was forwarded to the provider, causing a 400 const dataWithCorruptModelParams = { @@ -1327,17 +1378,11 @@ describe('Agent Controllers - Mass Assignment Protection', () => { await createAgentHandler(mockReq, mockRes); - expect(mockRes.status).toHaveBeenCalledWith(201); - - const createdAgent = mockRes.json.mock.calls[0][0]; - expect(createdAgent.model_parameters.max_tokens).toBeUndefined(); - expect(createdAgent.model_parameters.maxContextTokens).toBe(256000); - expect(createdAgent.model_parameters.fileTokenLimit).toBe(256000); - expect(createdAgent.model_parameters.useResponsesApi).toBe(true); - - const agentInDb = await Agent.findOne({ id: createdAgent.id }); - expect(agentInDb.model_parameters.max_tokens).toBeUndefined(); - expect(agentInDb.model_parameters.maxContextTokens).toBe(256000); + expect(mockRes.status).toHaveBeenCalledWith(400); + expect(await Agent.countDocuments()).toBe(0); + expect(mockRes.json).toHaveBeenCalledWith( + expect.objectContaining({ error: 'Invalid request data' }), + ); }); test('should handle invalid avatar format', async () => { @@ -1990,7 +2035,7 @@ describe('Agent Controllers - Mass Assignment Protection', () => { ); }); - test('should sanitize corrupt numeric model_parameters on update', async () => { + test('should reject corrupt numeric updates and preserve the existing agent', async () => { mockReq.user.id = existingAgentAuthorId.toString(); mockReq.params.id = existingAgentId; mockReq.body = { @@ -2004,16 +2049,10 @@ describe('Agent Controllers - Mass Assignment Protection', () => { await updateAgentHandler(mockReq, mockRes); - expect(mockRes.json).toHaveBeenCalled(); - - const updatedAgent = mockRes.json.mock.calls[0][0]; - expect(updatedAgent.model_parameters.max_tokens).toBeUndefined(); - expect(updatedAgent.model_parameters.maxContextTokens).toBe(256000); - expect(updatedAgent.model_parameters.temperature).toBe(0.7); - + expect(mockRes.status).toHaveBeenCalledWith(400); const agentInDb = await Agent.findOne({ id: existingAgentId }); - expect(agentInDb.model_parameters.max_tokens).toBeUndefined(); - expect(agentInDb.model_parameters.maxContextTokens).toBe(256000); + expect(agentInDb.name).not.toBe('Healed Agent'); + expect(agentInDb.model_parameters?.maxContextTokens).not.toBe(256000); }); test('should reject update with unauthorized fields (mass assignment protection)', async () => { diff --git a/client/src/components/SidePanel/Agents/__tests__/parameters.spec.ts b/client/src/components/SidePanel/Agents/__tests__/parameters.spec.ts index ddd3f31da53..9647eea2c95 100644 --- a/client/src/components/SidePanel/Agents/__tests__/parameters.spec.ts +++ b/client/src/components/SidePanel/Agents/__tests__/parameters.spec.ts @@ -1,4 +1,10 @@ -import { agentParamSettings, EModelEndpoint } from 'librechat-data-provider'; +import { + agentParamSettings, + EModelEndpoint, + SettingTypes, + ComponentTypes, + ReasoningParameterFormat, +} from 'librechat-data-provider'; import type { AgentModelParameters, TModelReasoning } from 'librechat-data-provider'; import { pruneAgentModelParameters, resolveAgentParameterSettings } from '../parameters'; @@ -188,3 +194,142 @@ describe('resolveAgentParameterSettings: OpenRouter reasoning efforts', () => { expect(effort?.options).toEqual(['', 'low', 'high']); }); }); + +describe('administrator-declared inference controls', () => { + const endpointsConfig = { + Gateway: { + order: 0, + type: EModelEndpoint.custom, + customParams: { + defaultParamsEndpoint: 'custom', + paramDefinitions: [ + { + key: 'min_p', + type: SettingTypes.Number, + component: ComponentTypes.Input, + label: 'Minimum Probability', + default: 0.1, + }, + { + key: 'foo_generation_mode', + type: SettingTypes.Enum, + component: ComponentTypes.Dropdown, + label: 'Generation Mode', + options: ['fast', 'accurate'], + default: 'accurate', + }, + { key: 'modelLabel', type: SettingTypes.String, component: ComponentTypes.Input }, + ], + }, + }, + }; + const stored = { + temperature: 0.3, + maxContextTokens: null, + max_context_tokens: null, + max_output_tokens: null, + top_p: null, + frequency_penalty: null, + presence_penalty: null, + min_p: 0.7, + foo_generation_mode: 'fast', + }; + const resolve = (drops: string[] = []) => + resolveAgentParameterSettings({ + provider: 'Gateway', + model: 'permitted', + webSearchAllowed: true, + endpointsConfig, + startupConfig: { endpointsDropParamsMap: { Gateway: drops } }, + }); + + it('offers typed controls and retains their saved values on unrelated edits', () => { + const settings = resolve(); + expect(settings.visibleParameters).toEqual( + expect.arrayContaining([ + expect.objectContaining({ key: 'min_p', label: 'Minimum Probability' }), + expect.objectContaining({ key: 'foo_generation_mode', options: ['fast', 'accurate'] }), + ]), + ); + expect(settings.parameters.map(({ key }) => key)).not.toContain('modelLabel'); + expect(pruneAgentModelParameters(stored, settings)).toBe(stored); + }); + + it('honors administrator drops for both rendering and persistence', () => { + const settings = resolve(['min_p']); + expect(settings.visibleParameters.map(({ key }) => key)).not.toContain('min_p'); + expect(pruneAgentModelParameters(stored, settings)).not.toHaveProperty('min_p'); + expect(pruneAgentModelParameters(stored, settings)).toHaveProperty( + 'foo_generation_mode', + 'fast', + ); + }); +}); + +describe('OpenRouter cache preferences do not depend on reasoning', () => { + it.each(['reasoning', 'reasoning_effort'])( + 'keeps cache controls when %s is dropped', + (dropped) => { + const settings = resolveAgentParameterSettings({ + provider: 'Gateway', + model: 'permitted', + webSearchAllowed: true, + endpointsConfig: { + Gateway: { + order: 0, + type: EModelEndpoint.custom, + customParams: { + defaultParamsEndpoint: 'openrouter', + reasoningFormat: ReasoningParameterFormat.disabled, + }, + }, + }, + startupConfig: { endpointsDropParamsMap: { Gateway: [dropped] } }, + }); + expect(settings.visibleParameters.map(({ key }) => key)).toEqual( + expect.arrayContaining(['promptCache', 'promptCacheTtl']), + ); + expect(settings.visibleParameters.map(({ key }) => key)).not.toContain('reasoning_effort'); + const saved = { ...defaults, promptCache: false, promptCacheTtl: '1h' as const }; + expect(pruneAgentModelParameters(saved, settings)).toBe(saved); + }, + ); +}); + +describe('legacy reasoning control visibility', () => { + it.each([false, true])( + 'preserves stored values but hides an explicitly disabled legacy control (%s)', + (disabled) => { + const settings = resolveAgentParameterSettings({ + provider: 'Gateway', + model: 'permitted', + webSearchAllowed: true, + endpointsConfig: { + Gateway: { + order: 0, + type: EModelEndpoint.custom, + customParams: { + defaultParamsEndpoint: 'openrouter', + reasoningFormat: disabled + ? ReasoningParameterFormat.disabled + : ReasoningParameterFormat.reasoningEffort, + paramDefinitions: [ + { + key: 'include_reasoning', + type: SettingTypes.Boolean, + component: ComponentTypes.Checkbox, + }, + ], + }, + }, + }, + }); + expect(settings.visibleParameters.some(({ key }) => key === 'include_reasoning')).toBe( + !disabled, + ); + expect(settings.parameters.some(({ key }) => key === 'include_reasoning')).toBe(true); + const saved = { ...defaults, include_reasoning: true, promptCache: false }; + expect(pruneAgentModelParameters(saved, settings)).toBe(saved); + }, + ); +}); diff --git a/client/src/components/SidePanel/Agents/parameters.ts b/client/src/components/SidePanel/Agents/parameters.ts index 30c9431044b..850fc15f5ab 100644 --- a/client/src/components/SidePanel/Agents/parameters.ts +++ b/client/src/components/SidePanel/Agents/parameters.ts @@ -1,11 +1,13 @@ import keyBy from 'lodash/keyBy'; import { agentParamSettings, + tConversationSchema, applyModelAwareDefaults, getEndpointField, getSettingsKeys, normalizeEndpointName, resolveDropParamsUIKeys, + ReasoningParameterFormat, } from 'librechat-data-provider'; import type { AgentModelParameters, @@ -34,6 +36,13 @@ export type ResolvedAgentParameterSettings = { }; const NON_CONTROL_PARAMETER_KEYS = new Set(['model']); +const REASONING_PARAMETER_KEYS = new Set([ + 'include_reasoning', + 'reasoning_effort', + 'reasoning_summary', + 'reasoning_mode', + 'reasoning_context', +]); export function resolveAgentParameterSettings({ endpointsConfig, @@ -73,7 +82,14 @@ export function resolveAgentParameterSettings({ const overriddenParamsMap = keyBy(overriddenParams ?? [], 'key'); /** Keep endpoint-recognized keys independent of model visibility. Only explicit * administrator drops remove keys from the schema used by both save paths. */ - const endpointParams = (defaultParams ?? []).filter( + const declaredExtras = (overriddenParams ?? []).filter( + (param): param is SettingDefinition => + param.key != null && + param.type != null && + param.component != null && + !Object.prototype.hasOwnProperty.call(tConversationSchema.shape, param.key), + ); + const endpointParams = [...(defaultParams ?? []), ...declaredExtras].filter( (param) => param != null && !dropParamsSet.has(param.key), ); const applyOverride = (param: SettingDefinition) => @@ -87,7 +103,12 @@ export function resolveAgentParameterSettings({ modelReasoning, ) .map(applyOverride) - .filter((param) => param.key !== 'web_search' || webSearchAllowed); + .filter((param) => param.key !== 'web_search' || webSearchAllowed) + .filter( + (param) => + customParams?.reasoningFormat !== ReasoningParameterFormat.disabled || + !REASONING_PARAMETER_KEYS.has(param.key), + ); return { schemaResolved: defaultParams != null || overriddenParams != null, diff --git a/packages/api/src/agents/__tests__/initialize.test.ts b/packages/api/src/agents/__tests__/initialize.test.ts index 6d12b44d5e5..b31fb565b23 100644 --- a/packages/api/src/agents/__tests__/initialize.test.ts +++ b/packages/api/src/agents/__tests__/initialize.test.ts @@ -6246,3 +6246,238 @@ describe('initializeAgent turn delivery routing', () => { expect(filesOrder).toBeLessThan(toolsOrder); }); }); + +describe('initializeAgent — inference trust boundary', () => { + beforeEach(() => { + jest.clearAllMocks(); + }); + + it.each([true, false])( + 'projects saved settings and request overrides (initial: %s)', + async (isInitialAgent) => { + const { agent, req, res, loadTools, db } = createMocks({ + provider: Providers.OPENAI, + model: 'gpt-4o-mini', + }); + Object.assign(agent.model_parameters, { + temperature: 0.2, + apiKey: 'untrusted-secret', + modelKwargs: { model: 'gpt-4o', service_tier: 'flex' }, + configuration: { baseURL: 'https://unchecked.invalid' }, + callbacks: ['not-user-configurable'], + fallbacks: [{ provider: 'openAI', clientOptions: { model: 'gpt-4o' } }], + unknownConstructorField: true, + }); + mockExtractLibreChatParams.mockImplementation(realUtils.extractLibreChatParams); + + await initializeAgent( + { + req, + res, + agent, + loadTools, + endpointOption: { + endpoint: EModelEndpoint.agents, + model_parameters: { temperature: 0.4, modelKwargs: { model: 'gpt-4o' } }, + }, + allowedProviders: new Set([Providers.OPENAI]), + isInitialAgent, + }, + db, + ); + + const { results } = mockGetProviderConfig.mock; + const { getOptions } = results[results.length - 1].value; + expect(getOptions).toHaveBeenCalledWith( + expect.objectContaining({ + model_parameters: { + model: 'gpt-4o-mini', + temperature: isInitialAgent ? 0.4 : 0.2, + }, + }), + ); + }, + ); + + it.each(['custom', 'anthropic', 'google'])( + 'retains canonical settings and endpoint identity for same-agent memory reentry (%s)', + async (mode) => { + const { agent, req, res, loadTools, db } = createMocks({ + provider: 'Gateway', + overrideProvider: Providers.OPENAI, + model: 'permitted', + }); + Object.assign(agent.model_parameters, { min_p: 0.7, modelKwargs: { min_p: 0.9 } }); + const getOptions = jest + .fn() + .mockImplementation(async ({ model_parameters, inferenceParameters }) => ({ + llmConfig: { + model: model_parameters.model, + apiKey: 'trusted-key', + modelKwargs: { min_p: inferenceParameters?.min_p ?? 0.1 }, + }, + configOptions: { baseURL: 'https://trusted.invalid' }, + })); + mockGetProviderConfig.mockImplementation(({ provider }) => { + if (provider !== 'Gateway') throw new Error('Lost original endpoint identity'); + return { + getOptions, + overrideProvider: Providers.OPENAI, + customEndpointConfig: { + customParams: { + defaultParamsEndpoint: mode, + paramDefinitions: [ + { key: 'min_p', type: 'number', default: 0.1, range: { min: 0, max: 1 } }, + ], + }, + }, + }; + }); + mockExtractLibreChatParams.mockImplementation(realUtils.extractLibreChatParams); + const params = { + req, + res, + agent, + loadTools, + endpointOption: { endpoint: EModelEndpoint.agents }, + allowedProviders: new Set([Providers.OPENAI, 'Gateway']), + isInitialAgent: true, + }; + const main = await initializeAgent(params, db); + const mainConfig = main.model_parameters; + expect(mainConfig).toMatchObject({ modelKwargs: { min_p: 0.7 } }); + for (let attempt = 0; attempt < 3; attempt++) { + const memory = await initializeAgent( + { ...params, agent: { ...main }, isInitialAgent: false }, + db, + ); + expect(memory.model_parameters).toMatchObject({ modelKwargs: { min_p: 0.7 } }); + expect(memory).not.toBe(main); + expect(main.model_parameters).toBe(mainConfig); + expect(getOptions).toHaveBeenLastCalledWith( + expect.objectContaining({ + endpoint: 'Gateway', + model_parameters: { model: 'permitted' }, + inferenceParameters: { min_p: 0.7 }, + }), + ); + } + await expect( + initializeAgent( + { + ...params, + agent: main, + allowedProviders: new Set([Providers.OPENAI]), + isInitialAgent: false, + }, + db, + ), + ).rejects.toThrow('invalid_agent_provider'); + }, + ); + + it('uses the trusted URL profile before projecting saved and request caching controls', async () => { + const key = 'TEST_INFERENCE_GATEWAY_URL'; + const previous = process.env[key]; + process.env[key] = 'https://openrouter.ai/api/v1'; + try { + const { agent, req, res, loadTools, db } = createMocks({ + provider: 'Gateway', + overrideProvider: Providers.OPENAI, + model: 'permitted', + }); + Object.assign(agent.model_parameters, { promptCache: false, promptCacheTtl: '1h' }); + const endpoint = { + name: 'Gateway', + apiKey: 'synthetic-key', + baseURL: '${TEST_INFERENCE_GATEWAY_URL}', + models: { default: ['permitted'], fetch: false }, + }; + if (req.config == null) throw new Error('Expected the request configuration fixture'); + req.config.endpoints = { custom: [endpoint] }; + const getOptions = jest + .fn() + .mockResolvedValue({ llmConfig: { model: 'permitted', promptCache: false } }); + mockGetProviderConfig.mockReturnValue({ + getOptions, + overrideProvider: Providers.OPENAI, + customEndpointConfig: endpoint, + }); + mockExtractLibreChatParams.mockImplementation(realUtils.extractLibreChatParams); + await initializeAgent( + { + req, + res, + agent, + loadTools, + endpointOption: { endpoint: EModelEndpoint.agents }, + allowedProviders: new Set([Providers.OPENAI, 'Gateway']), + isInitialAgent: true, + }, + db, + ); + expect(getOptions).toHaveBeenCalledWith( + expect.objectContaining({ + endpoint: 'Gateway', + model_parameters: { model: 'permitted', promptCache: false, promptCacheTtl: '1h' }, + }), + ); + } finally { + if (previous === undefined) delete process.env[key]; + else process.env[key] = previous; + } + }); + + it('constructs custom request kwargs only from administrator-declared inference values', async () => { + const { agent, req, res, loadTools, db } = createMocks({ + provider: 'Gateway', + overrideProvider: Providers.OPENAI, + model: 'permitted', + }); + Object.assign(agent.model_parameters, { min_p: 0.1, modelKwargs: { model: 'restricted' } }); + const getOptions = jest.fn().mockResolvedValue({ + llmConfig: { + model: 'permitted', + apiKey: 'trusted-key', + configuration: { baseURL: 'https://trusted.invalid' }, + modelKwargs: { seed: 7 }, + }, + }); + mockGetProviderConfig.mockReturnValue({ + getOptions, + overrideProvider: Providers.OPENAI, + customEndpointConfig: { + customParams: { + defaultParamsEndpoint: 'custom', + paramDefinitions: [{ key: 'min_p', type: 'number', range: { min: 0, max: 1 } }], + }, + }, + }); + mockExtractLibreChatParams.mockImplementation(realUtils.extractLibreChatParams); + const initialized = await initializeAgent( + { + req, + res, + agent, + loadTools, + endpointOption: { endpoint: EModelEndpoint.agents }, + allowedProviders: new Set([Providers.OPENAI, 'Gateway']), + isInitialAgent: true, + }, + db, + ); + + expect(getOptions).toHaveBeenCalledWith( + expect.objectContaining({ + model_parameters: { model: 'permitted' }, + inferenceParameters: { min_p: 0.1 }, + }), + ); + expect(initialized.model_parameters).toMatchObject({ + model: 'permitted', + apiKey: 'trusted-key', + configuration: { baseURL: 'https://trusted.invalid' }, + modelKwargs: { seed: 7 }, + }); + }); +}); diff --git a/packages/api/src/agents/index.ts b/packages/api/src/agents/index.ts index b8c34f34abd..b3270f56a0b 100644 --- a/packages/api/src/agents/index.ts +++ b/packages/api/src/agents/index.ts @@ -45,6 +45,7 @@ export * from './orphans'; export * from './ownerContact'; export * from './migration'; export * from './parameters'; +export * from './input'; export * from './plan'; export * from './prewarm'; export * from './ptc'; diff --git a/packages/api/src/agents/inference.extensions.spec.ts b/packages/api/src/agents/inference.extensions.spec.ts new file mode 100644 index 00000000000..f06edce9754 --- /dev/null +++ b/packages/api/src/agents/inference.extensions.spec.ts @@ -0,0 +1,591 @@ +import { Providers, initializeModel } from '@librechat/agents'; +import { + THINKING_BLOCK_BINDING, + THINKING_BINDING_BETA, + ThinkingDisplay, + SettingTypes, + bedrockInputParser, + bedrockOutputParser, +} from 'librechat-data-provider'; +import type { Agent, TEndpoint } from 'librechat-data-provider'; +import type { OpenAIConfigOptions, OpenAIConfiguration } from '~/types'; +import { + resolveAgentInferenceParameters, + restoreAgentInferenceInput, + retainAgentInferenceInput, +} from './inference'; +import { getLLMConfig as getAnthropicLLMConfig } from '~/endpoints/anthropic/llm'; +import { getOpenAIConfig } from '~/endpoints/openai/config'; + +const inferenceParameters = { foo_generation_mode: 'fast', min_p: 0.2, seed: 7 }; +const paramDefinitions = [ + { + key: 'foo_generation_mode', + type: SettingTypes.Enum, + default: 'accurate', + options: ['fast', 'accurate'], + }, + { key: 'min_p', type: SettingTypes.Number, default: 0.1 }, + { key: 'seed', type: SettingTypes.Number, default: 2 }, +]; + +const wireCases: Array<{ + name: string; + extras: Record; + addParams?: Record; + expectedMode: string; + expectedSeed: number; +}> = [ + { + name: 'explicit beats default and min_p drop wins', + extras: inferenceParameters, + expectedMode: 'fast', + expectedSeed: 7, + }, + { + name: 'missing values use defaults and min_p drop wins', + extras: {}, + expectedMode: 'accurate', + expectedSeed: 2, + }, + { + name: 'admin add beats explicit and last min_p drop wins', + extras: inferenceParameters, + addParams: { foo_generation_mode: 'admin', min_p: 0.9, seed: 9 }, + expectedMode: 'admin', + expectedSeed: 9, + }, +]; + +const customSettings = (defaultParamsEndpoint: string): NonNullable => ({ + defaultParamsEndpoint, + paramDefinitions, +}); + +function options(defaultParamsEndpoint: string): OpenAIConfigOptions { + return { + streaming: false, + reverseProxyUrl: 'https://gateway.example/v1', + modelOptions: { model: 'test-model' }, + inferenceParameters, + customParams: customSettings(defaultParamsEndpoint), + dropParams: ['min_p'], + }; +} + +describe('canonical runtime inference provenance', () => { + const makeAgent = (modelParameters: Record): Agent => ({ + id: 'agent_runtime_input', + name: 'Runtime input', + description: null, + created_at: 0, + avatar: null, + model: 'test-model', + provider: 'openAI', + model_parameters: modelParameters as Agent['model_parameters'], + }); + + it.each(['custom', 'anthropic', 'google'])( + 'retains explicit extensions through repeated real-adapter reentry (%s)', + (mode) => { + let agent = makeAgent({ min_p: 0.7, temperature: 0.4 }); + agent.provider = 'Gateway'; + for (let attempt = 0; attempt < 3; attempt++) { + const canonical = restoreAgentInferenceInput(agent); + expect(canonical.provider).toBe('Gateway'); + const projected = resolveAgentInferenceParameters({ + provider: canonical.provider, + model: canonical.model ?? '', + parameters: canonical.model_parameters, + customParams: customSettings(mode), + }); + expect(projected.customParameters).toEqual({ min_p: 0.7 }); + const configured = getOpenAIConfig( + 'test-key', + { + ...options(mode), + dropParams: undefined, + modelOptions: { ...projected.parameters, model: 'test-model' }, + inferenceParameters: projected.customParameters, + }, + 'Gateway', + ); + agent = { + ...canonical, + provider: Providers.OPENAI, + model_parameters: configured.llmConfig as Agent['model_parameters'], + }; + retainAgentInferenceInput(agent, 'Gateway', { + ...projected.parameters, + ...projected.customParameters, + }); + expect(agent.model_parameters).toMatchObject({ modelKwargs: { min_p: 0.7 } }); + } + const restricted = resolveAgentInferenceParameters({ + provider: 'Gateway', + model: 'test-model', + parameters: restoreAgentInferenceInput(agent).model_parameters, + customParams: { + defaultParamsEndpoint: mode, + paramDefinitions: [ + { key: 'min_p', type: SettingTypes.Number, range: { min: 0, max: 0.5 } }, + ], + }, + }); + expect(restricted.customParameters).toEqual({}); + expect(restricted.issues).toContainEqual({ path: ['min_p'], code: 'invalid_parameter' }); + }, + ); + + it('retains flat extensions when the native adapter emits invocation kwargs', () => { + let agent = makeAgent({ min_p: 0.7, thinking: false }); + agent.provider = 'NativeGateway'; + agent.model = 'claude-3-5-sonnet-20241022'; + for (let attempt = 0; attempt < 3; attempt++) { + const canonical = restoreAgentInferenceInput(agent); + expect(canonical.provider).toBe('NativeGateway'); + const projected = resolveAgentInferenceParameters({ + provider: 'anthropic', + model: canonical.model ?? '', + parameters: canonical.model_parameters, + customParams: customSettings('anthropic'), + }); + expect(projected.customParameters).toEqual({ min_p: 0.7 }); + const configured = getAnthropicLLMConfig('test-key', { + modelOptions: { ...projected.parameters, model: canonical.model ?? '' }, + inferenceParameters: projected.customParameters, + defaultParams: { min_p: 0.1 }, + }); + agent = { + ...canonical, + provider: Providers.ANTHROPIC, + model_parameters: configured.llmConfig as Agent['model_parameters'], + }; + retainAgentInferenceInput(agent, 'NativeGateway', { + ...projected.parameters, + ...projected.customParameters, + }); + expect(agent.model_parameters).toMatchObject({ invocationKwargs: { min_p: 0.7 } }); + } + }); + + it('does not trust JSON input, caller replacements, or copied lookalike provenance', () => { + const runtime = makeAgent({ modelKwargs: { min_p: 0.9 } }); + retainAgentInferenceInput(runtime, 'Gateway', { min_p: 0.7 }); + const copy = { ...runtime }; + expect(restoreAgentInferenceInput(copy).model_parameters).toEqual({ min_p: 0.7 }); + const external = JSON.parse(JSON.stringify(runtime)); + expect(restoreAgentInferenceInput(external)).toBe(external); + expect( + resolveAgentInferenceParameters({ + parameters: external.model_parameters, + provider: 'Gateway', + model: 'test-model', + customParams: customSettings('custom'), + }).customParameters, + ).toEqual({}); + const replacement = { + ...runtime, + model_parameters: makeAgent({ min_p: 0.3 }).model_parameters, + }; + expect(restoreAgentInferenceInput(replacement)).toBe(replacement); + const retargeted = { ...runtime, provider: 'google' }; + expect(restoreAgentInferenceInput(retargeted)).toBe(retargeted); + const lookalike = makeAgent({ modelKwargs: { min_p: 0.9 } }); + Object.assign(lookalike, { + agentInferenceSource: { provider: 'Gateway', parameters: { min_p: 0.7 } }, + }); + expect(restoreAgentInferenceInput(lookalike)).toBe(lookalike); + }); +}); + +describe('legacy Bedrock inference preferences', () => { + it.each([ + ['us.anthropic.claude-sonnet-5', { type: 'disabled' }, 'disabled'], + ['us.anthropic.claude-sonnet-5-5', { type: 'between_tools' }, 'between_tools'], + [ + 'us.anthropic.claude-opus-5-5', + { type: 'adaptive', display: 'omitted', block_binding: THINKING_BLOCK_BINDING }, + 'adaptive', + ], + ['us.anthropic.claude-3-7-sonnet', { type: 'enabled', budget_tokens: 4096 }, 'enabled'], + ])( + 'recovers only known preferences from a legacy record for %s', + (model, thinking, expectedType) => { + const parameters = { + additionalModelRequestFields: { + thinking, + output_config: { effort: 'low' }, + apiKey: 'untrusted', + model: 'restricted', + configuration: { baseURL: 'https://unchecked.invalid' }, + }, + }; + const projected = resolveAgentInferenceParameters({ provider: 'bedrock', model, parameters }); + expect(projected.parameters.thinking).toBe( + !['disabled', 'between_tools'].includes(expectedType), + ); + expect(projected.parameters).not.toHaveProperty('additionalModelRequestFields'); + expect(projected.parameters).not.toHaveProperty('apiKey'); + expect(projected.parameters).not.toHaveProperty('configuration'); + expect(projected.issues).toEqual([ + { path: ['additionalModelRequestFields'], code: 'unsupported_parameter' }, + ]); + const configured = bedrockOutputParser( + bedrockInputParser.parse({ ...projected.parameters, model }), + ); + expect(configured.additionalModelRequestFields).toMatchObject({ + thinking: { type: expectedType }, + }); + expect(JSON.stringify(configured)).not.toContain('untrusted'); + if (expectedType === 'adaptive') { + expect(configured.additionalModelRequestFields).toMatchObject({ + thinking: { display: 'omitted', block_binding: THINKING_BLOCK_BINDING }, + output_config: { effort: 'low' }, + }); + } + if (expectedType === 'enabled') { + expect(configured.additionalModelRequestFields).toMatchObject({ + thinking: { budget_tokens: 4096 }, + }); + } + }, + ); + + it('keeps flat preferences authoritative and drops malformed nested values', () => { + const projected = resolveAgentInferenceParameters({ + provider: 'bedrock', + model: 'us.anthropic.claude-sonnet-5', + parameters: { + thinking: false, + effort: 'medium', + additionalModelRequestFields: { + thinking: { type: 'adaptive' }, + output_config: { effort: 'low' }, + }, + }, + }); + expect(projected.parameters).toMatchObject({ thinking: false, effort: 'medium' }); + expect( + resolveAgentInferenceParameters({ + provider: 'bedrock', + model: 'us.anthropic.claude-sonnet-5', + parameters: { + additionalModelRequestFields: { + thinking: { type: 'disabled', unknownControl: true }, + output_config: { effort: 'invalid' }, + }, + }, + }).parameters, + ).toEqual({}); + }); + + it('recovers a recognized non-Anthropic reasoning preference without forwarding its envelope', () => { + const projected = resolveAgentInferenceParameters({ + provider: 'bedrock', + model: 'moonshot.kimi', + parameters: { + additionalModelRequestFields: { + reasoning_config: 'low', + configuration: { apiKey: 'untrusted' }, + }, + }, + }); + expect(projected.parameters).toEqual({ reasoning_effort: 'low' }); + const configured = bedrockOutputParser( + bedrockInputParser.parse({ ...projected.parameters, model: 'moonshot.kimi' }), + ); + expect(configured.additionalModelRequestFields).toEqual({ reasoning_config: 'low' }); + }); +}); + +describe('inference transition invariants', () => { + it.each(['claude-opus-5-5', 'claude-sonnet-5-5', 'claude-opus-4-7'])( + 'preserves the generated thinking preference across repeated initialization for %s', + (model) => { + let resolved = getAnthropicLLMConfig('test-key', { + modelOptions: { model, thinking: true, thinkingDisplay: ThinkingDisplay.omitted }, + }).llmConfig; + const initialThinking = structuredClone(resolved.thinking); + for (let attempt = 0; attempt < 3; attempt++) { + const projected = resolveAgentInferenceParameters({ + provider: 'anthropic', + model, + parameters: resolved, + }); + expect(projected.issues).not.toContainEqual({ + path: ['thinking'], + code: 'invalid_parameter', + }); + expect(projected.parameters.thinkingDisplay).toBe('omitted'); + expect(projected.parameters).not.toHaveProperty('block_binding'); + expect(projected.parameters).not.toHaveProperty('apiKey'); + expect(projected.parameters).not.toHaveProperty('clientOptions'); + resolved = getAnthropicLLMConfig('test-key', { + modelOptions: { ...projected.parameters, model }, + }).llmConfig; + expect(resolved.thinking).toEqual(initialThinking); + } + }, + ); + + it('regenerates a known binding for the current model instead of forwarding it', () => { + const projected = resolveAgentInferenceParameters({ + provider: 'anthropic', + model: 'claude-opus-4-7', + parameters: { + thinking: { type: 'adaptive', display: 'omitted', block_binding: THINKING_BLOCK_BINDING }, + }, + }); + expect(projected).toEqual({ + parameters: { thinking: true, thinkingDisplay: 'omitted' }, + customParameters: {}, + issues: [], + }); + const { llmConfig } = getAnthropicLLMConfig('test-key', { + modelOptions: { ...projected.parameters, model: 'claude-opus-4-7' }, + }); + expect(llmConfig.thinking).toEqual({ type: 'adaptive', display: 'omitted' }); + expect(llmConfig.clientOptions?.defaultHeaders).toHaveProperty( + 'anthropic-beta', + expect.not.stringContaining(THINKING_BINDING_BETA), + ); + }); + + it.each([ + { prefix_mismatch_behavior: 'keep_block' }, + { prefix_mismatch_behavior: 'drop_block', configuration: { apiKey: 'untrusted' } }, + ])('rejects an unrecognized binding %j', (block_binding) => { + expect( + resolveAgentInferenceParameters({ + provider: 'anthropic', + model: 'claude-opus-5-5', + parameters: { thinking: { type: 'adaptive', display: 'omitted', block_binding } }, + }).issues, + ).toEqual([{ path: ['thinking'], code: 'invalid_parameter' }]); + }); +}); + +describe('validated inference extension adapter seam', () => { + it.each(['custom', 'anthropic', 'google'])( + 'applies explicit extras before defaults and drops in %s mode', + (mode) => { + const result = getOpenAIConfig('test-key', options(mode), 'gateway'); + expect(result.llmConfig.modelKwargs).toMatchObject({ foo_generation_mode: 'fast', seed: 7 }); + expect(result.llmConfig.modelKwargs).not.toHaveProperty('min_p'); + expect(result.llmConfig).not.toHaveProperty('foo_generation_mode'); + expect(result.llmConfig).not.toHaveProperty('inferenceParameters'); + expect(inferenceParameters).toEqual({ foo_generation_mode: 'fast', min_p: 0.2, seed: 7 }); + }, + ); + + it.each(['custom', 'anthropic', 'google'])( + 'fills only missing extras and lets admin additions win in %s mode', + (mode) => { + const defaults = getOpenAIConfig( + 'test-key', + { ...options(mode), inferenceParameters: {} }, + 'gateway', + ); + expect(defaults.llmConfig.modelKwargs).toMatchObject({ foo_generation_mode: 'accurate' }); + const result = getOpenAIConfig( + 'test-key', + { + ...options(mode), + addParams: { foo_generation_mode: 'admin', min_p: 0.9, seed: 9 }, + }, + 'gateway', + ); + expect(result.llmConfig.modelKwargs).toMatchObject({ foo_generation_mode: 'admin' }); + expect(result.llmConfig.modelKwargs).not.toHaveProperty('min_p'); + expect( + result.llmConfig.modelKwargs?.seed ?? + (result.llmConfig as unknown as Record).seed, + ).toBe(9); + }, + ); + + it('does not accept raw model kwargs but preserves trusted admin wire kwargs', () => { + const result = getOpenAIConfig( + 'test-key', + { + ...options('custom'), + modelOptions: { + model: 'test-model', + modelKwargs: { untrusted: true }, + } as OpenAIConfigOptions['modelOptions'], + addParams: { modelKwargs: { server_field: 'configured' } }, + }, + 'gateway', + ); + expect(result.llmConfig.modelKwargs).toMatchObject({ + foo_generation_mode: 'fast', + server_field: 'configured', + }); + expect(result.llmConfig.modelKwargs).not.toHaveProperty('untrusted'); + expect(result.llmConfig.modelKwargs).not.toHaveProperty('modelKwargs'); + }); + + it.each( + ['custom', 'anthropic', 'google'].flatMap((mode) => + wireCases.map((testCase) => ({ mode, ...testCase })), + ), + )('never recreates a dropped envelope ($mode: $name)', async ({ mode, extras, addParams }) => { + const { llmConfig, configOptions } = getOpenAIConfig( + 'test-key', + { + ...options(mode), + inferenceParameters: extras, + addParams, + dropParams: ['modelKwargs'], + }, + 'gateway', + ); + expect(llmConfig).not.toHaveProperty('modelKwargs'); + const bodies: Record[] = []; + const fetch: NonNullable['fetch']> = async (_url, init) => { + bodies.push(JSON.parse(String(init?.body))); + return Response.json({ + id: 'chatcmpl-drop', + object: 'chat.completion', + created: 1, + model: 'test-model', + choices: [ + { index: 0, message: { role: 'assistant', content: 'ok' }, finish_reason: 'stop' }, + ], + usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 }, + }); + }; + const model = initializeModel({ + provider: Providers.OPENAI, + clientOptions: { + ...llmConfig, + verbosity: undefined, + configuration: { ...configOptions, fetch }, + }, + }); + await model.invoke('hello'); + expect(bodies).toHaveLength(1); + for (const key of Object.keys(inferenceParameters)) expect(bodies[0]).not.toHaveProperty(key); + }); + + it.each(wireCases)( + 'native Anthropic drops the entire invocation envelope: $name', + ({ extras, addParams }) => { + const { llmConfig } = getAnthropicLLMConfig('test-key', { + modelOptions: { model: 'claude-3-5-sonnet-20241022', thinking: false }, + inferenceParameters: extras, + defaultParams: { foo_generation_mode: 'accurate', seed: 2 }, + addParams, + dropParams: ['invocationKwargs'], + }); + expect(llmConfig).not.toHaveProperty('invocationKwargs'); + }, + ); + + it('composes native Anthropic invocation extras before defaults, admin additions, and drops', () => { + const result = getAnthropicLLMConfig('test-key', { + modelOptions: { model: 'claude-3-5-sonnet-20241022', user: 'user-1', thinking: false }, + inferenceParameters, + defaultParams: { foo_generation_mode: 'accurate', seed: 2 }, + addParams: { seed: 9, min_p: 0.9 }, + dropParams: ['min_p'], + }); + expect(result.llmConfig.invocationKwargs).toMatchObject({ + foo_generation_mode: 'fast', + seed: 9, + metadata: { user_id: 'user-1' }, + }); + expect(result.llmConfig.invocationKwargs).not.toHaveProperty('min_p'); + expect(result.llmConfig).not.toHaveProperty('foo_generation_mode'); + }); + + it.each( + ['custom', 'anthropic', 'google'].flatMap((mode) => + wireCases.map((testCase) => ({ mode, ...testCase })), + ), + )( + 'OpenAI SDK wire, $mode hint: $name', + async ({ mode, extras, addParams, expectedMode, expectedSeed }) => { + const bodies: Record[] = []; + const fetch: NonNullable['fetch']> = async (_url, init) => { + bodies.push(JSON.parse(String(init?.body))); + return Response.json({ + id: 'chatcmpl-test', + object: 'chat.completion', + created: 1, + model: 'test-model', + choices: [ + { index: 0, message: { role: 'assistant', content: 'ok' }, finish_reason: 'stop' }, + ], + usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 }, + }); + }; + const { llmConfig, configOptions } = getOpenAIConfig( + 'test-key', + { + ...options(mode), + inferenceParameters: extras, + addParams, + }, + 'gateway', + ); + const model = initializeModel({ + provider: Providers.OPENAI, + clientOptions: { + ...llmConfig, + verbosity: undefined, + configuration: { ...configOptions, fetch }, + }, + }); + await model.invoke('hello'); + expect(bodies).toHaveLength(1); + expect(bodies[0]).toMatchObject({ foo_generation_mode: expectedMode, seed: expectedSeed }); + expect(bodies[0]).not.toHaveProperty('min_p'); + expect(bodies[0]).not.toHaveProperty('inferenceParameters'); + expect(bodies[0]).not.toHaveProperty('modelKwargs'); + }, + ); + + it.each(wireCases)( + 'native Anthropic SDK wire: $name', + async ({ extras, addParams, expectedMode, expectedSeed }) => { + const bodies: Record[] = []; + const { llmConfig } = getAnthropicLLMConfig('test-key', { + modelOptions: { model: 'claude-3-5-sonnet-20241022', stream: false, thinking: false }, + inferenceParameters: extras, + defaultParams: { foo_generation_mode: 'accurate', min_p: 0.1, seed: 2 }, + addParams, + dropParams: ['min_p'], + }); + const model = initializeModel({ + provider: Providers.ANTHROPIC, + clientOptions: { + ...llmConfig, + streaming: false, + clientOptions: { + ...llmConfig.clientOptions, + fetch: async (_url, init) => { + bodies.push(JSON.parse(String(init?.body))); + return Response.json({ + id: 'msg-test', + type: 'message', + role: 'assistant', + model: 'claude-3-5-sonnet-20241022', + content: [{ type: 'text', text: 'ok' }], + stop_reason: 'end_turn', + stop_sequence: null, + usage: { input_tokens: 1, output_tokens: 1 }, + }); + }, + }, + }, + }); + await model.invoke('hello'); + expect(bodies).toHaveLength(1); + expect(bodies[0]).toMatchObject({ foo_generation_mode: expectedMode, seed: expectedSeed }); + expect(bodies[0]).not.toHaveProperty('min_p'); + expect(bodies[0]).not.toHaveProperty('inferenceParameters'); + }, + ); +}); diff --git a/packages/api/src/agents/inference.request.spec.ts b/packages/api/src/agents/inference.request.spec.ts new file mode 100644 index 00000000000..2cf708053a8 --- /dev/null +++ b/packages/api/src/agents/inference.request.spec.ts @@ -0,0 +1,98 @@ +import { ChatOpenAI } from '@librechat/agents/llm/openai'; +import { resolveAgentInferenceParameters } from './inference'; +import { getOpenAILLMConfig } from '~/endpoints/openai/llm'; + +const buildRequest = (parameters: Record) => { + const { parameters: inference } = resolveAgentInferenceParameters({ + parameters, + provider: 'openAI', + model: 'gpt-4o-mini', + }); + const { llmConfig } = getOpenAILLMConfig({ + apiKey: 'synthetic-server-key', + streaming: true, + endpoint: 'openAI', + modelOptions: { ...inference, model: 'gpt-4o-mini' }, + }); + const client = new ChatOpenAI(llmConfig as ConstructorParameters[0]); + return { request: client.invocationParams({}), client }; +}; + +describe('inference parameters at the SDK request boundary', () => { + it('cannot replace the authorized model, transport, tools, or identity', () => { + const { request, client } = buildRequest({ + temperature: 0.3, + model: 'gpt-4o', + modelName: 'gpt-4o', + modelKwargs: { model: 'gpt-4o', messages: ['untrusted'], tools: ['untrusted'] }, + apiKey: 'untrusted-key', + configuration: { apiKey: 'untrusted-key', baseURL: 'https://unchecked.invalid' }, + fallbacks: [{ provider: 'openAI', clientOptions: { model: 'gpt-4o' } }], + tools: ['untrusted'], + user: 'untrusted-user', + futureConstructorControl: { arbitrary: true }, + }); + expect(request.model).toBe('gpt-4o-mini'); + expect(request.temperature).toBe(0.3); + expect(request.tools).toBeUndefined(); + expect(request).not.toHaveProperty('messages'); + expect(request.user).not.toBe('untrusted-user'); + expect(client.apiKey).toBe('synthetic-server-key'); + expect(JSON.stringify(request)).not.toContain('untrusted'); + }); + + it.each(['gpt-4o-search-preview', 'gpt-4o-mini-search-preview'])( + 'does not reintroduce unsupported sampling after normalizing top_p for %s', + (model) => { + const { parameters } = resolveAgentInferenceParameters({ + parameters: { top_p: 0.8, temperature: 0.4, max_tokens: 256 }, + provider: 'openAI', + model, + }); + const { llmConfig } = getOpenAILLMConfig({ + apiKey: 'synthetic-server-key', + streaming: false, + endpoint: 'openAI', + modelOptions: { ...parameters, model }, + }); + const client = new ChatOpenAI(llmConfig as ConstructorParameters[0]); + const request = client.invocationParams({}); + expect(request.top_p).toBeUndefined(); + expect(request.temperature).toBeUndefined(); + expect(request).toMatchObject({ max_tokens: 256 }); + }, + ); + + it('honors an administrator top_p drop after alias normalization', () => { + const { parameters } = resolveAgentInferenceParameters({ + parameters: { topP: 0.8 }, + provider: 'openAI', + model: 'gpt-4o-mini', + }); + const { llmConfig } = getOpenAILLMConfig({ + apiKey: 'synthetic-server-key', + streaming: false, + endpoint: 'openAI', + modelOptions: { ...parameters, model: 'gpt-4o-mini' }, + dropParams: ['top_p'], + }); + const client = new ChatOpenAI(llmConfig as ConstructorParameters[0]); + expect(client.invocationParams({}).top_p).toBeUndefined(); + }); + + it('retains supported inference settings in the final wire payload', () => { + const { request } = buildRequest({ + temperature: '0.4', + top_p: 0.8, + max_tokens: '256', + stop: ['END'], + }); + expect(request).toMatchObject({ + model: 'gpt-4o-mini', + temperature: 0.4, + top_p: 0.8, + max_tokens: 256, + stop: ['END'], + }); + }); +}); diff --git a/packages/api/src/agents/inference.spec.ts b/packages/api/src/agents/inference.spec.ts new file mode 100644 index 00000000000..ec38d550637 --- /dev/null +++ b/packages/api/src/agents/inference.spec.ts @@ -0,0 +1,555 @@ +import { SettingTypes } from 'librechat-data-provider'; +import type { TEndpoint } from 'librechat-data-provider'; +import { resolveAgentInferenceParameters } from './inference'; + +type CustomParams = NonNullable; +const custom = (paramDefinitions: CustomParams['paramDefinitions']): CustomParams => ({ + defaultParamsEndpoint: 'custom', + paramDefinitions, +}); +const resolve = (parameters: Record, provider = 'openAI', model = 'test-model') => + resolveAgentInferenceParameters({ parameters, provider, model }); + +describe('resolveAgentInferenceParameters', () => { + it('projects safe inference fields and LibreChat controls without modifying input', () => { + const parameters = { + temperature: '0.7', + max_tokens: '4096', + maxContextTokens: '32000', + fileTokenLimit: 12000, + resendFiles: false, + imageDetail: 'high', + promptPrefix: 'Be concise', + disableStreaming: true, + reasoning_effort: 'low', + stop: ['END'], + }; + expect(resolve(parameters)).toEqual({ + parameters: { + ...parameters, + temperature: 0.7, + max_tokens: 4096, + maxContextTokens: 32000, + }, + customParameters: {}, + issues: [], + }); + expect(parameters.temperature).toBe('0.7'); + }); + + it.each(['openAI', 'openai', 'azureOpenAI', 'custom', 'some-private-endpoint'])( + 'uses the OpenAI projection for %s', + (provider) => { + expect( + resolve({ topP: '0.8', maxOutputTokens: '2048', frequencyPenalty: '-1' }, provider), + ).toEqual({ + parameters: { top_p: 0.8, max_tokens: 2048, frequency_penalty: -1 }, + customParameters: {}, + issues: [], + }); + }, + ); + + it.each(['google', 'vertexai'])( + 'uses Google enums and aliases for %s without admitting OpenAI-only fields', + (provider) => { + expect( + resolve( + { + top_p: '0.9', + top_k: '40', + max_tokens: '1024', + thinking_budget: '-1', + thinkingLevel: 'low', + url_context: true, + frequency_penalty: 1, + }, + provider, + ), + ).toEqual({ + parameters: { + topP: 0.9, + topK: 40, + maxOutputTokens: 1024, + thinkingBudget: -1, + thinkingLevel: 'low', + url_context: true, + }, + customParameters: {}, + issues: [{ path: ['frequency_penalty'], code: 'unsupported_parameter' }], + }); + }, + ); + + it('preserves the explicit unlimited context sentinel', () => { + expect(resolve({ maxContextTokens: '-1', max_context_tokens: -1 }).parameters).toEqual({ + maxContextTokens: -1, + }); + expect(resolve({ maxContextTokens: -2 }).issues).toEqual([ + { path: ['maxContextTokens'], code: 'invalid_parameter' }, + ]); + }); + + it.each(['openrouter', 'custom'])( + 'preserves OpenRouter prompt caching flags for %s', + (provider) => { + expect( + resolveAgentInferenceParameters({ + parameters: { promptCache: true, promptCacheTtl: '1h' }, + provider, + model: 'anthropic/claude', + customParams: { defaultParamsEndpoint: 'openrouter' }, + }), + ).toEqual({ + parameters: { promptCache: true, promptCacheTtl: '1h' }, + customParameters: {}, + issues: [], + }); + }, + ); + + it('uses the Anthropic projection and rejects invalid provider enums', () => { + expect( + resolve( + { + temperature: 0.8, + max_tokens: 2048, + top_p: 0.9, + thinking: true, + thinkingBudget: 1024, + promptCache: true, + promptCacheTtl: '1h', + thinkingDisplay: 'not-a-display', + reasoning_effort: 'low', + }, + 'anthropic', + ), + ).toEqual({ + parameters: { + temperature: 0.8, + maxOutputTokens: 2048, + topP: 0.9, + thinking: true, + thinkingBudget: 1024, + promptCache: true, + promptCacheTtl: '1h', + }, + customParameters: {}, + issues: [ + { path: ['thinkingDisplay'], code: 'invalid_parameter' }, + { path: ['reasoning_effort'], code: 'unsupported_parameter' }, + ], + }); + }); + + it('preserves Bedrock public controls without accepting opaque request fields', () => { + const result = resolve( + { + maxOutputTokens: 4096, + thinkingBudget: 1024, + system: 'Answer briefly', + region: 'us-east-1', + additionalModelRequestFields: { arbitrary: true }, + }, + 'bedrock', + 'us.anthropic.claude-sonnet', + ); + expect(result.parameters).toEqual({ + maxTokens: 4096, + thinkingBudget: 1024, + system: 'Answer briefly', + region: 'us-east-1', + }); + expect(result.issues).toEqual([ + { path: ['additionalModelRequestFields'], code: 'unsupported_parameter' }, + ]); + expect(resolve({ thinkingBudget: 1024 }, 'bedrock', 'amazon.nova').issues).toEqual([ + { path: ['thinkingBudget'], code: 'unsupported_parameter' }, + ]); + }); + + it('preserves safe Bedrock regions and reasoning, with server-configured region constraints', () => { + const parameters = { region: 'us-west-2', reasoning_effort: 'low' }; + expect(resolve(parameters, 'bedrock', 'moonshot.kimi').parameters).toEqual(parameters); + expect( + resolveAgentInferenceParameters({ + parameters, + provider: 'bedrock', + model: 'moonshot.kimi', + availableRegions: ['us-east-1'], + }), + ).toMatchObject({ + parameters: { reasoning_effort: 'low' }, + issues: [{ path: ['region'], code: 'invalid_parameter' }], + }); + expect(resolve({ region: 'us-west-2@unchecked.invalid' }, 'bedrock').issues).toEqual([ + { path: ['region'], code: 'invalid_parameter' }, + ]); + }); + + it.each([ + [{ type: 'disabled' }, { thinking: false }], + [{ type: 'between_tools' }, { thinking: false }], + [ + { type: 'adaptive', display: 'omitted' }, + { thinking: true, thinkingDisplay: 'omitted' }, + ], + [ + { type: 'enabled', budget_tokens: 4096 }, + { thinking: true, thinkingBudget: 4096 }, + ], + ])('normalizes known legacy thinking %j before provider configuration', (thinking, expected) => { + expect(resolve({ thinking }, 'anthropic', 'claude-opus-4-7')).toEqual({ + parameters: expected, + customParameters: {}, + issues: [], + }); + expect( + resolve({ thinking: { ...thinking, configuration: { apiKey: 'secret' } } }, 'anthropic') + .issues, + ).toEqual([{ path: ['thinking'], code: 'invalid_parameter' }]); + }); + + it.each(['mistral', 'mistralai', 'mistral-compatible'])( + 'uses the narrow Mistral projection for %s', + (provider) => { + expect(resolve({ max_tokens: 1024, top_p: 0.8, reasoning_effort: 'low' }, provider)).toEqual({ + parameters: { maxTokens: 1024, topP: 0.8 }, + customParameters: {}, + issues: [{ path: ['reasoning_effort'], code: 'unsupported_parameter' }], + }); + }, + ); + + it('honors only recognized custom provider hints without overriding native providers', () => { + const parameters = { thinkingBudget: 1024, max_tokens: 2048 }; + expect( + resolveAgentInferenceParameters({ + parameters, + provider: 'custom', + model: 'test-model', + customParams: { defaultParamsEndpoint: 'anthropic' }, + }).parameters, + ).toEqual({ thinkingBudget: 1024, maxOutputTokens: 2048 }); + expect( + resolveAgentInferenceParameters({ + parameters, + provider: 'openAI', + model: 'test-model', + customParams: { defaultParamsEndpoint: 'google' }, + }).parameters, + ).toEqual({ thinkingBudget: 1024, maxOutputTokens: 2048 }); + expect( + resolveAgentInferenceParameters({ + parameters, + provider: 'custom', + model: 'test-model', + customParams: { defaultParamsEndpoint: 'some-private-endpoint' }, + }).parameters, + ).toEqual({ max_tokens: 2048 }); + expect( + resolveAgentInferenceParameters({ + parameters: { thinkingBudget: -1 }, + provider: 'anthropic', + model: 'test-model', + customParams: { defaultParamsEndpoint: 'google' }, + }).issues, + ).toEqual([{ path: ['thinkingBudget'], code: 'invalid_parameter' }]); + }); + + it('ignores absent and empty values while reporting each nonempty invalid field', () => { + expect( + resolve({ + temperature: 'placeholder-secret', + top_p: 0, + max_tokens: Infinity, + fileTokenLimit: true, + promptCache: null, + apiKey: '', + model: undefined, + promptPrefix: ' ', + imageDetail: 'not-a-detail', + stop: ['END', { secret: 'never-echo-me' }], + }), + ).toEqual({ + parameters: { top_p: 0 }, + customParameters: {}, + issues: [ + { path: ['temperature'], code: 'invalid_parameter' }, + { path: ['max_tokens'], code: 'invalid_parameter' }, + { path: ['fileTokenLimit'], code: 'invalid_parameter' }, + { path: ['imageDetail'], code: 'invalid_parameter' }, + { path: ['stop'], code: 'invalid_parameter' }, + ], + }); + expect( + resolveAgentInferenceParameters({ parameters: null, provider: 'openAI', model: '' }), + ).toEqual({ parameters: {}, customParameters: {}, issues: [] }); + }); + + it.each([NaN, Infinity, -Infinity, {}, [], true, 'not-a-number'])( + 'rejects unsafe numeric input %p', + (temperature) => { + expect(resolve({ temperature, max_tokens: 10 })).toEqual({ + parameters: { max_tokens: 10 }, + customParameters: {}, + issues: [{ path: ['temperature'], code: 'invalid_parameter' }], + }); + }, + ); + + it('rejects out-of-range values and retains valid siblings', () => { + expect( + resolve({ temperature: 3, top_p: -1, max_tokens: 0.5, frequency_penalty: -1.5 }), + ).toEqual({ + parameters: { frequency_penalty: -1.5 }, + customParameters: {}, + issues: [ + { path: ['temperature'], code: 'invalid_parameter' }, + { path: ['top_p'], code: 'invalid_parameter' }, + { path: ['max_tokens'], code: 'invalid_parameter' }, + ], + }); + }); + + it('keeps canonical values over aliases regardless of input order', () => { + expect(resolve({ topP: 0.2, top_p: 0.8 }).parameters).toEqual({ top_p: 0.8 }); + expect(resolve({ top_p: 0.8, topP: 0.2 }).parameters).toEqual({ top_p: 0.8 }); + expect(resolve({ stopSequences: ['END'], reasoningEffort: 'low' }).parameters).toEqual({ + stop: ['END'], + reasoning_effort: 'low', + }); + }); + + it('returns only declared, positively allowed, typed custom inference fields separately', () => { + const result = resolveAgentInferenceParameters({ + provider: 'custom', + model: 'test-model', + parameters: { + temperature: 0.5, + seed: '42', + do_sample: false, + reasoning_format: 'hidden', + samplers: ['top_k', 'top_p'], + include_reasoning: true, + min_p: 0.1, + }, + customParams: custom([ + { key: 'seed', type: SettingTypes.Number, range: { min: 0, max: 100 } }, + { key: 'do_sample', type: SettingTypes.Boolean }, + { key: 'reasoning_format', type: SettingTypes.Enum, options: ['hidden', 'raw'] }, + { key: 'samplers', type: SettingTypes.Array, minTags: 1, maxTags: 3 }, + { key: 'include_reasoning', type: SettingTypes.Boolean }, + ]), + }); + expect(result).toEqual({ + parameters: { temperature: 0.5 }, + customParameters: { + seed: 42, + do_sample: false, + reasoning_format: 'hidden', + samplers: ['top_k', 'top_p'], + include_reasoning: true, + }, + issues: [{ path: ['min_p'], code: 'unsupported_parameter' }], + }); + }); + + it('accepts future admin-declared flat extras without a hardcoded sampling-key list', () => { + expect( + resolveAgentInferenceParameters({ + provider: 'custom', + model: 'test-model', + parameters: { foo_generation_mode: 'fast', undeclared_generation_mode: 'fast' }, + customParams: custom([ + { key: 'foo_generation_mode', type: SettingTypes.Enum, options: ['fast', 'accurate'] }, + ]), + }), + ).toEqual({ + parameters: {}, + customParameters: { foo_generation_mode: 'fast' }, + issues: [{ path: ['undeclared_generation_mode'], code: 'unsupported_parameter' }], + }); + }); + + it('intersects builtin validation with admin ranges and enum options through aliases', () => { + const customParams = custom([ + { key: 'temperature', type: SettingTypes.Number, range: { min: 0, max: 0.5 } }, + { key: 'top_p', type: SettingTypes.Number, range: { min: 0, max: 0.5 } }, + { key: 'reasoning_effort', type: SettingTypes.String, options: ['low'] }, + { key: 'max_tokens', range: { min: 10, max: 100 } }, + ]); + expect( + resolveAgentInferenceParameters({ + provider: 'openAI', + model: 'test-model', + customParams, + parameters: { temperature: 0.8, topP: 0.8, reasoningEffort: 'high', maxOutputTokens: 5 }, + }).issues, + ).toEqual([ + { path: ['temperature'], code: 'invalid_parameter' }, + { path: ['topP'], code: 'invalid_parameter' }, + { path: ['reasoningEffort'], code: 'invalid_parameter' }, + { path: ['maxOutputTokens'], code: 'invalid_parameter' }, + ]); + expect( + resolveAgentInferenceParameters({ + provider: 'openAI', + model: 'test-model', + customParams, + parameters: { temperature: 0.4, topP: 0.4, reasoningEffort: 'low', maxOutputTokens: 50 }, + }), + ).toEqual({ + parameters: { temperature: 0.4, top_p: 0.4, reasoning_effort: 'low', max_tokens: 50 }, + customParameters: {}, + issues: [], + }); + expect( + resolveAgentInferenceParameters({ + provider: 'openAI', + model: 'test-model', + parameters: { temperature: 3, reasoning_effort: 'invented' }, + customParams: custom([ + { key: 'temperature', type: SettingTypes.Number, range: { min: 0, max: 100 } }, + { key: 'reasoning_effort', type: SettingTypes.Enum, options: ['invented'] }, + ]), + }).issues, + ).toEqual([ + { path: ['temperature'], code: 'invalid_parameter' }, + { path: ['reasoning_effort'], code: 'invalid_parameter' }, + ]); + }); + + it('enforces custom ranges, enum membership, string limits, and scalar array elements', () => { + expect( + resolveAgentInferenceParameters({ + provider: 'custom', + model: 'test-model', + parameters: { + seed: 101, + do_sample: 'true', + reasoning_format: 'other', + samplers: [{ apiKey: 'never-echo' }], + include_reasoning: true, + mirostat: 'toolong', + stop_token_ids: [1, 2], + }, + customParams: custom([ + { key: 'seed', type: SettingTypes.Number, range: { min: 0, max: 100 } }, + { key: 'do_sample', type: SettingTypes.Boolean }, + { key: 'reasoning_format', type: SettingTypes.Enum, options: ['hidden'] }, + { key: 'samplers', type: SettingTypes.Array, maxTags: 2 }, + { key: 'include_reasoning', type: SettingTypes.Boolean }, + { key: 'mirostat', type: SettingTypes.String, minText: 1, maxText: 3 }, + { key: 'stop_token_ids', type: SettingTypes.Array, minTags: 1, maxTags: 4 }, + ]), + }), + ).toEqual({ + parameters: {}, + customParameters: { include_reasoning: true, stop_token_ids: [1, 2] }, + issues: [ + { path: ['seed'], code: 'invalid_parameter' }, + { path: ['do_sample'], code: 'invalid_parameter' }, + { path: ['reasoning_format'], code: 'invalid_parameter' }, + { path: ['samplers'], code: 'invalid_parameter' }, + { path: ['mirostat'], code: 'invalid_parameter' }, + ], + }); + }); + + it('does not synthesize custom defaults or allow declarations to widen builtin fields', () => { + expect( + resolveAgentInferenceParameters({ + provider: 'openAI', + model: 'test-model', + parameters: { temperature: 'high', thinkingBudget: 1024 }, + customParams: custom([ + { key: 'temperature', type: SettingTypes.String }, + { key: 'thinkingBudget', type: SettingTypes.Number }, + { key: 'seed', type: SettingTypes.Number, default: 42 }, + ]), + }), + ).toEqual({ + parameters: {}, + customParameters: {}, + issues: [ + { path: ['temperature'], code: 'invalid_parameter' }, + { path: ['thinkingBudget'], code: 'unsupported_parameter' }, + ], + }); + }); + + it.each([ + 'model', + 'modelName', + 'models', + 'route', + 'fallbacks', + 'fallbackClient', + 'modelId', + 'timeoutMs', + 'abortSignal', + 'transport', + 'azureOpenAIApiKey', + 'modelKwargs', + 'configuration', + 'apiKey', + 'baseURL', + 'baseUrl', + 'user', + 'userId', + 'tools', + 'tool_choice', + 'toolChoice', + 'input', + 'contents', + 'prompt', + 'title', + 'modelLabel', + 'createdAt', + 'greeting', + 'default_headers', + 'fallback', + 'provider', + 'providers', + 'callback', + 'request_options', + 'callbacks', + 'timeout', + 'retries', + 'maxRetries', + 'signal', + 'fetch', + 'tags', + 'metadata', + 'client', + 'clientOptions', + 'httpAgent', + 'headers', + 'defaultHeaders', + 'endpoint', + 'conversationId', + 'messages', + 'presetOverride', + 'additionalModelRequestFields', + 'reasoning', + 'thinkingConfig', + 'response_format', + '__proto__', + 'constructor', + 'prototype', + ])('cannot custom-declare control or opaque field %s', (key) => { + const result = resolveAgentInferenceParameters({ + provider: 'custom', + model: 'test-model', + parameters: { [key]: 'never-echo-this-value', temperature: 0.5 }, + customParams: custom([{ key, type: SettingTypes.String }]), + }); + expect(result).toEqual({ + parameters: { temperature: 0.5 }, + customParameters: {}, + issues: [{ path: [key], code: 'unsupported_parameter' }], + }); + expect(JSON.stringify(result)).not.toContain('never-echo-this-value'); + }); +}); diff --git a/packages/api/src/agents/inference.ts b/packages/api/src/agents/inference.ts new file mode 100644 index 00000000000..ac7d98ebc1b --- /dev/null +++ b/packages/api/src/agents/inference.ts @@ -0,0 +1,560 @@ +import { z } from 'zod'; +import { tConversationSchema, THINKING_BLOCK_BINDING } from 'librechat-data-provider'; +import type { Agent, TEndpoint } from 'librechat-data-provider'; +import { sanitizeModelParameters } from './parameters'; + +type InferenceIssue = { + path: string[]; + code: 'unsupported_parameter' | 'invalid_parameter'; +}; + +type InferenceResult = { + parameters: Record; + customParameters: Record; + issues: InferenceIssue[]; +}; + +// Runtime-only provenance survives object spreads, but cannot arrive through JSON input. +const inferenceSource = Symbol('agentInferenceSource'); +type AgentWithInferenceSource = Agent & { + [inferenceSource]?: { + provider: string; + resolvedProvider: string; + resolvedParameters: Agent['model_parameters']; + parameters: Record; + }; +}; + +/** Restore only input captured for this exact SDK configuration, never a submitted envelope. */ +export function restoreAgentInferenceInput(agent: Agent): Agent { + const source = (agent as AgentWithInferenceSource)[inferenceSource]; + if ( + source == null || + source.resolvedProvider !== agent.provider || + source.resolvedParameters !== agent.model_parameters + ) { + return agent; + } + return { + ...agent, + provider: source.provider, + model_parameters: { ...source.parameters } as Agent['model_parameters'], + }; +} + +/** Capture validated flat values before this runtime instance is reused for another run. */ +export function retainAgentInferenceInput( + agent: Agent, + provider: string, + parameters: Record, +): void { + (agent as AgentWithInferenceSource)[inferenceSource] = { + provider, + resolvedProvider: agent.provider, + resolvedParameters: agent.model_parameters, + parameters: { ...parameters }, + }; +} + +type Family = 'openai' | 'anthropic' | 'google' | 'bedrock' | 'mistral'; +type CustomDefinition = NonNullable< + NonNullable['paramDefinitions'] +>[number]; +type Fields = Readonly>; + +const flat = tConversationSchema.shape; +const tokens = z.number().finite().int().nonnegative(); +const probability = z.number().finite().min(0).max(1); +const penalty = z.number().finite().min(-2).max(2); +const region = z + .string() + .max(63) + .regex(/^[a-z][a-z0-9]*(?:-[a-z0-9]+)+-\d+$/); +const legacyThinking = z + .object({ + type: z.enum(['enabled', 'adaptive', 'disabled', 'between_tools']), + display: flat.thinkingDisplay, + budget_tokens: tokens.optional(), + block_binding: z + .object({ + prefix_mismatch_behavior: z.literal(THINKING_BLOCK_BINDING.prefix_mismatch_behavior), + }) + .strict() + .optional(), + }) + .strict(); +const common: Fields = { + maxContextTokens: flat.maxContextTokens.pipe(z.number().finite().int().min(-1)), + fileTokenLimit: flat.fileTokenLimit.pipe(tokens), + resendFiles: flat.resendFiles, + imageDetail: flat.imageDetail, + promptPrefix: flat.promptPrefix, + disableStreaming: flat.disableStreaming, +}; +const openai: Fields = { + ...common, + temperature: flat.temperature.pipe(z.number().finite().min(0).max(2)), + top_p: flat.top_p.pipe(probability), + frequency_penalty: flat.frequency_penalty.pipe(penalty), + presence_penalty: flat.presence_penalty.pipe(penalty), + max_tokens: flat.max_tokens.pipe(tokens), + stop: flat.stop, + reasoning_effort: flat.reasoning_effort, + reasoning_summary: flat.reasoning_summary, + reasoning_mode: flat.reasoning_mode, + reasoning_context: flat.reasoning_context, + verbosity: flat.verbosity, + useResponsesApi: flat.useResponsesApi, + web_search: flat.web_search, +}; +const openrouter: Fields = { + ...openai, + promptCache: flat.promptCache, + promptCacheTtl: flat.promptCacheTtl, +}; +const anthropic: Fields = { + ...common, + temperature: flat.temperature.pipe(probability), + topP: flat.topP.pipe(probability), + topK: flat.topK.pipe(tokens), + maxOutputTokens: flat.maxOutputTokens.pipe(tokens), + stop: flat.stop, + promptCache: flat.promptCache, + promptCacheTtl: flat.promptCacheTtl, + thinking: flat.thinking, + thinkingBudget: flat.thinkingBudget.pipe(tokens), + effort: flat.effort, + thinkingDisplay: flat.thinkingDisplay, + web_search: flat.web_search, +}; +const google: Fields = { + ...common, + temperature: flat.temperature.pipe(z.number().finite().min(0).max(2)), + topP: flat.topP.pipe(probability), + topK: flat.topK.pipe(tokens), + maxOutputTokens: flat.maxOutputTokens.pipe(tokens), + stop: flat.stop, + thinking: flat.thinking, + thinkingBudget: flat.thinkingBudget.pipe(z.number().finite().int().min(-1)), + thinkingLevel: flat.thinkingLevel, + web_search: flat.web_search, + url_context: flat.url_context, +}; +const bedrock: Fields = { + ...common, + region, + reasoning_effort: flat.reasoning_effort, + temperature: flat.temperature.pipe(probability), + topP: flat.topP.pipe(probability), + topK: flat.topK.pipe(tokens), + maxTokens: flat.maxTokens.pipe(tokens), + stop: flat.stop, + system: flat.system, + promptCache: flat.promptCache, + promptCacheTtl: flat.promptCacheTtl, +}; +const bedrockAnthropic: Fields = { + ...bedrock, + thinking: flat.thinking, + thinkingBudget: flat.thinkingBudget.pipe(tokens), + effort: flat.effort, + thinkingDisplay: flat.thinkingDisplay, +}; +const mistral: Fields = { + ...common, + temperature: flat.temperature.pipe(probability), + topP: flat.topP.pipe(probability), + maxTokens: flat.maxTokens.pipe(tokens), + stop: flat.stop, +}; + +const profiles: Record = { openai, anthropic, google, bedrock, mistral }; +const builtinKeys = new Set(Object.values(profiles).flatMap((fields) => Object.keys(fields))); +const aliases: Readonly> = { + max_context_tokens: 'maxContextTokens', + frequencyPenalty: 'frequency_penalty', + presencePenalty: 'presence_penalty', + thinking_budget: 'thinkingBudget', + top_k: 'topK', + stopSequences: 'stop', + stop_sequences: 'stop', + reasoningEffort: 'reasoning_effort', + reasoningSummary: 'reasoning_summary', + reasoningMode: 'reasoning_mode', + reasoningContext: 'reasoning_context', +}; + +// Admin declarations describe flat wire extras, never clients or request identity. +const conversationKeys = new Set( + Object.keys(flat).map((key) => key.replace(/_/g, '').toLowerCase()), +); +const reservedCustomKeys = new Set( + [ + 'model', + 'modelName', + 'modelId', + 'modelProvider', + 'models', + 'modelList', + 'modelKwargs', + 'invocationKwargs', + 'route', + 'routing', + 'fallback', + 'fallbacks', + 'fallbackClient', + 'fallbackClients', + 'provider', + 'providers', + 'messages', + 'input', + 'contents', + 'prompt', + 'system', + 'context', + 'examples', + 'instructions', + 'additionalInstructions', + 'tools', + 'toolChoice', + 'functions', + 'functionCall', + 'parallelToolCalls', + 'includeServerSideToolInvocations', + 'callbacks', + 'callback', + 'callbackManager', + 'timeout', + 'requestTimeout', + 'timeoutMs', + 'requestTimeoutMs', + 'retries', + 'maxRetries', + 'signal', + 'abortSignal', + 'abortController', + 'fetch', + 'tags', + 'metadata', + 'client', + 'clientOptions', + 'httpClient', + 'httpOptions', + 'fetchOptions', + 'transport', + 'connectionOptions', + 'agent', + 'maxConcurrency', + 'verbose', + 'configuration', + 'config', + 'httpAgent', + 'httpsAgent', + 'headers', + 'defaultHeaders', + 'defaultQuery', + 'query', + 'body', + 'extraBody', + 'method', + 'requestOptions', + 'apiKey', + 'apiKeys', + 'openAIApiKey', + 'anthropicApiKey', + 'googleApiKey', + 'azureOpenAIApiKey', + 'azureOpenAIApiInstanceName', + 'azureOpenAIApiDeploymentName', + 'azureOpenAIApiVersion', + 'apiVersion', + 'safetyIdentifier', + 'apiBase', + 'apiBaseURL', + 'baseURL', + 'basePath', + 'url', + 'endpoint', + 'endpoints', + 'deployment', + 'deploymentName', + 'deploymentId', + 'project', + 'projectId', + 'organization', + 'organizationId', + 'user', + 'userId', + 'credentials', + 'credential', + 'accessKeyId', + 'secretAccessKey', + 'sessionToken', + 'password', + 'token', + 'auth', + 'authorization', + 'connection', + 'proxy', + 'region', + 'stream', + 'streaming', + 'cache', + 'store', + 'previousResponseId', + 'conversation', + 'conversationId', + 'sessionId', + 'threadId', + 'assistantId', + 'agentId', + 'presetOverride', + 'additionalModelRequestFields', + 'reasoning', + 'thinkingConfig', + 'responseFormat', + 'generationConfig', + 'safetySettings', + 'constructor', + 'prototype', + ].map((key) => key.replace(/_/g, '').toLowerCase()), +); + +function isCustomKey(key: string): boolean { + return ( + /^[a-zA-Z][a-zA-Z0-9_]*$/.test(key) && + !key.toLowerCase().startsWith('lc_') && + !conversationKeys.has(key.replace(/_/g, '').toLowerCase()) && + !reservedCustomKeys.has(key.replace(/_/g, '').toLowerCase()) + ); +} + +function knownFamily(provider: string): Family | undefined { + switch (provider.toLowerCase()) { + case 'openai': + case 'azureopenai': + return 'openai'; + case 'anthropic': + return 'anthropic'; + case 'google': + case 'vertexai': + return 'google'; + case 'bedrock': + return 'bedrock'; + case 'mistral': + case 'mistralai': + return 'mistral'; + default: + return undefined; + } +} + +function parameterKey(key: string, family: Family): string { + if (key === 'topP' || key === 'top_p') { + return family === 'openai' ? 'top_p' : 'topP'; + } + if (['max_tokens', 'maxTokens', 'maxOutputTokens', 'max_output_tokens'].includes(key)) { + if (family === 'openai') { + return 'max_tokens'; + } + return family === 'bedrock' || family === 'mistral' ? 'maxTokens' : 'maxOutputTokens'; + } + return Object.prototype.hasOwnProperty.call(aliases, key) ? aliases[key] : key; +} + +function customSchema( + definition: CustomDefinition, + builtinValue?: unknown, +): z.ZodTypeAny | undefined { + let type: string = definition.type ?? typeof builtinValue; + if (definition.type == null && Array.isArray(builtinValue)) { + type = 'array'; + } + if (definition.type == null && definition.options != null && typeof builtinValue === 'string') { + type = 'enum'; + } + switch (type) { + case 'number': { + let schema = z.number().finite(); + const range = definition.range; + if (range) { + schema = schema.min(range.min).max(range.max); + } + const positiveMin = range?.positiveMin; + return positiveMin == null + ? schema + : schema.refine((value) => value === range?.min || value >= positiveMin); + } + case 'boolean': + return z.boolean(); + case 'string': { + let schema = z.string(); + if (definition.minText != null) { + schema = schema.min(definition.minText); + } + if (definition.maxText != null) { + schema = schema.max(definition.maxText); + } + return schema; + } + case 'enum': { + const options = definition.options; + return options?.length ? z.string().refine((value) => options.includes(value)) : undefined; + } + case 'array': { + let schema = z.array(z.union([z.string(), z.number().finite()])); + if (definition.minTags != null) { + schema = schema.min(definition.minTags); + } + if (definition.maxTags != null) { + schema = schema.max(definition.maxTags); + } + const options = definition.options; + return options == null + ? schema + : schema.refine((values) => + values.every((value) => typeof value === 'string' && options.includes(value)), + ); + } + default: + return undefined; + } +} + +/** Extra YAML controls must satisfy the same policy used to project their wire values. */ +export function isAgentInferenceExtension(definition: CustomDefinition): boolean { + if (!isCustomKey(definition.key) || builtinKeys.has(definition.key)) { + return false; + } + const schema = customSchema(definition); + return ( + schema != null && + (definition.default === undefined || schema.safeParse(definition.default).success) + ); +} + +/** Projects persisted or submitted settings without admitting SDK constructor options. */ +export function resolveAgentInferenceParameters({ + parameters, + provider, + model, + customParams, + availableRegions, +}: { + parameters: Record | null | undefined; + provider: string; + model: string; + customParams?: TEndpoint['customParams']; + availableRegions?: readonly string[]; +}): InferenceResult { + const nativeFamily = + knownFamily(provider) ?? (provider.toLowerCase().startsWith('mistral') ? 'mistral' : undefined); + const family = + ((nativeFamily == null || nativeFamily === 'openai') && + knownFamily(customParams?.defaultParamsEndpoint ?? '')) || + nativeFamily || + 'openai'; + const isOpenRouter = + provider.toLowerCase() === 'openrouter' || + customParams?.defaultParamsEndpoint?.toLowerCase() === 'openrouter'; + let fields = profiles[family]; + if (family === 'bedrock' && /anthropic|claude/i.test(model)) { + fields = bedrockAnthropic; + } + if (family === 'openai' && isOpenRouter) { + fields = openrouter; + } + const definitions = new Map( + (customParams?.paramDefinitions ?? []).map((definition) => [ + parameterKey(definition.key, family), + definition, + ]), + ); + const result: InferenceResult = { parameters: {}, customParameters: {}, issues: [] }; + if (parameters == null) { + return result; + } + const source = { ...parameters }; + const stored = source.additionalModelRequestFields; + if ( + family === 'bedrock' && + stored != null && + typeof stored === 'object' && + !Array.isArray(stored) + ) { + const preferences = stored as Record; + if (fields === bedrockAnthropic) { + const thinking = legacyThinking.safeParse(preferences.thinking); + if (source.thinking === undefined && thinking.success) { + source.thinking = thinking.data; + } + const output = z.object({ effort: flat.effort }).safeParse(preferences.output_config); + if (source.effort === undefined && output.success) { + source.effort = output.data.effort; + } + } else { + const effort = flat.reasoning_effort.safeParse(preferences.reasoning_config); + if (source.reasoning_effort === undefined && effort.success) { + source.reasoning_effort = effort.data; + } + } + } + if (family === 'anthropic' || (family === 'bedrock' && fields === bedrockAnthropic)) { + const thinking = legacyThinking.safeParse(source.thinking); + if (thinking.success) { + source.thinking = !['disabled', 'between_tools'].includes(thinking.data.type); + source.thinkingDisplay ??= thinking.data.display; + source.thinkingBudget ??= thinking.data.budget_tokens; + } + } + const sanitized = sanitizeModelParameters(source); + for (const [key, value] of Object.entries(source)) { + if (value == null || (typeof value === 'string' && value.trim() === '')) { + continue; + } + const target = parameterKey(key, family); + const builtin = Object.prototype.hasOwnProperty.call(fields, target); + const definition = definitions.get(target); + let schema = builtin ? fields[target] : undefined; + if (!builtin && !builtinKeys.has(target) && isCustomKey(key) && definition != null) { + schema = customSchema(definition); + } + if (schema == null) { + result.issues.push({ path: [key], code: 'unsupported_parameter' }); + continue; + } + let candidate = sanitized[key]; + if (!builtin && definition?.type === 'number' && typeof candidate === 'string') { + candidate = Number(candidate); + } + const parsed = schema.safeParse(candidate); + if ( + !parsed.success || + parsed.data == null || + (target === 'region' && + availableRegions != null && + !availableRegions.includes(String(parsed.data))) + ) { + result.issues.push({ path: [key], code: 'invalid_parameter' }); + continue; + } + if (builtin && definition != null) { + const constraint = customSchema(definition, parsed.data); + if ( + constraint == null || + !constraint.safeParse(parsed.data).success || + (definition.options != null && + !Array.isArray(parsed.data) && + !definition.options.includes(String(parsed.data))) + ) { + result.issues.push({ path: [key], code: 'invalid_parameter' }); + continue; + } + } + const destination = builtin ? result.parameters : result.customParameters; + if (key === target || !Object.prototype.hasOwnProperty.call(destination, target)) { + destination[target] = parsed.data; + } + } + return result; +} diff --git a/packages/api/src/agents/initialize.ts b/packages/api/src/agents/initialize.ts index 1bed769edd7..51a57bc6a81 100644 --- a/packages/api/src/agents/initialize.ts +++ b/packages/api/src/agents/initialize.ts @@ -111,6 +111,11 @@ import { resolveAttachedWorkspaceCommandTimeoutMax, resolveAttachedWorkspaceCommandTimeoutDefault, } from '~/code/command'; +import { + resolveAgentInferenceParameters, + restoreAgentInferenceInput, + retainAgentInferenceInput, +} from './inference'; import { formatChatProjectInstructions, hydrateChatProjectContextResources, @@ -139,8 +144,10 @@ import { hasActiveFileFieldPolicy } from '~/protection'; import { applyBackgroundToolCalls } from './background'; import { applyTurnDelivery } from './files/delivery'; import { generateArtifactsPrompt } from '~/prompts'; +import { getAgentInferencePolicy } from './input'; import { getProviderConfig } from '~/endpoints'; import { primeResources } from './resources'; +import { pinAgentModel } from './pin'; /** * Fraction of context budget reserved as headroom when no explicit maxContextTokens is set. @@ -1174,7 +1181,7 @@ export async function initializeAgent( db?: InitializeAgentDbMethods, ): Promise { const { - agent: inputAgent, + agent: suppliedAgent, loadTools, requestFiles = [], authorizedRunFiles, @@ -1193,6 +1200,7 @@ export async function initializeAgent( } const shouldUseChatProjectContext = useChatProjectContext ?? params.req?.chatProjectContextEnabled === true; + const inputAgent = restoreAgentInferenceInput(suppliedAgent); const agent = shouldUseChatProjectContext ? { ...inputAgent } : inputAgent; const { user, appConfig } = runtime; const requestFileOwnerId = user?.id; @@ -1498,18 +1506,6 @@ export async function initializeAgent( const effectiveToolNames = extraAllowedToolNames.length > 0 ? [...baseToolNames, ...extraAllowedToolNames] : baseToolNames; const requestedToolNames = effectiveToolNames; - const _modelOptions = structuredClone( - Object.assign( - { model: agent.model }, - agent.model_parameters ?? { model: agent.model }, - isInitialAgent === true ? endpointOption?.model_parameters : {}, - ), - ); - - const { resendFiles, maxContextTokens, imageDetail, modelOptions } = extractLibreChatParams( - _modelOptions as Record, - ); - const provider = agent.provider; agent.endpoint = provider; @@ -1525,10 +1521,18 @@ export async function initializeAgent( agent.provider = overrideProvider; } - const finalModelOptions = { - ...modelOptions, - model: agent.model, - }; + const inference = resolveAgentInferenceParameters({ + parameters: { + ...agent.model_parameters, + ...(isInitialAgent === true ? endpointOption?.model_parameters : undefined), + }, + ...getAgentInferencePolicy(provider, appConfig, customEndpointConfig), + model: agent.model ?? '', + }); + const { resendFiles, maxContextTokens, imageDetail, modelOptions } = extractLibreChatParams( + inference.parameters, + ); + const finalModelOptions = pinAgentModel(modelOptions, agent.model); const options: InitializeResultBase = await getOptions({ runtime: { @@ -1538,6 +1542,7 @@ export async function initializeAgent( }, endpoint: provider, model_parameters: finalModelOptions, + inferenceParameters: inference.customParameters, db, }); @@ -2502,6 +2507,10 @@ export async function initializeAgent( if (options.configOptions) { (agent.model_parameters as Record).configuration = options.configOptions; } + retainAgentInferenceInput(agent, provider, { + ...inference.parameters, + ...inference.customParameters, + }); /** * Resolves an `instructionsPrompt` link before special-vars substitution, so diff --git a/packages/api/src/agents/input.spec.ts b/packages/api/src/agents/input.spec.ts new file mode 100644 index 00000000000..7fb4782d693 --- /dev/null +++ b/packages/api/src/agents/input.spec.ts @@ -0,0 +1,123 @@ +import { z } from 'zod'; +import { SettingTypes } from 'librechat-data-provider'; +import type { AppConfig } from '@librechat/data-schemas'; +import { + getAgentInferencePolicy, + parseAgentInferenceInput, + applyAgentInferenceInput, +} from './input'; + +const endpoint = { + name: 'Gateway', + apiKey: 'server-secret', + baseURL: 'https://gateway.invalid/v1', + models: { default: ['permitted'], fetch: false }, + customParams: { + defaultParamsEndpoint: 'custom', + paramDefinitions: [{ key: 'min_p', type: SettingTypes.Number, range: { min: 0, max: 1 } }], + }, +}; +const appConfig = { endpoints: { custom: [endpoint] } } satisfies Pick; + +const input = (parameters: Record) => + parseAgentInferenceInput({ parameters, provider: 'openAI', model: 'permitted' }); + +describe('Agent inference input', () => { + it('does not turn an omitted parameter patch into a mutable field', () => { + const patch = {}; + applyAgentInferenceInput(patch, { provider: 'openAI', model: 'permitted' }); + expect(Object.keys(patch)).toEqual([]); + }); + it('preserves omission and numeric normalization', () => { + expect(parseAgentInferenceInput({ provider: 'openAI' })).toBeUndefined(); + expect(input({ temperature: '0.7', max_tokens: '', top_p: 0 })).toEqual({ + temperature: 0.7, + top_p: 0, + }); + }); + + it.each(['modelKwargs', 'configuration', 'fallbacks', 'apiKey', 'baseURL', 'callbacks'])( + 'rejects the SDK control %s without disclosing its value', + (key) => { + const secret = 'never-return-this-submitted-value'; + try { + input({ temperature: 0.5, [key]: { nested: secret } }); + throw new Error('Expected invalid input'); + } catch (error) { + expect(error).toBeInstanceOf(z.ZodError); + expect(JSON.stringify(error)).not.toContain(secret); + expect((error as z.ZodError).issues).toEqual([ + expect.objectContaining({ path: ['model_parameters', key] }), + ]); + } + }, + ); + + it('does not let the redundant model field select a different model', () => { + expect(input({ model: 'permitted', temperature: 0.4 })).toEqual({ temperature: 0.4 }); + expect(() => input({ model: 'restricted' })).toThrow(z.ZodError); + }); + + it('uses only the selected endpoint definitions, without copying its transport', () => { + expect(getAgentInferencePolicy('Gateway', appConfig)).toEqual({ + provider: 'Gateway', + customParams: endpoint.customParams, + }); + const params = { temperature: 0.3, min_p: 0.1 }; + expect( + parseAgentInferenceInput({ + parameters: params, + provider: 'Gateway', + model: 'permitted', + appConfig, + }), + ).toEqual(params); + expect(() => input({ min_p: 0.1 })).toThrow(z.ZodError); + expect(getAgentInferencePolicy('gAtEwAy', appConfig).customParams).toBeUndefined(); + expect( + parseAgentInferenceInput({ + parameters: {}, + provider: 'Gateway', + model: 'permitted', + appConfig, + }), + ).not.toHaveProperty('apiKey'); + }); + + it('validates Bedrock regions against the already-loaded administrator choices', () => { + const request = { + parameters: { region: 'us-west-2', reasoning_effort: 'low' }, + provider: 'bedrock', + model: 'moonshot.kimi', + appConfig: { endpoints: { bedrock: { availableRegions: ['us-west-2'] } } }, + }; + expect(parseAgentInferenceInput(request)).toEqual(request.parameters); + expect(() => + parseAgentInferenceInput({ + ...request, + appConfig: { endpoints: { bedrock: { availableRegions: ['us-east-1'] } } }, + }), + ).toThrow(z.ZodError); + }); + + it('still rejects Bedrock SDK envelopes on new writes despite recovering known legacy preferences', () => { + expect(() => + parseAgentInferenceInput({ + provider: 'bedrock', + model: 'us.anthropic.claude-sonnet-5', + parameters: { additionalModelRequestFields: { thinking: { type: 'disabled' } } }, + }), + ).toThrow(z.ZodError); + }); + + it('validates a custom value against administrator constraints', () => { + expect(() => + parseAgentInferenceInput({ + parameters: { min_p: 2 }, + provider: 'Gateway', + model: 'permitted', + appConfig, + }), + ).toThrow(z.ZodError); + }); +}); diff --git a/packages/api/src/agents/input.ts b/packages/api/src/agents/input.ts new file mode 100644 index 00000000000..c7d7fecb920 --- /dev/null +++ b/packages/api/src/agents/input.ts @@ -0,0 +1,92 @@ +import { z } from 'zod'; +import { normalizeEndpointName, extractEnvVariable } from 'librechat-data-provider'; +import type { AppConfig } from '@librechat/data-schemas'; +import type { TEndpoint } from 'librechat-data-provider'; +import { resolveCustomEndpointParameters } from '../endpoints/custom/config'; +import { resolveAgentInferenceParameters } from './inference'; + +/** Uses the caller's loaded configuration without database reads or credential resolution. */ +export function getAgentInferencePolicy( + provider: string, + appConfig?: Pick, + customEndpoint?: Partial, +): { + provider: string; + customParams?: TEndpoint['customParams']; + availableRegions?: readonly string[]; +} { + const endpoint = + customEndpoint ?? + appConfig?.endpoints?.custom?.find( + (entry) => normalizeEndpointName(entry.name) === normalizeEndpointName(provider), + ); + return { + provider: endpoint?.provider ?? provider, + customParams: endpoint + ? resolveCustomEndpointParameters(endpoint, extractEnvVariable(endpoint.baseURL ?? '')) + : undefined, + availableRegions: appConfig?.endpoints?.bedrock?.availableRegions, + }; +} + +type InferenceWrite = { + provider?: string | null; + model?: string | null; + model_parameters?: Record; +}; + +/** Validate a request-owned patch without adding absent fields to partial updates. */ +export function applyAgentInferenceInput( + update: InferenceWrite, + previous?: Pick, + appConfig?: Pick, +): void { + if (update.model_parameters == null) { + return; + } + update.model_parameters = parseAgentInferenceInput({ + parameters: update.model_parameters, + provider: update.provider ?? previous?.provider, + model: update.model ?? previous?.model, + appConfig, + }); +} + +/** Reject unsupported new input without disclosing submitted values in validation errors. */ +export function parseAgentInferenceInput({ + parameters, + provider, + model, + appConfig, +}: { + parameters?: Record; + provider?: string | null; + model?: string | null; + appConfig?: Pick; +}): Record | undefined { + if (parameters == null) { + return undefined; + } + const { model: suppliedModel, ...settings } = parameters; + const result = resolveAgentInferenceParameters({ + parameters: settings, + ...getAgentInferencePolicy(provider ?? '', appConfig), + model: model ?? '', + }); + if (suppliedModel != null && suppliedModel !== '' && suppliedModel !== model) { + result.issues.push({ path: ['model'], code: 'unsupported_parameter' }); + } + if (result.issues.length > 0) { + throw new z.ZodError( + result.issues.map((issue) => ({ + code: z.ZodIssueCode.custom, + path: ['model_parameters', ...issue.path], + message: + issue.code === 'unsupported_parameter' + ? 'This inference parameter is not permitted for the selected endpoint.' + : 'Invalid inference parameter value.', + })), + ); + } + return { ...result.parameters, ...result.customParameters }; +} diff --git a/packages/api/src/agents/pin.spec.ts b/packages/api/src/agents/pin.spec.ts new file mode 100644 index 00000000000..8b1898c388b --- /dev/null +++ b/packages/api/src/agents/pin.spec.ts @@ -0,0 +1,205 @@ +import { Providers } from '@librechat/agents'; +import { AuthKeys, EModelEndpoint } from 'librechat-data-provider'; +import type { Agent } from 'librechat-data-provider'; +import type { Request, Response } from 'express'; +import { getOpenAILLMConfig } from '~/endpoints/openai/llm'; +import { getGoogleConfig } from '~/endpoints/google/llm'; +import { validateAgentModel } from './validation'; +import { pinAgentModel } from './pin'; + +describe('pinAgentModel', () => { + it.each([undefined, null, []])('removes modelName without object kwargs (%p)', (modelKwargs) => { + expect( + pinAgentModel( + { + modelName: 'gpt-4o', + temperature: 0.2, + fallbacks: [{ provider: Providers.OPENAI, clientOptions: { model: 'gpt-4o' } }], + modelKwargs, + }, + 'gpt-4o-mini', + ), + ).toEqual({ model: 'gpt-4o-mini', temperature: 0.2, modelKwargs }); + }); + + it('copies routing kwargs without mutating saved parameters', () => { + const modelOptions = Object.freeze({ + model: 'gpt-4o', + modelName: 'gpt-4o', + temperature: 0.2, + fallbacks: [{ provider: Providers.OPENAI, clientOptions: { model: 'gpt-4o' } }], + modelKwargs: Object.freeze({ + model: 'gpt-4o', + models: ['gpt-4o'], + route: 'fallback', + fallbacks: ['gpt-4o'], + service_tier: 'flex', + }), + }); + + const pinned = pinAgentModel(modelOptions, 'gpt-4o-mini'); + + expect(pinned).toEqual({ + model: 'gpt-4o-mini', + temperature: 0.2, + modelKwargs: { service_tier: 'flex' }, + }); + expect(pinned.modelKwargs).not.toBe(modelOptions.modelKwargs); + expect(modelOptions).toHaveProperty('modelName', 'gpt-4o'); + expect(modelOptions).toHaveProperty('fallbacks', [ + { provider: Providers.OPENAI, clientOptions: { model: 'gpt-4o' } }, + ]); + expect(modelOptions.modelKwargs).toEqual({ + model: 'gpt-4o', + models: ['gpt-4o'], + route: 'fallback', + fallbacks: ['gpt-4o'], + service_tier: 'flex', + }); + }); +}); + +describe('pinned provider configuration', () => { + it.each([false, true])('pins OpenAI-compatible options (OpenRouter: %s)', (useOpenRouter) => { + const modelOptions = pinAgentModel( + { + model: 'gpt-4o', + modelName: 'gpt-4o', + temperature: 0.2, + fallbacks: [ + { + provider: Providers.OPENAI, + clientOptions: { + model: 'gpt-4o', + configuration: { baseURL: 'https://unchecked.invalid' }, + }, + }, + ], + modelKwargs: { + model: 'gpt-4o', + models: ['gpt-4o'], + route: 'fallback', + fallbacks: ['gpt-4o'], + service_tier: 'flex', + }, + }, + 'gpt-4o-mini', + ); + + const { llmConfig } = getOpenAILLMConfig({ + apiKey: 'test-api-key', + streaming: true, + endpoint: EModelEndpoint.openAI, + useOpenRouter, + modelOptions, + }); + + expect(llmConfig).toMatchObject({ model: 'gpt-4o-mini', temperature: 0.2 }); + expect(llmConfig).not.toHaveProperty('modelName'); + expect(llmConfig).not.toHaveProperty('fallbacks'); + expect(llmConfig.modelKwargs ?? {}).not.toHaveProperty('service_tier'); + }); + + it('pins Google options while preserving server-configured transport', () => { + const modelOptions = pinAgentModel( + { + model: 'gemini-2.5-pro', + modelName: 'gemini-2.5-pro', + temperature: 0.2, + maxOutputTokens: 1024, + apiKey: 'untrusted-api-key', + apiVersion: 'v1alpha', + baseUrl: 'https://unchecked.invalid', + customHeaders: { Authorization: 'untrusted-authorization' }, + fallbacks: [{ provider: Providers.OPENAI, clientOptions: { model: 'gpt-4o' } }], + }, + 'gemini-2.5-flash', + ); + + const { provider, llmConfig } = getGoogleConfig( + { [AuthKeys.GOOGLE_API_KEY]: 'server-api-key' }, + { + modelOptions, + reverseProxyUrl: 'https://configured.invalid/google', + authHeader: true, + headers: { 'X-Gateway': 'configured' }, + }, + ); + + expect(provider).toBe(Providers.GOOGLE); + expect(llmConfig).toMatchObject({ + model: 'gemini-2.5-flash', + temperature: 0.2, + maxOutputTokens: 1024, + apiKey: 'server-api-key', + baseUrl: 'https://configured.invalid/google', + customHeaders: { 'X-Gateway': 'configured', Authorization: 'Bearer server-api-key' }, + }); + expect(llmConfig).not.toHaveProperty('modelName'); + expect(llmConfig).not.toHaveProperty('fallbacks'); + expect(llmConfig).not.toHaveProperty('apiVersion'); + }); + + it('pins Vertex options without replacing the configured project', () => { + const modelOptions = pinAgentModel( + { + modelName: 'gemini-2.5-pro', + temperature: 0.2, + apiVersion: 'v1alpha', + authOptions: { projectId: 'unchecked-project' }, + }, + 'gemini-2.5-flash', + ); + + const { provider, llmConfig } = getGoogleConfig( + { [AuthKeys.GOOGLE_SERVICE_KEY]: { project_id: 'configured-project' } }, + { modelOptions, forceVertex: true, projectId: 'configured-project' }, + ); + + expect(provider).toBe(Providers.VERTEXAI); + expect(llmConfig).toMatchObject({ + model: 'gemini-2.5-flash', + temperature: 0.2, + authOptions: { projectId: 'configured-project' }, + }); + expect(llmConfig).not.toHaveProperty('modelName'); + expect(llmConfig).not.toHaveProperty('apiVersion'); + }); + + it('still rejects a disallowed declared model when aliases name an allowed model', async () => { + const logViolation = jest.fn().mockResolvedValue(undefined); + const modelParameters = { + temperature: 0.2, + maxContextTokens: null, + max_context_tokens: null, + max_output_tokens: null, + top_p: null, + frequency_penalty: null, + presence_penalty: null, + modelName: 'gpt-4o-mini', + modelKwargs: { model: 'gpt-4o-mini' }, + }; + const agent: Agent = { + id: 'restricted-agent', + name: 'Restricted agent', + description: null, + created_at: 0, + avatar: null, + provider: Providers.OPENAI, + model: 'gpt-4o', + model_parameters: modelParameters, + }; + + const validation = await validateAgentModel({ + req: {} as Request, + res: {} as Response, + agent, + modelsConfig: { [EModelEndpoint.openAI]: ['gpt-4o-mini'] }, + logViolation, + }); + + expect(validation.isValid).toBe(false); + expect(validation.error?.message).toContain('illegal_model_request'); + expect(logViolation).toHaveBeenCalledTimes(1); + }); +}); diff --git a/packages/api/src/agents/pin.ts b/packages/api/src/agents/pin.ts new file mode 100644 index 00000000000..a4121e2e39b --- /dev/null +++ b/packages/api/src/agents/pin.ts @@ -0,0 +1,24 @@ +/** + * Keys in free-form `modelKwargs` that OpenAI-compatible clients send verbatim in the request body + * and that select a different model: the model itself, plus OpenRouter (`models`, `route`) and + * LiteLLM (`fallbacks`) server-side fallback routing. + */ +const MODEL_ROUTING_KWARGS = new Set(['model', 'models', 'route', 'fallbacks']); + +type PinnedModelOptions = Omit & { model: M }; + +/** Final model guard after inference projection; server-owned deployment mapping runs later. */ +export function pinAgentModel, M>( + modelOptions: T, + model: M, +): PinnedModelOptions { + const { fallbacks: _fallbacks, modelName: _modelName, ...options } = modelOptions; + const { modelKwargs } = options; + if (modelKwargs == null || typeof modelKwargs !== 'object' || Array.isArray(modelKwargs)) { + return { ...options, model }; + } + const forwardedKwargs = Object.fromEntries( + Object.entries(modelKwargs).filter(([key]) => !MODEL_ROUTING_KWARGS.has(key)), + ); + return { ...options, model, modelKwargs: forwardedKwargs }; +} diff --git a/packages/api/src/agents/profile.spec.ts b/packages/api/src/agents/profile.spec.ts new file mode 100644 index 00000000000..99da40f147c --- /dev/null +++ b/packages/api/src/agents/profile.spec.ts @@ -0,0 +1,530 @@ +import yaml from 'js-yaml'; +import { HumanMessage, SystemMessage } from '@langchain/core/messages'; +import { Providers, initializeModel, addTailCacheControl } from '@librechat/agents'; +import { + endpointSchema, + extractEnvVariable, + EModelEndpoint, + ReasoningParameterFormat, + SettingTypes, + ComponentTypes, + EImageOutputType, + FileSources, +} from 'librechat-data-provider'; +import type { TEndpoint } from 'librechat-data-provider'; +import type { AppConfig } from '@librechat/data-schemas'; +import { loadCustomEndpointsConfig, usesOpenRouterCatalog } from '../endpoints/custom/config'; +import { getAgentInferencePolicy, parseAgentInferenceInput } from './input'; +import { initializeCustom } from '../endpoints/custom/initialize'; +import { resolveAgentInferenceParameters } from './inference'; +import { getOpenAIConfig } from '../endpoints/openai/config'; +import { createCustomConfigLoader } from '../app/loader'; + +const variable = 'TEST_INFERENCE_GATEWAY_URL'; +const original = process.env[variable]; +const endpoint = (options: Partial = {}) => ({ + name: 'Gateway', + apiKey: 'synthetic-key', + baseURL: '${TEST_INFERENCE_GATEWAY_URL}', + models: { default: ['permitted'], fetch: false }, + ...options, +}); + +afterEach(() => { + if (original === undefined) delete process.env[variable]; + else process.env[variable] = original; +}); + +describe('trusted inference profile consistency', () => { + it.each([undefined, { defaultParamsEndpoint: 'custom' }])( + 'resolves the configured URL before discovery, write validation, and runtime projection', + async (customParams) => { + process.env[variable] = 'https://openrouter.ai/api/v1'; + const text = yaml.dump({ + version: '1.3.17', + endpoints: { custom: [endpoint({ customParams })] }, + }); + const config = await createCustomConfigLoader({ + defaultConfigPath: 'inference-profile.yaml', + loadLocal: () => text, + redactConfig: (value) => value, + })(false); + const loaded = endpointSchema.parse(config?.endpoints?.custom?.[0]); + const appConfig = { endpoints: { custom: [loaded] } }; + const advertised = loadCustomEndpointsConfig([loaded])?.Gateway; + const policy = getAgentInferencePolicy('Gateway', appConfig); + expect(policy.customParams).toEqual(advertised?.customParams); + expect(policy.customParams?.defaultParamsEndpoint).toBe('openrouter'); + const values = { promptCache: false, promptCacheTtl: '1h' }; + const parameters = parseAgentInferenceInput({ + provider: 'Gateway', + model: 'permitted', + parameters: values, + appConfig, + }); + expect(parameters).toEqual(values); + const runtime = resolveAgentInferenceParameters({ + ...getAgentInferencePolicy('Gateway', appConfig, loaded), + model: 'permitted', + parameters: values, + }); + expect(runtime.parameters).toEqual(values); + const { llmConfig, configOptions, provider } = getOpenAIConfig( + 'synthetic-key', + { + reverseProxyUrl: extractEnvVariable(loaded.baseURL), + customParams: loaded.customParams, + modelOptions: { ...runtime.parameters, model: 'permitted' }, + }, + 'Gateway', + ); + expect(llmConfig.promptCache).not.toBe(true); + const bodies: Record[] = []; + const client = initializeModel({ + provider: provider ?? Providers.OPENAI, + clientOptions: { + ...llmConfig, + streaming: false, + configuration: { + ...configOptions, + fetch: async (_url: unknown, init: { body?: unknown } | undefined) => { + bodies.push(JSON.parse(String(init?.body))); + return Response.json({ + id: 'chatcmpl-cache', + object: 'chat.completion', + created: 1, + model: 'permitted', + choices: [ + { + index: 0, + message: { role: 'assistant', content: 'ok' }, + finish_reason: 'stop', + }, + ], + usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 }, + }); + }, + }, + }, + }); + await client.invoke('hello'); + expect(bodies).toHaveLength(1); + expect(JSON.stringify(bodies[0])).not.toContain('cache_control'); + expect(llmConfig).not.toHaveProperty('promptCacheTtl'); + expect(loaded.baseURL).toBe('${TEST_INFERENCE_GATEWAY_URL}'); + expect(loaded.customParams?.defaultParamsEndpoint).not.toBe('openrouter'); + }, + ); + + it.each([ + { customParams: { defaultParamsEndpoint: EModelEndpoint.google } }, + { provider: EModelEndpoint.anthropic as const }, + ])('preserves explicit profile and capability decisions: %j', (overrides) => { + process.env[variable] = 'https://openrouter.ai/api/v1'; + const loaded = endpointSchema.parse(endpoint(overrides)); + const policy = getAgentInferencePolicy('Gateway', { endpoints: { custom: [loaded] } }); + expect(policy.customParams).toEqual(loadCustomEndpointsConfig([loaded])?.Gateway?.customParams); + expect(policy.customParams?.defaultParamsEndpoint).not.toBe('openrouter'); + }); + + it.each([ + 'https://openrouter.ai.untrusted.invalid/v1', + 'user_provided', + 'https://gateway.invalid/v1', + ])('does not infer a profile from untrusted or unrelated destinations: %s', (url) => { + process.env[variable] = url; + const loaded = endpointSchema.parse(endpoint()); + const appConfig = { endpoints: { custom: [loaded] } }; + expect(getAgentInferencePolicy('Gateway', appConfig).customParams).toBeUndefined(); + expect(() => + parseAgentInferenceInput({ + provider: 'Gateway', + model: 'permitted', + parameters: { promptCache: false, baseURL: 'https://openrouter.ai' }, + appConfig, + }), + ).toThrow(); + }); +}); + +describe('OpenRouter identity is independent of reasoning capability', () => { + const policies: Array & { name: string }> = [ + { name: 'reasoning object drop', dropParams: ['reasoning'] }, + { name: 'effort drop', dropParams: ['reasoning_effort'] }, + { + name: 'disabled reasoning', + customParams: { + defaultParamsEndpoint: 'custom', + reasoningFormat: ReasoningParameterFormat.disabled, + }, + }, + ]; + const cases = ['literal', 'environment'].flatMap((urlKind) => + [undefined, 'custom', 'openrouter'].flatMap((profile) => + policies.flatMap((policy) => + [false, true].map((cache) => ({ urlKind, profile, policy, cache })), + ), + ), + ); + + it.each(cases)( + '$urlKind, $profile, $policy.name, cache=$cache', + async ({ urlKind, profile, policy, cache }) => { + process.env[variable] = 'https://openrouter.ai/api/v1'; + const configured = endpoint({ + ...policy, + name: 'Gateway', + baseURL: + urlKind === 'literal' ? 'https://openrouter.ai/api/v1' : '${TEST_INFERENCE_GATEWAY_URL}', + customParams: { + ...policy.customParams, + defaultParamsEndpoint: profile ?? policy.customParams?.defaultParamsEndpoint ?? 'custom', + }, + }); + const text = yaml.dump({ version: '1.3.17', endpoints: { custom: [configured] } }); + const config = await createCustomConfigLoader({ + defaultConfigPath: 'cache-reasoning.yaml', + loadLocal: () => text, + redactConfig: (value) => value, + })(false); + const loaded = endpointSchema.parse(config?.endpoints?.custom?.[0]); + const appConfig = { endpoints: { custom: [loaded] } }; + const advertised = loadCustomEndpointsConfig([loaded])?.Gateway?.customParams; + const effective = getAgentInferencePolicy('Gateway', appConfig); + expect(effective.customParams).toEqual(advertised); + expect(effective.customParams?.defaultParamsEndpoint).toBe('openrouter'); + expect(effective.customParams?.reasoningFormat).toBe(ReasoningParameterFormat.disabled); + expect(usesOpenRouterCatalog(loaded, extractEnvVariable(loaded.baseURL))).toBe(false); + const values = { promptCache: cache, promptCacheTtl: '1h' }; + expect( + parseAgentInferenceInput({ + provider: 'Gateway', + model: 'permitted', + parameters: values, + appConfig, + }), + ).toEqual(values); + const projected = resolveAgentInferenceParameters({ + ...effective, + model: 'permitted', + parameters: { ...values, reasoning_effort: 'high' }, + }); + expect(projected.parameters).toMatchObject(values); + const result = getOpenAIConfig( + 'synthetic-key', + { + reverseProxyUrl: extractEnvVariable(loaded.baseURL), + customParams: effective.customParams, + dropParams: loaded.dropParams, + modelOptions: { ...projected.parameters, model: 'permitted' }, + }, + 'Gateway', + ); + expect(result.llmConfig.promptCache === true).toBe(cache); + const bodies: Record[] = []; + const client = initializeModel({ + provider: result.provider ?? Providers.OPENAI, + clientOptions: { + ...result.llmConfig, + streaming: false, + configuration: { + ...result.configOptions, + fetch: async (_url: unknown, init: { body?: unknown } | undefined) => { + bodies.push(JSON.parse(String(init?.body))); + return Response.json({ + id: 'chatcmpl-policy', + object: 'chat.completion', + created: 1, + model: 'permitted', + choices: [ + { + index: 0, + message: { role: 'assistant', content: 'ok' }, + finish_reason: 'stop', + }, + ], + usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 }, + }); + }, + }, + }, + }); + const messages = [new SystemMessage('Synthetic system context'), new HumanMessage('hello')]; + const prepared = + result.llmConfig.promptCache === true ? addTailCacheControl(messages, '1h') : messages; + await client.invoke(prepared); + expect(bodies).toHaveLength(1); + expect(JSON.stringify(bodies[0]).includes('cache_control')).toBe(cache); + expect(bodies[0]).not.toHaveProperty('reasoning'); + expect(bodies[0]).not.toHaveProperty('reasoning_effort'); + expect(bodies[0]).not.toHaveProperty('include_reasoning'); + }, + ); +}); + +describe('disabled reasoning owns all legacy wire aliases', () => { + it.each([false, true])( + 'suppresses a declared legacy opt-in while retaining cache=%s', + async (cache) => { + const configured = endpoint({ + baseURL: 'https://openrouter.ai/api/v1', + customParams: { + defaultParamsEndpoint: 'openrouter', + reasoningFormat: ReasoningParameterFormat.disabled, + paramDefinitions: [ + { + key: 'include_reasoning', + type: SettingTypes.Boolean, + component: ComponentTypes.Checkbox, + default: false, + }, + ], + }, + }); + const text = yaml.dump({ version: '1.3.17', endpoints: { custom: [configured] } }); + const config = await createCustomConfigLoader({ + defaultConfigPath: 'legacy-reasoning.yaml', + loadLocal: () => text, + redactConfig: (value) => value, + })(false); + const loaded = endpointSchema.parse(config?.endpoints?.custom?.[0]); + const appConfig = { endpoints: { custom: [loaded] } }; + const policy = getAgentInferencePolicy('Gateway', appConfig); + const input = parseAgentInferenceInput({ + provider: 'Gateway', + model: 'permitted', + appConfig, + parameters: { promptCache: cache, include_reasoning: true }, + }); + const projected = resolveAgentInferenceParameters({ + ...policy, + model: 'permitted', + parameters: input, + }); + expect(projected.customParameters).toEqual({ include_reasoning: true }); + const result = getOpenAIConfig( + 'synthetic-key', + { + reverseProxyUrl: loaded.baseURL, + customParams: policy.customParams, + modelOptions: { ...projected.parameters, model: 'permitted' }, + inferenceParameters: projected.customParameters, + }, + 'Gateway', + ); + const bodies: Record[] = []; + const client = initializeModel({ + provider: result.provider ?? Providers.OPENAI, + clientOptions: { + ...result.llmConfig, + streaming: false, + configuration: { + ...result.configOptions, + fetch: async (_url: unknown, init: { body?: unknown } | undefined) => { + bodies.push(JSON.parse(String(init?.body))); + return Response.json({ + id: 'chatcmpl-legacy-flag', + object: 'chat.completion', + created: 1, + model: 'permitted', + choices: [ + { + index: 0, + message: { role: 'assistant', content: 'ok' }, + finish_reason: 'stop', + }, + ], + usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 }, + }); + }, + }, + }, + }); + const messages = [new SystemMessage('Synthetic system context'), new HumanMessage('hello')]; + await client.invoke(cache ? addTailCacheControl(messages, '1h') : messages); + expect(bodies).toHaveLength(1); + expect(bodies[0]).not.toHaveProperty('include_reasoning'); + expect(bodies[0]).not.toHaveProperty('reasoning'); + expect(bodies[0]).not.toHaveProperty('reasoning_effort'); + expect(JSON.stringify(bodies[0]).includes('cache_control')).toBe(cache); + }, + ); +}); + +describe('enabled generic inference extensions', () => { + it('preserves a declared legacy opt-in when reasoning is not disabled', async () => { + const configured = endpointSchema.parse( + endpoint({ + baseURL: 'https://gateway.invalid/v1', + customParams: { + defaultParamsEndpoint: 'custom', + reasoningFormat: ReasoningParameterFormat.reasoningEffort, + paramDefinitions: [ + { + key: 'include_reasoning', + type: SettingTypes.Boolean, + component: ComponentTypes.Checkbox, + }, + ], + }, + }), + ); + const appConfig: AppConfig = { + config: {}, + fileStrategy: FileSources.local, + imageOutputType: EImageOutputType.PNG, + endpoints: { custom: [configured] }, + }; + const values = parseAgentInferenceInput({ + provider: 'Gateway', + model: 'permitted', + appConfig, + parameters: { include_reasoning: true }, + }); + const projected = resolveAgentInferenceParameters({ + ...getAgentInferencePolicy('Gateway', appConfig), + model: 'permitted', + parameters: values, + }); + const result = await initializeCustom({ + runtime: { appConfig, requestBody: {} }, + endpoint: 'Gateway', + model_parameters: { ...projected.parameters, model: 'permitted' }, + inferenceParameters: projected.customParameters, + db: { + getUserKey: jest.fn().mockResolvedValue(''), + getUserKeyValues: jest.fn().mockResolvedValue({}), + }, + }); + expect(result.llmConfig).toMatchObject({ modelKwargs: { include_reasoning: true } }); + expect(configured.customParams?.reasoningFormat).toBe(ReasoningParameterFormat.reasoningEffort); + }); +}); + +describe('production custom initialization owns the effective inference profile', () => { + const cases = [ + { + name: 'OpenRouter reasoning drop', + url: 'https://openrouter.ai/api/v1', + dropParams: ['reasoning'], + }, + { + name: 'OpenRouter effort drop', + url: 'https://openrouter.ai/api/v1', + dropParams: ['reasoning_effort'], + }, + ...['custom', 'openrouter', 'google', 'anthropic'].map((profile) => ({ + name: `explicit disabled ${profile}`, + url: profile === 'openrouter' ? 'https://openrouter.ai/api/v1' : 'https://gateway.invalid/v1', + customParams: { + defaultParamsEndpoint: profile, + reasoningFormat: ReasoningParameterFormat.disabled, + }, + })), + ]; + + it.each( + ['literal', 'environment'].flatMap((urlKind) => cases.map((policy) => ({ urlKind, policy }))), + )( + '$urlKind, $policy.name applies the same policy through initializeCustom and SDK construction', + async ({ urlKind, policy }) => { + process.env[variable] = policy.url; + const configured = endpoint({ + baseURL: urlKind === 'literal' ? policy.url : '${TEST_INFERENCE_GATEWAY_URL}', + ...('dropParams' in policy ? { dropParams: policy.dropParams } : {}), + customParams: { + ...('customParams' in policy ? policy.customParams : { defaultParamsEndpoint: 'custom' }), + paramDefinitions: [ + { + key: 'include_reasoning', + type: SettingTypes.Boolean, + component: ComponentTypes.Checkbox, + default: false, + }, + ], + }, + }); + const text = yaml.dump({ version: '1.3.17', endpoints: { custom: [configured] } }); + const config = await createCustomConfigLoader({ + defaultConfigPath: 'production-profile.yaml', + loadLocal: () => text, + redactConfig: (value) => value, + })(false); + const loaded = endpointSchema.parse(config?.endpoints?.custom?.[0]); + const appConfig: AppConfig = { + config: {}, + fileStrategy: FileSources.local, + imageOutputType: EImageOutputType.PNG, + endpoints: { custom: [loaded] }, + }; + const effective = getAgentInferencePolicy('Gateway', appConfig); + expect(effective.customParams?.reasoningFormat).toBe(ReasoningParameterFormat.disabled); + const hasCacheProfile = effective.customParams?.defaultParamsEndpoint === 'openrouter'; + const supplied = { + ...(hasCacheProfile ? { promptCache: false, promptCacheTtl: '1h' } : {}), + include_reasoning: true, + }; + const parsed = parseAgentInferenceInput({ + provider: 'Gateway', + model: 'permitted', + parameters: supplied, + appConfig, + }); + const projected = resolveAgentInferenceParameters({ + ...effective, + model: 'permitted', + parameters: parsed, + }); + const db = { + getUserKey: jest.fn().mockResolvedValue(''), + getUserKeyValues: jest.fn().mockResolvedValue({}), + }; + const result = await initializeCustom({ + runtime: { appConfig, requestBody: {} }, + endpoint: 'Gateway', + db, + model_parameters: { ...projected.parameters, model: 'permitted' }, + inferenceParameters: projected.customParameters, + }); + expect(db.getUserKeyValues).not.toHaveBeenCalled(); + if (hasCacheProfile) { + expect(result.llmConfig).not.toHaveProperty('promptCache', true); + } + const bodies: Record[] = []; + const client = initializeModel({ + provider: result.provider ?? Providers.OPENAI, + clientOptions: { + ...result.llmConfig, + streaming: false, + configuration: { + ...result.configOptions, + fetch: async (_url: unknown, init: { body?: unknown } | undefined) => { + bodies.push(JSON.parse(String(init?.body))); + return Response.json({ + id: 'chatcmpl-real-initializer', + object: 'chat.completion', + created: 1, + model: 'permitted', + choices: [ + { + index: 0, + message: { role: 'assistant', content: 'ok' }, + finish_reason: 'stop', + }, + ], + usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 }, + }); + }, + }, + }, + }); + await client.invoke('hello'); + expect(bodies).toHaveLength(1); + expect(bodies[0]).not.toHaveProperty('reasoning'); + expect(bodies[0]).not.toHaveProperty('reasoning_effort'); + expect(bodies[0]).not.toHaveProperty('include_reasoning'); + expect(JSON.stringify(bodies[0])).not.toContain('apiKey'); + expect(loaded.customParams).toEqual(config?.endpoints?.custom?.[0].customParams); + }, + ); +}); diff --git a/packages/api/src/app/loader.inference.spec.ts b/packages/api/src/app/loader.inference.spec.ts new file mode 100644 index 00000000000..4866c172bdf --- /dev/null +++ b/packages/api/src/app/loader.inference.spec.ts @@ -0,0 +1,140 @@ +import yaml from 'js-yaml'; +import { ChatOpenAI } from '@librechat/agents/llm/openai'; +import { SettingTypes, ComponentTypes, endpointSchema } from 'librechat-data-provider'; +import type { TEndpoint } from 'librechat-data-provider'; +import { createCustomConfigLoader, ConfigReloadError } from './loader'; +import { resolveAgentInferenceParameters } from '../agents/inference'; +import { getOpenAIConfig } from '../endpoints/openai/config'; +import { parseAgentInferenceInput } from '../agents/input'; + +type Definition = NonNullable['paramDefinitions']>[number]; + +const load = (definitions: Definition[], mode: 'startup' | 'reload' = 'startup') => { + const text = yaml.dump({ + version: '1.3.17', + endpoints: { + custom: [ + { + name: 'Gateway', + apiKey: 'synthetic-server-key', + baseURL: 'https://gateway.invalid/v1', + models: { default: ['permitted'], fetch: false }, + customParams: { defaultParamsEndpoint: 'custom', paramDefinitions: definitions }, + }, + ], + }, + }); + return createCustomConfigLoader({ + defaultConfigPath: 'inference.yaml', + loadLocal: () => text, + redactConfig: (config) => config, + })(false, { mode }); +}; + +const definitions: Definition[] = [ + { + key: 'min_p', + type: SettingTypes.Number, + component: ComponentTypes.Input, + range: { min: 0, max: 1 }, + default: 0.1, + }, + { key: 'seed', type: SettingTypes.Number, component: ComponentTypes.Input, default: 7 }, + { + key: 'foo_generation_mode', + type: SettingTypes.Enum, + component: ComponentTypes.Dropdown, + options: ['fast', 'accurate'], + default: 'accurate', + }, +]; + +describe('typed inference declarations in YAML', () => { + it.each(['startup', 'reload'] as const)( + 'loads safe declarations in %s mode and uses them at the SDK boundary', + async (mode) => { + const config = await load(definitions, mode); + expect(config).not.toBeNull(); + const endpoint = endpointSchema.parse(config?.endpoints?.custom?.[0]); + expect(endpoint?.customParams?.paramDefinitions).toMatchObject( + definitions.map((definition) => ({ ...definition, optionType: 'custom' })), + ); + const parameters = parseAgentInferenceInput({ + provider: 'Gateway', + model: 'permitted', + parameters: { min_p: 0.7, seed: 11, foo_generation_mode: 'fast' }, + appConfig: { endpoints: { custom: endpoint ? [endpoint] : [] } }, + }); + const projected = resolveAgentInferenceParameters({ + provider: 'Gateway', + model: 'permitted', + parameters, + customParams: endpoint?.customParams, + }); + expect(projected.issues).toEqual([]); + const { llmConfig } = getOpenAIConfig( + 'synthetic-server-key', + { + streaming: false, + modelOptions: { model: 'permitted', ...projected.parameters }, + customParams: endpoint?.customParams, + inferenceParameters: projected.customParameters, + }, + 'Gateway', + ); + const client = new ChatOpenAI(llmConfig as ConstructorParameters[0]); + expect(client.invocationParams({})).toMatchObject({ + model: 'permitted', + min_p: 0.7, + seed: 11, + foo_generation_mode: 'fast', + }); + expect(() => + parseAgentInferenceInput({ + provider: 'Gateway', + model: 'permitted', + parameters: { min_p: 2 }, + appConfig: { endpoints: { custom: endpoint ? [endpoint] : [] } }, + }), + ).toThrow(); + }, + ); + + it.each([ + 'modelKwargs', + 'configuration', + 'fallbacks', + 'apiKey', + 'baseURL', + 'model', + 'tools', + 'messages', + ])('does not admit a typed SDK control declaration: %s', async (key) => { + const definition = { key, type: SettingTypes.String, component: ComponentTypes.Input }; + await expect(load([definition])).rejects.toThrow('invalid key'); + await expect(load([definition], 'reload')).rejects.toBeInstanceOf(ConfigReloadError); + }); + + it('rejects undeclared, untyped, malformed enum and invalid-default extensions', async () => { + await expect(load([{ key: 'min_p', component: ComponentTypes.Input }])).rejects.toThrow( + 'invalid key', + ); + await expect( + load([{ key: 'extra_mode', type: SettingTypes.Enum, component: ComponentTypes.Dropdown }]), + ).rejects.toThrow('invalid key'); + await expect(load([{ ...definitions[0], default: 2 }])).rejects.toThrow('invalid key'); + await expect( + load([{ key: 'reasoning', type: SettingTypes.String, component: ComponentTypes.Input }]), + ).rejects.toThrow('invalid key'); + }); + + it('preserves existing shorthand parameter declarations', async () => { + const config = await load([{ key: 'temperature', default: 0.3 }]); + expect(config?.endpoints?.custom?.[0].customParams?.paramDefinitions?.[0]).toMatchObject({ + key: 'temperature', + type: 'number', + default: 0.3, + optionType: 'custom', + }); + }); +}); diff --git a/packages/api/src/app/loader.ts b/packages/api/src/app/loader.ts index 25cdad6cdc3..80c3510113d 100644 --- a/packages/api/src/app/loader.ts +++ b/packages/api/src/app/loader.ts @@ -14,6 +14,7 @@ import { } from 'librechat-data-provider'; import type { TCustomConfig, TEndpoint } from 'librechat-data-provider'; import type { ZodIssue } from 'zod'; +import { isAgentInferenceExtension } from '../agents/inference'; import { checkConfigTheme } from './theme'; type CustomParams = NonNullable; @@ -123,8 +124,8 @@ function parseCustomParams(endpointName: string | undefined, customParams: Custo defaultParams.map((param) => [param.key, param as CustomParamDefinition]), ); const invalidKeys = paramDefinitions - .map((param) => param.key) - .filter((key) => !defaultParamsMap.has(key)); + .filter((param) => !defaultParamsMap.has(param.key) && !isAgentInferenceExtension(param)) + .map((param) => param.key); if (invalidKeys.length > 0) { throw new Error( `paramDefinitions of "${endpointName}" endpoint contains invalid key(s). ` + diff --git a/packages/api/src/endpoints/anthropic/llm.ts b/packages/api/src/endpoints/anthropic/llm.ts index 888c8d75016..8970b68e2c0 100644 --- a/packages/api/src/endpoints/anthropic/llm.ts +++ b/packages/api/src/endpoints/anthropic/llm.ts @@ -10,6 +10,7 @@ import { bindsThinkingBlocks, isThinkingOffConfig, anthropicSettings, + tConversationSchema, removeNullishValues, ThinkingDisplay, AuthKeys, @@ -206,6 +207,7 @@ function getLLMConfig( mergedOptions.maxOutputTokens || anthropicSettings.maxOutputTokens.reset(mergedOptions.model), clientOptions: {}, invocationKwargs: { + ...options.inferenceParameters, metadata: { user_id: mergedOptions.user, }, @@ -323,11 +325,20 @@ function getLLMConfig( continue; } + if (requestOptions.invocationKwargs?.[key] !== undefined) { + continue; + } if (knownAnthropicParams.has(key)) { /** Route known Anthropic params to requestOptions only if undefined */ applyDefaultParams(requestOptions as Record, { [key]: value }); + } else if ( + options.inferenceParameters != null && + !Object.prototype.hasOwnProperty.call(tConversationSchema.shape, key) + ) { + const kwargs = requestOptions.invocationKwargs ?? {}; + applyDefaultParams(kwargs, { [key]: value }); + requestOptions.invocationKwargs = kwargs; } - /** Leave other params for transform to handle - they might be OpenAI params */ } } @@ -344,9 +355,16 @@ function getLLMConfig( if (knownAnthropicParams.has(key)) { /** Route known Anthropic params to requestOptions */ + if (requestOptions.invocationKwargs) { + delete requestOptions.invocationKwargs[key]; + } (requestOptions as Record)[key] = value; + } else if ( + options.inferenceParameters != null && + !Object.prototype.hasOwnProperty.call(tConversationSchema.shape, key) + ) { + requestOptions.invocationKwargs = { ...requestOptions.invocationKwargs, [key]: value }; } - /** Leave other params for transform to handle - they might be OpenAI params */ } } diff --git a/packages/api/src/endpoints/custom/config.spec.ts b/packages/api/src/endpoints/custom/config.spec.ts index 3759b86eae1..5a4a8e9a5dc 100644 --- a/packages/api/src/endpoints/custom/config.spec.ts +++ b/packages/api/src/endpoints/custom/config.spec.ts @@ -67,7 +67,10 @@ describe('loadCustomEndpointsConfig: host-implied reasoning support', () => { it.each([['reasoning_effort'], ['reasoning']])( 'does not advertise reasoning that dropParams %j removes', (...dropParams) => { - expect(load({ baseURL: 'https://openrouter.ai/api/v1', dropParams })).toBeUndefined(); + expect(load({ baseURL: 'https://openrouter.ai/api/v1', dropParams })).toEqual({ + defaultParamsEndpoint: 'openrouter', + reasoningFormat: ReasoningParameterFormat.disabled, + }); }, ); @@ -123,13 +126,13 @@ describe('loadCustomEndpointsConfig: host-implied reasoning support', () => { }, ); - it('leaves an endpoint that disables reasoning unmarked', () => { + it('keeps the OpenRouter cache profile when reasoning is disabled', () => { const params = load({ baseURL: 'https://openrouter.ai/api/v1', customParams: { reasoningFormat: ReasoningParameterFormat.disabled }, }); - expect(params?.defaultParamsEndpoint).toBeUndefined(); + expect(params?.defaultParamsEndpoint).toBe('openrouter'); expect(params?.reasoningFormat).toBe(ReasoningParameterFormat.disabled); }); @@ -143,10 +146,13 @@ describe('loadCustomEndpointsConfig: host-implied reasoning support', () => { expect(load({ baseURL: 'user_provided' })).toBeUndefined(); }); - it('does not mark an endpoint that drops the effort', () => { + it('keeps the cache profile while dropping the effort', () => { expect( load({ baseURL: 'https://openrouter.ai/api/v1', dropParams: ['reasoning_effort'] }), - ).toBeUndefined(); + ).toEqual({ + defaultParamsEndpoint: 'openrouter', + reasoningFormat: ReasoningParameterFormat.disabled, + }); }); it('keeps an administrator-defined reasoning_effort authoritative', () => { @@ -308,6 +314,13 @@ describe('usesOpenRouterCatalog', () => { ); }); + it.each([['reasoning'], ['reasoning_effort']])( + 'does not read the catalogue when reasoning is dropped: %j', + (...dropParams) => { + expect(uses({ dropParams })).toBe(false); + }, + ); + it('is false for a native provider endpoint', () => { expect(uses({ provider: EModelEndpoint.anthropic })).toBe(false); }); diff --git a/packages/api/src/endpoints/custom/config.ts b/packages/api/src/endpoints/custom/config.ts index 1f492cfeac8..66625b7598f 100644 --- a/packages/api/src/endpoints/custom/config.ts +++ b/packages/api/src/endpoints/custom/config.ts @@ -23,18 +23,15 @@ type CustomParams = NonNullable; * Whether an endpoint is OpenRouter's, from its resolved base URL host or from the administrator * marking its parameter set as OpenRouter's (a proxy in front of OpenRouter has another host). A * name or icon says nothing about what the server behind it accepts, so neither counts. Anything - * the administrator stated wins: a native `provider`, another params endpoint, or disabled - * reasoning. Only OpenRouter takes an effort for every model it serves: OpenAI and xAI depend on the - * model (gpt-4o takes no effort, grok-4.6 and grok-4.7 take different sets, GPT-5.6 with tools - * needs a Responses route a custom endpoint does not get), and this is decided per endpoint, before - * a model is chosen. + * the administrator stated wins: a native `provider` or another params endpoint. Disabling + * reasoning does not change the endpoint's identity or caching controls. */ export function isOpenRouterEndpoint( endpoint: Pick, resolvedBaseURL: string, ): boolean { const params = endpoint.customParams as Partial | undefined; - if (endpoint.provider != null || params?.reasoningFormat === ReasoningParameterFormat.disabled) { + if (endpoint.provider != null) { return false; } const paramsEndpoint = params?.defaultParamsEndpoint; @@ -58,43 +55,57 @@ export function usesOpenRouterCatalog( ): boolean { return ( isOpenRouterEndpoint(endpoint, resolvedBaseURL) && + endpoint.customParams?.reasoningFormat !== ReasoningParameterFormat.disabled && !hasExplicitReasoningEffort(endpoint.customParams?.paramDefinitions) && endpoint.dropParams?.some((param) => reasoningDropParams.has(param)) !== true ); } /** - * Declares reasoning support for an OpenRouter endpoint so the effort control appears without - * per-endpoint config, and marks the endpoint's parameter set as OpenRouter's so the client knows to - * read the catalog. The marking comes from the resolved host: the config loader only sees the - * unresolved name and URL. A `reasoningFormat` the administrator set is kept, and reasoning - * parameter definitions suppress only the format. An endpoint that drops the effort is left alone. + * Preserve the OpenRouter parameter profile independently of its reasoning capability. + * Drops disable reasoning advertisement, not caching; explicit formats and definitions otherwise win. */ function withOpenRouterReasoning( endpoint: Pick, resolvedBaseURL: string, ): TEndpoint['customParams'] { const customParams = endpoint.customParams; - if ( - !isOpenRouterEndpoint(endpoint, resolvedBaseURL) || - endpoint.dropParams?.some((param) => reasoningDropParams.has(param)) === true - ) { + if (!isOpenRouterEndpoint(endpoint, resolvedBaseURL)) { return customParams; } const params = (customParams ?? {}) as Partial; + const dropsReasoning = endpoint.dropParams?.some((param) => reasoningDropParams.has(param)); const declaresReasoning = params.paramDefinitions?.some((setting) => reasoningSettingKeys.some((key) => key === setting.key), ); return { ...params, defaultParamsEndpoint: ProviderId.openrouter, - ...(params.reasoningFormat == null && + ...(dropsReasoning === true && { reasoningFormat: ReasoningParameterFormat.disabled }), + ...(dropsReasoning !== true && + params.reasoningFormat == null && declaresReasoning !== true && { reasoningFormat: ReasoningParameterFormat.reasoningEffort, }), } as TEndpoint['customParams']; } +/** The trusted parameter profile shared by discovery, write validation, and execution. */ +export function resolveCustomEndpointParameters( + endpoint: Pick, + resolvedBaseURL: string, +): TEndpoint['customParams'] { + const { provider, customParams } = endpoint; + if ( + provider != null && + (customParams?.defaultParamsEndpoint == null || + customParams.defaultParamsEndpoint === EModelEndpoint.custom) + ) { + return { ...customParams, defaultParamsEndpoint: provider }; + } + return withOpenRouterReasoning(endpoint, resolvedBaseURL); +} + /** * Load config endpoints from the cached configuration object * @param customEndpointsConfig - The configuration object @@ -142,12 +153,10 @@ export function loadCustomEndpointsConfig( * (e.g. `maxOutputTokens`/`thinking` for Anthropic, not OpenAI `max_tokens`), * unless an admin explicitly chose a non-default `defaultParamsEndpoint`. */ - const resolvedCustomParams = - provider != null && - (customParams?.defaultParamsEndpoint == null || - customParams.defaultParamsEndpoint === EModelEndpoint.custom) - ? { ...customParams, defaultParamsEndpoint: provider } - : withOpenRouterReasoning({ provider, customParams, dropParams }, resolvedBaseURL); + const resolvedCustomParams = resolveCustomEndpointParameters( + { provider, customParams, dropParams }, + resolvedBaseURL, + ); customEndpointsConfig[name] = { type: EModelEndpoint.custom, diff --git a/packages/api/src/endpoints/custom/initialize.ts b/packages/api/src/endpoints/custom/initialize.ts index ae9a138a657..c17e8a56c7b 100644 --- a/packages/api/src/endpoints/custom/initialize.ts +++ b/packages/api/src/endpoints/custom/initialize.ts @@ -21,6 +21,7 @@ import { isUserProvided, checkUserKeyExpiry } from '~/utils'; import { withSupportedEffort } from '~/endpoints/reasoning'; import { getOpenAIConfig } from '~/endpoints/openai/config'; import { getScopedTokenConfigKey } from '~/endpoints/keys'; +import { resolveCustomEndpointParameters } from './config'; import { getCustomEndpointConfig } from '~/app/config'; import { resolveEndpointRuntime } from '~/types'; import { fetchModels } from '~/endpoints/models'; @@ -140,6 +141,7 @@ function buildAnthropicCustomConfig({ apiKey, baseURL, modelOptions, + inferenceParameters, endpointConfig, userProvidesURL, allowedAddresses, @@ -147,12 +149,14 @@ function buildAnthropicCustomConfig({ apiKey: string; baseURL: string; modelOptions: AnthropicModelOptions; + inferenceParameters?: Record; endpointConfig: Partial; userProvidesURL: boolean; allowedAddresses?: string[] | null; }): InitializeResultBase { const result = getAnthropicLLMConfig(apiKey, { modelOptions, + inferenceParameters, proxy: PROXY ?? undefined, reverseProxyUrl: baseURL, baseURLIsUserProvided: userProvidesURL, @@ -183,21 +187,21 @@ function buildAnthropicCustomConfig({ export async function initializeCustom( params: ProviderInitializeParams, ): Promise { - const { endpoint, model_parameters, db } = params; + const { endpoint, model_parameters, inferenceParameters, db } = params; const { appConfig, user, requestBody } = resolveEndpointRuntime(params); const { key: expiresAt } = requestBody; - const endpointConfig = getCustomEndpointConfig({ + const configuredEndpoint = getCustomEndpointConfig({ endpoint, appConfig, }); - if (!endpointConfig) { + if (!configuredEndpoint) { throw new Error(`Config not found for the ${endpoint} custom endpoint.`); } - const CUSTOM_API_KEY = extractEnvVariable(endpointConfig.apiKey ?? ''); - const CUSTOM_BASE_URL = extractEnvVariable(endpointConfig.baseURL ?? ''); + const CUSTOM_API_KEY = extractEnvVariable(configuredEndpoint.apiKey ?? ''); + const CUSTOM_BASE_URL = extractEnvVariable(configuredEndpoint.baseURL ?? ''); if (CUSTOM_API_KEY.match(envVarRegex)) { throw new Error(`Missing API Key for ${endpoint}.`); @@ -207,6 +211,10 @@ export async function initializeCustom( throw new Error(`Missing Base URL for ${endpoint}.`); } + const endpointConfig = { + ...configuredEndpoint, + customParams: resolveCustomEndpointParameters(configuredEndpoint, CUSTOM_BASE_URL), + }; const userProvidesKey = isUserProvided(CUSTOM_API_KEY); const userProvidesURL = isUserProvided(CUSTOM_BASE_URL); @@ -347,6 +355,7 @@ export async function initializeCustom( apiKey, baseURL, modelOptions: modelOptions as AnthropicModelOptions, + inferenceParameters, endpointConfig, userProvidesURL, allowedAddresses: appConfig?.endpoints?.allowedAddresses, @@ -354,8 +363,9 @@ export async function initializeCustom( options.endpointTokenConfig = endpointTokenConfig; } else { const finalClientOptions = { - modelOptions, ...clientOptions, + modelOptions, + inferenceParameters, }; options = getOpenAIConfig(apiKey, finalClientOptions, endpoint); if (options != null) { diff --git a/packages/api/src/endpoints/google/llm.spec.ts b/packages/api/src/endpoints/google/llm.spec.ts index 678a031d100..0d7d60a5cfa 100644 --- a/packages/api/src/endpoints/google/llm.spec.ts +++ b/packages/api/src/endpoints/google/llm.spec.ts @@ -575,6 +575,21 @@ describe('getGoogleConfig', () => { }); }); + /** The SDK builds the request URL from `apiVersion`; only server config may choose it. */ + it('should ignore a model option apiVersion override', () => { + const credentials = { [AuthKeys.GOOGLE_API_KEY]: 'server-api-key' }; + + const result = getGoogleConfig(credentials, { + modelOptions: { + model: 'gemini-2.5-flash', + apiVersion: 'v1alpha', + } as t.GoogleParameters, + }); + + expect(result.llmConfig).not.toHaveProperty('apiVersion'); + expect(result.llmConfig).toHaveProperty('model', 'gemini-2.5-flash'); + }); + it('should ignore non-Google Vertex AI endpoint overrides from additional params', () => { process.env.GOOGLE_LOC = 'us'; diff --git a/packages/api/src/endpoints/google/llm.ts b/packages/api/src/endpoints/google/llm.ts index e57028b1976..b5b8d470759 100644 --- a/packages/api/src/endpoints/google/llm.ts +++ b/packages/api/src/endpoints/google/llm.ts @@ -114,6 +114,7 @@ const vertexMultiRegionEndpointHosts = new Set(vertexMultiRegionEndpoints.values const blockedModelOptionParams = [ 'apiKey', + 'apiVersion', 'baseUrl', 'baseURL', 'endpoint', diff --git a/packages/api/src/endpoints/openai/config.ts b/packages/api/src/endpoints/openai/config.ts index fba96de48f2..cd014c5aba5 100644 --- a/packages/api/src/endpoints/openai/config.ts +++ b/packages/api/src/endpoints/openai/config.ts @@ -3,11 +3,11 @@ import { Agent, fetch as undiciFetch } from 'undici'; import { KnownEndpoints, EModelEndpoint, ReasoningParameterFormat } from 'librechat-data-provider'; import type { Dispatcher } from 'undici'; import type * as t from '~/types'; +import { getOpenAILLMConfig, extractDefaultParams, removeReasoningPayload } from './llm'; import { getGoogleConfig, stripGeminiFlashBlockedParams } from '~/endpoints/google/llm'; import { getLLMConfig as getAnthropicLLMConfig } from '~/endpoints/anthropic/llm'; import { createSSRFSafeAgents, createSSRFSafeUndiciConnect } from '~/auth'; import { getDirectDispatcher, getProxyDispatcher } from '~/utils/proxy'; -import { getOpenAILLMConfig, extractDefaultParams } from './llm'; import { constructAzureResponsesURL } from '~/utils/azure'; import { transformToOpenAIConfig } from './transform'; import { createFetch } from '~/utils/generators'; @@ -103,6 +103,7 @@ export function getOpenAIConfig( directEndpoint, streaming = true, modelOptions = {}, + inferenceParameters, reverseProxyUrl: baseURL, } = options; const shouldProtectUserBaseURL = options.baseURLIsUserProvided === true && !!baseURL; @@ -135,6 +136,7 @@ export function getOpenAIConfig( if (isAnthropic) { const anthropicResult = getAnthropicLLMConfig(apiKey, { modelOptions, + inferenceParameters, proxy: options.proxy, reverseProxyUrl: baseURL, addParams, @@ -184,6 +186,7 @@ export function getOpenAIConfig( ), dropParams, defaultParams, + inferenceParameters, tools: googleResult.tools, llmConfig: googleResult.llmConfig, fromEndpoint: EModelEndpoint.google, @@ -200,6 +203,7 @@ export function getOpenAIConfig( addParams, dropParams, defaultParams, + inferenceParameters, modelOptions, useOpenRouter, reasoningFormat: getReasoningFormat({ @@ -212,6 +216,13 @@ export function getOpenAIConfig( tools = openaiResult.tools; } + if (options.customParams?.reasoningFormat === ReasoningParameterFormat.disabled) { + removeReasoningPayload(llmConfig as Record); + if (llmConfig.modelKwargs) { + removeReasoningPayload(llmConfig.modelKwargs); + } + } + /** * Within-run `reasoning_content` replay applies across every param-format * branch above (OpenAI / Anthropic / Google gateway modes all resolve to the diff --git a/packages/api/src/endpoints/openai/llm.ts b/packages/api/src/endpoints/openai/llm.ts index b79e642ae46..53811082d2c 100644 --- a/packages/api/src/endpoints/openai/llm.ts +++ b/packages/api/src/endpoints/openai/llm.ts @@ -200,9 +200,10 @@ function removeReasoningField(target: Record, field: string) { target.reasoning = rest; } -function removeReasoningPayload(target: Record) { +export function removeReasoningPayload(target: Record): void { delete target.reasoning; delete target.reasoning_effort; + delete target.include_reasoning; } function deleteConfigParam({ @@ -214,6 +215,13 @@ function deleteConfigParam({ llmConfig: OpenAILLMConfig; modelKwargs: Record; }) { + if (param === 'top_p' || param === 'topP') { + delete llmConfig.topP; + delete llmConfig.top_p; + delete modelKwargs.topP; + delete modelKwargs.top_p; + return; + } if (param === 'reasoning_effort') { removeReasoningPayload(llmConfig as Record); removeReasoningPayload(modelKwargs); @@ -599,6 +607,7 @@ export function getOpenAILLMConfig({ addParams, dropParams, defaultParams, + inferenceParameters, useOpenRouter, reasoningFormat = ReasoningParameterFormat.reasoningEffort, modelOptions: _modelOptions, @@ -611,6 +620,7 @@ export function getOpenAILLMConfig({ addParams?: Record; dropParams?: string[]; defaultParams?: Record; + inferenceParameters?: Record; useOpenRouter?: boolean; reasoningFormat?: ReasoningParameterFormat; azure?: false | t.AzureOptions; @@ -634,9 +644,15 @@ export function getOpenAILLMConfig({ promptCacheTtl, frequency_penalty, presence_penalty, + top_p, + modelKwargs: _untrustedModelKwargs, ...modelOptions } = cleanedModelOptions as Partial< - t.OpenAIParameters & { promptCache?: boolean; promptCacheTtl?: '5m' | '1h' } + t.OpenAIParameters & { + promptCache?: boolean; + promptCacheTtl?: '5m' | '1h'; + modelKwargs?: unknown; + } >; const llmConfig = Object.assign( @@ -647,6 +663,9 @@ export function getOpenAILLMConfig({ modelOptions, ) as OpenAILLMConfig; + if (top_p != null) { + llmConfig.topP = top_p; + } if (frequency_penalty != null) { llmConfig.frequencyPenalty = frequency_penalty; } @@ -654,8 +673,8 @@ export function getOpenAILLMConfig({ llmConfig.presencePenalty = presence_penalty; } - const modelKwargs: Record = {}; - let hasModelKwargs = false; + const modelKwargs: Record = { ...inferenceParameters }; + let hasModelKwargs = Object.keys(modelKwargs).length > 0; let reasoningEffort = reasoning_effort; let reasoningSummary = reasoning_summary; let reasoningMode = reasoning_mode; @@ -729,6 +748,24 @@ export function getOpenAILLMConfig({ continue; } + if ( + key === 'modelKwargs' && + value != null && + typeof value === 'object' && + !Array.isArray(value) + ) { + applyDefaultParams(modelKwargs, value as Record); + hasModelKwargs = true; + continue; + } + if (key === 'seed' && inferenceParameters != null) { + applyDefaultParams(modelKwargs, { [key]: value }); + hasModelKwargs = true; + continue; + } + if (modelKwargs[key] !== undefined) { + continue; + } if (knownOpenAIParams.has(key)) { applyDefaultParams(llmConfig as Record, { [key]: value }); } else { @@ -797,7 +834,31 @@ export function getOpenAILLMConfig({ }) || hasModelKwargs; continue; } + if ( + key === 'modelKwargs' && + value != null && + typeof value === 'object' && + !Array.isArray(value) + ) { + Object.assign(modelKwargs, value); + hasModelKwargs = true; + continue; + } + if ( + inferenceParameters != null && + Object.prototype.hasOwnProperty.call(inferenceParameters, key) + ) { + modelKwargs[key] = value; + hasModelKwargs = true; + continue; + } + if (key === 'seed' && inferenceParameters != null) { + modelKwargs[key] = value; + hasModelKwargs = true; + continue; + } if (knownOpenAIParams.has(key)) { + delete modelKwargs[key]; (llmConfig as Record)[key] = value; } else { hasModelKwargs = true; @@ -806,7 +867,10 @@ export function getOpenAILLMConfig({ } } - if (useOpenRouter) { + if (reasoningFormat === ReasoningParameterFormat.disabled) { + removeReasoningPayload(llmConfig as Record); + removeReasoningPayload(modelKwargs); + } else if (useOpenRouter) { /** * OpenRouter uses a `reasoning` object — `summary` is not supported. * ChatOpenRouter treats `reasoning` and `include_reasoning` as mutually exclusive: @@ -1067,7 +1131,11 @@ export function getOpenAILLMConfig({ hasModelKwargs = true; } - if (hasModelKwargs && Object.keys(modelKwargs).length > 0) { + if ( + hasModelKwargs && + Object.keys(modelKwargs).length > 0 && + !dropParams?.includes('modelKwargs') + ) { llmConfig.modelKwargs = modelKwargs; } diff --git a/packages/api/src/endpoints/openai/transform.ts b/packages/api/src/endpoints/openai/transform.ts index 5d09dc724db..9d7b420c3d6 100644 --- a/packages/api/src/endpoints/openai/transform.ts +++ b/packages/api/src/endpoints/openai/transform.ts @@ -1,4 +1,4 @@ -import { EModelEndpoint } from 'librechat-data-provider'; +import { EModelEndpoint, tConversationSchema } from 'librechat-data-provider'; import type { GoogleAIToolType } from '@librechat/agents/langchain/google-common'; import type { ClientOptions } from '@librechat/agents'; import type * as t from '~/types'; @@ -32,6 +32,7 @@ export function transformToOpenAIConfig({ addParams, dropParams, defaultParams, + inferenceParameters, llmConfig, fromEndpoint, }: { @@ -39,6 +40,7 @@ export function transformToOpenAIConfig({ addParams?: Record; dropParams?: string[]; defaultParams?: Record; + inferenceParameters?: Record; llmConfig: ClientOptions; fromEndpoint: string; }): { @@ -91,6 +93,20 @@ export function transformToOpenAIConfig({ } } + if (inferenceParameters != null) { + for (const [key, value] of Object.entries(defaultParams ?? {})) { + if ( + !Object.prototype.hasOwnProperty.call(tConversationSchema.shape, key) && + (key === 'seed' || !Object.prototype.hasOwnProperty.call(openAIConfig, key)) && + modelKwargs[key] === undefined + ) { + modelKwargs[key] = value; + } + } + Object.assign(modelKwargs, inferenceParameters); + hasModelKwargs = Object.keys(modelKwargs).length > 0; + } + if (addParams && typeof addParams === 'object') { for (const [key, value] of Object.entries(addParams)) { /** Skip web_search / url_context - handled separately as native tools */ @@ -98,7 +114,13 @@ export function transformToOpenAIConfig({ continue; } - if (knownOpenAIParams.has(key)) { + if ( + inferenceParameters != null && + (key === 'seed' || Object.prototype.hasOwnProperty.call(inferenceParameters, key)) + ) { + modelKwargs[key] = value; + hasModelKwargs = true; + } else if (knownOpenAIParams.has(key)) { (openAIConfig as Record)[key] = value; } else { modelKwargs[key] = value; diff --git a/packages/api/src/types/anthropic.ts b/packages/api/src/types/anthropic.ts index fe9bf693b1f..dcadaba2a6a 100644 --- a/packages/api/src/types/anthropic.ts +++ b/packages/api/src/types/anthropic.ts @@ -55,6 +55,8 @@ export type AnthropicModelOptions = Partial; modelOptions?: AnthropicModelOptions; /** Proxy server URL */ proxy?: string | null; diff --git a/packages/api/src/types/endpoints.ts b/packages/api/src/types/endpoints.ts index d5a67e0b008..7566bdc41a0 100644 --- a/packages/api/src/types/endpoints.ts +++ b/packages/api/src/types/endpoints.ts @@ -55,6 +55,8 @@ interface InitializeParamsBase { endpoint: string; /** Model parameters from the request (includes model, temperature, topP, etc.) */ model_parameters?: Record; + /** Validated extra inference wire fields, separate from SDK constructor options. */ + inferenceParameters?: Record; /** Database methods for user key operations */ db: EndpointDbMethods; /** diff --git a/packages/api/src/types/openai.ts b/packages/api/src/types/openai.ts index 7c876677c19..3d96716cd0f 100644 --- a/packages/api/src/types/openai.ts +++ b/packages/api/src/types/openai.ts @@ -13,6 +13,8 @@ export type OpenAIModelOptions = Partial; * Configuration options for the getLLMConfig function */ export interface OpenAIConfigOptions { + /** Validated extra inference wire fields. */ + inferenceParameters?: Record; modelOptions?: OpenAIModelOptions; transportTimeouts?: { bodyTimeout: number; diff --git a/packages/data-schemas/src/types/app.ts b/packages/data-schemas/src/types/app.ts index fd63f71297d..2654cbca939 100644 --- a/packages/data-schemas/src/types/app.ts +++ b/packages/data-schemas/src/types/app.ts @@ -144,7 +144,7 @@ export interface AppConfig { /** Google endpoint configuration */ google?: Partial; /** Bedrock endpoint configuration */ - bedrock?: Partial; + bedrock?: Partial & { availableRegions?: string[] }; /** Anthropic endpoint configuration with optional Vertex AI support */ anthropic?: Partial & { /** Validated Vertex AI configuration */