diff --git a/apps/web/src/lib/ai-gateway/providers/provider-definitions.ts b/apps/web/src/lib/ai-gateway/providers/provider-definitions.ts index 6588d1ec04..b151f2a5ae 100644 --- a/apps/web/src/lib/ai-gateway/providers/provider-definitions.ts +++ b/apps/web/src/lib/ai-gateway/providers/provider-definitions.ts @@ -75,7 +75,7 @@ export default { apiKey: getEnvVariable('VERCEL_AI_GATEWAY_API_KEY'), supportedChatApis: ['chat_completions', 'messages', 'responses'], async transformRequest(context) { - applyVercelSettings(context.model, context.request, context.userByok); + await applyVercelSettings(context.model, context.request, context.userByok); }, }, } as const satisfies Record; diff --git a/apps/web/src/lib/ai-gateway/providers/vercel/index.test.ts b/apps/web/src/lib/ai-gateway/providers/vercel/index.test.ts index 5efa239edc..e0877f1bc0 100644 --- a/apps/web/src/lib/ai-gateway/providers/vercel/index.test.ts +++ b/apps/web/src/lib/ai-gateway/providers/vercel/index.test.ts @@ -1,6 +1,8 @@ import { describe, it, expect } from '@jest/globals'; import { + convertProviderOptions, getAnthropicProviderOptionsForVercel, + getVercelInferenceProvidersExcludingIgnored, hasCompatibleVercelInferenceProvider, passesVercelRoutingPercentage, } from '@/lib/ai-gateway/providers/vercel'; @@ -73,6 +75,59 @@ describe('hasCompatibleVercelInferenceProvider', () => { }); }); +describe('getVercelInferenceProvidersExcludingIgnored', () => { + it('returns available providers minus translated ignored providers', () => { + expect( + getVercelInferenceProvidersExcludingIgnored(['amazon-bedrock'], undefined, [ + 'anthropic', + 'bedrock', + 'vertex', + ]) + ).toEqual(['anthropic', 'vertex']); + }); + + it('intersects the available providers with only before excluding ignored providers', () => { + expect( + getVercelInferenceProvidersExcludingIgnored( + ['google-vertex'], + ['amazon-bedrock', 'google-vertex', 'openai'], + ['anthropic', 'bedrock', 'vertex'] + ) + ).toEqual(['bedrock']); + }); + + it('returns an empty list when all available providers are ignored', () => { + expect( + getVercelInferenceProvidersExcludingIgnored(['anthropic', 'amazon-bedrock'], undefined, [ + 'anthropic', + 'bedrock', + ]) + ).toEqual([]); + }); +}); + +describe('convertProviderOptions', () => { + it('emits only available non-ignored providers without changing provider.only', () => { + const request: GatewayRequest = { + kind: 'chat_completions', + body: { + model: 'anthropic/claude-sonnet-4.5', + messages: [{ role: 'user', content: 'hello' }], + provider: { + only: ['anthropic', 'amazon-bedrock'], + ignore: ['amazon-bedrock'], + }, + }, + }; + + const provider = request.body.provider; + const providerOptions = convertProviderOptions(request, ['anthropic', 'bedrock', 'vertex']); + + expect(providerOptions.gateway?.only).toEqual(['anthropic']); + expect(provider?.only).toEqual(['anthropic', 'amazon-bedrock']); + }); +}); + describe('passesVercelRoutingPercentage', () => { it('never passes at 0% and always passes at 100%', () => { for (let seed = 0; seed < 1_000; seed++) { diff --git a/apps/web/src/lib/ai-gateway/providers/vercel/index.ts b/apps/web/src/lib/ai-gateway/providers/vercel/index.ts index 9df72efcea..c9676d8038 100644 --- a/apps/web/src/lib/ai-gateway/providers/vercel/index.ts +++ b/apps/web/src/lib/ai-gateway/providers/vercel/index.ts @@ -52,6 +52,21 @@ export function hasCompatibleVercelInferenceProvider( ); } +export function getVercelInferenceProvidersExcludingIgnored( + ignoredProviders: string[], + onlyProviders: string[] | undefined, + vercelInferenceProviders: string[] +) { + const ignored = new Set(ignoredProviders.map(openRouterToVercelInferenceProviderId)); + const only = onlyProviders + ? new Set(onlyProviders.map(openRouterToVercelInferenceProviderId)) + : null; + + return vercelInferenceProviders.filter( + provider => !ignored.has(provider) && (!only || only.has(provider)) + ); +} + export function passesVercelRoutingPercentage(randomSeed: string, routingPercentage: number) { const routingSeed = 'vercel_routing_' + randomSeed; const wholePercentageBucket = getRandomNumber(routingSeed, 100); @@ -66,13 +81,6 @@ export async function shouldRouteToVercel( request: GatewayRequest, randomSeed: string ) { - if ((request.body.provider?.ignore?.length ?? 0) > 0) { - console.debug( - '[shouldRouteToVercel] not routing to Vercel because of unsupported provider options' - ); - return false; - } - if (!kiloExclusiveModel?.flags.includes('vercel-routing') && isDeepseekModel(requestedModel)) { // https://kilo-code.slack.com/archives/C0A4SA041DE/p1781743079721409 console.debug( @@ -97,11 +105,32 @@ export async function shouldRouteToVercel( return false; } - const only = request.body.provider?.only; - if (only) { + const provider = request.body.provider; + if (provider && (provider.only || provider.ignore?.length)) { + const { only, ignore } = provider; const vercelInferenceProviders = await getCachedVercelInferenceProviderIdsForModel(vercelModelId); - if (!hasCompatibleVercelInferenceProvider(only, vercelInferenceProviders)) { + + if (ignore?.length) { + if (!vercelInferenceProviders) { + console.debug( + '[shouldRouteToVercel] not routing to Vercel because inference provider data is unavailable' + ); + return false; + } + + const effectiveOnly = getVercelInferenceProvidersExcludingIgnored( + ignore, + only, + vercelInferenceProviders + ); + if (effectiveOnly.length === 0) { + console.debug( + '[shouldRouteToVercel] no inference providers remain after applying provider preferences' + ); + return false; + } + } else if (only && !hasCompatibleVercelInferenceProvider(only, vercelInferenceProviders)) { console.debug( '[shouldRouteToVercel] none of the requested inference providers are available on Vercel' ); @@ -112,11 +141,28 @@ export async function shouldRouteToVercel( return true; } -function convertProviderOptions(requestToMutate: GatewayRequest): VercelProviderConfig { +export function convertProviderOptions( + requestToMutate: GatewayRequest, + vercelInferenceProviders: string[] | null +): VercelProviderConfig { const provider = requestToMutate.body.provider; + const only = (() => { + if (!provider?.ignore?.length) { + return provider?.only?.map(openRouterToVercelInferenceProviderId); + } + if (!vercelInferenceProviders) { + throw new Error('Vercel inference provider data became unavailable during request transform'); + } + return getVercelInferenceProvidersExcludingIgnored( + provider.ignore, + provider.only, + vercelInferenceProviders + ); + })(); + return { gateway: { - only: provider?.only?.map(p => openRouterToVercelInferenceProviderId(p)), + only, order: provider?.order?.map(p => openRouterToVercelInferenceProviderId(p)), zeroDataRetention: provider?.zdr, disallowPromptTraining: provider?.data_collection === 'deny' || undefined, @@ -180,12 +226,13 @@ export function getVercelInferenceProviderConfigForUserByok( return [key, list]; } -export function applyVercelSettings( +export async function applyVercelSettings( requestedModel: string, requestToMutate: GatewayRequest, userByok: BYOKResult[] | null ) { - requestToMutate.body.model = mapModelIdToVercel(requestedModel); + const vercelModelId = mapModelIdToVercel(requestedModel); + requestToMutate.body.model = vercelModelId; if (userByok) { if (userByok.length === 0) { @@ -207,7 +254,13 @@ export function applyVercelSettings( }, }; } else { - requestToMutate.body.providerOptions = convertProviderOptions(requestToMutate); + const vercelInferenceProviders = requestToMutate.body.provider?.ignore?.length + ? await getCachedVercelInferenceProviderIdsForModel(vercelModelId) + : null; + requestToMutate.body.providerOptions = convertProviderOptions( + requestToMutate, + vercelInferenceProviders + ); } if (requestToMutate.body.providerOptions) {