diff --git a/apps/web/src/lib/ai-gateway/providers/gateway-models-cache.ts b/apps/web/src/lib/ai-gateway/providers/gateway-models-cache.ts index 21a8a808ad..2f58abe13c 100644 --- a/apps/web/src/lib/ai-gateway/providers/gateway-models-cache.ts +++ b/apps/web/src/lib/ai-gateway/providers/gateway-models-cache.ts @@ -60,14 +60,17 @@ export function extractVercelInferenceProviderIdsFromModel(model: StoredModel): } const VercelInferenceProvidersSchema = z.array(z.string()); -const vercelInferenceProviderFetchers = new Map Promise>(); +const vercelInferenceProviderFetchers = new Map< + string, + () => Promise | null> +>(); export function getCachedVercelInferenceProviderIdsForModel( modelId: string -): Promise { +): Promise | null> { let fetchProviders = vercelInferenceProviderFetchers.get(modelId); if (!fetchProviders) { - fetchProviders = createCachedFetch( + fetchProviders = createCachedFetch | null>( async () => { const raw = await redisClient.get(vercelInferenceProvidersRedisKey(modelId)); if (raw === null) { diff --git a/apps/web/src/lib/ai-gateway/providers/get-provider.ts b/apps/web/src/lib/ai-gateway/providers/get-provider.ts index c892134d44..22acbd4dcb 100644 --- a/apps/web/src/lib/ai-gateway/providers/get-provider.ts +++ b/apps/web/src/lib/ai-gateway/providers/get-provider.ts @@ -231,7 +231,7 @@ export async function getProvider(input: GetProviderInput): Promise; diff --git a/apps/web/src/lib/ai-gateway/providers/vercel/index.ts b/apps/web/src/lib/ai-gateway/providers/vercel/index.ts index 9df72efcea..e5afc07b44 100644 --- a/apps/web/src/lib/ai-gateway/providers/vercel/index.ts +++ b/apps/web/src/lib/ai-gateway/providers/vercel/index.ts @@ -26,7 +26,6 @@ import { } from '@/lib/ai-gateway/providers/gateway-models-cache'; import type { AnthropicProviderOptions } from '@ai-sdk/anthropic'; import { isDeepseekModel } from '@/lib/ai-gateway/providers/deepseek'; -import type { KiloExclusiveModel } from '@/lib/ai-gateway/providers/kilo-exclusive-model'; const getVercelRoutingPercentage = createCachedFetch( async () => { @@ -41,7 +40,7 @@ const getVercelRoutingPercentage = createCachedFetch( export function hasCompatibleVercelInferenceProvider( openRouterInferenceProviders: string[], - vercelInferenceProviders: string[] | null + vercelInferenceProviders: ReadonlyArray | null ) { if (!vercelInferenceProviders) { return true; @@ -60,9 +59,23 @@ export function passesVercelRoutingPercentage(randomSeed: string, routingPercent return wholePercentageBucket + fractionalPercentageBucket / 1_000 < routingPercentage; } +async function getAcceptableDeepSeekProviders( + vercelDeepseekModelId: string, + only: string[] | undefined +) { + const availableProviders = + await getCachedVercelInferenceProviderIdsForModel(vercelDeepseekModelId); + const acceptableProviders = only + ? new Set(only.map(p => openRouterToVercelInferenceProviderId(p))).intersection( + new Set(availableProviders) + ) + : new Set(availableProviders); + acceptableProviders.delete(VercelUserByokInferenceProviderIdSchema.enum.novita); // https://kilo-code.slack.com/archives/C0A4SA041DE/p1781743079721409 + return acceptableProviders; +} + export async function shouldRouteToVercel( requestedModel: string, - kiloExclusiveModel: KiloExclusiveModel | null, request: GatewayRequest, randomSeed: string ) { @@ -73,14 +86,6 @@ export async function shouldRouteToVercel( return false; } - if (!kiloExclusiveModel?.flags.includes('vercel-routing') && isDeepseekModel(requestedModel)) { - // https://kilo-code.slack.com/archives/C0A4SA041DE/p1781743079721409 - console.debug( - '[shouldRouteToVercel] not routing to Vercel because some of its DeepSeek providers have tool call issues' - ); - return false; - } - console.debug('[shouldRouteToVercel] randomizing user to either OpenRouter or Vercel'); const routingPercentage = await getVercelRoutingPercentage(); @@ -98,6 +103,17 @@ export async function shouldRouteToVercel( } const only = request.body.provider?.only; + + if ( + isDeepseekModel(requestedModel) && + (await getAcceptableDeepSeekProviders(requestedModel, only)).size == 0 + ) { + console.debug( + '[shouldRouteToVercel] not routing to Vercel because some of its DeepSeek providers have tool call issues' + ); + return false; + } + if (only) { const vercelInferenceProviders = await getCachedVercelInferenceProviderIdsForModel(vercelModelId); @@ -112,11 +128,21 @@ export async function shouldRouteToVercel( return true; } -function convertProviderOptions(requestToMutate: GatewayRequest): VercelProviderConfig { +async function convertProviderOptions( + requestToMutate: GatewayRequest +): Promise { + const vercelModelId = requestToMutate.body.model; const provider = requestToMutate.body.provider; + const acceptableDeepSeekProviders = + vercelModelId && isDeepseekModel(vercelModelId) + ? await getAcceptableDeepSeekProviders(vercelModelId, provider?.only) + : undefined; return { gateway: { - only: provider?.only?.map(p => openRouterToVercelInferenceProviderId(p)), + only: + acceptableDeepSeekProviders && acceptableDeepSeekProviders.size > 0 + ? [...acceptableDeepSeekProviders] + : provider?.only?.map(p => openRouterToVercelInferenceProviderId(p)), order: provider?.order?.map(p => openRouterToVercelInferenceProviderId(p)), zeroDataRetention: provider?.zdr, disallowPromptTraining: provider?.data_collection === 'deny' || undefined, @@ -180,7 +206,7 @@ export function getVercelInferenceProviderConfigForUserByok( return [key, list]; } -export function applyVercelSettings( +export async function applyVercelSettings( requestedModel: string, requestToMutate: GatewayRequest, userByok: BYOKResult[] | null @@ -207,7 +233,7 @@ export function applyVercelSettings( }, }; } else { - requestToMutate.body.providerOptions = convertProviderOptions(requestToMutate); + requestToMutate.body.providerOptions = await convertProviderOptions(requestToMutate); } if (requestToMutate.body.providerOptions) {