Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -75,7 +75,7 @@ export default {
apiKey: getEnvVariable('VERCEL_AI_GATEWAY_API_KEY'),
supportedChatApis: ['chat_completions', 'messages', 'responses'],
async transformRequest(context) {
applyVercelSettings(context.model, context.request, context.userByok);
await applyVercelSettings(context.model, context.request, context.userByok);
},
},
} as const satisfies Record<string, Provider>;
55 changes: 55 additions & 0 deletions apps/web/src/lib/ai-gateway/providers/vercel/index.test.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
import { describe, it, expect } from '@jest/globals';
import {
convertProviderOptions,
getAnthropicProviderOptionsForVercel,
getVercelInferenceProvidersExcludingIgnored,
hasCompatibleVercelInferenceProvider,
passesVercelRoutingPercentage,
} from '@/lib/ai-gateway/providers/vercel';
Expand Down Expand Up @@ -73,6 +75,59 @@ describe('hasCompatibleVercelInferenceProvider', () => {
});
});

describe('getVercelInferenceProvidersExcludingIgnored', () => {
it('returns available providers minus translated ignored providers', () => {
expect(
getVercelInferenceProvidersExcludingIgnored(['amazon-bedrock'], undefined, [
'anthropic',
'bedrock',
'vertex',
])
).toEqual(['anthropic', 'vertex']);
});

it('intersects the available providers with only before excluding ignored providers', () => {
expect(
getVercelInferenceProvidersExcludingIgnored(
['google-vertex'],
['amazon-bedrock', 'google-vertex', 'openai'],
['anthropic', 'bedrock', 'vertex']
)
).toEqual(['bedrock']);
});

it('returns an empty list when all available providers are ignored', () => {
expect(
getVercelInferenceProvidersExcludingIgnored(['anthropic', 'amazon-bedrock'], undefined, [
'anthropic',
'bedrock',
])
).toEqual([]);
});
});

describe('convertProviderOptions', () => {
it('emits only available non-ignored providers without changing provider.only', () => {
const request: GatewayRequest = {
kind: 'chat_completions',
body: {
model: 'anthropic/claude-sonnet-4.5',
messages: [{ role: 'user', content: 'hello' }],
provider: {
only: ['anthropic', 'amazon-bedrock'],
ignore: ['amazon-bedrock'],
},
},
};

const provider = request.body.provider;
const providerOptions = convertProviderOptions(request, ['anthropic', 'bedrock', 'vertex']);

expect(providerOptions.gateway?.only).toEqual(['anthropic']);
expect(provider?.only).toEqual(['anthropic', 'amazon-bedrock']);
});
});

describe('passesVercelRoutingPercentage', () => {
it('never passes at 0% and always passes at 100%', () => {
for (let seed = 0; seed < 1_000; seed++) {
Expand Down
83 changes: 68 additions & 15 deletions apps/web/src/lib/ai-gateway/providers/vercel/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,21 @@ export function hasCompatibleVercelInferenceProvider(
);
}

export function getVercelInferenceProvidersExcludingIgnored(
ignoredProviders: string[],
onlyProviders: string[] | undefined,
vercelInferenceProviders: string[]
) {
const ignored = new Set(ignoredProviders.map(openRouterToVercelInferenceProviderId));
const only = onlyProviders
? new Set(onlyProviders.map(openRouterToVercelInferenceProviderId))
: null;

return vercelInferenceProviders.filter(
provider => !ignored.has(provider) && (!only || only.has(provider))
);
}

export function passesVercelRoutingPercentage(randomSeed: string, routingPercentage: number) {
const routingSeed = 'vercel_routing_' + randomSeed;
const wholePercentageBucket = getRandomNumber(routingSeed, 100);
Expand All @@ -66,13 +81,6 @@ export async function shouldRouteToVercel(
request: GatewayRequest,
randomSeed: string
) {
if ((request.body.provider?.ignore?.length ?? 0) > 0) {
console.debug(
'[shouldRouteToVercel] not routing to Vercel because of unsupported provider options'
);
return false;
}

if (!kiloExclusiveModel?.flags.includes('vercel-routing') && isDeepseekModel(requestedModel)) {
// https://kilo-code.slack.com/archives/C0A4SA041DE/p1781743079721409
console.debug(
Expand All @@ -97,11 +105,32 @@ export async function shouldRouteToVercel(
return false;
}

const only = request.body.provider?.only;
if (only) {
const provider = request.body.provider;
if (provider && (provider.only || provider.ignore?.length)) {
const { only, ignore } = provider;
const vercelInferenceProviders =
await getCachedVercelInferenceProviderIdsForModel(vercelModelId);
if (!hasCompatibleVercelInferenceProvider(only, vercelInferenceProviders)) {

if (ignore?.length) {
if (!vercelInferenceProviders) {
console.debug(
'[shouldRouteToVercel] not routing to Vercel because inference provider data is unavailable'
);
return false;
}

const effectiveOnly = getVercelInferenceProvidersExcludingIgnored(
ignore,
only,
vercelInferenceProviders
);
if (effectiveOnly.length === 0) {
console.debug(
'[shouldRouteToVercel] no inference providers remain after applying provider preferences'
);
return false;
}
} else if (only && !hasCompatibleVercelInferenceProvider(only, vercelInferenceProviders)) {
console.debug(
'[shouldRouteToVercel] none of the requested inference providers are available on Vercel'
);
Expand All @@ -112,11 +141,28 @@ export async function shouldRouteToVercel(
return true;
}

function convertProviderOptions(requestToMutate: GatewayRequest): VercelProviderConfig {
export function convertProviderOptions(
requestToMutate: GatewayRequest,
vercelInferenceProviders: string[] | null
): VercelProviderConfig {
const provider = requestToMutate.body.provider;
const only = (() => {
if (!provider?.ignore?.length) {
return provider?.only?.map(openRouterToVercelInferenceProviderId);
}
if (!vercelInferenceProviders) {
throw new Error('Vercel inference provider data became unavailable during request transform');
}
return getVercelInferenceProvidersExcludingIgnored(
provider.ignore,
provider.only,
vercelInferenceProviders
);
})();

return {
gateway: {
only: provider?.only?.map(p => openRouterToVercelInferenceProviderId(p)),
only,
order: provider?.order?.map(p => openRouterToVercelInferenceProviderId(p)),
zeroDataRetention: provider?.zdr,
disallowPromptTraining: provider?.data_collection === 'deny' || undefined,
Expand Down Expand Up @@ -180,12 +226,13 @@ export function getVercelInferenceProviderConfigForUserByok(
return [key, list];
}

export function applyVercelSettings(
export async function applyVercelSettings(
requestedModel: string,
requestToMutate: GatewayRequest,
userByok: BYOKResult[] | null
) {
requestToMutate.body.model = mapModelIdToVercel(requestedModel);
const vercelModelId = mapModelIdToVercel(requestedModel);
requestToMutate.body.model = vercelModelId;

if (userByok) {
if (userByok.length === 0) {
Expand All @@ -207,7 +254,13 @@ export function applyVercelSettings(
},
};
} else {
requestToMutate.body.providerOptions = convertProviderOptions(requestToMutate);
const vercelInferenceProviders = requestToMutate.body.provider?.ignore?.length
? await getCachedVercelInferenceProviderIdsForModel(vercelModelId)
: null;
requestToMutate.body.providerOptions = convertProviderOptions(
requestToMutate,
vercelInferenceProviders
);
}

if (requestToMutate.body.providerOptions) {
Expand Down