diff --git a/docs/reference/PROVIDER_REFERENCE.md b/docs/reference/PROVIDER_REFERENCE.md index 67046599ada..14396f99c5f 100644 --- a/docs/reference/PROVIDER_REFERENCE.md +++ b/docs/reference/PROVIDER_REFERENCE.md @@ -126,7 +126,7 @@ Use the dashboard at `/dashboard/providers` to enable, configure, and test each | `agentrouter` | `agentrouter` | AgentRouter | API key, aggregator | [link](https://agentrouter.org) | $200 free credits on signup - multi-model routing gateway | | `agnes` | `agnes` | Agnes AI | API key, video | [link](https://agnes-ai.com) | Get API key at agnes-ai.com | | `ai21` | `ai21` | AI21 Labs | API key | [link](https://www.ai21.com) | $10 trial credits on signup (valid 3 months), no credit card required | -| `aimlapi` | `aiml` | AI/ML API | API key, aggregator | [link](https://aimlapi.com) | Free tier paused (2026) — AI/ML API is now pay-as-you-go only (min $20 top-up); no recurring free credits. | +| `aimlapi` | `aiml` | aimlapi.com | API key, aggregator | [link](https://aimlapi.com) | Free tier paused (2026) — AI/ML API is now pay-as-you-go only (min $20 top-up); no recurring free credits. | | `ainative` | `ainative` | AINative Studio | API key | [link](https://ainative.studio) | Create a free API key at ainative.studio (no card), then paste it here as a Bearer token. | | `aion` | `aion` | Aion Labs | API key | [link](https://www.aionlabs.ai) | Create a free API key at aionlabs.ai (no card), then paste it here as a Bearer token. | | `alibaba` | `ali` | Alibaba Cloud Model Studio | API key | [link](https://bailian.console.alibabacloud.com/) | — | diff --git a/open-sse/config/providers/registry/aimlapi/index.ts b/open-sse/config/providers/registry/aimlapi/index.ts index e84d248a677..575416ef077 100644 --- a/open-sse/config/providers/registry/aimlapi/index.ts +++ b/open-sse/config/providers/registry/aimlapi/index.ts @@ -8,14 +8,48 @@ export const aimlapiProvider: RegistryEntry = { baseUrl: "https://api.aimlapi.com/v1/chat/completions", authType: "apikey", authHeader: "bearer", - // $0.025/day free credits — 200+ models via single aggregator endpoint + // Attribution, same mechanism the openrouter/orcarouter/cline entries use: + // BaseExecutor.buildHeadersPreamble() spreads config.headers into a fresh + // object per request, so these are scoped to this provider's own dispatch and + // cannot ride a request to another upstream, and the constant below is never + // mutated. HTTP-Referer/X-Title follow the OpenRouter convention and identify + // the CALLING app (OmniRoute), not the upstream gateway. + headers: { + "HTTP-Referer": "https://github.com/diegosouzapw/OmniRoute", + "X-Title": "OmniRoute", + "X-AIMLAPI-Source": "agent/omniroute", + "X-AIMLAPI-Partner-ID": "part_T2iNtMuQ3JBmEPwyOKCLOxaP", + }, + // Static fallback ONLY — the live catalog (353 chat models) is discovered via + // PROVIDER_MODELS_CONFIG.aimlapi and supersedes this list whenever the fetch + // succeeds, so these entries are what a user sees when discovery is down. + // + // Each id was verified twice on 2026-09-03: present on the catalog's + // `openai/chat-completions` surface (as an id or an alias) AND answering 200 to + // a real POST /v1/chat/completions. Catalog membership alone is not enough — + // `llama-3.3-70b-versatile` is listed as a chat model, advertises tools and + // structured output, and still 404s "model does not exist" on inference. + // + // All six are the UNPREFIXED spelling, deliberately. A `vendor/model` id in + // this registry is matched as an exact model id by parseModel(), which then + // reports provider = null: seeding "anthropic/claude-sonnet-4-6" here makes + // that string stop resolving to the anthropic provider everywhere in the app, + // collapsing its context window from 1M to the 128k default and dropping the + // provider prefix that #8716 exists to preserve. The previous seed was right + // about this, and the bare aliases carry no such prefix. + // + // Anthropic ids use the DOTTED spelling: the catalog carries both + // `claude-sonnet-4-6` and `claude-sonnet-4.6` as separate entries, and the + // dashed one advertises only `streaming` in `capabilities` while the dotted one + // advertises tools, vision, reasoning and structured output. Same for + // claude-opus-4.7 / 4.8. models: [ - { id: "gpt-4o", name: "GPT-4o (via AI/ML API)" }, - { id: "claude-3-5-sonnet-20241022", name: "Claude 3.5 Sonnet (via AI/ML API)" }, - { id: "gemini-1.5-pro", name: "Gemini 1.5 Pro (via AI/ML API)" }, - { id: "meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo", name: "Llama 3.1 70B (via AI/ML API)" }, - { id: "deepseek-chat", name: "DeepSeek Chat (via AI/ML API)" }, - { id: "mistral-large-latest", name: "Mistral Large (via AI/ML API)" }, + { id: "gpt-5", name: "GPT-5 (via aimlapi.com)" }, + { id: "claude-sonnet-4.6", name: "Claude 4.6 Sonnet (via aimlapi.com)" }, + { id: "gemini-2.5-pro", name: "Gemini 2.5 Pro (via aimlapi.com)" }, + { id: "glm-5", name: "GLM-5 (via aimlapi.com)" }, + { id: "deepseek-chat", name: "DeepSeek V3 (via aimlapi.com)" }, + { id: "mistral-large", name: "Mistral Large (via aimlapi.com)" }, ], passthroughModels: true, }; diff --git a/open-sse/translator/paramSupport.ts b/open-sse/translator/paramSupport.ts index 85dcec35ae4..31cab98d19d 100644 --- a/open-sse/translator/paramSupport.ts +++ b/open-sse/translator/paramSupport.ts @@ -6,6 +6,10 @@ // - `provider` (optional) limits the rule to a single provider id. // - `match` is a RegExp tested against the model id OR a predicate (model -> boolean). // - `drop` is the list of param keys to remove when the rule fires. +// - `dropIfNull` removes a key only when its value is literally `null`, leaving +// real values untouched. For upstreams that reject `null` on an optional +// param instead of treating it as "unset" — the OpenAI SDKs serialise an +// unset optional as `null`, so such a param arrives on ordinary requests. // - `clampToModelMaxOutput` clamps max_tokens/max_completion_tokens/max_output_tokens // down to the model's catalog `maxOutputTokens` ceiling, when one is set. // - `maxOutputCap` clamps the same keys down to a fixed endpoint-imposed ceiling @@ -23,6 +27,7 @@ type StripRule = { provider?: string; match: RegExp | ((model: string) => boolean); drop?: string[]; + dropIfNull?: string[]; clampToModelMaxOutput?: boolean; maxOutputCap?: number; }; @@ -93,6 +98,49 @@ const STRIP_RULES: StripRule[] = [ // to read), hence the fixed cap. { provider: "azure-openai", match: /^gpt-4o-mini/i, maxOutputCap: 16384 }, { provider: "azure-ai", match: /^gpt-4o-mini/i, maxOutputCap: 16384 }, + // AI/ML API validates optional fields with a strict schema that rejects a + // literal `null` ("Expected number, received null") instead of reading null as + // "unset". The OpenAI SDKs serialise an unset optional as `null`, so an SDK + // client that never touches temperature still puts `temperature: null` on the + // wire — and OmniRoute relays the caller's body verbatim, so the request 400s. + // + // Field-by-field sweep against POST /v1/chat/completions, 2026-09-03: + // 400 on null, every model: seed, tools, tool_choice, response_format, + // stream, stream_options, parallel_tool_calls, + // max_tokens, max_completion_tokens, + // reasoning_effort + // 400 on null, model-dependent: temperature, top_p (400 on + // claude-sonnet-4.6 and deepseek-chat, 200 on + // gpt-5 — so an integration smoke-tested only + // against gpt-5 looks healthy) + // 200 on null (left alone): stop, presence_penalty, frequency_penalty, n, + // user, logprobs, logit_bias, top_logprobs, + // metadata + // + // Provider-wide, and only for the null value: an explicit `temperature: 0` or + // `stream: false` is forwarded untouched. Dropping is the correct reading — + // every field here means "unset" when the caller sends null. + // + // The 400 body's top-level `message` is generic; `details[].path` / `.reason` + // name the offending field, which is what this list was built from. + { + provider: "aimlapi", + match: /.*/, + dropIfNull: [ + "temperature", + "top_p", + "seed", + "tools", + "tool_choice", + "response_format", + "stream", + "stream_options", + "parallel_tool_calls", + "max_tokens", + "max_completion_tokens", + "reasoning_effort", + ], + }, ]; function matches(rule: StripRule, model: string): boolean { @@ -156,6 +204,9 @@ export function stripUnsupportedParams( for (const key of rule.drop ?? []) { if (rec[key] !== undefined) delete rec[key]; } + for (const key of rule.dropIfNull ?? []) { + if (rec[key] === null) delete rec[key]; + } applyMaxOutputClamp(rule, provider, model, rec); } diff --git a/src/app/(dashboard)/dashboard/providers/featuredProviders.ts b/src/app/(dashboard)/dashboard/providers/featuredProviders.ts index 08905b5ee31..4ba5d34557e 100644 --- a/src/app/(dashboard)/dashboard/providers/featuredProviders.ts +++ b/src/app/(dashboard)/dashboard/providers/featuredProviders.ts @@ -52,6 +52,18 @@ const KIMI_PROVIDER_IDS: readonly string[] = [ /** Cheaper Inference (api.cheaperinference.com) — apikey category, single id. */ const CHEAPERINFERENCE_PROVIDER_IDS: readonly string[] = ["cheaperinference"]; +/** + * aimlapi.com (api.aimlapi.com) — apikey category, single id. + * + * FORK-ONLY. This pin is not part of the upstream provider set and must not be + * carried into an upstream pull request; it is isolated in its own commit so it + * can be dropped wholesale. It is deliberately ranked BELOW the two sponsors the + * operator ranked explicitly on 2026-07-31 — those ranks encode a real + * commitment and are asserted in `featured-providers-rank.test.ts`, so they are + * not ours to reorder. + */ +const AIMLAPI_PROVIDER_IDS: readonly string[] = ["aimlapi"]; + /** * Explicit sponsor ordering for the dashboard provider grids. * @@ -67,6 +79,7 @@ const CHEAPERINFERENCE_PROVIDER_IDS: readonly string[] = ["cheaperinference"]; const FEATURED_PROVIDER_RANKS: ReadonlyMap = new Map([ ...KIMI_PROVIDER_IDS.map((id) => [id, 1] as const), ...CHEAPERINFERENCE_PROVIDER_IDS.map((id) => [id, 2] as const), + ...AIMLAPI_PROVIDER_IDS.map((id) => [id, 3] as const), ]); /** Brand accent per sponsor family, keyed by any of that family's provider ids. */ diff --git a/src/app/api/providers/[id]/models/discovery/providerModelsConfig.ts b/src/app/api/providers/[id]/models/discovery/providerModelsConfig.ts index cc45e6b7277..7fd3705fd42 100644 --- a/src/app/api/providers/[id]/models/discovery/providerModelsConfig.ts +++ b/src/app/api/providers/[id]/models/discovery/providerModelsConfig.ts @@ -27,6 +27,18 @@ import { isDashscopeTextModelId } from "@omniroute/open-sse/services/dashscopeTe import { extractZaiToken } from "@omniroute/open-sse/services/zaiWebCredentials.ts"; import { normalizeOpenAiLikeModelsResponse } from "./normalizers"; +/** + * `type` values AI/ML API uses for models routable through /v1/chat/completions. + * + * `openai/chat-completions` is the current vocabulary (353 of 936 catalog rows on + * 2026-09-03); `chat-completion` is the pre-rename spelling, kept so a future + * rename in either direction cannot silently empty the provider's model list + * again. Everything else the catalog publishes — video, image, TTS, STT, + * `openai/responses/submit`, `anthropic/messages`, batches — is a different + * upstream surface and must not reach a chat model picker. + */ +export const AIMLAPI_CHAT_MODEL_TYPES = new Set(["openai/chat-completions", "chat-completion"]); + const QWEN_CLOUD_TEXT_MODEL_IDS = new Set(QWEN_CLOUD_TEXT_MODELS.map((model) => model.id)); const ALIBABA_MODEL_STUDIO_MODEL_IDS = new Set( ALIBABA_MODEL_STUDIO_MODELS.map((model) => model.id) @@ -472,17 +484,32 @@ export const PROVIDER_MODELS_CONFIG: Record = parseResponse: (data) => data.data || [], }, aimlapi: { - // #5570: AI/ML API's live catalog (400+ models) lives at the public, - // auth-free /models database endpoint (NOT /v1/models). The registry has no - // modelsUrl, so without this entry the route fell back to a stale 6-model - // seed. Response is a bare array of { id, type, info: { name } }. + // #5570: AI/ML API's live catalog lives at the public, auth-free /models + // database endpoint (NOT /v1/models). The registry has no modelsUrl, so + // without this entry the route falls back to the static seed. + // + // The two premises in the original #5570 comment went stale and silently + // zeroed this provider out (936 catalog rows -> 0 discovered models -> the + // 6-entry static seed): + // 1. the response is NOT a bare array — it is the OpenAI-style envelope + // `{ "object": "list", "data": [...] }`, so `Array.isArray(data)` was + // false and every row was discarded before the filter even ran; + // 2. the chat `type` vocabulary is now `openai/chat-completions`; the old + // `chat-completion` spelling matches 0 of 936 rows (verified against the + // live catalog 2026-09-03). + // Both spellings are accepted below so a future vocabulary change degrades to + // "older models missing" rather than "provider has no models", and the bare + // array is still unwrapped for the same reason. Non-chat rows (video, image, + // TTS, batches) are dropped outright — the previous `chat.length ? chat : all` + // fallback would have surfaced 583 non-chat ids in a chat model picker, which + // is what hid defect 2 from the tests. url: "https://api.aimlapi.com/models", method: "GET", headers: { "Content-Type": "application/json" }, parseResponse: (data) => { - const all = Array.isArray(data) ? data : []; - const chat = all.filter((m) => m?.type === "chat-completion"); - return (chat.length > 0 ? chat : all) + const rows = Array.isArray(data) ? data : Array.isArray(data?.data) ? data.data : []; + return rows + .filter((m) => AIMLAPI_CHAT_MODEL_TYPES.has(m?.type)) .map((m) => ({ id: m?.id, name: m?.info?.name || m?.id })) .filter((m) => typeof m.id === "string" && m.id); }, diff --git a/src/shared/constants/providers/apikey/gateways.ts b/src/shared/constants/providers/apikey/gateways.ts index 1a37aab7f32..214b6b3c474 100644 --- a/src/shared/constants/providers/apikey/gateways.ts +++ b/src/shared/constants/providers/apikey/gateways.ts @@ -15,7 +15,8 @@ export const APIKEY_PROVIDERS_GATEWAYS = { color: "#6366F1", textIcon: "1M", website: "https://1min.ai", - authHint: "Create an API key at https://docs.1min.ai/docs/api/create-api-key, then paste it here.", + authHint: + "Create an API key at https://docs.1min.ai/docs/api/create-api-key, then paste it here.", apiHint: "1min.ai uses a proprietary chat API (single prompt string + SSE) instead of OpenAI chat/completions. OmniRoute flattens OpenAI messages into a labeled prompt and translates the SSE stream.", passthroughModels: true, @@ -47,7 +48,8 @@ export const APIKEY_PROVIDERS_GATEWAYS = { website: "https://freebuff.com", hasFree: true, serviceKinds: ["llm"], - authHint: "Enter Freebuff / Codebuff Auth Token (obtained via CLI login or automated harvester).", + authHint: + "Enter Freebuff / Codebuff Auth Token (obtained via CLI login or automated harvester).", freeNote: "Free Codebuff / Freebuff AI models.", apiHint: "Token is authenticated against Codebuff upstream session pool.", passthroughModels: true, @@ -907,7 +909,7 @@ export const APIKEY_PROVIDERS_GATEWAYS = { id: "aimlapi", serviceKinds: ["llm"], alias: "aiml", - name: "AI/ML API", + name: "aimlapi.com", icon: "hub", color: "#6366F1", textIcon: "AI", diff --git a/tests/snapshots/provider/translate-path.json b/tests/snapshots/provider/translate-path.json index 43150e2b22d..e0752b0d617 100644 --- a/tests/snapshots/provider/translate-path.json +++ b/tests/snapshots/provider/translate-path.json @@ -189,16 +189,28 @@ "apiKey": { "Accept": "text/event-stream", "Authorization": "Bearer ", - "Content-Type": "application/json" + "Content-Type": "application/json", + "HTTP-Referer": "https://github.com/diegosouzapw/OmniRoute", + "X-AIMLAPI-Partner-ID": "part_T2iNtMuQ3JBmEPwyOKCLOxaP", + "X-AIMLAPI-Source": "agent/omniroute", + "X-Title": "OmniRoute" }, "nonStream": { "Authorization": "Bearer ", - "Content-Type": "application/json" + "Content-Type": "application/json", + "HTTP-Referer": "https://github.com/diegosouzapw/OmniRoute", + "X-AIMLAPI-Partner-ID": "part_T2iNtMuQ3JBmEPwyOKCLOxaP", + "X-AIMLAPI-Source": "agent/omniroute", + "X-Title": "OmniRoute" }, "oauth": { "Accept": "text/event-stream", "Authorization": "Bearer ", - "Content-Type": "application/json" + "Content-Type": "application/json", + "HTTP-Referer": "https://github.com/diegosouzapw/OmniRoute", + "X-AIMLAPI-Partner-ID": "part_T2iNtMuQ3JBmEPwyOKCLOxaP", + "X-AIMLAPI-Source": "agent/omniroute", + "X-Title": "OmniRoute" } }, "url": { diff --git a/tests/unit/aimlapi-attribution-headers.test.ts b/tests/unit/aimlapi-attribution-headers.test.ts new file mode 100644 index 00000000000..588234c67fe --- /dev/null +++ b/tests/unit/aimlapi-attribution-headers.test.ts @@ -0,0 +1,88 @@ +// AI/ML API partner attribution. +// +// A malformed partner id is NOT rejected upstream — the request succeeds and the +// usage is simply recorded as untagged. There is no runtime signal for a typo, so +// the id's shape has to be asserted here or nothing catches it. +import test from "node:test"; +import assert from "node:assert/strict"; + +import { REGISTRY } from "@omniroute/open-sse/config/providers/index.ts"; +import { aimlapiProvider } from "@omniroute/open-sse/config/providers/registry/aimlapi/index.ts"; +import { generateLegacyProviders } from "@omniroute/open-sse/config/providerRegistry.ts"; +import { getDefaultExecutor } from "@omniroute/open-sse/executors/defaultResolver.ts"; +import { APIKEY_PROVIDERS } from "@/shared/constants/providers/apikey/index"; + +/** Gateway contract: /^part_[A-Za-z0-9]{1,64}$/ — no dashes, no underscores. */ +const PARTNER_ID_PATTERN = /^part_[A-Za-z0-9]{1,64}$/; +/** `/`, channel a closed enum, client lowercase alnum + dashes. */ +const SOURCE_PATTERN = /^(web|agent|mcp)\/[a-z0-9-]{1,32}$/; + +test("aimlapi declares all four attribution headers", () => { + const headers = aimlapiProvider.headers ?? {}; + assert.equal(headers["X-AIMLAPI-Partner-ID"], "part_T2iNtMuQ3JBmEPwyOKCLOxaP"); + assert.equal(headers["X-AIMLAPI-Source"], "agent/omniroute"); + assert.equal(headers["HTTP-Referer"], "https://github.com/diegosouzapw/OmniRoute"); + assert.equal(headers["X-Title"], "OmniRoute"); +}); + +test("the partner id matches the gateway's pattern", () => { + // A malformed id is accepted by the API and silently earns nothing, so this + // assertion is the only place a typo surfaces. + const partnerId = aimlapiProvider.headers?.["X-AIMLAPI-Partner-ID"] ?? ""; + assert.match(partnerId, PARTNER_ID_PATTERN); + assert.match(aimlapiProvider.headers?.["X-AIMLAPI-Source"] ?? "", SOURCE_PATTERN); +}); + +test("HTTP-Referer and X-Title identify OmniRoute, not the upstream gateway", () => { + // These are the OpenRouter-convention analytics headers: they name the calling + // application. Pointing them at aimlapi.com would attribute OmniRoute's traffic + // to the provider it is calling. + const headers = aimlapiProvider.headers ?? {}; + assert.ok(!headers["HTTP-Referer"].includes("aimlapi.com")); + assert.ok(!headers["X-Title"].toLowerCase().includes("aimlapi")); +}); + +test("attribution is scoped to aimlapi and cannot ride a request to another provider", () => { + const carriers = Object.entries(REGISTRY) + .filter(([, entry]) => + Object.keys({ ...(entry.headers ?? {}), ...(entry.extraHeaders ?? {}) }).some((key) => + key.toLowerCase().startsWith("x-aimlapi-") + ) + ) + .map(([id]) => id); + assert.deepEqual(carriers, ["aimlapi"]); +}); + +test("the executor emits the headers without mutating the shared registry constant", () => { + const before = JSON.stringify(aimlapiProvider.headers); + const executor = getDefaultExecutor("aimlapi"); + + const first = executor.buildHeaders({ apiKey: "test-key" } as never, false); + const second = executor.buildHeaders({ apiKey: "test-key" } as never, false); + + assert.equal(first["X-AIMLAPI-Partner-ID"], "part_T2iNtMuQ3JBmEPwyOKCLOxaP"); + assert.equal(second["X-AIMLAPI-Partner-ID"], "part_T2iNtMuQ3JBmEPwyOKCLOxaP"); + // A fresh object per request — mutating one built header map must not leak into + // the next request or back into the registry entry. + assert.notEqual(first, second); + first["X-AIMLAPI-Partner-ID"] = "part_tampered"; + assert.equal( + executor.buildHeaders({ apiKey: "test-key" } as never, false)["X-AIMLAPI-Partner-ID"], + "part_T2iNtMuQ3JBmEPwyOKCLOxaP" + ); + assert.equal(JSON.stringify(aimlapiProvider.headers), before); +}); + +test("the generated legacy provider map carries the headers through to dispatch", () => { + const legacy = generateLegacyProviders().aimlapi; + assert.equal(legacy.headers?.["X-AIMLAPI-Partner-ID"], "part_T2iNtMuQ3JBmEPwyOKCLOxaP"); + assert.equal(legacy.headers?.["X-AIMLAPI-Source"], "agent/omniroute"); +}); + +test("the user-facing provider label is exactly aimlapi.com", () => { + // The machine id (`aimlapi`) and alias (`aiml`) are what users' configs and the + // DB reference, so they must not change; only the display string does. + assert.equal(APIKEY_PROVIDERS.aimlapi.name, "aimlapi.com"); + assert.equal(APIKEY_PROVIDERS.aimlapi.id, "aimlapi"); + assert.equal(APIKEY_PROVIDERS.aimlapi.alias, "aiml"); +}); diff --git a/tests/unit/aimlapi-catalog-repair.test.ts b/tests/unit/aimlapi-catalog-repair.test.ts new file mode 100644 index 00000000000..ab13b9f177c --- /dev/null +++ b/tests/unit/aimlapi-catalog-repair.test.ts @@ -0,0 +1,97 @@ +// AI/ML API (aimlapi) — regression guard for the 2026-09 discovery repair. +// +// The provider shipped fully wired and still showed users a 6-entry model list, +// four of whose ids no longer existed upstream. Three independent defects: +// +// 1. discovery ran `Array.isArray(data)` against `{ object: "list", data: [...] }` +// → every catalog row discarded → the route fell back to the static seed; +// 2. the chat filter used `chat-completion`, a spelling the catalog stopped +// publishing (`openai/chat-completions` matches 353 of 936 rows); +// 3. four seed ids (claude-3-5-sonnet-20241022, gemini-1.5-pro, +// meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo, mistral-large-latest) are +// absent from the catalog as both id AND alias. +// +// Defects 1 and 2 are covered by provider-models-discovery-split.test.ts. This +// file guards the seed list and pins the shape of the ids so a future refresh +// cannot reintroduce a dead spelling. +import test from "node:test"; +import assert from "node:assert/strict"; + +import { aimlapiProvider } from "@omniroute/open-sse/config/providers/registry/aimlapi/index.ts"; + +/** + * Ids that look plausible and are NOT in the AI/ML API catalog (as id or alias), + * re-verified 2026-09-03. Four of these shipped in this registry entry; the rest + * are the spellings other aggregators publish, which is how they get copied in. + */ +const KNOWN_DEAD_IDS = [ + "claude-3-5-sonnet-20241022", + "gemini-1.5-pro", + "meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo", + "mistral-large-latest", + "gpt-5.5", + "anthropic/claude-sonnet-4-5", + "z-ai/glm-5.2", + "moonshotai/kimi-k2.6", + "meta-llama/llama-4-maverick", + // In the catalog as a chat model, advertises tools + structured output, and + // still 404s "model does not exist" on POST /v1/chat/completions (2026-09-03). + "llama-3.3-70b-versatile", + // Dashed Anthropic spellings exist but advertise only `streaming`; the dotted + // ids are the ones carrying the real capability set. + "claude-sonnet-4-6", +]; + +/** + * Ids verified twice on 2026-09-03: present on the catalog's + * `openai/chat-completions` surface (each is an alias of a canonical + * `vendor/model` id — checking ids alone would wrongly call all six dead) AND + * answering 200 to a real POST /v1/chat/completions. Both checks are needed: + * `llama-3.3-70b-versatile` passes the first and fails the second. + */ +const VERIFIED_SEED_IDS = [ + "gpt-5", + "claude-sonnet-4.6", + "gemini-2.5-pro", + "glm-5", + "deepseek-chat", + "mistral-large", +]; + +test("aimlapi seed catalog carries only catalog-verified ids", () => { + assert.deepEqual( + aimlapiProvider.models.map((m) => m.id), + VERIFIED_SEED_IDS + ); +}); + +test("aimlapi seed catalog contains no known-dead model id", () => { + const seeded = new Set(aimlapiProvider.models.map((m) => m.id)); + for (const dead of KNOWN_DEAD_IDS) { + assert.ok(!seeded.has(dead), `dead model id "${dead}" must not be seeded`); + } +}); + +test("aimlapi seed ids carry no vendor prefix", () => { + // Not cosmetic. A "vendor/model" id in this registry is matched by parseModel() + // as an exact model id, which makes it report provider = null — so seeding + // "anthropic/claude-sonnet-4-6" here stops that string resolving to the + // anthropic provider app-wide (context window 1M -> the 128k default, and combo + // targets lose the provider prefix that #8716 guards). Bare aliases are inert. + for (const model of aimlapiProvider.models) { + assert.ok( + !model.id.includes("/"), + `"${model.id}" must not carry a vendor prefix — it would shadow that provider's own model resolution` + ); + assert.ok(model.name.length > 0, `"${model.id}" must have a display name`); + } +}); + +test("aimlapi still points at the OpenAI-compatible chat endpoint", () => { + // /v1/completions does not exist on this API (404); only /v1/chat/completions + // and /v1/responses do. + assert.equal(aimlapiProvider.baseUrl, "https://api.aimlapi.com/v1/chat/completions"); + assert.equal(aimlapiProvider.format, "openai"); + assert.equal(aimlapiProvider.authHeader, "bearer"); + assert.equal(aimlapiProvider.passthroughModels, true); +}); diff --git a/tests/unit/aimlapi-fork-placement.test.ts b/tests/unit/aimlapi-fork-placement.test.ts new file mode 100644 index 00000000000..c7bbbdd4af4 --- /dev/null +++ b/tests/unit/aimlapi-fork-placement.test.ts @@ -0,0 +1,51 @@ +// FORK-ONLY placement guard — see the commit that introduced this file. +// +// This test exists only to pin the fork's dashboard placement for aimlapi.com. It +// is NOT an upstream concern and is deliberately confined to the same commit as +// the pin itself, so dropping that commit drops this file with it. +import test from "node:test"; +import assert from "node:assert/strict"; + +import { + getFeaturedProviderRank, + isFeaturedProviderId, +} from "@/app/(dashboard)/dashboard/providers/featuredProviders"; +import { sortProviderEntriesFeaturedFirst } from "@/app/(dashboard)/dashboard/providers/providerPageUtils"; + +const entry = (providerId: string, name: string) => + ({ + providerId, + provider: { id: providerId, name }, + stats: { total: 0 }, + displayAuthType: "apikey" as const, + toggleAuthType: "apikey" as const, + }) as never; + +test("aimlapi.com is pinned in the dashboard grids", () => { + assert.equal(isFeaturedProviderId("aimlapi"), true); + assert.equal(getFeaturedProviderRank("aimlapi"), 3); +}); + +test("the pin does not displace the operator's two ranked sponsors", () => { + // Kimi (1) and Cheaper Inference (2) encode an explicit operator decision dated + // 2026-07-31. The fork pin sits under them, never over them. + assert.equal(getFeaturedProviderRank("moonshot"), 1); + assert.equal(getFeaturedProviderRank("cheaperinference"), 2); +}); + +test("aimlapi.com sorts above unranked aggregators despite the alphabet", () => { + // The aggregator grid is sorted by display name, so "aimlapi.com" would + // otherwise fall between "AgentRouter" and "AnyAPI AI". The rank is the only + // sanctioned lever for placement here — the catalog's key order does not reach + // the rendered grid at all. + const sorted = sortProviderEntriesFeaturedFirst([ + entry("openrouter", "OpenRouter"), + entry("agentrouter", "AgentRouter"), + entry("aimlapi", "aimlapi.com"), + entry("cheaperinference", "Cheaper Inference"), + ]); + assert.deepEqual( + sorted.map((e) => e.providerId), + ["cheaperinference", "aimlapi", "agentrouter", "openrouter"] + ); +}); diff --git a/tests/unit/aimlapi-null-param-strip.test.ts b/tests/unit/aimlapi-null-param-strip.test.ts new file mode 100644 index 00000000000..7ec1945b2b9 --- /dev/null +++ b/tests/unit/aimlapi-null-param-strip.test.ts @@ -0,0 +1,116 @@ +// AI/ML API validates optional fields with a strict schema that rejects a literal +// `null` ("Expected number, received null") rather than reading it as "unset". +// +// This is not an exotic payload: the OpenAI SDKs serialise an unset optional as +// `null`, so a client that never touches temperature still puts `temperature: +// null` on the wire, and OmniRoute relays the body verbatim. +// +// Field sweep against POST /v1/chat/completions on 2026-09-03 — the lists below +// are transcribed from `details[].path` in the 400 bodies (the top-level +// `message` is generic and names no field). +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { stripUnsupportedParams } from "../../open-sse/translator/paramSupport.ts"; + +/** 400 on null. temperature/top_p 400 on claude-sonnet-4.6 and deepseek-chat but + * 200 on gpt-5, so the rejection is model-dependent; the rest 400 everywhere. */ +const REJECTED_WHEN_NULL = [ + "temperature", + "top_p", + "seed", + "tools", + "tool_choice", + "response_format", + "stream", + "stream_options", + "parallel_tool_calls", + "max_tokens", + "max_completion_tokens", + "reasoning_effort", +]; + +/** 200 with a null value — stripping these would be scope the rule has not earned. */ +const ACCEPTED_WHEN_NULL = [ + "stop", + "presence_penalty", + "frequency_penalty", + "n", + "user", + "logprobs", + "logit_bias", + "top_logprobs", + "metadata", +]; + +test("aimlapi: every field the upstream rejects as null is dropped before dispatch", () => { + const body: Record = { + model: "claude-sonnet-4.6", + messages: [{ role: "user", content: "hi" }], + }; + for (const key of REJECTED_WHEN_NULL) body[key] = null; + stripUnsupportedParams("aimlapi", "claude-sonnet-4.6", body); + + for (const key of REJECTED_WHEN_NULL) { + assert.equal(key in body, false, `${key} must not reach the upstream as null`); + } + assert.equal(body.model, "claude-sonnet-4.6"); + assert.ok(Array.isArray(body.messages), "the payload itself must survive"); +}); + +test("aimlapi: real values are never touched — only the literal null is dropped", () => { + // Falsy-but-meaningful values are the trap here: a plain `drop` rule would + // discard a deliberate temperature: 0 or stream: false. + const body: Record = { + model: "claude-sonnet-4.6", + temperature: 0, + top_p: 0.95, + seed: 42, + stream: false, + max_tokens: 16, + parallel_tool_calls: false, + tools: [], + reasoning_effort: "low", + }; + stripUnsupportedParams("aimlapi", "claude-sonnet-4.6", body); + + assert.equal(body.temperature, 0); + assert.equal(body.top_p, 0.95); + assert.equal(body.seed, 42); + assert.equal(body.stream, false); + assert.equal(body.max_tokens, 16); + assert.equal(body.parallel_tool_calls, false); + assert.deepEqual(body.tools, []); + assert.equal(body.reasoning_effort, "low"); +}); + +test("aimlapi: params the upstream accepts as null are left alone", () => { + const body: Record = { model: "claude-sonnet-4.6" }; + for (const key of ACCEPTED_WHEN_NULL) body[key] = null; + stripUnsupportedParams("aimlapi", "claude-sonnet-4.6", body); + + for (const key of ACCEPTED_WHEN_NULL) { + assert.equal(body[key], null, `${key} returns 200 with null and must be preserved`); + } +}); + +test("aimlapi: the two lists are disjoint", () => { + // Guards against a future edit moving a field into both lists, which would make + // one of the assertions above vacuous. + for (const key of REJECTED_WHEN_NULL) { + assert.equal(ACCEPTED_WHEN_NULL.includes(key), false, `${key} cannot be in both lists`); + } +}); + +test("the null-drop is scoped to aimlapi and does not leak to other providers", () => { + const body: Record = { + model: "gpt-5.4", + temperature: null, + seed: null, + stream: null, + }; + stripUnsupportedParams("openai", "gpt-5.4", body); + + assert.equal(body.temperature, null, "other providers keep their body verbatim"); + assert.equal(body.seed, null); + assert.equal(body.stream, null); +}); diff --git a/tests/unit/executors-strip-unsupported-params.test.ts b/tests/unit/executors-strip-unsupported-params.test.ts index 9e98a9f9de4..8ed1bd39386 100644 --- a/tests/unit/executors-strip-unsupported-params.test.ts +++ b/tests/unit/executors-strip-unsupported-params.test.ts @@ -162,8 +162,12 @@ test("STRIP_RULES is non-empty and every rule has a drop list or a clamp mechani assert.ok(__STRIP_RULES_FOR_TEST.length > 0); for (const rule of __STRIP_RULES_FOR_TEST) { const hasDrop = Array.isArray(rule.drop) && rule.drop.length > 0; + const hasDropIfNull = Array.isArray(rule.dropIfNull) && rule.dropIfNull.length > 0; const hasClamp = rule.clampToModelMaxOutput === true || Number.isFinite(rule.maxOutputCap); - assert.ok(hasDrop || hasClamp, "rule must either drop params or clamp max output"); + assert.ok( + hasDrop || hasDropIfNull || hasClamp, + "rule must either drop params (always or only when null) or clamp max output" + ); assert.ok(typeof rule.match === "function" || rule.match instanceof RegExp); } }); @@ -193,11 +197,19 @@ test("stripUnsupportedParams: volcengine kimi-k2-5-260127 also clamps max_comple test("stripUnsupportedParams: volcengine non-kimi model (glm-4-7-251222) is NOT clamped by the kimi rule", () => { const body: Record = { max_tokens: 65536 }; stripUnsupportedParams("volcengine", "glm-4-7-251222", body); - assert.equal(body.max_tokens, 65536, "kimi-specific cap must not apply to other volcengine models"); + assert.equal( + body.max_tokens, + 65536, + "kimi-specific cap must not apply to other volcengine models" + ); }); test("stripUnsupportedParams: kimi rule is provider-scoped (no-op for non-volcengine providers)", () => { const body: Record = { max_tokens: 65536 }; stripUnsupportedParams("kimi", "kimi-k2-5-260127", body); - assert.equal(body.max_tokens, 65536, "the Ark-specific cap must not leak to other kimi-hosting providers"); + assert.equal( + body.max_tokens, + 65536, + "the Ark-specific cap must not leak to other kimi-hosting providers" + ); }); diff --git a/tests/unit/provider-models-discovery-split.test.ts b/tests/unit/provider-models-discovery-split.test.ts index 72184e01694..359afe0490a 100644 --- a/tests/unit/provider-models-discovery-split.test.ts +++ b/tests/unit/provider-models-discovery-split.test.ts @@ -142,7 +142,41 @@ test("providerModelsConfig keeps the aimlapi live catalog entry", () => { assert.equal(PROVIDER_MODELS_CONFIG.aimlapi.url, "https://api.aimlapi.com/models"); }); -test("providerModelsConfig aimlapi.parseResponse keeps only chat-completion models when present", () => { +// RED before the fix: the live endpoint answers with the OpenAI-style envelope +// `{ object: "list", data: [...] }`, so `Array.isArray(data)` was false and the +// parser returned [] for every real response — the whole provider degraded to its +// static seed. The old test only ever fed it a bare array, which is why a green +// suite coexisted with a provider that discovered nothing. +test("providerModelsConfig aimlapi.parseResponse unwraps the { object, data } catalog envelope", () => { + const parsed = PROVIDER_MODELS_CONFIG.aimlapi.parseResponse({ + object: "list", + data: [ + { id: "openai/gpt-5", type: "openai/chat-completions", info: { name: "GPT-5" } }, + { id: "flux/flux-pro", type: "openai/image-generations", info: { name: "FLUX Pro" } }, + ], + }); + assert.deepEqual(parsed, [{ id: "openai/gpt-5", name: "GPT-5" }]); +}); + +// RED before the fix: `chat-completion` matched 0 of the 936 live catalog rows — +// the vocabulary is `openai/chat-completions`. Both spellings stay accepted so a +// rename in either direction degrades to "some models missing" rather than "the +// provider has no models". +test("providerModelsConfig aimlapi.parseResponse accepts both chat type spellings", () => { + const parsed = PROVIDER_MODELS_CONFIG.aimlapi.parseResponse({ + object: "list", + data: [ + { id: "current", type: "openai/chat-completions", info: { name: "Current" } }, + { id: "legacy", type: "chat-completion", info: { name: "Legacy" } }, + ], + }); + assert.deepEqual(parsed, [ + { id: "current", name: "Current" }, + { id: "legacy", name: "Legacy" }, + ]); +}); + +test("providerModelsConfig aimlapi.parseResponse still accepts a bare array", () => { const parsed = PROVIDER_MODELS_CONFIG.aimlapi.parseResponse([ { id: "chat-1", type: "chat-completion", info: { name: "Chat 1" } }, { id: "img-1", type: "image" }, @@ -150,6 +184,21 @@ test("providerModelsConfig aimlapi.parseResponse keeps only chat-completion mode assert.deepEqual(parsed, [{ id: "chat-1", name: "Chat 1" }]); }); +// The old parser fell through to `all` when the chat filter matched nothing, which +// would have published 583 video/image/TTS/batch ids into a chat model picker and +// masked the broken type filter. An unrecognised catalog must yield nothing. +test("providerModelsConfig aimlapi.parseResponse never falls back to non-chat rows", () => { + const parsed = PROVIDER_MODELS_CONFIG.aimlapi.parseResponse({ + object: "list", + data: [ + { id: "veo/veo-3", type: "internal/video-generations/submit" }, + { id: "eleven/tts", type: "internal/text-to-speech" }, + { id: "anthropic/claude-opus-5", type: "anthropic/messages" }, + ], + }); + assert.deepEqual(parsed, []); +}); + test("providerModelsConfig grok-cli.parseResponse preserves exact supported reasoning efforts", () => { const parsed = PROVIDER_MODELS_CONFIG["grok-cli"].parseResponse({ models: [ diff --git a/tests/unit/provider-models-route.test.ts b/tests/unit/provider-models-route.test.ts index a8d6d2dd80a..e1fde44dd99 100644 --- a/tests/unit/provider-models-route.test.ts +++ b/tests/unit/provider-models-route.test.ts @@ -89,13 +89,25 @@ test("provider models route fetches the live AI/ML API catalog from the auth-fre apiKey: "aiml-key", }); let calledUrl = ""; + // The real endpoint answers with the OpenAI-style envelope and the + // `openai/chat-completions` type. The previous version of this mock returned a + // bare array of `chat-completion` rows — neither of which the live catalog has + // produced since the schema change — so the route passed here while returning + // zero models in production. globalThis.fetch = async (url) => { calledUrl = String(url); - return Response.json([ - { id: "openai/gpt-5.5", type: "chat-completion", info: { name: "GPT-5.5" } }, - { id: "zhipu/glm-5.2", type: "chat-completion", info: { name: "GLM 5.2" } }, - { id: "flux/flux-pro", type: "image", info: { name: "FLUX Pro" } }, - ]); + return Response.json({ + object: "list", + data: [ + { id: "openai/gpt-5", type: "openai/chat-completions", info: { name: "GPT-5" } }, + { + id: "anthropic/claude-sonnet-4-6", + type: "openai/chat-completions", + info: { name: "Claude 4.6 Sonnet" }, + }, + { id: "flux/flux-pro", type: "openai/image-generations", info: { name: "FLUX Pro" } }, + ], + }); }; const response = await callRoute(connection.id); @@ -107,7 +119,7 @@ test("provider models route fetches the live AI/ML API catalog from the auth-fre assert.equal(body.source, "api"); assert.equal(calledUrl, "https://api.aimlapi.com/models"); const ids = body.models.map((m: any) => m.id); - assert.ok(ids.includes("openai/gpt-5.5") && ids.includes("zhipu/glm-5.2")); + assert.ok(ids.includes("openai/gpt-5") && ids.includes("anthropic/claude-sonnet-4-6")); assert.ok(!ids.includes("flux/flux-pro"), "non-chat model types are filtered out"); });