diff --git a/scripts/test-layout/layout.json b/scripts/test-layout/layout.json index c65f7c378b..74bf21b604 100644 --- a/scripts/test-layout/layout.json +++ b/scripts/test-layout/layout.json @@ -648,6 +648,7 @@ "devin-adapter.test.ts": "providers", "devin-cli-authmode-migration.test.ts": "providers", "devin-effort-ladder.test.ts": "providers", + "devin-live-models.test.ts": "providers", "devin-login.test.ts": "providers", "devin-provider-merge-migration.test.ts": "providers", "devin-hardening.test.ts": "providers", diff --git a/src/adapters/devin/cloud-direct/catalog.ts b/src/adapters/devin/cloud-direct/catalog.ts index 4dcb42b082..1f45027c90 100644 --- a/src/adapters/devin/cloud-direct/catalog.ts +++ b/src/adapters/devin/cloud-direct/catalog.ts @@ -297,6 +297,19 @@ export function clearCachedCatalog(): void { cacheEpoch++; } +/** + * Test seam: install a catalog as the live cache entry. Mirrors + * clearCachedCatalog's invalidation — the in-flight slot is dropped and the + * epoch bumped — so a fetch racing the seed cannot overwrite it, and a null + * entry resets the cache between tests. + */ +export function setCachedCatalogForTests(entry: CacheEntry | null): void { + cached = entry; + inFlight = null; + inFlightKey = null; + cacheEpoch++; +} + /** * Tier-disabled error — thrown by the chat pre-flight when the catalog lists * a model as `disabled: true` for this account. The message names the model diff --git a/src/adapters/devin/live-models.ts b/src/adapters/devin/live-models.ts index bfd75e70bb..ea07ea5a49 100644 --- a/src/adapters/devin/live-models.ts +++ b/src/adapters/devin/live-models.ts @@ -139,7 +139,13 @@ export const DEVIN_MODEL_EFFORTS: Record = { export const DEVIN_DEFAULT_EFFORTS = ["low", "medium", "high", "xhigh", "max"]; export type DevinUsableModelsResult = - | { ok: true; models: string[]; contextWindows: Record; efforts: Record } + | { + ok: true; + models: string[]; + contextWindows: Record; + efforts: Record; + inputModalities: Record; + } | { ok: false; error: "auth" | "http" | "empty" | "unknown"; detail?: string }; /** @@ -160,6 +166,8 @@ export async function fetchDevinUsableModels(opts: { const contextWindows: Record = {}; // Effort rungs per base, recovered from the suffixes the collapse strips. const rungs = new Map>(); + // supportsImages votes per base; only rows that asserted field #5 vote. + const imageVotes = new Map(); for (const entry of catalog.byUid.values()) { if (entry.disabled) continue; // Skip internal enum constants (e.g. MODEL_GPT_5_2_LOW, MODEL_PRIVATE_*). @@ -183,6 +191,15 @@ export async function fetchDevinUsableModels(opts: { const seen = contextWindows[base]; contextWindows[base] = seen === undefined ? entry.contextWindow : Math.min(seen, entry.contextWindow); } + if (entry.supportsImages !== undefined) { + let votes = imageVotes.get(base); + if (!votes) { + votes = { sawTrue: false, sawFalse: false }; + imageVotes.set(base, votes); + } + if (entry.supportsImages) votes.sawTrue = true; + else votes.sawFalse = true; + } } if (bases.size === 0) return { ok: false, error: "empty" }; const efforts: Record = {}; @@ -191,7 +208,21 @@ export async function fetchDevinUsableModels(opts: { // would draw a picker whose only option is the value already in effect. if (set.size > 1) efforts[base] = sortDevinRungs(set); } - return { ok: true, models: [...bases].sort(), contextWindows, efforts }; + // supportsImages arrives tri-state per catalog row, so the collapse votes: + // a row that never asserted field #5 abstains, which keeps an unsuffixed + // unknown row from poisoning a base whose effort variants were measured + // image-capable. Unanimous measured rows advertise; measured disagreement + // advertises nothing, because a single measured false is not outvoted by + // its siblings. One accepted mismatch: resolveWireModelUid prefers the + // plain UID when the catalog lists it, so a base advertised + // ["text","image"] on variant evidence can still route a no-effort request + // to a plain row that never asserted the field. + const inputModalities: Record = {}; + for (const [base, votes] of imageVotes) { + if (votes.sawTrue && votes.sawFalse) continue; + inputModalities[base] = votes.sawTrue ? ["text", "image"] : ["text"]; + } + return { ok: true, models: [...bases].sort(), contextWindows, efforts, inputModalities }; } catch (error) { const message = error instanceof Error ? error.message : String(error); if (/unauth|401|invalid token|login/i.test(message)) return { ok: false, error: "auth", detail: message }; diff --git a/src/codex/catalog/provider-fetch.ts b/src/codex/catalog/provider-fetch.ts index f76e0b98fb..a6791783d7 100644 --- a/src/codex/catalog/provider-fetch.ts +++ b/src/codex/catalog/provider-fetch.ts @@ -1745,6 +1745,12 @@ async function fetchProviderModelsWithAuth( // away, and every client that keys an effort control off this field — // the Pi-shaped exports — renders no control at all. ...(liveResult.efforts[id]?.length ? { reasoningEfforts: liveResult.efforts[id] } : {}), + // The account catalog's per-base supportsImages vote collapses to one + // modalities value. It spreads before the hints so exact + // modelCapabilities declarations, the legacy modelInputModalities + // record and the vision-sidecar rewrite keep winning — the live + // value survives only when none of them applies. + ...(liveResult.inputModalities[id]?.length ? { inputModalities: liveResult.inputModalities[id] } : {}), ...catalogHintsFromProviderConfig(name, prov, id, contextCap, metadataModelIdCaseFold, captured.effectiveAlias), } as CatalogModel; }); diff --git a/structure/adapters/registry.md b/structure/adapters/registry.md index 9bc0569599..c43fc06484 100644 --- a/structure/adapters/registry.md +++ b/structure/adapters/registry.md @@ -56,7 +56,9 @@ Some adapters share another adapter's routed-tool semantics while retaining inde `ClientModelConfig` field #4 as the per-account disabled gate, field #18 as the per-account context window, and field #5 as an optional `supportsImages` tri-state — a present value asserts image support or its absence, while an omitted field stays unknown (the #1796 - precedent). + precedent). `src/adapters/devin/live-models.ts` collapses that tri-state across each base + model's effort variants: unmeasured rows abstain, unanimous measured rows advertise + `["text"]` or `["text", "image"]`, and measured disagreement stays unadvertised. The registry records those relationships with `contractParent`. A parent relationship does **not** mean the registry recursively constructs a parent adapter and injects it into the child. Azure and MiMo keep owning their existing internal composition. This avoids making production constructors depend on test/conformance needs and keeps this authority refactor behavior-neutral. diff --git a/structure/catalog.md b/structure/catalog.md index 8cce6268b0..57c7bbdcd9 100644 --- a/structure/catalog.md +++ b/structure/catalog.md @@ -95,6 +95,10 @@ Provider live-model lists are cached with a configured TTL (`src/codex/model-cac deleting, or editing a provider's shape clears that per-provider cache; a disabled-only change deliberately does not, because a disabled provider is already excluded from the catalog gather instead. Codex's own `models_cache.json` is a different cache, invalidated by catalog refresh. +A Devin live row spreads its measured `contextWindow`, `reasoningEfforts` and +`inputModalities` before `catalogHintsFromProviderConfig`, so exact `modelCapabilities` +declarations, the legacy `modelInputModalities` record and the vision-sidecar rewrite keep +precedence and a live value survives only when none of them applies. For `liveModels: false`, a static provider publishes the ordered union of `models` and `retainModels`. When `models` is absent or empty, its configured `defaultModel` seeds that diff --git a/tests/fixtures/test-layout-expected.json b/tests/fixtures/test-layout-expected.json index 145cbfc22b..dc8eceebb2 100644 --- a/tests/fixtures/test-layout-expected.json +++ b/tests/fixtures/test-layout-expected.json @@ -482,6 +482,7 @@ "devin-effort-ladder.test.ts": "providers", "devin-hardening.test.ts": "providers", "devin-image-passthrough.test.ts": "providers", + "devin-live-models.test.ts": "providers", "devin-prompt-cache.test.ts": "providers", "devin-stream-deadline.test.ts": "providers", "digitalocean-scaleway-provider.test.ts": "providers", diff --git a/tests/providers/devin-live-models.test.ts b/tests/providers/devin-live-models.test.ts new file mode 100644 index 0000000000..56dd99507c --- /dev/null +++ b/tests/providers/devin-live-models.test.ts @@ -0,0 +1,180 @@ +/** + * Devin live-discovery collapse and advertised-catalog propagation for + * ClientModelConfig field #5 (supportsImages). + * + * Catalogs are hand-encoded protobuf run through the real parser + * (parseCatalogBuffer) and installed through setCachedCatalogForTests, so the + * tests cover the collapse in fetchDevinUsableModels and the Devin branch of + * fetchProviderModels without touching the network. KEY is unique to this + * file and HOST is the stripped default host: getCachedCatalog hits only on + * an exact (apiKey, host) match with a fresh fetchedAt. + */ +import { afterEach, beforeEach, describe, expect, spyOn, test } from "bun:test"; +import * as oauth from "../../src/oauth"; +import { fetchDevinUsableModels } from "../../src/adapters/devin/live-models"; +import { parseCatalogBuffer, setCachedCatalogForTests } from "../../src/adapters/devin/cloud-direct/catalog"; +import { encodeMessage, encodeString, encodeVarintField } from "../../src/adapters/devin/cloud-direct/wire"; +import { fetchProviderModels } from "../../src/codex/catalog/provider-fetch"; +import { clearModelCache, providerCacheGenerations } from "../../src/codex/model-cache"; +import type { OcxProviderConfig } from "../../src/types"; + +const HOST = "https://server.codeium.com"; +const KEY = "devin-live-models-test-key"; + +/** One ClientModelConfig body; field #5 stays absent unless opts asserts it. */ +function catalogEntry( + uid: string, + opts: { disabled?: boolean; supportsImages?: boolean; contextWindow?: number } = {}, +): Buffer { + return Buffer.concat([ + encodeString(1, uid), + ...(opts.disabled === true ? [encodeVarintField(4, 1)] : []), + // encodeVarintField(5, 0) is a measured text-only vote — real bytes, not + // an omission — while leaving field #5 out keeps the row unknown. + ...(opts.supportsImages !== undefined ? [encodeVarintField(5, opts.supportsImages ? 1 : 0)] : []), + ...(opts.contextWindow !== undefined ? [encodeVarintField(18, opts.contextWindow)] : []), + encodeString(22, uid), + ]); +} + +function seedCatalog(...entries: Buffer[]): void { + setCachedCatalogForTests(parseCatalogBuffer( + Buffer.concat(entries.map((entry) => encodeMessage(1, entry))), + KEY, + HOST, + )); +} + +beforeEach(() => { + setCachedCatalogForTests(null); + clearModelCache("devin-test"); + providerCacheGenerations.delete("devin-test"); +}); +afterEach(() => { + setCachedCatalogForTests(null); + clearModelCache("devin-test"); + providerCacheGenerations.delete("devin-test"); +}); + +describe("devin live model discovery", () => { + test("collapses per-variant supportsImages votes into per-base input modalities", async () => { + seedCatalog( + // Unanimous measured rows advertise. + catalogEntry("vision-model", { supportsImages: true, contextWindow: 262_000 }), + catalogEntry("vision-model-high", { supportsImages: true, contextWindow: 1_000_000 }), + catalogEntry("text-model-low", { supportsImages: false }), + catalogEntry("text-model-high", { supportsImages: false }), + // An unsuffixed row that never asserted field #5 abstains instead of + // poisoning a measured image base. + catalogEntry("abstain-model"), + catalogEntry("abstain-model-high", { supportsImages: true }), + // Measured disagreement stays unadvertised — a single false is not + // outvoted by its siblings. + catalogEntry("split-model", { supportsImages: true }), + catalogEntry("split-model-low", { supportsImages: true }), + catalogEntry("split-model-high", { supportsImages: false }), + catalogEntry("mixed-model-low", { supportsImages: true }), + catalogEntry("mixed-model-high", { supportsImages: false }), + // Zero measured rows advertise nothing. + catalogEntry("mystery-model"), + catalogEntry("mystery-model-high"), + // Disabled and MODEL_* rows are skipped before they can vote: if the + // disabled true voted, text-off-model would read as disagreement. + catalogEntry("text-off-model", { supportsImages: false }), + catalogEntry("text-off-model-high", { disabled: true, supportsImages: true }), + catalogEntry("ghost-model-high", { disabled: true, supportsImages: true }), + catalogEntry("MODEL_INTERNAL_VISION", { supportsImages: true }), + ); + const result = await fetchDevinUsableModels({ apiKey: KEY, baseUrl: HOST }); + if (!result.ok) throw new Error(`expected ok, got ${result.error}`); + expect(result.models).toEqual([ + "abstain-model", + "mixed-model", + "mystery-model", + "split-model", + "text-model", + "text-off-model", + "vision-model", + ]); + expect(result.inputModalities).toEqual({ + "vision-model": ["text", "image"], + "text-model": ["text"], + "abstain-model": ["text", "image"], + "text-off-model": ["text"], + }); + // The collapse adds a field; the existing projections are unchanged. + expect(result.contextWindows["vision-model"]).toBe(262_000); + expect(result.efforts["text-model"]).toEqual(["low", "high"]); + }); + + test("a catalog with no measured rows still carries an empty record", async () => { + seedCatalog(catalogEntry("plain-model"), catalogEntry("plain-model-high")); + const result = await fetchDevinUsableModels({ apiKey: KEY, baseUrl: HOST }); + if (!result.ok) throw new Error(`expected ok, got ${result.error}`); + expect(result.inputModalities).toEqual({}); + }); +}); + +describe("devin advertised catalog input modalities", () => { + // Devin is an oauth provider, so discovery resolves its bearer through + // resolveModelsAuthToken; the tests lend it a token rather than an account + // store (the same seam the Copilot oauth cases use). + let authSpy: ReturnType | undefined; + beforeEach(() => { + authSpy = spyOn(oauth, "resolveModelsAuthToken").mockResolvedValue(KEY); + }); + afterEach(() => { + authSpy?.mockRestore(); + authSpy = undefined; + }); + + const devinProvider = (extra: Partial = {}): OcxProviderConfig => ({ + adapter: "devin", + baseUrl: HOST, + apiKey: KEY, + authMode: "oauth", + liveModels: true, + ...extra, + } as OcxProviderConfig); + + test("a measured image base advertises text and image", async () => { + seedCatalog(catalogEntry("img-model", { supportsImages: true })); + const models = await fetchProviderModels("devin-test", devinProvider(), 60_000); + expect(models.map((model) => model.id)).toEqual(["img-model"]); + expect(models[0]?.inputModalities).toEqual(["text", "image"]); + }); + + test("an exact modelCapabilities declaration overwrites the live value", async () => { + seedCatalog(catalogEntry("img-model", { supportsImages: true })); + const models = await fetchProviderModels("devin-test", devinProvider({ + modelCapabilities: { "img-model": { inputModalities: ["audio"] } }, + }), 60_000); + expect(models[0]?.inputModalities).toEqual(["audio"]); + }); + + test("an exact text-only declaration still takes the sidecar path", async () => { + // A text-only modelCapabilities entry makes the row a vision-sidecar + // consumer (src/vision/eligibility.ts): the declaration governs runtime + // eligibility while the catalog keeps attachments unblocked. + seedCatalog(catalogEntry("img-model", { supportsImages: true })); + const models = await fetchProviderModels("devin-test", devinProvider({ + modelCapabilities: { "img-model": { inputModalities: ["text"] } }, + }), 60_000); + expect(models[0]?.inputModalities).toEqual(["text", "image"]); + }); + + test("a noVisionModels entry upgrades a live text-only row through the sidecar", async () => { + seedCatalog(catalogEntry("side-model", { supportsImages: false })); + const models = await fetchProviderModels("devin-test", devinProvider({ + noVisionModels: ["side-model"], + }), 60_000); + expect(models[0]?.inputModalities).toEqual(["text", "image"]); + }); + + test("a measured text-only base is not upgraded without a sidecar consumer", async () => { + seedCatalog(catalogEntry("plain-model", { supportsImages: false })); + const models = await fetchProviderModels("devin-test", devinProvider(), 60_000); + expect(models.map((model) => model.id)).toEqual(["plain-model"]); + expect(models[0]?.inputModalities).toEqual(["text"]); + }); +});