Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 12 additions & 0 deletions docs-site/src/content/docs/guides/providers.md
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,18 @@ Shipped v1 configs migrate automatically to marker 2 and one option-aware row. T
is retained once at `~/.opencodex/config.json.pre-openai-tiers-v2.bak`; restore it with
`cp ~/.opencodex/config.json.pre-openai-tiers-v2.bak ~/.opencodex/config.json`.

## Anthropic image input

The built-in Claude model seeds advertise text and image input for both `anthropic` (OAuth) and
`anthropic-apikey`, consistent with [Anthropic's model overview](https://platform.claude.com/docs/en/models/overview).
Explicit per-model input-modality overrides remain authoritative; unknown models are not assumed
image-capable. This applies across integrations wherever the client's configuration supports image
capability metadata: OpenClaw exports a declared `input` array, and Kimi Code exports
`capabilities: ["image_in"]` only for image-capable models. OpenClaw omits `input` when no supported
modalities are declared; Kimi omits `capabilities` for unknown or text-only models. Clients without
a supported capability field keep their existing configuration shape. After updating opencodex,
regenerate or refresh the client configuration managed by opencodex to receive the updated metadata.

## Auth modes

Provider configs accept three `authMode` values (`key` is the default). The built-in registry also
Expand Down
10 changes: 7 additions & 3 deletions src/clients/config-export.ts
Original file line number Diff line number Diff line change
Expand Up @@ -812,6 +812,7 @@ export interface OpenclawModelEntry {
id: string;
name: string;
contextWindow?: number;
input?: string[];
}

export interface OpenclawProviderBlock {
Expand Down Expand Up @@ -839,15 +840,15 @@ export interface KimiProviderBlock {
/**
* `max_context_size` is mandatory and must be positive, so a model with no
* authoritative context window is omitted from the document entirely rather
* than guessed at. `capabilities` is never emitted: our catalog does not
* assert them, and Kimi's own inference works off OpenAI-style name prefixes
* that a routed selector will not match.
* than guessed at. Catalog image input becomes `image_in`; other capabilities
* are not inferred from routed model names.
*/
export interface KimiModelBlock {
provider: string;
model: string;
max_context_size: number;
display_name?: string;
capabilities?: ["image_in"];
}

export interface KimiGeneratedConfig {
Expand Down Expand Up @@ -974,10 +975,12 @@ function buildHermesClientConfig(ctx: ExportContext): HermesGeneratedConfig {
function buildOpenclawClientConfig(ctx: ExportContext): OpenclawGeneratedConfig {
const models: OpenclawModelEntry[] = normalizeExportModels(ctx.models).map(model => {
const context = authoritativeContextWindow(model.contextWindow);
const input = [...new Set(model.inputModalities?.filter(value => ["text", "image", "video", "audio"].includes(value)))];
return {
id: model.namespaced,
name: exportModelLabel(model),
...(context !== undefined ? { contextWindow: context } : {}),
...(input.length > 0 ? { input } : {}),
};
});
const headers = proxyAdmissionHeaders(ctx.config, OPENCLAW_API_KEY_ENV_REF);
Expand Down Expand Up @@ -1015,6 +1018,7 @@ function buildKimiClientConfig(ctx: ExportContext): KimiGeneratedConfig {
model: model.namespaced,
max_context_size: context,
...(model.displayName ? { display_name: model.displayName } : {}),
...(model.inputModalities?.includes("image") ? { capabilities: ["image_in"] as ["image_in"] } : {}),
};
}
return {
Expand Down
3 changes: 3 additions & 0 deletions src/providers/registry/entries-core.ts
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ import type { ProviderRegistryEntry } from "./types";
import {
ANTHROPIC_MODELS,
ANTHROPIC_MODEL_CONTEXT_WINDOWS,
ANTHROPIC_MODEL_INPUT_MODALITIES,
ANTHROPIC_DEFAULT_MAX_OUTPUT_TOKENS,
ANTHROPIC_MODEL_REASONING_EFFORTS,
ZAI_GLM_52_REASONING_EFFORTS,
Expand Down Expand Up @@ -383,6 +384,7 @@ export const PROVIDER_REGISTRY_CORE: readonly ProviderRegistryEntry[] = [
note: "Log in with your Claude account",
models: [...ANTHROPIC_MODELS],
modelContextWindows: { ...ANTHROPIC_MODEL_CONTEXT_WINDOWS },
modelInputModalities: { ...ANTHROPIC_MODEL_INPUT_MODALITIES },
modelReasoningEfforts: { ...ANTHROPIC_MODEL_REASONING_EFFORTS },
// Codex omits max_output_tokens; without a provider budget the Anthropic adapter
// falls back to 8192, which truncates long answers with stop_reason=max_tokens.
Expand All @@ -403,6 +405,7 @@ export const PROVIDER_REGISTRY_CORE: readonly ProviderRegistryEntry[] = [
models: [...ANTHROPIC_MODELS],
liveModels: true,
modelContextWindows: { ...ANTHROPIC_MODEL_CONTEXT_WINDOWS },
modelInputModalities: { ...ANTHROPIC_MODEL_INPUT_MODALITIES },
modelReasoningEfforts: { ...ANTHROPIC_MODEL_REASONING_EFFORTS },
defaultMaxOutputTokens: ANTHROPIC_DEFAULT_MAX_OUTPUT_TOKENS,
defaultModel: "claude-sonnet-5",
Expand Down
4 changes: 4 additions & 0 deletions src/providers/registry/model-seeds.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,10 @@ import type { ProviderModelDiscoverySpec } from "./types";
// always on, per the official models overview and pricing page (platform.claude.com).
export const ANTHROPIC_MODELS = ["claude-fable-5-1", "claude-fable-5", "claude-sonnet-5", "claude-opus-5", "claude-opus-4-8", "claude-opus-4-7", "claude-opus-4-6", "claude-sonnet-4-6", "claude-haiku-4-5"];
export const ANTHROPIC_MODEL_CONTEXT_WINDOWS: Record<string, number> = { "claude-fable-5-1": 1_000_000, "claude-sonnet-5": 1_000_000, "claude-fable-5": 1_000_000, "claude-opus-5": 1_000_000, "claude-opus-4-8": 1_000_000, "claude-opus-4-7": 1_000_000, "claude-opus-4-6": 1_000_000, "claude-sonnet-4-6": 1_000_000, "claude-haiku-4-5": 200_000 };
// All seeded Claude models support vision: https://platform.claude.com/docs/en/models/overview
export const ANTHROPIC_MODEL_INPUT_MODALITIES: Record<string, string[]> = Object.fromEntries(
ANTHROPIC_MODELS.map(id => [id, ["text", "image"]]),
);
// Every current Claude family accepts at least 64k output tokens (Haiku 4.5 / Sonnet 4.x
// through Opus 5 and Fable 5). Anthropic caps max_tokens per model server-side, so a
// larger request never over-allocates; it only stops the 8192 truncation.
Expand Down
18 changes: 18 additions & 0 deletions structure/clients/integrations.md
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,24 @@ selector, preserving the underlying provider, model ID, modalities, limits, and
False or missing metadata never causes local inference, so old or disabled remote hubs remain
authoritative. Existing client configs receive the entries on export or managed refresh.

## Model input capability exports

All registered integrations consume the shared catalog, including [Anthropic seed image metadata](../runtime.md#capability-aware-image-admission), through their existing schema-specific exports:

| Client | Per-model output |
| --- | --- |
| OpenCode | `attachment`, `modalities.input` |
| Pi, OMP, Prime, Aside, omo, Gajae, DSH | `input` (text/image only) |
| ZCode | `modalities.input` (text/image only) |
| Cline | `modalities.input`, `supportsVision` |
| Hermes | `supports_vision` (see below) |
| OpenClaw | `input`, filtered to declared text/image/video/audio; omitted when none remain |
| Kimi Code | `capabilities: ["image_in"]` only for declared image input; omitted for unknown/text-only models |
| MiniMax Code | No per-model image capability field emitted |
| Raycast | `abilities.vision.supported` |

No exporter infers image support from a model name. Existing client eligibility filters and ownership/refresh rules remain unchanged; exports do not add fields to schemas without a supported mapping.

## Hermes Model Capabilities

Hermes cannot infer custom-provider capabilities from its built-in registry. The OpenCodex
Expand Down
2 changes: 2 additions & 0 deletions structure/providers/xai-grok.md
Original file line number Diff line number Diff line change
Expand Up @@ -96,6 +96,8 @@ privately to final dispatch; preliminary route selection does not inject Go-only

Devin CLI credential path composition in `src/oauth/devin/cli-import.ts` follows the selected platform: Windows uses Win32 APPDATA paths, other platforms use POSIX XDG-data paths. The explicit absolute override remains verbatim; credential parsing and login behavior are unchanged.

[Anthropic seed image metadata](../runtime.md#capability-aware-image-admission) is provider-scoped; xAI model metadata and transport behavior remain unchanged.

Provider-scoped catalog hints remain isolated by provider in `src/providers/registry/entries-core.ts`. The
OpenCode Go `deepseek-v4.1-flash` 1,048,576-token context hint does not change xAI model metadata or
transport behavior.
Expand Down
6 changes: 6 additions & 0 deletions structure/runtime.md
Original file line number Diff line number Diff line change
Expand Up @@ -408,6 +408,12 @@ The [explicit model-capability contract](config.md#explicit-per-model-capability

## Capability-aware image admission

The `anthropic` OAuth and `anthropic-apikey` presets in `src/providers/registry/entries-core.ts`
declare `modelInputModalities: ["text", "image"]` per model for the nine Claude seeds in
`src/providers/registry/model-seeds.ts`. Existing enrichment fills missing entries while preserving
explicit operator overrides; unknown models receive no new declaration. Client eligibility filters
and Anthropic image wire handling remain unchanged.

`src/vision/plan.ts` prevents raw image bytes from reaching any target whose effective capability is positively known to exclude image input. Evidence from the resolved runtime provider and explicit operator declarations takes precedence, followed by backend-specific/registry/vendor metadata. A proven text-only target is preprocessed through the configured Vision Sidecar; a positively image-capable target receives the image directly. Genuinely unknown custom models retain the existing compatibility path rather than being guessed text-only.

Canonical ChatGPT Codex forwarding uses the generated `openai-codex` capability bundle rather than the public `openai` bundle. This matters when the two backends differ: for example, the vendored metadata records `gpt-5.3-codex-spark` as text-only on `openai-codex` while the public OpenAI row lists image input. The native Chat fast path and web-search image verbalization consume the same effective-capability decision.
Expand Down
2 changes: 2 additions & 0 deletions structure/subagents.md
Original file line number Diff line number Diff line change
Expand Up @@ -371,6 +371,8 @@ The [explicit model-capability contract](config.md#explicit-per-model-capability

Exact [model input declarations](config.md#explicit-per-model-capability-declarations) now feed text-only eligibility and catalog hints; existing image-description/omission handling consumes them before the main upstream send.

[Anthropic seed image metadata](runtime.md#capability-aware-image-admission) supplies missing capability evidence; subagent selection and eligibility rules remain unchanged.

Provider-scoped approval reviewer settings are projected by the [catalog owner](catalog.md#provider-scoped-approval-reviewer); this surface retains its existing routing, transport and account-selection behavior.

Renamed fixed-key providers receive [missing reasoning metadata](catalog.md#renamed-destination-reasoning-metadata) during derivation; explicit per-model entries and provider defaults retain precedence.
2 changes: 2 additions & 0 deletions structure/transports/inventory.md
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,8 @@ changes translated message placement only; endpoint selection and transport stay

Shared parsing and streaming follow the [request-copy](byte-accounting.md#request-copy-accounting) and [stream-buffer accounting](byte-accounting.md#stream-buffer-accounting) contracts. Response-attached WebSocket telemetry follows the [stage record identity contract](responses.md#passthrough-sse-stream-shapes-314).

[Anthropic seed image metadata](../runtime.md#capability-aware-image-admission) supplies missing capability evidence; transport selection and image wire handling remain unchanged.

## Transport inventory

The sections above cover the transports with load-bearing invariants. The rest of the transport
Expand Down
27 changes: 26 additions & 1 deletion tests/claude-integration/claude-model-info.test.ts
Original file line number Diff line number Diff line change
@@ -1,8 +1,33 @@
import { describe, expect, test } from "bun:test";
import { buildAnthropicModelInfos, nativeEffectiveLadder } from "../../src/claude/model-info";
import { nativeEffortClamp } from "../../src/codex/catalog";
import { gatherRoutedModels, nativeEffortClamp } from "../../src/codex/catalog";

describe("anthropic-flavor ModelInfo discovery entries (devlog 130 B4b)", () => {
test.each(["anthropic", "anthropic-apikey"])("%s registry image inputs reach Claude discovery aliases", async (provider) => {
const models = await gatherRoutedModels({
port: 10100,
defaultProvider: provider,
providers: {
[provider]: {
adapter: "anthropic",
baseUrl: "https://api.anthropic.com",
authMode: provider === "anthropic" ? "oauth" : "key",
liveModels: false,
},
},
});
const routed = models.filter(model => model.provider === provider);
expect(routed.length).toBeGreaterThan(0);
for (const idStyle of ["readable", "desktop3p"] as const) {
const infos = buildAnthropicModelInfos([], routed, undefined, idStyle);
expect(infos.length).toBeGreaterThanOrEqual(routed.length);
expect(infos.some(info => info.id.endsWith("[1m]"))).toBe(true);
for (const info of infos) {
expect(info.capabilities.image_input.supported).toBe(true);
}
}
});

test("routed model with adapter-reported ladder advertises exactly those rungs", () => {
const [info] = buildAnthropicModelInfos([], [{
provider: "cursor", id: "gpt-5.6-luna",
Expand Down
23 changes: 23 additions & 0 deletions tests/codex-integration/catalog-input-modality-enum.test.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
import { beforeEach, describe, expect, test } from "bun:test";
import { ensureStrictCatalogFields } from "../../src/codex/catalog/parsing";
import { catalogHintsFromModelsApiItem } from "../../src/codex/catalog/provider-fetch";
import { buildCatalogEntries, gatherRoutedModels } from "../../src/codex/catalog";
import type { OcxConfig } from "../../src/types";

/**
Expand All @@ -12,6 +13,28 @@ import type { OcxConfig } from "../../src/types";
* verbatim and the Codex app reported `unknown variant 'video'` while showing zero apps.
*/
describe("catalog input_modalities stay inside the enum Codex accepts", () => {
test.each(["anthropic", "anthropic-apikey"])("%s registry image inputs reach the Codex catalog", async (provider) => {
const models = await gatherRoutedModels({
port: 10100,
defaultProvider: provider,
providers: {
[provider]: {
adapter: "anthropic",
baseUrl: "https://api.anthropic.com",
authMode: provider === "anthropic" ? "oauth" : "key",
liveModels: false,
},
},
});
const routed = models.filter(model => model.provider === provider);
expect(routed.length).toBeGreaterThan(0);
const entries = buildCatalogEntries(null, [], routed);
expect(entries).toHaveLength(routed.length);
for (const entry of entries) {
expect(entry.input_modalities).toEqual(["text", "image"]);
}
});

test("an out-of-enum modality is dropped rather than written through", () => {
const entry = ensureStrictCatalogFields(
{ slug: "zenmux/meta-muse-spark-1.1", input_modalities: ["text", "image", "audio", "video"] },
Expand Down
35 changes: 32 additions & 3 deletions tests/config/client-config-export-new-clients.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -133,6 +133,29 @@ describe("hermes", () => {
});

describe("openclaw", () => {
test("declares image input only from catalog capabilities", () => {
const doc = buildClientConfig("openclaw", ctx()) as OpenclawGeneratedConfig;
const models = doc.models.providers[OPENCODE_PROVIDER_ID]!.models;
expect(models.find(model => model.id === "anthropic/claude-opus-4-8")).toHaveProperty("input", ["text", "image"]);
expect(models.find(model => model.id === "gpt-5.5")).toHaveProperty("input", ["text"]);
expect(models.find(model => model.id === "local/no-window")).not.toHaveProperty("input");
});

test("filters unsupported modalities without inventing image input", () => {
const doc = buildClientConfig("openclaw", {
...ctx(),
models: [
{ namespaced: "p/mixed", provider: "p", id: "mixed", inputModalities: ["text", "image", "image", "audio", "video", "pdf"] },
{ namespaced: "p/unknown", provider: "p", id: "unknown", inputModalities: [] },
{ namespaced: "p/foreign", provider: "p", id: "foreign", inputModalities: ["pdf"] },
],
}) as OpenclawGeneratedConfig;
const models = doc.models.providers[OPENCODE_PROVIDER_ID]!.models;
expect(models.find(model => model.id === "p/mixed")?.input).toEqual(["text", "image", "audio", "video"]);
expect(models.find(model => model.id === "p/unknown")).not.toHaveProperty("input");
expect(models.find(model => model.id === "p/foreign")).not.toHaveProperty("input");
});

test("merges with the bundled catalog and omits a window it cannot assert", () => {
const doc = buildClientConfig("openclaw", ctx()) as OpenclawGeneratedConfig;
expect(doc.models.mode).toBe("merge");
Expand Down Expand Up @@ -252,9 +275,15 @@ describe("kimi", () => {
expect(doc.providers[OPENCODE_PROVIDER_ID]!.api_key).toBe(LOOPBACK_API_KEY_PLACEHOLDER);
});

test("never emits capabilities it cannot assert", () => {
const { text } = buildClientConfigText("kimi", ctx());
expect(text).not.toContain("capabilities");
test("declares image_in only for catalog-backed image models", () => {
const doc = buildClientConfig("kimi", ctx()) as KimiGeneratedConfig;
expect(doc.models[`${OPENCODE_PROVIDER_ID}/anthropic/claude-opus-4-8`])
.toHaveProperty("capabilities", ["image_in"]);
expect(doc.models[`${OPENCODE_PROVIDER_ID}/gpt-5.5`]).not.toHaveProperty("capabilities");
const unknown = buildClientConfig("kimi", {
...ctx(), models: [{ namespaced: "local/unknown", provider: "local", id: "unknown", contextWindow: 32_000 }],
}) as KimiGeneratedConfig;
expect(unknown.models[`${OPENCODE_PROVIDER_ID}/local/unknown`]).not.toHaveProperty("capabilities");
});

test("KIMI_CODE_HOME wins over the default", () => {
Expand Down
7 changes: 3 additions & 4 deletions tests/config/client-config-new-clients.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -121,11 +121,10 @@ describe("kimi", () => {
expect(doc.providers[OPENCODE_PROVIDER_ID]!.type).toBe("openai");
});

test("never asserts capabilities it cannot know", () => {
test("asserts image input only when the catalog declares it", () => {
const doc = buildClientConfig("kimi", ctx()) as KimiGeneratedConfig;
for (const model of Object.values(doc.models)) {
expect(model).not.toHaveProperty("capabilities");
}
expect(doc.models[kimiModelAlias("anthropic/claude-opus-4-8")]?.capabilities).toEqual(["image_in"]);
expect(doc.models[kimiModelAlias("gpt-5.5")]).not.toHaveProperty("capabilities");
});

test("its document round-trips through the TOML parser", () => {
Expand Down
21 changes: 21 additions & 0 deletions tests/providers/provider-registry-parity.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -752,6 +752,27 @@ describe("provider registry parity", () => {
expect(KEY_LOGIN_PROVIDERS["anthropic-apikey"].modelContextWindows).toEqual(anthropicOauth?.modelContextWindows);
});

test("Anthropic providers seed image input while preserving explicit model overrides", () => {
for (const id of ["anthropic", "anthropic-apikey"]) {
const entry = PROVIDER_REGISTRY.find(entry => entry.id === id)!;
const seed = providerConfigSeed(entry);
expect(entry.models!.length).toBeGreaterThan(0);
for (const model of entry.models!) {
expect(seed.modelInputModalities?.[model]).toEqual(["text", "image"]);
}

const provider: OcxProviderConfig = {
adapter: "anthropic",
baseUrl: "https://api.anthropic.com",
modelInputModalities: { "claude-sonnet-5": ["text"] },
};
enrichProviderFromRegistry(id, provider);
expect(provider.modelInputModalities?.["claude-sonnet-5"]).toEqual(["text"]);
expect(provider.modelInputModalities?.["claude-fable-5-1"]).toEqual(["text", "image"]);
expect(provider.modelInputModalities?.["unknown-model"]).toBeUndefined();
}
});

test("Anthropic providers advertise an effort ladder for every model on both auth flows", () => {
const anthropicOauth = PROVIDER_REGISTRY.find(entry => entry.id === "anthropic");
const apiKey = KEY_LOGIN_PROVIDERS["anthropic-apikey"];
Expand Down
Loading
Loading