diff --git a/src/vision/image-rewrite.ts b/src/vision/image-rewrite.ts new file mode 100644 index 0000000000..9274a9e9cb --- /dev/null +++ b/src/vision/image-rewrite.ts @@ -0,0 +1,106 @@ +import type { OcxContentPart, OcxParsedRequest, OcxTextContent } from "../types"; +import type { TranslatorBudget } from "../lib/translator-budget"; + +export const descriptionEncoder = new TextEncoder(); + +/** A user/developer/toolResult message can carry images (toolResult: e.g. Codex view_image output). */ +export function carriesImages(role: string): boolean { + return role === "user" || role === "developer" || role === "toolResult"; +} + + +const IMAGE_OMITTED_TEXT = "[image omitted: this model is text-only and the vision sidecar is unavailable (no ChatGPT login)]"; + +function isPlainRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +/** + * Keep the native Responses passthrough body aligned with image replacements made in the parsed + * message graph. The passthrough adapter serializes `_rawBody`, while translated adapters serialize + * `context.messages`; updating only the latter would send the original pixels to a text-only + * Responses upstream even after the vision sidecar produced a caption. + * + * Rewrites only image-bearing user/developer messages and tool outputs. All other native Responses + * items (reasoning, calls, ids, compaction, and provider-specific metadata) remain byte-structurally + * untouched. + */ +export function syncRawBodyImageDescriptions(parsed: OcxParsedRequest, descriptions: readonly string[]): void { + const rawBody = parsed._rawBody; + if (!isPlainRecord(rawBody) || !Array.isArray(rawBody.input)) return; + + let nextDescription = 0; + const rewriteImages = (value: unknown, nonEmptyImageUrlsOnly: boolean): unknown => { + if (Array.isArray(value)) { + let changed = false; + const rewritten = value.map(entry => { + const next = rewriteImages(entry, nonEmptyImageUrlsOnly); + if (next !== entry) changed = true; + return next; + }); + return changed ? rewritten : value; + } + if (!isPlainRecord(value)) return value; + if (value.type === "input_image" && typeof value.image_url === "string") { + if (nonEmptyImageUrlsOnly && value.image_url.length === 0) { + return { type: "input_text", text: IMAGE_OMITTED_TEXT }; + } + const description = descriptions[nextDescription++]; + return { type: "input_text", text: description ?? IMAGE_OMITTED_TEXT }; + } + return value; + }; + + let changed = false; + const input = rawBody.input.map(item => { + if (!isPlainRecord(item)) return item; + const type = typeof item.type === "string" ? item.type : (typeof item.role === "string" ? "message" : ""); + const role = typeof item.role === "string" ? item.role : ""; + const isMessageContent = ( + (type === "message" && (role === "user" || role === "developer")) + || type === "agent_message" + ); + const field = isMessageContent + ? "content" + : (type === "function_call_output" || type === "custom_tool_call_output") + ? "output" + : undefined; + if (!field) return item; + const rewritten = rewriteImages(item[field], isMessageContent); + if (rewritten === item[field]) return item; + changed = true; + return { ...item, [field]: rewritten }; + }); + + if (changed) rawBody.input = input; +} + +/** + * Fail-closed image strip for sidecar-covered models when NO sidecar plan exists (no forward + * provider / missing forwarded auth / sidecar disabled): the upstream is text-only, so forwarding + * raw images would 400 or silently confuse it. Replace each image with an explicit marker so the + * model (and the user, via its reply) knows the image was dropped rather than ignored. + */ +export function stripImagesInPlace(parsed: OcxParsedRequest, translatorBudget?: TranslatorBudget): boolean { + let stripped = false; + const descriptions: string[] = []; + for (const msg of parsed.context.messages) { + if (!carriesImages(msg.role) || !Array.isArray(msg.content)) continue; + const parts = msg.content as OcxContentPart[]; + if (!parts.some(p => p.type === "image")) continue; + msg.content = parts.map(p => { + if (p.type !== "image") return p; + const replacement = { type: "text", text: IMAGE_OMITTED_TEXT } as OcxContentPart; + descriptions.push((replacement as OcxTextContent).text); + const reservation = translatorBudget?.reserveTransient( + descriptionEncoder.encode((replacement as OcxTextContent).text).byteLength, + { kind: "request_copies" }, + ); + reservation?.commitRetained(); + return replacement; + }); + stripped = true; + } + syncRawBodyImageDescriptions(parsed, descriptions); + return stripped; +} diff --git a/src/vision/index.ts b/src/vision/index.ts index 3f85258624..5c01a445e3 100644 --- a/src/vision/index.ts +++ b/src/vision/index.ts @@ -1,26 +1,16 @@ import { createHash } from "node:crypto"; -import type { OcxConfig, OcxContentPart, OcxMessage, OcxParsedRequest, OcxProviderConfig, OcxTextContent } from "../types"; -import type { VisionReasoningEffort } from "../reasoning-effort"; -import { describeImage, type DescribeOutcome, type VisionSettings } from "./describe"; +import type { OcxContentPart, OcxMessage, OcxParsedRequest, OcxTextContent } from "../types"; +import { describeImage, type DescribeOutcome } from "./describe"; import { describeImageAnthropic } from "./anthropic-describe"; import { describeImageRouted } from "./routed-describe"; -import { isModelVisionSidecarConsumer as isModelTextOnly, modelAcceptsImageInput } from "./eligibility"; -import { normalizeVisionReasoningForModel } from "./reasoning"; import type { CodexAuthContext } from "../codex/auth-context"; -import { resolveSidecarAuth } from "../sidecar/auth"; -import type { ResolvedOpenAiForwardSidecar } from "../providers/openai-sidecar"; import type { SidecarOutcomeRecorder } from "../web-search/executor"; import { enforceAppOwnedMemoryBudget } from "../lib/app-owned-memory"; import type { TranslatorBudget } from "../lib/translator-budget"; -import { - DEFAULT_VISION_TIMEOUT_MS, - MAX_VISION_TIMEOUT_MS, - MIN_VISION_TIMEOUT_MS, -} from "./timeout-bounds"; +import type { VisionPlan } from "./plan"; +import { carriesImages, descriptionEncoder, syncRawBodyImageDescriptions } from "./image-rewrite"; export { describeImage } from "./describe"; - -/** Backward-compatible request-time name for the shared vision-sidecar consumer predicate. */ export { isModelVisionSidecarConsumer as isModelTextOnly } from "./eligibility"; export { describeImageAnthropic, parseAnthropicVisionSSE } from "./anthropic-describe"; export { @@ -33,19 +23,25 @@ export { visionEligibleModelOptions, } from "./eligibility"; export type { VisionCandidateModel, VisionModelOption, VisionSidecarBackend } from "./eligibility"; +export { DEFAULT_VISION_TIMEOUT_MS, MAX_VISION_TIMEOUT_MS, MIN_VISION_TIMEOUT_MS } from "./timeout-bounds"; export { - DEFAULT_VISION_TIMEOUT_MS, - MAX_VISION_TIMEOUT_MS, - MIN_VISION_TIMEOUT_MS, -}; + DEFAULT_MAX_DESCRIPTIONS_PER_TURN, + resolveMaxDescriptionsPerTurn, + isValidVisionTimeoutMs, + resolveVisionTimeoutMs, + findAnthropicVisionProvider, + resolveVisionBackend, + resolveOpenAiVisionModel, + resolveEffectiveVisionModel, + shouldResolveOpenAiVisionSidecar, + planVisionSidecar, +} from "./plan"; +export type { AnthropicVisionProvider, VisionPlan } from "./plan"; +export { stripImagesInPlace } from "./image-rewrite"; + -const DEFAULT_VISION_MODEL = "gpt-5.4-mini"; -const DEFAULT_ANTHROPIC_VISION_MODEL = "claude-sonnet-5"; -const DEFAULT_REASONING: VisionReasoningEffort = "low"; -export const DEFAULT_MAX_DESCRIPTIONS_PER_TURN = 8; const DESCRIPTION_CACHE_MAX_ENTRIES = 256; export const VISION_DESCRIPTION_CACHE_MAX_BYTES = 1024 * 1024; -const descriptionEncoder = new TextEncoder(); /** Max images described in parallel — keeps first-token latency bounded without flooding the backend. */ const VISION_CONCURRENCY = 3; /** Per-image description hard cap (chars) so multi-image turns can't blow the main model's context. */ @@ -160,25 +156,6 @@ export function evictOldestVisionDescriptionForBudget(): number { return descriptionCache.evictOldest?.() ?? 0; } -/** Runtime config is permissive: zero is intentional; malformed values fall back to the bounded default. */ -export function resolveMaxDescriptionsPerTurn(value: unknown): number { - if (value === 0) return 0; - return typeof value === "number" && Number.isInteger(value) && value > 0 - ? value - : DEFAULT_MAX_DESCRIPTIONS_PER_TURN; -} - -export function isValidVisionTimeoutMs(value: unknown): value is number { - return typeof value === "number" - && Number.isInteger(value) - && value >= MIN_VISION_TIMEOUT_MS - && value <= MAX_VISION_TIMEOUT_MS; -} - -/** Runtime config is permissive: malformed or out-of-range values fall back to the default. */ -export function resolveVisionTimeoutMs(value: unknown): number { - return isValidVisionTimeoutMs(value) ? value : DEFAULT_VISION_TIMEOUT_MS; -} /** Run `worker` over `items` with bounded concurrency, preserving input order in the result array. */ async function runBounded(items: T[], limit: number, worker: (item: T) => Promise): Promise { @@ -198,176 +175,7 @@ function clamp(s: string, max: number): string { return s.length <= max ? s : `${s.slice(0, max)}\n…[description truncated]`; } -export interface AnthropicVisionProvider { - providerName: string; - provider: OcxProviderConfig; -} - -/** - * First enabled Anthropic OAuth provider whose active stored account is not marked for reauth. - * Delegates to the shared sidecar auth module (#2188) — same predicate as web-search. - */ -export function findAnthropicVisionProvider(config: OcxConfig): AnthropicVisionProvider | undefined { - const auth = resolveSidecarAuth(config); - if (!auth.isAnthropicAuth || !auth.anthropicProviderName || !auth.anthropicProvider) return undefined; - return { providerName: auth.anthropicProviderName, provider: auth.anthropicProvider }; -} -export function resolveVisionBackend( - explicit: "openai" | "anthropic" | "routed" | undefined, - anthropicSidecar: AnthropicVisionProvider | undefined, -): "openai" | "anthropic" { - if (explicit === "openai" || explicit === "anthropic") return explicit; - // "routed" collapses to the legacy default order until its describe executor - // lands (roadmap 170 → 180 revised): a persisted routed backend without a - // dispatchable arm degrades exactly like unset rather than crashing. wp3 - // replaces this collapse with the real routed arm in planVisionSidecar. - return anthropicSidecar ? "anthropic" : "openai"; -} - -/** Native model used by the OpenAI vision helper, including its bounded default. */ -export function resolveOpenAiVisionModel(config: Pick): string { - const configured = config.visionSidecar?.model; - // Namespaced routed ids never reach the forward executor (see - // resolveEffectiveVisionModel). - return configured && !configured.includes("/") ? configured : DEFAULT_VISION_MODEL; -} - -/** Effective describer model for the backend `planVisionSidecar` selected. */ -export function resolveEffectiveVisionModel( - config: Pick, - backend: "openai" | "anthropic", -): string { - const configured = config.visionSidecar?.model; - // A namespaced "provider/model" id belongs to the routed backend only; the - // forward/OAuth executors POST the model string verbatim, so it falls back - // to the side's default here (PUT coherence rejects new writes of this - // shape, but a legacy or hand-edited config must not break the executor). - const usable = configured && !configured.includes("/") ? configured : undefined; - return backend === "anthropic" - ? usable || DEFAULT_ANTHROPIC_VISION_MODEL - : usable || DEFAULT_VISION_MODEL; -} - -/** A user/developer/toolResult message can carry images (toolResult: e.g. Codex view_image output). */ -function carriesImages(role: string): boolean { - return role === "user" || role === "developer" || role === "toolResult"; -} - -function messagesHaveImage(parsed: OcxParsedRequest): boolean { - return parsed.context.messages.some(m => - carriesImages(m.role) && Array.isArray(m.content) && (m.content as OcxContentPart[]).some(p => p.type === "image")); -} - -export function shouldResolveOpenAiVisionSidecar( - config: OcxConfig, - provider: OcxProviderConfig, - modelId: string, - parsed: OcxParsedRequest, -): boolean { - if (!isModelTextOnly(provider, modelId) || !messagesHaveImage(parsed)) return false; - const cfg = config.visionSidecar ?? {}; - if (cfg.enabled === false) return false; - return resolveVisionBackend(cfg.backend, findAnthropicVisionProvider(config)) === "openai"; -} - -export interface VisionPlan { - backend: "openai" | "anthropic" | "routed"; - forwardSidecar?: ResolvedOpenAiForwardSidecar; - anthropicSidecar?: AnthropicVisionProvider; - /** Namespaced "provider/model" describer for the routed backend (roadmap 180). */ - routedModel?: string; - /** Loopback dispatch inputs for the routed backend. */ - routedConfig?: Pick; - settings: VisionSettings; - maxDescriptionsPerTurn: number; -} - -/** - * Decide whether the vision sidecar should pre-describe images for this request, returning the plan - * if so. Active when: the routed model is in `provider.noVisionModels`, the request actually carries - * an image, the sidecar isn't disabled, and the selected backend has usable auth. Returns undefined - * otherwise (the caller strips images before sending to a text-only model). - */ -export function planVisionSidecar( - config: OcxConfig, - provider: OcxProviderConfig, - modelId: string, - parsed: OcxParsedRequest, - openAiSidecar?: ResolvedOpenAiForwardSidecar, -): VisionPlan | undefined { - if (!isModelTextOnly(provider, modelId)) return undefined; - if (!messagesHaveImage(parsed)) return undefined; - const cfg = config.visionSidecar ?? {}; - if (cfg.enabled === false) return undefined; - - // Routed arm (roadmap 180 revised): explicit backend + NAMESPACED explicit - // model only — never inferred from credential availability. Plan-time - // fence: the target must not be provably blind, and must not itself be a - // model this planner would re-enter for (belt; the terminal marker on the - // loopback request is the braces). - if (cfg.backend === "routed") { - const routedModel = cfg.model; - const sep = routedModel ? routedModel.indexOf("/") : -1; - if (routedModel && sep > 0) { - const targetProvider = routedModel.slice(0, sep); - const targetId = routedModel.slice(sep + 1); - const targetProviderConfig = config.providers?.[targetProvider]; - const targetVisible = modelAcceptsImageInput(config, { provider: targetProvider, id: targetId }) !== false - && !(targetProviderConfig && isModelTextOnly(targetProviderConfig, targetId)); - if (targetVisible) { - return { - backend: "routed", - routedModel, - routedConfig: { port: config.port, ...(config.apiKeys ? { apiKeys: config.apiKeys } : {}) }, - settings: { - model: routedModel, - reasoning: DEFAULT_REASONING, - timeoutMs: resolveVisionTimeoutMs(cfg.timeoutMs), - }, - maxDescriptionsPerTurn: resolveMaxDescriptionsPerTurn(cfg.maxDescriptionsPerTurn), - }; - } - } - // Misconfigured routed backend (bare id, unknown provider, or provably - // blind target): fall through to the legacy default order below rather - // than dispatching a describe that cannot work. - } - - const anthropicSidecar = findAnthropicVisionProvider(config); - const backend = resolveVisionBackend(cfg.backend, anthropicSidecar); - // A namespaced routed model must never reach the forward/OAuth executors - // (they POST the string verbatim); the effective-model resolver falls back - // to each side's default in that case. - const model = resolveEffectiveVisionModel(config, backend); - const maxDescriptionsPerTurn = resolveMaxDescriptionsPerTurn(cfg.maxDescriptionsPerTurn); - - if (backend === "anthropic") { - if (!anthropicSidecar) return undefined; - return { - backend, - anthropicSidecar, - settings: { - model, - reasoning: normalizeVisionReasoningForModel(model, cfg.reasoning) ?? DEFAULT_REASONING, - timeoutMs: resolveVisionTimeoutMs(cfg.timeoutMs), - }, - maxDescriptionsPerTurn, - }; - } - - if (!openAiSidecar) return undefined; - return { - backend, - forwardSidecar: openAiSidecar, - settings: { - model, - reasoning: normalizeVisionReasoningForModel(model, cfg.reasoning) ?? DEFAULT_REASONING, - timeoutMs: resolveVisionTimeoutMs(cfg.timeoutMs), - }, - maxDescriptionsPerTurn, - }; -} interface ImageJob { imageUrl: string; @@ -385,71 +193,6 @@ function renderDescription(out: { text: string; error?: string }): OcxTextConten }; } -const IMAGE_OMITTED_TEXT = "[image omitted: this model is text-only and the vision sidecar is unavailable (no ChatGPT login)]"; - -function isPlainRecord(value: unknown): value is Record { - return typeof value === "object" && value !== null && !Array.isArray(value); -} - -/** - * Keep the native Responses passthrough body aligned with image replacements made in the parsed - * message graph. The passthrough adapter serializes `_rawBody`, while translated adapters serialize - * `context.messages`; updating only the latter would send the original pixels to a text-only - * Responses upstream even after the vision sidecar produced a caption. - * - * Rewrites only image-bearing user/developer messages and tool outputs. All other native Responses - * items (reasoning, calls, ids, compaction, and provider-specific metadata) remain byte-structurally - * untouched. - */ -function syncRawBodyImageDescriptions(parsed: OcxParsedRequest, descriptions: readonly string[]): void { - const rawBody = parsed._rawBody; - if (!isPlainRecord(rawBody) || !Array.isArray(rawBody.input)) return; - - let nextDescription = 0; - const rewriteImages = (value: unknown, nonEmptyImageUrlsOnly: boolean): unknown => { - if (Array.isArray(value)) { - let changed = false; - const rewritten = value.map(entry => { - const next = rewriteImages(entry, nonEmptyImageUrlsOnly); - if (next !== entry) changed = true; - return next; - }); - return changed ? rewritten : value; - } - if (!isPlainRecord(value)) return value; - if (value.type === "input_image" && typeof value.image_url === "string") { - if (nonEmptyImageUrlsOnly && value.image_url.length === 0) { - return { type: "input_text", text: IMAGE_OMITTED_TEXT }; - } - const description = descriptions[nextDescription++]; - return { type: "input_text", text: description ?? IMAGE_OMITTED_TEXT }; - } - return value; - }; - - let changed = false; - const input = rawBody.input.map(item => { - if (!isPlainRecord(item)) return item; - const type = typeof item.type === "string" ? item.type : (typeof item.role === "string" ? "message" : ""); - const role = typeof item.role === "string" ? item.role : ""; - const isMessageContent = ( - (type === "message" && (role === "user" || role === "developer")) - || type === "agent_message" - ); - const field = isMessageContent - ? "content" - : (type === "function_call_output" || type === "custom_tool_call_output") - ? "output" - : undefined; - if (!field) return item; - const rewritten = rewriteImages(item[field], isMessageContent); - if (rewritten === item[field]) return item; - changed = true; - return { ...item, [field]: rewritten }; - }); - - if (changed) rawBody.input = input; -} function sha256(value: string | Uint8Array): string { return createHash("sha256").update(value).digest("hex"); @@ -636,32 +379,3 @@ export async function describeImagesInPlace( syncRawBodyImageDescriptions(parsed, descriptions); } -/** - * Fail-closed image strip for sidecar-covered models when NO sidecar plan exists (no forward - * provider / missing forwarded auth / sidecar disabled): the upstream is text-only, so forwarding - * raw images would 400 or silently confuse it. Replace each image with an explicit marker so the - * model (and the user, via its reply) knows the image was dropped rather than ignored. - */ -export function stripImagesInPlace(parsed: OcxParsedRequest, translatorBudget?: TranslatorBudget): boolean { - let stripped = false; - const descriptions: string[] = []; - for (const msg of parsed.context.messages) { - if (!carriesImages(msg.role) || !Array.isArray(msg.content)) continue; - const parts = msg.content as OcxContentPart[]; - if (!parts.some(p => p.type === "image")) continue; - msg.content = parts.map(p => { - if (p.type !== "image") return p; - const replacement = { type: "text", text: IMAGE_OMITTED_TEXT } as OcxContentPart; - descriptions.push((replacement as OcxTextContent).text); - const reservation = translatorBudget?.reserveTransient( - descriptionEncoder.encode((replacement as OcxTextContent).text).byteLength, - { kind: "request_copies" }, - ); - reservation?.commitRetained(); - return replacement; - }); - stripped = true; - } - syncRawBodyImageDescriptions(parsed, descriptions); - return stripped; -} diff --git a/src/vision/plan.ts b/src/vision/plan.ts new file mode 100644 index 0000000000..90e1e336dd --- /dev/null +++ b/src/vision/plan.ts @@ -0,0 +1,200 @@ +import type { OcxConfig, OcxContentPart, OcxParsedRequest, OcxProviderConfig } from "../types"; +import type { VisionReasoningEffort } from "../reasoning-effort"; +import type { VisionSettings } from "./describe"; +import type { ResolvedOpenAiForwardSidecar } from "../providers/openai-sidecar"; +import { isModelVisionSidecarConsumer as isModelTextOnly, modelAcceptsImageInput } from "./eligibility"; +import { normalizeVisionReasoningForModel } from "./reasoning"; +import { resolveSidecarAuth } from "../sidecar/auth"; +import { DEFAULT_VISION_TIMEOUT_MS, MAX_VISION_TIMEOUT_MS, MIN_VISION_TIMEOUT_MS } from "./timeout-bounds"; +import { carriesImages } from "./image-rewrite"; + +const DEFAULT_VISION_MODEL = "gpt-5.4-mini"; +const DEFAULT_ANTHROPIC_VISION_MODEL = "claude-sonnet-5"; +const DEFAULT_REASONING: VisionReasoningEffort = "low"; +export const DEFAULT_MAX_DESCRIPTIONS_PER_TURN = 8; + +/** Runtime config is permissive: zero is intentional; malformed values fall back to the bounded default. */ +export function resolveMaxDescriptionsPerTurn(value: unknown): number { + if (value === 0) return 0; + return typeof value === "number" && Number.isInteger(value) && value > 0 + ? value + : DEFAULT_MAX_DESCRIPTIONS_PER_TURN; +} + +export function isValidVisionTimeoutMs(value: unknown): value is number { + return typeof value === "number" + && Number.isInteger(value) + && value >= MIN_VISION_TIMEOUT_MS + && value <= MAX_VISION_TIMEOUT_MS; +} + +/** Runtime config is permissive: malformed or out-of-range values fall back to the default. */ +export function resolveVisionTimeoutMs(value: unknown): number { + return isValidVisionTimeoutMs(value) ? value : DEFAULT_VISION_TIMEOUT_MS; +} + +export interface AnthropicVisionProvider { + providerName: string; + provider: OcxProviderConfig; +} + +/** + * First enabled Anthropic OAuth provider whose active stored account is not marked for reauth. + * Delegates to the shared sidecar auth module (#2188) — same predicate as web-search. + */ +export function findAnthropicVisionProvider(config: OcxConfig): AnthropicVisionProvider | undefined { + const auth = resolveSidecarAuth(config); + if (!auth.isAnthropicAuth || !auth.anthropicProviderName || !auth.anthropicProvider) return undefined; + return { providerName: auth.anthropicProviderName, provider: auth.anthropicProvider }; +} + +export function resolveVisionBackend( + explicit: "openai" | "anthropic" | "routed" | undefined, + anthropicSidecar: AnthropicVisionProvider | undefined, +): "openai" | "anthropic" { + if (explicit === "openai" || explicit === "anthropic") return explicit; + // "routed" collapses to the legacy default order until its describe executor + // lands (roadmap 170 → 180 revised): a persisted routed backend without a + // dispatchable arm degrades exactly like unset rather than crashing. wp3 + // replaces this collapse with the real routed arm in planVisionSidecar. + return anthropicSidecar ? "anthropic" : "openai"; +} + +/** Native model used by the OpenAI vision helper, including its bounded default. */ +export function resolveOpenAiVisionModel(config: Pick): string { + const configured = config.visionSidecar?.model; + // Namespaced routed ids never reach the forward executor (see + // resolveEffectiveVisionModel). + return configured && !configured.includes("/") ? configured : DEFAULT_VISION_MODEL; +} + +/** Effective describer model for the backend `planVisionSidecar` selected. */ +export function resolveEffectiveVisionModel( + config: Pick, + backend: "openai" | "anthropic", +): string { + const configured = config.visionSidecar?.model; + // A namespaced "provider/model" id belongs to the routed backend only; the + // forward/OAuth executors POST the model string verbatim, so it falls back + // to the side's default here (PUT coherence rejects new writes of this + // shape, but a legacy or hand-edited config must not break the executor). + const usable = configured && !configured.includes("/") ? configured : undefined; + return backend === "anthropic" + ? usable || DEFAULT_ANTHROPIC_VISION_MODEL + : usable || DEFAULT_VISION_MODEL; +} + +function messagesHaveImage(parsed: OcxParsedRequest): boolean { + return parsed.context.messages.some(m => + carriesImages(m.role) && Array.isArray(m.content) && (m.content as OcxContentPart[]).some(p => p.type === "image")); +} + +export function shouldResolveOpenAiVisionSidecar( + config: OcxConfig, + provider: OcxProviderConfig, + modelId: string, + parsed: OcxParsedRequest, +): boolean { + if (!isModelTextOnly(provider, modelId) || !messagesHaveImage(parsed)) return false; + const cfg = config.visionSidecar ?? {}; + if (cfg.enabled === false) return false; + return resolveVisionBackend(cfg.backend, findAnthropicVisionProvider(config)) === "openai"; +} + +export interface VisionPlan { + backend: "openai" | "anthropic" | "routed"; + forwardSidecar?: ResolvedOpenAiForwardSidecar; + anthropicSidecar?: AnthropicVisionProvider; + /** Namespaced "provider/model" describer for the routed backend (roadmap 180). */ + routedModel?: string; + /** Loopback dispatch inputs for the routed backend. */ + routedConfig?: Pick; + settings: VisionSettings; + maxDescriptionsPerTurn: number; +} + +/** + * Decide whether the vision sidecar should pre-describe images for this request, returning the plan + * if so. Active when: the routed model is in `provider.noVisionModels`, the request actually carries + * an image, the sidecar isn't disabled, and the selected backend has usable auth. Returns undefined + * otherwise (the caller strips images before sending to a text-only model). + */ +export function planVisionSidecar( + config: OcxConfig, + provider: OcxProviderConfig, + modelId: string, + parsed: OcxParsedRequest, + openAiSidecar?: ResolvedOpenAiForwardSidecar, +): VisionPlan | undefined { + if (!isModelTextOnly(provider, modelId)) return undefined; + if (!messagesHaveImage(parsed)) return undefined; + const cfg = config.visionSidecar ?? {}; + if (cfg.enabled === false) return undefined; + + // Routed arm (roadmap 180 revised): explicit backend + NAMESPACED explicit + // model only — never inferred from credential availability. Plan-time + // fence: the target must not be provably blind, and must not itself be a + // model this planner would re-enter for (belt; the terminal marker on the + // loopback request is the braces). + if (cfg.backend === "routed") { + const routedModel = cfg.model; + const sep = routedModel ? routedModel.indexOf("/") : -1; + if (routedModel && sep > 0) { + const targetProvider = routedModel.slice(0, sep); + const targetId = routedModel.slice(sep + 1); + const targetProviderConfig = config.providers?.[targetProvider]; + const targetVisible = modelAcceptsImageInput(config, { provider: targetProvider, id: targetId }) !== false + && !(targetProviderConfig && isModelTextOnly(targetProviderConfig, targetId)); + if (targetVisible) { + return { + backend: "routed", + routedModel, + routedConfig: { port: config.port, ...(config.apiKeys ? { apiKeys: config.apiKeys } : {}) }, + settings: { + model: routedModel, + reasoning: DEFAULT_REASONING, + timeoutMs: resolveVisionTimeoutMs(cfg.timeoutMs), + }, + maxDescriptionsPerTurn: resolveMaxDescriptionsPerTurn(cfg.maxDescriptionsPerTurn), + }; + } + } + // Misconfigured routed backend (bare id, unknown provider, or provably + // blind target): fall through to the legacy default order below rather + // than dispatching a describe that cannot work. + } + + const anthropicSidecar = findAnthropicVisionProvider(config); + const backend = resolveVisionBackend(cfg.backend, anthropicSidecar); + // A namespaced routed model must never reach the forward/OAuth executors + // (they POST the string verbatim); the effective-model resolver falls back + // to each side's default in that case. + const model = resolveEffectiveVisionModel(config, backend); + const maxDescriptionsPerTurn = resolveMaxDescriptionsPerTurn(cfg.maxDescriptionsPerTurn); + + if (backend === "anthropic") { + if (!anthropicSidecar) return undefined; + return { + backend, + anthropicSidecar, + settings: { + model, + reasoning: normalizeVisionReasoningForModel(model, cfg.reasoning) ?? DEFAULT_REASONING, + timeoutMs: resolveVisionTimeoutMs(cfg.timeoutMs), + }, + maxDescriptionsPerTurn, + }; + } + + if (!openAiSidecar) return undefined; + return { + backend, + forwardSidecar: openAiSidecar, + settings: { + model, + reasoning: normalizeVisionReasoningForModel(model, cfg.reasoning) ?? DEFAULT_REASONING, + timeoutMs: resolveVisionTimeoutMs(cfg.timeoutMs), + }, + maxDescriptionsPerTurn, + }; +} diff --git a/tests/vision/vision-cache.test.ts b/tests/vision/vision-cache.test.ts index dce18586f0..dda3a48ddf 100644 --- a/tests/vision/vision-cache.test.ts +++ b/tests/vision/vision-cache.test.ts @@ -356,3 +356,16 @@ describe("vision description cache and per-turn cap", () => { expect(visionDescriptionRetainedStoreSnapshot().bytes).toBe(before.bytes - released); }); }); + +test("vision planning and image-rewrite seams preserve boundary identity and dependency direction", async () => { + const boundary = await import("../../src/vision"); + const planning = await import("../../src/vision/plan"); + const rewrite = await import("../../src/vision/image-rewrite"); + const { readFileSync } = await import("node:fs"); + const { repoPath } = await import("../helpers/repo-root"); + + expect(boundary.resolveMaxDescriptionsPerTurn).toBe(planning.resolveMaxDescriptionsPerTurn); + expect(boundary.stripImagesInPlace).toBe(rewrite.stripImagesInPlace); + expect(readFileSync(repoPath("src/vision/image-rewrite.ts"), "utf8")).not.toMatch(/from\s+["']\.\/(plan|index)["']/); + expect(readFileSync(repoPath("src/vision/plan.ts"), "utf8")).not.toMatch(/from\s+["']\.\/index["']/); +});