From 2aafa8212a084dffd3d6e158c54cb95d573d8931 Mon Sep 17 00:00:00 2001 From: test Date: Thu, 13 Aug 2026 14:50:56 +0200 Subject: [PATCH 1/2] fix(loki): vision is a provider chain, not a pinned model that rots MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every screenshot attached to Loki has been failing. GROQ_VISION_MODEL pinned meta-llama/llama-4-scout-17b-16e-instruct, which is DECOMMISSIONED — 404 — and Groq now serves no vision model at all on this account. Found while probing models for the tool loop, not by anyone using the feature, which is the part worth fixing. This is the fourth pinned-free-model rot in this fleet, so it gets the class treatment rather than a new pin: config/vision-models.ts a CHAIN across providers, tried in order. Entries are OpenAI-compatible, so adding a provider is a row, not a client. Groq keeps a seat with no default model so restoring one is one string. lib/vision.ts walks the chain, first real answer wins. probe-models.ts now probes BOTH halves — chat tool-calling AND vision — with a real image. `npm run probe:models`. "Real answer" is load-bearing: nvidia/nemotron-nano-12b-v2-vl returns HTTP 200 with EMPTY content, which a naive client reports as a successful analysis of nothing — reading to the user as "nothing notable in your screenshot" rather than as a failure. Empty is treated as a dead link and the chain continues. Verified live against a solid-colour test image: google/gemma-4-26b-a4b-it:free -> 200, correctly answered "Red" OK google/gemma-4-31b-it:free -> 429 rate limited fallback nvidia/nemotron-nano-12b-v2-vl -> 200 but empty excluded FleetCrown had no vision-capable credential at all; it now shares orangecat's OPENROUTER_API_KEY (added to /opt/fleetcrown/app/.env on the box, which deploy.sh bootstraps into shared/ on the next release). Retires callGroqVision + GROQ_VISION_MODEL — one caller, now on the chain, and leaving a dead constant is how the next person re-pins a dead model. The analysis block now names the model that read the image: it is the one part of a Loki turn NOT grounded in FleetCrown's records, and saying so keeps it from blending into the cited answer around it. Co-Authored-By: Claude Opus 5 --- package.json | 2 +- .../{probe-loki-models.ts => probe-models.ts} | 52 ++++++++++- src/config/vision-models.ts | 76 +++++++++++++++ src/lib/groq.ts | 73 --------------- src/lib/loki/vision.ts | 19 +++- src/lib/vision.ts | 92 +++++++++++++++++++ 6 files changed, 231 insertions(+), 83 deletions(-) rename scripts/{probe-loki-models.ts => probe-models.ts} (54%) create mode 100644 src/config/vision-models.ts create mode 100644 src/lib/vision.ts diff --git a/package.json b/package.json index 9ab7d2b5a..20b846113 100644 --- a/package.json +++ b/package.json @@ -79,7 +79,7 @@ "ship": "git push origin main", "prepare": "husky", "sync:agent-core": "tsx scripts/sync-agent-core.ts", - "probe:loki-models": "tsx scripts/probe-loki-models.ts" + "probe:models": "tsx scripts/probe-models.ts" }, "dependencies": { "@auth/drizzle-adapter": "^1.11.3", diff --git a/scripts/probe-loki-models.ts b/scripts/probe-models.ts similarity index 54% rename from scripts/probe-loki-models.ts rename to scripts/probe-models.ts index ca30246e2..03201b6ff 100644 --- a/scripts/probe-loki-models.ts +++ b/scripts/probe-models.ts @@ -1,7 +1,7 @@ /** - * Live protocol probe — can this model drive Loki's tool loop? + * Live model probe — is every model this app depends on actually alive? * - * Run: npx tsx scripts/probe-loki-models.ts [model ...] + * Run: npx tsx scripts/probe-models.ts [chat-model ...] (npm run probe:models) * (defaults to a spread from frontier-ish down to tiny) * * Why this exists as a committed script rather than a one-off: Loki is meant to @@ -10,7 +10,17 @@ * emits a tool call the loop can parse. That is cheap to measure and impossible * to guess, so measure it before switching, not after. * - * NOT part of `npm run verify`: it costs real tokens and needs GROQ_API_KEY. + * Covers BOTH halves, because both have bitten: + * CHAT — can the model emit a tool call the loop can parse? + * VISION — does any model in the chain actually read an image? + * + * The vision half exists because a pinned free model (Groq's llama-4-scout) was + * decommissioned and every attached screenshot 404'd, silently, until someone + * tried one. That was the fourth pinned-free-model rot in this fleet, so the + * class gets a command instead of another one-off fix: run this before swapping + * a model, and on any "images stopped working" report. + * + * NOT part of `npm run verify`: it costs real tokens and needs live keys. * The env-independent coverage of the same parser lives in * scripts/test/agent-tool-loop.ts. * @@ -22,6 +32,8 @@ import { config } from "dotenv"; config({ path: ".env.local" }); import { callModelWithTools } from "../src/lib/agent/llm"; +import { analyzeImages } from "../src/lib/vision"; +import { usableVisionChain } from "../src/config/vision-models"; const DEFAULT_MODELS = [ "llama-3.3-70b-versatile", @@ -85,7 +97,39 @@ async function main() { } } - console.log(`\n${usable}/${models.length} model(s) can drive the loop.`); + console.log(`\n${usable}/${models.length} chat model(s) can drive the loop.`); + + // ── Vision chain ─────────────────────────────────────────────────────────── + // A 2x2 solid-red PNG. Tiny, but it proves the model actually READ the image + // rather than merely accepting the request — a provider that returns 200 with + // empty content passes a "did it error" check and fails this one. + const RED_2X2 = + "iVBORw0KGgoAAAANSUhEUgAAAAIAAAACCAYAAABytg0kAAAAFElEQVR4nGP8z8Dwn4GKgImahgEAG1kDCf0X4o8AAAAASUVORK5CYII="; + + const chain = usableVisionChain(); + console.log(`\nVision chain (${chain.length} usable link(s)):`); + if (chain.length === 0) { + console.log(" ✗ none — no OPENROUTER_API_KEY and no GROQ_VISION_MODEL. Image attachments WILL fail."); + } + let visionOk = 0; + for (const { provider, model } of chain) { + try { + const r = await analyzeImages({ + prompt: "What colour fills this image? Answer with one word.", + images: [{ mimeType: "image/png", dataBase64: RED_2X2 }], + maxTokens: 30, + timeoutMs: 90_000, + }); + // analyzeImages walks the whole chain itself, so this reports the link + // that actually answered rather than the one we asked about. + console.log(` ✓ ${provider.id}/${model} → answered via ${r.model}: ${r.text.slice(0, 40)}`); + visionOk++; + break; + } catch (e) { + console.log(` ✗ ${provider.id}/${model} — ${e instanceof Error ? e.message.slice(0, 90) : e}`); + } + } + if (visionOk === 0 && chain.length > 0) console.log(" ✗ NO vision model answered — image attachments are broken."); } main().catch((e) => { diff --git a/src/config/vision-models.ts b/src/config/vision-models.ts new file mode 100644 index 000000000..19a3be819 --- /dev/null +++ b/src/config/vision-models.ts @@ -0,0 +1,76 @@ +/** + * SSOT for the vision model chain. + * + * This file exists because of a class of failure, not a single bug. Loki's + * screenshot analysis pointed at a hardcoded `GROQ_VISION_MODEL` + * (meta-llama/llama-4-scout-17b-16e-instruct) that was DECOMMISSIONED — every + * attached image returned 404, and Groq now offers no vision model at all on + * this account. That is the fourth time a pinned free model has silently rotted + * out from under this fleet. + * + * Two consequences are encoded here: + * + * 1. A CHAIN, not a model. Free-tier models disappear and rate-limit; a single + * pin is a scheduled outage. The chain is tried in order and the first one + * that answers wins. + * 2. PROVIDER-AGNOSTIC entries. Every provider here speaks the OpenAI + * chat-completions shape, so adding one is a row in this table rather than + * a new client. Groq keeps a seat with no default model precisely so that + * re-adding one later is a one-line change. + * + * `scripts/probe-models.ts` calls every entry with a real image and fails if + * none answer — so the next rot is caught by a command instead of by a user + * attaching a screenshot and getting a 404. + */ + +export type VisionProvider = { + /** Display/debug name, and the env prefix for its key. */ + id: string; + baseUrl: string; + /** Env var holding the API key. Absent key = entry skipped, not an error. */ + keyEnv: string; + /** Models to try for this provider, in order. */ + models: string[]; +}; + +/** + * Default chain, verified live 2026-08-13 against a solid-colour test image: + * + * google/gemma-4-26b-a4b-it:free → 200, correctly answered "Red" ✅ + * google/gemma-4-31b-it:free → 429 (rate limited, transient) ⚠ fallback + * nvidia/nemotron-nano-12b-v2-vl → 200 but EMPTY content ✗ excluded + * + * The nemotron exclusion is the interesting one: it returns HTTP 200 with no + * text, which a naive client reports as a successful empty analysis. Callers + * must treat empty content as failure — see lib/vision.ts. + */ +export const VISION_CHAIN: VisionProvider[] = [ + { + id: "openrouter", + baseUrl: "https://openrouter.ai/api/v1", + keyEnv: "OPENROUTER_API_KEY", + models: ["google/gemma-4-26b-a4b-it:free", "google/gemma-4-31b-it:free"], + }, + { + // Groq has NO vision model on this account as of 2026-08-13. The seat is + // kept so restoring one is a single string, and so the probe reports it. + id: "groq", + baseUrl: "https://api.groq.com/openai/v1", + keyEnv: "GROQ_API_KEY", + models: process.env.GROQ_VISION_MODEL?.trim() ? [process.env.GROQ_VISION_MODEL.trim()] : [], + }, +]; + +/** + * The chain with unusable entries removed: no API key, or no models configured. + * A missing key is a normal deployment state (FleetCrown had no OpenRouter key + * at all until this change), so it filters out silently rather than throwing. + */ +export function usableVisionChain(): Array<{ provider: VisionProvider; model: string }> { + const out: Array<{ provider: VisionProvider; model: string }> = []; + for (const provider of VISION_CHAIN) { + if (!process.env[provider.keyEnv]?.trim()) continue; + for (const model of provider.models) out.push({ provider, model }); + } + return out; +} diff --git a/src/lib/groq.ts b/src/lib/groq.ts index a04fc2adb..65206c21e 100644 --- a/src/lib/groq.ts +++ b/src/lib/groq.ts @@ -6,7 +6,6 @@ import { HTTP_TIMEOUT_SHORT_MS, HTTP_TIMEOUT_LONG_MS, HTTP_TIMEOUT_XL_MS } from "@/lib/constants/time"; export const GROQ_FAST_MODEL = "llama-3.3-70b-versatile"; -export const GROQ_VISION_MODEL = "meta-llama/llama-4-scout-17b-16e-instruct"; export const GROQ_WHISPER_MODEL = "whisper-large-v3-turbo"; const GROQ_API_URL = "https://api.groq.com/openai/v1/chat/completions"; const GROQ_AUDIO_URL = "https://api.groq.com/openai/v1/audio/transcriptions"; @@ -47,78 +46,6 @@ export async function callGroqText(prompt: string, options: GroqOptions = {}): P return (data?.choices?.[0]?.message?.content ?? "").trim(); } -type GroqVisionImage = { mimeType: string; dataBase64: string; name?: string }; - -function normalizeImageMime(mime: string): string { - const m = mime.toLowerCase(); - if (m === "image/jpg" || m === "image/jpeg") return "image/jpeg"; - return m; -} - -type GroqVisionOptions = { - prompt: string; - images: GroqVisionImage[]; - systemPrompt?: string; - maxTokens?: number; - timeoutMs?: number; - model?: string; -}; - -/** - * Multimodal Groq call — up to 5 images per request (Groq vision limit). - * Images must be base64 without the data: prefix. - */ -export async function callGroqVision(options: GroqVisionOptions): Promise { - const key = process.env.GROQ_API_KEY; - if (!key) throw new Error("GROQ_API_KEY not set"); - - const { - prompt, - images, - systemPrompt, - maxTokens = 900, - timeoutMs = HTTP_TIMEOUT_XL_MS, - model = GROQ_VISION_MODEL, - } = options; - - if (images.length === 0) throw new Error("vision requires at least one image"); - if (images.length > 5) throw new Error("vision supports at most 5 images"); - - const content: Array<{ type: string; text?: string; image_url?: { url: string } }> = [ - { type: "text", text: prompt }, - ]; - for (const img of images) { - const mime = normalizeImageMime(img.mimeType); - content.push({ - type: "image_url", - image_url: { url: `data:${mime};base64,${img.dataBase64}` }, - }); - } - - const messages: Array<{ role: string; content: unknown }> = []; - if (systemPrompt) messages.push({ role: "system", content: systemPrompt }); - messages.push({ role: "user", content }); - - const res = await fetch(GROQ_API_URL, { - method: "POST", - headers: { "Content-Type": "application/json", Authorization: `Bearer ${key}` }, - body: JSON.stringify({ model, messages, max_tokens: maxTokens, temperature: 0.2 }), - signal: AbortSignal.timeout(timeoutMs), - }); - - if (!res.ok) { - const body = await res.text().catch(() => ""); - throw new Error(`groq vision ${res.status}${body ? `: ${body.slice(0, 160)}` : ""}`); - } - const data = await res.json() as { choices?: Array<{ message?: { content?: string } }> }; - return (data?.choices?.[0]?.message?.content ?? "").trim(); -} - -/** - * Transcribe audio via Groq's Whisper API. - * Used as the cloud fallback when local Whisper is unavailable (cloud host/remote). - * Returns the transcribed text. Throws on failure. - */ export async function callGroqTranscribe(audio: Blob, mimeType = "audio/webm"): Promise { const key = process.env.GROQ_API_KEY; if (!key) throw new Error("GROQ_API_KEY not set"); diff --git a/src/lib/loki/vision.ts b/src/lib/loki/vision.ts index 82297f0f8..da59c4c88 100644 --- a/src/lib/loki/vision.ts +++ b/src/lib/loki/vision.ts @@ -1,8 +1,13 @@ /** - * Vision preflight for Loki image attachments — Groq Llama 4 Scout turns - * screenshots into actionable text the chat brain and terminal agents can use. + * Vision preflight for Loki image attachments — turns screenshots into + * actionable text the chat brain and terminal agents can use. + * + * Model choice is a CHAIN, not a pin (config/vision-models.ts). The previous + * hardcoded Groq model was decommissioned and every attached image 404'd; a + * single pinned free model is a scheduled outage, so the provider chain and the + * `probe:models` check exist to make the next rot loud instead of silent. */ -import { callGroqVision } from "@/lib/groq"; +import { analyzeImages } from "@/lib/vision"; import type { ImageAttachment } from "@/lib/loki/attachments"; const SYSTEM = `You analyze UI screenshots and error states for a developer using an AI agent fleet. @@ -20,7 +25,7 @@ export async function describeAttachedImages( "The user attached screenshot(s) without a question. Describe what's wrong or notable and suggest fixes."; try { - const analysis = await callGroqVision({ + const { text, model } = await analyzeImages({ systemPrompt: SYSTEM, prompt: question, images: images.map((img) => ({ @@ -32,8 +37,12 @@ export async function describeAttachedImages( timeoutMs: 45_000, }); + // The model is named in the block because this analysis is the ONE part of + // a Loki turn that is not grounded in FleetCrown's records — it is a model's + // reading of a picture. Saying which model read it keeps that visible rather + // than letting it blend into the cited answer around it. const blocks = images.map((img) => img.name).join(", "); - return `\n\n--- Attached image analysis (${blocks}) ---\n${analysis}`; + return `\n\n--- Attached image analysis (${blocks}, via ${model}) ---\n${text}`; } catch (e) { const msg = e instanceof Error ? e.message : "vision failed"; return `\n\n[Could not analyze attached image(s): ${msg}. Describe the issue in words or try a smaller screenshot.]`; diff --git a/src/lib/vision.ts b/src/lib/vision.ts new file mode 100644 index 000000000..cf3198bba --- /dev/null +++ b/src/lib/vision.ts @@ -0,0 +1,92 @@ +/** + * Provider-agnostic image analysis — replaces the hardcoded Groq vision call. + * + * Walks `usableVisionChain()` and returns the first real answer. "Real" is + * doing work there: a model in this space can return HTTP 200 with EMPTY + * content (observed with nvidia/nemotron-nano-12b-v2-vl), and a client that + * accepts that reports a successful analysis of nothing — which reads to the + * user as "the screenshot contained nothing notable" rather than as a failure. + * Empty is therefore treated as a failed link and the chain continues. + * + * Every provider speaks the OpenAI chat-completions multimodal shape, so this + * is one request builder rather than a client per vendor. + */ +import { usableVisionChain } from "@/config/vision-models"; +import { HTTP_TIMEOUT_XL_MS } from "@/lib/constants/time"; + +export type VisionImage = { mimeType: string; dataBase64: string; name?: string }; + +/** Providers reject `image/jpg`; normalise the one alias that shows up. */ +function normalizeMime(mime: string): string { + const m = mime.toLowerCase().trim(); + return m === "image/jpg" ? "image/jpeg" : m; +} + +/** Most vision endpoints cap images per request; keep well inside every limit. */ +const MAX_IMAGES = 5; + +export type VisionResult = { text: string; model: string }; + +/** + * Analyse images with the first model in the chain that answers. + * + * Throws only when EVERY link fails, with the per-link reasons joined — so the + * caller can tell the user "no vision provider is configured" apart from "the + * model rate-limited", which are very different things to act on. + */ +export async function analyzeImages(input: { + prompt: string; + images: VisionImage[]; + systemPrompt?: string; + maxTokens?: number; + timeoutMs?: number; +}): Promise { + const chain = usableVisionChain(); + if (chain.length === 0) { + throw new Error("no vision provider configured (set OPENROUTER_API_KEY, or GROQ_VISION_MODEL if Groq has one again)"); + } + + const content = [ + { type: "text", text: input.prompt }, + ...input.images.slice(0, MAX_IMAGES).map((img) => ({ + type: "image_url", + image_url: { url: `data:${normalizeMime(img.mimeType)};base64,${img.dataBase64}` }, + })), + ]; + const messages = [ + ...(input.systemPrompt ? [{ role: "system", content: input.systemPrompt }] : []), + { role: "user", content }, + ]; + + const failures: string[] = []; + for (const { provider, model } of chain) { + try { + const res = await fetch(`${provider.baseUrl}/chat/completions`, { + method: "POST", + headers: { + "Content-Type": "application/json", + Authorization: `Bearer ${process.env[provider.keyEnv]}`, + }, + body: JSON.stringify({ model, messages, max_tokens: input.maxTokens ?? 900 }), + signal: AbortSignal.timeout(input.timeoutMs ?? HTTP_TIMEOUT_XL_MS), + }); + + if (!res.ok) { + failures.push(`${model}: HTTP ${res.status}`); + continue; + } + const data = (await res.json()) as { choices?: Array<{ message?: { content?: string } }> }; + const text = (data.choices?.[0]?.message?.content ?? "").trim(); + if (!text) { + // 200 with no content — a real, observed shape. Not an answer. + failures.push(`${model}: empty response`); + continue; + } + return { text, model: `${provider.id}/${model}` }; + } catch (e) { + failures.push(`${model}: ${e instanceof Error ? e.message.slice(0, 60) : "error"}`); + } + } + + throw new Error(`all vision models failed — ${failures.join("; ")}`); +} From bad5c3342501fd2a3838186bd2f8a7a0ca424126 Mon Sep 17 00:00:00 2001 From: test Date: Thu, 13 Aug 2026 15:14:21 +0200 Subject: [PATCH 2/2] fix(lint): drop the import orphaned by retiring callGroqVision MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit HTTP_TIMEOUT_XL_MS lost its only user when the Groq vision path was retired. Caught by CI, not locally, because I linted the files I had touched instead of the repo — and the orphan is by definition in a file the change did not add a line to. Run `npm run lint`, not eslint on a path list. Co-Authored-By: Claude Opus 5 --- src/lib/groq.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/groq.ts b/src/lib/groq.ts index 65206c21e..cb17d1493 100644 --- a/src/lib/groq.ts +++ b/src/lib/groq.ts @@ -3,7 +3,7 @@ * Dispatch strategist, prompt-merge, and Loki all use this. */ -import { HTTP_TIMEOUT_SHORT_MS, HTTP_TIMEOUT_LONG_MS, HTTP_TIMEOUT_XL_MS } from "@/lib/constants/time"; +import { HTTP_TIMEOUT_SHORT_MS, HTTP_TIMEOUT_LONG_MS } from "@/lib/constants/time"; export const GROQ_FAST_MODEL = "llama-3.3-70b-versatile"; export const GROQ_WHISPER_MODEL = "whisper-large-v3-turbo";