diff --git a/package.json b/package.json index 9ab7d2b5..20b84611 100644 --- a/package.json +++ b/package.json @@ -79,7 +79,7 @@ "ship": "git push origin main", "prepare": "husky", "sync:agent-core": "tsx scripts/sync-agent-core.ts", - "probe:loki-models": "tsx scripts/probe-loki-models.ts" + "probe:models": "tsx scripts/probe-models.ts" }, "dependencies": { "@auth/drizzle-adapter": "^1.11.3", diff --git a/scripts/probe-loki-models.ts b/scripts/probe-models.ts similarity index 54% rename from scripts/probe-loki-models.ts rename to scripts/probe-models.ts index ca30246e..03201b6f 100644 --- a/scripts/probe-loki-models.ts +++ b/scripts/probe-models.ts @@ -1,7 +1,7 @@ /** - * Live protocol probe — can this model drive Loki's tool loop? + * Live model probe — is every model this app depends on actually alive? * - * Run: npx tsx scripts/probe-loki-models.ts [model ...] + * Run: npx tsx scripts/probe-models.ts [chat-model ...] (npm run probe:models) * (defaults to a spread from frontier-ish down to tiny) * * Why this exists as a committed script rather than a one-off: Loki is meant to @@ -10,7 +10,17 @@ * emits a tool call the loop can parse. That is cheap to measure and impossible * to guess, so measure it before switching, not after. * - * NOT part of `npm run verify`: it costs real tokens and needs GROQ_API_KEY. + * Covers BOTH halves, because both have bitten: + * CHAT — can the model emit a tool call the loop can parse? + * VISION — does any model in the chain actually read an image? + * + * The vision half exists because a pinned free model (Groq's llama-4-scout) was + * decommissioned and every attached screenshot 404'd, silently, until someone + * tried one. That was the fourth pinned-free-model rot in this fleet, so the + * class gets a command instead of another one-off fix: run this before swapping + * a model, and on any "images stopped working" report. + * + * NOT part of `npm run verify`: it costs real tokens and needs live keys. * The env-independent coverage of the same parser lives in * scripts/test/agent-tool-loop.ts. * @@ -22,6 +32,8 @@ import { config } from "dotenv"; config({ path: ".env.local" }); import { callModelWithTools } from "../src/lib/agent/llm"; +import { analyzeImages } from "../src/lib/vision"; +import { usableVisionChain } from "../src/config/vision-models"; const DEFAULT_MODELS = [ "llama-3.3-70b-versatile", @@ -85,7 +97,39 @@ async function main() { } } - console.log(`\n${usable}/${models.length} model(s) can drive the loop.`); + console.log(`\n${usable}/${models.length} chat model(s) can drive the loop.`); + + // ── Vision chain ─────────────────────────────────────────────────────────── + // A 2x2 solid-red PNG. Tiny, but it proves the model actually READ the image + // rather than merely accepting the request — a provider that returns 200 with + // empty content passes a "did it error" check and fails this one. + const RED_2X2 = + "iVBORw0KGgoAAAANSUhEUgAAAAIAAAACCAYAAABytg0kAAAAFElEQVR4nGP8z8Dwn4GKgImahgEAG1kDCf0X4o8AAAAASUVORK5CYII="; + + const chain = usableVisionChain(); + console.log(`\nVision chain (${chain.length} usable link(s)):`); + if (chain.length === 0) { + console.log(" ✗ none — no OPENROUTER_API_KEY and no GROQ_VISION_MODEL. Image attachments WILL fail."); + } + let visionOk = 0; + for (const { provider, model } of chain) { + try { + const r = await analyzeImages({ + prompt: "What colour fills this image? Answer with one word.", + images: [{ mimeType: "image/png", dataBase64: RED_2X2 }], + maxTokens: 30, + timeoutMs: 90_000, + }); + // analyzeImages walks the whole chain itself, so this reports the link + // that actually answered rather than the one we asked about. + console.log(` ✓ ${provider.id}/${model} → answered via ${r.model}: ${r.text.slice(0, 40)}`); + visionOk++; + break; + } catch (e) { + console.log(` ✗ ${provider.id}/${model} — ${e instanceof Error ? e.message.slice(0, 90) : e}`); + } + } + if (visionOk === 0 && chain.length > 0) console.log(" ✗ NO vision model answered — image attachments are broken."); } main().catch((e) => { diff --git a/src/config/vision-models.ts b/src/config/vision-models.ts new file mode 100644 index 00000000..19a3be81 --- /dev/null +++ b/src/config/vision-models.ts @@ -0,0 +1,76 @@ +/** + * SSOT for the vision model chain. + * + * This file exists because of a class of failure, not a single bug. Loki's + * screenshot analysis pointed at a hardcoded `GROQ_VISION_MODEL` + * (meta-llama/llama-4-scout-17b-16e-instruct) that was DECOMMISSIONED — every + * attached image returned 404, and Groq now offers no vision model at all on + * this account. That is the fourth time a pinned free model has silently rotted + * out from under this fleet. + * + * Two consequences are encoded here: + * + * 1. A CHAIN, not a model. Free-tier models disappear and rate-limit; a single + * pin is a scheduled outage. The chain is tried in order and the first one + * that answers wins. + * 2. PROVIDER-AGNOSTIC entries. Every provider here speaks the OpenAI + * chat-completions shape, so adding one is a row in this table rather than + * a new client. Groq keeps a seat with no default model precisely so that + * re-adding one later is a one-line change. + * + * `scripts/probe-models.ts` calls every entry with a real image and fails if + * none answer — so the next rot is caught by a command instead of by a user + * attaching a screenshot and getting a 404. + */ + +export type VisionProvider = { + /** Display/debug name, and the env prefix for its key. */ + id: string; + baseUrl: string; + /** Env var holding the API key. Absent key = entry skipped, not an error. */ + keyEnv: string; + /** Models to try for this provider, in order. */ + models: string[]; +}; + +/** + * Default chain, verified live 2026-08-13 against a solid-colour test image: + * + * google/gemma-4-26b-a4b-it:free → 200, correctly answered "Red" ✅ + * google/gemma-4-31b-it:free → 429 (rate limited, transient) ⚠ fallback + * nvidia/nemotron-nano-12b-v2-vl → 200 but EMPTY content ✗ excluded + * + * The nemotron exclusion is the interesting one: it returns HTTP 200 with no + * text, which a naive client reports as a successful empty analysis. Callers + * must treat empty content as failure — see lib/vision.ts. + */ +export const VISION_CHAIN: VisionProvider[] = [ + { + id: "openrouter", + baseUrl: "https://openrouter.ai/api/v1", + keyEnv: "OPENROUTER_API_KEY", + models: ["google/gemma-4-26b-a4b-it:free", "google/gemma-4-31b-it:free"], + }, + { + // Groq has NO vision model on this account as of 2026-08-13. The seat is + // kept so restoring one is a single string, and so the probe reports it. + id: "groq", + baseUrl: "https://api.groq.com/openai/v1", + keyEnv: "GROQ_API_KEY", + models: process.env.GROQ_VISION_MODEL?.trim() ? [process.env.GROQ_VISION_MODEL.trim()] : [], + }, +]; + +/** + * The chain with unusable entries removed: no API key, or no models configured. + * A missing key is a normal deployment state (FleetCrown had no OpenRouter key + * at all until this change), so it filters out silently rather than throwing. + */ +export function usableVisionChain(): Array<{ provider: VisionProvider; model: string }> { + const out: Array<{ provider: VisionProvider; model: string }> = []; + for (const provider of VISION_CHAIN) { + if (!process.env[provider.keyEnv]?.trim()) continue; + for (const model of provider.models) out.push({ provider, model }); + } + return out; +} diff --git a/src/lib/groq.ts b/src/lib/groq.ts index a04fc2ad..cb17d149 100644 --- a/src/lib/groq.ts +++ b/src/lib/groq.ts @@ -3,10 +3,9 @@ * Dispatch strategist, prompt-merge, and Loki all use this. */ -import { HTTP_TIMEOUT_SHORT_MS, HTTP_TIMEOUT_LONG_MS, HTTP_TIMEOUT_XL_MS } from "@/lib/constants/time"; +import { HTTP_TIMEOUT_SHORT_MS, HTTP_TIMEOUT_LONG_MS } from "@/lib/constants/time"; export const GROQ_FAST_MODEL = "llama-3.3-70b-versatile"; -export const GROQ_VISION_MODEL = "meta-llama/llama-4-scout-17b-16e-instruct"; export const GROQ_WHISPER_MODEL = "whisper-large-v3-turbo"; const GROQ_API_URL = "https://api.groq.com/openai/v1/chat/completions"; const GROQ_AUDIO_URL = "https://api.groq.com/openai/v1/audio/transcriptions"; @@ -47,78 +46,6 @@ export async function callGroqText(prompt: string, options: GroqOptions = {}): P return (data?.choices?.[0]?.message?.content ?? "").trim(); } -type GroqVisionImage = { mimeType: string; dataBase64: string; name?: string }; - -function normalizeImageMime(mime: string): string { - const m = mime.toLowerCase(); - if (m === "image/jpg" || m === "image/jpeg") return "image/jpeg"; - return m; -} - -type GroqVisionOptions = { - prompt: string; - images: GroqVisionImage[]; - systemPrompt?: string; - maxTokens?: number; - timeoutMs?: number; - model?: string; -}; - -/** - * Multimodal Groq call — up to 5 images per request (Groq vision limit). - * Images must be base64 without the data: prefix. - */ -export async function callGroqVision(options: GroqVisionOptions): Promise { - const key = process.env.GROQ_API_KEY; - if (!key) throw new Error("GROQ_API_KEY not set"); - - const { - prompt, - images, - systemPrompt, - maxTokens = 900, - timeoutMs = HTTP_TIMEOUT_XL_MS, - model = GROQ_VISION_MODEL, - } = options; - - if (images.length === 0) throw new Error("vision requires at least one image"); - if (images.length > 5) throw new Error("vision supports at most 5 images"); - - const content: Array<{ type: string; text?: string; image_url?: { url: string } }> = [ - { type: "text", text: prompt }, - ]; - for (const img of images) { - const mime = normalizeImageMime(img.mimeType); - content.push({ - type: "image_url", - image_url: { url: `data:${mime};base64,${img.dataBase64}` }, - }); - } - - const messages: Array<{ role: string; content: unknown }> = []; - if (systemPrompt) messages.push({ role: "system", content: systemPrompt }); - messages.push({ role: "user", content }); - - const res = await fetch(GROQ_API_URL, { - method: "POST", - headers: { "Content-Type": "application/json", Authorization: `Bearer ${key}` }, - body: JSON.stringify({ model, messages, max_tokens: maxTokens, temperature: 0.2 }), - signal: AbortSignal.timeout(timeoutMs), - }); - - if (!res.ok) { - const body = await res.text().catch(() => ""); - throw new Error(`groq vision ${res.status}${body ? `: ${body.slice(0, 160)}` : ""}`); - } - const data = await res.json() as { choices?: Array<{ message?: { content?: string } }> }; - return (data?.choices?.[0]?.message?.content ?? "").trim(); -} - -/** - * Transcribe audio via Groq's Whisper API. - * Used as the cloud fallback when local Whisper is unavailable (cloud host/remote). - * Returns the transcribed text. Throws on failure. - */ export async function callGroqTranscribe(audio: Blob, mimeType = "audio/webm"): Promise { const key = process.env.GROQ_API_KEY; if (!key) throw new Error("GROQ_API_KEY not set"); diff --git a/src/lib/loki/vision.ts b/src/lib/loki/vision.ts index 82297f0f..da59c4c8 100644 --- a/src/lib/loki/vision.ts +++ b/src/lib/loki/vision.ts @@ -1,8 +1,13 @@ /** - * Vision preflight for Loki image attachments — Groq Llama 4 Scout turns - * screenshots into actionable text the chat brain and terminal agents can use. + * Vision preflight for Loki image attachments — turns screenshots into + * actionable text the chat brain and terminal agents can use. + * + * Model choice is a CHAIN, not a pin (config/vision-models.ts). The previous + * hardcoded Groq model was decommissioned and every attached image 404'd; a + * single pinned free model is a scheduled outage, so the provider chain and the + * `probe:models` check exist to make the next rot loud instead of silent. */ -import { callGroqVision } from "@/lib/groq"; +import { analyzeImages } from "@/lib/vision"; import type { ImageAttachment } from "@/lib/loki/attachments"; const SYSTEM = `You analyze UI screenshots and error states for a developer using an AI agent fleet. @@ -20,7 +25,7 @@ export async function describeAttachedImages( "The user attached screenshot(s) without a question. Describe what's wrong or notable and suggest fixes."; try { - const analysis = await callGroqVision({ + const { text, model } = await analyzeImages({ systemPrompt: SYSTEM, prompt: question, images: images.map((img) => ({ @@ -32,8 +37,12 @@ export async function describeAttachedImages( timeoutMs: 45_000, }); + // The model is named in the block because this analysis is the ONE part of + // a Loki turn that is not grounded in FleetCrown's records — it is a model's + // reading of a picture. Saying which model read it keeps that visible rather + // than letting it blend into the cited answer around it. const blocks = images.map((img) => img.name).join(", "); - return `\n\n--- Attached image analysis (${blocks}) ---\n${analysis}`; + return `\n\n--- Attached image analysis (${blocks}, via ${model}) ---\n${text}`; } catch (e) { const msg = e instanceof Error ? e.message : "vision failed"; return `\n\n[Could not analyze attached image(s): ${msg}. Describe the issue in words or try a smaller screenshot.]`; diff --git a/src/lib/vision.ts b/src/lib/vision.ts new file mode 100644 index 00000000..cf3198bb --- /dev/null +++ b/src/lib/vision.ts @@ -0,0 +1,92 @@ +/** + * Provider-agnostic image analysis — replaces the hardcoded Groq vision call. + * + * Walks `usableVisionChain()` and returns the first real answer. "Real" is + * doing work there: a model in this space can return HTTP 200 with EMPTY + * content (observed with nvidia/nemotron-nano-12b-v2-vl), and a client that + * accepts that reports a successful analysis of nothing — which reads to the + * user as "the screenshot contained nothing notable" rather than as a failure. + * Empty is therefore treated as a failed link and the chain continues. + * + * Every provider speaks the OpenAI chat-completions multimodal shape, so this + * is one request builder rather than a client per vendor. + */ +import { usableVisionChain } from "@/config/vision-models"; +import { HTTP_TIMEOUT_XL_MS } from "@/lib/constants/time"; + +export type VisionImage = { mimeType: string; dataBase64: string; name?: string }; + +/** Providers reject `image/jpg`; normalise the one alias that shows up. */ +function normalizeMime(mime: string): string { + const m = mime.toLowerCase().trim(); + return m === "image/jpg" ? "image/jpeg" : m; +} + +/** Most vision endpoints cap images per request; keep well inside every limit. */ +const MAX_IMAGES = 5; + +export type VisionResult = { text: string; model: string }; + +/** + * Analyse images with the first model in the chain that answers. + * + * Throws only when EVERY link fails, with the per-link reasons joined — so the + * caller can tell the user "no vision provider is configured" apart from "the + * model rate-limited", which are very different things to act on. + */ +export async function analyzeImages(input: { + prompt: string; + images: VisionImage[]; + systemPrompt?: string; + maxTokens?: number; + timeoutMs?: number; +}): Promise { + const chain = usableVisionChain(); + if (chain.length === 0) { + throw new Error("no vision provider configured (set OPENROUTER_API_KEY, or GROQ_VISION_MODEL if Groq has one again)"); + } + + const content = [ + { type: "text", text: input.prompt }, + ...input.images.slice(0, MAX_IMAGES).map((img) => ({ + type: "image_url", + image_url: { url: `data:${normalizeMime(img.mimeType)};base64,${img.dataBase64}` }, + })), + ]; + const messages = [ + ...(input.systemPrompt ? [{ role: "system", content: input.systemPrompt }] : []), + { role: "user", content }, + ]; + + const failures: string[] = []; + for (const { provider, model } of chain) { + try { + const res = await fetch(`${provider.baseUrl}/chat/completions`, { + method: "POST", + headers: { + "Content-Type": "application/json", + Authorization: `Bearer ${process.env[provider.keyEnv]}`, + }, + body: JSON.stringify({ model, messages, max_tokens: input.maxTokens ?? 900 }), + signal: AbortSignal.timeout(input.timeoutMs ?? HTTP_TIMEOUT_XL_MS), + }); + + if (!res.ok) { + failures.push(`${model}: HTTP ${res.status}`); + continue; + } + const data = (await res.json()) as { choices?: Array<{ message?: { content?: string } }> }; + const text = (data.choices?.[0]?.message?.content ?? "").trim(); + if (!text) { + // 200 with no content — a real, observed shape. Not an answer. + failures.push(`${model}: empty response`); + continue; + } + return { text, model: `${provider.id}/${model}` }; + } catch (e) { + failures.push(`${model}: ${e instanceof Error ? e.message.slice(0, 60) : "error"}`); + } + } + + throw new Error(`all vision models failed — ${failures.join("; ")}`); +}