fix(loki): vision is a provider chain, not a pinned model that rots

package.json

@@ -79,7 +79,7 @@
     "ship": "git push origin main",
     "prepare": "husky",
     "sync:agent-core": "tsx scripts/sync-agent-core.ts",
-    "probe:loki-models": "tsx scripts/probe-loki-models.ts"
+    "probe:models": "tsx scripts/probe-models.ts"
   },
   "dependencies": {
     "@auth/drizzle-adapter": "^1.11.3",

scripts/probe-loki-models.ts → scripts/probe-models.ts

@@ -1,7 +1,7 @@
 /**
- * Live protocol probe — can this model drive Loki's tool loop?
+ * Live model probe — is every model this app depends on actually alive?
  *
- * Run: npx tsx scripts/probe-loki-models.ts [model ...]
+ * Run: npx tsx scripts/probe-models.ts [chat-model ...]   (npm run probe:models)
  *      (defaults to a spread from frontier-ish down to tiny)
  *
  * Why this exists as a committed script rather than a one-off: Loki is meant to
@@ -10,7 +10,17 @@
  * emits a tool call the loop can parse. That is cheap to measure and impossible
  * to guess, so measure it before switching, not after.
  *
- * NOT part of `npm run verify`: it costs real tokens and needs GROQ_API_KEY.
+ * Covers BOTH halves, because both have bitten:
+ *   CHAT   — can the model emit a tool call the loop can parse?
+ *   VISION — does any model in the chain actually read an image?
+ *
+ * The vision half exists because a pinned free model (Groq's llama-4-scout) was
+ * decommissioned and every attached screenshot 404'd, silently, until someone
+ * tried one. That was the fourth pinned-free-model rot in this fleet, so the
+ * class gets a command instead of another one-off fix: run this before swapping
+ * a model, and on any "images stopped working" report.
+ *
+ * NOT part of `npm run verify`: it costs real tokens and needs live keys.
  * The env-independent coverage of the same parser lives in
  * scripts/test/agent-tool-loop.ts.
  *
@@ -22,6 +32,8 @@ import { config } from "dotenv";
 config({ path: ".env.local" });
 
 import { callModelWithTools } from "../src/lib/agent/llm";
+import { analyzeImages } from "../src/lib/vision";
+import { usableVisionChain } from "../src/config/vision-models";
 
 const DEFAULT_MODELS = [
   "llama-3.3-70b-versatile",
@@ -85,7 +97,39 @@ async function main() {
     }
   }
 
-  console.log(`\n${usable}/${models.length} model(s) can drive the loop.`);
+  console.log(`\n${usable}/${models.length} chat model(s) can drive the loop.`);
+
+  // ── Vision chain ───────────────────────────────────────────────────────────
+  // A 2x2 solid-red PNG. Tiny, but it proves the model actually READ the image
+  // rather than merely accepting the request — a provider that returns 200 with
+  // empty content passes a "did it error" check and fails this one.
+  const RED_2X2 =
+    "iVBORw0KGgoAAAANSUhEUgAAAAIAAAACCAYAAABytg0kAAAAFElEQVR4nGP8z8Dwn4GKgImahgEAG1kDCf0X4o8AAAAASUVORK5CYII=";
+
+  const chain = usableVisionChain();
+  console.log(`\nVision chain (${chain.length} usable link(s)):`);
+  if (chain.length === 0) {
+    console.log("  ✗ none — no OPENROUTER_API_KEY and no GROQ_VISION_MODEL. Image attachments WILL fail.");
+  }
+  let visionOk = 0;
+  for (const { provider, model } of chain) {
+    try {
+      const r = await analyzeImages({
+        prompt: "What colour fills this image? Answer with one word.",
+        images: [{ mimeType: "image/png", dataBase64: RED_2X2 }],
+        maxTokens: 30,
+        timeoutMs: 90_000,
+      });
+      // analyzeImages walks the whole chain itself, so this reports the link
+      // that actually answered rather than the one we asked about.
+      console.log(`  ✓ ${provider.id}/${model} → answered via ${r.model}: ${r.text.slice(0, 40)}`);
+      visionOk++;
+      break;
+    } catch (e) {
+      console.log(`  ✗ ${provider.id}/${model} — ${e instanceof Error ? e.message.slice(0, 90) : e}`);
+    }
+  }
+  if (visionOk === 0 && chain.length > 0) console.log("  ✗ NO vision model answered — image attachments are broken.");
 }
 
 main().catch((e) => {

src/config/vision-models.ts

@@ -0,0 +1,76 @@
+/**
+ * SSOT for the vision model chain.
+ *
+ * This file exists because of a class of failure, not a single bug. Loki's
+ * screenshot analysis pointed at a hardcoded `GROQ_VISION_MODEL`
+ * (meta-llama/llama-4-scout-17b-16e-instruct) that was DECOMMISSIONED — every
+ * attached image returned 404, and Groq now offers no vision model at all on
+ * this account. That is the fourth time a pinned free model has silently rotted
+ * out from under this fleet.
+ *
+ * Two consequences are encoded here:
+ *
+ *   1. A CHAIN, not a model. Free-tier models disappear and rate-limit; a single
+ *      pin is a scheduled outage. The chain is tried in order and the first one
+ *      that answers wins.
+ *   2. PROVIDER-AGNOSTIC entries. Every provider here speaks the OpenAI
+ *      chat-completions shape, so adding one is a row in this table rather than
+ *      a new client. Groq keeps a seat with no default model precisely so that
+ *      re-adding one later is a one-line change.
+ *
+ * `scripts/probe-models.ts` calls every entry with a real image and fails if
+ * none answer — so the next rot is caught by a command instead of by a user
+ * attaching a screenshot and getting a 404.
+ */
+
+export type VisionProvider = {
+  /** Display/debug name, and the env prefix for its key. */
+  id: string;
+  baseUrl: string;
+  /** Env var holding the API key. Absent key = entry skipped, not an error. */
+  keyEnv: string;
+  /** Models to try for this provider, in order. */
+  models: string[];
+};
+
+/**
+ * Default chain, verified live 2026-08-13 against a solid-colour test image:
+ *
+ *   google/gemma-4-26b-a4b-it:free   → 200, correctly answered "Red"   ✅
+ *   google/gemma-4-31b-it:free       → 429 (rate limited, transient)   ⚠ fallback
+ *   nvidia/nemotron-nano-12b-v2-vl   → 200 but EMPTY content           ✗ excluded
+ *
+ * The nemotron exclusion is the interesting one: it returns HTTP 200 with no
+ * text, which a naive client reports as a successful empty analysis. Callers
+ * must treat empty content as failure — see lib/vision.ts.
+ */
+export const VISION_CHAIN: VisionProvider[] = [
+  {
+    id: "openrouter",
+    baseUrl: "https://openrouter.ai/api/v1",
+    keyEnv: "OPENROUTER_API_KEY",
+    models: ["google/gemma-4-26b-a4b-it:free", "google/gemma-4-31b-it:free"],
+  },
+  {
+    // Groq has NO vision model on this account as of 2026-08-13. The seat is
+    // kept so restoring one is a single string, and so the probe reports it.
+    id: "groq",
+    baseUrl: "https://api.groq.com/openai/v1",
+    keyEnv: "GROQ_API_KEY",
+    models: process.env.GROQ_VISION_MODEL?.trim() ? [process.env.GROQ_VISION_MODEL.trim()] : [],
+  },
+];
+
+/**
+ * The chain with unusable entries removed: no API key, or no models configured.
+ * A missing key is a normal deployment state (FleetCrown had no OpenRouter key
+ * at all until this change), so it filters out silently rather than throwing.
+ */
+export function usableVisionChain(): Array<{ provider: VisionProvider; model: string }> {
+  const out: Array<{ provider: VisionProvider; model: string }> = [];
+  for (const provider of VISION_CHAIN) {
+    if (!process.env[provider.keyEnv]?.trim()) continue;
+    for (const model of provider.models) out.push({ provider, model });
+  }
+  return out;
+}

src/lib/groq.ts

@@ -3,10 +3,9 @@
  * Dispatch strategist, prompt-merge, and Loki all use this.
  */
 
-import { HTTP_TIMEOUT_SHORT_MS, HTTP_TIMEOUT_LONG_MS, HTTP_TIMEOUT_XL_MS } from "@/lib/constants/time";
+import { HTTP_TIMEOUT_SHORT_MS, HTTP_TIMEOUT_LONG_MS } from "@/lib/constants/time";
 
 export const GROQ_FAST_MODEL = "llama-3.3-70b-versatile";
-export const GROQ_VISION_MODEL = "meta-llama/llama-4-scout-17b-16e-instruct";
 export const GROQ_WHISPER_MODEL = "whisper-large-v3-turbo";
 const GROQ_API_URL = "https://api.groq.com/openai/v1/chat/completions";
 const GROQ_AUDIO_URL = "https://api.groq.com/openai/v1/audio/transcriptions";
@@ -47,78 +46,6 @@ export async function callGroqText(prompt: string, options: GroqOptions = {}): P
   return (data?.choices?.[0]?.message?.content ?? "").trim();
 }
 
-type GroqVisionImage = { mimeType: string; dataBase64: string; name?: string };
-
-function normalizeImageMime(mime: string): string {
-  const m = mime.toLowerCase();
-  if (m === "image/jpg" || m === "image/jpeg") return "image/jpeg";
-  return m;
-}
-
-type GroqVisionOptions = {
-  prompt: string;
-  images: GroqVisionImage[];
-  systemPrompt?: string;
-  maxTokens?: number;
-  timeoutMs?: number;
-  model?: string;
-};
-
-/**
- * Multimodal Groq call — up to 5 images per request (Groq vision limit).
- * Images must be base64 without the data: prefix.
- */
-export async function callGroqVision(options: GroqVisionOptions): Promise<string> {
-  const key = process.env.GROQ_API_KEY;
-  if (!key) throw new Error("GROQ_API_KEY not set");
-
-  const {
-    prompt,
-    images,
-    systemPrompt,
-    maxTokens = 900,
-    timeoutMs = HTTP_TIMEOUT_XL_MS,
-    model = GROQ_VISION_MODEL,
-  } = options;
-
-  if (images.length === 0) throw new Error("vision requires at least one image");
-  if (images.length > 5) throw new Error("vision supports at most 5 images");
-
-  const content: Array<{ type: string; text?: string; image_url?: { url: string } }> = [
-    { type: "text", text: prompt },
-  ];
-  for (const img of images) {
-    const mime = normalizeImageMime(img.mimeType);
-    content.push({
-      type: "image_url",
-      image_url: { url: `data:${mime};base64,${img.dataBase64}` },
-    });
-  }
-
-  const messages: Array<{ role: string; content: unknown }> = [];
-  if (systemPrompt) messages.push({ role: "system", content: systemPrompt });
-  messages.push({ role: "user", content });
-
-  const res = await fetch(GROQ_API_URL, {
-    method: "POST",
-    headers: { "Content-Type": "application/json", Authorization: `Bearer ${key}` },
-    body: JSON.stringify({ model, messages, max_tokens: maxTokens, temperature: 0.2 }),
-    signal: AbortSignal.timeout(timeoutMs),
-  });
-
-  if (!res.ok) {
-    const body = await res.text().catch(() => "");
-    throw new Error(`groq vision ${res.status}${body ? `: ${body.slice(0, 160)}` : ""}`);
-  }
-  const data = await res.json() as { choices?: Array<{ message?: { content?: string } }> };
-  return (data?.choices?.[0]?.message?.content ?? "").trim();
-}
-
-/**
- * Transcribe audio via Groq's Whisper API.
- * Used as the cloud fallback when local Whisper is unavailable (cloud host/remote).
- * Returns the transcribed text. Throws on failure.
- */
 export async function callGroqTranscribe(audio: Blob, mimeType = "audio/webm"): Promise<string> {
   const key = process.env.GROQ_API_KEY;
   if (!key) throw new Error("GROQ_API_KEY not set");

src/lib/loki/vision.ts

@@ -1,8 +1,13 @@
 /**
- * Vision preflight for Loki image attachments — Groq Llama 4 Scout turns
- * screenshots into actionable text the chat brain and terminal agents can use.
+ * Vision preflight for Loki image attachments — turns screenshots into
+ * actionable text the chat brain and terminal agents can use.
+ *
+ * Model choice is a CHAIN, not a pin (config/vision-models.ts). The previous
+ * hardcoded Groq model was decommissioned and every attached image 404'd; a
+ * single pinned free model is a scheduled outage, so the provider chain and the
+ * `probe:models` check exist to make the next rot loud instead of silent.
  */
-import { callGroqVision } from "@/lib/groq";
+import { analyzeImages } from "@/lib/vision";
 import type { ImageAttachment } from "@/lib/loki/attachments";
 
 const SYSTEM = `You analyze UI screenshots and error states for a developer using an AI agent fleet.
@@ -20,7 +25,7 @@ export async function describeAttachedImages(
     "The user attached screenshot(s) without a question. Describe what's wrong or notable and suggest fixes.";
 
   try {
-    const analysis = await callGroqVision({
+    const { text, model } = await analyzeImages({
       systemPrompt: SYSTEM,
       prompt: question,
       images: images.map((img) => ({
@@ -32,8 +37,12 @@ export async function describeAttachedImages(
       timeoutMs: 45_000,
     });
 
+    // The model is named in the block because this analysis is the ONE part of
+    // a Loki turn that is not grounded in FleetCrown's records — it is a model's
+    // reading of a picture. Saying which model read it keeps that visible rather
+    // than letting it blend into the cited answer around it.
     const blocks = images.map((img) => img.name).join(", ");
-    return `\n\n--- Attached image analysis (${blocks}) ---\n${analysis}`;
+    return `\n\n--- Attached image analysis (${blocks}, via ${model}) ---\n${text}`;
   } catch (e) {
     const msg = e instanceof Error ? e.message : "vision failed";
     return `\n\n[Could not analyze attached image(s): ${msg}. Describe the issue in words or try a smaller screenshot.]`;

src/lib/vision.ts

@@ -0,0 +1,92 @@
+/**
+ * Provider-agnostic image analysis — replaces the hardcoded Groq vision call.
+ *
+ * Walks `usableVisionChain()` and returns the first real answer. "Real" is
+ * doing work there: a model in this space can return HTTP 200 with EMPTY
+ * content (observed with nvidia/nemotron-nano-12b-v2-vl), and a client that
+ * accepts that reports a successful analysis of nothing — which reads to the
+ * user as "the screenshot contained nothing notable" rather than as a failure.
+ * Empty is therefore treated as a failed link and the chain continues.
+ *
+ * Every provider speaks the OpenAI chat-completions multimodal shape, so this
+ * is one request builder rather than a client per vendor.
+ */
+import { usableVisionChain } from "@/config/vision-models";
+import { HTTP_TIMEOUT_XL_MS } from "@/lib/constants/time";
+
+export type VisionImage = { mimeType: string; dataBase64: string; name?: string };
+
+/** Providers reject `image/jpg`; normalise the one alias that shows up. */
+function normalizeMime(mime: string): string {
+  const m = mime.toLowerCase().trim();
+  return m === "image/jpg" ? "image/jpeg" : m;
+}
+
+/** Most vision endpoints cap images per request; keep well inside every limit. */
+const MAX_IMAGES = 5;
+
+export type VisionResult = { text: string; model: string };
+
+/**
+ * Analyse images with the first model in the chain that answers.
+ *
+ * Throws only when EVERY link fails, with the per-link reasons joined — so the
+ * caller can tell the user "no vision provider is configured" apart from "the
+ * model rate-limited", which are very different things to act on.
+ */
+export async function analyzeImages(input: {
+  prompt: string;
+  images: VisionImage[];
+  systemPrompt?: string;
+  maxTokens?: number;
+  timeoutMs?: number;
+}): Promise<VisionResult> {
+  const chain = usableVisionChain();
+  if (chain.length === 0) {
+    throw new Error("no vision provider configured (set OPENROUTER_API_KEY, or GROQ_VISION_MODEL if Groq has one again)");
+  }
+
+  const content = [
+    { type: "text", text: input.prompt },
+    ...input.images.slice(0, MAX_IMAGES).map((img) => ({
+      type: "image_url",
+      image_url: { url: `data:${normalizeMime(img.mimeType)};base64,${img.dataBase64}` },
+    })),
+  ];
+  const messages = [
+    ...(input.systemPrompt ? [{ role: "system", content: input.systemPrompt }] : []),
+    { role: "user", content },
+  ];
+
+  const failures: string[] = [];
+  for (const { provider, model } of chain) {
+    try {
+      const res = await fetch(`${provider.baseUrl}/chat/completions`, {
+        method: "POST",
+        headers: {
+          "Content-Type": "application/json",
+          Authorization: `Bearer ${process.env[provider.keyEnv]}`,
+        },
+        body: JSON.stringify({ model, messages, max_tokens: input.maxTokens ?? 900 }),
+        signal: AbortSignal.timeout(input.timeoutMs ?? HTTP_TIMEOUT_XL_MS),
+      });
+
+      if (!res.ok) {
+        failures.push(`${model}: HTTP ${res.status}`);
+        continue;
+      }
+      const data = (await res.json()) as { choices?: Array<{ message?: { content?: string } }> };
+      const text = (data.choices?.[0]?.message?.content ?? "").trim();
+      if (!text) {
+        // 200 with no content — a real, observed shape. Not an answer.
+        failures.push(`${model}: empty response`);
+        continue;
+      }
+      return { text, model: `${provider.id}/${model}` };
+    } catch (e) {
+      failures.push(`${model}: ${e instanceof Error ? e.message.slice(0, 60) : "error"}`);
+    }
+  }
+
+  throw new Error(`all vision models failed — ${failures.join("; ")}`);
+}