HackingDave · TheComplianceAide · Jun 6, 2026 · Jun 6, 2026
diff --git a/README.md b/README.md
@@ -6,11 +6,25 @@ Independent, automated benchmarking of frontier AI coding models. Tracks perform
 
 ## Why This Exists
 
-AI model providers update their models constantly, and sometimes performance degrades without any announcement. ModelRegression runs the same 30 tests against every model daily, scores the results, and surfaces regressions automatically. No vendor self-reporting — just independent, reproducible benchmarks.
+AI model providers update their models constantly, and sometimes performance degrades without any announcement. ModelRegression runs the same 33 tests against every model daily, scores the results, and surfaces regressions automatically. No vendor self-reporting — just independent, reproducible benchmarks.
+
+## Trust Status
+
+The dashboard now answers the blunt question security teams actually care about:
+
+> Can I trust this model today?
+
+Each model gets a plain-English status:
+
+- **Good** - useful for security review support, with human verification.
+- **Watch** - helpful, but do not let it make the final call.
+- **Do not trust alone** - keep it away from incident response, risky fixes, and compliance decisions unless a human is driving.
+
+The status is derived from the latest composite score, Security Awareness score, Code Thoroughness score, active regression flags, and the weakest security-awareness evidence from the current run. Every claim links back to receipts.
 
 ## What Gets Tested
 
-30 tests across 10 categories, each targeting a different dimension of coding ability:
+33 tests across 11 categories, each targeting a different dimension of coding and agentic ability:
 
 | Category | What It Measures |
 |---|---|
@@ -24,15 +38,19 @@ AI model providers update their models constantly, and sometimes performance deg
 | Instruction Following | Schema compliance, constraint adherence, multi-step chains |
 | Code Quality | Idiomatic Python, TypeScript best practices, clean architecture |
 | Performance Efficiency | Algorithm complexity, streaming, query optimization |
+| Computer-Use Planning | Windows/macOS GUI state reasoning, recovery planning, and verified-completion discipline |
 
 ## Models Tracked
 
 - **Claude Opus 4.8** (Anthropic) — via `claude` CLI
 - **Claude Sonnet 4.6** (Anthropic) — via `claude` CLI
 - **GPT-5.5** (OpenAI) — via `codex` CLI
 - **Grok** (xAI) — via `agent` CLI
+- **OpenRouter models** (open-weight/open-source candidates) - opt-in via `OPENROUTER_MODEL_IDS` or a pinned manifest
 
-Models are tested through their official CLI tools, not direct API calls. This tests the full stack that developers actually use.
+Default frontier models are tested through their official CLI tools. OpenRouter models are tested through OpenRouter's chat-completions API so the same suite can cover dozens of open-weight and API-hosted models.
+
+See [docs/research-basis.md](docs/research-basis.md) for the benchmark design rationale, including recent 2026 computer-use and cybersecurity benchmark references.
 
 ## Architecture
 
@@ -42,8 +60,8 @@ Models are tested through their official CLI tools, not direct API calls. This t
                 │   (cron job) │  Python 3.13 + SQLite
                 └──────┬───────┘
                        │
-            benchmark suite runs 30 tests
-            against each model via CLI tools
+            benchmark suite runs 33 tests
+            against each model via configured adapters
                        │
                        ▼
                 ┌──────────────┐
@@ -107,12 +125,12 @@ modelregression/
 │   ├── regression_detector.py    #   Regression detection logic
 │   ├── outage_monitor.py         #   Health checks + outage tracking
 │   ├── run_benchmarks.sh         #   Cron entry point (full pipeline)
-│   └── tests/                    #   Test implementations (30 tests)
+│   └── tests/                    #   Test implementations (33 tests)
 │       ├── base.py               #     Base test class
 │       ├── long_reasoning.py     #     3 long reasoning tests
 │       ├── coding_tasks.py       #     3 coding task tests
 │       ├── bug_fixes.py          #     3 bug fix tests
-│       └── ...                   #     (10 categories x 3 tests each)
+│       └── ...                   #     (11 categories x 3 tests each)
 ├── config/                       # Nginx configuration
 ├── deploy.sh                     # Blue-green atomic deployment
 └── ecosystem.config.js           # PM2 process configuration
@@ -156,12 +174,55 @@ python export_json.py --output ../public/data
 
 ### CLI Tool Prerequisites
 
-The benchmark engine calls models through their official CLI tools. You need these installed and authenticated:
+The benchmark engine calls default frontier models through their official CLI tools. You need these installed and authenticated:
 
 - **[Claude Code](https://docs.anthropic.com/en/docs/claude-code)** (`claude`) — for Anthropic models
 - **[Codex CLI](https://github.com/openai/codex)** (`codex`) — for OpenAI models
 - **[Grok Agent](https://docs.x.ai/docs/grok-agent)** (`agent`) — for xAI models
 
+### OpenRouter Model Sweeps
+
+OpenRouter support is opt-in so daily runs do not suddenly benchmark dozens of additional models or incur unexpected cost. Broad sweeps should be pinned in a manifest first; benchmark runs do not fetch the live OpenRouter catalog during config import.
+
+```powershell
+# Controlled OpenRouter run against explicit model IDs
+$env:OPENROUTER_API_KEY = "<your key>"
+$env:OPENROUTER_MODEL_IDS = "meta-llama/llama-3.3-70b-instruct,qwen/qwen3.7-plus"
+py -3 benchmark\runner.py --schedule daily
+
+# Generate and review a broad open-weight candidate manifest
+$env:OPENROUTER_API_KEY = "<your key>"
+py -3 benchmark\openrouter_manifest.py --limit 50 --output benchmark\manifests\openrouter_models.json
+
+# Run from the pinned manifest
+$env:OPENROUTER_MANIFEST = "benchmark\manifests\openrouter_models.json"
+py -3 benchmark\runner.py --schedule daily
+
+# Optional: only include OpenRouter :free variants while generating the manifest
+py -3 benchmark\openrouter_manifest.py --free-only --limit 50 --output benchmark\manifests\openrouter_models.json
+```
+
+The manifest generator currently matches provider/model prefixes such as `meta-llama/`, `mistralai/`, `qwen/`, `deepseek/`, `google/gemma`, `nvidia/`, `microsoft/`, and other open-weight candidate families. Treat these as open-weight candidates until licenses are verified. Override with repeated `--prefix` flags when the catalog changes.
+
+Useful safety controls:
+
+```powershell
+$env:OPENROUTER_MAX_MODELS = "50"          # fail fast if the manifest is larger
+$env:MAX_PARALLEL_MODELS = "4"             # global model-level workers
+$env:OPENROUTER_PARALLEL_TESTS = "1"       # per-OpenRouter-model test workers
+```
+
+### OpenRouter Pricing
+
+The JSON export also refreshes `public/data/openrouter-pricing.json` from OpenRouter's model catalog each day. Prices are normalized from OpenRouter's per-token values into dollars per 1M input tokens, dollars per 1M output tokens, and a simple 1M-in + 1M-out blended number for dashboard display.
+
+```powershell
+# Standalone refresh if you only want to update the price sheet
+py -3 benchmark\openrouter_pricing.py --output public\data\openrouter-pricing.json
+```
+
+Negative OpenRouter router sentinel prices are ignored because they do not represent a real per-token price.
+
 ### Deployment
 
 Deployment uses a blue-green strategy with zero-downtime swaps:

diff --git a/app/categories/[slug]/page.tsx b/app/categories/[slug]/page.tsx
@@ -1,8 +1,7 @@
 import { notFound } from "next/navigation";
 import Link from "next/link";
 import type { Metadata } from "next";
-import { getCategoryDetail, getAllCategorySlugs } from "@/lib/data";
-import { getModel } from "@/lib/models";
+import { getAllModels, getCategoryDetail, getAllCategorySlugs } from "@/lib/data";
 import {
   cn,
   formatScore,
@@ -83,12 +82,13 @@ export default async function CategoryDetailPage({
     notFound();
   }
 
-  const { category, tests, models } = detail;
+  const modelInfos = getAllModels();
+  const { category, tests, models: categoryModels } = detail;
 
   // ---- Build line chart data from all models' history ----
   // Collect all unique timestamps
   const timestampSet = new Set<string>();
-  for (const [, modelData] of Object.entries(models)) {
+  for (const [, modelData] of Object.entries(categoryModels)) {
     for (const h of modelData.history) {
       timestampSet.add(h.timestamp);
     }
@@ -97,7 +97,7 @@ export default async function CategoryDetailPage({
 
   // Build lookup maps for fast access
   const modelHistoryMaps: Record<string, Map<string, number>> = {};
-  for (const [modelId, modelData] of Object.entries(models)) {
+  for (const [modelId, modelData] of Object.entries(categoryModels)) {
     const map = new Map<string, number>();
     for (const h of modelData.history) {
       map.set(h.timestamp, h.score);
@@ -110,7 +110,7 @@ export default async function CategoryDetailPage({
       timestamp: ts,
       models: {},
     };
-    for (const modelId of Object.keys(models)) {
+    for (const modelId of Object.keys(categoryModels)) {
       const val = modelHistoryMaps[modelId].get(ts);
       if (val !== undefined) {
         point.models[modelId] = val;
@@ -120,10 +120,10 @@ export default async function CategoryDetailPage({
   });
 
   // ---- Rank models by current score ----
-  const rankedModels = Object.entries(models)
+  const rankedModels = Object.entries(categoryModels)
     .map(([modelId, data]) => ({
       modelId,
-      info: getModel(modelId),
+      info: modelInfos.find((m) => m.id === modelId),
       score: data.currentScore,
       sparkline: data.history.slice(-7).map((h) => h.score),
     }))
@@ -234,6 +234,7 @@ export default async function CategoryDetailPage({
             data={chartData}
             height={400}
             showLegend={true}
+            models={modelInfos}
           />
         </div>
       </ScrollReveal>

diff --git a/app/compare/compare-client.tsx b/app/compare/compare-client.tsx
@@ -2,24 +2,30 @@
 
 import { useState } from "react";
 import { cn } from "@/lib/utils";
-import { MODELS } from "@/lib/models";
 import { CATEGORIES } from "@/lib/categories";
 import { formatScore, getScoreColor } from "@/lib/utils";
 import { CategoryRadarChart } from "@/components/charts/category-radar-chart";
 import { PerformanceLineChart } from "@/components/charts/performance-line-chart";
 import { ScrollReveal } from "@/components/shared/scroll-reveal";
 import { Check } from "lucide-react";
-import type { TrendData, LatestRun } from "@/lib/types";
+import type { TrendData, LatestRun, ModelInfo } from "@/lib/types";
 
 interface CompareClientProps {
   scores: Record<string, Record<string, number>>;
   trends: TrendData;
   latest: LatestRun;
+  models: ModelInfo[];
 }
 
-export function CompareClient({ scores, trends, latest }: CompareClientProps) {
+export function CompareClient({ scores, trends, latest, models }: CompareClientProps) {
   const [selected, setSelected] = useState<string[]>(
-    MODELS.map((m) => m.id)
+    models
+      .filter((m) => latest.models[m.id])
+      .sort((a, b) =>
+        (latest.models[a.id]?.rank ?? 999) - (latest.models[b.id]?.rank ?? 999)
+      )
+      .slice(0, Math.min(models.length, 6))
+      .map((m) => m.id)
   );
 
   const toggle = (id: string) => {
@@ -42,7 +48,7 @@ export function CompareClient({ scores, trends, latest }: CompareClientProps) {
             Select Models to Compare
           </h3>
           <div className="flex flex-wrap gap-2">
-            {MODELS.map((model) => {
+            {models.map((model) => {
               const isSelected = selected.includes(model.id);
               return (
                 <button
@@ -85,6 +91,7 @@ export function CompareClient({ scores, trends, latest }: CompareClientProps) {
             <CategoryRadarChart
               scores={filteredScores}
               modelFilter={selected}
+              models={models}
               height={450}
             />
           </div>
@@ -101,6 +108,7 @@ export function CompareClient({ scores, trends, latest }: CompareClientProps) {
             <PerformanceLineChart
               data={trends.data}
               modelFilter={selected}
+              models={models}
               height={350}
             />
           </div>
@@ -123,7 +131,7 @@ export function CompareClient({ scores, trends, latest }: CompareClientProps) {
                     <th className="text-left text-[11px] font-medium text-muted-foreground py-3 px-4 w-44">
                       Category
                     </th>
-                    {MODELS.filter((m) => selected.includes(m.id)).map(
+                    {models.filter((m) => selected.includes(m.id)).map(
                       (model) => (
                         <th
                           key={model.id}
@@ -138,7 +146,7 @@ export function CompareClient({ scores, trends, latest }: CompareClientProps) {
                 </thead>
                 <tbody>
                   {CATEGORIES.map((cat) => {
-                    const selectedModels = MODELS.filter((m) =>
+                    const selectedModels = models.filter((m) =>
                       selected.includes(m.id)
                     );
                     const catScores = selectedModels.map(
@@ -185,7 +193,7 @@ export function CompareClient({ scores, trends, latest }: CompareClientProps) {
                     <td className="py-3 px-4 text-sm font-semibold text-foreground">
                       Composite
                     </td>
-                    {MODELS.filter((m) => selected.includes(m.id)).map(
+                    {models.filter((m) => selected.includes(m.id)).map(
                       (model) => {
                         const result = latest.models[model.id];
                         return (

diff --git a/app/compare/page.tsx b/app/compare/page.tsx
@@ -1,14 +1,14 @@
-import { getLatestRun, getTrends } from "@/lib/data";
-import { MODELS } from "@/lib/models";
+import { getAllModels, getLatestRun, getTrends } from "@/lib/data";
 import { CATEGORIES } from "@/lib/categories";
 import { CompareClient } from "./compare-client";
 
 export default function ComparePage() {
+  const models = getAllModels();
   const latest = getLatestRun();
   const trends = getTrends("daily");
 
   const allScores: Record<string, Record<string, number>> = {};
-  for (const model of MODELS) {
+  for (const model of models) {
     allScores[model.id] = {};
     const result = latest.models[model.id];
     if (result) {
@@ -34,6 +34,7 @@ export default function ComparePage() {
         scores={allScores}
         trends={trends}
         latest={latest}
+        models={models}
       />
     </div>
   );

diff --git a/app/evidence/[runId]/[testId]/page.tsx b/app/evidence/[runId]/[testId]/page.tsx
@@ -1,5 +1,4 @@
-import { getEvidence, getAllEvidencePaths } from "@/lib/data";
-import { getModel } from "@/lib/models";
+import { getAllModels, getEvidence, getAllEvidencePaths } from "@/lib/data";
 import { getCategory } from "@/lib/categories";
 import { formatScore, getScoreColor, cn } from "@/lib/utils";
 import { ScrollReveal } from "@/components/shared/scroll-reveal";
@@ -42,6 +41,7 @@ export default async function EvidencePage({
     );
   }
 
+  const models = getAllModels();
   const category = getCategory(evidence.test.category);
 
   return (
@@ -97,7 +97,7 @@ export default async function EvidencePage({
         {Object.entries(evidence.results)
           .sort((a, b) => (b[1].score ?? -1) - (a[1].score ?? -1))
           .map(([modelId, result]) => {
-            const model = getModel(modelId);
+            const model = models.find((m) => m.id === modelId);
             if (!model) return null;
 
             return (

diff --git a/app/layout.tsx b/app/layout.tsx
@@ -7,6 +7,9 @@ export const metadata: Metadata = {
   title: "ModelRegression.com — AI Model Performance Tracker",
   description:
     "Independent automated benchmarking of frontier AI models. Track regressions, compare performance, and see which model is best — updated daily.",
+  icons: {
+    icon: "/images/stick-figure-shooting-graph.svg",
+  },
   openGraph: {
     title: "ModelRegression.com — AI Model Performance Tracker",
     description:
@@ -28,7 +31,7 @@ export default function RootLayout({
 }) {
   return (
     <html lang="en" className="dark">
-      <body className="min-h-screen bg-background antialiased">
+      <body className="min-h-screen overflow-x-hidden bg-background antialiased">
         <div className="relative min-h-screen flex flex-col">
           {/* Background effects */}
           <div className="fixed inset-0 bg-grid pointer-events-none" />