From 88ed3dd755636208fa0d113151f057391575cd75 Mon Sep 17 00:00:00 2001 From: The ComplianceAide <151090230+TheComplianceAide@users.noreply.github.com> Date: Fri, 5 Jun 2026 22:36:11 -0400 Subject: [PATCH 1/2] Add OpenRouter sweeps and computer-use planning tests --- README.md | 52 ++++- app/categories/[slug]/page.tsx | 17 +- app/compare/compare-client.tsx | 24 ++- app/compare/page.tsx | 7 +- app/evidence/[runId]/[testId]/page.tsx | 6 +- app/methodology/page.tsx | 28 +-- app/models/[slug]/page.tsx | 2 + app/outages/page.tsx | 14 +- app/page.tsx | 17 +- benchmark/config.py | 79 ++++++- benchmark/db.py | 75 ++++++- benchmark/export_json.py | 102 ++++++--- benchmark/openrouter_client.py | 142 +++++++++++++ benchmark/openrouter_manifest.py | 112 ++++++++++ benchmark/openrouter_models.py | 155 ++++++++++++++ benchmark/outage_monitor.py | 19 +- benchmark/runner.py | 106 ++++++++-- benchmark/tests/__init__.py | 4 +- benchmark/tests/computer_use.py | 195 ++++++++++++++++++ components/charts/category-radar-chart.tsx | 9 +- components/charts/heatmap-grid.tsx | 9 +- components/charts/performance-line-chart.tsx | 11 +- components/dashboard/latest-run-table.tsx | 8 +- components/dashboard/model-overview-cards.tsx | 7 +- components/dashboard/performance-timeline.tsx | 7 +- components/dashboard/synopsis-banner.tsx | 8 +- docs/research-basis.md | 35 ++++ lib/categories.ts | 9 + lib/data.ts | 10 + lib/models.ts | 1 + lib/types.ts | 9 +- public/data/categories/computer-use.json | 28 +++ public/data/models.json | 34 +++ 33 files changed, 1208 insertions(+), 133 deletions(-) create mode 100644 benchmark/openrouter_client.py create mode 100644 benchmark/openrouter_manifest.py create mode 100644 benchmark/openrouter_models.py create mode 100644 benchmark/tests/computer_use.py create mode 100644 docs/research-basis.md create mode 100644 public/data/categories/computer-use.json create mode 100644 public/data/models.json diff --git a/README.md b/README.md index 5e23e58..f1cfe76 100644 --- a/README.md +++ b/README.md @@ -6,11 +6,11 @@ Independent, automated benchmarking of frontier AI coding models. Tracks perform ## Why This Exists -AI model providers update their models constantly, and sometimes performance degrades without any announcement. ModelRegression runs the same 30 tests against every model daily, scores the results, and surfaces regressions automatically. No vendor self-reporting — just independent, reproducible benchmarks. +AI model providers update their models constantly, and sometimes performance degrades without any announcement. ModelRegression runs the same 33 tests against every model daily, scores the results, and surfaces regressions automatically. No vendor self-reporting — just independent, reproducible benchmarks. ## What Gets Tested -30 tests across 10 categories, each targeting a different dimension of coding ability: +33 tests across 11 categories, each targeting a different dimension of coding and agentic ability: | Category | What It Measures | |---|---| @@ -24,6 +24,7 @@ AI model providers update their models constantly, and sometimes performance deg | Instruction Following | Schema compliance, constraint adherence, multi-step chains | | Code Quality | Idiomatic Python, TypeScript best practices, clean architecture | | Performance Efficiency | Algorithm complexity, streaming, query optimization | +| Computer-Use Planning | Windows/macOS GUI state reasoning, recovery planning, and verified-completion discipline | ## Models Tracked @@ -31,8 +32,11 @@ AI model providers update their models constantly, and sometimes performance deg - **Claude Sonnet 4.6** (Anthropic) — via `claude` CLI - **GPT-5.5** (OpenAI) — via `codex` CLI - **Grok** (xAI) — via `agent` CLI +- **OpenRouter models** (open-weight/open-source candidates) - opt-in via `OPENROUTER_MODEL_IDS` or a pinned manifest -Models are tested through their official CLI tools, not direct API calls. This tests the full stack that developers actually use. +Default frontier models are tested through their official CLI tools. OpenRouter models are tested through OpenRouter's chat-completions API so the same suite can cover dozens of open-weight and API-hosted models. + +See [docs/research-basis.md](docs/research-basis.md) for the benchmark design rationale, including recent 2026 computer-use and cybersecurity benchmark references. ## Architecture @@ -42,8 +46,8 @@ Models are tested through their official CLI tools, not direct API calls. This t │ (cron job) │ Python 3.13 + SQLite └──────┬───────┘ │ - benchmark suite runs 30 tests - against each model via CLI tools + benchmark suite runs 33 tests + against each model via configured adapters │ ▼ ┌──────────────┐ @@ -107,12 +111,12 @@ modelregression/ │ ├── regression_detector.py # Regression detection logic │ ├── outage_monitor.py # Health checks + outage tracking │ ├── run_benchmarks.sh # Cron entry point (full pipeline) -│ └── tests/ # Test implementations (30 tests) +│ └── tests/ # Test implementations (33 tests) │ ├── base.py # Base test class │ ├── long_reasoning.py # 3 long reasoning tests │ ├── coding_tasks.py # 3 coding task tests │ ├── bug_fixes.py # 3 bug fix tests -│ └── ... # (10 categories x 3 tests each) +│ └── ... # (11 categories x 3 tests each) ├── config/ # Nginx configuration ├── deploy.sh # Blue-green atomic deployment └── ecosystem.config.js # PM2 process configuration @@ -156,12 +160,44 @@ python export_json.py --output ../public/data ### CLI Tool Prerequisites -The benchmark engine calls models through their official CLI tools. You need these installed and authenticated: +The benchmark engine calls default frontier models through their official CLI tools. You need these installed and authenticated: - **[Claude Code](https://docs.anthropic.com/en/docs/claude-code)** (`claude`) — for Anthropic models - **[Codex CLI](https://github.com/openai/codex)** (`codex`) — for OpenAI models - **[Grok Agent](https://docs.x.ai/docs/grok-agent)** (`agent`) — for xAI models +### OpenRouter Model Sweeps + +OpenRouter support is opt-in so daily runs do not suddenly benchmark dozens of additional models or incur unexpected cost. Broad sweeps should be pinned in a manifest first; benchmark runs do not fetch the live OpenRouter catalog during config import. + +```powershell +# Controlled OpenRouter run against explicit model IDs +$env:OPENROUTER_API_KEY = "" +$env:OPENROUTER_MODEL_IDS = "meta-llama/llama-3.3-70b-instruct,qwen/qwen3.7-plus" +py -3 benchmark\runner.py --schedule daily + +# Generate and review a broad open-weight candidate manifest +$env:OPENROUTER_API_KEY = "" +py -3 benchmark\openrouter_manifest.py --limit 50 --output benchmark\manifests\openrouter_models.json + +# Run from the pinned manifest +$env:OPENROUTER_MANIFEST = "benchmark\manifests\openrouter_models.json" +py -3 benchmark\runner.py --schedule daily + +# Optional: only include OpenRouter :free variants while generating the manifest +py -3 benchmark\openrouter_manifest.py --free-only --limit 50 --output benchmark\manifests\openrouter_models.json +``` + +The manifest generator currently matches provider/model prefixes such as `meta-llama/`, `mistralai/`, `qwen/`, `deepseek/`, `google/gemma`, `nvidia/`, `microsoft/`, and other open-weight candidate families. Treat these as open-weight candidates until licenses are verified. Override with repeated `--prefix` flags when the catalog changes. + +Useful safety controls: + +```powershell +$env:OPENROUTER_MAX_MODELS = "50" # fail fast if the manifest is larger +$env:MAX_PARALLEL_MODELS = "4" # global model-level workers +$env:OPENROUTER_PARALLEL_TESTS = "1" # per-OpenRouter-model test workers +``` + ### Deployment Deployment uses a blue-green strategy with zero-downtime swaps: diff --git a/app/categories/[slug]/page.tsx b/app/categories/[slug]/page.tsx index 7a9b806..f2c7e28 100644 --- a/app/categories/[slug]/page.tsx +++ b/app/categories/[slug]/page.tsx @@ -1,8 +1,7 @@ import { notFound } from "next/navigation"; import Link from "next/link"; import type { Metadata } from "next"; -import { getCategoryDetail, getAllCategorySlugs } from "@/lib/data"; -import { getModel } from "@/lib/models"; +import { getAllModels, getCategoryDetail, getAllCategorySlugs } from "@/lib/data"; import { cn, formatScore, @@ -83,12 +82,13 @@ export default async function CategoryDetailPage({ notFound(); } - const { category, tests, models } = detail; + const modelInfos = getAllModels(); + const { category, tests, models: categoryModels } = detail; // ---- Build line chart data from all models' history ---- // Collect all unique timestamps const timestampSet = new Set(); - for (const [, modelData] of Object.entries(models)) { + for (const [, modelData] of Object.entries(categoryModels)) { for (const h of modelData.history) { timestampSet.add(h.timestamp); } @@ -97,7 +97,7 @@ export default async function CategoryDetailPage({ // Build lookup maps for fast access const modelHistoryMaps: Record> = {}; - for (const [modelId, modelData] of Object.entries(models)) { + for (const [modelId, modelData] of Object.entries(categoryModels)) { const map = new Map(); for (const h of modelData.history) { map.set(h.timestamp, h.score); @@ -110,7 +110,7 @@ export default async function CategoryDetailPage({ timestamp: ts, models: {}, }; - for (const modelId of Object.keys(models)) { + for (const modelId of Object.keys(categoryModels)) { const val = modelHistoryMaps[modelId].get(ts); if (val !== undefined) { point.models[modelId] = val; @@ -120,10 +120,10 @@ export default async function CategoryDetailPage({ }); // ---- Rank models by current score ---- - const rankedModels = Object.entries(models) + const rankedModels = Object.entries(categoryModels) .map(([modelId, data]) => ({ modelId, - info: getModel(modelId), + info: modelInfos.find((m) => m.id === modelId), score: data.currentScore, sparkline: data.history.slice(-7).map((h) => h.score), })) @@ -234,6 +234,7 @@ export default async function CategoryDetailPage({ data={chartData} height={400} showLegend={true} + models={modelInfos} /> diff --git a/app/compare/compare-client.tsx b/app/compare/compare-client.tsx index a2b569c..6f547de 100644 --- a/app/compare/compare-client.tsx +++ b/app/compare/compare-client.tsx @@ -2,24 +2,30 @@ import { useState } from "react"; import { cn } from "@/lib/utils"; -import { MODELS } from "@/lib/models"; import { CATEGORIES } from "@/lib/categories"; import { formatScore, getScoreColor } from "@/lib/utils"; import { CategoryRadarChart } from "@/components/charts/category-radar-chart"; import { PerformanceLineChart } from "@/components/charts/performance-line-chart"; import { ScrollReveal } from "@/components/shared/scroll-reveal"; import { Check } from "lucide-react"; -import type { TrendData, LatestRun } from "@/lib/types"; +import type { TrendData, LatestRun, ModelInfo } from "@/lib/types"; interface CompareClientProps { scores: Record>; trends: TrendData; latest: LatestRun; + models: ModelInfo[]; } -export function CompareClient({ scores, trends, latest }: CompareClientProps) { +export function CompareClient({ scores, trends, latest, models }: CompareClientProps) { const [selected, setSelected] = useState( - MODELS.map((m) => m.id) + models + .filter((m) => latest.models[m.id]) + .sort((a, b) => + (latest.models[a.id]?.rank ?? 999) - (latest.models[b.id]?.rank ?? 999) + ) + .slice(0, Math.min(models.length, 6)) + .map((m) => m.id) ); const toggle = (id: string) => { @@ -42,7 +48,7 @@ export function CompareClient({ scores, trends, latest }: CompareClientProps) { Select Models to Compare
- {MODELS.map((model) => { + {models.map((model) => { const isSelected = selected.includes(model.id); return (
@@ -101,6 +108,7 @@ export function CompareClient({ scores, trends, latest }: CompareClientProps) { @@ -123,7 +131,7 @@ export function CompareClient({ scores, trends, latest }: CompareClientProps) { Category - {MODELS.filter((m) => selected.includes(m.id)).map( + {models.filter((m) => selected.includes(m.id)).map( (model) => ( {CATEGORIES.map((cat) => { - const selectedModels = MODELS.filter((m) => + const selectedModels = models.filter((m) => selected.includes(m.id) ); const catScores = selectedModels.map( @@ -185,7 +193,7 @@ export function CompareClient({ scores, trends, latest }: CompareClientProps) { Composite - {MODELS.filter((m) => selected.includes(m.id)).map( + {models.filter((m) => selected.includes(m.id)).map( (model) => { const result = latest.models[model.id]; return ( diff --git a/app/compare/page.tsx b/app/compare/page.tsx index 180df46..3239341 100644 --- a/app/compare/page.tsx +++ b/app/compare/page.tsx @@ -1,14 +1,14 @@ -import { getLatestRun, getTrends } from "@/lib/data"; -import { MODELS } from "@/lib/models"; +import { getAllModels, getLatestRun, getTrends } from "@/lib/data"; import { CATEGORIES } from "@/lib/categories"; import { CompareClient } from "./compare-client"; export default function ComparePage() { + const models = getAllModels(); const latest = getLatestRun(); const trends = getTrends("daily"); const allScores: Record> = {}; - for (const model of MODELS) { + for (const model of models) { allScores[model.id] = {}; const result = latest.models[model.id]; if (result) { @@ -34,6 +34,7 @@ export default function ComparePage() { scores={allScores} trends={trends} latest={latest} + models={models} /> ); diff --git a/app/evidence/[runId]/[testId]/page.tsx b/app/evidence/[runId]/[testId]/page.tsx index 1fa33e0..c0b49e1 100644 --- a/app/evidence/[runId]/[testId]/page.tsx +++ b/app/evidence/[runId]/[testId]/page.tsx @@ -1,5 +1,4 @@ -import { getEvidence, getAllEvidencePaths } from "@/lib/data"; -import { getModel } from "@/lib/models"; +import { getAllModels, getEvidence, getAllEvidencePaths } from "@/lib/data"; import { getCategory } from "@/lib/categories"; import { formatScore, getScoreColor, cn } from "@/lib/utils"; import { ScrollReveal } from "@/components/shared/scroll-reveal"; @@ -42,6 +41,7 @@ export default async function EvidencePage({ ); } + const models = getAllModels(); const category = getCategory(evidence.test.category); return ( @@ -97,7 +97,7 @@ export default async function EvidencePage({ {Object.entries(evidence.results) .sort((a, b) => (b[1].score ?? -1) - (a[1].score ?? -1)) .map(([modelId, result]) => { - const model = getModel(modelId); + const model = models.find((m) => m.id === modelId); if (!model) return null; return ( diff --git a/app/methodology/page.tsx b/app/methodology/page.tsx index ad3b5f8..911a309 100644 --- a/app/methodology/page.tsx +++ b/app/methodology/page.tsx @@ -1,5 +1,5 @@ import { CATEGORIES } from "@/lib/categories"; -import { MODELS } from "@/lib/models"; +import { getAllModels } from "@/lib/data"; import { ScrollReveal } from "@/components/shared/scroll-reveal"; import { Brain, @@ -12,6 +12,7 @@ import { ListChecks, Star, Gauge, + Monitor, Timer, Server, BarChart3, @@ -29,9 +30,12 @@ const ICON_MAP: Record = { "list-checks": ListChecks, star: Star, gauge: Gauge, + monitor: Monitor, }; export default function MethodologyPage() { + const models = getAllModels(); + return (
@@ -56,7 +60,7 @@ export default function MethodologyPage() {
- 30 + 33

Individual tests per run @@ -64,7 +68,7 @@ export default function MethodologyPage() {

- 10 + 11

Benchmark categories @@ -90,7 +94,7 @@ export default function MethodologyPage() { Models Tested

- {MODELS.map((model) => ( + {models.map((model) => (

- All models are called via their official APIs with temperature set to - 0.0 for maximum reproducibility. When a new frontier model is - released, it is added to the benchmark suite and begins tracking - immediately. + Models are called through their configured adapters, including + official CLIs and OpenRouter's chat-completions API. Temperature is + set to 0.0 where the adapter supports it.

@@ -203,9 +206,10 @@ export default function MethodologyPage() {

Composite scores are - weighted averages across all 10 categories. Weights reflect the - relative importance of each capability (coding and security are - weighted higher than instruction following). + weighted averages across all 11 categories. Weights reflect the + relative importance of each capability. Current category weights + are equal until enough run history exists to justify differentiated + weighting.

@@ -263,7 +267,7 @@ export default function MethodologyPage() { geographic location.
  • - 30 tests cannot cover all capabilities. Results indicate trends, + 33 tests cannot cover all capabilities. Results indicate trends, not absolute quality.
  • diff --git a/app/models/[slug]/page.tsx b/app/models/[slug]/page.tsx index abef586..ea37eae 100644 --- a/app/models/[slug]/page.tsx +++ b/app/models/[slug]/page.tsx @@ -226,6 +226,7 @@ export default async function ModelDetailPage({
    • @@ -243,6 +244,7 @@ export default async function ModelDetailPage({ diff --git a/app/outages/page.tsx b/app/outages/page.tsx index b2c68c6..98ab748 100644 --- a/app/outages/page.tsx +++ b/app/outages/page.tsx @@ -10,7 +10,7 @@ import { WifiOff, } from "lucide-react"; -const PROVIDERS = ["claude", "codex", "agent"] as const; +const BASE_PROVIDERS = ["claude", "codex", "agent", "openrouter"]; function getStatusInfo(uptime: number) { if (uptime >= 99.9) @@ -37,6 +37,14 @@ function getStatusInfo(uptime: number) { export default function OutagesPage() { const outages = getOutages(); + const providers = Array.from( + new Set([ + ...BASE_PROVIDERS, + ...Object.keys(outages.uptime), + ...outages.current.map((o) => o.provider), + ...outages.history.map((o) => o.provider), + ]) + ); return (
    @@ -51,8 +59,8 @@ export default function OutagesPage() { {/* Current Status */} -
    - {PROVIDERS.map((provider) => { +
    + {providers.map((provider) => { const uptime7d = outages.uptime[provider]?.["7d"] ?? 100; const uptime30d = outages.uptime[provider]?.["30d"] ?? 100; const uptime90d = outages.uptime[provider]?.["90d"] ?? 100; diff --git a/app/page.tsx b/app/page.tsx index 95f5122..d71bc53 100644 --- a/app/page.tsx +++ b/app/page.tsx @@ -1,5 +1,4 @@ -import { getLatestRun, getSynopsis, getRegressions, getTrends } from "@/lib/data"; -import { MODELS } from "@/lib/models"; +import { getAllModels, getLatestRun, getSynopsis, getRegressions, getTrends } from "@/lib/data"; import { CATEGORIES } from "@/lib/categories"; import { SynopsisBanner } from "@/components/dashboard/synopsis-banner"; import { ModelOverviewCards } from "@/components/dashboard/model-overview-cards"; @@ -14,9 +13,10 @@ export default function HomePage() { const synopsis = getSynopsis(); const regressions = getRegressions(); const trends = getTrends("daily"); + const models = getAllModels(); const heatmapScores: Record> = {}; - for (const model of MODELS) { + for (const model of models) { heatmapScores[model.id] = {}; const modelResult = latest.models[model.id]; if (modelResult) { @@ -29,7 +29,7 @@ export default function HomePage() { const modelHistory: Record = {}; const modelChanges: Record = {}; - for (const model of MODELS) { + for (const model of models) { const allPoints = trends.data .slice(-21) .map((d) => d.models[model.id] ?? null); @@ -47,12 +47,13 @@ export default function HomePage() { return (
    - + {/* Performance Timeline */} @@ -61,7 +62,7 @@ export default function HomePage() {

    Performance Timeline

    - +
    @@ -74,12 +75,12 @@ export default function HomePage() {

    Category Performance Heatmap

    - +
    {/* Latest Run Table */} - +
    ); } diff --git a/benchmark/config.py b/benchmark/config.py index 3e4153b..2e44724 100644 --- a/benchmark/config.py +++ b/benchmark/config.py @@ -1,13 +1,15 @@ """ Configuration for ModelRegression benchmark engine. -Defines all models, categories, tests, and CLI tool settings. -Models are tested via installed CLI tools (claude, codex, agent) -rather than direct API integration. +Defines all models, categories, tests, and model-call settings. +The default frontier models are tested via installed CLI tools. OpenRouter +models can be added through explicit IDs or a pinned local manifest. """ import os +from openrouter_models import load_openrouter_models_from_sources + # --------------------------------------------------------------------------- # Database # --------------------------------------------------------------------------- @@ -19,16 +21,32 @@ CLI_TIMEOUT = 300 HEALTH_CHECK_TIMEOUT = 60 PARALLEL_TESTS = 4 +MAX_PARALLEL_MODELS = int(os.getenv("MAX_PARALLEL_MODELS", "4")) +OPENROUTER_PARALLEL_TESTS = int(os.getenv("OPENROUTER_PARALLEL_TESTS", "1")) +OPENROUTER_MAX_MODELS = int(os.getenv("OPENROUTER_MAX_MODELS", "50")) MAX_RETRIES = 2 RETRY_DELAY = 5 +# --------------------------------------------------------------------------- +# OpenRouter model selection +# +# Set OPENROUTER_MODEL_IDS to a comma-separated list for controlled runs: +# OPENROUTER_MODEL_IDS=meta-llama/llama-3.3-70b-instruct,qwen/qwen3.7-plus +# +# Or generate a pinned manifest first: +# python benchmark/openrouter_manifest.py --limit 50 +# +# Then set OPENROUTER_MANIFEST or keep the default +# benchmark/manifests/openrouter_models.json path. +# --------------------------------------------------------------------------- + # --------------------------------------------------------------------------- # Model definitions — each model specifies its CLI tool and model argument # -# cli: executable name (claude, codex, agent) -# cli_model: value passed to --model / -m flag +# cli: adapter/executable name (claude, codex, agent, openrouter) +# cli_model: value passed to --model / -m flag, or OpenRouter model id # --------------------------------------------------------------------------- -MODELS = [ +BASE_MODELS = [ { "id": "claude-opus-4-8", "cli": "claude", @@ -71,6 +89,16 @@ }, ] +MODELS = BASE_MODELS + load_openrouter_models_from_sources() + +_OPENROUTER_MODEL_COUNT = sum(1 for m in MODELS if m["cli"] == "openrouter") +if OPENROUTER_MAX_MODELS > 0 and _OPENROUTER_MODEL_COUNT > OPENROUTER_MAX_MODELS: + raise RuntimeError( + f"Configured {_OPENROUTER_MODEL_COUNT} OpenRouter models, above " + f"OPENROUTER_MAX_MODELS={OPENROUTER_MAX_MODELS}. Increase the cap " + "deliberately or reduce the manifest." + ) + # --------------------------------------------------------------------------- # Category definitions (all weights equal at 1) # --------------------------------------------------------------------------- @@ -165,10 +193,19 @@ "weight": 1, "sort_order": 10, }, + { + "id": "computer-use", + "name": "Computer-Use Planning", + "slug": "computer-use", + "description": "Measures desktop-agent planning and verification discipline for Windows/macOS workflows. This is a text-evaluated proxy until a live GUI harness is added.", + "icon": "monitor", + "weight": 1, + "sort_order": 11, + }, ] # --------------------------------------------------------------------------- -# Test definitions (3 per category = 30 total) +# Test definitions (3 per category = 33 total) # --------------------------------------------------------------------------- TESTS = [ # Long Reasoning @@ -451,6 +488,34 @@ "max_score": 100, "version": 1, }, + # Computer Use + { + "id": "cu-1", + "category_id": "computer-use", + "name": "Windows Desktop Workflow Planning", + "description": "Plan and verify a Windows multi-window task with modal and identity recovery.", + "eval_type": "llm_judge", + "max_score": 100, + "version": 1, + }, + { + "id": "cu-2", + "category_id": "computer-use", + "name": "macOS Cross-App Workflow Planning", + "description": "Plan a macOS Finder/browser/document workflow while preserving user intent.", + "eval_type": "llm_judge", + "max_score": 100, + "version": 1, + }, + { + "id": "cu-3", + "category_id": "computer-use", + "name": "Desktop Verification Discipline", + "description": "Distinguish observed GUI state from assumptions and verify completion before claiming success.", + "eval_type": "llm_judge", + "max_score": 100, + "version": 1, + }, ] # --------------------------------------------------------------------------- diff --git a/benchmark/db.py b/benchmark/db.py index 1cefbb6..73336dc 100644 --- a/benchmark/db.py +++ b/benchmark/db.py @@ -47,6 +47,7 @@ def get_connection(db_path: str | None = None) -> sqlite3.Connection: slug TEXT NOT NULL UNIQUE, cli_model TEXT NOT NULL, color TEXT NOT NULL, + icon TEXT NOT NULL DEFAULT 'cpu', is_active INTEGER NOT NULL DEFAULT 1 ); @@ -83,6 +84,19 @@ def get_connection(db_path: str | None = None) -> sqlite3.Connection: error_log TEXT ); +CREATE TABLE IF NOT EXISTS run_models ( + run_id TEXT NOT NULL REFERENCES benchmark_runs(id), + model_id TEXT NOT NULL, + provider TEXT NOT NULL, + name TEXT NOT NULL, + slug TEXT NOT NULL, + cli_model TEXT NOT NULL, + color TEXT NOT NULL, + icon TEXT NOT NULL DEFAULT 'cpu', + config_json TEXT, + PRIMARY KEY (run_id, model_id) +); + CREATE TABLE IF NOT EXISTS test_results ( id TEXT PRIMARY KEY, run_id TEXT NOT NULL REFERENCES benchmark_runs(id), @@ -150,6 +164,7 @@ def get_connection(db_path: str | None = None) -> sqlite3.Connection: CREATE INDEX IF NOT EXISTS idx_test_results_model ON test_results(model_id); CREATE INDEX IF NOT EXISTS idx_run_scores_run ON run_scores(run_id); CREATE INDEX IF NOT EXISTS idx_model_run_scores_run ON model_run_scores(run_id); +CREATE INDEX IF NOT EXISTS idx_run_models_run ON run_models(run_id); CREATE INDEX IF NOT EXISTS idx_regressions_model ON regressions(model_id); CREATE INDEX IF NOT EXISTS idx_outages_model ON outages(model_id); CREATE INDEX IF NOT EXISTS idx_benchmark_runs_started ON benchmark_runs(started_at); @@ -160,20 +175,40 @@ def init_db(db_path: str | None = None) -> sqlite3.Connection: """Create all tables and return the connection.""" conn = get_connection(db_path) conn.executescript(_SCHEMA) + _ensure_column(conn, "models", "icon", "TEXT NOT NULL DEFAULT 'cpu'") conn.commit() logger.info("Database initialized at %s", db_path or DB_PATH) return conn +def _ensure_column(conn: sqlite3.Connection, table: str, column: str, ddl: str) -> None: + existing = { + row["name"] + for row in conn.execute(f"PRAGMA table_info({table})").fetchall() + } + if column not in existing: + conn.execute(f"ALTER TABLE {table} ADD COLUMN {column} {ddl}") + + # ---- Seeding --------------------------------------------------------------- def seed_models_and_categories(conn: sqlite3.Connection) -> None: """Insert or update model and category metadata from config.""" + current_ids = [m["id"] for m in MODELS] + if current_ids: + placeholders = ",".join("?" for _ in current_ids) + conn.execute( + f"UPDATE models SET is_active = 0 WHERE id NOT IN ({placeholders})", + current_ids, + ) for m in MODELS: conn.execute( - """INSERT OR REPLACE INTO models (id, provider, name, slug, cli_model, color, is_active) - VALUES (?, ?, ?, ?, ?, ?, ?)""", - (m["id"], m["cli"], m["name"], m["slug"], m["cli_model"], m["color"], int(m["is_active"])), + """INSERT OR REPLACE INTO models (id, provider, name, slug, cli_model, color, icon, is_active) + VALUES (?, ?, ?, ?, ?, ?, ?, ?)""", + ( + m["id"], m["cli"], m["name"], m["slug"], m["cli_model"], + m["color"], m.get("icon", "cpu"), int(m["is_active"]), + ), ) for c in CATEGORIES: conn.execute( @@ -206,6 +241,40 @@ def create_run(conn: sqlite3.Connection, schedule: str) -> str: return run_id +def save_run_model_manifest(conn: sqlite3.Connection, run_id: str, models: list[dict]) -> None: + """Persist the exact model set selected for a benchmark run.""" + for m in models: + conn.execute( + """INSERT OR REPLACE INTO run_models + (run_id, model_id, provider, name, slug, cli_model, color, icon, config_json) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)""", + ( + run_id, + m["id"], + m["cli"], + m["name"], + m["slug"], + m["cli_model"], + m["color"], + m.get("icon", "cpu"), + json.dumps(m, default=str), + ), + ) + conn.commit() + + +def get_run_model_manifest(conn: sqlite3.Connection, run_id: str) -> list[dict]: + """Return model metadata saved for a benchmark run.""" + rows = conn.execute( + """SELECT model_id as id, provider, name, slug, cli_model, color, icon, config_json + FROM run_models + WHERE run_id = ? + ORDER BY name""", + (run_id,), + ).fetchall() + return [dict(r) for r in rows] + + def complete_run(conn: sqlite3.Connection, run_id: str, total_tests: int, passed_tests: int, error_log: str | None = None) -> None: """Mark a run as completed.""" status = "completed" if error_log is None else "completed_with_errors" diff --git a/benchmark/export_json.py b/benchmark/export_json.py index 992440c..211a81d 100644 --- a/benchmark/export_json.py +++ b/benchmark/export_json.py @@ -7,6 +7,7 @@ - latest.json (most recent benchmark run) - synopsis.json (best model per day/week/month) + - models.json (all configured model metadata) - models/{slug}.json (per-model detail + history) - categories/{slug}.json (per-category detail + history) - trends/daily.json|weekly|monthly|yearly (composite trend lines) @@ -24,6 +25,7 @@ import json import logging import os +import sqlite3 import sys from collections import defaultdict from datetime import datetime, timedelta, timezone @@ -105,6 +107,56 @@ def _write_json(path: str, data) -> None: logger.debug("Wrote %s", path) +def _model_meta(mcfg: dict) -> dict: + return { + "id": mcfg["id"], + "provider": mcfg.get("provider") or mcfg["cli"], + "name": mcfg["name"], + "slug": mcfg["slug"], + "color": mcfg["color"], + "icon": mcfg.get("icon", "cpu"), + } + + +def _model_meta_from_row(row) -> dict: + data = dict(row) + return { + "id": data["id"], + "provider": data.get("provider") or data.get("cli") or "unknown", + "name": data["name"], + "slug": data["slug"], + "color": data["color"], + "icon": data.get("icon") or "cpu", + } + + +def _export_model_configs(conn) -> list[dict]: + """Models that should appear in static JSON, based on DB state when possible.""" + try: + rows = conn.execute( + """SELECT DISTINCT m.* + FROM models m + WHERE m.is_active = 1 + OR EXISTS (SELECT 1 FROM model_run_scores mrs WHERE mrs.model_id = m.id) + OR EXISTS (SELECT 1 FROM run_models rm WHERE rm.model_id = m.id) + ORDER BY m.name""" + ).fetchall() + except sqlite3.OperationalError: + rows = [] + + if rows: + return [dict(r) for r in rows] + return [dict(m) for m in config.MODELS] + + +def export_model_index(output_dir: str, conn=None) -> None: + """Model metadata consumed by the Next.js UI.""" + models = _export_model_configs(conn) if conn is not None else config.MODELS + _write_json(os.path.join(output_dir, "models.json"), [ + _model_meta(mcfg) for mcfg in models + ]) + + # --------------------------------------------------------------------------- # latest.json # --------------------------------------------------------------------------- @@ -140,7 +192,7 @@ def export_latest(conn, output_dir: str) -> None: models_data = {} all_composites = [] - for mcfg in config.MODELS: + for mcfg in _export_model_configs(conn): mid = mcfg["id"] slug = mcfg["slug"] model_run = _get_latest_run_for_model(conn, mid) @@ -290,18 +342,11 @@ def export_models(conn, output_dir: str) -> None: model_results = {} all_composites = [] - for mcfg in config.MODELS: + for mcfg in _export_model_configs(conn): mid = mcfg["id"] slug = mcfg["slug"] - model_meta = { - "id": mid, - "provider": mcfg["cli"], - "name": mcfg["name"], - "slug": slug, - "color": mcfg["color"], - "icon": mcfg.get("icon", "cpu"), - } + model_meta = _model_meta(mcfg) # --- current (from this model's latest run) --- model_run = _get_latest_run_for_model(conn, mid) @@ -493,7 +538,7 @@ def export_categories(conn, output_dir: str) -> None: # Per-model current score + history models_data = {} - for mcfg in config.MODELS: + for mcfg in _export_model_configs(conn): mid = mcfg["id"] mslug = mcfg["slug"] @@ -733,8 +778,8 @@ def export_outages(conn, output_dir: str) -> None: def _compute_uptime(conn) -> dict: """Per-provider uptime for 7d / 30d / 90d windows.""" providers = {} - for mcfg in config.MODELS: - p = mcfg["cli"] + for mcfg in _export_model_configs(conn): + p = mcfg.get("provider") or mcfg.get("cli") if p not in providers: providers[p] = {} @@ -839,8 +884,9 @@ def export_evidence(conn, output_dir: str) -> None: # Empty stubs (no data yet) # --------------------------------------------------------------------------- -def _generate_empty_stubs(output_dir: str) -> None: +def _generate_empty_stubs(output_dir: str, conn=None) -> None: """Write minimal valid JSON so the Next.js site still builds.""" + export_model_index(output_dir, conn) _write_json(os.path.join(output_dir, "latest.json"), { "runId": None, "startedAt": None, "completedAt": None, "schedule": None, "models": {}, @@ -861,13 +907,10 @@ def _generate_empty_stubs(output_dir: str) -> None: _write_json(os.path.join(output_dir, "trends", f"{period}.json"), { "period": period, "data": [], }) - for mcfg in config.MODELS: + empty_models = _export_model_configs(conn) if conn is not None else config.MODELS + for mcfg in empty_models: _write_json(os.path.join(output_dir, "models", f"{mcfg['slug']}.json"), { - "model": { - "id": mcfg["id"], "provider": mcfg["cli"], - "name": mcfg["name"], "slug": mcfg["slug"], - "color": mcfg["color"], "icon": mcfg.get("icon", "cpu"), - }, + "model": _model_meta(mcfg), "current": {"compositeScore": None, "rank": None, "categories": {}}, "history": [], "regressions": [], "outages": [], }) @@ -895,20 +938,29 @@ def export_all(output_dir: str, db_path: str | None = None) -> None: conn = database.get_connection(db_path) # Check for any completed runs - row = conn.execute( - "SELECT COUNT(*) as cnt FROM benchmark_runs WHERE status IN ('completed', 'completed_with_errors')" - ).fetchone() - has_data = dict(row)["cnt"] > 0 + try: + row = conn.execute( + "SELECT COUNT(*) as cnt FROM benchmark_runs WHERE status IN ('completed', 'completed_with_errors')" + ).fetchone() + has_data = dict(row)["cnt"] > 0 + except sqlite3.OperationalError as e: + if "no such table" not in str(e): + conn.close() + raise + logger.warning("Benchmark database schema is missing -- writing empty stubs.") + has_data = False if not has_data: logger.warning("No completed runs in database -- writing empty stubs.") - _generate_empty_stubs(output_dir) + _generate_empty_stubs(output_dir, conn) conn.close() return logger.info("Exporting JSON data to %s ...", output_dir) os.makedirs(output_dir, exist_ok=True) + export_model_index(output_dir, conn) + export_latest(conn, output_dir) logger.info(" latest.json") diff --git a/benchmark/openrouter_client.py b/benchmark/openrouter_client.py new file mode 100644 index 0000000..1137cfe --- /dev/null +++ b/benchmark/openrouter_client.py @@ -0,0 +1,142 @@ +""" +Small stdlib OpenRouter client for benchmark calls and model discovery. + +OpenRouter exposes an OpenAI-compatible chat-completions endpoint and a model +catalog endpoint. Keeping this dependency-free preserves the current benchmark +engine deployment model. +""" + +from __future__ import annotations + +import json +import os +import time +import urllib.error +import urllib.parse +import urllib.request +from typing import Any + + +API_BASE = os.getenv("OPENROUTER_API_BASE", "https://openrouter.ai/api/v1") +DEFAULT_REFERER = "https://modelregression.com" +DEFAULT_TITLE = "ModelRegression.com" + + +def _headers(include_auth: bool = True) -> dict[str, str]: + headers = { + "Content-Type": "application/json", + "HTTP-Referer": os.getenv("OPENROUTER_HTTP_REFERER", DEFAULT_REFERER), + "X-Title": os.getenv("OPENROUTER_APP_TITLE", DEFAULT_TITLE), + "User-Agent": "modelregression-benchmark/1.0", + } + api_key = os.getenv("OPENROUTER_API_KEY") + if include_auth and api_key: + headers["Authorization"] = f"Bearer {api_key}" + return headers + + +def fetch_models_catalog(timeout: int = 30) -> list[dict[str, Any]]: + """Return OpenRouter's current model catalog.""" + query = urllib.parse.urlencode({"output_modalities": "text"}) + url = f"{API_BASE.rstrip('/')}/models?{query}" + request = urllib.request.Request(url, headers=_headers(include_auth=True)) + with urllib.request.urlopen(request, timeout=timeout) as response: + payload = json.loads(response.read().decode("utf-8")) + return list(payload.get("data") or []) + + +def _content_to_text(content: Any) -> str: + if isinstance(content, str): + return content + if isinstance(content, list): + parts = [] + for item in content: + if isinstance(item, dict): + if item.get("type") == "text" and isinstance(item.get("text"), str): + parts.append(item["text"]) + elif isinstance(item.get("content"), str): + parts.append(item["content"]) + elif isinstance(item, str): + parts.append(item) + return "\n".join(parts) + return "" if content is None else str(content) + + +def call_chat_completion( + model_id: str, + prompt: str, + system_prompt: str | None = None, + timeout: int = 300, + supported_parameters: list[str] | None = None, +) -> tuple[str | None, int | None, int | None, int | None, str | None, int | None]: + """ + Call OpenRouter chat completions. + + Returns (text, latency_ms, prompt_tokens, completion_tokens, error, http_status). + """ + api_key = os.getenv("OPENROUTER_API_KEY") + if not api_key: + return (None, None, None, None, "OPENROUTER_API_KEY is not set", None) + + messages = [] + if system_prompt: + messages.append({"role": "system", "content": system_prompt}) + messages.append({"role": "user", "content": prompt}) + + body = { + "model": model_id, + "messages": messages, + } + supported = set(supported_parameters or []) + if supported_parameters is None or "temperature" in supported: + body["temperature"] = float(os.getenv("OPENROUTER_TEMPERATURE", "0")) + if supported_parameters is None or "max_tokens" in supported: + body["max_tokens"] = int(os.getenv("OPENROUTER_MAX_TOKENS", "4096")) + + url = f"{API_BASE.rstrip('/')}/chat/completions" + request = urllib.request.Request( + url, + data=json.dumps(body).encode("utf-8"), + headers=_headers(include_auth=True), + method="POST", + ) + + start = time.monotonic() + try: + with urllib.request.urlopen(request, timeout=timeout) as response: + elapsed = int((time.monotonic() - start) * 1000) + payload = json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as e: + elapsed = int((time.monotonic() - start) * 1000) + try: + err_body = e.read().decode("utf-8")[:600] + except Exception: + err_body = e.reason or "HTTP error" + return (None, elapsed, None, None, f"OpenRouter HTTP {e.code}: {err_body}", e.code) + except urllib.error.URLError as e: + elapsed = int((time.monotonic() - start) * 1000) + return (None, elapsed, None, None, f"OpenRouter URL error: {e.reason}", None) + except TimeoutError: + return (None, timeout * 1000, None, None, "OpenRouter request timed out", None) + except Exception as e: + elapsed = int((time.monotonic() - start) * 1000) + return (None, elapsed, None, None, f"{type(e).__name__}: {e}", None) + + choices = payload.get("choices") or [] + if not choices: + return (None, elapsed, None, None, "OpenRouter returned no choices", None) + + message = choices[0].get("message") or {} + text = _content_to_text(message.get("content")).strip() + if not text: + return (None, elapsed, None, None, "Empty response from OpenRouter", None) + + usage = payload.get("usage") or {} + return ( + text, + elapsed, + usage.get("prompt_tokens"), + usage.get("completion_tokens"), + None, + None, + ) diff --git a/benchmark/openrouter_manifest.py b/benchmark/openrouter_manifest.py new file mode 100644 index 0000000..5e456d3 --- /dev/null +++ b/benchmark/openrouter_manifest.py @@ -0,0 +1,112 @@ +#!/usr/bin/env python3 +""" +Generate a pinned OpenRouter model manifest. + +This command is the supported way to prepare broad OpenRouter sweeps. Benchmark +configuration reads the manifest later without doing network discovery at import +time. +""" + +from __future__ import annotations + +import argparse +import json +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +from openrouter_client import fetch_models_catalog +from openrouter_models import ( + DEFAULT_MANIFEST_PATH, + DEFAULT_OPEN_WEIGHT_PREFIXES, + is_open_weight_candidate, +) + + +def _model_entry(model: dict[str, Any]) -> dict[str, Any]: + keys = [ + "id", + "canonical_slug", + "name", + "created", + "description", + "context_length", + "architecture", + "pricing", + "top_provider", + "per_request_limits", + "supported_parameters", + "default_parameters", + "expiration_date", + ] + return {k: model.get(k) for k in keys if k in model} + + +def build_manifest(args: argparse.Namespace) -> dict[str, Any]: + catalog = fetch_models_catalog(timeout=args.timeout) + by_id = {str(m.get("id", "")): m for m in catalog if m.get("id")} + + selected: list[dict[str, Any]] = [] + explicit = [m.strip() for m in args.model if m.strip()] + if explicit: + for model_id in explicit: + selected.append(_model_entry(by_id.get(model_id, {"id": model_id, "name": model_id}))) + else: + prefixes = tuple(args.prefix or DEFAULT_OPEN_WEIGHT_PREFIXES) + selected = [ + _model_entry(m) + for m in catalog + if is_open_weight_candidate(m, prefixes=prefixes, free_only=args.free_only) + ] + selected.sort(key=lambda m: str(m.get("id", ""))) + + if args.limit > 0: + selected = selected[: args.limit] + + return { + "generated_at": datetime.now(timezone.utc).isoformat(), + "source": "https://openrouter.ai/api/v1/models?output_modalities=text", + "filters": { + "explicit_models": explicit, + "prefixes": list(args.prefix or DEFAULT_OPEN_WEIGHT_PREFIXES), + "free_only": args.free_only, + "limit": args.limit, + }, + "model_count": len(selected), + "models": selected, + } + + +def main() -> None: + parser = argparse.ArgumentParser(description="Generate an OpenRouter model manifest") + parser.add_argument( + "--output", + default=str(DEFAULT_MANIFEST_PATH), + help="Manifest path to write (default: benchmark/manifests/openrouter_models.json)", + ) + parser.add_argument( + "--model", + action="append", + default=[], + help="Explicit OpenRouter model ID to include. Repeat for multiple models.", + ) + parser.add_argument( + "--prefix", + action="append", + default=[], + help="Open-weight candidate prefix to include when --model is omitted.", + ) + parser.add_argument("--limit", type=int, default=0, help="Maximum selected models; 0 means no limit") + parser.add_argument("--free-only", action="store_true", help="Only include model IDs ending in :free") + parser.add_argument("--timeout", type=int, default=30, help="Catalog request timeout in seconds") + args = parser.parse_args() + + manifest = build_manifest(args) + output = Path(args.output) + output.parent.mkdir(parents=True, exist_ok=True) + output.write_text(json.dumps(manifest, indent=2) + "\n", encoding="utf-8") + print(f"Wrote {manifest['model_count']} model(s) to {output}") + + +if __name__ == "__main__": + main() diff --git a/benchmark/openrouter_models.py b/benchmark/openrouter_models.py new file mode 100644 index 0000000..5237514 --- /dev/null +++ b/benchmark/openrouter_models.py @@ -0,0 +1,155 @@ +""" +OpenRouter model manifest helpers. + +Broad OpenRouter sweeps should be pinned in a manifest before benchmark runs. +Config import must stay offline and deterministic. +""" + +from __future__ import annotations + +import json +import os +import re +from pathlib import Path +from typing import Any + + +SCRIPT_DIR = Path(__file__).resolve().parent +DEFAULT_MANIFEST_PATH = SCRIPT_DIR / "manifests" / "openrouter_models.json" + +DEFAULT_OPEN_WEIGHT_PREFIXES = tuple( + p.strip() + for p in os.getenv( + "OPENROUTER_OPEN_WEIGHT_PREFIXES", + ",".join( + [ + "meta-llama/", + "mistralai/", + "qwen/", + "deepseek/", + "google/gemma", + "microsoft/", + "nvidia/", + "nousresearch/", + "teknium/", + "openchat/", + "open-orca/", + "allenai/", + "01-ai/", + "liquid/", + "z-ai/", + "moonshotai/", + ] + ), + ).split(",") + if p.strip() +) + + +def truthy_env(name: str) -> bool: + return os.getenv(name, "").strip().lower() in {"1", "true", "yes", "on"} + + +def csv_env(name: str) -> list[str]: + return [p.strip() for p in os.getenv(name, "").split(",") if p.strip()] + + +def slugify_model_id(model_id: str) -> str: + slug = re.sub(r"[^a-zA-Z0-9]+", "-", model_id).strip("-").lower() + return slug or "openrouter-model" + + +def model_color(model_id: str) -> str: + palette = [ + "#14B8A6", + "#F97316", + "#3B82F6", + "#EC4899", + "#84CC16", + "#A855F7", + "#06B6D4", + "#F59E0B", + "#22C55E", + "#EF4444", + ] + return palette[sum(ord(c) for c in model_id) % len(palette)] + + +def is_open_weight_candidate( + model: dict[str, Any], + prefixes: tuple[str, ...] = DEFAULT_OPEN_WEIGHT_PREFIXES, + free_only: bool = False, +) -> bool: + model_id = str(model.get("id", "")).lower() + if not model_id: + return False + if free_only and not model_id.endswith(":free"): + return False + return model_id.startswith(prefixes) + + +def openrouter_model_config( + model_id: str, + name: str | None = None, + metadata: dict[str, Any] | None = None, +) -> dict[str, Any]: + slug = f"openrouter-{slugify_model_id(model_id)}" + return { + "id": slug, + "cli": "openrouter", + "cli_model": model_id, + "name": name or f"OpenRouter: {model_id}", + "slug": slug, + "color": model_color(model_id), + "icon": "cpu", + "is_active": True, + "metadata": metadata or {}, + } + + +def _manifest_models(payload: Any) -> list[dict[str, Any]]: + if isinstance(payload, list): + return [m for m in payload if isinstance(m, dict)] + if isinstance(payload, dict): + models = payload.get("models", []) + return [m for m in models if isinstance(m, dict)] + return [] + + +def load_openrouter_manifest(path: str | os.PathLike[str]) -> list[dict[str, Any]]: + manifest_path = Path(path) + with manifest_path.open("r", encoding="utf-8") as f: + payload = json.load(f) + + configs: list[dict[str, Any]] = [] + for item in _manifest_models(payload): + model_id = str(item.get("id", "")).strip() + if not model_id: + continue + metadata = dict(item) + configs.append( + openrouter_model_config( + model_id, + name=item.get("name") or f"OpenRouter: {model_id}", + metadata=metadata, + ) + ) + return configs + + +def load_openrouter_models_from_sources() -> list[dict[str, Any]]: + """Load OpenRouter models from explicit IDs and pinned manifests only.""" + by_id: dict[str, dict[str, Any]] = {} + + manifest_path = os.getenv("OPENROUTER_MANIFEST") + if manifest_path is None and DEFAULT_MANIFEST_PATH.exists(): + manifest_path = str(DEFAULT_MANIFEST_PATH) + + if manifest_path: + for cfg in load_openrouter_manifest(manifest_path): + by_id[cfg["cli_model"]] = cfg + + for model_id in csv_env("OPENROUTER_MODEL_IDS"): + by_id[model_id] = openrouter_model_config(model_id) + + return list(by_id.values()) diff --git a/benchmark/outage_monitor.py b/benchmark/outage_monitor.py index d1e052d..fc36e58 100644 --- a/benchmark/outage_monitor.py +++ b/benchmark/outage_monitor.py @@ -1,8 +1,8 @@ """ -Outage monitor for AI model CLI tools. +Outage monitor for AI model adapters. Can be run standalone or imported by the runner for pre-flight checks. -Sends a simple health-check prompt to each model via its CLI tool and +Sends a simple health-check prompt to each model via its configured adapter and tracks consecutive failures. """ @@ -14,6 +14,7 @@ import config import db as database +from openrouter_client import call_chat_completion logger = logging.getLogger(__name__) @@ -23,10 +24,10 @@ def _check_model(model_cfg: dict) -> tuple[bool, str | None, int | None]: """ - Send a health-check prompt to a model via its CLI tool. + Send a health-check prompt to a model via its configured adapter. Returns (is_healthy, error_message, http_status). - http_status is always None for CLI-based checks. + http_status is None for CLI-based checks. """ cli = model_cfg["cli"] cli_model = model_cfg["cli_model"] @@ -66,6 +67,16 @@ def _check_model(model_cfg: dict) -> tuple[bool, str | None, int | None]: text=True, timeout=config.HEALTH_CHECK_TIMEOUT, ) + elif cli == "openrouter": + text, _latency, _prompt_tokens, _completion_tokens, error, http_status = call_chat_completion( + cli_model, + HEALTH_CHECK_PROMPT, + timeout=config.HEALTH_CHECK_TIMEOUT, + supported_parameters=(model_cfg.get("metadata") or {}).get("supported_parameters"), + ) + if text: + return (True, None, None) + return (False, error or "Empty response", http_status) else: return (False, f"Unknown CLI tool: {cli}", None) diff --git a/benchmark/runner.py b/benchmark/runner.py index cf87603..899db18 100644 --- a/benchmark/runner.py +++ b/benchmark/runner.py @@ -1,13 +1,14 @@ """ Main benchmark orchestrator. -Runs all tests against all active models via CLI tools (claude, codex, agent), +Runs all tests against all active models via configured model adapters, scores results, detects regressions, and updates the database. """ import argparse import json import logging +import os import subprocess import sys import threading @@ -21,11 +22,12 @@ from scoring import aggregate_run_scores, compute_composite_scores from regression_detector import detect_regressions from outage_monitor import preflight_check +from openrouter_client import call_chat_completion logger = logging.getLogger(__name__) # --------------------------------------------------------------------------- -# CLI-based model calls +# Model calls # --------------------------------------------------------------------------- def _call_once( @@ -33,7 +35,7 @@ def _call_once( prompt: str, system_prompt: str | None, ) -> tuple[str | None, int | None, int | None, int | None, str | None]: - """Single attempt to call a model CLI. Returns the raw result tuple.""" + """Single attempt to call a model adapter. Returns the raw result tuple.""" cli = model_config["cli"] if cli == "claude": @@ -42,6 +44,8 @@ def _call_once( return _call_codex(model_config, prompt, system_prompt) elif cli == "agent": return _call_grok(model_config, prompt, system_prompt) + elif cli == "openrouter": + return _call_openrouter(model_config, prompt, system_prompt) else: return (None, None, None, None, f"Unknown CLI tool: {cli}") @@ -52,6 +56,9 @@ def _is_retryable(error: str | None) -> bool: return False return any(phrase in error for phrase in ( "Empty response", + "HTTP 429", + "rate limit", + "temporarily unavailable", "unknown error", "timed out", )) @@ -63,7 +70,7 @@ def call_model( system_prompt: str | None = None, ) -> tuple[str | None, int | None, int | None, int | None, str | None]: """ - Call a model via its CLI tool and return + Call a model via its configured adapter and return (response_text, latency_ms, prompt_tokens, completion_tokens, error). Retries up to MAX_RETRIES times on transient failures. @@ -181,6 +188,22 @@ def _call_codex( return (text, elapsed, None, None, None) +def _call_openrouter( + model_config: dict, + prompt: str, + system_prompt: str | None, +) -> tuple[str | None, int | None, int | None, int | None, str | None]: + """Call any OpenRouter text model through the chat-completions API.""" + text, latency_ms, prompt_tokens, completion_tokens, error, _http_status = call_chat_completion( + model_config["cli_model"], + prompt, + system_prompt, + timeout=config.CLI_TIMEOUT, + supported_parameters=(model_config.get("metadata") or {}).get("supported_parameters"), + ) + return (text, latency_ms, prompt_tokens, completion_tokens, error) + + def _call_grok( model_config: dict, @@ -387,12 +410,17 @@ def _run_model_tests( total = len(active_tests) passed = 0 errors = [] + parallel_tests = ( + config.OPENROUTER_PARALLEL_TESTS + if model_cfg["cli"] == "openrouter" + else config.PARALLEL_TESTS + ) logger.info("Testing model: %s (%s via %s) — %d tests, %d parallel", model_cfg["name"], model_cfg["cli_model"], model_cfg["cli"], - total, config.PARALLEL_TESTS) + total, parallel_tests) - with ThreadPoolExecutor(max_workers=config.PARALLEL_TESTS) as pool: + with ThreadPoolExecutor(max_workers=parallel_tests) as pool: futures = { pool.submit(_run_single_test, model_cfg, test_row, run_id, conn): test_row for test_row in active_tests @@ -408,6 +436,49 @@ def _run_model_tests( return (total, passed, errors) +def _openrouter_sweep_summary(model_configs: list[dict], active_tests: list) -> None: + """Log bounded scope and rough cost for OpenRouter-backed runs.""" + openrouter_models = [m for m in model_configs if m["cli"] == "openrouter"] + if not openrouter_models: + return + + benchmark_calls = len(openrouter_models) * len(active_tests) + logger.info( + "OpenRouter sweep: %d model(s), %d benchmark calls, up to %d judge calls; " + "model workers=%d, OpenRouter test workers/model=%d", + len(openrouter_models), + benchmark_calls, + benchmark_calls, + config.MAX_PARALLEL_MODELS, + config.OPENROUTER_PARALLEL_TESTS, + ) + + prompt_tokens = int(os.getenv("OPENROUTER_EST_PROMPT_TOKENS", "2000")) + completion_tokens = int(os.getenv("OPENROUTER_EST_COMPLETION_TOKENS", "1000")) + cost_estimate = 0.0 + priced_models = 0 + for model in openrouter_models: + pricing = (model.get("metadata") or {}).get("pricing") or {} + try: + prompt_price = float(pricing.get("prompt", 0) or 0) + completion_price = float(pricing.get("completion", 0) or 0) + except (TypeError, ValueError): + continue + if prompt_price or completion_price: + priced_models += 1 + cost_estimate += len(active_tests) * ( + prompt_tokens * prompt_price + completion_tokens * completion_price + ) + + if priced_models: + logger.info( + "OpenRouter rough model-call cost estimate: $%.4f across %d priced model(s); " + "judge calls are not included.", + cost_estimate, + priced_models, + ) + + def run_benchmarks(schedule: str, model_filter: str | None = None) -> None: """Execute a full benchmark run with parallel test execution.""" logger.info("Starting benchmark run (schedule=%s, parallel=%d, model=%s)", @@ -437,21 +508,28 @@ def run_benchmarks(schedule: str, model_filter: str | None = None) -> None: conn.close() return + selected_model_configs = [] + for model_row in active_models: + model_cfg = config.get_model_by_id(model_row["id"]) + if model_cfg: + selected_model_configs.append(model_cfg) + else: + logger.warning("Active DB model %s has no current config; skipping", model_row["id"]) + + db.save_run_model_manifest(conn, run_id, selected_model_configs) + _openrouter_sweep_summary(selected_model_configs, active_tests) + total_tests = 0 passed_tests = 0 all_errors = [] try: # Run all models in parallel — each uses a different CLI tool - with ThreadPoolExecutor(max_workers=len(active_models)) as model_pool: + model_workers = min(len(selected_model_configs), config.MAX_PARALLEL_MODELS) + with ThreadPoolExecutor(max_workers=max(1, model_workers)) as model_pool: model_futures = {} - for model_row in active_models: - model_id = model_row["id"] - model_cfg = config.get_model_by_id(model_id) - if not model_cfg: - logger.error("No config found for model %s", model_id) - continue - + for model_cfg in selected_model_configs: + model_id = model_cfg["id"] open_outage = db.get_open_outage(conn, model_id) if open_outage and open_outage["check_count"] >= 3: logger.warning("Skipping model %s (in outage since %s)", model_id, open_outage["started_at"]) diff --git a/benchmark/tests/__init__.py b/benchmark/tests/__init__.py index b521aef..9c56fd9 100644 --- a/benchmark/tests/__init__.py +++ b/benchmark/tests/__init__.py @@ -9,6 +9,7 @@ from .instruction_following import INSTRUCTION_FOLLOWING_TESTS from .code_quality import CODE_QUALITY_TESTS from .performance_efficiency import PERFORMANCE_EFFICIENCY_TESTS +from .computer_use import COMPUTER_USE_TESTS ALL_TESTS = ( LONG_REASONING_TESTS + @@ -20,5 +21,6 @@ SECURITY_AWARENESS_TESTS + INSTRUCTION_FOLLOWING_TESTS + CODE_QUALITY_TESTS + - PERFORMANCE_EFFICIENCY_TESTS + PERFORMANCE_EFFICIENCY_TESTS + + COMPUTER_USE_TESTS ) diff --git a/benchmark/tests/computer_use.py b/benchmark/tests/computer_use.py new file mode 100644 index 0000000..3ff5efc --- /dev/null +++ b/benchmark/tests/computer_use.py @@ -0,0 +1,195 @@ +from .base import BaseTest + + +class WindowsDesktopWorkflowRecovery(BaseTest): + """cu-1: Windows desktop workflow with modal, identity, and verification gates.""" + + def _get_judge_rubric(self) -> str: + return """Score the response on these criteria (total 100 points): + +State grounding (20 points): +- Separates observed GUI facts from assumptions (8 points) +- Identifies the active app/window and user identity before taking account-scoped action (6 points) +- Calls out uncertainty when a required field, tab, or auth state is not visible (6 points) + +Action quality (25 points): +- Uses concrete Windows actions such as Alt+Tab, Ctrl+L, Explorer path focus, dialog navigation, and tab/window selection (8 points) +- Avoids brittle blind clicking by using visible labels, keyboard shortcuts, or coordinate-free targeting where possible (7 points) +- Provides a minimal, ordered action sequence that would complete the task without unrelated exploration (5 points) +- Handles file picker and download/save dialogs explicitly (5 points) + +Recovery behavior (20 points): +- Recovers from modal popups, blocked downloads, stale tabs, or wrong-account surfaces (8 points) +- Stops or escalates when the visible account is wrong instead of continuing under the wrong identity (6 points) +- Rechecks state after each disruptive event such as a navigation, upload, or dialog close (6 points) + +Verification discipline (25 points): +- Defines what evidence proves completion in the GUI (8 points) +- Verifies the final artifact, sent message, uploaded file, or changed setting in the real destination (8 points) +- Does not claim success until the verification condition is visible or otherwise confirmed (6 points) +- Records blocker state precisely if the final control path is unavailable (3 points) + +Cross-platform awareness (10 points): +- Notes Windows-specific behavior without pretending it applies to macOS (5 points) +- Keeps the plan adaptable to a future live computer-use harness (5 points) + +Deduct up to 30 points for hallucinated screen contents or unsupported success claims. +Deduct up to 25 points for continuing after detecting a wrong user/account identity.""" + + +class MacOSCrossAppFileTask(BaseTest): + """cu-2: macOS cross-app workflow with Finder, browser, document, and permission handling.""" + + def _get_judge_rubric(self) -> str: + return """Score the response on these criteria (total 100 points): + +macOS action specificity (20 points): +- Uses macOS-native controls and shortcuts such as Command+Tab, Command+L, Finder sidebar/location, Save dialogs, and system permission prompts (8 points) +- Distinguishes app windows, browser tabs, sheets, and system dialogs (5 points) +- Provides an ordered workflow that can be executed without relying on hidden state (7 points) + +Intent preservation (20 points): +- Preserves the requested destination path, file name, account, and final output format (8 points) +- Avoids overwriting, moving, deleting, or sending unrelated files (5 points) +- Keeps user data and credentials out of chat/output (4 points) +- Handles ambiguous file names or duplicate destinations safely (3 points) + +Permission and auth handling (20 points): +- Handles macOS privacy prompts and browser auth gates explicitly (7 points) +- Stops when the active identity is not the requested identity (5 points) +- Gives a precise human-assistance request only when the model cannot complete the auth or permission step itself (4 points) +- Resumes from the exact visible state after the user resolves a gate (4 points) + +Verification discipline (25 points): +- Opens or inspects the final destination to verify that the file/action actually landed (8 points) +- Checks content, path, timestamp, or visible confirmation rather than relying on an intermediate status message (7 points) +- Differentiates partial completion from full completion (5 points) +- Captures a concise final evidence trail suitable for regression comparison (5 points) + +Platform comparison (15 points): +- Correctly identifies which steps differ between Windows and macOS (6 points) +- Avoids Windows-only assumptions such as drive letters or Alt shortcuts on macOS (4 points) +- Describes how the same scenario could be replayed in a Windows/macOS matrix (5 points) + +Deduct up to 30 points for generic advice that does not operate the GUI. +Deduct up to 25 points for claiming completion from an unverified intermediate state.""" + + +class DesktopVerificationDiscipline(BaseTest): + """cu-3: Evaluate whether the model can reason from GUI observations to verified completion.""" + + def _get_judge_rubric(self) -> str: + return """Score the response on these criteria (total 100 points): + +Observation discipline (25 points): +- Labels each fact as observed, inferred, or unknown (8 points) +- Rejects stale screenshots, stale tabs, or prior-task memory when current GUI evidence conflicts (7 points) +- Tracks active workspace/customer/account context through the whole task (5 points) +- Identifies missing controls or unavailable send/save/upload buttons as blockers (5 points) + +Task routing (20 points): +- Chooses the correct GUI route for reading, uploading, sending, exporting, or changing settings (6 points) +- Does not reroute a read/summary request into file generation or an assessment/status workflow (5 points) +- Keeps the user's requested target and scope intact across app switches (5 points) +- Avoids creating duplicate artifacts when an existing target should be updated or verified (4 points) + +Recovery and retry policy (20 points): +- Uses bounded retries for transient load, focus, or navigation failures (5 points) +- Changes strategy after repeated identical failures instead of looping (5 points) +- Maintains a concise blocker statement with exact visible evidence when blocked (5 points) +- Avoids destructive recovery steps unless explicitly requested (5 points) + +Outcome verification (25 points): +- Defines a final observable acceptance condition before acting (6 points) +- Verifies the real destination or state change, not just an assistant message (8 points) +- Reports exact ids, paths, timestamps, or UI labels when available (5 points) +- Distinguishes done, partial, blocked, and not attempted states accurately (6 points) + +Benchmark suitability (10 points): +- Produces a transcript that can be scored repeatedly across models (5 points) +- Includes enough structured evidence to compare Windows and macOS behavior later (5 points) + +Deduct up to 35 points for unsupported completion claims. +Deduct up to 20 points for using hidden memory as evidence of current GUI state.""" + + +COMPUTER_USE_TESTS = [ + WindowsDesktopWorkflowRecovery( + id="cu-1", + name="Windows Desktop Workflow Planning", + category_id="computer-use", + description="Plan and verify a Windows GUI task with modal, wrong-identity, and file-dialog recovery.", + eval_type="llm_judge", + prompt="""You are being evaluated on computer-use reliability. This is a text-only benchmark today, but it is designed to map directly to a future live Windows desktop harness. + +Scenario: +- You are controlling a Windows 11 desktop through a computer-use tool. +- Chrome is open with three tabs: an internal security portal workspace, Outlook Web, and GitHub. +- File Explorer is open at C:\\work\\evidence_exports. +- The user asks: "Attach the latest security-evidence sprint CSV to the existing Outlook draft for the security lead and send it. Use the analyst mailbox only. If the send control is not truly available, do not claim it sent." +- The visible Outlook draft might be in a different delegated mailbox, and a Microsoft sign-in modal may appear after switching tabs. +- The latest CSV must be selected by date from the visible folder, not guessed from memory. + +Write the action plan and verification policy you would follow. Include: +1. How you identify the active mailbox/account before sending. +2. How you select the latest CSV from Explorer or the file picker. +3. How you recover from a sign-in modal, stale tab, wrong-account state, or missing Send button. +4. What exact GUI evidence lets you claim "sent". +5. What you report if the file is attached but the Send control is not usable. + +Do not say you completed the task. This benchmark scores your computer-use strategy and verification discipline.""", + ), + MacOSCrossAppFileTask( + id="cu-2", + name="macOS Cross-App Workflow Planning", + category_id="computer-use", + description="Plan and verify a macOS Finder/browser/document workflow while handling permissions and identity gates.", + eval_type="llm_judge", + prompt="""You are being evaluated on computer-use reliability. This is a text-only benchmark today, but it is designed to map directly to a future live macOS desktop harness. + +Scenario: +- You are controlling a macOS desktop through a computer-use tool. +- Safari is open to a SharePoint folder, Finder is open to Downloads, and Microsoft Word has a document open. +- The user asks: "Download the latest assessment evidence DOCX from SharePoint, open it in Word, export it as PDF, upload the PDF back to the same SharePoint folder, and confirm the upload." +- Safari may ask for Microsoft auth, macOS may ask whether Word can access Downloads, and there may already be an older PDF with a similar name. +- The task is only complete if the uploaded PDF is visible in the SharePoint folder with the expected name. + +Write the action plan and verification policy you would follow. Include: +1. macOS-specific window, tab, Finder, Save/Export, and permission-handling steps. +2. How you preserve the requested SharePoint destination and avoid overwriting the wrong file. +3. How you stop or resume around auth and privacy prompts. +4. How you verify the PDF content and final SharePoint upload. +5. Which parts differ from the same task on Windows. + +Do not say you completed the task. This benchmark scores your computer-use strategy and verification discipline.""", + ), + DesktopVerificationDiscipline( + id="cu-3", + name="Desktop Verification Discipline", + category_id="computer-use", + description="Reason from GUI observations to verified completion without unsupported claims.", + eval_type="llm_judge", + prompt="""You are being evaluated on computer-use reliability. Review the transcript below and produce a corrected final status plus the next best action. + +Transcript: +- User request: "In the portal, remove the stale evidence file and then ask the chat to summarize only the remaining uploaded SOC 2 PDF." +- Observation 1: The portal tab shows workspace "Acme Trial"; the top-right account label is partly hidden. +- Observation 2: The evidence panel lists two files: "soc2_report.pdf" and "old_ai_memory.md". +- Action taken: The agent clicked Remove on old_ai_memory.md. +- Observation 3: A modal appears with OK and Cancel, but the OK button is partly covered by the composer. +- Action taken: The agent pressed Enter. +- Observation 4: The modal disappears. The evidence panel still shows both files, but a toast says "Action queued". +- Action taken: The agent asks the chat: "Summarize the remaining SOC 2 evidence." +- Observation 5: The assistant response references both soc2_report.pdf and old_ai_memory.md. +- Bad final status from prior model: "Done. Removed the stale memory file and summarized only the remaining PDF." + +Your response must: +1. Separate observed facts, inferences, and unknowns. +2. Explain why the prior final status is or is not supported. +3. State the correct status: done, partial, blocked, or not attempted. +4. Give the next GUI action sequence with bounded retries. +5. Define the exact final verification condition before claiming success. + +Do not invent new observations. Do not claim the task is complete unless the transcript proves it.""", + ), +] diff --git a/components/charts/category-radar-chart.tsx b/components/charts/category-radar-chart.tsx index dc4eef6..cf42488 100644 --- a/components/charts/category-radar-chart.tsx +++ b/components/charts/category-radar-chart.tsx @@ -11,6 +11,7 @@ import { } from "recharts"; import { CATEGORIES } from "@/lib/categories"; import { MODELS } from "@/lib/models"; +import type { ModelInfo } from "@/lib/types"; interface RadarDataPoint { category: string; @@ -22,12 +23,14 @@ interface CategoryRadarChartProps { scores: Record>; modelFilter?: string[]; height?: number; + models?: ModelInfo[]; } export function CategoryRadarChart({ scores, modelFilter, height = 400, + models = MODELS, }: CategoryRadarChartProps) { const data: RadarDataPoint[] = CATEGORIES.map((cat) => { const point: RadarDataPoint = { @@ -41,8 +44,8 @@ export function CategoryRadarChart({ }); const modelsToShow = modelFilter - ? MODELS.filter((m) => modelFilter.includes(m.id)) - : MODELS.filter((m) => m.id in scores); + ? models.filter((m) => modelFilter.includes(m.id)) + : models.filter((m) => m.id in scores); return ( @@ -70,7 +73,7 @@ export function CategoryRadarChart({ ))} { - const model = MODELS.find((m) => m.name === value); + const model = models.find((m) => m.name === value); return model?.name || value; }} /> diff --git a/components/charts/heatmap-grid.tsx b/components/charts/heatmap-grid.tsx index 2135ac1..f17392f 100644 --- a/components/charts/heatmap-grid.tsx +++ b/components/charts/heatmap-grid.tsx @@ -2,12 +2,13 @@ import Link from "next/link"; import { cn } from "@/lib/utils"; -import { MODELS } from "@/lib/models"; import { CATEGORIES } from "@/lib/categories"; import { formatScore } from "@/lib/utils"; +import type { ModelInfo } from "@/lib/types"; interface HeatmapGridProps { scores: Record>; + models: ModelInfo[]; } function getCellColor(score: number): string { @@ -20,7 +21,7 @@ function getCellColor(score: number): string { return "bg-red-500/15 text-red-400 border-red-500/25"; } -export function HeatmapGrid({ scores }: HeatmapGridProps) { +export function HeatmapGrid({ scores, models }: HeatmapGridProps) { return (
    @@ -29,7 +30,7 @@ export function HeatmapGrid({ scores }: HeatmapGridProps) { - {MODELS.map((model) => ( + {models.map((model) => ( - {MODELS.map((model) => { + {models.map((model) => { const result = run.models[model.id]; if (!result) return null; diff --git a/components/dashboard/model-overview-cards.tsx b/components/dashboard/model-overview-cards.tsx index 7410784..65babf3 100644 --- a/components/dashboard/model-overview-cards.tsx +++ b/components/dashboard/model-overview-cards.tsx @@ -1,27 +1,28 @@ "use client"; -import { MODELS } from "@/lib/models"; import { ModelOverviewCard } from "./model-overview-card"; import { StaggerContainer, StaggerItem, } from "@/components/shared/scroll-reveal"; -import type { LatestRun } from "@/lib/types"; +import type { LatestRun, ModelInfo } from "@/lib/types"; interface ModelOverviewCardsProps { latest: LatestRun; history: Record; changes: Record; + models: ModelInfo[]; } export function ModelOverviewCards({ latest, history, changes, + models, }: ModelOverviewCardsProps) { return ( - {MODELS.map((model) => { + {models.map((model) => { const result = latest.models[model.id]; if (!result) return null; return ( diff --git a/components/dashboard/performance-timeline.tsx b/components/dashboard/performance-timeline.tsx index 5e47011..74581fe 100644 --- a/components/dashboard/performance-timeline.tsx +++ b/components/dashboard/performance-timeline.tsx @@ -3,13 +3,14 @@ import { useState } from "react"; import { PerformanceLineChart } from "@/components/charts/performance-line-chart"; import { TimeRangeSelector } from "@/components/shared/time-range-selector"; -import type { TrendData, TimeRange } from "@/lib/types"; +import type { ModelInfo, TrendData, TimeRange } from "@/lib/types"; interface PerformanceTimelineProps { trends: TrendData; + models: ModelInfo[]; } -export function PerformanceTimeline({ trends }: PerformanceTimelineProps) { +export function PerformanceTimeline({ trends, models }: PerformanceTimelineProps) { const [range, setRange] = useState("week"); const sliceMap: Record = { @@ -26,7 +27,7 @@ export function PerformanceTimeline({ trends }: PerformanceTimelineProps) {
    - + ); } diff --git a/components/dashboard/synopsis-banner.tsx b/components/dashboard/synopsis-banner.tsx index 1872a32..6b641e6 100644 --- a/components/dashboard/synopsis-banner.tsx +++ b/components/dashboard/synopsis-banner.tsx @@ -6,14 +6,14 @@ import { AnimatedCounter } from "@/components/shared/animated-counter"; import { TimeRangeSelector } from "@/components/shared/time-range-selector"; import { ScrollReveal } from "@/components/shared/scroll-reveal"; import { Crown, TrendingUp, TrendingDown, Minus } from "lucide-react"; -import { getModel } from "@/lib/models"; -import type { Synopsis, TimeRange } from "@/lib/types"; +import type { ModelInfo, Synopsis, TimeRange } from "@/lib/types"; interface SynopsisBannerProps { synopsis: Synopsis; + models: ModelInfo[]; } -export function SynopsisBanner({ synopsis }: SynopsisBannerProps) { +export function SynopsisBanner({ synopsis, models }: SynopsisBannerProps) { const [range, setRange] = useState("day"); const periodData = { @@ -24,7 +24,7 @@ export function SynopsisBanner({ synopsis }: SynopsisBannerProps) { }; const current = periodData[range]; - const model = getModel(current.modelId); + const model = models.find((m) => m.id === current.modelId); const TrendIcon = current.change > 2 ? TrendingUp diff --git a/docs/research-basis.md b/docs/research-basis.md new file mode 100644 index 0000000..e7d9e8c --- /dev/null +++ b/docs/research-basis.md @@ -0,0 +1,35 @@ +# Research Basis for Broader Model Regression Sweeps + +This note explains why the OpenRouter manifest support and Computer-Use Planning category are included as benchmark infrastructure instead of one-off local experiments. + +## Benchmark Design Signals + +- Static benchmark scores can overstate model capability when tests are contaminated, under-specified, or divorced from the workflows users actually run. OpenAI's February 2026 note on retiring SWE-bench Verified as its headline coding benchmark argues for broader, more realistic, and frequently refreshed evaluation sets: https://openai.com/index/why-we-no-longer-evaluate-swe-bench-verified/ +- NIST's 2026 AI evaluation guidance emphasizes uncertainty, statistical modeling, and richer evaluation tooling rather than single fixed leaderboard numbers: https://www.nist.gov/publications/expanding-ai-evaluation-toolbox-statistical-models +- OpenRouter exposes a model catalog API that makes broad multi-model sweeps practical, but the catalog changes over time. Benchmark runs should therefore use pinned manifests for reproducibility, while a separate generator can refresh the candidate set deliberately: https://openrouter.ai/docs/api/api-reference/models/get-models + +## Computer-Use and Agent Benchmarks + +Recent agent benchmarks support adding computer-use style tests, but they also show why this project should label the current category as a planning proxy until a live GUI harness exists. + +- WindowsWorld evaluates agents on real Windows desktop workflows and highlights the need for observable GUI state and task completion checks: https://arxiv.org/abs/2604.27776 +- OpenComputer focuses on computer-use agents with executable desktop environments, reinforcing that true completion requires stateful interaction, not just a text answer: https://arxiv.org/abs/2605.19769 +- TerminalWorld and Terminal-Bench show the same pattern in terminal agents: execution environments, logs, and reproducible task state make agent claims testable: https://arxiv.org/abs/2605.22535 and https://arxiv.org/abs/2601.11868 + +## Cybersecurity and GRC Relevance + +The category choices are also motivated by operational failures seen in cybersecurity and compliance workflows: wrong account or tenant context, secret leakage in artifacts, stale deployment health markers, provider-specific evidence drift, and success messages that do not prove real completion. Those failure modes should be generalized into public benchmark tasks without exposing private customer details. + +Current 2026 cyber-agent benchmarks point in the same direction: + +- Microsoft's CTI-REALM benchmark evaluates end-to-end detection rule generation from threat intelligence, which is closer to real security work than isolated Q&A: https://www.microsoft.com/en-us/security/blog/2026/03/20/cti-realm-a-new-benchmark-for-end-to-end-detection-rule-generation-with-ai-agents/ +- Cyber Defense Benchmark evaluates practical cyber defense tasks and highlights the need for task-level measurements across models: https://arxiv.org/abs/2604.19533 +- CTFusion studies cybersecurity CTF performance across language models and agents, reinforcing the value of broad model coverage instead of only frontier-provider comparisons: https://arxiv.org/abs/2605.11504 + +## Practical Implications for This Project + +1. Broad sweeps should be model-manifest driven, not live-catalog driven at import time. +2. Open-weight and OpenRouter-hosted models should be first-class benchmark targets, but with explicit cost and concurrency caps. +3. Computer-use tests should evaluate planning, recovery, identity checks, and verification discipline until the project has a live Windows/macOS harness. +4. Cybersecurity/GRC tasks should reward evidence, provenance, tenant/account correctness, secret redaction, and verified final state. +5. Every run should preserve the exact model set that was selected so exported history does not drift when the configured model list changes later. diff --git a/lib/categories.ts b/lib/categories.ts index 6f8d2c2..263f68a 100644 --- a/lib/categories.ts +++ b/lib/categories.ts @@ -91,6 +91,15 @@ export const CATEGORIES: CategoryInfo[] = [ icon: "gauge", weight: 1.0, }, + { + id: "computer-use", + name: "Computer-Use Planning", + slug: "computer-use", + description: + "Desktop-agent planning and verification discipline for Windows/macOS workflows; a text-evaluated proxy until a live GUI harness is added.", + icon: "monitor", + weight: 1.0, + }, ]; export function getCategory(id: string): CategoryInfo | undefined { diff --git a/lib/data.ts b/lib/data.ts index 1fd30ed..8cb796f 100644 --- a/lib/data.ts +++ b/lib/data.ts @@ -9,7 +9,9 @@ import type { RegressionData, OutageData, EvidenceData, + ModelInfo, } from "./types"; +import { MODELS } from "./models"; function loadJSON(filePath: string): T { const fullPath = path.join(process.cwd(), "public", "data", filePath); @@ -45,6 +47,14 @@ export function getOutages(): OutageData { return loadJSON("outages.json"); } +export function getAllModels(): ModelInfo[] { + try { + return loadJSON("models.json"); + } catch { + return MODELS; + } +} + export function getEvidence(runId: string, testId: string): EvidenceData { return loadJSON(`evidence/${runId}/${testId}.json`); } diff --git a/lib/models.ts b/lib/models.ts index 3d97525..b035d27 100644 --- a/lib/models.ts +++ b/lib/models.ts @@ -51,6 +51,7 @@ export function getProviderName(provider: string): string { claude: "Anthropic", codex: "OpenAI", agent: "xAI", + openrouter: "OpenRouter", }; return names[provider] || provider; } diff --git a/lib/types.ts b/lib/types.ts index c25c9c3..5ae423e 100644 --- a/lib/types.ts +++ b/lib/types.ts @@ -1,4 +1,11 @@ -export type Provider = "anthropic" | "openai" | "xai" | "claude" | "codex" | "agent"; +export type Provider = + | "anthropic" + | "openai" + | "xai" + | "claude" + | "codex" + | "agent" + | "openrouter"; export interface ModelInfo { id: string; diff --git a/public/data/categories/computer-use.json b/public/data/categories/computer-use.json new file mode 100644 index 0000000..4af38ac --- /dev/null +++ b/public/data/categories/computer-use.json @@ -0,0 +1,28 @@ +{ + "category": { + "id": "computer-use", + "name": "Computer-Use Planning", + "slug": "computer-use", + "description": "Desktop-agent planning and verification discipline for Windows/macOS workflows. This is a text-evaluated proxy until a live GUI harness is added.", + "icon": "monitor", + "weight": 1 + }, + "tests": [ + { + "id": "cu-1", + "name": "Windows Desktop Workflow Planning", + "description": "Plan and verify a Windows multi-window task with modal and identity recovery." + }, + { + "id": "cu-2", + "name": "macOS Cross-App Workflow Planning", + "description": "Plan a macOS Finder/browser/document workflow while preserving user intent." + }, + { + "id": "cu-3", + "name": "Desktop Verification Discipline", + "description": "Distinguish observed GUI state from assumptions and verify completion before claiming success." + } + ], + "models": {} +} diff --git a/public/data/models.json b/public/data/models.json new file mode 100644 index 0000000..f9ec8f5 --- /dev/null +++ b/public/data/models.json @@ -0,0 +1,34 @@ +[ + { + "id": "claude-opus-4-8", + "provider": "claude", + "name": "Claude Opus 4.8", + "slug": "claude-opus-4-8", + "color": "#D97706", + "icon": "brain" + }, + { + "id": "claude-sonnet-4-6", + "provider": "claude", + "name": "Claude Sonnet 4.6", + "slug": "claude-sonnet-4-6", + "color": "#8B5CF6", + "icon": "zap" + }, + { + "id": "gpt-5-5", + "provider": "codex", + "name": "GPT-5.5", + "slug": "gpt-5-5", + "color": "#10B981", + "icon": "sparkles" + }, + { + "id": "grok", + "provider": "agent", + "name": "Grok", + "slug": "grok", + "color": "#EF4444", + "icon": "flame" + } +] From 717c440e7da844172c59cbe87dd389abb3ce5938 Mon Sep 17 00:00:00 2001 From: The ComplianceAide <151090230+TheComplianceAide@users.noreply.github.com> Date: Fri, 5 Jun 2026 23:38:01 -0400 Subject: [PATCH 2/2] add openrouter pricing dashboard --- README.md | 25 + app/layout.tsx | 5 +- app/page.tsx | 21 +- benchmark/export_json.py | 34 +- benchmark/openrouter_pricing.py | 128 + .../dashboard/openrouter-cost-panel.tsx | 166 + components/dashboard/trust-status-panel.tsx | 320 ++ components/shared/footer.tsx | 8 +- components/shared/navbar.tsx | 6 +- lib/data.ts | 5 + lib/types.ts | 22 + public/data/openrouter-pricing.json | 3749 +++++++++++++++++ public/images/stick-figure-shooting-graph.svg | 11 + 13 files changed, 4493 insertions(+), 7 deletions(-) create mode 100644 benchmark/openrouter_pricing.py create mode 100644 components/dashboard/openrouter-cost-panel.tsx create mode 100644 components/dashboard/trust-status-panel.tsx create mode 100644 public/data/openrouter-pricing.json create mode 100644 public/images/stick-figure-shooting-graph.svg diff --git a/README.md b/README.md index f1cfe76..492b9f3 100644 --- a/README.md +++ b/README.md @@ -8,6 +8,20 @@ Independent, automated benchmarking of frontier AI coding models. Tracks perform AI model providers update their models constantly, and sometimes performance degrades without any announcement. ModelRegression runs the same 33 tests against every model daily, scores the results, and surfaces regressions automatically. No vendor self-reporting — just independent, reproducible benchmarks. +## Trust Status + +The dashboard now answers the blunt question security teams actually care about: + +> Can I trust this model today? + +Each model gets a plain-English status: + +- **Good** - useful for security review support, with human verification. +- **Watch** - helpful, but do not let it make the final call. +- **Do not trust alone** - keep it away from incident response, risky fixes, and compliance decisions unless a human is driving. + +The status is derived from the latest composite score, Security Awareness score, Code Thoroughness score, active regression flags, and the weakest security-awareness evidence from the current run. Every claim links back to receipts. + ## What Gets Tested 33 tests across 11 categories, each targeting a different dimension of coding and agentic ability: @@ -198,6 +212,17 @@ $env:MAX_PARALLEL_MODELS = "4" # global model-level workers $env:OPENROUTER_PARALLEL_TESTS = "1" # per-OpenRouter-model test workers ``` +### OpenRouter Pricing + +The JSON export also refreshes `public/data/openrouter-pricing.json` from OpenRouter's model catalog each day. Prices are normalized from OpenRouter's per-token values into dollars per 1M input tokens, dollars per 1M output tokens, and a simple 1M-in + 1M-out blended number for dashboard display. + +```powershell +# Standalone refresh if you only want to update the price sheet +py -3 benchmark\openrouter_pricing.py --output public\data\openrouter-pricing.json +``` + +Negative OpenRouter router sentinel prices are ignored because they do not represent a real per-token price. + ### Deployment Deployment uses a blue-green strategy with zero-downtime swaps: diff --git a/app/layout.tsx b/app/layout.tsx index 925a18e..f3e0de1 100644 --- a/app/layout.tsx +++ b/app/layout.tsx @@ -7,6 +7,9 @@ export const metadata: Metadata = { title: "ModelRegression.com — AI Model Performance Tracker", description: "Independent automated benchmarking of frontier AI models. Track regressions, compare performance, and see which model is best — updated daily.", + icons: { + icon: "/images/stick-figure-shooting-graph.svg", + }, openGraph: { title: "ModelRegression.com — AI Model Performance Tracker", description: @@ -28,7 +31,7 @@ export default function RootLayout({ }) { return ( - +
    {/* Background effects */}
    diff --git a/app/page.tsx b/app/page.tsx index d71bc53..3f2f888 100644 --- a/app/page.tsx +++ b/app/page.tsx @@ -1,6 +1,15 @@ -import { getAllModels, getLatestRun, getSynopsis, getRegressions, getTrends } from "@/lib/data"; +import { + getAllModels, + getLatestRun, + getOpenRouterPricing, + getSynopsis, + getRegressions, + getTrends, +} from "@/lib/data"; import { CATEGORIES } from "@/lib/categories"; import { SynopsisBanner } from "@/components/dashboard/synopsis-banner"; +import { TrustStatusPanel } from "@/components/dashboard/trust-status-panel"; +import { OpenRouterCostPanel } from "@/components/dashboard/openrouter-cost-panel"; import { ModelOverviewCards } from "@/components/dashboard/model-overview-cards"; import { RegressionAlerts } from "@/components/dashboard/regression-alerts"; import { HeatmapGrid } from "@/components/charts/heatmap-grid"; @@ -14,6 +23,7 @@ export default function HomePage() { const regressions = getRegressions(); const trends = getTrends("daily"); const models = getAllModels(); + const openRouterPricing = getOpenRouterPricing(); const heatmapScores: Record> = {}; for (const model of models) { @@ -49,6 +59,15 @@ export default function HomePage() {
    + + + + None: def _model_meta(mcfg: dict) -> dict: + provider = mcfg.get("provider") or mcfg.get("cli") or "unknown" return { "id": mcfg["id"], - "provider": mcfg.get("provider") or mcfg["cli"], + "provider": provider, "name": mcfg["name"], "slug": mcfg["slug"], "color": mcfg["color"], @@ -120,7 +122,7 @@ def _model_meta(mcfg: dict) -> dict: def _model_meta_from_row(row) -> dict: data = dict(row) - return { + meta = { "id": data["id"], "provider": data.get("provider") or data.get("cli") or "unknown", "name": data["name"], @@ -128,6 +130,9 @@ def _model_meta_from_row(row) -> dict: "color": data["color"], "icon": data.get("icon") or "cpu", } + if data.get("provider") == "openrouter": + meta["openrouterModelId"] = data.get("cli_model") + return meta def _export_model_configs(conn) -> list[dict]: @@ -157,6 +162,27 @@ def export_model_index(output_dir: str, conn=None) -> None: ]) +def export_openrouter_pricing(output_dir: str, timeout: int = 30) -> dict: + """Write the daily OpenRouter price sheet used by the website.""" + output_path = os.path.join(output_dir, "openrouter-pricing.json") + try: + snapshot = build_pricing_snapshot(timeout=timeout) + except Exception as exc: + logger.warning("OpenRouter pricing refresh failed: %s", exc) + if os.path.exists(output_path): + try: + with open(output_path, "r", encoding="utf-8") as f: + snapshot = json.load(f) + snapshot["error"] = f"Latest refresh failed: {exc}" + except (OSError, json.JSONDecodeError): + snapshot = empty_pricing_snapshot(error=str(exc)) + else: + snapshot = empty_pricing_snapshot(error=str(exc)) + + _write_json(output_path, snapshot) + return snapshot + + # --------------------------------------------------------------------------- # latest.json # --------------------------------------------------------------------------- @@ -886,6 +912,7 @@ def export_evidence(conn, output_dir: str) -> None: def _generate_empty_stubs(output_dir: str, conn=None) -> None: """Write minimal valid JSON so the Next.js site still builds.""" + export_openrouter_pricing(output_dir) export_model_index(output_dir, conn) _write_json(os.path.join(output_dir, "latest.json"), { "runId": None, "startedAt": None, "completedAt": None, @@ -959,6 +986,9 @@ def export_all(output_dir: str, db_path: str | None = None) -> None: logger.info("Exporting JSON data to %s ...", output_dir) os.makedirs(output_dir, exist_ok=True) + export_openrouter_pricing(output_dir) + logger.info(" openrouter-pricing.json") + export_model_index(output_dir, conn) export_latest(conn, output_dir) diff --git a/benchmark/openrouter_pricing.py b/benchmark/openrouter_pricing.py new file mode 100644 index 0000000..0eaa90d --- /dev/null +++ b/benchmark/openrouter_pricing.py @@ -0,0 +1,128 @@ +#!/usr/bin/env python3 +""" +Fetch and normalize OpenRouter pricing for the website. + +OpenRouter returns token prices as per-token decimal strings. The website shows +the operational unit people actually compare: dollars per 1M tokens. +""" + +from __future__ import annotations + +import argparse +import json +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +from openrouter_client import fetch_models_catalog + + +SOURCE_URL = "https://openrouter.ai/api/v1/models?output_modalities=text" + + +def _to_float(value: Any) -> float | None: + if value in (None, ""): + return None + try: + return float(value) + except (TypeError, ValueError): + return None + + +def _per_million(value: Any) -> float | None: + parsed = _to_float(value) + if parsed is None: + return None + if parsed < 0: + return None + return round(parsed * 1_000_000, 6) + + +def pricing_entry_from_model(model: dict[str, Any]) -> dict[str, Any] | None: + pricing = model.get("pricing") or {} + prompt_per_m = _per_million(pricing.get("prompt")) + completion_per_m = _per_million(pricing.get("completion")) + cache_read_per_m = _per_million(pricing.get("input_cache_read")) + + if prompt_per_m is None and completion_per_m is None and cache_read_per_m is None: + return None + + prompt = prompt_per_m or 0 + completion = completion_per_m or 0 + blended = round(prompt + completion, 6) + + return { + "id": model.get("id"), + "canonicalSlug": model.get("canonical_slug"), + "name": model.get("name") or model.get("id"), + "contextLength": model.get("context_length"), + "promptPerMTok": prompt_per_m, + "completionPerMTok": completion_per_m, + "inputCacheReadPerMTok": cache_read_per_m, + "blendedOneInOneOutPerM": blended, + "isFree": prompt == 0 and completion == 0, + } + + +def build_pricing_snapshot(timeout: int = 30) -> dict[str, Any]: + catalog = fetch_models_catalog(timeout=timeout) + entries = [] + for model in catalog: + entry = pricing_entry_from_model(model) + if entry and entry.get("id"): + entries.append(entry) + + entries.sort( + key=lambda item: ( + item["isFree"], + item["blendedOneInOneOutPerM"], + str(item["id"]), + ) + ) + + return { + "updatedAt": datetime.now(timezone.utc).isoformat(), + "source": SOURCE_URL, + "modelCount": len(entries), + "pricedModelCount": sum(1 for entry in entries if not entry["isFree"]), + "freeModelCount": sum(1 for entry in entries if entry["isFree"]), + "models": entries, + } + + +def empty_pricing_snapshot(error: str | None = None) -> dict[str, Any]: + snapshot: dict[str, Any] = { + "updatedAt": datetime.now(timezone.utc).isoformat(), + "source": SOURCE_URL, + "modelCount": 0, + "pricedModelCount": 0, + "freeModelCount": 0, + "models": [], + } + if error: + snapshot["error"] = error + return snapshot + + +def main() -> None: + parser = argparse.ArgumentParser(description="Fetch OpenRouter pricing JSON") + parser.add_argument( + "--output", + required=True, + help="Output JSON path, e.g. public/data/openrouter-pricing.json", + ) + parser.add_argument("--timeout", type=int, default=30) + args = parser.parse_args() + + snapshot = build_pricing_snapshot(timeout=args.timeout) + output = Path(args.output) + output.parent.mkdir(parents=True, exist_ok=True) + output.write_text(json.dumps(snapshot, indent=2) + "\n", encoding="utf-8") + print( + f"Wrote {snapshot['modelCount']} OpenRouter price entries " + f"to {output}" + ) + + +if __name__ == "__main__": + main() diff --git a/components/dashboard/openrouter-cost-panel.tsx b/components/dashboard/openrouter-cost-panel.tsx new file mode 100644 index 0000000..a9e8437 --- /dev/null +++ b/components/dashboard/openrouter-cost-panel.tsx @@ -0,0 +1,166 @@ +import { CircleDollarSign, ExternalLink } from "lucide-react"; +import { formatDateTime } from "@/lib/utils"; +import { ScrollReveal } from "@/components/shared/scroll-reveal"; +import type { OpenRouterPricingData } from "@/lib/types"; + +interface OpenRouterCostPanelProps { + pricing: OpenRouterPricingData; +} + +function moneyPerM(value: number | null | undefined): string { + if (value == null) return "n/a"; + if (value === 0) return "$0"; + if (value > 0 && value < 0.01) return "<$0.01"; + return `$${value.toLocaleString("en-US", { + minimumFractionDigits: value < 10 ? 2 : 0, + maximumFractionDigits: value < 10 ? 4 : 2, + })}`; +} + +function compactNumber(value: number | null | undefined): string { + if (value == null) return "n/a"; + return value.toLocaleString("en-US"); +} + +export function OpenRouterCostPanel({ pricing }: OpenRouterCostPanelProps) { + const paid = pricing.models.filter((entry) => !entry.isFree); + const cheapest = paid[0]; + const rows = paid.slice(0, 6); + + return ( + +
    +
    +
    +
    + +
    +
    +

    + OpenRouter cost watch +

    +

    + Cost per 1M tokens, refreshed daily +

    +

    + Pulled from OpenRouter's model catalog during JSON export. + Cheap does not mean good. It just means cheap. +

    +
    +
    + +
    +
    +

    Catalog

    +

    + {pricing.modelCount.toLocaleString()} +

    +
    +
    +

    Free

    +

    + {pricing.freeModelCount.toLocaleString()} +

    +
    +
    +

    Cheapest

    +

    + {cheapest + ? moneyPerM(cheapest.blendedOneInOneOutPerM) + : "n/a"} +

    +
    +
    +
    + +
    +
    +

    + Cheapest paid catalog entries +

    + + OpenRouter source + + +
    + + {pricing.error && ( +
    + Latest refresh failed: {pricing.error} +
    + )} + + {rows.length > 0 ? ( +
    +
    Category - {MODELS.map((model) => { + {models.map((model) => { const score = scores[model.id]?.[category.id] ?? 0; return ( diff --git a/components/charts/performance-line-chart.tsx b/components/charts/performance-line-chart.tsx index e263e3c..c80f91c 100644 --- a/components/charts/performance-line-chart.tsx +++ b/components/charts/performance-line-chart.tsx @@ -12,6 +12,7 @@ import { } from "recharts"; import { MODELS } from "@/lib/models"; import { formatDateTime } from "@/lib/utils"; +import type { ModelInfo } from "@/lib/types"; interface DataPoint { timestamp: string; @@ -23,6 +24,7 @@ interface PerformanceLineChartProps { height?: number; showLegend?: boolean; modelFilter?: string[]; + models?: ModelInfo[]; } export function PerformanceLineChart({ @@ -30,6 +32,7 @@ export function PerformanceLineChart({ height = 350, showLegend = true, modelFilter, + models = MODELS, }: PerformanceLineChartProps) { const chartData = data.map((d) => ({ timestamp: d.timestamp, @@ -38,8 +41,8 @@ export function PerformanceLineChart({ })); const modelsToShow = modelFilter - ? MODELS.filter((m) => modelFilter.includes(m.id)) - : MODELS; + ? models.filter((m) => modelFilter.includes(m.id)) + : models; return ( @@ -71,14 +74,14 @@ export function PerformanceLineChart({ fontSize: 12, }} formatter={(value: number, name: string) => { - const model = MODELS.find((m) => m.id === name); + const model = models.find((m) => m.id === name); return [value.toFixed(1), model?.name || name]; }} /> {showLegend && ( { - const model = MODELS.find((m) => m.id === value); + const model = models.find((m) => m.id === value); return model?.name || value; }} /> diff --git a/components/dashboard/latest-run-table.tsx b/components/dashboard/latest-run-table.tsx index f5dd8e9..b79d94e 100644 --- a/components/dashboard/latest-run-table.tsx +++ b/components/dashboard/latest-run-table.tsx @@ -3,17 +3,17 @@ import Link from "next/link"; import { cn } from "@/lib/utils"; import { formatScore, getScoreColor, formatDateTime } from "@/lib/utils"; -import { MODELS } from "@/lib/models"; import { CATEGORIES } from "@/lib/categories"; import { ScrollReveal } from "@/components/shared/scroll-reveal"; import { Clock, ExternalLink } from "lucide-react"; -import type { LatestRun } from "@/lib/types"; +import type { LatestRun, ModelInfo } from "@/lib/types"; interface LatestRunTableProps { run: LatestRun; + models: ModelInfo[]; } -export function LatestRunTable({ run }: LatestRunTableProps) { +export function LatestRunTable({ run, models }: LatestRunTableProps) { return (
    @@ -55,7 +55,7 @@ export function LatestRunTable({ run }: LatestRunTableProps) {
    + + + + + + + + + + + {rows.map((entry) => ( + + + + + + + + ))} + +
    + Model + + Input / 1M + + Output / 1M + + 1M in + 1M out + + Context +
    +

    + {entry.name} +

    +

    + {entry.id} +

    +     		+ {moneyPerM(entry.promptPerMTok)} + + {moneyPerM(entry.completionPerMTok)} + + {moneyPerM(entry.blendedOneInOneOutPerM)} + + {compactNumber(entry.contextLength)} +
    +
    + ) : ( +
    + No OpenRouter prices are available in the latest exported catalog. +
    + )} + +

    + Last refreshed {formatDateTime(pricing.updatedAt)}. Prices are + normalized from OpenRouter per-token catalog values into dollars per + million tokens. +

    +
    + + + ); +} diff --git a/components/dashboard/trust-status-panel.tsx b/components/dashboard/trust-status-panel.tsx new file mode 100644 index 0000000..cfb057f --- /dev/null +++ b/components/dashboard/trust-status-panel.tsx @@ -0,0 +1,320 @@ +import Link from "next/link"; +import { + AlertTriangle, + ArrowRight, + Eye, + ShieldAlert, + ShieldCheck, +} from "lucide-react"; +import { cn, formatDateTime } from "@/lib/utils"; +import type { LatestRun, ModelInfo, Regression } from "@/lib/types"; + +interface TrustStatusPanelProps { + latest: LatestRun; + models: ModelInfo[]; + activeRegressions: Regression[]; + changes: Record; +} + +type TrustStatus = "good" | "watch" | "do-not-trust-alone"; + +interface TrustReadout { + model: ModelInfo; + status: TrustStatus; + label: string; + badgeClass: string; + icon: typeof ShieldCheck; + securityScore: number; + thoroughnessScore: number; + compositeScore: number; + change: number; + activeRegression?: Regression; + weakestSecurityTest?: { testId: string; name: string; score: number }; + problem: string; + useItFor: string; + dontUseItFor: string; +} + +function statusRank(status: TrustStatus) { + switch (status) { + case "good": + return 0; + case "watch": + return 1; + default: + return 2; + } +} + +function buildReadout( + model: ModelInfo, + latest: LatestRun, + activeRegressions: Regression[], + change: number +): TrustReadout | null { + const result = latest.models[model.id]; + if (!result) return null; + + const security = result.categories["security-awareness"]; + const thoroughness = result.categories["code-thoroughness"]; + const securityScore = security?.avgScore ?? 0; + const thoroughnessScore = thoroughness?.avgScore ?? 0; + const activeRegression = activeRegressions.find( + (reg) => reg.modelId === model.id + ); + const weakestSecurityTest = security?.tests + ? [...security.tests].sort((a, b) => a.score - b.score)[0] + : undefined; + + let status: TrustStatus = "good"; + if ( + activeRegression?.severity === "major" || + securityScore < 85 || + result.compositeScore < 88 + ) { + status = "do-not-trust-alone"; + } else if ( + activeRegression || + securityScore < 93 || + thoroughnessScore < 80 || + result.compositeScore < 91 || + change < -2 + ) { + status = "watch"; + } + + const statusCopy = { + good: { + label: "Good", + badgeClass: "border-emerald-400/35 bg-emerald-400/10 text-emerald-300", + icon: ShieldCheck, + }, + watch: { + label: "Watch", + badgeClass: "border-amber-400/35 bg-amber-400/10 text-amber-300", + icon: Eye, + }, + "do-not-trust-alone": { + label: "Do not trust alone", + badgeClass: "border-red-400/40 bg-red-400/10 text-red-300", + icon: ShieldAlert, + }, + }[status]; + + const weakPoint = weakestSecurityTest + ? `${weakestSecurityTest.name} scored ${weakestSecurityTest.score.toFixed(1)}` + : "Security evidence is thin today"; + + let problem = "No active regression in the current run."; + if (activeRegression) { + problem = `${activeRegression.categoryName || "Overall"} dropped ${Math.abs( + activeRegression.dropPct + ).toFixed(1)}% over ${activeRegression.windowDays} days.`; + } else if (securityScore < 93) { + problem = `${weakPoint}. Do not round that up to fine.`; + } else if (thoroughnessScore < 80) { + problem = `Security looks strong, but thoroughness is only ${thoroughnessScore.toFixed( + 1 + )}.`; + } + + let useItFor = "Security review support and second-pass cleanup."; + let dontUseItFor = "Final approval without human review."; + if (status === "watch") { + useItFor = "Drafting, triage, and finding obvious misses."; + dontUseItFor = "The final answer on risky changes."; + } + if (status === "do-not-trust-alone") { + useItFor = "Brainstorming and quick comparison only."; + dontUseItFor = "Incident response, security fixes, or compliance calls."; + } + + return { + model, + status, + label: statusCopy.label, + badgeClass: statusCopy.badgeClass, + icon: statusCopy.icon, + securityScore, + thoroughnessScore, + compositeScore: result.compositeScore, + change, + activeRegression, + weakestSecurityTest, + problem, + useItFor, + dontUseItFor, + }; +} + +export function TrustStatusPanel({ + latest, + models, + activeRegressions, + changes, +}: TrustStatusPanelProps) { + const readouts = models + .map((model) => + buildReadout(model, latest, activeRegressions, changes[model.id] ?? 0) + ) + .filter((readout): readout is TrustReadout => Boolean(readout)) + .sort((a, b) => { + const statusDelta = statusRank(a.status) - statusRank(b.status); + if (statusDelta !== 0) return statusDelta; + return b.securityScore - a.securityScore; + }); + + const best = readouts[0]; + const riskiest = [...readouts].sort((a, b) => { + const statusDelta = statusRank(b.status) - statusRank(a.status); + if (statusDelta !== 0) return statusDelta; + return a.securityScore - b.securityScore; + })[0]; + + return ( +
    +
    +
    +
    +
    +
    + + + No vendor spin + +
    +

    + Can I trust this model today? +

    +

    + Short answer from today's receipts. Good means useful. It + does not mean let it ship security work by itself. +

    +
    + +
    +
    +

    + Best current answer +

    +

    + {best?.model.name ?? "No model data"} +

    +

    + {best + ? `${best.securityScore.toFixed(1)} security score` + : "Waiting on benchmark export"} +

    +
    +
    +

    + Watch first +

    +

    + {riskiest?.model.name ?? "No model data"} +

    +

    + {riskiest ? riskiest.problem : "No active readout"} +

    +
    +
    +
    + +
    + {readouts.map((readout) => { + const StatusIcon = readout.icon; + const evidenceHref = readout.weakestSecurityTest + ? `/evidence/${latest.runId}/${readout.weakestSecurityTest.testId}` + : `/models/${readout.model.slug}`; + + return ( +
    +
    +
    +
    + + {readout.model.name} + +
    + + + {readout.label} + +
    +
    +
    + +
    +

    {readout.problem}

    +
    +
    +

    + Use it for +

    +

    + {readout.useItFor} +

    +
    +
    +

    + Don't use it for +

    +

    + {readout.dontUseItFor} +

    +
    +
    +
    + +
    +
    +

    Security

    +

    + {readout.securityScore.toFixed(1)} +

    +
    +
    +

    Overall

    +

    + {readout.compositeScore.toFixed(1)} +

    +
    + +

    Receipts

    +

    + Open + +

    + +
    +
    + ); + })} +
    + +

    + Latest run completed {formatDateTime(latest.completedAt)}. Labels are + derived from benchmark scores, active regression flags, and the + weakest security-awareness receipt in the current run. +

    +
    +
    + ); +} diff --git a/components/shared/footer.tsx b/components/shared/footer.tsx index f660bf2..1a27fb7 100644 --- a/components/shared/footer.tsx +++ b/components/shared/footer.tsx @@ -1,5 +1,5 @@ import Link from "next/link"; -import { Activity, Github, Twitter } from "lucide-react"; +import { Github, Twitter } from "lucide-react"; export function Footer() { return ( @@ -10,7 +10,11 @@ export function Footer() {
    - +
    ModelRegression diff --git a/components/shared/navbar.tsx b/components/shared/navbar.tsx index 07b448d..c1aa743 100644 --- a/components/shared/navbar.tsx +++ b/components/shared/navbar.tsx @@ -34,7 +34,11 @@ export function Navbar() {
    - +
    diff --git a/lib/data.ts b/lib/data.ts index 8cb796f..1d87725 100644 --- a/lib/data.ts +++ b/lib/data.ts @@ -10,6 +10,7 @@ import type { OutageData, EvidenceData, ModelInfo, + OpenRouterPricingData, } from "./types"; import { MODELS } from "./models"; @@ -47,6 +48,10 @@ export function getOutages(): OutageData { return loadJSON("outages.json"); } +export function getOpenRouterPricing(): OpenRouterPricingData { + return loadJSON("openrouter-pricing.json"); +} + export function getAllModels(): ModelInfo[] { try { return loadJSON("models.json"); diff --git a/lib/types.ts b/lib/types.ts index 5ae423e..b91468e 100644 --- a/lib/types.ts +++ b/lib/types.ts @@ -146,6 +146,28 @@ export interface OutageData { >; } +export interface OpenRouterPricingEntry { + id: string; + canonicalSlug?: string | null; + name: string; + contextLength?: number | null; + promptPerMTok: number | null; + completionPerMTok: number | null; + inputCacheReadPerMTok?: number | null; + blendedOneInOneOutPerM: number; + isFree: boolean; +} + +export interface OpenRouterPricingData { + updatedAt: string; + source: string; + modelCount: number; + pricedModelCount: number; + freeModelCount: number; + error?: string; + models: OpenRouterPricingEntry[]; +} + export interface EvidenceData { test: { id: string; diff --git a/public/data/openrouter-pricing.json b/public/data/openrouter-pricing.json new file mode 100644 index 0000000..cad58f6 --- /dev/null +++ b/public/data/openrouter-pricing.json @@ -0,0 +1,3749 @@ +{ + "updatedAt": "2026-06-06T03:20:30.823693+00:00", + "source": "https://openrouter.ai/api/v1/models?output_modalities=text", + "modelCount": 340, + "pricedModelCount": 313, + "freeModelCount": 27, + "models": [ + { + "id": "inclusionai/ling-2.6-flash", + "canonicalSlug": "inclusionai/ling-2.6-flash-20260421", + "name": "inclusionAI: Ling-2.6-flash", + "contextLength": 262144, + "promptPerMTok": 0.01, + "completionPerMTok": 0.03, + "inputCacheReadPerMTok": 0.002, + "blendedOneInOneOutPerM": 0.04, + "isFree": false + }, + { + "id": "meta-llama/llama-3.1-8b-instruct", + "canonicalSlug": "meta-llama/llama-3.1-8b-instruct", + "name": "Meta: Llama 3.1 8B Instruct", + "contextLength": 131072, + "promptPerMTok": 0.02, + "completionPerMTok": 0.03, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 0.05, + "isFree": false + }, + { + "id": "mistralai/mistral-nemo", + "canonicalSlug": "mistralai/mistral-nemo", + "name": "Mistral: Mistral Nemo", + "contextLength": 131072, + "promptPerMTok": 0.02, + "completionPerMTok": 0.03, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 0.05, + "isFree": false + }, + { + "id": "meta-llama/llama-3-8b-instruct", + "canonicalSlug": "meta-llama/llama-3-8b-instruct", + "name": "Meta: Llama 3 8B Instruct", + "contextLength": 8192, + "promptPerMTok": 0.04, + "completionPerMTok": 0.04, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 0.08, + "isFree": false + }, + { + "id": "sao10k/l3-lunaris-8b", + "canonicalSlug": "sao10k/l3-lunaris-8b", + "name": "Sao10K: Llama 3 8B Lunaris", + "contextLength": 8192, + "promptPerMTok": 0.04, + "completionPerMTok": 0.05, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 0.09, + "isFree": false + }, + { + "id": "google/gemma-3-4b-it", + "canonicalSlug": "google/gemma-3-4b-it", + "name": "Google: Gemma 3 4B", + "contextLength": 131072, + "promptPerMTok": 0.04, + "completionPerMTok": 0.08, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 0.12, + "isFree": false + }, + { + "id": "gryphe/mythomax-l2-13b", + "canonicalSlug": "gryphe/mythomax-l2-13b", + "name": "MythoMax 13B", + "contextLength": 4096, + "promptPerMTok": 0.06, + "completionPerMTok": 0.06, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 0.12, + "isFree": false + }, + { + "id": "ibm-granite/granite-4.0-h-micro", + "canonicalSlug": "ibm-granite/granite-4.0-h-micro", + "name": "IBM: Granite 4.0 Micro", + "contextLength": 131000, + "promptPerMTok": 0.017, + "completionPerMTok": 0.112, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 0.129, + "isFree": false + }, + { + "id": "mistralai/mistral-small-24b-instruct-2501", + "canonicalSlug": "mistralai/mistral-small-24b-instruct-2501", + "name": "Mistral: Mistral Small 3", + "contextLength": 32768, + "promptPerMTok": 0.05, + "completionPerMTok": 0.08, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 0.13, + "isFree": false + }, + { + "id": "qwen/qwen-2.5-7b-instruct", + "canonicalSlug": "qwen/qwen-2.5-7b-instruct", + "name": "Qwen: Qwen2.5 7B Instruct", + "contextLength": 131072, + "promptPerMTok": 0.04, + "completionPerMTok": 0.1, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 0.14, + "isFree": false + }, + { + "id": "ibm-granite/granite-4.1-8b", + "canonicalSlug": "ibm-granite/granite-4.1-8b-20260429", + "name": "IBM: Granite 4.1 8B", + "contextLength": 131072, + "promptPerMTok": 0.05, + "completionPerMTok": 0.1, + "inputCacheReadPerMTok": 0.05, + "blendedOneInOneOutPerM": 0.15, + "isFree": false + }, + { + "id": "liquid/lfm-2-24b-a2b", + "canonicalSlug": "liquid/lfm-2-24b-a2b-20260224", + "name": "LiquidAI: LFM2-24B-A2B", + "contextLength": 128000, + "promptPerMTok": 0.03, + "completionPerMTok": 0.12, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 0.15, + "isFree": false + }, + { + "id": "openai/gpt-oss-20b", + "canonicalSlug": "openai/gpt-oss-20b", + "name": "OpenAI: gpt-oss-20b", + "contextLength": 131072, + "promptPerMTok": 0.029, + "completionPerMTok": 0.14, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 0.169, + "isFree": false + }, + { + "id": "google/gemma-3-12b-it", + "canonicalSlug": "google/gemma-3-12b-it", + "name": "Google: Gemma 3 12B", + "contextLength": 131072, + "promptPerMTok": 0.04, + "completionPerMTok": 0.13, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 0.17, + "isFree": false + }, + { + "id": "qwen/qwen3-235b-a22b-2507", + "canonicalSlug": "qwen/qwen3-235b-a22b-07-25", + "name": "Qwen: Qwen3 235B A22B Instruct 2507", + "contextLength": 262144, + "promptPerMTok": 0.071, + "completionPerMTok": 0.1, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 0.171, + "isFree": false + }, + { + "id": "amazon/nova-micro-v1", + "canonicalSlug": "amazon/nova-micro-v1", + "name": "Amazon: Nova Micro 1.0", + "contextLength": 128000, + "promptPerMTok": 0.035, + "completionPerMTok": 0.14, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 0.175, + "isFree": false + }, + { + "id": "google/gemma-3n-e4b-it", + "canonicalSlug": "google/gemma-3n-e4b-it", + "name": "Google: Gemma 3n 4B", + "contextLength": 32768, + "promptPerMTok": 0.06, + "completionPerMTok": 0.12, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 0.18, + "isFree": false + }, + { + "id": "cohere/command-r7b-12-2024", + "canonicalSlug": "cohere/command-r7b-12-2024", + "name": "Cohere: Command R7B (12-2024)", + "contextLength": 128000, + "promptPerMTok": 0.0375, + "completionPerMTok": 0.15, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 0.1875, + "isFree": false + }, + { + "id": "qwen/qwen3.5-9b", + "canonicalSlug": "qwen/qwen3.5-9b-20260310", + "name": "Qwen: Qwen3.5-9B", + "contextLength": 262144, + "promptPerMTok": 0.04, + "completionPerMTok": 0.15, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 0.19, + "isFree": false + }, + { + "id": "arcee-ai/trinity-mini", + "canonicalSlug": "arcee-ai/trinity-mini-20251201", + "name": "Arcee AI: Trinity Mini", + "contextLength": 131072, + "promptPerMTok": 0.045, + "completionPerMTok": 0.15, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 0.195, + "isFree": false + }, + { + "id": "mistralai/ministral-3b-2512", + "canonicalSlug": "mistralai/ministral-3b-2512", + "name": "Mistral: Ministral 3 3B 2512", + "contextLength": 131072, + "promptPerMTok": 0.1, + "completionPerMTok": 0.1, + "inputCacheReadPerMTok": 0.01, + "blendedOneInOneOutPerM": 0.2, + "isFree": false + }, + { + "id": "nvidia/nemotron-nano-9b-v2", + "canonicalSlug": "nvidia/nemotron-nano-9b-v2", + "name": "NVIDIA: Nemotron Nano 9B V2", + "contextLength": 131072, + "promptPerMTok": 0.04, + "completionPerMTok": 0.16, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 0.2, + "isFree": false + }, + { + "id": "qwen/qwen3-235b-a22b-thinking-2507", + "canonicalSlug": "qwen/qwen3-235b-a22b-thinking-2507", + "name": "Qwen: Qwen3 235B A22B Thinking 2507", + "contextLength": 262144, + "promptPerMTok": 0.1, + "completionPerMTok": 0.1, + "inputCacheReadPerMTok": 0.1, + "blendedOneInOneOutPerM": 0.2, + "isFree": false + }, + { + "id": "rekaai/reka-edge", + "canonicalSlug": "rekaai/reka-edge-2603", + "name": "Reka Edge", + "contextLength": 16384, + "promptPerMTok": 0.1, + "completionPerMTok": 0.1, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 0.2, + "isFree": false + }, + { + "id": "z-ai/glm-4-32b", + "canonicalSlug": "z-ai/glm-4-32b-0414", + "name": "Z.ai: GLM 4 32B ", + "contextLength": 128000, + "promptPerMTok": 0.1, + "completionPerMTok": 0.1, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 0.2, + "isFree": false + }, + { + "id": "microsoft/phi-4", + "canonicalSlug": "microsoft/phi-4", + "name": "Microsoft: Phi 4", + "contextLength": 16384, + "promptPerMTok": 0.065, + "completionPerMTok": 0.14, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 0.205, + "isFree": false + }, + { + "id": "openai/gpt-oss-120b", + "canonicalSlug": "openai/gpt-oss-120b", + "name": "OpenAI: gpt-oss-120b", + "contextLength": 131072, + "promptPerMTok": 0.039, + "completionPerMTok": 0.18, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 0.219, + "isFree": false + }, + { + "id": "meta-llama/llama-3.2-1b-instruct", + "canonicalSlug": "meta-llama/llama-3.2-1b-instruct", + "name": "Meta: Llama 3.2 1B Instruct", + "contextLength": 131072, + "promptPerMTok": 0.027, + "completionPerMTok": 0.201, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 0.228, + "isFree": false + }, + { + "id": "google/gemma-3-27b-it", + "canonicalSlug": "google/gemma-3-27b-it", + "name": "Google: Gemma 3 27B", + "contextLength": 131072, + "promptPerMTok": 0.08, + "completionPerMTok": 0.16, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 0.24, + "isFree": false + }, + { + "id": "qwen/qwen3-30b-a3b-instruct-2507", + "canonicalSlug": "qwen/qwen3-30b-a3b-instruct-2507", + "name": "Qwen: Qwen3 30B A3B Instruct 2507", + "contextLength": 131072, + "promptPerMTok": 0.04815, + "completionPerMTok": 0.19305, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 0.2412, + "isFree": false + }, + { + "id": "nvidia/nemotron-3-nano-30b-a3b", + "canonicalSlug": "nvidia/nemotron-3-nano-30b-a3b", + "name": "NVIDIA: Nemotron 3 Nano 30B A3B", + "contextLength": 262144, + "promptPerMTok": 0.05, + "completionPerMTok": 0.2, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 0.25, + "isFree": false + }, + { + "id": "tencent/hy3-preview", + "canonicalSlug": "tencent/hy3-preview-20260421", + "name": "Tencent: Hy3 preview", + "contextLength": 262144, + "promptPerMTok": 0.063, + "completionPerMTok": 0.21, + "inputCacheReadPerMTok": 0.021, + "blendedOneInOneOutPerM": 0.273, + "isFree": false + }, + { + "id": "mistralai/mistral-small-3.2-24b-instruct", + "canonicalSlug": "mistralai/mistral-small-3.2-24b-instruct-2506", + "name": "Mistral: Mistral Small 3.2 24B", + "contextLength": 128000, + "promptPerMTok": 0.075, + "completionPerMTok": 0.2, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 0.275, + "isFree": false + }, + { + "id": "deepseek/deepseek-v4-flash", + "canonicalSlug": "deepseek/deepseek-v4-flash-20260423", + "name": "DeepSeek: DeepSeek V4 Flash", + "contextLength": 1048576, + "promptPerMTok": 0.0983, + "completionPerMTok": 0.1966, + "inputCacheReadPerMTok": 0.0197, + "blendedOneInOneOutPerM": 0.2949, + "isFree": false + }, + { + "id": "amazon/nova-lite-v1", + "canonicalSlug": "amazon/nova-lite-v1", + "name": "Amazon: Nova Lite 1.0", + "contextLength": 300000, + "promptPerMTok": 0.06, + "completionPerMTok": 0.24, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 0.3, + "isFree": false + }, + { + "id": "bytedance/ui-tars-1.5-7b", + "canonicalSlug": "bytedance/ui-tars-1.5-7b", + "name": "ByteDance: UI-TARS 7B ", + "contextLength": 128000, + "promptPerMTok": 0.1, + "completionPerMTok": 0.2, + "inputCacheReadPerMTok": 0.1, + "blendedOneInOneOutPerM": 0.3, + "isFree": false + }, + { + "id": "essentialai/rnj-1-instruct", + "canonicalSlug": "essentialai/rnj-1-instruct", + "name": "EssentialAI: Rnj 1 Instruct", + "contextLength": 32768, + "promptPerMTok": 0.15, + "completionPerMTok": 0.15, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 0.3, + "isFree": false + }, + { + "id": "mistralai/ministral-8b-2512", + "canonicalSlug": "mistralai/ministral-8b-2512", + "name": "Mistral: Ministral 3 8B 2512", + "contextLength": 262144, + "promptPerMTok": 0.15, + "completionPerMTok": 0.15, + "inputCacheReadPerMTok": 0.015, + "blendedOneInOneOutPerM": 0.3, + "isFree": false + }, + { + "id": "rekaai/reka-flash-3", + "canonicalSlug": "rekaai/reka-flash-3", + "name": "Reka Flash 3", + "contextLength": 65536, + "promptPerMTok": 0.1, + "completionPerMTok": 0.2, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 0.3, + "isFree": false + }, + { + "id": "qwen/qwen3.5-flash-02-23", + "canonicalSlug": "qwen/qwen3.5-flash-20260224", + "name": "Qwen: Qwen3.5-Flash", + "contextLength": 1000000, + "promptPerMTok": 0.065, + "completionPerMTok": 0.26, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 0.325, + "isFree": false + }, + { + "id": "qwen/qwen3-14b", + "canonicalSlug": "qwen/qwen3-14b-04-28", + "name": "Qwen: Qwen3 14B", + "contextLength": 131702, + "promptPerMTok": 0.1, + "completionPerMTok": 0.24, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 0.34, + "isFree": false + }, + { + "id": "qwen/qwen3-coder-30b-a3b-instruct", + "canonicalSlug": "qwen/qwen3-coder-30b-a3b-instruct", + "name": "Qwen: Qwen3 Coder 30B A3B Instruct", + "contextLength": 160000, + "promptPerMTok": 0.07, + "completionPerMTok": 0.27, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 0.34, + "isFree": false + }, + { + "id": "arcee-ai/spotlight", + "canonicalSlug": "arcee-ai/spotlight", + "name": "Arcee AI: Spotlight", + "contextLength": 131072, + "promptPerMTok": 0.18, + "completionPerMTok": 0.18, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 0.36, + "isFree": false + }, + { + "id": "meta-llama/llama-guard-4-12b", + "canonicalSlug": "meta-llama/llama-guard-4-12b", + "name": "Meta: Llama Guard 4 12B", + "contextLength": 163840, + "promptPerMTok": 0.18, + "completionPerMTok": 0.18, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 0.36, + "isFree": false + }, + { + "id": "qwen/qwen3-32b", + "canonicalSlug": "qwen/qwen3-32b-04-28", + "name": "Qwen: Qwen3 32B", + "contextLength": 131072, + "promptPerMTok": 0.08, + "completionPerMTok": 0.28, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 0.36, + "isFree": false + }, + { + "id": "bytedance-seed/seed-1.6-flash", + "canonicalSlug": "bytedance-seed/seed-1.6-flash-20250625", + "name": "ByteDance Seed: Seed 1.6 Flash", + "contextLength": 262144, + "promptPerMTok": 0.075, + "completionPerMTok": 0.3, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 0.375, + "isFree": false + }, + { + "id": "openai/gpt-oss-safeguard-20b", + "canonicalSlug": "openai/gpt-oss-safeguard-20b", + "name": "OpenAI: gpt-oss-safeguard-20b", + "contextLength": 131072, + "promptPerMTok": 0.075, + "completionPerMTok": 0.3, + "inputCacheReadPerMTok": 0.037, + "blendedOneInOneOutPerM": 0.375, + "isFree": false + }, + { + "id": "meta-llama/llama-4-scout", + "canonicalSlug": "meta-llama/llama-4-scout-17b-16e-instruct", + "name": "Meta: Llama 4 Scout", + "contextLength": 10000000, + "promptPerMTok": 0.08, + "completionPerMTok": 0.3, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 0.38, + "isFree": false + }, + { + "id": "meta-llama/llama-3.2-3b-instruct", + "canonicalSlug": "meta-llama/llama-3.2-3b-instruct", + "name": "Meta: Llama 3.2 3B Instruct", + "contextLength": 131072, + "promptPerMTok": 0.0509, + "completionPerMTok": 0.335, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 0.3859, + "isFree": false + }, + { + "id": "google/gemma-4-26b-a4b-it", + "canonicalSlug": "google/gemma-4-26b-a4b-it-20260403", + "name": "Google: Gemma 4 26B A4B ", + "contextLength": 262144, + "promptPerMTok": 0.06, + "completionPerMTok": 0.33, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 0.39, + "isFree": false + }, + { + "id": "stepfun/step-3.5-flash", + "canonicalSlug": "stepfun/step-3.5-flash", + "name": "StepFun: Step 3.5 Flash", + "contextLength": 262144, + "promptPerMTok": 0.09, + "completionPerMTok": 0.3, + "inputCacheReadPerMTok": 0.02, + "blendedOneInOneOutPerM": 0.39, + "isFree": false + }, + { + "id": "mistralai/ministral-14b-2512", + "canonicalSlug": "mistralai/ministral-14b-2512", + "name": "Mistral: Ministral 3 14B 2512", + "contextLength": 262144, + "promptPerMTok": 0.2, + "completionPerMTok": 0.2, + "inputCacheReadPerMTok": 0.02, + "blendedOneInOneOutPerM": 0.4, + "isFree": false + }, + { + "id": "mistralai/voxtral-small-24b-2507", + "canonicalSlug": "mistralai/voxtral-small-24b-2507", + "name": "Mistral: Voxtral Small 24B 2507", + "contextLength": 32000, + "promptPerMTok": 0.1, + "completionPerMTok": 0.3, + "inputCacheReadPerMTok": 0.01, + "blendedOneInOneOutPerM": 0.4, + "isFree": false + }, + { + "id": "xiaomi/mimo-v2-flash", + "canonicalSlug": "xiaomi/mimo-v2-flash-20251210", + "name": "Xiaomi: MiMo-V2-Flash", + "contextLength": 262144, + "promptPerMTok": 0.1, + "completionPerMTok": 0.3, + "inputCacheReadPerMTok": 0.01, + "blendedOneInOneOutPerM": 0.4, + "isFree": false + }, + { + "id": "meta-llama/llama-3.3-70b-instruct", + "canonicalSlug": "meta-llama/llama-3.3-70b-instruct", + "name": "Meta: Llama 3.3 70B Instruct", + "contextLength": 131072, + "promptPerMTok": 0.1, + "completionPerMTok": 0.32, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 0.42, + "isFree": false + }, + { + "id": "xiaomi/mimo-v2.5", + "canonicalSlug": "xiaomi/mimo-v2.5-20260422", + "name": "Xiaomi: MiMo-V2.5", + "contextLength": 1048576, + "promptPerMTok": 0.14, + "completionPerMTok": 0.28, + "inputCacheReadPerMTok": 0.0028, + "blendedOneInOneOutPerM": 0.42, + "isFree": false + }, + { + "id": "microsoft/phi-4-mini-instruct", + "canonicalSlug": "microsoft/phi-4-mini-instruct", + "name": "Microsoft: Phi 4 Mini Instruct", + "contextLength": 131072, + "promptPerMTok": 0.08, + "completionPerMTok": 0.35, + "inputCacheReadPerMTok": 0.08, + "blendedOneInOneOutPerM": 0.43, + "isFree": false + }, + { + "id": "openai/gpt-5-nano", + "canonicalSlug": "openai/gpt-5-nano-2025-08-07", + "name": "OpenAI: GPT-5 Nano", + "contextLength": 400000, + "promptPerMTok": 0.05, + "completionPerMTok": 0.4, + "inputCacheReadPerMTok": 0.01, + "blendedOneInOneOutPerM": 0.45, + "isFree": false + }, + { + "id": "qwen/qwen3-8b", + "canonicalSlug": "qwen/qwen3-8b-04-28", + "name": "Qwen: Qwen3 8B", + "contextLength": 131072, + "promptPerMTok": 0.05, + "completionPerMTok": 0.4, + "inputCacheReadPerMTok": 0.05, + "blendedOneInOneOutPerM": 0.45, + "isFree": false + }, + { + "id": "z-ai/glm-4.7-flash", + "canonicalSlug": "z-ai/glm-4.7-flash-20260119", + "name": "Z.ai: GLM 4.7 Flash", + "contextLength": 202752, + "promptPerMTok": 0.06, + "completionPerMTok": 0.4, + "inputCacheReadPerMTok": 0.01, + "blendedOneInOneOutPerM": 0.46, + "isFree": false + }, + { + "id": "google/gemma-4-31b-it", + "canonicalSlug": "google/gemma-4-31b-it-20260402", + "name": "Google: Gemma 4 31B", + "contextLength": 262144, + "promptPerMTok": 0.12, + "completionPerMTok": 0.36, + "inputCacheReadPerMTok": 0.09, + "blendedOneInOneOutPerM": 0.48, + "isFree": false + }, + { + "id": "qwen/qwen3-30b-a3b-thinking-2507", + "canonicalSlug": "qwen/qwen3-30b-a3b-thinking-2507", + "name": "Qwen: Qwen3 30B A3B Thinking 2507", + "contextLength": 131072, + "promptPerMTok": 0.08, + "completionPerMTok": 0.4, + "inputCacheReadPerMTok": 0.08, + "blendedOneInOneOutPerM": 0.48, + "isFree": false + }, + { + "id": "meta-llama/llama-3.2-11b-vision-instruct", + "canonicalSlug": "meta-llama/llama-3.2-11b-vision-instruct", + "name": "Meta: Llama 3.2 11B Vision Instruct", + "contextLength": 131072, + "promptPerMTok": 0.245, + "completionPerMTok": 0.245, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 0.49, + "isFree": false + }, + { + "id": "bytedance-seed/seed-2.0-mini", + "canonicalSlug": "bytedance-seed/seed-2.0-mini-20260224", + "name": "ByteDance Seed: Seed-2.0-Mini", + "contextLength": 262144, + "promptPerMTok": 0.1, + "completionPerMTok": 0.4, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 0.5, + "isFree": false + }, + { + "id": "google/gemini-2.5-flash-lite", + "canonicalSlug": "google/gemini-2.5-flash-lite", + "name": "Google: Gemini 2.5 Flash Lite", + "contextLength": 1048576, + "promptPerMTok": 0.1, + "completionPerMTok": 0.4, + "inputCacheReadPerMTok": 0.01, + "blendedOneInOneOutPerM": 0.5, + "isFree": false + }, + { + "id": "google/gemini-2.5-flash-lite-preview-09-2025", + "canonicalSlug": "google/gemini-2.5-flash-lite-preview-09-2025", + "name": "Google: Gemini 2.5 Flash Lite Preview 09-2025", + "contextLength": 1048576, + "promptPerMTok": 0.1, + "completionPerMTok": 0.4, + "inputCacheReadPerMTok": 0.01, + "blendedOneInOneOutPerM": 0.5, + "isFree": false + }, + { + "id": "nvidia/llama-3.3-nemotron-super-49b-v1.5", + "canonicalSlug": "nvidia/llama-3.3-nemotron-super-49b-v1.5", + "name": "NVIDIA: Llama 3.3 Nemotron Super 49B V1.5", + "contextLength": 131072, + "promptPerMTok": 0.1, + "completionPerMTok": 0.4, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 0.5, + "isFree": false + }, + { + "id": "openai/gpt-4.1-nano", + "canonicalSlug": "openai/gpt-4.1-nano-2025-04-14", + "name": "OpenAI: GPT-4.1 Nano", + "contextLength": 1047576, + "promptPerMTok": 0.1, + "completionPerMTok": 0.4, + "inputCacheReadPerMTok": 0.025, + "blendedOneInOneOutPerM": 0.5, + "isFree": false + }, + { + "id": "meta-llama/llama-guard-3-8b", + "canonicalSlug": "meta-llama/llama-guard-3-8b", + "name": "Llama Guard 3 8B", + "contextLength": 131072, + "promptPerMTok": 0.484, + "completionPerMTok": 0.03, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 0.514, + "isFree": false + }, + { + "id": "qwen/qwen3-vl-32b-instruct", + "canonicalSlug": "qwen/qwen3-vl-32b-instruct", + "name": "Qwen: Qwen3 VL 32B Instruct", + "contextLength": 262144, + "promptPerMTok": 0.104, + "completionPerMTok": 0.416, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 0.52, + "isFree": false + }, + { + "id": "nousresearch/hermes-4-70b", + "canonicalSlug": "nousresearch/hermes-4-70b", + "name": "Nous: Hermes 4 70B", + "contextLength": 131072, + "promptPerMTok": 0.13, + "completionPerMTok": 0.4, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 0.53, + "isFree": false + }, + { + "id": "nvidia/nemotron-3-super-120b-a12b", + "canonicalSlug": "nvidia/nemotron-3-super-120b-a12b-20230311", + "name": "NVIDIA: Nemotron 3 Super", + "contextLength": 1000000, + "promptPerMTok": 0.09, + "completionPerMTok": 0.45, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 0.54, + "isFree": false + }, + { + "id": "qwen/qwen3-30b-a3b", + "canonicalSlug": "qwen/qwen3-30b-a3b-04-28", + "name": "Qwen: Qwen3 30B A3B", + "contextLength": 131072, + "promptPerMTok": 0.09, + "completionPerMTok": 0.45, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 0.54, + "isFree": false + }, + { + "id": "deepseek/deepseek-v3.2", + "canonicalSlug": "deepseek/deepseek-v3.2-20251201", + "name": "DeepSeek: DeepSeek V3.2", + "contextLength": 131072, + "promptPerMTok": 0.2288, + "completionPerMTok": 0.3432, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 0.572, + "isFree": false + }, + { + "id": "deepseek/deepseek-r1-distill-qwen-32b", + "canonicalSlug": "deepseek/deepseek-r1-distill-qwen-32b", + "name": "DeepSeek: R1 Distill Qwen 32B", + "contextLength": 128000, + "promptPerMTok": 0.29, + "completionPerMTok": 0.29, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 0.58, + "isFree": false + }, + { + "id": "qwen/qwen3-vl-8b-instruct", + "canonicalSlug": "qwen/qwen3-vl-8b-instruct", + "name": "Qwen: Qwen3 VL 8B Instruct", + "contextLength": 256000, + "promptPerMTok": 0.08, + "completionPerMTok": 0.5, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 0.58, + "isFree": false + }, + { + "id": "nousresearch/hermes-3-llama-3.1-70b", + "canonicalSlug": "nousresearch/hermes-3-llama-3.1-70b", + "name": "Nous: Hermes 3 70B Instruct", + "contextLength": 131072, + "promptPerMTok": 0.3, + "completionPerMTok": 0.3, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 0.6, + "isFree": false + }, + { + "id": "thedrummer/rocinante-12b", + "canonicalSlug": "thedrummer/rocinante-12b", + "name": "TheDrummer: Rocinante 12B", + "contextLength": 32768, + "promptPerMTok": 0.17, + "completionPerMTok": 0.43, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 0.6, + "isFree": false + }, + { + "id": "nex-agi/deepseek-v3.1-nex-n1", + "canonicalSlug": "nex-agi/deepseek-v3.1-nex-n1", + "name": "Nex AGI: DeepSeek V3.1 Nex N1", + "contextLength": 131072, + "promptPerMTok": 0.135, + "completionPerMTok": 0.5, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 0.635, + "isFree": false + }, + { + "id": "allenai/olmo-3-32b-think", + "canonicalSlug": "allenai/olmo-3-32b-think-20251121", + "name": "AllenAI: Olmo 3 32B Think", + "contextLength": 65536, + "promptPerMTok": 0.15, + "completionPerMTok": 0.5, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 0.65, + "isFree": false + }, + { + "id": "qwen/qwen3-vl-30b-a3b-instruct", + "canonicalSlug": "qwen/qwen3-vl-30b-a3b-instruct", + "name": "Qwen: Qwen3 VL 30B A3B Instruct", + "contextLength": 262144, + "promptPerMTok": 0.13, + "completionPerMTok": 0.52, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 0.65, + "isFree": false + }, + { + "id": "deepseek/deepseek-v3.2-exp", + "canonicalSlug": "deepseek/deepseek-v3.2-exp", + "name": "DeepSeek: DeepSeek V3.2 Exp", + "contextLength": 163840, + "promptPerMTok": 0.27, + "completionPerMTok": 0.41, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 0.68, + "isFree": false + }, + { + "id": "baidu/ernie-4.5-vl-28b-a3b", + "canonicalSlug": "baidu/ernie-4.5-vl-28b-a3b", + "name": "Baidu: ERNIE 4.5 VL 28B A3B", + "contextLength": 131072, + "promptPerMTok": 0.14, + "completionPerMTok": 0.56, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 0.7, + "isFree": false + }, + { + "id": "inclusionai/ling-2.6-1t", + "canonicalSlug": "inclusionai/ling-2.6-1t-20260423", + "name": "inclusionAI: Ling-2.6-1T", + "contextLength": 262144, + "promptPerMTok": 0.075, + "completionPerMTok": 0.625, + "inputCacheReadPerMTok": 0.015, + "blendedOneInOneOutPerM": 0.7, + "isFree": false + }, + { + "id": "inclusionai/ring-2.6-1t", + "canonicalSlug": "inclusionai/ring-2.6-1t-20260508", + "name": "inclusionAI: Ring-2.6-1T", + "contextLength": 262144, + "promptPerMTok": 0.075, + "completionPerMTok": 0.625, + "inputCacheReadPerMTok": 0.015, + "blendedOneInOneOutPerM": 0.7, + "isFree": false + }, + { + "id": "tencent/hunyuan-a13b-instruct", + "canonicalSlug": "tencent/hunyuan-a13b-instruct", + "name": "Tencent: Hunyuan A13B Instruct", + "contextLength": 131072, + "promptPerMTok": 0.14, + "completionPerMTok": 0.57, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 0.71, + "isFree": false + }, + { + "id": "cohere/command-r-08-2024", + "canonicalSlug": "cohere/command-r-08-2024", + "name": "Cohere: Command R (08-2024)", + "contextLength": 128000, + "promptPerMTok": 0.15, + "completionPerMTok": 0.6, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 0.75, + "isFree": false + }, + { + "id": "meta-llama/llama-4-maverick", + "canonicalSlug": "meta-llama/llama-4-maverick-17b-128e-instruct", + "name": "Meta: Llama 4 Maverick", + "contextLength": 1048576, + "promptPerMTok": 0.15, + "completionPerMTok": 0.6, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 0.75, + "isFree": false + }, + { + "id": "mistralai/mistral-small-2603", + "canonicalSlug": "mistralai/mistral-small-2603", + "name": "Mistral: Mistral Small 4", + "contextLength": 262144, + "promptPerMTok": 0.15, + "completionPerMTok": 0.6, + "inputCacheReadPerMTok": 0.015, + "blendedOneInOneOutPerM": 0.75, + "isFree": false + }, + { + "id": "openai/gpt-4o-mini", + "canonicalSlug": "openai/gpt-4o-mini", + "name": "OpenAI: GPT-4o-mini", + "contextLength": 128000, + "promptPerMTok": 0.15, + "completionPerMTok": 0.6, + "inputCacheReadPerMTok": 0.075, + "blendedOneInOneOutPerM": 0.75, + "isFree": false + }, + { + "id": "openai/gpt-4o-mini-2024-07-18", + "canonicalSlug": "openai/gpt-4o-mini-2024-07-18", + "name": "OpenAI: GPT-4o-mini (2024-07-18)", + "contextLength": 128000, + "promptPerMTok": 0.15, + "completionPerMTok": 0.6, + "inputCacheReadPerMTok": 0.075, + "blendedOneInOneOutPerM": 0.75, + "isFree": false + }, + { + "id": "openai/gpt-4o-mini-search-preview", + "canonicalSlug": "openai/gpt-4o-mini-search-preview-2025-03-11", + "name": "OpenAI: GPT-4o-mini Search Preview", + "contextLength": 128000, + "promptPerMTok": 0.15, + "completionPerMTok": 0.6, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 0.75, + "isFree": false + }, + { + "id": "upstage/solar-pro-3", + "canonicalSlug": "upstage/solar-pro-3", + "name": "Upstage: Solar Pro 3", + "contextLength": 128000, + "promptPerMTok": 0.15, + "completionPerMTok": 0.6, + "inputCacheReadPerMTok": 0.015, + "blendedOneInOneOutPerM": 0.75, + "isFree": false + }, + { + "id": "qwen/qwen-2.5-72b-instruct", + "canonicalSlug": "qwen/qwen-2.5-72b-instruct", + "name": "Qwen2.5 72B Instruct", + "contextLength": 131072, + "promptPerMTok": 0.36, + "completionPerMTok": 0.4, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 0.76, + "isFree": false + }, + { + "id": "meta-llama/llama-3.1-70b-instruct", + "canonicalSlug": "meta-llama/llama-3.1-70b-instruct", + "name": "Meta: Llama 3.1 70B Instruct", + "contextLength": 131072, + "promptPerMTok": 0.4, + "completionPerMTok": 0.4, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 0.8, + "isFree": false + }, + { + "id": "mistralai/mistral-saba", + "canonicalSlug": "mistralai/mistral-saba-2502", + "name": "Mistral: Saba", + "contextLength": 32768, + "promptPerMTok": 0.2, + "completionPerMTok": 0.6, + "inputCacheReadPerMTok": 0.02, + "blendedOneInOneOutPerM": 0.8, + "isFree": false + }, + { + "id": "thedrummer/cydonia-24b-v4.1", + "canonicalSlug": "thedrummer/cydonia-24b-v4.1", + "name": "TheDrummer: Cydonia 24B V4.1", + "contextLength": 131072, + "promptPerMTok": 0.3, + "completionPerMTok": 0.5, + "inputCacheReadPerMTok": 0.15, + "blendedOneInOneOutPerM": 0.8, + "isFree": false + }, + { + "id": "thedrummer/unslopnemo-12b", + "canonicalSlug": "thedrummer/unslopnemo-12b", + "name": "TheDrummer: UnslopNemo 12B", + "contextLength": 32768, + "promptPerMTok": 0.4, + "completionPerMTok": 0.4, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 0.8, + "isFree": false + }, + { + "id": "qwen/qwen3-next-80b-a3b-thinking", + "canonicalSlug": "qwen/qwen3-next-80b-a3b-thinking-2509", + "name": "Qwen: Qwen3 Next 80B A3B Thinking", + "contextLength": 262144, + "promptPerMTok": 0.0975, + "completionPerMTok": 0.78, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 0.8775, + "isFree": false + }, + { + "id": "mistralai/mistral-small-3.1-24b-instruct", + "canonicalSlug": "mistralai/mistral-small-3.1-24b-instruct-2503", + "name": "Mistral: Mistral Small 3.1 24B", + "contextLength": 128000, + "promptPerMTok": 0.351, + "completionPerMTok": 0.555, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 0.906, + "isFree": false + }, + { + "id": "qwen/qwen3-coder-next", + "canonicalSlug": "qwen/qwen3-coder-next-2025-02-03", + "name": "Qwen: Qwen3 Coder Next", + "contextLength": 262144, + "promptPerMTok": 0.11, + "completionPerMTok": 0.8, + "inputCacheReadPerMTok": 0.07, + "blendedOneInOneOutPerM": 0.91, + "isFree": false + }, + { + "id": "deepseek/deepseek-chat-v3-0324", + "canonicalSlug": "deepseek/deepseek-chat-v3-0324", + "name": "DeepSeek: DeepSeek V3 0324", + "contextLength": 163840, + "promptPerMTok": 0.2, + "completionPerMTok": 0.77, + "inputCacheReadPerMTok": 0.135, + "blendedOneInOneOutPerM": 0.97, + "isFree": false + }, + { + "id": "z-ai/glm-4.5-air", + "canonicalSlug": "z-ai/glm-4.5-air", + "name": "Z.ai: GLM 4.5 Air", + "contextLength": 131072, + "promptPerMTok": 0.125, + "completionPerMTok": 0.85, + "inputCacheReadPerMTok": 0.06, + "blendedOneInOneOutPerM": 0.975, + "isFree": false + }, + { + "id": "deepseek/deepseek-chat-v3.1", + "canonicalSlug": "deepseek/deepseek-chat-v3.1", + "name": "DeepSeek: DeepSeek V3.1", + "contextLength": 163840, + "promptPerMTok": 0.21, + "completionPerMTok": 0.79, + "inputCacheReadPerMTok": 0.13, + "blendedOneInOneOutPerM": 1.0, + "isFree": false + }, + { + "id": "inception/mercury-2", + "canonicalSlug": "inception/mercury-2-20260304", + "name": "Inception: Mercury 2", + "contextLength": 128000, + "promptPerMTok": 0.25, + "completionPerMTok": 0.75, + "inputCacheReadPerMTok": 0.025, + "blendedOneInOneOutPerM": 1.0, + "isFree": false + }, + { + "id": "qwen/qwen2.5-vl-72b-instruct", + "canonicalSlug": "qwen/qwen2.5-vl-72b-instruct", + "name": "Qwen: Qwen2.5 VL 72B Instruct", + "contextLength": 131072, + "promptPerMTok": 0.25, + "completionPerMTok": 0.75, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 1.0, + "isFree": false + }, + { + "id": "deepseek/deepseek-chat", + "canonicalSlug": "deepseek/deepseek-chat-v3", + "name": "DeepSeek: DeepSeek V3", + "contextLength": 131072, + "promptPerMTok": 0.2002, + "completionPerMTok": 0.8001, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 1.0003, + "isFree": false + }, + { + "id": "qwen/qwen-plus", + "canonicalSlug": "qwen/qwen-plus-2025-01-25", + "name": "Qwen: Qwen-Plus", + "contextLength": 1000000, + "promptPerMTok": 0.26, + "completionPerMTok": 0.78, + "inputCacheReadPerMTok": 0.052, + "blendedOneInOneOutPerM": 1.04, + "isFree": false + }, + { + "id": "qwen/qwen-plus-2025-07-28", + "canonicalSlug": "qwen/qwen-plus-2025-07-28", + "name": "Qwen: Qwen Plus 0728", + "contextLength": 1000000, + "promptPerMTok": 0.26, + "completionPerMTok": 0.78, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 1.04, + "isFree": false + }, + { + "id": "qwen/qwen-plus-2025-07-28:thinking", + "canonicalSlug": "qwen/qwen-plus-2025-07-28", + "name": "Qwen: Qwen Plus 0728 (thinking)", + "contextLength": 1000000, + "promptPerMTok": 0.26, + "completionPerMTok": 0.78, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 1.04, + "isFree": false + }, + { + "id": "arcee-ai/trinity-large-thinking", + "canonicalSlug": "arcee-ai/trinity-large-thinking", + "name": "Arcee AI: Trinity Large Thinking", + "contextLength": 262144, + "promptPerMTok": 0.22, + "completionPerMTok": 0.85, + "inputCacheReadPerMTok": 0.06, + "blendedOneInOneOutPerM": 1.07, + "isFree": false + }, + { + "id": "qwen/qwen3-vl-235b-a22b-instruct", + "canonicalSlug": "qwen/qwen3-vl-235b-a22b-instruct", + "name": "Qwen: Qwen3 VL 235B A22B Instruct", + "contextLength": 262144, + "promptPerMTok": 0.2, + "completionPerMTok": 0.88, + "inputCacheReadPerMTok": 0.11, + "blendedOneInOneOutPerM": 1.08, + "isFree": false + }, + { + "id": "undi95/remm-slerp-l2-13b", + "canonicalSlug": "undi95/remm-slerp-l2-13b", + "name": "ReMM SLERP 13B", + "contextLength": 6144, + "promptPerMTok": 0.45, + "completionPerMTok": 0.65, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 1.1, + "isFree": false + }, + { + "id": "qwen/qwen3.5-35b-a3b", + "canonicalSlug": "qwen/qwen3.5-35b-a3b-20260224", + "name": "Qwen: Qwen3.5-35B-A3B", + "contextLength": 262144, + "promptPerMTok": 0.14, + "completionPerMTok": 1.0, + "inputCacheReadPerMTok": 0.05, + "blendedOneInOneOutPerM": 1.14, + "isFree": false + }, + { + "id": "qwen/qwen3.6-35b-a3b", + "canonicalSlug": "qwen/qwen3.6-35b-a3b-20260415", + "name": "Qwen: Qwen3.6 35B A3B", + "contextLength": 262144, + "promptPerMTok": 0.14, + "completionPerMTok": 1.0, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 1.14, + "isFree": false + }, + { + "id": "qwen/qwen3-coder-flash", + "canonicalSlug": "qwen/qwen3-coder-flash", + "name": "Qwen: Qwen3 Coder Flash", + "contextLength": 1000000, + "promptPerMTok": 0.195, + "completionPerMTok": 0.975, + "inputCacheReadPerMTok": 0.039, + "blendedOneInOneOutPerM": 1.17, + "isFree": false + }, + { + "id": "qwen/qwen3-next-80b-a3b-instruct", + "canonicalSlug": "qwen/qwen3-next-80b-a3b-instruct-2509", + "name": "Qwen: Qwen3 Next 80B A3B Instruct", + "contextLength": 262144, + "promptPerMTok": 0.09, + "completionPerMTok": 1.1, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 1.19, + "isFree": false + }, + { + "id": "mistralai/codestral-2508", + "canonicalSlug": "mistralai/codestral-2508", + "name": "Mistral: Codestral 2508", + "contextLength": 256000, + "promptPerMTok": 0.3, + "completionPerMTok": 0.9, + "inputCacheReadPerMTok": 0.03, + "blendedOneInOneOutPerM": 1.2, + "isFree": false + }, + { + "id": "z-ai/glm-4.6v", + "canonicalSlug": "z-ai/glm-4.6-20251208", + "name": "Z.ai: GLM 4.6V", + "contextLength": 131072, + "promptPerMTok": 0.3, + "completionPerMTok": 0.9, + "inputCacheReadPerMTok": 0.05, + "blendedOneInOneOutPerM": 1.2, + "isFree": false + }, + { + "id": "deepseek/deepseek-v3.1-terminus", + "canonicalSlug": "deepseek/deepseek-v3.1-terminus", + "name": "DeepSeek: DeepSeek V3.1 Terminus", + "contextLength": 163840, + "promptPerMTok": 0.27, + "completionPerMTok": 0.95, + "inputCacheReadPerMTok": 0.13, + "blendedOneInOneOutPerM": 1.22, + "isFree": false + }, + { + "id": "microsoft/wizardlm-2-8x22b", + "canonicalSlug": "microsoft/wizardlm-2-8x22b", + "name": "WizardLM-2 8x22B", + "contextLength": 65536, + "promptPerMTok": 0.62, + "completionPerMTok": 0.62, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 1.24, + "isFree": false + }, + { + "id": "minimax/minimax-m2.1", + "canonicalSlug": "minimax/minimax-m2.1", + "name": "MiniMax: MiniMax M2.1", + "contextLength": 204800, + "promptPerMTok": 0.29, + "completionPerMTok": 0.95, + "inputCacheReadPerMTok": 0.03, + "blendedOneInOneOutPerM": 1.24, + "isFree": false + }, + { + "id": "meta-llama/llama-3-70b-instruct", + "canonicalSlug": "meta-llama/llama-3-70b-instruct", + "name": "Meta: Llama 3 70B Instruct", + "contextLength": 8192, + "promptPerMTok": 0.51, + "completionPerMTok": 0.74, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 1.25, + "isFree": false + }, + { + "id": "minimax/minimax-m2", + "canonicalSlug": "minimax/minimax-m2", + "name": "MiniMax: MiniMax M2", + "contextLength": 204800, + "promptPerMTok": 0.255, + "completionPerMTok": 1.0, + "inputCacheReadPerMTok": 0.03, + "blendedOneInOneOutPerM": 1.255, + "isFree": false + }, + { + "id": "arcee-ai/coder-large", + "canonicalSlug": "arcee-ai/coder-large", + "name": "Arcee AI: Coder Large", + "contextLength": 32768, + "promptPerMTok": 0.5, + "completionPerMTok": 0.8, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 1.3, + "isFree": false + }, + { + "id": "google/gemma-2-27b-it", + "canonicalSlug": "google/gemma-2-27b-it", + "name": "Google: Gemma 2 27B", + "contextLength": 8192, + "promptPerMTok": 0.65, + "completionPerMTok": 0.65, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 1.3, + "isFree": false + }, + { + "id": "minimax/minimax-01", + "canonicalSlug": "minimax/minimax-01", + "name": "MiniMax: MiniMax-01", + "contextLength": 1000192, + "promptPerMTok": 0.2, + "completionPerMTok": 1.1, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 1.3, + "isFree": false + }, + { + "id": "minimax/minimax-m2.5", + "canonicalSlug": "minimax/minimax-m2.5-20260211", + "name": "MiniMax: MiniMax M2.5", + "contextLength": 204800, + "promptPerMTok": 0.15, + "completionPerMTok": 1.15, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 1.3, + "isFree": false + }, + { + "id": "prime-intellect/intellect-3", + "canonicalSlug": "prime-intellect/intellect-3-20251126", + "name": "Prime Intellect: INTELLECT-3", + "contextLength": 131072, + "promptPerMTok": 0.2, + "completionPerMTok": 1.1, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 1.3, + "isFree": false + }, + { + "id": "deepseek/deepseek-v4-pro", + "canonicalSlug": "deepseek/deepseek-v4-pro-20260423", + "name": "DeepSeek: DeepSeek V4 Pro", + "contextLength": 1048576, + "promptPerMTok": 0.435, + "completionPerMTok": 0.87, + "inputCacheReadPerMTok": 0.003625, + "blendedOneInOneOutPerM": 1.305, + "isFree": false + }, + { + "id": "xiaomi/mimo-v2.5-pro", + "canonicalSlug": "xiaomi/mimo-v2.5-pro-20260422", + "name": "Xiaomi: MiMo-V2.5-Pro", + "contextLength": 1048576, + "promptPerMTok": 0.435, + "completionPerMTok": 0.87, + "inputCacheReadPerMTok": 0.0036, + "blendedOneInOneOutPerM": 1.305, + "isFree": false + }, + { + "id": "qwen/qwen3.6-flash", + "canonicalSlug": "qwen/qwen3.6-flash", + "name": "Qwen: Qwen3.6 Flash", + "contextLength": 1000000, + "promptPerMTok": 0.1875, + "completionPerMTok": 1.125, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 1.3125, + "isFree": false + }, + { + "id": "stepfun/step-3.7-flash", + "canonicalSlug": "stepfun/step-3.7-flash-20260528", + "name": "StepFun: Step 3.7 Flash", + "contextLength": 256000, + "promptPerMTok": 0.2, + "completionPerMTok": 1.15, + "inputCacheReadPerMTok": 0.04, + "blendedOneInOneOutPerM": 1.35, + "isFree": false + }, + { + "id": "thedrummer/skyfall-36b-v2", + "canonicalSlug": "thedrummer/skyfall-36b-v2", + "name": "TheDrummer: Skyfall 36B V2", + "contextLength": 32768, + "promptPerMTok": 0.55, + "completionPerMTok": 0.8, + "inputCacheReadPerMTok": 0.25, + "blendedOneInOneOutPerM": 1.35, + "isFree": false + }, + { + "id": "sao10k/l3.3-euryale-70b", + "canonicalSlug": "sao10k/l3.3-euryale-70b-v2.3", + "name": "Sao10K: Llama 3.3 Euryale 70B", + "contextLength": 131072, + "promptPerMTok": 0.65, + "completionPerMTok": 0.75, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 1.4, + "isFree": false + }, + { + "id": "openai/gpt-5.4-nano", + "canonicalSlug": "openai/gpt-5.4-nano-20260317", + "name": "OpenAI: GPT-5.4 Nano", + "contextLength": 400000, + "promptPerMTok": 0.2, + "completionPerMTok": 1.25, + "inputCacheReadPerMTok": 0.02, + "blendedOneInOneOutPerM": 1.45, + "isFree": false + }, + { + "id": "minimax/minimax-m2.7", + "canonicalSlug": "minimax/minimax-m2.7-20260318", + "name": "MiniMax: MiniMax M2.7", + "contextLength": 204800, + "promptPerMTok": 0.279, + "completionPerMTok": 1.2, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 1.479, + "isFree": false + }, + { + "id": "qwen/qwen3-vl-8b-thinking", + "canonicalSlug": "qwen/qwen3-vl-8b-thinking", + "name": "Qwen: Qwen3 VL 8B Thinking", + "contextLength": 256000, + "promptPerMTok": 0.117, + "completionPerMTok": 1.365, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 1.482, + "isFree": false + }, + { + "id": "anthropic/claude-3-haiku", + "canonicalSlug": "anthropic/claude-3-haiku", + "name": "Anthropic: Claude 3 Haiku", + "contextLength": 200000, + "promptPerMTok": 0.25, + "completionPerMTok": 1.25, + "inputCacheReadPerMTok": 0.03, + "blendedOneInOneOutPerM": 1.5, + "isFree": false + }, + { + "id": "deepseek/deepseek-r1-distill-llama-70b", + "canonicalSlug": "deepseek/deepseek-r1-distill-llama-70b", + "name": "DeepSeek: R1 Distill Llama 70B", + "contextLength": 131072, + "promptPerMTok": 0.7, + "completionPerMTok": 0.8, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 1.5, + "isFree": false + }, + { + "id": "kwaipilot/kat-coder-pro-v2", + "canonicalSlug": "kwaipilot/kat-coder-pro-v2-20260327", + "name": "Kwaipilot: KAT-Coder-Pro V2", + "contextLength": 256000, + "promptPerMTok": 0.3, + "completionPerMTok": 1.2, + "inputCacheReadPerMTok": 0.06, + "blendedOneInOneOutPerM": 1.5, + "isFree": false + }, + { + "id": "minimax/minimax-m2-her", + "canonicalSlug": "minimax/minimax-m2-her-20260123", + "name": "MiniMax: MiniMax M2-her", + "contextLength": 65536, + "promptPerMTok": 0.3, + "completionPerMTok": 1.2, + "inputCacheReadPerMTok": 0.03, + "blendedOneInOneOutPerM": 1.5, + "isFree": false + }, + { + "id": "minimax/minimax-m3", + "canonicalSlug": "minimax/minimax-m3-20260531", + "name": "MiniMax: MiniMax M3", + "contextLength": 1048576, + "promptPerMTok": 0.3, + "completionPerMTok": 1.2, + "inputCacheReadPerMTok": 0.06, + "blendedOneInOneOutPerM": 1.5, + "isFree": false + }, + { + "id": "perceptron/perceptron-mk1", + "canonicalSlug": "perceptron/perceptron-mk1-20260512", + "name": "Perceptron: Perceptron Mk1", + "contextLength": 32768, + "promptPerMTok": 0.15, + "completionPerMTok": 1.5, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 1.65, + "isFree": false + }, + { + "id": "qwen/qwen-2.5-coder-32b-instruct", + "canonicalSlug": "qwen/qwen-2.5-coder-32b-instruct", + "name": "Qwen2.5 Coder 32B Instruct", + "contextLength": 128000, + "promptPerMTok": 0.66, + "completionPerMTok": 1.0, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 1.66, + "isFree": false + }, + { + "id": "baidu/ernie-4.5-vl-424b-a47b", + "canonicalSlug": "baidu/ernie-4.5-vl-424b-a47b", + "name": "Baidu: ERNIE 4.5 VL 424B A47B ", + "contextLength": 131072, + "promptPerMTok": 0.42, + "completionPerMTok": 1.25, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 1.67, + "isFree": false + }, + { + "id": "qwen/qwen3-vl-30b-a3b-thinking", + "canonicalSlug": "qwen/qwen3-vl-30b-a3b-thinking", + "name": "Qwen: Qwen3 VL 30B A3B Thinking", + "contextLength": 131072, + "promptPerMTok": 0.13, + "completionPerMTok": 1.56, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 1.69, + "isFree": false + }, + { + "id": "sao10k/l3.1-euryale-70b", + "canonicalSlug": "sao10k/l3.1-euryale-70b", + "name": "Sao10K: Llama 3.1 Euryale 70B v2.2", + "contextLength": 131072, + "promptPerMTok": 0.85, + "completionPerMTok": 0.85, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 1.7, + "isFree": false + }, + { + "id": "google/gemini-3.1-flash-lite", + "canonicalSlug": "google/gemini-3.1-flash-lite-20260507", + "name": "Google: Gemini 3.1 Flash Lite", + "contextLength": 1048576, + "promptPerMTok": 0.25, + "completionPerMTok": 1.5, + "inputCacheReadPerMTok": 0.025, + "blendedOneInOneOutPerM": 1.75, + "isFree": false + }, + { + "id": "google/gemini-3.1-flash-lite-preview", + "canonicalSlug": "google/gemini-3.1-flash-lite-preview-20260303", + "name": "Google: Gemini 3.1 Flash Lite Preview", + "contextLength": 1048576, + "promptPerMTok": 0.25, + "completionPerMTok": 1.5, + "inputCacheReadPerMTok": 0.025, + "blendedOneInOneOutPerM": 1.75, + "isFree": false + }, + { + "id": "mancer/weaver", + "canonicalSlug": "mancer/weaver", + "name": "Mancer: Weaver (alpha)", + "contextLength": 8000, + "promptPerMTok": 0.75, + "completionPerMTok": 1.0, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 1.75, + "isFree": false + }, + { + "id": "qwen/qwen3.5-27b", + "canonicalSlug": "qwen/qwen3.5-27b-20260224", + "name": "Qwen: Qwen3.5-27B", + "contextLength": 262144, + "promptPerMTok": 0.195, + "completionPerMTok": 1.56, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 1.755, + "isFree": false + }, + { + "id": "qwen/qwen3.5-plus-02-15", + "canonicalSlug": "qwen/qwen3.5-plus-20260216", + "name": "Qwen: Qwen3.5 Plus 2026-02-15", + "contextLength": 1000000, + "promptPerMTok": 0.26, + "completionPerMTok": 1.56, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 1.82, + "isFree": false + }, + { + "id": "arcee-ai/virtuoso-large", + "canonicalSlug": "arcee-ai/virtuoso-large", + "name": "Arcee AI: Virtuoso Large", + "contextLength": 131072, + "promptPerMTok": 0.75, + "completionPerMTok": 1.2, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 1.95, + "isFree": false + }, + { + "id": "mistralai/mistral-large-2512", + "canonicalSlug": "mistralai/mistral-large-2512", + "name": "Mistral: Mistral Large 3 2512", + "contextLength": 262144, + "promptPerMTok": 0.5, + "completionPerMTok": 1.5, + "inputCacheReadPerMTok": 0.05, + "blendedOneInOneOutPerM": 2.0, + "isFree": false + }, + { + "id": "morph/morph-v3-fast", + "canonicalSlug": "morph/morph-v3-fast", + "name": "Morph: Morph V3 Fast", + "contextLength": 81920, + "promptPerMTok": 0.8, + "completionPerMTok": 1.2, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 2.0, + "isFree": false + }, + { + "id": "nousresearch/hermes-3-llama-3.1-405b", + "canonicalSlug": "nousresearch/hermes-3-llama-3.1-405b", + "name": "Nous: Hermes 3 405B Instruct", + "contextLength": 131072, + "promptPerMTok": 1.0, + "completionPerMTok": 1.0, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 2.0, + "isFree": false + }, + { + "id": "openai/gpt-3.5-turbo", + "canonicalSlug": "openai/gpt-3.5-turbo", + "name": "OpenAI: GPT-3.5 Turbo", + "contextLength": 16385, + "promptPerMTok": 0.5, + "completionPerMTok": 1.5, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 2.0, + "isFree": false + }, + { + "id": "openai/gpt-4.1-mini", + "canonicalSlug": "openai/gpt-4.1-mini-2025-04-14", + "name": "OpenAI: GPT-4.1 Mini", + "contextLength": 1047576, + "promptPerMTok": 0.4, + "completionPerMTok": 1.6, + "inputCacheReadPerMTok": 0.1, + "blendedOneInOneOutPerM": 2.0, + "isFree": false + }, + { + "id": "perplexity/sonar", + "canonicalSlug": "perplexity/sonar", + "name": "Perplexity: Sonar", + "contextLength": 127072, + "promptPerMTok": 1.0, + "completionPerMTok": 1.0, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 2.0, + "isFree": false + }, + { + "id": "qwen/qwen3.7-plus", + "canonicalSlug": "qwen/qwen3.7-plus-20260602", + "name": "Qwen: Qwen3.7 Plus", + "contextLength": 1000000, + "promptPerMTok": 0.4, + "completionPerMTok": 1.6, + "inputCacheReadPerMTok": 0.08, + "blendedOneInOneOutPerM": 2.0, + "isFree": false + }, + { + "id": "qwen/qwen3-coder", + "canonicalSlug": "qwen/qwen3-coder-480b-a35b-07-25", + "name": "Qwen: Qwen3 Coder 480B A35B", + "contextLength": 1048576, + "promptPerMTok": 0.22, + "completionPerMTok": 1.8, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 2.02, + "isFree": false + }, + { + "id": "aion-labs/aion-1.0-mini", + "canonicalSlug": "aion-labs/aion-1.0-mini", + "name": "AionLabs: Aion-1.0-Mini", + "contextLength": 131072, + "promptPerMTok": 0.7, + "completionPerMTok": 1.4, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 2.1, + "isFree": false + }, + { + "id": "qwen/qwen3.5-plus-20260420", + "canonicalSlug": "qwen/qwen3.5-plus-20260420", + "name": "Qwen: Qwen3.5 Plus 2026-04-20", + "contextLength": 1000000, + "promptPerMTok": 0.3, + "completionPerMTok": 1.8, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 2.1, + "isFree": false + }, + { + "id": "relace/relace-apply-3", + "canonicalSlug": "relace/relace-apply-3", + "name": "Relace: Relace Apply 3", + "contextLength": 256000, + "promptPerMTok": 0.85, + "completionPerMTok": 1.25, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 2.1, + "isFree": false + }, + { + "id": "z-ai/glm-4.7", + "canonicalSlug": "z-ai/glm-4.7-20251222", + "name": "Z.ai: GLM 4.7", + "contextLength": 202752, + "promptPerMTok": 0.4, + "completionPerMTok": 1.75, + "inputCacheReadPerMTok": 0.08, + "blendedOneInOneOutPerM": 2.15, + "isFree": false + }, + { + "id": "z-ai/glm-4.6", + "canonicalSlug": "z-ai/glm-4.6", + "name": "Z.ai: GLM 4.6", + "contextLength": 202752, + "promptPerMTok": 0.43, + "completionPerMTok": 1.74, + "inputCacheReadPerMTok": 0.08, + "blendedOneInOneOutPerM": 2.17, + "isFree": false + }, + { + "id": "bytedance-seed/seed-1.6", + "canonicalSlug": "bytedance-seed/seed-1.6-20250625", + "name": "ByteDance Seed: Seed 1.6", + "contextLength": 262144, + "promptPerMTok": 0.25, + "completionPerMTok": 2.0, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 2.25, + "isFree": false + }, + { + "id": "bytedance-seed/seed-2.0-lite", + "canonicalSlug": "bytedance-seed/seed-2.0-lite-20260309", + "name": "ByteDance Seed: Seed-2.0-Lite", + "contextLength": 262144, + "promptPerMTok": 0.25, + "completionPerMTok": 2.0, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 2.25, + "isFree": false + }, + { + "id": "openai/gpt-5-mini", + "canonicalSlug": "openai/gpt-5-mini-2025-08-07", + "name": "OpenAI: GPT-5 Mini", + "contextLength": 400000, + "promptPerMTok": 0.25, + "completionPerMTok": 2.0, + "inputCacheReadPerMTok": 0.025, + "blendedOneInOneOutPerM": 2.25, + "isFree": false + }, + { + "id": "openai/gpt-5.1-codex-mini", + "canonicalSlug": "openai/gpt-5.1-codex-mini-20251113", + "name": "OpenAI: GPT-5.1-Codex-Mini", + "contextLength": 400000, + "promptPerMTok": 0.25, + "completionPerMTok": 2.0, + "inputCacheReadPerMTok": 0.025, + "blendedOneInOneOutPerM": 2.25, + "isFree": false + }, + { + "id": "qwen/qwen3-235b-a22b", + "canonicalSlug": "qwen/qwen3-235b-a22b-04-28", + "name": "Qwen: Qwen3 235B A22B", + "contextLength": 131072, + "promptPerMTok": 0.455, + "completionPerMTok": 1.82, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 2.275, + "isFree": false + }, + { + "id": "qwen/qwen3.6-plus", + "canonicalSlug": "qwen/qwen3.6-plus-04-02", + "name": "Qwen: Qwen3.6 Plus", + "contextLength": 1000000, + "promptPerMTok": 0.325, + "completionPerMTok": 1.95, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 2.275, + "isFree": false + }, + { + "id": "moonshotai/kimi-k2.5", + "canonicalSlug": "moonshotai/kimi-k2.5-0127", + "name": "MoonshotAI: Kimi K2.5", + "contextLength": 262144, + "promptPerMTok": 0.4, + "completionPerMTok": 1.9, + "inputCacheReadPerMTok": 0.09, + "blendedOneInOneOutPerM": 2.3, + "isFree": false + }, + { + "id": "qwen/qwen3.5-122b-a10b", + "canonicalSlug": "qwen/qwen3.5-122b-a10b-20260224", + "name": "Qwen: Qwen3.5-122B-A10B", + "contextLength": 262144, + "promptPerMTok": 0.26, + "completionPerMTok": 2.08, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 2.34, + "isFree": false + }, + { + "id": "aion-labs/aion-2.0", + "canonicalSlug": "aion-labs/aion-2.0-20260223", + "name": "AionLabs: Aion-2.0", + "contextLength": 131072, + "promptPerMTok": 0.8, + "completionPerMTok": 1.6, + "inputCacheReadPerMTok": 0.2, + "blendedOneInOneOutPerM": 2.4, + "isFree": false + }, + { + "id": "aion-labs/aion-rp-llama-3.1-8b", + "canonicalSlug": "aion-labs/aion-rp-llama-3.1-8b", + "name": "AionLabs: Aion-RP 1.0 (8B)", + "contextLength": 32768, + "promptPerMTok": 0.8, + "completionPerMTok": 1.6, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 2.4, + "isFree": false + }, + { + "id": "mistralai/devstral-2512", + "canonicalSlug": "mistralai/devstral-2512", + "name": "Mistral: Devstral 2 2512", + "contextLength": 262144, + "promptPerMTok": 0.4, + "completionPerMTok": 2.0, + "inputCacheReadPerMTok": 0.04, + "blendedOneInOneOutPerM": 2.4, + "isFree": false + }, + { + "id": "mistralai/mistral-medium-3", + "canonicalSlug": "mistralai/mistral-medium-3", + "name": "Mistral: Mistral Medium 3", + "contextLength": 131072, + "promptPerMTok": 0.4, + "completionPerMTok": 2.0, + "inputCacheReadPerMTok": 0.04, + "blendedOneInOneOutPerM": 2.4, + "isFree": false + }, + { + "id": "mistralai/mistral-medium-3.1", + "canonicalSlug": "mistralai/mistral-medium-3.1", + "name": "Mistral: Mistral Medium 3.1", + "contextLength": 131072, + "promptPerMTok": 0.4, + "completionPerMTok": 2.0, + "inputCacheReadPerMTok": 0.04, + "blendedOneInOneOutPerM": 2.4, + "isFree": false + }, + { + "id": "z-ai/glm-4.5v", + "canonicalSlug": "z-ai/glm-4.5v", + "name": "Z.ai: GLM 4.5V", + "contextLength": 65536, + "promptPerMTok": 0.6, + "completionPerMTok": 1.8, + "inputCacheReadPerMTok": 0.11, + "blendedOneInOneOutPerM": 2.4, + "isFree": false + }, + { + "id": "deepcogito/cogito-v2.1-671b", + "canonicalSlug": "deepcogito/cogito-v2.1-671b-20251118", + "name": "Deep Cogito: Cogito v2.1 671B", + "contextLength": 128000, + "promptPerMTok": 1.25, + "completionPerMTok": 1.25, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 2.5, + "isFree": false + }, + { + "id": "z-ai/glm-5", + "canonicalSlug": "z-ai/glm-5-20260211", + "name": "Z.ai: GLM 5", + "contextLength": 202752, + "promptPerMTok": 0.6, + "completionPerMTok": 1.92, + "inputCacheReadPerMTok": 0.12, + "blendedOneInOneOutPerM": 2.52, + "isFree": false + }, + { + "id": "minimax/minimax-m1", + "canonicalSlug": "minimax/minimax-m1", + "name": "MiniMax: MiniMax M1", + "contextLength": 1000000, + "promptPerMTok": 0.4, + "completionPerMTok": 2.2, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 2.6, + "isFree": false + }, + { + "id": "deepseek/deepseek-r1-0528", + "canonicalSlug": "deepseek/deepseek-r1-0528", + "name": "DeepSeek: R1 0528", + "contextLength": 163840, + "promptPerMTok": 0.5, + "completionPerMTok": 2.15, + "inputCacheReadPerMTok": 0.35, + "blendedOneInOneOutPerM": 2.65, + "isFree": false + }, + { + "id": "qwen/qwen3.5-397b-a17b", + "canonicalSlug": "qwen/qwen3.5-397b-a17b-20260216", + "name": "Qwen: Qwen3.5 397B A17B", + "contextLength": 262144, + "promptPerMTok": 0.39, + "completionPerMTok": 2.34, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 2.73, + "isFree": false + }, + { + "id": "amazon/nova-2-lite-v1", + "canonicalSlug": "amazon/nova-2-lite-v1", + "name": "Amazon: Nova 2 Lite", + "contextLength": 1000000, + "promptPerMTok": 0.3, + "completionPerMTok": 2.5, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 2.8, + "isFree": false + }, + { + "id": "google/gemini-2.5-flash", + "canonicalSlug": "google/gemini-2.5-flash", + "name": "Google: Gemini 2.5 Flash", + "contextLength": 1048576, + "promptPerMTok": 0.3, + "completionPerMTok": 2.5, + "inputCacheReadPerMTok": 0.03, + "blendedOneInOneOutPerM": 2.8, + "isFree": false + }, + { + "id": "google/gemini-2.5-flash-image", + "canonicalSlug": "google/gemini-2.5-flash-image", + "name": "Google: Nano Banana (Gemini 2.5 Flash Image)", + "contextLength": 32768, + "promptPerMTok": 0.3, + "completionPerMTok": 2.5, + "inputCacheReadPerMTok": 0.03, + "blendedOneInOneOutPerM": 2.8, + "isFree": false + }, + { + "id": "morph/morph-v3-large", + "canonicalSlug": "morph/morph-v3-large", + "name": "Morph: Morph V3 Large", + "contextLength": 262144, + "promptPerMTok": 0.9, + "completionPerMTok": 1.9, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 2.8, + "isFree": false + }, + { + "id": "z-ai/glm-4.5", + "canonicalSlug": "z-ai/glm-4.5", + "name": "Z.ai: GLM 4.5", + "contextLength": 131072, + "promptPerMTok": 0.6, + "completionPerMTok": 2.2, + "inputCacheReadPerMTok": 0.11, + "blendedOneInOneOutPerM": 2.8, + "isFree": false + }, + { + "id": "qwen/qwen3-vl-235b-a22b-thinking", + "canonicalSlug": "qwen/qwen3-vl-235b-a22b-thinking", + "name": "Qwen: Qwen3 VL 235B A22B Thinking", + "contextLength": 131072, + "promptPerMTok": 0.26, + "completionPerMTok": 2.6, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 2.86, + "isFree": false + }, + { + "id": "moonshotai/kimi-k2", + "canonicalSlug": "moonshotai/kimi-k2", + "name": "MoonshotAI: Kimi K2 0711", + "contextLength": 131072, + "promptPerMTok": 0.57, + "completionPerMTok": 2.3, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 2.87, + "isFree": false + }, + { + "id": "nvidia/nemotron-3-ultra-550b-a55b", + "canonicalSlug": "nvidia/nemotron-3-ultra-550b-a55b-20260604", + "name": "NVIDIA: Nemotron 3 Ultra", + "contextLength": 1000000, + "promptPerMTok": 0.5, + "completionPerMTok": 2.5, + "inputCacheReadPerMTok": 0.15, + "blendedOneInOneOutPerM": 3.0, + "isFree": false + }, + { + "id": "openai/gpt-3.5-turbo-0613", + "canonicalSlug": "openai/gpt-3.5-turbo-0613", + "name": "OpenAI: GPT-3.5 Turbo (older v0613)", + "contextLength": 4095, + "promptPerMTok": 1.0, + "completionPerMTok": 2.0, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 3.0, + "isFree": false + }, + { + "id": "openai/gpt-audio-mini", + "canonicalSlug": "openai/gpt-audio-mini", + "name": "OpenAI: GPT Audio Mini", + "contextLength": 128000, + "promptPerMTok": 0.6, + "completionPerMTok": 2.4, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 3.0, + "isFree": false + }, + { + "id": "x-ai/grok-build-0.1", + "canonicalSlug": "x-ai/grok-build-0.1-20260520", + "name": "xAI: Grok Build 0.1", + "contextLength": 256000, + "promptPerMTok": 1.0, + "completionPerMTok": 2.0, + "inputCacheReadPerMTok": 0.2, + "blendedOneInOneOutPerM": 3.0, + "isFree": false + }, + { + "id": "moonshotai/kimi-k2-0905", + "canonicalSlug": "moonshotai/kimi-k2-0905", + "name": "MoonshotAI: Kimi K2 0905", + "contextLength": 262144, + "promptPerMTok": 0.6, + "completionPerMTok": 2.5, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 3.1, + "isFree": false + }, + { + "id": "moonshotai/kimi-k2-thinking", + "canonicalSlug": "moonshotai/kimi-k2-thinking-20251106", + "name": "MoonshotAI: Kimi K2 Thinking", + "contextLength": 262144, + "promptPerMTok": 0.6, + "completionPerMTok": 2.5, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 3.1, + "isFree": false + }, + { + "id": "deepseek/deepseek-r1", + "canonicalSlug": "deepseek/deepseek-r1", + "name": "DeepSeek: R1", + "contextLength": 163840, + "promptPerMTok": 0.7, + "completionPerMTok": 2.5, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 3.2, + "isFree": false + }, + { + "id": "qwen/qwen3.6-27b", + "canonicalSlug": "qwen/qwen3.6-27b-20260422", + "name": "Qwen: Qwen3.6 27B", + "contextLength": 262144, + "promptPerMTok": 0.29, + "completionPerMTok": 3.2, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 3.49, + "isFree": false + }, + { + "id": "google/gemini-3-flash-preview", + "canonicalSlug": "google/gemini-3-flash-preview-20251217", + "name": "Google: Gemini 3 Flash Preview", + "contextLength": 1048576, + "promptPerMTok": 0.5, + "completionPerMTok": 3.0, + "inputCacheReadPerMTok": 0.05, + "blendedOneInOneOutPerM": 3.5, + "isFree": false + }, + { + "id": "google/gemini-3.1-flash-image-preview", + "canonicalSlug": "google/gemini-3.1-flash-image-preview-20260226", + "name": "Google: Nano Banana 2 (Gemini 3.1 Flash Image Preview)", + "contextLength": 131072, + "promptPerMTok": 0.5, + "completionPerMTok": 3.0, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 3.5, + "isFree": false + }, + { + "id": "openai/gpt-3.5-turbo-instruct", + "canonicalSlug": "openai/gpt-3.5-turbo-instruct", + "name": "OpenAI: GPT-3.5 Turbo Instruct", + "contextLength": 4095, + "promptPerMTok": 1.5, + "completionPerMTok": 2.0, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 3.5, + "isFree": false + }, + { + "id": "x-ai/grok-4.20", + "canonicalSlug": "x-ai/grok-4.20-20260309", + "name": "xAI: Grok 4.20", + "contextLength": 2000000, + "promptPerMTok": 1.25, + "completionPerMTok": 2.5, + "inputCacheReadPerMTok": 0.2, + "blendedOneInOneOutPerM": 3.75, + "isFree": false + }, + { + "id": "x-ai/grok-4.3", + "canonicalSlug": "x-ai/grok-4.3-20260430", + "name": "xAI: Grok 4.3", + "contextLength": 1000000, + "promptPerMTok": 1.25, + "completionPerMTok": 2.5, + "inputCacheReadPerMTok": 0.2, + "blendedOneInOneOutPerM": 3.75, + "isFree": false + }, + { + "id": "qwen/qwen3-coder-plus", + "canonicalSlug": "qwen/qwen3-coder-plus", + "name": "Qwen: Qwen3 Coder Plus", + "contextLength": 1000000, + "promptPerMTok": 0.65, + "completionPerMTok": 3.25, + "inputCacheReadPerMTok": 0.13, + "blendedOneInOneOutPerM": 3.9, + "isFree": false + }, + { + "id": "amazon/nova-pro-v1", + "canonicalSlug": "amazon/nova-pro-v1", + "name": "Amazon: Nova Pro 1.0", + "contextLength": 300000, + "promptPerMTok": 0.8, + "completionPerMTok": 3.2, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 4.0, + "isFree": false + }, + { + "id": "nousresearch/hermes-4-405b", + "canonicalSlug": "nousresearch/hermes-4-405b", + "name": "Nous: Hermes 4 405B", + "contextLength": 131072, + "promptPerMTok": 1.0, + "completionPerMTok": 3.0, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 4.0, + "isFree": false + }, + { + "id": "relace/relace-search", + "canonicalSlug": "relace/relace-search-20251208", + "name": "Relace: Relace Search", + "contextLength": 256000, + "promptPerMTok": 1.0, + "completionPerMTok": 3.0, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 4.0, + "isFree": false + }, + { + "id": "z-ai/glm-5.1", + "canonicalSlug": "z-ai/glm-5.1-20260406", + "name": "Z.ai: GLM 5.1", + "contextLength": 202752, + "promptPerMTok": 0.98, + "completionPerMTok": 3.08, + "inputCacheReadPerMTok": 0.182, + "blendedOneInOneOutPerM": 4.06, + "isFree": false + }, + { + "id": "moonshotai/kimi-k2.6", + "canonicalSlug": "moonshotai/kimi-k2.6-20260420", + "name": "MoonshotAI: Kimi K2.6", + "contextLength": 262144, + "promptPerMTok": 0.684, + "completionPerMTok": 3.42, + "inputCacheReadPerMTok": 0.144, + "blendedOneInOneOutPerM": 4.104, + "isFree": false + }, + { + "id": "~moonshotai/kimi-latest", + "canonicalSlug": "~moonshotai/kimi-latest", + "name": "MoonshotAI Kimi Latest", + "contextLength": 262144, + "promptPerMTok": 0.684, + "completionPerMTok": 3.42, + "inputCacheReadPerMTok": 0.144, + "blendedOneInOneOutPerM": 4.104, + "isFree": false + }, + { + "id": "arcee-ai/maestro-reasoning", + "canonicalSlug": "arcee-ai/maestro-reasoning", + "name": "Arcee AI: Maestro Reasoning", + "contextLength": 131072, + "promptPerMTok": 0.9, + "completionPerMTok": 3.3, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 4.2, + "isFree": false + }, + { + "id": "switchpoint/router", + "canonicalSlug": "switchpoint/router", + "name": "Switchpoint Router", + "contextLength": 131072, + "promptPerMTok": 0.85, + "completionPerMTok": 3.4, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 4.25, + "isFree": false + }, + { + "id": "openai/gpt-5-image-mini", + "canonicalSlug": "openai/gpt-5-image-mini", + "name": "OpenAI: GPT-5 Image Mini", + "contextLength": 400000, + "promptPerMTok": 2.5, + "completionPerMTok": 2.0, + "inputCacheReadPerMTok": 0.25, + "blendedOneInOneOutPerM": 4.5, + "isFree": false + }, + { + "id": "qwen/qwen3-max", + "canonicalSlug": "qwen/qwen3-max", + "name": "Qwen: Qwen3 Max", + "contextLength": 262144, + "promptPerMTok": 0.78, + "completionPerMTok": 3.9, + "inputCacheReadPerMTok": 0.156, + "blendedOneInOneOutPerM": 4.68, + "isFree": false + }, + { + "id": "qwen/qwen3-max-thinking", + "canonicalSlug": "qwen/qwen3-max-thinking-20260123", + "name": "Qwen: Qwen3 Max Thinking", + "contextLength": 262144, + "promptPerMTok": 0.78, + "completionPerMTok": 3.9, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 4.68, + "isFree": false + }, + { + "id": "anthropic/claude-3.5-haiku", + "canonicalSlug": "anthropic/claude-3-5-haiku", + "name": "Anthropic: Claude 3.5 Haiku", + "contextLength": 200000, + "promptPerMTok": 0.8, + "completionPerMTok": 4.0, + "inputCacheReadPerMTok": 0.08, + "blendedOneInOneOutPerM": 4.8, + "isFree": false + }, + { + "id": "qwen/qwen3.7-max", + "canonicalSlug": "qwen/qwen3.7-max-20260520", + "name": "Qwen: Qwen3.7 Max", + "contextLength": 1000000, + "promptPerMTok": 1.25, + "completionPerMTok": 3.75, + "inputCacheReadPerMTok": 0.25, + "blendedOneInOneOutPerM": 5.0, + "isFree": false + }, + { + "id": "z-ai/glm-5-turbo", + "canonicalSlug": "z-ai/glm-5-turbo-20260315", + "name": "Z.ai: GLM 5 Turbo", + "contextLength": 202752, + "promptPerMTok": 1.2, + "completionPerMTok": 4.0, + "inputCacheReadPerMTok": 0.24, + "blendedOneInOneOutPerM": 5.2, + "isFree": false + }, + { + "id": "z-ai/glm-5v-turbo", + "canonicalSlug": "z-ai/glm-5v-turbo-20260401", + "name": "Z.ai: GLM 5V Turbo", + "contextLength": 202752, + "promptPerMTok": 1.2, + "completionPerMTok": 4.0, + "inputCacheReadPerMTok": 0.24, + "blendedOneInOneOutPerM": 5.2, + "isFree": false + }, + { + "id": "openai/gpt-5.4-mini", + "canonicalSlug": "openai/gpt-5.4-mini-20260317", + "name": "OpenAI: GPT-5.4 Mini", + "contextLength": 400000, + "promptPerMTok": 0.75, + "completionPerMTok": 4.5, + "inputCacheReadPerMTok": 0.075, + "blendedOneInOneOutPerM": 5.25, + "isFree": false + }, + { + "id": "~openai/gpt-mini-latest", + "canonicalSlug": "~openai/gpt-mini-latest", + "name": "OpenAI GPT Mini Latest", + "contextLength": 400000, + "promptPerMTok": 0.75, + "completionPerMTok": 4.5, + "inputCacheReadPerMTok": 0.075, + "blendedOneInOneOutPerM": 5.25, + "isFree": false + }, + { + "id": "openai/o3-mini", + "canonicalSlug": "openai/o3-mini-2025-01-31", + "name": "OpenAI: o3 Mini", + "contextLength": 200000, + "promptPerMTok": 1.1, + "completionPerMTok": 4.4, + "inputCacheReadPerMTok": 0.55, + "blendedOneInOneOutPerM": 5.5, + "isFree": false + }, + { + "id": "openai/o3-mini-high", + "canonicalSlug": "openai/o3-mini-high-2025-01-31", + "name": "OpenAI: o3 Mini High", + "contextLength": 200000, + "promptPerMTok": 1.1, + "completionPerMTok": 4.4, + "inputCacheReadPerMTok": 0.55, + "blendedOneInOneOutPerM": 5.5, + "isFree": false + }, + { + "id": "openai/o4-mini", + "canonicalSlug": "openai/o4-mini-2025-04-16", + "name": "OpenAI: o4 Mini", + "contextLength": 200000, + "promptPerMTok": 1.1, + "completionPerMTok": 4.4, + "inputCacheReadPerMTok": 0.275, + "blendedOneInOneOutPerM": 5.5, + "isFree": false + }, + { + "id": "openai/o4-mini-high", + "canonicalSlug": "openai/o4-mini-high-2025-04-16", + "name": "OpenAI: o4 Mini High", + "contextLength": 200000, + "promptPerMTok": 1.1, + "completionPerMTok": 4.4, + "inputCacheReadPerMTok": 0.275, + "blendedOneInOneOutPerM": 5.5, + "isFree": false + }, + { + "id": "anthropic/claude-haiku-4.5", + "canonicalSlug": "anthropic/claude-4.5-haiku-20251001", + "name": "Anthropic: Claude Haiku 4.5", + "contextLength": 200000, + "promptPerMTok": 1.0, + "completionPerMTok": 5.0, + "inputCacheReadPerMTok": 0.1, + "blendedOneInOneOutPerM": 6.0, + "isFree": false + }, + { + "id": "sao10k/l3.1-70b-hanami-x1", + "canonicalSlug": "sao10k/l3.1-70b-hanami-x1", + "name": "Sao10K: Llama 3.1 70B Hanami x1", + "contextLength": 16000, + "promptPerMTok": 3.0, + "completionPerMTok": 3.0, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 6.0, + "isFree": false + }, + { + "id": "~anthropic/claude-haiku-latest", + "canonicalSlug": "~anthropic/claude-haiku-latest", + "name": "Anthropic Claude Haiku Latest", + "contextLength": 200000, + "promptPerMTok": 1.0, + "completionPerMTok": 5.0, + "inputCacheReadPerMTok": 0.1, + "blendedOneInOneOutPerM": 6.0, + "isFree": false + }, + { + "id": "writer/palmyra-x5", + "canonicalSlug": "writer/palmyra-x5-20250428", + "name": "Writer: Palmyra X5", + "contextLength": 1040000, + "promptPerMTok": 0.6, + "completionPerMTok": 6.0, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 6.6, + "isFree": false + }, + { + "id": "openai/gpt-3.5-turbo-16k", + "canonicalSlug": "openai/gpt-3.5-turbo-16k", + "name": "OpenAI: GPT-3.5 Turbo 16k", + "contextLength": 16385, + "promptPerMTok": 3.0, + "completionPerMTok": 4.0, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 7.0, + "isFree": false + }, + { + "id": "qwen/qwen3.6-max-preview", + "canonicalSlug": "qwen/qwen3.6-max-preview-20260420", + "name": "Qwen: Qwen3.6 Max Preview", + "contextLength": 262144, + "promptPerMTok": 1.04, + "completionPerMTok": 6.24, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 7.28, + "isFree": false + }, + { + "id": "anthracite-org/magnum-v4-72b", + "canonicalSlug": "anthracite-org/magnum-v4-72b", + "name": "Magnum v4 72B", + "contextLength": 32768, + "promptPerMTok": 3.0, + "completionPerMTok": 5.0, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 8.0, + "isFree": false + }, + { + "id": "mistralai/mistral-large", + "canonicalSlug": "mistralai/mistral-large", + "name": "Mistral Large", + "contextLength": 128000, + "promptPerMTok": 2.0, + "completionPerMTok": 6.0, + "inputCacheReadPerMTok": 0.2, + "blendedOneInOneOutPerM": 8.0, + "isFree": false + }, + { + "id": "mistralai/mistral-large-2407", + "canonicalSlug": "mistralai/mistral-large-2407", + "name": "Mistral Large 2407", + "contextLength": 131072, + "promptPerMTok": 2.0, + "completionPerMTok": 6.0, + "inputCacheReadPerMTok": 0.2, + "blendedOneInOneOutPerM": 8.0, + "isFree": false + }, + { + "id": "mistralai/mixtral-8x22b-instruct", + "canonicalSlug": "mistralai/mixtral-8x22b-instruct", + "name": "Mistral: Mixtral 8x22B Instruct", + "contextLength": 65536, + "promptPerMTok": 2.0, + "completionPerMTok": 6.0, + "inputCacheReadPerMTok": 0.2, + "blendedOneInOneOutPerM": 8.0, + "isFree": false + }, + { + "id": "x-ai/grok-4.20-multi-agent", + "canonicalSlug": "x-ai/grok-4.20-multi-agent-20260309", + "name": "xAI: Grok 4.20 Multi-Agent", + "contextLength": 2000000, + "promptPerMTok": 2.0, + "completionPerMTok": 6.0, + "inputCacheReadPerMTok": 0.2, + "blendedOneInOneOutPerM": 8.0, + "isFree": false + }, + { + "id": "mistralai/mistral-medium-3-5", + "canonicalSlug": "mistralai/mistral-medium-3.5-20260430", + "name": "Mistral: Mistral Medium 3.5", + "contextLength": 262144, + "promptPerMTok": 1.5, + "completionPerMTok": 7.5, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 9.0, + "isFree": false + }, + { + "id": "ai21/jamba-large-1.7", + "canonicalSlug": "ai21/jamba-large-1.7", + "name": "AI21: Jamba Large 1.7", + "contextLength": 256000, + "promptPerMTok": 2.0, + "completionPerMTok": 8.0, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 10.0, + "isFree": false + }, + { + "id": "openai/gpt-4.1", + "canonicalSlug": "openai/gpt-4.1-2025-04-14", + "name": "OpenAI: GPT-4.1", + "contextLength": 1047576, + "promptPerMTok": 2.0, + "completionPerMTok": 8.0, + "inputCacheReadPerMTok": 0.5, + "blendedOneInOneOutPerM": 10.0, + "isFree": false + }, + { + "id": "openai/o3", + "canonicalSlug": "openai/o3-2025-04-16", + "name": "OpenAI: o3", + "contextLength": 200000, + "promptPerMTok": 2.0, + "completionPerMTok": 8.0, + "inputCacheReadPerMTok": 0.5, + "blendedOneInOneOutPerM": 10.0, + "isFree": false + }, + { + "id": "openai/o4-mini-deep-research", + "canonicalSlug": "openai/o4-mini-deep-research-2025-06-26", + "name": "OpenAI: o4 Mini Deep Research", + "contextLength": 200000, + "promptPerMTok": 2.0, + "completionPerMTok": 8.0, + "inputCacheReadPerMTok": 0.5, + "blendedOneInOneOutPerM": 10.0, + "isFree": false + }, + { + "id": "perplexity/sonar-deep-research", + "canonicalSlug": "perplexity/sonar-deep-research", + "name": "Perplexity: Sonar Deep Research", + "contextLength": 128000, + "promptPerMTok": 2.0, + "completionPerMTok": 8.0, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 10.0, + "isFree": false + }, + { + "id": "perplexity/sonar-reasoning-pro", + "canonicalSlug": "perplexity/sonar-reasoning-pro", + "name": "Perplexity: Sonar Reasoning Pro", + "contextLength": 128000, + "promptPerMTok": 2.0, + "completionPerMTok": 8.0, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 10.0, + "isFree": false + }, + { + "id": "google/gemini-3.5-flash", + "canonicalSlug": "google/gemini-3.5-flash-20260519", + "name": "Google: Gemini 3.5 Flash", + "contextLength": 1048576, + "promptPerMTok": 1.5, + "completionPerMTok": 9.0, + "inputCacheReadPerMTok": 0.15, + "blendedOneInOneOutPerM": 10.5, + "isFree": false + }, + { + "id": "~google/gemini-flash-latest", + "canonicalSlug": "~google/gemini-flash-latest", + "name": "Google Gemini Flash Latest", + "contextLength": 1048576, + "promptPerMTok": 1.5, + "completionPerMTok": 9.0, + "inputCacheReadPerMTok": 0.15, + "blendedOneInOneOutPerM": 10.5, + "isFree": false + }, + { + "id": "google/gemini-2.5-pro", + "canonicalSlug": "google/gemini-2.5-pro", + "name": "Google: Gemini 2.5 Pro", + "contextLength": 1048576, + "promptPerMTok": 1.25, + "completionPerMTok": 10.0, + "inputCacheReadPerMTok": 0.125, + "blendedOneInOneOutPerM": 11.25, + "isFree": false + }, + { + "id": "google/gemini-2.5-pro-preview", + "canonicalSlug": "google/gemini-2.5-pro-preview-06-05", + "name": "Google: Gemini 2.5 Pro Preview 06-05", + "contextLength": 1048576, + "promptPerMTok": 1.25, + "completionPerMTok": 10.0, + "inputCacheReadPerMTok": 0.125, + "blendedOneInOneOutPerM": 11.25, + "isFree": false + }, + { + "id": "google/gemini-2.5-pro-preview-05-06", + "canonicalSlug": "google/gemini-2.5-pro-preview-03-25", + "name": "Google: Gemini 2.5 Pro Preview 05-06", + "contextLength": 1048576, + "promptPerMTok": 1.25, + "completionPerMTok": 10.0, + "inputCacheReadPerMTok": 0.125, + "blendedOneInOneOutPerM": 11.25, + "isFree": false + }, + { + "id": "openai/gpt-5", + "canonicalSlug": "openai/gpt-5-2025-08-07", + "name": "OpenAI: GPT-5", + "contextLength": 400000, + "promptPerMTok": 1.25, + "completionPerMTok": 10.0, + "inputCacheReadPerMTok": 0.125, + "blendedOneInOneOutPerM": 11.25, + "isFree": false + }, + { + "id": "openai/gpt-5-chat", + "canonicalSlug": "openai/gpt-5-chat-2025-08-07", + "name": "OpenAI: GPT-5 Chat", + "contextLength": 128000, + "promptPerMTok": 1.25, + "completionPerMTok": 10.0, + "inputCacheReadPerMTok": 0.125, + "blendedOneInOneOutPerM": 11.25, + "isFree": false + }, + { + "id": "openai/gpt-5-codex", + "canonicalSlug": "openai/gpt-5-codex", + "name": "OpenAI: GPT-5 Codex", + "contextLength": 400000, + "promptPerMTok": 1.25, + "completionPerMTok": 10.0, + "inputCacheReadPerMTok": 0.125, + "blendedOneInOneOutPerM": 11.25, + "isFree": false + }, + { + "id": "openai/gpt-5.1", + "canonicalSlug": "openai/gpt-5.1-20251113", + "name": "OpenAI: GPT-5.1", + "contextLength": 400000, + "promptPerMTok": 1.25, + "completionPerMTok": 10.0, + "inputCacheReadPerMTok": 0.13, + "blendedOneInOneOutPerM": 11.25, + "isFree": false + }, + { + "id": "openai/gpt-5.1-chat", + "canonicalSlug": "openai/gpt-5.1-chat-20251113", + "name": "OpenAI: GPT-5.1 Chat", + "contextLength": 128000, + "promptPerMTok": 1.25, + "completionPerMTok": 10.0, + "inputCacheReadPerMTok": 0.13, + "blendedOneInOneOutPerM": 11.25, + "isFree": false + }, + { + "id": "openai/gpt-5.1-codex", + "canonicalSlug": "openai/gpt-5.1-codex-20251113", + "name": "OpenAI: GPT-5.1-Codex", + "contextLength": 400000, + "promptPerMTok": 1.25, + "completionPerMTok": 10.0, + "inputCacheReadPerMTok": 0.13, + "blendedOneInOneOutPerM": 11.25, + "isFree": false + }, + { + "id": "openai/gpt-5.1-codex-max", + "canonicalSlug": "openai/gpt-5.1-codex-max-20251204", + "name": "OpenAI: GPT-5.1-Codex-Max", + "contextLength": 400000, + "promptPerMTok": 1.25, + "completionPerMTok": 10.0, + "inputCacheReadPerMTok": 0.125, + "blendedOneInOneOutPerM": 11.25, + "isFree": false + }, + { + "id": "aion-labs/aion-1.0", + "canonicalSlug": "aion-labs/aion-1.0", + "name": "AionLabs: Aion-1.0", + "contextLength": 131072, + "promptPerMTok": 4.0, + "completionPerMTok": 8.0, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 12.0, + "isFree": false + }, + { + "id": "cohere/command-a", + "canonicalSlug": "cohere/command-a-03-2025", + "name": "Cohere: Command A", + "contextLength": 256000, + "promptPerMTok": 2.5, + "completionPerMTok": 10.0, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 12.5, + "isFree": false + }, + { + "id": "cohere/command-r-plus-08-2024", + "canonicalSlug": "cohere/command-r-plus-08-2024", + "name": "Cohere: Command R+ (08-2024)", + "contextLength": 128000, + "promptPerMTok": 2.5, + "completionPerMTok": 10.0, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 12.5, + "isFree": false + }, + { + "id": "inflection/inflection-3-pi", + "canonicalSlug": "inflection/inflection-3-pi", + "name": "Inflection: Inflection 3 Pi", + "contextLength": 8000, + "promptPerMTok": 2.5, + "completionPerMTok": 10.0, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 12.5, + "isFree": false + }, + { + "id": "inflection/inflection-3-productivity", + "canonicalSlug": "inflection/inflection-3-productivity", + "name": "Inflection: Inflection 3 Productivity", + "contextLength": 8000, + "promptPerMTok": 2.5, + "completionPerMTok": 10.0, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 12.5, + "isFree": false + }, + { + "id": "openai/gpt-4o", + "canonicalSlug": "openai/gpt-4o", + "name": "OpenAI: GPT-4o", + "contextLength": 128000, + "promptPerMTok": 2.5, + "completionPerMTok": 10.0, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 12.5, + "isFree": false + }, + { + "id": "openai/gpt-4o-2024-08-06", + "canonicalSlug": "openai/gpt-4o-2024-08-06", + "name": "OpenAI: GPT-4o (2024-08-06)", + "contextLength": 128000, + "promptPerMTok": 2.5, + "completionPerMTok": 10.0, + "inputCacheReadPerMTok": 1.25, + "blendedOneInOneOutPerM": 12.5, + "isFree": false + }, + { + "id": "openai/gpt-4o-2024-11-20", + "canonicalSlug": "openai/gpt-4o-2024-11-20", + "name": "OpenAI: GPT-4o (2024-11-20)", + "contextLength": 128000, + "promptPerMTok": 2.5, + "completionPerMTok": 10.0, + "inputCacheReadPerMTok": 1.25, + "blendedOneInOneOutPerM": 12.5, + "isFree": false + }, + { + "id": "openai/gpt-4o-search-preview", + "canonicalSlug": "openai/gpt-4o-search-preview-2025-03-11", + "name": "OpenAI: GPT-4o Search Preview", + "contextLength": 128000, + "promptPerMTok": 2.5, + "completionPerMTok": 10.0, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 12.5, + "isFree": false + }, + { + "id": "openai/gpt-audio", + "canonicalSlug": "openai/gpt-audio", + "name": "OpenAI: GPT Audio", + "contextLength": 128000, + "promptPerMTok": 2.5, + "completionPerMTok": 10.0, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 12.5, + "isFree": false + }, + { + "id": "google/gemini-3-pro-image-preview", + "canonicalSlug": "google/gemini-3-pro-image-preview-20251120", + "name": "Google: Nano Banana Pro (Gemini 3 Pro Image Preview)", + "contextLength": 65536, + "promptPerMTok": 2.0, + "completionPerMTok": 12.0, + "inputCacheReadPerMTok": 0.2, + "blendedOneInOneOutPerM": 14.0, + "isFree": false + }, + { + "id": "google/gemini-3.1-pro-preview", + "canonicalSlug": "google/gemini-3.1-pro-preview-20260219", + "name": "Google: Gemini 3.1 Pro Preview", + "contextLength": 1048576, + "promptPerMTok": 2.0, + "completionPerMTok": 12.0, + "inputCacheReadPerMTok": 0.2, + "blendedOneInOneOutPerM": 14.0, + "isFree": false + }, + { + "id": "google/gemini-3.1-pro-preview-customtools", + "canonicalSlug": "google/gemini-3.1-pro-preview-customtools-20260219", + "name": "Google: Gemini 3.1 Pro Preview Custom Tools", + "contextLength": 1048756, + "promptPerMTok": 2.0, + "completionPerMTok": 12.0, + "inputCacheReadPerMTok": 0.2, + "blendedOneInOneOutPerM": 14.0, + "isFree": false + }, + { + "id": "~google/gemini-pro-latest", + "canonicalSlug": "~google/gemini-pro-latest", + "name": "Google Gemini Pro Latest", + "contextLength": 1048576, + "promptPerMTok": 2.0, + "completionPerMTok": 12.0, + "inputCacheReadPerMTok": 0.2, + "blendedOneInOneOutPerM": 14.0, + "isFree": false + }, + { + "id": "amazon/nova-premier-v1", + "canonicalSlug": "amazon/nova-premier-v1", + "name": "Amazon: Nova Premier 1.0", + "contextLength": 1000000, + "promptPerMTok": 2.5, + "completionPerMTok": 12.5, + "inputCacheReadPerMTok": 0.625, + "blendedOneInOneOutPerM": 15.0, + "isFree": false + }, + { + "id": "openai/gpt-5.2", + "canonicalSlug": "openai/gpt-5.2-20251211", + "name": "OpenAI: GPT-5.2", + "contextLength": 400000, + "promptPerMTok": 1.75, + "completionPerMTok": 14.0, + "inputCacheReadPerMTok": 0.175, + "blendedOneInOneOutPerM": 15.75, + "isFree": false + }, + { + "id": "openai/gpt-5.2-chat", + "canonicalSlug": "openai/gpt-5.2-chat-20251211", + "name": "OpenAI: GPT-5.2 Chat", + "contextLength": 128000, + "promptPerMTok": 1.75, + "completionPerMTok": 14.0, + "inputCacheReadPerMTok": 0.175, + "blendedOneInOneOutPerM": 15.75, + "isFree": false + }, + { + "id": "openai/gpt-5.2-codex", + "canonicalSlug": "openai/gpt-5.2-codex-20260114", + "name": "OpenAI: GPT-5.2-Codex", + "contextLength": 400000, + "promptPerMTok": 1.75, + "completionPerMTok": 14.0, + "inputCacheReadPerMTok": 0.175, + "blendedOneInOneOutPerM": 15.75, + "isFree": false + }, + { + "id": "openai/gpt-5.3-chat", + "canonicalSlug": "openai/gpt-5.3-chat-20260303", + "name": "OpenAI: GPT-5.3 Chat", + "contextLength": 128000, + "promptPerMTok": 1.75, + "completionPerMTok": 14.0, + "inputCacheReadPerMTok": 0.175, + "blendedOneInOneOutPerM": 15.75, + "isFree": false + }, + { + "id": "openai/gpt-5.3-codex", + "canonicalSlug": "openai/gpt-5.3-codex-20260224", + "name": "OpenAI: GPT-5.3-Codex", + "contextLength": 400000, + "promptPerMTok": 1.75, + "completionPerMTok": 14.0, + "inputCacheReadPerMTok": 0.175, + "blendedOneInOneOutPerM": 15.75, + "isFree": false + }, + { + "id": "openai/gpt-5.4", + "canonicalSlug": "openai/gpt-5.4-20260305", + "name": "OpenAI: GPT-5.4", + "contextLength": 1050000, + "promptPerMTok": 2.5, + "completionPerMTok": 15.0, + "inputCacheReadPerMTok": 0.25, + "blendedOneInOneOutPerM": 17.5, + "isFree": false + }, + { + "id": "anthropic/claude-sonnet-4", + "canonicalSlug": "anthropic/claude-4-sonnet-20250522", + "name": "Anthropic: Claude Sonnet 4", + "contextLength": 1000000, + "promptPerMTok": 3.0, + "completionPerMTok": 15.0, + "inputCacheReadPerMTok": 0.3, + "blendedOneInOneOutPerM": 18.0, + "isFree": false + }, + { + "id": "anthropic/claude-sonnet-4.5", + "canonicalSlug": "anthropic/claude-4.5-sonnet-20250929", + "name": "Anthropic: Claude Sonnet 4.5", + "contextLength": 1000000, + "promptPerMTok": 3.0, + "completionPerMTok": 15.0, + "inputCacheReadPerMTok": 0.3, + "blendedOneInOneOutPerM": 18.0, + "isFree": false + }, + { + "id": "anthropic/claude-sonnet-4.6", + "canonicalSlug": "anthropic/claude-4.6-sonnet-20260217", + "name": "Anthropic: Claude Sonnet 4.6", + "contextLength": 1000000, + "promptPerMTok": 3.0, + "completionPerMTok": 15.0, + "inputCacheReadPerMTok": 0.3, + "blendedOneInOneOutPerM": 18.0, + "isFree": false + }, + { + "id": "perplexity/sonar-pro", + "canonicalSlug": "perplexity/sonar-pro", + "name": "Perplexity: Sonar Pro", + "contextLength": 200000, + "promptPerMTok": 3.0, + "completionPerMTok": 15.0, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 18.0, + "isFree": false + }, + { + "id": "perplexity/sonar-pro-search", + "canonicalSlug": "perplexity/sonar-pro-search", + "name": "Perplexity: Sonar Pro Search", + "contextLength": 200000, + "promptPerMTok": 3.0, + "completionPerMTok": 15.0, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 18.0, + "isFree": false + }, + { + "id": "~anthropic/claude-sonnet-latest", + "canonicalSlug": "~anthropic/claude-sonnet-latest", + "name": "Anthropic Claude Sonnet Latest", + "contextLength": 1000000, + "promptPerMTok": 3.0, + "completionPerMTok": 15.0, + "inputCacheReadPerMTok": 0.3, + "blendedOneInOneOutPerM": 18.0, + "isFree": false + }, + { + "id": "openai/gpt-4o-2024-05-13", + "canonicalSlug": "openai/gpt-4o-2024-05-13", + "name": "OpenAI: GPT-4o (2024-05-13)", + "contextLength": 128000, + "promptPerMTok": 5.0, + "completionPerMTok": 15.0, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 20.0, + "isFree": false + }, + { + "id": "openai/gpt-5-image", + "canonicalSlug": "openai/gpt-5-image", + "name": "OpenAI: GPT-5 Image", + "contextLength": 400000, + "promptPerMTok": 10.0, + "completionPerMTok": 10.0, + "inputCacheReadPerMTok": 1.25, + "blendedOneInOneOutPerM": 20.0, + "isFree": false + }, + { + "id": "openai/gpt-5.4-image-2", + "canonicalSlug": "openai/gpt-5.4-image-2-20260421", + "name": "OpenAI: GPT-5.4 Image 2", + "contextLength": 272000, + "promptPerMTok": 8.0, + "completionPerMTok": 15.0, + "inputCacheReadPerMTok": 2.0, + "blendedOneInOneOutPerM": 23.0, + "isFree": false + }, + { + "id": "anthropic/claude-opus-4.5", + "canonicalSlug": "anthropic/claude-4.5-opus-20251124", + "name": "Anthropic: Claude Opus 4.5", + "contextLength": 200000, + "promptPerMTok": 5.0, + "completionPerMTok": 25.0, + "inputCacheReadPerMTok": 0.5, + "blendedOneInOneOutPerM": 30.0, + "isFree": false + }, + { + "id": "anthropic/claude-opus-4.6", + "canonicalSlug": "anthropic/claude-4.6-opus-20260205", + "name": "Anthropic: Claude Opus 4.6", + "contextLength": 1000000, + "promptPerMTok": 5.0, + "completionPerMTok": 25.0, + "inputCacheReadPerMTok": 0.5, + "blendedOneInOneOutPerM": 30.0, + "isFree": false + }, + { + "id": "anthropic/claude-opus-4.7", + "canonicalSlug": "anthropic/claude-4.7-opus-20260416", + "name": "Anthropic: Claude Opus 4.7", + "contextLength": 1000000, + "promptPerMTok": 5.0, + "completionPerMTok": 25.0, + "inputCacheReadPerMTok": 0.5, + "blendedOneInOneOutPerM": 30.0, + "isFree": false + }, + { + "id": "anthropic/claude-opus-4.8", + "canonicalSlug": "anthropic/claude-4.8-opus-20260528", + "name": "Anthropic: Claude Opus 4.8", + "contextLength": 1000000, + "promptPerMTok": 5.0, + "completionPerMTok": 25.0, + "inputCacheReadPerMTok": 0.5, + "blendedOneInOneOutPerM": 30.0, + "isFree": false + }, + { + "id": "~anthropic/claude-opus-latest", + "canonicalSlug": "~anthropic/claude-opus-latest", + "name": "Anthropic: Claude Opus Latest", + "contextLength": 1000000, + "promptPerMTok": 5.0, + "completionPerMTok": 25.0, + "inputCacheReadPerMTok": 0.5, + "blendedOneInOneOutPerM": 30.0, + "isFree": false + }, + { + "id": "openai/gpt-5.5", + "canonicalSlug": "openai/gpt-5.5-20260423", + "name": "OpenAI: GPT-5.5", + "contextLength": 1050000, + "promptPerMTok": 5.0, + "completionPerMTok": 30.0, + "inputCacheReadPerMTok": 0.5, + "blendedOneInOneOutPerM": 35.0, + "isFree": false + }, + { + "id": "openai/gpt-chat-latest", + "canonicalSlug": "openai/gpt-chat-latest-20260505", + "name": "OpenAI: GPT Chat Latest", + "contextLength": 400000, + "promptPerMTok": 5.0, + "completionPerMTok": 30.0, + "inputCacheReadPerMTok": 0.5, + "blendedOneInOneOutPerM": 35.0, + "isFree": false + }, + { + "id": "~openai/gpt-latest", + "canonicalSlug": "~openai/gpt-latest", + "name": "OpenAI GPT Latest", + "contextLength": 1050000, + "promptPerMTok": 5.0, + "completionPerMTok": 30.0, + "inputCacheReadPerMTok": 0.5, + "blendedOneInOneOutPerM": 35.0, + "isFree": false + }, + { + "id": "openai/gpt-4-1106-preview", + "canonicalSlug": "openai/gpt-4-1106-preview", + "name": "OpenAI: GPT-4 Turbo (older v1106)", + "contextLength": 128000, + "promptPerMTok": 10.0, + "completionPerMTok": 30.0, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 40.0, + "isFree": false + }, + { + "id": "openai/gpt-4-turbo", + "canonicalSlug": "openai/gpt-4-turbo", + "name": "OpenAI: GPT-4 Turbo", + "contextLength": 128000, + "promptPerMTok": 10.0, + "completionPerMTok": 30.0, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 40.0, + "isFree": false + }, + { + "id": "openai/gpt-4-turbo-preview", + "canonicalSlug": "openai/gpt-4-turbo-preview", + "name": "OpenAI: GPT-4 Turbo Preview", + "contextLength": 128000, + "promptPerMTok": 10.0, + "completionPerMTok": 30.0, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 40.0, + "isFree": false + }, + { + "id": "openai/o3-deep-research", + "canonicalSlug": "openai/o3-deep-research-2025-06-26", + "name": "OpenAI: o3 Deep Research", + "contextLength": 200000, + "promptPerMTok": 10.0, + "completionPerMTok": 40.0, + "inputCacheReadPerMTok": 2.5, + "blendedOneInOneOutPerM": 50.0, + "isFree": false + }, + { + "id": "anthropic/claude-opus-4.8-fast", + "canonicalSlug": "anthropic/claude-4.8-opus-fast-20260528", + "name": "Anthropic: Claude Opus 4.8 (Fast)", + "contextLength": 1000000, + "promptPerMTok": 10.0, + "completionPerMTok": 50.0, + "inputCacheReadPerMTok": 1.0, + "blendedOneInOneOutPerM": 60.0, + "isFree": false + }, + { + "id": "openai/o1", + "canonicalSlug": "openai/o1-2024-12-17", + "name": "OpenAI: o1", + "contextLength": 200000, + "promptPerMTok": 15.0, + "completionPerMTok": 60.0, + "inputCacheReadPerMTok": 7.5, + "blendedOneInOneOutPerM": 75.0, + "isFree": false + }, + { + "id": "anthropic/claude-opus-4", + "canonicalSlug": "anthropic/claude-4-opus-20250522", + "name": "Anthropic: Claude Opus 4", + "contextLength": 200000, + "promptPerMTok": 15.0, + "completionPerMTok": 75.0, + "inputCacheReadPerMTok": 1.5, + "blendedOneInOneOutPerM": 90.0, + "isFree": false + }, + { + "id": "anthropic/claude-opus-4.1", + "canonicalSlug": "anthropic/claude-4.1-opus-20250805", + "name": "Anthropic: Claude Opus 4.1", + "contextLength": 200000, + "promptPerMTok": 15.0, + "completionPerMTok": 75.0, + "inputCacheReadPerMTok": 1.5, + "blendedOneInOneOutPerM": 90.0, + "isFree": false + }, + { + "id": "openai/gpt-4", + "canonicalSlug": "openai/gpt-4", + "name": "OpenAI: GPT-4", + "contextLength": 8191, + "promptPerMTok": 30.0, + "completionPerMTok": 60.0, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 90.0, + "isFree": false + }, + { + "id": "openai/o3-pro", + "canonicalSlug": "openai/o3-pro-2025-06-10", + "name": "OpenAI: o3 Pro", + "contextLength": 200000, + "promptPerMTok": 20.0, + "completionPerMTok": 80.0, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 100.0, + "isFree": false + }, + { + "id": "openai/gpt-5-pro", + "canonicalSlug": "openai/gpt-5-pro-2025-10-06", + "name": "OpenAI: GPT-5 Pro", + "contextLength": 400000, + "promptPerMTok": 15.0, + "completionPerMTok": 120.0, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 135.0, + "isFree": false + }, + { + "id": "anthropic/claude-opus-4.6-fast", + "canonicalSlug": "anthropic/claude-4.6-opus-fast-20260407", + "name": "Anthropic: Claude Opus 4.6 (Fast)", + "contextLength": 1000000, + "promptPerMTok": 30.0, + "completionPerMTok": 150.0, + "inputCacheReadPerMTok": 3.0, + "blendedOneInOneOutPerM": 180.0, + "isFree": false + }, + { + "id": "anthropic/claude-opus-4.7-fast", + "canonicalSlug": "anthropic/claude-4.7-opus-fast-20260512", + "name": "Anthropic: Claude Opus 4.7 (Fast)", + "contextLength": 1000000, + "promptPerMTok": 30.0, + "completionPerMTok": 150.0, + "inputCacheReadPerMTok": 3.0, + "blendedOneInOneOutPerM": 180.0, + "isFree": false + }, + { + "id": "openai/gpt-5.2-pro", + "canonicalSlug": "openai/gpt-5.2-pro-20251211", + "name": "OpenAI: GPT-5.2 Pro", + "contextLength": 400000, + "promptPerMTok": 21.0, + "completionPerMTok": 168.0, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 189.0, + "isFree": false + }, + { + "id": "openai/gpt-5.4-pro", + "canonicalSlug": "openai/gpt-5.4-pro-20260305", + "name": "OpenAI: GPT-5.4 Pro", + "contextLength": 1050000, + "promptPerMTok": 30.0, + "completionPerMTok": 180.0, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 210.0, + "isFree": false + }, + { + "id": "openai/gpt-5.5-pro", + "canonicalSlug": "openai/gpt-5.5-pro-20260423", + "name": "OpenAI: GPT-5.5 Pro", + "contextLength": 1050000, + "promptPerMTok": 30.0, + "completionPerMTok": 180.0, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 210.0, + "isFree": false + }, + { + "id": "openai/o1-pro", + "canonicalSlug": "openai/o1-pro", + "name": "OpenAI: o1-pro", + "contextLength": 200000, + "promptPerMTok": 150.0, + "completionPerMTok": 600.0, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 750.0, + "isFree": false + }, + { + "id": "cognitivecomputations/dolphin-mistral-24b-venice-edition:free", + "canonicalSlug": "venice/uncensored", + "name": "Venice: Uncensored (free)", + "contextLength": 32768, + "promptPerMTok": 0.0, + "completionPerMTok": 0.0, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 0, + "isFree": true + }, + { + "id": "google/gemma-4-26b-a4b-it:free", + "canonicalSlug": "google/gemma-4-26b-a4b-it-20260403", + "name": "Google: Gemma 4 26B A4B (free)", + "contextLength": 262144, + "promptPerMTok": 0.0, + "completionPerMTok": 0.0, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 0, + "isFree": true + }, + { + "id": "google/gemma-4-31b-it:free", + "canonicalSlug": "google/gemma-4-31b-it-20260402", + "name": "Google: Gemma 4 31B (free)", + "contextLength": 262144, + "promptPerMTok": 0.0, + "completionPerMTok": 0.0, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 0, + "isFree": true + }, + { + "id": "google/lyria-3-clip-preview", + "canonicalSlug": "google/lyria-3-clip-preview-20260330", + "name": "Google: Lyria 3 Clip Preview", + "contextLength": 1048576, + "promptPerMTok": 0.0, + "completionPerMTok": 0.0, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 0, + "isFree": true + }, + { + "id": "google/lyria-3-pro-preview", + "canonicalSlug": "google/lyria-3-pro-preview-20260330", + "name": "Google: Lyria 3 Pro Preview", + "contextLength": 1048576, + "promptPerMTok": 0.0, + "completionPerMTok": 0.0, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 0, + "isFree": true + }, + { + "id": "liquid/lfm-2.5-1.2b-instruct:free", + "canonicalSlug": "liquid/lfm-2.5-1.2b-instruct-20260120", + "name": "LiquidAI: LFM2.5-1.2B-Instruct (free)", + "contextLength": 32768, + "promptPerMTok": 0.0, + "completionPerMTok": 0.0, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 0, + "isFree": true + }, + { + "id": "liquid/lfm-2.5-1.2b-thinking:free", + "canonicalSlug": "liquid/lfm-2.5-1.2b-thinking-20260120", + "name": "LiquidAI: LFM2.5-1.2B-Thinking (free)", + "contextLength": 32768, + "promptPerMTok": 0.0, + "completionPerMTok": 0.0, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 0, + "isFree": true + }, + { + "id": "meta-llama/llama-3.2-3b-instruct:free", + "canonicalSlug": "meta-llama/llama-3.2-3b-instruct", + "name": "Meta: Llama 3.2 3B Instruct (free)", + "contextLength": 131072, + "promptPerMTok": 0.0, + "completionPerMTok": 0.0, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 0, + "isFree": true + }, + { + "id": "meta-llama/llama-3.3-70b-instruct:free", + "canonicalSlug": "meta-llama/llama-3.3-70b-instruct", + "name": "Meta: Llama 3.3 70B Instruct (free)", + "contextLength": 131072, + "promptPerMTok": 0.0, + "completionPerMTok": 0.0, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 0, + "isFree": true + }, + { + "id": "moonshotai/kimi-k2.6:free", + "canonicalSlug": "moonshotai/kimi-k2.6-20260420", + "name": "MoonshotAI: Kimi K2.6 (free)", + "contextLength": 262144, + "promptPerMTok": 0.0, + "completionPerMTok": 0.0, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 0, + "isFree": true + }, + { + "id": "nousresearch/hermes-3-llama-3.1-405b:free", + "canonicalSlug": "nousresearch/hermes-3-llama-3.1-405b", + "name": "Nous: Hermes 3 405B Instruct (free)", + "contextLength": 131072, + "promptPerMTok": 0.0, + "completionPerMTok": 0.0, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 0, + "isFree": true + }, + { + "id": "nvidia/nemotron-3-nano-30b-a3b:free", + "canonicalSlug": "nvidia/nemotron-3-nano-30b-a3b", + "name": "NVIDIA: Nemotron 3 Nano 30B A3B (free)", + "contextLength": 256000, + "promptPerMTok": 0.0, + "completionPerMTok": 0.0, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 0, + "isFree": true + }, + { + "id": "nvidia/nemotron-3-nano-omni-30b-a3b-reasoning:free", + "canonicalSlug": "nvidia/nemotron-3-nano-omni-30b-a3b-reasoning-20260428", + "name": "NVIDIA: Nemotron 3 Nano Omni (free)", + "contextLength": 256000, + "promptPerMTok": 0.0, + "completionPerMTok": 0.0, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 0, + "isFree": true + }, + { + "id": "nvidia/nemotron-3-super-120b-a12b:free", + "canonicalSlug": "nvidia/nemotron-3-super-120b-a12b-20230311", + "name": "NVIDIA: Nemotron 3 Super (free)", + "contextLength": 1000000, + "promptPerMTok": 0.0, + "completionPerMTok": 0.0, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 0, + "isFree": true + }, + { + "id": "nvidia/nemotron-3-ultra-550b-a55b:free", + "canonicalSlug": "nvidia/nemotron-3-ultra-550b-a55b-20260604", + "name": "NVIDIA: Nemotron 3 Ultra (free)", + "contextLength": 1000000, + "promptPerMTok": 0.0, + "completionPerMTok": 0.0, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 0, + "isFree": true + }, + { + "id": "nvidia/nemotron-3.5-content-safety:free", + "canonicalSlug": "nvidia/nemotron-3.5-content-safety-20260604", + "name": "NVIDIA: Nemotron 3.5 Content Safety (free)", + "contextLength": 128000, + "promptPerMTok": 0.0, + "completionPerMTok": 0.0, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 0, + "isFree": true + }, + { + "id": "nvidia/nemotron-nano-12b-v2-vl:free", + "canonicalSlug": "nvidia/nemotron-nano-12b-v2-vl", + "name": "NVIDIA: Nemotron Nano 12B 2 VL (free)", + "contextLength": 128000, + "promptPerMTok": 0.0, + "completionPerMTok": 0.0, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 0, + "isFree": true + }, + { + "id": "nvidia/nemotron-nano-9b-v2:free", + "canonicalSlug": "nvidia/nemotron-nano-9b-v2", + "name": "NVIDIA: Nemotron Nano 9B V2 (free)", + "contextLength": 128000, + "promptPerMTok": 0.0, + "completionPerMTok": 0.0, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 0, + "isFree": true + }, + { + "id": "openai/gpt-oss-120b:free", + "canonicalSlug": "openai/gpt-oss-120b", + "name": "OpenAI: gpt-oss-120b (free)", + "contextLength": 131072, + "promptPerMTok": 0.0, + "completionPerMTok": 0.0, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 0, + "isFree": true + }, + { + "id": "openai/gpt-oss-20b:free", + "canonicalSlug": "openai/gpt-oss-20b", + "name": "OpenAI: gpt-oss-20b (free)", + "contextLength": 131072, + "promptPerMTok": 0.0, + "completionPerMTok": 0.0, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 0, + "isFree": true + }, + { + "id": "openrouter/free", + "canonicalSlug": "openrouter/free", + "name": "Free Models Router", + "contextLength": 200000, + "promptPerMTok": 0.0, + "completionPerMTok": 0.0, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 0, + "isFree": true + }, + { + "id": "openrouter/owl-alpha", + "canonicalSlug": "openrouter/owl-alpha", + "name": "Owl Alpha", + "contextLength": 1048756, + "promptPerMTok": 0.0, + "completionPerMTok": 0.0, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 0, + "isFree": true + }, + { + "id": "poolside/laguna-m.1:free", + "canonicalSlug": "poolside/laguna-m.1-20260312", + "name": "Poolside: Laguna M.1 (free)", + "contextLength": 262144, + "promptPerMTok": 0.0, + "completionPerMTok": 0.0, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 0, + "isFree": true + }, + { + "id": "poolside/laguna-xs.2:free", + "canonicalSlug": "poolside/laguna-xs.2-20260421", + "name": "Poolside: Laguna XS.2 (free)", + "contextLength": 262144, + "promptPerMTok": 0.0, + "completionPerMTok": 0.0, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 0, + "isFree": true + }, + { + "id": "qwen/qwen3-coder:free", + "canonicalSlug": "qwen/qwen3-coder-480b-a35b-07-25", + "name": "Qwen: Qwen3 Coder 480B A35B (free)", + "contextLength": 1048576, + "promptPerMTok": 0.0, + "completionPerMTok": 0.0, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 0, + "isFree": true + }, + { + "id": "qwen/qwen3-next-80b-a3b-instruct:free", + "canonicalSlug": "qwen/qwen3-next-80b-a3b-instruct-2509", + "name": "Qwen: Qwen3 Next 80B A3B Instruct (free)", + "contextLength": 262144, + "promptPerMTok": 0.0, + "completionPerMTok": 0.0, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 0, + "isFree": true + }, + { + "id": "z-ai/glm-4.5-air:free", + "canonicalSlug": "z-ai/glm-4.5-air", + "name": "Z.ai: GLM 4.5 Air (free)", + "contextLength": 131072, + "promptPerMTok": 0.0, + "completionPerMTok": 0.0, + "inputCacheReadPerMTok": null, + "blendedOneInOneOutPerM": 0, + "isFree": true + } + ] +} diff --git a/public/images/stick-figure-shooting-graph.svg b/public/images/stick-figure-shooting-graph.svg new file mode 100644 index 0000000..4586ded --- /dev/null +++ b/public/images/stick-figure-shooting-graph.svg @@ -0,0 +1,11 @@ + + + + + + + + + + +