Phase 14: registry data accuracy

docs/registry-refresh.md

@@ -0,0 +1,36 @@
+# Refreshing the model registry
+
+`src/norefund/config/default_models.yaml` and `model_architectures.yaml` are
+read against primary sources, not updated from memory or from a previous
+model's own claims about itself. Pricing comes from each provider's own
+pricing page, re-read on the day the file is edited: OpenAI
+(`https://developers.openai.com/api/docs/pricing`), Anthropic
+(`https://platform.claude.com/docs/en/about-claude/pricing`), Google
+(`https://ai.google.dev/gemini-api/docs/pricing`), and DeepSeek
+(`https://api-docs.deepseek.com/quick_start/pricing`). Architecture fields
+(layer counts, head dimensions, MLA parameters, context windows) come from
+the model's published HuggingFace `config.json`, and parameter counts are
+cross-checked against the HF API's safetensors total
+(`GET /api/models/{repo}?blobs=true`, the `safetensors.total` field) rather
+than trusted from a model card's rounded number.
+
+`pricing_verified_on` on a `ModelInfo` entry moves only when a human (or an
+agent acting on a human's behalf) has actually opened the provider's page
+that day and read the number off it -- never bumped to "look current"
+without doing that, and never copied forward from an older PR or plan
+document, which can itself have drifted by the time anyone acts on it (this
+happened during the Phase 14 registry refresh: a reference doc's own
+pricing figures, recorded four days earlier, were already wrong by the time
+they were used). `tests/test_registry_data.py` enforces the structural
+invariants this depends on -- every architecture id has a matching registry
+entry, the two files agree on shared fields, every priced entry carries a
+verification date -- but it cannot check whether a given date's price is
+still correct; only re-reading the source page can.
+
+When a licence-gated HuggingFace repo needs an ungated substitute (see the
+comments on the `meta:llama-*` and `google:gemma-2-*` entries), verify
+identity via the HF tree API's git blob oid for the file in question
+(`GET /api/models/{repo}/tree/main`) rather than trusting a mirror's name --
+an identical oid between the official repo and the mirror proves the file
+is byte-identical without needing to authenticate to the gated repo or
+download either copy.

frontend/src/lib/costing.test.ts

@@ -10,7 +10,22 @@ import {
 } from "./costing";
 
 // Mirrors tests/test_costing.py's _MODEL.
-const MODEL = { input_price_per_million: 2.0, output_price_per_million: 8.0 };
+const MODEL = {
+  input_price_per_million: 2.0,
+  output_price_per_million: 8.0,
+  long_context_threshold: null,
+  long_context_input_price_per_million: null,
+  long_context_output_price_per_million: null,
+};
+
+// Mirrors tests/test_costing.py's _TIERED_MODEL.
+const TIERED_MODEL = {
+  input_price_per_million: 2.0,
+  output_price_per_million: 8.0,
+  long_context_threshold: 200_000,
+  long_context_input_price_per_million: 4.0,
+  long_context_output_price_per_million: 16.0,
+};
 
 describe("contextUsagePct", () => {
   it("is null when the window is zero", () => {
@@ -64,14 +79,51 @@ describe("inputCost / outputCost / totalCost", () => {
     expect(outputCost(1_000_000, MODEL)).toBeCloseTo(8.0);
   });
   it("is zero for a free model", () => {
-    const free = { input_price_per_million: 0, output_price_per_million: 0 };
+    const free = {
+      input_price_per_million: 0,
+      output_price_per_million: 0,
+      long_context_threshold: null,
+      long_context_input_price_per_million: null,
+      long_context_output_price_per_million: null,
+    };
     expect(inputCost(100_000, free)).toBe(0);
   });
   it("sums input and output", () => {
     expect(totalCost(1_000_000, 1_000_000, MODEL)).toBeCloseTo(10.0);
   });
 });
 
+describe("context-tiered pricing", () => {
+  it("uses the short rate exactly at the threshold", () => {
+    expect(inputCost(200_000, TIERED_MODEL)).toBeCloseTo((200_000 / 1_000_000) * 2.0);
+  });
+  it("uses the short rate just below the threshold", () => {
+    expect(inputCost(199_999, TIERED_MODEL)).toBeCloseTo((199_999 / 1_000_000) * 2.0);
+  });
+  it("uses the long rate just above the threshold", () => {
+    expect(inputCost(200_001, TIERED_MODEL)).toBeCloseTo((200_001 / 1_000_000) * 4.0);
+  });
+  it("tiers output cost by prompt size, not output size", () => {
+    const cost = outputCost(500_000, TIERED_MODEL, 1_000);
+    expect(cost).toBeCloseTo((500_000 / 1_000_000) * 8.0);
+  });
+  it("uses the long output rate when the prompt crosses the threshold", () => {
+    const cost = outputCost(1_000, TIERED_MODEL, 200_001);
+    expect(cost).toBeCloseTo((1_000 / 1_000_000) * 16.0);
+  });
+  it("defaults the prompt size to the output's own count when omitted", () => {
+    expect(outputCost(200_001, TIERED_MODEL)).toBeCloseTo((200_001 / 1_000_000) * 16.0);
+  });
+  it("totalCost tiers output by the input size", () => {
+    const cost = totalCost(300_000, 1_000, TIERED_MODEL);
+    const expected = (300_000 / 1_000_000) * 4.0 + (1_000 / 1_000_000) * 16.0;
+    expect(cost).toBeCloseTo(expected);
+  });
+  it("is a no-op for a model with no tier fields", () => {
+    expect(inputCost(10_000_000, MODEL)).toBeCloseTo((10_000_000 / 1_000_000) * 2.0);
+  });
+});
+
 describe("convertCurrency", () => {
   it("applies the rate for the target currency", () => {
     const rates = { base: "USD", rates: { EUR: 0.5 }, fetched_at: null };

frontend/src/lib/costing.ts

@@ -28,26 +28,58 @@ export function minChunks(tokens: number, window: number): number {
   return Math.ceil(tokens / usable);
 }
 
+type TieredPricingFields = Pick<
+  ModelInfo,
+  | "long_context_threshold"
+  | "long_context_input_price_per_million"
+  | "long_context_output_price_per_million"
+>;
+
+/** Whether promptTokens crosses this model's long-context tier -- decided by
+ *  the prompt (input) size, per every provider that offers one, not by the
+ *  completion size even when pricing output. */
+function longContextActive(promptTokens: number, model: TieredPricingFields): boolean {
+  return (
+    model.long_context_threshold !== null && promptTokens > model.long_context_threshold
+  );
+}
+
 export function inputCost(
   tokens: number,
-  model: Pick<ModelInfo, "input_price_per_million">,
+  model: Pick<ModelInfo, "input_price_per_million"> & TieredPricingFields,
 ): number {
-  return (tokens / 1_000_000) * model.input_price_per_million;
+  const rate =
+    longContextActive(tokens, model) && model.long_context_input_price_per_million !== null
+      ? model.long_context_input_price_per_million
+      : model.input_price_per_million;
+  return (tokens / 1_000_000) * rate;
 }
 
+/** promptTokens is the input size that decides which price tier applies.
+ *  Defaults to tokens itself when omitted, which is only correct for
+ *  flat-priced models (long_context_threshold null) -- callers pricing a
+ *  tiered model must pass the real prompt size. */
 export function outputCost(
   tokens: number,
-  model: Pick<ModelInfo, "output_price_per_million">,
+  model: Pick<ModelInfo, "output_price_per_million"> & TieredPricingFields,
+  promptTokens?: number,
 ): number {
-  return (tokens / 1_000_000) * model.output_price_per_million;
+  const promptSize = promptTokens ?? tokens;
+  const rate =
+    longContextActive(promptSize, model) &&
+    model.long_context_output_price_per_million !== null
+      ? model.long_context_output_price_per_million
+      : model.output_price_per_million;
+  return (tokens / 1_000_000) * rate;
 }
 
 export function totalCost(
   inputTokens: number,
   outputTokens: number,
-  model: Pick<ModelInfo, "input_price_per_million" | "output_price_per_million">,
+  model: Pick<ModelInfo, "input_price_per_million" | "output_price_per_million"> &
+    TieredPricingFields,
 ): number {
-  return inputCost(inputTokens, model) + outputCost(outputTokens, model);
+  return inputCost(inputTokens, model) + outputCost(outputTokens, model, inputTokens);
 }
 
 /** Convert a USD amount into toCurrency using the given rates -- mirrors

frontend/src/lib/types.ts

@@ -16,6 +16,11 @@ export interface ModelInfo {
   currency: string;
   docs_url: string | null;
   tokenizer_is_approximate: boolean;
+  long_context_threshold: number | null;
+  long_context_input_price_per_million: number | null;
+  long_context_output_price_per_million: number | null;
+  pricing_note: string | null;
+  pricing_verified_on: string | null;
 }
 
 export interface AnalysisResult {
@@ -102,6 +107,8 @@ export interface ModelArchitecture {
   attention_type: string;
   kv_lora_rank: number;
   qk_rope_head_dim: number;
+  sliding_window: number;
+  sliding_window_pattern: number;
   docs_url: string | null;
 }
 

frontend/src/views/Calculator/index.tsx

@@ -50,7 +50,10 @@ export default function Calculator() {
   const pct = inputTokens === null ? null : contextUsagePct(inputTokens, model.context_window);
   const fits = inputTokens === null ? null : fitsInContext(inputTokens, model.context_window);
   const inCost = inputTokens === null ? null : inputCost(inputTokens, model);
-  const outCost = outputTokens === null ? null : outputCost(outputTokens, model);
+  const outCost =
+    outputTokens === null
+      ? null
+      : outputCost(outputTokens, model, inputTokens ?? undefined);
   const total = inCost === null || outCost === null ? null : inCost + outCost;
 
   return (

frontend/src/views/Compare/sort.test.ts

@@ -15,6 +15,11 @@ function model(id: string): ModelInfo {
     currency: "USD",
     docs_url: null,
     tokenizer_is_approximate: false,
+    long_context_threshold: null,
+    long_context_input_price_per_million: null,
+    long_context_output_price_per_million: null,
+    pricing_note: null,
+    pricing_verified_on: null,
   };
 }
 

frontend/src/views/Registry/index.tsx

@@ -31,16 +31,33 @@ export default function Registry() {
     () => Array.from(new Set(models.map((m) => m.provider))).sort(),
     [models],
   );
+  // The most recent date any priced entry was checked against its
+  // provider's actual page -- not every entry shares one date, so the
+  // true statement is "at least this recently," not "as of exactly this
+  // day." See docs/registry-refresh.md for the refresh procedure.
+  const latestPricingVerification = useMemo(() => {
+    const dates = models
+      .map((m) => m.pricing_verified_on)
+      .filter((d): d is string => d !== null);
+    return dates.length > 0 ? dates.reduce((a, b) => (a > b ? a : b)) : null;
+  }, [models]);
   const visible =
     activeProvider === "All" ? models : models.filter((m) => m.provider === activeProvider);
 
   return (
     <div className="flex flex-col gap-5 p-6">
       <div className="flex flex-wrap items-start justify-between gap-4">
-        <p className="type-body text-muted-foreground">
-          {models.length} models across {providers.length} providers. Locally stored
-          pricing data.
-        </p>
+        <div>
+          <p className="type-body text-muted-foreground">
+            {models.length} models across {providers.length} providers. Locally stored
+            pricing data.
+          </p>
+          {latestPricingVerification && (
+            <p className="type-small text-muted-foreground">
+              Prices verified {latestPricingVerification}
+            </p>
+          )}
+        </div>
         <ProviderFilter
           providers={providers}
           active={activeProvider}