From 24000f24a137443f80f984d6fa70ca50d9c7996b Mon Sep 17 00:00:00 2001 From: Phantom-VK Date: Wed, 26 Aug 2026 11:30:57 +0530 Subject: [PATCH 1/6] fix(config): correct DeepSeek context, Mistral 7B version, H200 memory and the missing Llama 3 8B architecture Verified against live sources on 2026-08-26, not copied from the phase doc: - DeepSeek V3: config.json's max_position_embeddings is 163840; default_models.yaml said 128000, disagreeing with model_architectures.yaml's own 163840 for the same id. - Mistral 7B: default_models.yaml tokenized v0.1 (config.json sliding_window: 4096, so its 32768 is nominal) while model_architectures.yaml's docs_url pointed at v0.3 (sliding_window: null, honestly 32768). Standardised on v0.3. - H200: nvidia-smi reports 143771 MiB = 140.4 GiB, not the marketed 141 GB; this file's own header already states the GiB-not-marketing rule for every other entry. - meta:llama-3-8b had no architecture entry, so it silently couldn't appear in the Fit Check. The official repo is licence-gated (config.json not fetchable without an accepted licence), so verified against two public mirrors (NousResearch, unsloth) with identical config.json and matching safetensors total (8030261248). - Corrected model_architectures.yaml's header claim that every field comes from config.json: the three Qwen2.5 entries' max_context_length is the model card's YaRN-extended figure, not config.json's native 32768 (no rope_scaling block on any of the three). test_architectures.py's hardcoded architecture count updated 13 -> 14. --- src/norefund/config/default_models.yaml | 7 +++-- src/norefund/config/hardware.yaml | 5 ++- src/norefund/config/model_architectures.yaml | 33 +++++++++++++++++++- tests/test_architectures.py | 8 +++-- 4 files changed, 46 insertions(+), 7 deletions(-) diff --git a/src/norefund/config/default_models.yaml b/src/norefund/config/default_models.yaml index 79e32eb..830b7b4 100644 --- a/src/norefund/config/default_models.yaml +++ b/src/norefund/config/default_models.yaml @@ -84,7 +84,7 @@ provider: "DeepSeek" tokenizer_backend: "hf" tokenizer_name: "deepseek-ai/DeepSeek-V3" - context_window: 128000 + context_window: 163840 # config.json's max_position_embeddings; matches model_architectures.yaml input_price_per_million: 0.27 output_price_per_million: 1.10 currency: "USD" @@ -105,7 +105,10 @@ display_name: "Mistral 7B (self-hosted)" provider: "Mistral" tokenizer_backend: "hf" - tokenizer_name: "mistralai/Mistral-7B-v0.1" + # v0.3, not v0.1: v0.1's config.json has sliding_window: 4096, so its + # nominal 32768 context is not honestly usable; v0.3 has sliding_window: + # null, matching model_architectures.yaml's docs_url for this id. + tokenizer_name: "mistralai/Mistral-7B-Instruct-v0.3" context_window: 32768 input_price_per_million: 0.00 output_price_per_million: 0.00 diff --git a/src/norefund/config/hardware.yaml b/src/norefund/config/hardware.yaml index 31e2a36..64327f2 100644 --- a/src/norefund/config/hardware.yaml +++ b/src/norefund/config/hardware.yaml @@ -49,7 +49,10 @@ vendor: "NVIDIA" accelerator: "NVIDIA H200" device_count: 1 - memory_gib_per_device: 141.0 + # NVIDIA markets "141 GB" but nvidia-smi reports 143771 MiB = 140.4 GiB. + # Follow this file's own GiB rule (see header) rather than the marketing + # figure -- do not "correct" this back to 141.0. + memory_gib_per_device: 140.4 memory_kind: "discrete" usable_memory_fraction: 0.90 diff --git a/src/norefund/config/model_architectures.yaml b/src/norefund/config/model_architectures.yaml index 6a87ca8..dcfe470 100644 --- a/src/norefund/config/model_architectures.yaml +++ b/src/norefund/config/model_architectures.yaml @@ -2,7 +2,10 @@ # self-host VRAM fit calculator (core/selfhost.py). # # Every number here is copied from the model's published HuggingFace -# config.json (linked via docs_url), not derived or guessed. In particular: +# config.json (linked via docs_url), not derived or guessed, with one +# exception: max_context_length for the three Qwen2.5 entries comes from +# the model card, not config.json -- see the comment on those entries. +# In particular: # # - head_dim is NOT always hidden_size / n_attention_heads. Gemma 2 9B is the # counter-example: hidden_size=3584, n_attention_heads=16 gives 224, but @@ -14,6 +17,27 @@ # (Mixtral, DeepSeek V3). See core/selfhost.py for why weight memory uses # total_params, not active_params. +- id: meta:llama-3-8b + display_name: "Llama 3 8B" + family: "Llama 3" + vendor: "Meta" + # The official meta-llama/Meta-Llama-3-8B repo is licence-gated, so + # config.json isn't fetchable without an accepted licence + HF token. + # Verified instead against NousResearch/Meta-Llama-3-8B and + # unsloth/llama-3-8b, both public mirrors carrying an identical + # config.json; total_params cross-checked against the mirror's own + # safetensors total (8030261248), not copied from this file. + total_params: 8030261248 + active_params: 8030261248 + n_layers: 32 + n_attention_heads: 32 + n_kv_heads: 8 + head_dim: 128 + hidden_size: 4096 + max_context_length: 8192 + attention_type: "gqa" + docs_url: "https://huggingface.co/meta-llama/Meta-Llama-3-8B" + - id: meta:llama-3.1-8b display_name: "Llama 3.1 8B" family: "Llama 3" @@ -85,6 +109,9 @@ n_kv_heads: 4 head_dim: 128 hidden_size: 3584 + # config.json's native max_position_embeddings is 32768 with no + # rope_scaling block; 131072 is the model card's claimed extended + # window via YaRN, not something config.json states on its own. max_context_length: 131072 attention_type: "gqa" docs_url: "https://huggingface.co/Qwen/Qwen2.5-7B-Instruct" @@ -100,6 +127,8 @@ n_kv_heads: 8 head_dim: 128 hidden_size: 5120 + # See qwen:qwen2.5-7b above -- config.json says 32768, no rope_scaling; + # 131072 is the model card's YaRN-extended figure. max_context_length: 131072 attention_type: "gqa" docs_url: "https://huggingface.co/Qwen/Qwen2.5-32B-Instruct" @@ -115,6 +144,8 @@ n_kv_heads: 8 head_dim: 128 hidden_size: 8192 + # See qwen:qwen2.5-7b above -- config.json says 32768, no rope_scaling; + # 131072 is the model card's YaRN-extended figure. max_context_length: 131072 attention_type: "gqa" docs_url: "https://huggingface.co/Qwen/Qwen2.5-72B-Instruct" diff --git a/tests/test_architectures.py b/tests/test_architectures.py index 756a8b3..30af9d0 100644 --- a/tests/test_architectures.py +++ b/tests/test_architectures.py @@ -17,10 +17,12 @@ def test_load_architectures_returns_dict_of_model_architecture(): assert all(isinstance(a, ModelArchitecture) for a in architectures.values()) -def test_exactly_thirteen_entries(): +def test_exactly_fourteen_entries(): # 13 rows in PLAN.md's table (Llama 3.1 70B and 3.3 70B are separate - # entries with identical numbers, despite the "~12 models" decision). - assert len(list_architectures()) == 13 + # entries with identical numbers, despite the "~12 models" decision), + # plus meta:llama-3-8b (Phase 14: every self-hosted model in + # default_models.yaml must have an architecture entry). + assert len(list_architectures()) == 14 def test_all_ids_unique(): From c7a82927166aad634ffea94a9b3f93160a23edf5 Mon Sep 17 00:00:00 2001 From: Phantom-VK Date: Wed, 26 Aug 2026 11:34:34 +0530 Subject: [PATCH 2/6] feat(costing): support context-tiered pricing in the model registry Chose the recommended option in GUI_REBUILD/14-REGISTRY-DATA.md over the flat-schema alternative: OpenAI and Google both now charge roughly double above a prompt-size threshold, which is exactly the regime this app exists for -- a flat rate would understate a 600k-token document by about half, confidently. ModelInfo gains long_context_threshold/long_context_input_price_per_million/ long_context_output_price_per_million/pricing_note, all optional so every flat-priced entry is untouched. costing.py's input_cost/output_cost pick the tier from the prompt (input) token count, not the completion size -- output_cost takes an explicit prompt_token_count for this since the two are pried apart at every call site that has both. compare.py's call site updated to pass it; portfolio.py's use of total_cost already threads it through correctly. frontend/src/lib/costing.ts mirrors the same logic (the sanctioned core/ duplication for the Calculator's bridge-free recompute), with boundary tests in both test_costing.py and costing.test.ts at threshold-1/threshold/threshold+1. --- frontend/src/lib/costing.test.ts | 56 ++++++++++++++++++++++- frontend/src/lib/costing.ts | 44 +++++++++++++++--- frontend/src/lib/types.ts | 4 ++ frontend/src/views/Calculator/index.tsx | 5 +- frontend/src/views/Compare/sort.test.ts | 4 ++ src/norefund/core/compare.py | 4 +- src/norefund/core/costing.py | 49 +++++++++++++++++--- src/norefund/core/models_registry.py | 6 +++ tests/test_costing.py | 61 +++++++++++++++++++++++++ 9 files changed, 216 insertions(+), 17 deletions(-) diff --git a/frontend/src/lib/costing.test.ts b/frontend/src/lib/costing.test.ts index ce837f2..22d8785 100644 --- a/frontend/src/lib/costing.test.ts +++ b/frontend/src/lib/costing.test.ts @@ -10,7 +10,22 @@ import { } from "./costing"; // Mirrors tests/test_costing.py's _MODEL. -const MODEL = { input_price_per_million: 2.0, output_price_per_million: 8.0 }; +const MODEL = { + input_price_per_million: 2.0, + output_price_per_million: 8.0, + long_context_threshold: null, + long_context_input_price_per_million: null, + long_context_output_price_per_million: null, +}; + +// Mirrors tests/test_costing.py's _TIERED_MODEL. +const TIERED_MODEL = { + input_price_per_million: 2.0, + output_price_per_million: 8.0, + long_context_threshold: 200_000, + long_context_input_price_per_million: 4.0, + long_context_output_price_per_million: 16.0, +}; describe("contextUsagePct", () => { it("is null when the window is zero", () => { @@ -64,7 +79,13 @@ describe("inputCost / outputCost / totalCost", () => { expect(outputCost(1_000_000, MODEL)).toBeCloseTo(8.0); }); it("is zero for a free model", () => { - const free = { input_price_per_million: 0, output_price_per_million: 0 }; + const free = { + input_price_per_million: 0, + output_price_per_million: 0, + long_context_threshold: null, + long_context_input_price_per_million: null, + long_context_output_price_per_million: null, + }; expect(inputCost(100_000, free)).toBe(0); }); it("sums input and output", () => { @@ -72,6 +93,37 @@ describe("inputCost / outputCost / totalCost", () => { }); }); +describe("context-tiered pricing", () => { + it("uses the short rate exactly at the threshold", () => { + expect(inputCost(200_000, TIERED_MODEL)).toBeCloseTo((200_000 / 1_000_000) * 2.0); + }); + it("uses the short rate just below the threshold", () => { + expect(inputCost(199_999, TIERED_MODEL)).toBeCloseTo((199_999 / 1_000_000) * 2.0); + }); + it("uses the long rate just above the threshold", () => { + expect(inputCost(200_001, TIERED_MODEL)).toBeCloseTo((200_001 / 1_000_000) * 4.0); + }); + it("tiers output cost by prompt size, not output size", () => { + const cost = outputCost(500_000, TIERED_MODEL, 1_000); + expect(cost).toBeCloseTo((500_000 / 1_000_000) * 8.0); + }); + it("uses the long output rate when the prompt crosses the threshold", () => { + const cost = outputCost(1_000, TIERED_MODEL, 200_001); + expect(cost).toBeCloseTo((1_000 / 1_000_000) * 16.0); + }); + it("defaults the prompt size to the output's own count when omitted", () => { + expect(outputCost(200_001, TIERED_MODEL)).toBeCloseTo((200_001 / 1_000_000) * 16.0); + }); + it("totalCost tiers output by the input size", () => { + const cost = totalCost(300_000, 1_000, TIERED_MODEL); + const expected = (300_000 / 1_000_000) * 4.0 + (1_000 / 1_000_000) * 16.0; + expect(cost).toBeCloseTo(expected); + }); + it("is a no-op for a model with no tier fields", () => { + expect(inputCost(10_000_000, MODEL)).toBeCloseTo((10_000_000 / 1_000_000) * 2.0); + }); +}); + describe("convertCurrency", () => { it("applies the rate for the target currency", () => { const rates = { base: "USD", rates: { EUR: 0.5 }, fetched_at: null }; diff --git a/frontend/src/lib/costing.ts b/frontend/src/lib/costing.ts index 0e38519..80df103 100644 --- a/frontend/src/lib/costing.ts +++ b/frontend/src/lib/costing.ts @@ -28,26 +28,58 @@ export function minChunks(tokens: number, window: number): number { return Math.ceil(tokens / usable); } +type TieredPricingFields = Pick< + ModelInfo, + | "long_context_threshold" + | "long_context_input_price_per_million" + | "long_context_output_price_per_million" +>; + +/** Whether promptTokens crosses this model's long-context tier -- decided by + * the prompt (input) size, per every provider that offers one, not by the + * completion size even when pricing output. */ +function longContextActive(promptTokens: number, model: TieredPricingFields): boolean { + return ( + model.long_context_threshold !== null && promptTokens > model.long_context_threshold + ); +} + export function inputCost( tokens: number, - model: Pick, + model: Pick & TieredPricingFields, ): number { - return (tokens / 1_000_000) * model.input_price_per_million; + const rate = + longContextActive(tokens, model) && model.long_context_input_price_per_million !== null + ? model.long_context_input_price_per_million + : model.input_price_per_million; + return (tokens / 1_000_000) * rate; } +/** promptTokens is the input size that decides which price tier applies. + * Defaults to tokens itself when omitted, which is only correct for + * flat-priced models (long_context_threshold null) -- callers pricing a + * tiered model must pass the real prompt size. */ export function outputCost( tokens: number, - model: Pick, + model: Pick & TieredPricingFields, + promptTokens?: number, ): number { - return (tokens / 1_000_000) * model.output_price_per_million; + const promptSize = promptTokens ?? tokens; + const rate = + longContextActive(promptSize, model) && + model.long_context_output_price_per_million !== null + ? model.long_context_output_price_per_million + : model.output_price_per_million; + return (tokens / 1_000_000) * rate; } export function totalCost( inputTokens: number, outputTokens: number, - model: Pick, + model: Pick & + TieredPricingFields, ): number { - return inputCost(inputTokens, model) + outputCost(outputTokens, model); + return inputCost(inputTokens, model) + outputCost(outputTokens, model, inputTokens); } /** Convert a USD amount into toCurrency using the given rates -- mirrors diff --git a/frontend/src/lib/types.ts b/frontend/src/lib/types.ts index 4517c68..4385b22 100644 --- a/frontend/src/lib/types.ts +++ b/frontend/src/lib/types.ts @@ -16,6 +16,10 @@ export interface ModelInfo { currency: string; docs_url: string | null; tokenizer_is_approximate: boolean; + long_context_threshold: number | null; + long_context_input_price_per_million: number | null; + long_context_output_price_per_million: number | null; + pricing_note: string | null; } export interface AnalysisResult { diff --git a/frontend/src/views/Calculator/index.tsx b/frontend/src/views/Calculator/index.tsx index b7d0e28..b7b4b19 100644 --- a/frontend/src/views/Calculator/index.tsx +++ b/frontend/src/views/Calculator/index.tsx @@ -50,7 +50,10 @@ export default function Calculator() { const pct = inputTokens === null ? null : contextUsagePct(inputTokens, model.context_window); const fits = inputTokens === null ? null : fitsInContext(inputTokens, model.context_window); const inCost = inputTokens === null ? null : inputCost(inputTokens, model); - const outCost = outputTokens === null ? null : outputCost(outputTokens, model); + const outCost = + outputTokens === null + ? null + : outputCost(outputTokens, model, inputTokens ?? undefined); const total = inCost === null || outCost === null ? null : inCost + outCost; return ( diff --git a/frontend/src/views/Compare/sort.test.ts b/frontend/src/views/Compare/sort.test.ts index e431982..dcba614 100644 --- a/frontend/src/views/Compare/sort.test.ts +++ b/frontend/src/views/Compare/sort.test.ts @@ -15,6 +15,10 @@ function model(id: string): ModelInfo { currency: "USD", docs_url: null, tokenizer_is_approximate: false, + long_context_threshold: null, + long_context_input_price_per_million: null, + long_context_output_price_per_million: null, + pricing_note: null, }; } diff --git a/src/norefund/core/compare.py b/src/norefund/core/compare.py index 1eeea59..97350fb 100644 --- a/src/norefund/core/compare.py +++ b/src/norefund/core/compare.py @@ -83,7 +83,9 @@ def _build_comparison( min_chunks_needed=min_chunks(token_count, model.context_window), output_tokens=output_tokens, input_cost=compute_input_cost(token_count, model), - output_cost=compute_output_cost(output_tokens, model), + output_cost=compute_output_cost( + output_tokens, model, prompt_token_count=token_count + ), total_cost=compute_total_cost(token_count, output_tokens, model), tokenizer_is_approximate=model.tokenizer_is_approximate, error=None, diff --git a/src/norefund/core/costing.py b/src/norefund/core/costing.py index d421bae..92836a4 100644 --- a/src/norefund/core/costing.py +++ b/src/norefund/core/costing.py @@ -35,15 +35,50 @@ def min_chunks(token_count: int, context_window: int) -> int: return math.ceil(token_count / usable) -def input_cost(token_count: int, model: ModelInfo) -> float: - """USD cost for processing token_count input tokens.""" - return (token_count / 1_000_000) * model.input_price_per_million +def _long_context_active(prompt_token_count: int, model: ModelInfo) -> bool: + """Whether prompt_token_count crosses this model's long-context tier. + + The tier is decided by the prompt (input) size, per every provider that + offers one -- not by the completion size, even when pricing output. + """ + return ( + model.long_context_threshold is not None + and prompt_token_count > model.long_context_threshold + ) -def output_cost(token_count: int, model: ModelInfo) -> float: - """USD cost for token_count output tokens.""" - return (token_count / 1_000_000) * model.output_price_per_million +def input_cost(token_count: int, model: ModelInfo) -> float: + """USD cost for processing token_count input/prompt tokens.""" + rate = model.input_price_per_million + if ( + _long_context_active(token_count, model) + and model.long_context_input_price_per_million is not None + ): + rate = model.long_context_input_price_per_million + return (token_count / 1_000_000) * rate + + +def output_cost( + token_count: int, model: ModelInfo, *, prompt_token_count: int | None = None +) -> float: + """USD cost for token_count output tokens. + + prompt_token_count is the input/prompt size that decides which price + tier applies. Defaults to token_count itself when omitted, which is + only correct for flat-priced models (long_context_threshold is None) -- + callers pricing a tiered model must pass the real prompt size. + """ + prompt_size = token_count if prompt_token_count is None else prompt_token_count + rate = model.output_price_per_million + if ( + _long_context_active(prompt_size, model) + and model.long_context_output_price_per_million is not None + ): + rate = model.long_context_output_price_per_million + return (token_count / 1_000_000) * rate def total_cost(input_tokens: int, output_tokens: int, model: ModelInfo) -> float: - return input_cost(input_tokens, model) + output_cost(output_tokens, model) + return input_cost(input_tokens, model) + output_cost( + output_tokens, model, prompt_token_count=input_tokens + ) diff --git a/src/norefund/core/models_registry.py b/src/norefund/core/models_registry.py index 5d96f95..767e9ab 100644 --- a/src/norefund/core/models_registry.py +++ b/src/norefund/core/models_registry.py @@ -27,6 +27,12 @@ class ModelInfo: currency: str = "USD" docs_url: str | None = None # Optional link to provider pricing/docs page tokenizer_is_approximate: bool = False # True when no real local tokenizer exists + # Context-tiered pricing (OpenAI, Google): prompts above this many tokens + # bill at the long_context_* rate instead. None means flat pricing. + long_context_threshold: int | None = None + long_context_input_price_per_million: float | None = None + long_context_output_price_per_million: float | None = None + pricing_note: str | None = None # e.g. DeepSeek's off-peak halving def load_models(path: Path = _DEFAULT_REGISTRY_PATH) -> dict[str, ModelInfo]: diff --git a/tests/test_costing.py b/tests/test_costing.py index 3e54887..a3a4938 100644 --- a/tests/test_costing.py +++ b/tests/test_costing.py @@ -84,3 +84,64 @@ def test_output_cost_calculation(): def test_total_cost_sum(): cost = total_cost(1_000_000, 1_000_000, _MODEL) assert cost == 10.0 # $2.0 input + $8.0 output + + +_TIERED_MODEL = ModelInfo( + id="test:tiered-model", + display_name="Test Tiered Model", + provider="Test", + tokenizer_backend="tiktoken", + tokenizer_name="cl100k_base", + context_window=1_000_000, + input_price_per_million=2.0, + output_price_per_million=8.0, + long_context_threshold=200_000, + long_context_input_price_per_million=4.0, + long_context_output_price_per_million=16.0, +) + + +def test_input_cost_at_threshold_uses_short_rate(): + # Boundary is "above the threshold", not "at or above" -- exactly + # long_context_threshold tokens still bills at the short-context rate. + assert input_cost(200_000, _TIERED_MODEL) == (200_000 / 1_000_000) * 2.0 + + +def test_input_cost_below_threshold_uses_short_rate(): + assert input_cost(199_999, _TIERED_MODEL) == (199_999 / 1_000_000) * 2.0 + + +def test_input_cost_above_threshold_uses_long_rate(): + assert input_cost(200_001, _TIERED_MODEL) == (200_001 / 1_000_000) * 4.0 + + +def test_output_cost_tier_follows_prompt_size_not_output_size(): + # A short prompt with a huge completion stays on the short-context + # output rate -- the tier is decided by the prompt, not the output. + cost = output_cost(500_000, _TIERED_MODEL, prompt_token_count=1_000) + assert cost == (500_000 / 1_000_000) * 8.0 + + +def test_output_cost_above_threshold_uses_long_rate(): + cost = output_cost(1_000, _TIERED_MODEL, prompt_token_count=200_001) + assert cost == (1_000 / 1_000_000) * 16.0 + + +def test_output_cost_defaults_prompt_size_to_own_token_count(): + # No prompt_token_count given: falls back to the output's own count. + # Only correct for flat models, but must not crash for a tiered one. + cost = output_cost(200_001, _TIERED_MODEL) + assert cost == (200_001 / 1_000_000) * 16.0 + + +def test_total_cost_tiers_output_by_input_size(): + # 300k input (above threshold) + 1k output must bill output at the + # long-context rate, driven by the input size, not the output size. + cost = total_cost(300_000, 1_000, _TIERED_MODEL) + expected = (300_000 / 1_000_000) * 4.0 + (1_000 / 1_000_000) * 16.0 + assert cost == expected + + +def test_flat_model_ignores_missing_tier_fields(): + # _MODEL has no long_context_threshold -- tiering must be a no-op. + assert input_cost(10_000_000, _MODEL) == (10_000_000 / 1_000_000) * 2.0 From c0a06459382045485176dc49df6f61472fa3b00a Mon Sep 17 00:00:00 2001 From: Phantom-VK Date: Wed, 26 Aug 2026 11:43:47 +0530 Subject: [PATCH 3/6] fix(config): refresh the priced model roster against current provider pricing Every priced entry was live-verified against each provider's actual pricing page on 2026-08-26 -- not copied from GUI_REBUILD/14-REGISTRY-DATA.md, whose own reference figures (recorded 2026-08-22) had already drifted: Gemini 2.0 Flash, which the doc called "unchanged, correct," was shut down 2026-06-01. - OpenAI: gpt-4o/gpt-4o-mini/gpt-4.1 replaced with the current gpt-5.6-sol/ terra/luna lineup. Confirmed via developers.openai.com/api/docs/pricing and each model's own page: 1,050,000 context window, tiered pricing above 272,000 input tokens (2x input, 1.5x output). Tokenizer stays "gpt-4o" (o200k_base) -- confirmed still current for the 5.6 family. - Anthropic: claude-3-5-sonnet/claude-3-haiku (retired) replaced with claude-sonnet-5 ($2/$10, 1M context) and claude-haiku-4.5 ($1/$5, 200K context), both flat-priced -- Anthropic's own pricing page states the full 1M window bills at standard rates with no long-context tier. - Google: gemini-2.0-flash/gemini-1.5-pro (both gone) replaced with gemini-3.5-flash (flat $1.50/$9.00) and gemini-3.1-pro-preview (tiered above 200,000 input tokens: $2/$12 to $4/$18), both 1,048,576 context. - DeepSeek: deepseek-v3 (delisted) replaced with deepseek-v4-flash. Storing the peak/cache-miss rate ($0.44/$1.32) since this app's one-off analyses never hit a warm cache; off-peak halving noted via pricing_note. Corrected an error in the phase doc itself, which had the peak/off-peak hours backwards. Added ModelInfo.pricing_verified_on (ISO date), surfaced in both the Python dataclass and its TypeScript mirror, with a dataclass/interface parity test that caught the TS side being out of sync. Updated every test that referenced a now-removed model id (openai:gpt-4o, anthropic:claude-3-5-sonnet, etc.) to the new ids; tests that construct their own synthetic ModelInfo fixtures (test_compare.py, test_resources.py) were unaffected since they never read the real registry file. --- frontend/src/lib/types.ts | 1 + frontend/src/views/Compare/sort.test.ts | 1 + src/norefund/config/default_models.yaml | 127 +++++++++++++++--------- src/norefund/core/models_registry.py | 1 + tests/test_models_registry.py | 22 ++-- tests/test_service.py | 22 ++-- tests/test_tokenization.py | 2 +- 7 files changed, 106 insertions(+), 70 deletions(-) diff --git a/frontend/src/lib/types.ts b/frontend/src/lib/types.ts index 4385b22..0a18448 100644 --- a/frontend/src/lib/types.ts +++ b/frontend/src/lib/types.ts @@ -20,6 +20,7 @@ export interface ModelInfo { long_context_input_price_per_million: number | null; long_context_output_price_per_million: number | null; pricing_note: string | null; + pricing_verified_on: string | null; } export interface AnalysisResult { diff --git a/frontend/src/views/Compare/sort.test.ts b/frontend/src/views/Compare/sort.test.ts index dcba614..e6eb5bc 100644 --- a/frontend/src/views/Compare/sort.test.ts +++ b/frontend/src/views/Compare/sort.test.ts @@ -19,6 +19,7 @@ function model(id: string): ModelInfo { long_context_input_price_per_million: null, long_context_output_price_per_million: null, pricing_note: null, + pricing_verified_on: null, }; } diff --git a/src/norefund/config/default_models.yaml b/src/norefund/config/default_models.yaml index 830b7b4..2da312c 100644 --- a/src/norefund/config/default_models.yaml +++ b/src/norefund/config/default_models.yaml @@ -1,94 +1,127 @@ -- id: openai:gpt-4o - display_name: "GPT-4o" +# Priced entries verified live against each provider's pricing page on +# 2026-08-26 (see pricing_verified_on on each entry) -- not copied from +# GUI_REBUILD/14-REGISTRY-DATA.md's own 2026-08-22 figures, which had +# already drifted by the time this file was edited (Gemini 2.0 Flash, one +# of the doc's "unchanged, correct" entries, had been shut down 2026-06-01). + +- id: openai:gpt-5.6-sol + display_name: "GPT-5.6 Sol" provider: "OpenAI" tokenizer_backend: "tiktoken" - tokenizer_name: "gpt-4o" - context_window: 128000 - input_price_per_million: 2.50 - output_price_per_million: 10.00 + tokenizer_name: "gpt-4o" # GPT-5.6 family shares gpt-4o's o200k_base encoding — this is exact, not an approximation + context_window: 1050000 + input_price_per_million: 4.00 + output_price_per_million: 20.00 + long_context_threshold: 272000 # >272k input tokens: 2x input, 1.5x output, for the full request + long_context_input_price_per_million: 8.00 + long_context_output_price_per_million: 30.00 currency: "USD" - docs_url: "https://openai.com/api/pricing/" + docs_url: "https://developers.openai.com/api/docs/pricing" + pricing_verified_on: "2026-08-26" -- id: openai:gpt-4o-mini - display_name: "GPT-4o Mini" +- id: openai:gpt-5.6-terra + display_name: "GPT-5.6 Terra" provider: "OpenAI" tokenizer_backend: "tiktoken" - tokenizer_name: "gpt-4o-mini" - context_window: 128000 - input_price_per_million: 0.15 - output_price_per_million: 0.60 + tokenizer_name: "gpt-4o" + context_window: 1050000 + input_price_per_million: 2.00 + output_price_per_million: 12.00 + long_context_threshold: 272000 + long_context_input_price_per_million: 4.00 + long_context_output_price_per_million: 18.00 currency: "USD" - docs_url: "https://openai.com/api/pricing/" + docs_url: "https://developers.openai.com/api/docs/pricing" + pricing_verified_on: "2026-08-26" -- id: openai:gpt-4.1 - display_name: "GPT-4.1" +- id: openai:gpt-5.6-luna + display_name: "GPT-5.6 Luna" provider: "OpenAI" tokenizer_backend: "tiktoken" - tokenizer_name: "gpt-4o" # GPT-4.1 shares gpt-4o's o200k_base encoding — this is exact, not an approximation - context_window: 1047576 - input_price_per_million: 2.00 - output_price_per_million: 8.00 + tokenizer_name: "gpt-4o" + context_window: 1050000 + input_price_per_million: 0.20 + output_price_per_million: 1.20 + long_context_threshold: 272000 + long_context_input_price_per_million: 0.40 + long_context_output_price_per_million: 1.80 currency: "USD" - docs_url: "https://openai.com/api/pricing/" + docs_url: "https://developers.openai.com/api/docs/pricing" + pricing_verified_on: "2026-08-26" -- id: anthropic:claude-3-5-sonnet - display_name: "Claude 3.5 Sonnet" +- id: anthropic:claude-sonnet-5 + display_name: "Claude Sonnet 5" provider: "Anthropic" tokenizer_backend: "tiktoken" tokenizer_name: "cl100k_base" - context_window: 200000 - input_price_per_million: 3.00 - output_price_per_million: 15.00 + context_window: 1000000 + input_price_per_million: 2.00 + output_price_per_million: 10.00 currency: "USD" - docs_url: "https://www.anthropic.com/pricing#anthropic-api" + docs_url: "https://platform.claude.com/docs/en/about-claude/pricing" tokenizer_is_approximate: true # Anthropic does not publish a local tokenizer + pricing_verified_on: "2026-08-26" -- id: anthropic:claude-3-haiku - display_name: "Claude 3 Haiku" +- id: anthropic:claude-haiku-4.5 + display_name: "Claude Haiku 4.5" provider: "Anthropic" tokenizer_backend: "tiktoken" tokenizer_name: "cl100k_base" context_window: 200000 - input_price_per_million: 0.25 - output_price_per_million: 1.25 + input_price_per_million: 1.00 + output_price_per_million: 5.00 currency: "USD" - docs_url: "https://www.anthropic.com/pricing#anthropic-api" + docs_url: "https://platform.claude.com/docs/en/about-claude/pricing" tokenizer_is_approximate: true # Anthropic does not publish a local tokenizer + pricing_verified_on: "2026-08-26" -- id: google:gemini-2.0-flash - display_name: "Gemini 2.0 Flash" +- id: google:gemini-3.5-flash + display_name: "Gemini 3.5 Flash" provider: "Google" tokenizer_backend: "tiktoken" tokenizer_name: "cl100k_base" context_window: 1048576 - input_price_per_million: 0.10 - output_price_per_million: 0.40 + input_price_per_million: 1.50 + output_price_per_million: 9.00 currency: "USD" docs_url: "https://ai.google.dev/gemini-api/docs/pricing" tokenizer_is_approximate: true # Google does not publish a local Gemini tokenizer + pricing_verified_on: "2026-08-26" -- id: google:gemini-1.5-pro - display_name: "Gemini 1.5 Pro" +- id: google:gemini-3.1-pro-preview + display_name: "Gemini 3.1 Pro Preview" provider: "Google" tokenizer_backend: "tiktoken" tokenizer_name: "cl100k_base" - context_window: 2000000 - input_price_per_million: 3.50 - output_price_per_million: 10.50 + context_window: 1048576 + input_price_per_million: 2.00 + output_price_per_million: 12.00 + long_context_threshold: 200000 # >200k input tokens: input and output both roughly double + long_context_input_price_per_million: 4.00 + long_context_output_price_per_million: 18.00 currency: "USD" docs_url: "https://ai.google.dev/gemini-api/docs/pricing" tokenizer_is_approximate: true # Google does not publish a local Gemini tokenizer + pricing_verified_on: "2026-08-26" -- id: deepseek:deepseek-v3 - display_name: "DeepSeek V3" +- id: deepseek:deepseek-v4-flash + display_name: "DeepSeek V4 Flash" provider: "DeepSeek" tokenizer_backend: "hf" - tokenizer_name: "deepseek-ai/DeepSeek-V3" - context_window: 163840 # config.json's max_position_embeddings; matches model_architectures.yaml - input_price_per_million: 0.27 - output_price_per_million: 1.10 + tokenizer_name: "deepseek-ai/DeepSeek-V4-Flash" + context_window: 1048576 # config.json's max_position_embeddings + # Storing the peak / cache-miss rate: this app's one-off document + # analyses never hit a warm cache, and peak is the provider's own + # "assume the worst case" hours. Off-peak (all hours except + # 01:00-04:00 and 06:00-10:00 UTC Mon-Fri -- these ARE peak, not + # off-peak; the source doc that preceded this file had that backwards) + # halves both rates to $0.22/$0.66. + input_price_per_million: 0.44 + output_price_per_million: 1.32 currency: "USD" docs_url: "https://api-docs.deepseek.com/quick_start/pricing" + pricing_note: "Off-peak (all hours except 01:00-04:00 and 06:00-10:00 UTC Mon-Fri) halves both rates to $0.22/$0.66." + pricing_verified_on: "2026-08-26" - id: meta:llama-3-8b display_name: "Llama 3 8B (self-hosted)" diff --git a/src/norefund/core/models_registry.py b/src/norefund/core/models_registry.py index 767e9ab..0fbb9de 100644 --- a/src/norefund/core/models_registry.py +++ b/src/norefund/core/models_registry.py @@ -33,6 +33,7 @@ class ModelInfo: long_context_input_price_per_million: float | None = None long_context_output_price_per_million: float | None = None pricing_note: str | None = None # e.g. DeepSeek's off-peak halving + pricing_verified_on: str | None = None # ISO date a human read the pricing page def load_models(path: Path = _DEFAULT_REGISTRY_PATH) -> dict[str, ModelInfo]: diff --git a/tests/test_models_registry.py b/tests/test_models_registry.py index b9f30da..774b766 100644 --- a/tests/test_models_registry.py +++ b/tests/test_models_registry.py @@ -14,15 +14,15 @@ def test_all_models_are_model_info(): assert isinstance(model, ModelInfo) -def test_gpt4o_present(): +def test_gpt_sol_present(): models = load_models() - assert "openai:gpt-4o" in models + assert "openai:gpt-5.6-sol" in models -def test_gpt4o_fields(): - model = get_model("openai:gpt-4o") - assert model.context_window == 128_000 - assert model.input_price_per_million == 2.50 +def test_gpt_sol_fields(): + model = get_model("openai:gpt-5.6-sol") + assert model.context_window == 1_050_000 + assert model.input_price_per_million == 4.00 assert model.tokenizer_backend == "tiktoken" @@ -67,10 +67,10 @@ def test_models_with_no_public_tokenizer_are_flagged_approximate(): """Claude and Gemini have no publicly cacheable local tokenizer, so their counts come from a tiktoken approximation and must say so.""" approximate_ids = [ - "anthropic:claude-3-5-sonnet", - "anthropic:claude-3-haiku", - "google:gemini-2.0-flash", - "google:gemini-1.5-pro", + "anthropic:claude-sonnet-5", + "anthropic:claude-haiku-4.5", + "google:gemini-3.5-flash", + "google:gemini-3.1-pro-preview", ] for model_id in approximate_ids: assert get_model(model_id).tokenizer_is_approximate is True @@ -78,7 +78,7 @@ def test_models_with_no_public_tokenizer_are_flagged_approximate(): def test_models_with_real_hf_tokenizer_are_not_flagged_approximate(): real_tokenizer_ids = [ - "deepseek:deepseek-v3", + "deepseek:deepseek-v4-flash", "meta:llama-3-8b", "mistral:mistral-7b", ] diff --git a/tests/test_service.py b/tests/test_service.py index 665a85c..66ba3ed 100644 --- a/tests/test_service.py +++ b/tests/test_service.py @@ -25,55 +25,55 @@ def sample_folder(tmp_path: Path) -> Path: def test_analyze_file_returns_result(sample_txt: Path): - result = analyze_file(sample_txt, "openai:gpt-4o") + result = analyze_file(sample_txt, "openai:gpt-5.6-sol") assert isinstance(result, AnalysisResult) def test_analyze_file_token_count_positive(sample_txt: Path): _skip_unless_cached("o200k_base") - result = analyze_file(sample_txt, "openai:gpt-4o") + result = analyze_file(sample_txt, "openai:gpt-5.6-sol") assert result.token_count > 0 def test_analyze_file_correct_model(sample_txt: Path): - result = analyze_file(sample_txt, "openai:gpt-4o") - assert result.model_id == "openai:gpt-4o" + result = analyze_file(sample_txt, "openai:gpt-5.6-sol") + assert result.model_id == "openai:gpt-5.6-sol" def test_analyze_file_fits_small_doc(sample_txt: Path): # A tiny txt file must fit in any model's context window _skip_unless_cached("o200k_base") - result = analyze_file(sample_txt, "openai:gpt-4o") + result = analyze_file(sample_txt, "openai:gpt-5.6-sol") assert result.fits_in_context is True assert result.min_chunks_needed == 1 def test_analyze_file_context_usage_under_100(sample_txt: Path): _skip_unless_cached("o200k_base") - result = analyze_file(sample_txt, "openai:gpt-4o") + result = analyze_file(sample_txt, "openai:gpt-5.6-sol") assert result.context_usage_pct < 100 def test_analyze_file_char_and_word_count(sample_txt: Path): _skip_unless_cached("o200k_base") - result = analyze_file(sample_txt, "openai:gpt-4o") + result = analyze_file(sample_txt, "openai:gpt-5.6-sol") assert result.char_count > 0 assert result.word_count > 0 def test_analyze_folder_returns_list(sample_folder: Path): - results = analyze_folder(sample_folder, "openai:gpt-4o") + results = analyze_folder(sample_folder, "openai:gpt-5.6-sol") assert isinstance(results, list) def test_analyze_folder_ignores_unsupported(sample_folder: Path): - results = analyze_folder(sample_folder, "openai:gpt-4o") + results = analyze_folder(sample_folder, "openai:gpt-5.6-sol") # Only .txt and .md should be picked up, not .exe assert len(results) == 2 def test_analyze_folder_all_results_valid(sample_folder: Path): - results = analyze_folder(sample_folder, "openai:gpt-4o") + results = analyze_folder(sample_folder, "openai:gpt-5.6-sol") assert all(isinstance(r, AnalysisResult) for r in results) @@ -82,6 +82,6 @@ def test_analyze_folder_does_not_descend_into_subfolders(sample_folder: Path): nested.mkdir() (nested / "c.txt").write_text("File C content, inside a subfolder") - results = analyze_folder(sample_folder, "openai:gpt-4o") + results = analyze_folder(sample_folder, "openai:gpt-5.6-sol") assert {Path(r.file_path).name for r in results} == {"a.txt", "b.md"} diff --git a/tests/test_tokenization.py b/tests/test_tokenization.py index b65869f..2bf770d 100644 --- a/tests/test_tokenization.py +++ b/tests/test_tokenization.py @@ -53,7 +53,7 @@ def test_tiktoken_fallback_unknown_model(): def test_get_tokenizer_returns_tiktoken_for_gpt4o(): _skip_unless_cached("o200k_base") - model = get_model("openai:gpt-4o") + model = get_model("openai:gpt-5.6-sol") backend = get_tokenizer(model) assert isinstance(backend, TikTokenBackend) From 5ac2d84e8d5a25f809ec36d82ea2a6293e565e31 Mon Sep 17 00:00:00 2001 From: Phantom-VK Date: Wed, 26 Aug 2026 11:48:40 +0530 Subject: [PATCH 4/6] fix(resources): make every registry tokenizer downloadable without a licence gate Chose Option A from GUI_REBUILD/14-REGISTRY-DATA.md: point tokenizer_name at an ungated mirror for all 7 licence-gated repos (every meta-llama/* entry, both google/gemma-2-*), rather than surfacing the gate in the UI. Every self-hosted model's tokenizer now downloads with no HF token and no licence click. Verified identity without an HF token: the official repos' file content isn't fetchable unauthenticated (confirmed: all 7 report gated=manual via the HF API), but the HF tree API's git blob oid for tokenizer.json is not gate-restricted. An identical oid between the official repo and its mirror proves the LFS pointer -- and therefore the actual tokenizer content it references -- is byte-identical, without downloading either file. All 7 pairs matched: - meta-llama/Meta-Llama-3-8B <-> NousResearch/Meta-Llama-3-8B - meta-llama/Llama-3.1-{8B,70B,405B}-Instruct <-> NousResearch (8B, 70B) / unsloth (405B, as a 4-bit repo carrying the unmodified tokenizer) -- all three share one oid, confirming one tokenizer across the family - meta-llama/Llama-3.3-70B-Instruct <-> unsloth/Llama-3.3-70B-Instruct - google/gemma-2-{9b,27b}-it <-> unsloth mirrors -- both official entries and both mirrors share one oid across the two sizes Spot-verified 3 of the 7 mirrors with a real unauthenticated hf_hub_download (NousResearch/Meta-Llama-3-8B, unsloth/gemma-2-9b-it, unsloth/Llama-3.3-70B-Instruct) -- all succeeded, sizes matching the tree API exactly. docs_url stays on the official model page in every entry; only tokenizer_name (what actually gets downloaded) moved to the mirror, with a comment recording the pairing and the verification method so the substitution isn't mysterious. resources/probe.py's existing _GATED_HF_PREFIXES notes logic is now dead for these 7 entries specifically (nothing in the registry starts with those prefixes anymore) but left in place as a generic guard for probe_hf(), which takes an arbitrary repo id. --- src/norefund/config/default_models.yaml | 39 ++++++++++++++++++++----- 1 file changed, 32 insertions(+), 7 deletions(-) diff --git a/src/norefund/config/default_models.yaml b/src/norefund/config/default_models.yaml index 2da312c..bc7f188 100644 --- a/src/norefund/config/default_models.yaml +++ b/src/norefund/config/default_models.yaml @@ -127,7 +127,13 @@ display_name: "Llama 3 8B (self-hosted)" provider: "Meta" tokenizer_backend: "hf" - tokenizer_name: "meta-llama/Meta-Llama-3-8B" + # meta-llama/Meta-Llama-3-8B is licence-gated (manual approval); this + # download would fail for any user without an accepted licence + HF + # token. NousResearch/Meta-Llama-3-8B carries a byte-identical + # tokenizer.json (verified via the HF tree API's git blob oid, since + # the gated repo's own content isn't fetchable without auth: both are + # b197f72effb9d5ed16ee0f5663e11e4cfac2ba62). + tokenizer_name: "NousResearch/Meta-Llama-3-8B" context_window: 8192 input_price_per_million: 0.00 output_price_per_million: 0.00 @@ -152,7 +158,10 @@ display_name: "Llama 3.1 8B (self-hosted)" provider: "Meta" tokenizer_backend: "hf" - tokenizer_name: "meta-llama/Llama-3.1-8B-Instruct" + # meta-llama/Llama-3.1-8B-Instruct is licence-gated. NousResearch's + # mirror is byte-identical (git blob oid 5cc5f00a5b203e90a27a3bd60d1ec393b07971e8 + # on both, verified via the HF tree API without needing gated-repo auth). + tokenizer_name: "NousResearch/Meta-Llama-3.1-8B-Instruct" context_window: 131072 input_price_per_million: 0.00 output_price_per_million: 0.00 @@ -163,7 +172,10 @@ display_name: "Llama 3.1 70B (self-hosted)" provider: "Meta" tokenizer_backend: "hf" - tokenizer_name: "meta-llama/Llama-3.1-70B-Instruct" + # meta-llama/Llama-3.1-70B-Instruct is licence-gated. Same tokenizer as + # the 8B/405B Instruct models in this family (identical git blob oid + # 5cc5f00a5b203e90a27a3bd60d1ec393b07971e8), mirrored ungated here. + tokenizer_name: "NousResearch/Meta-Llama-3.1-70B-Instruct" context_window: 131072 input_price_per_million: 0.00 output_price_per_million: 0.00 @@ -174,7 +186,10 @@ display_name: "Llama 3.3 70B (self-hosted)" provider: "Meta" tokenizer_backend: "hf" - tokenizer_name: "meta-llama/Llama-3.3-70B-Instruct" + # meta-llama/Llama-3.3-70B-Instruct is licence-gated. unsloth's mirror + # is byte-identical (git blob oid 1c1d8d5c9024994f1d3b00f9662b8dd89ca13cf2 + # on both). + tokenizer_name: "unsloth/Llama-3.3-70B-Instruct" context_window: 131072 input_price_per_million: 0.00 output_price_per_million: 0.00 @@ -185,7 +200,11 @@ display_name: "Llama 3.1 405B (self-hosted)" provider: "Meta" tokenizer_backend: "hf" - tokenizer_name: "meta-llama/Llama-3.1-405B-Instruct" + # meta-llama/Llama-3.1-405B-Instruct is licence-gated. Same tokenizer as + # the 8B/70B Instruct models in this family (identical git blob oid + # 5cc5f00a5b203e90a27a3bd60d1ec393b07971e8); unsloth's 4-bit mirror + # keeps the original, unmodified tokenizer. + tokenizer_name: "unsloth/Meta-Llama-3.1-405B-Instruct-bnb-4bit" context_window: 131072 input_price_per_million: 0.00 output_price_per_million: 0.00 @@ -251,7 +270,10 @@ display_name: "Gemma 2 9B (self-hosted)" provider: "Google" tokenizer_backend: "hf" - tokenizer_name: "google/gemma-2-9b-it" + # google/gemma-2-9b-it is licence-gated. unsloth's mirror is + # byte-identical (git blob oid af0eac5c0056f83b8f3fcdb79165f8847111c305 + # on both -- same tokenizer as gemma-2-27b-it too). + tokenizer_name: "unsloth/gemma-2-9b-it" context_window: 8192 input_price_per_million: 0.00 output_price_per_million: 0.00 @@ -262,7 +284,10 @@ display_name: "Gemma 2 27B (self-hosted)" provider: "Google" tokenizer_backend: "hf" - tokenizer_name: "google/gemma-2-27b-it" + # google/gemma-2-27b-it is licence-gated. unsloth's mirror is + # byte-identical (git blob oid af0eac5c0056f83b8f3fcdb79165f8847111c305 + # on both -- same tokenizer as gemma-2-9b-it too). + tokenizer_name: "unsloth/gemma-2-27b-it" context_window: 8192 input_price_per_million: 0.00 output_price_per_million: 0.00 From 5b3f73def3df5fcf420958b58ace48aae2119bf6 Mon Sep 17 00:00:00 2001 From: Phantom-VK Date: Wed, 26 Aug 2026 11:54:46 +0530 Subject: [PATCH 5/6] fix(selfhost): account for Gemma 2's sliding-window attention in KV cache sizing Both Gemma 2 configs alternate local sliding-window (4096) and global full attention 1:1 across layers (confirmed against config.json and Google's own Gemma 2 writeup) -- kv_cache_bytes_per_token treated every layer as full-context GQA, overstating KV cache by a real, measurable amount at any context beyond the window. kv_cache_bytes_per_token's per-token-times-context_length signature can't express a cost that isn't linear in context length once a window caps it. Added kv_cache_bytes(architecture, context_length, dtype) alongside it (the clearer of the two options in the phase doc, per its own reasoning) rather than smuggling a context dependency into a per-token name. Both share a new _kv_cache_bytes_per_layer_per_token helper so each rounds once, at the end, instead of compounding error by dividing an already-rounded total by n_layers. ModelArchitecture gains sliding_window/sliding_window_pattern (both default 0 = full attention, so every existing entry is untouched). selfhost.py's two call sites switched from kv_cache_bytes_per_token(...) * context_length to kv_cache_bytes(...) directly. Verified by hand: at 2x the window (8192 tokens), Gemma 2 9B's KV cache is exactly 75% of the naive all-layers-full figure (21 of 42 layers capped to 4096 tokens each) -- a real 25% reduction, not an approximate one. On a 24GB card, comparing current code against the pre-fix all-layers-full math, Gemma 2 9B's memory-bound context ceiling (past its own 8192 architectural max, where KV pressure is what actually binds) rises from roughly 9,728 to 15,616 tokens. New tests: Gemma 2 9B at 2x window vs at-or-under the window, a GQA model with sliding_window=0 (every architecture before this phase) matching its old behavior exactly, and DeepSeek V3's MLA path unchanged. --- frontend/src/lib/types.ts | 2 + src/norefund/config/model_architectures.yaml | 7 ++ src/norefund/core/architectures.py | 2 + src/norefund/core/selfhost.py | 77 ++++++++++++++------ tests/test_selfhost.py | 59 +++++++++++++++ 5 files changed, 125 insertions(+), 22 deletions(-) diff --git a/frontend/src/lib/types.ts b/frontend/src/lib/types.ts index 0a18448..8e1d646 100644 --- a/frontend/src/lib/types.ts +++ b/frontend/src/lib/types.ts @@ -107,6 +107,8 @@ export interface ModelArchitecture { attention_type: string; kv_lora_rank: number; qk_rope_head_dim: number; + sliding_window: number; + sliding_window_pattern: number; docs_url: string | null; } diff --git a/src/norefund/config/model_architectures.yaml b/src/norefund/config/model_architectures.yaml index dcfe470..f071f8e 100644 --- a/src/norefund/config/model_architectures.yaml +++ b/src/norefund/config/model_architectures.yaml @@ -208,6 +208,10 @@ hidden_size: 3584 max_context_length: 8192 attention_type: "gqa" + # Alternates local sliding-window (4096) and global full attention 1:1 + # every other layer -- see core/selfhost.py's kv_cache_bytes. + sliding_window: 4096 + sliding_window_pattern: 2 docs_url: "https://huggingface.co/google/gemma-2-9b-it" - id: google:gemma-2-27b @@ -223,6 +227,9 @@ hidden_size: 4608 max_context_length: 8192 attention_type: "gqa" + # See google:gemma-2-9b above -- same 1:1 local/global alternation. + sliding_window: 4096 + sliding_window_pattern: 2 docs_url: "https://huggingface.co/google/gemma-2-27b-it" - id: deepseek:deepseek-v3 diff --git a/src/norefund/core/architectures.py b/src/norefund/core/architectures.py index 696e158..cde8f2a 100644 --- a/src/norefund/core/architectures.py +++ b/src/norefund/core/architectures.py @@ -40,6 +40,8 @@ class ModelArchitecture: attention_type: AttentionType kv_lora_rank: int = 0 # MLA only (e.g. DeepSeek V3) qk_rope_head_dim: int = 0 # MLA only (e.g. DeepSeek V3) + sliding_window: int = 0 # 0 = full attention on every layer + sliding_window_pattern: int = 0 # every Nth layer is full attention (Gemma 2: 2) docs_url: str | None = None diff --git a/src/norefund/core/selfhost.py b/src/norefund/core/selfhost.py index 2e3c663..d638ca6 100644 --- a/src/norefund/core/selfhost.py +++ b/src/norefund/core/selfhost.py @@ -87,32 +87,67 @@ def weight_bytes(architecture: ModelArchitecture, quantization: str) -> int | No return round(architecture.total_params * bpw / 8) -def kv_cache_bytes_per_token( - architecture: ModelArchitecture, kv_cache_dtype: str = "fp16" -) -> int | None: - """KV-cache memory for one token, one sequence.""" +def _kv_cache_bytes_per_layer_per_token( + architecture: ModelArchitecture, kv_cache_dtype: str +) -> float | None: + """Unrounded KV-cache bytes for one layer, one token, one sequence. + + Shared by kv_cache_bytes_per_token and kv_cache_bytes so both round + only once, at the end, rather than compounding rounding error by + dividing an already-rounded per-token total by n_layers. + """ bytes_per_element = bytes_per_kv_element(kv_cache_dtype) if bytes_per_element is None: return None if architecture.attention_type == "gqa": - return round( - 2 - * architecture.n_layers - * architecture.n_kv_heads - * architecture.head_dim - * bytes_per_element - ) + return 2 * architecture.n_kv_heads * architecture.head_dim * bytes_per_element if architecture.attention_type == "mla": if architecture.kv_lora_rank <= 0: return None - return round( - architecture.n_layers - * (architecture.kv_lora_rank + architecture.qk_rope_head_dim) - * bytes_per_element - ) + return ( + architecture.kv_lora_rank + architecture.qk_rope_head_dim + ) * bytes_per_element return None +def kv_cache_bytes_per_token( + architecture: ModelArchitecture, kv_cache_dtype: str = "fp16" +) -> int | None: + """KV-cache memory for one token, one sequence, as if every layer were + full attention. Ignores sliding-window; see kv_cache_bytes for + context-length-aware sizing that accounts for it.""" + per_layer = _kv_cache_bytes_per_layer_per_token(architecture, kv_cache_dtype) + if per_layer is None: + return None + return round(architecture.n_layers * per_layer) + + +def kv_cache_bytes( + architecture: ModelArchitecture, context_length: int, kv_cache_dtype: str = "fp16" +) -> int | None: + """KV-cache memory for one sequence at context_length. + + Sliding-window KV cost is not linear in context length the way full + attention's is: a windowed layer caches only min(context_length, + sliding_window) tokens, while a full-attention layer caches the whole + context_length. architecture.sliding_window == 0 (every architecture + except Gemma 2 today) means every layer is full attention, which + reduces this to kv_cache_bytes_per_token(...) * context_length. + """ + per_layer = _kv_cache_bytes_per_layer_per_token(architecture, kv_cache_dtype) + if per_layer is None or architecture.n_layers <= 0: + return None + if architecture.sliding_window <= 0 or architecture.sliding_window_pattern <= 0: + return round(architecture.n_layers * per_layer * context_length) + + full_layers = architecture.n_layers // architecture.sliding_window_pattern + windowed_layers = architecture.n_layers - full_layers + windowed_context = min(context_length, architecture.sliding_window) + return round( + per_layer * (full_layers * context_length + windowed_layers * windowed_context) + ) + + def activation_bytes(architecture: ModelArchitecture, context_length: int) -> int: if context_length <= 0: return 0 @@ -135,11 +170,10 @@ def estimate_memory( kv_cache_dtype: str = "fp16", ) -> MemoryEstimate | None: weights = weight_bytes(architecture, quantization) - kv_per_token = kv_cache_bytes_per_token(architecture, kv_cache_dtype) - if weights is None or kv_per_token is None or context_length <= 0: + kv_per_sequence = kv_cache_bytes(architecture, context_length, kv_cache_dtype) + if weights is None or kv_per_sequence is None or context_length <= 0: return None - kv_per_sequence = kv_per_token * context_length kv_total = kv_per_sequence * max(concurrency, 0) activation = activation_bytes(architecture, context_length) overhead = framework_overhead_bytes(hardware) @@ -165,8 +199,8 @@ def max_concurrent_requests( """How many concurrent full-context sequences could fit, independent of the requested concurrency.""" weights = weight_bytes(architecture, quantization) - kv_per_token = kv_cache_bytes_per_token(architecture, kv_cache_dtype) - if weights is None or kv_per_token is None or context_length <= 0: + kv_per_sequence = kv_cache_bytes(architecture, context_length, kv_cache_dtype) + if weights is None or kv_per_sequence is None or context_length <= 0: return None activation = activation_bytes(architecture, context_length) @@ -175,7 +209,6 @@ def max_concurrent_requests( if available_for_kv <= 0: return 0 - kv_per_sequence = kv_per_token * context_length if kv_per_sequence <= 0: return None return available_for_kv // kv_per_sequence diff --git a/tests/test_selfhost.py b/tests/test_selfhost.py index adc4f64..4ae2004 100644 --- a/tests/test_selfhost.py +++ b/tests/test_selfhost.py @@ -14,6 +14,7 @@ estimate_memory, evaluate_fit, framework_overhead_bytes, + kv_cache_bytes, kv_cache_bytes_per_token, max_concurrent_requests, usable_memory_bytes, @@ -100,6 +101,24 @@ attention_type="gqa", ) +_GEMMA_2_9B = ModelArchitecture( + id="google:gemma-2-9b", + display_name="Gemma 2 9B", + family="Gemma 2", + vendor="Google", + total_params=9_240_000_000, + active_params=9_240_000_000, + n_layers=42, + n_attention_heads=16, + n_kv_heads=8, + head_dim=256, + hidden_size=3584, + max_context_length=8192, + attention_type="gqa", + sliding_window=4096, + sliding_window_pattern=2, # alternates local/global 1:1 +) + _DEEPSEEK_V3 = ModelArchitecture( id="deepseek:deepseek-v3", display_name="DeepSeek V3", @@ -284,6 +303,46 @@ def test_llama_70b_kv_cache_uses_kv_heads_not_attention_heads(): assert kv_cache_bytes_per_token(_LLAMA_70B) == 327_680 +# --- Gemma 2 sliding-window KV cache (Task 4, Phase 14) --- + + +def test_gemma2_9b_sliding_window_below_full_figure_at_2x_window(): + # 42 layers alternate 1:1 (21 full, 21 windowed to 4096). At 8192 + # (2x the window), the windowed half only pays for 4096 tokens each, + # giving exactly 75% of the naive all-layers-full figure -- a real + # ~25% reduction, not just "some" reduction. + full_figure = kv_cache_bytes_per_token(_GEMMA_2_9B) * 8192 + windowed = kv_cache_bytes(_GEMMA_2_9B, 8192) + assert windowed == round(full_figure * 0.75) + assert windowed < full_figure + + +def test_gemma2_9b_sliding_window_equals_full_figure_at_or_under_window(): + # At exactly the window size, every layer -- windowed or not -- caches + # the whole context, so this must equal the naive figure exactly. + assert kv_cache_bytes(_GEMMA_2_9B, 4096) == kv_cache_bytes_per_token( + _GEMMA_2_9B + ) * 4096 + + +def test_gqa_model_with_no_sliding_window_matches_per_token_figure(): + # _LLAMA_70B has sliding_window=0 (the default) -- kv_cache_bytes must + # return exactly what multiplying the existing per-token figure by + # context_length already gave, for every architecture written before + # this phase. + context_length = 32_768 + assert kv_cache_bytes(_LLAMA_70B, context_length) == ( + kv_cache_bytes_per_token(_LLAMA_70B) * context_length + ) + + +def test_deepseek_v3_mla_path_unchanged_by_sliding_window_support(): + context_length = 65_536 + assert kv_cache_bytes(_DEEPSEEK_V3, context_length) == ( + kv_cache_bytes_per_token(_DEEPSEEK_V3) * context_length + ) + + # --- Hand-computed scenarios --- From 73164d50f43633b01d0e7fd9d8d67d131f65ed89 Mon Sep 17 00:00:00 2001 From: Phantom-VK Date: Wed, 26 Aug 2026 11:59:12 +0530 Subject: [PATCH 6/6] test(config): add registry cross-consistency checks and price verification dates tests/test_registry_data.py: pure, no network, no fixtures. Checks every architecture id has a matching default_models.yaml entry (and vice versa for self-hosted models), that files sharing an id agree on context window, that every priced entry carries a docs_url and pricing_verified_on, that pricing_verified_on parses as a real date, and that head_dim * n_attention_heads either equals hidden_size or the entry explains why (Gemma 2's head_dim=256 exception). No test asserts a date is recent -- a time-bomb that reddens CI on a quiet Tuesday teaches people to ignore CI. This test caught a real gap the moment it was written: Task 2's roster refresh replaced the priced deepseek:deepseek-v3 entry with deepseek:deepseek-v4-flash, but model_architectures.yaml still has a deepseek:deepseek-v3 entry (it's a real, separately self-hostable open-weight model, not superseded the way its hosted-API pricing is) -- so the Fit Check would have had nothing to look up for it. Added it back as its own zero-priced self-hosted entry, alongside the priced V4-Flash one. Surfaced pricing_verified_on in the Registry view as a single muted line ("Prices verified ") showing the most recent verification date across all priced entries -- not per-row, per the phase doc's own UI guidance. docs/registry-refresh.md: three paragraphs naming the four pricing URLs, the config.json/safetensors-total lookup method, the gated-repo identity-verification technique from Task 3, and the rule that pricing_verified_on only moves when a human has actually reread the source page that day -- including the concrete lesson from this same phase (the reference doc's own 2026-08-22 figures had already drifted by 2026-08-26). --- docs/registry-refresh.md | 36 +++++++ frontend/src/views/Registry/index.tsx | 25 ++++- src/norefund/config/default_models.yaml | 18 ++++ tests/test_registry_data.py | 124 ++++++++++++++++++++++++ 4 files changed, 199 insertions(+), 4 deletions(-) create mode 100644 docs/registry-refresh.md create mode 100644 tests/test_registry_data.py diff --git a/docs/registry-refresh.md b/docs/registry-refresh.md new file mode 100644 index 0000000..eb08512 --- /dev/null +++ b/docs/registry-refresh.md @@ -0,0 +1,36 @@ +# Refreshing the model registry + +`src/norefund/config/default_models.yaml` and `model_architectures.yaml` are +read against primary sources, not updated from memory or from a previous +model's own claims about itself. Pricing comes from each provider's own +pricing page, re-read on the day the file is edited: OpenAI +(`https://developers.openai.com/api/docs/pricing`), Anthropic +(`https://platform.claude.com/docs/en/about-claude/pricing`), Google +(`https://ai.google.dev/gemini-api/docs/pricing`), and DeepSeek +(`https://api-docs.deepseek.com/quick_start/pricing`). Architecture fields +(layer counts, head dimensions, MLA parameters, context windows) come from +the model's published HuggingFace `config.json`, and parameter counts are +cross-checked against the HF API's safetensors total +(`GET /api/models/{repo}?blobs=true`, the `safetensors.total` field) rather +than trusted from a model card's rounded number. + +`pricing_verified_on` on a `ModelInfo` entry moves only when a human (or an +agent acting on a human's behalf) has actually opened the provider's page +that day and read the number off it -- never bumped to "look current" +without doing that, and never copied forward from an older PR or plan +document, which can itself have drifted by the time anyone acts on it (this +happened during the Phase 14 registry refresh: a reference doc's own +pricing figures, recorded four days earlier, were already wrong by the time +they were used). `tests/test_registry_data.py` enforces the structural +invariants this depends on -- every architecture id has a matching registry +entry, the two files agree on shared fields, every priced entry carries a +verification date -- but it cannot check whether a given date's price is +still correct; only re-reading the source page can. + +When a licence-gated HuggingFace repo needs an ungated substitute (see the +comments on the `meta:llama-*` and `google:gemma-2-*` entries), verify +identity via the HF tree API's git blob oid for the file in question +(`GET /api/models/{repo}/tree/main`) rather than trusting a mirror's name -- +an identical oid between the official repo and the mirror proves the file +is byte-identical without needing to authenticate to the gated repo or +download either copy. diff --git a/frontend/src/views/Registry/index.tsx b/frontend/src/views/Registry/index.tsx index 28e8e68..09a2d2f 100644 --- a/frontend/src/views/Registry/index.tsx +++ b/frontend/src/views/Registry/index.tsx @@ -31,16 +31,33 @@ export default function Registry() { () => Array.from(new Set(models.map((m) => m.provider))).sort(), [models], ); + // The most recent date any priced entry was checked against its + // provider's actual page -- not every entry shares one date, so the + // true statement is "at least this recently," not "as of exactly this + // day." See docs/registry-refresh.md for the refresh procedure. + const latestPricingVerification = useMemo(() => { + const dates = models + .map((m) => m.pricing_verified_on) + .filter((d): d is string => d !== null); + return dates.length > 0 ? dates.reduce((a, b) => (a > b ? a : b)) : null; + }, [models]); const visible = activeProvider === "All" ? models : models.filter((m) => m.provider === activeProvider); return (
-

- {models.length} models across {providers.length} providers. Locally stored - pricing data. -

+
+

+ {models.length} models across {providers.length} providers. Locally stored + pricing data. +

+ {latestPricingVerification && ( +

+ Prices verified {latestPricingVerification} +

+ )} +
dict[str, str]: + """Map architecture id -> its raw YAML block (comments included). + + ModelArchitecture strips comments on load, but the head_dim exception + check needs to see them -- re-read the raw file and split on each + top-level "- id:" entry rather than adding a YAML-comment-preserving + dependency for one test. + """ + text = _DEFAULT_ARCHITECTURE_PATH.read_text(encoding="utf-8") + entries = re.split(r"(?=^- id: )", text, flags=re.MULTILINE) + blocks: dict[str, str] = {} + for entry in entries: + match = re.match(r"^- id: (\S+)", entry) + if match: + blocks[match.group(1)] = entry + return blocks + + +def test_every_architecture_id_exists_in_model_registry(): + for arch_id in _architectures: + assert arch_id in _models, ( + f"model_architectures.yaml has '{arch_id}' with no matching " + f"entry in default_models.yaml" + ) + + +def test_every_self_hosted_model_has_an_architecture_entry(): + # Self-hosted == a real (non-tiktoken) tokenizer backend and no price. + # This is exactly the check that would have caught meta:llama-3-8b + # having no architecture entry before Phase 14. + for model in _models.values(): + is_self_hosted = ( + model.tokenizer_backend != "tiktoken" + and model.input_price_per_million == 0 + and model.output_price_per_million == 0 + ) + if is_self_hosted: + assert model.id in _architectures, ( + f"'{model.id}' is self-hosted (backend={model.tokenizer_backend}, " + f"zero price) but has no model_architectures.yaml entry, so it " + f"can never appear in the Fit Check" + ) + + +def test_shared_ids_agree_on_context_window(): + # This is exactly the check that would have caught DeepSeek V3's two + # files disagreeing (128000 vs 163840) before Phase 14. + shared_ids = set(_models) & set(_architectures) + assert shared_ids, "expected at least one id in both registries" + for shared_id in shared_ids: + model = _models[shared_id] + arch = _architectures[shared_id] + assert model.context_window == arch.max_context_length, ( + f"'{shared_id}': default_models.yaml says context_window=" + f"{model.context_window}, model_architectures.yaml says " + f"max_context_length={arch.max_context_length}" + ) + + +def test_every_priced_entry_has_docs_url_and_verification_date(): + priced = [m for m in _models.values() if m.input_price_per_million > 0] + assert priced, "expected at least one priced entry" + for model in priced: + assert model.docs_url is not None, f"'{model.id}' has a price but no docs_url" + assert model.pricing_verified_on is not None, ( + f"'{model.id}' has a price but no pricing_verified_on" + ) + + +def test_every_pricing_verified_on_parses_as_a_date(): + for model in _models.values(): + if model.pricing_verified_on is not None: + # Raises ValueError (failing the test) if it isn't a real + # ISO date -- deliberately not asserting anything about how + # recent it is. + datetime.date.fromisoformat(model.pricing_verified_on) + + +def test_head_dim_times_attention_heads_matches_hidden_size_or_is_explained(): + blocks = _architecture_yaml_blocks() + for arch in _architectures.values(): + if arch.head_dim * arch.n_attention_heads == arch.hidden_size: + continue + block = blocks.get(arch.id, "") + assert "head_dim" in block, ( + f"'{arch.id}': head_dim ({arch.head_dim}) * n_attention_heads " + f"({arch.n_attention_heads}) != hidden_size ({arch.hidden_size}), " + f"and model_architectures.yaml has no comment on this entry " + f"explaining why (see the Gemma 2 9B entry for the expected form)" + ) + + +def test_registry_yaml_path_is_the_real_bundled_file(): + # Sanity check on the test's own re-read of the raw file: make sure + # it's reading the same path the dataclass loader used, not a stale + # copy elsewhere on disk. + assert _DEFAULT_ARCHITECTURE_PATH.exists() + assert _DEFAULT_REGISTRY_PATH.exists()