diff --git a/docs/registry-refresh.md b/docs/registry-refresh.md new file mode 100644 index 0000000..eb08512 --- /dev/null +++ b/docs/registry-refresh.md @@ -0,0 +1,36 @@ +# Refreshing the model registry + +`src/norefund/config/default_models.yaml` and `model_architectures.yaml` are +read against primary sources, not updated from memory or from a previous +model's own claims about itself. Pricing comes from each provider's own +pricing page, re-read on the day the file is edited: OpenAI +(`https://developers.openai.com/api/docs/pricing`), Anthropic +(`https://platform.claude.com/docs/en/about-claude/pricing`), Google +(`https://ai.google.dev/gemini-api/docs/pricing`), and DeepSeek +(`https://api-docs.deepseek.com/quick_start/pricing`). Architecture fields +(layer counts, head dimensions, MLA parameters, context windows) come from +the model's published HuggingFace `config.json`, and parameter counts are +cross-checked against the HF API's safetensors total +(`GET /api/models/{repo}?blobs=true`, the `safetensors.total` field) rather +than trusted from a model card's rounded number. + +`pricing_verified_on` on a `ModelInfo` entry moves only when a human (or an +agent acting on a human's behalf) has actually opened the provider's page +that day and read the number off it -- never bumped to "look current" +without doing that, and never copied forward from an older PR or plan +document, which can itself have drifted by the time anyone acts on it (this +happened during the Phase 14 registry refresh: a reference doc's own +pricing figures, recorded four days earlier, were already wrong by the time +they were used). `tests/test_registry_data.py` enforces the structural +invariants this depends on -- every architecture id has a matching registry +entry, the two files agree on shared fields, every priced entry carries a +verification date -- but it cannot check whether a given date's price is +still correct; only re-reading the source page can. + +When a licence-gated HuggingFace repo needs an ungated substitute (see the +comments on the `meta:llama-*` and `google:gemma-2-*` entries), verify +identity via the HF tree API's git blob oid for the file in question +(`GET /api/models/{repo}/tree/main`) rather than trusting a mirror's name -- +an identical oid between the official repo and the mirror proves the file +is byte-identical without needing to authenticate to the gated repo or +download either copy. diff --git a/frontend/src/lib/costing.test.ts b/frontend/src/lib/costing.test.ts index ce837f2..22d8785 100644 --- a/frontend/src/lib/costing.test.ts +++ b/frontend/src/lib/costing.test.ts @@ -10,7 +10,22 @@ import { } from "./costing"; // Mirrors tests/test_costing.py's _MODEL. -const MODEL = { input_price_per_million: 2.0, output_price_per_million: 8.0 }; +const MODEL = { + input_price_per_million: 2.0, + output_price_per_million: 8.0, + long_context_threshold: null, + long_context_input_price_per_million: null, + long_context_output_price_per_million: null, +}; + +// Mirrors tests/test_costing.py's _TIERED_MODEL. +const TIERED_MODEL = { + input_price_per_million: 2.0, + output_price_per_million: 8.0, + long_context_threshold: 200_000, + long_context_input_price_per_million: 4.0, + long_context_output_price_per_million: 16.0, +}; describe("contextUsagePct", () => { it("is null when the window is zero", () => { @@ -64,7 +79,13 @@ describe("inputCost / outputCost / totalCost", () => { expect(outputCost(1_000_000, MODEL)).toBeCloseTo(8.0); }); it("is zero for a free model", () => { - const free = { input_price_per_million: 0, output_price_per_million: 0 }; + const free = { + input_price_per_million: 0, + output_price_per_million: 0, + long_context_threshold: null, + long_context_input_price_per_million: null, + long_context_output_price_per_million: null, + }; expect(inputCost(100_000, free)).toBe(0); }); it("sums input and output", () => { @@ -72,6 +93,37 @@ describe("inputCost / outputCost / totalCost", () => { }); }); +describe("context-tiered pricing", () => { + it("uses the short rate exactly at the threshold", () => { + expect(inputCost(200_000, TIERED_MODEL)).toBeCloseTo((200_000 / 1_000_000) * 2.0); + }); + it("uses the short rate just below the threshold", () => { + expect(inputCost(199_999, TIERED_MODEL)).toBeCloseTo((199_999 / 1_000_000) * 2.0); + }); + it("uses the long rate just above the threshold", () => { + expect(inputCost(200_001, TIERED_MODEL)).toBeCloseTo((200_001 / 1_000_000) * 4.0); + }); + it("tiers output cost by prompt size, not output size", () => { + const cost = outputCost(500_000, TIERED_MODEL, 1_000); + expect(cost).toBeCloseTo((500_000 / 1_000_000) * 8.0); + }); + it("uses the long output rate when the prompt crosses the threshold", () => { + const cost = outputCost(1_000, TIERED_MODEL, 200_001); + expect(cost).toBeCloseTo((1_000 / 1_000_000) * 16.0); + }); + it("defaults the prompt size to the output's own count when omitted", () => { + expect(outputCost(200_001, TIERED_MODEL)).toBeCloseTo((200_001 / 1_000_000) * 16.0); + }); + it("totalCost tiers output by the input size", () => { + const cost = totalCost(300_000, 1_000, TIERED_MODEL); + const expected = (300_000 / 1_000_000) * 4.0 + (1_000 / 1_000_000) * 16.0; + expect(cost).toBeCloseTo(expected); + }); + it("is a no-op for a model with no tier fields", () => { + expect(inputCost(10_000_000, MODEL)).toBeCloseTo((10_000_000 / 1_000_000) * 2.0); + }); +}); + describe("convertCurrency", () => { it("applies the rate for the target currency", () => { const rates = { base: "USD", rates: { EUR: 0.5 }, fetched_at: null }; diff --git a/frontend/src/lib/costing.ts b/frontend/src/lib/costing.ts index 0e38519..80df103 100644 --- a/frontend/src/lib/costing.ts +++ b/frontend/src/lib/costing.ts @@ -28,26 +28,58 @@ export function minChunks(tokens: number, window: number): number { return Math.ceil(tokens / usable); } +type TieredPricingFields = Pick< + ModelInfo, + | "long_context_threshold" + | "long_context_input_price_per_million" + | "long_context_output_price_per_million" +>; + +/** Whether promptTokens crosses this model's long-context tier -- decided by + * the prompt (input) size, per every provider that offers one, not by the + * completion size even when pricing output. */ +function longContextActive(promptTokens: number, model: TieredPricingFields): boolean { + return ( + model.long_context_threshold !== null && promptTokens > model.long_context_threshold + ); +} + export function inputCost( tokens: number, - model: Pick, + model: Pick & TieredPricingFields, ): number { - return (tokens / 1_000_000) * model.input_price_per_million; + const rate = + longContextActive(tokens, model) && model.long_context_input_price_per_million !== null + ? model.long_context_input_price_per_million + : model.input_price_per_million; + return (tokens / 1_000_000) * rate; } +/** promptTokens is the input size that decides which price tier applies. + * Defaults to tokens itself when omitted, which is only correct for + * flat-priced models (long_context_threshold null) -- callers pricing a + * tiered model must pass the real prompt size. */ export function outputCost( tokens: number, - model: Pick, + model: Pick & TieredPricingFields, + promptTokens?: number, ): number { - return (tokens / 1_000_000) * model.output_price_per_million; + const promptSize = promptTokens ?? tokens; + const rate = + longContextActive(promptSize, model) && + model.long_context_output_price_per_million !== null + ? model.long_context_output_price_per_million + : model.output_price_per_million; + return (tokens / 1_000_000) * rate; } export function totalCost( inputTokens: number, outputTokens: number, - model: Pick, + model: Pick & + TieredPricingFields, ): number { - return inputCost(inputTokens, model) + outputCost(outputTokens, model); + return inputCost(inputTokens, model) + outputCost(outputTokens, model, inputTokens); } /** Convert a USD amount into toCurrency using the given rates -- mirrors diff --git a/frontend/src/lib/types.ts b/frontend/src/lib/types.ts index 4517c68..8e1d646 100644 --- a/frontend/src/lib/types.ts +++ b/frontend/src/lib/types.ts @@ -16,6 +16,11 @@ export interface ModelInfo { currency: string; docs_url: string | null; tokenizer_is_approximate: boolean; + long_context_threshold: number | null; + long_context_input_price_per_million: number | null; + long_context_output_price_per_million: number | null; + pricing_note: string | null; + pricing_verified_on: string | null; } export interface AnalysisResult { @@ -102,6 +107,8 @@ export interface ModelArchitecture { attention_type: string; kv_lora_rank: number; qk_rope_head_dim: number; + sliding_window: number; + sliding_window_pattern: number; docs_url: string | null; } diff --git a/frontend/src/views/Calculator/index.tsx b/frontend/src/views/Calculator/index.tsx index b7d0e28..b7b4b19 100644 --- a/frontend/src/views/Calculator/index.tsx +++ b/frontend/src/views/Calculator/index.tsx @@ -50,7 +50,10 @@ export default function Calculator() { const pct = inputTokens === null ? null : contextUsagePct(inputTokens, model.context_window); const fits = inputTokens === null ? null : fitsInContext(inputTokens, model.context_window); const inCost = inputTokens === null ? null : inputCost(inputTokens, model); - const outCost = outputTokens === null ? null : outputCost(outputTokens, model); + const outCost = + outputTokens === null + ? null + : outputCost(outputTokens, model, inputTokens ?? undefined); const total = inCost === null || outCost === null ? null : inCost + outCost; return ( diff --git a/frontend/src/views/Compare/sort.test.ts b/frontend/src/views/Compare/sort.test.ts index e431982..e6eb5bc 100644 --- a/frontend/src/views/Compare/sort.test.ts +++ b/frontend/src/views/Compare/sort.test.ts @@ -15,6 +15,11 @@ function model(id: string): ModelInfo { currency: "USD", docs_url: null, tokenizer_is_approximate: false, + long_context_threshold: null, + long_context_input_price_per_million: null, + long_context_output_price_per_million: null, + pricing_note: null, + pricing_verified_on: null, }; } diff --git a/frontend/src/views/Registry/index.tsx b/frontend/src/views/Registry/index.tsx index 28e8e68..09a2d2f 100644 --- a/frontend/src/views/Registry/index.tsx +++ b/frontend/src/views/Registry/index.tsx @@ -31,16 +31,33 @@ export default function Registry() { () => Array.from(new Set(models.map((m) => m.provider))).sort(), [models], ); + // The most recent date any priced entry was checked against its + // provider's actual page -- not every entry shares one date, so the + // true statement is "at least this recently," not "as of exactly this + // day." See docs/registry-refresh.md for the refresh procedure. + const latestPricingVerification = useMemo(() => { + const dates = models + .map((m) => m.pricing_verified_on) + .filter((d): d is string => d !== null); + return dates.length > 0 ? dates.reduce((a, b) => (a > b ? a : b)) : null; + }, [models]); const visible = activeProvider === "All" ? models : models.filter((m) => m.provider === activeProvider); return (
-

- {models.length} models across {providers.length} providers. Locally stored - pricing data. -

+
+

+ {models.length} models across {providers.length} providers. Locally stored + pricing data. +

+ {latestPricingVerification && ( +

+ Prices verified {latestPricingVerification} +

+ )} +
272k input tokens: 2x input, 1.5x output, for the full request + long_context_input_price_per_million: 8.00 + long_context_output_price_per_million: 30.00 currency: "USD" - docs_url: "https://openai.com/api/pricing/" + docs_url: "https://developers.openai.com/api/docs/pricing" + pricing_verified_on: "2026-08-26" -- id: openai:gpt-4o-mini - display_name: "GPT-4o Mini" +- id: openai:gpt-5.6-terra + display_name: "GPT-5.6 Terra" provider: "OpenAI" tokenizer_backend: "tiktoken" - tokenizer_name: "gpt-4o-mini" - context_window: 128000 - input_price_per_million: 0.15 - output_price_per_million: 0.60 + tokenizer_name: "gpt-4o" + context_window: 1050000 + input_price_per_million: 2.00 + output_price_per_million: 12.00 + long_context_threshold: 272000 + long_context_input_price_per_million: 4.00 + long_context_output_price_per_million: 18.00 currency: "USD" - docs_url: "https://openai.com/api/pricing/" + docs_url: "https://developers.openai.com/api/docs/pricing" + pricing_verified_on: "2026-08-26" -- id: openai:gpt-4.1 - display_name: "GPT-4.1" +- id: openai:gpt-5.6-luna + display_name: "GPT-5.6 Luna" provider: "OpenAI" tokenizer_backend: "tiktoken" - tokenizer_name: "gpt-4o" # GPT-4.1 shares gpt-4o's o200k_base encoding — this is exact, not an approximation - context_window: 1047576 - input_price_per_million: 2.00 - output_price_per_million: 8.00 + tokenizer_name: "gpt-4o" + context_window: 1050000 + input_price_per_million: 0.20 + output_price_per_million: 1.20 + long_context_threshold: 272000 + long_context_input_price_per_million: 0.40 + long_context_output_price_per_million: 1.80 currency: "USD" - docs_url: "https://openai.com/api/pricing/" + docs_url: "https://developers.openai.com/api/docs/pricing" + pricing_verified_on: "2026-08-26" -- id: anthropic:claude-3-5-sonnet - display_name: "Claude 3.5 Sonnet" +- id: anthropic:claude-sonnet-5 + display_name: "Claude Sonnet 5" provider: "Anthropic" tokenizer_backend: "tiktoken" tokenizer_name: "cl100k_base" - context_window: 200000 - input_price_per_million: 3.00 - output_price_per_million: 15.00 + context_window: 1000000 + input_price_per_million: 2.00 + output_price_per_million: 10.00 currency: "USD" - docs_url: "https://www.anthropic.com/pricing#anthropic-api" + docs_url: "https://platform.claude.com/docs/en/about-claude/pricing" tokenizer_is_approximate: true # Anthropic does not publish a local tokenizer + pricing_verified_on: "2026-08-26" -- id: anthropic:claude-3-haiku - display_name: "Claude 3 Haiku" +- id: anthropic:claude-haiku-4.5 + display_name: "Claude Haiku 4.5" provider: "Anthropic" tokenizer_backend: "tiktoken" tokenizer_name: "cl100k_base" context_window: 200000 - input_price_per_million: 0.25 - output_price_per_million: 1.25 + input_price_per_million: 1.00 + output_price_per_million: 5.00 currency: "USD" - docs_url: "https://www.anthropic.com/pricing#anthropic-api" + docs_url: "https://platform.claude.com/docs/en/about-claude/pricing" tokenizer_is_approximate: true # Anthropic does not publish a local tokenizer + pricing_verified_on: "2026-08-26" -- id: google:gemini-2.0-flash - display_name: "Gemini 2.0 Flash" +- id: google:gemini-3.5-flash + display_name: "Gemini 3.5 Flash" provider: "Google" tokenizer_backend: "tiktoken" tokenizer_name: "cl100k_base" context_window: 1048576 - input_price_per_million: 0.10 - output_price_per_million: 0.40 + input_price_per_million: 1.50 + output_price_per_million: 9.00 currency: "USD" docs_url: "https://ai.google.dev/gemini-api/docs/pricing" tokenizer_is_approximate: true # Google does not publish a local Gemini tokenizer + pricing_verified_on: "2026-08-26" -- id: google:gemini-1.5-pro - display_name: "Gemini 1.5 Pro" +- id: google:gemini-3.1-pro-preview + display_name: "Gemini 3.1 Pro Preview" provider: "Google" tokenizer_backend: "tiktoken" tokenizer_name: "cl100k_base" - context_window: 2000000 - input_price_per_million: 3.50 - output_price_per_million: 10.50 + context_window: 1048576 + input_price_per_million: 2.00 + output_price_per_million: 12.00 + long_context_threshold: 200000 # >200k input tokens: input and output both roughly double + long_context_input_price_per_million: 4.00 + long_context_output_price_per_million: 18.00 currency: "USD" docs_url: "https://ai.google.dev/gemini-api/docs/pricing" tokenizer_is_approximate: true # Google does not publish a local Gemini tokenizer + pricing_verified_on: "2026-08-26" + +- id: deepseek:deepseek-v4-flash + display_name: "DeepSeek V4 Flash" + provider: "DeepSeek" + tokenizer_backend: "hf" + tokenizer_name: "deepseek-ai/DeepSeek-V4-Flash" + context_window: 1048576 # config.json's max_position_embeddings + # Storing the peak / cache-miss rate: this app's one-off document + # analyses never hit a warm cache, and peak is the provider's own + # "assume the worst case" hours. Off-peak (all hours except + # 01:00-04:00 and 06:00-10:00 UTC Mon-Fri -- these ARE peak, not + # off-peak; the source doc that preceded this file had that backwards) + # halves both rates to $0.22/$0.66. + input_price_per_million: 0.44 + output_price_per_million: 1.32 + currency: "USD" + docs_url: "https://api-docs.deepseek.com/quick_start/pricing" + pricing_note: "Off-peak (all hours except 01:00-04:00 and 06:00-10:00 UTC Mon-Fri) halves both rates to $0.22/$0.66." + pricing_verified_on: "2026-08-26" - id: deepseek:deepseek-v3 - display_name: "DeepSeek V3" + display_name: "DeepSeek V3 (self-hosted)" provider: "DeepSeek" tokenizer_backend: "hf" + # Separate from deepseek:deepseek-v4-flash above -- V3 is superseded as + # a hosted API product (delisted from DeepSeek's own pricing page) but + # remains a real, self-hostable open-weight model with its own + # model_architectures.yaml entry (MLA parameters, kv_lora_rank=512), + # which needs a matching entry here or the Fit Check has nothing to + # look up. Zero-priced since nobody pays DeepSeek per-token to run + # their own weights. tokenizer_name: "deepseek-ai/DeepSeek-V3" - context_window: 128000 - input_price_per_million: 0.27 - output_price_per_million: 1.10 + context_window: 163840 # matches model_architectures.yaml's max_context_length + input_price_per_million: 0.00 + output_price_per_million: 0.00 currency: "USD" - docs_url: "https://api-docs.deepseek.com/quick_start/pricing" + docs_url: "https://huggingface.co/deepseek-ai/DeepSeek-V3" - id: meta:llama-3-8b display_name: "Llama 3 8B (self-hosted)" provider: "Meta" tokenizer_backend: "hf" - tokenizer_name: "meta-llama/Meta-Llama-3-8B" + # meta-llama/Meta-Llama-3-8B is licence-gated (manual approval); this + # download would fail for any user without an accepted licence + HF + # token. NousResearch/Meta-Llama-3-8B carries a byte-identical + # tokenizer.json (verified via the HF tree API's git blob oid, since + # the gated repo's own content isn't fetchable without auth: both are + # b197f72effb9d5ed16ee0f5663e11e4cfac2ba62). + tokenizer_name: "NousResearch/Meta-Llama-3-8B" context_window: 8192 input_price_per_million: 0.00 output_price_per_million: 0.00 @@ -105,7 +162,10 @@ display_name: "Mistral 7B (self-hosted)" provider: "Mistral" tokenizer_backend: "hf" - tokenizer_name: "mistralai/Mistral-7B-v0.1" + # v0.3, not v0.1: v0.1's config.json has sliding_window: 4096, so its + # nominal 32768 context is not honestly usable; v0.3 has sliding_window: + # null, matching model_architectures.yaml's docs_url for this id. + tokenizer_name: "mistralai/Mistral-7B-Instruct-v0.3" context_window: 32768 input_price_per_million: 0.00 output_price_per_million: 0.00 @@ -116,7 +176,10 @@ display_name: "Llama 3.1 8B (self-hosted)" provider: "Meta" tokenizer_backend: "hf" - tokenizer_name: "meta-llama/Llama-3.1-8B-Instruct" + # meta-llama/Llama-3.1-8B-Instruct is licence-gated. NousResearch's + # mirror is byte-identical (git blob oid 5cc5f00a5b203e90a27a3bd60d1ec393b07971e8 + # on both, verified via the HF tree API without needing gated-repo auth). + tokenizer_name: "NousResearch/Meta-Llama-3.1-8B-Instruct" context_window: 131072 input_price_per_million: 0.00 output_price_per_million: 0.00 @@ -127,7 +190,10 @@ display_name: "Llama 3.1 70B (self-hosted)" provider: "Meta" tokenizer_backend: "hf" - tokenizer_name: "meta-llama/Llama-3.1-70B-Instruct" + # meta-llama/Llama-3.1-70B-Instruct is licence-gated. Same tokenizer as + # the 8B/405B Instruct models in this family (identical git blob oid + # 5cc5f00a5b203e90a27a3bd60d1ec393b07971e8), mirrored ungated here. + tokenizer_name: "NousResearch/Meta-Llama-3.1-70B-Instruct" context_window: 131072 input_price_per_million: 0.00 output_price_per_million: 0.00 @@ -138,7 +204,10 @@ display_name: "Llama 3.3 70B (self-hosted)" provider: "Meta" tokenizer_backend: "hf" - tokenizer_name: "meta-llama/Llama-3.3-70B-Instruct" + # meta-llama/Llama-3.3-70B-Instruct is licence-gated. unsloth's mirror + # is byte-identical (git blob oid 1c1d8d5c9024994f1d3b00f9662b8dd89ca13cf2 + # on both). + tokenizer_name: "unsloth/Llama-3.3-70B-Instruct" context_window: 131072 input_price_per_million: 0.00 output_price_per_million: 0.00 @@ -149,7 +218,11 @@ display_name: "Llama 3.1 405B (self-hosted)" provider: "Meta" tokenizer_backend: "hf" - tokenizer_name: "meta-llama/Llama-3.1-405B-Instruct" + # meta-llama/Llama-3.1-405B-Instruct is licence-gated. Same tokenizer as + # the 8B/70B Instruct models in this family (identical git blob oid + # 5cc5f00a5b203e90a27a3bd60d1ec393b07971e8); unsloth's 4-bit mirror + # keeps the original, unmodified tokenizer. + tokenizer_name: "unsloth/Meta-Llama-3.1-405B-Instruct-bnb-4bit" context_window: 131072 input_price_per_million: 0.00 output_price_per_million: 0.00 @@ -215,7 +288,10 @@ display_name: "Gemma 2 9B (self-hosted)" provider: "Google" tokenizer_backend: "hf" - tokenizer_name: "google/gemma-2-9b-it" + # google/gemma-2-9b-it is licence-gated. unsloth's mirror is + # byte-identical (git blob oid af0eac5c0056f83b8f3fcdb79165f8847111c305 + # on both -- same tokenizer as gemma-2-27b-it too). + tokenizer_name: "unsloth/gemma-2-9b-it" context_window: 8192 input_price_per_million: 0.00 output_price_per_million: 0.00 @@ -226,7 +302,10 @@ display_name: "Gemma 2 27B (self-hosted)" provider: "Google" tokenizer_backend: "hf" - tokenizer_name: "google/gemma-2-27b-it" + # google/gemma-2-27b-it is licence-gated. unsloth's mirror is + # byte-identical (git blob oid af0eac5c0056f83b8f3fcdb79165f8847111c305 + # on both -- same tokenizer as gemma-2-9b-it too). + tokenizer_name: "unsloth/gemma-2-27b-it" context_window: 8192 input_price_per_million: 0.00 output_price_per_million: 0.00 diff --git a/src/norefund/config/hardware.yaml b/src/norefund/config/hardware.yaml index 31e2a36..64327f2 100644 --- a/src/norefund/config/hardware.yaml +++ b/src/norefund/config/hardware.yaml @@ -49,7 +49,10 @@ vendor: "NVIDIA" accelerator: "NVIDIA H200" device_count: 1 - memory_gib_per_device: 141.0 + # NVIDIA markets "141 GB" but nvidia-smi reports 143771 MiB = 140.4 GiB. + # Follow this file's own GiB rule (see header) rather than the marketing + # figure -- do not "correct" this back to 141.0. + memory_gib_per_device: 140.4 memory_kind: "discrete" usable_memory_fraction: 0.90 diff --git a/src/norefund/config/model_architectures.yaml b/src/norefund/config/model_architectures.yaml index 6a87ca8..f071f8e 100644 --- a/src/norefund/config/model_architectures.yaml +++ b/src/norefund/config/model_architectures.yaml @@ -2,7 +2,10 @@ # self-host VRAM fit calculator (core/selfhost.py). # # Every number here is copied from the model's published HuggingFace -# config.json (linked via docs_url), not derived or guessed. In particular: +# config.json (linked via docs_url), not derived or guessed, with one +# exception: max_context_length for the three Qwen2.5 entries comes from +# the model card, not config.json -- see the comment on those entries. +# In particular: # # - head_dim is NOT always hidden_size / n_attention_heads. Gemma 2 9B is the # counter-example: hidden_size=3584, n_attention_heads=16 gives 224, but @@ -14,6 +17,27 @@ # (Mixtral, DeepSeek V3). See core/selfhost.py for why weight memory uses # total_params, not active_params. +- id: meta:llama-3-8b + display_name: "Llama 3 8B" + family: "Llama 3" + vendor: "Meta" + # The official meta-llama/Meta-Llama-3-8B repo is licence-gated, so + # config.json isn't fetchable without an accepted licence + HF token. + # Verified instead against NousResearch/Meta-Llama-3-8B and + # unsloth/llama-3-8b, both public mirrors carrying an identical + # config.json; total_params cross-checked against the mirror's own + # safetensors total (8030261248), not copied from this file. + total_params: 8030261248 + active_params: 8030261248 + n_layers: 32 + n_attention_heads: 32 + n_kv_heads: 8 + head_dim: 128 + hidden_size: 4096 + max_context_length: 8192 + attention_type: "gqa" + docs_url: "https://huggingface.co/meta-llama/Meta-Llama-3-8B" + - id: meta:llama-3.1-8b display_name: "Llama 3.1 8B" family: "Llama 3" @@ -85,6 +109,9 @@ n_kv_heads: 4 head_dim: 128 hidden_size: 3584 + # config.json's native max_position_embeddings is 32768 with no + # rope_scaling block; 131072 is the model card's claimed extended + # window via YaRN, not something config.json states on its own. max_context_length: 131072 attention_type: "gqa" docs_url: "https://huggingface.co/Qwen/Qwen2.5-7B-Instruct" @@ -100,6 +127,8 @@ n_kv_heads: 8 head_dim: 128 hidden_size: 5120 + # See qwen:qwen2.5-7b above -- config.json says 32768, no rope_scaling; + # 131072 is the model card's YaRN-extended figure. max_context_length: 131072 attention_type: "gqa" docs_url: "https://huggingface.co/Qwen/Qwen2.5-32B-Instruct" @@ -115,6 +144,8 @@ n_kv_heads: 8 head_dim: 128 hidden_size: 8192 + # See qwen:qwen2.5-7b above -- config.json says 32768, no rope_scaling; + # 131072 is the model card's YaRN-extended figure. max_context_length: 131072 attention_type: "gqa" docs_url: "https://huggingface.co/Qwen/Qwen2.5-72B-Instruct" @@ -177,6 +208,10 @@ hidden_size: 3584 max_context_length: 8192 attention_type: "gqa" + # Alternates local sliding-window (4096) and global full attention 1:1 + # every other layer -- see core/selfhost.py's kv_cache_bytes. + sliding_window: 4096 + sliding_window_pattern: 2 docs_url: "https://huggingface.co/google/gemma-2-9b-it" - id: google:gemma-2-27b @@ -192,6 +227,9 @@ hidden_size: 4608 max_context_length: 8192 attention_type: "gqa" + # See google:gemma-2-9b above -- same 1:1 local/global alternation. + sliding_window: 4096 + sliding_window_pattern: 2 docs_url: "https://huggingface.co/google/gemma-2-27b-it" - id: deepseek:deepseek-v3 diff --git a/src/norefund/core/architectures.py b/src/norefund/core/architectures.py index 696e158..cde8f2a 100644 --- a/src/norefund/core/architectures.py +++ b/src/norefund/core/architectures.py @@ -40,6 +40,8 @@ class ModelArchitecture: attention_type: AttentionType kv_lora_rank: int = 0 # MLA only (e.g. DeepSeek V3) qk_rope_head_dim: int = 0 # MLA only (e.g. DeepSeek V3) + sliding_window: int = 0 # 0 = full attention on every layer + sliding_window_pattern: int = 0 # every Nth layer is full attention (Gemma 2: 2) docs_url: str | None = None diff --git a/src/norefund/core/compare.py b/src/norefund/core/compare.py index 1eeea59..97350fb 100644 --- a/src/norefund/core/compare.py +++ b/src/norefund/core/compare.py @@ -83,7 +83,9 @@ def _build_comparison( min_chunks_needed=min_chunks(token_count, model.context_window), output_tokens=output_tokens, input_cost=compute_input_cost(token_count, model), - output_cost=compute_output_cost(output_tokens, model), + output_cost=compute_output_cost( + output_tokens, model, prompt_token_count=token_count + ), total_cost=compute_total_cost(token_count, output_tokens, model), tokenizer_is_approximate=model.tokenizer_is_approximate, error=None, diff --git a/src/norefund/core/costing.py b/src/norefund/core/costing.py index d421bae..92836a4 100644 --- a/src/norefund/core/costing.py +++ b/src/norefund/core/costing.py @@ -35,15 +35,50 @@ def min_chunks(token_count: int, context_window: int) -> int: return math.ceil(token_count / usable) -def input_cost(token_count: int, model: ModelInfo) -> float: - """USD cost for processing token_count input tokens.""" - return (token_count / 1_000_000) * model.input_price_per_million +def _long_context_active(prompt_token_count: int, model: ModelInfo) -> bool: + """Whether prompt_token_count crosses this model's long-context tier. + + The tier is decided by the prompt (input) size, per every provider that + offers one -- not by the completion size, even when pricing output. + """ + return ( + model.long_context_threshold is not None + and prompt_token_count > model.long_context_threshold + ) -def output_cost(token_count: int, model: ModelInfo) -> float: - """USD cost for token_count output tokens.""" - return (token_count / 1_000_000) * model.output_price_per_million +def input_cost(token_count: int, model: ModelInfo) -> float: + """USD cost for processing token_count input/prompt tokens.""" + rate = model.input_price_per_million + if ( + _long_context_active(token_count, model) + and model.long_context_input_price_per_million is not None + ): + rate = model.long_context_input_price_per_million + return (token_count / 1_000_000) * rate + + +def output_cost( + token_count: int, model: ModelInfo, *, prompt_token_count: int | None = None +) -> float: + """USD cost for token_count output tokens. + + prompt_token_count is the input/prompt size that decides which price + tier applies. Defaults to token_count itself when omitted, which is + only correct for flat-priced models (long_context_threshold is None) -- + callers pricing a tiered model must pass the real prompt size. + """ + prompt_size = token_count if prompt_token_count is None else prompt_token_count + rate = model.output_price_per_million + if ( + _long_context_active(prompt_size, model) + and model.long_context_output_price_per_million is not None + ): + rate = model.long_context_output_price_per_million + return (token_count / 1_000_000) * rate def total_cost(input_tokens: int, output_tokens: int, model: ModelInfo) -> float: - return input_cost(input_tokens, model) + output_cost(output_tokens, model) + return input_cost(input_tokens, model) + output_cost( + output_tokens, model, prompt_token_count=input_tokens + ) diff --git a/src/norefund/core/models_registry.py b/src/norefund/core/models_registry.py index 5d96f95..0fbb9de 100644 --- a/src/norefund/core/models_registry.py +++ b/src/norefund/core/models_registry.py @@ -27,6 +27,13 @@ class ModelInfo: currency: str = "USD" docs_url: str | None = None # Optional link to provider pricing/docs page tokenizer_is_approximate: bool = False # True when no real local tokenizer exists + # Context-tiered pricing (OpenAI, Google): prompts above this many tokens + # bill at the long_context_* rate instead. None means flat pricing. + long_context_threshold: int | None = None + long_context_input_price_per_million: float | None = None + long_context_output_price_per_million: float | None = None + pricing_note: str | None = None # e.g. DeepSeek's off-peak halving + pricing_verified_on: str | None = None # ISO date a human read the pricing page def load_models(path: Path = _DEFAULT_REGISTRY_PATH) -> dict[str, ModelInfo]: diff --git a/src/norefund/core/selfhost.py b/src/norefund/core/selfhost.py index 2e3c663..d638ca6 100644 --- a/src/norefund/core/selfhost.py +++ b/src/norefund/core/selfhost.py @@ -87,32 +87,67 @@ def weight_bytes(architecture: ModelArchitecture, quantization: str) -> int | No return round(architecture.total_params * bpw / 8) -def kv_cache_bytes_per_token( - architecture: ModelArchitecture, kv_cache_dtype: str = "fp16" -) -> int | None: - """KV-cache memory for one token, one sequence.""" +def _kv_cache_bytes_per_layer_per_token( + architecture: ModelArchitecture, kv_cache_dtype: str +) -> float | None: + """Unrounded KV-cache bytes for one layer, one token, one sequence. + + Shared by kv_cache_bytes_per_token and kv_cache_bytes so both round + only once, at the end, rather than compounding rounding error by + dividing an already-rounded per-token total by n_layers. + """ bytes_per_element = bytes_per_kv_element(kv_cache_dtype) if bytes_per_element is None: return None if architecture.attention_type == "gqa": - return round( - 2 - * architecture.n_layers - * architecture.n_kv_heads - * architecture.head_dim - * bytes_per_element - ) + return 2 * architecture.n_kv_heads * architecture.head_dim * bytes_per_element if architecture.attention_type == "mla": if architecture.kv_lora_rank <= 0: return None - return round( - architecture.n_layers - * (architecture.kv_lora_rank + architecture.qk_rope_head_dim) - * bytes_per_element - ) + return ( + architecture.kv_lora_rank + architecture.qk_rope_head_dim + ) * bytes_per_element return None +def kv_cache_bytes_per_token( + architecture: ModelArchitecture, kv_cache_dtype: str = "fp16" +) -> int | None: + """KV-cache memory for one token, one sequence, as if every layer were + full attention. Ignores sliding-window; see kv_cache_bytes for + context-length-aware sizing that accounts for it.""" + per_layer = _kv_cache_bytes_per_layer_per_token(architecture, kv_cache_dtype) + if per_layer is None: + return None + return round(architecture.n_layers * per_layer) + + +def kv_cache_bytes( + architecture: ModelArchitecture, context_length: int, kv_cache_dtype: str = "fp16" +) -> int | None: + """KV-cache memory for one sequence at context_length. + + Sliding-window KV cost is not linear in context length the way full + attention's is: a windowed layer caches only min(context_length, + sliding_window) tokens, while a full-attention layer caches the whole + context_length. architecture.sliding_window == 0 (every architecture + except Gemma 2 today) means every layer is full attention, which + reduces this to kv_cache_bytes_per_token(...) * context_length. + """ + per_layer = _kv_cache_bytes_per_layer_per_token(architecture, kv_cache_dtype) + if per_layer is None or architecture.n_layers <= 0: + return None + if architecture.sliding_window <= 0 or architecture.sliding_window_pattern <= 0: + return round(architecture.n_layers * per_layer * context_length) + + full_layers = architecture.n_layers // architecture.sliding_window_pattern + windowed_layers = architecture.n_layers - full_layers + windowed_context = min(context_length, architecture.sliding_window) + return round( + per_layer * (full_layers * context_length + windowed_layers * windowed_context) + ) + + def activation_bytes(architecture: ModelArchitecture, context_length: int) -> int: if context_length <= 0: return 0 @@ -135,11 +170,10 @@ def estimate_memory( kv_cache_dtype: str = "fp16", ) -> MemoryEstimate | None: weights = weight_bytes(architecture, quantization) - kv_per_token = kv_cache_bytes_per_token(architecture, kv_cache_dtype) - if weights is None or kv_per_token is None or context_length <= 0: + kv_per_sequence = kv_cache_bytes(architecture, context_length, kv_cache_dtype) + if weights is None or kv_per_sequence is None or context_length <= 0: return None - kv_per_sequence = kv_per_token * context_length kv_total = kv_per_sequence * max(concurrency, 0) activation = activation_bytes(architecture, context_length) overhead = framework_overhead_bytes(hardware) @@ -165,8 +199,8 @@ def max_concurrent_requests( """How many concurrent full-context sequences could fit, independent of the requested concurrency.""" weights = weight_bytes(architecture, quantization) - kv_per_token = kv_cache_bytes_per_token(architecture, kv_cache_dtype) - if weights is None or kv_per_token is None or context_length <= 0: + kv_per_sequence = kv_cache_bytes(architecture, context_length, kv_cache_dtype) + if weights is None or kv_per_sequence is None or context_length <= 0: return None activation = activation_bytes(architecture, context_length) @@ -175,7 +209,6 @@ def max_concurrent_requests( if available_for_kv <= 0: return 0 - kv_per_sequence = kv_per_token * context_length if kv_per_sequence <= 0: return None return available_for_kv // kv_per_sequence diff --git a/tests/test_architectures.py b/tests/test_architectures.py index 756a8b3..30af9d0 100644 --- a/tests/test_architectures.py +++ b/tests/test_architectures.py @@ -17,10 +17,12 @@ def test_load_architectures_returns_dict_of_model_architecture(): assert all(isinstance(a, ModelArchitecture) for a in architectures.values()) -def test_exactly_thirteen_entries(): +def test_exactly_fourteen_entries(): # 13 rows in PLAN.md's table (Llama 3.1 70B and 3.3 70B are separate - # entries with identical numbers, despite the "~12 models" decision). - assert len(list_architectures()) == 13 + # entries with identical numbers, despite the "~12 models" decision), + # plus meta:llama-3-8b (Phase 14: every self-hosted model in + # default_models.yaml must have an architecture entry). + assert len(list_architectures()) == 14 def test_all_ids_unique(): diff --git a/tests/test_costing.py b/tests/test_costing.py index 3e54887..a3a4938 100644 --- a/tests/test_costing.py +++ b/tests/test_costing.py @@ -84,3 +84,64 @@ def test_output_cost_calculation(): def test_total_cost_sum(): cost = total_cost(1_000_000, 1_000_000, _MODEL) assert cost == 10.0 # $2.0 input + $8.0 output + + +_TIERED_MODEL = ModelInfo( + id="test:tiered-model", + display_name="Test Tiered Model", + provider="Test", + tokenizer_backend="tiktoken", + tokenizer_name="cl100k_base", + context_window=1_000_000, + input_price_per_million=2.0, + output_price_per_million=8.0, + long_context_threshold=200_000, + long_context_input_price_per_million=4.0, + long_context_output_price_per_million=16.0, +) + + +def test_input_cost_at_threshold_uses_short_rate(): + # Boundary is "above the threshold", not "at or above" -- exactly + # long_context_threshold tokens still bills at the short-context rate. + assert input_cost(200_000, _TIERED_MODEL) == (200_000 / 1_000_000) * 2.0 + + +def test_input_cost_below_threshold_uses_short_rate(): + assert input_cost(199_999, _TIERED_MODEL) == (199_999 / 1_000_000) * 2.0 + + +def test_input_cost_above_threshold_uses_long_rate(): + assert input_cost(200_001, _TIERED_MODEL) == (200_001 / 1_000_000) * 4.0 + + +def test_output_cost_tier_follows_prompt_size_not_output_size(): + # A short prompt with a huge completion stays on the short-context + # output rate -- the tier is decided by the prompt, not the output. + cost = output_cost(500_000, _TIERED_MODEL, prompt_token_count=1_000) + assert cost == (500_000 / 1_000_000) * 8.0 + + +def test_output_cost_above_threshold_uses_long_rate(): + cost = output_cost(1_000, _TIERED_MODEL, prompt_token_count=200_001) + assert cost == (1_000 / 1_000_000) * 16.0 + + +def test_output_cost_defaults_prompt_size_to_own_token_count(): + # No prompt_token_count given: falls back to the output's own count. + # Only correct for flat models, but must not crash for a tiered one. + cost = output_cost(200_001, _TIERED_MODEL) + assert cost == (200_001 / 1_000_000) * 16.0 + + +def test_total_cost_tiers_output_by_input_size(): + # 300k input (above threshold) + 1k output must bill output at the + # long-context rate, driven by the input size, not the output size. + cost = total_cost(300_000, 1_000, _TIERED_MODEL) + expected = (300_000 / 1_000_000) * 4.0 + (1_000 / 1_000_000) * 16.0 + assert cost == expected + + +def test_flat_model_ignores_missing_tier_fields(): + # _MODEL has no long_context_threshold -- tiering must be a no-op. + assert input_cost(10_000_000, _MODEL) == (10_000_000 / 1_000_000) * 2.0 diff --git a/tests/test_models_registry.py b/tests/test_models_registry.py index b9f30da..774b766 100644 --- a/tests/test_models_registry.py +++ b/tests/test_models_registry.py @@ -14,15 +14,15 @@ def test_all_models_are_model_info(): assert isinstance(model, ModelInfo) -def test_gpt4o_present(): +def test_gpt_sol_present(): models = load_models() - assert "openai:gpt-4o" in models + assert "openai:gpt-5.6-sol" in models -def test_gpt4o_fields(): - model = get_model("openai:gpt-4o") - assert model.context_window == 128_000 - assert model.input_price_per_million == 2.50 +def test_gpt_sol_fields(): + model = get_model("openai:gpt-5.6-sol") + assert model.context_window == 1_050_000 + assert model.input_price_per_million == 4.00 assert model.tokenizer_backend == "tiktoken" @@ -67,10 +67,10 @@ def test_models_with_no_public_tokenizer_are_flagged_approximate(): """Claude and Gemini have no publicly cacheable local tokenizer, so their counts come from a tiktoken approximation and must say so.""" approximate_ids = [ - "anthropic:claude-3-5-sonnet", - "anthropic:claude-3-haiku", - "google:gemini-2.0-flash", - "google:gemini-1.5-pro", + "anthropic:claude-sonnet-5", + "anthropic:claude-haiku-4.5", + "google:gemini-3.5-flash", + "google:gemini-3.1-pro-preview", ] for model_id in approximate_ids: assert get_model(model_id).tokenizer_is_approximate is True @@ -78,7 +78,7 @@ def test_models_with_no_public_tokenizer_are_flagged_approximate(): def test_models_with_real_hf_tokenizer_are_not_flagged_approximate(): real_tokenizer_ids = [ - "deepseek:deepseek-v3", + "deepseek:deepseek-v4-flash", "meta:llama-3-8b", "mistral:mistral-7b", ] diff --git a/tests/test_registry_data.py b/tests/test_registry_data.py new file mode 100644 index 0000000..9c2c6bd --- /dev/null +++ b/tests/test_registry_data.py @@ -0,0 +1,124 @@ +"""Cross-consistency checks between default_models.yaml and +model_architectures.yaml, and basic sanity checks on the pricing metadata. + +Pure, no network, no fixtures -- reads the same bundled YAML the app ships. +These exist because the Phase 14 audit found real drift (a missing +architecture entry, two files disagreeing about one model's context +window) that nothing in the repo could have caught automatically. See +docs/registry-refresh.md for how to re-verify and update this data. + +No test here may assert a date is "recent" -- a time-bomb that reddens CI +on a quiet Tuesday teaches people to ignore CI, per this file's own +design goal of a guard that doesn't rot. +""" + +from __future__ import annotations + +import datetime +import re + +from norefund.core.architectures import _DEFAULT_ARCHITECTURE_PATH, list_architectures +from norefund.core.models_registry import _DEFAULT_REGISTRY_PATH, list_models + +_models = {m.id: m for m in list_models()} +_architectures = {a.id: a for a in list_architectures()} + + +def _architecture_yaml_blocks() -> dict[str, str]: + """Map architecture id -> its raw YAML block (comments included). + + ModelArchitecture strips comments on load, but the head_dim exception + check needs to see them -- re-read the raw file and split on each + top-level "- id:" entry rather than adding a YAML-comment-preserving + dependency for one test. + """ + text = _DEFAULT_ARCHITECTURE_PATH.read_text(encoding="utf-8") + entries = re.split(r"(?=^- id: )", text, flags=re.MULTILINE) + blocks: dict[str, str] = {} + for entry in entries: + match = re.match(r"^- id: (\S+)", entry) + if match: + blocks[match.group(1)] = entry + return blocks + + +def test_every_architecture_id_exists_in_model_registry(): + for arch_id in _architectures: + assert arch_id in _models, ( + f"model_architectures.yaml has '{arch_id}' with no matching " + f"entry in default_models.yaml" + ) + + +def test_every_self_hosted_model_has_an_architecture_entry(): + # Self-hosted == a real (non-tiktoken) tokenizer backend and no price. + # This is exactly the check that would have caught meta:llama-3-8b + # having no architecture entry before Phase 14. + for model in _models.values(): + is_self_hosted = ( + model.tokenizer_backend != "tiktoken" + and model.input_price_per_million == 0 + and model.output_price_per_million == 0 + ) + if is_self_hosted: + assert model.id in _architectures, ( + f"'{model.id}' is self-hosted (backend={model.tokenizer_backend}, " + f"zero price) but has no model_architectures.yaml entry, so it " + f"can never appear in the Fit Check" + ) + + +def test_shared_ids_agree_on_context_window(): + # This is exactly the check that would have caught DeepSeek V3's two + # files disagreeing (128000 vs 163840) before Phase 14. + shared_ids = set(_models) & set(_architectures) + assert shared_ids, "expected at least one id in both registries" + for shared_id in shared_ids: + model = _models[shared_id] + arch = _architectures[shared_id] + assert model.context_window == arch.max_context_length, ( + f"'{shared_id}': default_models.yaml says context_window=" + f"{model.context_window}, model_architectures.yaml says " + f"max_context_length={arch.max_context_length}" + ) + + +def test_every_priced_entry_has_docs_url_and_verification_date(): + priced = [m for m in _models.values() if m.input_price_per_million > 0] + assert priced, "expected at least one priced entry" + for model in priced: + assert model.docs_url is not None, f"'{model.id}' has a price but no docs_url" + assert model.pricing_verified_on is not None, ( + f"'{model.id}' has a price but no pricing_verified_on" + ) + + +def test_every_pricing_verified_on_parses_as_a_date(): + for model in _models.values(): + if model.pricing_verified_on is not None: + # Raises ValueError (failing the test) if it isn't a real + # ISO date -- deliberately not asserting anything about how + # recent it is. + datetime.date.fromisoformat(model.pricing_verified_on) + + +def test_head_dim_times_attention_heads_matches_hidden_size_or_is_explained(): + blocks = _architecture_yaml_blocks() + for arch in _architectures.values(): + if arch.head_dim * arch.n_attention_heads == arch.hidden_size: + continue + block = blocks.get(arch.id, "") + assert "head_dim" in block, ( + f"'{arch.id}': head_dim ({arch.head_dim}) * n_attention_heads " + f"({arch.n_attention_heads}) != hidden_size ({arch.hidden_size}), " + f"and model_architectures.yaml has no comment on this entry " + f"explaining why (see the Gemma 2 9B entry for the expected form)" + ) + + +def test_registry_yaml_path_is_the_real_bundled_file(): + # Sanity check on the test's own re-read of the raw file: make sure + # it's reading the same path the dataclass loader used, not a stale + # copy elsewhere on disk. + assert _DEFAULT_ARCHITECTURE_PATH.exists() + assert _DEFAULT_REGISTRY_PATH.exists() diff --git a/tests/test_selfhost.py b/tests/test_selfhost.py index adc4f64..4ae2004 100644 --- a/tests/test_selfhost.py +++ b/tests/test_selfhost.py @@ -14,6 +14,7 @@ estimate_memory, evaluate_fit, framework_overhead_bytes, + kv_cache_bytes, kv_cache_bytes_per_token, max_concurrent_requests, usable_memory_bytes, @@ -100,6 +101,24 @@ attention_type="gqa", ) +_GEMMA_2_9B = ModelArchitecture( + id="google:gemma-2-9b", + display_name="Gemma 2 9B", + family="Gemma 2", + vendor="Google", + total_params=9_240_000_000, + active_params=9_240_000_000, + n_layers=42, + n_attention_heads=16, + n_kv_heads=8, + head_dim=256, + hidden_size=3584, + max_context_length=8192, + attention_type="gqa", + sliding_window=4096, + sliding_window_pattern=2, # alternates local/global 1:1 +) + _DEEPSEEK_V3 = ModelArchitecture( id="deepseek:deepseek-v3", display_name="DeepSeek V3", @@ -284,6 +303,46 @@ def test_llama_70b_kv_cache_uses_kv_heads_not_attention_heads(): assert kv_cache_bytes_per_token(_LLAMA_70B) == 327_680 +# --- Gemma 2 sliding-window KV cache (Task 4, Phase 14) --- + + +def test_gemma2_9b_sliding_window_below_full_figure_at_2x_window(): + # 42 layers alternate 1:1 (21 full, 21 windowed to 4096). At 8192 + # (2x the window), the windowed half only pays for 4096 tokens each, + # giving exactly 75% of the naive all-layers-full figure -- a real + # ~25% reduction, not just "some" reduction. + full_figure = kv_cache_bytes_per_token(_GEMMA_2_9B) * 8192 + windowed = kv_cache_bytes(_GEMMA_2_9B, 8192) + assert windowed == round(full_figure * 0.75) + assert windowed < full_figure + + +def test_gemma2_9b_sliding_window_equals_full_figure_at_or_under_window(): + # At exactly the window size, every layer -- windowed or not -- caches + # the whole context, so this must equal the naive figure exactly. + assert kv_cache_bytes(_GEMMA_2_9B, 4096) == kv_cache_bytes_per_token( + _GEMMA_2_9B + ) * 4096 + + +def test_gqa_model_with_no_sliding_window_matches_per_token_figure(): + # _LLAMA_70B has sliding_window=0 (the default) -- kv_cache_bytes must + # return exactly what multiplying the existing per-token figure by + # context_length already gave, for every architecture written before + # this phase. + context_length = 32_768 + assert kv_cache_bytes(_LLAMA_70B, context_length) == ( + kv_cache_bytes_per_token(_LLAMA_70B) * context_length + ) + + +def test_deepseek_v3_mla_path_unchanged_by_sliding_window_support(): + context_length = 65_536 + assert kv_cache_bytes(_DEEPSEEK_V3, context_length) == ( + kv_cache_bytes_per_token(_DEEPSEEK_V3) * context_length + ) + + # --- Hand-computed scenarios --- diff --git a/tests/test_service.py b/tests/test_service.py index 665a85c..66ba3ed 100644 --- a/tests/test_service.py +++ b/tests/test_service.py @@ -25,55 +25,55 @@ def sample_folder(tmp_path: Path) -> Path: def test_analyze_file_returns_result(sample_txt: Path): - result = analyze_file(sample_txt, "openai:gpt-4o") + result = analyze_file(sample_txt, "openai:gpt-5.6-sol") assert isinstance(result, AnalysisResult) def test_analyze_file_token_count_positive(sample_txt: Path): _skip_unless_cached("o200k_base") - result = analyze_file(sample_txt, "openai:gpt-4o") + result = analyze_file(sample_txt, "openai:gpt-5.6-sol") assert result.token_count > 0 def test_analyze_file_correct_model(sample_txt: Path): - result = analyze_file(sample_txt, "openai:gpt-4o") - assert result.model_id == "openai:gpt-4o" + result = analyze_file(sample_txt, "openai:gpt-5.6-sol") + assert result.model_id == "openai:gpt-5.6-sol" def test_analyze_file_fits_small_doc(sample_txt: Path): # A tiny txt file must fit in any model's context window _skip_unless_cached("o200k_base") - result = analyze_file(sample_txt, "openai:gpt-4o") + result = analyze_file(sample_txt, "openai:gpt-5.6-sol") assert result.fits_in_context is True assert result.min_chunks_needed == 1 def test_analyze_file_context_usage_under_100(sample_txt: Path): _skip_unless_cached("o200k_base") - result = analyze_file(sample_txt, "openai:gpt-4o") + result = analyze_file(sample_txt, "openai:gpt-5.6-sol") assert result.context_usage_pct < 100 def test_analyze_file_char_and_word_count(sample_txt: Path): _skip_unless_cached("o200k_base") - result = analyze_file(sample_txt, "openai:gpt-4o") + result = analyze_file(sample_txt, "openai:gpt-5.6-sol") assert result.char_count > 0 assert result.word_count > 0 def test_analyze_folder_returns_list(sample_folder: Path): - results = analyze_folder(sample_folder, "openai:gpt-4o") + results = analyze_folder(sample_folder, "openai:gpt-5.6-sol") assert isinstance(results, list) def test_analyze_folder_ignores_unsupported(sample_folder: Path): - results = analyze_folder(sample_folder, "openai:gpt-4o") + results = analyze_folder(sample_folder, "openai:gpt-5.6-sol") # Only .txt and .md should be picked up, not .exe assert len(results) == 2 def test_analyze_folder_all_results_valid(sample_folder: Path): - results = analyze_folder(sample_folder, "openai:gpt-4o") + results = analyze_folder(sample_folder, "openai:gpt-5.6-sol") assert all(isinstance(r, AnalysisResult) for r in results) @@ -82,6 +82,6 @@ def test_analyze_folder_does_not_descend_into_subfolders(sample_folder: Path): nested.mkdir() (nested / "c.txt").write_text("File C content, inside a subfolder") - results = analyze_folder(sample_folder, "openai:gpt-4o") + results = analyze_folder(sample_folder, "openai:gpt-5.6-sol") assert {Path(r.file_path).name for r in results} == {"a.txt", "b.md"} diff --git a/tests/test_tokenization.py b/tests/test_tokenization.py index b65869f..2bf770d 100644 --- a/tests/test_tokenization.py +++ b/tests/test_tokenization.py @@ -53,7 +53,7 @@ def test_tiktoken_fallback_unknown_model(): def test_get_tokenizer_returns_tiktoken_for_gpt4o(): _skip_unless_cached("o200k_base") - model = get_model("openai:gpt-4o") + model = get_model("openai:gpt-5.6-sol") backend = get_tokenizer(model) assert isinstance(backend, TikTokenBackend)