diff --git a/CLAUDE.md b/CLAUDE.md index 65e167e..842167b 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -43,7 +43,7 @@ Aggregation (agg/aggOpenAI) └─ Consolidates usage buckets by model/key/workspace ↓ Analysis Engine (findIssues/findIssuesOpenAI) - └─ Runs the vendor's rule set (9 Anthropic / 13 OpenAI) with confidence + └─ Runs the vendor's rule set (10 Anthropic / 14 OpenAI) with confidence scoring; optional NIM consensus merges LLM proposals on top, priced by lib/{vendor}/costing.ts — the LLM never emits dollar figures ↓ @@ -78,7 +78,7 @@ History page → Detail routes → /recommendations, /analytics, /raw-data - Service-level analytics: tracks spending by service type (embeddings, completions, audio, images, etc.) - Per-project service breakdown charts showing monthly usage across all services -### Analysis Engine (22 rules: 9 Anthropic, 13 OpenAI) +### Analysis Engine (24 rules: 10 Anthropic, 14 OpenAI) Located in `lib/anthropic/analysis.ts` and `lib/openai/analysis.ts` — the `RULE n:` comment headers in those files are the authoritative list. The core @@ -110,6 +110,12 @@ categories: 6. **Legacy Model** (Anthropic only) - Outdated generation (g=1) still in use +7. **Cross-Vendor Comparison** (both vendors, one org-level finding per report) + - Reprices the whole org on the other vendor's nearest-equivalent tiers + - Both sides priced from the pricing tables (never OpenAI's billed `r.cost`) + - Always INFO, always `sav: 0` — never contributes to report savings + - Gated off below $50/mo org spend or a delta under 10% of that spend + **Confidence Scoring:** Multi-signal weighted system (0-1 scale) - Signals: usage volume, consistency (CoV), active days, input variance, temporal patterns @@ -126,6 +132,7 @@ categories: - `lib/anthropic/costing.ts` - deterministic optimized-cost formulas per category (OpenAI parallel in `lib/openai/costing.ts`) - `lib/nim/analysis.ts` - NIM LLM proposals, deterministic pricing, consensus merge (`source: rules|llm|both`) - `lib/savingsCap.ts` - caps cumulative per-row savings at the row's spend, applied by every engine output +- `lib/crossVendor.ts` - cross-vendor tier mapping + comparison math, imported by both engines (rules-only, never NIM) - `lib/anthropic/pricing.ts` - Model pricing table, cost calculation (`tc()` function) - `lib/openai/*` - Parallel implementations for OpenAI - `lib/formatters.ts` - Currency (`$`) and percentage (`P`) formatters diff --git a/PRODUCT.md b/PRODUCT.md index b32e152..8f29cfb 100644 --- a/PRODUCT.md +++ b/PRODUCT.md @@ -11,7 +11,7 @@ Built as part of Nick's Omilia internship; doubles as a portfolio project. ## Where it stands (2026-08-06) -- **22 detection rules** (9 Anthropic, 13 OpenAI) with weighted-signal +- **24 detection rules** (10 Anthropic, 14 OpenAI) with weighted-signal confidence scoring and per-finding signal trails. - **Consensus engine**: optional NVIDIA NIM augmentation runs alongside the rules, proposes findings from a fixed category set, and never prices — @@ -25,7 +25,7 @@ Built as part of Nick's Omilia internship; doubles as a portfolio project. incident month) or a lean startup (1-2 minor findings, on purpose). Each run is internally coherent across its 6 months; the generators are pure and clock-free, reproducible via explicit seeds (`DEMO_SEED` in tests). -- 117 vitest tests pin the money math; CI runs type-check, lint, format, test. +- 138 vitest tests pin the money math; CI runs type-check, lint, format, test. - AGPL-3.0. Corrected 2026-08-08: `prOpenAI` matched pricing-table keys in insertion @@ -47,8 +47,13 @@ Next candidates, in rough order (none committed): responses; needs a real key and a real llama round-trip. The demo now makes exactly one NIM call per run (current month only), so it doubles as the cheapest live test. -2. **Cross-vendor comparison** — both pricing tables are in the repo; "this - workload on the other vendor" is a finding single-vendor tools can't make. + +Shipped from this list 2026-08-08: **cross-vendor comparison** — one org-level +INFO finding per report reprices the whole analyzed org on the other vendor's +nearest-equivalent tiers (`src/lib/crossVendor.ts`, imported by both engines). +Both sides are priced from the pricing tables, never from OpenAI's billed +`r.cost`, which has no Anthropic counterpart. It contributes zero savings by +construction and gates itself off below $50/mo or a sub-10% delta. Shipped from this list 2026-08-06: temporal/scheduling rules (weekly-cadence 5d/4c, steady-batch 5c with a weekend-parity gate that keeps batch advice off diff --git a/README.md b/README.md index b3bbdeb..b2fe959 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,7 @@ _an LLM spend auditor that reads your Admin API and tells you where the money le ![telemetry](https://img.shields.io/badge/telemetry-0-3fb950?style=flat-square&labelColor=111111) ![keys](https://img.shields.io/badge/your_API_keys-never_leave_the_browser-3fb950?style=flat-square&labelColor=111111) ![access](https://img.shields.io/badge/access-read--only._we_touch_nothing-d4a017?style=flat-square&labelColor=111111) -![rules](https://img.shields.io/badge/detection_rules-22-d4a017?style=flat-square&labelColor=111111) +![rules](https://img.shields.io/badge/detection_rules-24-d4a017?style=flat-square&labelColor=111111) ![engine](https://img.shields.io/badge/AI_augmentation-detects%2C_never_prices-3fb950?style=flat-square&labelColor=111111) ![savings]() @@ -32,7 +32,7 @@ _an LLM spend auditor that reads your Admin API and tells you where the money le Most teams discover their LLM bill the way you discover a parking ticket. TokenPilot reads your **Anthropic and OpenAI Admin APIs**, runs your actual -token volumes through a **22-rule detection engine** (9 Anthropic, 13 OpenAI), +token volumes through a **24-rule detection engine** (10 Anthropic, 14 OpenAI), and hands back confidence-scored savings recommendations in about 60 seconds. No theoretical benchmarks — every dollar figure is computed from _your_ usage. @@ -41,7 +41,7 @@ the tab does, and never gets sent anywhere except the vendor it belongs to. ```console nick@tokenpilot:~$ audit --vendor anthropic -[✓] 9 rules executed. 4 findings. estimated waste: not zero. +[✓] 10 rules executed. 4 findings. estimated waste: not zero. [i] your keys stayed in this tab. as is tradition. ``` @@ -58,6 +58,7 @@ nick@tokenpilot:~$ audit --vendor anthropic | 07 | **reasoning-model overkill** | o-series reasoning premiums spent on non-reasoning work | | 08 | **quality upgrade** | the reverse case — places a smarter model would pay for itself | | 09 | **legacy model usage** | deprecated models quietly costing more for less | +| 10 | **cross-vendor comparison** | your exact token volumes repriced on the other vendor's tiers | Each finding gets **multi-signal confidence scoring** — volume, consistency, active days, temporal patterns — and carries its **signal trail** ("based on: diff --git a/src/__tests__/cross-vendor.test.ts b/src/__tests__/cross-vendor.test.ts new file mode 100644 index 0000000..440a380 --- /dev/null +++ b/src/__tests__/cross-vendor.test.ts @@ -0,0 +1,346 @@ +import { describe, it, expect } from "vitest"; +import { + anthropicToOpenAI, + compareVendors, + counterpartModel, + crossVendorFinding, + MIN_ORG_SPEND, + openAIToAnthropic, +} from "@/lib/crossVendor"; +import { capRowSavings } from "@/lib/savingsCap"; +import { findIssues } from "@/lib/anthropic/analysis"; +import { findIssuesOpenAI } from "@/lib/openai/analysis"; +import type { OpenAIAggregatedRow } from "@/lib/openai/analysis"; +import { tcOpenAI } from "@/lib/openai/pricing"; +import { tc } from "@/lib/anthropic/pricing"; +import type { AggregatedRow, Finding } from "@/types"; +import { AnthropicCategory, OpenAICategory, Severity } from "@/types/analysis"; + +const XV = "Cross-Vendor Comparison"; + +const isCrossVendor = (f: Finding) => f.cat === XV; + +function anthRow(over: Partial = {}): AggregatedRow { + return { + model: "claude-opus-4-6", + kid: "key_1", + wid: "ws_1", + inp: 100e6, + out: 5e6, + cached: 0, + cacheCreated: 0, + reqs: 5_000, + activeDays: 25, + ...over, + }; +} + +function oaiRow(over: Partial = {}): OpenAIAggregatedRow { + return { + model: "gpt-4o", + project_id: "proj_1", + line_item: "completions", + cost: 0, + inp: 100e6, + out: 5e6, + reqs: 5_000, + activeDays: 25, + ...over, + }; +} + +/* ─── 1. Tier mapping round-trips on current-generation models ─── */ + +describe("tier mapping", () => { + it("round-trips current-generation models in both directions", () => { + // Anthropic → OpenAI → Anthropic + expect(anthropicToOpenAI("opus-4-6")).toBe("o3"); + expect(openAIToAnthropic("o3")).toBe("opus-4-6"); + + expect(anthropicToOpenAI("sonnet-4-6")).toBe("gpt-4o"); + expect(openAIToAnthropic("gpt-4o")).toBe("sonnet-4-6"); + + expect(anthropicToOpenAI("haiku-4-5")).toBe("gpt-4o-mini"); + expect(openAIToAnthropic("gpt-4o-mini")).toBe("haiku-4-5"); + + // The same three, expressed as a loop over both directions. + for (const m of ["opus-4-6", "sonnet-4-6", "haiku-4-5"]) { + expect(openAIToAnthropic(anthropicToOpenAI(m))).toBe(m); + } + for (const m of ["o3", "gpt-4o", "gpt-4o-mini"]) { + expect(anthropicToOpenAI(openAIToAnthropic(m))).toBe(m); + } + }); + + it("maps the rest of both tables to their capability class", () => { + // Anthropic → OpenAI, including dated and legacy ids. + expect(anthropicToOpenAI("claude-opus-4-5-20260101")).toBe("o3"); + expect(anthropicToOpenAI("claude-3-opus")).toBe("o3"); + expect(anthropicToOpenAI("claude-sonnet-3-5")).toBe("gpt-4o"); + expect(anthropicToOpenAI("claude-3-haiku")).toBe("gpt-4o-mini"); + + // OpenAI → Anthropic, per the documented mapping. + expect(openAIToAnthropic("o1")).toBe("opus-4-6"); + expect(openAIToAnthropic("o1-preview")).toBe("opus-4-6"); + expect(openAIToAnthropic("o1-mini")).toBe("sonnet-4-6"); + expect(openAIToAnthropic("o3-mini")).toBe("sonnet-4-6"); + expect(openAIToAnthropic("gpt-4-turbo")).toBe("sonnet-4-6"); + expect(openAIToAnthropic("gpt-4")).toBe("sonnet-4-6"); + expect(openAIToAnthropic("gpt-4-32k")).toBe("sonnet-4-6"); + expect(openAIToAnthropic("gpt-3.5-turbo")).toBe("haiku-4-5"); + }); + + it("counterpartModel dispatches on vendor", () => { + expect(counterpartModel("anthropic", "claude-haiku-4-5")).toBe( + "gpt-4o-mini" + ); + expect(counterpartModel("openai", "gpt-4o-mini")).toBe("haiku-4-5"); + }); +}); + +/* ─── 2. The finding never moves report totals ─── */ + +describe("savings neutrality", () => { + const savings = (fs: Finding[]) => fs.reduce((s, f) => s + f.sav, 0); + const highConfSavings = (fs: Finding[]) => + fs.filter((f) => f.conf >= 0.65).reduce((s, f) => s + f.sav, 0); + + it("leaves report.savings and highConfSavings identical", () => { + // An org large enough that the comparison fires alongside real findings. + const rows = [ + anthRow(), + anthRow({ + model: "claude-sonnet-4-6", + kid: "key_2", + inp: 60e6, + out: 4e6, + }), + anthRow({ model: "claude-haiku-4-5", kid: "key_3", inp: 40e6, out: 2e6 }), + ]; + const findings = findIssues(rows, [], []); + + expect(findings.some(isCrossVendor)).toBe(true); + + const without = findings.filter((f) => !isCrossVendor(f)); + expect(savings(findings)).toBe(savings(without)); + expect(highConfSavings(findings)).toBe(highConfSavings(without)); + }); + + it("is always zero-savings, opt === cur, and INFO", () => { + const f = crossVendorFinding("anthropic", [ + { model: "claude-opus-4-6", inp: 100e6, out: 5e6 }, + ])!; + + expect(f).not.toBeNull(); + expect(f.sav).toBe(0); + expect(f.opt).toBe(f.cur); + expect(f.sev).toBe(Severity.INFO); + expect(f.cat).toBe(AnthropicCategory.CROSS_VENDOR_COMPARISON); + expect(f.source).toBe("rules"); + expect(f.impact).toContain("not in savings total"); + }); + + it("uses the OpenAI category on the OpenAI side", () => { + const f = crossVendorFinding("openai", [ + { model: "o1", inp: 40e6, out: 2e6 }, + ])!; + expect(f.cat).toBe(OpenAICategory.CROSS_VENDOR_COMPARISON); + expect(f.ws).toBe("All projects"); + }); +}); + +/* ─── 3. capRowSavings keeps it (regression on the regex change) ─── */ + +describe("capRowSavings", () => { + it("keeps the zero-savings cross-vendor finding", () => { + const f = crossVendorFinding("anthropic", [ + { model: "claude-opus-4-6", inp: 100e6, out: 5e6 }, + ])!; + expect(capRowSavings([f])).toHaveLength(1); + }); + + it("keeps it alongside cost findings that consume the row headroom", () => { + const f = crossVendorFinding("openai", [ + { model: "gpt-4o", inp: 100e6, out: 5e6 }, + ])!; + const rows = findIssuesOpenAI([oaiRow()], []); + expect(capRowSavings([...rows, f]).some(isCrossVendor)).toBe(true); + }); +}); + +/* ─── 4. The gate ─── */ + +describe("gate", () => { + it("skips orgs under $50/mo", () => { + // Opus at these volumes lands well under the floor. + const rows = [{ model: "claude-opus-4-6", inp: 1e6, out: 0.1e6 }]; + expect(compareVendors("anthropic", rows).cur).toBeLessThan(MIN_ORG_SPEND); + expect(crossVendorFinding("anthropic", rows)).toBeNull(); + }); + + it("skips a delta under 10% of org spend", () => { + // GPT-4 Turbo ($5/$15) vs Sonnet 4.6 ($3/$15): identical output pricing, + // so an output-heavy workload lands inside the 10% band — "roughly the + // same", which is noise rather than a finding. + const rows = [{ model: "gpt-4-turbo", inp: 5e6, out: 10e6 }]; + const c = compareVendors("openai", rows); + + expect(c.cur).toBeGreaterThanOrEqual(MIN_ORG_SPEND); + expect(Math.abs(c.delta) / c.cur).toBeLessThan(0.1); + expect(crossVendorFinding("openai", rows)).toBeNull(); + }); + + it("fires once a big enough org shows a big enough delta", () => { + const rows = [{ model: "claude-opus-4-6", inp: 100e6, out: 5e6 }]; + const c = compareVendors("anthropic", rows); + + expect(c.cur).toBeGreaterThanOrEqual(MIN_ORG_SPEND); + expect(Math.abs(c.delta) / c.cur).toBeGreaterThanOrEqual(0.1); + expect(crossVendorFinding("anthropic", rows)).not.toBeNull(); + }); + + it("emits at most one finding per report", () => { + const rows = [ + anthRow(), + anthRow({ model: "claude-sonnet-4-6", kid: "key_2" }), + anthRow({ model: "claude-haiku-4-5", kid: "key_3" }), + ]; + expect(findIssues(rows, [], []).filter(isCrossVendor)).toHaveLength(1); + }); +}); + +/* ─── 5. Both sides priced from the tables, never from r.cost ─── */ + +describe("table pricing on both sides", () => { + it("ignores an OpenAI row's billed cost in favour of the table figure", () => { + // A row whose billed spend is wildly out of line with what the pricing + // table says those tokens cost. Every other finding trusts r.cost; this + // one must not, because the Anthropic side has no billed equivalent. + const inp = 100e6; + const out = 5e6; + const table = tcOpenAI("gpt-4o", inp, out); + const billed = table * 10; + + const withBilled = findIssuesOpenAI( + [oaiRow({ cost: billed, inp, out })], + [] + ).find(isCrossVendor)!; + const withoutBilled = findIssuesOpenAI( + [oaiRow({ cost: 0, inp, out })], + [] + ).find(isCrossVendor)!; + + expect(withBilled).toBeDefined(); + expect(withBilled.cur).toBeCloseTo(table, 6); + expect(withBilled.cur).not.toBeCloseTo(billed, 6); + // Billed spend changes nothing about the comparison. + expect(withBilled.cur).toBeCloseTo(withoutBilled.cur, 6); + expect(withBilled.impact).toBe(withoutBilled.impact); + }); + + it("prices the counterpart side from the other vendor's table", () => { + const inp = 100e6; + const out = 5e6; + const c = compareVendors("anthropic", [ + { model: "claude-opus-4-6", inp, out }, + ]); + + expect(c.cur).toBeCloseTo(tc("opus-4-6", inp, out), 6); + expect(c.alt).toBeCloseTo(tcOpenAI("o3", inp, out), 6); + expect(c.delta).toBeCloseTo(c.alt - c.cur, 6); + }); + + it("ignores cached tokens on both sides, matching tc()", () => { + const bare = compareVendors("anthropic", [ + { model: "claude-opus-4-6", inp: 100e6, out: 5e6 }, + ]); + const cachedHeavy = compareVendors("anthropic", [ + { model: "claude-opus-4-6", inp: 100e6, out: 5e6, cached: 400e6 }, + ]); + + expect(cachedHeavy.cur).toBeCloseTo(bare.cur, 6); + expect(cachedHeavy.alt).toBeCloseTo(bare.alt, 6); + // Cache reads still inform the confidence signal and the caveat text. + expect(cachedHeavy.cacheRate).toBeGreaterThan(0.1); + expect(bare.cacheRate).toBe(0); + }); +}); + +/* ─── Rollup and narrative ─── */ + +describe("per-tier rollup", () => { + const rows = [ + { model: "claude-opus-4-6", inp: 100e6, out: 5e6 }, + { model: "claude-sonnet-4-6", inp: 60e6, out: 4e6 }, + { model: "claude-haiku-4-5", inp: 40e6, out: 2e6 }, + ]; + + it("groups to one class per capability tier, biggest spend first", () => { + const c = compareVendors("anthropic", rows); + + expect(c.tiers.map((t) => t.key)).toEqual(["o3", "gpt-4o", "gpt-4o-mini"]); + expect(c.tiers.map((t) => t.ownLabel)).toEqual([ + "Opus 4.6", + "Sonnet 4.6", + "Haiku 4.5", + ]); + expect(c.cur).toBeCloseTo( + c.tiers.reduce((s, t) => s + t.own, 0), + 6 + ); + expect(c.alt).toBeCloseTo( + c.tiers.reduce((s, t) => s + t.other, 0), + 6 + ); + }); + + it("labels each tier line with the counterpart and its delta", () => { + const f = crossVendorFinding("anthropic", rows)!; + + expect(f.reason).toContain("Opus 4.6"); + expect(f.reason).toContain("Sonnet 4.6"); + expect(f.reason).toMatch(/\([-+]\d+%\)/); + expect(f.reason.match(/→/g)).toHaveLength(3); + }); + + it("scores the four documented signals", () => { + // Small org, mixed tiers, no cache: only the direct-match and cache + // signals are met (0.25 + 0.20 of a 1.00 total). + const f = crossVendorFinding("anthropic", rows)!; + const c = compareVendors("anthropic", rows); + + expect(c.directMatchShare).toBe(1); + expect(f.conf).toBeGreaterThan(0); + expect(f.conf).toBeLessThanOrEqual(1); + }); + + it("drops the direct-match share when models fall through to the default", () => { + const c = compareVendors("anthropic", [ + { model: "claude-opus-4-6", inp: 100e6, out: 5e6 }, + { model: "some-unreleased-opus", inp: 100e6, out: 5e6 }, + ]); + expect(c.directMatchShare).toBeGreaterThan(0); + expect(c.directMatchShare).toBeLessThan(1); + }); +}); + +describe("action caveats", () => { + it("always names the repricing and migration-cost caveats", () => { + const f = crossVendorFinding("anthropic", [ + { model: "claude-opus-4-6", inp: 100e6, out: 5e6 }, + ])!; + + expect(f.action).toContain("not a quality claim"); + expect(f.action).toContain("Migration cost is excluded"); + expect(f.action).not.toContain("90% off"); + }); + + it("adds the cache-discount caveat once the org cache rate reaches 10%", () => { + const f = crossVendorFinding("anthropic", [ + { model: "claude-opus-4-6", inp: 100e6, out: 5e6, cached: 400e6 }, + ])!; + + expect(f.action).toContain("90% off"); + expect(f.action).toContain("50% off"); + }); +}); diff --git a/src/lib/anthropic/analysis.ts b/src/lib/anthropic/analysis.ts index 6f7e4c8..2d1bf25 100644 --- a/src/lib/anthropic/analysis.ts +++ b/src/lib/anthropic/analysis.ts @@ -28,6 +28,7 @@ import { } from "./costing"; import { $, P } from "@/lib/formatters"; import { capRowSavings } from "@/lib/savingsCap"; +import { crossVendorFinding } from "@/lib/crossVendor"; /* ═══════════════════ AGGREGATION ═══════════════════ */ @@ -808,6 +809,13 @@ After routing, return to this view next month — you should see spend distribut }); } + /* ─── CROSS-VENDOR COMPARISON ─── */ + // One org-level, zero-savings INFO finding: the whole workload repriced on + // OpenAI's nearest-equivalent tiers. Self-gating — returns null when the + // org is too small or the two vendors land within 10% of each other. + const crossVendor = crossVendorFinding("anthropic", rows); + if (crossVendor) out.push(crossVendor); + return capRowSavings(out).sort((a, b) => { const sv: Record = { [Severity.CRITICAL]: 0, diff --git a/src/lib/crossVendor.ts b/src/lib/crossVendor.ts new file mode 100644 index 0000000..3981cab --- /dev/null +++ b/src/lib/crossVendor.ts @@ -0,0 +1,350 @@ +/* ═══════════════════ CROSS-VENDOR COMPARISON ═══════════════════ */ +/* + * Both pricing tables live in this repo, so "what would this exact workload + * cost on the other vendor?" is answerable without a single extra API call — + * and it's the one finding a single-vendor tool structurally cannot make. + * + * This module owns the tier mapping (which model on vendor B stands in for a + * model on vendor A) and the comparison math. Both engines import it; neither + * reaches into the other vendor's pricing table directly. + * + * Two deliberate constraints: + * + * 1. BOTH sides are priced from the pricing tables (`tc` / `tcOpenAI`) on the + * row's input and output tokens. The OpenAI engine normally prefers the + * real billed figure from the Costs API (`r.cost`), but the Anthropic side + * has no equivalent — mixing billed spend against table pricing would + * compare two different things. `r.cost` remains the source of truth for + * `cur` on every other finding; only this comparison overrides it. + * + * 2. The finding never contributes savings. It is an INFO-only repricing of + * identical token volumes, not an optimization the user can bank, so `sav` + * is always 0 and `opt` always equals `cur`. + */ + +import type { Finding, FindingSignal } from "@/types"; +import { + AnthropicCategory, + AnthropicModelTier, + OpenAICategory, + OpenAIModelTier, + Severity, +} from "@/types/analysis"; +import { confidenceScore } from "@/lib/anthropic/analysis"; +import { pr, tc } from "@/lib/anthropic/pricing"; +import { prOpenAI, tcOpenAI } from "@/lib/openai/pricing"; +import { $ } from "@/lib/formatters"; + +export type Vendor = "anthropic" | "openai"; + +/** Everything the comparison needs from one aggregated usage row. */ +export interface CrossVendorRow { + model: string; + inp: number; // input tokens / mo + out: number; // output tokens / mo + cached?: number; // cache read tokens / mo (Anthropic only) +} + +/** Org spend below this is too small for a migration argument to mean much. */ +export const MIN_ORG_SPEND = 50; + +/** A delta this thin reads as "roughly the same" — noise, not a finding. */ +export const MIN_DELTA_SHARE = 0.1; + +/* ─── TIER MAPPING ─── */ + +/** + * Anthropic → OpenAI: the current-generation OpenAI model in the same + * capability class. Driven by the pricing table's tier, so dated model ids + * ("claude-opus-4-6-20260101") map the same as bare ones. + */ +export function anthropicToOpenAI(model: string): string { + switch (pr(model).t) { + case AnthropicModelTier.OPUS: + return "o3"; + case AnthropicModelTier.HAIKU: + return "gpt-4o-mini"; + default: + return "gpt-4o"; + } +} + +/** + * OpenAI → Anthropic: the current-generation Claude model in the same + * capability class. + * + * The mini variants are matched by name before falling back to the pricing + * tier, because `OpenAIModelTier` lumps o1 with o1-mini and o3 with o3-mini — + * and because `prOpenAI` resolves "gpt-4o-mini" through its "gpt-4o" entry, + * so its tier can't distinguish the two either. + */ +export function openAIToAnthropic(model: string): string { + const k = (model || "").toLowerCase(); + if (k.includes("gpt-4o-mini") || k.includes("gpt-3.5")) return "haiku-4-5"; + if (k.includes("o1-mini") || k.includes("o3-mini")) return "sonnet-4-6"; + + switch (prOpenAI(model).t) { + case OpenAIModelTier.O1: + case OpenAIModelTier.O3: + return "opus-4-6"; + case OpenAIModelTier.GPT4O_MINI: + case OpenAIModelTier.GPT3_5: + return "haiku-4-5"; + default: + // GPT-4o, GPT-4 Turbo, GPT-4, GPT-4 32k, and anything unrecognized. + return "sonnet-4-6"; + } +} + +/** The counterpart model for a row, in whichever direction applies. */ +export function counterpartModel(vendor: Vendor, model: string): string { + return vendor === "anthropic" + ? anthropicToOpenAI(model) + : openAIToAnthropic(model); +} + +/* ─── COMPARISON MATH ─── */ + +/** One capability class: what the org spends there now vs. on the other side. */ +export interface CrossVendorTier { + /** The counterpart model id — the anchor both sides are grouped by. */ + key: string; + ownLabel: string; + otherLabel: string; + own: number; // $/mo on the current vendor + other: number; // $/mo on the counterpart vendor +} + +export interface CrossVendorComparison { + vendor: Vendor; + cur: number; // org total on the current vendor, table-priced + alt: number; // org total on the counterpart vendor, table-priced + delta: number; // alt - cur; negative means the other vendor is cheaper + /** Tier classes carrying spend, biggest first. */ + tiers: CrossVendorTier[]; + /** Share of org spend priced from a real table entry (not the g:0 default). */ + directMatchShare: number; + /** Org-wide cache read rate; always 0 for OpenAI, which exposes no stats. */ + cacheRate: number; +} + +/* + * KNOWN UPSTREAM ISSUE (pre-existing, not introduced here): `prOpenAI` walks + * MP_OPENAI in insertion order with `k.includes(key)`, so "gpt-4o-mini" + * matches the "gpt-4o" entry first and prices at $2.50/$10 instead of + * $0.15/$0.60 — likewise "o1-mini" → o1, "o3-mini" → o3, "gpt-4-32k" → gpt-4. + * + * That inflates the Haiku ↔ GPT-4o-mini class in both directions and can flip + * the sign of its per-tier delta. It is left alone deliberately: `prOpenAI` + * feeds every existing OpenAI rule's `cur` and savings, so correcting it here + * would move the whole engine and the calibration sweep with it. Pricing both + * sides through the same resolver at least keeps each printed label and its + * dollar figure consistent with one another. Fix belongs in its own change. + */ +const priceOn = (vendor: Vendor, model: string, inp: number, out: number) => + vendor === "anthropic" ? tc(model, inp, out) : tcOpenAI(model, inp, out); + +const labelOn = (vendor: Vendor, model: string) => + vendor === "anthropic" ? pr(model).l : prOpenAI(model).l; + +const generationOn = (vendor: Vendor, model: string) => + vendor === "anthropic" ? pr(model).g : prOpenAI(model).g; + +/** + * Reprice every row on the other vendor's nearest-equivalent tier and roll the + * result up per capability class, then to an org total. + * + * Cached and cache-created tokens are ignored throughout, matching how `tc()` + * computes `cur` for every other finding. + */ +export function compareVendors( + vendor: Vendor, + rows: CrossVendorRow[] +): CrossVendorComparison { + const other: Vendor = vendor === "anthropic" ? "openai" : "anthropic"; + + // Per class: own/other spend, plus the own-side model carrying the most of + // it, which supplies the class label in the reason text. + const classes = new Map< + string, + { own: number; other: number; topModel: string; topSpend: number } + >(); + + let cur = 0; + let alt = 0; + let directMatched = 0; + let inp = 0; + let cached = 0; + + for (const r of rows) { + inp += r.inp; + cached += r.cached ?? 0; + + const ownCost = priceOn(vendor, r.model, r.inp, r.out); + if (ownCost <= 0) continue; + + const key = counterpartModel(vendor, r.model); + const otherCost = priceOn(other, key, r.inp, r.out); + + cur += ownCost; + alt += otherCost; + if (generationOn(vendor, r.model) > 0) directMatched += ownCost; + + const c = classes.get(key) ?? { + own: 0, + other: 0, + topModel: r.model, + topSpend: 0, + }; + c.own += ownCost; + c.other += otherCost; + if (ownCost > c.topSpend) { + c.topSpend = ownCost; + c.topModel = r.model; + } + classes.set(key, c); + } + + const tiers: CrossVendorTier[] = [...classes.entries()] + .map(([key, c]) => ({ + key, + ownLabel: labelOn(vendor, c.topModel), + otherLabel: labelOn(other, key), + own: c.own, + other: c.other, + })) + .sort((a, b) => b.own - a.own); + + return { + vendor, + cur, + alt, + delta: alt - cur, + tiers, + directMatchShare: cur > 0 ? directMatched / cur : 0, + cacheRate: inp + cached > 0 ? cached / (inp + cached) : 0, + }; +} + +/* ─── FINDING ─── */ + +const VENDOR_NAME: Record = { + anthropic: "Anthropic", + openai: "OpenAI", +}; + +/** Signed percentage of a tier's own spend, e.g. "-65%" / "+41%". */ +function tierDelta(own: number, other: number): string { + if (own <= 0) return "n/a"; + const pct = Math.round(((other - own) / own) * 100); + return `${pct > 0 ? "+" : ""}${pct}%`; +} + +function buildReason(c: CrossVendorComparison): string { + return c.tiers + .map((t, i) => { + const suffix = i === 0 ? "/mo" : ""; + return `${t.ownLabel} ${$(t.own)}${suffix} → ${t.otherLabel} ${$( + t.other + )}${suffix} (${tierDelta(t.own, t.other)}).`; + }) + .join(" "); +} + +function buildAction(c: CrossVendorComparison): string { + const other = VENDOR_NAME[c.vendor === "anthropic" ? "openai" : "anthropic"]; + const parts = [ + `This reprices identical token volumes on ${other}'s nearest-equivalent tiers. It is a price comparison, not a quality claim — equivalent tier does not mean equivalent output, and only your own evals can settle that.`, + `Migration cost is excluded: prompt rewrites, tool-calling differences, and the re-evaluation run needed before you would trust the swap.`, + ]; + + // Cache economics don't survive the move intact: Anthropic reads cached + // input at 90% off, OpenAI's automatic caching at 50% off. A cache-heavy + // workload therefore keeps less of the delta than a table repricing shows. + if (c.cacheRate >= 0.1) { + parts.push( + `${Math.round(c.cacheRate * 100)}% of your input is served from cache. Anthropic cache reads are 90% off while OpenAI automatic caching is 50% off, so a cache-heavy workload like this keeps less of the delta than these numbers show.` + ); + } + + return parts.join(" "); +} + +/** + * The org-level cross-vendor finding, or null when the gate rejects it: too + * little spend for the argument to matter, or a delta thin enough that the + * honest answer is "roughly the same". + * + * Always INFO, always zero savings — see the header note. + */ +export function crossVendorFinding( + vendor: Vendor, + rows: CrossVendorRow[] +): Finding | null { + const c = compareVendors(vendor, rows); + + if (c.cur < MIN_ORG_SPEND) return null; + if (Math.abs(c.delta) < c.cur * MIN_DELTA_SHARE) return null; + + const signals: FindingSignal[] = [ + { + weight: 0.3, + met: Math.abs(c.delta) >= c.cur * 0.4, + label: "delta ≥ 40% of org spend", + }, + { weight: 0.25, met: c.cur >= 500, label: "org spend ≥ $500/mo" }, + { + weight: 0.25, + met: c.directMatchShare >= 0.8, + label: "80%+ of spend on directly mapped tiers", + }, + { weight: 0.2, met: c.cacheRate < 0.1, label: "cache rate < 10%" }, + ]; + const conf = confidenceScore(signals); + + const totals = rows.reduce( + (a, r) => ({ + inp: a.inp + r.inp, + out: a.out + r.out, + cached: a.cached + (r.cached ?? 0), + }), + { inp: 0, out: 0, cached: 0 } + ); + + return { + id: "cross-vendor-comparison", + name: "Organization", + ws: vendor === "anthropic" ? "All workspaces" : "All projects", + model: "N/A", + ml: "Cross-Vendor Comparison", + inp: totals.inp, + out: totals.out, + cached: totals.cached, + reqs: 0, + ao: 0, + ai: 0, + ratio: totals.out > 0 ? totals.inp / totals.out : 0, + cr: c.cacheRate, + cur: c.cur, + opt: c.cur, + sav: 0, + reason: buildReason(c), + action: buildAction(c), + sev: Severity.INFO, + cat: + vendor === "anthropic" + ? AnthropicCategory.CROSS_VENDOR_COMPARISON + : OpenAICategory.CROSS_VENDOR_COMPARISON, + conf, + impact: `${c.delta < 0 ? "-" : "+"}${$(Math.abs(c.delta))}/mo if migrated (not in savings total)`, + activeDays: 0, + temporal: { + burstiness: 0, + consistency: 0, + batchCandidate: false, + meanDaily: 0, + }, + source: "rules", + signals, + }; +} diff --git a/src/lib/openai/analysis.ts b/src/lib/openai/analysis.ts index 894a1b9..c82b711 100644 --- a/src/lib/openai/analysis.ts +++ b/src/lib/openai/analysis.ts @@ -20,6 +20,7 @@ import { } from "./costing"; import { $, P } from "@/lib/formatters"; import { capRowSavings } from "@/lib/savingsCap"; +import { crossVendorFinding } from "@/lib/crossVendor"; import type { OpenAIUsageData } from "./api"; /* ═══════════════════ SERVICE TYPES ═══════════════════ */ @@ -851,6 +852,21 @@ export function findIssuesOpenAI( }); } + /* ─── CROSS-VENDOR COMPARISON ─── */ + // One org-level, zero-savings INFO finding: the whole workload repriced on + // Anthropic's nearest-equivalent tiers. Both sides are priced from the + // pricing tables — deliberately not from `r.cost`, which has no Anthropic + // counterpart. Self-gating on org spend and delta size. + const crossVendor = crossVendorFinding( + "openai", + rows.map((r) => ({ + model: r.model || r.line_item || "", + inp: r.inp, + out: r.out, + })) + ); + if (crossVendor) out.push(crossVendor); + return capRowSavings(out).sort((a, b) => { const sv: Record = { [Severity.CRITICAL]: 0, diff --git a/src/lib/savingsCap.ts b/src/lib/savingsCap.ts index 5761131..25a6855 100644 --- a/src/lib/savingsCap.ts +++ b/src/lib/savingsCap.ts @@ -9,11 +9,12 @@ import type { Finding } from "@/types"; -// Categories where a zero-savings finding is still worth surfacing (quality -// or organizational wins). Everything else exists to save money, so a finding -// the cap zeroes out is noise and gets dropped. +// Categories where a zero-savings finding is still worth surfacing (quality, +// organizational, or informational wins — the cross-vendor comparison is +// zero-savings by construction). Everything else exists to save money, so a +// finding the cap zeroes out is noise and gets dropped. export const KEEP_ZERO_SAVINGS = - /upgrade|organization|workspace|project|quality/i; + /upgrade|organization|workspace|project|quality|cross-vendor/i; const slug = (c: string) => c.replace(/[^a-z0-9]/gi, "-").toLowerCase(); diff --git a/src/types/analysis.ts b/src/types/analysis.ts index cc177a6..1cfe8f0 100644 --- a/src/types/analysis.ts +++ b/src/types/analysis.ts @@ -36,6 +36,7 @@ export enum AnthropicCategory { BATCH_API_MIGRATION = "Batch API Migration", MODEL_UPGRADE = "Model Upgrade", WORKSPACE_ORGANIZATION = "Workspace Organization", + CROSS_VENDOR_COMPARISON = "Cross-Vendor Comparison", } export enum OpenAICategory { @@ -49,6 +50,7 @@ export enum OpenAICategory { REASONING_MODEL_OVERKILL = "Reasoning Model Overkill", HIGH_IMPACT_OPPORTUNITY = "High-Impact Opportunity", PROJECT_ORGANIZATION = "Project Organization", + CROSS_VENDOR_COMPARISON = "Cross-Vendor Comparison", } // Backward compatibility