diff --git a/PRODUCT.md b/PRODUCT.md index 943a2a1..394bcb4 100644 --- a/PRODUCT.md +++ b/PRODUCT.md @@ -19,9 +19,13 @@ Built as part of Nick's Omilia internship; doubles as a portfolio project. (`src/lib/*/costing.ts`). Per-finding provenance (`rules` / `llm` / `both`), graceful degradation to rules-only when NIM is unavailable. - **Trust guarantees**: per-row savings capped at the row's spend, stamped - pricing-table date, ROI footer on AI-augmented reports, deterministic - 8-workspace demo (`DEMO_SEED`). -- 90 vitest tests pin the money math; CI runs type-check, lint, format, test. + pricing-table date, ROI footer on AI-augmented reports. +- **Persona demo**: every "sample data" click generates a fresh org β€” a + sprawling enterprise (every rule fires, compounding growth arc, one + incident month) or a lean startup (1-2 minor findings, on purpose). Each + run is internally coherent across its 6 months; the generators are pure + and clock-free, reproducible via explicit seeds (`DEMO_SEED` in tests). +- 96 vitest tests pin the money math; CI runs type-check, lint, format, test. - AGPL-3.0. ## Where it's headed diff --git a/README.md b/README.md index c25c694..d6c1686 100644 --- a/README.md +++ b/README.md @@ -103,8 +103,11 @@ Pick a vendor, paste the key, click **Analyze**. Then browse findings by severity, spend by workspace, and history by month. The key is forgotten the moment you close the tab β€” TokenPilot has the memory of a goldfish, on purpose. -No key handy? **Try with sample data** generates a seeded 8-workspace demo -org β€” same org, same numbers, every single run. There's also a `mock-server/` +No key handy? **Try with sample data** conjures a fresh fake org on every +click β€” pick a sprawling enterprise (every rule fires, spend compounding +month over month, one runaway-agent incident in the middle) or a lean startup +(barely anything to fix, on purpose). Each run stays internally coherent: +one org, one story, across all six months. There's also a `mock-server/` for developing without burning real API calls. ## πŸ”© Under the hood @@ -129,9 +132,9 @@ flowchart LR | ids | ULID | sortable, unique, no coordination needed | Pre-commit: Husky + lint-staged run Prettier on everything; commitlint guards -the messages. A vitest suite (90 tests) pins the money math β€” every costing -formula, the consensus merge, and demo determinism. Full architecture notes -live in `CLAUDE.md`. +the messages. A vitest suite (96 tests) pins the money math β€” every costing +formula, the consensus merge, and the demo generators' purity. Full +architecture notes live in `CLAUDE.md`. ```bash npm run dev # dev server diff --git a/src/__tests__/demo-determinism.test.ts b/src/__tests__/demo-determinism.test.ts index 45cb6d8..0a65971 100644 --- a/src/__tests__/demo-determinism.test.ts +++ b/src/__tests__/demo-determinism.test.ts @@ -1,17 +1,28 @@ import { describe, it, expect } from "vitest"; -import { demoAnthropic, demoOpenAI, DEMO_SEED } from "@/lib/demo"; +import { + demoAnthropic, + demoOpenAI, + DEMO_SEED, + type DemoPersona, +} from "@/lib/demo"; import { agg, findIssues } from "@/lib/anthropic/analysis"; import { aggOpenAI, findIssuesOpenAI } from "@/lib/openai/analysis"; import { tc } from "@/lib/anthropic/pricing"; import { tcOpenAI } from "@/lib/openai/pricing"; // Fixed base period so the test itself is clock-independent. Mirrors -// startDemo's loop: the base month plus the 5 months before it. +// startDemo's loop: the base month plus the 5 months before it, oldest +// first, with monthsAgo counting down to 0 at the base month. const BASE_YEAR = 2026; const BASE_MONTH = 6; // July (0-indexed) -function demoMonths(): { y: number; m: number }[] { - const out: { y: number; m: number }[] = []; +// Two arbitrary but fixed seeds β€” the generators must behave for any seed, +// these just make the assertions reproducible. +const SEED_A = 1234567; +const SEED_B = 89101112; + +function demoMonths(): { y: number; m: number; monthsAgo: number }[] { + const out: { y: number; m: number; monthsAgo: number }[] = []; for (let i = 5; i >= 0; i--) { let m = BASE_MONTH - i; let y = BASE_YEAR; @@ -19,15 +30,15 @@ function demoMonths(): { y: number; m: number }[] { m += 12; y--; } - out.push({ y, m }); + out.push({ y, m, monthsAgo: i }); } return out; } // Same report-building pipeline startDemo runs per month (rule engine path). -function runAnthropicDemo() { - return demoMonths().map(({ y, m }) => { - const d = demoAnthropic(y, m, DEMO_SEED); +function runAnthropicDemo(seed: number, persona: DemoPersona) { + return demoMonths().map(({ y, m, monthsAgo }) => { + const d = demoAnthropic(y, m, seed, persona, monthsAgo); const bk = agg(d.bk); const bm = agg(d.bm); const src = bk.length ? bk : bm; @@ -49,13 +60,17 @@ function runAnthropicDemo() { savings: findings.reduce((s, f) => s + f.sav, 0), tokens: ti + to, findings, + cacheWrites: d.bw.reduce( + (s, b) => s + (b.cache_creation_input_tokens || 0), + 0 + ), }; }); } -function runOpenAIDemo() { - return demoMonths().map(({ y, m }) => { - const d = demoOpenAI(y, m, DEMO_SEED); +function runOpenAIDemo(seed: number, persona: DemoPersona) { + return demoMonths().map(({ y, m, monthsAgo }) => { + const d = demoOpenAI(y, m, seed, persona, monthsAgo); const rows = aggOpenAI(d.usage); let spend = 0; @@ -81,56 +96,172 @@ function runOpenAIDemo() { }); } -describe("demo mode determinism", () => { - it("two runs of the full 6-month Anthropic demo produce byte-identical reports", () => { - expect(JSON.stringify(runAnthropicDemo())).toBe( - JSON.stringify(runAnthropicDemo()) - ); +describe("demo purity and determinism", () => { + const personas: DemoPersona[] = ["enterprise", "startup"]; + + it("two calls with identical (year, month, seed, persona, monthsAgo) are byte-identical", () => { + for (const persona of personas) { + expect( + JSON.stringify(demoAnthropic(BASE_YEAR, BASE_MONTH, SEED_A, persona, 2)) + ).toBe( + JSON.stringify(demoAnthropic(BASE_YEAR, BASE_MONTH, SEED_A, persona, 2)) + ); + expect( + JSON.stringify(demoOpenAI(BASE_YEAR, BASE_MONTH, SEED_A, persona, 2)) + ).toBe( + JSON.stringify(demoOpenAI(BASE_YEAR, BASE_MONTH, SEED_A, persona, 2)) + ); + } }); - it("two runs of the full 6-month OpenAI demo produce byte-identical reports", () => { - expect(JSON.stringify(runOpenAIDemo())).toBe( - JSON.stringify(runOpenAIDemo()) + it("two runs of the full 6-month demo produce byte-identical reports (both vendors)", () => { + expect(JSON.stringify(runAnthropicDemo(DEMO_SEED, "enterprise"))).toBe( + JSON.stringify(runAnthropicDemo(DEMO_SEED, "enterprise")) + ); + expect(JSON.stringify(runOpenAIDemo(DEMO_SEED, "enterprise"))).toBe( + JSON.stringify(runOpenAIDemo(DEMO_SEED, "enterprise")) ); }); - it("all 6 months of a run come from the same org", () => { - const anth = runAnthropicDemo(); - expect(new Set(anth.map((r) => r.org.name)).size).toBe(1); - const oai = runOpenAIDemo(); - expect(new Set(oai.map((r) => r.org.name)).size).toBe(1); + it("a different seed produces a different org with different data", () => { + const a = demoAnthropic(BASE_YEAR, BASE_MONTH, SEED_A); + const b = demoAnthropic(BASE_YEAR, BASE_MONTH, SEED_B); + expect(JSON.stringify(a.bm)).not.toBe(JSON.stringify(b.bm)); + expect(a.org.name).not.toBe(b.org.name); + expect(a.org.id).not.toBe(b.org.id); + + const oa = demoOpenAI(BASE_YEAR, BASE_MONTH, SEED_A); + const ob = demoOpenAI(BASE_YEAR, BASE_MONTH, SEED_B); + expect(JSON.stringify(oa.usage)).not.toBe(JSON.stringify(ob.usage)); + expect(oa.org.name).not.toBe(ob.org.name); }); - it("a different seed produces different data", () => { - const a = demoAnthropic(BASE_YEAR, BASE_MONTH, DEMO_SEED); - const b = demoAnthropic(BASE_YEAR, BASE_MONTH, DEMO_SEED + 1); - expect(JSON.stringify(a.bm)).not.toBe(JSON.stringify(b.bm)); + it("all 6 months of a run come from the same org (both personas)", () => { + for (const persona of personas) { + const anth = runAnthropicDemo(SEED_A, persona); + expect(new Set(anth.map((r) => r.org.name)).size).toBe(1); + const oai = runOpenAIDemo(SEED_A, persona); + expect(new Set(oai.map((r) => r.org.name)).size).toBe(1); + } }); +}); - it("the demo org has 8 workspaces plus busy default-workspace traffic", () => { - const d = demoAnthropic(BASE_YEAR, BASE_MONTH, DEMO_SEED); +describe("enterprise persona", () => { + it("has 8 workspaces plus busy default-workspace traffic", () => { + const d = demoAnthropic(BASE_YEAR, BASE_MONTH, SEED_A, "enterprise", 0); expect(d.ws).toHaveLength(8); // Some traffic deliberately carries no workspace_id β†’ default workspace. expect(d.bw.some((b) => !b.workspace_id)).toBe(true); expect(d.bw.some((b) => b.workspace_id === "ws_prod")).toBe(true); }); - it("the demo's varied workloads fire most rule categories", () => { - const cats = new Set( - runAnthropicDemo().flatMap((r) => r.findings.map((f) => f.cat as string)) - ); - for (const expected of [ - "Model Downgrade β†’ Haiku", - "Model Downgrade β†’ Sonnet", - "RAG Optimization", - "Prompt Caching", - "Batch API Migration", - "Model Upgrade", - "Workspace Organization", - ]) { - expect(cats, `expected category "${expected}" to fire`).toContain( - expected + it("fires at least 6 distinct Anthropic categories in the current month", () => { + for (const seed of [SEED_A, SEED_B, DEMO_SEED]) { + const [current] = runAnthropicDemo(seed, "enterprise").slice(-1); + const cats = new Set(current.findings.map((f) => f.cat as string)); + for (const expected of [ + "Model Downgrade β†’ Haiku", + "Model Downgrade β†’ Sonnet", + "RAG Optimization", + "Prompt Caching", + "Batch API Migration", + "Model Upgrade", + "Workspace Organization", + ]) { + expect(cats, `expected category "${expected}" to fire`).toContain( + expected + ); + } + expect(cats.size).toBeGreaterThanOrEqual(6); + } + }); + + it("fires at least 6 distinct OpenAI categories in the current month", () => { + for (const seed of [SEED_A, SEED_B, DEMO_SEED]) { + const [current] = runOpenAIDemo(seed, "enterprise").slice(-1); + const cats = new Set(current.findings.map((f) => f.cat as string)); + for (const expected of [ + "Model Downgrade β†’ GPT-4o-mini", + "RAG Optimization", + "Prompt Caching", + "Batch API Migration", + "Reasoning Model Overkill", + "Model Upgrade", + "Prompt Optimization", + "High-Impact Opportunity", + ]) { + expect(cats, `expected category "${expected}" to fire`).toContain( + expected + ); + } + expect(cats.size).toBeGreaterThanOrEqual(6); + } + }); + + it("6-month spend is strictly increasing, except around the incident month", () => { + // monthsAgo 2 β†’ index 3 in the oldest-first array. The incident bumps + // that month's spend, so the following month may legitimately dip. + const incidentIdx = 3; + for (const seed of [SEED_A, SEED_B, DEMO_SEED]) { + const months = runAnthropicDemo(seed, "enterprise"); + for (let i = 0; i < months.length - 1; i++) { + if (i === incidentIdx) continue; + expect( + months[i + 1].spend, + `seed ${seed}: spend should grow from month ${i} to ${i + 1}` + ).toBeGreaterThan(months[i].spend); + } + // The incident month itself must sit visibly above the prior month. + expect(months[incidentIdx].spend).toBeGreaterThan( + months[incidentIdx - 1].spend * 1.1 ); } }); + + it("the incident month's cache-write volume spikes vs its neighbors", () => { + const incidentIdx = 3; + for (const seed of [SEED_A, SEED_B, DEMO_SEED]) { + const months = runAnthropicDemo(seed, "enterprise"); + const spike = months[incidentIdx].cacheWrites; + expect(spike).toBeGreaterThan(months[incidentIdx - 1].cacheWrites * 2.5); + expect(spike).toBeGreaterThan(months[incidentIdx + 1].cacheWrites * 2.5); + } + }); +}); + +describe("startup persona", () => { + it("produces at most 3 findings per vendor, with modest savings", () => { + for (const seed of [SEED_A, SEED_B, DEMO_SEED]) { + const [anth] = runAnthropicDemo(seed, "startup").slice(-1); + expect(anth.findings.length).toBeGreaterThanOrEqual(1); + expect(anth.findings.length).toBeLessThanOrEqual(3); + expect(anth.savings).toBeLessThan(anth.spend * 0.3); + + const [oai] = runOpenAIDemo(seed, "startup").slice(-1); + expect(oai.findings.length).toBeGreaterThanOrEqual(1); + expect(oai.findings.length).toBeLessThanOrEqual(3); + expect(oai.savings).toBeLessThan(oai.spend * 0.3); + } + }); + + it("is a small org: 3 Anthropic workspaces, 2 OpenAI projects", () => { + const d = demoAnthropic(BASE_YEAR, BASE_MONTH, SEED_A, "startup", 0); + expect(d.ws).toHaveLength(3); + // All startup traffic is properly workspace-tagged β€” no default-workspace + // mess, so no workspace-organization finding. + expect(d.bw.every((b) => !!b.workspace_id)).toBe(true); + + const o = demoOpenAI(BASE_YEAR, BASE_MONTH, SEED_A, "startup", 0); + expect(o.projects).toHaveLength(2); + }); + + it("keeps the startup's findings minor β€” no critical severity", () => { + for (const seed of [SEED_A, SEED_B, DEMO_SEED]) { + const [anth] = runAnthropicDemo(seed, "startup").slice(-1); + const [oai] = runOpenAIDemo(seed, "startup").slice(-1); + for (const f of [...anth.findings, ...oai.findings]) { + expect(f.sev).not.toBe("critical"); + } + } + }); }); diff --git a/src/app/page.tsx b/src/app/page.tsx index 40ae42d..db61856 100644 --- a/src/app/page.tsx +++ b/src/app/page.tsx @@ -18,7 +18,7 @@ import { import { toSummariesAnthropic, toSummariesOpenAI } from "@/lib/nim/adapters"; import { tc } from "@/lib/anthropic/pricing"; import { tcOpenAI } from "@/lib/openai/pricing"; -import { demoAnthropic, demoOpenAI, DEMO_SEED } from "@/lib/demo"; +import { demoAnthropic, demoOpenAI, type DemoPersona } from "@/lib/demo"; import type { Report } from "@/types"; import Footer from "@/components/Footer"; import { FadeUp } from "@/components/motion/FadeUp"; @@ -179,6 +179,9 @@ function HomeContent() { const [nimAvailable, setNimAvailable] = useState(false); const [useNim, setUseNim] = useState(false); + // Demo persona: which org archetype "Try with sample data" simulates. + const [demoPersona, setDemoPersona] = useState("enterprise"); + useEffect(() => { fetch("/api/nim") .then((r) => r.json()) @@ -580,6 +583,11 @@ function HomeContent() { const year = now.getFullYear(); const month = now.getMonth(); const id = generateId(); + // One fresh random seed per click: every demo run is a new org, but the + // same seed threads through all six monthly calls so the run stays + // internally coherent. Randomness lives here in the UI layer only β€” + // the generators in demo.ts stay pure. + const seed = Date.now() % 2147483647; if (vendor === Vendor.OPENAI) { setStep("Simulating 6 months of OpenAI usage..."); @@ -591,7 +599,7 @@ function HomeContent() { y--; } - const d = demoOpenAI(y, m, DEMO_SEED); + const d = demoOpenAI(y, m, seed, demoPersona, i); const rows = aggOpenAI(d.usage); let spend = 0; @@ -690,7 +698,7 @@ function HomeContent() { y--; } - const d = demoAnthropic(y, m, DEMO_SEED); + const d = demoAnthropic(y, m, seed, demoPersona, i); const bk = agg(d.bk); const bm = agg(d.bm); const src = bk.length ? bk : bm; @@ -913,6 +921,34 @@ function HomeContent() { Try with sample data β†’ + {/* Demo persona picker: which org archetype to simulate */} +
+ {( + [ + ["enterprise", "Sprawling enterprise"], + ["startup", "Lean startup"], + ] as [DemoPersona, string][] + ).map(([p, label]) => ( + + ))} +
+ {/* Error display */} {err && (
number { @@ -35,44 +42,96 @@ function hashStr(s: string): number { interface BusinessProfile { orgName: string; - scale: number; - cacheAffinity: number; - weekendFactor: number; - volatility: number; + scale: number; // org size multiplier applied to every workload + weekendFactor: number; // weekend traffic as a share of weekday traffic + jitter: number; // daily noise band around 1 (kept small so trends read) + growthRate: number; // month-over-month volume growth toward the present } -const ORG_NAMES = [ - "Acme Corp", - "Northwind Labs", - "Vertex Dynamics", - "BlueHarbor AI", - "Quantra Systems", - "Helio Industries", - "Mosswood Software", - "Ironclad Analytics", - "Skyline Robotics", - "Cobalt & Finch", +const ORG_PREFIXES = [ + "Acme", + "Northwind", + "Vertex", + "BlueHarbor", + "Quantra", + "Helio", + "Mosswood", + "Ironclad", + "Skyline", + "Cobalt", + "Latchford", + "Ferrous", +]; + +const ORG_SUFFIXES = [ + "Corp", + "Labs", + "Systems", + "Dynamics", + "Software", + "Analytics", + "Robotics", + "Industries", ]; -function newProfile(seed: number): BusinessProfile { - const r = makeRand(seed); +function newProfile(seed: number, persona: DemoPersona): BusinessProfile { + const r = makeRand(seed ^ hashStr("profile")); + const orgName = `${ORG_PREFIXES[Math.floor(r() * ORG_PREFIXES.length)]} ${ + ORG_SUFFIXES[Math.floor(r() * ORG_SUFFIXES.length)] + }`; + const weekendFactor = 0.55 + r() * 0.25; + if (persona === "startup") { + return { + orgName, + scale: 0.9 + r() * 0.4, + weekendFactor, + jitter: 0.16, + growthRate: 0.03 + r() * 0.03, + }; + } return { - orgName: ORG_NAMES[Math.floor(r() * ORG_NAMES.length)], - scale: 0.4 + r() * r() * 7, - cacheAffinity: 0.2 + r() * 1.6, - weekendFactor: 0.1 + r() * 0.7, - volatility: 0.6 + r() * 0.8, + orgName, + scale: 0.7 + r() * 1.8, + weekendFactor, + jitter: 0.16, + growthRate: 0.1 + r() * 0.04, }; } +// ─── 6-month arc ───────────────────────────────────────────────────────────── +// +// Each run's months form a deliberate shape instead of independent noise: +// volumes compound by the profile's seeded growth rate toward the present +// (monthsAgo 0 = current month, 5 = oldest), normalized for month length so +// the trend survives short months and the analytics forecast has a real +// slope to fit. Legacy workloads decay ~20%/mo instead (a migration in +// progress), and the enterprise Agent Platform workspace has one incident +// month at monthsAgo 2: a runaway agent loop that multiplies cache writes +// and input volume before being fixed. + +const DEMO_WINDOW = 5; // oldest monthsAgo in a 6-month run +const INCIDENT_MONTHS_AGO = 2; +const INCIDENT = { inp: 10, out: 4, reqs: 8, cacheWrite: 4.2 }; + +function arcMult(profile: BusinessProfile, monthsAgo: number): number { + return Math.pow(1 + profile.growthRate, -monthsAgo); +} + +function decayMult(monthsAgo: number): number { + return Math.pow(0.8, DEMO_WINDOW - monthsAgo); +} + // ─── Anthropic ─────────────────────────────────────────────────────────────── // -// 8 workspaces plus an overloaded default. Each workspace runs a distinct, -// realistic workload pattern so most rule categories fire in a demo run: -// caching miss, Haiku downgrade, RAG bloat, batch candidate, Opusβ†’Sonnet, -// legacy model, cache-write waste, plus a quiet staging workspace. Traffic -// with no workspace_id lands in the (biggest-spending) default workspace, -// which also triggers the org-structure finding. +// Enterprise: 8 workspaces plus an overloaded default. Each workspace runs a +// distinct, realistic workload pattern so most rule categories fire in a demo +// run: caching miss, Haiku downgrade, RAG bloat, batch candidate, +// Opusβ†’Sonnet, legacy model, cache-write waste, plus a quiet staging +// workspace. Traffic with no workspace_id lands in the (biggest-spending) +// default workspace, which also triggers the org-structure finding. +// +// Startup: 3 workspaces of mostly well-optimized traffic. Only production's +// never-enabled prompt caching should fire β€” a small, honest finding. const ANTH_WORKSPACES: Workspace[] = [ { id: "ws_prod", name: "Production API", display_name: "Production API" }, @@ -104,6 +163,19 @@ const ANTH_WORKSPACES: Workspace[] = [ created_at: `2024-0${(i % 8) + 1}-01T00:00:00Z`, })); +const STARTUP_ANTH_WORKSPACES: Workspace[] = [ + { id: "ws_prod", name: "Production", display_name: "Production" }, + { + id: "ws_internal", + name: "Internal Tools", + display_name: "Internal Tools", + }, + { id: "ws_staging", name: "Staging", display_name: "Staging" }, +].map((w, i) => ({ + ...w, + created_at: `2025-0${i + 1}-01T00:00:00Z`, +})); + // One entry per (workspace, api key, model) workload. Daily base volumes are // sized so every pattern clears its rule thresholds even at the low end of // the profile's scale range. wid undefined β†’ default workspace. @@ -118,6 +190,8 @@ interface AnthWorkload { cacheWrite?: number; // daily base cache-creation tokens cacheReads?: number; // daily base cache-read tokens (overrides cacheRate) mondayBoost?: boolean; // bursty Monday spikes β†’ batch candidate + decay?: boolean; // shrinks ~20%/mo instead of growing (migration story) + incidentSpike?: boolean; // hit by the enterprise incident month } const ANTH_WORKLOADS: AnthWorkload[] = [ @@ -197,7 +271,8 @@ const ANTH_WORKLOADS: AnthWorkload[] = [ reqs: 100, cacheRate: 0.1, }, - // Legacy summarizer: still on Claude 3 Opus β†’ legacy model upgrade. + // Legacy summarizer: still on Claude 3 Opus, being migrated away β€” + // decays across the window but the legacy finding still fires today. { wid: "ws_legacy", key: "key_legacy", @@ -206,8 +281,10 @@ const ANTH_WORKLOADS: AnthWorkload[] = [ out: 15_000, reqs: 80, cacheRate: 0, + decay: true, }, - // Agent platform: writes big cache prefixes it rarely reads back. + // Agent platform: writes big cache prefixes it rarely reads back. Also the + // site of the incident month's runaway loop. { wid: "ws_agents", key: "key_agents", @@ -218,6 +295,7 @@ const ANTH_WORKLOADS: AnthWorkload[] = [ cacheRate: 0, cacheWrite: 400_000, cacheReads: 20_000, + incidentSpike: true, }, // Staging: light, well-behaved traffic β€” no findings expected. { @@ -231,6 +309,40 @@ const ANTH_WORKLOADS: AnthWorkload[] = [ }, ]; +const STARTUP_ANTH_WORKLOADS: AnthWorkload[] = [ + // Production: healthy model choice and shape, but prompt caching was never + // enabled β€” the run's one deliberate (small) finding. + { + wid: "ws_prod", + key: "key_prod", + model: "claude-sonnet-4-6-20250514", + inp: 1_000_000, + out: 120_000, + reqs: 400, + cacheRate: 0.02, + }, + // Internal tools: Haiku, well cached β€” nothing to flag. + { + wid: "ws_internal", + key: "key_internal", + model: "claude-haiku-4-5-20250514", + inp: 1_200_000, + out: 250_000, + reqs: 500, + cacheRate: 0.3, + }, + // Staging: light, clean traffic. + { + wid: "ws_staging", + key: "key_staging", + model: "claude-sonnet-4-6-20250514", + inp: 60_000, + out: 15_000, + reqs: 60, + cacheRate: 0.35, + }, +]; + interface AnthEntry { bucket_start: string; model: string; @@ -247,33 +359,49 @@ function genAnthropicEntries( profile: BusinessProfile, seed: number, year: number, - month: number + month: number, + persona: DemoPersona, + monthsAgo: number ): AnthEntry[] { const rand = makeRand(seed ^ (year * 12 + month) ^ hashStr("anth")); - const { scale } = profile; - const daysInMonth = new Date(year, month + 1, 0).getDate(); + const { scale, jitter } = profile; + const workloads = + persona === "startup" ? STARTUP_ANTH_WORKLOADS : ANTH_WORKLOADS; + const daysInMonth = new Date(Date.UTC(year, month + 1, 0)).getUTCDate(); + // Normalize monthly volume for month length so the arc isn't distorted by + // 28-day months. + const monthNorm = 30 / daysInMonth; + const growth = arcMult(profile, monthsAgo); const entries: AnthEntry[] = []; for (let day = 1; day <= daysInMonth; day++) { - const date = new Date(year, month, day); - const isWeekend = date.getDay() === 0 || date.getDay() === 6; - const isMonday = date.getDay() === 1; + const dow = new Date(Date.UTC(year, month, day)).getUTCDay(); + const isWeekend = dow === 0 || dow === 6; + const isMonday = dow === 1; const ds = `${year}-${String(month + 1).padStart(2, "0")}-${String(day).padStart(2, "0")}T00:00:00Z`; - for (const w of ANTH_WORKLOADS) { + for (const w of workloads) { const wm = isWeekend ? profile.weekendFactor : 1; - const v = 0.5 + rand() * profile.volatility; - let mult = wm * v * scale; + const v = 1 - jitter / 2 + rand() * jitter; + const trend = w.decay ? decayMult(monthsAgo) : growth; + let mult = wm * v * scale * trend * monthNorm; if (w.mondayBoost) mult *= isMonday ? 8 : 0.3; - const inp = Math.floor(w.inp * mult); - const out = Math.floor(w.out * mult); - const reqs = Math.floor(w.reqs * mult); + const incident = + persona === "enterprise" && + monthsAgo === INCIDENT_MONTHS_AGO && + !!w.incidentSpike; + + const inp = Math.floor(w.inp * mult * (incident ? INCIDENT.inp : 1)); + const out = Math.floor(w.out * mult * (incident ? INCIDENT.out : 1)); + const reqs = Math.floor(w.reqs * mult * (incident ? INCIDENT.reqs : 1)); const cache = w.cacheReads !== undefined ? Math.floor(w.cacheReads * mult) : Math.floor(inp * w.cacheRate * (0.8 + rand() * 0.4)); - const cacheCreated = w.cacheWrite ? Math.floor(w.cacheWrite * mult) : 0; + const cacheCreated = w.cacheWrite + ? Math.floor(w.cacheWrite * mult * (incident ? INCIDENT.cacheWrite : 1)) + : 0; if (inp === 0 && out === 0) continue; @@ -297,13 +425,25 @@ function genAnthropicEntries( export function demoAnthropic( year: number, month: number, - seed: number = DEMO_SEED + seed: number = DEMO_SEED, + persona: DemoPersona = "enterprise", + monthsAgo: number = 0 ): PullResult { - const profile = newProfile(seed); + const profile = newProfile(seed, persona); - const org: Organization = { id: "demo_org_01", name: profile.orgName }; - const ws = ANTH_WORKSPACES; - const entries = genAnthropicEntries(profile, seed, year, month); + const org: Organization = { + id: `demo_org_${(seed >>> 0).toString(36)}`, + name: profile.orgName, + }; + const ws = persona === "startup" ? STARTUP_ANTH_WORKSPACES : ANTH_WORKSPACES; + const entries = genAnthropicEntries( + profile, + seed, + year, + month, + persona, + monthsAgo + ); const bm: UsageBucket[] = entries.map((e) => ({ bucket_start: e.bucket_start, @@ -315,10 +455,15 @@ export function demoAnthropic( request_count: e.request_count, })); + // The by-key report mirrors the real Admin API for enterprise: no + // workspace_id, so the untagged mess lands in the default workspace and + // triggers the org-structure finding. The startup tags its keys properly β€” + // its report should attribute spend cleanly and stay finding-quiet. const bk: UsageBucket[] = entries.map((e) => ({ bucket_start: e.bucket_start, model: e.model, api_key_id: e.api_key_id, + ...(persona === "startup" ? { workspace_id: e.workspace_id } : {}), input_tokens: e.input_tokens, output_tokens: e.output_tokens, cache_read_input_tokens: e.cache_read_input_tokens, @@ -337,31 +482,33 @@ export function demoAnthropic( request_count: e.request_count, })); - const now = new Date().toISOString(); + // Deterministic "fetched at end of month" stamp β€” demo.ts never reads the + // clock, so identical inputs stay byte-identical. + const fetchedAt = new Date(Date.UTC(year, month + 1, 1)).toISOString(); const raw = { organization: { endpoint: "/v1/organizations/me", - fetched_at: now, + fetched_at: fetchedAt, response: org, }, workspaces: { endpoint: "/v1/organizations/workspaces", - fetched_at: now, + fetched_at: fetchedAt, response: { data: ws }, }, usage_by_model: { endpoint: "/v1/organizations/usage_report/messages", - fetched_at: now, + fetched_at: fetchedAt, results: bm, }, usage_by_key: { endpoint: "/v1/organizations/usage_report/messages", - fetched_at: now, + fetched_at: fetchedAt, results: bk, }, usage_by_workspace: { endpoint: "/v1/organizations/usage_report/messages", - fetched_at: now, + fetched_at: fetchedAt, results: bw, }, }; @@ -370,171 +517,368 @@ export function demoAnthropic( } // ─── OpenAI ────────────────────────────────────────────────────────────────── +// +// Mirrors the Anthropic side: a curated per-project workload table, one +// completions entry per rule story, instead of a per-day scenario roulette. +// Enterprise projects each tell one story (mini-downgrade router, RAG bloat, +// steady batch enrichment, Monday-only evals, o1 reasoning overkill, legacy +// GPT-4, prompt bloat, caching miss) on top of an overloaded default +// project. The startup runs two clean mini-based projects. Non-completions +// services stay as light background traffic so the multi-service table is +// populated, and every cost row is derived from the same token volumes at +// pricing-table rates. + +interface OaiProject { + id: string; + name: string; + created_at: number; + organization_id: string; +} -const OAI_PROJECTS = [ +const OAI_ENT_PROJECTS: OaiProject[] = [ + { id: "proj_router", name: "Support Router" }, + { id: "proj_rag", name: "Docs Assistant" }, + { id: "proj_enrich", name: "Data Enrichment" }, + { id: "proj_evals", name: "Model Evals" }, + { id: "proj_reason", name: "Reasoning Pipeline" }, + { id: "proj_legacy", name: "Legacy Chat" }, + { id: "proj_content", name: "Content Studio" }, + { id: "proj_realtime", name: "Realtime Assistant" }, +].map((p, i) => ({ + ...p, + created_at: 1709251200 + i * 2_592_000, + organization_id: "org_demo", +})); + +const OAI_STARTUP_PROJECTS: OaiProject[] = [ + { id: "proj_app", name: "Product API" }, + { id: "proj_tools", name: "Internal Tools" }, +].map((p, i) => ({ + ...p, + created_at: 1735689600 + i * 2_592_000, + organization_id: "org_demo", +})); + +// pid undefined β†’ default project. +interface OaiCompletionsWorkload { + pid?: string; + model: string; + inp: number; // daily base input tokens + out: number; // daily base output tokens + reqs: number; // daily base requests + mondayOnly?: boolean; // eval batches that only run Mondays β†’ bursty +} + +const OAI_ENT_WORKLOADS: OaiCompletionsWorkload[] = [ + // Overloaded default project: the org's main traffic never got segmented + // β†’ high-impact opportunity in the "Default project" bucket. + { + model: "gpt-4o", + inp: 1_000_000, + out: 150_000, + reqs: 400, + }, + { + model: "gpt-4o-mini", + inp: 500_000, + out: 120_000, + reqs: 500, + }, + // Support router: o1-mini emitting tiny classification outputs β†’ GPT-4o-mini + // downgrade (a reasoning model on routing traffic). { - id: "proj_main", - name: "Main App", - created_at: 1709251200, - organization_id: "org_demo", + pid: "proj_router", + model: "o1-mini", + inp: 200_000, + out: 8_000, + reqs: 140, + }, + // Docs assistant: enormous retrieval context per request β†’ RAG bloat. + { + pid: "proj_rag", + model: "gpt-4o", + inp: 700_000, + out: 25_000, + reqs: 25, }, + // Data enrichment: steady high-volume traffic β†’ batch API (rule 4b). { - id: "proj_analytics", - name: "Analytics Service", - created_at: 1711929600, - organization_id: "org_demo", + pid: "proj_enrich", + model: "gpt-4o", + inp: 500_000, + out: 100_000, + reqs: 300, + }, + // Model evals: Monday-only bursts β†’ batch API (rule 4). + { + pid: "proj_evals", + model: "gpt-4o", + inp: 2_000_000, + out: 200_000, + reqs: 800, + mondayOnly: true, + }, + // Reasoning pipeline: o1 on short, simple outputs β†’ reasoning overkill. + { + pid: "proj_reason", + model: "o1", + inp: 150_000, + out: 15_000, + reqs: 60, + }, + // Legacy chat: still on GPT-4 β†’ upgrade to GPT-4o. + { + pid: "proj_legacy", + model: "gpt-4", + inp: 40_000, + out: 8_000, + reqs: 30, + }, + // Content studio: verbose 15k-token prompts β†’ prompt bloat (rule 8). + { + pid: "proj_content", + model: "gpt-4o", + inp: 300_000, + out: 12_000, + reqs: 20, + }, + // Realtime assistant: big repeated context, no caching β†’ caching (rule 0). + { + pid: "proj_realtime", + model: "gpt-4o", + inp: 400_000, + out: 40_000, + reqs: 70, }, ]; -interface ServiceConfig { - endpoint: string; - models: string[]; - baseTokens: number; - baseCost: number; +const OAI_STARTUP_WORKLOADS: OaiCompletionsWorkload[] = [ + // Product API: right-sized on mini, but repeated context isn't cached β€” + // the run's one deliberate (small) finding. + { + pid: "proj_app", + model: "gpt-4o-mini", + inp: 250_000, + out: 40_000, + reqs: 100, + }, + // Internal tools: small and clean. + { + pid: "proj_tools", + model: "gpt-4o-mini", + inp: 20_000, + out: 5_000, + reqs: 80, + }, +]; + +// Background traffic for the non-completions services. Costs derive from the +// volumes at realistic per-unit rates; volumes are kept small enough that +// none of the token-based rules can fire on them. +interface OaiServiceWorkload { + pid?: string; + service: string; + model: string; + tokens?: number; // daily tokens (input-only) + seconds?: number; // daily audio seconds (whisper) + units?: number; // daily images / sessions / store-days + reqs: number; // daily requests + tokenRate?: number; // $/MTok + unitCost?: number; // $/unit } -const OAI_SERVICES: ServiceConfig[] = [ +const OAI_ENT_SERVICES: OaiServiceWorkload[] = [ + { + pid: "proj_rag", + service: "embeddings", + model: "text-embedding-3-large", + tokens: 400_000, + reqs: 800, + tokenRate: 0.13, + }, { - endpoint: "completions", - models: ["gpt-4o", "gpt-4o-mini", "o3-mini"], - baseTokens: 15000, - baseCost: 0.015, + pid: "proj_realtime", + service: "audio_speeches", + model: "tts-1", + tokens: 40_000, + reqs: 60, + tokenRate: 15, }, { - endpoint: "embeddings", - models: ["text-embedding-3-large", "text-embedding-3-small"], - baseTokens: 5000, - baseCost: 0.00013, + pid: "proj_realtime", + service: "audio_transcriptions", + model: "whisper-1", + seconds: 7_200, + reqs: 80, }, { - endpoint: "audio_speeches", - models: ["tts-1", "tts-1-hd"], - baseTokens: 1000, - baseCost: 0.03, + pid: "proj_content", + service: "images", + model: "dall-e-3", + units: 25, + reqs: 25, + unitCost: 0.04, }, { - endpoint: "audio_transcriptions", - models: ["whisper-1"], - baseTokens: 2000, - baseCost: 0.006, + service: "moderations", + model: "text-moderation-latest", + tokens: 20_000, + reqs: 300, + tokenRate: 0, }, { - endpoint: "images", - models: ["dall-e-3", "dall-e-2"], - baseTokens: 500, - baseCost: 0.04, + pid: "proj_rag", + service: "vector_stores", + model: "vector-store", + units: 3, + reqs: 40, + unitCost: 0.1, }, { - endpoint: "moderations", - models: ["text-moderation-latest"], - baseTokens: 3000, - baseCost: 0.0001, + pid: "proj_enrich", + service: "code_interpreter_sessions", + model: "code-interpreter", + units: 12, + reqs: 12, + unitCost: 0.03, }, +]; + +const OAI_STARTUP_SERVICES: OaiServiceWorkload[] = [ { - endpoint: "vector_stores", - models: ["vector-store"], - baseTokens: 0, - baseCost: 0.01, + pid: "proj_app", + service: "embeddings", + model: "text-embedding-3-small", + tokens: 150_000, + reqs: 300, + tokenRate: 0.02, }, { - endpoint: "code_interpreter_sessions", - models: ["code-interpreter"], - baseTokens: 10000, - baseCost: 0.02, + pid: "proj_app", + service: "moderations", + model: "text-moderation-latest", + tokens: 15_000, + reqs: 150, + tokenRate: 0, }, ]; export function demoOpenAI( year: number, month: number, - seed: number = DEMO_SEED + seed: number = DEMO_SEED, + persona: DemoPersona = "enterprise", + monthsAgo: number = 0 ): OpenAIPullResult { - const profile = newProfile(seed); + const profile = newProfile(seed, persona); const rand = makeRand(seed ^ (year * 12 + month) ^ hashStr("oai")); - const { scale } = profile; - const daysInMonth = new Date(year, month + 1, 0).getDate(); + const { scale, jitter } = profile; + const daysInMonth = new Date(Date.UTC(year, month + 1, 0)).getUTCDate(); + const monthNorm = 30 / daysInMonth; + const growth = arcMult(profile, monthsAgo); + + const projects = + persona === "startup" ? OAI_STARTUP_PROJECTS : OAI_ENT_PROJECTS; + const completions = + persona === "startup" ? OAI_STARTUP_WORKLOADS : OAI_ENT_WORKLOADS; + const services = + persona === "startup" ? OAI_STARTUP_SERVICES : OAI_ENT_SERVICES; + const projectName: Record = {}; + for (const p of projects) projectName[p.id] = p.name; const usageRows: OpenAIUsageData["data"] = []; const costResults: OpenAICostsData["data"] = []; for (let day = 1; day <= daysInMonth; day++) { - const date = new Date(year, month, day); - const isWeekend = date.getDay() === 0 || date.getDay() === 6; - const isMonday = date.getDay() === 1; - const ts = Math.floor(date.getTime() / 1000); - const scenario = rand(); + const dow = new Date(Date.UTC(year, month, day)).getUTCDay(); + const isWeekend = dow === 0 || dow === 6; + const isMonday = dow === 1; + const ts = Math.floor(Date.UTC(year, month, day) / 1000); const dayCostResults: (typeof costResults)[0]["results"] = []; + const pushCost = (cost: number, model: string, pid?: string) => { + if (cost <= 0) return; + dayCostResults.push({ + object: "organization.costs.result", + amount: { value: parseFloat(cost.toFixed(6)), currency: "usd" }, + line_item: model, + project_id: pid, + project_name: pid ? projectName[pid] : undefined, + organization_id: "org_demo", + organization_name: profile.orgName, + }); + }; - for (const svc of OAI_SERVICES) { - if (rand() > 0.75) continue; - - const project = OAI_PROJECTS[Math.floor(rand() * OAI_PROJECTS.length)]; - const model = svc.models[Math.floor(rand() * svc.models.length)]; + for (const w of completions) { + if (w.mondayOnly && !isMonday) continue; const wm = isWeekend ? profile.weekendFactor : 1; - const v = 0.5 + rand() * profile.volatility; - - let tokens: number, cost: number, reqs: number; - - if (svc.endpoint === "completions") { - if (scenario < 0.15) { - // Low output β†’ model downgrade candidate - tokens = Math.floor(2500 * wm * v * scale); - cost = tokens * (svc.baseCost / 1000); - reqs = Math.floor(80 * wm * v * scale); - } else if (scenario < 0.3) { - // High input β†’ RAG bloat candidate - tokens = Math.floor((15000 + rand() * 10000) * wm * v * scale); - cost = tokens * (svc.baseCost / 1000); - reqs = Math.floor(30 * wm * v * scale); - } else if (scenario < 0.45) { - // High volume β†’ batch API candidate - tokens = Math.floor(25000 * wm * v * scale); - cost = tokens * (svc.baseCost / 1000); - reqs = Math.floor(200 * wm * (isMonday ? 1.5 : 0.7) * scale); - } else { - tokens = Math.floor(svc.baseTokens * wm * v * scale); - cost = tokens * (svc.baseCost / 1000); - reqs = Math.floor(60 * wm * v * scale); - } - } else if (svc.endpoint === "audio_transcriptions") { - const minutes = Math.floor((svc.baseTokens * wm * v * scale) / 60); - cost = minutes * 0.006; - reqs = Math.floor(20 * wm * v * scale); - tokens = minutes * 60; - } else { - tokens = Math.floor(svc.baseTokens * wm * v * scale); - cost = tokens * (svc.baseCost / 1000); - reqs = Math.floor(20 * wm * v * scale); - } + const v = 1 - jitter / 2 + rand() * jitter; + const mult = wm * v * scale * growth * monthNorm; - if (cost === 0 && tokens === 0) continue; + const inp = Math.floor(w.inp * mult); + const out = Math.floor(w.out * mult); + const reqs = Math.max(1, Math.floor(w.reqs * mult)); + if (inp === 0 && out === 0) continue; usageRows.push({ aggregation_timestamp: ts, - n_requests: Math.max(1, reqs), - operation: svc.endpoint, - snapshot_id: model, - n_context_tokens_total: Math.floor(tokens * 0.7), - n_generated_tokens_total: Math.floor(tokens * 0.3), - model, - service: svc.endpoint, + n_requests: reqs, + operation: "completions", + snapshot_id: w.model, + n_context_tokens_total: inp, + n_generated_tokens_total: out, + model: w.model, + service: "completions", bucket_start_time: ts, - project_id: project.id, - input_tokens: Math.floor(tokens * 0.7), - output_tokens: Math.floor(tokens * 0.3), - num_model_requests: Math.max(1, reqs), + project_id: w.pid, + input_tokens: inp, + output_tokens: out, + num_model_requests: reqs, }); - dayCostResults.push({ - object: "organization.costs.result", - amount: { value: parseFloat(cost.toFixed(6)), currency: "usd" }, - line_item: model, - project_id: project.id, - project_name: project.name, - organization_id: "org_demo", - organization_name: profile.orgName, - }); + // Costs API figures come from the same token volumes at pricing-table + // rates, so cost-based and token-based views never contradict. + pushCost(tcOpenAI(w.model, inp, out), w.model, w.pid); + } + + for (const s of services) { + const wm = isWeekend ? profile.weekendFactor : 1; + const v = 1 - jitter / 2 + rand() * jitter; + const mult = wm * v * scale * growth * monthNorm; + const reqs = Math.max(1, Math.floor(s.reqs * mult)); + + let cost = 0; + const row: OpenAIUsageData["data"][0] = { + aggregation_timestamp: ts, + n_requests: reqs, + operation: s.service, + snapshot_id: s.model, + n_context_tokens_total: 0, + n_generated_tokens_total: 0, + model: s.model, + service: s.service, + bucket_start_time: ts, + project_id: s.pid, + num_model_requests: reqs, + }; + + if (s.seconds !== undefined) { + const seconds = Math.floor(s.seconds * mult); + row.seconds = seconds; + cost = (seconds / 60) * 0.006; // whisper $0.006/min + } else if (s.tokens !== undefined) { + const tokens = Math.floor(s.tokens * mult); + row.n_context_tokens_total = tokens; + row.input_tokens = tokens; + row.output_tokens = 0; + cost = (tokens / 1e6) * (s.tokenRate ?? 0); + } else if (s.units !== undefined) { + const units = Math.max(1, Math.floor(s.units * mult)); + cost = units * (s.unitCost ?? 0); + } + + usageRows.push(row); + pushCost(cost, s.model, s.pid); } if (dayCostResults.length > 0) { @@ -552,23 +896,26 @@ export function demoOpenAI( has_more: false, }; - const org = { id: "org_demo", name: profile.orgName }; - const now = new Date().toISOString(); + const org = { + id: `org_demo_${(seed >>> 0).toString(36)}`, + name: profile.orgName, + }; + const fetchedAt = new Date(Date.UTC(year, month + 1, 1)).toISOString(); const raw = { completions: { endpoint: "/v1/organization/usage/completions", - fetched_at: now, + fetched_at: fetchedAt, response: { data: [], has_more: false }, }, costs: { endpoint: "/v1/organization/costs", - fetched_at: now, + fetched_at: fetchedAt, response: costs, }, projects: { endpoint: "/v1/organization/projects", - fetched_at: now, - response: { data: OAI_PROJECTS }, + fetched_at: fetchedAt, + response: { data: projects }, }, }; @@ -576,7 +923,7 @@ export function demoOpenAI( org, usage: { data: usageRows }, costs, - projects: OAI_PROJECTS, + projects, raw, }; }