diff --git a/src/__tests__/anthropic-analysis.test.ts b/src/__tests__/anthropic-analysis.test.ts index f45d91a..4c560b2 100644 --- a/src/__tests__/anthropic-analysis.test.ts +++ b/src/__tests__/anthropic-analysis.test.ts @@ -374,6 +374,26 @@ describe("findIssues", () => { } }); + it("stores labeled detection signals on rule findings", () => { + const day = (n: number) => `2024-01-${String(n).padStart(2, "0")}`; + const buckets = Array.from({ length: 25 }, (_, i) => + sonnetBucket(day(i + 1), 500_000, 60_000, 800) + ); + const rows = agg(buckets); + const downgrade = findIssues(rows, [], buckets).find( + (f) => f.cat === "Model Downgrade → Haiku" + ); + expect(downgrade).toBeDefined(); + expect(downgrade!.source).toBe("rules"); + expect(downgrade!.signals!.length).toBeGreaterThan(0); + for (const s of downgrade!.signals!) { + expect(s.label).toBeTruthy(); + expect(typeof s.met).toBe("boolean"); + expect(s.weight).toBeGreaterThan(0); + } + expect(downgrade!.signals!.some((s) => s.met)).toBe(true); + }); + it("savings are positive and less than current cost", () => { const day = (n: number) => `2024-01-${String(n).padStart(2, "0")}`; const buckets = Array.from({ length: 25 }, (_, i) => diff --git a/src/__tests__/costing.test.ts b/src/__tests__/costing.test.ts new file mode 100644 index 0000000..41c54e1 --- /dev/null +++ b/src/__tests__/costing.test.ts @@ -0,0 +1,460 @@ +import { describe, it, expect } from "vitest"; +import { + cacheWriteEconomics, + costBatchDiscount, + costEnableCaching, + costGenerationUpgrade, + costHaikuDowngrade, + costRagReduction, + costSonnetDowngrade, + optimizedCostAnthropic, + type CostRow, +} from "@/lib/anthropic/costing"; +import { + costBatchDiscountOpenAI, + costEnableCachingOpenAI, + costGpt4oDowngrade, + costLegacyGpt4Upgrade, + costMiniDowngrade, + costModelUpgradeOpenAI, + costPromptTrim, + costRagReductionOpenAI, + costTenPercentTrim, + optimizedCostOpenAI, + type OpenAICostRow, +} from "@/lib/openai/costing"; +import { agg, findIssues } from "@/lib/anthropic/analysis"; +import { findIssuesOpenAI } from "@/lib/openai/analysis"; +import { tc } from "@/lib/anthropic/pricing"; +import { tcOpenAI } from "@/lib/openai/pricing"; +import { AnthropicCategory, OpenAICategory } from "@/types/analysis"; +import type { UsageBucket } from "@/types"; + +const day = (n: number) => `2024-01-${String(n).padStart(2, "0")}T00:00:00Z`; + +function anthBuckets( + model: string, + perDay: Partial, + days = 25 +): UsageBucket[] { + return Array.from({ length: days }, (_, i) => ({ + bucket_start: day(i + 1), + model, + api_key_id: "key_test", + workspace_id: "ws_test", + input_tokens: 0, + output_tokens: 0, + request_count: 0, + ...perDay, + })); +} + +function anthCostRow(model: string, buckets: UsageBucket[]): CostRow { + const [r] = agg(buckets); + return { + model, + inp: r.inp, + out: r.out, + cached: r.cached, + cacheCreated: r.cacheCreated, + cur: tc(model, r.inp, r.out), + conf: 0, + }; +} + +/* ─── Anthropic: costing functions match the rule engine's own output ─── */ + +describe("Anthropic costing functions vs rule engine", () => { + it("costHaikuDowngrade matches rule 1's optimized cost", () => { + const buckets = anthBuckets("claude-sonnet-4-6", { + input_tokens: 500_000, + output_tokens: 60_000, + request_count: 800, + }); + const rows = agg(buckets); + const f = findIssues(rows, [], buckets).find( + (x) => x.cat === AnthropicCategory.MODEL_DOWNGRADE_HAIKU + ); + expect(f).toBeDefined(); + expect(f!.opt).toBeCloseTo( + costHaikuDowngrade(anthCostRow("claude-sonnet-4-6", buckets)), + 10 + ); + }); + + it("costRagReduction matches rule 2's optimized cost", () => { + const buckets = anthBuckets("claude-sonnet-4-6", { + input_tokens: 12_000_000, + output_tokens: 200_000, + request_count: 200, + }); + const rows = agg(buckets); + const f = findIssues(rows, [], buckets).find( + (x) => x.cat === AnthropicCategory.RAG_OPTIMIZATION + ); + expect(f).toBeDefined(); + expect(f!.opt).toBeCloseTo( + costRagReduction(anthCostRow("claude-sonnet-4-6", buckets), f!.conf), + 10 + ); + }); + + it("costRagReduction reprices Opus rows at Sonnet", () => { + const row: CostRow = { + model: "claude-opus-4-5", + inp: 10_000_000, + out: 100_000, + cached: 0, + cacheCreated: 0, + cur: tc("claude-opus-4-5", 10_000_000, 100_000), + conf: 0, + }; + // 50% input reduction at Sonnet ($3/$15) rates + expect(costRagReduction(row, 0.9)).toBeCloseTo( + (5_000_000 / 1e6) * 3 + (100_000 / 1e6) * 15, + 10 + ); + }); + + it("costEnableCaching matches rule 3's optimized cost", () => { + const buckets = anthBuckets("claude-sonnet-4-6", { + input_tokens: 3_000_000, + output_tokens: 200_000, + request_count: 300, + cache_read_input_tokens: 0, + }); + const rows = agg(buckets); + const f = findIssues(rows, [], buckets).find( + (x) => x.cat === AnthropicCategory.PROMPT_CACHING + ); + expect(f).toBeDefined(); + expect(f!.opt).toBeCloseTo( + costEnableCaching(anthCostRow("claude-sonnet-4-6", buckets)), + 10 + ); + }); + + it("costSonnetDowngrade matches rule 4's optimized cost", () => { + const buckets = anthBuckets("claude-opus-4-5", { + input_tokens: 1_000_000, + output_tokens: 300_000, + request_count: 200, + }); + const rows = agg(buckets); + const f = findIssues(rows, [], buckets).find( + (x) => x.cat === AnthropicCategory.MODEL_DOWNGRADE_SONNET + ); + expect(f).toBeDefined(); + expect(f!.opt).toBeCloseTo( + costSonnetDowngrade(anthCostRow("claude-opus-4-5", buckets)), + 10 + ); + }); + + it("costBatchDiscount matches rule 5's optimized cost (50% off)", () => { + // Bursty: big spikes with dead days in between. + const buckets: UsageBucket[] = Array.from({ length: 20 }, (_, i) => ({ + bucket_start: day(i + 1), + model: "claude-sonnet-4-6", + api_key_id: "key_test", + workspace_id: "ws_test", + input_tokens: i % 5 === 0 ? 3_000_000 : 0, + output_tokens: i % 5 === 0 ? 500_000 : 0, + request_count: i % 5 === 0 ? 2000 : 0, + })); + const rows = agg(buckets); + const f = findIssues(rows, [], buckets).find( + (x) => x.cat === AnthropicCategory.BATCH_API_MIGRATION + ); + expect(f).toBeDefined(); + const row = anthCostRow("claude-sonnet-4-6", buckets); + expect(f!.opt).toBeCloseTo(costBatchDiscount(row), 10); + expect(costBatchDiscount(row)).toBeCloseTo(row.cur * 0.5, 10); + }); + + it("cacheWriteEconomics matches rule 5b's optimized cost", () => { + const buckets = anthBuckets("claude-sonnet-4-6", { + input_tokens: 400_000, + output_tokens: 300_000, + request_count: 100, + cache_creation_input_tokens: 400_000, + cache_read_input_tokens: 20_000, + }); + const rows = agg(buckets); + const f = findIssues(rows, [], buckets).find( + (x) => x.cat === AnthropicCategory.PROMPT_CACHING + ); + expect(f).toBeDefined(); + const econ = cacheWriteEconomics(anthCostRow("claude-sonnet-4-6", buckets)); + expect(econ).not.toBeNull(); + expect(f!.opt).toBeCloseTo(econ!.opt, 10); + expect(econ!.netCacheCost).toBeGreaterThan(1); + }); + + it("cacheWriteEconomics returns null without cache writes", () => { + expect( + cacheWriteEconomics({ + model: "claude-sonnet-4-6", + inp: 1e6, + out: 1e5, + cached: 0, + cacheCreated: 0, + cur: 5, + conf: 0, + }) + ).toBeNull(); + }); + + it("costGenerationUpgrade matches rule 6 and never exceeds current cost", () => { + const buckets = anthBuckets("claude-3-opus-20240229", { + input_tokens: 200_000, + output_tokens: 40_000, + request_count: 50, + }); + const rows = agg(buckets); + const f = findIssues(rows, [], buckets).find( + (x) => x.cat === AnthropicCategory.MODEL_UPGRADE + ); + expect(f).toBeDefined(); + const row = anthCostRow("claude-3-opus-20240229", buckets); + expect(f!.opt).toBeCloseTo(costGenerationUpgrade(row), 10); + expect(costGenerationUpgrade(row)).toBeLessThanOrEqual(row.cur); + }); +}); + +describe("optimizedCostAnthropic dispatcher", () => { + const row: CostRow = { + model: "claude-opus-4-6", + inp: 10_000_000, + out: 500_000, + cached: 0, + cacheCreated: 0, + cur: tc("claude-opus-4-6", 10_000_000, 500_000), + conf: 0.8, + }; + + it("routes each category to its formula", () => { + expect( + optimizedCostAnthropic(AnthropicCategory.MODEL_DOWNGRADE_HAIKU, row) + ).toBeCloseTo(costHaikuDowngrade(row), 10); + expect( + optimizedCostAnthropic(AnthropicCategory.MODEL_DOWNGRADE_SONNET, row) + ).toBeCloseTo(costSonnetDowngrade(row), 10); + expect( + optimizedCostAnthropic(AnthropicCategory.RAG_OPTIMIZATION, row) + ).toBeCloseTo(costRagReduction(row, row.conf), 10); + expect( + optimizedCostAnthropic(AnthropicCategory.PROMPT_CACHING, row) + ).toBeCloseTo(costEnableCaching(row), 10); + expect( + optimizedCostAnthropic(AnthropicCategory.BATCH_API_MIGRATION, row) + ).toBeCloseTo(row.cur * 0.5, 10); + expect( + optimizedCostAnthropic(AnthropicCategory.MODEL_UPGRADE, row) + ).toBeCloseTo(costGenerationUpgrade(row), 10); + }); + + it("returns null for categories with no formula or unusable rows", () => { + expect( + optimizedCostAnthropic(AnthropicCategory.WORKSPACE_ORGANIZATION, row) + ).toBeNull(); + expect( + optimizedCostAnthropic(AnthropicCategory.PROMPT_OPTIMIZATION, row) + ).toBeNull(); + expect( + optimizedCostAnthropic(AnthropicCategory.RAG_OPTIMIZATION, { + ...row, + inp: 0, + }) + ).toBeNull(); + expect( + optimizedCostAnthropic(AnthropicCategory.MODEL_DOWNGRADE_HAIKU, { + ...row, + inp: 0, + out: 0, + }) + ).toBeNull(); + }); +}); + +/* ─── OpenAI: costing functions match the rule engine's own output ─── */ + +function oaiRow(over: Partial> = {}) { + return { + model: "gpt-4o-2024-08-06", + project_id: "proj_1", + line_item: "GPT-4o", + cost: 0, + inp: 5_000_000, + out: 400_000, + reqs: 5000, + activeDays: 25, + ...over, + }; +} + +function oaiCostRow(r: ReturnType): OpenAICostRow { + return { + model: (r.model as string) || (r.line_item as string) || "", + inp: r.inp as number, + out: r.out as number, + cur: + (r.cost as number) > 0 + ? (r.cost as number) + : tcOpenAI(r.model as string, r.inp as number, r.out as number), + conf: 0, + }; +} + +describe("OpenAI costing functions vs rule engine", () => { + it("costMiniDowngrade matches the mini-downgrade rules", () => { + const r = oaiRow({ cost: 80 }); + const f = findIssuesOpenAI([r], []).find( + (x) => x.cat === OpenAICategory.MODEL_DOWNGRADE_MINI + ); + expect(f).toBeDefined(); + expect(f!.opt).toBeCloseTo(costMiniDowngrade(oaiCostRow(r)), 10); + }); + + it("costRagReductionOpenAI matches rule 2's optimized cost", () => { + const r = oaiRow({ inp: 20_000_000, out: 500_000, reqs: 1500, cost: 120 }); + const f = findIssuesOpenAI([r], []).find( + (x) => x.cat === OpenAICategory.RAG_OPTIMIZATION + ); + expect(f).toBeDefined(); + expect(f!.opt).toBeCloseTo( + costRagReductionOpenAI(oaiCostRow(r), f!.conf), + 10 + ); + }); + + it("costEnableCachingOpenAI matches rule 0's optimized cost", () => { + const r = oaiRow({ inp: 10_000_000, out: 2_000_000, reqs: 2000, cost: 60 }); + const f = findIssuesOpenAI([r], []).find( + (x) => x.cat === OpenAICategory.PROMPT_CACHING + ); + expect(f).toBeDefined(); + expect(f!.opt).toBeCloseTo(costEnableCachingOpenAI(oaiCostRow(r)), 10); + }); + + it("costBatchDiscountOpenAI matches the batch rules (50% off)", () => { + const r = oaiRow({ + inp: 30_000_000, + out: 3_000_000, + reqs: 30_000, + cost: 200, + activeDays: 28, + }); + const f = findIssuesOpenAI([r], []).find( + (x) => x.cat === OpenAICategory.BATCH_API_MIGRATION + ); + expect(f).toBeDefined(); + expect(f!.opt).toBeCloseTo(costBatchDiscountOpenAI(oaiCostRow(r)), 10); + expect(f!.opt).toBeCloseTo(f!.cur * 0.5, 10); + }); + + it("costGpt4oDowngrade matches the o-series overkill rule", () => { + const r = oaiRow({ + model: "o1-preview", + line_item: "o1", + inp: 2_000_000, + out: 150_000, + reqs: 1000, + cost: 45, + }); + const f = findIssuesOpenAI([r], []).find( + (x) => x.cat === OpenAICategory.REASONING_MODEL_OVERKILL + ); + expect(f).toBeDefined(); + expect(f!.opt).toBeCloseTo(costGpt4oDowngrade(oaiCostRow(r)), 10); + }); + + it("costPromptTrim matches rule 8's optimized cost", () => { + const r = oaiRow({ + inp: 30_000_000, + out: 500_000, + reqs: 2000, + cost: 100, + }); + const f = findIssuesOpenAI([r], []).find( + (x) => x.cat === OpenAICategory.PROMPT_OPTIMIZATION + ); + expect(f).toBeDefined(); + expect(f!.opt).toBeCloseTo(costPromptTrim(oaiCostRow(r)), 10); + }); + + it("costLegacyGpt4Upgrade matches rule 7's optimized cost", () => { + const r = oaiRow({ + model: "gpt-4-0613", + line_item: "GPT-4", + inp: 1_000_000, + out: 100_000, + reqs: 300, + cost: 40, + }); + const f = findIssuesOpenAI([r], []).find( + (x) => x.cat === OpenAICategory.MODEL_UPGRADE + ); + expect(f).toBeDefined(); + expect(f!.opt).toBeCloseTo(costLegacyGpt4Upgrade(oaiCostRow(r)), 10); + }); + + it("costTenPercentTrim matches the high-impact rules", () => { + const r = oaiRow({ cost: 90 }); + const f = findIssuesOpenAI([r], []).find( + (x) => x.cat === OpenAICategory.HIGH_IMPACT_OPPORTUNITY + ); + expect(f).toBeDefined(); + expect(f!.opt).toBeCloseTo(costTenPercentTrim(oaiCostRow(r)), 10); + }); +}); + +describe("optimizedCostOpenAI dispatcher", () => { + const row: OpenAICostRow = { + model: "gpt-4o", + inp: 5_000_000, + out: 400_000, + cur: tcOpenAI("gpt-4o", 5_000_000, 400_000), + conf: 0.8, + }; + + it("routes each category to its formula", () => { + expect( + optimizedCostOpenAI(OpenAICategory.MODEL_DOWNGRADE_MINI, row) + ).toBeCloseTo(costMiniDowngrade(row), 10); + expect( + optimizedCostOpenAI(OpenAICategory.RAG_OPTIMIZATION, row) + ).toBeCloseTo(costRagReductionOpenAI(row, row.conf), 10); + expect(optimizedCostOpenAI(OpenAICategory.PROMPT_CACHING, row)).toBeCloseTo( + costEnableCachingOpenAI(row), + 10 + ); + expect( + optimizedCostOpenAI(OpenAICategory.BATCH_API_MIGRATION, row) + ).toBeCloseTo(row.cur * 0.5, 10); + expect( + optimizedCostOpenAI(OpenAICategory.PROMPT_OPTIMIZATION, row) + ).toBeCloseTo(costPromptTrim(row), 10); + expect(optimizedCostOpenAI(OpenAICategory.MODEL_UPGRADE, row)).toBeCloseTo( + costModelUpgradeOpenAI(row), + 10 + ); + }); + + it("returns null for uncostable categories and token-less rows", () => { + expect( + optimizedCostOpenAI(OpenAICategory.PROJECT_ORGANIZATION, row) + ).toBeNull(); + const noTokens: OpenAICostRow = { ...row, inp: 0, out: 0, cur: 50 }; + expect( + optimizedCostOpenAI(OpenAICategory.MODEL_DOWNGRADE_MINI, noTokens) + ).toBeNull(); + expect( + optimizedCostOpenAI(OpenAICategory.RAG_OPTIMIZATION, noTokens) + ).toBeNull(); + // Whole-workload discounts still work from cost alone. + expect( + optimizedCostOpenAI(OpenAICategory.BATCH_API_MIGRATION, noTokens) + ).toBeCloseTo(25, 10); + }); +}); diff --git a/src/__tests__/demo-determinism.test.ts b/src/__tests__/demo-determinism.test.ts index 6495931..45cb6d8 100644 --- a/src/__tests__/demo-determinism.test.ts +++ b/src/__tests__/demo-determinism.test.ts @@ -106,4 +106,31 @@ describe("demo mode determinism", () => { const b = demoAnthropic(BASE_YEAR, BASE_MONTH, DEMO_SEED + 1); expect(JSON.stringify(a.bm)).not.toBe(JSON.stringify(b.bm)); }); + + it("the demo org has 8 workspaces plus busy default-workspace traffic", () => { + const d = demoAnthropic(BASE_YEAR, BASE_MONTH, DEMO_SEED); + expect(d.ws).toHaveLength(8); + // Some traffic deliberately carries no workspace_id → default workspace. + expect(d.bw.some((b) => !b.workspace_id)).toBe(true); + expect(d.bw.some((b) => b.workspace_id === "ws_prod")).toBe(true); + }); + + it("the demo's varied workloads fire most rule categories", () => { + const cats = new Set( + runAnthropicDemo().flatMap((r) => r.findings.map((f) => f.cat as string)) + ); + for (const expected of [ + "Model Downgrade → Haiku", + "Model Downgrade → Sonnet", + "RAG Optimization", + "Prompt Caching", + "Batch API Migration", + "Model Upgrade", + "Workspace Organization", + ]) { + expect(cats, `expected category "${expected}" to fire`).toContain( + expected + ); + } + }); }); diff --git a/src/__tests__/nim-analysis.test.ts b/src/__tests__/nim-analysis.test.ts index 94fa32b..3947a67 100644 --- a/src/__tests__/nim-analysis.test.ts +++ b/src/__tests__/nim-analysis.test.ts @@ -1,11 +1,18 @@ -import { describe, it, expect } from "vitest"; +import { describe, it, expect, vi, afterEach } from "vitest"; import { - mergeLlmFindings, buildSystemPrompt, - type UsageSummary, + findIssuesLLM, + mergeConsensus, + priceLlmFindings, + resolveCategory, + validCategories, type AnalysisContext, + type LlmProposal, + type UsageSummary, } from "@/lib/nim/analysis"; -import { Severity } from "@/types/analysis"; +import { costEnableCaching, costHaikuDowngrade } from "@/lib/anthropic/costing"; +import { AnthropicCategory, Severity } from "@/types/analysis"; +import type { Finding } from "@/types"; const row: UsageSummary = { id: "key1-ws1", @@ -28,212 +35,317 @@ const ctx: AnalysisContext = { workspaceCount: 1, }; -describe("mergeLlmFindings", () => { - it("clamps savings to [0, currentCost] and grounds dollar figures", () => { - const [f] = mergeLlmFindings( +const proposal = (over: Partial = {}): LlmProposal => ({ + rowId: "key1-ws1", + category: AnthropicCategory.PROMPT_CACHING, + severity: "warning", + confidence: 0.7, + reason: "r", + action: "a", + ...over, +}); + +describe("priceLlmFindings", () => { + it("prices proposals deterministically from the costing module", () => { + const [f] = priceLlmFindings( [ - { - rowId: "key1-ws1", + proposal({ category: "Model Downgrade → Haiku", severity: "critical", confidence: 0.9, - savingsMonthly: 9999, // absurd; must clamp to cur - reason: "low output", - action: "switch model", - }, + }), ], [row], ctx ); - expect(f.sav).toBe(200); - expect(f.opt).toBe(0); - expect(f.cur).toBe(200); + const expectedOpt = costHaikuDowngrade({ + model: row.model, + inp: row.inp, + out: row.out, + cached: 0, + cacheCreated: 0, + cur: row.cur, + conf: 0.9, + }); + expect(f.opt).toBeCloseTo(expectedOpt, 10); + expect(f.sav).toBeCloseTo(row.cur - expectedOpt, 10); expect(f.sev).toBe(Severity.CRITICAL); expect(f.model).toBe("claude-opus-4-6"); + expect(f.source).toBe("llm"); + expect(f.cat).toBe(AnthropicCategory.MODEL_DOWNGRADE_HAIKU); }); - it("clamps negative savings to 0 and confidence to [0,1]", () => { - const [f] = mergeLlmFindings( - [ - { - rowId: "key1-ws1", - category: "Model Upgrade", // quality category survives zero savings - severity: "info", - confidence: 5, - savingsMonthly: -50, - reason: "r", - action: "a", - }, - ], - [row], - ctx + it("prices a caching proposal with the enable-caching formula", () => { + const [f] = priceLlmFindings([proposal()], [row], ctx); + expect(f.opt).toBeCloseTo( + costEnableCaching({ + model: row.model, + inp: row.inp, + out: row.out, + cached: 0, + cacheCreated: 0, + cur: row.cur, + conf: 0.7, + }), + 10 ); - expect(f.sav).toBe(0); - expect(f.conf).toBe(1); - expect(f.impact).toBe("Quality improvement"); }); - it("handles org-level findings with no matching row", () => { - const [f] = mergeLlmFindings( - [ - { - rowId: "org", - category: "Workspace Organization", - severity: "info", - confidence: 0.8, - savingsMonthly: 0, - reason: "all spend in default", - action: "split workspaces", - }, - ], + it("drops proposals whose category is outside the fixed enum", () => { + const out = priceLlmFindings( + [proposal({ category: "Made Up Category" })], [row], ctx ); - expect(f.cur).toBe(200); // falls back to totalSpend - expect(f.name).toBe("Organization"); + expect(out).toHaveLength(0); }); - it("drops findings without a reason and sorts by severity then savings", () => { - const out = mergeLlmFindings( + it("drops uncostable cost categories but keeps org categories at $0", () => { + const out = priceLlmFindings( [ - { - rowId: "key1-ws1", - category: "A", + // RAG on a row with no matching id → no metrics → uncostable → dropped + proposal({ rowId: "missing", category: "RAG Optimization" }), + // Org-structure finding survives with zero savings + proposal({ + rowId: "org", + category: "Workspace Organization", severity: "info", - confidence: 0.5, - savingsMonthly: 10, - reason: "r", - action: "a", - }, - { - rowId: "bad", - category: "B", - severity: "critical", - confidence: 0.5, - savingsMonthly: 10, - reason: "", - action: "a", - }, - { - rowId: "key1-ws1", - category: "C", - severity: "critical", - confidence: 0.5, - savingsMonthly: 50, - reason: "r", - action: "a", - }, + }), ], [row], ctx ); - expect(out).toHaveLength(2); // empty-reason dropped - expect(out[0].sev).toBe(Severity.CRITICAL); - expect(out[1].sev).toBe(Severity.INFO); + expect(out).toHaveLength(1); + expect(out[0].cat).toBe(AnthropicCategory.WORKSPACE_ORGANIZATION); + expect(out[0].sav).toBe(0); + expect(out[0].cur).toBe(200); // falls back to totalSpend + expect(out[0].name).toBe("Organization"); + expect(out[0].impact).toBe("Quality improvement"); }); -}); -describe("mergeLlmFindings guardrails", () => { - // A cheap Haiku row — downgrades to pricier tiers must be rejected. - const haikuRow: UsageSummary = { - ...row, - id: "key2-ws1", - name: "key2", - model: "claude-haiku-3", - ml: "Haiku 3", - cur: 0.03, - }; - const f = (over: Partial>) => ({ - rowId: "key1-ws1", - category: "X", - severity: "warning", - confidence: 0.7, - savingsMonthly: 10, - reason: "r", - action: "a", - ...over, + it("clamps confidence to [0,1]", () => { + const [f] = priceLlmFindings([proposal({ confidence: 5 })], [row], ctx); + expect(f.conf).toBe(1); }); it("drops a 'downgrade' whose target is not cheaper than the row's model", () => { - const out = mergeLlmFindings( + const haikuRow: UsageSummary = { + ...row, + id: "key2-ws1", + model: "claude-haiku-3", + ml: "Haiku 3", + cur: 5, + }; + const out = priceLlmFindings( [ - f({ - rowId: "key2-ws1", - category: "Model Downgrade → Sonnet", - savingsMonthly: 0.01, - }), - f({ - rowId: "key2-ws1", - category: "Model Downgrade → Haiku", - savingsMonthly: 0.01, - }), + proposal({ rowId: "key2-ws1", category: "Model Downgrade → Sonnet" }), + proposal({ rowId: "key2-ws1", category: "Model Downgrade → Haiku" }), ], [haikuRow], ctx ); - // Haiku→Sonnet (upgrade) and Haiku→Haiku (no-op) both dropped. expect(out).toHaveLength(0); }); it("keeps only the single best downgrade per row", () => { - const out = mergeLlmFindings( + const out = priceLlmFindings( [ - f({ category: "Model Downgrade → Sonnet", savingsMonthly: 80 }), - f({ category: "Model Downgrade → Haiku", savingsMonthly: 90 }), + proposal({ category: "Model Downgrade → Sonnet" }), + proposal({ category: "Model Downgrade → Haiku" }), ], [row], ctx ); + // Haiku repricing saves more than Sonnet repricing on an Opus row. expect(out).toHaveLength(1); - expect(out[0].cat).toContain("Haiku"); - expect(out[0].sav).toBe(90); + expect(out[0].cat).toBe(AnthropicCategory.MODEL_DOWNGRADE_HAIKU); }); - it("caps cumulative savings per row at the row's spend", () => { - const out = mergeLlmFindings( + it("dedupes repeated (row, category) proposals", () => { + const out = priceLlmFindings([proposal(), proposal()], [row], ctx); + expect(out).toHaveLength(1); + }); + + it("drops proposals without a reason and sorts by severity then savings", () => { + const out = priceLlmFindings( [ - f({ category: "Prompt Caching", savingsMonthly: 150 }), - f({ category: "RAG Optimization", savingsMonthly: 150 }), + proposal({ category: "Batch API Migration", severity: "info" }), + proposal({ reason: "", severity: "critical" }), + proposal({ category: "Prompt Caching", severity: "critical" }), ], - [row], // cur = 200 + [row], + ctx + ); + expect(out).toHaveLength(2); // empty-reason dropped + expect(out[0].sev).toBe(Severity.CRITICAL); + expect(out[1].sev).toBe(Severity.INFO); + }); +}); + +describe("resolveCategory", () => { + it("canonicalizes arrow and case drift", () => { + expect(resolveCategory("anthropic", "model downgrade -> haiku")).toBe( + AnthropicCategory.MODEL_DOWNGRADE_HAIKU + ); + expect(resolveCategory("anthropic", "Prompt Caching")).toBe( + AnthropicCategory.PROMPT_CACHING + ); + expect(resolveCategory("anthropic", "nonsense")).toBeNull(); + }); +}); + +/* ─── Consensus merge ─── */ + +const mkRuleFinding = (over: Partial = {}): Finding => ({ + id: "key1-ws1-prompt-caching", + name: "key1", + ws: "production", + model: "claude-opus-4-6", + ml: "Opus 4.6", + inp: 10_000_000, + out: 100_000, + cached: 0, + reqs: 1000, + ao: 100, + ai: 10_000, + ratio: 100, + cr: 0, + cur: 200, + opt: 120, + sav: 80, + reason: "rule reason", + action: "rule action", + sev: Severity.WARNING, + cat: AnthropicCategory.PROMPT_CACHING, + conf: 0.8, + impact: "$80.00/mo (40%)", + activeDays: 25, + temporal: { + burstiness: 0.2, + consistency: 0.8, + batchCandidate: false, + meanDaily: 300, + }, + source: "rules", + ...over, +}); + +describe("mergeConsensus", () => { + it("merges findings found by both engines, keeping the rule's text and price", () => { + const ruleF = mkRuleFinding({ conf: 0.7 }); + const [llmF] = priceLlmFindings( + [proposal({ confidence: 0.6, reason: "llm reason" })], + [row], ctx ); - const total = out.reduce((s, x) => s + x.sav, 0); - expect(total).toBeLessThanOrEqual(200); - expect(total).toBeCloseTo(200); + const merged = mergeConsensus([ruleF], [llmF]); + expect(merged).toHaveLength(1); + expect(merged[0].source).toBe("both"); + expect(merged[0].reason).toBe("rule reason"); + expect(merged[0].action).toBe("rule action"); + expect(merged[0].sav).toBe(80); // rule's deterministic price kept + expect(merged[0].conf).toBeCloseTo(0.8); // max(0.7, 0.6) + 0.1 }); - it("drops zero-savings cost findings as noise", () => { - const out = mergeLlmFindings( - [f({ category: "Batch API Migration", savingsMonthly: 0 })], + it("caps the consensus confidence boost at 0.95", () => { + const ruleF = mkRuleFinding({ conf: 0.92 }); + const [llmF] = priceLlmFindings( + [proposal({ confidence: 0.9 })], [row], ctx ); - expect(out).toHaveLength(0); + const merged = mergeConsensus([ruleF], [llmF]); + expect(merged[0].conf).toBe(0.95); + }); + + it("keeps rules-only findings unchanged with source 'rules'", () => { + const ruleF = mkRuleFinding(); + const merged = mergeConsensus([ruleF], []); + expect(merged).toHaveLength(1); + expect(merged[0].source).toBe("rules"); + expect(merged[0].conf).toBe(0.8); + expect(merged[0].sav).toBe(80); + }); + + it("keeps LLM-only findings with their deterministic price and source 'llm'", () => { + const ruleF = mkRuleFinding(); // prompt caching from rules + const [llmF] = priceLlmFindings( + [proposal({ category: "Batch API Migration", reason: "bursty" })], + [row], + ctx + ); + const merged = mergeConsensus([ruleF], [llmF]); + expect(merged).toHaveLength(2); + const llmOnly = merged.find((f) => f.source === "llm"); + expect(llmOnly).toBeDefined(); + expect(llmOnly!.cat).toBe(AnthropicCategory.BATCH_API_MIGRATION); + expect(llmOnly!.sav).toBeCloseTo(100, 10); // 50% of $200, priced by code + expect(llmOnly!.reason).toBe("bursty"); }); - it("drops cost findings the cumulative cap zeroed out (no $0 'quality' noise)", () => { - // First finding claims the row's whole cost; later ones get capped to $0 - // and must not survive as junk "Quality improvement" rows. - const out = mergeLlmFindings( + it("merges org-structure findings from both engines despite different id shapes", () => { + const ruleOrg = mkRuleFinding({ + id: "workspace-organization-unused", + cat: AnthropicCategory.WORKSPACE_ORGANIZATION, + sev: Severity.INFO, + sav: 0, + conf: 1.0, + }); + const [llmOrg] = priceLlmFindings( [ - f({ category: "Prompt Caching", savingsMonthly: 200 }), // = full cur - f({ category: "RAG Optimization", savingsMonthly: 50 }), - f({ category: "Batch API Migration", savingsMonthly: 50 }), + proposal({ + rowId: "org", + category: "Workspace Organization", + severity: "info", + }), ], - [row], // cur = 200 + [row], ctx ); - expect(out).toHaveLength(1); - expect(out[0].cat).toBe("Prompt Caching"); - expect(out.every((x) => x.sav > 0)).toBe(true); + const merged = mergeConsensus([ruleOrg], [llmOrg]); + expect(merged).toHaveLength(1); + expect(merged[0].source).toBe("both"); }); }); -describe("buildSystemPrompt", () => { - it("includes the analysis rules and vendor-appropriate models", () => { +/* ─── LLM contract: the model never sees or emits a savings field ─── */ + +describe("LLM payload schema", () => { + it("the system prompt contains no savings field and lists valid categories", () => { + for (const vendor of ["anthropic", "openai"] as const) { + const prompt = buildSystemPrompt(vendor); + expect(prompt).not.toContain("savingsMonthly"); + expect(prompt).not.toMatch(/"savings/i); + for (const c of validCategories(vendor)) { + expect(prompt).toContain(`"${c}"`); + } + } expect(buildSystemPrompt("anthropic")).toContain("claude-haiku-4-5"); expect(buildSystemPrompt("openai")).toContain("gpt-4o-mini"); expect(buildSystemPrompt("anthropic")).toContain("RAG CONTEXT BLOAT"); }); + + it("the request payload sent to NIM contains no savings field", async () => { + let captured = ""; + vi.stubGlobal( + "fetch", + vi.fn(async (_url: string, init: RequestInit) => { + captured = String(init.body); + return new Response( + JSON.stringify({ + choices: [{ message: { content: '{"findings": []}' } }], + }), + { status: 200 } + ); + }) + ); + await findIssuesLLM([row], ctx); + expect(captured).not.toBe(""); + expect(captured).not.toContain("savingsMonthly"); + expect(captured.toLowerCase()).not.toContain('"savings'); + }); + + afterEach(() => { + vi.unstubAllGlobals(); + }); }); diff --git a/src/__tests__/nim-fallback.test.ts b/src/__tests__/nim-fallback.test.ts index bfaf219..a181ddc 100644 --- a/src/__tests__/nim-fallback.test.ts +++ b/src/__tests__/nim-fallback.test.ts @@ -4,7 +4,7 @@ import type { Finding } from "@/types"; import { Severity, AnthropicCategory } from "@/types/analysis"; const ruleFinding: Finding = { - id: "key1-ws1-caching", + id: "key1-ws1-prompt-caching", name: "key1", ws: "production", model: "claude-opus-4-6", @@ -33,10 +33,22 @@ const ruleFinding: Finding = { batchCandidate: false, meanDaily: 300_000, }, + source: "rules", }; -describe("analyzeWithFallback", () => { - it("falls back to the rule engine and sets the notice when the LLM throws", async () => { +const llmFinding: Finding = { + ...ruleFinding, + id: "key1-ws1-batch-api-migration", + cat: AnthropicCategory.BATCH_API_MIGRATION, + sav: 100, + opt: 100, + reason: "bursty traffic", + action: "move to batch", + source: "llm", +}; + +describe("analyzeWithFallback (consensus)", () => { + it("degrades to rules-only findings with the notice when the LLM throws", async () => { vi.spyOn(console, "warn").mockImplementation(() => {}); let rulesRan = false; @@ -56,7 +68,7 @@ describe("analyzeWithFallback", () => { vi.restoreAllMocks(); }); - it("returns the LLM findings and usage with engine 'llm' on success", async () => { + it("runs BOTH engines and merges with engine 'hybrid' on success", async () => { const usage = { promptTokens: 4000, completionTokens: 800, @@ -65,17 +77,38 @@ describe("analyzeWithFallback", () => { let rulesRan = false; const out = await analyzeWithFallback( - async () => ({ findings: [ruleFinding], usage }), + async () => ({ findings: [llmFinding], usage }), () => { rulesRan = true; - return []; + return [ruleFinding]; } ); - expect(rulesRan).toBe(false); - expect(out.findings).toEqual([ruleFinding]); - expect(out.engine).toBe("llm"); + // Rules always run, even when the LLM succeeds. + expect(rulesRan).toBe(true); + expect(out.engine).toBe("hybrid"); expect(out.notice).toBeUndefined(); expect(out.llmUsage).toEqual(usage); + expect(out.findings).toHaveLength(2); + expect(out.findings.map((f) => f.source).sort()).toEqual(["llm", "rules"]); + }); + + it("marks a finding both engines agree on as source 'both' with boosted confidence", async () => { + const llmDuplicate: Finding = { + ...ruleFinding, + reason: "llm version", + conf: 0.6, + source: "llm", + }; + + const out = await analyzeWithFallback( + async () => ({ findings: [llmDuplicate] }), + () => [ruleFinding] + ); + + expect(out.findings).toHaveLength(1); + expect(out.findings[0].source).toBe("both"); + expect(out.findings[0].reason).toBe(ruleFinding.reason); + expect(out.findings[0].conf).toBeCloseTo(0.9); // max(0.8, 0.6) + 0.1 }); }); diff --git a/src/app/history/[id]/recommendations/page.tsx b/src/app/history/[id]/recommendations/page.tsx index 86a75f7..3eee5fc 100644 --- a/src/app/history/[id]/recommendations/page.tsx +++ b/src/app/history/[id]/recommendations/page.tsx @@ -586,7 +586,11 @@ function RecommendationsPageContent() { className="px-2 py-0.5 rounded-full border border-ink-border bg-ink-elevated text-[10px] font-mono text-bone-subtle" title="Which analysis engine produced this report" > - {(r.engine ?? "rules") === "llm" ? "LLM analysis" : "Rule engine"} + {(r.engine ?? "rules") === "hybrid" + ? "Rules + AI consensus" + : (r.engine ?? "rules") === "llm" + ? "LLM analysis" + : "Rule engine"} {r && r.savings > 0 && ( @@ -1195,7 +1199,7 @@ function RecommendationsPageContent() {

Prices as of {pricingDate}

- {r.engine === "llm" && r.llmUsage && ( + {(r.engine === "llm" || r.engine === "hybrid") && r.llmUsage && (

This analysis cost ~$ {r.llmUsage.costUsd < 0.01 diff --git a/src/app/page.tsx b/src/app/page.tsx index 41b5db2..40ae42d 100644 --- a/src/app/page.tsx +++ b/src/app/page.tsx @@ -172,9 +172,10 @@ function HomeContent() { const [step, setStep] = useState(""); const [tickerItems, setTickerItems] = useState(DEFAULT_TICKER); - // NVIDIA NIM LLM analysis: a persisted on/off setting. The key lives in a - // server env var (NIM_API_KEY); the toggle only shows when the server reports - // NIM is configured. + // NVIDIA NIM AI augmentation: a persisted on/off setting. When on, the rule + // engine still runs and an LLM augments it (consensus merge) — it never + // replaces the rules. The key lives in a server env var (NIM_API_KEY); the + // toggle only shows when the server reports NIM is configured. const [nimAvailable, setNimAvailable] = useState(false); const [useNim, setUseNim] = useState(false); @@ -257,9 +258,10 @@ function HomeContent() { router.push(`/?${params.toString()}`); }; - // Choose NIM-guided LLM analysis when the setting is on, else the rule engine. - // A NIM failure falls back to the deterministic rules and stamps a notice on - // the report instead of erroring out. Used by both the real analysis and the demo. + // When AI augmentation is on, run BOTH engines: rules first, then the LLM, + // merged with per-finding provenance (engine "hybrid"). A NIM failure + // degrades to the rules-only findings with a notice — the report is never + // empty. Used by both the real analysis and the demo. const nimOn = () => useNim && nimAvailable; const analyzeAnthropic = async ( @@ -269,7 +271,7 @@ function HomeContent() { spend: number ): Promise => { if (nimOn()) { - setStep("Analyzing usage with NVIDIA NIM..."); + setStep("Augmenting rule analysis with NVIDIA NIM..."); return analyzeWithFallback( () => findIssuesLLM(toSummariesAnthropic(src, ws, buckets), { @@ -289,7 +291,7 @@ function HomeContent() { spend: number ): Promise => { if (nimOn()) { - setStep("Analyzing usage with NVIDIA NIM..."); + setStep("Augmenting rule analysis with NVIDIA NIM..."); return analyzeWithFallback( () => findIssuesLLM(toSummariesOpenAI(rows, projects), { @@ -875,7 +877,7 @@ function HomeContent() { : "Console → API Keys → Admin Keys → Create admin key (read-only)"}

- {/* AI analysis setting — only shown when NIM is configured server-side */} + {/* AI augmentation setting — only shown when NIM is configured server-side */} {nimAvailable && ( )} diff --git a/src/components/Row.tsx b/src/components/Row.tsx index b9d0a6f..4e9cb83 100644 --- a/src/components/Row.tsx +++ b/src/components/Row.tsx @@ -17,6 +17,36 @@ interface RowProps { index: number; } +/** Per-finding provenance badge: which engine(s) surfaced this finding. */ +function SourceBadge({ source }: { source: NonNullable }) { + const style = + source === "both" + ? "border-moss/30 bg-moss/10 text-moss-light" + : source === "llm" + ? "border-info/30 bg-info/10 text-info" + : "border-ink-border bg-ink text-bone-subtle"; + const label = + source === "both" + ? "Rules + AI" + : source === "llm" + ? "AI-spotted" + : "Rules"; + return ( + + {label} + + ); +} + function Row({ f, open, toggle, vendor, index }: RowProps) { const isAnthropic = vendor === Vendor.ANTHROPIC; const isOpenAI = vendor === Vendor.OPENAI; @@ -56,11 +86,14 @@ function Row({ f, open, toggle, vendor, index }: RowProps) { {/* Title + meta */}
- - {f.name} + + + {f.name} + + {f.source && }

{f.cat} · {f.ws} · {f.ml} @@ -156,6 +189,56 @@ function Row({ f, open, toggle, vendor, index }: RowProps) { this pattern

+ {/* Provenance + detection signal trail */} + {(f.source === "llm" || + f.source === "both" || + (f.signals && f.signals.length > 0)) && ( +
+ {f.source === "llm" && ( +

+ Spotted by AI · priced deterministically from your usage + data +

+ )} + {f.source === "both" && ( +

+ ✓ Confirmed by AI +

+ )} + {f.signals && f.signals.length > 0 && ( +
+

+ Based on: +

+
+ {f.signals.map((s, i) => ( + + {s.met ? "✓" : "○"} {s.label} + + ))} +
+
+ )} + {f.source === "llm" && ( +

+ Priced from this row's data: {T(f.inp)} input ·{" "} + {T(f.out)} output tokens · {f.reqs.toLocaleString()}{" "} + requests · {$(f.cur)}/mo current cost +

+ )} +
+ )} + {/* Solution sections */}
{/* ========== ANTHROPIC SOLUTIONS ========== */} diff --git a/src/lib/anthropic/analysis.ts b/src/lib/anthropic/analysis.ts index 565171c..9fc7386 100644 --- a/src/lib/anthropic/analysis.ts +++ b/src/lib/anthropic/analysis.ts @@ -3,6 +3,7 @@ import type { AggregatedRow, Finding, + FindingSignal, TemporalPattern, Workspace, UsageBucket, @@ -13,6 +14,18 @@ import { Severity, } from "@/types/analysis"; import { pr, tc } from "./pricing"; +import { + cacheWriteEconomics, + costBatchDiscount, + costEnableCaching, + costGenerationUpgrade, + costHaikuDowngrade, + costRagReduction, + costSonnetDowngrade, + ragReductionFactor, + upgradeTarget, + type CostRow, +} from "./costing"; import { $, P } from "@/lib/formatters"; /* ═══════════════════ AGGREGATION ═══════════════════ */ @@ -198,6 +211,17 @@ export function findIssues( const temporal = analyzeTemporalPattern(rawBuckets || [], r.kid, r.model); const inputVariance = ai > 5000 ? "high" : ai > 1000 ? "medium" : "low"; + // Shared input to the costing module — all optimized costs derive from it. + const costRow: CostRow = { + model: r.model, + inp: r.inp, + out: r.out, + cached: r.cached, + cacheCreated: r.cacheCreated, + cur, + conf: 0, + }; + // Initialize set for this API key if not exists const keyId = r.kid || r.model; if (!addedCategoriesByKey[keyId]) { @@ -211,7 +235,8 @@ export function findIssues( reason: string, action: string, severity: Finding["sev"], - confidence: number + confidence: number, + signals?: FindingSignal[] ) => { // Skip if this category was already added for this API key if (addedCategoriesByKey[keyId].has(category)) { @@ -248,23 +273,28 @@ export function findIssues( impact, activeDays: r.activeDays, temporal, + source: "rules", + signals, }); } }; /* ─── RULE 1: Model Downgrade → Haiku ─── */ if ((isO || isS) && ao > 0 && ao < 150 && r.reqs > 50) { - const signals = [ - { weight: 0.3, met: ao < 80 }, - { weight: 0.25, met: r.reqs > 500 }, - { weight: 0.2, met: inputVariance !== "high" }, - { weight: 0.15, met: ai < 2000 }, - { weight: 0.1, met: r.activeDays > 20 }, + const signals: FindingSignal[] = [ + { weight: 0.3, met: ao < 80, label: "avg output < 80 tok" }, + { weight: 0.25, met: r.reqs > 500, label: "500+ requests" }, + { + weight: 0.2, + met: inputVariance !== "high", + label: "input not oversized", + }, + { weight: 0.15, met: ai < 2000, label: "avg input < 2k tok" }, + { weight: 0.1, met: r.activeDays > 20, label: "20+ active days" }, ]; const conf = confidenceScore(signals); if (conf >= 0.4) { - const h = pr("haiku-4-5"); - const opt = (r.inp / 1e6) * h.i + (r.out / 1e6) * h.o; + const opt = costHaikuDowngrade(costRow); const reason = `Avg output ${ao} tokens across ${r.reqs.toLocaleString()} reqs (avg input ${ai.toLocaleString()} tok). Pattern consistent with classification, routing, or extraction.`; const action = `Switch to claude-haiku-4-5. Run a 100-request A/B test first — if accuracy delta <2%, ship it.`; const sev = conf >= 0.65 ? Severity.CRITICAL : Severity.WARNING; @@ -274,27 +304,25 @@ export function findIssues( reason, action, sev, - conf + conf, + signals ); } } /* ─── RULE 2: RAG Context Bloat ─── */ if (ratio > 12 && !isH && r.inp > 10e6) { - const signals = [ - { weight: 0.3, met: ratio > 20 }, - { weight: 0.25, met: ai > 8000 }, - { weight: 0.2, met: ao < 500 }, - { weight: 0.15, met: r.reqs > 100 }, - { weight: 0.1, met: cr < 0.1 }, + const signals: FindingSignal[] = [ + { weight: 0.3, met: ratio > 20, label: "in:out ratio > 20:1" }, + { weight: 0.25, met: ai > 8000, label: "avg input > 8k tok" }, + { weight: 0.2, met: ao < 500, label: "avg output < 500 tok" }, + { weight: 0.15, met: r.reqs > 100, label: "100+ requests" }, + { weight: 0.1, met: cr < 0.1, label: "cache rate < 10%" }, ]; const conf = confidenceScore(signals); if (conf >= 0.4) { - const reductionFactor = conf >= 0.7 ? 0.5 : 0.6; - const targetP = isO ? pr("sonnet-4-6") : p; - const opt = - ((r.inp * reductionFactor) / 1e6) * targetP.i + - (r.out / 1e6) * targetP.o; + const reductionFactor = ragReductionFactor(conf); + const opt = costRagReduction(costRow, conf); const reason = `Input:output ratio ${ratio.toFixed(0)}:1 (~${ai.toLocaleString()} tok/req input, ~${ao} output). ${(r.inp / 1e6).toFixed(1)}M input tokens/mo. Retrieval pulling too many chunks.`; const action = `Audit retrieval pipeline: reduce top-k, add reranking, tighten chunk size. ${isO ? "Downgrade to Sonnet — RAG quality is retrieval-bound, not model-bound." : ""} Conservative: ${Math.round((1 - reductionFactor) * 100)}% input reduction.`; const sev = conf >= 0.65 ? Severity.CRITICAL : Severity.WARNING; @@ -304,26 +332,23 @@ export function findIssues( reason, action, sev, - conf + conf, + signals ); } } /* ─── RULE 3: Prompt Caching Miss ─── */ if (cr < 0.05 && r.inp > 20e6 && !isH) { - const signals = [ - { weight: 0.35, met: cr < 0.01 }, - { weight: 0.25, met: r.reqs > 200 }, - { weight: 0.2, met: ai > 3000 }, - { weight: 0.2, met: r.activeDays > 15 }, + const signals: FindingSignal[] = [ + { weight: 0.35, met: cr < 0.01, label: "≈0 cache reads" }, + { weight: 0.25, met: r.reqs > 200, label: "200+ requests" }, + { weight: 0.2, met: ai > 3000, label: "avg input > 3k tok" }, + { weight: 0.2, met: r.activeDays > 15, label: "15+ active days" }, ]; const conf = confidenceScore(signals); if (conf >= 0.4) { - const cacheable = r.inp * 0.6; - const opt = - ((r.inp - cacheable) / 1e6) * p.i + - (cacheable / 1e6) * p.i * 0.1 + - (r.out / 1e6) * p.o; + const opt = costEnableCaching(costRow); const reason = `${(r.inp / 1e6).toFixed(1)}M input at ${(cr * 100).toFixed(1)}% cache rate. ${cr < 0.01 ? "Caching appears disabled." : "Minimal caching."} System prompts re-sent every request without caching.`; const action = `Enable prompt caching on static prefixes (system prompt, tool defs, persistent context). Add cache_control breakpoints in message array. ~90% savings on cached portion.`; const sev = conf >= 0.65 ? Severity.CRITICAL : Severity.WARNING; @@ -333,24 +358,24 @@ export function findIssues( reason, action, sev, - conf + conf, + signals ); } } /* ─── RULE 4: Opus Overkill → Sonnet ─── */ if (isO && ao >= 150 && r.inp > 5e6) { - const signals = [ - { weight: 0.3, met: ao < 1500 }, - { weight: 0.25, met: r.reqs > 100 }, - { weight: 0.2, met: ratio < 10 }, - { weight: 0.15, met: ai < 10000 }, - { weight: 0.1, met: r.activeDays > 15 }, + const signals: FindingSignal[] = [ + { weight: 0.3, met: ao < 1500, label: "avg output < 1.5k tok" }, + { weight: 0.25, met: r.reqs > 100, label: "100+ requests" }, + { weight: 0.2, met: ratio < 10, label: "in:out ratio < 10:1" }, + { weight: 0.15, met: ai < 10000, label: "avg input < 10k tok" }, + { weight: 0.1, met: r.activeDays > 15, label: "15+ active days" }, ]; const conf = confidenceScore(signals); if (conf >= 0.4) { - const s = pr("sonnet-4-6"); - const opt = (r.inp / 1e6) * s.i + (r.out / 1e6) * s.o; + const opt = costSonnetDowngrade(costRow); const reason = `${p.l} with avg ${ao} tok output, ${r.reqs.toLocaleString()} reqs. Moderate complexity where Sonnet performs within 5% of Opus.`; const action = `A/B test Sonnet 4.6 on 10% traffic split. If quality holds, migrate fully. Opus→Sonnet saves ~80%.`; const sev = conf >= 0.65 ? Severity.WARNING : Severity.INFO; @@ -360,22 +385,27 @@ export function findIssues( reason, action, sev, - conf + conf, + signals ); } } /* ─── RULE 5: Batch API Candidate ─── */ if (temporal.batchCandidate && r.reqs > 200 && cur > 5) { - const signals = [ - { weight: 0.35, met: temporal.burstiness > 1.5 }, - { weight: 0.25, met: r.reqs > 500 }, - { weight: 0.2, met: !isH }, - { weight: 0.2, met: r.activeDays < 25 }, + const signals: FindingSignal[] = [ + { + weight: 0.35, + met: temporal.burstiness > 1.5, + label: "high burstiness (CoV > 1.5)", + }, + { weight: 0.25, met: r.reqs > 500, label: "500+ requests" }, + { weight: 0.2, met: !isH, label: "premium-tier model" }, + { weight: 0.2, met: r.activeDays < 25, label: "< 25 active days" }, ]; const conf = confidenceScore(signals); if (conf >= 0.4) { - const opt = cur * 0.5; + const opt = costBatchDiscount(costRow); const reason = `Bursty traffic (CoV: ${temporal.burstiness.toFixed(1)}, ~${Math.round(temporal.meanDaily)} reqs/day avg). ${r.reqs.toLocaleString()} total reqs with periodic spikes — batch processing or eval runs.`; const action = `Migrate to Batch API for 50% cost reduction. Processes within 24hrs. If not latency-sensitive, this is free money.`; const sev = conf >= 0.65 ? Severity.WARNING : Severity.INFO; @@ -385,7 +415,8 @@ export function findIssues( reason, action, sev, - conf + conf, + signals ); } } @@ -394,14 +425,11 @@ export function findIssues( // Cache writes cost 25% MORE than regular input; reads cost 90% LESS. // Break-even: each written token only needs 0.28 reads to pay for itself. // Fire when writes are large but reads are sparse → cache invalidating before reuse. - if (r.cacheCreated > 2e6 && r.cached / r.cacheCreated < 1.0) { - const reuseFactor = r.cacheCreated > 0 ? r.cached / r.cacheCreated : 0; - const writeExtra = (r.cacheCreated / 1e6) * p.i * 0.25; - const readSaving = (r.cached / 1e6) * p.i * 0.9; - const netCacheCost = writeExtra - readSaving; + const cacheEcon = cacheWriteEconomics(costRow); + if (cacheEcon && r.cacheCreated > 2e6 && cacheEcon.reuseFactor < 1.0) { + const { reuseFactor, netCacheCost, opt } = cacheEcon; if (netCacheCost > 1) { const conf = Math.min(0.9, 0.5 + (1.0 - reuseFactor) * 0.5); - const opt = cur - netCacheCost * 0.6; const reason = `Wrote ${(r.cacheCreated / 1e6).toFixed(1)}M cache tokens but only read back ${(r.cached / 1e6).toFixed(1)}M (${(reuseFactor * 100).toFixed(0)}% reuse). Cache paying ${$(netCacheCost)}/mo more than it saves — invalidating before adequate reuse.`; const action = `Extend cache TTL (up to 5 min default, up to 60 min with extended cache). Ensure system prompt structure is stable between requests. Avoid inserting dynamic content before the cache breakpoint.`; addFinding( @@ -417,17 +445,10 @@ export function findIssues( /* ─── RULE 6: Legacy Model ─── */ if (p.g > 0 && p.g < 3 && cur > 2) { - const newer = - p.t === AnthropicModelTier.OPUS - ? pr("opus-4-6") - : p.t === AnthropicModelTier.SONNET - ? pr("sonnet-4-6") - : pr("haiku-4-5"); - const newerCost = (r.inp / 1e6) * newer.i + (r.out / 1e6) * newer.o; - const savOrCost = cur - newerCost; + const newer = upgradeTarget(r.model); + const opt = costGenerationUpgrade(costRow); const conf = 0.8; - const opt = Math.min(cur, newerCost); - const reason = `Running ${p.l} (gen ${p.g}). ${newer.l} offers better performance${savOrCost > 0 ? " at lower cost" : ""}.`; + const reason = `Running ${p.l} (gen ${p.g}). ${newer.l} offers better performance${opt < cur ? " at lower cost" : ""}.`; const action = `Update model string to ${newer.l.toLowerCase().replace(/ /g, "-")}. Drop-in replacement — test on staging, then ship.`; const sev = Severity.INFO; addFinding( @@ -547,6 +568,7 @@ After setup, you'll be able to see: "Production: $450/mo, Staging: $120/mo, Dev: batchCandidate: false, meanDaily: 0, }, + source: "rules", }); } @@ -623,6 +645,7 @@ After routing, return to this view next month — you should see spend distribut batchCandidate: false, meanDaily: 0, }, + source: "rules", }); } diff --git a/src/lib/anthropic/costing.ts b/src/lib/anthropic/costing.ts new file mode 100644 index 0000000..2fdccfb --- /dev/null +++ b/src/lib/anthropic/costing.ts @@ -0,0 +1,151 @@ +/* ═══════════════════ ANTHROPIC COSTING MODULE ═══════════════════ */ +/* + * Deterministic optimized-cost formulas for every finding category. The rule + * engine (findIssues) and the LLM consensus path both price findings through + * these functions, so a dollar figure never comes from an LLM — only from the + * row's actual token volumes and the pricing table. + */ + +import { AnthropicCategory, AnthropicModelTier } from "@/types/analysis"; +import { pr } from "./pricing"; + +/** Row metrics every costing formula works from. */ +export interface CostRow { + model: string; + inp: number; // input tokens / mo + out: number; // output tokens / mo + cached: number; // cache read tokens / mo + cacheCreated: number; // cache write tokens / mo + cur: number; // current monthly cost (USD) + conf: number; // confidence 0-1, drives conservative reduction factors +} + +/** Rule 1: reprice the row's tokens at Haiku rates. */ +export function costHaikuDowngrade(row: CostRow): number { + const h = pr("haiku-4-5"); + return (row.inp / 1e6) * h.i + (row.out / 1e6) * h.o; +} + +/** Rule 4: reprice the row's tokens at Sonnet rates. */ +export function costSonnetDowngrade(row: CostRow): number { + const s = pr("sonnet-4-6"); + return (row.inp / 1e6) * s.i + (row.out / 1e6) * s.o; +} + +/** + * Rule 2: input reduction from tighter retrieval. High confidence justifies a + * 50% cut, otherwise a conservative 40%. Opus rows are also repriced at Sonnet + * (RAG quality is retrieval-bound, not model-bound). + */ +export function costRagReduction(row: CostRow, conf: number): number { + const p = pr(row.model); + const reductionFactor = conf >= 0.7 ? 0.5 : 0.6; + const targetP = p.t === AnthropicModelTier.OPUS ? pr("sonnet-4-6") : p; + return ( + ((row.inp * reductionFactor) / 1e6) * targetP.i + + (row.out / 1e6) * targetP.o + ); +} + +/** Rule 2's reduction factor, exposed so reason/action text can cite it. */ +export function ragReductionFactor(conf: number): number { + return conf >= 0.7 ? 0.5 : 0.6; +} + +/** Rule 3: 60% of input assumed cacheable at 90% off after enabling caching. */ +export function costEnableCaching(row: CostRow): number { + const p = pr(row.model); + const cacheable = row.inp * 0.6; + return ( + ((row.inp - cacheable) / 1e6) * p.i + + (cacheable / 1e6) * p.i * 0.1 + + (row.out / 1e6) * p.o + ); +} + +/** + * Rule 5b: cache write break-even economics. Writes cost 25% more than plain + * input; reads 90% less. Returns null when the row has no cache writes. + */ +export function cacheWriteEconomics(row: CostRow): { + reuseFactor: number; + writeExtra: number; + readSaving: number; + netCacheCost: number; + opt: number; +} | null { + if (row.cacheCreated <= 0) return null; + const p = pr(row.model); + const reuseFactor = row.cached / row.cacheCreated; + const writeExtra = (row.cacheCreated / 1e6) * p.i * 0.25; + const readSaving = (row.cached / 1e6) * p.i * 0.9; + const netCacheCost = writeExtra - readSaving; + return { + reuseFactor, + writeExtra, + readSaving, + netCacheCost, + opt: row.cur - netCacheCost * 0.6, + }; +} + +/** Rules 5/4b: Batch API's 50% discount on the whole workload. */ +export function costBatchDiscount(row: CostRow): number { + return row.cur * 0.5; +} + +/** + * Rule 6: reprice at the newest same-tier model; never above current cost + * (an upgrade is recommended for quality even when it isn't cheaper). + */ +export function costGenerationUpgrade(row: CostRow): number { + const p = pr(row.model); + const newer = + p.t === AnthropicModelTier.OPUS + ? pr("opus-4-6") + : p.t === AnthropicModelTier.SONNET + ? pr("sonnet-4-6") + : pr("haiku-4-5"); + const newerCost = (row.inp / 1e6) * newer.i + (row.out / 1e6) * newer.o; + return Math.min(row.cur, newerCost); +} + +/** The newest same-tier model's pricing entry, for reason/action text. */ +export function upgradeTarget(model: string) { + const p = pr(model); + return p.t === AnthropicModelTier.OPUS + ? pr("opus-4-6") + : p.t === AnthropicModelTier.SONNET + ? pr("sonnet-4-6") + : pr("haiku-4-5"); +} + +/** + * Category dispatcher for the consensus path: given an LLM-proposed category + * and the row's real metrics, return the deterministic optimized monthly cost, + * or null when that category can't be costed for that row. + */ +export function optimizedCostAnthropic( + cat: AnthropicCategory, + row: CostRow +): number | null { + switch (cat) { + case AnthropicCategory.MODEL_DOWNGRADE_HAIKU: + return row.inp + row.out > 0 ? costHaikuDowngrade(row) : null; + case AnthropicCategory.MODEL_DOWNGRADE_SONNET: + return row.inp + row.out > 0 ? costSonnetDowngrade(row) : null; + case AnthropicCategory.RAG_OPTIMIZATION: + return row.inp > 0 ? costRagReduction(row, row.conf) : null; + case AnthropicCategory.PROMPT_CACHING: + return row.inp > 0 ? costEnableCaching(row) : null; + case AnthropicCategory.BATCH_API_MIGRATION: + return row.cur > 0 ? costBatchDiscount(row) : null; + case AnthropicCategory.MODEL_UPGRADE: + return row.inp + row.out > 0 ? costGenerationUpgrade(row) : null; + default: + // No deterministic formula (e.g. Prompt Optimization, Workspace + // Organization) — the caller decides whether a zero-savings finding + // survives. + return null; + } +} diff --git a/src/lib/demo.ts b/src/lib/demo.ts index 914452c..71825ab 100644 --- a/src/lib/demo.ts +++ b/src/lib/demo.ts @@ -66,41 +66,180 @@ function newProfile(seed: number): BusinessProfile { } // ─── Anthropic ─────────────────────────────────────────────────────────────── +// +// 8 workspaces plus an overloaded default. Each workspace runs a distinct, +// realistic workload pattern so most rule categories fire in a demo run: +// caching miss, Haiku downgrade, RAG bloat, batch candidate, Opus→Sonnet, +// legacy model, cache-write waste, plus a quiet staging workspace. Traffic +// with no workspace_id lands in the (biggest-spending) default workspace, +// which also triggers the org-structure finding. const ANTH_WORKSPACES: Workspace[] = [ + { id: "ws_prod", name: "Production API", display_name: "Production API" }, { - id: "ws_prod", - name: "Production", - display_name: "Production", - created_at: "2024-01-01T00:00:00Z", + id: "ws_support", + name: "Support Chatbot", + display_name: "Support Chatbot", }, { - id: "ws_dev", - name: "Development", - display_name: "Development", - created_at: "2024-03-01T00:00:00Z", + id: "ws_rag", + name: "Knowledge Base RAG", + display_name: "Knowledge Base RAG", }, -]; - -const ANTH_API_KEYS = ["key_aaa111bbb222", "key_ccc333ddd444"]; + { id: "ws_evals", name: "Nightly Evals", display_name: "Nightly Evals" }, + { + id: "ws_research", + name: "Research Sandbox", + display_name: "Research Sandbox", + }, + { + id: "ws_legacy", + name: "Legacy Summarizer", + display_name: "Legacy Summarizer", + }, + { id: "ws_agents", name: "Agent Platform", display_name: "Agent Platform" }, + { id: "ws_staging", name: "Staging", display_name: "Staging" }, +].map((w, i) => ({ + ...w, + created_at: `2024-0${(i % 8) + 1}-01T00:00:00Z`, +})); + +// One entry per (workspace, api key, model) workload. Daily base volumes are +// sized so every pattern clears its rule thresholds even at the low end of +// the profile's scale range. wid undefined → default workspace. +interface AnthWorkload { + wid?: string; + key: string; + model: string; + inp: number; // daily base input tokens + out: number; // daily base output tokens + reqs: number; // daily base requests + cacheRate: number; // share of input served from cache + cacheWrite?: number; // daily base cache-creation tokens + cacheReads?: number; // daily base cache-read tokens (overrides cacheRate) + mondayBoost?: boolean; // bursty Monday spikes → batch candidate +} -const ANTH_MODELS = [ - { name: "claude-3-5-sonnet-20241022", inp: 8000, out: 2000 }, - { name: "claude-3-opus-20240229", inp: 12000, out: 3000 }, - { name: "claude-3-haiku-20240307", inp: 3000, out: 800 }, - { name: "claude-sonnet-4-6-20250514", inp: 7000, out: 1800 }, - { name: "claude-opus-4-6-20250514", inp: 11000, out: 2800 }, - { name: "claude-haiku-4-5-20250514", inp: 2500, out: 600 }, +const ANTH_WORKLOADS: AnthWorkload[] = [ + // Overloaded default workspace: the org's main app never got segmented. + { + key: "key_default_app", + model: "claude-opus-4-6-20250514", + inp: 2_400_000, + out: 250_000, + reqs: 700, + cacheRate: 0.15, + }, + { + key: "key_default_app", + model: "claude-sonnet-4-6-20250514", + inp: 800_000, + out: 200_000, + reqs: 600, + cacheRate: 0.25, + }, + { + key: "key_default_misc", + model: "claude-haiku-4-5-20250514", + inp: 400_000, + out: 100_000, + reqs: 300, + cacheRate: 0.1, + }, + // Production: heavy volume, caching never enabled → prompt caching miss. + { + wid: "ws_prod", + key: "key_prod", + model: "claude-sonnet-4-6-20250514", + inp: 3_500_000, + out: 450_000, + reqs: 700, + cacheRate: 0.004, + }, + // Support chatbot: Opus emitting tiny outputs → Haiku downgrade. + { + wid: "ws_support", + key: "key_support", + model: "claude-opus-4-6-20250514", + inp: 300_000, + out: 15_000, + reqs: 250, + cacheRate: 0.03, + }, + // RAG service: enormous retrieval context per request → RAG bloat. + { + wid: "ws_rag", + key: "key_rag", + model: "claude-sonnet-4-6-20250514", + inp: 1_600_000, + out: 25_000, + reqs: 60, + cacheRate: 0.2, + }, + // Nightly evals: Monday spikes, quiet otherwise → batch API candidate. + { + wid: "ws_evals", + key: "key_evals", + model: "claude-sonnet-4-6-20250514", + inp: 500_000, + out: 120_000, + reqs: 300, + cacheRate: 0.1, + mondayBoost: true, + }, + // Research: Opus on moderate-complexity work → Opus→Sonnet downgrade. + { + wid: "ws_research", + key: "key_research", + model: "claude-opus-4-6-20250514", + inp: 700_000, + out: 70_000, + reqs: 100, + cacheRate: 0.1, + }, + // Legacy summarizer: still on Claude 3 Opus → legacy model upgrade. + { + wid: "ws_legacy", + key: "key_legacy", + model: "claude-3-opus-20240229", + inp: 150_000, + out: 15_000, + reqs: 80, + cacheRate: 0, + }, + // Agent platform: writes big cache prefixes it rarely reads back. + { + wid: "ws_agents", + key: "key_agents", + model: "claude-sonnet-4-6-20250514", + inp: 300_000, + out: 40_000, + reqs: 120, + cacheRate: 0, + cacheWrite: 400_000, + cacheReads: 20_000, + }, + // Staging: light, well-behaved traffic — no findings expected. + { + wid: "ws_staging", + key: "key_staging", + model: "claude-haiku-4-5-20250514", + inp: 200_000, + out: 50_000, + reqs: 100, + cacheRate: 0.15, + }, ]; interface AnthEntry { bucket_start: string; model: string; api_key_id: string; - workspace_id: string; + workspace_id?: string; input_tokens: number; output_tokens: number; cache_read_input_tokens: number; + cache_creation_input_tokens: number; request_count: number; } @@ -120,59 +259,33 @@ function genAnthropicEntries( const isWeekend = date.getDay() === 0 || date.getDay() === 6; const isMonday = date.getDay() === 1; const ds = `${year}-${String(month + 1).padStart(2, "0")}-${String(day).padStart(2, "0")}T00:00:00Z`; - const scenario = rand(); - for (const m of ANTH_MODELS) { - const ws = - ANTH_WORKSPACES[Math.floor(rand() * ANTH_WORKSPACES.length)].id; - const key = ANTH_API_KEYS[Math.floor(rand() * ANTH_API_KEYS.length)]; + for (const w of ANTH_WORKLOADS) { const wm = isWeekend ? profile.weekendFactor : 1; const v = 0.5 + rand() * profile.volatility; - - let inp: number, out: number, cache: number, reqs: number; - - if (scenario < 0.15) { - // Low output ratio → model downgrade candidate - inp = Math.floor(2000 * wm * v * scale); - out = Math.floor((50 + rand() * 100) * wm * v * scale); - reqs = Math.floor(80 * wm * v * scale); - cache = Math.floor(inp * (0.02 + rand() * 0.03)); - } else if (scenario < 0.3) { - // High input:output → RAG context bloat candidate - inp = Math.floor((15000 + rand() * 10000) * wm * v * scale); - out = Math.floor(400 * wm * v * scale); - reqs = Math.floor(30 * wm * v * scale); - cache = Math.floor(inp * (0.15 + rand() * 0.25)); - } else if (scenario < 0.45) { - // High volume, low cache → prompt caching miss candidate - inp = Math.floor(25000 * wm * v * scale); - out = Math.floor(2500 * wm * v * scale); - reqs = Math.floor(150 * wm * v * scale); - cache = Math.floor(inp * (0.01 + rand() * 0.03)); - } else if (scenario < 0.55) { - // Bursty Monday pattern → batch API candidate - inp = Math.floor(8000 * wm * v * scale); - out = Math.floor(2000 * wm * v * scale); - reqs = Math.floor(200 * wm * (isMonday ? 1.5 : 0.7) * scale); - cache = Math.floor(inp * (0.1 + rand() * 0.3) * profile.cacheAffinity); - } else { - // Baseline usage - inp = Math.floor(m.inp * wm * v * scale); - out = Math.floor(m.out * wm * v * scale); - reqs = Math.floor(80 * wm * v * scale); - cache = Math.floor(inp * (0.1 + rand() * 0.3) * profile.cacheAffinity); - } + let mult = wm * v * scale; + if (w.mondayBoost) mult *= isMonday ? 8 : 0.3; + + const inp = Math.floor(w.inp * mult); + const out = Math.floor(w.out * mult); + const reqs = Math.floor(w.reqs * mult); + const cache = + w.cacheReads !== undefined + ? Math.floor(w.cacheReads * mult) + : Math.floor(inp * w.cacheRate * (0.8 + rand() * 0.4)); + const cacheCreated = w.cacheWrite ? Math.floor(w.cacheWrite * mult) : 0; if (inp === 0 && out === 0) continue; entries.push({ bucket_start: ds, - model: m.name, - api_key_id: key, - workspace_id: ws, + model: w.model, + api_key_id: w.key, + workspace_id: w.wid, input_tokens: inp, output_tokens: out, cache_read_input_tokens: Math.min(cache, Math.floor(inp * 0.9)), + cache_creation_input_tokens: cacheCreated, request_count: Math.max(1, reqs), }); } @@ -198,6 +311,7 @@ export function demoAnthropic( input_tokens: e.input_tokens, output_tokens: e.output_tokens, cache_read_input_tokens: e.cache_read_input_tokens, + cache_creation_input_tokens: e.cache_creation_input_tokens, request_count: e.request_count, })); @@ -208,6 +322,7 @@ export function demoAnthropic( input_tokens: e.input_tokens, output_tokens: e.output_tokens, cache_read_input_tokens: e.cache_read_input_tokens, + cache_creation_input_tokens: e.cache_creation_input_tokens, request_count: e.request_count, })); @@ -218,6 +333,7 @@ export function demoAnthropic( input_tokens: e.input_tokens, output_tokens: e.output_tokens, cache_read_input_tokens: e.cache_read_input_tokens, + cache_creation_input_tokens: e.cache_creation_input_tokens, request_count: e.request_count, })); diff --git a/src/lib/nim/adapters.ts b/src/lib/nim/adapters.ts index 46175b3..5d1c9f5 100644 --- a/src/lib/nim/adapters.ts +++ b/src/lib/nim/adapters.ts @@ -50,7 +50,9 @@ export function toSummariesOpenAI( const p = prOpenAI(r.model || r.line_item || ""); const cur = r.cost > 0 ? r.cost : tcOpenAI(r.model, r.inp, r.out); return { - id: `${r.project_id || "default"}-${r.model || r.line_item || "?"}`, + // Mirrors findIssuesOpenAI's finding-id prefix so consensus merging can + // match rule and LLM findings on the same row. + id: `${r.model || r.line_item || "unknown"}-${r.project_id || "x"}`, name: r.model || r.line_item || "?", ws: r.project_id ? pn[r.project_id] || r.project_id : "Default project", model: r.model || r.line_item || "?", diff --git a/src/lib/nim/analysis.ts b/src/lib/nim/analysis.ts index f0f2a40..99727a3 100644 --- a/src/lib/nim/analysis.ts +++ b/src/lib/nim/analysis.ts @@ -1,12 +1,13 @@ /* ═══════════════════ NVIDIA NIM — LLM-GUIDED ANALYSIS ═══════════════════ */ /* - * Drop-in alternative to the hardcoded rule engines (findIssues / - * findIssuesOpenAI). Instead of fixed thresholds, we hand a structured usage - * summary to an LLM hosted on NVIDIA NIM and let it reason about which - * optimizations apply. The rules below are *guidance* for the model, not code. + * AI augmentation layer over the deterministic rule engines (findIssues / + * findIssuesOpenAI). The LLM's job is detection and explanation ONLY: it + * proposes {row, category, severity, confidence, reason, action}. Every + * dollar figure is computed here from the row's real metrics via the vendor + * costing modules — the model never emits a savings number. * - * The deterministic metrics (cost, savings clamping, ids) are still computed - * here so dollar figures stay grounded and the model can't hallucinate numbers. + * When the AI toggle is on, BOTH engines run and their findings merge with + * per-finding provenance (source: "rules" | "llm" | "both"). */ import type { @@ -15,7 +16,9 @@ import type { LlmUsage, TemporalPattern, } from "@/types"; -import { Severity } from "@/types/analysis"; +import { AnthropicCategory, OpenAICategory, Severity } from "@/types/analysis"; +import { optimizedCostAnthropic } from "@/lib/anthropic/costing"; +import { optimizedCostOpenAI } from "@/lib/openai/costing"; /** Default NIM-hosted model. OpenAI-compatible chat completions. */ export const NIM_DEFAULT_MODEL = "meta/llama-3.3-70b-instruct"; @@ -69,18 +72,53 @@ export const ANALYSIS_RULES = ` Action: split by environment/team/product for cost attribution. (Quality/visibility win, savings = 0.) -Only emit a finding when the data actually supports it. Be conservative with -savings — prefer underestimating. Skip rows costing under ~$0.50/mo. +Only emit a finding when the data actually supports it. Skip rows costing under +~$0.50/mo. `.trim(); +/** + * Category values the LLM may emit per vendor. Only categories the costing + * module can price (plus the zero-savings org/quality ones) are offered. + */ +export function validCategories(vendor: "anthropic" | "openai"): string[] { + if (vendor === "openai") { + return [ + OpenAICategory.MODEL_DOWNGRADE_MINI, + OpenAICategory.MODEL_DOWNGRADE_4O, + OpenAICategory.RAG_OPTIMIZATION, + OpenAICategory.PROMPT_CACHING, + OpenAICategory.PROMPT_OPTIMIZATION, + OpenAICategory.BATCH_API_MIGRATION, + OpenAICategory.MODEL_UPGRADE, + OpenAICategory.REASONING_MODEL_OVERKILL, + OpenAICategory.HIGH_IMPACT_OPPORTUNITY, + OpenAICategory.PROJECT_ORGANIZATION, + ]; + } + return [ + AnthropicCategory.MODEL_DOWNGRADE_HAIKU, + AnthropicCategory.MODEL_DOWNGRADE_SONNET, + AnthropicCategory.RAG_OPTIMIZATION, + AnthropicCategory.PROMPT_CACHING, + AnthropicCategory.BATCH_API_MIGRATION, + AnthropicCategory.MODEL_UPGRADE, + AnthropicCategory.WORKSPACE_ORGANIZATION, + ]; +} + export function buildSystemPrompt(vendor: "anthropic" | "openai"): string { const small = vendor === "openai" ? "gpt-4o-mini" : "claude-haiku-4-5"; const mid = vendor === "openai" ? "gpt-4o" : "claude-sonnet-4-6"; + const categories = validCategories(vendor) + .map((c) => `"${c}"`) + .join(", "); return `You are TokenPilot's LLM cost-optimization analyst for ${vendor === "openai" ? "OpenAI" : "Anthropic"} API usage. You receive a JSON array of per-row usage summaries (one row = a model used by an API key inside a workspace/project) plus org-level context. Apply the rules below -and return concrete, actionable cost-savings findings. +and propose concrete, actionable findings. You DETECT and EXPLAIN only — all +dollar amounts are computed separately from the row's real usage data, so do not +mention specific dollar savings. ANALYSIS RULES: ${ANALYSIS_RULES} @@ -92,27 +130,29 @@ Respond with ONLY a JSON object, no prose, of the form: "findings": [ { "rowId": "", - "category": "short label, e.g. 'Model Downgrade → Haiku', 'RAG Optimization', 'Prompt Caching', 'Batch API Migration', 'Model Upgrade', 'Workspace Organization'", + "category": "one of the exact values listed below", "severity": "critical | warning | info", "confidence": 0.0, - "savingsMonthly": 0.0, "reason": "1-3 sentences. Cite ONLY the actual values from THIS row.", "action": "concrete next step the engineer can take." } ] } +VALID CATEGORY VALUES (use exactly one of these strings, verbatim): +${categories} + GROUNDING (critical): In "reason", quote only the real numbers from the row you are analyzing — its monthlyCostUsd, inputTokens, cacheReadRate, avgOutputPerReq, requests, and its own model id. NEVER repeat the threshold numbers written in the rules above (e.g. ">20M tok/mo", "<5%", ">5000 tok/req") — those are triggers, not this row's data. NEVER mention another row's model or numbers. If a row's real numbers don't clear a -rule's threshold, do not emit that finding for it. +rule's threshold, do not emit that finding for it. Do NOT state dollar amounts — +savings are priced deterministically from the row's data after you respond. -Severity guide: critical = >$100/mo or >20% of this row's cost; warning = meaningful -savings; info = small or quality-only. savingsMonthly is the estimated monthly USD -saved (0 for quality-only or org-structure findings) and must not exceed the row's -current monthly cost. confidence is 0-1. Emit at most one finding per category per row.`; +Severity guide: critical = large or urgent waste on this row; warning = meaningful +optimization; info = small or quality-only. confidence is 0-1. Emit at most one +finding per category per row.`; } /* ─────────────── INPUT SUMMARY (vendor-agnostic) ─────────────── */ @@ -147,12 +187,12 @@ const EMPTY_TEMPORAL: TemporalPattern = { meanDaily: 0, }; -interface LlmFinding { +/** What the LLM returns per proposal — note: no savings field of any kind. */ +export interface LlmProposal { rowId: string; category: string; severity: string; confidence: number; - savingsMonthly: number; reason: string; action: string; } @@ -205,6 +245,12 @@ function toSeverity(s: string): Severity { } } +const slug = (c: string) => c.replace(/[^a-z0-9]/gi, "-").toLowerCase(); + +// Looser form for matching LLM-emitted category strings: consecutive +// separators collapse so "→" and "->" normalize identically. +const canon = (c: string) => slug(c).replace(/-+/g, "-"); + // Cost tier rank from a model id/label — lower = cheaper. Heuristic, but enough // to catch the LLM mislabelling an upgrade as a "downgrade". ponytail: extend // the regexes if a new tier shows up. @@ -224,67 +270,123 @@ function downgradeTargetRank(category: string): number | null { return tierRank(target); } +// Categories where a zero-savings finding is still worth surfacing (quality +// or organizational wins), matching the rule engines' behavior. const KEEP_ZERO_SAVINGS = /upgrade|organization|workspace|project|quality/i; interface Candidate { finding: Finding; rowId: string; - cur: number; sav: number; conf: number; isDowngrade: boolean; } /** - * Merge the LLM's reasoning back onto deterministic per-row metrics, then apply - * guardrails so a weak model can't produce contradictory or impossible advice: - * 1. Per-finding savings clamped to [0, row cost]. - * 2. Drop "downgrades" whose target isn't actually cheaper than the row's model. - * 3. Keep only the single best downgrade per row (no Sonnet AND Haiku at once). - * 4. Drop zero-savings cost findings (noise); keep zero-savings quality/org ones. - * 5. Cap cumulative savings per row at the row's spend (no >100% savings). + * Resolve an LLM-emitted category string to the vendor's canonical enum value. + * Tolerates "->" vs "→" and case drift; returns null for anything not in the + * fixed category set. + */ +export function resolveCategory( + vendor: "anthropic" | "openai", + category: string +): AnthropicCategory | OpenAICategory | null { + const want = canon(category || ""); + for (const c of validCategories(vendor)) { + if (canon(c) === want) return c as AnthropicCategory | OpenAICategory; + } + return null; +} + +/** + * Price the LLM's proposals deterministically and turn them into Findings. + * The LLM contributes detection + explanation; the costing module contributes + * every number. Guardrails: + * 1. Categories outside the vendor's fixed enum are dropped. + * 2. "Downgrades" whose target isn't actually cheaper than the row's model + * are dropped (tier sanity). + * 3. At most one downgrade per row (no Sonnet AND Haiku at once). + * 4. Proposals whose category can't be costed for that row are dropped — + * except zero-savings org/quality categories, which keep savings 0. * Exported for testing. */ -export function mergeLlmFindings( - llm: LlmFinding[], +export function priceLlmFindings( + llm: LlmProposal[], rows: UsageSummary[], ctx: AnalysisContext ): Finding[] { const byId = new Map(rows.map((r) => [r.id, r])); const candidates: Candidate[] = []; + const seen = new Set(); for (const f of llm || []) { if (!f || !f.reason) continue; + const cat = resolveCategory(ctx.vendor, f.category); + if (!cat) continue; // guardrail 1: unknown category + + const dedupeKey = `${f.rowId}|${slug(cat)}`; + if (seen.has(dedupeKey)) continue; + const r = byId.get(f.rowId); const cur = r ? r.cur : ctx.totalSpend; - const sav = Math.max(0, Math.min(f.savingsMonthly || 0, cur)); const conf = Math.max(0, Math.min(f.confidence ?? 0.5, 1)); - const category = f.category || "Optimization"; // Guardrail 2: a "downgrade" to an equal/pricier tier isn't a saving. - const targetRank = downgradeTargetRank(category); + const targetRank = downgradeTargetRank(cat); const isDowngrade = targetRank !== null; if (r && isDowngrade && targetRank! >= tierRank(r.model || r.ml)) continue; - // Guardrail 4: drop zero-savings cost findings; keep quality/org ones. - if (sav <= 0 && !KEEP_ZERO_SAVINGS.test(category)) continue; + // Deterministic pricing — the only source of dollar figures. + const priced = r + ? ctx.vendor === "openai" + ? optimizedCostOpenAI(cat as OpenAICategory, { + model: r.model || r.ml || "", + inp: r.inp, + out: r.out, + cur, + conf, + }) + : optimizedCostAnthropic(cat as AnthropicCategory, { + model: r.model, + inp: r.inp, + out: r.out, + cached: r.cached, + cacheCreated: r.cacheCreated, + cur, + conf, + }) + : null; + + let sav: number; + let opt: number; + if (priced === null) { + // Guardrail 4: uncostable — only org/quality categories survive, at $0. + if (!KEEP_ZERO_SAVINGS.test(cat)) continue; + sav = 0; + opt = cur; + } else { + sav = Math.max(0, cur - priced); + opt = cur - sav; + // A cost finding the pricing says saves nothing is noise. + if (sav <= 0 && !KEEP_ZERO_SAVINGS.test(cat)) continue; + } + seen.add(dedupeKey); const ratio = r && r.out > 0 ? r.inp / r.out : 0; const cr = r && r.inp + r.cached > 0 ? r.cached / (r.inp + r.cached) : 0; - const slug = category.replace(/[^a-z0-9]/gi, "-").toLowerCase(); + const pct = cur > 0 ? Math.round((sav / cur) * 100) : 0; candidates.push({ rowId: f.rowId, - cur, sav, conf, isDowngrade, finding: { - id: `${f.rowId}-${slug}`, + id: `${f.rowId}-${slug(cat)}`, name: r ? r.name : "Organization", ws: r ? r.ws : "All workspaces", model: r ? r.model : "N/A", - ml: r ? r.ml : category, + ml: r ? r.ml : cat, inp: r?.inp ?? 0, out: r?.out ?? 0, cached: r?.cached ?? 0, @@ -294,16 +396,18 @@ export function mergeLlmFindings( ratio, cr, cur, - opt: cur, // filled after capping - sav, // filled after capping + opt, + sav, reason: f.reason, action: f.action || "", sev: toSeverity(f.severity), - cat: category as Finding["cat"], + cat, conf, - impact: "", + impact: + sav > 0 ? `$${sav.toFixed(2)}/mo (${pct}%)` : "Quality improvement", activeDays: r?.activeDays ?? 0, temporal: r?.temporal ?? EMPTY_TEMPORAL, + source: "llm", }, }); } @@ -321,41 +425,73 @@ export function mergeLlmFindings( (c) => !c.isDowngrade || bestDowngrade.get(c.rowId) === c ); - // Guardrail 5: cap cumulative savings per row at its spend (highest first). - const headroom = new Map(); - for (const c of [...kept].sort((a, b) => b.sav - a.sav)) { - const left = headroom.get(c.rowId) ?? c.cur; - const sav = Math.min(c.sav, Math.max(0, left)); - headroom.set(c.rowId, left - sav); - c.finding.sav = sav; - c.finding.opt = Math.max(0, c.cur - sav); - const pct = c.cur > 0 ? Math.round((sav / c.cur) * 100) : 0; - c.finding.impact = - sav > 0 ? `$${sav.toFixed(2)}/mo (${pct}%)` : "Quality improvement"; + return kept.map((c) => c.finding).sort(bySeverityThenSavings); +} + +/* ─────────────── CONSENSUS MERGE ─────────────── */ + +const sv: Record = { + [Severity.CRITICAL]: 0, + [Severity.WARNING]: 1, + [Severity.INFO]: 2, + [Severity.OK]: 3, +}; + +function bySeverityThenSavings(a: Finding, b: Finding): number { + return sv[a.sev] !== sv[b.sev] ? sv[a.sev] - sv[b.sev] : b.sav - a.sav; +} + +// A finding's merge key: the row it belongs to plus its category. Rule and LLM +// finding ids share the shape `${rowId}-${categorySlug}`, so stripping the +// category slug recovers the row id. Org-level findings (workspace/project +// organization) collapse to a shared "org" row so both engines' versions of +// the same structural insight merge into one. +export function consensusKey(f: Finding): string { + const catSlug = slug(f.cat as string); + if (/organization/i.test(f.cat as string)) return `org|${catSlug}`; + const suffix = `-${catSlug}`; + const rowId = f.id.endsWith(suffix) ? f.id.slice(0, -suffix.length) : f.id; + return `${rowId}|${catSlug}`; +} + +/** + * Merge both engines' findings by (row, category): + * - found by both → one finding, rule's text kept, source "both", + * confidence = min(0.95, max(ruleConf, llmConf) + 0.1) + * - rules only → unchanged, source "rules" + * - LLM only → LLM's text, deterministic price, source "llm" + */ +export function mergeConsensus( + ruleFindings: Finding[], + llmFindings: Finding[] +): Finding[] { + const llmByKey = new Map(llmFindings.map((f) => [consensusKey(f), f])); + const merged: Finding[] = []; + + for (const rf of ruleFindings) { + const key = consensusKey(rf); + const lf = llmByKey.get(key); + if (lf) { + llmByKey.delete(key); + merged.push({ + ...rf, + source: "both", + conf: Math.min(0.95, Math.max(rf.conf, lf.conf) + 0.1), + }); + } else { + merged.push({ ...rf, source: rf.source ?? "rules" }); + } } - // Guardrail 4 (final pass): a cost finding the cumulative cap zeroed out is - // noise, not a "quality improvement" — drop it. Quality/org findings stay. - const final = kept.filter( - (c) => c.finding.sav > 0 || KEEP_ZERO_SAVINGS.test(c.finding.cat as string) - ); + for (const lf of llmByKey.values()) { + merged.push({ ...lf, source: "llm" }); + } - // Same severity+savings ordering the hardcoded engine uses. - const sv: Record = { - [Severity.CRITICAL]: 0, - [Severity.WARNING]: 1, - [Severity.INFO]: 2, - [Severity.OK]: 3, - }; - return final - .map((c) => c.finding) - .sort((a, b) => - sv[a.sev] !== sv[b.sev] ? sv[a.sev] - sv[b.sev] : b.sav - a.sav - ); + return merged.sort(bySeverityThenSavings); } /** Extract a JSON object even if the model wraps it in prose / code fences. */ -function parseFindings(content: string): LlmFinding[] { +function parseFindings(content: string): LlmProposal[] { let txt = content.trim(); const fence = txt.match(/```(?:json)?\s*([\s\S]*?)```/); if (fence) txt = fence[1].trim(); @@ -383,27 +519,39 @@ export const LLM_FALLBACK_NOTICE = "LLM unavailable, showing deterministic analysis"; /** - * Run the LLM analysis; on any failure fall back to the deterministic rule - * engine and surface a notice so the report says which engine actually ran. + * Consensus run: the rule engine always runs first (synchronously), then the + * LLM augments it. On success the two merge with per-finding provenance + * (engine "hybrid"); on any LLM failure the report degrades to the rules-only + * findings with a notice — it is never empty, because the rules already ran. */ export async function analyzeWithFallback( llm: () => Promise, rules: () => Finding[] ): Promise { + const ruleFindings = rules(); try { const res = await llm(); - return { findings: res.findings, engine: "llm", llmUsage: res.usage }; + return { + findings: mergeConsensus(ruleFindings, res.findings), + engine: "hybrid", + llmUsage: res.usage, + }; } catch (e) { - console.warn("LLM analysis failed, falling back to rule engine:", e); - return { findings: rules(), engine: "rules", notice: LLM_FALLBACK_NOTICE }; + console.warn("LLM augmentation failed, showing rule-engine findings:", e); + return { + findings: ruleFindings, + engine: "rules", + notice: LLM_FALLBACK_NOTICE, + }; } } /** - * Run LLM-guided analysis via NVIDIA NIM. Returns Findings in the same shape as - * the hardcoded engines, plus the NIM call's own token usage when the response - * reports it. Throws on transport/parse failure so callers can fall back to - * the rule engine (see analyzeWithFallback). + * Run LLM-guided detection via NVIDIA NIM and price each proposal with the + * vendor costing module. Returns Findings in the same shape as the rule + * engines (source "llm"), plus the NIM call's own token usage when the + * response reports it. Throws on transport/parse failure so callers can + * degrade to rules-only (see analyzeWithFallback). */ export async function findIssuesLLM( rows: UsageSummary[], @@ -473,7 +621,7 @@ export async function findIssuesLLM( : undefined; return { - findings: mergeLlmFindings(parseFindings(content), rows, ctx), + findings: priceLlmFindings(parseFindings(content), rows, ctx), usage, }; } diff --git a/src/lib/openai/analysis.ts b/src/lib/openai/analysis.ts index e698678..ac62d5f 100644 --- a/src/lib/openai/analysis.ts +++ b/src/lib/openai/analysis.ts @@ -1,8 +1,22 @@ /* ═══════════════════ OpenAI ANALYSIS ENGINE ═══════════════════ */ -import type { Finding } from "@/types"; +import type { Finding, FindingSignal } from "@/types"; import { OpenAICategory, Severity } from "@/types/analysis"; import { prOpenAI, tcOpenAI } from "./pricing"; +import { + costBatchDiscountOpenAI, + costEnableCachingOpenAI, + costGpt4oDowngrade, + costLegacyGpt4Upgrade, + costMiniDowngrade, + costModelUpgradeOpenAI, + costPromptTrim, + costRagReductionOpenAI, + costTenPercentTrim, + ragReductionFactorOpenAI, + upgradeTargetOpenAI, + type OpenAICostRow, +} from "./costing"; import { $, P } from "@/lib/formatters"; import type { OpenAIUsageData } from "./api"; @@ -211,6 +225,15 @@ export function findIssuesOpenAI( // If using costs API without token data, skip token-based rules const hasTokenData = r.inp > 0 || r.out > 0; + // Shared input to the costing module — all optimized costs derive from it. + const costRow: OpenAICostRow = { + model: r.model || r.line_item || "", + inp: r.inp, + out: r.out, + cur, + conf: 0, + }; + const isGPT4O = r.model.toLowerCase().includes("gpt-4o") && !r.model.toLowerCase().includes("mini"); @@ -233,7 +256,8 @@ export function findIssuesOpenAI( action: string, severity: Severity, confidence: number, - impact?: string + impact?: string, + signals?: FindingSignal[] ) => { // Skip if this category was already added for this model if (addedCategories.has(category)) { @@ -279,6 +303,8 @@ export function findIssuesOpenAI( batchCandidate: false, meanDaily: r.activeDays > 0 ? r.reqs / r.activeDays : 0, }, + source: "rules", + signals, }); } }; @@ -313,7 +339,7 @@ export function findIssuesOpenAI( // WARNING if >20% of total spend OR >$20, otherwise INFO const sev = spendPercent > 0.2 || cur > 20 ? Severity.WARNING : Severity.INFO; - const opt = cur * 0.9; // Conservative 10% optimization potential + const opt = costTenPercentTrim(costRow); // Conservative 10% optimization potential addFinding( OpenAICategory.HIGH_IMPACT_OPPORTUNITY, opt, @@ -328,20 +354,18 @@ export function findIssuesOpenAI( // Detect patterns where prompt caching could save costs // Large, consistent input + many requests = prime candidate if (hasTokenData && ai > 2000 && r.reqs > 100 && r.inp > 5e6) { - const signals = [ - { weight: 0.3, met: ai > 5000 }, // Large prompts - { weight: 0.25, met: r.reqs > 500 }, // High volume - { weight: 0.2, met: ratio > 3 }, // Input-heavy - { weight: 0.15, met: r.activeDays > 15 }, // Sustained usage - { weight: 0.1, met: !isGPT4OMini }, // Higher-tier model + const signals: FindingSignal[] = [ + { weight: 0.3, met: ai > 5000, label: "avg input > 5k tok" }, + { weight: 0.25, met: r.reqs > 500, label: "500+ requests" }, + { weight: 0.2, met: ratio > 3, label: "input-heavy (ratio > 3:1)" }, + { weight: 0.15, met: r.activeDays > 15, label: "15+ active days" }, + { weight: 0.1, met: !isGPT4OMini, label: "higher-tier model" }, ]; const conf = confidenceScore(signals); if (conf >= 0.45) { - // Estimate 50% of input tokens could be cached (conservative) - const cacheableTokens = r.inp * 0.5; - // Cached tokens cost 50% less on input, 0 on subsequent reads - const cacheSavings = (cacheableTokens / 1e6) * p.i * 0.5; - const opt = cur - cacheSavings; + // 50% of input assumed cacheable at half input price (conservative) + const opt = costEnableCachingOpenAI(costRow); + const cacheSavings = cur - opt; const reason = `Large avg input (~${ai.toLocaleString()} tok/req) across ${r.reqs.toLocaleString()} requests. ${(r.inp / 1e6).toFixed(1)}M input tokens/mo with likely repeated system prompts or context.`; const action = `Implement prompt caching for system prompts, instructions, or RAG context. OpenAI caches up to ${isGPT4OMini ? "5min" : "1hr"}. Potential 40-60% input cost reduction.`; const impact = `~${$(cacheSavings)}/mo (${P(cacheSavings, cur)}%) if 50% cacheable`; @@ -353,7 +377,8 @@ export function findIssuesOpenAI( action, sev, conf, - impact + impact, + signals ); } } @@ -366,17 +391,16 @@ export function findIssuesOpenAI( ao < 200 && r.reqs > 50 ) { - const signals = [ - { weight: 0.3, met: ao < 100 }, - { weight: 0.25, met: r.reqs > 500 }, - { weight: 0.2, met: ai < 3000 }, - { weight: 0.15, met: ai < 2000 }, - { weight: 0.1, met: r.activeDays > 20 }, + const signals: FindingSignal[] = [ + { weight: 0.3, met: ao < 100, label: "avg output < 100 tok" }, + { weight: 0.25, met: r.reqs > 500, label: "500+ requests" }, + { weight: 0.2, met: ai < 3000, label: "avg input < 3k tok" }, + { weight: 0.15, met: ai < 2000, label: "avg input < 2k tok" }, + { weight: 0.1, met: r.activeDays > 20, label: "20+ active days" }, ]; const conf = confidenceScore(signals); if (conf >= 0.4) { - const mini = prOpenAI("gpt-4o-mini"); - const opt = (r.inp / 1e6) * mini.i + (r.out / 1e6) * mini.o; + const opt = costMiniDowngrade(costRow); const reason = `Avg output ${ao} tokens across ${r.reqs.toLocaleString()} reqs (avg input ${ai.toLocaleString()} tok). Pattern suggests classification/routing tasks.`; const action = `Switch to GPT-4o-mini. Run A/B test on 100 requests — if quality holds, migrate. Saves ~95%.`; const sev = conf >= 0.65 ? Severity.CRITICAL : Severity.WARNING; @@ -386,25 +410,26 @@ export function findIssuesOpenAI( reason, action, sev, - conf + conf, + undefined, + signals ); } } /* ─── RULE 2: RAG Context Bloat ─── */ if (hasTokenData && ratio > 12 && !isGPT4OMini && r.inp > 10e6) { - const signals = [ - { weight: 0.3, met: ratio > 20 }, - { weight: 0.25, met: ai > 8000 }, - { weight: 0.2, met: ao < 500 }, - { weight: 0.15, met: r.reqs > 100 }, - { weight: 0.1, met: true }, + const signals: FindingSignal[] = [ + { weight: 0.3, met: ratio > 20, label: "in:out ratio > 20:1" }, + { weight: 0.25, met: ai > 8000, label: "avg input > 8k tok" }, + { weight: 0.2, met: ao < 500, label: "avg output < 500 tok" }, + { weight: 0.15, met: r.reqs > 100, label: "100+ requests" }, + { weight: 0.1, met: true, label: "input-dominant workload" }, ]; const conf = confidenceScore(signals); if (conf >= 0.4) { - const reductionFactor = conf >= 0.7 ? 0.5 : 0.6; - const opt = - ((r.inp * reductionFactor) / 1e6) * p.i + (r.out / 1e6) * p.o; + const reductionFactor = ragReductionFactorOpenAI(conf); + const opt = costRagReductionOpenAI(costRow, conf); const reason = `Input:output ratio ${ratio.toFixed(0)}:1 (~${ai.toLocaleString()} tok/req input, ~${ao} output). ${(r.inp / 1e6).toFixed(1)}M input tokens/mo. RAG pulling too many chunks.`; const action = `Audit retrieval: reduce top-k, add reranking, tighten chunk size. Conservative: ${Math.round((1 - reductionFactor) * 100)}% input reduction.`; const sev = conf >= 0.65 ? Severity.CRITICAL : Severity.WARNING; @@ -414,24 +439,25 @@ export function findIssuesOpenAI( reason, action, sev, - conf + conf, + undefined, + signals ); } } /* ─── RULE 3: GPT-4o Overkill → GPT-4o-mini ─── */ if (hasTokenData && isGPT4O && ao >= 200 && r.inp > 5e6) { - const signals = [ - { weight: 0.3, met: ao < 1500 }, - { weight: 0.25, met: r.reqs > 100 }, - { weight: 0.2, met: ratio < 10 }, - { weight: 0.15, met: ai < 10000 }, - { weight: 0.1, met: r.activeDays > 15 }, + const signals: FindingSignal[] = [ + { weight: 0.3, met: ao < 1500, label: "avg output < 1.5k tok" }, + { weight: 0.25, met: r.reqs > 100, label: "100+ requests" }, + { weight: 0.2, met: ratio < 10, label: "in:out ratio < 10:1" }, + { weight: 0.15, met: ai < 10000, label: "avg input < 10k tok" }, + { weight: 0.1, met: r.activeDays > 15, label: "15+ active days" }, ]; const conf = confidenceScore(signals); if (conf >= 0.4) { - const mini = prOpenAI("gpt-4o-mini"); - const opt = (r.inp / 1e6) * mini.i + (r.out / 1e6) * mini.o; + const opt = costMiniDowngrade(costRow); const reason = `GPT-4o with avg ${ao} tok output, ${r.reqs.toLocaleString()} reqs. Moderate complexity where GPT-4o-mini performs comparably.`; const action = `A/B test GPT-4o-mini on 10% traffic. If quality holds, migrate. Saves ~94%.`; const sev = conf >= 0.65 ? Severity.WARNING : Severity.INFO; @@ -441,7 +467,9 @@ export function findIssuesOpenAI( reason, action, sev, - conf + conf, + undefined, + signals ); } } @@ -455,15 +483,15 @@ export function findIssuesOpenAI( const bursty = avgDaily > 100 && r.activeDays < 25; if (bursty) { - const signals = [ - { weight: 0.35, met: avgDaily > 200 }, - { weight: 0.25, met: r.reqs > 500 }, - { weight: 0.2, met: !isGPT4OMini }, - { weight: 0.2, met: r.activeDays < 25 }, + const signals: FindingSignal[] = [ + { weight: 0.35, met: avgDaily > 200, label: "200+ reqs/day" }, + { weight: 0.25, met: r.reqs > 500, label: "500+ requests" }, + { weight: 0.2, met: !isGPT4OMini, label: "higher-tier model" }, + { weight: 0.2, met: r.activeDays < 25, label: "< 25 active days" }, ]; const conf = confidenceScore(signals); if (conf >= 0.4) { - const opt = cur * 0.5; // Batch API offers 50% discount + const opt = costBatchDiscountOpenAI(costRow); // 50% batch discount const reason = `Bursty traffic (~${Math.round(avgDaily)} reqs/day avg). ${r.reqs.toLocaleString()} total reqs — likely batch processing.`; const action = `Migrate to Batch API for 50% cost reduction. Processes within 24hrs.`; const sev = conf >= 0.65 ? Severity.WARNING : Severity.INFO; @@ -473,7 +501,9 @@ export function findIssuesOpenAI( reason, action, sev, - conf + conf, + undefined, + signals ); } } @@ -489,15 +519,15 @@ export function findIssuesOpenAI( r.activeDays >= 20 ) { const avgDaily = r.reqs / r.activeDays; - const signals = [ - { weight: 0.4, met: cur > 80 }, - { weight: 0.3, met: r.reqs > 5000 }, - { weight: 0.2, met: !isGPT4OMini }, - { weight: 0.1, met: avgDaily < 2000 }, + const signals: FindingSignal[] = [ + { weight: 0.4, met: cur > 80, label: "spend > $80/mo" }, + { weight: 0.3, met: r.reqs > 5000, label: "5k+ requests" }, + { weight: 0.2, met: !isGPT4OMini, label: "higher-tier model" }, + { weight: 0.1, met: avgDaily < 2000, label: "steady daily volume" }, ]; const conf = confidenceScore(signals); if (conf >= 0.4) { - const opt = cur * 0.5; + const opt = costBatchDiscountOpenAI(costRow); const reason = `${r.reqs.toLocaleString()} requests/mo (~${Math.round(avgDaily)}/day, ${r.activeDays} active days). Steady high-volume pattern — Batch API gives 50% off for async workloads with 24hr turnaround.`; const action = `If any of these calls are latency-tolerant (evals, data processing, nightly jobs), migrate to Batch API. Zero code change required beyond switching endpoint to /v1/batches.`; const sev = conf >= 0.65 ? Severity.WARNING : Severity.INFO; @@ -507,7 +537,9 @@ export function findIssuesOpenAI( reason, action, sev, - conf + conf, + undefined, + signals ); } } @@ -520,17 +552,16 @@ export function findIssuesOpenAI( r.model.toLowerCase().includes("o3")) && r.reqs > 50 ) { - const signals = [ - { weight: 0.35, met: ao < 500 }, // Short outputs suggest simple tasks - { weight: 0.25, met: ai < 5000 }, // Short inputs suggest simple prompts - { weight: 0.2, met: r.reqs > 200 }, // High volume - { weight: 0.15, met: ratio < 8 }, // Not context-heavy - { weight: 0.05, met: cur > 10 }, // Significant spend + const signals: FindingSignal[] = [ + { weight: 0.35, met: ao < 500, label: "avg output < 500 tok" }, + { weight: 0.25, met: ai < 5000, label: "avg input < 5k tok" }, + { weight: 0.2, met: r.reqs > 200, label: "200+ requests" }, + { weight: 0.15, met: ratio < 8, label: "not context-heavy" }, + { weight: 0.05, met: cur > 10, label: "spend > $10/mo" }, ]; const conf = confidenceScore(signals); if (conf >= 0.5) { - const gpt4o = prOpenAI("gpt-4o"); - const opt = (r.inp / 1e6) * gpt4o.i + (r.out / 1e6) * gpt4o.o; + const opt = costGpt4oDowngrade(costRow); const reason = `Using ${r.model} for ${r.reqs.toLocaleString()} reqs with avg ${ao} tok output. O-series excels at complex reasoning, but pattern suggests simpler tasks.`; const action = `Test GPT-4o on representative sample. O-series adds 60-80% cost premium for reasoning - verify it's needed. Consider GPT-4o or 4o-mini.`; const sev = conf >= 0.7 ? Severity.WARNING : Severity.INFO; @@ -540,7 +571,9 @@ export function findIssuesOpenAI( reason, action, sev, - conf + conf, + undefined, + signals ); } } @@ -554,7 +587,7 @@ export function findIssuesOpenAI( const reason = `This pattern represents ${P(cur, totalSpend)}% of total OpenAI spend (${$(cur)}/${$(totalSpend)}/mo). ${r.reqs.toLocaleString()} reqs, avg ${ai.toLocaleString()} in / ${ao} out tokens.`; const action = `High-impact optimization target. Review: (1) Model choice, (2) Prompt efficiency, (3) Request patterns. Even 10% reduction = ${$(cur * 0.1)}/mo.`; const sev = Severity.INFO; - const opt = cur * 0.9; // Assume 10% optimization potential + const opt = costTenPercentTrim(costRow); // Assume 10% optimization potential addFinding( OpenAICategory.HIGH_IMPACT_OPPORTUNITY, opt, @@ -568,18 +601,17 @@ export function findIssuesOpenAI( /* ─── RULE 8: Token Efficiency - Prompt Bloat ─── */ // Detect unnecessarily verbose prompts or inefficient formatting if (hasTokenData && ai > 8000 && r.reqs > 100 && cur > 3) { - const signals = [ - { weight: 0.3, met: ai > 12000 }, // Very large inputs - { weight: 0.25, met: ao < ai * 0.05 }, // Output is tiny compared to input - { weight: 0.2, met: r.reqs > 500 }, // High volume amplifies waste - { weight: 0.15, met: ratio > 15 }, // Extreme input/output ratio - { weight: 0.1, met: true }, + const signals: FindingSignal[] = [ + { weight: 0.3, met: ai > 12000, label: "avg input > 12k tok" }, + { weight: 0.25, met: ao < ai * 0.05, label: "output < 5% of input" }, + { weight: 0.2, met: r.reqs > 500, label: "500+ requests" }, + { weight: 0.15, met: ratio > 15, label: "in:out ratio > 15:1" }, + { weight: 0.1, met: true, label: "verbose prompt pattern" }, ]; const conf = confidenceScore(signals); if (conf >= 0.5) { // Estimate 25% token reduction through optimization - const optimizedInput = r.inp * 0.75; - const opt = (optimizedInput / 1e6) * p.i + (r.out / 1e6) * p.o; + const opt = costPromptTrim(costRow); const reason = `Avg ${ai.toLocaleString()} input tokens/req producing ${ao} output tokens. ${(r.inp / 1e6).toFixed(1)}M input tokens/mo. Suggests verbose prompts, redundant context, or inefficient formatting.`; const action = `Audit prompts: (1) Remove instructional bloat, (2) Use structured outputs, (3) Compress examples, (4) Trim RAG context. Target 25% reduction = ${$(cur - opt)}/mo.`; const sev = conf >= 0.7 ? Severity.WARNING : Severity.INFO; @@ -589,7 +621,9 @@ export function findIssuesOpenAI( reason, action, sev, - conf + conf, + undefined, + signals ); } } @@ -597,17 +631,16 @@ export function findIssuesOpenAI( /* ─── RULE 7: GPT-4 Legacy → GPT-4o Upgrade ─── */ // Detect old GPT-4 models that should upgrade to GPT-4o if (hasTokenData && isGPT4 && r.reqs > 50 && cur > 2) { - const signals = [ - { weight: 0.3, met: r.reqs > 100 }, - { weight: 0.25, met: cur > 5 }, - { weight: 0.2, met: r.activeDays > 3 }, - { weight: 0.15, met: true }, // Always recommend upgrading from legacy GPT-4 - { weight: 0.1, met: ao < 500 }, + const signals: FindingSignal[] = [ + { weight: 0.3, met: r.reqs > 100, label: "100+ requests" }, + { weight: 0.25, met: cur > 5, label: "spend > $5/mo" }, + { weight: 0.2, met: r.activeDays > 3, label: "3+ active days" }, + { weight: 0.15, met: true, label: "legacy GPT-4 in use" }, + { weight: 0.1, met: ao < 500, label: "avg output < 500 tok" }, ]; const conf = confidenceScore(signals); if (conf >= 0.4) { - const gpt4o = prOpenAI("gpt-4o"); - const opt = (r.inp / 1e6) * gpt4o.i + (r.out / 1e6) * gpt4o.o; + const opt = costLegacyGpt4Upgrade(costRow); const reason = `Using legacy ${r.model} (${r.reqs.toLocaleString()} reqs, avg ${ao} tok output). GPT-4o offers better performance at similar or lower cost.`; const action = `Upgrade to GPT-4o (gpt-4o-2024-08-06). Drop-in replacement with better reasoning, faster speed, and lower cost. Test on staging first.`; const sev = Severity.INFO; @@ -617,7 +650,9 @@ export function findIssuesOpenAI( reason, action, sev, - conf + conf, + undefined, + signals ); } } @@ -625,16 +660,10 @@ export function findIssuesOpenAI( /* ─── RULE 9: Legacy Model ─── */ if (p.g > 0 && p.g < 4 && cur > 2) { // Suggest upgrading to newer models - const newer = isO1 - ? prOpenAI("o3") - : isGPT4 - ? prOpenAI("gpt-4o") - : prOpenAI("gpt-4o-mini"); - const newerCost = (r.inp / 1e6) * newer.i + (r.out / 1e6) * newer.o; - const savOrCost = cur - newerCost; + const newer = upgradeTargetOpenAI(costRow.model); + const opt = costModelUpgradeOpenAI(costRow); const conf = 0.8; - const opt = Math.min(cur, newerCost); - const reason = `Running ${p.l} (gen ${p.g}). ${newer.l} offers better performance${savOrCost > 0 ? " at lower cost" : ""}.`; + const reason = `Running ${p.l} (gen ${p.g}). ${newer.l} offers better performance${opt < cur ? " at lower cost" : ""}.`; const action = `Update model to ${newer.l.toLowerCase().replace(/ /g, "-")}. Test on staging first.`; const sev = Severity.INFO; addFinding(OpenAICategory.MODEL_UPGRADE, opt, reason, action, sev, conf); @@ -692,6 +721,7 @@ export function findIssuesOpenAI( batchCandidate: false, meanDaily: 0, }, + source: "rules", }); } @@ -736,6 +766,7 @@ export function findIssuesOpenAI( batchCandidate: false, meanDaily: 0, }, + source: "rules", }); } diff --git a/src/lib/openai/costing.ts b/src/lib/openai/costing.ts new file mode 100644 index 0000000..632f38f --- /dev/null +++ b/src/lib/openai/costing.ts @@ -0,0 +1,137 @@ +/* ═══════════════════ OpenAI COSTING MODULE ═══════════════════ */ +/* + * Deterministic optimized-cost formulas for every OpenAI finding category. + * The rule engine (findIssuesOpenAI) and the LLM consensus path both price + * findings through these functions — dollar figures always come from the + * row's actual metrics and the pricing table, never from an LLM. + */ + +import { OpenAICategory } from "@/types/analysis"; +import { prOpenAI } from "./pricing"; + +/** Row metrics every OpenAI costing formula works from. */ +export interface OpenAICostRow { + model: string; + inp: number; // input tokens / mo (0 when only costs-API data exists) + out: number; // output tokens / mo + cur: number; // current monthly cost (USD) + conf: number; // confidence 0-1, drives conservative reduction factors +} + +const hasTokenData = (row: OpenAICostRow) => row.inp > 0 || row.out > 0; + +/** Rules 1/3: reprice the row's tokens at GPT-4o-mini rates. */ +export function costMiniDowngrade(row: OpenAICostRow): number { + const mini = prOpenAI("gpt-4o-mini"); + return (row.inp / 1e6) * mini.i + (row.out / 1e6) * mini.o; +} + +/** Rule 5 (and Downgrade → GPT-4o): reprice at GPT-4o rates. */ +export function costGpt4oDowngrade(row: OpenAICostRow): number { + const gpt4o = prOpenAI("gpt-4o"); + return (row.inp / 1e6) * gpt4o.i + (row.out / 1e6) * gpt4o.o; +} + +/** + * Rule 2: input reduction from tighter retrieval — 50% cut at high + * confidence, conservative 40% otherwise. + */ +export function costRagReductionOpenAI( + row: OpenAICostRow, + conf: number +): number { + const p = prOpenAI(row.model); + const reductionFactor = conf >= 0.7 ? 0.5 : 0.6; + return ((row.inp * reductionFactor) / 1e6) * p.i + (row.out / 1e6) * p.o; +} + +/** Rule 2's reduction factor, exposed so reason/action text can cite it. */ +export function ragReductionFactorOpenAI(conf: number): number { + return conf >= 0.7 ? 0.5 : 0.6; +} + +/** Rule 0: 50% of input assumed cacheable at half input price. */ +export function costEnableCachingOpenAI(row: OpenAICostRow): number { + const p = prOpenAI(row.model); + const cacheSavings = ((row.inp * 0.5) / 1e6) * p.i * 0.5; + return row.cur - cacheSavings; +} + +/** Rules 4/4b: Batch API's 50% discount on the whole workload. */ +export function costBatchDiscountOpenAI(row: OpenAICostRow): number { + return row.cur * 0.5; +} + +/** Rule 8: 25% input-token reduction from tighter prompts. */ +export function costPromptTrim(row: OpenAICostRow): number { + const p = prOpenAI(row.model); + return ((row.inp * 0.75) / 1e6) * p.i + (row.out / 1e6) * p.o; +} + +/** Rules 0b/6: conservative 10% optimization potential on the workload. */ +export function costTenPercentTrim(row: OpenAICostRow): number { + return row.cur * 0.9; +} + +/** Rule 7: reprice legacy GPT-4 tokens at GPT-4o rates (may exceed cur). */ +export function costLegacyGpt4Upgrade(row: OpenAICostRow): number { + const gpt4o = prOpenAI("gpt-4o"); + return (row.inp / 1e6) * gpt4o.i + (row.out / 1e6) * gpt4o.o; +} + +/** + * Rule 9: reprice at the newest comparable model; never above current cost + * (an upgrade is recommended for quality even when it isn't cheaper). + */ +export function costModelUpgradeOpenAI(row: OpenAICostRow): number { + const newer = upgradeTargetOpenAI(row.model); + const newerCost = (row.inp / 1e6) * newer.i + (row.out / 1e6) * newer.o; + return Math.min(row.cur, newerCost); +} + +/** The newest comparable model's pricing entry, for reason/action text. */ +export function upgradeTargetOpenAI(model: string) { + const m = (model || "").toLowerCase(); + const isO1 = m.includes("o1") && !m.includes("o3"); + const isGPT4 = m.includes("gpt-4") && !m.includes("gpt-4o"); + return isO1 + ? prOpenAI("o3") + : isGPT4 + ? prOpenAI("gpt-4o") + : prOpenAI("gpt-4o-mini"); +} + +/** + * Category dispatcher for the consensus path: given an LLM-proposed category + * and the row's real metrics, return the deterministic optimized monthly cost, + * or null when that category can't be costed for that row (e.g. token-based + * repricing without token data). + */ +export function optimizedCostOpenAI( + cat: OpenAICategory, + row: OpenAICostRow +): number | null { + switch (cat) { + case OpenAICategory.MODEL_DOWNGRADE_MINI: + return hasTokenData(row) ? costMiniDowngrade(row) : null; + case OpenAICategory.MODEL_DOWNGRADE_4O: + case OpenAICategory.REASONING_MODEL_OVERKILL: + return hasTokenData(row) ? costGpt4oDowngrade(row) : null; + case OpenAICategory.RAG_OPTIMIZATION: + return row.inp > 0 ? costRagReductionOpenAI(row, row.conf) : null; + case OpenAICategory.PROMPT_CACHING: + return row.inp > 0 ? costEnableCachingOpenAI(row) : null; + case OpenAICategory.PROMPT_OPTIMIZATION: + return row.inp > 0 ? costPromptTrim(row) : null; + case OpenAICategory.BATCH_API_MIGRATION: + return row.cur > 0 ? costBatchDiscountOpenAI(row) : null; + case OpenAICategory.HIGH_IMPACT_OPPORTUNITY: + return row.cur > 0 ? costTenPercentTrim(row) : null; + case OpenAICategory.MODEL_UPGRADE: + return hasTokenData(row) ? costModelUpgradeOpenAI(row) : null; + default: + // No deterministic formula (e.g. Project Organization) — the caller + // decides whether a zero-savings finding survives. + return null; + } +} diff --git a/src/types/analysis.ts b/src/types/analysis.ts index f25fd37..4ee2e68 100644 --- a/src/types/analysis.ts +++ b/src/types/analysis.ts @@ -81,6 +81,20 @@ export interface AggregatedRow { activeDays: number; } +// Which engine surfaced a finding in a consensus run. "rules" = deterministic +// rule engine; "llm" = AI-spotted (priced deterministically); "both" = found +// independently by both engines. Optional so reports stored before consensus +// runs existed still parse. +export type FindingSource = "rules" | "llm" | "both"; + +// One detection signal behind a rule finding: its confidence weight, whether +// the row met it, and a short human-readable label for the "Based on:" trail. +export interface FindingSignal { + weight: number; + met: boolean; + label: string; +} + export interface Finding { id: string; name: string; @@ -106,6 +120,8 @@ export interface Finding { impact: string; activeDays: number; temporal: TemporalPattern; + source?: FindingSource; + signals?: FindingSignal[]; } export interface WorkspaceSpend { @@ -115,8 +131,10 @@ export interface WorkspaceSpend { } // Which engine produced a report's findings. "rules" = deterministic rule -// engine; "llm" = NVIDIA NIM LLM-guided analysis. -export type AnalysisEngine = "rules" | "llm"; +// engine; "hybrid" = consensus run (rules + NVIDIA NIM LLM augmentation); +// "llm" stays valid so reports stored by the old LLM-replaces-rules mode +// still parse and render. +export type AnalysisEngine = "rules" | "llm" | "hybrid"; // Token usage of the NIM analysis call itself, for the ROI footer on // LLM-engine reports.