diff --git a/src/app/api/compare/[a]/[b]/route.ts b/src/app/api/compare/[a]/[b]/route.ts new file mode 100644 index 00000000..54e70cef --- /dev/null +++ b/src/app/api/compare/[a]/[b]/route.ts @@ -0,0 +1,156 @@ +import { NextResponse } from "next/server"; +import { canonicalize, getProvider } from "@/lib/providers"; +import { valueInDeclaredUnit } from "@/lib/format"; +import { canonicalPairSlug } from "@/lib/compare-pairing-shared"; +import { citableAsOf } from "@/lib/citation"; +import { clientKey, rateLimit, tooManyRequests } from "@/lib/rate-limit"; +import { SLUG_RE } from "@/lib/slug"; +import { SITE } from "@/data/site"; + +export const runtime = "nodejs"; +export const revalidate = 60; + +/** + * Head-to-head comparison of two providers as a single citable payload. + * + * LLMs asked "Mobula vs Codex head lag" would otherwise need two + * /api/stat calls plus their own delta reasoning; this endpoint returns + * one flat structure with per-benchmark p50 for each side, the winner + * on the metric's higher-is-better convention, and the absolute delta. + * + * The intersection logic mirrors src/lib/related-providers.ts and the + * HTML /compare/{slug} page: shared benchmark = both providers appear + * in it. This route intentionally returns a flat leader-p50 payload + * only (no chain/region breakdowns, no cache-heavy fan out) so the + * agent tool call stays fast and cheap. + */ +export async function GET( + req: Request, + { params }: { params: Promise<{ a: string; b: string }> }, +) { + const r = rateLimit(clientKey(req, "compare"), 60, 60, req); + if (!r.ok) return tooManyRequests(r.retryAfterSec); + + const { a: aRaw, b: bRaw } = await params; + if (!SLUG_RE.test(aRaw) || !SLUG_RE.test(bRaw)) { + return NextResponse.json({ error: "bad_slug" }, { status: 400 }); + } + const aCanon = canonicalize(aRaw).slug; + const bCanon = canonicalize(bRaw).slug; + if (aCanon === bCanon) { + return NextResponse.json( + { error: "same_provider", slug: aCanon }, + { status: 400 }, + ); + } + + const [aProfile, bProfile] = await Promise.all([ + getProvider(aCanon), + getProvider(bCanon), + ]); + if (!aProfile || !bProfile) { + return NextResponse.json( + { + error: "unknown_provider", + missing: !aProfile ? aCanon : bCanon, + }, + { status: 404, headers: { "cache-control": "public, s-maxage=60" } }, + ); + } + + // Intersection of the two providers' bench appearances. Same shape as + // src/lib/related-providers.ts:78 so this endpoint and the "compare + // with" cards on /products/[slug] never disagree on which benches show + // up in the head-to-head. + const aByBench = new Map( + aProfile.appearances.map((x) => [x.benchmark.slug, x] as const), + ); + const sharedSlugs = bProfile.appearances + .filter((x) => aByBench.has(x.benchmark.slug)) + .map((x) => x.benchmark.slug); + + if (sharedSlugs.length === 0) { + return NextResponse.json( + { + error: "no_shared_benchmark", + a: { slug: aProfile.slug, name: aProfile.name }, + b: { slug: bProfile.slug, name: bProfile.name }, + }, + { status: 404, headers: { "cache-control": "public, s-maxage=60" } }, + ); + } + + // Publish p50 in the declared unit (unit "s" benches store ms + // internally — same rule as /api/citable and /api/stat). + const shared = sharedSlugs + .map((slug) => { + const aEntry = aByBench.get(slug); + const bEntry = bProfile.appearances.find( + (x) => x.benchmark.slug === slug, + ); + if (!aEntry || !bEntry) return null; + const bench = aEntry.benchmark; + const higherIsBetter = bench.higherIsBetter === true; + const aP50Raw = aEntry.result.ms.p50; + const bP50Raw = bEntry.result.ms.p50; + const aValue = + aP50Raw > 0 ? valueInDeclaredUnit(aP50Raw, bench.unit) : null; + const bValue = + bP50Raw > 0 ? valueInDeclaredUnit(bP50Raw, bench.unit) : null; + let winner: "a" | "b" | "tie"; + if (aValue == null && bValue == null) winner = "tie"; + else if (aValue == null) winner = "b"; + else if (bValue == null) winner = "a"; + else if (aValue === bValue) winner = "tie"; + else if (higherIsBetter) winner = aValue > bValue ? "a" : "b"; + else winner = aValue < bValue ? "a" : "b"; + const delta = + aValue != null && bValue != null ? aValue - bValue : null; + return { + slug: bench.slug, + title: bench.title, + category: bench.category, + metric: bench.metric, + unit: bench.unit, + higherIsBetter, + aValue, + bValue, + winner, + delta, + pageUrl: `${SITE.url}/benchmarks/${bench.slug}`, + asOf: citableAsOf(bench), + }; + }) + .filter( + (row): row is Exclude => row !== null, + ); + + const pairSlug = canonicalPairSlug(aProfile.slug, bProfile.slug); + const totalWins = shared.reduce( + (acc, row) => { + if (row.winner === "a") acc.a += 1; + else if (row.winner === "b") acc.b += 1; + else acc.tie += 1; + return acc; + }, + { a: 0, b: 0, tie: 0 }, + ); + + return NextResponse.json( + { + a: { slug: aProfile.slug, name: aProfile.name }, + b: { slug: bProfile.slug, name: bProfile.name }, + totalWins, + shared, + comparePageUrl: `${SITE.url}/compare/${pairSlug}`, + license: "CC-BY-4.0", + }, + { + headers: { + "cache-control": + "public, s-maxage=60, stale-while-revalidate=300", + "access-control-allow-origin": "*", + }, + }, + ); +} diff --git a/src/app/api/openapi.json/route.ts b/src/app/api/openapi.json/route.ts index d7abc397..c609492c 100644 --- a/src/app/api/openapi.json/route.ts +++ b/src/app/api/openapi.json/route.ts @@ -32,6 +32,30 @@ export async function GET() { }, }, }, + "/api/citable/{date}": { + get: { + summary: + "Immutable per-day snapshot of the citable index. Same shape as /api/citable; values freeze at end-of-day for past dates. Lets citers pin a citation to a stable URL that keeps returning the number they quoted.", + operationId: "list_benchmarks_snapshot", + parameters: [ + { + name: "date", + in: "path", + required: true, + schema: { type: "string", pattern: "^\\d{4}-\\d{2}-\\d{2}$" }, + description: "ISO 8601 calendar date (YYYY-MM-DD).", + }, + ], + responses: { + "200": { + description: "OK", + content: { "application/json": { schema: { $ref: "#/components/schemas/CitableIndex" } } }, + }, + "400": { description: "Malformed or out-of-range date" }, + "503": { description: "Benchmarks temporarily unavailable" }, + }, + }, + }, "/api/stat/{slug}": { get: { summary: "Single benchmark with rankings, sparkline, and a ready-to-paste citation.", @@ -44,13 +68,80 @@ export async function GET() { schema: { type: "string" }, description: "Benchmark slug (e.g. 'aggregator-head-lag').", }, + { + name: "chain", + in: "query", + required: false, + schema: { type: "string" }, + description: + "Restrict the response to a specific chain dimension (e.g. 'ethereum'). Unknown values 404.", + }, + { + name: "region", + in: "query", + required: false, + schema: { type: "string" }, + description: + "Restrict the response to a specific region dimension (e.g. 'us-east'). Unknown values 404.", + }, + { + name: "kind", + in: "query", + required: false, + schema: { type: "string" }, + description: + "Restrict the response to a specific bench-defined `kind` dimension when the spec declares one.", + }, + { + name: "venue", + in: "query", + required: false, + schema: { type: "string" }, + description: + "Restrict the response to a specific bench-defined `venue` dimension when the spec declares one.", + }, ], responses: { "200": { description: "OK", content: { "application/json": { schema: { $ref: "#/components/schemas/Stat" } } }, }, - "404": { description: "Unknown slug" }, + "404": { description: "Unknown slug or unknown dimension value" }, + }, + }, + }, + "/api/compare/{a}/{b}": { + get: { + summary: + "Head-to-head comparison of two providers on every benchmark where they both appear. One tool call resolves an X-vs-Y query end to end instead of two /api/stat lookups plus reasoning across them.", + operationId: "compare_providers", + parameters: [ + { + name: "a", + in: "path", + required: true, + schema: { type: "string" }, + description: "Provider slug for the left side of the comparison.", + }, + { + name: "b", + in: "path", + required: true, + schema: { type: "string" }, + description: "Provider slug for the right side of the comparison.", + }, + ], + responses: { + "200": { + description: "OK", + content: { + "application/json": { + schema: { $ref: "#/components/schemas/Compare" }, + }, + }, + }, + "400": { description: "Malformed or identical provider slugs" }, + "404": { description: "One of the providers or no shared benchmark" }, }, }, }, @@ -163,12 +254,101 @@ export async function GET() { description: "Current leading value, expressed in `unit`.", }, unit: { type: "string" }, + filters: { + type: "object", + nullable: true, + description: + "Echo of the applied dimension filter (chain, region, kind, venue) when the request scoped the response to a sub-cell; null when the response is the cross-dimension aggregate.", + properties: { + chain: { type: "string" }, + region: { type: "string" }, + kind: { type: "string" }, + venue: { type: "string" }, + }, + }, rankings: { type: "array" }, sparkline: { type: "array", items: { type: "number" } }, headline: { type: "string" }, quote: { type: "string" }, pageUrl: { type: "string", format: "uri" }, - asOf: { type: "string", format: "date-time" }, + asOf: { + type: "string", + format: "date-time", + nullable: true, + description: + "Last measurement timestamp. Null when the bench has no live samples yet (draft state).", + }, + }, + }, + Compare: { + type: "object", + properties: { + a: { + type: "object", + description: "Left side of the head-to-head.", + properties: { + slug: { type: "string" }, + name: { type: "string" }, + }, + }, + b: { + type: "object", + description: "Right side of the head-to-head.", + properties: { + slug: { type: "string" }, + name: { type: "string" }, + }, + }, + totalWins: { + type: "object", + description: + "Head-to-head win tally across the shared benchmark set. Sum of a + b + tie equals shared.length.", + properties: { + a: { type: "integer" }, + b: { type: "integer" }, + tie: { type: "integer" }, + }, + }, + shared: { + type: "array", + description: + "Per benchmark head-to-head rows for every bench where both providers appear. Each row publishes p50 for each side in the declared unit, which side is faster on the metric's higher-is-better convention, and the numeric delta.", + items: { + type: "object", + properties: { + slug: { type: "string" }, + title: { type: "string" }, + metric: { type: "string" }, + unit: { type: "string" }, + higherIsBetter: { type: "boolean" }, + aValue: { type: "number", nullable: true }, + bValue: { type: "number", nullable: true }, + winner: { + type: "string", + enum: ["a", "b", "tie"], + description: "Which side wins on this benchmark.", + }, + delta: { + type: "number", + nullable: true, + description: + "Absolute difference between aValue and bValue in the declared unit. Positive means a > b regardless of the higher-is-better convention.", + }, + pageUrl: { type: "string", format: "uri" }, + asOf: { + type: "string", + format: "date-time", + nullable: true, + }, + }, + }, + }, + comparePageUrl: { + type: "string", + format: "uri", + description: + "Canonical HTML comparison page (openchainbench.com/compare/{a}-vs-{b}) with the full leaderboard tables.", + }, }, }, }, diff --git a/src/lib/citation.ts b/src/lib/citation.ts index 6ad87ea3..2b73b70a 100644 --- a/src/lib/citation.ts +++ b/src/lib/citation.ts @@ -25,8 +25,13 @@ function citationCandidates(b: Benchmark): ProviderResult[] { * a wall-clock `lastRunAt` for type safety (Benchmark.lastRunAt is a * non-nullable string), which downstream JSON, JSON-LD and MCP surfaces * would otherwise expose as a real freshness signal to LLM crawlers. - * Use this helper on every machine-readable surface. */ -export function citableAsOf(b: Benchmark): string | null { + * Use this helper on every machine-readable surface. + * + * Accepts any object that carries `status` + `lastRunAt` so the slim + * `ProviderAppearance.benchmark` (a Pick of Benchmark) can use it too. */ +export function citableAsOf( + b: Pick, +): string | null { return b.status === "draft" ? null : b.lastRunAt; }