From 9daf09d8dcfc9e28a8675ab5a0851a98010578f3 Mon Sep 17 00:00:00 2001 From: Flotapponnier <160007691+Flotapponnier@users.noreply.github.com> Date: Wed, 8 Jul 2026 17:26:55 +0200 Subject: [PATCH 1/2] alerting: rpc error-rate + keyed quota + ws disconnect rules (gaps from 2026-07-08 audit) (#1002) Co-authored-by: Florent Tapponnier --- .../monitoring/prometheus/alert_rules.yml | 47 +++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/infrastructure/monitoring/prometheus/alert_rules.yml b/infrastructure/monitoring/prometheus/alert_rules.yml index b4501d99..233c7354 100644 --- a/infrastructure/monitoring/prometheus/alert_rules.yml +++ b/infrastructure/monitoring/prometheus/alert_rules.yml @@ -118,3 +118,50 @@ groups: annotations: summary: '{{ if eq $externalLabels.environment "staging" }}[STAGING] {{ end }}Latency monitor service is down' description: "The aggregator latency monitor has been down for 2 minutes. No metrics are being collected." + + - name: bench_coverage_alerts + interval: 60s + rules: + # RPC provider error-rate (no-key + keyed): the 1RPC/dRPC per-IP + # throttling incident (2026-07-08) ran for days unalerted. Fires + # when a (provider, chain) cell errors on >50% of calls for 1h. + - alert: RpcProviderErrorRate + expr: | + ( + sum by (provider, chain) (rate(rpc_call_total{result!="ok"}[30m])) + / sum by (provider, chain) (rate(rpc_call_total[30m])) + ) > 0.5 + for: 1h + labels: + severity: warning + alert_type: rpc_error_rate + app: rpc_benches + annotations: + summary: '{{ $labels.provider }} erroring on {{ $labels.chain }} ({{ $value | humanizePercentage }})' + description: "More than half of probes to {{ $labels.provider }} on {{ $labels.chain }} failed over the last 30m, sustained 1h. Rate limit, dead endpoint, or our own footprint (check probe cadence before blaming the provider)." + + # Keyed free-tier quota guard approaching: the guard pauses at 90%, + # this warns at 85% so a human can react before data gaps appear. + - alert: KeyedQuotaNearExhaustion + expr: rpc_keyed_quota_used_ratio > 0.85 + for: 10m + labels: + severity: warning + alert_type: quota + app: rpc_benches + annotations: + summary: 'Keyed RPC quota {{ $labels.provider }} at {{ $value | humanizePercentage }} of monthly budget' + description: "The 90% guard will pause probing soon; leaderboard gaps follow. Consider a cadence reduction or a bigger plan." + + # Aggregator WS silently down: connection-state gauge from the + # reconnect instrumentation (PR #950). + - alert: AggregatorWSDisconnected + expr: ws_connected == 0 + for: 10m + labels: + severity: warning + alert_type: ws_state + app: aggregator_latency_monitor + annotations: + summary: '{{ $labels.aggregator }} WebSocket disconnected ({{ $labels.region }})' + description: "ws_connected has been 0 for 10 minutes; reconnect loop is running but not succeeding. Check JWT/cookie/proxy for codex, API key for mobula." From a6e23bd4df15776338262c8d456e010c802501d3 Mon Sep 17 00:00:00 2001 From: Flotapponnier <160007691+Flotapponnier@users.noreply.github.com> Date: Wed, 8 Jul 2026 17:37:47 +0200 Subject: [PATCH 2/2] prod gate: rpc-hub cohort snapshot + answers built on gated benches (#1003) Co-authored-by: Florent Tapponnier --- src/lib/answers.ts | 13 +++++++++++++ src/lib/removed-benches.ts | 10 ++++++++++ src/lib/rpc-hub-stats.ts | 38 +++++++++++++++++++++++++++++++++++--- src/middleware.ts | 13 +++++++++++-- 4 files changed, 69 insertions(+), 5 deletions(-) diff --git a/src/lib/answers.ts b/src/lib/answers.ts index 84ed3acc..343d9e04 100644 --- a/src/lib/answers.ts +++ b/src/lib/answers.ts @@ -17,6 +17,10 @@ import { cache } from "react"; import yaml from "js-yaml"; import { z } from "zod"; import { loadBenchmark } from "@/lib/spec"; +import { + REMOVED_ANSWER_SLUGS, + REMOVED_BENCH_SLUGS, +} from "@/lib/removed-benches"; import type { Benchmark } from "@/types/benchmark"; const ANSWERS_DIR = path.join(process.cwd(), "answers"); @@ -85,6 +89,15 @@ export const loadAllAnswers = cache(async (): Promise => { ); return parsed .filter((a): a is Answer => a !== null && a.status === "live") + // Prod-only gate: answers built on staging-pipeline benches never + // reach the prod listing, sitemap or tag clouds. Direct URL hits + // get a 410 from middleware. + .filter( + (a) => + process.env.VERCEL_ENV !== "production" || + (!REMOVED_ANSWER_SLUGS.has(a.slug) && + !REMOVED_BENCH_SLUGS.has(a.benchmark)), + ) .sort((a, b) => a.slug.localeCompare(b.slug)); }); diff --git a/src/lib/removed-benches.ts b/src/lib/removed-benches.ts index d50e50bc..32de4dca 100644 --- a/src/lib/removed-benches.ts +++ b/src/lib/removed-benches.ts @@ -15,6 +15,16 @@ * Moving a bench to production = remove its slug here, bump the * bench-set cache keys in src/lib/spec.ts, ship dev to main. */ +/** + * Answer pages (answers/.yml) whose referenced benchmark is in + * REMOVED_BENCH_SLUGS. Same treatment: 410 on prod direct hits, dropped + * from the answers listing and sitemap on prod, normal on staging. + */ +export const REMOVED_ANSWER_SLUGS = new Set([ + "which-evm-aggregator-has-the-fastest-quote", + "which-solana-rpc-lands-the-most-transactions", +]); + export const REMOVED_BENCH_SLUGS = new Set([ // retired for good "bridge-revenue", diff --git a/src/lib/rpc-hub-stats.ts b/src/lib/rpc-hub-stats.ts index 8936a86f..8126bc52 100644 --- a/src/lib/rpc-hub-stats.ts +++ b/src/lib/rpc-hub-stats.ts @@ -28,6 +28,7 @@ import { filterSig, loadSpecsUncached, } from "@/lib/materialize/load"; +import { REMOVED_BENCH_SLUGS } from "@/lib/removed-benches"; import { readMaterialized, storeConfigured } from "@/lib/materialize/store"; import { chainLabelForSlug } from "@/lib/chains"; import type { Benchmark, ProviderResult } from "@/types/benchmark"; @@ -381,11 +382,40 @@ export async function buildRpcHubSnapshotFresh(): Promise * KV-only; the Vercel side never touches Prometheus. Null means the * page renders its "warming up" empty state. */ +/** Prod-only gate on the worker-written snapshot. The worker builds the + * cohort blob from the FULL spec set (it runs outside Vercel, no + * VERCEL_ENV), so staging-pipeline chains like monad-rpc reach the + * shared KV; drop them at read time and recompute the pivot + totals + * so provider aggregates only span the chains actually shown. */ +function gateSnapshotForProd(snap: RpcHubSnapshot): RpcHubSnapshot { + if (process.env.VERCEL_ENV !== "production") return snap; + const chains = snap.chains.filter((c) => !REMOVED_BENCH_SLUGS.has(c.slug)); + if (chains.length === snap.chains.length) return snap; + const providersPivot = buildPivot(chains); + return { + ...snap, + chains, + providersPivot, + totals: { + ...snap.totals, + chains: chains.length, + uniqueProviders: providersPivot.length, + }, + }; +} + async function fetchRpcHubRaw(): Promise { const snapshot = await readCohortSnapshot(RPC_HUB_KEY); - if (snapshot) return snapshot.data; + if (snapshot) return gateSnapshotForProd(snapshot.data); const fresh = await buildRpcHubSnapshotFresh().catch(() => null); - if (fresh && cohortSnapshotConfigured()) { + // No writeback on production: fresh is built from the prod-gated spec + // set there, and the KV blob is shared with staging (worker + preview + // deployments own its full-set content). + if ( + fresh && + cohortSnapshotConfigured() && + process.env.VERCEL_ENV !== "production" + ) { try { await writeCohortSnapshot(RPC_HUB_KEY, fresh); } catch (err) { @@ -401,10 +431,12 @@ async function fetchRpcHubRaw(): Promise { const fetchRpcHubCached = unstable_cache( fetchRpcHubRaw, + // v4: prod-only gate drops staging-pipeline chains from the shared + // worker blob at read time; env in key since the set differs per env. // v3: pivot rows gained medianSuccessPct/errors24h + per-chain // successPct/sampleSize; chains gained unresponsive[] rows. // v2: chains gained unresponsiveCount (unresponsive provider rows). - ["rpc-hub-cohort-v3"], + ["rpc-hub-cohort-v4", process.env.VERCEL_ENV === "production" ? "prod" : "all"], { revalidate: 60, tags: ["rpc-cohort"] }, ); diff --git a/src/middleware.ts b/src/middleware.ts index 544dcb0b..d2194e71 100644 --- a/src/middleware.ts +++ b/src/middleware.ts @@ -48,10 +48,14 @@ const CANONICAL_NO_QUERY = new Set([ // own module (not here) so the spec loader and the materialize worker // can import it without pulling next/server. Re-exported for the // sitemap, which historically imports it from "@/middleware". -import { REMOVED_BENCH_SLUGS } from "@/lib/removed-benches"; +import { + REMOVED_ANSWER_SLUGS, + REMOVED_BENCH_SLUGS, +} from "@/lib/removed-benches"; export { REMOVED_BENCH_SLUGS }; const BENCH_PATH = /^\/benchmarks\/([a-z0-9][a-z0-9-]{0,79})\/?$/; +const ANSWER_PATH = /^\/answers\/([a-z0-9][a-z0-9-]{0,79})\/?$/; // `/compare/-vs-` with both sides as standard provider slug // shapes (lowercase alphanumeric + hyphens). The `-vs-` delimiter is // matched literally; provider slugs themselves can contain hyphens @@ -70,7 +74,11 @@ export function middleware(req: NextRequest) { if (process.env.VERCEL_ENV === "production") { const m = pathname.match(BENCH_PATH); - if (m && REMOVED_BENCH_SLUGS.has(m[1])) { + const a = pathname.match(ANSWER_PATH); + if ( + (m && REMOVED_BENCH_SLUGS.has(m[1])) || + (a && REMOVED_ANSWER_SLUGS.has(a[1])) + ) { return new NextResponse( `410 Gone

410 Gone

This benchmark has been retired. See the current catalog.

`, { status: 410, headers: { "Content-Type": "text/html; charset=utf-8" } }, @@ -111,6 +119,7 @@ export const config = { "/api/freshness", "/api/openapi.json", "/benchmarks/:slug*", + "/answers/:slug*", "/compare/:slug*", ], };