ship: prod gate follow-up (rpc hub snapshot + answers)

infrastructure/monitoring/prometheus/alert_rules.yml

@@ -118,3 +118,50 @@ groups:
         annotations:
           summary: '{{ if eq $externalLabels.environment "staging" }}[STAGING] {{ end }}Latency monitor service is down'
           description: "The aggregator latency monitor has been down for 2 minutes. No metrics are being collected."
+
+  - name: bench_coverage_alerts
+    interval: 60s
+    rules:
+      # RPC provider error-rate (no-key + keyed): the 1RPC/dRPC per-IP
+      # throttling incident (2026-07-08) ran for days unalerted. Fires
+      # when a (provider, chain) cell errors on >50% of calls for 1h.
+      - alert: RpcProviderErrorRate
+        expr: |
+          (
+            sum by (provider, chain) (rate(rpc_call_total{result!="ok"}[30m]))
+            / sum by (provider, chain) (rate(rpc_call_total[30m]))
+          ) > 0.5
+        for: 1h
+        labels:
+          severity: warning
+          alert_type: rpc_error_rate
+          app: rpc_benches
+        annotations:
+          summary: '{{ $labels.provider }} erroring on {{ $labels.chain }} ({{ $value | humanizePercentage }})'
+          description: "More than half of probes to {{ $labels.provider }} on {{ $labels.chain }} failed over the last 30m, sustained 1h. Rate limit, dead endpoint, or our own footprint (check probe cadence before blaming the provider)."
+
+      # Keyed free-tier quota guard approaching: the guard pauses at 90%,
+      # this warns at 85% so a human can react before data gaps appear.
+      - alert: KeyedQuotaNearExhaustion
+        expr: rpc_keyed_quota_used_ratio > 0.85
+        for: 10m
+        labels:
+          severity: warning
+          alert_type: quota
+          app: rpc_benches
+        annotations:
+          summary: 'Keyed RPC quota {{ $labels.provider }} at {{ $value | humanizePercentage }} of monthly budget'
+          description: "The 90% guard will pause probing soon; leaderboard gaps follow. Consider a cadence reduction or a bigger plan."
+
+      # Aggregator WS silently down: connection-state gauge from the
+      # reconnect instrumentation (PR #950).
+      - alert: AggregatorWSDisconnected
+        expr: ws_connected == 0
+        for: 10m
+        labels:
+          severity: warning
+          alert_type: ws_state
+          app: aggregator_latency_monitor
+        annotations:
+          summary: '{{ $labels.aggregator }} WebSocket disconnected ({{ $labels.region }})'
+          description: "ws_connected has been 0 for 10 minutes; reconnect loop is running but not succeeding. Check JWT/cookie/proxy for codex, API key for mobula."

src/lib/answers.ts

@@ -17,6 +17,10 @@ import { cache } from "react";
 import yaml from "js-yaml";
 import { z } from "zod";
 import { loadBenchmark } from "@/lib/spec";
+import {
+  REMOVED_ANSWER_SLUGS,
+  REMOVED_BENCH_SLUGS,
+} from "@/lib/removed-benches";
 import type { Benchmark } from "@/types/benchmark";
 
 const ANSWERS_DIR = path.join(process.cwd(), "answers");
@@ -85,6 +89,15 @@ export const loadAllAnswers = cache(async (): Promise<Answer[]> => {
   );
   return parsed
     .filter((a): a is Answer => a !== null && a.status === "live")
+    // Prod-only gate: answers built on staging-pipeline benches never
+    // reach the prod listing, sitemap or tag clouds. Direct URL hits
+    // get a 410 from middleware.
+    .filter(
+      (a) =>
+        process.env.VERCEL_ENV !== "production" ||
+        (!REMOVED_ANSWER_SLUGS.has(a.slug) &&
+          !REMOVED_BENCH_SLUGS.has(a.benchmark)),
+    )
     .sort((a, b) => a.slug.localeCompare(b.slug));
 });
 

src/lib/removed-benches.ts

@@ -15,6 +15,16 @@
  * Moving a bench to production = remove its slug here, bump the
  * bench-set cache keys in src/lib/spec.ts, ship dev to main.
  */
+/**
+ * Answer pages (answers/<slug>.yml) whose referenced benchmark is in
+ * REMOVED_BENCH_SLUGS. Same treatment: 410 on prod direct hits, dropped
+ * from the answers listing and sitemap on prod, normal on staging.
+ */
+export const REMOVED_ANSWER_SLUGS = new Set([
+  "which-evm-aggregator-has-the-fastest-quote",
+  "which-solana-rpc-lands-the-most-transactions",
+]);
+
 export const REMOVED_BENCH_SLUGS = new Set([
   // retired for good
   "bridge-revenue",

src/lib/rpc-hub-stats.ts

@@ -28,6 +28,7 @@ import {
   filterSig,
   loadSpecsUncached,
 } from "@/lib/materialize/load";
+import { REMOVED_BENCH_SLUGS } from "@/lib/removed-benches";
 import { readMaterialized, storeConfigured } from "@/lib/materialize/store";
 import { chainLabelForSlug } from "@/lib/chains";
 import type { Benchmark, ProviderResult } from "@/types/benchmark";
@@ -381,11 +382,40 @@ export async function buildRpcHubSnapshotFresh(): Promise<RpcHubSnapshot | null>
  * KV-only; the Vercel side never touches Prometheus. Null means the
  * page renders its "warming up" empty state.
  */
+/** Prod-only gate on the worker-written snapshot. The worker builds the
+ *  cohort blob from the FULL spec set (it runs outside Vercel, no
+ *  VERCEL_ENV), so staging-pipeline chains like monad-rpc reach the
+ *  shared KV; drop them at read time and recompute the pivot + totals
+ *  so provider aggregates only span the chains actually shown. */
+function gateSnapshotForProd(snap: RpcHubSnapshot): RpcHubSnapshot {
+  if (process.env.VERCEL_ENV !== "production") return snap;
+  const chains = snap.chains.filter((c) => !REMOVED_BENCH_SLUGS.has(c.slug));
+  if (chains.length === snap.chains.length) return snap;
+  const providersPivot = buildPivot(chains);
+  return {
+    ...snap,
+    chains,
+    providersPivot,
+    totals: {
+      ...snap.totals,
+      chains: chains.length,
+      uniqueProviders: providersPivot.length,
+    },
+  };
+}
+
 async function fetchRpcHubRaw(): Promise<RpcHubSnapshot | null> {
   const snapshot = await readCohortSnapshot<RpcHubSnapshot>(RPC_HUB_KEY);
-  if (snapshot) return snapshot.data;
+  if (snapshot) return gateSnapshotForProd(snapshot.data);
   const fresh = await buildRpcHubSnapshotFresh().catch(() => null);
-  if (fresh && cohortSnapshotConfigured()) {
+  // No writeback on production: fresh is built from the prod-gated spec
+  // set there, and the KV blob is shared with staging (worker + preview
+  // deployments own its full-set content).
+  if (
+    fresh &&
+    cohortSnapshotConfigured() &&
+    process.env.VERCEL_ENV !== "production"
+  ) {
     try {
       await writeCohortSnapshot(RPC_HUB_KEY, fresh);
     } catch (err) {
@@ -401,10 +431,12 @@ async function fetchRpcHubRaw(): Promise<RpcHubSnapshot | null> {
 
 const fetchRpcHubCached = unstable_cache(
   fetchRpcHubRaw,
+  // v4: prod-only gate drops staging-pipeline chains from the shared
+  // worker blob at read time; env in key since the set differs per env.
   // v3: pivot rows gained medianSuccessPct/errors24h + per-chain
   // successPct/sampleSize; chains gained unresponsive[] rows.
   // v2: chains gained unresponsiveCount (unresponsive provider rows).
-  ["rpc-hub-cohort-v3"],
+  ["rpc-hub-cohort-v4", process.env.VERCEL_ENV === "production" ? "prod" : "all"],
   { revalidate: 60, tags: ["rpc-cohort"] },
 );
 

src/middleware.ts

@@ -48,10 +48,14 @@ const CANONICAL_NO_QUERY = new Set([
 // own module (not here) so the spec loader and the materialize worker
 // can import it without pulling next/server. Re-exported for the
 // sitemap, which historically imports it from "@/middleware".
-import { REMOVED_BENCH_SLUGS } from "@/lib/removed-benches";
+import {
+  REMOVED_ANSWER_SLUGS,
+  REMOVED_BENCH_SLUGS,
+} from "@/lib/removed-benches";
 export { REMOVED_BENCH_SLUGS };
 
 const BENCH_PATH = /^\/benchmarks\/([a-z0-9][a-z0-9-]{0,79})\/?$/;
+const ANSWER_PATH = /^\/answers\/([a-z0-9][a-z0-9-]{0,79})\/?$/;
 // `/compare/<a>-vs-<b>` with both sides as standard provider slug
 // shapes (lowercase alphanumeric + hyphens). The `-vs-` delimiter is
 // matched literally; provider slugs themselves can contain hyphens
@@ -70,7 +74,11 @@ export function middleware(req: NextRequest) {
 
   if (process.env.VERCEL_ENV === "production") {
     const m = pathname.match(BENCH_PATH);
-    if (m && REMOVED_BENCH_SLUGS.has(m[1])) {
+    const a = pathname.match(ANSWER_PATH);
+    if (
+      (m && REMOVED_BENCH_SLUGS.has(m[1])) ||
+      (a && REMOVED_ANSWER_SLUGS.has(a[1]))
+    ) {
       return new NextResponse(
         `<!doctype html><html lang="en"><head><meta charset="utf-8"><title>410 Gone</title><meta name="robots" content="noindex"></head><body><h1>410 Gone</h1><p>This benchmark has been retired. See the <a href="/benchmarks">current catalog</a>.</p></body></html>`,
         { status: 410, headers: { "Content-Type": "text/html; charset=utf-8" } },
@@ -111,6 +119,7 @@ export const config = {
     "/api/freshness",
     "/api/openapi.json",
     "/benchmarks/:slug*",
+    "/answers/:slug*",
     "/compare/:slug*",
   ],
 };