diff --git a/benchmarks/aggregator-head-lag.yml b/benchmarks/aggregator-head-lag.yml index e72b0c81..acc2d4a1 100644 --- a/benchmarks/aggregator-head-lag.yml +++ b/benchmarks/aggregator-head-lag.yml @@ -83,6 +83,14 @@ source: https://github.com/ChainBench/OpenChainBench/tree/main/harnesses/aggrega prometheus: window: 24h + # Live-feed sanity gate. Sums event-arrival changes across every + # aggregator series in the last 15min. If this is 0, ALL our probes + # are silent (harness crash / Prom scrape failing / config drift) and + # the per-provider "Feed down" badges would be misleading, so the UI + # suppresses them. If > 0, at least one provider is actively receiving + # events from us, which means a provider showing 0 changes on its own + # is genuinely silent from Codex/Mobula/Gecko upstream. + probe_ok: sum(changes(head_lag_seconds[15m])) faq: - q: "Which crypto data API has the lowest latency right now?" @@ -154,6 +162,12 @@ providers: success: clamp_max(count_over_time(head_lag_seconds{aggregator="mobula"}[24h]) / 5760, 1) sample_size: sum(count_over_time(head_lag_seconds{aggregator="mobula"}[24h])) series: avg_over_time(head_lag_seconds{aggregator="mobula"}[1h]) * 1000 + # Total gauge-value changes on this aggregator across chains and + # regions in the last 15min. Non-zero = fresh events arrived; 0 = + # every subscription has been silent for 15+ minutes (the gauge is + # frozen at its last value). Threshold is > 0 rather than a rate + # so quiet chains at night don't false-flag the aggregate view. + live_activity: sum(changes(head_lag_seconds{aggregator="mobula"}[15m])) regions: - region: us-east p50: quantile_over_time(0.50, head_lag_seconds{aggregator="mobula", region="us-east"}[24h]) * 1000 @@ -177,6 +191,7 @@ providers: success: clamp_max(count_over_time(head_lag_seconds{aggregator="codex"}[24h]) / 5760, 1) sample_size: sum(count_over_time(head_lag_seconds{aggregator="codex"}[24h])) series: avg_over_time(head_lag_seconds{aggregator="codex"}[1h]) * 1000 + live_activity: sum(changes(head_lag_seconds{aggregator="codex"}[15m])) regions: - region: us-east p50: quantile_over_time(0.50, head_lag_seconds{aggregator="codex", region="us-east"}[24h]) * 1000 @@ -200,6 +215,7 @@ providers: success: clamp_max(count_over_time(head_lag_seconds{aggregator="geckoterminal"}[24h]) / 5760, 1) sample_size: sum(count_over_time(head_lag_seconds{aggregator="geckoterminal"}[24h])) series: avg_over_time(head_lag_seconds{aggregator="geckoterminal"}[1h]) * 1000 + live_activity: sum(changes(head_lag_seconds{aggregator="geckoterminal"}[15m])) regions: - region: us-east p50: quantile_over_time(0.50, head_lag_seconds{aggregator="geckoterminal", region="us-east"}[24h]) * 1000 diff --git a/src/app/benchmarks/[slug]/page.tsx b/src/app/benchmarks/[slug]/page.tsx index 14215e18..702c6b7f 100644 --- a/src/app/benchmarks/[slug]/page.tsx +++ b/src/app/benchmarks/[slug]/page.tsx @@ -613,6 +613,37 @@ export default async function BenchmarkPage({ )} + {/* Live-feed outage banner. Renders when the spec declared + live_activity queries and one or more providers came back + "down" (short-window activity = 0, bench-level probe_ok + confirmed our end is fine). Placed above the fold so a reader + landing on the page during an incident sees the caveat before + reading last-known percentiles as current truth. */} + {(() => { + const downProviders = benchmark.results.filter( + (r) => r.liveStatus === "down" + ); + if (downProviders.length === 0) return null; + const names = downProviders.map((r) => r.name).join(", "); + return ( +
+

+ Live feed silent: {names} +

+

+ No new events received from{" "} + {downProviders.length === 1 ? "this feed" : "these feeds"} in + the last several minutes. Percentiles and rankings shown are + the last-known values from the 24-hour window and update as + soon as fresh events resume. +

+
+ ); + })()} + {/* SEO-tuned intro paragraph rendered server-side under the H1 so long-tail query phrases land in the first ~200 words crawlers weight heavily. Optional - omitted when the YAML doesn't set it. */} diff --git a/src/components/ledger-table.tsx b/src/components/ledger-table.tsx index b98c53de..057e6cd0 100644 --- a/src/components/ledger-table.tsx +++ b/src/components/ledger-table.tsx @@ -781,6 +781,14 @@ function Row({ )} + {!isMuted && r.liveStatus === "down" && ( + + + + Feed down + + + )} {!isMuted && r.dataConfidence === "low" && ( ): Spec success: inject(p.queries.success), sample_size: inject(p.queries.sample_size), series: inject(p.queries.series), + live_activity: inject(p.queries.live_activity), regions: p.queries.regions?.map((r) => ({ ...r, p50: inject(r.p50), @@ -546,6 +547,9 @@ function applyDimensionsToSpec(spec: Spec, labels: Record): Spec } : p.queries, })), + prometheus: spec.prometheus + ? { ...spec.prometheus, probe_ok: inject(spec.prometheus.probe_ok) } + : spec.prometheus, }; } @@ -654,6 +658,19 @@ async function tryLoadLive( const sevenDaysSec = 7 * 86_400; const thirtyDaysSec = 30 * 86_400; + // Bench-level "is our end fine" gate. Fetched once per sweep; feeds + // every provider's liveStatus verdict below so a broken harness / Prom + // scrape can't fake-flag every provider as down at once. Absent when + // the spec doesn't declare probe_ok — in which case we trust each + // provider's live_activity unconditionally. + const probeOkQuery = spec.prometheus?.probe_ok; + const probeOk = probeOkQuery + ? await prom.scalar(probeOkQuery) + : null; + // Interpret: >0 or null-when-not-declared → trust per-provider verdicts. + // Explicit 0 (or NaN) → the probe itself is down; suppress all badges. + const trustLiveVerdicts = !probeOkQuery || (probeOk != null && probeOk > 0); + for (const p of spec.providers) { const q = p.queries; if (!q) return null; @@ -663,12 +680,13 @@ async function tryLoadLive( q.p90 ? prom.scalar(q.p90) : Promise.resolve(null), q.p99 ? prom.scalar(q.p99) : Promise.resolve(null), ]); - const [mean, success, sampleSize, slotP50, slotP99] = await Promise.all([ + const [mean, success, sampleSize, slotP50, slotP99, liveActivity] = await Promise.all([ q.mean ? prom.scalar(q.mean) : Promise.resolve(null), q.success ? prom.scalar(q.success) : Promise.resolve(null), q.sample_size ? prom.scalar(q.sample_size) : Promise.resolve(null), q.slot_p50 ? prom.scalar(q.slot_p50) : Promise.resolve(null), q.slot_p99 ? prom.scalar(q.slot_p99) : Promise.resolve(null), + q.live_activity ? prom.scalar(q.live_activity) : Promise.resolve(null), ]); // One retry on the load-bearing percentiles. A null here is either @@ -734,6 +752,22 @@ async function tryLoadLive( continue; } + // liveStatus: only computed when the spec declares live_activity. + // "unknown" wins whenever we can't tell (probe_ok says our side is + // broken, or the activity query returned no sample) — never falls + // through to "down" on ambiguous data, since a false red pill on + // a live provider is worse than a missing pill on a real outage. + let liveStatus: "healthy" | "down" | "unknown" | undefined; + if (q.live_activity) { + if (!trustLiveVerdicts || liveActivity == null) { + liveStatus = "unknown"; + } else if (liveActivity > 0) { + liveStatus = "healthy"; + } else { + liveStatus = "down"; + } + } + liveResults.push({ name: p.name, slug: p.slug, @@ -750,6 +784,7 @@ async function tryLoadLive( secondary: p.secondary, query: q.p50, formula: p.formula, + liveStatus, }); if (q.series) { diff --git a/src/lib/snapshot.ts b/src/lib/snapshot.ts index 89e658bd..bc397e3d 100644 --- a/src/lib/snapshot.ts +++ b/src/lib/snapshot.ts @@ -86,6 +86,7 @@ const ProviderResultSchema = z.object({ meta: StalenessMetaSchema.optional(), query: z.string().optional(), formula: z.string().optional(), + liveStatus: z.enum(["healthy", "down", "unknown"]).optional(), }); const RegionPointSchema = z.object({ diff --git a/src/lib/spec-schema.ts b/src/lib/spec-schema.ts index 75535ac0..1bed87b5 100644 --- a/src/lib/spec-schema.ts +++ b/src/lib/spec-schema.ts @@ -80,6 +80,14 @@ const queries = z success: promql.optional(), sample_size: promql.optional(), series: promql.optional(), + /** Short-window "is this provider's live feed producing new events + * right now" probe. Instant query returning a scalar count > 0 when + * fresh events arrived in the past few minutes, 0 when the source + * is silent. Distinct from `success` (24h rolling reliability), + * which is too slow to move on a 30-60 min outage. Rendered as a + * "Feed down" pill + top-of-page banner when 0 and probe_ok + * confirms our end is fine. */ + live_activity: promql.optional(), /** Optional slot-level companion queries. Solana-native benches set * these to surface slot_delta p50/p99 alongside the ms columns. The * ms numbers are wall-clock derived; slot_delta is the canonical @@ -309,6 +317,15 @@ export const SpecSchema = z .string() .regex(/^[a-zA-Z_:][a-zA-Z0-9_:]*$/, "Must be a bare metric name") .optional(), + /** Bench-level sanity check for the per-provider `live_activity` + * probe: an instant query that must be > 0 for the "Feed down" + * UI to trust its per-provider verdicts. Meant to answer "are + * ANY of our probes still emitting". Zero means our end + * (harness / Prom scrape) is broken, not the provider — so the + * UI suppresses all per-provider down badges to avoid falsely + * blaming every source at once. When omitted, per-provider + * activity is trusted unconditionally. */ + probe_ok: promql.optional(), }) .optional(), diff --git a/src/types/benchmark.ts b/src/types/benchmark.ts index 47d29a00..59db3d02 100644 --- a/src/types/benchmark.ts +++ b/src/types/benchmark.ts @@ -94,6 +94,20 @@ export type ProviderResult = { * headline value is computed. Rendered as the leaderboard-row * hover tooltip. Authored per-bench in YAML (provider.formula). */ formula?: string; + /** Short-window liveness verdict derived at load time from the spec's + * `queries.live_activity` scalar and the bench-level `probe_ok` + * gate. Only populated when the spec declares those queries. + * - "healthy": recent events arrived (activity > 0). + * - "down": no recent events AND probe_ok confirmed our end is + * fine — the provider's live feed is silent, values + * shown are last-known. Renderers should badge it. + * - "unknown": probe_ok reports our side is broken (harness or + * Prom scrape), OR the activity query itself + * returned no sample. Suppress the badge either way + * so a probe hiccup can't fake-flag every provider. + * Absent when the spec doesn't declare live_activity; UI must + * behave identically to today for those benches. */ + liveStatus?: "healthy" | "down" | "unknown"; }; /** One provider's standing inside a (chain, region) ranking cell. */