diff --git a/benchmarks/aggregator-head-lag.yml b/benchmarks/aggregator-head-lag.yml index e72b0c81..d9fade57 100644 --- a/benchmarks/aggregator-head-lag.yml +++ b/benchmarks/aggregator-head-lag.yml @@ -83,6 +83,14 @@ source: https://github.com/ChainBench/OpenChainBench/tree/main/harnesses/aggrega prometheus: window: 24h + # Live-feed sanity gate. Sums event-arrival changes across every + # aggregator series in the last 15min. If this is 0, ALL our probes + # are silent (harness crash / Prom scrape failing / config drift) and + # the per-provider "Feed down" badges would be misleading, so the UI + # suppresses them. If > 0, at least one provider is actively receiving + # events from us, which means a provider showing 0 changes on its own + # is genuinely silent from Codex/Mobula/Gecko upstream. + probe_ok: sum(changes(head_lag_seconds[15m])) faq: - q: "Which crypto data API has the lowest latency right now?" @@ -153,17 +161,32 @@ providers: mean: avg_over_time(head_lag_seconds{aggregator="mobula"}[24h]) * 1000 success: clamp_max(count_over_time(head_lag_seconds{aggregator="mobula"}[24h]) / 5760, 1) sample_size: sum(count_over_time(head_lag_seconds{aggregator="mobula"}[24h])) - series: avg_over_time(head_lag_seconds{aggregator="mobula"}[1h]) * 1000 + # Hybrid `unless` guard: drop the sample when the gauge saw no + # fresh event in the last 15 min AND we have enough scrapes in + # that window to trust the changes() reading (>5 samples). The + # count_over_time gate skips false-drops on backfilled data + # that Prom compacted to ~1 sample per hour — on that vintage + # changes([15m]) reads 0 by construction, without the gate the + # 30D view lost its first ~8 days to a fake wall-to-wall gap. + # Modern data at 15s cadence carries ~60 samples per 15 min so + # the guard still fires the moment a WS goes silent. + series: avg_over_time(head_lag_seconds{aggregator="mobula"}[1h]) * 1000 unless (changes(head_lag_seconds{aggregator="mobula"}[15m]) == 0 and count_over_time(head_lag_seconds{aggregator="mobula"}[15m]) > 5) + # Total gauge-value changes on this aggregator across chains and + # regions in the last 15min. Non-zero = fresh events arrived; 0 = + # every subscription has been silent for 15+ minutes (the gauge is + # frozen at its last value). Threshold is > 0 rather than a rate + # so quiet chains at night don't false-flag the aggregate view. + live_activity: sum(changes(head_lag_seconds{aggregator="mobula"}[15m])) regions: - region: us-east p50: quantile_over_time(0.50, head_lag_seconds{aggregator="mobula", region="us-east"}[24h]) * 1000 - series: avg_over_time(head_lag_seconds{aggregator="mobula", region="us-east"}[1h]) * 1000 + series: avg_over_time(head_lag_seconds{aggregator="mobula", region="us-east"}[1h]) * 1000 unless (changes(head_lag_seconds{aggregator="mobula", region="us-east"}[15m]) == 0 and count_over_time(head_lag_seconds{aggregator="mobula", region="us-east"}[15m]) > 5) - region: eu-west p50: quantile_over_time(0.50, head_lag_seconds{aggregator="mobula", region="eu-west"}[24h]) * 1000 - series: avg_over_time(head_lag_seconds{aggregator="mobula", region="eu-west"}[1h]) * 1000 + series: avg_over_time(head_lag_seconds{aggregator="mobula", region="eu-west"}[1h]) * 1000 unless (changes(head_lag_seconds{aggregator="mobula", region="eu-west"}[15m]) == 0 and count_over_time(head_lag_seconds{aggregator="mobula", region="eu-west"}[15m]) > 5) - region: ap-southeast p50: quantile_over_time(0.50, head_lag_seconds{aggregator="mobula", region="sgp"}[24h]) * 1000 - series: avg_over_time(head_lag_seconds{aggregator="mobula", region="sgp"}[1h]) * 1000 + series: avg_over_time(head_lag_seconds{aggregator="mobula", region="sgp"}[1h]) * 1000 unless (changes(head_lag_seconds{aggregator="mobula", region="sgp"}[15m]) == 0 and count_over_time(head_lag_seconds{aggregator="mobula", region="sgp"}[15m]) > 5) - slug: codex name: Codex @@ -176,17 +199,18 @@ providers: mean: avg_over_time(head_lag_seconds{aggregator="codex"}[24h]) * 1000 success: clamp_max(count_over_time(head_lag_seconds{aggregator="codex"}[24h]) / 5760, 1) sample_size: sum(count_over_time(head_lag_seconds{aggregator="codex"}[24h])) - series: avg_over_time(head_lag_seconds{aggregator="codex"}[1h]) * 1000 + series: avg_over_time(head_lag_seconds{aggregator="codex"}[1h]) * 1000 unless (changes(head_lag_seconds{aggregator="codex"}[15m]) == 0 and count_over_time(head_lag_seconds{aggregator="codex"}[15m]) > 5) + live_activity: sum(changes(head_lag_seconds{aggregator="codex"}[15m])) regions: - region: us-east p50: quantile_over_time(0.50, head_lag_seconds{aggregator="codex", region="us-east"}[24h]) * 1000 - series: avg_over_time(head_lag_seconds{aggregator="codex", region="us-east"}[1h]) * 1000 + series: avg_over_time(head_lag_seconds{aggregator="codex", region="us-east"}[1h]) * 1000 unless (changes(head_lag_seconds{aggregator="codex", region="us-east"}[15m]) == 0 and count_over_time(head_lag_seconds{aggregator="codex", region="us-east"}[15m]) > 5) - region: eu-west p50: quantile_over_time(0.50, head_lag_seconds{aggregator="codex", region="eu-west"}[24h]) * 1000 - series: avg_over_time(head_lag_seconds{aggregator="codex", region="eu-west"}[1h]) * 1000 + series: avg_over_time(head_lag_seconds{aggregator="codex", region="eu-west"}[1h]) * 1000 unless (changes(head_lag_seconds{aggregator="codex", region="eu-west"}[15m]) == 0 and count_over_time(head_lag_seconds{aggregator="codex", region="eu-west"}[15m]) > 5) - region: ap-southeast p50: quantile_over_time(0.50, head_lag_seconds{aggregator="codex", region="sgp"}[24h]) * 1000 - series: avg_over_time(head_lag_seconds{aggregator="codex", region="sgp"}[1h]) * 1000 + series: avg_over_time(head_lag_seconds{aggregator="codex", region="sgp"}[1h]) * 1000 unless (changes(head_lag_seconds{aggregator="codex", region="sgp"}[15m]) == 0 and count_over_time(head_lag_seconds{aggregator="codex", region="sgp"}[15m]) > 5) - slug: geckoterminal name: GeckoTerminal @@ -199,15 +223,16 @@ providers: mean: avg_over_time(head_lag_seconds{aggregator="geckoterminal"}[24h]) * 1000 success: clamp_max(count_over_time(head_lag_seconds{aggregator="geckoterminal"}[24h]) / 5760, 1) sample_size: sum(count_over_time(head_lag_seconds{aggregator="geckoterminal"}[24h])) - series: avg_over_time(head_lag_seconds{aggregator="geckoterminal"}[1h]) * 1000 + series: avg_over_time(head_lag_seconds{aggregator="geckoterminal"}[1h]) * 1000 unless (changes(head_lag_seconds{aggregator="geckoterminal"}[15m]) == 0 and count_over_time(head_lag_seconds{aggregator="geckoterminal"}[15m]) > 5) + live_activity: sum(changes(head_lag_seconds{aggregator="geckoterminal"}[15m])) regions: - region: us-east p50: quantile_over_time(0.50, head_lag_seconds{aggregator="geckoterminal", region="us-east"}[24h]) * 1000 - series: avg_over_time(head_lag_seconds{aggregator="geckoterminal", region="us-east"}[1h]) * 1000 + series: avg_over_time(head_lag_seconds{aggregator="geckoterminal", region="us-east"}[1h]) * 1000 unless (changes(head_lag_seconds{aggregator="geckoterminal", region="us-east"}[15m]) == 0 and count_over_time(head_lag_seconds{aggregator="geckoterminal", region="us-east"}[15m]) > 5) - region: eu-west p50: quantile_over_time(0.50, head_lag_seconds{aggregator="geckoterminal", region="eu-west"}[24h]) * 1000 - series: avg_over_time(head_lag_seconds{aggregator="geckoterminal", region="eu-west"}[1h]) * 1000 + series: avg_over_time(head_lag_seconds{aggregator="geckoterminal", region="eu-west"}[1h]) * 1000 unless (changes(head_lag_seconds{aggregator="geckoterminal", region="eu-west"}[15m]) == 0 and count_over_time(head_lag_seconds{aggregator="geckoterminal", region="eu-west"}[15m]) > 5) - region: ap-southeast p50: quantile_over_time(0.50, head_lag_seconds{aggregator="geckoterminal", region="sgp"}[24h]) * 1000 - series: avg_over_time(head_lag_seconds{aggregator="geckoterminal", region="sgp"}[1h]) * 1000 + series: avg_over_time(head_lag_seconds{aggregator="geckoterminal", region="sgp"}[1h]) * 1000 unless (changes(head_lag_seconds{aggregator="geckoterminal", region="sgp"}[15m]) == 0 and count_over_time(head_lag_seconds{aggregator="geckoterminal", region="sgp"}[15m]) > 5) diff --git a/src/app/benchmarks/[slug]/page.tsx b/src/app/benchmarks/[slug]/page.tsx index 14215e18..702c6b7f 100644 --- a/src/app/benchmarks/[slug]/page.tsx +++ b/src/app/benchmarks/[slug]/page.tsx @@ -613,6 +613,37 @@ export default async function BenchmarkPage({ )} + {/* Live-feed outage banner. Renders when the spec declared + live_activity queries and one or more providers came back + "down" (short-window activity = 0, bench-level probe_ok + confirmed our end is fine). Placed above the fold so a reader + landing on the page during an incident sees the caveat before + reading last-known percentiles as current truth. */} + {(() => { + const downProviders = benchmark.results.filter( + (r) => r.liveStatus === "down" + ); + if (downProviders.length === 0) return null; + const names = downProviders.map((r) => r.name).join(", "); + return ( +
+

+ Live feed silent: {names} +

+

+ No new events received from{" "} + {downProviders.length === 1 ? "this feed" : "these feeds"} in + the last several minutes. Percentiles and rankings shown are + the last-known values from the 24-hour window and update as + soon as fresh events resume. +

+
+ ); + })()} + {/* SEO-tuned intro paragraph rendered server-side under the H1 so long-tail query phrases land in the first ~200 words crawlers weight heavily. Optional - omitted when the YAML doesn't set it. */} diff --git a/src/components/ledger-table.tsx b/src/components/ledger-table.tsx index f06ed03f..638f0b8c 100644 --- a/src/components/ledger-table.tsx +++ b/src/components/ledger-table.tsx @@ -786,6 +786,14 @@ function Row({ )} + {!isMuted && r.liveStatus === "down" && ( + + + + Feed down + + + )} {!isMuted && r.dataConfidence === "low" && ( void; onToggleExclude?: (slug: string) => void; onResetExcluded?: () => void; + /** Only render the downtime bands + pill labels when the parent bench + * opted in (spec declared live_activity). Otherwise natural nulls in + * any bench's series24h would fire a "DATA MISSING" pill on charts + * that never asked for the feature (observed on rpc-reliability). */ + showDowntime?: boolean; }; export function Chart({ @@ -33,6 +38,7 @@ export function Chart({ onZoom, onToggleExclude, onResetExcluded, + showDowntime = false, }: ChartProps) { const W = 1000; const H = 360; @@ -386,6 +392,11 @@ export function Chart({ {/* X tick labels */} + {/* Downtime bands. Rendered before the lines so the stroke stays + on top of the highlight. Contiguous null-buckets (from spec + `unless changes == 0`) become visible red columns. */} + {showDowntime && } + {/* Areas + lines */} diff --git a/src/components/time-series-chart/index.tsx b/src/components/time-series-chart/index.tsx index 6e105ae1..519ae13e 100644 --- a/src/components/time-series-chart/index.tsx +++ b/src/components/time-series-chart/index.tsx @@ -475,6 +475,7 @@ export function TimeSeriesChart({ onZoom={setZoom} onToggleExclude={toggle} onResetExcluded={excluded.size > 0 ? reset : undefined} + showDowntime={benchmark.results.some((r) => r.liveStatus != null)} /> )} diff --git a/src/components/time-series-chart/series.tsx b/src/components/time-series-chart/series.tsx index 22e685cd..ca2ffa77 100644 --- a/src/components/time-series-chart/series.tsx +++ b/src/components/time-series-chart/series.tsx @@ -16,6 +16,177 @@ type SeriesPathsProps = { unit: string; }; +type DowntimeBandsProps = { + drawn: DrawnLine[]; + padT: number; + innerH: number; +}; + +/** Colored rectangles that highlight the exact window each line was + * silent (contiguous gap indices), plus a high-contrast pill label + * above each band naming the provider that dropped off. Pills stack + * vertically when multiple providers went silent at the same time so + * simultaneous outages read as separate lines instead of colliding + * as unreadable overlapping text. Rendered under SeriesPaths so the + * line stroke stays on top; skipped for excluded (legend-toggled) + * providers so their bands disappear with the line. */ +export function DowntimeBands({ drawn, padT, innerH }: DowntimeBandsProps) { + // Collect every band across every visible provider before rendering. + // Two-pass so we can assign each band a vertical stack slot based on + // how many EARLIER-placed bands its X-range overlaps — labels then + // don't collide when three providers went dark in the same window. + type Band = { + slug: string; + name: string; + color: string; + x: number; + w: number; + slot: number; + }; + const all: Band[] = []; + for (const d of drawn) { + if (d.excluded) continue; + // Only count gap runs that start AFTER the first observed sample. + // A leading run of nulls (Prom retention shorter than the visible + // window, harness started mid-range, or provider added recently) + // is "we didn't measure yet", not "provider was down for a week". + // Without this guard the 30D view on aggregator-head-lag rendered + // a chart-wide band during Prom's initial fill period. + let seenData = false; + let runStart: number | null = null; + const push = (endX: number) => { + if (runStart == null) return; + if (!seenData) { + runStart = null; + return; + } + const startX = d.pts[runStart].x; + all.push({ + slug: d.slug, + name: d.name, + color: d.color, + x: startX, + w: Math.max(6, endX - startX), + slot: 0, + }); + runStart = null; + }; + for (let i = 0; i < d.pts.length; i++) { + const p = d.pts[i]; + if (p.gap) { + if (runStart == null) runStart = i; + } else { + push(d.pts[i].x); + seenData = true; + } + } + // Trailing gap that runs to the current time — meaningful only + // when the series had prior data (same seenData guard). + if (runStart != null) push(d.pts[d.pts.length - 1].x); + } + if (all.length === 0) return null; + + // Slot assignment: for each band, pick the smallest slot not used + // by another band whose X-range overlaps this one. Naive O(n²) is + // fine — a chart has at most ~20 bands in the pathological case. + all.sort((a, b) => a.x - b.x); + for (let i = 0; i < all.length; i++) { + const b = all[i]; + const used = new Set(); + for (let j = 0; j < i; j++) { + const p = all[j]; + if (p.x < b.x + b.w && p.x + p.w > b.x) used.add(p.slot); + } + let slot = 0; + while (used.has(slot)) slot++; + b.slot = slot; + } + + // Pill geometry constants. Pills sit ABOVE the plot area so they + // never occlude data. Small padding on the container's padT keeps + // them within the chart frame. + const PILL_H = 15; + const PILL_GAP = 3; + + return ( + + {/* Bands first (behind), then all pills on top so no band tint + can bleed onto a label from a taller-slot pill. */} + {all.map((b, i) => ( + + + + + + ))} + {all.map((b, i) => { + const cx = b.x + b.w / 2; + // "NAME DATA MISSING" width estimate: ~5.5 px per char plus + // 14 px of horizontal padding (7 each side). "DATA MISSING" + // stays neutral: absence of samples could be the provider + // going down, our harness losing its WebSocket, or a Prom + // scrape failure — the pill doesn't blame either side. + const label = `${b.name.toUpperCase()} DATA MISSING`; + const pillW = Math.max(80, label.length * 5.5 + 14); + const pillY = padT + 2 + b.slot * (PILL_H + PILL_GAP); + const pillX = Math.max(0, cx - pillW / 2); + return ( + + + + {label} + + + ); + })} + + ); +} + export function SeriesPaths({ drawn, unit }: SeriesPathsProps) { return ( <> diff --git a/src/lib/materialize/load.ts b/src/lib/materialize/load.ts index 6abd6d1d..150e8828 100644 --- a/src/lib/materialize/load.ts +++ b/src/lib/materialize/load.ts @@ -367,6 +367,58 @@ function activeFilterLabels(opts: BenchmarkFilters): Record { return out; } + +/** + * Nullify every bucket in `coarse` whose covering time-range overlaps + * a null bucket in `fine`. Both series are dense right-anchored ("now" + * = last index), so index i in a series of length N maps to time-ago + * (N - 1 - i) / (N - 1) of the window. Each contiguous null RUN on the + * fine grid nulls every coarse bucket its range straddles, so a short + * outage detected on the 24h grid stays visible as a contiguous band + * on the 7d and 30d grids (not two neighbouring pills). + * + * No-op when either input is missing or too short. Mutates `coarse`. + * Exported for tests. + */ +export function propagateNullsToCoarser( + fine: (number | null)[] | null | undefined, + fineWindowSec: number, + coarse: (number | null)[] | null | undefined, + coarseWindowSec: number, +): void { + if (!fine || !coarse || fine.length < 2 || coarse.length < 2) return; + if (fineWindowSec <= 0 || coarseWindowSec <= 0) return; + const fineStep = fineWindowSec / (fine.length - 1); + const coarseStep = coarseWindowSec / (coarse.length - 1); + // Right-anchored: last index = now. Map a fine index to the same + // ABSOLUTE time-ago on the coarse grid (both grids share "now"; the + // step sizes are what differ, not the reference point). Previous + // math scaled by the fine window fraction, which shrank a 7 h ago + // outage down to 40 h ago once projected onto the 7 d grid. + const toIdxCoarse = (i: number): number => { + const secondsAgo = (fine.length - 1 - i) * fineStep; + return (coarse.length - 1) - secondsAgo / coarseStep; + }; + let runStart: number | null = null; + const closeRun = (endExclusive: number) => { + if (runStart == null) return; + const startX = toIdxCoarse(runStart); + const endX = toIdxCoarse(endExclusive); + const lo = Math.max(0, Math.floor(Math.min(startX, endX))); + const hi = Math.min(coarse.length - 1, Math.ceil(Math.max(startX, endX))); + for (let k = lo; k <= hi; k++) coarse[k] = null; + runStart = null; + }; + for (let i = 0; i < fine.length; i++) { + if (fine[i] === null) { + if (runStart == null) runStart = i; + } else { + closeRun(i); + } + } + closeRun(fine.length); +} + /** * Run the spec's `rank_matrix_query` (one instant vector with a sample per * (provider[, chain][, region])) and fold it into full per-cell rankings. @@ -539,6 +591,7 @@ function applyDimensionsToSpec(spec: Spec, labels: Record): Spec success: inject(p.queries.success), sample_size: inject(p.queries.sample_size), series: inject(p.queries.series), + live_activity: inject(p.queries.live_activity), regions: p.queries.regions?.map((r) => ({ ...r, p50: inject(r.p50), @@ -547,6 +600,9 @@ function applyDimensionsToSpec(spec: Spec, labels: Record): Spec } : p.queries, })), + prometheus: spec.prometheus + ? { ...spec.prometheus, probe_ok: inject(spec.prometheus.probe_ok) } + : spec.prometheus, }; } @@ -655,6 +711,19 @@ async function tryLoadLive( const sevenDaysSec = 7 * 86_400; const thirtyDaysSec = 30 * 86_400; + // Bench-level "is our end fine" gate. Fetched once per sweep; feeds + // every provider's liveStatus verdict below so a broken harness / Prom + // scrape can't fake-flag every provider as down at once. Absent when + // the spec doesn't declare probe_ok — in which case we trust each + // provider's live_activity unconditionally. + const probeOkQuery = spec.prometheus?.probe_ok; + const probeOk = probeOkQuery + ? await prom.scalar(probeOkQuery) + : null; + // Interpret: >0 or null-when-not-declared → trust per-provider verdicts. + // Explicit 0 (or NaN) → the probe itself is down; suppress all badges. + const trustLiveVerdicts = !probeOkQuery || (probeOk != null && probeOk > 0); + for (const p of spec.providers) { const q = p.queries; if (!q) return null; @@ -664,12 +733,13 @@ async function tryLoadLive( q.p90 ? prom.scalar(q.p90) : Promise.resolve(null), q.p99 ? prom.scalar(q.p99) : Promise.resolve(null), ]); - const [mean, success, sampleSize, slotP50, slotP99] = await Promise.all([ + const [mean, success, sampleSize, slotP50, slotP99, liveActivity] = await Promise.all([ q.mean ? prom.scalar(q.mean) : Promise.resolve(null), q.success ? prom.scalar(q.success) : Promise.resolve(null), q.sample_size ? prom.scalar(q.sample_size) : Promise.resolve(null), q.slot_p50 ? prom.scalar(q.slot_p50) : Promise.resolve(null), q.slot_p99 ? prom.scalar(q.slot_p99) : Promise.resolve(null), + q.live_activity ? prom.scalar(q.live_activity) : Promise.resolve(null), ]); // One retry on the load-bearing percentiles. A null here is either @@ -735,6 +805,22 @@ async function tryLoadLive( continue; } + // liveStatus: only computed when the spec declares live_activity. + // "unknown" wins whenever we can't tell (probe_ok says our side is + // broken, or the activity query returned no sample) — never falls + // through to "down" on ambiguous data, since a false red pill on + // a live provider is worse than a missing pill on a real outage. + let liveStatus: "healthy" | "down" | "unknown" | undefined; + if (q.live_activity) { + if (!trustLiveVerdicts || liveActivity == null) { + liveStatus = "unknown"; + } else if (liveActivity > 0) { + liveStatus = "healthy"; + } else { + liveStatus = "down"; + } + } + liveResults.push({ name: p.name, slug: p.slug, @@ -751,6 +837,7 @@ async function tryLoadLive( secondary: p.secondary, query: q.p50, formula: p.formula, + liveStatus, }); if (q.series) { @@ -759,6 +846,15 @@ async function tryLoadLive( prom.series(q.series, sevenDaysSec, 84), prom.series(q.series, thirtyDaysSec, 60), ]); + // Propagate short outages captured on the fine 24h grid up to + // the coarser 7d/30d grids. Only when the bench opted in via + // `live_activity` — other benches carry natural nulls (sparse + // scrapes, backfill edges) that shouldn't fire a "DATA MISSING" + // pill on the chart. + if (q.live_activity && s24 && s24.length > 0) { + propagateNullsToCoarser(s24, winSec, s7, sevenDaysSec); + propagateNullsToCoarser(s24, winSec, s30, thirtyDaysSec); + } if (s24 && s24.length > 0) series24h[p.slug] = s24; if (s7 && s7.length > 0) series7d[p.slug] = s7; if (s30 && s30.length > 0) series30d[p.slug] = s30; @@ -784,6 +880,11 @@ async function tryLoadLive( ); regions[p.slug] = points.map(({ region: rg, p50: v }) => ({ region: rg, p50: v })); for (const pt of points) { + // Same short-outage propagation as the global series above. + if (q.live_activity && pt.series24 && pt.series24.length > 0) { + propagateNullsToCoarser(pt.series24, winSec, pt.series7, sevenDaysSec); + propagateNullsToCoarser(pt.series24, winSec, pt.series30, thirtyDaysSec); + } if (pt.series24 && pt.series24.length > 0) { (seriesByRegion24h[p.slug] ??= {})[pt.region] = pt.series24; } diff --git a/src/lib/snapshot.ts b/src/lib/snapshot.ts index 89e658bd..bc397e3d 100644 --- a/src/lib/snapshot.ts +++ b/src/lib/snapshot.ts @@ -86,6 +86,7 @@ const ProviderResultSchema = z.object({ meta: StalenessMetaSchema.optional(), query: z.string().optional(), formula: z.string().optional(), + liveStatus: z.enum(["healthy", "down", "unknown"]).optional(), }); const RegionPointSchema = z.object({ diff --git a/src/lib/spec-schema.ts b/src/lib/spec-schema.ts index 2aa96b9f..36bbcd87 100644 --- a/src/lib/spec-schema.ts +++ b/src/lib/spec-schema.ts @@ -80,6 +80,14 @@ const queries = z success: promql.optional(), sample_size: promql.optional(), series: promql.optional(), + /** Short-window "is this provider's live feed producing new events + * right now" probe. Instant query returning a scalar count > 0 when + * fresh events arrived in the past few minutes, 0 when the source + * is silent. Distinct from `success` (24h rolling reliability), + * which is too slow to move on a 30-60 min outage. Rendered as a + * "Feed down" pill + top-of-page banner when 0 and probe_ok + * confirms our end is fine. */ + live_activity: promql.optional(), /** Optional slot-level companion queries. Solana-native benches set * these to surface slot_delta p50/p99 alongside the ms columns. The * ms numbers are wall-clock derived; slot_delta is the canonical @@ -309,6 +317,15 @@ export const SpecSchema = z .string() .regex(/^[a-zA-Z_:][a-zA-Z0-9_:]*$/, "Must be a bare metric name") .optional(), + /** Bench-level sanity check for the per-provider `live_activity` + * probe: an instant query that must be > 0 for the "Feed down" + * UI to trust its per-provider verdicts. Meant to answer "are + * ANY of our probes still emitting". Zero means our end + * (harness / Prom scrape) is broken, not the provider — so the + * UI suppresses all per-provider down badges to avoid falsely + * blaming every source at once. When omitted, per-provider + * activity is trusted unconditionally. */ + probe_ok: promql.optional(), }) .optional(), diff --git a/src/types/benchmark.ts b/src/types/benchmark.ts index d61cff87..d7ef0e4e 100644 --- a/src/types/benchmark.ts +++ b/src/types/benchmark.ts @@ -94,6 +94,20 @@ export type ProviderResult = { * headline value is computed. Rendered as the leaderboard-row * hover tooltip. Authored per-bench in YAML (provider.formula). */ formula?: string; + /** Short-window liveness verdict derived at load time from the spec's + * `queries.live_activity` scalar and the bench-level `probe_ok` + * gate. Only populated when the spec declares those queries. + * - "healthy": recent events arrived (activity > 0). + * - "down": no recent events AND probe_ok confirmed our end is + * fine — the provider's live feed is silent, values + * shown are last-known. Renderers should badge it. + * - "unknown": probe_ok reports our side is broken (harness or + * Prom scrape), OR the activity query itself + * returned no sample. Suppress the badge either way + * so a probe hiccup can't fake-flag every provider. + * Absent when the spec doesn't declare live_activity; UI must + * behave identically to today for those benches. */ + liveStatus?: "healthy" | "down" | "unknown"; }; /** One provider's standing inside a (chain, region) ranking cell. */