From 7c1d743f4cbce19e9a05aec32f5c08d185d48fa2 Mon Sep 17 00:00:00 2001 From: Flotapponnier <160007691+Flotapponnier@users.noreply.github.com> Date: Thu, 16 Jul 2026 12:40:34 +0200 Subject: [PATCH 01/11] live-feed status: Feed down pill + banner on aggregator-head-lag (#1193) Short-window changes() probe per provider surfaces silent live feeds that the 24h quantile aggregate hides (Codex WS blackout 08:23-09:04 UTC today moved the p50 by nothing but the whole feed was mute for 40min). Bench-level probe_ok gate suppresses badges when our harness side is broken so a Railway crash cannot fake-flag every provider. Co-authored-by: Florent Tapponnier --- benchmarks/aggregator-head-lag.yml | 16 +++++++++++++ src/app/benchmarks/[slug]/page.tsx | 31 +++++++++++++++++++++++++ src/components/ledger-table.tsx | 8 +++++++ src/lib/materialize/load.ts | 37 +++++++++++++++++++++++++++++- src/lib/snapshot.ts | 1 + src/lib/spec-schema.ts | 17 ++++++++++++++ src/types/benchmark.ts | 14 +++++++++++ 7 files changed, 123 insertions(+), 1 deletion(-) diff --git a/benchmarks/aggregator-head-lag.yml b/benchmarks/aggregator-head-lag.yml index e72b0c81..acc2d4a1 100644 --- a/benchmarks/aggregator-head-lag.yml +++ b/benchmarks/aggregator-head-lag.yml @@ -83,6 +83,14 @@ source: https://github.com/ChainBench/OpenChainBench/tree/main/harnesses/aggrega prometheus: window: 24h + # Live-feed sanity gate. Sums event-arrival changes across every + # aggregator series in the last 15min. If this is 0, ALL our probes + # are silent (harness crash / Prom scrape failing / config drift) and + # the per-provider "Feed down" badges would be misleading, so the UI + # suppresses them. If > 0, at least one provider is actively receiving + # events from us, which means a provider showing 0 changes on its own + # is genuinely silent from Codex/Mobula/Gecko upstream. + probe_ok: sum(changes(head_lag_seconds[15m])) faq: - q: "Which crypto data API has the lowest latency right now?" @@ -154,6 +162,12 @@ providers: success: clamp_max(count_over_time(head_lag_seconds{aggregator="mobula"}[24h]) / 5760, 1) sample_size: sum(count_over_time(head_lag_seconds{aggregator="mobula"}[24h])) series: avg_over_time(head_lag_seconds{aggregator="mobula"}[1h]) * 1000 + # Total gauge-value changes on this aggregator across chains and + # regions in the last 15min. Non-zero = fresh events arrived; 0 = + # every subscription has been silent for 15+ minutes (the gauge is + # frozen at its last value). Threshold is > 0 rather than a rate + # so quiet chains at night don't false-flag the aggregate view. + live_activity: sum(changes(head_lag_seconds{aggregator="mobula"}[15m])) regions: - region: us-east p50: quantile_over_time(0.50, head_lag_seconds{aggregator="mobula", region="us-east"}[24h]) * 1000 @@ -177,6 +191,7 @@ providers: success: clamp_max(count_over_time(head_lag_seconds{aggregator="codex"}[24h]) / 5760, 1) sample_size: sum(count_over_time(head_lag_seconds{aggregator="codex"}[24h])) series: avg_over_time(head_lag_seconds{aggregator="codex"}[1h]) * 1000 + live_activity: sum(changes(head_lag_seconds{aggregator="codex"}[15m])) regions: - region: us-east p50: quantile_over_time(0.50, head_lag_seconds{aggregator="codex", region="us-east"}[24h]) * 1000 @@ -200,6 +215,7 @@ providers: success: clamp_max(count_over_time(head_lag_seconds{aggregator="geckoterminal"}[24h]) / 5760, 1) sample_size: sum(count_over_time(head_lag_seconds{aggregator="geckoterminal"}[24h])) series: avg_over_time(head_lag_seconds{aggregator="geckoterminal"}[1h]) * 1000 + live_activity: sum(changes(head_lag_seconds{aggregator="geckoterminal"}[15m])) regions: - region: us-east p50: quantile_over_time(0.50, head_lag_seconds{aggregator="geckoterminal", region="us-east"}[24h]) * 1000 diff --git a/src/app/benchmarks/[slug]/page.tsx b/src/app/benchmarks/[slug]/page.tsx index 14215e18..702c6b7f 100644 --- a/src/app/benchmarks/[slug]/page.tsx +++ b/src/app/benchmarks/[slug]/page.tsx @@ -613,6 +613,37 @@ export default async function BenchmarkPage({ )} + {/* Live-feed outage banner. Renders when the spec declared + live_activity queries and one or more providers came back + "down" (short-window activity = 0, bench-level probe_ok + confirmed our end is fine). Placed above the fold so a reader + landing on the page during an incident sees the caveat before + reading last-known percentiles as current truth. */} + {(() => { + const downProviders = benchmark.results.filter( + (r) => r.liveStatus === "down" + ); + if (downProviders.length === 0) return null; + const names = downProviders.map((r) => r.name).join(", "); + return ( +
+

+ Live feed silent: {names} +

+

+ No new events received from{" "} + {downProviders.length === 1 ? "this feed" : "these feeds"} in + the last several minutes. Percentiles and rankings shown are + the last-known values from the 24-hour window and update as + soon as fresh events resume. +

+
+ ); + })()} + {/* SEO-tuned intro paragraph rendered server-side under the H1 so long-tail query phrases land in the first ~200 words crawlers weight heavily. Optional - omitted when the YAML doesn't set it. */} diff --git a/src/components/ledger-table.tsx b/src/components/ledger-table.tsx index f06ed03f..638f0b8c 100644 --- a/src/components/ledger-table.tsx +++ b/src/components/ledger-table.tsx @@ -786,6 +786,14 @@ function Row({ )} + {!isMuted && r.liveStatus === "down" && ( + + + + Feed down + + + )} {!isMuted && r.dataConfidence === "low" && ( ): Spec success: inject(p.queries.success), sample_size: inject(p.queries.sample_size), series: inject(p.queries.series), + live_activity: inject(p.queries.live_activity), regions: p.queries.regions?.map((r) => ({ ...r, p50: inject(r.p50), @@ -547,6 +548,9 @@ function applyDimensionsToSpec(spec: Spec, labels: Record): Spec } : p.queries, })), + prometheus: spec.prometheus + ? { ...spec.prometheus, probe_ok: inject(spec.prometheus.probe_ok) } + : spec.prometheus, }; } @@ -655,6 +659,19 @@ async function tryLoadLive( const sevenDaysSec = 7 * 86_400; const thirtyDaysSec = 30 * 86_400; + // Bench-level "is our end fine" gate. Fetched once per sweep; feeds + // every provider's liveStatus verdict below so a broken harness / Prom + // scrape can't fake-flag every provider as down at once. Absent when + // the spec doesn't declare probe_ok — in which case we trust each + // provider's live_activity unconditionally. + const probeOkQuery = spec.prometheus?.probe_ok; + const probeOk = probeOkQuery + ? await prom.scalar(probeOkQuery) + : null; + // Interpret: >0 or null-when-not-declared → trust per-provider verdicts. + // Explicit 0 (or NaN) → the probe itself is down; suppress all badges. + const trustLiveVerdicts = !probeOkQuery || (probeOk != null && probeOk > 0); + for (const p of spec.providers) { const q = p.queries; if (!q) return null; @@ -664,12 +681,13 @@ async function tryLoadLive( q.p90 ? prom.scalar(q.p90) : Promise.resolve(null), q.p99 ? prom.scalar(q.p99) : Promise.resolve(null), ]); - const [mean, success, sampleSize, slotP50, slotP99] = await Promise.all([ + const [mean, success, sampleSize, slotP50, slotP99, liveActivity] = await Promise.all([ q.mean ? prom.scalar(q.mean) : Promise.resolve(null), q.success ? prom.scalar(q.success) : Promise.resolve(null), q.sample_size ? prom.scalar(q.sample_size) : Promise.resolve(null), q.slot_p50 ? prom.scalar(q.slot_p50) : Promise.resolve(null), q.slot_p99 ? prom.scalar(q.slot_p99) : Promise.resolve(null), + q.live_activity ? prom.scalar(q.live_activity) : Promise.resolve(null), ]); // One retry on the load-bearing percentiles. A null here is either @@ -735,6 +753,22 @@ async function tryLoadLive( continue; } + // liveStatus: only computed when the spec declares live_activity. + // "unknown" wins whenever we can't tell (probe_ok says our side is + // broken, or the activity query returned no sample) — never falls + // through to "down" on ambiguous data, since a false red pill on + // a live provider is worse than a missing pill on a real outage. + let liveStatus: "healthy" | "down" | "unknown" | undefined; + if (q.live_activity) { + if (!trustLiveVerdicts || liveActivity == null) { + liveStatus = "unknown"; + } else if (liveActivity > 0) { + liveStatus = "healthy"; + } else { + liveStatus = "down"; + } + } + liveResults.push({ name: p.name, slug: p.slug, @@ -751,6 +785,7 @@ async function tryLoadLive( secondary: p.secondary, query: q.p50, formula: p.formula, + liveStatus, }); if (q.series) { diff --git a/src/lib/snapshot.ts b/src/lib/snapshot.ts index 89e658bd..bc397e3d 100644 --- a/src/lib/snapshot.ts +++ b/src/lib/snapshot.ts @@ -86,6 +86,7 @@ const ProviderResultSchema = z.object({ meta: StalenessMetaSchema.optional(), query: z.string().optional(), formula: z.string().optional(), + liveStatus: z.enum(["healthy", "down", "unknown"]).optional(), }); const RegionPointSchema = z.object({ diff --git a/src/lib/spec-schema.ts b/src/lib/spec-schema.ts index 2aa96b9f..36bbcd87 100644 --- a/src/lib/spec-schema.ts +++ b/src/lib/spec-schema.ts @@ -80,6 +80,14 @@ const queries = z success: promql.optional(), sample_size: promql.optional(), series: promql.optional(), + /** Short-window "is this provider's live feed producing new events + * right now" probe. Instant query returning a scalar count > 0 when + * fresh events arrived in the past few minutes, 0 when the source + * is silent. Distinct from `success` (24h rolling reliability), + * which is too slow to move on a 30-60 min outage. Rendered as a + * "Feed down" pill + top-of-page banner when 0 and probe_ok + * confirms our end is fine. */ + live_activity: promql.optional(), /** Optional slot-level companion queries. Solana-native benches set * these to surface slot_delta p50/p99 alongside the ms columns. The * ms numbers are wall-clock derived; slot_delta is the canonical @@ -309,6 +317,15 @@ export const SpecSchema = z .string() .regex(/^[a-zA-Z_:][a-zA-Z0-9_:]*$/, "Must be a bare metric name") .optional(), + /** Bench-level sanity check for the per-provider `live_activity` + * probe: an instant query that must be > 0 for the "Feed down" + * UI to trust its per-provider verdicts. Meant to answer "are + * ANY of our probes still emitting". Zero means our end + * (harness / Prom scrape) is broken, not the provider — so the + * UI suppresses all per-provider down badges to avoid falsely + * blaming every source at once. When omitted, per-provider + * activity is trusted unconditionally. */ + probe_ok: promql.optional(), }) .optional(), diff --git a/src/types/benchmark.ts b/src/types/benchmark.ts index d61cff87..d7ef0e4e 100644 --- a/src/types/benchmark.ts +++ b/src/types/benchmark.ts @@ -94,6 +94,20 @@ export type ProviderResult = { * headline value is computed. Rendered as the leaderboard-row * hover tooltip. Authored per-bench in YAML (provider.formula). */ formula?: string; + /** Short-window liveness verdict derived at load time from the spec's + * `queries.live_activity` scalar and the bench-level `probe_ok` + * gate. Only populated when the spec declares those queries. + * - "healthy": recent events arrived (activity > 0). + * - "down": no recent events AND probe_ok confirmed our end is + * fine — the provider's live feed is silent, values + * shown are last-known. Renderers should badge it. + * - "unknown": probe_ok reports our side is broken (harness or + * Prom scrape), OR the activity query itself + * returned no sample. Suppress the badge either way + * so a probe hiccup can't fake-flag every provider. + * Absent when the spec doesn't declare live_activity; UI must + * behave identically to today for those benches. */ + liveStatus?: "healthy" | "down" | "unknown"; }; /** One provider's standing inside a (chain, region) ranking cell. */ From e5fed65255fdf39625a71f84795c8cc7e6b44f85 Mon Sep 17 00:00:00 2001 From: Flotapponnier <160007691+Flotapponnier@users.noreply.github.com> Date: Thu, 16 Jul 2026 12:50:32 +0200 Subject: [PATCH 02/11] series gaps: null the chart line when a feed goes silent (unless changes==0) (#1195) Companion to feed-down banner: makes the 40min Codex blackout from this morning (08:23-09:04 UTC) visible on the 24h chart instead of a flat line covering the outage. Per-series match preserves the aggregate line as long as one chain still receives events. Co-authored-by: Florent Tapponnier --- benchmarks/aggregator-head-lag.yml | 31 ++++++++++++++++++------------ 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/benchmarks/aggregator-head-lag.yml b/benchmarks/aggregator-head-lag.yml index acc2d4a1..f3eb9cd8 100644 --- a/benchmarks/aggregator-head-lag.yml +++ b/benchmarks/aggregator-head-lag.yml @@ -161,7 +161,14 @@ providers: mean: avg_over_time(head_lag_seconds{aggregator="mobula"}[24h]) * 1000 success: clamp_max(count_over_time(head_lag_seconds{aggregator="mobula"}[24h]) / 5760, 1) sample_size: sum(count_over_time(head_lag_seconds{aggregator="mobula"}[24h])) - series: avg_over_time(head_lag_seconds{aggregator="mobula"}[1h]) * 1000 + # `unless ... == 0` drops the sample when the gauge saw no fresh + # event in the last 15 min. Without this, the head_lag gauge stays + # frozen at its last value during a full WS outage and the chart + # draws a flat line where the reader would expect a gap. The + # per-series match means a partial outage on one chain leaves the + # other chains' points untouched (aggregate line stays continuous + # when at least one chain is still receiving events). + series: avg_over_time(head_lag_seconds{aggregator="mobula"}[1h]) * 1000 unless changes(head_lag_seconds{aggregator="mobula"}[15m]) == 0 # Total gauge-value changes on this aggregator across chains and # regions in the last 15min. Non-zero = fresh events arrived; 0 = # every subscription has been silent for 15+ minutes (the gauge is @@ -171,13 +178,13 @@ providers: regions: - region: us-east p50: quantile_over_time(0.50, head_lag_seconds{aggregator="mobula", region="us-east"}[24h]) * 1000 - series: avg_over_time(head_lag_seconds{aggregator="mobula", region="us-east"}[1h]) * 1000 + series: avg_over_time(head_lag_seconds{aggregator="mobula", region="us-east"}[1h]) * 1000 unless changes(head_lag_seconds{aggregator="mobula", region="us-east"}[15m]) == 0 - region: eu-west p50: quantile_over_time(0.50, head_lag_seconds{aggregator="mobula", region="eu-west"}[24h]) * 1000 - series: avg_over_time(head_lag_seconds{aggregator="mobula", region="eu-west"}[1h]) * 1000 + series: avg_over_time(head_lag_seconds{aggregator="mobula", region="eu-west"}[1h]) * 1000 unless changes(head_lag_seconds{aggregator="mobula", region="eu-west"}[15m]) == 0 - region: ap-southeast p50: quantile_over_time(0.50, head_lag_seconds{aggregator="mobula", region="sgp"}[24h]) * 1000 - series: avg_over_time(head_lag_seconds{aggregator="mobula", region="sgp"}[1h]) * 1000 + series: avg_over_time(head_lag_seconds{aggregator="mobula", region="sgp"}[1h]) * 1000 unless changes(head_lag_seconds{aggregator="mobula", region="sgp"}[15m]) == 0 - slug: codex name: Codex @@ -190,18 +197,18 @@ providers: mean: avg_over_time(head_lag_seconds{aggregator="codex"}[24h]) * 1000 success: clamp_max(count_over_time(head_lag_seconds{aggregator="codex"}[24h]) / 5760, 1) sample_size: sum(count_over_time(head_lag_seconds{aggregator="codex"}[24h])) - series: avg_over_time(head_lag_seconds{aggregator="codex"}[1h]) * 1000 + series: avg_over_time(head_lag_seconds{aggregator="codex"}[1h]) * 1000 unless changes(head_lag_seconds{aggregator="codex"}[15m]) == 0 live_activity: sum(changes(head_lag_seconds{aggregator="codex"}[15m])) regions: - region: us-east p50: quantile_over_time(0.50, head_lag_seconds{aggregator="codex", region="us-east"}[24h]) * 1000 - series: avg_over_time(head_lag_seconds{aggregator="codex", region="us-east"}[1h]) * 1000 + series: avg_over_time(head_lag_seconds{aggregator="codex", region="us-east"}[1h]) * 1000 unless changes(head_lag_seconds{aggregator="codex", region="us-east"}[15m]) == 0 - region: eu-west p50: quantile_over_time(0.50, head_lag_seconds{aggregator="codex", region="eu-west"}[24h]) * 1000 - series: avg_over_time(head_lag_seconds{aggregator="codex", region="eu-west"}[1h]) * 1000 + series: avg_over_time(head_lag_seconds{aggregator="codex", region="eu-west"}[1h]) * 1000 unless changes(head_lag_seconds{aggregator="codex", region="eu-west"}[15m]) == 0 - region: ap-southeast p50: quantile_over_time(0.50, head_lag_seconds{aggregator="codex", region="sgp"}[24h]) * 1000 - series: avg_over_time(head_lag_seconds{aggregator="codex", region="sgp"}[1h]) * 1000 + series: avg_over_time(head_lag_seconds{aggregator="codex", region="sgp"}[1h]) * 1000 unless changes(head_lag_seconds{aggregator="codex", region="sgp"}[15m]) == 0 - slug: geckoterminal name: GeckoTerminal @@ -214,16 +221,16 @@ providers: mean: avg_over_time(head_lag_seconds{aggregator="geckoterminal"}[24h]) * 1000 success: clamp_max(count_over_time(head_lag_seconds{aggregator="geckoterminal"}[24h]) / 5760, 1) sample_size: sum(count_over_time(head_lag_seconds{aggregator="geckoterminal"}[24h])) - series: avg_over_time(head_lag_seconds{aggregator="geckoterminal"}[1h]) * 1000 + series: avg_over_time(head_lag_seconds{aggregator="geckoterminal"}[1h]) * 1000 unless changes(head_lag_seconds{aggregator="geckoterminal"}[15m]) == 0 live_activity: sum(changes(head_lag_seconds{aggregator="geckoterminal"}[15m])) regions: - region: us-east p50: quantile_over_time(0.50, head_lag_seconds{aggregator="geckoterminal", region="us-east"}[24h]) * 1000 - series: avg_over_time(head_lag_seconds{aggregator="geckoterminal", region="us-east"}[1h]) * 1000 + series: avg_over_time(head_lag_seconds{aggregator="geckoterminal", region="us-east"}[1h]) * 1000 unless changes(head_lag_seconds{aggregator="geckoterminal", region="us-east"}[15m]) == 0 - region: eu-west p50: quantile_over_time(0.50, head_lag_seconds{aggregator="geckoterminal", region="eu-west"}[24h]) * 1000 - series: avg_over_time(head_lag_seconds{aggregator="geckoterminal", region="eu-west"}[1h]) * 1000 + series: avg_over_time(head_lag_seconds{aggregator="geckoterminal", region="eu-west"}[1h]) * 1000 unless changes(head_lag_seconds{aggregator="geckoterminal", region="eu-west"}[15m]) == 0 - region: ap-southeast p50: quantile_over_time(0.50, head_lag_seconds{aggregator="geckoterminal", region="sgp"}[24h]) * 1000 - series: avg_over_time(head_lag_seconds{aggregator="geckoterminal", region="sgp"}[1h]) * 1000 + series: avg_over_time(head_lag_seconds{aggregator="geckoterminal", region="sgp"}[1h]) * 1000 unless changes(head_lag_seconds{aggregator="geckoterminal", region="sgp"}[15m]) == 0 From 9eaa50171160275d1cf76abb70b088a87dc34e20 Mon Sep 17 00:00:00 2001 From: Flotapponnier <160007691+Flotapponnier@users.noreply.github.com> Date: Thu, 16 Jul 2026 13:52:22 +0200 Subject: [PATCH 03/11] chart: red band overlay for null-bucket downtime windows (#1203) Companion to #1195 (series null gaps). The gaps themselves render as line breaks but a break can read as 'the chart missed a sample'. A soft red column behind the break spans the exact outage window and makes the reader understand it was a real feed-side silence. Per line + per contiguous gap range, extended by one step so 1-bucket outages stay visible instead of collapsing to a zero-width sliver. Hidden for excluded (legend-toggled) providers. Co-authored-by: Florent Tapponnier --- src/components/time-series-chart/chart.tsx | 7 ++- src/components/time-series-chart/series.tsx | 61 +++++++++++++++++++++ 2 files changed, 67 insertions(+), 1 deletion(-) diff --git a/src/components/time-series-chart/chart.tsx b/src/components/time-series-chart/chart.tsx index 6998b116..29d88f38 100644 --- a/src/components/time-series-chart/chart.tsx +++ b/src/components/time-series-chart/chart.tsx @@ -6,7 +6,7 @@ import { type LineWithColor, } from "./scales"; import { YAxis, XAxis } from "./axis"; -import { SeriesGradients, SeriesPaths, HoverMarkers, type DrawnLine } from "./series"; +import { SeriesGradients, SeriesPaths, DowntimeBands, HoverMarkers, type DrawnLine } from "./series"; import { Legend } from "./legend"; import { Tooltip } from "./tooltip"; @@ -386,6 +386,11 @@ export function Chart({ {/* X tick labels */} + {/* Downtime bands. Rendered before the lines so the stroke stays + on top of the highlight. Contiguous null-buckets (from spec + `unless changes == 0`) become visible red columns. */} + + {/* Areas + lines */} diff --git a/src/components/time-series-chart/series.tsx b/src/components/time-series-chart/series.tsx index 22e685cd..7769bca2 100644 --- a/src/components/time-series-chart/series.tsx +++ b/src/components/time-series-chart/series.tsx @@ -16,6 +16,67 @@ type SeriesPathsProps = { unit: string; }; +type DowntimeBandsProps = { + drawn: DrawnLine[]; + padT: number; + innerH: number; +}; + +/** Semi-transparent red rectangles that highlight the exact window each + * line was silent (contiguous gap indices). Rendered under SeriesPaths + * so the line stroke stays on top; drawn only for visible (non-excluded) + * lines so toggling a provider off in the legend hides its band with + * it. A single-point gap (1 bucket) still renders as a narrow band by + * extending one step to the right — makes a short outage visible + * instead of being an invisible zero-width sliver. */ +export function DowntimeBands({ drawn, padT, innerH }: DowntimeBandsProps) { + return ( + <> + {drawn.map((d) => { + if (d.excluded) return null; + // Collect contiguous gap ranges as [startX, endX] pairs. endX is + // extended by one step (or the innerH) so a 1-point gap has a + // real visible width. + const bands: { x: number; w: number }[] = []; + let runStart: number | null = null; + for (let i = 0; i < d.pts.length; i++) { + const p = d.pts[i]; + if (p.gap) { + if (runStart == null) runStart = i; + } else if (runStart != null) { + const startX = d.pts[runStart].x; + const endX = d.pts[i].x; + bands.push({ x: startX, w: Math.max(2, endX - startX) }); + runStart = null; + } + } + // Trailing gap that runs to the current time. + if (runStart != null) { + const startX = d.pts[runStart].x; + const endX = d.pts[d.pts.length - 1].x; + bands.push({ x: startX, w: Math.max(2, endX - startX) }); + } + if (bands.length === 0) return null; + return ( + + {bands.map((b, i) => ( + + ))} + + ); + })} + + ); +} + export function SeriesPaths({ drawn, unit }: SeriesPathsProps) { return ( <> From c5d83c0d7edac9355a06f38deb9593bb97aadf83 Mon Sep 17 00:00:00 2001 From: Flotapponnier <160007691+Flotapponnier@users.noreply.github.com> Date: Thu, 16 Jul 2026 14:54:10 +0200 Subject: [PATCH 04/11] downtime bands: provider-colored tint + name/silent label + edge markers (#1210) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Feedback on #1203: the neutral red band read as 'something happened here' without telling the reader WHO. Tint each band in the provider's own color, add dashed edges at the outage boundaries, and label 'PROVIDER · SILENT' above the band when it fits (with a caret that survives on narrow single-bucket bands). Works uniformly for Mobula/Codex/Gecko or any future provider with a live_activity query. Co-authored-by: Florent Tapponnier --- src/components/time-series-chart/series.tsx | 103 +++++++++++++++----- 1 file changed, 80 insertions(+), 23 deletions(-) diff --git a/src/components/time-series-chart/series.tsx b/src/components/time-series-chart/series.tsx index 7769bca2..587f1099 100644 --- a/src/components/time-series-chart/series.tsx +++ b/src/components/time-series-chart/series.tsx @@ -22,21 +22,23 @@ type DowntimeBandsProps = { innerH: number; }; -/** Semi-transparent red rectangles that highlight the exact window each - * line was silent (contiguous gap indices). Rendered under SeriesPaths - * so the line stroke stays on top; drawn only for visible (non-excluded) - * lines so toggling a provider off in the legend hides its band with - * it. A single-point gap (1 bucket) still renders as a narrow band by - * extending one step to the right — makes a short outage visible - * instead of being an invisible zero-width sliver. */ +/** Colored rectangles that highlight the exact window each line was + * silent (contiguous gap indices). Each band tints in the provider's + * own color so multiple simultaneous outages read distinctly (e.g. + * Codex teal + Mobula orange bands overlapping look like two events, + * not one). A small label sits above the band with the provider name + * and a short reason so a reader doesn't need to hover the line to + * understand who dropped off. Rendered under SeriesPaths so the line + * stroke stays on top; skipped for excluded (legend-toggled) + * providers so their bands disappear with the line. A single-point + * gap still renders (min width 4 px) — a short outage matters just + * as much as a long one. */ export function DowntimeBands({ drawn, padT, innerH }: DowntimeBandsProps) { return ( <> {drawn.map((d) => { if (d.excluded) return null; - // Collect contiguous gap ranges as [startX, endX] pairs. endX is - // extended by one step (or the innerH) so a 1-point gap has a - // real visible width. + // Collect contiguous gap ranges as [startX, endX] pairs. const bands: { x: number; w: number }[] = []; let runStart: number | null = null; for (let i = 0; i < d.pts.length; i++) { @@ -46,7 +48,7 @@ export function DowntimeBands({ drawn, padT, innerH }: DowntimeBandsProps) { } else if (runStart != null) { const startX = d.pts[runStart].x; const endX = d.pts[i].x; - bands.push({ x: startX, w: Math.max(2, endX - startX) }); + bands.push({ x: startX, w: Math.max(4, endX - startX) }); runStart = null; } } @@ -54,22 +56,77 @@ export function DowntimeBands({ drawn, padT, innerH }: DowntimeBandsProps) { if (runStart != null) { const startX = d.pts[runStart].x; const endX = d.pts[d.pts.length - 1].x; - bands.push({ x: startX, w: Math.max(2, endX - startX) }); + bands.push({ x: startX, w: Math.max(4, endX - startX) }); } if (bands.length === 0) return null; return ( - {bands.map((b, i) => ( - - ))} + {bands.map((b, i) => { + // Label only when the band is wide enough that the text + // fits without overflowing. On narrow bands (1 bucket) + // the caret above the band carries the meaning alone. + const cx = b.x + b.w / 2; + const showLabel = b.w >= 42; + return ( + + {/* Tinted fill spans the outage window. */} + + {/* Left + right edges: dashed provider-colored strokes so + the boundaries of the outage are unambiguous even + when the tint is subtle. */} + + + {/* Caret above the band, then the label. The caret + alone survives on very narrow bands where the text + would be truncated. */} + + {showLabel && ( + + {d.name.toUpperCase()} · SILENT + + )} + + ); + })} ); })} From 6b2d23b3172da3af4fab3a14826d0856ce72a981 Mon Sep 17 00:00:00 2001 From: Flotapponnier <160007691+Flotapponnier@users.noreply.github.com> Date: Thu, 16 Jul 2026 15:06:41 +0200 Subject: [PATCH 05/11] downtime bands: pill labels + vertical stacking on overlap (#1211) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Screenshot review: the small text-only label above the band was still too subtle to read as 'this provider is down', and 3 simultaneous provider outages produced overlapping unreadable labels stacked on the same y. Replace with high-contrast rounded pills (white text on provider color) that stack vertically when their X-ranges overlap. Renders as '● NAME DOWN' in bold small caps so a reader gets the event and the provider identity at a glance. Co-authored-by: Florent Tapponnier --- src/components/time-series-chart/series.tsx | 249 ++++++++++++-------- 1 file changed, 147 insertions(+), 102 deletions(-) diff --git a/src/components/time-series-chart/series.tsx b/src/components/time-series-chart/series.tsx index 587f1099..0a04fa8c 100644 --- a/src/components/time-series-chart/series.tsx +++ b/src/components/time-series-chart/series.tsx @@ -23,114 +23,159 @@ type DowntimeBandsProps = { }; /** Colored rectangles that highlight the exact window each line was - * silent (contiguous gap indices). Each band tints in the provider's - * own color so multiple simultaneous outages read distinctly (e.g. - * Codex teal + Mobula orange bands overlapping look like two events, - * not one). A small label sits above the band with the provider name - * and a short reason so a reader doesn't need to hover the line to - * understand who dropped off. Rendered under SeriesPaths so the line - * stroke stays on top; skipped for excluded (legend-toggled) - * providers so their bands disappear with the line. A single-point - * gap still renders (min width 4 px) — a short outage matters just - * as much as a long one. */ + * silent (contiguous gap indices), plus a high-contrast pill label + * above each band naming the provider that dropped off. Pills stack + * vertically when multiple providers went silent at the same time so + * simultaneous outages read as separate lines instead of colliding + * as unreadable overlapping text. Rendered under SeriesPaths so the + * line stroke stays on top; skipped for excluded (legend-toggled) + * providers so their bands disappear with the line. */ export function DowntimeBands({ drawn, padT, innerH }: DowntimeBandsProps) { + // Collect every band across every visible provider before rendering. + // Two-pass so we can assign each band a vertical stack slot based on + // how many EARLIER-placed bands its X-range overlaps — labels then + // don't collide when three providers went dark in the same window. + type Band = { + slug: string; + name: string; + color: string; + x: number; + w: number; + slot: number; + }; + const all: Band[] = []; + for (const d of drawn) { + if (d.excluded) continue; + let runStart: number | null = null; + const push = (endX: number) => { + if (runStart == null) return; + const startX = d.pts[runStart].x; + all.push({ + slug: d.slug, + name: d.name, + color: d.color, + x: startX, + w: Math.max(6, endX - startX), + slot: 0, + }); + runStart = null; + }; + for (let i = 0; i < d.pts.length; i++) { + const p = d.pts[i]; + if (p.gap) { + if (runStart == null) runStart = i; + } else { + push(d.pts[i].x); + } + } + // Trailing gap that runs to the current time. + if (runStart != null) push(d.pts[d.pts.length - 1].x); + } + if (all.length === 0) return null; + + // Slot assignment: for each band, pick the smallest slot not used + // by another band whose X-range overlaps this one. Naive O(n²) is + // fine — a chart has at most ~20 bands in the pathological case. + all.sort((a, b) => a.x - b.x); + for (let i = 0; i < all.length; i++) { + const b = all[i]; + const used = new Set(); + for (let j = 0; j < i; j++) { + const p = all[j]; + if (p.x < b.x + b.w && p.x + p.w > b.x) used.add(p.slot); + } + let slot = 0; + while (used.has(slot)) slot++; + b.slot = slot; + } + + // Pill geometry constants. Pills sit ABOVE the plot area so they + // never occlude data. Small padding on the container's padT keeps + // them within the chart frame. + const PILL_H = 15; + const PILL_GAP = 3; + return ( - <> - {drawn.map((d) => { - if (d.excluded) return null; - // Collect contiguous gap ranges as [startX, endX] pairs. - const bands: { x: number; w: number }[] = []; - let runStart: number | null = null; - for (let i = 0; i < d.pts.length; i++) { - const p = d.pts[i]; - if (p.gap) { - if (runStart == null) runStart = i; - } else if (runStart != null) { - const startX = d.pts[runStart].x; - const endX = d.pts[i].x; - bands.push({ x: startX, w: Math.max(4, endX - startX) }); - runStart = null; - } - } - // Trailing gap that runs to the current time. - if (runStart != null) { - const startX = d.pts[runStart].x; - const endX = d.pts[d.pts.length - 1].x; - bands.push({ x: startX, w: Math.max(4, endX - startX) }); - } - if (bands.length === 0) return null; + + {/* Bands first (behind), then all pills on top so no band tint + can bleed onto a label from a taller-slot pill. */} + {all.map((b, i) => ( + + + + + + ))} + {all.map((b, i) => { + const cx = b.x + b.w / 2; + // "● NAME DOWN" width estimate: ~5.5 px per char plus fixed + // padding for the dot and the "DOWN" suffix. + const label = `${b.name.toUpperCase()} DOWN`; + const pillW = Math.max(58, label.length * 5.5 + 22); + const pillY = padT + 2 + b.slot * (PILL_H + PILL_GAP); + const pillX = Math.max(0, cx - pillW / 2); return ( - - {bands.map((b, i) => { - // Label only when the band is wide enough that the text - // fits without overflowing. On narrow bands (1 bucket) - // the caret above the band carries the meaning alone. - const cx = b.x + b.w / 2; - const showLabel = b.w >= 42; - return ( - - {/* Tinted fill spans the outage window. */} - - {/* Left + right edges: dashed provider-colored strokes so - the boundaries of the outage are unambiguous even - when the tint is subtle. */} - - - {/* Caret above the band, then the label. The caret - alone survives on very narrow bands where the text - would be truncated. */} - - {showLabel && ( - - {d.name.toUpperCase()} · SILENT - - )} - - ); - })} + + + {/* Small circle indicator, left of the label. */} + + + {label} + ); })} - + ); } From 49169dfbb6fbc5ac7dfd227f74a11ae8417939c1 Mon Sep 17 00:00:00 2001 From: Flotapponnier <160007691+Flotapponnier@users.noreply.github.com> Date: Thu, 16 Jul 2026 15:15:23 +0200 Subject: [PATCH 06/11] downtime pill: drop the leading dot indicator, center the label (#1213) Co-authored-by: Florent Tapponnier --- src/components/time-series-chart/series.tsx | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/src/components/time-series-chart/series.tsx b/src/components/time-series-chart/series.tsx index 0a04fa8c..fba23cf7 100644 --- a/src/components/time-series-chart/series.tsx +++ b/src/components/time-series-chart/series.tsx @@ -133,10 +133,10 @@ export function DowntimeBands({ drawn, padT, innerH }: DowntimeBandsProps) { ))} {all.map((b, i) => { const cx = b.x + b.w / 2; - // "● NAME DOWN" width estimate: ~5.5 px per char plus fixed - // padding for the dot and the "DOWN" suffix. + // "NAME DOWN" width estimate: ~5.5 px per char plus 14 px of + // horizontal padding (7 each side). const label = `${b.name.toUpperCase()} DOWN`; - const pillW = Math.max(58, label.length * 5.5 + 22); + const pillW = Math.max(50, label.length * 5.5 + 14); const pillY = padT + 2 + b.slot * (PILL_H + PILL_GAP); const pillX = Math.max(0, cx - pillW / 2); return ( @@ -151,16 +151,8 @@ export function DowntimeBands({ drawn, padT, innerH }: DowntimeBandsProps) { fill={b.color} opacity={0.95} /> - {/* Small circle indicator, left of the label. */} - Date: Thu, 16 Jul 2026 15:25:56 +0200 Subject: [PATCH 07/11] downtime pill: 'DATA MISSING' wording + skip leading gaps (#1215) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two fixes based on 30D view review: - Leading gaps (Prom retention shorter than the visible window, harness started mid-range, provider added recently) rendered as a chart-wide band claiming every provider was down for the missing period. Skip runs that started before the first observed sample. - Rename 'DOWN' to 'DATA MISSING'. A silent series can be the provider going down, our harness losing its WebSocket, or a Prom scrape failure — the pill stays neutral instead of blaming the provider. Co-authored-by: Florent Tapponnier --- src/components/time-series-chart/series.tsx | 26 +++++++++++++++++---- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/src/components/time-series-chart/series.tsx b/src/components/time-series-chart/series.tsx index fba23cf7..ca2ffa77 100644 --- a/src/components/time-series-chart/series.tsx +++ b/src/components/time-series-chart/series.tsx @@ -46,9 +46,20 @@ export function DowntimeBands({ drawn, padT, innerH }: DowntimeBandsProps) { const all: Band[] = []; for (const d of drawn) { if (d.excluded) continue; + // Only count gap runs that start AFTER the first observed sample. + // A leading run of nulls (Prom retention shorter than the visible + // window, harness started mid-range, or provider added recently) + // is "we didn't measure yet", not "provider was down for a week". + // Without this guard the 30D view on aggregator-head-lag rendered + // a chart-wide band during Prom's initial fill period. + let seenData = false; let runStart: number | null = null; const push = (endX: number) => { if (runStart == null) return; + if (!seenData) { + runStart = null; + return; + } const startX = d.pts[runStart].x; all.push({ slug: d.slug, @@ -66,9 +77,11 @@ export function DowntimeBands({ drawn, padT, innerH }: DowntimeBandsProps) { if (runStart == null) runStart = i; } else { push(d.pts[i].x); + seenData = true; } } - // Trailing gap that runs to the current time. + // Trailing gap that runs to the current time — meaningful only + // when the series had prior data (same seenData guard). if (runStart != null) push(d.pts[d.pts.length - 1].x); } if (all.length === 0) return null; @@ -133,10 +146,13 @@ export function DowntimeBands({ drawn, padT, innerH }: DowntimeBandsProps) { ))} {all.map((b, i) => { const cx = b.x + b.w / 2; - // "NAME DOWN" width estimate: ~5.5 px per char plus 14 px of - // horizontal padding (7 each side). - const label = `${b.name.toUpperCase()} DOWN`; - const pillW = Math.max(50, label.length * 5.5 + 14); + // "NAME DATA MISSING" width estimate: ~5.5 px per char plus + // 14 px of horizontal padding (7 each side). "DATA MISSING" + // stays neutral: absence of samples could be the provider + // going down, our harness losing its WebSocket, or a Prom + // scrape failure — the pill doesn't blame either side. + const label = `${b.name.toUpperCase()} DATA MISSING`; + const pillW = Math.max(80, label.length * 5.5 + 14); const pillY = padT + 2 + b.slot * (PILL_H + PILL_GAP); const pillX = Math.max(0, cx - pillW / 2); return ( From ade41d2ca8d64f0426e10f17de1b7dfba9ce1b89 Mon Sep 17 00:00:00 2001 From: Flotapponnier <160007691+Flotapponnier@users.noreply.github.com> Date: Thu, 16 Jul 2026 16:01:44 +0200 Subject: [PATCH 08/11] aggregator-head-lag: restore outage detection with backfill-safe guard (#1224) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Follow-up to #1221. Dropping the guard entirely restored the 30D range but silently hid today's Codex morning blackout on the 24H view (frozen gauge value = flat line, indistinguishable from a healthy quiet period). Bring the guard back, gated on count_over_time > 5 samples in the window. Modern data at 15s cadence carries ~60 samples per 15 min so the gate fires the moment a WS goes silent. Backfilled data compacted to ~1 sample/hour has count <= 2 in a 15m window so the gate stays open and every point Prom kept is served — no more fake wall-to-wall gap at the start of the 30D range. Co-authored-by: Florent Tapponnier --- benchmarks/aggregator-head-lag.yml | 40 ++++++++++++++++-------------- 1 file changed, 21 insertions(+), 19 deletions(-) diff --git a/benchmarks/aggregator-head-lag.yml b/benchmarks/aggregator-head-lag.yml index f3eb9cd8..d9fade57 100644 --- a/benchmarks/aggregator-head-lag.yml +++ b/benchmarks/aggregator-head-lag.yml @@ -161,14 +161,16 @@ providers: mean: avg_over_time(head_lag_seconds{aggregator="mobula"}[24h]) * 1000 success: clamp_max(count_over_time(head_lag_seconds{aggregator="mobula"}[24h]) / 5760, 1) sample_size: sum(count_over_time(head_lag_seconds{aggregator="mobula"}[24h])) - # `unless ... == 0` drops the sample when the gauge saw no fresh - # event in the last 15 min. Without this, the head_lag gauge stays - # frozen at its last value during a full WS outage and the chart - # draws a flat line where the reader would expect a gap. The - # per-series match means a partial outage on one chain leaves the - # other chains' points untouched (aggregate line stays continuous - # when at least one chain is still receiving events). - series: avg_over_time(head_lag_seconds{aggregator="mobula"}[1h]) * 1000 unless changes(head_lag_seconds{aggregator="mobula"}[15m]) == 0 + # Hybrid `unless` guard: drop the sample when the gauge saw no + # fresh event in the last 15 min AND we have enough scrapes in + # that window to trust the changes() reading (>5 samples). The + # count_over_time gate skips false-drops on backfilled data + # that Prom compacted to ~1 sample per hour — on that vintage + # changes([15m]) reads 0 by construction, without the gate the + # 30D view lost its first ~8 days to a fake wall-to-wall gap. + # Modern data at 15s cadence carries ~60 samples per 15 min so + # the guard still fires the moment a WS goes silent. + series: avg_over_time(head_lag_seconds{aggregator="mobula"}[1h]) * 1000 unless (changes(head_lag_seconds{aggregator="mobula"}[15m]) == 0 and count_over_time(head_lag_seconds{aggregator="mobula"}[15m]) > 5) # Total gauge-value changes on this aggregator across chains and # regions in the last 15min. Non-zero = fresh events arrived; 0 = # every subscription has been silent for 15+ minutes (the gauge is @@ -178,13 +180,13 @@ providers: regions: - region: us-east p50: quantile_over_time(0.50, head_lag_seconds{aggregator="mobula", region="us-east"}[24h]) * 1000 - series: avg_over_time(head_lag_seconds{aggregator="mobula", region="us-east"}[1h]) * 1000 unless changes(head_lag_seconds{aggregator="mobula", region="us-east"}[15m]) == 0 + series: avg_over_time(head_lag_seconds{aggregator="mobula", region="us-east"}[1h]) * 1000 unless (changes(head_lag_seconds{aggregator="mobula", region="us-east"}[15m]) == 0 and count_over_time(head_lag_seconds{aggregator="mobula", region="us-east"}[15m]) > 5) - region: eu-west p50: quantile_over_time(0.50, head_lag_seconds{aggregator="mobula", region="eu-west"}[24h]) * 1000 - series: avg_over_time(head_lag_seconds{aggregator="mobula", region="eu-west"}[1h]) * 1000 unless changes(head_lag_seconds{aggregator="mobula", region="eu-west"}[15m]) == 0 + series: avg_over_time(head_lag_seconds{aggregator="mobula", region="eu-west"}[1h]) * 1000 unless (changes(head_lag_seconds{aggregator="mobula", region="eu-west"}[15m]) == 0 and count_over_time(head_lag_seconds{aggregator="mobula", region="eu-west"}[15m]) > 5) - region: ap-southeast p50: quantile_over_time(0.50, head_lag_seconds{aggregator="mobula", region="sgp"}[24h]) * 1000 - series: avg_over_time(head_lag_seconds{aggregator="mobula", region="sgp"}[1h]) * 1000 unless changes(head_lag_seconds{aggregator="mobula", region="sgp"}[15m]) == 0 + series: avg_over_time(head_lag_seconds{aggregator="mobula", region="sgp"}[1h]) * 1000 unless (changes(head_lag_seconds{aggregator="mobula", region="sgp"}[15m]) == 0 and count_over_time(head_lag_seconds{aggregator="mobula", region="sgp"}[15m]) > 5) - slug: codex name: Codex @@ -197,18 +199,18 @@ providers: mean: avg_over_time(head_lag_seconds{aggregator="codex"}[24h]) * 1000 success: clamp_max(count_over_time(head_lag_seconds{aggregator="codex"}[24h]) / 5760, 1) sample_size: sum(count_over_time(head_lag_seconds{aggregator="codex"}[24h])) - series: avg_over_time(head_lag_seconds{aggregator="codex"}[1h]) * 1000 unless changes(head_lag_seconds{aggregator="codex"}[15m]) == 0 + series: avg_over_time(head_lag_seconds{aggregator="codex"}[1h]) * 1000 unless (changes(head_lag_seconds{aggregator="codex"}[15m]) == 0 and count_over_time(head_lag_seconds{aggregator="codex"}[15m]) > 5) live_activity: sum(changes(head_lag_seconds{aggregator="codex"}[15m])) regions: - region: us-east p50: quantile_over_time(0.50, head_lag_seconds{aggregator="codex", region="us-east"}[24h]) * 1000 - series: avg_over_time(head_lag_seconds{aggregator="codex", region="us-east"}[1h]) * 1000 unless changes(head_lag_seconds{aggregator="codex", region="us-east"}[15m]) == 0 + series: avg_over_time(head_lag_seconds{aggregator="codex", region="us-east"}[1h]) * 1000 unless (changes(head_lag_seconds{aggregator="codex", region="us-east"}[15m]) == 0 and count_over_time(head_lag_seconds{aggregator="codex", region="us-east"}[15m]) > 5) - region: eu-west p50: quantile_over_time(0.50, head_lag_seconds{aggregator="codex", region="eu-west"}[24h]) * 1000 - series: avg_over_time(head_lag_seconds{aggregator="codex", region="eu-west"}[1h]) * 1000 unless changes(head_lag_seconds{aggregator="codex", region="eu-west"}[15m]) == 0 + series: avg_over_time(head_lag_seconds{aggregator="codex", region="eu-west"}[1h]) * 1000 unless (changes(head_lag_seconds{aggregator="codex", region="eu-west"}[15m]) == 0 and count_over_time(head_lag_seconds{aggregator="codex", region="eu-west"}[15m]) > 5) - region: ap-southeast p50: quantile_over_time(0.50, head_lag_seconds{aggregator="codex", region="sgp"}[24h]) * 1000 - series: avg_over_time(head_lag_seconds{aggregator="codex", region="sgp"}[1h]) * 1000 unless changes(head_lag_seconds{aggregator="codex", region="sgp"}[15m]) == 0 + series: avg_over_time(head_lag_seconds{aggregator="codex", region="sgp"}[1h]) * 1000 unless (changes(head_lag_seconds{aggregator="codex", region="sgp"}[15m]) == 0 and count_over_time(head_lag_seconds{aggregator="codex", region="sgp"}[15m]) > 5) - slug: geckoterminal name: GeckoTerminal @@ -221,16 +223,16 @@ providers: mean: avg_over_time(head_lag_seconds{aggregator="geckoterminal"}[24h]) * 1000 success: clamp_max(count_over_time(head_lag_seconds{aggregator="geckoterminal"}[24h]) / 5760, 1) sample_size: sum(count_over_time(head_lag_seconds{aggregator="geckoterminal"}[24h])) - series: avg_over_time(head_lag_seconds{aggregator="geckoterminal"}[1h]) * 1000 unless changes(head_lag_seconds{aggregator="geckoterminal"}[15m]) == 0 + series: avg_over_time(head_lag_seconds{aggregator="geckoterminal"}[1h]) * 1000 unless (changes(head_lag_seconds{aggregator="geckoterminal"}[15m]) == 0 and count_over_time(head_lag_seconds{aggregator="geckoterminal"}[15m]) > 5) live_activity: sum(changes(head_lag_seconds{aggregator="geckoterminal"}[15m])) regions: - region: us-east p50: quantile_over_time(0.50, head_lag_seconds{aggregator="geckoterminal", region="us-east"}[24h]) * 1000 - series: avg_over_time(head_lag_seconds{aggregator="geckoterminal", region="us-east"}[1h]) * 1000 unless changes(head_lag_seconds{aggregator="geckoterminal", region="us-east"}[15m]) == 0 + series: avg_over_time(head_lag_seconds{aggregator="geckoterminal", region="us-east"}[1h]) * 1000 unless (changes(head_lag_seconds{aggregator="geckoterminal", region="us-east"}[15m]) == 0 and count_over_time(head_lag_seconds{aggregator="geckoterminal", region="us-east"}[15m]) > 5) - region: eu-west p50: quantile_over_time(0.50, head_lag_seconds{aggregator="geckoterminal", region="eu-west"}[24h]) * 1000 - series: avg_over_time(head_lag_seconds{aggregator="geckoterminal", region="eu-west"}[1h]) * 1000 unless changes(head_lag_seconds{aggregator="geckoterminal", region="eu-west"}[15m]) == 0 + series: avg_over_time(head_lag_seconds{aggregator="geckoterminal", region="eu-west"}[1h]) * 1000 unless (changes(head_lag_seconds{aggregator="geckoterminal", region="eu-west"}[15m]) == 0 and count_over_time(head_lag_seconds{aggregator="geckoterminal", region="eu-west"}[15m]) > 5) - region: ap-southeast p50: quantile_over_time(0.50, head_lag_seconds{aggregator="geckoterminal", region="sgp"}[24h]) * 1000 - series: avg_over_time(head_lag_seconds{aggregator="geckoterminal", region="sgp"}[1h]) * 1000 unless changes(head_lag_seconds{aggregator="geckoterminal", region="sgp"}[15m]) == 0 + series: avg_over_time(head_lag_seconds{aggregator="geckoterminal", region="sgp"}[1h]) * 1000 unless (changes(head_lag_seconds{aggregator="geckoterminal", region="sgp"}[15m]) == 0 and count_over_time(head_lag_seconds{aggregator="geckoterminal", region="sgp"}[15m]) > 5) From 458937caef0cd8fe708956481883ac83bf5d9560 Mon Sep 17 00:00:00 2001 From: Flotapponnier <160007691+Flotapponnier@users.noreply.github.com> Date: Thu, 16 Jul 2026 16:26:56 +0200 Subject: [PATCH 09/11] loader: propagate 24h series null-runs to 7d/30d (#1229) Fine-grained outages (24h step = 20 min buckets) rarely align with coarser evaluation timestamps (7d step = 2h, 30d step = 12h) so the Prom query returns a healthy average and the reader sees the pill on 24H but nothing on 7D/30D. Post-process: for every contiguous null run detected on s24, translate the time-ago range to bucket indices on s7 and s30 and null every coarse bucket the run straddles. Contiguity preserved so a single short outage doesn't split into two adjacent pills after propagation. Same treatment applied to per-region series. Co-authored-by: Florent Tapponnier --- src/lib/materialize/load.ts | 56 +++++++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/src/lib/materialize/load.ts b/src/lib/materialize/load.ts index 4f9f8932..7e8cd303 100644 --- a/src/lib/materialize/load.ts +++ b/src/lib/materialize/load.ts @@ -367,6 +367,48 @@ function activeFilterLabels(opts: BenchmarkFilters): Record { return out; } + +/** + * Nullify every bucket in `coarse` whose covering time-range overlaps + * a null bucket in `fine`. Both series are dense right-anchored ("now" + * = last index), so index i in a series of length N maps to time-ago + * (N - 1 - i) / (N - 1) of the window. Each contiguous null RUN on the + * fine grid nulls every coarse bucket its range straddles, so a short + * outage detected on the 24h grid stays visible as a contiguous band + * on the 7d and 30d grids (not two neighbouring pills). + * + * No-op when either input is missing or too short. Mutates `coarse`. + * Exported for tests. + */ +export function propagateNullsToCoarser( + fine: (number | null)[] | null | undefined, + coarse: (number | null)[] | null | undefined, +): void { + if (!fine || !coarse || fine.length < 2 || coarse.length < 2) return; + const denomFine = fine.length - 1; + const denomCoarse = coarse.length - 1; + const toIdxCoarse = (i: number) => + denomCoarse * (1 - (denomFine - i) / denomFine); + let runStart: number | null = null; + const closeRun = (endExclusive: number) => { + if (runStart == null) return; + const startX = toIdxCoarse(runStart); + const endX = toIdxCoarse(endExclusive); + const lo = Math.max(0, Math.floor(Math.min(startX, endX))); + const hi = Math.min(coarse.length - 1, Math.ceil(Math.max(startX, endX))); + for (let k = lo; k <= hi; k++) coarse[k] = null; + runStart = null; + }; + for (let i = 0; i < fine.length; i++) { + if (fine[i] === null) { + if (runStart == null) runStart = i; + } else { + closeRun(i); + } + } + closeRun(fine.length); +} + /** * Run the spec's `rank_matrix_query` (one instant vector with a sample per * (provider[, chain][, region])) and fold it into full per-cell rankings. @@ -794,6 +836,15 @@ async function tryLoadLive( prom.series(q.series, sevenDaysSec, 84), prom.series(q.series, thirtyDaysSec, 60), ]); + // Propagate short outages captured on the fine 24h grid up to + // the coarser 7d/30d grids. A 40 min blackout at 20 min step + // shows as 2-3 nulls on s24, but on s7 (2h step) or s30 (12h + // step) the outage rarely aligns with an evaluation window and + // Prom returns a healthy average. + if (s24 && s24.length > 0) { + propagateNullsToCoarser(s24, s7); + propagateNullsToCoarser(s24, s30); + } if (s24 && s24.length > 0) series24h[p.slug] = s24; if (s7 && s7.length > 0) series7d[p.slug] = s7; if (s30 && s30.length > 0) series30d[p.slug] = s30; @@ -819,6 +870,11 @@ async function tryLoadLive( ); regions[p.slug] = points.map(({ region: rg, p50: v }) => ({ region: rg, p50: v })); for (const pt of points) { + // Same short-outage propagation as the global series above. + if (pt.series24 && pt.series24.length > 0) { + propagateNullsToCoarser(pt.series24, pt.series7); + propagateNullsToCoarser(pt.series24, pt.series30); + } if (pt.series24 && pt.series24.length > 0) { (seriesByRegion24h[p.slug] ??= {})[pt.region] = pt.series24; } From 0a7b99f9394736ca0aace3c1bdd904d47576e800 Mon Sep 17 00:00:00 2001 From: Flotapponnier <160007691+Flotapponnier@users.noreply.github.com> Date: Thu, 16 Jul 2026 16:36:51 +0200 Subject: [PATCH 10/11] propagateNullsToCoarser: use absolute time-ago instead of window fraction (#1230) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bug: the previous math scaled by the fine window fraction, which projected a 7h-ago outage down to 40h-ago on the 7d grid (the pill landed at -1.7d instead of -6h). Both grids share 'now' as their right anchor and only the step sizes differ — take the fine idx's absolute time-ago and divide by the coarse step to find the covering bucket. Co-authored-by: Florent Tapponnier --- src/lib/materialize/load.ts | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/src/lib/materialize/load.ts b/src/lib/materialize/load.ts index 7e8cd303..9f559537 100644 --- a/src/lib/materialize/load.ts +++ b/src/lib/materialize/load.ts @@ -382,13 +382,23 @@ function activeFilterLabels(opts: BenchmarkFilters): Record { */ export function propagateNullsToCoarser( fine: (number | null)[] | null | undefined, + fineWindowSec: number, coarse: (number | null)[] | null | undefined, + coarseWindowSec: number, ): void { if (!fine || !coarse || fine.length < 2 || coarse.length < 2) return; - const denomFine = fine.length - 1; - const denomCoarse = coarse.length - 1; - const toIdxCoarse = (i: number) => - denomCoarse * (1 - (denomFine - i) / denomFine); + if (fineWindowSec <= 0 || coarseWindowSec <= 0) return; + const fineStep = fineWindowSec / (fine.length - 1); + const coarseStep = coarseWindowSec / (coarse.length - 1); + // Right-anchored: last index = now. Map a fine index to the same + // ABSOLUTE time-ago on the coarse grid (both grids share "now"; the + // step sizes are what differ, not the reference point). Previous + // math scaled by the fine window fraction, which shrank a 7 h ago + // outage down to 40 h ago once projected onto the 7 d grid. + const toIdxCoarse = (i: number): number => { + const secondsAgo = (fine.length - 1 - i) * fineStep; + return (coarse.length - 1) - secondsAgo / coarseStep; + }; let runStart: number | null = null; const closeRun = (endExclusive: number) => { if (runStart == null) return; @@ -842,8 +852,8 @@ async function tryLoadLive( // step) the outage rarely aligns with an evaluation window and // Prom returns a healthy average. if (s24 && s24.length > 0) { - propagateNullsToCoarser(s24, s7); - propagateNullsToCoarser(s24, s30); + propagateNullsToCoarser(s24, winSec, s7, sevenDaysSec); + propagateNullsToCoarser(s24, winSec, s30, thirtyDaysSec); } if (s24 && s24.length > 0) series24h[p.slug] = s24; if (s7 && s7.length > 0) series7d[p.slug] = s7; @@ -872,8 +882,8 @@ async function tryLoadLive( for (const pt of points) { // Same short-outage propagation as the global series above. if (pt.series24 && pt.series24.length > 0) { - propagateNullsToCoarser(pt.series24, pt.series7); - propagateNullsToCoarser(pt.series24, pt.series30); + propagateNullsToCoarser(pt.series24, winSec, pt.series7, sevenDaysSec); + propagateNullsToCoarser(pt.series24, winSec, pt.series30, thirtyDaysSec); } if (pt.series24 && pt.series24.length > 0) { (seriesByRegion24h[p.slug] ??= {})[pt.region] = pt.series24; From e46e86039598335460b40ca1e903cd8ddf978a9b Mon Sep 17 00:00:00 2001 From: Flotapponnier <160007691+Flotapponnier@users.noreply.github.com> Date: Thu, 16 Jul 2026 16:52:54 +0200 Subject: [PATCH 11/11] downtime bands: opt-in via live_activity, skip on other benches (#1235) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Screenshot review: rpc-reliability rendered a 'LAVANODIES DATA MISSING' pill even though its spec never declared live_activity — natural nulls in its series24h (sparse scrapes, backfill edges) triggered the DowntimeBands render. Gate the render on benchmark.results.some(r => r.liveStatus != null), which is only set when the spec declared live_activity. Same gate on propagateNullsToCoarser so other benches' 7d/30d series stay untouched. Co-authored-by: Florent Tapponnier --- src/components/time-series-chart/chart.tsx | 8 +++++++- src/components/time-series-chart/index.tsx | 1 + src/lib/materialize/load.ts | 12 ++++++------ 3 files changed, 14 insertions(+), 7 deletions(-) diff --git a/src/components/time-series-chart/chart.tsx b/src/components/time-series-chart/chart.tsx index 29d88f38..bdaca025 100644 --- a/src/components/time-series-chart/chart.tsx +++ b/src/components/time-series-chart/chart.tsx @@ -22,6 +22,11 @@ type ChartProps = { onZoom?: (next: { startFrac: number; endFrac: number } | null) => void; onToggleExclude?: (slug: string) => void; onResetExcluded?: () => void; + /** Only render the downtime bands + pill labels when the parent bench + * opted in (spec declared live_activity). Otherwise natural nulls in + * any bench's series24h would fire a "DATA MISSING" pill on charts + * that never asked for the feature (observed on rpc-reliability). */ + showDowntime?: boolean; }; export function Chart({ @@ -33,6 +38,7 @@ export function Chart({ onZoom, onToggleExclude, onResetExcluded, + showDowntime = false, }: ChartProps) { const W = 1000; const H = 360; @@ -389,7 +395,7 @@ export function Chart({ {/* Downtime bands. Rendered before the lines so the stroke stays on top of the highlight. Contiguous null-buckets (from spec `unless changes == 0`) become visible red columns. */} - + {showDowntime && } {/* Areas + lines */} diff --git a/src/components/time-series-chart/index.tsx b/src/components/time-series-chart/index.tsx index 6e105ae1..519ae13e 100644 --- a/src/components/time-series-chart/index.tsx +++ b/src/components/time-series-chart/index.tsx @@ -475,6 +475,7 @@ export function TimeSeriesChart({ onZoom={setZoom} onToggleExclude={toggle} onResetExcluded={excluded.size > 0 ? reset : undefined} + showDowntime={benchmark.results.some((r) => r.liveStatus != null)} /> )} diff --git a/src/lib/materialize/load.ts b/src/lib/materialize/load.ts index 9f559537..150e8828 100644 --- a/src/lib/materialize/load.ts +++ b/src/lib/materialize/load.ts @@ -847,11 +847,11 @@ async function tryLoadLive( prom.series(q.series, thirtyDaysSec, 60), ]); // Propagate short outages captured on the fine 24h grid up to - // the coarser 7d/30d grids. A 40 min blackout at 20 min step - // shows as 2-3 nulls on s24, but on s7 (2h step) or s30 (12h - // step) the outage rarely aligns with an evaluation window and - // Prom returns a healthy average. - if (s24 && s24.length > 0) { + // the coarser 7d/30d grids. Only when the bench opted in via + // `live_activity` — other benches carry natural nulls (sparse + // scrapes, backfill edges) that shouldn't fire a "DATA MISSING" + // pill on the chart. + if (q.live_activity && s24 && s24.length > 0) { propagateNullsToCoarser(s24, winSec, s7, sevenDaysSec); propagateNullsToCoarser(s24, winSec, s30, thirtyDaysSec); } @@ -881,7 +881,7 @@ async function tryLoadLive( regions[p.slug] = points.map(({ region: rg, p50: v }) => ({ region: rg, p50: v })); for (const pt of points) { // Same short-outage propagation as the global series above. - if (pt.series24 && pt.series24.length > 0) { + if (q.live_activity && pt.series24 && pt.series24.length > 0) { propagateNullsToCoarser(pt.series24, winSec, pt.series7, sevenDaysSec); propagateNullsToCoarser(pt.series24, winSec, pt.series30, thirtyDaysSec); }