From 00e7124cb6934815d92d95183a22e39799dce7d9 Mon Sep 17 00:00:00 2001 From: Florent Tapponnier Date: Thu, 16 Jul 2026 12:50:07 +0200 Subject: [PATCH] series gaps: null the chart line when a feed goes silent (unless changes==0) Companion to feed-down banner: makes the 40min Codex blackout from this morning (08:23-09:04 UTC) visible on the 24h chart instead of a flat line covering the outage. Per-series match preserves the aggregate line as long as one chain still receives events. --- benchmarks/aggregator-head-lag.yml | 31 ++++++++++++++++++------------ 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/benchmarks/aggregator-head-lag.yml b/benchmarks/aggregator-head-lag.yml index acc2d4a1..f3eb9cd8 100644 --- a/benchmarks/aggregator-head-lag.yml +++ b/benchmarks/aggregator-head-lag.yml @@ -161,7 +161,14 @@ providers: mean: avg_over_time(head_lag_seconds{aggregator="mobula"}[24h]) * 1000 success: clamp_max(count_over_time(head_lag_seconds{aggregator="mobula"}[24h]) / 5760, 1) sample_size: sum(count_over_time(head_lag_seconds{aggregator="mobula"}[24h])) - series: avg_over_time(head_lag_seconds{aggregator="mobula"}[1h]) * 1000 + # `unless ... == 0` drops the sample when the gauge saw no fresh + # event in the last 15 min. Without this, the head_lag gauge stays + # frozen at its last value during a full WS outage and the chart + # draws a flat line where the reader would expect a gap. The + # per-series match means a partial outage on one chain leaves the + # other chains' points untouched (aggregate line stays continuous + # when at least one chain is still receiving events). + series: avg_over_time(head_lag_seconds{aggregator="mobula"}[1h]) * 1000 unless changes(head_lag_seconds{aggregator="mobula"}[15m]) == 0 # Total gauge-value changes on this aggregator across chains and # regions in the last 15min. Non-zero = fresh events arrived; 0 = # every subscription has been silent for 15+ minutes (the gauge is @@ -171,13 +178,13 @@ providers: regions: - region: us-east p50: quantile_over_time(0.50, head_lag_seconds{aggregator="mobula", region="us-east"}[24h]) * 1000 - series: avg_over_time(head_lag_seconds{aggregator="mobula", region="us-east"}[1h]) * 1000 + series: avg_over_time(head_lag_seconds{aggregator="mobula", region="us-east"}[1h]) * 1000 unless changes(head_lag_seconds{aggregator="mobula", region="us-east"}[15m]) == 0 - region: eu-west p50: quantile_over_time(0.50, head_lag_seconds{aggregator="mobula", region="eu-west"}[24h]) * 1000 - series: avg_over_time(head_lag_seconds{aggregator="mobula", region="eu-west"}[1h]) * 1000 + series: avg_over_time(head_lag_seconds{aggregator="mobula", region="eu-west"}[1h]) * 1000 unless changes(head_lag_seconds{aggregator="mobula", region="eu-west"}[15m]) == 0 - region: ap-southeast p50: quantile_over_time(0.50, head_lag_seconds{aggregator="mobula", region="sgp"}[24h]) * 1000 - series: avg_over_time(head_lag_seconds{aggregator="mobula", region="sgp"}[1h]) * 1000 + series: avg_over_time(head_lag_seconds{aggregator="mobula", region="sgp"}[1h]) * 1000 unless changes(head_lag_seconds{aggregator="mobula", region="sgp"}[15m]) == 0 - slug: codex name: Codex @@ -190,18 +197,18 @@ providers: mean: avg_over_time(head_lag_seconds{aggregator="codex"}[24h]) * 1000 success: clamp_max(count_over_time(head_lag_seconds{aggregator="codex"}[24h]) / 5760, 1) sample_size: sum(count_over_time(head_lag_seconds{aggregator="codex"}[24h])) - series: avg_over_time(head_lag_seconds{aggregator="codex"}[1h]) * 1000 + series: avg_over_time(head_lag_seconds{aggregator="codex"}[1h]) * 1000 unless changes(head_lag_seconds{aggregator="codex"}[15m]) == 0 live_activity: sum(changes(head_lag_seconds{aggregator="codex"}[15m])) regions: - region: us-east p50: quantile_over_time(0.50, head_lag_seconds{aggregator="codex", region="us-east"}[24h]) * 1000 - series: avg_over_time(head_lag_seconds{aggregator="codex", region="us-east"}[1h]) * 1000 + series: avg_over_time(head_lag_seconds{aggregator="codex", region="us-east"}[1h]) * 1000 unless changes(head_lag_seconds{aggregator="codex", region="us-east"}[15m]) == 0 - region: eu-west p50: quantile_over_time(0.50, head_lag_seconds{aggregator="codex", region="eu-west"}[24h]) * 1000 - series: avg_over_time(head_lag_seconds{aggregator="codex", region="eu-west"}[1h]) * 1000 + series: avg_over_time(head_lag_seconds{aggregator="codex", region="eu-west"}[1h]) * 1000 unless changes(head_lag_seconds{aggregator="codex", region="eu-west"}[15m]) == 0 - region: ap-southeast p50: quantile_over_time(0.50, head_lag_seconds{aggregator="codex", region="sgp"}[24h]) * 1000 - series: avg_over_time(head_lag_seconds{aggregator="codex", region="sgp"}[1h]) * 1000 + series: avg_over_time(head_lag_seconds{aggregator="codex", region="sgp"}[1h]) * 1000 unless changes(head_lag_seconds{aggregator="codex", region="sgp"}[15m]) == 0 - slug: geckoterminal name: GeckoTerminal @@ -214,16 +221,16 @@ providers: mean: avg_over_time(head_lag_seconds{aggregator="geckoterminal"}[24h]) * 1000 success: clamp_max(count_over_time(head_lag_seconds{aggregator="geckoterminal"}[24h]) / 5760, 1) sample_size: sum(count_over_time(head_lag_seconds{aggregator="geckoterminal"}[24h])) - series: avg_over_time(head_lag_seconds{aggregator="geckoterminal"}[1h]) * 1000 + series: avg_over_time(head_lag_seconds{aggregator="geckoterminal"}[1h]) * 1000 unless changes(head_lag_seconds{aggregator="geckoterminal"}[15m]) == 0 live_activity: sum(changes(head_lag_seconds{aggregator="geckoterminal"}[15m])) regions: - region: us-east p50: quantile_over_time(0.50, head_lag_seconds{aggregator="geckoterminal", region="us-east"}[24h]) * 1000 - series: avg_over_time(head_lag_seconds{aggregator="geckoterminal", region="us-east"}[1h]) * 1000 + series: avg_over_time(head_lag_seconds{aggregator="geckoterminal", region="us-east"}[1h]) * 1000 unless changes(head_lag_seconds{aggregator="geckoterminal", region="us-east"}[15m]) == 0 - region: eu-west p50: quantile_over_time(0.50, head_lag_seconds{aggregator="geckoterminal", region="eu-west"}[24h]) * 1000 - series: avg_over_time(head_lag_seconds{aggregator="geckoterminal", region="eu-west"}[1h]) * 1000 + series: avg_over_time(head_lag_seconds{aggregator="geckoterminal", region="eu-west"}[1h]) * 1000 unless changes(head_lag_seconds{aggregator="geckoterminal", region="eu-west"}[15m]) == 0 - region: ap-southeast p50: quantile_over_time(0.50, head_lag_seconds{aggregator="geckoterminal", region="sgp"}[24h]) * 1000 - series: avg_over_time(head_lag_seconds{aggregator="geckoterminal", region="sgp"}[1h]) * 1000 + series: avg_over_time(head_lag_seconds{aggregator="geckoterminal", region="sgp"}[1h]) * 1000 unless changes(head_lag_seconds{aggregator="geckoterminal", region="sgp"}[15m]) == 0