cherry-pick: aggregator-head-lag downtime pills → main

benchmarks/aggregator-head-lag.yml

@@ -83,6 +83,14 @@ source: https://github.com/ChainBench/OpenChainBench/tree/main/harnesses/aggrega
 
 prometheus:
   window: 24h
+  # Live-feed sanity gate. Sums event-arrival changes across every
+  # aggregator series in the last 15min. If this is 0, ALL our probes
+  # are silent (harness crash / Prom scrape failing / config drift) and
+  # the per-provider "Feed down" badges would be misleading, so the UI
+  # suppresses them. If > 0, at least one provider is actively receiving
+  # events from us, which means a provider showing 0 changes on its own
+  # is genuinely silent from Codex/Mobula/Gecko upstream.
+  probe_ok: sum(changes(head_lag_seconds[15m]))
 
 faq:
   - q: "Which crypto data API has the lowest latency right now?"
@@ -153,17 +161,32 @@ providers:
       mean: avg_over_time(head_lag_seconds{aggregator="mobula"}[24h]) * 1000
       success: clamp_max(count_over_time(head_lag_seconds{aggregator="mobula"}[24h]) / 5760, 1)
       sample_size: sum(count_over_time(head_lag_seconds{aggregator="mobula"}[24h]))
-      series: avg_over_time(head_lag_seconds{aggregator="mobula"}[1h]) * 1000
+      # Hybrid `unless` guard: drop the sample when the gauge saw no
+      # fresh event in the last 15 min AND we have enough scrapes in
+      # that window to trust the changes() reading (>5 samples). The
+      # count_over_time gate skips false-drops on backfilled data
+      # that Prom compacted to ~1 sample per hour — on that vintage
+      # changes([15m]) reads 0 by construction, without the gate the
+      # 30D view lost its first ~8 days to a fake wall-to-wall gap.
+      # Modern data at 15s cadence carries ~60 samples per 15 min so
+      # the guard still fires the moment a WS goes silent.
+      series: avg_over_time(head_lag_seconds{aggregator="mobula"}[1h]) * 1000 unless (changes(head_lag_seconds{aggregator="mobula"}[15m]) == 0 and count_over_time(head_lag_seconds{aggregator="mobula"}[15m]) > 5)
+      # Total gauge-value changes on this aggregator across chains and
+      # regions in the last 15min. Non-zero = fresh events arrived; 0 =
+      # every subscription has been silent for 15+ minutes (the gauge is
+      # frozen at its last value). Threshold is > 0 rather than a rate
+      # so quiet chains at night don't false-flag the aggregate view.
+      live_activity: sum(changes(head_lag_seconds{aggregator="mobula"}[15m]))
       regions:
         - region: us-east
           p50: quantile_over_time(0.50, head_lag_seconds{aggregator="mobula", region="us-east"}[24h]) * 1000
-          series: avg_over_time(head_lag_seconds{aggregator="mobula", region="us-east"}[1h]) * 1000
+          series: avg_over_time(head_lag_seconds{aggregator="mobula", region="us-east"}[1h]) * 1000 unless (changes(head_lag_seconds{aggregator="mobula", region="us-east"}[15m]) == 0 and count_over_time(head_lag_seconds{aggregator="mobula", region="us-east"}[15m]) > 5)
         - region: eu-west
           p50: quantile_over_time(0.50, head_lag_seconds{aggregator="mobula", region="eu-west"}[24h]) * 1000
-          series: avg_over_time(head_lag_seconds{aggregator="mobula", region="eu-west"}[1h]) * 1000
+          series: avg_over_time(head_lag_seconds{aggregator="mobula", region="eu-west"}[1h]) * 1000 unless (changes(head_lag_seconds{aggregator="mobula", region="eu-west"}[15m]) == 0 and count_over_time(head_lag_seconds{aggregator="mobula", region="eu-west"}[15m]) > 5)
         - region: ap-southeast
           p50: quantile_over_time(0.50, head_lag_seconds{aggregator="mobula", region="sgp"}[24h]) * 1000
-          series: avg_over_time(head_lag_seconds{aggregator="mobula", region="sgp"}[1h]) * 1000
+          series: avg_over_time(head_lag_seconds{aggregator="mobula", region="sgp"}[1h]) * 1000 unless (changes(head_lag_seconds{aggregator="mobula", region="sgp"}[15m]) == 0 and count_over_time(head_lag_seconds{aggregator="mobula", region="sgp"}[15m]) > 5)
 
   - slug: codex
     name: Codex
@@ -176,17 +199,18 @@ providers:
       mean: avg_over_time(head_lag_seconds{aggregator="codex"}[24h]) * 1000
       success: clamp_max(count_over_time(head_lag_seconds{aggregator="codex"}[24h]) / 5760, 1)
       sample_size: sum(count_over_time(head_lag_seconds{aggregator="codex"}[24h]))
-      series: avg_over_time(head_lag_seconds{aggregator="codex"}[1h]) * 1000
+      series: avg_over_time(head_lag_seconds{aggregator="codex"}[1h]) * 1000 unless (changes(head_lag_seconds{aggregator="codex"}[15m]) == 0 and count_over_time(head_lag_seconds{aggregator="codex"}[15m]) > 5)
+      live_activity: sum(changes(head_lag_seconds{aggregator="codex"}[15m]))
       regions:
         - region: us-east
           p50: quantile_over_time(0.50, head_lag_seconds{aggregator="codex", region="us-east"}[24h]) * 1000
-          series: avg_over_time(head_lag_seconds{aggregator="codex", region="us-east"}[1h]) * 1000
+          series: avg_over_time(head_lag_seconds{aggregator="codex", region="us-east"}[1h]) * 1000 unless (changes(head_lag_seconds{aggregator="codex", region="us-east"}[15m]) == 0 and count_over_time(head_lag_seconds{aggregator="codex", region="us-east"}[15m]) > 5)
         - region: eu-west
           p50: quantile_over_time(0.50, head_lag_seconds{aggregator="codex", region="eu-west"}[24h]) * 1000
-          series: avg_over_time(head_lag_seconds{aggregator="codex", region="eu-west"}[1h]) * 1000
+          series: avg_over_time(head_lag_seconds{aggregator="codex", region="eu-west"}[1h]) * 1000 unless (changes(head_lag_seconds{aggregator="codex", region="eu-west"}[15m]) == 0 and count_over_time(head_lag_seconds{aggregator="codex", region="eu-west"}[15m]) > 5)
         - region: ap-southeast
           p50: quantile_over_time(0.50, head_lag_seconds{aggregator="codex", region="sgp"}[24h]) * 1000
-          series: avg_over_time(head_lag_seconds{aggregator="codex", region="sgp"}[1h]) * 1000
+          series: avg_over_time(head_lag_seconds{aggregator="codex", region="sgp"}[1h]) * 1000 unless (changes(head_lag_seconds{aggregator="codex", region="sgp"}[15m]) == 0 and count_over_time(head_lag_seconds{aggregator="codex", region="sgp"}[15m]) > 5)
 
   - slug: geckoterminal
     name: GeckoTerminal
@@ -199,15 +223,16 @@ providers:
       mean: avg_over_time(head_lag_seconds{aggregator="geckoterminal"}[24h]) * 1000
       success: clamp_max(count_over_time(head_lag_seconds{aggregator="geckoterminal"}[24h]) / 5760, 1)
       sample_size: sum(count_over_time(head_lag_seconds{aggregator="geckoterminal"}[24h]))
-      series: avg_over_time(head_lag_seconds{aggregator="geckoterminal"}[1h]) * 1000
+      series: avg_over_time(head_lag_seconds{aggregator="geckoterminal"}[1h]) * 1000 unless (changes(head_lag_seconds{aggregator="geckoterminal"}[15m]) == 0 and count_over_time(head_lag_seconds{aggregator="geckoterminal"}[15m]) > 5)
+      live_activity: sum(changes(head_lag_seconds{aggregator="geckoterminal"}[15m]))
       regions:
         - region: us-east
           p50: quantile_over_time(0.50, head_lag_seconds{aggregator="geckoterminal", region="us-east"}[24h]) * 1000
-          series: avg_over_time(head_lag_seconds{aggregator="geckoterminal", region="us-east"}[1h]) * 1000
+          series: avg_over_time(head_lag_seconds{aggregator="geckoterminal", region="us-east"}[1h]) * 1000 unless (changes(head_lag_seconds{aggregator="geckoterminal", region="us-east"}[15m]) == 0 and count_over_time(head_lag_seconds{aggregator="geckoterminal", region="us-east"}[15m]) > 5)
         - region: eu-west
           p50: quantile_over_time(0.50, head_lag_seconds{aggregator="geckoterminal", region="eu-west"}[24h]) * 1000
-          series: avg_over_time(head_lag_seconds{aggregator="geckoterminal", region="eu-west"}[1h]) * 1000
+          series: avg_over_time(head_lag_seconds{aggregator="geckoterminal", region="eu-west"}[1h]) * 1000 unless (changes(head_lag_seconds{aggregator="geckoterminal", region="eu-west"}[15m]) == 0 and count_over_time(head_lag_seconds{aggregator="geckoterminal", region="eu-west"}[15m]) > 5)
         - region: ap-southeast
           p50: quantile_over_time(0.50, head_lag_seconds{aggregator="geckoterminal", region="sgp"}[24h]) * 1000
-          series: avg_over_time(head_lag_seconds{aggregator="geckoterminal", region="sgp"}[1h]) * 1000
+          series: avg_over_time(head_lag_seconds{aggregator="geckoterminal", region="sgp"}[1h]) * 1000 unless (changes(head_lag_seconds{aggregator="geckoterminal", region="sgp"}[15m]) == 0 and count_over_time(head_lag_seconds{aggregator="geckoterminal", region="sgp"}[15m]) > 5)
 

src/app/benchmarks/[slug]/page.tsx

@@ -613,6 +613,37 @@ export default async function BenchmarkPage({
         </div>
       )}
 
+      {/* Live-feed outage banner. Renders when the spec declared
+          live_activity queries and one or more providers came back
+          "down" (short-window activity = 0, bench-level probe_ok
+          confirmed our end is fine). Placed above the fold so a reader
+          landing on the page during an incident sees the caveat before
+          reading last-known percentiles as current truth. */}
+      {(() => {
+        const downProviders = benchmark.results.filter(
+          (r) => r.liveStatus === "down"
+        );
+        if (downProviders.length === 0) return null;
+        const names = downProviders.map((r) => r.name).join(", ");
+        return (
+          <div
+            role="status"
+            className="mt-6 max-w-3xl rounded-md border border-danger/40 bg-danger/10 px-4 py-3 text-[14px] leading-relaxed text-ink"
+          >
+            <p className="label-mono mb-1 text-danger">
+              Live feed silent: {names}
+            </p>
+            <p>
+              No new events received from{" "}
+              {downProviders.length === 1 ? "this feed" : "these feeds"} in
+              the last several minutes. Percentiles and rankings shown are
+              the last-known values from the 24-hour window and update as
+              soon as fresh events resume.
+            </p>
+          </div>
+        );
+      })()}
+
       {/* SEO-tuned intro paragraph rendered server-side under the H1 so
           long-tail query phrases land in the first ~200 words crawlers
           weight heavily. Optional - omitted when the YAML doesn't set it. */}

src/components/ledger-table.tsx

@@ -786,6 +786,14 @@ function Row({
                 </span>
               </Hint>
             )}
+            {!isMuted && r.liveStatus === "down" && (
+              <Hint label="The provider's live feed has produced no new events in the last several minutes. The percentiles shown are the last-known values from the 24-hour window and will keep displaying until fresh events resume. Rolling reliability (Success column) barely moves on a short outage, which is why this pill exists.">
+                <span className="inline-flex items-center gap-1 shrink-0 font-sans text-[10px] uppercase tracking-[0.14em] text-ink-muted">
+                  <span className="inline-block w-1.5 h-1.5 rounded-full bg-[var(--color-danger,#b0402e)]" aria-hidden />
+                  Feed down
+                </span>
+              </Hint>
+            )}
             {!isMuted && r.dataConfidence === "low" && (
               <Hint
                 label={

src/components/time-series-chart/chart.tsx

@@ -6,7 +6,7 @@
   type LineWithColor,
 } from "./scales";
 import { YAxis, XAxis } from "./axis";
-import { SeriesGradients, SeriesPaths, HoverMarkers, type DrawnLine } from "./series";
+import { SeriesGradients, SeriesPaths, DowntimeBands, HoverMarkers, type DrawnLine } from "./series";
 import { Legend } from "./legend";
 import { Tooltip } from "./tooltip";
 
@@ -22,6 +22,11 @@
   onZoom?: (next: { startFrac: number; endFrac: number } | null) => void;
   onToggleExclude?: (slug: string) => void;
   onResetExcluded?: () => void;
+  /** Only render the downtime bands + pill labels when the parent bench
+   *  opted in (spec declared live_activity). Otherwise natural nulls in
+   *  any bench's series24h would fire a "DATA MISSING" pill on charts
+   *  that never asked for the feature (observed on rpc-reliability). */
+  showDowntime?: boolean;
 };
 
 export function Chart({
@@ -33,6 +38,7 @@
   onZoom,
   onToggleExclude,
   onResetExcluded,
+  showDowntime = false,
 }: ChartProps) {
   const W = 1000;
   const H = 360;
@@ -292,7 +298,7 @@
       // line draws a break but the dot keeps rendering at 0 / NaN.
       return { ...l, color, pts, linePath, fillPath, lastX, lastY, last, isGap };
     });
   }, [slicedLines, padL, padT, innerW, innerH, lo, yRange, expectedPoints]);
 
   // Hover line position: mirror the right-anchored placement used by drawn().
   // hover.idx is in [0, numPoints-1]; we map it to the visible chart via the
@@ -386,6 +392,11 @@
         {/* X tick labels */}
         <XAxis xTicks={xTicks} padL={padL} padT={padT} innerW={innerW} innerH={innerH} />
 
+        {/* Downtime bands. Rendered before the lines so the stroke stays
+            on top of the highlight. Contiguous null-buckets (from spec
+            `unless changes == 0`) become visible red columns. */}
+        {showDowntime && <DowntimeBands drawn={drawn} padT={padT} innerH={innerH} />}
+
         {/* Areas + lines */}
         <SeriesPaths drawn={drawn} unit={unit} />
 

src/components/time-series-chart/index.tsx

@@ -475,6 +475,7 @@ export function TimeSeriesChart({
           onZoom={setZoom}
           onToggleExclude={toggle}
           onResetExcluded={excluded.size > 0 ? reset : undefined}
+          showDowntime={benchmark.results.some((r) => r.liveStatus != null)}
         />
       )}
     </figure>

src/components/time-series-chart/series.tsx

@@ -16,6 +16,177 @@ type SeriesPathsProps = {
   unit: string;
 };
 
+type DowntimeBandsProps = {
+  drawn: DrawnLine[];
+  padT: number;
+  innerH: number;
+};
+
+/** Colored rectangles that highlight the exact window each line was
+ *  silent (contiguous gap indices), plus a high-contrast pill label
+ *  above each band naming the provider that dropped off. Pills stack
+ *  vertically when multiple providers went silent at the same time so
+ *  simultaneous outages read as separate lines instead of colliding
+ *  as unreadable overlapping text. Rendered under SeriesPaths so the
+ *  line stroke stays on top; skipped for excluded (legend-toggled)
+ *  providers so their bands disappear with the line. */
+export function DowntimeBands({ drawn, padT, innerH }: DowntimeBandsProps) {
+  // Collect every band across every visible provider before rendering.
+  // Two-pass so we can assign each band a vertical stack slot based on
+  // how many EARLIER-placed bands its X-range overlaps — labels then
+  // don't collide when three providers went dark in the same window.
+  type Band = {
+    slug: string;
+    name: string;
+    color: string;
+    x: number;
+    w: number;
+    slot: number;
+  };
+  const all: Band[] = [];
+  for (const d of drawn) {
+    if (d.excluded) continue;
+    // Only count gap runs that start AFTER the first observed sample.
+    // A leading run of nulls (Prom retention shorter than the visible
+    // window, harness started mid-range, or provider added recently)
+    // is "we didn't measure yet", not "provider was down for a week".
+    // Without this guard the 30D view on aggregator-head-lag rendered
+    // a chart-wide band during Prom's initial fill period.
+    let seenData = false;
+    let runStart: number | null = null;
+    const push = (endX: number) => {
+      if (runStart == null) return;
+      if (!seenData) {
+        runStart = null;
+        return;
+      }
+      const startX = d.pts[runStart].x;
+      all.push({
+        slug: d.slug,
+        name: d.name,
+        color: d.color,
+        x: startX,
+        w: Math.max(6, endX - startX),
+        slot: 0,
+      });
+      runStart = null;
+    };
+    for (let i = 0; i < d.pts.length; i++) {
+      const p = d.pts[i];
+      if (p.gap) {
+        if (runStart == null) runStart = i;
+      } else {
+        push(d.pts[i].x);
+        seenData = true;
+      }
+    }
+    // Trailing gap that runs to the current time — meaningful only
+    // when the series had prior data (same seenData guard).
+    if (runStart != null) push(d.pts[d.pts.length - 1].x);
+  }
+  if (all.length === 0) return null;
+
+  // Slot assignment: for each band, pick the smallest slot not used
+  // by another band whose X-range overlaps this one. Naive O(n²) is
+  // fine — a chart has at most ~20 bands in the pathological case.
+  all.sort((a, b) => a.x - b.x);
+  for (let i = 0; i < all.length; i++) {
+    const b = all[i];
+    const used = new Set<number>();
+    for (let j = 0; j < i; j++) {
+      const p = all[j];
+      if (p.x < b.x + b.w && p.x + p.w > b.x) used.add(p.slot);
+    }
+    let slot = 0;
+    while (used.has(slot)) slot++;
+    b.slot = slot;
+  }
+
+  // Pill geometry constants. Pills sit ABOVE the plot area so they
+  // never occlude data. Small padding on the container's padT keeps
+  // them within the chart frame.
+  const PILL_H = 15;
+  const PILL_GAP = 3;
+
+  return (
+    <g className="ts-downtime">
+      {/* Bands first (behind), then all pills on top so no band tint
+          can bleed onto a label from a taller-slot pill. */}
+      {all.map((b, i) => (
+        <g key={`band-${b.slug}-${i}`}>
+          <rect
+            x={b.x}
+            y={padT}
+            width={b.w}
+            height={innerH}
+            fill={b.color}
+            opacity={0.1}
+          />
+          <line
+            x1={b.x}
+            x2={b.x}
+            y1={padT}
+            y2={padT + innerH}
+            stroke={b.color}
+            strokeWidth={0.8}
+            strokeDasharray="3 3"
+            opacity={0.55}
+          />
+          <line
+            x1={b.x + b.w}
+            x2={b.x + b.w}
+            y1={padT}
+            y2={padT + innerH}
+            stroke={b.color}
+            strokeWidth={0.8}
+            strokeDasharray="3 3"
+            opacity={0.55}
+          />
+        </g>
+      ))}
+      {all.map((b, i) => {
+        const cx = b.x + b.w / 2;
+        // "NAME DATA MISSING" width estimate: ~5.5 px per char plus
+        // 14 px of horizontal padding (7 each side). "DATA MISSING"
+        // stays neutral: absence of samples could be the provider
+        // going down, our harness losing its WebSocket, or a Prom
+        // scrape failure — the pill doesn't blame either side.
+        const label = `${b.name.toUpperCase()} DATA MISSING`;
+        const pillW = Math.max(80, label.length * 5.5 + 14);
+        const pillY = padT + 2 + b.slot * (PILL_H + PILL_GAP);
+        const pillX = Math.max(0, cx - pillW / 2);
+        return (
+          <g key={`pill-${b.slug}-${i}`}>
+            <rect
+              x={pillX}
+              y={pillY}
+              width={pillW}
+              height={PILL_H}
+              rx={PILL_H / 2}
+              ry={PILL_H / 2}
+              fill={b.color}
+              opacity={0.95}
+            />
+            <text
+              x={pillX + pillW / 2}
+              y={pillY + PILL_H / 2 + 0.5}
+              textAnchor="middle"
+              dominantBaseline="middle"
+              fontFamily="var(--font-sans)"
+              fontSize="9.5"
+              fontWeight="700"
+              letterSpacing="0.06em"
+              fill="var(--color-paper, #ffffff)"
+            >
+              {label}
+            </text>
+          </g>
+        );
+      })}
+    </g>
+  );
+}
+
 export function SeriesPaths({ drawn, unit }: SeriesPathsProps) {
   return (
     <>