From 738d6533066b44c0c5bf644bb7dfaa1ebdb0f090 Mon Sep 17 00:00:00 2001 From: Florent Tapponnier Date: Wed, 15 Jul 2026 00:52:01 +0200 Subject: [PATCH 01/12] add bench 081 ws-head-latency: WS block-push lag harness + spec --- benchmarks/ws-head-latency.yml | 127 +++++++++++ harnesses/ws-head-latency/Dockerfile | 22 ++ .../ws-head-latency/cmd/script/aggregator.go | 190 ++++++++++++++++ .../ws-head-latency/cmd/script/config.go | 70 ++++++ .../ws-head-latency/cmd/script/evm_ws.go | 212 ++++++++++++++++++ .../ws-head-latency/cmd/script/loghub.go | 114 ++++++++++ harnesses/ws-head-latency/cmd/script/main.go | 38 ++++ .../ws-head-latency/cmd/script/metrics.go | 81 +++++++ .../ws-head-latency/cmd/script/solana_ws.go | 80 +++++++ harnesses/ws-head-latency/go.mod | 21 ++ harnesses/ws-head-latency/go.sum | 48 ++++ 11 files changed, 1003 insertions(+) create mode 100644 benchmarks/ws-head-latency.yml create mode 100644 harnesses/ws-head-latency/Dockerfile create mode 100644 harnesses/ws-head-latency/cmd/script/aggregator.go create mode 100644 harnesses/ws-head-latency/cmd/script/config.go create mode 100644 harnesses/ws-head-latency/cmd/script/evm_ws.go create mode 100644 harnesses/ws-head-latency/cmd/script/loghub.go create mode 100644 harnesses/ws-head-latency/cmd/script/main.go create mode 100644 harnesses/ws-head-latency/cmd/script/metrics.go create mode 100644 harnesses/ws-head-latency/cmd/script/solana_ws.go create mode 100644 harnesses/ws-head-latency/go.mod create mode 100644 harnesses/ws-head-latency/go.sum diff --git a/benchmarks/ws-head-latency.yml b/benchmarks/ws-head-latency.yml new file mode 100644 index 00000000..49a42ebd --- /dev/null +++ b/benchmarks/ws-head-latency.yml @@ -0,0 +1,127 @@ +# OpenChainBench. Bench № 081 + +slug: ws-head-latency +number: "081" +title: Fastest WebSocket newHeads push, live block-push lag across RPC providers +seo_title: "Fastest WebSocket RPC newHeads push 2026" +seo_description: "{{best_name}} pushes Ethereum newHeads first. Live WebSocket block-push lag for PublicNode, dRPC and Tenderly, measured per block against the earliest arrival in the cohort." +subtitle: Per-block WebSocket push lag in milliseconds versus the earliest provider to deliver the same Ethereum head, measured continuously from one eu-west vantage point. + +category: RPCs +status: live +metric: Block push lag +unit: ms +higher_is_better: false + +seo_intro: | + Every trading bot, MEV searcher, liquidation keeper and live dashboard + that subscribes to `eth_subscribe("newHeads")` cares about one number + first: how many milliseconds after the network produces a block does my + RPC provider actually push it to me? Marketing pages say "real-time + WebSocket streaming"; this page says who delivers block N first and by + how much the others trail. The harness holds one persistent WebSocket + per provider from the same eu-west host, timestamps every newHeads + frame on receipt, and scores each Ethereum block as a race: the + earliest arrival sets T0, every other provider's lag is its arrival + minus T0. The headline is pinned to Ethereum, where all three keyless + providers compete; the same harness also races Base heads and Solana + slots for cross-chain context. Because the measurement is relative, + the leader reads near zero by construction, so the win-rate and the + shape of the trailing distribution (p50 vs p99) are the numbers to + read, not the absolute magnitude. {{best_name}} currently leads at + {{best_p50}} (p50, 24 h) across {{count}} measured providers. + +abstract: | + We race RPC providers on WebSocket block delivery. One harness holds a + persistent `eth_subscribe("newHeads")` connection to each provider from + the same host and timestamps every head frame on receipt. Per block, + the earliest arrival across providers defines T0; each provider's + sample is its own arrival minus T0, in milliseconds. A block's cohort + stays open 5 seconds after the first arrival, then it is scored: + lag histogram per provider, a win for the earliest, and a gap counter + for any live provider that never delivered the block. The measurement + is relative to the fastest provider observed from our vantage point, + not an absolute network latency; the same-path subtraction is exactly + what makes the ordering fair. Solana runs the identical race on + `slotSubscribe` slot notifications. + +methodology: + - "Providers measured (keyless WebSocket endpoints): PublicNode (wss://ethereum-rpc.publicnode.com), dRPC (wss://eth.drpc.org), Tenderly (wss://mainnet.gateway.tenderly.co). Keyed providers (Alchemy, Infura, Chainstack) join the cohort when contributor keys are wired; the harness reads them from env vars and skips them cleanly when unset." + - "Chains: Ethereum (headline, all three providers compete), Base via newHeads and Solana via slotSubscribe as secondary cohorts. Headline queries on this page are pinned to chain=ethereum so the ranking never mixes block cadences." + - "Race scoring: the first provider to deliver block N sets T0. Each provider's sample for block N is arrival(N) minus T0 in milliseconds. The cohort closes 5 seconds after the first arrival; a provider arriving later than that is scored as a missed block (ws_block_gap_total), not as a huge latency sample." + - "Relative, not absolute: a single vantage point cannot separate its own network path from provider pipeline time, so we subtract the two arrivals over the same path instead. Consequence: the fastest provider reads ~0 by construction, and the honest readings are win rate, the trailers' lag distribution, and gap counts, not the leader's absolute number." + - "Blocks where only one provider delivered within the window emit no lag samples and no win: a one-horse race carries no relative information. Gap counters still increment for live providers that missed the block." + - "Connection discipline: one persistent WebSocket per (provider, chain), client ping every 30s, read deadline 60s, plus a 90s head watchdog that forces a reconnect when a provider keeps the socket heartbeat alive but silently drops the subscription (a failure mode we have observed in production on other benches). Reconnects use exponential backoff 2s to 60s and are counted in ws_reconnects_total." + - "Coverage score: ws_head_blocks_seen_total per provider divided by the cohort maximum over the same 24h window, clamped to 1. A provider that pushes fast but skips blocks loses on this column rather than hiding in the latency percentiles." + - "Aggregation: per-provider lag histogram (5ms to 10s exponential buckets), p50/p90/p99 via histogram_quantile over the 24h rate. Duplicate announcements of the same block on one connection are dropped; per-provider liveness is exported as ws_health (1 = connected and delivered a message in the last 120s)." + - "Cheat vector disclosure: a provider could theoretically pre-announce heads it has not fully validated to win the race. We keyed the cohort on block number as pushed; hash-level cross-checking against a reference node is on the v2 list. Reading win rate together with the gap counter already flags a provider that pushes early but wrong (retracted heads show up as cohort inconsistencies and reorg noise)." + - "Vantage point: a single eu-west host. Geography moves absolute arrival times but affects all providers over the same path; still, a provider whose nearest edge is far from eu-west is disadvantaged, and a us-east probe is the planned second vantage before any cross-region claim is made." + +findings: + - "{{best_name}} leads the Ethereum cohort at {{best_p50}} (p50, 24 h) across {{count}} measured providers. Remember the construction: the per-block winner defines zero, so the leader's near-zero p50 mostly says it wins most races; the informative numbers are the trailers' medians and the p99 tails." + - "{{name:drpc}} trails the winner by {{p50:drpc}} at the median. That is the extra time a liquidation bot or MEV searcher on dRPC's keyless endpoint waits for block N after the fastest feed already delivered it." + - "{{name:tenderly}} trails by {{p50:tenderly}} (p50, 24 h). Tenderly's gateway is built around simulation and developer tooling rather than raw head-push fan-out, and the push pipeline depth shows up in this race." + - "The p99 column is the integration-grade number: {{p99:publicnode}} / {{p99:drpc}} / {{p99:tenderly}}. A provider with a tight p50 but a multi-second p99 will occasionally hand your bot a stale head exactly when the chain is busiest." + - "Win rate and block coverage are the robust readings. A provider can look decent on median lag while silently skipping blocks; the success column (blocks seen vs cohort max) and ws_block_gap_total keep that visible." + +source: https://github.com/ChainBench/OpenChainBench/tree/main/harnesses/ws-head-latency + +expected_n: 7000 + +prometheus: + window: 24h + freshness_metric: ws_head_blocks_seen_total + +faq: + - q: "Which RPC provider pushes new Ethereum blocks first over WebSocket?" + a: "{{best_name}} currently wins most per-block races, with a median lag of {{best_p50}} versus the earliest arrival (p50, 24 h) across {{count}} measured keyless providers. The leaderboard re-scores on every Ethereum block, roughly every 12 seconds, so the answer reflects sustained live measurement, not a one-off test. Check the win rate and the trailers' p99 before wiring a latency-sensitive bot to any single provider." + - q: "Why does the fastest provider show a lag near zero?" + a: "By construction. Each block is scored as a race: the earliest arrival among providers defines T0 and every sample is measured against it, so the provider that wins most races accumulates mostly zero samples. This is deliberate. A single vantage point cannot measure absolute chain-to-client latency without a reference node co-located with block producers; it can measure, fairly, which provider delivers the same block earlier over the same network path. Read the leader's win rate and the trailers' distributions, not the leader's absolute number." + - q: "Is this absolute latency from block production to my client?" + a: "No. The number is relative lag versus the fastest provider observed from our eu-west host. Absolute delivery latency includes the propagation time from the block producer to each provider's infrastructure plus the path to you, which shifts with your geography. What transfers to your integration is the ordering and the spread: if a provider trails the winner by 800 ms at the median from our vantage point, its pipeline is structurally behind, and that gap does not disappear because you probe from another region." + - q: "Could a provider cheat by pushing heads before validating them?" + a: "In principle yes: pre-announcing an unvalidated head wins the race at the cost of occasionally pushing a block that gets reorged or retracted. That is why the bench exports more than the lag histogram. A provider that pushes early but wrong shows up as cohort inconsistencies, elevated reorg noise and gaps, and the coverage column (blocks seen versus the cohort maximum) penalises skipped or retracted blocks. Hash-level cross-checking against an independent reference node is planned for v2 of the harness." + - q: "How is WebSocket head lag measured on OpenChainBench?" + a: "One Go harness holds a persistent eth_subscribe('newHeads') WebSocket per provider from the same eu-west host, plus slotSubscribe for the Solana cohort. Every frame is timestamped on receipt with time.Now() precision. A block's cohort stays open for 5 seconds after the first arrival, then each provider's lag versus the earliest is observed into a Prometheus histogram (5 ms to 10 s exponential buckets); p50/p90/p99 come from histogram_quantile over the 24 h window. Connections carry a 90 s silent-subscription watchdog and exponential-backoff reconnects so a flaky endpoint degrades to health=0 instead of corrupting the race." + - q: "Why are only PublicNode, dRPC and Tenderly listed?" + a: "V1 measures the keyless tier: endpoints anyone can open a WebSocket to without an account, which is also the tier where claims are hardest to verify and marketing is loudest. Alchemy, Infura and Chainstack require API keys; the harness already accepts them via environment variables and they join the leaderboard once contributor keys are wired into the deployment. Keyed tiers usually run on better-provisioned infrastructure, so expect the keyless numbers here to be the floor, not the ceiling, of each vendor's performance." + +providers: + - slug: publicnode + name: PublicNode + tag: Keyless WS, wss://ethereum-rpc.publicnode.com + formula: "Median ms behind the earliest provider to push each Ethereum newHeads frame, from one persistent WebSocket per provider on the same eu-west host, histogram_quantile over 24h. The per-block winner scores 0 by construction." + queries: + p50: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="publicnode", chain="ethereum"}[24h]))) + p90: histogram_quantile(0.90, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="publicnode", chain="ethereum"}[24h]))) + p99: histogram_quantile(0.99, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="publicnode", chain="ethereum"}[24h]))) + mean: sum(rate(ws_head_lag_milliseconds_sum{provider="publicnode", chain="ethereum"}[24h])) / sum(rate(ws_head_lag_milliseconds_count{provider="publicnode", chain="ethereum"}[24h])) + success: clamp_max(sum(increase(ws_head_blocks_seen_total{provider="publicnode", chain="ethereum"}[24h])) / max(sum by (provider) (increase(ws_head_blocks_seen_total{chain="ethereum"}[24h]))), 1) + sample_size: sum(increase(ws_head_lag_milliseconds_count{provider="publicnode", chain="ethereum"}[24h])) + series: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="publicnode", chain="ethereum"}[1h]))) + + - slug: drpc + name: dRPC + tag: Keyless WS, wss://eth.drpc.org + formula: "Median ms behind the earliest provider to push each Ethereum newHeads frame, from one persistent WebSocket per provider on the same eu-west host, histogram_quantile over 24h. Lag is relative to the per-block winner." + queries: + p50: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="drpc", chain="ethereum"}[24h]))) + p90: histogram_quantile(0.90, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="drpc", chain="ethereum"}[24h]))) + p99: histogram_quantile(0.99, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="drpc", chain="ethereum"}[24h]))) + mean: sum(rate(ws_head_lag_milliseconds_sum{provider="drpc", chain="ethereum"}[24h])) / sum(rate(ws_head_lag_milliseconds_count{provider="drpc", chain="ethereum"}[24h])) + success: clamp_max(sum(increase(ws_head_blocks_seen_total{provider="drpc", chain="ethereum"}[24h])) / max(sum by (provider) (increase(ws_head_blocks_seen_total{chain="ethereum"}[24h]))), 1) + sample_size: sum(increase(ws_head_lag_milliseconds_count{provider="drpc", chain="ethereum"}[24h])) + series: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="drpc", chain="ethereum"}[1h]))) + + - slug: tenderly + name: Tenderly + tag: Keyless WS, wss://mainnet.gateway.tenderly.co + formula: "Median ms behind the earliest provider to push each Ethereum newHeads frame, from one persistent WebSocket per provider on the same eu-west host, histogram_quantile over 24h. Lag is relative to the per-block winner." + queries: + p50: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="tenderly", chain="ethereum"}[24h]))) + p90: histogram_quantile(0.90, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="tenderly", chain="ethereum"}[24h]))) + p99: histogram_quantile(0.99, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="tenderly", chain="ethereum"}[24h]))) + mean: sum(rate(ws_head_lag_milliseconds_sum{provider="tenderly", chain="ethereum"}[24h])) / sum(rate(ws_head_lag_milliseconds_count{provider="tenderly", chain="ethereum"}[24h])) + success: clamp_max(sum(increase(ws_head_blocks_seen_total{provider="tenderly", chain="ethereum"}[24h])) / max(sum by (provider) (increase(ws_head_blocks_seen_total{chain="ethereum"}[24h]))), 1) + sample_size: sum(increase(ws_head_lag_milliseconds_count{provider="tenderly", chain="ethereum"}[24h])) + series: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="tenderly", chain="ethereum"}[1h]))) diff --git a/harnesses/ws-head-latency/Dockerfile b/harnesses/ws-head-latency/Dockerfile new file mode 100644 index 00000000..63108cfc --- /dev/null +++ b/harnesses/ws-head-latency/Dockerfile @@ -0,0 +1,22 @@ +FROM golang:1.24-alpine AS builder + +WORKDIR /app +RUN apk add --no-cache git + +COPY go.mod go.sum ./ +RUN go mod download + +COPY . . + +RUN CGO_ENABLED=0 GOOS=linux go build -o /app/monitor ./cmd/script + +FROM debian:bookworm-slim + +WORKDIR /app +RUN apt-get update && apt-get install -y ca-certificates && rm -rf /var/lib/apt/lists/* + +COPY --from=builder /app/monitor /app/monitor + +EXPOSE 2112 + +CMD ["/app/monitor"] diff --git a/harnesses/ws-head-latency/cmd/script/aggregator.go b/harnesses/ws-head-latency/cmd/script/aggregator.go new file mode 100644 index 00000000..87e384bc --- /dev/null +++ b/harnesses/ws-head-latency/cmd/script/aggregator.go @@ -0,0 +1,190 @@ +package main + +import ( + "fmt" + "sort" + "sync" + "time" +) + +// Per-chain arrival cohort. +// +// Every (provider, chain) connection reports "provider P saw block N at +// time T" into the chain's aggregator. The first report for a block +// opens a collection window (settleWindow); when it expires the cohort +// is scored: +// +// T0 = earliest arrival among providers +// lag(provider) = arrival(provider) - T0 → histogram sample +// winner = provider at T0 → wins counter +// gap = provider healthy on this chain but absent from the +// cohort → gap counter +// +// The measurement is deliberately RELATIVE: one vantage point cannot +// separate its own network path from provider pipeline latency, but the +// per-block delta between providers over the same path is a fair +// ordering. The fastest provider reads ~0 by construction; the spec's +// editorial copy discloses this. + +const ( + // settleWindow is how long a block cohort stays open after the first + // arrival. 5s covers everything we've observed (worst straggler + // ~1.5s on ethereum); providers later than this are scored as a gap, + // which is the honest reading for a push feed. + settleWindow = 5 * time.Second + // staleAfter bounds "healthy": a provider absent from a cohort only + // counts as a gap when it delivered *something* on this chain within + // staleAfter (otherwise it's just disconnected, which ws_health and + // ws_reconnects_total already report). + staleAfter = 120 * time.Second +) + +type cohortEntry struct { + arrivals map[string]time.Time +} + +type aggregator struct { + chain string + + mu sync.Mutex + blocks map[int64]*cohortEntry + maxFinalized int64 + // lastMsg tracks per-provider liveness on this chain (any accepted + // head/slot). Used for gap attribution and the ws_health sweeper. + lastMsg map[string]time.Time +} + +var ( + aggregatorsMu sync.Mutex + aggregators = map[string]*aggregator{} +) + +func aggregatorFor(chain string) *aggregator { + aggregatorsMu.Lock() + defer aggregatorsMu.Unlock() + if a, ok := aggregators[chain]; ok { + return a + } + a := &aggregator{ + chain: chain, + blocks: map[int64]*cohortEntry{}, + lastMsg: map[string]time.Time{}, + } + aggregators[chain] = a + return a +} + +// record ingests one arrival. Returns false when the arrival was +// dropped (duplicate from the same provider, or a straggler for an +// already-scored block). +func (a *aggregator) record(provider string, height int64, now time.Time) bool { + a.mu.Lock() + defer a.mu.Unlock() + + a.lastMsg[provider] = now + + // Straggler for a cohort that already settled. From a push-feed + // consumer's perspective this block was missed (the gap counter was + // bumped at settle time); opening a fresh single-provider cohort + // here would fabricate a lag=0 "win", so drop it. + if height <= a.maxFinalized { + return false + } + + e, ok := a.blocks[height] + if !ok { + e = &cohortEntry{arrivals: map[string]time.Time{}} + a.blocks[height] = e + time.AfterFunc(settleWindow, func() { a.settle(height) }) + } + if _, dup := e.arrivals[provider]; dup { + return false + } + e.arrivals[provider] = now + headBlocksSeen.WithLabelValues(provider, a.chain).Inc() + return true +} + +// settle scores one block cohort and releases it. +func (a *aggregator) settle(height int64) { + a.mu.Lock() + e, ok := a.blocks[height] + if !ok { + a.mu.Unlock() + return + } + delete(a.blocks, height) + if height > a.maxFinalized { + a.maxFinalized = height + } + // Snapshot healthy providers for gap attribution while still locked. + settleTime := time.Now() + healthy := make([]string, 0, len(a.lastMsg)) + for p, t := range a.lastMsg { + if settleTime.Sub(t) <= staleAfter { + healthy = append(healthy, p) + } + } + a.mu.Unlock() + + // A single-provider cohort carries no relative information: lag + // would be 0 by definition and the "win" unearned. Count gaps for + // the healthy absentees (they truly missed a block others pushed) + // but emit no lag samples. + if len(e.arrivals) >= 2 { + providers := make([]string, 0, len(e.arrivals)) + for p := range e.arrivals { + providers = append(providers, p) + } + // Deterministic tie-break: earliest arrival wins; equal + // timestamps resolve alphabetically. + sort.Strings(providers) + winner := providers[0] + t0 := e.arrivals[winner] + for _, p := range providers[1:] { + if e.arrivals[p].Before(t0) { + winner, t0 = p, e.arrivals[p] + } + } + for _, p := range providers { + lagMs := float64(e.arrivals[p].Sub(t0).Milliseconds()) + headLagHist.WithLabelValues(p, a.chain).Observe(lagMs) + } + headWins.WithLabelValues(winner, a.chain).Inc() + fmt.Printf("[%s] block=%d cohort=%d winner=%s\n", a.chain, height, len(e.arrivals), winner) + } + + for _, p := range healthy { + if _, saw := e.arrivals[p]; !saw { + headBlockGap.WithLabelValues(p, a.chain).Inc() + } + } +} + +// startHealthSweeper flips ws_health to 0 for any (provider, chain) +// that stopped delivering messages, per the gauge's contract (1 = +// connected AND received a message in the last 120s). The connection +// goroutines set it back to 1 on every accepted message. +func startHealthSweeper() { + go func() { + t := time.NewTicker(15 * time.Second) + defer t.Stop() + for now := range t.C { + aggregatorsMu.Lock() + aggs := make([]*aggregator, 0, len(aggregators)) + for _, a := range aggregators { + aggs = append(aggs, a) + } + aggregatorsMu.Unlock() + for _, a := range aggs { + a.mu.Lock() + for p, last := range a.lastMsg { + if now.Sub(last) > staleAfter { + wsHealth.WithLabelValues(p, a.chain).Set(0) + } + } + a.mu.Unlock() + } + } + }() +} diff --git a/harnesses/ws-head-latency/cmd/script/config.go b/harnesses/ws-head-latency/cmd/script/config.go new file mode 100644 index 00000000..1c30cfaa --- /dev/null +++ b/harnesses/ws-head-latency/cmd/script/config.go @@ -0,0 +1,70 @@ +package main + +import ( + "os" + "strings" +) + +// Endpoint is one (provider, chain) WebSocket subscription. Each entry +// gets its own goroutine holding a persistent connection; the arrival +// timestamps from every endpoint on the same chain are pooled by the +// per-chain cohort (see cohort.go logic in evm_ws.go / solana_ws.go and +// the shared aggregator below). +type Endpoint struct { + Provider string + Chain string // ethereum | base | solana + Kind string // "evm" (eth_subscribe newHeads) | "solana" (slotSubscribe) + URL string +} + +// endpoints is the source of truth for the measured cohort. All default +// URLs are public and keyless, verified live at harness inception: +// +// publicnode: ethereum / base / solana all push events +// drpc: ethereum verified; base + solana declared and allowed to +// fail at runtime (per-connection reconnect loop degrades +// gracefully, never crashes the process) +// tenderly: ethereum only (no keyless base/solana WS) +// +// Optional keyed providers are appended from env vars so contributor +// keys can widen the cohort without a rebuild. Every default URL is +// also env-overridable via WS_URL__. +func endpoints() []Endpoint { + eps := []Endpoint{ + {Provider: "publicnode", Chain: "ethereum", Kind: "evm", URL: envDefault("WS_URL_PUBLICNODE_ETHEREUM", "wss://ethereum-rpc.publicnode.com")}, + {Provider: "publicnode", Chain: "base", Kind: "evm", URL: envDefault("WS_URL_PUBLICNODE_BASE", "wss://base-rpc.publicnode.com")}, + {Provider: "publicnode", Chain: "solana", Kind: "solana", URL: envDefault("WS_URL_PUBLICNODE_SOLANA", "wss://solana-rpc.publicnode.com")}, + {Provider: "drpc", Chain: "ethereum", Kind: "evm", URL: envDefault("WS_URL_DRPC_ETHEREUM", "wss://eth.drpc.org")}, + {Provider: "drpc", Chain: "base", Kind: "evm", URL: envDefault("WS_URL_DRPC_BASE", "wss://base.drpc.org")}, + {Provider: "drpc", Chain: "solana", Kind: "solana", URL: envDefault("WS_URL_DRPC_SOLANA", "wss://solana.drpc.org")}, + {Provider: "tenderly", Chain: "ethereum", Kind: "evm", URL: envDefault("WS_URL_TENDERLY_ETHEREUM", "wss://mainnet.gateway.tenderly.co")}, + {Provider: "solana-labs", Chain: "solana", Kind: "solana", URL: envDefault("WS_URL_SOLANALABS_SOLANA", "wss://api.mainnet-beta.solana.com")}, + } + // Keyed providers, skipped when the env var is unset. URLs carry the + // key inline (e.g. wss://eth-mainnet.g.alchemy.com/v2/) so they + // live in the deployment env, never in the repo. + for _, opt := range []struct{ env, provider, chain string }{ + {"WS_URL_ALCHEMY_ETHEREUM", "alchemy", "ethereum"}, + {"WS_URL_INFURA_ETHEREUM", "infura", "ethereum"}, + {"WS_URL_CHAINSTACK_ETHEREUM", "chainstack", "ethereum"}, + } { + if url := strings.TrimSpace(os.Getenv(opt.env)); url != "" { + eps = append(eps, Endpoint{Provider: opt.provider, Chain: opt.chain, Kind: "evm", URL: url}) + } + } + return eps +} + +func envDefault(key, def string) string { + if v := strings.TrimSpace(os.Getenv(key)); v != "" { + return v + } + return def +} + +// listenAddr defaults to :2112, the OCB harness convention (scraped by +// the shared Prometheus). LISTEN_ADDR overrides for local runs so a dev +// copy doesn't collide with a deployed harness. +func listenAddr() string { + return envDefault("LISTEN_ADDR", ":2112") +} diff --git a/harnesses/ws-head-latency/cmd/script/evm_ws.go b/harnesses/ws-head-latency/cmd/script/evm_ws.go new file mode 100644 index 00000000..b69fa750 --- /dev/null +++ b/harnesses/ws-head-latency/cmd/script/evm_ws.go @@ -0,0 +1,212 @@ +package main + +import ( + "encoding/json" + "fmt" + "strconv" + "strings" + "sync/atomic" + "time" + + "github.com/gorilla/websocket" +) + +// Persistent WebSocket machinery, ported from the l2-block-time +// harness. One goroutine per (provider, chain) endpoint: +// +// - dial with timeout, subscribe, read loop with rolling deadline +// - client ping every 30s; pong handler extends the read deadline +// - head watchdog: the pong handler keeps the read deadline alive, so +// a provider that heartbeats while silently dropping the +// subscription (observed on publicnode) would wedge the goroutine +// forever. If no head/slot arrives for headWatchdog, slam the +// connection closed and let the outer loop redial. +// - reconnect with exponential backoff 2s → 60s; per-connection, so a +// dead endpoint (e.g. drpc/base if delisted) degrades to health=0 + +// periodic redials without touching the rest of the cohort. + +const ( + dialTimeout = 15 * time.Second + readDeadline = 60 * time.Second + pingEvery = 30 * time.Second + minBackoff = 2 * time.Second + maxBackoff = 60 * time.Second + // headWatchdog must sit well above the slowest cadence in the panel + // (ethereum, 12s slots; solana pushes multiple times per second). + // 90s of silence is unambiguously a broken subscription. + headWatchdog = 90 * time.Second +) + +type rpcReq struct { + JSONRPC string `json:"jsonrpc"` + ID int `json:"id"` + Method string `json:"method"` + Params []any `json:"params"` +} + +// evmEnvelope matches both the eth_subscription push frame and the +// subscribe ack. +type evmEnvelope struct { + Method string `json:"method"` + Params *struct { + Result json.RawMessage `json:"result"` + } `json:"params,omitempty"` +} + +type evmHead struct { + Number string `json:"number"` + Hash string `json:"hash"` +} + +// StartEndpoints spawns the reconnect loop for every configured +// (provider, chain) connection. +func StartEndpoints() { + for _, ep := range endpoints() { + ep := ep + go func() { + backoff := minBackoff + for { + start := time.Now() + var err error + switch ep.Kind { + case "solana": + err = runSolanaConn(ep) + default: + err = runEVMConn(ep) + } + wsHealth.WithLabelValues(ep.Provider, ep.Chain).Set(0) + wsReconnects.WithLabelValues(ep.Provider, ep.Chain).Inc() + if err != nil { + fmt.Printf("[%s/%s] WS error: %v (reconnecting in %v)\n", ep.Provider, ep.Chain, err, backoff) + } + // A connection that held for a while earns a fresh + // backoff; only consecutive fast failures escalate. + if time.Since(start) > 5*time.Minute { + backoff = minBackoff + } + time.Sleep(backoff) + if backoff < maxBackoff { + backoff *= 2 + if backoff > maxBackoff { + backoff = maxBackoff + } + } + } + }() + } +} + +// dialAndGuard opens the connection and installs ping + watchdog +// goroutines shared by the EVM and Solana read loops. The returned +// cleanup must be deferred; bumpHead must be called on every accepted +// head/slot. +func dialAndGuard(ep Endpoint) (conn *websocket.Conn, bumpHead func(), cleanup func(), err error) { + dialer := *websocket.DefaultDialer + dialer.HandshakeTimeout = dialTimeout + conn, _, err = dialer.Dial(ep.URL, nil) + if err != nil { + return nil, nil, nil, fmt.Errorf("dial: %w", err) + } + + done := make(chan struct{}) + var lastHeadUnixNano atomic.Int64 + lastHeadUnixNano.Store(time.Now().UnixNano()) + + go func() { + t := time.NewTicker(pingEvery) + defer t.Stop() + for { + select { + case <-done: + return + case <-t.C: + _ = conn.WriteControl(websocket.PingMessage, nil, time.Now().Add(5*time.Second)) + } + } + }() + go func() { + t := time.NewTicker(15 * time.Second) + defer t.Stop() + for { + select { + case <-done: + return + case now := <-t.C: + silent := now.Sub(time.Unix(0, lastHeadUnixNano.Load())) + if silent > headWatchdog { + fmt.Printf("[%s/%s] watchdog: no event for %s, forcing reconnect\n", ep.Provider, ep.Chain, silent.Round(time.Second)) + _ = conn.Close() + return + } + } + } + }() + + conn.SetPongHandler(func(string) error { + _ = conn.SetReadDeadline(time.Now().Add(readDeadline)) + return nil + }) + + bumpHead = func() { lastHeadUnixNano.Store(time.Now().UnixNano()) } + cleanup = func() { + close(done) + _ = conn.Close() + } + return conn, bumpHead, cleanup, nil +} + +func runEVMConn(ep Endpoint) error { + conn, bumpHead, cleanup, err := dialAndGuard(ep) + if err != nil { + return err + } + defer cleanup() + + if err := conn.WriteJSON(rpcReq{ + JSONRPC: "2.0", ID: 1, Method: "eth_subscribe", + Params: []any{"newHeads"}, + }); err != nil { + return fmt.Errorf("subscribe: %w", err) + } + + wsHealth.WithLabelValues(ep.Provider, ep.Chain).Set(1) + fmt.Printf("[%s/%s] connected, subscribed to newHeads\n", ep.Provider, ep.Chain) + + agg := aggregatorFor(ep.Chain) + for { + _ = conn.SetReadDeadline(time.Now().Add(readDeadline)) + _, msg, err := conn.ReadMessage() + if err != nil { + return fmt.Errorf("read: %w", err) + } + + var env evmEnvelope + if err := json.Unmarshal(msg, &env); err != nil { + continue + } + if env.Method != "eth_subscription" || env.Params == nil { + continue // subscribe ack or unrelated frame + } + var head evmHead + if err := json.Unmarshal(env.Params.Result, &head); err != nil { + continue + } + blockNum := parseHexInt64(head.Number) + if blockNum == 0 { + continue + } + now := time.Now() + bumpHead() + wsHealth.WithLabelValues(ep.Provider, ep.Chain).Set(1) + agg.record(ep.Provider, blockNum, now) + } +} + +func parseHexInt64(s string) int64 { + s = strings.TrimPrefix(s, "0x") + if s == "" { + return 0 + } + n, _ := strconv.ParseInt(s, 16, 64) + return n +} diff --git a/harnesses/ws-head-latency/cmd/script/loghub.go b/harnesses/ws-head-latency/cmd/script/loghub.go new file mode 100644 index 00000000..9dd74488 --- /dev/null +++ b/harnesses/ws-head-latency/cmd/script/loghub.go @@ -0,0 +1,114 @@ +package main + +import ( + "bufio" + "fmt" + "io" + "net/http" + "os" + "strconv" + "sync" + "time" +) + +// Auto-generated by the loghub inline pattern. Captures stdout/stderr into a +// bounded ring buffer and exposes GET /logs?tail=N protected by X-Logs-Token +// matching the LOGS_TOKEN env var. +// +// Keep in sync across miniapps (was previously the shared/loghub package; we +// inline because Railway's per-harness Docker build context can't reach a +// sibling shared module via go.mod replace). + +const logRingMax = 5000 + +type logRing struct { + mu sync.Mutex + lines []string + max int +} + +var globalLogRing = &logRing{max: logRingMax} + +func (b *logRing) push(line string) { + entry := time.Now().UTC().Format("2006-01-02T15:04:05.000Z") + " " + line + b.mu.Lock() + if len(b.lines) >= b.max { + b.lines = append(b.lines[1:], entry) + } else { + b.lines = append(b.lines, entry) + } + b.mu.Unlock() +} + +func (b *logRing) snapshot(tail int) []string { + b.mu.Lock() + defer b.mu.Unlock() + if tail <= 0 || tail >= len(b.lines) { + out := make([]string, len(b.lines)) + copy(out, b.lines) + return out + } + start := len(b.lines) - tail + out := make([]string, tail) + copy(out, b.lines[start:]) + return out +} + +var logSetupOnce sync.Once + +// installLogCapture replaces os.Stdout (and os.Stderr) with the write-end of a +// pipe, then spawns a goroutine that fan-outs every line to the original +// stdout AND the in-memory ring buffer. Call exactly once, very early in +// main(). +func installLogCapture() { logSetupOnce.Do(doInstallLogCapture) } + +func doInstallLogCapture() { + originalStdout := os.Stdout + originalStderr := os.Stderr + r, w, err := os.Pipe() + if err != nil { + fmt.Fprintf(originalStdout, "[loghub] pipe failed: %v (/logs will be empty)\n", err) + return + } + os.Stdout = w + os.Stderr = w + + go func() { + scanner := bufio.NewScanner(r) + buf := make([]byte, 0, 1024*1024) + scanner.Buffer(buf, 1024*1024) + for scanner.Scan() { + line := scanner.Text() + fmt.Fprintln(originalStdout, line) + globalLogRing.push(line) + } + _, _ = io.Copy(originalStdout, r) + _ = originalStderr + }() +} + +// logsHandler returns an http.Handler for GET /logs?tail=N. Requires header +// X-Logs-Token to match the LOGS_TOKEN env var. Returns 403 if env unset. +func logsHandler() http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + expected := os.Getenv("LOGS_TOKEN") + if expected == "" { + http.Error(w, "logs disabled: LOGS_TOKEN unset", http.StatusForbidden) + return + } + if r.Header.Get("X-Logs-Token") != expected { + http.Error(w, "forbidden", http.StatusForbidden) + return + } + tail := 500 + if t := r.URL.Query().Get("tail"); t != "" { + if n, err := strconv.Atoi(t); err == nil && n > 0 { + tail = n + } + } + w.Header().Set("Content-Type", "text/plain; charset=utf-8") + for _, l := range globalLogRing.snapshot(tail) { + fmt.Fprintln(w, l) + } + }) +} diff --git a/harnesses/ws-head-latency/cmd/script/main.go b/harnesses/ws-head-latency/cmd/script/main.go new file mode 100644 index 00000000..2079aeaa --- /dev/null +++ b/harnesses/ws-head-latency/cmd/script/main.go @@ -0,0 +1,38 @@ +package main + +import ( + "fmt" + "os" + "os/signal" + "syscall" +) + +func main() { + installLogCapture() // capture stdout into /logs ring buffer + fmt.Println("=== WS Head Latency Monitor ===") + fmt.Println("Bench № 081 — per-block WebSocket push lag across RPC providers, relative to the earliest arrival in the cohort.") + fmt.Println() + + eps := endpoints() + addr := listenAddr() + fmt.Printf("Connections tracked: %d\n", len(eps)) + for _, ep := range eps { + fmt.Printf(" %-12s %-9s %s\n", ep.Provider, ep.Chain, ep.URL) + } + fmt.Printf("Metrics server: %s/metrics\n\n", addr) + + go func() { + if err := StartMetricsServer(addr); err != nil { + fmt.Printf("[fatal] metrics server: %v\n", err) + os.Exit(1) + } + }() + + StartEndpoints() + startHealthSweeper() + + sig := make(chan os.Signal, 1) + signal.Notify(sig, os.Interrupt, syscall.SIGTERM) + s := <-sig + fmt.Printf("\n[shutdown] received %v\n", s) +} diff --git a/harnesses/ws-head-latency/cmd/script/metrics.go b/harnesses/ws-head-latency/cmd/script/metrics.go new file mode 100644 index 00000000..e2940ee6 --- /dev/null +++ b/harnesses/ws-head-latency/cmd/script/metrics.go @@ -0,0 +1,81 @@ +package main + +import ( + "net/http" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" + "github.com/prometheus/client_golang/prometheus/promhttp" +) + +// Prom metrics for the ws-head-latency harness. Names follow the +// `__` OCB convention. All series carry +// {provider, chain} so the spec can pin headline queries to one chain +// while keeping base/solana series available for future tabs. +var ( + headLagHist = promauto.NewHistogramVec( + prometheus.HistogramOpts{ + Name: "ws_head_lag_milliseconds", + Help: "Per-block push lag vs the earliest provider in the cohort, in milliseconds. Relative measurement: the fastest provider on a block reads 0 by construction.", + // 5ms → 10.24s exponential; covers same-frame ties through + // multi-second stragglers. + Buckets: prometheus.ExponentialBuckets(5, 2, 12), + }, + []string{"provider", "chain"}, + ) + + headWins = promauto.NewCounterVec( + prometheus.CounterOpts{ + Name: "ws_head_wins_total", + Help: "Blocks where this provider was the first in the cohort to push the head/slot. Only counted when at least 2 providers saw the block.", + }, + []string{"provider", "chain"}, + ) + + headBlocksSeen = promauto.NewCounterVec( + prometheus.CounterOpts{ + Name: "ws_head_blocks_seen_total", + Help: "Distinct blocks/slots this provider pushed within the cohort settle window. Ratio against the cohort max is the coverage score.", + }, + []string{"provider", "chain"}, + ) + + wsReconnects = promauto.NewCounterVec( + prometheus.CounterOpts{ + Name: "ws_reconnects_total", + Help: "WebSocket reconnections per (provider, chain) since process start. Sustained non-zero rate flags an unstable endpoint.", + }, + []string{"provider", "chain"}, + ) + + wsHealth = promauto.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "ws_health", + Help: "1 when the subscription is connected and delivered a message in the last 120s, 0 otherwise.", + }, + []string{"provider", "chain"}, + ) + + headBlockGap = promauto.NewCounterVec( + prometheus.CounterOpts{ + Name: "ws_block_gap_total", + Help: "Blocks the cohort saw but this provider missed (absent from the 5s settle window while its connection was otherwise live).", + }, + []string{"provider", "chain"}, + ) +) + +// StartMetricsServer binds /metrics + /health + /logs on addr. Blocking +// call, run in its own goroutine. +func StartMetricsServer(addr string) error { + mux := http.NewServeMux() + mux.Handle("/metrics", promhttp.Handler()) + mux.Handle("/logs", logsHandler()) + mux.HandleFunc("/health", func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte("ok")) + }) + mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte("ws-head-latency harness · OpenChainBench № 081")) + }) + return http.ListenAndServe(addr, mux) +} diff --git a/harnesses/ws-head-latency/cmd/script/solana_ws.go b/harnesses/ws-head-latency/cmd/script/solana_ws.go new file mode 100644 index 00000000..6ebf8cc3 --- /dev/null +++ b/harnesses/ws-head-latency/cmd/script/solana_ws.go @@ -0,0 +1,80 @@ +package main + +import ( + "encoding/json" + "fmt" + "time" +) + +// Solana slot push via `slotSubscribe`. The notification frame looks +// like: +// +// {"jsonrpc":"2.0","method":"slotNotification", +// "params":{"result":{"parent":364,"root":362,"slot":365},"subscription":0}} +// +// The slot number is the cohort key, playing the role the block number +// plays on EVM chains. Solana slots tick every ~400ms, so the cohort +// churn is higher but the settle/score path is identical. + +type solanaEnvelope struct { + Method string `json:"method"` + Params *struct { + Result struct { + Slot int64 `json:"slot"` + } `json:"result"` + } `json:"params,omitempty"` + // Subscribe ack: {"jsonrpc":"2.0","result":23784,"id":1} + Error *struct { + Code int `json:"code"` + Message string `json:"message"` + } `json:"error,omitempty"` +} + +func runSolanaConn(ep Endpoint) error { + conn, bumpHead, cleanup, err := dialAndGuard(ep) + if err != nil { + return err + } + defer cleanup() + + if err := conn.WriteJSON(rpcReq{ + JSONRPC: "2.0", ID: 1, Method: "slotSubscribe", + Params: []any{}, + }); err != nil { + return fmt.Errorf("subscribe: %w", err) + } + + wsHealth.WithLabelValues(ep.Provider, ep.Chain).Set(1) + fmt.Printf("[%s/%s] connected, subscribed to slotSubscribe\n", ep.Provider, ep.Chain) + + agg := aggregatorFor(ep.Chain) + for { + _ = conn.SetReadDeadline(time.Now().Add(readDeadline)) + _, msg, err := conn.ReadMessage() + if err != nil { + return fmt.Errorf("read: %w", err) + } + + var env solanaEnvelope + if err := json.Unmarshal(msg, &env); err != nil { + continue + } + if env.Error != nil { + // Endpoint answered the subscribe with a JSON-RPC error + // (method disabled, quota, ...). Surface it and let the + // outer loop back off instead of holding a dead socket. + return fmt.Errorf("subscribe error %d: %s", env.Error.Code, env.Error.Message) + } + if env.Method != "slotNotification" || env.Params == nil { + continue // subscribe ack or unrelated frame + } + slot := env.Params.Result.Slot + if slot == 0 { + continue + } + now := time.Now() + bumpHead() + wsHealth.WithLabelValues(ep.Provider, ep.Chain).Set(1) + agg.record(ep.Provider, slot, now) + } +} diff --git a/harnesses/ws-head-latency/go.mod b/harnesses/ws-head-latency/go.mod new file mode 100644 index 00000000..3bca706e --- /dev/null +++ b/harnesses/ws-head-latency/go.mod @@ -0,0 +1,21 @@ +module ws-head-latency + +go 1.24.0 + +require ( + github.com/gorilla/websocket v1.5.3 + github.com/prometheus/client_golang v1.23.2 +) + +require ( + github.com/beorn7/perks v1.0.1 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/kr/text v0.2.0 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/prometheus/client_model v0.6.2 // indirect + github.com/prometheus/common v0.66.1 // indirect + github.com/prometheus/procfs v0.16.1 // indirect + go.yaml.in/yaml/v2 v2.4.2 // indirect + golang.org/x/sys v0.35.0 // indirect + google.golang.org/protobuf v1.36.8 // indirect +) diff --git a/harnesses/ws-head-latency/go.sum b/harnesses/ws-head-latency/go.sum new file mode 100644 index 00000000..02807ba4 --- /dev/null +++ b/harnesses/ws-head-latency/go.sum @@ -0,0 +1,48 @@ +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= +github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= +github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o= +github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg= +github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= +github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= +github.com/prometheus/common v0.66.1 h1:h5E0h5/Y8niHc5DlaLlWLArTQI7tMrsfQjHV+d9ZoGs= +github.com/prometheus/common v0.66.1/go.mod h1:gcaUsgf3KfRSwHY4dIMXLPV0K/Wg1oZ8+SbZk/HH/dA= +github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg= +github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is= +github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= +github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI= +go.yaml.in/yaml/v2 v2.4.2/go.mod h1:081UH+NErpNdqlCXm3TtEran0rJZGxAYx9hb/ELlsPU= +golang.org/x/sys v0.35.0 h1:vz1N37gP5bs89s7He8XuIYXpyY0+QlsKmzipCbUtyxI= +golang.org/x/sys v0.35.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +google.golang.org/protobuf v1.36.8 h1:xHScyCOEuuwZEc6UtSOvPbAT4zRh0xcNRYekJwfqyMc= +google.golang.org/protobuf v1.36.8/go.mod h1:fuxRtAxBytpl4zzqUh6/eyUujkJdNiuEkXntxiD/uRU= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= From bb503835b5d4f3db7e46741f4a385cf845a68f77 Mon Sep 17 00:00:00 2001 From: Florent Tapponnier Date: Wed, 15 Jul 2026 00:55:23 +0200 Subject: [PATCH 02/12] add bench 084 indexer-latency: HyperSync vs The Graph vs Alchemy time-to-queryable --- benchmarks/indexer-latency.yml | 127 +++++++ harnesses/indexer-latency/Dockerfile | 22 ++ .../indexer-latency/cmd/script/alchemy.go | 39 ++ .../indexer-latency/cmd/script/config.go | 96 +++++ .../indexer-latency/cmd/script/hypersync.go | 69 ++++ .../indexer-latency/cmd/script/loghub.go | 114 ++++++ harnesses/indexer-latency/cmd/script/main.go | 340 ++++++++++++++++++ .../indexer-latency/cmd/script/metrics.go | 71 ++++ .../indexer-latency/cmd/script/thegraph.go | 69 ++++ harnesses/indexer-latency/go.mod | 18 + harnesses/indexer-latency/go.sum | 46 +++ 11 files changed, 1011 insertions(+) create mode 100644 benchmarks/indexer-latency.yml create mode 100644 harnesses/indexer-latency/Dockerfile create mode 100644 harnesses/indexer-latency/cmd/script/alchemy.go create mode 100644 harnesses/indexer-latency/cmd/script/config.go create mode 100644 harnesses/indexer-latency/cmd/script/hypersync.go create mode 100644 harnesses/indexer-latency/cmd/script/loghub.go create mode 100644 harnesses/indexer-latency/cmd/script/main.go create mode 100644 harnesses/indexer-latency/cmd/script/metrics.go create mode 100644 harnesses/indexer-latency/cmd/script/thegraph.go create mode 100644 harnesses/indexer-latency/go.mod create mode 100644 harnesses/indexer-latency/go.sum diff --git a/benchmarks/indexer-latency.yml b/benchmarks/indexer-latency.yml new file mode 100644 index 00000000..23419275 --- /dev/null +++ b/benchmarks/indexer-latency.yml @@ -0,0 +1,127 @@ +# OpenChainBench. Bench № 084 + +slug: indexer-latency +number: "084" +title: Fastest blockchain indexer. Envio HyperSync, The Graph, Alchemy +seo_title: "Fastest blockchain indexer 2026: HyperSync vs The Graph vs Alchemy" +seo_description: "How fast do indexing pipelines make new onchain events queryable? Envio HyperSync, The Graph and Alchemy measured live on organic Ethereum USDC transfers, second by second." +subtitle: Seconds between a USDC transfer confirming on Ethereum and the moment each indexing pipeline first makes that block's data queryable, measured continuously on organic transfers. +category: Aggregators +status: live +metric: Time to queryable +unit: s +higher_is_better: false + +seo_intro: | + Every dapp backend, analytics product and trading system sits on an + indexing pipeline, and in 2026 the choice of that pipeline is a live + question again. Dune is sunsetting Sim, teams are re-evaluating + their data stack, and Envio markets HyperSync as "10x faster" than + alternatives without publishing comparable numbers. This benchmark + measures the claim the only way that cannot be gamed. we pick a + random, organic USDC transfer from the newest Ethereum block, then + poll each indexing pipeline until that block's data is queryable. + Because the ground truth is a random real transaction there is + nothing a provider could special case, and every measurement is + publicly re-verifiable from the tx hash. Cohort. Envio HyperSync + (low level log query engine), The Graph (decentralized subgraph + network, probed through the canonical Uniswap V3 subgraph) and + Alchemy (indexed transfers API), all probed on the identical + schedule from the same host. This cohort is indexing + infrastructure; wallet data APIs (Zerion, Moralis, Mobula and + friends) are raced separately in bench 070, with zero provider + overlap between the two. + +abstract: | + We measure time to queryable for indexing infrastructure. the delay + between an organic USDC transfer confirming on Ethereum (T0 = the + instant our own keyless reference RPC first shows the containing + block) and the first poll at which each provider can return that + block's data. One probe event per 5 minutes; each event uses a fresh + random transfer from a fresh block. The poll schedule is front + loaded (1s to 240s, identical for every provider) and detection is + parser free where the API returns the tx hash verbatim. Because the + three pipelines expose different query surfaces, the probed entity + differs per provider and is disclosed. HyperSync is asked for the + exact Transfer log, Alchemy for the exact transfer via its indexed + transfers API, The Graph for a block pinned query on the canonical + Uniswap V3 subgraph that only succeeds once the subgraph has indexed + the reference block. The measured question is identical in all three + cases. is block N data queryable at time T? Events not queryable + within 240 seconds count as timeouts and feed the reliability + column. Per provider monthly quota guards keep the probe inside + every free tier. + +methodology: + - "Ground truth: one probe event per 5 minutes. The harness watches new Ethereum blocks through its own keyless reference RPC (publicnode, never a cohort member's endpoint) and picks one random USDC Transfer log from the newest block. T0 is the instant the harness first observes the containing block; the same T0 is used for every provider." + - "Anti-gaming by construction: the probe target is a random organic USDC transfer, different on every event, so no provider can pre-warm a known benchmark entity. Every measurement is re-verifiable by anyone from the public tx hash. No canary contract is deployed; the benchmark piggybacks on the natural high frequency USDC stream at zero onchain cost." + - "Poll schedule: each provider is polled at 1, 2, 3, 5, 8, 12, 20, 30, 45, 60, 90, 120, 180 and 240 seconds after T0. Reported lag is the first poll at which block N's data is queryable, an upper bound with resolution equal to the gap between consecutive polls." + - "Probed entity per provider, disclosed because the pipelines expose different query surfaces: HyperSync is queried via POST /query for the exact Transfer log in block N; Alchemy via alchemy_getAssetTransfers with a fromBlock=N filter and the USDC contract; The Graph via a block pinned GraphQL query on the canonical Uniswap V3 Ethereum subgraph, which graph-node rejects until the subgraph has indexed block N. The measured question is identical: is block N data queryable at time T." + - "The Graph number therefore includes the full subgraph pipeline (firehose, indexer sync, gateway routing) rather than a raw data lookup. That is the honest comparison: it is the path every subgraph consumer actually waits on, but readers should know it is a pipeline measurement, not a single database read." + - "Detection is parser free where possible: HyperSync and Alchemy responses are scanned for the tx hash substring, so no provider gains or loses from response schema differences. The Graph probe succeeds when the block pinned query returns data without a block range error." + - "Classification: found (lag recorded), timeout (not queryable within 240s), error (provider errored on every poll of the event), quota_paused (monthly guard tripped), disabled (credential not configured). Percentiles are computed from the histogram of found lags over 24h; the timeout rate is published alongside because latency without reliability is a misleading ranking signal." + - "Quota fairness: per provider monthly call budgets sized to each free tier with headroom, enforced by a guard that pauses probing at 90 percent. The Graph joins every second event (100k gateway queries/month free plan); HyperSync and Alchemy join every event. Sample sizes per provider are published." + - "Excluded from v1 with reasons: Goldsky (requires deploying and paying for your own subgraph, no neutral shared surface to probe), Dune (2500 free credits/month cannot sustain continuous probing), Bitquery (free tier is a one month trial), Moralis (already raced in bench 070 and its free key is saturated there), SubQuery and Ponder (no measurable hosted free tier)." + - "Cohort positioning: this bench races indexing infrastructure. Wallet data APIs (Zerion, Moralis, Allium, GoldRush, Mobula) are a different product layer and are raced separately in bench 070 with an identical T0 and poll design; the two cohorts share zero providers by construction." + - "Scope: Ethereum mainnet, single probe region. Time to queryable is measured in seconds while cross region network deltas are milliseconds, so multi region probing would add cost without changing the ranking." + +findings: + - "{{best_name}} currently leads at {{best_p50}} (p50 of found lags, 24h) on organic Ethereum USDC transfers." + - "{{name:hypersync}} sits at {{p50:hypersync}}. Envio's marketing claims 10x faster retrieval than alternatives; this page is the first continuously measured public number behind that claim." + - "{{name:thegraph}} runs at {{p50:thegraph}} through the canonical Uniswap V3 subgraph. This is a full pipeline measurement (firehose, indexer, gateway), the path every subgraph consumer actually waits on." + - "{{name:alchemy}} sits at {{p50:alchemy}} on its indexed transfers API, the same surface portfolio and activity feeds built on Alchemy consume." + - "Read the reliability column before the latency one: a pipeline that occasionally never surfaces an event within 240 seconds breaks downstream consumers in a way a slow median does not." + +faq: + - q: "Which blockchain indexer makes new events queryable fastest?" + a: "Per the live leaderboard above: {{best_name}} at {{best_p50}} (p50 over 24h of organic Ethereum USDC transfers). The ranking re-sorts continuously as probe events land every 5 minutes. Check the reliability column too; a pipeline that occasionally never surfaces an event within 240 seconds is worse for most consumers than one that is a second slower on median." + - q: "How is indexer latency measured here?" + a: "The harness picks a random organic USDC transfer from the newest Ethereum block, records T0 when its own keyless reference RPC first shows the block, then polls each provider on an identical front loaded schedule (1s to 240s) until that block's data is queryable. HyperSync and Alchemy are asked for the exact transfer; The Graph is asked a block pinned query on the canonical Uniswap V3 subgraph that only succeeds once block N is indexed. The full harness is open source and each measurement can be re-verified from the public tx hash." + - q: "Is comparing HyperSync, The Graph and Alchemy apples to apples?" + a: "Not perfectly, and the asymmetry is disclosed rather than hidden. HyperSync and Alchemy serve raw or lightly transformed data; The Graph serves subgraph entities, so its number includes the whole firehose plus indexer plus gateway pipeline. All three answer the same operational question a builder has: after an event confirms onchain, how long until my query layer can see it. That is the number this page publishes, per pipeline, with the probed entity documented." + - q: "Why are Goldsky, Dune, SubQuery and Ponder missing?" + a: "Each lacks a neutral, continuously probeable free surface in v1. Goldsky requires deploying your own subgraph, so any probe would measure our deployment, not the platform baseline. Dune's free tier (2500 credits/month) cannot sustain a 5 minute cadence. Bitquery's free tier is a one month trial. SubQuery and Ponder have no measurable hosted free tier. Moralis is already raced in the wallet data cohort of bench 070. Providers change tiers often; the cohort is revisited when a fair probe becomes possible." + - q: "Why does indexer latency matter in 2026?" + a: "Because the indexing market is consolidating and re-forming at once. Dune is sunsetting Sim, pushing teams to re-evaluate their data stack, while Envio advertises HyperSync as 10x faster with no public comparable numbers. For trading systems, alerting and activity feeds, the seconds between confirmation and queryability are the product. This page is the only continuously measured, provider neutral comparison of that number across indexing infrastructure." + +source: https://github.com/ChainBench/OpenChainBench/tree/main/harnesses/indexer-latency + +prometheus: + window: 24h + freshness_metric: idx_health + expected_freshness_seconds: 1800 + +providers: + - slug: hypersync + name: Envio HyperSync + tag: Low level log query engine, claims 10x faster retrieval + formula: "Median (p50) over 24h of seconds between our reference RPC first showing an Ethereum block and HyperSync /query returning that block's probe USDC Transfer log, from the shared histogram estimator." + queries: + p50: histogram_quantile(0.5, sum by (le) (increase(idx_latency_seconds_bucket{provider="hypersync"}[24h]))) + p90: histogram_quantile(0.9, sum by (le) (increase(idx_latency_seconds_bucket{provider="hypersync"}[24h]))) + p99: histogram_quantile(0.99, sum by (le) (increase(idx_latency_seconds_bucket{provider="hypersync"}[24h]))) + success: sum(increase(idx_probe_total{provider="hypersync", result="found"}[24h])) / sum(increase(idx_probe_total{provider="hypersync", result=~"found|timeout"}[24h])) + sample_size: sum(increase(idx_probe_total{provider="hypersync", result=~"found|timeout"}[24h])) + series: avg_over_time(idx_latency_last_seconds{provider="hypersync"}[1h]) + - slug: thegraph + name: The Graph + tag: Decentralized subgraph network, probed via Uniswap V3 subgraph + formula: "Median (p50) over 24h of seconds until a block pinned query on the canonical Uniswap V3 subgraph succeeds for the reference block. Includes the full firehose, indexer and gateway pipeline; joins every second event." + queries: + p50: histogram_quantile(0.5, sum by (le) (increase(idx_latency_seconds_bucket{provider="thegraph"}[24h]))) + p90: histogram_quantile(0.9, sum by (le) (increase(idx_latency_seconds_bucket{provider="thegraph"}[24h]))) + p99: histogram_quantile(0.99, sum by (le) (increase(idx_latency_seconds_bucket{provider="thegraph"}[24h]))) + success: sum(increase(idx_probe_total{provider="thegraph", result="found"}[24h])) / sum(increase(idx_probe_total{provider="thegraph", result=~"found|timeout"}[24h])) + sample_size: sum(increase(idx_probe_total{provider="thegraph", result=~"found|timeout"}[24h])) + series: avg_over_time(idx_latency_last_seconds{provider="thegraph"}[1h]) + - slug: alchemy + name: Alchemy + tag: Indexed transfers API on the Alchemy data platform + formula: "Median (p50) over 24h of seconds between our reference RPC first showing a block and alchemy_getAssetTransfers returning that block's probe USDC transfer with a fromBlock filter, from the shared histogram estimator." + queries: + p50: histogram_quantile(0.5, sum by (le) (increase(idx_latency_seconds_bucket{provider="alchemy"}[24h]))) + p90: histogram_quantile(0.9, sum by (le) (increase(idx_latency_seconds_bucket{provider="alchemy"}[24h]))) + p99: histogram_quantile(0.99, sum by (le) (increase(idx_latency_seconds_bucket{provider="alchemy"}[24h]))) + success: sum(increase(idx_probe_total{provider="alchemy", result="found"}[24h])) / sum(increase(idx_probe_total{provider="alchemy", result=~"found|timeout"}[24h])) + sample_size: sum(increase(idx_probe_total{provider="alchemy", result=~"found|timeout"}[24h])) + series: avg_over_time(idx_latency_last_seconds{provider="alchemy"}[1h]) diff --git a/harnesses/indexer-latency/Dockerfile b/harnesses/indexer-latency/Dockerfile new file mode 100644 index 00000000..63108cfc --- /dev/null +++ b/harnesses/indexer-latency/Dockerfile @@ -0,0 +1,22 @@ +FROM golang:1.24-alpine AS builder + +WORKDIR /app +RUN apk add --no-cache git + +COPY go.mod go.sum ./ +RUN go mod download + +COPY . . + +RUN CGO_ENABLED=0 GOOS=linux go build -o /app/monitor ./cmd/script + +FROM debian:bookworm-slim + +WORKDIR /app +RUN apt-get update && apt-get install -y ca-certificates && rm -rf /var/lib/apt/lists/* + +COPY --from=builder /app/monitor /app/monitor + +EXPOSE 2112 + +CMD ["/app/monitor"] diff --git a/harnesses/indexer-latency/cmd/script/alchemy.go b/harnesses/indexer-latency/cmd/script/alchemy.go new file mode 100644 index 00000000..f7c77525 --- /dev/null +++ b/harnesses/indexer-latency/cmd/script/alchemy.go @@ -0,0 +1,39 @@ +package main + +import ( + "fmt" + "net/http" + "strings" +) + +// Alchemy poller. +// +// Probed entity: the exact USDC Transfer (matched by tx hash) via +// alchemy_getAssetTransfers with fromBlock=toBlock=N and the USDC +// contract filter. This is Alchemy's indexed transfers pipeline (not a +// raw eth_getLogs proxy), which is what apps built on Alchemy's data +// APIs actually consume. ~150 CU/call on the 30M CU/month free tier. +// +// The key rides in the URL path per Alchemy convention; sanitize() +// strips URLs from any logged error so it can never leak. + +func checkAlchemy(bn uint64, txHash string) (bool, error) { + apiCalls.WithLabelValues("alchemy").Inc() + blk := fmt.Sprintf("0x%x", bn) + body := fmt.Sprintf( + `{"jsonrpc":"2.0","id":1,"method":"alchemy_getAssetTransfers","params":[{"fromBlock":"%s","toBlock":"%s","contractAddresses":["%s"],"category":["erc20"],"maxCount":"0x3e8","withMetadata":false,"excludeZeroValue":false}]}`, + blk, blk, usdcContract, + ) + u := "https://eth-mainnet.g.alchemy.com/v2/" + alchemyAPIKey() + req, err := http.NewRequest("POST", u, strings.NewReader(body)) + if err != nil { + return false, err + } + req.Header.Set("Content-Type", "application/json") + req.Header.Set("User-Agent", "OpenChainBench/1.0 (+https://openchainbench.com)") + resp, err := httpClient.Do(req) + if err != nil { + return false, err + } + return bodyContains(resp, txHash) +} diff --git a/harnesses/indexer-latency/cmd/script/config.go b/harnesses/indexer-latency/cmd/script/config.go new file mode 100644 index 00000000..dd931b58 --- /dev/null +++ b/harnesses/indexer-latency/cmd/script/config.go @@ -0,0 +1,96 @@ +package main + +import ( + "os" + "strconv" + "strings" + "time" +) + +// indexer-latency — bench №084. +// +// Cohort: indexing INFRASTRUCTURE (HyperSync, The Graph, Alchemy raw +// data API). Deliberately zero provider overlap with bench 070 +// (indexing-freshness), which races wallet-data APIs (Zerion, Moralis, +// Allium, GoldRush, Mobula). Same measured question, different layer of +// the stack: how fast does new onchain data become queryable through +// each indexing pipeline. + +// Provider is one indexing pipeline we race. Enabled reports whether +// the required credential is present; a provider without one runs in +// the "disabled" state (counted, health=0) so the harness degrades +// gracefully instead of exiting. +// +// EveryN throttles participation for quota-tight free tiers: the +// provider only joins every Nth probe event. Budget is the monthly +// API-call budget for the guard (90% cutoff, calendar-month reset). +type Provider struct { + Slug string + EveryN int + Budget int64 + Enabled func() bool + // Check asks the provider whether the probe event (USDC Transfer + // tx in block bn) is queryable yet. The probed entity differs per + // provider (raw log vs subgraph block head) — disclosed in the + // bench methodology; the question is identical: "is block N data + // queryable at time T?". + Check func(bn uint64, txHash string) (bool, error) +} + +var cohort = []Provider{ + // HyperSync free tier (token required since Nov 2025) is sized in + // the hundreds of millions of rows/month; our row volume is tiny, + // so the guard is request-count based with wide headroom. + {Slug: "hypersync", EveryN: 1, Budget: 400_000, Enabled: func() bool { return hypersyncToken() != "" }, Check: checkHyperSync}, + // The Graph free plan: 100k gateway queries/month. EveryN=2 on the + // 5-min event cadence = 1 probe event / 10 min → 144 events/day × + // 14 polls ≈ 60k queries/month worst case, inside the 90k budget. + {Slug: "thegraph", EveryN: 2, Budget: 90_000, Enabled: func() bool { return graphAPIKey() != "" }, Check: checkTheGraph}, + // Alchemy free tier: 30M CU/month; alchemy_getAssetTransfers ≈ 150 + // CU/call. 288 events/day × 14 polls ≈ 121k calls/month ≈ 18M CU. + // Budget 180k calls ≈ 27M CU keeps the guard inside the tier. + {Slug: "alchemy", EveryN: 1, Budget: 180_000, Enabled: func() bool { return alchemyAPIKey() != "" }, Check: checkAlchemy}, +} + +const chainSlug = "ethereum" + +// USDC on Ethereum mainnet + the ERC-20 Transfer topic — the natural +// high-frequency event stream we piggyback on (no canary contract, $0). +const ( + usdcContract = "0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48" + transferTopic = "0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef" +) + +func hypersyncToken() string { return strings.TrimSpace(os.Getenv("HYPERSYNC_TOKEN")) } +func graphAPIKey() string { return strings.TrimSpace(os.Getenv("GRAPH_API_KEY")) } +func alchemyAPIKey() string { return strings.TrimSpace(os.Getenv("ALCHEMY_API_KEY")) } + +// rpcHTTP: OUR keyless reference RPC. T0 = the instant it first shows +// the block containing the probe event. publicnode by default; never a +// cohort member's RPC (that would gift them a head start). +func rpcHTTP() string { + if v := strings.TrimSpace(os.Getenv("INDEXER_RPC_HTTP")); v != "" { + return v + } + return "https://ethereum-rpc.publicnode.com" +} + +// eventInterval: one probe event (fresh USDC Transfer picked from a new +// block) per interval. 5 min default → 288 events/day, which keeps +// every provider inside its monthly free quota given the poll schedule +// (The Graph additionally joins only every 2nd event). +func eventInterval() time.Duration { + if v := strings.TrimSpace(os.Getenv("IDX_EVENT_SECONDS")); v != "" { + if n, err := strconv.Atoi(v); err == nil && n >= 30 { + return time.Duration(n) * time.Second + } + } + return 5 * time.Minute +} + +// pollSchedule: seconds after T0 at which each provider is polled. +// Front-loaded because HyperSync claims sub-second freshness while +// subgraph pipelines can lag minutes; capped at 240s (< event interval) +// after which the event counts as "timeout". Measured lag is an upper +// bound with resolution equal to the gap between consecutive polls. +var pollSchedule = []int{1, 2, 3, 5, 8, 12, 20, 30, 45, 60, 90, 120, 180, 240} diff --git a/harnesses/indexer-latency/cmd/script/hypersync.go b/harnesses/indexer-latency/cmd/script/hypersync.go new file mode 100644 index 00000000..2dc721be --- /dev/null +++ b/harnesses/indexer-latency/cmd/script/hypersync.go @@ -0,0 +1,69 @@ +package main + +import ( + "encoding/json" + "fmt" + "io" + "net/http" + "strings" +) + +// Envio HyperSync poller. +// +// Probed entity: the exact USDC Transfer log (matched by tx hash) via +// POST /query scoped to [bn, bn+1). While HyperSync's archive height is +// below bn the query legitimately returns no logs (next_block <= bn), +// so substring detection on the tx hash is a correct "queryable yet?" +// signal. /query requires a free bearer token since Nov 2025 +// (HYPERSYNC_TOKEN); without it the provider runs disabled. + +const hypersyncBase = "https://eth.hypersync.xyz" + +func checkHyperSync(bn uint64, txHash string) (bool, error) { + apiCalls.WithLabelValues("hypersync").Inc() + body := fmt.Sprintf( + `{"from_block":%d,"to_block":%d,"logs":[{"address":["%s"],"topics":[["%s"]]}],"field_selection":{"log":["transaction_hash","block_number","log_index"]}}`, + bn, bn+1, usdcContract, transferTopic, + ) + req, err := http.NewRequest("POST", hypersyncBase+"/query", strings.NewReader(body)) + if err != nil { + return false, err + } + req.Header.Set("Authorization", "Bearer "+hypersyncToken()) + req.Header.Set("Content-Type", "application/json") + req.Header.Set("User-Agent", "OpenChainBench/1.0 (+https://openchainbench.com)") + resp, err := httpClient.Do(req) + if err != nil { + return false, err + } + return bodyContains(resp, txHash) +} + +// hypersyncHeight hits the keyless /height endpoint. Used at startup as +// a reachability proof (and logged) even when HYPERSYNC_TOKEN is unset. +func hypersyncHeight() (uint64, error) { + req, err := http.NewRequest("GET", hypersyncBase+"/height", nil) + if err != nil { + return 0, err + } + req.Header.Set("User-Agent", "OpenChainBench/1.0 (+https://openchainbench.com)") + resp, err := httpClient.Do(req) + if err != nil { + return 0, err + } + defer resp.Body.Close() + raw, err := io.ReadAll(io.LimitReader(resp.Body, 1<<20)) + if err != nil { + return 0, err + } + if resp.StatusCode != 200 { + return 0, fmt.Errorf("status %d", resp.StatusCode) + } + var out struct { + Height uint64 `json:"height"` + } + if err := json.Unmarshal(raw, &out); err != nil { + return 0, err + } + return out.Height, nil +} diff --git a/harnesses/indexer-latency/cmd/script/loghub.go b/harnesses/indexer-latency/cmd/script/loghub.go new file mode 100644 index 00000000..9dd74488 --- /dev/null +++ b/harnesses/indexer-latency/cmd/script/loghub.go @@ -0,0 +1,114 @@ +package main + +import ( + "bufio" + "fmt" + "io" + "net/http" + "os" + "strconv" + "sync" + "time" +) + +// Auto-generated by the loghub inline pattern. Captures stdout/stderr into a +// bounded ring buffer and exposes GET /logs?tail=N protected by X-Logs-Token +// matching the LOGS_TOKEN env var. +// +// Keep in sync across miniapps (was previously the shared/loghub package; we +// inline because Railway's per-harness Docker build context can't reach a +// sibling shared module via go.mod replace). + +const logRingMax = 5000 + +type logRing struct { + mu sync.Mutex + lines []string + max int +} + +var globalLogRing = &logRing{max: logRingMax} + +func (b *logRing) push(line string) { + entry := time.Now().UTC().Format("2006-01-02T15:04:05.000Z") + " " + line + b.mu.Lock() + if len(b.lines) >= b.max { + b.lines = append(b.lines[1:], entry) + } else { + b.lines = append(b.lines, entry) + } + b.mu.Unlock() +} + +func (b *logRing) snapshot(tail int) []string { + b.mu.Lock() + defer b.mu.Unlock() + if tail <= 0 || tail >= len(b.lines) { + out := make([]string, len(b.lines)) + copy(out, b.lines) + return out + } + start := len(b.lines) - tail + out := make([]string, tail) + copy(out, b.lines[start:]) + return out +} + +var logSetupOnce sync.Once + +// installLogCapture replaces os.Stdout (and os.Stderr) with the write-end of a +// pipe, then spawns a goroutine that fan-outs every line to the original +// stdout AND the in-memory ring buffer. Call exactly once, very early in +// main(). +func installLogCapture() { logSetupOnce.Do(doInstallLogCapture) } + +func doInstallLogCapture() { + originalStdout := os.Stdout + originalStderr := os.Stderr + r, w, err := os.Pipe() + if err != nil { + fmt.Fprintf(originalStdout, "[loghub] pipe failed: %v (/logs will be empty)\n", err) + return + } + os.Stdout = w + os.Stderr = w + + go func() { + scanner := bufio.NewScanner(r) + buf := make([]byte, 0, 1024*1024) + scanner.Buffer(buf, 1024*1024) + for scanner.Scan() { + line := scanner.Text() + fmt.Fprintln(originalStdout, line) + globalLogRing.push(line) + } + _, _ = io.Copy(originalStdout, r) + _ = originalStderr + }() +} + +// logsHandler returns an http.Handler for GET /logs?tail=N. Requires header +// X-Logs-Token to match the LOGS_TOKEN env var. Returns 403 if env unset. +func logsHandler() http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + expected := os.Getenv("LOGS_TOKEN") + if expected == "" { + http.Error(w, "logs disabled: LOGS_TOKEN unset", http.StatusForbidden) + return + } + if r.Header.Get("X-Logs-Token") != expected { + http.Error(w, "forbidden", http.StatusForbidden) + return + } + tail := 500 + if t := r.URL.Query().Get("tail"); t != "" { + if n, err := strconv.Atoi(t); err == nil && n > 0 { + tail = n + } + } + w.Header().Set("Content-Type", "text/plain; charset=utf-8") + for _, l := range globalLogRing.snapshot(tail) { + fmt.Fprintln(w, l) + } + }) +} diff --git a/harnesses/indexer-latency/cmd/script/main.go b/harnesses/indexer-latency/cmd/script/main.go new file mode 100644 index 00000000..0bccfe70 --- /dev/null +++ b/harnesses/indexer-latency/cmd/script/main.go @@ -0,0 +1,340 @@ +package main + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "math/rand" + "net/http" + "os" + "os/signal" + "strconv" + "strings" + "sync" + "syscall" + "time" +) + +// indexer-latency — bench №084. +// +// How fast does indexing infrastructure make new onchain events +// queryable? One probe event per interval: pick a fresh USDC Transfer +// from the newest Ethereum block via our keyless reference RPC +// (T0 = the instant WE first see the block), then poll each indexing +// pipeline on a front-loaded schedule until block N's data is +// queryable (lag = poll time − T0) or 240s passes (timeout). +// +// $0 methodology: no canary contract, we piggyback on the natural +// high-frequency USDC stream. Ground truth is a random organic +// transfer, different every event, publicly re-verifiable by tx hash. + +func main() { + installLogCapture() // capture stdout into /logs ring buffer + + fmt.Println("=== Indexer Latency Harness (bench 084) ===") + fmt.Println("OpenChainBench — onchain event → indexer time-to-queryable.") + fmt.Printf("Reference RPC: %s\n", rpcHTTP()) + + // Keyless reachability proof: HyperSync /height vs our RPC head. + if h, err := hypersyncHeight(); err == nil { + fmt.Printf("HyperSync /height (keyless): %d\n", h) + } else { + fmt.Printf("HyperSync /height (keyless): unreachable (%s)\n", sanitize(err)) + } + + enabled := 0 + fmt.Printf("Chain: %s | event interval: %s | poll cap: %ds\n", chainSlug, eventInterval(), pollSchedule[len(pollSchedule)-1]) + for _, p := range cohort { + state := "disabled (credential missing)" + if p.Enabled() { + state = fmt.Sprintf("enabled everyN=%d budget=%d calls/mo", p.EveryN, p.Budget) + enabled++ + } + fmt.Printf(" - %-10s %s\n", p.Slug, state) + } + if enabled == 0 { + fmt.Println("[warn] no provider credentials set (HYPERSYNC_TOKEN / GRAPH_API_KEY / ALCHEMY_API_KEY)") + fmt.Println("[warn] running in degraded mode: events sampled, all providers report result=disabled") + } + + addr := ":2112" + if v := strings.TrimSpace(os.Getenv("METRICS_ADDR")); v != "" { + addr = v + } + fmt.Printf("Metrics server: %s/metrics\n\n", addr) + go func() { + if err := StartMetricsServer(addr); err != nil { + fmt.Printf("[fatal] metrics server: %v\n", err) + os.Exit(1) + } + }() + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + go runEvents(ctx) + + sig := make(chan os.Signal, 1) + signal.Notify(sig, os.Interrupt, syscall.SIGTERM) + s := <-sig + fmt.Printf("\n[shutdown] %v\n", s) + cancel() +} + +func runEvents(ctx context.Context) { + t := time.NewTicker(eventInterval()) + defer t.Stop() + var lastSeen uint64 + eventN := 0 + for { + eventN++ + bn, txHash, t0 := waitFreshEvent(ctx, lastSeen) + if bn == 0 { + return // ctx cancelled + } + lastSeen = bn + fmt.Printf("[event %d] block=%d usdc-transfer tx=%s\n", eventN, bn, txHash[:14]+"…") + + var wg sync.WaitGroup + for _, p := range cohort { + if eventN%p.EveryN != 0 { + continue + } + if !p.Enabled() { + probeTotal.WithLabelValues(p.Slug, "disabled").Inc() + health.WithLabelValues(p.Slug).Set(0) + fmt.Printf(" %-10s disabled (credential missing)\n", p.Slug) + continue + } + if !quota.allow(p) { + probeTotal.WithLabelValues(p.Slug, "quota_paused").Inc() + fmt.Printf(" %-10s quota guard tripped — paused until month rollover\n", p.Slug) + continue + } + wg.Add(1) + go func(p Provider) { + defer wg.Done() + raceProvider(ctx, p, bn, txHash, t0) + }(p) + } + wg.Wait() + + select { + case <-ctx.Done(): + return + case <-t.C: + } + } +} + +// raceProvider polls one indexing pipeline on the shared front-loaded +// schedule until block bn's probe event is queryable, or the schedule +// is exhausted (timeout), or every poll errored (error). +func raceProvider(ctx context.Context, p Provider, bn uint64, txHash string, t0 time.Time) { + var lastErr error + for _, after := range pollSchedule { + wait := time.Until(t0.Add(time.Duration(after) * time.Second)) + if wait > 0 { + select { + case <-ctx.Done(): + return + case <-time.After(wait): + } + } + found, err := p.Check(bn, txHash) + if err != nil { + lastErr = err + continue + } + if found { + lag := time.Since(t0).Seconds() + latencyHist.WithLabelValues(p.Slug).Observe(lag) + latencyLast.WithLabelValues(p.Slug).Set(lag) + probeTotal.WithLabelValues(p.Slug, "found").Inc() + health.WithLabelValues(p.Slug).Set(1) + fmt.Printf(" %-10s queryable in %.1fs\n", p.Slug, lag) + return + } + } + health.WithLabelValues(p.Slug).Set(0) + if lastErr != nil { + probeTotal.WithLabelValues(p.Slug, "error").Inc() + fmt.Printf(" %-10s error: %s\n", p.Slug, sanitize(lastErr)) + return + } + probeTotal.WithLabelValues(p.Slug, "timeout").Inc() + fmt.Printf(" %-10s TIMEOUT (not queryable within %ds)\n", p.Slug, pollSchedule[len(pollSchedule)-1]) +} + +// --------------------------------------------------------------------------- +// Reference sampler: T0 from OUR keyless RPC (publicnode), never a +// cohort member's endpoint. 500ms head polling bounds the T0 error at +// +500ms, identical for every provider. +// --------------------------------------------------------------------------- + +var httpClient = &http.Client{Timeout: 8 * time.Second} + +func rpcCall(method, params string) (json.RawMessage, error) { + body := fmt.Sprintf(`{"jsonrpc":"2.0","method":"%s","params":%s,"id":%d}`, method, params, time.Now().UnixNano()) + req, err := http.NewRequest("POST", rpcHTTP(), strings.NewReader(body)) + if err != nil { + return nil, err + } + req.Header.Set("Content-Type", "application/json") + req.Header.Set("User-Agent", "OpenChainBench/1.0 (+https://openchainbench.com)") + resp, err := httpClient.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + raw, err := io.ReadAll(io.LimitReader(resp.Body, 8<<20)) + if err != nil { + return nil, err + } + var env struct { + Result json.RawMessage `json:"result"` + Error *struct { + Message string `json:"message"` + } `json:"error"` + } + if err := json.Unmarshal(raw, &env); err != nil { + return nil, err + } + if env.Error != nil { + return nil, fmt.Errorf("rpc: %s", env.Error.Message) + } + return env.Result, nil +} + +func headBlock() (uint64, error) { + res, err := rpcCall("eth_blockNumber", "[]") + if err != nil { + return 0, err + } + var hexN string + if err := json.Unmarshal(res, &hexN); err != nil { + return 0, err + } + return strconv.ParseUint(strings.TrimPrefix(hexN, "0x"), 16, 64) +} + +type rpcLog struct { + TransactionHash string `json:"transactionHash"` +} + +// usdcTransfersInBlock returns the tx hashes of USDC Transfer logs in +// block bn, via eth_getLogs on the reference RPC. +func usdcTransfersInBlock(bn uint64) ([]rpcLog, error) { + params := fmt.Sprintf(`[{"fromBlock":"0x%x","toBlock":"0x%x","address":"%s","topics":["%s"]}]`, bn, bn, usdcContract, transferTopic) + res, err := rpcCall("eth_getLogs", params) + if err != nil { + return nil, err + } + var logs []rpcLog + if err := json.Unmarshal(res, &logs); err != nil { + return nil, err + } + return logs, nil +} + +// waitFreshEvent polls the reference RPC head until a block newer than +// lastSeen containing at least one USDC Transfer shows up. T0 is the +// instant we first observed the new head. Returns (0, "", zero) only +// when ctx is cancelled. +func waitFreshEvent(ctx context.Context, lastSeen uint64) (uint64, string, time.Time) { + for { + select { + case <-ctx.Done(): + return 0, "", time.Time{} + default: + } + bn, err := headBlock() + if err != nil { + fmt.Printf("[sampler] head poll error: %s\n", sanitize(err)) + time.Sleep(2 * time.Second) + continue + } + if bn > lastSeen { + t0 := time.Now() + logs, err := usdcTransfersInBlock(bn) + if err != nil { + fmt.Printf("[sampler] getLogs error: %s\n", sanitize(err)) + time.Sleep(time.Second) + continue + } + if len(logs) > 0 { + pick := logs[rand.Intn(len(logs))] + if strings.HasPrefix(pick.TransactionHash, "0x") && len(pick.TransactionHash) == 66 { + return bn, strings.ToLower(pick.TransactionHash), t0 + } + } + // Rare on ETH mainnet: block without a USDC transfer. Skip it. + lastSeen = bn + } + time.Sleep(500 * time.Millisecond) + } +} + +// --------------------------------------------------------------------------- +// Shared helpers. +// --------------------------------------------------------------------------- + +// bodyContains — deliberately parser-free detection: lowercase the raw +// response body and look for the tx hash substring. Immune to +// per-provider schema churn; no response shape is favoured. +func bodyContains(resp *http.Response, txHash string) (bool, error) { + defer resp.Body.Close() + raw, err := io.ReadAll(io.LimitReader(resp.Body, 4<<20)) + if err != nil { + return false, err + } + if resp.StatusCode != 200 { + return false, fmt.Errorf("status %d", resp.StatusCode) + } + return bytes.Contains(bytes.ToLower(raw), []byte(strings.ToLower(txHash))), nil +} + +// sanitize strips anything URL-ish from provider errors before logging +// (the Alchemy key rides in its URL; never let it reach stdout/loghub). +func sanitize(err error) string { + if err == nil { + return "" + } + s := err.Error() + if i := strings.Index(s, "http"); i >= 0 { + return s[:i] + "" + } + return s +} + +// --------------------------------------------------------------------------- +// Monthly quota guard (same design as indexing-freshness / rpc-keyed-latency). +// --------------------------------------------------------------------------- + +type quotaGuard struct { + mu sync.Mutex + month string + counts map[string]int64 +} + +var quota = "aGuard{counts: make(map[string]int64)} + +func (q *quotaGuard) allow(p Provider) bool { + q.mu.Lock() + defer q.mu.Unlock() + m := time.Now().UTC().Format("2006-01") + if m != q.month { + q.month = m + q.counts = make(map[string]int64) + } + used := q.counts[p.Slug] + ratio := float64(used) / float64(p.Budget) + quotaUsedRatio.WithLabelValues(p.Slug).Set(ratio) + if ratio >= 0.90 { + return false + } + // Reserve the worst case for one event (full poll schedule). + q.counts[p.Slug] = used + int64(len(pollSchedule)) + return true +} diff --git a/harnesses/indexer-latency/cmd/script/metrics.go b/harnesses/indexer-latency/cmd/script/metrics.go new file mode 100644 index 00000000..4ee048d7 --- /dev/null +++ b/harnesses/indexer-latency/cmd/script/metrics.go @@ -0,0 +1,71 @@ +package main + +import ( + "net/http" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" + "github.com/prometheus/client_golang/prometheus/promhttp" +) + +var ( + latencyHist = promauto.NewHistogramVec( + prometheus.HistogramOpts{ + Name: "idx_latency_seconds", + Help: "Histogram of time-to-queryable lags: seconds between our reference RPC first showing block N and the provider first returning block N's probe event.", + Buckets: []float64{1, 2, 3, 5, 8, 12, 20, 30, 45, 60, 90, 120, 180, 240, 360, 600}, + }, + []string{"provider"}, + ) + + latencyLast = promauto.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "idx_latency_last_seconds", + Help: "Latest observed time-to-queryable lag per provider (drives the bench page time series).", + }, + []string{"provider"}, + ) + + probeTotal = promauto.NewCounterVec( + prometheus.CounterOpts{ + Name: "idx_probe_total", + Help: "Probe outcomes per provider: found | timeout (not queryable within 240s) | error | quota_paused | disabled (credential missing).", + }, + []string{"provider", "result"}, + ) + + quotaUsedRatio = promauto.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "idx_quota_used_ratio", + Help: "Fraction of the provider's monthly call budget consumed (probing pauses at 0.90, calendar-month reset).", + }, + []string{"provider"}, + ) + + health = promauto.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "idx_health", + Help: "1 when the provider's last probe found the event, 0 on timeout/error/disabled. Doubles as the harness freshness signal.", + }, + []string{"provider"}, + ) + + apiCalls = promauto.NewCounterVec( + prometheus.CounterOpts{ + Name: "idx_api_calls_total", + Help: "API calls issued per provider (feeds the monthly quota guard).", + }, + []string{"provider"}, + ) +) + +func StartMetricsServer(addr string) error { + mux := http.NewServeMux() + mux.Handle("/metrics", promhttp.Handler()) + mux.Handle("/logs", logsHandler()) + mux.HandleFunc("/health", func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("ok")) + }) + return http.ListenAndServe(addr, mux) +} diff --git a/harnesses/indexer-latency/cmd/script/thegraph.go b/harnesses/indexer-latency/cmd/script/thegraph.go new file mode 100644 index 00000000..cd97e45d --- /dev/null +++ b/harnesses/indexer-latency/cmd/script/thegraph.go @@ -0,0 +1,69 @@ +package main + +import ( + "encoding/json" + "fmt" + "io" + "net/http" + "strings" +) + +// The Graph poller. +// +// The Graph only serves what subgraphs index, so we cannot ask it for a +// raw USDC Transfer log. Probed entity instead: the canonical Uniswap +// V3 Ethereum subgraph queried WITH a block constraint pinned to the +// reference block N. graph-node rejects block-constrained queries until +// the subgraph has indexed block N ("only indexed up to block ..."), so +// a successful data response is exactly "block N data queryable". This +// per-provider entity asymmetry is disclosed in the bench methodology; +// the measured question (block N queryable at T?) is identical. +// +// Gateway free plan (100k queries/month) requires GRAPH_API_KEY; the +// key travels in the Authorization header, never in the URL. + +const uniswapV3SubgraphID = "5zvR82QoaXYFyDEKLZ9t6v9adgnptxYpKpSbxtgVENFV" + +func checkTheGraph(bn uint64, _ string) (bool, error) { + apiCalls.WithLabelValues("thegraph").Inc() + q := fmt.Sprintf(`{"query":"{ pools(block: {number: %d}, first: 1) { id } }"}`, bn) + u := "https://gateway.thegraph.com/api/subgraphs/id/" + uniswapV3SubgraphID + req, err := http.NewRequest("POST", u, strings.NewReader(q)) + if err != nil { + return false, err + } + req.Header.Set("Authorization", "Bearer "+graphAPIKey()) + req.Header.Set("Content-Type", "application/json") + req.Header.Set("User-Agent", "OpenChainBench/1.0 (+https://openchainbench.com)") + resp, err := httpClient.Do(req) + if err != nil { + return false, err + } + defer resp.Body.Close() + raw, err := io.ReadAll(io.LimitReader(resp.Body, 4<<20)) + if err != nil { + return false, err + } + if resp.StatusCode != 200 { + return false, fmt.Errorf("status %d", resp.StatusCode) + } + var out struct { + Data json.RawMessage `json:"data"` + Errors []struct { + Message string `json:"message"` + } `json:"errors"` + } + if err := json.Unmarshal(raw, &out); err != nil { + return false, err + } + if len(out.Errors) > 0 { + msg := strings.ToLower(out.Errors[0].Message) + // graph-node's "block N not yet indexed" family of errors is the + // expected pre-sync answer, not an API failure. + if strings.Contains(msg, "indexed up to") || strings.Contains(msg, "has only indexed") || strings.Contains(msg, "missing block") || strings.Contains(msg, "not yet") { + return false, nil + } + return false, fmt.Errorf("graphql: %s", out.Errors[0].Message) + } + return len(out.Data) > 0 && string(out.Data) != "null", nil +} diff --git a/harnesses/indexer-latency/go.mod b/harnesses/indexer-latency/go.mod new file mode 100644 index 00000000..c07b9b31 --- /dev/null +++ b/harnesses/indexer-latency/go.mod @@ -0,0 +1,18 @@ +module indexer-latency + +go 1.24.0 + +require github.com/prometheus/client_golang v1.23.2 + +require ( + github.com/beorn7/perks v1.0.1 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/kr/text v0.2.0 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/prometheus/client_model v0.6.2 // indirect + github.com/prometheus/common v0.66.1 // indirect + github.com/prometheus/procfs v0.16.1 // indirect + go.yaml.in/yaml/v2 v2.4.2 // indirect + golang.org/x/sys v0.35.0 // indirect + google.golang.org/protobuf v1.36.8 // indirect +) diff --git a/harnesses/indexer-latency/go.sum b/harnesses/indexer-latency/go.sum new file mode 100644 index 00000000..d6b8ca98 --- /dev/null +++ b/harnesses/indexer-latency/go.sum @@ -0,0 +1,46 @@ +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= +github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o= +github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg= +github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= +github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= +github.com/prometheus/common v0.66.1 h1:h5E0h5/Y8niHc5DlaLlWLArTQI7tMrsfQjHV+d9ZoGs= +github.com/prometheus/common v0.66.1/go.mod h1:gcaUsgf3KfRSwHY4dIMXLPV0K/Wg1oZ8+SbZk/HH/dA= +github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg= +github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is= +github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= +github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI= +go.yaml.in/yaml/v2 v2.4.2/go.mod h1:081UH+NErpNdqlCXm3TtEran0rJZGxAYx9hb/ELlsPU= +golang.org/x/sys v0.35.0 h1:vz1N37gP5bs89s7He8XuIYXpyY0+QlsKmzipCbUtyxI= +golang.org/x/sys v0.35.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +google.golang.org/protobuf v1.36.8 h1:xHScyCOEuuwZEc6UtSOvPbAT4zRh0xcNRYekJwfqyMc= +google.golang.org/protobuf v1.36.8/go.mod h1:fuxRtAxBytpl4zzqUh6/eyUujkJdNiuEkXntxiD/uRU= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= From 0b867e7c253e7b3c92d1cdeb26a6f27c235d60e2 Mon Sep 17 00:00:00 2001 From: Florent Tapponnier Date: Wed, 15 Jul 2026 01:05:43 +0200 Subject: [PATCH 03/12] idx-latency: only classify error when every poll errored, else timeout --- harnesses/indexer-latency/cmd/script/main.go | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/harnesses/indexer-latency/cmd/script/main.go b/harnesses/indexer-latency/cmd/script/main.go index 0bccfe70..dbfa255e 100644 --- a/harnesses/indexer-latency/cmd/script/main.go +++ b/harnesses/indexer-latency/cmd/script/main.go @@ -133,6 +133,7 @@ func runEvents(ctx context.Context) { // is exhausted (timeout), or every poll errored (error). func raceProvider(ctx context.Context, p Provider, bn uint64, txHash string, t0 time.Time) { var lastErr error + polls, errs := 0, 0 for _, after := range pollSchedule { wait := time.Until(t0.Add(time.Duration(after) * time.Second)) if wait > 0 { @@ -142,8 +143,10 @@ func raceProvider(ctx context.Context, p Provider, bn uint64, txHash string, t0 case <-time.After(wait): } } + polls++ found, err := p.Check(bn, txHash) if err != nil { + errs++ lastErr = err continue } @@ -158,12 +161,20 @@ func raceProvider(ctx context.Context, p Provider, bn uint64, txHash string, t0 } } health.WithLabelValues(p.Slug).Set(0) - if lastErr != nil { + // "error" only when EVERY poll of the event errored (matches the spec's + // classification). An event with at least one clean "not yet" answer is a + // timeout: reclassifying it as error would drop it from the success-rate + // denominator and let transient blips inflate a provider's score. + if errs == polls && lastErr != nil { probeTotal.WithLabelValues(p.Slug, "error").Inc() fmt.Printf(" %-10s error: %s\n", p.Slug, sanitize(lastErr)) return } probeTotal.WithLabelValues(p.Slug, "timeout").Inc() + if lastErr != nil { + fmt.Printf(" %-10s TIMEOUT (not queryable within %ds; %d/%d polls errored, last: %s)\n", p.Slug, pollSchedule[len(pollSchedule)-1], errs, polls, sanitize(lastErr)) + return + } fmt.Printf(" %-10s TIMEOUT (not queryable within %ds)\n", p.Slug, pollSchedule[len(pollSchedule)-1]) } From d1172d37e15e38234e6f63fcfeb761bc17b3dc7a Mon Sep 17 00:00:00 2001 From: Florent Tapponnier Date: Wed, 15 Jul 2026 01:05:43 +0200 Subject: [PATCH 04/12] bump bench-set cache keys for 084 (lockstep rule) --- src/lib/spec.ts | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/lib/spec.ts b/src/lib/spec.ts index efcd21b9..3fc4fdec 100644 --- a/src/lib/spec.ts +++ b/src/lib/spec.ts @@ -283,7 +283,9 @@ const loadBenchmarkUnfilteredCached = unstable_cache( // buckets (gap rendering fix). Cached v26 entries hold the old // hole-compressed arrays whose indices no longer map onto the nominal // step grid the chart back-computes timestamps from. - ["bench-unfiltered-v27", process.env.VERCEL_ENV === "production" ? "prod" : "all"], +// v28: bench vague 2 ships 081-085 (ws-head-latency, oracle-freshness, + // rpc-reliability, indexer-latency, evm-block-builders). Bench SET changed. + ["bench-unfiltered-v28", process.env.VERCEL_ENV === "production" ? "prod" : "all"], { revalidate: 300, tags: ["benchmarks"] }, ); @@ -438,7 +440,8 @@ const loadAllBenchmarksCached = unstable_cache( // gated catalog to /products for 30+ min after the deploy). // v29: bumped with bench-unfiltered-v26 (monad-rpc + megaeth-rpc ship). // v30: bumped with bench-unfiltered-v27 (dense series with nulls). - ["all-benchmarks-v30", process.env.VERCEL_ENV === "production" ? "prod" : "all"], +// v31: bumped with bench-unfiltered-v28 (bench vague 2, 081-085). + ["all-benchmarks-v31", process.env.VERCEL_ENV === "production" ? "prod" : "all"], { revalidate: 300, tags: ["benchmarks"] }, ); export const loadAllBenchmarks = cache(loadAllBenchmarksCached); @@ -517,7 +520,8 @@ const loadBenchmarkFiltered = unstable_cache( // v16: bumped with the bench 074 ship (lockstep rule, see all-benchmarks-v28). // v17: bumped with the monad-rpc + megaeth-rpc ship (lockstep rule). // v18: bumped with bench-unfiltered-v27 (dense series with nulls). - ["bench-filters-v18", process.env.VERCEL_ENV === "production" ? "prod" : "all"], + // v19: bumped with bench-unfiltered-v28 (bench vague 2, 081-085). + ["bench-filters-v19", process.env.VERCEL_ENV === "production" ? "prod" : "all"], { revalidate: 300, tags: ["benchmarks"] } ); From 92048dd298356d5446cbb1fd66a1950973583f9f Mon Sep 17 00:00:00 2001 From: Florent Tapponnier Date: Wed, 15 Jul 2026 01:00:02 +0200 Subject: [PATCH 05/12] add bench 082 oracle-freshness: staleness metrics in oracle-deviation harness (multi-chain chainlink, pyth publish_time fix, redstone), bump bench-set cache keys --- benchmarks/oracle-freshness.yml | 167 +++++++++++++++++ harnesses/oracle-deviation/README.md | 39 +++- .../oracle-deviation/cmd/script/chainlink.go | 81 +++++++- .../oracle-deviation/cmd/script/config.go | 65 +++++++ .../oracle-deviation/cmd/script/freshness.go | 176 ++++++++++++++++++ harnesses/oracle-deviation/cmd/script/main.go | 15 +- .../oracle-deviation/cmd/script/metrics.go | 53 ++++++ harnesses/oracle-deviation/cmd/script/pyth.go | 21 ++- .../oracle-deviation/cmd/script/redstone.go | 106 +++++++++++ 9 files changed, 710 insertions(+), 13 deletions(-) create mode 100644 benchmarks/oracle-freshness.yml create mode 100644 harnesses/oracle-deviation/cmd/script/freshness.go create mode 100644 harnesses/oracle-deviation/cmd/script/redstone.go diff --git a/benchmarks/oracle-freshness.yml b/benchmarks/oracle-freshness.yml new file mode 100644 index 00000000..69b526d4 --- /dev/null +++ b/benchmarks/oracle-freshness.yml @@ -0,0 +1,167 @@ +# OpenChainBench. Bench № 082 + +slug: oracle-freshness +number: "082" +title: Chainlink vs Pyth vs RedStone, live oracle freshness +seo_title: "Chainlink vs Pyth vs RedStone oracle staleness 2026" +seo_description: "How old is the oracle price your protocol settles on? Chainlink (Ethereum, Arbitrum, Base), Pyth Hermes and RedStone ranked live by ETH/USD staleness in seconds, p50/p99 over 24h, keyless and reproducible." +subtitle: Seconds since each oracle's own last-update timestamp for ETH/USD, polled every 30 seconds across Chainlink on three chains, Pyth via Hermes and RedStone via its public gateway. Paired with a stale-but-moved flag that fires only when an old price has also been left behind by the market. +category: Trading +status: live +metric: Oracle staleness +unit: s +higher_is_better: false + +disclaimer: | + Read before ranking. The three oracles use different update models, and that difference IS the measurement. Chainlink pushes on-chain only on a deviation trigger or heartbeat, so hundreds of seconds of staleness in a calm market is by design, not failure. Pyth and RedStone publish continuously off-chain and are pulled on demand. The dangerous state is not "old", it is "old while the market moved": that is the companion `ocb_oracle_stale_but_moved` signal, not the headline. + +seo_intro: | + Every lending market, perp DEX and stablecoin issuer settles on an + oracle price that is, by construction, some number of seconds old. + This page publishes that number live. We read each oracle's own + update timestamp (not our fetch time): Chainlink's on-chain + `latestRoundData().updatedAt` on Ethereum, Arbitrum and Base, Pyth's + Hermes `publish_time`, and the timestamp of RedStone's latest signed + data package, and publish `now - last_update` as a Prometheus gauge + in seconds, quantiled over 24 hours. Why it matters: liquidation + cascades are a freshness problem before they are a price problem. + When ETH drops 3% and the on-chain feed has not updated for ten + minutes, underwater positions sit unliquidated, accruing bad debt; + when the round finally lands, every overdue liquidation fires in the + same block, keepers race, gas spikes, and the forced selling deepens + the move that caused it. The honest complication, stated up front: + Chainlink is a push oracle whose feeds are SUPPOSED to be stale in + calm markets (updates cost gas, so they land only on a 0.5% + deviation or a 1 hour heartbeat), while Pyth and RedStone are + pull-model oracles whose off-chain publication runs at 1 to 25 + second cadence and whose on-chain freshness depends on who pulls. + We are comparing different update models on the same clock, which is + exactly the design decision an integrator is making when they pick + one. The signal that separates "calm market, by design" from "your + liquidation engine is trading on a fossil" is the stale-but-moved + flag: staleness above 300 seconds while the CEX print has moved more + than 0.5% since the oracle's last update. + +abstract: | + We measure oracle freshness by reading each oracle's own + last-update timestamp every 30 seconds and publishing + `ocb_oracle_staleness_seconds{oracle, pair, chain}` (refreshed every + 5 seconds in-harness so the gauge climbs monotonically between + polls). Timestamps come from the source, never from our clock: + Chainlink's `latestRoundData().updatedAt` via keyless `eth_call` on + Ethereum, Arbitrum and Base; Pyth's `publish_time` from the Hermes + REST batch endpoint (Pyth is pull-based, so Hermes IS the price an + integrator would put on-chain, making publish_time the honest + freshness measure); RedStone's signed data-package timestamp from + its keyless per-symbol API. Two companion series complete the + picture: `ocb_oracle_update_events_total` counts observed update + timestamp advances, and `ocb_oracle_stale_but_moved` flags the + dangerous state, staleness above 300 s while the Binance/Coinbase + reference price moved more than 0.5% since the update. The headline + ranks the three oracles by p50 ETH/USD staleness over 24 h + (Chainlink pinned to its Ethereum mainnet feed; the Arbitrum and + Base series are queryable under the `chain` label). Lower is + fresher, but the disclaimer applies: push and pull oracles are + different machines, and the leaderboard quantifies the trade-off + rather than declaring one model wrong. + +methodology: + - "Staleness definition. `staleness = now - last_update_ts`, where last_update_ts is the oracle's OWN declared timestamp, never our fetch time: Chainlink's on-chain `latestRoundData().updatedAt`, Pyth's Hermes `publish_time`, RedStone's signed data-package `timestamp`. Published as `ocb_oracle_staleness_seconds{oracle, pair, chain}` in seconds." + - "Chainlink leg. Keyless `eth_call(latestRoundData())` against the canonical ETH/USD AggregatorV3 contracts on three chains: Ethereum mainnet (0x5f4eC3Df...8419), Arbitrum One (0x639Fe6ab...a612) and Base (0x71041ddd...Bb70), via public RPCs with a fallback per chain. The headline pins `chain=\"ethereum\"`; Arbitrum and Base are live under the same metric's `chain` label." + - "Pyth leg. Hermes REST batch endpoint, no key. Pyth is a pull oracle: there is no continuously-updated on-chain price to read, integrators fetch a signed update from Hermes and post it in their own transaction. Hermes `publish_time` is therefore the freshness of the price you would actually put on-chain, and that is what we measure (`chain=\"hermes\"`). At fetch time the publish_time is typically 1-2 s old; see the observation-lag bullet for what the quantiles report." + - "RedStone leg. Keyless per-symbol price API (`provider=redstone`, the primary production service), which returns the latest signed data package's value and timestamp (`chain=\"gateway\"`). We poll per symbol because the full data-packages gateway payload for the service is ~1.7 MB per request; the per-symbol object is a few hundred bytes. Packages are produced on a roughly 10 s cadence; at fetch time the package timestamp is typically 15-25 s old." + - "Cadence and aggregation. Every (oracle, pair, chain) is polled every 30 s; the staleness gauge is additionally re-published every 5 s in-harness so it climbs between polls instead of stair-stepping. Leaderboard numbers are `quantile_over_time` p50/p90/p99 over 24 h. Total added load is ~8 requests/minute on top of the host harness's 80, all keyless." + - "Observation lag, stated honestly. For oracles that publish faster than our 30 s poll (Pyth, RedStone), the gauge keeps aging until the next poll, so reported quantiles sit about half a poll interval above the true publication age: Pyth's p50 reads mid-teens of seconds even though Hermes is 1-2 s old at every fetch. The numbers are upper bounds with a shared floor set by our cadence, negligible against push-feed staleness measured in hundreds of seconds." + - "Update events. `ocb_oracle_update_events_total` increments each time the source-declared timestamp moves forward between observations. With a 30 s poll it undercounts sources that update faster than the poll (Pyth publishes near-continuously), so treat it as a lower bound on update frequency; it is exact for Chainlink's minutes-to-hours round cadence." + - "Stale-but-moved. `ocb_oracle_stale_but_moved` is 1 when staleness exceeds 300 s AND the CEX reference (Binance, Coinbase fallback, from the same harness's bench 025 pollers) has moved more than 0.5% since the update was observed. 0.5% matches the widest deviation trigger configured on the measured Chainlink feeds: if the market moved past the feed's own trigger and no round landed, the feed is late by its own standard." + - "Threshold rationale. 300 s sits below Chainlink's 1 h heartbeat and above healthy deviation-triggered gaps (feasibility runs saw fresh rounds 398-1069 s apart on quiet hours, sub-minute during moves). Alone it flags nothing; with the 0.5% move condition it isolates the window where liquidations execute on a price the market has left. Cold-start caveat: after a restart the CEX reference snapshots at boot, converging on the first real update." + - "Push vs pull is the comparison, not a flaw. Chainlink pays gas per update so it updates sparsely by design; Pyth and RedStone publish off-chain continuously and shift the on-chain freshness question to whoever pulls. Ranking them on the same staleness clock quantifies the trade-off an integrator accepts with each model. The per-oracle tag states the model so the table is read correctly." + - "Isolation from bench 025. This bench shares the oracle-deviation harness but emits NEW metric families with an `oracle`/`chain` label set. Bench 025's metrics, labels and error counters are untouched; freshness-only sources (RedStone, Arbitrum/Base Chainlink) never enter 025's deviation store, and freshness scrape failures land in `ocb_oracle_freshness_scrape_errors_total`, not 025's error counter." + - "Excluded in v1, with reasons. Chronicle: reads are whitelist-gated on-chain (`toll` access control), so there is no keyless public read to reproduce. API3: the measured-pair proxy addresses we probed were dead or unverifiable without an integration contract. Both are candidates for a follow-up once a reproducible keyless read path exists. Uniswap TWAP is out of scope: it is a derivation of the same CEX prints, not an oracle network with its own update timestamp." + - "Reproducibility. Every leg is a keyless public endpoint: `eth_call` with selector 0xfeaf968c on the published feed addresses, the Hermes batch REST endpoint, and the RedStone per-symbol API. Anyone can replay any sample with curl and get the same timestamps we do." + +findings: + - "{{best_name}} is currently the freshest at {{best_p50}} (p50 ETH/USD staleness, 24 h) across {{count}} measured oracles. Remember the disclaimer before reading this as a quality ranking: the leader is fresh because its update model publishes continuously off-chain, not because the others are broken." + - "{{name:pyth}} sits at {{p50:pyth}}. Hermes aggregates publisher prices at sub-second cadence and its publish_time is 1-2 s old at every fetch; the reported p50 is higher because our gauge ages between 30 s polls (upper-bound semantics, see methodology). Two honest asterisks, then: the number carries our observation lag, and the on-chain freshness of a Pyth integration is ultimately set by how often the integrating protocol pulls." + - "{{name:chainlink}} reads {{p50:chainlink}} on the Ethereum mainnet ETH/USD feed. That number is the deviation-trigger economics made visible: rounds land only when the off-chain aggregate drifts ~0.5% or the 1 h heartbeat expires, so calm hours accumulate hundreds of seconds of staleness by design. The Arbitrum and Base deployments of the same feed are published under the chain label for comparison." + - "{{name:redstone}} lands at {{p50:redstone}}. RedStone's primary production service signs a fresh data package roughly every 10 s, and its per-symbol API served packages 15-25 s old at fetch during feasibility: slower than Hermes, orders of magnitude fresher than a push feed between rounds, with the same observation-lag and pull-model asterisks as Pyth." + - "The alarm metric is not on this leaderboard. `ocb_oracle_stale_but_moved` fires only when a feed is over 300 s old AND the CEX print moved over 0.5% since its last update, which is the state where liquidation engines execute on a price the market has left. In calm markets it should read 0 for every oracle, including a 900-second-stale Chainlink feed." + +faq: + - q: "What is oracle staleness and why should a protocol care?" + a: "Staleness is the age of the oracle price at the moment your contract reads it: now minus the oracle's own last-update timestamp. Every on-chain settlement (a liquidation, a mint, a perp funding tick) executes at that possibly-old price. If ETH moves 3% while a feed sits unchanged for ten minutes, positions that should liquidate do not, bad debt accrues, and when the update finally lands every overdue liquidation fires in the same block. That burst, keepers racing to close stale-priced positions all at once, is how freshness problems become liquidation cascades. This bench publishes the age live, per oracle, so the risk is sized before integration." + - q: "Chainlink shows hundreds of seconds of staleness. Is it broken?" + a: "No, and this is the most important caveat on the page. Chainlink is a push oracle: node operators pay gas to post each round on-chain, so feeds are configured to update only when the off-chain aggregate drifts past a deviation trigger (0.5% on the measured ETH/USD feeds) or a heartbeat expires (1 hour). Between triggers the on-chain price legitimately ages. A 900-second-old Chainlink price in a flat market is the system working as designed and costs integrators nothing. The failure mode worth alarming on is captured by the separate stale_but_moved flag: an old price after the market has moved past the feed's own trigger threshold." + - q: "Why is Pyth measured at Hermes instead of on-chain, and is that fair?" + a: "Because pull oracles have no continuously-updated on-chain price to read. A Pyth integrator fetches a signed price update from Hermes and posts it inside their own transaction; the on-chain contract only stores whatever was last pushed by someone. Measuring Pyth's freshness on-chain would measure the pull habits of whichever protocols happen to share the feed, not the oracle. Hermes publish_time is the age of the price actually available to any integrator at the moment of the pull, which is the like-for-like comparison against Chainlink's updatedAt. The trade-off is stated openly: Hermes is 1-2 s old at pull time (our reported quantiles add up to 30 s of observation lag from the poll cadence), and a protocol that pulls lazily inherits its own staleness on top." + - q: "What exactly does stale_but_moved measure and why 300 seconds and 0.5%?" + a: "It is 1 when two conditions hold at once: the oracle's price is older than 300 seconds, and the CEX reference (Binance, Coinbase fallback) has moved more than 0.5% since that update was observed. 0.5% mirrors the widest deviation trigger on the measured Chainlink feeds, so the flag fires precisely when a feed is late by its own configured standard. 300 s sits above every healthy deviation-triggered gap we observed during fast markets and far below the 1 h heartbeat, isolating genuine lag without penalizing calm-market aging. Both thresholds are documented constants in the harness source and can be recomputed from the raw staleness and price series." + - q: "Why are Chronicle and API3 not on the leaderboard?" + a: "Both failed the bench's reproducibility bar for v1. Chronicle gates on-chain reads behind a whitelist (its toll access control), so there is no keyless public read a third party can replay. For API3, the measured-pair proxy addresses we probed were dead or unverifiable without deploying an integration contract. OpenChainBench only publishes numbers anyone can reproduce with curl and a public RPC, so both are excluded and documented rather than measured badly. If either ships a keyless read path for the measured pairs, they are straightforward to add: the harness's freshness tracker is oracle-agnostic." + +source: https://github.com/ChainBench/OpenChainBench/tree/main/harnesses/oracle-deviation + +prometheus: + window: 24h + freshness_metric: ocb_oracle_staleness_seconds + +# Metric families emitted by the oracle-deviation harness for this bench +# (bench 025's families are separate and untouched): +# ocb_oracle_staleness_seconds{oracle, pair, chain} gauge (s) +# ocb_oracle_update_events_total{oracle, pair, chain} counter +# ocb_oracle_stale_but_moved{oracle, pair, chain} gauge (0/1) +# ocb_oracle_freshness_scrape_errors_total{oracle, pair, chain} counter +# +# oracle is "chainlink" | "pyth" | "redstone". chain is where the +# freshness timestamp lives: ethereum | arbitrum | base for Chainlink +# deployments, "hermes" for Pyth (publish_time), "gateway" for +# RedStone (signed data-package timestamp). +# +# The headline ranks the three oracles by p50 ETH/USD staleness over +# 24h; Chainlink is pinned to its Ethereum mainnet feed and its +# Arbitrum + Base series remain queryable under the chain label. +# Success rate divides freshness scrape failures by the 30s poll +# cadence (1 expected attempt per 30s per series). + +providers: + - slug: chainlink + name: Chainlink + tag: "Push onchain · ETH/USD · ethereum, arbitrum, base" + type: protocol + formula: "p50 over 24h of seconds since the Chainlink ETH/USD aggregator's on-chain updatedAt on Ethereum mainnet. Deviation-triggered push feed: high staleness in calm markets is by design. Arbitrum and Base series live under the chain label." + queries: + p50: quantile_over_time(0.50, ocb_oracle_staleness_seconds{oracle="chainlink", pair="ETH/USD", chain="ethereum"}[24h]) + p90: quantile_over_time(0.90, ocb_oracle_staleness_seconds{oracle="chainlink", pair="ETH/USD", chain="ethereum"}[24h]) + p99: quantile_over_time(0.99, ocb_oracle_staleness_seconds{oracle="chainlink", pair="ETH/USD", chain="ethereum"}[24h]) + mean: avg_over_time(ocb_oracle_staleness_seconds{oracle="chainlink", pair="ETH/USD", chain="ethereum"}[24h]) + success: clamp_min(clamp_max(1 - sum(rate(ocb_oracle_freshness_scrape_errors_total{oracle="chainlink", pair="ETH/USD", chain="ethereum"}[24h])) / (1/30), 1), 0) + sample_size: count_over_time(ocb_oracle_staleness_seconds{oracle="chainlink", pair="ETH/USD", chain="ethereum"}[24h]) + series: ocb_oracle_staleness_seconds{oracle="chainlink", pair="ETH/USD", chain="ethereum"} + + - slug: pyth + name: Pyth + tag: "Pull via Hermes · ETH/USD · publish_time" + type: protocol + formula: "p50 over 24h of seconds since the Hermes publish_time of Pyth's ETH/USD feed. Pull oracle: Hermes is the price an integrator would post on-chain, so publish_time is the freshness of what is available to pull." + queries: + p50: quantile_over_time(0.50, ocb_oracle_staleness_seconds{oracle="pyth", pair="ETH/USD", chain="hermes"}[24h]) + p90: quantile_over_time(0.90, ocb_oracle_staleness_seconds{oracle="pyth", pair="ETH/USD", chain="hermes"}[24h]) + p99: quantile_over_time(0.99, ocb_oracle_staleness_seconds{oracle="pyth", pair="ETH/USD", chain="hermes"}[24h]) + mean: avg_over_time(ocb_oracle_staleness_seconds{oracle="pyth", pair="ETH/USD", chain="hermes"}[24h]) + success: clamp_min(clamp_max(1 - sum(rate(ocb_oracle_freshness_scrape_errors_total{oracle="pyth", pair="ETH/USD", chain="hermes"}[24h])) / (1/30), 1), 0) + sample_size: count_over_time(ocb_oracle_staleness_seconds{oracle="pyth", pair="ETH/USD", chain="hermes"}[24h]) + series: ocb_oracle_staleness_seconds{oracle="pyth", pair="ETH/USD", chain="hermes"} + + - slug: redstone + name: RedStone + tag: "Modular push-pull · ETH/USD · signed package timestamp" + type: protocol + formula: "p50 over 24h of seconds since the timestamp of RedStone's latest signed ETH data package on the keyless per-symbol API (primary production service). Packages are produced on a roughly 10s cadence." + queries: + p50: quantile_over_time(0.50, ocb_oracle_staleness_seconds{oracle="redstone", pair="ETH/USD", chain="gateway"}[24h]) + p90: quantile_over_time(0.90, ocb_oracle_staleness_seconds{oracle="redstone", pair="ETH/USD", chain="gateway"}[24h]) + p99: quantile_over_time(0.99, ocb_oracle_staleness_seconds{oracle="redstone", pair="ETH/USD", chain="gateway"}[24h]) + mean: avg_over_time(ocb_oracle_staleness_seconds{oracle="redstone", pair="ETH/USD", chain="gateway"}[24h]) + success: clamp_min(clamp_max(1 - sum(rate(ocb_oracle_freshness_scrape_errors_total{oracle="redstone", pair="ETH/USD", chain="gateway"}[24h])) / (1/30), 1), 0) + sample_size: count_over_time(ocb_oracle_staleness_seconds{oracle="redstone", pair="ETH/USD", chain="gateway"}[24h]) + series: ocb_oracle_staleness_seconds{oracle="redstone", pair="ETH/USD", chain="gateway"} diff --git a/harnesses/oracle-deviation/README.md b/harnesses/oracle-deviation/README.md index 1d41b04e..107e80fe 100644 --- a/harnesses/oracle-deviation/README.md +++ b/harnesses/oracle-deviation/README.md @@ -1,7 +1,11 @@ -# oracle-deviation — OpenChainBench № 025 +# oracle-deviation — OpenChainBench № 025 + № 082 Compares four price oracles in real time across ten USD pairs and -publishes the per-pair deviation as Prometheus gauges. +publishes the per-pair deviation as Prometheus gauges (bench № 025). +The same process also tracks oracle freshness (bench № 082): +staleness in seconds derived from each oracle's own last-update +timestamp, for Chainlink on three chains, Pyth via Hermes and +RedStone via its public per-symbol API. ## Sources @@ -12,6 +16,14 @@ publishes the per-pair deviation as Prometheus gauges. | Binance | REST `/api/v3/ticker/price` | none | 30 s | | Coinbase | REST `/products/

/ticker` | none | 30 s | +Freshness-only sources (№ 082, never enter the № 025 deviation matrix): + +| Source | Transport | Auth | Cadence | +| ------------------- | ------------------------------------------------ | ---- | ------- | +| Chainlink Arbitrum | `eth_call` on AggregatorV3 (ETH/USD) | none | 30 s | +| Chainlink Base | `eth_call` on AggregatorV3 (ETH/USD) | none | 30 s | +| RedStone | REST `/prices?symbol=&provider=redstone` | none | 30 s | + ## Pairs BTC, ETH, SOL, BNB, XRP, ADA, DOGE, AVAX, LINK, MATIC — all quoted @@ -48,6 +60,25 @@ ocb_oracle_scrape_errors_total{source, pair} counter ocb_oracle_last_round_age_seconds{source="chainlink", pair} gauge ``` +Bench № 082 (freshness) families. New label set (`oracle`, `pair`, +`chain`) instead of adding a `chain` label to the № 025 families, +which would have split their series and changed the live bench's +numbers: + +``` +ocb_oracle_staleness_seconds{oracle, pair, chain} gauge (s) +ocb_oracle_update_events_total{oracle, pair, chain} counter +ocb_oracle_stale_but_moved{oracle, pair, chain} gauge (0/1) +ocb_oracle_freshness_scrape_errors_total{oracle, pair, chain} counter +``` + +`chain` is where the freshness timestamp lives: `ethereum` / +`arbitrum` / `base` for Chainlink deployments, `hermes` for Pyth +(publish_time), `gateway` for RedStone (signed data-package +timestamp). `stale_but_moved` is 1 when staleness > 300 s AND the +Binance/Coinbase reference moved > 0.5 % since the update was +observed (thresholds documented in `freshness.go`). + The headline ranking signal for the OCB leaderboard is `ocb_oracle_max_deviation_pct` — a per-pair quantile over time (`quantile_over_time(0.95, ocb_oracle_max_deviation_pct[24h])`) @@ -76,3 +107,7 @@ curl -s :2112/metrics | grep ocb_oracle_max_deviation_pct | ------------------------ | ------------------------------------------- | | `ORACLE_RPC_PRIMARY` | `https://ethereum-rpc.publicnode.com` | | `ORACLE_RPC_FALLBACK` | `https://eth.llamarpc.com` | +| `ORACLE_RPC_ARBITRUM` | `https://arbitrum-one-rpc.publicnode.com` | +| `ORACLE_RPC_BASE` | `https://base-rpc.publicnode.com` | +| `ORACLE_REDSTONE_URL` | `https://api.redstone.finance/prices` | +| `ORACLE_METRICS_ADDR` | `:2112` (local-run override only) | diff --git a/harnesses/oracle-deviation/cmd/script/chainlink.go b/harnesses/oracle-deviation/cmd/script/chainlink.go index fdd292de..6bf7c6f7 100644 --- a/harnesses/oracle-deviation/cmd/script/chainlink.go +++ b/harnesses/oracle-deviation/cmd/script/chainlink.go @@ -66,9 +66,11 @@ type rpcResp struct { type chainlinkPoller struct { httpClient *http.Client - // rotated[i] is the index of the next RPC to try; on consecutive - // errors the poller flips to the fallback. Cheap atomic instead - // of a mutex since the field is only read/written by this poller. + // rpcs is the ordered endpoint list for the chain this poller + // reads from. rpcIdx rotates through it on transient errors. + // Cheap atomic instead of a mutex since the field is only + // read/written by this poller. + rpcs []string rpcIdx atomic.Int32 // decimals cache (feed → decimals). Populated lazily on first // successful read. @@ -80,9 +82,10 @@ type chainlinkPoller struct { failureCount map[string]int } -func newChainlinkPoller() *chainlinkPoller { +func newChainlinkPoller(rpcs []string) *chainlinkPoller { return &chainlinkPoller{ httpClient: &http.Client{Timeout: httpTimeout}, + rpcs: rpcs, decimalsCache: make(map[string]int), unsupported: make(map[string]bool), failureCount: make(map[string]int), @@ -90,10 +93,11 @@ func newChainlinkPoller() *chainlinkPoller { } func (c *chainlinkPoller) rpcURL() string { - if c.rpcIdx.Load()%2 == 0 { - return rpcEndpoint() + idx := int(c.rpcIdx.Load()) % len(c.rpcs) + if idx < 0 { + idx = -idx } - return rpcEndpointFallback() + return c.rpcs[idx] } // runChainlink loops every pair every pollInterval. The 10 pairs are @@ -101,7 +105,7 @@ func (c *chainlinkPoller) rpcURL() string { // (one batched request would be ideal — public RPCs often refuse // JSON-RPC batches, so we stick to one call per pair). func runChainlink(ctx context.Context, specs []PairSpec) { - c := newChainlinkPoller() + c := newChainlinkPoller([]string{rpcEndpoint(), rpcEndpointFallback()}) t := time.NewTicker(pollInterval) defer t.Stop() @@ -127,6 +131,7 @@ func runChainlink(ctx context.Context, specs []PairSpec) { cancel() if err != nil { oracleScrapeErrors.WithLabelValues(string(SourceChainlink), string(s.Pair)).Inc() + freshnessScrapeErrors.WithLabelValues(string(SourceChainlink), string(s.Pair), ChainEthereum).Inc() c.failureCount[s.ChainlinkFeed]++ // Distinguish reverts (permanent — bad address / // non-AggregatorV3 contract) from transient HTTP @@ -150,6 +155,10 @@ func runChainlink(ctx context.Context, specs []PairSpec) { // values (which would tag Chainlink's heartbeat lag as // "deviation" — that artifact is exactly what Fix C kills). recordPriceAt(SourceChainlink, s.Pair, price, time.Unix(updatedAt, 0)) + // Feed the № 082 freshness tracker with the same on-chain + // updatedAt (chain=ethereum). Freshness for other chains + // comes from runChainlinkChains. + recordFreshness(string(SourceChainlink), s.Pair, ChainEthereum, time.Unix(updatedAt, 0)) ageS := time.Since(time.Unix(updatedAt, 0)).Seconds() if ageS < 0 { ageS = 0 @@ -169,6 +178,62 @@ func runChainlink(ctx context.Context, specs []PairSpec) { } } +// runChainlinkChains polls the non-Ethereum Chainlink deployments +// (bench № 082, freshness only). Same read path as the mainnet +// poller, one dedicated chainlinkPoller per chain so the RPC rotation +// and decimals cache stay chain-local. Successes feed ONLY the +// freshness tracker: these prices never enter the 025 deviation store +// (see ChainFeed doc in config.go). Errors go to the freshness error +// counter, never to ocb_oracle_scrape_errors_total. +func runChainlinkChains(ctx context.Context, feeds []ChainFeed) { + pollers := make(map[string]*chainlinkPoller) + for _, f := range feeds { + if _, ok := pollers[f.Chain]; !ok { + pollers[f.Chain] = newChainlinkPoller(f.RPCs) + } + } + t := time.NewTicker(pollInterval) + defer t.Stop() + + tick := func() { + for _, f := range feeds { + c := pollers[f.Chain] + if c.unsupported[f.Feed] { + freshnessScrapeErrors.WithLabelValues(string(SourceChainlink), string(f.Pair), f.Chain).Inc() + continue + } + pollCtx, cancel := context.WithTimeout(ctx, httpTimeout*2) + price, updatedAt, err := c.read(pollCtx, f.Feed) + cancel() + if err != nil || price <= 0 { + freshnessScrapeErrors.WithLabelValues(string(SourceChainlink), string(f.Pair), f.Chain).Inc() + c.failureCount[f.Feed]++ + if isRevert(err) && c.failureCount[f.Feed] >= 3 { + c.unsupported[f.Feed] = true + fmt.Printf("[chainlink-%s/%s] feed %s marked unsupported after 3 reverts\n", f.Chain, f.Pair, f.Feed) + } + if err != nil && !isRevert(err) { + c.rpcIdx.Add(1) + } + fmt.Printf("[chainlink-%s/%s] err: %v (price=%v)\n", f.Chain, f.Pair, err, price) + continue + } + c.failureCount[f.Feed] = 0 + recordFreshness(string(SourceChainlink), f.Pair, f.Chain, time.Unix(updatedAt, 0)) + } + } + + tick() + for { + select { + case <-ctx.Done(): + return + case <-t.C: + tick() + } + } +} + // read returns (price USD, updatedAt unix seconds, err). func (c *chainlinkPoller) read(ctx context.Context, feed string) (float64, int64, error) { // Fetch decimals first (cached). diff --git a/harnesses/oracle-deviation/cmd/script/config.go b/harnesses/oracle-deviation/cmd/script/config.go index db0f23b6..628e5b6a 100644 --- a/harnesses/oracle-deviation/cmd/script/config.go +++ b/harnesses/oracle-deviation/cmd/script/config.go @@ -83,6 +83,71 @@ func pairs() []PairSpec { } } +// ChainFeed wires one Chainlink AggregatorV3 deployment on a non- +// Ethereum chain to the freshness tracker (bench № 082). These feeds +// are freshness-only: they never enter the 025 deviation store (the +// price store is keyed by (pair, source), so an Arbitrum ETH/USD +// sample would silently overwrite the Ethereum one and change 025's +// deviation numbers). +type ChainFeed struct { + Pair Pair + Chain string // freshness `chain` label: arbitrum | base + Feed string // AggregatorV3 contract on that chain + RPCs []string +} + +// extraChainlinkFeeds lists the non-Ethereum Chainlink deployments the +// freshness bench tracks. Ethereum mainnet feeds are covered by the +// existing 025 poller (which now also feeds recordFreshness). +func extraChainlinkFeeds() []ChainFeed { + return []ChainFeed{ + { + Pair: "ETH/USD", + Chain: ChainArbitrum, + Feed: "0x639Fe6ab55C921f74e7fac1ee960C0B6293ba612", + RPCs: []string{ + envDefault("ORACLE_RPC_ARBITRUM", "https://arbitrum-one-rpc.publicnode.com"), + "https://arb1.arbitrum.io/rpc", + }, + }, + { + Pair: "ETH/USD", + Chain: ChainBase, + Feed: "0x71041dddad3595F9CEd3DcCFBe3D1F4b0a16Bb70", + RPCs: []string{ + envDefault("ORACLE_RPC_BASE", "https://base-rpc.publicnode.com"), + "https://mainnet.base.org", + }, + }, + } +} + +// RedstoneFeed maps a canonical OCB pair to RedStone's symbol on the +// public per-symbol price API. +type RedstoneFeed struct { + Pair Pair + Symbol string +} + +// redstoneFeeds is the freshness cohort for RedStone. ETH + BTC only +// in v1: the per-symbol endpoint costs one request per symbol per +// tick, and the bench's headline is ETH/USD. +func redstoneFeeds() []RedstoneFeed { + return []RedstoneFeed{ + {Pair: "ETH/USD", Symbol: "ETH"}, + {Pair: "BTC/USD", Symbol: "BTC"}, + } +} + +// redstoneBaseURL is RedStone's public per-symbol price API. The +// data-packages gateway payload for the whole redstone-primary-prod +// service is ~1.7 MB per request; the per-symbol endpoint returns a +// single small object with the signed package's value + timestamp, +// so we poll per symbol instead. +func redstoneBaseURL() string { + return envDefault("ORACLE_REDSTONE_URL", "https://api.redstone.finance/prices") +} + // rpcEndpoint is the Ethereum mainnet public RPC used by the // Chainlink poller to call latestRoundData() on the AggregatorV3 // contracts. Two free public endpoints are kept; the second is a diff --git a/harnesses/oracle-deviation/cmd/script/freshness.go b/harnesses/oracle-deviation/cmd/script/freshness.go new file mode 100644 index 00000000..1713f026 --- /dev/null +++ b/harnesses/oracle-deviation/cmd/script/freshness.go @@ -0,0 +1,176 @@ +package main + +import ( + "context" + "math" + "sync" + "time" +) + +// Freshness tracking for OCB bench № 082 (oracle-freshness). +// +// Every freshness-capable poller calls recordFreshness with the +// source-declared update timestamp: +// +// - Chainlink: latestRoundData().updatedAt, per chain (ethereum, +// arbitrum, base). Push oracle: staleness in the multiple hundreds +// of seconds is EXPECTED in calm markets (deviation trigger + +// heartbeat mechanics), not a failure. +// - Pyth: Hermes publish_time. Pull oracle: Hermes IS the price +// source integrators pull from, so publish_time is the honest +// freshness measure (typically 1-2s). +// - RedStone: the signed data-package timestamp from the public +// per-symbol API (typically 15-25s; packages are produced on a +// ~10s cadence). +// +// Because "old" is not the same as "wrong" for a push oracle, the +// bench pairs raw staleness with a second signal: stale_but_moved, +// which only fires when the feed is old AND the market has left it +// behind. Thresholds: +// +// staleThresholdSeconds (300s): half of Chainlink's typical 1h +// heartbeat divided by 6 is arbitrary; 300s was picked because +// every observed healthy update gap on the measured feeds +// (deviation-triggered) lands under it EXCEPT genuinely quiet +// periods, and because 5 minutes of price lag is where perp/ +// lending liquidation math starts to hurt. +// staleMoveThresholdPct (0.5%): matches the widest deviation +// trigger configured on the measured Chainlink feeds. If the CEX +// print moved more than the feed's own trigger and the feed still +// hasn't updated, the feed is late by its own standard. +const ( + staleThresholdSeconds = 300.0 + staleMoveThresholdPct = 0.5 + // freshnessTickInterval refreshes the staleness gauges between + // polls so the gauge grows monotonically instead of stair-stepping + // on the 30s poll cadence. + freshnessTickInterval = 5 * time.Second +) + +// chain label values. For Chainlink the label is the chain the +// aggregator contract lives on. Pyth and RedStone are not read from a +// chain at all (pull-model oracles); their freshness source is named +// instead. +const ( + ChainEthereum = "ethereum" + ChainArbitrum = "arbitrum" + ChainBase = "base" + ChainHermes = "hermes" // Pyth Hermes publish_time + ChainGateway = "gateway" // RedStone public data gateway +) + +type freshKey struct { + oracle string + pair Pair + chain string +} + +type freshState struct { + // lastUpdate is the source-declared timestamp of the feed's most + // recent update (NOT our fetch time). + lastUpdate time.Time + // cexAtUpdate is the CEX reference price snapshotted when we first + // observed this update. 0 when no fresh CEX sample was available. + // Cold-start caveat: for a feed whose current round predates the + // harness boot, the snapshot is taken at boot, not at the round's + // true landing time; it converges on the first real update event. + cexAtUpdate float64 +} + +var ( + freshMu sync.Mutex + fresh = make(map[freshKey]freshState) +) + +// recordFreshness ingests one observation of a feed's own update +// timestamp. Increments the update-events counter when the timestamp +// moved forward vs the previous observation, snapshots the CEX +// reference at that moment, then republishes the gauges. +func recordFreshness(oracle string, pair Pair, chain string, sourceTS time.Time) { + if sourceTS.Unix() <= 0 { + // A zero/absent timestamp would read as ~56 years of staleness. + freshnessScrapeErrors.WithLabelValues(oracle, string(pair), chain).Inc() + return + } + key := freshKey{oracle: oracle, pair: pair, chain: chain} + freshMu.Lock() + st, seen := fresh[key] + if !seen || sourceTS.After(st.lastUpdate) { + if seen { + oracleUpdateEvents.WithLabelValues(oracle, string(pair), chain).Inc() + } + ref, _ := cexRefPrice(pair) + st = freshState{lastUpdate: sourceTS, cexAtUpdate: ref} + fresh[key] = st + } + freshMu.Unlock() + publishFreshness(key, st) +} + +// publishFreshness sets the staleness + stale_but_moved gauges for one +// (oracle, pair, chain) from its stored state. +func publishFreshness(key freshKey, st freshState) { + stale := time.Since(st.lastUpdate).Seconds() + if stale < 0 { + // Source clock marginally ahead of ours (Hermes publish_time + // can lead by sub-second). Clamp instead of publishing a + // negative age. + stale = 0 + } + oracleStalenessSeconds.WithLabelValues(key.oracle, string(key.pair), key.chain).Set(stale) + + moved := 0.0 + if stale > staleThresholdSeconds && st.cexAtUpdate > 0 { + if cur, ok := cexRefPrice(key.pair); ok { + movePct := math.Abs(cur-st.cexAtUpdate) / st.cexAtUpdate * 100 + if movePct > staleMoveThresholdPct { + moved = 1 + } + } + } + oracleStaleButMoved.WithLabelValues(key.oracle, string(key.pair), key.chain).Set(moved) +} + +// cexRefPrice returns the freshest CEX print for a pair from the +// existing 025 price store: Binance first, Coinbase as fallback, both +// subject to the same 2x-poll-interval staleness guard the deviation +// calc uses. ok=false when neither has a fresh sample. +func cexRefPrice(pair Pair) (float64, bool) { + storeMu.RLock() + defer storeMu.RUnlock() + srcMap := store[pair] + if srcMap == nil { + return 0, false + } + for _, src := range []Source{SourceBinance, SourceCoinbase} { + if p, ok := srcMap[src]; ok && time.Since(p.TS) <= 2*pollInterval { + return p.Value, true + } + } + return 0, false +} + +// runFreshnessUpdater republishes every tracked freshness gauge on a +// short cadence so staleness keeps climbing between the 30s polls (a +// Prometheus scrape landing mid-window sees the true age, not the age +// as of the last poll). +func runFreshnessUpdater(ctx context.Context) { + t := time.NewTicker(freshnessTickInterval) + defer t.Stop() + for { + select { + case <-ctx.Done(): + return + case <-t.C: + freshMu.Lock() + snapshot := make(map[freshKey]freshState, len(fresh)) + for k, v := range fresh { + snapshot[k] = v + } + freshMu.Unlock() + for k, v := range snapshot { + publishFreshness(k, v) + } + } + } +} diff --git a/harnesses/oracle-deviation/cmd/script/main.go b/harnesses/oracle-deviation/cmd/script/main.go index c7b96cec..f3e9869c 100644 --- a/harnesses/oracle-deviation/cmd/script/main.go +++ b/harnesses/oracle-deviation/cmd/script/main.go @@ -320,6 +320,7 @@ func main() { installLogCapture() // capture stdout into /logs ring buffer fmt.Println("=== Oracle Deviation Harness ===") fmt.Println("OpenChainBench № 025 — 4 oracles × 10 pairs, max deviation gauge.") + fmt.Println("OpenChainBench № 082 — oracle freshness (chainlink eth/arb/base, pyth hermes, redstone gateway).") fmt.Println() specs := pairs() @@ -331,12 +332,16 @@ func main() { fmt.Println() fmt.Printf("RPC primary: %s\n", rpcEndpoint()) fmt.Printf("RPC fallback: %s\n", rpcEndpointFallback()) + // :2112 is the OCB convention (see top-of-file comment). The env + // override exists for LOCAL runs only, e.g. when another process + // already holds 2112 on a dev machine. Never set it in production. + metricsAddr := envDefault("ORACLE_METRICS_ADDR", ":2112") fmt.Printf("Poll cadence: %s per source per pair\n", pollInterval) - fmt.Println("Metrics server: :2112/metrics") + fmt.Printf("Metrics server: %s/metrics\n", metricsAddr) fmt.Println() go func() { - if err := StartMetricsServer(":2112"); err != nil { + if err := StartMetricsServer(metricsAddr); err != nil { fmt.Printf("[fatal] metrics server: %v\n", err) os.Exit(1) } @@ -351,6 +356,12 @@ func main() { go runCoinbase(ctx, specs) go runLatencyUpdater(ctx) + // № 082 freshness-only pollers. Kept out of the deviation store by + // construction (see ChainFeed / runRedstone docs). + go runChainlinkChains(ctx, extraChainlinkFeeds()) + go runRedstone(ctx, redstoneFeeds()) + go runFreshnessUpdater(ctx) + sig := make(chan os.Signal, 1) signal.Notify(sig, os.Interrupt, syscall.SIGTERM) s := <-sig diff --git a/harnesses/oracle-deviation/cmd/script/metrics.go b/harnesses/oracle-deviation/cmd/script/metrics.go index 9b47ca1b..b56f4ff3 100644 --- a/harnesses/oracle-deviation/cmd/script/metrics.go +++ b/harnesses/oracle-deviation/cmd/script/metrics.go @@ -106,6 +106,59 @@ var ( }, []string{"source", "pair"}, ) + + // --- Bench № 082 (oracle-freshness) metric families ------------- + // + // Deliberately NEW families instead of adding a `chain` label to + // the existing ones: bench 025's PromQL surface selects on + // {source, pair} and aggregates with count by (source); injecting + // a chain label there would split its series and silently change + // its numbers. Freshness gets its own label set: + // + // oracle: chainlink | pyth | redstone + // pair: canonical OCB pair (ETH/USD, BTC/USD, ...) + // chain: where the freshness timestamp lives. For Chainlink + // this is the chain the aggregator contract is deployed + // on (ethereum | arbitrum | base). Pyth is pull-based so + // its authoritative timestamp is the Hermes publish_time + // (chain="hermes"); RedStone's is the signed data-package + // timestamp from its public gateway (chain="gateway"). + + oracleStalenessSeconds = promauto.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "ocb_oracle_staleness_seconds", + Help: "Seconds between now and the oracle's own last-update timestamp (Chainlink latestRoundData.updatedAt per chain, Pyth Hermes publish_time, RedStone data-package timestamp). Refreshed every 5s between polls so staleness grows monotonically until the next observed update.", + }, + []string{"oracle", "pair", "chain"}, + ) + + oracleUpdateEvents = promauto.NewCounterVec( + prometheus.CounterOpts{ + Name: "ocb_oracle_update_events_total", + Help: "Count of observed oracle updates: increments each time the source-declared update timestamp moves forward. Bounded below by reality and above by the 30s poll cadence (two updates inside one poll window count once).", + }, + []string{"oracle", "pair", "chain"}, + ) + + oracleStaleButMoved = promauto.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "ocb_oracle_stale_but_moved", + Help: "1 when the feed is older than 300s AND the CEX reference price (Binance, Coinbase fallback) has moved more than 0.5% since the update was observed, else 0. Staleness alone is normal for deviation-triggered feeds; stale AND moved is the dangerous state.", + }, + []string{"oracle", "pair", "chain"}, + ) + + // Separate error counter for the freshness pollers. NOT merged into + // ocb_oracle_scrape_errors_total because bench 025's success queries + // sum that counter across all sources per pair; redstone / L2 + // chainlink failures must not depress 025's success rate. + freshnessScrapeErrors = promauto.NewCounterVec( + prometheus.CounterOpts{ + Name: "ocb_oracle_freshness_scrape_errors_total", + Help: "Failed freshness scrape attempts per (oracle, pair, chain). Kept separate from ocb_oracle_scrape_errors_total so bench 025's error-rate queries are unaffected by freshness-only sources.", + }, + []string{"oracle", "pair", "chain"}, + ) ) // StartMetricsServer binds /metrics + /health on addr. Blocking call — diff --git a/harnesses/oracle-deviation/cmd/script/pyth.go b/harnesses/oracle-deviation/cmd/script/pyth.go index ac4fd163..20e27206 100644 --- a/harnesses/oracle-deviation/cmd/script/pyth.go +++ b/harnesses/oracle-deviation/cmd/script/pyth.go @@ -56,6 +56,7 @@ func runPyth(ctx context.Context, specs []PairSpec) { if err != nil { for _, s := range specs { oracleScrapeErrors.WithLabelValues(string(SourcePyth), string(s.Pair)).Inc() + freshnessScrapeErrors.WithLabelValues(string(SourcePyth), string(s.Pair), ChainHermes).Inc() } fmt.Printf("[pyth] http err: %v\n", err) return @@ -65,6 +66,7 @@ func runPyth(ctx context.Context, specs []PairSpec) { if resp.StatusCode != 200 { for _, s := range specs { oracleScrapeErrors.WithLabelValues(string(SourcePyth), string(s.Pair)).Inc() + freshnessScrapeErrors.WithLabelValues(string(SourcePyth), string(s.Pair), ChainHermes).Inc() } fmt.Printf("[pyth] http %d: %s\n", resp.StatusCode, string(body)) return @@ -73,6 +75,7 @@ func runPyth(ctx context.Context, specs []PairSpec) { if err := json.Unmarshal(body, &entries); err != nil { for _, s := range specs { oracleScrapeErrors.WithLabelValues(string(SourcePyth), string(s.Pair)).Inc() + freshnessScrapeErrors.WithLabelValues(string(SourcePyth), string(s.Pair), ChainHermes).Inc() } fmt.Printf("[pyth] decode err: %v\n", err) return @@ -86,9 +89,24 @@ func runPyth(ctx context.Context, specs []PairSpec) { price, err := pythToFloat(e.Price.Price, e.Price.Expo) if err != nil { oracleScrapeErrors.WithLabelValues(string(SourcePyth), string(pair)).Inc() + freshnessScrapeErrors.WithLabelValues(string(SourcePyth), string(pair), ChainHermes).Inc() continue } - recordPrice(SourcePyth, pair, price) + // Bug fix: Hermes attaches publish_time to every price and + // we used to discard it (recordPrice stamped fetch time as + // SourceTS). Pyth is a pull oracle, so publish_time IS the + // authoritative freshness of the price integrators pull. + // Use it as SourceTS (typically 1-2s behind fetch, well + // inside the 10s alignment tolerance, so the 025 aligned + // deviation is unaffected in practice) and feed the № 082 + // freshness tracker from it. + if e.Price.PublishTime > 0 { + srcTS := time.Unix(e.Price.PublishTime, 0) + recordPriceAt(SourcePyth, pair, price, srcTS) + recordFreshness(string(SourcePyth), pair, ChainHermes, srcTS) + } else { + recordPrice(SourcePyth, pair, price) + } got[pair] = true } // Mark missing pairs as errored so the deviation map doesn't @@ -96,6 +114,7 @@ func runPyth(ctx context.Context, specs []PairSpec) { for _, s := range specs { if !got[s.Pair] { oracleScrapeErrors.WithLabelValues(string(SourcePyth), string(s.Pair)).Inc() + freshnessScrapeErrors.WithLabelValues(string(SourcePyth), string(s.Pair), ChainHermes).Inc() } } } diff --git a/harnesses/oracle-deviation/cmd/script/redstone.go b/harnesses/oracle-deviation/cmd/script/redstone.go new file mode 100644 index 00000000..a91930fb --- /dev/null +++ b/harnesses/oracle-deviation/cmd/script/redstone.go @@ -0,0 +1,106 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "time" +) + +// RedStone freshness poller (bench № 082 only). RedStone is a +// modular oracle: signed data packages are produced off-chain on a +// ~10s cadence and delivered on-chain either by relayers (push +// model) or attached to transactions by the integrator (pull model, +// "core"). The freshest public view of the primary production +// service is the keyless per-symbol price API, which returns the +// latest signed package's value and timestamp. We deliberately avoid +// the data-packages gateway's full-service payload (~1.7 MB per +// request for redstone-primary-prod) and poll one small per-symbol +// object per pair instead. +// +// RedStone samples feed ONLY the freshness metrics. They never enter +// the 025 deviation store: bench 025 is a fixed 4-source matrix +// (chainlink/pyth/binance/coinbase) and its queries count sources per +// pair, so adding a 5th source would silently change its numbers. + +type redstonePrice struct { + Symbol string `json:"symbol"` + Value float64 `json:"value"` + Timestamp int64 `json:"timestamp"` // unix milliseconds +} + +func runRedstone(ctx context.Context, feeds []RedstoneFeed) { + client := &http.Client{Timeout: httpTimeout} + t := time.NewTicker(pollInterval) + defer t.Stop() + + tick := func() { + for _, f := range feeds { + pollCtx, cancel := context.WithTimeout(ctx, httpTimeout) + ts, err := fetchRedstone(pollCtx, client, f.Symbol) + cancel() + if err != nil { + freshnessScrapeErrors.WithLabelValues("redstone", string(f.Pair), ChainGateway).Inc() + fmt.Printf("[redstone/%s] err: %v\n", f.Pair, err) + continue + } + recordFreshness("redstone", f.Pair, ChainGateway, ts) + } + } + + tick() + for { + select { + case <-ctx.Done(): + return + case <-t.C: + tick() + } + } +} + +// fetchRedstone returns the timestamp of the latest signed data +// package for one symbol. GET /prices?symbol=&provider=redstone&limit=1 +// answers with a one-element array; timestamp is unix ms. +func fetchRedstone(ctx context.Context, client *http.Client, symbol string) (time.Time, error) { + u := redstoneBaseURL() + "?symbol=" + url.QueryEscape(symbol) + "&provider=redstone&limit=1" + req, err := http.NewRequestWithContext(ctx, "GET", u, nil) + if err != nil { + return time.Time{}, err + } + req.Header.Set("User-Agent", "openchainbench-oracle-freshness/1.0") + resp, err := client.Do(req) + if err != nil { + return time.Time{}, err + } + defer resp.Body.Close() + body, _ := io.ReadAll(resp.Body) + if resp.StatusCode != 200 { + return time.Time{}, fmt.Errorf("http %d: %s", resp.StatusCode, truncate(string(body), 200)) + } + var arr []redstonePrice + if err := json.Unmarshal(body, &arr); err != nil { + return time.Time{}, fmt.Errorf("decode: %w", err) + } + if len(arr) == 0 { + return time.Time{}, fmt.Errorf("empty result for %s", symbol) + } + p := arr[0] + if p.Value <= 0 { + return time.Time{}, fmt.Errorf("non-positive value %v for %s", p.Value, symbol) + } + if p.Timestamp <= 0 { + return time.Time{}, fmt.Errorf("missing timestamp for %s", symbol) + } + return time.UnixMilli(p.Timestamp), nil +} + +func truncate(s string, n int) string { + if len(s) <= n { + return s + } + return s[:n] + "…" +} From 06242d3eb3e5752212ed9ce0d764de913bef7764 Mon Sep 17 00:00:00 2001 From: Florent Tapponnier Date: Wed, 15 Jul 2026 01:06:16 +0200 Subject: [PATCH 06/12] oracle-freshness: count unsupported mainnet feeds in freshness error counter --- harnesses/oracle-deviation/cmd/script/chainlink.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/harnesses/oracle-deviation/cmd/script/chainlink.go b/harnesses/oracle-deviation/cmd/script/chainlink.go index 6bf7c6f7..f8648564 100644 --- a/harnesses/oracle-deviation/cmd/script/chainlink.go +++ b/harnesses/oracle-deviation/cmd/script/chainlink.go @@ -122,8 +122,12 @@ func runChainlink(ctx context.Context, specs []PairSpec) { } if c.unsupported[s.ChainlinkFeed] { // Mark error every cycle so the deviation calc skips - // stale values, but skip the network round-trip. + // stale values, but skip the network round-trip. Also + // signal the freshness error counter (mirrors + // runChainlinkChains) so a dead feed's climbing + // staleness gauge is distinguishable from a quiet feed. oracleScrapeErrors.WithLabelValues(string(SourceChainlink), string(s.Pair)).Inc() + freshnessScrapeErrors.WithLabelValues(string(SourceChainlink), string(s.Pair), ChainEthereum).Inc() continue } pollCtx, cancel := context.WithTimeout(ctx, httpTimeout*2) From 4fe73b1c8cf3d26f5be7c0a39f0f0908be7980b3 Mon Sep 17 00:00:00 2001 From: Florent Tapponnier Date: Wed, 15 Jul 2026 01:05:44 +0200 Subject: [PATCH 07/12] add bench 085 evm-block-builders: ETH builder share via extraData + relay bidtraces + Arb/Base soft-conf lag, bump bench-set cache keys --- benchmarks/evm-block-builders.yml | 245 ++++++++++++++++ harnesses/evm-block-builders/Dockerfile | 22 ++ .../evm-block-builders/cmd/script/arb_feed.go | 270 ++++++++++++++++++ .../cmd/script/base_flashblocks.go | 257 +++++++++++++++++ .../evm-block-builders/cmd/script/builders.go | 192 +++++++++++++ .../evm-block-builders/cmd/script/config.go | 152 ++++++++++ .../evm-block-builders/cmd/script/loghub.go | 114 ++++++++ .../evm-block-builders/cmd/script/main.go | 76 +++++ .../evm-block-builders/cmd/script/metrics.go | 223 +++++++++++++++ .../evm-block-builders/cmd/script/relays.go | 134 +++++++++ harnesses/evm-block-builders/go.mod | 22 ++ harnesses/evm-block-builders/go.sum | 52 ++++ 12 files changed, 1759 insertions(+) create mode 100644 benchmarks/evm-block-builders.yml create mode 100644 harnesses/evm-block-builders/Dockerfile create mode 100644 harnesses/evm-block-builders/cmd/script/arb_feed.go create mode 100644 harnesses/evm-block-builders/cmd/script/base_flashblocks.go create mode 100644 harnesses/evm-block-builders/cmd/script/builders.go create mode 100644 harnesses/evm-block-builders/cmd/script/config.go create mode 100644 harnesses/evm-block-builders/cmd/script/loghub.go create mode 100644 harnesses/evm-block-builders/cmd/script/main.go create mode 100644 harnesses/evm-block-builders/cmd/script/metrics.go create mode 100644 harnesses/evm-block-builders/cmd/script/relays.go create mode 100644 harnesses/evm-block-builders/go.mod create mode 100644 harnesses/evm-block-builders/go.sum diff --git a/benchmarks/evm-block-builders.yml b/benchmarks/evm-block-builders.yml new file mode 100644 index 00000000..b2fddf65 --- /dev/null +++ b/benchmarks/evm-block-builders.yml @@ -0,0 +1,245 @@ +# OpenChainBench. Bench № 085 + +slug: evm-block-builders +number: "085" +title: Ethereum block builder market share +seo_title: "Ethereum block builders benchmark 2026, live builder market share" +seo_description: "Live benchmark of who builds Ethereum blocks. Builder market share attributed via block extraData, cross-checked against MEV-Boost relay bidtraces." +subtitle: Share of Ethereum mainnet blocks attributed to each block builder via the extraData self-label, cross-checked against relay bidtraces. Observational, no transactions sent. +category: Trading +status: live +metric: Block share (24h) +unit: pct +higher_is_better: true + +disclaimer: | + Attribution relies on the extraData tag each builder voluntarily stamps into its blocks. A builder can change or strip its tag at any time; unrecognized tags land in the Other row and every raw string is logged so the table grows. High share is a centralization datapoint, not a quality ranking. + +seo_intro: | + Who actually builds Ethereum? Since MEV-Boost, validators outsource + block construction to a handful of specialized builders, and the + builders sign their work: each stamps a self-label into the block's + extraData field ("Titan (titanbuilder.xyz)", "beaverbuild.org", and + so on). This benchmark decodes the extraData of every mainnet block, + attributes it to a curated table of known builder tags, and publishes + the live market share. The result is the concentration story in one + chart: 95 percent plus of Ethereum blocks are built by a handful of + private builders, and the top two regularly build most of the chain. + Blocks carrying an execution client's default tag (geth, reth, + nethermind, besu version strings) are counted as vanilla, meaning the + proposer built locally instead of outsourcing; that row is the live + measure of how much of Ethereum still self-builds. As an independent + cross-check the harness also polls the public bidtrace data APIs of + seven MEV-Boost relays (Flashbots, ultrasound, Agnostic, bloXroute + max-profit and regulated, Titan, Aestus) and counts delivered + payloads per relay, a signal that does not depend on any builder's + self-chosen label. + +abstract: | + We attribute every Ethereum mainnet block to its builder by decoding + the block's extraData field, the 32 bytes a builder uses to sign its + work in human-readable ASCII. The harness polls the head of the chain + through a keyless public RPC once per 12 second slot, backfills any + blocks missed between polls by number, decodes extraData to printable + ASCII and matches it against a curated substring table of known + builder tags (Titan, Quasar, Eureka, BuilderNet, beaverbuild, rsync, + Flashbots, Builder+ and more). Client-default tags (geth, reth, + nethermind, besu, erigon version strings) and empty extraData count + as vanilla, locally built blocks. Anything unrecognized counts as + Other, increments an unattributed counter, and logs the raw string so + the table can grow, the same philosophy as the tip-wallet table in + bench 016. The headline metric is each builder's percentage of all + blocks observed in the last 24 hours. Two companion measurements ship + in the same harness. First, an independent cross-check: the public + bidtrace APIs of seven MEV-Boost relays are polled every five minutes + and delivered payloads are counted per relay, deduped per relay by + slot high-water-mark; the same slot appearing on several relays is + normal because builders multi-home their bids. Second, an L2 + soft-confirmation study: the harness holds the Arbitrum sequencer + feed and the Base flashblocks stream open and measures how far those + preconfirmation channels run ahead of the public RPC head, emitted as + histograms for a future bench page. At inception a 50 block live + sample attributed 98 percent cleanly: Titan 19, Quasar 16, Eureka 6, + BuilderNet 2, beaverbuild 1, Builder+ 1, one vanilla block. + +methodology: + - "Source: keyless public Ethereum RPC (ethereum-rpc.publicnode.com by default, env-overridable). eth_getBlockByNumber(latest) every 12s, matching the mainnet slot time. Gaps between polls are backfilled by block number (capped at 10) so the counters see every block, not just the ones that happen to be head at poll time." + - "Attribution: extraData hex is decoded to printable ASCII and matched, lowercase, against a curated substring table (titan, quasar, eureka, buildernet, beaverbuild, rsync, builder0x69, flashbots, bloxroute, builder+/btcs, penguinbuild and more). First match wins; most specific substrings are ordered first." + - "Vanilla detection: empty extraData or execution-client default tags (geth, reth, nethermind, besu, erigon, go1.x, platform strings) count as builder=vanilla, meaning the proposer built the block locally instead of outsourcing to a builder. This row is the live self-built share of Ethereum." + - "Unattributed handling: any non-empty tag not in the table counts as builder=other, increments ebb_unattributed_total, and logs the raw extraData string. The table grows from operator logs instead of silently under-counting, the bench 016 pattern." + - "Self-labeling caveat: extraData is voluntary. A builder can rebrand, blank its tag, or impersonate another. The relay cross-check below is the independent signal; a large divergence between extraData share and relay-side delivered payloads would surface tag games." + - "Relay cross-check: seven MEV-Boost relay data APIs (Flashbots, ultrasound, Agnostic, bloXroute max-profit, bloXroute regulated, Titan, Aestus) are polled keyless every 5 min via GET /relay/v1/data/bidtraces/proposer_payload_delivered?limit=50. Delivered payloads are counted per relay (ebb_relay_payloads_total), deduped per relay by slot high-water-mark. The same slot on multiple relays is normal: builders multi-home bids, so relay counters measure relay share, not a partition of blocks." + - "Single vantage: one harness in one region observes the chain through one public RPC. Attribution is not latency-sensitive (extraData is consensus data, identical from every vantage), so single-vantage only affects liveness, which the health gauges expose." + - "Companion L2 metrics: the same harness measures Arbitrum sequencer-feed soft-confirmation lag (ebb_arb_softconf_lag_milliseconds, feed arrival to public RPC visibility, offset derived at runtime) and Base flashblocks cadence plus flashblock-to-RPC lag (ebb_base_flashblock_interval_milliseconds, ebb_base_softconf_lag_milliseconds). These are per-chain, not per-builder, so this spec surfaces builder share only; the L2 lag series feed a future bench page." + +findings: + - "{{best_name}} currently builds {{best_p50}} of Ethereum blocks observed over the last 24h, across {{count}} attribution rows. The top two builders regularly account for the majority of the chain, the concentration this bench exists to make visible." + - "{{name:beaverbuild}} sits at {{p50:beaverbuild}}. beaverbuild dominated 2023-2024 alongside rsync; its slide against Titan and the newer Quasar and Eureka operations is the clearest sign the builder market still churns even as it concentrates." + - "{{name:buildernet}} carries {{p50:buildernet}}. BuilderNet is the Flashbots-initiated attempt to mutualize block building into a multi-operator network precisely because of the concentration this page measures; its share is the decentralization counter-signal to watch." + - "{{name:vanilla}} blocks, built locally by the proposer without a builder, sit at {{p50:vanilla}}. This is the live measure of how much of Ethereum still self-builds rather than outsourcing to the MEV supply chain." + - "{{worst_name}} trails at {{worst_p50}}. The long tail matters less than the head here: the bench's story is that a validator set of a million keys funnels block construction through a single-digit number of firms." + +faq: + - q: "How is a block attributed to a builder?" + a: "Builders stamp a self-label into the 32-byte extraData field of every block they construct, for example 'Titan (titanbuilder.xyz)' or 'beaverbuild.org'. The harness decodes extraData to ASCII and matches it against a curated substring table. Empty or client-default extraData (geth, reth, nethermind version strings) means the proposer built the block locally and counts as vanilla. Unrecognized tags count as Other and are logged raw so the table grows. At inception, 98 percent of a live 50-block sample attributed cleanly." + - q: "Can a builder game this by changing its extraData?" + a: "Yes, extraData is voluntary self-labeling, and that is disclosed as the bench's main caveat. A builder could blank its tag (blocks would fall into Other, and the unattributed counter plus raw-string logging would surface the shift within hours) or imitate another builder's tag (detectable by divergence against relay bidtraces, which report the builder's signing pubkey rather than its label). In practice builders keep their tags stable because the label is marketing: searchers pick builders partly on public share stats." + - q: "Why do the p50, p90 and p99 columns show the same number?" + a: "This bench has no latency dimension, so the percentile slots are repurposed, same convention as bench 016. Every aggregate column carries the builder's 24h block share; the sample column carries the raw block count and the success column reports the harness's own RPC poll health, not anything about the builder. The ranking signal is the share itself." + - q: "What is the relay cross-check and why does it not sum to 100 percent?" + a: "Seven MEV-Boost relays expose a public data API listing every payload they delivered to a proposer. The harness counts delivered payloads per relay every five minutes. Because builders submit the same bid through several relays and every relay that carried the winning payload reports it, one block can legitimately appear on multiple relays. Relay counts therefore measure relay market share and act as an attribution sanity check independent of extraData; they are not a partition of blocks and do not sum to the block count." + - q: "Why does builder concentration matter?" + a: "Under MEV-Boost, validators outsource block construction, so whoever builds most blocks decides transaction ordering and inclusion for most of Ethereum. Concentration in two or three private firms creates censorship surface (a dominant builder filtering transactions affects most blocks), a single point of failure for liveness of the fee market, and information asymmetry for anyone trading on-chain. Projects like BuilderNet exist specifically to mutualize this layer; this bench provides the neutral number for whether concentration is getting better or worse." + +source: https://github.com/ChainBench/OpenChainBench/tree/main/harnesses/evm-block-builders + +prometheus: + window: 24h + expected_freshness_seconds: 120 + freshness_metric: ebb_eth_last_block + +# Real metrics emitted by the evm-block-builders harness: +# ebb_blocks_total{builder} counter (attributed blocks) +# ebb_unattributed_total counter +# ebb_eth_poll_health gauge (0|1) +# ebb_eth_last_block gauge (freshness) +# ebb_relay_payloads_total{relay} counter (cross-check) +# ebb_arb_softconf_lag_milliseconds histogram (future bench) +# ebb_base_flashblock_interval_milliseconds histogram (future bench) +# ebb_base_softconf_lag_milliseconds histogram (future bench) +# +# Each "provider" is one builder attribution row. The headline is the +# builder's share of all blocks observed in 24h (pct). p50/p90/p99/mean +# carry the same share scalar (016 convention); sample_size carries the +# raw block count; success carries harness poll health. + +providers: + - slug: titan + name: Titan + tag: titanbuilder.xyz, dominant private builder + formula: "Blocks whose extraData contains the Titan tag, as a percentage of all Ethereum blocks observed in the last 24h." + queries: + p50: 100 * sum(increase(ebb_blocks_total{builder="titan"}[24h])) / scalar(sum(increase(ebb_blocks_total[24h]))) + p90: 100 * sum(increase(ebb_blocks_total{builder="titan"}[24h])) / scalar(sum(increase(ebb_blocks_total[24h]))) + p99: 100 * sum(increase(ebb_blocks_total{builder="titan"}[24h])) / scalar(sum(increase(ebb_blocks_total[24h]))) + mean: 100 * sum(increase(ebb_blocks_total{builder="titan"}[24h])) / scalar(sum(increase(ebb_blocks_total[24h]))) + success: avg_over_time(ebb_eth_poll_health[24h]) + sample_size: sum(increase(ebb_blocks_total{builder="titan"}[24h])) + series: 100 * sum(rate(ebb_blocks_total{builder="titan"}[1h])) / scalar(sum(rate(ebb_blocks_total[1h]))) + + - slug: quasar + name: Quasar + tag: Fast-rising private builder + formula: "Blocks whose extraData contains the Quasar tag, as a percentage of all Ethereum blocks observed in the last 24h." + queries: + p50: 100 * sum(increase(ebb_blocks_total{builder="quasar"}[24h])) / scalar(sum(increase(ebb_blocks_total[24h]))) + p90: 100 * sum(increase(ebb_blocks_total{builder="quasar"}[24h])) / scalar(sum(increase(ebb_blocks_total[24h]))) + p99: 100 * sum(increase(ebb_blocks_total{builder="quasar"}[24h])) / scalar(sum(increase(ebb_blocks_total[24h]))) + mean: 100 * sum(increase(ebb_blocks_total{builder="quasar"}[24h])) / scalar(sum(increase(ebb_blocks_total[24h]))) + success: avg_over_time(ebb_eth_poll_health[24h]) + sample_size: sum(increase(ebb_blocks_total{builder="quasar"}[24h])) + series: 100 * sum(rate(ebb_blocks_total{builder="quasar"}[1h])) / scalar(sum(rate(ebb_blocks_total[1h]))) + + - slug: eureka + name: Eureka + tag: Private builder, newer entrant + formula: "Blocks whose extraData contains the Eureka tag, as a percentage of all Ethereum blocks observed in the last 24h." + queries: + p50: 100 * sum(increase(ebb_blocks_total{builder="eureka"}[24h])) / scalar(sum(increase(ebb_blocks_total[24h]))) + p90: 100 * sum(increase(ebb_blocks_total{builder="eureka"}[24h])) / scalar(sum(increase(ebb_blocks_total[24h]))) + p99: 100 * sum(increase(ebb_blocks_total{builder="eureka"}[24h])) / scalar(sum(increase(ebb_blocks_total[24h]))) + mean: 100 * sum(increase(ebb_blocks_total{builder="eureka"}[24h])) / scalar(sum(increase(ebb_blocks_total[24h]))) + success: avg_over_time(ebb_eth_poll_health[24h]) + sample_size: sum(increase(ebb_blocks_total{builder="eureka"}[24h])) + series: 100 * sum(rate(ebb_blocks_total{builder="eureka"}[1h])) / scalar(sum(rate(ebb_blocks_total[1h]))) + + - slug: buildernet + name: BuilderNet + tag: Flashbots-initiated multi-operator building network + formula: "Blocks whose extraData contains the BuilderNet tag, as a percentage of all Ethereum blocks observed in the last 24h." + queries: + p50: 100 * sum(increase(ebb_blocks_total{builder="buildernet"}[24h])) / scalar(sum(increase(ebb_blocks_total[24h]))) + p90: 100 * sum(increase(ebb_blocks_total{builder="buildernet"}[24h])) / scalar(sum(increase(ebb_blocks_total[24h]))) + p99: 100 * sum(increase(ebb_blocks_total{builder="buildernet"}[24h])) / scalar(sum(increase(ebb_blocks_total[24h]))) + mean: 100 * sum(increase(ebb_blocks_total{builder="buildernet"}[24h])) / scalar(sum(increase(ebb_blocks_total[24h]))) + success: avg_over_time(ebb_eth_poll_health[24h]) + sample_size: sum(increase(ebb_blocks_total{builder="buildernet"}[24h])) + series: 100 * sum(rate(ebb_blocks_total{builder="buildernet"}[1h])) / scalar(sum(rate(ebb_blocks_total[1h]))) + + - slug: beaverbuild + name: beaverbuild + tag: beaverbuild.org, former market leader + formula: "Blocks whose extraData contains the beaverbuild tag, as a percentage of all Ethereum blocks observed in the last 24h." + queries: + p50: 100 * sum(increase(ebb_blocks_total{builder="beaverbuild"}[24h])) / scalar(sum(increase(ebb_blocks_total[24h]))) + p90: 100 * sum(increase(ebb_blocks_total{builder="beaverbuild"}[24h])) / scalar(sum(increase(ebb_blocks_total[24h]))) + p99: 100 * sum(increase(ebb_blocks_total{builder="beaverbuild"}[24h])) / scalar(sum(increase(ebb_blocks_total[24h]))) + mean: 100 * sum(increase(ebb_blocks_total{builder="beaverbuild"}[24h])) / scalar(sum(increase(ebb_blocks_total[24h]))) + success: avg_over_time(ebb_eth_poll_health[24h]) + sample_size: sum(increase(ebb_blocks_total{builder="beaverbuild"}[24h])) + series: 100 * sum(rate(ebb_blocks_total{builder="beaverbuild"}[1h])) / scalar(sum(rate(ebb_blocks_total[1h]))) + + - slug: rsync + name: rsync + tag: rsync-builder.xyz + formula: "Blocks whose extraData contains the rsync tag, as a percentage of all Ethereum blocks observed in the last 24h." + queries: + p50: 100 * sum(increase(ebb_blocks_total{builder="rsync"}[24h])) / scalar(sum(increase(ebb_blocks_total[24h]))) + p90: 100 * sum(increase(ebb_blocks_total{builder="rsync"}[24h])) / scalar(sum(increase(ebb_blocks_total[24h]))) + p99: 100 * sum(increase(ebb_blocks_total{builder="rsync"}[24h])) / scalar(sum(increase(ebb_blocks_total[24h]))) + mean: 100 * sum(increase(ebb_blocks_total{builder="rsync"}[24h])) / scalar(sum(increase(ebb_blocks_total[24h]))) + success: avg_over_time(ebb_eth_poll_health[24h]) + sample_size: sum(increase(ebb_blocks_total{builder="rsync"}[24h])) + series: 100 * sum(rate(ebb_blocks_total{builder="rsync"}[1h])) / scalar(sum(rate(ebb_blocks_total[1h]))) + + - slug: flashbots + name: Flashbots + tag: The original MEV-Boost builder + formula: "Blocks whose extraData contains a Flashbots builder tag (flashbots or Illuminate Dmocratize), as a percentage of all Ethereum blocks observed in the last 24h." + queries: + p50: 100 * sum(increase(ebb_blocks_total{builder="flashbots"}[24h])) / scalar(sum(increase(ebb_blocks_total[24h]))) + p90: 100 * sum(increase(ebb_blocks_total{builder="flashbots"}[24h])) / scalar(sum(increase(ebb_blocks_total[24h]))) + p99: 100 * sum(increase(ebb_blocks_total{builder="flashbots"}[24h])) / scalar(sum(increase(ebb_blocks_total[24h]))) + mean: 100 * sum(increase(ebb_blocks_total{builder="flashbots"}[24h])) / scalar(sum(increase(ebb_blocks_total[24h]))) + success: avg_over_time(ebb_eth_poll_health[24h]) + sample_size: sum(increase(ebb_blocks_total{builder="flashbots"}[24h])) + series: 100 * sum(rate(ebb_blocks_total{builder="flashbots"}[1h])) / scalar(sum(rate(ebb_blocks_total[1h]))) + + - slug: btcs + name: Builder+ + tag: BTCS Builder+ + formula: "Blocks whose extraData contains the Builder+ or BTCS tag, as a percentage of all Ethereum blocks observed in the last 24h." + queries: + p50: 100 * sum(increase(ebb_blocks_total{builder="btcs"}[24h])) / scalar(sum(increase(ebb_blocks_total[24h]))) + p90: 100 * sum(increase(ebb_blocks_total{builder="btcs"}[24h])) / scalar(sum(increase(ebb_blocks_total[24h]))) + p99: 100 * sum(increase(ebb_blocks_total{builder="btcs"}[24h])) / scalar(sum(increase(ebb_blocks_total[24h]))) + mean: 100 * sum(increase(ebb_blocks_total{builder="btcs"}[24h])) / scalar(sum(increase(ebb_blocks_total[24h]))) + success: avg_over_time(ebb_eth_poll_health[24h]) + sample_size: sum(increase(ebb_blocks_total{builder="btcs"}[24h])) + series: 100 * sum(rate(ebb_blocks_total{builder="btcs"}[1h])) / scalar(sum(rate(ebb_blocks_total[1h]))) + + - slug: vanilla + name: Vanilla + tag: Locally built by the proposer, no builder + formula: "Blocks with empty or execution-client default extraData (geth, reth, nethermind, besu, erigon), meaning the proposer built locally, as a percentage of all blocks observed in 24h." + queries: + p50: 100 * sum(increase(ebb_blocks_total{builder="vanilla"}[24h])) / scalar(sum(increase(ebb_blocks_total[24h]))) + p90: 100 * sum(increase(ebb_blocks_total{builder="vanilla"}[24h])) / scalar(sum(increase(ebb_blocks_total[24h]))) + p99: 100 * sum(increase(ebb_blocks_total{builder="vanilla"}[24h])) / scalar(sum(increase(ebb_blocks_total[24h]))) + mean: 100 * sum(increase(ebb_blocks_total{builder="vanilla"}[24h])) / scalar(sum(increase(ebb_blocks_total[24h]))) + success: avg_over_time(ebb_eth_poll_health[24h]) + sample_size: sum(increase(ebb_blocks_total{builder="vanilla"}[24h])) + series: 100 * sum(rate(ebb_blocks_total{builder="vanilla"}[1h])) / scalar(sum(rate(ebb_blocks_total[1h]))) + + - slug: other + name: Other + tag: Unrecognized extraData tags, logged raw for table growth + formula: "Blocks whose extraData carries a non-empty tag not in the curated table, as a percentage of all blocks observed in 24h. Every raw string is logged so the attribution table grows." + queries: + p50: 100 * sum(increase(ebb_blocks_total{builder="other"}[24h])) / scalar(sum(increase(ebb_blocks_total[24h]))) + p90: 100 * sum(increase(ebb_blocks_total{builder="other"}[24h])) / scalar(sum(increase(ebb_blocks_total[24h]))) + p99: 100 * sum(increase(ebb_blocks_total{builder="other"}[24h])) / scalar(sum(increase(ebb_blocks_total[24h]))) + mean: 100 * sum(increase(ebb_blocks_total{builder="other"}[24h])) / scalar(sum(increase(ebb_blocks_total[24h]))) + success: avg_over_time(ebb_eth_poll_health[24h]) + sample_size: sum(increase(ebb_blocks_total{builder="other"}[24h])) + series: 100 * sum(rate(ebb_blocks_total{builder="other"}[1h])) / scalar(sum(rate(ebb_blocks_total[1h]))) diff --git a/harnesses/evm-block-builders/Dockerfile b/harnesses/evm-block-builders/Dockerfile new file mode 100644 index 00000000..7a6bc559 --- /dev/null +++ b/harnesses/evm-block-builders/Dockerfile @@ -0,0 +1,22 @@ +FROM golang:1.24-alpine AS builder + +WORKDIR /app +RUN apk add --no-cache git + +COPY go.mod go.sum ./ +RUN go mod download + +COPY . . + +RUN CGO_ENABLED=0 GOOS=linux go build -o /app/evm-block-builders ./cmd/script + +FROM debian:bookworm-slim + +WORKDIR /app +RUN apt-get update && apt-get install -y ca-certificates && rm -rf /var/lib/apt/lists/* + +COPY --from=builder /app/evm-block-builders /app/evm-block-builders + +EXPOSE 2112 + +CMD ["/app/evm-block-builders"] diff --git a/harnesses/evm-block-builders/cmd/script/arb_feed.go b/harnesses/evm-block-builders/cmd/script/arb_feed.go new file mode 100644 index 00000000..29c51a0f --- /dev/null +++ b/harnesses/evm-block-builders/cmd/script/arb_feed.go @@ -0,0 +1,270 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "sync" + "sync/atomic" + "time" + + "github.com/gorilla/websocket" +) + +// Arbitrum sequencer feed soft-confirmation lag. +// +// The sequencer feed (wss://arb1.arbitrum.io/feed) streams every L2 +// message the sequencer accepts, seconds before the block is queryable +// on a public RPC. That gap IS the soft-confirmation advantage: trading +// systems reading the feed act on state that RPC users haven't seen +// yet. We hold the feed open, timestamp each message's arrival keyed by +// sequence number, poll the public RPC head, and emit the feed→RPC +// visibility lag as a histogram. +// +// Feed sequenceNumber → L2 block number is a FIXED offset (genesis +// alignment; 22207817 at inception) but we derive it at runtime instead +// of hardcoding. Per calibration poll we sample cand = rpcHead - +// latestFeedSeq. While the feed is live, cand is trueOffset minus the +// 0-1 blocks in flight between feed and RPC (lag ~0.1s vs 0.25s block +// time), so trueOffset is the LARGEST frequently-seen candidate. A +// plain max is NOT safe: on connect the feed replays a short backlog, +// latestSeq briefly trails the chain, and cand overshoots (observed +// +3 → every lag inflated by ~750ms). We therefore ignore the first +// polls after the feed goes live and pick the highest candidate with +// meaningful support. The derived value is logged and exported +// (ebb_arb_derived_offset) so drift would be visible. + +const ( + arbDialTimeout = 15 * time.Second + arbReadDeadline = 60 * time.Second + arbWatchdog = 30 * time.Second + arbMinBackoff = 2 * time.Second + arbMaxBackoff = 60 * time.Second + arbCalibSamples = 60 // ~18s of head polls before freezing the offset + arbCalibWarmup = 10 // polls to discard while the feed drains its connect backlog + arbMaxLagSeconds = 30 // sanity bound: drop reconnect-gap outliers + arrivalsMax = 4000 +) + +type arbFeedState struct { + mu sync.Mutex + arrivals map[int64]time.Time // seq → first feed arrival + order []int64 // insertion order, for bounded pruning + latestSeq int64 +} + +func newArbFeedState() *arbFeedState { + return &arbFeedState{arrivals: make(map[int64]time.Time, arrivalsMax)} +} + +func (s *arbFeedState) record(seq int64, at time.Time) { + s.mu.Lock() + defer s.mu.Unlock() + if _, dup := s.arrivals[seq]; dup { + return // only first arrival counts + } + s.arrivals[seq] = at + s.order = append(s.order, seq) + if seq > s.latestSeq { + s.latestSeq = seq + } + for len(s.order) > arrivalsMax { + delete(s.arrivals, s.order[0]) + s.order = s.order[1:] + } +} + +func (s *arbFeedState) lookup(seq int64) (time.Time, bool) { + s.mu.Lock() + defer s.mu.Unlock() + t, ok := s.arrivals[seq] + return t, ok +} + +func (s *arbFeedState) latest() int64 { + s.mu.Lock() + defer s.mu.Unlock() + return s.latestSeq +} + +// arbFeedMsg is the minimal shape of a broadcast frame: +// +// {"version":1,"messages":[{"sequenceNumber":296300000,"message":{...}}]} +type arbFeedMsg struct { + Messages []struct { + SequenceNumber int64 `json:"sequenceNumber"` + } `json:"messages"` +} + +// runArbFeed holds the sequencer feed WS open (reconnect loop with +// backoff + silence watchdog, same pattern as l2-block-time). +func runArbFeed(ctx context.Context, state *arbFeedState) { + url := arbFeedURL() + fmt.Printf("[arb-feed] connecting %s\n", url) + backoff := arbMinBackoff + for ctx.Err() == nil { + err := readArbFeed(ctx, url, state) + arbFeedHealth.Set(0) + streamReconnects.WithLabelValues("arb-feed").Inc() + if err != nil { + fmt.Printf("[arb-feed] error: %v (reconnecting in %v)\n", err, backoff) + } + select { + case <-ctx.Done(): + return + case <-time.After(backoff): + } + if backoff < arbMaxBackoff { + backoff *= 2 + if backoff > arbMaxBackoff { + backoff = arbMaxBackoff + } + } + } +} + +func readArbFeed(ctx context.Context, url string, state *arbFeedState) error { + dialer := *websocket.DefaultDialer + dialer.HandshakeTimeout = arbDialTimeout + hdr := http.Header{"User-Agent": []string{harnessUserAgent}} + conn, _, err := dialer.Dial(url, hdr) + if err != nil { + return fmt.Errorf("dial: %w", err) + } + defer conn.Close() + arbFeedHealth.Set(1) + fmt.Println("[arb-feed] connected") + + // Silence watchdog: Arbitrum produces ~4 blocks/s, so 30s without a + // frame is unambiguously a wedged subscription. Atomic keeps the + // watchdog lock-free (same pattern as l2-block-time's head watchdog). + done := make(chan struct{}) + defer close(done) + var lastMsgUnixNano atomic.Int64 + lastMsgUnixNano.Store(time.Now().UnixNano()) + go func() { + t := time.NewTicker(10 * time.Second) + defer t.Stop() + for { + select { + case <-done: + return + case <-ctx.Done(): + _ = conn.Close() + return + case now := <-t.C: + silent := now.Sub(time.Unix(0, lastMsgUnixNano.Load())) + if silent > arbWatchdog { + fmt.Printf("[arb-feed] watchdog: silent %s, forcing reconnect\n", silent.Round(time.Second)) + _ = conn.Close() + return + } + } + } + }() + + for { + _ = conn.SetReadDeadline(time.Now().Add(arbReadDeadline)) + _, raw, err := conn.ReadMessage() + if err != nil { + return fmt.Errorf("read: %w", err) + } + now := time.Now() + lastMsgUnixNano.Store(now.UnixNano()) + + var msg arbFeedMsg + if err := json.Unmarshal(raw, &msg); err != nil { + continue // keepalive / non-broadcast frame + } + for _, m := range msg.Messages { + if m.SequenceNumber > 0 { + state.record(m.SequenceNumber, now) + arbFeedMessages.Inc() + } + } + } +} + +// runArbHeadPoller polls the Arbitrum RPC head, derives the +// seq→blockNumber offset during calibration, then emits the feed→RPC +// soft-confirmation lag for every head advance. +func runArbHeadPoller(ctx context.Context, state *arbFeedState) { + url := arbRPCURL() + fmt.Printf("[arb-rpc] head poll: %s every %s\n", url, l2HeadPollInterval) + + var ( + offset int64 + warmup int + candidates = map[int64]int{} + calibCount int + calibrated bool + prevHead int64 + ) + t := time.NewTicker(l2HeadPollInterval) + defer t.Stop() + for { + select { + case <-ctx.Done(): + return + case <-t.C: + } + + head, err := blockNumber(url) + if err != nil { + arbRPCErrors.Inc() + continue + } + pollAt := time.Now() + latestSeq := state.latest() + if latestSeq == 0 { + continue // feed not live yet + } + + if !calibrated { + // Discard the first polls after the feed goes live: the connect + // backlog replay makes latestSeq briefly trail the chain, which + // would overshoot the offset (and inflate every lag sample). + if warmup < arbCalibWarmup { + warmup++ + continue + } + candidates[head-latestSeq]++ + calibCount++ + if calibCount >= arbCalibSamples { + // Highest candidate with real support (>= 20% of samples). + // Live-feed candidates cluster on trueOffset and + // trueOffset-1 (0-1 blocks in flight); transient overshoots + // have negligible counts and are rejected here. + minSupport := calibCount / 5 + for cand, n := range candidates { + if n >= minSupport && cand > offset { + offset = cand + } + } + calibrated = true + prevHead = head + arbDerivedOffset.Set(float64(offset)) + fmt.Printf("[arb-rpc] calibrated seq→block offset=%d over %d samples (expected ≈22207817)\n", offset, calibCount) + } + continue + } + + if head <= prevHead { + continue + } + for b := prevHead + 1; b <= head; b++ { + arrival, ok := state.lookup(b - offset) + if !ok { + continue + } + lagMs := float64(pollAt.Sub(arrival).Milliseconds()) + if lagMs >= 0 && lagMs < arbMaxLagSeconds*1000 { + arbSoftConfLag.Observe(lagMs) + arbLagSamples.Inc() + fmt.Printf("[arb] block=%d softconf lag=%.0fms\n", b, lagMs) + } + } + prevHead = head + } +} diff --git a/harnesses/evm-block-builders/cmd/script/base_flashblocks.go b/harnesses/evm-block-builders/cmd/script/base_flashblocks.go new file mode 100644 index 00000000..4ecc9156 --- /dev/null +++ b/harnesses/evm-block-builders/cmd/script/base_flashblocks.go @@ -0,0 +1,257 @@ +package main + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "sync" + "sync/atomic" + "time" + + "github.com/andybalholm/brotli" + "github.com/gorilla/websocket" +) + +// Base flashblocks (preconfirmation) stream. +// +// wss://mainnet.flashblocks.base.org/ws pushes one frame per flashblock +// (~every 200-250ms; ~10 flashblocks per 2s Base block). Frames are +// binary, brotli-compressed JSON of the shape: +// +// {"payload_id":"0x..","index":3,"base":{...},"diff":{...}, +// "metadata":{"block_number":31234567,"receipts":{...},...}} +// +// Two measurements: +// 1. inter-frame cadence (ebb_base_flashblock_interval_milliseconds): +// is the advertised 200ms preconfirmation rhythm actually held? +// 2. soft-confirmation lag (ebb_base_softconf_lag_milliseconds): time +// from the FIRST flashblock of block N arriving on the stream to +// block N being visible on the public RPC head. This is the head +// start a flashblock consumer gets over an RPC poller. +// +// If a frame fails brotli+JSON parsing we still record cadence (the +// frame arrived) and count a parse failure; cadence-only degradation is +// acceptable and disclosed. + +const ( + baseDialTimeout = 15 * time.Second + baseReadDeadline = 60 * time.Second + baseWatchdog = 20 * time.Second + baseMinBackoff = 2 * time.Second + baseMaxBackoff = 60 * time.Second + baseMaxLagMs = 30 * 1000 + // A reconnect gap must not pollute the cadence histogram. + baseMaxIntervalMs = 10 * 1000 + baseArrivalsMax = 200 +) + +type baseFeedState struct { + mu sync.Mutex + arrivals map[int64]time.Time // block number → first flashblock arrival + order []int64 +} + +func newBaseFeedState() *baseFeedState { + return &baseFeedState{arrivals: make(map[int64]time.Time, baseArrivalsMax)} +} + +func (s *baseFeedState) recordFirst(block int64, at time.Time) bool { + s.mu.Lock() + defer s.mu.Unlock() + if _, dup := s.arrivals[block]; dup { + return false + } + s.arrivals[block] = at + s.order = append(s.order, block) + for len(s.order) > baseArrivalsMax { + delete(s.arrivals, s.order[0]) + s.order = s.order[1:] + } + return true +} + +func (s *baseFeedState) lookup(block int64) (time.Time, bool) { + s.mu.Lock() + defer s.mu.Unlock() + t, ok := s.arrivals[block] + return t, ok +} + +type flashblockFrame struct { + Index *int64 `json:"index"` + Metadata struct { + BlockNumber int64 `json:"block_number"` + } `json:"metadata"` +} + +// runBaseFlashblocks holds the flashblocks WS open with reconnect +// backoff + silence watchdog and feeds cadence + arrival state. +func runBaseFlashblocks(ctx context.Context, state *baseFeedState) { + url := baseFlashblocksURL() + fmt.Printf("[base-fb] connecting %s\n", url) + backoff := baseMinBackoff + for ctx.Err() == nil { + err := readBaseFlashblocks(ctx, url, state) + baseFeedHealth.Set(0) + streamReconnects.WithLabelValues("base-flashblocks").Inc() + if err != nil { + fmt.Printf("[base-fb] error: %v (reconnecting in %v)\n", err, backoff) + } + select { + case <-ctx.Done(): + return + case <-time.After(backoff): + } + if backoff < baseMaxBackoff { + backoff *= 2 + if backoff > baseMaxBackoff { + backoff = baseMaxBackoff + } + } + } +} + +func readBaseFlashblocks(ctx context.Context, url string, state *baseFeedState) error { + dialer := *websocket.DefaultDialer + dialer.HandshakeTimeout = baseDialTimeout + hdr := http.Header{"User-Agent": []string{harnessUserAgent}} + conn, _, err := dialer.Dial(url, hdr) + if err != nil { + return fmt.Errorf("dial: %w", err) + } + defer conn.Close() + baseFeedHealth.Set(1) + fmt.Println("[base-fb] connected") + + done := make(chan struct{}) + defer close(done) + var lastFrameUnixNano atomic.Int64 + lastFrameUnixNano.Store(time.Now().UnixNano()) + go func() { + t := time.NewTicker(5 * time.Second) + defer t.Stop() + for { + select { + case <-done: + return + case <-ctx.Done(): + _ = conn.Close() + return + case now := <-t.C: + silent := now.Sub(time.Unix(0, lastFrameUnixNano.Load())) + if silent > baseWatchdog { + fmt.Printf("[base-fb] watchdog: silent %s, forcing reconnect\n", silent.Round(time.Second)) + _ = conn.Close() + return + } + } + } + }() + + var lastFrame time.Time + logEvery := 0 + for { + _ = conn.SetReadDeadline(time.Now().Add(baseReadDeadline)) + _, raw, err := conn.ReadMessage() + if err != nil { + return fmt.Errorf("read: %w", err) + } + now := time.Now() + lastFrameUnixNano.Store(now.UnixNano()) + + // Cadence: frame arrival rhythm, parse success or not. + if !lastFrame.IsZero() { + ivMs := float64(now.Sub(lastFrame).Milliseconds()) + if ivMs >= 0 && ivMs < baseMaxIntervalMs { + baseFlashblockInterval.Observe(ivMs) + } + } + lastFrame = now + baseFrames.Inc() + + frame, err := decodeFlashblockFrame(raw) + if err != nil { + baseParseFailures.Inc() + continue + } + if frame.Metadata.BlockNumber > 0 { + if state.recordFirst(frame.Metadata.BlockNumber, now) { + logEvery++ + if logEvery%10 == 1 { // ~1 log line per 20s, not per 200ms frame + idx := int64(-1) + if frame.Index != nil { + idx = *frame.Index + } + fmt.Printf("[base-fb] first flashblock block=%d index=%d\n", frame.Metadata.BlockNumber, idx) + } + } + } + } +} + +// decodeFlashblockFrame handles both brotli-compressed and plain-JSON +// frames (the stream compressed at inception; tolerate both so a +// server-side change doesn't blind the bench). +func decodeFlashblockFrame(raw []byte) (*flashblockFrame, error) { + var frame flashblockFrame + if len(raw) > 0 && (raw[0] == '{' || raw[0] == '[') { + if err := json.Unmarshal(raw, &frame); err == nil { + return &frame, nil + } + } + dec, err := io.ReadAll(io.LimitReader(brotli.NewReader(bytes.NewReader(raw)), 16<<20)) + if err != nil { + return nil, fmt.Errorf("brotli: %w", err) + } + if err := json.Unmarshal(dec, &frame); err != nil { + return nil, fmt.Errorf("json: %w", err) + } + return &frame, nil +} + +// runBaseHeadPoller polls the Base RPC head and emits the +// flashblock→canonical-RPC soft-confirmation lag. +func runBaseHeadPoller(ctx context.Context, state *baseFeedState) { + url := baseRPCURL() + fmt.Printf("[base-rpc] head poll: %s every %s\n", url, l2HeadPollInterval) + + var prevHead int64 + t := time.NewTicker(l2HeadPollInterval) + defer t.Stop() + for { + select { + case <-ctx.Done(): + return + case <-t.C: + } + + head, err := blockNumber(url) + if err != nil { + baseRPCErrors.Inc() + continue + } + pollAt := time.Now() + if prevHead == 0 { + prevHead = head + continue + } + for b := prevHead + 1; b <= head; b++ { + arrival, ok := state.lookup(b) + if !ok { + continue + } + lagMs := float64(pollAt.Sub(arrival).Milliseconds()) + if lagMs >= 0 && lagMs < baseMaxLagMs { + baseSoftConfLag.Observe(lagMs) + baseLagSamples.Inc() + fmt.Printf("[base] block=%d softconf lag=%.0fms\n", b, lagMs) + } + } + if head > prevHead { + prevHead = head + } + } +} diff --git a/harnesses/evm-block-builders/cmd/script/builders.go b/harnesses/evm-block-builders/cmd/script/builders.go new file mode 100644 index 00000000..0156ee66 --- /dev/null +++ b/harnesses/evm-block-builders/cmd/script/builders.go @@ -0,0 +1,192 @@ +package main + +import ( + "context" + "encoding/hex" + "fmt" + "strings" + "time" +) + +// Ethereum builder attribution via block extraData. +// +// Same philosophy as bench 016 (solana-tx-landing): a curated table of +// known-entity tags, plus explicit logging of every string we could NOT +// attribute so the table grows from operator logs instead of silently +// under-counting. Builders self-label their blocks in extraData +// ("Titan (titanbuilder.xyz)", "beaverbuild.org", ...); vanilla +// (locally-built) blocks carry the execution client's default tag +// (geth/reth/besu/nethermind version strings) or are empty. +// +// extraData is self-reported: a builder could change or strip its tag +// at any time. The methodology in the spec discloses this; the relay +// bidtrace cross-check (relays.go) provides an independent signal. + +// builderRule maps a lowercase substring of the decoded extraData to a +// canonical builder slug. Order matters: first match wins, so put the +// most specific substrings first. +type builderRule struct { + substr string + slug string +} + +// builderTable is the curated attribution table. Slugs are the Prom +// label values the spec YAML queries. Verified against live mainnet +// blocks at bench inception (50-block sample: Titan 19, Quasar 16, +// Eureka 6, BuilderNet 2, beaverbuild 1, Builder+ 1, vanilla 1). +var builderTable = []builderRule{ + // Specific multi-word tags first. + {"bob-the-builder", "bobthebuilder"}, + {"bobthebuilder", "bobthebuilder"}, + {"builder0x69", "builder0x69"}, + {"penguinbuild", "penguinbuild"}, + {"beaverbuild", "beaverbuild"}, + {"buildernet", "buildernet"}, + {"builder+", "btcs"}, // BTCS "Builder+" + {"btcs", "btcs"}, + {"titan", "titan"}, + {"quasar", "quasar"}, + {"eureka", "eureka"}, + {"rsync", "rsync"}, + // Flashbots historically tags "Illuminate Dmocratize Dstribute". + {"flashbots", "flashbots"}, + {"illuminate", "flashbots"}, + {"dmocratize", "flashbots"}, + {"bloxroute", "bloxroute"}, + {"gambit", "gambit"}, + {"antbuilder", "antbuilder"}, + {"manifold", "manifold"}, + {"blockbeelder", "blockbeelder"}, + {"jetbldr", "jetbuilder"}, + {"payload", "payload"}, + {"boba-builder", "boba"}, +} + +// vanillaTags identify execution-client default extraData: the proposer +// built the block locally instead of outsourcing to a builder. These +// count as "vanilla", NOT as unattributed - a locally-built block is a +// positive decentralization signal, not a table gap. +var vanillaTags = []string{ + "geth", "go1.", "nethermind", "besu", "reth", "erigon", "linux", + "darwin", "windows", "ubuntu", +} + +// slugOther collects blocks whose extraData carries a tag we don't +// recognize. Every occurrence is logged with the raw string so the +// table can be grown (016 pattern). +const ( + slugOther = "other" + slugVanilla = "vanilla" +) + +// decodeExtraData turns the hex extraData field into a printable string +// for substring matching. Non-printable bytes (client tags like geth +// pack RLP fragments around the ASCII) are dropped. +func decodeExtraData(hexStr string) string { + raw, err := hex.DecodeString(strings.TrimPrefix(hexStr, "0x")) + if err != nil { + return "" + } + var b strings.Builder + for _, c := range raw { + if c >= 0x20 && c <= 0x7e { + b.WriteByte(c) + } + } + return b.String() +} + +// attributeBuilder maps decoded extraData to a builder slug. +// Returns (slug, attributed). attributed=false means the tag was +// non-empty and unrecognized (goes to "other" + unattributed counter). +func attributeBuilder(decoded string) (string, bool) { + lower := strings.ToLower(decoded) + if strings.TrimSpace(lower) == "" { + return slugVanilla, true + } + for _, r := range builderTable { + if strings.Contains(lower, r.substr) { + return r.slug, true + } + } + for _, t := range vanillaTags { + if strings.Contains(lower, t) { + return slugVanilla, true + } + } + return slugOther, false +} + +type ethBlock struct { + Number string `json:"number"` + ExtraData string `json:"extraData"` +} + +// runBuilderPoll polls the ETH head every slot (12s), attributes each +// new block's extraData and backfills small gaps by number so the +// market-share counters see every block, not just the ones that happen +// to be head at poll time. +func runBuilderPoll(ctx context.Context) { + url := ethRPCURL() + fmt.Printf("[eth] builder attribution poll: %s every %s\n", url, ethPollInterval) + + var lastSeen int64 + t := time.NewTicker(ethPollInterval) + defer t.Stop() + for { + var head ethBlock + err := jsonRPCCall(url, "eth_getBlockByNumber", []any{"latest", false}, &head) + if err != nil { + ethPollHealth.Set(0) + ethPollErrors.Inc() + fmt.Printf("[eth] poll error: %v\n", err) + } else { + ethPollHealth.Set(1) + headNum := parseHexInt64(head.Number) + if headNum > lastSeen { + // Backfill missed blocks (cap 10: a longer gap means we were + // down; counting a burst of stale blocks then would skew the + // 5m rate series without changing 24h share meaningfully). + start := headNum + if lastSeen > 0 { + start = lastSeen + 1 + if headNum-lastSeen > 10 { + start = headNum - 10 + 1 + } + } + for n := start; n < headNum; n++ { + var blk ethBlock + if err := jsonRPCCall(url, "eth_getBlockByNumber", + []any{fmt.Sprintf("0x%x", n), false}, &blk); err != nil { + fmt.Printf("[eth] backfill %d error: %v\n", n, err) + continue + } + recordBlock(n, blk.ExtraData) + } + recordBlock(headNum, head.ExtraData) + lastSeen = headNum + ethLastBlock.Set(float64(headNum)) + } + } + + select { + case <-ctx.Done(): + return + case <-t.C: + } + } +} + +func recordBlock(num int64, extraHex string) { + decoded := decodeExtraData(extraHex) + slug, attributed := attributeBuilder(decoded) + blocksTotal.WithLabelValues(slug).Inc() + if !attributed { + unattributedTotal.Inc() + // The whole point of the log line: grow the table from operator + // logs, exactly like 016 logs unattributable tip patterns. + fmt.Printf("[eth] UNATTRIBUTED block=%d extraData=%q raw=%s\n", num, decoded, extraHex) + } else { + fmt.Printf("[eth] block=%d builder=%s extraData=%q\n", num, slug, decoded) + } +} diff --git a/harnesses/evm-block-builders/cmd/script/config.go b/harnesses/evm-block-builders/cmd/script/config.go new file mode 100644 index 00000000..5181581a --- /dev/null +++ b/harnesses/evm-block-builders/cmd/script/config.go @@ -0,0 +1,152 @@ +package main + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "net/http" + "os" + "strconv" + "strings" + "time" +) + +// Endpoints. All public, keyless, verified live at bench inception +// (2026-07-14). Every URL is env-overridable without a rebuild so a +// flaky public endpoint can be swapped from the deploy env. +// +// publicnode 403s default Go/Python User-Agents (bench 067 lesson), so +// EVERY outbound HTTP request and WS dial in this harness sends +// harnessUserAgent - see newRequest / wsDialer. + +const ( + defaultEthRPC = "https://ethereum-rpc.publicnode.com" + defaultArbRPC = "https://arbitrum-one-rpc.publicnode.com" + defaultBaseRPC = "https://base-rpc.publicnode.com" + defaultArbFeedURL = "wss://arb1.arbitrum.io/feed" + defaultBaseFlashblocks = "wss://mainnet.flashblocks.base.org/ws" + + // Browser-prefixed but honest: identifies the harness while passing + // UA-based bot filters (publicnode 403s bare Go-http-client UAs). + harnessUserAgent = "Mozilla/5.0 (compatible; OpenChainBench-harness/1.0; +https://openchainbench.com)" + + // ETH mainnet slot time. One poll per slot samples ~every block; the + // gap-backfill in builders.go covers the occasional missed one. + ethPollInterval = 12 * time.Second + + // Relay bidtrace APIs are rate-limited and slow-moving (1 payload per + // 12s slot at most); 5 min polls with limit=50 never miss a slot. + relayPollInterval = 5 * time.Minute + + // L2 head pollers. 300ms against publicnode is comfortably inside the + // public rate limits (verified live) and fine-grained enough to + // resolve a ~100ms soft-conf lag. + l2HeadPollInterval = 300 * time.Millisecond +) + +func envDefault(key, def string) string { + if v := strings.TrimSpace(os.Getenv(key)); v != "" { + return v + } + return def +} + +func ethRPCURL() string { return envDefault("ETH_RPC_URL", defaultEthRPC) } +func arbRPCURL() string { return envDefault("ARB_RPC_URL", defaultArbRPC) } +func baseRPCURL() string { return envDefault("BASE_RPC_URL", defaultBaseRPC) } +func arbFeedURL() string { + return envDefault("ARB_FEED_URL", defaultArbFeedURL) +} +func baseFlashblocksURL() string { + return envDefault("BASE_FLASHBLOCKS_URL", defaultBaseFlashblocks) +} + +// listenAddr defaults to :2112 - OCB convention, scraped by the shared +// Prometheus. We deliberately ignore $PORT so a platform-injected public +// port doesn't move the listener away from the address Prom expects. +// METRICS_ADDR exists for local dev where another harness holds 2112. +func listenAddr() string { return envDefault("METRICS_ADDR", ":2112") } + +var httpClient = &http.Client{Timeout: 15 * time.Second} + +// newRequest builds an *http.Request with the harness User-Agent set. +func newRequest(method, url string, body []byte) (*http.Request, error) { + var rdr io.Reader + if body != nil { + rdr = bytes.NewReader(body) + } + req, err := http.NewRequest(method, url, rdr) + if err != nil { + return nil, err + } + req.Header.Set("User-Agent", harnessUserAgent) + req.Header.Set("Accept", "application/json") + if body != nil { + req.Header.Set("Content-Type", "application/json") + } + return req, nil +} + +// jsonRPCCall does a single JSON-RPC 2.0 request and unmarshals the +// `result` field into out. +func jsonRPCCall(url, method string, params []any, out any) error { + payload, err := json.Marshal(map[string]any{ + "jsonrpc": "2.0", "id": 1, "method": method, "params": params, + }) + if err != nil { + return err + } + req, err := newRequest(http.MethodPost, url, payload) + if err != nil { + return err + } + resp, err := httpClient.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(io.LimitReader(resp.Body, 200)) + return fmt.Errorf("%s: HTTP %d: %s", method, resp.StatusCode, string(body)) + } + var envl struct { + Result json.RawMessage `json:"result"` + Error *struct { + Code int `json:"code"` + Message string `json:"message"` + } `json:"error"` + } + if err := json.NewDecoder(resp.Body).Decode(&envl); err != nil { + return fmt.Errorf("%s: decode: %w", method, err) + } + if envl.Error != nil { + return fmt.Errorf("%s: rpc error %d: %s", method, envl.Error.Code, envl.Error.Message) + } + if envl.Result == nil { + return fmt.Errorf("%s: empty result", method) + } + return json.Unmarshal(envl.Result, out) +} + +// blockNumber fetches the current head number via eth_blockNumber. +func blockNumber(url string) (int64, error) { + var hexNum string + if err := jsonRPCCall(url, "eth_blockNumber", []any{}, &hexNum); err != nil { + return 0, err + } + n := parseHexInt64(hexNum) + if n == 0 { + return 0, fmt.Errorf("eth_blockNumber: unparseable %q", hexNum) + } + return n, nil +} + +func parseHexInt64(s string) int64 { + s = strings.TrimPrefix(s, "0x") + if s == "" { + return 0 + } + n, _ := strconv.ParseInt(s, 16, 64) + return n +} diff --git a/harnesses/evm-block-builders/cmd/script/loghub.go b/harnesses/evm-block-builders/cmd/script/loghub.go new file mode 100644 index 00000000..9dd74488 --- /dev/null +++ b/harnesses/evm-block-builders/cmd/script/loghub.go @@ -0,0 +1,114 @@ +package main + +import ( + "bufio" + "fmt" + "io" + "net/http" + "os" + "strconv" + "sync" + "time" +) + +// Auto-generated by the loghub inline pattern. Captures stdout/stderr into a +// bounded ring buffer and exposes GET /logs?tail=N protected by X-Logs-Token +// matching the LOGS_TOKEN env var. +// +// Keep in sync across miniapps (was previously the shared/loghub package; we +// inline because Railway's per-harness Docker build context can't reach a +// sibling shared module via go.mod replace). + +const logRingMax = 5000 + +type logRing struct { + mu sync.Mutex + lines []string + max int +} + +var globalLogRing = &logRing{max: logRingMax} + +func (b *logRing) push(line string) { + entry := time.Now().UTC().Format("2006-01-02T15:04:05.000Z") + " " + line + b.mu.Lock() + if len(b.lines) >= b.max { + b.lines = append(b.lines[1:], entry) + } else { + b.lines = append(b.lines, entry) + } + b.mu.Unlock() +} + +func (b *logRing) snapshot(tail int) []string { + b.mu.Lock() + defer b.mu.Unlock() + if tail <= 0 || tail >= len(b.lines) { + out := make([]string, len(b.lines)) + copy(out, b.lines) + return out + } + start := len(b.lines) - tail + out := make([]string, tail) + copy(out, b.lines[start:]) + return out +} + +var logSetupOnce sync.Once + +// installLogCapture replaces os.Stdout (and os.Stderr) with the write-end of a +// pipe, then spawns a goroutine that fan-outs every line to the original +// stdout AND the in-memory ring buffer. Call exactly once, very early in +// main(). +func installLogCapture() { logSetupOnce.Do(doInstallLogCapture) } + +func doInstallLogCapture() { + originalStdout := os.Stdout + originalStderr := os.Stderr + r, w, err := os.Pipe() + if err != nil { + fmt.Fprintf(originalStdout, "[loghub] pipe failed: %v (/logs will be empty)\n", err) + return + } + os.Stdout = w + os.Stderr = w + + go func() { + scanner := bufio.NewScanner(r) + buf := make([]byte, 0, 1024*1024) + scanner.Buffer(buf, 1024*1024) + for scanner.Scan() { + line := scanner.Text() + fmt.Fprintln(originalStdout, line) + globalLogRing.push(line) + } + _, _ = io.Copy(originalStdout, r) + _ = originalStderr + }() +} + +// logsHandler returns an http.Handler for GET /logs?tail=N. Requires header +// X-Logs-Token to match the LOGS_TOKEN env var. Returns 403 if env unset. +func logsHandler() http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + expected := os.Getenv("LOGS_TOKEN") + if expected == "" { + http.Error(w, "logs disabled: LOGS_TOKEN unset", http.StatusForbidden) + return + } + if r.Header.Get("X-Logs-Token") != expected { + http.Error(w, "forbidden", http.StatusForbidden) + return + } + tail := 500 + if t := r.URL.Query().Get("tail"); t != "" { + if n, err := strconv.Atoi(t); err == nil && n > 0 { + tail = n + } + } + w.Header().Set("Content-Type", "text/plain; charset=utf-8") + for _, l := range globalLogRing.snapshot(tail) { + fmt.Fprintln(w, l) + } + }) +} diff --git a/harnesses/evm-block-builders/cmd/script/main.go b/harnesses/evm-block-builders/cmd/script/main.go new file mode 100644 index 00000000..244d38ff --- /dev/null +++ b/harnesses/evm-block-builders/cmd/script/main.go @@ -0,0 +1,76 @@ +package main + +import ( + "context" + "fmt" + "os" + "os/signal" + "syscall" + "time" +) + +// Bench 085 "evm-block-builders": who builds Ethereum, plus L2 +// soft-confirmation lag. +// +// Four independent measurement loops share one process: +// 1. builders.go - ETH head poll, extraData → builder attribution +// 2. relays.go - MEV-Boost relay bidtrace pollers (cross-check) +// 3. arb_feed.go - Arbitrum sequencer feed → RPC visibility lag +// 4. base_flashblocks.go - Base flashblock cadence + → RPC lag +// +// Metrics server listens on :2112 - OCB convention; $PORT is ignored +// so Prometheus can scrape the expected address. + +func main() { + installLogCapture() // capture stdout into /logs ring buffer + fmt.Println("=== EVM Block Builders Harness (bench 085) ===") + fmt.Println("OpenChainBench: builder market share via extraData + L2 soft-confirmation lag.") + fmt.Println() + fmt.Printf("ETH RPC: %s (poll %s)\n", ethRPCURL(), ethPollInterval) + fmt.Printf("Relays: %d bidtrace APIs (poll %s)\n", len(relays()), relayPollInterval) + fmt.Printf("Arb feed: %s\n", arbFeedURL()) + fmt.Printf("Arb RPC: %s (poll %s)\n", arbRPCURL(), l2HeadPollInterval) + fmt.Printf("Base flashblocks: %s\n", baseFlashblocksURL()) + fmt.Printf("Base RPC: %s (poll %s)\n", baseRPCURL(), l2HeadPollInterval) + fmt.Println() + fmt.Println("Metrics server: :2112/metrics") + fmt.Println() + + go func() { + if err := StartMetricsServer(listenAddr()); err != nil { + fmt.Printf("[fatal] metrics server: %v\n", err) + // Give the loghub forwarding goroutine a beat to flush the + // fatal line to the real stdout before we exit. + time.Sleep(200 * time.Millisecond) + os.Exit(1) + } + }() + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + // Initialise health gauges to 0; each loop flips its own to 1. + ethPollHealth.Set(0) + arbFeedHealth.Set(0) + baseFeedHealth.Set(0) + for _, r := range relays() { + relayHealth.WithLabelValues(r.Slug).Set(0) + } + + go runBuilderPoll(ctx) + runRelayPolls(ctx) + + arbState := newArbFeedState() + go runArbFeed(ctx, arbState) + go runArbHeadPoller(ctx, arbState) + + baseState := newBaseFeedState() + go runBaseFlashblocks(ctx, baseState) + go runBaseHeadPoller(ctx, baseState) + + sig := make(chan os.Signal, 1) + signal.Notify(sig, os.Interrupt, syscall.SIGTERM) + s := <-sig + fmt.Printf("\n[shutdown] received %v\n", s) + cancel() +} diff --git a/harnesses/evm-block-builders/cmd/script/metrics.go b/harnesses/evm-block-builders/cmd/script/metrics.go new file mode 100644 index 00000000..93cfb14e --- /dev/null +++ b/harnesses/evm-block-builders/cmd/script/metrics.go @@ -0,0 +1,223 @@ +package main + +import ( + "net/http" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" + "github.com/prometheus/client_golang/prometheus/promhttp" +) + +// Prom metrics for bench 085 (evm-block-builders). Prefix `ebb_`. +// The spec YAML (benchmarks/evm-block-builders.yml) queries these +// directly via PromQL. +var ( + /* ---- Ethereum builder attribution (headline) ---- */ + + // Per-builder block counter, attributed via extraData substring + // table. Includes the synthetic slugs "vanilla" (client-default + // extraData → locally built) and "other" (unrecognized tag). + blocksTotal = promauto.NewCounterVec( + prometheus.CounterOpts{ + Name: "ebb_blocks_total", + Help: "Ethereum mainnet blocks attributed to each builder via the block's extraData self-label. builder=vanilla means client-default extraData (locally built); builder=other means an unrecognized tag (also logged raw for table growth).", + }, + []string{"builder"}, + ) + + // Blocks whose extraData carried an unrecognized tag. The raw string + // is logged so the attribution table can grow (016 pattern). + unattributedTotal = promauto.NewCounter( + prometheus.CounterOpts{ + Name: "ebb_unattributed_total", + Help: "Blocks whose extraData tag was non-empty but not in the curated builder table. Every occurrence logs the raw extraData for later table growth.", + }, + ) + + ethPollHealth = promauto.NewGauge( + prometheus.GaugeOpts{ + Name: "ebb_eth_poll_health", + Help: "1 when the Ethereum head poll is succeeding, 0 after the last poll errored.", + }, + ) + + ethPollErrors = promauto.NewCounter( + prometheus.CounterOpts{ + Name: "ebb_eth_poll_errors_total", + Help: "Ethereum RPC poll errors since process start.", + }, + ) + + ethLastBlock = promauto.NewGauge( + prometheus.GaugeOpts{ + Name: "ebb_eth_last_block", + Help: "Most recent Ethereum block number attributed by the harness. Freshness probe target.", + }, + ) + + /* ---- Relay bidtrace cross-check ---- */ + + relayPayloadsTotal = promauto.NewCounterVec( + prometheus.CounterOpts{ + Name: "ebb_relay_payloads_total", + Help: "Delivered payloads (proposer_payload_delivered bidtraces) counted per MEV-Boost relay, deduped per relay via a slot high-water-mark. The same slot on multiple relays is normal (multi-homed bids), so this measures relay share, not a partition of blocks.", + }, + []string{"relay"}, + ) + + relayPollErrors = promauto.NewCounterVec( + prometheus.CounterOpts{ + Name: "ebb_relay_poll_errors_total", + Help: "Bidtrace poll errors per relay since process start.", + }, + []string{"relay"}, + ) + + relayHealth = promauto.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "ebb_relay_health", + Help: "1 when the last bidtrace poll for this relay succeeded, 0 otherwise.", + }, + []string{"relay"}, + ) + + relayLastSlot = promauto.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "ebb_relay_last_slot", + Help: "Highest slot seen in bidtraces per relay.", + }, + []string{"relay"}, + ) + + /* ---- Arbitrum sequencer feed soft-confirmation lag ---- */ + + arbSoftConfLag = promauto.NewHistogram( + prometheus.HistogramOpts{ + Name: "ebb_arb_softconf_lag_milliseconds", + Help: "Lag from a block's sequencer-feed arrival to the same block being visible on the public Arbitrum RPC head. Quantized by the ~300ms RPC poll interval.", + Buckets: []float64{ + 25, 50, 75, 100, 150, 200, 300, 400, 500, 750, + 1000, 1500, 2000, 3000, 5000, 10000, + }, + }, + ) + + arbLagSamples = promauto.NewCounter( + prometheus.CounterOpts{ + Name: "ebb_arb_lag_samples_total", + Help: "Arbitrum soft-confirmation lag samples recorded.", + }, + ) + + arbFeedHealth = promauto.NewGauge( + prometheus.GaugeOpts{ + Name: "ebb_arb_feed_health", + Help: "1 while the Arbitrum sequencer feed WebSocket is connected, 0 otherwise.", + }, + ) + + arbFeedMessages = promauto.NewCounter( + prometheus.CounterOpts{ + Name: "ebb_arb_feed_messages_total", + Help: "Sequencer feed messages received.", + }, + ) + + arbDerivedOffset = promauto.NewGauge( + prometheus.GaugeOpts{ + Name: "ebb_arb_derived_offset", + Help: "Runtime-derived sequenceNumber→blockNumber offset (expected ≈22207817). Drift signals a feed format change.", + }, + ) + + arbRPCErrors = promauto.NewCounter( + prometheus.CounterOpts{ + Name: "ebb_arb_rpc_errors_total", + Help: "Arbitrum RPC head-poll errors since process start.", + }, + ) + + /* ---- Base flashblocks ---- */ + + baseFlashblockInterval = promauto.NewHistogram( + prometheus.HistogramOpts{ + Name: "ebb_base_flashblock_interval_milliseconds", + Help: "Wall-clock interval between consecutive flashblock frames on the Base flashblocks WebSocket (advertised rhythm: ~200ms).", + Buckets: []float64{ + 50, 100, 150, 200, 250, 300, 400, 500, 750, + 1000, 2000, 5000, + }, + }, + ) + + baseSoftConfLag = promauto.NewHistogram( + prometheus.HistogramOpts{ + Name: "ebb_base_softconf_lag_milliseconds", + Help: "Lag from the first flashblock of a Base block arriving on the stream to that block being visible on the public Base RPC head. Quantized by the ~300ms RPC poll interval.", + Buckets: []float64{ + 100, 250, 500, 750, 1000, 1250, 1500, 1750, 2000, + 2500, 3000, 5000, 10000, + }, + }, + ) + + baseLagSamples = promauto.NewCounter( + prometheus.CounterOpts{ + Name: "ebb_base_lag_samples_total", + Help: "Base soft-confirmation lag samples recorded.", + }, + ) + + baseFeedHealth = promauto.NewGauge( + prometheus.GaugeOpts{ + Name: "ebb_base_feed_health", + Help: "1 while the Base flashblocks WebSocket is connected, 0 otherwise.", + }, + ) + + baseFrames = promauto.NewCounter( + prometheus.CounterOpts{ + Name: "ebb_base_frames_total", + Help: "Flashblock frames received.", + }, + ) + + baseParseFailures = promauto.NewCounter( + prometheus.CounterOpts{ + Name: "ebb_base_parse_failures_total", + Help: "Flashblock frames that failed brotli+JSON decoding (cadence still recorded).", + }, + ) + + baseRPCErrors = promauto.NewCounter( + prometheus.CounterOpts{ + Name: "ebb_base_rpc_errors_total", + Help: "Base RPC head-poll errors since process start.", + }, + ) + + /* ---- Shared ---- */ + + streamReconnects = promauto.NewCounterVec( + prometheus.CounterOpts{ + Name: "ebb_stream_reconnects_total", + Help: "WebSocket reconnects per stream (arb-feed, base-flashblocks) since process start.", + }, + []string{"stream"}, + ) +) + +// StartMetricsServer binds /metrics + /health + /logs on addr. Blocking +// call, run in its own goroutine. +func StartMetricsServer(addr string) error { + mux := http.NewServeMux() + mux.Handle("/metrics", promhttp.Handler()) + mux.Handle("/logs", logsHandler()) + mux.HandleFunc("/health", func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte("ok")) + }) + mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte("evm-block-builders harness · OpenChainBench")) + }) + return http.ListenAndServe(addr, mux) +} diff --git a/harnesses/evm-block-builders/cmd/script/relays.go b/harnesses/evm-block-builders/cmd/script/relays.go new file mode 100644 index 00000000..eaba8212 --- /dev/null +++ b/harnesses/evm-block-builders/cmd/script/relays.go @@ -0,0 +1,134 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "sort" + "strconv" + "time" +) + +// MEV-Boost relay bidtrace pollers. +// +// Every major relay exposes the standard Relay Data API keyless: +// GET /relay/v1/data/bidtraces/proposer_payload_delivered?limit=N +// One entry per delivered payload (= one relayed block). We poll each +// relay every 5 min and count NEW slots per relay via a high-water-mark +// so restarts / overlapping windows never double-count within a relay. +// +// The same slot appearing on multiple relays is NORMAL (builders +// multi-home their bids; the winning payload is reported by every relay +// that carried it), so relay counters are a market-share signal for +// relays, not a partition of blocks. This is the independent +// cross-check on the extraData attribution: relay-side builder pubkeys +// don't depend on the builder's self-chosen extraData tag. + +type relayEndpoint struct { + Slug string + Base string +} + +// relays verified answering keyless at bench inception (2026-07-14). +func relays() []relayEndpoint { + return []relayEndpoint{ + {"flashbots", "https://boost-relay.flashbots.net"}, + {"ultrasound", "https://relay.ultrasound.money"}, + {"agnostic", "https://agnostic-relay.net"}, + {"bloxroute-max-profit", "https://bloxroute.max-profit.blxrbdn.com"}, + {"bloxroute-regulated", "https://bloxroute.regulated.blxrbdn.com"}, + {"titan", "https://titanrelay.xyz"}, + {"aestus", "https://aestus.live"}, + } +} + +type bidTrace struct { + Slot string `json:"slot"` +} + +// runRelayPolls launches one poller goroutine per relay. +func runRelayPolls(ctx context.Context) { + for _, r := range relays() { + r := r + go pollRelay(ctx, r) + } +} + +func pollRelay(ctx context.Context, r relayEndpoint) { + url := r.Base + "/relay/v1/data/bidtraces/proposer_payload_delivered?limit=50" + fmt.Printf("[relay:%s] polling %s every %s\n", r.Slug, url, relayPollInterval) + + var maxSeenSlot int64 + t := time.NewTicker(relayPollInterval) + defer t.Stop() + for { + slots, err := fetchDeliveredSlots(url) + if err != nil { + relayPollErrors.WithLabelValues(r.Slug).Inc() + relayHealth.WithLabelValues(r.Slug).Set(0) + fmt.Printf("[relay:%s] poll error: %v\n", r.Slug, err) + } else { + relayHealth.WithLabelValues(r.Slug).Set(1) + fresh := 0 + for _, s := range slots { + if s > maxSeenSlot { + fresh++ + } + } + if len(slots) > 0 { + sort.Slice(slots, func(i, j int) bool { return slots[i] > slots[j] }) + if slots[0] > maxSeenSlot { + maxSeenSlot = slots[0] + } + } + relayPayloadsTotal.WithLabelValues(r.Slug).Add(float64(fresh)) + relayLastSlot.WithLabelValues(r.Slug).Set(float64(maxSeenSlot)) + fmt.Printf("[relay:%s] delivered=%d fresh=%d maxSlot=%d\n", r.Slug, len(slots), fresh, maxSeenSlot) + } + + select { + case <-ctx.Done(): + return + case <-t.C: + } + } +} + +// fetchDeliveredSlots GETs the bidtraces endpoint and returns the slot +// numbers, deduped within the response (defensive: the same slot can +// appear twice on a relay after a reorged proposal). +func fetchDeliveredSlots(url string) ([]int64, error) { + req, err := newRequest(http.MethodGet, url, nil) + if err != nil { + return nil, err + } + resp, err := httpClient.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(io.LimitReader(resp.Body, 200)) + return nil, fmt.Errorf("HTTP %d: %s", resp.StatusCode, string(body)) + } + var traces []bidTrace + if err := json.NewDecoder(resp.Body).Decode(&traces); err != nil { + return nil, fmt.Errorf("decode: %w", err) + } + seen := make(map[int64]struct{}, len(traces)) + out := make([]int64, 0, len(traces)) + for _, tr := range traces { + n, err := strconv.ParseInt(tr.Slot, 10, 64) + if err != nil || n == 0 { + continue + } + if _, dup := seen[n]; dup { + continue + } + seen[n] = struct{}{} + out = append(out, n) + } + return out, nil +} diff --git a/harnesses/evm-block-builders/go.mod b/harnesses/evm-block-builders/go.mod new file mode 100644 index 00000000..6b4d23fc --- /dev/null +++ b/harnesses/evm-block-builders/go.mod @@ -0,0 +1,22 @@ +module evm-block-builders + +go 1.24.0 + +require ( + github.com/andybalholm/brotli v1.1.1 + github.com/gorilla/websocket v1.5.3 + github.com/prometheus/client_golang v1.23.2 +) + +require ( + github.com/beorn7/perks v1.0.1 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/kr/text v0.2.0 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/prometheus/client_model v0.6.2 // indirect + github.com/prometheus/common v0.66.1 // indirect + github.com/prometheus/procfs v0.16.1 // indirect + go.yaml.in/yaml/v2 v2.4.2 // indirect + golang.org/x/sys v0.35.0 // indirect + google.golang.org/protobuf v1.36.8 // indirect +) diff --git a/harnesses/evm-block-builders/go.sum b/harnesses/evm-block-builders/go.sum new file mode 100644 index 00000000..91fbc2f0 --- /dev/null +++ b/harnesses/evm-block-builders/go.sum @@ -0,0 +1,52 @@ +github.com/andybalholm/brotli v1.1.1 h1:PR2pgnyFznKEugtsUo0xLdDop5SKXd5Qf5ysW+7XdTA= +github.com/andybalholm/brotli v1.1.1/go.mod h1:05ib4cKhjx3OQYUY22hTVd34Bc8upXjOLL2rKwwZBoA= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= +github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= +github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o= +github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg= +github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= +github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= +github.com/prometheus/common v0.66.1 h1:h5E0h5/Y8niHc5DlaLlWLArTQI7tMrsfQjHV+d9ZoGs= +github.com/prometheus/common v0.66.1/go.mod h1:gcaUsgf3KfRSwHY4dIMXLPV0K/Wg1oZ8+SbZk/HH/dA= +github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg= +github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is= +github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= +github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU= +github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI= +go.yaml.in/yaml/v2 v2.4.2/go.mod h1:081UH+NErpNdqlCXm3TtEran0rJZGxAYx9hb/ELlsPU= +golang.org/x/sys v0.35.0 h1:vz1N37gP5bs89s7He8XuIYXpyY0+QlsKmzipCbUtyxI= +golang.org/x/sys v0.35.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +google.golang.org/protobuf v1.36.8 h1:xHScyCOEuuwZEc6UtSOvPbAT4zRh0xcNRYekJwfqyMc= +google.golang.org/protobuf v1.36.8/go.mod h1:fuxRtAxBytpl4zzqUh6/eyUujkJdNiuEkXntxiD/uRU= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= From 74521ea547a5a0e3cf0265f0efce7179ea35d13f Mon Sep 17 00:00:00 2001 From: Florent Tapponnier Date: Wed, 15 Jul 2026 01:12:40 +0200 Subject: [PATCH 08/12] ebb: don't count historical relay payloads on first poll, retry arb calibration on zero support, reset WS backoff after stable session --- .../evm-block-builders/cmd/script/arb_feed.go | 15 +++++++++++++++ .../cmd/script/base_flashblocks.go | 4 ++++ harnesses/evm-block-builders/cmd/script/config.go | 3 ++- harnesses/evm-block-builders/cmd/script/relays.go | 15 +++++++++++++-- 4 files changed, 34 insertions(+), 3 deletions(-) diff --git a/harnesses/evm-block-builders/cmd/script/arb_feed.go b/harnesses/evm-block-builders/cmd/script/arb_feed.go index 29c51a0f..48784d96 100644 --- a/harnesses/evm-block-builders/cmd/script/arb_feed.go +++ b/harnesses/evm-block-builders/cmd/script/arb_feed.go @@ -104,7 +104,11 @@ func runArbFeed(ctx context.Context, state *arbFeedState) { fmt.Printf("[arb-feed] connecting %s\n", url) backoff := arbMinBackoff for ctx.Err() == nil { + connStart := time.Now() err := readArbFeed(ctx, url, state) + if time.Since(connStart) > 5*time.Minute { + backoff = arbMinBackoff // stable session: don't carry stale backoff + } arbFeedHealth.Set(0) streamReconnects.WithLabelValues("arb-feed").Inc() if err != nil { @@ -242,6 +246,17 @@ func runArbHeadPoller(ctx context.Context, state *arbFeedState) { offset = cand } } + if offset <= 0 { + // No candidate reached support (feed stalled mid-window, + // so cand drifted and the counts spread out). Locking + // offset=0 would silence the lag histogram forever; + // discard the window and re-calibrate instead. + fmt.Printf("[arb-rpc] calibration failed (no candidate with >=%d support over %d samples), retrying\n", minSupport, calibCount) + candidates = map[int64]int{} + calibCount = 0 + warmup = 0 + continue + } calibrated = true prevHead = head arbDerivedOffset.Set(float64(offset)) diff --git a/harnesses/evm-block-builders/cmd/script/base_flashblocks.go b/harnesses/evm-block-builders/cmd/script/base_flashblocks.go index 4ecc9156..c4b37f74 100644 --- a/harnesses/evm-block-builders/cmd/script/base_flashblocks.go +++ b/harnesses/evm-block-builders/cmd/script/base_flashblocks.go @@ -94,7 +94,11 @@ func runBaseFlashblocks(ctx context.Context, state *baseFeedState) { fmt.Printf("[base-fb] connecting %s\n", url) backoff := baseMinBackoff for ctx.Err() == nil { + connStart := time.Now() err := readBaseFlashblocks(ctx, url, state) + if time.Since(connStart) > 5*time.Minute { + backoff = baseMinBackoff // stable session: don't carry stale backoff + } baseFeedHealth.Set(0) streamReconnects.WithLabelValues("base-flashblocks").Inc() if err != nil { diff --git a/harnesses/evm-block-builders/cmd/script/config.go b/harnesses/evm-block-builders/cmd/script/config.go index 5181581a..d5e08c65 100644 --- a/harnesses/evm-block-builders/cmd/script/config.go +++ b/harnesses/evm-block-builders/cmd/script/config.go @@ -18,7 +18,8 @@ import ( // // publicnode 403s default Go/Python User-Agents (bench 067 lesson), so // EVERY outbound HTTP request and WS dial in this harness sends -// harnessUserAgent - see newRequest / wsDialer. +// harnessUserAgent - see newRequest and the WS dial headers in +// arb_feed.go / base_flashblocks.go. const ( defaultEthRPC = "https://ethereum-rpc.publicnode.com" diff --git a/harnesses/evm-block-builders/cmd/script/relays.go b/harnesses/evm-block-builders/cmd/script/relays.go index eaba8212..91c7f922 100644 --- a/harnesses/evm-block-builders/cmd/script/relays.go +++ b/harnesses/evm-block-builders/cmd/script/relays.go @@ -61,6 +61,7 @@ func pollRelay(ctx context.Context, r relayEndpoint) { fmt.Printf("[relay:%s] polling %s every %s\n", r.Slug, url, relayPollInterval) var maxSeenSlot int64 + primed := false // first successful fetch seeds the high-water-mark only t := time.NewTicker(relayPollInterval) defer t.Stop() for { @@ -83,9 +84,19 @@ func pollRelay(ctx context.Context, r relayEndpoint) { maxSeenSlot = slots[0] } } - relayPayloadsTotal.WithLabelValues(r.Slug).Add(float64(fresh)) + if !primed { + // First fetch after (re)start returns up to 50 historical + // payloads that were already counted before the restart. + // Seed the high-water-mark from them WITHOUT incrementing: + // Prom increase() tolerates counter resets, but re-counting + // old slots would inflate the series after every restart. + primed = true + fmt.Printf("[relay:%s] primed high-water-mark maxSlot=%d (skipped %d historical)\n", r.Slug, maxSeenSlot, fresh) + } else { + relayPayloadsTotal.WithLabelValues(r.Slug).Add(float64(fresh)) + fmt.Printf("[relay:%s] delivered=%d fresh=%d maxSlot=%d\n", r.Slug, len(slots), fresh, maxSeenSlot) + } relayLastSlot.WithLabelValues(r.Slug).Set(float64(maxSeenSlot)) - fmt.Printf("[relay:%s] delivered=%d fresh=%d maxSlot=%d\n", r.Slug, len(slots), fresh, maxSeenSlot) } select { From 38cb86ce84b9b5dee1c2c3609a06c9b33c99c4bf Mon Sep 17 00:00:00 2001 From: Florent Tapponnier Date: Wed, 15 Jul 2026 01:06:49 +0200 Subject: [PATCH 09/12] add bench 083 rpc-reliability: consensus lag, hash quorum, fixed-vector integrity in rpc-capabilities harness, bump bench-set cache keys --- benchmarks/rpc-reliability.yml | 294 +++++++++++++++++ .../rpc-capabilities/cmd/script/config.go | 25 +- .../rpc-capabilities/cmd/script/consensus.go | 148 +++++++++ .../cmd/script/consensus_test.go | 144 +++++++++ .../rpc-capabilities/cmd/script/integrity.go | 304 ++++++++++++++++++ harnesses/rpc-capabilities/cmd/script/main.go | 10 +- .../rpc-capabilities/cmd/script/metrics.go | 51 +++ .../rpc-capabilities/cmd/script/probe.go | 41 ++- harnesses/rpc-capabilities/go.mod | 1 + 9 files changed, 1001 insertions(+), 17 deletions(-) create mode 100644 benchmarks/rpc-reliability.yml create mode 100644 harnesses/rpc-capabilities/cmd/script/consensus.go create mode 100644 harnesses/rpc-capabilities/cmd/script/consensus_test.go create mode 100644 harnesses/rpc-capabilities/cmd/script/integrity.go diff --git a/benchmarks/rpc-reliability.yml b/benchmarks/rpc-reliability.yml new file mode 100644 index 00000000..a4240116 --- /dev/null +++ b/benchmarks/rpc-reliability.yml @@ -0,0 +1,294 @@ +# OpenChainBench. Bench № 083 + +slug: rpc-reliability +number: "083" +title: Most reliable free public RPC, correctness incidents across 7 EVM chains +seo_title: "Most reliable free public RPC 2026" +seo_description: "Free public RPCs ranked by correctness incidents: block-hash quorum disagreements, eth_getLogs completeness and eth_getBalance state integrity, live and keyless." +subtitle: "Correctness incidents per 24 hours for free no-key RPC endpoints: block hashes disagreeing with the cross-provider quorum, eth_getLogs counts deviating from the majority, divergent eth_getBalance state and blocked or gated methods, measured continuously alongside consensus head lag." +category: RPCs +status: live +metric: Reliability incidents +unit: count +higher_is_better: false + +disclaimer: > + A zero row means no incident was caught in the window, not a proof of + correctness. Most providers post zero most days; this bench exists for + the days they do not, and for the errors: a blocked eth_getLogs or a + balance read gated behind a paid key at 40 blocks of depth counts as + an incident because it is one from the caller's seat. + +seo_intro: | + Bench 010 answers which free public RPC is fastest; this page answers + the question that should come first. which one is telling the truth. + A fast endpoint that serves a stale head, drops logs from an + eth_getLogs response, or answers eth_getBalance from a diverged state + view is worse than a slow honest one, and none of that shows up in a + latency percentile. So we cross-examine the same keyless provider + cohort the latency bench probes, continuously, on three axes. First, + consensus head lag: every 60 seconds each provider's reported + (height, hash) pair feeds a per-chain quorum map, and the gap between + a provider's head and the highest head any provider reported is + published as a gauge. Second, hash integrity: when at least two + providers agree on the hash at a height, a provider reporting a + different hash at that same height is counted, once per height, as a + mismatch incident. Third, fixed-vector integrity: every 5 minutes, + one chain at a time, every provider gets the identical eth_getLogs + query for the chain's canonical USDC contract over a 10-block window + behind the tip, plus an eth_getBalance read at the same fixed block; + answers are compared across providers and a deviation from the strict + majority is an incident. Errors count too, deliberately: 1RPC blocks + eth_getLogs outright with a -32000, and PublicNode began gating + keyless reads a few dozen blocks deep behind a personal token + (-32602) in July 2026. Those are reliability failures a latency bench + renders invisible, and here they are the headline. + +abstract: | + We measure the correctness of free no-key RPC endpoints, not their + speed. The harness reuses the bench-010 probe matrix (same providers, + same chains, same 60-second header fetch) and adds three metric + families. rpc_consensus_lag_blocks tracks each provider's head + against the highest head seen across providers for that chain. + rpc_hash_mismatch_total counts same-height hash disagreements with + the two-or-more-provider quorum, at most once per height, with a + strict-plurality rule so a live reorg splitting providers two against + two counts nobody. rpc_integrity_check_total classifies fixed-vector + checks (eth_getLogs for the chain's canonical USDC over a 10-block + window at a daily-rotating depth behind tip, and eth_getBalance of a + fixed well-known address at the same block) as ok, error or disagree + against the cross-provider majority. The headline is the raw + incident count over 24 hours: errors plus disagreements plus hash + mismatches. No weighted composite: weights invite gaming disputes, + and an incident is an incident. + +methodology: + - "Consensus lag: the bench-010 latency probe fetches the full latest header every 60 seconds per (provider, chain); its height and hash, previously discarded, feed this bench. rpc_consensus_lag_blocks is the gap between the provider's reported head and the highest head any probed provider reported for that chain (rolling max). Valid-but-stale responses still count as observations; failed probes delete the series so dead endpoints age out." + - "Hash quorum: per chain, a height-to-hash vote map. When one hash at height H is backed by at least 2 providers AND strictly more than any competing hash, it is canonical; a provider that reported a different hash at H is counted in rpc_hash_mismatch_total, at most once per (provider, height). A 2-2 split, the signature of a live reorg, resolves to no quorum and counts nobody." + - "Fixed-vector integrity, cadence: every 5 minutes ONE chain is checked, rotating through the 7 chains with a canonical high-traffic USDC deployment (Ethereum, Arbitrum, Optimism, Base, Polygon, BNB, Avalanche). Each provider on that chain gets 2 checks per round, so a single-chain provider accrues ~82 checks per day and a 7-chain provider ~575." + - "Logs completeness vector: eth_getLogs for the chain's canonical USDC contract (Circle native deployments; Binance-peg on BNB) over a 10-block window ending at tip minus N. Returned log counts are compared across providers; deviation from the strict majority count increments rpc_logs_disagreement_total and books a disagree incident. Live calibration on Ethereum returned 1371 logs consistently on drpc, tenderly and blastapi over the same fixed range." + - "Anti-gaming rotation: N rotates daily over {20, 30, 40, 50, 60} blocks, so a provider cannot special-case a published fixed range, and request ids rotate on every call (the same edge-cache defeat as the latency probe). All depths stay far inside non-archive territory, so a pruned-but-honest node is never penalized." + - "State consistency vector: eth_getBalance of a fixed well-known address (the same Vitalik address the archive-depth probe uses) at the tip-minus-N block. The hex answer must be byte-identical across providers; divergence from the strict majority increments rpc_state_disagreement_total. Live calibration showed byte-identical balances across all responding providers at a fixed block." + - "Errors are signal: rpc_integrity_check_total{result} is ok, error or disagree. error covers blocked methods (1RPC answers eth_getLogs with -32000), depth gating (PublicNode keyless regressed to under 150 blocks of archive depth in July 2026, returning -32602 asking for a personal token), and transport failures. The headline counts error and disagree alike: a method you cannot call is unreliable regardless of why." + - "Headline: incidents per 24h = increase of integrity checks with result error or disagree, plus increase of hash mismatches, from the eu-west vantage. Raw counts, no weighting; the p90/p99 columns are consensus-lag quantiles in blocks, published as raw companion dimensions." + - "Single-region disclosure: incidents are counted from the eu-west probe only. Correctness, unlike latency, is location-independent in principle, but every provider fronts with anycast, so what we audit is the serving cluster eu-west traffic lands on; a diverged cache in another region is invisible to this bench. The us-east and sgp replicas emit the same metrics for cross-checking on the time-series chart." + - "Quorum honesty: strict plurality everywhere. Two agreeing providers outvote one dissenter; two against two resolves to no quorum and books nothing; a lone answer with no second opinion is unverifiable, not wrong, and books ok. Disagreement requires being outvoted by an established majority." + - "Cohort: 12 keyless providers from the bench-010 audited matrix. Excluded: LlamaRPC (fully down, HTTP 521, already delisted from the harness), 1RPC (delisted 2026-07-09 at under 13 percent success; when probed live it also blocked eth_getLogs with -32000, exactly the incident class this bench counts), Ankr / NodeReal / GetBlock / Chainstack (key-gated). Avalanche's foundation endpoint is probed by the harness and visible on bench 010; it is left out here only to keep the cohort at 12." + - "Provider matrix and vector addresses are documented in harnesses/rpc-capabilities/cmd/script/config.go and integrity.go; the harness is shared with bench 010, so both benches see the identical provider set and probe budget." + +findings: + - "Consensus lag is real and measurable at 60-second resolution: calibration rounds of simultaneous eth_getBlockByNumber across the Ethereum cohort caught individual gateways one block behind the cross-provider tip in 2 of 4 rounds, while every provider at equal height returned byte-identical hashes. The lag gauge turns that from an anecdote into a 24h p90/p99 per provider." + - "The completeness gap is not hypothetical. On the same fixed 10-block USDC range, drpc, tenderly and blastapi each returned exactly 1371 logs, 1RPC refused the method with -32000, and PublicNode's keyless tier had regressed its archive depth to under 150 blocks, answering -32602 and asking for a personal token. Three providers agreed, two could not answer; both failure modes are incidents here." + - "State reads are the quiet good news: eth_getBalance at a fixed recent block came back byte-identical across every responding provider during calibration. rpc_state_disagreement_total exists for the day that stops being true, which is precisely the day you want a public counter." + - "{{name:cloudflare}} shows why errors must count: the endpoint answers HTTP 200 fast and fails inside the JSON-RPC body. A correctness bench that only compared successful answers would score it clean; counting error results books its blocked calls as the incidents they are." + - "Zero is the normal reading and that is the point. Free public RPCs mostly do tell the truth; the bench prices the exceptions, publicly timestamps regressions like PublicNode's July 2026 keyless depth gating, and gives the honest majority a way to prove they stayed honest." + +faq: + - q: "What counts as a reliability incident?" + a: "Three things, summed over 24 hours from the eu-west probe. An integrity check that errored (method blocked, depth gated behind a key, transport failure). An integrity check whose answer deviated from the strict cross-provider majority (an eth_getLogs count differing from the count at least two other providers agree on, or an eth_getBalance hex that is not byte-identical to the majority answer). And a block-hash mismatch, where the provider reported a hash at a height where at least two providers agreed on a different one. Raw counts, no weights; lower is better and zero is common." + - q: "Why do method errors count as incidents when this is a correctness bench?" + a: "Because from the caller's seat a method you cannot call is a reliability failure regardless of the reason. 1RPC blocks eth_getLogs with a -32000 error; PublicNode's keyless tier started answering balance reads a few dozen blocks deep with -32602 and a demand for a personal token. Neither returns wrong data, but both break the contract a developer assumes when pasting a no-key URL into a dapp. Counting them alongside genuine disagreements keeps the headline honest: it measures whether the endpoint reliably does what an EVM JSON-RPC endpoint is expected to do." + - q: "Can a provider game this benchmark by special-casing the test queries?" + a: "The vectors are designed to make that expensive. The eth_getLogs window is anchored at tip minus N where N rotates daily over five depths, so the exact range is never fixed; request ids rotate on every call, defeating body-keyed edge caches; and the USDC contracts are the busiest transfer contracts on each chain, so serving them correctly IS serving real traffic correctly. The head-lag and hash-quorum signals cannot be gamed at all without actually running well-synced nodes, because they are computed from the same latest-header probe the latency bench uses." + - q: "Does a reorg unfairly flag providers as mismatching?" + a: "No, by construction. During a live reorg two providers can legitimately hold different hashes at the same height. The quorum rule requires a strict plurality: one hash backed by at least two providers and strictly more than any competing hash. A two-against-two split resolves to no quorum and counts nobody, and a mismatch is booked at most once per provider per height. What remains after those guards, a provider outvoted by an established majority at a settled height, is the signal this bench exists to record." + - q: "Why is this measured from one region only?" + a: "The headline counts incidents from the eu-west vantage because correctness, unlike latency, does not depend on where the client stands: a node serving the canonical chain serves it to everyone. The honest caveat is anycast: each probe sees the serving cluster its region lands on, so a diverged cache serving only another continent is invisible to the eu-west counter. The us-east and Singapore replicas emit the same metric families, and the per-region series are on the chart for cross-checking; the headline stays single-region so the same incident is never double-counted." + +source: https://github.com/ChainBench/OpenChainBench/tree/main/harnesses/rpc-capabilities + +prometheus: + window: 24h + freshness_metric: rpc_consensus_lag_blocks + +# Real metrics emitted by the rpc-capabilities harness (bench-083 set): +# rpc_consensus_lag_blocks{provider, chain, region} gauge +# rpc_hash_mismatch_total{provider, chain, region} counter +# rpc_logs_count{provider, chain, region} gauge +# rpc_logs_disagreement_total{provider, chain, region} counter +# rpc_state_disagreement_total{provider, chain, region} counter +# rpc_integrity_check_total{provider, chain, region, check, result} +# counter, check in {logs, balance}, result in {ok, error, disagree} +# Counters are zero-initialized for the whole (provider x chain) matrix +# at harness start, so increase() reads 0 (not absent) for clean rows. +# Headline scopes region="eu-west" (see methodology); the chain tab +# injects chain="X" into every query. + +dimensions: + chain: + - { value: all, label: All chains } + - { value: ethereum, label: Ethereum } + - { value: polygon, label: Polygon } + - { value: arbitrum, label: Arbitrum } + - { value: optimism, label: Optimism } + - { value: base, label: Base } + - { value: bnb, label: BNB Chain } + - { value: avalanche, label: Avalanche } + +rank_matrix_query: sum by (provider, chain) (increase(rpc_integrity_check_total{region="eu-west",result=~"error|disagree"}[24h])) + sum by (provider, chain) (increase(rpc_hash_mismatch_total{region="eu-west"}[24h])) + +# ~82 integrity checks per provider per chain per day (2 checks per +# 5-min rotation over 7 chains); single-chain providers are the floor. +expected_n: 80 + +ledger_columns: + - { label: "Incidents 24h", slot: p50, unit: count } + - { label: "Head lag p90", slot: p90, unit: count } + - { label: "Head lag p99", slot: p99, unit: count } + +providers: + - slug: publicnode + name: PublicNode + tag: Allnodes-operated; keyless archive depth gated since July 2026 + formula: "Incidents in 24h (eu-west): integrity checks (USDC eth_getLogs count, eth_getBalance bytes) that errored or diverged from the cross-provider majority, plus block-hash quorum disagreements, summed over its measured chains." + queries: + p50: sum(increase(rpc_integrity_check_total{provider="publicnode",region="eu-west",result=~"error|disagree"}[24h])) + sum(increase(rpc_hash_mismatch_total{provider="publicnode",region="eu-west"}[24h])) + p90: avg(quantile_over_time(0.90, rpc_consensus_lag_blocks{provider="publicnode",region="eu-west"}[24h])) + p99: avg(quantile_over_time(0.99, rpc_consensus_lag_blocks{provider="publicnode",region="eu-west"}[24h])) + mean: avg(avg_over_time(rpc_consensus_lag_blocks{provider="publicnode",region="eu-west"}[24h])) + success: sum(increase(rpc_integrity_check_total{provider="publicnode",region="eu-west",result="ok"}[24h])) / sum(increase(rpc_integrity_check_total{provider="publicnode",region="eu-west"}[24h])) + sample_size: sum(increase(rpc_integrity_check_total{provider="publicnode",region="eu-west"}[24h])) + series: sum(increase(rpc_integrity_check_total{provider="publicnode",region="eu-west",result=~"error|disagree"}[1h])) + sum(increase(rpc_hash_mismatch_total{provider="publicnode",region="eu-west"}[1h])) + + - slug: drpc + name: dRPC + tag: Decentralized RPC mesh, consensus-checked routing + formula: "Incidents in 24h (eu-west): integrity checks (USDC eth_getLogs count, eth_getBalance bytes) that errored or diverged from the cross-provider majority, plus block-hash quorum disagreements, summed over its measured chains." + queries: + p50: sum(increase(rpc_integrity_check_total{provider="drpc",region="eu-west",result=~"error|disagree"}[24h])) + sum(increase(rpc_hash_mismatch_total{provider="drpc",region="eu-west"}[24h])) + p90: avg(quantile_over_time(0.90, rpc_consensus_lag_blocks{provider="drpc",region="eu-west"}[24h])) + p99: avg(quantile_over_time(0.99, rpc_consensus_lag_blocks{provider="drpc",region="eu-west"}[24h])) + mean: avg(avg_over_time(rpc_consensus_lag_blocks{provider="drpc",region="eu-west"}[24h])) + success: sum(increase(rpc_integrity_check_total{provider="drpc",region="eu-west",result="ok"}[24h])) / sum(increase(rpc_integrity_check_total{provider="drpc",region="eu-west"}[24h])) + sample_size: sum(increase(rpc_integrity_check_total{provider="drpc",region="eu-west"}[24h])) + series: sum(increase(rpc_integrity_check_total{provider="drpc",region="eu-west",result=~"error|disagree"}[1h])) + sum(increase(rpc_hash_mismatch_total{provider="drpc",region="eu-west"}[1h])) + + - slug: tenderly + name: Tenderly + tag: Multi-chain public gateway, no key + formula: "Incidents in 24h (eu-west): integrity checks (USDC eth_getLogs count, eth_getBalance bytes) that errored or diverged from the cross-provider majority, plus block-hash quorum disagreements, summed over its measured chains." + queries: + p50: sum(increase(rpc_integrity_check_total{provider="tenderly",region="eu-west",result=~"error|disagree"}[24h])) + sum(increase(rpc_hash_mismatch_total{provider="tenderly",region="eu-west"}[24h])) + p90: avg(quantile_over_time(0.90, rpc_consensus_lag_blocks{provider="tenderly",region="eu-west"}[24h])) + p99: avg(quantile_over_time(0.99, rpc_consensus_lag_blocks{provider="tenderly",region="eu-west"}[24h])) + mean: avg(avg_over_time(rpc_consensus_lag_blocks{provider="tenderly",region="eu-west"}[24h])) + success: sum(increase(rpc_integrity_check_total{provider="tenderly",region="eu-west",result="ok"}[24h])) / sum(increase(rpc_integrity_check_total{provider="tenderly",region="eu-west"}[24h])) + sample_size: sum(increase(rpc_integrity_check_total{provider="tenderly",region="eu-west"}[24h])) + series: sum(increase(rpc_integrity_check_total{provider="tenderly",region="eu-west",result=~"error|disagree"}[1h])) + sum(increase(rpc_hash_mismatch_total{provider="tenderly",region="eu-west"}[1h])) + + - slug: nodies + name: Nodies + tag: POKT Network's decentralized public RPC successor + formula: "Incidents in 24h (eu-west): integrity checks (USDC eth_getLogs count, eth_getBalance bytes) that errored or diverged from the cross-provider majority, plus block-hash quorum disagreements, summed over its measured chains." + queries: + p50: sum(increase(rpc_integrity_check_total{provider="nodies",region="eu-west",result=~"error|disagree"}[24h])) + sum(increase(rpc_hash_mismatch_total{provider="nodies",region="eu-west"}[24h])) + p90: avg(quantile_over_time(0.90, rpc_consensus_lag_blocks{provider="nodies",region="eu-west"}[24h])) + p99: avg(quantile_over_time(0.99, rpc_consensus_lag_blocks{provider="nodies",region="eu-west"}[24h])) + mean: avg(avg_over_time(rpc_consensus_lag_blocks{provider="nodies",region="eu-west"}[24h])) + success: sum(increase(rpc_integrity_check_total{provider="nodies",region="eu-west",result="ok"}[24h])) / sum(increase(rpc_integrity_check_total{provider="nodies",region="eu-west"}[24h])) + sample_size: sum(increase(rpc_integrity_check_total{provider="nodies",region="eu-west"}[24h])) + series: sum(increase(rpc_integrity_check_total{provider="nodies",region="eu-west",result=~"error|disagree"}[1h])) + sum(increase(rpc_hash_mismatch_total{provider="nodies",region="eu-west"}[1h])) + + - slug: lava + name: Lava + tag: Decentralized RPC mesh (ETH + Arbitrum no-key) + formula: "Incidents in 24h (eu-west) on Ethereum and Arbitrum: integrity checks (USDC eth_getLogs count, eth_getBalance bytes) that errored or diverged from the cross-provider majority, plus block-hash quorum disagreements." + queries: + p50: sum(increase(rpc_integrity_check_total{provider="lava",region="eu-west",result=~"error|disagree"}[24h])) + sum(increase(rpc_hash_mismatch_total{provider="lava",region="eu-west"}[24h])) + p90: avg(quantile_over_time(0.90, rpc_consensus_lag_blocks{provider="lava",region="eu-west"}[24h])) + p99: avg(quantile_over_time(0.99, rpc_consensus_lag_blocks{provider="lava",region="eu-west"}[24h])) + mean: avg(avg_over_time(rpc_consensus_lag_blocks{provider="lava",region="eu-west"}[24h])) + success: sum(increase(rpc_integrity_check_total{provider="lava",region="eu-west",result="ok"}[24h])) / sum(increase(rpc_integrity_check_total{provider="lava",region="eu-west"}[24h])) + sample_size: sum(increase(rpc_integrity_check_total{provider="lava",region="eu-west"}[24h])) + series: sum(increase(rpc_integrity_check_total{provider="lava",region="eu-west",result=~"error|disagree"}[1h])) + sum(increase(rpc_hash_mismatch_total{provider="lava",region="eu-west"}[1h])) + + - slug: meowrpc + name: MeowRPC + tag: Free public RPC (ETH + Arbitrum) + formula: "Incidents in 24h (eu-west) on Ethereum and Arbitrum: integrity checks (USDC eth_getLogs count, eth_getBalance bytes) that errored or diverged from the cross-provider majority, plus block-hash quorum disagreements." + queries: + p50: sum(increase(rpc_integrity_check_total{provider="meowrpc",region="eu-west",result=~"error|disagree"}[24h])) + sum(increase(rpc_hash_mismatch_total{provider="meowrpc",region="eu-west"}[24h])) + p90: avg(quantile_over_time(0.90, rpc_consensus_lag_blocks{provider="meowrpc",region="eu-west"}[24h])) + p99: avg(quantile_over_time(0.99, rpc_consensus_lag_blocks{provider="meowrpc",region="eu-west"}[24h])) + mean: avg(avg_over_time(rpc_consensus_lag_blocks{provider="meowrpc",region="eu-west"}[24h])) + success: sum(increase(rpc_integrity_check_total{provider="meowrpc",region="eu-west",result="ok"}[24h])) / sum(increase(rpc_integrity_check_total{provider="meowrpc",region="eu-west"}[24h])) + sample_size: sum(increase(rpc_integrity_check_total{provider="meowrpc",region="eu-west"}[24h])) + series: sum(increase(rpc_integrity_check_total{provider="meowrpc",region="eu-west",result=~"error|disagree"}[1h])) + sum(increase(rpc_hash_mismatch_total{provider="meowrpc",region="eu-west"}[1h])) + + - slug: flashbots + name: Flashbots + tag: Private-mempool read proxy, Ethereum only + formula: "Incidents in 24h (eu-west) on Ethereum: integrity checks (USDC eth_getLogs count, eth_getBalance bytes) that errored or diverged from the cross-provider majority, plus block-hash quorum disagreements." + queries: + p50: sum(increase(rpc_integrity_check_total{provider="flashbots",region="eu-west",result=~"error|disagree"}[24h])) + sum(increase(rpc_hash_mismatch_total{provider="flashbots",region="eu-west"}[24h])) + p90: avg(quantile_over_time(0.90, rpc_consensus_lag_blocks{provider="flashbots",region="eu-west"}[24h])) + p99: avg(quantile_over_time(0.99, rpc_consensus_lag_blocks{provider="flashbots",region="eu-west"}[24h])) + mean: avg(avg_over_time(rpc_consensus_lag_blocks{provider="flashbots",region="eu-west"}[24h])) + success: sum(increase(rpc_integrity_check_total{provider="flashbots",region="eu-west",result="ok"}[24h])) / sum(increase(rpc_integrity_check_total{provider="flashbots",region="eu-west"}[24h])) + sample_size: sum(increase(rpc_integrity_check_total{provider="flashbots",region="eu-west"}[24h])) + series: sum(increase(rpc_integrity_check_total{provider="flashbots",region="eu-west",result=~"error|disagree"}[1h])) + sum(increase(rpc_hash_mismatch_total{provider="flashbots",region="eu-west"}[1h])) + + - slug: cloudflare + name: Cloudflare + tag: Permissioned mode; the errors-are-signal exhibit + formula: "Incidents in 24h (eu-west) on Ethereum: integrity checks (USDC eth_getLogs count, eth_getBalance bytes) that errored or diverged from the cross-provider majority, plus block-hash quorum disagreements. Blocked methods count as errors." + queries: + p50: sum(increase(rpc_integrity_check_total{provider="cloudflare",region="eu-west",result=~"error|disagree"}[24h])) + sum(increase(rpc_hash_mismatch_total{provider="cloudflare",region="eu-west"}[24h])) + p90: avg(quantile_over_time(0.90, rpc_consensus_lag_blocks{provider="cloudflare",region="eu-west"}[24h])) + p99: avg(quantile_over_time(0.99, rpc_consensus_lag_blocks{provider="cloudflare",region="eu-west"}[24h])) + mean: avg(avg_over_time(rpc_consensus_lag_blocks{provider="cloudflare",region="eu-west"}[24h])) + success: sum(increase(rpc_integrity_check_total{provider="cloudflare",region="eu-west",result="ok"}[24h])) / sum(increase(rpc_integrity_check_total{provider="cloudflare",region="eu-west"}[24h])) + sample_size: sum(increase(rpc_integrity_check_total{provider="cloudflare",region="eu-west"}[24h])) + series: sum(increase(rpc_integrity_check_total{provider="cloudflare",region="eu-west",result=~"error|disagree"}[1h])) + sum(increase(rpc_hash_mismatch_total{provider="cloudflare",region="eu-west"}[1h])) + + - slug: base-official + name: Base + tag: Coinbase-operated, Base mainnet only + formula: "Incidents in 24h (eu-west) on Base: integrity checks (USDC eth_getLogs count, eth_getBalance bytes) that errored or diverged from the cross-provider majority, plus block-hash quorum disagreements." + queries: + p50: sum(increase(rpc_integrity_check_total{provider="base-official",region="eu-west",result=~"error|disagree"}[24h])) + sum(increase(rpc_hash_mismatch_total{provider="base-official",region="eu-west"}[24h])) + p90: avg(quantile_over_time(0.90, rpc_consensus_lag_blocks{provider="base-official",region="eu-west"}[24h])) + p99: avg(quantile_over_time(0.99, rpc_consensus_lag_blocks{provider="base-official",region="eu-west"}[24h])) + mean: avg(avg_over_time(rpc_consensus_lag_blocks{provider="base-official",region="eu-west"}[24h])) + success: sum(increase(rpc_integrity_check_total{provider="base-official",region="eu-west",result="ok"}[24h])) / sum(increase(rpc_integrity_check_total{provider="base-official",region="eu-west"}[24h])) + sample_size: sum(increase(rpc_integrity_check_total{provider="base-official",region="eu-west"}[24h])) + series: sum(increase(rpc_integrity_check_total{provider="base-official",region="eu-west",result=~"error|disagree"}[1h])) + sum(increase(rpc_hash_mismatch_total{provider="base-official",region="eu-west"}[1h])) + + - slug: arbitrum-official + name: Arbitrum + tag: Arbitrum Foundation RPC, Arbitrum One only + formula: "Incidents in 24h (eu-west) on Arbitrum One: integrity checks (USDC eth_getLogs count, eth_getBalance bytes) that errored or diverged from the cross-provider majority, plus block-hash quorum disagreements." + queries: + p50: sum(increase(rpc_integrity_check_total{provider="arbitrum-official",region="eu-west",result=~"error|disagree"}[24h])) + sum(increase(rpc_hash_mismatch_total{provider="arbitrum-official",region="eu-west"}[24h])) + p90: avg(quantile_over_time(0.90, rpc_consensus_lag_blocks{provider="arbitrum-official",region="eu-west"}[24h])) + p99: avg(quantile_over_time(0.99, rpc_consensus_lag_blocks{provider="arbitrum-official",region="eu-west"}[24h])) + mean: avg(avg_over_time(rpc_consensus_lag_blocks{provider="arbitrum-official",region="eu-west"}[24h])) + success: sum(increase(rpc_integrity_check_total{provider="arbitrum-official",region="eu-west",result="ok"}[24h])) / sum(increase(rpc_integrity_check_total{provider="arbitrum-official",region="eu-west"}[24h])) + sample_size: sum(increase(rpc_integrity_check_total{provider="arbitrum-official",region="eu-west"}[24h])) + series: sum(increase(rpc_integrity_check_total{provider="arbitrum-official",region="eu-west",result=~"error|disagree"}[1h])) + sum(increase(rpc_hash_mismatch_total{provider="arbitrum-official",region="eu-west"}[1h])) + + - slug: optimism-official + name: Optimism + tag: Optimism Foundation RPC, OP mainnet only + formula: "Incidents in 24h (eu-west) on Optimism: integrity checks (USDC eth_getLogs count, eth_getBalance bytes) that errored or diverged from the cross-provider majority, plus block-hash quorum disagreements." + queries: + p50: sum(increase(rpc_integrity_check_total{provider="optimism-official",region="eu-west",result=~"error|disagree"}[24h])) + sum(increase(rpc_hash_mismatch_total{provider="optimism-official",region="eu-west"}[24h])) + p90: avg(quantile_over_time(0.90, rpc_consensus_lag_blocks{provider="optimism-official",region="eu-west"}[24h])) + p99: avg(quantile_over_time(0.99, rpc_consensus_lag_blocks{provider="optimism-official",region="eu-west"}[24h])) + mean: avg(avg_over_time(rpc_consensus_lag_blocks{provider="optimism-official",region="eu-west"}[24h])) + success: sum(increase(rpc_integrity_check_total{provider="optimism-official",region="eu-west",result="ok"}[24h])) / sum(increase(rpc_integrity_check_total{provider="optimism-official",region="eu-west"}[24h])) + sample_size: sum(increase(rpc_integrity_check_total{provider="optimism-official",region="eu-west"}[24h])) + series: sum(increase(rpc_integrity_check_total{provider="optimism-official",region="eu-west",result=~"error|disagree"}[1h])) + sum(increase(rpc_hash_mismatch_total{provider="optimism-official",region="eu-west"}[1h])) + + - slug: binance + name: Binance + tag: BNB Chain dataseed RPC, BNB only + formula: "Incidents in 24h (eu-west) on BNB Chain: integrity checks (USDC eth_getLogs count, eth_getBalance bytes) that errored or diverged from the cross-provider majority, plus block-hash quorum disagreements." + queries: + p50: sum(increase(rpc_integrity_check_total{provider="binance",region="eu-west",result=~"error|disagree"}[24h])) + sum(increase(rpc_hash_mismatch_total{provider="binance",region="eu-west"}[24h])) + p90: avg(quantile_over_time(0.90, rpc_consensus_lag_blocks{provider="binance",region="eu-west"}[24h])) + p99: avg(quantile_over_time(0.99, rpc_consensus_lag_blocks{provider="binance",region="eu-west"}[24h])) + mean: avg(avg_over_time(rpc_consensus_lag_blocks{provider="binance",region="eu-west"}[24h])) + success: sum(increase(rpc_integrity_check_total{provider="binance",region="eu-west",result="ok"}[24h])) / sum(increase(rpc_integrity_check_total{provider="binance",region="eu-west"}[24h])) + sample_size: sum(increase(rpc_integrity_check_total{provider="binance",region="eu-west"}[24h])) + series: sum(increase(rpc_integrity_check_total{provider="binance",region="eu-west",result=~"error|disagree"}[1h])) + sum(increase(rpc_hash_mismatch_total{provider="binance",region="eu-west"}[1h])) diff --git a/harnesses/rpc-capabilities/cmd/script/config.go b/harnesses/rpc-capabilities/cmd/script/config.go index c759b0b5..52750b09 100644 --- a/harnesses/rpc-capabilities/cmd/script/config.go +++ b/harnesses/rpc-capabilities/cmd/script/config.go @@ -57,8 +57,12 @@ type Chain struct { // (drpc caches eth_blockNumber → only 2 clean providers), opBNB // (1rpc 429s at probe cadence, only 3 solid providers), Mode // (3 providers), Zora / Abstract / HyperEVM (≤2 keyless providers). +// +// Local-run filter: OCB_CHAINS=ethereum,base restricts the matrix to +// the listed slugs (unset or unmatched = full matrix). Used for local +// smoke runs; never set in production. func chains() []Chain { - return []Chain{ + all := []Chain{ // ─── Solana mainnet — added 2026-07-12, all 5 endpoints keyless // and live-verified (getSlot + getLatestBlockhash + getVersion, // mutually consistent advancing slots). Excluded by that sweep: @@ -364,6 +368,25 @@ func chains() []Chain { }, }, } + + filter := strings.TrimSpace(os.Getenv("OCB_CHAINS")) + if filter == "" { + return all + } + keep := make(map[string]bool) + for _, s := range strings.Split(filter, ",") { + keep[strings.TrimSpace(s)] = true + } + var out []Chain + for _, c := range all { + if keep[c.Slug] { + out = append(out, c) + } + } + if len(out) == 0 { + return all + } + return out } func envDefault(key, def string) string { diff --git a/harnesses/rpc-capabilities/cmd/script/consensus.go b/harnesses/rpc-capabilities/cmd/script/consensus.go new file mode 100644 index 00000000..0381ef56 --- /dev/null +++ b/harnesses/rpc-capabilities/cmd/script/consensus.go @@ -0,0 +1,148 @@ +package main + +import "sync" + +// consensus.go — bench 083 rpc-reliability, cross-provider head +// agreement. The latency probe already fetches the full latest header +// (anti-cache design) and used to throw the hash away; here every +// successful probe feeds a per-chain height→hash quorum map so we can +// publish two things the latency bench cannot see: +// +// rpc_consensus_lag_blocks — how far this provider's head sits +// behind the highest head any probed provider reported for the +// same chain (the chainTips rolling max). Zero for a provider at +// the shared tip; 1-2 for a gateway one block behind (observed +// live on 1rpc/tenderly); tens for something wedged. +// +// rpc_hash_mismatch_total — the provider's hash at height H +// disagrees with the hash a >=2-provider strict plurality agreed +// on at H. Same-height hash divergence is the serious signal +// (serving a non-canonical or fabricated block), so it is a +// counter, incremented at most once per (provider, height). +// +// Reorg honesty: during a real reorg two providers can legitimately +// sit on different hashes at the same height. quorumHash requires a +// strict plurality (>=2 votes AND strictly more than any competing +// hash), so a 2-2 split counts nobody. Only a provider outvoted by an +// established majority is flagged. + +const ( + // quorumMinProviders: minimum providers agreeing on one hash + // before that hash is treated as canonical at its height. + quorumMinProviders = 2 + // consensusPruneDepth: heights this far below the chain tip are + // dropped from the vote map so memory stays bounded (~2 blocks/s + // chains would otherwise grow forever). + consensusPruneDepth uint64 = 128 +) + +type heightVotes struct { + hash map[string]string // provider -> reported hash + flagged map[string]bool // providers already counted at this height +} + +type chainConsensus struct { + heights map[uint64]*heightVotes +} + +type consensusTracker struct { + mu sync.Mutex + chains map[string]*chainConsensus +} + +func newConsensusTracker() *consensusTracker { + return &consensusTracker{chains: make(map[string]*chainConsensus)} +} + +var consensus = newConsensusTracker() + +// observe records one valid head probe. Called from probeOne for +// results classified ok or stale (a stale block is still a valid +// (height, hash) observation — its lag is exactly the point). hash is +// "" on the Solana path (getSlot carries no hash): lag is emitted, +// quorum voting is skipped. +func (t *consensusTracker) observe(chain, provider string, height uint64, hash string) { + // Lag against the cross-provider rolling max the staleness check + // already maintains. The caller updates tips before observing, so + // a provider that IS the tip reads 0. Noise note: tips is updated + // asynchronously by per-provider goroutines staggered across the + // probe interval, so ±1 block of jitter on fast chains is expected + // and averages out in the 24h quantiles. + tip := tips.get(chain) + lag := 0.0 + if tip > height { + lag = float64(tip - height) + } + rpcConsensusLag.WithLabelValues(provider, chain, currentRegion).Set(lag) + + if hash == "" { + return + } + + t.mu.Lock() + defer t.mu.Unlock() + + cc := t.chains[chain] + if cc == nil { + cc = &chainConsensus{heights: make(map[uint64]*heightVotes)} + t.chains[chain] = cc + } + hv := cc.heights[height] + if hv == nil { + hv = &heightVotes{hash: make(map[string]string), flagged: make(map[string]bool)} + cc.heights[height] = hv + } + hv.hash[provider] = hash + + if q := quorumHash(hv.hash); q != "" { + for p, h := range hv.hash { + if h != q && !hv.flagged[p] { + hv.flagged[p] = true + rpcHashMismatch.WithLabelValues(p, chain, currentRegion).Inc() + } + } + } + + for h := range cc.heights { + if h+consensusPruneDepth < tip { + delete(cc.heights, h) + } + } +} + +// quorumHash returns the hash backed by >= quorumMinProviders votes +// AND strictly more votes than any competing hash. Ties (the 2-2 reorg +// split) return "" so no side is punished without a real majority. +func quorumHash(votes map[string]string) string { + counts := make(map[string]int, len(votes)) + for _, h := range votes { + counts[h]++ + } + best, bestN, secondN := "", 0, 0 + for h, n := range counts { + if n > bestN { + best, secondN, bestN = h, bestN, n + } else if n > secondN { + secondN = n + } + } + if bestN >= quorumMinProviders && bestN > secondN { + return best + } + return "" +} + +// initConsensusMetrics zero-initializes the mismatch counters for the +// full (provider × chain) matrix so `increase()` in the bench's +// incident queries resolves to 0 instead of an absent series for +// providers that never misbehave (which is, hopefully, most of them). +func initConsensusMetrics() { + for _, c := range chains() { + if c.Kind == "solana" { + continue // no hash to vote on + } + for _, p := range c.Providers { + rpcHashMismatch.WithLabelValues(p.Slug, c.Slug, currentRegion).Add(0) + } + } +} diff --git a/harnesses/rpc-capabilities/cmd/script/consensus_test.go b/harnesses/rpc-capabilities/cmd/script/consensus_test.go new file mode 100644 index 00000000..27e36d65 --- /dev/null +++ b/harnesses/rpc-capabilities/cmd/script/consensus_test.go @@ -0,0 +1,144 @@ +package main + +import ( + "testing" + + "github.com/prometheus/client_golang/prometheus/testutil" +) + +func TestQuorumHash(t *testing.T) { + cases := []struct { + name string + votes map[string]string + want string + }{ + {"empty", map[string]string{}, ""}, + {"single vote no quorum", map[string]string{"a": "0xaa"}, ""}, + {"two agree", map[string]string{"a": "0xaa", "b": "0xaa"}, "0xaa"}, + {"majority beats minority", map[string]string{"a": "0xaa", "b": "0xaa", "c": "0xbb"}, "0xaa"}, + {"reorg 2-2 split counts nobody", map[string]string{"a": "0xaa", "b": "0xaa", "c": "0xbb", "d": "0xbb"}, ""}, + {"3-2 split resolves", map[string]string{"a": "0xaa", "b": "0xaa", "c": "0xaa", "d": "0xbb", "e": "0xbb"}, "0xaa"}, + {"all distinct", map[string]string{"a": "0xaa", "b": "0xbb", "c": "0xcc"}, ""}, + } + for _, tc := range cases { + if got := quorumHash(tc.votes); got != tc.want { + t.Errorf("%s: quorumHash = %q, want %q", tc.name, got, tc.want) + } + } +} + +func TestObserveFlagsMinorityOnce(t *testing.T) { + tr := newConsensusTracker() + chain := "testchain-mismatch" + tips.update(chain, 100) + + mismatches := func(p string) float64 { + return testutil.ToFloat64(rpcHashMismatch.WithLabelValues(p, chain, currentRegion)) + } + + // First vote: no quorum yet, nobody flagged. + tr.observe(chain, "alpha", 100, "0xaa") + if got := mismatches("alpha"); got != 0 { + t.Fatalf("alpha flagged before any quorum existed: %v", got) + } + + // Quorum forms on 0xaa; gamma disagrees and is flagged exactly once. + tr.observe(chain, "beta", 100, "0xaa") + tr.observe(chain, "gamma", 100, "0xbb") + if got := mismatches("gamma"); got != 1 { + t.Fatalf("gamma mismatch count = %v, want 1", got) + } + if mismatches("alpha") != 0 || mismatches("beta") != 0 { + t.Fatalf("quorum members were flagged") + } + + // Re-reporting the same bad hash at the same height must not + // double-count: 60s probes revisit heights on slow chains. + tr.observe(chain, "gamma", 100, "0xbb") + if got := mismatches("gamma"); got != 1 { + t.Fatalf("gamma double-counted at same height: %v", got) + } + + // A different height is a new incident. + tr.observe(chain, "alpha", 101, "0xcc") + tr.observe(chain, "beta", 101, "0xcc") + tr.observe(chain, "gamma", 101, "0xdd") + if got := mismatches("gamma"); got != 2 { + t.Fatalf("gamma mismatch count after 2nd height = %v, want 2", got) + } +} + +func TestObserveReorgSplitCountsNobody(t *testing.T) { + tr := newConsensusTracker() + chain := "testchain-reorg" + tips.update(chain, 50) + + tr.observe(chain, "a", 50, "0xaa") + tr.observe(chain, "b", 50, "0xaa") + tr.observe(chain, "c", 50, "0xbb") + tr.observe(chain, "d", 50, "0xbb") + // c was flagged while 0xaa held a 2-1 plurality; d's vote made it + // 2-2, and from that point no NEW flags may appear. + before := testutil.ToFloat64(rpcHashMismatch.WithLabelValues("d", chain, currentRegion)) + if before != 0 { + t.Fatalf("d flagged on a 2-2 split: %v", before) + } +} + +func TestObserveConsensusLagGauge(t *testing.T) { + tr := newConsensusTracker() + chain := "testchain-lag" + tips.update(chain, 200) + + tr.observe(chain, "laggy", 197, "0xaa") + got := testutil.ToFloat64(rpcConsensusLag.WithLabelValues("laggy", chain, currentRegion)) + if got != 3 { + t.Fatalf("lag gauge = %v, want 3", got) + } + + // Provider at (or ahead of) the recorded tip reads 0. + tr.observe(chain, "fresh", 200, "0xaa") + if got := testutil.ToFloat64(rpcConsensusLag.WithLabelValues("fresh", chain, currentRegion)); got != 0 { + t.Fatalf("fresh lag gauge = %v, want 0", got) + } + + // Solana path: empty hash emits lag but never votes. + tr.observe(chain, "solananode", 195, "") + if got := testutil.ToFloat64(rpcConsensusLag.WithLabelValues("solananode", chain, currentRegion)); got != 5 { + t.Fatalf("solana lag gauge = %v, want 5", got) + } + if got := testutil.ToFloat64(rpcHashMismatch.WithLabelValues("solananode", chain, currentRegion)); got != 0 { + t.Fatalf("hashless observation voted: %v", got) + } +} + +func TestConsensusPruning(t *testing.T) { + tr := newConsensusTracker() + chain := "testchain-prune" + tips.update(chain, 10) + + tr.observe(chain, "a", 10, "0xaa") + tips.update(chain, 10+consensusPruneDepth+5) + tr.observe(chain, "a", 10+consensusPruneDepth+5, "0xbb") + + tr.mu.Lock() + defer tr.mu.Unlock() + if _, ok := tr.chains[chain].heights[10]; ok { + t.Fatalf("height 10 not pruned at tip %d", 10+consensusPruneDepth+5) + } +} + +func TestMajority(t *testing.T) { + if v, ok := majority(map[int]int{1371: 3, 1370: 1}); !ok || v != 1371 { + t.Fatalf("majority(3-1) = %v %v", v, ok) + } + if _, ok := majority(map[int]int{1371: 2, 1370: 2}); ok { + t.Fatalf("tie must not resolve") + } + if _, ok := majority(map[int]int{1371: 1}); ok { + t.Fatalf("single answer must not resolve") + } + if v, ok := majority(map[string]int{"0x1": 2}); !ok || v != "0x1" { + t.Fatalf("majority(2-0) = %v %v", v, ok) + } +} diff --git a/harnesses/rpc-capabilities/cmd/script/integrity.go b/harnesses/rpc-capabilities/cmd/script/integrity.go new file mode 100644 index 00000000..e8663ed4 --- /dev/null +++ b/harnesses/rpc-capabilities/cmd/script/integrity.go @@ -0,0 +1,304 @@ +package main + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "time" +) + +// integrity.go — bench 083 rpc-reliability, fixed-vector correctness +// checks. Every 5 minutes ONE chain (rotating through the set below) +// gets the same two questions asked of every provider serving it: +// +// logs — `eth_getLogs` for the chain's canonical USDC contract +// over a 10-block window ending at tip-N. Completeness: +// a provider silently dropping logs (or blocking the +// method, or gating the depth behind a paid tier) shows +// up against the cross-provider majority count. +// balance — `eth_getBalance` of a fixed well-known address at the +// same tip-N block. Consistency: the hex answer must be +// byte-identical across providers; the comparison works +// whether or not the balance is non-zero. +// +// Anti-gaming: N rotates daily over {20, 30, 40, 50, 60} blocks so a +// provider cannot special-case a fixed range, and every request id +// rotates (same edge-cache defeat as the latency probe). All depths +// stay far inside non-archive territory so pruned-but-honest nodes +// are never penalized; a provider that gates even 60-blocks-deep data +// behind a key (observed live: publicnode -32602 "Archive requests +// require a personal token") is emitting exactly the signal this +// bench exists to record. +// +// Errors ARE signal: result="error" on rpc_integrity_check_total is +// counted as an incident by the spec, because "method blocked" and +// "depth gated" are reliability failures from the caller's seat. + +const ( + integrityInterval = 5 * time.Minute + integrityTimeout = 15 * time.Second + // integrityCallSpacing: sequential per-provider spacing; meowrpc + // 429s on rapid bursts (same lesson as archive.go). + integrityCallSpacing = 1500 * time.Millisecond + // integrityLogsSpan: width of the getLogs window, in blocks. + integrityLogsSpan uint64 = 10 + // integrityBalanceAddr: same well-known address the archive loop + // uses (Vitalik). The check compares answers across providers + // byte-for-byte, so it is divergence we measure, not the value. + integrityBalanceAddr = archiveTestAddr +) + +// integrityVectors maps chain slug -> canonical USDC contract used as +// the fixed eth_getLogs vector. Only chains with a heavily traded +// canonical USDC participate (a quiet contract would return 0 logs +// everywhere and the completeness check would be vacuous). Native +// Circle deployments except BNB (Binance-peg, still the busiest +// USDC-family contract there). +var integrityVectors = map[string]string{ + "ethereum": "0xA0b86991c6218b36c1d19D4a2e9Eb0cE3606eB48", + "arbitrum": "0xaf88d065e77c8cC2239327C5EDb3A432268e5831", + "optimism": "0x0b2C639c533813f4Aa9D7837CAf62653d097Ff85", + "base": "0x833589fCD6eDb6E08f4c7C32D4f71b54bdA02913", + "polygon": "0x3c499c542cEF5E3811e1192ce70d8cC03d5c3359", + "bnb": "0x8AC76a51cc950d9822D68b83fE1Ad97B32Cd580d", + "avalanche": "0xB97EF9Ef8734C71904D8002F8b6Bc66Dd9c48a6E", +} + +// integrityOffset returns today's tip-N base depth: 20 + 10*(day%5), +// i.e. {20, 30, 40, 50, 60}, rotating at UTC midnight. +func integrityOffset() uint64 { + day := uint64(time.Now().UTC().Unix() / 86400) + return 20 + 10*(day%5) +} + +// StartIntegrityLoop rotates through the vector-equipped chains, one +// chain per 5-minute tick. Non-blocking; spawns its own goroutine. +func StartIntegrityLoop(ctx context.Context) { + var targets []Chain + for _, c := range chains() { + if c.Kind == "" && integrityVectors[c.Slug] != "" { + targets = append(targets, c) + } + } + if len(targets) == 0 { + return + } + initIntegrityMetrics(targets) + + go func() { + // Let the latency loop populate chainTips first: the vector + // block range is derived from the cross-provider tip. + select { + case <-ctx.Done(): + return + case <-time.After(90 * time.Second): + } + i := 0 + integrityTick(ctx, targets[i%len(targets)]) + i++ + t := time.NewTicker(integrityInterval) + defer t.Stop() + for { + select { + case <-ctx.Done(): + return + case <-t.C: + integrityTick(ctx, targets[i%len(targets)]) + i++ + } + } + }() +} + +type integrityObs struct { + p Provider + logsN int + logsErr error + balHex string + balErr error +} + +func integrityTick(ctx context.Context, c Chain) { + tip := tips.get(c.Slug) + off := integrityOffset() + if tip <= off+integrityLogsSpan { + fmt.Printf("[integrity/%s] no tip yet (tip=%d), skipping round\n", c.Slug, tip) + return + } + to := tip - off + from := to - (integrityLogsSpan - 1) + + obs := make([]integrityObs, 0, len(c.Providers)) + for _, p := range c.Providers { + o := integrityObs{p: p} + o.logsN, o.logsErr = fetchLogsCount(ctx, p.URL, integrityVectors[c.Slug], from, to) + if !integritySleep(ctx) { + return + } + o.balHex, o.balErr = fetchBalanceHex(ctx, p.URL, integrityBalanceAddr, to) + obs = append(obs, o) + if !integritySleep(ctx) { + return + } + } + + // Logs completeness: strict majority of the successful counts. + logsCounts := make(map[int]int) + for _, o := range obs { + if o.logsErr == nil { + logsCounts[o.logsN]++ + } + } + majLogs, logsQuorum := majority(logsCounts) + for _, o := range obs { + switch { + case o.logsErr != nil: + rpcLogsCount.DeleteLabelValues(o.p.Slug, c.Slug, currentRegion) + rpcIntegrityCheck.WithLabelValues(o.p.Slug, c.Slug, currentRegion, "logs", "error").Inc() + fmt.Printf("[integrity/%s/%s] logs range=%d-%d err=%v\n", c.Slug, o.p.Slug, from, to, o.logsErr) + case logsQuorum && o.logsN != majLogs: + rpcLogsCount.WithLabelValues(o.p.Slug, c.Slug, currentRegion).Set(float64(o.logsN)) + rpcLogsDisagreement.WithLabelValues(o.p.Slug, c.Slug, currentRegion).Inc() + rpcIntegrityCheck.WithLabelValues(o.p.Slug, c.Slug, currentRegion, "logs", "disagree").Inc() + fmt.Printf("[integrity/%s/%s] logs DISAGREE got=%d majority=%d range=%d-%d\n", c.Slug, o.p.Slug, o.logsN, majLogs, from, to) + default: + // No >=2 quorum this round (too few successes) also lands + // here: a lone answer is unverifiable, not wrong. + rpcLogsCount.WithLabelValues(o.p.Slug, c.Slug, currentRegion).Set(float64(o.logsN)) + rpcIntegrityCheck.WithLabelValues(o.p.Slug, c.Slug, currentRegion, "logs", "ok").Inc() + fmt.Printf("[integrity/%s/%s] logs ok count=%d range=%d-%d\n", c.Slug, o.p.Slug, o.logsN, from, to) + } + } + + // State consistency: strict majority of the raw hex answers. + balCounts := make(map[string]int) + for _, o := range obs { + if o.balErr == nil { + balCounts[o.balHex]++ + } + } + majBal, balQuorum := majority(balCounts) + for _, o := range obs { + switch { + case o.balErr != nil: + rpcIntegrityCheck.WithLabelValues(o.p.Slug, c.Slug, currentRegion, "balance", "error").Inc() + fmt.Printf("[integrity/%s/%s] balance block=%d err=%v\n", c.Slug, o.p.Slug, to, o.balErr) + case balQuorum && o.balHex != majBal: + rpcStateDisagreement.WithLabelValues(o.p.Slug, c.Slug, currentRegion).Inc() + rpcIntegrityCheck.WithLabelValues(o.p.Slug, c.Slug, currentRegion, "balance", "disagree").Inc() + fmt.Printf("[integrity/%s/%s] balance DISAGREE got=%s majority=%s block=%d\n", c.Slug, o.p.Slug, o.balHex, majBal, to) + default: + rpcIntegrityCheck.WithLabelValues(o.p.Slug, c.Slug, currentRegion, "balance", "ok").Inc() + fmt.Printf("[integrity/%s/%s] balance ok block=%d\n", c.Slug, o.p.Slug, to) + } + } +} + +func integritySleep(ctx context.Context) bool { + select { + case <-ctx.Done(): + return false + case <-time.After(integrityCallSpacing): + return true + } +} + +// majority returns the value backed by >=2 votes and strictly more +// than any competing value; ok=false when no such strict majority +// exists (all-distinct answers, or a tie). +func majority[T comparable](counts map[T]int) (T, bool) { + var best T + bestN, secondN := 0, 0 + for v, n := range counts { + if n > bestN { + best, secondN, bestN = v, bestN, n + } else if n > secondN { + secondN = n + } + } + return best, bestN >= 2 && bestN > secondN +} + +func fetchLogsCount(ctx context.Context, url, addr string, from, to uint64) (int, error) { + body := fmt.Sprintf( + `{"jsonrpc":"2.0","method":"eth_getLogs","params":[{"fromBlock":"0x%x","toBlock":"0x%x","address":"%s"}],"id":%d}`, + from, to, addr, time.Now().UnixNano(), + ) + raw, err := integrityPost(ctx, url, body) + if err != nil { + return 0, err + } + var r rpcBlockEnvelope + if err := json.Unmarshal(raw, &r); err != nil { + return 0, err + } + if r.Error != nil { + return 0, fmt.Errorf("rpc %d: %s", r.Error.Code, r.Error.Message) + } + var logs []json.RawMessage + if err := json.Unmarshal(r.Result, &logs); err != nil { + return 0, fmt.Errorf("non-array result") + } + return len(logs), nil +} + +func fetchBalanceHex(ctx context.Context, url, addr string, block uint64) (string, error) { + body := fmt.Sprintf( + `{"jsonrpc":"2.0","method":"eth_getBalance","params":["%s","0x%x"],"id":%d}`, + addr, block, time.Now().UnixNano(), + ) + raw, err := integrityPost(ctx, url, body) + if err != nil { + return "", err + } + var r rpcEnvelope + if err := json.Unmarshal(raw, &r); err != nil { + return "", err + } + if r.Error != nil { + return "", fmt.Errorf("rpc %d: %s", r.Error.Code, r.Error.Message) + } + if r.Result == "" { + return "", fmt.Errorf("empty result") + } + return r.Result, nil +} + +func integrityPost(ctx context.Context, url, body string) ([]byte, error) { + c, cancel := context.WithTimeout(ctx, integrityTimeout) + defer cancel() + req, _ := http.NewRequestWithContext(c, "POST", url, bytes.NewReader([]byte(body))) + req.Header.Set("Content-Type", "application/json") + req.Header.Set("User-Agent", "OpenChainBench/1.0 (+https://openchainbench.com)") + client := &http.Client{Timeout: integrityTimeout} + resp, err := client.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + if resp.StatusCode != 200 { + _, _ = io.Copy(io.Discard, resp.Body) + return nil, fmt.Errorf("http %d", resp.StatusCode) + } + return io.ReadAll(resp.Body) +} + +// initIntegrityMetrics zero-initializes every counter in the vector +// matrix so the bench's `increase()` queries read 0, not absent, for +// clean providers. +func initIntegrityMetrics(targets []Chain) { + for _, c := range targets { + for _, p := range c.Providers { + for _, check := range []string{"logs", "balance"} { + for _, result := range []string{"ok", "error", "disagree"} { + rpcIntegrityCheck.WithLabelValues(p.Slug, c.Slug, currentRegion, check, result).Add(0) + } + } + rpcLogsDisagreement.WithLabelValues(p.Slug, c.Slug, currentRegion).Add(0) + rpcStateDisagreement.WithLabelValues(p.Slug, c.Slug, currentRegion).Add(0) + } + } +} diff --git a/harnesses/rpc-capabilities/cmd/script/main.go b/harnesses/rpc-capabilities/cmd/script/main.go index 9bea8647..9b18b764 100644 --- a/harnesses/rpc-capabilities/cmd/script/main.go +++ b/harnesses/rpc-capabilities/cmd/script/main.go @@ -73,11 +73,16 @@ func main() { } } fmt.Println() - fmt.Println("Metrics server: :2112/metrics") + + // OCB_METRICS_ADDR: explicit local-run override only (e.g. a dev + // box where :2112 is taken). We still ignore Railway's injected + // $PORT on purpose; Prometheus scrapes :2112 in prod. + addr := envDefault("OCB_METRICS_ADDR", ":2112") + fmt.Printf("Metrics server: %s/metrics\n", addr) fmt.Println() go func() { - if err := StartMetricsServer(":2112"); err != nil { + if err := StartMetricsServer(addr); err != nil { fmt.Printf("[fatal] metrics server: %v\n", err) os.Exit(1) } @@ -88,6 +93,7 @@ func main() { StartProbeLoop(ctx) StartArchiveLoop(ctx) + StartIntegrityLoop(ctx) sig := make(chan os.Signal, 1) signal.Notify(sig, os.Interrupt, syscall.SIGTERM) diff --git a/harnesses/rpc-capabilities/cmd/script/metrics.go b/harnesses/rpc-capabilities/cmd/script/metrics.go index df15637f..304c860e 100644 --- a/harnesses/rpc-capabilities/cmd/script/metrics.go +++ b/harnesses/rpc-capabilities/cmd/script/metrics.go @@ -57,6 +57,57 @@ var ( }, []string{"provider", "chain", "region", "depth"}, ) + + // ─── Bench 083 rpc-reliability: correctness / integrity metrics ──── + // Emitted from the same probe matrix; consensus.go + integrity.go. + + rpcConsensusLag = promauto.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "rpc_consensus_lag_blocks", + Help: "Blocks (slots on Solana) between this provider's reported head and the highest head any probed provider reported for the same chain. Set on every valid head probe (ok or stale); deleted on failure so a dead endpoint ages out instead of freezing at its last lag.", + }, + []string{"provider", "chain", "region"}, + ) + + rpcHashMismatch = promauto.NewCounterVec( + prometheus.CounterOpts{ + Name: "rpc_hash_mismatch_total", + Help: "Times a provider reported a block hash at height H that disagrees with the hash at least 2 other-or-same providers agreed on at H (strict plurality). Incremented at most once per (provider, height); a 2-2 reorg split yields no quorum and nobody is counted.", + }, + []string{"provider", "chain", "region"}, + ) + + rpcLogsCount = promauto.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "rpc_logs_count", + Help: "Number of logs the provider returned for the fixed-vector `eth_getLogs` check (canonical USDC contract, 10-block range at a daily-rotating depth behind tip). Deleted when the call errors so a blocked method doesn't freeze a stale count.", + }, + []string{"provider", "chain", "region"}, + ) + + rpcLogsDisagreement = promauto.NewCounterVec( + prometheus.CounterOpts{ + Name: "rpc_logs_disagreement_total", + Help: "Times a provider's `eth_getLogs` count for the fixed vector deviated from the strict cross-provider majority count in the same round.", + }, + []string{"provider", "chain", "region"}, + ) + + rpcStateDisagreement = promauto.NewCounterVec( + prometheus.CounterOpts{ + Name: "rpc_state_disagreement_total", + Help: "Times a provider's `eth_getBalance` hex at the fixed recent block was not byte-identical to the strict cross-provider majority answer in the same round.", + }, + []string{"provider", "chain", "region"}, + ) + + rpcIntegrityCheck = promauto.NewCounterVec( + prometheus.CounterOpts{ + Name: "rpc_integrity_check_total", + Help: "Fixed-vector integrity check outcomes. check is `logs` (eth_getLogs USDC 10-block window) or `balance` (eth_getBalance at a fixed recent block). result is ok (matches majority), error (method blocked, archive gated, transport failure: errors are signal), or disagree (diverged from the >=2-provider majority).", + }, + []string{"provider", "chain", "region", "check", "result"}, + ) ) // StartMetricsServer binds /metrics + /health on addr. Blocking call — diff --git a/harnesses/rpc-capabilities/cmd/script/probe.go b/harnesses/rpc-capabilities/cmd/script/probe.go index d19f1925..5a0c55cb 100644 --- a/harnesses/rpc-capabilities/cmd/script/probe.go +++ b/harnesses/rpc-capabilities/cmd/script/probe.go @@ -83,6 +83,7 @@ type rpcBlockEnvelope struct { type blockHeader struct { Number string `json:"number"` + Hash string `json:"hash"` } // callLatestBlock issues `eth_getBlockByNumber("latest", false)` and @@ -95,8 +96,9 @@ type blockHeader struct { // node, which let cache-fronted gateways top the latency leaderboard // on cache hits rather than real RPC work. Fetching the full latest // header with a rotating request id defeats body-keyed edge caches; -// the header's `number` field keeps the staleness check intact. -func callLatestBlock(ctx context.Context, url string) (block uint64, result string, latencyMs float64, err error) { +// the header's `number` field keeps the staleness check intact and its +// `hash` feeds the bench-083 cross-provider quorum map (consensus.go). +func callLatestBlock(ctx context.Context, url string) (block uint64, hash string, result string, latencyMs float64, err error) { body := []byte(fmt.Sprintf( `{"jsonrpc":"2.0","method":"eth_getBlockByNumber","params":["latest",false],"id":%d}`, time.Now().UnixNano(), @@ -112,48 +114,49 @@ func callLatestBlock(ctx context.Context, url string) (block uint64, result stri if err != nil { if ctx.Err() != nil || strings.Contains(err.Error(), "deadline exceeded") || strings.Contains(err.Error(), "Timeout") { - return 0, "timeout", latencyMs, err + return 0, "", "timeout", latencyMs, err } - return 0, "http_err", latencyMs, err + return 0, "", "http_err", latencyMs, err } defer resp.Body.Close() if resp.StatusCode != 200 { _, _ = io.Copy(io.Discard, resp.Body) - return 0, "http_err", latencyMs, fmt.Errorf("status %d", resp.StatusCode) + return 0, "", "http_err", latencyMs, fmt.Errorf("status %d", resp.StatusCode) } raw, err := io.ReadAll(resp.Body) if err != nil { - return 0, "http_err", latencyMs, err + return 0, "", "http_err", latencyMs, err } var r rpcBlockEnvelope if err := json.Unmarshal(raw, &r); err != nil { - return 0, "http_err", latencyMs, err + return 0, "", "http_err", latencyMs, err } if r.Error != nil { - return 0, "jsonrpc_err", latencyMs, fmt.Errorf("rpc -%d: %s", r.Error.Code, r.Error.Message) + return 0, "", "jsonrpc_err", latencyMs, fmt.Errorf("rpc -%d: %s", r.Error.Code, r.Error.Message) } if len(r.Result) == 0 || string(r.Result) == "null" { - return 0, "jsonrpc_err", latencyMs, fmt.Errorf("empty result") + return 0, "", "jsonrpc_err", latencyMs, fmt.Errorf("empty result") } var hdr blockHeader if err := json.Unmarshal(r.Result, &hdr); err != nil { - return 0, "jsonrpc_err", latencyMs, err + return 0, "", "jsonrpc_err", latencyMs, err } if hdr.Number == "" { - return 0, "jsonrpc_err", latencyMs, fmt.Errorf("header missing number") + return 0, "", "jsonrpc_err", latencyMs, fmt.Errorf("header missing number") } n, err := strconv.ParseUint(strings.TrimPrefix(hdr.Number, "0x"), 16, 64) if err != nil { - return 0, "jsonrpc_err", latencyMs, err + return 0, "", "jsonrpc_err", latencyMs, err } - return n, "ok", latencyMs, nil + return n, hdr.Hash, "ok", latencyMs, nil } // StartProbeLoop spawns one goroutine per (chain × provider). Each // goroutine runs forever, ticking every probeInterval. func StartProbeLoop(ctx context.Context) { + initConsensusMetrics() for _, c := range chains() { c := c for _, p := range c.Providers { @@ -180,13 +183,14 @@ func probeOne(ctx context.Context, c Chain, p Provider) { probeCtx, cancel := context.WithTimeout(ctx, probeTimeout) defer cancel() var block uint64 + var hash string var result string var latency float64 var err error if c.Kind == "solana" { block, result, latency, err = callLatestSlot(probeCtx, p.URL) } else { - block, result, latency, err = callLatestBlock(probeCtx, p.URL) + block, hash, result, latency, err = callLatestBlock(probeCtx, p.URL) } if result == "ok" { @@ -200,6 +204,15 @@ func probeOne(ctx context.Context, c Chain, p Provider) { result = "stale" } } + // Bench 083: valid observations (fresh or stale, both carry a + // real height + hash) feed the consensus lag gauge and the + // height→hash quorum map; anything else deletes the lag series + // so a dead endpoint ages out instead of freezing. + if result == "ok" || result == "stale" { + consensus.observe(c.Slug, p.Slug, block, hash) + } else { + rpcConsensusLag.DeleteLabelValues(p.Slug, c.Slug, currentRegion) + } rpcCallTotal.WithLabelValues(p.Slug, c.Slug, currentRegion, result).Inc() if result == "ok" { // Latency is recorded ONLY for fresh, valid responses. Error diff --git a/harnesses/rpc-capabilities/go.mod b/harnesses/rpc-capabilities/go.mod index 74e2e1fb..3c5d672c 100644 --- a/harnesses/rpc-capabilities/go.mod +++ b/harnesses/rpc-capabilities/go.mod @@ -8,6 +8,7 @@ require ( github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/kr/text v0.2.0 // indirect + github.com/kylelemons/godebug v1.1.0 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/prometheus/client_model v0.6.2 // indirect github.com/prometheus/common v0.66.1 // indirect From 9b7109d091b207b8db6aa4f08dd032fa2f449226 Mon Sep 17 00:00:00 2001 From: Florent Tapponnier Date: Wed, 15 Jul 2026 01:12:51 +0200 Subject: [PATCH 10/12] bench 083: clamp_min rank_matrix_query so zero-incident providers keep cell ranks --- benchmarks/rpc-reliability.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/benchmarks/rpc-reliability.yml b/benchmarks/rpc-reliability.yml index a4240116..6906b7f5 100644 --- a/benchmarks/rpc-reliability.yml +++ b/benchmarks/rpc-reliability.yml @@ -125,7 +125,10 @@ dimensions: - { value: bnb, label: BNB Chain } - { value: avalanche, label: Avalanche } -rank_matrix_query: sum by (provider, chain) (increase(rpc_integrity_check_total{region="eu-west",result=~"error|disagree"}[24h])) + sum by (provider, chain) (increase(rpc_hash_mismatch_total{region="eu-west"}[24h])) +# clamp_min floor: the cell-rank loader drops samples <= 0 (guard against +# absent-series garbage on latency benches), but 0 incidents is this bench's +# BEST and most common value; the epsilon keeps clean providers ranked. +rank_matrix_query: clamp_min(sum by (provider, chain) (increase(rpc_integrity_check_total{region="eu-west",result=~"error|disagree"}[24h])) + sum by (provider, chain) (increase(rpc_hash_mismatch_total{region="eu-west"}[24h])), 0.001) # ~82 integrity checks per provider per chain per day (2 checks per # 5-min rotation over 7 chains); single-chain providers are the floor. From e062fa7f7b6fd63d2fe8a4a156949117dc3ac857 Mon Sep 17 00:00:00 2001 From: Florent Tapponnier Date: Wed, 15 Jul 2026 01:19:38 +0200 Subject: [PATCH 11/12] gate benches 081-085 behind staging until 48h clean data --- src/lib/removed-benches.ts | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/lib/removed-benches.ts b/src/lib/removed-benches.ts index c78b306e..67a2f3c1 100644 --- a/src/lib/removed-benches.ts +++ b/src/lib/removed-benches.ts @@ -43,4 +43,11 @@ export const REMOVED_BENCH_SLUGS = new Set([ "xstocks-peg", "usdy-nav-basis", "portfolio-chain-coverage", + // bench vague 2 (081-085): validating on staging until harnesses have + // 48h of clean data on the VPS, then ship dev -> main + "ws-head-latency", + "oracle-freshness", + "rpc-reliability", + "indexer-latency", + "evm-block-builders", ]); From c61ac003e9692a3e15f9541e4daaee57610af13e Mon Sep 17 00:00:00 2001 From: Florent Tapponnier Date: Wed, 15 Jul 2026 01:19:56 +0200 Subject: [PATCH 12/12] drop shipped RWA slugs from prod gate (live on main since #1162/#1167) --- src/lib/removed-benches.ts | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/lib/removed-benches.ts b/src/lib/removed-benches.ts index 67a2f3c1..e5a4c60c 100644 --- a/src/lib/removed-benches.ts +++ b/src/lib/removed-benches.ts @@ -36,12 +36,7 @@ export const REMOVED_BENCH_SLUGS = new Set([ "indexing-freshness", "rpc-keyed-latency", "explorer-chain-coverage", - "solana-rpc", - "tokenized-stock-peg", - "tokenized-stock-weekend-drift", "tokenized-stock-arb-latency", - "xstocks-peg", - "usdy-nav-basis", "portfolio-chain-coverage", // bench vague 2 (081-085): validating on staging until harnesses have // 48h of clean data on the VPS, then ship dev -> main