From 760feca0fb9ed98a65b5128b2b5fe945a4dd56c5 Mon Sep 17 00:00:00 2001 From: Florent Tapponnier Date: Sun, 26 Jul 2026 08:06:04 +0200 Subject: [PATCH 1/3] gas bench: dual buffer (primary + lag2) so same prediction can be graded at block+0 and block+2 --- harnesses/gas-estimation/cmd/script/buffer.go | 59 +++++++++++++++---- 1 file changed, 48 insertions(+), 11 deletions(-) diff --git a/harnesses/gas-estimation/cmd/script/buffer.go b/harnesses/gas-estimation/cmd/script/buffer.go index 0a72ed8d..8e9a8696 100644 --- a/harnesses/gas-estimation/cmd/script/buffer.go +++ b/harnesses/gas-estimation/cmd/script/buffer.go @@ -26,32 +26,44 @@ type Prediction struct { } // Buffer stores pending predictions keyed by the block number they -// target. When a block is mined, the realizer fetches the matching -// slice, computes errors, then deletes the entry. +// target. Two parallel maps back it: `pending` is drained by the +// primary grade (join at target block ±1) and `pendingLag2` is +// drained two blocks later so we can grade the same prediction +// against a block +2 further out — the red-team-flagged fix for +// the "next-block winner is whoever scraped the mempool 100 ms +// before us" latency-race bias. Every Add() writes into both maps +// so lag=0 and lag=2 see identical prediction populations. // // Sized so that a multi-minute realizer outage doesn't lose data // silently — entries older than `pendingTTLBlocks` are evicted on // every join. type Buffer struct { - mu sync.Mutex - pending map[uint64][]Prediction + mu sync.Mutex + pending map[uint64][]Prediction + pendingLag2 map[uint64][]Prediction } func NewBuffer() *Buffer { - return &Buffer{pending: make(map[uint64][]Prediction)} + return &Buffer{ + pending: make(map[uint64][]Prediction), + pendingLag2: make(map[uint64][]Prediction), + } } // Add appends a prediction for the given target block. Callers may // add multiple predictions for the same (block, oracle) — e.g. one // per tier — and the realizer will emit one error metric per entry. +// Prediction is inserted into both the primary map and the lag2 +// map so the two grade passes see the same population. func (b *Buffer) Add(targetBlock uint64, p Prediction) { b.mu.Lock() defer b.mu.Unlock() b.pending[targetBlock] = append(b.pending[targetBlock], p) + b.pendingLag2[targetBlock] = append(b.pendingLag2[targetBlock], p) } -// Take returns and removes all predictions targeting the given -// block. If none are waiting, returns nil. +// Take returns and removes all primary-side predictions targeting +// the given block. If none are waiting, returns nil. func (b *Buffer) Take(block uint64) []Prediction { b.mu.Lock() defer b.mu.Unlock() @@ -60,10 +72,23 @@ func (b *Buffer) Take(block uint64) []Prediction { return ps } +// TakeLag2 returns and removes all lag2-side predictions targeting +// the given block. The realizer calls this with (currentBlock-2), +// i.e. predictions that targeted a block 2 ahead of their capture +// evaluated against the block 2 further out. If none are waiting, +// returns nil. +func (b *Buffer) TakeLag2(block uint64) []Prediction { + b.mu.Lock() + defer b.mu.Unlock() + ps := b.pendingLag2[block] + delete(b.pendingLag2, block) + return ps +} + // EvictOlderThan removes any pending entries whose target block is // strictly less than `floor`. Called by the realizer after each -// successful join to keep the map bounded if predictions arrive for -// blocks the realizer never sees (network split, oracle clock skew). +// successful join to keep both maps bounded if predictions arrive +// for blocks the realizer never sees (network split, oracle skew). func (b *Buffer) EvictOlderThan(floor uint64) { b.mu.Lock() defer b.mu.Unlock() @@ -72,10 +97,22 @@ func (b *Buffer) EvictOlderThan(floor uint64) { delete(b.pending, k) } } + // Lag2 map needs a floor 2 blocks earlier so it retains the + // predictions the +2 join is still going to consume. + lag2Floor := floor + if lag2Floor >= 2 { + lag2Floor -= 2 + } + for k := range b.pendingLag2 { + if k < lag2Floor { + delete(b.pendingLag2, k) + } + } } -// Sizes returns a snapshot of pending count per oracle. Used by the -// metrics goroutine to surface buffer growth. +// Sizes returns a snapshot of pending count per oracle (primary +// map only — the lag2 map trails it by 2 blocks and would otherwise +// double-count for the /metrics buffer-growth gauge). func (b *Buffer) Sizes() map[Oracle]int { b.mu.Lock() defer b.mu.Unlock() From 2cebcc6adabe24b252a65086e8e0634a1688bb30 Mon Sep 17 00:00:00 2001 From: Florent Tapponnier Date: Sun, 26 Jul 2026 08:07:27 +0200 Subject: [PATCH 2/3] gas bench: emit over/under split + lag2 secondary grade histograms --- .../gas-estimation/cmd/script/metrics.go | 43 ++++++++++++ .../gas-estimation/cmd/script/realized.go | 68 +++++++++++++------ 2 files changed, 90 insertions(+), 21 deletions(-) diff --git a/harnesses/gas-estimation/cmd/script/metrics.go b/harnesses/gas-estimation/cmd/script/metrics.go index abbf4b9b..4b765f5e 100644 --- a/harnesses/gas-estimation/cmd/script/metrics.go +++ b/harnesses/gas-estimation/cmd/script/metrics.go @@ -124,6 +124,49 @@ var ( }, []string{"chain", "kind"}, ) + + // Over/under split: an inclusion-confidence oracle (over-predicts + // on purpose) and a percentile tracker (under-predicts on tail + // spikes) can post the same abs error but with opposite user + // consequences — over = overpay a few wei, under = tx stuck + // waiting for a spike to subside. Emitting the two branches as + // separate histograms lets the leaderboard rank them independently + // or compose them into an asymmetric loss (over-weight = 0.1, + // under-weight = 0.9 → pinball loss at τ=0.9) without ever + // hiding the split behind a single abs number. + gasErrorPriorityOverHist = promauto.NewHistogramVec( + prometheus.HistogramOpts{ + Name: "gas_error_priority_over_gwei_histogram", + Help: "Histogram of over-prediction gap (predicted − realized when predicted > realized, in gwei) per (oracle, tier, chain). Zero when the oracle under-predicts. Reads high for inclusion-confidence oracles by design (Etherscan, MetaMask 'high').", + Buckets: []float64{0.001, 0.005, 0.01, 0.05, 0.1, 0.25, 0.5, 1, 2, 5, 10, 25, 50, 100, 250}, + }, + []string{"oracle", "tier", "chain"}, + ) + + gasErrorPriorityUnderHist = promauto.NewHistogramVec( + prometheus.HistogramOpts{ + Name: "gas_error_priority_under_gwei_histogram", + Help: "Histogram of under-prediction gap (realized − predicted when predicted < realized, in gwei) per (oracle, tier, chain). Zero when the oracle over-predicts. Reads high for percentile trackers during spikes (PublicNode feeHistory, Owlracle) — that's when a wrong number actually costs the user their block.", + Buckets: []float64{0.001, 0.005, 0.01, 0.05, 0.1, 0.25, 0.5, 1, 2, 5, 10, 25, 50, 100, 250}, + }, + []string{"oracle", "tier", "chain"}, + ) + + // Lag2 grade: the primary histogram grades a prediction against + // the very next block, which rewards whichever oracle scraped the + // mempool 100 ms before we did (a latency race, not accuracy). + // This companion series grades the same prediction two blocks + // later — closer to what a wallet UX actually delivers (sign, + // broadcast, propagate). Same buckets so the two are directly + // comparable via `quantile_over_time`. + gasErrorPriorityLag2Hist = promauto.NewHistogramVec( + prometheus.HistogramOpts{ + Name: "gas_error_priority_lag2_gwei_histogram", + Help: "Histogram of |predicted − realized| in gwei, graded 2 blocks after the prediction's target (removes the very-next-block latency-race bias). Buckets identical to the primary histogram so they can be compared directly on the leaderboard.", + Buckets: []float64{0.001, 0.005, 0.01, 0.05, 0.1, 0.25, 0.5, 1, 2, 5, 10, 25, 50, 100, 250}, + }, + []string{"oracle", "tier", "chain"}, + ) ) // StartMetricsServer binds /metrics + /health on addr. Blocking call — diff --git a/harnesses/gas-estimation/cmd/script/realized.go b/harnesses/gas-estimation/cmd/script/realized.go index e618cfc6..920629c4 100644 --- a/harnesses/gas-estimation/cmd/script/realized.go +++ b/harnesses/gas-estimation/cmd/script/realized.go @@ -301,16 +301,6 @@ func processBlock(ctx context.Context, buf *Buffer, blockNum uint64, chain Chain gasRealizedPriority.WithLabelValues(string(TierP99), chain.Slug).Set(pcts.P99) } - // Join: pull every pending prediction for THIS block (exact - // match) plus the previous block (some oracles target a slightly - // different head than the realizer's RPC sees, ±1 tolerance - // covers it). - predictionsToCheck := buf.Take(blockNum) - predictionsToCheck = append(predictionsToCheck, buf.Take(blockNum-1)...) - if len(predictionsToCheck) == 0 { - return - } - // Every tier now grades against a real percentile of the mined // block. p75 and p99 used to be approximated as (p50+p90)/2 and // p90 respectively — replaced with real values so the leaderboard @@ -323,21 +313,57 @@ func processBlock(ctx context.Context, buf *Buffer, blockNum uint64, chain Chain TierP99: pcts.P99, } + // Primary join: pull every pending prediction for THIS block + // (exact match) plus the previous block (some oracles target a + // slightly different head than the realizer's RPC sees, ±1 + // tolerance covers it). + predictionsToCheck := buf.Take(blockNum) + predictionsToCheck = append(predictionsToCheck, buf.Take(blockNum-1)...) + now := time.Now() - for _, p := range predictionsToCheck { - ref, ok := realized[p.Tier] - if !ok || pcts.TxCount == 0 { - continue + if len(predictionsToCheck) > 0 && pcts.TxCount > 0 { + for _, p := range predictionsToCheck { + ref, ok := realized[p.Tier] + if !ok { + continue + } + errPriority := math.Abs(p.PriorityGwei - ref) + gasErrorPriorityGauge.WithLabelValues(string(p.Oracle), string(p.Tier), chain.Slug).Set(errPriority) + gasErrorPriorityHist.WithLabelValues(string(p.Oracle), string(p.Tier), chain.Slug).Observe(errPriority) + // Over/under split: only the side the oracle actually + // erred on gets a sample this cycle. Percentile trackers + // will build fat under histograms, inclusion-confidence + // oracles will build fat over histograms; the leaderboard + // can then rank on whichever side matches user intent. + if p.PriorityGwei > ref { + gasErrorPriorityOverHist.WithLabelValues(string(p.Oracle), string(p.Tier), chain.Slug).Observe(p.PriorityGwei - ref) + } else if p.PriorityGwei < ref { + gasErrorPriorityUnderHist.WithLabelValues(string(p.Oracle), string(p.Tier), chain.Slug).Observe(ref - p.PriorityGwei) + } + errBase := math.Abs(p.BaseGwei - baseGwei) + gasErrorBaseGauge.WithLabelValues(string(p.Oracle), chain.Slug).Set(errBase) + if !p.CapturedAt.IsZero() { + gasPredictionAge.WithLabelValues(string(p.Oracle), chain.Slug).Observe(now.Sub(p.CapturedAt).Seconds()) + } } - errPriority := math.Abs(p.PriorityGwei - ref) - gasErrorPriorityGauge.WithLabelValues(string(p.Oracle), string(p.Tier), chain.Slug).Set(errPriority) - gasErrorPriorityHist.WithLabelValues(string(p.Oracle), string(p.Tier), chain.Slug).Observe(errPriority) - errBase := math.Abs(p.BaseGwei - baseGwei) - gasErrorBaseGauge.WithLabelValues(string(p.Oracle), chain.Slug).Set(errBase) - if !p.CapturedAt.IsZero() { - gasPredictionAge.WithLabelValues(string(p.Oracle), chain.Slug).Observe(now.Sub(p.CapturedAt).Seconds()) + } + + // Lag2 join: at block N, grade predictions that targeted block + // N-2 against the current block. Same tier reference values. + // Emits into a separate histogram so the primary ranking is + // unaffected while the leaderboard can add a "lag=2" secondary + // column that removes the very-next-block latency-race bias. + if blockNum >= 2 && pcts.TxCount > 0 { + lag2Predictions := buf.TakeLag2(blockNum - 2) + for _, p := range lag2Predictions { + ref, ok := realized[p.Tier] + if !ok { + continue + } + gasErrorPriorityLag2Hist.WithLabelValues(string(p.Oracle), string(p.Tier), chain.Slug).Observe(math.Abs(p.PriorityGwei - ref)) } } + fmt.Printf("[realizer/%s] block=%d txs=%d base=%.3f p25/p50/p75/p90/p99=%.3f/%.3f/%.3f/%.3f/%.3f matched=%d\n", chain.Slug, blockNum, pcts.TxCount, baseGwei, pcts.P25, pcts.P50, pcts.P75, pcts.P90, pcts.P99, len(predictionsToCheck)) } From 9f36d543c3ad1ace2f07c74aac7c6a41bb736649 Mon Sep 17 00:00:00 2001 From: Florent Tapponnier Date: Sun, 26 Jul 2026 08:09:19 +0200 Subject: [PATCH 3/3] gas bench: document over/under split + lag2 grade in methodology --- benchmarks/gas-estimation.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/benchmarks/gas-estimation.yml b/benchmarks/gas-estimation.yml index e61ee54c..cce9adc0 100644 --- a/benchmarks/gas-estimation.yml +++ b/benchmarks/gas-estimation.yml @@ -93,6 +93,8 @@ methodology: - "Polling result classification. `ok`, `http_err`, `parse_err`, `throttled` (HTTP 429 or oracle-specific quota message), `timeout`. Counter `gas_oracle_call_total{oracle, result, chain}` powers the reliability column." - "Ranking metric. The table ranks on the p99 of the absolute gap over 24h, not the p50. At current fee levels the typical-minute gaps are fractions of a micro-gwei apart across oracles (ranking noise); the p99 captures behaviour during the volatile minutes where a wrong prediction either overpays or misses the block. The typical p50 and p90 gaps stay visible as secondary columns." - "Prediction age disclosure. `gas_prediction_age_seconds{oracle, chain}` records the wall-clock time between an oracle's poll returning and the realized block being graded. This surfaces the free-tier cadence asymmetry (Owlracle at 60s vs PublicNode at 12s vs Etherscan at 15s) so readers can see how much of an oracle's error is stale-sample rather than model quality; it is not part of the ranking, only a diagnostic column." + - "Over/under split. In parallel with the absolute-error histogram, the harness emits `gas_error_priority_over_gwei_histogram` (predicted > realized, cost = overpay) and `gas_error_priority_under_gwei_histogram` (predicted < realized, cost = transaction stuck). Same buckets. The two branches lets the leaderboard decompose asymmetric behaviour: inclusion-confidence oracles skew fat on the over side by design, percentile trackers skew fat on the under side during spikes." + - "Lag=2 secondary grade. `gas_error_priority_lag2_gwei_histogram` grades each prediction against the block 2 slots after its target, not the very-next block. Removes the latency-race bias where whoever scraped the mempool 100ms before us wins by construction; closer to the wallet UX (sign, broadcast, propagate) than block+1. Same buckets as the primary histogram so a leaderboard column can plot the two side by side." - "Covered rate. Share of time (24h) each oracle's posted p50-tier prediction was at or above the realized p50 priority fee, recorded as `ocb:gas_p50_covered:pct_24h` via a recording rule. Same-instant gauge comparison (current prediction vs latest realized block), a proxy pending a harness-side matched counter. High = errs on the side of inclusion; low = under-bids vs the block's realized median." - "Excluded by design. (a) BNB Chain (not EIP-1559). (b) L2 OP Stack chains (Optimism, Base, Arbitrum), priority fee ≈ 0 there because sequencer is centralised, the relevant cost is L1 data fee which is a different metric and belongs in a separate bench. (c) Rough median oracles that publish only a single number (no tier breakdown). (d) Solana, different fee model entirely (lamports per CU, MEV via Jito), separate bench needed." @@ -151,6 +153,9 @@ dimensions: # gas_pending_buffer_size{oracle, chain} gauge # gas_prediction_age_seconds{oracle, chain} histogram (poll-to-grade lag; discloses cadence asymmetry) # gas_realized_quorum_disagreement_total{chain, kind} counter (primary vs verify RPC baseFee mismatches) +# gas_error_priority_over_gwei_histogram{oracle,tier,chain} histogram (over-prediction branch of the split — overpay cost) +# gas_error_priority_under_gwei_histogram{oracle,tier,chain} histogram (under-prediction branch — tx-stuck risk) +# gas_error_priority_lag2_gwei_histogram{oracle,tier,chain} histogram (same as gwei_histogram but graded at target+2 blocks — removes next-block latency-race bias) # # Tier note: the leaderboard ranks oracles on their p50-tier # prediction error (standard speed, the tier wallets use by default).