From b311e4524e0e9a9d6a8b88e379455515333de72d Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Thu, 15 Oct 2026 05:19:44 -0700 Subject: [PATCH 1/3] op-reth: keep the proofs sync retry deadline across engine events The engine armed a fresh backoff timer on every iteration of its event loop, so once the retry delay exceeded the gap between incoming actions the failed sync step was never retried. The retry instant now lives on the engine and only moves when a sync step fails. A runner test grows the delay to 400 ms, feeds sync-target updates every 50 ms and asserts that the step runs again within four delays. Fixes ethereum-optimism/protocol-team#477. Co-Authored-By: Claude Fable 5.1 (cherry picked from commit f35c5ffeb62e259bd1e9bd0f6508c9ad930e8e82) (cherry picked from commit e19e4b68e90d3f63360ee8d75e43d704f82cf169) --- rust/op-reth/crates/trie/src/engine/runner.rs | 107 ++++++++++++++++-- 1 file changed, 99 insertions(+), 8 deletions(-) diff --git a/rust/op-reth/crates/trie/src/engine/runner.rs b/rust/op-reth/crates/trie/src/engine/runner.rs index 6fd850e90de..aabb8e54f04 100644 --- a/rust/op-reth/crates/trie/src/engine/runner.rs +++ b/rust/op-reth/crates/trie/src/engine/runner.rs @@ -43,6 +43,9 @@ where backpressure_threshold: u64, /// Current retry delay for the sync arm; grows on failure, resets on success. sync_backoff: Duration, + /// Earliest instant the next sync step may run: already past unless the last step failed. + /// Only a failed step moves it, so no other event can postpone a retry. + sync_retry_at: Instant, } impl Engine @@ -70,6 +73,7 @@ where persistence_threshold: DEFAULT_PERSISTENCE_THRESHOLD, backpressure_threshold: DEFAULT_BACKPRESSURE_THRESHOLD, sync_backoff: Duration::ZERO, + sync_retry_at: Instant::now(), } } @@ -153,8 +157,9 @@ where /// backpressure is active — callers naturally block in their bounded `send` until the /// in-flight save completes and memory is pruned. /// - **persistence**: signals a completed background save. - /// - **sync**: a zero-duration timer that fires immediately when the engine is behind its sync - /// target and not under backpressure; [`crossbeam_channel::never`] otherwise. + /// - **sync**: fires once `sync_retry_at` has passed, at once unless the last step failed, + /// while the engine is behind its sync target and not under backpressure; + /// [`crossbeam_channel::never`] otherwise. /// - **idle-flush**: fires after [`IDLE_FLUSH_INTERVAL`] when memory holds buffered blocks but /// the persistence threshold hasn't been reached and no save is in flight. Keeps a paused /// chain from leaving buffered blocks invisible to the proofs RPC indefinitely. @@ -171,11 +176,11 @@ where let incoming_rx: Receiver>> = if backpressure { crossbeam_channel::never() } else { self.incoming.clone() }; - // Fire when there is sync work to do; block indefinitely otherwise. After a failed - // sync step we delay by `sync_backoff` so a sustained failure (e.g. a stale sync target - // ahead of the provider's tip) doesn't spin the CPU and flood logs. + // Fire when there is sync work to do; block indefinitely otherwise. A failed step pushes + // `sync_retry_at` out so a sustained failure (e.g. a stale sync target ahead of the + // provider's tip) doesn't spin the CPU and flood logs. let sync_rx: Receiver = if self.needs_sync() && !backpressure { - crossbeam_channel::after(self.sync_backoff) + crossbeam_channel::at(self.sync_retry_at) } else { crossbeam_channel::never() }; @@ -198,6 +203,7 @@ where Ok(()) => self.sync_backoff = Duration::ZERO, Err(err) => { self.sync_backoff = next_sync_backoff(self.sync_backoff); + self.sync_retry_at = Instant::now() + self.sync_backoff; error!(target: "trie::engine::runner", ?err, backoff = ?self.sync_backoff, "Sync step failed"); } }, @@ -234,8 +240,93 @@ where #[cfg(test)] mod tests { - use super::{SYNC_BACKOFF_INITIAL, SYNC_BACKOFF_MAX, next_sync_backoff}; - use std::time::Duration; + use super::{Engine, SYNC_BACKOFF_INITIAL, SYNC_BACKOFF_MAX, next_sync_backoff}; + use crate::{ + InMemoryProofsStorage, OpProofStoragePruner, OpProofsInitProvider, OpProofsStore, + engine::{EngineAction, tasks::SyncToTask}, + }; + use alloy_consensus::Header; + use alloy_eips::BlockNumHash; + use crossbeam_channel::{Sender, bounded}; + use reth_chainspec::MAINNET; + use reth_db_common::init::init_genesis; + use reth_ethereum_primitives::Block; + use reth_evm_ethereum::EthEvmConfig; + use reth_primitives_traits::SealedHeader; + use reth_provider::{ + providers::BlockchainProvider, + test_utils::{MockNodeTypesWithDB, create_test_provider_factory_with_chain_spec}, + }; + use std::{ + ops::ControlFlow, + thread, + time::{Duration, Instant}, + }; + + type TestEngine = + Engine, InMemoryProofsStorage>; + + /// An engine whose store is anchored at genesis and whose sync target is block 1, which reth + /// reports as its head but does not hold, so that every sync step fails. + fn engine_behind_a_missing_block() -> (TestEngine, Sender>) { + let chain_spec = MAINNET.clone(); + let factory = create_test_provider_factory_with_chain_spec(chain_spec.clone()); + let genesis_hash = init_genesis(&factory).expect("init genesis"); + let provider = BlockchainProvider::new(factory).expect("blockchain provider"); + let missing_head = Header { number: 1, parent_hash: genesis_hash, ..Default::default() }; + provider + .canonical_in_memory_state() + .set_canonical_head(SealedHeader::seal_slow(missing_head)); + let storage = InMemoryProofsStorage::new(); + let init = storage.initialization_provider().expect("init provider"); + init.set_initial_state_anchor(BlockNumHash::new(0, genesis_hash)).expect("set anchor"); + init.commit_initial_state().expect("commit initial state"); + OpProofsInitProvider::commit(init).expect("commit"); + let pruner = OpProofStoragePruner::new(storage.clone(), provider.clone(), 1000); + let (actions, incoming) = bounded(10); + let mut engine = + Engine::new(EthEvmConfig::ethereum(chain_spec), provider, storage, pruner, incoming); + engine.state.sync_target = 1; + (engine, actions) + } + + fn step(engine: &mut TestEngine) { + assert_eq!(engine.process_next_event(), ControlFlow::Continue(())); + } + + /// Sync-target updates arriving faster than the retry delay neither postpone the retry + /// (ethereum-optimism/protocol-team#477) nor bring it forward. + #[test] + fn sync_retry_is_not_postponed_by_incoming_actions() { + let (mut engine, actions) = engine_behind_a_missing_block(); + // Three failed steps grow the retry delay to four times the initial one. + for _ in 0..3 { + step(&mut engine); + } + let third_failure = Instant::now(); + let retry_delay = engine.sync_backoff; + assert_eq!(retry_delay, 4 * SYNC_BACKOFF_INITIAL); + let action_gap = retry_delay / 8; + let patience = 10 * retry_delay; + + // Every step finds an action queued, which is what kept re-arming the retry timer. + let deadline = Instant::now() + patience; + while engine.sync_backoff == retry_delay { + assert!( + Instant::now() < deadline, + "no sync step within {patience:?} while actions kept arriving every {action_gap:?}" + ); + actions.send(EngineAction::SyncTo(SyncToTask { target: 1 })).expect("engine alive"); + step(&mut engine); + thread::sleep(action_gap); + } + + assert_eq!(engine.sync_backoff, 2 * retry_delay, "the step ran once and failed again"); + assert!( + third_failure.elapsed() >= retry_delay - action_gap, + "the step ran before its retry delay had passed" + ); + } #[test] fn zero_backoff_jumps_to_initial() { From f1832feef586a86a9623fd6826d10625206eb1ff Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Thu, 15 Oct 2026 05:19:45 -0700 Subject: [PATCH 2/3] op-reth: test the proofs catch-up under pruning and the FinishedHeight contract An ExEx unit test pins what FinishedHeight acknowledges for a committed chain the engine saved, scheduled for catch-up, or holds buffered: the notification tip in every case, which ethereum-optimism/protocol-team#478 tracks, so the fix has to change the assertion deliberately. A live test seeds a store at genesis, builds 70 blocks in reth, hands the engine the tip as a sync target the way the ExEx does, commits five more blocks along the index or execute path while the engine catches up, and has reth's bodies pruner follow the acknowledged height with no distance below it. SimulatedReth gains static files of a configurable size for transactions and a prune_bodies_through seam that deletes them as reth's bodies segment does; a small test pins that reth then answers BlockExpired for a pruned block. Pruning behind the store's persisted tip lets every cell reach the tip; pruning through the acknowledged tip stalls the catch-up, so those cells are ignored naming #478. Co-Authored-By: Claude Fable 5.1 (cherry picked from commit 25371807c1f5a0a30d89ebc4eeb6319e746fbfba) (cherry picked from commit e2b7a9577ed65d5ee579ed50a58f79f1654f8d6c) --- rust/op-reth/crates/exex/src/lib.rs | 157 +++++++++- rust/op-reth/crates/trie/tests/live.rs | 2 + .../trie/tests/live/catch_up_under_pruning.rs | 278 ++++++++++++++++++ .../crates/trie/tests/live/simulated_reth.rs | 57 +++- 4 files changed, 485 insertions(+), 9 deletions(-) create mode 100644 rust/op-reth/crates/trie/tests/live/catch_up_under_pruning.rs diff --git a/rust/op-reth/crates/exex/src/lib.rs b/rust/op-reth/crates/exex/src/lib.rs index 5f728d535a8..4130353ddc6 100644 --- a/rust/op-reth/crates/exex/src/lib.rs +++ b/rust/op-reth/crates/exex/src/lib.rs @@ -936,13 +936,26 @@ mod tests { store.provider_ro().expect("provider ro").get_latest_block().expect("latest block") } - /// Anchors `store` at the test context's genesis and spawns an engine flushing every block, + /// The engine's persistence and backpressure thresholds. + #[derive(Clone, Copy, Debug)] + struct FlushCadence { + backpressure: u64, + persistence: u64, + } + + impl FlushCadence { + const EVERY_BLOCK: Self = Self { backpressure: 2, persistence: 1 }; + const EVERY_FIFTH_BLOCK: Self = Self { backpressure: 10, persistence: 5 }; + } + + /// Anchors `store` at the test context's genesis and spawns an engine flushing at `cadence`, /// executing with a real EVM over `chain_spec` since the context's EVM refuses every call. fn engine_over( ctx: &ExExContext, handle: &TestExExHandle, chain_spec: Arc, store: S, + cadence: FlushCadence, ) -> EngineHandle { init_storage_at(store.clone(), handle.genesis.num_hash()); let pruner = OpProofStoragePruner::new(store.clone(), ctx.components.provider.clone(), 20); @@ -951,8 +964,8 @@ mod tests { ctx.components.provider.clone(), store, pruner, - 1, - 2, + cadence.persistence, + cadence.backpressure, ) } @@ -978,7 +991,8 @@ mod tests { let chain_spec = merged_chain_spec(); let (ctx, mut handle) = test_exex_context_with_chain_spec(chain_spec.clone()).await.expect("exex test context"); - let engine = engine_over(&ctx, &handle, chain_spec, store.clone()); + let engine = + engine_over(&ctx, &handle, chain_spec, store.clone(), FlushCadence::EVERY_BLOCK); let exex = build_test_exex(ctx, store.clone()); let old = commit_three_blocks(&exex, &mut handle, &engine); // reth switches from blocks 2 and 3 to blocks 2' through 4' on block 1. @@ -1034,7 +1048,8 @@ mod tests { let chain_spec = merged_chain_spec(); let (ctx, mut handle) = test_exex_context_with_chain_spec(chain_spec.clone()).await.expect("exex test context"); - let engine = engine_over(&ctx, &handle, chain_spec, store.clone()); + let engine = + engine_over(&ctx, &handle, chain_spec, store.clone(), FlushCadence::EVERY_BLOCK); let exex = build_test_exex(ctx, store.clone()); let old = commit_three_blocks(&exex, &mut handle, &engine); let stored_tip = old[2].num_hash(); @@ -1055,6 +1070,138 @@ mod tests { handle.assert_events_empty(); } + /// How the ExEx hands a block near the tip to the engine, chosen through the verification + /// interval the way `--proofs-history.verification-interval` chooses it. + #[derive(Clone, Copy, Debug)] + enum ProofsPath { + /// Every block's verification is due, so the engine executes it. + Execute, + /// Every block carries reth's trie data and no block's verification is due. + Index, + } + + impl ProofsPath { + const fn verification_interval(self) -> u64 { + match self { + Self::Execute => 1, + Self::Index => 0, + } + } + } + + /// Blocks in a chain the engine saves under either cadence: the slower cadence's persistence + /// threshold, which the faster one's divides, so that no block waits for an idle flush. + const SAVED_CHAIN_LENGTH: u64 = FlushCadence::EVERY_FIFTH_BLOCK.persistence; + + /// Blocks in a chain below the slower cadence's persistence threshold. + const SHORT_CHAIN_LENGTH: u64 = 3; + + const _: () = assert!(SHORT_CHAIN_LENGTH < FlushCadence::EVERY_FIFTH_BLOCK.persistence); + + /// What the engine has done with a committed chain by the time the ExEx acknowledges it. + #[derive(Clone, Copy, Debug)] + enum CommittedChain { + /// Near reth's head but shorter than the engine's persistence threshold. + Buffered, + /// The real-time threshold behind reth's head, so the ExEx only sets the engine's sync + /// target, which reth cannot serve since it holds none of the blocks. + CatchUp, + /// Near reth's head and long enough for the engine to save it. + Saved, + } + + impl CommittedChain { + const fn length(self) -> u64 { + match self { + Self::Buffered | Self::CatchUp => SHORT_CHAIN_LENGTH, + Self::Saved => SAVED_CHAIN_LENGTH, + } + } + + /// The store's persisted tip once the engine has flushed what its cadence lets it flush. + const fn persisted_tip(self, genesis: NumHash, tip: NumHash) -> NumHash { + match self { + Self::Buffered | Self::CatchUp => genesis, + Self::Saved => tip, + } + } + } + + /// Moves reth's head to the real-time threshold past `tip` without giving reth any block. + fn set_head_far_ahead(ctx: &ExExContext, tip: u64) { + let head = Header { number: tip + REAL_TIME_BLOCKS_THRESHOLD, ..Default::default() }; + ctx.components + .provider + .canonical_in_memory_state() + .set_canonical_head(SealedHeader::seal_slow(head)); + } + + /// Flushes until nothing the engine could still save is unsaved, and returns the persisted + /// tip. A flush waits for the save in flight and the engine starts the next save only after + /// replying, so one flush per block the engine could have buffered drains them all. + fn settle(engine: &EngineHandle, store: &S) -> NumHash { + for _ in 0..=SAVED_CHAIN_LENGTH { + engine.flush(); + } + latest_block(store) + } + + async fn run_finished_height_case( + chain: CommittedChain, + cadence: FlushCadence, + path: ProofsPath, + ) { + let store = InMemoryProofsStorage::new(); + let chain_spec = merged_chain_spec(); + let (ctx, mut handle) = + test_exex_context_with_chain_spec(chain_spec.clone()).await.expect("exex test context"); + let engine = engine_over(&ctx, &handle, chain_spec, store.clone(), cadence); + let exex = OpProofsExEx::builder(ctx, store.clone()) + .with_verification_interval(path.verification_interval()) + .build(); + let genesis = handle.genesis.num_hash(); + let blocks = empty_chain(genesis, 1..=chain.length(), 0x10); + let tip = blocks.last().expect("a chain has a tip").num_hash(); + match chain { + CommittedChain::Buffered | CommittedChain::Saved => make_canonical( + &exex.ctx, + NewCanonicalChain::Commit { new: blocks.iter().map(executed).collect() }, + ), + CommittedChain::CatchUp => set_head_far_ahead(&exex.ctx, tip.number), + } + let notification = + ExExNotification::ChainCommitted { new: Arc::new(chain_of(blocks, true)) }; + + exex.handle_notification(notification, &engine).expect("handle chain commit"); + + // The contract today (ethereum-optimism/protocol-team#478): the ExEx acknowledges the + // notification's tip once the engine has accepted the chain, before the engine has + // persisted any of it, and even when it has only scheduled it for catch-up. A fix changes + // the acknowledged height, so it must change this assertion deliberately. + handle.assert_event_finished_height(tip).expect("one FinishedHeight event"); + handle.assert_events_empty(); + assert_eq!(settle(&engine, &store), chain.persisted_tip(genesis, tip), "persisted tip"); + } + + /// What `FinishedHeight` acknowledges for a committed chain the engine holds buffered and + /// unflushed, has only scheduled for catch-up, or has saved. A chain shorter than the + /// persistence threshold stays buffered only under the slower cadence, so that row has no + /// every-block cell. + #[rstest] + #[case::buffered_every_fifth_block(CommittedChain::Buffered, FlushCadence::EVERY_FIFTH_BLOCK)] + #[case::catch_up_every_block(CommittedChain::CatchUp, FlushCadence::EVERY_BLOCK)] + #[case::catch_up_every_fifth_block(CommittedChain::CatchUp, FlushCadence::EVERY_FIFTH_BLOCK)] + #[case::saved_every_block(CommittedChain::Saved, FlushCadence::EVERY_BLOCK)] + #[case::saved_every_fifth_block(CommittedChain::Saved, FlushCadence::EVERY_FIFTH_BLOCK)] + #[tokio::test] + async fn finished_height_is_the_notification_tip( + #[case] chain: CommittedChain, + #[case] cadence: FlushCadence, + #[values(ProofsPath::Execute, ProofsPath::Index)] path: ProofsPath, + ) { + run_finished_height_case(chain, cadence, path).await + } + /// The trie data of a block is used unless the chain lacks it or the block's verification is /// due; an interval of zero never makes it due. #[rstest] diff --git a/rust/op-reth/crates/trie/tests/live.rs b/rust/op-reth/crates/trie/tests/live.rs index 6c825a23063..ef8614ca6d9 100644 --- a/rust/op-reth/crates/trie/tests/live.rs +++ b/rust/op-reth/crates/trie/tests/live.rs @@ -77,6 +77,8 @@ macro_rules! over_backend { }; } +#[path = "live/catch_up_under_pruning.rs"] +mod catch_up_under_pruning; #[path = "live/contracts.rs"] mod contracts; #[path = "live/index_execute_parity.rs"] diff --git a/rust/op-reth/crates/trie/tests/live/catch_up_under_pruning.rs b/rust/op-reth/crates/trie/tests/live/catch_up_under_pruning.rs new file mode 100644 index 00000000000..54564ba30e3 --- /dev/null +++ b/rust/op-reth/crates/trie/tests/live/catch_up_under_pruning.rs @@ -0,0 +1,278 @@ +//! Catches a store up from genesis over a chain longer than the ExEx's real-time threshold while +//! reth prunes the bodies the ExEx has acknowledged. +//! +//! A notification more than `REAL_TIME_BLOCKS_THRESHOLD` blocks behind reth's head makes the ExEx +//! set the engine's sync target to the notification's tip and acknowledge that tip to reth with +//! `FinishedHeight` at once; the blocks reth commits meanwhile reach the engine along the index +//! or execute path, where they raise the sync target while the engine is still behind, and are +//! acknowledged in turn. reth's pruner takes the acknowledged height as the ceiling of what it +//! may remove, so the bodies the sync loop still has to execute can disappear under it. The trie +//! crate cannot see the ExEx, so the cases model its two decisions: the sync target it sets and +//! the height it acknowledges. The pruner model prunes right up to that height and after every +//! acknowledgement, which is more than reth's pruner ever removes, so a case that passes here +//! passes under reth. The stores are MDBX, the backend a node runs; a case's cost is on reth's +//! side, where the proofs backend makes no difference. + +use super::{ + BlockSpec, FlushCadence, TxSpec, chain_spec_with_address, create_mdbx_proofs_storage, + initialize_storage, public_key_to_address, + simulated_reth::{RethPersistence, SimulatedReth, fixed_signer, index_executed_block}, + store_view::{BlockView, ProofTarget}, +}; +use alloy_eips::BlockNumHash; +use alloy_primitives::{Address, U256}; +use reth_chain_state::ExecutedBlock; +use reth_ethereum_primitives::Block; +use reth_optimism_trie::{ + OpProofsProviderRO, OpProofsStore, ProofWindowRange, + engine::{EngineError, EngineHandle}, +}; +use reth_provider::{BlockReader, ProviderError, TransactionVariant}; +use rstest::rstest; +use serial_test::serial; +use std::{ + thread, + time::{Duration, Instant}, +}; + +/// Blocks reth holds before the first notification: more than the ExEx's real-time threshold, +/// and a multiple of both cadences' persistence thresholds so that the engine saves all of them +/// without an idle flush. +const CATCH_UP_LENGTH: u64 = 70; + +/// Blocks reth commits while the engine catches up, again a multiple of both thresholds. +const REAL_TIME_LENGTH: u64 = 5; + +const _: () = assert!( + CATCH_UP_LENGTH.is_multiple_of(FlushCadence::EVERY_FIFTH_BLOCK.persistence) && + REAL_TIME_LENGTH.is_multiple_of(FlushCadence::EVERY_FIFTH_BLOCK.persistence) +); + +/// Blocks per static file of transactions: reth's pruner removes whole files, so this is the +/// step in which bodies disappear. Each file costs reth's persistence a write, and that +/// persistence dominates a case, so the files are as large as the slower cadence's saves. +const BLOCKS_PER_BODY_FILE: u64 = FlushCadence::EVERY_FIFTH_BLOCK.persistence; + +/// How long a case lets the engine catch up before calling it stalled. +const CATCH_UP_TIMEOUT: Duration = Duration::from_secs(30); + +const POLL_INTERVAL: Duration = Duration::from_millis(5); + +/// What the ExEx acknowledges to reth after a notification. +#[derive(Clone, Copy, Debug)] +enum AckPolicy { + /// The notification's tip, whether or not the store holds it, as the ExEx does today. + NotificationTip, + /// The newest block the store has persisted, which a pruner following it never gets ahead of. + PersistedTip, +} + +impl AckPolicy { + const fn acknowledged(self, notified: u64, persisted: u64) -> u64 { + match self { + Self::NotificationTip => notified, + Self::PersistedTip => persisted, + } + } +} + +/// How the ExEx hands a block near reth's head to the engine. +#[derive(Clone, Copy, Debug)] +enum ProofsPath { + Execute, + Index, +} + +impl ProofsPath { + fn hand_over( + self, + engine: &EngineHandle, + block: &ExecutedBlock, + ) -> Result<(), EngineError> { + match self { + Self::Execute => engine.execute_block(block.recovered_block()), + Self::Index => index_executed_block(engine, block), + } + } +} + +/// reth's bodies pruner, following what the ExEx acknowledges with no distance below it. +struct BodiesPruner { + policy: AckPolicy, + /// The newest block whose body is gone. + pruned_through: Option, +} + +impl BodiesPruner { + const fn new(policy: AckPolicy) -> Self { + Self { policy, pruned_through: None } + } + + /// Prunes what the ExEx's acknowledgement of a notification at `notified`, with the store + /// persisted through `persisted`, lets the pruner remove. + fn follow(&mut self, reth: &SimulatedReth, notified: u64, persisted: u64) -> eyre::Result<()> { + let acknowledged = self.policy.acknowledged(notified, persisted); + self.pruned_through = self.pruned_through.max(reth.prune_bodies_through(acknowledged)?); + Ok(()) + } +} + +fn recipient(number: u64) -> Address { + Address::repeat_byte(0x40 + u8::try_from(number).expect("short chain")) +} + +/// Block `number` of the chain: `number` wei to the recipient only this block pays, so that every +/// body carries a transaction for pruning to remove. +fn transfer_block(number: u64) -> BlockSpec { + BlockSpec::new(vec![TxSpec::transfer(recipient(number), U256::from(number))]) +} + +/// Every address the chain touches, none of which has storage. +fn proof_targets(sender: Address) -> Vec { + [sender, Address::ZERO] + .into_iter() + .chain((1..=CATCH_UP_LENGTH + REAL_TIME_LENGTH).map(recipient)) + .map(|address| ProofTarget { address, slots: Vec::new() }) + .collect() +} + +fn persisted_tip(store: &S) -> eyre::Result { + Ok(store.provider_ro()?.get_latest_block()?) +} + +/// Waits for the store to persist `tip` while the pruner follows the acknowledgement of a +/// notification at `tip`, and fails naming where the store stalled. +fn wait_for_catch_up( + store: &S, + reth: &SimulatedReth, + pruner: &mut BodiesPruner, + tip: BlockNumHash, +) -> eyre::Result<()> { + let deadline = Instant::now() + CATCH_UP_TIMEOUT; + loop { + let persisted = persisted_tip(store)?; + pruner.follow(reth, tip.number, persisted.number)?; + if persisted == tip { + return Ok(()); + } + eyre::ensure!( + Instant::now() < deadline, + "the store stalled at block {} below the tip {} after reth pruned the bodies through \ + block {:?}", + persisted.number, + tip.number, + pruner.pruned_through + ); + thread::sleep(POLL_INTERVAL); + } +} + +fn run_catch_up_case( + persistence: RethPersistence, + cadence: FlushCadence, + path: ProofsPath, + policy: AckPolicy, +) -> eyre::Result<()> { + let key_pair = fixed_signer(); + let sender = public_key_to_address(key_pair.public_key()); + let mut reth = SimulatedReth::with_body_files_of( + chain_spec_with_address(sender), + key_pair, + persistence, + BLOCKS_PER_BODY_FILE, + )?; + let store = create_mdbx_proofs_storage(); + initialize_storage(store.clone(), &reth.factory, reth.genesis())?; + for number in 1..=CATCH_UP_LENGTH { + reth.commit(&transfer_block(number))?; + } + let engine = reth.spawn_engine(store.clone(), cadence.persistence, cadence.backpressure); + let mut pruner = BodiesPruner::new(policy); + + // The first notification is the whole chain, which the ExEx hands over as a sync target and + // acknowledges at once. + let notified = reth.head().number; + engine.sync_to(notified)?; + pruner.follow(&reth, notified, persisted_tip(&store)?.number)?; + // reth keeps committing while the engine catches up; each block reaches the engine along the + // path and is acknowledged in turn. + for number in CATCH_UP_LENGTH + 1..=CATCH_UP_LENGTH + REAL_TIME_LENGTH { + let block = reth.commit(&transfer_block(number))?; + path.hand_over(&engine, &block)?; + pruner.follow(&reth, number, persisted_tip(&store)?.number)?; + } + let tip = reth.head(); + + wait_for_catch_up(&store, &reth, &mut pruner, tip)?; + + eyre::ensure!(pruner.pruned_through.is_some(), "reth pruned no bodies"); + let window = store.provider_ro()?.get_proof_window()?; + assert_eq!(window, ProofWindowRange { earliest: reth.genesis(), latest: tip }, "proof window"); + let targets = proof_targets(sender); + for number in [1, CATCH_UP_LENGTH, tip.number] { + BlockView::read(&store, number, &targets)? + .verify_proofs(number, reth.state_root(number))?; + } + Ok(()) +} + +/// reth refuses to serve a block below its lowest remaining transactions file, so the sync loop +/// meets a pruned body as a `BlockExpired` error rather than as an empty body. +#[test] +#[serial] +fn pruned_blocks_read_back_as_expired() -> eyre::Result<()> { + let key_pair = fixed_signer(); + let sender = public_key_to_address(key_pair.public_key()); + let persistence = RethPersistence::threshold(7); + let mut reth = SimulatedReth::with_body_files_of( + chain_spec_with_address(sender), + key_pair, + persistence, + 1, + )?; + // The first persistence cycle writes blocks 1 through 3. + for number in 1..=persistence.persistence_threshold + 1 { + reth.commit(&transfer_block(number))?; + } + + assert_eq!(reth.prune_bodies_through(2)?, Some(2)); + + let err = reth + .provider + .recovered_block(1.into(), TransactionVariant::NoHash) + .expect_err("block 1 is below the lowest remaining file"); + assert!( + matches!(err, ProviderError::BlockExpired { requested: 1, earliest_available: 3 }), + "got {err:?}" + ); + assert!(reth.provider.recovered_block(3.into(), TransactionVariant::NoHash)?.is_some()); + Ok(()) +} + +/// With the store's persisted tip as the pruner's ceiling, pruning never outruns the sync loop +/// and the store reaches the tip under either reth threshold, cadence and path. +#[rstest] +#[serial] +fn catch_up_completes_when_reth_prunes_bodies_behind_the_persisted_tip( + #[values(RethPersistence::threshold(7), RethPersistence::threshold(50))] + persistence: RethPersistence, + #[values(FlushCadence::EVERY_BLOCK, FlushCadence::EVERY_FIFTH_BLOCK)] cadence: FlushCadence, + #[values(ProofsPath::Execute, ProofsPath::Index)] path: ProofsPath, +) -> eyre::Result<()> { + run_catch_up_case(persistence, cadence, path, AckPolicy::PersistedTip) +} + +/// With the notification's tip as the pruner's ceiling, which is what the ExEx acknowledges +/// today, reth prunes bodies the sync loop has not executed yet and the catch-up stalls +/// (ethereum-optimism/protocol-team#478). +#[rstest] +#[serial] +#[ignore = "fails on develop: the ExEx acknowledges the notification tip before the sync loop has executed the bodies below it, so reth prunes them and the catch-up stalls (ethereum-optimism/protocol-team#478)"] +fn catch_up_completes_when_reth_prunes_bodies_through_the_acknowledged_tip( + #[values(RethPersistence::threshold(7), RethPersistence::threshold(50))] + persistence: RethPersistence, + #[values(FlushCadence::EVERY_BLOCK, FlushCadence::EVERY_FIFTH_BLOCK)] cadence: FlushCadence, + #[values(ProofsPath::Execute, ProofsPath::Index)] path: ProofsPath, +) -> eyre::Result<()> { + run_catch_up_case(persistence, cadence, path, AckPolicy::NotificationTip) +} diff --git a/rust/op-reth/crates/trie/tests/live/simulated_reth.rs b/rust/op-reth/crates/trie/tests/live/simulated_reth.rs index 04aacd3e6b0..2c59b83b682 100644 --- a/rust/op-reth/crates/trie/tests/live/simulated_reth.rs +++ b/rust/op-reth/crates/trie/tests/live/simulated_reth.rs @@ -7,6 +7,7 @@ use alloy_eips::BlockNumHash; use alloy_primitives::B256; use reth_chain_state::{CanonicalInMemoryState, ExecutedBlock, NewCanonicalChain}; use reth_chainspec::ChainSpec; +use reth_db::test_utils::{create_test_rw_db_with_datadir, tempdir_path}; use reth_db_common::init::init_genesis; use reth_ethereum_primitives::Block; use reth_evm::{ConfigureEvm, execute::Executor}; @@ -18,8 +19,8 @@ use reth_optimism_trie::{ use reth_primitives_traits::RecoveredBlock; use reth_provider::{ HashedPostStateProvider, ProviderFactory, SaveBlocksInput, StateProviderFactory, - StateRootProvider, - providers::BlockchainProvider, + StateRootProvider, StaticFileProviderFactory, StaticFileSegment, + providers::{BlockchainProvider, RocksDBBuilder, StaticFileProviderBuilder}, test_utils::{MockNodeTypesWithDB, create_test_provider_factory_with_chain_spec}, }; use reth_revm::database::StateProviderDatabase; @@ -136,10 +137,42 @@ impl SimulatedReth { chain_spec: Arc, key_pair: Keypair, persistence: RethPersistence, + ) -> eyre::Result { + let factory = create_test_provider_factory_with_chain_spec(chain_spec.clone()); + Self::over_factory(chain_spec, key_pair, persistence, factory) + } + + /// reth keeping its transactions in static files of `blocks_per_file` blocks, the unit in + /// which [`Self::prune_bodies_through`] removes them. + pub(super) fn with_body_files_of( + chain_spec: Arc, + key_pair: Keypair, + persistence: RethPersistence, + blocks_per_file: u64, + ) -> eyre::Result { + let datadir = tempdir_path(); + let static_files = datadir.join("static_files"); + std::fs::create_dir_all(&static_files)?; + let factory = ProviderFactory::new( + create_test_rw_db_with_datadir(&datadir), + chain_spec.clone(), + StaticFileProviderBuilder::read_write(static_files) + .with_blocks_per_file_for_segment(StaticFileSegment::Transactions, blocks_per_file) + .build()?, + RocksDBBuilder::new(datadir.join("rocksdb")).with_default_tables().build()?, + reth_tasks::Runtime::test(), + )?; + Self::over_factory(chain_spec, key_pair, persistence, factory) + } + + fn over_factory( + chain_spec: Arc, + key_pair: Keypair, + persistence: RethPersistence, + factory: ProviderFactory, ) -> eyre::Result { let overlay_manager = OverlayManager::default(); - let factory = create_test_provider_factory_with_chain_spec(chain_spec.clone()) - .with_overlay_manager(overlay_manager.clone()); + let factory = factory.with_overlay_manager(overlay_manager.clone()); init_genesis(&factory)?; let provider = BlockchainProvider::new(factory.clone())?; let genesis = BlockNumHash::new(0, chain_spec.genesis_hash()); @@ -308,6 +341,22 @@ impl SimulatedReth { Ok(new) } + /// Deletes bodies reth has persisted at or below `number` the way reth's bodies pruner does + /// with `number` as its target: whole static files of transactions, never the newest one, so + /// a file reaching past `number` survives. reth then refuses every block below its lowest + /// remaining file as `BlockExpired`. Returns the newest block this call pruned. + /// + /// reth's pruner keeps at least `MINIMUM_DISTANCE` blocks below the height the proofs ExEx + /// acknowledged and only runs after a save; a caller passing the acknowledged height itself + /// models the worst case. + pub(super) fn prune_bodies_through(&self, number: u64) -> eyre::Result> { + let deleted = self + .factory + .static_file_provider() + .delete_segment_below_block(StaticFileSegment::Transactions, number + 1)?; + Ok(deleted.iter().filter_map(|header| header.block_range()).map(|range| range.end()).max()) + } + /// Writes every canonical block to the database without evicting any from memory, the state /// a `proofs init` run finds on a node that has caught up. pub(super) fn persist_all(&mut self) -> eyre::Result<()> { From 3e265c366d547cb8ab1b0f8722d28a27ee7ebbd7 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Thu, 15 Oct 2026 14:11:21 -0700 Subject: [PATCH 3/3] op-reth: let in-memory proofs store readers wait for a writer The in-memory proofs store populated its cursors under `try_read`, which fails while a writer holds the lock. The engine thread reads the store through the hashed-account and trie cursors when it executes a block, and the persistence thread writes it under `inner.write()` when it saves the block before, so under the every-block cadence with the execute path the two collided and `execute_block` failed with `failed lock attempt`; when the trie cursor took the hit, reth's trie walker unwrapped the error and the engine thread died. That made `finished_height_is_the_notification_tip::case_4_saved_every_block::path_1_ProofsPath__Execute` flaky. MDBX never fails a reader against a writer, so the cursors now take a blocking `read()` like every other read in the store, and the `TryLockError` variant nothing produces any more is gone. The `try_read` dates from a tokio `RwLock` the synchronous cursor methods could not await and survived the switch to parking_lot. Co-Authored-By: Claude Fable 5.1 (cherry picked from commit 9ba9e6c67a7d3f7fa77701734df8440469366b53) (cherry picked from commit 56c9d40fd614f1485a75e5a15274e19b8fd29af0) --- rust/op-reth/crates/trie/src/error.rs | 3 -- rust/op-reth/crates/trie/src/in_memory.rs | 59 +++++++++++++++++++++-- 2 files changed, 55 insertions(+), 7 deletions(-) diff --git a/rust/op-reth/crates/trie/src/error.rs b/rust/op-reth/crates/trie/src/error.rs index db1d62a04c3..508356d4aa1 100644 --- a/rust/op-reth/crates/trie/src/error.rs +++ b/rust/op-reth/crates/trie/src/error.rs @@ -103,9 +103,6 @@ pub enum OpProofsStorageError { /// Error occurred while interacting with the database. #[error(transparent)] DatabaseError(DatabaseError), - /// Error occurred while trying to acquire a lock. - #[error("failed lock attempt")] - TryLockError, /// Error occurred during block execution. #[error(transparent)] ExecutionError(Arc), diff --git a/rust/op-reth/crates/trie/src/in_memory.rs b/rust/op-reth/crates/trie/src/in_memory.rs index 8d6242bbe1d..5480bc80429 100644 --- a/rust/op-reth/crates/trie/src/in_memory.rs +++ b/rust/op-reth/crates/trie/src/in_memory.rs @@ -25,7 +25,9 @@ use std::{collections::BTreeMap, sync::Arc}; /// In-memory implementation of [`OpProofsStore`] for testing purposes #[derive(Debug, Clone)] pub struct InMemoryProofsStorage { - /// Shared state across all instances + /// Shared state across all instances. A reader blocks while a writer holds it rather than + /// failing, since this store stands in for MDBX, whose read transactions never fail + /// against a writer. inner: Arc>, } @@ -164,7 +166,7 @@ impl InMemoryTrieCursor { return Ok(()); } - let storage = self.inner.try_read().ok_or(OpProofsStorageError::TryLockError)?; + let storage = self.inner.read(); // Common logic: collect latest values for each path let mut path_to_latest: std::collections::BTreeMap< @@ -323,7 +325,7 @@ impl InMemoryStorageCursor { return Ok(()); } - let storage = self.inner.try_read().ok_or(OpProofsStorageError::TryLockError)?; + let storage = self.inner.read(); // Collect latest values for each slot let mut slot_to_latest: std::collections::BTreeMap = @@ -597,7 +599,7 @@ impl OpProofsProviderRO for InMemoryProofsProvider { &self, max_block_number: u64, ) -> OpProofsStorageResult> { - let inner = self.inner.try_read().ok_or(OpProofsStorageError::TryLockError)?; + let inner = self.inner.read(); Ok(InMemoryAccountCursor::new(&inner, max_block_number)) } @@ -890,6 +892,8 @@ mod tests { use alloy_eips::NumHash; use alloy_primitives::U256; use reth_primitives_traits::Account; + use rstest::rstest; + use std::{sync::mpsc, thread}; #[test] fn test_in_memory_storage_basic_operations() -> Result<(), OpProofsStorageError> { @@ -940,4 +944,51 @@ mod tests { Ok(()) } + + fn read_account_trie(provider: &InMemoryProofsProvider) -> Result<(), DatabaseError> { + provider.account_trie_cursor(0)?.seek(Nibbles::default())?; + Ok(()) + } + + fn read_storage_trie(provider: &InMemoryProofsProvider) -> Result<(), DatabaseError> { + provider.storage_trie_cursor(B256::ZERO, 0)?.seek(Nibbles::default())?; + Ok(()) + } + + fn read_account_hashed(provider: &InMemoryProofsProvider) -> Result<(), DatabaseError> { + provider.account_hashed_cursor(0)?; + Ok(()) + } + + fn read_storage_hashed(provider: &InMemoryProofsProvider) -> Result<(), DatabaseError> { + provider.storage_hashed_cursor(B256::ZERO, 0)?.seek(B256::ZERO)?; + Ok(()) + } + + /// A read that starts while a writer holds the store waits for the writer and completes, + /// instead of failing. + #[rstest] + #[case::account_trie(read_account_trie)] + #[case::storage_trie(read_storage_trie)] + #[case::account_hashed(read_account_hashed)] + #[case::storage_hashed(read_storage_hashed)] + fn a_read_started_during_a_write_completes_once_the_write_is_done( + #[case] read: fn(&InMemoryProofsProvider) -> Result<(), DatabaseError>, + ) { + let storage = InMemoryProofsStorage::new(); + let provider = storage.provider_ro().expect("provider ro"); + let write_in_progress = storage.inner.write(); + let (reader_started_tx, reader_started_rx) = mpsc::channel(); + let reader = thread::spawn(move || { + reader_started_tx.send(()).expect("the test thread waits for the reader"); + read(&provider) + }); + reader_started_rx.recv().expect("the reader thread started"); + drop(write_in_progress); + + reader + .join() + .expect("the reader thread") + .expect("the read completes once the write is done"); + } }