diff --git a/rust/op-reth/crates/exex/src/lib.rs b/rust/op-reth/crates/exex/src/lib.rs index 5f728d535a8..4130353ddc6 100644 --- a/rust/op-reth/crates/exex/src/lib.rs +++ b/rust/op-reth/crates/exex/src/lib.rs @@ -936,13 +936,26 @@ mod tests { store.provider_ro().expect("provider ro").get_latest_block().expect("latest block") } - /// Anchors `store` at the test context's genesis and spawns an engine flushing every block, + /// The engine's persistence and backpressure thresholds. + #[derive(Clone, Copy, Debug)] + struct FlushCadence { + backpressure: u64, + persistence: u64, + } + + impl FlushCadence { + const EVERY_BLOCK: Self = Self { backpressure: 2, persistence: 1 }; + const EVERY_FIFTH_BLOCK: Self = Self { backpressure: 10, persistence: 5 }; + } + + /// Anchors `store` at the test context's genesis and spawns an engine flushing at `cadence`, /// executing with a real EVM over `chain_spec` since the context's EVM refuses every call. fn engine_over( ctx: &ExExContext, handle: &TestExExHandle, chain_spec: Arc, store: S, + cadence: FlushCadence, ) -> EngineHandle { init_storage_at(store.clone(), handle.genesis.num_hash()); let pruner = OpProofStoragePruner::new(store.clone(), ctx.components.provider.clone(), 20); @@ -951,8 +964,8 @@ mod tests { ctx.components.provider.clone(), store, pruner, - 1, - 2, + cadence.persistence, + cadence.backpressure, ) } @@ -978,7 +991,8 @@ mod tests { let chain_spec = merged_chain_spec(); let (ctx, mut handle) = test_exex_context_with_chain_spec(chain_spec.clone()).await.expect("exex test context"); - let engine = engine_over(&ctx, &handle, chain_spec, store.clone()); + let engine = + engine_over(&ctx, &handle, chain_spec, store.clone(), FlushCadence::EVERY_BLOCK); let exex = build_test_exex(ctx, store.clone()); let old = commit_three_blocks(&exex, &mut handle, &engine); // reth switches from blocks 2 and 3 to blocks 2' through 4' on block 1. @@ -1034,7 +1048,8 @@ mod tests { let chain_spec = merged_chain_spec(); let (ctx, mut handle) = test_exex_context_with_chain_spec(chain_spec.clone()).await.expect("exex test context"); - let engine = engine_over(&ctx, &handle, chain_spec, store.clone()); + let engine = + engine_over(&ctx, &handle, chain_spec, store.clone(), FlushCadence::EVERY_BLOCK); let exex = build_test_exex(ctx, store.clone()); let old = commit_three_blocks(&exex, &mut handle, &engine); let stored_tip = old[2].num_hash(); @@ -1055,6 +1070,138 @@ mod tests { handle.assert_events_empty(); } + /// How the ExEx hands a block near the tip to the engine, chosen through the verification + /// interval the way `--proofs-history.verification-interval` chooses it. + #[derive(Clone, Copy, Debug)] + enum ProofsPath { + /// Every block's verification is due, so the engine executes it. + Execute, + /// Every block carries reth's trie data and no block's verification is due. + Index, + } + + impl ProofsPath { + const fn verification_interval(self) -> u64 { + match self { + Self::Execute => 1, + Self::Index => 0, + } + } + } + + /// Blocks in a chain the engine saves under either cadence: the slower cadence's persistence + /// threshold, which the faster one's divides, so that no block waits for an idle flush. + const SAVED_CHAIN_LENGTH: u64 = FlushCadence::EVERY_FIFTH_BLOCK.persistence; + + /// Blocks in a chain below the slower cadence's persistence threshold. + const SHORT_CHAIN_LENGTH: u64 = 3; + + const _: () = assert!(SHORT_CHAIN_LENGTH < FlushCadence::EVERY_FIFTH_BLOCK.persistence); + + /// What the engine has done with a committed chain by the time the ExEx acknowledges it. + #[derive(Clone, Copy, Debug)] + enum CommittedChain { + /// Near reth's head but shorter than the engine's persistence threshold. + Buffered, + /// The real-time threshold behind reth's head, so the ExEx only sets the engine's sync + /// target, which reth cannot serve since it holds none of the blocks. + CatchUp, + /// Near reth's head and long enough for the engine to save it. + Saved, + } + + impl CommittedChain { + const fn length(self) -> u64 { + match self { + Self::Buffered | Self::CatchUp => SHORT_CHAIN_LENGTH, + Self::Saved => SAVED_CHAIN_LENGTH, + } + } + + /// The store's persisted tip once the engine has flushed what its cadence lets it flush. + const fn persisted_tip(self, genesis: NumHash, tip: NumHash) -> NumHash { + match self { + Self::Buffered | Self::CatchUp => genesis, + Self::Saved => tip, + } + } + } + + /// Moves reth's head to the real-time threshold past `tip` without giving reth any block. + fn set_head_far_ahead(ctx: &ExExContext, tip: u64) { + let head = Header { number: tip + REAL_TIME_BLOCKS_THRESHOLD, ..Default::default() }; + ctx.components + .provider + .canonical_in_memory_state() + .set_canonical_head(SealedHeader::seal_slow(head)); + } + + /// Flushes until nothing the engine could still save is unsaved, and returns the persisted + /// tip. A flush waits for the save in flight and the engine starts the next save only after + /// replying, so one flush per block the engine could have buffered drains them all. + fn settle(engine: &EngineHandle, store: &S) -> NumHash { + for _ in 0..=SAVED_CHAIN_LENGTH { + engine.flush(); + } + latest_block(store) + } + + async fn run_finished_height_case( + chain: CommittedChain, + cadence: FlushCadence, + path: ProofsPath, + ) { + let store = InMemoryProofsStorage::new(); + let chain_spec = merged_chain_spec(); + let (ctx, mut handle) = + test_exex_context_with_chain_spec(chain_spec.clone()).await.expect("exex test context"); + let engine = engine_over(&ctx, &handle, chain_spec, store.clone(), cadence); + let exex = OpProofsExEx::builder(ctx, store.clone()) + .with_verification_interval(path.verification_interval()) + .build(); + let genesis = handle.genesis.num_hash(); + let blocks = empty_chain(genesis, 1..=chain.length(), 0x10); + let tip = blocks.last().expect("a chain has a tip").num_hash(); + match chain { + CommittedChain::Buffered | CommittedChain::Saved => make_canonical( + &exex.ctx, + NewCanonicalChain::Commit { new: blocks.iter().map(executed).collect() }, + ), + CommittedChain::CatchUp => set_head_far_ahead(&exex.ctx, tip.number), + } + let notification = + ExExNotification::ChainCommitted { new: Arc::new(chain_of(blocks, true)) }; + + exex.handle_notification(notification, &engine).expect("handle chain commit"); + + // The contract today (ethereum-optimism/protocol-team#478): the ExEx acknowledges the + // notification's tip once the engine has accepted the chain, before the engine has + // persisted any of it, and even when it has only scheduled it for catch-up. A fix changes + // the acknowledged height, so it must change this assertion deliberately. + handle.assert_event_finished_height(tip).expect("one FinishedHeight event"); + handle.assert_events_empty(); + assert_eq!(settle(&engine, &store), chain.persisted_tip(genesis, tip), "persisted tip"); + } + + /// What `FinishedHeight` acknowledges for a committed chain the engine holds buffered and + /// unflushed, has only scheduled for catch-up, or has saved. A chain shorter than the + /// persistence threshold stays buffered only under the slower cadence, so that row has no + /// every-block cell. + #[rstest] + #[case::buffered_every_fifth_block(CommittedChain::Buffered, FlushCadence::EVERY_FIFTH_BLOCK)] + #[case::catch_up_every_block(CommittedChain::CatchUp, FlushCadence::EVERY_BLOCK)] + #[case::catch_up_every_fifth_block(CommittedChain::CatchUp, FlushCadence::EVERY_FIFTH_BLOCK)] + #[case::saved_every_block(CommittedChain::Saved, FlushCadence::EVERY_BLOCK)] + #[case::saved_every_fifth_block(CommittedChain::Saved, FlushCadence::EVERY_FIFTH_BLOCK)] + #[tokio::test] + async fn finished_height_is_the_notification_tip( + #[case] chain: CommittedChain, + #[case] cadence: FlushCadence, + #[values(ProofsPath::Execute, ProofsPath::Index)] path: ProofsPath, + ) { + run_finished_height_case(chain, cadence, path).await + } + /// The trie data of a block is used unless the chain lacks it or the block's verification is /// due; an interval of zero never makes it due. #[rstest] diff --git a/rust/op-reth/crates/trie/src/engine/runner.rs b/rust/op-reth/crates/trie/src/engine/runner.rs index 6fd850e90de..aabb8e54f04 100644 --- a/rust/op-reth/crates/trie/src/engine/runner.rs +++ b/rust/op-reth/crates/trie/src/engine/runner.rs @@ -43,6 +43,9 @@ where backpressure_threshold: u64, /// Current retry delay for the sync arm; grows on failure, resets on success. sync_backoff: Duration, + /// Earliest instant the next sync step may run: already past unless the last step failed. + /// Only a failed step moves it, so no other event can postpone a retry. + sync_retry_at: Instant, } impl Engine @@ -70,6 +73,7 @@ where persistence_threshold: DEFAULT_PERSISTENCE_THRESHOLD, backpressure_threshold: DEFAULT_BACKPRESSURE_THRESHOLD, sync_backoff: Duration::ZERO, + sync_retry_at: Instant::now(), } } @@ -153,8 +157,9 @@ where /// backpressure is active — callers naturally block in their bounded `send` until the /// in-flight save completes and memory is pruned. /// - **persistence**: signals a completed background save. - /// - **sync**: a zero-duration timer that fires immediately when the engine is behind its sync - /// target and not under backpressure; [`crossbeam_channel::never`] otherwise. + /// - **sync**: fires once `sync_retry_at` has passed, at once unless the last step failed, + /// while the engine is behind its sync target and not under backpressure; + /// [`crossbeam_channel::never`] otherwise. /// - **idle-flush**: fires after [`IDLE_FLUSH_INTERVAL`] when memory holds buffered blocks but /// the persistence threshold hasn't been reached and no save is in flight. Keeps a paused /// chain from leaving buffered blocks invisible to the proofs RPC indefinitely. @@ -171,11 +176,11 @@ where let incoming_rx: Receiver>> = if backpressure { crossbeam_channel::never() } else { self.incoming.clone() }; - // Fire when there is sync work to do; block indefinitely otherwise. After a failed - // sync step we delay by `sync_backoff` so a sustained failure (e.g. a stale sync target - // ahead of the provider's tip) doesn't spin the CPU and flood logs. + // Fire when there is sync work to do; block indefinitely otherwise. A failed step pushes + // `sync_retry_at` out so a sustained failure (e.g. a stale sync target ahead of the + // provider's tip) doesn't spin the CPU and flood logs. let sync_rx: Receiver = if self.needs_sync() && !backpressure { - crossbeam_channel::after(self.sync_backoff) + crossbeam_channel::at(self.sync_retry_at) } else { crossbeam_channel::never() }; @@ -198,6 +203,7 @@ where Ok(()) => self.sync_backoff = Duration::ZERO, Err(err) => { self.sync_backoff = next_sync_backoff(self.sync_backoff); + self.sync_retry_at = Instant::now() + self.sync_backoff; error!(target: "trie::engine::runner", ?err, backoff = ?self.sync_backoff, "Sync step failed"); } }, @@ -234,8 +240,93 @@ where #[cfg(test)] mod tests { - use super::{SYNC_BACKOFF_INITIAL, SYNC_BACKOFF_MAX, next_sync_backoff}; - use std::time::Duration; + use super::{Engine, SYNC_BACKOFF_INITIAL, SYNC_BACKOFF_MAX, next_sync_backoff}; + use crate::{ + InMemoryProofsStorage, OpProofStoragePruner, OpProofsInitProvider, OpProofsStore, + engine::{EngineAction, tasks::SyncToTask}, + }; + use alloy_consensus::Header; + use alloy_eips::BlockNumHash; + use crossbeam_channel::{Sender, bounded}; + use reth_chainspec::MAINNET; + use reth_db_common::init::init_genesis; + use reth_ethereum_primitives::Block; + use reth_evm_ethereum::EthEvmConfig; + use reth_primitives_traits::SealedHeader; + use reth_provider::{ + providers::BlockchainProvider, + test_utils::{MockNodeTypesWithDB, create_test_provider_factory_with_chain_spec}, + }; + use std::{ + ops::ControlFlow, + thread, + time::{Duration, Instant}, + }; + + type TestEngine = + Engine, InMemoryProofsStorage>; + + /// An engine whose store is anchored at genesis and whose sync target is block 1, which reth + /// reports as its head but does not hold, so that every sync step fails. + fn engine_behind_a_missing_block() -> (TestEngine, Sender>) { + let chain_spec = MAINNET.clone(); + let factory = create_test_provider_factory_with_chain_spec(chain_spec.clone()); + let genesis_hash = init_genesis(&factory).expect("init genesis"); + let provider = BlockchainProvider::new(factory).expect("blockchain provider"); + let missing_head = Header { number: 1, parent_hash: genesis_hash, ..Default::default() }; + provider + .canonical_in_memory_state() + .set_canonical_head(SealedHeader::seal_slow(missing_head)); + let storage = InMemoryProofsStorage::new(); + let init = storage.initialization_provider().expect("init provider"); + init.set_initial_state_anchor(BlockNumHash::new(0, genesis_hash)).expect("set anchor"); + init.commit_initial_state().expect("commit initial state"); + OpProofsInitProvider::commit(init).expect("commit"); + let pruner = OpProofStoragePruner::new(storage.clone(), provider.clone(), 1000); + let (actions, incoming) = bounded(10); + let mut engine = + Engine::new(EthEvmConfig::ethereum(chain_spec), provider, storage, pruner, incoming); + engine.state.sync_target = 1; + (engine, actions) + } + + fn step(engine: &mut TestEngine) { + assert_eq!(engine.process_next_event(), ControlFlow::Continue(())); + } + + /// Sync-target updates arriving faster than the retry delay neither postpone the retry + /// (ethereum-optimism/protocol-team#477) nor bring it forward. + #[test] + fn sync_retry_is_not_postponed_by_incoming_actions() { + let (mut engine, actions) = engine_behind_a_missing_block(); + // Three failed steps grow the retry delay to four times the initial one. + for _ in 0..3 { + step(&mut engine); + } + let third_failure = Instant::now(); + let retry_delay = engine.sync_backoff; + assert_eq!(retry_delay, 4 * SYNC_BACKOFF_INITIAL); + let action_gap = retry_delay / 8; + let patience = 10 * retry_delay; + + // Every step finds an action queued, which is what kept re-arming the retry timer. + let deadline = Instant::now() + patience; + while engine.sync_backoff == retry_delay { + assert!( + Instant::now() < deadline, + "no sync step within {patience:?} while actions kept arriving every {action_gap:?}" + ); + actions.send(EngineAction::SyncTo(SyncToTask { target: 1 })).expect("engine alive"); + step(&mut engine); + thread::sleep(action_gap); + } + + assert_eq!(engine.sync_backoff, 2 * retry_delay, "the step ran once and failed again"); + assert!( + third_failure.elapsed() >= retry_delay - action_gap, + "the step ran before its retry delay had passed" + ); + } #[test] fn zero_backoff_jumps_to_initial() { diff --git a/rust/op-reth/crates/trie/src/error.rs b/rust/op-reth/crates/trie/src/error.rs index db1d62a04c3..508356d4aa1 100644 --- a/rust/op-reth/crates/trie/src/error.rs +++ b/rust/op-reth/crates/trie/src/error.rs @@ -103,9 +103,6 @@ pub enum OpProofsStorageError { /// Error occurred while interacting with the database. #[error(transparent)] DatabaseError(DatabaseError), - /// Error occurred while trying to acquire a lock. - #[error("failed lock attempt")] - TryLockError, /// Error occurred during block execution. #[error(transparent)] ExecutionError(Arc), diff --git a/rust/op-reth/crates/trie/src/in_memory.rs b/rust/op-reth/crates/trie/src/in_memory.rs index 8d6242bbe1d..5480bc80429 100644 --- a/rust/op-reth/crates/trie/src/in_memory.rs +++ b/rust/op-reth/crates/trie/src/in_memory.rs @@ -25,7 +25,9 @@ use std::{collections::BTreeMap, sync::Arc}; /// In-memory implementation of [`OpProofsStore`] for testing purposes #[derive(Debug, Clone)] pub struct InMemoryProofsStorage { - /// Shared state across all instances + /// Shared state across all instances. A reader blocks while a writer holds it rather than + /// failing, since this store stands in for MDBX, whose read transactions never fail + /// against a writer. inner: Arc>, } @@ -164,7 +166,7 @@ impl InMemoryTrieCursor { return Ok(()); } - let storage = self.inner.try_read().ok_or(OpProofsStorageError::TryLockError)?; + let storage = self.inner.read(); // Common logic: collect latest values for each path let mut path_to_latest: std::collections::BTreeMap< @@ -323,7 +325,7 @@ impl InMemoryStorageCursor { return Ok(()); } - let storage = self.inner.try_read().ok_or(OpProofsStorageError::TryLockError)?; + let storage = self.inner.read(); // Collect latest values for each slot let mut slot_to_latest: std::collections::BTreeMap = @@ -597,7 +599,7 @@ impl OpProofsProviderRO for InMemoryProofsProvider { &self, max_block_number: u64, ) -> OpProofsStorageResult> { - let inner = self.inner.try_read().ok_or(OpProofsStorageError::TryLockError)?; + let inner = self.inner.read(); Ok(InMemoryAccountCursor::new(&inner, max_block_number)) } @@ -890,6 +892,8 @@ mod tests { use alloy_eips::NumHash; use alloy_primitives::U256; use reth_primitives_traits::Account; + use rstest::rstest; + use std::{sync::mpsc, thread}; #[test] fn test_in_memory_storage_basic_operations() -> Result<(), OpProofsStorageError> { @@ -940,4 +944,51 @@ mod tests { Ok(()) } + + fn read_account_trie(provider: &InMemoryProofsProvider) -> Result<(), DatabaseError> { + provider.account_trie_cursor(0)?.seek(Nibbles::default())?; + Ok(()) + } + + fn read_storage_trie(provider: &InMemoryProofsProvider) -> Result<(), DatabaseError> { + provider.storage_trie_cursor(B256::ZERO, 0)?.seek(Nibbles::default())?; + Ok(()) + } + + fn read_account_hashed(provider: &InMemoryProofsProvider) -> Result<(), DatabaseError> { + provider.account_hashed_cursor(0)?; + Ok(()) + } + + fn read_storage_hashed(provider: &InMemoryProofsProvider) -> Result<(), DatabaseError> { + provider.storage_hashed_cursor(B256::ZERO, 0)?.seek(B256::ZERO)?; + Ok(()) + } + + /// A read that starts while a writer holds the store waits for the writer and completes, + /// instead of failing. + #[rstest] + #[case::account_trie(read_account_trie)] + #[case::storage_trie(read_storage_trie)] + #[case::account_hashed(read_account_hashed)] + #[case::storage_hashed(read_storage_hashed)] + fn a_read_started_during_a_write_completes_once_the_write_is_done( + #[case] read: fn(&InMemoryProofsProvider) -> Result<(), DatabaseError>, + ) { + let storage = InMemoryProofsStorage::new(); + let provider = storage.provider_ro().expect("provider ro"); + let write_in_progress = storage.inner.write(); + let (reader_started_tx, reader_started_rx) = mpsc::channel(); + let reader = thread::spawn(move || { + reader_started_tx.send(()).expect("the test thread waits for the reader"); + read(&provider) + }); + reader_started_rx.recv().expect("the reader thread started"); + drop(write_in_progress); + + reader + .join() + .expect("the reader thread") + .expect("the read completes once the write is done"); + } } diff --git a/rust/op-reth/crates/trie/tests/live.rs b/rust/op-reth/crates/trie/tests/live.rs index 6c825a23063..ef8614ca6d9 100644 --- a/rust/op-reth/crates/trie/tests/live.rs +++ b/rust/op-reth/crates/trie/tests/live.rs @@ -77,6 +77,8 @@ macro_rules! over_backend { }; } +#[path = "live/catch_up_under_pruning.rs"] +mod catch_up_under_pruning; #[path = "live/contracts.rs"] mod contracts; #[path = "live/index_execute_parity.rs"] diff --git a/rust/op-reth/crates/trie/tests/live/catch_up_under_pruning.rs b/rust/op-reth/crates/trie/tests/live/catch_up_under_pruning.rs new file mode 100644 index 00000000000..54564ba30e3 --- /dev/null +++ b/rust/op-reth/crates/trie/tests/live/catch_up_under_pruning.rs @@ -0,0 +1,278 @@ +//! Catches a store up from genesis over a chain longer than the ExEx's real-time threshold while +//! reth prunes the bodies the ExEx has acknowledged. +//! +//! A notification more than `REAL_TIME_BLOCKS_THRESHOLD` blocks behind reth's head makes the ExEx +//! set the engine's sync target to the notification's tip and acknowledge that tip to reth with +//! `FinishedHeight` at once; the blocks reth commits meanwhile reach the engine along the index +//! or execute path, where they raise the sync target while the engine is still behind, and are +//! acknowledged in turn. reth's pruner takes the acknowledged height as the ceiling of what it +//! may remove, so the bodies the sync loop still has to execute can disappear under it. The trie +//! crate cannot see the ExEx, so the cases model its two decisions: the sync target it sets and +//! the height it acknowledges. The pruner model prunes right up to that height and after every +//! acknowledgement, which is more than reth's pruner ever removes, so a case that passes here +//! passes under reth. The stores are MDBX, the backend a node runs; a case's cost is on reth's +//! side, where the proofs backend makes no difference. + +use super::{ + BlockSpec, FlushCadence, TxSpec, chain_spec_with_address, create_mdbx_proofs_storage, + initialize_storage, public_key_to_address, + simulated_reth::{RethPersistence, SimulatedReth, fixed_signer, index_executed_block}, + store_view::{BlockView, ProofTarget}, +}; +use alloy_eips::BlockNumHash; +use alloy_primitives::{Address, U256}; +use reth_chain_state::ExecutedBlock; +use reth_ethereum_primitives::Block; +use reth_optimism_trie::{ + OpProofsProviderRO, OpProofsStore, ProofWindowRange, + engine::{EngineError, EngineHandle}, +}; +use reth_provider::{BlockReader, ProviderError, TransactionVariant}; +use rstest::rstest; +use serial_test::serial; +use std::{ + thread, + time::{Duration, Instant}, +}; + +/// Blocks reth holds before the first notification: more than the ExEx's real-time threshold, +/// and a multiple of both cadences' persistence thresholds so that the engine saves all of them +/// without an idle flush. +const CATCH_UP_LENGTH: u64 = 70; + +/// Blocks reth commits while the engine catches up, again a multiple of both thresholds. +const REAL_TIME_LENGTH: u64 = 5; + +const _: () = assert!( + CATCH_UP_LENGTH.is_multiple_of(FlushCadence::EVERY_FIFTH_BLOCK.persistence) && + REAL_TIME_LENGTH.is_multiple_of(FlushCadence::EVERY_FIFTH_BLOCK.persistence) +); + +/// Blocks per static file of transactions: reth's pruner removes whole files, so this is the +/// step in which bodies disappear. Each file costs reth's persistence a write, and that +/// persistence dominates a case, so the files are as large as the slower cadence's saves. +const BLOCKS_PER_BODY_FILE: u64 = FlushCadence::EVERY_FIFTH_BLOCK.persistence; + +/// How long a case lets the engine catch up before calling it stalled. +const CATCH_UP_TIMEOUT: Duration = Duration::from_secs(30); + +const POLL_INTERVAL: Duration = Duration::from_millis(5); + +/// What the ExEx acknowledges to reth after a notification. +#[derive(Clone, Copy, Debug)] +enum AckPolicy { + /// The notification's tip, whether or not the store holds it, as the ExEx does today. + NotificationTip, + /// The newest block the store has persisted, which a pruner following it never gets ahead of. + PersistedTip, +} + +impl AckPolicy { + const fn acknowledged(self, notified: u64, persisted: u64) -> u64 { + match self { + Self::NotificationTip => notified, + Self::PersistedTip => persisted, + } + } +} + +/// How the ExEx hands a block near reth's head to the engine. +#[derive(Clone, Copy, Debug)] +enum ProofsPath { + Execute, + Index, +} + +impl ProofsPath { + fn hand_over( + self, + engine: &EngineHandle, + block: &ExecutedBlock, + ) -> Result<(), EngineError> { + match self { + Self::Execute => engine.execute_block(block.recovered_block()), + Self::Index => index_executed_block(engine, block), + } + } +} + +/// reth's bodies pruner, following what the ExEx acknowledges with no distance below it. +struct BodiesPruner { + policy: AckPolicy, + /// The newest block whose body is gone. + pruned_through: Option, +} + +impl BodiesPruner { + const fn new(policy: AckPolicy) -> Self { + Self { policy, pruned_through: None } + } + + /// Prunes what the ExEx's acknowledgement of a notification at `notified`, with the store + /// persisted through `persisted`, lets the pruner remove. + fn follow(&mut self, reth: &SimulatedReth, notified: u64, persisted: u64) -> eyre::Result<()> { + let acknowledged = self.policy.acknowledged(notified, persisted); + self.pruned_through = self.pruned_through.max(reth.prune_bodies_through(acknowledged)?); + Ok(()) + } +} + +fn recipient(number: u64) -> Address { + Address::repeat_byte(0x40 + u8::try_from(number).expect("short chain")) +} + +/// Block `number` of the chain: `number` wei to the recipient only this block pays, so that every +/// body carries a transaction for pruning to remove. +fn transfer_block(number: u64) -> BlockSpec { + BlockSpec::new(vec![TxSpec::transfer(recipient(number), U256::from(number))]) +} + +/// Every address the chain touches, none of which has storage. +fn proof_targets(sender: Address) -> Vec { + [sender, Address::ZERO] + .into_iter() + .chain((1..=CATCH_UP_LENGTH + REAL_TIME_LENGTH).map(recipient)) + .map(|address| ProofTarget { address, slots: Vec::new() }) + .collect() +} + +fn persisted_tip(store: &S) -> eyre::Result { + Ok(store.provider_ro()?.get_latest_block()?) +} + +/// Waits for the store to persist `tip` while the pruner follows the acknowledgement of a +/// notification at `tip`, and fails naming where the store stalled. +fn wait_for_catch_up( + store: &S, + reth: &SimulatedReth, + pruner: &mut BodiesPruner, + tip: BlockNumHash, +) -> eyre::Result<()> { + let deadline = Instant::now() + CATCH_UP_TIMEOUT; + loop { + let persisted = persisted_tip(store)?; + pruner.follow(reth, tip.number, persisted.number)?; + if persisted == tip { + return Ok(()); + } + eyre::ensure!( + Instant::now() < deadline, + "the store stalled at block {} below the tip {} after reth pruned the bodies through \ + block {:?}", + persisted.number, + tip.number, + pruner.pruned_through + ); + thread::sleep(POLL_INTERVAL); + } +} + +fn run_catch_up_case( + persistence: RethPersistence, + cadence: FlushCadence, + path: ProofsPath, + policy: AckPolicy, +) -> eyre::Result<()> { + let key_pair = fixed_signer(); + let sender = public_key_to_address(key_pair.public_key()); + let mut reth = SimulatedReth::with_body_files_of( + chain_spec_with_address(sender), + key_pair, + persistence, + BLOCKS_PER_BODY_FILE, + )?; + let store = create_mdbx_proofs_storage(); + initialize_storage(store.clone(), &reth.factory, reth.genesis())?; + for number in 1..=CATCH_UP_LENGTH { + reth.commit(&transfer_block(number))?; + } + let engine = reth.spawn_engine(store.clone(), cadence.persistence, cadence.backpressure); + let mut pruner = BodiesPruner::new(policy); + + // The first notification is the whole chain, which the ExEx hands over as a sync target and + // acknowledges at once. + let notified = reth.head().number; + engine.sync_to(notified)?; + pruner.follow(&reth, notified, persisted_tip(&store)?.number)?; + // reth keeps committing while the engine catches up; each block reaches the engine along the + // path and is acknowledged in turn. + for number in CATCH_UP_LENGTH + 1..=CATCH_UP_LENGTH + REAL_TIME_LENGTH { + let block = reth.commit(&transfer_block(number))?; + path.hand_over(&engine, &block)?; + pruner.follow(&reth, number, persisted_tip(&store)?.number)?; + } + let tip = reth.head(); + + wait_for_catch_up(&store, &reth, &mut pruner, tip)?; + + eyre::ensure!(pruner.pruned_through.is_some(), "reth pruned no bodies"); + let window = store.provider_ro()?.get_proof_window()?; + assert_eq!(window, ProofWindowRange { earliest: reth.genesis(), latest: tip }, "proof window"); + let targets = proof_targets(sender); + for number in [1, CATCH_UP_LENGTH, tip.number] { + BlockView::read(&store, number, &targets)? + .verify_proofs(number, reth.state_root(number))?; + } + Ok(()) +} + +/// reth refuses to serve a block below its lowest remaining transactions file, so the sync loop +/// meets a pruned body as a `BlockExpired` error rather than as an empty body. +#[test] +#[serial] +fn pruned_blocks_read_back_as_expired() -> eyre::Result<()> { + let key_pair = fixed_signer(); + let sender = public_key_to_address(key_pair.public_key()); + let persistence = RethPersistence::threshold(7); + let mut reth = SimulatedReth::with_body_files_of( + chain_spec_with_address(sender), + key_pair, + persistence, + 1, + )?; + // The first persistence cycle writes blocks 1 through 3. + for number in 1..=persistence.persistence_threshold + 1 { + reth.commit(&transfer_block(number))?; + } + + assert_eq!(reth.prune_bodies_through(2)?, Some(2)); + + let err = reth + .provider + .recovered_block(1.into(), TransactionVariant::NoHash) + .expect_err("block 1 is below the lowest remaining file"); + assert!( + matches!(err, ProviderError::BlockExpired { requested: 1, earliest_available: 3 }), + "got {err:?}" + ); + assert!(reth.provider.recovered_block(3.into(), TransactionVariant::NoHash)?.is_some()); + Ok(()) +} + +/// With the store's persisted tip as the pruner's ceiling, pruning never outruns the sync loop +/// and the store reaches the tip under either reth threshold, cadence and path. +#[rstest] +#[serial] +fn catch_up_completes_when_reth_prunes_bodies_behind_the_persisted_tip( + #[values(RethPersistence::threshold(7), RethPersistence::threshold(50))] + persistence: RethPersistence, + #[values(FlushCadence::EVERY_BLOCK, FlushCadence::EVERY_FIFTH_BLOCK)] cadence: FlushCadence, + #[values(ProofsPath::Execute, ProofsPath::Index)] path: ProofsPath, +) -> eyre::Result<()> { + run_catch_up_case(persistence, cadence, path, AckPolicy::PersistedTip) +} + +/// With the notification's tip as the pruner's ceiling, which is what the ExEx acknowledges +/// today, reth prunes bodies the sync loop has not executed yet and the catch-up stalls +/// (ethereum-optimism/protocol-team#478). +#[rstest] +#[serial] +#[ignore = "fails on develop: the ExEx acknowledges the notification tip before the sync loop has executed the bodies below it, so reth prunes them and the catch-up stalls (ethereum-optimism/protocol-team#478)"] +fn catch_up_completes_when_reth_prunes_bodies_through_the_acknowledged_tip( + #[values(RethPersistence::threshold(7), RethPersistence::threshold(50))] + persistence: RethPersistence, + #[values(FlushCadence::EVERY_BLOCK, FlushCadence::EVERY_FIFTH_BLOCK)] cadence: FlushCadence, + #[values(ProofsPath::Execute, ProofsPath::Index)] path: ProofsPath, +) -> eyre::Result<()> { + run_catch_up_case(persistence, cadence, path, AckPolicy::NotificationTip) +} diff --git a/rust/op-reth/crates/trie/tests/live/simulated_reth.rs b/rust/op-reth/crates/trie/tests/live/simulated_reth.rs index 04aacd3e6b0..2c59b83b682 100644 --- a/rust/op-reth/crates/trie/tests/live/simulated_reth.rs +++ b/rust/op-reth/crates/trie/tests/live/simulated_reth.rs @@ -7,6 +7,7 @@ use alloy_eips::BlockNumHash; use alloy_primitives::B256; use reth_chain_state::{CanonicalInMemoryState, ExecutedBlock, NewCanonicalChain}; use reth_chainspec::ChainSpec; +use reth_db::test_utils::{create_test_rw_db_with_datadir, tempdir_path}; use reth_db_common::init::init_genesis; use reth_ethereum_primitives::Block; use reth_evm::{ConfigureEvm, execute::Executor}; @@ -18,8 +19,8 @@ use reth_optimism_trie::{ use reth_primitives_traits::RecoveredBlock; use reth_provider::{ HashedPostStateProvider, ProviderFactory, SaveBlocksInput, StateProviderFactory, - StateRootProvider, - providers::BlockchainProvider, + StateRootProvider, StaticFileProviderFactory, StaticFileSegment, + providers::{BlockchainProvider, RocksDBBuilder, StaticFileProviderBuilder}, test_utils::{MockNodeTypesWithDB, create_test_provider_factory_with_chain_spec}, }; use reth_revm::database::StateProviderDatabase; @@ -136,10 +137,42 @@ impl SimulatedReth { chain_spec: Arc, key_pair: Keypair, persistence: RethPersistence, + ) -> eyre::Result { + let factory = create_test_provider_factory_with_chain_spec(chain_spec.clone()); + Self::over_factory(chain_spec, key_pair, persistence, factory) + } + + /// reth keeping its transactions in static files of `blocks_per_file` blocks, the unit in + /// which [`Self::prune_bodies_through`] removes them. + pub(super) fn with_body_files_of( + chain_spec: Arc, + key_pair: Keypair, + persistence: RethPersistence, + blocks_per_file: u64, + ) -> eyre::Result { + let datadir = tempdir_path(); + let static_files = datadir.join("static_files"); + std::fs::create_dir_all(&static_files)?; + let factory = ProviderFactory::new( + create_test_rw_db_with_datadir(&datadir), + chain_spec.clone(), + StaticFileProviderBuilder::read_write(static_files) + .with_blocks_per_file_for_segment(StaticFileSegment::Transactions, blocks_per_file) + .build()?, + RocksDBBuilder::new(datadir.join("rocksdb")).with_default_tables().build()?, + reth_tasks::Runtime::test(), + )?; + Self::over_factory(chain_spec, key_pair, persistence, factory) + } + + fn over_factory( + chain_spec: Arc, + key_pair: Keypair, + persistence: RethPersistence, + factory: ProviderFactory, ) -> eyre::Result { let overlay_manager = OverlayManager::default(); - let factory = create_test_provider_factory_with_chain_spec(chain_spec.clone()) - .with_overlay_manager(overlay_manager.clone()); + let factory = factory.with_overlay_manager(overlay_manager.clone()); init_genesis(&factory)?; let provider = BlockchainProvider::new(factory.clone())?; let genesis = BlockNumHash::new(0, chain_spec.genesis_hash()); @@ -308,6 +341,22 @@ impl SimulatedReth { Ok(new) } + /// Deletes bodies reth has persisted at or below `number` the way reth's bodies pruner does + /// with `number` as its target: whole static files of transactions, never the newest one, so + /// a file reaching past `number` survives. reth then refuses every block below its lowest + /// remaining file as `BlockExpired`. Returns the newest block this call pruned. + /// + /// reth's pruner keeps at least `MINIMUM_DISTANCE` blocks below the height the proofs ExEx + /// acknowledged and only runs after a save; a caller passing the acknowledged height itself + /// models the worst case. + pub(super) fn prune_bodies_through(&self, number: u64) -> eyre::Result> { + let deleted = self + .factory + .static_file_provider() + .delete_segment_below_block(StaticFileSegment::Transactions, number + 1)?; + Ok(deleted.iter().filter_map(|header| header.block_range()).map(|range| range.end()).max()) + } + /// Writes every canonical block to the database without evicting any from memory, the state /// a `proofs init` run finds on a node that has caught up. pub(super) fn persist_all(&mut self) -> eyre::Result<()> {