From b8a450b82b538042aa6a6287428ecd35b7d2d5c2 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Thu, 15 Oct 2026 02:25:10 -0700 Subject: [PATCH 1/4] op-reth: test the proofs engine reorg and revert edges and check unwind targets Drive the proofs engine's reorg, unwind and revert tasks at the boundaries the model-based test may not reach, and add the two checks they found missing: an unwind now refuses a target at or below the window's earliest block, and one whose parent differs from the stored block below it, before touching anything. The engine buffer keeps the hashes of the blocks it persisted so a fork below the disk tip can be verified within a process lifetime. `EngineHandle::reorg` takes `ReorgBlock`s, each indexed from reth's trie data or executed through the EVM, and the ExEx picks per block with the rule its ChainCommitted path uses instead of failing a reorg without trie data. The in-memory store's `unwind_history` follows the trait contract: no-op above the latest block, an error at or below the earliest, no underflow. Co-Authored-By: Claude Fable 5.1 (cherry picked from commit 743644a6a35cfd2e5625df3f31156c23706d73a5) (cherry picked from commit 23f2d2dda1f5f8dabec479c7ee6fd8190848100e) --- rust/op-reth/crates/exex/Cargo.toml | 5 +- rust/op-reth/crates/exex/src/lib.rs | 304 ++++++- .../crates/trie/src/engine/buffer/state.rs | 74 +- rust/op-reth/crates/trie/src/engine/error.rs | 5 +- rust/op-reth/crates/trie/src/engine/handle.rs | 14 +- rust/op-reth/crates/trie/src/engine/mod.rs | 5 +- rust/op-reth/crates/trie/src/engine/state.rs | 84 +- .../crates/trie/src/engine/tasks/mod.rs | 1 + .../crates/trie/src/engine/tasks/reorg.rs | 84 +- rust/op-reth/crates/trie/src/in_memory.rs | 38 +- rust/op-reth/crates/trie/tests/lib.rs | 102 +++ rust/op-reth/crates/trie/tests/live.rs | 21 + .../trie/tests/live/index_execute_parity.rs | 21 +- .../trie/tests/live/reorg_revert_edges.rs | 789 ++++++++++++++++++ .../crates/trie/tests/live/simulated_reth.rs | 67 +- 15 files changed, 1485 insertions(+), 129 deletions(-) create mode 100644 rust/op-reth/crates/trie/tests/live/reorg_revert_edges.rs diff --git a/rust/op-reth/crates/exex/Cargo.toml b/rust/op-reth/crates/exex/Cargo.toml index f7435701a97..563d10a309a 100644 --- a/rust/op-reth/crates/exex/Cargo.toml +++ b/rust/op-reth/crates/exex/Cargo.toml @@ -25,7 +25,6 @@ reth-optimism-trie = { workspace = true, features = ["serde-bincode-compat"] } # alloy alloy-consensus.workspace = true -alloy-eips.workspace = true # misc eyre.workspace = true @@ -44,7 +43,11 @@ reth-primitives-traits.workspace = true reth-ethereum-primitives.workspace = true reth-evm-ethereum.workspace = true reth-exex-test-utils.workspace = true +reth-chain-state.workspace = true +reth-chainspec.workspace = true +rstest.workspace = true tempfile.workspace = true +alloy-eips.workspace = true [features] test-utils = [ diff --git a/rust/op-reth/crates/exex/src/lib.rs b/rust/op-reth/crates/exex/src/lib.rs index 0703cc89be9..8d48836f216 100644 --- a/rust/op-reth/crates/exex/src/lib.rs +++ b/rust/op-reth/crates/exex/src/lib.rs @@ -16,11 +16,14 @@ use reth_exex::{ExExContext, ExExEvent, ExExNotification}; use reth_node_api::{FullNodeComponents, NodePrimitives, NodeTypes}; use reth_optimism_trie::{ CanonicalInMemoryStateProvider, OpProofStoragePruner, OpProofsProviderRO, OpProofsProviderRw, - OpProofsStore, ProofWindowRange, engine::EngineHandle, proof::DatabaseStateRoot, + OpProofsStore, ProofWindowRange, + engine::{EngineHandle, ReorgBlock}, + proof::DatabaseStateRoot, }; use reth_provider::{BlockHashReader, BlockNumReader, HeaderProvider}; use reth_trie::{ - HashedPostState, HashedPostStateSorted, SortedTrieData, StateRoot, updates::TrieUpdatesSorted, + ComputedTrieData, HashedPostState, HashedPostStateSorted, LazyTrieData, SortedTrieData, + StateRoot, updates::TrieUpdatesSorted, }; use std::sync::Arc; use tracing::{debug, info, warn}; @@ -409,13 +412,8 @@ where // `Chain::blocks()` is a BTreeMap so iteration is already ordered oldest → newest. for (&block_number, block) in new.blocks() { - // Fast path: use pre-computed trie data only when verification is not due. - let should_verify = self.verification_interval > 0 && - block_number.is_multiple_of(self.verification_interval); - let precomputed = (!should_verify).then(|| new.trie_data_at(block_number)).flatten(); - - if let Some(d) = precomputed { - let SortedTrieData { hashed_state, trie_updates } = &d.get().sorted; + if let Some(trie_data) = self.precomputed_trie_data(&new, block_number) { + let SortedTrieData { hashed_state, trie_updates } = &trie_data.sorted; engine_handle.index_block( block.block_with_parent(), (**trie_updates).clone(), @@ -444,6 +442,26 @@ where Ok(()) } + /// reth's trie data for `block_number`, unless the block has none or its verification is due, + /// in which case the engine executes the block instead. + fn precomputed_trie_data<'chain>( + &self, + chain: &'chain Chain, + block_number: u64, + ) -> Option<&'chain ComputedTrieData> { + let verification_due = self.verification_interval > 0 && + block_number.is_multiple_of(self.verification_interval); + (!verification_due) + .then(|| chain.trie_data_at(block_number)) + .flatten() + .map(LazyTrieData::get) + } + + /// Switches the store to `new`: the engine unwinds to the fork block and takes the new + /// blocks in the way [`Self::handle_chain_committed`] would, indexing reth's trie data or + /// executing the block. reth notifies a reorg with both chains on one fork block; two + /// different fork blocks mean the notification is not a reorg the store can follow, so it + /// fails before the engine touches anything. fn handle_chain_reorged( &self, old: Arc>, @@ -466,33 +484,22 @@ where )); } - let mut block_updates: Vec<( - BlockWithParent, - Arc, - Arc, - )> = Vec::with_capacity(new.len()); - for block_number in new.blocks().keys() { - let block = new - .blocks() - .get(block_number) - .ok_or_else(|| eyre::eyre!("Missing block {} in new chain", block_number))?; - let trie_data = new - .trie_data_at(*block_number) - .ok_or_else(|| { - eyre::eyre!("Missing Trie data for block {} in new chain", block_number) - })? - .get(); - let trie_updates = &trie_data.sorted.trie_updates; - let hashed_state = &trie_data.sorted.hashed_state; - - block_updates.push(( - block.block_with_parent(), - trie_updates.clone(), - hashed_state.clone(), - )); - } - - engine_handle.reorg(block_updates)?; + let blocks = new + .blocks() + .iter() + .map(|(&block_number, block)| { + self.precomputed_trie_data(&new, block_number).map_or_else( + || ReorgBlock::Execute(Arc::clone(block)), + |trie_data| ReorgBlock::Index { + block: block.block_with_parent(), + hashed_state: trie_data.sorted.hashed_state.clone(), + trie_updates: trie_data.sorted.trie_updates.clone(), + }, + ) + }) + .collect(); + + engine_handle.reorg(blocks)?; Ok(()) } @@ -519,10 +526,15 @@ mod tests { use super::*; use alloy_consensus::{Header, private::alloy_primitives::B256}; use alloy_eips::{BlockNumHash, NumHash, eip1898::BlockWithParent}; + use reth_chain_state::{ExecutedBlock, NewCanonicalChain}; + use reth_chainspec::{ChainSpec, ChainSpecBuilder, MAINNET}; use reth_db::test_utils::tempdir_path; use reth_ethereum_primitives::{Block, Receipt}; + use reth_evm_ethereum::EthEvmConfig; use reth_execution_types::{Chain, ExecutionOutcome}; - use reth_exex_test_utils::{TestExExContext, TestExExHandle}; + use reth_exex_test_utils::{ + Adapter, TestExExContext, TestExExHandle, test_exex_context_with_chain_spec, + }; use reth_optimism_trie::{ BlockStateDiff, InMemoryProofsStorage, OpProofsInitProvider, OpProofsProviderRO, OpProofsProviderRw, OpProofsStore, db::MdbxProofsStorage, engine::EngineHandle, @@ -533,7 +545,8 @@ mod tests { ComputedTrieData, HashedPostState, HashedPostStateSorted, LazyTrieData, updates::TrieUpdatesSorted, }; - use std::{collections::BTreeMap, default::Default, sync::Arc}; + use rstest::rstest; + use std::{collections::BTreeMap, default::Default, ops::RangeInclusive, sync::Arc}; // ------------------------------------------------------------------------- // Helpers: deterministic blocks and deterministic Chain with precomputed updates @@ -838,6 +851,221 @@ mod tests { assert_eq!(latest, 12); } + /// Which store implementation a case runs over. + #[derive(Clone, Copy, Debug)] + enum Backend { + InMemory, + Mdbx, + } + + /// A mainnet-shaped chain past the merge from genesis, so that an empty block changes no + /// state when executed: no block reward, no withdrawals and no beacon root. + fn merged_chain_spec() -> Arc { + Arc::new( + ChainSpecBuilder::default() + .chain(MAINNET.chain) + .genesis(MAINNET.genesis.clone()) + .paris_activated() + .build(), + ) + } + + /// An empty block `number` on `parent`, with a hash only `tag` and `number` decide. Its + /// header carries the empty state root, which is what executing it on [`merged_chain_spec`] + /// over a store initialized without trie data computes. + fn empty_block(number: u64, parent: B256, tag: u8) -> RecoveredBlock { + let mut block: RecoveredBlock = Default::default(); + block.set_block_number(number); + block.set_parent_hash(parent); + block.set_hash(B256::new([[tag; 16], [number as u8; 16]].concat().try_into().unwrap())); + block + } + + /// Empty blocks `numbers` chained on `parent`, which reth's canonical in-memory state takes + /// in as `update` so that the engine finds their parents' state when it executes them. + fn canonical_empty_chain( + ctx: &ExExContext, + parent: BlockNumHash, + numbers: RangeInclusive, + tag: u8, + update: impl FnOnce(Vec) -> NewCanonicalChain, + ) -> Vec> { + let mut blocks: Vec> = Vec::new(); + let mut parent_hash = parent.hash; + for number in numbers { + let block = empty_block(number, parent_hash, tag); + parent_hash = block.hash(); + blocks.push(block); + } + let executed = blocks + .iter() + .map(|block| ExecutedBlock { + recovered_block: Arc::new(block.clone()), + ..Default::default() + }) + .collect(); + let tip = blocks.last().expect("a chain has a tip"); + let in_memory = ctx.components.provider.canonical_in_memory_state(); + in_memory.update_chain(update(executed)); + in_memory.set_canonical_head(SealedHeader::new(tip.header().clone(), tip.hash())); + blocks + } + + /// A chain of `blocks`, with empty trie data for each when `with_trie_data`. + fn chain_of( + blocks: Vec>, + with_trie_data: bool, + ) -> Chain { + let first_block = blocks.first().expect("a chain has a first block").number(); + let trie_data = blocks + .iter() + .filter(|_| with_trie_data) + .map(|block| (block.number(), LazyTrieData::ready(ComputedTrieData::default()))) + .collect(); + let execution_outcome: ExecutionOutcome = ExecutionOutcome { + bundle: Default::default(), + receipts: Vec::new(), + requests: Vec::new(), + first_block, + }; + Chain::new(blocks, execution_outcome, trie_data) + } + + fn latest_block(store: &S) -> NumHash { + store.provider_ro().expect("provider ro").get_latest_block().expect("latest block") + } + + /// Anchors `store` at the test context's genesis and spawns an engine flushing every block, + /// executing with a real EVM over `chain_spec` since the context's EVM refuses every call. + fn engine_over( + ctx: &ExExContext, + handle: &TestExExHandle, + chain_spec: Arc, + store: S, + ) -> EngineHandle { + let init = store.initialization_provider().expect("init"); + init.set_initial_state_anchor(handle.genesis.num_hash()).expect("anchor"); + init.commit_initial_state().expect("commit init"); + OpProofsInitProvider::commit(init).expect("commit"); + let pruner = OpProofStoragePruner::new(store.clone(), ctx.components.provider.clone(), 20); + EngineHandle::spawn_with_thresholds( + EthEvmConfig::ethereum(chain_spec), + ctx.components.provider.clone(), + store, + pruner, + 1, + 2, + ) + } + + /// Commits empty blocks 1 through 3 on genesis, with trie data, through the ExEx. + fn commit_three_blocks( + exex: &OpProofsExEx, + handle: &mut TestExExHandle, + engine: &EngineHandle, + ) -> Vec> { + let blocks = + canonical_empty_chain(&exex.ctx, handle.genesis.num_hash(), 1..=3, 0x10, |new| { + NewCanonicalChain::Commit { new } + }); + let notification = + ExExNotification::ChainCommitted { new: Arc::new(chain_of(blocks.clone(), true)) }; + exex.handle_notification(notification, engine).expect("handle chain commit"); + handle.assert_event_finished_height(blocks[2].num_hash()).expect("finished height"); + blocks + } + + async fn run_reorg_without_trie_data_case(store: S) { + let chain_spec = merged_chain_spec(); + let (ctx, mut handle) = + test_exex_context_with_chain_spec(chain_spec.clone()).await.expect("exex test context"); + let engine = engine_over(&ctx, &handle, chain_spec, store.clone()); + let exex = build_test_exex(ctx, store.clone()); + let old = commit_three_blocks(&exex, &mut handle, &engine); + // reth switches from blocks 2 and 3 to blocks 2' through 4' on block 1. + let old_from_fork = old[1..].to_vec(); + let new = canonical_empty_chain(&exex.ctx, old[0].num_hash(), 2..=4, 0x20, |new| { + NewCanonicalChain::Reorg { + new, + old: old_from_fork + .iter() + .map(|block| ExecutedBlock { + recovered_block: Arc::new(block.clone()), + ..Default::default() + }) + .collect(), + } + }); + let new_tip = new[2].num_hash(); + let notification = ExExNotification::ChainReorged { + old: Arc::new(chain_of(old_from_fork, true)), + new: Arc::new(chain_of(new, false)), + }; + + exex.handle_notification(notification, &engine) + .expect("a reorg without trie data executes the new chain"); + + engine.flush(); + assert_eq!(latest_block(&store), new_tip); + handle.assert_event_finished_height(new_tip).expect("finished height"); + } + + /// A reorg whose new chain carries no trie data executes the new blocks, as a commit without + /// trie data does, instead of failing. + #[rstest] + #[tokio::test] + async fn chain_reorged_without_trie_data_executes_the_new_chain( + #[values(Backend::InMemory, Backend::Mdbx)] backend: Backend, + ) { + match backend { + Backend::InMemory => { + run_reorg_without_trie_data_case(InMemoryProofsStorage::new()).await + } + Backend::Mdbx => { + let dir = tempdir_path(); + run_reorg_without_trie_data_case(Arc::new( + MdbxProofsStorage::new(dir.as_path()).expect("env"), + )) + .await + } + } + } + + /// A reorg whose old and new chains do not share a fork block fails before the engine + /// touches the store, and acknowledges no height. + #[tokio::test] + async fn chain_reorged_with_different_fork_blocks_fails_and_changes_nothing() { + let store = InMemoryProofsStorage::new(); + let chain_spec = merged_chain_spec(); + let (ctx, mut handle) = + test_exex_context_with_chain_spec(chain_spec.clone()).await.expect("exex test context"); + let engine = engine_over(&ctx, &handle, chain_spec, store.clone()); + let exex = build_test_exex(ctx, store.clone()); + let old = commit_three_blocks(&exex, &mut handle, &engine); + let stored_tip = old[2].num_hash(); + // The old chain forks at block 1, the new chain at block 2. + let mut new = Vec::new(); + let mut parent_hash = old[1].hash(); + for number in 3..=4 { + let block = empty_block(number, parent_hash, 0x20); + parent_hash = block.hash(); + new.push(block); + } + let notification = ExExNotification::ChainReorged { + old: Arc::new(chain_of(old[1..].to_vec(), true)), + new: Arc::new(chain_of(new, true)), + }; + + let err = exex + .handle_notification(notification, &engine) + .expect_err("the chains have different fork blocks"); + + assert!(err.to_string().starts_with("Fork blocks do not match"), "got {err}"); + engine.flush(); + assert_eq!(latest_block(&store), stored_tip); + handle.assert_events_empty(); + } + #[tokio::test] async fn handle_notification_chain_reorged_skips_beyond_stored_blocks() { // MDBX proofs storage diff --git a/rust/op-reth/crates/trie/src/engine/buffer/state.rs b/rust/op-reth/crates/trie/src/engine/buffer/state.rs index 06988618558..45b8b3c76bb 100644 --- a/rust/op-reth/crates/trie/src/engine/buffer/state.rs +++ b/rust/op-reth/crates/trie/src/engine/buffer/state.rs @@ -9,6 +9,10 @@ use alloy_primitives::{B256, map::HashMap}; use parking_lot::RwLock; use std::{collections::BTreeMap, sync::Arc}; +/// Persisted block hashes the buffer keeps. A reorg reaches at most reth's in-memory chain below +/// its head, far fewer blocks than this. +const RETAINED_PERSISTED_HASHES: usize = 1024; + /// Buffer for holding blocks waiting to be persisted. /// /// This acts as the in-memory "tip" of the chain for the trie calculator. @@ -18,6 +22,9 @@ struct TrieBuffer { blocks: RwLock>>, /// Mapping of block numbers to block hashes. numbers: RwLock>, + /// Hashes of the newest blocks that left the buffer through a save, so an unwind can check + /// its target against a block that is no longer buffered. + persisted_hashes: RwLock>, #[cfg(feature = "metrics")] metrics: BufferMetrics, } @@ -28,6 +35,7 @@ impl TrieBuffer { Self { blocks: RwLock::new(HashMap::default()), numbers: RwLock::new(BTreeMap::new()), + persisted_hashes: RwLock::new(BTreeMap::new()), #[cfg(feature = "metrics")] metrics: BufferMetrics::new_with_labels(&[] as &[(&str, &str)]), } @@ -55,18 +63,24 @@ impl TrieBuffer { self.blocks.read().len() } - /// Prunes blocks from the buffer that are strictly before the given block number. + /// Prunes blocks from the buffer that are strictly before the given block number, keeping + /// their hashes. fn prune(&self, number: u64) { let mut blocks = self.blocks.write(); let mut numbers = self.numbers.write(); + let mut persisted_hashes = self.persisted_hashes.write(); // Split off the entries we want to keep (number >= threshold). // The original map now contains only entries to prune (num < number). let to_keep = numbers.split_off(&number); - for hash in numbers.values() { + for (&pruned_number, hash) in numbers.iter() { blocks.remove(hash); + persisted_hashes.insert(pruned_number, *hash); } *numbers = to_keep; + while persisted_hashes.len() > RETAINED_PERSISTED_HASHES { + persisted_hashes.pop_first(); + } #[cfg(feature = "metrics")] self.metrics.buffer_size.set(blocks.len() as f64); @@ -85,10 +99,18 @@ impl TrieBuffer { for hash in to_remove.values() { blocks.remove(hash); } + self.persisted_hashes.write().split_off(&from); #[cfg(feature = "metrics")] self.metrics.buffer_size.set(blocks.len() as f64); } + + /// Hash of the block at `number` if it is buffered or left the buffer through a save. + fn hash_at(&self, number: u64) -> Option { + let numbers = self.numbers.read(); + let persisted_hashes = self.persisted_hashes.read(); + numbers.get(&number).or_else(|| persisted_hashes.get(&number)).copied() + } } /// Manager for the in-memory state of the live trie. @@ -129,11 +151,17 @@ impl TrieBufferState { self.inner.prune(number); } - /// Removes blocks starting from `from` (inclusive) through the tip. + /// Removes blocks starting from `from` (inclusive) through the tip, and forgets the hashes of + /// persisted blocks from `from` up. pub(in crate::engine) fn unwind(&self, from: u64) { self.inner.unwind(from); } + /// Hash of the block at `number` if it is buffered or was pruned from the buffer by a save. + pub(in crate::engine) fn hash_at(&self, number: u64) -> Option { + self.inner.hash_at(number) + } + /// Returns the highest buffered block as a [`NumHash`], or `None` if the buffer is empty. pub(in crate::engine) fn tip(&self) -> Option { let numbers = self.inner.numbers.read(); @@ -189,7 +217,7 @@ mod tests { use super::*; use crate::BlockStateDiff; use alloy_eips::NumHash; - use alloy_primitives::B256; + use alloy_primitives::{B256, U256}; /// Build a `(BlockWithParent, BlockStateDiff)` pair with predictable hashes. /// @@ -318,6 +346,44 @@ mod tests { assert_eq!(state.len(), 3); } + #[test] + fn hash_at_follows_blocks_through_prune_and_unwind() { + let state = TrieBufferState::new(); + for i in 1..=5u64 { + insert(&state, i, i as u8, (i - 1) as u8); + } + assert_eq!(state.hash_at(3), Some(B256::repeat_byte(0x03))); + assert_eq!(state.hash_at(6), None); + + state.prune(4); // 1, 2, 3 persisted; 4, 5 buffered + assert_eq!(state.hash_at(2), Some(B256::repeat_byte(0x02))); + assert_eq!(state.hash_at(5), Some(B256::repeat_byte(0x05))); + + state.unwind(3); // removes 3, 4, 5 whether persisted or buffered + assert_eq!(state.hash_at(2), Some(B256::repeat_byte(0x02))); + assert_eq!(state.hash_at(3), None); + assert_eq!(state.hash_at(5), None); + } + + #[test] + fn retained_persisted_hashes_drop_the_oldest_beyond_the_bound() { + let state = TrieBufferState::new(); + let extra = 5u64; + let highest = RETAINED_PERSISTED_HASHES as u64 + extra; + for i in 1..=highest { + state.insert( + BlockWithParent::new(B256::ZERO, NumHash::new(i, B256::from(U256::from(i)))), + BlockStateDiff::default(), + ); + } + + state.prune(highest + 1); + + assert_eq!(state.hash_at(extra), None); + assert_eq!(state.hash_at(extra + 1), Some(B256::from(U256::from(extra + 1)))); + assert_eq!(state.hash_at(highest), Some(B256::from(U256::from(highest)))); + } + #[test] fn blocks_ordered_returns_oldest_to_newest() { let state = TrieBufferState::new(); diff --git a/rust/op-reth/crates/trie/src/engine/error.rs b/rust/op-reth/crates/trie/src/engine/error.rs index fd9d6059483..5a92b995b78 100644 --- a/rust/op-reth/crates/trie/src/engine/error.rs +++ b/rust/op-reth/crates/trie/src/engine/error.rs @@ -34,14 +34,15 @@ pub enum EngineError { /// The collector engine thread terminated unexpectedly. #[error("Collector engine terminated unexpectedly")] EngineDied, - /// Block is at the correct number but its parent hash does not match the current tip. + /// A block's parent hash does not match the stored block below it: the tip for a new block, + /// the block below the target for an unwind or a reorg. #[error( "Parent hash mismatch at block {block_number}: expected {expected_parent_hash}, got {actual_parent_hash}" )] ParentHashMismatch { /// The block number where the mismatch occurred. block_number: u64, - /// The expected parent hash (current tip hash). + /// The hash of the stored block the block should build on. expected_parent_hash: B256, /// The actual parent hash from the block header. actual_parent_hash: B256, diff --git a/rust/op-reth/crates/trie/src/engine/handle.rs b/rust/op-reth/crates/trie/src/engine/handle.rs index 44ac2216655..2d2e3aa914c 100644 --- a/rust/op-reth/crates/trie/src/engine/handle.rs +++ b/rust/op-reth/crates/trie/src/engine/handle.rs @@ -6,7 +6,7 @@ use super::{ error::EngineError, runner::Engine, service_guard::ServiceGuard, - tasks::{ExecuteBlockTask, IndexBlockTask, ReorgTask, SyncToTask, UnwindTask}, + tasks::{ExecuteBlockTask, IndexBlockTask, ReorgBlock, ReorgTask, SyncToTask, UnwindTask}, }; use crate::{OpProofStoragePruner, OpProofsStore}; use alloy_eips::eip1898::BlockWithParent; @@ -135,12 +135,12 @@ impl EngineHandle }) } - /// Handle a chain reorg: unwind to the common ancestor then buffer new fork blocks. - pub fn reorg( - &self, - block_updates: Vec<(BlockWithParent, Arc, Arc)>, - ) -> Result<(), EngineError> { - self.send_and_recv(|reply| EngineAction::Reorg(ReorgTask { block_updates, reply })) + /// Handle a chain reorg: unwind to the common ancestor then buffer the new chain, indexing or + /// executing each block as its [`ReorgBlock`] says. Fails without unwinding when the first + /// block does not build on the stored block below it or when the unwind would reach the + /// window's earliest block. + pub fn reorg(&self, blocks: Vec>) -> Result<(), EngineError> { + self.send_and_recv(|reply| EngineAction::Reorg(ReorgTask { blocks, reply })) } /// Unwind indexed data back to `to` (first block number removed, inclusive). diff --git a/rust/op-reth/crates/trie/src/engine/mod.rs b/rust/op-reth/crates/trie/src/engine/mod.rs index a012edbd217..267a9cb21f6 100644 --- a/rust/op-reth/crates/trie/src/engine/mod.rs +++ b/rust/op-reth/crates/trie/src/engine/mod.rs @@ -11,6 +11,7 @@ mod buffer; pub mod persistence; mod tasks; +pub use tasks::ReorgBlock; mod canonical_state; pub use canonical_state::CanonicalInMemoryStateProvider; @@ -50,8 +51,8 @@ enum EngineAction { ExecuteBlock(tasks::ExecuteBlockTask), /// Index pre-computed trie updates for a block (no EVM execution). IndexBlock(tasks::IndexBlockTask), - /// Handle a reorg: unwind to the common ancestor then index the new chain. - Reorg(tasks::ReorgTask), + /// Handle a reorg: unwind to the common ancestor then index or execute the new chain. + Reorg(tasks::ReorgTask), /// Unwind indexed data back to a given block. Unwind(tasks::UnwindTask), /// Block the caller until any in-flight persistence completes. diff --git a/rust/op-reth/crates/trie/src/engine/state.rs b/rust/op-reth/crates/trie/src/engine/state.rs index 8ecdf0d61b3..8e96ad09a1a 100644 --- a/rust/op-reth/crates/trie/src/engine/state.rs +++ b/rust/op-reth/crates/trie/src/engine/state.rs @@ -8,7 +8,7 @@ use super::{ error::EngineError, persistence::{PersistenceHandle, error::PersistenceError}, }; -use crate::{OpProofStoragePruner, OpProofsProviderRO, OpProofsStore}; +use crate::{OpProofStoragePruner, OpProofsProviderRO, OpProofsStorageError, OpProofsStore}; use alloy_eips::{NumHash, eip1898::BlockWithParent}; use crossbeam_channel::{Receiver, RecvError, RecvTimeoutError, bounded}; use reth_evm::ConfigureEvm; @@ -210,11 +210,14 @@ where self.persistence.wait(&self.memory); } - /// Drain any in-flight save, unwind the persistence service to `to`, then - /// unwind the in-memory buffer to match. + /// Drain any in-flight save, check `to` against the proof window and the stored block below + /// it, unwind the persistence service to `to`, then unwind the in-memory buffer to match. pub(crate) fn unwind(&mut self, to: BlockWithParent) -> Result<(), EngineError> { #[cfg(feature = "metrics")] let start = Instant::now(); + // A save in flight can prune the window, so the checks run once it is done. + self.drain_persistence(); + self.check_unwind_target(to)?; self.persistence.unwind(to, &self.memory)?; self.memory.unwind(to.block.number); // Clamp `sync_target` to the post-unwind tip. Without this, a stale target left over @@ -226,6 +229,37 @@ where Ok(()) } + /// Fails an unwind that would remove the window's earliest block, or whose target does not + /// build on the stored block below it, before anything is unwound. + /// + /// Only the blocks this engine has buffered and the window's ends have a known hash: a block + /// persisted before a restart has none, so an unwind onto it goes unverified. + fn check_unwind_target(&self, to: BlockWithParent) -> Result<(), EngineError> { + let window = self.storage.provider_ro()?.get_proof_window()?; + if to.block.number <= window.earliest.number { + return Err(OpProofsStorageError::UnwindBeyondEarliest { + unwind_block_number: to.block.number, + earliest_block_number: window.earliest.number, + } + .into()); + } + let fork_number = to.block.number - 1; + let stored_fork_hash = self.memory.hash_at(fork_number).or_else(|| { + [window.latest, window.earliest] + .into_iter() + .find(|block| block.number == fork_number) + .map(|block| block.hash) + }); + match stored_fork_hash { + Some(stored) if stored != to.parent => Err(EngineError::ParentHashMismatch { + block_number: to.block.number, + expected_parent_hash: stored, + actual_parent_hash: to.parent, + }), + _ => Ok(()), + } + } + /// Advances `sync_target` to `block_number` if it is higher than the current target. pub(crate) const fn update_sync_target(&mut self, block_number: u64) { if block_number > self.sync_target { @@ -284,6 +318,8 @@ mod tests { EngineState::new(evm_config, blockchain_db, storage, pruner) } + /// An unwind target whose parent is unknown to the engine: the store holds only the anchor, + /// so nothing checks the parent of a block far above it. fn unwind_target(block_number: u64) -> BlockWithParent { BlockWithParent::new( B256::repeat_byte(0xAA), @@ -291,6 +327,11 @@ mod tests { ) } + /// An unwind target right above the anchor, whose parent the anchor's hash verifies. + fn unwind_target_above_anchor(parent: B256) -> BlockWithParent { + BlockWithParent::new(parent, NumHash::new(1, B256::repeat_byte(0x50))) + } + #[test] fn unwind_clamps_stale_sync_target_down_to_new_tip() { let mut state = make_engine_state(); @@ -323,7 +364,42 @@ mod tests { // `saturating_sub(1)` must not underflow when unwinding to block 1. let mut state = make_engine_state(); state.sync_target = 1_000_000; - state.unwind(unwind_target(1)).expect("unwind"); + state.unwind(unwind_target_above_anchor(B256::ZERO)).expect("unwind"); assert_eq!(state.sync_target, 0); } + + #[test] + fn unwind_at_the_anchor_fails_before_touching_anything() { + let mut state = make_engine_state(); + state.sync_target = 100; + let anchor = BlockWithParent::new(B256::ZERO, NumHash::new(0, B256::ZERO)); + let err = state.unwind(anchor).expect_err("the anchor is the window's earliest block"); + assert!( + matches!( + err, + EngineError::Storage(OpProofsStorageError::UnwindBeyondEarliest { + unwind_block_number: 0, + earliest_block_number: 0, + }) + ), + "got {err:?}" + ); + assert_eq!(state.sync_target, 100); + } + + #[test] + fn unwind_onto_a_parent_the_window_end_contradicts_fails() { + let mut state = make_engine_state(); + let err = state + .unwind(unwind_target_above_anchor(B256::repeat_byte(0xAA))) + .expect_err("the anchor is the stored block below the target"); + assert!( + matches!( + err, + EngineError::ParentHashMismatch { block_number: 1, expected_parent_hash, .. } + if expected_parent_hash == B256::ZERO + ), + "got {err:?}" + ); + } } diff --git a/rust/op-reth/crates/trie/src/engine/tasks/mod.rs b/rust/op-reth/crates/trie/src/engine/tasks/mod.rs index 8be7d8944dc..0feb1c75ed7 100644 --- a/rust/op-reth/crates/trie/src/engine/tasks/mod.rs +++ b/rust/op-reth/crates/trie/src/engine/tasks/mod.rs @@ -17,6 +17,7 @@ pub(super) use execute_block::{ExecuteBlockTask, run as execute_block}; #[cfg(any(test, feature = "test-utils"))] pub(super) use flush::FlushTask; pub(super) use index_block::IndexBlockTask; +pub use reorg::ReorgBlock; pub(super) use reorg::ReorgTask; pub(super) use sync_to::SyncToTask; pub(super) use unwind::UnwindTask; diff --git a/rust/op-reth/crates/trie/src/engine/tasks/reorg.rs b/rust/op-reth/crates/trie/src/engine/tasks/reorg.rs index d3cfae4d816..01ce1638e91 100644 --- a/rust/op-reth/crates/trie/src/engine/tasks/reorg.rs +++ b/rust/op-reth/crates/trie/src/engine/tasks/reorg.rs @@ -6,55 +6,80 @@ use crate::{ use alloy_eips::eip1898::BlockWithParent; use crossbeam_channel::Sender; use reth_evm::ConfigureEvm; -use reth_primitives_traits::BlockTy; +use reth_primitives_traits::{NodePrimitives, RecoveredBlock}; use reth_provider::{BlockHashReader, BlockReader, DatabaseProviderFactory, StageCheckpointReader}; use reth_trie_common::{HashedPostStateSorted, updates::TrieUpdatesSorted}; use std::{sync::Arc, time::Instant}; use tracing::{debug, info}; -pub(crate) struct ReorgTask { - pub(crate) block_updates: - Vec<(BlockWithParent, Arc, Arc)>, +/// A block of the chain a reorg switches to, with the way the engine takes it in. +#[derive(Debug)] +pub enum ReorgBlock { + /// Execute the block through the EVM, as `execute_block` does. + Execute(Arc>), + /// Buffer reth's pre-computed trie data, as `index_block` does. + Index { + /// The block the trie data belongs to. + block: BlockWithParent, + /// The hashed post state of the block. + hashed_state: Arc, + /// The trie updates of the block. + trie_updates: Arc, + }, +} + +impl ReorgBlock { + fn block_with_parent(&self) -> BlockWithParent { + match self { + Self::Execute(block) => block.block_with_parent(), + Self::Index { block, .. } => *block, + } + } +} + +pub(crate) struct ReorgTask { + pub(crate) blocks: Vec>, pub(crate) reply: Sender>, } -impl ReorgTask { +impl ReorgTask { pub(crate) fn execute(self, state: &mut EngineState) where - Evm: ConfigureEvm, + Evm: ConfigureEvm>, Provider: BlockHashReader + CanonicalInMemoryStateProvider + StageCheckpointReader + DatabaseProviderFactory - + BlockReader> + + BlockReader + Clone + 'static, Store: OpProofsStore + Clone + 'static, { - let _ = self.reply.send(run(state, self.block_updates)); + let _ = self.reply.send(run(state, self.blocks)); } } -fn run( +fn run( state: &mut EngineState, - block_updates: Vec<(BlockWithParent, Arc, Arc)>, + blocks: Vec>, ) -> Result<(), EngineError> where - Evm: ConfigureEvm, + Block: reth_primitives_traits::Block, + Evm: ConfigureEvm>, Provider: BlockHashReader + CanonicalInMemoryStateProvider + StageCheckpointReader + DatabaseProviderFactory - + BlockReader> + + BlockReader + Clone + 'static, Store: OpProofsStore + Clone + 'static, { - if block_updates.is_empty() { + let (Some(first), Some(last)) = (blocks.first(), blocks.last()) else { return Ok(()); - } - - let first = &block_updates[0].0; + }; + let first = first.block_with_parent(); + let last_block_number = last.block_with_parent().block.number; let tip = state.get_tip()?; if first.block.number > tip.number { @@ -74,21 +99,24 @@ where info!( target: "trie::engine::task", - reorg_depth = block_updates.len(), + reorg_depth = blocks.len(), common_ancestor = common_ancestor_number, "Handling reorg: unwinding and buffering new path" ); - state.unwind(*first)?; + state.unwind(first)?; - for (block, trie_updates, hashed_state) in &block_updates { - state.memory.insert( - *block, - BlockStateDiff { - sorted_trie_updates: (**trie_updates).clone(), - sorted_post_state: (**hashed_state).clone(), - }, - ); + for block in blocks { + match block { + ReorgBlock::Execute(block) => super::execute_block(&block, state)?, + ReorgBlock::Index { block, hashed_state, trie_updates } => state.memory.insert( + block, + BlockStateDiff { + sorted_trie_updates: Arc::unwrap_or_clone(trie_updates), + sorted_post_state: Arc::unwrap_or_clone(hashed_state), + }, + ), + } } let total_duration = start.elapsed(); @@ -98,8 +126,8 @@ where info!( target: "trie::engine::task", - start_block_number = block_updates.first().map(|(b, _, _)| b.block.number), - end_block_number = block_updates.last().map(|(b, _, _)| b.block.number), + start_block_number = first.block.number, + end_block_number = last_block_number, ?total_duration, "Trie updates rewound and buffered successfully", ); diff --git a/rust/op-reth/crates/trie/src/in_memory.rs b/rust/op-reth/crates/trie/src/in_memory.rs index 7564c5e5f08..8d6242bbe1d 100644 --- a/rust/op-reth/crates/trie/src/in_memory.rs +++ b/rust/op-reth/crates/trie/src/in_memory.rs @@ -725,20 +725,32 @@ impl OpProofsProviderRw for InMemoryProofsProvider { Ok(write_counts) } - fn unwind_history(&self, unwind_upto_block: BlockWithParent) -> OpProofsStorageResult<()> { + fn unwind_history(&self, to: BlockWithParent) -> OpProofsStorageResult<()> { let mut inner = self.inner.write(); - let unwind_upto_block_number = unwind_upto_block.block.number - 1; - - // Remove all updates after unwind_upto_block_number - inner.trie_updates.retain(|block, _| *block <= unwind_upto_block_number); - inner.post_states.retain(|block, _| *block <= unwind_upto_block_number); - inner.account_branches.retain(|(block, _), _| *block <= unwind_upto_block_number); - inner.storage_branches.retain(|(block, _, _), _| *block <= unwind_upto_block_number); - inner.hashed_accounts.retain(|(block, _), _| *block <= unwind_upto_block_number); - inner.hashed_storages.retain(|(block, _, _), _| *block <= unwind_upto_block_number); - - // After unwind the new tip is `unwind_upto_block`'s parent. - inner.latest_block = Some((unwind_upto_block_number, unwind_upto_block.parent)); + let (earliest_block_number, _) = + inner.earliest_block.ok_or(OpProofsStorageError::NoBlocksFound)?; + let (latest_block_number, _) = + inner.latest_block.ok_or(OpProofsStorageError::NoBlocksFound)?; + + if to.block.number > latest_block_number { + return Ok(()); + } + if to.block.number <= earliest_block_number { + return Err(OpProofsStorageError::UnwindBeyondEarliest { + unwind_block_number: to.block.number, + earliest_block_number, + }); + } + + // Remove every update from `to` up; the new tip is `to`'s parent. + let new_latest_block_number = to.block.number - 1; + inner.trie_updates.retain(|block, _| *block <= new_latest_block_number); + inner.post_states.retain(|block, _| *block <= new_latest_block_number); + inner.account_branches.retain(|(block, _), _| *block <= new_latest_block_number); + inner.storage_branches.retain(|(block, _, _), _| *block <= new_latest_block_number); + inner.hashed_accounts.retain(|(block, _), _| *block <= new_latest_block_number); + inner.hashed_storages.retain(|(block, _, _), _| *block <= new_latest_block_number); + inner.latest_block = Some((new_latest_block_number, to.parent)); Ok(()) } diff --git a/rust/op-reth/crates/trie/tests/lib.rs b/rust/op-reth/crates/trie/tests/lib.rs index 34efd3d7b8b..e2b43f66910 100644 --- a/rust/op-reth/crates/trie/tests/lib.rs +++ b/rust/op-reth/crates/trie/tests/lib.rs @@ -2108,3 +2108,105 @@ fn test_updates_take_precedence_over_removals( Ok(()) } + +// ============================================================================= +// Unwind +// ============================================================================= + +/// Hash of block `number` in the unwind tests' chain; genesis is the zero hash. +const fn unwind_chain_hash(number: u64) -> B256 { + match number { + 0 => B256::ZERO, + _ => B256::repeat_byte(number as u8), + } +} + +/// Block `number` of the unwind tests' chain, on block `number - 1`. +const fn unwind_chain_block(number: u64) -> BlockWithParent { + BlockWithParent::new( + unwind_chain_hash(number - 1), + NumHash::new(number, unwind_chain_hash(number)), + ) +} + +/// Anchors `storage` at block `anchor` and stores empty blocks through `latest`. +fn store_unwind_chain( + storage: &S, + anchor: u64, + latest: u64, +) -> Result<(), OpProofsStorageError> { + bootstrap_anchor(storage, BlockNumHash::new(anchor, unwind_chain_hash(anchor))); + let provider_rw = storage.provider_rw()?; + for number in anchor + 1..=latest { + provider_rw.store_trie_updates(unwind_chain_block(number), BlockStateDiff::default())?; + } + provider_rw.commit() +} + +/// `unwind_history` refuses a target at or below the earliest block and leaves the window alone. +#[test_case(InMemoryProofsStorage::new(); "InMemory")] +#[test_case(MdbxProofsStorage::new(TempDir::new().unwrap().path()).unwrap(); "Mdbx")] +#[serial] +fn test_unwind_history_rejects_target_at_or_below_earliest( + storage: S, +) -> Result<(), OpProofsStorageError> { + store_unwind_chain(&storage, 2, 5)?; + + for target in [2, 1] { + let provider_rw = storage.provider_rw()?; + let err = provider_rw.unwind_history(unwind_chain_block(target)).unwrap_err(); + assert!( + matches!( + err, + OpProofsStorageError::UnwindBeyondEarliest { + unwind_block_number, + earliest_block_number: 2, + } if unwind_block_number == target + ), + "unwinding to {target} gave {err:?}" + ); + provider_rw.commit()?; + } + + let provider = storage.provider_ro()?; + assert_eq!(provider.get_earliest_block()?, NumHash::new(2, unwind_chain_hash(2))); + assert_eq!(provider.get_latest_block()?, NumHash::new(5, unwind_chain_hash(5))); + Ok(()) +} + +/// `unwind_history` to the block above the earliest keeps only the earliest block. +#[test_case(InMemoryProofsStorage::new(); "InMemory")] +#[test_case(MdbxProofsStorage::new(TempDir::new().unwrap().path()).unwrap(); "Mdbx")] +#[serial] +fn test_unwind_history_to_block_above_earliest_keeps_only_earliest( + storage: S, +) -> Result<(), OpProofsStorageError> { + store_unwind_chain(&storage, 2, 5)?; + + let provider_rw = storage.provider_rw()?; + provider_rw.unwind_history(unwind_chain_block(3))?; + provider_rw.commit()?; + + let provider = storage.provider_ro()?; + assert_eq!(provider.get_earliest_block()?, NumHash::new(2, unwind_chain_hash(2))); + assert_eq!(provider.get_latest_block()?, NumHash::new(2, unwind_chain_hash(2))); + Ok(()) +} + +/// `unwind_history` above the latest block changes nothing. +#[test_case(InMemoryProofsStorage::new(); "InMemory")] +#[test_case(MdbxProofsStorage::new(TempDir::new().unwrap().path()).unwrap(); "Mdbx")] +#[serial] +fn test_unwind_history_above_latest_is_a_no_op( + storage: S, +) -> Result<(), OpProofsStorageError> { + store_unwind_chain(&storage, 0, 3)?; + + let provider_rw = storage.provider_rw()?; + provider_rw.unwind_history(unwind_chain_block(5))?; + provider_rw.commit()?; + + let provider = storage.provider_ro()?; + assert_eq!(provider.get_latest_block()?, NumHash::new(3, unwind_chain_hash(3))); + Ok(()) +} diff --git a/rust/op-reth/crates/trie/tests/live.rs b/rust/op-reth/crates/trie/tests/live.rs index a8f7b3a2cc0..1cfeba2e40e 100644 --- a/rust/op-reth/crates/trie/tests/live.rs +++ b/rust/op-reth/crates/trie/tests/live.rs @@ -43,6 +43,8 @@ mod contracts; mod index_execute_parity; #[path = "live/parent_states.rs"] mod parent_states; +#[path = "live/reorg_revert_edges.rs"] +mod reorg_revert_edges; #[path = "live/simulated_reth.rs"] mod simulated_reth; #[path = "live/store_view.rs"] @@ -59,6 +61,25 @@ const fn block_timestamp(number: u64) -> u64 { number * BLOCK_TIME } +/// The engine's persistence and backpressure thresholds. +#[derive(Clone, Copy, Debug)] +struct FlushCadence { + backpressure: u64, + persistence: u64, +} + +impl FlushCadence { + const EVERY_BLOCK: Self = Self { backpressure: 2, persistence: 1 }; + const EVERY_FIFTH_BLOCK: Self = Self { backpressure: 10, persistence: 5 }; +} + +/// Which store implementation a case runs over. +#[derive(Clone, Copy, Debug)] +enum Backend { + InMemory, + Mdbx, +} + fn create_mdbx_proofs_storage() -> Arc { let path = TempDir::new().unwrap().keep(); Arc::new(MdbxProofsStorage::new(&path).unwrap()) diff --git a/rust/op-reth/crates/trie/tests/live/index_execute_parity.rs b/rust/op-reth/crates/trie/tests/live/index_execute_parity.rs index 0698943b148..4b7cff921f2 100644 --- a/rust/op-reth/crates/trie/tests/live/index_execute_parity.rs +++ b/rust/op-reth/crates/trie/tests/live/index_execute_parity.rs @@ -9,7 +9,7 @@ //! comparison runs over the persisted history once everything is flushed. use super::{ - BlockSpec, TxSpec, block_timestamp, chain_spec_builder, + Backend, BlockSpec, FlushCadence, TxSpec, block_timestamp, chain_spec_builder, contracts::{ BLOCK_HASH_RECORDER_RUNTIME, BLOCK_HASH_SLOT, CALLED_SLOT, CREATE_AND_SELFDESTRUCT_SLOTS, SLOT_WRITER_SLOTS, block_number_calldata, create_and_selfdestruct_initcode, @@ -242,25 +242,6 @@ impl StateChange { } } -/// The engine's persistence and backpressure thresholds. -#[derive(Clone, Copy, Debug)] -struct FlushCadence { - backpressure: u64, - persistence: u64, -} - -impl FlushCadence { - const EVERY_BLOCK: Self = Self { backpressure: 2, persistence: 1 }; - const EVERY_FIFTH_BLOCK: Self = Self { backpressure: 10, persistence: 5 }; -} - -/// Which store implementation the indexed and executed stores use. -#[derive(Clone, Copy, Debug)] -enum Backend { - InMemory, - Mdbx, -} - /// How a backfill reads the trie state it rewinds from. #[derive(Clone, Copy, Debug)] enum BackfillMode { diff --git a/rust/op-reth/crates/trie/tests/live/reorg_revert_edges.rs b/rust/op-reth/crates/trie/tests/live/reorg_revert_edges.rs new file mode 100644 index 00000000000..c4552d4b08c --- /dev/null +++ b/rust/op-reth/crates/trie/tests/live/reorg_revert_edges.rs @@ -0,0 +1,789 @@ +//! Drives the engine's reorg and unwind tasks at the boundaries the model-based test may not +//! reach: a reorg whose first block does not build on the stored fork block, a target at or +//! below the window's earliest block, a revert that only touches the buffer, and a reorg or a +//! revert that arrives while a save is in flight. +//! +//! Every case commits the same transfer chain to reth, indexes it under the case's flush cadence +//! and settles, then acts. The engine's tip is read through the parent check of a probe block, +//! and the store's persisted history is held to a fresh store indexed over reth's final canonical +//! chain once a save has carried the case's blocks to disk. + +use super::{ + Backend, BlockSpec, FlushCadence, TxSpec, chain_spec_with_address, create_block_from_spec, + create_mdbx_proofs_storage, initialize_storage, public_key_to_address, + simulated_reth::{RethPersistence, SimulatedReth, fixed_signer}, + store_view::{BlockView, ProofTarget, assert_same_view}, +}; +use alloy_eips::{NumHash, eip1898::BlockWithParent}; +use alloy_primitives::{Address, B256, U256}; +use parking_lot::Mutex; +use reth_chain_state::ExecutedBlock; +use reth_chainspec::ChainSpec; +use reth_ethereum_primitives::Block; +use reth_optimism_trie::{ + InMemoryProofsStorage, OpProofsProviderRO, OpProofsStorageError, OpProofsStorageResult, + OpProofsStore, ProofWindowRange, + engine::{EngineError, EngineHandle, ReorgBlock}, +}; +use reth_trie_common::{HashedPostStateSorted, updates::TrieUpdatesSorted}; +use rstest::rstest; +use secp256k1::Keypair; +use serial_test::serial; +use std::{ + fmt, + ops::RangeInclusive, + sync::{ + Arc, + mpsc::{Receiver, SyncSender, sync_channel}, + }, + thread, +}; + +/// Blocks every case commits and settles before it acts. +const CHAIN_LENGTH: u64 = 7; + +/// Blocks a case may hold at most, which bounds the flushes a settle needs and the recipients. +const MAX_BLOCKS_PER_CASE: u64 = 2 * CHAIN_LENGTH; + +/// reth's persistence threshold, above any chain here so that reth keeps every block in memory, +/// where `reorg_with` can fork it anywhere and the reference store still finds genesis state. +const RETH_PERSISTENCE_THRESHOLD: u64 = 2 * MAX_BLOCKS_PER_CASE; + +/// Blocks the pruned window keeps, so that pruning moves the earliest block above the anchor +/// under either cadence. +const PRUNED_WINDOW: u64 = 3; + +/// A window no chain here fills, so that the earliest block stays the anchor. +const WHOLE_WINDOW: u64 = 1000; + +/// A parent hash no block here has. +const UNRELATED_PARENT: B256 = B256::repeat_byte(0xBA); + +const fn recipient(number: u64) -> Address { + Address::repeat_byte(0x40 + number as u8) +} + +const fn fork_recipient(number: u64) -> Address { + Address::repeat_byte(0x80 + number as u8) +} + +/// Block `number` of the chain: `number` wei to the recipient only this block pays. +fn chain_block(number: u64) -> BlockSpec { + BlockSpec::new(vec![TxSpec::transfer(recipient(number), U256::from(number))]) +} + +/// Blocks `numbers` of a fork: `number` wei each to a recipient the chain never pays. +fn fork_blocks(numbers: RangeInclusive) -> Vec { + numbers + .map(|number| { + BlockSpec::new(vec![TxSpec::transfer(fork_recipient(number), U256::from(number))]) + }) + .collect() +} + +/// Every address the chain or a fork can touch, none of which has storage. +fn proof_targets(sender: Address) -> Vec { + [sender, Address::ZERO] + .into_iter() + .chain( + (1..=MAX_BLOCKS_PER_CASE) + .flat_map(|number| [recipient(number), fork_recipient(number)]), + ) + .map(|address| ProofTarget { address, slots: Vec::new() }) + .collect() +} + +/// The disk tip once the engine has saved what its cadence lets it save: `buffered` blocks went +/// into the buffer above `base`, and the first save starts at the persistence threshold with +/// exactly that many blocks. Below twice the threshold no second save starts, so the outcome does +/// not depend on when the first save completes. +const fn settled_disk_tip(cadence: FlushCadence, base: u64, buffered: u64) -> u64 { + assert!( + cadence.persistence == 1 || buffered < 2 * cadence.persistence, + "a second save would race the commits" + ); + base + buffered - buffered % cadence.persistence +} + +/// How the engine receives the blocks of a new chain. +#[derive(Clone, Copy, Debug)] +enum ReorgPath { + Execute, + Index, +} + +/// `blocks` the way the ExEx hands a reorg to the engine along `path`. +fn reorg_blocks(blocks: &[ExecutedBlock], path: ReorgPath) -> Vec> { + blocks + .iter() + .map(|block| match path { + ReorgPath::Execute => ReorgBlock::Execute(Arc::clone(&block.recovered_block)), + ReorgPath::Index => ReorgBlock::Index { + block: block.recovered_block().block_with_parent(), + hashed_state: block.hashed_state(), + trie_updates: block.trie_updates(), + }, + }) + .collect() +} + +/// The reorg task's input for `block` without trie data, enough to reach the unwind checks. +fn reorg_block_without_data(block: BlockWithParent) -> ReorgBlock { + ReorgBlock::Index { + block, + hashed_state: Arc::new(HashedPostStateSorted::default()), + trie_updates: Arc::new(TrieUpdatesSorted::default()), + } +} + +/// Which engine task unwinds. +#[derive(Clone, Copy, Debug)] +enum Task { + Reorg, + Unwind, +} + +/// Where an unwind target sits relative to the proof window. +#[derive(Clone, Copy, Debug)] +enum Target { + /// The init anchor, block 0: the earliest block of a whole window, below a pruned one. + Anchor, + BelowEarliest, + Earliest, +} + +/// The store's end of a held write. +struct WriteHold { + entered: SyncSender<()>, + release: Receiver<()>, +} + +/// The case's end of a held write. +struct HeldWrite { + entered: Receiver<()>, + release: SyncSender<()>, +} + +impl HeldWrite { + /// Blocks until the engine's persistence thread is inside the held write. + fn wait_until_entered(&self) { + self.entered.recv().expect("the store keeps the hold until a write enters it"); + } + + /// Lets the held write go on, or the next write through if none is held yet. + fn release(self) { + let _ = self.release.send(()); + } +} + +/// A proofs store whose next write transaction, once a hold is armed, blocks until the hold is +/// released, which keeps a save of the engine in flight for as long as a case needs. +#[derive(Clone)] +struct StoreWithWriteGate { + inner: S, + next_write: Arc>>, +} + +impl StoreWithWriteGate { + fn new(inner: S) -> Self { + Self { inner, next_write: Arc::default() } + } + + fn hold_next_write(&self) -> HeldWrite { + let (entered_tx, entered_rx) = sync_channel(1); + let (release_tx, release_rx) = sync_channel(1); + *self.next_write.lock() = Some(WriteHold { entered: entered_tx, release: release_rx }); + HeldWrite { entered: entered_rx, release: release_tx } + } +} + +impl fmt::Debug for StoreWithWriteGate { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("StoreWithWriteGate").field("inner", &self.inner).finish_non_exhaustive() + } +} + +impl OpProofsStore for StoreWithWriteGate { + type ProviderRO<'a> + = S::ProviderRO<'a> + where + Self: 'a; + + type ProviderRw<'a> + = S::ProviderRw<'a> + where + Self: 'a; + + type Initializer<'a> + = S::Initializer<'a> + where + Self: 'a; + + fn provider_ro<'a>(&'a self) -> OpProofsStorageResult> { + self.inner.provider_ro() + } + + fn provider_rw<'a>(&'a self) -> OpProofsStorageResult> { + let hold = self.next_write.lock().take(); + if let Some(hold) = hold { + let _ = hold.entered.send(()); + let _ = hold.release.recv(); + } + self.inner.provider_rw() + } + + fn initialization_provider<'a>(&'a self) -> OpProofsStorageResult> { + self.inner.initialization_provider() + } +} + +/// What the store serves at one moment: its window and every persisted view in it. +struct DiskSnapshot { + views: Vec<(u64, BlockView)>, + window: ProofWindowRange, +} + +/// reth holding the chain, and the engine indexing it into the store under test. +struct Scenario { + cadence: FlushCadence, + chain_spec: Arc, + engine: EngineHandle, + gated: StoreWithWriteGate, + key_pair: Keypair, + reth: SimulatedReth, + store: S, + targets: Vec, +} + +impl Scenario { + /// Commits the chain to reth, indexes it into a store pruned to `window` blocks, and settles. + fn new( + make_store: impl FnOnce() -> S, + cadence: FlushCadence, + window: u64, + ) -> eyre::Result { + let key_pair = fixed_signer(); + let sender = public_key_to_address(key_pair.public_key()); + let chain_spec = chain_spec_with_address(sender); + let persistence = RethPersistence::threshold(RETH_PERSISTENCE_THRESHOLD); + let reth = SimulatedReth::new(chain_spec.clone(), key_pair, persistence)?; + let store = make_store(); + initialize_storage(store.clone(), &reth.factory, reth.head())?; + let gated = StoreWithWriteGate::new(store.clone()); + let engine = reth.spawn_engine_with_window( + gated.clone(), + cadence.persistence, + cadence.backpressure, + window, + ); + let mut scenario = Self { + cadence, + chain_spec, + engine, + gated, + key_pair, + reth, + store, + targets: proof_targets(sender), + }; + for number in 1..=CHAIN_LENGTH { + scenario.commit(&chain_block(number))?; + } + let disk_tip = scenario.settle()?; + assert_eq!(disk_tip.number, settled_disk_tip(cadence, 0, CHAIN_LENGTH), "settled disk tip"); + Ok(scenario) + } + + /// Commits `spec` to reth and indexes it. + fn commit(&mut self, spec: &BlockSpec) -> eyre::Result<()> { + let block = self.reth.commit(spec)?; + self.index(&block)?; + Ok(()) + } + + fn index(&self, block: &ExecutedBlock) -> Result<(), EngineError> { + self.engine.index_block( + block.recovered_block().block_with_parent(), + (*block.trie_updates()).clone(), + (*block.hashed_state()).clone(), + ) + } + + /// Commits enough blocks for a save to carry everything buffered to disk, and settles. + fn commit_through_a_save(&mut self) -> eyre::Result<()> { + for _ in 0..self.cadence.persistence { + let number = self.reth.head().number + 1; + self.commit(&chain_block(number))?; + } + self.settle()?; + Ok(()) + } + + /// Flushes until nothing the engine could still save is unsaved, and returns the disk tip. + /// A flush waits for the save in flight, and the engine starts the next save only after + /// replying, so one flush per block the case could have buffered drains them all. + fn settle(&self) -> eyre::Result { + for _ in 0..=MAX_BLOCKS_PER_CASE { + self.engine.flush(); + } + Ok(self.window()?.latest) + } + + fn window(&self) -> eyre::Result { + Ok(self.store.provider_ro()?.get_proof_window()?) + } + + /// Arms a hold on the engine's next write, the save the following commits start. + fn hold_next_save(&self) -> HeldWrite { + self.gated.hold_next_write() + } + + /// Commits blocks until the buffer reaches the persistence threshold and the save it starts + /// is inside `hold`. + fn commit_until_a_save_is_in_flight(&mut self, hold: &HeldWrite) -> eyre::Result<()> { + let buffered = self.reth.head().number - self.window()?.latest.number; + for _ in buffered..self.cadence.persistence { + let number = self.reth.head().number + 1; + self.commit(&chain_block(number))?; + } + hold.wait_until_entered(); + Ok(()) + } + + /// Replaces the engine with one started over the same store, as a restart does. + fn restart_engine(&mut self, window: u64) { + self.engine = self.reth.spawn_engine_with_window( + self.gated.clone(), + self.cadence.persistence, + self.cadence.backpressure, + window, + ); + } + + /// The canonical block `number` as an unwind target: the target's parent is the block below. + fn canonical_target(&self, number: u64) -> BlockWithParent { + let parent = match number { + 0 => B256::ZERO, + _ => self.reth.num_hash(number - 1).hash, + }; + BlockWithParent::new(parent, self.reth.num_hash(number)) + } + + /// A one-block reorg at `number` whose block builds on a parent no block here has. + fn reorg_on_unrelated_parent(&self, number: u64, path: ReorgPath) -> Vec> { + let block = match path { + ReorgPath::Execute => ReorgBlock::Execute(Arc::new(create_block_from_spec( + &BlockSpec::empty(), + number, + UNRELATED_PARENT, + &self.chain_spec, + self.key_pair, + &mut 0, + ))), + ReorgPath::Index => reorg_block_without_data(BlockWithParent::new( + UNRELATED_PARENT, + NumHash::new(number, B256::repeat_byte(0xB1)), + )), + }; + vec![block] + } + + fn disk_snapshot(&self) -> eyre::Result { + let window = self.window()?; + let views = (window.earliest.number..=window.latest.number) + .map(|number| Ok((number, BlockView::read(&self.store, number, &self.targets)?))) + .collect::>()?; + Ok(DiskSnapshot { views, window }) + } + + /// The store's window and every persisted view are as in `before`. + fn assert_disk_unchanged(&self, before: &DiskSnapshot) -> eyre::Result<()> { + assert_eq!(self.window()?, before.window, "proof window"); + for (number, expected) in &before.views { + let actual = BlockView::read(&self.store, *number, &self.targets)?; + assert_same_view(*number, "unwound", expected, &actual)?; + } + Ok(()) + } + + /// The engine's tip is `expected`, read through the parent check: a block numbered one above + /// it on an unrelated parent is refused with the tip's hash as the expected parent. + fn assert_tip(&self, expected: NumHash) { + let probe = BlockWithParent::new( + UNRELATED_PARENT, + NumHash::new(expected.number + 1, B256::repeat_byte(0xBB)), + ); + let err = self + .engine + .index_block(probe, TrieUpdatesSorted::default(), HashedPostStateSorted::default()) + .expect_err("the probe does not build on the tip"); + let EngineError::ParentHashMismatch { block_number, expected_parent_hash, .. } = err else { + panic!("expected a parent-hash mismatch, got {err:?}"); + }; + assert_eq!((block_number, expected_parent_hash), (expected.number + 1, expected.hash)); + } + + /// The store's persisted history reaches `through` and is what a fresh store indexed over + /// reth's canonical chain serves, with proofs that hash to reth's state roots. + fn assert_disk_matches_reth_through(&self, through: u64) -> eyre::Result<()> { + let window = self.window()?; + eyre::ensure!( + window.latest.number >= through, + "the disk tip {} is below block {through}", + window.latest.number + ); + eyre::ensure!( + self.reth.locate(window.latest).canonical, + "the disk tip {:?} is not canonical in reth", + window.latest + ); + let reference = self.reference_store()?; + for number in window.earliest.number..=window.latest.number { + let expected = BlockView::read(&reference, number, &self.targets)?; + let actual = BlockView::read(&self.store, number, &self.targets)?; + assert_same_view(number, "under test", &expected, &actual)?; + actual.verify_proofs(number, self.reth.state_root(number))?; + } + Ok(()) + } + + /// A fresh store indexed over reth's canonical chain from genesis. + fn reference_store(&self) -> eyre::Result { + eyre::ensure!( + !self.reth.locate(self.reth.num_hash(1)).persisted, + "reth's database must still hold genesis state" + ); + let store = InMemoryProofsStorage::new(); + initialize_storage(store.clone(), &self.reth.factory, self.reth.genesis())?; + let engine = self.reth.spawn_engine(store.clone(), 1, 2); + let head = self.reth.head().number; + for number in 1..=head { + let block = self.reth.executed_block(number); + engine.index_block( + block.recovered_block().block_with_parent(), + (*block.trie_updates()).clone(), + (*block.hashed_state()).clone(), + )?; + } + for _ in 0..=head { + engine.flush(); + } + eyre::ensure!( + store.provider_ro()?.get_latest_block()? == self.reth.head(), + "the reference store holds reth's whole chain" + ); + Ok(store) + } +} + +/// Runs `$run` over a fresh store of `$backend`. +macro_rules! over_backend { + ($backend:expr, $run:ident($($arg:expr),*)) => { + match $backend { + Backend::InMemory => $run(InMemoryProofsStorage::new, $($arg),*), + Backend::Mdbx => $run(create_mdbx_proofs_storage, $($arg),*), + } + }; +} + +fn run_reorg_case( + make_store: impl FnOnce() -> S, + cadence: FlushCadence, + path: ReorgPath, + fork: u64, +) -> eyre::Result<()> { + let mut scenario = Scenario::new(make_store, cadence, WHOLE_WINDOW)?; + let new_chain = scenario.reth.reorg_with(fork + 1, &fork_blocks(fork + 1..=fork + 3))?; + + scenario.engine.reorg(reorg_blocks(&new_chain, path))?; + + scenario.assert_tip(scenario.reth.head()); + scenario.commit_through_a_save()?; + scenario.assert_disk_matches_reth_through(fork + 3) +} + +/// A reorg replaces the stored blocks above the fork with the new chain, indexed or executed, +/// whether the fork is buffered (block 6 under the slower cadence) or on disk. +#[rstest] +#[case::indexed_from_block_six(ReorgPath::Index, 6)] +#[case::executed_from_block_three(ReorgPath::Execute, 3)] +#[serial] +fn reorg_replaces_the_stored_blocks_above_the_fork( + #[values(Backend::InMemory, Backend::Mdbx)] backend: Backend, + #[values(FlushCadence::EVERY_BLOCK, FlushCadence::EVERY_FIFTH_BLOCK)] cadence: FlushCadence, + #[case] path: ReorgPath, + #[case] fork: u64, +) -> eyre::Result<()> { + over_backend!(backend, run_reorg_case(cadence, path, fork)) +} + +fn run_wrong_fork_parent_case( + make_store: impl FnOnce() -> S, + cadence: FlushCadence, + path: ReorgPath, + fork: u64, +) -> eyre::Result<()> { + let mut scenario = Scenario::new(make_store, cadence, WHOLE_WINDOW)?; + let before = scenario.disk_snapshot()?; + + let err = scenario + .engine + .reorg(scenario.reorg_on_unrelated_parent(fork + 1, path)) + .expect_err("the reorg does not build on the stored fork block"); + + let EngineError::ParentHashMismatch { block_number, expected_parent_hash, actual_parent_hash } = + err + else { + panic!("expected a parent-hash mismatch, got {err:?}"); + }; + assert_eq!( + (block_number, expected_parent_hash, actual_parent_hash), + (fork + 1, scenario.reth.num_hash(fork).hash, UNRELATED_PARENT) + ); + scenario.assert_disk_unchanged(&before)?; + scenario.assert_tip(scenario.reth.head()); + scenario.commit_through_a_save()?; + scenario.assert_disk_matches_reth_through(CHAIN_LENGTH) +} + +/// A reorg whose first block does not build on the stored fork block fails with the stored hash +/// and changes nothing, whether the engine knows that block from its buffer (block 6 under the +/// slower cadence), from a save it made, or as the window's earliest block. +#[rstest] +#[case::indexed_on_block_six(ReorgPath::Index, 6)] +#[case::executed_on_block_two(ReorgPath::Execute, 2)] +#[case::indexed_on_the_anchor(ReorgPath::Index, 0)] +#[serial] +fn reorg_not_building_on_the_stored_fork_block_fails_and_changes_nothing( + #[values(Backend::InMemory, Backend::Mdbx)] backend: Backend, + #[values(FlushCadence::EVERY_BLOCK, FlushCadence::EVERY_FIFTH_BLOCK)] cadence: FlushCadence, + #[case] path: ReorgPath, + #[case] fork: u64, +) -> eyre::Result<()> { + over_backend!(backend, run_wrong_fork_parent_case(cadence, path, fork)) +} + +fn run_restarted_engine_case( + make_store: impl FnOnce() -> S, +) -> eyre::Result<()> { + let fork = 2; + let mut scenario = Scenario::new(make_store, FlushCadence::EVERY_BLOCK, WHOLE_WINDOW)?; + let before = scenario.disk_snapshot()?; + scenario.restart_engine(WHOLE_WINDOW); + + let err = scenario + .engine + .reorg(scenario.reorg_on_unrelated_parent(fork + 1, ReorgPath::Index)) + .expect_err("the reorg does not build on the stored fork block"); + + assert!(matches!(err, EngineError::ParentHashMismatch { .. }), "got {err:?}"); + scenario.assert_disk_unchanged(&before) +} + +/// A restarted engine knows no hash below its store's latest block, so a reorg onto a wrong +/// fork block there goes through unverified; checking it needs the store to keep every block's +/// hash. +#[rstest] +#[ignore = "a restarted engine cannot verify a fork below its disk tip; the store keeps no per-block hashes (#23391, tracked under #23385)"] +#[serial] +fn restarted_engine_rejects_a_reorg_not_building_on_a_block_below_its_disk_tip( + #[values(Backend::InMemory, Backend::Mdbx)] backend: Backend, +) -> eyre::Result<()> { + over_backend!(backend, run_restarted_engine_case()) +} + +fn run_below_window_case( + make_store: impl FnOnce() -> S, + cadence: FlushCadence, + task: Task, + target: Target, + window: u64, +) -> eyre::Result<()> { + let scenario = Scenario::new(make_store, cadence, window)?; + let before = scenario.disk_snapshot()?; + let earliest = before.window.earliest.number; + let number = match target { + Target::Anchor => 0, + Target::BelowEarliest => earliest - 1, + Target::Earliest => earliest, + }; + let to = scenario.canonical_target(number); + + let result = match task { + Task::Reorg => scenario.engine.reorg(vec![reorg_block_without_data(to)]), + Task::Unwind => scenario.engine.unwind(to), + }; + + let err = result.expect_err("the unwind would remove the window's earliest block"); + assert!( + matches!( + err, + EngineError::Storage(OpProofsStorageError::UnwindBeyondEarliest { + unwind_block_number, + earliest_block_number, + }) if unwind_block_number == number && earliest_block_number == earliest + ), + "got {err:?}" + ); + scenario.assert_disk_unchanged(&before)?; + scenario.assert_tip(scenario.reth.head()); + Ok(()) +} + +/// An unwind or a reorg whose target is at or below the window's earliest block, the init anchor +/// or a block pruning moved the window past, fails before touching the store or the buffer. +#[rstest] +#[case::unwind_at_the_earliest_block(Task::Unwind, Target::Earliest, PRUNED_WINDOW)] +#[case::reorg_below_the_earliest_block(Task::Reorg, Target::BelowEarliest, PRUNED_WINDOW)] +#[case::reorg_at_the_anchor_below_the_window(Task::Reorg, Target::Anchor, PRUNED_WINDOW)] +#[case::unwind_at_the_anchor_as_earliest_block(Task::Unwind, Target::Anchor, WHOLE_WINDOW)] +#[serial] +fn unwind_or_reorg_at_or_below_the_earliest_block_fails_and_changes_nothing( + #[values(Backend::InMemory, Backend::Mdbx)] backend: Backend, + #[values(FlushCadence::EVERY_BLOCK, FlushCadence::EVERY_FIFTH_BLOCK)] cadence: FlushCadence, + #[case] task: Task, + #[case] target: Target, + #[case] window: u64, +) -> eyre::Result<()> { + over_backend!(backend, run_below_window_case(cadence, task, target, window)) +} + +fn run_unwind_to_above_earliest_case( + make_store: impl FnOnce() -> S, + cadence: FlushCadence, +) -> eyre::Result<()> { + let mut scenario = Scenario::new(make_store, cadence, PRUNED_WINDOW)?; + let earliest = scenario.window()?.earliest; + + scenario.engine.unwind(scenario.canonical_target(earliest.number + 1))?; + + assert_eq!(scenario.window()?, ProofWindowRange { earliest, latest: earliest }); + scenario.assert_tip(earliest); + for number in earliest.number + 1..=CHAIN_LENGTH { + scenario.index(scenario.reth.executed_block(number))?; + } + scenario.commit_through_a_save()?; + scenario.assert_disk_matches_reth_through(CHAIN_LENGTH) +} + +/// An unwind to the block right above the earliest leaves the window at that one block, from +/// which the engine indexes the chain again. +#[rstest] +#[serial] +fn unwind_to_the_block_above_the_earliest_keeps_only_the_earliest( + #[values(Backend::InMemory, Backend::Mdbx)] backend: Backend, + #[values(FlushCadence::EVERY_BLOCK, FlushCadence::EVERY_FIFTH_BLOCK)] cadence: FlushCadence, +) -> eyre::Result<()> { + over_backend!(backend, run_unwind_to_above_earliest_case(cadence)) +} + +fn run_buffered_revert_case( + make_store: impl FnOnce() -> S, + revert_from: u64, +) -> eyre::Result<()> { + let cadence = FlushCadence::EVERY_FIFTH_BLOCK; + let mut scenario = Scenario::new(make_store, cadence, WHOLE_WINDOW)?; + let before = scenario.disk_snapshot()?; + assert!(before.window.latest.number < revert_from, "the reverted blocks are buffered"); + + scenario.engine.unwind(scenario.canonical_target(revert_from))?; + + scenario.assert_disk_unchanged(&before)?; + scenario.assert_tip(scenario.reth.num_hash(revert_from - 1)); + let new_chain = + scenario.reth.reorg_with(revert_from, &fork_blocks(revert_from..=CHAIN_LENGTH))?; + for block in &new_chain { + scenario.index(block)?; + } + scenario.commit_through_a_save()?; + scenario.assert_disk_matches_reth_through(CHAIN_LENGTH) +} + +/// A revert of blocks still in the buffer unwinds the buffer, leaves the disk untouched and +/// moves the tip to the block below the revert. Under the every-block cadence no block stays +/// buffered outside a save in flight, which the in-flight case covers. +#[rstest] +#[serial] +fn revert_of_buffered_blocks_unwinds_the_buffer_and_leaves_the_disk_untouched( + #[values(Backend::InMemory, Backend::Mdbx)] backend: Backend, + #[values(6, 7)] revert_from: u64, +) -> eyre::Result<()> { + over_backend!(backend, run_buffered_revert_case(revert_from)) +} + +fn run_revert_during_save_case( + make_store: impl FnOnce() -> S, + cadence: FlushCadence, +) -> eyre::Result<()> { + let mut scenario = Scenario::new(make_store, cadence, WHOLE_WINDOW)?; + let before = scenario.disk_snapshot()?; + let disk_tip = before.window.latest; + let hold = scenario.hold_next_save(); + scenario.commit_until_a_save_is_in_flight(&hold)?; + let revert = scenario.canonical_target(disk_tip.number + 1); + let engine = scenario.engine.clone(); + + let reverting = thread::spawn(move || engine.unwind(revert)); + hold.release(); + reverting.join().expect("the revert thread finishes")?; + + scenario.assert_disk_unchanged(&before)?; + scenario.assert_tip(disk_tip); + let new_chain = scenario + .reth + .reorg_with(disk_tip.number + 1, &fork_blocks(disk_tip.number + 1..=disk_tip.number + 3))?; + for block in &new_chain { + scenario.index(block)?; + } + scenario.commit_through_a_save()?; + scenario.assert_disk_matches_reth_through(disk_tip.number + 3) +} + +/// A revert of every block above the disk tip while their save is in flight waits for the save, +/// then removes them again: the disk ends as it was and the tip is the disk tip. +#[rstest] +#[serial] +fn revert_during_an_in_flight_save_waits_for_the_save( + #[values(Backend::InMemory, Backend::Mdbx)] backend: Backend, + #[values(FlushCadence::EVERY_BLOCK, FlushCadence::EVERY_FIFTH_BLOCK)] cadence: FlushCadence, +) -> eyre::Result<()> { + over_backend!(backend, run_revert_during_save_case(cadence)) +} + +fn run_reorg_during_save_case( + make_store: impl FnOnce() -> S, + cadence: FlushCadence, + path: ReorgPath, +) -> eyre::Result<()> { + let mut scenario = Scenario::new(make_store, cadence, WHOLE_WINDOW)?; + let fork = scenario.window()?.latest.number - 2; + let hold = scenario.hold_next_save(); + scenario.commit_until_a_save_is_in_flight(&hold)?; + let new_chain = scenario.reth.reorg_with(fork + 1, &fork_blocks(fork + 1..=fork + 3))?; + let blocks = reorg_blocks(&new_chain, path); + let engine = scenario.engine.clone(); + + let reorging = thread::spawn(move || engine.reorg(blocks)); + hold.release(); + reorging.join().expect("the reorg thread finishes")?; + + scenario.assert_tip(scenario.reth.head()); + let disk_tip = scenario.settle()?; + assert_eq!(disk_tip, scenario.reth.num_hash(settled_disk_tip(cadence, fork, 3))); + scenario.assert_disk_matches_reth_through(disk_tip.number)?; + scenario.commit_through_a_save()?; + scenario.assert_disk_matches_reth_through(fork + 3) +} + +/// A reorg from a fork on disk while a save of the blocks above it is in flight waits for the +/// save, unwinds the disk to the fork and buffers the new chain, so that the disk and the buffer +/// agree with reth. +#[rstest] +#[serial] +fn reorg_from_a_disk_fork_during_an_in_flight_save_waits_for_the_save( + #[values(Backend::InMemory, Backend::Mdbx)] backend: Backend, + #[values(FlushCadence::EVERY_BLOCK, FlushCadence::EVERY_FIFTH_BLOCK)] cadence: FlushCadence, + #[values(ReorgPath::Index, ReorgPath::Execute)] path: ReorgPath, +) -> eyre::Result<()> { + over_backend!(backend, run_reorg_during_save_case(cadence, path)) +} diff --git a/rust/op-reth/crates/trie/tests/live/simulated_reth.rs b/rust/op-reth/crates/trie/tests/live/simulated_reth.rs index 8e44c111255..2e9e4a467b3 100644 --- a/rust/op-reth/crates/trie/tests/live/simulated_reth.rs +++ b/rust/op-reth/crates/trie/tests/live/simulated_reth.rs @@ -144,13 +144,17 @@ impl SimulatedReth { }) } - fn num_hash(&self, number: u64) -> BlockNumHash { + pub(super) fn num_hash(&self, number: u64) -> BlockNumHash { match number { 0 => BlockNumHash::new(0, self.chain_spec.genesis_hash()), _ => self.block(number).num_hash(), } } + pub(super) fn genesis(&self) -> BlockNumHash { + self.num_hash(0) + } + pub(super) fn state_root(&self, number: u64) -> B256 { match number { 0 => self.chain_spec.genesis_header().state_root, @@ -163,10 +167,16 @@ impl SimulatedReth { } pub(super) fn block(&self, number: u64) -> &RecoveredBlock { - self.canonical[number as usize - 1].recovered_block() + self.executed_block(number).recovered_block() + } + + /// The canonical block at `number` with the trie data reth computed for it. + pub(super) fn executed_block(&self, number: u64) -> &ExecutedBlock { + &self.canonical[number as usize - 1] } - /// Spawns a proofs engine over `store` that reads reth's state through this chain's provider. + /// Spawns a proofs engine over `store` that reads reth's state through this chain's provider + /// and never prunes. pub(super) fn spawn_engine( &self, store: S, @@ -176,8 +186,26 @@ impl SimulatedReth { where S: OpProofsStore + Clone + 'static, { - let pruner = - OpProofStoragePruner::new(store.clone(), self.provider.clone(), PRUNE_INTERVAL_BLOCKS); + self.spawn_engine_with_window( + store, + persistence_threshold, + backpressure_threshold, + PRUNE_INTERVAL_BLOCKS, + ) + } + + /// Spawns a proofs engine over `store` whose saves prune the store down to `window` blocks. + pub(super) fn spawn_engine_with_window( + &self, + store: S, + persistence_threshold: u64, + backpressure_threshold: u64, + window: u64, + ) -> EngineHandle + where + S: OpProofsStore + Clone + 'static, + { + let pruner = OpProofStoragePruner::new(store.clone(), self.provider.clone(), window); EngineHandle::spawn_with_thresholds( self.evm_config.clone(), self.provider.clone(), @@ -225,20 +253,39 @@ impl SimulatedReth { /// Replaces the canonical blocks from `number` up with empty blocks on the same ancestor, the /// way a reorg to a sibling fork does. pub(super) fn reorg_from(&mut self, number: u64) -> eyre::Result<()> { + let depth = self.head().number + 1 - number; + self.reorg_with(number, &vec![BlockSpec::empty(); depth as usize])?; + Ok(()) + } + + /// Replaces the canonical blocks from `number` up with `specs` built on the same ancestor, + /// the way a reorg to a sibling fork does, and returns the new blocks. + pub(super) fn reorg_with( + &mut self, + number: u64, + specs: &[BlockSpec], + ) -> eyre::Result> { assert!(number > self.last_persisted.number, "only in-memory blocks can be reorged"); + let (first_spec, rest) = specs.split_first().expect("a reorg switches to a new block"); let old = self.canonical.split_off(number as usize - 1); - let fork = self.build_block(&BlockSpec::empty())?; + // The signer's nonce is the number of transactions the remaining chain holds. + self.nonce = self + .canonical + .iter() + .map(|block| block.recovered_block().body().transactions.len() as u64) + .sum(); + let fork = self.build_block(first_spec)?; assert_ne!( fork.recovered_block().hash(), old[0].recovered_block().hash(), "the fork must differ from the chain it replaces" ); - let depth = old.len(); + let mut new = vec![fork.clone()]; self.make_canonical(NewCanonicalChain::Reorg { new: vec![fork], old }); - for _ in 1..depth { - self.commit(&BlockSpec::empty())?; + for spec in rest { + new.push(self.commit(spec)?); } - Ok(()) + Ok(new) } /// Writes every canonical block to the database without evicting any from memory, the state From d4ae7175a1a8ccd80b9d2c88d706768e9a070470 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Thu, 15 Oct 2026 02:49:09 -0700 Subject: [PATCH 2/4] op-reth: address review of the proofs engine reorg edges Run every block of a reorg through the same code as index_block and execute_block, so an indexed block cannot land on a gap left by an executed block the engine had to skip, and cover a mixed reorg. Pin that the unwind checks run against the window as an in-flight save leaves it with a pruned window case, name the in-flight cases by the end state they assert, bound the wait on a held write, drop the wait in PersistenceState::unwind that its only caller now does first, log the two rejections, share the index helper and the backend macro across the live tests, and add a table for the ExEx's trie-data rule. Co-Authored-By: Claude Fable 5.1 (cherry picked from commit c7bb2d68ea3d69f869ad5b0ce22c6fbf9d767790) (cherry picked from commit afb5bb8730ad4a83299cc6c02018c62024973786) --- rust/op-reth/crates/exex/Cargo.toml | 2 +- rust/op-reth/crates/exex/src/lib.rs | 129 ++++--- .../crates/trie/src/engine/buffer/state.rs | 4 +- rust/op-reth/crates/trie/src/engine/handle.rs | 9 +- rust/op-reth/crates/trie/src/engine/state.rs | 45 ++- .../trie/src/engine/tasks/index_block.rs | 2 +- .../crates/trie/src/engine/tasks/mod.rs | 2 +- .../crates/trie/src/engine/tasks/reorg.rs | 13 +- rust/op-reth/crates/trie/tests/lib.rs | 35 +- rust/op-reth/crates/trie/tests/live.rs | 52 +-- .../trie/tests/live/index_execute_parity.rs | 16 +- .../trie/tests/live/reorg_revert_edges.rs | 356 +++++++++++------- .../crates/trie/tests/live/simulated_reth.rs | 21 +- 13 files changed, 411 insertions(+), 275 deletions(-) diff --git a/rust/op-reth/crates/exex/Cargo.toml b/rust/op-reth/crates/exex/Cargo.toml index 563d10a309a..641e9345697 100644 --- a/rust/op-reth/crates/exex/Cargo.toml +++ b/rust/op-reth/crates/exex/Cargo.toml @@ -32,6 +32,7 @@ futures-util.workspace = true tracing.workspace = true [dev-dependencies] +alloy-eips.workspace = true futures.workspace = true reth-db = { workspace = true, features = ["test-utils"] } reth-optimism-trie = { workspace = true, features = ["test-utils"] } @@ -47,7 +48,6 @@ reth-chain-state.workspace = true reth-chainspec.workspace = true rstest.workspace = true tempfile.workspace = true -alloy-eips.workspace = true [features] test-utils = [ diff --git a/rust/op-reth/crates/exex/src/lib.rs b/rust/op-reth/crates/exex/src/lib.rs index 8d48836f216..df52dcb2082 100644 --- a/rust/op-reth/crates/exex/src/lib.rs +++ b/rust/op-reth/crates/exex/src/lib.rs @@ -442,8 +442,8 @@ where Ok(()) } - /// reth's trie data for `block_number`, unless the block has none or its verification is due, - /// in which case the engine executes the block instead. + /// reth's trie data for `block_number`; `None` when the chain carries none for it or when the + /// verification interval makes this block's execution due. fn precomputed_trie_data<'chain>( &self, chain: &'chain Chain, @@ -457,11 +457,11 @@ where .map(LazyTrieData::get) } - /// Switches the store to `new`: the engine unwinds to the fork block and takes the new - /// blocks in the way [`Self::handle_chain_committed`] would, indexing reth's trie data or - /// executing the block. reth notifies a reorg with both chains on one fork block; two - /// different fork blocks mean the notification is not a reorg the store can follow, so it - /// fails before the engine touches anything. + /// Switches the store to `new`: the engine unwinds to the fork block and takes each new + /// block as [`Self::precomputed_trie_data`] decides, indexing reth's trie data or executing + /// the block. reth notifies a reorg with both chains on one fork block; two different fork + /// blocks mean the notification is not a reorg the store can follow, so it fails before the + /// engine touches anything. fn handle_chain_reorged( &self, old: Arc>, @@ -626,9 +626,9 @@ mod tests { init_storage_at(storage, NumHash::new(0, b256(0x00))); } - fn init_storage_at(storage: S, genesis: NumHash) { + fn init_storage_at(storage: S, anchor: BlockNumHash) { let init = storage.initialization_provider().expect("init"); - init.set_initial_state_anchor(genesis).expect("anchor"); + init.set_initial_state_anchor(anchor).expect("anchor"); init.commit_initial_state().expect("commit init"); OpProofsInitProvider::commit(init).expect("commit"); } @@ -881,14 +881,11 @@ mod tests { block } - /// Empty blocks `numbers` chained on `parent`, which reth's canonical in-memory state takes - /// in as `update` so that the engine finds their parents' state when it executes them. - fn canonical_empty_chain( - ctx: &ExExContext, + /// Empty blocks `numbers` chained on `parent`. + fn empty_chain( parent: BlockNumHash, numbers: RangeInclusive, tag: u8, - update: impl FnOnce(Vec) -> NewCanonicalChain, ) -> Vec> { let mut blocks: Vec> = Vec::new(); let mut parent_hash = parent.hash; @@ -897,31 +894,38 @@ mod tests { parent_hash = block.hash(); blocks.push(block); } - let executed = blocks - .iter() - .map(|block| ExecutedBlock { - recovered_block: Arc::new(block.clone()), - ..Default::default() - }) - .collect(); - let tip = blocks.last().expect("a chain has a tip"); - let in_memory = ctx.components.provider.canonical_in_memory_state(); - in_memory.update_chain(update(executed)); - in_memory.set_canonical_head(SealedHeader::new(tip.header().clone(), tip.hash())); blocks } + /// `block` as reth holds it in memory, with no execution output. + fn executed(block: &RecoveredBlock) -> ExecutedBlock { + ExecutedBlock { recovered_block: Arc::new(block.clone()), ..Default::default() } + } + + /// Makes `update` canonical in reth's in-memory state, so that the engine finds the state of + /// its blocks when it executes their children. + fn make_canonical(ctx: &ExExContext, update: NewCanonicalChain) { + let tip = update.tip(); + let head = SealedHeader::new(tip.header().clone(), tip.hash()); + let in_memory = ctx.components.provider.canonical_in_memory_state(); + in_memory.update_chain(update); + in_memory.set_canonical_head(head); + } + /// A chain of `blocks`, with empty trie data for each when `with_trie_data`. fn chain_of( blocks: Vec>, with_trie_data: bool, ) -> Chain { let first_block = blocks.first().expect("a chain has a first block").number(); - let trie_data = blocks - .iter() - .filter(|_| with_trie_data) - .map(|block| (block.number(), LazyTrieData::ready(ComputedTrieData::default()))) - .collect(); + let trie_data = if with_trie_data { + blocks + .iter() + .map(|block| (block.number(), LazyTrieData::ready(ComputedTrieData::default()))) + .collect() + } else { + BTreeMap::new() + }; let execution_outcome: ExecutionOutcome = ExecutionOutcome { bundle: Default::default(), receipts: Vec::new(), @@ -943,10 +947,7 @@ mod tests { chain_spec: Arc, store: S, ) -> EngineHandle { - let init = store.initialization_provider().expect("init"); - init.set_initial_state_anchor(handle.genesis.num_hash()).expect("anchor"); - init.commit_initial_state().expect("commit init"); - OpProofsInitProvider::commit(init).expect("commit"); + init_storage_at(store.clone(), handle.genesis.num_hash()); let pruner = OpProofStoragePruner::new(store.clone(), ctx.components.provider.clone(), 20); EngineHandle::spawn_with_thresholds( EthEvmConfig::ethereum(chain_spec), @@ -964,10 +965,11 @@ mod tests { handle: &mut TestExExHandle, engine: &EngineHandle, ) -> Vec> { - let blocks = - canonical_empty_chain(&exex.ctx, handle.genesis.num_hash(), 1..=3, 0x10, |new| { - NewCanonicalChain::Commit { new } - }); + let blocks = empty_chain(handle.genesis.num_hash(), 1..=3, 0x10); + make_canonical( + &exex.ctx, + NewCanonicalChain::Commit { new: blocks.iter().map(executed).collect() }, + ); let notification = ExExNotification::ChainCommitted { new: Arc::new(chain_of(blocks.clone(), true)) }; exex.handle_notification(notification, engine).expect("handle chain commit"); @@ -984,18 +986,14 @@ mod tests { let old = commit_three_blocks(&exex, &mut handle, &engine); // reth switches from blocks 2 and 3 to blocks 2' through 4' on block 1. let old_from_fork = old[1..].to_vec(); - let new = canonical_empty_chain(&exex.ctx, old[0].num_hash(), 2..=4, 0x20, |new| { + let new = empty_chain(old[0].num_hash(), 2..=4, 0x20); + make_canonical( + &exex.ctx, NewCanonicalChain::Reorg { - new, - old: old_from_fork - .iter() - .map(|block| ExecutedBlock { - recovered_block: Arc::new(block.clone()), - ..Default::default() - }) - .collect(), - } - }); + new: new.iter().map(executed).collect(), + old: old_from_fork.iter().map(executed).collect(), + }, + ); let new_tip = new[2].num_hash(); let notification = ExExNotification::ChainReorged { old: Arc::new(chain_of(old_from_fork, true)), @@ -1044,13 +1042,7 @@ mod tests { let old = commit_three_blocks(&exex, &mut handle, &engine); let stored_tip = old[2].num_hash(); // The old chain forks at block 1, the new chain at block 2. - let mut new = Vec::new(); - let mut parent_hash = old[1].hash(); - for number in 3..=4 { - let block = empty_block(number, parent_hash, 0x20); - parent_hash = block.hash(); - new.push(block); - } + let new = empty_chain(old[1].num_hash(), 3..=4, 0x20); let notification = ExExNotification::ChainReorged { old: Arc::new(chain_of(old[1..].to_vec(), true)), new: Arc::new(chain_of(new, true)), @@ -1066,6 +1058,31 @@ mod tests { handle.assert_events_empty(); } + /// The trie data of a block is used unless the chain lacks it or the block's verification is + /// due; an interval of zero never makes it due. + #[rstest] + #[case::no_interval_with_data(0, 2, true, true)] + #[case::no_interval_without_data(0, 2, false, false)] + #[case::block_not_due(2, 1, true, true)] + #[case::block_due(2, 2, true, false)] + #[case::block_due_without_data(2, 2, false, false)] + #[tokio::test] + async fn precomputed_trie_data_follows_the_verification_interval( + #[case] verification_interval: u64, + #[case] block_number: u64, + #[case] with_trie_data: bool, + #[case] expected: bool, + ) { + let (ctx, handle) = + reth_exex_test_utils::test_exex_context().await.expect("exex test context"); + let exex = OpProofsExEx::builder(ctx, InMemoryProofsStorage::new()) + .with_verification_interval(verification_interval) + .build(); + let chain = chain_of(empty_chain(handle.genesis.num_hash(), 1..=2, 0x10), with_trie_data); + + assert_eq!(exex.precomputed_trie_data(&chain, block_number).is_some(), expected); + } + #[tokio::test] async fn handle_notification_chain_reorged_skips_beyond_stored_blocks() { // MDBX proofs storage diff --git a/rust/op-reth/crates/trie/src/engine/buffer/state.rs b/rust/op-reth/crates/trie/src/engine/buffer/state.rs index 45b8b3c76bb..bb094abcaea 100644 --- a/rust/op-reth/crates/trie/src/engine/buffer/state.rs +++ b/rust/op-reth/crates/trie/src/engine/buffer/state.rs @@ -9,8 +9,8 @@ use alloy_primitives::{B256, map::HashMap}; use parking_lot::RwLock; use std::{collections::BTreeMap, sync::Arc}; -/// Persisted block hashes the buffer keeps. A reorg reaches at most reth's in-memory chain below -/// its head, far fewer blocks than this. +/// Persisted block hashes the buffer keeps; an unwind onto an older block has only the window's +/// ends to check against. const RETAINED_PERSISTED_HASHES: usize = 1024; /// Buffer for holding blocks waiting to be persisted. diff --git a/rust/op-reth/crates/trie/src/engine/handle.rs b/rust/op-reth/crates/trie/src/engine/handle.rs index 2d2e3aa914c..add5a23a460 100644 --- a/rust/op-reth/crates/trie/src/engine/handle.rs +++ b/rust/op-reth/crates/trie/src/engine/handle.rs @@ -136,9 +136,12 @@ impl EngineHandle } /// Handle a chain reorg: unwind to the common ancestor then buffer the new chain, indexing or - /// executing each block as its [`ReorgBlock`] says. Fails without unwinding when the first - /// block does not build on the stored block below it or when the unwind would reach the - /// window's earliest block. + /// executing each block as its [`ReorgBlock`] says. + /// + /// Fails without unwinding when the unwind would reach the window's earliest block, or when + /// the first block's parent differs from the stored block below it and the engine knows that + /// block's hash (it has buffered or persisted it, or it is a window end). A block that fails + /// after the unwind leaves the store unwound with the blocks before it buffered. pub fn reorg(&self, blocks: Vec>) -> Result<(), EngineError> { self.send_and_recv(|reply| EngineAction::Reorg(ReorgTask { blocks, reply })) } diff --git a/rust/op-reth/crates/trie/src/engine/state.rs b/rust/op-reth/crates/trie/src/engine/state.rs index 8e96ad09a1a..e6a696d7145 100644 --- a/rust/op-reth/crates/trie/src/engine/state.rs +++ b/rust/op-reth/crates/trie/src/engine/state.rs @@ -17,7 +17,7 @@ use reth_provider::{BlockHashReader, BlockReader, DatabaseProviderFactory, Stage use std::time::Duration; #[cfg(feature = "metrics")] use std::time::Instant; -use tracing::{error, info}; +use tracing::{error, info, warn}; /// Tracks all in-flight state for background persistence. pub(crate) struct PersistenceState { @@ -122,18 +122,9 @@ impl PersistenceState { Ok(()) } - /// Wait for any in-flight save, then send an unwind to the persistence service and - /// block until it completes. - pub(crate) fn unwind( - &mut self, - to: BlockWithParent, - memory: &TrieBufferState, - ) -> Result<(), EngineError> { - if self.in_flight.is_some() { - info!(target: "trie::engine::state", "Unwind waiting for in-flight persistence..."); - self.wait(memory); - } - + /// Send an unwind to the persistence service and block until it completes. The caller + /// drains any in-flight save first. + pub(crate) fn unwind(&self, to: BlockWithParent) -> Result<(), EngineError> { let (tx, rx) = bounded(1); self.handle.unwind(to, tx)?; @@ -218,7 +209,7 @@ where // A save in flight can prune the window, so the checks run once it is done. self.drain_persistence(); self.check_unwind_target(to)?; - self.persistence.unwind(to, &self.memory)?; + self.persistence.unwind(to)?; self.memory.unwind(to.block.number); // Clamp `sync_target` to the post-unwind tip. Without this, a stale target left over // from before a deep FCU rewind keeps `needs_sync()` perpetually true and spins the @@ -237,6 +228,13 @@ where fn check_unwind_target(&self, to: BlockWithParent) -> Result<(), EngineError> { let window = self.storage.provider_ro()?.get_proof_window()?; if to.block.number <= window.earliest.number { + warn!( + target: "trie::engine::state", + to_block = to.block.number, + earliest = window.earliest.number, + latest = window.latest.number, + "Refusing to unwind at or below the earliest block" + ); return Err(OpProofsStorageError::UnwindBeyondEarliest { unwind_block_number: to.block.number, earliest_block_number: window.earliest.number, @@ -251,11 +249,20 @@ where .map(|block| block.hash) }); match stored_fork_hash { - Some(stored) if stored != to.parent => Err(EngineError::ParentHashMismatch { - block_number: to.block.number, - expected_parent_hash: stored, - actual_parent_hash: to.parent, - }), + Some(stored) if stored != to.parent => { + warn!( + target: "trie::engine::state", + to_block = to.block.number, + stored_parent = ?stored, + target_parent = ?to.parent, + "Refusing to unwind onto a block the target does not build on" + ); + Err(EngineError::ParentHashMismatch { + block_number: to.block.number, + expected_parent_hash: stored, + actual_parent_hash: to.parent, + }) + } _ => Ok(()), } } diff --git a/rust/op-reth/crates/trie/src/engine/tasks/index_block.rs b/rust/op-reth/crates/trie/src/engine/tasks/index_block.rs index 9e8b526aad8..5fd809be987 100644 --- a/rust/op-reth/crates/trie/src/engine/tasks/index_block.rs +++ b/rust/op-reth/crates/trie/src/engine/tasks/index_block.rs @@ -37,7 +37,7 @@ impl IndexBlockTask { } } -fn run( +pub(crate) fn run( state: &mut EngineState, block: BlockWithParent, sorted_trie_updates: TrieUpdatesSorted, diff --git a/rust/op-reth/crates/trie/src/engine/tasks/mod.rs b/rust/op-reth/crates/trie/src/engine/tasks/mod.rs index 0feb1c75ed7..aaf462c27d4 100644 --- a/rust/op-reth/crates/trie/src/engine/tasks/mod.rs +++ b/rust/op-reth/crates/trie/src/engine/tasks/mod.rs @@ -16,7 +16,7 @@ mod unwind; pub(super) use execute_block::{ExecuteBlockTask, run as execute_block}; #[cfg(any(test, feature = "test-utils"))] pub(super) use flush::FlushTask; -pub(super) use index_block::IndexBlockTask; +pub(super) use index_block::{IndexBlockTask, run as index_block}; pub use reorg::ReorgBlock; pub(super) use reorg::ReorgTask; pub(super) use sync_to::SyncToTask; diff --git a/rust/op-reth/crates/trie/src/engine/tasks/reorg.rs b/rust/op-reth/crates/trie/src/engine/tasks/reorg.rs index 01ce1638e91..13a9fac2fc4 100644 --- a/rust/op-reth/crates/trie/src/engine/tasks/reorg.rs +++ b/rust/op-reth/crates/trie/src/engine/tasks/reorg.rs @@ -1,6 +1,6 @@ use super::super::state::EngineState; use crate::{ - BlockStateDiff, OpProofsStore, + OpProofsStore, engine::{CanonicalInMemoryStateProvider, EngineError}, }; use alloy_eips::eip1898::BlockWithParent; @@ -109,13 +109,12 @@ where for block in blocks { match block { ReorgBlock::Execute(block) => super::execute_block(&block, state)?, - ReorgBlock::Index { block, hashed_state, trie_updates } => state.memory.insert( + ReorgBlock::Index { block, hashed_state, trie_updates } => super::index_block( + state, block, - BlockStateDiff { - sorted_trie_updates: Arc::unwrap_or_clone(trie_updates), - sorted_post_state: Arc::unwrap_or_clone(hashed_state), - }, - ), + Arc::unwrap_or_clone(trie_updates), + Arc::unwrap_or_clone(hashed_state), + )?, } } diff --git a/rust/op-reth/crates/trie/tests/lib.rs b/rust/op-reth/crates/trie/tests/lib.rs index e2b43f66910..1b19760e124 100644 --- a/rust/op-reth/crates/trie/tests/lib.rs +++ b/rust/op-reth/crates/trie/tests/lib.rs @@ -2144,29 +2144,30 @@ fn store_unwind_chain( } /// `unwind_history` refuses a target at or below the earliest block and leaves the window alone. -#[test_case(InMemoryProofsStorage::new(); "InMemory")] -#[test_case(MdbxProofsStorage::new(TempDir::new().unwrap().path()).unwrap(); "Mdbx")] +#[test_case(InMemoryProofsStorage::new(), 2; "InMemory at earliest")] +#[test_case(InMemoryProofsStorage::new(), 1; "InMemory below earliest")] +#[test_case(MdbxProofsStorage::new(TempDir::new().unwrap().path()).unwrap(), 2; "Mdbx at earliest")] +#[test_case(MdbxProofsStorage::new(TempDir::new().unwrap().path()).unwrap(), 1; "Mdbx below earliest")] #[serial] fn test_unwind_history_rejects_target_at_or_below_earliest( storage: S, + target: u64, ) -> Result<(), OpProofsStorageError> { store_unwind_chain(&storage, 2, 5)?; - for target in [2, 1] { - let provider_rw = storage.provider_rw()?; - let err = provider_rw.unwind_history(unwind_chain_block(target)).unwrap_err(); - assert!( - matches!( - err, - OpProofsStorageError::UnwindBeyondEarliest { - unwind_block_number, - earliest_block_number: 2, - } if unwind_block_number == target - ), - "unwinding to {target} gave {err:?}" - ); - provider_rw.commit()?; - } + let provider_rw = storage.provider_rw()?; + let err = provider_rw.unwind_history(unwind_chain_block(target)).unwrap_err(); + assert!( + matches!( + err, + OpProofsStorageError::UnwindBeyondEarliest { + unwind_block_number, + earliest_block_number: 2, + } if unwind_block_number == target + ), + "unwinding to {target} gave {err:?}" + ); + provider_rw.commit()?; let provider = storage.provider_ro()?; assert_eq!(provider.get_earliest_block()?, NumHash::new(2, unwind_chain_hash(2))); diff --git a/rust/op-reth/crates/trie/tests/live.rs b/rust/op-reth/crates/trie/tests/live.rs index 1cfeba2e40e..6c825a23063 100644 --- a/rust/op-reth/crates/trie/tests/live.rs +++ b/rust/op-reth/crates/trie/tests/live.rs @@ -37,24 +37,11 @@ use std::sync::Arc; use tempfile::TempDir; use test_case::test_case; -#[path = "live/contracts.rs"] -mod contracts; -#[path = "live/index_execute_parity.rs"] -mod index_execute_parity; -#[path = "live/parent_states.rs"] -mod parent_states; -#[path = "live/reorg_revert_edges.rs"] -mod reorg_revert_edges; -#[path = "live/simulated_reth.rs"] -mod simulated_reth; -#[path = "live/store_view.rs"] -mod store_view; - /// Seconds between consecutive blocks. const BLOCK_TIME: u64 = 2; -/// Blocks between pruner runs, far beyond any test chain. -const PRUNE_INTERVAL_BLOCKS: u64 = 1000; +/// A proof window no test chain fills, so that the pruner never moves the earliest block. +const WHOLE_WINDOW: u64 = 1000; /// Timestamp of the block at `number`. const fn block_timestamp(number: u64) -> u64 { @@ -80,6 +67,29 @@ enum Backend { Mdbx, } +/// Runs `$run` over a fresh store of `$backend`. +macro_rules! over_backend { + ($backend:expr, $run:ident($($arg:expr),*)) => { + match $backend { + Backend::InMemory => $run(InMemoryProofsStorage::new, $($arg),*), + Backend::Mdbx => $run(create_mdbx_proofs_storage, $($arg),*), + } + }; +} + +#[path = "live/contracts.rs"] +mod contracts; +#[path = "live/index_execute_parity.rs"] +mod index_execute_parity; +#[path = "live/parent_states.rs"] +mod parent_states; +#[path = "live/reorg_revert_edges.rs"] +mod reorg_revert_edges; +#[path = "live/simulated_reth.rs"] +mod simulated_reth; +#[path = "live/store_view.rs"] +mod store_view; + fn create_mdbx_proofs_storage() -> Arc { let path = TempDir::new().unwrap().keep(); Arc::new(MdbxProofsStorage::new(&path).unwrap()) @@ -388,8 +398,7 @@ where // state between iterations (the new async-persistence architecture requires this). let evm_config = EthEvmConfig::ethereum(chain_spec.clone()); let blockchain_db = BlockchainProvider::new(provider_factory.clone())?; - let pruner = - OpProofStoragePruner::new(storage.clone(), blockchain_db.clone(), PRUNE_INTERVAL_BLOCKS); + let pruner = OpProofStoragePruner::new(storage.clone(), blockchain_db.clone(), WHOLE_WINDOW); let engine_handle = EngineHandle::spawn(evm_config, blockchain_db, storage, pruner); for (idx, block_spec) in scenario.blocks_after_initialization.iter().enumerate() { @@ -501,8 +510,7 @@ where ); let blockchain_db = BlockchainProvider::new(provider_factory).unwrap(); - let pruner = - OpProofStoragePruner::new(storage.clone(), blockchain_db.clone(), PRUNE_INTERVAL_BLOCKS); + let pruner = OpProofStoragePruner::new(storage.clone(), blockchain_db.clone(), WHOLE_WINDOW); let engine_handle = EngineHandle::spawn( EthEvmConfig::ethereum(chain_spec.clone()), blockchain_db, @@ -550,8 +558,7 @@ where // Generate a second block normally let blockchain_db = BlockchainProvider::new(provider_factory.clone()).unwrap(); - let pruner = - OpProofStoragePruner::new(storage.clone(), blockchain_db.clone(), PRUNE_INTERVAL_BLOCKS); + let pruner = OpProofStoragePruner::new(storage.clone(), blockchain_db.clone(), WHOLE_WINDOW); let engine_handle = EngineHandle::spawn( EthEvmConfig::ethereum(chain_spec.clone()), blockchain_db, @@ -624,8 +631,7 @@ where )?; let blockchain_db = BlockchainProvider::new(provider_factory.clone()).unwrap(); - let pruner = - OpProofStoragePruner::new(storage.clone(), blockchain_db.clone(), PRUNE_INTERVAL_BLOCKS); + let pruner = OpProofStoragePruner::new(storage.clone(), blockchain_db.clone(), WHOLE_WINDOW); let engine_handle = EngineHandle::spawn( EthEvmConfig::ethereum(chain_spec.clone()), blockchain_db, diff --git a/rust/op-reth/crates/trie/tests/live/index_execute_parity.rs b/rust/op-reth/crates/trie/tests/live/index_execute_parity.rs index 4b7cff921f2..1cb1ed32369 100644 --- a/rust/op-reth/crates/trie/tests/live/index_execute_parity.rs +++ b/rust/op-reth/crates/trie/tests/live/index_execute_parity.rs @@ -16,7 +16,7 @@ use super::{ initcode_returning, slot_writer_calldata, slot_writer_runtime, }, create_mdbx_proofs_storage, funded, initialize_storage, public_key_to_address, - simulated_reth::{RethPersistence, SimulatedReth, fixed_signer}, + simulated_reth::{RethPersistence, SimulatedReth, fixed_signer, index_executed_block}, store_view::{BlockView, ProofTarget, assert_same_view}, }; use alloy_eips::{ @@ -389,13 +389,8 @@ where reth.spawn_engine(executed.clone(), cadence.persistence, cadence.backpressure); for spec in &blocks { let reth_block = reth.commit(spec)?; - let block = reth_block.recovered_block(); - indexed_engine.index_block( - block.block_with_parent(), - (*reth_block.trie_updates()).clone(), - (*reth_block.hashed_state()).clone(), - )?; - executed_engine.execute_block(block)?; + index_executed_block(&indexed_engine, &reth_block)?; + executed_engine.execute_block(reth_block.recovered_block())?; } flush_until_persisted(&indexed_engine, &indexed, reth.head())?; flush_until_persisted(&executed_engine, &executed, reth.head())?; @@ -476,8 +471,5 @@ fn index_execute_and_backfill_stores_agree( #[values(Backend::InMemory, Backend::Mdbx)] backend: Backend, #[case] change: StateChange, ) -> eyre::Result<()> { - match backend { - Backend::InMemory => run_parity_case(InMemoryProofsStorage::new, cadence, change), - Backend::Mdbx => run_parity_case(create_mdbx_proofs_storage, cadence, change), - } + over_backend!(backend, run_parity_case(cadence, change)) } diff --git a/rust/op-reth/crates/trie/tests/live/reorg_revert_edges.rs b/rust/op-reth/crates/trie/tests/live/reorg_revert_edges.rs index c4552d4b08c..6462490adb8 100644 --- a/rust/op-reth/crates/trie/tests/live/reorg_revert_edges.rs +++ b/rust/op-reth/crates/trie/tests/live/reorg_revert_edges.rs @@ -1,7 +1,7 @@ -//! Drives the engine's reorg and unwind tasks at the boundaries the model-based test may not -//! reach: a reorg whose first block does not build on the stored fork block, a target at or -//! below the window's earliest block, a revert that only touches the buffer, and a reorg or a -//! revert that arrives while a save is in flight. +//! Drives the engine's reorg and unwind tasks at boundaries a random chain rarely reaches: a +//! reorg whose first block does not build on the stored fork block, a target at or below the +//! window's earliest block, a revert that only touches the buffer, and a reorg or a revert that +//! arrives while a save is in flight. //! //! Every case commits the same transfer chain to reth, indexes it under the case's flush cadence //! and settles, then acts. The engine's tip is read through the parent check of a probe block, @@ -9,9 +9,9 @@ //! chain once a save has carried the case's blocks to disk. use super::{ - Backend, BlockSpec, FlushCadence, TxSpec, chain_spec_with_address, create_block_from_spec, - create_mdbx_proofs_storage, initialize_storage, public_key_to_address, - simulated_reth::{RethPersistence, SimulatedReth, fixed_signer}, + Backend, BlockSpec, FlushCadence, TxSpec, WHOLE_WINDOW, chain_spec_with_address, + create_block_from_spec, create_mdbx_proofs_storage, initialize_storage, public_key_to_address, + simulated_reth::{RethPersistence, SimulatedReth, fixed_signer, index_executed_block}, store_view::{BlockView, ProofTarget, assert_same_view}, }; use alloy_eips::{NumHash, eip1898::BlockWithParent}; @@ -30,6 +30,7 @@ use rstest::rstest; use secp256k1::Keypair; use serial_test::serial; use std::{ + collections::BTreeMap, fmt, ops::RangeInclusive, sync::{ @@ -37,6 +38,7 @@ use std::{ mpsc::{Receiver, SyncSender, sync_channel}, }, thread, + time::Duration, }; /// Blocks every case commits and settles before it acts. @@ -53,12 +55,12 @@ const RETH_PERSISTENCE_THRESHOLD: u64 = 2 * MAX_BLOCKS_PER_CASE; /// under either cadence. const PRUNED_WINDOW: u64 = 3; -/// A window no chain here fills, so that the earliest block stays the anchor. -const WHOLE_WINDOW: u64 = 1000; - /// A parent hash no block here has. const UNRELATED_PARENT: B256 = B256::repeat_byte(0xBA); +/// How long a case waits for the engine's persistence thread to reach a held write. +const HELD_WRITE_TIMEOUT: Duration = Duration::from_secs(30); + const fn recipient(number: u64) -> Address { Address::repeat_byte(0x40 + number as u8) } @@ -110,21 +112,34 @@ const fn settled_disk_tip(cadence: FlushCadence, base: u64, buffered: u64) -> u6 enum ReorgPath { Execute, Index, + /// Executes every other block, starting with the first, as the ExEx does when the + /// verification interval falls inside the new chain. + Mixed, +} + +impl ReorgPath { + /// The way the ExEx hands the `position`th block of a reorg to the engine along this path. + fn reorg_block(self, position: usize, block: &ExecutedBlock) -> ReorgBlock { + let execute = match self { + Self::Execute => true, + Self::Index => false, + Self::Mixed => position.is_multiple_of(2), + }; + if execute { + ReorgBlock::Execute(Arc::clone(&block.recovered_block)) + } else { + ReorgBlock::Index { + block: block.recovered_block().block_with_parent(), + hashed_state: block.hashed_state(), + trie_updates: block.trie_updates(), + } + } + } } /// `blocks` the way the ExEx hands a reorg to the engine along `path`. fn reorg_blocks(blocks: &[ExecutedBlock], path: ReorgPath) -> Vec> { - blocks - .iter() - .map(|block| match path { - ReorgPath::Execute => ReorgBlock::Execute(Arc::clone(&block.recovered_block)), - ReorgPath::Index => ReorgBlock::Index { - block: block.recovered_block().block_with_parent(), - hashed_state: block.hashed_state(), - trie_updates: block.trie_updates(), - }, - }) - .collect() + blocks.iter().enumerate().map(|(position, block)| path.reorg_block(position, block)).collect() } /// The reorg task's input for `block` without trie data, enough to reach the unwind checks. @@ -138,27 +153,48 @@ fn reorg_block_without_data(block: BlockWithParent) -> ReorgBlock { /// Which engine task unwinds. #[derive(Clone, Copy, Debug)] -enum Task { +enum UnwindingTask { Reorg, Unwind, } +impl UnwindingTask { + fn run(self, engine: &EngineHandle, to: BlockWithParent) -> Result<(), EngineError> { + match self { + Self::Reorg => engine.reorg(vec![reorg_block_without_data(to)]), + Self::Unwind => engine.unwind(to), + } + } +} + /// Where an unwind target sits relative to the proof window. #[derive(Clone, Copy, Debug)] -enum Target { +enum UnwindTarget { /// The init anchor, block 0: the earliest block of a whole window, below a pruned one. Anchor, BelowEarliest, Earliest, } -/// The store's end of a held write. -struct WriteHold { +impl UnwindTarget { + const fn block_number(self, earliest: u64) -> u64 { + match self { + Self::Anchor => 0, + Self::BelowEarliest => { + earliest.checked_sub(1).expect("the window must have been pruned past the anchor") + } + Self::Earliest => earliest, + } + } +} + +/// The store's side of a held write: signals the write and waits to be let through. +struct WriteBarrier { entered: SyncSender<()>, release: Receiver<()>, } -/// The case's end of a held write. +/// The case's side of a held write. struct HeldWrite { entered: Receiver<()>, release: SyncSender<()>, @@ -167,7 +203,9 @@ struct HeldWrite { impl HeldWrite { /// Blocks until the engine's persistence thread is inside the held write. fn wait_until_entered(&self) { - self.entered.recv().expect("the store keeps the hold until a write enters it"); + self.entered + .recv_timeout(HELD_WRITE_TIMEOUT) + .expect("the engine's persistence thread reaches the held write"); } /// Lets the held write go on, or the next write through if none is held yet. @@ -181,7 +219,7 @@ impl HeldWrite { #[derive(Clone)] struct StoreWithWriteGate { inner: S, - next_write: Arc>>, + next_write: Arc>>, } impl StoreWithWriteGate { @@ -192,7 +230,7 @@ impl StoreWithWriteGate { fn hold_next_write(&self) -> HeldWrite { let (entered_tx, entered_rx) = sync_channel(1); let (release_tx, release_rx) = sync_channel(1); - *self.next_write.lock() = Some(WriteHold { entered: entered_tx, release: release_rx }); + *self.next_write.lock() = Some(WriteBarrier { entered: entered_tx, release: release_rx }); HeldWrite { entered: entered_rx, release: release_tx } } } @@ -224,10 +262,12 @@ impl OpProofsStore for StoreWithWriteGate { } fn provider_rw<'a>(&'a self) -> OpProofsStorageResult> { - let hold = self.next_write.lock().take(); - if let Some(hold) = hold { - let _ = hold.entered.send(()); - let _ = hold.release.recv(); + // The lock must be released before blocking, or a case arming the next hold meanwhile + // deadlocks against this thread. + let barrier = self.next_write.lock().take(); + if let Some(barrier) = barrier { + let _ = barrier.entered.send(()); + let _ = barrier.release.recv(); } self.inner.provider_rw() } @@ -239,7 +279,7 @@ impl OpProofsStore for StoreWithWriteGate { /// What the store serves at one moment: its window and every persisted view in it. struct DiskSnapshot { - views: Vec<(u64, BlockView)>, + views: BTreeMap, window: ProofWindowRange, } @@ -248,11 +288,12 @@ struct Scenario { cadence: FlushCadence, chain_spec: Arc, engine: EngineHandle, - gated: StoreWithWriteGate, + gated_store: StoreWithWriteGate, key_pair: Keypair, reth: SimulatedReth, store: S, targets: Vec, + window: u64, } impl Scenario { @@ -269,22 +310,18 @@ impl Scenario { let reth = SimulatedReth::new(chain_spec.clone(), key_pair, persistence)?; let store = make_store(); initialize_storage(store.clone(), &reth.factory, reth.head())?; - let gated = StoreWithWriteGate::new(store.clone()); - let engine = reth.spawn_engine_with_window( - gated.clone(), - cadence.persistence, - cadence.backpressure, - window, - ); + let gated_store = StoreWithWriteGate::new(store.clone()); + let engine = Self::spawn_engine(&reth, &gated_store, cadence, window); let mut scenario = Self { cadence, chain_spec, engine, - gated, + gated_store, key_pair, reth, store, targets: proof_targets(sender), + window, }; for number in 1..=CHAIN_LENGTH { scenario.commit(&chain_block(number))?; @@ -294,21 +331,45 @@ impl Scenario { Ok(scenario) } + fn spawn_engine( + reth: &SimulatedReth, + gated_store: &StoreWithWriteGate, + cadence: FlushCadence, + window: u64, + ) -> EngineHandle { + reth.spawn_engine_with_window( + gated_store.clone(), + cadence.persistence, + cadence.backpressure, + window, + ) + } + + /// Replaces the engine with one started over the same store, as a restart does. + fn restart_engine(self) -> Self { + let Self { + cadence, + chain_spec, + engine, + gated_store, + key_pair, + reth, + store, + targets, + window, + } = self; + drop(engine); + let engine = Self::spawn_engine(&reth, &gated_store, cadence, window); + Self { cadence, chain_spec, engine, gated_store, key_pair, reth, store, targets, window } + } + /// Commits `spec` to reth and indexes it. fn commit(&mut self, spec: &BlockSpec) -> eyre::Result<()> { let block = self.reth.commit(spec)?; - self.index(&block)?; + index_executed_block(&self.engine, &block)?; Ok(()) } - fn index(&self, block: &ExecutedBlock) -> Result<(), EngineError> { - self.engine.index_block( - block.recovered_block().block_with_parent(), - (*block.trie_updates()).clone(), - (*block.hashed_state()).clone(), - ) - } - /// Commits enough blocks for a save to carry everything buffered to disk, and settles. fn commit_through_a_save(&mut self) -> eyre::Result<()> { for _ in 0..self.cadence.persistence { @@ -326,22 +387,22 @@ impl Scenario { for _ in 0..=MAX_BLOCKS_PER_CASE { self.engine.flush(); } - Ok(self.window()?.latest) + Ok(self.proof_window()?.latest) } - fn window(&self) -> eyre::Result { + fn proof_window(&self) -> eyre::Result { Ok(self.store.provider_ro()?.get_proof_window()?) } /// Arms a hold on the engine's next write, the save the following commits start. fn hold_next_save(&self) -> HeldWrite { - self.gated.hold_next_write() + self.gated_store.hold_next_write() } /// Commits blocks until the buffer reaches the persistence threshold and the save it starts /// is inside `hold`. fn commit_until_a_save_is_in_flight(&mut self, hold: &HeldWrite) -> eyre::Result<()> { - let buffered = self.reth.head().number - self.window()?.latest.number; + let buffered = self.reth.head().number - self.proof_window()?.latest.number; for _ in buffered..self.cadence.persistence { let number = self.reth.head().number + 1; self.commit(&chain_block(number))?; @@ -350,16 +411,6 @@ impl Scenario { Ok(()) } - /// Replaces the engine with one started over the same store, as a restart does. - fn restart_engine(&mut self, window: u64) { - self.engine = self.reth.spawn_engine_with_window( - self.gated.clone(), - self.cadence.persistence, - self.cadence.backpressure, - window, - ); - } - /// The canonical block `number` as an unwind target: the target's parent is the block below. fn canonical_target(&self, number: u64) -> BlockWithParent { let parent = match number { @@ -372,14 +423,16 @@ impl Scenario { /// A one-block reorg at `number` whose block builds on a parent no block here has. fn reorg_on_unrelated_parent(&self, number: u64, path: ReorgPath) -> Vec> { let block = match path { - ReorgPath::Execute => ReorgBlock::Execute(Arc::new(create_block_from_spec( - &BlockSpec::empty(), - number, - UNRELATED_PARENT, - &self.chain_spec, - self.key_pair, - &mut 0, - ))), + ReorgPath::Execute | ReorgPath::Mixed => { + ReorgBlock::Execute(Arc::new(create_block_from_spec( + &BlockSpec::empty(), + number, + UNRELATED_PARENT, + &self.chain_spec, + self.key_pair, + &mut 0, + ))) + } ReorgPath::Index => reorg_block_without_data(BlockWithParent::new( UNRELATED_PARENT, NumHash::new(number, B256::repeat_byte(0xB1)), @@ -389,7 +442,7 @@ impl Scenario { } fn disk_snapshot(&self) -> eyre::Result { - let window = self.window()?; + let window = self.proof_window()?; let views = (window.earliest.number..=window.latest.number) .map(|number| Ok((number, BlockView::read(&self.store, number, &self.targets)?))) .collect::>()?; @@ -398,10 +451,10 @@ impl Scenario { /// The store's window and every persisted view are as in `before`. fn assert_disk_unchanged(&self, before: &DiskSnapshot) -> eyre::Result<()> { - assert_eq!(self.window()?, before.window, "proof window"); + assert_eq!(self.proof_window()?, before.window, "proof window"); for (number, expected) in &before.views { let actual = BlockView::read(&self.store, *number, &self.targets)?; - assert_same_view(*number, "unwound", expected, &actual)?; + assert_same_view(*number, "under test", expected, &actual)?; } Ok(()) } @@ -426,7 +479,7 @@ impl Scenario { /// The store's persisted history reaches `through` and is what a fresh store indexed over /// reth's canonical chain serves, with proofs that hash to reth's state roots. fn assert_disk_matches_reth_through(&self, through: u64) -> eyre::Result<()> { - let window = self.window()?; + let window = self.proof_window()?; eyre::ensure!( window.latest.number >= through, "the disk tip {} is below block {through}", @@ -458,12 +511,7 @@ impl Scenario { let engine = self.reth.spawn_engine(store.clone(), 1, 2); let head = self.reth.head().number; for number in 1..=head { - let block = self.reth.executed_block(number); - engine.index_block( - block.recovered_block().block_with_parent(), - (*block.trie_updates()).clone(), - (*block.hashed_state()).clone(), - )?; + index_executed_block(&engine, self.reth.executed_block(number))?; } for _ in 0..=head { engine.flush(); @@ -476,16 +524,6 @@ impl Scenario { } } -/// Runs `$run` over a fresh store of `$backend`. -macro_rules! over_backend { - ($backend:expr, $run:ident($($arg:expr),*)) => { - match $backend { - Backend::InMemory => $run(InMemoryProofsStorage::new, $($arg),*), - Backend::Mdbx => $run(create_mdbx_proofs_storage, $($arg),*), - } - }; -} - fn run_reorg_case( make_store: impl FnOnce() -> S, cadence: FlushCadence, @@ -502,11 +540,12 @@ fn run_reorg_case( scenario.assert_disk_matches_reth_through(fork + 3) } -/// A reorg replaces the stored blocks above the fork with the new chain, indexed or executed, -/// whether the fork is buffered (block 6 under the slower cadence) or on disk. +/// A reorg replaces the stored blocks above the fork with the new chain, indexed, executed or +/// both, whether the fork is buffered (block 6 under the slower cadence) or on disk. #[rstest] #[case::indexed_from_block_six(ReorgPath::Index, 6)] #[case::executed_from_block_three(ReorgPath::Execute, 3)] +#[case::mixed_from_block_four(ReorgPath::Mixed, 4)] #[serial] fn reorg_replaces_the_stored_blocks_above_the_fork( #[values(Backend::InMemory, Backend::Mdbx)] backend: Backend, @@ -567,9 +606,9 @@ fn run_restarted_engine_case( make_store: impl FnOnce() -> S, ) -> eyre::Result<()> { let fork = 2; - let mut scenario = Scenario::new(make_store, FlushCadence::EVERY_BLOCK, WHOLE_WINDOW)?; + let scenario = Scenario::new(make_store, FlushCadence::EVERY_BLOCK, WHOLE_WINDOW)?; let before = scenario.disk_snapshot()?; - scenario.restart_engine(WHOLE_WINDOW); + let scenario = scenario.restart_engine(); let err = scenario .engine @@ -595,26 +634,19 @@ fn restarted_engine_rejects_a_reorg_not_building_on_a_block_below_its_disk_tip( fn run_below_window_case( make_store: impl FnOnce() -> S, cadence: FlushCadence, - task: Task, - target: Target, + task: UnwindingTask, + target: UnwindTarget, window: u64, ) -> eyre::Result<()> { let scenario = Scenario::new(make_store, cadence, window)?; let before = scenario.disk_snapshot()?; let earliest = before.window.earliest.number; - let number = match target { - Target::Anchor => 0, - Target::BelowEarliest => earliest - 1, - Target::Earliest => earliest, - }; - let to = scenario.canonical_target(number); + let number = target.block_number(earliest); - let result = match task { - Task::Reorg => scenario.engine.reorg(vec![reorg_block_without_data(to)]), - Task::Unwind => scenario.engine.unwind(to), - }; + let err = task + .run(&scenario.engine, scenario.canonical_target(number)) + .expect_err("the unwind would remove the window's earliest block"); - let err = result.expect_err("the unwind would remove the window's earliest block"); assert!( matches!( err, @@ -633,16 +665,28 @@ fn run_below_window_case( /// An unwind or a reorg whose target is at or below the window's earliest block, the init anchor /// or a block pruning moved the window past, fails before touching the store or the buffer. #[rstest] -#[case::unwind_at_the_earliest_block(Task::Unwind, Target::Earliest, PRUNED_WINDOW)] -#[case::reorg_below_the_earliest_block(Task::Reorg, Target::BelowEarliest, PRUNED_WINDOW)] -#[case::reorg_at_the_anchor_below_the_window(Task::Reorg, Target::Anchor, PRUNED_WINDOW)] -#[case::unwind_at_the_anchor_as_earliest_block(Task::Unwind, Target::Anchor, WHOLE_WINDOW)] +#[case::unwind_at_the_earliest_block(UnwindingTask::Unwind, UnwindTarget::Earliest, PRUNED_WINDOW)] +#[case::reorg_below_the_earliest_block( + UnwindingTask::Reorg, + UnwindTarget::BelowEarliest, + PRUNED_WINDOW +)] +#[case::reorg_at_the_anchor_below_the_window( + UnwindingTask::Reorg, + UnwindTarget::Anchor, + PRUNED_WINDOW +)] +#[case::unwind_at_the_anchor_as_earliest_block( + UnwindingTask::Unwind, + UnwindTarget::Anchor, + WHOLE_WINDOW +)] #[serial] fn unwind_or_reorg_at_or_below_the_earliest_block_fails_and_changes_nothing( #[values(Backend::InMemory, Backend::Mdbx)] backend: Backend, #[values(FlushCadence::EVERY_BLOCK, FlushCadence::EVERY_FIFTH_BLOCK)] cadence: FlushCadence, - #[case] task: Task, - #[case] target: Target, + #[case] task: UnwindingTask, + #[case] target: UnwindTarget, #[case] window: u64, ) -> eyre::Result<()> { over_backend!(backend, run_below_window_case(cadence, task, target, window)) @@ -653,14 +697,14 @@ fn run_unwind_to_above_earliest_case( cadence: FlushCadence, ) -> eyre::Result<()> { let mut scenario = Scenario::new(make_store, cadence, PRUNED_WINDOW)?; - let earliest = scenario.window()?.earliest; + let earliest = scenario.proof_window()?.earliest; scenario.engine.unwind(scenario.canonical_target(earliest.number + 1))?; - assert_eq!(scenario.window()?, ProofWindowRange { earliest, latest: earliest }); + assert_eq!(scenario.proof_window()?, ProofWindowRange { earliest, latest: earliest }); scenario.assert_tip(earliest); for number in earliest.number + 1..=CHAIN_LENGTH { - scenario.index(scenario.reth.executed_block(number))?; + index_executed_block(&scenario.engine, scenario.reth.executed_block(number))?; } scenario.commit_through_a_save()?; scenario.assert_disk_matches_reth_through(CHAIN_LENGTH) @@ -677,6 +721,57 @@ fn unwind_to_the_block_above_the_earliest_keeps_only_the_earliest( over_backend!(backend, run_unwind_to_above_earliest_case(cadence)) } +fn run_unwind_at_a_block_the_save_prunes_case( + make_store: impl FnOnce() -> S, + cadence: FlushCadence, +) -> eyre::Result<()> { + let mut scenario = Scenario::new(make_store, cadence, PRUNED_WINDOW)?; + let earliest_before = scenario.proof_window()?.earliest.number; + let hold = scenario.hold_next_save(); + scenario.commit_until_a_save_is_in_flight(&hold)?; + let head = scenario.reth.head(); + // The save carries the chain through the head and prunes the window to end at the head. + let target = head.number - PRUNED_WINDOW; + assert!(target > earliest_before, "the target is inside the window until the save prunes it"); + let to = scenario.canonical_target(target); + let engine = scenario.engine.clone(); + + let unwinding = thread::spawn(move || engine.unwind(to)); + hold.release(); + let err = unwinding + .join() + .expect("the unwind thread finishes") + .expect_err("the save moved the earliest block up to the target"); + + assert!( + matches!( + err, + EngineError::Storage(OpProofsStorageError::UnwindBeyondEarliest { + unwind_block_number, + earliest_block_number, + }) if unwind_block_number == target && earliest_block_number == target + ), + "got {err:?}" + ); + assert_eq!( + scenario.proof_window()?, + ProofWindowRange { earliest: scenario.reth.num_hash(target), latest: head } + ); + scenario.assert_tip(head); + scenario.assert_disk_matches_reth_through(head.number) +} + +/// An unwind whose target the in-flight save's pruning moves the window past is refused against +/// the window as the save leaves it, before the persistence service sees the unwind. +#[rstest] +#[serial] +fn unwind_at_a_block_the_in_flight_save_prunes_fails_after_the_save( + #[values(Backend::InMemory, Backend::Mdbx)] backend: Backend, + #[values(FlushCadence::EVERY_BLOCK, FlushCadence::EVERY_FIFTH_BLOCK)] cadence: FlushCadence, +) -> eyre::Result<()> { + over_backend!(backend, run_unwind_at_a_block_the_save_prunes_case(cadence)) +} + fn run_buffered_revert_case( make_store: impl FnOnce() -> S, revert_from: u64, @@ -693,7 +788,7 @@ fn run_buffered_revert_case( let new_chain = scenario.reth.reorg_with(revert_from, &fork_blocks(revert_from..=CHAIN_LENGTH))?; for block in &new_chain { - scenario.index(block)?; + index_executed_block(&scenario.engine, block)?; } scenario.commit_through_a_save()?; scenario.assert_disk_matches_reth_through(CHAIN_LENGTH) @@ -733,17 +828,18 @@ fn run_revert_during_save_case( .reth .reorg_with(disk_tip.number + 1, &fork_blocks(disk_tip.number + 1..=disk_tip.number + 3))?; for block in &new_chain { - scenario.index(block)?; + index_executed_block(&scenario.engine, block)?; } scenario.commit_through_a_save()?; scenario.assert_disk_matches_reth_through(disk_tip.number + 3) } -/// A revert of every block above the disk tip while their save is in flight waits for the save, -/// then removes them again: the disk ends as it was and the tip is the disk tip. +/// A revert of every block above the disk tip, sent while their save is in flight, ends with the +/// disk as it was and the tip at the disk tip: the engine either drains the save before the +/// unwind or sees its completion first, and both orders must end the same. #[rstest] #[serial] -fn revert_during_an_in_flight_save_waits_for_the_save( +fn revert_sent_during_an_in_flight_save_ends_with_the_disk_as_it_was( #[values(Backend::InMemory, Backend::Mdbx)] backend: Backend, #[values(FlushCadence::EVERY_BLOCK, FlushCadence::EVERY_FIFTH_BLOCK)] cadence: FlushCadence, ) -> eyre::Result<()> { @@ -756,7 +852,7 @@ fn run_reorg_during_save_case( path: ReorgPath, ) -> eyre::Result<()> { let mut scenario = Scenario::new(make_store, cadence, WHOLE_WINDOW)?; - let fork = scenario.window()?.latest.number - 2; + let fork = scenario.proof_window()?.latest.number - 2; let hold = scenario.hold_next_save(); scenario.commit_until_a_save_is_in_flight(&hold)?; let new_chain = scenario.reth.reorg_with(fork + 1, &fork_blocks(fork + 1..=fork + 3))?; @@ -775,12 +871,12 @@ fn run_reorg_during_save_case( scenario.assert_disk_matches_reth_through(fork + 3) } -/// A reorg from a fork on disk while a save of the blocks above it is in flight waits for the -/// save, unwinds the disk to the fork and buffers the new chain, so that the disk and the buffer -/// agree with reth. +/// A reorg from a fork on disk, sent while a save of the blocks above it is in flight, ends with +/// the disk unwound to the fork and the new chain buffered, so that the disk and the buffer agree +/// with reth whichever of the save and the reorg the engine handles first. #[rstest] #[serial] -fn reorg_from_a_disk_fork_during_an_in_flight_save_waits_for_the_save( +fn reorg_sent_during_an_in_flight_save_ends_consistent_with_reth( #[values(Backend::InMemory, Backend::Mdbx)] backend: Backend, #[values(FlushCadence::EVERY_BLOCK, FlushCadence::EVERY_FIFTH_BLOCK)] cadence: FlushCadence, #[values(ReorgPath::Index, ReorgPath::Execute)] path: ReorgPath, diff --git a/rust/op-reth/crates/trie/tests/live/simulated_reth.rs b/rust/op-reth/crates/trie/tests/live/simulated_reth.rs index 2e9e4a467b3..bb82865589e 100644 --- a/rust/op-reth/crates/trie/tests/live/simulated_reth.rs +++ b/rust/op-reth/crates/trie/tests/live/simulated_reth.rs @@ -1,7 +1,7 @@ //! reth's database and canonical in-memory state, driven the way reth's engine tree and //! persistence task drive them. -use super::{BlockSpec, PRUNE_INTERVAL_BLOCKS, create_block_from_spec}; +use super::{BlockSpec, WHOLE_WINDOW, create_block_from_spec}; use alloy_consensus::BlockHeader; use alloy_eips::BlockNumHash; use alloy_primitives::B256; @@ -11,7 +11,10 @@ use reth_db_common::init::init_genesis; use reth_ethereum_primitives::Block; use reth_evm::{ConfigureEvm, execute::Executor}; use reth_evm_ethereum::EthEvmConfig; -use reth_optimism_trie::{OpProofStoragePruner, OpProofsStore, engine::EngineHandle}; +use reth_optimism_trie::{ + OpProofStoragePruner, OpProofsStore, + engine::{EngineError, EngineHandle}, +}; use reth_primitives_traits::RecoveredBlock; use reth_provider::{ HashedPostStateProvider, ProviderFactory, SaveBlocksInput, StateProviderFactory, @@ -28,6 +31,18 @@ use std::sync::Arc; /// reth's default `--engine.memory-block-buffer-target`. const MEMORY_BLOCK_BUFFER_TARGET: u64 = 5; +/// Hands `block` to `engine` the way the ExEx indexes a block reth validated. +pub(super) fn index_executed_block( + engine: &EngineHandle, + block: &ExecutedBlock, +) -> Result<(), EngineError> { + engine.index_block( + block.recovered_block().block_with_parent(), + (*block.trie_updates()).clone(), + (*block.hashed_state()).clone(), + ) +} + /// A fixed signer, so a failing case reproduces with the same addresses and trie shape. pub(super) fn fixed_signer() -> Keypair { Keypair::from_seckey_byte_array(&Secp256k1::new(), [0x11; 32]).expect("valid secret key") @@ -190,7 +205,7 @@ impl SimulatedReth { store, persistence_threshold, backpressure_threshold, - PRUNE_INTERVAL_BLOCKS, + WHOLE_WINDOW, ) } From 78665253f800ae8b3d5676e0e8216b9e3267b975 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Thu, 15 Oct 2026 21:10:19 -0700 Subject: [PATCH 3/4] op-reth: persist the simulated reth through the proofs window's earliest block The reorg and revert edge cases expect every save to prune the store to `disk tip - window`. A pruner that holds the window back at reth's persisted head (the reth 2.6 pin's, #23358) pruned nothing, because the simulated reth never persisted: the cases that unwind at or below the earliest block found the window unpruned, and the unwind at the block an in-flight save prunes went through. The scenario now persists reth through the block each expected save prunes the window to before it indexes the blocks, so that both pruners prune to the same height, and the reference store starts at reth's database tip, the oldest state reth's database holds, instead of genesis. Co-Authored-By: Claude Fable 5.1 (cherry picked from commit 5c55d97557c16b7cec4fa9406607682f840357a3) (cherry picked from commit fdf9fad8bfca21fbbcdd30755aeb5d6e52343923) --- .../trie/tests/live/reorg_revert_edges.rs | 69 +++++++++++-------- .../crates/trie/tests/live/simulated_reth.rs | 17 ++++- 2 files changed, 56 insertions(+), 30 deletions(-) diff --git a/rust/op-reth/crates/trie/tests/live/reorg_revert_edges.rs b/rust/op-reth/crates/trie/tests/live/reorg_revert_edges.rs index 6462490adb8..88d3155eefa 100644 --- a/rust/op-reth/crates/trie/tests/live/reorg_revert_edges.rs +++ b/rust/op-reth/crates/trie/tests/live/reorg_revert_edges.rs @@ -47,8 +47,9 @@ const CHAIN_LENGTH: u64 = 7; /// Blocks a case may hold at most, which bounds the flushes a settle needs and the recipients. const MAX_BLOCKS_PER_CASE: u64 = 2 * CHAIN_LENGTH; -/// reth's persistence threshold, above any chain here so that reth keeps every block in memory, -/// where `reorg_with` can fork it anywhere and the reference store still finds genesis state. +/// reth's persistence threshold, above any chain here so that reth persists nothing on its own: +/// a case persists reth through the block its window starts at, which keeps `reorg_with` free +/// to fork the chain anywhere above. const RETH_PERSISTENCE_THRESHOLD: u64 = 2 * MAX_BLOCKS_PER_CASE; /// Blocks the pruned window keeps, so that pruning moves the earliest block above the anchor @@ -309,7 +310,7 @@ impl Scenario { let persistence = RethPersistence::threshold(RETH_PERSISTENCE_THRESHOLD); let reth = SimulatedReth::new(chain_spec.clone(), key_pair, persistence)?; let store = make_store(); - initialize_storage(store.clone(), &reth.factory, reth.head())?; + initialize_storage(store.clone(), &reth.factory, reth.genesis())?; let gated_store = StoreWithWriteGate::new(store.clone()); let engine = Self::spawn_engine(&reth, &gated_store, cadence, window); let mut scenario = Self { @@ -323,11 +324,10 @@ impl Scenario { targets: proof_targets(sender), window, }; - for number in 1..=CHAIN_LENGTH { - scenario.commit(&chain_block(number))?; - } + let settled_tip = settled_disk_tip(cadence, 0, CHAIN_LENGTH); + scenario.commit_blocks_toward_a_save_at(CHAIN_LENGTH, settled_tip)?; let disk_tip = scenario.settle()?; - assert_eq!(disk_tip.number, settled_disk_tip(cadence, 0, CHAIN_LENGTH), "settled disk tip"); + assert_eq!(disk_tip.number, settled_tip, "settled disk tip"); Ok(scenario) } @@ -363,19 +363,31 @@ impl Scenario { Self { cadence, chain_spec, engine, gated_store, key_pair, reth, store, targets, window } } - /// Commits `spec` to reth and indexes it. - fn commit(&mut self, spec: &BlockSpec) -> eyre::Result<()> { - let block = self.reth.commit(spec)?; - index_executed_block(&self.engine, &block)?; + /// Commits `count` chain blocks to reth, persists reth through the block the save landing at + /// `save_tip` prunes the window to, and indexes the blocks. reth must be persisted exactly + /// through that block: any lower and a pruner holding the window back at reth's persisted + /// head prunes less than one that does not; any higher and the reference store, which starts + /// at reth's database tip, cannot serve the earliest block. + fn commit_blocks_toward_a_save_at(&mut self, count: u64, save_tip: u64) -> eyre::Result<()> { + let blocks = + (0..count).map(|_| self.commit_chain_block()).collect::>>()?; + self.reth.persist_through(save_tip.saturating_sub(self.window))?; + for block in &blocks { + index_executed_block(&self.engine, block)?; + } Ok(()) } + /// Commits the next block of the chain to reth. + fn commit_chain_block(&mut self) -> eyre::Result { + let number = self.reth.head().number + 1; + self.reth.commit(&chain_block(number)) + } + /// Commits enough blocks for a save to carry everything buffered to disk, and settles. fn commit_through_a_save(&mut self) -> eyre::Result<()> { - for _ in 0..self.cadence.persistence { - let number = self.reth.head().number + 1; - self.commit(&chain_block(number))?; - } + let save_tip = self.settle()?.number + self.cadence.persistence; + self.commit_blocks_toward_a_save_at(self.cadence.persistence, save_tip)?; self.settle()?; Ok(()) } @@ -402,11 +414,9 @@ impl Scenario { /// Commits blocks until the buffer reaches the persistence threshold and the save it starts /// is inside `hold`. fn commit_until_a_save_is_in_flight(&mut self, hold: &HeldWrite) -> eyre::Result<()> { - let buffered = self.reth.head().number - self.proof_window()?.latest.number; - for _ in buffered..self.cadence.persistence { - let number = self.reth.head().number + 1; - self.commit(&chain_block(number))?; - } + let save_tip = self.proof_window()?.latest.number + self.cadence.persistence; + let count = save_tip - self.reth.head().number; + self.commit_blocks_toward_a_save_at(count, save_tip)?; hold.wait_until_entered(); Ok(()) } @@ -490,6 +500,13 @@ impl Scenario { "the disk tip {:?} is not canonical in reth", window.latest ); + let database_tip = self.reth.persisted().number; + eyre::ensure!( + window.earliest.number >= database_tip, + "the earliest block {} is below reth's database tip {database_tip}, where the \ + reference store starts", + window.earliest.number + ); let reference = self.reference_store()?; for number in window.earliest.number..=window.latest.number { let expected = BlockView::read(&reference, number, &self.targets)?; @@ -500,17 +517,15 @@ impl Scenario { Ok(()) } - /// A fresh store indexed over reth's canonical chain from genesis. + /// A fresh store initialized at reth's database tip, the only block whose trie reth's + /// database holds, and indexed over reth's canonical chain above it. fn reference_store(&self) -> eyre::Result { - eyre::ensure!( - !self.reth.locate(self.reth.num_hash(1)).persisted, - "reth's database must still hold genesis state" - ); + let anchor = self.reth.persisted(); let store = InMemoryProofsStorage::new(); - initialize_storage(store.clone(), &self.reth.factory, self.reth.genesis())?; + initialize_storage(store.clone(), &self.reth.factory, anchor)?; let engine = self.reth.spawn_engine(store.clone(), 1, 2); let head = self.reth.head().number; - for number in 1..=head { + for number in anchor.number + 1..=head { index_executed_block(&engine, self.reth.executed_block(number))?; } for _ in 0..=head { diff --git a/rust/op-reth/crates/trie/tests/live/simulated_reth.rs b/rust/op-reth/crates/trie/tests/live/simulated_reth.rs index bb82865589e..04aacd3e6b0 100644 --- a/rust/op-reth/crates/trie/tests/live/simulated_reth.rs +++ b/rust/op-reth/crates/trie/tests/live/simulated_reth.rs @@ -170,6 +170,11 @@ impl SimulatedReth { self.num_hash(0) } + /// Newest block in the database. + pub(super) const fn persisted(&self) -> BlockNumHash { + self.last_persisted + } + pub(super) fn state_root(&self, number: u64) -> B256 { match number { 0 => self.chain_spec.genesis_header().state_root, @@ -306,9 +311,15 @@ impl SimulatedReth { /// Writes every canonical block to the database without evicting any from memory, the state /// a `proofs init` run finds on a node that has caught up. pub(super) fn persist_all(&mut self) -> eyre::Result<()> { - let head = self.head().number; - if head > self.last_persisted.number { - self.persist(head)?; + self.persist_through(self.head().number) + } + + /// Writes the canonical blocks through `number` to the database without evicting any from + /// memory. + pub(super) fn persist_through(&mut self, number: u64) -> eyre::Result<()> { + assert!(number <= self.head().number, "only canonical blocks can be persisted"); + if number > self.last_persisted.number { + self.persist(number)?; } Ok(()) } From 38e8eca25544039543ffc8fbad1de2c4bea4a50f Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Mon, 9 Nov 2026 20:46:48 +0100 Subject: [PATCH 4/4] op-reth: adapt the proofs engine edge tests to the reth v2.6.0 bump The ExEx keeps alloy-eips as a regular dependency, the sorted trie types the bump dropped leave the top-level import, and the bump's canonical-block test helper takes a name that does not collide with the reorg tests' make_canonical. Co-Authored-By: Claude Fable 5.1 --- rust/Cargo.lock | 3 +++ rust/op-reth/crates/exex/Cargo.toml | 2 +- rust/op-reth/crates/exex/src/lib.rs | 11 ++++------- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/rust/Cargo.lock b/rust/Cargo.lock index fa72de56c71..cc19affc315 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -13951,6 +13951,8 @@ dependencies = [ "eyre", "futures", "futures-util", + "reth-chain-state", + "reth-chainspec", "reth-db", "reth-ethereum-primitives", "reth-evm-ethereum", @@ -13965,6 +13967,7 @@ dependencies = [ "reth-primitives-traits", "reth-provider", "reth-trie", + "rstest", "tempfile", "tokio", "tracing", diff --git a/rust/op-reth/crates/exex/Cargo.toml b/rust/op-reth/crates/exex/Cargo.toml index 641e9345697..e9371534b18 100644 --- a/rust/op-reth/crates/exex/Cargo.toml +++ b/rust/op-reth/crates/exex/Cargo.toml @@ -25,6 +25,7 @@ reth-optimism-trie = { workspace = true, features = ["serde-bincode-compat"] } # alloy alloy-consensus.workspace = true +alloy-eips.workspace = true # misc eyre.workspace = true @@ -32,7 +33,6 @@ futures-util.workspace = true tracing.workspace = true [dev-dependencies] -alloy-eips.workspace = true futures.workspace = true reth-db = { workspace = true, features = ["test-utils"] } reth-optimism-trie = { workspace = true, features = ["test-utils"] } diff --git a/rust/op-reth/crates/exex/src/lib.rs b/rust/op-reth/crates/exex/src/lib.rs index df52dcb2082..5f728d535a8 100644 --- a/rust/op-reth/crates/exex/src/lib.rs +++ b/rust/op-reth/crates/exex/src/lib.rs @@ -21,10 +21,7 @@ use reth_optimism_trie::{ proof::DatabaseStateRoot, }; use reth_provider::{BlockHashReader, BlockNumReader, HeaderProvider}; -use reth_trie::{ - ComputedTrieData, HashedPostState, HashedPostStateSorted, LazyTrieData, SortedTrieData, - StateRoot, updates::TrieUpdatesSorted, -}; +use reth_trie::{ComputedTrieData, HashedPostState, LazyTrieData, SortedTrieData, StateRoot}; use std::sync::Arc; use tracing::{debug, info, warn}; @@ -1385,7 +1382,7 @@ mod tests { } /// Makes `block` reth's canonical block at its height. - fn make_canonical( + fn make_canonical_block( handle: &reth_exex_test_utils::TestExExHandle, provider: &impl reth_provider::CanonChainTracker
, block: &RecoveredBlock, @@ -1432,7 +1429,7 @@ mod tests { // reth's canonical block 1 has the state root execution produces, standing in for the // block that replaced the notification's block in a reorg. let canonical = block_one_with_state_root(current_state_hash); - make_canonical(&handle, &provider, &canonical); + make_canonical_block(&handle, &provider, &canonical); let notif = ExExNotification::ChainCommitted { new: Arc::new(mk_chain_without_updates(1, 1)) }; @@ -1494,7 +1491,7 @@ mod tests { // the one execution produces. Executing it again would fail the same way, so the failure // stops the ExEx. let block = mk_block(1); - make_canonical(&handle, &provider, &block); + make_canonical_block(&handle, &provider, &block); let chain = if with_trie_data { mk_chain_with_updates(1, 1, None) } else {