diff --git a/rust/.config/nextest.toml b/rust/.config/nextest.toml index 9cb49f19920..50a953a4333 100644 --- a/rust/.config/nextest.toml +++ b/rust/.config/nextest.toml @@ -31,6 +31,12 @@ filter = 'test(=log_estimate_bounds_measured_pgu) | test(=consolidation_tests::c slow-timeout = { period = "1m", terminate-after = 10 } threads-required = "num-test-threads" +# The proofs engine model test runs its whole proptest as one test, and a failing case shrinks +# for up to a minute on top of the run. +[[profile.default.overrides]] +filter = "binary(engine_model)" +slow-timeout = { period = "1m", terminate-after = 5 } + # E2E tests using the testsuite framework from crates/e2e-test-utils. [[profile.default.overrides]] filter = "binary(e2e_testsuite)" diff --git a/rust/Cargo.lock b/rust/Cargo.lock index cc19affc315..fcee1c9c24e 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -14312,6 +14312,7 @@ dependencies = [ "metrics", "mockall", "parking_lot", + "proptest", "reth-chain-state", "reth-chainspec", "reth-codecs", diff --git a/rust/op-reth/crates/trie/Cargo.toml b/rust/op-reth/crates/trie/Cargo.toml index 5ba6a8d34c8..5fd15bfa395 100644 --- a/rust/op-reth/crates/trie/Cargo.toml +++ b/rust/op-reth/crates/trie/Cargo.toml @@ -82,6 +82,7 @@ secp256k1 = { workspace = true, features = ["rand", "std"] } mockall.workspace = true rstest.workspace = true eyre.workspace = true +proptest.workspace = true # misc serial_test.workspace = true diff --git a/rust/op-reth/crates/trie/src/engine/handle.rs b/rust/op-reth/crates/trie/src/engine/handle.rs index add5a23a460..4b80a7ef0be 100644 --- a/rust/op-reth/crates/trie/src/engine/handle.rs +++ b/rust/op-reth/crates/trie/src/engine/handle.rs @@ -9,6 +9,8 @@ use super::{ tasks::{ExecuteBlockTask, IndexBlockTask, ReorgBlock, ReorgTask, SyncToTask, UnwindTask}, }; use crate::{OpProofStoragePruner, OpProofsStore}; +#[cfg(any(test, feature = "test-utils"))] +use alloy_eips::NumHash; use alloy_eips::eip1898::BlockWithParent; use crossbeam_channel::{Sender, bounded}; use reth_evm::ConfigureEvm; @@ -170,4 +172,16 @@ impl EngineHandle let _ = reply_rx.recv(); } } + + /// The engine's tip: the newest buffered block, or the store's latest block when nothing is + /// buffered (test/utility only). + #[cfg(any(test, feature = "test-utils"))] + pub fn tip(&self) -> Result { + use super::tasks::TipTask; + let (reply_tx, reply_rx) = bounded(1); + self.sender + .send(EngineAction::Tip(TipTask { reply: reply_tx })) + .map_err(|_| EngineError::EngineDied)?; + reply_rx.recv().map_err(|_| EngineError::EngineDied)? + } } diff --git a/rust/op-reth/crates/trie/src/engine/mod.rs b/rust/op-reth/crates/trie/src/engine/mod.rs index 267a9cb21f6..975c39a5544 100644 --- a/rust/op-reth/crates/trie/src/engine/mod.rs +++ b/rust/op-reth/crates/trie/src/engine/mod.rs @@ -60,6 +60,9 @@ enum EngineAction { Flush(tasks::FlushTask), /// Update the sync catch-up target (fire-and-forget). SyncTo(tasks::SyncToTask), + /// Report the engine's tip. + #[cfg(any(test, feature = "test-utils"))] + Tip(tasks::TipTask), } impl EngineAction { @@ -85,6 +88,8 @@ impl EngineAction { #[cfg(any(test, feature = "test-utils"))] Self::Flush(task) => task.execute(state), Self::SyncTo(task) => task.execute(state), + #[cfg(any(test, feature = "test-utils"))] + Self::Tip(task) => task.execute(state), } } } diff --git a/rust/op-reth/crates/trie/src/engine/tasks/mod.rs b/rust/op-reth/crates/trie/src/engine/tasks/mod.rs index aaf462c27d4..8091e14a82b 100644 --- a/rust/op-reth/crates/trie/src/engine/tasks/mod.rs +++ b/rust/op-reth/crates/trie/src/engine/tasks/mod.rs @@ -11,6 +11,8 @@ mod index_block; mod reorg; mod reth_auxiliary; mod sync_to; +#[cfg(any(test, feature = "test-utils"))] +mod tip; mod unwind; pub(super) use execute_block::{ExecuteBlockTask, run as execute_block}; @@ -20,4 +22,6 @@ pub(super) use index_block::{IndexBlockTask, run as index_block}; pub use reorg::ReorgBlock; pub(super) use reorg::ReorgTask; pub(super) use sync_to::SyncToTask; +#[cfg(any(test, feature = "test-utils"))] +pub(super) use tip::TipTask; pub(super) use unwind::UnwindTask; diff --git a/rust/op-reth/crates/trie/src/engine/tasks/tip.rs b/rust/op-reth/crates/trie/src/engine/tasks/tip.rs new file mode 100644 index 00000000000..bcf714315fd --- /dev/null +++ b/rust/op-reth/crates/trie/src/engine/tasks/tip.rs @@ -0,0 +1,31 @@ +use super::super::state::EngineState; +use crate::{ + OpProofsStore, + engine::{CanonicalInMemoryStateProvider, EngineError}, +}; +use alloy_eips::NumHash; +use crossbeam_channel::Sender; +use reth_evm::ConfigureEvm; +use reth_primitives_traits::BlockTy; +use reth_provider::{BlockHashReader, BlockReader, DatabaseProviderFactory, StageCheckpointReader}; + +pub(crate) struct TipTask { + pub(crate) reply: Sender>, +} + +impl TipTask { + pub(crate) fn execute(self, state: &EngineState) + where + Evm: ConfigureEvm, + Provider: BlockHashReader + + CanonicalInMemoryStateProvider + + StageCheckpointReader + + DatabaseProviderFactory + + BlockReader> + + Clone + + 'static, + Store: OpProofsStore + Clone + 'static, + { + let _ = self.reply.send(state.get_tip()); + } +} diff --git a/rust/op-reth/crates/trie/tests/engine_model.rs b/rust/op-reth/crates/trie/tests/engine_model.rs new file mode 100644 index 00000000000..e80e75b9ccd --- /dev/null +++ b/rust/op-reth/crates/trie/tests/engine_model.rs @@ -0,0 +1,123 @@ +//! Model-based test of the proofs engine: random interleavings of commits, reorgs, reverts, sync +//! targets, flushes, failed saves, stale notifications and crashes, driven over reth's +//! persistence model and held to a reference model after every step. The deterministic boundary +//! cases live in the live target; this target is for the interleavings. + +use proptest::prelude::*; + +#[path = "engine_model/actions.rs"] +mod actions; +#[path = "live/chain.rs"] +#[allow(dead_code, reason = "each test target uses part of this module")] +mod chain; +#[path = "engine_model/faults.rs"] +mod faults; +#[path = "engine_model/harness.rs"] +mod harness; +#[path = "engine_model/invariants.rs"] +mod invariants; +#[path = "engine_model/model.rs"] +mod model; +#[path = "live/simulated_reth.rs"] +#[allow(dead_code, reason = "each test target uses part of this module")] +mod simulated_reth; +#[path = "live/store_view.rs"] +#[allow(dead_code, reason = "each test target uses part of this module")] +mod store_view; + +use actions::{Action, Actions, Config, Rebuild}; +use chain::*; +use harness::Harness; +use proptest::test_runner::contextualize_config; +use simulated_reth::RethPersistence; +use std::sync::atomic::{AtomicBool, Ordering}; + +/// Cases per run unless `PROPTEST_CASES` says otherwise. +const DEFAULT_CASES: u32 = 24; + +/// The default count and a shrinking budget that fits the test's time limit, with the +/// `PROPTEST_*` environment variables applied on top. +fn proptest_config() -> ProptestConfig { + contextualize_config(ProptestConfig { + cases: DEFAULT_CASES, + failure_persistence: None, + max_shrink_iters: 256, + max_shrink_time: 60_000, + ..ProptestConfig::default() + }) +} + +fn run_sequence(config: Config, actions: &[Action]) -> eyre::Result<()> { + Harness::new(config)?.run(actions) +} + +static FIRST_FAILURE_PRINTED: AtomicBool = AtomicBool::new(false); + +/// Runs a case and prints the first failure as found, before shrinking, so that a run cut short +/// by a time limit still shows what failed. +fn run_case(config: Config, actions: &[Action]) -> Result<(), TestCaseError> { + run_sequence(config, actions).map_err(|err| { + if !FIRST_FAILURE_PRINTED.swap(true, Ordering::SeqCst) { + eprintln!("first failing case, before shrinking: {config:?}\n{err:?}\n"); + } + TestCaseError::fail(format!("{err:?}")) + }) +} + +proptest! { + #![proptest_config(proptest_config())] + + /// Under every cell of the matrix, the engine's tip and the store's window stay on the + /// canonical chain and within what the model allows after each random step, every valid + /// event is taken, every reorg on a wrong parent or below the window is refused, and the + /// engine settles within the step bound. + #[test] + fn engine_and_store_follow_the_model( + config in actions::config(), + actions in actions::sequence(Actions::DEFAULT), + ) { + run_case(config, &actions)?; + } + + /// As above, with crashes after which reth rebuilds a different chain, as a sequencer does. + #[test] + #[ignore = "fails on develop: after an unclean stop the store keeps blocks reth lost, and a chain rebuilt differently leaves the store's latest block off the canonical chain (ethereum-optimism/protocol-team#464; the harness's restart must gain the startup reconcile of #23358 to pass)"] + fn engine_and_store_follow_the_model_through_crashes_rebuilding_a_different_chain( + config in actions::config(), + actions in actions::sequence(Actions::WITH_TRACKED_BUGS), + ) { + run_case(config, &actions)?; + } +} + +/// The matrix cell the replayed sequences run under. +fn replay_config(cadence: FlushCadence, persistence_threshold: u64) -> Config { + Config { + body_pruning: false, + cadence, + persistence: RethPersistence::threshold(persistence_threshold), + verification_interval: 0, + window: WHOLE_WINDOW, + } +} + +/// The minimal sequence the random run shrinks to for protocol-team#464: one block saved, then a +/// crash reth comes back from with a different block 1, which the store keeps serving. +#[test] +#[ignore = "fails on develop: the store's latest block 1 is not on the chain reth rebuilt after the crash (ethereum-optimism/protocol-team#464; the harness's restart must gain the startup reconcile of #23358 to pass)"] +fn store_follows_reth_through_a_crash_rebuilding_a_different_chain() -> eyre::Result<()> { + run_sequence( + replay_config(FlushCadence::EVERY_BLOCK, 50), + &[Action::Commit { count: 1 }, Action::Crash(Rebuild::Different)], + ) +} + +/// The ExEx acknowledges a notification's tip before the sync loop has executed the bodies +/// below it, so a reth that prunes bodies through the acknowledged height takes them from under +/// the sync loop and the catch-up stalls. +#[test] +#[ignore = "fails on develop: reth prunes the bodies the sync loop still needs because the ExEx acknowledges the notification tip, and the engine never settles (ethereum-optimism/protocol-team#478)"] +fn sync_loop_completes_while_reth_prunes_bodies_through_the_acknowledged_tip() -> eyre::Result<()> { + let config = Config { body_pruning: true, ..replay_config(FlushCadence::EVERY_BLOCK, 7) }; + run_sequence(config, &[Action::SyncTo { count: 40 }, Action::Commit { count: 1 }]) +} diff --git a/rust/op-reth/crates/trie/tests/engine_model/actions.rs b/rust/op-reth/crates/trie/tests/engine_model/actions.rs new file mode 100644 index 00000000000..54d3e78a641 --- /dev/null +++ b/rust/op-reth/crates/trie/tests/engine_model/actions.rs @@ -0,0 +1,126 @@ +//! What a step can do to reth, the ExEx and the node, and the configuration a run samples. + +use super::{FlushCadence, WHOLE_WINDOW, simulated_reth::RethPersistence}; +use proptest::{collection::vec, prelude::*, sample::select}; + +/// Steps per case at most. +const MAX_STEPS: usize = 24; + +/// A proof window the chains here outgrow, so that pruning moves the earliest block. +const PRUNED_WINDOW: u64 = 4; + +/// How reth rebuilds the blocks a crash lost: from the same consensus-layer chain, as a verifier +/// does, or as different blocks, as a sequencer does. +#[derive(Clone, Copy, Debug)] +pub(crate) enum Rebuild { + Different, + Same, +} + +#[derive(Clone, Copy, Debug)] +pub(crate) enum Action { + /// A one-block reorg on a parent no block has, at a fork point the engine knows, which it + /// must refuse. + BogusReorg { offset: u64 }, + /// reth commits `count` blocks, each delivered to the engine as a live notification. + Commit { count: u64 }, + /// reth and the engine stop uncleanly, losing what either held in memory, and restart. + Crash(Rebuild), + /// The next save fails at its write transaction, and enough blocks follow to start it. + FailNextSave, + /// Waits for the save in flight, if any. + Flush, + /// reth replaces its newest `depth` in-memory blocks with `length` new ones and delivers + /// the reorg. + Reorg { depth: u64, length: u64 }, + /// reth drops its newest `depth` in-memory blocks and delivers the revert. + Revert { depth: u64 }, + /// reth commits `depth` blocks and reorgs them away before their notifications reach the + /// engine, then the reorg's does. + StaleCommit { depth: u64, length: u64 }, + /// reth commits `count` blocks the engine only learns of through a sync target, as the ExEx + /// hands over a notification far behind reth's head. + SyncTo { count: u64 }, +} + +/// Which extra actions a run may take. +#[derive(Clone, Copy, Debug)] +pub(crate) struct Actions { + /// A crash after which reth rebuilds a different chain (ethereum-optimism/protocol-team#464). + pub(crate) crash_rebuilding_a_different_chain: bool, +} + +impl Actions { + pub(crate) const DEFAULT: Self = Self { crash_rebuilding_a_different_chain: false }; + pub(crate) const WITH_TRACKED_BUGS: Self = Self { crash_rebuilding_a_different_chain: true }; +} + +fn rebuild(actions: Actions) -> BoxedStrategy { + if actions.crash_rebuilding_a_different_chain { + prop_oneof![Just(Rebuild::Same), Just(Rebuild::Different)].boxed() + } else { + Just(Rebuild::Same).boxed() + } +} + +fn action(actions: Actions) -> impl Strategy { + prop_oneof![ + 6 => (1..=4u64).prop_map(|count| Action::Commit { count }), + 3 => (1..=8u64, 1..=4u64).prop_map(|(depth, length)| Action::Reorg { depth, length }), + 2 => (1..=4u64).prop_map(|depth| Action::Revert { depth }), + 3 => (1..=12u64).prop_map(|count| Action::SyncTo { count }), + 3 => Just(Action::Flush), + 1 => Just(Action::FailNextSave), + 2 => (1..=3u64, 1..=3u64).prop_map(|(depth, length)| Action::StaleCommit { depth, length }), + 1 => (0..=8u64).prop_map(|offset| Action::BogusReorg { offset }), + 1 => rebuild(actions).prop_map(Action::Crash), + ] +} + +pub(crate) fn sequence(actions: Actions) -> impl Strategy> { + vec(action(actions), 1..=MAX_STEPS) +} + +/// One cell of the matrix. +#[derive(Clone, Copy, Debug)] +pub(crate) struct Config { + /// reth prunes block bodies through what the ExEx acknowledges, the notification's tip. + pub(crate) body_pruning: bool, + pub(crate) cadence: FlushCadence, + pub(crate) persistence: RethPersistence, + /// Every block at a multiple of this is executed rather than indexed; 0 indexes every block. + pub(crate) verification_interval: u64, + pub(crate) window: u64, +} + +impl Config { + pub(crate) const fn executes(&self, number: u64) -> bool { + self.verification_interval > 0 && number.is_multiple_of(self.verification_interval) + } +} + +/// reth's persistence settings: the thresholds of develop and of reth 2.6, the latter also with +/// its default masking window, which reth only accepts below the threshold. +fn persistence() -> impl Strategy { + select(vec![7u64, 50]).prop_flat_map(|threshold| { + let masking = if threshold == 50 { vec![0u64, 30] } else { vec![0u64] }; + select(masking) + .prop_map(move |masking| RethPersistence::threshold(threshold).masking(masking)) + }) +} + +pub(crate) fn config() -> impl Strategy { + ( + select(vec![FlushCadence::EVERY_BLOCK, FlushCadence::EVERY_FIFTH_BLOCK]), + persistence(), + select(vec![0u64, 1, 4]), + select(vec![WHOLE_WINDOW, PRUNED_WINDOW]), + ) + .prop_map(|(cadence, persistence, verification_interval, window)| Config { + body_pruning: false, + cadence, + persistence, + verification_interval, + window, + }) +} diff --git a/rust/op-reth/crates/trie/tests/engine_model/faults.rs b/rust/op-reth/crates/trie/tests/engine_model/faults.rs new file mode 100644 index 00000000000..783a0069815 --- /dev/null +++ b/rust/op-reth/crates/trie/tests/engine_model/faults.rs @@ -0,0 +1,71 @@ +//! A proofs store whose next write transaction can be made to fail, the way a full disk or a +//! database error fails a save. + +use reth_db::DatabaseError; +use reth_optimism_trie::{OpProofsStorageResult, OpProofsStore}; +use std::{ + fmt, + sync::{ + Arc, + atomic::{AtomicBool, Ordering}, + }, +}; + +#[derive(Clone)] +pub(crate) struct StoreWithSaveFaults { + fail_next_write: Arc, + inner: S, +} + +impl StoreWithSaveFaults { + pub(crate) fn new(inner: S) -> Self { + Self { fail_next_write: Arc::default(), inner } + } + + pub(crate) fn fail_next_write(&self) { + self.fail_next_write.store(true, Ordering::SeqCst); + } + + /// Withdraws a failure that no write has consumed yet; true if there was one. + pub(crate) fn disarm(&self) -> bool { + self.fail_next_write.swap(false, Ordering::SeqCst) + } +} + +impl fmt::Debug for StoreWithSaveFaults { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("StoreWithSaveFaults").field("inner", &self.inner).finish_non_exhaustive() + } +} + +impl OpProofsStore for StoreWithSaveFaults { + type ProviderRO<'a> + = S::ProviderRO<'a> + where + Self: 'a; + + type ProviderRw<'a> + = S::ProviderRw<'a> + where + Self: 'a; + + type Initializer<'a> + = S::Initializer<'a> + where + Self: 'a; + + fn provider_ro<'a>(&'a self) -> OpProofsStorageResult> { + self.inner.provider_ro() + } + + fn provider_rw<'a>(&'a self) -> OpProofsStorageResult> { + if self.fail_next_write.swap(false, Ordering::SeqCst) { + return Err(DatabaseError::Other("injected write failure".to_owned()).into()); + } + self.inner.provider_rw() + } + + fn initialization_provider<'a>(&'a self) -> OpProofsStorageResult> { + self.inner.initialization_provider() + } +} diff --git a/rust/op-reth/crates/trie/tests/engine_model/harness.rs b/rust/op-reth/crates/trie/tests/engine_model/harness.rs new file mode 100644 index 00000000000..c56e73baeda --- /dev/null +++ b/rust/op-reth/crates/trie/tests/engine_model/harness.rs @@ -0,0 +1,580 @@ +//! Drives reth, the ExEx's decisions and the engine through a sequence of actions; the checks +//! after every step are in `invariants.rs`. + +use super::{ + BlockSpec, TxSpec, + actions::{Action, Config, Rebuild}, + chain_spec_with_address, + faults::StoreWithSaveFaults, + initialize_storage, + model::{Model, ModelBlock, Reply}, + public_key_to_address, + simulated_reth::{ + SimulatedReth, UNRELATED_PARENT, fixed_signer, index_executed_block, reorg_block, + reorg_block_without_data, + }, + store_view::ProofTarget, +}; +use alloy_consensus::BlockHeader; +use alloy_eips::{NumHash, eip1898::BlockWithParent}; +use alloy_primitives::{Address, B256, U256}; +use reth_chain_state::ExecutedBlock; +use reth_ethereum_primitives::Block; +use reth_optimism_trie::{ + MdbxProofsStorage, OpProofsProviderRO, OpProofsStorageError, OpProofsStore, ProofWindowRange, + engine::{EngineError, EngineHandle, ReorgBlock}, +}; +use std::{ + mem, + ops::ControlFlow, + sync::{Arc, mpsc}, + thread, + time::Duration, +}; +use tempfile::TempDir; + +/// How long one engine call, or the engine settling after a step, may take before the case +/// fails instead of hanging. +pub(super) const STEP_TIMEOUT: Duration = Duration::from_secs(30); + +/// Recipients the chain pays in turn. +const RECIPIENTS: u64 = 4; + +const fn recipient(serial: u64) -> Address { + Address::repeat_byte(0x40 + (serial % RECIPIENTS) as u8) +} + +/// The block with `serial`: `serial` wei to its recipient, so that no two blocks are alike. +fn block_spec(serial: u64) -> BlockSpec { + BlockSpec::new(vec![TxSpec::transfer(recipient(serial), U256::from(serial))]) +} + +fn model_block(block: &ExecutedBlock, serial: u64, persisted_before_commit: u64) -> ModelBlock { + let recovered = block.recovered_block(); + ModelBlock { + hash: recovered.hash(), + parent: recovered.parent_hash(), + persisted_before_commit, + serial, + state_root: recovered.state_root(), + } +} + +/// The engine and the store it runs over, dropped engine first so that every handle on the +/// store is gone before the store is reopened. +struct ProofsNode { + engine: EngineHandle, + store: StoreWithSaveFaults>, +} + +pub(super) struct Harness { + pub(super) config: Config, + /// Set when an engine call hung; the node is then leaked rather than joined. + engine_hung: bool, + /// The fork point of the last reorg or revert, a height the store had to unwind to. + pub(super) last_fork: Option, + log: Vec, + pub(super) model: Model, + node: Option, + pub(super) reth: SimulatedReth, + serial: u64, + store_dir: TempDir, + pub(super) targets: Vec, +} + +impl Drop for Harness { + fn drop(&mut self) { + if self.engine_hung { + mem::forget(self.node.take()); + } + } +} + +impl Harness { + pub(super) fn new(config: Config) -> eyre::Result { + let key_pair = fixed_signer(); + let sender = public_key_to_address(key_pair.public_key()); + let chain_spec = chain_spec_with_address(sender); + let genesis = ModelBlock { + hash: chain_spec.genesis_hash(), + parent: B256::ZERO, + persisted_before_commit: 0, + serial: 0, + state_root: chain_spec.genesis_header().state_root, + }; + let reth = if config.body_pruning { + SimulatedReth::with_body_files_of(chain_spec, key_pair, config.persistence, 1)? + } else { + SimulatedReth::new(chain_spec, key_pair, config.persistence)? + }; + let store_dir = TempDir::new()?; + let store = StoreWithSaveFaults::new(Arc::new(MdbxProofsStorage::new(store_dir.path())?)); + initialize_storage(store.clone(), &reth.factory, reth.genesis())?; + let engine = Self::spawn_engine(&reth, &store, config); + let targets = ProofTarget::accounts( + [sender, Address::ZERO].into_iter().chain((0..RECIPIENTS).map(recipient)), + ); + Ok(Self { + config, + engine_hung: false, + last_fork: None, + log: Vec::new(), + model: Model::new(genesis, config.window), + node: Some(ProofsNode { engine, store }), + reth, + serial: 0, + store_dir, + targets, + }) + } + + fn spawn_engine( + reth: &SimulatedReth, + store: &StoreWithSaveFaults>, + config: Config, + ) -> EngineHandle { + reth.spawn_engine_with_window( + store.clone(), + config.cadence.persistence, + config.cadence.backpressure, + config.window, + ) + } + + const fn node(&self) -> &ProofsNode { + self.node.as_ref().expect("the node is up between steps") + } + + const fn engine(&self) -> &EngineHandle { + &self.node().engine + } + + pub(super) const fn store(&self) -> &StoreWithSaveFaults> { + &self.node().store + } + + /// Runs `actions` in order, checking the invariants after each, and reports a failure with + /// the steps that led to it; `ENGINE_MODEL_TRANSCRIPT` in the environment prints the steps + /// of a passing case too. + pub(super) fn run(&mut self, actions: &[Action]) -> eyre::Result<()> { + let result = self.run_steps(actions); + if std::env::var_os("ENGINE_MODEL_TRANSCRIPT").is_some() { + eprintln!("{}\n", self.log.join("\n")); + } + result.map_err(|err| eyre::eyre!("{err:#}\n--- steps ---\n{}", self.log.join("\n"))) + } + + fn run_steps(&mut self, actions: &[Action]) -> eyre::Result<()> { + for (index, action) in actions.iter().enumerate() { + self.log(format!("step {index}: {action:?}")); + if self.step(*action)?.is_break() { + return Ok(()); + } + self.check_invariants()?; + } + self.settle_store()?; + self.check_settled() + } + + pub(super) fn log(&mut self, line: String) { + self.log.push(line); + } + + fn step(&mut self, action: Action) -> eyre::Result> { + match action { + Action::BogusReorg { offset } => self.bogus_reorg(offset)?, + Action::Commit { count } => { + for _ in 0..count { + let block = self.commit_block()?; + self.deliver(&block)?; + } + } + Action::Crash(rebuild) => self.crash(rebuild)?, + Action::FailNextSave => self.fail_next_save()?, + Action::Flush => self.flush()?, + Action::Reorg { depth, length } => return self.reorg(depth, length), + Action::Revert { depth } => return self.revert(depth), + Action::StaleCommit { depth, length } => return self.stale_commit(depth, length), + Action::SyncTo { count } => { + for _ in 0..count { + self.commit_block()?; + } + let head = self.reth.head().number; + // The ExEx acknowledges the tip as soon as it has queued the sync target, so + // reth's pruner can run before the engine's loop reads its first block. + self.prune_bodies_through(head)?; + self.engine().sync_to(head)?; + self.model.sync_to(head); + } + } + Ok(ControlFlow::Continue(())) + } + + /// Runs `call` against the engine on its own thread and fails the case if it does not return + /// within [`STEP_TIMEOUT`], which a stalled engine would otherwise turn into a hang. + pub(super) fn call( + &mut self, + what: &str, + call: impl FnOnce(EngineHandle) -> T + Send + 'static, + ) -> eyre::Result { + let engine = self.engine().clone(); + let (reply_tx, reply_rx) = mpsc::channel(); + thread::spawn(move || { + let _ = reply_tx.send(call(engine)); + }); + match reply_rx.recv_timeout(STEP_TIMEOUT) { + Ok(result) => Ok(result), + Err(mpsc::RecvTimeoutError::Timeout) => { + self.engine_hung = true; + Err(eyre::eyre!("the engine did not reply to {what} within {STEP_TIMEOUT:?}")) + } + Err(mpsc::RecvTimeoutError::Disconnected) => { + Err(eyre::eyre!("the engine call {what} panicked")) + } + } + } + + pub(super) fn window(&self) -> eyre::Result { + Ok(self.store().provider_ro()?.get_proof_window()?) + } + + const fn next_serial(&mut self) -> u64 { + self.serial += 1; + self.serial + } + + /// Commits a new block to reth and to the model. + fn commit_block(&mut self) -> eyre::Result { + let serial = self.next_serial(); + let persisted_before_commit = self.reth.persisted().number; + let block = self.reth.commit(&block_spec(serial))?; + self.model.push(model_block(&block, serial, persisted_before_commit)); + Ok(block) + } + + /// Replaces reth's chain above `fork` with `length` new blocks, in reth and in the model. + fn fork_chain(&mut self, fork: u64, length: u64) -> eyre::Result> { + let serials: Vec = (0..length).map(|_| self.next_serial()).collect(); + let specs: Vec = serials.iter().copied().map(block_spec).collect(); + // Every block of the fork becomes canonical with reth persisted at least this far. + let persisted_before_the_fork = self.reth.persisted().number; + let new = self.reth.reorg_with(fork + 1, &specs)?; + self.model.truncate(fork); + for (block, serial) in new.iter().zip(serials) { + self.model.push(model_block(block, serial, persisted_before_the_fork)); + } + Ok(new) + } + + /// Hands `block` to the engine the way the ExEx does for a committed block, along the index + /// or the execute path, and acknowledges it to reth. + fn deliver(&mut self, block: &ExecutedBlock) -> eyre::Result<()> { + let recovered = block.recovered_block(); + let (number, num_hash, parent) = + (recovered.number(), recovered.num_hash(), recovered.parent_hash()); + let executes = self.config.executes(number); + let parent_available = + !executes || self.model.is_canonical(NumHash::new(number - 1, parent)); + let result = if executes { + let recovered = Arc::clone(&block.recovered_block); + self.call("execute_block", move |engine| engine.execute_block(&recovered))? + } else { + let block = block.clone(); + self.call("index_block", move |engine| index_executed_block(&engine, &block))? + }; + let reply = match result { + Ok(()) => Reply::Taken, + Err(EngineError::ParentHashMismatch { + block_number, + expected_parent_hash, + actual_parent_hash, + }) if (block_number, actual_parent_hash) == (number, parent) => { + self.log(format!( + " the engine refused block {number}: its tip at {} is {expected_parent_hash}", + number - 1 + )); + Reply::Refused { tip_hash: expected_parent_hash } + } + Err(err) => { + return Err(eyre::eyre!("the engine refused block {number}: {err}")); + } + }; + self.model.deliver_block(num_hash, parent, parent_available, reply)?; + self.prune_bodies_through(number) + } + + /// reth's bodies pruner following the ExEx's acknowledgement of a notification at `number`, + /// which is the notification's tip whether or not the store holds it. + fn prune_bodies_through(&mut self, number: u64) -> eyre::Result<()> { + if self.config.body_pruning && + let Some(pruned) = self.reth.prune_bodies_through(number)? + { + self.log(format!(" reth pruned bodies through block {pruned}")); + } + Ok(()) + } + + /// `blocks` the way the ExEx hands a reorg to the engine, indexed or executed as the + /// verification interval says. + fn reorg_blocks(&self, blocks: &[ExecutedBlock]) -> Vec> { + blocks + .iter() + .map(|block| reorg_block(block, self.config.executes(block.recovered_block().number()))) + .collect() + } + + /// The fork point of a reorg or revert `depth` blocks deep, clamped to what reth holds in + /// memory; none when nothing is. + fn fork_point(&mut self, depth: u64) -> Option { + let head = self.reth.head().number; + let in_memory = head - self.reth.persisted().number; + if in_memory == 0 { + self.log(" reth holds no block in memory, nothing to reorg".to_owned()); + return None; + } + Some(head - depth.min(in_memory)) + } + + /// Checks the engine's reply to a reorg or revert whose first removed block is `first`. A + /// refusal must name a block at or below the earliest the window held around the call; it + /// ends the case, since reth has moved on without the store. An `Ok` must not have unwound + /// at or below the window's earliest block. + fn unwind_outcome( + &mut self, + what: &str, + result: Result<(), EngineError>, + first: u64, + earliest_before: u64, + ) -> eyre::Result> { + match result { + Ok(()) => { + eyre::ensure!( + first > earliest_before, + "the engine applied a {what} from block {first}, at or below its earliest \ + block {earliest_before}" + ); + Ok(ControlFlow::Continue(())) + } + Err(EngineError::Storage(OpProofsStorageError::UnwindBeyondEarliest { + unwind_block_number, + earliest_block_number, + })) => { + let earliest_after = self.window()?.earliest.number; + eyre::ensure!( + unwind_block_number == first && + unwind_block_number <= earliest_block_number && + (earliest_before..=earliest_after).contains(&earliest_block_number), + "the engine refused a {what} from block {first} citing block \ + {unwind_block_number} and earliest block {earliest_block_number}, while the \ + window's earliest was {earliest_before} before and {earliest_after} after" + ); + self.log(format!( + " the engine refused the {what} from block {first} at or below its earliest \ + block {earliest_block_number}; reth moved on without the store" + )); + Ok(ControlFlow::Break(())) + } + Err(err) => { + Err(eyre::eyre!("the engine refused a valid {what} from block {first}: {err}")) + } + } + } + + /// Delivers a reorg to `new` and, if the engine took it, records it in the model. + fn deliver_reorg( + &mut self, + new: &[ExecutedBlock], + head_before: u64, + ) -> eyre::Result> { + let first = new[0].recovered_block().number(); + let last = new.last().expect("a reorg has a block").recovered_block().num_hash(); + let blocks = self.reorg_blocks(new); + let earliest_before = self.window()?.earliest.number; + let result = self.call("reorg", move |engine| engine.reorg(blocks))?; + self.last_fork = Some(first - 1); + let outcome = self.unwind_outcome("reorg", result, first, earliest_before)?; + if outcome.is_continue() { + self.model.reorg(first, last, head_before); + self.prune_bodies_through(self.reth.head().number)?; + } + Ok(outcome) + } + + fn reorg(&mut self, depth: u64, length: u64) -> eyre::Result> { + let head_before = self.reth.head().number; + let Some(fork) = self.fork_point(depth) else { + return Ok(ControlFlow::Continue(())); + }; + let new = self.fork_chain(fork, length)?; + self.deliver_reorg(&new, head_before) + } + + fn revert(&mut self, depth: u64) -> eyre::Result> { + let head_before = self.reth.head().number; + let Some(fork) = self.fork_point(depth) else { + return Ok(ControlFlow::Continue(())); + }; + let first = fork + 1; + let to = self.model.block_with_parent(first); + self.reth.revert_to(fork); + self.model.truncate(fork); + let earliest_before = self.window()?.earliest.number; + let result = self.call("unwind", move |engine| engine.unwind(to))?; + self.last_fork = Some(fork); + let outcome = self.unwind_outcome("revert", result, first, earliest_before)?; + if outcome.is_continue() { + self.model.revert(first, head_before); + } + Ok(outcome) + } + + /// reth commits `depth` blocks and reorgs them away before the ExEx processes their + /// notifications, so the engine takes the stale blocks first and the reorg after. + fn stale_commit(&mut self, depth: u64, length: u64) -> eyre::Result> { + // reth persists all but its newest `memory_block_buffer_target` blocks, and only + // in-memory blocks can be reorged away. + let depth = depth.min(self.reth.persistence.memory_block_buffer_target); + let fork = self.reth.head().number; + let stale: Vec = + (0..depth).map(|_| self.commit_block()).collect::>()?; + let head_before = self.reth.head().number; + let new = self.fork_chain(fork, length)?; + for block in &stale { + self.deliver(block)?; + } + self.deliver_reorg(&new, head_before) + } + + pub(super) fn flush(&mut self) -> eyre::Result<()> { + self.call("flush", |engine| engine.flush())?; + let latest = self.window()?.latest.number; + self.model.note_persisted(latest); + Ok(()) + } + + /// Arms a failure for the next write, commits enough blocks to start a save, and flushes + /// twice: once for a save already in flight and once for the one that then starts and fails. + fn fail_next_save(&mut self) -> eyre::Result<()> { + self.store().fail_next_write(); + for _ in 0..self.config.cadence.persistence { + let block = self.commit_block()?; + self.deliver(&block)?; + } + self.flush()?; + self.flush()?; + if self.store().disarm() { + self.log(" no save started, the injected failure is withdrawn".to_owned()); + } + Ok(()) + } + + /// A one-block reorg on a parent no block has, at a fork point the engine knows the hash of, + /// which it must refuse, or skip while its tip is still below the fork point. + fn bogus_reorg(&mut self, offset: u64) -> eyre::Result<()> { + let points = self.model.known_fork_points(); + if points.is_empty() { + self.log(" the engine knows no fork point for certain, nothing to refuse".to_owned()); + return Ok(()); + } + let fork = points[(offset % points.len() as u64) as usize]; + let first = fork + 1; + let bogus = reorg_block_without_data(BlockWithParent::new( + UNRELATED_PARENT, + NumHash::new(first, B256::repeat_byte(0xB1)), + )); + let stored_fork_hash = self.model.num_hash(fork).hash; + let result = self.call("reorg", move |engine| engine.reorg(vec![bogus]))?; + let refused = match result { + Ok(()) => false, + Err(EngineError::ParentHashMismatch { + block_number, + expected_parent_hash, + actual_parent_hash, + }) => { + eyre::ensure!( + (block_number, expected_parent_hash, actual_parent_hash) == + (first, stored_fork_hash, UNRELATED_PARENT), + "the engine refused the reorg at block {first} with a mismatch at block \ + {block_number} expecting {expected_parent_hash} and seeing \ + {actual_parent_hash}, not the stored block {fork} {stored_fork_hash}" + ); + true + } + Err(EngineError::Storage(OpProofsStorageError::UnwindBeyondEarliest { + unwind_block_number, + earliest_block_number, + })) => { + eyre::ensure!( + unwind_block_number == first && first <= earliest_block_number, + "the engine refused the reorg at block {first} citing block \ + {unwind_block_number} and earliest block {earliest_block_number}" + ); + true + } + Err(err) => { + return Err(eyre::eyre!( + "unexpected reply to a reorg at block {first} not building on the stored block \ + {fork}: {err}" + )); + } + }; + self.model.bogus_reorg(first, refused)?; + self.log(format!( + " the engine {} the reorg at block {first} on a parent no block has", + if refused { "refused" } else { "skipped" } + )); + Ok(()) + } + + /// reth and the node stop and restart: reth loses what it held only in memory, the engine + /// its buffer (its save in flight completes, as the engine thread drains it on drop), and + /// both come back the way a restart brings them back today: a fresh engine over the reopened + /// store, a sync target at reth's head, and reth's rebuilt blocks as live notifications. + fn crash(&mut self, rebuild: Rebuild) -> eyre::Result<()> { + let persisted = self.reth.persisted().number; + let lost = self.model.serials_above(persisted); + self.log(match lost.len() { + 0 => format!(" crash: reth loses no block ({rebuild:?})"), + count => format!( + " crash: reth loses blocks {}..={} and rebuilds them ({rebuild:?})", + persisted + 1, + persisted + count as u64 + ), + }); + self.node = None; + self.reth.crash_and_restart()?; + let store = + StoreWithSaveFaults::new(Arc::new(MdbxProofsStorage::new(self.store_dir.path())?)); + let engine = Self::spawn_engine(&self.reth, &store, self.config); + self.node = Some(ProofsNode { engine, store }); + let tip = self.call("tip", |engine| engine.tip())??; + self.model.restart(tip)?; + self.log(format!( + " restart: the store serves through block {}, reth's head is {persisted}", + tip.number + )); + self.engine().sync_to(persisted)?; + self.model.sync_to(persisted); + match rebuild { + Rebuild::Same => { + for serial in lost { + let block = self.reth.commit(&block_spec(serial))?; + let number = block.recovered_block().number(); + eyre::ensure!( + block.recovered_block().hash() == self.model.block(number).hash, + "reth rebuilt block {number} differently from the chain it lost" + ); + self.deliver(&block)?; + } + } + Rebuild::Different => { + self.model.truncate(persisted); + for _ in lost { + let block = self.commit_block()?; + self.deliver(&block)?; + } + } + } + Ok(()) + } +} diff --git a/rust/op-reth/crates/trie/tests/engine_model/invariants.rs b/rust/op-reth/crates/trie/tests/engine_model/invariants.rs new file mode 100644 index 00000000000..e7c58438f2d --- /dev/null +++ b/rust/op-reth/crates/trie/tests/engine_model/invariants.rs @@ -0,0 +1,175 @@ +//! What the harness checks after every step and at the end of a case. + +use super::{ + harness::{Harness, STEP_TIMEOUT}, + store_view::BlockView, +}; +use alloy_eips::NumHash; +use reth_optimism_trie::{OpProofsProviderRO, OpProofsStore, ProofWindowRange}; +use std::{ + thread, + time::{Duration, Instant}, +}; + +const POLL_INTERVAL: Duration = Duration::from_millis(2); + +/// Flushes a settle may need before the store stops changing. +const MAX_SETTLE_FLUSHES: usize = 64; + +impl Harness { + pub(super) fn check_invariants(&mut self) -> eyre::Result<()> { + eyre::ensure!( + self.model.num_hash(self.model.head()) == self.reth.head(), + "the model's head {:?} is not reth's {:?}", + self.model.num_hash(self.model.head()), + self.reth.head() + ); + let tip = self.settle_engine()?; + let (window, views) = self.read_window_and_views(self.last_fork)?; + self.ensure_canonical("earliest", window.earliest)?; + self.ensure_canonical("latest", window.latest)?; + eyre::ensure!( + window.latest.number <= tip.number, + "the store's latest block {} is above the engine's tip {}", + window.latest.number, + tip.number + ); + eyre::ensure!( + window.latest.number >= self.model.persisted_at_least(), + "the store's latest block {} is below block {}, which it had persisted", + window.latest.number, + self.model.persisted_at_least() + ); + let earliest_bounds = self.model.earliest_bounds(window.latest.number); + eyre::ensure!( + earliest_bounds.contains(&window.earliest.number), + "the store holds blocks {}..={}, whose earliest block lies outside {:?}, what its \ + window of {} allows with reth persisted through {} before block {} was committed", + window.earliest.number, + window.latest.number, + earliest_bounds, + self.model.window(), + self.model.block(window.latest.number).persisted_before_commit, + window.latest.number + ); + self.model.note_persisted(window.latest.number); + for (number, view) in views { + view.verify_proofs(number, self.model.block(number).state_root)?; + } + self.log(format!( + " engine tip {}, store blocks {}..={}, reth head {}", + tip.number, + window.earliest.number, + window.latest.number, + self.reth.head().number + )); + Ok(()) + } + + /// Flushes until a flush changes nothing, which leaves fewer blocks buffered than start a + /// save. + pub(super) fn settle_store(&mut self) -> eyre::Result<()> { + let mut latest = self.window()?.latest; + for _ in 0..MAX_SETTLE_FLUSHES { + self.flush()?; + let now = self.window()?.latest; + if now == latest { + return Ok(()); + } + latest = now; + } + Err(eyre::eyre!("the store kept changing over {MAX_SETTLE_FLUSHES} flushes")) + } + + pub(super) fn check_settled(&mut self) -> eyre::Result<()> { + let tip = self.settle_engine()?; + let latest = self.window()?.latest.number; + let unsaved = tip.number.checked_sub(latest).ok_or_else(|| { + eyre::eyre!( + "the store's latest block {latest} is above the engine's tip {}", + tip.number + ) + })?; + eyre::ensure!( + unsaved < self.config.cadence.persistence, + "{unsaved} blocks stay buffered above the store's latest block {latest} after \ + settling, enough to start a save at the persistence threshold {}", + self.config.cadence.persistence + ); + self.log(format!("settled: engine tip {}, store latest {latest}", tip.number)); + Ok(()) + } + + /// Waits for the engine's tip to reach one the model predicts, and narrows the model to it. + /// A tip off every path the model allows fails at once. + fn settle_engine(&mut self) -> eyre::Result { + let expected = self.model.settled_tips(); + let deadline = Instant::now() + STEP_TIMEOUT; + loop { + let tip = self.call("tip", |engine| engine.tip())??; + if expected.contains(&tip) { + self.model.observe_settled(tip)?; + return Ok(tip); + } + eyre::ensure!( + self.model.on_path(tip), + "the engine's tip {tip:?} is on no path the model allows: {:?}", + self.model.candidates() + ); + if Instant::now() >= deadline { + let window = self.window()?; + eyre::bail!( + "the engine did not settle within {STEP_TIMEOUT:?}: its tip is {tip:?}, the \ + model expects one of {expected:?}, the store holds blocks {}..={}", + window.earliest.number, + window.latest.number + ); + } + thread::sleep(POLL_INTERVAL); + } + } + + /// The store's window and, read in the same transaction so that a save landing meanwhile + /// cannot prune a height from under the read, its views at the window's ends and at `fork` + /// when the window holds it. + fn read_window_and_views( + &self, + fork: Option, + ) -> eyre::Result<(ProofWindowRange, Vec<(u64, BlockView)>)> { + let provider = self.store().provider_ro()?; + let window = provider.get_proof_window()?; + let mut heights = vec![window.earliest.number, window.latest.number]; + if let Some(fork) = fork && + (window.earliest.number..=window.latest.number).contains(&fork) + { + heights.push(fork); + } + heights.sort_unstable(); + heights.dedup(); + let views = heights + .into_iter() + .map(|number| Ok((number, BlockView::read_from(&provider, number, &self.targets)?))) + .collect::>()?; + Ok((window, views)) + } + + fn describe_canonical(&self, number: u64) -> String { + if number <= self.model.head() { + format!("{}", self.model.block(number).hash) + } else { + format!("beyond the canonical head {}", self.model.head()) + } + } + + fn ensure_canonical(&self, what: &str, block: NumHash) -> eyre::Result<()> { + eyre::ensure!( + self.model.is_canonical(block), + "the store's {what} block {}:{} is not on the canonical chain, whose block {} is {}", + block.number, + block.hash, + block.number, + self.describe_canonical(block.number) + ); + Ok(()) + } +} diff --git a/rust/op-reth/crates/trie/tests/engine_model/model.rs b/rust/op-reth/crates/trie/tests/engine_model/model.rs new file mode 100644 index 00000000000..111a65f9c92 --- /dev/null +++ b/rust/op-reth/crates/trie/tests/engine_model/model.rs @@ -0,0 +1,535 @@ +//! The reference model: the canonical chain as the consensus layer holds it, and where the engine +//! and its store must end up once they settle. It knows nothing of the engine's internals; what +//! it cannot predict exactly, because a save or the sync loop runs in the background, it bounds, +//! and the harness narrows the bound with what it observes. + +use alloy_eips::{NumHash, eip1898::BlockWithParent}; +use alloy_primitives::B256; +use std::ops::RangeInclusive; + +/// A canonical block: what the store must agree with at its height. +#[derive(Clone, Copy, Debug)] +pub(super) struct ModelBlock { + pub(super) hash: B256, + pub(super) parent: B256, + /// reth's persisted head just before reth committed the block; reth's persisted head never + /// falls, so every save of the block sees at least this. + pub(super) persisted_before_commit: u64, + /// The serial of the transfer the block carries; rebuilding it on the same parent gives the + /// same block. + pub(super) serial: u64, + pub(super) state_root: B256, +} + +/// Where the engine may be: a tip it has reached for certain and the target its sync loop works +/// toward, so its actual tip lies between the two. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(super) struct EngineCandidate { + pub(super) target: u64, + pub(super) tip: NumHash, +} + +/// The engine's reply to a block from a live notification. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(super) enum Reply { + /// A parent-hash mismatch naming the engine's tip at the height below the block. + Refused { tip_hash: B256 }, + /// `Ok`, whether the block was buffered, ignored or taken as a sync target. + Taken, +} + +pub(super) struct Model { + /// Every state the engine may be in; an observation keeps the ones it confirms. No + /// candidate's way to its settled tip passes through another's settled tip, so the first + /// settled tip the harness observes is the one the engine stays at. + candidates: Vec, + /// The canonical chain, genesis first. + canonical: Vec, + /// The highest head the chain has had, which no save's tip exceeds. + highest_head: u64, + /// The lowest block whose hash the engine has held since it started, through its tip. + known_from: u64, + /// The store has persisted at least through this block. + persisted_at_least: u64, + /// The proof window the store keeps. + window: u64, +} + +impl Model { + pub(super) fn new(genesis: ModelBlock, window: u64) -> Self { + Self { + candidates: vec![EngineCandidate { target: 0, tip: NumHash::new(0, genesis.hash) }], + canonical: vec![genesis], + highest_head: 0, + known_from: 1, + persisted_at_least: 0, + window, + } + } + + pub(super) const fn head(&self) -> u64 { + self.canonical.len() as u64 - 1 + } + + pub(super) fn block(&self, number: u64) -> &ModelBlock { + &self.canonical[number as usize] + } + + pub(super) fn num_hash(&self, number: u64) -> NumHash { + NumHash::new(number, self.block(number).hash) + } + + pub(super) fn block_with_parent(&self, number: u64) -> BlockWithParent { + BlockWithParent::new(self.block(number).parent, self.num_hash(number)) + } + + pub(super) fn is_canonical(&self, block: NumHash) -> bool { + block.number <= self.head() && self.block(block.number).hash == block.hash + } + + pub(super) fn push(&mut self, block: ModelBlock) { + self.canonical.push(block); + self.highest_head = self.highest_head.max(self.head()); + } + + /// Keeps the chain through `head`. + pub(super) fn truncate(&mut self, head: u64) { + self.canonical.truncate(head as usize + 1); + } + + /// The serials of the blocks above `number`, the order to rebuild them in. + pub(super) fn serials_above(&self, number: u64) -> Vec { + self.canonical[number as usize + 1..].iter().map(|block| block.serial).collect() + } + + pub(super) const fn window(&self) -> u64 { + self.window + } + + /// Where the store's earliest block may lie while its latest block is `latest`. Every save + /// with tip `t` prunes to `t - window`, or not past reth's persisted head when the pruner + /// caps it there. The save that wrote `latest` had a tip at or above it and saw reth + /// persisted at least through `persisted_before_commit` of `latest`, and the earliest block + /// never moves down, which gives the lower end. No save's tip exceeds the highest head, + /// which gives the upper end. + pub(super) fn earliest_bounds(&self, latest: u64) -> RangeInclusive { + let persisted_before_the_save = self.block(latest).persisted_before_commit; + let lowest = latest.saturating_sub(self.window).min(persisted_before_the_save); + let highest = self.highest_head.saturating_sub(self.window); + lowest..=highest + } + + pub(super) const fn persisted_at_least(&self) -> u64 { + self.persisted_at_least + } + + /// The store was seen persisted through `latest`. + pub(super) fn note_persisted(&mut self, latest: u64) { + self.persisted_at_least = self.persisted_at_least.max(latest); + } + + pub(super) fn candidates(&self) -> &[EngineCandidate] { + &self.candidates + } + + /// Fork points whose hash the engine holds for certain: the blocks it took in since it + /// started, and genesis while the window is never pruned. + pub(super) fn known_fork_points(&self) -> Vec { + let mut points: Vec = self + .highest_reach() + .map(|reach| (self.known_from..=reach).collect()) + .unwrap_or_default(); + if self.window >= self.head() { + points.push(0); + } + points + } + + /// How far the sync loop can carry `candidate` over a chain whose head was `head`: to its + /// target while its tip is canonical, nowhere once the tip is on an abandoned chain. + fn reach(&self, candidate: &EngineCandidate, head: u64) -> u64 { + if self.is_canonical(candidate.tip) { + candidate.target.min(head).max(candidate.tip.number) + } else { + candidate.tip.number + } + } + + fn highest_reach(&self) -> Option { + self.candidates.iter().map(|candidate| self.reach(candidate, self.head())).max() + } + + /// The tip `candidate` settles on once the sync loop has done what it can. + fn settled_tip(&self, candidate: &EngineCandidate) -> NumHash { + if self.is_canonical(candidate.tip) { + self.num_hash(self.reach(candidate, self.head())) + } else { + candidate.tip + } + } + + /// The tips the engine may settle on. + pub(super) fn settled_tips(&self) -> Vec { + let mut tips = Vec::new(); + for candidate in &self.candidates { + let tip = self.settled_tip(candidate); + if !tips.contains(&tip) { + tips.push(tip); + } + } + tips + } + + /// Whether `tip` lies on some candidate's way to settling: the candidate's tip, or a + /// canonical block between it and the candidate's reach. The sync loop only moves forward + /// along the canonical chain, so a tip off every path is a wrong engine state at once. + pub(super) fn on_path(&self, tip: NumHash) -> bool { + let head = self.head(); + self.candidates.iter().any(|candidate| { + tip == candidate.tip || + (self.is_canonical(tip) && + (candidate.tip.number..=self.reach(candidate, head)) + .contains(&tip.number)) + }) + } + + /// Keeps the candidates that settle on `observed` and pins them to it. + pub(super) fn observe_settled(&mut self, observed: NumHash) -> eyre::Result<()> { + let head = self.head(); + let settled: Vec = self + .candidates + .iter() + .filter(|candidate| self.settled_tip(candidate) == observed) + .map(|candidate| EngineCandidate { + target: candidate.target.min(head).max(observed.number), + tip: observed, + }) + .collect(); + eyre::ensure!( + !settled.is_empty(), + "the engine settled at {observed:?}, which none of {:?} predicts", + self.candidates + ); + self.replace_candidates(settled); + Ok(()) + } + + /// The engine replied `reply` to `block` from a live notification. It buffers a block that + /// is the next one and builds on its tip, on the execute path only while the parent's state + /// is still in reth; it refuses the next block when the block does not build on its tip, + /// naming the tip; it raises its sync target for a block beyond the next one; it ignores a + /// block at or below its tip. + pub(super) fn deliver_block( + &mut self, + block: NumHash, + parent: B256, + parent_available: bool, + reply: Reply, + ) -> eyre::Result<()> { + let consistent: Vec = self + .candidates + .iter() + .flat_map(|candidate| self.deliveries(candidate, block, parent, parent_available)) + .filter(|(_, predicted)| *predicted == reply) + .map(|(candidate, _)| candidate) + .collect(); + eyre::ensure!( + !consistent.is_empty(), + "the engine replied {reply:?} to block {} on parent {parent}, which no state it \ + could be in allows: {:?}", + block.number, + self.candidates + ); + self.replace_candidates(consistent); + Ok(()) + } + + /// What `candidate` becomes on `block`, with the reply each outcome gives, for every tip the + /// candidate's sync loop may have reached by then. + fn deliveries( + &self, + candidate: &EngineCandidate, + block: NumHash, + parent: B256, + parent_available: bool, + ) -> Vec<(EngineCandidate, Reply)> { + let number = block.number; + let lowest = candidate.tip.number; + if number <= lowest { + return vec![(*candidate, Reply::Taken)]; + } + let reach = self.reach(candidate, self.head()); + let mut next = Vec::new(); + if lowest < number - 1 { + let target = candidate.target.max(number); + next.push((EngineCandidate { target, ..*candidate }, Reply::Taken)); + } + if number - 1 <= reach { + let tip_below = + if lowest == number - 1 { candidate.tip } else { self.num_hash(number - 1) }; + if tip_below.hash == parent { + let tip = if parent_available { block } else { tip_below }; + next.push((EngineCandidate { target: candidate.target, tip }, Reply::Taken)); + } else { + let refused = EngineCandidate { target: candidate.target, tip: tip_below }; + next.push((refused, Reply::Refused { tip_hash: tip_below.hash })); + } + } + if reach >= number { + next.push((*candidate, Reply::Taken)); + } + next + } + + /// The engine replied `Ok` to a reorg from `first` to a chain ending at `last`, delivered + /// after reth switched to it while its head stood at `head_before`. + pub(super) fn reorg(&mut self, first: u64, last: NumHash, head_before: u64) { + self.unwind_from(first, last, head_before); + } + + /// The engine replied `Ok` to a revert of the blocks from `first` up, delivered after reth + /// dropped them while its head stood at `head_before`. + pub(super) fn revert(&mut self, first: u64, head_before: u64) { + self.unwind_from(first, self.num_hash(first - 1), head_before); + } + + /// The engine skipped the unwind where `first` was above its tip; elsewhere the store and + /// the buffer unwound to the block below `first` and the tip is `tip_after`. + fn unwind_from(&mut self, first: u64, tip_after: NumHash, head_before: u64) { + let widest_head = head_before.max(self.head()); + let mut next = Vec::new(); + for candidate in &self.candidates { + if candidate.tip.number < first { + next.push(*candidate); + } + if self.reach(candidate, widest_head) >= first { + next.push(EngineCandidate { + target: candidate.target.min(first - 1), + tip: tip_after, + }); + } + } + self.replace_candidates(next); + self.known_from = self.known_from.min(first); + self.persisted_at_least = self.persisted_at_least.min(first - 1); + } + + pub(super) fn sync_to(&mut self, target: u64) { + for candidate in &mut self.candidates { + candidate.target = candidate.target.max(target); + } + } + + /// The engine's reply to a reorg at `first` whose first block builds on no block it holds: + /// refused where its tip was at or above `first`, which takes a candidate reaching that far, + /// and skipped where its tip was below. + pub(super) fn bogus_reorg(&mut self, first: u64, refused: bool) -> eyre::Result<()> { + let head = self.head(); + let consistent: Vec = self + .candidates + .iter() + .copied() + .filter(|candidate| { + if refused { + self.reach(candidate, head) >= first + } else { + candidate.tip.number < first + } + }) + .collect(); + eyre::ensure!( + !consistent.is_empty(), + "the engine {} a reorg at block {first} on a parent no block has, which no state it \ + could be in allows: {:?}", + if refused { "refused" } else { "skipped" }, + self.candidates + ); + self.replace_candidates(consistent); + Ok(()) + } + + /// A fresh engine over the store reports `tip`, which the store had persisted: nothing is + /// buffered and nothing below the tip is known. + pub(super) fn restart(&mut self, tip: NumHash) -> eyre::Result<()> { + eyre::ensure!( + tip.number >= self.persisted_at_least, + "a fresh engine reports tip {tip:?} below the persisted bound {}", + self.persisted_at_least + ); + eyre::ensure!( + self.highest_reach().is_some_and(|reach| tip.number <= reach), + "a fresh engine reports tip {tip:?} above anything the old engine could have held \ + ({:?})", + self.candidates + ); + self.candidates = vec![EngineCandidate { target: tip.number, tip }]; + self.known_from = tip.number + 1; + self.persisted_at_least = tip.number; + Ok(()) + } + + fn replace_candidates(&mut self, next: Vec) { + let mut unique: Vec = Vec::with_capacity(next.len()); + for candidate in next { + if !unique.contains(&candidate) { + unique.push(candidate); + } + } + assert!(!unique.is_empty(), "a step left the engine with no possible state"); + self.candidates = unique; + } +} + +#[cfg(test)] +mod tests { + use super::*; + use rstest::rstest; + + const WINDOW: u64 = 1000; + + /// The canonical block at `number`: hash and parent are the block numbers. + fn canonical(number: u64) -> NumHash { + NumHash::new(number, B256::with_last_byte(number as u8)) + } + + /// A block at `number` on an abandoned chain. + fn stale(number: u64) -> NumHash { + NumHash::new(number, B256::with_last_byte(0xA0 + number as u8)) + } + + fn block(number: u64) -> ModelBlock { + ModelBlock { + hash: canonical(number).hash, + parent: canonical(number.saturating_sub(1)).hash, + persisted_before_commit: 0, + serial: number, + state_root: B256::ZERO, + } + } + + /// A chain through `head` with the engine in `candidates`. + fn model(head: u64, candidates: &[EngineCandidate]) -> Model { + let mut model = Model::new(block(0), WINDOW); + for number in 1..=head { + model.push(block(number)); + } + model.candidates = candidates.to_vec(); + model + } + + const fn at(tip: NumHash, target: u64) -> EngineCandidate { + EngineCandidate { target, tip } + } + + #[rstest] + #[case::at_or_below_the_tip(3, &[at(canonical(3), 3)], 2, true, &[at(canonical(3), 3)])] + #[case::next_block(4, &[at(canonical(3), 3)], 4, true, &[at(canonical(4), 3)])] + #[case::next_block_without_parent_state(4, &[at(canonical(3), 3)], 4, false, &[at(canonical(3), 3)])] + #[case::beyond_the_next_block(5, &[at(canonical(3), 3)], 5, true, &[at(canonical(3), 5)])] + #[case::while_syncing(7, &[at(canonical(3), 6)], 7, true, &[at(canonical(3), 7), at(canonical(7), 6)])] + fn deliver_block_taken( + #[case] head: u64, + #[case] before: &[EngineCandidate], + #[case] number: u64, + #[case] parent_available: bool, + #[case] after: &[EngineCandidate], + ) { + let mut model = model(head, before); + model + .deliver_block( + canonical(number), + canonical(number - 1).hash, + parent_available, + Reply::Taken, + ) + .unwrap(); + assert_eq!(model.candidates(), after); + } + + #[rstest] + #[case::refused_citing_the_tip(Reply::Refused { tip_hash: stale(3).hash }, true)] + #[case::refused_citing_another_block(Reply::Refused { tip_hash: canonical(3).hash }, false)] + #[case::taken(Reply::Taken, false)] + fn deliver_block_on_a_stale_tip(#[case] reply: Reply, #[case] consistent: bool) { + let mut model = model(4, &[at(stale(3), 3)]); + let result = model.deliver_block(canonical(4), canonical(3).hash, true, reply); + assert_eq!(result.is_ok(), consistent, "{result:?}"); + } + + #[rstest] + #[case::skipped_below_the_tip(&[at(canonical(2), 2)], &[at(canonical(2), 2)])] + #[case::applied_at_the_tip(&[at(canonical(5), 5)], &[at(canonical(5), 3)])] + #[case::either_while_syncing(&[at(canonical(2), 5)], &[at(canonical(2), 5), at(canonical(5), 3)])] + fn reorg_from_block_four( + #[case] before: &[EngineCandidate], + #[case] after: &[EngineCandidate], + ) { + let mut model = model(5, before); + model.reorg(4, canonical(5), 5); + assert_eq!(model.candidates(), after); + } + + #[test] + fn revert_from_block_four_moves_the_tip_below_it() { + let mut model = model(3, &[at(canonical(5), 5)]); + model.revert(4, 5); + assert_eq!(model.candidates(), &[at(canonical(3), 3)]); + } + + #[test] + fn settling_keeps_the_candidates_that_reach_the_observed_tip() { + let mut model = model(7, &[at(canonical(3), 7), at(canonical(7), 6)]); + assert_eq!(model.settled_tips(), vec![canonical(7)]); + assert!(model.on_path(canonical(5))); + assert!(!model.on_path(stale(5))); + model.observe_settled(canonical(7)).unwrap(); + assert_eq!(model.candidates(), &[at(canonical(7), 7)]); + assert!(model.observe_settled(canonical(6)).is_err()); + } + + #[rstest] + #[case::at_the_tip(3, true)] + #[case::above_the_reach(4, false)] + #[case::below_the_persisted_bound(1, false)] + fn restart_bounds(#[case] tip: u64, #[case] consistent: bool) { + let mut model = model(3, &[at(canonical(3), 3)]); + model.note_persisted(2); + assert_eq!(model.restart(canonical(tip)).is_ok(), consistent); + } + + /// A chain through `head` under `window`, whose block `n` was committed with reth persisted + /// through `min(n, persisted)`. + fn model_with_reth_persisted(window: u64, head: u64, persisted: u64) -> Model { + let mut model = Model::new(block(0), window); + for number in 1..=head { + let persisted_before_commit = persisted.min(number); + model.push(ModelBlock { persisted_before_commit, ..block(number) }); + } + model + } + + #[rstest] + #[case::whole_window(WINDOW, 7, 7, 7, 0..=0)] + #[case::reth_persisted_through_the_prune_target(4, 8, 8, 8, 4..=4)] + #[case::reth_persisted_below_the_prune_target(4, 8, 8, 3, 3..=4)] + #[case::latest_unwound_below_the_highest_head(4, 8, 5, 8, 1..=4)] + #[case::latest_within_a_window_of_genesis(4, 8, 2, 8, 0..=4)] + fn earliest_bounds( + #[case] window: u64, + #[case] head: u64, + #[case] latest: u64, + #[case] persisted: u64, + #[case] expected: RangeInclusive, + ) { + let model = model_with_reth_persisted(window, head, persisted); + assert_eq!(model.earliest_bounds(latest), expected); + } + + #[test] + fn earliest_bounds_keep_the_highest_head_through_a_reorg() { + let mut model = model_with_reth_persisted(4, 8, 5); + model.truncate(5); + model.push(ModelBlock { persisted_before_commit: 5, ..block(6) }); + assert_eq!(model.earliest_bounds(6), 2..=4); + } +} diff --git a/rust/op-reth/crates/trie/tests/live.rs b/rust/op-reth/crates/trie/tests/live.rs index ef8614ca6d9..441e95ed9e8 100644 --- a/rust/op-reth/crates/trie/tests/live.rs +++ b/rust/op-reth/crates/trie/tests/live.rs @@ -1,19 +1,13 @@ //! End-to-end test of the live trie collector. -use alloy_consensus::{ - BlockHeader, Header, TxEip2930, - constants::{EMPTY_WITHDRAWALS, ETH_TO_WEI}, -}; -use alloy_eips::{BlockNumHash, eip4895::Withdrawals}; -use alloy_genesis::{Genesis, GenesisAccount}; -use alloy_primitives::{Address, B256, Bytes, TxKind, U256, keccak256}; +use alloy_consensus::BlockHeader; +use alloy_eips::BlockNumHash; +use alloy_primitives::{Address, B256, Bytes, TxKind, U256}; use derive_more::Constructor; -use reth_chainspec::{ - ChainSpec, ChainSpecBuilder, EthereumHardfork, EthereumHardforks, MAINNET, MIN_TRANSACTION_GAS, -}; +use reth_chainspec::ChainSpec; use reth_db::Database; use reth_db_common::init::init_genesis; -use reth_ethereum_primitives::{Block, BlockBody, Receipt, Transaction, TransactionSigned}; +use reth_ethereum_primitives::{Block, BlockBody, Receipt}; use reth_evm::{ConfigureEvm, execute::Executor}; use reth_evm_ethereum::EthEvmConfig; use reth_node_api::{NodePrimitives, NodeTypesWithDB}; @@ -26,7 +20,7 @@ use reth_optimism_trie::{ use reth_primitives_traits::{Block as _, RecoveredBlock}; use reth_provider::{ BlockWriter as _, ExecutionOutcome, HashedPostStateProvider, LatestStateProviderRef, - ProviderFactory, StateRootProvider, StorageSettingsCache, + ProviderFactory, StateRootProvider, providers::{BlockchainProvider, ProviderNodeTypes}, test_utils::create_test_provider_factory_with_chain_spec, }; @@ -34,39 +28,8 @@ use reth_revm::database::StateProviderDatabase; use secp256k1::{Keypair, Secp256k1, rand::rng}; use serial_test::serial; use std::sync::Arc; -use tempfile::TempDir; use test_case::test_case; -/// Seconds between consecutive blocks. -const BLOCK_TIME: u64 = 2; - -/// A proof window no test chain fills, so that the pruner never moves the earliest block. -const WHOLE_WINDOW: u64 = 1000; - -/// Timestamp of the block at `number`. -const fn block_timestamp(number: u64) -> u64 { - number * BLOCK_TIME -} - -/// The engine's persistence and backpressure thresholds. -#[derive(Clone, Copy, Debug)] -struct FlushCadence { - backpressure: u64, - persistence: u64, -} - -impl FlushCadence { - const EVERY_BLOCK: Self = Self { backpressure: 2, persistence: 1 }; - const EVERY_FIFTH_BLOCK: Self = Self { backpressure: 10, persistence: 5 }; -} - -/// Which store implementation a case runs over. -#[derive(Clone, Copy, Debug)] -enum Backend { - InMemory, - Mdbx, -} - /// Runs `$run` over a fresh store of `$backend`. macro_rules! over_backend { ($backend:expr, $run:ident($($arg:expr),*)) => { @@ -79,6 +42,8 @@ macro_rules! over_backend { #[path = "live/catch_up_under_pruning.rs"] mod catch_up_under_pruning; +#[path = "live/chain.rs"] +mod chain; #[path = "live/contracts.rs"] mod contracts; #[path = "live/index_execute_parity.rs"] @@ -88,86 +53,12 @@ mod parent_states; #[path = "live/reorg_revert_edges.rs"] mod reorg_revert_edges; #[path = "live/simulated_reth.rs"] +#[allow(dead_code, reason = "each test target uses part of this module")] mod simulated_reth; #[path = "live/store_view.rs"] mod store_view; -fn create_mdbx_proofs_storage() -> Arc { - let path = TempDir::new().unwrap().keep(); - Arc::new(MdbxProofsStorage::new(&path).unwrap()) -} - -/// Converts a secp256k1 public key to an Ethereum address. -fn public_key_to_address(pubkey: secp256k1::PublicKey) -> Address { - let hash = keccak256(&pubkey.serialize_uncompressed()[1..]); - Address::from_slice(&hash[12..]) -} - -/// Signs a transaction with the given keypair. -fn sign_tx_with_key_pair(key_pair: Keypair, tx: Transaction) -> TransactionSigned { - use alloy_consensus::SignableTransaction; - use reth_primitives_traits::crypto::secp256k1::sign_message; - let secret = B256::from_slice(&key_pair.secret_bytes()); - let sig = sign_message(secret, tx.signature_hash()).unwrap(); - tx.into_signed(sig).into() -} - -/// Specification for a transaction within a block -#[derive(Debug, Clone)] -struct TxSpec { - /// Call recipient or contract creation. - to: TxKind, - /// Transaction input, including initcode for deployments. - data: Bytes, - /// Maximum execution gas. - gas_limit: u64, - /// Value to transfer (in wei) - value: U256, - /// Nonce for the transaction (will be automatically assigned if None) - nonce: Option, -} - -impl TxSpec { - /// Create a simple transfer transaction - const fn transfer(to: Address, value: U256) -> Self { - Self { - to: TxKind::Call(to), - data: Bytes::new(), - gas_limit: MIN_TRANSACTION_GAS, - value, - nonce: None, - } - } - - /// Create a contract deployment running `initcode`. - const fn create(initcode: Bytes, gas_limit: u64) -> Self { - Self { to: TxKind::Create, data: initcode, gas_limit, value: U256::ZERO, nonce: None } - } - - /// Create a call to `contract` with `data` as input. - const fn call(contract: Address, data: Bytes, gas_limit: u64) -> Self { - Self { to: TxKind::Call(contract), data, gas_limit, value: U256::ZERO, nonce: None } - } - - /// Send `value` along with the transaction. - fn with_value(self, value: U256) -> Self { - Self { value, ..self } - } -} - -/// Specification for a block in the test chain -#[derive(Debug, Clone, Constructor)] -struct BlockSpec { - /// Transactions to include in this block - txs: Vec, -} - -impl BlockSpec { - /// A block without transactions. - const fn empty() -> Self { - Self { txs: Vec::new() } - } -} +use chain::*; /// Configuration for a test scenario #[derive(Debug, Constructor)] @@ -178,95 +69,6 @@ struct TestScenario { blocks_after_initialization: Vec, } -/// A mainnet-shaped chain spec whose genesis allocation is `alloc`, with the hardforks still to -/// be chosen. -fn chain_spec_builder( - alloc: impl IntoIterator, -) -> ChainSpecBuilder { - ChainSpecBuilder::default() - .chain(MAINNET.chain) - .genesis(Genesis { alloc: alloc.into_iter().collect(), ..MAINNET.genesis.clone() }) -} - -/// A genesis allocation funding `address`. -fn funded(address: Address) -> (Address, GenesisAccount) { - (address, GenesisAccount { balance: U256::from(10 * ETH_TO_WEI), ..Default::default() }) -} - -/// Helper to create a chain spec with a genesis account funded -fn chain_spec_with_address(address: Address) -> Arc { - Arc::new(chain_spec_builder([funded(address)]).paris_activated().build()) -} - -/// Creates a block from a spec, executing transactions with the given keypair -fn create_block_from_spec( - spec: &BlockSpec, - block_number: u64, - parent_hash: B256, - chain_spec: &Arc, - key_pair: Keypair, - nonce_counter: &mut u64, -) -> RecoveredBlock { - let transactions: Vec = spec - .txs - .iter() - .map(|tx_spec| { - let nonce = tx_spec.nonce.unwrap_or_else(|| { - let current = *nonce_counter; - *nonce_counter += 1; - current - }); - - sign_tx_with_key_pair( - key_pair, - TxEip2930 { - chain_id: chain_spec.chain.id(), - nonce, - gas_limit: tx_spec.gas_limit, - input: tx_spec.data.clone(), - gas_price: 1_500_000_000, - to: tx_spec.to, - value: tx_spec.value, - ..Default::default() - } - .into(), - ) - }) - .collect(); - - let gas_total = spec.txs.iter().map(|tx| tx.gas_limit).sum::(); - let timestamp = block_timestamp(block_number); - let shanghai = chain_spec.is_shanghai_active_at_timestamp(timestamp); - let cancun = chain_spec.is_cancun_active_at_timestamp(timestamp); - - Block { - header: Header { - parent_hash, - receipts_root: alloy_primitives::b256!( - "0xd3a6acf9a244d78b33831df95d472c4128ea85bf079a1d41e32ed0b7d2244c9e" - ), - difficulty: chain_spec.fork(EthereumHardfork::Paris).ttd().expect("Paris TTD"), - number: block_number, - gas_limit: gas_total.max(MIN_TRANSACTION_GAS), - gas_used: gas_total, - timestamp, - state_root: B256::ZERO, // Will be calculated by executor - withdrawals_root: shanghai.then_some(EMPTY_WITHDRAWALS), - blob_gas_used: cancun.then_some(0), - excess_blob_gas: cancun.then_some(0), - parent_beacon_block_root: cancun.then(|| keccak256(block_number.to_be_bytes())), - ..Default::default() - }, - body: BlockBody { - transactions, - withdrawals: shanghai.then(Withdrawals::default), - ..Default::default() - }, - } - .try_into_recovered() - .unwrap() -} - /// Executes a block and returns the updated block with correct state root fn execute_block( block: &mut RecoveredBlock, @@ -330,26 +132,6 @@ where Ok(()) } -/// Seeds `storage` with reth's state and trie at `block`, the proofs store's starting point. -fn initialize_storage( - storage: S, - provider_factory: &ProviderFactory, - block: BlockNumHash, -) -> eyre::Result<()> -where - N: NodeTypesWithDB, - S: OpProofsStore + Send + Sync + Clone + std::fmt::Debug + 'static, -{ - let trie_layout = if provider_factory.cached_storage_settings().is_v2() { - RethTrieStorageLayout::Packed - } else { - RethTrieStorageLayout::Legacy - }; - let tx = provider_factory.db_ref().tx()?; - InitializationJob::new(storage, tx, trie_layout).run(block.number, block.hash)?; - Ok(()) -} - /// Runs a test scenario with the given configuration fn run_test_scenario( scenario: TestScenario, diff --git a/rust/op-reth/crates/trie/tests/live/catch_up_under_pruning.rs b/rust/op-reth/crates/trie/tests/live/catch_up_under_pruning.rs index 54564ba30e3..06aa14ff619 100644 --- a/rust/op-reth/crates/trie/tests/live/catch_up_under_pruning.rs +++ b/rust/op-reth/crates/trie/tests/live/catch_up_under_pruning.rs @@ -129,11 +129,11 @@ fn transfer_block(number: u64) -> BlockSpec { /// Every address the chain touches, none of which has storage. fn proof_targets(sender: Address) -> Vec { - [sender, Address::ZERO] - .into_iter() - .chain((1..=CATCH_UP_LENGTH + REAL_TIME_LENGTH).map(recipient)) - .map(|address| ProofTarget { address, slots: Vec::new() }) - .collect() + ProofTarget::accounts( + [sender, Address::ZERO] + .into_iter() + .chain((1..=CATCH_UP_LENGTH + REAL_TIME_LENGTH).map(recipient)), + ) } fn persisted_tip(store: &S) -> eyre::Result { diff --git a/rust/op-reth/crates/trie/tests/live/chain.rs b/rust/op-reth/crates/trie/tests/live/chain.rs new file mode 100644 index 00000000000..22f2657882f --- /dev/null +++ b/rust/op-reth/crates/trie/tests/live/chain.rs @@ -0,0 +1,241 @@ +//! The test chain the live targets share: a funded signer, block and transaction specs, the +//! block builder, the store constructors and the store's initialization at a block. + +use alloy_consensus::{ + Header, TxEip2930, + constants::{EMPTY_WITHDRAWALS, ETH_TO_WEI}, +}; +use alloy_eips::{BlockNumHash, eip4895::Withdrawals}; +use alloy_genesis::{Genesis, GenesisAccount}; +use alloy_primitives::{Address, B256, Bytes, TxKind, U256, keccak256}; +use derive_more::Constructor; +use reth_chainspec::{ChainSpec, ChainSpecBuilder, EthereumHardfork, EthereumHardforks, MAINNET}; +use reth_db::Database; +use reth_ethereum_primitives::{Block, BlockBody, Transaction, TransactionSigned}; +use reth_node_api::NodeTypesWithDB; +use reth_optimism_trie::{ + MdbxProofsStorage, OpProofsStore, RethTrieStorageLayout, initialize::InitializationJob, +}; +use reth_primitives_traits::{Block as _, RecoveredBlock}; +use reth_provider::{ProviderFactory, StorageSettingsCache}; +use secp256k1::Keypair; +use std::sync::Arc; +use tempfile::TempDir; + +pub(super) use reth_chainspec::MIN_TRANSACTION_GAS; + +/// Seconds between consecutive blocks. +const BLOCK_TIME: u64 = 2; + +/// A proof window no test chain fills, so that the pruner never moves the earliest block. +pub(super) const WHOLE_WINDOW: u64 = 1000; + +/// Timestamp of the block at `number`. +pub(super) const fn block_timestamp(number: u64) -> u64 { + number * BLOCK_TIME +} + +/// The engine's persistence and backpressure thresholds. +#[derive(Clone, Copy, Debug)] +pub(super) struct FlushCadence { + pub(super) backpressure: u64, + pub(super) persistence: u64, +} + +impl FlushCadence { + pub(super) const EVERY_BLOCK: Self = Self { backpressure: 2, persistence: 1 }; + pub(super) const EVERY_FIFTH_BLOCK: Self = Self { backpressure: 10, persistence: 5 }; +} + +/// Which store implementation a case runs over. +#[derive(Clone, Copy, Debug)] +pub(super) enum Backend { + InMemory, + Mdbx, +} + +pub(super) fn create_mdbx_proofs_storage() -> Arc { + let path = TempDir::new().unwrap().keep(); + Arc::new(MdbxProofsStorage::new(&path).unwrap()) +} + +/// Converts a secp256k1 public key to an Ethereum address. +pub(super) fn public_key_to_address(pubkey: secp256k1::PublicKey) -> Address { + let hash = keccak256(&pubkey.serialize_uncompressed()[1..]); + Address::from_slice(&hash[12..]) +} + +/// Signs a transaction with the given keypair. +fn sign_tx_with_key_pair(key_pair: Keypair, tx: Transaction) -> TransactionSigned { + use alloy_consensus::SignableTransaction; + use reth_primitives_traits::crypto::secp256k1::sign_message; + let secret = B256::from_slice(&key_pair.secret_bytes()); + let sig = sign_message(secret, tx.signature_hash()).unwrap(); + tx.into_signed(sig).into() +} + +/// Specification for a transaction within a block +#[derive(Debug, Clone)] +pub(super) struct TxSpec { + /// Call recipient or contract creation. + pub(super) to: TxKind, + /// Transaction input, including initcode for deployments. + pub(super) data: Bytes, + /// Maximum execution gas. + pub(super) gas_limit: u64, + /// Value to transfer (in wei) + pub(super) value: U256, + /// Nonce for the transaction (will be automatically assigned if None) + pub(super) nonce: Option, +} + +impl TxSpec { + /// Create a simple transfer transaction + pub(super) const fn transfer(to: Address, value: U256) -> Self { + Self { + to: TxKind::Call(to), + data: Bytes::new(), + gas_limit: MIN_TRANSACTION_GAS, + value, + nonce: None, + } + } + + /// Create a contract deployment running `initcode`. + pub(super) const fn create(initcode: Bytes, gas_limit: u64) -> Self { + Self { to: TxKind::Create, data: initcode, gas_limit, value: U256::ZERO, nonce: None } + } + + /// Create a call to `contract` with `data` as input. + pub(super) const fn call(contract: Address, data: Bytes, gas_limit: u64) -> Self { + Self { to: TxKind::Call(contract), data, gas_limit, value: U256::ZERO, nonce: None } + } + + /// Send `value` along with the transaction. + pub(super) fn with_value(self, value: U256) -> Self { + Self { value, ..self } + } +} + +/// Specification for a block in the test chain +#[derive(Debug, Clone, Constructor)] +pub(super) struct BlockSpec { + /// Transactions to include in this block + pub(super) txs: Vec, +} + +impl BlockSpec { + /// A block without transactions. + pub(super) const fn empty() -> Self { + Self { txs: Vec::new() } + } +} + +/// A mainnet-shaped chain spec whose genesis allocation is `alloc`, with the hardforks still to +/// be chosen. +pub(super) fn chain_spec_builder( + alloc: impl IntoIterator, +) -> ChainSpecBuilder { + ChainSpecBuilder::default() + .chain(MAINNET.chain) + .genesis(Genesis { alloc: alloc.into_iter().collect(), ..MAINNET.genesis.clone() }) +} + +/// A genesis allocation funding `address`. +pub(super) fn funded(address: Address) -> (Address, GenesisAccount) { + (address, GenesisAccount { balance: U256::from(10 * ETH_TO_WEI), ..Default::default() }) +} + +/// Helper to create a chain spec with a genesis account funded +pub(super) fn chain_spec_with_address(address: Address) -> Arc { + Arc::new(chain_spec_builder([funded(address)]).paris_activated().build()) +} + +/// Creates a block from a spec, executing transactions with the given keypair +pub(super) fn create_block_from_spec( + spec: &BlockSpec, + block_number: u64, + parent_hash: B256, + chain_spec: &Arc, + key_pair: Keypair, + nonce_counter: &mut u64, +) -> RecoveredBlock { + let transactions: Vec = spec + .txs + .iter() + .map(|tx_spec| { + let nonce = tx_spec.nonce.unwrap_or_else(|| { + let current = *nonce_counter; + *nonce_counter += 1; + current + }); + + sign_tx_with_key_pair( + key_pair, + TxEip2930 { + chain_id: chain_spec.chain.id(), + nonce, + gas_limit: tx_spec.gas_limit, + input: tx_spec.data.clone(), + gas_price: 1_500_000_000, + to: tx_spec.to, + value: tx_spec.value, + ..Default::default() + } + .into(), + ) + }) + .collect(); + + let gas_total = spec.txs.iter().map(|tx| tx.gas_limit).sum::(); + let timestamp = block_timestamp(block_number); + let shanghai = chain_spec.is_shanghai_active_at_timestamp(timestamp); + let cancun = chain_spec.is_cancun_active_at_timestamp(timestamp); + + Block { + header: Header { + parent_hash, + receipts_root: alloy_primitives::b256!( + "0xd3a6acf9a244d78b33831df95d472c4128ea85bf079a1d41e32ed0b7d2244c9e" + ), + difficulty: chain_spec.fork(EthereumHardfork::Paris).ttd().expect("Paris TTD"), + number: block_number, + gas_limit: gas_total.max(MIN_TRANSACTION_GAS), + gas_used: gas_total, + timestamp, + state_root: B256::ZERO, // Will be calculated by executor + withdrawals_root: shanghai.then_some(EMPTY_WITHDRAWALS), + blob_gas_used: cancun.then_some(0), + excess_blob_gas: cancun.then_some(0), + parent_beacon_block_root: cancun.then(|| keccak256(block_number.to_be_bytes())), + ..Default::default() + }, + body: BlockBody { + transactions, + withdrawals: shanghai.then(Withdrawals::default), + ..Default::default() + }, + } + .try_into_recovered() + .unwrap() +} + +/// Seeds `storage` with reth's state and trie at `block`, the proofs store's starting point. +pub(super) fn initialize_storage( + storage: S, + provider_factory: &ProviderFactory, + block: BlockNumHash, +) -> eyre::Result<()> +where + N: NodeTypesWithDB, + S: OpProofsStore + Send + Sync + Clone + std::fmt::Debug + 'static, +{ + let trie_layout = if provider_factory.cached_storage_settings().is_v2() { + RethTrieStorageLayout::Packed + } else { + RethTrieStorageLayout::Legacy + }; + let tx = provider_factory.db_ref().tx()?; + InitializationJob::new(storage, tx, trie_layout).run(block.number, block.hash)?; + Ok(()) +} diff --git a/rust/op-reth/crates/trie/tests/live/reorg_revert_edges.rs b/rust/op-reth/crates/trie/tests/live/reorg_revert_edges.rs index 88d3155eefa..869b4ebedfe 100644 --- a/rust/op-reth/crates/trie/tests/live/reorg_revert_edges.rs +++ b/rust/op-reth/crates/trie/tests/live/reorg_revert_edges.rs @@ -11,7 +11,10 @@ use super::{ Backend, BlockSpec, FlushCadence, TxSpec, WHOLE_WINDOW, chain_spec_with_address, create_block_from_spec, create_mdbx_proofs_storage, initialize_storage, public_key_to_address, - simulated_reth::{RethPersistence, SimulatedReth, fixed_signer, index_executed_block}, + simulated_reth::{ + RethPersistence, SimulatedReth, UNRELATED_PARENT, fixed_signer, index_executed_block, + reorg_block, reorg_block_without_data, + }, store_view::{BlockView, ProofTarget, assert_same_view}, }; use alloy_eips::{NumHash, eip1898::BlockWithParent}; @@ -56,9 +59,6 @@ const RETH_PERSISTENCE_THRESHOLD: u64 = 2 * MAX_BLOCKS_PER_CASE; /// under either cadence. const PRUNED_WINDOW: u64 = 3; -/// A parent hash no block here has. -const UNRELATED_PARENT: B256 = B256::repeat_byte(0xBA); - /// How long a case waits for the engine's persistence thread to reach a held write. const HELD_WRITE_TIMEOUT: Duration = Duration::from_secs(30); @@ -86,14 +86,9 @@ fn fork_blocks(numbers: RangeInclusive) -> Vec { /// Every address the chain or a fork can touch, none of which has storage. fn proof_targets(sender: Address) -> Vec { - [sender, Address::ZERO] - .into_iter() - .chain( - (1..=MAX_BLOCKS_PER_CASE) - .flat_map(|number| [recipient(number), fork_recipient(number)]), - ) - .map(|address| ProofTarget { address, slots: Vec::new() }) - .collect() + ProofTarget::accounts([sender, Address::ZERO].into_iter().chain( + (1..=MAX_BLOCKS_PER_CASE).flat_map(|number| [recipient(number), fork_recipient(number)]), + )) } /// The disk tip once the engine has saved what its cadence lets it save: `buffered` blocks went @@ -126,15 +121,7 @@ impl ReorgPath { Self::Index => false, Self::Mixed => position.is_multiple_of(2), }; - if execute { - ReorgBlock::Execute(Arc::clone(&block.recovered_block)) - } else { - ReorgBlock::Index { - block: block.recovered_block().block_with_parent(), - hashed_state: block.hashed_state(), - trie_updates: block.trie_updates(), - } - } + reorg_block(block, execute) } } @@ -143,15 +130,6 @@ fn reorg_blocks(blocks: &[ExecutedBlock], path: ReorgPath) -> Vec ReorgBlock { - ReorgBlock::Index { - block, - hashed_state: Arc::new(HashedPostStateSorted::default()), - trie_updates: Arc::new(TrieUpdatesSorted::default()), - } -} - /// Which engine task unwinds. #[derive(Clone, Copy, Debug)] enum UnwindingTask { diff --git a/rust/op-reth/crates/trie/tests/live/simulated_reth.rs b/rust/op-reth/crates/trie/tests/live/simulated_reth.rs index 2c59b83b682..27d432aa483 100644 --- a/rust/op-reth/crates/trie/tests/live/simulated_reth.rs +++ b/rust/op-reth/crates/trie/tests/live/simulated_reth.rs @@ -3,7 +3,7 @@ use super::{BlockSpec, WHOLE_WINDOW, create_block_from_spec}; use alloy_consensus::BlockHeader; -use alloy_eips::BlockNumHash; +use alloy_eips::{BlockNumHash, eip1898::BlockWithParent}; use alloy_primitives::B256; use reth_chain_state::{CanonicalInMemoryState, ExecutedBlock, NewCanonicalChain}; use reth_chainspec::ChainSpec; @@ -14,9 +14,9 @@ use reth_evm::{ConfigureEvm, execute::Executor}; use reth_evm_ethereum::EthEvmConfig; use reth_optimism_trie::{ OpProofStoragePruner, OpProofsStore, - engine::{EngineError, EngineHandle}, + engine::{EngineError, EngineHandle, ReorgBlock}, }; -use reth_primitives_traits::RecoveredBlock; +use reth_primitives_traits::{RecoveredBlock, SealedHeader}; use reth_provider::{ HashedPostStateProvider, ProviderFactory, SaveBlocksInput, StateProviderFactory, StateRootProvider, StaticFileProviderFactory, StaticFileSegment, @@ -25,13 +25,39 @@ use reth_provider::{ }; use reth_revm::database::StateProviderDatabase; use reth_storage_overlay::OverlayManager; -use reth_trie_common::ComputedTrieData; +use reth_trie_common::{ComputedTrieData, HashedPostStateSorted, updates::TrieUpdatesSorted}; use secp256k1::{Keypair, Secp256k1}; use std::sync::Arc; /// reth's default `--engine.memory-block-buffer-target`. const MEMORY_BLOCK_BUFFER_TARGET: u64 = 5; +/// A parent hash no block here has. +pub(super) const UNRELATED_PARENT: B256 = B256::repeat_byte(0xBA); + +/// `block` the way the ExEx hands a block of a reorg to the engine: executed, or indexed with +/// the trie data reth computed for it. +pub(super) fn reorg_block(block: &ExecutedBlock, execute: bool) -> ReorgBlock { + if execute { + ReorgBlock::Execute(Arc::clone(&block.recovered_block)) + } else { + ReorgBlock::Index { + block: block.recovered_block().block_with_parent(), + hashed_state: block.hashed_state(), + trie_updates: block.trie_updates(), + } + } +} + +/// The reorg task's input for `block` without trie data, enough to reach the unwind checks. +pub(super) fn reorg_block_without_data(block: BlockWithParent) -> ReorgBlock { + ReorgBlock::Index { + block, + hashed_state: Arc::new(HashedPostStateSorted::default()), + trie_updates: Arc::new(TrieUpdatesSorted::default()), + } +} + /// Hands `block` to `engine` the way the ExEx indexes a block reth validated. pub(super) fn index_executed_block( engine: &EngineHandle, @@ -321,12 +347,7 @@ impl SimulatedReth { assert!(number > self.last_persisted.number, "only in-memory blocks can be reorged"); let (first_spec, rest) = specs.split_first().expect("a reorg switches to a new block"); let old = self.canonical.split_off(number as usize - 1); - // The signer's nonce is the number of transactions the remaining chain holds. - self.nonce = self - .canonical - .iter() - .map(|block| block.recovered_block().body().transactions.len() as u64) - .sum(); + self.nonce = self.transactions_in_chain(); let fork = self.build_block(first_spec)?; assert_ne!( fork.recovered_block().hash(), @@ -357,6 +378,27 @@ impl SimulatedReth { Ok(deleted.iter().filter_map(|header| header.block_range()).map(|range| range.end()).max()) } + /// Drops the canonical blocks above `number` without replacing them, the way a forkchoice + /// update back to an ancestor does. Only in-memory blocks can go. + pub(super) fn revert_to(&mut self, number: u64) { + assert!(number >= self.last_persisted.number, "only in-memory blocks can be reverted"); + let old = self.canonical.split_off(number as usize); + self.nonce = self.transactions_in_chain(); + self.in_memory.update_chain(NewCanonicalChain::Reorg { new: Vec::new(), old }); + self.in_memory.set_canonical_head(self.sealed_header(number)); + } + + /// Loses every block held only in memory and starts over from the database, the way reth + /// does after an unclean stop: the head is the persisted tip and nothing is in memory. + pub(super) fn crash_and_restart(&mut self) -> eyre::Result<()> { + self.provider = BlockchainProvider::new(self.factory.clone())?; + self.in_memory = self.provider.canonical_in_memory_state(); + self.canonical.truncate(self.last_persisted.number as usize); + self.last_evicted = self.last_persisted; + self.nonce = self.transactions_in_chain(); + Ok(()) + } + /// Writes every canonical block to the database without evicting any from memory, the state /// a `proofs init` run finds on a node that has caught up. pub(super) fn persist_all(&mut self) -> eyre::Result<()> { @@ -373,6 +415,21 @@ impl SimulatedReth { Ok(()) } + fn sealed_header(&self, number: u64) -> SealedHeader { + match number { + 0 => self.chain_spec.sealed_genesis_header(), + _ => self.block(number).clone_sealed_header(), + } + } + + /// The signer's nonce: the number of transactions the canonical chain holds. + fn transactions_in_chain(&self) -> u64 { + self.canonical + .iter() + .map(|block| block.recovered_block().body().transactions.len() as u64) + .sum() + } + fn make_canonical(&mut self, update: NewCanonicalChain) { let tip = update.tip().clone_sealed_header(); if let NewCanonicalChain::Reorg { old, .. } = &update { diff --git a/rust/op-reth/crates/trie/tests/live/store_view.rs b/rust/op-reth/crates/trie/tests/live/store_view.rs index 795ac337f00..3124e498d3b 100644 --- a/rust/op-reth/crates/trie/tests/live/store_view.rs +++ b/rust/op-reth/crates/trie/tests/live/store_view.rs @@ -21,6 +21,13 @@ pub(super) struct ProofTarget { pub(super) slots: Vec, } +impl ProofTarget { + /// Targets for `addresses`, none of which has storage. + pub(super) fn accounts(addresses: impl IntoIterator) -> Vec { + addresses.into_iter().map(|address| Self { address, slots: Vec::new() }).collect() + } +} + /// One account's storage: its slots and its storage trie nodes. pub(super) struct AccountStorage { pub(super) slots: BTreeMap, @@ -42,7 +49,16 @@ impl BlockView { number: u64, targets: &[ProofTarget], ) -> eyre::Result { - let provider = store.provider_ro()?; + Self::read_from(&store.provider_ro()?, number, targets) + } + + /// Reads the view through `provider`, so that several views and the window come from one + /// read transaction while saves go on. + pub(super) fn read_from( + provider: &P, + number: u64, + targets: &[ProofTarget], + ) -> eyre::Result { let accounts = collect_hashed(provider.account_hashed_cursor(number)?)?; let hashed_addresses: BTreeSet = accounts .keys()