From 9485637b4a17f7ae9622f74cfd95e900e7301b97 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Thu, 15 Oct 2026 06:52:03 -0700 Subject: [PATCH 1/4] op-reth: model-based test of the proofs engine state machine A proptest-driven integration target that runs random interleavings of commits, reorgs, reverts, sync targets, flushes, failed saves, stale notifications and crashes over reth's persistence model, and holds the proofs engine and its store to a reference model after every step. The matrix samples reth's persistence threshold and masking window, the engine's flush cadence, the proofs path and the proof window per case. Two ignored cases reproduce tracked bugs: a crash after which reth rebuilds a different chain leaves the store's latest block off the canonical chain (ethereum-optimism/protocol-team#464), and a reth that prunes bodies through the acknowledged tip stalls the sync loop (ethereum-optimism/protocol-team#478). Seams: EngineHandle::tip() under the test-utils cfg next to flush(), the live target's chain and reorg helpers moved to tests/live/chain.rs and simulated_reth.rs so both targets share them, SimulatedReth::{persisted, revert_to, crash_and_restart}, BlockView::read_from and ProofTarget::accounts. nextest gives the engine_model binary a longer slow-timeout, since its whole proptest is one test. Co-Authored-By: Claude Fable 5.1 (cherry picked from commit 9e94c089b1b587f9596206f7f990f21a6e0f608b) (cherry picked from commit 8ebb72bfd2800c851a37898be3ac25a57261f504) --- rust/.config/nextest.toml | 6 + rust/Cargo.lock | 1 + rust/op-reth/crates/trie/Cargo.toml | 1 + rust/op-reth/crates/trie/src/engine/handle.rs | 14 + rust/op-reth/crates/trie/src/engine/mod.rs | 5 + .../crates/trie/src/engine/tasks/mod.rs | 4 + .../crates/trie/src/engine/tasks/tip.rs | 30 + .../op-reth/crates/trie/tests/engine_model.rs | 123 ++++ .../crates/trie/tests/engine_model/actions.rs | 126 ++++ .../crates/trie/tests/engine_model/faults.rs | 71 +++ .../crates/trie/tests/engine_model/harness.rs | 569 ++++++++++++++++++ .../trie/tests/engine_model/invariants.rs | 170 ++++++ .../crates/trie/tests/engine_model/model.rs | 477 +++++++++++++++ rust/op-reth/crates/trie/tests/live.rs | 240 +------- .../trie/tests/live/catch_up_under_pruning.rs | 10 +- rust/op-reth/crates/trie/tests/live/chain.rs | 241 ++++++++ .../trie/tests/live/reorg_revert_edges.rs | 38 +- .../crates/trie/tests/live/simulated_reth.rs | 77 ++- .../crates/trie/tests/live/store_view.rs | 18 +- 19 files changed, 1945 insertions(+), 276 deletions(-) create mode 100644 rust/op-reth/crates/trie/src/engine/tasks/tip.rs create mode 100644 rust/op-reth/crates/trie/tests/engine_model.rs create mode 100644 rust/op-reth/crates/trie/tests/engine_model/actions.rs create mode 100644 rust/op-reth/crates/trie/tests/engine_model/faults.rs create mode 100644 rust/op-reth/crates/trie/tests/engine_model/harness.rs create mode 100644 rust/op-reth/crates/trie/tests/engine_model/invariants.rs create mode 100644 rust/op-reth/crates/trie/tests/engine_model/model.rs create mode 100644 rust/op-reth/crates/trie/tests/live/chain.rs diff --git a/rust/.config/nextest.toml b/rust/.config/nextest.toml index 9cb49f19920..50a953a4333 100644 --- a/rust/.config/nextest.toml +++ b/rust/.config/nextest.toml @@ -31,6 +31,12 @@ filter = 'test(=log_estimate_bounds_measured_pgu) | test(=consolidation_tests::c slow-timeout = { period = "1m", terminate-after = 10 } threads-required = "num-test-threads" +# The proofs engine model test runs its whole proptest as one test, and a failing case shrinks +# for up to a minute on top of the run. +[[profile.default.overrides]] +filter = "binary(engine_model)" +slow-timeout = { period = "1m", terminate-after = 5 } + # E2E tests using the testsuite framework from crates/e2e-test-utils. [[profile.default.overrides]] filter = "binary(e2e_testsuite)" diff --git a/rust/Cargo.lock b/rust/Cargo.lock index cc19affc315..fcee1c9c24e 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -14312,6 +14312,7 @@ dependencies = [ "metrics", "mockall", "parking_lot", + "proptest", "reth-chain-state", "reth-chainspec", "reth-codecs", diff --git a/rust/op-reth/crates/trie/Cargo.toml b/rust/op-reth/crates/trie/Cargo.toml index 5ba6a8d34c8..5fd15bfa395 100644 --- a/rust/op-reth/crates/trie/Cargo.toml +++ b/rust/op-reth/crates/trie/Cargo.toml @@ -82,6 +82,7 @@ secp256k1 = { workspace = true, features = ["rand", "std"] } mockall.workspace = true rstest.workspace = true eyre.workspace = true +proptest.workspace = true # misc serial_test.workspace = true diff --git a/rust/op-reth/crates/trie/src/engine/handle.rs b/rust/op-reth/crates/trie/src/engine/handle.rs index add5a23a460..4b80a7ef0be 100644 --- a/rust/op-reth/crates/trie/src/engine/handle.rs +++ b/rust/op-reth/crates/trie/src/engine/handle.rs @@ -9,6 +9,8 @@ use super::{ tasks::{ExecuteBlockTask, IndexBlockTask, ReorgBlock, ReorgTask, SyncToTask, UnwindTask}, }; use crate::{OpProofStoragePruner, OpProofsStore}; +#[cfg(any(test, feature = "test-utils"))] +use alloy_eips::NumHash; use alloy_eips::eip1898::BlockWithParent; use crossbeam_channel::{Sender, bounded}; use reth_evm::ConfigureEvm; @@ -170,4 +172,16 @@ impl EngineHandle let _ = reply_rx.recv(); } } + + /// The engine's tip: the newest buffered block, or the store's latest block when nothing is + /// buffered (test/utility only). + #[cfg(any(test, feature = "test-utils"))] + pub fn tip(&self) -> Result { + use super::tasks::TipTask; + let (reply_tx, reply_rx) = bounded(1); + self.sender + .send(EngineAction::Tip(TipTask { reply: reply_tx })) + .map_err(|_| EngineError::EngineDied)?; + reply_rx.recv().map_err(|_| EngineError::EngineDied)? + } } diff --git a/rust/op-reth/crates/trie/src/engine/mod.rs b/rust/op-reth/crates/trie/src/engine/mod.rs index 267a9cb21f6..975c39a5544 100644 --- a/rust/op-reth/crates/trie/src/engine/mod.rs +++ b/rust/op-reth/crates/trie/src/engine/mod.rs @@ -60,6 +60,9 @@ enum EngineAction { Flush(tasks::FlushTask), /// Update the sync catch-up target (fire-and-forget). SyncTo(tasks::SyncToTask), + /// Report the engine's tip. + #[cfg(any(test, feature = "test-utils"))] + Tip(tasks::TipTask), } impl EngineAction { @@ -85,6 +88,8 @@ impl EngineAction { #[cfg(any(test, feature = "test-utils"))] Self::Flush(task) => task.execute(state), Self::SyncTo(task) => task.execute(state), + #[cfg(any(test, feature = "test-utils"))] + Self::Tip(task) => task.execute(state), } } } diff --git a/rust/op-reth/crates/trie/src/engine/tasks/mod.rs b/rust/op-reth/crates/trie/src/engine/tasks/mod.rs index aaf462c27d4..8091e14a82b 100644 --- a/rust/op-reth/crates/trie/src/engine/tasks/mod.rs +++ b/rust/op-reth/crates/trie/src/engine/tasks/mod.rs @@ -11,6 +11,8 @@ mod index_block; mod reorg; mod reth_auxiliary; mod sync_to; +#[cfg(any(test, feature = "test-utils"))] +mod tip; mod unwind; pub(super) use execute_block::{ExecuteBlockTask, run as execute_block}; @@ -20,4 +22,6 @@ pub(super) use index_block::{IndexBlockTask, run as index_block}; pub use reorg::ReorgBlock; pub(super) use reorg::ReorgTask; pub(super) use sync_to::SyncToTask; +#[cfg(any(test, feature = "test-utils"))] +pub(super) use tip::TipTask; pub(super) use unwind::UnwindTask; diff --git a/rust/op-reth/crates/trie/src/engine/tasks/tip.rs b/rust/op-reth/crates/trie/src/engine/tasks/tip.rs new file mode 100644 index 00000000000..d405fd56f5a --- /dev/null +++ b/rust/op-reth/crates/trie/src/engine/tasks/tip.rs @@ -0,0 +1,30 @@ +use super::super::state::EngineState; +use crate::{OpProofsStore, engine::EngineError}; +use alloy_eips::NumHash; +use crossbeam_channel::Sender; +use reth_evm::ConfigureEvm; +use reth_primitives_traits::BlockTy; +use reth_provider::{ + BlockHashReader, BlockReader, DatabaseProviderFactory, StateProviderFactory, StateReader, +}; + +pub(crate) struct TipTask { + pub(crate) reply: Sender>, +} + +impl TipTask { + pub(crate) fn execute(self, state: &EngineState) + where + Evm: ConfigureEvm, + Provider: BlockHashReader + + StateReader + + DatabaseProviderFactory + + StateProviderFactory + + BlockReader> + + Clone + + 'static, + Store: OpProofsStore + Clone + 'static, + { + let _ = self.reply.send(state.get_tip()); + } +} diff --git a/rust/op-reth/crates/trie/tests/engine_model.rs b/rust/op-reth/crates/trie/tests/engine_model.rs new file mode 100644 index 00000000000..e80e75b9ccd --- /dev/null +++ b/rust/op-reth/crates/trie/tests/engine_model.rs @@ -0,0 +1,123 @@ +//! Model-based test of the proofs engine: random interleavings of commits, reorgs, reverts, sync +//! targets, flushes, failed saves, stale notifications and crashes, driven over reth's +//! persistence model and held to a reference model after every step. The deterministic boundary +//! cases live in the live target; this target is for the interleavings. + +use proptest::prelude::*; + +#[path = "engine_model/actions.rs"] +mod actions; +#[path = "live/chain.rs"] +#[allow(dead_code, reason = "each test target uses part of this module")] +mod chain; +#[path = "engine_model/faults.rs"] +mod faults; +#[path = "engine_model/harness.rs"] +mod harness; +#[path = "engine_model/invariants.rs"] +mod invariants; +#[path = "engine_model/model.rs"] +mod model; +#[path = "live/simulated_reth.rs"] +#[allow(dead_code, reason = "each test target uses part of this module")] +mod simulated_reth; +#[path = "live/store_view.rs"] +#[allow(dead_code, reason = "each test target uses part of this module")] +mod store_view; + +use actions::{Action, Actions, Config, Rebuild}; +use chain::*; +use harness::Harness; +use proptest::test_runner::contextualize_config; +use simulated_reth::RethPersistence; +use std::sync::atomic::{AtomicBool, Ordering}; + +/// Cases per run unless `PROPTEST_CASES` says otherwise. +const DEFAULT_CASES: u32 = 24; + +/// The default count and a shrinking budget that fits the test's time limit, with the +/// `PROPTEST_*` environment variables applied on top. +fn proptest_config() -> ProptestConfig { + contextualize_config(ProptestConfig { + cases: DEFAULT_CASES, + failure_persistence: None, + max_shrink_iters: 256, + max_shrink_time: 60_000, + ..ProptestConfig::default() + }) +} + +fn run_sequence(config: Config, actions: &[Action]) -> eyre::Result<()> { + Harness::new(config)?.run(actions) +} + +static FIRST_FAILURE_PRINTED: AtomicBool = AtomicBool::new(false); + +/// Runs a case and prints the first failure as found, before shrinking, so that a run cut short +/// by a time limit still shows what failed. +fn run_case(config: Config, actions: &[Action]) -> Result<(), TestCaseError> { + run_sequence(config, actions).map_err(|err| { + if !FIRST_FAILURE_PRINTED.swap(true, Ordering::SeqCst) { + eprintln!("first failing case, before shrinking: {config:?}\n{err:?}\n"); + } + TestCaseError::fail(format!("{err:?}")) + }) +} + +proptest! { + #![proptest_config(proptest_config())] + + /// Under every cell of the matrix, the engine's tip and the store's window stay on the + /// canonical chain and within what the model allows after each random step, every valid + /// event is taken, every reorg on a wrong parent or below the window is refused, and the + /// engine settles within the step bound. + #[test] + fn engine_and_store_follow_the_model( + config in actions::config(), + actions in actions::sequence(Actions::DEFAULT), + ) { + run_case(config, &actions)?; + } + + /// As above, with crashes after which reth rebuilds a different chain, as a sequencer does. + #[test] + #[ignore = "fails on develop: after an unclean stop the store keeps blocks reth lost, and a chain rebuilt differently leaves the store's latest block off the canonical chain (ethereum-optimism/protocol-team#464; the harness's restart must gain the startup reconcile of #23358 to pass)"] + fn engine_and_store_follow_the_model_through_crashes_rebuilding_a_different_chain( + config in actions::config(), + actions in actions::sequence(Actions::WITH_TRACKED_BUGS), + ) { + run_case(config, &actions)?; + } +} + +/// The matrix cell the replayed sequences run under. +fn replay_config(cadence: FlushCadence, persistence_threshold: u64) -> Config { + Config { + body_pruning: false, + cadence, + persistence: RethPersistence::threshold(persistence_threshold), + verification_interval: 0, + window: WHOLE_WINDOW, + } +} + +/// The minimal sequence the random run shrinks to for protocol-team#464: one block saved, then a +/// crash reth comes back from with a different block 1, which the store keeps serving. +#[test] +#[ignore = "fails on develop: the store's latest block 1 is not on the chain reth rebuilt after the crash (ethereum-optimism/protocol-team#464; the harness's restart must gain the startup reconcile of #23358 to pass)"] +fn store_follows_reth_through_a_crash_rebuilding_a_different_chain() -> eyre::Result<()> { + run_sequence( + replay_config(FlushCadence::EVERY_BLOCK, 50), + &[Action::Commit { count: 1 }, Action::Crash(Rebuild::Different)], + ) +} + +/// The ExEx acknowledges a notification's tip before the sync loop has executed the bodies +/// below it, so a reth that prunes bodies through the acknowledged height takes them from under +/// the sync loop and the catch-up stalls. +#[test] +#[ignore = "fails on develop: reth prunes the bodies the sync loop still needs because the ExEx acknowledges the notification tip, and the engine never settles (ethereum-optimism/protocol-team#478)"] +fn sync_loop_completes_while_reth_prunes_bodies_through_the_acknowledged_tip() -> eyre::Result<()> { + let config = Config { body_pruning: true, ..replay_config(FlushCadence::EVERY_BLOCK, 7) }; + run_sequence(config, &[Action::SyncTo { count: 40 }, Action::Commit { count: 1 }]) +} diff --git a/rust/op-reth/crates/trie/tests/engine_model/actions.rs b/rust/op-reth/crates/trie/tests/engine_model/actions.rs new file mode 100644 index 00000000000..54d3e78a641 --- /dev/null +++ b/rust/op-reth/crates/trie/tests/engine_model/actions.rs @@ -0,0 +1,126 @@ +//! What a step can do to reth, the ExEx and the node, and the configuration a run samples. + +use super::{FlushCadence, WHOLE_WINDOW, simulated_reth::RethPersistence}; +use proptest::{collection::vec, prelude::*, sample::select}; + +/// Steps per case at most. +const MAX_STEPS: usize = 24; + +/// A proof window the chains here outgrow, so that pruning moves the earliest block. +const PRUNED_WINDOW: u64 = 4; + +/// How reth rebuilds the blocks a crash lost: from the same consensus-layer chain, as a verifier +/// does, or as different blocks, as a sequencer does. +#[derive(Clone, Copy, Debug)] +pub(crate) enum Rebuild { + Different, + Same, +} + +#[derive(Clone, Copy, Debug)] +pub(crate) enum Action { + /// A one-block reorg on a parent no block has, at a fork point the engine knows, which it + /// must refuse. + BogusReorg { offset: u64 }, + /// reth commits `count` blocks, each delivered to the engine as a live notification. + Commit { count: u64 }, + /// reth and the engine stop uncleanly, losing what either held in memory, and restart. + Crash(Rebuild), + /// The next save fails at its write transaction, and enough blocks follow to start it. + FailNextSave, + /// Waits for the save in flight, if any. + Flush, + /// reth replaces its newest `depth` in-memory blocks with `length` new ones and delivers + /// the reorg. + Reorg { depth: u64, length: u64 }, + /// reth drops its newest `depth` in-memory blocks and delivers the revert. + Revert { depth: u64 }, + /// reth commits `depth` blocks and reorgs them away before their notifications reach the + /// engine, then the reorg's does. + StaleCommit { depth: u64, length: u64 }, + /// reth commits `count` blocks the engine only learns of through a sync target, as the ExEx + /// hands over a notification far behind reth's head. + SyncTo { count: u64 }, +} + +/// Which extra actions a run may take. +#[derive(Clone, Copy, Debug)] +pub(crate) struct Actions { + /// A crash after which reth rebuilds a different chain (ethereum-optimism/protocol-team#464). + pub(crate) crash_rebuilding_a_different_chain: bool, +} + +impl Actions { + pub(crate) const DEFAULT: Self = Self { crash_rebuilding_a_different_chain: false }; + pub(crate) const WITH_TRACKED_BUGS: Self = Self { crash_rebuilding_a_different_chain: true }; +} + +fn rebuild(actions: Actions) -> BoxedStrategy { + if actions.crash_rebuilding_a_different_chain { + prop_oneof![Just(Rebuild::Same), Just(Rebuild::Different)].boxed() + } else { + Just(Rebuild::Same).boxed() + } +} + +fn action(actions: Actions) -> impl Strategy { + prop_oneof![ + 6 => (1..=4u64).prop_map(|count| Action::Commit { count }), + 3 => (1..=8u64, 1..=4u64).prop_map(|(depth, length)| Action::Reorg { depth, length }), + 2 => (1..=4u64).prop_map(|depth| Action::Revert { depth }), + 3 => (1..=12u64).prop_map(|count| Action::SyncTo { count }), + 3 => Just(Action::Flush), + 1 => Just(Action::FailNextSave), + 2 => (1..=3u64, 1..=3u64).prop_map(|(depth, length)| Action::StaleCommit { depth, length }), + 1 => (0..=8u64).prop_map(|offset| Action::BogusReorg { offset }), + 1 => rebuild(actions).prop_map(Action::Crash), + ] +} + +pub(crate) fn sequence(actions: Actions) -> impl Strategy> { + vec(action(actions), 1..=MAX_STEPS) +} + +/// One cell of the matrix. +#[derive(Clone, Copy, Debug)] +pub(crate) struct Config { + /// reth prunes block bodies through what the ExEx acknowledges, the notification's tip. + pub(crate) body_pruning: bool, + pub(crate) cadence: FlushCadence, + pub(crate) persistence: RethPersistence, + /// Every block at a multiple of this is executed rather than indexed; 0 indexes every block. + pub(crate) verification_interval: u64, + pub(crate) window: u64, +} + +impl Config { + pub(crate) const fn executes(&self, number: u64) -> bool { + self.verification_interval > 0 && number.is_multiple_of(self.verification_interval) + } +} + +/// reth's persistence settings: the thresholds of develop and of reth 2.6, the latter also with +/// its default masking window, which reth only accepts below the threshold. +fn persistence() -> impl Strategy { + select(vec![7u64, 50]).prop_flat_map(|threshold| { + let masking = if threshold == 50 { vec![0u64, 30] } else { vec![0u64] }; + select(masking) + .prop_map(move |masking| RethPersistence::threshold(threshold).masking(masking)) + }) +} + +pub(crate) fn config() -> impl Strategy { + ( + select(vec![FlushCadence::EVERY_BLOCK, FlushCadence::EVERY_FIFTH_BLOCK]), + persistence(), + select(vec![0u64, 1, 4]), + select(vec![WHOLE_WINDOW, PRUNED_WINDOW]), + ) + .prop_map(|(cadence, persistence, verification_interval, window)| Config { + body_pruning: false, + cadence, + persistence, + verification_interval, + window, + }) +} diff --git a/rust/op-reth/crates/trie/tests/engine_model/faults.rs b/rust/op-reth/crates/trie/tests/engine_model/faults.rs new file mode 100644 index 00000000000..783a0069815 --- /dev/null +++ b/rust/op-reth/crates/trie/tests/engine_model/faults.rs @@ -0,0 +1,71 @@ +//! A proofs store whose next write transaction can be made to fail, the way a full disk or a +//! database error fails a save. + +use reth_db::DatabaseError; +use reth_optimism_trie::{OpProofsStorageResult, OpProofsStore}; +use std::{ + fmt, + sync::{ + Arc, + atomic::{AtomicBool, Ordering}, + }, +}; + +#[derive(Clone)] +pub(crate) struct StoreWithSaveFaults { + fail_next_write: Arc, + inner: S, +} + +impl StoreWithSaveFaults { + pub(crate) fn new(inner: S) -> Self { + Self { fail_next_write: Arc::default(), inner } + } + + pub(crate) fn fail_next_write(&self) { + self.fail_next_write.store(true, Ordering::SeqCst); + } + + /// Withdraws a failure that no write has consumed yet; true if there was one. + pub(crate) fn disarm(&self) -> bool { + self.fail_next_write.swap(false, Ordering::SeqCst) + } +} + +impl fmt::Debug for StoreWithSaveFaults { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("StoreWithSaveFaults").field("inner", &self.inner).finish_non_exhaustive() + } +} + +impl OpProofsStore for StoreWithSaveFaults { + type ProviderRO<'a> + = S::ProviderRO<'a> + where + Self: 'a; + + type ProviderRw<'a> + = S::ProviderRw<'a> + where + Self: 'a; + + type Initializer<'a> + = S::Initializer<'a> + where + Self: 'a; + + fn provider_ro<'a>(&'a self) -> OpProofsStorageResult> { + self.inner.provider_ro() + } + + fn provider_rw<'a>(&'a self) -> OpProofsStorageResult> { + if self.fail_next_write.swap(false, Ordering::SeqCst) { + return Err(DatabaseError::Other("injected write failure".to_owned()).into()); + } + self.inner.provider_rw() + } + + fn initialization_provider<'a>(&'a self) -> OpProofsStorageResult> { + self.inner.initialization_provider() + } +} diff --git a/rust/op-reth/crates/trie/tests/engine_model/harness.rs b/rust/op-reth/crates/trie/tests/engine_model/harness.rs new file mode 100644 index 00000000000..1491ee74352 --- /dev/null +++ b/rust/op-reth/crates/trie/tests/engine_model/harness.rs @@ -0,0 +1,569 @@ +//! Drives reth, the ExEx's decisions and the engine through a sequence of actions; the checks +//! after every step are in `invariants.rs`. + +use super::{ + BlockSpec, TxSpec, + actions::{Action, Config, Rebuild}, + chain_spec_with_address, + faults::StoreWithSaveFaults, + initialize_storage, + model::{Model, ModelBlock, Reply}, + public_key_to_address, + simulated_reth::{ + SimulatedReth, UNRELATED_PARENT, fixed_signer, index_executed_block, reorg_block, + reorg_block_without_data, + }, + store_view::ProofTarget, +}; +use alloy_consensus::BlockHeader; +use alloy_eips::{NumHash, eip1898::BlockWithParent}; +use alloy_primitives::{Address, B256, U256}; +use reth_chain_state::ExecutedBlock; +use reth_ethereum_primitives::Block; +use reth_optimism_trie::{ + MdbxProofsStorage, OpProofsProviderRO, OpProofsStorageError, OpProofsStore, ProofWindowRange, + engine::{EngineError, EngineHandle, ReorgBlock}, +}; +use std::{ + mem, + ops::ControlFlow, + sync::{Arc, mpsc}, + thread, + time::Duration, +}; +use tempfile::TempDir; + +/// How long one engine call, or the engine settling after a step, may take before the case +/// fails instead of hanging. +pub(super) const STEP_TIMEOUT: Duration = Duration::from_secs(30); + +/// Recipients the chain pays in turn. +const RECIPIENTS: u64 = 4; + +const fn recipient(serial: u64) -> Address { + Address::repeat_byte(0x40 + (serial % RECIPIENTS) as u8) +} + +/// The block with `serial`: `serial` wei to its recipient, so that no two blocks are alike. +fn block_spec(serial: u64) -> BlockSpec { + BlockSpec::new(vec![TxSpec::transfer(recipient(serial), U256::from(serial))]) +} + +fn model_block(block: &ExecutedBlock, serial: u64) -> ModelBlock { + let recovered = block.recovered_block(); + ModelBlock { + hash: recovered.hash(), + parent: recovered.parent_hash(), + serial, + state_root: recovered.state_root(), + } +} + +/// The engine and the store it runs over, dropped engine first so that every handle on the +/// store is gone before the store is reopened. +struct ProofsNode { + engine: EngineHandle, + store: StoreWithSaveFaults>, +} + +pub(super) struct Harness { + pub(super) config: Config, + /// Set when an engine call hung; the node is then leaked rather than joined. + engine_hung: bool, + /// The fork point of the last reorg or revert, a height the store had to unwind to. + pub(super) last_fork: Option, + log: Vec, + pub(super) model: Model, + node: Option, + pub(super) reth: SimulatedReth, + serial: u64, + store_dir: TempDir, + pub(super) targets: Vec, +} + +impl Drop for Harness { + fn drop(&mut self) { + if self.engine_hung { + mem::forget(self.node.take()); + } + } +} + +impl Harness { + pub(super) fn new(config: Config) -> eyre::Result { + let key_pair = fixed_signer(); + let sender = public_key_to_address(key_pair.public_key()); + let chain_spec = chain_spec_with_address(sender); + let genesis = ModelBlock { + hash: chain_spec.genesis_hash(), + parent: B256::ZERO, + serial: 0, + state_root: chain_spec.genesis_header().state_root, + }; + let reth = if config.body_pruning { + SimulatedReth::with_body_files_of(chain_spec, key_pair, config.persistence, 1)? + } else { + SimulatedReth::new(chain_spec, key_pair, config.persistence)? + }; + let store_dir = TempDir::new()?; + let store = StoreWithSaveFaults::new(Arc::new(MdbxProofsStorage::new(store_dir.path())?)); + initialize_storage(store.clone(), &reth.factory, reth.genesis())?; + let engine = Self::spawn_engine(&reth, &store, config); + let targets = ProofTarget::accounts( + [sender, Address::ZERO].into_iter().chain((0..RECIPIENTS).map(recipient)), + ); + Ok(Self { + config, + engine_hung: false, + last_fork: None, + log: Vec::new(), + model: Model::new(genesis, config.window), + node: Some(ProofsNode { engine, store }), + reth, + serial: 0, + store_dir, + targets, + }) + } + + fn spawn_engine( + reth: &SimulatedReth, + store: &StoreWithSaveFaults>, + config: Config, + ) -> EngineHandle { + reth.spawn_engine_with_window( + store.clone(), + config.cadence.persistence, + config.cadence.backpressure, + config.window, + ) + } + + const fn node(&self) -> &ProofsNode { + self.node.as_ref().expect("the node is up between steps") + } + + const fn engine(&self) -> &EngineHandle { + &self.node().engine + } + + pub(super) const fn store(&self) -> &StoreWithSaveFaults> { + &self.node().store + } + + /// Runs `actions` in order, checking the invariants after each, and reports a failure with + /// the steps that led to it; `ENGINE_MODEL_TRANSCRIPT` in the environment prints the steps + /// of a passing case too. + pub(super) fn run(&mut self, actions: &[Action]) -> eyre::Result<()> { + let result = self.run_steps(actions); + if std::env::var_os("ENGINE_MODEL_TRANSCRIPT").is_some() { + eprintln!("{}\n", self.log.join("\n")); + } + result.map_err(|err| eyre::eyre!("{err:#}\n--- steps ---\n{}", self.log.join("\n"))) + } + + fn run_steps(&mut self, actions: &[Action]) -> eyre::Result<()> { + for (index, action) in actions.iter().enumerate() { + self.log(format!("step {index}: {action:?}")); + if self.step(*action)?.is_break() { + return Ok(()); + } + self.check_invariants()?; + } + self.settle_store()?; + self.check_settled() + } + + pub(super) fn log(&mut self, line: String) { + self.log.push(line); + } + + fn step(&mut self, action: Action) -> eyre::Result> { + match action { + Action::BogusReorg { offset } => self.bogus_reorg(offset)?, + Action::Commit { count } => { + for _ in 0..count { + let block = self.commit_block()?; + self.deliver(&block)?; + } + } + Action::Crash(rebuild) => self.crash(rebuild)?, + Action::FailNextSave => self.fail_next_save()?, + Action::Flush => self.flush()?, + Action::Reorg { depth, length } => return self.reorg(depth, length), + Action::Revert { depth } => return self.revert(depth), + Action::StaleCommit { depth, length } => return self.stale_commit(depth, length), + Action::SyncTo { count } => { + for _ in 0..count { + self.commit_block()?; + } + let head = self.reth.head().number; + // The ExEx acknowledges the tip as soon as it has queued the sync target, so + // reth's pruner can run before the engine's loop reads its first block. + self.prune_bodies_through(head)?; + self.engine().sync_to(head)?; + self.model.sync_to(head); + } + } + Ok(ControlFlow::Continue(())) + } + + /// Runs `call` against the engine on its own thread and fails the case if it does not return + /// within [`STEP_TIMEOUT`], which a stalled engine would otherwise turn into a hang. + pub(super) fn call( + &mut self, + what: &str, + call: impl FnOnce(EngineHandle) -> T + Send + 'static, + ) -> eyre::Result { + let engine = self.engine().clone(); + let (reply_tx, reply_rx) = mpsc::channel(); + thread::spawn(move || { + let _ = reply_tx.send(call(engine)); + }); + match reply_rx.recv_timeout(STEP_TIMEOUT) { + Ok(result) => Ok(result), + Err(mpsc::RecvTimeoutError::Timeout) => { + self.engine_hung = true; + eyre::bail!("the engine did not reply to {what} within {STEP_TIMEOUT:?}") + } + Err(mpsc::RecvTimeoutError::Disconnected) => { + eyre::bail!("the engine call {what} panicked") + } + } + } + + pub(super) fn window(&self) -> eyre::Result { + Ok(self.store().provider_ro()?.get_proof_window()?) + } + + const fn next_serial(&mut self) -> u64 { + self.serial += 1; + self.serial + } + + /// Commits a new block to reth and to the model. + fn commit_block(&mut self) -> eyre::Result { + let serial = self.next_serial(); + let block = self.reth.commit(&block_spec(serial))?; + self.model.push(model_block(&block, serial)); + Ok(block) + } + + /// Replaces reth's chain above `fork` with `length` new blocks, in reth and in the model. + fn fork_chain(&mut self, fork: u64, length: u64) -> eyre::Result> { + let serials: Vec = (0..length).map(|_| self.next_serial()).collect(); + let specs: Vec = serials.iter().copied().map(block_spec).collect(); + let new = self.reth.reorg_with(fork + 1, &specs)?; + self.model.truncate(fork); + for (block, serial) in new.iter().zip(serials) { + self.model.push(model_block(block, serial)); + } + Ok(new) + } + + /// Hands `block` to the engine the way the ExEx does for a committed block, along the index + /// or the execute path, and acknowledges it to reth. + fn deliver(&mut self, block: &ExecutedBlock) -> eyre::Result<()> { + let recovered = block.recovered_block(); + let (number, num_hash, parent) = + (recovered.number(), recovered.num_hash(), recovered.parent_hash()); + let executes = self.config.executes(number); + let parent_available = + !executes || self.model.is_canonical(NumHash::new(number - 1, parent)); + let result = if executes { + let recovered = Arc::clone(&block.recovered_block); + self.call("execute_block", move |engine| engine.execute_block(&recovered))? + } else { + let block = block.clone(); + self.call("index_block", move |engine| index_executed_block(&engine, &block))? + }; + let reply = match result { + Ok(()) => Reply::Taken, + Err(EngineError::ParentHashMismatch { + block_number, + expected_parent_hash, + actual_parent_hash, + }) if (block_number, actual_parent_hash) == (number, parent) => { + self.log(format!( + " the engine refused block {number}: its tip at {} is {expected_parent_hash}", + number - 1 + )); + Reply::Refused { tip_hash: expected_parent_hash } + } + Err(err) => eyre::bail!("the engine refused block {number}: {err}"), + }; + self.model.deliver_block(num_hash, parent, parent_available, reply)?; + self.prune_bodies_through(number) + } + + /// reth's bodies pruner following the ExEx's acknowledgement of a notification at `number`, + /// which is the notification's tip whether or not the store holds it. + fn prune_bodies_through(&mut self, number: u64) -> eyre::Result<()> { + if self.config.body_pruning && + let Some(pruned) = self.reth.prune_bodies_through(number)? + { + self.log(format!(" reth pruned bodies through block {pruned}")); + } + Ok(()) + } + + /// `blocks` the way the ExEx hands a reorg to the engine, indexed or executed as the + /// verification interval says. + fn reorg_blocks(&self, blocks: &[ExecutedBlock]) -> Vec> { + blocks + .iter() + .map(|block| reorg_block(block, self.config.executes(block.recovered_block().number()))) + .collect() + } + + /// The fork point of a reorg or revert `depth` blocks deep, clamped to what reth holds in + /// memory; none when nothing is. + fn fork_point(&mut self, depth: u64) -> Option { + let head = self.reth.head().number; + let in_memory = head - self.reth.persisted().number; + if in_memory == 0 { + self.log(" reth holds no block in memory, nothing to reorg".to_owned()); + return None; + } + Some(head - depth.min(in_memory)) + } + + /// Checks the engine's reply to a reorg or revert whose first removed block is `first`. A + /// refusal must name a block at or below the earliest the window held around the call; it + /// ends the case, since reth has moved on without the store. An `Ok` must not have unwound + /// at or below the window's earliest block. + fn unwind_outcome( + &mut self, + what: &str, + result: Result<(), EngineError>, + first: u64, + earliest_before: u64, + ) -> eyre::Result> { + match result { + Ok(()) => { + eyre::ensure!( + first > earliest_before, + "the engine applied a {what} from block {first}, at or below its earliest \ + block {earliest_before}" + ); + Ok(ControlFlow::Continue(())) + } + Err(EngineError::Storage(OpProofsStorageError::UnwindBeyondEarliest { + unwind_block_number, + earliest_block_number, + })) => { + let earliest_after = self.window()?.earliest.number; + eyre::ensure!( + unwind_block_number == first && + unwind_block_number <= earliest_block_number && + (earliest_before..=earliest_after).contains(&earliest_block_number), + "the engine refused a {what} from block {first} citing block \ + {unwind_block_number} and earliest block {earliest_block_number}, while the \ + window's earliest was {earliest_before} before and {earliest_after} after" + ); + self.log(format!( + " the engine refused the {what} from block {first} at or below its earliest \ + block {earliest_block_number}; reth moved on without the store" + )); + Ok(ControlFlow::Break(())) + } + Err(err) => eyre::bail!("the engine refused a valid {what} from block {first}: {err}"), + } + } + + /// Delivers a reorg to `new` and, if the engine took it, records it in the model. + fn deliver_reorg( + &mut self, + new: &[ExecutedBlock], + head_before: u64, + ) -> eyre::Result> { + let first = new[0].recovered_block().number(); + let last = new.last().expect("a reorg has a block").recovered_block().num_hash(); + let blocks = self.reorg_blocks(new); + let earliest_before = self.window()?.earliest.number; + let result = self.call("reorg", move |engine| engine.reorg(blocks))?; + self.last_fork = Some(first - 1); + let outcome = self.unwind_outcome("reorg", result, first, earliest_before)?; + if outcome.is_continue() { + self.model.reorg(first, last, head_before); + self.prune_bodies_through(self.reth.head().number)?; + } + Ok(outcome) + } + + fn reorg(&mut self, depth: u64, length: u64) -> eyre::Result> { + let head_before = self.reth.head().number; + let Some(fork) = self.fork_point(depth) else { + return Ok(ControlFlow::Continue(())); + }; + let new = self.fork_chain(fork, length)?; + self.deliver_reorg(&new, head_before) + } + + fn revert(&mut self, depth: u64) -> eyre::Result> { + let head_before = self.reth.head().number; + let Some(fork) = self.fork_point(depth) else { + return Ok(ControlFlow::Continue(())); + }; + let first = fork + 1; + let to = self.model.block_with_parent(first); + self.reth.revert_to(fork); + self.model.truncate(fork); + let earliest_before = self.window()?.earliest.number; + let result = self.call("unwind", move |engine| engine.unwind(to))?; + self.last_fork = Some(fork); + let outcome = self.unwind_outcome("revert", result, first, earliest_before)?; + if outcome.is_continue() { + self.model.revert(first, head_before); + } + Ok(outcome) + } + + /// reth commits `depth` blocks and reorgs them away before the ExEx processes their + /// notifications, so the engine takes the stale blocks first and the reorg after. + fn stale_commit(&mut self, depth: u64, length: u64) -> eyre::Result> { + // reth persists all but its newest `memory_block_buffer_target` blocks, and only + // in-memory blocks can be reorged away. + let depth = depth.min(self.reth.persistence.memory_block_buffer_target); + let fork = self.reth.head().number; + let stale: Vec = + (0..depth).map(|_| self.commit_block()).collect::>()?; + let head_before = self.reth.head().number; + let new = self.fork_chain(fork, length)?; + for block in &stale { + self.deliver(block)?; + } + self.deliver_reorg(&new, head_before) + } + + pub(super) fn flush(&mut self) -> eyre::Result<()> { + self.call("flush", |engine| engine.flush())?; + let latest = self.window()?.latest.number; + self.model.note_persisted(latest); + Ok(()) + } + + /// Arms a failure for the next write, commits enough blocks to start a save, and flushes + /// twice: once for a save already in flight and once for the one that then starts and fails. + fn fail_next_save(&mut self) -> eyre::Result<()> { + self.store().fail_next_write(); + for _ in 0..self.config.cadence.persistence { + let block = self.commit_block()?; + self.deliver(&block)?; + } + self.flush()?; + self.flush()?; + if self.store().disarm() { + self.log(" no save started, the injected failure is withdrawn".to_owned()); + } + Ok(()) + } + + /// A one-block reorg on a parent no block has, at a fork point the engine knows the hash of, + /// which it must refuse, or skip while its tip is still below the fork point. + fn bogus_reorg(&mut self, offset: u64) -> eyre::Result<()> { + let points = self.model.known_fork_points(); + if points.is_empty() { + self.log(" the engine knows no fork point for certain, nothing to refuse".to_owned()); + return Ok(()); + } + let fork = points[(offset % points.len() as u64) as usize]; + let first = fork + 1; + let bogus = reorg_block_without_data(BlockWithParent::new( + UNRELATED_PARENT, + NumHash::new(first, B256::repeat_byte(0xB1)), + )); + let stored_fork_hash = self.model.num_hash(fork).hash; + let result = self.call("reorg", move |engine| engine.reorg(vec![bogus]))?; + let refused = match result { + Ok(()) => false, + Err(EngineError::ParentHashMismatch { + block_number, + expected_parent_hash, + actual_parent_hash, + }) => { + eyre::ensure!( + (block_number, expected_parent_hash, actual_parent_hash) == + (first, stored_fork_hash, UNRELATED_PARENT), + "the engine refused the reorg at block {first} with a mismatch at block \ + {block_number} expecting {expected_parent_hash} and seeing \ + {actual_parent_hash}, not the stored block {fork} {stored_fork_hash}" + ); + true + } + Err(EngineError::Storage(OpProofsStorageError::UnwindBeyondEarliest { + unwind_block_number, + earliest_block_number, + })) => { + eyre::ensure!( + unwind_block_number == first && first <= earliest_block_number, + "the engine refused the reorg at block {first} citing block \ + {unwind_block_number} and earliest block {earliest_block_number}" + ); + true + } + Err(err) => eyre::bail!( + "unexpected reply to a reorg at block {first} not building on the stored block \ + {fork}: {err}" + ), + }; + self.model.bogus_reorg(first, refused)?; + self.log(format!( + " the engine {} the reorg at block {first} on a parent no block has", + if refused { "refused" } else { "skipped" } + )); + Ok(()) + } + + /// reth and the node stop and restart: reth loses what it held only in memory, the engine + /// its buffer (its save in flight completes, as the engine thread drains it on drop), and + /// both come back the way a restart brings them back today: a fresh engine over the reopened + /// store, a sync target at reth's head, and reth's rebuilt blocks as live notifications. + fn crash(&mut self, rebuild: Rebuild) -> eyre::Result<()> { + let persisted = self.reth.persisted().number; + let lost = self.model.serials_above(persisted); + self.log(match lost.len() { + 0 => format!(" crash: reth loses no block ({rebuild:?})"), + count => format!( + " crash: reth loses blocks {}..={} and rebuilds them ({rebuild:?})", + persisted + 1, + persisted + count as u64 + ), + }); + self.node = None; + self.reth.crash_and_restart()?; + let store = + StoreWithSaveFaults::new(Arc::new(MdbxProofsStorage::new(self.store_dir.path())?)); + let engine = Self::spawn_engine(&self.reth, &store, self.config); + self.node = Some(ProofsNode { engine, store }); + let tip = self.call("tip", |engine| engine.tip())??; + self.model.restart(tip)?; + self.log(format!( + " restart: the store serves through block {}, reth's head is {persisted}", + tip.number + )); + self.engine().sync_to(persisted)?; + self.model.sync_to(persisted); + match rebuild { + Rebuild::Same => { + for serial in lost { + let block = self.reth.commit(&block_spec(serial))?; + let number = block.recovered_block().number(); + eyre::ensure!( + block.recovered_block().hash() == self.model.block(number).hash, + "reth rebuilt block {number} differently from the chain it lost" + ); + self.deliver(&block)?; + } + } + Rebuild::Different => { + self.model.truncate(persisted); + for _ in lost { + let block = self.commit_block()?; + self.deliver(&block)?; + } + } + } + Ok(()) + } +} diff --git a/rust/op-reth/crates/trie/tests/engine_model/invariants.rs b/rust/op-reth/crates/trie/tests/engine_model/invariants.rs new file mode 100644 index 00000000000..9243915d873 --- /dev/null +++ b/rust/op-reth/crates/trie/tests/engine_model/invariants.rs @@ -0,0 +1,170 @@ +//! What the harness checks after every step and at the end of a case. + +use super::{ + harness::{Harness, STEP_TIMEOUT}, + store_view::BlockView, +}; +use alloy_eips::NumHash; +use reth_optimism_trie::{OpProofsProviderRO, OpProofsStore, ProofWindowRange}; +use std::{ + thread, + time::{Duration, Instant}, +}; + +const POLL_INTERVAL: Duration = Duration::from_millis(2); + +/// Flushes a settle may need before the store stops changing. +const MAX_SETTLE_FLUSHES: usize = 64; + +impl Harness { + pub(super) fn check_invariants(&mut self) -> eyre::Result<()> { + eyre::ensure!( + self.model.num_hash(self.model.head()) == self.reth.head(), + "the model's head {:?} is not reth's {:?}", + self.model.num_hash(self.model.head()), + self.reth.head() + ); + let tip = self.settle_engine()?; + let (window, views) = self.read_window_and_views(self.last_fork)?; + self.ensure_canonical("earliest", window.earliest)?; + self.ensure_canonical("latest", window.latest)?; + eyre::ensure!( + window.latest.number <= tip.number, + "the store's latest block {} is above the engine's tip {}", + window.latest.number, + tip.number + ); + eyre::ensure!( + window.latest.number >= self.model.persisted_at_least(), + "the store's latest block {} is below block {}, which it had persisted", + window.latest.number, + self.model.persisted_at_least() + ); + eyre::ensure!( + window.latest.number - window.earliest.number <= self.model.window(), + "the store holds blocks {}..={}, more than its window of {}", + window.earliest.number, + window.latest.number, + self.model.window() + ); + self.model.note_persisted(window.latest.number); + for (number, view) in views { + view.verify_proofs(number, self.model.block(number).state_root)?; + } + self.log(format!( + " engine tip {}, store blocks {}..={}, reth head {}", + tip.number, + window.earliest.number, + window.latest.number, + self.reth.head().number + )); + Ok(()) + } + + /// Flushes until a flush changes nothing, which leaves fewer blocks buffered than start a + /// save. + pub(super) fn settle_store(&mut self) -> eyre::Result<()> { + let mut latest = self.window()?.latest; + for _ in 0..MAX_SETTLE_FLUSHES { + self.flush()?; + let now = self.window()?.latest; + if now == latest { + return Ok(()); + } + latest = now; + } + eyre::bail!("the store kept changing over {MAX_SETTLE_FLUSHES} flushes") + } + + pub(super) fn check_settled(&mut self) -> eyre::Result<()> { + let tip = self.settle_engine()?; + let latest = self.window()?.latest.number; + let unsaved = tip.number.checked_sub(latest).ok_or_else(|| { + eyre::eyre!( + "the store's latest block {latest} is above the engine's tip {}", + tip.number + ) + })?; + eyre::ensure!( + unsaved < self.config.cadence.persistence, + "{unsaved} blocks stay buffered above the store's latest block {latest} after \ + settling, enough to start a save at the persistence threshold {}", + self.config.cadence.persistence + ); + self.log(format!("settled: engine tip {}, store latest {latest}", tip.number)); + Ok(()) + } + + /// Waits for the engine's tip to reach one the model predicts, and narrows the model to it. + /// A tip off every path the model allows fails at once. + fn settle_engine(&mut self) -> eyre::Result { + let expected = self.model.settled_tips(); + let deadline = Instant::now() + STEP_TIMEOUT; + loop { + let tip = self.call("tip", |engine| engine.tip())??; + if expected.contains(&tip) { + self.model.observe_settled(tip)?; + return Ok(tip); + } + eyre::ensure!( + self.model.on_path(tip), + "the engine's tip {tip:?} is on no path the model allows: {:?}", + self.model.candidates() + ); + if Instant::now() >= deadline { + let window = self.window()?; + eyre::bail!( + "the engine did not settle within {STEP_TIMEOUT:?}: its tip is {tip:?}, the \ + model expects one of {expected:?}, the store holds blocks {}..={}", + window.earliest.number, + window.latest.number + ); + } + thread::sleep(POLL_INTERVAL); + } + } + + /// The store's window and, read in the same transaction so that a save landing meanwhile + /// cannot prune a height from under the read, its views at the window's ends and at `fork` + /// when the window holds it. + fn read_window_and_views( + &self, + fork: Option, + ) -> eyre::Result<(ProofWindowRange, Vec<(u64, BlockView)>)> { + let provider = self.store().provider_ro()?; + let window = provider.get_proof_window()?; + let mut heights = vec![window.earliest.number, window.latest.number]; + if let Some(fork) = fork && + (window.earliest.number..=window.latest.number).contains(&fork) + { + heights.push(fork); + } + heights.sort_unstable(); + heights.dedup(); + let views = heights + .into_iter() + .map(|number| Ok((number, BlockView::read_from(&provider, number, &self.targets)?))) + .collect::>()?; + Ok((window, views)) + } + + fn describe_canonical(&self, number: u64) -> String { + if number <= self.model.head() { + format!("{}", self.model.block(number).hash) + } else { + format!("beyond the canonical head {}", self.model.head()) + } + } + + fn ensure_canonical(&self, what: &str, block: NumHash) -> eyre::Result<()> { + eyre::ensure!( + self.model.is_canonical(block), + "the store's {what} block {}:{} is not on the canonical chain, whose block {} is {}", + block.number, + block.hash, + block.number, + self.describe_canonical(block.number) + ); + Ok(()) + } +} diff --git a/rust/op-reth/crates/trie/tests/engine_model/model.rs b/rust/op-reth/crates/trie/tests/engine_model/model.rs new file mode 100644 index 00000000000..e32f68dbd6b --- /dev/null +++ b/rust/op-reth/crates/trie/tests/engine_model/model.rs @@ -0,0 +1,477 @@ +//! The reference model: the canonical chain as the consensus layer holds it, and where the engine +//! and its store must end up once they settle. It knows nothing of the engine's internals; what +//! it cannot predict exactly, because a save or the sync loop runs in the background, it bounds, +//! and the harness narrows the bound with what it observes. + +use alloy_eips::{NumHash, eip1898::BlockWithParent}; +use alloy_primitives::B256; + +/// A canonical block: what the store must agree with at its height. +#[derive(Clone, Copy, Debug)] +pub(super) struct ModelBlock { + pub(super) hash: B256, + pub(super) parent: B256, + /// The serial of the transfer the block carries; rebuilding it on the same parent gives the + /// same block. + pub(super) serial: u64, + pub(super) state_root: B256, +} + +/// Where the engine may be: a tip it has reached for certain and the target its sync loop works +/// toward, so its actual tip lies between the two. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(super) struct EngineCandidate { + pub(super) target: u64, + pub(super) tip: NumHash, +} + +/// The engine's reply to a block from a live notification. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(super) enum Reply { + /// A parent-hash mismatch naming the engine's tip at the height below the block. + Refused { tip_hash: B256 }, + /// `Ok`, whether the block was buffered, ignored or taken as a sync target. + Taken, +} + +pub(super) struct Model { + /// Every state the engine may be in; an observation keeps the ones it confirms. No + /// candidate's way to its settled tip passes through another's settled tip, so the first + /// settled tip the harness observes is the one the engine stays at. + candidates: Vec, + /// The canonical chain, genesis first. + canonical: Vec, + /// The lowest block whose hash the engine has held since it started, through its tip. + known_from: u64, + /// The store has persisted at least through this block. + persisted_at_least: u64, + /// The proof window the store keeps. + window: u64, +} + +impl Model { + pub(super) fn new(genesis: ModelBlock, window: u64) -> Self { + Self { + candidates: vec![EngineCandidate { target: 0, tip: NumHash::new(0, genesis.hash) }], + canonical: vec![genesis], + known_from: 1, + persisted_at_least: 0, + window, + } + } + + pub(super) const fn head(&self) -> u64 { + self.canonical.len() as u64 - 1 + } + + pub(super) fn block(&self, number: u64) -> &ModelBlock { + &self.canonical[number as usize] + } + + pub(super) fn num_hash(&self, number: u64) -> NumHash { + NumHash::new(number, self.block(number).hash) + } + + pub(super) fn block_with_parent(&self, number: u64) -> BlockWithParent { + BlockWithParent::new(self.block(number).parent, self.num_hash(number)) + } + + pub(super) fn is_canonical(&self, block: NumHash) -> bool { + block.number <= self.head() && self.block(block.number).hash == block.hash + } + + pub(super) fn push(&mut self, block: ModelBlock) { + self.canonical.push(block); + } + + /// Keeps the chain through `head`. + pub(super) fn truncate(&mut self, head: u64) { + self.canonical.truncate(head as usize + 1); + } + + /// The serials of the blocks above `number`, the order to rebuild them in. + pub(super) fn serials_above(&self, number: u64) -> Vec { + self.canonical[number as usize + 1..].iter().map(|block| block.serial).collect() + } + + pub(super) const fn window(&self) -> u64 { + self.window + } + + pub(super) const fn persisted_at_least(&self) -> u64 { + self.persisted_at_least + } + + /// The store was seen persisted through `latest`. + pub(super) fn note_persisted(&mut self, latest: u64) { + self.persisted_at_least = self.persisted_at_least.max(latest); + } + + pub(super) fn candidates(&self) -> &[EngineCandidate] { + &self.candidates + } + + /// Fork points whose hash the engine holds for certain: the blocks it took in since it + /// started, and genesis while the window is never pruned. + pub(super) fn known_fork_points(&self) -> Vec { + let mut points: Vec = self + .highest_reach() + .map(|reach| (self.known_from..=reach).collect()) + .unwrap_or_default(); + if self.window >= self.head() { + points.push(0); + } + points + } + + /// How far the sync loop can carry `candidate` over a chain whose head was `head`: to its + /// target while its tip is canonical, nowhere once the tip is on an abandoned chain. + fn reach(&self, candidate: &EngineCandidate, head: u64) -> u64 { + if self.is_canonical(candidate.tip) { + candidate.target.min(head).max(candidate.tip.number) + } else { + candidate.tip.number + } + } + + fn highest_reach(&self) -> Option { + self.candidates.iter().map(|candidate| self.reach(candidate, self.head())).max() + } + + /// The tip `candidate` settles on once the sync loop has done what it can. + fn settled_tip(&self, candidate: &EngineCandidate) -> NumHash { + if self.is_canonical(candidate.tip) { + self.num_hash(self.reach(candidate, self.head())) + } else { + candidate.tip + } + } + + /// The tips the engine may settle on. + pub(super) fn settled_tips(&self) -> Vec { + let mut tips = Vec::new(); + for candidate in &self.candidates { + let tip = self.settled_tip(candidate); + if !tips.contains(&tip) { + tips.push(tip); + } + } + tips + } + + /// Whether `tip` lies on some candidate's way to settling: the candidate's tip, or a + /// canonical block between it and the candidate's reach. The sync loop only moves forward + /// along the canonical chain, so a tip off every path is a wrong engine state at once. + pub(super) fn on_path(&self, tip: NumHash) -> bool { + let head = self.head(); + self.candidates.iter().any(|candidate| { + tip == candidate.tip || + (self.is_canonical(tip) && + (candidate.tip.number..=self.reach(candidate, head)) + .contains(&tip.number)) + }) + } + + /// Keeps the candidates that settle on `observed` and pins them to it. + pub(super) fn observe_settled(&mut self, observed: NumHash) -> eyre::Result<()> { + let head = self.head(); + let settled: Vec = self + .candidates + .iter() + .filter(|candidate| self.settled_tip(candidate) == observed) + .map(|candidate| EngineCandidate { + target: candidate.target.min(head).max(observed.number), + tip: observed, + }) + .collect(); + eyre::ensure!( + !settled.is_empty(), + "the engine settled at {observed:?}, which none of {:?} predicts", + self.candidates + ); + self.replace_candidates(settled); + Ok(()) + } + + /// The engine replied `reply` to `block` from a live notification. It buffers a block that + /// is the next one and builds on its tip, on the execute path only while the parent's state + /// is still in reth; it refuses the next block when the block does not build on its tip, + /// naming the tip; it raises its sync target for a block beyond the next one; it ignores a + /// block at or below its tip. + pub(super) fn deliver_block( + &mut self, + block: NumHash, + parent: B256, + parent_available: bool, + reply: Reply, + ) -> eyre::Result<()> { + let consistent: Vec = self + .candidates + .iter() + .flat_map(|candidate| self.deliveries(candidate, block, parent, parent_available)) + .filter(|(_, predicted)| *predicted == reply) + .map(|(candidate, _)| candidate) + .collect(); + eyre::ensure!( + !consistent.is_empty(), + "the engine replied {reply:?} to block {} on parent {parent}, which no state it \ + could be in allows: {:?}", + block.number, + self.candidates + ); + self.replace_candidates(consistent); + Ok(()) + } + + /// What `candidate` becomes on `block`, with the reply each outcome gives, for every tip the + /// candidate's sync loop may have reached by then. + fn deliveries( + &self, + candidate: &EngineCandidate, + block: NumHash, + parent: B256, + parent_available: bool, + ) -> Vec<(EngineCandidate, Reply)> { + let number = block.number; + let lowest = candidate.tip.number; + if number <= lowest { + return vec![(*candidate, Reply::Taken)]; + } + let reach = self.reach(candidate, self.head()); + let mut next = Vec::new(); + if lowest < number - 1 { + let target = candidate.target.max(number); + next.push((EngineCandidate { target, ..*candidate }, Reply::Taken)); + } + if number - 1 <= reach { + let tip_below = + if lowest == number - 1 { candidate.tip } else { self.num_hash(number - 1) }; + if tip_below.hash == parent { + let tip = if parent_available { block } else { tip_below }; + next.push((EngineCandidate { target: candidate.target, tip }, Reply::Taken)); + } else { + let refused = EngineCandidate { target: candidate.target, tip: tip_below }; + next.push((refused, Reply::Refused { tip_hash: tip_below.hash })); + } + } + if reach >= number { + next.push((*candidate, Reply::Taken)); + } + next + } + + /// The engine replied `Ok` to a reorg from `first` to a chain ending at `last`, delivered + /// after reth switched to it while its head stood at `head_before`. + pub(super) fn reorg(&mut self, first: u64, last: NumHash, head_before: u64) { + self.unwind_from(first, last, head_before); + } + + /// The engine replied `Ok` to a revert of the blocks from `first` up, delivered after reth + /// dropped them while its head stood at `head_before`. + pub(super) fn revert(&mut self, first: u64, head_before: u64) { + self.unwind_from(first, self.num_hash(first - 1), head_before); + } + + /// The engine skipped the unwind where `first` was above its tip; elsewhere the store and + /// the buffer unwound to the block below `first` and the tip is `tip_after`. + fn unwind_from(&mut self, first: u64, tip_after: NumHash, head_before: u64) { + let widest_head = head_before.max(self.head()); + let mut next = Vec::new(); + for candidate in &self.candidates { + if candidate.tip.number < first { + next.push(*candidate); + } + if self.reach(candidate, widest_head) >= first { + next.push(EngineCandidate { + target: candidate.target.min(first - 1), + tip: tip_after, + }); + } + } + self.replace_candidates(next); + self.known_from = self.known_from.min(first); + self.persisted_at_least = self.persisted_at_least.min(first - 1); + } + + pub(super) fn sync_to(&mut self, target: u64) { + for candidate in &mut self.candidates { + candidate.target = candidate.target.max(target); + } + } + + /// The engine's reply to a reorg at `first` whose first block builds on no block it holds: + /// refused where its tip was at or above `first`, which takes a candidate reaching that far, + /// and skipped where its tip was below. + pub(super) fn bogus_reorg(&mut self, first: u64, refused: bool) -> eyre::Result<()> { + let head = self.head(); + let consistent: Vec = self + .candidates + .iter() + .copied() + .filter(|candidate| { + if refused { + self.reach(candidate, head) >= first + } else { + candidate.tip.number < first + } + }) + .collect(); + eyre::ensure!( + !consistent.is_empty(), + "the engine {} a reorg at block {first} on a parent no block has, which no state it \ + could be in allows: {:?}", + if refused { "refused" } else { "skipped" }, + self.candidates + ); + self.replace_candidates(consistent); + Ok(()) + } + + /// A fresh engine over the store reports `tip`, which the store had persisted: nothing is + /// buffered and nothing below the tip is known. + pub(super) fn restart(&mut self, tip: NumHash) -> eyre::Result<()> { + eyre::ensure!( + tip.number >= self.persisted_at_least, + "a fresh engine reports tip {tip:?} below the persisted bound {}", + self.persisted_at_least + ); + eyre::ensure!( + self.highest_reach().is_some_and(|reach| tip.number <= reach), + "a fresh engine reports tip {tip:?} above anything the old engine could have held \ + ({:?})", + self.candidates + ); + self.candidates = vec![EngineCandidate { target: tip.number, tip }]; + self.known_from = tip.number + 1; + self.persisted_at_least = tip.number; + Ok(()) + } + + fn replace_candidates(&mut self, next: Vec) { + let mut unique: Vec = Vec::with_capacity(next.len()); + for candidate in next { + if !unique.contains(&candidate) { + unique.push(candidate); + } + } + assert!(!unique.is_empty(), "a step left the engine with no possible state"); + self.candidates = unique; + } +} + +#[cfg(test)] +mod tests { + use super::*; + use rstest::rstest; + + const WINDOW: u64 = 1000; + + /// The canonical block at `number`: hash and parent are the block numbers. + fn canonical(number: u64) -> NumHash { + NumHash::new(number, B256::with_last_byte(number as u8)) + } + + /// A block at `number` on an abandoned chain. + fn stale(number: u64) -> NumHash { + NumHash::new(number, B256::with_last_byte(0xA0 + number as u8)) + } + + fn block(number: u64) -> ModelBlock { + ModelBlock { + hash: canonical(number).hash, + parent: canonical(number.saturating_sub(1)).hash, + serial: number, + state_root: B256::ZERO, + } + } + + /// A chain through `head` with the engine in `candidates`. + fn model(head: u64, candidates: &[EngineCandidate]) -> Model { + let mut model = Model::new(block(0), WINDOW); + for number in 1..=head { + model.push(block(number)); + } + model.candidates = candidates.to_vec(); + model + } + + const fn at(tip: NumHash, target: u64) -> EngineCandidate { + EngineCandidate { target, tip } + } + + #[rstest] + #[case::at_or_below_the_tip(3, &[at(canonical(3), 3)], 2, true, &[at(canonical(3), 3)])] + #[case::next_block(4, &[at(canonical(3), 3)], 4, true, &[at(canonical(4), 3)])] + #[case::next_block_without_parent_state(4, &[at(canonical(3), 3)], 4, false, &[at(canonical(3), 3)])] + #[case::beyond_the_next_block(5, &[at(canonical(3), 3)], 5, true, &[at(canonical(3), 5)])] + #[case::while_syncing(7, &[at(canonical(3), 6)], 7, true, &[at(canonical(3), 7), at(canonical(7), 6)])] + fn deliver_block_taken( + #[case] head: u64, + #[case] before: &[EngineCandidate], + #[case] number: u64, + #[case] parent_available: bool, + #[case] after: &[EngineCandidate], + ) { + let mut model = model(head, before); + model + .deliver_block( + canonical(number), + canonical(number - 1).hash, + parent_available, + Reply::Taken, + ) + .unwrap(); + assert_eq!(model.candidates(), after); + } + + #[rstest] + #[case::refused_citing_the_tip(Reply::Refused { tip_hash: stale(3).hash }, true)] + #[case::refused_citing_another_block(Reply::Refused { tip_hash: canonical(3).hash }, false)] + #[case::taken(Reply::Taken, false)] + fn deliver_block_on_a_stale_tip(#[case] reply: Reply, #[case] consistent: bool) { + let mut model = model(4, &[at(stale(3), 3)]); + let result = model.deliver_block(canonical(4), canonical(3).hash, true, reply); + assert_eq!(result.is_ok(), consistent, "{result:?}"); + } + + #[rstest] + #[case::skipped_below_the_tip(&[at(canonical(2), 2)], &[at(canonical(2), 2)])] + #[case::applied_at_the_tip(&[at(canonical(5), 5)], &[at(canonical(5), 3)])] + #[case::either_while_syncing(&[at(canonical(2), 5)], &[at(canonical(2), 5), at(canonical(5), 3)])] + fn reorg_from_block_four( + #[case] before: &[EngineCandidate], + #[case] after: &[EngineCandidate], + ) { + let mut model = model(5, before); + model.reorg(4, canonical(5), 5); + assert_eq!(model.candidates(), after); + } + + #[test] + fn revert_from_block_four_moves_the_tip_below_it() { + let mut model = model(3, &[at(canonical(5), 5)]); + model.revert(4, 5); + assert_eq!(model.candidates(), &[at(canonical(3), 3)]); + } + + #[test] + fn settling_keeps_the_candidates_that_reach_the_observed_tip() { + let mut model = model(7, &[at(canonical(3), 7), at(canonical(7), 6)]); + assert_eq!(model.settled_tips(), vec![canonical(7)]); + assert!(model.on_path(canonical(5))); + assert!(!model.on_path(stale(5))); + model.observe_settled(canonical(7)).unwrap(); + assert_eq!(model.candidates(), &[at(canonical(7), 7)]); + assert!(model.observe_settled(canonical(6)).is_err()); + } + + #[rstest] + #[case::at_the_tip(3, true)] + #[case::above_the_reach(4, false)] + #[case::below_the_persisted_bound(1, false)] + fn restart_bounds(#[case] tip: u64, #[case] consistent: bool) { + let mut model = model(3, &[at(canonical(3), 3)]); + model.note_persisted(2); + assert_eq!(model.restart(canonical(tip)).is_ok(), consistent); + } +} diff --git a/rust/op-reth/crates/trie/tests/live.rs b/rust/op-reth/crates/trie/tests/live.rs index ef8614ca6d9..97ed8f0ed87 100644 --- a/rust/op-reth/crates/trie/tests/live.rs +++ b/rust/op-reth/crates/trie/tests/live.rs @@ -1,19 +1,12 @@ //! End-to-end test of the live trie collector. -use alloy_consensus::{ - BlockHeader, Header, TxEip2930, - constants::{EMPTY_WITHDRAWALS, ETH_TO_WEI}, -}; -use alloy_eips::{BlockNumHash, eip4895::Withdrawals}; -use alloy_genesis::{Genesis, GenesisAccount}; -use alloy_primitives::{Address, B256, Bytes, TxKind, U256, keccak256}; +use alloy_consensus::BlockHeader; +use alloy_eips::BlockNumHash; +use alloy_primitives::{Address, B256, U256}; use derive_more::Constructor; -use reth_chainspec::{ - ChainSpec, ChainSpecBuilder, EthereumHardfork, EthereumHardforks, MAINNET, MIN_TRANSACTION_GAS, -}; -use reth_db::Database; +use reth_chainspec::ChainSpec; use reth_db_common::init::init_genesis; -use reth_ethereum_primitives::{Block, BlockBody, Receipt, Transaction, TransactionSigned}; +use reth_ethereum_primitives::{Block, BlockBody, Receipt}; use reth_evm::{ConfigureEvm, execute::Executor}; use reth_evm_ethereum::EthEvmConfig; use reth_node_api::{NodePrimitives, NodeTypesWithDB}; @@ -21,12 +14,11 @@ use reth_optimism_trie::{ MdbxProofsStorage, OpProofStoragePruner, OpProofsProviderRO, OpProofsStorage, OpProofsStore, RethTrieStorageLayout, engine::{EngineError, EngineHandle}, - initialize::InitializationJob, }; use reth_primitives_traits::{Block as _, RecoveredBlock}; use reth_provider::{ BlockWriter as _, ExecutionOutcome, HashedPostStateProvider, LatestStateProviderRef, - ProviderFactory, StateRootProvider, StorageSettingsCache, + ProviderFactory, StateRootProvider, providers::{BlockchainProvider, ProviderNodeTypes}, test_utils::create_test_provider_factory_with_chain_spec, }; @@ -34,39 +26,8 @@ use reth_revm::database::StateProviderDatabase; use secp256k1::{Keypair, Secp256k1, rand::rng}; use serial_test::serial; use std::sync::Arc; -use tempfile::TempDir; use test_case::test_case; -/// Seconds between consecutive blocks. -const BLOCK_TIME: u64 = 2; - -/// A proof window no test chain fills, so that the pruner never moves the earliest block. -const WHOLE_WINDOW: u64 = 1000; - -/// Timestamp of the block at `number`. -const fn block_timestamp(number: u64) -> u64 { - number * BLOCK_TIME -} - -/// The engine's persistence and backpressure thresholds. -#[derive(Clone, Copy, Debug)] -struct FlushCadence { - backpressure: u64, - persistence: u64, -} - -impl FlushCadence { - const EVERY_BLOCK: Self = Self { backpressure: 2, persistence: 1 }; - const EVERY_FIFTH_BLOCK: Self = Self { backpressure: 10, persistence: 5 }; -} - -/// Which store implementation a case runs over. -#[derive(Clone, Copy, Debug)] -enum Backend { - InMemory, - Mdbx, -} - /// Runs `$run` over a fresh store of `$backend`. macro_rules! over_backend { ($backend:expr, $run:ident($($arg:expr),*)) => { @@ -79,6 +40,8 @@ macro_rules! over_backend { #[path = "live/catch_up_under_pruning.rs"] mod catch_up_under_pruning; +#[path = "live/chain.rs"] +mod chain; #[path = "live/contracts.rs"] mod contracts; #[path = "live/index_execute_parity.rs"] @@ -88,86 +51,12 @@ mod parent_states; #[path = "live/reorg_revert_edges.rs"] mod reorg_revert_edges; #[path = "live/simulated_reth.rs"] +#[allow(dead_code, reason = "each test target uses part of this module")] mod simulated_reth; #[path = "live/store_view.rs"] mod store_view; -fn create_mdbx_proofs_storage() -> Arc { - let path = TempDir::new().unwrap().keep(); - Arc::new(MdbxProofsStorage::new(&path).unwrap()) -} - -/// Converts a secp256k1 public key to an Ethereum address. -fn public_key_to_address(pubkey: secp256k1::PublicKey) -> Address { - let hash = keccak256(&pubkey.serialize_uncompressed()[1..]); - Address::from_slice(&hash[12..]) -} - -/// Signs a transaction with the given keypair. -fn sign_tx_with_key_pair(key_pair: Keypair, tx: Transaction) -> TransactionSigned { - use alloy_consensus::SignableTransaction; - use reth_primitives_traits::crypto::secp256k1::sign_message; - let secret = B256::from_slice(&key_pair.secret_bytes()); - let sig = sign_message(secret, tx.signature_hash()).unwrap(); - tx.into_signed(sig).into() -} - -/// Specification for a transaction within a block -#[derive(Debug, Clone)] -struct TxSpec { - /// Call recipient or contract creation. - to: TxKind, - /// Transaction input, including initcode for deployments. - data: Bytes, - /// Maximum execution gas. - gas_limit: u64, - /// Value to transfer (in wei) - value: U256, - /// Nonce for the transaction (will be automatically assigned if None) - nonce: Option, -} - -impl TxSpec { - /// Create a simple transfer transaction - const fn transfer(to: Address, value: U256) -> Self { - Self { - to: TxKind::Call(to), - data: Bytes::new(), - gas_limit: MIN_TRANSACTION_GAS, - value, - nonce: None, - } - } - - /// Create a contract deployment running `initcode`. - const fn create(initcode: Bytes, gas_limit: u64) -> Self { - Self { to: TxKind::Create, data: initcode, gas_limit, value: U256::ZERO, nonce: None } - } - - /// Create a call to `contract` with `data` as input. - const fn call(contract: Address, data: Bytes, gas_limit: u64) -> Self { - Self { to: TxKind::Call(contract), data, gas_limit, value: U256::ZERO, nonce: None } - } - - /// Send `value` along with the transaction. - fn with_value(self, value: U256) -> Self { - Self { value, ..self } - } -} - -/// Specification for a block in the test chain -#[derive(Debug, Clone, Constructor)] -struct BlockSpec { - /// Transactions to include in this block - txs: Vec, -} - -impl BlockSpec { - /// A block without transactions. - const fn empty() -> Self { - Self { txs: Vec::new() } - } -} +use chain::*; /// Configuration for a test scenario #[derive(Debug, Constructor)] @@ -178,95 +67,6 @@ struct TestScenario { blocks_after_initialization: Vec, } -/// A mainnet-shaped chain spec whose genesis allocation is `alloc`, with the hardforks still to -/// be chosen. -fn chain_spec_builder( - alloc: impl IntoIterator, -) -> ChainSpecBuilder { - ChainSpecBuilder::default() - .chain(MAINNET.chain) - .genesis(Genesis { alloc: alloc.into_iter().collect(), ..MAINNET.genesis.clone() }) -} - -/// A genesis allocation funding `address`. -fn funded(address: Address) -> (Address, GenesisAccount) { - (address, GenesisAccount { balance: U256::from(10 * ETH_TO_WEI), ..Default::default() }) -} - -/// Helper to create a chain spec with a genesis account funded -fn chain_spec_with_address(address: Address) -> Arc { - Arc::new(chain_spec_builder([funded(address)]).paris_activated().build()) -} - -/// Creates a block from a spec, executing transactions with the given keypair -fn create_block_from_spec( - spec: &BlockSpec, - block_number: u64, - parent_hash: B256, - chain_spec: &Arc, - key_pair: Keypair, - nonce_counter: &mut u64, -) -> RecoveredBlock { - let transactions: Vec = spec - .txs - .iter() - .map(|tx_spec| { - let nonce = tx_spec.nonce.unwrap_or_else(|| { - let current = *nonce_counter; - *nonce_counter += 1; - current - }); - - sign_tx_with_key_pair( - key_pair, - TxEip2930 { - chain_id: chain_spec.chain.id(), - nonce, - gas_limit: tx_spec.gas_limit, - input: tx_spec.data.clone(), - gas_price: 1_500_000_000, - to: tx_spec.to, - value: tx_spec.value, - ..Default::default() - } - .into(), - ) - }) - .collect(); - - let gas_total = spec.txs.iter().map(|tx| tx.gas_limit).sum::(); - let timestamp = block_timestamp(block_number); - let shanghai = chain_spec.is_shanghai_active_at_timestamp(timestamp); - let cancun = chain_spec.is_cancun_active_at_timestamp(timestamp); - - Block { - header: Header { - parent_hash, - receipts_root: alloy_primitives::b256!( - "0xd3a6acf9a244d78b33831df95d472c4128ea85bf079a1d41e32ed0b7d2244c9e" - ), - difficulty: chain_spec.fork(EthereumHardfork::Paris).ttd().expect("Paris TTD"), - number: block_number, - gas_limit: gas_total.max(MIN_TRANSACTION_GAS), - gas_used: gas_total, - timestamp, - state_root: B256::ZERO, // Will be calculated by executor - withdrawals_root: shanghai.then_some(EMPTY_WITHDRAWALS), - blob_gas_used: cancun.then_some(0), - excess_blob_gas: cancun.then_some(0), - parent_beacon_block_root: cancun.then(|| keccak256(block_number.to_be_bytes())), - ..Default::default() - }, - body: BlockBody { - transactions, - withdrawals: shanghai.then(Withdrawals::default), - ..Default::default() - }, - } - .try_into_recovered() - .unwrap() -} - /// Executes a block and returns the updated block with correct state root fn execute_block( block: &mut RecoveredBlock, @@ -330,26 +130,6 @@ where Ok(()) } -/// Seeds `storage` with reth's state and trie at `block`, the proofs store's starting point. -fn initialize_storage( - storage: S, - provider_factory: &ProviderFactory, - block: BlockNumHash, -) -> eyre::Result<()> -where - N: NodeTypesWithDB, - S: OpProofsStore + Send + Sync + Clone + std::fmt::Debug + 'static, -{ - let trie_layout = if provider_factory.cached_storage_settings().is_v2() { - RethTrieStorageLayout::Packed - } else { - RethTrieStorageLayout::Legacy - }; - let tx = provider_factory.db_ref().tx()?; - InitializationJob::new(storage, tx, trie_layout).run(block.number, block.hash)?; - Ok(()) -} - /// Runs a test scenario with the given configuration fn run_test_scenario( scenario: TestScenario, diff --git a/rust/op-reth/crates/trie/tests/live/catch_up_under_pruning.rs b/rust/op-reth/crates/trie/tests/live/catch_up_under_pruning.rs index 54564ba30e3..06aa14ff619 100644 --- a/rust/op-reth/crates/trie/tests/live/catch_up_under_pruning.rs +++ b/rust/op-reth/crates/trie/tests/live/catch_up_under_pruning.rs @@ -129,11 +129,11 @@ fn transfer_block(number: u64) -> BlockSpec { /// Every address the chain touches, none of which has storage. fn proof_targets(sender: Address) -> Vec { - [sender, Address::ZERO] - .into_iter() - .chain((1..=CATCH_UP_LENGTH + REAL_TIME_LENGTH).map(recipient)) - .map(|address| ProofTarget { address, slots: Vec::new() }) - .collect() + ProofTarget::accounts( + [sender, Address::ZERO] + .into_iter() + .chain((1..=CATCH_UP_LENGTH + REAL_TIME_LENGTH).map(recipient)), + ) } fn persisted_tip(store: &S) -> eyre::Result { diff --git a/rust/op-reth/crates/trie/tests/live/chain.rs b/rust/op-reth/crates/trie/tests/live/chain.rs new file mode 100644 index 00000000000..22f2657882f --- /dev/null +++ b/rust/op-reth/crates/trie/tests/live/chain.rs @@ -0,0 +1,241 @@ +//! The test chain the live targets share: a funded signer, block and transaction specs, the +//! block builder, the store constructors and the store's initialization at a block. + +use alloy_consensus::{ + Header, TxEip2930, + constants::{EMPTY_WITHDRAWALS, ETH_TO_WEI}, +}; +use alloy_eips::{BlockNumHash, eip4895::Withdrawals}; +use alloy_genesis::{Genesis, GenesisAccount}; +use alloy_primitives::{Address, B256, Bytes, TxKind, U256, keccak256}; +use derive_more::Constructor; +use reth_chainspec::{ChainSpec, ChainSpecBuilder, EthereumHardfork, EthereumHardforks, MAINNET}; +use reth_db::Database; +use reth_ethereum_primitives::{Block, BlockBody, Transaction, TransactionSigned}; +use reth_node_api::NodeTypesWithDB; +use reth_optimism_trie::{ + MdbxProofsStorage, OpProofsStore, RethTrieStorageLayout, initialize::InitializationJob, +}; +use reth_primitives_traits::{Block as _, RecoveredBlock}; +use reth_provider::{ProviderFactory, StorageSettingsCache}; +use secp256k1::Keypair; +use std::sync::Arc; +use tempfile::TempDir; + +pub(super) use reth_chainspec::MIN_TRANSACTION_GAS; + +/// Seconds between consecutive blocks. +const BLOCK_TIME: u64 = 2; + +/// A proof window no test chain fills, so that the pruner never moves the earliest block. +pub(super) const WHOLE_WINDOW: u64 = 1000; + +/// Timestamp of the block at `number`. +pub(super) const fn block_timestamp(number: u64) -> u64 { + number * BLOCK_TIME +} + +/// The engine's persistence and backpressure thresholds. +#[derive(Clone, Copy, Debug)] +pub(super) struct FlushCadence { + pub(super) backpressure: u64, + pub(super) persistence: u64, +} + +impl FlushCadence { + pub(super) const EVERY_BLOCK: Self = Self { backpressure: 2, persistence: 1 }; + pub(super) const EVERY_FIFTH_BLOCK: Self = Self { backpressure: 10, persistence: 5 }; +} + +/// Which store implementation a case runs over. +#[derive(Clone, Copy, Debug)] +pub(super) enum Backend { + InMemory, + Mdbx, +} + +pub(super) fn create_mdbx_proofs_storage() -> Arc { + let path = TempDir::new().unwrap().keep(); + Arc::new(MdbxProofsStorage::new(&path).unwrap()) +} + +/// Converts a secp256k1 public key to an Ethereum address. +pub(super) fn public_key_to_address(pubkey: secp256k1::PublicKey) -> Address { + let hash = keccak256(&pubkey.serialize_uncompressed()[1..]); + Address::from_slice(&hash[12..]) +} + +/// Signs a transaction with the given keypair. +fn sign_tx_with_key_pair(key_pair: Keypair, tx: Transaction) -> TransactionSigned { + use alloy_consensus::SignableTransaction; + use reth_primitives_traits::crypto::secp256k1::sign_message; + let secret = B256::from_slice(&key_pair.secret_bytes()); + let sig = sign_message(secret, tx.signature_hash()).unwrap(); + tx.into_signed(sig).into() +} + +/// Specification for a transaction within a block +#[derive(Debug, Clone)] +pub(super) struct TxSpec { + /// Call recipient or contract creation. + pub(super) to: TxKind, + /// Transaction input, including initcode for deployments. + pub(super) data: Bytes, + /// Maximum execution gas. + pub(super) gas_limit: u64, + /// Value to transfer (in wei) + pub(super) value: U256, + /// Nonce for the transaction (will be automatically assigned if None) + pub(super) nonce: Option, +} + +impl TxSpec { + /// Create a simple transfer transaction + pub(super) const fn transfer(to: Address, value: U256) -> Self { + Self { + to: TxKind::Call(to), + data: Bytes::new(), + gas_limit: MIN_TRANSACTION_GAS, + value, + nonce: None, + } + } + + /// Create a contract deployment running `initcode`. + pub(super) const fn create(initcode: Bytes, gas_limit: u64) -> Self { + Self { to: TxKind::Create, data: initcode, gas_limit, value: U256::ZERO, nonce: None } + } + + /// Create a call to `contract` with `data` as input. + pub(super) const fn call(contract: Address, data: Bytes, gas_limit: u64) -> Self { + Self { to: TxKind::Call(contract), data, gas_limit, value: U256::ZERO, nonce: None } + } + + /// Send `value` along with the transaction. + pub(super) fn with_value(self, value: U256) -> Self { + Self { value, ..self } + } +} + +/// Specification for a block in the test chain +#[derive(Debug, Clone, Constructor)] +pub(super) struct BlockSpec { + /// Transactions to include in this block + pub(super) txs: Vec, +} + +impl BlockSpec { + /// A block without transactions. + pub(super) const fn empty() -> Self { + Self { txs: Vec::new() } + } +} + +/// A mainnet-shaped chain spec whose genesis allocation is `alloc`, with the hardforks still to +/// be chosen. +pub(super) fn chain_spec_builder( + alloc: impl IntoIterator, +) -> ChainSpecBuilder { + ChainSpecBuilder::default() + .chain(MAINNET.chain) + .genesis(Genesis { alloc: alloc.into_iter().collect(), ..MAINNET.genesis.clone() }) +} + +/// A genesis allocation funding `address`. +pub(super) fn funded(address: Address) -> (Address, GenesisAccount) { + (address, GenesisAccount { balance: U256::from(10 * ETH_TO_WEI), ..Default::default() }) +} + +/// Helper to create a chain spec with a genesis account funded +pub(super) fn chain_spec_with_address(address: Address) -> Arc { + Arc::new(chain_spec_builder([funded(address)]).paris_activated().build()) +} + +/// Creates a block from a spec, executing transactions with the given keypair +pub(super) fn create_block_from_spec( + spec: &BlockSpec, + block_number: u64, + parent_hash: B256, + chain_spec: &Arc, + key_pair: Keypair, + nonce_counter: &mut u64, +) -> RecoveredBlock { + let transactions: Vec = spec + .txs + .iter() + .map(|tx_spec| { + let nonce = tx_spec.nonce.unwrap_or_else(|| { + let current = *nonce_counter; + *nonce_counter += 1; + current + }); + + sign_tx_with_key_pair( + key_pair, + TxEip2930 { + chain_id: chain_spec.chain.id(), + nonce, + gas_limit: tx_spec.gas_limit, + input: tx_spec.data.clone(), + gas_price: 1_500_000_000, + to: tx_spec.to, + value: tx_spec.value, + ..Default::default() + } + .into(), + ) + }) + .collect(); + + let gas_total = spec.txs.iter().map(|tx| tx.gas_limit).sum::(); + let timestamp = block_timestamp(block_number); + let shanghai = chain_spec.is_shanghai_active_at_timestamp(timestamp); + let cancun = chain_spec.is_cancun_active_at_timestamp(timestamp); + + Block { + header: Header { + parent_hash, + receipts_root: alloy_primitives::b256!( + "0xd3a6acf9a244d78b33831df95d472c4128ea85bf079a1d41e32ed0b7d2244c9e" + ), + difficulty: chain_spec.fork(EthereumHardfork::Paris).ttd().expect("Paris TTD"), + number: block_number, + gas_limit: gas_total.max(MIN_TRANSACTION_GAS), + gas_used: gas_total, + timestamp, + state_root: B256::ZERO, // Will be calculated by executor + withdrawals_root: shanghai.then_some(EMPTY_WITHDRAWALS), + blob_gas_used: cancun.then_some(0), + excess_blob_gas: cancun.then_some(0), + parent_beacon_block_root: cancun.then(|| keccak256(block_number.to_be_bytes())), + ..Default::default() + }, + body: BlockBody { + transactions, + withdrawals: shanghai.then(Withdrawals::default), + ..Default::default() + }, + } + .try_into_recovered() + .unwrap() +} + +/// Seeds `storage` with reth's state and trie at `block`, the proofs store's starting point. +pub(super) fn initialize_storage( + storage: S, + provider_factory: &ProviderFactory, + block: BlockNumHash, +) -> eyre::Result<()> +where + N: NodeTypesWithDB, + S: OpProofsStore + Send + Sync + Clone + std::fmt::Debug + 'static, +{ + let trie_layout = if provider_factory.cached_storage_settings().is_v2() { + RethTrieStorageLayout::Packed + } else { + RethTrieStorageLayout::Legacy + }; + let tx = provider_factory.db_ref().tx()?; + InitializationJob::new(storage, tx, trie_layout).run(block.number, block.hash)?; + Ok(()) +} diff --git a/rust/op-reth/crates/trie/tests/live/reorg_revert_edges.rs b/rust/op-reth/crates/trie/tests/live/reorg_revert_edges.rs index 88d3155eefa..869b4ebedfe 100644 --- a/rust/op-reth/crates/trie/tests/live/reorg_revert_edges.rs +++ b/rust/op-reth/crates/trie/tests/live/reorg_revert_edges.rs @@ -11,7 +11,10 @@ use super::{ Backend, BlockSpec, FlushCadence, TxSpec, WHOLE_WINDOW, chain_spec_with_address, create_block_from_spec, create_mdbx_proofs_storage, initialize_storage, public_key_to_address, - simulated_reth::{RethPersistence, SimulatedReth, fixed_signer, index_executed_block}, + simulated_reth::{ + RethPersistence, SimulatedReth, UNRELATED_PARENT, fixed_signer, index_executed_block, + reorg_block, reorg_block_without_data, + }, store_view::{BlockView, ProofTarget, assert_same_view}, }; use alloy_eips::{NumHash, eip1898::BlockWithParent}; @@ -56,9 +59,6 @@ const RETH_PERSISTENCE_THRESHOLD: u64 = 2 * MAX_BLOCKS_PER_CASE; /// under either cadence. const PRUNED_WINDOW: u64 = 3; -/// A parent hash no block here has. -const UNRELATED_PARENT: B256 = B256::repeat_byte(0xBA); - /// How long a case waits for the engine's persistence thread to reach a held write. const HELD_WRITE_TIMEOUT: Duration = Duration::from_secs(30); @@ -86,14 +86,9 @@ fn fork_blocks(numbers: RangeInclusive) -> Vec { /// Every address the chain or a fork can touch, none of which has storage. fn proof_targets(sender: Address) -> Vec { - [sender, Address::ZERO] - .into_iter() - .chain( - (1..=MAX_BLOCKS_PER_CASE) - .flat_map(|number| [recipient(number), fork_recipient(number)]), - ) - .map(|address| ProofTarget { address, slots: Vec::new() }) - .collect() + ProofTarget::accounts([sender, Address::ZERO].into_iter().chain( + (1..=MAX_BLOCKS_PER_CASE).flat_map(|number| [recipient(number), fork_recipient(number)]), + )) } /// The disk tip once the engine has saved what its cadence lets it save: `buffered` blocks went @@ -126,15 +121,7 @@ impl ReorgPath { Self::Index => false, Self::Mixed => position.is_multiple_of(2), }; - if execute { - ReorgBlock::Execute(Arc::clone(&block.recovered_block)) - } else { - ReorgBlock::Index { - block: block.recovered_block().block_with_parent(), - hashed_state: block.hashed_state(), - trie_updates: block.trie_updates(), - } - } + reorg_block(block, execute) } } @@ -143,15 +130,6 @@ fn reorg_blocks(blocks: &[ExecutedBlock], path: ReorgPath) -> Vec ReorgBlock { - ReorgBlock::Index { - block, - hashed_state: Arc::new(HashedPostStateSorted::default()), - trie_updates: Arc::new(TrieUpdatesSorted::default()), - } -} - /// Which engine task unwinds. #[derive(Clone, Copy, Debug)] enum UnwindingTask { diff --git a/rust/op-reth/crates/trie/tests/live/simulated_reth.rs b/rust/op-reth/crates/trie/tests/live/simulated_reth.rs index 2c59b83b682..27d432aa483 100644 --- a/rust/op-reth/crates/trie/tests/live/simulated_reth.rs +++ b/rust/op-reth/crates/trie/tests/live/simulated_reth.rs @@ -3,7 +3,7 @@ use super::{BlockSpec, WHOLE_WINDOW, create_block_from_spec}; use alloy_consensus::BlockHeader; -use alloy_eips::BlockNumHash; +use alloy_eips::{BlockNumHash, eip1898::BlockWithParent}; use alloy_primitives::B256; use reth_chain_state::{CanonicalInMemoryState, ExecutedBlock, NewCanonicalChain}; use reth_chainspec::ChainSpec; @@ -14,9 +14,9 @@ use reth_evm::{ConfigureEvm, execute::Executor}; use reth_evm_ethereum::EthEvmConfig; use reth_optimism_trie::{ OpProofStoragePruner, OpProofsStore, - engine::{EngineError, EngineHandle}, + engine::{EngineError, EngineHandle, ReorgBlock}, }; -use reth_primitives_traits::RecoveredBlock; +use reth_primitives_traits::{RecoveredBlock, SealedHeader}; use reth_provider::{ HashedPostStateProvider, ProviderFactory, SaveBlocksInput, StateProviderFactory, StateRootProvider, StaticFileProviderFactory, StaticFileSegment, @@ -25,13 +25,39 @@ use reth_provider::{ }; use reth_revm::database::StateProviderDatabase; use reth_storage_overlay::OverlayManager; -use reth_trie_common::ComputedTrieData; +use reth_trie_common::{ComputedTrieData, HashedPostStateSorted, updates::TrieUpdatesSorted}; use secp256k1::{Keypair, Secp256k1}; use std::sync::Arc; /// reth's default `--engine.memory-block-buffer-target`. const MEMORY_BLOCK_BUFFER_TARGET: u64 = 5; +/// A parent hash no block here has. +pub(super) const UNRELATED_PARENT: B256 = B256::repeat_byte(0xBA); + +/// `block` the way the ExEx hands a block of a reorg to the engine: executed, or indexed with +/// the trie data reth computed for it. +pub(super) fn reorg_block(block: &ExecutedBlock, execute: bool) -> ReorgBlock { + if execute { + ReorgBlock::Execute(Arc::clone(&block.recovered_block)) + } else { + ReorgBlock::Index { + block: block.recovered_block().block_with_parent(), + hashed_state: block.hashed_state(), + trie_updates: block.trie_updates(), + } + } +} + +/// The reorg task's input for `block` without trie data, enough to reach the unwind checks. +pub(super) fn reorg_block_without_data(block: BlockWithParent) -> ReorgBlock { + ReorgBlock::Index { + block, + hashed_state: Arc::new(HashedPostStateSorted::default()), + trie_updates: Arc::new(TrieUpdatesSorted::default()), + } +} + /// Hands `block` to `engine` the way the ExEx indexes a block reth validated. pub(super) fn index_executed_block( engine: &EngineHandle, @@ -321,12 +347,7 @@ impl SimulatedReth { assert!(number > self.last_persisted.number, "only in-memory blocks can be reorged"); let (first_spec, rest) = specs.split_first().expect("a reorg switches to a new block"); let old = self.canonical.split_off(number as usize - 1); - // The signer's nonce is the number of transactions the remaining chain holds. - self.nonce = self - .canonical - .iter() - .map(|block| block.recovered_block().body().transactions.len() as u64) - .sum(); + self.nonce = self.transactions_in_chain(); let fork = self.build_block(first_spec)?; assert_ne!( fork.recovered_block().hash(), @@ -357,6 +378,27 @@ impl SimulatedReth { Ok(deleted.iter().filter_map(|header| header.block_range()).map(|range| range.end()).max()) } + /// Drops the canonical blocks above `number` without replacing them, the way a forkchoice + /// update back to an ancestor does. Only in-memory blocks can go. + pub(super) fn revert_to(&mut self, number: u64) { + assert!(number >= self.last_persisted.number, "only in-memory blocks can be reverted"); + let old = self.canonical.split_off(number as usize); + self.nonce = self.transactions_in_chain(); + self.in_memory.update_chain(NewCanonicalChain::Reorg { new: Vec::new(), old }); + self.in_memory.set_canonical_head(self.sealed_header(number)); + } + + /// Loses every block held only in memory and starts over from the database, the way reth + /// does after an unclean stop: the head is the persisted tip and nothing is in memory. + pub(super) fn crash_and_restart(&mut self) -> eyre::Result<()> { + self.provider = BlockchainProvider::new(self.factory.clone())?; + self.in_memory = self.provider.canonical_in_memory_state(); + self.canonical.truncate(self.last_persisted.number as usize); + self.last_evicted = self.last_persisted; + self.nonce = self.transactions_in_chain(); + Ok(()) + } + /// Writes every canonical block to the database without evicting any from memory, the state /// a `proofs init` run finds on a node that has caught up. pub(super) fn persist_all(&mut self) -> eyre::Result<()> { @@ -373,6 +415,21 @@ impl SimulatedReth { Ok(()) } + fn sealed_header(&self, number: u64) -> SealedHeader { + match number { + 0 => self.chain_spec.sealed_genesis_header(), + _ => self.block(number).clone_sealed_header(), + } + } + + /// The signer's nonce: the number of transactions the canonical chain holds. + fn transactions_in_chain(&self) -> u64 { + self.canonical + .iter() + .map(|block| block.recovered_block().body().transactions.len() as u64) + .sum() + } + fn make_canonical(&mut self, update: NewCanonicalChain) { let tip = update.tip().clone_sealed_header(); if let NewCanonicalChain::Reorg { old, .. } = &update { diff --git a/rust/op-reth/crates/trie/tests/live/store_view.rs b/rust/op-reth/crates/trie/tests/live/store_view.rs index 795ac337f00..3124e498d3b 100644 --- a/rust/op-reth/crates/trie/tests/live/store_view.rs +++ b/rust/op-reth/crates/trie/tests/live/store_view.rs @@ -21,6 +21,13 @@ pub(super) struct ProofTarget { pub(super) slots: Vec, } +impl ProofTarget { + /// Targets for `addresses`, none of which has storage. + pub(super) fn accounts(addresses: impl IntoIterator) -> Vec { + addresses.into_iter().map(|address| Self { address, slots: Vec::new() }).collect() + } +} + /// One account's storage: its slots and its storage trie nodes. pub(super) struct AccountStorage { pub(super) slots: BTreeMap, @@ -42,7 +49,16 @@ impl BlockView { number: u64, targets: &[ProofTarget], ) -> eyre::Result { - let provider = store.provider_ro()?; + Self::read_from(&store.provider_ro()?, number, targets) + } + + /// Reads the view through `provider`, so that several views and the window come from one + /// read transaction while saves go on. + pub(super) fn read_from( + provider: &P, + number: u64, + targets: &[ProofTarget], + ) -> eyre::Result { let accounts = collect_hashed(provider.account_hashed_cursor(number)?)?; let hashed_addresses: BTreeSet = accounts .keys() From 1669599148c6f7d6412c2408317f977f7e21fe74 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Thu, 15 Oct 2026 21:10:35 -0700 Subject: [PATCH 2/4] op-reth: bound the proofs engine model's earliest block by reth's persisted head The model held the store to `latest - earliest <= window`, which only a pruner that always prunes to `latest - window` satisfies. The reth 2.6 pin's pruner (#23358) prunes no further than reth's persisted head, so the store legitimately keeps more under a short window while reth's persistence lags. Each model block now records reth's persisted head when reth committed it. A save that writes block `latest` ran after that commit and reth's persisted head never moves down, so the save pruned at least to `min(latest - window, persisted_at_commit(latest))`; and no save's tip exceeds the highest head the chain has had, so no save pruned past `highest_head - window`. The earliest block must lie between the two, which holds under either pruner; on this pin the lower end is weaker than the old `latest - window`, the price of being pin-agnostic. Keeping reth persisted through `latest - window` at every save would have let the old invariant stand, but it is not equivalent: under a window of 4 it caps reth's in-memory chain at 4 blocks, which shrinks the reorgs, reverts and crash losses the pruned-window cells exercise, and it keeps the cap from ever binding, so the run would never see the pruner hold the window back. Co-Authored-By: Claude Fable 5.1 (cherry picked from commit 3f76589e74127d568a3d44aaca0a1e2986e206dd) (cherry picked from commit e76692968a0bc7e2e2ac5612b103d4cc27120a66) --- .../crates/trie/tests/engine_model/harness.rs | 11 +++- .../trie/tests/engine_model/invariants.rs | 11 +++- .../crates/trie/tests/engine_model/model.rs | 58 +++++++++++++++++++ 3 files changed, 74 insertions(+), 6 deletions(-) diff --git a/rust/op-reth/crates/trie/tests/engine_model/harness.rs b/rust/op-reth/crates/trie/tests/engine_model/harness.rs index 1491ee74352..b01371df361 100644 --- a/rust/op-reth/crates/trie/tests/engine_model/harness.rs +++ b/rust/op-reth/crates/trie/tests/engine_model/harness.rs @@ -49,11 +49,12 @@ fn block_spec(serial: u64) -> BlockSpec { BlockSpec::new(vec![TxSpec::transfer(recipient(serial), U256::from(serial))]) } -fn model_block(block: &ExecutedBlock, serial: u64) -> ModelBlock { +fn model_block(block: &ExecutedBlock, serial: u64, persisted_before_commit: u64) -> ModelBlock { let recovered = block.recovered_block(); ModelBlock { hash: recovered.hash(), parent: recovered.parent_hash(), + persisted_before_commit, serial, state_root: recovered.state_root(), } @@ -97,6 +98,7 @@ impl Harness { let genesis = ModelBlock { hash: chain_spec.genesis_hash(), parent: B256::ZERO, + persisted_before_commit: 0, serial: 0, state_root: chain_spec.genesis_header().state_root, }; @@ -244,8 +246,9 @@ impl Harness { /// Commits a new block to reth and to the model. fn commit_block(&mut self) -> eyre::Result { let serial = self.next_serial(); + let persisted_before_commit = self.reth.persisted().number; let block = self.reth.commit(&block_spec(serial))?; - self.model.push(model_block(&block, serial)); + self.model.push(model_block(&block, serial, persisted_before_commit)); Ok(block) } @@ -253,10 +256,12 @@ impl Harness { fn fork_chain(&mut self, fork: u64, length: u64) -> eyre::Result> { let serials: Vec = (0..length).map(|_| self.next_serial()).collect(); let specs: Vec = serials.iter().copied().map(block_spec).collect(); + // Every block of the fork becomes canonical with reth persisted at least this far. + let persisted_before_the_fork = self.reth.persisted().number; let new = self.reth.reorg_with(fork + 1, &specs)?; self.model.truncate(fork); for (block, serial) in new.iter().zip(serials) { - self.model.push(model_block(block, serial)); + self.model.push(model_block(block, serial, persisted_before_the_fork)); } Ok(new) } diff --git a/rust/op-reth/crates/trie/tests/engine_model/invariants.rs b/rust/op-reth/crates/trie/tests/engine_model/invariants.rs index 9243915d873..8338d899a6d 100644 --- a/rust/op-reth/crates/trie/tests/engine_model/invariants.rs +++ b/rust/op-reth/crates/trie/tests/engine_model/invariants.rs @@ -40,12 +40,17 @@ impl Harness { window.latest.number, self.model.persisted_at_least() ); + let earliest_bounds = self.model.earliest_bounds(window.latest.number); eyre::ensure!( - window.latest.number - window.earliest.number <= self.model.window(), - "the store holds blocks {}..={}, more than its window of {}", + earliest_bounds.contains(&window.earliest.number), + "the store holds blocks {}..={}, whose earliest block lies outside {:?}, what its \ + window of {} allows with reth persisted through {} before block {} was committed", window.earliest.number, window.latest.number, - self.model.window() + earliest_bounds, + self.model.window(), + self.model.block(window.latest.number).persisted_before_commit, + window.latest.number ); self.model.note_persisted(window.latest.number); for (number, view) in views { diff --git a/rust/op-reth/crates/trie/tests/engine_model/model.rs b/rust/op-reth/crates/trie/tests/engine_model/model.rs index e32f68dbd6b..111a65f9c92 100644 --- a/rust/op-reth/crates/trie/tests/engine_model/model.rs +++ b/rust/op-reth/crates/trie/tests/engine_model/model.rs @@ -5,12 +5,16 @@ use alloy_eips::{NumHash, eip1898::BlockWithParent}; use alloy_primitives::B256; +use std::ops::RangeInclusive; /// A canonical block: what the store must agree with at its height. #[derive(Clone, Copy, Debug)] pub(super) struct ModelBlock { pub(super) hash: B256, pub(super) parent: B256, + /// reth's persisted head just before reth committed the block; reth's persisted head never + /// falls, so every save of the block sees at least this. + pub(super) persisted_before_commit: u64, /// The serial of the transfer the block carries; rebuilding it on the same parent gives the /// same block. pub(super) serial: u64, @@ -41,6 +45,8 @@ pub(super) struct Model { candidates: Vec, /// The canonical chain, genesis first. canonical: Vec, + /// The highest head the chain has had, which no save's tip exceeds. + highest_head: u64, /// The lowest block whose hash the engine has held since it started, through its tip. known_from: u64, /// The store has persisted at least through this block. @@ -54,6 +60,7 @@ impl Model { Self { candidates: vec![EngineCandidate { target: 0, tip: NumHash::new(0, genesis.hash) }], canonical: vec![genesis], + highest_head: 0, known_from: 1, persisted_at_least: 0, window, @@ -82,6 +89,7 @@ impl Model { pub(super) fn push(&mut self, block: ModelBlock) { self.canonical.push(block); + self.highest_head = self.highest_head.max(self.head()); } /// Keeps the chain through `head`. @@ -98,6 +106,19 @@ impl Model { self.window } + /// Where the store's earliest block may lie while its latest block is `latest`. Every save + /// with tip `t` prunes to `t - window`, or not past reth's persisted head when the pruner + /// caps it there. The save that wrote `latest` had a tip at or above it and saw reth + /// persisted at least through `persisted_before_commit` of `latest`, and the earliest block + /// never moves down, which gives the lower end. No save's tip exceeds the highest head, + /// which gives the upper end. + pub(super) fn earliest_bounds(&self, latest: u64) -> RangeInclusive { + let persisted_before_the_save = self.block(latest).persisted_before_commit; + let lowest = latest.saturating_sub(self.window).min(persisted_before_the_save); + let highest = self.highest_head.saturating_sub(self.window); + lowest..=highest + } + pub(super) const fn persisted_at_least(&self) -> u64 { self.persisted_at_least } @@ -380,6 +401,7 @@ mod tests { ModelBlock { hash: canonical(number).hash, parent: canonical(number.saturating_sub(1)).hash, + persisted_before_commit: 0, serial: number, state_root: B256::ZERO, } @@ -474,4 +496,40 @@ mod tests { model.note_persisted(2); assert_eq!(model.restart(canonical(tip)).is_ok(), consistent); } + + /// A chain through `head` under `window`, whose block `n` was committed with reth persisted + /// through `min(n, persisted)`. + fn model_with_reth_persisted(window: u64, head: u64, persisted: u64) -> Model { + let mut model = Model::new(block(0), window); + for number in 1..=head { + let persisted_before_commit = persisted.min(number); + model.push(ModelBlock { persisted_before_commit, ..block(number) }); + } + model + } + + #[rstest] + #[case::whole_window(WINDOW, 7, 7, 7, 0..=0)] + #[case::reth_persisted_through_the_prune_target(4, 8, 8, 8, 4..=4)] + #[case::reth_persisted_below_the_prune_target(4, 8, 8, 3, 3..=4)] + #[case::latest_unwound_below_the_highest_head(4, 8, 5, 8, 1..=4)] + #[case::latest_within_a_window_of_genesis(4, 8, 2, 8, 0..=4)] + fn earliest_bounds( + #[case] window: u64, + #[case] head: u64, + #[case] latest: u64, + #[case] persisted: u64, + #[case] expected: RangeInclusive, + ) { + let model = model_with_reth_persisted(window, head, persisted); + assert_eq!(model.earliest_bounds(latest), expected); + } + + #[test] + fn earliest_bounds_keep_the_highest_head_through_a_reorg() { + let mut model = model_with_reth_persisted(4, 8, 5); + model.truncate(5); + model.push(ModelBlock { persisted_before_commit: 5, ..block(6) }); + assert_eq!(model.earliest_bounds(6), 2..=4); + } } From e7a41c62b0b7b2128e55d9686d0d4ee1a0b970fa Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Mon, 9 Nov 2026 23:27:28 +0100 Subject: [PATCH 3/4] op-reth: adapt the proofs engine tip task to the reth v2.6.0 bump The engine state now asks its provider for the canonical in-memory state and the stage checkpoints instead of a state reader and factory, so the tip task bounds its provider the same way. Co-Authored-By: Claude Fable 5.1 --- rust/op-reth/crates/trie/src/engine/tasks/tip.rs | 13 +++++++------ rust/op-reth/crates/trie/tests/live.rs | 4 +++- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/rust/op-reth/crates/trie/src/engine/tasks/tip.rs b/rust/op-reth/crates/trie/src/engine/tasks/tip.rs index d405fd56f5a..bcf714315fd 100644 --- a/rust/op-reth/crates/trie/src/engine/tasks/tip.rs +++ b/rust/op-reth/crates/trie/src/engine/tasks/tip.rs @@ -1,12 +1,13 @@ use super::super::state::EngineState; -use crate::{OpProofsStore, engine::EngineError}; +use crate::{ + OpProofsStore, + engine::{CanonicalInMemoryStateProvider, EngineError}, +}; use alloy_eips::NumHash; use crossbeam_channel::Sender; use reth_evm::ConfigureEvm; use reth_primitives_traits::BlockTy; -use reth_provider::{ - BlockHashReader, BlockReader, DatabaseProviderFactory, StateProviderFactory, StateReader, -}; +use reth_provider::{BlockHashReader, BlockReader, DatabaseProviderFactory, StageCheckpointReader}; pub(crate) struct TipTask { pub(crate) reply: Sender>, @@ -17,9 +18,9 @@ impl TipTask { where Evm: ConfigureEvm, Provider: BlockHashReader - + StateReader + + CanonicalInMemoryStateProvider + + StageCheckpointReader + DatabaseProviderFactory - + StateProviderFactory + BlockReader> + Clone + 'static, diff --git a/rust/op-reth/crates/trie/tests/live.rs b/rust/op-reth/crates/trie/tests/live.rs index 97ed8f0ed87..441e95ed9e8 100644 --- a/rust/op-reth/crates/trie/tests/live.rs +++ b/rust/op-reth/crates/trie/tests/live.rs @@ -2,9 +2,10 @@ use alloy_consensus::BlockHeader; use alloy_eips::BlockNumHash; -use alloy_primitives::{Address, B256, U256}; +use alloy_primitives::{Address, B256, Bytes, TxKind, U256}; use derive_more::Constructor; use reth_chainspec::ChainSpec; +use reth_db::Database; use reth_db_common::init::init_genesis; use reth_ethereum_primitives::{Block, BlockBody, Receipt}; use reth_evm::{ConfigureEvm, execute::Executor}; @@ -14,6 +15,7 @@ use reth_optimism_trie::{ MdbxProofsStorage, OpProofStoragePruner, OpProofsProviderRO, OpProofsStorage, OpProofsStore, RethTrieStorageLayout, engine::{EngineError, EngineHandle}, + initialize::InitializationJob, }; use reth_primitives_traits::{Block as _, RecoveredBlock}; use reth_provider::{ From ebb44a7fb01d3dbdeeaaaa2d1b9af388ca967d9f Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Tue, 10 Nov 2026 00:08:21 +0100 Subject: [PATCH 4/4] op-reth: keep the engine model's refusal errors out of expression position Co-Authored-By: Claude Fable 5.1 --- .../crates/trie/tests/engine_model/harness.rs | 22 ++++++++++++------- .../trie/tests/engine_model/invariants.rs | 2 +- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/rust/op-reth/crates/trie/tests/engine_model/harness.rs b/rust/op-reth/crates/trie/tests/engine_model/harness.rs index b01371df361..c56e73baeda 100644 --- a/rust/op-reth/crates/trie/tests/engine_model/harness.rs +++ b/rust/op-reth/crates/trie/tests/engine_model/harness.rs @@ -226,10 +226,10 @@ impl Harness { Ok(result) => Ok(result), Err(mpsc::RecvTimeoutError::Timeout) => { self.engine_hung = true; - eyre::bail!("the engine did not reply to {what} within {STEP_TIMEOUT:?}") + Err(eyre::eyre!("the engine did not reply to {what} within {STEP_TIMEOUT:?}")) } Err(mpsc::RecvTimeoutError::Disconnected) => { - eyre::bail!("the engine call {what} panicked") + Err(eyre::eyre!("the engine call {what} panicked")) } } } @@ -295,7 +295,9 @@ impl Harness { )); Reply::Refused { tip_hash: expected_parent_hash } } - Err(err) => eyre::bail!("the engine refused block {number}: {err}"), + Err(err) => { + return Err(eyre::eyre!("the engine refused block {number}: {err}")); + } }; self.model.deliver_block(num_hash, parent, parent_available, reply)?; self.prune_bodies_through(number) @@ -372,7 +374,9 @@ impl Harness { )); Ok(ControlFlow::Break(())) } - Err(err) => eyre::bail!("the engine refused a valid {what} from block {first}: {err}"), + Err(err) => { + Err(eyre::eyre!("the engine refused a valid {what} from block {first}: {err}")) + } } } @@ -507,10 +511,12 @@ impl Harness { ); true } - Err(err) => eyre::bail!( - "unexpected reply to a reorg at block {first} not building on the stored block \ - {fork}: {err}" - ), + Err(err) => { + return Err(eyre::eyre!( + "unexpected reply to a reorg at block {first} not building on the stored block \ + {fork}: {err}" + )); + } }; self.model.bogus_reorg(first, refused)?; self.log(format!( diff --git a/rust/op-reth/crates/trie/tests/engine_model/invariants.rs b/rust/op-reth/crates/trie/tests/engine_model/invariants.rs index 8338d899a6d..e7c58438f2d 100644 --- a/rust/op-reth/crates/trie/tests/engine_model/invariants.rs +++ b/rust/op-reth/crates/trie/tests/engine_model/invariants.rs @@ -78,7 +78,7 @@ impl Harness { } latest = now; } - eyre::bail!("the store kept changing over {MAX_SETTLE_FLUSHES} flushes") + Err(eyre::eyre!("the store kept changing over {MAX_SETTLE_FLUSHES} flushes")) } pub(super) fn check_settled(&mut self) -> eyre::Result<()> {