Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions rust/.config/nextest.toml
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,12 @@ filter = 'test(=log_estimate_bounds_measured_pgu) | test(=consolidation_tests::c
slow-timeout = { period = "1m", terminate-after = 10 }
threads-required = "num-test-threads"

# The proofs engine model test runs its whole proptest as one test, and a failing case shrinks
# for up to a minute on top of the run.
[[profile.default.overrides]]
filter = "binary(engine_model)"
slow-timeout = { period = "1m", terminate-after = 5 }

# E2E tests using the testsuite framework from crates/e2e-test-utils.
[[profile.default.overrides]]
filter = "binary(e2e_testsuite)"
Expand Down
1 change: 1 addition & 0 deletions rust/Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions rust/op-reth/crates/trie/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -82,6 +82,7 @@ secp256k1 = { workspace = true, features = ["rand", "std"] }
mockall.workspace = true
rstest.workspace = true
eyre.workspace = true
proptest.workspace = true

# misc
serial_test.workspace = true
Expand Down
14 changes: 14 additions & 0 deletions rust/op-reth/crates/trie/src/engine/handle.rs
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,8 @@ use super::{
tasks::{ExecuteBlockTask, IndexBlockTask, ReorgBlock, ReorgTask, SyncToTask, UnwindTask},
};
use crate::{OpProofStoragePruner, OpProofsStore};
#[cfg(any(test, feature = "test-utils"))]
use alloy_eips::NumHash;
use alloy_eips::eip1898::BlockWithParent;
use crossbeam_channel::{Sender, bounded};
use reth_evm::ConfigureEvm;
Expand Down Expand Up @@ -170,4 +172,16 @@ impl<Block: reth_primitives_traits::Block + Send + 'static> EngineHandle<Block>
let _ = reply_rx.recv();
}
}

/// The engine's tip: the newest buffered block, or the store's latest block when nothing is
/// buffered (test/utility only).
#[cfg(any(test, feature = "test-utils"))]
pub fn tip(&self) -> Result<NumHash, EngineError> {
use super::tasks::TipTask;
let (reply_tx, reply_rx) = bounded(1);
self.sender
.send(EngineAction::Tip(TipTask { reply: reply_tx }))
.map_err(|_| EngineError::EngineDied)?;
reply_rx.recv().map_err(|_| EngineError::EngineDied)?
}
}
5 changes: 5 additions & 0 deletions rust/op-reth/crates/trie/src/engine/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,9 @@ enum EngineAction<Block: reth_primitives_traits::Block> {
Flush(tasks::FlushTask),
/// Update the sync catch-up target (fire-and-forget).
SyncTo(tasks::SyncToTask),
/// Report the engine's tip.
#[cfg(any(test, feature = "test-utils"))]
Tip(tasks::TipTask),
}

impl<Block: reth_primitives_traits::Block> EngineAction<Block> {
Expand All @@ -85,6 +88,8 @@ impl<Block: reth_primitives_traits::Block> EngineAction<Block> {
#[cfg(any(test, feature = "test-utils"))]
Self::Flush(task) => task.execute(state),
Self::SyncTo(task) => task.execute(state),
#[cfg(any(test, feature = "test-utils"))]
Self::Tip(task) => task.execute(state),
}
}
}
4 changes: 4 additions & 0 deletions rust/op-reth/crates/trie/src/engine/tasks/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,8 @@ mod index_block;
mod reorg;
mod reth_auxiliary;
mod sync_to;
#[cfg(any(test, feature = "test-utils"))]
mod tip;
mod unwind;

pub(super) use execute_block::{ExecuteBlockTask, run as execute_block};
Expand All @@ -20,4 +22,6 @@ pub(super) use index_block::{IndexBlockTask, run as index_block};
pub use reorg::ReorgBlock;
pub(super) use reorg::ReorgTask;
pub(super) use sync_to::SyncToTask;
#[cfg(any(test, feature = "test-utils"))]
pub(super) use tip::TipTask;
pub(super) use unwind::UnwindTask;
31 changes: 31 additions & 0 deletions rust/op-reth/crates/trie/src/engine/tasks/tip.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
use super::super::state::EngineState;
use crate::{
OpProofsStore,
engine::{CanonicalInMemoryStateProvider, EngineError},
};
use alloy_eips::NumHash;
use crossbeam_channel::Sender;
use reth_evm::ConfigureEvm;
use reth_primitives_traits::BlockTy;
use reth_provider::{BlockHashReader, BlockReader, DatabaseProviderFactory, StageCheckpointReader};

pub(crate) struct TipTask {
pub(crate) reply: Sender<Result<NumHash, EngineError>>,
}

impl TipTask {
pub(crate) fn execute<Evm, Provider, Store>(self, state: &EngineState<Evm, Provider, Store>)
where
Evm: ConfigureEvm,
Provider: BlockHashReader
+ CanonicalInMemoryStateProvider
+ StageCheckpointReader
+ DatabaseProviderFactory
+ BlockReader<Block = BlockTy<Evm::Primitives>>
+ Clone
+ 'static,
Store: OpProofsStore + Clone + 'static,
{
let _ = self.reply.send(state.get_tip());
}
}
123 changes: 123 additions & 0 deletions rust/op-reth/crates/trie/tests/engine_model.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,123 @@
//! Model-based test of the proofs engine: random interleavings of commits, reorgs, reverts, sync
//! targets, flushes, failed saves, stale notifications and crashes, driven over reth's
//! persistence model and held to a reference model after every step. The deterministic boundary
//! cases live in the live target; this target is for the interleavings.

use proptest::prelude::*;

#[path = "engine_model/actions.rs"]
mod actions;
#[path = "live/chain.rs"]
#[allow(dead_code, reason = "each test target uses part of this module")]
mod chain;
#[path = "engine_model/faults.rs"]
mod faults;
#[path = "engine_model/harness.rs"]
mod harness;
#[path = "engine_model/invariants.rs"]
mod invariants;
#[path = "engine_model/model.rs"]
mod model;
#[path = "live/simulated_reth.rs"]
#[allow(dead_code, reason = "each test target uses part of this module")]
mod simulated_reth;
#[path = "live/store_view.rs"]
#[allow(dead_code, reason = "each test target uses part of this module")]
mod store_view;

use actions::{Action, Actions, Config, Rebuild};
use chain::*;
use harness::Harness;
use proptest::test_runner::contextualize_config;
use simulated_reth::RethPersistence;
use std::sync::atomic::{AtomicBool, Ordering};

/// Cases per run unless `PROPTEST_CASES` says otherwise.
const DEFAULT_CASES: u32 = 24;

/// The default count and a shrinking budget that fits the test's time limit, with the
/// `PROPTEST_*` environment variables applied on top.
fn proptest_config() -> ProptestConfig {
contextualize_config(ProptestConfig {
cases: DEFAULT_CASES,
failure_persistence: None,
max_shrink_iters: 256,
max_shrink_time: 60_000,
..ProptestConfig::default()
})
}

fn run_sequence(config: Config, actions: &[Action]) -> eyre::Result<()> {
Harness::new(config)?.run(actions)
}

static FIRST_FAILURE_PRINTED: AtomicBool = AtomicBool::new(false);

/// Runs a case and prints the first failure as found, before shrinking, so that a run cut short
/// by a time limit still shows what failed.
fn run_case(config: Config, actions: &[Action]) -> Result<(), TestCaseError> {
run_sequence(config, actions).map_err(|err| {
if !FIRST_FAILURE_PRINTED.swap(true, Ordering::SeqCst) {
eprintln!("first failing case, before shrinking: {config:?}\n{err:?}\n");
}
TestCaseError::fail(format!("{err:?}"))
})
}

proptest! {
#![proptest_config(proptest_config())]

/// Under every cell of the matrix, the engine's tip and the store's window stay on the
/// canonical chain and within what the model allows after each random step, every valid
/// event is taken, every reorg on a wrong parent or below the window is refused, and the
/// engine settles within the step bound.
#[test]
fn engine_and_store_follow_the_model(
config in actions::config(),
actions in actions::sequence(Actions::DEFAULT),
) {
run_case(config, &actions)?;
}

/// As above, with crashes after which reth rebuilds a different chain, as a sequencer does.
#[test]
#[ignore = "fails on develop: after an unclean stop the store keeps blocks reth lost, and a chain rebuilt differently leaves the store's latest block off the canonical chain (ethereum-optimism/protocol-team#464; the harness's restart must gain the startup reconcile of #23358 to pass)"]
fn engine_and_store_follow_the_model_through_crashes_rebuilding_a_different_chain(
config in actions::config(),
actions in actions::sequence(Actions::WITH_TRACKED_BUGS),
) {
run_case(config, &actions)?;
}
}

/// The matrix cell the replayed sequences run under.
fn replay_config(cadence: FlushCadence, persistence_threshold: u64) -> Config {
Config {
body_pruning: false,
cadence,
persistence: RethPersistence::threshold(persistence_threshold),
verification_interval: 0,
window: WHOLE_WINDOW,
}
}

/// The minimal sequence the random run shrinks to for protocol-team#464: one block saved, then a
/// crash reth comes back from with a different block 1, which the store keeps serving.
#[test]
#[ignore = "fails on develop: the store's latest block 1 is not on the chain reth rebuilt after the crash (ethereum-optimism/protocol-team#464; the harness's restart must gain the startup reconcile of #23358 to pass)"]
fn store_follows_reth_through_a_crash_rebuilding_a_different_chain() -> eyre::Result<()> {
run_sequence(
replay_config(FlushCadence::EVERY_BLOCK, 50),
&[Action::Commit { count: 1 }, Action::Crash(Rebuild::Different)],
)
}

/// The ExEx acknowledges a notification's tip before the sync loop has executed the bodies
/// below it, so a reth that prunes bodies through the acknowledged height takes them from under
/// the sync loop and the catch-up stalls.
#[test]
#[ignore = "fails on develop: reth prunes the bodies the sync loop still needs because the ExEx acknowledges the notification tip, and the engine never settles (ethereum-optimism/protocol-team#478)"]
fn sync_loop_completes_while_reth_prunes_bodies_through_the_acknowledged_tip() -> eyre::Result<()> {
let config = Config { body_pruning: true, ..replay_config(FlushCadence::EVERY_BLOCK, 7) };
run_sequence(config, &[Action::SyncTo { count: 40 }, Action::Commit { count: 1 }])
}
126 changes: 126 additions & 0 deletions rust/op-reth/crates/trie/tests/engine_model/actions.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,126 @@
//! What a step can do to reth, the ExEx and the node, and the configuration a run samples.

use super::{FlushCadence, WHOLE_WINDOW, simulated_reth::RethPersistence};
use proptest::{collection::vec, prelude::*, sample::select};

/// Steps per case at most.
const MAX_STEPS: usize = 24;

/// A proof window the chains here outgrow, so that pruning moves the earliest block.
const PRUNED_WINDOW: u64 = 4;

/// How reth rebuilds the blocks a crash lost: from the same consensus-layer chain, as a verifier
/// does, or as different blocks, as a sequencer does.
#[derive(Clone, Copy, Debug)]
pub(crate) enum Rebuild {
Different,
Same,
}

#[derive(Clone, Copy, Debug)]
pub(crate) enum Action {
/// A one-block reorg on a parent no block has, at a fork point the engine knows, which it
/// must refuse.
BogusReorg { offset: u64 },
/// reth commits `count` blocks, each delivered to the engine as a live notification.
Commit { count: u64 },
/// reth and the engine stop uncleanly, losing what either held in memory, and restart.
Crash(Rebuild),
/// The next save fails at its write transaction, and enough blocks follow to start it.
FailNextSave,
/// Waits for the save in flight, if any.
Flush,
/// reth replaces its newest `depth` in-memory blocks with `length` new ones and delivers
/// the reorg.
Reorg { depth: u64, length: u64 },
/// reth drops its newest `depth` in-memory blocks and delivers the revert.
Revert { depth: u64 },
/// reth commits `depth` blocks and reorgs them away before their notifications reach the
/// engine, then the reorg's does.
StaleCommit { depth: u64, length: u64 },
/// reth commits `count` blocks the engine only learns of through a sync target, as the ExEx
/// hands over a notification far behind reth's head.
SyncTo { count: u64 },
}

/// Which extra actions a run may take.
#[derive(Clone, Copy, Debug)]
pub(crate) struct Actions {
/// A crash after which reth rebuilds a different chain (ethereum-optimism/protocol-team#464).
pub(crate) crash_rebuilding_a_different_chain: bool,
}

impl Actions {
pub(crate) const DEFAULT: Self = Self { crash_rebuilding_a_different_chain: false };
pub(crate) const WITH_TRACKED_BUGS: Self = Self { crash_rebuilding_a_different_chain: true };
}

fn rebuild(actions: Actions) -> BoxedStrategy<Rebuild> {
if actions.crash_rebuilding_a_different_chain {
prop_oneof![Just(Rebuild::Same), Just(Rebuild::Different)].boxed()
} else {
Just(Rebuild::Same).boxed()
}
}

fn action(actions: Actions) -> impl Strategy<Value = Action> {
prop_oneof![
6 => (1..=4u64).prop_map(|count| Action::Commit { count }),
3 => (1..=8u64, 1..=4u64).prop_map(|(depth, length)| Action::Reorg { depth, length }),
2 => (1..=4u64).prop_map(|depth| Action::Revert { depth }),
3 => (1..=12u64).prop_map(|count| Action::SyncTo { count }),
3 => Just(Action::Flush),
1 => Just(Action::FailNextSave),
2 => (1..=3u64, 1..=3u64).prop_map(|(depth, length)| Action::StaleCommit { depth, length }),
1 => (0..=8u64).prop_map(|offset| Action::BogusReorg { offset }),
1 => rebuild(actions).prop_map(Action::Crash),
]
}

pub(crate) fn sequence(actions: Actions) -> impl Strategy<Value = Vec<Action>> {
vec(action(actions), 1..=MAX_STEPS)
}

/// One cell of the matrix.
#[derive(Clone, Copy, Debug)]
pub(crate) struct Config {
/// reth prunes block bodies through what the ExEx acknowledges, the notification's tip.
pub(crate) body_pruning: bool,
pub(crate) cadence: FlushCadence,
pub(crate) persistence: RethPersistence,
/// Every block at a multiple of this is executed rather than indexed; 0 indexes every block.
pub(crate) verification_interval: u64,
pub(crate) window: u64,
}

impl Config {
pub(crate) const fn executes(&self, number: u64) -> bool {
self.verification_interval > 0 && number.is_multiple_of(self.verification_interval)
}
}

/// reth's persistence settings: the thresholds of develop and of reth 2.6, the latter also with
/// its default masking window, which reth only accepts below the threshold.
fn persistence() -> impl Strategy<Value = RethPersistence> {
select(vec![7u64, 50]).prop_flat_map(|threshold| {
let masking = if threshold == 50 { vec![0u64, 30] } else { vec![0u64] };
select(masking)
.prop_map(move |masking| RethPersistence::threshold(threshold).masking(masking))
})
}

pub(crate) fn config() -> impl Strategy<Value = Config> {
(
select(vec![FlushCadence::EVERY_BLOCK, FlushCadence::EVERY_FIFTH_BLOCK]),
persistence(),
select(vec![0u64, 1, 4]),
select(vec![WHOLE_WINDOW, PRUNED_WINDOW]),
)
.prop_map(|(cadence, persistence, verification_interval, window)| Config {
body_pruning: false,
cadence,
persistence,
verification_interval,
window,
})
}
Loading
Loading