diff --git a/rust/op-reth/crates/trie/src/engine/handle.rs b/rust/op-reth/crates/trie/src/engine/handle.rs index 4b80a7ef0be..c369bf31a1b 100644 --- a/rust/op-reth/crates/trie/src/engine/handle.rs +++ b/rust/op-reth/crates/trie/src/engine/handle.rs @@ -1,8 +1,7 @@ //! [`EngineHandle`] — the public, cloneable, Send + Sync interface. use super::{ - CanonicalInMemoryStateProvider, DEFAULT_BACKPRESSURE_THRESHOLD, DEFAULT_PERSISTENCE_THRESHOLD, - EngineAction, + CanonicalInMemoryStateProvider, EngineAction, EngineConfig, error::EngineError, runner::Engine, service_guard::ServiceGuard, @@ -12,7 +11,7 @@ use crate::{OpProofStoragePruner, OpProofsStore}; #[cfg(any(test, feature = "test-utils"))] use alloy_eips::NumHash; use alloy_eips::eip1898::BlockWithParent; -use crossbeam_channel::{Sender, bounded}; +use crossbeam_channel::{Receiver, Sender, bounded, select_biased}; use reth_evm::ConfigureEvm; use reth_primitives_traits::{NodePrimitives, RecoveredBlock}; use reth_provider::{BlockHashReader, BlockReader, DatabaseProviderFactory, StageCheckpointReader}; @@ -23,9 +22,13 @@ use tracing::error; /// A thin, cloneable handle used to communicate with the collector engine. /// /// Every public method (except [`Self::sync_to`]) sends an engine action to the -/// engine thread and blocks on a one-shot reply channel. +/// engine thread and blocks on a one-shot reply channel, or until the engine exits. #[derive(Debug, Clone)] pub struct EngineHandle { + /// Disconnects when the engine thread exits by any path, so that no call waits on a reply + /// the engine will never send: an action queued when the engine exits stays in the channel, + /// with its reply sender, for as long as a handle lives. + engine_exited: Receiver<()>, sender: Sender>, _service_guard: Arc, } @@ -49,14 +52,7 @@ impl EngineHandle + 'static, Store: OpProofsStore + Clone + 'static, { - Self::spawn_with_thresholds( - evm_config, - provider, - storage, - pruner, - DEFAULT_PERSISTENCE_THRESHOLD, - DEFAULT_BACKPRESSURE_THRESHOLD, - ) + Self::spawn_with_config(evm_config, provider, storage, pruner, EngineConfig::default()) } /// Spawn the collector engine with custom persistence and backpressure thresholds. @@ -68,6 +64,30 @@ impl EngineHandle persistence_threshold: u64, backpressure_threshold: u64, ) -> Self + where + Evm: ConfigureEvm> + 'static, + Provider: BlockHashReader + + CanonicalInMemoryStateProvider + + StageCheckpointReader + + DatabaseProviderFactory + + BlockReader + + Clone + + 'static, + Store: OpProofsStore + Clone + 'static, + { + let config = + EngineConfig { backpressure_threshold, persistence_threshold, ..Default::default() }; + Self::spawn_with_config(evm_config, provider, storage, pruner, config) + } + + /// Spawn the collector engine with `config`. + pub fn spawn_with_config( + evm_config: Evm, + provider: Provider, + storage: Store, + pruner: OpProofStoragePruner, + config: EngineConfig, + ) -> Self where Evm: ConfigureEvm> + 'static, Provider: BlockHashReader @@ -80,9 +100,8 @@ impl EngineHandle Store: OpProofsStore + Clone + 'static, { let (tx, rx) = bounded(10); - let engine = Engine::new(evm_config, provider, storage, pruner, rx) - .with_persistence_threshold(persistence_threshold) - .with_backpressure_threshold(backpressure_threshold); + let (exited, engine_exited) = bounded(0); + let engine = Engine::new(evm_config, provider, storage, pruner, rx, exited, config); let join_handle = thread::Builder::new() .name("live-trie-collector".into()) @@ -98,7 +117,15 @@ impl EngineHandle }) .expect("failed to spawn live-trie-collector thread"); - Self { sender: tx, _service_guard: Arc::new(ServiceGuard::new(join_handle)) } + Self { engine_exited, sender: tx, _service_guard: Arc::new(ServiceGuard::new(join_handle)) } + } + + /// Waits for the reply to an action, or for the engine to exit; a reply already there wins. + fn recv_reply(&self, reply: &Receiver) -> Result { + select_biased! { + recv(reply) -> reply => reply.map_err(|_| EngineError::EngineDied), + recv(self.engine_exited) -> _ => Err(EngineError::EngineDied), + } } fn send_and_recv( @@ -107,7 +134,7 @@ impl EngineHandle ) -> Result<(), EngineError> { let (reply_tx, reply_rx) = bounded(1); self.sender.send(make_action(reply_tx)).map_err(|_| EngineError::EngineDied)?; - reply_rx.recv().map_err(|_| EngineError::EngineDied)? + self.recv_reply(&reply_rx)? } /// Execute a block through the EVM and buffer the resulting trie updates. @@ -169,7 +196,7 @@ impl EngineHandle use super::tasks::FlushTask; let (reply_tx, reply_rx) = bounded(1); if self.sender.send(EngineAction::Flush(FlushTask { reply: reply_tx })).is_ok() { - let _ = reply_rx.recv(); + let _ = self.recv_reply(&reply_rx); } } @@ -182,6 +209,6 @@ impl EngineHandle self.sender .send(EngineAction::Tip(TipTask { reply: reply_tx })) .map_err(|_| EngineError::EngineDied)?; - reply_rx.recv().map_err(|_| EngineError::EngineDied)? + self.recv_reply(&reply_rx)? } } diff --git a/rust/op-reth/crates/trie/src/engine/mod.rs b/rust/op-reth/crates/trie/src/engine/mod.rs index 975c39a5544..e39fd067803 100644 --- a/rust/op-reth/crates/trie/src/engine/mod.rs +++ b/rust/op-reth/crates/trie/src/engine/mod.rs @@ -28,22 +28,49 @@ mod runner; mod service_guard; mod state; +use std::time::Duration; + /// Default number of blocks to keep in memory before persisting. const DEFAULT_PERSISTENCE_THRESHOLD: u64 = 5; /// Default number of blocks where we block execution to allow persistence to catch up. const DEFAULT_BACKPRESSURE_THRESHOLD: u64 = 10; -/// Default timeout for waiting on a persistence save/unwind operation (in seconds). -const DEFAULT_PERSISTENCE_TIMEOUT_SECS: u64 = 60; +/// Default timeout for waiting on a persistence save/unwind operation. +const DEFAULT_PERSISTENCE_TIMEOUT: Duration = Duration::from_secs(60); -/// How long the engine waits with a non-empty memory buffer before flushing it even if the -/// persistence threshold has not been reached. +/// How long the engine waits by default with a non-empty memory buffer before flushing it even +/// if the persistence threshold has not been reached. /// /// Without this, a paused chain (e.g. fault-proof tests that freeze the sequencer) would leave /// buffered blocks unpersisted indefinitely, breaking the proofs RPC's strict "is this block /// persisted?" check. -const IDLE_FLUSH_INTERVAL: std::time::Duration = std::time::Duration::from_secs(10); +const DEFAULT_IDLE_FLUSH_INTERVAL: Duration = Duration::from_secs(10); + +/// How the engine paces its saves and how long it waits on its persistence service. +#[derive(Clone, Copy, Debug)] +pub struct EngineConfig { + /// Buffered blocks at which the engine stops taking actions until the save in flight + /// completes. + pub backpressure_threshold: u64, + /// How long buffered blocks below the persistence threshold wait before a save starts anyway. + pub idle_flush_interval: Duration, + /// Buffered blocks that start a save. + pub persistence_threshold: u64, + /// How long the engine waits for a save or an unwind before giving up on it. + pub persistence_timeout: Duration, +} + +impl Default for EngineConfig { + fn default() -> Self { + Self { + backpressure_threshold: DEFAULT_BACKPRESSURE_THRESHOLD, + idle_flush_interval: DEFAULT_IDLE_FLUSH_INTERVAL, + persistence_threshold: DEFAULT_PERSISTENCE_THRESHOLD, + persistence_timeout: DEFAULT_PERSISTENCE_TIMEOUT, + } + } +} /// Messages sent from [`EngineHandle`] to the engine thread. enum EngineAction { diff --git a/rust/op-reth/crates/trie/src/engine/runner.rs b/rust/op-reth/crates/trie/src/engine/runner.rs index aabb8e54f04..44c87522927 100644 --- a/rust/op-reth/crates/trie/src/engine/runner.rs +++ b/rust/op-reth/crates/trie/src/engine/runner.rs @@ -1,11 +1,11 @@ //! [`Engine`] — the thin event-loop dispatcher. use super::{ - CanonicalInMemoryStateProvider, DEFAULT_BACKPRESSURE_THRESHOLD, DEFAULT_PERSISTENCE_THRESHOLD, - EngineAction, IDLE_FLUSH_INTERVAL, error::EngineError, state::EngineState as State, + CanonicalInMemoryStateProvider, EngineAction, EngineConfig, error::EngineError, + state::EngineState as State, }; use crate::{OpProofStoragePruner, OpProofsStore}; -use crossbeam_channel::Receiver; +use crossbeam_channel::{Receiver, Sender}; use reth_evm::ConfigureEvm; use reth_primitives_traits::BlockTy; use reth_provider::{ @@ -37,10 +37,13 @@ where Evm: ConfigureEvm, Provider: CanonicalInMemoryStateProvider + DatabaseProviderFactory + BlockReader, { - state: State, + /// Dropped when the engine exits by any path, which every call waiting on the handle takes + /// as `EngineDied`. Must stay declared first: fields drop in declaration order, and dropping + /// `state` joins the persistence thread, which a slow save can hold indefinitely. + _exited: Sender<()>, incoming: Receiver>>, - persistence_threshold: u64, - backpressure_threshold: u64, + state: State, + config: EngineConfig, /// Current retry delay for the sync arm; grows on failure, resets on success. sync_backoff: Duration, /// Earliest instant the next sync step may run: already past unless the last step failed. @@ -66,27 +69,19 @@ where storage: Store, pruner: OpProofStoragePruner, incoming: Receiver>>, + exited: Sender<()>, + config: EngineConfig, ) -> Self { Self { - state: State::new(evm_config, provider, storage, pruner), + _exited: exited, incoming, - persistence_threshold: DEFAULT_PERSISTENCE_THRESHOLD, - backpressure_threshold: DEFAULT_BACKPRESSURE_THRESHOLD, + state: State::new(evm_config, provider, storage, pruner, config.persistence_timeout), + config, sync_backoff: Duration::ZERO, sync_retry_at: Instant::now(), } } - pub(super) const fn with_persistence_threshold(mut self, threshold: u64) -> Self { - self.persistence_threshold = threshold; - self - } - - pub(super) const fn with_backpressure_threshold(mut self, threshold: u64) -> Self { - self.backpressure_threshold = threshold; - self - } - /// Returns `true` if the engine is behind its sync target. fn needs_sync(&self) -> bool { let current_tip = self.state.get_tip().map(|t| t.number).unwrap_or(0); @@ -96,12 +91,12 @@ where /// Returns `true` if the buffer is above the backpressure threshold with a save in-flight. fn backpressure_active(&self) -> bool { self.state.persistence.in_flight.is_some() && - self.state.memory.len() as u64 >= self.backpressure_threshold + self.state.memory.len() as u64 >= self.config.backpressure_threshold } /// Start a background persistence save if the memory buffer has reached the threshold. fn maybe_start_save(&mut self) { - if self.state.memory.len() as u64 >= self.persistence_threshold && + if self.state.memory.len() as u64 >= self.config.persistence_threshold && let Err(e) = self.state.advance_persistence() { error!(target: "trie::engine::runner", ?e, "Failed to start persistence save"); @@ -154,13 +149,13 @@ where /// /// Four receivers compete in a single `select!`: /// - **action**: a new [`EngineAction`] from a caller, or [`crossbeam_channel::never`] while - /// backpressure is active — callers naturally block in their bounded `send` until the - /// in-flight save completes and memory is pruned. + /// backpressure is active — callers' actions then wait in the channel until the in-flight + /// save completes and memory is pruned. /// - **persistence**: signals a completed background save. /// - **sync**: fires once `sync_retry_at` has passed, at once unless the last step failed, /// while the engine is behind its sync target and not under backpressure; /// [`crossbeam_channel::never`] otherwise. - /// - **idle-flush**: fires after [`IDLE_FLUSH_INTERVAL`] when memory holds buffered blocks but + /// - **idle-flush**: fires after the idle flush interval when memory holds buffered blocks but /// the persistence threshold hasn't been reached and no save is in flight. Keeps a paused /// chain from leaving buffered blocks invisible to the proofs RPC indefinitely. /// @@ -188,7 +183,7 @@ where // Arm the idle-flush timer only when there's buffered work that nothing else will flush. let idle_flush_rx: Receiver = if !self.state.memory.is_empty() && !save_in_flight && !backpressure { - crossbeam_channel::after(IDLE_FLUSH_INTERVAL) + crossbeam_channel::after(self.config.idle_flush_interval) } else { crossbeam_channel::never() }; @@ -217,10 +212,10 @@ where /// Runs the main loop of the engine, processing incoming actions. pub(super) fn run(mut self) { debug_assert!( - self.persistence_threshold < self.backpressure_threshold, + self.config.persistence_threshold < self.config.backpressure_threshold, "backpressure_threshold ({}) must be greater than persistence_threshold ({})", - self.backpressure_threshold, - self.persistence_threshold, + self.config.backpressure_threshold, + self.config.persistence_threshold, ); debug!(target: "trie::engine::runner", "Collector engine started"); @@ -230,6 +225,10 @@ where ControlFlow::Continue(()) => {} } self.maybe_start_save(); + if self.state.persistence.is_disconnected() { + error!(target: "trie::engine::runner", "Persistence service gone, stopping the collector engine"); + break; + } } debug!(target: "trie::engine::runner", "Collector engine shutting down, draining in-flight persist"); @@ -243,7 +242,7 @@ mod tests { use super::{Engine, SYNC_BACKOFF_INITIAL, SYNC_BACKOFF_MAX, next_sync_backoff}; use crate::{ InMemoryProofsStorage, OpProofStoragePruner, OpProofsInitProvider, OpProofsStore, - engine::{EngineAction, tasks::SyncToTask}, + engine::{EngineAction, EngineConfig, tasks::SyncToTask}, }; use alloy_consensus::Header; use alloy_eips::BlockNumHash; @@ -284,8 +283,16 @@ mod tests { OpProofsInitProvider::commit(init).expect("commit"); let pruner = OpProofStoragePruner::new(storage.clone(), provider.clone(), 1000); let (actions, incoming) = bounded(10); - let mut engine = - Engine::new(EthEvmConfig::ethereum(chain_spec), provider, storage, pruner, incoming); + let (exited, _engine_exited) = bounded(0); + let mut engine = Engine::new( + EthEvmConfig::ethereum(chain_spec), + provider, + storage, + pruner, + incoming, + exited, + EngineConfig::default(), + ); engine.state.sync_target = 1; (engine, actions) } diff --git a/rust/op-reth/crates/trie/src/engine/state.rs b/rust/op-reth/crates/trie/src/engine/state.rs index e6a696d7145..4a416a19fa6 100644 --- a/rust/op-reth/crates/trie/src/engine/state.rs +++ b/rust/op-reth/crates/trie/src/engine/state.rs @@ -3,7 +3,7 @@ #[cfg(feature = "metrics")] use super::metrics::EngineMetrics; use super::{ - CanonicalInMemoryStateProvider, DEFAULT_PERSISTENCE_TIMEOUT_SECS, + CanonicalInMemoryStateProvider, buffer::state::TrieBufferState, error::EngineError, persistence::{PersistenceHandle, error::PersistenceError}, @@ -21,17 +21,26 @@ use tracing::{error, info, warn}; /// Tracks all in-flight state for background persistence. pub(crate) struct PersistenceState { + /// Set once the service is found gone, which only its thread dying brings about: nothing + /// can be saved or unwound after that. + disconnected: bool, /// Handle to the persistence service. handle: PersistenceHandle, /// Reply channel for the in-flight save. Present only while a save is running. /// /// Exposed `pub(crate)` so the engine loop can include it in a `select!`. pub(crate) in_flight: Option, PersistenceError>>>, + /// How long a wait on the service lasts before it is given up. + timeout: Duration, } impl PersistenceState { - const fn new(handle: PersistenceHandle) -> Self { - Self { handle, in_flight: None } + const fn new(handle: PersistenceHandle, timeout: Duration) -> Self { + Self { disconnected: false, handle, in_flight: None, timeout } + } + + pub(crate) const fn is_disconnected(&self) -> bool { + self.disconnected } /// Blocking: wait for the in-flight save to finish and prune `memory`. @@ -40,7 +49,7 @@ impl PersistenceState { pub(crate) fn wait(&mut self, memory: &TrieBufferState) { let Some(rx) = self.in_flight.take() else { return }; - match rx.recv_timeout(Duration::from_secs(DEFAULT_PERSISTENCE_TIMEOUT_SECS)) { + match rx.recv_timeout(self.timeout) { Ok(Ok(Some(last_persisted))) => { info!( target: "trie::engine::state", @@ -57,6 +66,7 @@ impl PersistenceState { error!(target: "trie::engine::state", "Persistence timeout while waiting"); } Err(RecvTimeoutError::Disconnected) => { + self.disconnected = true; error!(target: "trie::engine::state", "Persistence service disconnected while waiting"); } } @@ -85,6 +95,7 @@ impl PersistenceState { error!(target: "trie::engine::state", ?e, "Background persistence save failed"); } Err(_) => { + self.disconnected = true; error!(target: "trie::engine::state", "Persistence service disconnected unexpectedly"); } } @@ -116,7 +127,10 @@ impl PersistenceState { ); let (tx, rx) = bounded(1); - self.handle.save_updates(blocks, tx)?; + if let Err(err) = self.handle.save_updates(blocks, tx) { + self.disconnected = true; + return Err(err.into()); + } self.in_flight = Some(rx); Ok(()) @@ -124,15 +138,21 @@ impl PersistenceState { /// Send an unwind to the persistence service and block until it completes. The caller /// drains any in-flight save first. - pub(crate) fn unwind(&self, to: BlockWithParent) -> Result<(), EngineError> { + pub(crate) fn unwind(&mut self, to: BlockWithParent) -> Result<(), EngineError> { let (tx, rx) = bounded(1); - self.handle.unwind(to, tx)?; + if let Err(err) = self.handle.unwind(to, tx) { + self.disconnected = true; + return Err(err.into()); + } - match rx.recv_timeout(Duration::from_secs(DEFAULT_PERSISTENCE_TIMEOUT_SECS)) { + match rx.recv_timeout(self.timeout) { Ok(Ok(())) => Ok(()), Ok(Err(e)) => Err(e.into()), Err(RecvTimeoutError::Timeout) => Err(EngineError::PersistenceTimeout), - Err(RecvTimeoutError::Disconnected) => Err(EngineError::PersistenceDisconnected), + Err(RecvTimeoutError::Disconnected) => { + self.disconnected = true; + Err(EngineError::PersistenceDisconnected) + } } } } @@ -174,6 +194,7 @@ where provider: Provider, storage: Store, pruner: OpProofStoragePruner, + persistence_timeout: Duration, ) -> Self { let persistence_handle = PersistenceHandle::spawn(pruner, storage.clone()); Self { @@ -181,7 +202,7 @@ where provider, storage, memory: TrieBufferState::new(), - persistence: PersistenceState::new(persistence_handle), + persistence: PersistenceState::new(persistence_handle, persistence_timeout), sync_target: 0, #[cfg(feature = "metrics")] metrics: EngineMetrics::new_with_labels(&[] as &[(&str, &str)]), @@ -287,7 +308,7 @@ where #[cfg(test)] mod tests { use super::*; - use crate::{OpProofsInitProvider, db::MdbxProofsStorage}; + use crate::{OpProofsInitProvider, db::MdbxProofsStorage, engine::EngineConfig}; use alloy_eips::{BlockNumHash, NumHash, eip1898::BlockWithParent}; use alloy_primitives::B256; use reth_chainspec::MAINNET; @@ -322,7 +343,8 @@ mod tests { let storage = bootstrap_storage(); let pruner = OpProofStoragePruner::new(storage.clone(), blockchain_db.clone(), 1000); let evm_config = EthEvmConfig::ethereum(chain_spec); - EngineState::new(evm_config, blockchain_db, storage, pruner) + let persistence_timeout = EngineConfig::default().persistence_timeout; + EngineState::new(evm_config, blockchain_db, storage, pruner, persistence_timeout) } /// An unwind target whose parent is unknown to the engine: the store holds only the anchor, diff --git a/rust/op-reth/crates/trie/tests/engine_liveness.rs b/rust/op-reth/crates/trie/tests/engine_liveness.rs new file mode 100644 index 00000000000..fa63f6b5bfa --- /dev/null +++ b/rust/op-reth/crates/trie/tests/engine_liveness.rs @@ -0,0 +1,445 @@ +//! Liveness of the proofs engine over a slow, stalling or dying persistence service. +//! +//! The engine waits for the save in flight before every unwind and gates actions under +//! backpressure, so a slow store stalls the ExEx, and reth behind it, for as long as the engine +//! waits. These cases feed the engine over a store whose writes outlast its idle flush interval +//! or its persistence timeout and check that every call returns rather than hangs, that +//! backpressure holds and releases, that a failed save is retried, that a reorg or a revert under +//! a save in flight completes or times out as the persistence timeout says, and that a dead +//! persistence service or a panic on the engine thread reaches callers, gated ones included, as +//! `EngineDied` instead of a hang. + +#[path = "live/chain.rs"] +#[allow(dead_code, reason = "each test target uses part of this module")] +mod chain; +#[path = "live/simulated_reth.rs"] +#[allow(dead_code, reason = "each test target uses part of this module")] +mod simulated_reth; +#[path = "live/store_hooks.rs"] +#[allow(dead_code, reason = "each test target uses part of this module")] +mod store_hooks; +#[path = "live/store_view.rs"] +#[allow(dead_code, reason = "each test target uses part of this module")] +mod store_view; +#[path = "live/waits.rs"] +#[allow(dead_code, reason = "each test target uses part of this module")] +mod waits; + +use alloy_consensus::BlockHeader; +use alloy_eips::NumHash; +use chain::*; +use eyre::WrapErr; +use reth_chain_state::ExecutedBlock; +use reth_ethereum_primitives::Block; +use reth_optimism_trie::{ + MdbxProofsStorage, + engine::{EngineConfig, EngineError, EngineHandle, ReorgBlock}, +}; +use rstest::rstest; +use simulated_reth::{ + DEFAULT_PERSISTENCE_THRESHOLD, RethPersistence, SimulatedReth, fixed_signer, + index_executed_block, reorg_block, +}; +use std::{ + sync::Arc, + time::{Duration, Instant}, +}; +use store_hooks::{HeldWrite, StoreWithHooks}; +use store_view::{BlockView, ProofTarget, proof_window}; +use waits::{PendingCall, wait_until}; + +/// The engine's idle flush interval and persistence timeout, lowered from their defaults so +/// that a save slower than either fits the time budget. +const IDLE_FLUSH_INTERVAL: Duration = Duration::from_millis(100); +const PERSISTENCE_TIMEOUT: Duration = Duration::from_millis(400); + +/// A save slower than the idle flush interval, within the persistence timeout. +const SAVE_OVER_IDLE_FLUSH: Duration = Duration::from_millis(200); + +/// A save slower than the persistence timeout. +const SAVE_OVER_PERSISTENCE_TIMEOUT: Duration = Duration::from_millis(800); + +/// How long a call or a wait may take before it counts as hung; the saves in its way take at +/// most a few seconds. +const CALL_TIMEOUT: Duration = Duration::from_secs(10); + +/// How long a reply that must come at once may take. +const PROMPT_REPLY_TIMEOUT: Duration = Duration::from_secs(2); + +/// How long a call that must stay blocked is watched. +const BLOCKED_PROBE_DURATION: Duration = Duration::from_millis(300); + +/// How reth leaves the chain the engine holds. +#[derive(Clone, Copy, Debug)] +enum Unwind { + /// reth replaces its two newest blocks. + Reorg, + /// reth drops its newest block. + Revert, +} + +/// A gated engine call: the hold on the save it waits for, and the call itself. +type GatedCall = (HeldWrite, PendingCall>); + +/// reth, the engine and the store it runs over, with every engine call bounded. +struct Node { + cadence: FlushCadence, + engine: EngineHandle, + reth: SimulatedReth, + serial: u64, + store: StoreWithHooks>, + targets: Vec, +} + +impl Node { + /// A node whose store opens every write transaction `save_delay` late. + fn new(cadence: FlushCadence, save_delay: Duration) -> eyre::Result { + let key_pair = fixed_signer(); + let sender = public_key_to_address(key_pair.public_key()); + let chain_spec = chain_spec_with_address(sender); + let persistence = RethPersistence::threshold(DEFAULT_PERSISTENCE_THRESHOLD); + let reth = SimulatedReth::new(chain_spec, key_pair, persistence)?; + let store = StoreWithHooks::new(create_mdbx_proofs_storage()); + initialize_storage(store.clone(), &reth.factory, reth.genesis())?; + store.delay_writes(save_delay); + let config = EngineConfig { + backpressure_threshold: cadence.backpressure, + idle_flush_interval: IDLE_FLUSH_INTERVAL, + persistence_threshold: cadence.persistence, + persistence_timeout: PERSISTENCE_TIMEOUT, + }; + let engine = reth.spawn_engine_with_config(store.clone(), config, WHOLE_WINDOW); + let targets = proof_targets_for_transfers(sender); + Ok(Self { cadence, engine, reth, serial: 0, store, targets }) + } + + fn start_call( + &self, + call_name: &'static str, + call: impl FnOnce(EngineHandle) -> T + Send + 'static, + ) -> PendingCall { + PendingCall::spawn(self.engine.clone(), call_name, call) + } + + /// Runs `call` against the engine, bounded by `bound`. + fn call( + &self, + call_name: &'static str, + bound: Duration, + call: impl FnOnce(EngineHandle) -> T + Send + 'static, + ) -> eyre::Result<(T, Duration)> { + Ok(self.start_call(call_name, call).wait(bound)?) + } + + const fn next_serial(&mut self) -> u64 { + self.serial += 1; + self.serial + } + + /// Commits the next block to reth. + fn next_block(&mut self) -> eyre::Result { + let serial = self.next_serial(); + self.reth.commit(&serial_transfer_block(serial)) + } + + /// Commits the next block to reth and hands it to the engine, which must take it. + fn index_next_block(&mut self) -> eyre::Result<()> { + let block = self.next_block()?; + let number = block.recovered_block().number(); + let (result, _) = self.call("index_block", CALL_TIMEOUT, move |engine| { + index_executed_block(&engine, &block) + })?; + result.map_err(|err| eyre::eyre!("the engine refused block {number}: {err}")) + } + + /// Commits and indexes `count` blocks back to back. + fn feed(&mut self, count: u64) -> eyre::Result<()> { + for _ in 0..count { + self.index_next_block()?; + } + Ok(()) + } + + /// Feeds the blocks that bring an empty buffer to the persistence threshold, by which a save + /// has started: at the threshold, or at the idle flush when a feed gap outlasted it. + fn fill_to_persistence_threshold(&mut self) -> eyre::Result<()> { + self.feed(self.cadence.persistence) + } + + /// Starts a save and returns once the store has it, so that its delay runs from here. + fn start_save(&mut self) -> eyre::Result<()> { + let hold = self.store.hold_next_write(); + self.fill_to_persistence_threshold()?; + hold.wait_until_entered(); + hold.release(); + Ok(()) + } + + fn tip(&self) -> eyre::Result { + Ok(self.call("tip", CALL_TIMEOUT, |engine| engine.tip())?.0?) + } + + fn store_latest(&self) -> eyre::Result { + Ok(proof_window(self.store.inner())?.latest) + } + + /// Waits until the store holds reth's head, which the engine's saves and then its idle flush + /// bring about on their own once nothing more is fed. Polls the store rather than the engine: + /// an engine action would postpone the idle flush this waits for. + fn settle(&self) -> eyre::Result<()> { + let head = self.reth.head(); + let mut latest = None; + let settled = wait_until("the store reaching reth's head", CALL_TIMEOUT, || { + latest = Some(self.store_latest()?); + Ok(latest == Some(head)) + }); + settled.wrap_err_with(|| { + format!("the store's latest block is {latest:?}, reth's head {head:?}") + }) + } + + /// The engine's buffer, what its tip holds above the store's latest block, must not exceed + /// the backpressure threshold: at the threshold the engine takes nothing in until the save + /// in flight completes. + fn ensure_buffer_within_backpressure(&self) -> eyre::Result<()> { + let tip = self.tip()?; + let latest = self.store_latest()?; + let buffered = tip.number.checked_sub(latest.number).ok_or_else(|| { + eyre::eyre!("the store's latest block {latest:?} is above the engine's tip {tip:?}") + })?; + eyre::ensure!( + buffered <= self.cadence.backpressure, + "the engine holds {buffered} blocks above the store's latest block {}, more than \ + its backpressure threshold {}", + latest.number, + self.cadence.backpressure + ); + Ok(()) + } + + /// Holds the next save inside the store, fills the buffer to the backpressure threshold and + /// issues one more `index_block`, which must stay blocked while the save is held. The node + /// must hold nothing buffered before. + fn gate_a_call(&mut self) -> eyre::Result { + let hold = self.store.hold_next_write(); + self.fill_to_persistence_threshold()?; + hold.wait_until_entered(); + self.feed(self.cadence.backpressure - self.cadence.persistence)?; + let block = self.next_block()?; + let gated = self.start_call("index_block under backpressure", move |engine| { + index_executed_block(&engine, &block) + }); + gated.ensure_blocked_for(BLOCKED_PROBE_DURATION)?; + Ok((hold, gated)) + } + + /// A call gated by backpressure must return once the held save is released. + fn probe_backpressure(&mut self) -> eyre::Result<()> { + self.settle()?; + let (hold, gated) = self.gate_a_call()?; + hold.release(); + let (result, _) = gated.wait(CALL_TIMEOUT)?; + result.map_err(|err| eyre::eyre!("the engine refused the block it had gated: {err}")) + } + + /// Fails the next save: the engine must take the next block meanwhile, retry the save and + /// catch the store up. + fn fail_a_save(&mut self) -> eyre::Result<()> { + self.settle()?; + self.store.fail_next_write(); + self.fill_to_persistence_threshold()?; + self.index_next_block()?; + self.settle() + } + + /// Blocks fed back to back, the buffer held within the backpressure threshold, a gated call + /// released, a failed save retried, and the store at reth's head after all of it. + fn feed_under_backpressure_and_a_failed_save(&mut self) -> eyre::Result<()> { + self.feed(3 * self.cadence.backpressure)?; + self.ensure_buffer_within_backpressure()?; + self.probe_backpressure()?; + self.fail_a_save()?; + self.verify_store_at_head() + } + + /// Starts a save and, while it is in flight, hands the engine `unwind` of the chain whose + /// newest block the save holds. + fn unwind_under_a_save( + &mut self, + unwind: Unwind, + ) -> eyre::Result<(Result<(), EngineError>, Duration)> { + self.settle()?; + self.start_save()?; + match unwind { + Unwind::Reorg => self.reorg(), + Unwind::Revert => self.revert(), + } + } + + /// Replaces reth's two newest blocks and hands the reorg to the engine. + fn reorg(&mut self) -> eyre::Result<(Result<(), EngineError>, Duration)> { + let first = self.reth.head().number - 1; + let serials = [self.next_serial(), self.next_serial()]; + let specs: Vec = serials.into_iter().map(serial_transfer_block).collect(); + let new = self.reth.reorg_with(first, &specs)?; + let blocks: Vec> = + new.iter().map(|block| reorg_block(block, false)).collect(); + self.call("reorg", CALL_TIMEOUT, move |engine| engine.reorg(blocks)) + } + + /// Drops reth's newest block and hands the revert to the engine. + fn revert(&mut self) -> eyre::Result<(Result<(), EngineError>, Duration)> { + let head = self.reth.head().number; + let to = self.reth.block(head).block_with_parent(); + self.reth.revert_to(head - 1); + self.call("unwind", CALL_TIMEOUT, move |engine| engine.unwind(to)) + } + + /// The store's latest block must be reth's head, serving proofs that hash to its state root. + fn verify_store_at_head(&self) -> eyre::Result<()> { + let head = self.reth.head(); + let latest = self.store_latest()?; + eyre::ensure!( + latest == head, + "the store's latest block {latest:?} is not reth's head {head:?}" + ); + BlockView::read(self.store.inner(), head.number, &self.targets)? + .verify_proofs(head.number, self.reth.state_root(head.number)) + } +} + +/// Blocks fed back to back over a store slower than the engine's idle flush interval: every call +/// returns, the buffer never outgrows the backpressure threshold, a call gated by backpressure +/// stays blocked until the save in flight completes, a failed save is retried, and a reorg or a +/// revert issued under a save in flight completes once the save has, after which the store +/// serves reth's head. +#[rstest] +fn engine_stays_live_over_a_slow_store( + #[values(FlushCadence::EVERY_BLOCK, FlushCadence::EVERY_FIFTH_BLOCK)] cadence: FlushCadence, + #[values(Unwind::Reorg, Unwind::Revert)] unwind: Unwind, +) -> eyre::Result<()> { + let mut node = Node::new(cadence, SAVE_OVER_IDLE_FLUSH)?; + node.feed_under_backpressure_and_a_failed_save()?; + let (result, took) = node.unwind_under_a_save(unwind)?; + result.map_err(|err| eyre::eyre!("the {unwind:?} under a save in flight failed: {err}"))?; + eyre::ensure!( + took >= SAVE_OVER_IDLE_FLUSH, + "the {unwind:?} returned after {took:?}, before the save in flight could complete" + ); + node.settle()?; + node.verify_store_at_head() +} + +/// With a save slower than the persistence timeout in flight, the engine gives up waiting for it +/// before unwinding, and gives up on its own unwind when the store, still busy with the save, +/// does not answer within a second timeout: the reorg or revert ends with `PersistenceTimeout` +/// after at least one timeout, and the engine still answers. The engine then holds a chain the +/// store no longer does, so nothing further is fed. +#[rstest] +fn an_unwind_under_a_save_slower_than_the_persistence_timeout_times_out( + #[values(FlushCadence::EVERY_BLOCK, FlushCadence::EVERY_FIFTH_BLOCK)] cadence: FlushCadence, + #[values(Unwind::Reorg, Unwind::Revert)] unwind: Unwind, +) -> eyre::Result<()> { + let mut node = Node::new(cadence, SAVE_OVER_PERSISTENCE_TIMEOUT)?; + node.feed_under_backpressure_and_a_failed_save()?; + let (result, took) = node.unwind_under_a_save(unwind)?; + eyre::ensure!( + matches!(result, Err(EngineError::PersistenceTimeout)), + "the {unwind:?} under a save slower than the persistence timeout ended with {result:?} \ + instead of a persistence timeout" + ); + eyre::ensure!( + took >= PERSISTENCE_TIMEOUT, + "the {unwind:?} returned after {took:?}, before the persistence timeout" + ); + node.tip()?; + Ok(()) +} + +/// Once the persistence service's thread has died, nothing can be saved any more: the engine +/// must stop and refuse every further call with `EngineDied`, which the ExEx turns into an +/// error, rather than keep taking blocks into a buffer it can never flush. +#[rstest] +fn a_dead_persistence_service_reaches_callers_as_engine_died( + #[values(FlushCadence::EVERY_BLOCK, FlushCadence::EVERY_FIFTH_BLOCK)] cadence: FlushCadence, +) -> eyre::Result<()> { + let mut node = Node::new(cadence, Duration::ZERO)?; + node.store.panic_in_next_write(); + + // The save that kills the service starts by the persistence threshold at the latest. + let deadline = Instant::now() + CALL_TIMEOUT; + loop { + let block = node.next_block()?; + let (result, _) = node.call("index_block", CALL_TIMEOUT, move |engine| { + index_executed_block(&engine, &block) + })?; + match result { + Err(EngineError::EngineDied) => return Ok(()), + Ok(()) => eyre::ensure!( + Instant::now() < deadline, + "the engine kept taking blocks for {CALL_TIMEOUT:?} after the panic was armed in \ + its persistence service" + ), + Err(err) => { + return Err(eyre::eyre!( + "the engine refused a block with {err} instead of EngineDied" + )); + } + } + } +} + +/// A caller gated by backpressure waits for the save in flight; when that save kills the +/// persistence service instead of completing, the caller must get `EngineDied` at once rather +/// than wait forever on an action the engine will never take. +#[rstest] +fn a_dead_persistence_service_reaches_a_gated_caller_as_engine_died( + #[values(FlushCadence::EVERY_BLOCK, FlushCadence::EVERY_FIFTH_BLOCK)] cadence: FlushCadence, +) -> eyre::Result<()> { + let mut node = Node::new(cadence, Duration::ZERO)?; + let (hold, gated) = node.gate_a_call()?; + hold.release_into_panic(); + let (result, _) = gated.wait(PROMPT_REPLY_TIMEOUT)?; + eyre::ensure!( + matches!(result, Err(EngineError::EngineDied)), + "the gated caller got {result:?} instead of EngineDied once the save killed the \ + persistence service" + ); + Ok(()) +} + +/// A panic on the engine thread drops the reply of the call it was serving, and must reach every +/// other caller as `EngineDied` at once, even while the persistence thread is still inside a +/// save: no caller may have to wait for the store to learn that the engine is gone. +#[rstest] +fn an_engine_panic_reaches_waiting_callers_as_engine_died( + #[values(FlushCadence::EVERY_BLOCK, FlushCadence::EVERY_FIFTH_BLOCK)] cadence: FlushCadence, +) -> eyre::Result<()> { + let mut node = Node::new(cadence, Duration::ZERO)?; + let hold = node.store.hold_next_write(); + node.fill_to_persistence_threshold()?; + hold.wait_until_entered(); + + node.store.panic_in_next_read(); + let block = node.next_block()?; + let (result, _) = node.call("execute_block", CALL_TIMEOUT, move |engine| { + engine.execute_block(block.recovered_block()) + })?; + eyre::ensure!( + matches!(result, Err(EngineError::EngineDied)), + "the call the engine panicked in ended with {result:?} instead of EngineDied" + ); + eyre::ensure!( + !node.store.is_next_read_armed(), + "the engine never opened the read view the panic was armed in" + ); + + let (result, _) = + node.call("tip after the engine died", PROMPT_REPLY_TIMEOUT, |engine| engine.tip())?; + eyre::ensure!( + matches!(result, Err(EngineError::EngineDied)), + "a caller after the engine's death got {result:?} instead of EngineDied" + ); + hold.release(); + Ok(()) +} diff --git a/rust/op-reth/crates/trie/tests/engine_model.rs b/rust/op-reth/crates/trie/tests/engine_model.rs index e80e75b9ccd..bc9dee5ed2c 100644 --- a/rust/op-reth/crates/trie/tests/engine_model.rs +++ b/rust/op-reth/crates/trie/tests/engine_model.rs @@ -10,8 +10,6 @@ mod actions; #[path = "live/chain.rs"] #[allow(dead_code, reason = "each test target uses part of this module")] mod chain; -#[path = "engine_model/faults.rs"] -mod faults; #[path = "engine_model/harness.rs"] mod harness; #[path = "engine_model/invariants.rs"] @@ -21,9 +19,15 @@ mod model; #[path = "live/simulated_reth.rs"] #[allow(dead_code, reason = "each test target uses part of this module")] mod simulated_reth; +#[path = "live/store_hooks.rs"] +#[allow(dead_code, reason = "each test target uses part of this module")] +mod store_hooks; #[path = "live/store_view.rs"] #[allow(dead_code, reason = "each test target uses part of this module")] mod store_view; +#[path = "live/waits.rs"] +#[allow(dead_code, reason = "each test target uses part of this module")] +mod waits; use actions::{Action, Actions, Config, Rebuild}; use chain::*; diff --git a/rust/op-reth/crates/trie/tests/engine_model/faults.rs b/rust/op-reth/crates/trie/tests/engine_model/faults.rs deleted file mode 100644 index 783a0069815..00000000000 --- a/rust/op-reth/crates/trie/tests/engine_model/faults.rs +++ /dev/null @@ -1,71 +0,0 @@ -//! A proofs store whose next write transaction can be made to fail, the way a full disk or a -//! database error fails a save. - -use reth_db::DatabaseError; -use reth_optimism_trie::{OpProofsStorageResult, OpProofsStore}; -use std::{ - fmt, - sync::{ - Arc, - atomic::{AtomicBool, Ordering}, - }, -}; - -#[derive(Clone)] -pub(crate) struct StoreWithSaveFaults { - fail_next_write: Arc, - inner: S, -} - -impl StoreWithSaveFaults { - pub(crate) fn new(inner: S) -> Self { - Self { fail_next_write: Arc::default(), inner } - } - - pub(crate) fn fail_next_write(&self) { - self.fail_next_write.store(true, Ordering::SeqCst); - } - - /// Withdraws a failure that no write has consumed yet; true if there was one. - pub(crate) fn disarm(&self) -> bool { - self.fail_next_write.swap(false, Ordering::SeqCst) - } -} - -impl fmt::Debug for StoreWithSaveFaults { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("StoreWithSaveFaults").field("inner", &self.inner).finish_non_exhaustive() - } -} - -impl OpProofsStore for StoreWithSaveFaults { - type ProviderRO<'a> - = S::ProviderRO<'a> - where - Self: 'a; - - type ProviderRw<'a> - = S::ProviderRw<'a> - where - Self: 'a; - - type Initializer<'a> - = S::Initializer<'a> - where - Self: 'a; - - fn provider_ro<'a>(&'a self) -> OpProofsStorageResult> { - self.inner.provider_ro() - } - - fn provider_rw<'a>(&'a self) -> OpProofsStorageResult> { - if self.fail_next_write.swap(false, Ordering::SeqCst) { - return Err(DatabaseError::Other("injected write failure".to_owned()).into()); - } - self.inner.provider_rw() - } - - fn initialization_provider<'a>(&'a self) -> OpProofsStorageResult> { - self.inner.initialization_provider() - } -} diff --git a/rust/op-reth/crates/trie/tests/engine_model/harness.rs b/rust/op-reth/crates/trie/tests/engine_model/harness.rs index c56e73baeda..8066036513f 100644 --- a/rust/op-reth/crates/trie/tests/engine_model/harness.rs +++ b/rust/op-reth/crates/trie/tests/engine_model/harness.rs @@ -2,53 +2,35 @@ //! after every step are in `invariants.rs`. use super::{ - BlockSpec, TxSpec, + BlockSpec, actions::{Action, Config, Rebuild}, - chain_spec_with_address, - faults::StoreWithSaveFaults, - initialize_storage, + chain_spec_with_address, initialize_storage, model::{Model, ModelBlock, Reply}, - public_key_to_address, + proof_targets_for_transfers, public_key_to_address, serial_transfer_block, simulated_reth::{ SimulatedReth, UNRELATED_PARENT, fixed_signer, index_executed_block, reorg_block, reorg_block_without_data, }, - store_view::ProofTarget, + store_hooks::StoreWithHooks, + store_view::{ProofTarget, proof_window}, + waits::{CallError, PendingCall}, }; use alloy_consensus::BlockHeader; use alloy_eips::{NumHash, eip1898::BlockWithParent}; -use alloy_primitives::{Address, B256, U256}; +use alloy_primitives::B256; use reth_chain_state::ExecutedBlock; use reth_ethereum_primitives::Block; use reth_optimism_trie::{ - MdbxProofsStorage, OpProofsProviderRO, OpProofsStorageError, OpProofsStore, ProofWindowRange, + MdbxProofsStorage, OpProofsStorageError, ProofWindowRange, engine::{EngineError, EngineHandle, ReorgBlock}, }; -use std::{ - mem, - ops::ControlFlow, - sync::{Arc, mpsc}, - thread, - time::Duration, -}; +use std::{mem, ops::ControlFlow, sync::Arc, time::Duration}; use tempfile::TempDir; /// How long one engine call, or the engine settling after a step, may take before the case /// fails instead of hanging. pub(super) const STEP_TIMEOUT: Duration = Duration::from_secs(30); -/// Recipients the chain pays in turn. -const RECIPIENTS: u64 = 4; - -const fn recipient(serial: u64) -> Address { - Address::repeat_byte(0x40 + (serial % RECIPIENTS) as u8) -} - -/// The block with `serial`: `serial` wei to its recipient, so that no two blocks are alike. -fn block_spec(serial: u64) -> BlockSpec { - BlockSpec::new(vec![TxSpec::transfer(recipient(serial), U256::from(serial))]) -} - fn model_block(block: &ExecutedBlock, serial: u64, persisted_before_commit: u64) -> ModelBlock { let recovered = block.recovered_block(); ModelBlock { @@ -64,7 +46,7 @@ fn model_block(block: &ExecutedBlock, serial: u64, persisted_before_commit: u64) /// store is gone before the store is reopened. struct ProofsNode { engine: EngineHandle, - store: StoreWithSaveFaults>, + store: StoreWithHooks>, } pub(super) struct Harness { @@ -108,12 +90,10 @@ impl Harness { SimulatedReth::new(chain_spec, key_pair, config.persistence)? }; let store_dir = TempDir::new()?; - let store = StoreWithSaveFaults::new(Arc::new(MdbxProofsStorage::new(store_dir.path())?)); + let store = StoreWithHooks::new(Arc::new(MdbxProofsStorage::new(store_dir.path())?)); initialize_storage(store.clone(), &reth.factory, reth.genesis())?; let engine = Self::spawn_engine(&reth, &store, config); - let targets = ProofTarget::accounts( - [sender, Address::ZERO].into_iter().chain((0..RECIPIENTS).map(recipient)), - ); + let targets = proof_targets_for_transfers(sender); Ok(Self { config, engine_hung: false, @@ -130,7 +110,7 @@ impl Harness { fn spawn_engine( reth: &SimulatedReth, - store: &StoreWithSaveFaults>, + store: &StoreWithHooks>, config: Config, ) -> EngineHandle { reth.spawn_engine_with_window( @@ -149,7 +129,7 @@ impl Harness { &self.node().engine } - pub(super) const fn store(&self) -> &StoreWithSaveFaults> { + pub(super) const fn store(&self) -> &StoreWithHooks> { &self.node().store } @@ -214,28 +194,21 @@ impl Harness { /// within [`STEP_TIMEOUT`], which a stalled engine would otherwise turn into a hang. pub(super) fn call( &mut self, - what: &str, + call_name: &'static str, call: impl FnOnce(EngineHandle) -> T + Send + 'static, ) -> eyre::Result { - let engine = self.engine().clone(); - let (reply_tx, reply_rx) = mpsc::channel(); - thread::spawn(move || { - let _ = reply_tx.send(call(engine)); - }); - match reply_rx.recv_timeout(STEP_TIMEOUT) { - Ok(result) => Ok(result), - Err(mpsc::RecvTimeoutError::Timeout) => { + match PendingCall::spawn(self.engine().clone(), call_name, call).wait(STEP_TIMEOUT) { + Ok((result, _)) => Ok(result), + Err(err @ CallError::Hung { .. }) => { self.engine_hung = true; - Err(eyre::eyre!("the engine did not reply to {what} within {STEP_TIMEOUT:?}")) - } - Err(mpsc::RecvTimeoutError::Disconnected) => { - Err(eyre::eyre!("the engine call {what} panicked")) + Err(err.into()) } + Err(err) => Err(err.into()), } } pub(super) fn window(&self) -> eyre::Result { - Ok(self.store().provider_ro()?.get_proof_window()?) + proof_window(self.store()) } const fn next_serial(&mut self) -> u64 { @@ -247,7 +220,7 @@ impl Harness { fn commit_block(&mut self) -> eyre::Result { let serial = self.next_serial(); let persisted_before_commit = self.reth.persisted().number; - let block = self.reth.commit(&block_spec(serial))?; + let block = self.reth.commit(&serial_transfer_block(serial))?; self.model.push(model_block(&block, serial, persisted_before_commit)); Ok(block) } @@ -255,7 +228,7 @@ impl Harness { /// Replaces reth's chain above `fork` with `length` new blocks, in reth and in the model. fn fork_chain(&mut self, fork: u64, length: u64) -> eyre::Result> { let serials: Vec = (0..length).map(|_| self.next_serial()).collect(); - let specs: Vec = serials.iter().copied().map(block_spec).collect(); + let specs: Vec = serials.iter().copied().map(serial_transfer_block).collect(); // Every block of the fork becomes canonical with reth persisted at least this far. let persisted_before_the_fork = self.reth.persisted().number; let new = self.reth.reorg_with(fork + 1, &specs)?; @@ -462,7 +435,7 @@ impl Harness { } self.flush()?; self.flush()?; - if self.store().disarm() { + if self.store().disarm_next_write() { self.log(" no save started, the injected failure is withdrawn".to_owned()); } Ok(()) @@ -543,8 +516,7 @@ impl Harness { }); self.node = None; self.reth.crash_and_restart()?; - let store = - StoreWithSaveFaults::new(Arc::new(MdbxProofsStorage::new(self.store_dir.path())?)); + let store = StoreWithHooks::new(Arc::new(MdbxProofsStorage::new(self.store_dir.path())?)); let engine = Self::spawn_engine(&self.reth, &store, self.config); self.node = Some(ProofsNode { engine, store }); let tip = self.call("tip", |engine| engine.tip())??; @@ -558,7 +530,7 @@ impl Harness { match rebuild { Rebuild::Same => { for serial in lost { - let block = self.reth.commit(&block_spec(serial))?; + let block = self.reth.commit(&serial_transfer_block(serial))?; let number = block.recovered_block().number(); eyre::ensure!( block.recovered_block().hash() == self.model.block(number).hash, diff --git a/rust/op-reth/crates/trie/tests/live.rs b/rust/op-reth/crates/trie/tests/live.rs index 441e95ed9e8..3067af60723 100644 --- a/rust/op-reth/crates/trie/tests/live.rs +++ b/rust/op-reth/crates/trie/tests/live.rs @@ -43,6 +43,7 @@ macro_rules! over_backend { #[path = "live/catch_up_under_pruning.rs"] mod catch_up_under_pruning; #[path = "live/chain.rs"] +#[allow(dead_code, reason = "each test target uses part of this module")] mod chain; #[path = "live/contracts.rs"] mod contracts; @@ -55,6 +56,9 @@ mod reorg_revert_edges; #[path = "live/simulated_reth.rs"] #[allow(dead_code, reason = "each test target uses part of this module")] mod simulated_reth; +#[path = "live/store_hooks.rs"] +#[allow(dead_code, reason = "each test target uses part of this module")] +mod store_hooks; #[path = "live/store_view.rs"] mod store_view; diff --git a/rust/op-reth/crates/trie/tests/live/chain.rs b/rust/op-reth/crates/trie/tests/live/chain.rs index 22f2657882f..34439de5f33 100644 --- a/rust/op-reth/crates/trie/tests/live/chain.rs +++ b/rust/op-reth/crates/trie/tests/live/chain.rs @@ -1,6 +1,7 @@ //! The test chain the live targets share: a funded signer, block and transaction specs, the //! block builder, the store constructors and the store's initialization at a block. +use super::store_view::ProofTarget; use alloy_consensus::{ Header, TxEip2930, constants::{EMPTY_WITHDRAWALS, ETH_TO_WEI}, @@ -131,6 +132,23 @@ impl BlockSpec { } } +/// Recipients the transfer chain pays in turn. +pub(super) const RECIPIENTS: u64 = 4; + +pub(super) const fn recipient(serial: u64) -> Address { + Address::repeat_byte(0x40 + (serial % RECIPIENTS) as u8) +} + +/// The block with `serial`: `serial` wei to its recipient, so that no two blocks are alike. +pub(super) fn serial_transfer_block(serial: u64) -> BlockSpec { + BlockSpec::new(vec![TxSpec::transfer(recipient(serial), U256::from(serial))]) +} + +/// Proof targets for the transfer chain: the sender, the zero address and every recipient. +pub(super) fn proof_targets_for_transfers(sender: Address) -> Vec { + ProofTarget::accounts([sender, Address::ZERO].into_iter().chain((0..RECIPIENTS).map(recipient))) +} + /// A mainnet-shaped chain spec whose genesis allocation is `alloc`, with the hardforks still to /// be chosen. pub(super) fn chain_spec_builder( diff --git a/rust/op-reth/crates/trie/tests/live/parent_states.rs b/rust/op-reth/crates/trie/tests/live/parent_states.rs index 48dff707207..25172dfc966 100644 --- a/rust/op-reth/crates/trie/tests/live/parent_states.rs +++ b/rust/op-reth/crates/trie/tests/live/parent_states.rs @@ -15,6 +15,7 @@ use super::{ }, create_mdbx_proofs_storage, initialize_storage, public_key_to_address, simulated_reth::{Location, RethPersistence, SimulatedReth, fixed_signer}, + store_hooks::StoreWithHooks, }; use alloy_consensus::BlockHeader; use alloy_eips::BlockNumHash; @@ -22,15 +23,15 @@ use alloy_primitives::{Address, Bytes, U256}; use parking_lot::Mutex; use reth_ethereum_primitives::Block; use reth_optimism_trie::{ - MdbxProofsStorage, OpProofsProviderRO, OpProofsStorageResult, OpProofsStore, - engine::EngineError, provider::OpProofsStateProviderRef, + MdbxProofsStorage, OpProofsProviderRO, OpProofsStore, engine::EngineError, + provider::OpProofsStateProviderRef, }; use reth_primitives_traits::RecoveredBlock; use reth_provider::{StateProvider, StateRootProvider, noop::NoopProvider}; use reth_trie_common::HashedPostState; use rstest::rstest; use serial_test::serial; -use std::{fmt, sync::Arc}; +use std::sync::Arc; /// Covers the contract's deployment and either of its calls. const CONTRACT_GAS_LIMIT: u64 = 100_000; @@ -112,73 +113,6 @@ enum Outcome { StateRootMismatch, } -type ReadHook = Box; - -/// A proofs store that runs an action on reth right before the engine opens its next read view. -/// -/// The engine's execute task snapshots reth's parent state and then opens the proofs read view it -/// executes against, so an action armed here lands between the two. The engine also opens a read -/// view to find its tip whenever its block buffer is empty; the cases keep the preceding blocks -/// buffered so that read never happens while an action is armed. -#[derive(Clone)] -struct StoreWithReadHook { - inner: S, - on_next_read: Arc>>, -} - -impl StoreWithReadHook { - fn new(inner: S) -> Self { - Self { inner, on_next_read: Arc::default() } - } - - fn on_next_read(&self, action: impl FnOnce() + Send + 'static) { - *self.on_next_read.lock() = Some(Box::new(action)); - } - - fn is_armed(&self) -> bool { - self.on_next_read.lock().is_some() - } -} - -impl fmt::Debug for StoreWithReadHook { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("StoreWithReadHook").field("inner", &self.inner).finish_non_exhaustive() - } -} - -impl OpProofsStore for StoreWithReadHook { - type ProviderRO<'a> - = S::ProviderRO<'a> - where - Self: 'a; - - type ProviderRw<'a> - = S::ProviderRw<'a> - where - Self: 'a; - - type Initializer<'a> - = S::Initializer<'a> - where - Self: 'a; - - fn provider_ro<'a>(&'a self) -> OpProofsStorageResult> { - let action = self.on_next_read.lock().take(); - if let Some(action) = action { - action(); - } - self.inner.provider_ro() - } - - fn provider_rw<'a>(&'a self) -> OpProofsStorageResult> { - self.inner.provider_rw() - } - - fn initialization_provider<'a>(&'a self) -> OpProofsStorageResult> { - self.inner.initialization_provider() - } -} - /// The chain reth holds before the engine runs: `leading` empty blocks, the deployment, the call /// and the `BLOCKHASH` read, with the indices of the deployment and of the block under test. struct ProofChain { @@ -280,9 +214,11 @@ fn arrange_parent_state( Ok(()) } -/// Arms the reth transitions that happen while the engine executes the block. +/// Arms the reth transitions that happen while the engine executes the block: the engine's +/// execute task snapshots reth's parent state and then opens the proofs read view it executes +/// against, so a hook on the next read lands between the two. fn arm_in_flight_transition( - store: &StoreWithReadHook>, + store: &StoreWithHooks>, reth: &Arc>, parent_state: ParentState, chain: &ProofChain, @@ -291,13 +227,15 @@ fn arm_in_flight_transition( let deploy_number = chain.deploy().number(); let reth = Arc::clone(reth); match parent_state { - ParentState::EvictedWhileExecuting => store.on_next_read(move || { + ParentState::EvictedWhileExecuting => store.before_next_read(move || { reth.lock() .commit_fillers_until(|reth| reth.locate(parent) == Location::PERSISTED_AND_EVICTED) .expect("reth persists and evicts the parent"); + Ok(()) }), - ParentState::ReorgedWhileExecuting => store.on_next_read(move || { + ParentState::ReorgedWhileExecuting => store.before_next_read(move || { reth.lock().reorg_from(deploy_number).expect("reth reorgs below the block"); + Ok(()) }), _ => {} } @@ -372,7 +310,7 @@ fn run_parent_state_case( )?; arrange_parent_state(&mut reth, parent_state, &chain)?; - let store_with_hook = StoreWithReadHook::new(store.clone()); + let hooked_store = StoreWithHooks::new(store.clone()); // The engine writes its buffer to the store once it holds this many blocks, and `flush` only // waits for that write, so the threshold is the number of blocks the case stores. It also // keeps the preceding blocks buffered while the block under test executes, which the read @@ -381,16 +319,16 @@ fn run_parent_state_case( Outcome::Stored => chain.preceding().len() + 1, Outcome::StateRootMismatch => chain.preceding().len(), } as u64; - let engine = reth.spawn_engine(store_with_hook.clone(), stored_blocks, stored_blocks + 1); + let engine = reth.spawn_engine(hooked_store.clone(), stored_blocks, stored_blocks + 1); let reth = Arc::new(Mutex::new(reth)); for block in chain.preceding() { engine.execute_block(block)?; } - arm_in_flight_transition(&store_with_hook, &reth, parent_state, &chain); + arm_in_flight_transition(&hooked_store, &reth, parent_state, &chain); let result = engine.execute_block(chain.under_test()); assert!( - !store_with_hook.is_armed(), + !hooked_store.is_next_read_armed(), "the engine never opened a proofs read view, so the in-flight transition never ran" ); engine.flush(); diff --git a/rust/op-reth/crates/trie/tests/live/reorg_revert_edges.rs b/rust/op-reth/crates/trie/tests/live/reorg_revert_edges.rs index 869b4ebedfe..e2b33ad3864 100644 --- a/rust/op-reth/crates/trie/tests/live/reorg_revert_edges.rs +++ b/rust/op-reth/crates/trie/tests/live/reorg_revert_edges.rs @@ -15,34 +15,24 @@ use super::{ RethPersistence, SimulatedReth, UNRELATED_PARENT, fixed_signer, index_executed_block, reorg_block, reorg_block_without_data, }, - store_view::{BlockView, ProofTarget, assert_same_view}, + store_hooks::{HeldWrite, StoreWithHooks}, + store_view::{BlockView, ProofTarget, assert_same_view, proof_window}, }; use alloy_eips::{NumHash, eip1898::BlockWithParent}; use alloy_primitives::{Address, B256, U256}; -use parking_lot::Mutex; use reth_chain_state::ExecutedBlock; use reth_chainspec::ChainSpec; use reth_ethereum_primitives::Block; use reth_optimism_trie::{ - InMemoryProofsStorage, OpProofsProviderRO, OpProofsStorageError, OpProofsStorageResult, - OpProofsStore, ProofWindowRange, + InMemoryProofsStorage, OpProofsProviderRO, OpProofsStorageError, OpProofsStore, + ProofWindowRange, engine::{EngineError, EngineHandle, ReorgBlock}, }; use reth_trie_common::{HashedPostStateSorted, updates::TrieUpdatesSorted}; use rstest::rstest; use secp256k1::Keypair; use serial_test::serial; -use std::{ - collections::BTreeMap, - fmt, - ops::RangeInclusive, - sync::{ - Arc, - mpsc::{Receiver, SyncSender, sync_channel}, - }, - thread, - time::Duration, -}; +use std::{collections::BTreeMap, ops::RangeInclusive, sync::Arc, thread}; /// Blocks every case commits and settles before it acts. const CHAIN_LENGTH: u64 = 7; @@ -59,9 +49,6 @@ const RETH_PERSISTENCE_THRESHOLD: u64 = 2 * MAX_BLOCKS_PER_CASE; /// under either cadence. const PRUNED_WINDOW: u64 = 3; -/// How long a case waits for the engine's persistence thread to reach a held write. -const HELD_WRITE_TIMEOUT: Duration = Duration::from_secs(30); - const fn recipient(number: u64) -> Address { Address::repeat_byte(0x40 + number as u8) } @@ -167,95 +154,6 @@ impl UnwindTarget { } } -/// The store's side of a held write: signals the write and waits to be let through. -struct WriteBarrier { - entered: SyncSender<()>, - release: Receiver<()>, -} - -/// The case's side of a held write. -struct HeldWrite { - entered: Receiver<()>, - release: SyncSender<()>, -} - -impl HeldWrite { - /// Blocks until the engine's persistence thread is inside the held write. - fn wait_until_entered(&self) { - self.entered - .recv_timeout(HELD_WRITE_TIMEOUT) - .expect("the engine's persistence thread reaches the held write"); - } - - /// Lets the held write go on, or the next write through if none is held yet. - fn release(self) { - let _ = self.release.send(()); - } -} - -/// A proofs store whose next write transaction, once a hold is armed, blocks until the hold is -/// released, which keeps a save of the engine in flight for as long as a case needs. -#[derive(Clone)] -struct StoreWithWriteGate { - inner: S, - next_write: Arc>>, -} - -impl StoreWithWriteGate { - fn new(inner: S) -> Self { - Self { inner, next_write: Arc::default() } - } - - fn hold_next_write(&self) -> HeldWrite { - let (entered_tx, entered_rx) = sync_channel(1); - let (release_tx, release_rx) = sync_channel(1); - *self.next_write.lock() = Some(WriteBarrier { entered: entered_tx, release: release_rx }); - HeldWrite { entered: entered_rx, release: release_tx } - } -} - -impl fmt::Debug for StoreWithWriteGate { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("StoreWithWriteGate").field("inner", &self.inner).finish_non_exhaustive() - } -} - -impl OpProofsStore for StoreWithWriteGate { - type ProviderRO<'a> - = S::ProviderRO<'a> - where - Self: 'a; - - type ProviderRw<'a> - = S::ProviderRw<'a> - where - Self: 'a; - - type Initializer<'a> - = S::Initializer<'a> - where - Self: 'a; - - fn provider_ro<'a>(&'a self) -> OpProofsStorageResult> { - self.inner.provider_ro() - } - - fn provider_rw<'a>(&'a self) -> OpProofsStorageResult> { - // The lock must be released before blocking, or a case arming the next hold meanwhile - // deadlocks against this thread. - let barrier = self.next_write.lock().take(); - if let Some(barrier) = barrier { - let _ = barrier.entered.send(()); - let _ = barrier.release.recv(); - } - self.inner.provider_rw() - } - - fn initialization_provider<'a>(&'a self) -> OpProofsStorageResult> { - self.inner.initialization_provider() - } -} - /// What the store serves at one moment: its window and every persisted view in it. struct DiskSnapshot { views: BTreeMap, @@ -267,7 +165,7 @@ struct Scenario { cadence: FlushCadence, chain_spec: Arc, engine: EngineHandle, - gated_store: StoreWithWriteGate, + gated_store: StoreWithHooks, key_pair: Keypair, reth: SimulatedReth, store: S, @@ -289,7 +187,7 @@ impl Scenario { let reth = SimulatedReth::new(chain_spec.clone(), key_pair, persistence)?; let store = make_store(); initialize_storage(store.clone(), &reth.factory, reth.genesis())?; - let gated_store = StoreWithWriteGate::new(store.clone()); + let gated_store = StoreWithHooks::new(store.clone()); let engine = Self::spawn_engine(&reth, &gated_store, cadence, window); let mut scenario = Self { cadence, @@ -311,7 +209,7 @@ impl Scenario { fn spawn_engine( reth: &SimulatedReth, - gated_store: &StoreWithWriteGate, + gated_store: &StoreWithHooks, cadence: FlushCadence, window: u64, ) -> EngineHandle { @@ -381,7 +279,7 @@ impl Scenario { } fn proof_window(&self) -> eyre::Result { - Ok(self.store.provider_ro()?.get_proof_window()?) + proof_window(&self.store) } /// Arms a hold on the engine's next write, the save the following commits start. diff --git a/rust/op-reth/crates/trie/tests/live/simulated_reth.rs b/rust/op-reth/crates/trie/tests/live/simulated_reth.rs index 27d432aa483..cc36c39542a 100644 --- a/rust/op-reth/crates/trie/tests/live/simulated_reth.rs +++ b/rust/op-reth/crates/trie/tests/live/simulated_reth.rs @@ -14,7 +14,7 @@ use reth_evm::{ConfigureEvm, execute::Executor}; use reth_evm_ethereum::EthEvmConfig; use reth_optimism_trie::{ OpProofStoragePruner, OpProofsStore, - engine::{EngineError, EngineHandle, ReorgBlock}, + engine::{EngineConfig, EngineError, EngineHandle, ReorgBlock}, }; use reth_primitives_traits::{RecoveredBlock, SealedHeader}; use reth_provider::{ @@ -32,6 +32,9 @@ use std::sync::Arc; /// reth's default `--engine.memory-block-buffer-target`. const MEMORY_BLOCK_BUFFER_TARGET: u64 = 5; +/// reth's default `--engine.persistence-threshold`. +pub(super) const DEFAULT_PERSISTENCE_THRESHOLD: u64 = 7; + /// A parent hash no block here has. pub(super) const UNRELATED_PARENT: B256 = B256::repeat_byte(0xBA); @@ -281,17 +284,32 @@ impl SimulatedReth { backpressure_threshold: u64, window: u64, ) -> EngineHandle + where + S: OpProofsStore + Clone + 'static, + { + let config = + EngineConfig { backpressure_threshold, persistence_threshold, ..Default::default() }; + self.spawn_engine_with_config(store, config, window) + } + + /// Spawns a proofs engine over `store` with `config`, pruning the store down to `window` + /// blocks. + pub(super) fn spawn_engine_with_config( + &self, + store: S, + config: EngineConfig, + window: u64, + ) -> EngineHandle where S: OpProofsStore + Clone + 'static, { let pruner = OpProofStoragePruner::new(store.clone(), self.provider.clone(), window); - EngineHandle::spawn_with_thresholds( + EngineHandle::spawn_with_config( self.evm_config.clone(), self.provider.clone(), store, pruner, - persistence_threshold, - backpressure_threshold, + config, ) } diff --git a/rust/op-reth/crates/trie/tests/live/store_hooks.rs b/rust/op-reth/crates/trie/tests/live/store_hooks.rs new file mode 100644 index 00000000000..1511737a82e --- /dev/null +++ b/rust/op-reth/crates/trie/tests/live/store_hooks.rs @@ -0,0 +1,188 @@ +//! A proofs store whose transactions a case can delay, hold, fail or make panic: the faults a +//! slow or broken disk puts in the engine's way, injected where the engine opens a read or a +//! write transaction. + +use parking_lot::Mutex; +use reth_db::DatabaseError; +use reth_optimism_trie::{OpProofsStorageResult, OpProofsStore}; +use std::{ + fmt, + sync::{ + Arc, + mpsc::{Receiver, SyncSender, sync_channel}, + }, + thread, + time::Duration, +}; + +/// How long a case waits for the engine's persistence thread to reach a held write. +const HELD_WRITE_TIMEOUT: Duration = Duration::from_secs(30); + +/// Runs on the thread opening the transaction, before the inner store opens it; an error is +/// returned in the transaction's place. +type StoreHook = Box OpProofsStorageResult<()> + Send>; + +#[derive(Default)] +struct Hooks { + next_read: Option, + next_write: Option, + write_delay: Duration, +} + +/// How a held write goes on once released. +enum Release { + Panic, + Proceed, +} + +#[derive(Clone)] +pub(super) struct StoreWithHooks { + /// Taken under the lock and run after it is released: a hook may block, and a case arming + /// the next hook meanwhile would deadlock against the lock. + hooks: Arc>, + inner: S, +} + +impl StoreWithHooks { + pub(super) fn new(inner: S) -> Self { + Self { hooks: Arc::default(), inner } + } + + /// The wrapped store, for reads that must not consume an armed hook. + pub(super) const fn inner(&self) -> &S { + &self.inner + } + + /// Runs `hook` on the thread opening the next read transaction, right before it is opened. + /// Any read consumes it, a case's own reads through this wrapper included. + pub(super) fn before_next_read( + &self, + hook: impl FnOnce() -> OpProofsStorageResult<()> + Send + 'static, + ) { + self.hooks.lock().next_read = Some(Box::new(hook)); + } + + /// Runs `hook` on the thread opening the next write transaction, right before it is opened: + /// the engine's persistence thread, at its next save or unwind. + fn before_next_write(&self, hook: impl FnOnce() -> OpProofsStorageResult<()> + Send + 'static) { + self.hooks.lock().next_write = Some(Box::new(hook)); + } + + pub(super) fn is_next_read_armed(&self) -> bool { + self.hooks.lock().next_read.is_some() + } + + /// Withdraws a write hook no write has consumed yet; true if there was one. + pub(super) fn disarm_next_write(&self) -> bool { + self.hooks.lock().next_write.take().is_some() + } + + /// Every write transaction from now on opens `delay` late, as on a slow disk. + pub(super) fn delay_writes(&self, delay: Duration) { + self.hooks.lock().write_delay = delay; + } + + /// The next write transaction fails to open, as on a full disk. + pub(super) fn fail_next_write(&self) { + self.before_next_write(|| { + Err(DatabaseError::Other("injected write failure".to_owned()).into()) + }); + } + + /// The next write transaction blocks until the returned hold is released, which keeps a + /// save of the engine in flight for as long as a case needs. + pub(super) fn hold_next_write(&self) -> HeldWrite { + let (entered_tx, entered_rx) = sync_channel(1); + let (release_tx, release_rx) = sync_channel(1); + self.before_next_write(move || { + let _ = entered_tx.send(()); + match release_rx.recv() { + Ok(Release::Panic) => panic!("injected panic in a held write transaction"), + Ok(Release::Proceed) | Err(_) => Ok(()), + } + }); + HeldWrite { entered: entered_rx, release: release_tx } + } + + /// The thread opening the next read transaction panics, as a bug in the store would. + pub(super) fn panic_in_next_read(&self) { + self.before_next_read(|| panic!("injected panic in a read transaction")); + } + + /// The thread opening the next write transaction panics, as a bug in the store would. + pub(super) fn panic_in_next_write(&self) { + self.before_next_write(|| panic!("injected panic in a write transaction")); + } +} + +/// The case's side of a held write. Dropping it lets the write go on too. +pub(super) struct HeldWrite { + entered: Receiver<()>, + release: SyncSender, +} + +impl HeldWrite { + /// Blocks until the engine's persistence thread is inside the held write. + pub(super) fn wait_until_entered(&self) { + self.entered + .recv_timeout(HELD_WRITE_TIMEOUT) + .expect("the engine's persistence thread reaches the held write"); + } + + /// Lets the held write go on, or the next write through if none is held yet. + pub(super) fn release(self) { + let _ = self.release.send(Release::Proceed); + } + + /// Lets the held write go on into a panic, as a store that dies mid-write would. + pub(super) fn release_into_panic(self) { + let _ = self.release.send(Release::Panic); + } +} + +impl fmt::Debug for StoreWithHooks { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("StoreWithHooks").field("inner", &self.inner).finish_non_exhaustive() + } +} + +impl OpProofsStore for StoreWithHooks { + type ProviderRO<'a> + = S::ProviderRO<'a> + where + Self: 'a; + + type ProviderRw<'a> + = S::ProviderRw<'a> + where + Self: 'a; + + type Initializer<'a> + = S::Initializer<'a> + where + Self: 'a; + + fn provider_ro<'a>(&'a self) -> OpProofsStorageResult> { + let hook = self.hooks.lock().next_read.take(); + if let Some(hook) = hook { + hook()?; + } + self.inner.provider_ro() + } + + fn provider_rw<'a>(&'a self) -> OpProofsStorageResult> { + let (hook, delay) = { + let mut hooks = self.hooks.lock(); + (hooks.next_write.take(), hooks.write_delay) + }; + if let Some(hook) = hook { + hook()?; + } + thread::sleep(delay); + self.inner.provider_rw() + } + + fn initialization_provider<'a>(&'a self) -> OpProofsStorageResult> { + self.inner.initialization_provider() + } +} diff --git a/rust/op-reth/crates/trie/tests/live/store_view.rs b/rust/op-reth/crates/trie/tests/live/store_view.rs index 3124e498d3b..3a1ca0a01c0 100644 --- a/rust/op-reth/crates/trie/tests/live/store_view.rs +++ b/rust/op-reth/crates/trie/tests/live/store_view.rs @@ -2,7 +2,9 @@ //! and the comparison of two such views that names the first differing table and key. use alloy_primitives::{Address, B256, U256, keccak256}; -use reth_optimism_trie::{OpProofsProviderRO, OpProofsStore, proof::DatabaseProof}; +use reth_optimism_trie::{ + OpProofsProviderRO, OpProofsStore, ProofWindowRange, proof::DatabaseProof, +}; use reth_primitives_traits::Account; use reth_trie::{ hashed_cursor::HashedCursor, @@ -15,6 +17,11 @@ use std::{ fmt, }; +/// The store's window: its earliest and latest persisted blocks. +pub(super) fn proof_window(store: &S) -> eyre::Result { + Ok(store.provider_ro()?.get_proof_window()?) +} + /// An address whose proof, with these slots, every store must serve identically. pub(super) struct ProofTarget { pub(super) address: Address, diff --git a/rust/op-reth/crates/trie/tests/live/waits.rs b/rust/op-reth/crates/trie/tests/live/waits.rs new file mode 100644 index 00000000000..3ac0f6375bc --- /dev/null +++ b/rust/op-reth/crates/trie/tests/live/waits.rs @@ -0,0 +1,90 @@ +//! Bounded waits on the engine and on the store, so that a stall fails a case instead of +//! hanging it. + +use reth_ethereum_primitives::Block; +use reth_optimism_trie::engine::EngineHandle; +use std::{ + sync::mpsc, + thread, + time::{Duration, Instant}, +}; +use thiserror::Error; + +const POLL_INTERVAL: Duration = Duration::from_millis(10); + +/// Why a bounded wait on an engine call ended without its result. +#[derive(Debug, Error)] +pub(super) enum CallError { + #[error("{call_name} did not return within {bound:?}")] + Hung { bound: Duration, call_name: &'static str }, + #[error("{call_name} panicked")] + Panicked { call_name: &'static str }, +} + +/// An engine call running on its own thread. +pub(super) struct PendingCall { + call_name: &'static str, + reply: mpsc::Receiver, + started: Instant, +} + +impl PendingCall { + pub(super) fn spawn( + engine: EngineHandle, + call_name: &'static str, + call: impl FnOnce(EngineHandle) -> T + Send + 'static, + ) -> Self { + let (reply_tx, reply_rx) = mpsc::channel(); + thread::spawn(move || { + let _ = reply_tx.send(call(engine)); + }); + Self { call_name, reply: reply_rx, started: Instant::now() } + } + + /// Fails if the call returns within `probe`, while it had to stay blocked. + pub(super) fn ensure_blocked_for(&self, probe: Duration) -> eyre::Result<()> { + match self.reply.recv_timeout(probe) { + Err(mpsc::RecvTimeoutError::Timeout) => Ok(()), + Ok(_) => Err(eyre::eyre!( + "{} returned after {:?} while it had to stay blocked", + self.call_name, + self.started.elapsed() + )), + Err(mpsc::RecvTimeoutError::Disconnected) => { + Err(CallError::Panicked { call_name: self.call_name }.into()) + } + } + } + + /// The call's result and how long it took, or why neither is there once `bound` has passed + /// since the call started. + pub(super) fn wait(self, bound: Duration) -> Result<(T, Duration), CallError> { + let remaining = bound.saturating_sub(self.started.elapsed()); + match self.reply.recv_timeout(remaining) { + Ok(value) => Ok((value, self.started.elapsed())), + Err(mpsc::RecvTimeoutError::Timeout) => { + Err(CallError::Hung { bound, call_name: self.call_name }) + } + Err(mpsc::RecvTimeoutError::Disconnected) => { + Err(CallError::Panicked { call_name: self.call_name }) + } + } + } +} + +/// Polls `condition` until it holds, or fails once `bound` has passed; `what` names what is +/// waited for. +pub(super) fn wait_until( + what: &str, + bound: Duration, + mut condition: impl FnMut() -> eyre::Result, +) -> eyre::Result<()> { + let deadline = Instant::now() + bound; + loop { + if condition()? { + return Ok(()); + } + eyre::ensure!(Instant::now() < deadline, "{what} did not happen within {bound:?}"); + thread::sleep(POLL_INTERVAL); + } +}