Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
65 changes: 46 additions & 19 deletions rust/op-reth/crates/trie/src/engine/handle.rs
Original file line number Diff line number Diff line change
@@ -1,8 +1,7 @@
//! [`EngineHandle`] — the public, cloneable, Send + Sync interface.

use super::{
CanonicalInMemoryStateProvider, DEFAULT_BACKPRESSURE_THRESHOLD, DEFAULT_PERSISTENCE_THRESHOLD,
EngineAction,
CanonicalInMemoryStateProvider, EngineAction, EngineConfig,
error::EngineError,
runner::Engine,
service_guard::ServiceGuard,
Expand All @@ -12,7 +11,7 @@ use crate::{OpProofStoragePruner, OpProofsStore};
#[cfg(any(test, feature = "test-utils"))]
use alloy_eips::NumHash;
use alloy_eips::eip1898::BlockWithParent;
use crossbeam_channel::{Sender, bounded};
use crossbeam_channel::{Receiver, Sender, bounded, select_biased};
use reth_evm::ConfigureEvm;
use reth_primitives_traits::{NodePrimitives, RecoveredBlock};
use reth_provider::{BlockHashReader, BlockReader, DatabaseProviderFactory, StageCheckpointReader};
Expand All @@ -23,9 +22,13 @@ use tracing::error;
/// A thin, cloneable handle used to communicate with the collector engine.
///
/// Every public method (except [`Self::sync_to`]) sends an engine action to the
/// engine thread and blocks on a one-shot reply channel.
/// engine thread and blocks on a one-shot reply channel, or until the engine exits.
#[derive(Debug, Clone)]
pub struct EngineHandle<Block: reth_primitives_traits::Block> {
/// Disconnects when the engine thread exits by any path, so that no call waits on a reply
/// the engine will never send: an action queued when the engine exits stays in the channel,
/// with its reply sender, for as long as a handle lives.
engine_exited: Receiver<()>,
sender: Sender<EngineAction<Block>>,
_service_guard: Arc<ServiceGuard>,
}
Expand All @@ -49,14 +52,7 @@ impl<Block: reth_primitives_traits::Block + Send + 'static> EngineHandle<Block>
+ 'static,
Store: OpProofsStore + Clone + 'static,
{
Self::spawn_with_thresholds(
evm_config,
provider,
storage,
pruner,
DEFAULT_PERSISTENCE_THRESHOLD,
DEFAULT_BACKPRESSURE_THRESHOLD,
)
Self::spawn_with_config(evm_config, provider, storage, pruner, EngineConfig::default())
}

/// Spawn the collector engine with custom persistence and backpressure thresholds.
Expand All @@ -68,6 +64,30 @@ impl<Block: reth_primitives_traits::Block + Send + 'static> EngineHandle<Block>
persistence_threshold: u64,
backpressure_threshold: u64,
) -> Self
where
Evm: ConfigureEvm<Primitives: NodePrimitives<Block = Block>> + 'static,
Provider: BlockHashReader
+ CanonicalInMemoryStateProvider
+ StageCheckpointReader
+ DatabaseProviderFactory
+ BlockReader<Block = Block>
+ Clone
+ 'static,
Store: OpProofsStore + Clone + 'static,
{
let config =
EngineConfig { backpressure_threshold, persistence_threshold, ..Default::default() };
Self::spawn_with_config(evm_config, provider, storage, pruner, config)
}

/// Spawn the collector engine with `config`.
pub fn spawn_with_config<Evm, Provider, Store>(
evm_config: Evm,
provider: Provider,
storage: Store,
pruner: OpProofStoragePruner<Store, Provider>,
config: EngineConfig,
) -> Self
where
Evm: ConfigureEvm<Primitives: NodePrimitives<Block = Block>> + 'static,
Provider: BlockHashReader
Expand All @@ -80,9 +100,8 @@ impl<Block: reth_primitives_traits::Block + Send + 'static> EngineHandle<Block>
Store: OpProofsStore + Clone + 'static,
{
let (tx, rx) = bounded(10);
let engine = Engine::new(evm_config, provider, storage, pruner, rx)
.with_persistence_threshold(persistence_threshold)
.with_backpressure_threshold(backpressure_threshold);
let (exited, engine_exited) = bounded(0);
let engine = Engine::new(evm_config, provider, storage, pruner, rx, exited, config);

let join_handle = thread::Builder::new()
.name("live-trie-collector".into())
Expand All @@ -98,7 +117,15 @@ impl<Block: reth_primitives_traits::Block + Send + 'static> EngineHandle<Block>
})
.expect("failed to spawn live-trie-collector thread");

Self { sender: tx, _service_guard: Arc::new(ServiceGuard::new(join_handle)) }
Self { engine_exited, sender: tx, _service_guard: Arc::new(ServiceGuard::new(join_handle)) }
}

/// Waits for the reply to an action, or for the engine to exit; a reply already there wins.
fn recv_reply<Reply>(&self, reply: &Receiver<Reply>) -> Result<Reply, EngineError> {
select_biased! {
recv(reply) -> reply => reply.map_err(|_| EngineError::EngineDied),
recv(self.engine_exited) -> _ => Err(EngineError::EngineDied),
}
}

fn send_and_recv(
Expand All @@ -107,7 +134,7 @@ impl<Block: reth_primitives_traits::Block + Send + 'static> EngineHandle<Block>
) -> Result<(), EngineError> {
let (reply_tx, reply_rx) = bounded(1);
self.sender.send(make_action(reply_tx)).map_err(|_| EngineError::EngineDied)?;
reply_rx.recv().map_err(|_| EngineError::EngineDied)?
self.recv_reply(&reply_rx)?
}

/// Execute a block through the EVM and buffer the resulting trie updates.
Expand Down Expand Up @@ -169,7 +196,7 @@ impl<Block: reth_primitives_traits::Block + Send + 'static> EngineHandle<Block>
use super::tasks::FlushTask;
let (reply_tx, reply_rx) = bounded(1);
if self.sender.send(EngineAction::Flush(FlushTask { reply: reply_tx })).is_ok() {
let _ = reply_rx.recv();
let _ = self.recv_reply(&reply_rx);
}
}

Expand All @@ -182,6 +209,6 @@ impl<Block: reth_primitives_traits::Block + Send + 'static> EngineHandle<Block>
self.sender
.send(EngineAction::Tip(TipTask { reply: reply_tx }))
.map_err(|_| EngineError::EngineDied)?;
reply_rx.recv().map_err(|_| EngineError::EngineDied)?
self.recv_reply(&reply_rx)?
}
}
37 changes: 32 additions & 5 deletions rust/op-reth/crates/trie/src/engine/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -28,22 +28,49 @@ mod runner;
mod service_guard;
mod state;

use std::time::Duration;

/// Default number of blocks to keep in memory before persisting.
const DEFAULT_PERSISTENCE_THRESHOLD: u64 = 5;

/// Default number of blocks where we block execution to allow persistence to catch up.
const DEFAULT_BACKPRESSURE_THRESHOLD: u64 = 10;

/// Default timeout for waiting on a persistence save/unwind operation (in seconds).
const DEFAULT_PERSISTENCE_TIMEOUT_SECS: u64 = 60;
/// Default timeout for waiting on a persistence save/unwind operation.
const DEFAULT_PERSISTENCE_TIMEOUT: Duration = Duration::from_secs(60);

/// How long the engine waits with a non-empty memory buffer before flushing it even if the
/// persistence threshold has not been reached.
/// How long the engine waits by default with a non-empty memory buffer before flushing it even
/// if the persistence threshold has not been reached.
///
/// Without this, a paused chain (e.g. fault-proof tests that freeze the sequencer) would leave
/// buffered blocks unpersisted indefinitely, breaking the proofs RPC's strict "is this block
/// persisted?" check.
const IDLE_FLUSH_INTERVAL: std::time::Duration = std::time::Duration::from_secs(10);
const DEFAULT_IDLE_FLUSH_INTERVAL: Duration = Duration::from_secs(10);

/// How the engine paces its saves and how long it waits on its persistence service.
#[derive(Clone, Copy, Debug)]
pub struct EngineConfig {
/// Buffered blocks at which the engine stops taking actions until the save in flight
/// completes.
pub backpressure_threshold: u64,
/// How long buffered blocks below the persistence threshold wait before a save starts anyway.
pub idle_flush_interval: Duration,
/// Buffered blocks that start a save.
pub persistence_threshold: u64,
/// How long the engine waits for a save or an unwind before giving up on it.
pub persistence_timeout: Duration,
}

impl Default for EngineConfig {
fn default() -> Self {
Self {
backpressure_threshold: DEFAULT_BACKPRESSURE_THRESHOLD,
idle_flush_interval: DEFAULT_IDLE_FLUSH_INTERVAL,
persistence_threshold: DEFAULT_PERSISTENCE_THRESHOLD,
persistence_timeout: DEFAULT_PERSISTENCE_TIMEOUT,
}
}
}

/// Messages sent from [`EngineHandle`] to the engine thread.
enum EngineAction<Block: reth_primitives_traits::Block> {
Expand Down
69 changes: 38 additions & 31 deletions rust/op-reth/crates/trie/src/engine/runner.rs
Original file line number Diff line number Diff line change
@@ -1,11 +1,11 @@
//! [`Engine`] — the thin event-loop dispatcher.

use super::{
CanonicalInMemoryStateProvider, DEFAULT_BACKPRESSURE_THRESHOLD, DEFAULT_PERSISTENCE_THRESHOLD,
EngineAction, IDLE_FLUSH_INTERVAL, error::EngineError, state::EngineState as State,
CanonicalInMemoryStateProvider, EngineAction, EngineConfig, error::EngineError,
state::EngineState as State,
};
use crate::{OpProofStoragePruner, OpProofsStore};
use crossbeam_channel::Receiver;
use crossbeam_channel::{Receiver, Sender};
use reth_evm::ConfigureEvm;
use reth_primitives_traits::BlockTy;
use reth_provider::{
Expand Down Expand Up @@ -37,10 +37,13 @@ where
Evm: ConfigureEvm,
Provider: CanonicalInMemoryStateProvider + DatabaseProviderFactory + BlockReader,
{
state: State<Evm, Provider, Store>,
/// Dropped when the engine exits by any path, which every call waiting on the handle takes
/// as `EngineDied`. Must stay declared first: fields drop in declaration order, and dropping
/// `state` joins the persistence thread, which a slow save can hold indefinitely.
_exited: Sender<()>,
incoming: Receiver<EngineAction<BlockTy<Evm::Primitives>>>,
persistence_threshold: u64,
backpressure_threshold: u64,
state: State<Evm, Provider, Store>,
config: EngineConfig,
/// Current retry delay for the sync arm; grows on failure, resets on success.
sync_backoff: Duration,
/// Earliest instant the next sync step may run: already past unless the last step failed.
Expand All @@ -66,27 +69,19 @@ where
storage: Store,
pruner: OpProofStoragePruner<Store, Provider>,
incoming: Receiver<EngineAction<BlockTy<Evm::Primitives>>>,
exited: Sender<()>,
config: EngineConfig,
) -> Self {
Self {
state: State::new(evm_config, provider, storage, pruner),
_exited: exited,
incoming,
persistence_threshold: DEFAULT_PERSISTENCE_THRESHOLD,
backpressure_threshold: DEFAULT_BACKPRESSURE_THRESHOLD,
state: State::new(evm_config, provider, storage, pruner, config.persistence_timeout),
config,
sync_backoff: Duration::ZERO,
sync_retry_at: Instant::now(),
}
}

pub(super) const fn with_persistence_threshold(mut self, threshold: u64) -> Self {
self.persistence_threshold = threshold;
self
}

pub(super) const fn with_backpressure_threshold(mut self, threshold: u64) -> Self {
self.backpressure_threshold = threshold;
self
}

/// Returns `true` if the engine is behind its sync target.
fn needs_sync(&self) -> bool {
let current_tip = self.state.get_tip().map(|t| t.number).unwrap_or(0);
Expand All @@ -96,12 +91,12 @@ where
/// Returns `true` if the buffer is above the backpressure threshold with a save in-flight.
fn backpressure_active(&self) -> bool {
self.state.persistence.in_flight.is_some() &&
self.state.memory.len() as u64 >= self.backpressure_threshold
self.state.memory.len() as u64 >= self.config.backpressure_threshold
}

/// Start a background persistence save if the memory buffer has reached the threshold.
fn maybe_start_save(&mut self) {
if self.state.memory.len() as u64 >= self.persistence_threshold &&
if self.state.memory.len() as u64 >= self.config.persistence_threshold &&
let Err(e) = self.state.advance_persistence()
{
error!(target: "trie::engine::runner", ?e, "Failed to start persistence save");
Expand Down Expand Up @@ -154,13 +149,13 @@ where
///
/// Four receivers compete in a single `select!`:
/// - **action**: a new [`EngineAction`] from a caller, or [`crossbeam_channel::never`] while
/// backpressure is active — callers naturally block in their bounded `send` until the
/// in-flight save completes and memory is pruned.
/// backpressure is active — callers' actions then wait in the channel until the in-flight
/// save completes and memory is pruned.
/// - **persistence**: signals a completed background save.
/// - **sync**: fires once `sync_retry_at` has passed, at once unless the last step failed,
/// while the engine is behind its sync target and not under backpressure;
/// [`crossbeam_channel::never`] otherwise.
/// - **idle-flush**: fires after [`IDLE_FLUSH_INTERVAL`] when memory holds buffered blocks but
/// - **idle-flush**: fires after the idle flush interval when memory holds buffered blocks but
/// the persistence threshold hasn't been reached and no save is in flight. Keeps a paused
/// chain from leaving buffered blocks invisible to the proofs RPC indefinitely.
///
Expand Down Expand Up @@ -188,7 +183,7 @@ where
// Arm the idle-flush timer only when there's buffered work that nothing else will flush.
let idle_flush_rx: Receiver<Instant> =
if !self.state.memory.is_empty() && !save_in_flight && !backpressure {
crossbeam_channel::after(IDLE_FLUSH_INTERVAL)
crossbeam_channel::after(self.config.idle_flush_interval)
} else {
crossbeam_channel::never()
};
Expand Down Expand Up @@ -217,10 +212,10 @@ where
/// Runs the main loop of the engine, processing incoming actions.
pub(super) fn run(mut self) {
debug_assert!(
self.persistence_threshold < self.backpressure_threshold,
self.config.persistence_threshold < self.config.backpressure_threshold,
"backpressure_threshold ({}) must be greater than persistence_threshold ({})",
self.backpressure_threshold,
self.persistence_threshold,
self.config.backpressure_threshold,
self.config.persistence_threshold,
);
debug!(target: "trie::engine::runner", "Collector engine started");

Expand All @@ -230,6 +225,10 @@ where
ControlFlow::Continue(()) => {}
}
self.maybe_start_save();
if self.state.persistence.is_disconnected() {
error!(target: "trie::engine::runner", "Persistence service gone, stopping the collector engine");
break;
}
}

debug!(target: "trie::engine::runner", "Collector engine shutting down, draining in-flight persist");
Expand All @@ -243,7 +242,7 @@ mod tests {
use super::{Engine, SYNC_BACKOFF_INITIAL, SYNC_BACKOFF_MAX, next_sync_backoff};
use crate::{
InMemoryProofsStorage, OpProofStoragePruner, OpProofsInitProvider, OpProofsStore,
engine::{EngineAction, tasks::SyncToTask},
engine::{EngineAction, EngineConfig, tasks::SyncToTask},
};
use alloy_consensus::Header;
use alloy_eips::BlockNumHash;
Expand Down Expand Up @@ -284,8 +283,16 @@ mod tests {
OpProofsInitProvider::commit(init).expect("commit");
let pruner = OpProofStoragePruner::new(storage.clone(), provider.clone(), 1000);
let (actions, incoming) = bounded(10);
let mut engine =
Engine::new(EthEvmConfig::ethereum(chain_spec), provider, storage, pruner, incoming);
let (exited, _engine_exited) = bounded(0);
let mut engine = Engine::new(
EthEvmConfig::ethereum(chain_spec),
provider,
storage,
pruner,
incoming,
exited,
EngineConfig::default(),
);
engine.state.sync_target = 1;
(engine, actions)
}
Expand Down
Loading
Loading