Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
157 changes: 152 additions & 5 deletions rust/op-reth/crates/exex/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -936,13 +936,26 @@ mod tests {
store.provider_ro().expect("provider ro").get_latest_block().expect("latest block")
}

/// Anchors `store` at the test context's genesis and spawns an engine flushing every block,
/// The engine's persistence and backpressure thresholds.
#[derive(Clone, Copy, Debug)]
struct FlushCadence {
backpressure: u64,
persistence: u64,
}

impl FlushCadence {
const EVERY_BLOCK: Self = Self { backpressure: 2, persistence: 1 };
const EVERY_FIFTH_BLOCK: Self = Self { backpressure: 10, persistence: 5 };
}

/// Anchors `store` at the test context's genesis and spawns an engine flushing at `cadence`,
/// executing with a real EVM over `chain_spec` since the context's EVM refuses every call.
fn engine_over<S: OpProofsStore + Clone + 'static>(
ctx: &ExExContext<Adapter>,
handle: &TestExExHandle,
chain_spec: Arc<ChainSpec>,
store: S,
cadence: FlushCadence,
) -> EngineHandle<Block> {
init_storage_at(store.clone(), handle.genesis.num_hash());
let pruner = OpProofStoragePruner::new(store.clone(), ctx.components.provider.clone(), 20);
Expand All @@ -951,8 +964,8 @@ mod tests {
ctx.components.provider.clone(),
store,
pruner,
1,
2,
cadence.persistence,
cadence.backpressure,
)
}

Expand All @@ -978,7 +991,8 @@ mod tests {
let chain_spec = merged_chain_spec();
let (ctx, mut handle) =
test_exex_context_with_chain_spec(chain_spec.clone()).await.expect("exex test context");
let engine = engine_over(&ctx, &handle, chain_spec, store.clone());
let engine =
engine_over(&ctx, &handle, chain_spec, store.clone(), FlushCadence::EVERY_BLOCK);
let exex = build_test_exex(ctx, store.clone());
let old = commit_three_blocks(&exex, &mut handle, &engine);
// reth switches from blocks 2 and 3 to blocks 2' through 4' on block 1.
Expand Down Expand Up @@ -1034,7 +1048,8 @@ mod tests {
let chain_spec = merged_chain_spec();
let (ctx, mut handle) =
test_exex_context_with_chain_spec(chain_spec.clone()).await.expect("exex test context");
let engine = engine_over(&ctx, &handle, chain_spec, store.clone());
let engine =
engine_over(&ctx, &handle, chain_spec, store.clone(), FlushCadence::EVERY_BLOCK);
let exex = build_test_exex(ctx, store.clone());
let old = commit_three_blocks(&exex, &mut handle, &engine);
let stored_tip = old[2].num_hash();
Expand All @@ -1055,6 +1070,138 @@ mod tests {
handle.assert_events_empty();
}

/// How the ExEx hands a block near the tip to the engine, chosen through the verification
/// interval the way `--proofs-history.verification-interval` chooses it.
#[derive(Clone, Copy, Debug)]
enum ProofsPath {
/// Every block's verification is due, so the engine executes it.
Execute,
/// Every block carries reth's trie data and no block's verification is due.
Index,
}

impl ProofsPath {
const fn verification_interval(self) -> u64 {
match self {
Self::Execute => 1,
Self::Index => 0,
}
}
}

/// Blocks in a chain the engine saves under either cadence: the slower cadence's persistence
/// threshold, which the faster one's divides, so that no block waits for an idle flush.
const SAVED_CHAIN_LENGTH: u64 = FlushCadence::EVERY_FIFTH_BLOCK.persistence;

/// Blocks in a chain below the slower cadence's persistence threshold.
const SHORT_CHAIN_LENGTH: u64 = 3;

const _: () = assert!(SHORT_CHAIN_LENGTH < FlushCadence::EVERY_FIFTH_BLOCK.persistence);

/// What the engine has done with a committed chain by the time the ExEx acknowledges it.
#[derive(Clone, Copy, Debug)]
enum CommittedChain {
/// Near reth's head but shorter than the engine's persistence threshold.
Buffered,
/// The real-time threshold behind reth's head, so the ExEx only sets the engine's sync
/// target, which reth cannot serve since it holds none of the blocks.
CatchUp,
/// Near reth's head and long enough for the engine to save it.
Saved,
}

impl CommittedChain {
const fn length(self) -> u64 {
match self {
Self::Buffered | Self::CatchUp => SHORT_CHAIN_LENGTH,
Self::Saved => SAVED_CHAIN_LENGTH,
}
}

/// The store's persisted tip once the engine has flushed what its cadence lets it flush.
const fn persisted_tip(self, genesis: NumHash, tip: NumHash) -> NumHash {
match self {
Self::Buffered | Self::CatchUp => genesis,
Self::Saved => tip,
}
}
}

/// Moves reth's head to the real-time threshold past `tip` without giving reth any block.
fn set_head_far_ahead(ctx: &ExExContext<Adapter>, tip: u64) {
let head = Header { number: tip + REAL_TIME_BLOCKS_THRESHOLD, ..Default::default() };
ctx.components
.provider
.canonical_in_memory_state()
.set_canonical_head(SealedHeader::seal_slow(head));
}

/// Flushes until nothing the engine could still save is unsaved, and returns the persisted
/// tip. A flush waits for the save in flight and the engine starts the next save only after
/// replying, so one flush per block the engine could have buffered drains them all.
fn settle<S: OpProofsStore>(engine: &EngineHandle<Block>, store: &S) -> NumHash {
for _ in 0..=SAVED_CHAIN_LENGTH {
engine.flush();
}
latest_block(store)
}

async fn run_finished_height_case(
chain: CommittedChain,
cadence: FlushCadence,
path: ProofsPath,
) {
let store = InMemoryProofsStorage::new();
let chain_spec = merged_chain_spec();
let (ctx, mut handle) =
test_exex_context_with_chain_spec(chain_spec.clone()).await.expect("exex test context");
let engine = engine_over(&ctx, &handle, chain_spec, store.clone(), cadence);
let exex = OpProofsExEx::builder(ctx, store.clone())
.with_verification_interval(path.verification_interval())
.build();
let genesis = handle.genesis.num_hash();
let blocks = empty_chain(genesis, 1..=chain.length(), 0x10);
let tip = blocks.last().expect("a chain has a tip").num_hash();
match chain {
CommittedChain::Buffered | CommittedChain::Saved => make_canonical(
&exex.ctx,
NewCanonicalChain::Commit { new: blocks.iter().map(executed).collect() },
),
CommittedChain::CatchUp => set_head_far_ahead(&exex.ctx, tip.number),
}
let notification =
ExExNotification::ChainCommitted { new: Arc::new(chain_of(blocks, true)) };

exex.handle_notification(notification, &engine).expect("handle chain commit");

// The contract today (ethereum-optimism/protocol-team#478): the ExEx acknowledges the
// notification's tip once the engine has accepted the chain, before the engine has
// persisted any of it, and even when it has only scheduled it for catch-up. A fix changes
// the acknowledged height, so it must change this assertion deliberately.
handle.assert_event_finished_height(tip).expect("one FinishedHeight event");
handle.assert_events_empty();
assert_eq!(settle(&engine, &store), chain.persisted_tip(genesis, tip), "persisted tip");
}

/// What `FinishedHeight` acknowledges for a committed chain the engine holds buffered and
/// unflushed, has only scheduled for catch-up, or has saved. A chain shorter than the
/// persistence threshold stays buffered only under the slower cadence, so that row has no
/// every-block cell.
#[rstest]
#[case::buffered_every_fifth_block(CommittedChain::Buffered, FlushCadence::EVERY_FIFTH_BLOCK)]
#[case::catch_up_every_block(CommittedChain::CatchUp, FlushCadence::EVERY_BLOCK)]
#[case::catch_up_every_fifth_block(CommittedChain::CatchUp, FlushCadence::EVERY_FIFTH_BLOCK)]
#[case::saved_every_block(CommittedChain::Saved, FlushCadence::EVERY_BLOCK)]
#[case::saved_every_fifth_block(CommittedChain::Saved, FlushCadence::EVERY_FIFTH_BLOCK)]
#[tokio::test]
async fn finished_height_is_the_notification_tip(
#[case] chain: CommittedChain,
#[case] cadence: FlushCadence,
#[values(ProofsPath::Execute, ProofsPath::Index)] path: ProofsPath,
) {
run_finished_height_case(chain, cadence, path).await
}

/// The trie data of a block is used unless the chain lacks it or the block's verification is
/// due; an interval of zero never makes it due.
#[rstest]
Expand Down
107 changes: 99 additions & 8 deletions rust/op-reth/crates/trie/src/engine/runner.rs
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,9 @@ where
backpressure_threshold: u64,
/// Current retry delay for the sync arm; grows on failure, resets on success.
sync_backoff: Duration,
/// Earliest instant the next sync step may run: already past unless the last step failed.
/// Only a failed step moves it, so no other event can postpone a retry.
sync_retry_at: Instant,
}

impl<Evm, Provider, Store> Engine<Evm, Provider, Store>
Expand Down Expand Up @@ -70,6 +73,7 @@ where
persistence_threshold: DEFAULT_PERSISTENCE_THRESHOLD,
backpressure_threshold: DEFAULT_BACKPRESSURE_THRESHOLD,
sync_backoff: Duration::ZERO,
sync_retry_at: Instant::now(),
}
}

Expand Down Expand Up @@ -153,8 +157,9 @@ where
/// backpressure is active — callers naturally block in their bounded `send` until the
/// in-flight save completes and memory is pruned.
/// - **persistence**: signals a completed background save.
/// - **sync**: a zero-duration timer that fires immediately when the engine is behind its sync
/// target and not under backpressure; [`crossbeam_channel::never`] otherwise.
/// - **sync**: fires once `sync_retry_at` has passed, at once unless the last step failed,
/// while the engine is behind its sync target and not under backpressure;
/// [`crossbeam_channel::never`] otherwise.
/// - **idle-flush**: fires after [`IDLE_FLUSH_INTERVAL`] when memory holds buffered blocks but
/// the persistence threshold hasn't been reached and no save is in flight. Keeps a paused
/// chain from leaving buffered blocks invisible to the proofs RPC indefinitely.
Expand All @@ -171,11 +176,11 @@ where
let incoming_rx: Receiver<EngineAction<BlockTy<Evm::Primitives>>> =
if backpressure { crossbeam_channel::never() } else { self.incoming.clone() };

// Fire when there is sync work to do; block indefinitely otherwise. After a failed
// sync step we delay by `sync_backoff` so a sustained failure (e.g. a stale sync target
// ahead of the provider's tip) doesn't spin the CPU and flood logs.
// Fire when there is sync work to do; block indefinitely otherwise. A failed step pushes
// `sync_retry_at` out so a sustained failure (e.g. a stale sync target ahead of the
// provider's tip) doesn't spin the CPU and flood logs.
let sync_rx: Receiver<Instant> = if self.needs_sync() && !backpressure {
crossbeam_channel::after(self.sync_backoff)
crossbeam_channel::at(self.sync_retry_at)
} else {
crossbeam_channel::never()
};
Expand All @@ -198,6 +203,7 @@ where
Ok(()) => self.sync_backoff = Duration::ZERO,
Err(err) => {
self.sync_backoff = next_sync_backoff(self.sync_backoff);
self.sync_retry_at = Instant::now() + self.sync_backoff;
error!(target: "trie::engine::runner", ?err, backoff = ?self.sync_backoff, "Sync step failed");
}
},
Expand Down Expand Up @@ -234,8 +240,93 @@ where

#[cfg(test)]
mod tests {
use super::{SYNC_BACKOFF_INITIAL, SYNC_BACKOFF_MAX, next_sync_backoff};
use std::time::Duration;
use super::{Engine, SYNC_BACKOFF_INITIAL, SYNC_BACKOFF_MAX, next_sync_backoff};
use crate::{
InMemoryProofsStorage, OpProofStoragePruner, OpProofsInitProvider, OpProofsStore,
engine::{EngineAction, tasks::SyncToTask},
};
use alloy_consensus::Header;
use alloy_eips::BlockNumHash;
use crossbeam_channel::{Sender, bounded};
use reth_chainspec::MAINNET;
use reth_db_common::init::init_genesis;
use reth_ethereum_primitives::Block;
use reth_evm_ethereum::EthEvmConfig;
use reth_primitives_traits::SealedHeader;
use reth_provider::{
providers::BlockchainProvider,
test_utils::{MockNodeTypesWithDB, create_test_provider_factory_with_chain_spec},
};
use std::{
ops::ControlFlow,
thread,
time::{Duration, Instant},
};

type TestEngine =
Engine<EthEvmConfig, BlockchainProvider<MockNodeTypesWithDB>, InMemoryProofsStorage>;

/// An engine whose store is anchored at genesis and whose sync target is block 1, which reth
/// reports as its head but does not hold, so that every sync step fails.
fn engine_behind_a_missing_block() -> (TestEngine, Sender<EngineAction<Block>>) {
let chain_spec = MAINNET.clone();
let factory = create_test_provider_factory_with_chain_spec(chain_spec.clone());
let genesis_hash = init_genesis(&factory).expect("init genesis");
let provider = BlockchainProvider::new(factory).expect("blockchain provider");
let missing_head = Header { number: 1, parent_hash: genesis_hash, ..Default::default() };
provider
.canonical_in_memory_state()
.set_canonical_head(SealedHeader::seal_slow(missing_head));
let storage = InMemoryProofsStorage::new();
let init = storage.initialization_provider().expect("init provider");
init.set_initial_state_anchor(BlockNumHash::new(0, genesis_hash)).expect("set anchor");
init.commit_initial_state().expect("commit initial state");
OpProofsInitProvider::commit(init).expect("commit");
let pruner = OpProofStoragePruner::new(storage.clone(), provider.clone(), 1000);
let (actions, incoming) = bounded(10);
let mut engine =
Engine::new(EthEvmConfig::ethereum(chain_spec), provider, storage, pruner, incoming);
engine.state.sync_target = 1;
(engine, actions)
}

fn step(engine: &mut TestEngine) {
assert_eq!(engine.process_next_event(), ControlFlow::Continue(()));
}

/// Sync-target updates arriving faster than the retry delay neither postpone the retry
/// (ethereum-optimism/protocol-team#477) nor bring it forward.
#[test]
fn sync_retry_is_not_postponed_by_incoming_actions() {
let (mut engine, actions) = engine_behind_a_missing_block();
// Three failed steps grow the retry delay to four times the initial one.
for _ in 0..3 {
step(&mut engine);
}
let third_failure = Instant::now();
let retry_delay = engine.sync_backoff;
assert_eq!(retry_delay, 4 * SYNC_BACKOFF_INITIAL);
let action_gap = retry_delay / 8;
let patience = 10 * retry_delay;

// Every step finds an action queued, which is what kept re-arming the retry timer.
let deadline = Instant::now() + patience;
while engine.sync_backoff == retry_delay {
assert!(
Instant::now() < deadline,
"no sync step within {patience:?} while actions kept arriving every {action_gap:?}"
);
actions.send(EngineAction::SyncTo(SyncToTask { target: 1 })).expect("engine alive");
step(&mut engine);
thread::sleep(action_gap);
}

assert_eq!(engine.sync_backoff, 2 * retry_delay, "the step ran once and failed again");
assert!(
third_failure.elapsed() >= retry_delay - action_gap,
"the step ran before its retry delay had passed"
);
}

#[test]
fn zero_backoff_jumps_to_initial() {
Expand Down
3 changes: 0 additions & 3 deletions rust/op-reth/crates/trie/src/error.rs
Original file line number Diff line number Diff line change
Expand Up @@ -103,9 +103,6 @@ pub enum OpProofsStorageError {
/// Error occurred while interacting with the database.
#[error(transparent)]
DatabaseError(DatabaseError),
/// Error occurred while trying to acquire a lock.
#[error("failed lock attempt")]
TryLockError,
/// Error occurred during block execution.
#[error(transparent)]
ExecutionError(Arc<BlockExecutionError>),
Expand Down
Loading
Loading