diff --git a/Cargo.lock b/Cargo.lock index 475ef86c651..790692cff43 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8485,6 +8485,7 @@ dependencies = [ "reth-payload-primitives", "reth-primitives-traits", "reth-provider", + "reth-prune-types", "reth-rpc-api", "reth-rpc-builder", "reth-rpc-eth-api", @@ -8493,6 +8494,7 @@ dependencies = [ "reth-tasks", "reth-tokio-util", "reth-tracing", + "reth-transaction-pool", "reth-trie", "reth-trie-db", "revm", diff --git a/crates/e2e-test-utils/Cargo.toml b/crates/e2e-test-utils/Cargo.toml index fcf902070fa..f314cf93fbd 100644 --- a/crates/e2e-test-utils/Cargo.toml +++ b/crates/e2e-test-utils/Cargo.toml @@ -41,6 +41,7 @@ reth-db-common.workspace = true reth-primitives-traits.workspace = true reth-trie.workspace = true reth-trie-db.workspace = true +reth-transaction-pool.workspace = true revm.workspace = true tempfile.workspace = true @@ -70,6 +71,9 @@ serde_json.workspace = true tracing.workspace = true derive_more.workspace = true +[dev-dependencies] +reth-prune-types.workspace = true + [[test]] name = "e2e_testsuite" path = "tests/e2e-testsuite/main.rs" diff --git a/crates/e2e-test-utils/src/node.rs b/crates/e2e-test-utils/src/node.rs index 7eb43c73b7d..43b2b1cc665 100644 --- a/crates/e2e-test-utils/src/node.rs +++ b/crates/e2e-test-utils/src/node.rs @@ -30,6 +30,7 @@ use reth_rpc_api::TestingBuildBlockRequestV1; use reth_rpc_builder::auth::AuthServerHandle; use reth_rpc_eth_api::helpers::{EthApiSpec, EthTransactions, TraceExt}; use reth_stages_types::StageId; +use reth_transaction_pool::TransactionPool; use std::{pin::Pin, sync::Arc, time::Duration}; use tokio_stream::StreamExt; use url::Url; @@ -168,6 +169,30 @@ where }) } + /// Sets the timestamp of the next payload built by [`Self::new_payload`] and the methods + /// built on it, such as [`Self::advance_block`]. + /// + /// Later payloads continue from this timestamp, one second apart. The attributes generator is + /// called with the new timestamp, so fork-dependent attributes follow it. + /// + /// Returns an error if the timestamp is not greater than the timestamp of the current latest + /// block, which is the parent of the next payload. + pub fn set_next_payload_timestamp(&mut self, timestamp: u64) -> eyre::Result<()> { + let latest = self + .inner + .provider + .sealed_header_by_number_or_tag(BlockNumberOrTag::Latest)? + .ok_or_else(|| eyre!("latest block not found"))?; + ensure!( + timestamp > latest.timestamp(), + "next payload timestamp {timestamp} must be greater than the latest block timestamp {}", + latest.timestamp() + ); + // The payload context increments its timestamp before generating the next attributes. + self.payload.timestamp = timestamp - 1; + Ok(()) + } + /// Creates a new payload from given attributes generator /// expects a payload attribute event and waits until the payload is built. /// @@ -199,16 +224,24 @@ where Ok(payload) } - /// Advances the node forward one block + /// Advances the node forward one block by building a payload and importing it with + /// [`Self::import_payload`]. pub async fn advance_block(&mut self) -> eyre::Result { - let payload = self.build_and_submit_payload().await?; + let payload = self.new_payload().await?; - // trigger forkchoice update via engine api to commit the block to the blockchain - self.update_forkchoice(payload.block().hash(), payload.block().hash()).await?; + self.import_payload(payload.clone()).await?; Ok(payload) } + /// Advances the node forward one block like [`Self::advance_block`] and waits until the + /// transaction pool processed the new block, see [`Self::wait_for_pool_head`]. + pub async fn advance_block_synced(&mut self) -> eyre::Result { + let payload = self.advance_block().await?; + self.wait_for_pool_head(payload.block().hash()).await?; + Ok(payload) + } + /// Waits for block to be available on node. /// /// Returns an error if the block is not available within [`WAIT_TIMEOUT`]. @@ -271,6 +304,29 @@ where .await } + /// Waits until the transaction pool of the node processed the canonical state update that made + /// the block with the given hash its head. + /// + /// The pool maintenance task processes new heads in the background, so right after e.g. a + /// forkchoice update the pool can still contain the transactions mined in the new block or use + /// outdated sender nonces and pending fees. The pool updates its last seen block together with + /// these, so they are up to date once this returns. Exceptions are reorgs, after which the + /// maintenance task re-injects the transactions of the old chain only afterwards, so wait for + /// them with [`Self::wait_for_pool`], and commits deeper than the maximum update depth of the + /// maintenance task, e.g. after a long sync, which only update the last seen block. + /// + /// Returns an error if the pool does not process the block within [`WAIT_TIMEOUT`], e.g. + /// because the block is not the canonical head, the pool already processed a newer head, or + /// the block was synced by backfill, which does not notify the pool. + pub async fn wait_for_pool_head(&self, hash: B256) -> eyre::Result<()> { + let pool = &self.inner.pool; + poll_until(format!("transaction pool to process block {hash}"), move || { + let ready = pool.block_info().last_seen_block_hash == hash; + async move { Ok(ready.then_some(())) } + }) + .await + } + /// Asserts that a new block has been added to the blockchain and the tx has been included in /// the block, at any position. /// @@ -344,10 +400,10 @@ where .await .map_err(|_| eyre!("timed out syncing to block {block}"))??; - // Hack to make sure that all components have time to process canonical state update. - // Otherwise, this might result in e.g "nonce too low" errors when advancing chain further, - // making tests flaky. - tokio::time::sleep(Duration::from_millis(1000)).await; + // The transaction pool processes the canonical state update in the background, and + // advancing the chain before it did can fail with e.g. "nonce too low" errors. Blocks + // synced by backfill don't notify the pool, so wait for at most a second. + let _ = tokio::time::timeout(Duration::from_secs(1), self.wait_for_pool_head(block)).await; Ok(()) } @@ -390,6 +446,30 @@ where Ok(block_hash) } + /// Submits a payload to the engine and makes its block the canonical head, returning the block + /// hash. + /// + /// The forkchoice update marks the block as head, safe, and finalized block, like + /// [`Self::advance_block`] does, so the node can't reorg to a chain without the block + /// afterwards. The engine only reports the forkchoice update valid after making the head + /// canonical, so the block is the latest block of the node once this returns. The transaction + /// pool processes the new block in the background, see [`Self::wait_for_pool`]. + /// + /// The parent of the payload must be known to the node, e.g. to import a payload built by + /// another node into its peers. Returns an error if the engine does not report the forkchoice + /// update valid, e.g. because the payload is invalid. + pub async fn import_payload(&self, payload: Payload::BuiltPayload) -> eyre::Result { + let block_hash = self.submit_payload(payload).await?; + let updated = self.update_forkchoice(block_hash, block_hash).await?; + ensure!( + updated.is_valid(), + "forkchoice update to block {block_hash} is not valid: {}", + updated.payload_status.status + ); + + Ok(block_hash) + } + /// Returns the RPC URL. pub fn rpc_url(&self) -> Url { let addr = self.inner.rpc_server_handle().http_local_addr().unwrap(); @@ -522,20 +602,26 @@ where mod tests { use super::*; use crate::NodeHelperType; - use reth_node_ethereum::EthereumNode; + use reth_node_ethereum::{EthEngineTypes, EthereumNode}; fn assert_send(_: T) {} /// Tests of downstream nodes await these helpers in spawned tasks, so their futures must be /// `Send`. #[expect(dead_code)] - fn test_helper_futures_are_send(node: &mut NodeHelperType) { + fn test_helper_futures_are_send( + node: &mut NodeHelperType, + payload: ::BuiltPayload, + ) { assert_send(node.advance_block()); + assert_send(node.advance_block_synced()); assert_send(node.inject_and_advance(Bytes::new())); assert_send(node.wait_block(0, B256::ZERO, false)); assert_send(node.wait_unwind(0)); assert_send(node.wait_for_pool(|_| true)); + assert_send(node.wait_for_pool_head(B256::ZERO)); assert_send(node.assert_new_block(B256::ZERO, B256::ZERO, 0)); assert_send(node.sync_to(B256::ZERO)); + assert_send(node.import_payload(payload)); } } diff --git a/crates/e2e-test-utils/src/payload.rs b/crates/e2e-test-utils/src/payload.rs index fe0d4452c9f..c920b1bf065 100644 --- a/crates/e2e-test-utils/src/payload.rs +++ b/crates/e2e-test-utils/src/payload.rs @@ -10,6 +10,13 @@ use tokio_stream::wrappers::BroadcastStream; pub struct PayloadTestContext { pub payload_event_stream: BroadcastStream>, payload_builder: PayloadBuilderHandle, + /// Timestamp of the last generated payload attributes, or the starting point if none were + /// generated yet. + /// + /// [`Self::next_attributes`] increments it before generating the next attributes. Prefer + /// [`NodeTestContext::set_next_payload_timestamp`] over setting it directly. + /// + /// [`NodeTestContext::set_next_payload_timestamp`]: crate::node::NodeTestContext::set_next_payload_timestamp pub timestamp: u64, #[debug(skip)] attributes_generator: Box T::PayloadAttributes + Send + Sync>, diff --git a/crates/e2e-test-utils/tests/rocksdb/main.rs b/crates/e2e-test-utils/tests/rocksdb/main.rs index 15d5d00b131..c2f6dd12a1b 100644 --- a/crates/e2e-test-utils/tests/rocksdb/main.rs +++ b/crates/e2e-test-utils/tests/rocksdb/main.rs @@ -9,7 +9,8 @@ use reth_chainspec::EthereumHardfork; use reth_db::tables; use reth_e2e_test_utils::{transaction::TransactionTestContext, wait::poll_until, E2ETestSetupExt}; use reth_node_ethereum::EthereumNode; -use reth_provider::RocksDBProviderFactory; +use reth_provider::{PruneCheckpointReader, RocksDBProviderFactory}; +use reth_prune_types::PruneSegment; use std::time::Duration; /// Polls RPC until the given `tx_hash` is visible as pending (not yet mined). @@ -40,6 +41,25 @@ async fn poll_tx_in_rocksdb(provider: &P, tx_hash: B2 .await } +/// Polls until the pruner has pruned `segment` up to at least `block`. +/// +/// The persistence service acknowledges saved blocks before it runs the pruner in a separate +/// commit, so a block being visible in `RocksDB` does not mean the pruner has run for it yet. +async fn wait_for_prune_checkpoint( + provider: &P, + segment: PruneSegment, + block: u64, +) -> Result<()> { + poll_until(format!("{segment} to be pruned up to block {block}"), || async move { + let checkpoint = provider.get_prune_checkpoint(segment)?; + Ok(checkpoint + .and_then(|checkpoint| checkpoint.block_number) + .is_some_and(|n| n >= block) + .then_some(())) + }) + .await +} + /// Smoke test: node boots with `RocksDB` routing enabled. #[tokio::test] async fn test_rocksdb_node_startup() -> Result<()> { @@ -665,12 +685,21 @@ async fn test_rocksdb_account_history_pruning() -> Result<()> { let payload = node.advance_block().await?; assert_eq!(payload.block().number(), nonce + 1); - // Let the persistence cycle (save_blocks → pruner → commit) complete before - // producing the next block, so each cycle processes exactly one block. The - // transaction becomes visible in RocksDB with the commit. + // Wait for the block's save to commit before producing the next block, so each save + // processes exactly one block. The transaction becomes visible in RocksDB with the + // commit, and the persistence service runs the pruner before it handles the next save. poll_tx_in_rocksdb(&node.inner.provider, tx_hash).await?; } + // The pruner runs after the save of the last block is acknowledged, so wait for it to prune + // the blocks outside the retention window. + wait_for_prune_checkpoint( + &node.inner.provider, + PruneSegment::AccountHistory, + TOTAL_BLOCKS - PRUNE_DISTANCE, + ) + .await?; + // Read the AccountsHistory shard for `sender` directly from RocksDB. // This is the data structure corrupted by the race. let rocksdb = node.inner.provider.rocksdb_provider(); @@ -843,6 +872,15 @@ async fn test_rocksdb_storage_history_pruning() -> Result<()> { poll_tx_in_rocksdb(&node.inner.provider, tx_hash).await?; } + // The pruner runs after the save of the last block is acknowledged, so wait for it to prune + // the blocks outside the retention window. + wait_for_prune_checkpoint( + &node.inner.provider, + PruneSegment::StorageHistory, + TOTAL_BLOCKS - PRUNE_DISTANCE, + ) + .await?; + // Read StoragesHistory shard for (contract_address, slot 0) directly from RocksDB let rocksdb = node.inner.provider.rocksdb_provider(); let shards = rocksdb.storage_history_shards(contract_address, storage_slot).unwrap(); diff --git a/crates/ethereum/node/tests/e2e/blobs.rs b/crates/ethereum/node/tests/e2e/blobs.rs index feef4cc18a9..02532194737 100644 --- a/crates/ethereum/node/tests/e2e/blobs.rs +++ b/crates/ethereum/node/tests/e2e/blobs.rs @@ -1,4 +1,4 @@ -use alloy_eips::Decodable2718; +use alloy_eips::{merge::SLOT_DURATION_SECS, Decodable2718}; use reth_chainspec::EthereumHardfork; use reth_e2e_test_utils::{ test_chain_spec, test_chain_spec_builder, transaction::TransactionTestContext, E2ETestSetupExt, @@ -7,10 +7,7 @@ use reth_ethereum_engine_primitives::BlobSidecars; use reth_ethereum_primitives::PooledTransactionVariant; use reth_node_ethereum::EthereumNode; use reth_transaction_pool::TransactionPool; -use std::{ - sync::Arc, - time::{Duration, SystemTime, UNIX_EPOCH}, -}; +use std::sync::Arc; #[tokio::test] async fn can_handle_blobs() -> eyre::Result<()> { @@ -51,7 +48,9 @@ async fn can_handle_blobs() -> eyre::Result<()> { let block_hash = node.submit_payload(payload).await?; node.update_forkchoice(genesis_hash, block_hash).await?; - tokio::time::sleep(std::time::Duration::from_secs(3)).await; + // wait for the pool to process the reorg, and then re-inject the blob tx + node.wait_for_pool_head(block_hash).await?; + node.wait_for_pool(|pool| pool.contains(&blob_tx_hash)).await?; // expects the blob tx to be back in the pool let envelope = node.rpc.envelope_by_hash(blob_tx_hash).await?; @@ -66,7 +65,6 @@ async fn can_send_legacy_sidecar_post_activation() -> eyre::Result<()> { reth_tracing::init_test_tracing(); let chain_spec = test_chain_spec(EthereumHardfork::Osaka); - let genesis_hash = chain_spec.genesis_hash(); let (mut node, wallet) = EthereumNode::test_setup(1, chain_spec) .with_rpc_modifier(|rpc| rpc.with_force_blob_sidecar_upcasting()) .build_single() @@ -90,10 +88,8 @@ async fn can_send_legacy_sidecar_post_activation() -> eyre::Result<()> { // build a payload let blob_payload = node.new_payload().await?; - // submit the blob payload - let blob_block_hash = node.submit_payload(blob_payload).await?; - - node.update_forkchoice(genesis_hash, blob_block_hash).await?; + // import the blob payload + node.import_payload(blob_payload).await?; Ok(()) } @@ -102,25 +98,19 @@ async fn can_send_legacy_sidecar_post_activation() -> eyre::Result<()> { async fn blob_conversion_at_osaka() -> eyre::Result<()> { reth_tracing::init_test_tracing(); - let current_timestamp = SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_secs(); - // Osaka activates in 2 slots - let osaka_timestamp = current_timestamp + 24; + // Keep genesis outside the conversion window. Importing the Prague payload below starts + // conversion two slots before Osaka, after all legacy sidecar assertions have completed. + let prague_timestamp = 2 * SLOT_DURATION_SECS; + let osaka_timestamp = prague_timestamp + 2 * SLOT_DURATION_SECS; let chain_spec = Arc::new( test_chain_spec_builder().prague_activated().with_osaka_at(osaka_timestamp).build(), ); - let genesis_hash = chain_spec.genesis_hash(); let (mut node, wallet) = EthereumNode::test_setup(1, chain_spec) .with_rpc_modifier(|rpc| rpc.with_force_blob_sidecar_upcasting()) .build_single() .await?; - // build a dummy payload at `current_timestamp` - let raw_tx = TransactionTestContext::transfer_tx_bytes(1, wallet.signer(0)).await; - node.rpc.inject_tx(raw_tx).await?; - node.payload.timestamp = current_timestamp - 1; - node.advance_block().await?; - // build blob txs let first_blob = TransactionTestContext::tx_with_blobs_bytes(1, wallet.signer(1)).await?; let second_blob = TransactionTestContext::tx_with_blobs_bytes(1, wallet.signer(2)).await?; @@ -151,8 +141,10 @@ async fn blob_conversion_at_osaka() -> eyre::Result<()> { TransactionTestContext::validate_sidecar(envelope); // build last Prague payload - node.payload.timestamp = current_timestamp + 1; + node.set_next_payload_timestamp(prague_timestamp)?; let prague_payload = node.new_payload().await?; + assert_eq!(prague_payload.block().timestamp, prague_timestamp); + assert!(prague_payload.block().body().transactions().any(|tx| *tx.hash() == blob_tx_hash)); assert!(matches!(prague_payload.sidecars(), BlobSidecars::Eip4844(_))); // inject second blob tx to the pool @@ -164,7 +156,14 @@ async fn blob_conversion_at_osaka() -> eyre::Result<()> { // validate sidecar TransactionTestContext::validate_sidecar(envelope); - tokio::time::sleep(Duration::from_secs(6)).await; + // Import the Prague payload to trigger conversion only after checking both legacy sidecars. + node.import_payload(prague_payload).await?; + + // wait for the pool to convert the sidecar ahead of the Osaka activation + node.wait_for_pool(|pool| { + pool.get_blob(blob_tx_hash).ok().flatten().is_some_and(|sidecar| sidecar.is_eip7594()) + }) + .await?; // fetch second blob tx from rpc again let envelope = node.rpc.envelope_by_hash(blob_tx_hash).await?; @@ -173,18 +172,16 @@ async fn blob_conversion_at_osaka() -> eyre::Result<()> { // validate sidecar TransactionTestContext::validate_sidecar(envelope); - // submit the Prague payload - node.update_forkchoice(genesis_hash, node.submit_payload(prague_payload).await?).await?; - // Build first Osaka payload - node.payload.timestamp = osaka_timestamp - 1; + node.set_next_payload_timestamp(osaka_timestamp)?; let osaka_payload = node.new_payload().await?; + assert_eq!(osaka_payload.block().timestamp, osaka_timestamp); // Assert that it includes the second blob tx with eip7594 sidecar assert!(osaka_payload.block().body().transactions().any(|tx| *tx.hash() == blob_tx_hash)); assert!(matches!(osaka_payload.sidecars(), BlobSidecars::Eip7594(_))); - node.update_forkchoice(genesis_hash, node.submit_payload(osaka_payload).await?).await?; + node.import_payload(osaka_payload).await?; Ok(()) } diff --git a/crates/ethereum/node/tests/e2e/custom_genesis.rs b/crates/ethereum/node/tests/e2e/custom_genesis.rs index 0b11d6a1e14..12127e9143f 100644 --- a/crates/ethereum/node/tests/e2e/custom_genesis.rs +++ b/crates/ethereum/node/tests/e2e/custom_genesis.rs @@ -92,3 +92,32 @@ async fn can_advance_on_genesis_newer_than_payload_timestamp() -> eyre::Result<( Ok(()) } + +/// Tests that the next payload is built at the timestamp set on the test context, that later +/// payloads continue from it, and that timestamps not after the latest block are rejected. +#[tokio::test] +async fn can_set_next_payload_timestamp() -> eyre::Result<()> { + reth_tracing::init_test_tracing(); + + let genesis_timestamp = 2_000_000_000; + let mut genesis = test_genesis(); + genesis.timestamp = genesis_timestamp; + let chain_spec = + Arc::new(test_chain_spec_builder().genesis(genesis).cancun_activated().build()); + + let (mut node, _) = EthereumNode::test_setup(1, chain_spec).build_single().await?; + + assert_eq!( + node.set_next_payload_timestamp(genesis_timestamp).unwrap_err().to_string(), + "next payload timestamp 2000000000 must be greater than the latest block timestamp 2000000000" + ); + + let timestamp = genesis_timestamp + 100; + node.set_next_payload_timestamp(timestamp)?; + assert_eq!(node.advance_block().await?.block().timestamp, timestamp); + assert_eq!(node.advance_block().await?.block().timestamp, timestamp + 1); + + assert!(node.set_next_payload_timestamp(timestamp + 1).is_err()); + + Ok(()) +} diff --git a/crates/ethereum/node/tests/e2e/p2p.rs b/crates/ethereum/node/tests/e2e/p2p.rs index cb076baf906..9aa52c74c78 100644 --- a/crates/ethereum/node/tests/e2e/p2p.rs +++ b/crates/ethereum/node/tests/e2e/p2p.rs @@ -242,7 +242,7 @@ async fn test_long_reorg() -> eyre::Result<()> { second_node.sync_to(head.header.hash).await?; // Produce a fork chain with blocks 21..60 - second_node.payload.timestamp = head.header.timestamp; + second_node.set_next_payload_timestamp(head.header.timestamp + 1)?; advance_with_random_transactions(&mut second_node, 40, &mut rng, true).await?; // Reorg first node from 100th block to new 60th block. @@ -332,7 +332,7 @@ async fn test_reorg_through_backfill() -> eyre::Result<()> { second_node.sync_to(head.header.hash).await?; // Produce an unfinalized fork chain with 30 blocks - second_node.payload.timestamp = head.header.timestamp; + second_node.set_next_payload_timestamp(head.header.timestamp + 1)?; advance_with_random_transactions(&mut second_node, 30, &mut rng, false).await?; // Now reorg second node to the finalized canonical head diff --git a/crates/ethereum/node/tests/e2e/pool.rs b/crates/ethereum/node/tests/e2e/pool.rs index 9b68ff9ff80..56ddcfb703c 100644 --- a/crates/ethereum/node/tests/e2e/pool.rs +++ b/crates/ethereum/node/tests/e2e/pool.rs @@ -289,3 +289,34 @@ async fn maintain_txpool_commit() -> eyre::Result<()> { Ok(()) } + +// Test that the pool processed the new block once `advance_block_synced` returns. +#[tokio::test] +async fn advance_block_synced_waits_for_pool() -> eyre::Result<()> { + reth_tracing::init_test_tracing(); + + let (mut node, wallet) = + EthereumNode::test_setup_for(EthereumHardfork::Cancun).build_single().await?; + + for nonce in 0..3 { + let raw_tx = TransactionTestContext::transfer_tx_bytes_with_nonce( + wallet.chain_id, + wallet.inner.clone(), + nonce, + ) + .await; + let tx_hash = node.rpc.inject_tx(raw_tx).await?; + + let payload = node.advance_block_synced().await?; + let block = payload.block(); + assert!(block.body().transactions().any(|tx| *tx.hash() == tx_hash)); + + // the mined transaction is removed from the pool without waiting any further + let info = node.inner.pool.block_info(); + assert_eq!(info.last_seen_block_hash, block.hash()); + assert_eq!(info.last_seen_block_number, block.header().number); + assert!(node.inner.pool.is_empty()); + } + + Ok(()) +}