From 87be7482a80668dd4165b07310a16e6f8dda607b Mon Sep 17 00:00:00 2001 From: Sergei Shulepov Date: Wed, 12 Aug 2026 01:16:46 -0400 Subject: [PATCH 001/294] fix(network): missed on_bad_message call (#26659) --- crates/net/network/src/session/active.rs | 122 ++++++++++++++++++++++- 1 file changed, 118 insertions(+), 4 deletions(-) diff --git a/crates/net/network/src/session/active.rs b/crates/net/network/src/session/active.rs index d3985f75966..34e22aee46e 100644 --- a/crates/net/network/src/session/active.rs +++ b/crates/net/network/src/session/active.rs @@ -290,6 +290,12 @@ impl ActiveSession { self.update_request_timeout(req.timestamp, Instant::now()); } RequestState::Waiting(request) => { + // The peer replied to a request id we handed out, but with the wrong + // response type. This cancels the pending request, so it must cost the + // peer reputation. Without the penalty the peer can kill any request we + // send it, repeatedly and for free. + debug!(target: "net::session", ?request_id, remote_peer_id=?self.remote_peer_id, "received response of wrong type"); + self.on_bad_message(); request.send_bad_response(); } RequestState::TimedOut => { @@ -475,7 +481,11 @@ impl ActiveSession { } } RequestState::Waiting(request) => { - // A different PeerRequest kind was pending for this id. + // A different PeerRequest kind was pending for this id: the peer answered an `eth` + // request with a `snap` message. Same as the wrong-type case above, this cancels + // the pending request and must be penalized. + debug!(target: "net::session", ?request_id, msg_id=?msg.message_id(), remote_peer_id=?self.remote_peer_id, "received snap response for non-snap request"); + self.on_bad_message(); request.send_bad_response(); } RequestState::TimedOut => { @@ -1316,9 +1326,10 @@ mod tests { use reth_chainspec::MAINNET; use reth_ecies::stream::ECIESStream; use reth_eth_wire::{ - handshake::EthHandshake, protocol::Protocol, EthNetworkPrimitives, EthStream, - GetBlockAccessLists, GetBlockBodies, HelloMessageWithProtocols, P2PStream, StatusBuilder, - UnauthedEthStream, UnauthedP2PStream, UnifiedStatus, + handshake::EthHandshake, protocol::Protocol, BlockBodies, BlockHeaders, + EthNetworkPrimitives, EthStream, GetBlockAccessLists, GetBlockBodies, + HelloMessageWithProtocols, P2PStream, StatusBuilder, UnauthedEthStream, UnauthedP2PStream, + UnifiedStatus, }; use reth_eth_wire_types::{ message::MAX_MESSAGE_SIZE, @@ -1533,6 +1544,22 @@ mod tests { (id, rx) } + /// Dispatches an `eth` `GetBlockBodies` request via + /// [`ActiveSession::on_internal_peer_request`] and returns the session-assigned request id plus + /// the caller's response receiver. + fn dispatch_block_bodies_request( + session: &mut ActiveSession, + ) -> (u64, oneshot::Receiver>) { + let (response, rx) = oneshot::channel(); + let deadline = session.request_deadline(); + session.on_internal_peer_request( + PeerRequest::GetBlockBodies { request: GetBlockBodies(Vec::new()), response }, + deadline, + ); + let id = *session.inflight_requests.keys().next().expect("eth request tracked"); + (id, rx) + } + #[tokio::test(flavor = "multi_thread")] async fn test_disconnect() { let mut builder = SessionBuilder::default(); @@ -1851,6 +1878,93 @@ mod tests { )); } + #[tokio::test(flavor = "multi_thread")] + async fn wrong_type_eth_response_is_penalized() { + let mut builder = SessionBuilder::default(); + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let local_addr = listener.local_addr().unwrap(); + let fut = builder.with_client_stream(local_addr, async move |client_stream| { + let _client_stream = client_stream; + tokio::time::sleep(Duration::from_secs(60)).await; + }); + tokio::task::spawn(fut); + let (incoming, _) = listener.accept().await.unwrap(); + let mut session = builder.connect_incoming(incoming).await; + + let (id, rx) = dispatch_block_bodies_request(&mut session); + + // The peer knows this id because we sent it. Answering the GetBlockBodies with BlockHeaders + // cancels our request, so it must cost the peer reputation, otherwise the peer can kill + // every request we make for free. + let outcome = session.on_incoming_message(EthMessage::BlockHeaders(RequestPair { + request_id: id, + message: BlockHeaders(Vec::new()), + })); + assert!(matches!(outcome, OnIncomingMessageOutcome::Ok)); + assert!(!session.inflight_requests.contains_key(&id)); + assert_eq!(rx.await.unwrap().unwrap_err(), RequestError::BadResponse); + assert!(matches!( + futures::FutureExt::now_or_never(builder.active_session_rx.next()).flatten(), + Some(ActiveSessionMessage::BadMessage { .. }) + )); + } + + #[tokio::test(flavor = "multi_thread")] + async fn matching_eth_response_is_not_penalized() { + let mut builder = SessionBuilder::default(); + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let local_addr = listener.local_addr().unwrap(); + let fut = builder.with_client_stream(local_addr, async move |client_stream| { + let _client_stream = client_stream; + tokio::time::sleep(Duration::from_secs(60)).await; + }); + tokio::task::spawn(fut); + let (incoming, _) = listener.accept().await.unwrap(); + let mut session = builder.connect_incoming(incoming).await; + + let (id, rx) = dispatch_block_bodies_request(&mut session); + + let outcome = session.on_incoming_message(EthMessage::BlockBodies(RequestPair { + request_id: id, + message: BlockBodies(Vec::new()), + })); + assert!(matches!(outcome, OnIncomingMessageOutcome::Ok)); + assert!(!session.inflight_requests.contains_key(&id)); + assert!(rx.await.unwrap().is_ok()); + assert!(futures::FutureExt::now_or_never(builder.active_session_rx.next()) + .flatten() + .is_none()); + } + + #[tokio::test(flavor = "multi_thread")] + async fn snap_response_to_eth_request_is_penalized() { + let mut builder = snap_session_builder(); + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let local_addr = listener.local_addr().unwrap(); + let fut = builder.with_client_stream(local_addr, async move |client_stream| { + let _client_stream = client_stream; + tokio::time::sleep(Duration::from_secs(60)).await; + }); + tokio::task::spawn(fut); + let (incoming, _) = listener.accept().await.unwrap(); + let mut session = builder.connect_incoming(incoming).await; + + // `eth` and `snap/2` requests share one id space, so a snap response can land on a pending + // eth request. It cancels that request just the same and is penalized the same way. + let (id, rx) = dispatch_block_bodies_request(&mut session); + + let outcome = session.on_incoming_snap_message(SnapProtocolMessage::AccountRange( + AccountRangeMessage { request_id: id, accounts: Vec::new(), proof: Vec::new() }, + )); + assert!(matches!(outcome, OnIncomingMessageOutcome::Ok)); + assert!(!session.inflight_requests.contains_key(&id)); + assert_eq!(rx.await.unwrap().unwrap_err(), RequestError::BadResponse); + assert!(matches!( + futures::FutureExt::now_or_never(builder.active_session_rx.next()).flatten(), + Some(ActiveSessionMessage::BadMessage { .. }) + )); + } + #[tokio::test(flavor = "multi_thread")] async fn get_snap_request_rejected_without_negotiated_snap() { // A plain `eth`-only session: no `snap/2` was negotiated. From 0c3efffa90215400426d0f6dedf81a7a1475b7ee Mon Sep 17 00:00:00 2001 From: Soubhik Singha Mahapatra <160333583+Soubhik-10@users.noreply.github.com> Date: Wed, 12 Aug 2026 12:49:58 +0530 Subject: [PATCH 002/294] fix(engine): handle little-endian cell bitvectors (#26650) --- crates/net/network-api/src/custody.rs | 18 +++++++++++ crates/rpc/rpc-engine-api/src/engine_api.rs | 35 ++++++++++++++++----- 2 files changed, 46 insertions(+), 7 deletions(-) diff --git a/crates/net/network-api/src/custody.rs b/crates/net/network-api/src/custody.rs index fe2f47e7766..ba7cb9bfb71 100644 --- a/crates/net/network-api/src/custody.rs +++ b/crates/net/network-api/src/custody.rs @@ -51,6 +51,11 @@ impl CellCustody { } } } + + /// Updates the custody bitmap from the Engine API's little-endian bitvector encoding. + pub fn set_from_engine_api(&self, custody_columns: B128) { + self.set(B128::from(u128::from_le_bytes(custody_columns.into()))); + } } const CELL_CUSTODY_NONE: u8 = 0; @@ -63,3 +68,16 @@ struct CellCustodyInner { high: AtomicU64, low: AtomicU64, } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn sets_engine_api_bitvector_as_numeric_mask() { + let custody = CellCustody::default(); + custody.set_from_engine_api(B128::from(0b1010u128.to_le_bytes())); + + assert_eq!(custody.get(), B128::from(0b1010u128)); + } +} diff --git a/crates/rpc/rpc-engine-api/src/engine_api.rs b/crates/rpc/rpc-engine-api/src/engine_api.rs index 8053becc1f6..639ef9340a8 100644 --- a/crates/rpc/rpc-engine-api/src/engine_api.rs +++ b/crates/rpc/rpc-engine-api/src/engine_api.rs @@ -395,7 +395,7 @@ where custody_columns: Option, ) -> EngineApiResult { if let Some(custody_columns) = custody_columns { - self.inner.cell_custody.set(custody_columns); + self.inner.cell_custody.set_from_engine_api(custody_columns); } self.validate_and_execute_forkchoice(EngineApiMessageVersion::V4, state, payload_attrs) .await @@ -1065,6 +1065,9 @@ where versioned_hashes: Vec, indices_bitarray: B128, ) -> EngineApiResult>>> { + // Engine API bitvectors are little-endian, while B128 integer conversions are + // big-endian. The transaction pool uses the latter representation as a numeric cell mask. + let indices_bitarray = B128::from(u128::from_le_bytes(indices_bitarray.into())); let current_timestamp = SystemTime::now().duration_since(SystemTime::UNIX_EPOCH).unwrap_or_default().as_secs(); if !self.inner.chain_spec.is_amsterdam_active_at_timestamp(current_timestamp) { @@ -1994,10 +1997,26 @@ mod tests { TestNetworkInfo { syncing: true }, ); - let res = api.get_blobs_v4_metered(vec![B256::ZERO], B128::from(1u128)); + let res = api.get_blobs_v4_metered(vec![B256::ZERO], B128::from(1u128.to_le_bytes())); assert_matches!(res, Ok(None)); } + #[test] + fn engine_bitvector_uses_little_endian_cell_indices() { + assert_eq!( + B128::from(u128::from_le_bytes(B128::from(1u128.to_le_bytes()).into())), + B128::from(1u128) + ); + assert_eq!( + B128::from(u128::from_le_bytes(B128::from((1u128 << 127).to_le_bytes()).into())), + B128::from(1u128 << 127) + ); + assert_eq!( + B128::from(u128::from_le_bytes(B128::from(((1u128 << 64) - 1).to_le_bytes()).into(),)), + B128::from((1u128 << 64) - 1) + ); + } + #[tokio::test] async fn fcu_v4_updates_shared_cell_custody_before_forkchoice_result() { let chain_spec: Arc = @@ -2032,7 +2051,8 @@ mod tests { safe_block_hash: B256::ZERO, finalized_block_hash: B256::ZERO, }; - let custody_columns = B128::from(0b1010u128); + let custody_columns = B128::from(0b1010u128.to_le_bytes()); + let expected_custody_columns = B128::from(0b1010u128); let api_task = tokio::spawn(async move { api.fork_choice_updated_v4(state, None, Some(custody_columns)).await @@ -2049,7 +2069,7 @@ mod tests { } other => panic!("unexpected engine message: {other:?}"), }; - assert_eq!(cell_custody.get(), custody_columns); + assert_eq!(cell_custody.get(), expected_custody_columns); response_tx .send(Ok(OnForkChoiceUpdated::valid(PayloadStatus::from_status( @@ -2061,7 +2081,7 @@ mod tests { .await .expect("api task should not panic") .expect("forkchoiceUpdatedV4 should succeed"); - assert_eq!(cell_custody.get(), custody_columns); + assert_eq!(cell_custody.get(), expected_custody_columns); } #[tokio::test] @@ -2107,7 +2127,8 @@ mod tests { slot_number: None, target_gas_limit: None, }; - let custody_columns = B128::from(0b1010u128); + let custody_columns = B128::from(0b1010u128.to_le_bytes()); + let expected_custody_columns = B128::from(0b1010u128); let api_task = tokio::spawn(async move { api.fork_choice_updated_v4(state, Some(payload_attributes), Some(custody_columns)).await @@ -2127,7 +2148,7 @@ mod tests { } other => panic!("unexpected engine message: {other:?}"), }; - assert_eq!(cell_custody.get(), custody_columns); + assert_eq!(cell_custody.get(), expected_custody_columns); response_tx .send(Ok(OnForkChoiceUpdated::valid(PayloadStatus::from_status( From f59df2809316d9417fda6f0b05fc28adcd7d416c Mon Sep 17 00:00:00 2001 From: Derek Cofausper <256792747+decofe@users.noreply.github.com> Date: Wed, 12 Aug 2026 13:07:39 +0200 Subject: [PATCH 003/294] ci(docker): build partial persistence nightly image (#26662) Co-authored-by: Alexey Shekhirin <5773434+shekhirin@users.noreply.github.com> --- .github/scripts/verify_image_arch.sh | 1 + .github/workflows/docker.yml | 1 + docker-bake.hcl | 12 +++++++++++- 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/.github/scripts/verify_image_arch.sh b/.github/scripts/verify_image_arch.sh index 9746b97edd2..842e3352b18 100755 --- a/.github/scripts/verify_image_arch.sh +++ b/.github/scripts/verify_image_arch.sh @@ -42,6 +42,7 @@ verify_image() { if [[ "$TARGETS" == *"nightly"* ]]; then verify_image "${REGISTRY}/reth:nightly" amd64 arm64 + verify_image "${REGISTRY}/reth:nightly-partial-persistence" amd64 arm64 verify_image "${REGISTRY}/reth:nightly-profiling" amd64 verify_image "${REGISTRY}/reth:nightly-edge-profiling" amd64 else diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 5441c81cfd9..83b97c5e53b 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -96,6 +96,7 @@ jobs: echo "ethereum.tags=${REGISTRY}/reth:nightly" echo "ethereum.args.RETH_ENGINE_TXPOOL_PREWARMING=true" echo "ethereum.args.RETH_ENGINE_SENDER_RECOVERY_CACHE=true" + echo "ethereum-partial-persistence.tags=${REGISTRY}/reth:nightly-partial-persistence" echo "EOF" } >> "$GITHUB_OUTPUT" diff --git a/docker-bake.hcl b/docker-bake.hcl index 6078af6b74e..82b8992a63e 100644 --- a/docker-bake.hcl +++ b/docker-bake.hcl @@ -35,7 +35,7 @@ group "default" { } group "nightly" { - targets = ["ethereum", "ethereum-profiling"] + targets = ["ethereum", "ethereum-profiling", "ethereum-partial-persistence"] } // Base target with shared configuration @@ -71,6 +71,16 @@ target "ethereum" { tags = ["${REGISTRY}/reth:${TAG}"] } +target "ethereum-partial-persistence" { + inherits = ["_base"] + args = { + BINARY = "reth" + MANIFEST_PATH = "bin/reth" + FEATURES = "partial-persistence" + } + tags = ["${REGISTRY}/reth:nightly-partial-persistence"] +} + target "ethereum-profiling" { inherits = ["_base_profiling"] args = { From 390b050c64a8d5d42c70d15c7d74d42f73943b8c Mon Sep 17 00:00:00 2001 From: Emma Jamieson-Hoare Date: Wed, 12 Aug 2026 14:53:55 +0100 Subject: [PATCH 004/294] chore: bump reth version to 2.5.0 (#26663) --- Cargo.lock | 228 +++++++++++++++++++-------------------- Cargo.toml | 2 +- docs/vocs/vocs.config.ts | 2 +- 3 files changed, 116 insertions(+), 116 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b33be037034..f84bacfb442 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3269,7 +3269,7 @@ dependencies = [ [[package]] name = "ef-test-runner" -version = "2.4.1" +version = "2.5.0" dependencies = [ "clap", "ef-tests", @@ -3277,7 +3277,7 @@ dependencies = [ [[package]] name = "ef-tests" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-consensus", "alloy-eips", @@ -3730,7 +3730,7 @@ dependencies = [ [[package]] name = "example-full-contract-state" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-primitives", "eyre", @@ -3848,7 +3848,7 @@ dependencies = [ [[package]] name = "exex-subscription" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-primitives", "clap", @@ -7661,7 +7661,7 @@ checksum = "1e061d1b48cb8d38042de4ae0a7a6401009d6143dc80d2e2d6f31f0bdd6470c7" [[package]] name = "reth" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-node-bindings", "alloy-primitives", @@ -7711,7 +7711,7 @@ dependencies = [ [[package]] name = "reth-basic-payload-builder" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-consensus", "alloy-eips", @@ -7737,7 +7737,7 @@ dependencies = [ [[package]] name = "reth-bb" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-consensus", "alloy-eips", @@ -7784,7 +7784,7 @@ dependencies = [ [[package]] name = "reth-chain-state" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-consensus", "alloy-eips", @@ -7815,7 +7815,7 @@ dependencies = [ [[package]] name = "reth-chainspec" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-chains", "alloy-consensus", @@ -7835,7 +7835,7 @@ dependencies = [ [[package]] name = "reth-cli" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-genesis", "clap", @@ -7847,7 +7847,7 @@ dependencies = [ [[package]] name = "reth-cli-commands" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-chains", "alloy-consensus", @@ -7937,7 +7937,7 @@ dependencies = [ [[package]] name = "reth-cli-runner" -version = "2.4.1" +version = "2.5.0" dependencies = [ "reth-tasks", "tokio", @@ -7946,7 +7946,7 @@ dependencies = [ [[package]] name = "reth-cli-util" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-eips", "alloy-primitives", @@ -8000,7 +8000,7 @@ dependencies = [ [[package]] name = "reth-config" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-primitives", "eyre", @@ -8018,7 +8018,7 @@ dependencies = [ [[package]] name = "reth-consensus" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-consensus", "alloy-eip7928", @@ -8031,7 +8031,7 @@ dependencies = [ [[package]] name = "reth-consensus-common" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-consensus", "alloy-eips", @@ -8045,7 +8045,7 @@ dependencies = [ [[package]] name = "reth-consensus-debug-client" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-consensus", "alloy-eips", @@ -8069,7 +8069,7 @@ dependencies = [ [[package]] name = "reth-db" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-consensus", "alloy-primitives", @@ -8105,7 +8105,7 @@ dependencies = [ [[package]] name = "reth-db-api" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-consensus", "alloy-primitives", @@ -8133,7 +8133,7 @@ dependencies = [ [[package]] name = "reth-db-common" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-consensus", "alloy-genesis", @@ -8164,7 +8164,7 @@ dependencies = [ [[package]] name = "reth-db-models" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-eips", "alloy-primitives", @@ -8180,7 +8180,7 @@ dependencies = [ [[package]] name = "reth-discv4" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-primitives", "alloy-rlp", @@ -8206,7 +8206,7 @@ dependencies = [ [[package]] name = "reth-discv5" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-primitives", "alloy-rlp", @@ -8231,7 +8231,7 @@ dependencies = [ [[package]] name = "reth-dns-discovery" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-chains", "alloy-primitives", @@ -8259,7 +8259,7 @@ dependencies = [ [[package]] name = "reth-downloaders" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-consensus", "alloy-eips", @@ -8297,7 +8297,7 @@ dependencies = [ [[package]] name = "reth-e2e-test-utils" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-consensus", "alloy-eips", @@ -8353,7 +8353,7 @@ dependencies = [ [[package]] name = "reth-ecies" -version = "2.4.1" +version = "2.5.0" dependencies = [ "aes", "alloy-primitives", @@ -8379,7 +8379,7 @@ dependencies = [ [[package]] name = "reth-engine-local" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-consensus", "alloy-primitives", @@ -8401,7 +8401,7 @@ dependencies = [ [[package]] name = "reth-engine-primitives" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-consensus", "alloy-eips", @@ -8427,7 +8427,7 @@ dependencies = [ [[package]] name = "reth-engine-tree" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-consensus", "alloy-eip7928", @@ -8497,7 +8497,7 @@ dependencies = [ [[package]] name = "reth-engine-util" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-consensus", "alloy-primitives", @@ -8526,7 +8526,7 @@ dependencies = [ [[package]] name = "reth-era" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-consensus", "alloy-eips", @@ -8551,7 +8551,7 @@ dependencies = [ [[package]] name = "reth-era-downloader" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-primitives", "bytes", @@ -8569,7 +8569,7 @@ dependencies = [ [[package]] name = "reth-era-utils" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-consensus", "alloy-primitives", @@ -8600,7 +8600,7 @@ dependencies = [ [[package]] name = "reth-errors" -version = "2.4.1" +version = "2.5.0" dependencies = [ "reth-consensus", "reth-execution-errors", @@ -8610,7 +8610,7 @@ dependencies = [ [[package]] name = "reth-eth-wire" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-chains", "alloy-consensus", @@ -8648,7 +8648,7 @@ dependencies = [ [[package]] name = "reth-eth-wire-types" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-chains", "alloy-consensus", @@ -8676,7 +8676,7 @@ dependencies = [ [[package]] name = "reth-ethereum" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-rpc-types-engine", "alloy-rpc-types-eth", @@ -8716,7 +8716,7 @@ dependencies = [ [[package]] name = "reth-ethereum-cli" -version = "2.4.1" +version = "2.5.0" dependencies = [ "clap", "eyre", @@ -8739,7 +8739,7 @@ dependencies = [ [[package]] name = "reth-ethereum-consensus" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-consensus", "alloy-eips", @@ -8755,7 +8755,7 @@ dependencies = [ [[package]] name = "reth-ethereum-engine-primitives" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-eips", "alloy-primitives", @@ -8771,7 +8771,7 @@ dependencies = [ [[package]] name = "reth-ethereum-forks" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-eip2124", "alloy-hardforks 0.4.7", @@ -8784,7 +8784,7 @@ dependencies = [ [[package]] name = "reth-ethereum-payload-builder" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-consensus", "alloy-eips", @@ -8813,7 +8813,7 @@ dependencies = [ [[package]] name = "reth-ethereum-primitives" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-consensus", "alloy-eips", @@ -8834,7 +8834,7 @@ dependencies = [ [[package]] name = "reth-etl" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-primitives", "rayon", @@ -8844,7 +8844,7 @@ dependencies = [ [[package]] name = "reth-evm" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-consensus", "alloy-eip7928", @@ -8870,7 +8870,7 @@ dependencies = [ [[package]] name = "reth-evm-ethereum" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-consensus", "alloy-eips", @@ -8895,7 +8895,7 @@ dependencies = [ [[package]] name = "reth-execution-cache" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-primitives", "fixed-cache", @@ -8913,7 +8913,7 @@ dependencies = [ [[package]] name = "reth-execution-errors" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-evm", "alloy-primitives", @@ -8925,7 +8925,7 @@ dependencies = [ [[package]] name = "reth-execution-types" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-consensus", "alloy-eips", @@ -8946,7 +8946,7 @@ dependencies = [ [[package]] name = "reth-exex" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-consensus", "alloy-eips", @@ -8991,7 +8991,7 @@ dependencies = [ [[package]] name = "reth-exex-test-utils" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-eips", "eyre", @@ -9022,7 +9022,7 @@ dependencies = [ [[package]] name = "reth-exex-types" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-eips", "alloy-primitives", @@ -9039,7 +9039,7 @@ dependencies = [ [[package]] name = "reth-fs-util" -version = "2.4.1" +version = "2.5.0" dependencies = [ "serde", "serde_json", @@ -9048,7 +9048,7 @@ dependencies = [ [[package]] name = "reth-invalid-block-hooks" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-consensus", "alloy-eips", @@ -9079,7 +9079,7 @@ dependencies = [ [[package]] name = "reth-ipc" -version = "2.4.1" +version = "2.5.0" dependencies = [ "bytes", "futures", @@ -9101,7 +9101,7 @@ dependencies = [ [[package]] name = "reth-libmdbx" -version = "2.4.1" +version = "2.5.0" dependencies = [ "bitflags 2.13.0", "byteorder", @@ -9118,7 +9118,7 @@ dependencies = [ [[package]] name = "reth-mdbx-sys" -version = "2.4.1" +version = "2.5.0" dependencies = [ "bindgen", "cc", @@ -9126,7 +9126,7 @@ dependencies = [ [[package]] name = "reth-metrics" -version = "2.4.1" +version = "2.5.0" dependencies = [ "futures", "libc", @@ -9139,7 +9139,7 @@ dependencies = [ [[package]] name = "reth-net-banlist" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-primitives", "ipnet", @@ -9147,7 +9147,7 @@ dependencies = [ [[package]] name = "reth-net-nat" -version = "2.4.1" +version = "2.5.0" dependencies = [ "futures-util", "if-addrs", @@ -9161,7 +9161,7 @@ dependencies = [ [[package]] name = "reth-network" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-consensus", "alloy-eip7928", @@ -9230,7 +9230,7 @@ dependencies = [ [[package]] name = "reth-network-api" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-consensus", "alloy-primitives", @@ -9254,7 +9254,7 @@ dependencies = [ [[package]] name = "reth-network-p2p" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-consensus", "alloy-eip7928", @@ -9278,7 +9278,7 @@ dependencies = [ [[package]] name = "reth-network-peers" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-primitives", "alloy-rlp", @@ -9295,7 +9295,7 @@ dependencies = [ [[package]] name = "reth-network-types" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-eip2124", "humantime-serde", @@ -9308,7 +9308,7 @@ dependencies = [ [[package]] name = "reth-nippy-jar" -version = "2.4.1" +version = "2.5.0" dependencies = [ "anyhow", "bincode", @@ -9326,7 +9326,7 @@ dependencies = [ [[package]] name = "reth-node-api" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-rpc-types-engine", "eyre", @@ -9349,7 +9349,7 @@ dependencies = [ [[package]] name = "reth-node-builder" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-consensus", "alloy-eips", @@ -9420,7 +9420,7 @@ dependencies = [ [[package]] name = "reth-node-core" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-consensus", "alloy-eips", @@ -9476,7 +9476,7 @@ dependencies = [ [[package]] name = "reth-node-ethereum" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-consensus", "alloy-contract", @@ -9545,7 +9545,7 @@ dependencies = [ [[package]] name = "reth-node-ethstats" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-consensus", "alloy-primitives", @@ -9568,7 +9568,7 @@ dependencies = [ [[package]] name = "reth-node-events" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-consensus", "alloy-eips", @@ -9591,7 +9591,7 @@ dependencies = [ [[package]] name = "reth-node-metrics" -version = "2.4.1" +version = "2.5.0" dependencies = [ "bytes", "eyre", @@ -9620,7 +9620,7 @@ dependencies = [ [[package]] name = "reth-node-types" -version = "2.4.1" +version = "2.5.0" dependencies = [ "reth-chainspec", "reth-db-api", @@ -9631,7 +9631,7 @@ dependencies = [ [[package]] name = "reth-payload-builder" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-consensus", "alloy-primitives", @@ -9654,7 +9654,7 @@ dependencies = [ [[package]] name = "reth-payload-builder-primitives" -version = "2.4.1" +version = "2.5.0" dependencies = [ "pin-project", "reth-payload-primitives", @@ -9665,7 +9665,7 @@ dependencies = [ [[package]] name = "reth-payload-primitives" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-consensus", "alloy-eips", @@ -9689,7 +9689,7 @@ dependencies = [ [[package]] name = "reth-payload-util" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-consensus", "alloy-primitives", @@ -9698,7 +9698,7 @@ dependencies = [ [[package]] name = "reth-payload-validator" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-consensus", "alloy-rpc-types-engine", @@ -9740,7 +9740,7 @@ dependencies = [ [[package]] name = "reth-provider" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-consensus", "alloy-eip7928", @@ -9794,7 +9794,7 @@ dependencies = [ [[package]] name = "reth-prune" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-consensus", "alloy-primitives", @@ -9825,7 +9825,7 @@ dependencies = [ [[package]] name = "reth-prune-types" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-primitives", "arbitrary", @@ -9845,7 +9845,7 @@ dependencies = [ [[package]] name = "reth-revm" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-consensus", "alloy-primitives", @@ -9861,7 +9861,7 @@ dependencies = [ [[package]] name = "reth-rpc" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-consensus", "alloy-dyn-abi", @@ -9941,7 +9941,7 @@ dependencies = [ [[package]] name = "reth-rpc-api" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-eips", "alloy-genesis", @@ -9971,7 +9971,7 @@ dependencies = [ [[package]] name = "reth-rpc-builder" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-eips", "alloy-network", @@ -10029,7 +10029,7 @@ dependencies = [ [[package]] name = "reth-rpc-convert" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-consensus", "alloy-evm", @@ -10049,7 +10049,7 @@ dependencies = [ [[package]] name = "reth-rpc-e2e-tests" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-genesis", "alloy-rpc-types-engine", @@ -10069,7 +10069,7 @@ dependencies = [ [[package]] name = "reth-rpc-engine-api" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-eips", "alloy-primitives", @@ -10105,7 +10105,7 @@ dependencies = [ [[package]] name = "reth-rpc-eth-api" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-consensus", "alloy-dyn-abi", @@ -10151,7 +10151,7 @@ dependencies = [ [[package]] name = "reth-rpc-eth-types" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-chains", "alloy-consensus", @@ -10203,7 +10203,7 @@ dependencies = [ [[package]] name = "reth-rpc-layer" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-rpc-types-engine", "http", @@ -10220,7 +10220,7 @@ dependencies = [ [[package]] name = "reth-rpc-server-types" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-eips", "alloy-primitives", @@ -10250,7 +10250,7 @@ dependencies = [ [[package]] name = "reth-stages" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-consensus", "alloy-eips", @@ -10312,7 +10312,7 @@ dependencies = [ [[package]] name = "reth-stages-api" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-eips", "alloy-primitives", @@ -10346,7 +10346,7 @@ dependencies = [ [[package]] name = "reth-stages-types" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-primitives", "arbitrary", @@ -10362,7 +10362,7 @@ dependencies = [ [[package]] name = "reth-static-file" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-primitives", "assert_matches", @@ -10385,7 +10385,7 @@ dependencies = [ [[package]] name = "reth-static-file-types" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-primitives", "clap", @@ -10403,7 +10403,7 @@ dependencies = [ [[package]] name = "reth-storage-api" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-consensus", "alloy-eip7928", @@ -10430,7 +10430,7 @@ dependencies = [ [[package]] name = "reth-storage-errors" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-eips", "alloy-primitives", @@ -10446,7 +10446,7 @@ dependencies = [ [[package]] name = "reth-storage-overlay" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-consensus", "alloy-eips", @@ -10475,7 +10475,7 @@ dependencies = [ [[package]] name = "reth-storage-rpc-provider" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-consensus", "alloy-eips", @@ -10504,7 +10504,7 @@ dependencies = [ [[package]] name = "reth-tasks" -version = "2.4.1" +version = "2.5.0" dependencies = [ "crossbeam-utils", "dashmap", @@ -10524,7 +10524,7 @@ dependencies = [ [[package]] name = "reth-testing-utils" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-consensus", "alloy-eips", @@ -10540,7 +10540,7 @@ dependencies = [ [[package]] name = "reth-tokio-util" -version = "2.4.1" +version = "2.5.0" dependencies = [ "tokio", "tokio-stream", @@ -10549,7 +10549,7 @@ dependencies = [ [[package]] name = "reth-tracing" -version = "2.4.1" +version = "2.5.0" dependencies = [ "clap", "eyre", @@ -10568,7 +10568,7 @@ dependencies = [ [[package]] name = "reth-tracing-otlp" -version = "2.4.1" +version = "2.5.0" dependencies = [ "base64 0.22.1", "clap", @@ -10586,7 +10586,7 @@ dependencies = [ [[package]] name = "reth-transaction-pool" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-consensus", "alloy-eips", @@ -10637,7 +10637,7 @@ dependencies = [ [[package]] name = "reth-trie" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-consensus", "alloy-eips", @@ -10669,7 +10669,7 @@ dependencies = [ [[package]] name = "reth-trie-common" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-consensus", "alloy-eips", @@ -10701,7 +10701,7 @@ dependencies = [ [[package]] name = "reth-trie-db" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-consensus", "alloy-primitives", @@ -10728,7 +10728,7 @@ dependencies = [ [[package]] name = "reth-trie-parallel" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-evm", "alloy-primitives", @@ -10756,7 +10756,7 @@ dependencies = [ [[package]] name = "reth-trie-sparse" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-primitives", "alloy-rlp", diff --git a/Cargo.toml b/Cargo.toml index b23f6556a63..c82dd492175 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,5 +1,5 @@ [workspace.package] -version = "2.4.1" +version = "2.5.0" edition = "2024" rust-version = "1.95" license = "MIT OR Apache-2.0" diff --git a/docs/vocs/vocs.config.ts b/docs/vocs/vocs.config.ts index bb8a8fba54b..29e1e41dae4 100644 --- a/docs/vocs/vocs.config.ts +++ b/docs/vocs/vocs.config.ts @@ -23,7 +23,7 @@ export default defineConfig({ { text: 'Rustdocs', link: '/docs' }, { text: 'GitHub', link: 'https://github.com/paradigmxyz/reth' }, { - text: 'v2.4.1', + text: 'v2.5.0', items: [ { text: 'Releases', From 057a055c4e422de53798661c5ba21baced903932 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Wed, 12 Aug 2026 16:07:36 +0200 Subject: [PATCH 005/294] chore(deps): update Alloy to 2.4.0 (#26666) --- Cargo.lock | 161 +++++++++++++++++++++++++---------------------------- Cargo.toml | 58 +++++++++---------- 2 files changed, 106 insertions(+), 113 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f84bacfb442..4b86aaa27e9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -121,9 +121,9 @@ dependencies = [ [[package]] name = "alloy-consensus" -version = "2.3.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51b9eb70841f76b69b1205bb57b551b7d7afd8407ad57cbda02e833877507c16" +checksum = "99bdf6a3e99089d97e137009956928b41d645035b220cb303eae88182882e9ca" dependencies = [ "alloy-eips", "alloy-primitives", @@ -149,9 +149,9 @@ dependencies = [ [[package]] name = "alloy-consensus-any" -version = "2.3.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f89486e1bd2c5c56522cf18e73460c3fe7b54d50f5e80131466e62b7632edfcb" +checksum = "18d98872f189ea87063912de45e7e0d6ae44700a8836f8bfa79b3265f0c38bfe" dependencies = [ "alloy-consensus", "alloy-eips", @@ -164,9 +164,9 @@ dependencies = [ [[package]] name = "alloy-contract" -version = "2.3.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4c0dadb2468adc8aaaaad4c9a6cfa6bd055fe6596c40017067b37671f2c45f5" +checksum = "f69e65d1b6adc7e46c0e93c2cf3cb4e8646e66a2ced4ba38d542b724acd54774" dependencies = [ "alloy-consensus", "alloy-dyn-abi", @@ -182,7 +182,9 @@ dependencies = [ "futures-util", "serde_json", "thiserror 2.0.18", + "tokio", "tracing", + "wasmtimer", ] [[package]] @@ -265,9 +267,9 @@ dependencies = [ [[package]] name = "alloy-eips" -version = "2.3.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fddc9c1868d6331b56926de8ab4c4a80d8dcb7bc4f8f7bebc6a89d0548f79af2" +checksum = "74bffa6f6440f1ee3c8cd7bc9dc2b01fd837641ef9bd566bd9f8922e75ff8e6c" dependencies = [ "alloy-eip2124", "alloy-eip2930", @@ -311,9 +313,9 @@ dependencies = [ [[package]] name = "alloy-genesis" -version = "2.3.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "921b3dccecc4e9dd571c9f9a728d913c2fc7a8ecdfce6005ed99751817523d0d" +checksum = "f48592ff562892af6a2ebb6e214ed1d2795a6c1dc601c601ef0863583c799003" dependencies = [ "alloy-eips", "alloy-primitives", @@ -365,9 +367,9 @@ dependencies = [ [[package]] name = "alloy-json-rpc" -version = "2.3.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "989d701ece9bccec3294991ce4b4a39dd1cecd0979ea58e86b9878679591b4f5" +checksum = "31de46f2b1948ac9e378212fd79cb963745cbb29cfe08a13c5b675d7e0934a24" dependencies = [ "alloy-primitives", "alloy-sol-types", @@ -380,9 +382,9 @@ dependencies = [ [[package]] name = "alloy-network" -version = "2.3.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "acfca49fbbe3b30de8114b9cfd89029f87cb33ca7e8a5f5f48a809caaf085126" +checksum = "fa3ea75a876a4cd302c15dad628cc55f1f0b2db7fd4865f57f4e24cea2629298" dependencies = [ "alloy-consensus", "alloy-consensus-any", @@ -406,9 +408,9 @@ dependencies = [ [[package]] name = "alloy-network-primitives" -version = "2.3.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2af7b76520abc91b9e0f1aa987d42df55a66494f94a1881cdadd9b8f67e0b924" +checksum = "f93f09f2bee068dcd80bfdf057109d64e37b318f21d3d64851139ca741366dd8" dependencies = [ "alloy-consensus", "alloy-eips", @@ -419,9 +421,9 @@ dependencies = [ [[package]] name = "alloy-node-bindings" -version = "2.3.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48b81973f768c49fe233d6e43da093765d60c1f9bcc81c82ba4ef31c8e273b4b" +checksum = "623d5c6b07f1adf11940eff4b8e665d9b7d9db969fbfe872816878906b267152" dependencies = [ "alloy-genesis", "alloy-hardforks 0.2.13", @@ -473,9 +475,9 @@ dependencies = [ [[package]] name = "alloy-provider" -version = "2.3.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a3fb53b31526b0f97cdbfaf9d8bb72b3e4a9ac7965148d79ecec4c733c8558e" +checksum = "4e9070296bd2902aac4d9defd85608116eceb319526746257ab97096da2c5603" dependencies = [ "alloy-chains", "alloy-consensus", @@ -502,7 +504,7 @@ dependencies = [ "either", "futures", "futures-utils-wasm", - "lru 0.16.4", + "lru", "parking_lot", "pin-project", "reqwest", @@ -517,9 +519,9 @@ dependencies = [ [[package]] name = "alloy-pubsub" -version = "2.3.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "197325ef17fcdf5dfa21999a2a33877b82bb5009d5cc9b5c7f5ddb840fa3b6bb" +checksum = "e6f4d232e145165d28fb3e614b934a7af0b0b7b4c67f9c96afc96552114e7de4" dependencies = [ "alloy-json-rpc", "alloy-primitives", @@ -561,9 +563,9 @@ dependencies = [ [[package]] name = "alloy-rpc-client" -version = "2.3.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f5274f9ada7f558d2feaba16fadcb96022512566cf424147f9cfa2000bc2e458" +checksum = "87da7170c18c43a5def51f5f166d0260776b721e184c349fae1cd5466cb67362" dependencies = [ "alloy-json-rpc", "alloy-primitives", @@ -587,9 +589,9 @@ dependencies = [ [[package]] name = "alloy-rpc-types" -version = "2.3.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12a456d697d4f639d20ac242981de13b94f911c99199e4175e85eaf52b8f785c" +checksum = "88813dc63f261e25d158f07063a71e926f36b3ee8076c1e6d8396348452c5f69" dependencies = [ "alloy-primitives", "alloy-rpc-types-engine", @@ -600,9 +602,9 @@ dependencies = [ [[package]] name = "alloy-rpc-types-admin" -version = "2.3.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f36ee47a0f4e901b95180fbe86c92fec3ea73f12c88c5ec70d1cb41abb8d911" +checksum = "1612dde5f2f6bdc16d3e8b68f4b6c90a2e6db3556c416224fee529871a0fdabc" dependencies = [ "alloy-genesis", "alloy-primitives", @@ -612,9 +614,9 @@ dependencies = [ [[package]] name = "alloy-rpc-types-anvil" -version = "2.3.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d7e911c628067c406effef5137d0e253d98e117cdb81d47aacfe2816199297e" +checksum = "2771c42b80531e99e4449e0a3ffd8f9830073f1d38576ac1643d2573fe6d922f" dependencies = [ "alloy-primitives", "alloy-rpc-types-eth", @@ -624,9 +626,9 @@ dependencies = [ [[package]] name = "alloy-rpc-types-any" -version = "2.3.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5493c88bfdd02271de47f28d69481191a2af6b2907901a3a8c7aff79a6b574eb" +checksum = "6574846e24e090607d78325e90256c9a5ce4010cf0e663b23d06e01fd74356f0" dependencies = [ "alloy-consensus-any", "alloy-network-primitives", @@ -639,9 +641,9 @@ dependencies = [ [[package]] name = "alloy-rpc-types-beacon" -version = "2.3.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8573342cde98a4d40120b7bc0063fbda42cfd87fb305ab834704177b60d8691" +checksum = "d2f2e2476007e47ae60bc0f048a63963004adcd6fdc3737a200e6115b534f084" dependencies = [ "alloy-eips", "alloy-primitives", @@ -659,9 +661,9 @@ dependencies = [ [[package]] name = "alloy-rpc-types-debug" -version = "2.3.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bddf06191e2b12eaaeb3a50ef29a8f3416fb74c61421b80f9d7554c4c1d59a56" +checksum = "6083d2989d456e321721ea9b392c48edb949abbe0a8d449feaf6fa74addcd647" dependencies = [ "alloy-primitives", "alloy-rlp", @@ -672,9 +674,9 @@ dependencies = [ [[package]] name = "alloy-rpc-types-engine" -version = "2.3.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "866fa880cfd95681c979bf3f8385964c2317ef7423c31c0de53159f01112fedf" +checksum = "436d9fe64a92e47ff668d766e75bd7eaafabafb68abaab235848bd7a3c10a8e0" dependencies = [ "alloy-consensus", "alloy-eips", @@ -693,9 +695,9 @@ dependencies = [ [[package]] name = "alloy-rpc-types-eth" -version = "2.3.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d3b805bb9cafaf1362c28b377e2035e49fc5ae0a5ee3ea61bafd9a5d33959e1" +checksum = "acc71ae98333ab278782d59423bd8317e7658954c356e1f4ce559fcea4f4da85" dependencies = [ "alloy-consensus", "alloy-consensus-any", @@ -715,9 +717,9 @@ dependencies = [ [[package]] name = "alloy-rpc-types-mev" -version = "2.3.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b973745b97a364c0215f681b2fb3cb6f0bbeda4dcdba5f56c50f999719ec96f" +checksum = "a4db8cb3571bcd2ef13f79e4094be9f786b57608be4e915baddb061ffe39b057" dependencies = [ "alloy-consensus", "alloy-eips", @@ -730,9 +732,9 @@ dependencies = [ [[package]] name = "alloy-rpc-types-trace" -version = "2.3.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a6fae28f1bda2586b117a5754b49366160770e63e09ac93fc3d872015a5a993" +checksum = "eca9bf40e0b129268f874feff8cb73e5de8fc9a371a6b34635afbb18f437bc7d" dependencies = [ "alloy-primitives", "alloy-rpc-types-eth", @@ -744,9 +746,9 @@ dependencies = [ [[package]] name = "alloy-rpc-types-txpool" -version = "2.3.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4962825b8819e54028cbdaadeba128a613afa55c91b9a88e0b0dc257772a14d7" +checksum = "6045fcec2d380218f5847569b1f9388bdd3bf1edb31ebf14a94b364a81247428" dependencies = [ "alloy-primitives", "alloy-rpc-types-eth", @@ -756,9 +758,9 @@ dependencies = [ [[package]] name = "alloy-serde" -version = "2.3.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ad7efdd5e8fe92f5bab87956d6d3f717d1614c4ab6ad0d1f869b16e83b0dff9" +checksum = "6f11544d72bc87eab3cf4addc61d1f735bf88f3769482eca3deefc0c2558d2d0" dependencies = [ "alloy-primitives", "arbitrary", @@ -768,9 +770,9 @@ dependencies = [ [[package]] name = "alloy-signer" -version = "2.3.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "afa150cd076471ef21aa7b181d8186cb86c3239518859a2e5d80fbec74f0ec38" +checksum = "e76dfc1a6c8337e59624b9f6f02245566b7d24e8dab0eb842d57d4fd8777adc3" dependencies = [ "alloy-primitives", "async-trait", @@ -783,9 +785,9 @@ dependencies = [ [[package]] name = "alloy-signer-local" -version = "2.3.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f125bb730d4772538835d4a31e8e5ed5068e2a991619ba32711fd53ad8c93032" +checksum = "1a9f88d471c719ce2bdf949191f8a1c08bffd0c030606c6a8da0d414454e8bfe" dependencies = [ "alloy-consensus", "alloy-network", @@ -872,9 +874,9 @@ dependencies = [ [[package]] name = "alloy-transport" -version = "2.3.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3cd5dbd32cae5d1bc3afaf455064ebd9bcce70b57de220468908717c225225dd" +checksum = "f9399b22a1d2a82c8c30fd527541ba61e574795d0b95f3b902d3473d11489be5" dependencies = [ "alloy-json-rpc", "auto_impl", @@ -895,9 +897,9 @@ dependencies = [ [[package]] name = "alloy-transport-http" -version = "2.3.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0b7b1f0e528ae788d05937f5b351afae3b4c1e08f17a061d868bd28e56aa9e3" +checksum = "59b55d7ccf45bec99db692c1266413498d2c9776c390cf148aa590422e31b2d3" dependencies = [ "alloy-json-rpc", "alloy-transport", @@ -911,9 +913,9 @@ dependencies = [ [[package]] name = "alloy-transport-ipc" -version = "2.3.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b964b4c048eea647a701a6af07307c64995c4abd42d46ba0b3be9593ae3c333c" +checksum = "5843087378580a14098ada3e9184a92f3706828857ad483558ef2fa1080b2dfb" dependencies = [ "alloy-json-rpc", "alloy-pubsub", @@ -931,9 +933,9 @@ dependencies = [ [[package]] name = "alloy-transport-ws" -version = "2.3.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "efecf9efc5d56276e115b13b36ff27311b8cd6f72affc9788a10ff46b55b2616" +checksum = "5bf93caeb6d84d20060acba53479c9cb55f5364a3d49ab876ad00ef074ce794d" dependencies = [ "alloy-pubsub", "alloy-transport", @@ -970,9 +972,9 @@ dependencies = [ [[package]] name = "alloy-tx-macros" -version = "2.3.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3e2386593cef5b12c7d439dbcf62d7d51c3a50dc943909b42ae34b48f1b69dc" +checksum = "de9792a54c0c8cc009883436ff1a60eb091095aced1c167a764bedbbfa0eee79" dependencies = [ "darling 0.23.0", "proc-macro2", @@ -2973,7 +2975,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ab67060fc6b8ef687992d439ca0fa36e7ed17e9a0b16b25b601e8757df720de" dependencies = [ "data-encoding", - "syn 1.0.109", + "syn 2.0.118", ] [[package]] @@ -3427,7 +3429,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.59.0", + "windows-sys 0.52.0", ] [[package]] @@ -5171,7 +5173,7 @@ checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" dependencies = [ "hermit-abi", "libc", - "windows-sys 0.59.0", + "windows-sys 0.52.0", ] [[package]] @@ -5830,18 +5832,9 @@ dependencies = [ [[package]] name = "lru" -version = "0.16.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f66e8d5d03f609abc3a39e6f08e4164ebf1447a732906d39eb9b99b7919ef39" -dependencies = [ - "hashbrown 0.16.1", -] - -[[package]] -name = "lru" -version = "0.18.0" +version = "0.18.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a860605968fce16869fd239cf4237a82f3ac470723415db603b0e8b6c8d4fb9" +checksum = "5d2f2f9b4ba7e6b24d95e7e899329d35be83bcded72c8540cdd5368932d1d90a" dependencies = [ "hashbrown 0.17.0", ] @@ -7440,7 +7433,7 @@ dependencies = [ "hashbrown 0.17.0", "itertools 0.14.0", "kasuari", - "lru 0.18.0", + "lru", "palette", "serde", "strum 0.28.0", @@ -10928,9 +10921,9 @@ dependencies = [ [[package]] name = "revm-inspectors" -version = "0.42.0" +version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3958b5de5c2c08c6a49b8d94f6ead3e9cd7980c43d6ed240aa3d6485d53269b5" +checksum = "90c136a072540987ee442e9824631835520b13e5756f20780a637ba2e11f9c28" dependencies = [ "alloy-primitives", "alloy-rpc-types-eth", @@ -11371,7 +11364,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.59.0", + "windows-sys 0.52.0", ] [[package]] @@ -11451,7 +11444,7 @@ dependencies = [ "security-framework", "security-framework-sys", "webpki-root-certs 1.0.7", - "windows-sys 0.59.0", + "windows-sys 0.52.0", ] [[package]] @@ -12343,7 +12336,7 @@ dependencies = [ "getrandom 0.4.2", "once_cell", "rustix", - "windows-sys 0.59.0", + "windows-sys 0.52.0", ] [[package]] @@ -13606,7 +13599,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.52.0", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index c82dd492175..016f005b1fb 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -433,7 +433,7 @@ reth-zstd-compressors = { version = "0.6.0", default-features = false } # revm revm = { version = "42.0.1", default-features = false } revmc = { git = "https://github.com/paradigmxyz/revmc", branch = "main", default-features = false, features = ["llvm-prefer-static"] } -revm-inspectors = "0.42.0" +revm-inspectors = "0.42.2" # eth alloy-dyn-abi = "1.6.1" @@ -449,34 +449,34 @@ alloy-trie = { version = "0.9.4", default-features = false } alloy-hardforks = "0.4.7" -alloy-consensus = { version = "2.3.0", default-features = false } -alloy-contract = { version = "2.3.0", default-features = false } -alloy-eips = { version = "2.3.0", default-features = false } -alloy-genesis = { version = "2.3.0", default-features = false } -alloy-json-rpc = { version = "2.3.0", default-features = false } -alloy-network = { version = "2.3.0", default-features = false } -alloy-network-primitives = { version = "2.3.0", default-features = false } -alloy-node-bindings = "2.3.0" -alloy-provider = { version = "2.3.0", features = ["reqwest", "debug-api"], default-features = false } -alloy-pubsub = { version = "2.3.0", default-features = false } -alloy-rpc-client = { version = "2.3.0", default-features = false } -alloy-rpc-types = { version = "2.3.0", features = ["eth"], default-features = false } -alloy-rpc-types-admin = { version = "2.3.0", default-features = false } -alloy-rpc-types-anvil = { version = "2.3.0", default-features = false } -alloy-rpc-types-beacon = { version = "2.3.0", default-features = false } -alloy-rpc-types-debug = { version = "2.3.0", default-features = false } -alloy-rpc-types-engine = { version = "2.3.0", default-features = false } -alloy-rpc-types-eth = { version = "2.3.0", default-features = false } -alloy-rpc-types-mev = { version = "2.3.0", default-features = false } -alloy-rpc-types-trace = { version = "2.3.0", default-features = false } -alloy-rpc-types-txpool = { version = "2.3.0", default-features = false } -alloy-serde = { version = "2.3.0", default-features = false } -alloy-signer = { version = "2.3.0", default-features = false } -alloy-signer-local = { version = "2.3.0", default-features = false } -alloy-transport = { version = "2.3.0" } -alloy-transport-http = { version = "2.3.0", features = ["reqwest-rustls-tls"], default-features = false } -alloy-transport-ipc = { version = "2.3.0", default-features = false } -alloy-transport-ws = { version = "2.3.0", default-features = false } +alloy-consensus = { version = "2.4.0", default-features = false } +alloy-contract = { version = "2.4.0", default-features = false } +alloy-eips = { version = "2.4.0", default-features = false } +alloy-genesis = { version = "2.4.0", default-features = false } +alloy-json-rpc = { version = "2.4.0", default-features = false } +alloy-network = { version = "2.4.0", default-features = false } +alloy-network-primitives = { version = "2.4.0", default-features = false } +alloy-node-bindings = "2.4.0" +alloy-provider = { version = "2.4.0", features = ["reqwest", "debug-api"], default-features = false } +alloy-pubsub = { version = "2.4.0", default-features = false } +alloy-rpc-client = { version = "2.4.0", default-features = false } +alloy-rpc-types = { version = "2.4.0", features = ["eth"], default-features = false } +alloy-rpc-types-admin = { version = "2.4.0", default-features = false } +alloy-rpc-types-anvil = { version = "2.4.0", default-features = false } +alloy-rpc-types-beacon = { version = "2.4.0", default-features = false } +alloy-rpc-types-debug = { version = "2.4.0", default-features = false } +alloy-rpc-types-engine = { version = "2.4.0", default-features = false } +alloy-rpc-types-eth = { version = "2.4.0", default-features = false } +alloy-rpc-types-mev = { version = "2.4.0", default-features = false } +alloy-rpc-types-trace = { version = "2.4.0", default-features = false } +alloy-rpc-types-txpool = { version = "2.4.0", default-features = false } +alloy-serde = { version = "2.4.0", default-features = false } +alloy-signer = { version = "2.4.0", default-features = false } +alloy-signer-local = { version = "2.4.0", default-features = false } +alloy-transport = { version = "2.4.0" } +alloy-transport-http = { version = "2.4.0", features = ["reqwest-rustls-tls"], default-features = false } +alloy-transport-ipc = { version = "2.4.0", default-features = false } +alloy-transport-ws = { version = "2.4.0", default-features = false } # misc either = { version = "1.16.0", default-features = false } From 86963c9c30c58c36b72bde11aa969a257eedbb92 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Wed, 12 Aug 2026 16:46:57 +0200 Subject: [PATCH 006/294] feat(storage): re-enable RocksDB BAL storage (#26653) --- crates/cli/commands/src/common.rs | 16 ++++--- crates/engine/tree/src/persistence.rs | 37 ++++++++++++++++ crates/node/builder/src/launch/common.rs | 19 ++++---- crates/storage/provider/src/bal/rocksdb.rs | 14 ++---- .../provider/src/providers/rocksdb/metrics.rs | 2 + .../src/providers/rocksdb/provider.rs | 43 +++++++++++-------- 6 files changed, 86 insertions(+), 45 deletions(-) diff --git a/crates/cli/commands/src/common.rs b/crates/cli/commands/src/common.rs index 2748ba5ec9c..77cd552633f 100644 --- a/crates/cli/commands/src/common.rs +++ b/crates/cli/commands/src/common.rs @@ -27,8 +27,7 @@ use reth_provider::{ BlockchainProvider, NodeTypesForProvider, RocksDBProvider, StaticFileProvider, StaticFileProviderBuilder, }, - BalConfig, BalStoreHandle, InMemoryBalStore, ProviderFactory, StaticFileProviderFactory, - StorageSettings, + BalStoreHandle, ProviderFactory, RocksDBBalStore, StaticFileProviderFactory, StorageSettings, }; use reth_stages::{sets::DefaultStages, Pipeline, PipelineTarget}; use reth_static_file::StaticFileProducer; @@ -196,11 +195,14 @@ impl EnvironmentArgs { where C: ChainSpecParser, { - let balstore_cache_size = - self.db.balstore_cache_size.unwrap_or(BalConfig::DEFAULT_IN_MEMORY_RETENTION_DISTANCE); - let bal_store = BalStoreHandle::new(InMemoryBalStore::new( - BalConfig::with_in_memory_retention_distance(balstore_cache_size), - )); + let bal_store = self + .db + .balstore_cache_size + .map(|distance| { + RocksDBBalStore::with_buffer_retention_distance(rocksdb_provider.clone(), distance) + }) + .unwrap_or_else(|| RocksDBBalStore::new(rocksdb_provider.clone())); + let bal_store = BalStoreHandle::new(bal_store); let factory = ProviderFactory::>::new( db, self.chain.clone(), diff --git a/crates/engine/tree/src/persistence.rs b/crates/engine/tree/src/persistence.rs index feef9a6e368..113e7a0f47a 100644 --- a/crates/engine/tree/src/persistence.rs +++ b/crates/engine/tree/src/persistence.rs @@ -566,6 +566,43 @@ mod tests { assert_eq!(result.last_state_trie_block, result.last_block); } + #[test] + fn test_save_blocks_flushes_bal_store() { + use reth_provider::{RocksDBBalStore, RocksDBProviderFactory}; + + reth_tracing::init_test_tracing(); + let provider = create_test_provider_factory(); + let bal_store = BalStoreHandle::new(RocksDBBalStore::new(provider.rocksdb_provider())); + let provider = provider.with_bal_store(bal_store); + init_genesis(&provider).unwrap(); + + let mut test_block_builder = TestBlockBuilder::eth(); + let executed = test_block_builder.get_executed_block_with_number(1, B256::random()); + let num_hash = executed.recovered_block().num_hash(); + let raw_bal = Bytes::from_static(&[0xc0]); + + provider.bal_store().insert(num_hash, RawBal::new(raw_bal.clone())).unwrap(); + + let (_finished_exex_height_tx, finished_exex_height_rx) = + tokio::sync::watch::channel(FinishedExExHeight::NoExExs); + let pruner = + Pruner::new_with_factory(provider.clone(), vec![], 5, 0, None, finished_exex_height_rx); + let (sync_metrics_tx, _sync_metrics_rx) = unbounded_channel(); + let handle = PersistenceHandle::::spawn_service( + provider.clone(), + pruner, + sync_metrics_tx, + ); + let (tx, rx) = crossbeam_channel::bounded(1); + + handle.save_blocks(full_save_input(vec![executed]), tx).unwrap(); + + let result = rx.recv_timeout(std::time::Duration::from_secs(10)).expect("test timed out"); + assert_eq!(result.last_block, num_hash); + let persisted_store = RocksDBBalStore::new(provider.rocksdb_provider()); + assert_eq!(persisted_store.get_by_hash(num_hash.hash).unwrap(), Some(raw_bal)); + } + #[test] fn test_save_blocks_multiple_blocks() { reth_tracing::init_test_tracing(); diff --git a/crates/node/builder/src/launch/common.rs b/crates/node/builder/src/launch/common.rs index 498c65b3fab..43b6cc93bfc 100644 --- a/crates/node/builder/src/launch/common.rs +++ b/crates/node/builder/src/launch/common.rs @@ -71,10 +71,10 @@ use reth_node_metrics::{ }; use reth_provider::{ providers::{NodeTypesForProvider, ProviderNodeTypes, RocksDBProvider, StaticFileProvider}, - BalConfig, BalStoreHandle, BlockHashReader, BlockNumReader, DBProvider, - DatabaseProviderFactory, InMemoryBalStore, MetadataProvider, MetadataWriter, ProviderError, - ProviderFactory, ProviderResult, RocksDBProviderFactory, StageCheckpointReader, - StaticFileProviderBuilder, StaticFileProviderFactory, StorageSettingsCache, + BalStoreHandle, BlockHashReader, BlockNumReader, DBProvider, DatabaseProviderFactory, + MetadataProvider, MetadataWriter, ProviderError, ProviderFactory, ProviderResult, + RocksDBBalStore, RocksDBProviderFactory, StageCheckpointReader, StaticFileProviderBuilder, + StaticFileProviderFactory, StorageSettingsCache, }; use reth_prune::{PruneMode, PruneModes, PrunerBuilder}; use reth_rpc_builder::config::RethRpcServerConfig; @@ -524,14 +524,15 @@ where .build()? }; - let balstore_cache_size = self + let bal_store = self .node_config() .db .balstore_cache_size - .unwrap_or(BalConfig::DEFAULT_IN_MEMORY_RETENTION_DISTANCE); - let bal_store = BalStoreHandle::new(InMemoryBalStore::new( - BalConfig::with_in_memory_retention_distance(balstore_cache_size), - )); + .map(|distance| { + RocksDBBalStore::with_buffer_retention_distance(rocksdb_provider.clone(), distance) + }) + .unwrap_or_else(|| RocksDBBalStore::new(rocksdb_provider.clone())); + let bal_store = BalStoreHandle::new(bal_store); let factory = ProviderFactory::new( self.right().clone(), self.chain_spec(), diff --git a/crates/storage/provider/src/bal/rocksdb.rs b/crates/storage/provider/src/bal/rocksdb.rs index 99765824b5a..d5886d1e8d9 100644 --- a/crates/storage/provider/src/bal/rocksdb.rs +++ b/crates/storage/provider/src/bal/rocksdb.rs @@ -352,21 +352,13 @@ struct RocksDBBalEntry { #[cfg(test)] mod tests { use super::*; - use crate::providers::{RocksDBBuilder, RocksDBProvider}; + use crate::providers::RocksDBBuilder; use alloy_primitives::B256; use tokio_stream::StreamExt; - fn test_rocksdb(dir: &tempfile::TempDir) -> RocksDBProvider { - RocksDBBuilder::new(dir.path()) - .with_table::() - .with_table::() - .build() - .unwrap() - } - fn test_store() -> (tempfile::TempDir, RocksDBBalStore) { let dir = tempfile::tempdir().unwrap(); - let rocksdb = test_rocksdb(&dir); + let rocksdb = RocksDBBuilder::new(dir.path()).with_default_tables().build().unwrap(); (dir, RocksDBBalStore::new(rocksdb)) } @@ -455,7 +447,7 @@ mod tests { #[test] fn configured_buffer_retention_distance_is_used() { let dir = tempfile::tempdir().unwrap(); - let rocksdb = test_rocksdb(&dir); + let rocksdb = RocksDBBuilder::new(dir.path()).with_default_tables().build().unwrap(); let store = RocksDBBalStore::with_buffer_retention_distance(rocksdb, 64); let old = NumHash::new(1, B256::with_last_byte(1)); let tip = NumHash::new(34, B256::with_last_byte(2)); diff --git a/crates/storage/provider/src/providers/rocksdb/metrics.rs b/crates/storage/provider/src/providers/rocksdb/metrics.rs index 3971ed978ba..c22edf65176 100644 --- a/crates/storage/provider/src/providers/rocksdb/metrics.rs +++ b/crates/storage/provider/src/providers/rocksdb/metrics.rs @@ -8,6 +8,8 @@ use strum::{EnumIter, IntoEnumIterator}; pub(super) const ROCKSDB_TABLES: &[&str] = &[ Tables::TransactionHashNumbers.name(), + Tables::BlockAccessLists.name(), + Tables::BlockAccessListBlockNumbers.name(), Tables::StoragesHistory.name(), Tables::AccountsHistory.name(), ]; diff --git a/crates/storage/provider/src/providers/rocksdb/provider.rs b/crates/storage/provider/src/providers/rocksdb/provider.rs index d7198e7c74a..bae0a137b37 100644 --- a/crates/storage/provider/src/providers/rocksdb/provider.rs +++ b/crates/storage/provider/src/providers/rocksdb/provider.rs @@ -315,10 +315,14 @@ impl RocksDBBuilder { /// - [`tables::TransactionHashNumbers`] - Transaction hash to number mapping /// - [`tables::AccountsHistory`] - Account history index /// - [`tables::StoragesHistory`] - Storage history index + /// - [`tables::BlockAccessLists`] - Block access list payloads + /// - [`tables::BlockAccessListBlockNumbers`] - Block access list hash index pub fn with_default_tables(self) -> Self { self.with_table::() .with_table::() .with_table::() + .with_table::() + .with_table::() } /// Enables metrics. @@ -2918,22 +2922,25 @@ mod tests { provider.put::(key.clone(), &value).unwrap(); assert!(provider.get::(key).unwrap().is_some()); - drop(provider); - - let column_families = DB::list_cf(&Options::default(), temp_dir.path()).unwrap(); - assert!(!column_families.iter().any(|name| name == tables::BlockAccessLists::NAME)); - assert!(!column_families - .iter() - .any(|name| name == tables::BlockAccessListBlockNumbers::NAME)); + let bal_key = + reth_db_api::models::StoredBlockAccessListKey::new(1, B256::with_last_byte(1)); + let bal_value = + reth_db_api::models::StoredBlockAccessList::new(Bytes::from_static(&[0xc0])); + provider.put::(bal_key, &bal_value).unwrap(); + assert_eq!(provider.get::(bal_key).unwrap(), Some(bal_value)); + provider + .put::(bal_key.hash(), &bal_key.number()) + .unwrap(); + assert_eq!( + provider.get::(bal_key.hash()).unwrap(), + Some(bal_key.number()) + ); } #[test] fn block_access_lists_store_large_payloads_in_blob_files() { let temp_dir = TempDir::new().unwrap(); - let provider = RocksDBBuilder::new(temp_dir.path()) - .with_table::() - .build() - .unwrap(); + let provider = RocksDBBuilder::new(temp_dir.path()).with_default_tables().build().unwrap(); let bal_key = reth_db_api::models::StoredBlockAccessListKey::new(1, B256::with_last_byte(1)); let bal_value = reth_db_api::models::StoredBlockAccessList::new(Bytes::from(vec![ @@ -2989,12 +2996,7 @@ mod tests { 1 ])); - let provider = RocksDBBuilder::new(temp_dir.path()) - .with_default_tables() - .with_table::() - .with_table::() - .build() - .unwrap(); + let provider = RocksDBBuilder::new(temp_dir.path()).with_default_tables().build().unwrap(); provider.put::(bal_key, &bal_value).unwrap(); provider .put::(bal_key.hash(), &bal_key.number()) @@ -3002,7 +3004,12 @@ mod tests { provider.flush(&[tables::BlockAccessLists::NAME]).unwrap(); drop(provider); - let provider = RocksDBBuilder::new(temp_dir.path()).with_default_tables().build().unwrap(); + let provider = RocksDBBuilder::new(temp_dir.path()) + .with_table::() + .with_table::() + .with_table::() + .build() + .unwrap(); assert_eq!(provider.get::(bal_key).unwrap(), Some(bal_value)); assert_eq!( provider.get::(bal_key.hash()).unwrap(), From fd1f845f4dd603d204990d3a9e6ebf04cfb2d4d2 Mon Sep 17 00:00:00 2001 From: stevencartavia <112043913+stevencartavia@users.noreply.github.com> Date: Wed, 12 Aug 2026 12:02:13 -0600 Subject: [PATCH 007/294] feat(rpc): implement debug_traceChain subscription (#26582) Co-authored-by: Matthias Seitz --- crates/ethereum/node/tests/e2e/rpc.rs | 92 +++++++++- crates/rpc/rpc-api/src/debug.rs | 68 +++++++- crates/rpc/rpc-api/src/lib.rs | 1 + crates/rpc/rpc/src/debug.rs | 236 ++++++++++++++++++++++++-- 4 files changed, 376 insertions(+), 21 deletions(-) diff --git a/crates/ethereum/node/tests/e2e/rpc.rs b/crates/ethereum/node/tests/e2e/rpc.rs index 338326cd4f4..d70bd07aff8 100644 --- a/crates/ethereum/node/tests/e2e/rpc.rs +++ b/crates/ethereum/node/tests/e2e/rpc.rs @@ -1,8 +1,10 @@ use crate::utils::{eth_payload_attributes, eth_payload_attributes_amsterdam}; -use alloy_eips::{eip2718::Encodable2718, eip7910::EthConfig}; +use alloy_eips::{eip2718::Encodable2718, eip7910::EthConfig, BlockNumberOrTag}; use alloy_genesis::Genesis; use alloy_primitives::{Address, Bytes, B256, U256}; -use alloy_provider::{network::EthereumWallet, Provider, ProviderBuilder, SendableTx}; +use alloy_provider::{ + ext::DebugApi, network::EthereumWallet, Provider, ProviderBuilder, SendableTx, +}; use alloy_rpc_types_beacon::relay::{ BidTrace, BuilderBlockValidationRequestV3, BuilderBlockValidationRequestV4, BuilderBlockValidationRequestV6, SignedBidSubmissionV3, SignedBidSubmissionV4, @@ -13,9 +15,11 @@ use alloy_rpc_types_engine::{ ExecutionPayloadV3, PraguePayloadFields, }; use alloy_rpc_types_eth::TransactionRequest; +use alloy_rpc_types_trace::geth::GethDebugTracingOptions; +use jsonrpsee::core::client::{Subscription, SubscriptionClientT}; use rand::{rngs::StdRng, Rng, SeedableRng}; use reth_chainspec::{ChainSpecBuilder, EthChainSpec, MAINNET}; -use reth_e2e_test_utils::setup_engine; +use reth_e2e_test_utils::{setup_engine, E2ETestSetupBuilder}; use reth_network::{types::NatResolver, PeersInfo}; use reth_node_builder::{NodeBuilder, NodeHandle}; use reth_node_core::{ @@ -25,7 +29,8 @@ use reth_node_core::{ use reth_node_ethereum::EthereumNode; use reth_payload_primitives::BuiltPayload; use reth_primitives_traits::Block as _; -use reth_rpc_api::servers::AdminApiServer; +use reth_rpc_api::{servers::AdminApiServer, ChainBlockTraceResult}; +use reth_rpc_server_types::RpcModuleSelection; use reth_tasks::Runtime; use std::{ net::{IpAddr, Ipv4Addr}, @@ -136,6 +141,85 @@ async fn test_fee_history() -> eyre::Result<()> { Ok(()) } +#[tokio::test] +async fn test_debug_trace_chain_subscription() -> eyre::Result<()> { + reth_tracing::init_test_tracing(); + + let chain_spec = Arc::new( + ChainSpecBuilder::default() + .chain(MAINNET.chain) + .genesis(serde_json::from_str(include_str!("../assets/genesis.json")).unwrap()) + .cancun_activated() + .build(), + ); + let (mut nodes, wallet) = + E2ETestSetupBuilder::::new(1, chain_spec, eth_payload_attributes) + .with_node_config_modifier(|config| { + config.with_rpc( + RpcServerArgs::default() + .with_unused_ports() + .with_http() + .with_http_api(RpcModuleSelection::All) + .with_ws() + .with_ws_api(RpcModuleSelection::All), + ) + }) + .build() + .await?; + let mut node = nodes.pop().unwrap(); + let provider = ProviderBuilder::new() + .wallet(EthereumWallet::new(wallet.wallet_gen().swap_remove(0))) + .connect_http(node.rpc_url()); + + // Geth suppresses empty intermediate blocks but always emits the end block. + node.advance_block().await?; + let _ = GasWaster::deploy_builder(&provider, U256::from(5)).send().await?; + let _ = GasWaster::deploy_builder(&provider, U256::from(7)).send().await?; + node.advance_block().await?; + node.advance_block().await?; + + let client = node.inner.rpc_server_handle().ws_client().await.unwrap(); + let invalid: Result, _> = client + .subscribe( + "debug_subscribe", + jsonrpsee::rpc_params!["traceChain", "0x3", "0x3"], + "debug_unsubscribe", + ) + .await; + assert!(invalid.is_err(), "equal endpoints must be rejected"); + + let mut subscription: Subscription = client + .subscribe( + "debug_subscribe", + jsonrpsee::rpc_params![ + "traceChain", + BlockNumberOrTag::Number(0), + BlockNumberOrTag::Number(3), + GethDebugTracingOptions::default() + ], + "debug_unsubscribe", + ) + .await?; + + let traced = subscription.next().await.unwrap()?; + assert_eq!(traced.block, U256::from(2)); + let expected = provider + .debug_trace_block_by_number( + BlockNumberOrTag::Number(2), + GethDebugTracingOptions::default(), + ) + .await?; + assert_eq!(expected.len(), 2, "both transactions must be traced"); + assert_eq!(traced.traces, expected.into_iter().map(Some).collect::>()); + + let terminal = subscription.next().await.unwrap()?; + assert_eq!(terminal.block, U256::from(3)); + assert!(terminal.traces.is_empty()); + subscription.unsubscribe().await?; + + Ok(()) +} + #[tokio::test] async fn test_flashbots_validate_v3() -> eyre::Result<()> { reth_tracing::init_test_tracing(); diff --git a/crates/rpc/rpc-api/src/debug.rs b/crates/rpc/rpc-api/src/debug.rs index 964621d7d2d..7474fe76da2 100644 --- a/crates/rpc/rpc-api/src/debug.rs +++ b/crates/rpc/rpc-api/src/debug.rs @@ -1,14 +1,15 @@ use alloy_eips::{BlockId, BlockNumberOrTag}; use alloy_genesis::ChainConfig; use alloy_json_rpc::RpcObject; -use alloy_primitives::{Address, Bytes, B256, U64}; +use alloy_primitives::{Address, Bytes, B256, U256, U64}; use alloy_rpc_types_debug::ExecutionWitness; use alloy_rpc_types_eth::{Account, AccountInfo, Bundle, Index, StateContext}; use alloy_rpc_types_trace::geth::{ - BlockTraceResult, GethDebugTracingCallOptions, GethDebugTracingOptions, GethTrace, TraceResult, + GethDebugTracingCallOptions, GethDebugTracingOptions, GethTrace, TraceResult, }; use jsonrpsee::{core::RpcResult, proc_macros::rpc}; use reth_trie_common::{updates::TrieUpdates, ExecutionWitnessMode, HashedPostState}; +use serde::{Deserialize, Serialize}; /// Debug rpc interface. #[cfg_attr(not(feature = "client"), rpc(server, namespace = "debug"))] @@ -48,14 +49,20 @@ pub trait DebugApi { #[method(name = "clearTxpool")] async fn debug_clear_txpool(&self) -> RpcResult<()>; - /// Returns the structured logs created during the execution of EVM between two blocks - /// (excluding start) as a JSON object. - #[method(name = "traceChain")] - async fn debug_trace_chain( + /// Subscribes to structured logs created during EVM execution between two blocks, excluding + /// the start block and including the end block. + #[subscription( + name = "subscribe" => "subscription", + unsubscribe = "unsubscribe", + item = ChainBlockTraceResult + )] + async fn debug_subscribe( &self, + subscription: String, start_exclusive: BlockNumberOrTag, end_inclusive: BlockNumberOrTag, - ) -> RpcResult>; + opts: Option, + ) -> jsonrpsee::core::SubscriptionResult; /// The `debug_traceBlock` method will return a full stack trace of all invoked opcodes of all /// transaction that were included in this block. @@ -375,3 +382,50 @@ pub trait DebugApi { opts: Option, ) -> RpcResult>; } + +/// Results for one block emitted by a `traceChain` debug subscription. +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct ChainBlockTraceResult { + /// Block number corresponding to the trace task. + pub block: U256, + /// Block hash corresponding to the trace task. + pub hash: B256, + /// Trace results produced by the task. + /// + /// Geth leaves entries after the first transaction-level tracing failure as `null`. + pub traces: Vec>, +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn chain_block_trace_result_matches_geth_wire_shape() { + let block_hash = B256::repeat_byte(0x11); + let tx_hash = B256::repeat_byte(0x22); + let result = ChainBlockTraceResult { + block: U256::from(2), + hash: block_hash, + traces: vec![ + Some(TraceResult::Error { + error: "trace failed".to_string(), + tx_hash: Some(tx_hash), + }), + None, + ], + }; + + assert_eq!( + serde_json::to_value(result).unwrap(), + serde_json::json!({ + "block": "0x2", + "hash": block_hash, + "traces": [ + {"error": "trace failed", "txHash": tx_hash}, + null + ] + }) + ); + } +} diff --git a/crates/rpc/rpc-api/src/lib.rs b/crates/rpc/rpc-api/src/lib.rs index a6c82b185dc..10a45d26f59 100644 --- a/crates/rpc/rpc-api/src/lib.rs +++ b/crates/rpc/rpc-api/src/lib.rs @@ -32,6 +32,7 @@ mod txpool; mod validation; mod web3; +pub use debug::ChainBlockTraceResult; pub use reth::RethJitAction; pub use testing::{TestingBuildBlockRequestV1, TESTING_BUILD_BLOCK_V1, TESTING_COMMIT_BLOCK_V1}; diff --git a/crates/rpc/rpc/src/debug.rs b/crates/rpc/rpc/src/debug.rs index 9860eae2fc7..f5db8205032 100644 --- a/crates/rpc/rpc/src/debug.rs +++ b/crates/rpc/rpc/src/debug.rs @@ -2,7 +2,7 @@ use alloy_consensus::{constants::KECCAK_EMPTY, transaction::TxHashRef, BlockHead use alloy_eips::{eip2718::Encodable2718, BlockId, BlockNumberOrTag}; use alloy_evm::{env::BlockEnvironment, Evm}; use alloy_genesis::ChainConfig; -use alloy_primitives::{hex::decode, uint, Address, Bytes, B256, U64}; +use alloy_primitives::{hex::decode, uint, Address, Bytes, B256, U256, U64}; use alloy_rlp::{Decodable, Encodable}; use alloy_rpc_types::BlockTransactionsKind; use alloy_rpc_types_debug::ExecutionWitness; @@ -10,11 +10,11 @@ use alloy_rpc_types_eth::{ state::EvmOverrides, Account, AccountInfo, BlockError, Bundle, Index, StateContext, }; use alloy_rpc_types_trace::geth::{ - BlockTraceResult, GethDebugTracingCallOptions, GethDebugTracingOptions, GethTrace, TraceResult, + GethDebugTracingCallOptions, GethDebugTracingOptions, GethTrace, TraceResult, }; use async_trait::async_trait; use futures::Stream; -use jsonrpsee::core::RpcResult; +use jsonrpsee::{core::RpcResult, PendingSubscriptionSink, SubscriptionMessage}; use parking_lot::RwLock; use reth_chainspec::{ChainSpecProvider, EthChainSpec, EthereumHardforks}; use reth_engine_primitives::ConsensusEngineEvent; @@ -24,7 +24,7 @@ use reth_primitives_traits::{ Block as BlockTrait, BlockBody, BlockTy, ReceiptWithBloom, RecoveredBlock, }; use reth_revm::{db::State, witness::ExecutionWitnessRecord}; -use reth_rpc_api::DebugApiServer; +use reth_rpc_api::{ChainBlockTraceResult, DebugApiServer}; use reth_rpc_convert::RpcTxReq; use reth_rpc_eth_api::{ helpers::{EthTransactions, TraceExt}, @@ -72,6 +72,7 @@ where let inner = Arc::new(DebugApiInner { eth_api, blocking_task_guard, + task_spawner: executor.clone(), bad_block_store: bad_block_store.clone(), }); @@ -164,6 +165,95 @@ where .await } + /// Traces a block for `traceChain`, preserving transaction-level tracing failures in the + /// subscription result instead of failing the entire range. + async fn trace_chain_block( + &self, + block: Arc>>, + opts: GethDebugTracingOptions, + ) -> Result>, Eth::Error> { + let evm_env = self.eth_api().evm_env_for_header(block.sealed_block().sealed_header())?; + + self.eth_api() + .spawn_with_state_at_block(block.parent_hash(), move |eth_api, mut db| { + let tx_count = block.body().transactions().len(); + let mut results = Vec::with_capacity(tx_count); + + eth_api.apply_pre_execution_changes(&block, &mut db)?; + + let block_env = evm_env.block_env.clone(); + let mut transactions = block.transactions_recovered().enumerate().peekable(); + let inspector = match DebugInspector::new(opts) { + Ok(inspector) => inspector, + Err(err) => { + if let Some((_, tx)) = transactions.peek() { + results.push(Some(TraceResult::Error { + error: err.to_string(), + tx_hash: Some(*tx.tx_hash()), + })); + } + results.resize(tx_count, None); + return Ok(results) + } + }; + let mut evm = + eth_api.evm_config().evm_with_env_and_inspector(&mut db, evm_env, inspector); + + while let Some((index, tx)) = transactions.next() { + let tx_hash = *tx.tx_hash(); + let tx_env = eth_api.evm_config().tx_env(tx); + let res = match evm.transact(tx_env.clone()) { + Ok(res) => res, + Err(err) => { + results.push(Some(TraceResult::Error { + error: err.to_string(), + tx_hash: Some(tx_hash), + })); + break + } + }; + + let (db, inspector, _) = evm.components_mut(); + let result = match inspector.get_result( + Some(TransactionContext { + block_hash: Some(block.hash()), + tx_hash: Some(tx_hash), + tx_index: Some(index), + }), + &tx_env, + &block_env, + &res, + db, + ) { + Ok(result) => result, + Err(err) => { + results.push(Some(TraceResult::Error { + error: err.to_string(), + tx_hash: Some(tx_hash), + })); + break + } + }; + + results.push(Some(TraceResult::Success { result, tx_hash: Some(tx_hash) })); + if let Some((_, next_tx)) = transactions.peek() { + if let Err(err) = inspector.fuse() { + results.push(Some(TraceResult::Error { + error: err.to_string(), + tx_hash: Some(*next_tx.tx_hash()), + })); + break + } + db.commit(res.state); + } + } + + results.resize(tx_count, None); + Ok(results) + }) + .await + } + /// Replays the given block and returns the trace of each transaction. /// /// This expects a rlp encoded block @@ -862,13 +952,137 @@ where Ok(()) } - /// Handler for `debug_traceChain` - async fn debug_trace_chain( + /// Handler for `debug_subscribe("traceChain", ...)`. + async fn debug_subscribe( &self, - _start_exclusive: BlockNumberOrTag, - _end_inclusive: BlockNumberOrTag, - ) -> RpcResult> { - Err(internal_rpc_err("unimplemented")) + pending: PendingSubscriptionSink, + subscription: String, + start_exclusive: BlockNumberOrTag, + end_inclusive: BlockNumberOrTag, + opts: Option, + ) -> jsonrpsee::core::SubscriptionResult { + if subscription != "traceChain" { + pending + .reject(EthApiError::InvalidParams(format!( + "unsupported debug subscription: {subscription}" + ))) + .await; + return Ok(()) + } + + let start_id = BlockId::Number(start_exclusive); + let start = match self.eth_api().recovered_block(start_id).await { + Ok(Some(block)) => block, + Ok(None) => { + pending.reject(EthApiError::TracingBlockNotFound(start_id)).await; + return Ok(()) + } + Err(err) => { + pending.reject(err).await; + return Ok(()) + } + }; + let end_id = BlockId::Number(end_inclusive); + let end = match self.eth_api().recovered_block(end_id).await { + Ok(Some(block)) => block, + Ok(None) => { + pending.reject(EthApiError::TracingBlockNotFound(end_id)).await; + return Ok(()) + } + Err(err) => { + pending.reject(err).await; + return Ok(()) + } + }; + + if start.number() >= end.number() { + pending + .reject(EthApiError::InvalidParams(format!( + "end block (#{}) needs to come after start block (#{})", + end.number(), + start.number() + ))) + .await; + return Ok(()) + } + + let sink = pending.accept().await?; + let this = self.clone(); + let task_spawner = self.inner.task_spawner.clone(); + task_spawner.spawn_task(async move { + let end_number = end.number(); + let opts = opts.unwrap_or_default(); + + for number in (start.number() + 1)..=end_number { + if sink.is_closed() { + break + } + + let block_id = BlockId::Number(number.into()); + let block = match this.eth_api().recovered_block(block_id).await { + Ok(Some(block)) => block, + Ok(None) => { + tracing::warn!(target: "rpc::debug", %number, "Chain tracing block not found"); + break + } + Err(err) => { + tracing::warn!(target: "rpc::debug", %number, %err, "Failed to load chain tracing block"); + break + } + }; + let permit = tokio::select! { + _ = sink.closed() => break, + permit = this.acquire_trace_permit() => match permit { + Ok(permit) => permit, + Err(err) => { + tracing::debug!(target: "rpc::debug", %err, "Failed to acquire trace permit"); + break + } + } + }; + if sink.is_closed() { + break + } + let traces = match this.trace_chain_block(block.clone(), opts.clone()).await { + Ok(traces) => traces, + Err(err) => { + tracing::warn!(target: "rpc::debug", %number, %err, "Failed to trace chain block"); + break + } + }; + drop(permit); + + if traces.is_empty() && number != end_number { + continue + } + let result = ChainBlockTraceResult { + block: U256::from(number), + hash: block.hash(), + traces, + }; + let message = match SubscriptionMessage::new( + sink.method_name(), + sink.subscription_id(), + &result, + ) { + Ok(message) => message, + Err(err) => { + tracing::warn!(target: "rpc::debug", %number, %err, "Failed to serialize chain trace"); + break + } + }; + if sink.send(message).await.is_err() { + break + } + } + + // Dropping jsonrpsee's final sink unregisters the subscription without notifying the + // client that this finite stream completed. Retain it so the subscription remains + // explicitly unsubscribable until the client unsubscribes or disconnects. + sink.closed().await; + }); + + Ok(()) } /// Handler for `debug_traceBlock` @@ -1207,6 +1421,8 @@ struct DebugApiInner { eth_api: Eth, // restrict the number of concurrent calls to blocking calls blocking_task_guard: BlockingTaskGuard, + /// Spawns long-running subscription tasks. + task_spawner: Runtime, /// Cache for bad blocks. bad_block_store: BadBlockStore>, } From 722628f965e7dbcebd7079bcb7db60c76761f2ff Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Wed, 12 Aug 2026 20:25:15 +0200 Subject: [PATCH 008/294] refactor(rpc): use Alloy trace chain result type (#26669) --- crates/ethereum/node/tests/e2e/rpc.rs | 4 +- crates/rpc/rpc-api/src/debug.rs | 53 ++------------------------- crates/rpc/rpc-api/src/lib.rs | 1 - crates/rpc/rpc/src/debug.rs | 5 ++- 4 files changed, 8 insertions(+), 55 deletions(-) diff --git a/crates/ethereum/node/tests/e2e/rpc.rs b/crates/ethereum/node/tests/e2e/rpc.rs index d70bd07aff8..d4ec107410f 100644 --- a/crates/ethereum/node/tests/e2e/rpc.rs +++ b/crates/ethereum/node/tests/e2e/rpc.rs @@ -15,7 +15,7 @@ use alloy_rpc_types_engine::{ ExecutionPayloadV3, PraguePayloadFields, }; use alloy_rpc_types_eth::TransactionRequest; -use alloy_rpc_types_trace::geth::GethDebugTracingOptions; +use alloy_rpc_types_trace::geth::{ChainBlockTraceResult, GethDebugTracingOptions}; use jsonrpsee::core::client::{Subscription, SubscriptionClientT}; use rand::{rngs::StdRng, Rng, SeedableRng}; use reth_chainspec::{ChainSpecBuilder, EthChainSpec, MAINNET}; @@ -29,7 +29,7 @@ use reth_node_core::{ use reth_node_ethereum::EthereumNode; use reth_payload_primitives::BuiltPayload; use reth_primitives_traits::Block as _; -use reth_rpc_api::{servers::AdminApiServer, ChainBlockTraceResult}; +use reth_rpc_api::servers::AdminApiServer; use reth_rpc_server_types::RpcModuleSelection; use reth_tasks::Runtime; use std::{ diff --git a/crates/rpc/rpc-api/src/debug.rs b/crates/rpc/rpc-api/src/debug.rs index 7474fe76da2..571da6f8b15 100644 --- a/crates/rpc/rpc-api/src/debug.rs +++ b/crates/rpc/rpc-api/src/debug.rs @@ -1,15 +1,15 @@ use alloy_eips::{BlockId, BlockNumberOrTag}; use alloy_genesis::ChainConfig; use alloy_json_rpc::RpcObject; -use alloy_primitives::{Address, Bytes, B256, U256, U64}; +use alloy_primitives::{Address, Bytes, B256, U64}; use alloy_rpc_types_debug::ExecutionWitness; use alloy_rpc_types_eth::{Account, AccountInfo, Bundle, Index, StateContext}; use alloy_rpc_types_trace::geth::{ - GethDebugTracingCallOptions, GethDebugTracingOptions, GethTrace, TraceResult, + ChainBlockTraceResult, GethDebugTracingCallOptions, GethDebugTracingOptions, GethTrace, + TraceResult, }; use jsonrpsee::{core::RpcResult, proc_macros::rpc}; use reth_trie_common::{updates::TrieUpdates, ExecutionWitnessMode, HashedPostState}; -use serde::{Deserialize, Serialize}; /// Debug rpc interface. #[cfg_attr(not(feature = "client"), rpc(server, namespace = "debug"))] @@ -382,50 +382,3 @@ pub trait DebugApi { opts: Option, ) -> RpcResult>; } - -/// Results for one block emitted by a `traceChain` debug subscription. -#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] -pub struct ChainBlockTraceResult { - /// Block number corresponding to the trace task. - pub block: U256, - /// Block hash corresponding to the trace task. - pub hash: B256, - /// Trace results produced by the task. - /// - /// Geth leaves entries after the first transaction-level tracing failure as `null`. - pub traces: Vec>, -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn chain_block_trace_result_matches_geth_wire_shape() { - let block_hash = B256::repeat_byte(0x11); - let tx_hash = B256::repeat_byte(0x22); - let result = ChainBlockTraceResult { - block: U256::from(2), - hash: block_hash, - traces: vec![ - Some(TraceResult::Error { - error: "trace failed".to_string(), - tx_hash: Some(tx_hash), - }), - None, - ], - }; - - assert_eq!( - serde_json::to_value(result).unwrap(), - serde_json::json!({ - "block": "0x2", - "hash": block_hash, - "traces": [ - {"error": "trace failed", "txHash": tx_hash}, - null - ] - }) - ); - } -} diff --git a/crates/rpc/rpc-api/src/lib.rs b/crates/rpc/rpc-api/src/lib.rs index 10a45d26f59..a6c82b185dc 100644 --- a/crates/rpc/rpc-api/src/lib.rs +++ b/crates/rpc/rpc-api/src/lib.rs @@ -32,7 +32,6 @@ mod txpool; mod validation; mod web3; -pub use debug::ChainBlockTraceResult; pub use reth::RethJitAction; pub use testing::{TestingBuildBlockRequestV1, TESTING_BUILD_BLOCK_V1, TESTING_COMMIT_BLOCK_V1}; diff --git a/crates/rpc/rpc/src/debug.rs b/crates/rpc/rpc/src/debug.rs index f5db8205032..1b3d42a355d 100644 --- a/crates/rpc/rpc/src/debug.rs +++ b/crates/rpc/rpc/src/debug.rs @@ -10,7 +10,8 @@ use alloy_rpc_types_eth::{ state::EvmOverrides, Account, AccountInfo, BlockError, Bundle, Index, StateContext, }; use alloy_rpc_types_trace::geth::{ - GethDebugTracingCallOptions, GethDebugTracingOptions, GethTrace, TraceResult, + ChainBlockTraceResult, GethDebugTracingCallOptions, GethDebugTracingOptions, GethTrace, + TraceResult, }; use async_trait::async_trait; use futures::Stream; @@ -24,7 +25,7 @@ use reth_primitives_traits::{ Block as BlockTrait, BlockBody, BlockTy, ReceiptWithBloom, RecoveredBlock, }; use reth_revm::{db::State, witness::ExecutionWitnessRecord}; -use reth_rpc_api::{ChainBlockTraceResult, DebugApiServer}; +use reth_rpc_api::DebugApiServer; use reth_rpc_convert::RpcTxReq; use reth_rpc_eth_api::{ helpers::{EthTransactions, TraceExt}, From 70fb52e5fc7e6fb799937005ac294c8fffba5a61 Mon Sep 17 00:00:00 2001 From: Derek Cofausper <256792747+decofe@users.noreply.github.com> Date: Wed, 12 Aug 2026 23:24:15 +0200 Subject: [PATCH 009/294] refactor(network): assert negotiated protocol installation (#26671) Co-authored-by: Sergei Shulepov <2205845+pepyakin@users.noreply.github.com> --- crates/net/network/src/session/mod.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/crates/net/network/src/session/mod.rs b/crates/net/network/src/session/mod.rs index db95a71d984..1dad5666baa 100644 --- a/crates/net/network/src/session/mod.rs +++ b/crates/net/network/src/session/mod.rs @@ -1247,11 +1247,14 @@ async fn authenticate_stream( let cap = handler.protocol().cap; let remote_peer_id = authenticated_peer_id; + // The unsupported-handler pass above guarantees that every remaining handler has a + // matching negotiated capability. The multiplexer retains the same immutable set of + // shared capabilities, so installing one of these handlers cannot fail. multiplex_stream .install_protocol(&cap, move |conn| { handler.into_connection(direction, remote_peer_id, conn) }) - .ok(); + .expect("remaining handler capability was negotiated"); } let (multiplex_stream, their_status) = match multiplex_stream From 61b0950fe0d0ffbd3756423c7ce206875daa6460 Mon Sep 17 00:00:00 2001 From: Soubhik Singha Mahapatra <160333583+Soubhik-10@users.noreply.github.com> Date: Thu, 13 Aug 2026 09:54:01 +0000 Subject: [PATCH 010/294] feat(net): add outbound GetCells transaction client (#26673) --- crates/net/network/src/transactions/mod.rs | 29 +++++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/crates/net/network/src/transactions/mod.rs b/crates/net/network/src/transactions/mod.rs index 206a8b5b80a..3b820f4f9bd 100644 --- a/crates/net/network/src/transactions/mod.rs +++ b/crates/net/network/src/transactions/mod.rs @@ -48,7 +48,7 @@ use alloy_rlp::Encodable; use constants::SOFT_LIMIT_COUNT_HASHES_IN_NEW_POOLED_TRANSACTIONS_BROADCAST_MESSAGE; use futures::{stream::FuturesUnordered, Future, StreamExt}; use reth_eth_wire::{ - BroadcastPoolTransactions, DedupPayload, EthNetworkPrimitives, EthVersion, + BroadcastPoolTransactions, Cells, DedupPayload, EthNetworkPrimitives, EthVersion, GetCells, GetPooledTransactions, HandleMempoolData, HandleVersionedMempoolData, LazyEncoded, LazyEncodedTransaction, NetworkPrimitives, NewPooledTransactionHashes, NewPooledTransactionHashes66, NewPooledTransactionHashes68, NewPooledTransactionHashes72, @@ -225,6 +225,33 @@ impl TransactionsHandle { rx.await?.map(|res| Some(res.0)) } + + /// Requests cells from a specific eth/72 peer. + /// + /// This is the transport-level client primitive for EIP-8070. The caller supplies the + /// transaction hashes and the common cell mask for the request. Announcement-driven batching, + /// retries, cell verification, and blob reconstruction are layered above this method. + /// + /// Returns `None` if the peer is no longer connected. + pub async fn get_cells_from( + &self, + peer_id: PeerId, + hashes: Vec, + cell_mask: alloy_primitives::B128, + ) -> Result, RequestError> { + if hashes.is_empty() { + return Ok(Some(Cells { cell_mask, ..Default::default() })) + } + + let Some(peer) = self.peer_handle(peer_id).await? else { return Ok(None) }; + + let (tx, rx) = oneshot::channel(); + let request = + PeerRequest::GetCells { request: GetCells { hashes, cell_mask }, response: tx }; + peer.try_send(request).map_err(|_| RequestError::ChannelClosed)?; + + rx.await?.map(Some) + } } /// Manages transactions on top of the p2p network. From 050abcf9be8d09c69ddd073847264e2f32b4d92c Mon Sep 17 00:00:00 2001 From: snowingfox <1503401882@qq.com> Date: Thu, 13 Aug 2026 10:29:52 +0000 Subject: [PATCH 011/294] fix(txpool): count blob pool transactions in queued_transactions_count (#26677) Co-authored-by: Matthias Seitz --- crates/transaction-pool/src/pool/txpool.rs | 25 +++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/crates/transaction-pool/src/pool/txpool.rs b/crates/transaction-pool/src/pool/txpool.rs index a0e6782e5c8..b33047c576b 100644 --- a/crates/transaction-pool/src/pool/txpool.rs +++ b/crates/transaction-pool/src/pool/txpool.rs @@ -504,7 +504,7 @@ impl TxPool { /// Returns the number of transactions in parked pools pub(crate) fn queued_transactions_count(&self) -> usize { - self.basefee_pool.len() + self.queued_pool.len() + self.basefee_pool.len() + self.queued_pool.len() + self.blob_pool.len() } /// Returns queued and pending transactions for the specified sender @@ -2515,6 +2515,29 @@ mod tests { assert!(pool.blob_pool.is_empty()); } + #[test] + fn test_queued_count_includes_blob_pool() { + let on_chain_balance = U256::MAX; + let on_chain_nonce = 0; + let mut f = MockTransactionFactory::default(); + let mut pool = TxPool::new(MockOrdering::default(), Default::default()); + let tx = MockTransaction::eip4844().inc_price().inc_limit(); + + // set block info so the tx is underpriced w.r.t. blob fee and lands in the blob pool + let mut block_info = pool.block_info(); + block_info.pending_blob_fee = Some(tx.max_fee_per_blob_gas().unwrap() + 1); + pool.set_block_info(block_info); + + let validated = f.validated(tx); + pool.add_transaction(validated, on_chain_balance, on_chain_nonce, None).unwrap(); + + assert_eq!(pool.blob_pool.len(), 1); + assert!(pool.pending_pool.is_empty()); + + // blob pool transactions are parked and must be reported as queued + assert_eq!(pool.queued_transactions_count(), 1); + } + /// A struct representing a txpool promotion test instance #[derive(Debug, PartialEq, Eq, Clone, Hash)] struct PromotionTest { From dccc43c22b58841ea6fb90264049166f58001195 Mon Sep 17 00:00:00 2001 From: knows <99597038+evmKnows@users.noreply.github.com> Date: Thu, 13 Aug 2026 12:05:13 +0000 Subject: [PATCH 012/294] feat(rpc): allow selecting HTTP response compression algorithms (#26668) Co-authored-by: Matthias Seitz --- crates/node/core/src/args/rpc_server.rs | 48 +++++++++++++++++++ crates/rpc/rpc-builder/src/config.rs | 3 +- crates/rpc/rpc-builder/src/lib.rs | 40 +++++++++++++--- crates/rpc/rpc-layer/src/compression_layer.rs | 25 +++++++++- docs/vocs/docs/pages/cli/reth/node.mdx | 9 ++++ 5 files changed, 116 insertions(+), 9 deletions(-) diff --git a/crates/node/core/src/args/rpc_server.rs b/crates/node/core/src/args/rpc_server.rs index e04df70db74..79b17023a00 100644 --- a/crates/node/core/src/args/rpc_server.rs +++ b/crates/node/core/src/args/rpc_server.rs @@ -54,6 +54,7 @@ pub struct DefaultRpcServerArgs { http_addr: IpAddr, http_port: u16, http_disable_compression: bool, + http_compression_algorithms: Option>, http_api: Option, http_corsdomain: Option, ws: bool, @@ -131,6 +132,12 @@ impl DefaultRpcServerArgs { self } + /// Set the default allowed HTTP response compression algorithms + pub fn with_http_compression_algorithms(mut self, v: Option>) -> Self { + self.http_compression_algorithms = v; + self + } + /// Set the default HTTP API modules pub fn with_http_api(mut self, v: Option) -> Self { self.http_api = v; @@ -379,6 +386,7 @@ impl Default for DefaultRpcServerArgs { http_addr: Ipv4Addr::LOCALHOST.into(), http_port: constants::DEFAULT_HTTP_RPC_PORT, http_disable_compression: false, + http_compression_algorithms: None, http_api: None, http_corsdomain: None, ws: false, @@ -444,6 +452,22 @@ pub struct RpcServerArgs { #[arg(long = "http.disable-compression", default_value_t = DefaultRpcServerArgs::get_global().http_disable_compression)] pub http_disable_compression: bool, + /// Comma-separated list of allowed compression algorithms for HTTP responses. + /// + /// If not specified, all supported algorithms are enabled. + /// + /// Client `Accept-Encoding` quality values select among allowed algorithms; ties prefer + /// zstd > br > gzip > deflate. Omitted quality values default to 1; without an acceptable + /// allowed algorithm, the response is uncompressed. List order is ignored. + #[arg( + long = "http.compression", + value_name = "ALGOS", + value_delimiter = ',', + value_parser = ["zstd", "gzip", "deflate", "br"], + default_value = Resettable::from(DefaultRpcServerArgs::get_global().http_compression_algorithms.as_ref().map(|v| v.join(",").into())) + )] + pub http_compression_algorithms: Option>, + /// Rpc Modules to be configured for the HTTP server #[arg(long = "http.api", value_parser = RpcModuleSelectionValueParser::default(), default_value = Resettable::from(DefaultRpcServerArgs::get_global().http_api.as_ref().map(|v| v.to_string().into())))] pub http_api: Option, @@ -839,6 +863,7 @@ impl Default for RpcServerArgs { http_addr, http_port, http_disable_compression, + http_compression_algorithms, http_api, http_corsdomain, ws, @@ -885,6 +910,7 @@ impl Default for RpcServerArgs { http_addr, http_port, http_disable_compression, + http_compression_algorithms, http_api, http_corsdomain, ws, @@ -1005,6 +1031,27 @@ mod tests { assert_eq!(apis, expected); } + #[test] + fn http_compression_algorithms_are_optional() { + let args = CommandParser::::parse_from(["reth"]).args; + assert!(args.http_compression_algorithms.is_none()); + + let args = CommandParser::::parse_from([ + "reth", + "--http.compression", + "zstd,gzip,deflate,br", + ]) + .args; + assert_eq!( + args.http_compression_algorithms.as_deref().unwrap(), + ["zstd", "gzip", "deflate", "br"] + ); + + let result = + CommandParser::::try_parse_from(["reth", "--http.compression", "gizp"]); + assert!(result.is_err()); + } + #[test] fn rpc_server_args_default_sanity_test() { let default_args = RpcServerArgs::default(); @@ -1058,6 +1105,7 @@ mod tests { http_addr: "127.0.0.1".parse().unwrap(), http_port: 8545, http_disable_compression: false, + http_compression_algorithms: None, http_api: Some(RpcModuleSelection::try_from_selection(["eth", "admin"]).unwrap()), http_corsdomain: Some("*".to_string()), ws: true, diff --git a/crates/rpc/rpc-builder/src/config.rs b/crates/rpc/rpc-builder/src/config.rs index 6518ba6deac..4dc6e2edc81 100644 --- a/crates/rpc/rpc-builder/src/config.rs +++ b/crates/rpc/rpc-builder/src/config.rs @@ -213,7 +213,8 @@ impl RethRpcServerConfig for RpcServerArgs { .with_http_address(socket_address) .with_http(self.http_ws_server_builder()) .with_http_cors(self.http_corsdomain.clone()) - .with_http_disable_compression(self.http_disable_compression); + .with_http_disable_compression(self.http_disable_compression) + .with_http_compression_algorithms(self.http_compression_algorithms.clone()); } if self.ws { diff --git a/crates/rpc/rpc-builder/src/lib.rs b/crates/rpc/rpc-builder/src/lib.rs index 3e818dabc56..e32884d83c5 100644 --- a/crates/rpc/rpc-builder/src/lib.rs +++ b/crates/rpc/rpc-builder/src/lib.rs @@ -1087,6 +1087,10 @@ pub struct RpcServerConfig { http_addr: Option, /// Control whether http responses should be compressed http_disable_compression: bool, + /// Allowed compression algorithms for HTTP responses. + /// + /// If `None`, all supported algorithms are enabled. + http_compression_algorithms: Option>, /// Configs for WS server ws_server_config: Option, /// Allowed CORS Domains for ws. @@ -1115,6 +1119,7 @@ impl Default for RpcServerConfig { http_cors_domains: None, http_addr: None, http_disable_compression: false, + http_compression_algorithms: None, ws_server_config: None, ws_cors_domains: None, ws_addr: None, @@ -1180,6 +1185,7 @@ impl RpcServerConfig { http_cors_domains: self.http_cors_domains, http_addr: self.http_addr, http_disable_compression: self.http_disable_compression, + http_compression_algorithms: self.http_compression_algorithms, ws_server_config: self.ws_server_config, ws_cors_domains: self.ws_cors_domains, ws_addr: self.ws_addr, @@ -1208,15 +1214,24 @@ impl RpcServerConfig { self } + /// Configure the cors domains for HTTP + pub fn with_http_cors(mut self, cors_domain: Option) -> Self { + self.http_cors_domains = cors_domain; + self + } + /// Configure whether HTTP responses should be compressed pub const fn with_http_disable_compression(mut self, http_disable_compression: bool) -> Self { self.http_disable_compression = http_disable_compression; self } - /// Configure the cors domains for HTTP - pub fn with_http_cors(mut self, cors_domain: Option) -> Self { - self.http_cors_domains = cors_domain; + /// Configure the allowed compression algorithms for HTTP responses. + /// + /// If `None`, all supported algorithms are enabled. See + /// [`CompressionLayer::with_algorithms`] for how the algorithms are applied. + pub fn with_http_compression_algorithms(mut self, algos: Option>) -> Self { + self.http_compression_algorithms = algos; self } @@ -1329,12 +1344,19 @@ impl RpcServerConfig { } /// Returns a [`CompressionLayer`] that adds compression support (gzip, deflate, brotli, zstd) - /// based on the client's `Accept-Encoding` header - fn maybe_compression_layer(disable_compression: bool) -> Option { + /// based on the client's `Accept-Encoding` header. + fn maybe_compression_layer( + disable_compression: bool, + algos: Option<&[String]>, + ) -> Option { if disable_compression { None } else { - Some(CompressionLayer::new()) + match algos { + // All supported algorithms are enabled when none are specified + None => Some(CompressionLayer::new()), + Some(algos) => Some(CompressionLayer::with_algorithms(algos)), + } } } @@ -1409,6 +1431,7 @@ impl RpcServerConfig { .option_layer(Self::maybe_jwt_layer(self.jwt_secret)) .option_layer(Self::maybe_compression_layer( self.http_disable_compression, + self.http_compression_algorithms.as_deref(), )), ) .set_rpc_middleware( @@ -1493,7 +1516,10 @@ impl RpcServerConfig { tower::ServiceBuilder::new() .option_layer(Self::maybe_cors_layer(self.http_cors_domains.clone())?) .option_layer(Self::maybe_jwt_layer(self.jwt_secret)) - .option_layer(Self::maybe_compression_layer(self.http_disable_compression)), + .option_layer(Self::maybe_compression_layer( + self.http_disable_compression, + self.http_compression_algorithms.as_deref(), + )), ) .set_rpc_middleware( RpcServiceBuilder::default() diff --git a/crates/rpc/rpc-layer/src/compression_layer.rs b/crates/rpc/rpc-layer/src/compression_layer.rs index 5dbf82bf511..b8129c29ae4 100644 --- a/crates/rpc/rpc-layer/src/compression_layer.rs +++ b/crates/rpc/rpc-layer/src/compression_layer.rs @@ -9,7 +9,7 @@ use tower_http::compression::{Compression, CompressionLayer as TowerCompressionL /// This layer is a wrapper around [`tower_http::compression::CompressionLayer`] that integrates /// with jsonrpsee's HTTP types. It automatically compresses responses based on the client's -/// Accept-Encoding header. +/// `Accept-Encoding` header. #[expect(missing_debug_implementations)] #[derive(Clone)] pub struct CompressionLayer { @@ -23,6 +23,29 @@ impl CompressionLayer { inner_layer: TowerCompressionLayer::new().gzip(true).br(true).deflate(true).zstd(true), } } + + /// Creates a compression layer allowing only the listed algorithms. + /// + /// The client's `Accept-Encoding` quality values determine selection among allowed algorithms; + /// ties use `tower_http`'s fixed **zstd > br > gzip > deflate** order. The order of `algos` is + /// ignored. Omitted quality values default to 1; if no allowed algorithm is acceptable, the + /// response is uncompressed. + pub fn with_algorithms(algos: &[impl AsRef]) -> Self { + // Clear tower_http's enabled-by-default algorithms to make `algos` an allowlist. + let mut layer = TowerCompressionLayer::new().no_zstd().no_gzip().no_deflate().no_br(); + + for algo in algos { + match algo.as_ref() { + "zstd" => layer = layer.zstd(true), + "deflate" => layer = layer.deflate(true), + "gzip" => layer = layer.gzip(true), + "br" | "brotli" => layer = layer.br(true), + _ => {} + } + } + + Self { inner_layer: layer } + } } impl Default for CompressionLayer { diff --git a/docs/vocs/docs/pages/cli/reth/node.mdx b/docs/vocs/docs/pages/cli/reth/node.mdx index d3835ed45d4..c243fc6a033 100644 --- a/docs/vocs/docs/pages/cli/reth/node.mdx +++ b/docs/vocs/docs/pages/cli/reth/node.mdx @@ -332,6 +332,15 @@ RPC: --http.disable-compression Disable compression for HTTP responses + --http.compression + Comma-separated list of allowed compression algorithms for HTTP responses. + + If not specified, all supported algorithms are enabled. + + Client `Accept-Encoding` quality values select among allowed algorithms; ties prefer zstd > br > gzip > deflate. Omitted quality values default to 1; without an acceptable allowed algorithm, the response is uncompressed. List order is ignored. + + [possible values: zstd, gzip, deflate, br] + --http.api Rpc Modules to be configured for the HTTP server From 6456156e97c032e89a85af9d3dd25ef50c7309a2 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Thu, 13 Aug 2026 14:26:26 +0200 Subject: [PATCH 013/294] feat(engine): add inclusion list rpc stub (#26682) --- crates/rpc/rpc-api/src/engine.rs | 6 ++++++ crates/rpc/rpc-builder/tests/it/auth.rs | 1 + crates/rpc/rpc-engine-api/src/capabilities.rs | 2 ++ crates/rpc/rpc-engine-api/src/engine_api.rs | 16 ++++++++++++++++ 4 files changed, 25 insertions(+) diff --git a/crates/rpc/rpc-api/src/engine.rs b/crates/rpc/rpc-api/src/engine.rs index 61256141070..582944132b6 100644 --- a/crates/rpc/rpc-api/src/engine.rs +++ b/crates/rpc/rpc-api/src/engine.rs @@ -229,6 +229,12 @@ pub trait EngineApi { payload_id: PayloadId, ) -> RpcResult; + /// Returns transactions selected from the local transaction pool for the FOCIL inclusion list. + /// + /// See also . + #[method(name = "getInclusionListV1")] + async fn get_inclusion_list_v1(&self) -> RpcResult>; + /// See also #[method(name = "getPayloadBodiesByHashV1")] async fn get_payload_bodies_by_hash_v1( diff --git a/crates/rpc/rpc-builder/tests/it/auth.rs b/crates/rpc/rpc-builder/tests/it/auth.rs index 236448f7f5e..033fbf36abc 100644 --- a/crates/rpc/rpc-builder/tests/it/auth.rs +++ b/crates/rpc/rpc-builder/tests/it/auth.rs @@ -110,6 +110,7 @@ where EngineApiClient::fork_choice_updated_v1(client, ForkchoiceState::default(), None).await; EngineApiClient::get_payload_v1(client, PayloadId::new([0, 0, 0, 0, 0, 0, 0, 0])).await; EngineApiClient::get_payload_v2(client, PayloadId::new([0, 0, 0, 0, 0, 0, 0, 0])).await; + EngineApiClient::get_inclusion_list_v1(client).await; EngineApiClient::get_payload_bodies_by_hash_v1(client, vec![]).await; EngineApiClient::get_payload_bodies_by_range_v1(client, U64::ZERO, U64::from(1u64)).await; EngineApiClient::exchange_capabilities(client, vec![]).await; diff --git a/crates/rpc/rpc-engine-api/src/capabilities.rs b/crates/rpc/rpc-engine-api/src/capabilities.rs index aec3fc934c3..5e0e0af043d 100644 --- a/crates/rpc/rpc-engine-api/src/capabilities.rs +++ b/crates/rpc/rpc-engine-api/src/capabilities.rs @@ -29,6 +29,7 @@ pub const CAPABILITIES: &[&str] = &[ "engine_getPayloadV4", "engine_getPayloadV5", "engine_getPayloadV6", + "engine_getInclusionListV1", "engine_newPayloadV1", "engine_newPayloadV2", "engine_newPayloadV3", @@ -250,6 +251,7 @@ mod tests { assert!(!is_critical_method("engine_getPayloadBodiesByHashV1")); assert!(!is_critical_method("engine_getPayloadBodiesByRangeV1")); assert!(!is_critical_method("engine_getClientVersionV1")); + assert!(!is_critical_method("engine_getInclusionListV1")); } #[test] diff --git a/crates/rpc/rpc-engine-api/src/engine_api.rs b/crates/rpc/rpc-engine-api/src/engine_api.rs index 639ef9340a8..109bd5ccfdb 100644 --- a/crates/rpc/rpc-engine-api/src/engine_api.rs +++ b/crates/rpc/rpc-engine-api/src/engine_api.rs @@ -1432,6 +1432,14 @@ where Ok(self.get_payload_v6_metered(payload_id).await?) } + /// Handler for `engine_getInclusionListV1`. + /// + /// See also . + async fn get_inclusion_list_v1(&self) -> RpcResult> { + trace!(target: "rpc::engine", "Serving engine_getInclusionListV1"); + Ok(Vec::new()) + } + /// Handler for `engine_getPayloadBodiesByHashV1` /// See also async fn get_payload_bodies_by_hash_v1( @@ -1690,6 +1698,14 @@ mod tests { assert_eq!(res.unwrap(), vec![client]); } + #[tokio::test] + async fn get_inclusion_list_v1_returns_empty_list() { + let (_, api) = setup_engine_api(); + + let res = EngineApiServer::get_inclusion_list_v1(&api).await.unwrap(); + assert!(res.is_empty()); + } + #[tokio::test] async fn has_blobs_returns_ordered_availability() { let (_, api) = setup_engine_api(); From 130ee0080a8ed115dcd3e527fe079d736e2a0768 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Thu, 13 Aug 2026 14:26:58 +0200 Subject: [PATCH 014/294] fix(txpool): include blob pool in queued transactions (#26679) --- crates/transaction-pool/src/pool/blob.rs | 22 ++++++++++++++- crates/transaction-pool/src/pool/txpool.rs | 31 +++++++++++++++++++++- 2 files changed, 51 insertions(+), 2 deletions(-) diff --git a/crates/transaction-pool/src/pool/blob.rs b/crates/transaction-pool/src/pool/blob.rs index 86476cdd855..01dde166d72 100644 --- a/crates/transaction-pool/src/pool/blob.rs +++ b/crates/transaction-pool/src/pool/blob.rs @@ -1,11 +1,14 @@ use super::txpool::PendingFees; use crate::{ - identifier::TransactionId, pool::size::SizeTracker, traits::BestTransactionsAttributes, + identifier::{SenderId, TransactionId}, + pool::size::SizeTracker, + traits::BestTransactionsAttributes, PoolTransaction, SubPoolLimit, ValidPoolTransaction, }; use std::{ cmp::Ordering, collections::{BTreeMap, BTreeSet}, + ops::Bound::Unbounded, sync::Arc, }; @@ -65,6 +68,23 @@ impl BlobTransactions { id } + /// Returns an iterator over all transactions in the pool + pub(crate) fn all(&self) -> impl ExactSizeIterator>> + '_ { + self.by_id.values().map(|tx| tx.transaction.clone()) + } + + /// Returns an iterator over all transactions for the given sender, using a `BTree` range + /// query. + pub(crate) fn txs_by_sender( + &self, + sender: SenderId, + ) -> impl Iterator>> + '_ { + self.by_id + .range((sender.start_bound(), Unbounded)) + .take_while(move |(other, _)| sender == other.sender) + .map(|(_, tx)| Arc::clone(&tx.transaction)) + } + /// Removes the transaction from the pool pub(crate) fn remove_transaction( &mut self, diff --git a/crates/transaction-pool/src/pool/txpool.rs b/crates/transaction-pool/src/pool/txpool.rs index b33047c576b..cb7f99a0a8d 100644 --- a/crates/transaction-pool/src/pool/txpool.rs +++ b/crates/transaction-pool/src/pool/txpool.rs @@ -499,7 +499,7 @@ impl TxPool { /// Returns all transactions from parked pools pub(crate) fn queued_transactions(&self) -> Vec>> { - self.basefee_pool.all().chain(self.queued_pool.all()).collect() + self.basefee_pool.all().chain(self.queued_pool.all()).chain(self.blob_pool.all()).collect() } /// Returns the number of transactions in parked pools @@ -522,6 +522,7 @@ impl TxPool { ) -> Vec>> { let mut txs = self.basefee_pool.txs_by_sender(sender); txs.extend(self.queued_pool.txs_by_sender(sender)); + txs.extend(self.blob_pool.txs_by_sender(sender)); txs } @@ -3669,6 +3670,34 @@ mod tests { assert!(tx_meta.state.contains(TxState::ENOUGH_FEE_CAP_BLOCK)); } + #[test] + fn queued_transactions_include_blob_pool() { + let mut f = MockTransactionFactory::default(); + let mut pool = TxPool::new(MockOrdering::default(), Default::default()); + + let initial_blob_fee = pool.all_transactions.pending_fees.blob_fee; + let tx = MockTransaction::eip4844() + .with_max_fee(500) + .with_priority_fee(1) + .with_blob_fee(initial_blob_fee + 100); + let validated = f.validated(tx); + let id = *validated.id(); + let sender = validated.sender_id(); + pool.add_transaction(validated, U256::from(1_000_000), 0, None).unwrap(); + + // Raise the base fee beyond the transaction's cap so it gets parked in the blob pool. + pool.update_basefee(600, |_| {}); + assert_eq!(pool.blob_pool.len(), 1); + + let queued = pool.queued_transactions(); + assert_eq!(queued.len(), 1); + assert_eq!(queued[0].id(), &id); + + let by_sender = pool.queued_txs_by_sender(sender); + assert_eq!(by_sender.len(), 1); + assert_eq!(by_sender[0].id(), &id); + } + #[test] fn apply_fee_updates_demotes_after_basefee_rise() { let mut f = MockTransactionFactory::default(); From ebab368d5995c2f81998cda0a52a58ee8b7968ab Mon Sep 17 00:00:00 2001 From: knows <99597038+evmKnows@users.noreply.github.com> Date: Thu, 13 Aug 2026 12:42:55 +0000 Subject: [PATCH 015/294] feat(rpc): Add request decompression (#20277) --- Cargo.lock | 3 + Cargo.toml | 1 + crates/node/core/src/args/rpc_server.rs | 53 +++ crates/rpc/rpc-builder/src/config.rs | 6 +- crates/rpc/rpc-builder/src/lib.rs | 49 +- crates/rpc/rpc-layer/Cargo.toml | 7 +- .../rpc/rpc-layer/src/decompression_layer.rs | 432 ++++++++++++++++++ crates/rpc/rpc-layer/src/lib.rs | 2 + docs/vocs/docs/pages/cli/reth/node.mdx | 11 +- 9 files changed, 560 insertions(+), 4 deletions(-) create mode 100644 crates/rpc/rpc-layer/src/decompression_layer.rs diff --git a/Cargo.lock b/Cargo.lock index 4b86aaa27e9..b45bab0635e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -10199,6 +10199,8 @@ name = "reth-rpc-layer" version = "2.5.0" dependencies = [ "alloy-rpc-types-engine", + "brotli", + "flate2", "http", "http-body-util", "jsonrpsee", @@ -10209,6 +10211,7 @@ dependencies = [ "tower", "tower-http", "tracing", + "zstd", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 016f005b1fb..936ed1c0c1e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -500,6 +500,7 @@ dyn-clone = "1.0.17" eyre = "0.6" fdlimit = "0.3.0" fixed-map = { version = "0.9", default-features = false } +flate2 = "1.0" humantime = "2.1" imbl = "7" humantime-serde = "1.1" diff --git a/crates/node/core/src/args/rpc_server.rs b/crates/node/core/src/args/rpc_server.rs index 79b17023a00..eb0f2432adc 100644 --- a/crates/node/core/src/args/rpc_server.rs +++ b/crates/node/core/src/args/rpc_server.rs @@ -55,6 +55,7 @@ pub struct DefaultRpcServerArgs { http_port: u16, http_disable_compression: bool, http_compression_algorithms: Option>, + http_decompression_algorithms: Option>, http_api: Option, http_corsdomain: Option, ws: bool, @@ -138,6 +139,12 @@ impl DefaultRpcServerArgs { self } + /// Set the default allowed HTTP request decompression algorithms + pub fn with_http_decompression_algorithms(mut self, v: Option>) -> Self { + self.http_decompression_algorithms = v; + self + } + /// Set the default HTTP API modules pub fn with_http_api(mut self, v: Option) -> Self { self.http_api = v; @@ -387,6 +394,7 @@ impl Default for DefaultRpcServerArgs { http_port: constants::DEFAULT_HTTP_RPC_PORT, http_disable_compression: false, http_compression_algorithms: None, + http_decompression_algorithms: None, http_api: None, http_corsdomain: None, ws: false, @@ -468,6 +476,18 @@ pub struct RpcServerArgs { )] pub http_compression_algorithms: Option>, + /// Comma-separated list of allowed decompression algorithms for HTTP requests. + /// + /// Request decompression is disabled when not specified. + #[arg( + long = "http.decompression", + value_name = "ALGOS", + value_delimiter = ',', + value_parser = ["zstd", "gzip", "deflate", "br"], + default_value = Resettable::from(DefaultRpcServerArgs::get_global().http_decompression_algorithms.as_ref().map(|v| v.join(",").into())) + )] + pub http_decompression_algorithms: Option>, + /// Rpc Modules to be configured for the HTTP server #[arg(long = "http.api", value_parser = RpcModuleSelectionValueParser::default(), default_value = Resettable::from(DefaultRpcServerArgs::get_global().http_api.as_ref().map(|v| v.to_string().into())))] pub http_api: Option, @@ -556,6 +576,9 @@ pub struct RpcServerArgs { pub rpc_disable_metrics: bool, /// Set the maximum RPC request payload size for both HTTP and WS in megabytes. + /// + /// For compressed HTTP requests, this limit applies to both the compressed and decompressed + /// payloads. #[arg(long = "rpc.max-request-size", alias = "rpc-max-request-size", default_value_t = DefaultRpcServerArgs::get_global().rpc_max_request_size)] pub rpc_max_request_size: MaxU32, @@ -864,6 +887,7 @@ impl Default for RpcServerArgs { http_port, http_disable_compression, http_compression_algorithms, + http_decompression_algorithms, http_api, http_corsdomain, ws, @@ -911,6 +935,7 @@ impl Default for RpcServerArgs { http_port, http_disable_compression, http_compression_algorithms, + http_decompression_algorithms, http_api, http_corsdomain, ws, @@ -1060,6 +1085,33 @@ mod tests { assert_eq!(args, default_args); } + #[test] + fn http_request_decompression_is_opt_in() { + let args = CommandParser::::parse_from(["reth"]).args; + assert!(args.http_decompression_algorithms.is_none()); + + let args = CommandParser::::parse_from([ + "reth", + "--http.decompression", + "zstd,gzip,deflate,br", + ]) + .args; + assert_eq!( + args.http_decompression_algorithms.as_deref().unwrap(), + ["zstd", "gzip", "deflate", "br"] + ); + } + + #[test] + fn invalid_http_request_decompression_algorithm_is_rejected() { + let result = CommandParser::::try_parse_from([ + "reth", + "--http.decompression", + "gizp", + ]); + assert!(result.is_err()); + } + #[test] fn test_rpc_disable_metrics_arg() { let args = CommandParser::::parse_from(["reth"]).args; @@ -1106,6 +1158,7 @@ mod tests { http_port: 8545, http_disable_compression: false, http_compression_algorithms: None, + http_decompression_algorithms: None, http_api: Some(RpcModuleSelection::try_from_selection(["eth", "admin"]).unwrap()), http_corsdomain: Some("*".to_string()), ws: true, diff --git a/crates/rpc/rpc-builder/src/config.rs b/crates/rpc/rpc-builder/src/config.rs index 4dc6e2edc81..894a0c70a65 100644 --- a/crates/rpc/rpc-builder/src/config.rs +++ b/crates/rpc/rpc-builder/src/config.rs @@ -214,7 +214,11 @@ impl RethRpcServerConfig for RpcServerArgs { .with_http(self.http_ws_server_builder()) .with_http_cors(self.http_corsdomain.clone()) .with_http_disable_compression(self.http_disable_compression) - .with_http_compression_algorithms(self.http_compression_algorithms.clone()); + .with_http_compression_algorithms(self.http_compression_algorithms.clone()) + .with_http_decompression( + self.http_decompression_algorithms.clone(), + self.rpc_max_request_size_bytes(), + ); } if self.ws { diff --git a/crates/rpc/rpc-builder/src/lib.rs b/crates/rpc/rpc-builder/src/lib.rs index e32884d83c5..b2eeab22815 100644 --- a/crates/rpc/rpc-builder/src/lib.rs +++ b/crates/rpc/rpc-builder/src/lib.rs @@ -53,7 +53,9 @@ use reth_rpc_eth_api::{ RpcConverter, RpcHeader, RpcNodeCore, RpcReceipt, RpcTransaction, RpcTxReq, }; use reth_rpc_eth_types::{receipt::EthReceiptConverter, EthConfig, EthSubscriptionIdProvider}; -use reth_rpc_layer::{AuthLayer, Claims, CompressionLayer, JwtAuthValidator, JwtSecret}; +use reth_rpc_layer::{ + AuthLayer, Claims, CompressionLayer, DecompressionLayer, JwtAuthValidator, JwtSecret, +}; pub use reth_rpc_server_types::RethRpcModule; use reth_storage_api::{ BlockReader, ChangeSetReader, FullRpcProvider, NodePrimitivesProvider, StateProviderFactory, @@ -1091,6 +1093,12 @@ pub struct RpcServerConfig { /// /// If `None`, all supported algorithms are enabled. http_compression_algorithms: Option>, + /// Allowed decompression algorithms for HTTP requests. + /// + /// If `None`, request decompression is disabled. + http_decompression_algorithms: Option>, + /// Maximum allowed HTTP request body size in bytes (decompressed). + http_max_request_body_size: Option, /// Configs for WS server ws_server_config: Option, /// Allowed CORS Domains for ws. @@ -1120,6 +1128,8 @@ impl Default for RpcServerConfig { http_addr: None, http_disable_compression: false, http_compression_algorithms: None, + http_decompression_algorithms: None, + http_max_request_body_size: None, ws_server_config: None, ws_cors_domains: None, ws_addr: None, @@ -1186,6 +1196,8 @@ impl RpcServerConfig { http_addr: self.http_addr, http_disable_compression: self.http_disable_compression, http_compression_algorithms: self.http_compression_algorithms, + http_decompression_algorithms: self.http_decompression_algorithms, + http_max_request_body_size: self.http_max_request_body_size, ws_server_config: self.ws_server_config, ws_cors_domains: self.ws_cors_domains, ws_addr: self.ws_addr, @@ -1235,6 +1247,15 @@ impl RpcServerConfig { self } + /// Configure decompression for HTTP requests. + /// + /// If `algos` is `None`, request decompression is disabled. + pub fn with_http_decompression(mut self, algos: Option>, max_size: u32) -> Self { + self.http_decompression_algorithms = algos; + self.http_max_request_body_size = Some(max_size); + self + } + /// Configures the [`SocketAddr`] of the http server /// /// Default is [`Ipv4Addr::LOCALHOST`] and @@ -1360,6 +1381,24 @@ impl RpcServerConfig { } } + /// Returns a [`DecompressionLayer`] that allows decompression of incoming requests + /// based on the `Content-Encoding` request header. + /// + /// Decompression is disabled by default, and only enabled when algorithms are explicitly + /// specified via the `algos` parameter. + fn maybe_decompression_layer( + max_request_body_size: Option, + algos: Option<&[String]>, + ) -> Option { + let algos = algos?; + if algos.is_empty() { + return None; + } + + let max = max_request_body_size?; + Some(DecompressionLayer::new(algos, max as usize)) + } + /// Builds and starts the configured server(s): http, ws, ipc. /// /// If both http and ws are on the same port, they are combined into one server. @@ -1429,6 +1468,10 @@ impl RpcServerConfig { tower::ServiceBuilder::new() .option_layer(Self::maybe_cors_layer(cors)?) .option_layer(Self::maybe_jwt_layer(self.jwt_secret)) + .option_layer(Self::maybe_decompression_layer( + self.http_max_request_body_size, + self.http_decompression_algorithms.as_deref(), + )) .option_layer(Self::maybe_compression_layer( self.http_disable_compression, self.http_compression_algorithms.as_deref(), @@ -1516,6 +1559,10 @@ impl RpcServerConfig { tower::ServiceBuilder::new() .option_layer(Self::maybe_cors_layer(self.http_cors_domains.clone())?) .option_layer(Self::maybe_jwt_layer(self.jwt_secret)) + .option_layer(Self::maybe_decompression_layer( + self.http_max_request_body_size, + self.http_decompression_algorithms.as_deref(), + )) .option_layer(Self::maybe_compression_layer( self.http_disable_compression, self.http_compression_algorithms.as_deref(), diff --git a/crates/rpc/rpc-layer/Cargo.toml b/crates/rpc/rpc-layer/Cargo.toml index 8d128d2c5f0..b9507fc2add 100644 --- a/crates/rpc/rpc-layer/Cargo.toml +++ b/crates/rpc/rpc-layer/Cargo.toml @@ -14,8 +14,10 @@ workspace = true alloy-rpc-types-engine = { workspace = true, features = ["jwt-aws-lc-rs", "serde"] } http.workspace = true +http-body-util.workspace = true jsonrpsee-http-client.workspace = true pin-project.workspace = true +tokio = { workspace = true, features = ["sync"] } tower.workspace = true tower-http = { workspace = true, features = ["full"] } tracing.workspace = true @@ -24,4 +26,7 @@ tracing.workspace = true reqwest.workspace = true tokio = { workspace = true, features = ["macros"] } jsonrpsee = { workspace = true, features = ["server"] } -http-body-util.workspace = true + +flate2.workspace = true +brotli.workspace = true +zstd.workspace = true diff --git a/crates/rpc/rpc-layer/src/decompression_layer.rs b/crates/rpc/rpc-layer/src/decompression_layer.rs new file mode 100644 index 00000000000..077003c5e98 --- /dev/null +++ b/crates/rpc/rpc-layer/src/decompression_layer.rs @@ -0,0 +1,432 @@ +use http::StatusCode; +use http_body_util::{BodyExt, LengthLimitError, Limited}; +use jsonrpsee_http_client::{HttpBody, HttpRequest, HttpResponse}; +use std::{ + future::{poll_fn, Future}, + pin::Pin, + sync::Arc, + task::{Context, Poll}, +}; +use tokio::sync::Semaphore; +use tower::{Layer, Service}; +use tower_http::decompression::{ + RequestDecompression, RequestDecompressionLayer as TowerDecompressionLayer, +}; +use tracing::debug; + +/// Maximum number of request bodies that may be decompressed concurrently. +/// +/// Each in-flight decompression can materialize up to the configured maximum body size, so this +/// bounds the worst-case memory usage at `MAX_CONCURRENT_DECOMPRESSIONS * max_body_size` (120 MiB +/// with the default 15 MiB request size limit). Excess requests wait for a permit instead of +/// being rejected. +const MAX_CONCURRENT_DECOMPRESSIONS: usize = 8; + +/// This layer is a wrapper around [`tower_http::decompression::RequestDecompressionLayer`] that +/// integrates with jsonrpsee's HTTP types. +#[expect(missing_debug_implementations)] +#[derive(Clone)] +pub struct DecompressionLayer { + inner_layer: TowerDecompressionLayer, + /// Maximum size in bytes for both compressed and decompressed bodies. + max_body_size: usize, + /// Bounds concurrent decompression work across all services created from this layer. + decompression_permits: Arc, +} + +impl DecompressionLayer { + /// Creates a new decompression layer from a list of algorithm names. + /// Supported: zstd, gzip, deflate, br + pub fn new(algos: &[impl AsRef], max_body_size: usize) -> Self { + // Start with all algorithms explicitly disabled + let mut layer = TowerDecompressionLayer::new().no_zstd().no_gzip().no_deflate().no_br(); + + // Only enable the algorithms that were explicitly passed. + for algo in algos { + match algo.as_ref() { + "zstd" => layer = layer.zstd(true), + "gzip" => layer = layer.gzip(true), + "deflate" => layer = layer.deflate(true), + "br" | "brotli" => layer = layer.br(true), + _ => {} + } + } + + Self { + inner_layer: layer, + max_body_size, + decompression_permits: Arc::new(Semaphore::new(MAX_CONCURRENT_DECOMPRESSIONS)), + } + } +} + +impl Layer for DecompressionLayer { + type Service = DecompressionService; + + fn layer(&self, inner: S) -> Self::Service { + DecompressionService { + decompression: self.inner_layer.layer(InnerService { + inner, + max_body_size: self.max_body_size, + decompression_permits: self.decompression_permits.clone(), + }), + max_body_size: self.max_body_size, + } + } +} + +/// Service that performs request decompression with body size limiting. +/// +/// Created by [`DecompressionLayer`]. +#[expect(missing_debug_implementations)] +#[derive(Clone)] +pub struct DecompressionService { + decompression: RequestDecompression>, + max_body_size: usize, +} + +/// Inner service wrapper to handle type conversion between jsonrpsee and `tower_http` +/// with body size limiting. +#[derive(Clone)] +struct InnerService { + inner: S, + max_body_size: usize, + decompression_permits: Arc, +} + +/// Marks a request whose body Tower will decompress before it reaches [`InnerService`]. +#[derive(Clone, Copy, Debug)] +struct CompressedBody { + content_length: Option, +} + +impl Service>> + for InnerService +where + S: Service + Clone + Send + 'static, + S::Future: Send + 'static, +{ + type Response = http::Response; + type Error = S::Error; + type Future = Pin> + Send>>; + + fn poll_ready(&mut self, _cx: &mut Context<'_>) -> Poll> { + Poll::Ready(Ok(())) + } + + fn call( + &mut self, + req: http::Request>, + ) -> Self::Future { + let mut inner = self.inner.clone(); + let max_body_size = self.max_body_size; + let decompression_permits = self.decompression_permits.clone(); + Box::pin(async move { + let (mut parts, body) = req.into_parts(); + + let Some(compressed) = parts.extensions.remove::() else { + poll_fn(|cx| inner.poll_ready(cx)).await?; + return inner.call(HttpRequest::from_parts(parts, HttpBody::new(body))).await; + }; + + if compressed.content_length.is_some_and(|length| length > max_body_size as u64) { + return Ok(err_response(StatusCode::PAYLOAD_TOO_LARGE, "Payload Too Large")); + } + + // HTTP/2 allows many concurrent streams per connection, so bound how many bodies are + // decompressed and materialized at once to cap memory and CPU usage. + let permit = decompression_permits + .acquire_owned() + .await + .expect("decompression semaphore is never closed"); + + let body = match Limited::new(body, max_body_size).collect().await { + Ok(body) => body, + Err(err) if err.is::() => { + return Ok(err_response(StatusCode::PAYLOAD_TOO_LARGE, "Payload Too Large")); + } + Err(err) => { + debug!(target: "rpc::decompression", %err, "Failed to decompress request body"); + return Ok(err_response(StatusCode::BAD_REQUEST, "Invalid compressed body")); + } + }; + + // Decompression is done; release the permit before dispatching the request. + drop(permit); + + poll_fn(|cx| inner.poll_ready(cx)).await?; + inner.call(HttpRequest::from_parts(parts, HttpBody::new(body))).await + }) + } +} + +impl Service for DecompressionService +where + S: Service + Clone + Send + 'static, + S::Future: Send + 'static, +{ + type Response = HttpResponse; + type Error = S::Error; + type Future = Pin> + Send>>; + + fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll> { + self.decompression.poll_ready(cx) + } + + fn call(&mut self, mut req: HttpRequest) -> Self::Future { + // RFC 9110 §8.4.1: content-coding tokens are case-insensitive, but `tower_http` matches + // lowercase bytes exactly, so normalize the header before decompression. + if let Some(encoding) = req.headers().get(http::header::CONTENT_ENCODING) && + encoding.as_bytes().iter().any(u8::is_ascii_uppercase) && + let Ok(normalized) = + http::HeaderValue::from_bytes(&encoding.as_bytes().to_ascii_lowercase()) + { + req.headers_mut().insert(http::header::CONTENT_ENCODING, normalized); + } + + if req + .headers() + .get(http::header::CONTENT_ENCODING) + .is_some_and(|encoding| encoding.as_bytes() != b"identity") + { + let content_length = req + .headers() + .get(http::header::CONTENT_LENGTH) + .and_then(|value| value.to_str().ok()) + .and_then(|value| value.parse().ok()); + req.extensions_mut().insert(CompressedBody { content_length }); + + let (parts, body) = req.into_parts(); + req = HttpRequest::from_parts( + parts, + HttpBody::new(Limited::new(body, self.max_body_size)), + ); + } + + let fut = self.decompression.call(req); + + Box::pin(async move { Ok(fut.await?.map(HttpBody::new)) }) + } +} + +#[inline] +fn err_response(status: StatusCode, msg: &'static str) -> HttpResponse { + http::Response::builder() + .status(status) + .header(http::header::CONTENT_TYPE, "text/plain") + .body(HttpBody::from(msg)) + .expect("static error response is valid") +} + +#[cfg(test)] +mod tests { + use super::*; + use http::header::CONTENT_ENCODING; + use http_body_util::BodyExt; + use jsonrpsee_http_client::{HttpRequest, HttpResponse}; + use std::{convert::Infallible, future::ready, io::Write}; + + const TEST_DATA: &str = r#"{"method":"test","params":["test data"],"id":1}"#; + const DEFAULT_MAX_SIZE: usize = 15 * 1024 * 1024; + + type Compressor = fn(&[u8]) -> Vec; + + #[derive(Clone)] + struct MockEchoService; + + impl Service for MockEchoService { + type Response = HttpResponse; + type Error = Infallible; + type Future = std::future::Ready>; + + fn poll_ready( + &mut self, + _: &mut std::task::Context<'_>, + ) -> std::task::Poll> { + std::task::Poll::Ready(Ok(())) + } + + fn call(&mut self, req: HttpRequest) -> Self::Future { + let (_parts, body) = req.into_parts(); + ready(Ok(HttpResponse::builder().status(200).body(body).unwrap())) + } + } + + fn setup_service( + algorithms: &[&str], + max_size: usize, + ) -> DecompressionService { + DecompressionLayer::new(algorithms, max_size).layer(MockEchoService) + } + + async fn get_response_body(response: HttpResponse) -> Vec { + response.into_body().collect().await.unwrap().to_bytes().to_vec() + } + + fn compress_gzip(data: &[u8]) -> Vec { + let mut encoder = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::default()); + encoder.write_all(data).unwrap(); + encoder.finish().unwrap() + } + + fn compress_deflate(data: &[u8]) -> Vec { + let mut encoder = + flate2::write::ZlibEncoder::new(Vec::new(), flate2::Compression::default()); + encoder.write_all(data).unwrap(); + encoder.finish().unwrap() + } + + fn compress_brotli(data: &[u8]) -> Vec { + let mut compressed = Vec::new(); + { + let mut encoder = brotli::CompressorWriter::new(&mut compressed, 4096, 11, 22); + encoder.write_all(data).unwrap(); + encoder.flush().unwrap(); + } + compressed + } + + fn compress_zstd(data: &[u8]) -> Vec { + let mut compressed = Vec::new(); + { + let mut encoder = zstd::Encoder::new(&mut compressed, 3).unwrap(); + encoder.write_all(data).unwrap(); + encoder.finish().unwrap(); + } + compressed + } + + fn build_compressed_request(encoding: &str, body: Vec) -> HttpRequest { + HttpRequest::builder() + .header(CONTENT_ENCODING, encoding) + .body(HttpBody::from(body)) + .unwrap() + } + + #[tokio::test] + async fn configured_algorithms_are_decompressed_without_content_length() { + let cases: [(&str, Compressor); 4] = [ + ("zstd", compress_zstd), + ("gzip", compress_gzip), + ("deflate", compress_deflate), + ("br", compress_brotli), + ]; + + for (algorithm, compress) in cases { + let mut service = setup_service(&[algorithm], DEFAULT_MAX_SIZE); + let request = build_compressed_request(algorithm, compress(TEST_DATA.as_bytes())); + let response = service.call(request).await.unwrap(); + + assert_eq!(response.status(), StatusCode::OK, "algorithm: {algorithm}"); + assert_eq!(get_response_body(response).await, TEST_DATA.as_bytes()); + } + } + + #[tokio::test] + async fn oversized_decompressed_body_is_rejected() { + const MAX_SIZE: usize = 1024; + let mut service = setup_service(&["gzip"], MAX_SIZE); + let body = vec![0; MAX_SIZE + 1]; + let response = + service.call(build_compressed_request("gzip", compress_gzip(&body))).await.unwrap(); + + assert_eq!(response.status(), StatusCode::PAYLOAD_TOO_LARGE); + } + + #[tokio::test] + async fn oversized_compressed_body_without_content_length_is_rejected() { + let compressed = compress_gzip(TEST_DATA.as_bytes()); + let max_size = compressed.len() - 1; + assert!(TEST_DATA.len() <= max_size); + + let mut service = setup_service(&["gzip"], max_size); + let response = service.call(build_compressed_request("gzip", compressed)).await.unwrap(); + + assert_eq!(response.status(), StatusCode::PAYLOAD_TOO_LARGE); + } + + #[tokio::test] + async fn oversized_compressed_content_length_is_rejected() { + let compressed = compress_gzip(TEST_DATA.as_bytes()); + let max_size = compressed.len(); + let mut request = build_compressed_request("gzip", compressed); + request.headers_mut().insert(http::header::CONTENT_LENGTH, (max_size + 1).into()); + + let mut service = setup_service(&["gzip"], max_size); + let response = service.call(request).await.unwrap(); + + assert_eq!(response.status(), StatusCode::PAYLOAD_TOO_LARGE); + } + + #[tokio::test] + async fn malformed_compressed_body_is_rejected() { + let mut service = setup_service(&["gzip"], DEFAULT_MAX_SIZE); + let response = service + .call(build_compressed_request("gzip", b"not a gzip stream".to_vec())) + .await + .unwrap(); + + assert_eq!(response.status(), StatusCode::BAD_REQUEST); + } + + #[tokio::test] + async fn disabled_algorithm_is_rejected() { + let mut service = setup_service(&["gzip"], DEFAULT_MAX_SIZE); + let mut request = build_compressed_request("zstd", compress_zstd(TEST_DATA.as_bytes())); + request.headers_mut().insert(http::header::CONTENT_LENGTH, (DEFAULT_MAX_SIZE + 1).into()); + + let response = service.call(request).await.unwrap(); + assert_eq!(response.status(), StatusCode::UNSUPPORTED_MEDIA_TYPE); + } + + #[tokio::test] + async fn mixed_case_content_codings_are_accepted() { + let cases: [(&str, Compressor); 3] = + [("GZip", compress_gzip), ("ZSTD", compress_zstd), ("Br", compress_brotli)]; + + for (encoding, compress) in cases { + let mut service = + setup_service(&[encoding.to_ascii_lowercase().as_str()], DEFAULT_MAX_SIZE); + let request = build_compressed_request(encoding, compress(TEST_DATA.as_bytes())); + let response = service.call(request).await.unwrap(); + + assert_eq!(response.status(), StatusCode::OK, "encoding: {encoding}"); + assert_eq!(get_response_body(response).await, TEST_DATA.as_bytes()); + } + } + + #[tokio::test] + async fn concurrent_compressed_requests_all_complete() { + let service = setup_service(&["gzip"], DEFAULT_MAX_SIZE); + let mut tasks = tokio::task::JoinSet::new(); + + // Spawn more requests than there are decompression permits to ensure permits are + // released and waiting requests complete. + for _ in 0..4 * MAX_CONCURRENT_DECOMPRESSIONS { + let mut service = service.clone(); + tasks.spawn(async move { + let request = build_compressed_request("gzip", compress_gzip(TEST_DATA.as_bytes())); + service.call(request).await.unwrap() + }); + } + + while let Some(response) = tasks.join_next().await { + assert_eq!(response.unwrap().status(), StatusCode::OK); + } + } + + #[tokio::test] + async fn identity_and_unencoded_bodies_are_not_eagerly_collected() { + for encoding in [None, Some("identity"), Some("Identity")] { + let mut service = setup_service(&["gzip"], DEFAULT_MAX_SIZE); + let body = Limited::new(HttpBody::from(TEST_DATA), 0); + let mut request = HttpRequest::builder(); + if let Some(encoding) = encoding { + request = request.header(CONTENT_ENCODING, encoding); + } + + let response = service.call(request.body(HttpBody::new(body)).unwrap()).await.unwrap(); + assert_eq!(response.status(), StatusCode::OK); + } + } +} diff --git a/crates/rpc/rpc-layer/src/lib.rs b/crates/rpc/rpc-layer/src/lib.rs index b7f6f29cbb9..abd777b55e4 100644 --- a/crates/rpc/rpc-layer/src/lib.rs +++ b/crates/rpc/rpc-layer/src/lib.rs @@ -14,10 +14,12 @@ use jsonrpsee_http_client::HttpResponse; mod auth_client_layer; mod auth_layer; mod compression_layer; +mod decompression_layer; mod jwt_validator; pub use auth_layer::{AuthService, ResponseFuture}; pub use compression_layer::CompressionLayer; +pub use decompression_layer::DecompressionLayer; // Export alloy JWT types pub use alloy_rpc_types_engine::{Claims, JwtError, JwtSecret}; diff --git a/docs/vocs/docs/pages/cli/reth/node.mdx b/docs/vocs/docs/pages/cli/reth/node.mdx index c243fc6a033..ed65fd43fe2 100644 --- a/docs/vocs/docs/pages/cli/reth/node.mdx +++ b/docs/vocs/docs/pages/cli/reth/node.mdx @@ -341,6 +341,13 @@ RPC: [possible values: zstd, gzip, deflate, br] + --http.decompression + Comma-separated list of allowed decompression algorithms for HTTP requests. + + Request decompression is disabled when not specified. + + [possible values: zstd, gzip, deflate, br] + --http.api Rpc Modules to be configured for the HTTP server @@ -422,7 +429,9 @@ RPC: Disable built-in RPC request metrics --rpc.max-request-size - Set the maximum RPC request payload size for both HTTP and WS in megabytes + Set the maximum RPC request payload size for both HTTP and WS in megabytes. + + For compressed HTTP requests, this limit applies to both the compressed and decompressed payloads. [default: 15] From b5c38a7ef7aacf43df9c98419b04b58576e17447 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Thu, 13 Aug 2026 12:53:47 +0000 Subject: [PATCH 016/294] refactor(txpool): return iterators from txs_by_sender accessors (#26681) --- crates/transaction-pool/src/pool/parked.rs | 6 +++--- crates/transaction-pool/src/pool/pending.rs | 6 +++--- crates/transaction-pool/src/pool/txpool.rs | 11 ++++++----- 3 files changed, 12 insertions(+), 11 deletions(-) diff --git a/crates/transaction-pool/src/pool/parked.rs b/crates/transaction-pool/src/pool/parked.rs index e3efd30967f..7bd524fcde5 100644 --- a/crates/transaction-pool/src/pool/parked.rs +++ b/crates/transaction-pool/src/pool/parked.rs @@ -151,16 +151,16 @@ impl ParkedPool { .collect() } - /// Returns all transactions for the given sender, using a `BTree` range query. + /// Returns an iterator over all transactions for the given sender, using a `BTree` range + /// query. pub(crate) fn txs_by_sender( &self, sender: SenderId, - ) -> Vec>> { + ) -> impl Iterator>> + '_ { self.by_id .range((sender.start_bound(), Unbounded)) .take_while(move |(other, _)| sender == other.sender) .map(|(_, tx)| Arc::clone(&tx.transaction)) - .collect() } #[cfg(test)] diff --git a/crates/transaction-pool/src/pool/pending.rs b/crates/transaction-pool/src/pool/pending.rs index 7ca27ad9b66..b273758b86a 100644 --- a/crates/transaction-pool/src/pool/pending.rs +++ b/crates/transaction-pool/src/pool/pending.rs @@ -583,16 +583,16 @@ impl PendingPool { .map(|(tx_id, _)| tx_id) } - /// Returns all transactions for the given sender, using a `BTree` range query. + /// Returns an iterator over all transactions for the given sender, using a `BTree` range + /// query. pub(crate) fn txs_by_sender( &self, sender: SenderId, - ) -> Vec>> { + ) -> impl Iterator>> + '_ { self.by_id .range((sender.start_bound(), Unbounded)) .take_while(move |(other, _)| sender == other.sender) .map(|(_, tx)| tx.transaction.clone()) - .collect() } /// Retrieves a transaction with the given ID from the pool, if it exists. diff --git a/crates/transaction-pool/src/pool/txpool.rs b/crates/transaction-pool/src/pool/txpool.rs index cb7f99a0a8d..0fd49c7ad35 100644 --- a/crates/transaction-pool/src/pool/txpool.rs +++ b/crates/transaction-pool/src/pool/txpool.rs @@ -494,7 +494,7 @@ impl TxPool { &self, sender: SenderId, ) -> Vec>> { - self.pending_pool.txs_by_sender(sender) + self.pending_pool.txs_by_sender(sender).collect() } /// Returns all transactions from parked pools @@ -520,10 +520,11 @@ impl TxPool { &self, sender: SenderId, ) -> Vec>> { - let mut txs = self.basefee_pool.txs_by_sender(sender); - txs.extend(self.queued_pool.txs_by_sender(sender)); - txs.extend(self.blob_pool.txs_by_sender(sender)); - txs + self.basefee_pool + .txs_by_sender(sender) + .chain(self.queued_pool.txs_by_sender(sender)) + .chain(self.blob_pool.txs_by_sender(sender)) + .collect() } /// Returns `true` if the transaction with the given hash is already included in this pool. From e2671fa888b7d56384b03b384c86dc55793c91e8 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Thu, 13 Aug 2026 16:07:01 +0200 Subject: [PATCH 017/294] refactor: default unspecified payload attributes (#26684) --- .../src/testsuite/actions/produce_blocks.rs | 4 ++-- crates/e2e-test-utils/src/testsuite/setup.rs | 4 ++-- crates/e2e-test-utils/tests/e2e-testsuite/main.rs | 2 +- crates/e2e-test-utils/tests/rocksdb/main.rs | 4 ++-- crates/engine/tree/src/tree/tests.rs | 8 ++++---- crates/ethereum/node/src/engine_ssz_containers.rs | 8 +++++--- crates/ethereum/node/tests/e2e/eth.rs | 4 ++-- crates/ethereum/node/tests/e2e/utils.rs | 12 ++++++------ crates/ethereum/node/tests/it/testing.rs | 4 +++- crates/payload/builder/src/service.rs | 2 +- crates/payload/primitives/src/traits.rs | 10 ++++++---- crates/rpc/rpc-engine-api/src/engine_api.rs | 6 +++--- 12 files changed, 37 insertions(+), 31 deletions(-) diff --git a/crates/e2e-test-utils/src/testsuite/actions/produce_blocks.rs b/crates/e2e-test-utils/src/testsuite/actions/produce_blocks.rs index 39d3af4b626..5915ee03101 100644 --- a/crates/e2e-test-utils/src/testsuite/actions/produce_blocks.rs +++ b/crates/e2e-test-utils/src/testsuite/actions/produce_blocks.rs @@ -228,7 +228,7 @@ where withdrawals: Some(vec![]), parent_beacon_block_root: Some(B256::ZERO), slot_number: None, - target_gas_limit: None, + ..Default::default() }; env.active_node_state_mut()? @@ -302,7 +302,7 @@ where withdrawals: Some(vec![]), parent_beacon_block_root: Some(B256::ZERO), slot_number: None, - target_gas_limit: None, + ..Default::default() }; let fresh_fcu_result = EngineApiClient::::fork_choice_updated_v3( diff --git a/crates/e2e-test-utils/src/testsuite/setup.rs b/crates/e2e-test-utils/src/testsuite/setup.rs index 4909545d8ee..79f4a86d0e8 100644 --- a/crates/e2e-test-utils/src/testsuite/setup.rs +++ b/crates/e2e-test-utils/src/testsuite/setup.rs @@ -270,7 +270,7 @@ where withdrawals: Some(vec![]), parent_beacon_block_root: Some(B256::ZERO), slot_number: None, - target_gas_limit: None, + ..Default::default() }; crate::setup_import::setup_engine_with_chain_import( @@ -298,7 +298,7 @@ where withdrawals: Some(vec![]), parent_beacon_block_root: Some(B256::ZERO), slot_number: None, - target_gas_limit: None, + ..Default::default() } .into() } diff --git a/crates/e2e-test-utils/tests/e2e-testsuite/main.rs b/crates/e2e-test-utils/tests/e2e-testsuite/main.rs index 920fd160df9..023a50b343c 100644 --- a/crates/e2e-test-utils/tests/e2e-testsuite/main.rs +++ b/crates/e2e-test-utils/tests/e2e-testsuite/main.rs @@ -161,7 +161,7 @@ async fn test_testsuite_assert_mine_block() -> Result<()> { withdrawals: None, parent_beacon_block_root: None, slot_number: None, - target_gas_limit: None, + ..Default::default() }, )); diff --git a/crates/e2e-test-utils/tests/rocksdb/main.rs b/crates/e2e-test-utils/tests/rocksdb/main.rs index f278978e383..dc55ae8430e 100644 --- a/crates/e2e-test-utils/tests/rocksdb/main.rs +++ b/crates/e2e-test-utils/tests/rocksdb/main.rs @@ -83,7 +83,7 @@ fn test_chain_spec() -> Arc { } /// Returns test payload attributes for the given timestamp. -const fn test_attributes_generator(timestamp: u64) -> PayloadAttributes { +fn test_attributes_generator(timestamp: u64) -> PayloadAttributes { PayloadAttributes { timestamp, prev_randao: B256::ZERO, @@ -91,7 +91,7 @@ const fn test_attributes_generator(timestamp: u64) -> PayloadAttributes { withdrawals: Some(vec![]), parent_beacon_block_root: Some(B256::ZERO), slot_number: None, - target_gas_limit: None, + ..Default::default() } } diff --git a/crates/engine/tree/src/tree/tests.rs b/crates/engine/tree/src/tree/tests.rs index 53cbf1d4e5f..9fb6bb4086e 100644 --- a/crates/engine/tree/src/tree/tests.rs +++ b/crates/engine/tree/src/tree/tests.rs @@ -900,7 +900,7 @@ fn process_payload_attributes_shares_sparse_trie_during_validation_fallback() { withdrawals: None, parent_beacon_block_root: None, slot_number: None, - target_gas_limit: None, + ..Default::default() }, &head, state, @@ -1682,7 +1682,7 @@ async fn test_fcu_with_canonical_ancestor_below_finalized_is_rejected() { withdrawals: None, parent_beacon_block_root: None, slot_number: None, - target_gas_limit: None, + ..Default::default() }; for attrs in [Some(payload_attributes), None] { let err = test_harness @@ -1716,7 +1716,7 @@ async fn test_fcu_with_canonical_ancestor_below_finalized_is_rejected() { withdrawals: None, parent_beacon_block_root: None, slot_number: None, - target_gas_limit: None, + ..Default::default() }; let outcome = test_harness .tree @@ -1759,7 +1759,7 @@ async fn test_fcu_with_canonical_ancestor_above_finalized_starts_payload_build() withdrawals: None, parent_beacon_block_root: None, slot_number: None, - target_gas_limit: None, + ..Default::default() }; let outcome = test_harness .tree diff --git a/crates/ethereum/node/src/engine_ssz_containers.rs b/crates/ethereum/node/src/engine_ssz_containers.rs index 14513d1520f..be042493a0c 100644 --- a/crates/ethereum/node/src/engine_ssz_containers.rs +++ b/crates/ethereum/node/src/engine_ssz_containers.rs @@ -491,7 +491,7 @@ impl From for LegacyPayloadAttributes { withdrawals: None, parent_beacon_block_root: None, slot_number: None, - target_gas_limit: None, + ..Default::default() } } } @@ -521,7 +521,7 @@ impl From for LegacyPayloadAttributes { withdrawals: Some(value.withdrawals), parent_beacon_block_root: None, slot_number: None, - target_gas_limit: None, + ..Default::default() } } } @@ -551,7 +551,7 @@ impl From for LegacyPayloadAttributes { withdrawals: Some(value.withdrawals), parent_beacon_block_root: Some(value.parent_beacon_block_root), slot_number: None, - target_gas_limit: None, + ..Default::default() } } } @@ -576,6 +576,7 @@ impl TryFrom for PayloadAttributesCancun { } impl From for LegacyPayloadAttributes { + #[allow(clippy::needless_update)] fn from(value: PayloadAttributesAmsterdam) -> Self { Self { timestamp: value.timestamp, @@ -585,6 +586,7 @@ impl From for LegacyPayloadAttributes { parent_beacon_block_root: Some(value.parent_beacon_block_root), slot_number: Some(value.slot_number), target_gas_limit: Some(value.target_gas_limit), + ..Default::default() } } } diff --git a/crates/ethereum/node/tests/e2e/eth.rs b/crates/ethereum/node/tests/e2e/eth.rs index 61991332d58..a44e6b27190 100644 --- a/crates/ethereum/node/tests/e2e/eth.rs +++ b/crates/ethereum/node/tests/e2e/eth.rs @@ -244,7 +244,7 @@ async fn test_testing_build_block_v1_osaka() -> eyre::Result<()> { withdrawals: Some(vec![]), parent_beacon_block_root: Some(B256::ZERO), slot_number: None, - target_gas_limit: None, + ..Default::default() }; let request = TestingBuildBlockRequestV1 { @@ -333,7 +333,7 @@ async fn test_engine_ssz_proxy_can_mine_block() -> eyre::Result<()> { withdrawals: Some(vec![]), parent_beacon_block_root: Some(B256::ZERO), slot_number: None, - target_gas_limit: None, + ..Default::default() }; let envelope = node diff --git a/crates/ethereum/node/tests/e2e/utils.rs b/crates/ethereum/node/tests/e2e/utils.rs index 9cef9c0a27d..863731ba36c 100644 --- a/crates/ethereum/node/tests/e2e/utils.rs +++ b/crates/ethereum/node/tests/e2e/utils.rs @@ -17,7 +17,7 @@ use reth_node_ethereum::EthereumNode; use reth_provider::FullProvider; /// Helper function to create a new eth payload attributes -pub(crate) const fn eth_payload_attributes(timestamp: u64) -> PayloadAttributes { +pub(crate) fn eth_payload_attributes(timestamp: u64) -> PayloadAttributes { PayloadAttributes { timestamp, prev_randao: B256::ZERO, @@ -25,13 +25,13 @@ pub(crate) const fn eth_payload_attributes(timestamp: u64) -> PayloadAttributes withdrawals: Some(vec![]), parent_beacon_block_root: Some(B256::ZERO), slot_number: None, - target_gas_limit: None, + ..Default::default() } } /// Helper function to create pre-Cancun (Shanghai) payload attributes. /// No `parent_beacon_block_root` field. -pub(crate) const fn eth_payload_attributes_shanghai(timestamp: u64) -> PayloadAttributes { +pub(crate) fn eth_payload_attributes_shanghai(timestamp: u64) -> PayloadAttributes { PayloadAttributes { timestamp, prev_randao: B256::ZERO, @@ -39,7 +39,7 @@ pub(crate) const fn eth_payload_attributes_shanghai(timestamp: u64) -> PayloadAt withdrawals: Some(vec![]), parent_beacon_block_root: None, slot_number: None, - target_gas_limit: None, + ..Default::default() } } @@ -49,7 +49,7 @@ pub(crate) const fn eth_payload_attributes_shanghai(timestamp: u64) -> PayloadAt /// `slot_number` in the attributes once Amsterdam is active. Tests use the timestamp as a /// deterministic dummy slot because the exact beacon slot is irrelevant for these local e2e /// payloads. -pub(crate) const fn eth_payload_attributes_amsterdam(timestamp: u64) -> PayloadAttributes { +pub(crate) fn eth_payload_attributes_amsterdam(timestamp: u64) -> PayloadAttributes { PayloadAttributes { timestamp, prev_randao: B256::ZERO, @@ -57,7 +57,7 @@ pub(crate) const fn eth_payload_attributes_amsterdam(timestamp: u64) -> PayloadA withdrawals: Some(vec![]), parent_beacon_block_root: Some(B256::ZERO), slot_number: Some(timestamp), - target_gas_limit: None, + ..Default::default() } } diff --git a/crates/ethereum/node/tests/it/testing.rs b/crates/ethereum/node/tests/it/testing.rs index df483211b9a..c86080ab403 100644 --- a/crates/ethereum/node/tests/it/testing.rs +++ b/crates/ethereum/node/tests/it/testing.rs @@ -60,7 +60,7 @@ async fn testing_rpc_build_block_works() -> eyre::Result<()> { withdrawals: None, parent_beacon_block_root: None, slot_number: None, - target_gas_limit: None, + ..Default::default() }; let request = TestingBuildBlockRequestV1 { @@ -134,6 +134,7 @@ async fn testing_rpc_commit_block_works() -> eyre::Result<()> { let chain = ctx.config().chain.clone(); let timestamp = chain.genesis().timestamp + 1; let target_gas_limit = chain.genesis().gas_limit - 100; + #[allow(clippy::needless_update)] let payload_attributes = EthPayloadAttributes { timestamp, prev_randao: B256::ZERO, @@ -142,6 +143,7 @@ async fn testing_rpc_commit_block_works() -> eyre::Result<()> { parent_beacon_block_root: Some(B256::ZERO), slot_number: Some(timestamp), target_gas_limit: Some(target_gas_limit), + ..Default::default() }; tokio::spawn(async move { diff --git a/crates/payload/builder/src/service.rs b/crates/payload/builder/src/service.rs index 4d6e4a62f70..b9e2abac366 100644 --- a/crates/payload/builder/src/service.rs +++ b/crates/payload/builder/src/service.rs @@ -693,7 +693,7 @@ mod tests { withdrawals: None, parent_beacon_block_root: None, slot_number: None, - target_gas_limit: None, + ..Default::default() }, parent_hash: B256::ZERO, resources: PayloadBuilderResources::default().with_lease(lease), diff --git a/crates/payload/primitives/src/traits.rs b/crates/payload/primitives/src/traits.rs index 4d46e6eb0fa..068cd0db55c 100644 --- a/crates/payload/primitives/src/traits.rs +++ b/crates/payload/primitives/src/traits.rs @@ -253,7 +253,7 @@ mod tests { withdrawals: None, parent_beacon_block_root: None, slot_number: None, - target_gas_limit: None, + ..Default::default() }; // Verify that the generated payload ID matches the expected value @@ -292,7 +292,7 @@ mod tests { ]), parent_beacon_block_root: None, slot_number: None, - target_gas_limit: None, + ..Default::default() }; // Verify that the generated payload ID matches the expected value @@ -326,7 +326,7 @@ mod tests { .unwrap(), ), slot_number: None, - target_gas_limit: None, + ..Default::default() }; // Verify that the generated payload ID matches the expected value @@ -351,7 +351,7 @@ mod tests { withdrawals: Some(vec![]), parent_beacon_block_root: Some(B256::from_slice(&[2; 32])), slot_number: Some(1), - target_gas_limit: None, + ..Default::default() }; let first = payload_id(&parent, &attributes); @@ -365,6 +365,7 @@ mod tests { let parent = B256::from_str("0x9876543210abcdef9876543210abcdef9876543210abcdef9876543210abcdef") .unwrap(); + #[allow(clippy::needless_update)] let mut attributes = EthPayloadAttributes { timestamp: 1622553200, prev_randao: B256::from_slice(&[1; 32]), @@ -376,6 +377,7 @@ mod tests { parent_beacon_block_root: Some(B256::from_slice(&[2; 32])), slot_number: Some(1), target_gas_limit: Some(30_000_000), + ..Default::default() }; let first = payload_id(&parent, &attributes); diff --git a/crates/rpc/rpc-engine-api/src/engine_api.rs b/crates/rpc/rpc-engine-api/src/engine_api.rs index 109bd5ccfdb..ad12529f0bb 100644 --- a/crates/rpc/rpc-engine-api/src/engine_api.rs +++ b/crates/rpc/rpc-engine-api/src/engine_api.rs @@ -2141,7 +2141,7 @@ mod tests { withdrawals: Some(vec![]), parent_beacon_block_root: None, slot_number: None, - target_gas_limit: None, + ..Default::default() }; let custody_columns = B128::from(0b1010u128.to_le_bytes()); let expected_custody_columns = B128::from(0b1010u128); @@ -2198,7 +2198,7 @@ mod tests { // Invalid for V3/Cancun, but should be ignored if forkchoice is SYNCING. parent_beacon_block_root: None, slot_number: None, - target_gas_limit: None, + ..Default::default() }; let api_task = tokio::spawn(async move { @@ -2247,7 +2247,7 @@ mod tests { withdrawals: Some(vec![]), parent_beacon_block_root: None, slot_number: None, - target_gas_limit: None, + ..Default::default() }; let api_task = tokio::spawn(async move { From 6f8b3699d707ecbd4d3fdfee42ea308545612d89 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Thu, 13 Aug 2026 16:07:14 +0200 Subject: [PATCH 018/294] chore(deps): bump alloy-hardforks to 0.4.8 (#26685) --- Cargo.lock | 12 ++++++------ Cargo.toml | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b45bab0635e..dadcfd17d95 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -299,7 +299,7 @@ checksum = "6b307a52f91f7f3aad56b5cb4f54e5eb17dd0b3f4b2c82823301499fe1c6fdaf" dependencies = [ "alloy-consensus", "alloy-eips", - "alloy-hardforks 0.4.7", + "alloy-hardforks 0.4.8", "alloy-primitives", "alloy-rpc-types-engine", "alloy-rpc-types-eth", @@ -341,9 +341,9 @@ dependencies = [ [[package]] name = "alloy-hardforks" -version = "0.4.7" +version = "0.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83ba208044232d14d4adbfa77e57d6329f51bc1acc21f5667bb7db72d88a0831" +checksum = "865371fb677c005f7cb0ea9c2847a1ba4fdb042caf6428b1769fee20368bbfc7" dependencies = [ "alloy-chains", "alloy-eip2124", @@ -7735,7 +7735,7 @@ dependencies = [ "alloy-consensus", "alloy-eips", "alloy-evm", - "alloy-hardforks 0.4.7", + "alloy-hardforks 0.4.8", "alloy-primitives", "alloy-rlp", "alloy-rpc-types", @@ -8648,7 +8648,7 @@ dependencies = [ "alloy-eip7928", "alloy-eips", "alloy-genesis", - "alloy-hardforks 0.4.7", + "alloy-hardforks 0.4.8", "alloy-primitives", "alloy-rlp", "alloy-trie", @@ -8767,7 +8767,7 @@ name = "reth-ethereum-forks" version = "2.5.0" dependencies = [ "alloy-eip2124", - "alloy-hardforks 0.4.7", + "alloy-hardforks 0.4.8", "alloy-primitives", "arbitrary", "auto_impl", diff --git a/Cargo.toml b/Cargo.toml index 936ed1c0c1e..65ffdcbd8d0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -447,7 +447,7 @@ alloy-evm = { version = "0.38.0", default-features = false } alloy-rlp = { version = "0.3.16", default-features = false, features = ["core-net"] } alloy-trie = { version = "0.9.4", default-features = false } -alloy-hardforks = "0.4.7" +alloy-hardforks = "0.4.8" alloy-consensus = { version = "2.4.0", default-features = false } alloy-contract = { version = "2.4.0", default-features = false } From 5e2cf1c8af043b0355d82dd97231fb386e20736b Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Thu, 13 Aug 2026 16:21:32 +0200 Subject: [PATCH 019/294] chore(deps): bump alloy to 2.4.1 (#26687) --- Cargo.lock | 124 ++++++++++++++++++++++++++--------------------------- Cargo.toml | 56 ++++++++++++------------ 2 files changed, 90 insertions(+), 90 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index dadcfd17d95..9cab6e4c18c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -121,9 +121,9 @@ dependencies = [ [[package]] name = "alloy-consensus" -version = "2.4.0" +version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "99bdf6a3e99089d97e137009956928b41d645035b220cb303eae88182882e9ca" +checksum = "12a542fe1e6a48cfd010a9e8eba1235436de58ee0f1415e588c27d5d556e98df" dependencies = [ "alloy-eips", "alloy-primitives", @@ -149,9 +149,9 @@ dependencies = [ [[package]] name = "alloy-consensus-any" -version = "2.4.0" +version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "18d98872f189ea87063912de45e7e0d6ae44700a8836f8bfa79b3265f0c38bfe" +checksum = "05ac1559726a5e71d6dbab59cbd8ad0988c762105d2ae41ae4f3241a3166262d" dependencies = [ "alloy-consensus", "alloy-eips", @@ -164,9 +164,9 @@ dependencies = [ [[package]] name = "alloy-contract" -version = "2.4.0" +version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f69e65d1b6adc7e46c0e93c2cf3cb4e8646e66a2ced4ba38d542b724acd54774" +checksum = "b86722084d6d07822a6c67e39ae794d437fb9de12fa4127d4f9fd93f89651121" dependencies = [ "alloy-consensus", "alloy-dyn-abi", @@ -267,9 +267,9 @@ dependencies = [ [[package]] name = "alloy-eips" -version = "2.4.0" +version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74bffa6f6440f1ee3c8cd7bc9dc2b01fd837641ef9bd566bd9f8922e75ff8e6c" +checksum = "d40246adf7468b430fce87945ece8a54ef27fa760dd9f670f5162a412007bdb2" dependencies = [ "alloy-eip2124", "alloy-eip2930", @@ -313,9 +313,9 @@ dependencies = [ [[package]] name = "alloy-genesis" -version = "2.4.0" +version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f48592ff562892af6a2ebb6e214ed1d2795a6c1dc601c601ef0863583c799003" +checksum = "4951473e6ff25cdf82eb87b6a1a5fbb4af9c97528398a4a62fbf516198ec5028" dependencies = [ "alloy-eips", "alloy-primitives", @@ -367,9 +367,9 @@ dependencies = [ [[package]] name = "alloy-json-rpc" -version = "2.4.0" +version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31de46f2b1948ac9e378212fd79cb963745cbb29cfe08a13c5b675d7e0934a24" +checksum = "7e5722ea29a9a85ecdba2516c58f50d71e77cd25f8ce7e9f4ee8570ddeee6df1" dependencies = [ "alloy-primitives", "alloy-sol-types", @@ -382,9 +382,9 @@ dependencies = [ [[package]] name = "alloy-network" -version = "2.4.0" +version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa3ea75a876a4cd302c15dad628cc55f1f0b2db7fd4865f57f4e24cea2629298" +checksum = "e2e4d278913f00ab611f5367448a3a800fed173ab791c3cc67b7a1b0d7afd305" dependencies = [ "alloy-consensus", "alloy-consensus-any", @@ -408,9 +408,9 @@ dependencies = [ [[package]] name = "alloy-network-primitives" -version = "2.4.0" +version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f93f09f2bee068dcd80bfdf057109d64e37b318f21d3d64851139ca741366dd8" +checksum = "e4e52ae56d1acaa6b46e9d7015b8ae6c10f3d1c213a32ae70b17426a03ae6729" dependencies = [ "alloy-consensus", "alloy-eips", @@ -421,9 +421,9 @@ dependencies = [ [[package]] name = "alloy-node-bindings" -version = "2.4.0" +version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "623d5c6b07f1adf11940eff4b8e665d9b7d9db969fbfe872816878906b267152" +checksum = "cdbb4a1bc03b411a3880d410ccc48d866b72d753ab1080605631890955b9caa2" dependencies = [ "alloy-genesis", "alloy-hardforks 0.2.13", @@ -475,9 +475,9 @@ dependencies = [ [[package]] name = "alloy-provider" -version = "2.4.0" +version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e9070296bd2902aac4d9defd85608116eceb319526746257ab97096da2c5603" +checksum = "386124639a5c386329b9f2ef2b42969badbf7bba40422b513e3e5739c516dc34" dependencies = [ "alloy-chains", "alloy-consensus", @@ -519,9 +519,9 @@ dependencies = [ [[package]] name = "alloy-pubsub" -version = "2.4.0" +version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6f4d232e145165d28fb3e614b934a7af0b0b7b4c67f9c96afc96552114e7de4" +checksum = "ca2c32b323d8001b6e883444b7433804be0e6d55e4dbfd64e3bd2ba025282b77" dependencies = [ "alloy-json-rpc", "alloy-primitives", @@ -563,9 +563,9 @@ dependencies = [ [[package]] name = "alloy-rpc-client" -version = "2.4.0" +version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87da7170c18c43a5def51f5f166d0260776b721e184c349fae1cd5466cb67362" +checksum = "3dfd3ebc1756426a6a9e17979d4467632aff01f359136c50e5320ac6e32d6b8d" dependencies = [ "alloy-json-rpc", "alloy-primitives", @@ -589,9 +589,9 @@ dependencies = [ [[package]] name = "alloy-rpc-types" -version = "2.4.0" +version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88813dc63f261e25d158f07063a71e926f36b3ee8076c1e6d8396348452c5f69" +checksum = "417dc664965e36fc43a4f9c8eb8b2d14067a29c430e23470d2c923c1c977bac7" dependencies = [ "alloy-primitives", "alloy-rpc-types-engine", @@ -602,9 +602,9 @@ dependencies = [ [[package]] name = "alloy-rpc-types-admin" -version = "2.4.0" +version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1612dde5f2f6bdc16d3e8b68f4b6c90a2e6db3556c416224fee529871a0fdabc" +checksum = "8bfdc2a2cde178ad3a79594b4fabbc0f2f11be5a98a3363083e91575fc95335c" dependencies = [ "alloy-genesis", "alloy-primitives", @@ -614,9 +614,9 @@ dependencies = [ [[package]] name = "alloy-rpc-types-anvil" -version = "2.4.0" +version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2771c42b80531e99e4449e0a3ffd8f9830073f1d38576ac1643d2573fe6d922f" +checksum = "5a07bb602ee07767e393b5862f719a1d061deca15e617fbb1f7af49cd49576e4" dependencies = [ "alloy-primitives", "alloy-rpc-types-eth", @@ -626,9 +626,9 @@ dependencies = [ [[package]] name = "alloy-rpc-types-any" -version = "2.4.0" +version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6574846e24e090607d78325e90256c9a5ce4010cf0e663b23d06e01fd74356f0" +checksum = "cd0a6cce3f97e5d9c0048081b5d173e0e406c2056c52c012e578a1d451ff0cdc" dependencies = [ "alloy-consensus-any", "alloy-network-primitives", @@ -641,9 +641,9 @@ dependencies = [ [[package]] name = "alloy-rpc-types-beacon" -version = "2.4.0" +version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2f2e2476007e47ae60bc0f048a63963004adcd6fdc3737a200e6115b534f084" +checksum = "905e705ef91d2ed6cf08afbec0f9d79318175785eb146656441e8c644786ee5f" dependencies = [ "alloy-eips", "alloy-primitives", @@ -661,9 +661,9 @@ dependencies = [ [[package]] name = "alloy-rpc-types-debug" -version = "2.4.0" +version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6083d2989d456e321721ea9b392c48edb949abbe0a8d449feaf6fa74addcd647" +checksum = "32f6d7c9462a3cf3bb046d0b870b86f1ce8c6892157491abebba62913e6f53cb" dependencies = [ "alloy-primitives", "alloy-rlp", @@ -674,9 +674,9 @@ dependencies = [ [[package]] name = "alloy-rpc-types-engine" -version = "2.4.0" +version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "436d9fe64a92e47ff668d766e75bd7eaafabafb68abaab235848bd7a3c10a8e0" +checksum = "b6e23b5864b3ed3479285ae9af1924266d60cac839dc559d50da868a0734c1d3" dependencies = [ "alloy-consensus", "alloy-eips", @@ -695,9 +695,9 @@ dependencies = [ [[package]] name = "alloy-rpc-types-eth" -version = "2.4.0" +version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "acc71ae98333ab278782d59423bd8317e7658954c356e1f4ce559fcea4f4da85" +checksum = "1eb4c87dfdde83cd97e03a8c0c5e986b52e14075e02266e322ac6c7eb02e7399" dependencies = [ "alloy-consensus", "alloy-consensus-any", @@ -717,9 +717,9 @@ dependencies = [ [[package]] name = "alloy-rpc-types-mev" -version = "2.4.0" +version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4db8cb3571bcd2ef13f79e4094be9f786b57608be4e915baddb061ffe39b057" +checksum = "e97bfdac1ec57f53ab2a0dacfedf382aeba1245ecb4ad7ecd8aeb4438a6c3b9b" dependencies = [ "alloy-consensus", "alloy-eips", @@ -732,9 +732,9 @@ dependencies = [ [[package]] name = "alloy-rpc-types-trace" -version = "2.4.0" +version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eca9bf40e0b129268f874feff8cb73e5de8fc9a371a6b34635afbb18f437bc7d" +checksum = "7d45ae3eadb97bf7933086205b4be0fef82c208bfc3007759e9afdcb6efa74c5" dependencies = [ "alloy-primitives", "alloy-rpc-types-eth", @@ -746,9 +746,9 @@ dependencies = [ [[package]] name = "alloy-rpc-types-txpool" -version = "2.4.0" +version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6045fcec2d380218f5847569b1f9388bdd3bf1edb31ebf14a94b364a81247428" +checksum = "ce0d7a9ab748d011daac12902ab9aedf8232d9b070bec6344cadd8509d8a6798" dependencies = [ "alloy-primitives", "alloy-rpc-types-eth", @@ -758,9 +758,9 @@ dependencies = [ [[package]] name = "alloy-serde" -version = "2.4.0" +version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f11544d72bc87eab3cf4addc61d1f735bf88f3769482eca3deefc0c2558d2d0" +checksum = "2ff89f75b8a9d00f659750e9220c31b42d67c69ffc43b2ac8911dfd0506eb4a6" dependencies = [ "alloy-primitives", "arbitrary", @@ -770,9 +770,9 @@ dependencies = [ [[package]] name = "alloy-signer" -version = "2.4.0" +version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e76dfc1a6c8337e59624b9f6f02245566b7d24e8dab0eb842d57d4fd8777adc3" +checksum = "69e1f6fae3ade809daa4c32bee35a30a439bb307209d7b42f3995988b5cc74b8" dependencies = [ "alloy-primitives", "async-trait", @@ -785,9 +785,9 @@ dependencies = [ [[package]] name = "alloy-signer-local" -version = "2.4.0" +version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a9f88d471c719ce2bdf949191f8a1c08bffd0c030606c6a8da0d414454e8bfe" +checksum = "621cc39c34b85875858ef149598f3d5f9797dff9a3247f3606fcc06f9b20acc8" dependencies = [ "alloy-consensus", "alloy-network", @@ -874,9 +874,9 @@ dependencies = [ [[package]] name = "alloy-transport" -version = "2.4.0" +version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9399b22a1d2a82c8c30fd527541ba61e574795d0b95f3b902d3473d11489be5" +checksum = "06f4ca8484d6295d5165f67de8fdb00ae630ba585efb606f003171286b56441c" dependencies = [ "alloy-json-rpc", "auto_impl", @@ -897,9 +897,9 @@ dependencies = [ [[package]] name = "alloy-transport-http" -version = "2.4.0" +version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59b55d7ccf45bec99db692c1266413498d2c9776c390cf148aa590422e31b2d3" +checksum = "8342aa5a7f8dee6d606307eebbc847d9799a823c7e4f37b9c1e32b08715f73c8" dependencies = [ "alloy-json-rpc", "alloy-transport", @@ -913,9 +913,9 @@ dependencies = [ [[package]] name = "alloy-transport-ipc" -version = "2.4.0" +version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5843087378580a14098ada3e9184a92f3706828857ad483558ef2fa1080b2dfb" +checksum = "184e03e1845edd5534f309d3169d93969a819ad1609108d9378fd33059de7547" dependencies = [ "alloy-json-rpc", "alloy-pubsub", @@ -933,9 +933,9 @@ dependencies = [ [[package]] name = "alloy-transport-ws" -version = "2.4.0" +version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5bf93caeb6d84d20060acba53479c9cb55f5364a3d49ab876ad00ef074ce794d" +checksum = "ee146fb5859e612e95570642a29def87c060c7ce47d14a0caa5ab7ee8570ec2c" dependencies = [ "alloy-pubsub", "alloy-transport", @@ -972,9 +972,9 @@ dependencies = [ [[package]] name = "alloy-tx-macros" -version = "2.4.0" +version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de9792a54c0c8cc009883436ff1a60eb091095aced1c167a764bedbbfa0eee79" +checksum = "1d0626c4f3b2028f7e8db32f53c22d9ecd5939f772317b7c4b6fe5219cb0589a" dependencies = [ "darling 0.23.0", "proc-macro2", diff --git a/Cargo.toml b/Cargo.toml index 65ffdcbd8d0..f81b7c0d7a3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -449,34 +449,34 @@ alloy-trie = { version = "0.9.4", default-features = false } alloy-hardforks = "0.4.8" -alloy-consensus = { version = "2.4.0", default-features = false } -alloy-contract = { version = "2.4.0", default-features = false } -alloy-eips = { version = "2.4.0", default-features = false } -alloy-genesis = { version = "2.4.0", default-features = false } -alloy-json-rpc = { version = "2.4.0", default-features = false } -alloy-network = { version = "2.4.0", default-features = false } -alloy-network-primitives = { version = "2.4.0", default-features = false } -alloy-node-bindings = "2.4.0" -alloy-provider = { version = "2.4.0", features = ["reqwest", "debug-api"], default-features = false } -alloy-pubsub = { version = "2.4.0", default-features = false } -alloy-rpc-client = { version = "2.4.0", default-features = false } -alloy-rpc-types = { version = "2.4.0", features = ["eth"], default-features = false } -alloy-rpc-types-admin = { version = "2.4.0", default-features = false } -alloy-rpc-types-anvil = { version = "2.4.0", default-features = false } -alloy-rpc-types-beacon = { version = "2.4.0", default-features = false } -alloy-rpc-types-debug = { version = "2.4.0", default-features = false } -alloy-rpc-types-engine = { version = "2.4.0", default-features = false } -alloy-rpc-types-eth = { version = "2.4.0", default-features = false } -alloy-rpc-types-mev = { version = "2.4.0", default-features = false } -alloy-rpc-types-trace = { version = "2.4.0", default-features = false } -alloy-rpc-types-txpool = { version = "2.4.0", default-features = false } -alloy-serde = { version = "2.4.0", default-features = false } -alloy-signer = { version = "2.4.0", default-features = false } -alloy-signer-local = { version = "2.4.0", default-features = false } -alloy-transport = { version = "2.4.0" } -alloy-transport-http = { version = "2.4.0", features = ["reqwest-rustls-tls"], default-features = false } -alloy-transport-ipc = { version = "2.4.0", default-features = false } -alloy-transport-ws = { version = "2.4.0", default-features = false } +alloy-consensus = { version = "2.4.1", default-features = false } +alloy-contract = { version = "2.4.1", default-features = false } +alloy-eips = { version = "2.4.1", default-features = false } +alloy-genesis = { version = "2.4.1", default-features = false } +alloy-json-rpc = { version = "2.4.1", default-features = false } +alloy-network = { version = "2.4.1", default-features = false } +alloy-network-primitives = { version = "2.4.1", default-features = false } +alloy-node-bindings = "2.4.1" +alloy-provider = { version = "2.4.1", features = ["reqwest", "debug-api"], default-features = false } +alloy-pubsub = { version = "2.4.1", default-features = false } +alloy-rpc-client = { version = "2.4.1", default-features = false } +alloy-rpc-types = { version = "2.4.1", features = ["eth"], default-features = false } +alloy-rpc-types-admin = { version = "2.4.1", default-features = false } +alloy-rpc-types-anvil = { version = "2.4.1", default-features = false } +alloy-rpc-types-beacon = { version = "2.4.1", default-features = false } +alloy-rpc-types-debug = { version = "2.4.1", default-features = false } +alloy-rpc-types-engine = { version = "2.4.1", default-features = false } +alloy-rpc-types-eth = { version = "2.4.1", default-features = false } +alloy-rpc-types-mev = { version = "2.4.1", default-features = false } +alloy-rpc-types-trace = { version = "2.4.1", default-features = false } +alloy-rpc-types-txpool = { version = "2.4.1", default-features = false } +alloy-serde = { version = "2.4.1", default-features = false } +alloy-signer = { version = "2.4.1", default-features = false } +alloy-signer-local = { version = "2.4.1", default-features = false } +alloy-transport = { version = "2.4.1" } +alloy-transport-http = { version = "2.4.1", features = ["reqwest-rustls-tls"], default-features = false } +alloy-transport-ipc = { version = "2.4.1", default-features = false } +alloy-transport-ws = { version = "2.4.1", default-features = false } # misc either = { version = "1.16.0", default-features = false } From 4895ee4cd24c8a7b063dd24c65b11a86ed43abe5 Mon Sep 17 00:00:00 2001 From: Soubhik Singha Mahapatra <160333583+Soubhik-10@users.noreply.github.com> Date: Thu, 13 Aug 2026 14:28:03 +0000 Subject: [PATCH 020/294] feat(chainspec): add Bogota hardfork support (#26686) Co-authored-by: Matthias Seitz --- crates/chainspec/src/spec.rs | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/crates/chainspec/src/spec.rs b/crates/chainspec/src/spec.rs index df67be280cd..e72e0dcc216 100644 --- a/crates/chainspec/src/spec.rs +++ b/crates/chainspec/src/spec.rs @@ -312,6 +312,7 @@ pub fn create_chain_config( prague_time: timestamp(EthereumHardfork::Prague), osaka_time: timestamp(EthereumHardfork::Osaka), amsterdam_time: timestamp(EthereumHardfork::Amsterdam), + bogota_time: timestamp(EthereumHardfork::Bogota), bpo1_time: timestamp(EthereumHardfork::Bpo1), bpo2_time: timestamp(EthereumHardfork::Bpo2), bpo3_time: timestamp(EthereumHardfork::Bpo3), @@ -896,6 +897,7 @@ impl From for ChainSpec { (EthereumHardfork::Bpo4.boxed(), genesis.config.bpo4_time), (EthereumHardfork::Bpo5.boxed(), genesis.config.bpo5_time), (EthereumHardfork::Amsterdam.boxed(), genesis.config.amsterdam_time), + (EthereumHardfork::Bogota.boxed(), genesis.config.bogota_time), ]; let mut time_hardforks = time_hardfork_opts @@ -1215,6 +1217,19 @@ impl ChainSpecBuilder { self } + /// Enable Bogota at genesis. + pub fn bogota_activated(mut self) -> Self { + self = self.amsterdam_activated(); + self.hardforks.insert(EthereumHardfork::Bogota, ForkCondition::Timestamp(0)); + self + } + + /// Enable Bogota at the given timestamp. + pub fn with_bogota_at(mut self, timestamp: u64) -> Self { + self.hardforks.insert(EthereumHardfork::Bogota, ForkCondition::Timestamp(timestamp)); + self + } + /// Build the resulting [`ChainSpec`]. /// /// # Panics From 3641f10830f44f71c2469fdf7e818ccd4a4b0e76 Mon Sep 17 00:00:00 2001 From: spencer Date: Thu, 13 Aug 2026 17:11:43 +0200 Subject: [PATCH 021/294] fix(chainspec): honor genesis slotNumber instead of hardcoding 0 (EIP-7843) (#26680) Co-authored-by: Matthias Seitz --- crates/chainspec/src/spec.rs | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/crates/chainspec/src/spec.rs b/crates/chainspec/src/spec.rs index e72e0dcc216..ac0a247c37b 100644 --- a/crates/chainspec/src/spec.rs +++ b/crates/chainspec/src/spec.rs @@ -82,11 +82,11 @@ pub fn make_genesis_header(genesis: &Genesis, hardforks: &ChainHardforks) -> Hea .active_at_timestamp(genesis.timestamp) .then_some(EMPTY_BLOCK_ACCESS_LIST_HASH); - // If Amsterdam is activated at genesis we set slot number to 0 + // If Amsterdam is activated at genesis we set slot number to the provided genesis or 0 let slot_number = hardforks .fork(EthereumHardfork::Amsterdam) .active_at_timestamp(genesis.timestamp) - .then_some(0); + .then_some(genesis.slot_number.unwrap_or(0)); Header { number: genesis.number.unwrap_or_default(), @@ -2671,6 +2671,28 @@ Post-merge hard forks (timestamp based): assert_eq!(ForkHash::from(genesis_hash), expected_forkhash); } + #[test] + fn test_amsterdam_genesis_slot_number() { + // a genesis-provided slot number is used as-is + let genesis = + Genesis { gas_limit: 0x2fefd8u64, ..Default::default() }.with_slot_number(Some(999)); + let chainspec = ChainSpecBuilder::default() + .chain(Chain::from_id(1337)) + .genesis(genesis) + .amsterdam_activated() + .build(); + assert_eq!(chainspec.genesis_header().slot_number, Some(999)); + + // an omitted slot number defaults to 0 + let genesis = Genesis { gas_limit: 0x2fefd8u64, ..Default::default() }; + let chainspec = ChainSpecBuilder::default() + .chain(Chain::from_id(1337)) + .genesis(genesis) + .amsterdam_activated() + .build(); + assert_eq!(chainspec.genesis_header().slot_number, Some(0)); + } + #[test] fn holesky_paris_activated_at_genesis() { assert!(HOLESKY From 4cf0facecda7b4d474c739acef1c0fc2c69a122c Mon Sep 17 00:00:00 2001 From: Soubhik Singha Mahapatra <160333583+Soubhik-10@users.noreply.github.com> Date: Thu, 13 Aug 2026 16:11:50 +0000 Subject: [PATCH 022/294] feat(rpc): add Bogota Engine API stubs (#26691) Co-authored-by: Matthias Seitz --- crates/rpc/rpc-api/src/engine.rs | 26 ++++++- crates/rpc/rpc-engine-api/src/capabilities.rs | 11 ++- crates/rpc/rpc-engine-api/src/engine_api.rs | 71 ++++++++++++++++++- 3 files changed, 103 insertions(+), 5 deletions(-) diff --git a/crates/rpc/rpc-api/src/engine.rs b/crates/rpc/rpc-api/src/engine.rs index 582944132b6..0faadc412b7 100644 --- a/crates/rpc/rpc-api/src/engine.rs +++ b/crates/rpc/rpc-api/src/engine.rs @@ -13,7 +13,7 @@ use alloy_primitives::{Address, BlockHash, Bytes, B128, B256, U256, U64}; use alloy_rpc_types_engine::{ ClientVersionV1, ExecutionPayloadBodiesV1, ExecutionPayloadBodiesV2, ExecutionPayloadInputV2, ExecutionPayloadV1, ExecutionPayloadV3, ExecutionPayloadV4, ForkchoiceState, ForkchoiceUpdated, - PayloadId, PayloadStatus, + ForkchoiceUpdatedResponseV2, PayloadId, PayloadStatus, PayloadStatusV2, }; use alloy_rpc_types_eth::{ state::StateOverride, BlockOverrides, EIP1186AccountProofResponse, Filter, SyncStatus, @@ -87,6 +87,19 @@ pub trait EngineApi { execution_requests: RequestsOrHash, ) -> RpcResult; + /// Post Bogota payload handler stub. + /// + /// See also + #[method(name = "newPayloadV6")] + async fn new_payload_v6( + &self, + payload: ExecutionPayloadV4, + versioned_hashes: Vec, + parent_beacon_block_root: B256, + execution_requests: RequestsOrHash, + inclusion_list_transactions: Vec, + ) -> RpcResult; + /// See also /// /// Caution: This should not accept the `withdrawals` field in the payload attributes. @@ -151,6 +164,17 @@ pub trait EngineApi { custody_columns: Option, ) -> RpcResult; + /// Post Bogota forkchoice update stub. + /// + /// See also + #[method(name = "forkchoiceUpdatedV5")] + async fn fork_choice_updated_v5( + &self, + fork_choice_state: ForkchoiceState, + payload_attributes: Option, + custody_columns: Option, + ) -> RpcResult; + /// See also /// /// Returns the most recent version of the payload that is available in the corresponding diff --git a/crates/rpc/rpc-engine-api/src/capabilities.rs b/crates/rpc/rpc-engine-api/src/capabilities.rs index 5e0e0af043d..c69227c5051 100644 --- a/crates/rpc/rpc-engine-api/src/capabilities.rs +++ b/crates/rpc/rpc-engine-api/src/capabilities.rs @@ -11,8 +11,13 @@ const CRITICAL_METHOD_PREFIXES: &[&str] = &["engine_forkchoiceUpdated", "engine_getPayload", "engine_newPayload"]; /// Engine API methods for upcoming hardforks that should not warn while they are still unstable. -const UNSTABLE_METHODS: &[&str] = - &["engine_forkchoiceUpdatedV4", "engine_getPayloadV6", "engine_newPayloadV5"]; +const UNSTABLE_METHODS: &[&str] = &[ + "engine_forkchoiceUpdatedV4", + "engine_forkchoiceUpdatedV5", + "engine_getPayloadV6", + "engine_newPayloadV5", + "engine_newPayloadV6", +]; /// All Engine API capabilities supported by Reth (Ethereum mainnet). /// @@ -22,6 +27,7 @@ pub const CAPABILITIES: &[&str] = &[ "engine_forkchoiceUpdatedV2", "engine_forkchoiceUpdatedV3", "engine_forkchoiceUpdatedV4", + "engine_forkchoiceUpdatedV5", "engine_getClientVersionV1", "engine_getPayloadV1", "engine_getPayloadV2", @@ -35,6 +41,7 @@ pub const CAPABILITIES: &[&str] = &[ "engine_newPayloadV3", "engine_newPayloadV4", "engine_newPayloadV5", + "engine_newPayloadV6", "engine_getPayloadBodiesByHashV1", "engine_getPayloadBodiesByHashV2", "engine_getPayloadBodiesByRangeV1", diff --git a/crates/rpc/rpc-engine-api/src/engine_api.rs b/crates/rpc/rpc-engine-api/src/engine_api.rs index ad12529f0bb..2635ad3051c 100644 --- a/crates/rpc/rpc-engine-api/src/engine_api.rs +++ b/crates/rpc/rpc-engine-api/src/engine_api.rs @@ -12,8 +12,8 @@ use alloy_rpc_types_engine::{ CancunPayloadFields, ClientVersionV1, ExecutionData, ExecutionPayloadBodiesV1, ExecutionPayloadBodiesV2, ExecutionPayloadBodyV1, ExecutionPayloadBodyV2, ExecutionPayloadInputV2, ExecutionPayloadSidecar, ExecutionPayloadV1, ExecutionPayloadV3, - ExecutionPayloadV4, ForkchoiceState, ForkchoiceUpdated, PayloadId, PayloadStatus, - PraguePayloadFields, + ExecutionPayloadV4, ForkchoiceState, ForkchoiceUpdated, ForkchoiceUpdatedResponseV2, PayloadId, + PayloadStatus, PayloadStatusV2, PraguePayloadFields, }; use async_trait::async_trait; use jsonrpsee_core::{server::RpcModule, RpcResult}; @@ -292,6 +292,19 @@ where Ok(res?) } + /// Handler stub for `engine_newPayloadV6`. + /// + /// See also + pub async fn new_payload_v6( + &self, + payload: PayloadT::ExecutionData, + ) -> EngineApiResult { + let _ = payload; + Err(EngineApiError::EngineObjectValidationError( + reth_payload_primitives::EngineObjectValidationError::UnsupportedFork, + )) + } + /// Returns whether the engine accepts execution requests hash. pub fn accept_execution_requests_hash(&self) -> bool { self.inner.accept_execution_requests_hash @@ -414,6 +427,21 @@ where res } + /// Handler stub for `engine_forkchoiceUpdatedV5`. + /// + /// See also + pub async fn fork_choice_updated_v5( + &self, + state: ForkchoiceState, + payload_attrs: Option, + custody_columns: Option, + ) -> EngineApiResult { + let _ = (state, payload_attrs, custody_columns); + Err(EngineApiError::EngineObjectValidationError( + reth_payload_primitives::EngineObjectValidationError::UnsupportedFork, + )) + } + /// Helper function for retrieving the build payload by id. async fn get_built_payload( &self, @@ -1281,6 +1309,30 @@ where Ok(self.new_payload_v5_metered(payload).await?) } + /// Handler for the stubbed `engine_newPayloadV6` endpoint. + /// + /// See also + async fn new_payload_v6( + &self, + payload: ExecutionPayloadV4, + versioned_hashes: Vec, + parent_beacon_block_root: B256, + execution_requests: RequestsOrHash, + inclusion_list_transactions: Vec, + ) -> RpcResult { + trace!(target: "rpc::engine", "Serving engine_newPayloadV6 stub"); + let payload = ExecutionData { + payload: payload.into(), + sidecar: ExecutionPayloadSidecar::v6( + CancunPayloadFields { versioned_hashes, parent_beacon_block_root }, + PraguePayloadFields { requests: execution_requests }, + inclusion_list_transactions.into(), + ), + }; + + Ok(self.new_payload_v6(payload).await?) + } + /// Handler for `engine_forkchoiceUpdatedV1` /// See also /// @@ -1332,6 +1384,21 @@ where .await?) } + /// Handler for the stubbed `engine_forkchoiceUpdatedV5` endpoint. + /// + /// See also + async fn fork_choice_updated_v5( + &self, + fork_choice_state: ForkchoiceState, + payload_attributes: Option, + custody_columns: Option, + ) -> RpcResult { + trace!(target: "rpc::engine", "Serving engine_forkchoiceUpdatedV5 stub"); + Ok(self + .fork_choice_updated_v5(fork_choice_state, payload_attributes, custody_columns) + .await?) + } + /// Handler for `engine_getPayloadV1` /// /// Returns the most recent version of the payload that is available in the corresponding From 63810e8956288abe40eac903af7fff1a86a99fb4 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Fri, 14 Aug 2026 00:00:40 +0200 Subject: [PATCH 023/294] fix(stages): validate block access list hash on import path (EIP-7928) (#26696) Co-authored-by: spencer-tb --- Cargo.lock | 6 ++++-- Cargo.toml | 2 +- crates/cli/commands/Cargo.toml | 2 ++ crates/cli/commands/src/re_execute.rs | 15 ++++++++++++++- crates/stages/stages/Cargo.toml | 1 + crates/stages/stages/src/stages/execution/mod.rs | 16 +++++++++++++++- 6 files changed, 37 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9cab6e4c18c..e776aad8344 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -252,9 +252,9 @@ dependencies = [ [[package]] name = "alloy-eip7928" -version = "0.4.5" +version = "0.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3b12337f74cbfa451cb04dac173974814a6ff463079e1793aa09600ba8813ab" +checksum = "5074ab00e3f04a5b377bd7398d18ede2dd8110c178a10295abaaa52e2761b6ca" dependencies = [ "alloy-primitives", "alloy-rlp", @@ -7844,6 +7844,7 @@ version = "2.5.0" dependencies = [ "alloy-chains", "alloy-consensus", + "alloy-eip7928", "alloy-eips", "alloy-primitives", "alloy-rlp", @@ -10249,6 +10250,7 @@ name = "reth-stages" version = "2.5.0" dependencies = [ "alloy-consensus", + "alloy-eip7928", "alloy-eips", "alloy-genesis", "alloy-primitives", diff --git a/Cargo.toml b/Cargo.toml index f81b7c0d7a3..7ac7acc2e88 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -442,7 +442,7 @@ alloy-sol-types = { version = "1.6.1", default-features = false } alloy-chains = { version = "0.2.33", default-features = false } alloy-eip2124 = { version = "0.2.0", default-features = false } -alloy-eip7928 = { version = "0.4.5", default-features = false, features = ["rlp"] } +alloy-eip7928 = { version = "0.4.7", default-features = false, features = ["rlp"] } alloy-evm = { version = "0.38.0", default-features = false } alloy-rlp = { version = "0.3.16", default-features = false, features = ["core-net"] } alloy-trie = { version = "0.9.4", default-features = false } diff --git a/crates/cli/commands/Cargo.toml b/crates/cli/commands/Cargo.toml index 3d64068cc39..f28d39c3218 100644 --- a/crates/cli/commands/Cargo.toml +++ b/crates/cli/commands/Cargo.toml @@ -59,6 +59,7 @@ reth-discv4.workspace = true reth-discv5.workspace = true # ethereum +alloy-eip7928.workspace = true alloy-eips.workspace = true alloy-primitives.workspace = true alloy-rlp.workspace = true @@ -122,6 +123,7 @@ arbitrary = [ "reth-db/arbitrary", "reth-chainspec/arbitrary", "alloy-chains/arbitrary", + "alloy-eip7928/arbitrary", "alloy-eips/arbitrary", "alloy-primitives/arbitrary", "reth-codecs/test-utils", diff --git a/crates/cli/commands/src/re_execute.rs b/crates/cli/commands/src/re_execute.rs index 6058f2f6e21..2cd537d042e 100644 --- a/crates/cli/commands/src/re_execute.rs +++ b/crates/cli/commands/src/re_execute.rs @@ -5,6 +5,7 @@ use crate::common::{ EnvironmentArgs, }; use alloy_consensus::{transaction::TxHashRef, BlockHeader, TxReceipt}; +use alloy_eip7928::bal::Bal; use alloy_primitives::{Address, B256, U256}; use clap::Parser; use eyre::WrapErr; @@ -149,6 +150,8 @@ impl let evm_config = evm_config.with_jit_support(); let executor_lifetime = Duration::from_secs(600); let provider = provider_factory.database_provider_ro()?.disable_long_read_transaction_safety(); + // Reused across blocks for BAL hash encoding. + let mut bal_buf = Vec::new(); let db_at = { |block_number: u64| { @@ -199,8 +202,12 @@ impl } }; + let bal_hash = executor + .take_bal() + .map(|bal| Bal::from(bal).compute_hash_with_buf(&mut bal_buf)); + if let Err(err) = consensus - .validate_block_post_execution(&block, &result, None,None) + .validate_block_post_execution(&block, &result, None, bal_hash) .wrap_err_with(|| { format!( "Failed to validate block {} {}", @@ -260,6 +267,12 @@ impl } } + if skip_invalid_blocks { + executor = + evm_config.batch_executor(db_at(block.number())); + let _ = info_tx.send((block, err)); + continue 'blocks; + } return Err(err); } let _ = stats_tx.send((block.number(), block.gas_used())); diff --git a/crates/stages/stages/Cargo.toml b/crates/stages/stages/Cargo.toml index 46eba3e0786..2bd2fafb102 100644 --- a/crates/stages/stages/Cargo.toml +++ b/crates/stages/stages/Cargo.toml @@ -45,6 +45,7 @@ reth-trie-db = { workspace = true, features = ["metrics"] } reth-testing-utils = { workspace = true, optional = true } +alloy-eip7928.workspace = true alloy-primitives.workspace = true alloy-consensus.workspace = true alloy-rlp.workspace = true diff --git a/crates/stages/stages/src/stages/execution/mod.rs b/crates/stages/stages/src/stages/execution/mod.rs index 61253135c85..e2a19febb98 100644 --- a/crates/stages/stages/src/stages/execution/mod.rs +++ b/crates/stages/stages/src/stages/execution/mod.rs @@ -1,5 +1,6 @@ use crate::stages::MERKLE_STAGE_DEFAULT_INCREMENTAL_THRESHOLD; use alloy_consensus::BlockHeader; +use alloy_eip7928::bal::Bal; use alloy_primitives::BlockNumber; use num_traits::Zero; use reth_chainspec::{ChainSpecProvider, EthereumHardforks}; @@ -333,6 +334,8 @@ where let mut blocks = Vec::new(); let mut results = Vec::new(); + // Reused across blocks for BAL hash encoding. + let mut bal_buf = Vec::new(); for block_number in start_block..=max_block { // Fetch the block let fetch_block_start = Instant::now(); @@ -359,8 +362,19 @@ where }) })?; + let built_bal = executor.take_bal().map(Bal::from); + if let Some(bal) = &built_bal && + let Err(err) = bal.validate_gas_limit(block.header().gas_limit()) + { + return Err(StageError::Block { + block: Box::new(block.block_with_parent()), + error: BlockErrorKind::Validation(err.into()), + }) + } + let bal_hash = built_bal.as_ref().map(|bal| bal.compute_hash_with_buf(&mut bal_buf)); + if let Err(err) = - self.consensus.validate_block_post_execution(&block, &result, None, None) + self.consensus.validate_block_post_execution(&block, &result, None, bal_hash) { return Err(StageError::Block { block: Box::new(block.block_with_parent()), From 4e6d0c29c0d76367b5a27a48d820ab7c00d0f4b6 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Fri, 14 Aug 2026 00:13:11 +0200 Subject: [PATCH 024/294] chore(deps): bump alloy-eip7928 to 0.4.8 (#26700) --- Cargo.lock | 4 ++-- Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e776aad8344..e8a09a309f8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -252,9 +252,9 @@ dependencies = [ [[package]] name = "alloy-eip7928" -version = "0.4.7" +version = "0.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5074ab00e3f04a5b377bd7398d18ede2dd8110c178a10295abaaa52e2761b6ca" +checksum = "3fb59cca9e58f5c624becc3cffb32772dc278f31eeae4606844a88592e8d2672" dependencies = [ "alloy-primitives", "alloy-rlp", diff --git a/Cargo.toml b/Cargo.toml index 7ac7acc2e88..7b0b7346903 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -442,7 +442,7 @@ alloy-sol-types = { version = "1.6.1", default-features = false } alloy-chains = { version = "0.2.33", default-features = false } alloy-eip2124 = { version = "0.2.0", default-features = false } -alloy-eip7928 = { version = "0.4.7", default-features = false, features = ["rlp"] } +alloy-eip7928 = { version = "0.4.8", default-features = false, features = ["rlp"] } alloy-evm = { version = "0.38.0", default-features = false } alloy-rlp = { version = "0.3.16", default-features = false, features = ["core-net"] } alloy-trie = { version = "0.9.4", default-features = false } From 20fab813c08ab9f5c7f7a2db1d3c304d717fe46c Mon Sep 17 00:00:00 2001 From: Sergei Shulepov Date: Thu, 13 Aug 2026 22:23:54 +0000 Subject: [PATCH 025/294] fix(network): better checking of ping / pong (#26698) --- crates/net/eth-wire/src/errors/eth.rs | 1 + crates/net/eth-wire/src/errors/p2p.rs | 4 + crates/net/eth-wire/src/p2pstream.rs | 127 +++++++++++++++++++++++--- crates/net/network/src/error.rs | 4 + 4 files changed, 123 insertions(+), 13 deletions(-) diff --git a/crates/net/eth-wire/src/errors/eth.rs b/crates/net/eth-wire/src/errors/eth.rs index 97c069c95d8..d18367ccadd 100644 --- a/crates/net/eth-wire/src/errors/eth.rs +++ b/crates/net/eth-wire/src/errors/eth.rs @@ -84,6 +84,7 @@ impl EthStreamError { P2PStreamError::MessageTooBig { .. } | P2PStreamError::UnknownReservedMessageId(_) | P2PStreamError::EmptyProtocolMessage | + P2PStreamError::InvalidPingPongPayload(_) | P2PStreamError::UnknownDisconnectReason(_) ) ) diff --git a/crates/net/eth-wire/src/errors/p2p.rs b/crates/net/eth-wire/src/errors/p2p.rs index b55a408e0df..e95186bd43c 100644 --- a/crates/net/eth-wire/src/errors/p2p.rs +++ b/crates/net/eth-wire/src/errors/p2p.rs @@ -43,6 +43,10 @@ pub enum P2PStreamError { #[error("empty protocol message received")] EmptyProtocolMessage, + /// Ping or Pong payload is not an RLP empty list. + #[error("invalid ping/pong payload for p2p message id: {0:#x}")] + InvalidPingPongPayload(u8), + /// Error related to the Pinger. #[error(transparent)] PingerError(#[from] PingerError), diff --git a/crates/net/eth-wire/src/p2pstream.rs b/crates/net/eth-wire/src/p2pstream.rs index c9eb041ff0b..141c656c15f 100644 --- a/crates/net/eth-wire/src/p2pstream.rs +++ b/crates/net/eth-wire/src/p2pstream.rs @@ -495,6 +495,22 @@ where } } + if id == P2PMessageID::Ping as u8 || id == P2PMessageID::Pong as u8 { + validate_ping_pong_payload(&mut this.decoder, id, &bytes[1..])?; + + if id == P2PMessageID::Ping as u8 { + trace!("Received Ping, Sending Pong"); + this.send_pong(); + // This is required because the `Sink` may not be polled externally, and if + // that happens, the pong will never be sent. + cx.waker().wake_by_ref(); + } else { + // if we were waiting for a pong, this will reset the pinger state + this.pinger.on_pong()?; + } + continue + } + // first check that the compressed message length does not exceed the max // payload size let decompressed_len = snap::raw::decompress_len(&bytes[1..])?; @@ -521,13 +537,6 @@ where })?; match id { - _ if id == P2PMessageID::Ping as u8 => { - trace!("Received Ping, Sending Pong"); - this.send_pong(); - // This is required because the `Sink` may not be polled externally, and if - // that happens, the pong will never be sent. - cx.waker().wake_by_ref(); - } _ if id == P2PMessageID::Hello as u8 => { // we have received a hello message outside of the handshake, so we will return // an error @@ -535,10 +544,6 @@ where P2PHandshakeError::HelloNotInHandshake, )))) } - _ if id == P2PMessageID::Pong as u8 => { - // if we were waiting for a pong, this will reset the pinger state - this.pinger.on_pong()? - } _ if id == P2PMessageID::Disconnect as u8 => { // At this point, the `decompress_buf` contains the snappy decompressed // disconnect message. @@ -591,6 +596,25 @@ where } } +/// Validates a compressed Ping or Pong payload without allocating from its advertised size. +fn validate_ping_pong_payload( + decoder: &mut snap::raw::Decoder, + message_id: u8, + compressed_payload: &[u8], +) -> Result<(), P2PStreamError> { + if snap::raw::decompress_len(compressed_payload)? != 1 { + return Err(P2PStreamError::InvalidPingPongPayload(message_id)) + } + + let mut payload = [0u8; 1]; + decoder.decompress(compressed_payload, &mut payload)?; + if payload != [EMPTY_LIST_CODE] { + return Err(P2PStreamError::InvalidPingPongPayload(message_id)) + } + + Ok(()) +} + impl Sink for P2PStream where S: Sink + Unpin, @@ -879,6 +903,7 @@ mod tests { /// A sink that records started frames and counts flushes, to observe batching behavior. #[derive(Default)] struct FlushCountingTransport { + incoming: VecDeque>, sent: Vec, flushes: usize, } @@ -886,8 +911,11 @@ mod tests { impl Stream for FlushCountingTransport { type Item = io::Result; - fn poll_next(self: Pin<&mut Self>, _: &mut Context<'_>) -> Poll> { - Poll::Pending + fn poll_next(mut self: Pin<&mut Self>, _: &mut Context<'_>) -> Poll> { + match self.incoming.pop_front() { + Some(item) => Poll::Ready(Some(item)), + None => Poll::Pending, + } } } @@ -924,6 +952,79 @@ mod tests { .unwrap() } + fn stream_with_incoming(frame: BytesMut) -> P2PStream { + let mut transport = FlushCountingTransport::default(); + transport.incoming.push_back(Ok(frame)); + P2PStream::new(transport, eth_shared_capabilities()) + } + + fn compressed_p2p_message(message_id: P2PMessageID, payload: &[u8]) -> BytesMut { + let mut encoder = snap::raw::Encoder::new(); + let mut scratch = Vec::new(); + let message = + compress_frame(&mut encoder, &mut scratch, message_id as u8, payload).unwrap(); + BytesMut::from(message.as_ref()) + } + + #[tokio::test] + async fn rejects_ping_pong_with_oversized_payload_before_decompression() { + const SIXTEEN_MIB_SNAPPY_HEADER: [u8; 4] = [0x80, 0x80, 0x80, 0x08]; + + for message_id in [P2PMessageID::Ping, P2PMessageID::Pong] { + let frame = BytesMut::from( + [ + message_id as u8, + SIXTEEN_MIB_SNAPPY_HEADER[0], + SIXTEEN_MIB_SNAPPY_HEADER[1], + SIXTEEN_MIB_SNAPPY_HEADER[2], + SIXTEEN_MIB_SNAPPY_HEADER[3], + ] + .as_slice(), + ); + assert_eq!(snap::raw::decompress_len(&frame[1..]).unwrap(), MAX_PAYLOAD_SIZE); + + let mut stream = stream_with_incoming(frame); + let waker = noop_waker_ref(); + let mut cx = Context::from_waker(waker); + + match Pin::new(&mut stream).poll_next(&mut cx) { + Poll::Ready(Some(Err(P2PStreamError::InvalidPingPongPayload(id)))) => { + assert_eq!(id, message_id as u8) + } + result => panic!("unexpected poll result: {result:?}"), + } + assert!(stream.outgoing_messages.is_empty()); + } + } + + #[tokio::test] + async fn rejects_ping_pong_with_non_list_payload() { + for message_id in [P2PMessageID::Ping, P2PMessageID::Pong] { + let frame = compressed_p2p_message(message_id, &[alloy_rlp::EMPTY_STRING_CODE]); + let mut stream = stream_with_incoming(frame); + let waker = noop_waker_ref(); + let mut cx = Context::from_waker(waker); + + assert!(matches!( + Pin::new(&mut stream).poll_next(&mut cx), + Poll::Ready(Some(Err(P2PStreamError::InvalidPingPongPayload(id)))) + if id == message_id as u8 + )); + assert!(stream.outgoing_messages.is_empty()); + } + } + + #[tokio::test] + async fn accepts_ping_with_empty_list_payload() { + let mut stream = stream_with_incoming(BytesMut::from(SNAPPY_PING_MESSAGE)); + let waker = noop_waker_ref(); + let mut cx = Context::from_waker(waker); + + assert!(Pin::new(&mut stream).poll_next(&mut cx).is_pending()); + assert_eq!(stream.outgoing_messages.len(), 1); + assert_eq!(stream.outgoing_messages.front().unwrap().as_ref(), SNAPPY_PONG_MESSAGE); + } + #[tokio::test] async fn poll_ready_drains_full_subprotocol_queue_without_flushing_inner() { let mut stream = diff --git a/crates/net/network/src/error.rs b/crates/net/network/src/error.rs index 7e3a84ba4eb..6c4e1c440bd 100644 --- a/crates/net/network/src/error.rs +++ b/crates/net/network/src/error.rs @@ -223,6 +223,7 @@ impl SessionError for EthStreamError { P2PStreamError::UnknownDisconnectReason(_) | P2PStreamError::MessageTooBig { .. } | P2PStreamError::EmptyProtocolMessage | + P2PStreamError::InvalidPingPongPayload(_) | P2PStreamError::PingerError(_) | P2PStreamError::Snap(_), ) => Some(BackoffKind::Medium), @@ -342,6 +343,9 @@ mod tests { P2PHandshakeError::NoResponse, )); assert_eq!(err.should_backoff(), Some(BackoffKind::Low)); + + let err = EthStreamError::P2PStreamError(P2PStreamError::InvalidPingPongPayload(0x02)); + assert_eq!(err.should_backoff(), Some(BackoffKind::Medium)); } #[test] From ae81966b68e71028b9efeead3e4a9bcb38ffe049 Mon Sep 17 00:00:00 2001 From: Sergei Shulepov Date: Thu, 13 Aug 2026 23:40:35 +0000 Subject: [PATCH 026/294] fix(network): pace pings. (#26702) --- crates/net/eth-wire/src/errors/eth.rs | 1 + crates/net/eth-wire/src/errors/p2p.rs | 4 + crates/net/eth-wire/src/p2pstream.rs | 108 +++++++++++++++++++++++++- crates/net/network/src/error.rs | 7 +- 4 files changed, 118 insertions(+), 2 deletions(-) diff --git a/crates/net/eth-wire/src/errors/eth.rs b/crates/net/eth-wire/src/errors/eth.rs index d18367ccadd..00c06108617 100644 --- a/crates/net/eth-wire/src/errors/eth.rs +++ b/crates/net/eth-wire/src/errors/eth.rs @@ -85,6 +85,7 @@ impl EthStreamError { P2PStreamError::UnknownReservedMessageId(_) | P2PStreamError::EmptyProtocolMessage | P2PStreamError::InvalidPingPongPayload(_) | + P2PStreamError::TooManyPings | P2PStreamError::UnknownDisconnectReason(_) ) ) diff --git a/crates/net/eth-wire/src/errors/p2p.rs b/crates/net/eth-wire/src/errors/p2p.rs index e95186bd43c..7622d5f2fac 100644 --- a/crates/net/eth-wire/src/errors/p2p.rs +++ b/crates/net/eth-wire/src/errors/p2p.rs @@ -47,6 +47,10 @@ pub enum P2PStreamError { #[error("invalid ping/pong payload for p2p message id: {0:#x}")] InvalidPingPongPayload(u8), + /// Incoming ping rate limit exceeded. + #[error("too many pings received")] + TooManyPings, + /// Error related to the Pinger. #[error(transparent)] PingerError(#[from] PingerError), diff --git a/crates/net/eth-wire/src/p2pstream.rs b/crates/net/eth-wire/src/p2pstream.rs index 141c656c15f..21b87dfcf76 100644 --- a/crates/net/eth-wire/src/p2pstream.rs +++ b/crates/net/eth-wire/src/p2pstream.rs @@ -21,7 +21,7 @@ use std::{ io, pin::Pin, task::{ready, Context, Poll}, - time::Duration, + time::{Duration, Instant}, }; use tokio_stream::Stream; use tracing::{debug, trace}; @@ -61,6 +61,9 @@ const PING_TIMEOUT: Duration = Duration::from_secs(15); /// when the peer is responsive. const PING_INTERVAL: Duration = Duration::from_secs(60); +/// Maximum number of incoming pings that can arrive in a burst. +const PING_TOKEN_BUCKET_CAPACITY: u8 = 5; + /// [`MAX_P2P_CAPACITY`] is the maximum number of messages that can be buffered to be sent in the /// `p2p` stream. /// @@ -275,6 +278,9 @@ pub struct P2PStream { /// The state machine used for keeping track of the peer's ping status. pinger: Pinger, + /// Per-connection limit for incoming ping bursts. + ping_token_bucket: PingTokenBucket, + /// The supported capability for this stream. shared_capabilities: SharedCapabilities, @@ -308,6 +314,7 @@ impl P2PStream { compress_scratch: Vec::new(), decoder: snap::raw::Decoder::new(), pinger: Pinger::new(PING_INTERVAL, PING_TIMEOUT), + ping_token_bucket: PingTokenBucket::new(Instant::now()), shared_capabilities, outgoing_messages: VecDeque::new(), outgoing_message_buffer_capacity: MAX_P2P_CAPACITY, @@ -358,6 +365,39 @@ impl P2PStream { } } +/// Per-connection bucket that restores one incoming ping token per second. +#[derive(Debug)] +struct PingTokenBucket { + tokens: u8, + last_refill: Instant, +} + +impl PingTokenBucket { + const fn new(now: Instant) -> Self { + Self { tokens: PING_TOKEN_BUCKET_CAPACITY, last_refill: now } + } + + fn try_take(&mut self, now: Instant) -> bool { + let refill = now.saturating_duration_since(self.last_refill).as_secs(); + if refill > 0 { + self.tokens = u64::from(self.tokens) + .saturating_add(refill) + .min(u64::from(PING_TOKEN_BUCKET_CAPACITY)) as u8; + self.last_refill += Duration::from_secs(refill); + } + + if self.tokens == 0 { + return false + } + + if self.tokens == PING_TOKEN_BUCKET_CAPACITY { + self.last_refill = now; + } + self.tokens -= 1; + true + } +} + /// Gracefully disconnects the connection by sending a disconnect message and stop reading new /// messages. pub trait DisconnectP2P { @@ -459,6 +499,8 @@ where return Poll::Ready(None) } + let mut ping_batch_time = None; + // we should loop here to ensure we don't return Poll::Pending if we have a message to // return behind any pings we need to respond to while let Poll::Ready(res) = this.inner.poll_next_unpin(cx) { @@ -499,6 +541,14 @@ where validate_ping_pong_payload(&mut this.decoder, id, &bytes[1..])?; if id == P2PMessageID::Ping as u8 { + // Use the timestamp of the first ping for every ping in this poll, so buffered + // pings form one burst. The tradeoff is that a poll that takes more than one + // second can reject a later ping. Considered acceptable. + let now = *ping_batch_time.get_or_insert_with(Instant::now); + if !this.ping_token_bucket.try_take(now) { + return Poll::Ready(Some(Err(P2PStreamError::TooManyPings))) + } + trace!("Received Ping, Sending Pong"); this.send_pong(); // This is required because the `Sink` may not be polled externally, and if @@ -958,6 +1008,12 @@ mod tests { P2PStream::new(transport, eth_shared_capabilities()) } + fn stream_with_incoming_pings(count: usize) -> P2PStream { + let mut transport = FlushCountingTransport::default(); + transport.incoming.extend((0..count).map(|_| Ok(BytesMut::from(SNAPPY_PING_MESSAGE)))); + P2PStream::new(transport, eth_shared_capabilities()) + } + fn compressed_p2p_message(message_id: P2PMessageID, payload: &[u8]) -> BytesMut { let mut encoder = snap::raw::Encoder::new(); let mut scratch = Vec::new(); @@ -1025,6 +1081,56 @@ mod tests { assert_eq!(stream.outgoing_messages.front().unwrap().as_ref(), SNAPPY_PONG_MESSAGE); } + #[test] + fn ping_token_bucket_limits_bursts_and_refills() { + let now = Instant::now(); + let mut bucket = PingTokenBucket::new(now); + + for _ in 0..PING_TOKEN_BUCKET_CAPACITY { + assert!(bucket.try_take(now)); + } + assert!(!bucket.try_take(now)); + + let one_refill = now + Duration::from_secs(1); + assert!(bucket.try_take(one_refill)); + assert!(!bucket.try_take(one_refill)); + + // The extra half second checks that refill time accumulated while the bucket is full is + // discarded. Otherwise, the next token could arrive less than one second after the burst. + let full_refill = one_refill + + Duration::from_secs(u64::from(PING_TOKEN_BUCKET_CAPACITY)) + + Duration::from_millis(500); + for _ in 0..PING_TOKEN_BUCKET_CAPACITY { + assert!(bucket.try_take(full_refill)); + } + assert!(!bucket.try_take(full_refill)); + assert!(!bucket.try_take(full_refill + Duration::from_millis(999))); + assert!(bucket.try_take(full_refill + Duration::from_secs(1))); + } + + #[tokio::test] + async fn accepts_ping_burst_at_token_bucket_capacity() { + let mut stream = stream_with_incoming_pings(usize::from(PING_TOKEN_BUCKET_CAPACITY)); + let waker = noop_waker_ref(); + let mut cx = Context::from_waker(waker); + + assert!(Pin::new(&mut stream).poll_next(&mut cx).is_pending()); + assert_eq!(stream.outgoing_messages.len(), usize::from(PING_TOKEN_BUCKET_CAPACITY)); + } + + #[tokio::test] + async fn rejects_ping_burst_over_token_bucket_capacity() { + let mut stream = stream_with_incoming_pings(usize::from(PING_TOKEN_BUCKET_CAPACITY) + 1); + let waker = noop_waker_ref(); + let mut cx = Context::from_waker(waker); + + assert!(matches!( + Pin::new(&mut stream).poll_next(&mut cx), + Poll::Ready(Some(Err(P2PStreamError::TooManyPings))) + )); + assert_eq!(stream.outgoing_messages.len(), usize::from(PING_TOKEN_BUCKET_CAPACITY)); + } + #[tokio::test] async fn poll_ready_drains_full_subprotocol_queue_without_flushing_inner() { let mut stream = diff --git a/crates/net/network/src/error.rs b/crates/net/network/src/error.rs index 6c4e1c440bd..daf43d55713 100644 --- a/crates/net/network/src/error.rs +++ b/crates/net/network/src/error.rs @@ -216,7 +216,7 @@ impl SessionError for EthStreamError { P2PStreamError::HandshakeError(P2PHandshakeError::NoResponse) | P2PStreamError::PingTimeout, ) => Some(BackoffKind::Low), - // malformed messages + // malformed or abusive messages Self::P2PStreamError( P2PStreamError::Rlp(_) | P2PStreamError::UnknownReservedMessageId(_) | @@ -224,6 +224,7 @@ impl SessionError for EthStreamError { P2PStreamError::MessageTooBig { .. } | P2PStreamError::EmptyProtocolMessage | P2PStreamError::InvalidPingPongPayload(_) | + P2PStreamError::TooManyPings | P2PStreamError::PingerError(_) | P2PStreamError::Snap(_), ) => Some(BackoffKind::Medium), @@ -346,6 +347,10 @@ mod tests { let err = EthStreamError::P2PStreamError(P2PStreamError::InvalidPingPongPayload(0x02)); assert_eq!(err.should_backoff(), Some(BackoffKind::Medium)); + + let err = EthStreamError::P2PStreamError(P2PStreamError::TooManyPings); + assert!(err.is_protocol_breach()); + assert_eq!(err.should_backoff(), Some(BackoffKind::Medium)); } #[test] From dd637cefe1f79f6e4815a2389a36412296cde4ad Mon Sep 17 00:00:00 2001 From: Sergei Shulepov Date: Fri, 14 Aug 2026 00:20:47 +0000 Subject: [PATCH 027/294] feat(network): ingress limits (#26660) Co-authored-by: Matthias Seitz --- crates/net/eth-wire/src/errors/eth.rs | 1 + crates/net/eth-wire/src/errors/p2p.rs | 26 +- crates/net/eth-wire/src/multiplex.rs | 693 ++++++++++++++++++++++---- crates/net/eth-wire/src/p2pstream.rs | 177 ++++++- crates/net/eth-wire/src/protocol.rs | 81 +++ crates/net/network/src/error.rs | 31 ++ crates/net/network/src/protocol.rs | 15 +- crates/net/network/src/session/mod.rs | 5 +- 8 files changed, 931 insertions(+), 98 deletions(-) diff --git a/crates/net/eth-wire/src/errors/eth.rs b/crates/net/eth-wire/src/errors/eth.rs index 00c06108617..f8826a73838 100644 --- a/crates/net/eth-wire/src/errors/eth.rs +++ b/crates/net/eth-wire/src/errors/eth.rs @@ -83,6 +83,7 @@ impl EthStreamError { P2PStreamError::Snap(_) | P2PStreamError::MessageTooBig { .. } | P2PStreamError::UnknownReservedMessageId(_) | + P2PStreamError::UnknownSubprotocolMessageId(_) | P2PStreamError::EmptyProtocolMessage | P2PStreamError::InvalidPingPongPayload(_) | P2PStreamError::TooManyPings | diff --git a/crates/net/eth-wire/src/errors/p2p.rs b/crates/net/eth-wire/src/errors/p2p.rs index 7622d5f2fac..02890d77564 100644 --- a/crates/net/eth-wire/src/errors/p2p.rs +++ b/crates/net/eth-wire/src/errors/p2p.rs @@ -5,7 +5,7 @@ use std::io; use reth_eth_wire_types::{DisconnectReason, UnknownDisconnectReason}; use reth_primitives_traits::GotExpected; -use crate::{capability::SharedCapabilityError, ProtocolVersion}; +use crate::{capability::SharedCapabilityError, Capability, ProtocolVersion}; /// Errors when sending/receiving p2p messages. These should result in kicking the peer. #[derive(thiserror::Error, Debug)] @@ -35,6 +35,30 @@ pub enum P2PStreamError { max_size: usize, }, + /// A subprotocol message exceeds the limit declared by its local handler. + #[error( + "message for subprotocol {capability} has size {message_size}, exceeding max length {max_size}" + )] + SubprotocolMessageTooBig { + /// The negotiated capability whose limit was exceeded. + capability: Capability, + /// The frame size, including the capability-local message ID. + message_size: usize, + /// The maximum frame size declared by the protocol handler. + max_size: usize, + }, + + /// A subprotocol's bounded inbound queue has no remaining capacity. + #[error("inbound buffer for subprotocol {capability} is full")] + SubprotocolInboundBufferFull { + /// The negotiated capability whose queue is full. + capability: Capability, + }, + + /// An incoming message ID is outside all negotiated subprotocol ranges. + #[error("unknown subprotocol message id: {0:#04x}")] + UnknownSubprotocolMessageId(u8), + /// Unknown reserved P2P message ID error. #[error("unknown reserved p2p message id: {0}")] UnknownReservedMessageId(u8), diff --git a/crates/net/eth-wire/src/multiplex.rs b/crates/net/eth-wire/src/multiplex.rs index 4d0f1e97cf2..b06fc8f5d14 100644 --- a/crates/net/eth-wire/src/multiplex.rs +++ b/crates/net/eth-wire/src/multiplex.rs @@ -13,7 +13,10 @@ use std::{ future::Future, io, pin::{pin, Pin}, - sync::Arc, + sync::{ + atomic::{AtomicUsize, Ordering}, + Arc, + }, task::{ready, Context, Poll}, }; @@ -22,6 +25,7 @@ use crate::{ errors::{EthStreamError, P2PStreamError}, handshake::EthRlpxHandshake, p2pstream::DisconnectP2P, + protocol::ProtocolIngressLimits, CanDisconnect, Capability, DisconnectReason, EthStream, P2PStream, UnifiedStatus, HANDSHAKE_TIMEOUT, }; @@ -29,8 +33,9 @@ use bytes::{Bytes, BytesMut}; use futures::{Sink, SinkExt, Stream, StreamExt, TryStream, TryStreamExt}; use reth_eth_wire_types::NetworkPrimitives; use reth_ethereum_forks::ForkFilter; +use reth_metrics::metrics::counter; use tokio::sync::{mpsc, mpsc::UnboundedSender}; -use tokio_stream::wrappers::UnboundedReceiverStream; +use tokio_stream::wrappers::{ReceiverStream, UnboundedReceiverStream}; /// A Stream and Sink type that wraps a raw rlpx stream [`P2PStream`] and handles message ID /// multiplexing. @@ -47,6 +52,7 @@ impl RlpxProtocolMultiplexer { conn, protocols: Default::default(), out_buffer: Default::default(), + inbound_budget: Arc::new(InboundBudget::new(MAX_MUX_IN_BUFFER_BYTES)), }, } } @@ -64,7 +70,21 @@ impl RlpxProtocolMultiplexer { F: FnOnce(ProtocolConnection) -> Proto, Proto: Stream + Send + 'static, { - self.inner.install_protocol(cap, f) + self.inner.install_protocol(cap, ProtocolIngressLimits::default(), f) + } + + /// Installs a new protocol with local limits for inbound messages. + pub fn install_protocol_with_limits( + &mut self, + cap: &Capability, + limits: ProtocolIngressLimits, + f: F, + ) -> Result<(), UnsupportedCapabilityError> + where + F: FnOnce(ProtocolConnection) -> Proto, + Proto: Stream + Send + 'static, + { + self.inner.install_protocol(cap, limits, f) } /// Returns the [`SharedCapabilities`] of the underlying raw p2p stream @@ -86,13 +106,10 @@ impl RlpxProtocolMultiplexer { return Err(P2PStreamError::CapabilityNotShared) }; - let (to_primary, from_wire) = mpsc::unbounded_channel(); + let (to_primary, from_wire) = + self.inner.inbound_channel(&shared_cap, ProtocolIngressLimits::default()); let (to_wire, from_primary) = mpsc::unbounded_channel(); - let proxy = ProtocolProxy { - shared_cap: shared_cap.clone(), - from_wire: UnboundedReceiverStream::new(from_wire), - to_wire, - }; + let proxy = ProtocolProxy { shared_cap: shared_cap.clone(), from_wire, to_wire }; let st = primary(proxy); Ok(RlpxSatelliteStream { @@ -104,6 +121,7 @@ impl RlpxProtocolMultiplexer { shared_cap, }, next_outbound: 0, + producer_polls_since_inbound: 0, }) } @@ -155,47 +173,20 @@ impl RlpxProtocolMultiplexer { return Err(P2PStreamError::CapabilityNotShared.into()) }; - let (to_primary, from_wire) = mpsc::unbounded_channel(); + let (to_primary, from_wire) = + self.inner.inbound_channel(&shared_cap, ProtocolIngressLimits::default()); let (to_wire, mut from_primary) = mpsc::unbounded_channel(); - let proxy = ProtocolProxy { - shared_cap: shared_cap.clone(), - from_wire: UnboundedReceiverStream::new(from_wire), - to_wire, - }; + let proxy = ProtocolProxy { shared_cap: shared_cap.clone(), from_wire, to_wire }; let f = handshake(proxy); let mut f = pin!(f); + let mut inbound_messages = 0; // this polls the connection and the primary stream concurrently until the handshake is // complete loop { tokio::select! { biased; - Some(Ok(msg)) = self.inner.conn.next() => { - // Ensure the message belongs to the primary protocol - let Some(offset) = msg.first().copied() - else { - return Err(P2PStreamError::EmptyProtocolMessage.into()) - }; - if let Some(cap) = self.shared_capabilities().find_by_relative_offset(offset).cloned() { - if cap == shared_cap { - // delegate to primary - let _ = to_primary.send(msg); - } else { - // delegate to satellite - self.inner.delegate_message(&cap, msg); - } - } else { - return Err(P2PStreamError::UnknownReservedMessageId(offset).into()) - } - } - Some(msg) = from_primary.recv() => { - self.inner.conn.send(msg).await.map_err(Into::into)?; - } - // Poll all subprotocols for new messages - msg = ProtocolsPoller::new(&mut self.inner.protocols) => { - self.inner.conn.send(msg.map_err(Into::into)?).await.map_err(Into::into)?; - } res = &mut f => { let (st, extra) = res?; return Ok(( @@ -208,10 +199,50 @@ impl RlpxProtocolMultiplexer { shared_cap, }, next_outbound: 0, + producer_polls_since_inbound: 0, }, extra, )) } + Some(msg) = from_primary.recv() => { + self.inner.conn.send(msg).await.map_err(Into::into)?; + } + // Polling a satellite drives its inbound consumer even when it has no outbound + // message. This must happen before reading another frame from the socket. + msg = ProtocolsPoller::new(&mut self.inner.protocols) => { + self.inner.conn.send(msg.map_err(Into::into)?).await.map_err(Into::into)?; + } + incoming = self.inner.conn.next() => { + let msg = match incoming { + Some(Ok(msg)) => msg, + Some(Err(err)) => return Err(err.into()), + None => { + return Err(P2PStreamError::Io(io::ErrorKind::UnexpectedEof.into()).into()) + } + }; + // Ensure the message belongs to the primary protocol + let Some(offset) = msg.first().copied() + else { + return Err(P2PStreamError::EmptyProtocolMessage.into()) + }; + if let Some(cap) = self.shared_capabilities().find_by_relative_offset(offset).cloned() { + if cap == shared_cap { + // delegate to primary + to_primary.try_send(msg).map_err(Into::into)?; + } else { + // delegate to satellite + self.inner.delegate_message(&cap, msg).map_err(Into::into)?; + } + } else { + return Err(P2PStreamError::UnknownReservedMessageId(offset).into()) + } + + inbound_messages += 1; + if inbound_messages == MAX_INBOUND_MESSAGES_PER_POLL { + inbound_messages = 0; + yield_once().await; + } + } } } } @@ -257,6 +288,8 @@ struct MultiplexInner { protocols: VecDeque, /// Buffer for outgoing messages on the wire. out_buffer: OutBuffer, + /// Byte budget shared by every inbound protocol queue on this connection. + inbound_budget: Arc, } impl MultiplexInner { @@ -265,19 +298,24 @@ impl MultiplexInner { } /// Delegates a message to the matching protocol. - fn delegate_message(&self, cap: &SharedCapability, msg: BytesMut) -> bool { + fn delegate_message( + &self, + cap: &SharedCapability, + msg: BytesMut, + ) -> Result { for proto in &self.protocols { if proto.shared_cap == *cap { - proto.send_raw(msg); - return true + proto.send_raw(msg)?; + return Ok(true) } } - false + Ok(false) } fn install_protocol( &mut self, cap: &Capability, + limits: ProtocolIngressLimits, f: F, ) -> Result<(), UnsupportedCapabilityError> where @@ -286,20 +324,29 @@ impl MultiplexInner { { let shared_cap = self.conn.shared_capabilities().ensure_matching_capability(cap).cloned()?; - let (to_satellite, rx) = mpsc::unbounded_channel(); - let proto_conn = ProtocolConnection { from_wire: UnboundedReceiverStream::new(rx) }; + self.conn.set_protocol_ingress_limits(&shared_cap, limits); + let (to_satellite, from_wire) = self.inbound_channel(&shared_cap, limits); + let proto_conn = ProtocolConnection { from_wire }; let st = f(proto_conn); let st = ProtocolStream { shared_cap, to_satellite, satellite_st: Box::pin(st) }; self.protocols.push_back(st); Ok(()) } + + fn inbound_channel( + &self, + capability: &SharedCapability, + limits: ProtocolIngressLimits, + ) -> (InboundSender, InboundReceiver) { + inbound_channel(capability, limits, Arc::clone(&self.inbound_budget)) + } } /// Represents a protocol in the multiplexer that is used as the primary protocol. #[derive(Debug)] struct PrimaryProtocol { /// Channel to send messages to the primary protocol. - to_primary: UnboundedSender, + to_primary: InboundSender, /// Receiver for messages from the primary protocol. from_primary: UnboundedReceiverStream, /// Shared capability of the primary protocol. @@ -315,7 +362,7 @@ struct PrimaryProtocol { pub struct ProtocolProxy { shared_cap: SharedCapability, /// Receives _non-empty_ messages from the wire - from_wire: UnboundedReceiverStream, + from_wire: InboundReceiver, /// Sends _non-empty_ messages from the wire to_wire: UnboundedSender, } @@ -451,12 +498,127 @@ impl CanDisconnect for UnauthProxy { } } +#[derive(Debug)] +struct InboundSender { + inner: mpsc::Sender, + connection_budget: Arc, + protocol_budget: Arc, + capability: Capability, +} + +impl InboundSender { + fn try_send(&self, frame: BytesMut) -> Result<(), P2PStreamError> { + let size = frame.capacity().max(frame.len()); + let Some(connection_guard) = InboundBudget::try_reserve(&self.connection_budget, size) + else { + return Err(self.buffer_full()) + }; + let Some(protocol_guard) = InboundBudget::try_reserve(&self.protocol_budget, size) else { + return Err(self.buffer_full()) + }; + + let frame = BudgetedInbound { + frame, + _connection_guard: connection_guard, + _protocol_guard: protocol_guard, + }; + self.inner.try_send(frame).map_err(|err| match err { + mpsc::error::TrySendError::Full(_) => self.buffer_full(), + mpsc::error::TrySendError::Closed(_) => { + P2PStreamError::Io(io::ErrorKind::BrokenPipe.into()) + } + }) + } + + fn buffer_full(&self) -> P2PStreamError { + counter!("p2pstream.subprotocol_inbound_buffer_full").increment(1); + P2PStreamError::SubprotocolInboundBufferFull { capability: self.capability.clone() } + } +} + +#[derive(Debug)] +struct InboundReceiver { + inner: ReceiverStream, +} + +impl Stream for InboundReceiver { + type Item = BytesMut; + + fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + self.inner.poll_next_unpin(cx).map(|frame| frame.map(BudgetedInbound::into_frame)) + } +} + +#[derive(Debug)] +struct BudgetedInbound { + frame: BytesMut, + _connection_guard: InboundBudgetGuard, + _protocol_guard: InboundBudgetGuard, +} + +impl BudgetedInbound { + fn into_frame(self) -> BytesMut { + let Self { frame, _connection_guard, _protocol_guard } = self; + frame + } +} + +#[derive(Debug)] +struct InboundBudget { + used: AtomicUsize, + max: usize, +} + +impl InboundBudget { + const fn new(max: usize) -> Self { + Self { used: AtomicUsize::new(0), max } + } + + fn try_reserve(budget: &Arc, size: usize) -> Option { + budget + .used + .try_update(Ordering::Relaxed, Ordering::Relaxed, |used| { + used.checked_add(size).filter(|next| *next <= budget.max) + }) + .ok() + .map(|_| InboundBudgetGuard { size, budget: Arc::clone(budget) }) + } +} + +#[derive(Debug)] +struct InboundBudgetGuard { + size: usize, + budget: Arc, +} + +impl Drop for InboundBudgetGuard { + fn drop(&mut self) { + self.budget.used.fetch_sub(self.size, Ordering::Relaxed); + } +} + +fn inbound_channel( + capability: &SharedCapability, + limits: ProtocolIngressLimits, + connection_budget: Arc, +) -> (InboundSender, InboundReceiver) { + let (tx, rx) = mpsc::channel(limits.max_buffered_messages()); + let sender = InboundSender { + inner: tx, + connection_budget, + protocol_budget: Arc::new(InboundBudget::new(limits.max_buffered_bytes())), + capability: capability.capability().into_owned(), + }; + let receiver = InboundReceiver { inner: ReceiverStream::new(rx) }; + (sender, receiver) +} + /// A connection channel to receive _`non_empty`_ messages for the negotiated protocol. /// /// This is a [Stream] that returns raw bytes of the received messages for this protocol. #[derive(Debug)] pub struct ProtocolConnection { - from_wire: UnboundedReceiverStream, + from_wire: InboundReceiver, } impl Stream for ProtocolConnection { @@ -475,6 +637,8 @@ pub struct RlpxSatelliteStream { primary: PrimaryProtocol, /// Round-robin cursor for the next outbound producer to poll. next_outbound: usize, + /// Number of round-robin producers polled since the last inbound frame. + producer_polls_since_inbound: usize, } impl RlpxSatelliteStream { @@ -491,7 +655,29 @@ impl RlpxSatelliteStream { F: FnOnce(ProtocolConnection) -> Proto, Proto: Stream + Send + 'static, { - self.inner.install_protocol(cap, f) + let result = self.inner.install_protocol(cap, ProtocolIngressLimits::default(), f); + if result.is_ok() { + self.producer_polls_since_inbound = 0; + } + result + } + + /// Installs a new protocol with local limits for inbound messages. + pub fn install_protocol_with_limits( + &mut self, + cap: &Capability, + limits: ProtocolIngressLimits, + f: F, + ) -> Result<(), UnsupportedCapabilityError> + where + F: FnOnce(ProtocolConnection) -> Proto, + Proto: Stream + Send + 'static, + { + let result = self.inner.install_protocol(cap, limits, f); + if result.is_ok() { + self.producer_polls_since_inbound = 0; + } + result } /// Returns the primary protocol. @@ -544,6 +730,9 @@ impl RlpxSatelliteStream { let producer = self.next_outbound; self.next_outbound = (self.next_outbound + 1) % producers; + if self.producer_polls_since_inbound < producers { + self.producer_polls_since_inbound += 1; + } let msg = if producer == 0 { match self.primary.from_primary.poll_next_unpin(cx) { @@ -588,6 +777,7 @@ where fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { let this = self.get_mut(); + let mut inbound_messages = 0; loop { // first drain the primary stream @@ -634,52 +824,62 @@ where } match this.poll_outbound_producers(cx) { + Ok(ProducerPoll::Full) + if this.producer_polls_since_inbound < this.inner.protocols.len() + 1 => + { + // A full buffer prevents protocol streams from being polled. Pause ingress + // until every consumer has had a chance to drain the previous frame. + if conn_ready { + cx.waker().wake_by_ref(); + } + return Poll::Pending + } Ok(ProducerPoll::Pending | ProducerPoll::Full) => {} Ok(ProducerPoll::Closed) => return Poll::Ready(None), Err(err) => return Poll::Ready(Some(Err(P2PStreamError::Io(err).into()))), } let mut delegated = false; - loop { - // pull messages from connection - match this.inner.conn.poll_next_unpin(cx) { - Poll::Ready(Some(Ok(msg))) => { - delegated = true; - let Some(offset) = msg.first().copied() else { - return Poll::Ready(Some(Err( - P2PStreamError::EmptyProtocolMessage.into() - ))) - }; - // delegate the multiplexed message to the correct protocol - if let Some(cap) = - this.inner.conn.shared_capabilities().find_by_relative_offset(offset) - { - if cap == &this.primary.shared_cap { - // delegate to primary - let _ = this.primary.to_primary.send(msg); - } else { - // delegate to installed satellite if any - for proto in &this.inner.protocols { - if proto.shared_cap == *cap { - proto.send_raw(msg); - break - } - } - } - } else { - return Poll::Ready(Some(Err(P2PStreamError::UnknownReservedMessageId( - offset, - ) - .into()))) - } + // Pull one message before returning to the top of the loop, where every protocol gets + // another chance to consume its inbound queue. + match this.inner.conn.poll_next_unpin(cx) { + Poll::Ready(Some(Ok(msg))) => { + delegated = true; + let Some(offset) = msg.first().copied() else { + return Poll::Ready(Some(Err(P2PStreamError::EmptyProtocolMessage.into()))) + }; + let Some(cap) = this + .inner + .conn + .shared_capabilities() + .find_by_relative_offset(offset) + .cloned() + else { + return Poll::Ready(Some(Err(P2PStreamError::UnknownReservedMessageId( + offset, + ) + .into()))) + }; + + let result = if cap == this.primary.shared_cap { + this.primary.to_primary.try_send(msg) + } else { + this.inner.delegate_message(&cap, msg).map(|_| ()) + }; + if let Err(err) = result { + return Poll::Ready(Some(Err(err.into()))) } - Poll::Ready(Some(Err(err))) => return Poll::Ready(Some(Err(err.into()))), - Poll::Ready(None) => { - // connection closed - return Poll::Ready(None) + this.producer_polls_since_inbound = 0; + + inbound_messages += 1; + if inbound_messages == MAX_INBOUND_MESSAGES_PER_POLL { + cx.waker().wake_by_ref(); + return Poll::Pending } - Poll::Pending => break, } + Poll::Ready(Some(Err(err))) => return Poll::Ready(Some(Err(err.into()))), + Poll::Ready(None) => return Poll::Ready(None), + Poll::Pending => {} } if !conn_ready || (!delegated && this.inner.out_buffer.is_empty()) { @@ -725,7 +925,7 @@ where struct ProtocolStream { shared_cap: SharedCapability, /// the channel shared with the satellite stream - to_satellite: UnboundedSender, + to_satellite: InboundSender, satellite_st: Pin + Send>>, } @@ -757,8 +957,9 @@ impl ProtocolStream { } /// Sends the message to the satellite stream. - fn send_raw(&self, msg: BytesMut) { - let _ = self.unmask_id(msg).map(|msg| self.to_satellite.send(msg)); + fn send_raw(&self, msg: BytesMut) -> Result<(), P2PStreamError> { + let msg = self.unmask_id(msg).map_err(P2PStreamError::from)?; + self.to_satellite.try_send(msg) } } @@ -818,6 +1019,25 @@ impl<'a> Future for ProtocolsPoller<'a> { } } +fn yield_once() -> impl Future { + let mut yielded = false; + std::future::poll_fn(move |cx| { + if yielded { + Poll::Ready(()) + } else { + yielded = true; + cx.waker().wake_by_ref(); + Poll::Pending + } + }) +} + +/// Aggregate byte budget for inbound messages queued across all protocols on one connection. +const MAX_MUX_IN_BUFFER_BYTES: usize = 32 * 1024 * 1024; + +/// Maximum network messages delegated before yielding the connection task. +const MAX_INBOUND_MESSAGES_PER_POLL: usize = 32; + /// Soft cap for per-connection outbound `RLPx` messages waiting in the multiplexer. /// /// The cap is soft because the next message size is only known after polling a protocol stream. @@ -888,7 +1108,7 @@ mod tests { }; use futures::{stream, task::noop_waker_ref}; use reth_eth_wire_types::EthNetworkPrimitives; - use std::task::Poll; + use std::{sync::atomic::AtomicBool, task::Poll}; use tokio::{net::TcpListener, sync::oneshot}; use tokio_util::codec::Decoder; @@ -905,6 +1125,24 @@ mod tests { } } + #[derive(Debug)] + struct BurstingPrimary { + proxy: ProtocolProxy, + remaining: usize, + } + + impl Stream for BurstingPrimary { + type Item = Result<(), P2PStreamError>; + + fn poll_next(mut self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll> { + if self.remaining > 0 { + self.remaining -= 1; + self.proxy.try_send(Bytes::from_static(&[0, 0])).unwrap(); + } + Poll::Pending + } + } + #[derive(Debug)] struct StalledTransport; @@ -945,6 +1183,279 @@ mod tests { } } + #[derive(Debug)] + struct InboundFramesTransport { + frames: VecDeque, + writable: AtomicBool, + } + + impl Stream for InboundFramesTransport { + type Item = io::Result; + + fn poll_next(mut self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll> { + Poll::Ready(self.frames.pop_front().map(Ok)) + } + } + + impl Sink for InboundFramesTransport { + type Error = io::Error; + + fn poll_ready( + self: Pin<&mut Self>, + _cx: &mut Context<'_>, + ) -> Poll> { + if self.writable.load(Ordering::Relaxed) { + Poll::Ready(Ok(())) + } else { + Poll::Pending + } + } + + fn start_send(self: Pin<&mut Self>, _item: Bytes) -> Result<(), Self::Error> { + Ok(()) + } + + fn poll_flush(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + self.poll_ready(cx) + } + + fn poll_close(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + self.poll_ready(cx) + } + } + + #[derive(Debug)] + struct HoldingProtocol { + _conn: ProtocolConnection, + } + + impl Stream for HoldingProtocol { + type Item = BytesMut; + + fn poll_next(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll> { + Poll::Pending + } + } + + #[derive(Debug)] + struct DrainingProtocol { + conn: ProtocolConnection, + } + + impl Stream for DrainingProtocol { + type Item = BytesMut; + + fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + while matches!(self.conn.poll_next_unpin(cx), Poll::Ready(Some(_))) {} + Poll::Pending + } + } + + #[derive(Debug)] + struct SignalingDrainingProtocol { + conn: ProtocolConnection, + drained: Option>, + } + + impl Stream for SignalingDrainingProtocol { + type Item = BytesMut; + + fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + if matches!(self.conn.poll_next_unpin(cx), Poll::Ready(Some(_))) && + let Some(drained) = self.drained.take() + { + let _ = drained.send(()); + } + Poll::Pending + } + } + + fn compressed_frame(message_id: u8, payload: &[u8]) -> BytesMut { + let mut encoded = vec![0; snap::raw::max_compress_len(payload.len()) + 1]; + encoded[0] = message_id; + let len = snap::raw::Encoder::new().compress(payload, &mut encoded[1..]).unwrap(); + encoded.truncate(len + 1); + BytesMut::from(encoded.as_slice()) + } + + fn test_multiplexer( + frame_count: usize, + ) -> (RlpxProtocolMultiplexer, SharedCapability, SharedCapability) { + let (hello, _) = test_hello(); + let shared_capabilities = + SharedCapabilities::try_new(hello.protocols.clone(), hello.message().capabilities) + .unwrap(); + let eth = shared_capabilities.eth().unwrap().clone(); + let test = shared_capabilities.find(&TestProtoMessage::capability()).unwrap().clone(); + let frames = + (0..frame_count).map(|_| compressed_frame(test.message_id_offset(), &[0])).collect(); + let conn = P2PStream::new( + InboundFramesTransport { frames, writable: AtomicBool::new(true) }, + shared_capabilities, + ); + (RlpxProtocolMultiplexer::new(conn), eth, test) + } + + fn shared_test_capability(name: &'static str, offset: u8) -> SharedCapability { + SharedCapability::UnknownCapability { + cap: Capability::new_static(name, 1), + offset, + messages: 1, + } + } + + #[tokio::test] + async fn inbound_channel_enforces_and_releases_protocol_limits() { + let capability = shared_test_capability("test", 0x10); + let limits = + ProtocolIngressLimits::new(16).with_max_buffered_bytes(4).with_max_buffered_messages(1); + let aggregate = Arc::new(InboundBudget::new(16)); + let (sender, mut receiver) = inbound_channel(&capability, limits, aggregate); + + sender.try_send(BytesMut::zeroed(4)).unwrap(); + assert!(matches!( + sender.try_send(BytesMut::zeroed(1)), + Err(P2PStreamError::SubprotocolInboundBufferFull { .. }) + )); + + assert_eq!(receiver.next().await.unwrap().len(), 4); + sender.try_send(BytesMut::zeroed(1)).unwrap(); + } + + #[tokio::test] + async fn inbound_channels_share_connection_byte_budget() { + let aggregate = Arc::new(InboundBudget::new(6)); + let limits = ProtocolIngressLimits::new(16) + .with_max_buffered_bytes(16) + .with_max_buffered_messages(2); + let (first, mut first_receiver) = + inbound_channel(&shared_test_capability("aaa", 0x10), limits, Arc::clone(&aggregate)); + let (second, _second_receiver) = + inbound_channel(&shared_test_capability("bbb", 0x11), limits, aggregate); + + first.try_send(BytesMut::zeroed(4)).unwrap(); + assert!(matches!( + second.try_send(BytesMut::zeroed(4)), + Err(P2PStreamError::SubprotocolInboundBufferFull { .. }) + )); + + let _ = first_receiver.next().await.unwrap(); + second.try_send(BytesMut::zeroed(4)).unwrap(); + } + + #[tokio::test] + async fn stalled_satellite_fails_when_its_inbound_queue_is_full() { + let (mut mux, eth, test) = test_multiplexer(2); + let limits = + ProtocolIngressLimits::new(2).with_max_buffered_bytes(2).with_max_buffered_messages(1); + mux.install_protocol_with_limits(&TestProtoMessage::capability(), limits, |conn| { + HoldingProtocol { _conn: conn } + }) + .unwrap(); + let mut stream = mux + .into_satellite_stream(eth.capability().as_ref(), |proxy| PendingPrimary { + _proxy: proxy, + }) + .unwrap(); + + let mut cx = Context::from_waker(noop_waker_ref()); + assert!(matches!( + Pin::new(&mut stream).poll_next(&mut cx), + Poll::Ready(Some(Err(P2PStreamError::SubprotocolInboundBufferFull { + capability, + }))) if capability == test.capability().into_owned() + )); + } + + #[tokio::test] + async fn satellite_is_polled_between_inbound_frames() { + let (mut mux, eth, _) = test_multiplexer(2 * MAX_INBOUND_MESSAGES_PER_POLL); + let limits = + ProtocolIngressLimits::new(2).with_max_buffered_bytes(2).with_max_buffered_messages(1); + mux.install_protocol_with_limits(&TestProtoMessage::capability(), limits, |conn| { + DrainingProtocol { conn } + }) + .unwrap(); + let mut stream = mux + .into_satellite_stream(eth.capability().as_ref(), |proxy| PendingPrimary { + _proxy: proxy, + }) + .unwrap(); + + let mut cx = Context::from_waker(noop_waker_ref()); + assert!(Pin::new(&mut stream).poll_next(&mut cx).is_pending()); + assert_eq!(stream.inner.conn.inner().frames.len(), MAX_INBOUND_MESSAGES_PER_POLL); + } + + #[tokio::test] + async fn outbound_backpressure_preserves_inbound_fairness() { + let (hello, _) = test_hello(); + let shared_capabilities = + SharedCapabilities::try_new(hello.protocols.clone(), hello.message().capabilities) + .unwrap(); + let eth = shared_capabilities.eth().unwrap().clone(); + let test = shared_capabilities.find(&TestProtoMessage::capability()).unwrap().clone(); + let frames = (0..2) + .map(|_| compressed_frame(test.message_id_offset(), &[0])) + .collect::>(); + let conn = P2PStream::new( + InboundFramesTransport { frames, writable: AtomicBool::new(false) }, + shared_capabilities, + ); + let mut mux = RlpxProtocolMultiplexer::new(conn); + let limits = + ProtocolIngressLimits::new(2).with_max_buffered_bytes(2).with_max_buffered_messages(1); + mux.install_protocol_with_limits(&TestProtoMessage::capability(), limits, |conn| { + DrainingProtocol { conn } + }) + .unwrap(); + let mut stream = mux + .into_satellite_stream(eth.capability().as_ref(), |proxy| BurstingPrimary { + proxy, + remaining: 3, + }) + .unwrap(); + stream.inner.conn.start_send_unpin(Bytes::from_static(&[0, 0])).unwrap(); + stream.inner.conn.start_send_unpin(Bytes::from_static(&[0, 0])).unwrap(); + stream.inner.out_buffer.max_bytes = 1; + stream.inner.out_buffer.push_back(Bytes::from_static(&[0, 0])); + + let mut cx = Context::from_waker(noop_waker_ref()); + assert!(Pin::new(&mut stream).poll_next(&mut cx).is_pending()); + assert_eq!(stream.inner.conn.inner().frames.len(), 2); + + assert!(Pin::new(&mut stream).poll_next(&mut cx).is_pending()); + assert_eq!(stream.inner.conn.inner().frames.len(), 2); + + stream.inner.conn.inner().writable.store(true, Ordering::Relaxed); + assert!(Pin::new(&mut stream).poll_next(&mut cx).is_pending()); + assert_eq!(stream.inner.conn.inner().frames.len(), 2); + + assert!(matches!(Pin::new(&mut stream).poll_next(&mut cx), Poll::Ready(None))); + assert_eq!(stream.inner.conn.inner().frames.len(), 0); + } + + #[tokio::test] + async fn satellite_is_polled_between_frames_during_primary_handshake() { + let (mut mux, eth, _) = test_multiplexer(2); + let limits = + ProtocolIngressLimits::new(2).with_max_buffered_bytes(2).with_max_buffered_messages(1); + let (drained_tx, drained_rx) = oneshot::channel(); + mux.install_protocol_with_limits(&TestProtoMessage::capability(), limits, |conn| { + SignalingDrainingProtocol { conn, drained: Some(drained_tx) } + }) + .unwrap(); + + let stream = mux + .into_satellite_stream_with_handshake(eth.capability().as_ref(), async move |proxy| { + drained_rx.await.unwrap(); + Ok::<_, P2PStreamError>(PendingPrimary { _proxy: proxy }) + }) + .await; + + assert!(stream.is_ok()); + } + #[tokio::test] async fn satellite_mux_stops_polling_protocols_when_out_buffer_is_full() { let (hello, _) = test_hello(); diff --git a/crates/net/eth-wire/src/p2pstream.rs b/crates/net/eth-wire/src/p2pstream.rs index 21b87dfcf76..721102a0ed1 100644 --- a/crates/net/eth-wire/src/p2pstream.rs +++ b/crates/net/eth-wire/src/p2pstream.rs @@ -3,6 +3,7 @@ use crate::{ disconnect::CanDisconnect, errors::{P2PHandshakeError, P2PStreamError}, pinger::{Pinger, PingerEvent}, + protocol::ProtocolIngressLimits, DisconnectReason, HelloMessage, HelloMessageWithProtocols, }; use alloy_primitives::{ @@ -284,6 +285,9 @@ pub struct P2PStream { /// The supported capability for this stream. shared_capabilities: SharedCapabilities, + /// Explicit frame limits registered by installed subprotocol handlers. + inbound_protocol_limits: Vec, + /// Outgoing messages buffered for sending to the underlying stream. outgoing_messages: VecDeque, @@ -316,6 +320,7 @@ impl P2PStream { pinger: Pinger::new(PING_INTERVAL, PING_TIMEOUT), ping_token_bucket: PingTokenBucket::new(Instant::now()), shared_capabilities, + inbound_protocol_limits: Vec::new(), outgoing_messages: VecDeque::new(), outgoing_message_buffer_capacity: MAX_P2P_CAPACITY, disconnecting: false, @@ -347,6 +352,40 @@ impl P2PStream { &self.shared_capabilities } + pub(crate) fn set_protocol_ingress_limits( + &mut self, + capability: &crate::capability::SharedCapability, + limits: ProtocolIngressLimits, + ) { + let Some(max_frame_bytes) = limits.max_frame_bytes() else { return }; + if capability.num_messages() == 0 { + return + } + + let start = capability.message_id_offset(); + let end = u16::from(start) + u16::from(capability.num_messages()); + let configured = InboundProtocolLimit { + capability: capability.capability().into_owned(), + start, + end, + max_frame_bytes, + }; + + if let Some(current) = self + .inbound_protocol_limits + .iter_mut() + .find(|current| current.capability == configured.capability) + { + *current = configured; + } else { + self.inbound_protocol_limits.push(configured); + } + } + + fn inbound_protocol_limit(&self, message_id: u8) -> Option<&InboundProtocolLimit> { + self.inbound_protocol_limits.iter().find(|limit| limit.contains(message_id)) + } + /// Returns `true` if the stream has outgoing capacity. fn has_outgoing_capacity(&self) -> bool { self.outgoing_messages.len() < self.outgoing_message_buffer_capacity @@ -398,6 +437,20 @@ impl PingTokenBucket { } } +#[derive(Debug)] +struct InboundProtocolLimit { + capability: crate::Capability, + start: u8, + end: u16, + max_frame_bytes: usize, +} + +impl InboundProtocolLimit { + const fn contains(&self, message_id: u8) -> bool { + message_id >= self.start && (message_id as u16) < self.end + } +} + /// Gracefully disconnects the connection by sending a disconnect message and stop reading new /// messages. pub trait DisconnectP2P { @@ -561,6 +614,11 @@ where continue } + if id > MAX_RESERVED_MESSAGE_ID && this.shared_capabilities.find_by_offset(id).is_none() + { + return Poll::Ready(Some(Err(P2PStreamError::UnknownSubprotocolMessageId(id)))) + } + // first check that the compressed message length does not exceed the max // payload size let decompressed_len = snap::raw::decompress_len(&bytes[1..])?; @@ -571,9 +629,21 @@ where }))) } + let frame_len = decompressed_len + 1; + if let Some(limit) = this.inbound_protocol_limit(id) && + frame_len > limit.max_frame_bytes + { + counter!("p2pstream.subprotocol_message_too_big").increment(1); + return Poll::Ready(Some(Err(P2PStreamError::SubprotocolMessageTooBig { + capability: limit.capability.clone(), + message_size: frame_len, + max_size: limit.max_frame_bytes, + }))) + } + // create a buffer to hold the decompressed message, adding a byte to the length for // the message ID byte, which is the first byte in this buffer - let mut decompress_buf = BytesMut::zeroed(decompressed_len + 1); + let mut decompress_buf = BytesMut::zeroed(frame_len); // each message following a successful handshake is compressed with snappy, so we need // to decompress the message before we can decode it. @@ -943,8 +1013,8 @@ fn compress_frame( mod tests { use super::*; use crate::{ - capability::SharedCapability, test_utils::eth_hello, Capability, EthVersion, - ProtocolVersion, + capability::SharedCapability, protocol::Protocol, test_utils::eth_hello, Capability, + EthVersion, ProtocolVersion, }; use futures::task::noop_waker_ref; use tokio::net::{TcpListener, TcpStream}; @@ -958,6 +1028,39 @@ mod tests { flushes: usize, } + #[derive(Default)] + struct InboundTransport { + incoming: VecDeque, + } + + impl Stream for InboundTransport { + type Item = io::Result; + + fn poll_next(mut self: Pin<&mut Self>, _: &mut Context<'_>) -> Poll> { + Poll::Ready(self.incoming.pop_front().map(Ok)) + } + } + + impl Sink for InboundTransport { + type Error = io::Error; + + fn poll_ready(self: Pin<&mut Self>, _: &mut Context<'_>) -> Poll> { + Poll::Ready(Ok(())) + } + + fn start_send(self: Pin<&mut Self>, _item: Bytes) -> Result<(), Self::Error> { + Ok(()) + } + + fn poll_flush(self: Pin<&mut Self>, _: &mut Context<'_>) -> Poll> { + Poll::Ready(Ok(())) + } + + fn poll_close(self: Pin<&mut Self>, _: &mut Context<'_>) -> Poll> { + Poll::Ready(Ok(())) + } + } + impl Stream for FlushCountingTransport { type Item = io::Result; @@ -1131,6 +1234,74 @@ mod tests { assert_eq!(stream.outgoing_messages.len(), usize::from(PING_TOKEN_BUCKET_CAPACITY)); } + #[tokio::test] + async fn rejects_subprotocol_frame_before_decompression_when_declared_size_exceeds_limit() { + let cap = Capability::new_static("test", 1); + let shared_capabilities = + SharedCapabilities::try_new(vec![Protocol::new(cap.clone(), 1)], vec![cap.clone()]) + .unwrap(); + let shared_capability = shared_capabilities.find(&cap).unwrap().clone(); + let wire_id = shared_capability.message_id_offset(); + let mut encoder = snap::raw::Encoder::new(); + let mut scratch = Vec::new(); + let accepted = compress_frame(&mut encoder, &mut scratch, wire_id, &[0; 3]).unwrap(); + let oversized = compress_frame(&mut encoder, &mut scratch, wire_id, &[0; 4]).unwrap(); + let transport = InboundTransport { + incoming: [accepted, oversized] + .into_iter() + .map(|frame| BytesMut::from(frame.as_ref())) + .collect(), + }; + let mut stream = P2PStream::new(transport, shared_capabilities); + stream.set_protocol_ingress_limits(&shared_capability, ProtocolIngressLimits::new(4)); + + let frame = stream.next().await.unwrap().unwrap(); + assert_eq!(frame.len(), 4); + + assert!(matches!( + stream.next().await.unwrap().unwrap_err(), + P2PStreamError::SubprotocolMessageTooBig { + capability, + message_size: 5, + max_size: 4, + } if capability == cap + )); + } + + #[tokio::test] + async fn zero_message_protocol_does_not_replace_neighboring_frame_limit() { + let zero = Capability::new_static("aaa", 1); + let limited = Capability::new_static("bbb", 1); + let shared_capabilities = SharedCapabilities::try_new( + vec![Protocol::new(zero.clone(), 0), Protocol::new(limited.clone(), 1)], + vec![zero.clone(), limited.clone()], + ) + .unwrap(); + let zero_shared = shared_capabilities.find(&zero).unwrap().clone(); + let limited_shared = shared_capabilities.find(&limited).unwrap().clone(); + assert_eq!(zero_shared.message_id_offset(), limited_shared.message_id_offset()); + + let mut encoder = snap::raw::Encoder::new(); + let mut scratch = Vec::new(); + let oversized = + compress_frame(&mut encoder, &mut scratch, limited_shared.message_id_offset(), &[0; 4]) + .unwrap(); + let transport = + InboundTransport { incoming: VecDeque::from([BytesMut::from(oversized.as_ref())]) }; + let mut stream = P2PStream::new(transport, shared_capabilities); + stream.set_protocol_ingress_limits(&limited_shared, ProtocolIngressLimits::new(4)); + stream.set_protocol_ingress_limits(&zero_shared, ProtocolIngressLimits::new(1)); + + assert!(matches!( + stream.next().await.unwrap().unwrap_err(), + P2PStreamError::SubprotocolMessageTooBig { + capability, + message_size: 5, + max_size: 4, + } if capability == limited + )); + } + #[tokio::test] async fn poll_ready_drains_full_subprotocol_queue_without_flushing_inner() { let mut stream = diff --git a/crates/net/eth-wire/src/protocol.rs b/crates/net/eth-wire/src/protocol.rs index acb6dae09ee..cd4bd2fd413 100644 --- a/crates/net/eth-wire/src/protocol.rs +++ b/crates/net/eth-wire/src/protocol.rs @@ -69,6 +69,87 @@ impl Protocol { } } +/// Local limits for inbound messages of an `RLPx` subprotocol. +/// +/// These limits are not advertised to the remote peer and do not affect message ID negotiation. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct ProtocolIngressLimits { + max_frame_bytes: Option, + max_buffered_bytes: usize, + max_buffered_messages: usize, +} + +impl ProtocolIngressLimits { + /// Default byte budget for messages waiting to be polled by a protocol connection. + pub const DEFAULT_MAX_BUFFERED_BYTES: usize = 32 * 1024 * 1024; + + /// Default message budget for messages waiting to be polled by a protocol connection. + pub const DEFAULT_MAX_BUFFERED_MESSAGES: usize = 1024; + + /// Creates limits with an explicit maximum inbound frame size. + /// + /// The frame size includes the capability-local message ID byte. + /// + /// # Panics + /// + /// Panics if `max_frame_bytes` is zero. + pub const fn new(max_frame_bytes: usize) -> Self { + assert!(max_frame_bytes > 0, "maximum frame size must be non-zero"); + Self { max_frame_bytes: Some(max_frame_bytes), ..Self::default_values() } + } + + /// Sets the maximum number of buffered frame bytes. + /// + /// # Panics + /// + /// Panics if `max_buffered_bytes` is zero. + pub const fn with_max_buffered_bytes(mut self, max_buffered_bytes: usize) -> Self { + assert!(max_buffered_bytes > 0, "maximum buffered bytes must be non-zero"); + self.max_buffered_bytes = max_buffered_bytes; + self + } + + /// Sets the maximum number of buffered messages. + /// + /// # Panics + /// + /// Panics if `max_buffered_messages` is zero. + pub const fn with_max_buffered_messages(mut self, max_buffered_messages: usize) -> Self { + assert!(max_buffered_messages > 0, "maximum buffered messages must be non-zero"); + self.max_buffered_messages = max_buffered_messages; + self + } + + /// Returns the explicit maximum inbound frame size, if configured. + pub const fn max_frame_bytes(&self) -> Option { + self.max_frame_bytes + } + + /// Returns the maximum number of buffered frame bytes. + pub const fn max_buffered_bytes(&self) -> usize { + self.max_buffered_bytes + } + + /// Returns the maximum number of buffered messages. + pub const fn max_buffered_messages(&self) -> usize { + self.max_buffered_messages + } + + const fn default_values() -> Self { + Self { + max_frame_bytes: None, + max_buffered_bytes: Self::DEFAULT_MAX_BUFFERED_BYTES, + max_buffered_messages: Self::DEFAULT_MAX_BUFFERED_MESSAGES, + } + } +} + +impl Default for ProtocolIngressLimits { + fn default() -> Self { + Self::default_values() + } +} + impl From for Protocol { fn from(version: EthVersion) -> Self { Self::eth(version) diff --git a/crates/net/network/src/error.rs b/crates/net/network/src/error.rs index daf43d55713..ccb6cd9c770 100644 --- a/crates/net/network/src/error.rs +++ b/crates/net/network/src/error.rs @@ -148,6 +148,7 @@ impl SessionError for EthStreamError { DisconnectReason::ProtocolBreach ) ) | P2PStreamError::UnknownReservedMessageId(_) | + P2PStreamError::UnknownSubprotocolMessageId(_) | P2PStreamError::EmptyProtocolMessage | P2PStreamError::ParseSharedCapability(_) | P2PStreamError::CapabilityNotShared | @@ -220,14 +221,19 @@ impl SessionError for EthStreamError { Self::P2PStreamError( P2PStreamError::Rlp(_) | P2PStreamError::UnknownReservedMessageId(_) | + P2PStreamError::UnknownSubprotocolMessageId(_) | P2PStreamError::UnknownDisconnectReason(_) | P2PStreamError::MessageTooBig { .. } | + P2PStreamError::SubprotocolMessageTooBig { .. } | P2PStreamError::EmptyProtocolMessage | P2PStreamError::InvalidPingPongPayload(_) | P2PStreamError::TooManyPings | P2PStreamError::PingerError(_) | P2PStreamError::Snap(_), ) => Some(BackoffKind::Medium), + Self::P2PStreamError(P2PStreamError::SubprotocolInboundBufferFull { .. }) => { + Some(BackoffKind::Low) + } Self::EthHandshakeError(EthHandshakeError::InvalidFork(_)) => { // the remote can come back online after updating client version, so we can back off // for a bit @@ -353,6 +359,31 @@ mod tests { assert_eq!(err.should_backoff(), Some(BackoffKind::Medium)); } + #[test] + fn subprotocol_ingress_errors_have_distinct_reputation_outcomes() { + let capability = reth_eth_wire::Capability::new_static("test", 1); + let unknown = + EthStreamError::P2PStreamError(P2PStreamError::UnknownSubprotocolMessageId(0xff)); + assert!(unknown.is_protocol_breach()); + assert!(unknown.is_fatal_protocol_error()); + + let oversized = EthStreamError::P2PStreamError(P2PStreamError::SubprotocolMessageTooBig { + capability: capability.clone(), + message_size: 5, + max_size: 4, + }); + assert!(!oversized.is_protocol_breach()); + assert!(!oversized.is_fatal_protocol_error()); + assert_eq!(oversized.should_backoff(), Some(BackoffKind::Medium)); + + let full = EthStreamError::P2PStreamError(P2PStreamError::SubprotocolInboundBufferFull { + capability, + }); + assert!(!full.is_protocol_breach()); + assert!(!full.is_fatal_protocol_error()); + assert_eq!(full.should_backoff(), Some(BackoffKind::Low)); + } + #[test] fn test_address_in_use_message() { let addr = SocketAddr::V4(SocketAddrV4::new(Ipv4Addr::new(127, 0, 0, 1), 1234)); diff --git a/crates/net/network/src/protocol.rs b/crates/net/network/src/protocol.rs index 6479ac77e5b..09dc804131b 100644 --- a/crates/net/network/src/protocol.rs +++ b/crates/net/network/src/protocol.rs @@ -5,7 +5,9 @@ use alloy_primitives::bytes::BytesMut; use futures::Stream; use reth_eth_wire::{ - capability::SharedCapabilities, multiplex::ProtocolConnection, protocol::Protocol, + capability::SharedCapabilities, + multiplex::ProtocolConnection, + protocol::{Protocol, ProtocolIngressLimits}, }; use reth_network_api::{Direction, PeerId}; use std::{ @@ -50,6 +52,11 @@ pub trait ConnectionHandler: Send + Sync + 'static { /// This will be negotiated with the remote peer. fn protocol(&self) -> Protocol; + /// Returns local resource limits for inbound messages of this protocol. + fn inbound_limits(&self) -> ProtocolIngressLimits { + ProtocolIngressLimits::default() + } + /// Invoked when the `RLPx` connection has been established by the peer does not share the /// protocol. fn on_unsupported_by_peer( @@ -200,6 +207,8 @@ impl DynProtocolHandler for T { pub(crate) trait DynConnectionHandler: Send + Sync + 'static { fn protocol(&self) -> Protocol; + fn inbound_limits(&self) -> ProtocolIngressLimits; + fn on_unsupported_by_peer( self: Box, supported: &SharedCapabilities, @@ -220,6 +229,10 @@ impl DynConnectionHandler for T { T::protocol(self) } + fn inbound_limits(&self) -> ProtocolIngressLimits { + T::inbound_limits(self) + } + fn on_unsupported_by_peer( self: Box, supported: &SharedCapabilities, diff --git a/crates/net/network/src/session/mod.rs b/crates/net/network/src/session/mod.rs index 1dad5666baa..0111ab85c50 100644 --- a/crates/net/network/src/session/mod.rs +++ b/crates/net/network/src/session/mod.rs @@ -1244,14 +1244,15 @@ async fn authenticate_stream( // install additional handlers for handler in extra_handlers.into_iter() { - let cap = handler.protocol().cap; + let protocol = handler.protocol(); + let limits = handler.inbound_limits(); let remote_peer_id = authenticated_peer_id; // The unsupported-handler pass above guarantees that every remaining handler has a // matching negotiated capability. The multiplexer retains the same immutable set of // shared capabilities, so installing one of these handlers cannot fail. multiplex_stream - .install_protocol(&cap, move |conn| { + .install_protocol_with_limits(&protocol.cap, limits, move |conn| { handler.into_connection(direction, remote_peer_id, conn) }) .expect("remaining handler capability was negotiated"); From 8b32854d812d7a5edc7b421b1c5a43a4174a8c78 Mon Sep 17 00:00:00 2001 From: jihoon1125 <67624104+jihoon1125@users.noreply.github.com> Date: Fri, 14 Aug 2026 02:24:05 +0000 Subject: [PATCH 028/294] docs(engine): clarify BlockStatus::Valid does not imply canonical (#26688) --- crates/engine/tree/src/tree/mod.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/crates/engine/tree/src/tree/mod.rs b/crates/engine/tree/src/tree/mod.rs index f0bb957fb3f..ef141728faa 100644 --- a/crates/engine/tree/src/tree/mod.rs +++ b/crates/engine/tree/src/tree/mod.rs @@ -3667,7 +3667,11 @@ impl Drop for PayloadBuildLease { /// is valid or not. #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub enum BlockStatus { - /// The block is valid and block extends canonical chain. + /// The block is valid: its parent state was available, so it was executed and inserted into + /// the tree. + /// + /// Note: this does not imply the block extends the canonical chain. Blocks on a fork are + /// executed and inserted the same way and report this status as well. Valid, /// The block may be valid and has an unknown missing ancestor. Disconnected { From c6e4cbf207f0d07931f4d3204eb4ad490f257831 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Fri, 14 Aug 2026 11:03:39 +0000 Subject: [PATCH 029/294] fix(engine): stop prewarm workers on terminate signal (#26703) --- .../src/tree/payload_processor/prewarm.rs | 73 +++++++++++++++++-- 1 file changed, 67 insertions(+), 6 deletions(-) diff --git a/crates/engine/tree/src/tree/payload_processor/prewarm.rs b/crates/engine/tree/src/tree/payload_processor/prewarm.rs index 9de0c0a250f..4ac2174d1c0 100644 --- a/crates/engine/tree/src/tree/payload_processor/prewarm.rs +++ b/crates/engine/tree/src/tree/payload_processor/prewarm.rs @@ -487,6 +487,9 @@ where } PrewarmTaskEvent::Terminate { execution_outcome, valid_block_rx } => { trace!(target: "engine::tree::payload_processor::prewarm", "Received termination signal"); + // `Terminate` can arrive without `TerminateTransactionExecution` when the + // handle is dropped on an execution error, so stop workers before waiting. + self.ctx.stop(); final_execution_outcome = Some(execution_outcome.map(|outcome| (outcome, valid_block_rx))); @@ -824,11 +827,58 @@ fn multiproof_targets_from_withdrawals(withdrawals: &[Withdrawal]) -> MultiProof #[cfg(test)] mod tests { use super::*; + use alloy_consensus::transaction::Recovered; use alloy_eip7928::{ AccountChanges, BalanceChange, BlockAccessIndex, CodeChange, NonceChange, SlotChanges, StorageChange, }; use alloy_primitives::{address, bytes}; + use reth_chainspec::ChainSpec; + use reth_ethereum_primitives::{EthPrimitives, TransactionSigned}; + use reth_evm::{execute::WithTxEnv, TxEnvFor}; + use reth_evm_ethereum::EthEvmConfig; + use reth_provider::test_utils::MockEthProvider; + use reth_storage_overlay::OverlayManager; + + #[test] + fn terminate_event_stops_transaction_execution() { + let terminate_execution = Arc::new(AtomicBool::new(false)); + let ctx = PrewarmContext { + env: ExecutionEnv::test_default(), + evm_config: EthEvmConfig::new(Arc::new(ChainSpec::default())), + saved_cache: None, + provider: StateProviderBuilder::::new( + MockEthProvider::default(), + B256::ZERO, + OverlayManager::default(), + ), + bal_prewarm_pool: None, + metrics: PrewarmMetrics::default(), + cache_metrics: None, + cache_state_metrics: None, + terminate_execution: Arc::clone(&terminate_execution), + executed_tx_index: Arc::new(AtomicUsize::new(0)), + precompile_cache_disabled: false, + precompile_cache_map: PrecompileCacheMap::default(), + disable_bal_parallel_state_root: false, + disable_bal_batch_io: false, + }; + let (task, actions_tx) = + PrewarmCacheTask::new(Runtime::test(), PayloadExecutionCache::default(), ctx); + actions_tx + .send(PrewarmTaskEvent::Terminate { + execution_outcome: None, + valid_block_rx: mpsc::channel().1, + }) + .unwrap(); + + task.run::, Recovered>>( + PrewarmMode::Skipped, + actions_tx, + ); + + assert!(terminate_execution.load(Ordering::Relaxed)); + } #[test] fn bal_read_only_account_does_not_change_state_root() { @@ -888,13 +938,23 @@ mod tests { /// execution path without cloning the expensive `BundleState`. #[derive(Debug)] pub enum PrewarmTaskEvent { - /// Forcefully terminate all remaining transaction execution. + /// Signals the prewarm workers to stop executing further transactions. + /// + /// This only sets the termination flag the workers poll; the task keeps running to save the + /// cache. Sent once the authoritative execution no longer needs prewarming, so the workers do + /// not race ahead on transactions that will never be used. TerminateTransactionExecution, - /// Forcefully terminate the task on demand and update the shared cache with the given output - /// before exiting. + /// Tears the whole task down: stops the workers, optionally saves the warmed cache from the + /// final output, and exits. + /// + /// Sent when execution completed successfully (carrying the output to save) or when the task + /// handle is dropped (carrying no output, e.g. after an execution error). Handling this event + /// also stops the workers, since a teardown may arrive without a preceding + /// [`TerminateTransactionExecution`](Self::TerminateTransactionExecution). Terminate { - /// The final execution outcome. Using `Arc` allows sharing with the main execution - /// path without cloning the expensive `BundleState`. + /// The final execution outcome, or `None` when the task is torn down without one (e.g. a + /// dropped handle). Using `Arc` allows sharing with the main execution path without + /// cloning the expensive `BundleState`. execution_outcome: Option>>, /// Receiver for the block validation result. /// @@ -902,7 +962,8 @@ pub enum PrewarmTaskEvent { /// updated cache but only save it once we know the block is valid. valid_block_rx: mpsc::Receiver<()>, }, - /// Finished executing all transactions + /// Emitted by the worker-dispatch side once every dispatched transaction has finished or been + /// cancelled, reporting how many were executed. FinishedTxExecution { /// Number of transactions executed executed_transactions: usize, From 1bd905027e45752cdb60a82560e1f69eee58264e Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Fri, 14 Aug 2026 12:04:34 +0000 Subject: [PATCH 030/294] fix(rpc): add Amsterdam system contracts to eth_config (#26705) --- crates/rpc/rpc-eth-api/src/helpers/config.rs | 31 +++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/crates/rpc/rpc-eth-api/src/helpers/config.rs b/crates/rpc/rpc-eth-api/src/helpers/config.rs index 860d0af65f8..18c5c65f315 100644 --- a/crates/rpc/rpc-eth-api/src/helpers/config.rs +++ b/crates/rpc/rpc-eth-api/src/helpers/config.rs @@ -6,7 +6,7 @@ use alloy_eips::{ eip7910::{EthConfig, EthForkConfig, SystemContract}, }; use alloy_evm::precompiles::Precompile; -use alloy_primitives::Address; +use alloy_primitives::{address, Address}; use jsonrpsee::{core::RpcResult, proc_macros::rpc}; use reth_chainspec::{ChainSpecProvider, EthChainSpec, EthereumHardforks, Hardforks, Head}; use reth_errors::{ProviderError, RethError}; @@ -67,6 +67,10 @@ where .extend(SystemContract::prague(chain_spec.deposit_contract().map(|c| c.address))); } + if chain_spec.is_amsterdam_active_at_timestamp(timestamp) { + system_contracts.extend(amsterdam_system_contracts()); + } + // Fork config only exists for timestamp-based hardforks. let fork_id = chain_spec .fork_id(&Head { timestamp, number: u64::MAX, ..Default::default() }) @@ -169,6 +173,31 @@ where } } +/// Address of the builder deposit system contract introduced in Amsterdam +/// ([EIP-8282](https://eips.ethereum.org/EIPS/eip-8282)). +const BUILDER_DEPOSIT_CONTRACT_ADDRESS: Address = + address!("0x0000BFF46984E3725691FA540A8C7589300D8282"); + +/// Address of the builder exit system contract introduced in Amsterdam +/// ([EIP-8282](https://eips.ethereum.org/EIPS/eip-8282)). +const BUILDER_EXIT_CONTRACT_ADDRESS: Address = + address!("0x000064D678505AD48F8CCB093BC65613800E8282"); + +/// Returns the system contracts introduced in the Amsterdam hardfork +/// ([EIP-8282](https://eips.ethereum.org/EIPS/eip-8282) builder execution requests). +fn amsterdam_system_contracts() -> [(SystemContract, Address); 2] { + [ + ( + SystemContract::Other("BUILDER_DEPOSIT_CONTRACT_ADDRESS".to_string()), + BUILDER_DEPOSIT_CONTRACT_ADDRESS, + ), + ( + SystemContract::Other("BUILDER_EXIT_CONTRACT_ADDRESS".to_string()), + BUILDER_EXIT_CONTRACT_ADDRESS, + ), + ] +} + fn evm_to_precompiles_map( evm: impl Evm, ) -> BTreeMap { From bbdaad9a157c69a2099674569af6d384e921a558 Mon Sep 17 00:00:00 2001 From: Alexey Shekhirin Date: Fri, 14 Aug 2026 14:38:46 +0000 Subject: [PATCH 031/294] fix(trie): construct witness from depth-first ordered nodes (#26707) --- crates/trie/common/src/proofs.rs | 67 +++++++++++++++++++++++++++++++- 1 file changed, 65 insertions(+), 2 deletions(-) diff --git a/crates/trie/common/src/proofs.rs b/crates/trie/common/src/proofs.rs index e1b65bc15c2..bb8f17094a7 100644 --- a/crates/trie/common/src/proofs.rs +++ b/crates/trie/common/src/proofs.rs @@ -531,12 +531,12 @@ impl DecodedMultiProofV2 { } } - account_nodes.sort_by(|(a, _, _), (b, _, _)| b.cmp(a)); + account_nodes.sort_by(|(a, _, _), (b, _, _)| crate::depth_first_cmp(a, b)); let account_proofs = ProofTrieNodeV2::from_sorted_trie_nodes(account_nodes); let mut storage_proofs = B256Map::default(); for (account, mut nodes) in storage_nodes { - nodes.sort_by(|(a, _, _), (b, _, _)| b.cmp(a)); + nodes.sort_by(|(a, _, _), (b, _, _)| crate::depth_first_cmp(a, b)); storage_proofs.insert(account, ProofTrieNodeV2::from_sorted_trie_nodes(nodes)); } @@ -1062,6 +1062,69 @@ pub mod triehash { #[cfg(test)] mod tests { use super::*; + use alloy_trie::{ + nodes::{BranchNode, LeafNode, RlpNode}, + TrieMask, + }; + + #[test] + fn witness_nodes_are_depth_first_ordered() { + fn insert_node(witness: &mut B256Map, node: impl alloy_rlp::Encodable) -> RlpNode { + let encoded = alloy_rlp::encode(node); + witness.insert(keccak256(&encoded), encoded.clone().into()); + RlpNode::from_rlp(&encoded) + } + + let mut witness = B256Map::default(); + let leaf_key = Nibbles::from_nibbles([0; 63]); + + let storage_leaf_0 = insert_node( + &mut witness, + LeafNode::new(leaf_key, encode_fixed_size(&U256::from(1)).to_vec()), + ); + let storage_leaf_1 = insert_node( + &mut witness, + LeafNode::new(leaf_key, encode_fixed_size(&U256::from(2)).to_vec()), + ); + let storage_root = insert_node( + &mut witness, + BranchNode::new(vec![storage_leaf_0, storage_leaf_1], TrieMask::new(0b11)), + ); + + let account_leaf_0 = insert_node( + &mut witness, + LeafNode::new( + leaf_key, + alloy_rlp::encode(TrieAccount { + storage_root: storage_root.as_hash().expect("storage root is hashed"), + ..Default::default() + }), + ), + ); + let account_leaf_1 = insert_node( + &mut witness, + LeafNode::new(leaf_key, alloy_rlp::encode(TrieAccount::default())), + ); + let state_root = insert_node( + &mut witness, + BranchNode::new(vec![account_leaf_0, account_leaf_1], TrieMask::new(0b11)), + ) + .as_hash() + .expect("state root is hashed"); + + let proof = DecodedMultiProofV2::from_witness(state_root, &witness).unwrap(); + let expected_paths = + [Nibbles::from_nibbles([0]), Nibbles::from_nibbles([1]), Nibbles::default()]; + + assert_eq!( + proof.account_proofs.iter().map(|node| node.path).collect::>(), + expected_paths + ); + assert_eq!( + proof.storage_proofs[&B256::ZERO].iter().map(|node| node.path).collect::>(), + expected_paths + ); + } #[test] fn test_multiproof_extend_account_proofs() { From 3e0847e1f688c4a7cae6460fd23c0896df39301e Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Fri, 14 Aug 2026 14:51:08 +0000 Subject: [PATCH 032/294] perf(engine): reuse scratch buffer for BAL hash encoding (#26701) --- crates/engine/tree/src/tree/payload_validator.rs | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/crates/engine/tree/src/tree/payload_validator.rs b/crates/engine/tree/src/tree/payload_validator.rs index 9c4dde82d67..65300c02964 100644 --- a/crates/engine/tree/src/tree/payload_validator.rs +++ b/crates/engine/tree/src/tree/payload_validator.rs @@ -106,7 +106,7 @@ use crate::tree::{ PayloadHandle, StateProviderBuilder, StateProviderDatabase, TreeConfig, WaitForCaches, }; use alloy_consensus::transaction::{Either, TxHashRef}; -use alloy_eip7928::{bal::DecodedBal, compute_block_access_list_hash, BlockAccessList}; +use alloy_eip7928::{bal::DecodedBal, compute_block_access_list_hash_with_buf, BlockAccessList}; use alloy_eips::{eip1898::BlockWithParent, eip4895::Withdrawal, NumHash}; use alloy_evm::Evm; use alloy_primitives::{ @@ -295,6 +295,8 @@ where /// None if txpool prewarming is disabled. #[debug(skip)] txpool_prewarm: Option>, + /// Scratch buffer reused for BAL hash encoding across validated blocks. + bal_hash_buf: Vec, } impl BasicEngineValidator @@ -354,6 +356,7 @@ where overlay_manager, state_root_strategy: Arc::new(DefaultStateRootStrategy::default()), txpool_prewarm: None, + bal_hash_buf: Vec::new(), } } @@ -1303,7 +1306,7 @@ where /// The `hashed_state` handle wraps the background hashed post state computation. #[instrument(level = "debug", target = "engine::tree::payload_validator", skip_all)] fn validate_post_execution>>( - &self, + &mut self, block: &RecoveredBlock, parent_block: &SealedHeader, output: &BlockExecutionOutput, @@ -1322,8 +1325,9 @@ where let _enter = debug_span!(target: "engine::tree::payload_validator", "validate_block_post_execution") .entered(); - let block_access_list_hash = - built_bal.as_ref().map(|bal| compute_block_access_list_hash(bal)); + let block_access_list_hash = built_bal + .as_ref() + .map(|bal| compute_block_access_list_hash_with_buf(bal, &mut self.bal_hash_buf)); if let Err(err) = self.consensus.validate_block_post_execution( block, From 94cf8fdaf37edb94c72c27b920f28b65dba9e940 Mon Sep 17 00:00:00 2001 From: spencer Date: Fri, 14 Aug 2026 15:31:25 +0000 Subject: [PATCH 033/294] fix(engine): reject undecodable block access list bytes as invalid params (#26694) Co-authored-by: Matthias Seitz --- Cargo.lock | 2 + crates/consensus/consensus/Cargo.toml | 2 + crates/consensus/consensus/src/lib.rs | 6 ++ crates/engine/primitives/Cargo.toml | 2 + crates/engine/primitives/src/error.rs | 71 ++++++++++++++++- crates/engine/tree/src/tree/mod.rs | 14 ++-- .../engine/tree/src/tree/payload_validator.rs | 5 +- .../node/tests/e2e/invalid_payload.rs | 79 ++++++++++++++++++- crates/rpc/rpc-engine-api/src/error.rs | 11 +++ crates/rpc/rpc/src/validation.rs | 3 +- 10 files changed, 177 insertions(+), 18 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e8a09a309f8..a9ca903a753 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8017,6 +8017,7 @@ dependencies = [ "alloy-consensus", "alloy-eip7928", "alloy-primitives", + "alloy-rlp", "auto_impl", "reth-execution-types", "reth-primitives-traits", @@ -8400,6 +8401,7 @@ dependencies = [ "alloy-consensus", "alloy-eips", "alloy-primitives", + "alloy-rlp", "alloy-rpc-types-engine", "auto_impl", "futures", diff --git a/crates/consensus/consensus/Cargo.toml b/crates/consensus/consensus/Cargo.toml index 5fa7c181525..4969a8538b7 100644 --- a/crates/consensus/consensus/Cargo.toml +++ b/crates/consensus/consensus/Cargo.toml @@ -18,6 +18,7 @@ reth-primitives-traits.workspace = true # ethereum alloy-primitives.workspace = true alloy-consensus.workspace = true +alloy-rlp.workspace = true alloy-eip7928.workspace = true # misc @@ -33,5 +34,6 @@ std = [ "reth-execution-types/std", "thiserror/std", "alloy-eip7928/std", + "alloy-rlp/std", ] test-utils = ["reth-primitives-traits/test-utils"] diff --git a/crates/consensus/consensus/src/lib.rs b/crates/consensus/consensus/src/lib.rs index 59ab0354b38..f9726ac45e3 100644 --- a/crates/consensus/consensus/src/lib.rs +++ b/crates/consensus/consensus/src/lib.rs @@ -498,6 +498,12 @@ pub enum ConsensusError { /// EIP-7928: Error when the block access list hash doesn't match the expected value. #[error("block access list hash mismatch: {0}")] BlockAccessListHashMismatch(GotExpectedBoxed), + /// EIP-7928: Error when the raw block access list bytes cannot be RLP-decoded. + /// + /// Note: this is not a block validity error, the received block access list bytes themselves + /// are malformed request params. + #[error("failed to decode block access list: {0}")] + BlockAccessListDecode(alloy_rlp::Error), /// EIP-7928: Error when the block access list cannot be decoded or converted. #[error("invalid block access list: {0}")] BlockAccessListInvalid(String), diff --git a/crates/engine/primitives/Cargo.toml b/crates/engine/primitives/Cargo.toml index e73046c027f..c1a55fed607 100644 --- a/crates/engine/primitives/Cargo.toml +++ b/crates/engine/primitives/Cargo.toml @@ -41,6 +41,7 @@ thiserror.workspace = true [dev-dependencies] alloy-primitives = { workspace = true, features = ["getrandom"] } +alloy-rlp.workspace = true [features] default = ["std"] @@ -63,4 +64,5 @@ std = [ "serde/std", "thiserror/std", "reth-evm?/std", + "alloy-rlp/std", ] diff --git a/crates/engine/primitives/src/error.rs b/crates/engine/primitives/src/error.rs index def17707fb9..103b1671cfe 100644 --- a/crates/engine/primitives/src/error.rs +++ b/crates/engine/primitives/src/error.rs @@ -12,6 +12,12 @@ pub enum BeaconOnNewPayloadError { /// Thrown when the engine task is unavailable/stopped. #[error("beacon consensus engine task stopped")] EngineUnavailable, + /// Thrown when the payload params are malformed, e.g. a field's raw bytes cannot be decoded. + /// + /// Per the engine API spec this must be rejected with an invalid params error instead of an + /// `INVALID` payload status. + #[error(transparent)] + InvalidParams(Box), /// An internal error occurred, not necessarily related to the payload. #[error(transparent)] Internal(Box), @@ -24,6 +30,15 @@ impl BeaconOnNewPayloadError { } } +impl From for BeaconOnNewPayloadError { + fn from(error: InsertBlockFatalError) -> Self { + match error { + InsertBlockFatalError::InvalidParams(err) => Self::InvalidParams(err), + error => Self::internal(error), + } + } +} + /// Represents error cases for an applied forkchoice update. /// /// This represents all possible error cases, that must be returned as JSON RPC errors back to the @@ -73,15 +88,24 @@ impl InsertBlockErrorKind { /// Returns an [`InsertBlockValidationError`] if the error is caused by an invalid block. /// - /// Returns an [`InsertBlockFatalError`] if the error is caused by an error that is not - /// validation related or is otherwise fatal. + /// Returns an [`InsertBlockFatalError`] if the failure is not attributable to the block + /// itself, either an internal error or malformed request params. /// - /// This is intended to be used to determine if we should respond `INVALID` as a response when - /// processing a new block. + /// This split decides how `newPayload` responds: validation errors become an `INVALID` + /// payload status and mark the block hash as invalid, while fatal errors are returned as + /// actual errors to the caller. This distinction is required because responding `INVALID` + /// has consensus meaning (the block is rejected and its hash cached as invalid), which must + /// not happen for failures the block is not responsible for. pub fn ensure_validation_error( self, ) -> Result { match self { + // Undecodable block access list bytes are malformed request params, not an invalid + // block, and must be rejected with an invalid params error instead of an `INVALID` + // payload status. + Self::Consensus(ConsensusError::BlockAccessListDecode(err)) => { + Err(InsertBlockFatalError::InvalidParams(Box::new(err))) + } Self::Consensus(err) => Ok(InsertBlockValidationError::Consensus(err)), Self::Execution(err) => match err { BlockExecutionError::Validation(err) => { @@ -98,6 +122,17 @@ impl InsertBlockErrorKind { } /// Error variants that are not caused by invalid blocks. +/// +/// "Fatal" means block processing failed for a reason other than the block itself being invalid. +/// This includes errors caused by additional payload data that is not part of the block, such as +/// undecodable block access list bytes: their malformation says nothing about the validity of +/// the block and is therefore treated differently with respect to the `PayloadStatus`. +/// +/// These failures must not be answered with an `INVALID` payload status or mark the block hash +/// as invalid. Instead they are propagated as actual errors: for `newPayload` they convert into +/// [`BeaconOnNewPayloadError`] and are returned as a JSON-RPC error object instead of a +/// `PayloadStatus` result, [`Self::InvalidParams`] as invalid params (`-32602`) and all other +/// variants as internal error (`-32603`). #[derive(Debug, thiserror::Error)] pub enum InsertBlockFatalError { /// A provider error. @@ -106,6 +141,10 @@ pub enum InsertBlockFatalError { /// An internal or fatal block execution error. #[error(transparent)] BlockExecutionError(#[from] InternalBlockExecutionError), + /// The payload params are malformed, e.g. undecodable block access list bytes, and the + /// request must be rejected with an invalid params error. + #[error(transparent)] + InvalidParams(Box), } /// Error variants that are caused by invalid blocks. @@ -118,3 +157,27 @@ pub enum InsertBlockValidationError { #[error(transparent)] Validation(#[from] BlockValidationError), } + +#[cfg(test)] +mod tests { + use super::*; + + // Undecodable block access list bytes are malformed request params and must not be treated + // as a block validation error. + #[test] + fn bal_decode_error_is_invalid_params() { + let err = InsertBlockErrorKind::Consensus(ConsensusError::BlockAccessListDecode( + alloy_rlp::Error::UnexpectedString, + )); + assert!(matches!( + err.ensure_validation_error(), + Err(InsertBlockFatalError::InvalidParams(_)) + )); + assert!(matches!( + BeaconOnNewPayloadError::from(InsertBlockFatalError::InvalidParams(Box::new( + alloy_rlp::Error::UnexpectedString + ))), + BeaconOnNewPayloadError::InvalidParams(_) + )); + } +} diff --git a/crates/engine/tree/src/tree/mod.rs b/crates/engine/tree/src/tree/mod.rs index ef141728faa..1e48cb9da90 100644 --- a/crates/engine/tree/src/tree/mod.rs +++ b/crates/engine/tree/src/tree/mod.rs @@ -18,8 +18,8 @@ use reth_chain_state::{ }; use reth_consensus::{Consensus, FullConsensus}; use reth_engine_primitives::{ - BeaconEngineMessage, BeaconOnNewPayloadError, ConsensusEngineEvent, ExecutionPayload, - ForkchoiceStateTracker, NewPayloadTimings, OnForkChoiceUpdated, SlowBlockInfo, + BeaconEngineMessage, ConsensusEngineEvent, ExecutionPayload, ForkchoiceStateTracker, + NewPayloadTimings, OnForkChoiceUpdated, SlowBlockInfo, }; use reth_errors::{ConsensusError, ProviderResult}; use reth_evm::ConfigureEvm; @@ -1830,9 +1830,7 @@ where // emit response if let Err(err) = - tx.send(output.map(|o| o.outcome).map_err(|e| { - BeaconOnNewPayloadError::Internal(Box::new(e)) - })) + tx.send(output.map(|o| o.outcome).map_err(Into::into)) { warn!(target: "engine::tree", payload=?num_hash, elapsed=?start.elapsed(), "Failed to deliver newPayload response, receiver dropped (request cancelled): {err:?}"); self.metrics @@ -1916,10 +1914,8 @@ where .map(|wait| wait.execution_cache), sparse_trie_wait: cache_wait.map(|wait| wait.sparse_trie), }; - if let Err(err) = - tx.send(output.map(|o| (o.outcome, timings)).map_err(|e| { - BeaconOnNewPayloadError::Internal(Box::new(e)) - })) + if let Err(err) = tx + .send(output.map(|o| (o.outcome, timings)).map_err(Into::into)) { error!( target: "engine::tree", diff --git a/crates/engine/tree/src/tree/payload_validator.rs b/crates/engine/tree/src/tree/payload_validator.rs index 65300c02964..991a4963e2e 100644 --- a/crates/engine/tree/src/tree/payload_validator.rs +++ b/crates/engine/tree/src/tree/payload_validator.rs @@ -565,10 +565,11 @@ where .in_scope(|| self.evm_env_for(&input)) .map_err(NewPayloadError::other)?; - // Extract the decoded BAL, if valid and available. + // Extract the decoded BAL, if present. Undecodable block access list bytes are malformed + // request params, not an invalid block. let decoded_bal = ensure_ok!(input .try_decoded_access_list() - .map_err(|err| ConsensusError::BlockAccessListInvalid(err.to_string()))) + .map_err(ConsensusError::BlockAccessListDecode)) .map(Arc::new); if let Some(decoded_bal) = decoded_bal.as_deref() { diff --git a/crates/ethereum/node/tests/e2e/invalid_payload.rs b/crates/ethereum/node/tests/e2e/invalid_payload.rs index 7e830982155..7d9cc1110e7 100644 --- a/crates/ethereum/node/tests/e2e/invalid_payload.rs +++ b/crates/ethereum/node/tests/e2e/invalid_payload.rs @@ -3,9 +3,11 @@ //! This module tests the scenario where a node receives invalid payloads (e.g., with modified //! state roots) before receiving valid ones, ensuring the node can recover and continue. -use crate::utils::eth_payload_attributes; -use alloy_primitives::B256; +use crate::utils::{eth_payload_attributes, eth_payload_attributes_amsterdam}; +use alloy_eips::eip7685::RequestsOrHash; +use alloy_primitives::{keccak256, Bytes, B256}; use alloy_rpc_types_engine::{ExecutionPayloadV3, PayloadStatusEnum}; +use jsonrpsee::types::error::INVALID_PARAMS_CODE; use rand::{rngs::StdRng, Rng, SeedableRng}; use reth_chainspec::{ChainSpecBuilder, MAINNET}; use reth_e2e_test_utils::{setup_engine, transaction::TransactionTestContext}; @@ -355,3 +357,76 @@ async fn can_handle_invalid_payload_with_transactions() -> eyre::Result<()> { Ok(()) } + +/// Tests that `engine_newPayloadV5` rejects undecodable block access list bytes with an invalid +/// params error (`-32602`) instead of an `INVALID` payload status. +#[tokio::test] +async fn undecodable_bal_is_rejected_as_invalid_params() -> eyre::Result<()> { + reth_tracing::init_test_tracing(); + + let chain_spec = Arc::new( + ChainSpecBuilder::default() + .chain(MAINNET.chain) + .genesis(serde_json::from_str(include_str!("../assets/genesis.json")).unwrap()) + .amsterdam_activated() + .build(), + ); + + let (mut nodes, wallet) = setup_engine::( + 1, + chain_spec, + false, + Default::default(), + eth_payload_attributes_amsterdam, + ) + .await?; + let mut node = nodes.pop().unwrap(); + + // Build a valid Amsterdam payload without making it canonical. + let raw_tx = TransactionTestContext::transfer_tx_bytes(1, wallet.inner).await; + node.rpc.inject_tx(raw_tx).await?; + let payload = node.new_payload().await?; + let block = payload.block().clone(); + let envelope = payload.try_into_v6()?; + let valid_payload = envelope.execution_payload; + + // Corrupt the block access list bytes and recompute the block hash for the corresponding + // header, so the payload converts cleanly and the undecodable bytes are the only defect. + let garbage = Bytes::from_static(b"not-rlp"); + let mut header = block.header().clone(); + header.block_access_list_hash = Some(keccak256(&garbage)); + + let mut corrupted = valid_payload.clone(); + corrupted.block_access_list = garbage; + corrupted.payload_inner.payload_inner.payload_inner.block_hash = header.hash_slow(); + + let engine = node.auth_server_handle().http_client(); + let parent_beacon_block_root = block.header().parent_beacon_block_root.unwrap(); + let err = EngineApiClient::::new_payload_v5( + &engine, + corrupted, + vec![], + parent_beacon_block_root, + RequestsOrHash::Requests(envelope.execution_requests.clone()), + ) + .await + .unwrap_err(); + + let jsonrpsee::core::client::Error::Call(err) = err else { + panic!("expected an invalid params error object, got {err:?}") + }; + assert_eq!(err.code(), INVALID_PARAMS_CODE); + + // The same block with well-formed block access list bytes is processed normally. + let status = EngineApiClient::::new_payload_v5( + &engine, + valid_payload, + vec![], + parent_beacon_block_root, + RequestsOrHash::Requests(envelope.execution_requests), + ) + .await?; + assert!(matches!(status.status, PayloadStatusEnum::Valid)); + + Ok(()) +} diff --git a/crates/rpc/rpc-engine-api/src/error.rs b/crates/rpc/rpc-engine-api/src/error.rs index 3c7fd2c8a66..9b5709109ea 100644 --- a/crates/rpc/rpc-engine-api/src/error.rs +++ b/crates/rpc/rpc-engine-api/src/error.rs @@ -140,6 +140,7 @@ impl From for jsonrpsee_types::error::ErrorObject<'static> { ) } EngineApiError::InvalidBodiesRange { .. } | + EngineApiError::NewPayload(BeaconOnNewPayloadError::InvalidParams(_)) | EngineApiError::EngineObjectValidationError( EngineObjectValidationError::Payload(_) | EngineObjectValidationError::InvalidParams(_), @@ -301,6 +302,16 @@ mod tests { EngineApiError::UnknownPayload, ); + // Malformed payload params, e.g. undecodable block access list bytes, are rejected with + // an invalid params error instead of an `INVALID` payload status. + ensure_engine_rpc_error( + INVALID_PARAMS_CODE, + INVALID_PARAMS_MSG, + EngineApiError::NewPayload(BeaconOnNewPayloadError::InvalidParams( + "undecodable block access list".into(), + )), + ); + // Per the Shanghai Engine API spec, FCU V2 must return -38003 when the wrong // PayloadAttributes version is used. // Spec: https://github.com/ethereum/execution-apis/blob/main/src/engine/shanghai.md diff --git a/crates/rpc/rpc/src/validation.rs b/crates/rpc/rpc/src/validation.rs index 33751dec024..2d0a1136bbf 100644 --- a/crates/rpc/rpc/src/validation.rs +++ b/crates/rpc/rpc/src/validation.rs @@ -726,7 +726,8 @@ impl From for ErrorObject<'static> { ValidationApiError::Blob(_) => invalid_params_rpc_err(error.to_string()), ValidationApiError::Consensus( - error @ (ConsensusError::BlockAccessListCostMoreThanGasLimit(_) | + error @ (ConsensusError::BlockAccessListDecode(_) | + ConsensusError::BlockAccessListCostMoreThanGasLimit(_) | ConsensusError::BlockAccessListHashMismatch(_)), ) => invalid_params_rpc_err(error.to_string()), ValidationApiError::MissingLatestBlock | From 90ea46d0665408804e5fa24c6f05e06c09b232bb Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Fri, 14 Aug 2026 15:57:51 +0000 Subject: [PATCH 034/294] fix(engine): keep messages responsive during pending handoff (#26708) --- crates/engine/tree/src/tree/mod.rs | 23 ++++++++++------ crates/engine/tree/src/tree/tests.rs | 39 ++++++++++------------------ 2 files changed, 29 insertions(+), 33 deletions(-) diff --git a/crates/engine/tree/src/tree/mod.rs b/crates/engine/tree/src/tree/mod.rs index 1e48cb9da90..cc8c001f653 100644 --- a/crates/engine/tree/src/tree/mod.rs +++ b/crates/engine/tree/src/tree/mod.rs @@ -710,17 +710,24 @@ where /// Blocks until the next event that can safely be processed is ready. /// - /// A pending persisted handoff only accepts the notification that all active payload builds - /// have finished. This keeps queued engine messages from starting replacement payload jobs - /// before the handoff can reclaim the in-memory overlay. Otherwise, uses biased selection to - /// prioritize persistence completion to update in-memory state and unblock further writes. + /// A pending persisted handoff keeps processing engine messages while prioritizing the + /// notification that all active payload builds have finished. This keeps the engine responsive + /// without reclaiming the in-memory overlay while a payload job may still access it. Otherwise, + /// uses biased selection to prioritize persistence completion to update in-memory state and + /// unblock further writes. fn wait_for_event(&mut self) -> LoopEvent { if self.pending_persisted_handoff.is_some() { self.metrics.engine.backpressure_active.set(0.0); - return match self.payload_build_finished.recv() { - Ok(()) => LoopEvent::PayloadBuildFinished, - Err(_) => LoopEvent::Disconnected, - }; + return crossbeam_channel::select_biased! { + recv(self.payload_build_finished) -> result => match result { + Ok(()) => LoopEvent::PayloadBuildFinished, + Err(_) => LoopEvent::Disconnected, + }, + recv(self.incoming) -> msg => match msg { + Ok(m) => LoopEvent::EngineMessage(m), + Err(_) => LoopEvent::Disconnected, + }, + } } // Take ownership of persistence rx if present diff --git a/crates/engine/tree/src/tree/tests.rs b/crates/engine/tree/src/tree/tests.rs index 9fb6bb4086e..14421f40a05 100644 --- a/crates/engine/tree/src/tree/tests.rs +++ b/crates/engine/tree/src/tree/tests.rs @@ -709,7 +709,7 @@ fn persistence_handoff_waits_for_active_payload_jobs() { } #[test] -fn pending_persisted_handoff_blocks_engine_messages_until_payload_build_finishes() { +fn pending_persisted_handoff_keeps_engine_messages_responsive() { let blocks: Vec<_> = TestBlockBuilder::eth().get_executed_blocks(1..4).collect(); let mut test_harness = TestHarness::with_config( MAINNET.clone(), @@ -732,34 +732,23 @@ fn pending_persisted_handoff_blocks_engine_messages_until_payload_build_finishes .unwrap(); assert!(test_harness.tree.pending_persisted_handoff.is_some()); - // Forkchoice updates received while the handoff is pending could start replacement payload - // builds and keep the final completion notification from reaching the tree. Queue a generic - // engine message to prove the gate leaves every message upstream until then. + // Engine messages remain processable while the handoff waits for the active payload build. test_harness.to_tree_tx.send(FromEngine::Event(FromOrchestrator::BackfillSyncStarted)).unwrap(); + assert_matches!( + test_harness.tree.wait_for_event(), + super::LoopEvent::EngineMessage(FromEngine::Event(FromOrchestrator::BackfillSyncStarted)) + ); + assert!(test_harness.tree.pending_persisted_handoff.is_some()); + assert!(test_harness.tree.state.tree_state.contains_hash(&persisted.hash)); - let (started_tx, started_rx) = std::sync::mpsc::sync_channel(0); - let (event_tx, event_rx) = std::sync::mpsc::sync_channel(1); - std::thread::scope(|scope| { - let tree = &mut test_harness.tree; - scope.spawn(move || { - started_tx.send(()).unwrap(); - event_tx - .send(matches!(tree.wait_for_event(), super::LoopEvent::PayloadBuildFinished)) - .unwrap(); - }); - - started_rx.recv().unwrap(); - assert!( - event_rx.recv_timeout(Duration::from_millis(100)).is_err(), - "a pending handoff must prevent queued engine messages from being processed" - ); - - drop(payload_build); - assert_eq!(event_rx.recv_timeout(Duration::from_secs(1)), Ok(true)); - }); - + // Once the final lease drops, its completion is prioritized over queued engine messages so the + // handoff gets the first opportunity to reclaim the overlay. + test_harness.to_tree_tx.send(FromEngine::Event(FromOrchestrator::BackfillSyncStarted)).unwrap(); + drop(payload_build); + assert_matches!(test_harness.tree.wait_for_event(), super::LoopEvent::PayloadBuildFinished); test_harness.tree.on_payload_build_finished().unwrap(); assert!(test_harness.tree.pending_persisted_handoff.is_none()); + assert!(!test_harness.tree.state.tree_state.contains_hash(&persisted.hash)); assert_matches!( test_harness.tree.wait_for_event(), super::LoopEvent::EngineMessage(FromEngine::Event(FromOrchestrator::BackfillSyncStarted)) From 4dfc9f0fcd13cb61bb635b163580b1642c00f22d Mon Sep 17 00:00:00 2001 From: Soubhik Singha Mahapatra <160333583+Soubhik-10@users.noreply.github.com> Date: Fri, 14 Aug 2026 16:07:02 +0000 Subject: [PATCH 035/294] feat(engine): add Bogota Engine API method bodies (#26706) Co-authored-by: Matthias Seitz --- crates/payload/primitives/src/lib.rs | 14 +++- crates/payload/primitives/src/traits.rs | 5 ++ crates/rpc/rpc-engine-api/src/engine_api.rs | 85 +++++++++++++++------ crates/rpc/rpc-engine-api/src/metrics.rs | 4 + 4 files changed, 85 insertions(+), 23 deletions(-) diff --git a/crates/payload/primitives/src/lib.rs b/crates/payload/primitives/src/lib.rs index 946284961b2..08c3f2a27a0 100644 --- a/crates/payload/primitives/src/lib.rs +++ b/crates/payload/primitives/src/lib.rs @@ -279,7 +279,10 @@ pub fn validate_slot_number_presence( } EngineApiMessageVersion::V5 => { - if message_validation_kind == MessageValidationKind::Payload { + if matches!( + message_validation_kind, + MessageValidationKind::Payload | MessageValidationKind::PayloadAttributes + ) { if is_amsterdam_active && !has_slot_number { return Err(message_validation_kind .to_error(VersionSpecificValidationError::NoSlotNumberPostAmsterdam)) @@ -769,6 +772,15 @@ mod tests { ); assert_matches!(res, Ok(())); + let res = validate_slot_number_presence( + &chain_spec, + EngineApiMessageVersion::V5, + MessageValidationKind::PayloadAttributes, + 0, + true, + ); + assert_matches!(res, Ok(())); + let res = validate_block_access_list_presence( &chain_spec, EngineApiMessageVersion::V5, diff --git a/crates/payload/primitives/src/traits.rs b/crates/payload/primitives/src/traits.rs index 068cd0db55c..981c0547a2f 100644 --- a/crates/payload/primitives/src/traits.rs +++ b/crates/payload/primitives/src/traits.rs @@ -99,6 +99,11 @@ pub trait PayloadAttributes: fn target_gas_limit(&self) -> Option { None } + + /// Returns the EIP-7805 inclusion-list transactions supplied by the consensus layer. + fn inclusion_list_transactions(&self) -> Option<&[Bytes]> { + None + } } impl PayloadAttributes for EthPayloadAttributes { diff --git a/crates/rpc/rpc-engine-api/src/engine_api.rs b/crates/rpc/rpc-engine-api/src/engine_api.rs index 2635ad3051c..3e3093850ba 100644 --- a/crates/rpc/rpc-engine-api/src/engine_api.rs +++ b/crates/rpc/rpc-engine-api/src/engine_api.rs @@ -9,11 +9,11 @@ use alloy_eips::{ }; use alloy_primitives::{BlockHash, BlockNumber, Bytes, Sealable, B128, B256, U64}; use alloy_rpc_types_engine::{ - CancunPayloadFields, ClientVersionV1, ExecutionData, ExecutionPayloadBodiesV1, - ExecutionPayloadBodiesV2, ExecutionPayloadBodyV1, ExecutionPayloadBodyV2, - ExecutionPayloadInputV2, ExecutionPayloadSidecar, ExecutionPayloadV1, ExecutionPayloadV3, - ExecutionPayloadV4, ForkchoiceState, ForkchoiceUpdated, ForkchoiceUpdatedResponseV2, PayloadId, - PayloadStatus, PayloadStatusV2, PraguePayloadFields, + BogotaPayloadFields, CancunPayloadFields, ClientVersionV1, ExecutionData, + ExecutionPayloadBodiesV1, ExecutionPayloadBodiesV2, ExecutionPayloadBodyV1, + ExecutionPayloadBodyV2, ExecutionPayloadInputV2, ExecutionPayloadSidecar, ExecutionPayloadV1, + ExecutionPayloadV3, ExecutionPayloadV4, ForkchoiceState, ForkchoiceUpdated, + ForkchoiceUpdatedResponseV2, PayloadId, PayloadStatus, PayloadStatusV2, PraguePayloadFields, }; use async_trait::async_trait; use jsonrpsee_core::{server::RpcModule, RpcResult}; @@ -292,17 +292,34 @@ where Ok(res?) } - /// Handler stub for `engine_newPayloadV6`. + /// Handler for `engine_newPayloadV6`. /// /// See also pub async fn new_payload_v6( &self, payload: PayloadT::ExecutionData, ) -> EngineApiResult { - let _ = payload; - Err(EngineApiError::EngineObjectValidationError( - reth_payload_primitives::EngineObjectValidationError::UnsupportedFork, - )) + let payload_or_attrs = PayloadOrAttributes::< + '_, + PayloadT::ExecutionData, + PayloadT::PayloadAttributes, + >::from_execution_payload(&payload); + self.inner + .validator + .validate_version_specific_fields(EngineApiMessageVersion::V6, payload_or_attrs)?; + + Ok(self.inner.beacon_consensus.new_payload(payload).await?.into()) + } + + /// Metrics version of `new_payload_v6`. + pub async fn new_payload_v6_metered( + &self, + payload: PayloadT::ExecutionData, + ) -> EngineApiResult { + let start = Instant::now(); + let result = Self::new_payload_v6(self, payload).await; + self.inner.metrics.latency.new_payload_v6.record(start.elapsed()); + result } /// Returns whether the engine accepts execution requests hash. @@ -427,7 +444,7 @@ where res } - /// Handler stub for `engine_forkchoiceUpdatedV5`. + /// Handler for `engine_forkchoiceUpdatedV5`. /// /// See also pub async fn fork_choice_updated_v5( @@ -436,10 +453,28 @@ where payload_attrs: Option, custody_columns: Option, ) -> EngineApiResult { - let _ = (state, payload_attrs, custody_columns); - Err(EngineApiError::EngineObjectValidationError( - reth_payload_primitives::EngineObjectValidationError::UnsupportedFork, - )) + if let Some(custody_columns) = custody_columns { + self.inner.cell_custody.set_from_engine_api(custody_columns); + } + + // Todo: Validate IL and populate `inclusion_list_satisfied` properly and test. + Ok(self + .validate_and_execute_forkchoice(EngineApiMessageVersion::V5, state, payload_attrs) + .await? + .into()) + } + + /// Metrics version of `fork_choice_updated_v5` + pub async fn fork_choice_updated_v5_metered( + &self, + state: ForkchoiceState, + payload_attrs: Option, + custody_columns: Option, + ) -> EngineApiResult { + let start = Instant::now(); + let res = Self::fork_choice_updated_v5(self, state, payload_attrs, custody_columns).await; + self.inner.metrics.latency.fork_choice_updated_v5.record(start.elapsed()); + res } /// Helper function for retrieving the build payload by id. @@ -1309,7 +1344,7 @@ where Ok(self.new_payload_v5_metered(payload).await?) } - /// Handler for the stubbed `engine_newPayloadV6` endpoint. + /// Handler for `engine_newPayloadV6`. /// /// See also async fn new_payload_v6( @@ -1320,17 +1355,23 @@ where execution_requests: RequestsOrHash, inclusion_list_transactions: Vec, ) -> RpcResult { - trace!(target: "rpc::engine", "Serving engine_newPayloadV6 stub"); + trace!(target: "rpc::engine", "Serving engine_newPayloadV6"); + if execution_requests.is_hash() && !self.inner.accept_execution_requests_hash { + return Err(EngineApiError::UnexpectedRequestsHash.into()); + } + let payload = ExecutionData { payload: payload.into(), sidecar: ExecutionPayloadSidecar::v6( CancunPayloadFields { versioned_hashes, parent_beacon_block_root }, PraguePayloadFields { requests: execution_requests }, - inclusion_list_transactions.into(), + BogotaPayloadFields { inclusion_list_transactions }, ), }; - Ok(self.new_payload_v6(payload).await?) + // TODO: perform structural validation of the inclusion list transactions and populate + // `inclusion_list_satisfied` for VALID payloads + Ok(self.new_payload_v6_metered(payload).await?) } /// Handler for `engine_forkchoiceUpdatedV1` @@ -1384,7 +1425,7 @@ where .await?) } - /// Handler for the stubbed `engine_forkchoiceUpdatedV5` endpoint. + /// Handler for `engine_forkchoiceUpdatedV5`. /// /// See also async fn fork_choice_updated_v5( @@ -1393,9 +1434,9 @@ where payload_attributes: Option, custody_columns: Option, ) -> RpcResult { - trace!(target: "rpc::engine", "Serving engine_forkchoiceUpdatedV5 stub"); + trace!(target: "rpc::engine", "Serving engine_forkchoiceUpdatedV5"); Ok(self - .fork_choice_updated_v5(fork_choice_state, payload_attributes, custody_columns) + .fork_choice_updated_v5_metered(fork_choice_state, payload_attributes, custody_columns) .await?) } diff --git a/crates/rpc/rpc-engine-api/src/metrics.rs b/crates/rpc/rpc-engine-api/src/metrics.rs index 87048445cf4..653a53fb3c7 100644 --- a/crates/rpc/rpc-engine-api/src/metrics.rs +++ b/crates/rpc/rpc-engine-api/src/metrics.rs @@ -24,6 +24,8 @@ pub(crate) struct EngineApiLatencyMetrics { pub(crate) new_payload_v4: Histogram, /// Latency for `engine_newPayloadV5` pub(crate) new_payload_v5: Histogram, + /// Latency for `engine_newPayloadV6` + pub(crate) new_payload_v6: Histogram, /// Latency for `engine_forkchoiceUpdatedV1` pub(crate) fork_choice_updated_v1: Histogram, /// Latency for `engine_forkchoiceUpdatedV2` @@ -32,6 +34,8 @@ pub(crate) struct EngineApiLatencyMetrics { pub(crate) fork_choice_updated_v3: Histogram, /// Latency for `engine_forkchoiceUpdatedV4` pub(crate) fork_choice_updated_v4: Histogram, + /// Latency for `engine_forkchoiceUpdatedV5` + pub(crate) fork_choice_updated_v5: Histogram, /// Latency for `engine_getPayloadV1` pub(crate) get_payload_v1: Histogram, /// Latency for `engine_getPayloadV2` From 2e5f4c09dbeb0a725d753fe23484f6f7f91b37c2 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Fri, 14 Aug 2026 20:48:23 +0200 Subject: [PATCH 036/294] fix(engine): enforce Bogota fork timeframe in engine method validation (#26711) --- crates/payload/primitives/src/lib.rs | 139 ++++++++++++++++++++++++++- 1 file changed, 137 insertions(+), 2 deletions(-) diff --git a/crates/payload/primitives/src/lib.rs b/crates/payload/primitives/src/lib.rs index 08c3f2a27a0..932d45141ae 100644 --- a/crates/payload/primitives/src/lib.rs +++ b/crates/payload/primitives/src/lib.rs @@ -62,8 +62,10 @@ pub trait PayloadTypes: Send + Sync + Unpin + core::fmt::Debug + Clone + 'static /// * If V2, this ensures that the payload timestamp is pre-Cancun. /// * If V3, this ensures that the payload timestamp is within the Cancun timestamp. /// * If V4, this ensures that the payload timestamp is within the Prague timestamp. -/// * If V5, this ensures that the payload timestamp is within the Osaka timestamp. -/// * If V6, this ensures that the payload timestamp is within the Amsterdam timestamp. +/// * If V5, this ensures that the payload timestamp is within the Osaka timestamp, and within the +/// Bogota timestamp for `engine_forkchoiceUpdatedV5` payload attributes. +/// * If V6, this ensures that the payload timestamp is within the Amsterdam timestamp, and within +/// the Bogota timestamp for `engine_newPayloadV6` payloads. /// /// Additionally, it ensures that `engine_getPayloadV4` is not used for an Osaka payload and that /// staggered endpoint upgrades reject the next fork once a newer method version is required. @@ -186,6 +188,51 @@ pub fn validate_payload_timestamp( return Err(EngineObjectValidationError::UnsupportedFork) } + let is_bogota = chain_spec.is_bogota_active_at_timestamp(timestamp); + + // Staggered endpoint upgrades must reject Bogota payloads until the Bogota-specific method + // version is used. + // + // From the Engine API spec: + // + // + // For `engine_newPayloadV5` and `engine_forkchoiceUpdatedV4`: + // + // 1. Client software MUST return -38005: Unsupported fork error if the timestamp of payload is + // greater than or equal to the Bogota activation timestamp. + if is_bogota && + matches!( + (version, kind), + (EngineApiMessageVersion::V4, MessageValidationKind::PayloadAttributes) | + (EngineApiMessageVersion::V5, MessageValidationKind::Payload) + ) + { + return Err(EngineObjectValidationError::UnsupportedFork) + } + + if !is_bogota && + matches!( + (version, kind), + (EngineApiMessageVersion::V5, MessageValidationKind::PayloadAttributes) | + (EngineApiMessageVersion::V6, MessageValidationKind::Payload) + ) + { + // From the Engine API spec: + // + // + // For `engine_newPayloadV6`: + // + // 1. Client software MUST return -38005: Unsupported fork error if the timestamp of the + // payload does not fall within the time frame of the Bogota fork. + // + // For `engine_forkchoiceUpdatedV5`: + // + // 2. Client software MUST return -38005: Unsupported fork error if the payloadAttributes is + // set and the payloadAttributes.timestamp does not fall within the time frame of the + // Bogota fork. + return Err(EngineObjectValidationError::UnsupportedFork) + } + Ok(()) } @@ -750,6 +797,94 @@ mod tests { assert_matches!(res, Ok(())); } + #[test] + fn validate_bogota_fork_timeframe() { + // Amsterdam active, Bogota not scheduled: the Bogota-specific methods must reject + let chain_spec = ChainSpecBuilder::mainnet().amsterdam_activated().build(); + + // `engine_newPayloadV6` requires a Bogota timestamp + let res = validate_payload_timestamp( + &chain_spec, + EngineApiMessageVersion::V6, + 0, + MessageValidationKind::Payload, + ); + assert_matches!(res, Err(EngineObjectValidationError::UnsupportedFork)); + + // `engine_forkchoiceUpdatedV5` requires Bogota payload attributes + let res = validate_payload_timestamp( + &chain_spec, + EngineApiMessageVersion::V5, + 0, + MessageValidationKind::PayloadAttributes, + ); + assert_matches!(res, Err(EngineObjectValidationError::UnsupportedFork)); + + // the Amsterdam methods remain valid pre-Bogota + let res = validate_payload_timestamp( + &chain_spec, + EngineApiMessageVersion::V5, + 0, + MessageValidationKind::Payload, + ); + assert_matches!(res, Ok(())); + let res = validate_payload_timestamp( + &chain_spec, + EngineApiMessageVersion::V4, + 0, + MessageValidationKind::PayloadAttributes, + ); + assert_matches!(res, Ok(())); + } + + #[test] + fn validate_bogota_staggered_version_restrictions() { + let chain_spec = ChainSpecBuilder::mainnet().bogota_activated().build(); + + // `engine_newPayloadV5` must reject Bogota payloads + let res = validate_payload_timestamp( + &chain_spec, + EngineApiMessageVersion::V5, + 0, + MessageValidationKind::Payload, + ); + assert_matches!(res, Err(EngineObjectValidationError::UnsupportedFork)); + + // `engine_forkchoiceUpdatedV4` must reject Bogota payload attributes + let res = validate_payload_timestamp( + &chain_spec, + EngineApiMessageVersion::V4, + 0, + MessageValidationKind::PayloadAttributes, + ); + assert_matches!(res, Err(EngineObjectValidationError::UnsupportedFork)); + + // the Bogota-specific methods are accepted + let res = validate_payload_timestamp( + &chain_spec, + EngineApiMessageVersion::V6, + 0, + MessageValidationKind::Payload, + ); + assert_matches!(res, Ok(())); + let res = validate_payload_timestamp( + &chain_spec, + EngineApiMessageVersion::V5, + 0, + MessageValidationKind::PayloadAttributes, + ); + assert_matches!(res, Ok(())); + + // Bogota defines no new getPayload version, `engine_getPayloadV6` remains valid + let res = validate_payload_timestamp( + &chain_spec, + EngineApiMessageVersion::V6, + 0, + MessageValidationKind::GetPayload, + ); + assert_matches!(res, Ok(())); + } + #[test] fn validate_amsterdam_slot_and_bal_presence() { let chain_spec = ChainSpecBuilder::mainnet().amsterdam_activated().build(); From d5bdb687eeb0d3471fe156174e08246bce9662a3 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Sat, 15 Aug 2026 11:03:18 +0200 Subject: [PATCH 037/294] docs(rpc): document trace chain subscription (#26732) --- docs/vocs/docs/pages/jsonrpc/debug.mdx | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/docs/vocs/docs/pages/jsonrpc/debug.mdx b/docs/vocs/docs/pages/jsonrpc/debug.mdx index a341f5ae32e..299b0ec52b2 100644 --- a/docs/vocs/docs/pages/jsonrpc/debug.mdx +++ b/docs/vocs/docs/pages/jsonrpc/debug.mdx @@ -53,13 +53,25 @@ Returns an array of recent bad blocks that the client has seen on the network. | ------ | ------------------------------------------------ | | RPC | `{"method": "debug_getBadBlocks", "params": []}` | -## `debug_traceChain` +## `debug_subscribe`, `debug_unsubscribe` -Returns the structured logs created during the execution of EVM between two blocks (excluding start) as a JSON object. +Subscribes to structured logs created during EVM execution between two blocks. The start block is +excluded and the end block is included. Pass `"traceChain"` as the subscription type and optionally +provide tracing options. -| Client | Method invocation | -| ------ | -------------------------------------------------------------------- | -| RPC | `{"method": "debug_traceChain", "params": [start_block, end_block]}` | +The subscription returns an ID and emits `debug_subscription` notifications containing the block +number and its transaction traces. Empty intermediate blocks are omitted, but the end block is +always emitted. Pass the subscription ID to `debug_unsubscribe` to stop receiving notifications. + +| Client | Method invocation | +| ------ | ------------------------------------------------------------------------------------------------- | +| RPC | `{"method": "debug_subscribe", "params": ["traceChain", start_block, end_block, opts]}` | +| RPC | `{"method": "debug_unsubscribe", "params": [subscription_id]}` | + +:::note +Subscriptions are only available over WebSocket and IPC transports, as HTTP does not support +server-initiated messages. +::: ## `debug_traceBlock` @@ -195,4 +207,4 @@ Returns the contract bytecode associated with the given code hash at the specifi | Client | Method invocation | | ------ | ------------------------------------------------------------------------ | -| RPC | `{"method": "debug_codeByHash", "params": [code_hash, block]}` | \ No newline at end of file +| RPC | `{"method": "debug_codeByHash", "params": [code_hash, block]}` | From 5327b68d5b91eaf69cf07a9737d605f562f771c7 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Sat, 15 Aug 2026 11:52:23 +0200 Subject: [PATCH 038/294] docs(rpc): fix trace_rawTransaction example (#26735) --- docs/vocs/docs/pages/jsonrpc/trace.mdx | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/docs/vocs/docs/pages/jsonrpc/trace.mdx b/docs/vocs/docs/pages/jsonrpc/trace.mdx index e962ddf5c0d..0b40b2449be 100644 --- a/docs/vocs/docs/pages/jsonrpc/trace.mdx +++ b/docs/vocs/docs/pages/jsonrpc/trace.mdx @@ -294,21 +294,24 @@ Traces a call to `eth_sendRawTransaction` without making the call, returning the ### Example +The example below can be reproduced with `reth node --dev --http --http.api eth,trace`. The signed EIP-1559 transaction is from the first prefunded dev account, and the explicit `0x0` block parameter pins the trace state to genesis. + ```js -// > {"jsonrpc":"2.0","id":1,"method":"trace_rawTransaction","params":["0xd46e8dd67c5d32be8d46e8dd67c5d32be8058bb8eb970870f072445675058bb8eb970870f072445675",["trace"]]} +// > {"jsonrpc":"2.0","id":1,"method":"trace_rawTransaction","params":["0x02f86e82053980843b9aca008502540be400830186a094000000000000000000000000000000000000dead8080c080a0db09ddf33a2ded1d2f9255abf3d1e7cab03df637fa204c7f777aaa518a375237a01e7948bacb7bb5e12292f7f4c44464deb001ab5f85d486d7738cd6b634193ec8",["trace"],"0x0"]} { - "id": 1, "jsonrpc": "2.0", + "id": 1, "result": { "output": "0x", "stateDiff": null, "trace": [{ + "type": "call", "action": { + "from": "0xf39fd6e51aad88f6f4ce6ab8827279cfffb92266", "callType": "call", - "from": "0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b", - "to": "0x6295ee1b4f6dd65047762f924ecd367c17eabf8f", - "gas": "0x186a0", + "gas": "0x13498", "input": "0x", + "to": "0x000000000000000000000000000000000000dead", "value": "0x0" }, "result": { @@ -316,8 +319,7 @@ Traces a call to `eth_sendRawTransaction` without making the call, returning the "output": "0x" }, "subtraces": 0, - "traceAddress": [], - "type": "call" + "traceAddress": [] }], "vmTrace": null } From aa90fb124e87d6cabed2b8f3f9d9b02eb1d37b45 Mon Sep 17 00:00:00 2001 From: Soubhik Singha Mahapatra <160333583+Soubhik-10@users.noreply.github.com> Date: Sun, 16 Aug 2026 06:17:30 +0000 Subject: [PATCH 039/294] feat(engine): build FOCIL inclusion lists (#26737) Co-authored-by: Matthias Seitz --- crates/rpc/rpc-engine-api/Cargo.toml | 1 + crates/rpc/rpc-engine-api/src/engine_api.rs | 137 +++++++++++++++++++- crates/rpc/rpc-engine-api/src/metrics.rs | 2 + 3 files changed, 135 insertions(+), 5 deletions(-) diff --git a/crates/rpc/rpc-engine-api/Cargo.toml b/crates/rpc/rpc-engine-api/Cargo.toml index f73a48e8212..a5c9cb166a6 100644 --- a/crates/rpc/rpc-engine-api/Cargo.toml +++ b/crates/rpc/rpc-engine-api/Cargo.toml @@ -51,6 +51,7 @@ reth-ethereum-engine-primitives.workspace = true reth-provider = { workspace = true, features = ["test-utils"] } reth-ethereum-primitives.workspace = true reth-payload-builder = { workspace = true, features = ["test-utils"] } +reth-transaction-pool = { workspace = true, features = ["test-utils"] } reth-testing-utils.workspace = true alloy-rlp.workspace = true reth-node-ethereum.workspace = true diff --git a/crates/rpc/rpc-engine-api/src/engine_api.rs b/crates/rpc/rpc-engine-api/src/engine_api.rs index 3e3093850ba..1b619bd53d6 100644 --- a/crates/rpc/rpc-engine-api/src/engine_api.rs +++ b/crates/rpc/rpc-engine-api/src/engine_api.rs @@ -6,6 +6,7 @@ use alloy_eips::{ eip4844::{BlobAndProofV1, BlobAndProofV2, BlobCellsAndProofsV1}, eip4895::Withdrawals, eip7685::RequestsOrHash, + Encodable2718, }; use alloy_primitives::{BlockHash, BlockNumber, Bytes, Sealable, B128, B256, U64}; use alloy_rpc_types_engine::{ @@ -14,6 +15,7 @@ use alloy_rpc_types_engine::{ ExecutionPayloadBodyV2, ExecutionPayloadInputV2, ExecutionPayloadSidecar, ExecutionPayloadV1, ExecutionPayloadV3, ExecutionPayloadV4, ForkchoiceState, ForkchoiceUpdated, ForkchoiceUpdatedResponseV2, PayloadId, PayloadStatus, PayloadStatusV2, PraguePayloadFields, + MAX_BYTES_PER_INCLUSION_LIST, }; use async_trait::async_trait; use jsonrpsee_core::{server::RpcModule, RpcResult}; @@ -29,7 +31,7 @@ use reth_primitives_traits::{Block, BlockBody}; use reth_rpc_api::{EngineApiServer, IntoEngineApiRpcModule}; use reth_storage_api::{BalProvider, BlockReader, HeaderProvider, StateProviderFactory}; use reth_tasks::Runtime; -use reth_transaction_pool::TransactionPool; +use reth_transaction_pool::{BestTransactions, PoolTransaction, TransactionPool}; use std::{ sync::Arc, time::{Instant, SystemTime}, @@ -477,6 +479,35 @@ where res } + /// Builds an EIP-7805 inclusion list from the local transaction pool. + pub fn get_inclusion_list_v1(&self) -> EngineApiResult> { + let mut total_size = 0; + let mut inclusion_list = Vec::new(); + + for pool_tx in self.inner.tx_pool.best_transactions().without_blobs().without_updates() { + let encoded: Bytes = pool_tx.transaction.consensus_ref().encoded_2718().into(); + let new_size = total_size + alloy_rlp::Encodable::length(&encoded); + if new_size + alloy_rlp::length_of_length(new_size) > + MAX_BYTES_PER_INCLUSION_LIST as usize + { + break + } + + total_size = new_size; + inclusion_list.push(encoded); + } + + Ok(inclusion_list) + } + + /// Metrics version of `get_inclusion_list_v1`. + pub fn get_inclusion_list_v1_metered(&self) -> EngineApiResult> { + let start = Instant::now(); + let result = Self::get_inclusion_list_v1(self); + self.inner.metrics.latency.get_inclusion_list_v1.record(start.elapsed()); + result + } + /// Helper function for retrieving the build payload by id. async fn get_built_payload( &self, @@ -1545,7 +1576,7 @@ where /// See also . async fn get_inclusion_list_v1(&self) -> RpcResult> { trace!(target: "rpc::engine", "Serving engine_getInclusionListV1"); - Ok(Vec::new()) + Ok(self.get_inclusion_list_v1_metered()?) } /// Handler for `engine_getPayloadBodiesByHashV1` @@ -1743,17 +1774,44 @@ mod tests { use reth_chainspec::{ChainSpec, ChainSpecBuilder, MAINNET}; use reth_engine_primitives::{BeaconEngineMessage, OnForkChoiceUpdated}; use reth_ethereum_engine_primitives::EthEngineTypes; - use reth_ethereum_primitives::Block; + use reth_ethereum_primitives::{Block, TransactionSigned}; use reth_network_api::{ noop::NoopNetwork, EthProtocolInfo, NetworkError, NetworkInfo, NetworkStatus, }; use reth_node_ethereum::EthereumEngineValidator; use reth_payload_builder::test_utils::spawn_test_payload_service; + use reth_primitives_traits::SignedTransaction; use reth_provider::{test_utils::MockEthProvider, BalStoreHandle, InMemoryBalStore, RawBal}; use reth_tasks::Runtime; - use reth_transaction_pool::noop::NoopTransactionPool; + use reth_transaction_pool::{ + blobstore::InMemoryBlobStore, + noop::NoopTransactionPool, + test_utils::{OkValidator, TransactionBuilder}, + CoinbaseTipOrdering, EthPooledTransaction, Pool, PoolTransaction, TransactionOrigin, + }; use tokio::sync::mpsc::{unbounded_channel, UnboundedReceiver}; + type EthTestPool = Pool< + OkValidator, + CoinbaseTipOrdering, + InMemoryBlobStore, + >; + + fn eth_test_pool() -> EthTestPool { + Pool::new( + OkValidator::default(), + CoinbaseTipOrdering::default(), + InMemoryBlobStore::default(), + Default::default(), + ) + } + + fn pooled_transaction(transaction: TransactionSigned) -> EthPooledTransaction { + let transaction = transaction.try_into_recovered().unwrap(); + let encoded_length = transaction.encode_2718_len(); + EthPooledTransaction::new(transaction, encoded_length) + } + fn setup_engine_api() -> ( EngineApiTestHandle, EngineApi< @@ -1764,6 +1822,18 @@ mod tests { ChainSpec, >, ) { + setup_engine_api_with_pool(NoopTransactionPool::default()) + } + + fn setup_engine_api_with_pool( + tx_pool: Pool, + ) -> ( + EngineApiTestHandle, + EngineApi, EthEngineTypes, Pool, EthereumEngineValidator, ChainSpec>, + ) + where + Pool: TransactionPool + 'static, + { let client = ClientVersionV1 { code: ClientCode::RH, name: "Reth".to_string(), @@ -1781,7 +1851,7 @@ mod tests { chain_spec.clone(), ConsensusEngineHandle::new(to_engine), payload_store.into(), - NoopTransactionPool::default(), + tx_pool, task_executor, client, EngineCapabilities::default(), @@ -1814,6 +1884,63 @@ mod tests { assert!(res.is_empty()); } + #[tokio::test] + async fn get_inclusion_list_v1_stops_at_size_limit() { + let pool = eth_test_pool(); + let first = pooled_transaction( + TransactionBuilder::default() + .max_fee_per_gas(3_000_000_000u128) + .input(vec![0; 4_200]) + .into_legacy(), + ); + let second = pooled_transaction( + TransactionBuilder::default() + .max_fee_per_gas(2_000_000_000u128) + .input(vec![0; 4_200]) + .into_legacy(), + ); + let third = pooled_transaction( + TransactionBuilder::default().max_fee_per_gas(1_000_000_000u128).into_legacy(), + ); + let expected: Bytes = first.consensus_ref().encoded_2718().into(); + + pool.add_transaction(TransactionOrigin::External, first).await.unwrap(); + pool.add_transaction(TransactionOrigin::External, second).await.unwrap(); + pool.add_transaction(TransactionOrigin::External, third).await.unwrap(); + let (_, api) = setup_engine_api_with_pool(pool); + + let res = EngineApiServer::get_inclusion_list_v1(&api).await.unwrap(); + assert_eq!(res, vec![expected]); + assert!( + alloy_rlp::list_length::(&res) <= MAX_BYTES_PER_INCLUSION_LIST as usize + ); + } + + #[tokio::test] + async fn get_inclusion_list_v1_excludes_blob_transactions() { + let pool = eth_test_pool(); + let blob = pooled_transaction( + TransactionBuilder::default() + .max_fee_per_gas(2_000_000_000u128) + .max_priority_fee_per_gas(1_000_000_000u128) + .into_eip4844(), + ); + let non_blob = pooled_transaction( + TransactionBuilder::default() + .max_fee_per_gas(1_000_000_000u128) + .max_priority_fee_per_gas(1_000_000_000u128) + .into_eip1559(), + ); + let expected: Bytes = non_blob.consensus_ref().encoded_2718().into(); + + pool.add_transaction(TransactionOrigin::External, blob).await.unwrap(); + pool.add_transaction(TransactionOrigin::External, non_blob).await.unwrap(); + let (_, api) = setup_engine_api_with_pool(pool); + + let res = EngineApiServer::get_inclusion_list_v1(&api).await.unwrap(); + assert_eq!(res, vec![expected]); + } + #[tokio::test] async fn has_blobs_returns_ordered_availability() { let (_, api) = setup_engine_api(); diff --git a/crates/rpc/rpc-engine-api/src/metrics.rs b/crates/rpc/rpc-engine-api/src/metrics.rs index 653a53fb3c7..2d49e26f8f0 100644 --- a/crates/rpc/rpc-engine-api/src/metrics.rs +++ b/crates/rpc/rpc-engine-api/src/metrics.rs @@ -36,6 +36,8 @@ pub(crate) struct EngineApiLatencyMetrics { pub(crate) fork_choice_updated_v4: Histogram, /// Latency for `engine_forkchoiceUpdatedV5` pub(crate) fork_choice_updated_v5: Histogram, + /// Latency for `engine_getInclusionListV1` + pub(crate) get_inclusion_list_v1: Histogram, /// Latency for `engine_getPayloadV1` pub(crate) get_payload_v1: Histogram, /// Latency for `engine_getPayloadV2` From bac4cde24027ee4f0307e4718f7eb271f6a2c6a0 Mon Sep 17 00:00:00 2001 From: Derek Cofausper <256792747+decofe@users.noreply.github.com> Date: Mon, 17 Aug 2026 11:02:43 +0000 Subject: [PATCH 040/294] fix(rpc): preserve gas limit in eth_sendTransaction (#26743) Co-authored-by: Matthias Seitz <19890894+mattsse@users.noreply.github.com> --- .../rpc-eth-api/src/helpers/transaction.rs | 11 ++++--- crates/rpc/rpc/src/eth/helpers/transaction.rs | 31 ++++++++++++++++++- 2 files changed, 36 insertions(+), 6 deletions(-) diff --git a/crates/rpc/rpc-eth-api/src/helpers/transaction.rs b/crates/rpc/rpc-eth-api/src/helpers/transaction.rs index 5909eb7338d..bbbdbfc467a 100644 --- a/crates/rpc/rpc-eth-api/src/helpers/transaction.rs +++ b/crates/rpc/rpc-eth-api/src/helpers/transaction.rs @@ -509,11 +509,12 @@ pub trait EthTransactions: LoadTransaction { let chain_id = self.chain_id(); request.as_mut().set_chain_id(chain_id.to()); - let estimated_gas = self - .estimate_gas_at(request.clone(), BlockId::pending(), EvmOverrides::default()) - .await?; - let gas_limit = estimated_gas; - request.as_mut().set_gas_limit(gas_limit.to()); + if request.as_ref().gas_limit().is_none() { + let estimated_gas = self + .estimate_gas_at(request.clone(), BlockId::pending(), EvmOverrides::default()) + .await?; + request.as_mut().set_gas_limit(estimated_gas.to()); + } let transaction = self.sign_request(&from, request).await?.with_signer(from); diff --git a/crates/rpc/rpc/src/eth/helpers/transaction.rs b/crates/rpc/rpc/src/eth/helpers/transaction.rs index ecc1c431e1b..87c2d4588e7 100644 --- a/crates/rpc/rpc/src/eth/helpers/transaction.rs +++ b/crates/rpc/rpc/src/eth/helpers/transaction.rs @@ -132,7 +132,7 @@ where #[cfg(test)] mod tests { use super::*; - use crate::eth::helpers::types::EthRpcConverter; + use crate::eth::helpers::{signer::DevSigner, types::EthRpcConverter}; use alloy_consensus::{ BlobTransactionSidecar, Block, Header, SidecarBuilder, SimpleCoder, Transaction, }; @@ -289,6 +289,35 @@ mod tests { assert_eq!(eth_api.pool().len(), 1); } + #[tokio::test] + async fn send_transaction_preserves_provided_gas_limit() { + let signers = DevSigner::random_signers(1); + let address = signers[0].accounts()[0]; + let accounts = AddressMap::from_iter([( + address, + ExtendedAccount::new(0, U256::from(10_000_000_000_000_000_000u64)), + )]); + let eth_api = mock_eth_api(accounts); + eth_api.signers().write().extend(signers); + + let provided_gas_limit = 90_000; + let tx_req = TransactionRequest { + from: Some(address), + to: Some(address.into()), + gas: Some(provided_gas_limit), + gas_price: Some(1_000_000_000), + ..Default::default() + }; + + let hash = eth_api + .send_transaction_request(tx_req) + .await + .expect("send_transaction should succeed"); + let pooled = eth_api.pool().get(&hash).expect("transaction should be in the pool"); + + assert_eq!(pooled.transaction.gas_limit(), provided_gas_limit); + } + #[tokio::test] async fn test_fill_transaction_fills_chain_id() { let address = Address::random(); From faa8ca9847a4fd6213f770a30bcd532d01a79dab Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 17 Aug 2026 11:03:53 +0000 Subject: [PATCH 041/294] chore(deps): bump the ci-weekly group across 1 directory with 4 updates (#26745) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/bench-benchmarkoor.yml | 2 +- .github/workflows/bench-scheduled.yml | 2 +- .github/workflows/bench.yml | 2 +- .github/workflows/check-alloy.yml | 2 +- .github/workflows/compact.yml | 4 +- .github/workflows/e2e.yml | 12 +++--- .github/workflows/integration.yml | 12 +++--- .github/workflows/lint.yml | 50 ++++++++++++------------ .github/workflows/stage.yml | 4 +- .github/workflows/sync-era.yml | 4 +- .github/workflows/sync.yml | 4 +- .github/workflows/unit.yml | 16 ++++---- 12 files changed, 57 insertions(+), 57 deletions(-) diff --git a/.github/workflows/bench-benchmarkoor.yml b/.github/workflows/bench-benchmarkoor.yml index e2cdce401b3..c9e69ef6bb3 100644 --- a/.github/workflows/bench-benchmarkoor.yml +++ b/.github/workflows/bench-benchmarkoor.yml @@ -198,7 +198,7 @@ jobs: - uses: dtolnay/rust-toolchain@stable - - uses: mozilla-actions/sccache-action@9e7fa8a12102821edf02ca5dbea1acd0f89a2696 # v0.0.10 + - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 continue-on-error: true - name: Install dependencies diff --git a/.github/workflows/bench-scheduled.yml b/.github/workflows/bench-scheduled.yml index 9d9fa094b41..45562f0babd 100644 --- a/.github/workflows/bench-scheduled.yml +++ b/.github/workflows/bench-scheduled.yml @@ -349,7 +349,7 @@ jobs: - uses: rui314/setup-mold@9c9c13bf4c3f1adef0cc596abc155580bcb04444 # v1 - uses: dtolnay/rust-toolchain@stable - - uses: mozilla-actions/sccache-action@9e7fa8a12102821edf02ca5dbea1acd0f89a2696 # v0.0.10 + - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 continue-on-error: true - uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0 diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index 58f4c400bbb..7113a02c8f4 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -935,7 +935,7 @@ jobs: - uses: rui314/setup-mold@9c9c13bf4c3f1adef0cc596abc155580bcb04444 # v1 - uses: dtolnay/rust-toolchain@stable - - uses: mozilla-actions/sccache-action@9e7fa8a12102821edf02ca5dbea1acd0f89a2696 # v0.0.10 + - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 continue-on-error: true - uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0 diff --git a/.github/workflows/check-alloy.yml b/.github/workflows/check-alloy.yml index 64e68867669..59d53f851ea 100644 --- a/.github/workflows/check-alloy.yml +++ b/.github/workflows/check-alloy.yml @@ -41,7 +41,7 @@ jobs: - uses: dtolnay/rust-toolchain@stable - - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 + - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 with: cache-on-failure: true diff --git a/.github/workflows/compact.yml b/.github/workflows/compact.yml index 86f2592881a..5dc14b85d31 100644 --- a/.github/workflows/compact.yml +++ b/.github/workflows/compact.yml @@ -35,8 +35,8 @@ jobs: - uses: rui314/setup-mold@9c9c13bf4c3f1adef0cc596abc155580bcb04444 # v1 - run: .github/scripts/install_llvm.sh ubuntu - uses: dtolnay/rust-toolchain@stable - - uses: mozilla-actions/sccache-action@9e7fa8a12102821edf02ca5dbea1acd0f89a2696 # v0.0.10 - - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 + - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 + - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 with: cache-on-failure: true - name: Checkout base diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 1abce78da42..a55655b01f9 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -35,11 +35,11 @@ jobs: - uses: rui314/setup-mold@9c9c13bf4c3f1adef0cc596abc155580bcb04444 # v1 - run: .github/scripts/install_llvm.sh ubuntu - uses: dtolnay/rust-toolchain@stable - - uses: mozilla-actions/sccache-action@9e7fa8a12102821edf02ca5dbea1acd0f89a2696 # v0.0.10 - - uses: taiki-e/install-action@41049aa56687c35e0afa74eed4f09cec4f9afabf # v2.85.2 + - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 + - uses: taiki-e/install-action@7f4eb899022d8fe70b20c4f3de697aa85c309026 # v2.85.11 with: tool: nextest - - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 + - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 with: cache-targets: false cache-on-failure: true @@ -70,11 +70,11 @@ jobs: - uses: rui314/setup-mold@9c9c13bf4c3f1adef0cc596abc155580bcb04444 # v1 - run: .github/scripts/install_llvm.sh ubuntu - uses: dtolnay/rust-toolchain@stable - - uses: mozilla-actions/sccache-action@9e7fa8a12102821edf02ca5dbea1acd0f89a2696 # v0.0.10 - - uses: taiki-e/install-action@41049aa56687c35e0afa74eed4f09cec4f9afabf # v2.85.2 + - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 + - uses: taiki-e/install-action@7f4eb899022d8fe70b20c4f3de697aa85c309026 # v2.85.11 with: tool: nextest - - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 + - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 with: cache-on-failure: true - name: Run RocksDB e2e tests diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 591bc31304e..e324f0139dd 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -44,11 +44,11 @@ jobs: - uses: dtolnay/rust-toolchain@stable - name: Install Geth run: .github/scripts/install_geth.sh - - uses: taiki-e/install-action@41049aa56687c35e0afa74eed4f09cec4f9afabf # v2.85.2 + - uses: taiki-e/install-action@7f4eb899022d8fe70b20c4f3de697aa85c309026 # v2.85.11 with: tool: nextest - - uses: mozilla-actions/sccache-action@9e7fa8a12102821edf02ca5dbea1acd0f89a2696 # v0.0.10 - - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 + - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 + - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 with: cache-targets: false cache-on-failure: true @@ -85,11 +85,11 @@ jobs: - uses: rui314/setup-mold@9c9c13bf4c3f1adef0cc596abc155580bcb04444 # v1 - run: .github/scripts/install_llvm.sh ubuntu - uses: dtolnay/rust-toolchain@stable - - uses: taiki-e/install-action@41049aa56687c35e0afa74eed4f09cec4f9afabf # v2.85.2 + - uses: taiki-e/install-action@7f4eb899022d8fe70b20c4f3de697aa85c309026 # v2.85.11 with: tool: nextest - - uses: mozilla-actions/sccache-action@9e7fa8a12102821edf02ca5dbea1acd0f89a2696 # v0.0.10 - - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 + - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 + - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 with: cache-on-failure: true - name: run era/era1/ere files integration tests diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 562185f766b..b87e294f04c 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -34,8 +34,8 @@ jobs: - uses: dtolnay/rust-toolchain@clippy with: components: clippy - - uses: mozilla-actions/sccache-action@9e7fa8a12102821edf02ca5dbea1acd0f89a2696 # v0.0.10 - - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 + - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 + - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 with: cache-on-failure: true - if: "${{ matrix.type == 'book' }}" @@ -62,8 +62,8 @@ jobs: - uses: dtolnay/rust-toolchain@nightly with: components: clippy - - uses: mozilla-actions/sccache-action@9e7fa8a12102821edf02ca5dbea1acd0f89a2696 # v0.0.10 - - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 + - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 + - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 with: cache-on-failure: true - run: cargo clippy --workspace --lib --examples --tests --benches --all-features --locked @@ -83,11 +83,11 @@ jobs: - uses: dtolnay/rust-toolchain@stable with: target: wasm32-wasip1 - - uses: taiki-e/install-action@41049aa56687c35e0afa74eed4f09cec4f9afabf # v2.85.2 + - uses: taiki-e/install-action@7f4eb899022d8fe70b20c4f3de697aa85c309026 # v2.85.11 with: tool: cargo-hack - - uses: mozilla-actions/sccache-action@9e7fa8a12102821edf02ca5dbea1acd0f89a2696 # v0.0.10 - - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 + - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 + - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 with: cache-on-failure: true - uses: dcarbone/install-jq-action@4fcb5062d7ce9bc4382d1a352d19ba3ba2c317c1 # v4.0.1 @@ -109,11 +109,11 @@ jobs: - uses: dtolnay/rust-toolchain@stable with: target: riscv32imac-unknown-none-elf - - uses: taiki-e/install-action@41049aa56687c35e0afa74eed4f09cec4f9afabf # v2.85.2 + - uses: taiki-e/install-action@7f4eb899022d8fe70b20c4f3de697aa85c309026 # v2.85.11 with: tool: cargo-hack - - uses: mozilla-actions/sccache-action@9e7fa8a12102821edf02ca5dbea1acd0f89a2696 # v0.0.10 - - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 + - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 + - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 with: cache-on-failure: true - uses: dcarbone/install-jq-action@4fcb5062d7ce9bc4382d1a352d19ba3ba2c317c1 # v4.0.1 @@ -137,11 +137,11 @@ jobs: - uses: rui314/setup-mold@9c9c13bf4c3f1adef0cc596abc155580bcb04444 # v1 - run: .github/scripts/install_llvm.sh ubuntu - uses: dtolnay/rust-toolchain@stable - - uses: taiki-e/install-action@41049aa56687c35e0afa74eed4f09cec4f9afabf # v2.85.2 + - uses: taiki-e/install-action@7f4eb899022d8fe70b20c4f3de697aa85c309026 # v2.85.11 with: tool: cargo-hack - - uses: mozilla-actions/sccache-action@9e7fa8a12102821edf02ca5dbea1acd0f89a2696 # v0.0.10 - - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 + - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 + - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 with: cache-on-failure: true - run: cargo hack check --workspace --partition ${{ matrix.partition }}/${{ matrix.total_partitions }} @@ -161,8 +161,8 @@ jobs: - uses: dtolnay/rust-toolchain@master with: toolchain: "1.95" # MSRV - - uses: mozilla-actions/sccache-action@9e7fa8a12102821edf02ca5dbea1acd0f89a2696 # v0.0.10 - - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 + - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 + - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 with: cache-on-failure: true - run: cargo build --bin reth --workspace @@ -182,8 +182,8 @@ jobs: - uses: rui314/setup-mold@9c9c13bf4c3f1adef0cc596abc155580bcb04444 # v1 - run: .github/scripts/install_llvm.sh ubuntu - uses: dtolnay/rust-toolchain@nightly - - uses: mozilla-actions/sccache-action@9e7fa8a12102821edf02ca5dbea1acd0f89a2696 # v0.0.10 - - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 + - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 + - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 with: cache-on-failure: true - run: cargo docs --document-private-items @@ -206,7 +206,7 @@ jobs: - uses: dtolnay/rust-toolchain@nightly with: components: rustfmt - - uses: mozilla-actions/sccache-action@9e7fa8a12102821edf02ca5dbea1acd0f89a2696 # v0.0.10 + - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 - name: Run fmt run: cargo fmt --all --check @@ -223,11 +223,11 @@ jobs: - uses: rui314/setup-mold@9c9c13bf4c3f1adef0cc596abc155580bcb04444 # v1 - run: .github/scripts/install_llvm.sh ubuntu - uses: dtolnay/rust-toolchain@nightly - - uses: mozilla-actions/sccache-action@9e7fa8a12102821edf02ca5dbea1acd0f89a2696 # v0.0.10 - - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 + - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 + - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 with: cache-on-failure: true - - uses: taiki-e/install-action@41049aa56687c35e0afa74eed4f09cec4f9afabf # v2.85.2 + - uses: taiki-e/install-action@7f4eb899022d8fe70b20c4f3de697aa85c309026 # v2.85.11 with: tool: cargo-udeps - run: cargo udeps --workspace --lib --examples --tests --benches --all-features --locked @@ -245,8 +245,8 @@ jobs: - uses: rui314/setup-mold@9c9c13bf4c3f1adef0cc596abc155580bcb04444 # v1 - run: .github/scripts/install_llvm.sh ubuntu - uses: dtolnay/rust-toolchain@nightly - - uses: mozilla-actions/sccache-action@9e7fa8a12102821edf02ca5dbea1acd0f89a2696 # v0.0.10 - - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 + - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 + - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 with: cache-on-failure: true - run: cargo build --bin reth --workspace @@ -322,9 +322,9 @@ jobs: persist-credentials: false - run: .github/scripts/install_llvm.sh ubuntu - uses: dtolnay/rust-toolchain@stable - - uses: mozilla-actions/sccache-action@9e7fa8a12102821edf02ca5dbea1acd0f89a2696 # v0.0.10 + - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 - uses: rui314/setup-mold@9c9c13bf4c3f1adef0cc596abc155580bcb04444 # v1 - - uses: taiki-e/cache-cargo-install-action@417450f3c33ee20393705369577571770643d4c7 # v3.0.7 + - uses: taiki-e/cache-cargo-install-action@9ee83daaa7b96a6fab930949ecf1122bba04a389 # v3.0.8 with: tool: zepter - name: Eagerly pull dependencies diff --git a/.github/workflows/stage.yml b/.github/workflows/stage.yml index 8d0e2bf9d04..66454965ef5 100644 --- a/.github/workflows/stage.yml +++ b/.github/workflows/stage.yml @@ -39,8 +39,8 @@ jobs: - uses: rui314/setup-mold@9c9c13bf4c3f1adef0cc596abc155580bcb04444 # v1 - run: .github/scripts/install_llvm.sh ubuntu - uses: dtolnay/rust-toolchain@stable - - uses: mozilla-actions/sccache-action@9e7fa8a12102821edf02ca5dbea1acd0f89a2696 # v0.0.10 - - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 + - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 + - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 with: cache-on-failure: true - name: Build reth diff --git a/.github/workflows/sync-era.yml b/.github/workflows/sync-era.yml index bbd1ee0b472..66684d314e3 100644 --- a/.github/workflows/sync-era.yml +++ b/.github/workflows/sync-era.yml @@ -44,8 +44,8 @@ jobs: - uses: rui314/setup-mold@9c9c13bf4c3f1adef0cc596abc155580bcb04444 # v1 - run: .github/scripts/install_llvm.sh ubuntu - uses: dtolnay/rust-toolchain@stable - - uses: mozilla-actions/sccache-action@9e7fa8a12102821edf02ca5dbea1acd0f89a2696 # v0.0.10 - - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 + - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 + - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 with: cache-on-failure: true - name: Build ${{ matrix.chain.bin }} diff --git a/.github/workflows/sync.yml b/.github/workflows/sync.yml index 7a2b8c69801..e74797669e8 100644 --- a/.github/workflows/sync.yml +++ b/.github/workflows/sync.yml @@ -44,8 +44,8 @@ jobs: - uses: rui314/setup-mold@9c9c13bf4c3f1adef0cc596abc155580bcb04444 # v1 - run: .github/scripts/install_llvm.sh ubuntu - uses: dtolnay/rust-toolchain@stable - - uses: mozilla-actions/sccache-action@9e7fa8a12102821edf02ca5dbea1acd0f89a2696 # v0.0.10 - - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 + - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 + - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 with: cache-on-failure: true - name: Build ${{ matrix.chain.bin }} diff --git a/.github/workflows/unit.yml b/.github/workflows/unit.yml index 74986cbb3b9..d2153a5bc41 100644 --- a/.github/workflows/unit.yml +++ b/.github/workflows/unit.yml @@ -42,12 +42,12 @@ jobs: - uses: rui314/setup-mold@9c9c13bf4c3f1adef0cc596abc155580bcb04444 # v1 - run: .github/scripts/install_llvm.sh ubuntu - uses: dtolnay/rust-toolchain@stable - - uses: mozilla-actions/sccache-action@9e7fa8a12102821edf02ca5dbea1acd0f89a2696 # v0.0.10 - - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 + - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 + - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 with: cache-targets: false cache-on-failure: true - - uses: taiki-e/install-action@41049aa56687c35e0afa74eed4f09cec4f9afabf # v2.85.2 + - uses: taiki-e/install-action@7f4eb899022d8fe70b20c4f3de697aa85c309026 # v2.85.11 with: tool: nextest - if: "${{ matrix.type == 'book' }}" @@ -97,11 +97,11 @@ jobs: - uses: rui314/setup-mold@9c9c13bf4c3f1adef0cc596abc155580bcb04444 # v1 - run: .github/scripts/install_llvm.sh ubuntu - uses: dtolnay/rust-toolchain@stable - - uses: taiki-e/install-action@41049aa56687c35e0afa74eed4f09cec4f9afabf # v2.85.2 + - uses: taiki-e/install-action@7f4eb899022d8fe70b20c4f3de697aa85c309026 # v2.85.11 with: tool: nextest - - uses: mozilla-actions/sccache-action@9e7fa8a12102821edf02ca5dbea1acd0f89a2696 # v0.0.10 - - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 + - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 + - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 with: cache-on-failure: true - run: cargo nextest run --no-fail-fast --cargo-profile hivetests -p ef-tests --features "asm-keccak ef-tests" @@ -121,8 +121,8 @@ jobs: - uses: rui314/setup-mold@9c9c13bf4c3f1adef0cc596abc155580bcb04444 # v1 - run: .github/scripts/install_llvm.sh ubuntu - uses: dtolnay/rust-toolchain@stable - - uses: mozilla-actions/sccache-action@9e7fa8a12102821edf02ca5dbea1acd0f89a2696 # v0.0.10 - - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 + - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 + - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 with: cache-on-failure: true - name: Run doctests From 4e515276f219bdad941cf87b4f16a127305e97ea Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Tue, 18 Aug 2026 14:03:50 +0200 Subject: [PATCH 042/294] chore(deps): bump h2 to 0.4.16 (#26753) --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a9ca903a753..9e4d002ed6e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4375,9 +4375,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.4.14" +version = "0.4.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "171fefbc92fe4a4de27e0698d6a5b392d6a0e333506bc49133760b3bcf948733" +checksum = "a9f37a958b41b3b19ee2707c06439c0e9e547e847223eb791ecb0cb821c65e27" dependencies = [ "atomic-waker", "bytes", From 1a8322e0fc663294d30facc5d4cb6203b1ae90b6 Mon Sep 17 00:00:00 2001 From: Soubhik Singha Mahapatra <160333583+Soubhik-10@users.noreply.github.com> Date: Tue, 18 Aug 2026 20:28:29 +0000 Subject: [PATCH 043/294] ci(hive): install ripgrep for failure logs (#26757) --- .github/workflows/hive.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/hive.yml b/.github/workflows/hive.yml index 860c54d71d1..5e007f154e0 100644 --- a/.github/workflows/hive.yml +++ b/.github/workflows/hive.yml @@ -217,6 +217,9 @@ jobs: persist-credentials: false fetch-depth: 0 + - name: Install ripgrep + run: sudo apt-get update && sudo apt-get install -y ripgrep + - name: Download hive assets uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: @@ -400,6 +403,9 @@ jobs: persist-credentials: false fetch-depth: 0 + - name: Install ripgrep + run: sudo apt-get update && sudo apt-get install -y ripgrep + - name: Download hive assets uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: From 5e04902bfefb44eb80b1f5a2cd4ae218904bc1fb Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Wed, 19 Aug 2026 12:12:37 +0000 Subject: [PATCH 044/294] fix(engine): enforce state-gas admission on the BAL execution path (#26719) --- .../src/tree/payload_processor/bal/execute.rs | 94 ++++++++++++++++++- 1 file changed, 89 insertions(+), 5 deletions(-) diff --git a/crates/engine/tree/src/tree/payload_processor/bal/execute.rs b/crates/engine/tree/src/tree/payload_processor/bal/execute.rs index 507ba400932..dca199d9251 100644 --- a/crates/engine/tree/src/tree/payload_processor/bal/execute.rs +++ b/crates/engine/tree/src/tree/payload_processor/bal/execute.rs @@ -239,7 +239,7 @@ impl AbortGuard { } } -/// Mirrors `EthBlockExecutor`'s cumulative gas admission check in the ordered BAL commit loop. +/// Mirrors `EthBlockExecutor`'s gas admission checks in the ordered BAL commit loop. #[derive(Debug)] struct BlockGasTracker { block_gas_limit: u64, @@ -247,6 +247,7 @@ struct BlockGasTracker { tx_gas_limit_cap: Option, cumulative_tx_gas_used: u64, block_regular_gas_used: u64, + block_state_gas_used: u64, } impl BlockGasTracker { @@ -261,9 +262,26 @@ impl BlockGasTracker { tx_gas_limit_cap, cumulative_tx_gas_used: 0, block_regular_gas_used: 0, + block_state_gas_used: 0, } } + /// Verifies that the transaction's gas limit fits the block's remaining gas budget(s): the + /// admission check `EthBlockExecutor::execute_transaction_without_commit` performs before + /// executing a transaction. + /// + /// The commit loop never calls that entry point — workers execute speculatively and their + /// results are committed directly via `commit_transaction` — so the check must be replayed + /// here for BAL and serial execution to reach the same block validity verdict. + /// + /// Pre-Amsterdam there is one budget: the tx gas limit, capped by `tx_gas_limit_cap` + /// (EIP-7825), must fit `block_gas_limit - cumulative_tx_gas_used`. + /// + /// Amsterdam (EIP-8037) splits gas into two lanes, each budgeted at `block_gas_limit`: + /// - regular: the capped tx gas limit must fit the remaining regular budget + /// - state: the full, uncapped tx gas limit must fit the remaining state budget, since state + /// gas is drawn from the reservoir above `tx_gas_limit_cap` (execution-specs + /// `check_block_gas_capacity`) fn validate_tx_limit(&self, tx_gas_limit: u64) -> Result<(), BlockExecutionError> { let block_gas_used = if self.enable_amsterdam_eip8037 { self.block_regular_gas_used @@ -282,6 +300,18 @@ impl BlockGasTracker { .into()); } + if self.enable_amsterdam_eip8037 { + let state_gas_available = + self.block_gas_limit.saturating_sub(self.block_state_gas_used); + if tx_gas_limit > state_gas_available { + return Err(BlockValidationError::TransactionGasLimitMoreThanAvailableBlockGas { + transaction_gas_limit: tx_gas_limit, + block_available_gas: state_gas_available, + } + .into()); + } + } + Ok(()) } @@ -290,6 +320,8 @@ impl BlockGasTracker { self.cumulative_tx_gas_used = self.cumulative_tx_gas_used.saturating_add(gas.tx_gas_used()); self.block_regular_gas_used = self.block_regular_gas_used.saturating_add(gas.block_regular_gas_used()); + self.block_state_gas_used = + self.block_state_gas_used.saturating_add(gas.block_state_gas_used()); } } @@ -1144,7 +1176,8 @@ mod tests { #[test] fn gas_tracker_non_amsterdam_uses_cumulative_gas() { - // All-state-gas results keep block_regular_gas_used at 0, so a second tx that fits + // A half-state-gas result keeps both Amsterdam budgets (regular and state) at + // 300_000 used while cumulative_tx_gas_used is 600_000, so a second tx that fits // within the block limit but not the remaining cumulative budget proves that // non-Amsterdam reads cumulative_tx_gas_used while Amsterdam does not. use revm::{ @@ -1158,7 +1191,7 @@ mod tests { let first_tx_gas = 600_000u64; let second_tx_gas_limit = 500_000u64; // fits in total limit but not after cumulative deduction - let gas = ResultGas::new_with_state_gas(first_tx_gas, 0, 0, first_tx_gas); + let gas = ResultGas::new_with_state_gas(first_tx_gas, 0, 0, first_tx_gas / 2); let fake_result: ResultAndState = ExecResultAndState::new( ExecutionResult::Success { @@ -1178,12 +1211,63 @@ mod tests { "non-Amsterdam tracker must reject tx that exceeds remaining cumulative gas", ); - // Amsterdam: block_available_gas = 1_000_000 - 0 = 1_000_000 → accept 500_000. + // Amsterdam: both regular and state budgets have 700_000 left → accept 500_000. let mut amsterdam = BlockGasTracker::new(block_gas_limit, true, None); amsterdam.record_result(&fake_result); assert!( amsterdam.validate_tx_limit(second_tx_gas_limit).is_ok(), - "Amsterdam tracker must accept the same tx since block_regular_gas_used stays 0", + "Amsterdam tracker must accept the same tx since per-dimension budgets still fit", + ); + } + + #[test] + fn gas_tracker_amsterdam_enforces_state_gas_budget() { + // An all-state-gas result leaves the regular budget untouched, so only the + // state-gas admission check can reject the second transaction. The tx's full gas + // limit counts against the state budget — tx_gas_limit_cap does not bound it. + use revm::{ + context::result::{ + ExecResultAndState, ExecutionResult, Output, ResultGas, SuccessReason, + }, + state::EvmState, + }; + + let block_gas_limit = 1_000_000u64; + let first_tx_state_gas = 600_000u64; + let second_tx_gas_limit = 500_000u64; // exceeds the remaining 400_000 state budget + + let gas = ResultGas::new_with_state_gas(first_tx_state_gas, 0, 0, first_tx_state_gas); + let fake_result: ResultAndState = + ExecResultAndState::new( + ExecutionResult::Success { + reason: SuccessReason::Return, + gas, + logs: vec![], + output: Output::Call(Default::default()), + }, + EvmState::default(), + ); + + // Regular budget is full (block_regular_gas_used = 0) but the state budget has + // only 400_000 left → reject 500_000. + let mut amsterdam = BlockGasTracker::new(block_gas_limit, true, None); + amsterdam.record_result(&fake_result); + assert!( + amsterdam.validate_tx_limit(second_tx_gas_limit).is_err(), + "Amsterdam tracker must reject tx whose gas limit exceeds the remaining state budget", + ); + assert!( + amsterdam.validate_tx_limit(400_000).is_ok(), + "Amsterdam tracker must accept tx whose gas limit exactly fits the state budget", + ); + + // With a cap of 400_000 the capped regular check passes, but the full 500_000 + // limit still counts against the state budget → reject. + let mut capped = BlockGasTracker::new(block_gas_limit, true, Some(400_000)); + capped.record_result(&fake_result); + assert!( + capped.validate_tx_limit(second_tx_gas_limit).is_err(), + "tx_gas_limit_cap must not bound the state-gas admission check", ); } From d7a5c943f9b9902258476ed1bc6dd2a72f8be86d Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Wed, 19 Aug 2026 12:14:01 +0000 Subject: [PATCH 045/294] feat(rpc): expose raw block transactions on auth server (#26760) --- crates/rpc/rpc-api/src/engine.rs | 18 +++++++++++++++++- crates/rpc/rpc/src/engine.rs | 26 +++++++++++++++++++++++++- 2 files changed, 42 insertions(+), 2 deletions(-) diff --git a/crates/rpc/rpc-api/src/engine.rs b/crates/rpc/rpc-api/src/engine.rs index 0faadc412b7..43d6b15b618 100644 --- a/crates/rpc/rpc-api/src/engine.rs +++ b/crates/rpc/rpc-api/src/engine.rs @@ -16,7 +16,7 @@ use alloy_rpc_types_engine::{ ForkchoiceUpdatedResponseV2, PayloadId, PayloadStatus, PayloadStatusV2, }; use alloy_rpc_types_eth::{ - state::StateOverride, BlockOverrides, EIP1186AccountProofResponse, Filter, SyncStatus, + state::StateOverride, BlockOverrides, EIP1186AccountProofResponse, Filter, Index, SyncStatus, }; use alloy_serde::JsonStorageKey; use jsonrpsee::{core::RpcResult, proc_macros::rpc, RpcModule}; @@ -430,6 +430,22 @@ pub trait EngineEthApi RpcResult>>; + /// Returns the EIP-2718 encoded transaction by block hash and transaction index position. + #[method(name = "getRawTransactionByBlockHashAndIndex")] + async fn raw_transaction_by_block_hash_and_index( + &self, + hash: B256, + index: Index, + ) -> RpcResult>; + + /// Returns the EIP-2718 encoded transaction by block number and transaction index position. + #[method(name = "getRawTransactionByBlockNumberAndIndex")] + async fn raw_transaction_by_block_number_and_index( + &self, + number: BlockNumberOrTag, + index: Index, + ) -> RpcResult>; + /// Sends signed transaction, returning its hash. #[method(name = "sendRawTransaction")] async fn send_raw_transaction(&self, bytes: Bytes) -> RpcResult; diff --git a/crates/rpc/rpc/src/engine.rs b/crates/rpc/rpc/src/engine.rs index c7ee36f0b7b..4b987ee7939 100644 --- a/crates/rpc/rpc/src/engine.rs +++ b/crates/rpc/rpc/src/engine.rs @@ -1,7 +1,7 @@ use alloy_eips::{BlockId, BlockNumberOrTag}; use alloy_primitives::{Address, Bytes, B256, U256, U64}; use alloy_rpc_types_eth::{ - state::StateOverride, BlockOverrides, EIP1186AccountProofResponse, Filter, SyncStatus, + state::StateOverride, BlockOverrides, EIP1186AccountProofResponse, Filter, Index, SyncStatus, }; use alloy_serde::JsonStorageKey; use jsonrpsee::core::RpcResult as Result; @@ -122,6 +122,30 @@ where self.eth.block_receipts(block_id).instrument(engine_span!()).await } + /// Handler for: `eth_getRawTransactionByBlockHashAndIndex` + async fn raw_transaction_by_block_hash_and_index( + &self, + hash: B256, + index: Index, + ) -> Result> { + self.eth + .raw_transaction_by_block_hash_and_index(hash, index) + .instrument(engine_span!()) + .await + } + + /// Handler for: `eth_getRawTransactionByBlockNumberAndIndex` + async fn raw_transaction_by_block_number_and_index( + &self, + number: BlockNumberOrTag, + index: Index, + ) -> Result> { + self.eth + .raw_transaction_by_block_number_and_index(number, index) + .instrument(engine_span!()) + .await + } + /// Handler for: `eth_sendRawTransaction` async fn send_raw_transaction(&self, bytes: Bytes) -> Result { self.eth.send_raw_transaction(bytes).instrument(engine_span!()).await From 64e5f9c6d749db16da09f067297253ee36edd929 Mon Sep 17 00:00:00 2001 From: Derek Cofausper <256792747+decofe@users.noreply.github.com> Date: Wed, 19 Aug 2026 12:50:29 +0000 Subject: [PATCH 046/294] refactor(chain-state): add trie data reference collectors (#26752) Co-authored-by: Matthias Seitz <19890894+mattsse@users.noreply.github.com> Co-authored-by: Matthias Seitz --- crates/chain-state/src/in_memory.rs | 30 +++++++++++++++++++ .../src/providers/database/provider.rs | 20 +++---------- 2 files changed, 34 insertions(+), 16 deletions(-) diff --git a/crates/chain-state/src/in_memory.rs b/crates/chain-state/src/in_memory.rs index fb68a6a6094..d58da478321 100644 --- a/crates/chain-state/src/in_memory.rs +++ b/crates/chain-state/src/in_memory.rs @@ -876,6 +876,21 @@ impl ExecutedBlock { self.trie_data().sorted.hashed_state } + /// Returns a reference to the hashed state result of the execution outcome. + /// + /// May wait for trie data if the deferred task hasn't completed. + #[inline] + pub fn hashed_state_ref(&self) -> &HashedPostStateSorted { + &self.trie_data.get().sorted.hashed_state + } + + /// Returns references to the hashed state results of the executed blocks. + /// + /// May wait for trie data if any deferred task hasn't completed. + pub fn hashed_state_refs(blocks: &[Self]) -> Vec<&HashedPostStateSorted> { + blocks.iter().map(Self::hashed_state_ref).collect() + } + /// Returns the trie updates resulting from the execution outcome. /// /// May wait for trie data if the deferred task hasn't completed. @@ -884,6 +899,21 @@ impl ExecutedBlock { self.trie_data().sorted.trie_updates } + /// Returns a reference to the trie updates resulting from the execution outcome. + /// + /// May wait for trie data if the deferred task hasn't completed. + #[inline] + pub fn trie_updates_ref(&self) -> &TrieUpdatesSorted { + &self.trie_data.get().sorted.trie_updates + } + + /// Returns references to the trie updates of the executed blocks. + /// + /// May wait for trie data if any deferred task hasn't completed. + pub fn trie_updates_refs(blocks: &[Self]) -> Vec<&TrieUpdatesSorted> { + blocks.iter().map(Self::trie_updates_ref).collect() + } + /// Returns a [`BlockNumber`] of the block. #[inline] pub fn block_number(&self) -> BlockNumber { diff --git a/crates/storage/provider/src/providers/database/provider.rs b/crates/storage/provider/src/providers/database/provider.rs index f29f84a5327..7c78f972ac7 100644 --- a/crates/storage/provider/src/providers/database/provider.rs +++ b/crates/storage/provider/src/providers/database/provider.rs @@ -774,14 +774,8 @@ impl DatabaseProvider>(); - let mask = state_trie_masking_blocks - .iter() - .map(|block| block.trie_data.get().sorted.hashed_state.as_ref()) - .collect::>(); + let batch = ExecutedBlock::hashed_state_refs(state_trie_blocks); + let mask = ExecutedBlock::hashed_state_refs(state_trie_masking_blocks); let merged_hashed_state = HashedPostStateSorted::disjointed_merge_batch(&batch, &mask); if !merged_hashed_state.is_empty() { @@ -790,14 +784,8 @@ impl DatabaseProvider>(); - let mask = state_trie_masking_blocks - .iter() - .map(|block| block.trie_data.get().sorted.trie_updates.as_ref()) - .collect::>(); + let batch = ExecutedBlock::trie_updates_refs(state_trie_blocks); + let mask = ExecutedBlock::trie_updates_refs(state_trie_masking_blocks); let merged_trie = TrieUpdatesSorted::disjointed_merge_batch(&batch, &mask); if !merged_trie.is_empty() { self.write_trie_updates_sorted(&merged_trie)?; From b7a7ed8c0a7ecd53845e83830b9e70b49b5d32c7 Mon Sep 17 00:00:00 2001 From: Derek Cofausper <256792747+decofe@users.noreply.github.com> Date: Wed, 19 Aug 2026 14:44:16 +0000 Subject: [PATCH 047/294] fix(provider): handle storage wipes in batched persistence (#26750) Co-authored-by: Matthias Seitz <19890894+mattsse@users.noreply.github.com> Co-authored-by: joshieDo <93316087+joshieDo@users.noreply.github.com> --- .../src/providers/database/provider.rs | 137 +++++++++++++++++- 1 file changed, 136 insertions(+), 1 deletion(-) diff --git a/crates/storage/provider/src/providers/database/provider.rs b/crates/storage/provider/src/providers/database/provider.rs index 7c78f972ac7..65a9c7edf0c 100644 --- a/crates/storage/provider/src/providers/database/provider.rs +++ b/crates/storage/provider/src/providers/database/provider.rs @@ -786,7 +786,25 @@ impl DatabaseProvider = test_block_builder.get_executed_blocks(1..4).collect(); + let address = B256::with_last_byte(1); + let storage_path = Nibbles::from_nibbles([0x1, 0x2]); + + let provider_rw = factory.provider_rw().unwrap(); + save_genesis(&provider_rw, &genesis).unwrap(); + provider_rw + .write_trie_updates_sorted(&TrieUpdatesSorted::new( + vec![], + B256Map::from_iter([( + address, + StorageTrieUpdatesSorted { + is_deleted: false, + storage_nodes: vec![(storage_path, Some(BranchNodeCompact::default()))], + }, + )]), + )) + .unwrap(); + provider_rw.commit().unwrap(); + + let wipe = TrieUpdatesSorted::new( + vec![], + B256Map::from_iter([( + address, + StorageTrieUpdatesSorted { is_deleted: true, storage_nodes: vec![] }, + )]), + ); + let wipe_block = &blocks[2]; + blocks[2] = ExecutedBlock::new( + Arc::clone(&wipe_block.recovered_block), + Arc::clone(&wipe_block.execution_output), + ComputedTrieData::new(Arc::new(HashedPostStateSorted::default()), Arc::new(wipe)), + ); + + let provider_rw = factory.provider_rw().unwrap(); + let input = SaveBlocksInput::new(blocks, 0, 0, 3, 3); + provider_rw.save_blocks(&input).unwrap(); + provider_rw.commit().unwrap(); + + let provider = factory.provider().unwrap(); + let checkpoint = provider.get_stage_checkpoint(StageId::Finish).unwrap().unwrap(); + assert_eq!(checkpoint.block_number, 3); + let storage_entries = provider + .tx_ref() + .cursor_dup_read::() + .unwrap() + .walk_dup(Some(address), None) + .unwrap() + .collect::, _>>() + .unwrap(); + assert!(storage_entries.is_empty()); + } + + #[cfg(feature = "partial-persistence")] + #[test] + fn test_save_blocks_batches_transient_storage_wipe() { + use alloy_primitives::map::B256Set; + use reth_trie::{updates::TrieUpdates, HashBuilder, HashedPostStateSorted}; + + let factory = create_test_provider_factory(); + factory.set_storage_settings_cache(StorageSettings::v2()); + + let mut test_block_builder = TestBlockBuilder::eth().with_state(); + let genesis = test_block_builder.get_executed_blocks(0..1).next().unwrap(); + let blocks: Vec<_> = test_block_builder.get_executed_blocks(1..4).collect(); + + let provider_rw = factory.provider_rw().unwrap(); + save_genesis(&provider_rw, &genesis).unwrap(); + provider_rw.commit().unwrap(); + + // A contract created and self-destructed in the same transaction leaves an empty whole- + // storage wipe in the trie updates, even though the account never reaches persisted state. + let ephemeral_account = B256::with_last_byte(0x57); + let hashed_state = HashedPostStateSorted::default(); + let mut trie_updates = TrieUpdates::default(); + trie_updates.finalize( + HashBuilder::default(), + Default::default(), + B256Set::from_iter([ephemeral_account]), + ); + let trie_updates = trie_updates.into_sorted(); + assert!(trie_updates.storage_tries_ref()[&ephemeral_account].is_deleted); + let selfdestruct_block = ExecutedBlock::new( + Arc::clone(&blocks[2].recovered_block), + Arc::clone(&blocks[2].execution_output), + ComputedTrieData::new(Arc::new(hashed_state), Arc::new(trie_updates)), + ); + + let provider_rw = factory.provider_rw().unwrap(); + let input = SaveBlocksInput::new( + vec![blocks[0].clone(), blocks[1].clone(), selfdestruct_block], + 0, + 0, + 3, + 3, + ); + provider_rw.save_blocks(&input).unwrap(); + provider_rw.commit().unwrap(); + + let checkpoint = + factory.provider().unwrap().get_stage_checkpoint(StageId::Finish).unwrap().unwrap(); + assert_eq!(checkpoint.block_number, 3); + } + #[cfg(feature = "partial-persistence")] #[test] fn test_save_blocks_partial_cycles_do_not_duplicate_static_file_writes() { From df69a6ac99af6e2f29461a5695b0b7df36919627 Mon Sep 17 00:00:00 2001 From: Arsenii Kulikov Date: Wed, 19 Aug 2026 17:47:43 +0000 Subject: [PATCH 048/294] fix(dev): canonicalize mined blocks immediately (#26761) --- crates/engine/local/src/miner.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/crates/engine/local/src/miner.rs b/crates/engine/local/src/miner.rs index c9047814a38..cc0ab6c7684 100644 --- a/crates/engine/local/src/miner.rs +++ b/crates/engine/local/src/miner.rs @@ -283,6 +283,8 @@ where self.last_block_hashes.pop_front(); } + self.update_forkchoice_state().await?; + Ok(()) } } From 27c8307bfd594e8105023c3b7f7d51c3b5851276 Mon Sep 17 00:00:00 2001 From: Francis Li Date: Thu, 20 Aug 2026 09:45:54 +0000 Subject: [PATCH 049/294] feat(payload): finalize pending build on resolve (#26759) Co-authored-by: Amp Co-authored-by: Arsenii Kulikov --- crates/payload/basic/src/lib.rs | 14 +++++--- crates/revm/src/cancelled.rs | 62 ++++++++++++++++++++++++++++++--- 2 files changed, 66 insertions(+), 10 deletions(-) diff --git a/crates/payload/basic/src/lib.rs b/crates/payload/basic/src/lib.rs index f9c18dfc00d..fb67012927f 100644 --- a/crates/payload/basic/src/lib.rs +++ b/crates/payload/basic/src/lib.rs @@ -416,7 +416,7 @@ where trace!(target: "payload_builder", id = %self.config.payload_id(), "spawn new payload build task"); let (tx, rx) = oneshot::channel(); let cancel = CancelOnDrop::default(); - let _cancel = cancel.clone(); + let pending_cancel = cancel.clone(); let guard = self.payload_task_guard.clone(); let payload_config = self.config.clone(); let best_payload = self.best_payload.payload().cloned(); @@ -446,7 +446,7 @@ where }); }); - self.pending_block = Some(PendingPayload { _cancel, payload: rx }); + self.pending_block = Some(PendingPayload { cancel: pending_cancel, payload: rx }); } } @@ -567,6 +567,10 @@ where let mut empty_payload = None; if best_payload.is_none() { + if let Some(pending) = maybe_better.as_ref() { + pending.cancel.request_finalization(); + } + debug!(target: "payload_builder", id=%self.config.payload_id(), "no best payload yet to resolve, building empty payload"); let args = BuildArguments { @@ -734,8 +738,8 @@ where /// A future that resolves to the result of the block building job. #[derive(Debug)] pub struct PendingPayload

{ - /// The marker to cancel the job on drop - _cancel: CancelOnDrop, + /// Cancels the job on drop and carries cooperative control signals. + cancel: CancelOnDrop, /// The channel to send the result to. payload: oneshot::Receiver, PayloadBuilderError>>, } @@ -746,7 +750,7 @@ impl

PendingPayload

{ cancel: CancelOnDrop, payload: oneshot::Receiver, PayloadBuilderError>>, ) -> Self { - Self { _cancel: cancel, payload } + Self { cancel, payload } } } diff --git a/crates/revm/src/cancelled.rs b/crates/revm/src/cancelled.rs index e535882355c..591cd3d376d 100644 --- a/crates/revm/src/cancelled.rs +++ b/crates/revm/src/cancelled.rs @@ -1,26 +1,53 @@ use alloc::sync::Arc; -use core::sync::atomic::AtomicBool; +use core::sync::atomic::{AtomicBool, AtomicU8, Ordering}; -/// A marker that can be used to cancel execution. +/// Payload building is still in progress. +const RUNNING: u8 = 0; +/// Payload building should stop accepting transactions and seal the accumulated work. +const FINALIZATION_REQUESTED: u8 = 1; +/// Payload building should stop and discard the accumulated work. +const CANCELLED: u8 = 2; + +/// Cancels execution on drop and supports cooperative finalization. /// /// If dropped, it will set the `cancelled` flag to true. /// /// This is most useful when a payload job needs to be cancelled. #[derive(Default, Clone, Debug)] -pub struct CancelOnDrop(Arc); +pub struct CancelOnDrop(Arc); // === impl CancelOnDrop === impl CancelOnDrop { + /// Returns true if the current work should be interrupted. + pub fn is_interrupted(&self) -> bool { + self.0.load(Ordering::Relaxed) != RUNNING + } + /// Returns true if the job was cancelled. pub fn is_cancelled(&self) -> bool { - self.0.load(core::sync::atomic::Ordering::Relaxed) + self.0.load(Ordering::Relaxed) == CANCELLED + } + + /// Requests that the current work be finalized without cancelling it. + pub fn request_finalization(&self) { + let _ = self.0.compare_exchange( + RUNNING, + FINALIZATION_REQUESTED, + Ordering::Relaxed, + Ordering::Relaxed, + ); + } + + /// Returns true if finalization was requested. + pub fn is_finalization_requested(&self) -> bool { + self.0.load(Ordering::Relaxed) == FINALIZATION_REQUESTED } } impl Drop for CancelOnDrop { fn drop(&mut self) { - self.0.store(true, core::sync::atomic::Ordering::Relaxed); + self.0.store(CANCELLED, Ordering::Relaxed); } } @@ -55,6 +82,7 @@ mod tests { #[test] fn test_default_cancelled() { let c = CancelOnDrop::default(); + assert!(!c.is_interrupted()); assert!(!c.is_cancelled()); } @@ -122,6 +150,7 @@ mod tests { drop(cancel); // The cloned instance should now see the cancelled flag as true + assert!(cloned_cancel.is_interrupted()); assert!(cloned_cancel.is_cancelled()); } @@ -140,8 +169,31 @@ mod tests { // Drop one clone - this should cancel all instances drop(clone1); + assert!(cancel.is_interrupted()); assert!(cancel.is_cancelled()); assert!(clone2.is_cancelled()); assert!(clone3.is_cancelled()); } + + #[test] + fn test_cancel_on_drop_finalization_request() { + let cancel = CancelOnDrop::default(); + let clone = cancel.clone(); + + cancel.request_finalization(); + + assert!(clone.is_interrupted()); + assert!(clone.is_finalization_requested()); + assert!(!clone.is_cancelled()); + + drop(cancel); + + assert!(clone.is_cancelled()); + assert!(!clone.is_finalization_requested()); + + clone.request_finalization(); + + assert!(clone.is_cancelled()); + assert!(!clone.is_finalization_requested()); + } } From a31ff8b46b2f2c5282cbc3fb4e55690c431a4263 Mon Sep 17 00:00:00 2001 From: Derek Cofausper <256792747+decofe@users.noreply.github.com> Date: Thu, 20 Aug 2026 12:28:19 +0000 Subject: [PATCH 050/294] chore: bump versions to 2.5.1 (#26771) Co-authored-by: Emma Jamieson-Hoare <21029500+emmajam@users.noreply.github.com> --- Cargo.lock | 228 +++++++++++++++++++-------------------- Cargo.toml | 2 +- docs/vocs/vocs.config.ts | 2 +- 3 files changed, 116 insertions(+), 116 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9e4d002ed6e..14b5a2e981e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3271,7 +3271,7 @@ dependencies = [ [[package]] name = "ef-test-runner" -version = "2.5.0" +version = "2.5.1" dependencies = [ "clap", "ef-tests", @@ -3279,7 +3279,7 @@ dependencies = [ [[package]] name = "ef-tests" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-consensus", "alloy-eips", @@ -3732,7 +3732,7 @@ dependencies = [ [[package]] name = "example-full-contract-state" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-primitives", "eyre", @@ -3850,7 +3850,7 @@ dependencies = [ [[package]] name = "exex-subscription" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-primitives", "clap", @@ -7654,7 +7654,7 @@ checksum = "1e061d1b48cb8d38042de4ae0a7a6401009d6143dc80d2e2d6f31f0bdd6470c7" [[package]] name = "reth" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-node-bindings", "alloy-primitives", @@ -7704,7 +7704,7 @@ dependencies = [ [[package]] name = "reth-basic-payload-builder" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-consensus", "alloy-eips", @@ -7730,7 +7730,7 @@ dependencies = [ [[package]] name = "reth-bb" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-consensus", "alloy-eips", @@ -7777,7 +7777,7 @@ dependencies = [ [[package]] name = "reth-chain-state" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-consensus", "alloy-eips", @@ -7808,7 +7808,7 @@ dependencies = [ [[package]] name = "reth-chainspec" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-chains", "alloy-consensus", @@ -7828,7 +7828,7 @@ dependencies = [ [[package]] name = "reth-cli" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-genesis", "clap", @@ -7840,7 +7840,7 @@ dependencies = [ [[package]] name = "reth-cli-commands" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-chains", "alloy-consensus", @@ -7931,7 +7931,7 @@ dependencies = [ [[package]] name = "reth-cli-runner" -version = "2.5.0" +version = "2.5.1" dependencies = [ "reth-tasks", "tokio", @@ -7940,7 +7940,7 @@ dependencies = [ [[package]] name = "reth-cli-util" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-eips", "alloy-primitives", @@ -7994,7 +7994,7 @@ dependencies = [ [[package]] name = "reth-config" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-primitives", "eyre", @@ -8012,7 +8012,7 @@ dependencies = [ [[package]] name = "reth-consensus" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-consensus", "alloy-eip7928", @@ -8026,7 +8026,7 @@ dependencies = [ [[package]] name = "reth-consensus-common" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-consensus", "alloy-eips", @@ -8040,7 +8040,7 @@ dependencies = [ [[package]] name = "reth-consensus-debug-client" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-consensus", "alloy-eips", @@ -8064,7 +8064,7 @@ dependencies = [ [[package]] name = "reth-db" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-consensus", "alloy-primitives", @@ -8100,7 +8100,7 @@ dependencies = [ [[package]] name = "reth-db-api" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-consensus", "alloy-primitives", @@ -8128,7 +8128,7 @@ dependencies = [ [[package]] name = "reth-db-common" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-consensus", "alloy-genesis", @@ -8159,7 +8159,7 @@ dependencies = [ [[package]] name = "reth-db-models" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-eips", "alloy-primitives", @@ -8175,7 +8175,7 @@ dependencies = [ [[package]] name = "reth-discv4" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-primitives", "alloy-rlp", @@ -8201,7 +8201,7 @@ dependencies = [ [[package]] name = "reth-discv5" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-primitives", "alloy-rlp", @@ -8226,7 +8226,7 @@ dependencies = [ [[package]] name = "reth-dns-discovery" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-chains", "alloy-primitives", @@ -8254,7 +8254,7 @@ dependencies = [ [[package]] name = "reth-downloaders" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-consensus", "alloy-eips", @@ -8292,7 +8292,7 @@ dependencies = [ [[package]] name = "reth-e2e-test-utils" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-consensus", "alloy-eips", @@ -8348,7 +8348,7 @@ dependencies = [ [[package]] name = "reth-ecies" -version = "2.5.0" +version = "2.5.1" dependencies = [ "aes", "alloy-primitives", @@ -8374,7 +8374,7 @@ dependencies = [ [[package]] name = "reth-engine-local" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-consensus", "alloy-primitives", @@ -8396,7 +8396,7 @@ dependencies = [ [[package]] name = "reth-engine-primitives" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-consensus", "alloy-eips", @@ -8423,7 +8423,7 @@ dependencies = [ [[package]] name = "reth-engine-tree" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-consensus", "alloy-eip7928", @@ -8493,7 +8493,7 @@ dependencies = [ [[package]] name = "reth-engine-util" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-consensus", "alloy-primitives", @@ -8522,7 +8522,7 @@ dependencies = [ [[package]] name = "reth-era" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-consensus", "alloy-eips", @@ -8547,7 +8547,7 @@ dependencies = [ [[package]] name = "reth-era-downloader" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-primitives", "bytes", @@ -8565,7 +8565,7 @@ dependencies = [ [[package]] name = "reth-era-utils" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-consensus", "alloy-primitives", @@ -8596,7 +8596,7 @@ dependencies = [ [[package]] name = "reth-errors" -version = "2.5.0" +version = "2.5.1" dependencies = [ "reth-consensus", "reth-execution-errors", @@ -8606,7 +8606,7 @@ dependencies = [ [[package]] name = "reth-eth-wire" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-chains", "alloy-consensus", @@ -8644,7 +8644,7 @@ dependencies = [ [[package]] name = "reth-eth-wire-types" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-chains", "alloy-consensus", @@ -8672,7 +8672,7 @@ dependencies = [ [[package]] name = "reth-ethereum" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-rpc-types-engine", "alloy-rpc-types-eth", @@ -8712,7 +8712,7 @@ dependencies = [ [[package]] name = "reth-ethereum-cli" -version = "2.5.0" +version = "2.5.1" dependencies = [ "clap", "eyre", @@ -8735,7 +8735,7 @@ dependencies = [ [[package]] name = "reth-ethereum-consensus" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-consensus", "alloy-eips", @@ -8751,7 +8751,7 @@ dependencies = [ [[package]] name = "reth-ethereum-engine-primitives" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-eips", "alloy-primitives", @@ -8767,7 +8767,7 @@ dependencies = [ [[package]] name = "reth-ethereum-forks" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-eip2124", "alloy-hardforks 0.4.8", @@ -8780,7 +8780,7 @@ dependencies = [ [[package]] name = "reth-ethereum-payload-builder" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-consensus", "alloy-eips", @@ -8809,7 +8809,7 @@ dependencies = [ [[package]] name = "reth-ethereum-primitives" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-consensus", "alloy-eips", @@ -8830,7 +8830,7 @@ dependencies = [ [[package]] name = "reth-etl" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-primitives", "rayon", @@ -8840,7 +8840,7 @@ dependencies = [ [[package]] name = "reth-evm" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-consensus", "alloy-eip7928", @@ -8866,7 +8866,7 @@ dependencies = [ [[package]] name = "reth-evm-ethereum" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-consensus", "alloy-eips", @@ -8891,7 +8891,7 @@ dependencies = [ [[package]] name = "reth-execution-cache" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-primitives", "fixed-cache", @@ -8909,7 +8909,7 @@ dependencies = [ [[package]] name = "reth-execution-errors" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-evm", "alloy-primitives", @@ -8921,7 +8921,7 @@ dependencies = [ [[package]] name = "reth-execution-types" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-consensus", "alloy-eips", @@ -8942,7 +8942,7 @@ dependencies = [ [[package]] name = "reth-exex" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-consensus", "alloy-eips", @@ -8987,7 +8987,7 @@ dependencies = [ [[package]] name = "reth-exex-test-utils" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-eips", "eyre", @@ -9018,7 +9018,7 @@ dependencies = [ [[package]] name = "reth-exex-types" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-eips", "alloy-primitives", @@ -9035,7 +9035,7 @@ dependencies = [ [[package]] name = "reth-fs-util" -version = "2.5.0" +version = "2.5.1" dependencies = [ "serde", "serde_json", @@ -9044,7 +9044,7 @@ dependencies = [ [[package]] name = "reth-invalid-block-hooks" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-consensus", "alloy-eips", @@ -9075,7 +9075,7 @@ dependencies = [ [[package]] name = "reth-ipc" -version = "2.5.0" +version = "2.5.1" dependencies = [ "bytes", "futures", @@ -9097,7 +9097,7 @@ dependencies = [ [[package]] name = "reth-libmdbx" -version = "2.5.0" +version = "2.5.1" dependencies = [ "bitflags 2.13.0", "byteorder", @@ -9114,7 +9114,7 @@ dependencies = [ [[package]] name = "reth-mdbx-sys" -version = "2.5.0" +version = "2.5.1" dependencies = [ "bindgen", "cc", @@ -9122,7 +9122,7 @@ dependencies = [ [[package]] name = "reth-metrics" -version = "2.5.0" +version = "2.5.1" dependencies = [ "futures", "libc", @@ -9135,7 +9135,7 @@ dependencies = [ [[package]] name = "reth-net-banlist" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-primitives", "ipnet", @@ -9143,7 +9143,7 @@ dependencies = [ [[package]] name = "reth-net-nat" -version = "2.5.0" +version = "2.5.1" dependencies = [ "futures-util", "if-addrs", @@ -9157,7 +9157,7 @@ dependencies = [ [[package]] name = "reth-network" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-consensus", "alloy-eip7928", @@ -9226,7 +9226,7 @@ dependencies = [ [[package]] name = "reth-network-api" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-consensus", "alloy-primitives", @@ -9250,7 +9250,7 @@ dependencies = [ [[package]] name = "reth-network-p2p" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-consensus", "alloy-eip7928", @@ -9274,7 +9274,7 @@ dependencies = [ [[package]] name = "reth-network-peers" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-primitives", "alloy-rlp", @@ -9291,7 +9291,7 @@ dependencies = [ [[package]] name = "reth-network-types" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-eip2124", "humantime-serde", @@ -9304,7 +9304,7 @@ dependencies = [ [[package]] name = "reth-nippy-jar" -version = "2.5.0" +version = "2.5.1" dependencies = [ "anyhow", "bincode", @@ -9322,7 +9322,7 @@ dependencies = [ [[package]] name = "reth-node-api" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-rpc-types-engine", "eyre", @@ -9345,7 +9345,7 @@ dependencies = [ [[package]] name = "reth-node-builder" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-consensus", "alloy-eips", @@ -9416,7 +9416,7 @@ dependencies = [ [[package]] name = "reth-node-core" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-consensus", "alloy-eips", @@ -9472,7 +9472,7 @@ dependencies = [ [[package]] name = "reth-node-ethereum" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-consensus", "alloy-contract", @@ -9541,7 +9541,7 @@ dependencies = [ [[package]] name = "reth-node-ethstats" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-consensus", "alloy-primitives", @@ -9564,7 +9564,7 @@ dependencies = [ [[package]] name = "reth-node-events" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-consensus", "alloy-eips", @@ -9587,7 +9587,7 @@ dependencies = [ [[package]] name = "reth-node-metrics" -version = "2.5.0" +version = "2.5.1" dependencies = [ "bytes", "eyre", @@ -9616,7 +9616,7 @@ dependencies = [ [[package]] name = "reth-node-types" -version = "2.5.0" +version = "2.5.1" dependencies = [ "reth-chainspec", "reth-db-api", @@ -9627,7 +9627,7 @@ dependencies = [ [[package]] name = "reth-payload-builder" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-consensus", "alloy-primitives", @@ -9650,7 +9650,7 @@ dependencies = [ [[package]] name = "reth-payload-builder-primitives" -version = "2.5.0" +version = "2.5.1" dependencies = [ "pin-project", "reth-payload-primitives", @@ -9661,7 +9661,7 @@ dependencies = [ [[package]] name = "reth-payload-primitives" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-consensus", "alloy-eips", @@ -9685,7 +9685,7 @@ dependencies = [ [[package]] name = "reth-payload-util" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-consensus", "alloy-primitives", @@ -9694,7 +9694,7 @@ dependencies = [ [[package]] name = "reth-payload-validator" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-consensus", "alloy-rpc-types-engine", @@ -9736,7 +9736,7 @@ dependencies = [ [[package]] name = "reth-provider" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-consensus", "alloy-eip7928", @@ -9790,7 +9790,7 @@ dependencies = [ [[package]] name = "reth-prune" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-consensus", "alloy-primitives", @@ -9821,7 +9821,7 @@ dependencies = [ [[package]] name = "reth-prune-types" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-primitives", "arbitrary", @@ -9841,7 +9841,7 @@ dependencies = [ [[package]] name = "reth-revm" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-consensus", "alloy-primitives", @@ -9857,7 +9857,7 @@ dependencies = [ [[package]] name = "reth-rpc" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-consensus", "alloy-dyn-abi", @@ -9937,7 +9937,7 @@ dependencies = [ [[package]] name = "reth-rpc-api" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-eips", "alloy-genesis", @@ -9967,7 +9967,7 @@ dependencies = [ [[package]] name = "reth-rpc-builder" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-eips", "alloy-network", @@ -10025,7 +10025,7 @@ dependencies = [ [[package]] name = "reth-rpc-convert" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-consensus", "alloy-evm", @@ -10045,7 +10045,7 @@ dependencies = [ [[package]] name = "reth-rpc-e2e-tests" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-genesis", "alloy-rpc-types-engine", @@ -10065,7 +10065,7 @@ dependencies = [ [[package]] name = "reth-rpc-engine-api" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-eips", "alloy-primitives", @@ -10101,7 +10101,7 @@ dependencies = [ [[package]] name = "reth-rpc-eth-api" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-consensus", "alloy-dyn-abi", @@ -10147,7 +10147,7 @@ dependencies = [ [[package]] name = "reth-rpc-eth-types" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-chains", "alloy-consensus", @@ -10199,7 +10199,7 @@ dependencies = [ [[package]] name = "reth-rpc-layer" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-rpc-types-engine", "brotli", @@ -10219,7 +10219,7 @@ dependencies = [ [[package]] name = "reth-rpc-server-types" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-eips", "alloy-primitives", @@ -10249,7 +10249,7 @@ dependencies = [ [[package]] name = "reth-stages" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-consensus", "alloy-eip7928", @@ -10312,7 +10312,7 @@ dependencies = [ [[package]] name = "reth-stages-api" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-eips", "alloy-primitives", @@ -10346,7 +10346,7 @@ dependencies = [ [[package]] name = "reth-stages-types" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-primitives", "arbitrary", @@ -10362,7 +10362,7 @@ dependencies = [ [[package]] name = "reth-static-file" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-primitives", "assert_matches", @@ -10385,7 +10385,7 @@ dependencies = [ [[package]] name = "reth-static-file-types" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-primitives", "clap", @@ -10403,7 +10403,7 @@ dependencies = [ [[package]] name = "reth-storage-api" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-consensus", "alloy-eip7928", @@ -10430,7 +10430,7 @@ dependencies = [ [[package]] name = "reth-storage-errors" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-eips", "alloy-primitives", @@ -10446,7 +10446,7 @@ dependencies = [ [[package]] name = "reth-storage-overlay" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-consensus", "alloy-eips", @@ -10475,7 +10475,7 @@ dependencies = [ [[package]] name = "reth-storage-rpc-provider" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-consensus", "alloy-eips", @@ -10504,7 +10504,7 @@ dependencies = [ [[package]] name = "reth-tasks" -version = "2.5.0" +version = "2.5.1" dependencies = [ "crossbeam-utils", "dashmap", @@ -10524,7 +10524,7 @@ dependencies = [ [[package]] name = "reth-testing-utils" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-consensus", "alloy-eips", @@ -10540,7 +10540,7 @@ dependencies = [ [[package]] name = "reth-tokio-util" -version = "2.5.0" +version = "2.5.1" dependencies = [ "tokio", "tokio-stream", @@ -10549,7 +10549,7 @@ dependencies = [ [[package]] name = "reth-tracing" -version = "2.5.0" +version = "2.5.1" dependencies = [ "clap", "eyre", @@ -10568,7 +10568,7 @@ dependencies = [ [[package]] name = "reth-tracing-otlp" -version = "2.5.0" +version = "2.5.1" dependencies = [ "base64 0.22.1", "clap", @@ -10586,7 +10586,7 @@ dependencies = [ [[package]] name = "reth-transaction-pool" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-consensus", "alloy-eips", @@ -10637,7 +10637,7 @@ dependencies = [ [[package]] name = "reth-trie" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-consensus", "alloy-eips", @@ -10669,7 +10669,7 @@ dependencies = [ [[package]] name = "reth-trie-common" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-consensus", "alloy-eips", @@ -10701,7 +10701,7 @@ dependencies = [ [[package]] name = "reth-trie-db" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-consensus", "alloy-primitives", @@ -10728,7 +10728,7 @@ dependencies = [ [[package]] name = "reth-trie-parallel" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-evm", "alloy-primitives", @@ -10756,7 +10756,7 @@ dependencies = [ [[package]] name = "reth-trie-sparse" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-primitives", "alloy-rlp", diff --git a/Cargo.toml b/Cargo.toml index 7b0b7346903..283f9b8d4c8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,5 +1,5 @@ [workspace.package] -version = "2.5.0" +version = "2.5.1" edition = "2024" rust-version = "1.95" license = "MIT OR Apache-2.0" diff --git a/docs/vocs/vocs.config.ts b/docs/vocs/vocs.config.ts index 29e1e41dae4..941f6e83ed6 100644 --- a/docs/vocs/vocs.config.ts +++ b/docs/vocs/vocs.config.ts @@ -23,7 +23,7 @@ export default defineConfig({ { text: 'Rustdocs', link: '/docs' }, { text: 'GitHub', link: 'https://github.com/paradigmxyz/reth' }, { - text: 'v2.5.0', + text: 'v2.5.1', items: [ { text: 'Releases', From 69b94b4e89a593ddf38be2a21dcbb6b61b98a110 Mon Sep 17 00:00:00 2001 From: Jimmy Chen Date: Thu, 20 Aug 2026 14:52:07 +0000 Subject: [PATCH 051/294] fix(engine): serve getBlobsV4 from Osaka (#26768) Co-authored-by: Matthias Seitz --- crates/rpc/rpc-engine-api/src/engine_api.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/rpc/rpc-engine-api/src/engine_api.rs b/crates/rpc/rpc-engine-api/src/engine_api.rs index 1b619bd53d6..3def16b346b 100644 --- a/crates/rpc/rpc-engine-api/src/engine_api.rs +++ b/crates/rpc/rpc-engine-api/src/engine_api.rs @@ -1164,7 +1164,7 @@ where let indices_bitarray = B128::from(u128::from_le_bytes(indices_bitarray.into())); let current_timestamp = SystemTime::now().duration_since(SystemTime::UNIX_EPOCH).unwrap_or_default().as_secs(); - if !self.inner.chain_spec.is_amsterdam_active_at_timestamp(current_timestamp) { + if !self.inner.chain_spec.is_osaka_active_at_timestamp(current_timestamp) { return Err(EngineApiError::EngineObjectValidationError( reth_payload_primitives::EngineObjectValidationError::UnsupportedFork, )); @@ -2224,7 +2224,7 @@ mod tests { #[tokio::test] async fn get_blobs_v4_returns_null_when_syncing() { let chain_spec: Arc = - Arc::new(ChainSpecBuilder::mainnet().amsterdam_activated().build()); + Arc::new(ChainSpecBuilder::mainnet().osaka_activated().build()); let provider = Arc::new(MockEthProvider::default()); let payload_store = spawn_test_payload_service::(); let (to_engine, _engine_rx) = unbounded_channel::>(); From dc83c609a8336c1d3e29b467ddbc9d896908bd14 Mon Sep 17 00:00:00 2001 From: Alexey Shekhirin Date: Thu, 20 Aug 2026 20:08:51 +0000 Subject: [PATCH 052/294] feat(node): testing node with persistent datadir (#26774) --- crates/node/builder/src/builder/mod.rs | 53 ++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/crates/node/builder/src/builder/mod.rs b/crates/node/builder/src/builder/mod.rs index d7952de9329..a8b8ec8b8e4 100644 --- a/crates/node/builder/src/builder/mod.rs +++ b/crates/node/builder/src/builder/mod.rs @@ -283,6 +283,33 @@ impl NodeBuilder { WithLaunchContext { builder: self.with_database(db), task_executor } } + + /// Creates a preconfigured test node whose datadir is preserved when the node is dropped. + /// + /// The caller owns cleanup of `datadir`. This is useful for tests that stop a node and launch + /// a new instance against the same database and static files. + #[cfg(feature = "test-utils")] + pub fn testing_node_with_persistent_datadir( + mut self, + task_executor: TaskExecutor, + datadir: impl Into, + ) -> WithLaunchContext, ChainSpec>> { + let path = reth_node_core::dirs::MaybePlatformPath::::from(datadir.into()); + self.config = self.config.with_datadir_args(reth_node_core::args::DatadirArgs { + datadir: path.clone(), + ..Default::default() + }); + + let data_dir = + path.unwrap_or_chain_default(self.config.chain.chain(), self.config.datadir.clone()); + let db_path = data_dir.data_dir().join("db"); + let db = reth_db::init_db(&db_path, reth_db::mdbx::DatabaseArguments::test()) + .unwrap_or_else(|error| { + panic!("could not create test database at {db_path:?}: {error}") + }); + + WithLaunchContext { builder: self.with_database(Arc::new(db)), task_executor } + } } impl NodeBuilder @@ -1042,3 +1069,29 @@ impl std::fmt::Debug for BuilderContext { .finish() } } + +#[cfg(all(test, feature = "test-utils"))] +mod tests { + use super::*; + use reth_chainspec::ChainSpec; + use reth_tasks::Runtime; + + #[test] + fn persistent_test_datadir_can_be_reopened() { + let root = tempfile::tempdir().unwrap(); + let datadir = root.path().join("node"); + let runtime = Runtime::test(); + + let config = || NodeConfig::new(Arc::new(ChainSpec::::default())); + let first = NodeBuilder::new(config()) + .testing_node_with_persistent_datadir(runtime.clone(), datadir.clone()); + assert!(datadir.join("db").exists()); + drop(first); + assert!(datadir.join("db").exists()); + + let reopened = NodeBuilder::new(config()) + .testing_node_with_persistent_datadir(runtime, datadir.clone()); + drop(reopened); + assert!(datadir.join("db").exists()); + } +} From d892c93b3abf15b7573ca600124024258cf888ec Mon Sep 17 00:00:00 2001 From: Arsenii Kulikov Date: Fri, 21 Aug 2026 09:09:14 +0000 Subject: [PATCH 053/294] feat(download): expose prepared snapshot context (#26777) --- .../cli/commands/src/download/config_gen.rs | 1 + crates/cli/commands/src/download/manifest.rs | 35 ++++++++++ crates/cli/commands/src/download/mod.rs | 68 +++++++++++++------ crates/cli/commands/src/download/planning.rs | 2 + crates/ethereum/cli/src/app.rs | 3 +- 5 files changed, 86 insertions(+), 23 deletions(-) diff --git a/crates/cli/commands/src/download/config_gen.rs b/crates/cli/commands/src/download/config_gen.rs index ed7ee2e904a..34f70b10af2 100644 --- a/crates/cli/commands/src/download/config_gen.rs +++ b/crates/cli/commands/src/download/config_gen.rs @@ -294,6 +294,7 @@ mod tests { base_url: None, reth_version: None, components: BTreeMap::new(), + extensions: Default::default(), } } diff --git a/crates/cli/commands/src/download/manifest.rs b/crates/cli/commands/src/download/manifest.rs index 174ec1c59d0..a0329024953 100644 --- a/crates/cli/commands/src/download/manifest.rs +++ b/crates/cli/commands/src/download/manifest.rs @@ -52,6 +52,12 @@ pub struct SnapshotManifest { pub reth_version: Option, /// Available snapshot components. pub components: BTreeMap, + /// Chain-specific manifest fields not interpreted by Reth. + /// + /// Extensions are retained so downstream commands can consume snapshot metadata without + /// refetching or reparsing the manifest selected by Reth. + #[serde(default, flatten)] + pub extensions: BTreeMap, } /// Manifest entry for a single snapshot component. @@ -728,6 +734,7 @@ pub fn generate_manifest( base_url: base_url.map(str::to_owned), reth_version: Some(reth_node_core::version::version_metadata().short_version.to_string()), components, + extensions: Default::default(), }) } @@ -1015,9 +1022,30 @@ mod tests { base_url: Some("https://example.com".to_string()), reth_version: None, components, + extensions: Default::default(), } } + #[test] + fn manifest_preserves_extensions() { + let manifest: SnapshotManifest = serde_json::from_str( + r#"{ + "block": 1, + "chain_id": 1, + "storage_version": 2, + "timestamp": 0, + "components": {}, + "consensus": { "archive": "consensus.tar.zst" } + }"#, + ) + .unwrap(); + + assert_eq!( + manifest.extensions.get("consensus"), + Some(&serde_json::json!({ "archive": "consensus.tar.zst" })) + ); + } + #[test] fn archive_urls_for_distance_all() { let m = test_manifest(); @@ -1067,6 +1095,7 @@ mod tests { base_url: Some("https://example.com".to_string()), reth_version: None, components, + extensions: Default::default(), }; let urls = m.archive_urls_for_distance(SnapshotComponentType::RocksdbIndices, Some(10)); @@ -1126,6 +1155,7 @@ mod tests { base_url: Some("https://example.com".to_string()), reth_version: None, components, + extensions: Default::default(), } } @@ -1218,6 +1248,7 @@ mod tests { base_url: Some("https://example.com".to_string()), reth_version: None, components, + extensions: Default::default(), }; let urls = m.archive_urls(SnapshotComponentType::StorageChangesets); assert_eq!(urls.len(), 49); @@ -1299,6 +1330,7 @@ mod tests { base_url: Some("https://example.com".to_string()), reth_version: None, components, + extensions: Default::default(), }; assert_eq!(manifest.output_size_for_distance(SnapshotComponentType::State, None), 1_000); @@ -1360,6 +1392,7 @@ mod tests { base_url: Some("https://example.com".to_string()), reth_version: None, components, + extensions: Default::default(), }; let state = m.snapshot_archives_for_distance(SnapshotComponentType::State, None); @@ -1477,6 +1510,7 @@ mod tests { base_url: Some("https://example.com/mainnet".to_string()), reth_version: None, components, + extensions: Default::default(), }; let urls = m.archive_urls(SnapshotComponentType::Headers); @@ -1560,6 +1594,7 @@ mod tests { base_url: Some("https://example.com/mainnet".to_string()), reth_version: None, components, + extensions: Default::default(), }; let ComponentManifest::Chunked(chunked) = diff --git a/crates/cli/commands/src/download/mod.rs b/crates/cli/commands/src/download/mod.rs index 3d00f4609c9..d7279afda21 100644 --- a/crates/cli/commands/src/download/mod.rs +++ b/crates/cli/commands/src/download/mod.rs @@ -456,24 +456,21 @@ pub struct DownloadCommand { impl> DownloadCommand { /// Runs the download command in single-archive or manifest mode. - pub async fn execute(self) -> Result<()> { + pub async fn execute(self) -> Result> { let chain = self.env.chain.chain(); - let chain_id = chain.id(); // --list: print available snapshots and exit if self.list { - let entries = fetch_snapshot_api_entries(chain_id).await?; - print_snapshot_listing(&entries, chain_id); - return Ok(()); + let entries = fetch_snapshot_api_entries(chain.id()).await?; + print_snapshot_listing(&entries, chain.id()); + return Ok(None); } - let data_dir = self.env.datadir.clone().resolve_datadir(chain); - - let cancel_token = CancellationToken::new(); - let _cancel_guard = cancel_token.drop_guard(); - // Legacy single-URL mode: download one archive and extract it if let Some(ref url) = self.url { + let cancel_token = CancellationToken::new(); + let _cancel_guard = cancel_token.drop_guard(); + let data_dir = self.env.datadir.clone().resolve_datadir(chain); let target_dir = data_dir.data_dir(); if self.force { clear_existing_datadir(target_dir)?; @@ -498,17 +495,22 @@ impl> DownloadCo .await?; info!(target: "reth::cli", "Snapshot downloaded and extracted successfully"); - return Ok(()); + return Ok(None); } let ResolvedDownload { manifest, selections, preset, planned } = - self.resolve_download(chain_id).await?; + self.resolve_download(chain.id()).await?; + let data_dir = self.env.datadir.clone().resolve_datadir(chain).data_dir().to_path_buf(); + let prepared = PreparedSnapshotDownload { manifest, data_dir }; if self.print_plan_json { - DownloadPlan::from_planned(&manifest, &planned).write_json(std::io::stdout().lock())?; - return Ok(()) + DownloadPlan::from_planned(&prepared.manifest, &planned) + .write_json(std::io::stdout().lock())?; + return Ok(Some(prepared)) } - let target_dir = data_dir.data_dir(); + let target_dir = prepared.data_dir.as_path(); + let cancel_token = CancellationToken::new(); + let _cancel_guard = cancel_token.drop_guard(); if self.force { clear_existing_datadir(target_dir)?; } @@ -535,16 +537,28 @@ impl> DownloadCo ) .await?; - self.finalize_modular_download(&selections, &manifest, preset, target_dir, &data_dir.db())?; + self.finalize_modular_download( + &selections, + &prepared.manifest, + preset, + target_dir, + &target_dir.join("db"), + )?; - Ok(()) + Ok(Some(prepared)) } - /// Resolves the exact modular archive plan without downloading or modifying the data dir. - pub async fn plan(&self) -> Result { - let chain_id = self.env.chain.chain().id(); - let resolved = self.resolve_download(chain_id).await?; - Ok(DownloadPlan::from_planned(&resolved.manifest, &resolved.planned)) + /// Resolves the exact modular archive plan and manifest context without downloading or + /// modifying the data dir. + pub async fn plan(&self) -> Result<(DownloadPlan, PreparedSnapshotDownload)> { + let chain = self.env.chain.chain(); + let resolved = self.resolve_download(chain.id()).await?; + let plan = DownloadPlan::from_planned(&resolved.manifest, &resolved.planned); + let prepared = PreparedSnapshotDownload { + manifest: resolved.manifest, + data_dir: self.env.datadir.clone().resolve_datadir(chain).data_dir().to_path_buf(), + }; + Ok((plan, prepared)) } async fn resolve_download(&self, chain_id: u64) -> Result { @@ -1034,6 +1048,15 @@ impl DownloadCommand { } } +/// A modular snapshot download after manifest and data-directory resolution. +#[derive(Debug)] +pub struct PreparedSnapshotDownload { + /// Manifest selected by the command, with a normalized `base_url`. + pub manifest: SnapshotManifest, + /// Chain-resolved directory where Reth installs the snapshot. + pub data_dir: PathBuf, +} + struct ResolvedDownload { manifest: SnapshotManifest, selections: BTreeMap, @@ -1089,6 +1112,7 @@ mod tests { base_url: Some("https://example.com".to_string()), reth_version: None, components, + extensions: Default::default(), } } diff --git a/crates/cli/commands/src/download/planning.rs b/crates/cli/commands/src/download/planning.rs index f087926d7d0..22bb8b96bc9 100644 --- a/crates/cli/commands/src/download/planning.rs +++ b/crates/cli/commands/src/download/planning.rs @@ -418,6 +418,7 @@ mod tests { base_url: Some("https://example.com".to_string()), reth_version: None, components, + extensions: Default::default(), }; let selections = BTreeMap::from([ @@ -494,6 +495,7 @@ mod tests { base_url: Some("https://example.com/mainnet".to_string()), reth_version: None, components, + extensions: Default::default(), }; let selections = BTreeMap::from([(SnapshotComponentType::Transactions, ComponentSelection::All)]); diff --git a/crates/ethereum/cli/src/app.rs b/crates/ethereum/cli/src/app.rs index cb3d732757a..44fe4dbde98 100644 --- a/crates/ethereum/cli/src/app.rs +++ b/crates/ethereum/cli/src/app.rs @@ -201,7 +201,8 @@ where Commands::Db(command) => { runner.run_blocking_command_until_exit(|ctx| command.execute::(ctx)) } - Commands::Download(command) => runner.run_blocking_until_ctrl_c(command.execute::()), + Commands::Download(command) => runner + .run_blocking_until_ctrl_c(async move { command.execute::().await.map(|_| ()) }), Commands::SnapshotManifest(command) => command.execute(), Commands::Stage(command) => { runner.run_command_until_exit(|ctx| command.execute::(ctx, components)) From 7b3432d950030875b23c5c4df7c9f53a7caa4868 Mon Sep 17 00:00:00 2001 From: monem <119044801+pucedoteth@users.noreply.github.com> Date: Fri, 21 Aug 2026 14:53:24 +0000 Subject: [PATCH 054/294] fix(rpc): reject block timestamp overflow in eth_simulateV1 (#26767) Co-authored-by: Matthias Seitz --- crates/rpc/rpc-eth-types/src/simulate.rs | 55 +++++++++++++++++++++++- 1 file changed, 53 insertions(+), 2 deletions(-) diff --git a/crates/rpc/rpc-eth-types/src/simulate.rs b/crates/rpc/rpc-eth-types/src/simulate.rs index 15f96d45f74..90e88592c6f 100644 --- a/crates/rpc/rpc-eth-types/src/simulate.rs +++ b/crates/rpc/rpc-eth-types/src/simulate.rs @@ -216,7 +216,13 @@ where if gap > 1 { for i in 1..gap { let filler_number = prev_number + i; - let filler_time = prev_timestamp + timestamp_increment; + let filler_time = + prev_timestamp.checked_add(timestamp_increment).ok_or_else(|| { + EthApiError::other(EthSimulateError::BlockTimestampInvalid { + got: prev_timestamp, + parent: prev_timestamp, + }) + })?; out.push(SimBlock { block_overrides: Some(BlockOverrides { number: Some(U256::from(filler_number)), @@ -241,7 +247,12 @@ where } t } else { - let t = prev_timestamp + timestamp_increment; + let t = prev_timestamp.checked_add(timestamp_increment).ok_or_else(|| { + EthApiError::other(EthSimulateError::BlockTimestampInvalid { + got: prev_timestamp, + parent: prev_timestamp, + }) + })?; overrides.time = Some(t); t }; @@ -755,6 +766,46 @@ mod tests { assert!(matches!(err, EthApiError::Other(_))); } + #[test] + fn sanitize_chain_rejects_timestamp_overflow() { + // A block may set any timestamp above its parent's, including `u64::MAX`. The following + // block then defaults to `prev + increment`, which must not wrap. + let parent = parent_at(0, 0); + let blocks: Vec> = vec![ + SimBlock { + block_overrides: Some(BlockOverrides { + time: Some(u64::MAX), + ..Default::default() + }), + ..Default::default() + }, + SimBlock::default(), + ]; + + let err = sanitize_chain(blocks, &parent, Chain::mainnet().id(), 256).unwrap_err(); + assert!(matches!(err, EthApiError::Other(_))); + } + + #[test] + fn sanitize_chain_rejects_filler_timestamp_overflow() { + // Same, but the wrap would happen while generating filler blocks for a number gap. + let parent = parent_at(0, 0); + let blocks = vec![ + SimBlock { + block_overrides: Some(BlockOverrides { + number: Some(U256::from(1)), + time: Some(u64::MAX), + ..Default::default() + }), + ..Default::default() + }, + block_with_number(4), + ]; + + let err = sanitize_chain(blocks, &parent, Chain::mainnet().id(), 256).unwrap_err(); + assert!(matches!(err, EthApiError::Other(_))); + } + #[test] fn sanitize_chain_enforces_max_blocks() { let parent = parent_at(0, 0); From de401d9b403481fdfe86a6d1841f77d460f41fc9 Mon Sep 17 00:00:00 2001 From: will-2012 <117156346+will-2012@users.noreply.github.com> Date: Mon, 24 Aug 2026 11:28:03 +0000 Subject: [PATCH 055/294] fix(rpc): stop cancelled blocking IO tasks (#26776) Co-authored-by: Matthias Seitz --- .../rpc-eth-api/src/helpers/blocking_task.rs | 18 +++++++++++++++--- crates/rpc/rpc/src/eth/filter.rs | 16 ++++++++++++---- 2 files changed, 27 insertions(+), 7 deletions(-) diff --git a/crates/rpc/rpc-eth-api/src/helpers/blocking_task.rs b/crates/rpc/rpc-eth-api/src/helpers/blocking_task.rs index de29c934609..5336d016455 100644 --- a/crates/rpc/rpc-eth-api/src/helpers/blocking_task.rs +++ b/crates/rpc/rpc-eth-api/src/helpers/blocking_task.rs @@ -164,6 +164,9 @@ pub trait SpawnBlocking: EthApiTypes + Clone + Send + Sync + 'static { let (tx, rx) = oneshot::channel(); let this = self.clone(); self.io_task_spawner().spawn_blocking_task(async move { + if tx.is_closed() { + return + } let res = f(this); let _ = tx.send(res); }); @@ -184,11 +187,20 @@ pub trait SpawnBlocking: EthApiTypes + Clone + Send + Sync + 'static { F: FnOnce(Self) -> Fut + Send + 'static, R: Send + 'static, { - let (tx, rx) = oneshot::channel(); + let (mut tx, rx) = oneshot::channel(); let this = self.clone(); self.io_task_spawner().spawn_blocking_task(async move { - let res = f(this).await; - let _ = tx.send(res); + let fut = f(this); + tokio::pin!(fut); + let res = tokio::select! { + // Futures executed here may perform blocking work before their first yield. + biased; + _ = tx.closed() => None, + res = &mut fut => Some(res), + }; + if let Some(res) = res { + let _ = tx.send(res); + } }); async move { rx.await.map_err(|_| EthApiError::InternalEthError)? } diff --git a/crates/rpc/rpc/src/eth/filter.rs b/crates/rpc/rpc/src/eth/filter.rs index f201b2f9418..da5a5d5da66 100644 --- a/crates/rpc/rpc/src/eth/filter.rs +++ b/crates/rpc/rpc/src/eth/filter.rs @@ -658,12 +658,20 @@ where return Err(EthFilterError::QueryExceedsMaxBlocks(max_blocks_per_filter)) } - let (tx, rx) = oneshot::channel(); + let (mut tx, rx) = oneshot::channel(); let this = self.clone(); self.task_spawner.spawn_blocking_task(async move { - let res = - this.get_logs_in_block_range_inner(&filter, from_block, to_block, limits).await; - let _ = tx.send(res); + let fut = this.get_logs_in_block_range_inner(&filter, from_block, to_block, limits); + tokio::pin!(fut); + let res = tokio::select! { + // Range scans perform blocking reads before their first yield. + biased; + _ = tx.closed() => None, + res = &mut fut => Some(res), + }; + if let Some(res) = res { + let _ = tx.send(res); + } }); rx.await.map_err(|_| EthFilterError::InternalError)? From b160aa942a2e1af092c71e440bd9242eadea0174 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 24 Aug 2026 11:40:21 +0000 Subject: [PATCH 056/294] chore(deps): bump the ci-weekly group with 4 updates (#26785) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/bench-scheduled.yml | 4 +-- .github/workflows/bench.yml | 4 +-- .github/workflows/check-alloy.yml | 2 +- .github/workflows/compact.yml | 2 +- .github/workflows/docker-test.yml | 2 +- .github/workflows/e2e.yml | 8 +++--- .github/workflows/integration.yml | 8 +++--- .github/workflows/lint.yml | 32 +++++++++++----------- .github/workflows/release-reproducible.yml | 2 +- .github/workflows/release.yml | 2 +- .github/workflows/reproducible-build.yml | 2 +- .github/workflows/stage.yml | 2 +- .github/workflows/sync-era.yml | 2 +- .github/workflows/sync.yml | 2 +- .github/workflows/unit.yml | 10 +++---- 15 files changed, 42 insertions(+), 42 deletions(-) diff --git a/.github/workflows/bench-scheduled.yml b/.github/workflows/bench-scheduled.yml index 45562f0babd..5e08ac69436 100644 --- a/.github/workflows/bench-scheduled.yml +++ b/.github/workflows/bench-scheduled.yml @@ -347,11 +347,11 @@ jobs: const jobUrl = job ? job.html_url : `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`; core.exportVariable('BENCH_JOB_URL', jobUrl); - - uses: rui314/setup-mold@9c9c13bf4c3f1adef0cc596abc155580bcb04444 # v1 + - uses: rui314/setup-mold@7e4f20ad28a2e8ca6fd0892ccf72e2abb706b9c3 # v1 - uses: dtolnay/rust-toolchain@stable - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 continue-on-error: true - - uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0 + - uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1 - name: Install dependencies env: diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index 7113a02c8f4..4ed409f4c94 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -933,11 +933,11 @@ jobs: }); } - - uses: rui314/setup-mold@9c9c13bf4c3f1adef0cc596abc155580bcb04444 # v1 + - uses: rui314/setup-mold@7e4f20ad28a2e8ca6fd0892ccf72e2abb706b9c3 # v1 - uses: dtolnay/rust-toolchain@stable - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 continue-on-error: true - - uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0 + - uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1 - name: Mask OTLP endpoints uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 diff --git a/.github/workflows/check-alloy.yml b/.github/workflows/check-alloy.yml index 59d53f851ea..93df79f1dbb 100644 --- a/.github/workflows/check-alloy.yml +++ b/.github/workflows/check-alloy.yml @@ -35,7 +35,7 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - - uses: rui314/setup-mold@9c9c13bf4c3f1adef0cc596abc155580bcb04444 # v1 + - uses: rui314/setup-mold@7e4f20ad28a2e8ca6fd0892ccf72e2abb706b9c3 # v1 - run: .github/scripts/install_llvm.sh ubuntu diff --git a/.github/workflows/compact.yml b/.github/workflows/compact.yml index 5dc14b85d31..52ad1ff8374 100644 --- a/.github/workflows/compact.yml +++ b/.github/workflows/compact.yml @@ -32,7 +32,7 @@ jobs: uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - - uses: rui314/setup-mold@9c9c13bf4c3f1adef0cc596abc155580bcb04444 # v1 + - uses: rui314/setup-mold@7e4f20ad28a2e8ca6fd0892ccf72e2abb706b9c3 # v1 - run: .github/scripts/install_llvm.sh ubuntu - uses: dtolnay/rust-toolchain@stable - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 diff --git a/.github/workflows/docker-test.yml b/.github/workflows/docker-test.yml index 91b57deebf7..bda764e0c6f 100644 --- a/.github/workflows/docker-test.yml +++ b/.github/workflows/docker-test.yml @@ -68,7 +68,7 @@ jobs: # Docker build (forks) - name: Set up Docker Buildx if: steps.fork.outputs.is_fork == 'true' - uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0 + uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4.3.0 - name: Build reth image (Docker) if: steps.fork.outputs.is_fork == 'true' diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index a55655b01f9..40ae0fb086c 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -32,11 +32,11 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - - uses: rui314/setup-mold@9c9c13bf4c3f1adef0cc596abc155580bcb04444 # v1 + - uses: rui314/setup-mold@7e4f20ad28a2e8ca6fd0892ccf72e2abb706b9c3 # v1 - run: .github/scripts/install_llvm.sh ubuntu - uses: dtolnay/rust-toolchain@stable - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 - - uses: taiki-e/install-action@7f4eb899022d8fe70b20c4f3de697aa85c309026 # v2.85.11 + - uses: taiki-e/install-action@b6b84cf49ebfe0176417bdce007c624f0db37f20 # v2.86.2 with: tool: nextest - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 @@ -67,11 +67,11 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - - uses: rui314/setup-mold@9c9c13bf4c3f1adef0cc596abc155580bcb04444 # v1 + - uses: rui314/setup-mold@7e4f20ad28a2e8ca6fd0892ccf72e2abb706b9c3 # v1 - run: .github/scripts/install_llvm.sh ubuntu - uses: dtolnay/rust-toolchain@stable - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 - - uses: taiki-e/install-action@7f4eb899022d8fe70b20c4f3de697aa85c309026 # v2.85.11 + - uses: taiki-e/install-action@b6b84cf49ebfe0176417bdce007c624f0db37f20 # v2.86.2 with: tool: nextest - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index e324f0139dd..dde997fa0e6 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -39,12 +39,12 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - - uses: rui314/setup-mold@9c9c13bf4c3f1adef0cc596abc155580bcb04444 # v1 + - uses: rui314/setup-mold@7e4f20ad28a2e8ca6fd0892ccf72e2abb706b9c3 # v1 - run: .github/scripts/install_llvm.sh ubuntu - uses: dtolnay/rust-toolchain@stable - name: Install Geth run: .github/scripts/install_geth.sh - - uses: taiki-e/install-action@7f4eb899022d8fe70b20c4f3de697aa85c309026 # v2.85.11 + - uses: taiki-e/install-action@b6b84cf49ebfe0176417bdce007c624f0db37f20 # v2.86.2 with: tool: nextest - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 @@ -82,10 +82,10 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - - uses: rui314/setup-mold@9c9c13bf4c3f1adef0cc596abc155580bcb04444 # v1 + - uses: rui314/setup-mold@7e4f20ad28a2e8ca6fd0892ccf72e2abb706b9c3 # v1 - run: .github/scripts/install_llvm.sh ubuntu - uses: dtolnay/rust-toolchain@stable - - uses: taiki-e/install-action@7f4eb899022d8fe70b20c4f3de697aa85c309026 # v2.85.11 + - uses: taiki-e/install-action@b6b84cf49ebfe0176417bdce007c624f0db37f20 # v2.86.2 with: tool: nextest - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index b87e294f04c..30fc52850ed 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -29,7 +29,7 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - - uses: rui314/setup-mold@9c9c13bf4c3f1adef0cc596abc155580bcb04444 # v1 + - uses: rui314/setup-mold@7e4f20ad28a2e8ca6fd0892ccf72e2abb706b9c3 # v1 - run: .github/scripts/install_llvm.sh ubuntu - uses: dtolnay/rust-toolchain@clippy with: @@ -57,7 +57,7 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - - uses: rui314/setup-mold@9c9c13bf4c3f1adef0cc596abc155580bcb04444 # v1 + - uses: rui314/setup-mold@7e4f20ad28a2e8ca6fd0892ccf72e2abb706b9c3 # v1 - run: .github/scripts/install_llvm.sh ubuntu - uses: dtolnay/rust-toolchain@nightly with: @@ -79,11 +79,11 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - - uses: rui314/setup-mold@9c9c13bf4c3f1adef0cc596abc155580bcb04444 # v1 + - uses: rui314/setup-mold@7e4f20ad28a2e8ca6fd0892ccf72e2abb706b9c3 # v1 - uses: dtolnay/rust-toolchain@stable with: target: wasm32-wasip1 - - uses: taiki-e/install-action@7f4eb899022d8fe70b20c4f3de697aa85c309026 # v2.85.11 + - uses: taiki-e/install-action@b6b84cf49ebfe0176417bdce007c624f0db37f20 # v2.86.2 with: tool: cargo-hack - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 @@ -105,11 +105,11 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - - uses: rui314/setup-mold@9c9c13bf4c3f1adef0cc596abc155580bcb04444 # v1 + - uses: rui314/setup-mold@7e4f20ad28a2e8ca6fd0892ccf72e2abb706b9c3 # v1 - uses: dtolnay/rust-toolchain@stable with: target: riscv32imac-unknown-none-elf - - uses: taiki-e/install-action@7f4eb899022d8fe70b20c4f3de697aa85c309026 # v2.85.11 + - uses: taiki-e/install-action@b6b84cf49ebfe0176417bdce007c624f0db37f20 # v2.86.2 with: tool: cargo-hack - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 @@ -134,10 +134,10 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - - uses: rui314/setup-mold@9c9c13bf4c3f1adef0cc596abc155580bcb04444 # v1 + - uses: rui314/setup-mold@7e4f20ad28a2e8ca6fd0892ccf72e2abb706b9c3 # v1 - run: .github/scripts/install_llvm.sh ubuntu - uses: dtolnay/rust-toolchain@stable - - uses: taiki-e/install-action@7f4eb899022d8fe70b20c4f3de697aa85c309026 # v2.85.11 + - uses: taiki-e/install-action@b6b84cf49ebfe0176417bdce007c624f0db37f20 # v2.86.2 with: tool: cargo-hack - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 @@ -156,7 +156,7 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - - uses: rui314/setup-mold@9c9c13bf4c3f1adef0cc596abc155580bcb04444 # v1 + - uses: rui314/setup-mold@7e4f20ad28a2e8ca6fd0892ccf72e2abb706b9c3 # v1 - run: .github/scripts/install_llvm.sh ubuntu - uses: dtolnay/rust-toolchain@master with: @@ -179,7 +179,7 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - - uses: rui314/setup-mold@9c9c13bf4c3f1adef0cc596abc155580bcb04444 # v1 + - uses: rui314/setup-mold@7e4f20ad28a2e8ca6fd0892ccf72e2abb706b9c3 # v1 - run: .github/scripts/install_llvm.sh ubuntu - uses: dtolnay/rust-toolchain@nightly - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 @@ -202,7 +202,7 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - - uses: rui314/setup-mold@9c9c13bf4c3f1adef0cc596abc155580bcb04444 # v1 + - uses: rui314/setup-mold@7e4f20ad28a2e8ca6fd0892ccf72e2abb706b9c3 # v1 - uses: dtolnay/rust-toolchain@nightly with: components: rustfmt @@ -220,14 +220,14 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - - uses: rui314/setup-mold@9c9c13bf4c3f1adef0cc596abc155580bcb04444 # v1 + - uses: rui314/setup-mold@7e4f20ad28a2e8ca6fd0892ccf72e2abb706b9c3 # v1 - run: .github/scripts/install_llvm.sh ubuntu - uses: dtolnay/rust-toolchain@nightly - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 with: cache-on-failure: true - - uses: taiki-e/install-action@7f4eb899022d8fe70b20c4f3de697aa85c309026 # v2.85.11 + - uses: taiki-e/install-action@b6b84cf49ebfe0176417bdce007c624f0db37f20 # v2.86.2 with: tool: cargo-udeps - run: cargo udeps --workspace --lib --examples --tests --benches --all-features --locked @@ -242,7 +242,7 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - - uses: rui314/setup-mold@9c9c13bf4c3f1adef0cc596abc155580bcb04444 # v1 + - uses: rui314/setup-mold@7e4f20ad28a2e8ca6fd0892ccf72e2abb706b9c3 # v1 - run: .github/scripts/install_llvm.sh ubuntu - uses: dtolnay/rust-toolchain@nightly - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 @@ -305,7 +305,7 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - - uses: rui314/setup-mold@9c9c13bf4c3f1adef0cc596abc155580bcb04444 # v1 + - uses: rui314/setup-mold@7e4f20ad28a2e8ca6fd0892ccf72e2abb706b9c3 # v1 - uses: dtolnay/rust-toolchain@stable - name: Ensure no arbitrary or proptest dependency on default build run: cargo tree --package reth -e=features,no-dev | grep -Eq "arbitrary|proptest" && exit 1 || exit 0 @@ -323,7 +323,7 @@ jobs: - run: .github/scripts/install_llvm.sh ubuntu - uses: dtolnay/rust-toolchain@stable - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 - - uses: rui314/setup-mold@9c9c13bf4c3f1adef0cc596abc155580bcb04444 # v1 + - uses: rui314/setup-mold@7e4f20ad28a2e8ca6fd0892ccf72e2abb706b9c3 # v1 - uses: taiki-e/cache-cargo-install-action@9ee83daaa7b96a6fab930949ecf1122bba04a389 # v3.0.8 with: tool: zepter diff --git a/.github/workflows/release-reproducible.yml b/.github/workflows/release-reproducible.yml index 93c46014e33..98adf750cfb 100644 --- a/.github/workflows/release-reproducible.yml +++ b/.github/workflows/release-reproducible.yml @@ -55,7 +55,7 @@ jobs: ref: ${{ needs.extract-version.outputs.VERSION }} - name: Set up Docker Buildx - uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0 + uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4.3.0 - name: Log in to GitHub Container Registry uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 3e3e6fa753b..5dc76c32844 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -112,7 +112,7 @@ jobs: with: persist-credentials: false ref: ${{ env.RELEASE_REF }} - - uses: rui314/setup-mold@9c9c13bf4c3f1adef0cc596abc155580bcb04444 # v1 + - uses: rui314/setup-mold@7e4f20ad28a2e8ca6fd0892ccf72e2abb706b9c3 # v1 - uses: dtolnay/rust-toolchain@stable with: target: ${{ matrix.configs.target }} diff --git a/.github/workflows/reproducible-build.yml b/.github/workflows/reproducible-build.yml index 5251696c777..58980f10012 100644 --- a/.github/workflows/reproducible-build.yml +++ b/.github/workflows/reproducible-build.yml @@ -31,7 +31,7 @@ jobs: target: x86_64-unknown-linux-gnu - name: Set up Docker Buildx - uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0 + uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4.3.0 - name: Build reproducible binary with Docker run: | diff --git a/.github/workflows/stage.yml b/.github/workflows/stage.yml index 66454965ef5..bb5c2422482 100644 --- a/.github/workflows/stage.yml +++ b/.github/workflows/stage.yml @@ -36,7 +36,7 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - - uses: rui314/setup-mold@9c9c13bf4c3f1adef0cc596abc155580bcb04444 # v1 + - uses: rui314/setup-mold@7e4f20ad28a2e8ca6fd0892ccf72e2abb706b9c3 # v1 - run: .github/scripts/install_llvm.sh ubuntu - uses: dtolnay/rust-toolchain@stable - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 diff --git a/.github/workflows/sync-era.yml b/.github/workflows/sync-era.yml index 66684d314e3..87774748908 100644 --- a/.github/workflows/sync-era.yml +++ b/.github/workflows/sync-era.yml @@ -41,7 +41,7 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - - uses: rui314/setup-mold@9c9c13bf4c3f1adef0cc596abc155580bcb04444 # v1 + - uses: rui314/setup-mold@7e4f20ad28a2e8ca6fd0892ccf72e2abb706b9c3 # v1 - run: .github/scripts/install_llvm.sh ubuntu - uses: dtolnay/rust-toolchain@stable - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 diff --git a/.github/workflows/sync.yml b/.github/workflows/sync.yml index e74797669e8..dd1f3ef826f 100644 --- a/.github/workflows/sync.yml +++ b/.github/workflows/sync.yml @@ -41,7 +41,7 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - - uses: rui314/setup-mold@9c9c13bf4c3f1adef0cc596abc155580bcb04444 # v1 + - uses: rui314/setup-mold@7e4f20ad28a2e8ca6fd0892ccf72e2abb706b9c3 # v1 - run: .github/scripts/install_llvm.sh ubuntu - uses: dtolnay/rust-toolchain@stable - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 diff --git a/.github/workflows/unit.yml b/.github/workflows/unit.yml index d2153a5bc41..f844ddfeb75 100644 --- a/.github/workflows/unit.yml +++ b/.github/workflows/unit.yml @@ -39,7 +39,7 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - - uses: rui314/setup-mold@9c9c13bf4c3f1adef0cc596abc155580bcb04444 # v1 + - uses: rui314/setup-mold@7e4f20ad28a2e8ca6fd0892ccf72e2abb706b9c3 # v1 - run: .github/scripts/install_llvm.sh ubuntu - uses: dtolnay/rust-toolchain@stable - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 @@ -47,7 +47,7 @@ jobs: with: cache-targets: false cache-on-failure: true - - uses: taiki-e/install-action@7f4eb899022d8fe70b20c4f3de697aa85c309026 # v2.85.11 + - uses: taiki-e/install-action@b6b84cf49ebfe0176417bdce007c624f0db37f20 # v2.86.2 with: tool: nextest - if: "${{ matrix.type == 'book' }}" @@ -94,10 +94,10 @@ jobs: mkdir -p testing/ef-tests/execution-spec-tests URL="https://github.com/ethereum/execution-spec-tests/releases/download/${EEST_TESTS_TAG}/fixtures_stable.tar.gz" curl -L "$URL" | tar -xz --strip-components=1 -C testing/ef-tests/execution-spec-tests - - uses: rui314/setup-mold@9c9c13bf4c3f1adef0cc596abc155580bcb04444 # v1 + - uses: rui314/setup-mold@7e4f20ad28a2e8ca6fd0892ccf72e2abb706b9c3 # v1 - run: .github/scripts/install_llvm.sh ubuntu - uses: dtolnay/rust-toolchain@stable - - uses: taiki-e/install-action@7f4eb899022d8fe70b20c4f3de697aa85c309026 # v2.85.11 + - uses: taiki-e/install-action@b6b84cf49ebfe0176417bdce007c624f0db37f20 # v2.86.2 with: tool: nextest - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 @@ -118,7 +118,7 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - - uses: rui314/setup-mold@9c9c13bf4c3f1adef0cc596abc155580bcb04444 # v1 + - uses: rui314/setup-mold@7e4f20ad28a2e8ca6fd0892ccf72e2abb706b9c3 # v1 - run: .github/scripts/install_llvm.sh ubuntu - uses: dtolnay/rust-toolchain@stable - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 From 97fba9d3d2320cf3ae3d3fa662c8d99259de1c66 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Mon, 24 Aug 2026 11:50:34 +0000 Subject: [PATCH 057/294] feat(txpool): add consensus encoding helper (#26739) --- crates/rpc/rpc-engine-api/src/engine_api.rs | 11 +++++----- crates/transaction-pool/src/maintain.rs | 9 ++------ .../transaction-pool/src/test_utils/mock.rs | 5 +++++ crates/transaction-pool/src/traits.rs | 22 +++++++++++++++++-- crates/transaction-pool/src/validate/mod.rs | 7 +++++- 5 files changed, 38 insertions(+), 16 deletions(-) diff --git a/crates/rpc/rpc-engine-api/src/engine_api.rs b/crates/rpc/rpc-engine-api/src/engine_api.rs index 3def16b346b..a012836b3a3 100644 --- a/crates/rpc/rpc-engine-api/src/engine_api.rs +++ b/crates/rpc/rpc-engine-api/src/engine_api.rs @@ -6,7 +6,6 @@ use alloy_eips::{ eip4844::{BlobAndProofV1, BlobAndProofV2, BlobCellsAndProofsV1}, eip4895::Withdrawals, eip7685::RequestsOrHash, - Encodable2718, }; use alloy_primitives::{BlockHash, BlockNumber, Bytes, Sealable, B128, B256, U64}; use alloy_rpc_types_engine::{ @@ -31,7 +30,7 @@ use reth_primitives_traits::{Block, BlockBody}; use reth_rpc_api::{EngineApiServer, IntoEngineApiRpcModule}; use reth_storage_api::{BalProvider, BlockReader, HeaderProvider, StateProviderFactory}; use reth_tasks::Runtime; -use reth_transaction_pool::{BestTransactions, PoolTransaction, TransactionPool}; +use reth_transaction_pool::{BestTransactions, TransactionPool}; use std::{ sync::Arc, time::{Instant, SystemTime}, @@ -485,7 +484,7 @@ where let mut inclusion_list = Vec::new(); for pool_tx in self.inner.tx_pool.best_transactions().without_blobs().without_updates() { - let encoded: Bytes = pool_tx.transaction.consensus_ref().encoded_2718().into(); + let encoded = pool_tx.encoded_2718_consensus(); let new_size = total_size + alloy_rlp::Encodable::length(&encoded); if new_size + alloy_rlp::length_of_length(new_size) > MAX_BYTES_PER_INCLUSION_LIST as usize @@ -1765,7 +1764,7 @@ struct EngineApiInner>(); let json_data = match serde_json::to_string(&local_transactions) { diff --git a/crates/transaction-pool/src/test_utils/mock.rs b/crates/transaction-pool/src/test_utils/mock.rs index 188e46499a6..83ca3c81741 100644 --- a/crates/transaction-pool/src/test_utils/mock.rs +++ b/crates/transaction-pool/src/test_utils/mock.rs @@ -17,6 +17,7 @@ use alloy_consensus::{ }; use alloy_eips::{ eip1559::MIN_PROTOCOL_BASE_FEE, + eip2718::Encodable2718, eip2930::AccessList, eip4844::{BlobTransactionSidecar, BlobTransactionValidationError, DATA_GAS_PER_BLOB}, eip7594::BlobTransactionSidecarVariant, @@ -712,6 +713,10 @@ impl PoolTransaction for MockTransaction { type Pooled = PooledTransactionVariant; + fn encoded_2718_consensus(&self) -> Bytes { + self.clone_into_consensus().encoded_2718().into() + } + fn consensus_ref(&self) -> Recovered<&Self::Consensus> { unimplemented!("mock transaction does not wrap a consensus transaction") } diff --git a/crates/transaction-pool/src/traits.rs b/crates/transaction-pool/src/traits.rs index 602d3ed02d6..ca20e7fa4d7 100644 --- a/crates/transaction-pool/src/traits.rs +++ b/crates/transaction-pool/src/traits.rs @@ -1362,6 +1362,13 @@ pub trait PoolTransaction: self.clone().into_consensus() } + /// Returns the EIP-2718 encoded consensus transaction. + /// + /// Implementations that synthesize the consensus representation should override this method. + fn encoded_2718_consensus(&self) -> Bytes { + self.consensus_ref().encoded_2718().into() + } + /// Returns a reference to the consensus transaction with the recovered sender. fn consensus_ref(&self) -> Recovered<&Self::Consensus>; @@ -1968,7 +1975,7 @@ impl Stream for NewSubpoolTransactionStream { #[cfg(test)] mod tests { use super::*; - use crate::blobstore::BlobCellAvailability; + use crate::{blobstore::BlobCellAvailability, test_utils::MockTransaction}; use alloy_consensus::{ EthereumTxEnvelope, SignableTransaction, TxEip1559, TxEip2930, TxEip4844, TxEip7702, TxEnvelope, TxLegacy, @@ -1976,6 +1983,16 @@ mod tests { use alloy_eips::{eip4844::DATA_GAS_PER_BLOB, eip7594::BlobCellMask}; use alloy_primitives::Signature; + #[test] + fn test_mock_consensus_encoding() { + let transaction = MockTransaction::legacy(); + + assert_eq!( + transaction.encoded_2718_consensus(), + transaction.into_consensus().encoded_2718() + ); + } + #[test] fn test_pool_size_invariants() { let pool_size = PoolSize { @@ -2016,7 +2033,7 @@ mod tests { #[test] fn test_eth_pooled_transaction_new_legacy() { // Create a legacy transaction with specific parameters - let tx = TxEnvelope::Legacy( + let tx = EthereumTxEnvelope::::Legacy( TxLegacy { gas_price: 10, gas_limit: 1000, @@ -2035,6 +2052,7 @@ mod tests { assert!(pooled_tx.blob_cell_availability.is_none()); assert_eq!(pooled_tx.blob_cell_availability().map(BlobCellAvailability::get), None); assert_eq!(pooled_tx.cost, U256::from(100) + U256::from(10 * 1000)); + assert_eq!(pooled_tx.encoded_2718_consensus(), transaction.encoded_2718()); } #[test] diff --git a/crates/transaction-pool/src/validate/mod.rs b/crates/transaction-pool/src/validate/mod.rs index 66d9b250ba4..9c941035c84 100644 --- a/crates/transaction-pool/src/validate/mod.rs +++ b/crates/transaction-pool/src/validate/mod.rs @@ -8,7 +8,7 @@ use crate::{ PriceBumpConfig, }; use alloy_eips::eip7702::SignedAuthorization; -use alloy_primitives::{Address, TxHash, B256, U256}; +use alloy_primitives::{Address, Bytes, TxHash, B256, U256}; use futures_util::future::Either; use reth_primitives_traits::{Block, Recovered, SealedBlock}; use std::{fmt, fmt::Debug, future::Future, time::Instant}; @@ -455,6 +455,11 @@ impl ValidPoolTransaction { self.transaction.clone_into_consensus() } + /// Returns the EIP-2718 encoded consensus transaction. + pub fn encoded_2718_consensus(&self) -> Bytes { + self.transaction.encoded_2718_consensus() + } + /// Determines whether a candidate transaction (`maybe_replacement`) is underpriced compared to /// an existing transaction in the pool. /// From 2ea36ecf60f2647ea66b8174cb8c05629275fc4c Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Mon, 24 Aug 2026 14:55:03 +0200 Subject: [PATCH 058/294] revert: "feat(storage): re-enable RocksDB BAL storage" (#26788) --- crates/cli/commands/src/common.rs | 16 +++---- crates/engine/tree/src/persistence.rs | 37 ---------------- crates/node/builder/src/launch/common.rs | 19 ++++---- crates/storage/provider/src/bal/rocksdb.rs | 14 ++++-- .../provider/src/providers/rocksdb/metrics.rs | 2 - .../src/providers/rocksdb/provider.rs | 43 ++++++++----------- 6 files changed, 45 insertions(+), 86 deletions(-) diff --git a/crates/cli/commands/src/common.rs b/crates/cli/commands/src/common.rs index 77cd552633f..2748ba5ec9c 100644 --- a/crates/cli/commands/src/common.rs +++ b/crates/cli/commands/src/common.rs @@ -27,7 +27,8 @@ use reth_provider::{ BlockchainProvider, NodeTypesForProvider, RocksDBProvider, StaticFileProvider, StaticFileProviderBuilder, }, - BalStoreHandle, ProviderFactory, RocksDBBalStore, StaticFileProviderFactory, StorageSettings, + BalConfig, BalStoreHandle, InMemoryBalStore, ProviderFactory, StaticFileProviderFactory, + StorageSettings, }; use reth_stages::{sets::DefaultStages, Pipeline, PipelineTarget}; use reth_static_file::StaticFileProducer; @@ -195,14 +196,11 @@ impl EnvironmentArgs { where C: ChainSpecParser, { - let bal_store = self - .db - .balstore_cache_size - .map(|distance| { - RocksDBBalStore::with_buffer_retention_distance(rocksdb_provider.clone(), distance) - }) - .unwrap_or_else(|| RocksDBBalStore::new(rocksdb_provider.clone())); - let bal_store = BalStoreHandle::new(bal_store); + let balstore_cache_size = + self.db.balstore_cache_size.unwrap_or(BalConfig::DEFAULT_IN_MEMORY_RETENTION_DISTANCE); + let bal_store = BalStoreHandle::new(InMemoryBalStore::new( + BalConfig::with_in_memory_retention_distance(balstore_cache_size), + )); let factory = ProviderFactory::>::new( db, self.chain.clone(), diff --git a/crates/engine/tree/src/persistence.rs b/crates/engine/tree/src/persistence.rs index 113e7a0f47a..feef9a6e368 100644 --- a/crates/engine/tree/src/persistence.rs +++ b/crates/engine/tree/src/persistence.rs @@ -566,43 +566,6 @@ mod tests { assert_eq!(result.last_state_trie_block, result.last_block); } - #[test] - fn test_save_blocks_flushes_bal_store() { - use reth_provider::{RocksDBBalStore, RocksDBProviderFactory}; - - reth_tracing::init_test_tracing(); - let provider = create_test_provider_factory(); - let bal_store = BalStoreHandle::new(RocksDBBalStore::new(provider.rocksdb_provider())); - let provider = provider.with_bal_store(bal_store); - init_genesis(&provider).unwrap(); - - let mut test_block_builder = TestBlockBuilder::eth(); - let executed = test_block_builder.get_executed_block_with_number(1, B256::random()); - let num_hash = executed.recovered_block().num_hash(); - let raw_bal = Bytes::from_static(&[0xc0]); - - provider.bal_store().insert(num_hash, RawBal::new(raw_bal.clone())).unwrap(); - - let (_finished_exex_height_tx, finished_exex_height_rx) = - tokio::sync::watch::channel(FinishedExExHeight::NoExExs); - let pruner = - Pruner::new_with_factory(provider.clone(), vec![], 5, 0, None, finished_exex_height_rx); - let (sync_metrics_tx, _sync_metrics_rx) = unbounded_channel(); - let handle = PersistenceHandle::::spawn_service( - provider.clone(), - pruner, - sync_metrics_tx, - ); - let (tx, rx) = crossbeam_channel::bounded(1); - - handle.save_blocks(full_save_input(vec![executed]), tx).unwrap(); - - let result = rx.recv_timeout(std::time::Duration::from_secs(10)).expect("test timed out"); - assert_eq!(result.last_block, num_hash); - let persisted_store = RocksDBBalStore::new(provider.rocksdb_provider()); - assert_eq!(persisted_store.get_by_hash(num_hash.hash).unwrap(), Some(raw_bal)); - } - #[test] fn test_save_blocks_multiple_blocks() { reth_tracing::init_test_tracing(); diff --git a/crates/node/builder/src/launch/common.rs b/crates/node/builder/src/launch/common.rs index 43b6cc93bfc..498c65b3fab 100644 --- a/crates/node/builder/src/launch/common.rs +++ b/crates/node/builder/src/launch/common.rs @@ -71,10 +71,10 @@ use reth_node_metrics::{ }; use reth_provider::{ providers::{NodeTypesForProvider, ProviderNodeTypes, RocksDBProvider, StaticFileProvider}, - BalStoreHandle, BlockHashReader, BlockNumReader, DBProvider, DatabaseProviderFactory, - MetadataProvider, MetadataWriter, ProviderError, ProviderFactory, ProviderResult, - RocksDBBalStore, RocksDBProviderFactory, StageCheckpointReader, StaticFileProviderBuilder, - StaticFileProviderFactory, StorageSettingsCache, + BalConfig, BalStoreHandle, BlockHashReader, BlockNumReader, DBProvider, + DatabaseProviderFactory, InMemoryBalStore, MetadataProvider, MetadataWriter, ProviderError, + ProviderFactory, ProviderResult, RocksDBProviderFactory, StageCheckpointReader, + StaticFileProviderBuilder, StaticFileProviderFactory, StorageSettingsCache, }; use reth_prune::{PruneMode, PruneModes, PrunerBuilder}; use reth_rpc_builder::config::RethRpcServerConfig; @@ -524,15 +524,14 @@ where .build()? }; - let bal_store = self + let balstore_cache_size = self .node_config() .db .balstore_cache_size - .map(|distance| { - RocksDBBalStore::with_buffer_retention_distance(rocksdb_provider.clone(), distance) - }) - .unwrap_or_else(|| RocksDBBalStore::new(rocksdb_provider.clone())); - let bal_store = BalStoreHandle::new(bal_store); + .unwrap_or(BalConfig::DEFAULT_IN_MEMORY_RETENTION_DISTANCE); + let bal_store = BalStoreHandle::new(InMemoryBalStore::new( + BalConfig::with_in_memory_retention_distance(balstore_cache_size), + )); let factory = ProviderFactory::new( self.right().clone(), self.chain_spec(), diff --git a/crates/storage/provider/src/bal/rocksdb.rs b/crates/storage/provider/src/bal/rocksdb.rs index d5886d1e8d9..99765824b5a 100644 --- a/crates/storage/provider/src/bal/rocksdb.rs +++ b/crates/storage/provider/src/bal/rocksdb.rs @@ -352,13 +352,21 @@ struct RocksDBBalEntry { #[cfg(test)] mod tests { use super::*; - use crate::providers::RocksDBBuilder; + use crate::providers::{RocksDBBuilder, RocksDBProvider}; use alloy_primitives::B256; use tokio_stream::StreamExt; + fn test_rocksdb(dir: &tempfile::TempDir) -> RocksDBProvider { + RocksDBBuilder::new(dir.path()) + .with_table::() + .with_table::() + .build() + .unwrap() + } + fn test_store() -> (tempfile::TempDir, RocksDBBalStore) { let dir = tempfile::tempdir().unwrap(); - let rocksdb = RocksDBBuilder::new(dir.path()).with_default_tables().build().unwrap(); + let rocksdb = test_rocksdb(&dir); (dir, RocksDBBalStore::new(rocksdb)) } @@ -447,7 +455,7 @@ mod tests { #[test] fn configured_buffer_retention_distance_is_used() { let dir = tempfile::tempdir().unwrap(); - let rocksdb = RocksDBBuilder::new(dir.path()).with_default_tables().build().unwrap(); + let rocksdb = test_rocksdb(&dir); let store = RocksDBBalStore::with_buffer_retention_distance(rocksdb, 64); let old = NumHash::new(1, B256::with_last_byte(1)); let tip = NumHash::new(34, B256::with_last_byte(2)); diff --git a/crates/storage/provider/src/providers/rocksdb/metrics.rs b/crates/storage/provider/src/providers/rocksdb/metrics.rs index c22edf65176..3971ed978ba 100644 --- a/crates/storage/provider/src/providers/rocksdb/metrics.rs +++ b/crates/storage/provider/src/providers/rocksdb/metrics.rs @@ -8,8 +8,6 @@ use strum::{EnumIter, IntoEnumIterator}; pub(super) const ROCKSDB_TABLES: &[&str] = &[ Tables::TransactionHashNumbers.name(), - Tables::BlockAccessLists.name(), - Tables::BlockAccessListBlockNumbers.name(), Tables::StoragesHistory.name(), Tables::AccountsHistory.name(), ]; diff --git a/crates/storage/provider/src/providers/rocksdb/provider.rs b/crates/storage/provider/src/providers/rocksdb/provider.rs index bae0a137b37..d7198e7c74a 100644 --- a/crates/storage/provider/src/providers/rocksdb/provider.rs +++ b/crates/storage/provider/src/providers/rocksdb/provider.rs @@ -315,14 +315,10 @@ impl RocksDBBuilder { /// - [`tables::TransactionHashNumbers`] - Transaction hash to number mapping /// - [`tables::AccountsHistory`] - Account history index /// - [`tables::StoragesHistory`] - Storage history index - /// - [`tables::BlockAccessLists`] - Block access list payloads - /// - [`tables::BlockAccessListBlockNumbers`] - Block access list hash index pub fn with_default_tables(self) -> Self { self.with_table::() .with_table::() .with_table::() - .with_table::() - .with_table::() } /// Enables metrics. @@ -2922,25 +2918,22 @@ mod tests { provider.put::(key.clone(), &value).unwrap(); assert!(provider.get::(key).unwrap().is_some()); - let bal_key = - reth_db_api::models::StoredBlockAccessListKey::new(1, B256::with_last_byte(1)); - let bal_value = - reth_db_api::models::StoredBlockAccessList::new(Bytes::from_static(&[0xc0])); - provider.put::(bal_key, &bal_value).unwrap(); - assert_eq!(provider.get::(bal_key).unwrap(), Some(bal_value)); - provider - .put::(bal_key.hash(), &bal_key.number()) - .unwrap(); - assert_eq!( - provider.get::(bal_key.hash()).unwrap(), - Some(bal_key.number()) - ); + drop(provider); + + let column_families = DB::list_cf(&Options::default(), temp_dir.path()).unwrap(); + assert!(!column_families.iter().any(|name| name == tables::BlockAccessLists::NAME)); + assert!(!column_families + .iter() + .any(|name| name == tables::BlockAccessListBlockNumbers::NAME)); } #[test] fn block_access_lists_store_large_payloads_in_blob_files() { let temp_dir = TempDir::new().unwrap(); - let provider = RocksDBBuilder::new(temp_dir.path()).with_default_tables().build().unwrap(); + let provider = RocksDBBuilder::new(temp_dir.path()) + .with_table::() + .build() + .unwrap(); let bal_key = reth_db_api::models::StoredBlockAccessListKey::new(1, B256::with_last_byte(1)); let bal_value = reth_db_api::models::StoredBlockAccessList::new(Bytes::from(vec![ @@ -2996,7 +2989,12 @@ mod tests { 1 ])); - let provider = RocksDBBuilder::new(temp_dir.path()).with_default_tables().build().unwrap(); + let provider = RocksDBBuilder::new(temp_dir.path()) + .with_default_tables() + .with_table::() + .with_table::() + .build() + .unwrap(); provider.put::(bal_key, &bal_value).unwrap(); provider .put::(bal_key.hash(), &bal_key.number()) @@ -3004,12 +3002,7 @@ mod tests { provider.flush(&[tables::BlockAccessLists::NAME]).unwrap(); drop(provider); - let provider = RocksDBBuilder::new(temp_dir.path()) - .with_table::() - .with_table::() - .with_table::() - .build() - .unwrap(); + let provider = RocksDBBuilder::new(temp_dir.path()).with_default_tables().build().unwrap(); assert_eq!(provider.get::(bal_key).unwrap(), Some(bal_value)); assert_eq!( provider.get::(bal_key.hash()).unwrap(), From 5d3c60191ddc76a6af3591d2321beb2cdc64ca8f Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Mon, 24 Aug 2026 15:03:30 +0200 Subject: [PATCH 059/294] fix(net): make eth/72 announcements interoperable with geth (#26670) --- crates/net/eth-wire-types/src/broadcast.rs | 47 ++++- crates/net/network/src/transactions/mod.rs | 62 +++++- crates/net/network/tests/it/eth72.rs | 210 +++++++++++++++++++++ crates/net/network/tests/it/main.rs | 1 + 4 files changed, 307 insertions(+), 13 deletions(-) create mode 100644 crates/net/network/tests/it/eth72.rs diff --git a/crates/net/eth-wire-types/src/broadcast.rs b/crates/net/eth-wire-types/src/broadcast.rs index 64596203e8f..2c3921a7487 100644 --- a/crates/net/eth-wire-types/src/broadcast.rs +++ b/crates/net/eth-wire-types/src/broadcast.rs @@ -794,8 +794,13 @@ pub struct NewPooledTransactionHashes72 { /// /// Per [EIP-8070](https://eips.ethereum.org/EIPS/eip-8070), this is a `B_16` /// bitarray over `CELLS_PER_EXT_BLOB`; bit `i` is set when the announcer has column - /// `i` available for every type 3 transaction in the message. `None` encodes as RLP - /// `nil` and must be used when no type 3 transactions are announced. + /// `i` available for every type 3 transaction in the message. + /// + /// On the wire this field is always encoded as a 16 byte string, zero-filled when no + /// type 3 transactions are announced: go-ethereum decodes the mask into a fixed + /// `[16]byte` and rejects the RLP `nil` encoding the EIP text describes, so the + /// always-present form is the de facto network format. `None` is equivalent to a zero + /// mask; decoding additionally accepts the spec's `nil` encoding for compatibility. pub cell_mask: Option, } @@ -831,6 +836,12 @@ impl proptest::prelude::Arbitrary for NewPooledTransactionHashes72 { } impl NewPooledTransactionHashes72 { + /// Cell mask advertising availability of every cell. + /// + /// Used when announcing blob transactions whose full sidecar is available locally, since + /// every cell can be computed from the complete blob data. + pub const ALL_CELLS_MASK: B128 = B128::repeat_byte(0xff); + /// Returns a new instance with capacity for `capacity` entries and no cell mask. pub fn with_capacity(capacity: usize) -> Self { Self { @@ -851,6 +862,9 @@ impl NewPooledTransactionHashes72 { self.hashes.push(*tx.tx_hash()); self.sizes.push(tx.encode_2718_len()); self.types.push(tx.ty()); + if tx.is_eip4844() { + self.cell_mask = Some(Self::ALL_CELLS_MASK); + } } /// Appends the provided transactions @@ -886,7 +900,7 @@ impl NewPooledTransactionHashes72 { self.types.as_slice().length() + self.sizes.length() + self.hashes.length() + - self.cell_mask.as_ref().map_or(1, Encodable::length) + self.cell_mask.unwrap_or_default().length() } } @@ -896,11 +910,8 @@ impl Encodable for NewPooledTransactionHashes72 { self.types.as_slice().encode(out); self.sizes.encode(out); self.hashes.encode(out); - if let Some(cell_mask) = &self.cell_mask { - cell_mask.encode(out); - } else { - out.put_u8(alloy_rlp::EMPTY_STRING_CODE); - } + // A zero-filled mask when no cells are available, see the `cell_mask` field docs. + self.cell_mask.unwrap_or_default().encode(out); } fn length(&self) -> usize { @@ -924,10 +935,13 @@ impl Decodable for NewPooledTransactionHashes72 { return Err(alloy_rlp::Error::InputTooShort) }; let cell_mask = if first_byte == alloy_rlp::EMPTY_STRING_CODE { + // The EIP-8070 `nil` encoding, tolerated for compatibility. payload = &payload[1..]; None } else { - Some(B128::decode(&mut payload)?) + // A zero mask is the wire representation of "no cells available", see the + // `cell_mask` field docs. + Some(B128::decode(&mut payload)?).filter(|mask| !mask.is_zero()) }; if !payload.is_empty() { @@ -1658,6 +1672,8 @@ mod tests { #[test] fn eth_72_tx_hash_roundtrip() { let vectors = vec![ + // `None` is always encoded as a zero-filled 16 byte mask, matching go-ethereum's + // non-optional `[16]byte` field. ( NewPooledTransactionHashes72 { types: vec![], @@ -1665,7 +1681,7 @@ mod tests { hashes: vec![], cell_mask: None, }, - &hex!("c480c0c080")[..], + &hex!("d480c0c09000000000000000000000000000000000")[..], ), ( NewPooledTransactionHashes72 { @@ -1683,6 +1699,17 @@ mod tests { } } + #[test] + fn eth_72_decodes_spec_nil_cell_mask() { + // The EIP-8070 text encodes an absent mask as the RLP empty string; decoding stays + // lenient even though reth never produces this form. + let encoded = hex!("c480c0c080"); + + let decoded = NewPooledTransactionHashes72::decode(&mut encoded.as_ref()).unwrap(); + + assert_eq!(decoded.cell_mask, None); + } + #[test] fn eth_72_rejects_missing_cell_mask() { let encoded_eth68_payload = hex!("c380c0c0"); diff --git a/crates/net/network/src/transactions/mod.rs b/crates/net/network/src/transactions/mod.rs index 3b820f4f9bd..fdb6bf64125 100644 --- a/crates/net/network/src/transactions/mod.rs +++ b/crates/net/network/src/transactions/mod.rs @@ -1,6 +1,6 @@ //! Transactions management for the p2p network. -use alloy_consensus::transaction::TxHashRef; +use alloy_consensus::{constants::EIP4844_TX_TYPE_ID, transaction::TxHashRef}; use rayon::iter::{IntoParallelIterator, ParallelIterator}; use smallvec::SmallVec; @@ -544,6 +544,10 @@ impl TransactionsManager { // Blob sidecar errors: penalize but do NOT cache the hash as bad. // The transaction may be valid — only the sidecar from this peer was wrong. // Using regular penalties means repeated offenders still get disconnected. + // + // Eth/72 peers cannot reach this path with an elided sidecar: their blob + // transactions are dropped before import in `import_transactions`, so any sidecar + // failure here means actual blob data failed validation. if let Some(peers) = peers { for peer_id in peers { self.report_peer_bad_transactions(peer_id); @@ -1453,6 +1457,23 @@ where } } + // Eth/72 `PooledTransactions` responses elide blob payloads from type 3 transactions per + // EIP-8070, so their sidecars can never validate. Drop them before touching the pool; + // geth equivalently diverts these bodies into a buffer that is completed with cells + // fetched via `GetCells`, which is not implemented yet. + if peer.version() == EthVersion::Eth72 { + let len_before = transactions.len(); + transactions.retain(|tx| !tx.is_eip4844()); + let dropped = len_before - transactions.len(); + if dropped > 0 { + trace!(target: "net::tx", + peer_id=format!("{peer_id:#}"), + dropped, + "dropped blob transactions from eth72 response, cell fetching not implemented" + ); + } + } + // tracks the quality of the given transactions let mut has_bad_transactions = false; @@ -2085,7 +2106,12 @@ impl PooledTransactionsHashesBuilder { Self::Eth72(msg) => { msg.hashes.push(*pooled_tx.hash()); msg.sizes.push(pooled_tx.encoded_length()); - msg.types.push(pooled_tx.transaction.ty()); + let ty = pooled_tx.transaction.ty(); + msg.types.push(ty); + if ty == EIP4844_TX_TYPE_ID { + // The pool holds the full sidecar, so every cell can be served. + msg.cell_mask = Some(NewPooledTransactionHashes72::ALL_CELLS_MASK); + } } } } @@ -2126,7 +2152,12 @@ impl PooledTransactionsHashesBuilder { Self::Eth72(msg) => { msg.hashes.push(*tx.tx_hash()); msg.sizes.push(tx.propagation_size()); - msg.types.push(tx.tx_type()); + let ty = tx.tx_type(); + msg.types.push(ty); + if ty == EIP4844_TX_TYPE_ID { + // The pool holds the full sidecar, so every cell can be served. + msg.cell_mask = Some(NewPooledTransactionHashes72::ALL_CELLS_MASK); + } } } } @@ -3360,6 +3391,31 @@ mod tests { assert!(propagated.is_empty()); } + #[test] + fn test_eth72_hashes_builder_sets_cell_mask_for_blob_txs() { + let mut tx_gen = TransactionGenerator::new(rand::rng()); + + // no blob transactions: the mask stays unset and encodes as a zero mask + let mut builder = PooledTransactionsHashesBuilder::new(EthVersion::Eth72); + builder.push(&PropagateTransaction::pool_tx(valid_eth_pool_transaction( + tx_gen.gen_eip1559_pooled(), + ))); + let msg = builder.build(); + assert_eq!(msg.as_eth72().unwrap().cell_mask, None); + + // announcing a blob transaction advertises every cell as available + let mut builder = PooledTransactionsHashesBuilder::new(EthVersion::Eth72); + builder.push(&PropagateTransaction::pool_tx(valid_eth_pool_transaction( + tx_gen.gen_eip1559_pooled(), + ))); + builder.push_pooled(valid_eth_pool_transaction(tx_gen.gen_eip4844_pooled())); + let msg = builder.build(); + assert_eq!( + msg.as_eth72().unwrap().cell_mask, + Some(NewPooledTransactionHashes72::ALL_CELLS_MASK) + ); + } + #[tokio::test] async fn test_truncated_hash_announcement_not_marked_seen() { reth_tracing::init_test_tracing(); diff --git a/crates/net/network/tests/it/eth72.rs b/crates/net/network/tests/it/eth72.rs new file mode 100644 index 00000000000..dfa1a5230df --- /dev/null +++ b/crates/net/network/tests/it/eth72.rs @@ -0,0 +1,210 @@ +//! Eth/72 interop tests. +//! +//! A raw peer speaking the go-ethereum eth/72 wire format connects to a node and exercises the +//! announcement cell mask in both directions as well as a blob-elided `PooledTransactions` +//! response per [EIP-8070](https://eips.ethereum.org/EIPS/eip-8070). + +use alloy_consensus::{ + constants::EIP4844_TX_TYPE_ID, transaction::TxEip4844WithSidecar, Header, SignableTransaction, + TxEip4844, +}; +use alloy_eips::{ + eip2718::Encodable2718, + eip7594::{BlobTransactionSidecarEip7594, BlobTransactionSidecarVariant}, +}; +use alloy_primitives::{Address, B256, U256}; +use futures::{SinkExt, StreamExt}; +use reth_chainspec::MAINNET; +use reth_ecies::stream::ECIESStream; +use reth_eth_wire::{ + EthMessage, EthNetworkPrimitives, EthStream, EthVersion, HelloMessageWithProtocols, + NewPooledTransactionHashes72, P2PStream, StatusBuilder, UnauthedEthStream, UnauthedP2PStream, +}; +use reth_eth_wire_types::{message::RequestPair, PooledTransactions}; +use reth_ethereum_forks::EthereumHardfork; +use reth_ethereum_primitives::{Block, PooledTransactionVariant}; +use reth_network::{ + test_utils::{NetworkEventStream, PeerConfig, Testnet}, + transactions::config::{TransactionPropagationMode, TransactionsManagerConfig}, + NetworkEventListenerProvider, PeersInfo, +}; +use reth_network_peers::pk2id; +use reth_primitives_traits::{crypto::secp256k1::sign_message, SignerRecoverable}; +use reth_provider::test_utils::{ExtendedAccount, MockEthProvider}; +use reth_transaction_pool::{ + test_utils::TransactionGenerator, AddedTransactionOutcome, PoolTransaction, TransactionPool, +}; +use secp256k1::{SecretKey, SECP256K1}; +use std::time::Duration; +use tokio::net::TcpStream; + +type RawEthStream = EthStream>, EthNetworkPrimitives>; + +/// Returns the next eth message from the raw peer's stream, skipping unrelated traffic such as +/// block range updates. +async fn next_message(stream: &mut RawEthStream) -> EthMessage { + loop { + let message = tokio::time::timeout(Duration::from_secs(30), stream.next()) + .await + .expect("timed out awaiting eth message") + .expect("stream terminated") + .expect("stream errored"); + if !matches!(message, EthMessage::BlockRangeUpdate(_)) { + return message + } + } +} + +/// A signed EIP-4844 pooled transaction in the eth/72 shape: the sidecar keeps commitments and +/// cell proof metadata while the blob payloads are elided (fetched separately via `GetCells`). +fn blob_tx_without_blobs() -> PooledTransactionVariant { + let mut versioned_hash = B256::random(); + versioned_hash.0[0] = 0x01; + + let tx = TxEip4844 { + chain_id: 1, + nonce: 0, + gas_limit: 100_000, + max_fee_per_gas: 20_000_000_000, + max_priority_fee_per_gas: 1_000_000_000, + to: Address::random(), + value: U256::ZERO, + access_list: Default::default(), + blob_versioned_hashes: vec![versioned_hash], + max_fee_per_blob_gas: 20_000_000_000, + input: Default::default(), + }; + + let signature = sign_message(B256::random(), tx.signature_hash()).unwrap(); + let sidecar = BlobTransactionSidecarVariant::Eip7594(BlobTransactionSidecarEip7594 { + blobs: vec![], + commitments: vec![Default::default()], + cell_proofs: vec![], + }); + + PooledTransactionVariant::Eip4844( + TxEip4844WithSidecar::from_tx_and_sidecar(tx, sidecar).into_signed(signature), + ) +} + +#[tokio::test(flavor = "multi_thread")] +async fn test_eth72_blob_announcements_and_elided_pooled_response() { + reth_tracing::init_test_tracing(); + + let provider = MockEthProvider::default().with_genesis_block(); + + // A recent tip so the pool validator activates the blob fork checks; the network status + // itself stays at the genesis head. + let tip = Header { + number: 1, + parent_hash: MAINNET.genesis_hash(), + timestamp: 1_750_000_000, + gas_limit: 30_000_000, + base_fee_per_gas: Some(7), + excess_blob_gas: Some(0), + blob_gas_used: Some(0), + ..Default::default() + }; + provider.add_block(tip.hash_slow(), Block { header: tip, body: Default::default() }); + + let mut net = Testnet::create_with(0, provider.clone()).await; + net.add_peer_with_config(PeerConfig::with_protocols( + provider.clone(), + Some(EthVersion::Eth72.into()), + )) + .await + .unwrap(); + + // Announce hashes to every peer instead of broadcasting transactions in full, so the raw + // peer receives `NewPooledTransactionHashes` for non-blob transactions as well. + let tx_manager_config = TransactionsManagerConfig { + propagation_mode: TransactionPropagationMode::Max(0), + ..Default::default() + }; + let net = net.with_eth_pool_config(tx_manager_config); + let handle = net.spawn(); + + let node = &handle.peers()[0]; + let node_id = *node.peer_id(); + let node_pool = node.pool().unwrap(); + let mut events = NetworkEventStream::new(node.network().event_listener()); + + // connect a raw eth/72 peer + let raw_key = SecretKey::new(&mut rand_08::thread_rng()); + let raw_id = pk2id(&raw_key.public_key(SECP256K1)); + + let tcp = TcpStream::connect(node.local_addr()).await.unwrap(); + let ecies = ECIESStream::connect(tcp, raw_key, node_id).await.unwrap(); + let hello = HelloMessageWithProtocols::builder(raw_id) + .protocols(vec![EthVersion::Eth72.into()]) + .build(); + let (p2p, _their_hello) = UnauthedP2PStream::new(ecies).handshake(hello).await.unwrap(); + let version = p2p.shared_capabilities().eth_version().unwrap(); + assert_eq!(version, EthVersion::Eth72); + + let mut status = StatusBuilder::default().build(); + status.set_eth_version(version); + let fork_filter = MAINNET.hardfork_fork_filter(EthereumHardfork::Frontier).unwrap(); + let (mut eth_stream, _their_status) = UnauthedEthStream::new(p2p) + .handshake::(status, fork_filter) + .await + .unwrap(); + + assert_eq!(events.next_session_established().await.unwrap(), raw_id); + let baseline_reputation = node.peer_handle().peer_by_id(raw_id).await.unwrap().reputation(); + + // 1) node -> raw peer: a plain pending transaction is announced with the eth/72 message + let mut tx_gen = TransactionGenerator::new(rand::rng()); + let tx = tx_gen.gen_eip1559_pooled(); + provider.add_account(tx.sender(), ExtendedAccount::new(0, U256::from(100_000_000u64))); + let AddedTransactionOutcome { hash: pending_hash, .. } = + node_pool.add_external_transaction(tx).await.unwrap(); + + let announcement = match next_message(&mut eth_stream).await { + EthMessage::NewPooledTransactionHashes72(announcement) => announcement, + message => panic!("unexpected message {message:?}"), + }; + assert_eq!(announcement.hashes, vec![pending_hash]); + // no blob transactions were announced: the zero cell mask on the wire decodes to `None` + assert_eq!(announcement.cell_mask, None); + + // 2) raw peer -> node: a geth style blob announcement carrying the custody mask + let pooled = blob_tx_without_blobs(); + let blob_tx_hash = *pooled.tx_hash(); + let sender = pooled.recover_signer().unwrap(); + provider.add_account(sender, ExtendedAccount::new(0, U256::from(10u128.pow(19)))); + + let announcement = NewPooledTransactionHashes72 { + types: vec![EIP4844_TX_TYPE_ID], + sizes: vec![pooled.encode_2718_len()], + hashes: vec![blob_tx_hash], + cell_mask: Some(NewPooledTransactionHashes72::ALL_CELLS_MASK), + }; + eth_stream.send(EthMessage::NewPooledTransactionHashes72(announcement)).await.unwrap(); + + // the node must fetch the announced blob transaction + let request = match next_message(&mut eth_stream).await { + EthMessage::GetPooledTransactions(request) => request, + message => panic!("unexpected message {message:?}"), + }; + assert_eq!(request.message.0, vec![blob_tx_hash]); + + // 3) raw peer -> node: the eth/72 response elides the blob payloads + eth_stream + .send(EthMessage::PooledTransactions(RequestPair { + request_id: request.request_id, + message: PooledTransactions(vec![pooled]), + })) + .await + .unwrap(); + + // The blob-elided body is dropped before the pool import (cell fetching is not implemented + // yet), and the eth/72 peer served a spec compliant response so it must not be penalized. + tokio::time::sleep(Duration::from_secs(1)).await; + + assert!(!node_pool.contains(&blob_tx_hash)); + let reputation = node.peer_handle().peer_by_id(raw_id).await.unwrap().reputation(); + assert_eq!(reputation, baseline_reputation); + // the session survives serving the blob-elided response + assert_eq!(node.network().num_connected_peers(), 1); +} diff --git a/crates/net/network/tests/it/main.rs b/crates/net/network/tests/it/main.rs index b8688d8e5a9..939ff89f3b8 100644 --- a/crates/net/network/tests/it/main.rs +++ b/crates/net/network/tests/it/main.rs @@ -2,6 +2,7 @@ mod big_pooled_txs_req; mod connect; +mod eth72; mod multiplex; mod requests; mod session; From 00ff6505734fa896a38a83ff9381d7c29e814839 Mon Sep 17 00:00:00 2001 From: monem <119044801+pucedoteth@users.noreply.github.com> Date: Mon, 24 Aug 2026 14:08:24 +0000 Subject: [PATCH 060/294] fix(rpc): reject a chainId that is not the node's (#26782) Co-authored-by: Claude Opus 5 --- .../rpc-eth-api/src/helpers/transaction.rs | 24 ++++++++ crates/rpc/rpc/src/eth/helpers/transaction.rs | 60 +++++++++++++++++++ 2 files changed, 84 insertions(+) diff --git a/crates/rpc/rpc-eth-api/src/helpers/transaction.rs b/crates/rpc/rpc-eth-api/src/helpers/transaction.rs index bbbdbfc467a..c34a6fe2955 100644 --- a/crates/rpc/rpc-eth-api/src/helpers/transaction.rs +++ b/crates/rpc/rpc-eth-api/src/helpers/transaction.rs @@ -507,6 +507,18 @@ pub trait EthTransactions: LoadTransaction { } let chain_id = self.chain_id(); + // A caller that pins `chainId` is asserting which chain it means to sign for. + // Silently rewriting it would sign and submit on a different chain than asked + // for, so reject the mismatch instead. Matches geth's `setDefaults`. + if let Some(request_chain_id) = request.as_ref().chain_id() && + request_chain_id != chain_id.to::() + { + return Err(EthApiError::InvalidParams(format!( + "chainId does not match node's (have={request_chain_id}, want={})", + chain_id.to::() + )) + .into()) + } request.as_mut().set_chain_id(chain_id.to()); if request.as_ref().gas_limit().is_none() { @@ -556,6 +568,18 @@ pub trait EthTransactions: LoadTransaction { } let chain_id = self.chain_id(); + // A caller that pins `chainId` is asserting which chain it means to sign for. + // Silently rewriting it would sign and submit on a different chain than asked + // for, so reject the mismatch instead. Matches geth's `setDefaults`. + if let Some(request_chain_id) = request.as_ref().chain_id() && + request_chain_id != chain_id.to::() + { + return Err(EthApiError::InvalidParams(format!( + "chainId does not match node's (have={request_chain_id}, want={})", + chain_id.to::() + )) + .into()) + } request.as_mut().set_chain_id(chain_id.to()); if request.as_ref().has_eip4844_fields() && diff --git a/crates/rpc/rpc/src/eth/helpers/transaction.rs b/crates/rpc/rpc/src/eth/helpers/transaction.rs index 87c2d4588e7..eaf2af53043 100644 --- a/crates/rpc/rpc/src/eth/helpers/transaction.rs +++ b/crates/rpc/rpc/src/eth/helpers/transaction.rs @@ -318,6 +318,66 @@ mod tests { assert_eq!(pooled.transaction.gas_limit(), provided_gas_limit); } + #[tokio::test] + async fn send_transaction_rejects_mismatched_chain_id() { + let signers = DevSigner::random_signers(1); + let address = signers[0].accounts()[0]; + let accounts = AddressMap::from_iter([( + address, + ExtendedAccount::new(0, U256::from(10_000_000_000_000_000_000u64)), + )]); + let eth_api = mock_eth_api(accounts); + eth_api.signers().write().extend(signers); + + // The mock node is mainnet (chain id 1); the caller pins a different chain. + let tx_req = TransactionRequest { + from: Some(address), + to: Some(address.into()), + gas: Some(90_000), + gas_price: Some(1_000_000_000), + chain_id: Some(999), + ..Default::default() + }; + + let err = eth_api + .send_transaction_request(tx_req) + .await + .expect_err("a chain id that is not the node's must be rejected, not rewritten"); + assert!( + err.to_string().contains("chainId does not match node's"), + "unexpected error: {err}" + ); + assert!(eth_api.pool().is_empty(), "no transaction should have been submitted"); + } + + #[tokio::test] + async fn send_transaction_accepts_matching_chain_id() { + let signers = DevSigner::random_signers(1); + let address = signers[0].accounts()[0]; + let accounts = AddressMap::from_iter([( + address, + ExtendedAccount::new(0, U256::from(10_000_000_000_000_000_000u64)), + )]); + let eth_api = mock_eth_api(accounts); + eth_api.signers().write().extend(signers); + + let tx_req = TransactionRequest { + from: Some(address), + to: Some(address.into()), + gas: Some(90_000), + gas_price: Some(1_000_000_000), + chain_id: Some(1), + ..Default::default() + }; + + let hash = eth_api + .send_transaction_request(tx_req) + .await + .expect("a matching chain id must still be accepted"); + let pooled = eth_api.pool().get(&hash).expect("transaction should be in the pool"); + assert_eq!(pooled.transaction.chain_id(), Some(1)); + } + #[tokio::test] async fn test_fill_transaction_fills_chain_id() { let address = Address::random(); From 5e8d2a813e8f547b210b310b38c22731b8b216dc Mon Sep 17 00:00:00 2001 From: Derek Cofausper <256792747+decofe@users.noreply.github.com> Date: Tue, 25 Aug 2026 11:37:48 +0000 Subject: [PATCH 061/294] feat(engine): remove partial persistence feature flag (#26800) Co-authored-by: Brian Picciano <933154+mediocregopher@users.noreply.github.com> --- .github/scripts/verify_image_arch.sh | 1 - .github/workflows/docker.yml | 1 - bin/reth-bb/Cargo.toml | 2 - bin/reth/Cargo.toml | 1 - crates/net/network/src/test_utils/testnet.rs | 3 +- .../net/network/src/transactions/fetcher.rs | 5 +- crates/node/core/Cargo.toml | 2 - crates/node/core/src/args/engine.rs | 29 ++------- crates/node/core/src/args/network.rs | 6 +- crates/stages/types/Cargo.toml | 1 - crates/stages/types/src/checkpoints.rs | 62 ++----------------- crates/storage/provider/Cargo.toml | 4 -- .../src/providers/database/provider.rs | 53 +++++----------- crates/storage/storage-overlay/Cargo.toml | 1 - crates/storage/storage-overlay/src/builder.rs | 18 ------ .../storage/storage-overlay/src/provider.rs | 2 +- docker-bake.hcl | 12 +--- docs/vocs/docs/pages/cli/reth/node.mdx | 5 ++ 18 files changed, 36 insertions(+), 172 deletions(-) diff --git a/.github/scripts/verify_image_arch.sh b/.github/scripts/verify_image_arch.sh index 842e3352b18..9746b97edd2 100755 --- a/.github/scripts/verify_image_arch.sh +++ b/.github/scripts/verify_image_arch.sh @@ -42,7 +42,6 @@ verify_image() { if [[ "$TARGETS" == *"nightly"* ]]; then verify_image "${REGISTRY}/reth:nightly" amd64 arm64 - verify_image "${REGISTRY}/reth:nightly-partial-persistence" amd64 arm64 verify_image "${REGISTRY}/reth:nightly-profiling" amd64 verify_image "${REGISTRY}/reth:nightly-edge-profiling" amd64 else diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 83b97c5e53b..5441c81cfd9 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -96,7 +96,6 @@ jobs: echo "ethereum.tags=${REGISTRY}/reth:nightly" echo "ethereum.args.RETH_ENGINE_TXPOOL_PREWARMING=true" echo "ethereum.args.RETH_ENGINE_SENDER_RECOVERY_CACHE=true" - echo "ethereum-partial-persistence.tags=${REGISTRY}/reth:nightly-partial-persistence" echo "EOF" } >> "$GITHUB_OUTPUT" diff --git a/bin/reth-bb/Cargo.toml b/bin/reth-bb/Cargo.toml index e6f4492e776..4c8ff17ce93 100644 --- a/bin/reth-bb/Cargo.toml +++ b/bin/reth-bb/Cargo.toml @@ -122,8 +122,6 @@ min-trace-logs = [ "reth-node-core/min-trace-logs", ] -partial-persistence = ["reth-node-core/partial-persistence"] - [[bin]] name = "reth-bb" path = "src/main.rs" diff --git a/bin/reth/Cargo.toml b/bin/reth/Cargo.toml index d616349ceda..cc83d1b4265 100644 --- a/bin/reth/Cargo.toml +++ b/bin/reth/Cargo.toml @@ -199,7 +199,6 @@ min-trace-logs = [ "reth-node-core/min-trace-logs", ] -partial-persistence = ["reth-node-core/partial-persistence"] trie-debug = ["reth-node-builder/trie-debug", "reth-node-core/trie-debug"] [[bin]] diff --git a/crates/net/network/src/test_utils/testnet.rs b/crates/net/network/src/test_utils/testnet.rs index 4c52db50499..0b5d6b9b2a6 100644 --- a/crates/net/network/src/test_utils/testnet.rs +++ b/crates/net/network/src/test_utils/testnet.rs @@ -386,8 +386,7 @@ impl TestnetHandle { // add all peers to each other for (idx, handle) in self.peers.iter().enumerate().take(self.peers.len() - 1) { - for idx in (idx + 1)..self.peers.len() { - let neighbour = &self.peers[idx]; + for neighbour in &self.peers[idx + 1..] { handle.network.add_peer(*neighbour.peer_id(), neighbour.local_addr()); } } diff --git a/crates/net/network/src/transactions/fetcher.rs b/crates/net/network/src/transactions/fetcher.rs index f0230c21fd0..fee72ce45d7 100644 --- a/crates/net/network/src/transactions/fetcher.rs +++ b/crates/net/network/src/transactions/fetcher.rs @@ -184,10 +184,7 @@ impl TransactionFetcher { #[inline] pub fn is_idle(&self, peer_id: &PeerId) -> bool { let Some(inflight_count) = self.active_peers.peek(peer_id) else { return true }; - if *inflight_count < self.info.max_inflight_requests_per_peer { - return true - } - false + *inflight_count < self.info.max_inflight_requests_per_peer } /// Returns any idle peer for the given hash. diff --git a/crates/node/core/Cargo.toml b/crates/node/core/Cargo.toml index 39caeec04e2..8b9f1956a70 100644 --- a/crates/node/core/Cargo.toml +++ b/crates/node/core/Cargo.toml @@ -90,8 +90,6 @@ min-info-logs = ["tracing/release_max_level_info"] min-debug-logs = ["tracing/release_max_level_debug"] min-trace-logs = ["tracing/release_max_level_trace"] -partial-persistence = ["reth-stages-types/partial-persistence"] - # Debug recording for sparse trie mutations trie-debug = ["reth-engine-primitives/trie-debug"] diff --git a/crates/node/core/src/args/engine.rs b/crates/node/core/src/args/engine.rs index 06df7745ece..50bb3977a93 100644 --- a/crates/node/core/src/args/engine.rs +++ b/crates/node/core/src/args/engine.rs @@ -326,18 +326,11 @@ pub struct EngineArgs { #[arg(long = "engine.persistence-backpressure-threshold")] pub persistence_backpressure_threshold: Option, - /// Configure how many of the blocks being persisted should only mask state/trie writes instead - /// of durably persisting their state/trie updates in the current cycle. - #[cfg_attr( - feature = "partial-persistence", - arg( - long = "engine.num-state-masking-blocks", - default_value_t = DefaultEngineValues::get_global().num_state_masking_blocks - ) - )] - #[cfg_attr( - not(feature = "partial-persistence"), - arg(skip = DefaultEngineValues::get_global().num_state_masking_blocks) + /// EXPERIMENTAL: Configure how many of the blocks being persisted should only mask state/trie + /// writes instead of durably persisting their state/trie updates in the current cycle. + #[arg( + long = "engine.num-state-masking-blocks", + default_value_t = DefaultEngineValues::get_global().num_state_masking_blocks )] pub num_state_masking_blocks: u64, @@ -994,7 +987,6 @@ mod tests { assert!(err.contains("engine.persistence-threshold")); } - #[cfg(feature = "partial-persistence")] #[test] fn test_parse_num_state_masking_blocks() { let args = CommandParser::::parse_from([ @@ -1009,17 +1001,6 @@ mod tests { assert_eq!(args.tree_config().num_state_masking_blocks(), 7); } - #[cfg(not(feature = "partial-persistence"))] - #[test] - fn num_state_masking_blocks_is_hidden_without_partial_persistence() { - assert!(CommandParser::::try_parse_from([ - "reth", - "--engine.num-state-masking-blocks", - "1", - ]) - .is_err()); - } - #[test] fn validate_rejects_state_masking_window_at_or_above_threshold() { let args = EngineArgs { diff --git a/crates/node/core/src/args/network.rs b/crates/node/core/src/args/network.rs index fc2115f2dd5..6f212ce5499 100644 --- a/crates/node/core/src/args/network.rs +++ b/crates/node/core/src/args/network.rs @@ -1081,11 +1081,7 @@ impl DiscoveryArgs { /// /// Discv5 is enabled by default and can be disabled with `--disable-discv5-discovery`. const fn should_enable_discv5(&self) -> bool { - if self.disable_discovery || self.disable_discv5_discovery { - return false; - } - - true + !(self.disable_discovery || self.disable_discv5_discovery) } /// Set the discovery ports to zero, to allow the OS to assign random unused ports when diff --git a/crates/stages/types/Cargo.toml b/crates/stages/types/Cargo.toml index 6cf08a26724..83276b46f2d 100644 --- a/crates/stages/types/Cargo.toml +++ b/crates/stages/types/Cargo.toml @@ -35,7 +35,6 @@ modular-bitfield.workspace = true [features] default = ["std"] -partial-persistence = [] std = [ "alloy-primitives/std", "bytes?/std", diff --git a/crates/stages/types/src/checkpoints.rs b/crates/stages/types/src/checkpoints.rs index 3ace5e8e486..b56f8596f33 100644 --- a/crates/stages/types/src/checkpoints.rs +++ b/crates/stages/types/src/checkpoints.rs @@ -442,14 +442,8 @@ reth_codecs::impl_compression_for_compact!(StageCheckpoint); /// Saves the progress of the Finish stage. #[derive(Default, Debug, Copy, Clone, PartialEq, Eq)] #[cfg_attr(any(test, feature = "test-utils"), derive(arbitrary::Arbitrary))] -#[cfg_attr( - all(any(test, feature = "reth-codec"), feature = "partial-persistence"), - derive(reth_codecs::Compact) -)] -#[cfg_attr( - all(any(test, feature = "reth-codec"), feature = "partial-persistence"), - reth_codecs::add_arbitrary_tests(compact) -)] +#[cfg_attr(any(test, feature = "reth-codec"), derive(reth_codecs::Compact))] +#[cfg_attr(any(test, feature = "reth-codec"), reth_codecs::add_arbitrary_tests(compact))] #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] pub struct FinishCheckpoint { /// The highest block with a partially persisted state and trie. @@ -457,30 +451,9 @@ pub struct FinishCheckpoint { } impl FinishCheckpoint { - /// Returns the highest block with a partially persisted state and trie, if enabled. + /// Returns the highest block with a partially persisted state and trie. pub const fn partial_state_trie(&self) -> Option { - #[cfg(feature = "partial-persistence")] - { - self.partial_state_trie - } - #[cfg(not(feature = "partial-persistence"))] - { - None - } - } -} - -#[cfg(all(any(test, feature = "reth-codec"), not(feature = "partial-persistence")))] -impl reth_codecs::Compact for FinishCheckpoint { - fn to_compact(&self, _buf: &mut B) -> usize - where - B: bytes::BufMut + AsMut<[u8]>, - { - panic!("serializing FinishCheckpoint requires the `partial-persistence` feature") - } - - fn from_compact(_buf: &[u8], _len: usize) -> (Self, &[u8]) { - panic!("deserializing FinishCheckpoint requires the `partial-persistence` feature") + self.partial_state_trie } } @@ -511,10 +484,6 @@ pub enum StageUnitCheckpoint { /// The `MerkleChangeSets` stage has been removed. MerkleChangeSets(MerkleChangeSetsCheckpoint), /// Saves the progress of the Finish stage. - #[cfg_attr( - all(any(test, feature = "test-utils"), not(feature = "partial-persistence")), - arbitrary(skip) - )] Finish(FinishCheckpoint), } @@ -725,7 +694,6 @@ mod tests { assert_eq!(decoded, checkpoint); } - #[cfg(feature = "partial-persistence")] #[test] fn finish_checkpoint_roundtrip() { let finish_checkpoint = FinishCheckpoint { partial_state_trie: Some(21) }; @@ -738,26 +706,4 @@ mod tests { assert_eq!(decoded, checkpoint); assert_eq!(decoded.finish_stage_checkpoint().unwrap().partial_state_trie(), Some(21)); } - - #[cfg(not(feature = "partial-persistence"))] - #[test] - #[should_panic( - expected = "serializing FinishCheckpoint requires the `partial-persistence` feature" - )] - fn finish_checkpoint_serialization_requires_partial_persistence() { - let finish_checkpoint = FinishCheckpoint { partial_state_trie: Some(21) }; - assert_eq!(finish_checkpoint.partial_state_trie(), None); - - let checkpoint = StageCheckpoint::new(42).with_finish_stage_checkpoint(finish_checkpoint); - checkpoint.to_compact(&mut Vec::new()); - } - - #[cfg(not(feature = "partial-persistence"))] - #[test] - #[should_panic( - expected = "deserializing FinishCheckpoint requires the `partial-persistence` feature" - )] - fn finish_checkpoint_deserialization_requires_partial_persistence() { - let _ = FinishCheckpoint::from_compact(&[], 0); - } } diff --git a/crates/storage/provider/Cargo.toml b/crates/storage/provider/Cargo.toml index 1ff298889c1..4cdeeb69ddd 100644 --- a/crates/storage/provider/Cargo.toml +++ b/crates/storage/provider/Cargo.toml @@ -88,10 +88,6 @@ tokio-stream = { workspace = true, features = ["sync"] } [features] jemalloc = ["rocksdb/jemalloc"] -partial-persistence = [ - "reth-stages-types/partial-persistence", - "reth-storage-overlay/partial-persistence", -] test-utils = [ "reth-db/test-utils", "reth-nippy-jar/test-utils", diff --git a/crates/storage/provider/src/providers/database/provider.rs b/crates/storage/provider/src/providers/database/provider.rs index 65a9c7edf0c..fe53665a5a6 100644 --- a/crates/storage/provider/src/providers/database/provider.rs +++ b/crates/storage/provider/src/providers/database/provider.rs @@ -4089,9 +4089,7 @@ mod tests { map::{AddressMap, B256Map}, U256, }; - #[cfg(feature = "partial-persistence")] - use reth_chain_state::test_utils::TestBlockBuilder; - use reth_chain_state::ExecutedBlock; + use reth_chain_state::{test_utils::TestBlockBuilder, ExecutedBlock}; use reth_db_api::models::StorageSettings; use reth_ethereum_primitives::Receipt; use reth_execution_types::{AccountRevertInit, BlockExecutionOutput, BlockExecutionResult}; @@ -4235,14 +4233,10 @@ mod tests { StateWriteConfig::default(), ) .unwrap(); - for i in 0..3 { - provider_rw.insert_block(&data.blocks[i].0).unwrap(); + for (block, outcome) in data.blocks.iter().take(3) { + provider_rw.insert_block(block).unwrap(); provider_rw - .write_state( - &data.blocks[i].1, - crate::OriginalValuesKnown::No, - StateWriteConfig::default(), - ) + .write_state(outcome, crate::OriginalValuesKnown::No, StateWriteConfig::default()) .unwrap(); } provider_rw.commit().unwrap(); @@ -4274,14 +4268,10 @@ mod tests { .unwrap(); // insert blocks 1-3 with receipts - for i in 0..3 { - provider_rw.insert_block(&data.blocks[i].0).unwrap(); + for (block, outcome) in data.blocks.iter().take(3) { + provider_rw.insert_block(block).unwrap(); provider_rw - .write_state( - &data.blocks[i].1, - crate::OriginalValuesKnown::No, - StateWriteConfig::default(), - ) + .write_state(outcome, crate::OriginalValuesKnown::No, StateWriteConfig::default()) .unwrap(); } provider_rw.commit().unwrap(); @@ -4310,14 +4300,10 @@ mod tests { StateWriteConfig::default(), ) .unwrap(); - for i in 0..3 { - provider_rw.insert_block(&data.blocks[i].0).unwrap(); + for (block, outcome) in data.blocks.iter().take(3) { + provider_rw.insert_block(block).unwrap(); provider_rw - .write_state( - &data.blocks[i].1, - crate::OriginalValuesKnown::No, - StateWriteConfig::default(), - ) + .write_state(outcome, crate::OriginalValuesKnown::No, StateWriteConfig::default()) .unwrap(); } provider_rw.commit().unwrap(); @@ -4380,14 +4366,10 @@ mod tests { StateWriteConfig::default(), ) .unwrap(); - for i in 0..3 { - provider_rw.insert_block(&data.blocks[i].0).unwrap(); + for (block, outcome) in data.blocks.iter().take(3) { + provider_rw.insert_block(block).unwrap(); provider_rw - .write_state( - &data.blocks[i].1, - crate::OriginalValuesKnown::No, - StateWriteConfig::default(), - ) + .write_state(outcome, crate::OriginalValuesKnown::No, StateWriteConfig::default()) .unwrap(); } provider_rw.commit().unwrap(); @@ -4642,7 +4624,6 @@ mod tests { provider_rw.commit().unwrap(); } - #[cfg(feature = "partial-persistence")] #[test] fn test_save_blocks_only_masks_trie_with_deferred_blocks() { use reth_trie::{ @@ -4864,7 +4845,6 @@ mod tests { assert_eq!(masked_entries[0].1.nibbles.0, masked_storage_node); } - #[cfg(feature = "partial-persistence")] #[test] fn test_save_blocks_merges_storage_wipe_in_multi_block_batch() { use reth_trie::{ @@ -4930,7 +4910,6 @@ mod tests { assert!(storage_entries.is_empty()); } - #[cfg(feature = "partial-persistence")] #[test] fn test_save_blocks_batches_transient_storage_wipe() { use alloy_primitives::map::B256Set; @@ -4981,7 +4960,6 @@ mod tests { assert_eq!(checkpoint.block_number, 3); } - #[cfg(feature = "partial-persistence")] #[test] fn test_save_blocks_partial_cycles_do_not_duplicate_static_file_writes() { let factory = create_test_provider_factory(); @@ -5030,7 +5008,6 @@ mod tests { ); } - #[cfg(feature = "partial-persistence")] #[test] fn remove_block_and_execution_above_returns_persistence_frontiers() { let factory = create_test_provider_factory(); @@ -5043,6 +5020,10 @@ mod tests { save_genesis(&provider_rw, &genesis).unwrap(); provider_rw.commit().unwrap(); + for block in &blocks[2..] { + factory.overlay_manager().insert_block(block.clone()); + } + let provider_rw = factory.provider_rw().unwrap(); let input = SaveBlocksInput::new(blocks, 0, 0, 4, 2); provider_rw.save_blocks(&input).unwrap(); diff --git a/crates/storage/storage-overlay/Cargo.toml b/crates/storage/storage-overlay/Cargo.toml index 316ca8e8ef4..08a1d77a7c8 100644 --- a/crates/storage/storage-overlay/Cargo.toml +++ b/crates/storage/storage-overlay/Cargo.toml @@ -50,5 +50,4 @@ reth-trie = { workspace = true, features = ["test-utils"] } alloy-consensus.workspace = true [features] -partial-persistence = ["reth-stages-types/partial-persistence"] rayon = ["dep:rayon", "dep:reth-tasks"] diff --git a/crates/storage/storage-overlay/src/builder.rs b/crates/storage/storage-overlay/src/builder.rs index 2452da5cf30..ae2a9088c42 100644 --- a/crates/storage/storage-overlay/src/builder.rs +++ b/crates/storage/storage-overlay/src/builder.rs @@ -610,15 +610,11 @@ mod tests { use alloy_primitives::U256; use reth_chain_state::{test_utils::TestBlockBuilder, ExecutedBlock}; use reth_primitives_traits::Account; - #[cfg(feature = "partial-persistence")] use reth_provider::{ test_utils::{create_test_provider_factory, MockNodeTypesWithDB}, BlockWriter, ProviderFactory, }; - #[cfg(feature = "partial-persistence")] - #[cfg(feature = "partial-persistence")] use reth_stages_types::{FinishCheckpoint, StageCheckpoint}; - #[cfg(feature = "partial-persistence")] use reth_storage_api::StageCheckpointWriter; use reth_trie::{BranchNodeCompact, ComputedTrieData, HashedPostState, HashedStorage, Nibbles}; @@ -658,7 +654,6 @@ mod tests { .collect() } - #[cfg(feature = "partial-persistence")] fn setup_frontiers( state_trie_tip_index: usize, finish_tip_index: usize, @@ -683,17 +678,14 @@ mod tests { (factory, blocks) } - #[cfg(feature = "partial-persistence")] fn account_keys(overlay: &Overlay) -> Vec { overlay.hashed_post_state.accounts.iter().map(|(key, _)| *key).collect() } - #[cfg(feature = "partial-persistence")] fn account_node_paths(overlay: &Overlay) -> Vec { overlay.trie_updates.account_nodes_ref().iter().map(|(path, _)| *path).collect() } - #[cfg(feature = "partial-persistence")] #[test] fn managed_overlay_starts_at_state_trie_frontier() { let (factory, blocks) = setup_frontiers(1, 3); @@ -724,7 +716,6 @@ mod tests { } } - #[cfg(feature = "partial-persistence")] #[test] fn managed_overlay_skips_when_finish_is_the_anchor() { let (factory, blocks) = setup_frontiers(3, 3); @@ -742,7 +733,6 @@ mod tests { assert!(overlay.trie_updates.is_empty()); } - #[cfg(feature = "partial-persistence")] #[test] fn no_reverts_errors_when_reverts_are_required() { let (factory, blocks) = setup_frontiers(2, 3); @@ -757,7 +747,6 @@ mod tests { assert!(error.to_string().contains("reverts are disabled")); } - #[cfg(feature = "partial-persistence")] #[test] fn managed_overlay_uses_persisted_parent_even_if_retained() { let (factory, blocks) = setup_frontiers(2, 3); @@ -775,14 +764,8 @@ mod tests { panic!("persisted parent below Finish must require reverts") } } - - let overlay = builder.build_overlay(&provider).unwrap(); - - assert!(overlay.hashed_post_state.is_empty()); - assert!(overlay.trie_updates.is_empty()); } - #[cfg(feature = "partial-persistence")] #[test] fn overlay_after_state_trie_frontier_requires_managed_coverage() { let (factory, blocks) = setup_frontiers(1, 3); @@ -799,7 +782,6 @@ mod tests { ); } - #[cfg(feature = "partial-persistence")] #[test] fn managed_overlay_errors_if_parent_is_not_persisted_or_managed_across_frontiers() { let (factory, blocks) = setup_frontiers(1, 3); diff --git a/crates/storage/storage-overlay/src/provider.rs b/crates/storage/storage-overlay/src/provider.rs index c5f38269c0d..8f6e57728ce 100644 --- a/crates/storage/storage-overlay/src/provider.rs +++ b/crates/storage/storage-overlay/src/provider.rs @@ -250,7 +250,7 @@ where } } -#[cfg(all(test, feature = "partial-persistence"))] +#[cfg(test)] mod tests { use super::*; use crate::OverlayManager; diff --git a/docker-bake.hcl b/docker-bake.hcl index 82b8992a63e..6078af6b74e 100644 --- a/docker-bake.hcl +++ b/docker-bake.hcl @@ -35,7 +35,7 @@ group "default" { } group "nightly" { - targets = ["ethereum", "ethereum-profiling", "ethereum-partial-persistence"] + targets = ["ethereum", "ethereum-profiling"] } // Base target with shared configuration @@ -71,16 +71,6 @@ target "ethereum" { tags = ["${REGISTRY}/reth:${TAG}"] } -target "ethereum-partial-persistence" { - inherits = ["_base"] - args = { - BINARY = "reth" - MANIFEST_PATH = "bin/reth" - FEATURES = "partial-persistence" - } - tags = ["${REGISTRY}/reth:nightly-partial-persistence"] -} - target "ethereum-profiling" { inherits = ["_base_profiling"] args = { diff --git a/docs/vocs/docs/pages/cli/reth/node.mdx b/docs/vocs/docs/pages/cli/reth/node.mdx index ed65fd43fe2..d0e688744db 100644 --- a/docs/vocs/docs/pages/cli/reth/node.mdx +++ b/docs/vocs/docs/pages/cli/reth/node.mdx @@ -1011,6 +1011,11 @@ Engine: This value must be greater than `--engine.persistence-threshold`. + --engine.num-state-masking-blocks + EXPERIMENTAL: Configure how many of the blocks being persisted should only mask state/trie writes instead of durably persisting their state/trie updates in the current cycle + + [default: 0] + --engine.memory-block-buffer-target Configure the target number of blocks to keep in memory. From b472a11f8caae8fdd194460d772665a77f503ab8 Mon Sep 17 00:00:00 2001 From: Derek Cofausper <256792747+decofe@users.noreply.github.com> Date: Tue, 25 Aug 2026 13:21:28 +0000 Subject: [PATCH 062/294] feat(node): allow KZG warmup without EIP-4844 pool support (#26808) Co-authored-by: Matthias Seitz <19890894+mattsse@users.noreply.github.com> --- crates/ethereum/node/src/node.rs | 36 +++++++++++++++++++++++++++++--- 1 file changed, 33 insertions(+), 3 deletions(-) diff --git a/crates/ethereum/node/src/node.rs b/crates/ethereum/node/src/node.rs index 0746e679198..bc1b49ab7cf 100644 --- a/crates/ethereum/node/src/node.rs +++ b/crates/ethereum/node/src/node.rs @@ -639,10 +639,29 @@ where /// /// This contains various settings that can be configured and take precedence over the node's /// config. -#[derive(Debug, Default, Clone, Copy)] +#[derive(Debug, Clone, Copy)] #[non_exhaustive] pub struct EthereumPoolBuilder { - // TODO add options for txpool args + init_kzg_settings: bool, +} + +impl EthereumPoolBuilder { + /// Creates a new [`EthereumPoolBuilder`]. + pub const fn new() -> Self { + Self { init_kzg_settings: false } + } + + /// Sets whether to initialize KZG settings even if EIP-4844 support is disabled in the pool. + pub const fn with_init_kzg_settings(mut self, init_kzg_settings: bool) -> Self { + self.init_kzg_settings = init_kzg_settings; + self + } +} + +impl Default for EthereumPoolBuilder { + fn default() -> Self { + Self::new() + } } impl PoolBuilder for EthereumPoolBuilder @@ -698,7 +717,7 @@ where .with_additional_tasks(ctx.config().txpool.additional_validation_tasks) .build_with_tasks(ctx.task_executor().clone(), blob_store.clone()); - if validator.validator().eip4844() { + if validator.validator().eip4844() || self.init_kzg_settings { // initializing the KZG settings can be expensive, this should be done upfront so that // it doesn't impact the first block or the first gossiped blob transaction, so we // initialize this in the background @@ -788,3 +807,14 @@ where Ok(EthereumEngineValidator::new(ctx.config.chain.clone())) } } + +#[cfg(test)] +mod tests { + use super::EthereumPoolBuilder; + + #[test] + fn configures_kzg_settings_initialization() { + assert!(!EthereumPoolBuilder::new().init_kzg_settings); + assert!(EthereumPoolBuilder::new().with_init_kzg_settings(true).init_kzg_settings); + } +} From e1ec6b53be53b7e66064924c6c208d3f6c98e0d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A9a=20Narzis?= <78718413+lean-apple@users.noreply.github.com> Date: Tue, 25 Aug 2026 13:46:02 +0000 Subject: [PATCH 063/294] feat(downloader): add snap account range downloader (#26657) Co-authored-by: Matthias Seitz --- Cargo.lock | 3 + crates/net/downloaders/Cargo.toml | 9 +- crates/net/downloaders/src/lib.rs | 3 + crates/net/downloaders/src/snap/mod.rs | 726 ++++++++++++++++++++++++ crates/net/eth-wire-types/src/snap.rs | 10 +- crates/trie/common/Cargo.toml | 2 + crates/trie/common/src/lib.rs | 3 + crates/trie/common/src/range_proof.rs | 745 +++++++++++++++++++++++++ 8 files changed, 1497 insertions(+), 4 deletions(-) create mode 100644 crates/net/downloaders/src/snap/mod.rs create mode 100644 crates/trie/common/src/range_proof.rs diff --git a/Cargo.lock b/Cargo.lock index 14b5a2e981e..9cecf5f880e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8272,6 +8272,7 @@ dependencies = [ "reth-chainspec", "reth-config", "reth-consensus", + "reth-eth-wire-types", "reth-ethereum-primitives", "reth-metrics", "reth-network-p2p", @@ -8282,6 +8283,7 @@ dependencies = [ "reth-tasks", "reth-testing-utils", "reth-tracing", + "reth-trie-common", "tempfile", "thiserror 2.0.18", "tokio", @@ -10697,6 +10699,7 @@ dependencies = [ "serde", "serde_json", "serde_with", + "thiserror 2.0.18", ] [[package]] diff --git a/crates/net/downloaders/Cargo.toml b/crates/net/downloaders/Cargo.toml index 709c30fe510..b9f052962bc 100644 --- a/crates/net/downloaders/Cargo.toml +++ b/crates/net/downloaders/Cargo.toml @@ -15,11 +15,13 @@ workspace = true # reth reth-config.workspace = true reth-consensus.workspace = true +reth-eth-wire-types.workspace = true reth-network-p2p.workspace = true reth-network-peers.workspace = true reth-primitives-traits.workspace = true reth-storage-api.workspace = true reth-tasks.workspace = true +reth-trie-common.workspace = true # optional deps for the test-utils feature reth-ethereum-primitives = { workspace = true, optional = true } @@ -30,13 +32,13 @@ reth-testing-utils = { workspace = true, optional = true } alloy-consensus.workspace = true alloy-eips.workspace = true alloy-primitives.workspace = true -alloy-rlp = { workspace = true, optional = true } +alloy-rlp.workspace = true # async futures.workspace = true futures-util.workspace = true pin-project.workspace = true -tokio = { workspace = true, features = ["sync", "fs", "io-util"] } +tokio = { workspace = true, features = ["sync", "fs", "io-util", "rt"] } tokio-stream.workspace = true tokio-util = { workspace = true, features = ["codec"] } async-compression = { workspace = true, features = ["gzip", "tokio"], optional = true } @@ -72,7 +74,7 @@ itertools.workspace = true [features] default = [] -file-client = ["dep:async-compression", "dep:alloy-rlp", "dep:itertools"] +file-client = ["dep:async-compression", "dep:itertools"] test-utils = [ "tempfile", "reth-consensus/test-utils", @@ -84,4 +86,5 @@ test-utils = [ "dep:reth-ethereum-primitives", "reth-ethereum-primitives?/test-utils", "reth-tasks/test-utils", + "reth-trie-common/test-utils", ] diff --git a/crates/net/downloaders/src/lib.rs b/crates/net/downloaders/src/lib.rs index ed0a65ba95e..6178d533c12 100644 --- a/crates/net/downloaders/src/lib.rs +++ b/crates/net/downloaders/src/lib.rs @@ -25,6 +25,9 @@ pub mod headers; /// Common downloader metrics. pub mod metrics; +/// Downloaders for authenticated snap state ranges. +pub mod snap; + /// Module managing file-based data retrieval and buffering. /// /// Contains [`FileClient`](file_client::FileClient) to read block data from files, diff --git a/crates/net/downloaders/src/snap/mod.rs b/crates/net/downloaders/src/snap/mod.rs new file mode 100644 index 00000000000..ac5c47b90a6 --- /dev/null +++ b/crates/net/downloaders/src/snap/mod.rs @@ -0,0 +1,726 @@ +//! Downloads and verifies snap/2 account ranges against +//! [EIP-8189](https://eips.ethereum.org/EIPS/eip-8189) pivot state roots. +//! +//! Persistence and range selection are handled by the snap sync orchestrator. + +use alloy_primitives::B256; +use futures::{Future, FutureExt}; +use reth_eth_wire_types::snap::{AccountRangeMessage, GetAccountRangeMessage}; +use reth_network_p2p::{ + error::{PeerRequestResult, RequestError}, + priority::Priority, + snap::client::{SnapClient, SnapResponse}, +}; +use reth_network_peers::PeerId; +use reth_tasks::Runtime; +use reth_trie_common::{range_proof::verify_range_proof, TrieAccount, EMPTY_ROOT_HASH}; +use std::{ + pin::Pin, + task::{ready, Context, Poll}, +}; +use tracing::debug; + +const MAX_RETRIES: u8 = 2; + +/// Downloads and verifies one account range against its requested state root. +/// +/// Invalid responses penalize their peer and retry at high priority. Proof verification runs on +/// the blocking pool. +#[derive(Debug)] +pub struct AccountRangeDownloader { + client: C, + runtime: Runtime, + request: GetAccountRangeMessage, + fut: C::Output, + verification: Option, + retries: u8, +} + +impl AccountRangeDownloader { + /// Creates a downloader using `runtime` for proof verification and submits the initial request. + /// Returns an error when the origin exceeds the limit. + pub fn new( + client: C, + request: GetAccountRangeMessage, + runtime: Runtime, + ) -> Result { + if request.starting_hash > request.limit_hash { + return Err(InvalidAccountRange { + origin: request.starting_hash, + limit: request.limit_hash, + }) + } + let fut = client.get_account_range(request.clone()); + Ok(Self { client, runtime, request, fut, verification: None, retries: 0 }) + } + + // Raise retry priority so transient failures cannot leave range progress behind new work. + fn retry(&mut self) -> bool { + if self.retries >= MAX_RETRIES { + return false + } + self.retries += 1; + self.fut = + self.client.get_account_range_with_priority(self.request.clone(), Priority::High); + true + } + + // Verify peer-controlled proofs off the async worker while retaining peer attribution. + fn start_verification( + &mut self, + peer_id: PeerId, + response: SnapResponse, + ) -> Result, RequestError> { + let response = self.account_range_response(response)?; + + if response.accounts.is_empty() && response.proof.is_empty() { + return if self.request.root_hash == EMPTY_ROOT_HASH { + Ok(Some(AccountRangeOutcome::Verified(VerifiedAccountRange { + accounts: Vec::new(), + has_more: false, + }))) + } else { + Ok(Some(AccountRangeOutcome::Unavailable { peer_id })) + } + } + + let request = self.request.clone(); + let fut = self.runtime.spawn_blocking(move || verify_account_range(&request, response)); + self.verification = Some(VerificationTask { peer_id, fut }); + Ok(None) + } + + // Bind replies to the expected request before trusting peer-supplied data. + fn account_range_response( + &self, + response: SnapResponse, + ) -> Result { + let SnapResponse::AccountRange(response) = response else { + debug!(target: "downloaders::snap", "Expected account range response"); + return Err(RequestError::BadResponse) + }; + if response.request_id != self.request.request_id { + debug!( + target: "downloaders::snap", + expected = self.request.request_id, + got = response.request_id, + "Account range response id mismatch" + ); + return Err(RequestError::BadResponse) + } + Ok(response) + } + + // Keep validation and retry accounting together so peers are penalized exactly once. + fn handle_response( + &mut self, + response: PeerRequestResult, + ) -> Result, RequestError> { + match response { + Ok(response) => { + let (peer_id, response) = response.split(); + match self.start_verification(peer_id, response) { + Ok(outcome) => Ok(outcome), + Err(error) => { + debug!(target: "downloaders::snap", ?peer_id, %error, "Invalid account range response"); + self.client.report_bad_message(peer_id); + self.retry().then_some(None).ok_or(error) + } + } + } + // A wrong wire response is already penalized by the session. + Err(error) if error.is_retryable() || error == RequestError::BadResponse => { + debug!(target: "downloaders::snap", %error, "Account range request failed, retrying"); + self.retry().then_some(None).ok_or(error) + } + Err(error) => Err(error), + } + } + + // Preserve peer attribution until blocking verification finishes. + fn poll_verification( + &mut self, + cx: &mut Context<'_>, + ) -> Poll, RequestError>> { + let verification = self.verification.as_mut().expect("verification task is present"); + let result = ready!(verification.fut.poll_unpin(cx)); + let peer_id = verification.peer_id; + self.verification = None; + + match result { + Ok(Ok(range)) => Poll::Ready(Ok(Some(AccountRangeOutcome::Verified(range)))), + Ok(Err(error)) => { + debug!(target: "downloaders::snap", ?peer_id, %error, "Invalid account range response"); + self.client.report_bad_message(peer_id); + Poll::Ready(self.retry().then_some(None).ok_or(error)) + } + Err(error) => { + debug!(target: "downloaders::snap", %error, "Account range verification task failed"); + Poll::Ready(Err(RequestError::ChannelClosed)) + } + } + } +} + +impl Future for AccountRangeDownloader +where + C: SnapClient + Unpin + 'static, +{ + type Output = Result; + + // Finish an active verification before accepting another response. + fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + let this = self.get_mut(); + + loop { + if this.verification.is_some() { + match ready!(this.poll_verification(cx)) { + Ok(Some(outcome)) => return Poll::Ready(Ok(outcome)), + Ok(None) => {} + Err(error) => return Poll::Ready(Err(error)), + } + } + + let response = ready!(this.fut.poll_unpin(cx)); + match this.handle_response(response) { + Ok(Some(outcome)) => return Poll::Ready(Ok(outcome)), + Ok(None) => {} + Err(error) => return Poll::Ready(Err(error)), + } + } + } +} + +/// The result of an authenticated account-range request. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum AccountRangeOutcome { + /// The peer does not have the requested state root and was not penalized. + Unavailable { + /// The peer that answered. + peer_id: PeerId, + }, + /// An account range authenticated against the requested state root. + Verified(VerifiedAccountRange), +} + +// Couples blocking proof work with its responder so failures remain attributable. +#[derive(Debug)] +struct VerificationTask { + // Identifies the responder to penalize if verification rejects the range. + peer_id: PeerId, + // Carries the verified result back without blocking the async worker. + fut: tokio::task::JoinHandle>, +} + +/// A decoded account range authenticated against a state root. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct VerifiedAccountRange { + /// Accounts in strictly increasing hashed-key order. + pub accounts: Vec<(B256, TrieAccount)>, + /// Whether another request is needed to complete the requested interval. + pub has_more: bool, +} + +// Authenticate the full response before trimming its optional boundary account. +fn verify_account_range( + request: &GetAccountRangeMessage, + response: AccountRangeMessage, +) -> Result { + // Allow only the single out-of-range account needed as a boundary witness. + if response.accounts.iter().filter(|data| data.hash > request.limit_hash).nth(1).is_some() { + debug!(target: "downloaders::snap", "Account range runs past the requested limit"); + return Err(RequestError::BadResponse) + } + + // Decode first so malformed account values are attributed to the responder. + let mut accounts = response + .accounts + .into_iter() + .map(|data| { + data.into_trie_entry().map_err(|error| { + debug!(target: "downloaders::snap", %error, "Invalid account data"); + RequestError::BadResponse + }) + }) + .collect::, _>>()?; + let next = verify_proof(request, &accounts, &response.proof)?; + + // Authenticate the boundary account before removing it from the requested range. + accounts.truncate(accounts.partition_point(|(hash, _)| *hash <= request.limit_hash)); + let has_more = next.is_some_and(|next| next <= request.limit_hash); + + Ok(VerifiedAccountRange { accounts, has_more }) +} + +// Re-encode decoded accounts so the proof authenticates their canonical trie values. +fn verify_proof( + request: &GetAccountRangeMessage, + accounts: &[(B256, TrieAccount)], + proof: &[alloy_primitives::Bytes], +) -> Result, RequestError> { + let leaves = accounts.iter().map(|(hash, account)| (*hash, alloy_rlp::encode(account))); + verify_range_proof(request.root_hash, request.starting_hash, leaves, proof).map_err(|error| { + debug!(target: "downloaders::snap", %error, "Invalid account range proof"); + RequestError::BadResponse + }) +} + +/// An account-range request whose origin exceeds its limit. +#[derive(Clone, Copy, Debug, PartialEq, Eq, thiserror::Error)] +#[error("account range origin {origin} exceeds limit {limit}")] +pub struct InvalidAccountRange { + origin: B256, + limit: B256, +} + +#[cfg(test)] +mod tests { + use super::*; + use alloy_primitives::{Bytes, KECCAK256_EMPTY, U256}; + use futures::future::{ready, Ready}; + use reth_eth_wire_types::snap::{ + AccountData, ByteCodesMessage, GetBlockAccessListsMessage, GetByteCodesMessage, + GetStorageRangesMessage, + }; + use reth_network_p2p::download::DownloadClient; + use reth_network_peers::WithPeerId; + use reth_trie_common::{proof::ProofRetainer, HashBuilder, Nibbles}; + use std::{ + collections::VecDeque, + sync::{Arc, Mutex}, + }; + + const MAX_HASH: B256 = B256::new([0xff; B256::len_bytes()]); + + #[derive(Debug)] + struct TestSnapClient { + responses: Mutex>>, + reported: Mutex>, + priorities: Mutex>, + } + + impl TestSnapClient { + fn new(responses: impl IntoIterator>) -> Self { + Self { + responses: Mutex::new(responses.into_iter().collect()), + reported: Mutex::new(Vec::new()), + priorities: Mutex::new(Vec::new()), + } + } + + fn next(&self, priority: Priority) -> Ready> { + self.priorities.lock().unwrap().push(priority); + ready(self.responses.lock().unwrap().pop_front().expect("test response available")) + } + } + + impl DownloadClient for TestSnapClient { + fn report_bad_message(&self, peer_id: PeerId) { + self.reported.lock().unwrap().push(peer_id); + } + + fn num_connected_peers(&self) -> usize { + 1 + } + } + + impl SnapClient for TestSnapClient { + type Output = Ready>; + + fn get_account_range_with_priority( + &self, + _request: GetAccountRangeMessage, + priority: Priority, + ) -> Self::Output { + self.next(priority) + } + + fn get_storage_ranges(&self, _request: GetStorageRangesMessage) -> Self::Output { + ready(Err(RequestError::UnsupportedCapability)) + } + + fn get_storage_ranges_with_priority( + &self, + _request: GetStorageRangesMessage, + _priority: Priority, + ) -> Self::Output { + ready(Err(RequestError::UnsupportedCapability)) + } + + fn get_byte_codes(&self, _request: GetByteCodesMessage) -> Self::Output { + ready(Err(RequestError::UnsupportedCapability)) + } + + fn get_byte_codes_with_priority( + &self, + _request: GetByteCodesMessage, + _priority: Priority, + ) -> Self::Output { + ready(Err(RequestError::UnsupportedCapability)) + } + + fn get_block_access_lists_with_priority( + &self, + _request: GetBlockAccessListsMessage, + _priority: Priority, + ) -> Self::Output { + ready(Err(RequestError::UnsupportedCapability)) + } + } + + fn key(value: u64) -> B256 { + B256::left_padding_from(&value.to_be_bytes()) + } + + fn account(nonce: u64) -> TrieAccount { + TrieAccount { + nonce, + balance: U256::from(1), + storage_root: EMPTY_ROOT_HASH, + code_hash: KECCAK256_EMPTY, + } + } + + fn root(accounts: &[(B256, TrieAccount)]) -> B256 { + let mut builder = HashBuilder::default(); + for (key, account) in accounts { + builder.add_leaf(Nibbles::unpack(*key), &alloy_rlp::encode(account)); + } + builder.root() + } + + fn root_and_proof(accounts: &[(B256, TrieAccount)], targets: &[B256]) -> (B256, Vec) { + let targets = targets.iter().copied().map(Nibbles::unpack).collect(); + let mut builder = HashBuilder::default().with_proof_retainer(ProofRetainer::new(targets)); + for (key, account) in accounts { + builder.add_leaf(Nibbles::unpack(*key), &alloy_rlp::encode(account)); + } + let root = builder.root(); + let proof = builder + .take_proof_nodes() + .into_nodes_sorted() + .into_iter() + .map(|(_, node)| node) + .collect(); + (root, proof) + } + + fn request(root_hash: B256) -> GetAccountRangeMessage { + GetAccountRangeMessage { + request_id: 1, + root_hash, + starting_hash: B256::ZERO, + limit_hash: MAX_HASH, + response_bytes: 512 * 1024, + } + } + + fn response(peer: PeerId, message: AccountRangeMessage) -> PeerRequestResult { + Ok(WithPeerId::new(peer, SnapResponse::AccountRange(message))) + } + + fn downloader( + client: Arc, + request: GetAccountRangeMessage, + ) -> Result>, InvalidAccountRange> { + AccountRangeDownloader::new(client, request, Runtime::test()) + } + + #[test] + fn verifies_and_decodes_without_an_ambient_runtime() { + let accounts = vec![(key(1), account(7)), (key(2), account(8))]; + let root_hash = root(&accounts); + let peer = PeerId::random(); + let message = AccountRangeMessage { + request_id: 1, + accounts: accounts + .iter() + .map(|(key, account)| AccountData::from_trie_account(*key, account)) + .collect(), + proof: Vec::new(), + }; + let client = Arc::new(TestSnapClient::new([response(peer, message)])); + + let downloader = downloader(Arc::clone(&client), request(root_hash)).unwrap(); + let outcome = futures::executor::block_on(downloader).unwrap(); + + assert_eq!( + outcome, + AccountRangeOutcome::Verified(VerifiedAccountRange { accounts, has_more: false }) + ); + assert!(client.reported.lock().unwrap().is_empty()); + assert_eq!(*client.priorities.lock().unwrap(), [Priority::Normal]); + } + + #[tokio::test] + async fn invalid_peer_is_reported_and_request_is_retried_at_high_priority() { + let accounts = vec![(key(1), account(7))]; + let root_hash = root(&accounts); + let bad_peer = PeerId::random(); + let good_peer = PeerId::random(); + let bad = Ok(WithPeerId::new( + bad_peer, + SnapResponse::ByteCodes(ByteCodesMessage { request_id: 1, codes: Vec::new() }), + )); + let good = response( + good_peer, + AccountRangeMessage { + request_id: 1, + accounts: vec![AccountData::from_trie_account(accounts[0].0, &accounts[0].1)], + proof: Vec::new(), + }, + ); + let client = Arc::new(TestSnapClient::new([bad, good])); + + let outcome = downloader(Arc::clone(&client), request(root_hash)).unwrap().await.unwrap(); + + assert!(matches!(outcome, AccountRangeOutcome::Verified(_))); + assert_eq!(*client.reported.lock().unwrap(), [bad_peer]); + assert_eq!(*client.priorities.lock().unwrap(), [Priority::Normal, Priority::High]); + } + + #[tokio::test] + async fn unavailable_state_is_not_a_bad_peer_response() { + let peer = PeerId::random(); + let message = + AccountRangeMessage { request_id: 1, accounts: Vec::new(), proof: Vec::new() }; + let client = Arc::new(TestSnapClient::new([response(peer, message)])); + + let outcome = downloader(Arc::clone(&client), request(B256::repeat_byte(0x11))) + .unwrap() + .await + .unwrap(); + + assert_eq!(outcome, AccountRangeOutcome::Unavailable { peer_id: peer }); + assert!(client.reported.lock().unwrap().is_empty()); + } + + #[test] + fn invalid_request_range_is_rejected_before_submission() { + let client = Arc::new(TestSnapClient::new(std::iter::empty())); + let mut request = request(B256::repeat_byte(0x11)); + request.starting_hash = key(2); + request.limit_hash = key(1); + + assert!(matches!( + downloader(Arc::clone(&client), request), + Err(InvalidAccountRange { .. }) + )); + assert!(client.priorities.lock().unwrap().is_empty()); + } + + #[tokio::test] + async fn authenticates_then_trims_an_account_past_the_limit() { + let accounts = vec![(key(1), account(7)), (key(3), account(8)), (key(4), account(9))]; + let (root_hash, proof) = root_and_proof(&accounts, &[key(1), key(3)]); + let peer = PeerId::random(); + let message = AccountRangeMessage { + request_id: 1, + accounts: accounts[..2] + .iter() + .map(|(key, account)| AccountData::from_trie_account(*key, account)) + .collect(), + proof, + }; + let client = Arc::new(TestSnapClient::new([response(peer, message)])); + let mut request = request(root_hash); + request.limit_hash = key(2); + + let outcome = downloader(Arc::clone(&client), request).unwrap().await.unwrap(); + + assert_eq!( + outcome, + AccountRangeOutcome::Verified(VerifiedAccountRange { + accounts: vec![accounts[0]], + has_more: false, + }) + ); + assert!(client.reported.lock().unwrap().is_empty()); + } + + #[tokio::test] + async fn range_running_past_the_limit_is_rejected() { + let accounts = vec![(key(1), account(7)), (key(3), account(8)), (key(4), account(9))]; + let (root_hash, proof) = root_and_proof(&accounts, &[key(1), key(4)]); + let peer = PeerId::random(); + let message = AccountRangeMessage { + request_id: 1, + accounts: accounts + .iter() + .map(|(key, account)| AccountData::from_trie_account(*key, account)) + .collect(), + proof, + }; + let attempts = usize::from(MAX_RETRIES) + 1; + let client = Arc::new(TestSnapClient::new( + std::iter::repeat_with(|| response(peer, message.clone())).take(attempts), + )); + let mut request = request(root_hash); + request.limit_hash = key(2); + + let error = downloader(Arc::clone(&client), request).unwrap().await.unwrap_err(); + + assert_eq!(error, RequestError::BadResponse); + assert_eq!(client.reported.lock().unwrap().len(), attempts); + } + + #[tokio::test] + async fn account_at_the_limit_completes_the_requested_interval() { + let accounts = vec![(key(1), account(7)), (key(2), account(8)), (key(3), account(9))]; + let (root_hash, proof) = root_and_proof(&accounts, &[key(1), key(2)]); + let peer = PeerId::random(); + let message = AccountRangeMessage { + request_id: 1, + accounts: accounts[..2] + .iter() + .map(|(key, account)| AccountData::from_trie_account(*key, account)) + .collect(), + proof, + }; + let client = Arc::new(TestSnapClient::new([response(peer, message)])); + let mut request = request(root_hash); + request.limit_hash = key(2); + + let outcome = downloader(Arc::clone(&client), request).unwrap().await.unwrap(); + + assert_eq!( + outcome, + AccountRangeOutcome::Verified(VerifiedAccountRange { + accounts: accounts[..2].to_vec(), + has_more: false, + }) + ); + assert!(client.reported.lock().unwrap().is_empty()); + } + + // The first account after the limit proves an empty interval. + #[tokio::test] + async fn empty_interval_is_proven_by_the_first_account_after_the_limit() { + let accounts = vec![(key(1), account(7)), (key(9), account(8))]; + let (root_hash, proof) = root_and_proof(&accounts, &[key(3), key(9)]); + let peer = PeerId::random(); + let message = AccountRangeMessage { + request_id: 1, + accounts: vec![AccountData::from_trie_account(accounts[1].0, &accounts[1].1)], + proof, + }; + let client = Arc::new(TestSnapClient::new([response(peer, message)])); + let mut request = request(root_hash); + request.starting_hash = key(3); + request.limit_hash = key(5); + + let outcome = downloader(Arc::clone(&client), request).unwrap().await.unwrap(); + + assert_eq!( + outcome, + AccountRangeOutcome::Verified(VerifiedAccountRange { + accounts: Vec::new(), + has_more: false, + }) + ); + assert!(client.reported.lock().unwrap().is_empty()); + } + + // A proof that continues past the limit completes the requested interval. + #[tokio::test] + async fn range_ending_before_the_limit_needs_no_further_request() { + let accounts = vec![(key(1), account(7)), (key(9), account(8))]; + let (root_hash, proof) = root_and_proof(&accounts, &[key(1)]); + let peer = PeerId::random(); + let message = AccountRangeMessage { + request_id: 1, + accounts: vec![AccountData::from_trie_account(accounts[0].0, &accounts[0].1)], + proof, + }; + let client = Arc::new(TestSnapClient::new([response(peer, message)])); + let mut request = request(root_hash); + request.limit_hash = key(5); + + let outcome = downloader(Arc::clone(&client), request).unwrap().await.unwrap(); + + assert_eq!( + outcome, + AccountRangeOutcome::Verified(VerifiedAccountRange { + accounts: vec![accounts[0]], + has_more: false, + }) + ); + } + + #[tokio::test] + async fn range_ending_before_a_covered_key_reports_more() { + let accounts = vec![(key(1), account(7)), (key(3), account(8))]; + let (root_hash, proof) = root_and_proof(&accounts, &[key(1)]); + let peer = PeerId::random(); + let message = AccountRangeMessage { + request_id: 1, + accounts: vec![AccountData::from_trie_account(accounts[0].0, &accounts[0].1)], + proof, + }; + let client = Arc::new(TestSnapClient::new([response(peer, message)])); + let mut request = request(root_hash); + request.limit_hash = key(5); + + let outcome = downloader(Arc::clone(&client), request).unwrap().await.unwrap(); + + assert_eq!( + outcome, + AccountRangeOutcome::Verified(VerifiedAccountRange { + accounts: vec![accounts[0]], + has_more: true, + }) + ); + } + + #[tokio::test] + async fn request_errors_retry_without_duplicate_peer_penalties() { + let accounts = vec![(key(1), account(7))]; + let root_hash = root(&accounts); + let peer = PeerId::random(); + let good = response( + peer, + AccountRangeMessage { + request_id: 1, + accounts: vec![AccountData::from_trie_account(accounts[0].0, &accounts[0].1)], + proof: Vec::new(), + }, + ); + let client = Arc::new(TestSnapClient::new([ + Err(RequestError::Timeout), + Err(RequestError::BadResponse), + good, + ])); + + downloader(Arc::clone(&client), request(root_hash)).unwrap().await.unwrap(); + + assert!(client.reported.lock().unwrap().is_empty()); + assert_eq!( + *client.priorities.lock().unwrap(), + [Priority::Normal, Priority::High, Priority::High] + ); + } + + #[tokio::test] + async fn stops_after_the_retry_budget_is_exhausted() { + let peers = [PeerId::random(), PeerId::random(), PeerId::random()]; + let responses = peers.map(|peer| { + Ok(WithPeerId::new( + peer, + SnapResponse::ByteCodes(ByteCodesMessage { request_id: 1, codes: Vec::new() }), + )) + }); + let client = Arc::new(TestSnapClient::new(responses)); + + let error = downloader(Arc::clone(&client), request(B256::repeat_byte(0x11))) + .unwrap() + .await + .unwrap_err(); + + assert_eq!(error, RequestError::BadResponse); + assert_eq!(*client.reported.lock().unwrap(), peers); + assert_eq!( + *client.priorities.lock().unwrap(), + [Priority::Normal, Priority::High, Priority::High] + ); + } +} diff --git a/crates/net/eth-wire-types/src/snap.rs b/crates/net/eth-wire-types/src/snap.rs index 1711cc77645..e86e1efb855 100644 --- a/crates/net/eth-wire-types/src/snap.rs +++ b/crates/net/eth-wire-types/src/snap.rs @@ -140,6 +140,12 @@ impl AccountData { code_hash: SlimAccountBody::restore(&slim.code_hash, KECCAK256_EMPTY)?, }) } + + /// Consumes the wire value and returns its hashed key with the decoded trie account. + pub fn into_trie_entry(self) -> alloy_rlp::Result<(B256, TrieAccount)> { + let account = self.trie_account()?; + Ok((self.hash, account)) + } } /// Response containing a number of consecutive accounts and the Merkle proofs for the entire range. @@ -885,12 +891,14 @@ mod tests { #[test] fn slim_body_elides_empty_storage_and_code() { let account = trie_account(EMPTY_ROOT_HASH, KECCAK256_EMPTY); - let encoded = AccountData::from_trie_account(B256::repeat_byte(1), &account); + let hash = B256::repeat_byte(1); + let encoded = AccountData::from_trie_account(hash, &account); let body = SlimAccountBody::decode(&mut encoded.body.as_ref()).unwrap(); assert!(body.storage_root.is_empty()); assert!(body.code_hash.is_empty()); assert_eq!(encoded.trie_account().unwrap(), account); + assert_eq!(encoded.into_trie_entry().unwrap(), (hash, account)); } #[test] diff --git a/crates/trie/common/Cargo.toml b/crates/trie/common/Cargo.toml index c2d1ef596f8..7df50c735f6 100644 --- a/crates/trie/common/Cargo.toml +++ b/crates/trie/common/Cargo.toml @@ -28,6 +28,7 @@ bytes = { workspace = true, optional = true } derive_more.workspace = true itertools = { workspace = true, features = ["use_alloc"] } nybbles = { workspace = true, features = ["rlp"] } +thiserror.workspace = true # reth revm.workspace = true @@ -83,6 +84,7 @@ std = [ "serde?/std", "serde_with?/std", "serde_json/std", + "thiserror/std", "revm/std", "reth-codecs?/std", "alloy-eips/std", diff --git a/crates/trie/common/src/lib.rs b/crates/trie/common/src/lib.rs index 7aeaf0de258..6f2408c06db 100644 --- a/crates/trie/common/src/lib.rs +++ b/crates/trie/common/src/lib.rs @@ -63,6 +63,9 @@ pub use trie::{BranchNodeMasks, BranchNodeMasksMap, ProofTrieNode}; mod trie_node_v2; pub use trie_node_v2::*; +/// Merkle Patricia trie range-proof verification. +pub mod range_proof; + /// The implementation of a container for storing intermediate changes to a trie. /// The container indicates when the trie has been modified. pub mod prefix_set; diff --git a/crates/trie/common/src/range_proof.rs b/crates/trie/common/src/range_proof.rs new file mode 100644 index 00000000000..7f25d04f6ab --- /dev/null +++ b/crates/trie/common/src/range_proof.rs @@ -0,0 +1,745 @@ +//! Merkle Patricia trie range-proof verification. +//! +//! Reconstructs a trie root from consecutive hashed leaves and boundary proof nodes, rejecting +//! altered or incomplete ranges and reporting where the trie continues past the range. +//! Boundary paths are expanded, outside commitments are retained, and response leaves replace the +//! covered interior before the reconstructed root is compared with the requested root. + +use crate::{HashBuilder, Nibbles, RlpNode, TrieNode}; +use alloc::vec::Vec; +use alloy_primitives::{keccak256, map::B256Map, Bytes, B256}; +use alloy_rlp::Decodable; + +const KEY_NIBBLES: usize = B256::len_bytes() * 2; +const MAX_HASH: B256 = B256::new([0xff; B256::len_bytes()]); + +// Coordinates proof traversal and root reconstruction through one shared frontier. +struct RangeProofVerifier<'a> { + // Determines which trie paths belong to the response and which remain proof-owned. + range: ProofRange, + // Resolves hashed boundary references without depending on proof wire order. + nodes: ProofNodes<'a>, + // Accumulates the disjoint entries needed to reconstruct the requested root. + frontier: ProofFrontier, + // Tracks the lowest known path after the response to report whether the trie continues. + next: Option, +} + +impl<'a> RangeProofVerifier<'a> { + // Creates a verifier with fixed bounds so traversal cannot drift from the supplied leaf range. + fn new(left: B256, right: B256, proof: &'a [Bytes], frontier: ProofFrontier) -> Self { + Self { + range: ProofRange::new(left, right), + nodes: ProofNodes::new(proof), + frontier, + next: None, + } + } + + // Verifies the range by rebuilding its root, making omitted or altered leaves change the + // result. + fn verify(mut self, root: B256) -> Result, RangeProofError> { + self.visit_reference(Nibbles::new(), &RlpNode::word_rlp(&root))?; + + let got = self.frontier.root()?; + if got != root { + return Err(RangeProofError::RootMismatch { expected: root, got }) + } + Ok(self.next.as_ref().map(TriePath::lowest_key)) + } + + // Visits a trie reference, expanding only boundaries because response leaves replace the + // interior. + fn visit_reference( + &mut self, + prefix: Nibbles, + reference: &RlpNode, + ) -> Result<(), RangeProofError> { + match self.range.subtree_relation(&prefix)? { + SubtreeRelation::OutsideLeft => self.add_outside_reference(prefix, reference), + SubtreeRelation::OutsideRight => { + self.note_next(prefix); + self.add_outside_reference(prefix, reference) + } + SubtreeRelation::Inside => Ok(()), + SubtreeRelation::Boundary => { + let node = self.nodes.resolve(prefix, reference)?; + self.visit_node(node, prefix) + } + } + } + + // Visits a boundary node to expose the disjoint commitments needed for root reconstruction. + fn visit_node(&mut self, node: TrieNode, prefix: Nibbles) -> Result<(), RangeProofError> { + match node { + TrieNode::EmptyRoot => Ok(()), + TrieNode::Leaf(leaf) => { + let path = prefix.descend_leaf(&leaf.key)?; + match self.range.key_relation(&path) { + KeyRelation::Before => self.frontier.push_leaf(path, leaf.value), + KeyRelation::Inside => {} + KeyRelation::After => { + self.note_next(path); + self.frontier.push_leaf(path, leaf.value); + } + } + Ok(()) + } + TrieNode::Extension(extension) => { + self.visit_reference(prefix.descend_extension(&extension.key)?, &extension.child) + } + TrieNode::Branch(branch) => { + for (nibble, child) in branch + .as_ref() + .children() + .filter_map(|(nibble, child)| child.map(|child| (nibble, child))) + { + self.visit_reference(prefix.descend_child(nibble)?, child)?; + } + Ok(()) + } + } + } + + // Adds an outside reference without expanding hashes because returned leaves cannot overlap it. + fn add_outside_reference( + &mut self, + prefix: Nibbles, + reference: &RlpNode, + ) -> Result<(), RangeProofError> { + if let Some(hash) = reference.as_hash() { + self.frontier.push_subtree(prefix, hash); + return Ok(()) + } + self.add_outside_node(TrieNode::decode(&mut reference.as_slice())?, prefix) + } + + // Adds an inline outside node by descending until a retainable leaf or hashed child is reached. + fn add_outside_node(&mut self, node: TrieNode, prefix: Nibbles) -> Result<(), RangeProofError> { + match node { + TrieNode::EmptyRoot => Ok(()), + TrieNode::Leaf(leaf) => { + let path = prefix.descend_leaf(&leaf.key)?; + self.frontier.push_leaf(path, leaf.value); + Ok(()) + } + TrieNode::Extension(extension) => self + .add_outside_reference(prefix.descend_extension(&extension.key)?, &extension.child), + TrieNode::Branch(branch) => { + for (nibble, child) in branch + .as_ref() + .children() + .filter_map(|(nibble, child)| child.map(|child| (nibble, child))) + { + self.add_outside_reference(prefix.descend_child(nibble)?, child)?; + } + Ok(()) + } + } + } + + // Records the lowest right-side path needed to determine whether the interval is covered. + fn note_next(&mut self, path: Nibbles) { + if self.next.is_none_or(|next| path < next) { + self.next = Some(path); + } + } +} + +// Stores unpacked range bounds so recursive comparisons avoid repeatedly expanding hashed keys. +struct ProofRange { + // Inclusive origin paired with the proof's left boundary path. + left: Nibbles, + // Inclusive last leaf, or the maximum key when an empty response proves exhaustion. + right: Nibbles, +} + +impl ProofRange { + // Creates range bounds in the nibble representation used throughout trie traversal. + fn new(left: B256, right: B256) -> Self { + Self { left: Nibbles::unpack(left), right: Nibbles::unpack(right) } + } + + // Classifies a subtree prefix to avoid resolving subtries that cannot cross a boundary. + fn subtree_relation(&self, prefix: &Nibbles) -> Result { + if prefix.len() > KEY_NIBBLES { + return Err(RangeProofError::PathTooLong { path: *prefix }) + } + let left = self.left.slice(..prefix.len()); + let right = self.right.slice(..prefix.len()); + + Ok(if *prefix < left { + SubtreeRelation::OutsideLeft + } else if *prefix > right { + SubtreeRelation::OutsideRight + } else if *prefix > left && *prefix < right { + SubtreeRelation::Inside + } else { + SubtreeRelation::Boundary + }) + } + + // Classifies a complete key because boundary proof leaves may sit outside the supplied range. + fn key_relation(&self, path: &Nibbles) -> KeyRelation { + if path < &self.left { + KeyRelation::Before + } else if path > &self.right { + KeyRelation::After + } else { + KeyRelation::Inside + } + } +} + +// Prevents proof-owned subtries from being discarded or expanded unnecessarily by making each +// prefix's relationship to the requested range explicit. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum SubtreeRelation { + // Retains a subtree that lies entirely before the requested range. + OutsideLeft, + // Retains a subtree after the range and uses its prefix to bound the next key. + OutsideRight, + // Expands a subtree because it may contain both proof-owned and response-owned paths. + Boundary, + // Replaces a wholly covered subtree with the response leaves being authenticated. + Inside, +} + +// Prevents boundary proof leaves from being confused with response-owned interior leaves. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum KeyRelation { + // Retains a proof leaf needed to reconstruct the trie before the response range. + Before, + // Defers to the response value so the reconstructed root authenticates the supplied leaf. + Inside, + // Retains a proof leaf and records that the trie continues beyond the response range. + After, +} + +// Indexes proof blobs by commitment because proof wire order has no semantic meaning. +struct ProofNodes<'a>(B256Map<&'a [u8]>); + +impl<'a> ProofNodes<'a> { + // Builds the proof index once to avoid rescanning it for every boundary reference. + fn new(proof: &'a [Bytes]) -> Self { + Self(proof.iter().map(|node| (keccak256(node), node.as_ref())).collect()) + } + + // Resolves inline references directly and requires proof backing for hashed references. + fn resolve(&self, path: Nibbles, reference: &RlpNode) -> Result { + let Some(hash) = reference.as_hash() else { + return Ok(TrieNode::decode(&mut reference.as_slice())?) + }; + let node = self.0.get(&hash).ok_or(RangeProofError::MissingProofNode { path })?; + Ok(TrieNode::decode(&mut &node[..])?) + } +} + +// Collects disjoint leaves and subtree commitments for canonical root reconstruction. +#[derive(Default)] +struct ProofFrontier(Vec); + +impl ProofFrontier { + // Builds a frontier from validated leaves before HashBuilder enforces ordering with assertions. + fn from_leaves(origin: B256, leaves: I) -> Result<(Self, Option), RangeProofError> + where + I: IntoIterator, + V: Into>, + { + let mut frontier = Self::default(); + let mut previous = None; + + for (key, value) in leaves { + let value = value.into(); + if key < origin { + return Err(RangeProofError::LeafBeforeOrigin { key, origin }) + } + if previous.is_some_and(|previous| key <= previous) { + return Err(RangeProofError::NonMonotonicLeaves) + } + if value.is_empty() { + return Err(RangeProofError::EmptyLeafValue { key }) + } + previous = Some(key); + frontier.push_leaf(Nibbles::unpack(key), value); + } + + Ok((frontier, previous)) + } + + // Adds a leaf only at the fixed depth required by secure-trie hashed keys. + fn push_leaf(&mut self, path: Nibbles, value: Vec) { + debug_assert_eq!(path.len(), KEY_NIBBLES); + self.0.push(FrontierEntry::Leaf { path, value }); + } + + // Adds an opaque subtree at any prefix within the fixed hashed-key depth. + fn push_subtree(&mut self, path: Nibbles, hash: B256) { + debug_assert!(path.len() <= KEY_NIBBLES); + self.0.push(FrontierEntry::Subtree { path, hash }); + } + + // Reconstructs the root after sorting leaves and subtries into HashBuilder's strict path order. + fn root(mut self) -> Result { + // Outside subtries are disjoint from returned leaves, so sorting produces the strict path + // order required by HashBuilder. Reject duplicates before they reach its assertion. + self.0.sort_unstable_by_key(FrontierEntry::path); + let mut builder = HashBuilder::default(); + let mut previous = None; + + for entry in self.0 { + let path = entry.path(); + if previous.is_some_and(|previous| path <= previous) { + return Err(RangeProofError::DuplicateFrontierPath { path }) + } + previous = Some(path); + match entry { + FrontierEntry::Leaf { path, value } => builder.add_leaf(path, &value), + FrontierEntry::Subtree { path, hash } => builder.add_branch(path, hash, false), + } + } + Ok(builder.root()) + } +} + +// Unifies supplied leaves and proof-owned subtrees so HashBuilder receives one globally ordered +// stream without losing which payload each path carries. +#[derive(Clone, Debug)] +enum FrontierEntry { + // Carries a response value so root reconstruction authenticates the supplied leaf. + Leaf { path: Nibbles, value: Vec }, + // Carries an opaque commitment so proof-owned state outside the range remains unchanged. + Subtree { path: Nibbles, hash: B256 }, +} + +impl FrontierEntry { + // Exposes the common ordering key because HashBuilder requires strictly increasing paths. + const fn path(&self) -> Nibbles { + match self { + Self::Leaf { path, .. } | Self::Subtree { path, .. } => *path, + } + } +} + +/// Error returned when a trie range proof is invalid. +#[derive(Debug, thiserror::Error, PartialEq, Eq)] +pub enum RangeProofError { + /// The response leaves are not strictly increasing. + #[error("range leaves are not strictly increasing")] + NonMonotonicLeaves, + /// A returned leaf precedes the requested origin. + #[error("range leaf {key} precedes origin {origin}")] + LeafBeforeOrigin { + /// Hashed key of the offending leaf. + key: B256, + /// Inclusive origin the range was requested from. + origin: B256, + }, + /// A returned leaf has no value and would represent a deletion. + #[error("range leaf {key} has an empty value")] + EmptyLeafValue { + /// Hashed key of the valueless leaf. + key: B256, + }, + /// A proof node required on a range boundary is missing. + #[error("missing proof node at path {path:?}")] + MissingProofNode { + /// Trie path the missing node was referenced from. + path: Nibbles, + }, + /// An extension node consumes no nibble, so a crafted chain could recurse without descending. + #[error("extension node at path {path:?} has an empty key")] + EmptyExtensionKey { + /// Trie path the extension was reached at. + path: Nibbles, + }, + /// A proof path exceeds a hashed key's fixed length. + #[error("proof path {path:?} exceeds hashed key length")] + PathTooLong { + /// Trie path that could not be descended any further. + path: Nibbles, + }, + /// A leaf does not resolve to a complete hashed key. + #[error("leaf path {path:?} does not resolve to a hashed key")] + InvalidLeafPath { + /// Incomplete trie path the leaf terminated at. + path: Nibbles, + }, + /// Two reconstructed trie entries occupy the same path. + #[error("duplicate range-proof frontier path {path:?}")] + DuplicateFrontierPath { + /// Trie path claimed by more than one entry. + path: Nibbles, + }, + /// The reconstructed trie does not match the requested root. + #[error("range proof root mismatch: expected {expected}, got {got}")] + RootMismatch { + /// Root the range was requested against. + expected: B256, + /// Root reconstructed from the response. + got: B256, + }, + /// A trie node failed to decode. + #[error(transparent)] + Rlp(#[from] alloy_rlp::Error), +} + +/// Verifies a consecutive leaf range against `root`, starting at `origin`. +/// +/// Returns a lower bound for the next key, or `None` if the range exhausts the trie. +pub fn verify_range_proof( + root: B256, + origin: B256, + leaves: I, + proof: &[Bytes], +) -> Result, RangeProofError> +where + I: IntoIterator, + V: Into>, +{ + let (frontier, last_key) = ProofFrontier::from_leaves(origin, leaves)?; + + // Without boundary nodes, only the complete leaf set can reproduce the requested root. + if proof.is_empty() { + let got = frontier.root()?; + if got != root { + return Err(RangeProofError::RootMismatch { expected: root, got }) + } + return Ok(None) + } + + RangeProofVerifier::new(origin, last_key.unwrap_or(MAX_HASH), proof, frontier).verify(root) +} + +// Keeps path mutation behind one checked API because external `Nibbles` cannot have inherent +// methods and malformed proofs must not bypass secure-trie depth invariants. +trait TriePath: Sized { + // Produces the conservative bound needed when an opaque subtree hides its first exact key. + fn lowest_key(&self) -> B256; + + // Requires extensions to consume bounded path space so hostile proofs cannot recurse in place. + fn descend_extension(self, key: &Nibbles) -> Result; + + // Rejects branches beyond the key depth before mutating the path. + fn descend_child(self, nibble: u8) -> Result; + + // Requires leaves to resolve to one full hashed key before entering the frontier. + fn descend_leaf(self, key: &Nibbles) -> Result; + + // Shares the overflow guard used by extension and leaf descent. + fn join_checked(self, key: &Nibbles) -> Result; +} + +impl TriePath for Nibbles { + // Returns the subtree's lowest possible key because packing zero-fills unconsumed nibbles. + fn lowest_key(&self) -> B256 { + B256::right_padding_from(&self.pack()) + } + + // Descends an extension while rejecting empty keys that could recurse without consuming space. + fn descend_extension(self, key: &Nibbles) -> Result { + if key.is_empty() { + return Err(RangeProofError::EmptyExtensionKey { path: self }) + } + self.join_checked(key) + } + + // Descends one branch nibble while rejecting nodes below the fixed hashed-key depth. + fn descend_child(self, nibble: u8) -> Result { + if self.len() >= KEY_NIBBLES { + return Err(RangeProofError::PathTooLong { path: self }) + } + let mut path = self; + path.push(nibble); + Ok(path) + } + + // Completes a leaf path while rejecting leaves that do not resolve to one full hashed key. + fn descend_leaf(self, key: &Nibbles) -> Result { + let path = self.join_checked(key)?; + if path.len() != KEY_NIBBLES { + return Err(RangeProofError::InvalidLeafPath { path }) + } + Ok(path) + } + + // Joins path segments while rejecting proof nodes that exceed the fixed hashed-key depth. + fn join_checked(self, key: &Nibbles) -> Result { + if self.len() + key.len() > KEY_NIBBLES { + return Err(RangeProofError::PathTooLong { path: self }) + } + Ok(self.join(key)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{proof::ProofRetainer, BranchNode, ExtensionNode, TrieMask, EMPTY_ROOT_HASH}; + use alloc::{vec, vec::Vec}; + + fn key(value: u64) -> B256 { + B256::left_padding_from(&value.to_be_bytes()) + } + + fn encode_node(node: &TrieNode) -> Bytes { + alloy_rlp::encode(node).into() + } + + fn no_leaves() -> Vec<(B256, Vec)> { + Vec::new() + } + + fn value(byte: u8) -> Vec { + vec![byte; 64] + } + + fn build_proof(leaves: &[(B256, Vec)], targets: &[B256]) -> (B256, Vec) { + let targets = targets.iter().copied().map(Nibbles::unpack).collect(); + let mut builder = HashBuilder::default().with_proof_retainer(ProofRetainer::new(targets)); + for (key, value) in leaves { + builder.add_leaf(Nibbles::unpack(*key), value); + } + let root = builder.root(); + let proof = builder + .take_proof_nodes() + .into_nodes_sorted() + .into_iter() + .map(|(_, node)| node) + .collect(); + (root, proof) + } + + #[test] + fn partial_range_authenticates_and_reports_the_next_key() { + let leaves = + vec![(key(1), value(1)), (key(2), value(2)), (key(3), value(3)), (key(4), value(4))]; + let (root, proof) = build_proof(&leaves, &[key(2), key(3)]); + + assert_eq!( + verify_range_proof(root, key(2), leaves[1..3].to_vec(), &proof).unwrap(), + Some(key(4)) + ); + } + + // An unexpanded subtree reports its lowest possible key. + #[test] + fn unexpanded_right_subtree_reports_a_prefix_bound() { + let right = |tail: u8| { + let mut key = B256::ZERO; + key.0[0] = 0x40; + key.0[31] = tail; + key + }; + let leaves = vec![ + (key(1), value(1)), + (key(2), value(2)), + (right(1), value(3)), + (right(2), value(4)), + ]; + let (root, proof) = build_proof(&leaves, &[key(1), key(2)]); + + assert_eq!( + verify_range_proof(root, key(1), leaves[..2].to_vec(), &proof).unwrap(), + Some(B256::right_padding_from(&[0x40])) + ); + } + + #[test] + fn inline_outside_nodes_are_reconstructed() { + let leaves = + vec![(key(1), vec![1]), (key(2), vec![2]), (key(3), vec![3]), (key(4), vec![4])]; + let (root, proof) = build_proof(&leaves, &[key(2), key(3)]); + + assert_eq!( + verify_range_proof(root, key(2), leaves[1..3].to_vec(), &proof).unwrap(), + Some(key(4)) + ); + } + + #[test] + fn unused_proof_nodes_are_accepted() { + let leaves = + vec![(key(1), value(1)), (key(2), value(2)), (key(3), value(3)), (key(4), value(4))]; + let (root, mut proof) = build_proof(&leaves, &[key(2), key(3)]); + proof.push(Bytes::from_static(&[alloy_rlp::EMPTY_STRING_CODE])); + + assert_eq!( + verify_range_proof(root, key(2), leaves[1..3].to_vec(), &proof).unwrap(), + Some(key(4)) + ); + } + + // Reject paths before they exceed the fixed hashed-key length. + #[test] + fn node_paths_are_bounded_before_they_overflow_a_key() { + let dangling = RlpNode::word_rlp(&B256::repeat_byte(0xaa)); + + // An extension one nibble deep whose key spans a whole hashed key: 1 + 64 nibbles. + let overlong = encode_node(&TrieNode::Extension(ExtensionNode::new( + Nibbles::unpack(key(1)), + dangling.clone(), + ))); + let branch = encode_node(&TrieNode::Branch(BranchNode::new( + vec![RlpNode::word_rlp(&keccak256(&overlong))], + TrieMask::new(1), + ))); + let proof = vec![branch.clone(), overlong]; + + assert!(matches!( + verify_range_proof(keccak256(&branch), key(1), no_leaves(), &proof), + Err(RangeProofError::PathTooLong { .. }) + )); + + // A branch sitting at the full hashed-key depth, which has no room for a child. + let deep_branch = + encode_node(&TrieNode::Branch(BranchNode::new(vec![dangling], TrieMask::new(1)))); + let reach = encode_node(&TrieNode::Extension(ExtensionNode::new( + Nibbles::unpack(key(1)), + RlpNode::word_rlp(&keccak256(&deep_branch)), + ))); + let proof = vec![reach.clone(), deep_branch]; + + assert!(matches!( + verify_range_proof(keccak256(&reach), key(1), no_leaves(), &proof), + Err(RangeProofError::PathTooLong { .. }) + )); + } + + // Empty extensions could recurse indefinitely without increasing the path depth. + #[test] + fn empty_extension_keys_are_rejected() { + let empty_key = Nibbles::new(); + let mut proof = Vec::new(); + let mut child = RlpNode::word_rlp(&B256::repeat_byte(0xaa)); + + for _ in 0..64 { + let node = encode_node(&TrieNode::Extension(ExtensionNode::new(empty_key, child))); + child = RlpNode::word_rlp(&keccak256(&node)); + proof.push(node); + } + let root = keccak256(proof.last().unwrap()); + + assert!(matches!( + verify_range_proof(root, key(1), no_leaves(), &proof), + Err(RangeProofError::EmptyExtensionKey { .. }) + )); + } + + #[test] + fn missing_boundary_node_is_rejected() { + let leaves = + vec![(key(1), value(1)), (key(2), value(2)), (key(3), value(3)), (key(4), value(4))]; + let (root, _) = build_proof(&leaves, &[key(2), key(3)]); + let unrelated = [Bytes::from_static(&[alloy_rlp::EMPTY_STRING_CODE])]; + + assert!(matches!( + verify_range_proof(root, key(2), leaves[1..3].to_vec(), &unrelated), + Err(RangeProofError::MissingProofNode { .. }) + )); + } + + #[test] + fn boundary_proof_can_authenticate_an_exhausted_range() { + let leaves = + vec![(key(1), value(1)), (key(2), value(2)), (key(3), value(3)), (key(4), value(4))]; + let (root, proof) = build_proof(&leaves, &[key(2), key(4)]); + + assert_eq!(verify_range_proof(root, key(2), leaves[1..].to_vec(), &proof).unwrap(), None); + } + + #[test] + fn proof_free_full_trie_is_exhausted() { + let leaves = vec![(key(1), value(1)), (key(2), value(2)), (key(3), value(3))]; + let (root, _) = build_proof(&leaves, &[]); + + assert_eq!(verify_range_proof(root, B256::ZERO, leaves, &[]).unwrap(), None); + } + + #[test] + fn omitted_interior_leaf_changes_root() { + let leaves = + vec![(key(1), value(1)), (key(2), value(2)), (key(3), value(3)), (key(4), value(4))]; + let (root, proof) = build_proof(&leaves, &[key(2), key(4)]); + let returned = vec![(key(2), value(2)), (key(4), value(4))]; + + assert!(matches!( + verify_range_proof(root, key(2), returned, &proof), + Err(RangeProofError::RootMismatch { .. }) + )); + } + + #[test] + fn mutated_leaf_changes_root() { + let leaves = vec![(key(1), value(1)), (key(2), value(2)), (key(3), value(3))]; + let (root, proof) = build_proof(&leaves, &[key(2), key(3)]); + let returned = vec![(key(2), value(9)), (key(3), value(3))]; + + assert!(matches!( + verify_range_proof(root, key(2), returned, &proof), + Err(RangeProofError::RootMismatch { .. }) + )); + } + + #[test] + fn empty_tail_is_authenticated() { + let leaves = vec![(key(1), value(1)), (key(2), value(2))]; + let (root, proof) = build_proof(&leaves, &[key(3)]); + + assert_eq!( + verify_range_proof(root, key(3), core::iter::empty::<(B256, Vec)>(), &proof) + .unwrap(), + None + ); + } + + #[test] + fn empty_range_cannot_hide_a_right_leaf() { + let leaves = vec![(key(1), value(1)), (key(3), value(3))]; + let (root, proof) = build_proof(&leaves, &[key(2)]); + + assert!(matches!( + verify_range_proof(root, key(2), core::iter::empty::<(B256, Vec)>(), &proof,), + Err(RangeProofError::RootMismatch { .. }) + )); + } + + #[test] + fn rejects_non_monotonic_leaves_and_leaves_before_origin() { + let leaves = vec![(key(2), value(2)), (key(1), value(1))]; + + assert_eq!( + verify_range_proof(B256::ZERO, B256::ZERO, leaves, &[]), + Err(RangeProofError::NonMonotonicLeaves) + ); + assert!(matches!( + verify_range_proof(B256::ZERO, key(2), [(key(1), value(1))], &[],), + Err(RangeProofError::LeafBeforeOrigin { .. }) + )); + assert_eq!( + verify_range_proof(B256::ZERO, B256::ZERO, [(key(1), Vec::new())], &[]), + Err(RangeProofError::EmptyLeafValue { key: key(1) }) + ); + } + + #[test] + fn empty_root_accepts_only_an_empty_range() { + assert_eq!( + verify_range_proof( + EMPTY_ROOT_HASH, + B256::ZERO, + core::iter::empty::<(B256, Vec)>(), + &[] + ) + .unwrap(), + None + ); + assert!(matches!( + verify_range_proof(EMPTY_ROOT_HASH, B256::ZERO, [(key(1), value(1))], &[],), + Err(RangeProofError::RootMismatch { .. }) + )); + assert!( + verify_range_proof(EMPTY_ROOT_HASH, B256::ZERO, no_leaves(), &[Bytes::new()]).is_err() + ); + } +} From 62cd99e4056c222c6b56ebd3b822b8bc10fdde1a Mon Sep 17 00:00:00 2001 From: Derek Cofausper <256792747+decofe@users.noreply.github.com> Date: Tue, 25 Aug 2026 15:05:09 +0000 Subject: [PATCH 064/294] fix(downloader): address account range review feedback (#26810) Co-authored-by: Matthias Seitz <19890894+mattsse@users.noreply.github.com> Co-authored-by: Matthias Seitz --- crates/net/downloaders/src/snap/mod.rs | 76 ++++++++++++++++++++++---- crates/trie/common/src/range_proof.rs | 37 +++++++++++-- 2 files changed, 98 insertions(+), 15 deletions(-) diff --git a/crates/net/downloaders/src/snap/mod.rs b/crates/net/downloaders/src/snap/mod.rs index ac5c47b90a6..2c037ba3b95 100644 --- a/crates/net/downloaders/src/snap/mod.rs +++ b/crates/net/downloaders/src/snap/mod.rs @@ -26,7 +26,6 @@ const MAX_RETRIES: u8 = 2; /// /// Invalid responses penalize their peer and retry at high priority. Proof verification runs on /// the blocking pool. -#[derive(Debug)] pub struct AccountRangeDownloader { client: C, runtime: Runtime, @@ -36,6 +35,17 @@ pub struct AccountRangeDownloader { retries: u8, } +impl std::fmt::Debug for AccountRangeDownloader { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("AccountRangeDownloader") + .field("client", &self.client) + .field("request", &self.request) + .field("verification", &self.verification) + .field("retries", &self.retries) + .finish_non_exhaustive() + } +} + impl AccountRangeDownloader { /// Creates a downloader using `runtime` for proof verification and submits the initial request. /// Returns an error when the origin exceeds the limit. @@ -78,6 +88,7 @@ impl AccountRangeDownloader { Ok(Some(AccountRangeOutcome::Verified(VerifiedAccountRange { accounts: Vec::new(), has_more: false, + next: None, }))) } else { Ok(Some(AccountRangeOutcome::Unavailable { peer_id })) @@ -164,7 +175,7 @@ impl AccountRangeDownloader { impl Future for AccountRangeDownloader where - C: SnapClient + Unpin + 'static, + C: SnapClient + Unpin, { type Output = Result; @@ -217,8 +228,15 @@ struct VerificationTask { pub struct VerifiedAccountRange { /// Accounts in strictly increasing hashed-key order. pub accounts: Vec<(B256, TrieAccount)>, - /// Whether another request is needed to complete the requested interval. + /// Whether another request may be needed to complete the requested interval. + /// + /// Conservative: [`Self::next`] is only a lower bound when the trie continues inside a + /// subtree the proof left unexpanded, so this can be `true` for an interval that is already + /// complete. pub has_more: bool, + /// Authenticated lower bound for the first key after the response, or `None` when the range + /// exhausted the trie. + pub next: Option, } // Authenticate the full response before trimming its optional boundary account. @@ -249,7 +267,7 @@ fn verify_account_range( accounts.truncate(accounts.partition_point(|(hash, _)| *hash <= request.limit_hash)); let has_more = next.is_some_and(|next| next <= request.limit_hash); - Ok(VerifiedAccountRange { accounts, has_more }) + Ok(VerifiedAccountRange { accounts, has_more, next }) } // Re-encode decoded accounts so the proof authenticates their canonical trie values. @@ -259,18 +277,21 @@ fn verify_proof( proof: &[alloy_primitives::Bytes], ) -> Result, RequestError> { let leaves = accounts.iter().map(|(hash, account)| (*hash, alloy_rlp::encode(account))); - verify_range_proof(request.root_hash, request.starting_hash, leaves, proof).map_err(|error| { - debug!(target: "downloaders::snap", %error, "Invalid account range proof"); - RequestError::BadResponse - }) + verify_range_proof(request.root_hash, request.starting_hash, request.limit_hash, leaves, proof) + .map_err(|error| { + debug!(target: "downloaders::snap", %error, "Invalid account range proof"); + RequestError::BadResponse + }) } /// An account-range request whose origin exceeds its limit. #[derive(Clone, Copy, Debug, PartialEq, Eq, thiserror::Error)] #[error("account range origin {origin} exceeds limit {limit}")] pub struct InvalidAccountRange { - origin: B256, - limit: B256, + /// Inclusive origin the range was requested from. + pub origin: B256, + /// Inclusive limit the range was requested to. + pub limit: B256, } #[cfg(test)] @@ -446,7 +467,11 @@ mod tests { assert_eq!( outcome, - AccountRangeOutcome::Verified(VerifiedAccountRange { accounts, has_more: false }) + AccountRangeOutcome::Verified(VerifiedAccountRange { + accounts, + has_more: false, + next: None, + }) ); assert!(client.reported.lock().unwrap().is_empty()); assert_eq!(*client.priorities.lock().unwrap(), [Priority::Normal]); @@ -533,6 +558,7 @@ mod tests { AccountRangeOutcome::Verified(VerifiedAccountRange { accounts: vec![accounts[0]], has_more: false, + next: Some(key(4)), }) ); assert!(client.reported.lock().unwrap().is_empty()); @@ -588,6 +614,7 @@ mod tests { AccountRangeOutcome::Verified(VerifiedAccountRange { accounts: accounts[..2].to_vec(), has_more: false, + next: Some(key(3)), }) ); assert!(client.reported.lock().unwrap().is_empty()); @@ -616,6 +643,31 @@ mod tests { AccountRangeOutcome::Verified(VerifiedAccountRange { accounts: Vec::new(), has_more: false, + next: None, + }) + ); + assert!(client.reported.lock().unwrap().is_empty()); + } + + #[tokio::test] + async fn empty_interval_is_proven_without_a_boundary_account() { + let accounts = vec![(key(1), account(7)), (key(9), account(8))]; + let (root_hash, proof) = root_and_proof(&accounts, &[key(3), key(5)]); + let peer = PeerId::random(); + let message = AccountRangeMessage { request_id: 1, accounts: Vec::new(), proof }; + let client = Arc::new(TestSnapClient::new([response(peer, message)])); + let mut request = request(root_hash); + request.starting_hash = key(3); + request.limit_hash = key(5); + + let outcome = downloader(Arc::clone(&client), request).unwrap().await.unwrap(); + + assert_eq!( + outcome, + AccountRangeOutcome::Verified(VerifiedAccountRange { + accounts: Vec::new(), + has_more: false, + next: Some(key(9)), }) ); assert!(client.reported.lock().unwrap().is_empty()); @@ -643,6 +695,7 @@ mod tests { AccountRangeOutcome::Verified(VerifiedAccountRange { accounts: vec![accounts[0]], has_more: false, + next: Some(key(9)), }) ); } @@ -668,6 +721,7 @@ mod tests { AccountRangeOutcome::Verified(VerifiedAccountRange { accounts: vec![accounts[0]], has_more: true, + next: Some(key(3)), }) ); } diff --git a/crates/trie/common/src/range_proof.rs b/crates/trie/common/src/range_proof.rs index 7f25d04f6ab..11592b7ef41 100644 --- a/crates/trie/common/src/range_proof.rs +++ b/crates/trie/common/src/range_proof.rs @@ -11,7 +11,6 @@ use alloy_primitives::{keccak256, map::B256Map, Bytes, B256}; use alloy_rlp::Decodable; const KEY_NIBBLES: usize = B256::len_bytes() * 2; -const MAX_HASH: B256 = B256::new([0xff; B256::len_bytes()]); // Coordinates proof traversal and root reconstruction through one shared frontier. struct RangeProofVerifier<'a> { @@ -150,7 +149,7 @@ impl<'a> RangeProofVerifier<'a> { struct ProofRange { // Inclusive origin paired with the proof's left boundary path. left: Nibbles, - // Inclusive last leaf, or the maximum key when an empty response proves exhaustion. + // Inclusive last leaf, or the requested limit when the response is empty. right: Nibbles, } @@ -384,12 +383,16 @@ pub enum RangeProofError { Rlp(#[from] alloy_rlp::Error), } -/// Verifies a consecutive leaf range against `root`, starting at `origin`. +/// Verifies a consecutive leaf range against `root`, from `origin` through `limit`. +/// +/// When `leaves` is empty, `limit` supplies the response's right boundary so an empty interval can +/// be authenticated without requiring a leaf past the limit. /// /// Returns a lower bound for the next key, or `None` if the range exhausts the trie. pub fn verify_range_proof( root: B256, origin: B256, + limit: B256, leaves: I, proof: &[Bytes], ) -> Result, RangeProofError> @@ -408,7 +411,7 @@ where return Ok(None) } - RangeProofVerifier::new(origin, last_key.unwrap_or(MAX_HASH), proof, frontier).verify(root) + RangeProofVerifier::new(origin, last_key.unwrap_or(limit), proof, frontier).verify(root) } // Keeps path mutation behind one checked API because external `Nibbles` cannot have inherent @@ -478,6 +481,21 @@ mod tests { use crate::{proof::ProofRetainer, BranchNode, ExtensionNode, TrieMask, EMPTY_ROOT_HASH}; use alloc::{vec, vec::Vec}; + const MAX_HASH: B256 = B256::new([0xff; B256::len_bytes()]); + + fn verify_range_proof( + root: B256, + origin: B256, + leaves: I, + proof: &[Bytes], + ) -> Result, RangeProofError> + where + I: IntoIterator, + V: Into>, + { + super::verify_range_proof(root, origin, MAX_HASH, leaves, proof) + } + fn key(value: u64) -> B256 { B256::left_padding_from(&value.to_be_bytes()) } @@ -704,6 +722,17 @@ mod tests { )); } + #[test] + fn empty_interval_is_authenticated_through_its_limit() { + let leaves = vec![(key(1), value(1)), (key(3), value(3))]; + let (root, proof) = build_proof(&leaves, &[key(2)]); + + assert_eq!( + super::verify_range_proof(root, key(2), key(2), no_leaves(), &proof).unwrap(), + Some(key(3)) + ); + } + #[test] fn rejects_non_monotonic_leaves_and_leaves_before_origin() { let leaves = vec![(key(2), value(2)), (key(1), value(1))]; From 9e425fe997d6d384c6e97e6c2c0f0ea93195dd95 Mon Sep 17 00:00:00 2001 From: refcell Date: Tue, 25 Aug 2026 15:25:25 +0000 Subject: [PATCH 065/294] feat(net): allow updating the fork filter at runtime (#26794) Co-authored-by: Claude Co-authored-by: Matthias Seitz --- crates/net/network/src/manager.rs | 4 ++++ crates/net/network/src/network.rs | 24 +++++++++++++++++++++++- crates/net/network/src/session/mod.rs | 17 +++++++++++++++++ crates/net/p2p/src/sync.rs | 13 ++++++++++++- 4 files changed, 56 insertions(+), 2 deletions(-) diff --git a/crates/net/network/src/manager.rs b/crates/net/network/src/manager.rs index 515b27d690b..63255c41daa 100644 --- a/crates/net/network/src/manager.rs +++ b/crates/net/network/src/manager.rs @@ -776,6 +776,10 @@ impl NetworkManager { self.swarm.state_mut().update_fork_id(transition.current); } } + NetworkHandleMessage::SetForkFilter { fork_filter } => { + let fork_id = self.swarm.sessions_mut().set_fork_filter(fork_filter); + self.swarm.state_mut().update_fork_id(fork_id); + } NetworkHandleMessage::GetPeerInfos(tx) => { let _ = tx.send(self.get_peer_infos()); } diff --git a/crates/net/network/src/network.rs b/crates/net/network/src/network.rs index 18f381b3cbf..0ad5356bfee 100644 --- a/crates/net/network/src/network.rs +++ b/crates/net/network/src/network.rs @@ -12,7 +12,7 @@ use reth_eth_wire::{ BlockRangeUpdate, BroadcastPoolTransactions, DisconnectReason, EthNetworkPrimitives, NetworkPrimitives, NewPooledTransactionHashes, SharedTransactions, }; -use reth_ethereum_forks::Head; +use reth_ethereum_forks::{ForkFilter, Head}; use reth_network_api::{ events::{NetworkPeersEvents, PeerEvent, PeerEventStream}, test_utils::{PeersHandle, PeersHandleProvider}, @@ -112,6 +112,18 @@ impl NetworkHandle { self.send_message(NetworkHandleMessage::StatusUpdate { head }); } + /// Replaces the network's active [`ForkFilter`] with `fork_filter`, re-deriving the advertised + /// [`ForkId`](reth_ethereum_forks::ForkId) for future handshakes and updating the discovery + /// ENR entry. + /// + /// This lets a running node adopt a fork schedule that changed at runtime (e.g. an + /// L1-signalled upgrade) without a restart. The caller must build `fork_filter` from the + /// updated chain spec advanced to the node's current head, and should do so before the fork's + /// activation timestamp so the node announces the upcoming fork ahead of time. + pub fn set_fork_filter(&self, fork_filter: ForkFilter) { + self.send_message(NetworkHandleMessage::SetForkFilter { fork_filter }); + } + /// Announce a block over devp2p /// /// Caution: in `PoS` this is a noop because new blocks are no longer announced over devp2p. @@ -497,6 +509,11 @@ impl NetworkSyncUpdater for NetworkHandle { fn update_block_range(&self, update: reth_eth_wire::BlockRangeUpdate) { self.send_message(NetworkHandleMessage::InternalBlockRangeUpdate(update)); } + + /// Replaces the active fork filter to adopt a runtime fork-schedule change. + fn set_fork_filter(&self, fork_filter: ForkFilter) { + self.send_message(NetworkHandleMessage::SetForkFilter { fork_filter }); + } } impl BlockDownloaderProvider for NetworkHandle { @@ -614,6 +631,11 @@ pub(crate) enum NetworkHandleMessage), /// Gets `PeerInfo` for the specified peer IDs. diff --git a/crates/net/network/src/session/mod.rs b/crates/net/network/src/session/mod.rs index 0111ab85c50..9a5ea88d09a 100644 --- a/crates/net/network/src/session/mod.rs +++ b/crates/net/network/src/session/mod.rs @@ -263,6 +263,23 @@ impl SessionManager { transition } + /// Replaces the active [`ForkFilter`] with a newly derived one and returns the resulting + /// [`ForkId`]. + /// + /// This is used to adopt a fork schedule that changed at runtime (e.g. an L1-signalled + /// upgrade) without restarting the node. The caller is expected to build `fork_filter` from + /// the updated chain spec already advanced to the node's current head, so that + /// [`ForkFilter::current`] reflects the correct [`ForkId`] immediately. + /// + /// Every subsequent handshake clones this filter and advertises the returned [`ForkId`], so + /// the node's advertised fork identity stays aligned with the rules it now enforces. Existing + /// active sessions are not revalidated here; see [`SessionManager::active_sessions`]. + pub(crate) fn set_fork_filter(&mut self, fork_filter: ForkFilter) -> ForkId { + self.fork_filter = fork_filter; + self.status.forkid = self.fork_filter.current(); + self.status.forkid + } + /// An incoming TCP connection was received. This starts the authentication process to turn this /// stream into an active peer session. /// diff --git a/crates/net/p2p/src/sync.rs b/crates/net/p2p/src/sync.rs index 77b97116d19..6770c5d12a2 100644 --- a/crates/net/p2p/src/sync.rs +++ b/crates/net/p2p/src/sync.rs @@ -1,6 +1,6 @@ //! Traits used when interacting with the sync status of the network. -use alloy_eips::eip2124::Head; +use alloy_eips::eip2124::{ForkFilter, Head}; use reth_eth_wire_types::BlockRangeUpdate; /// A type that provides information about whether the node is currently syncing and the network is @@ -31,6 +31,17 @@ pub trait NetworkSyncUpdater: std::fmt::Debug + Send + Sync + 'static { /// Updates the advertised block range. fn update_block_range(&self, update: BlockRangeUpdate); + + /// Replaces the node's active [`ForkFilter`] to adopt a fork schedule that changed at runtime + /// (e.g. an L1-signalled upgrade) without a restart. + /// + /// The caller must build `fork_filter` from the updated chain spec advanced to the node's + /// current head, and should do so before the fork's activation timestamp so the node + /// announces the upcoming fork ahead of time. Defaults to a no-op for updaters that do not + /// track a fork filter. + fn set_fork_filter(&self, fork_filter: ForkFilter) { + let _ = fork_filter; + } } /// The state the network is currently in when it comes to synchronization. From 853a42db1c68b4aaaab918302a109986679053ac Mon Sep 17 00:00:00 2001 From: Sergei Shulepov Date: Tue, 25 Aug 2026 22:19:09 +0000 Subject: [PATCH 066/294] fix(rocksdb): better max_open_files config (#26812) --- Cargo.lock | 1 + crates/storage/provider/Cargo.toml | 3 + .../src/providers/rocksdb/provider.rs | 84 +++++++++++++++++-- 3 files changed, 82 insertions(+), 6 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9cecf5f880e..b2cc43d5524 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9749,6 +9749,7 @@ dependencies = [ "assert_matches", "eyre", "itertools 0.14.0", + "libc", "metrics", "notify", "parking_lot", diff --git a/crates/storage/provider/Cargo.toml b/crates/storage/provider/Cargo.toml index 4cdeeb69ddd..dfe79de19fe 100644 --- a/crates/storage/provider/Cargo.toml +++ b/crates/storage/provider/Cargo.toml @@ -69,6 +69,9 @@ rayon.workspace = true rocksdb.workspace = true +[target.'cfg(unix)'.dependencies] +libc.workspace = true + [dev-dependencies] reth-db = { workspace = true, features = ["test-utils"] } reth-primitives-traits = { workspace = true, features = ["arbitrary", "test-utils"] } diff --git a/crates/storage/provider/src/providers/rocksdb/provider.rs b/crates/storage/provider/src/providers/rocksdb/provider.rs index d7198e7c74a..9fae9514ebd 100644 --- a/crates/storage/provider/src/providers/rocksdb/provider.rs +++ b/crates/storage/provider/src/providers/rocksdb/provider.rs @@ -115,14 +115,22 @@ const DEFAULT_BLOCK_SIZE: usize = 16 * 1024; /// Default max background jobs for `RocksDB` compaction and flushing. const DEFAULT_MAX_BACKGROUND_JOBS: i32 = 6; -/// Default max open file descriptors for `RocksDB`. +/// Max open file descriptors for `RocksDB` when the process limit is low. /// /// Caps the number of SST file handles `RocksDB` keeps open simultaneously. /// Set to 512 to stay within the common default OS `ulimit -n` of 1024, /// leaving headroom for MDBX, static files, and other I/O. -/// `RocksDB` uses an internal table cache and re-opens files on demand, -/// so this has negligible performance impact on read-heavy workloads. -const DEFAULT_MAX_OPEN_FILES: i32 = 512; +const LIMITED_MAX_OPEN_FILES: i32 = 512; + +/// Keeps all `RocksDB` files open and avoids table cache lookups. +const KEEP_ALL_FILES_OPEN: i32 = -1; + +/// Minimum process file descriptor limit for keeping all `RocksDB` files open. +/// +/// Mature archive databases can use tens of thousands of file descriptors. This threshold keeps +/// unlimited mode for hosts configured for that workload while retaining a bounded table cache on +/// systems with low limits. +const HIGH_FILE_DESCRIPTOR_LIMIT: u64 = 128 * 1024; /// Default bytes per sync for `RocksDB` WAL writes (1 MB). const DEFAULT_BYTES_PER_SYNC: u64 = 1_048_576; @@ -234,7 +242,7 @@ impl RocksDBBuilder { options.set_log_level(log_level); - options.set_max_open_files(DEFAULT_MAX_OPEN_FILES); + options.set_max_open_files(select_max_open_files()); // Delete obsolete WAL files immediately after all column families have flushed. // Both set to 0 means "delete ASAP, no archival". @@ -415,7 +423,7 @@ impl RocksDBBuilder { // Open as secondary instance for catch-up capability. // Secondary needs max_open_files = -1 to keep all FDs open. let mut options = options; - options.set_max_open_files(-1); + options.set_max_open_files(KEEP_ALL_FILES_OPEN); let secondary_path = self .path @@ -2879,6 +2887,60 @@ const fn convert_log_level(level: LogLevel) -> rocksdb::LogLevel { } } +/// Selects the `RocksDB` `max_open_files` setting from the current file descriptor limit +/// balancing performance vs. compatibility. +/// +/// A mature database can use tens of thousands of file descriptors. With a +/// finite `max_open_files` value, `RocksDB` performs additional table-cache checks +/// even when enough capacity is available. The value `-1` keeps all table files +/// open and avoids this overhead. +/// +/// During normal startup, Reth raises the soft file descriptor limit to the +/// system hard limit before it opens `RocksDB`. Therefore, we expect the current +/// limit to be the hard limit. A survey of modern Linux distributions shows +/// common default limits of `1024:524288`. +/// +/// We use `-1` only when the current limit is above a conservative threshold. +/// This keeps enough file descriptors for other parts of the process. When the +/// limit is lower or cannot be read, we use a stricter fixed limit. +fn select_max_open_files() -> i32 { + let file_descriptor_limit = current_file_descriptor_limit(); + let max_open_files = max_open_files_for_limit(file_descriptor_limit); + + if max_open_files == LIMITED_MAX_OPEN_FILES { + tracing::warn!( + target: "providers::rocksdb", + ?file_descriptor_limit, + threshold = HIGH_FILE_DESCRIPTOR_LIMIT, + max_open_files, + "RocksDB will not keep all files open; performance may be reduced" + ); + } + + max_open_files +} + +const fn max_open_files_for_limit(file_descriptor_limit: Option) -> i32 { + match file_descriptor_limit { + Some(limit) if limit >= HIGH_FILE_DESCRIPTOR_LIMIT => KEEP_ALL_FILES_OPEN, + _ => LIMITED_MAX_OPEN_FILES, + } +} + +#[cfg(unix)] +#[allow(clippy::useless_conversion)] +fn current_file_descriptor_limit() -> Option { + let mut limit = libc::rlimit { rlim_cur: 0, rlim_max: 0 }; + // SAFETY: `limit` points to initialized writable memory for the kernel to populate. + let result = unsafe { libc::getrlimit(libc::RLIMIT_NOFILE, &raw mut limit) }; + (result == 0).then(|| limit.rlim_cur.into()) +} + +#[cfg(not(unix))] +const fn current_file_descriptor_limit() -> Option { + None +} + #[cfg(test)] mod tests { use super::*; @@ -2895,6 +2957,16 @@ mod tests { }; use tempfile::TempDir; + #[test] + fn max_open_files_adapts_to_file_descriptor_limit() { + assert_eq!(max_open_files_for_limit(Some(HIGH_FILE_DESCRIPTOR_LIMIT)), KEEP_ALL_FILES_OPEN); + assert_eq!( + max_open_files_for_limit(Some(HIGH_FILE_DESCRIPTOR_LIMIT - 1)), + LIMITED_MAX_OPEN_FILES + ); + assert_eq!(max_open_files_for_limit(None), LIMITED_MAX_OPEN_FILES); + } + #[test] fn test_with_default_tables_registers_required_column_families() { let temp_dir = TempDir::new().unwrap(); From cb7fc64c76ddc66e2e13c2f4ba7bfe50cad99ae5 Mon Sep 17 00:00:00 2001 From: Brian Picciano Date: Wed, 26 Aug 2026 07:52:48 +0000 Subject: [PATCH 067/294] fix(trie): materialize destroyed storage trie nodes (#26802) Co-authored-by: Brian Picciano <933154+mediocregopher@users.noreply.github.com> Co-authored-by: Derek Cofausper <256792747+decofe@users.noreply.github.com> --- crates/net/network/src/config.rs | 2 +- .../src/providers/database/provider.rs | 50 ---------- .../storage-overlay/src/changeset_cache.rs | 7 +- crates/trie/common/src/updates.rs | 39 +++++++- crates/trie/db/tests/trie.rs | 92 ++++++++++++++++++- crates/trie/trie/src/trie.rs | 25 ++++- 6 files changed, 148 insertions(+), 67 deletions(-) diff --git a/crates/net/network/src/config.rs b/crates/net/network/src/config.rs index 5958e333399..f604c42108a 100644 --- a/crates/net/network/src/config.rs +++ b/crates/net/network/src/config.rs @@ -186,7 +186,7 @@ where } } -/// Builder for [`NetworkConfig`](struct.NetworkConfig.html). +/// Builder for [`NetworkConfig`]. #[derive(Debug)] pub struct NetworkConfigBuilder { /// The node's secret key, from which the node's identity is derived. diff --git a/crates/storage/provider/src/providers/database/provider.rs b/crates/storage/provider/src/providers/database/provider.rs index fe53665a5a6..3dac39e0046 100644 --- a/crates/storage/provider/src/providers/database/provider.rs +++ b/crates/storage/provider/src/providers/database/provider.rs @@ -4910,56 +4910,6 @@ mod tests { assert!(storage_entries.is_empty()); } - #[test] - fn test_save_blocks_batches_transient_storage_wipe() { - use alloy_primitives::map::B256Set; - use reth_trie::{updates::TrieUpdates, HashBuilder, HashedPostStateSorted}; - - let factory = create_test_provider_factory(); - factory.set_storage_settings_cache(StorageSettings::v2()); - - let mut test_block_builder = TestBlockBuilder::eth().with_state(); - let genesis = test_block_builder.get_executed_blocks(0..1).next().unwrap(); - let blocks: Vec<_> = test_block_builder.get_executed_blocks(1..4).collect(); - - let provider_rw = factory.provider_rw().unwrap(); - save_genesis(&provider_rw, &genesis).unwrap(); - provider_rw.commit().unwrap(); - - // A contract created and self-destructed in the same transaction leaves an empty whole- - // storage wipe in the trie updates, even though the account never reaches persisted state. - let ephemeral_account = B256::with_last_byte(0x57); - let hashed_state = HashedPostStateSorted::default(); - let mut trie_updates = TrieUpdates::default(); - trie_updates.finalize( - HashBuilder::default(), - Default::default(), - B256Set::from_iter([ephemeral_account]), - ); - let trie_updates = trie_updates.into_sorted(); - assert!(trie_updates.storage_tries_ref()[&ephemeral_account].is_deleted); - let selfdestruct_block = ExecutedBlock::new( - Arc::clone(&blocks[2].recovered_block), - Arc::clone(&blocks[2].execution_output), - ComputedTrieData::new(Arc::new(hashed_state), Arc::new(trie_updates)), - ); - - let provider_rw = factory.provider_rw().unwrap(); - let input = SaveBlocksInput::new( - vec![blocks[0].clone(), blocks[1].clone(), selfdestruct_block], - 0, - 0, - 3, - 3, - ); - provider_rw.save_blocks(&input).unwrap(); - provider_rw.commit().unwrap(); - - let checkpoint = - factory.provider().unwrap().get_stage_checkpoint(StageId::Finish).unwrap().unwrap(); - assert_eq!(checkpoint.block_number, 3); - } - #[test] fn test_save_blocks_partial_cycles_do_not_duplicate_static_file_writes() { let factory = create_test_provider_factory(); diff --git a/crates/storage/storage-overlay/src/changeset_cache.rs b/crates/storage/storage-overlay/src/changeset_cache.rs index 1400a432579..79df0335e91 100644 --- a/crates/storage/storage-overlay/src/changeset_cache.rs +++ b/crates/storage/storage-overlay/src/changeset_cache.rs @@ -915,12 +915,7 @@ mod tests { let actual = reth_trie_db::compute_range_trie_changesets(&*provider, &state_trie_provider, 1..=3, 3) .unwrap(); - let storage_revert = actual - .storage_tries_ref() - .get(&hashed_address) - .expect("created account storage trie should be deleted by range revert"); - assert!(storage_revert.is_deleted()); - assert!(storage_revert.storage_nodes_ref().is_empty()); + assert!(actual.storage_tries_ref().get(&hashed_address).is_none()); let cache = ChangesetCache::new(); let overlay_manager = OverlayManager::::default(); diff --git a/crates/trie/common/src/updates.rs b/crates/trie/common/src/updates.rs index d5fbb7db395..f906798b059 100644 --- a/crates/trie/common/src/updates.rs +++ b/crates/trie/common/src/updates.rs @@ -7,7 +7,7 @@ use alloc::{ vec::Vec, }; use alloy_primitives::{ - map::{B256Map, B256Set, HashMap, HashSet}, + map::{B256Map, HashMap, HashSet}, FixedBytes, B256, }; @@ -141,7 +141,7 @@ impl TrieUpdates { &mut self, hash_builder: HashBuilder, removed_keys: HashSet, - destroyed_accounts: B256Set, + destroyed_storage_trie_nodes: B256Map>, ) { // Retrieve updated nodes from hash builder. let (_, updated_nodes) = hash_builder.split(); @@ -150,9 +150,10 @@ impl TrieUpdates { // Add deleted node paths. self.removed_nodes.extend(exclude_empty(removed_keys)); - // Add deleted storage tries for destroyed accounts. - for destroyed in destroyed_accounts { - self.storage_tries.entry(destroyed).or_default().set_deleted(true); + // Add removed storage trie nodes for destroyed accounts. + for (hashed_address, removed_nodes) in destroyed_storage_trie_nodes { + let storage_updates = self.storage_tries.entry(hashed_address).or_default(); + storage_updates.removed_nodes.extend(removed_nodes); } } @@ -942,6 +943,34 @@ mod tests { use super::*; use alloy_primitives::B256; + #[test] + fn test_finalize_keeps_storage_updates_after_destroyed_node_removal() { + let hashed_address = B256::with_last_byte(1); + let path = Nibbles::from_nibbles_unchecked([0x01]); + let node = BranchNodeCompact::default(); + let mut updates = TrieUpdates { + storage_tries: B256Map::from_iter([( + hashed_address, + StorageTrieUpdates { + storage_nodes: HashMap::from_iter([(path, node.clone())]), + ..Default::default() + }, + )]), + ..Default::default() + }; + + updates.finalize( + HashBuilder::default(), + Default::default(), + B256Map::from_iter([(hashed_address, vec![path])]), + ); + + assert_eq!( + updates.into_sorted().storage_tries[&hashed_address].storage_nodes, + vec![(path, Some(node))] + ); + } + #[test] fn test_trie_updates_sorted_extend_ref() { // Test extending with empty updates diff --git a/crates/trie/db/tests/trie.rs b/crates/trie/db/tests/trie.rs index 23a00203ffd..7367c06a592 100644 --- a/crates/trie/db/tests/trie.rs +++ b/crates/trie/db/tests/trie.rs @@ -2,14 +2,18 @@ use alloy_consensus::EMPTY_ROOT_HASH; use alloy_primitives::{ - address, b256, hex_literal::hex, keccak256, map::HashMap, Address, B256, U256, + address, b256, + hex_literal::hex, + keccak256, + map::{B256Set, HashMap}, + Address, B256, U256, }; use alloy_rlp::Encodable; use proptest::{prelude::ProptestConfig, proptest}; use proptest_arbitrary_interop::arb; use reth_db::{tables, test_utils::TempDatabase, DatabaseEnv}; use reth_db_api::{ - cursor::{DbCursorRO, DbCursorRW, DbDupCursorRO}, + cursor::{DbCursorRO, DbCursorRW, DbDupCursorRO, DbDupCursorRW}, transaction::{DbTx, DbTxMut}, }; use reth_primitives_traits::{Account, StorageEntry}; @@ -272,6 +276,90 @@ fn cleared_storage_emits_node_removals_without_deleted_flag() { }); } +#[test] +fn destroyed_account_storage_emits_node_removals() { + let factory = create_test_provider_factory(); + let tx = factory.provider_rw().unwrap(); + let hashed_address = B256::random(); + tx.tx_ref().put::(hashed_address, Account::default()).unwrap(); + for hashed_slot in [ + hex!("30af561000000000000000000000000000000000000000000000000000000000"), + hex!("30af569000000000000000000000000000000000000000000000000000000000"), + hex!("30af650000000000000000000000000000000000000000000000000000000000"), + hex!("30af6f0000000000000000000000000000000000000000000000000000000000"), + hex!("30af8f0000000000000000000000000000000000000000000000000000000000"), + hex!("3100000000000000000000000000000000000000000000000000000000000000"), + ] { + tx.tx_ref() + .put::( + hashed_address, + StorageEntry { key: B256::new(hashed_slot), value: U256::ONE }, + ) + .unwrap(); + } + + let initial_updates = reth_trie_db::with_adapter!(tx, |A| { + DbStateRoot::<_, A>::from_tx(tx.tx_ref()).root_with_updates().unwrap().1 + }); + tx.write_trie_updates(initial_updates).unwrap(); + + let initial_storage_updates = reth_trie_db::with_adapter!(tx, |A| { + DbStorageRoot::<_, A>::from_tx_hashed(tx.tx_ref(), hashed_address) + .root_with_updates() + .unwrap() + .2 + }); + let mut expected_removed_nodes: Vec<_> = + initial_storage_updates.storage_nodes_ref().keys().copied().collect(); + expected_removed_nodes.sort_unstable(); + assert!(!expected_removed_nodes.is_empty()); + let initial_storage_updates = initial_storage_updates.into_sorted(); + tx.write_storage_trie_updates_sorted(core::iter::once(( + &hashed_address, + &initial_storage_updates, + ))) + .unwrap(); + + let mut account_cursor = tx.tx_ref().cursor_write::().unwrap(); + account_cursor.seek_exact(hashed_address).unwrap(); + account_cursor.delete_current().unwrap(); + drop(account_cursor); + + let mut storage_cursor = tx.tx_ref().cursor_dup_write::().unwrap(); + storage_cursor.seek_exact(hashed_address).unwrap(); + storage_cursor.delete_current_duplicates().unwrap(); + drop(storage_cursor); + + let mut account_prefix_set = PrefixSetMut::default(); + account_prefix_set.insert(Nibbles::unpack(hashed_address)); + let (root, updates) = reth_trie_db::with_adapter!(tx, |A| { + DbStateRoot::<_, A>::from_tx(tx.tx_ref()) + .with_prefix_sets(TriePrefixSets { + account_prefix_set: account_prefix_set.freeze(), + destroyed_accounts: B256Set::from_iter([hashed_address]), + ..Default::default() + }) + .root_with_updates() + .unwrap() + }); + + assert_eq!(root, EMPTY_ROOT_HASH); + let storage_updates = &updates.storage_tries_ref()[&hashed_address]; + assert!(!storage_updates.is_deleted()); + let mut removed_nodes: Vec<_> = storage_updates.removed_nodes_ref().iter().copied().collect(); + removed_nodes.sort_unstable(); + assert_eq!(removed_nodes, expected_removed_nodes); + + tx.write_trie_updates(updates).unwrap(); + assert!(tx + .tx_ref() + .cursor_dup_read::() + .unwrap() + .seek_exact(hashed_address) + .unwrap() + .is_none()); +} + #[test] // This ensures that the walker goes over all the storage slots fn test_storage_root() { diff --git a/crates/trie/trie/src/trie.rs b/crates/trie/trie/src/trie.rs index 280ad5b7d6c..65efe86a27a 100644 --- a/crates/trie/trie/src/trie.rs +++ b/crates/trie/trie/src/trie.rs @@ -7,13 +7,13 @@ use crate::{ StateRootProgress, StorageRootProgress, }, stats::TrieTracker, - trie_cursor::{TrieCursor, TrieCursorFactory, TrieStorageCursor}, + trie_cursor::{TrieCursor, TrieCursorFactory, TrieCursorIter, TrieStorageCursor}, updates::{StorageTrieUpdates, TrieUpdates}, walker::TrieWalker, HashBuilder, Nibbles, TRIE_ACCOUNT_RLP_MAX_SIZE, }; use alloy_consensus::EMPTY_ROOT_HASH; -use alloy_primitives::{keccak256, Address, B256, U256}; +use alloy_primitives::{keccak256, map::B256Map, Address, B256, U256}; use alloy_rlp::{BufMut, Encodable}; use alloy_trie::proof::AddedRemovedKeys; use reth_execution_errors::{StateRootError, StorageRootError}; @@ -352,9 +352,28 @@ where let root = hash_builder.root(); + let mut destroyed_storage_trie_nodes = B256Map::default(); + if retain_updates && !self.prefix_sets.destroyed_accounts.is_empty() { + let mut storage_trie_cursor = match storage_trie_cursor { + Some(cursor) => cursor, + None => self.trie_cursor_factory.storage_trie_cursor(B256::ZERO)?, + }; + + for hashed_address in &self.prefix_sets.destroyed_accounts { + storage_trie_cursor.set_hashed_address(*hashed_address); + let nodes = TrieCursorIter::new(&mut storage_trie_cursor) + .map(|node| node.map(|(path, _)| path)) + .collect::, _>>()?; + + if !nodes.is_empty() { + destroyed_storage_trie_nodes.insert(*hashed_address, nodes); + } + } + } + let removed_keys = account_node_iter.walker.take_removed_keys(); let StateRootContext { mut trie_updates, hashed_entries_walked, .. } = storage_ctx; - trie_updates.finalize(hash_builder, removed_keys, self.prefix_sets.destroyed_accounts); + trie_updates.finalize(hash_builder, removed_keys, destroyed_storage_trie_nodes); let stats = tracker.finish(); From 8a8163e718f0d525bd731e6f22f16e4b0de31408 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Wed, 26 Aug 2026 11:50:31 +0000 Subject: [PATCH 068/294] fix(exex): support sparse WAL file IDs (#26819) --- crates/exex/exex/src/wal/mod.rs | 68 ++++++++++++++++++++++------- crates/exex/exex/src/wal/storage.rs | 34 ++++++--------- 2 files changed, 67 insertions(+), 35 deletions(-) diff --git a/crates/exex/exex/src/wal/mod.rs b/crates/exex/exex/src/wal/mod.rs index 0836e15b55a..e96d8b6cd3e 100644 --- a/crates/exex/exex/src/wal/mod.rs +++ b/crates/exex/exex/src/wal/mod.rs @@ -113,13 +113,14 @@ where /// Fills the block cache with the notifications from the storage. #[instrument(skip(self))] fn fill_block_cache(&self) -> WalResult<()> { - let Some(files_range) = self.storage.files_range()? else { return Ok(()) }; - self.next_file_id.store(files_range.end() + 1, Ordering::Relaxed); + let file_ids = self.storage.file_ids()?; + let Some(last_file_id) = file_ids.last() else { return Ok(()) }; + self.next_file_id.store(last_file_id + 1, Ordering::Relaxed); let mut block_cache = self.block_cache.write(); let mut notifications_size = 0; - for entry in self.storage.iter_notifications(files_range) { + for entry in self.storage.iter_notifications(file_ids) { let (file_id, size, notification) = entry?; notifications_size += size; @@ -198,11 +199,9 @@ where fn iter_notifications( &self, ) -> WalResult>> + '_>> { - let Some(range) = self.storage.files_range()? else { - return Ok(Box::new(std::iter::empty())) - }; - - Ok(Box::new(self.storage.iter_notifications(range).map(|entry| Ok(entry?.2)))) + Ok(Box::new( + self.storage.iter_notifications(self.storage.file_ids()?).map(|entry| Ok(entry?.2)), + )) } } @@ -238,6 +237,7 @@ mod tests { use crate::wal::{cache::CachedBlock, error::WalResult, Wal}; use alloy_primitives::B256; use itertools::Itertools; + use reth_ethereum_primitives::EthPrimitives; use reth_exex_types::ExExNotification; use reth_provider::Chain; use reth_testing_utils::generators::{ @@ -246,13 +246,11 @@ mod tests { use std::{collections::BTreeMap, sync::Arc}; fn read_notifications(wal: &Wal) -> WalResult> { - wal.inner.storage.files_range()?.map_or(Ok(Vec::new()), |range| { - wal.inner - .storage - .iter_notifications(range) - .map(|entry| entry.map(|(_, _, n)| n)) - .collect() - }) + wal.inner + .storage + .iter_notifications(wal.inner.storage.file_ids()?) + .map(|entry| entry.map(|(_, _, n)| n)) + .collect() } fn sort_committed_blocks( @@ -522,4 +520,44 @@ mod tests { Ok(()) } + + #[test] + fn test_finalize_with_sparse_file_ids() -> eyre::Result<()> { + let mut rng = generators::rng(); + + let temp_dir = tempfile::tempdir()?; + let wal = Wal::::new(&temp_dir)?; + + let blocks = random_block_range(&mut rng, 0..=5, BlockRangeParams::default()) + .into_iter() + .map(|block| block.try_recover()) + .collect::, _>>()?; + let chain = |range: std::ops::RangeInclusive| { + Arc::new(Chain::new(blocks[range].to_vec(), Default::default(), BTreeMap::new())) + }; + + let commit_to_four = ExExNotification::ChainCommitted { new: chain(1..=4) }; + let revert_to_two = ExExNotification::ChainReverted { old: chain(2..=4) }; + let commit_two = ExExNotification::ChainCommitted { new: chain(2..=2) }; + let commit_to_five = ExExNotification::ChainCommitted { new: chain(3..=5) }; + + wal.commit(&commit_to_four)?; + wal.commit(&revert_to_two)?; + wal.commit(&commit_two)?; + wal.commit(&commit_to_five)?; + assert_eq!(wal.inner.storage.file_ids()?, vec![0, 1, 2, 3]); + + // File 2 has a lower maximum block than the surrounding notifications, so finalizing it + // leaves a gap in the file ID sequence. + wal.finalize((blocks[2].number, blocks[2].hash()).into())?; + assert_eq!(wal.inner.storage.file_ids()?, vec![0, 1, 3]); + + let wal = Wal::::new(&temp_dir)?; + assert_eq!(read_notifications(&wal)?, vec![commit_to_four, revert_to_two, commit_to_five]); + + wal.commit(&commit_two)?; + assert_eq!(wal.inner.storage.file_ids()?, vec![0, 1, 3, 4]); + + Ok(()) + } } diff --git a/crates/exex/exex/src/wal/storage.rs b/crates/exex/exex/src/wal/storage.rs index 8c36e3f624d..45363130c23 100644 --- a/crates/exex/exex/src/wal/storage.rs +++ b/crates/exex/exex/src/wal/storage.rs @@ -1,6 +1,5 @@ use std::{ fs::File, - ops::RangeInclusive, path::{Path, PathBuf}, }; @@ -69,12 +68,9 @@ where } } - /// Returns the range of file IDs in the storage. - /// - /// If there are no files in the storage, returns `None`. - pub(super) fn files_range(&self) -> WalResult>> { - let mut min_id = None; - let mut max_id = None; + /// Returns the file IDs in the storage in ascending order. + pub(super) fn file_ids(&self) -> WalResult> { + let mut file_ids = Vec::new(); for entry in reth_fs_util::read_dir(&self.path)? { let entry = entry.map_err(|err| WalError::DirEntry(self.path.clone(), err))?; @@ -82,13 +78,12 @@ where if entry.path().extension() == Some(FILE_EXTENSION.as_ref()) { let file_name = entry.file_name(); let file_id = Self::parse_filename(&file_name.to_string_lossy())?; - - min_id = min_id.map_or(Some(file_id), |min_id: u32| Some(min_id.min(file_id))); - max_id = max_id.map_or(Some(file_id), |max_id: u32| Some(max_id.max(file_id))); + file_ids.push(file_id); } } - Ok(min_id.zip(max_id).map(|(min_id, max_id)| min_id..=max_id)) + file_ids.sort_unstable(); + Ok(file_ids) } /// Removes notifications from the storage according to the given list of file IDs. @@ -113,11 +108,11 @@ where Ok((deleted_total, deleted_size)) } - pub(super) fn iter_notifications( - &self, - range: RangeInclusive, - ) -> impl Iterator)>> + '_ { - range.map(move |id| { + pub(super) fn iter_notifications<'a>( + &'a self, + file_ids: impl IntoIterator + 'a, + ) -> impl Iterator)>> + 'a { + file_ids.into_iter().map(move |id| { let (notification, size) = self.read_notification(id)?.ok_or(WalError::FileNotFound(id))?; @@ -316,21 +311,20 @@ mod tests { } #[test] - fn test_files_range() -> eyre::Result<()> { + fn test_file_ids() -> eyre::Result<()> { let temp_dir = tempfile::tempdir()?; let storage: Storage = Storage::new(&temp_dir)?; // Create WAL files File::create(storage.file_path(1))?; - File::create(storage.file_path(2))?; File::create(storage.file_path(3))?; // Create non-WAL files that should be ignored File::create(temp_dir.path().join("0.tmp"))?; File::create(temp_dir.path().join("4.tmp"))?; - // Check files range - assert_eq!(storage.files_range()?, Some(1..=3)); + // Check existing file IDs are returned in order without filling the gap + assert_eq!(storage.file_ids()?, vec![1, 3]); Ok(()) } From c9b466c13823bfc73a94af590ca42a51eccaadf7 Mon Sep 17 00:00:00 2001 From: Brian Picciano Date: Wed, 26 Aug 2026 12:40:32 +0000 Subject: [PATCH 069/294] refactor(trie): remove storage wipe markers (#26811) Co-authored-by: Matthias Seitz --- .../engine/invalid-block-hooks/src/witness.rs | 7 +- .../src/tree/payload_processor/prewarm.rs | 2 +- .../tree/state_root_strategy/sparse_trie.rs | 2 +- crates/engine/tree/src/tree/trie_updates.rs | 29 +- crates/exex/exex/src/wal/storage.rs | 32 +- crates/revm/src/witness.rs | 11 +- crates/rpc/rpc/src/debug.rs | 1 - .../stages/stages/src/stages/execution/mod.rs | 1 - crates/stages/stages/tests/preimage.rs | 16 +- .../src/providers/database/provider.rs | 131 +--- crates/storage/provider/src/writer/mod.rs | 11 +- crates/storage/storage-api/src/state.rs | 2 +- .../storage-overlay/src/changeset_cache.rs | 5 +- crates/trie/common/src/added_removed_keys.rs | 45 -- crates/trie/common/src/hashed_state.rs | 576 ++++-------------- crates/trie/common/src/updates.rs | 347 +++-------- crates/trie/db/src/state.rs | 2 +- crates/trie/db/src/storage.rs | 2 +- crates/trie/db/src/trie_cursor.rs | 5 - crates/trie/db/tests/fuzz_in_memory_nodes.rs | 18 +- crates/trie/db/tests/post_state.rs | 132 +--- crates/trie/db/tests/trie.rs | 4 +- crates/trie/db/tests/witness.rs | 5 +- crates/trie/sparse/src/arena/mod.rs | 9 - crates/trie/sparse/src/state.rs | 44 +- crates/trie/sparse/src/traits.rs | 12 - crates/trie/sparse/src/trie.rs | 19 - crates/trie/sparse/tests/suite/main.rs | 6 +- crates/trie/sparse/tests/suite/wipe_clear.rs | 61 +- crates/trie/trie/benches/proof_v2.rs | 6 +- crates/trie/trie/src/changesets.rs | 229 +------ .../trie/trie/src/hashed_cursor/post_state.rs | 52 +- crates/trie/trie/src/test_utils.rs | 5 - crates/trie/trie/src/trie_cursor/in_memory.rs | 49 +- crates/trie/trie/src/witness.rs | 41 +- 35 files changed, 331 insertions(+), 1588 deletions(-) diff --git a/crates/engine/invalid-block-hooks/src/witness.rs b/crates/engine/invalid-block-hooks/src/witness.rs index 574b8545c29..c7ad116ece7 100644 --- a/crates/engine/invalid-block-hooks/src/witness.rs +++ b/crates/engine/invalid-block-hooks/src/witness.rs @@ -12,7 +12,7 @@ use reth_revm::{ }; use reth_rpc_api::DebugApiClient; use reth_tracing::tracing::warn; -use reth_trie::{updates::TrieUpdates, HashedStorage}; +use reth_trie::updates::TrieUpdates; use revm::{ bytecode::Bytecode, database::{ @@ -138,10 +138,7 @@ fn collect_execution_data( if let Some(account_data) = account.account { preimages.insert(hashed_address, alloy_rlp::encode(address).into()); - let storage = hashed_state - .storages - .entry(hashed_address) - .or_insert_with(|| HashedStorage::new(false)); + let storage = hashed_state.storages.entry(hashed_address).or_default(); for (slot, value) in account_data.storage { let slot_bytes = B256::from(slot); diff --git a/crates/engine/tree/src/tree/payload_processor/prewarm.rs b/crates/engine/tree/src/tree/payload_processor/prewarm.rs index 4ac2174d1c0..ed795224604 100644 --- a/crates/engine/tree/src/tree/payload_processor/prewarm.rs +++ b/crates/engine/tree/src/tree/payload_processor/prewarm.rs @@ -680,7 +680,7 @@ where // changes to start processing them before potentially hitting the db in the next step. if !account_changes.storage_changes.is_empty() { let hashed_address = *hashed_address.get_or_insert_with(|| keccak256(address)); - let mut storage_map = reth_trie::HashedStorage::new(false); + let mut storage_map = reth_trie::HashedStorage::default(); for slot_changes in &account_changes.storage_changes { let hashed_slot = keccak256(slot_changes.slot.to_be_bytes::<32>()); diff --git a/crates/engine/tree/src/tree/state_root_strategy/sparse_trie.rs b/crates/engine/tree/src/tree/state_root_strategy/sparse_trie.rs index 9e852d9fe6a..3f9cd9fe79b 100644 --- a/crates/engine/tree/src/tree/state_root_strategy/sparse_trie.rs +++ b/crates/engine/tree/src/tree/state_root_strategy/sparse_trie.rs @@ -1146,7 +1146,7 @@ mod tests { address, Some(Account { balance: U256::from(100), nonce: 1, bytecode_hash: None }), ); - let mut storage = reth_trie::HashedStorage::new(false); + let mut storage = reth_trie::HashedStorage::default(); storage.storage.insert(slot, value); hashed_state.storages.insert(address, storage); diff --git a/crates/engine/tree/src/tree/trie_updates.rs b/crates/engine/tree/src/tree/trie_updates.rs index a757d555714..87a1081f2ba 100644 --- a/crates/engine/tree/src/tree/trie_updates.rs +++ b/crates/engine/tree/src/tree/trie_updates.rs @@ -59,28 +59,16 @@ impl TrieUpdatesDiff { #[derive(Debug, Default)] struct StorageTrieUpdatesDiff { - is_deleted: Option>, storage_nodes: HashMap>>, removed_nodes: HashMap>, } impl StorageTrieUpdatesDiff { fn has_differences(&self) -> bool { - self.is_deleted.is_some() || - !self.storage_nodes.is_empty() || - !self.removed_nodes.is_empty() + !self.storage_nodes.is_empty() || !self.removed_nodes.is_empty() } fn log_differences(&self, address: B256) { - if let Some(EntryDiff { - task: task_deleted, - regular: regular_deleted, - database: database_not_exists, - }) = self.is_deleted - { - warn!(target: "engine::tree", ?address, ?task_deleted, ?regular_deleted, ?database_not_exists, "Difference in storage trie deletion"); - } - for (path, EntryDiff { task, regular, database }) in &self.storage_nodes { warn!(target: "engine::tree", ?address, ?path, ?task, ?regular, ?database, "Difference in storage trie updates"); } @@ -192,20 +180,7 @@ fn compare_storage_trie_updates( task: &mut StorageTrieUpdates, regular: &mut StorageTrieUpdates, ) -> Result { - // Check if the storage trie exists by seeking to the first entry - let database_not_exists = trie_cursor()?.seek(Nibbles::default())?.is_none(); - let mut diff = StorageTrieUpdatesDiff { - // If the deletion is a no-op, meaning that the entry is not in the - // database, do not add it to the diff. - is_deleted: (task.is_deleted != regular.is_deleted && !database_not_exists).then_some( - EntryDiff { - task: task.is_deleted, - regular: regular.is_deleted, - database: database_not_exists, - }, - ), - ..Default::default() - }; + let mut diff = StorageTrieUpdatesDiff::default(); // compare storage nodes let mut storage_trie_cursor = trie_cursor()?; diff --git a/crates/exex/exex/src/wal/storage.rs b/crates/exex/exex/src/wal/storage.rs index 45363130c23..6a23f86b3a9 100644 --- a/crates/exex/exex/src/wal/storage.rs +++ b/crates/exex/exex/src/wal/storage.rs @@ -183,8 +183,10 @@ mod tests { use reth_provider::Chain; use reth_testing_utils::generators::{self, random_block}; use reth_trie_common::{ - updates::{StorageTrieUpdates, TrieUpdates}, - BranchNodeCompact, ComputedTrieData, HashedPostState, HashedStorage, LazyTrieData, Nibbles, + serde_bincode_compat, + updates::{StorageTrieUpdates, StorageTrieUpdatesSorted, TrieUpdates}, + BranchNodeCompact, ComputedTrieData, HashedPostState, HashedStorage, HashedStorageSorted, + LazyTrieData, Nibbles, }; use std::{collections::BTreeMap, fs::File, sync::Arc}; @@ -215,6 +217,26 @@ mod tests { Ok(()) } + #[test] + fn test_decode_legacy_sorted_trie_data() -> eyre::Result<()> { + let storage_nodes = + vec![(Nibbles::from_nibbles_unchecked([0x01]), Some(BranchNodeCompact::default()))]; + let encoded = rmp_serde::encode::to_vec(&(false, &storage_nodes))?; + let decoded: serde_bincode_compat::updates::StorageTrieUpdatesSorted<'_> = + rmp_serde::decode::from_slice(&encoded)?; + let decoded: StorageTrieUpdatesSorted = decoded.into(); + assert_eq!(decoded.storage_nodes, storage_nodes); + + let storage_slots = vec![(B256::from([1; 32]), U256::from(1))]; + let encoded = rmp_serde::encode::to_vec(&(&storage_slots, false))?; + let decoded: serde_bincode_compat::hashed_state::HashedStorageSorted<'_> = + rmp_serde::decode::from_slice(&encoded)?; + let decoded: HashedStorageSorted = decoded.into(); + assert_eq!(decoded.storage_slots, storage_slots); + + Ok(()) + } + /// Generate a new WAL file for testing. /// /// Run this test with `--ignored` to generate a new test WAL file: @@ -270,7 +292,6 @@ mod tests { storage_tries: HashMap::from_iter([( hashed_address, StorageTrieUpdates { - is_deleted: false, storage_nodes: HashMap::from_iter([( Nibbles::from_nibbles_unchecked([0x04]), BranchNodeCompact::default(), @@ -287,10 +308,7 @@ mod tests { )]), storages: HashMap::from_iter([( hashed_address, - HashedStorage { - wiped: false, - storage: HashMap::from_iter([(storage_key, U256::from(101))]), - }, + HashedStorage { storage: HashMap::from_iter([(storage_key, U256::from(101))]) }, )]), }; diff --git a/crates/revm/src/witness.rs b/crates/revm/src/witness.rs index 3be8bad69e8..8f105eef11e 100644 --- a/crates/revm/src/witness.rs +++ b/crates/revm/src/witness.rs @@ -1,6 +1,6 @@ use alloc::vec::Vec; use alloy_primitives::{keccak256, Bytes, B256}; -use reth_trie::{ExecutionWitnessMode, HashedPostState, HashedStorage}; +use reth_trie::{ExecutionWitnessMode, HashedPostState}; use revm::database::State; /// Borrows finalized execution state for witness generation. @@ -106,10 +106,7 @@ impl<'a, DB> ExecutionWitnessRecord<'a, DB> { .accounts .insert(hashed_address, account.account.as_ref().map(|a| (&a.info).into())); - let storage = hashed_state - .storages - .entry(hashed_address) - .or_insert_with(|| HashedStorage::new(false)); + let storage = hashed_state.storages.entry(hashed_address).or_default(); if let Some(account) = &account.account { keys.push(address.to_vec().into()); @@ -138,6 +135,7 @@ mod tests { use alloy_primitives::{Address, U256}; use reth_storage_api::HashedPostStateProvider; use reth_storage_errors::provider::ProviderResult; + use reth_trie::HashedStorage; use revm::{ database::{states::CacheAccount, AccountStatus, BundleAccount, EmptyDB}, state::AccountInfo, @@ -157,7 +155,7 @@ mod tests { } #[test] - fn destroyed_account_storage_is_zero_expanded_without_wipe() { + fn destroyed_account_storage_is_zero_expanded() { let address = Address::with_last_byte(1); let hashed_address = keccak256(address); let hashed_slot = B256::with_last_byte(2); @@ -184,7 +182,6 @@ mod tests { let (hashed_state, _) = ExecutionWitnessRecord::new(&state).hashed_post_state(&provider).unwrap(); let storage = hashed_state.storages.get(&hashed_address).unwrap(); - assert!(!storage.wiped); assert_eq!(storage.storage.get(&hashed_slot), Some(&U256::ZERO)); } } diff --git a/crates/rpc/rpc/src/debug.rs b/crates/rpc/rpc/src/debug.rs index 1b3d42a355d..0ff27fdecf8 100644 --- a/crates/rpc/rpc/src/debug.rs +++ b/crates/rpc/rpc/src/debug.rs @@ -1534,7 +1534,6 @@ mod tests { let hashed_state = provider.hashed_post_state(&bundle_state).unwrap(); let storage = &hashed_state.storages[&hashed_address]; - assert!(!storage.wiped); assert_eq!(storage.storage[&hashed_old_slot], U256::ZERO); assert_eq!(storage.storage[&hashed_new_slot], new_value); } diff --git a/crates/stages/stages/src/stages/execution/mod.rs b/crates/stages/stages/src/stages/execution/mod.rs index e2a19febb98..e3e8121d11d 100644 --- a/crates/stages/stages/src/stages/execution/mod.rs +++ b/crates/stages/stages/src/stages/execution/mod.rs @@ -846,7 +846,6 @@ mod tests { let hashed_state = provider.latest().hashed_post_state(&state.bundle).unwrap(); let storage = &hashed_state.storages[&hashed_address]; - assert!(!storage.wiped); assert_eq!(storage.storage[&first_slot], U256::ZERO); assert_eq!(storage.storage[&second_slot], U256::ZERO); assert!(state.bundle.reverts.is_empty()); diff --git a/crates/stages/stages/tests/preimage.rs b/crates/stages/stages/tests/preimage.rs index 001f160e9aa..9f7b0436552 100644 --- a/crates/stages/stages/tests/preimage.rs +++ b/crates/stages/stages/tests/preimage.rs @@ -159,10 +159,10 @@ async fn test_pipeline_v2_selfdestruct_changesets_use_plain_slots() -> eyre::Res /// Prefunding ensures the CREATE2 target has original account state, while its init code performs /// no `SSTORE`. This verifies that destruction alone does not create an otherwise-empty -/// [`reth_trie::HashedStorage`] solely to carry `wiped = true`. +/// [`reth_trie::HashedStorage`] update. #[tokio::test(flavor = "multi_thread")] -async fn test_pipeline_v2_prefunded_create2_selfdestruct_does_not_wipe_storage() -> eyre::Result<()> -{ +async fn test_pipeline_v2_prefunded_create2_selfdestruct_does_not_create_storage_update( +) -> eyre::Result<()> { reth_tracing::init_test_tracing(); let scenario = setup_create2_selfdestruct_scenario()?; @@ -193,8 +193,8 @@ async fn test_pipeline_v2_prefunded_create2_selfdestruct_does_not_wipe_storage() } /// Scenario coverage: -/// 1. Single execution batch (`1..=2`) where block 1 writes and block 2 wipes. -/// 2. Block-by-block storage changesets still contain plain keys for wipe entries. +/// 1. Single execution batch (`1..=2`) where block 1 writes and block 2 destroys the account. +/// 2. Block-by-block storage changesets still contain plain keys for destroyed entries. /// /// Regression coverage for single execution-batch behavior. #[tokio::test(flavor = "multi_thread")] @@ -228,11 +228,11 @@ async fn test_pipeline_v2_single_batch_write_then_selfdestruct_changesets_plain_ /// Scenario coverage: /// 1. A slot appears only in intermediate reverts (not final bundle state). -/// 2. A later block in the same execution batch wipes the account. -/// 3. Wipe changesets still contain the required plain slot key/value. +/// 2. A later block in the same execution batch destroys the account. +/// 3. Destroyed-storage changesets still contain the required plain slot key/value. /// /// Covers the edge case where a slot appears in intermediate block reverts but not in the final -/// bundle state, then gets wiped by a later selfdestruct in the same execution batch. +/// bundle state, then is destroyed by a later selfdestruct in the same execution batch. #[tokio::test(flavor = "multi_thread")] async fn test_pipeline_v2_single_batch_reverted_slot_then_selfdestruct_changesets_plain_slots( ) -> eyre::Result<()> { diff --git a/crates/storage/provider/src/providers/database/provider.rs b/crates/storage/provider/src/providers/database/provider.rs index 3dac39e0046..6ced4f2624b 100644 --- a/crates/storage/provider/src/providers/database/provider.rs +++ b/crates/storage/provider/src/providers/database/provider.rs @@ -786,25 +786,8 @@ impl DatabaseProvider StateWriter let mut hashed_storage_cursor = self.tx_ref().cursor_dup_write::()?; for (hashed_address, storage) in sorted_storages { - if storage.is_wiped() && hashed_storage_cursor.seek_exact(*hashed_address)?.is_some() { - hashed_storage_cursor.delete_current_duplicates()?; - } - for (hashed_slot, value) in storage.storage_slots_ref() { let entry = StorageEntry { key: *hashed_slot, value: *value }; @@ -4474,14 +4453,14 @@ mod tests { ) .unwrap(); - // Add storage nodes for address2 (will be wiped) + // Add storage nodes for address2. storage_cursor .upsert( storage_address2, &StorageTrieEntry { nibbles: StoredNibblesSubKey(Nibbles::from_nibbles([0xa, 0xb])), node: BranchNodeCompact::new( - 0b1100_1100_1100_1100, // will be wiped + 0b1100_1100_1100_1100, 0b0000_0000_0000_0000, 0b0000_0000_0000_0000, vec![], @@ -4496,7 +4475,7 @@ mod tests { &StorageTrieEntry { nibbles: StoredNibblesSubKey(Nibbles::from_nibbles([0xc, 0xd])), node: BranchNodeCompact::new( - 0b0011_1100_0011_1100, // will be wiped + 0b0011_1100_0011_1100, 0b0000_0000_0000_0000, 0b0000_0000_0000_0000, vec![], @@ -4534,7 +4513,6 @@ mod tests { // Create sorted storage trie updates let storage_trie1 = StorageTrieUpdatesSorted { - is_deleted: false, storage_nodes: vec![ ( Nibbles::from_nibbles([0x1, 0x0]), @@ -4551,8 +4529,10 @@ mod tests { }; let storage_trie2 = StorageTrieUpdatesSorted { - is_deleted: true, // Wipe all storage for this address - storage_nodes: vec![], + storage_nodes: vec![ + (Nibbles::from_nibbles([0xa, 0xb]), None), + (Nibbles::from_nibbles([0xc, 0xd]), None), + ], }; let mut storage_tries = B256Map::default(); @@ -4564,9 +4544,9 @@ mod tests { // Write the sorted trie updates let num_entries = provider_rw.write_trie_updates_sorted(&trie_updates).unwrap(); - // We should have 2 account insertions + 1 account deletion + 1 storage insertion + 1 - // storage deletion = 5 - assert_eq!(num_entries, 5); + // We should have 2 account insertions + 1 account deletion + 1 storage insertion + 3 + // storage deletions = 7 + assert_eq!(num_entries, 7); // Verify account trie updates were written correctly let tx = provider_rw.tx_ref(); @@ -4613,13 +4593,13 @@ mod tests { "Remaining entry should be [0x1, 0x0]" ); - // Check storage for address2 was wiped + // Check storage for address2 was removed let storage_entries2: Vec<_> = storage_cursor .walk_dup(Some(storage_address2), None) .unwrap() .collect::, _>>() .unwrap(); - assert_eq!(storage_entries2.len(), 0, "Storage address2 should be empty after wipe"); + assert_eq!(storage_entries2.len(), 0, "Storage address2 should be empty after removal"); provider_rw.commit().unwrap(); } @@ -4667,17 +4647,11 @@ mod tests { B256Map::from_iter([ ( kept_storage, - HashedStorageSorted { - wiped: false, - storage_slots: vec![(kept_slot, U256::from(1))], - }, + HashedStorageSorted { storage_slots: vec![(kept_slot, U256::from(1))] }, ), ( masked_storage, - HashedStorageSorted { - wiped: false, - storage_slots: vec![(masked_slot, U256::from(2))], - }, + HashedStorageSorted { storage_slots: vec![(masked_slot, U256::from(2))] }, ), ]), ); @@ -4690,14 +4664,12 @@ mod tests { ( kept_storage, StorageTrieUpdatesSorted { - is_deleted: false, storage_nodes: vec![(kept_storage_node, Some(branch(0b1010)))], }, ), ( masked_storage, StorageTrieUpdatesSorted { - is_deleted: false, storage_nodes: vec![(masked_storage_node, Some(branch(0b0101)))], }, ), @@ -4717,10 +4689,7 @@ mod tests { vec![(masked_account, Some(Account { nonce: 3, ..Default::default() }))], B256Map::from_iter([( masked_storage, - HashedStorageSorted { - wiped: false, - storage_slots: vec![(masked_slot, U256::from(4))], - }, + HashedStorageSorted { storage_slots: vec![(masked_slot, U256::from(4))] }, )]), ); let deferred_trie_updates = TrieUpdatesSorted::new( @@ -4728,7 +4697,6 @@ mod tests { B256Map::from_iter([( masked_storage, StorageTrieUpdatesSorted { - is_deleted: false, storage_nodes: vec![(masked_storage_node, Some(branch(0b1100)))], }, )]), @@ -4845,71 +4813,6 @@ mod tests { assert_eq!(masked_entries[0].1.nibbles.0, masked_storage_node); } - #[test] - fn test_save_blocks_merges_storage_wipe_in_multi_block_batch() { - use reth_trie::{ - updates::{StorageTrieUpdatesSorted, TrieUpdatesSorted}, - BranchNodeCompact, HashedPostStateSorted, - }; - - let factory = create_test_provider_factory(); - factory.set_storage_settings_cache(StorageSettings::v2()); - - let mut test_block_builder = TestBlockBuilder::eth().with_state(); - let genesis = test_block_builder.get_executed_blocks(0..1).next().unwrap(); - let mut blocks: Vec<_> = test_block_builder.get_executed_blocks(1..4).collect(); - let address = B256::with_last_byte(1); - let storage_path = Nibbles::from_nibbles([0x1, 0x2]); - - let provider_rw = factory.provider_rw().unwrap(); - save_genesis(&provider_rw, &genesis).unwrap(); - provider_rw - .write_trie_updates_sorted(&TrieUpdatesSorted::new( - vec![], - B256Map::from_iter([( - address, - StorageTrieUpdatesSorted { - is_deleted: false, - storage_nodes: vec![(storage_path, Some(BranchNodeCompact::default()))], - }, - )]), - )) - .unwrap(); - provider_rw.commit().unwrap(); - - let wipe = TrieUpdatesSorted::new( - vec![], - B256Map::from_iter([( - address, - StorageTrieUpdatesSorted { is_deleted: true, storage_nodes: vec![] }, - )]), - ); - let wipe_block = &blocks[2]; - blocks[2] = ExecutedBlock::new( - Arc::clone(&wipe_block.recovered_block), - Arc::clone(&wipe_block.execution_output), - ComputedTrieData::new(Arc::new(HashedPostStateSorted::default()), Arc::new(wipe)), - ); - - let provider_rw = factory.provider_rw().unwrap(); - let input = SaveBlocksInput::new(blocks, 0, 0, 3, 3); - provider_rw.save_blocks(&input).unwrap(); - provider_rw.commit().unwrap(); - - let provider = factory.provider().unwrap(); - let checkpoint = provider.get_stage_checkpoint(StageId::Finish).unwrap().unwrap(); - assert_eq!(checkpoint.block_number, 3); - let storage_entries = provider - .tx_ref() - .cursor_dup_read::() - .unwrap() - .walk_dup(Some(address), None) - .unwrap() - .collect::, _>>() - .unwrap(); - assert!(storage_entries.is_empty()); - } - #[test] fn test_save_blocks_partial_cycles_do_not_duplicate_static_file_writes() { let factory = create_test_provider_factory(); diff --git a/crates/storage/provider/src/writer/mod.rs b/crates/storage/provider/src/writer/mod.rs index 6f0f30db36a..ef8fb25013b 100644 --- a/crates/storage/provider/src/writer/mod.rs +++ b/crates/storage/provider/src/writer/mod.rs @@ -35,7 +35,7 @@ mod tests { use std::{collections::BTreeMap, str::FromStr}; #[test] - fn wiped_entries_are_removed() { + fn zeroed_entries_are_removed() { let provider_factory = create_test_provider_factory(); let addresses = (0..10).map(|_| Address::random()).collect::>(); @@ -67,7 +67,10 @@ mod tests { let mut hashed_state = HashedPostState::default(); hashed_state.accounts.insert(destroyed_address_hashed, None); - hashed_state.storages.insert(destroyed_address_hashed, HashedStorage::new(true)); + hashed_state.storages.insert( + destroyed_address_hashed, + HashedStorage::from_iter([(hashed_slot, U256::ZERO)]), + ); let provider_rw = provider_factory.provider_rw().unwrap(); assert!(matches!(provider_rw.write_hashed_state(&hashed_state.into_sorted()), Ok(()))); @@ -1170,7 +1173,9 @@ mod tests { .into_iter() .map(|str| (B256::from_str(str).unwrap(), U256::from(1))), ); - updated_storage.wiped = true; + updated_storage + .storage + .extend(init_storage.storage.keys().map(|hashed_slot| (*hashed_slot, U256::ZERO))); let mut state = HashedPostState::default(); state.storages.insert(hashed_address, updated_storage.clone()); provider_rw.write_hashed_state(&state.clone().into_sorted()).unwrap(); diff --git a/crates/storage/storage-api/src/state.rs b/crates/storage/storage-api/src/state.rs index b099295e9ed..5c2617022ca 100644 --- a/crates/storage/storage-api/src/state.rs +++ b/crates/storage/storage-api/src/state.rs @@ -101,7 +101,7 @@ pub trait HashedPostStateProvider { /// updates for parent storage of accounts that were destroyed but remain in the post-state. /// /// Providers backed by an exact parent-state view also materialize terminally destroyed - /// accounts so the result can be persisted without storage wipe markers. + /// accounts with explicit zero-valued storage updates. fn hashed_post_state(&self, bundle_state: &BundleState) -> ProviderResult; } diff --git a/crates/storage/storage-overlay/src/changeset_cache.rs b/crates/storage/storage-overlay/src/changeset_cache.rs index 79df0335e91..54f713d8d54 100644 --- a/crates/storage/storage-overlay/src/changeset_cache.rs +++ b/crates/storage/storage-overlay/src/changeset_cache.rs @@ -155,10 +155,7 @@ where storage_nodes.push((*nibbles, node_value)); } - storage_tries.insert( - *hashed_address, - StorageTrieUpdatesSorted { storage_nodes, is_deleted: storage_changeset.is_deleted }, - ); + storage_tries.insert(*hashed_address, StorageTrieUpdatesSorted { storage_nodes }); } Ok(TrieUpdatesSorted::new(account_nodes, storage_tries)) diff --git a/crates/trie/common/src/added_removed_keys.rs b/crates/trie/common/src/added_removed_keys.rs index 00aeef83d1e..8b756c7b301 100644 --- a/crates/trie/common/src/added_removed_keys.rs +++ b/crates/trie/common/src/added_removed_keys.rs @@ -44,14 +44,6 @@ impl MultiAddedRemovedKeys { .map(|entry| entry.unwrap_or_default()) .unwrap_or_default(); - if storage.wiped { - self.storages.remove(hashed_address); - if account.is_empty() { - self.account.insert_removed(*hashed_address); - } - continue - } - let storage_removed_keys = self.storages.entry(*hashed_address).or_insert_with(default_added_removed_keys); @@ -128,32 +120,6 @@ mod tests { assert!(!storage_keys.is_removed(&slot2)); } - #[test] - fn test_update_with_state_wiped_storage() { - let mut multi_keys = MultiAddedRemovedKeys::new(); - let mut update = HashedPostState::default(); - - let addr = B256::random(); - let slot1 = B256::random(); - - // First add some removed keys - let mut storage = HashedStorage::default(); - storage.storage.insert(slot1, U256::ZERO); - update.storages.insert(addr, storage); - multi_keys.update_with_state(&update); - assert!(multi_keys.get_storage(&addr).is_some()); - - // Now wipe the storage - let mut update2 = HashedPostState::default(); - let wiped_storage = HashedStorage::new(true); - update2.storages.insert(addr, wiped_storage); - multi_keys.update_with_state(&update2); - - // Storage and account should be removed - assert!(multi_keys.get_storage(&addr).is_none()); - assert!(multi_keys.get_accounts().is_removed(&addr)); - } - #[test] fn test_update_with_state_account_tracking() { let mut multi_keys = MultiAddedRemovedKeys::new(); @@ -205,16 +171,5 @@ mod tests { // Account should not be marked as removed because it has balance assert!(!multi_keys.get_accounts().is_removed(&addr)); - - // Now wipe the storage - let mut update2 = HashedPostState::default(); - let wiped_storage = HashedStorage::new(true); - update2.storages.insert(addr, wiped_storage); - update2.accounts.insert(addr, Some(account)); - multi_keys.update_with_state(&update2); - - // Storage should be None, but account should not be removed. - assert!(multi_keys.get_storage(&addr).is_none()); - assert!(!multi_keys.get_accounts().is_removed(&addr)); } } diff --git a/crates/trie/common/src/hashed_state.rs b/crates/trie/common/src/hashed_state.rs index ecfbea13172..987eb35e7b5 100644 --- a/crates/trie/common/src/hashed_state.rs +++ b/crates/trie/common/src/hashed_state.rs @@ -172,9 +172,6 @@ impl HashedPostState { /// Partition the state update into two state updates: /// - First with accounts and storages slots that are present in the provided targets. /// - Second with all other. - /// - /// CAUTION: The state updates are expected to be applied in order, so that the storage wipes - /// are done correctly. pub fn partition_by_targets( mut self, targets: &MultiProofTargets, @@ -199,18 +196,8 @@ impl HashedPostState { false }); - // We do not check the wiped flag here, because targets only contain addresses - // and storage slots. So if there are no storage slots left, the storage update - // can be fully removed. let retain = !storage.storage.is_empty(); - // Since state updates are expected to be applied in order, we can only set the - // wiped flag in the second storage update if the first storage update is empty - // and will not be retained. - if !retain { - storage_not_in_targets.wiped = storage.wiped; - } - ( retain, storage_not_in_targets.is_empty().not().then_some(storage_not_in_targets), @@ -247,10 +234,7 @@ impl HashedPostState { /// Returns the number of items that will be considered during chunking in `[Self::chunks]`. pub fn chunking_length(&self) -> usize { self.accounts.len() + - self.storages - .values() - .map(|storage| if storage.wiped { 1 } else { 0 } + storage.storage.len()) - .sum::() + self.storages.values().map(|storage| storage.storage.len()).sum::() } /// Extend this hashed post state with contents of another. @@ -316,7 +300,7 @@ impl HashedPostState { for (hashed_address, sorted_storage) in &sorted.storages { match self.storages.entry(*hashed_address) { hash_map::Entry::Vacant(entry) => { - let mut new_storage = HashedStorage::new(false); + let mut new_storage = HashedStorage::default(); new_storage.extend_from_sorted(sorted_storage); entry.insert(new_storage); } @@ -423,27 +407,20 @@ impl FromParallelIterator<(B256, Option, Option)> for Ha #[derive(PartialEq, Eq, Clone, Debug, Default)] #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] pub struct HashedStorage { - /// Flag indicating whether the storage was wiped or not. - pub wiped: bool, /// Mapping of hashed storage slot to storage value. pub storage: B256Map, } impl HashedStorage { - /// Create new instance of [`HashedStorage`]. - pub fn new(wiped: bool) -> Self { - Self { wiped, storage: HashMap::default() } - } - /// Check if self is empty. pub fn is_empty(&self) -> bool { - !self.wiped && self.storage.is_empty() + self.storage.is_empty() } /// Create new hashed storage from iterator. #[expect(clippy::should_implement_trait)] pub fn from_iter(iter: impl IntoIterator) -> Self { - Self { wiped: false, storage: HashMap::from_iter(iter) } + Self { storage: HashMap::from_iter(iter) } } /// Create new hashed storage from plain storage. @@ -455,24 +432,16 @@ impl HashedStorage { /// Construct [`PrefixSetMut`] from hashed storage. pub fn construct_prefix_set(&self) -> PrefixSetMut { - if self.wiped { - PrefixSetMut::all() - } else { - let mut prefix_set = PrefixSetMut::with_capacity(self.storage.len()); - for hashed_slot in self.storage.keys() { - prefix_set.insert(Nibbles::unpack(hashed_slot)); - } - prefix_set + let mut prefix_set = PrefixSetMut::with_capacity(self.storage.len()); + for hashed_slot in self.storage.keys() { + prefix_set.insert(Nibbles::unpack(hashed_slot)); } + prefix_set } /// Extend hashed storage with contents of other. /// The entries in second hashed storage take precedence. pub fn extend(&mut self, other: &Self) { - if other.wiped { - self.wiped = true; - self.storage.clear(); - } self.storage.extend(other.storage.iter().map(|(&k, &v)| (k, v))); } @@ -480,11 +449,6 @@ impl HashedStorage { /// representation. This is more efficient than first converting to `HashedStorage` and /// then extending, as it avoids creating intermediate `HashMap` allocations. pub fn extend_from_sorted(&mut self, sorted: &HashedStorageSorted) { - if sorted.wiped { - self.wiped = true; - self.storage.clear(); - } - // Reserve capacity for all slots self.storage.reserve(sorted.storage_slots.len()); @@ -499,7 +463,7 @@ impl HashedStorage { let mut storage_slots: Vec<_> = self.storage.into_iter().collect(); storage_slots.sort_unstable_by_key(|(key, _)| *key); - HashedStorageSorted { storage_slots, wiped: self.wiped } + HashedStorageSorted { storage_slots } } /// Creates a sorted copy without consuming self. @@ -508,7 +472,7 @@ impl HashedStorage { let mut storage_slots: Vec<_> = self.storage.iter().map(|(&k, &v)| (k, v)).collect(); storage_slots.sort_unstable_by_key(|(key, _)| *key); - HashedStorageSorted { storage_slots, wiped: self.wiped } + HashedStorageSorted { storage_slots } } } @@ -570,20 +534,13 @@ impl HashedPostStateSorted { for (hashed_address, hashed_storage) in &self.storages { // Ensure account trie covers storage overlays even if account map is empty. account_prefix_set.insert(Nibbles::unpack(hashed_address)); - - let prefix_set = if hashed_storage.wiped { - PrefixSetMut::all() - } else { - let mut prefix_set = - PrefixSetMut::with_capacity(hashed_storage.storage_slots.len()); - prefix_set.extend_keys( - hashed_storage - .storage_slots - .iter() - .map(|(hashed_slot, _)| Nibbles::unpack(hashed_slot)), - ); - prefix_set - }; + let mut prefix_set = PrefixSetMut::with_capacity(hashed_storage.storage_slots.len()); + prefix_set.extend_keys( + hashed_storage + .storage_slots + .iter() + .map(|(hashed_slot, _)| Nibbles::unpack(hashed_slot)), + ); storage_prefix_sets.insert(*hashed_address, prefix_set); } @@ -652,8 +609,6 @@ impl HashedPostStateSorted { let accounts = kway_merge_sorted(items.iter().map(|i| i.as_ref().accounts.as_slice())); struct StorageAcc<'a> { - wiped: bool, - sealed: bool, slices: Vec<&'a [(B256, U256)]>, } @@ -661,21 +616,8 @@ impl HashedPostStateSorted { for item in items { for (addr, storage) in &item.as_ref().storages { - let entry = acc.entry(*addr).or_insert_with(|| StorageAcc { - wiped: false, - sealed: false, - slices: Vec::new(), - }); - - if entry.sealed { - continue; - } - + let entry = acc.entry(*addr).or_insert_with(|| StorageAcc { slices: Vec::new() }); entry.slices.push(storage.storage_slots.as_slice()); - if storage.wiped { - entry.wiped = true; - entry.sealed = true; - } } } @@ -683,7 +625,7 @@ impl HashedPostStateSorted { .into_iter() .map(|(addr, entry)| { let storage_slots = kway_merge_sorted(entry.slices); - (addr, HashedStorageSorted { wiped: entry.wiped, storage_slots }) + (addr, HashedStorageSorted { storage_slots }) }) .collect(); @@ -697,10 +639,6 @@ impl HashedPostStateSorted { /// level. For duplicate keys in the batch, later items take precedence over earlier ones. An /// overlapping entry is retained if any mask value is equal to the merged batch value. The /// order of the mask does not matter. An empty mask merges the batch without filtering. - /// - /// # Panics - /// - /// Panics if any batch or mask entry wipes an entire storage. pub fn disjointed_merge_batch<'a>(batch: &[&'a Self], mask: &[&'a Self]) -> Self { let account_count = batch.iter().map(|item| item.accounts.len()).sum(); let mut accounts = Vec::with_capacity(account_count); @@ -726,10 +664,6 @@ impl HashedPostStateSorted { for item in batch.iter().rev() { for (hashed_address, storage) in &item.storages { - assert!( - !storage.wiped, - "storage wipes are not supported by disjointed_merge_batch" - ); let entry = storages .entry(*hashed_address) .or_insert_with(|| StorageAcc { slot_count: 0, slices: Vec::new() }); @@ -744,10 +678,6 @@ impl HashedPostStateSorted { ); for item in mask { for (hashed_address, storage) in &item.storages { - assert!( - !storage.wiped, - "storage wipes are not supported by disjointed_merge_batch" - ); let entry = storage_masks.entry(*hashed_address).or_default(); entry.slices.push(storage.storage_slots.as_slice()); } @@ -769,10 +699,8 @@ impl HashedPostStateSorted { None => kway_merge_sorted(entry.slices), }; - (!storage_slots.is_empty() || mask.is_empty()).then_some(( - hashed_address, - HashedStorageSorted { wiped: false, storage_slots }, - )) + (!storage_slots.is_empty() || mask.is_empty()) + .then_some((hashed_address, HashedStorageSorted { storage_slots })) }) .collect(); @@ -798,16 +726,9 @@ impl AsRef for HashedPostStateSorted { pub struct HashedStorageSorted { /// Sorted collection of updated storage slots. [`U256::ZERO`] indicates a deleted value. pub storage_slots: Vec<(B256, U256)>, - /// Flag indicating whether the storage was wiped or not. - pub wiped: bool, } impl HashedStorageSorted { - /// Returns `true` if the account was wiped. - pub const fn is_wiped(&self) -> bool { - self.wiped - } - /// Returns reference to updated storage slots. pub fn storage_slots_ref(&self) -> &[(B256, U256)] { &self.storage_slots @@ -824,35 +745,16 @@ impl HashedStorageSorted { } /// Extends the storage slots updates with another set of sorted updates. - /// - /// If `other` is marked as deleted, this will be marked as deleted and all slots cleared. - /// Otherwise, nodes are merged with `other`'s values taking precedence for duplicates. pub fn extend_ref(&mut self, other: &Self) { - if other.wiped { - // If other is wiped, clear everything and copy from other - self.wiped = true; - self.storage_slots.clear(); - self.storage_slots.extend(other.storage_slots.iter().copied()); - return; - } - - // Extend the sorted non-zero valued slots extend_sorted_vec(&mut self.storage_slots, &other.storage_slots); } /// Batch-merge sorted hashed storage. Iterator yields **newest to oldest**. - /// If any update is wiped, prior data is discarded. pub fn merge_batch<'a>(updates: impl IntoIterator) -> Self { let updates: Vec<_> = updates.into_iter().collect(); - if updates.is_empty() { - return Self::default(); + Self { + storage_slots: kway_merge_sorted(updates.iter().map(|u| u.storage_slots.as_slice())), } - - let wipe_idx = updates.iter().position(|u| u.wiped); - let relevant = wipe_idx.map_or(&updates[..], |idx| &updates[..=idx]); - let storage_slots = kway_merge_sorted(relevant.iter().map(|u| u.storage_slots.as_slice())); - - Self { wiped: wipe_idx.is_some(), storage_slots } } } @@ -865,7 +767,7 @@ impl From for HashedStorage { storage.insert(slot, value); } - Self { wiped: sorted.wiped, storage } + Self { storage } } } @@ -893,10 +795,7 @@ impl From for HashedPostState { /// An iterator that yields chunks of the state updates of at most `size` account and storage /// targets. /// -/// # Notes -/// 1. Chunks are expected to be applied in order, because of storage wipes. If applied out of -/// order, it's possible to wipe more storage than in the original state update. -/// 2. For each account, chunks with storage updates come first, followed by account updates. +/// Storage updates for each account are yielded before its account update. #[derive(Debug)] pub struct ChunkedHashedPostState { flattened: alloc::vec::IntoIter<(B256, FlattenedHashedPostStateItem)>, @@ -904,10 +803,9 @@ pub struct ChunkedHashedPostState { } /// Order discriminant for sorting flattened state items. -/// Ordering: `StorageWipe` < `StorageUpdate` (by slot) < `Account` +/// Ordering: `StorageUpdate` (by slot) < `Account` #[derive(Debug, PartialEq, Eq, PartialOrd, Ord)] enum FlattenedStateOrder { - StorageWipe, StorageUpdate(B256), Account, } @@ -915,14 +813,12 @@ enum FlattenedStateOrder { #[derive(Debug)] enum FlattenedHashedPostStateItem { Account(Option), - StorageWipe, StorageUpdate { slot: B256, value: U256 }, } impl FlattenedHashedPostStateItem { const fn order(&self) -> FlattenedStateOrder { match self { - Self::StorageWipe => FlattenedStateOrder::StorageWipe, Self::StorageUpdate { slot, .. } => FlattenedStateOrder::StorageUpdate(*slot), Self::Account(_) => FlattenedStateOrder::Account, } @@ -935,21 +831,16 @@ impl ChunkedHashedPostState { .storages .into_iter() .flat_map(|(address, storage)| { - storage - .wiped - .then_some((address, FlattenedHashedPostStateItem::StorageWipe)) - .into_iter() - .chain(storage.storage.into_iter().map(move |(slot, value)| { - (address, FlattenedHashedPostStateItem::StorageUpdate { slot, value }) - })) + storage.storage.into_iter().map(move |(slot, value)| { + (address, FlattenedHashedPostStateItem::StorageUpdate { slot, value }) + }) }) .chain(hashed_post_state.accounts.into_iter().map(|(address, account)| { (address, FlattenedHashedPostStateItem::Account(account)) })) // Sort by address, then by item order to ensure correct application sequence: - // 1. Storage wipes (must come first to clear storage) - // 2. Storage updates (sorted by slot for determinism) - // 3. Account updates (can be applied last) + // 1. Storage updates (sorted by slot for determinism) + // 2. Account updates (can be applied last) .sorted_unstable_by_key(|(address, item)| (*address, item.order())); Self { flattened, size } @@ -970,9 +861,6 @@ impl Iterator for ChunkedHashedPostState { FlattenedHashedPostStateItem::Account(account) => { chunk.accounts.insert(address, account); } - FlattenedHashedPostStateItem::StorageWipe => { - chunk.storages.entry(address).or_default().wiped = true; - } FlattenedHashedPostStateItem::StorageUpdate { slot, value } => { chunk.storages.entry(address).or_default().storage.insert(slot, value); } @@ -1010,80 +898,6 @@ mod tests { core::iter::once((U256::from(1), StorageSlot::new_changed(original, present))).collect() } - #[test] - fn hashed_state_wiped_extension() { - let hashed_address = B256::default(); - let hashed_slot = B256::with_last_byte(64); - let hashed_slot2 = B256::with_last_byte(65); - - // Initialize post state storage - let original_slot_value = U256::from(123); - let mut hashed_state = HashedPostState::default().with_storages([( - hashed_address, - HashedStorage::from_iter([ - (hashed_slot, original_slot_value), - (hashed_slot2, original_slot_value), - ]), - )]); - - // Update single slot value - let updated_slot_value = U256::from(321); - let extension = HashedPostState::default().with_storages([( - hashed_address, - HashedStorage::from_iter([(hashed_slot, updated_slot_value)]), - )]); - hashed_state.extend(extension); - - let account_storage = hashed_state.storages.get(&hashed_address); - assert_eq!( - account_storage.and_then(|st| st.storage.get(&hashed_slot)), - Some(&updated_slot_value) - ); - assert_eq!( - account_storage.and_then(|st| st.storage.get(&hashed_slot2)), - Some(&original_slot_value) - ); - assert_eq!(account_storage.map(|st| st.wiped), Some(false)); - - // Wipe account storage - let wiped_extension = - HashedPostState::default().with_storages([(hashed_address, HashedStorage::new(true))]); - hashed_state.extend(wiped_extension); - - let account_storage = hashed_state.storages.get(&hashed_address); - assert_eq!(account_storage.map(|st| st.storage.is_empty()), Some(true)); - assert_eq!(account_storage.map(|st| st.wiped), Some(true)); - - // Reinitialize single slot value - hashed_state.extend(HashedPostState::default().with_storages([( - hashed_address, - HashedStorage::from_iter([(hashed_slot, original_slot_value)]), - )])); - let account_storage = hashed_state.storages.get(&hashed_address); - assert_eq!( - account_storage.and_then(|st| st.storage.get(&hashed_slot)), - Some(&original_slot_value) - ); - assert_eq!(account_storage.and_then(|st| st.storage.get(&hashed_slot2)), None); - assert_eq!(account_storage.map(|st| st.wiped), Some(true)); - - // Reinitialize single slot value - hashed_state.extend(HashedPostState::default().with_storages([( - hashed_address, - HashedStorage::from_iter([(hashed_slot2, updated_slot_value)]), - )])); - let account_storage = hashed_state.storages.get(&hashed_address); - assert_eq!( - account_storage.and_then(|st| st.storage.get(&hashed_slot)), - Some(&original_slot_value) - ); - assert_eq!( - account_storage.and_then(|st| st.storage.get(&hashed_slot2)), - Some(&updated_slot_value) - ); - assert_eq!(account_storage.map(|st| st.wiped), Some(true)); - } - #[test] fn test_hashed_post_state_from_bundle_state() { // Prepare a random Ethereum address as a key for the account. @@ -1143,7 +957,7 @@ mod tests { } #[test] - fn destroyed_accounts_emit_zero_storage_changes_without_wipe() { + fn destroyed_accounts_emit_zero_storage_changes() { let existing_contract = AccountInfo { code_hash: B256::repeat_byte(0x01), ..Default::default() }; let legacy_empty_account = AccountInfo::default(); @@ -1161,13 +975,12 @@ mod tests { let storage = bundle_hashed_storage(&account).unwrap(); let hashed_slot = keccak256(B256::from(U256::from(1))); - assert!(!storage.wiped); assert_eq!(storage.storage[&hashed_slot], U256::ZERO); } } #[test] - fn destroyed_recreated_accounts_preserve_storage_without_wipe() { + fn destroyed_recreated_accounts_preserve_storage() { let value = U256::from(2); let new_account = BundleAccount::new( None, @@ -1187,9 +1000,6 @@ mod tests { let new_storage = bundle_hashed_storage(&new_account).unwrap(); let existing_storage = bundle_hashed_storage(&existing_account).unwrap(); let hashed_slot = keccak256(B256::from(U256::from(1))); - - assert!(!new_storage.wiped); - assert!(!existing_storage.wiped); assert_eq!(new_storage.storage[&hashed_slot], value); assert_eq!(existing_storage.storage[&hashed_slot], value); } @@ -1227,8 +1037,8 @@ mod tests { let address_1 = Address::random(); let address_2 = Address::random(); - let storage_1 = (keccak256(address_1), HashedStorage::new(false)); - let storage_2 = (keccak256(address_2), HashedStorage::new(true)); + let storage_1 = (keccak256(address_1), HashedStorage::default()); + let storage_2 = (keccak256(address_2), HashedStorage::default()); // Add storages to the hashed post state. let hashed_state = HashedPostState::default().with_storages(vec![storage_1, storage_2]); @@ -1426,7 +1236,6 @@ mod tests { storages: B256Map::from_iter([( addr1, HashedStorage { - wiped: true, storage: B256Map::from_iter([(slot1, U256::ZERO), (slot2, U256::from(1))]), }, )]), @@ -1442,10 +1251,7 @@ mod tests { accounts: B256Map::from_iter([(addr1, Some(Default::default()))]), storages: B256Map::from_iter([( addr1, - HashedStorage { - wiped: true, - storage: B256Map::from_iter([(slot1, U256::ZERO)]) - } + HashedStorage { storage: B256Map::from_iter([(slot1, U256::ZERO)]) } )]), } ); @@ -1455,112 +1261,12 @@ mod tests { accounts: B256Map::from_iter([(addr2, Some(Default::default()))]), storages: B256Map::from_iter([( addr1, - HashedStorage { - wiped: false, - storage: B256Map::from_iter([(slot2, U256::from(1))]) - } + HashedStorage { storage: B256Map::from_iter([(slot2, U256::from(1))]) } )]), } ); } - #[test] - fn test_chunks() { - let addr1 = B256::from([1; 32]); - let addr2 = B256::from([2; 32]); - let slot1 = B256::from([1; 32]); - let slot2 = B256::from([2; 32]); - - let state = HashedPostState { - accounts: B256Map::from_iter([ - (addr1, Some(Default::default())), - (addr2, Some(Default::default())), - ]), - storages: B256Map::from_iter([( - addr2, - HashedStorage { - wiped: true, - storage: B256Map::from_iter([(slot1, U256::ZERO), (slot2, U256::from(1))]), - }, - )]), - }; - - let mut chunks = state.chunks(2); - assert_eq!( - chunks.next(), - Some(HashedPostState { - accounts: B256Map::from_iter([(addr1, Some(Default::default()))]), - storages: B256Map::from_iter([(addr2, HashedStorage::new(true)),]) - }) - ); - assert_eq!( - chunks.next(), - Some(HashedPostState { - accounts: B256Map::default(), - storages: B256Map::from_iter([( - addr2, - HashedStorage { - wiped: false, - storage: B256Map::from_iter([(slot1, U256::ZERO), (slot2, U256::from(1))]), - }, - )]) - }) - ); - assert_eq!( - chunks.next(), - Some(HashedPostState { - accounts: B256Map::from_iter([(addr2, Some(Default::default()))]), - storages: B256Map::default() - }) - ); - assert_eq!(chunks.next(), None); - } - - #[test] - fn test_chunks_ordering_guarantee() { - // Test that chunks preserve the ordering: wipe -> storage updates -> account - // Use chunk size of 1 to verify each item comes out in the correct order - let addr = B256::from([1; 32]); - let slot1 = B256::from([1; 32]); - let slot2 = B256::from([2; 32]); - - let state = HashedPostState { - accounts: B256Map::from_iter([(addr, Some(Default::default()))]), - storages: B256Map::from_iter([( - addr, - HashedStorage { - wiped: true, - storage: B256Map::from_iter([(slot1, U256::from(1)), (slot2, U256::from(2))]), - }, - )]), - }; - - let chunks: Vec<_> = state.chunks(1).collect(); - - // Should have 4 chunks: 1 wipe + 2 storage updates + 1 account - assert_eq!(chunks.len(), 4); - - // First chunk must be the storage wipe - assert!(chunks[0].accounts.is_empty()); - assert_eq!(chunks[0].storages.len(), 1); - assert!(chunks[0].storages.get(&addr).unwrap().wiped); - assert!(chunks[0].storages.get(&addr).unwrap().storage.is_empty()); - - // Next two chunks must be storage updates (order between them doesn't matter) - assert!(chunks[1].accounts.is_empty()); - assert!(!chunks[1].storages.get(&addr).unwrap().wiped); - assert_eq!(chunks[1].storages.get(&addr).unwrap().storage.len(), 1); - - assert!(chunks[2].accounts.is_empty()); - assert!(!chunks[2].storages.get(&addr).unwrap().wiped); - assert_eq!(chunks[2].storages.get(&addr).unwrap().storage.len(), 1); - - // Last chunk must be the account update - assert_eq!(chunks[3].accounts.len(), 1); - assert!(chunks[3].accounts.contains_key(&addr)); - assert!(chunks[3].storages.is_empty()); - } - #[test] fn test_hashed_post_state_sorted_extend_ref() { // Test extending accounts @@ -1607,7 +1313,6 @@ mod tests { (B256::from([3; 32]), U256::from(30)), (B256::from([5; 32]), U256::ZERO), ], - wiped: false, }; let storage2 = HashedStorageSorted { @@ -1617,7 +1322,6 @@ mod tests { (B256::from([4; 32]), U256::from(40)), (B256::from([6; 32]), U256::ZERO), ], - wiped: false, }; storage1.extend_ref(&storage2); @@ -1635,34 +1339,6 @@ mod tests { assert_eq!(storage1.storage_slots[4].1, U256::ZERO); assert_eq!(storage1.storage_slots[5].0, B256::from([6; 32])); assert_eq!(storage1.storage_slots[5].1, U256::ZERO); - assert!(!storage1.wiped); - - // Test wiped storage - let mut storage3 = HashedStorageSorted { - storage_slots: vec![ - (B256::from([1; 32]), U256::from(10)), - (B256::from([2; 32]), U256::ZERO), - ], - wiped: false, - }; - - let storage4 = HashedStorageSorted { - storage_slots: vec![ - (B256::from([3; 32]), U256::from(30)), - (B256::from([4; 32]), U256::ZERO), - ], - wiped: true, - }; - - storage3.extend_ref(&storage4); - - assert!(storage3.wiped); - // When wiped, should only have storage4's values - assert_eq!(storage3.storage_slots.len(), 2); - assert_eq!(storage3.storage_slots[0].0, B256::from([3; 32])); - assert_eq!(storage3.storage_slots[0].1, U256::from(30)); - assert_eq!(storage3.storage_slots[1].0, B256::from([4; 32])); - assert_eq!(storage3.storage_slots[1].1, U256::ZERO); } /// Test extending with sorted accounts merges correctly into `HashMap` @@ -1716,7 +1392,7 @@ mod tests { vec![(kept_account, Some(account(1))), (removed_account, Some(account(10)))], B256Map::from_iter([( kept_storage, - HashedStorageSorted { wiped: false, storage_slots: vec![(slot1, U256::from(1))] }, + HashedStorageSorted { storage_slots: vec![(slot1, U256::from(1))] }, )]), ); @@ -1725,7 +1401,6 @@ mod tests { B256Map::from_iter([( kept_storage, HashedStorageSorted { - wiped: false, storage_slots: vec![(slot1, U256::from(3)), (slot2, U256::from(4))], }, )]), @@ -1735,7 +1410,7 @@ mod tests { vec![(removed_account, None)], B256Map::from_iter([( kept_storage, - HashedStorageSorted { wiped: false, storage_slots: vec![(slot2, U256::ZERO)] }, + HashedStorageSorted { storage_slots: vec![(slot2, U256::ZERO)] }, )]), ); @@ -1753,10 +1428,7 @@ mod tests { assert_eq!(result.storages.len(), 1); assert_eq!( result.storages.get(&kept_storage), - Some(&HashedStorageSorted { - wiped: false, - storage_slots: vec![(slot1, U256::from(3))], - }) + Some(&HashedStorageSorted { storage_slots: vec![(slot1, U256::from(3))] }) ); } @@ -1769,13 +1441,7 @@ mod tests { let older = HashedPostStateSorted::new( vec![(address, Some(Account { nonce: 1, ..Default::default() }))], B256Map::from_iter([ - ( - storage, - HashedStorageSorted { - wiped: false, - storage_slots: vec![(slot, U256::from(1))], - }, - ), + (storage, HashedStorageSorted { storage_slots: vec![(slot, U256::from(1))] }), (empty_storage, HashedStorageSorted::default()), ]), ); @@ -1783,7 +1449,7 @@ mod tests { vec![(address, Some(Account { nonce: 2, ..Default::default() }))], B256Map::from_iter([( storage, - HashedStorageSorted { wiped: false, storage_slots: vec![(slot, U256::from(2))] }, + HashedStorageSorted { storage_slots: vec![(slot, U256::from(2))] }, )]), ); let expected = HashedPostStateSorted::merge_batch(vec![newer.clone(), older.clone()]); @@ -1834,38 +1500,20 @@ mod tests { let batch = HashedPostStateSorted::new( vec![(address, Some(account(1))), (deleted_address, None)], B256Map::from_iter([ - ( - storage, - HashedStorageSorted { - wiped: false, - storage_slots: vec![(slot, U256::from(1))], - }, - ), + (storage, HashedStorageSorted { storage_slots: vec![(slot, U256::from(1))] }), ( deleted_storage, - HashedStorageSorted { - wiped: false, - storage_slots: vec![(deleted_slot, U256::ZERO)], - }, + HashedStorageSorted { storage_slots: vec![(deleted_slot, U256::ZERO)] }, ), ]), ); let different_mask = HashedPostStateSorted::new( vec![(address, Some(account(2))), (deleted_address, Some(account(3)))], B256Map::from_iter([ - ( - storage, - HashedStorageSorted { - wiped: false, - storage_slots: vec![(slot, U256::from(2))], - }, - ), + (storage, HashedStorageSorted { storage_slots: vec![(slot, U256::from(2))] }), ( deleted_storage, - HashedStorageSorted { - wiped: false, - storage_slots: vec![(deleted_slot, U256::from(3))], - }, + HashedStorageSorted { storage_slots: vec![(deleted_slot, U256::from(3))] }, ), ]), ); @@ -1893,28 +1541,25 @@ mod tests { vec![], B256Map::from_iter([( storage, - HashedStorageSorted { wiped: false, storage_slots: vec![(slot, U256::from(1))] }, + HashedStorageSorted { storage_slots: vec![(slot, U256::from(1))] }, )]), ); let mask = HashedPostStateSorted::new( vec![], - B256Map::from_iter([( - storage, - HashedStorageSorted { wiped: false, storage_slots: vec![] }, - )]), + B256Map::from_iter([(storage, HashedStorageSorted { storage_slots: vec![] })]), ); let result = HashedPostStateSorted::disjointed_merge_batch(&[&batch], &[&mask]); assert_eq!( result.storages.get(&storage), - Some(&HashedStorageSorted { wiped: false, storage_slots: vec![(slot, U256::from(1))] }) + Some(&HashedStorageSorted { storage_slots: vec![(slot, U256::from(1))] }) ); } - /// Test non-wiped storage merges both zero and non-zero valued slots + /// Test storage merges both zero and non-zero valued slots #[test] - fn test_hashed_storage_extend_from_sorted_non_wiped() { + fn test_hashed_storage_extend_from_sorted() { let slot1 = B256::random(); let slot2 = B256::random(); let slot3 = B256::random(); @@ -1923,37 +1568,15 @@ mod tests { let sorted = HashedStorageSorted { storage_slots: vec![(slot2, U256::from(200)), (slot3, U256::ZERO)], - wiped: false, }; storage.extend_from_sorted(&sorted); - - assert!(!storage.wiped); assert_eq!(storage.storage.len(), 3); assert_eq!(storage.storage.get(&slot1), Some(&U256::from(100))); assert_eq!(storage.storage.get(&slot2), Some(&U256::from(200))); assert_eq!(storage.storage.get(&slot3), Some(&U256::ZERO)); } - /// Test wiped=true clears existing storage and only keeps new slots (critical edge case) - #[test] - fn test_hashed_storage_extend_from_sorted_wiped() { - let slot1 = B256::random(); - let slot2 = B256::random(); - - let mut storage = HashedStorage::from_iter([(slot1, U256::from(100))]); - - let sorted = - HashedStorageSorted { storage_slots: vec![(slot2, U256::from(200))], wiped: true }; - - storage.extend_from_sorted(&sorted); - - assert!(storage.wiped); - // After wipe, old storage should be cleared and only new storage remains - assert_eq!(storage.storage.len(), 1); - assert_eq!(storage.storage.get(&slot2), Some(&U256::from(200))); - } - #[test] fn test_hashed_post_state_chunking_length() { let addr1 = B256::from([1; 32]); @@ -1970,7 +1593,6 @@ mod tests { ( addr1, HashedStorage { - wiped: false, storage: B256Map::from_iter([ (slot1, U256::ZERO), (slot2, U256::ZERO), @@ -1981,7 +1603,6 @@ mod tests { ( addr2, HashedStorage { - wiped: true, storage: B256Map::from_iter([ (slot1, U256::ZERO), (slot2, U256::ZERO), @@ -1992,7 +1613,6 @@ mod tests { ( addr3, HashedStorage { - wiped: false, storage: B256Map::from_iter([ (slot1, U256::ZERO), (slot2, U256::ZERO), @@ -2034,20 +1654,13 @@ mod tests { ( addr1, HashedStorage { - wiped: false, storage: B256Map::from_iter([ (slot1, U256::from(10)), (slot2, U256::from(20)), ]), }, ), - ( - addr2, - HashedStorage { - wiped: true, - storage: B256Map::from_iter([(slot3, U256::ZERO)]), - }, - ), + (addr2, HashedStorage { storage: B256Map::from_iter([(slot3, U256::ZERO)]) }), ]), }; @@ -2069,7 +1682,6 @@ mod tests { let slot3 = B256::from([3; 32]); let storage = HashedStorage { - wiped: true, storage: B256Map::from_iter([ (slot1, U256::from(100)), (slot2, U256::ZERO), @@ -2085,7 +1697,6 @@ mod tests { // Verify the original storage is not consumed assert_eq!(storage.storage.len(), 3); - assert!(storage.wiped); } } @@ -2093,9 +1704,13 @@ mod tests { #[cfg(feature = "serde-bincode-compat")] pub mod serde_bincode_compat { use super::Account; - use alloc::borrow::Cow; + use alloc::{borrow::Cow, vec::Vec}; use alloy_primitives::{map::B256Map, B256, U256}; - use serde::{Deserialize, Deserializer, Serialize, Serializer}; + use core::fmt; + use serde::{ + de::{Error as _, SeqAccess, Visitor}, + Deserialize, Deserializer, Serialize, Serializer, + }; use serde_with::{DeserializeAs, SerializeAs}; /// Bincode-compatible [`super::HashedPostState`] serde implementation. @@ -2175,19 +1790,18 @@ pub mod serde_bincode_compat { /// ``` #[derive(Debug, Serialize, Deserialize)] pub struct HashedStorage<'a> { - wiped: bool, storage: Cow<'a, B256Map>, } impl<'a> From<&'a super::HashedStorage> for HashedStorage<'a> { fn from(value: &'a super::HashedStorage) -> Self { - Self { wiped: value.wiped, storage: Cow::Borrowed(&value.storage) } + Self { storage: Cow::Borrowed(&value.storage) } } } impl<'a> From> for super::HashedStorage { fn from(value: HashedStorage<'a>) -> Self { - Self { wiped: value.wiped, storage: value.storage.into_owned() } + Self { storage: value.storage.into_owned() } } } @@ -2284,21 +1898,63 @@ pub mod serde_bincode_compat { /// hashed_storage: HashedStorageSorted, /// } /// ``` - #[derive(Debug, Serialize, Deserialize)] + #[derive(Debug, Serialize)] pub struct HashedStorageSorted<'a> { storage_slots: Cow<'a, [(B256, U256)]>, - wiped: bool, + } + + impl<'de, 'a> Deserialize<'de> for HashedStorageSorted<'a> { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + struct HashedStorageSortedVisitor; + + impl<'de> Visitor<'de> for HashedStorageSortedVisitor { + type Value = Vec<(B256, U256)>; + + fn expecting(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str("hashed storage with an optional legacy wipe marker") + } + + fn visit_seq(self, mut seq: A) -> Result + where + A: SeqAccess<'de>, + { + let len = seq.size_hint().unwrap_or_default(); + if !matches!(len, 1 | 2) { + return Err(A::Error::invalid_length(len, &self)) + } + + let storage_slots = + seq.next_element()?.ok_or_else(|| A::Error::invalid_length(0, &self))?; + if len == 2 { + let _: bool = seq + .next_element()? + .ok_or_else(|| A::Error::invalid_length(1, &self))?; + } + + Ok(storage_slots) + } + } + + // Bincode uses the supplied tuple length for the current format, while MessagePack + // exposes the encoded sequence length so legacy ExEx WAL entries can still be read. + deserializer + .deserialize_tuple(1, HashedStorageSortedVisitor) + .map(|storage_slots| Self { storage_slots: Cow::Owned(storage_slots) }) + } } impl<'a> From<&'a super::HashedStorageSorted> for HashedStorageSorted<'a> { fn from(value: &'a super::HashedStorageSorted) -> Self { - Self { storage_slots: Cow::Borrowed(&value.storage_slots), wiped: value.wiped } + Self { storage_slots: Cow::Borrowed(&value.storage_slots) } } } impl<'a> From> for super::HashedStorageSorted { fn from(value: HashedStorageSorted<'a>) -> Self { - Self { storage_slots: value.storage_slots.into_owned(), wiped: value.wiped } + Self { storage_slots: value.storage_slots.into_owned() } } } @@ -2375,11 +2031,6 @@ pub mod serde_bincode_compat { let decoded: Data = bincode::deserialize(&encoded).unwrap(); assert_eq!(decoded, data); - data.hashed_storage.wiped = true; - let encoded = bincode::serialize(&data).unwrap(); - let decoded: Data = bincode::deserialize(&encoded).unwrap(); - assert_eq!(decoded, data); - data.hashed_storage.storage.insert(B256::random(), U256::from(1)); let encoded = bincode::serialize(&data).unwrap(); let decoded: Data = bincode::deserialize(&encoded).unwrap(); @@ -2410,10 +2061,7 @@ pub mod serde_bincode_compat { data.hashed_state.storages.insert( B256::random(), - HashedStorageSorted { - storage_slots: vec![(B256::from([1; 32]), U256::from(10))], - wiped: false, - }, + HashedStorageSorted { storage_slots: vec![(B256::from([1; 32]), U256::from(10))] }, ); let encoded = bincode::serialize(&data).unwrap(); let decoded: Data = bincode::deserialize(&encoded).unwrap(); @@ -2429,14 +2077,8 @@ pub mod serde_bincode_compat { hashed_storage: HashedStorageSorted, } - let mut data = Data { - hashed_storage: HashedStorageSorted { storage_slots: Vec::new(), wiped: false }, - }; - let encoded = bincode::serialize(&data).unwrap(); - let decoded: Data = bincode::deserialize(&encoded).unwrap(); - assert_eq!(decoded, data); - - data.hashed_storage.wiped = true; + let mut data = + Data { hashed_storage: HashedStorageSorted { storage_slots: Vec::new() } }; let encoded = bincode::serialize(&data).unwrap(); let decoded: Data = bincode::deserialize(&encoded).unwrap(); assert_eq!(decoded, data); diff --git a/crates/trie/common/src/updates.rs b/crates/trie/common/src/updates.rs index f906798b059..6a905c6e1ee 100644 --- a/crates/trie/common/src/updates.rs +++ b/crates/trie/common/src/updates.rs @@ -234,8 +234,6 @@ impl TrieUpdates { #[derive(PartialEq, Eq, Clone, Default, Debug)] #[cfg_attr(any(test, feature = "serde"), derive(serde::Serialize, serde::Deserialize))] pub struct StorageTrieUpdates { - /// Flag indicating whether the trie was deleted. - pub is_deleted: bool, /// Collection of updated storage trie nodes. #[cfg_attr(any(test, feature = "serde"), serde(with = "serde_nibbles_map"))] pub storage_nodes: HashMap, @@ -246,30 +244,16 @@ pub struct StorageTrieUpdates { #[cfg(feature = "test-utils")] impl StorageTrieUpdates { - /// Creates a new storage trie updates that are not marked as deleted. + /// Creates storage trie updates from the provided nodes. pub fn new(updates: impl IntoIterator) -> Self { Self { storage_nodes: exclude_empty_from_pair(updates).collect(), ..Default::default() } } } impl StorageTrieUpdates { - /// Returns empty storage trie updates with `deleted` set to `true`. - pub fn deleted() -> Self { - Self { - is_deleted: true, - storage_nodes: HashMap::default(), - removed_nodes: HashSet::default(), - } - } - /// Returns the length of updated nodes. pub fn len(&self) -> usize { - (self.is_deleted as usize) + self.storage_nodes.len() + self.removed_nodes.len() - } - - /// Returns `true` if the trie was deleted. - pub const fn is_deleted(&self) -> bool { - self.is_deleted + self.storage_nodes.len() + self.removed_nodes.len() } /// Returns reference to updated storage nodes. @@ -284,12 +268,7 @@ impl StorageTrieUpdates { /// Returns `true` if storage updates are empty. pub fn is_empty(&self) -> bool { - !self.is_deleted && self.storage_nodes.is_empty() && self.removed_nodes.is_empty() - } - - /// Sets `deleted` flag on the storage trie. - pub const fn set_deleted(&mut self, deleted: bool) { - self.is_deleted = deleted; + self.storage_nodes.is_empty() && self.removed_nodes.is_empty() } /// Extends storage trie updates. @@ -311,11 +290,6 @@ impl StorageTrieUpdates { } fn extend_common(&mut self, other: &Self) { - if other.is_deleted { - self.storage_nodes.clear(); - self.removed_nodes.clear(); - } - self.is_deleted |= other.is_deleted; self.storage_nodes.retain(|nibbles, _| !other.removed_nodes.contains(nibbles)); } @@ -326,12 +300,6 @@ impl StorageTrieUpdates { /// /// This is invoked from [`TrieUpdates::extend_from_sorted`] for each account. pub fn extend_from_sorted(&mut self, sorted: &StorageTrieUpdatesSorted) { - if sorted.is_deleted { - self.storage_nodes.clear(); - self.removed_nodes.clear(); - } - self.is_deleted |= sorted.is_deleted; - // Reserve capacity for storage nodes let new_nodes_count = sorted.storage_nodes.len(); self.storage_nodes.reserve(new_nodes_count); @@ -376,7 +344,7 @@ impl StorageTrieUpdates { storage_nodes.extend(self.removed_nodes.into_iter().map(|path| (path, None))); storage_nodes.sort_unstable_by_key(|a| a.0); - StorageTrieUpdatesSorted { is_deleted: self.is_deleted, storage_nodes } + StorageTrieUpdatesSorted { storage_nodes } } /// Creates a sorted copy without consuming self. @@ -397,13 +365,12 @@ impl StorageTrieUpdates { ); storage_nodes.sort_unstable_by_key(|a| a.0); - StorageTrieUpdatesSorted { is_deleted: self.is_deleted, storage_nodes } + StorageTrieUpdatesSorted { storage_nodes } } /// Convert storage trie updates into [`StorageTrieUpdatesSortedRef`]. pub fn into_sorted_ref(&self) -> StorageTrieUpdatesSortedRef<'_> { StorageTrieUpdatesSortedRef { - is_deleted: self.is_deleted, removed_nodes: self.removed_nodes.iter().collect::>(), storage_nodes: self.storage_nodes.iter().collect::>(), } @@ -673,8 +640,6 @@ impl TrieUpdatesSorted { kway_merge_sorted(items.iter().map(|i| i.as_ref().account_nodes.as_slice())); struct StorageAcc<'a> { - is_deleted: bool, - sealed: bool, slices: Vec<&'a [(Nibbles, Option)]>, } @@ -682,22 +647,8 @@ impl TrieUpdatesSorted { for item in items { for (addr, storage) in &item.as_ref().storage_tries { - let entry = acc.entry(*addr).or_insert_with(|| StorageAcc { - is_deleted: false, - sealed: false, - slices: Vec::new(), - }); - - if entry.sealed { - continue; - } - + let entry = acc.entry(*addr).or_insert_with(|| StorageAcc { slices: Vec::new() }); entry.slices.push(storage.storage_nodes.as_slice()); - - if storage.is_deleted { - entry.is_deleted = true; - entry.sealed = true; - } } } @@ -705,7 +656,7 @@ impl TrieUpdatesSorted { .into_iter() .map(|(addr, entry)| { let storage_nodes = kway_merge_sorted(entry.slices); - (addr, StorageTrieUpdatesSorted { is_deleted: entry.is_deleted, storage_nodes }) + (addr, StorageTrieUpdatesSorted { storage_nodes }) }) .collect(); @@ -719,10 +670,6 @@ impl TrieUpdatesSorted { /// node level. For duplicate keys in the batch, later items take precedence over earlier ones. /// An overlapping entry is retained if any mask value is equal to the merged batch value. The /// order of the mask does not matter. An empty mask merges the batch without filtering. - /// - /// # Panics - /// - /// Panics if any batch or mask entry deletes an entire storage trie. pub fn disjointed_merge_batch<'a>(batch: &[&'a Self], mask: &[&'a Self]) -> Self { let account_node_count = batch.iter().map(|item| item.account_nodes.len()).sum(); let mut account_nodes = Vec::with_capacity(account_node_count); @@ -748,10 +695,6 @@ impl TrieUpdatesSorted { for item in batch.iter().rev() { for (hashed_address, storage_trie) in &item.storage_tries { - assert!( - !storage_trie.is_deleted, - "storage wipes are not supported by disjointed_merge_batch" - ); let entry = storage_tries .entry(*hashed_address) .or_insert_with(|| StorageAcc { node_count: 0, slices: Vec::new() }); @@ -766,10 +709,6 @@ impl TrieUpdatesSorted { ); for item in mask { for (hashed_address, storage_trie) in &item.storage_tries { - assert!( - !storage_trie.is_deleted, - "storage wipes are not supported by disjointed_merge_batch" - ); let entry = storage_masks.entry(*hashed_address).or_default(); entry.slices.push(storage_trie.storage_nodes.as_slice()); } @@ -791,10 +730,8 @@ impl TrieUpdatesSorted { None => kway_merge_sorted(entry.slices), }; - (!storage_nodes.is_empty() || mask.is_empty()).then_some(( - hashed_address, - StorageTrieUpdatesSorted { is_deleted: false, storage_nodes }, - )) + (!storage_nodes.is_empty() || mask.is_empty()) + .then_some((hashed_address, StorageTrieUpdatesSorted { storage_nodes })) }) .collect(); @@ -835,8 +772,6 @@ impl From for TrieUpdates { #[derive(PartialEq, Eq, Clone, Default, Debug)] #[cfg_attr(any(test, feature = "serde"), derive(serde::Serialize))] pub struct StorageTrieUpdatesSortedRef<'a> { - /// Flag indicating whether the trie has been deleted/wiped. - pub is_deleted: bool, /// Sorted collection of updated storage nodes with corresponding paths. pub storage_nodes: BTreeMap<&'a Nibbles, &'a BranchNodeCompact>, /// The set of removed storage node keys. @@ -847,19 +782,12 @@ pub struct StorageTrieUpdatesSortedRef<'a> { #[derive(PartialEq, Eq, Clone, Default, Debug)] #[cfg_attr(any(test, feature = "serde"), derive(serde::Serialize, serde::Deserialize))] pub struct StorageTrieUpdatesSorted { - /// Flag indicating whether the trie has been deleted/wiped. - pub is_deleted: bool, /// Sorted collection of updated storage nodes with corresponding paths. None indicates a node /// is removed. pub storage_nodes: Vec<(Nibbles, Option)>, } impl StorageTrieUpdatesSorted { - /// Returns `true` if the trie was deleted. - pub const fn is_deleted(&self) -> bool { - self.is_deleted - } - /// Returns reference to updated storage nodes. pub fn storage_nodes_ref(&self) -> &[(Nibbles, Option)] { &self.storage_nodes @@ -876,36 +804,16 @@ impl StorageTrieUpdatesSorted { } /// Extends the storage trie updates with another set of sorted updates. - /// - /// If `other` is marked as deleted, this will be marked as deleted and all nodes cleared. - /// Otherwise, nodes are merged with `other`'s values taking precedence for duplicates. pub fn extend_ref(&mut self, other: &Self) { - if other.is_deleted { - self.is_deleted = true; - self.storage_nodes.clear(); - self.storage_nodes.extend(other.storage_nodes.iter().cloned()); - return; - } - - // Extend storage nodes extend_sorted_vec(&mut self.storage_nodes, &other.storage_nodes); - self.is_deleted = self.is_deleted || other.is_deleted; } /// Batch-merge sorted storage trie updates. Iterator yields **newest to oldest**. - /// If any update is deleted, older data is discarded. pub fn merge_batch<'a>(updates: impl IntoIterator) -> Self { let updates: Vec<_> = updates.into_iter().collect(); - if updates.is_empty() { - return Self::default(); + Self { + storage_nodes: kway_merge_sorted(updates.iter().map(|u| u.storage_nodes.as_slice())), } - - // Discard updates older than the first deletion since the trie was wiped at that point. - let del_idx = updates.iter().position(|u| u.is_deleted); - let relevant = del_idx.map_or(&updates[..], |idx| &updates[..=idx]); - let storage_nodes = kway_merge_sorted(relevant.iter().map(|u| u.storage_nodes.as_slice())); - - Self { is_deleted: del_idx.is_some(), storage_nodes } } } @@ -934,7 +842,7 @@ impl From for StorageTrieUpdates { } } - Self { is_deleted: sorted.is_deleted, storage_nodes, removed_nodes } + Self { storage_nodes, removed_nodes } } } @@ -1006,14 +914,12 @@ mod tests { // Test extending storage tries let storage_trie1 = StorageTrieUpdatesSorted { - is_deleted: false, storage_nodes: vec![( Nibbles::from_nibbles_unchecked([0x0a]), Some(BranchNodeCompact::default()), )], }; let storage_trie2 = StorageTrieUpdatesSorted { - is_deleted: false, storage_nodes: vec![(Nibbles::from_nibbles_unchecked([0x0b]), None)], }; @@ -1040,61 +946,6 @@ mod tests { assert_eq!(merged_storage.storage_nodes.len(), 2); } - #[test] - fn test_storage_trie_updates_sorted_extend_ref_deleted() { - // Test case 1: Extending with a deleted storage trie that has nodes - let mut storage1 = StorageTrieUpdatesSorted { - is_deleted: false, - storage_nodes: vec![ - (Nibbles::from_nibbles_unchecked([0x01]), Some(BranchNodeCompact::default())), - (Nibbles::from_nibbles_unchecked([0x02]), None), - ], - }; - - let storage2 = StorageTrieUpdatesSorted { - is_deleted: true, - storage_nodes: vec![ - (Nibbles::from_nibbles_unchecked([0x03]), Some(BranchNodeCompact::default())), - (Nibbles::from_nibbles_unchecked([0x04]), None), - ], - }; - - storage1.extend_ref(&storage2); - - // Should be marked as deleted - assert!(storage1.is_deleted); - // Original nodes should be cleared, but other's nodes should be added - assert_eq!(storage1.storage_nodes.len(), 2); - assert_eq!(storage1.storage_nodes[0].0, Nibbles::from_nibbles_unchecked([0x03])); - assert_eq!(storage1.storage_nodes[1].0, Nibbles::from_nibbles_unchecked([0x04])); - - // Test case 2: Extending a deleted storage trie with more nodes - let mut storage3 = StorageTrieUpdatesSorted { - is_deleted: true, - storage_nodes: vec![( - Nibbles::from_nibbles_unchecked([0x05]), - Some(BranchNodeCompact::default()), - )], - }; - - let storage4 = StorageTrieUpdatesSorted { - is_deleted: true, - storage_nodes: vec![ - (Nibbles::from_nibbles_unchecked([0x06]), Some(BranchNodeCompact::default())), - (Nibbles::from_nibbles_unchecked([0x07]), None), - ], - }; - - storage3.extend_ref(&storage4); - - // Should remain deleted - assert!(storage3.is_deleted); - // Should have nodes from other (original cleared then extended) - assert_eq!(storage3.storage_nodes.len(), 2); - assert_eq!(storage3.storage_nodes[0].0, Nibbles::from_nibbles_unchecked([0x06])); - assert_eq!(storage3.storage_nodes[1].0, Nibbles::from_nibbles_unchecked([0x07])); - } - #[test] fn test_trie_updates_sorted_disjointed_merge_batch() { let kept_node = Nibbles::from_nibbles_unchecked([0x01]); @@ -1107,7 +958,7 @@ mod tests { vec![(kept_node, Some(BranchNodeCompact::default())), (removed_node, None)], B256Map::from_iter([( kept_storage, - StorageTrieUpdatesSorted { is_deleted: false, storage_nodes: vec![(slot1, None)] }, + StorageTrieUpdatesSorted { storage_nodes: vec![(slot1, None)] }, )]), ); @@ -1116,7 +967,6 @@ mod tests { B256Map::from_iter([( kept_storage, StorageTrieUpdatesSorted { - is_deleted: false, storage_nodes: vec![(slot1, Some(BranchNodeCompact::default())), (slot2, None)], }, )]), @@ -1127,7 +977,6 @@ mod tests { B256Map::from_iter([( kept_storage, StorageTrieUpdatesSorted { - is_deleted: false, storage_nodes: vec![(slot2, Some(BranchNodeCompact::default()))], }, )]), @@ -1146,7 +995,6 @@ mod tests { assert_eq!( result.storage_tries.get(&kept_storage), Some(&StorageTrieUpdatesSorted { - is_deleted: false, storage_nodes: vec![(slot1, Some(BranchNodeCompact::default()))], }) ); @@ -1161,13 +1009,7 @@ mod tests { let older = TrieUpdatesSorted::new( vec![(node, Some(BranchNodeCompact::default()))], B256Map::from_iter([ - ( - storage, - StorageTrieUpdatesSorted { - is_deleted: false, - storage_nodes: vec![(storage_node, None)], - }, - ), + (storage, StorageTrieUpdatesSorted { storage_nodes: vec![(storage_node, None)] }), (empty_storage, StorageTrieUpdatesSorted::default()), ]), ); @@ -1176,7 +1018,6 @@ mod tests { B256Map::from_iter([( storage, StorageTrieUpdatesSorted { - is_deleted: false, storage_nodes: vec![(storage_node, Some(BranchNodeCompact::default()))], }, )]), @@ -1227,14 +1068,12 @@ mod tests { ( storage, StorageTrieUpdatesSorted { - is_deleted: false, storage_nodes: vec![(storage_node_path, Some(branch(0b0011_1100)))], }, ), ( deleted_storage, StorageTrieUpdatesSorted { - is_deleted: false, storage_nodes: vec![(deleted_storage_node_path, None)], }, ), @@ -1249,14 +1088,12 @@ mod tests { ( storage, StorageTrieUpdatesSorted { - is_deleted: false, storage_nodes: vec![(storage_node_path, Some(branch(0b1100_0011)))], }, ), ( deleted_storage, StorageTrieUpdatesSorted { - is_deleted: false, storage_nodes: vec![(deleted_storage_node_path, Some(branch(0b0000_1111)))], }, ), @@ -1290,7 +1127,6 @@ mod tests { B256Map::from_iter([( hashed_address, StorageTrieUpdatesSorted { - is_deleted: false, storage_nodes: vec![ (grandparent, Some(BranchNodeCompact::default())), (parent, Some(BranchNodeCompact::default())), @@ -1307,7 +1143,6 @@ mod tests { B256Map::from_iter([( hashed_address, StorageTrieUpdatesSorted { - is_deleted: false, storage_nodes: vec![ (grandparent, Some(different_node.clone())), (parent, Some(different_node)), @@ -1322,7 +1157,6 @@ mod tests { assert_eq!( result.storage_tries.get(&hashed_address), Some(&StorageTrieUpdatesSorted { - is_deleted: false, storage_nodes: vec![(child, Some(BranchNodeCompact::default()))], }) ); @@ -1338,17 +1172,13 @@ mod tests { B256Map::from_iter([( storage, StorageTrieUpdatesSorted { - is_deleted: false, storage_nodes: vec![(slot, Some(BranchNodeCompact::default()))], }, )]), ); let mask = TrieUpdatesSorted::new( vec![], - B256Map::from_iter([( - storage, - StorageTrieUpdatesSorted { is_deleted: false, storage_nodes: vec![] }, - )]), + B256Map::from_iter([(storage, StorageTrieUpdatesSorted { storage_nodes: vec![] })]), ); let result = TrieUpdatesSorted::disjointed_merge_batch(&[&batch], &[&mask]); @@ -1356,7 +1186,6 @@ mod tests { assert_eq!( result.storage_tries.get(&storage), Some(&StorageTrieUpdatesSorted { - is_deleted: false, storage_nodes: vec![(slot, Some(BranchNodeCompact::default()))], }) ); @@ -1370,7 +1199,6 @@ mod tests { let mut updates = TrieUpdates::default(); let storage_trie = StorageTrieUpdatesSorted { - is_deleted: false, storage_nodes: vec![ (Nibbles::from_nibbles_unchecked([0x0a]), Some(BranchNodeCompact::default())), (Nibbles::from_nibbles_unchecked([0x0b]), None), @@ -1386,56 +1214,14 @@ mod tests { assert_eq!(updates.storage_tries.len(), 1); let storage = updates.storage_tries.get(&hashed_address).unwrap(); - assert!(!storage.is_deleted); assert_eq!(storage.storage_nodes.len(), 1); assert!(storage.removed_nodes.contains(&Nibbles::from_nibbles_unchecked([0x0b]))); } - /// Test deleted=true clears old storage nodes before adding new ones (critical edge case) - #[test] - fn test_trie_updates_extend_from_sorted_with_deleted_storage() { - let hashed_address = B256::from([1; 32]); - - let mut updates = TrieUpdates::default(); - updates.storage_tries.insert( - hashed_address, - StorageTrieUpdates { - is_deleted: false, - storage_nodes: HashMap::from_iter([( - Nibbles::from_nibbles_unchecked([0x01]), - BranchNodeCompact::default(), - )]), - removed_nodes: Default::default(), - }, - ); - - let storage_trie = StorageTrieUpdatesSorted { - is_deleted: true, - storage_nodes: vec![( - Nibbles::from_nibbles_unchecked([0x0a]), - Some(BranchNodeCompact::default()), - )], - }; - - let sorted = TrieUpdatesSorted { - account_nodes: vec![], - storage_tries: B256Map::from_iter([(hashed_address, storage_trie)]), - }; - - updates.extend_from_sorted(&sorted); - - let storage = updates.storage_tries.get(&hashed_address).unwrap(); - assert!(storage.is_deleted); - // After deletion, old nodes should be cleared - assert_eq!(storage.storage_nodes.len(), 1); - assert!(storage.storage_nodes.contains_key(&Nibbles::from_nibbles_unchecked([0x0a]))); - } - - /// Test non-deleted storage merges nodes and tracks removed nodes + /// Test storage merges nodes and tracks removed nodes #[test] fn test_storage_trie_updates_extend_from_sorted_non_deleted() { let mut storage = StorageTrieUpdates { - is_deleted: false, storage_nodes: HashMap::from_iter([( Nibbles::from_nibbles_unchecked([0x01]), BranchNodeCompact::default(), @@ -1444,7 +1230,6 @@ mod tests { }; let sorted = StorageTrieUpdatesSorted { - is_deleted: false, storage_nodes: vec![ (Nibbles::from_nibbles_unchecked([0x02]), Some(BranchNodeCompact::default())), (Nibbles::from_nibbles_unchecked([0x03]), None), @@ -1452,40 +1237,10 @@ mod tests { }; storage.extend_from_sorted(&sorted); - - assert!(!storage.is_deleted); assert_eq!(storage.storage_nodes.len(), 2); assert!(storage.removed_nodes.contains(&Nibbles::from_nibbles_unchecked([0x03]))); } - /// Test deleted=true clears old nodes before extending (edge case) - #[test] - fn test_storage_trie_updates_extend_from_sorted_deleted() { - let mut storage = StorageTrieUpdates { - is_deleted: false, - storage_nodes: HashMap::from_iter([( - Nibbles::from_nibbles_unchecked([0x01]), - BranchNodeCompact::default(), - )]), - removed_nodes: Default::default(), - }; - - let sorted = StorageTrieUpdatesSorted { - is_deleted: true, - storage_nodes: vec![( - Nibbles::from_nibbles_unchecked([0x0a]), - Some(BranchNodeCompact::default()), - )], - }; - - storage.extend_from_sorted(&sorted); - - assert!(storage.is_deleted); - // Old nodes should be cleared when deleted - assert_eq!(storage.storage_nodes.len(), 1); - assert!(storage.storage_nodes.contains_key(&Nibbles::from_nibbles_unchecked([0x0a]))); - } - /// Test empty nibbles are filtered out during conversion (edge case bug) #[test] fn test_trie_updates_extend_from_sorted_filters_empty_nibbles() { @@ -1512,9 +1267,13 @@ mod tests { #[cfg(feature = "serde-bincode-compat")] pub mod serde_bincode_compat { use crate::{BranchNodeCompact, Nibbles}; - use alloc::borrow::Cow; + use alloc::{borrow::Cow, vec::Vec}; use alloy_primitives::map::{B256Map, HashMap, HashSet}; - use serde::{Deserialize, Deserializer, Serialize, Serializer}; + use core::fmt; + use serde::{ + de::{Error as _, SeqAccess, Visitor}, + Deserialize, Deserializer, Serialize, Serializer, + }; use serde_with::{DeserializeAs, SerializeAs}; /// Bincode-compatible [`super::TrieUpdates`] serde implementation. @@ -1598,7 +1357,6 @@ pub mod serde_bincode_compat { /// ``` #[derive(Debug, Serialize, Deserialize)] pub struct StorageTrieUpdates<'a> { - is_deleted: bool, storage_nodes: Cow<'a, HashMap>, removed_nodes: Cow<'a, HashSet>, } @@ -1606,7 +1364,6 @@ pub mod serde_bincode_compat { impl<'a> From<&'a super::StorageTrieUpdates> for StorageTrieUpdates<'a> { fn from(value: &'a super::StorageTrieUpdates) -> Self { Self { - is_deleted: value.is_deleted, storage_nodes: Cow::Borrowed(&value.storage_nodes), removed_nodes: Cow::Borrowed(&value.removed_nodes), } @@ -1616,7 +1373,6 @@ pub mod serde_bincode_compat { impl<'a> From> for super::StorageTrieUpdates { fn from(value: StorageTrieUpdates<'a>) -> Self { Self { - is_deleted: value.is_deleted, storage_nodes: value.storage_nodes.into_owned(), removed_nodes: value.removed_nodes.into_owned(), } @@ -1723,24 +1479,64 @@ pub mod serde_bincode_compat { /// trie_updates: StorageTrieUpdatesSorted, /// } /// ``` - #[derive(Debug, Serialize, Deserialize)] + #[derive(Debug, Serialize)] pub struct StorageTrieUpdatesSorted<'a> { - is_deleted: bool, storage_nodes: Cow<'a, [(Nibbles, Option)]>, } + impl<'de, 'a> Deserialize<'de> for StorageTrieUpdatesSorted<'a> { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + struct StorageTrieUpdatesSortedVisitor; + + impl<'de> Visitor<'de> for StorageTrieUpdatesSortedVisitor { + type Value = Vec<(Nibbles, Option)>; + + fn expecting(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str("storage trie updates with an optional legacy wipe marker") + } + + fn visit_seq(self, mut seq: A) -> Result + where + A: SeqAccess<'de>, + { + let len = seq.size_hint().unwrap_or_default(); + let storage_nodes = match len { + 1 => { + seq.next_element()?.ok_or_else(|| A::Error::invalid_length(0, &self))? + } + 2 => { + let _: bool = seq + .next_element()? + .ok_or_else(|| A::Error::invalid_length(0, &self))?; + seq.next_element()?.ok_or_else(|| A::Error::invalid_length(1, &self))? + } + _ => return Err(A::Error::invalid_length(len, &self)), + }; + + Ok(storage_nodes) + } + } + + // Bincode uses the supplied tuple length for the current format, while MessagePack + // exposes the encoded sequence length so legacy ExEx WAL entries can still be read. + deserializer + .deserialize_tuple(1, StorageTrieUpdatesSortedVisitor) + .map(|storage_nodes| Self { storage_nodes: Cow::Owned(storage_nodes) }) + } + } + impl<'a> From<&'a super::StorageTrieUpdatesSorted> for StorageTrieUpdatesSorted<'a> { fn from(value: &'a super::StorageTrieUpdatesSorted) -> Self { - Self { - is_deleted: value.is_deleted, - storage_nodes: Cow::Borrowed(&value.storage_nodes), - } + Self { storage_nodes: Cow::Borrowed(&value.storage_nodes) } } } impl<'a> From> for super::StorageTrieUpdatesSorted { fn from(value: StorageTrieUpdatesSorted<'a>) -> Self { - Self { is_deleted: value.is_deleted, storage_nodes: value.storage_nodes.into_owned() } + Self { storage_nodes: value.storage_nodes.into_owned() } } } @@ -1908,11 +1704,6 @@ pub mod serde_bincode_compat { let encoded = bincode::serialize(&data).unwrap(); let decoded: Data = bincode::deserialize(&encoded).unwrap(); assert_eq!(decoded, data); - - data.trie_updates.is_deleted = true; - let encoded = bincode::serialize(&data).unwrap(); - let decoded: Data = bincode::deserialize(&encoded).unwrap(); - assert_eq!(decoded, data); } } } diff --git a/crates/trie/db/src/state.rs b/crates/trie/db/src/state.rs index c0012fdae05..512dc96ccd3 100644 --- a/crates/trie/db/src/state.rs +++ b/crates/trie/db/src/state.rs @@ -325,7 +325,7 @@ impl DatabaseHashedPostState for HashedPostStateSorted { .into_iter() .map(|(address, mut slots)| { slots.sort_unstable_by_key(|(slot, _)| *slot); - (address, HashedStorageSorted { storage_slots: slots, wiped: false }) + (address, HashedStorageSorted { storage_slots: slots }) }) .collect(); diff --git a/crates/trie/db/src/storage.rs b/crates/trie/db/src/storage.rs index e82f883655d..884d6f15565 100644 --- a/crates/trie/db/src/storage.rs +++ b/crates/trie/db/src/storage.rs @@ -36,7 +36,7 @@ pub fn hashed_storage_from_reverts_with_provider

( where P: StorageChangeSetReader + BlockNumReader, { - let mut storage = HashedStorage::new(false); + let mut storage = HashedStorage::default(); let tip = provider.last_block_number()?; if from > tip { diff --git a/crates/trie/db/src/trie_cursor.rs b/crates/trie/db/src/trie_cursor.rs index 30ef564ffba..250d049d874 100644 --- a/crates/trie/db/src/trie_cursor.rs +++ b/crates/trie/db/src/trie_cursor.rs @@ -281,11 +281,6 @@ where &mut self, updates: &StorageTrieUpdatesSorted, ) -> Result { - // The storage trie for this account has to be deleted. - if updates.is_deleted() && self.cursor.seek_exact(self.hashed_address)?.is_some() { - self.cursor.delete_current_duplicates()?; - } - let mut num_entries = 0; for (nibbles, maybe_updated) in updates.storage_nodes.iter().filter(|(n, _)| !n.is_empty()) { diff --git a/crates/trie/db/tests/fuzz_in_memory_nodes.rs b/crates/trie/db/tests/fuzz_in_memory_nodes.rs index 229b01e3464..d4b2bcd20e6 100644 --- a/crates/trie/db/tests/fuzz_in_memory_nodes.rs +++ b/crates/trie/db/tests/fuzz_in_memory_nodes.rs @@ -2,11 +2,7 @@ use alloy_primitives::{B256, U256}; use proptest::prelude::*; -use reth_db::{ - cursor::{DbCursorRO, DbCursorRW, DbDupCursorRW}, - tables, - transaction::DbTxMut, -}; +use reth_db::{cursor::DbCursorRW, tables, transaction::DbTxMut}; use reth_primitives_traits::{Account, StorageEntry}; use reth_provider::test_utils::create_test_provider_factory; use reth_storage_api::StorageSettingsCache; @@ -85,7 +81,7 @@ proptest! { } #[test] - fn fuzz_in_memory_storage_nodes(mut init_storage: BTreeMap, storage_updates: [(bool, BTreeMap); 10]) { + fn fuzz_in_memory_storage_nodes(mut init_storage: BTreeMap, storage_updates: [BTreeMap; 10]) { let hashed_address = B256::random(); let factory = create_test_provider_factory(); let provider = factory.provider_rw().unwrap(); @@ -105,12 +101,9 @@ proptest! { DbStorageRoot::<_, A>::from_tx_hashed(provider.tx_ref(), hashed_address).root_with_updates().unwrap(); let mut storage = init_storage; - for (is_deleted, mut storage_update) in storage_updates { + for mut storage_update in storage_updates { // Insert state updates into database - if is_deleted && hashed_storage_cursor.seek_exact(hashed_address).unwrap().is_some() { - hashed_storage_cursor.delete_current_duplicates().unwrap(); - } - let mut hashed_storage = HashedStorage::new(is_deleted); + let mut hashed_storage = HashedStorage::default(); for (hashed_slot, value) in storage_update.clone() { hashed_storage_cursor .upsert(hashed_address, &StorageEntry { key: hashed_slot, value }) @@ -134,9 +127,6 @@ proptest! { storage_trie_nodes.extend(trie_updates); // Verify the result - if is_deleted { - storage.clear(); - } storage.append(&mut storage_update); let expected_root = storage_root_prehashed(storage.clone()); assert_eq!(expected_root, storage_root); diff --git a/crates/trie/db/tests/post_state.rs b/crates/trie/db/tests/post_state.rs index d3f8fe36484..b071487fbed 100644 --- a/crates/trie/db/tests/post_state.rs +++ b/crates/trie/db/tests/post_state.rs @@ -208,18 +208,7 @@ fn fuzz_hashed_account_cursor() { ); } -/// Tests `is_storage_empty()` correctly distinguishes wiped storage from storage with zero values. -/// -/// Key distinction: -/// - `wiped = true`: Storage cleared/deleted → empty -/// - `wiped = false` with zeros: Explicit zero values → not empty -/// -/// Test cases: -/// 1. No entries → empty -/// 2. Non-zero values → not empty -/// 3. Some zero values, not wiped → not empty -/// 4. Wiped + zero post-state → empty -/// 5. Wiped + non-zero post-state → not empty +/// Tests `is_storage_empty()` with empty, database-backed, and zero-value overlays. #[test] fn storage_is_empty() { let address = B256::random(); @@ -256,10 +245,9 @@ fn storage_is_empty() { assert!(!cursor.is_storage_empty().unwrap()); } - // Some zero values, but not wiped + // Some zero values { - let wiped = false; - let mut hashed_storage = HashedStorage::new(wiped); + let mut hashed_storage = HashedStorage::default(); hashed_storage.storage.insert(B256::with_last_byte(0), U256::ZERO); let mut hashed_post_state = HashedPostState::default(); @@ -272,56 +260,6 @@ fn storage_is_empty() { let mut cursor = factory.hashed_storage_cursor(address).unwrap(); assert!(!cursor.is_storage_empty().unwrap()); } - - // wiped storage, must be empty - { - let wiped = true; - let hashed_storage = HashedStorage::new(wiped); - - let mut hashed_post_state = HashedPostState::default(); - hashed_post_state.storages.insert(address, hashed_storage); - - let sorted = hashed_post_state.into_sorted(); - let tx = db.tx().unwrap(); - let factory = - HashedPostStateCursorFactory::new(DatabaseHashedCursorFactory::new(&tx), &sorted); - let mut cursor = factory.hashed_storage_cursor(address).unwrap(); - assert!(cursor.is_storage_empty().unwrap()); - } - - // wiped storage, but post state has zero-value entries - { - let wiped = true; - let mut hashed_storage = HashedStorage::new(wiped); - hashed_storage.storage.insert(B256::random(), U256::ZERO); - - let mut hashed_post_state = HashedPostState::default(); - hashed_post_state.storages.insert(address, hashed_storage); - - let sorted = hashed_post_state.into_sorted(); - let tx = db.tx().unwrap(); - let factory = - HashedPostStateCursorFactory::new(DatabaseHashedCursorFactory::new(&tx), &sorted); - let mut cursor = factory.hashed_storage_cursor(address).unwrap(); - assert!(cursor.is_storage_empty().unwrap()); - } - - // wiped storage, but post state has non-zero entries - { - let wiped = true; - let mut hashed_storage = HashedStorage::new(wiped); - hashed_storage.storage.insert(B256::random(), U256::from(1)); - - let mut hashed_post_state = HashedPostState::default(); - hashed_post_state.storages.insert(address, hashed_storage); - - let sorted = hashed_post_state.into_sorted(); - let tx = db.tx().unwrap(); - let factory = - HashedPostStateCursorFactory::new(DatabaseHashedCursorFactory::new(&tx), &sorted); - let mut cursor = factory.hashed_storage_cursor(address).unwrap(); - assert!(!cursor.is_storage_empty().unwrap()); - } } #[test] @@ -343,8 +281,7 @@ fn storage_cursor_correct_order() { }) .unwrap(); - let wiped = false; - let mut hashed_storage = HashedStorage::new(wiped); + let mut hashed_storage = HashedStorage::default(); for (slot, value) in &post_state_storage { hashed_storage.storage.insert(*slot, *value); } @@ -383,8 +320,7 @@ fn zero_value_storage_entries_are_discarded() { }) .unwrap(); - let wiped = false; - let mut hashed_storage = HashedStorage::new(wiped); + let mut hashed_storage = HashedStorage::default(); for (slot, value) in &post_state_storage { hashed_storage.storage.insert(*slot, *value); } @@ -402,40 +338,6 @@ fn zero_value_storage_entries_are_discarded() { assert_storage_cursor_order(&factory, expected); } -#[test] -fn wiped_storage_is_discarded() { - let address = B256::random(); - let db_storage = - (1..11).map(|key| (B256::with_last_byte(key), U256::from(key))).collect::>(); - let post_state_storage = (11..21) - .map(|key| (B256::with_last_byte(key), U256::from(key))) - .collect::>(); - - let db = create_test_rw_db(); - db.update(|tx| { - for (slot, value) in db_storage { - // insert zero value accounts to the database - tx.put::(address, StorageEntry { key: slot, value }).unwrap(); - } - }) - .unwrap(); - - let wiped = true; - let mut hashed_storage = HashedStorage::new(wiped); - for (slot, value) in &post_state_storage { - hashed_storage.storage.insert(*slot, *value); - } - - let mut hashed_post_state = HashedPostState::default(); - hashed_post_state.storages.insert(address, hashed_storage); - - let sorted = hashed_post_state.into_sorted(); - let tx = db.tx().unwrap(); - let factory = HashedPostStateCursorFactory::new(DatabaseHashedCursorFactory::new(&tx), &sorted); - let expected = std::iter::once((address, post_state_storage)); - assert_storage_cursor_order(&factory, expected); -} - #[test] fn post_state_storages_take_precedence() { let address = B256::random(); @@ -455,8 +357,7 @@ fn post_state_storages_take_precedence() { }) .unwrap(); - let wiped = false; - let mut hashed_storage = HashedStorage::new(wiped); + let mut hashed_storage = HashedStorage::default(); for (slot, value) in &storage { hashed_storage.storage.insert(*slot, *value); } @@ -476,7 +377,7 @@ fn fuzz_hashed_storage_cursor() { proptest!(ProptestConfig::with_cases(10), |( db_storages: BTreeMap>, - post_state_storages: BTreeMap)> + post_state_storages: BTreeMap> )| { let db = create_test_rw_db(); @@ -492,8 +393,8 @@ fn fuzz_hashed_storage_cursor() { let mut hashed_post_state = HashedPostState::default(); - for (address, (wiped, storage)) in &post_state_storages { - let mut hashed_storage = HashedStorage::new(*wiped); + for (address, storage) in &post_state_storages { + let mut hashed_storage = HashedStorage::default(); for (slot, value) in storage { hashed_storage.storage.insert(*slot, *value); } @@ -503,12 +404,8 @@ fn fuzz_hashed_storage_cursor() { let mut expected = db_storages; // overwrite or remove accounts from the expected result - for (key, (wiped, storage)) in post_state_storages { - let entry = expected.entry(key).or_default(); - if wiped { - entry.clear(); - } - entry.extend(storage); + for (key, storage) in post_state_storages { + expected.entry(key).or_default().extend(storage); } let sorted = hashed_post_state.into_sorted(); @@ -519,9 +416,8 @@ fn fuzz_hashed_storage_cursor() { } #[test] -fn all_storage_slots_deleted_not_wiped_exact_keys() { +fn all_storage_slots_deleted_exact_keys() { // This test reproduces an edge case where: - // - wiped = false // - All post state entries are deletions (None values) // - Database has corresponding entries // - Expected: NO leaves should be returned (all deleted) @@ -550,7 +446,7 @@ fn all_storage_slots_deleted_not_wiped_exact_keys() { .unwrap(); // Create post state with same keys but all Zero values (deletions) - let mut hashed_storage = HashedStorage::new(false); + let mut hashed_storage = HashedStorage::default(); for (key, _) in &db_entries { hashed_storage.storage.insert(*key, U256::ZERO); // Zero value = deletion } @@ -564,6 +460,8 @@ fn all_storage_slots_deleted_not_wiped_exact_keys() { let mut cursor = factory.hashed_storage_cursor(address).unwrap(); + assert!(cursor.is_storage_empty().unwrap()); + // Seek to beginning should return None (all slots are deleted) let result = cursor.seek(B256::ZERO).unwrap(); assert_eq!( diff --git a/crates/trie/db/tests/trie.rs b/crates/trie/db/tests/trie.rs index 7367c06a592..903b6e2b0c1 100644 --- a/crates/trie/db/tests/trie.rs +++ b/crates/trie/db/tests/trie.rs @@ -218,7 +218,7 @@ fn test_empty_storage_root() { } #[test] -fn cleared_storage_emits_node_removals_without_deleted_flag() { +fn cleared_storage_emits_node_removals() { let factory = create_test_provider_factory(); let tx = factory.provider_rw().unwrap(); let hashed_address = B256::random(); @@ -271,7 +271,6 @@ fn cleared_storage_emits_node_removals_without_deleted_flag() { .root_with_updates() .unwrap(); assert_eq!(root, EMPTY_ROOT_HASH); - assert!(!updates.is_deleted()); assert!(!updates.removed_nodes_ref().is_empty()); }); } @@ -345,7 +344,6 @@ fn destroyed_account_storage_emits_node_removals() { assert_eq!(root, EMPTY_ROOT_HASH); let storage_updates = &updates.storage_tries_ref()[&hashed_address]; - assert!(!storage_updates.is_deleted()); let mut removed_nodes: Vec<_> = storage_updates.removed_nodes_ref().iter().copied().collect(); removed_nodes.sort_unstable(); assert_eq!(removed_nodes, expected_removed_nodes); diff --git a/crates/trie/db/tests/witness.rs b/crates/trie/db/tests/witness.rs index 38d55688ff3..b15520dc949 100644 --- a/crates/trie/db/tests/witness.rs +++ b/crates/trie/db/tests/witness.rs @@ -148,7 +148,10 @@ fn includes_nodes_for_destroyed_storage_nodes() { ) .compute(HashedPostState { accounts: HashMap::from_iter([(hashed_address, Some(Account::default()))]), - storages: HashMap::from_iter([(hashed_address, HashedStorage::new(true))]), // destroyed + storages: HashMap::from_iter([( + hashed_address, + HashedStorage::from_iter([(hashed_slot, U256::ZERO)]), + )]), }) .unwrap(); assert!(witness.contains_key(&state_root)); diff --git a/crates/trie/sparse/src/arena/mod.rs b/crates/trie/sparse/src/arena/mod.rs index 97eaf1e510b..26ae9f62710 100644 --- a/crates/trie/sparse/src/arena/mod.rs +++ b/crates/trie/sparse/src/arena/mod.rs @@ -1052,7 +1052,6 @@ impl ArenaParallelSparseTrie { ) { if let Some(dst_updates) = dst.as_mut() { let src_updates = src.as_mut().expect("updates are enabled"); - debug_assert!(!src_updates.wiped, "subtrie updates should never have wiped=true"); // Source insertions cancel destination removals. for path in src_updates.updated_nodes.keys() { @@ -2622,14 +2621,6 @@ impl SparseTrie for ArenaParallelSparseTrie { } } - #[instrument(level = "trace", target = TRACE_TARGET, skip_all)] - fn wipe(&mut self) { - trace!(target: TRACE_TARGET, "Wiping arena trie"); - self.clear(); - *self.upper_arena[self.root].state_mut() = ArenaSparseNodeState::Dirty; - self.buffers.updates = self.buffers.updates.is_some().then(SparseTrieUpdates::wiped); - } - #[instrument(level = "trace", target = TRACE_TARGET, skip_all)] fn clear(&mut self) { #[cfg(feature = "trie-debug")] diff --git a/crates/trie/sparse/src/state.rs b/crates/trie/sparse/src/state.rs index 8d67c1157af..a0ed6c92acf 100644 --- a/crates/trie/sparse/src/state.rs +++ b/crates/trie/sparse/src/state.rs @@ -341,14 +341,6 @@ where any_err } - /// Wipe the storage trie at the provided address. - pub fn wipe_storage(&mut self, address: B256) -> SparseStateTrieResult<()> { - if let Some(trie) = self.storage.tries.get_mut(&address) { - trie.wipe()?; - } - Ok(()) - } - /// Calculates the hashes of subtries. /// /// If the trie has not been revealed, this function does nothing. @@ -413,7 +405,6 @@ where let trie = trie.as_revealed_mut().unwrap(); let updates = trie.take_updates(); let updates = StorageTrieUpdates { - is_deleted: updates.wiped, storage_nodes: updates.updated_nodes, removed_nodes: updates.removed_nodes, }; @@ -1053,7 +1044,7 @@ mod tests { let address_2 = b256!("0x1100000000000000000000000000000000000000000000000000000000000000"); let address_path_2 = Nibbles::unpack(address_2); let account_2 = Account::arbitrary(&mut arbitrary::Unstructured::new(&bytes)).unwrap(); - let mut trie_account_2 = account_2.into_trie_account(EMPTY_ROOT_HASH); + let trie_account_2 = account_2.into_trie_account(EMPTY_ROOT_HASH); let mut hash_builder = HashBuilder::default() .with_proof_retainer(ProofRetainer::from_iter([address_path_1, address_path_2])); @@ -1125,14 +1116,6 @@ mod tests { LeafUpdate::Changed(alloy_rlp::encode(trie_account_1)), ); - sparse.wipe_storage(address_2).unwrap(); - trie_account_2.storage_root = sparse.storage_root(&address_2, epoch(0)).unwrap(); - apply_account_update( - &mut sparse, - address_2, - LeafUpdate::Changed(alloy_rlp::encode(trie_account_2)), - ); - sparse.root(epoch(0)).unwrap(); let sparse_updates = sparse.take_trie_updates().unwrap(); @@ -1141,24 +1124,13 @@ mod tests { sparse_updates, TrieUpdates { account_nodes: HashMap::default(), - storage_tries: HashMap::from_iter([ - ( - b256!("0x1000000000000000000000000000000000000000000000000000000000000000"), - StorageTrieUpdates { - is_deleted: false, - storage_nodes: HashMap::default(), - removed_nodes: HashSet::from_iter([Nibbles::from_nibbles([0x1])]) - } - ), - ( - b256!("0x1100000000000000000000000000000000000000000000000000000000000000"), - StorageTrieUpdates { - is_deleted: true, - storage_nodes: HashMap::default(), - removed_nodes: HashSet::default() - } - ) - ]), + storage_tries: HashMap::from_iter([( + b256!("0x1000000000000000000000000000000000000000000000000000000000000000"), + StorageTrieUpdates { + storage_nodes: HashMap::default(), + removed_nodes: HashSet::from_iter([Nibbles::from_nibbles([0x1])]) + } + )]), removed_nodes: HashSet::default() } ); diff --git a/crates/trie/sparse/src/traits.rs b/crates/trie/sparse/src/traits.rs index 79a4a8d4972..2c0fd0f9e88 100644 --- a/crates/trie/sparse/src/traits.rs +++ b/crates/trie/sparse/src/traits.rs @@ -206,15 +206,6 @@ pub trait SparseTrie: Sized + Debug + Send + Sync { /// The accumulated updates, or an empty set if updates weren't being tracked. fn take_updates(&mut self) -> SparseTrieUpdates; - /// Removes all nodes and values from the trie, resetting it to a blank state - /// with only an empty root node. This is used when a storage root is deleted. - /// - /// This should not be used when intending to reuse the trie for a fresh account/storage root; - /// use `clear` for that. - /// - /// Note: All previously tracked changes to the trie are also removed. - fn wipe(&mut self); - /// This clears all data structures in the sparse trie, keeping the backing data structures /// allocated. An empty root node is inserted at the root. /// @@ -277,8 +268,6 @@ pub struct SparseTrieUpdates { pub updated_nodes: HashMap, /// Collection of removed intermediate nodes indexed by full path. pub removed_nodes: HashSet, - /// Flag indicating whether the trie was wiped. - pub wiped: bool, } impl SparseTrieUpdates { @@ -287,7 +276,6 @@ impl SparseTrieUpdates { Self { updated_nodes: HashMap::with_capacity_and_hasher(num_updated_nodes, Default::default()), removed_nodes: HashSet::with_capacity_and_hasher(num_removed_nodes, Default::default()), - wiped: false, } } } diff --git a/crates/trie/sparse/src/trie.rs b/crates/trie/sparse/src/trie.rs index 63ef6e1b81d..e4a3e429c56 100644 --- a/crates/trie/sparse/src/trie.rs +++ b/crates/trie/sparse/src/trie.rs @@ -159,16 +159,6 @@ impl RevealableSparseTrie { } } - /// Wipes the trie by removing all nodes and values, - /// and resetting the trie to only contain an empty root node. - /// - /// Note: This method will error if the trie is blinded. - pub fn wipe(&mut self) -> SparseTrieResult<()> { - let revealed = self.as_revealed_mut().ok_or(SparseTrieErrorKind::Blind)?; - revealed.wipe(); - Ok(()) - } - /// Calculates the root hash of the trie. /// /// This will update any remaining dirty nodes before computing the root hash. @@ -470,24 +460,15 @@ pub struct RlpNodeStackItem { } impl SparseTrieUpdates { - /// Create new wiped sparse trie updates. - pub fn wiped() -> Self { - Self { wiped: true, ..Default::default() } - } - /// Clears the updates, but keeps the backing data structures allocated. - /// - /// Sets `wiped` to `false`. pub fn clear(&mut self) { self.updated_nodes.clear(); self.removed_nodes.clear(); - self.wiped = false; } /// Extends the updates with another set of updates. pub fn extend(&mut self, other: Self) { self.updated_nodes.extend(other.updated_nodes); self.removed_nodes.extend(other.removed_nodes); - self.wiped |= other.wiped; } } diff --git a/crates/trie/sparse/tests/suite/main.rs b/crates/trie/sparse/tests/suite/main.rs index 4666d5b0c85..7fdab8685ab 100644 --- a/crates/trie/sparse/tests/suite/main.rs +++ b/crates/trie/sparse/tests/suite/main.rs @@ -12,7 +12,7 @@ //! - [`root`]: Tests for `root()` hash computation //! - [`take_updates`]: Tests for `take_updates` //! - [`prune`]: Tests for `prune` -//! - [`wipe_clear`]: Tests for `wipe` and `clear` +//! - [`wipe_clear`]: Tests for `clear` //! - [`get_leaf_value`]: Tests for `get_leaf_value` //! - [`find_leaf`]: Tests for `find_leaf` //! - [`lifecycle`]: Integration tests exercising multiple methods together @@ -285,10 +285,8 @@ sparse_trie_tests! { test_prune_only_descends_into_branch_root, test_prune_handles_small_subtrie_root_nodes, - // wipe / clear - test_wipe_resets_to_empty_root, + // clear test_clear_resets_trie_but_preserves_update_tracking, - test_wipe_produces_wiped_updates, test_clear_then_reuse_trie, // get_leaf_value diff --git a/crates/trie/sparse/tests/suite/wipe_clear.rs b/crates/trie/sparse/tests/suite/wipe_clear.rs index f3c789a93cd..ab4677a1cfd 100644 --- a/crates/trie/sparse/tests/suite/wipe_clear.rs +++ b/crates/trie/sparse/tests/suite/wipe_clear.rs @@ -1,33 +1,8 @@ use super::*; -/// Calling `wipe()` resets the trie so that -/// `root()` returns `EMPTY_ROOT_HASH`. -pub(super) fn test_wipe_resets_to_empty_root(new_trie: fn() -> T) { - let storage: BTreeMap = BTreeMap::from([ - (B256::with_last_byte(0x10), U256::from(1)), - (B256::with_last_byte(0x20), U256::from(2)), - (B256::with_last_byte(0x30), U256::from(3)), - (B256::with_last_byte(0x40), U256::from(4)), - (B256::with_last_byte(0x50), U256::from(5)), - ]); - - let harness = SuiteTestHarness::new(storage); - let mut trie: T = harness.init_trie_fully_revealed(false, new_trie); - - // Compute root to confirm the trie is populated. - let root_before = trie.root(epoch(0)); - assert_eq!(root_before, harness.original_root()); - assert_ne!(root_before, EMPTY_ROOT_HASH); - - // Wipe and verify empty root. - trie.wipe(); - let root_after = trie.root(epoch(0)); - assert_eq!(root_after, EMPTY_ROOT_HASH, "root must be EMPTY_ROOT_HASH after wipe"); -} - /// `clear()` resets the trie to empty but preserves /// update tracking mode. After clear, `root()` returns `EMPTY_ROOT_HASH` and -/// `take_updates()` returns empty (non-wiped) updates. +/// `take_updates()` returns empty updates. pub(super) fn test_clear_resets_trie_but_preserves_update_tracking( new_trie: fn() -> T, ) { @@ -51,44 +26,12 @@ pub(super) fn test_clear_resets_trie_but_preserves_update_tracking(new_trie: fn() -> T) { - let storage: BTreeMap = BTreeMap::from([ - (B256::with_last_byte(0x10), U256::from(1)), - (B256::with_last_byte(0x20), U256::from(2)), - (B256::with_last_byte(0x30), U256::from(3)), - ]); - - let harness = SuiteTestHarness::new(storage); - // retain_updates = true so update tracking is active - let mut trie: T = harness.init_trie_fully_revealed(true, new_trie); - - // Compute root to populate the trie fully. - let root_before = trie.root(epoch(0)); - assert_eq!(root_before, harness.original_root()); - assert_ne!(root_before, EMPTY_ROOT_HASH); - - // Wipe the trie. - trie.wipe(); - - // Root should be EMPTY_ROOT_HASH after wipe. - let root_after = trie.root(epoch(0)); - assert_eq!(root_after, EMPTY_ROOT_HASH, "root must be EMPTY_ROOT_HASH after wipe"); - - // take_updates should return updates with the wiped flag set. - let updates = trie.take_updates(); - assert!(updates.wiped, "wipe should produce wiped updates"); -} - /// A cleared trie can be fully re-initialized and used /// normally. After `clear()`, set a new root from a different dataset, reveal /// nodes, insert a leaf, and verify `root()` matches the reference. diff --git a/crates/trie/trie/benches/proof_v2.rs b/crates/trie/trie/benches/proof_v2.rs index 87dd20b9404..f007e19a604 100644 --- a/crates/trie/trie/benches/proof_v2.rs +++ b/crates/trie/trie/benches/proof_v2.rs @@ -43,10 +43,8 @@ fn generate_test_data( // Create HashedPostState with single account's storage let mut storages = B256Map::default(); - let hashed_storage = HashedStorage { - wiped: false, - storage: storage_map.iter().map(|(k, v)| (*k, *v)).collect(), - }; + let hashed_storage = + HashedStorage { storage: storage_map.iter().map(|(k, v)| (*k, *v)).collect() }; storages.insert(hashed_address, hashed_storage); let hashed_post_state = HashedPostState { accounts: B256Map::default(), storages }; diff --git a/crates/trie/trie/src/changesets.rs b/crates/trie/trie/src/changesets.rs index 6f0bc85f80d..30942e5eb61 100644 --- a/crates/trie/trie/src/changesets.rs +++ b/crates/trie/trie/src/changesets.rs @@ -18,17 +18,13 @@ //! //! And returns `TrieUpdatesSorted` containing the old node values. -use crate::trie_cursor::TrieCursorIter; +use crate::trie_cursor::{TrieCursor, TrieCursorFactory, TrieStorageCursor}; use alloy_primitives::{map::B256Map, B256}; -use itertools::{merge_join_by, EitherOrBoth}; use reth_storage_errors::db::DatabaseError; use reth_trie_common::{ updates::{StorageTrieUpdatesSorted, TrieUpdatesSorted}, BranchNodeCompact, Nibbles, }; -use std::cmp::Ordering; - -use crate::trie_cursor::{TrieCursor, TrieCursorFactory, TrieStorageCursor}; /// Result type for changeset operations. pub type ChangesetResult = Result; @@ -66,21 +62,12 @@ where for (hashed_address, storage_updates) in trie_updates.storage_tries_ref() { storage_cursor.set_hashed_address(*hashed_address); - let storage_changesets = if storage_updates.is_deleted() { - // Handle wiped storage - compute_wiped_storage_changesets(&mut storage_cursor, storage_updates)? - } else { - // Handle normal storage updates - compute_storage_changesets(&mut storage_cursor, storage_updates)? - }; + let storage_changesets = compute_storage_changesets(&mut storage_cursor, storage_updates)?; if !storage_changesets.is_empty() { storage_tries.insert( *hashed_address, - StorageTrieUpdatesSorted { - is_deleted: storage_updates.is_deleted(), - storage_nodes: storage_changesets, - }, + StorageTrieUpdatesSorted { storage_nodes: storage_changesets }, ); } } @@ -141,99 +128,6 @@ fn compute_storage_changesets( Ok(storage_changesets) } -/// Handles wiped storage trie changeset computation. -/// -/// When an account's storage is completely wiped (e.g., account is destroyed), -/// we need to capture not just the changed nodes, but ALL existing nodes in -/// the storage trie, since they all will be deleted. -/// -/// This uses an iterator-based approach to avoid allocating an intermediate Vec. -/// It merges two sorted iterators: -/// - Current values of changed paths -/// - All existing nodes in the storage trie -/// -/// # Arguments -/// -/// * `changed_cursor` - Cursor for looking up changed node values -/// * `wiped_cursor` - Cursor for iterating all nodes in the storage trie -/// * `hashed_address` - The hashed address of the account -/// * `storage_updates` - Storage trie updates for this account -fn compute_wiped_storage_changesets( - cursor: &mut impl TrieStorageCursor, - storage_updates: &StorageTrieUpdatesSorted, -) -> ChangesetResult)>> { - // Set the hashed address for this account's storage trie - // Create an iterator that yields all nodes in the storage trie - let all_nodes = TrieCursorIter::new(cursor); - - // Merge the two sorted iterators - let merged = storage_trie_wiped_changeset_iter( - storage_updates.storage_nodes.iter().map(|e| e.0), - all_nodes, - )?; - - // Collect into a Vec - let mut storage_changesets = Vec::new(); - for result in merged { - storage_changesets.push(result?); - } - - Ok(storage_changesets) -} - -/// Returns an iterator which produces the changeset values for an account whose storage was wiped -/// during a block. -/// -/// ## Arguments -/// -/// - `curr_values_of_changed` is an iterator over the current values of all trie nodes modified by -/// the block, ordered by path. -/// - `all_nodes` is an iterator over all existing trie nodes for the account, ordered by path. -/// -/// ## Returns -/// -/// An iterator of trie node paths and a `Some(node)` (indicating the node was wiped) or a `None` -/// (indicating the node was modified in the block but didn't previously exist). The iterator's -/// results will be ordered by path. -pub fn storage_trie_wiped_changeset_iter( - changed_paths: impl Iterator, - all_nodes: impl Iterator>, -) -> Result< - impl Iterator), DatabaseError>>, - DatabaseError, -> { - let all_nodes = all_nodes.map(|e| e.map(|(nibbles, node)| (nibbles, Some(node)))); - - let merged = merge_join_by(changed_paths, all_nodes, |a, b| match (a, b) { - (_, Err(_)) => Ordering::Greater, - (a, Ok(b)) => a.cmp(&b.0), - }); - - Ok(merged.map(|either_or| match either_or { - EitherOrBoth::Left(changed) => { - // A path of a changed node which was not found in the database. The current value of - // this path must be None, otherwise it would have also been returned by the `all_nodes` - // iter. - Ok((changed, None)) - } - EitherOrBoth::Right(wiped) => { - // A node was found in the db (indicating it was wiped) but was not a changed node. - // Return it as-is. - wiped - } - EitherOrBoth::Both(_changed, wiped) => { - // A path of a changed node was found with a previous value in the database. If the - // changed node had no previous value (None) it wouldn't be returned by `all_nodes` and - // so would be in the Left branch. - // - // Due to the ordering closure passed to `merge_join_by` it's not possible for wrapped - // to be an error here. - debug_assert!(wiped.is_ok(), "unreachable error condition: {wiped:?}"); - wiped - } - })) -} - #[cfg(test)] mod tests { use super::*; @@ -334,7 +228,6 @@ mod tests { storage_updates.insert( hashed_address, StorageTrieUpdatesSorted { - is_deleted: false, storage_nodes: vec![(path1, Some(new_node1)), (path3, Some(new_node3))], }, ); @@ -347,7 +240,6 @@ mod tests { // Check storage changesets assert_eq!(changesets.storage_tries_ref().len(), 1); let storage_changesets = changesets.storage_tries_ref().get(&hashed_address).unwrap(); - assert!(!storage_changesets.is_deleted); assert_eq!(storage_changesets.storage_nodes.len(), 2); // path1 should have the old node1 value @@ -358,119 +250,4 @@ mod tests { assert_eq!(storage_changesets.storage_nodes[1].0, path3); assert_eq!(storage_changesets.storage_nodes[1].1, None); } - - #[test] - fn test_wiped_storage() { - let hashed_address = B256::from([2u8; 32]); - - // Create initial storage trie with multiple nodes - let path1 = Nibbles::from_nibbles([0x1, 0x2]); - let path2 = Nibbles::from_nibbles([0x3, 0x4]); - let path3 = Nibbles::from_nibbles([0x5, 0x6]); - let node1 = BranchNodeCompact::new(0b1111, 0b0011, 0, vec![], None); - let node2 = BranchNodeCompact::new(0b1111, 0b0101, 0, vec![], None); - let node3 = BranchNodeCompact::new(0b1111, 0b1001, 0, vec![], None); - - let mut storage_nodes = BTreeMap::new(); - storage_nodes.insert(path1, node1.clone()); - storage_nodes.insert(path2, node2.clone()); - storage_nodes.insert(path3, node3.clone()); - - let mut storage_tries = B256Map::default(); - storage_tries.insert(B256::default(), BTreeMap::new()); // For cursor creation - storage_tries.insert(hashed_address, storage_nodes); - - let factory = MockTrieCursorFactory::new(BTreeMap::new(), storage_tries); - - // Create updates that modify path1 and mark storage as wiped - let new_node1 = BranchNodeCompact::new(0b1111, 0b1000, 0, vec![], None); - - let mut storage_updates = B256Map::default(); - storage_updates.insert( - hashed_address, - StorageTrieUpdatesSorted { - is_deleted: true, - storage_nodes: vec![(path1, Some(new_node1))], - }, - ); - - let updates = TrieUpdatesSorted::new(vec![], storage_updates); - - // Compute changesets - let changesets = compute_trie_changesets(&factory, &updates).unwrap(); - - // Check storage changesets - assert_eq!(changesets.storage_tries_ref().len(), 1); - let storage_changesets = changesets.storage_tries_ref().get(&hashed_address).unwrap(); - assert!(storage_changesets.is_deleted); - - // Should include all three nodes (changed path1 + wiped path2 and path3) - assert_eq!(storage_changesets.storage_nodes.len(), 3); - - // All paths should be present in sorted order - assert_eq!(storage_changesets.storage_nodes[0].0, path1); - assert_eq!(storage_changesets.storage_nodes[1].0, path2); - assert_eq!(storage_changesets.storage_nodes[2].0, path3); - - // All should have their old values - assert_eq!(storage_changesets.storage_nodes[0].1, Some(node1)); - assert_eq!(storage_changesets.storage_nodes[1].1, Some(node2)); - assert_eq!(storage_changesets.storage_nodes[2].1, Some(node3)); - } - - #[test] - fn test_wiped_storage_with_new_path() { - let hashed_address = B256::from([3u8; 32]); - - // Create initial storage trie with two nodes - let path1 = Nibbles::from_nibbles([0x1, 0x2]); - let path3 = Nibbles::from_nibbles([0x5, 0x6]); - let node1 = BranchNodeCompact::new(0b1111, 0b0011, 0, vec![], None); - let node3 = BranchNodeCompact::new(0b1111, 0b1001, 0, vec![], None); - - let mut storage_nodes = BTreeMap::new(); - storage_nodes.insert(path1, node1.clone()); - storage_nodes.insert(path3, node3.clone()); - - let mut storage_tries = B256Map::default(); - storage_tries.insert(B256::default(), BTreeMap::new()); // For cursor creation - storage_tries.insert(hashed_address, storage_nodes); - - let factory = MockTrieCursorFactory::new(BTreeMap::new(), storage_tries); - - // Create updates that add a new path2 that didn't exist before - let path2 = Nibbles::from_nibbles([0x3, 0x4]); - let new_node2 = BranchNodeCompact::new(0b1111, 0b0101, 0, vec![], None); - - let mut storage_updates = B256Map::default(); - storage_updates.insert( - hashed_address, - StorageTrieUpdatesSorted { - is_deleted: true, - storage_nodes: vec![(path2, Some(new_node2))], - }, - ); - - let updates = TrieUpdatesSorted::new(vec![], storage_updates); - - // Compute changesets - let changesets = compute_trie_changesets(&factory, &updates).unwrap(); - - // Check storage changesets - let storage_changesets = changesets.storage_tries_ref().get(&hashed_address).unwrap(); - assert!(storage_changesets.is_deleted); - - // Should include all three paths: existing path1, new path2, existing path3 - assert_eq!(storage_changesets.storage_nodes.len(), 3); - - // Check sorted order - assert_eq!(storage_changesets.storage_nodes[0].0, path1); - assert_eq!(storage_changesets.storage_nodes[1].0, path2); - assert_eq!(storage_changesets.storage_nodes[2].0, path3); - - // path1 and path3 have old values, path2 has None (didn't exist) - assert_eq!(storage_changesets.storage_nodes[0].1, Some(node1)); - assert_eq!(storage_changesets.storage_nodes[1].1, None); - assert_eq!(storage_changesets.storage_nodes[2].1, Some(node3)); - } } diff --git a/crates/trie/trie/src/hashed_cursor/post_state.rs b/crates/trie/trie/src/hashed_cursor/post_state.rs index 3a821d68a66..a5e6e593347 100644 --- a/crates/trie/trie/src/hashed_cursor/post_state.rs +++ b/crates/trie/trie/src/hashed_cursor/post_state.rs @@ -111,22 +111,13 @@ enum DbCursorState { NeedsPosition, Positioned((B256, V)), Exhausted, - Wiped, } impl DbCursorState { - const fn new(cursor_wiped: bool) -> Self { - if cursor_wiped { - Self::Wiped - } else { - Self::NeedsPosition - } - } - const fn entry(&self) -> Option<&(B256, V)> { match self { Self::Positioned(entry) => Some(entry), - Self::NeedsPosition | Self::Exhausted | Self::Wiped => None, + Self::NeedsPosition | Self::Exhausted => None, } } @@ -168,11 +159,10 @@ where post_state: &'a HashedPostStateSorted, hashed_address: B256, ) -> Self { - let (post_state_cursor, cursor_wiped) = - Self::get_storage_overlay(post_state, hashed_address); + let post_state_cursor = Self::get_storage_overlay(post_state, hashed_address); Self { cursor, - db_cursor_state: DbCursorState::new(cursor_wiped), + db_cursor_state: DbCursorState::NeedsPosition, post_state_cursor, last_key: None, #[cfg(debug_assertions)] @@ -181,16 +171,15 @@ where } } - /// Returns the storage overlay for `hashed_address` and whether it was wiped. + /// Returns the storage overlay for `hashed_address`. fn get_storage_overlay( post_state: &'a HashedPostStateSorted, hashed_address: B256, - ) -> (ForwardInMemoryCursor<'a, B256, U256>, bool) { + ) -> ForwardInMemoryCursor<'a, B256, U256> { let post_state_storage = post_state.storages.get(&hashed_address); - let cursor_wiped = post_state_storage.is_some_and(|u| u.is_wiped()); let storage_slots = post_state_storage.map(|u| u.storage_slots_ref()).unwrap_or(&[]); - (ForwardInMemoryCursor::new(storage_slots), cursor_wiped) + ForwardInMemoryCursor::new(storage_slots) } } @@ -199,9 +188,8 @@ where C: HashedCursor, V: HashedPostStateCursorValue, { - /// Returns a mutable reference to the underlying cursor if it's not wiped, None otherwise. - fn get_cursor_mut(&mut self) -> Option<&mut C> { - (!matches!(self.db_cursor_state, DbCursorState::Wiped)).then_some(&mut self.cursor) + const fn get_cursor_mut(&mut self) -> &mut C { + &mut self.cursor } /// Asserts that the next entry to be returned from the cursor is not previous to the last entry @@ -225,11 +213,11 @@ where let should_seek = match &self.db_cursor_state { DbCursorState::NeedsPosition => true, DbCursorState::Positioned((entry_key, _)) => entry_key < &key, - DbCursorState::Exhausted | DbCursorState::Wiped => false, + DbCursorState::Exhausted => false, }; if should_seek { - let entry = self.get_cursor_mut().map(|c| c.seek(key)).transpose()?.flatten(); + let entry = self.get_cursor_mut().seek(key)?; self.db_cursor_state.set_entry(entry); } @@ -245,7 +233,7 @@ where // Exhausted DB state is stable; only advance when the DB cursor is positioned at an entry. if matches!(self.db_cursor_state, DbCursorState::Positioned(_)) { - let entry = self.get_cursor_mut().map(|c| c.next()).transpose()?.flatten(); + let entry = self.get_cursor_mut().next()?; self.db_cursor_state.set_entry(entry); } @@ -412,23 +400,18 @@ where /// This function should be called before attempting to call [`HashedCursor::seek`] or /// [`HashedCursor::next`]. fn is_storage_empty(&mut self) -> Result { - // Storage is not empty if it has non-zero slots. - if self.post_state_cursor.has_any(|(_, value)| !value.is_zero()) { - return Ok(false); - } - - // If no non-zero slots in post state, check the database. - // Returns true if cursor is wiped. - self.get_cursor_mut().map_or(Ok(true), |c| c.is_storage_empty()) + let is_empty = self.seek(B256::ZERO)?.is_none(); + self.reset(); + Ok(is_empty) } fn set_hashed_address(&mut self, hashed_address: B256) { self.reset(); self.cursor.set_hashed_address(hashed_address); - let (post_state_cursor, cursor_wiped) = + let post_state_cursor = HashedPostStateCursor::::get_storage_overlay(self.post_state, hashed_address); self.post_state_cursor = post_state_cursor; - self.db_cursor_state = DbCursorState::new(cursor_wiped); + self.db_cursor_state = DbCursorState::NeedsPosition; } } @@ -444,7 +427,7 @@ mod tests { } fn storage_post_state(storage_slots: Vec<(B256, U256)>) -> HashedPostStateSorted { - let storage_sorted = reth_trie_common::HashedStorageSorted { storage_slots, wiped: false }; + let storage_sorted = reth_trie_common::HashedStorageSorted { storage_slots }; let mut storages = alloy_primitives::map::B256Map::default(); storages.insert(B256::ZERO, storage_sorted); HashedPostStateSorted::new(Vec::new(), storages) @@ -630,7 +613,6 @@ mod tests { let hashed_address = B256::ZERO; let storage_sorted = reth_trie_common::HashedStorageSorted { storage_slots: post_state_nodes, - wiped: false, }; let mut storages = alloy_primitives::map::B256Map::default(); storages.insert(hashed_address, storage_sorted); diff --git a/crates/trie/trie/src/test_utils.rs b/crates/trie/trie/src/test_utils.rs index 3244d1d3051..1b904b68f88 100644 --- a/crates/trie/trie/src/test_utils.rs +++ b/crates/trie/trie/src/test_utils.rs @@ -271,12 +271,7 @@ impl TrieTestHarness { /// /// A storage node is redundant if it exists in the starting set with the same value. /// A removed node is redundant if it was already absent from the starting set. - /// The `is_deleted` flag is cleared if it matches the starting value. pub fn minimize_trie_updates(&self, updates: &mut StorageTrieUpdates) { - if updates.is_deleted == self.storage_trie_updates.is_deleted { - updates.is_deleted = false; - } - // StorageTrieUpdates::finalize can leave the same path in both storage_nodes // and removed_nodes. Per into_sorted, updated nodes take precedence over // removed ones. Record which paths had an update before minimization so we diff --git a/crates/trie/trie/src/trie_cursor/in_memory.rs b/crates/trie/trie/src/trie_cursor/in_memory.rs index d2316ac27a2..1d704cc4abf 100644 --- a/crates/trie/trie/src/trie_cursor/in_memory.rs +++ b/crates/trie/trie/src/trie_cursor/in_memory.rs @@ -74,22 +74,13 @@ enum DbCursorState { NeedsPosition, Positioned((Nibbles, BranchNodeCompact)), Exhausted, - Wiped, } impl DbCursorState { - const fn new(cursor_wiped: bool) -> Self { - if cursor_wiped { - Self::Wiped - } else { - Self::NeedsPosition - } - } - const fn entry(&self) -> Option<&(Nibbles, BranchNodeCompact)> { match self { Self::Positioned(entry) => Some(entry), - Self::NeedsPosition | Self::Exhausted | Self::Wiped => None, + Self::NeedsPosition | Self::Exhausted => None, } } @@ -123,11 +114,10 @@ impl<'a, C: TrieCursor> InMemoryTrieCursor<'a, C> { trie_updates: &'a TrieUpdatesSorted, hashed_address: B256, ) -> Self { - let (in_memory_cursor, cursor_wiped) = - Self::get_storage_overlay(trie_updates, hashed_address); + let in_memory_cursor = Self::get_storage_overlay(trie_updates, hashed_address); Self { cursor, - db_cursor_state: DbCursorState::new(cursor_wiped), + db_cursor_state: DbCursorState::NeedsPosition, in_memory_cursor, last_key: None, #[cfg(debug_assertions)] @@ -136,21 +126,19 @@ impl<'a, C: TrieCursor> InMemoryTrieCursor<'a, C> { } } - /// Returns the storage overlay for `hashed_address` and whether it was deleted. + /// Returns the storage overlay for `hashed_address`. fn get_storage_overlay( trie_updates: &'a TrieUpdatesSorted, hashed_address: B256, - ) -> (ForwardInMemoryCursor<'a, Nibbles, Option>, bool) { + ) -> ForwardInMemoryCursor<'a, Nibbles, Option> { let storage_trie_updates = trie_updates.storage_tries_ref().get(&hashed_address); - let cursor_wiped = storage_trie_updates.is_some_and(|u| u.is_deleted()); let storage_nodes = storage_trie_updates.map(|u| u.storage_nodes_ref()).unwrap_or(&[]); - (ForwardInMemoryCursor::new(storage_nodes), cursor_wiped) + ForwardInMemoryCursor::new(storage_nodes) } - /// Returns a mutable reference to the underlying cursor if it's not wiped, None otherwise. - fn get_cursor_mut(&mut self) -> Option<&mut C> { - (!matches!(self.db_cursor_state, DbCursorState::Wiped)).then_some(&mut self.cursor) + const fn get_cursor_mut(&mut self) -> &mut C { + &mut self.cursor } /// Asserts that the next entry to be returned from the cursor is not previous to the last entry @@ -174,11 +162,11 @@ impl<'a, C: TrieCursor> InMemoryTrieCursor<'a, C> { let should_seek = match &self.db_cursor_state { DbCursorState::NeedsPosition => true, DbCursorState::Positioned((entry_key, _)) => entry_key < &key, - DbCursorState::Exhausted | DbCursorState::Wiped => false, + DbCursorState::Exhausted => false, }; if should_seek { - let entry = self.get_cursor_mut().map(|c| c.seek(key)).transpose()?.flatten(); + let entry = self.get_cursor_mut().seek(key)?; self.db_cursor_state.set_entry(entry); } @@ -193,10 +181,10 @@ impl<'a, C: TrieCursor> InMemoryTrieCursor<'a, C> { debug_assert!(!matches!(self.db_cursor_state, DbCursorState::NeedsPosition)); } - // Exhausted and wiped states are stable; only advance if the DB cursor currently points to - // an entry. + // The exhausted state is stable; only advance if the DB cursor currently points to an + // entry. if matches!(self.db_cursor_state, DbCursorState::Positioned(_)) { - let entry = self.get_cursor_mut().map(|c| c.next()).transpose()?.flatten(); + let entry = self.get_cursor_mut().next()?; self.db_cursor_state.set_entry(entry); } @@ -364,7 +352,7 @@ impl TrieCursor for InMemoryTrieCursor<'_, C> { fn current(&mut self) -> Result, DatabaseError> { match &self.last_key { Some(key) => Ok(Some(*key)), - None => Ok(self.get_cursor_mut().map(|c| c.current()).transpose()?.flatten()), + None => self.get_cursor_mut().current(), } } @@ -385,10 +373,9 @@ impl TrieStorageCursor for InMemoryTrieCursor<'_, C> { fn set_hashed_address(&mut self, hashed_address: B256) { self.reset(); self.cursor.set_hashed_address(hashed_address); - let (in_memory_cursor, cursor_wiped) = - Self::get_storage_overlay(self.trie_updates, hashed_address); + let in_memory_cursor = Self::get_storage_overlay(self.trie_updates, hashed_address); self.in_memory_cursor = in_memory_cursor; - self.db_cursor_state = DbCursorState::new(cursor_wiped); + self.db_cursor_state = DbCursorState::NeedsPosition; } } @@ -805,12 +792,12 @@ mod tests { } #[test] - fn test_all_storage_slots_deleted_not_wiped_exact_keys() { + fn test_all_storage_nodes_deleted_exact_keys() { use tracing::debug; reth_tracing::init_test_tracing(); // This test reproduces an edge case where: - // - cursor is not None (not wiped) + // - cursor is available // - All in-memory entries are deletions (None values) // - Database has corresponding entries // - Expected: NO leaves should be returned (all deleted) diff --git a/crates/trie/trie/src/witness.rs b/crates/trie/trie/src/witness.rs index 41ea9a067cf..7901dd0c7fc 100644 --- a/crates/trie/trie/src/witness.rs +++ b/crates/trie/trie/src/witness.rs @@ -1,15 +1,11 @@ use crate::{ - hashed_cursor::{HashedCursor, HashedCursorFactory}, - prefix_set::TriePrefixSetsMut, - proof::Proof, - proof_v2, - trie_cursor::TrieCursorFactory, - TRIE_ACCOUNT_RLP_MAX_SIZE, + hashed_cursor::HashedCursorFactory, prefix_set::TriePrefixSetsMut, proof::Proof, proof_v2, + trie_cursor::TrieCursorFactory, TRIE_ACCOUNT_RLP_MAX_SIZE, }; use alloy_primitives::{ keccak256, map::{B256Map, HashMap}, - Bytes, B256, U256, + Bytes, B256, }; use alloy_rlp::{Encodable, EMPTY_STRING_CODE}; use alloy_trie::{nodes::BranchNodeRef, EMPTY_ROOT_HASH}; @@ -109,19 +105,12 @@ where /// # Arguments /// /// `state` - state transition containing both modified and touched accounts and storage slots. - pub fn compute( - mut self, - mut state: HashedPostState, - ) -> Result, TrieWitnessError> { + pub fn compute(mut self, state: HashedPostState) -> Result, TrieWitnessError> { let is_state_empty = state.is_empty(); if is_state_empty && !self.always_include_root_node { return Ok(Default::default()) } - // Expand wiped storages into explicit zero-value entries for every existing slot, - // so that downstream code can treat all storages uniformly. - self.expand_wiped_storages(&mut state)?; - let proof_targets = if is_state_empty { MultiProofTargetsV2 { account_targets: vec![ProofV2Target::new(B256::ZERO)], @@ -376,28 +365,8 @@ where Ok(root_hash) } - /// Expand wiped storages into explicit zero-value entries for every existing slot in the - /// database. After this, all storages can be treated uniformly without special wiped handling. - fn expand_wiped_storages(&self, state: &mut HashedPostState) -> Result<(), StateProofError> { - for (hashed_address, storage) in &mut state.storages { - if !storage.wiped { - continue; - } - let mut storage_cursor = - self.hashed_cursor_factory.hashed_storage_cursor(*hashed_address)?; - let mut current_entry = storage_cursor.seek(B256::ZERO)?; - while let Some((hashed_slot, _)) = current_entry { - storage.storage.entry(hashed_slot).or_insert(U256::ZERO); - current_entry = storage_cursor.next()?; - } - storage.wiped = false; - } - Ok(()) - } - /// Retrieve proof targets for incoming hashed state. - /// Aggregates all accounts and slots present in the state. Wiped storages must have been - /// expanded via [`Self::expand_wiped_storages`] before calling this. + /// Aggregates all accounts and slots present in the state. fn get_proof_targets(state: &HashedPostState) -> MultiProofTargetsV2 { let mut targets = MultiProofTargetsV2::default(); for &hashed_address in state.accounts.keys() { From 134089c2d30fdfbcf9e3bc26159c8b224e22ce6e Mon Sep 17 00:00:00 2001 From: Brian Picciano Date: Wed, 26 Aug 2026 13:07:03 +0000 Subject: [PATCH 070/294] refactor(trie): remove unused multi key tracking (#26822) --- crates/trie/common/src/added_removed_keys.rs | 175 ------------------- crates/trie/common/src/hashed_state.rs | 104 ----------- crates/trie/common/src/lib.rs | 2 - 3 files changed, 281 deletions(-) delete mode 100644 crates/trie/common/src/added_removed_keys.rs diff --git a/crates/trie/common/src/added_removed_keys.rs b/crates/trie/common/src/added_removed_keys.rs deleted file mode 100644 index 8b756c7b301..00000000000 --- a/crates/trie/common/src/added_removed_keys.rs +++ /dev/null @@ -1,175 +0,0 @@ -//! Tracking of keys having been added and removed from the tries. - -use crate::HashedPostState; -use alloy_primitives::{map::B256Map, B256}; -use alloy_trie::proof::AddedRemovedKeys; - -/// Tracks added and removed keys across account and storage tries. -#[derive(Debug, Clone)] -pub struct MultiAddedRemovedKeys { - /// Added and removed accounts. - pub account: AddedRemovedKeys, - /// Added and removed storage keys for each account. - pub storages: B256Map, -} - -/// Returns [`AddedRemovedKeys`] with default parameters. This is necessary while we are not yet -/// tracking added keys. -pub fn default_added_removed_keys() -> AddedRemovedKeys { - AddedRemovedKeys::default().with_assume_added(true) -} - -impl Default for MultiAddedRemovedKeys { - fn default() -> Self { - Self::new() - } -} - -impl MultiAddedRemovedKeys { - /// Returns a new instance. - pub fn new() -> Self { - Self { account: default_added_removed_keys(), storages: Default::default() } - } - - /// Updates the set of removed keys based on a [`HashedPostState`]. - /// - /// Storage keys set to [`alloy_primitives::U256::ZERO`] are added to the set for their - /// respective account. Keys set to any other value are removed from their respective - /// account. - pub fn update_with_state(&mut self, update: &HashedPostState) { - for (hashed_address, storage) in &update.storages { - let account = update - .accounts - .get(hashed_address) - .map(|entry| entry.unwrap_or_default()) - .unwrap_or_default(); - - let storage_removed_keys = - self.storages.entry(*hashed_address).or_insert_with(default_added_removed_keys); - - for (key, val) in &storage.storage { - if val.is_zero() { - storage_removed_keys.insert_removed(*key); - } else { - storage_removed_keys.remove_removed(key); - } - } - - if !account.is_empty() { - self.account.remove_removed(hashed_address); - } - } - } - - /// Returns a [`AddedRemovedKeys`] for the storage trie of a particular account, if any. - pub fn get_storage(&self, hashed_address: &B256) -> Option<&AddedRemovedKeys> { - self.storages.get(hashed_address) - } - - /// Returns an [`AddedRemovedKeys`] for tracking account-level changes. - pub const fn get_accounts(&self) -> &AddedRemovedKeys { - &self.account - } - - /// Marks an account as existing, and therefore having storage. - pub fn touch_accounts(&mut self, addresses: impl Iterator) { - for address in addresses { - self.storages.entry(address).or_insert_with(default_added_removed_keys); - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::HashedStorage; - use alloy_primitives::U256; - use reth_primitives_traits::Account; - - #[test] - fn test_update_with_state_storage_keys_non_zero() { - let mut multi_keys = MultiAddedRemovedKeys::new(); - let mut update = HashedPostState::default(); - - let addr = B256::random(); - let slot1 = B256::random(); - let slot2 = B256::random(); - - // First mark slots as removed - let mut storage = HashedStorage::default(); - storage.storage.insert(slot1, U256::ZERO); - storage.storage.insert(slot2, U256::ZERO); - update.storages.insert(addr, storage); - multi_keys.update_with_state(&update); - - // Verify they are removed - assert!(multi_keys.get_storage(&addr).unwrap().is_removed(&slot1)); - assert!(multi_keys.get_storage(&addr).unwrap().is_removed(&slot2)); - - // Now update with non-zero values - let mut update2 = HashedPostState::default(); - let mut storage2 = HashedStorage::default(); - storage2.storage.insert(slot1, U256::from(100)); - storage2.storage.insert(slot2, U256::from(200)); - update2.storages.insert(addr, storage2); - multi_keys.update_with_state(&update2); - - // Slots should no longer be marked as removed - let storage_keys = multi_keys.get_storage(&addr).unwrap(); - assert!(!storage_keys.is_removed(&slot1)); - assert!(!storage_keys.is_removed(&slot2)); - } - - #[test] - fn test_update_with_state_account_tracking() { - let mut multi_keys = MultiAddedRemovedKeys::new(); - let mut update = HashedPostState::default(); - - let addr = B256::random(); - let slot = B256::random(); - - // Add storage with zero value and empty account - let mut storage = HashedStorage::default(); - storage.storage.insert(slot, U256::ZERO); - update.storages.insert(addr, storage); - // Account is implicitly empty (not in accounts map) - - multi_keys.update_with_state(&update); - - // Storage should have removed keys but account should not be removed - assert!(multi_keys.get_storage(&addr).unwrap().is_removed(&slot)); - assert!(!multi_keys.get_accounts().is_removed(&addr)); - - // Now clear all removed storage keys and keep account empty - let mut update2 = HashedPostState::default(); - let mut storage2 = HashedStorage::default(); - storage2.storage.insert(slot, U256::from(100)); // Non-zero removes from removed set - update2.storages.insert(addr, storage2); - - multi_keys.update_with_state(&update2); - - // Account should not be marked as removed still - assert!(!multi_keys.get_accounts().is_removed(&addr)); - } - - #[test] - fn test_update_with_state_account_with_balance() { - let mut multi_keys = MultiAddedRemovedKeys::new(); - let mut update = HashedPostState::default(); - - let addr = B256::random(); - - // Add account with non-empty state (has balance) - let account = Account { balance: U256::from(1000), nonce: 0, bytecode_hash: None }; - update.accounts.insert(addr, Some(account)); - - // Add empty storage - let storage = HashedStorage::default(); - update.storages.insert(addr, storage); - - multi_keys.update_with_state(&update); - - // Account should not be marked as removed because it has balance - assert!(!multi_keys.get_accounts().is_removed(&addr)); - } -} diff --git a/crates/trie/common/src/hashed_state.rs b/crates/trie/common/src/hashed_state.rs index 987eb35e7b5..3a8176464e2 100644 --- a/crates/trie/common/src/hashed_state.rs +++ b/crates/trie/common/src/hashed_state.rs @@ -1,7 +1,4 @@ -use core::ops::Not; - use crate::{ - added_removed_keys::MultiAddedRemovedKeys, prefix_set::{PrefixSetMut, TriePrefixSetsMut}, utils::{extend_sorted_vec, kway_merge_disjoint_sorted, kway_merge_sorted}, KeyHasher, MultiProofTargets, Nibbles, @@ -169,61 +166,6 @@ impl HashedPostState { targets } - /// Partition the state update into two state updates: - /// - First with accounts and storages slots that are present in the provided targets. - /// - Second with all other. - pub fn partition_by_targets( - mut self, - targets: &MultiProofTargets, - added_removed_keys: &MultiAddedRemovedKeys, - ) -> (Self, Self) { - let mut state_updates_not_in_targets = Self::default(); - - self.storages.retain(|&address, storage| { - let storage_added_removed_keys = added_removed_keys.get_storage(&address); - - let (retain, storage_not_in_targets) = match targets.get(&address) { - Some(storage_in_targets) => { - let mut storage_not_in_targets = HashedStorage::default(); - storage.storage.retain(|&slot, value| { - if storage_in_targets.contains(&slot) && - !storage_added_removed_keys.is_some_and(|k| k.is_removed(&slot)) - { - return true - } - - storage_not_in_targets.storage.insert(slot, *value); - false - }); - - let retain = !storage.storage.is_empty(); - - ( - retain, - storage_not_in_targets.is_empty().not().then_some(storage_not_in_targets), - ) - } - None => (false, Some(core::mem::take(storage))), - }; - - if let Some(storage_not_in_targets) = storage_not_in_targets { - state_updates_not_in_targets.storages.insert(address, storage_not_in_targets); - } - - retain - }); - self.accounts.retain(|&address, account| { - if targets.contains_key(&address) { - return true - } - - state_updates_not_in_targets.accounts.insert(address, *account); - false - }); - - (self, state_updates_not_in_targets) - } - /// Returns an iterator that yields chunks of the specified size. /// /// See [`ChunkedHashedPostState`] for more information. @@ -1221,52 +1163,6 @@ mod tests { assert!(target_slots.contains(&slot2)); } - #[test] - fn test_partition_by_targets() { - let addr1 = B256::random(); - let addr2 = B256::random(); - let slot1 = B256::random(); - let slot2 = B256::random(); - - let state = HashedPostState { - accounts: B256Map::from_iter([ - (addr1, Some(Default::default())), - (addr2, Some(Default::default())), - ]), - storages: B256Map::from_iter([( - addr1, - HashedStorage { - storage: B256Map::from_iter([(slot1, U256::ZERO), (slot2, U256::from(1))]), - }, - )]), - }; - let targets = MultiProofTargets::from_iter([(addr1, HashSet::from_iter([slot1]))]); - - let (with_targets, without_targets) = - state.partition_by_targets(&targets, &MultiAddedRemovedKeys::new()); - - assert_eq!( - with_targets, - HashedPostState { - accounts: B256Map::from_iter([(addr1, Some(Default::default()))]), - storages: B256Map::from_iter([( - addr1, - HashedStorage { storage: B256Map::from_iter([(slot1, U256::ZERO)]) } - )]), - } - ); - assert_eq!( - without_targets, - HashedPostState { - accounts: B256Map::from_iter([(addr2, Some(Default::default()))]), - storages: B256Map::from_iter([( - addr1, - HashedStorage { storage: B256Map::from_iter([(slot2, U256::from(1))]) } - )]), - } - ); - } - #[test] fn test_hashed_post_state_sorted_extend_ref() { // Test extending accounts diff --git a/crates/trie/common/src/lib.rs b/crates/trie/common/src/lib.rs index 6f2408c06db..55e88dca1e7 100644 --- a/crates/trie/common/src/lib.rs +++ b/crates/trie/common/src/lib.rs @@ -83,8 +83,6 @@ pub mod ordered_root; /// Buffer for trie updates. pub mod updates; -pub mod added_removed_keys; - /// Utilities used by other modules in this crate. mod utils; From 6917e61f1ab14e9f70e8b98ff6fa0e1304cd3099 Mon Sep 17 00:00:00 2001 From: nk_ysg Date: Wed, 26 Aug 2026 13:14:12 +0000 Subject: [PATCH 071/294] perf(trie): lazily copy encoded witness bytes (#26809) --- crates/trie/trie/src/witness.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/crates/trie/trie/src/witness.rs b/crates/trie/trie/src/witness.rs index 7901dd0c7fc..1a78c512767 100644 --- a/crates/trie/trie/src/witness.rs +++ b/crates/trie/trie/src/witness.rs @@ -318,16 +318,16 @@ where fn record_witness_node(&mut self, node: &TrieNodeV2, encoded: &mut Vec) { encoded.clear(); node.encode(encoded); - let bytes = Bytes::from(encoded.clone()); - self.witness.entry(keccak256(&bytes)).or_insert(bytes); + let hash = keccak256(encoded.as_slice()); + self.witness.entry(hash).or_insert_with(|| Bytes::copy_from_slice(encoded)); if let TrieNodeV2::Branch(branch) = node && !branch.key.is_empty() { encoded.clear(); BranchNodeRef::new(&branch.stack, branch.state_mask).encode(encoded); - let bytes = Bytes::from(encoded.clone()); - self.witness.entry(keccak256(&bytes)).or_insert(bytes); + let hash = keccak256(encoded.as_slice()); + self.witness.entry(hash).or_insert_with(|| Bytes::copy_from_slice(encoded)); } } From f12d52de2569f61306b0a581972ac44da8e56dd1 Mon Sep 17 00:00:00 2001 From: nk_ysg Date: Wed, 26 Aug 2026 13:18:06 +0000 Subject: [PATCH 072/294] fix(txpool): reject zero transaction batch size (#26806) Co-authored-by: Matthias Seitz --- crates/node/core/src/args/txpool.rs | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/crates/node/core/src/args/txpool.rs b/crates/node/core/src/args/txpool.rs index 0dc2ad027f6..1843bf88996 100644 --- a/crates/node/core/src/args/txpool.rs +++ b/crates/node/core/src/args/txpool.rs @@ -3,7 +3,10 @@ use crate::cli::config::RethTransactionPoolConfig; use alloy_eips::eip1559::{ETHEREUM_BLOCK_GAS_LIMIT_30M, MIN_PROTOCOL_BASE_FEE}; use alloy_primitives::Address; -use clap::{builder::Resettable, Args}; +use clap::{ + builder::{RangedU64ValueParser, Resettable}, + Args, +}; use reth_cli_util::{parse_duration_from_secs_or_ms, parsers::format_duration_as_secs_or_ms}; use reth_transaction_pool::{ blobstore::disk::DEFAULT_MAX_CACHED_BLOBS, @@ -409,7 +412,7 @@ pub struct TxPoolArgs { pub disable_transactions_backup: bool, /// Max batch size for transaction pool insertions - #[arg(long = "txpool.max-batch-size", default_value_t = DefaultTxPoolValues::get_global().max_batch_size)] + #[arg(long = "txpool.max-batch-size", value_parser = RangedU64ValueParser::::new().range(1..), default_value_t = DefaultTxPoolValues::get_global().max_batch_size)] pub max_batch_size: usize, } @@ -589,6 +592,17 @@ mod tests { assert!(result.is_err(), "Expected an error for invalid duration"); } + #[test] + fn txpool_max_batch_size_must_be_nonzero() { + let zero = + CommandParser::::try_parse_from(["reth", "--txpool.max-batch-size", "0"]); + assert!(zero.is_err()); + + let valid = + CommandParser::::parse_from(["reth", "--txpool.max-batch-size", "1"]); + assert_eq!(valid.args.max_batch_size, 1); + } + #[test] fn txpool_args() { let args = TxPoolArgs { From 4523fb457e985682fc7037f138ef5ca17b275310 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Wed, 26 Aug 2026 15:47:53 +0000 Subject: [PATCH 073/294] feat(net): advertise eth/70 and eth/71 by default (#26824) --- crates/net/eth-wire-types/src/version.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/crates/net/eth-wire-types/src/version.rs b/crates/net/eth-wire-types/src/version.rs index 260698248eb..1720c8459e6 100644 --- a/crates/net/eth-wire-types/src/version.rs +++ b/crates/net/eth-wire-types/src/version.rs @@ -47,10 +47,11 @@ pub enum EthVersion { impl EthVersion { /// The latest known eth version - pub const LATEST: Self = Self::Eth69; + pub const LATEST: Self = Self::Eth71; /// All known eth versions - pub const ALL_VERSIONS: &'static [Self] = &[Self::Eth69, Self::Eth68, Self::Eth67, Self::Eth66]; + pub const ALL_VERSIONS: &'static [Self] = + &[Self::Eth71, Self::Eth70, Self::Eth69, Self::Eth68, Self::Eth67, Self::Eth66]; /// Returns true if the version is eth/66 pub const fn is_eth66(&self) -> bool { From 8666dcd9426f5485ec0e68019d69b84fed4d22dd Mon Sep 17 00:00:00 2001 From: Derek Cofausper <256792747+decofe@users.noreply.github.com> Date: Wed, 26 Aug 2026 16:20:51 +0000 Subject: [PATCH 074/294] feat(revm): support additional execution witness state (#26828) Co-authored-by: Alexey Shekhirin <5773434+shekhirin@users.noreply.github.com> Co-authored-by: Alexey Shekhirin --- crates/revm/src/witness.rs | 67 ++++++++++++++++++++++++++++++++++---- 1 file changed, 61 insertions(+), 6 deletions(-) diff --git a/crates/revm/src/witness.rs b/crates/revm/src/witness.rs index 8f105eef11e..e22efd2b4c3 100644 --- a/crates/revm/src/witness.rs +++ b/crates/revm/src/witness.rs @@ -4,16 +4,27 @@ use reth_trie::{ExecutionWitnessMode, HashedPostState}; use revm::database::State; /// Borrows finalized execution state for witness generation. -#[derive(Debug, Clone, Copy)] +#[derive(Debug, Clone)] pub struct ExecutionWitnessRecord<'a, DB> { /// State after execution. state: &'a State, + /// Additional hashed state to include in the witness. + additional_state: Option, } impl<'a, DB> ExecutionWitnessRecord<'a, DB> { /// Creates a new record from the state after execution. pub const fn new(state: &'a State) -> Self { - Self { state } + Self { state, additional_state: None } + } + + /// Adds hashed state that should be included when generating the witness. + /// + /// State recorded during execution takes precedence over additional state for overlapping + /// accounts and storage slots. + pub fn with_additional_state(mut self, additional_state: HashedPostState) -> Self { + self.additional_state.get_or_insert_default().extend(additional_state); + self } /// Converts this record into a complete [`alloy_rpc_types_debug::ExecutionWitness`] by @@ -66,14 +77,14 @@ impl<'a, DB> ExecutionWitnessRecord<'a, DB> { } }; + let lowest_block_number = + self.state.block_hashes.lowest().map(|(block_number, _)| block_number); let (hashed_state, keys) = self.hashed_post_state(state_provider)?; let state = state_provider.witness(Default::default(), hashed_state, mode)?; let mut exec_witness = alloy_rpc_types_debug::ExecutionWitness { state, codes, keys, ..Default::default() }; - let lowest_block_number = - self.state.block_hashes.lowest().map(|(block_number, _)| block_number); let smallest = lowest_block_number.unwrap_or_else(|| block_number.saturating_sub(1)); let range = smallest..block_number; @@ -92,13 +103,13 @@ impl<'a, DB> ExecutionWitnessRecord<'a, DB> { #[cfg(feature = "witness")] fn hashed_post_state( - &self, + self, state_provider: &SP, ) -> reth_storage_errors::provider::ProviderResult<(HashedPostState, Vec)> where SP: reth_storage_api::HashedPostStateProvider + ?Sized, { - let mut hashed_state = HashedPostState::default(); + let mut hashed_state = self.additional_state.unwrap_or_default(); let mut keys = Vec::new(); for (address, account) in &self.state.cache.accounts { let hashed_address = keccak256(address); @@ -154,6 +165,18 @@ mod tests { } } + #[derive(Debug)] + struct StaticStateProvider(HashedPostState); + + impl HashedPostStateProvider for StaticStateProvider { + fn hashed_post_state( + &self, + _bundle_state: &revm::database::BundleState, + ) -> ProviderResult { + Ok(self.0.clone()) + } + } + #[test] fn destroyed_account_storage_is_zero_expanded() { let address = Address::with_last_byte(1); @@ -184,4 +207,36 @@ mod tests { let storage = hashed_state.storages.get(&hashed_address).unwrap(); assert_eq!(storage.storage.get(&hashed_slot), Some(&U256::ZERO)); } + + #[test] + fn additional_state_is_merged_with_executed_state() { + let address = Address::with_last_byte(1); + let hashed_address = keccak256(address); + let slot = U256::from(1); + let additional_slot = B256::with_last_byte(2); + + let mut state = State::builder().with_database(EmptyDB::default()).build(); + let account = CacheAccount::new_loaded( + AccountInfo::default(), + core::iter::once((slot, U256::from(2))).collect(), + ); + state.cache.accounts.insert(address, account); + + let additional_state = HashedPostState::default().with_storages([( + hashed_address, + HashedStorage::from_iter([ + (keccak256(B256::from(slot)), U256::from(1)), + (additional_slot, U256::from(3)), + ]), + )]); + let provider = StaticStateProvider(HashedPostState::default()); + + let (hashed_state, _) = ExecutionWitnessRecord::new(&state) + .with_additional_state(additional_state) + .hashed_post_state(&provider) + .unwrap(); + let storage = &hashed_state.storages[&hashed_address].storage; + assert_eq!(storage[&keccak256(B256::from(slot))], U256::from(2)); + assert_eq!(storage[&additional_slot], U256::from(3)); + } } From 24f7cd94b06421d375b8cfe27b45a8a86d107b56 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Wed, 26 Aug 2026 17:02:05 +0000 Subject: [PATCH 075/294] fix(engine): validate BALs for downloaded blocks (#26831) --- .../engine/tree/src/tree/payload_validator.rs | 41 +++++-- crates/ethereum/consensus/src/lib.rs | 16 +++ crates/ethereum/consensus/src/validation.rs | 8 +- crates/ethereum/node/tests/e2e/p2p.rs | 105 +++++++++++++++++- 4 files changed, 154 insertions(+), 16 deletions(-) diff --git a/crates/engine/tree/src/tree/payload_validator.rs b/crates/engine/tree/src/tree/payload_validator.rs index 991a4963e2e..4fc02821f1c 100644 --- a/crates/engine/tree/src/tree/payload_validator.rs +++ b/crates/engine/tree/src/tree/payload_validator.rs @@ -106,7 +106,10 @@ use crate::tree::{ PayloadHandle, StateProviderBuilder, StateProviderDatabase, TreeConfig, WaitForCaches, }; use alloy_consensus::transaction::{Either, TxHashRef}; -use alloy_eip7928::{bal::DecodedBal, compute_block_access_list_hash_with_buf, BlockAccessList}; +use alloy_eip7928::{ + bal::{Bal, DecodedBal}, + BlockAccessList, +}; use alloy_eips::{eip1898::BlockWithParent, eip4895::Withdrawal, NumHash}; use alloy_evm::Evm; use alloy_primitives::{ @@ -1010,7 +1013,7 @@ where { debug!(target: "engine::tree::payload_validator", "Executing block"); - let has_bal = env.decoded_bal.is_some(); + let has_bal = input.has_block_access_list(); let mut db = debug_span!(target: "engine::tree", "build_state_db").in_scope(|| { State::builder() .with_database(StateProviderDatabase::new(state_provider)) @@ -1326,16 +1329,24 @@ where let _enter = debug_span!(target: "engine::tree::payload_validator", "validate_block_post_execution") .entered(); - let block_access_list_hash = built_bal + let built_bal = built_bal.map(Bal::from); + let block_access_list_hash = + built_bal.as_ref().map(|bal| bal.compute_hash_with_buf(&mut self.bal_hash_buf)); + + let validation_result = built_bal .as_ref() - .map(|bal| compute_block_access_list_hash_with_buf(bal, &mut self.bal_hash_buf)); - - if let Err(err) = self.consensus.validate_block_post_execution( - block, - output, - receipt_root_bloom, - block_access_list_hash, - ) { + .map(|bal| bal.validate_gas_limit(block.gas_limit()).map_err(ConsensusError::from)) + .transpose() + .and_then(|_| { + self.consensus.validate_block_post_execution( + block, + output, + receipt_root_bloom, + block_access_list_hash, + ) + }); + + if let Err(err) = validation_result { // call post-block hook self.on_invalid_block(parent_block, block, output, None, ctx.state_mut()); return Err(err.into()) @@ -2030,6 +2041,14 @@ impl BlockOrPayload { } } + /// Returns whether execution must build a block access list. + pub fn has_block_access_list(&self) -> bool { + match self { + Self::Payload(payload) => payload.block_access_list().is_some(), + Self::Block(block) => block.block_access_list_hash().is_some(), + } + } + /// Returns the number of transactions in the payload or block. pub fn transaction_count(&self) -> usize where diff --git a/crates/ethereum/consensus/src/lib.rs b/crates/ethereum/consensus/src/lib.rs index 8bc2c6badb1..1973babb2fc 100644 --- a/crates/ethereum/consensus/src/lib.rs +++ b/crates/ethereum/consensus/src/lib.rs @@ -542,4 +542,20 @@ mod tests { .validate_header(&SealedHeader::seal_slow(header,)) .is_ok()); } + + #[test] + fn amsterdam_post_execution_requires_computed_block_access_list_hash() { + let chain_spec = Arc::new(ChainSpecBuilder::mainnet().amsterdam_activated().build()); + let block = prague_recovered_block_with_bal_hash(B256::ZERO); + let result = BlockExecutionResult::::default(); + let consensus = EthBeaconConsensus::new(chain_spec); + + assert!(matches!( + FullConsensus::::validate_block_post_execution( + &consensus, &block, &result, None, None, + ) + .unwrap_err(), + ConsensusError::BlockAccessListHashMissing + )); + } } diff --git a/crates/ethereum/consensus/src/validation.rs b/crates/ethereum/consensus/src/validation.rs index 5acb8a02e29..d0c11ff1b94 100644 --- a/crates/ethereum/consensus/src/validation.rs +++ b/crates/ethereum/consensus/src/validation.rs @@ -110,8 +110,12 @@ where !chain_spec.is_amsterdam_active_at_timestamp(block.header().timestamp()) && block.header().block_access_list_hash().is_some(); - if (chain_spec.is_amsterdam_active_at_timestamp(block.header().timestamp()) || - is_allowed_pre_amsterdam_bal_hash) && + let is_amsterdam = chain_spec.is_amsterdam_active_at_timestamp(block.header().timestamp()); + if is_amsterdam && block_access_list_hash.is_none() { + return Err(ConsensusError::BlockAccessListHashMissing) + } + + if (is_amsterdam || is_allowed_pre_amsterdam_bal_hash) && let Some(block_access_list_hash) = block_access_list_hash { let block_bal_hash = block.header().block_access_list_hash().unwrap_or_default(); diff --git a/crates/ethereum/node/tests/e2e/p2p.rs b/crates/ethereum/node/tests/e2e/p2p.rs index 229db7fd508..e77d9e316e7 100644 --- a/crates/ethereum/node/tests/e2e/p2p.rs +++ b/crates/ethereum/node/tests/e2e/p2p.rs @@ -1,19 +1,27 @@ -use crate::utils::{advance_with_random_transactions, eth_payload_attributes}; +use crate::utils::{ + advance_with_random_transactions, eth_payload_attributes, eth_payload_attributes_amsterdam, +}; use alloy_consensus::{SignableTransaction, TxEip1559, TxEnvelope}; use alloy_eips::Encodable2718; use alloy_network::TxSignerSync; +use alloy_primitives::B256; use alloy_provider::{Provider, ProviderBuilder}; -use futures::future::JoinAll; +use alloy_rpc_types_engine::{ForkchoiceState, PayloadStatusEnum}; +use futures::{future::JoinAll, StreamExt}; use rand::{rngs::StdRng, seq::IndexedRandom, Rng, SeedableRng}; use reth_chainspec::{ChainSpecBuilder, MAINNET}; use reth_e2e_test_utils::{ setup, setup_engine, setup_engine_with_connection, transaction::TransactionTestContext, wallet::Wallet, }; -use reth_network::{NetworkInfo, PeersInfo}; +use reth_engine_primitives::ConsensusEngineEvent; +use reth_ethereum_primitives::EthPrimitives; +use reth_network::{test_utils::Testnet, NetworkInfo, Peers, PeersInfo}; use reth_node_builder::{NodeBuilder, NodeHandle}; use reth_node_core::{args::NetworkArgs, node_config::NodeConfig}; use reth_node_ethereum::EthereumNode; +use reth_primitives_traits::SealedBlock; +use reth_provider::test_utils::MockEthProvider; use reth_rpc_api::EthApiServer; use reth_tasks::Runtime; use std::{net::UdpSocket, sync::Arc, time::Duration}; @@ -134,6 +142,97 @@ async fn can_sync() -> eyre::Result<()> { Ok(()) } +#[tokio::test] +async fn rejects_downloaded_block_with_invalid_bal_hash() -> eyre::Result<()> { + reth_tracing::init_test_tracing(); + + let chain_spec = Arc::new( + ChainSpecBuilder::default() + .chain(MAINNET.chain) + .genesis(serde_json::from_str(include_str!("../assets/genesis.json")).unwrap()) + .amsterdam_activated() + .build(), + ); + + let (mut nodes, wallet) = setup_engine::( + 1, + chain_spec.clone(), + false, + Default::default(), + eth_payload_attributes_amsterdam, + ) + .await?; + let mut node = nodes.pop().unwrap(); + + // Build a valid Amsterdam block without submitting it to the engine, then change only its BAL + // commitment so all other execution-derived header fields remain valid. + let raw_tx = TransactionTestContext::transfer_tx_bytes(1, wallet.inner).await; + node.rpc.inject_tx(raw_tx).await?; + let payload = node.new_payload().await?; + let mut invalid_block = payload.block().clone().into_block(); + let valid_bal_hash = invalid_block.header.block_access_list_hash.unwrap(); + invalid_block.header.block_access_list_hash = Some(B256::ZERO); + assert_ne!(valid_bal_hash, B256::ZERO); + + let invalid_block = SealedBlock::seal_slow(invalid_block); + let invalid_hash = invalid_block.hash(); + let genesis_hash = node.block_hash(0); + + // Serve the malformed block from a real eth protocol peer so the node must fetch it after the + // forkchoice update instead of receiving it through `newPayload`. + let peer_provider = Arc::new( + MockEthProvider::::new() + .with_chain_spec((*chain_spec).clone()) + .with_genesis_block(), + ); + peer_provider.add_block(invalid_hash, invalid_block.into_block()); + + let mut testnet = Testnet::create_with(1, peer_provider).await; + testnet.for_each_mut(|peer| peer.install_request_handler()); + let peer = testnet.peers()[0].handle(); + let peer_id = *peer.peer_id(); + let peer_addr = peer.local_addr(); + let _testnet = testnet.spawn(); + + node.inner.network.add_peer(peer_id, peer_addr); + assert_eq!(node.network.next_session_established().await, Some(peer_id)); + + let mut engine_events = node.inner.add_ons_handle.consensus_engine_events().new_listener(); + let state = ForkchoiceState { + head_block_hash: invalid_hash, + safe_block_hash: genesis_hash, + finalized_block_hash: genesis_hash, + }; + let response = + node.inner.add_ons_handle.beacon_engine_handle.fork_choice_updated(state, None).await?; + assert_eq!(response.payload_status.status, PayloadStatusEnum::Syncing); + + let error = tokio::time::timeout(Duration::from_secs(20), async { + while let Some(event) = engine_events.next().await { + match event { + ConsensusEngineEvent::InvalidBlock { block, error } + if block.hash() == invalid_hash => + { + return error + } + ConsensusEngineEvent::CanonicalBlockAdded(block, _) + if block.recovered_block().hash() == invalid_hash => + { + panic!("block with invalid BAL hash became canonical") + } + _ => {} + } + } + panic!("consensus engine event stream ended") + }) + .await?; + + assert!(error.contains("block access list hash mismatch"), "{error}"); + assert_eq!(node.current_forkchoice_state()?.head_block_hash, genesis_hash); + + Ok(()) +} + #[tokio::test] async fn e2e_test_send_transactions() -> eyre::Result<()> { reth_tracing::init_test_tracing(); From 23305f9d98e2424da527fdd8a46f762a18d9de3f Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Thu, 27 Aug 2026 03:57:00 +0200 Subject: [PATCH 076/294] feat(rpc): add recovered block lookup with cached BAL (#26832) --- crates/rpc/rpc-eth-types/src/cache/mod.rs | 77 ++++++++++++++++++++++- 1 file changed, 75 insertions(+), 2 deletions(-) diff --git a/crates/rpc/rpc-eth-types/src/cache/mod.rs b/crates/rpc/rpc-eth-types/src/cache/mod.rs index cdbb33e4a1f..2be7e50b45c 100644 --- a/crates/rpc/rpc-eth-types/src/cache/mod.rs +++ b/crates/rpc/rpc-eth-types/src/cache/mod.rs @@ -48,6 +48,8 @@ type ReceiptsResponseSender = oneshot::Sender = oneshot::Sender>>>; +type CachedBalResponseSender = oneshot::Sender>; + type CachedBlockAndReceiptsResponseSender = oneshot::Sender<(Option>>, Option>>)>; @@ -192,6 +194,25 @@ impl EthStateCache { Ok(block.zip(receipts)) } + /// Retrieves the block and its BAL if the BAL is already cached. + /// + /// The block is fetched if it is not cached. Returns `None` if the block does not exist. + pub async fn get_recovered_block_and_maybe_bal( + &self, + block_hash: B256, + ) -> ProviderResult< + Option<(Arc>, Option>>>)>, + > { + let (response_tx, rx) = oneshot::channel(); + let _ = self.to_service.send(CacheAction::GetCachedBal { block_hash, response_tx }); + + let block = self.get_recovered_block(block_hash); + let (block, bal) = futures::join!(block, rx); + + let bal = bal.map_err(|_| CacheServiceUnavailable)?.map(|cached| cached.0); + Ok(block?.map(|block| (block, bal))) + } + /// Retrieves receipts and blocks from cache if block is in the cache, otherwise only receipts. pub async fn get_receipts_and_maybe_block( &self, @@ -520,6 +541,9 @@ where let _ = response_tx.send(this.full_block_cache.get(&block_hash).cloned()); } + CacheAction::GetCachedBal { block_hash, response_tx } => { + let _ = response_tx.send(this.bal_cache.get(&block_hash).cloned()); + } CacheAction::GetCachedBlockAndReceipts { block_hash, response_tx } => { let block = this.full_block_cache.get(&block_hash).cloned(); let receipts = this.receipts_cache.get(&block_hash).cloned(); @@ -760,6 +784,10 @@ enum CacheAction { block_hash: B256, response_tx: CachedBlockResponseSender, }, + GetCachedBal { + block_hash: B256, + response_tx: CachedBalResponseSender, + }, GetCachedBlockAndReceipts { block_hash: B256, response_tx: CachedBlockAndReceiptsResponseSender, @@ -1193,6 +1221,45 @@ mod tests { assert_eq!(fetches.load(Ordering::SeqCst), 1); } + #[tokio::test] + async fn get_recovered_block_and_maybe_bal_does_not_fetch_bal() { + let bal_fetches = Arc::new(AtomicUsize::default()); + let block = test_block(); + let block_hash = block.hash(); + let provider = TestBalProvider::new(bal_fetches.clone()).with_block(block); + let cache = EthStateCache::::spawn_with( + provider, + EthStateCacheConfig { + max_blocks: 4, + max_receipts: 0, + max_headers: 0, + max_bals: 4, + max_concurrent_db_requests: 1, + max_cached_tx_hashes: 0, + }, + Runtime::test(), + ); + + let (returned_block, bal) = cache + .get_recovered_block_and_maybe_bal(block_hash) + .await + .unwrap() + .expect("block exists"); + assert_eq!(returned_block.hash(), block_hash); + assert!(bal.is_none()); + assert_eq!(bal_fetches.load(Ordering::SeqCst), 0); + + assert!(cache.get_bal(block_hash).await.unwrap().is_some()); + + let (_, bal) = cache + .get_recovered_block_and_maybe_bal(block_hash) + .await + .unwrap() + .expect("block exists"); + assert!(bal.is_some()); + assert_eq!(bal_fetches.load(Ordering::SeqCst), 1); + } + #[tokio::test] async fn concurrent_get_bal_requests_share_fetch() { let fetches = Arc::new(AtomicUsize::default()); @@ -1221,11 +1288,17 @@ mod tests { #[derive(Clone, Debug, Default)] struct TestBalProvider { bal_store: BalStoreHandle, + block: Option>, } impl TestBalProvider { fn new(fetches: Arc) -> Self { - Self { bal_store: BalStoreHandle::new(TestBalStore { fetches }) } + Self { bal_store: BalStoreHandle::new(TestBalStore { fetches }), block: None } + } + + fn with_block(mut self, block: RecoveredBlock) -> Self { + self.block = Some(block); + self } } @@ -1471,7 +1544,7 @@ mod tests { _id: BlockHashOrNumber, _transaction_kind: TransactionVariant, ) -> ProviderResult>> { - Ok(None) + Ok(self.block.clone()) } fn block_range( From 3eb0f03a08cd1fa0aa079443fe5c5e0ca8b6efdf Mon Sep 17 00:00:00 2001 From: Brian Picciano Date: Thu, 27 Aug 2026 10:06:42 +0000 Subject: [PATCH 077/294] fix(engine): base persistence threshold on in-memory chain (#26827) --- crates/engine/tree/src/tree/mod.rs | 4 +-- crates/engine/tree/src/tree/tests.rs | 43 +++++++++++++++++++--------- 2 files changed, 32 insertions(+), 15 deletions(-) diff --git a/crates/engine/tree/src/tree/mod.rs b/crates/engine/tree/src/tree/mod.rs index cc8c001f653..889fa7c3367 100644 --- a/crates/engine/tree/src/tree/mod.rs +++ b/crates/engine/tree/src/tree/mod.rs @@ -2273,8 +2273,8 @@ where return None } - if canonical_head_number.saturating_sub(prev_db_tip) <= - self.config.persistence_threshold() + if self.canonical_in_memory_state.canonical_chain().count() <= + self.config.persistence_threshold() as usize { return None } diff --git a/crates/engine/tree/src/tree/tests.rs b/crates/engine/tree/src/tree/tests.rs index 14421f40a05..2866f4fae5c 100644 --- a/crates/engine/tree/src/tree/tests.rs +++ b/crates/engine/tree/src/tree/tests.rs @@ -20,7 +20,7 @@ use alloy_rpc_types_engine::{ ForkchoiceUpdateError, }; use assert_matches::assert_matches; -use reth_chain_state::{test_utils::TestBlockBuilder, BlockState}; +use reth_chain_state::test_utils::TestBlockBuilder; use reth_chainspec::{ChainSpec, HOLESKY, MAINNET}; use reth_engine_primitives::{EngineApiValidator, ForkchoiceStatus, NoopInvalidBlockHook}; use reth_ethereum_consensus::EthBeaconConsensus; @@ -272,8 +272,6 @@ impl TestHarness { fn with_blocks(mut self, blocks: Vec) -> Self { let mut blocks_by_hash = B256Map::default(); let mut blocks_by_number = BTreeMap::new(); - let mut state_by_hash = B256Map::default(); - let mut hash_by_number = BTreeMap::new(); let mut parent_to_child: B256Map = B256Map::default(); let mut parent_hash = B256::ZERO; @@ -283,8 +281,6 @@ impl TestHarness { let number = sealed_block.number; blocks_by_hash.insert(hash, block.clone()); blocks_by_number.entry(number).or_insert_with(Vec::new).push(block.clone()); - state_by_hash.insert(hash, Arc::new(BlockState::new(block.clone()))); - hash_by_number.insert(number, hash); parent_to_child.entry(parent_hash).or_default().insert(hash); parent_hash = hash; } @@ -303,10 +299,11 @@ impl TestHarness { overlay_manager, }; - let last_executed_block = blocks.last().unwrap().clone(); - let pending = Some(BlockState::new(last_executed_block)); - self.tree.canonical_in_memory_state = - CanonicalInMemoryState::new(state_by_hash, hash_by_number, pending, None, None); + let canonical_in_memory_state = CanonicalInMemoryState::empty(); + canonical_in_memory_state.update_chain(NewCanonicalChain::Commit { new: blocks.clone() }); + canonical_in_memory_state + .set_canonical_head(blocks.last().unwrap().recovered_block().clone_sealed_header()); + self.tree.canonical_in_memory_state = canonical_in_memory_state; self.blocks = blocks.clone(); @@ -917,18 +914,16 @@ async fn test_in_memory_state_trait_impl() { for executed_block in blocks { let sealed_block = executed_block.recovered_block(); - let expected_state = BlockState::new(executed_block.clone()); - let actual_state_by_hash = test_harness.tree.canonical_in_memory_state.state_by_hash(sealed_block.hash()).unwrap(); - assert_eq!(expected_state, *actual_state_by_hash); + assert_eq!(executed_block, *actual_state_by_hash.block_ref()); let actual_state_by_number = test_harness .tree .canonical_in_memory_state .state_by_number(sealed_block.number) .unwrap(); - assert_eq!(expected_state, *actual_state_by_number); + assert_eq!(executed_block, *actual_state_by_number.block_ref()); } } @@ -1202,6 +1197,7 @@ async fn test_tree_state_on_new_head_reorg() { assert_eq!(new.len(), 2); assert_eq!(new[0].recovered_block().hash(), blocks[3].recovered_block().hash()); assert_eq!(new[1].recovered_block().hash(), blocks[4].recovered_block().hash()); + test_harness.tree.canonical_in_memory_state.update_chain(NewCanonicalChain::Commit { new }); } // should be a None persistence action before we advance persistence @@ -1420,6 +1416,27 @@ async fn test_get_canonical_blocks_to_persist() { ); } +#[test] +fn threshold_persistence_uses_canonical_in_memory_chain_length() { + let blocks: Vec<_> = TestBlockBuilder::eth().get_executed_blocks(0..10).collect(); + let mut test_harness = TestHarness::new(MAINNET.clone()).with_blocks(blocks.clone()); + test_harness.tree.config = TreeConfig::default() + .with_persistence_threshold(3) + .with_memory_block_buffer_target(0) + .with_num_state_masking_blocks(2); + test_harness.tree.persistence_state.last_persisted_block = + blocks[7].recovered_block().num_hash(); + test_harness.tree.persistence_state.last_state_trie_persisted_block = + blocks[5].recovered_block().num_hash(); + test_harness.tree.canonical_in_memory_state.remove_persisted_blocks_until( + blocks[7].recovered_block().num_hash(), + blocks[5].recovered_block().number(), + ); + + assert_eq!(test_harness.tree.canonical_in_memory_state.canonical_chain().count(), 4); + assert!(test_harness.tree.get_save_blocks_input(PersistTarget::Threshold).is_some()); +} + #[test] fn test_threshold_persistence_with_state_masking_blocks() { let mut test_harness = TestHarness::new(MAINNET.clone()); From 444d9fda59830148de83ea9856bd23e49a418de9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A9a=20Narzis?= <78718413+lean-apple@users.noreply.github.com> Date: Thu, 27 Aug 2026 10:26:02 +0000 Subject: [PATCH 078/294] feat(downloader): add authenticated snap storage ranges (#26814) --- crates/net/downloaders/src/snap/mod.rs | 511 ++++----- crates/net/downloaders/src/snap/request.rs | 179 +++ crates/net/downloaders/src/snap/storage.rs | 1003 +++++++++++++++++ crates/net/downloaders/src/snap/test_utils.rs | 113 ++ 4 files changed, 1514 insertions(+), 292 deletions(-) create mode 100644 crates/net/downloaders/src/snap/request.rs create mode 100644 crates/net/downloaders/src/snap/storage.rs create mode 100644 crates/net/downloaders/src/snap/test_utils.rs diff --git a/crates/net/downloaders/src/snap/mod.rs b/crates/net/downloaders/src/snap/mod.rs index 2c037ba3b95..dc76caff111 100644 --- a/crates/net/downloaders/src/snap/mod.rs +++ b/crates/net/downloaders/src/snap/mod.rs @@ -1,50 +1,41 @@ -//! Downloads and verifies snap/2 account ranges against +//! Downloads and verifies snap/2 ranges against //! [EIP-8189](https://eips.ethereum.org/EIPS/eip-8189) pivot state roots. //! //! Persistence and range selection are handled by the snap sync orchestrator. use alloy_primitives::B256; -use futures::{Future, FutureExt}; -use reth_eth_wire_types::snap::{AccountRangeMessage, GetAccountRangeMessage}; +use futures::Future; +use reth_eth_wire_types::snap::{ + AccountRangeMessage, GetAccountRangeMessage, GetStorageRangesMessage, +}; use reth_network_p2p::{ - error::{PeerRequestResult, RequestError}, - priority::Priority, + error::RequestError, snap::client::{SnapClient, SnapResponse}, }; use reth_network_peers::PeerId; use reth_tasks::Runtime; use reth_trie_common::{range_proof::verify_range_proof, TrieAccount, EMPTY_ROOT_HASH}; use std::{ + ops::Range, pin::Pin, - task::{ready, Context, Poll}, + task::{Context, Poll}, }; use tracing::debug; -const MAX_RETRIES: u8 = 2; +mod request; +mod storage; +#[cfg(test)] +mod test_utils; + +use request::{SnapVerifier, VerifyingRequest}; +pub use storage::*; /// Downloads and verifies one account range against its requested state root. /// /// Invalid responses penalize their peer and retry at high priority. Proof verification runs on /// the blocking pool. -pub struct AccountRangeDownloader { - client: C, - runtime: Runtime, - request: GetAccountRangeMessage, - fut: C::Output, - verification: Option, - retries: u8, -} - -impl std::fmt::Debug for AccountRangeDownloader { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("AccountRangeDownloader") - .field("client", &self.client) - .field("request", &self.request) - .field("verification", &self.verification) - .field("retries", &self.retries) - .finish_non_exhaustive() - } -} +#[derive(Debug)] +pub struct AccountRangeDownloader(VerifyingRequest); impl AccountRangeDownloader { /// Creates a downloader using `runtime` for proof verification and submits the initial request. @@ -60,116 +51,8 @@ impl AccountRangeDownloader { limit: request.limit_hash, }) } - let fut = client.get_account_range(request.clone()); - Ok(Self { client, runtime, request, fut, verification: None, retries: 0 }) - } - - // Raise retry priority so transient failures cannot leave range progress behind new work. - fn retry(&mut self) -> bool { - if self.retries >= MAX_RETRIES { - return false - } - self.retries += 1; - self.fut = - self.client.get_account_range_with_priority(self.request.clone(), Priority::High); - true - } - - // Verify peer-controlled proofs off the async worker while retaining peer attribution. - fn start_verification( - &mut self, - peer_id: PeerId, - response: SnapResponse, - ) -> Result, RequestError> { - let response = self.account_range_response(response)?; - - if response.accounts.is_empty() && response.proof.is_empty() { - return if self.request.root_hash == EMPTY_ROOT_HASH { - Ok(Some(AccountRangeOutcome::Verified(VerifiedAccountRange { - accounts: Vec::new(), - has_more: false, - next: None, - }))) - } else { - Ok(Some(AccountRangeOutcome::Unavailable { peer_id })) - } - } - - let request = self.request.clone(); - let fut = self.runtime.spawn_blocking(move || verify_account_range(&request, response)); - self.verification = Some(VerificationTask { peer_id, fut }); - Ok(None) - } - - // Bind replies to the expected request before trusting peer-supplied data. - fn account_range_response( - &self, - response: SnapResponse, - ) -> Result { - let SnapResponse::AccountRange(response) = response else { - debug!(target: "downloaders::snap", "Expected account range response"); - return Err(RequestError::BadResponse) - }; - if response.request_id != self.request.request_id { - debug!( - target: "downloaders::snap", - expected = self.request.request_id, - got = response.request_id, - "Account range response id mismatch" - ); - return Err(RequestError::BadResponse) - } - Ok(response) - } - - // Keep validation and retry accounting together so peers are penalized exactly once. - fn handle_response( - &mut self, - response: PeerRequestResult, - ) -> Result, RequestError> { - match response { - Ok(response) => { - let (peer_id, response) = response.split(); - match self.start_verification(peer_id, response) { - Ok(outcome) => Ok(outcome), - Err(error) => { - debug!(target: "downloaders::snap", ?peer_id, %error, "Invalid account range response"); - self.client.report_bad_message(peer_id); - self.retry().then_some(None).ok_or(error) - } - } - } - // A wrong wire response is already penalized by the session. - Err(error) if error.is_retryable() || error == RequestError::BadResponse => { - debug!(target: "downloaders::snap", %error, "Account range request failed, retrying"); - self.retry().then_some(None).ok_or(error) - } - Err(error) => Err(error), - } - } - - // Preserve peer attribution until blocking verification finishes. - fn poll_verification( - &mut self, - cx: &mut Context<'_>, - ) -> Poll, RequestError>> { - let verification = self.verification.as_mut().expect("verification task is present"); - let result = ready!(verification.fut.poll_unpin(cx)); - let peer_id = verification.peer_id; - self.verification = None; - - match result { - Ok(Ok(range)) => Poll::Ready(Ok(Some(AccountRangeOutcome::Verified(range)))), - Ok(Err(error)) => { - debug!(target: "downloaders::snap", ?peer_id, %error, "Invalid account range response"); - self.client.report_bad_message(peer_id); - Poll::Ready(self.retry().then_some(None).ok_or(error)) - } - Err(error) => { - debug!(target: "downloaders::snap", %error, "Account range verification task failed"); - Poll::Ready(Err(RequestError::ChannelClosed)) - } - } + let verifier = request.clone(); + Ok(Self(VerifyingRequest::new(client, request, verifier, runtime))) } } @@ -179,26 +62,8 @@ where { type Output = Result; - // Finish an active verification before accepting another response. fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { - let this = self.get_mut(); - - loop { - if this.verification.is_some() { - match ready!(this.poll_verification(cx)) { - Ok(Some(outcome)) => return Poll::Ready(Ok(outcome)), - Ok(None) => {} - Err(error) => return Poll::Ready(Err(error)), - } - } - - let response = ready!(this.fut.poll_unpin(cx)); - match this.handle_response(response) { - Ok(Some(outcome)) => return Poll::Ready(Ok(outcome)), - Ok(None) => {} - Err(error) => return Poll::Ready(Err(error)), - } - } + self.get_mut().0.poll_verified(cx) } } @@ -214,32 +79,168 @@ pub enum AccountRangeOutcome { Verified(VerifiedAccountRange), } -// Couples blocking proof work with its responder so failures remain attributable. -#[derive(Debug)] -struct VerificationTask { - // Identifies the responder to penalize if verification rejects the range. - peer_id: PeerId, - // Carries the verified result back without blocking the async worker. - fut: tokio::task::JoinHandle>, -} - /// A decoded account range authenticated against a state root. #[derive(Clone, Debug, PartialEq, Eq)] pub struct VerifiedAccountRange { + // Root the accounts were proven against. Private so a range cannot be relabelled with a root + // that did not authenticate it. + state_root: B256, + // Accounts as the response returned them, in the order every positional check assumes. + accounts: Vec<(B256, TrieAccount)>, + // Whether the requested interval may continue past this response. + has_more: bool, + // First key after the response, or none when the range ran out of trie. + next: Option, +} + +impl VerifiedAccountRange { + /// State root the accounts were authenticated against. + pub const fn state_root(&self) -> B256 { + self.state_root + } + /// Accounts in strictly increasing hashed-key order. - pub accounts: Vec<(B256, TrieAccount)>, - /// Whether another request may be needed to complete the requested interval. + pub fn accounts(&self) -> &[(B256, TrieAccount)] { + &self.accounts + } + + /// Whether another request may be needed to complete the interval. /// - /// Conservative: [`Self::next`] is only a lower bound when the trie continues inside a - /// subtree the proof left unexpanded, so this can be `true` for an interval that is already - /// complete. - pub has_more: bool, + /// Conservative: can be `true` for an interval that is already complete. + pub const fn has_more(&self) -> bool { + self.has_more + } + /// Authenticated lower bound for the first key after the response, or `None` when the range /// exhausted the trie. - pub next: Option, + pub const fn next(&self) -> Option { + self.next + } + + /// Borrows the accounts together with the root that authenticated them. + pub fn batch(&self) -> VerifiedAccountBatch<'_> { + VerifiedAccountBatch { state_root: self.state_root, accounts: &self.accounts } + } + + /// Borrows a positional subrange of the accounts, so a caller can request storage in bounded + /// chunks without losing the root that authenticated them. + /// + /// `None` when the range falls outside the response. + pub fn batch_range(&self, range: Range) -> Option> { + self.accounts + .get(range) + .map(|accounts| VerifiedAccountBatch { state_root: self.state_root, accounts }) + } +} + +/// Accounts and the state root they were authenticated against. +/// +/// Only obtainable from [`VerifiedAccountRange::batch`], so requests built from it can always be +/// checked against the root the accounts came from. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct VerifiedAccountBatch<'a> { + // Root and accounts travel together, so neither can be swapped for another generation's. + state_root: B256, + // Accounts the root authenticated, in response order. + accounts: &'a [(B256, TrieAccount)], } -// Authenticate the full response before trimming its optional boundary account. +impl<'a> VerifiedAccountBatch<'a> { + /// State root the accounts were authenticated against. + pub const fn state_root(&self) -> B256 { + self.state_root + } + + /// Accounts in the order the range returned them. + pub const fn accounts(&self) -> &'a [(B256, TrieAccount)] { + self.accounts + } + + // Confirms the batch is the one `request` was built from, so every returned range is checked + // against the root that authenticated its account. + pub(super) fn verify_batch( + &self, + request: &GetStorageRangesMessage, + ) -> Result<(), InvalidStorageRangeRequest> { + if request.root_hash != self.state_root { + return Err(InvalidStorageRangeRequest::StateRootMismatch { + requested: request.root_hash, + authenticated: self.state_root, + }) + } + if request.account_hashes.len() != self.accounts.len() { + return Err(InvalidStorageRangeRequest::AccountCount { + requested: request.account_hashes.len(), + supplied: self.accounts.len(), + }) + } + for (index, (requested, (supplied, _))) in + request.account_hashes.iter().zip(self.accounts).enumerate() + { + if requested != supplied { + return Err(InvalidStorageRangeRequest::AccountMismatch { + index, + requested: *requested, + supplied: *supplied, + }) + } + } + Ok(()) + } + + // The accounts from `from` onwards under the same state root, or none if the batch is shorter. + pub(super) fn slice(&self, from: usize) -> Option { + self.accounts.get(from..).map(|accounts| Self { state_root: self.state_root, accounts }) + } +} + +/// An account-range request whose origin exceeds its limit. +#[derive(Clone, Copy, Debug, PartialEq, Eq, thiserror::Error)] +#[error("account range origin {origin} exceeds limit {limit}")] +pub struct InvalidAccountRange { + /// Inclusive origin the range was requested from. + pub origin: B256, + /// Inclusive limit the range was requested to. + pub limit: B256, +} + +// The request itself carries everything needed to authenticate its response. +impl SnapVerifier for GetAccountRangeMessage { + type Request = Self; + type Output = AccountRangeOutcome; + + fn verify(self, peer_id: PeerId, response: SnapResponse) -> Result { + let SnapResponse::AccountRange(response) = response else { + debug!(target: "downloaders::snap", "Expected account range response"); + return Err(RequestError::BadResponse) + }; + if response.request_id != self.request_id { + debug!( + target: "downloaders::snap", + expected = self.request_id, + got = response.request_id, + "Account range response id mismatch" + ); + return Err(RequestError::BadResponse) + } + if response.accounts.is_empty() && response.proof.is_empty() { + return if self.root_hash == EMPTY_ROOT_HASH { + Ok(AccountRangeOutcome::Verified(VerifiedAccountRange { + state_root: self.root_hash, + accounts: Vec::new(), + has_more: false, + next: None, + })) + } else { + Ok(AccountRangeOutcome::Unavailable { peer_id }) + } + } + + verify_account_range(&self, response).map(AccountRangeOutcome::Verified) + } +} + +// Authenticates the full response before trimming its optional boundary account. fn verify_account_range( request: &GetAccountRangeMessage, response: AccountRangeMessage, @@ -267,10 +268,10 @@ fn verify_account_range( accounts.truncate(accounts.partition_point(|(hash, _)| *hash <= request.limit_hash)); let has_more = next.is_some_and(|next| next <= request.limit_hash); - Ok(VerifiedAccountRange { accounts, has_more, next }) + Ok(VerifiedAccountRange { state_root: request.root_hash, accounts, has_more, next }) } -// Re-encode decoded accounts so the proof authenticates their canonical trie values. +// Re-encodes decoded accounts so the proof authenticates their canonical trie values. fn verify_proof( request: &GetAccountRangeMessage, accounts: &[(B256, TrieAccount)], @@ -284,111 +285,18 @@ fn verify_proof( }) } -/// An account-range request whose origin exceeds its limit. -#[derive(Clone, Copy, Debug, PartialEq, Eq, thiserror::Error)] -#[error("account range origin {origin} exceeds limit {limit}")] -pub struct InvalidAccountRange { - /// Inclusive origin the range was requested from. - pub origin: B256, - /// Inclusive limit the range was requested to. - pub limit: B256, -} - #[cfg(test)] mod tests { - use super::*; + use super::{request::MAX_RETRIES, test_utils::TestSnapClient, *}; use alloy_primitives::{Bytes, KECCAK256_EMPTY, U256}; - use futures::future::{ready, Ready}; - use reth_eth_wire_types::snap::{ - AccountData, ByteCodesMessage, GetBlockAccessListsMessage, GetByteCodesMessage, - GetStorageRangesMessage, - }; - use reth_network_p2p::download::DownloadClient; + use reth_eth_wire_types::snap::{AccountData, ByteCodesMessage}; + use reth_network_p2p::{error::PeerRequestResult, priority::Priority}; use reth_network_peers::WithPeerId; use reth_trie_common::{proof::ProofRetainer, HashBuilder, Nibbles}; - use std::{ - collections::VecDeque, - sync::{Arc, Mutex}, - }; + use std::sync::Arc; const MAX_HASH: B256 = B256::new([0xff; B256::len_bytes()]); - #[derive(Debug)] - struct TestSnapClient { - responses: Mutex>>, - reported: Mutex>, - priorities: Mutex>, - } - - impl TestSnapClient { - fn new(responses: impl IntoIterator>) -> Self { - Self { - responses: Mutex::new(responses.into_iter().collect()), - reported: Mutex::new(Vec::new()), - priorities: Mutex::new(Vec::new()), - } - } - - fn next(&self, priority: Priority) -> Ready> { - self.priorities.lock().unwrap().push(priority); - ready(self.responses.lock().unwrap().pop_front().expect("test response available")) - } - } - - impl DownloadClient for TestSnapClient { - fn report_bad_message(&self, peer_id: PeerId) { - self.reported.lock().unwrap().push(peer_id); - } - - fn num_connected_peers(&self) -> usize { - 1 - } - } - - impl SnapClient for TestSnapClient { - type Output = Ready>; - - fn get_account_range_with_priority( - &self, - _request: GetAccountRangeMessage, - priority: Priority, - ) -> Self::Output { - self.next(priority) - } - - fn get_storage_ranges(&self, _request: GetStorageRangesMessage) -> Self::Output { - ready(Err(RequestError::UnsupportedCapability)) - } - - fn get_storage_ranges_with_priority( - &self, - _request: GetStorageRangesMessage, - _priority: Priority, - ) -> Self::Output { - ready(Err(RequestError::UnsupportedCapability)) - } - - fn get_byte_codes(&self, _request: GetByteCodesMessage) -> Self::Output { - ready(Err(RequestError::UnsupportedCapability)) - } - - fn get_byte_codes_with_priority( - &self, - _request: GetByteCodesMessage, - _priority: Priority, - ) -> Self::Output { - ready(Err(RequestError::UnsupportedCapability)) - } - - fn get_block_access_lists_with_priority( - &self, - _request: GetBlockAccessListsMessage, - _priority: Priority, - ) -> Self::Output { - ready(Err(RequestError::UnsupportedCapability)) - } - } - fn key(value: u64) -> B256 { B256::left_padding_from(&value.to_be_bytes()) } @@ -468,13 +376,14 @@ mod tests { assert_eq!( outcome, AccountRangeOutcome::Verified(VerifiedAccountRange { + state_root: root_hash, accounts, has_more: false, next: None, }) ); - assert!(client.reported.lock().unwrap().is_empty()); - assert_eq!(*client.priorities.lock().unwrap(), [Priority::Normal]); + assert!(client.reported().is_empty()); + assert_eq!(*client.priorities(), [Priority::Normal]); } #[tokio::test] @@ -500,8 +409,8 @@ mod tests { let outcome = downloader(Arc::clone(&client), request(root_hash)).unwrap().await.unwrap(); assert!(matches!(outcome, AccountRangeOutcome::Verified(_))); - assert_eq!(*client.reported.lock().unwrap(), [bad_peer]); - assert_eq!(*client.priorities.lock().unwrap(), [Priority::Normal, Priority::High]); + assert_eq!(*client.reported(), [bad_peer]); + assert_eq!(*client.priorities(), [Priority::Normal, Priority::High]); } #[tokio::test] @@ -517,7 +426,25 @@ mod tests { .unwrap(); assert_eq!(outcome, AccountRangeOutcome::Unavailable { peer_id: peer }); - assert!(client.reported.lock().unwrap().is_empty()); + assert!(client.reported().is_empty()); + } + + #[test] + fn batch_range_narrows_the_accounts_and_keeps_their_root() { + let accounts = vec![(key(1), account(7)), (key(2), account(8)), (key(3), account(9))]; + let root_hash = root(&accounts); + let range = VerifiedAccountRange { + state_root: root_hash, + accounts: accounts.clone(), + has_more: false, + next: None, + }; + + let chunk = range.batch_range(1..3).expect("chunk is inside the range"); + assert_eq!(chunk.accounts(), &accounts[1..3]); + assert_eq!(chunk.state_root(), root_hash); + + assert_eq!(range.batch_range(2..4), None); } #[test] @@ -531,7 +458,7 @@ mod tests { downloader(Arc::clone(&client), request), Err(InvalidAccountRange { .. }) )); - assert!(client.priorities.lock().unwrap().is_empty()); + assert!(client.priorities().is_empty()); } #[tokio::test] @@ -556,12 +483,13 @@ mod tests { assert_eq!( outcome, AccountRangeOutcome::Verified(VerifiedAccountRange { + state_root: root_hash, accounts: vec![accounts[0]], has_more: false, next: Some(key(4)), }) ); - assert!(client.reported.lock().unwrap().is_empty()); + assert!(client.reported().is_empty()); } #[tokio::test] @@ -587,7 +515,7 @@ mod tests { let error = downloader(Arc::clone(&client), request).unwrap().await.unwrap_err(); assert_eq!(error, RequestError::BadResponse); - assert_eq!(client.reported.lock().unwrap().len(), attempts); + assert_eq!(client.reported().len(), attempts); } #[tokio::test] @@ -612,12 +540,13 @@ mod tests { assert_eq!( outcome, AccountRangeOutcome::Verified(VerifiedAccountRange { + state_root: root_hash, accounts: accounts[..2].to_vec(), has_more: false, next: Some(key(3)), }) ); - assert!(client.reported.lock().unwrap().is_empty()); + assert!(client.reported().is_empty()); } // The first account after the limit proves an empty interval. @@ -641,12 +570,13 @@ mod tests { assert_eq!( outcome, AccountRangeOutcome::Verified(VerifiedAccountRange { + state_root: root_hash, accounts: Vec::new(), has_more: false, next: None, }) ); - assert!(client.reported.lock().unwrap().is_empty()); + assert!(client.reported().is_empty()); } #[tokio::test] @@ -665,12 +595,13 @@ mod tests { assert_eq!( outcome, AccountRangeOutcome::Verified(VerifiedAccountRange { + state_root: root_hash, accounts: Vec::new(), has_more: false, next: Some(key(9)), }) ); - assert!(client.reported.lock().unwrap().is_empty()); + assert!(client.reported().is_empty()); } // A proof that continues past the limit completes the requested interval. @@ -693,6 +624,7 @@ mod tests { assert_eq!( outcome, AccountRangeOutcome::Verified(VerifiedAccountRange { + state_root: root_hash, accounts: vec![accounts[0]], has_more: false, next: Some(key(9)), @@ -719,6 +651,7 @@ mod tests { assert_eq!( outcome, AccountRangeOutcome::Verified(VerifiedAccountRange { + state_root: root_hash, accounts: vec![accounts[0]], has_more: true, next: Some(key(3)), @@ -747,11 +680,8 @@ mod tests { downloader(Arc::clone(&client), request(root_hash)).unwrap().await.unwrap(); - assert!(client.reported.lock().unwrap().is_empty()); - assert_eq!( - *client.priorities.lock().unwrap(), - [Priority::Normal, Priority::High, Priority::High] - ); + assert!(client.reported().is_empty()); + assert_eq!(*client.priorities(), [Priority::Normal, Priority::High, Priority::High]); } #[tokio::test] @@ -771,10 +701,7 @@ mod tests { .unwrap_err(); assert_eq!(error, RequestError::BadResponse); - assert_eq!(*client.reported.lock().unwrap(), peers); - assert_eq!( - *client.priorities.lock().unwrap(), - [Priority::Normal, Priority::High, Priority::High] - ); + assert_eq!(*client.reported(), peers); + assert_eq!(*client.priorities(), [Priority::Normal, Priority::High, Priority::High]); } } diff --git a/crates/net/downloaders/src/snap/request.rs b/crates/net/downloaders/src/snap/request.rs new file mode 100644 index 00000000000..6ef539c5270 --- /dev/null +++ b/crates/net/downloaders/src/snap/request.rs @@ -0,0 +1,179 @@ +//! Shared request execution for authenticated snap responses. +//! +//! Retries, peer attribution and blocking verification live here so every range downloader +//! penalizes and reissues in exactly the same way. + +use futures::FutureExt; +use reth_eth_wire_types::snap::{GetAccountRangeMessage, GetStorageRangesMessage}; +use reth_network_p2p::{ + error::RequestError, + priority::Priority, + snap::client::{SnapClient, SnapResponse}, +}; +use reth_network_peers::PeerId; +use reth_tasks::Runtime; +use std::{ + fmt, + task::{ready, Context, Poll}, +}; +use tracing::debug; + +/// Number of retries allowed after the initial request fails. +pub(super) const MAX_RETRIES: u8 = 2; + +/// Drives one snap request until its response is verified or retries are exhausted. +pub(super) struct VerifyingRequest { + // Sends each attempt and receives the penalty for an invalid response. + client: C, + // Verification runs here so peer-controlled proof work stays off the async worker. + runtime: Runtime, + // Retained so a retry reissues the identical request. + request: V::Request, + // Cloned per attempt, because verification moves onto the blocking pool. + verifier: V, + // The response currently in flight. + fut: C::Output, + // Present only while a response is being authenticated. + verification: Option>, + // Attempts already spent against `MAX_RETRIES`. + retries: u8, +} + +impl VerifyingRequest +where + C: SnapClient, + V: SnapVerifier, +{ + /// Submits `request` and prepares to authenticate its response with `verifier`. + pub(super) fn new(client: C, request: V::Request, verifier: V, runtime: Runtime) -> Self { + let fut = request.send(&client, Priority::Normal); + Self { client, runtime, request, verifier, fut, verification: None, retries: 0 } + } + + /// Polls until the request yields a verified response or a terminal error. + /// + /// An active verification finishes before another response is accepted, so a peer stays + /// attributable for the work done on its behalf. + pub(super) fn poll_verified( + &mut self, + cx: &mut Context<'_>, + ) -> Poll> { + loop { + if self.verification.is_some() { + match ready!(self.poll_verification(cx)) { + Ok(Some(output)) => return Poll::Ready(Ok(output)), + Ok(None) => {} + Err(error) => return Poll::Ready(Err(error)), + } + } + + match ready!(self.fut.poll_unpin(cx)) { + Ok(response) => { + let (peer_id, response) = response.split(); + let verifier = self.verifier.clone(); + let fut = + self.runtime.spawn_blocking(move || verifier.verify(peer_id, response)); + self.verification = Some(VerificationTask { peer_id, fut }); + } + // A wrong wire response is already penalized by the session. + Err(error) if error.is_retryable() || error == RequestError::BadResponse => { + debug!(target: "downloaders::snap", %error, "Snap request failed, retrying"); + if !self.retry() { + return Poll::Ready(Err(error)) + } + } + Err(error) => return Poll::Ready(Err(error)), + } + } + } + + // Raise retry priority so transient failures cannot leave range progress behind new work. + fn retry(&mut self) -> bool { + if self.retries >= MAX_RETRIES { + return false + } + self.retries += 1; + self.fut = self.request.send(&self.client, Priority::High); + true + } + + // The responder stays attached until blocking verification completes. + fn poll_verification( + &mut self, + cx: &mut Context<'_>, + ) -> Poll, RequestError>> { + let verification = self.verification.as_mut().expect("verification task is present"); + let result = ready!(verification.fut.poll_unpin(cx)); + let peer_id = verification.peer_id; + self.verification = None; + + match result { + Ok(Ok(output)) => Poll::Ready(Ok(Some(output))), + Ok(Err(error)) => { + debug!(target: "downloaders::snap", ?peer_id, %error, "Invalid snap response"); + self.client.report_bad_message(peer_id); + Poll::Ready(self.retry().then_some(None).ok_or(error)) + } + // A panic or a shutting-down runtime is local, so it must not penalize the responder. + Err(error) => { + debug!(target: "downloaders::snap", %error, "Snap verification task failed"); + Poll::Ready(Err(RequestError::ChannelClosed)) + } + } + } +} + +// The opaque client future cannot be printed without imposing an unnecessary bound. +impl fmt::Debug for VerifyingRequest +where + C: SnapClient, + V: SnapVerifier + fmt::Debug, + V::Request: fmt::Debug, +{ + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("VerifyingRequest") + .field("client", &self.client) + .field("request", &self.request) + .field("verifier", &self.verifier) + .field("verifying", &self.verification.is_some()) + .field("retries", &self.retries) + .finish_non_exhaustive() + } +} + +/// A snap request that can be reissued at a chosen priority. +pub(super) trait SnapRequest { + /// Sends this request through `client`. + fn send(&self, client: &C, priority: Priority) -> C::Output; +} + +impl SnapRequest for GetAccountRangeMessage { + fn send(&self, client: &C, priority: Priority) -> C::Output { + client.get_account_range_with_priority(self.clone(), priority) + } +} + +impl SnapRequest for GetStorageRangesMessage { + fn send(&self, client: &C, priority: Priority) -> C::Output { + client.get_storage_ranges_with_priority(self.clone(), priority) + } +} + +/// Authenticates a snap response against the request that asked for it. +pub(super) trait SnapVerifier: Clone + Send + 'static { + /// Request type this verifier authenticates responses for. + type Request: SnapRequest; + /// Verified output returned to the downloader. + type Output: Send + 'static; + + /// Verifies `response`, retaining the responder for non-error outcomes. + fn verify(self, peer_id: PeerId, response: SnapResponse) -> Result; +} + +// Proof failures remain attributable after leaving the async worker. +struct VerificationTask { + // The responder to penalize if verification rejects its response. + peer_id: PeerId, + // Returns the verified output without blocking the async worker. + fut: tokio::task::JoinHandle>, +} diff --git a/crates/net/downloaders/src/snap/storage.rs b/crates/net/downloaders/src/snap/storage.rs new file mode 100644 index 00000000000..9c17865f56f --- /dev/null +++ b/crates/net/downloaders/src/snap/storage.rs @@ -0,0 +1,1003 @@ +//! Downloads storage ranges and authenticates them against account storage roots. +//! +//! Responses remain positional, and only the final partial range may use the shared boundary +//! proof defined by the Snap protocol. + +use super::{ + request::{SnapVerifier, VerifyingRequest}, + VerifiedAccountBatch, +}; +use alloy_primitives::{B256, U256}; +use futures::Future; +use reth_eth_wire_types::snap::{ + GetStorageRangesMessage, RangeBound, StorageData, StorageRangesMessage, +}; +use reth_network_p2p::{ + error::RequestError, + snap::client::{SnapClient, SnapResponse}, +}; +use reth_network_peers::PeerId; +use reth_tasks::Runtime; +use reth_trie_common::{range_proof::verify_range_proof, EMPTY_ROOT_HASH}; +use std::{ + pin::Pin, + task::{Context, Poll}, +}; +use tracing::debug; + +// Keeps storage requests inclusive through the full trie keyspace. +const MAX_HASH: B256 = B256::new([0xff; B256::len_bytes()]); + +/// Downloads storage ranges authenticated by a verified account range. +#[derive(Debug)] +pub struct StorageRangeDownloader(VerifyingRequest); + +impl StorageRangeDownloader { + /// Validates the authenticated batch against the request before submitting it. + /// + /// `batch` must hold the accounts in `request.account_hashes` order, under the same state + /// root; their storage roots authenticate the returned ranges. Pairing a batch with a request + /// for another root would penalize a peer that answered honestly. + /// + /// Accounts with empty storage roots must be omitted because Snap responses do not preserve + /// an outer-list position for them. Use [`super::VerifiedAccountRange::batch_range`] to select + /// the accounts included in the request without losing their state-root provenance. + pub fn new( + client: C, + request: GetStorageRangesMessage, + batch: VerifiedAccountBatch<'_>, + runtime: Runtime, + ) -> Result { + let origin = request.starting_hash.unwrap_or(B256::ZERO); + let limit = request.limit_hash.unwrap_or(MAX_HASH); + if origin > limit { + return Err(InvalidStorageRangeRequest::ReversedBounds { origin, limit }) + } + if request.account_hashes.is_empty() { + return Err(InvalidStorageRangeRequest::NoAccounts) + } + // Servers disagree on whether a finite limit applies only to the first account or every + // account, so the final proof cannot be assigned reliably for this request shape. + if request.account_hashes.len() > 1 && origin == B256::ZERO && limit != MAX_HASH { + return Err(InvalidStorageRangeRequest::LimitedMultipleAccounts { + accounts: request.account_hashes.len(), + }) + } + batch.verify_batch(&request)?; + let mut storage_roots = Vec::with_capacity(batch.accounts().len()); + for (index, (account_hash, account)) in batch.accounts().iter().enumerate() { + if account.storage_root == EMPTY_ROOT_HASH { + return Err(InvalidStorageRangeRequest::EmptyStorageRoot { + index, + account_hash: *account_hash, + }) + } + storage_roots.push(account.storage_root); + } + + let verifier = StorageRangeVerifier { request: request.clone(), storage_roots }; + Ok(Self(VerifyingRequest::new(client, request, verifier, runtime))) + } +} + +impl Future for StorageRangeDownloader +where + C: SnapClient + Unpin, +{ + type Output = Result; + + fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + self.get_mut().0.poll_verified(cx) + } +} + +/// Result of an authenticated storage-ranges request. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum StorageRangeOutcome { + /// The responder lacks the requested state or account. + Unavailable { + /// Peer that returned the empty response. + peer_id: PeerId, + }, + /// Ranges authenticated against their account storage roots. + Verified(VerifiedStorageRanges), +} + +/// Positional storage ranges authenticated against their accounts. +/// +/// Carries the request it answers so [`Self::follow_up`] can resume it. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct VerifiedStorageRanges { + // Slots per account. Private so they stay consistent with the position to resume at. + ranges: Vec, + // Where a follow-up resumes, or none when every requested account is complete. + continuation: Option, + // The request these ranges answer. Retained so a follow-up cannot be built against another. + request: Box, +} + +impl VerifiedStorageRanges { + /// Consumes the result and returns the ranges it authenticated. + pub fn into_ranges(self) -> Vec { + self.ranges + } + + /// Resumes this response, narrowing `batch` to the accounts the new request covers. + /// + /// `Ok(None)` once every requested account is complete, and an error when `batch` is not the + /// one the answered request was built from. Each resumption asks for strictly less than the + /// last, and only its first account stays bounded. + pub fn follow_up<'a>( + &self, + request_id: u64, + batch: VerifiedAccountBatch<'a>, + ) -> Result< + Option<(GetStorageRangesMessage, VerifiedAccountBatch<'a>)>, + InvalidStorageRangeRequest, + > { + batch.verify_batch(&self.request)?; + let Some(continuation) = self.continuation else { return Ok(None) }; + let (index, starting_hash, limit_hash) = match continuation { + StorageRangeContinuation::Partial { account_index: 0, starting_hash, .. } => { + (0, starting_hash.into(), self.request.limit_hash) + } + StorageRangeContinuation::Partial { account_index, starting_hash, .. } => { + (account_index, starting_hash.into(), RangeBound::default()) + } + StorageRangeContinuation::NextAccount { account_index, .. } => { + (account_index, RangeBound::default(), RangeBound::default()) + } + }; + + let request = GetStorageRangesMessage { + request_id, + account_hashes: self.request.account_hashes[index..].to_vec(), + starting_hash, + limit_hash, + ..(*self.request).clone() + }; + let narrowed = batch.slice(index).expect("checked batch covers the continuation"); + Ok(Some((request, narrowed))) + } +} + +/// Decoded storage slots authenticated against one account. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct VerifiedStorageRange { + /// Hashed address of the owning account. + pub account_hash: B256, + /// Non-zero slots in increasing hashed-key order. + pub slots: Vec<(B256, U256)>, +} + +// Resume position for an incomplete storage-ranges response. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum StorageRangeContinuation { + /// The final returned account remains incomplete. + Partial { + /// Position in the original account list. + account_index: usize, + /// Hashed address at `account_index`. + account_hash: B256, + /// Inclusive slot origin for the next request. + starting_hash: B256, + }, + /// A later requested account was not returned. + NextAccount { + /// Position in the original account list. + account_index: usize, + /// Hashed address at `account_index`. + account_hash: B256, + }, +} + +/// A storage request that does not match its authenticated accounts. +#[derive(Clone, Copy, Debug, PartialEq, Eq, thiserror::Error)] +pub enum InvalidStorageRangeRequest { + /// The request targets a different state root than the one that authenticated the accounts. + #[error( + "storage range requests state root {requested}, but accounts are authenticated by {authenticated}" + )] + StateRootMismatch { + /// State root in the wire request. + requested: B256, + /// State root the account batch was verified against. + authenticated: B256, + }, + /// No account was requested. + #[error("storage range request contains no accounts")] + NoAccounts, + /// A finite limit with a zero origin was requested for multiple accounts. + #[error("limited storage range request contains {accounts} accounts")] + LimitedMultipleAccounts { + /// Number of requested accounts. + accounts: usize, + }, + /// The inclusive bounds are reversed. + #[error("storage range origin {origin} exceeds limit {limit}")] + ReversedBounds { + /// Requested inclusive origin. + origin: B256, + /// Requested inclusive limit. + limit: B256, + }, + /// The request and authenticated batch have different lengths. + #[error("storage range request has {requested} accounts but {supplied} were supplied")] + AccountCount { + /// Number of requested accounts. + requested: usize, + /// Number of authenticated accounts. + supplied: usize, + }, + /// A requested account differs from its authenticated position. + #[error( + "storage range account {index} requests {requested}, but authenticated account is {supplied}" + )] + AccountMismatch { + /// Position of the mismatch. + index: usize, + /// Hash in the wire request. + requested: B256, + /// Hash in the authenticated batch. + supplied: B256, + }, + /// An account has no storage to download. + #[error("storage range account {index} ({account_hash}) has an empty storage root")] + EmptyStorageRoot { + /// Position of the account in the request. + index: usize, + /// Hashed address of the account. + account_hash: B256, + }, +} + +// Checks a storage response against the request and the account roots it was built from. +#[derive(Clone, Debug)] +struct StorageRangeVerifier { + // The request being answered, which every response is bound to. + request: GetStorageRangesMessage, + // Authenticated storage root per requested account, in the same order. + storage_roots: Vec, +} + +impl SnapVerifier for StorageRangeVerifier { + type Request = GetStorageRangesMessage; + type Output = StorageRangeOutcome; + + fn verify(self, peer_id: PeerId, response: SnapResponse) -> Result { + self.verify_response(peer_id, response) + } +} + +impl StorageRangeVerifier { + // Every returned list must match the account at the same request position. + fn verify_response( + &self, + peer_id: PeerId, + response: SnapResponse, + ) -> Result { + let Some(response) = self.accepted_response(response)? else { + return Ok(StorageRangeOutcome::Unavailable { peer_id }) + }; + + // Only the last returned range may carry the response's shared boundary proof. + let proof_index = + response.slots.len().checked_sub(1).filter(|_| !response.proof.is_empty()); + let mut ranges = Vec::with_capacity(response.slots.len()); + let mut bounded_next = None; + + for (index, slots) in response.slots.iter().enumerate() { + let proof = if proof_index == Some(index) { response.proof.as_slice() } else { &[] }; + let verified = self.verify_range(index, slots, proof)?; + ranges.push(verified.range); + bounded_next = verified.within_limit; + } + + let continuation = self.continuation(&ranges, bounded_next); + Ok(StorageRangeOutcome::Verified(VerifiedStorageRanges { + ranges, + continuation, + request: Box::new(self.request.clone()), + })) + } + + // Binds the reply to this request before any peer-supplied data is trusted. `None` means the + // responder simply lacks the state. + fn accepted_response( + &self, + response: SnapResponse, + ) -> Result, RequestError> { + let SnapResponse::StorageRanges(mut response) = response else { + debug!(target: "downloaders::snap", "Expected storage ranges response"); + return Err(RequestError::BadResponse) + }; + if response.request_id != self.request.request_id { + debug!( + target: "downloaders::snap", + expected = self.request.request_id, + got = response.request_id, + "Storage ranges response id mismatch" + ); + return Err(RequestError::BadResponse) + } + if response.slots.len() > self.request.account_hashes.len() { + debug!(target: "downloaders::snap", "Storage response contains extra ranges"); + return Err(RequestError::BadResponse) + } + if response.slots.is_empty() { + if response.proof.is_empty() { + return Ok(None) + } + // A boundary proof can authenticate an empty suffix without an encoded inner list. + response.slots.push(Vec::new()); + } + Ok(Some(response)) + } + + // Authenticates one account's slots against its storage root. + fn verify_range( + &self, + index: usize, + slots: &[StorageData], + proof: &[alloy_primitives::Bytes], + ) -> Result { + let account_hash = self.request.account_hashes[index]; + // Only the first account inherits the request bounds; the rest are whole tries. + let (origin, limit) = if index == 0 { + ( + self.request.starting_hash.unwrap_or(B256::ZERO), + self.request.limit_hash.unwrap_or(MAX_HASH), + ) + } else { + (B256::ZERO, MAX_HASH) + }; + + // One boundary slot may sit past the inclusive limit and is verified before removal. + if slots.iter().filter(|slot| slot.hash > limit).nth(1).is_some() { + debug!(target: "downloaders::snap", %account_hash, "Storage range exceeds limit"); + return Err(RequestError::BadResponse) + } + + let mut decoded = Self::decode_slots(account_hash, origin, slots)?; + let leaves = decoded.iter().map(|(hash, value)| (*hash, alloy_rlp::encode(value))); + let next = verify_range_proof(self.storage_roots[index], origin, limit, leaves, proof) + .map_err(|error| { + debug!( + target: "downloaders::snap", + %account_hash, + %error, + "Invalid storage range proof" + ); + RequestError::BadResponse + })?; + + // Resuming at or before the origin would reissue this request forever, so a peer that + // reports one has withheld a slot it was asked for. + if next.is_some_and(|next| next <= origin) { + debug!(target: "downloaders::snap", %account_hash, "Storage range does not advance"); + return Err(RequestError::BadResponse) + } + + decoded.truncate(decoded.partition_point(|(hash, _)| *hash <= limit)); + Ok(VerifiedRange { + range: VerifiedStorageRange { account_hash, slots: decoded }, + within_limit: next.filter(|next| *next <= limit), + }) + } + + // A response is incomplete either part way through its last account, or because a requested + // account was omitted entirely. + fn continuation( + &self, + ranges: &[VerifiedStorageRange], + final_next: Option, + ) -> Option { + if let Some(starting_hash) = final_next { + return Some(StorageRangeContinuation::Partial { + account_index: ranges.len() - 1, + account_hash: ranges.last().expect("a response range exists").account_hash, + starting_hash, + }) + } + let account_index = ranges.len(); + self.request.account_hashes.get(account_index).copied().map(|account_hash| { + StorageRangeContinuation::NextAccount { account_index, account_hash } + }) + } + + // Storage values are canonical non-zero trie leaves. + fn decode_slots( + account_hash: B256, + origin: B256, + slots: &[StorageData], + ) -> Result, RequestError> { + let mut decoded = Vec::with_capacity(slots.len()); + let mut previous = None; + + for slot in slots { + if slot.hash < origin || previous.is_some_and(|previous| slot.hash <= previous) { + debug!( + target: "downloaders::snap", + %account_hash, + "Storage slots precede origin or are not strictly ordered" + ); + return Err(RequestError::BadResponse) + } + let value = slot.value().map_err(|error| { + debug!(target: "downloaders::snap", %account_hash, %error, "Invalid storage value"); + RequestError::BadResponse + })?; + if value.is_zero() { + debug!(target: "downloaders::snap", %account_hash, "Storage trie contains zero leaf"); + return Err(RequestError::BadResponse) + } + previous = Some(slot.hash); + decoded.push((slot.hash, value)); + } + Ok(decoded) + } +} + +// One account's verified slots and whether it continues within the request limit. +struct VerifiedRange { + // The account's slots, once its proof checked out. + range: VerifiedStorageRange, + // First slot after the range when it remains within the requested limit. + within_limit: Option, +} + +#[cfg(test)] +mod tests { + use super::{ + super::{request::MAX_RETRIES, test_utils::TestSnapClient, VerifiedAccountRange}, + *, + }; + use alloy_primitives::{Bytes, KECCAK256_EMPTY}; + use reth_network_p2p::{error::PeerRequestResult, priority::Priority}; + use reth_network_peers::WithPeerId; + use reth_trie_common::{ + proof::ProofRetainer, HashBuilder, Nibbles, TrieAccount, EMPTY_ROOT_HASH, + }; + use std::sync::Arc; + + fn key(value: u64) -> B256 { + B256::left_padding_from(&value.to_be_bytes()) + } + + // Storage leaves are RLP-encoded non-zero integers, so the trie value is the encoded slot. + fn slots(values: &[(B256, u64)]) -> Vec<(B256, U256)> { + values.iter().map(|(hash, value)| (*hash, U256::from(*value))).collect() + } + + fn storage_root(slots: &[(B256, U256)], targets: &[B256]) -> (B256, Vec) { + let targets = targets.iter().copied().map(Nibbles::unpack).collect(); + let mut builder = HashBuilder::default().with_proof_retainer(ProofRetainer::new(targets)); + for (hash, value) in slots { + builder.add_leaf(Nibbles::unpack(*hash), &alloy_rlp::encode(value)); + } + let root = builder.root(); + let proof = builder + .take_proof_nodes() + .into_nodes_sorted() + .into_iter() + .map(|(_, node)| node) + .collect(); + (root, proof) + } + + // Every request in these tests targets the state root that authenticated its accounts. + const STATE_ROOT: B256 = B256::repeat_byte(0xaa); + + // Accounts reach the downloader only through a range that authenticated them. + fn verified_range(accounts: &[(B256, TrieAccount)]) -> VerifiedAccountRange { + VerifiedAccountRange { + state_root: STATE_ROOT, + accounts: accounts.to_vec(), + has_more: false, + next: None, + } + } + + fn account(storage_root: B256) -> TrieAccount { + TrieAccount { nonce: 1, balance: U256::from(2), storage_root, code_hash: KECCAK256_EMPTY } + } + + fn wire_slots(slots: &[(B256, U256)]) -> Vec { + slots.iter().map(|(hash, value)| StorageData::from_value(*hash, *value)).collect() + } + + fn request(accounts: &[(B256, TrieAccount)]) -> GetStorageRangesMessage { + GetStorageRangesMessage { + request_id: 1, + root_hash: STATE_ROOT, + account_hashes: accounts.iter().map(|(hash, _)| *hash).collect(), + starting_hash: B256::ZERO.into(), + limit_hash: MAX_HASH.into(), + response_bytes: 512 * 1024, + } + } + + fn response( + peer: PeerId, + request_id: u64, + slots: Vec>, + proof: Vec, + ) -> PeerRequestResult { + Ok(WithPeerId::new( + peer, + SnapResponse::StorageRanges(StorageRangesMessage { request_id, slots, proof }), + )) + } + + fn rejecting_client( + response: impl Fn(PeerId) -> PeerRequestResult, + ) -> Arc { + let peer = PeerId::random(); + Arc::new(TestSnapClient::new((0..=MAX_RETRIES).map(|_| response(peer)))) + } + + fn downloader( + client: C, + request: GetStorageRangesMessage, + accounts: &[(B256, TrieAccount)], + ) -> Result, InvalidStorageRangeRequest> { + let range = verified_range(accounts); + StorageRangeDownloader::new(client, request, range.batch(), Runtime::test()) + } + + #[tokio::test] + async fn complete_ranges_for_multiple_accounts_are_verified() { + let first = slots(&[(key(1), 11), (key(2), 12)]); + let second = slots(&[(key(3), 13)]); + let (first_root, _) = storage_root(&first, &[]); + let (second_root, proof) = storage_root(&second, &[key(3)]); + let accounts = vec![(key(100), account(first_root)), (key(200), account(second_root))]; + let client = Arc::new(TestSnapClient::new([response( + PeerId::random(), + 1, + vec![wire_slots(&first), wire_slots(&second)], + proof, + )])); + + let outcome = + downloader(Arc::clone(&client), request(&accounts), &accounts).unwrap().await.unwrap(); + + let StorageRangeOutcome::Verified(verified) = outcome else { panic!("verified ranges") }; + assert_eq!(verified.ranges.len(), 2); + assert_eq!(verified.ranges[0].slots, first); + assert_eq!(verified.ranges[1].slots, second); + assert_eq!(verified.continuation, None); + assert!(client.reported().is_empty()); + } + + #[tokio::test] + async fn partial_final_range_reports_its_slot_continuation() { + let all = slots(&[(key(1), 11), (key(2), 12), (key(3), 13)]); + let (root, proof) = storage_root(&all, &[B256::ZERO, key(1)]); + let accounts = vec![(key(100), account(root))]; + let client = Arc::new(TestSnapClient::new([response( + PeerId::random(), + 1, + vec![wire_slots(&all[..1])], + proof, + )])); + + let outcome = + downloader(Arc::clone(&client), request(&accounts), &accounts).unwrap().await.unwrap(); + + let StorageRangeOutcome::Verified(verified) = outcome else { panic!("verified ranges") }; + assert_eq!(verified.ranges[0].slots, all[..1]); + assert_eq!( + verified.continuation, + Some(StorageRangeContinuation::Partial { + account_index: 0, + account_hash: key(100), + starting_hash: key(2), + }) + ); + assert!(client.reported().is_empty()); + } + + #[tokio::test] + async fn an_omitted_account_resumes_at_the_next_position() { + let first = slots(&[(key(1), 11)]); + let (first_root, _) = storage_root(&first, &[]); + let accounts = + vec![(key(100), account(first_root)), (key(200), account(B256::repeat_byte(0xbb)))]; + let client = Arc::new(TestSnapClient::new([response( + PeerId::random(), + 1, + vec![wire_slots(&first)], + Vec::new(), + )])); + + let outcome = + downloader(Arc::clone(&client), request(&accounts), &accounts).unwrap().await.unwrap(); + + let StorageRangeOutcome::Verified(verified) = outcome else { panic!("verified ranges") }; + assert_eq!(verified.ranges.len(), 1); + assert_eq!( + verified.continuation, + Some(StorageRangeContinuation::NextAccount { + account_index: 1, + account_hash: key(200), + }) + ); + } + + #[test] + fn a_mixed_batch_with_an_empty_storage_root_is_refused_before_submission() { + let accounts = vec![ + (key(100), account(EMPTY_ROOT_HASH)), + (key(200), account(B256::repeat_byte(0xbb))), + ]; + let client = TestSnapClient::new([response(PeerId::random(), 1, Vec::new(), Vec::new())]); + + assert_eq!( + downloader(&client, request(&accounts), &accounts).unwrap_err(), + InvalidStorageRangeRequest::EmptyStorageRoot { index: 0, account_hash: key(100) } + ); + assert!(client.priorities().is_empty()); + + let range = verified_range(&accounts); + assert!(StorageRangeDownloader::new( + &client, + request(&accounts[1..]), + range.batch_range(1..2).unwrap(), + Runtime::test(), + ) + .is_ok()); + assert_eq!(*client.priorities(), [Priority::Normal]); + } + + #[tokio::test] + async fn a_response_without_slots_or_proof_reports_the_state_unavailable() { + let batches = [ + vec![(key(100), account(B256::repeat_byte(0xbb)))], + vec![ + (key(100), account(B256::repeat_byte(0xaa))), + (key(200), account(B256::repeat_byte(0xbb))), + ], + ]; + for accounts in batches { + let peer = PeerId::random(); + let client = Arc::new(TestSnapClient::new([response(peer, 1, Vec::new(), Vec::new())])); + + let outcome = downloader(Arc::clone(&client), request(&accounts), &accounts) + .unwrap() + .await + .unwrap(); + + assert_eq!(outcome, StorageRangeOutcome::Unavailable { peer_id: peer }); + // Lacking the state is not misbehaviour. + assert!(client.reported().is_empty()); + } + } + + #[tokio::test] + async fn a_mismatched_request_id_exhausts_the_retry_budget() { + let all = slots(&[(key(1), 11)]); + let (root, _) = storage_root(&all, &[]); + let accounts = vec![(key(100), account(root))]; + + let client = rejecting_client(|peer| response(peer, 9, vec![wire_slots(&all)], Vec::new())); + let error = downloader(Arc::clone(&client), request(&accounts), &accounts) + .unwrap() + .await + .unwrap_err(); + + assert_eq!(error, RequestError::BadResponse); + // One penalty per attempt, then the budget is spent. + assert_eq!(client.reported().len(), usize::from(MAX_RETRIES) + 1); + } + + #[tokio::test] + async fn a_range_proved_against_another_root_is_rejected() { + let all = slots(&[(key(1), 11)]); + let accounts = vec![(key(100), account(B256::repeat_byte(0xcc)))]; + + let client = rejecting_client(|peer| response(peer, 1, vec![wire_slots(&all)], Vec::new())); + let error = downloader(Arc::clone(&client), request(&accounts), &accounts) + .unwrap() + .await + .unwrap_err(); + + assert_eq!(error, RequestError::BadResponse); + assert_eq!(client.reported().len(), usize::from(MAX_RETRIES) + 1); + } + + #[tokio::test] + async fn slots_that_are_not_strictly_ordered_penalize_the_peer_before_a_retry_succeeds() { + let all = slots(&[(key(1), 11), (key(2), 12)]); + let (root, _) = storage_root(&all, &[]); + let accounts = vec![(key(100), account(root))]; + let bad_peer = PeerId::random(); + let good_peer = PeerId::random(); + let mut reversed = wire_slots(&all); + reversed.reverse(); + let client = Arc::new(TestSnapClient::new([ + response(bad_peer, 1, vec![reversed], Vec::new()), + response(good_peer, 1, vec![wire_slots(&all)], Vec::new()), + ])); + + let outcome = + downloader(Arc::clone(&client), request(&accounts), &accounts).unwrap().await.unwrap(); + + let StorageRangeOutcome::Verified(verified) = outcome else { panic!("verified ranges") }; + assert_eq!(verified.ranges[0].slots, all); + assert_eq!(*client.reported(), [bad_peer]); + assert_eq!(*client.priorities(), [Priority::Normal, Priority::High]); + } + + #[test] + fn a_request_that_does_not_match_its_accounts_is_refused() { + let accounts = vec![(key(100), account(EMPTY_ROOT_HASH))]; + let client = TestSnapClient::new([]); + + let mut mismatched = request(&accounts); + mismatched.account_hashes = vec![key(999)]; + let error = downloader(&client, mismatched, &accounts).unwrap_err(); + assert_eq!( + error, + InvalidStorageRangeRequest::AccountMismatch { + index: 0, + requested: key(999), + supplied: key(100), + } + ); + + let mut empty = request(&accounts); + empty.account_hashes.clear(); + assert_eq!( + downloader(&client, empty, &accounts).unwrap_err(), + InvalidStorageRangeRequest::NoAccounts + ); + + let mut reversed = request(&accounts); + reversed.starting_hash = key(9).into(); + reversed.limit_hash = key(1).into(); + assert_eq!( + downloader(&client, reversed, &accounts).unwrap_err(), + InvalidStorageRangeRequest::ReversedBounds { origin: key(9), limit: key(1) } + ); + } + + #[test] + fn a_request_for_another_state_root_is_refused() { + let accounts = vec![(key(100), account(EMPTY_ROOT_HASH))]; + let client = TestSnapClient::new([]); + + let mut other_root = request(&accounts); + other_root.root_hash = B256::repeat_byte(0xcc); + assert_eq!( + downloader(&client, other_root, &accounts).unwrap_err(), + InvalidStorageRangeRequest::StateRootMismatch { + requested: B256::repeat_byte(0xcc), + authenticated: STATE_ROOT, + } + ); + } + + #[tokio::test] + async fn a_trie_continuing_past_the_limit_needs_no_follow_up() { + let all = slots(&[(key(1), 11), (key(2), 12), (key(3), 13)]); + let (root, proof) = storage_root(&all, &[B256::ZERO, key(2)]); + let accounts = vec![(key(100), account(root))]; + let client = Arc::new(TestSnapClient::new([response( + PeerId::random(), + 1, + vec![wire_slots(&all[..2])], + proof, + )])); + let mut bounded = request(&accounts); + bounded.limit_hash = key(2).into(); + + let outcome = downloader(Arc::clone(&client), bounded, &accounts).unwrap().await.unwrap(); + + let StorageRangeOutcome::Verified(verified) = outcome else { panic!("verified ranges") }; + let range = verified_range(&accounts); + assert_eq!(verified.follow_up(2, range.batch()).unwrap(), None); + assert_eq!(verified.into_ranges()[0].slots, all[..2]); + } + + #[tokio::test] + async fn a_follow_up_keeps_the_limit_only_at_the_first_account() { + let all = slots(&[(key(1), 11), (key(2), 12), (key(3), 13)]); + let (root, proof) = storage_root(&all, &[B256::ZERO, key(1)]); + let accounts = vec![(key(100), account(root))]; + let client = Arc::new(TestSnapClient::new([response( + PeerId::random(), + 1, + vec![wire_slots(&all[..1])], + proof, + )])); + let mut bounded = request(&accounts); + bounded.limit_hash = key(5).into(); + + let outcome = + downloader(Arc::clone(&client), bounded.clone(), &accounts).unwrap().await.unwrap(); + + let StorageRangeOutcome::Verified(verified) = outcome else { panic!("verified ranges") }; + let range = verified_range(&accounts); + let (follow_up, narrowed) = verified.follow_up(2, range.batch()).unwrap().unwrap(); + assert_eq!(narrowed.accounts(), &accounts[..]); + assert_eq!(follow_up.request_id, 2); + assert_eq!(follow_up.root_hash, bounded.root_hash); + assert_eq!(follow_up.account_hashes, bounded.account_hashes); + assert_eq!(follow_up.starting_hash, key(2).into()); + assert_eq!(follow_up.limit_hash, key(5).into()); + } + + // Every account after the first is authenticated as a whole trie, so its follow-up must drop + // the bounds the first account was requested with. + #[tokio::test] + async fn a_follow_up_past_the_first_account_is_unbounded() { + let first = slots(&[(key(1), 11)]); + let second = slots(&[(key(1), 21), (key(2), 22)]); + let (first_root, _) = storage_root(&first, &[]); + let (second_root, proof) = storage_root(&second, &[B256::ZERO, key(1)]); + let accounts = vec![(key(100), account(first_root)), (key(200), account(second_root))]; + let client = Arc::new(TestSnapClient::new([response( + PeerId::random(), + 1, + vec![wire_slots(&first), wire_slots(&second[..1])], + proof, + )])); + + let outcome = + downloader(Arc::clone(&client), request(&accounts), &accounts).unwrap().await.unwrap(); + + let StorageRangeOutcome::Verified(verified) = outcome else { panic!("verified ranges") }; + assert_eq!( + verified.continuation, + Some(StorageRangeContinuation::Partial { + account_index: 1, + account_hash: key(200), + starting_hash: key(2), + }) + ); + + let range = verified_range(&accounts); + let (follow_up, narrowed) = verified.follow_up(2, range.batch()).unwrap().unwrap(); + assert_eq!(narrowed.accounts(), &accounts[1..]); + assert_eq!(follow_up.account_hashes, vec![key(200)]); + assert_eq!(follow_up.starting_hash, key(2).into()); + assert_eq!(follow_up.limit_hash, RangeBound::default()); + } + + // Each follow-up must resume strictly past the last one, so the loop terminates. + #[tokio::test] + async fn resuming_a_partial_range_advances_until_the_trie_is_exhausted() { + let all = slots(&[(key(1), 11), (key(2), 12), (key(3), 13)]); + let (root, _) = storage_root(&all, &[]); + let accounts = vec![(key(100), account(root))]; + let range = verified_range(&accounts); + let mut request = request(&accounts); + let mut origins = Vec::new(); + let mut collected = Vec::new(); + + for served in 0..all.len() { + let origin = request.starting_hash.unwrap_or(B256::ZERO); + origins.push(origin); + let (_, proof) = storage_root(&all, &[origin, all[served].0]); + let client = Arc::new(TestSnapClient::new([response( + PeerId::random(), + request.request_id, + vec![wire_slots(&all[served..=served])], + proof, + )])); + + let outcome = + downloader(Arc::clone(&client), request.clone(), &accounts).unwrap().await.unwrap(); + let StorageRangeOutcome::Verified(verified) = outcome else { + panic!("verified ranges") + }; + collected.extend(verified.ranges[0].slots.clone()); + + let Some((follow_up, _)) = + verified.follow_up(request.request_id + 1, range.batch()).unwrap() + else { + assert_eq!(served, all.len() - 1); + break + }; + request = follow_up; + } + + assert_eq!(collected, all); + assert_eq!(origins, vec![B256::ZERO, key(2), key(3)]); + } + + // A non-zero origin makes the first account unambiguously partial, so servers stop there and + // the shared proof can still be assigned to it. + #[test] + fn a_zero_origin_limited_request_must_not_carry_several_accounts() { + let accounts = vec![ + (key(100), account(B256::repeat_byte(0xaa))), + (key(200), account(B256::repeat_byte(0xbb))), + ]; + // Accepted requests are submitted on construction, so the client must have replies ready. + let client = + TestSnapClient::new([response(PeerId::random(), 1, vec![Vec::new()], Vec::new())]); + + let mut bounded = request(&accounts); + bounded.limit_hash = key(5).into(); + assert_eq!( + downloader(&client, bounded.clone(), &accounts).unwrap_err(), + InvalidStorageRangeRequest::LimitedMultipleAccounts { accounts: 2 } + ); + + bounded.starting_hash = key(1).into(); + assert!(downloader(&client, bounded, &accounts).is_ok()); + } + + // Consecutive account transitions must chain: each follow-up is driven by the batch the + // previous one returned, never by accounts the caller re-derived. + #[tokio::test] + async fn consecutive_account_transitions_carry_their_batch_forward() { + let served = slots(&[(key(1), 11)]); + let (root, _) = storage_root(&served, &[]); + let accounts = + vec![(key(100), account(root)), (key(200), account(root)), (key(300), account(root))]; + let range = verified_range(&accounts); + let mut request = request(&accounts); + let mut batch = range.batch(); + + for remaining in (1..accounts.len()).rev() { + let client = Arc::new(TestSnapClient::new([response( + PeerId::random(), + request.request_id, + vec![wire_slots(&served)], + Vec::new(), + )])); + let outcome = StorageRangeDownloader::new( + Arc::clone(&client), + request.clone(), + batch, + Runtime::test(), + ) + .unwrap() + .await + .unwrap(); + + let StorageRangeOutcome::Verified(verified) = outcome else { + panic!("verified ranges") + }; + let (follow_up, narrowed) = + verified.follow_up(request.request_id + 1, batch).unwrap().unwrap(); + assert_eq!(follow_up.account_hashes.len(), remaining); + assert_eq!(narrowed.accounts(), &accounts[accounts.len() - remaining..]); + request = follow_up; + batch = narrowed; + } + + assert_eq!(request.account_hashes, vec![key(300)]); + } + + #[tokio::test] + async fn a_follow_up_refuses_a_batch_from_another_request() { + let served = slots(&[(key(1), 11)]); + let (root, _) = storage_root(&served, &[]); + let accounts = vec![(key(100), account(root)), (key(200), account(root))]; + let client = Arc::new(TestSnapClient::new([response( + PeerId::random(), + 1, + vec![wire_slots(&served)], + Vec::new(), + )])); + + let outcome = + downloader(Arc::clone(&client), request(&accounts), &accounts).unwrap().await.unwrap(); + let StorageRangeOutcome::Verified(verified) = outcome else { panic!("verified ranges") }; + + let others = vec![(key(900), account(root)), (key(901), account(root))]; + let other_range = verified_range(&others); + assert_eq!( + verified.follow_up(2, other_range.batch()).unwrap_err(), + InvalidStorageRangeRequest::AccountMismatch { + index: 0, + requested: key(100), + supplied: key(900), + } + ); + } +} diff --git a/crates/net/downloaders/src/snap/test_utils.rs b/crates/net/downloaders/src/snap/test_utils.rs new file mode 100644 index 00000000000..ace9e52d388 --- /dev/null +++ b/crates/net/downloaders/src/snap/test_utils.rs @@ -0,0 +1,113 @@ +//! Scripted snap client shared by the range downloader tests. + +use futures::future::{ready, Ready}; +use reth_eth_wire_types::snap::{ + GetAccountRangeMessage, GetBlockAccessListsMessage, GetByteCodesMessage, + GetStorageRangesMessage, +}; +use reth_network_p2p::{ + download::DownloadClient, + error::{PeerRequestResult, RequestError}, + priority::Priority, + snap::client::{SnapClient, SnapResponse}, +}; +use reth_network_peers::PeerId; +use std::{ + collections::VecDeque, + sync::{Mutex, MutexGuard}, +}; + +/// Answers account and storage requests from one queue, so retries across request kinds stay +/// observable in submission order. +#[derive(Debug)] +pub(super) struct TestSnapClient { + responses: Mutex>>, + reported: Mutex>, + priorities: Mutex>, +} + +impl TestSnapClient { + /// Creates a client that returns `responses` in request order. + pub(super) fn new( + responses: impl IntoIterator>, + ) -> Self { + Self { + responses: Mutex::new(responses.into_iter().collect()), + reported: Mutex::new(Vec::new()), + priorities: Mutex::new(Vec::new()), + } + } + + /// Returns peers reported for invalid messages. + pub(super) fn reported(&self) -> MutexGuard<'_, Vec> { + self.reported.lock().unwrap() + } + + /// Returns request priorities in submission order. + pub(super) fn priorities(&self) -> MutexGuard<'_, Vec> { + self.priorities.lock().unwrap() + } + + fn next(&self, priority: Priority) -> Ready> { + self.priorities.lock().unwrap().push(priority); + ready(self.responses.lock().unwrap().pop_front().expect("test response available")) + } +} + +impl DownloadClient for TestSnapClient { + fn report_bad_message(&self, peer_id: PeerId) { + self.reported.lock().unwrap().push(peer_id); + } + + fn num_connected_peers(&self) -> usize { + 1 + } +} + +impl SnapClient for TestSnapClient { + type Output = Ready>; + + fn get_account_range_with_priority( + &self, + _request: GetAccountRangeMessage, + priority: Priority, + ) -> Self::Output { + self.next(priority) + } + + fn get_storage_ranges(&self, request: GetStorageRangesMessage) -> Self::Output { + self.get_storage_ranges_with_priority(request, Priority::Normal) + } + + fn get_storage_ranges_with_priority( + &self, + _request: GetStorageRangesMessage, + priority: Priority, + ) -> Self::Output { + self.next(priority) + } + + fn get_byte_codes(&self, _request: GetByteCodesMessage) -> Self::Output { + unsupported() + } + + fn get_byte_codes_with_priority( + &self, + _request: GetByteCodesMessage, + _priority: Priority, + ) -> Self::Output { + unsupported() + } + + fn get_block_access_lists_with_priority( + &self, + _request: GetBlockAccessListsMessage, + _priority: Priority, + ) -> Self::Output { + unsupported() + } +} + +fn unsupported() -> Ready> { + ready(Err(RequestError::UnsupportedCapability)) +} From e5094e1d68878a4d8bb9baefb96c1344c1435aa7 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Thu, 27 Aug 2026 10:56:16 +0000 Subject: [PATCH 079/294] fix(engine): separate malformed input from fatal errors (#26829) --- Cargo.lock | 1 - crates/consensus/consensus/Cargo.toml | 2 - crates/consensus/consensus/src/lib.rs | 6 - crates/engine/primitives/src/error.rs | 117 +++++++++++------- crates/engine/tree/src/tree/error.rs | 3 +- crates/engine/tree/src/tree/metrics.rs | 6 +- crates/engine/tree/src/tree/mod.rs | 45 ++++--- .../engine/tree/src/tree/payload_validator.rs | 11 +- crates/engine/tree/src/tree/tests.rs | 17 +++ crates/rpc/rpc/src/validation.rs | 3 +- 10 files changed, 132 insertions(+), 79 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b2cc43d5524..0876b9c2e54 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8017,7 +8017,6 @@ dependencies = [ "alloy-consensus", "alloy-eip7928", "alloy-primitives", - "alloy-rlp", "auto_impl", "reth-execution-types", "reth-primitives-traits", diff --git a/crates/consensus/consensus/Cargo.toml b/crates/consensus/consensus/Cargo.toml index 4969a8538b7..5fa7c181525 100644 --- a/crates/consensus/consensus/Cargo.toml +++ b/crates/consensus/consensus/Cargo.toml @@ -18,7 +18,6 @@ reth-primitives-traits.workspace = true # ethereum alloy-primitives.workspace = true alloy-consensus.workspace = true -alloy-rlp.workspace = true alloy-eip7928.workspace = true # misc @@ -34,6 +33,5 @@ std = [ "reth-execution-types/std", "thiserror/std", "alloy-eip7928/std", - "alloy-rlp/std", ] test-utils = ["reth-primitives-traits/test-utils"] diff --git a/crates/consensus/consensus/src/lib.rs b/crates/consensus/consensus/src/lib.rs index f9726ac45e3..59ab0354b38 100644 --- a/crates/consensus/consensus/src/lib.rs +++ b/crates/consensus/consensus/src/lib.rs @@ -498,12 +498,6 @@ pub enum ConsensusError { /// EIP-7928: Error when the block access list hash doesn't match the expected value. #[error("block access list hash mismatch: {0}")] BlockAccessListHashMismatch(GotExpectedBoxed), - /// EIP-7928: Error when the raw block access list bytes cannot be RLP-decoded. - /// - /// Note: this is not a block validity error, the received block access list bytes themselves - /// are malformed request params. - #[error("failed to decode block access list: {0}")] - BlockAccessListDecode(alloy_rlp::Error), /// EIP-7928: Error when the block access list cannot be decoded or converted. #[error("invalid block access list: {0}")] BlockAccessListInvalid(String), diff --git a/crates/engine/primitives/src/error.rs b/crates/engine/primitives/src/error.rs index 103b1671cfe..939a9ac7f89 100644 --- a/crates/engine/primitives/src/error.rs +++ b/crates/engine/primitives/src/error.rs @@ -32,9 +32,15 @@ impl BeaconOnNewPayloadError { impl From for BeaconOnNewPayloadError { fn from(error: InsertBlockFatalError) -> Self { + Self::internal(error) + } +} + +impl From for BeaconOnNewPayloadError { + fn from(error: InsertBlockProcessingError) -> Self { match error { - InsertBlockFatalError::InvalidParams(err) => Self::InvalidParams(err), - error => Self::internal(error), + InsertBlockProcessingError::MalformedInput(error) => Self::InvalidParams(error), + InsertBlockProcessingError::Fatal(error) => Self::internal(error), } } } @@ -69,6 +75,9 @@ pub enum InsertBlockErrorKind { /// Block violated consensus rules. #[error(transparent)] Consensus(#[from] ConsensusError), + /// Supplemental block access list bytes could not be decoded. + #[error(transparent)] + BlockAccessListDecode(#[from] BlockAccessListDecodeError), /// Block execution failed. #[error(transparent)] Execution(#[from] BlockExecutionError), @@ -86,53 +95,71 @@ impl InsertBlockErrorKind { matches!(self, Self::Consensus(_) | Self::Execution(BlockExecutionError::Validation(_))) } - /// Returns an [`InsertBlockValidationError`] if the error is caused by an invalid block. - /// - /// Returns an [`InsertBlockFatalError`] if the failure is not attributable to the block - /// itself, either an internal error or malformed request params. + /// Returns an [`InsertBlockValidationError`] if the failure invalidates the block, or an + /// [`InsertBlockProcessingError`] if the failure is not attributable to the block itself. /// - /// This split decides how `newPayload` responds: validation errors become an `INVALID` - /// payload status and mark the block hash as invalid, while fatal errors are returned as - /// actual errors to the caller. This distinction is required because responding `INVALID` - /// has consensus meaning (the block is rejected and its hash cached as invalid), which must - /// not happen for failures the block is not responsible for. + /// This distinction controls whether the block may be returned as `INVALID` and its hash cached + /// as invalid. Because `INVALID` has consensus meaning, malformed supplemental input and + /// internal failures must remain processing errors instead. pub fn ensure_validation_error( self, - ) -> Result { + ) -> Result { match self { - // Undecodable block access list bytes are malformed request params, not an invalid - // block, and must be rejected with an invalid params error instead of an `INVALID` - // payload status. - Self::Consensus(ConsensusError::BlockAccessListDecode(err)) => { - Err(InsertBlockFatalError::InvalidParams(Box::new(err))) + Self::BlockAccessListDecode(error) => { + Err(InsertBlockProcessingError::MalformedInput(Box::new(error))) } Self::Consensus(err) => Ok(InsertBlockValidationError::Consensus(err)), Self::Execution(err) => match err { BlockExecutionError::Validation(err) => { Ok(InsertBlockValidationError::Validation(err)) } - BlockExecutionError::Internal(error) => { - Err(InsertBlockFatalError::BlockExecutionError(error)) - } + BlockExecutionError::Internal(error) => Err(InsertBlockProcessingError::Fatal( + InsertBlockFatalError::BlockExecutionError(error), + )), }, - Self::Provider(err) => Err(InsertBlockFatalError::Provider(err)), - Self::Other(err) => Err(InternalBlockExecutionError::Other(err).into()), + Self::Provider(err) => { + Err(InsertBlockProcessingError::Fatal(InsertBlockFatalError::Provider(err))) + } + Self::Other(err) => Err(InsertBlockProcessingError::Fatal( + InternalBlockExecutionError::Other(err).into(), + )), } } } -/// Error variants that are not caused by invalid blocks. -/// -/// "Fatal" means block processing failed for a reason other than the block itself being invalid. -/// This includes errors caused by additional payload data that is not part of the block, such as -/// undecodable block access list bytes: their malformation says nothing about the validity of -/// the block and is therefore treated differently with respect to the `PayloadStatus`. +/// Error decoding supplemental block access list bytes. +#[derive(Debug, thiserror::Error)] +#[error("failed to decode block access list: {0}")] +pub struct BlockAccessListDecodeError(#[source] Box); + +impl BlockAccessListDecodeError { + /// Creates a new block access list decode error. + pub fn new(error: E) -> Self + where + E: core::error::Error + Send + Sync + 'static, + { + Self(Box::new(error)) + } +} + +/// An error that occurs while processing a block but does not invalidate the block itself. /// -/// These failures must not be answered with an `INVALID` payload status or mark the block hash -/// as invalid. Instead they are propagated as actual errors: for `newPayload` they convert into -/// [`BeaconOnNewPayloadError`] and are returned as a JSON-RPC error object instead of a -/// `PayloadStatus` result, [`Self::InvalidParams`] as invalid params (`-32602`) and all other -/// variants as internal error (`-32603`). +/// These errors must not produce an `INVALID` +/// [`PayloadStatus`](alloy_rpc_types_engine::PayloadStatus) or cause the block hash to be cached as +/// invalid. Malformed supplemental input remains distinct from fatal processing failures so +/// `newPayload` can reject it as invalid params, while internal ingestion paths can discard it +/// without terminating the engine task. +#[derive(Debug, thiserror::Error)] +pub enum InsertBlockProcessingError { + /// Supplemental input is malformed, but the block itself is not known to be invalid. + #[error(transparent)] + MalformedInput(Box), + /// Block processing cannot continue because of an internal failure. + #[error(transparent)] + Fatal(#[from] InsertBlockFatalError), +} + +/// Internal errors that prevent block processing from continuing. #[derive(Debug, thiserror::Error)] pub enum InsertBlockFatalError { /// A provider error. @@ -141,10 +168,6 @@ pub enum InsertBlockFatalError { /// An internal or fatal block execution error. #[error(transparent)] BlockExecutionError(#[from] InternalBlockExecutionError), - /// The payload params are malformed, e.g. undecodable block access list bytes, and the - /// request must be rejected with an invalid params error. - #[error(transparent)] - InvalidParams(Box), } /// Error variants that are caused by invalid blocks. @@ -165,19 +188,25 @@ mod tests { // Undecodable block access list bytes are malformed request params and must not be treated // as a block validation error. #[test] - fn bal_decode_error_is_invalid_params() { - let err = InsertBlockErrorKind::Consensus(ConsensusError::BlockAccessListDecode( + fn ensure_insert_block_validation_error() { + let err = InsertBlockErrorKind::BlockAccessListDecode(BlockAccessListDecodeError::new( alloy_rlp::Error::UnexpectedString, )); assert!(matches!( err.ensure_validation_error(), - Err(InsertBlockFatalError::InvalidParams(_)) + Err(InsertBlockProcessingError::MalformedInput(_)) + )); + + assert!(matches!( + InsertBlockErrorKind::Consensus(ConsensusError::BlockAccessListHashMissing) + .ensure_validation_error(), + Ok(InsertBlockValidationError::Consensus(_)) )); + assert!(matches!( - BeaconOnNewPayloadError::from(InsertBlockFatalError::InvalidParams(Box::new( - alloy_rlp::Error::UnexpectedString - ))), - BeaconOnNewPayloadError::InvalidParams(_) + InsertBlockErrorKind::Provider(ProviderError::BestBlockNotFound) + .ensure_validation_error(), + Err(InsertBlockProcessingError::Fatal(InsertBlockFatalError::Provider(_))) )); } } diff --git a/crates/engine/tree/src/tree/error.rs b/crates/engine/tree/src/tree/error.rs index e0b19c9b6c8..80665fff564 100644 --- a/crates/engine/tree/src/tree/error.rs +++ b/crates/engine/tree/src/tree/error.rs @@ -4,7 +4,8 @@ use crate::tree::payload_processor::bal::BalExecutionError; use alloy_consensus::BlockHeader; use reth_consensus::ConsensusError; pub use reth_engine_primitives::{ - InsertBlockErrorKind, InsertBlockFatalError, InsertBlockValidationError, + BlockAccessListDecodeError, InsertBlockErrorKind, InsertBlockFatalError, + InsertBlockProcessingError, InsertBlockValidationError, }; use reth_errors::ProviderError; use reth_payload_primitives::NewPayloadError; diff --git a/crates/engine/tree/src/tree/metrics.rs b/crates/engine/tree/src/tree/metrics.rs index e7eaec53f79..3532f1b2c29 100644 --- a/crates/engine/tree/src/tree/metrics.rs +++ b/crates/engine/tree/src/tree/metrics.rs @@ -1,6 +1,6 @@ -use crate::tree::{error::InsertBlockFatalError, TreeOutcome}; +use crate::tree::TreeOutcome; use alloy_rpc_types_engine::{PayloadStatus, PayloadStatusEnum}; -use reth_engine_primitives::{ForkchoiceStatus, OnForkChoiceUpdated}; +use reth_engine_primitives::{ForkchoiceStatus, InsertBlockProcessingError, OnForkChoiceUpdated}; use reth_errors::ProviderError; use reth_evm::metrics::ExecutorMetrics; use reth_execution_types::BlockExecutionOutput; @@ -443,7 +443,7 @@ impl NewPayloadStatusMetrics { &mut self, start: Instant, latest_forkchoice_updated_at: &mut Option, - result: &Result, InsertBlockFatalError>, + result: &Result, InsertBlockProcessingError>, gas_used: u64, ) { let finish = Instant::now(); diff --git a/crates/engine/tree/src/tree/mod.rs b/crates/engine/tree/src/tree/mod.rs index 889fa7c3367..75a7bec431f 100644 --- a/crates/engine/tree/src/tree/mod.rs +++ b/crates/engine/tree/src/tree/mod.rs @@ -11,7 +11,9 @@ use alloy_primitives::{map::B256Map, B256}; use alloy_rpc_types_engine::{ ForkchoiceState, PayloadStatus, PayloadStatusEnum, PayloadValidationError, }; -use error::{InsertBlockError, InsertBlockFatalError, InsertBlockValidationError}; +use error::{ + InsertBlockError, InsertBlockFatalError, InsertBlockProcessingError, InsertBlockValidationError, +}; use reth_chain_state::{ CanonicalInMemoryState, ExecutedBlock, ExecutionTimingStats, MemoryOverlayStateProvider, NewCanonicalChain, @@ -837,7 +839,7 @@ where fn on_new_payload( &mut self, payload: T::ExecutionData, - ) -> Result, InsertBlockFatalError> { + ) -> Result, InsertBlockProcessingError> { let _thread_resource_usage = self.metrics.engine.new_payload.measure_thread_resource_usage(); trace!(target: "engine::tree", "invoked new payload"); @@ -913,7 +915,7 @@ where fn try_insert_payload( &mut self, payload: T::ExecutionData, - ) -> Result { + ) -> Result { let block_hash = payload.block_hash(); let num_hash = payload.num_hash(); let parent_hash = payload.parent_hash(); @@ -948,9 +950,9 @@ where Err(error) => { let status = match error { InsertPayloadError::Block(error) => self.on_insert_block_error(error)?, - InsertPayloadError::Payload(error) => { - self.on_new_payload_error(error, num_hash, parent_hash)? - } + InsertPayloadError::Payload(error) => self + .on_new_payload_error(error, num_hash, parent_hash) + .map_err(InsertBlockFatalError::from)?, }; Ok(TryInsertPayloadResult { status, already_seen: false }) @@ -969,7 +971,7 @@ where fn try_buffer_payload( &mut self, payload: T::ExecutionData, - ) -> Result { + ) -> Result { let parent_hash = payload.parent_hash(); let num_hash = payload.num_hash(); @@ -977,12 +979,14 @@ where // if the block is well-formed, buffer it for later Ok(block) => { if let Err(error) = self.buffer_block(block) { - Ok(self.on_insert_block_error(error)?) + self.on_insert_block_error(error) } else { Ok(PayloadStatus::from_status(PayloadStatusEnum::Syncing)) } } - Err(error) => Ok(self.on_new_payload_error(error, num_hash, parent_hash)?), + Err(error) => Ok(self + .on_new_payload_error(error, num_hash, parent_hash) + .map_err(InsertBlockFatalError::from)?), } } @@ -2717,7 +2721,7 @@ where Err(err) => { if let InsertPayloadError::Block(err) = err { debug!(target: "engine::tree", ?err, "failed to connect buffered block to tree"); - if let Err(fatal) = self.on_insert_block_error(err) { + if let Err(fatal) = self.on_internal_insert_block_error(err) { warn!(target: "engine::tree", %fatal, "fatal error occurred while connecting buffered blocks"); return Err(fatal) } @@ -3092,7 +3096,7 @@ where Err(err) => { if let InsertPayloadError::Block(err) = err { debug!(target: "engine::tree", err=%err.kind(), "failed to insert downloaded block"); - if let Err(fatal) = self.on_insert_block_error(err) { + if let Err(fatal) = self.on_internal_insert_block_error(err) { warn!(target: "engine::tree", %fatal, "fatal error occurred while inserting downloaded block"); return Err(fatal) } @@ -3294,11 +3298,9 @@ where fn on_insert_block_error( &mut self, error: InsertBlockError, - ) -> Result { + ) -> Result { let (block, error) = error.split(); - // if invalid block, we check the validation error. Otherwise return the fatal - // error. let validation_err = error.ensure_validation_error()?; // If the error was due to an invalid payload, the payload is added to the @@ -3311,7 +3313,9 @@ where %validation_err, "Invalid block error on new payload", ); - let latest_valid_hash = self.latest_valid_hash_for_invalid_payload(block.parent_hash())?; + let latest_valid_hash = self + .latest_valid_hash_for_invalid_payload(block.parent_hash()) + .map_err(InsertBlockFatalError::from)?; // keep track of the invalid header unless the consensus impl considers it transient let is_transient = match &validation_err { @@ -3340,6 +3344,17 @@ where )) } + /// Handles an insertion error from a non-RPC source, where malformed input can be discarded. + fn on_internal_insert_block_error( + &mut self, + error: InsertBlockError, + ) -> Result<(), InsertBlockFatalError> { + match self.on_insert_block_error(error) { + Ok(_) | Err(InsertBlockProcessingError::MalformedInput(_)) => Ok(()), + Err(InsertBlockProcessingError::Fatal(error)) => Err(error), + } + } + /// Handles a [`NewPayloadError`] by converting it to a [`PayloadStatus`]. fn on_new_payload_error( &mut self, diff --git a/crates/engine/tree/src/tree/payload_validator.rs b/crates/engine/tree/src/tree/payload_validator.rs index 4fc02821f1c..4b72a2b7847 100644 --- a/crates/engine/tree/src/tree/payload_validator.rs +++ b/crates/engine/tree/src/tree/payload_validator.rs @@ -96,7 +96,9 @@ //! `INVALID_PAYLOAD_ATTRIBUTES` without rolling back the forkchoice update. use crate::tree::{ - error::{InsertBlockError, InsertBlockErrorKind, InsertPayloadError}, + error::{ + BlockAccessListDecodeError, InsertBlockError, InsertBlockErrorKind, InsertPayloadError, + }, instrumented_state::{InstrumentedStateProvider, StateProviderMetrics, StateProviderStats}, payload_processor::PayloadProcessor, precompile_cache::{CachedPrecompile, CachedPrecompileMetrics, PrecompileCacheMap}, @@ -570,10 +572,9 @@ where // Extract the decoded BAL, if present. Undecodable block access list bytes are malformed // request params, not an invalid block. - let decoded_bal = ensure_ok!(input - .try_decoded_access_list() - .map_err(ConsensusError::BlockAccessListDecode)) - .map(Arc::new); + let decoded_bal = + ensure_ok!(input.try_decoded_access_list().map_err(BlockAccessListDecodeError::new)) + .map(Arc::new); if let Some(decoded_bal) = decoded_bal.as_deref() { // Reject oversized BAL sidecars before executing the block. diff --git a/crates/engine/tree/src/tree/tests.rs b/crates/engine/tree/src/tree/tests.rs index 2866f4fae5c..fa1b0130324 100644 --- a/crates/engine/tree/src/tree/tests.rs +++ b/crates/engine/tree/src/tree/tests.rs @@ -2,6 +2,7 @@ use super::*; use crate::{ persistence::PersistenceAction, tree::{ + error::{BlockAccessListDecodeError, InsertBlockErrorKind}, payload_validator::{BasicEngineValidator, TreeCtx, ValidationOutcome}, persistence_state::CurrentPersistenceAction, PersistTarget, TreeConfig, @@ -563,6 +564,22 @@ fn test_tree_persist_block_batch() { } } +#[test] +fn malformed_input_is_non_fatal_insert_outcome() { + let mut test_harness = TestHarness::new(MAINNET.clone()); + let block = test_harness.block_builder.generate_random_block(1, B256::ZERO); + let error = InsertBlockError::new( + block, + InsertBlockErrorKind::BlockAccessListDecode(BlockAccessListDecodeError::new( + alloy_rlp::Error::UnexpectedString, + )), + ); + assert_matches!( + test_harness.tree.on_insert_block_error(error), + Err(InsertBlockProcessingError::MalformedInput(_)) + ); +} + #[tokio::test] async fn test_tree_persist_blocks() { let tree_config = TreeConfig::default(); diff --git a/crates/rpc/rpc/src/validation.rs b/crates/rpc/rpc/src/validation.rs index 2d0a1136bbf..33751dec024 100644 --- a/crates/rpc/rpc/src/validation.rs +++ b/crates/rpc/rpc/src/validation.rs @@ -726,8 +726,7 @@ impl From for ErrorObject<'static> { ValidationApiError::Blob(_) => invalid_params_rpc_err(error.to_string()), ValidationApiError::Consensus( - error @ (ConsensusError::BlockAccessListDecode(_) | - ConsensusError::BlockAccessListCostMoreThanGasLimit(_) | + error @ (ConsensusError::BlockAccessListCostMoreThanGasLimit(_) | ConsensusError::BlockAccessListHashMismatch(_)), ) => invalid_params_rpc_err(error.to_string()), ValidationApiError::MissingLatestBlock | From b77a6bfd901ad01b6d450e743872f29e9bd8c358 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Thu, 27 Aug 2026 11:08:48 +0000 Subject: [PATCH 080/294] refactor(rpc): remove dead header cache (#26836) --- crates/node/core/src/args/rpc_state_cache.rs | 3 +- crates/rpc/rpc-builder/src/config.rs | 1 - crates/rpc/rpc-eth-types/src/cache/config.rs | 7 +- crates/rpc/rpc-eth-types/src/cache/mod.rs | 219 +------------------ crates/rpc/rpc-server-types/src/constants.rs | 2 +- docs/vocs/docs/pages/cli/reth/node.mdx | 5 - 6 files changed, 6 insertions(+), 231 deletions(-) diff --git a/crates/node/core/src/args/rpc_state_cache.rs b/crates/node/core/src/args/rpc_state_cache.rs index 641eef772ba..eea316055d8 100644 --- a/crates/node/core/src/args/rpc_state_cache.rs +++ b/crates/node/core/src/args/rpc_state_cache.rs @@ -22,11 +22,12 @@ pub struct RpcStateCacheArgs { )] pub max_receipts: u32, - /// Max number of headers in cache. + /// Legacy no-op retained for CLI compatibility. #[arg( long = "rpc-cache.max-headers", alias = "rpc-cache.max-envs", default_value_t = DEFAULT_HEADER_CACHE_MAX_LEN, + hide = true, )] pub max_headers: u32, diff --git a/crates/rpc/rpc-builder/src/config.rs b/crates/rpc/rpc-builder/src/config.rs index 894a0c70a65..93496950290 100644 --- a/crates/rpc/rpc-builder/src/config.rs +++ b/crates/rpc/rpc-builder/src/config.rs @@ -125,7 +125,6 @@ impl RethRpcServerConfig for RpcServerArgs { EthStateCacheConfig { max_blocks: self.rpc_state_cache.max_blocks, max_receipts: self.rpc_state_cache.max_receipts, - max_headers: self.rpc_state_cache.max_headers, max_bals: self.rpc_state_cache.max_bals, max_concurrent_db_requests: self.rpc_state_cache.max_concurrent_db_requests, max_cached_tx_hashes: self.rpc_state_cache.max_cached_tx_hashes, diff --git a/crates/rpc/rpc-eth-types/src/cache/config.rs b/crates/rpc/rpc-eth-types/src/cache/config.rs index 2ffd60dcd30..4fc1b6e90a7 100644 --- a/crates/rpc/rpc-eth-types/src/cache/config.rs +++ b/crates/rpc/rpc-eth-types/src/cache/config.rs @@ -4,7 +4,7 @@ use serde::{Deserialize, Serialize}; use reth_rpc_server_types::constants::cache::{ DEFAULT_BAL_CACHE_MAX_LEN, DEFAULT_BLOCK_CACHE_MAX_LEN, DEFAULT_CONCURRENT_DB_REQUESTS, - DEFAULT_HEADER_CACHE_MAX_LEN, DEFAULT_MAX_CACHED_TX_HASHES, DEFAULT_RECEIPT_CACHE_MAX_LEN, + DEFAULT_MAX_CACHED_TX_HASHES, DEFAULT_RECEIPT_CACHE_MAX_LEN, }; /// Settings for the [`EthStateCache`](super::EthStateCache). @@ -19,10 +19,6 @@ pub struct EthStateCacheConfig { /// /// Default is 2000. pub max_receipts: u32, - /// Max number of headers in cache. - /// - /// Default is 1000. - pub max_headers: u32, /// Max number of EVM BALs in cache. /// /// Default is 1000. @@ -40,7 +36,6 @@ impl Default for EthStateCacheConfig { Self { max_blocks: DEFAULT_BLOCK_CACHE_MAX_LEN, max_receipts: DEFAULT_RECEIPT_CACHE_MAX_LEN, - max_headers: DEFAULT_HEADER_CACHE_MAX_LEN, max_bals: DEFAULT_BAL_CACHE_MAX_LEN, max_concurrent_db_requests: DEFAULT_CONCURRENT_DB_REQUESTS, max_cached_tx_hashes: DEFAULT_MAX_CACHED_TX_HASHES, diff --git a/crates/rpc/rpc-eth-types/src/cache/mod.rs b/crates/rpc/rpc-eth-types/src/cache/mod.rs index 2be7e50b45c..f8b3461eaa6 100644 --- a/crates/rpc/rpc-eth-types/src/cache/mod.rs +++ b/crates/rpc/rpc-eth-types/src/cache/mod.rs @@ -2,11 +2,11 @@ use super::{EthStateCacheConfig, MultiConsumerLruCache}; use crate::block::CachedTransaction; -use alloy_consensus::{transaction::TxHashRef, BlockHeader}; +use alloy_consensus::transaction::TxHashRef; use alloy_eip7928::bal::DecodedBal; use alloy_eips::BlockHashOrNumber; use alloy_primitives::{Address, Bytes, TxHash, B256}; -use futures::{stream::FuturesOrdered, Stream, StreamExt}; +use futures::{Stream, StreamExt}; use reth_chain_state::CanonStateNotification; use reth_errors::{ProviderError, ProviderResult}; use reth_execution_types::Chain; @@ -53,12 +53,6 @@ type CachedBalResponseSender = oneshot::Sender>; type CachedBlockAndReceiptsResponseSender = oneshot::Sender<(Option>>, Option>>)>; -/// The type that can send the response to a requested header -type HeaderResponseSender = oneshot::Sender>; - -/// The type that can send the response with a chain of cached blocks -type CachedParentBlocksResponseSender = oneshot::Sender>>>; - /// The type that can send the response for a transaction hash lookup type TransactionHashResponseSender = oneshot::Sender>>; @@ -71,8 +65,6 @@ type BlockLruCache = type ReceiptsLruCache = MultiConsumerLruCache>, L, ReceiptsResponseSender>; -type HeaderLruCache = MultiConsumerLruCache>; - type BalLruCache = MultiConsumerLruCache; /// Provides async access to cached eth data @@ -103,7 +95,6 @@ impl EthStateCache { let EthStateCacheConfig { max_blocks, max_receipts, - max_headers, max_bals, max_concurrent_db_requests, max_cached_tx_hashes, @@ -114,7 +105,6 @@ impl EthStateCache { provider, full_block_cache: BlockLruCache::new(max_blocks, "blocks"), receipts_cache: ReceiptsLruCache::new(max_receipts, "receipts"), - headers_cache: HeaderLruCache::new(max_headers, "headers"), bal_cache: BalLruCache::new(max_bals, "bals"), action_tx: to_service.clone(), action_rx: UnboundedReceiverStream::new(rx), @@ -241,57 +231,6 @@ impl EthStateCache { rx.await.map_err(|_| CacheServiceUnavailable.into()) } - /// Streams cached receipts and blocks for a list of block hashes, preserving input order. - #[expect(clippy::type_complexity)] - pub fn get_receipts_and_maybe_block_stream<'a>( - &'a self, - hashes: Vec, - ) -> impl Stream< - Item = ProviderResult< - Option<(Arc>, Option>>)>, - >, - > + 'a { - let futures = hashes.into_iter().map(move |hash| self.get_receipts_and_maybe_block(hash)); - - futures.collect::>() - } - - /// Requests the header for the given hash. - /// - /// Returns an error if the header is not found. - pub async fn get_header(&self, block_hash: B256) -> ProviderResult { - let (response_tx, rx) = oneshot::channel(); - let _ = self.to_service.send(CacheAction::GetHeader { block_hash, response_tx }); - rx.await.map_err(|_| CacheServiceUnavailable)? - } - - /// Retrieves a chain of connected blocks from the cache, starting from the given block hash - /// and traversing down through parent hashes. Returns blocks in descending order (newest - /// first). - /// This is useful for efficiently retrieving a sequence of blocks that might already be in - /// cache without making separate database requests. - /// Returns `None` if no blocks are found in the cache, otherwise returns `Some(Vec<...>)` - /// with at least one block. - pub async fn get_cached_parent_blocks( - &self, - block_hash: B256, - max_blocks: usize, - ) -> Option>>> { - let (response_tx, rx) = oneshot::channel(); - let _ = self.to_service.send(CacheAction::GetCachedParentBlocks { - block_hash, - max_blocks, - response_tx, - }); - - let blocks = rx.await.unwrap_or_default(); - if blocks.is_empty() { - None - } else { - Some(blocks) - } - } - /// Looks up a transaction by its hash in the cache index. /// /// Returns the cached block, transaction index, and optionally receipts if the transaction @@ -352,13 +291,11 @@ pub(crate) struct EthStateCacheService< Tasks, LimitBlocks = ByLength, LimitReceipts = ByLength, - LimitHeaders = ByLength, LimitBals = ByLength, > where Provider: BlockReader + BalProvider, LimitBlocks: Limiter>>, LimitReceipts: Limiter>>, - LimitHeaders: Limiter, LimitBals: Limiter, { /// The type used to lookup data from disk @@ -367,11 +304,6 @@ pub(crate) struct EthStateCacheService< full_block_cache: BlockLruCache, /// The LRU cache for block receipts grouped by the block hash. receipts_cache: ReceiptsLruCache, - /// The LRU cache for headers. - /// - /// Headers are cached because they are required to populate the environment for execution - /// (evm). - headers_cache: HeaderLruCache, /// The LRU cache for revm BALs grouped by the block hash. bal_cache: BalLruCache, /// Sender half of the action channel. @@ -481,15 +413,6 @@ where } } - fn on_reorg_header(&mut self, block_hash: B256, res: ProviderResult) { - if let Some(queued) = self.headers_cache.remove(&block_hash) { - // send the response to queued senders - for tx in queued { - let _ = tx.send(res.clone()); - } - } - } - fn on_reorg_bal(&mut self, block_hash: B256, res: ProviderResult>) { if let Some(queued) = self.bal_cache.remove(&block_hash) { for tx in queued { @@ -503,14 +426,12 @@ where let min_capacity = 2; self.full_block_cache.shrink_to(min_capacity); self.receipts_cache.shrink_to(min_capacity); - self.headers_cache.shrink_to(min_capacity); self.bal_cache.shrink_to(min_capacity); } fn update_cached_metrics(&self) { self.full_block_cache.update_cached_metrics(); self.receipts_cache.update_cached_metrics(); - self.headers_cache.update_cached_metrics(); self.bal_cache.update_cached_metrics(); } } @@ -602,39 +523,6 @@ where }); } } - CacheAction::GetHeader { block_hash, response_tx } => { - // check if the header is cached - if let Some(header) = this.headers_cache.get(&block_hash).cloned() { - let _ = response_tx.send(Ok(header)); - continue - } - - // it's possible we have the entire block cached - if let Some(block) = this.full_block_cache.get(&block_hash) { - let _ = response_tx.send(Ok(block.clone_header())); - continue - } - - // header is not in the cache, request it if this is the first - // consumer - if this.headers_cache.queue(block_hash, response_tx) { - let provider = this.provider.clone(); - let action_tx = this.action_tx.clone(); - let rate_limiter = this.rate_limiter.clone(); - let mut action_sender = - ActionSender::new(CacheKind::Header, block_hash, action_tx); - this.action_task_spawner.spawn_blocking_task(async move { - // Acquire permit - let _permit = rate_limiter.acquire().await; - let header = provider.header(block_hash).and_then(|header| { - header.ok_or_else(|| { - ProviderError::HeaderNotFound(block_hash.into()) - }) - }); - action_sender.send_header(header); - }); - } - } CacheAction::GetBal { block_hash, response_tx } => { if let Some(bal) = this.bal_cache.get(&block_hash).cloned() { let _ = response_tx.send(Ok(Some(bal))); @@ -675,20 +563,6 @@ where this.on_new_block(block_hash, Err(e)); } }, - CacheAction::HeaderResult { block_hash, res } => { - let res = *res; - if let Some(queued) = this.headers_cache.remove(&block_hash) { - // send the response to queued senders - for tx in queued { - let _ = tx.send(res.clone()); - } - } - - // cache good header - if let Ok(data) = res { - this.headers_cache.insert(block_hash, data); - } - } CacheAction::CacheNewCanonicalChain { chain_change } => { for block in chain_change.blocks { // Index transactions before caching the block @@ -706,11 +580,9 @@ where CacheAction::RemoveReorgedChain { chain_change } => { for block in chain_change.blocks { let block_hash = block.hash(); - let header = block.clone_header(); // Remove transaction index entries for reorged blocks this.remove_block_transactions(&block); this.on_reorg_block(block_hash, Ok(Some(block))); - this.on_reorg_header(block_hash, Ok(header)); this.on_reorg_bal(block_hash, Ok(None)); } @@ -721,30 +593,6 @@ where ); } } - CacheAction::GetCachedParentBlocks { - block_hash, - max_blocks, - response_tx, - } => { - let mut blocks = Vec::new(); - let mut current_hash = block_hash; - - // Start with the requested block - while blocks.len() < max_blocks { - if let Some(block) = - this.full_block_cache.get(¤t_hash).cloned() - { - // Get the parent hash for the next iteration - current_hash = block.header().parent_hash(); - blocks.push(block); - } else { - // Break the loop if we can't find the current block - break; - } - } - - let _ = response_tx.send(blocks); - } CacheAction::GetTransactionByHash { tx_hash, response_tx } => { let result = this.tx_hash_index.get(&tx_hash).and_then(|(block_hash, idx)| { @@ -768,10 +616,6 @@ enum CacheAction { block_hash: B256, response_tx: BlockWithSendersResponseSender, }, - GetHeader { - block_hash: B256, - response_tx: HeaderResponseSender, - }, GetReceipts { block_hash: B256, response_tx: ReceiptsResponseSender, @@ -800,10 +644,6 @@ enum CacheAction { block_hash: B256, res: ProviderResult>>>, }, - HeaderResult { - block_hash: B256, - res: Box>, - }, BalResult { block_hash: B256, res: ProviderResult>, @@ -814,11 +654,6 @@ enum CacheAction { RemoveReorgedChain { chain_change: ChainChange, }, - GetCachedParentBlocks { - block_hash: B256, - max_blocks: usize, - response_tx: CachedParentBlocksResponseSender, - }, /// Look up a transaction's cached data by its hash GetTransactionByHash { tx_hash: TxHash, @@ -861,7 +696,6 @@ impl ChainChange { enum CacheKind { Block, Receipt, - Header, Bal, } @@ -897,15 +731,6 @@ impl ActionSender { } } - fn send_header(&mut self, header: Result<::Header, ProviderError>) { - if let Some(tx) = self.tx.take() { - let _ = tx.send(CacheAction::HeaderResult { - block_hash: self.blockhash, - res: Box::new(header), - }); - } - } - fn send_bal(&mut self, bal: Result, ProviderError>) { if let Some(tx) = self.tx.take() { let _ = tx.send(CacheAction::BalResult { block_hash: self.blockhash, res: bal }); @@ -924,10 +749,6 @@ impl Drop for ActionSender { block_hash: self.blockhash, res: Err(CacheServiceUnavailable.into()), }, - CacheKind::Header => CacheAction::HeaderResult { - block_hash: self.blockhash, - res: Box::new(Err(CacheServiceUnavailable.into())), - }, CacheKind::Bal => CacheAction::BalResult { block_hash: self.blockhash, res: Err(CacheServiceUnavailable.into()), @@ -1065,7 +886,6 @@ mod tests { EthStateCacheConfig { max_blocks: 4, max_receipts: 4, - max_headers: 4, max_bals: 4, max_concurrent_db_requests: 1, max_cached_tx_hashes: 16, @@ -1094,38 +914,6 @@ mod tests { ) } - #[test] - fn reorg_evicts_cached_headers() { - let mut service = test_service(); - let block_hash = B256::repeat_byte(0x11); - - assert!(service - .headers_cache - .insert(block_hash, Header { number: 42, ..Default::default() })); - assert!(service.headers_cache.get(&block_hash).is_some()); - - service.on_reorg_header(block_hash, Ok(Header { number: 7, ..Default::default() })); - - assert!(service.headers_cache.get(&block_hash).is_none()); - } - - #[test] - fn reorg_forwards_header_to_queued_requests() { - let mut service = test_service(); - let block_hash = B256::repeat_byte(0x22); - let (response_tx, mut response_rx) = oneshot::channel(); - let header = Header { number: 7, ..Default::default() }; - - assert!(service.headers_cache.queue(block_hash, response_tx)); - - service.on_reorg_header(block_hash, Ok(header)); - - let header = - response_rx.try_recv().expect("queued header response").expect("header result"); - - assert_eq!(header.number, 7); - } - #[test] fn reorg_removes_tx_hash_index_entries_unconditionally() { let mut service = test_service(); @@ -1206,7 +994,6 @@ mod tests { EthStateCacheConfig { max_blocks: 0, max_receipts: 0, - max_headers: 0, max_bals: 4, max_concurrent_db_requests: 1, max_cached_tx_hashes: 0, @@ -1232,7 +1019,6 @@ mod tests { EthStateCacheConfig { max_blocks: 4, max_receipts: 0, - max_headers: 0, max_bals: 4, max_concurrent_db_requests: 1, max_cached_tx_hashes: 0, @@ -1269,7 +1055,6 @@ mod tests { EthStateCacheConfig { max_blocks: 0, max_receipts: 0, - max_headers: 0, max_bals: 4, max_concurrent_db_requests: 1, max_cached_tx_hashes: 0, diff --git a/crates/rpc/rpc-server-types/src/constants.rs b/crates/rpc/rpc-server-types/src/constants.rs index b8e97e57e94..042768b67c9 100644 --- a/crates/rpc/rpc-server-types/src/constants.rs +++ b/crates/rpc/rpc-server-types/src/constants.rs @@ -120,7 +120,7 @@ pub mod cache { /// Default cache size for the receipts cache: 2000 receipts. pub const DEFAULT_RECEIPT_CACHE_MAX_LEN: u32 = 2000; - /// Default cache size for the header cache: 1000 headers. + /// Legacy default for the no-op RPC header cache option. pub const DEFAULT_HEADER_CACHE_MAX_LEN: u32 = 1000; /// Default cache size for the EVM BAL cache: 1000 block access lists. diff --git a/docs/vocs/docs/pages/cli/reth/node.mdx b/docs/vocs/docs/pages/cli/reth/node.mdx index d0e688744db..b234af4b03e 100644 --- a/docs/vocs/docs/pages/cli/reth/node.mdx +++ b/docs/vocs/docs/pages/cli/reth/node.mdx @@ -539,11 +539,6 @@ RPC State Cache: [default: 2000] - --rpc-cache.max-headers - Max number of headers in cache - - [default: 1000] - --rpc-cache.max-bals Max number of block access lists in cache From 4ca70ab669a3474910072d665b13789e9ec6eed0 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Thu, 27 Aug 2026 11:15:38 +0000 Subject: [PATCH 081/294] refactor(rpc): remove state provider wrapper (#26837) --- Cargo.lock | 1 - crates/rpc/rpc-eth-api/src/helpers/bal.rs | 6 +- crates/rpc/rpc-eth-api/src/helpers/call.rs | 7 +- crates/rpc/rpc-eth-types/Cargo.toml | 1 - crates/rpc/rpc-eth-types/src/cache/db.rs | 184 +-------------------- crates/rpc/rpc/src/trace.rs | 3 +- 6 files changed, 10 insertions(+), 192 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0876b9c2e54..687f2df1b90 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -10186,7 +10186,6 @@ dependencies = [ "reth-storage-api", "reth-tasks", "reth-transaction-pool", - "reth-trie", "revm", "revm-inspectors", "schnellru", diff --git a/crates/rpc/rpc-eth-api/src/helpers/bal.rs b/crates/rpc/rpc-eth-api/src/helpers/bal.rs index bf6c79ab5f3..b3c37b7606c 100644 --- a/crates/rpc/rpc-eth-api/src/helpers/bal.rs +++ b/crates/rpc/rpc-eth-api/src/helpers/bal.rs @@ -6,9 +6,7 @@ use alloy_rpc_types_eth::BlockId; use reth_errors::RethError; use reth_evm::{block::BlockExecutor, ConfigureEvm, Evm}; use reth_revm::{database::StateProviderDatabase, State}; -use reth_rpc_eth_types::{ - cache::db::StateProviderTraitObjWrapper, error::FromEthApiError, EthApiError, -}; +use reth_rpc_eth_types::{error::FromEthApiError, EthApiError}; use reth_storage_api::StateProviderFactory; use crate::{ @@ -45,7 +43,7 @@ pub trait GetBlockAccessList: Trace + Call + LoadBlock + RpcNodeCoreExt { .map_err(Self::Error::from_eth_err)?; let mut db = State::builder() - .with_database(StateProviderDatabase::new(StateProviderTraitObjWrapper(state))) + .with_database(StateProviderDatabase::new(state)) .with_bal_builder() .build(); diff --git a/crates/rpc/rpc-eth-api/src/helpers/call.rs b/crates/rpc/rpc-eth-api/src/helpers/call.rs index 060d2365298..b6e6c6cf410 100644 --- a/crates/rpc/rpc-eth-api/src/helpers/call.rs +++ b/crates/rpc/rpc-eth-api/src/helpers/call.rs @@ -33,7 +33,6 @@ use reth_revm::{ }; use reth_rpc_convert::{RpcConvert, RpcTxReq}; use reth_rpc_eth_types::{ - cache::db::StateProviderTraitObjWrapper, error::{AsEthApiError, FromEthApiError}, simulate::{self, EthSimulateError}, EthApiError, StateCacheDb, @@ -101,7 +100,7 @@ pub trait EthCall: EstimateCall + Call + LoadPendingBlock + LoadBlock + FullEthA let max_simulate_blocks = self.max_simulate_blocks(); self.spawn_with_state_at_block(block, move |this, db| { - let state_provider = db.database.0 .0; + let state_provider = db.database.0; let mut db = State::builder() .with_database(StateProviderDatabase::new(&state_provider)) .with_bundle_update() @@ -612,9 +611,7 @@ pub trait Call: let at = at.into(); self.spawn_blocking_io_fut(async move |this| { let state = this.state_at_block_id(at).await?; - let db = State::builder() - .with_database(StateProviderDatabase::new(StateProviderTraitObjWrapper(state))) - .build(); + let db = State::builder().with_database(StateProviderDatabase::new(state)).build(); f(this, db) }) } diff --git a/crates/rpc/rpc-eth-types/Cargo.toml b/crates/rpc/rpc-eth-types/Cargo.toml index 3785c89b2bc..66179c7c9c5 100644 --- a/crates/rpc/rpc-eth-types/Cargo.toml +++ b/crates/rpc/rpc-eth-types/Cargo.toml @@ -26,7 +26,6 @@ reth-rpc-server-types.workspace = true reth-rpc-convert.workspace = true reth-tasks.workspace = true reth-transaction-pool.workspace = true -reth-trie.workspace = true # ethereum alloy-chains.workspace = true diff --git a/crates/rpc/rpc-eth-types/src/cache/db.rs b/crates/rpc/rpc-eth-types/src/cache/db.rs index 644b1114a0b..00bb6737c0a 100644 --- a/crates/rpc/rpc-eth-types/src/cache/db.rs +++ b/crates/rpc/rpc-eth-types/src/cache/db.rs @@ -1,184 +1,8 @@ -//! Helper types to workaround 'higher-ranked lifetime error' -//! in default implementation of -//! `reth_rpc_eth_api::helpers::Call`. +//! EVM database types used by the RPC state cache. -use alloy_primitives::{Address, B256, U256}; -use reth_errors::ProviderResult; use reth_revm::database::StateProviderDatabase; -use reth_storage_api::{BytecodeReader, HashedPostStateProvider, StateProvider, StateProviderBox}; -use reth_trie::{HashedStorage, MultiProofTargets}; -use revm::database::{BundleState, State}; +use reth_storage_api::StateProviderBox; +use revm::database::State; /// Helper alias type for the state's [`State`] -pub type StateCacheDb = State>; - -/// Hack to get around 'higher-ranked lifetime error', see -/// -/// -/// Apparently, when dealing with our RPC code, compiler is struggling to prove lifetimes around -/// [`StateProvider`] trait objects. This type is a workaround which should help the compiler to -/// understand that there are no lifetimes involved. -#[expect(missing_debug_implementations)] -pub struct StateProviderTraitObjWrapper(pub StateProviderBox); - -impl reth_storage_api::StateRootProvider for StateProviderTraitObjWrapper { - fn state_root( - &self, - hashed_state: reth_trie::HashedPostState, - ) -> reth_errors::ProviderResult { - self.0.state_root(hashed_state) - } - - fn state_root_from_nodes( - &self, - input: reth_trie::TrieInput, - ) -> reth_errors::ProviderResult { - self.0.state_root_from_nodes(input) - } - - fn state_root_with_updates( - &self, - hashed_state: reth_trie::HashedPostState, - ) -> reth_errors::ProviderResult<(B256, reth_trie::updates::TrieUpdates)> { - self.0.state_root_with_updates(hashed_state) - } - - fn state_root_from_nodes_with_updates( - &self, - input: reth_trie::TrieInput, - ) -> reth_errors::ProviderResult<(B256, reth_trie::updates::TrieUpdates)> { - self.0.state_root_from_nodes_with_updates(input) - } -} - -impl reth_storage_api::StorageRootProvider for StateProviderTraitObjWrapper { - fn storage_root( - &self, - address: Address, - hashed_storage: HashedStorage, - ) -> ProviderResult { - self.0.storage_root(address, hashed_storage) - } - - fn storage_proof( - &self, - address: Address, - slot: B256, - hashed_storage: HashedStorage, - ) -> ProviderResult { - self.0.storage_proof(address, slot, hashed_storage) - } - - fn storage_multiproof( - &self, - address: Address, - slots: &[B256], - hashed_storage: HashedStorage, - ) -> ProviderResult { - self.0.storage_multiproof(address, slots, hashed_storage) - } -} - -impl reth_storage_api::StateProofProvider for StateProviderTraitObjWrapper { - fn proof( - &self, - input: reth_trie::TrieInput, - address: Address, - slots: &[B256], - ) -> reth_errors::ProviderResult { - self.0.proof(input, address, slots) - } - - fn multiproof( - &self, - input: reth_trie::TrieInput, - targets: MultiProofTargets, - ) -> ProviderResult { - self.0.multiproof(input, targets) - } - - fn witness( - &self, - input: reth_trie::TrieInput, - target: reth_trie::HashedPostState, - mode: reth_trie::ExecutionWitnessMode, - ) -> reth_errors::ProviderResult> { - self.0.witness(input, target, mode) - } -} - -impl reth_storage_api::AccountReader for StateProviderTraitObjWrapper { - fn basic_account( - &self, - address: &Address, - ) -> reth_errors::ProviderResult> { - self.0.basic_account(address) - } -} - -impl reth_storage_api::BlockHashReader for StateProviderTraitObjWrapper { - fn block_hash( - &self, - block_number: alloy_primitives::BlockNumber, - ) -> reth_errors::ProviderResult> { - self.0.block_hash(block_number) - } - - fn convert_block_hash( - &self, - hash_or_number: alloy_rpc_types_eth::BlockHashOrNumber, - ) -> reth_errors::ProviderResult> { - self.0.convert_block_hash(hash_or_number) - } - - fn canonical_hashes_range( - &self, - start: alloy_primitives::BlockNumber, - end: alloy_primitives::BlockNumber, - ) -> reth_errors::ProviderResult> { - self.0.canonical_hashes_range(start, end) - } -} - -impl HashedPostStateProvider for StateProviderTraitObjWrapper { - fn hashed_post_state( - &self, - bundle_state: &BundleState, - ) -> ProviderResult { - self.0.hashed_post_state(bundle_state) - } -} - -impl StateProvider for StateProviderTraitObjWrapper { - fn storage( - &self, - account: Address, - storage_key: alloy_primitives::StorageKey, - ) -> reth_errors::ProviderResult> { - self.0.storage(account, storage_key) - } - - fn account_code( - &self, - addr: &Address, - ) -> reth_errors::ProviderResult> { - self.0.account_code(addr) - } - - fn account_balance(&self, addr: &Address) -> reth_errors::ProviderResult> { - self.0.account_balance(addr) - } - - fn account_nonce(&self, addr: &Address) -> reth_errors::ProviderResult> { - self.0.account_nonce(addr) - } -} - -impl BytecodeReader for StateProviderTraitObjWrapper { - fn bytecode_by_hash( - &self, - code_hash: &B256, - ) -> reth_errors::ProviderResult> { - self.0.bytecode_by_hash(code_hash) - } -} +pub type StateCacheDb = State>; diff --git a/crates/rpc/rpc/src/trace.rs b/crates/rpc/rpc/src/trace.rs index 43b1158890a..9ab456b253a 100644 --- a/crates/rpc/rpc/src/trace.rs +++ b/crates/rpc/rpc/src/trace.rs @@ -16,7 +16,7 @@ use alloy_rpc_types_trace::{ tracerequest::TraceCallRequest, }; use async_trait::async_trait; -use futures::StreamExt; +use futures::{FutureExt, StreamExt}; use jsonrpsee::core::RpcResult; use reth_chainspec::{ChainSpecProvider, EthereumHardforks}; use reth_primitives_traits::{BlockBody, BlockHeader}; @@ -113,6 +113,7 @@ where .map_err(Eth::Error::from_eth_err)?; Ok(trace_res) }) + .boxed() .await } From 5ba8e25567b069f181a81677a73f4a6fff73ac8c Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Thu, 27 Aug 2026 12:06:02 +0000 Subject: [PATCH 082/294] fix(engine): classify invalid BAL input as malformed (#26844) --- crates/consensus/consensus/src/lib.rs | 3 -- crates/engine/tree/src/tree/error.rs | 1 + .../src/tree/payload_processor/bal/error.rs | 4 +++ .../src/tree/payload_processor/bal/execute.rs | 35 +++++++++++++++---- 4 files changed, 34 insertions(+), 9 deletions(-) diff --git a/crates/consensus/consensus/src/lib.rs b/crates/consensus/consensus/src/lib.rs index 59ab0354b38..0223f2d5df7 100644 --- a/crates/consensus/consensus/src/lib.rs +++ b/crates/consensus/consensus/src/lib.rs @@ -498,9 +498,6 @@ pub enum ConsensusError { /// EIP-7928: Error when the block access list hash doesn't match the expected value. #[error("block access list hash mismatch: {0}")] BlockAccessListHashMismatch(GotExpectedBoxed), - /// EIP-7928: Error when the block access list cannot be decoded or converted. - #[error("invalid block access list: {0}")] - BlockAccessListInvalid(String), /// Any additional consensus error, for example L2-specific errors. #[error(transparent)] Other(#[from] Arc), diff --git a/crates/engine/tree/src/tree/error.rs b/crates/engine/tree/src/tree/error.rs index 80665fff564..d2a823e0227 100644 --- a/crates/engine/tree/src/tree/error.rs +++ b/crates/engine/tree/src/tree/error.rs @@ -112,6 +112,7 @@ impl From for InsertBlockErrorKind { fn from(e: BalExecutionError) -> Self { match e { BalExecutionError::Consensus(inner) => Self::Consensus(inner), + BalExecutionError::BlockAccessListDecode(inner) => Self::BlockAccessListDecode(inner), BalExecutionError::Execution(inner) => Self::Execution(inner), BalExecutionError::Provider(inner) => Self::Provider(inner), BalExecutionError::Other(inner) => Self::Other(inner), diff --git a/crates/engine/tree/src/tree/payload_processor/bal/error.rs b/crates/engine/tree/src/tree/payload_processor/bal/error.rs index 894c0a54b94..e49c3998c62 100644 --- a/crates/engine/tree/src/tree/payload_processor/bal/error.rs +++ b/crates/engine/tree/src/tree/payload_processor/bal/error.rs @@ -1,5 +1,6 @@ //! Errors for the BAL execution path. +use reth_engine_primitives::BlockAccessListDecodeError; use reth_errors::{BlockExecutionError, ConsensusError, ProviderError}; /// Errors surfaced by `execute_block`. @@ -8,6 +9,9 @@ pub enum BalExecutionError { /// Block violated consensus rules while running the BAL path. #[error(transparent)] Consensus(#[from] ConsensusError), + /// The block access list could not be decoded for execution. + #[error(transparent)] + BlockAccessListDecode(#[from] BlockAccessListDecodeError), /// Worker or canonical EVM failure. #[error("evm execution failed: {0}")] Execution(#[from] BlockExecutionError), diff --git a/crates/engine/tree/src/tree/payload_processor/bal/execute.rs b/crates/engine/tree/src/tree/payload_processor/bal/execute.rs index dca199d9251..4b1bc14d331 100644 --- a/crates/engine/tree/src/tree/payload_processor/bal/execute.rs +++ b/crates/engine/tree/src/tree/payload_processor/bal/execute.rs @@ -25,6 +25,7 @@ use alloy_evm::{ }; use alloy_primitives::Address; use crossbeam_channel::{Receiver, Sender}; +use reth_engine_primitives::BlockAccessListDecodeError; use reth_evm::{execute::ExecutableTxFor, ConfigureEvm, Database, EvmEnvFor, ExecutionCtxFor}; use reth_primitives_traits::ReceiptTy; use reth_provider::BlockExecutionOutput; @@ -193,11 +194,8 @@ fn convert_alloy_to_revm_bal(alloy_bal: &AlloyBal) -> Result, BalEx // is triggered then the execution is reverted, and as such no actual code change event takes // place. Therefore, if we do observe such a bytecode in a BAL then that means the BAL is // invalid as no legal execution should've led to this bytecode deployment. - let received_bal_revm = RevmBal::clone_from_alloy(alloy_bal.as_vec()).map_err(|e| { - BalExecutionError::Consensus(reth_consensus::ConsensusError::BlockAccessListInvalid( - format!("{e:?}"), - )) - })?; + let received_bal_revm = + RevmBal::clone_from_alloy(alloy_bal.as_vec()).map_err(BlockAccessListDecodeError::new)?; Ok(Arc::new(received_bal_revm)) } @@ -328,8 +326,11 @@ impl BlockGasTracker { #[cfg(test)] mod tests { use super::*; + use crate::tree::error::{InsertBlockErrorKind, InsertBlockProcessingError}; use alloy_consensus::{BlockHeader, Header}; - use alloy_eip7928::{bal::Bal as AlloyBal, BlockAccessList}; + use alloy_eip7928::{ + bal::Bal as AlloyBal, AccountChanges, BlockAccessIndex, BlockAccessList, CodeChange, + }; use alloy_eips::{ eip2935::{HISTORY_STORAGE_ADDRESS, HISTORY_STORAGE_CODE}, eip4788::{BEACON_ROOTS_ADDRESS, BEACON_ROOTS_CODE}, @@ -449,6 +450,28 @@ mod tests { state.take_built_alloy_bal().expect("with_bal_builder was set") } + #[test] + fn invalid_bal_bytecode_is_malformed_input() { + let alloy_bal = vec![AccountChanges { + address: Address::ZERO, + code_changes: vec![CodeChange::new( + BlockAccessIndex::new(1), + vec![0xef, 0x01, 0xde].into(), + )], + ..Default::default() + }] + .into(); + + let error = convert_alloy_to_revm_bal(&alloy_bal).unwrap_err(); + assert!(matches!(&error, BalExecutionError::BlockAccessListDecode(_))); + let error = InsertBlockErrorKind::from(error); + assert!(matches!(&error, InsertBlockErrorKind::BlockAccessListDecode(_))); + assert!(matches!( + error.ensure_validation_error(), + Err(InsertBlockProcessingError::MalformedInput(_)) + )); + } + #[test] fn empty_block_happy_path_round_trip() { // Two-pass end-to-end: From c12b8f287f27a20ec4f7c706f8da0deb7471757d Mon Sep 17 00:00:00 2001 From: Derek Cofausper <256792747+decofe@users.noreply.github.com> Date: Thu, 27 Aug 2026 13:07:59 +0000 Subject: [PATCH 083/294] chore(deps): bump revm to 43.0.0 (#26830) Co-authored-by: Matthias Seitz <19890894+mattsse@users.noreply.github.com> Co-authored-by: rakita Co-authored-by: Matthias Seitz --- Cargo.lock | 108 +++++++++--------- Cargo.toml | 16 +-- .../custom-beacon-withdrawals/src/main.rs | 2 +- 3 files changed, 63 insertions(+), 63 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 687f2df1b90..e58fb0b55ed 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -293,9 +293,9 @@ dependencies = [ [[package]] name = "alloy-evm" -version = "0.38.0" +version = "0.39.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b307a52f91f7f3aad56b5cb4f54e5eb17dd0b3f4b2c82823301499fe1c6fdaf" +checksum = "7b6bd5f7e5ce9858bac99275e3f548c3b63bb3e5806da0142dc0b2f518d3ec4a" dependencies = [ "alloy-consensus", "alloy-eips", @@ -1967,7 +1967,7 @@ version = "3.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6dee98b0db6a962de883bf5d20362dee4d7ca0d12fe39a7c6c73c844e1cd7c1f" dependencies = [ - "darling 0.21.3", + "darling 0.23.0", "ident_case", "prettyplease", "proc-macro2", @@ -3429,7 +3429,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.52.0", + "windows-sys 0.60.2", ] [[package]] @@ -5173,7 +5173,7 @@ checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" dependencies = [ "hermit-abi", "libc", - "windows-sys 0.52.0", + "windows-sys 0.60.2", ] [[package]] @@ -6241,7 +6241,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.60.2", ] [[package]] @@ -7962,9 +7962,9 @@ dependencies = [ [[package]] name = "reth-codecs" -version = "0.6.0" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f0dd7f542fbcae0e490ebe13f3eababcc05547375246ff7f62700b533cf3f01" +checksum = "d34b0b409ecf930ed3850e92bd9b4d4f8a8c397eef23e8ba5b1a1a02682a94bd" dependencies = [ "alloy-consensus", "alloy-eips", @@ -7983,9 +7983,9 @@ dependencies = [ [[package]] name = "reth-codecs-derive" -version = "0.6.0" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7e508fcbb8cb775639dbbb56b81b62756d4322f28e76c60a52aa32575f993fe" +checksum = "ebcb49ab084e764e7d20f710e2c9daa439380d0fc33f8bfee8c372bfdbc1f437" dependencies = [ "proc-macro2", "quote", @@ -9704,9 +9704,9 @@ dependencies = [ [[package]] name = "reth-primitives-traits" -version = "0.6.0" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4666a8d5b246650b23e6767a45e46334d352557f473e91c2ea7211cc29291cca" +checksum = "9060738a07ca1b0d8ede3ce21d8187c57cc22ee9e2991b29cbbc663280f843cd" dependencies = [ "alloy-consensus", "alloy-eips", @@ -10235,9 +10235,9 @@ dependencies = [ [[package]] name = "reth-rpc-traits" -version = "0.6.0" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b417a50d3fa9b58bb1a22e46fb47e0866006f24abffa1fbe75ae60116b5b2efa" +checksum = "f410d6bff1fd059ebaec2edeff1326f103f61162158411325d78158f94b2b4a9" dependencies = [ "alloy-consensus", "alloy-network", @@ -10789,18 +10789,18 @@ dependencies = [ [[package]] name = "reth-zstd-compressors" -version = "0.6.0" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "80addacffc5df9398c55919f8993922d2b688f2aa5753708fdd658ca3cc63e39" +checksum = "08f1ebb37e81a596ce16e81bec6a0c9accbdb914cf19b9190eff69ed00395900" dependencies = [ "zstd", ] [[package]] name = "revm" -version = "42.0.1" +version = "43.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7c7fcd426beab3b91300a6d0d5eb8d583c0fad0fba451126e19fd51443eca62" +checksum = "b0d1681d52ee61171b838db4924827f63d659d4c3ad559aa3a57003f3dda3f7c" dependencies = [ "revm-bytecode", "revm-context", @@ -10817,9 +10817,9 @@ dependencies = [ [[package]] name = "revm-bytecode" -version = "42.0.0" +version = "43.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "537356bdec80dbc04e4e6bb688b3441ff7cf18eecdcf3cac23150633c46467f1" +checksum = "c8d38e16b72ec1c836ec8490562bfb21003165ea8bc496086bf345680a6e6c41" dependencies = [ "bitvec", "paste", @@ -10830,9 +10830,9 @@ dependencies = [ [[package]] name = "revm-context" -version = "42.0.0" +version = "43.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc6512f05f83378e29b2b016fa5bd7f5a533e3625548be4113732e1a536c2564" +checksum = "67822069c254eb83e95fa5912a0d1324dd176f6ea05e8f110671ad242e27d61b" dependencies = [ "bitvec", "cfg-if", @@ -10847,9 +10847,9 @@ dependencies = [ [[package]] name = "revm-context-interface" -version = "42.0.0" +version = "43.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "103fff818b10311034c0c11ff8760294ee0e3c320daab3c5717990fe8ee2617b" +checksum = "d97d0c7f8bdf65601d35ed27cf6b0f0bd436087f0a97a52aa767a49bbcea8373" dependencies = [ "alloy-eip2930", "alloy-eip7702", @@ -10863,9 +10863,9 @@ dependencies = [ [[package]] name = "revm-database" -version = "42.0.0" +version = "43.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df72760eab0852100e748c7cb03ce0221626d60ac14534118c84c5cc894fb4c9" +checksum = "31d764afa9a1c9278c05d40461b07319d21ccb39c1bdf0e9dde7e5ea7d9be53d" dependencies = [ "alloy-eips", "derive_more", @@ -10879,9 +10879,9 @@ dependencies = [ [[package]] name = "revm-database-interface" -version = "42.0.0" +version = "43.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9286c059a11be1dee814cafdc3a3ade57521c8caa050785a52c19a9d894f400" +checksum = "d2a27e4440c2ef4932afdaa28a1d1ac8601504c7030c1a387b5e307d07d864d1" dependencies = [ "auto_impl", "either", @@ -10893,9 +10893,9 @@ dependencies = [ [[package]] name = "revm-handler" -version = "42.0.1" +version = "43.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09126175c3fa482e71e80cc28c6bb17e681911e472b94d4f7f73dd6780411798" +checksum = "ab0b5923be8784704d119be5381d76a602f65e4a9991650bb87291565bd0d8ec" dependencies = [ "auto_impl", "derive-where", @@ -10912,9 +10912,9 @@ dependencies = [ [[package]] name = "revm-inspector" -version = "42.0.1" +version = "43.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4aa55b208b52f0c083ea68147c78866eeecadd7c0071a614f477de316b60648e" +checksum = "a988ad0879bd81b269cad53c235a8ce87cd44a7c1de7d21a6ff3dd9e9f3e70a9" dependencies = [ "auto_impl", "either", @@ -10930,9 +10930,9 @@ dependencies = [ [[package]] name = "revm-inspectors" -version = "0.42.2" +version = "0.43.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90c136a072540987ee442e9824631835520b13e5756f20780a637ba2e11f9c28" +checksum = "79b609cb78fcb3a8c10f3190847c9bb0223423cf46e2de30a61ea9e37a3e6736" dependencies = [ "alloy-primitives", "alloy-rpc-types-eth", @@ -10950,9 +10950,9 @@ dependencies = [ [[package]] name = "revm-interpreter" -version = "42.0.0" +version = "43.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd310927437752ce4bac6997b385772823b84843f04c82e151993bec1265a5ed" +checksum = "7ddc442e3c5006ebb65f479e9e8fded7fc10f300c040276e825eee81e00c94cc" dependencies = [ "revm-bytecode", "revm-context-interface", @@ -10963,9 +10963,9 @@ dependencies = [ [[package]] name = "revm-precompile" -version = "42.0.1" +version = "43.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2431091420999795df6a200b9f7933defe3d654710560a895e6bcabbd9954a5" +checksum = "d85f7a8906482f8d18f2a2c793bc4daf254b29465af4285268384505ce6ee308" dependencies = [ "ark-bls12-381", "ark-bn254", @@ -10990,9 +10990,9 @@ dependencies = [ [[package]] name = "revm-primitives" -version = "42.0.0" +version = "43.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ef5564c1bce219fd08beca43d4f39fdcbc19193fa4f604c0c6b58950a13194a" +checksum = "f1f4ba1a5757fe4398262290422c371556fcebf236f762f7ed7ff2c56768a63c" dependencies = [ "alloy-primitives", "once_cell", @@ -11001,9 +11001,9 @@ dependencies = [ [[package]] name = "revm-state" -version = "42.0.0" +version = "43.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c4ae66db3abfcb565854dbfc95513bfa440b3dc390ddf8e50b0363d12a6fa79" +checksum = "7cd02082cf1baca01e8bc17456c45136bdc66632395a98db1c1a2afe4c6ea87f" dependencies = [ "alloy-eip7928", "bitflags 2.13.0", @@ -11016,7 +11016,7 @@ dependencies = [ [[package]] name = "revmc" version = "0.1.0" -source = "git+https://github.com/paradigmxyz/revmc?branch=main#cf68a87f627299a9c49bcc333a8a317c9b312a3d" +source = "git+https://github.com/paradigmxyz/revmc?branch=main#68835cf19661307d293c26dd06bd1e59ccae5144" dependencies = [ "revm-bytecode", "revm-context-interface", @@ -11036,7 +11036,7 @@ dependencies = [ [[package]] name = "revmc-backend" version = "0.1.0" -source = "git+https://github.com/paradigmxyz/revmc?branch=main#cf68a87f627299a9c49bcc333a8a317c9b312a3d" +source = "git+https://github.com/paradigmxyz/revmc?branch=main#68835cf19661307d293c26dd06bd1e59ccae5144" dependencies = [ "eyre", "ruint", @@ -11045,12 +11045,12 @@ dependencies = [ [[package]] name = "revmc-build" version = "0.1.0" -source = "git+https://github.com/paradigmxyz/revmc?branch=main#cf68a87f627299a9c49bcc333a8a317c9b312a3d" +source = "git+https://github.com/paradigmxyz/revmc?branch=main#68835cf19661307d293c26dd06bd1e59ccae5144" [[package]] name = "revmc-builtins" version = "0.1.0" -source = "git+https://github.com/paradigmxyz/revmc?branch=main#cf68a87f627299a9c49bcc333a8a317c9b312a3d" +source = "git+https://github.com/paradigmxyz/revmc?branch=main#68835cf19661307d293c26dd06bd1e59ccae5144" dependencies = [ "paste", "revm-bytecode", @@ -11065,7 +11065,7 @@ dependencies = [ [[package]] name = "revmc-codegen" version = "0.1.0" -source = "git+https://github.com/paradigmxyz/revmc?branch=main#cf68a87f627299a9c49bcc333a8a317c9b312a3d" +source = "git+https://github.com/paradigmxyz/revmc?branch=main#68835cf19661307d293c26dd06bd1e59ccae5144" dependencies = [ "alloy-primitives", "bitflags 2.13.0", @@ -11096,7 +11096,7 @@ dependencies = [ [[package]] name = "revmc-context" version = "0.1.0" -source = "git+https://github.com/paradigmxyz/revmc?branch=main#cf68a87f627299a9c49bcc333a8a317c9b312a3d" +source = "git+https://github.com/paradigmxyz/revmc?branch=main#68835cf19661307d293c26dd06bd1e59ccae5144" dependencies = [ "revm-context", "revm-context-interface", @@ -11111,7 +11111,7 @@ dependencies = [ [[package]] name = "revmc-llvm" version = "0.1.0" -source = "git+https://github.com/paradigmxyz/revmc?branch=main#cf68a87f627299a9c49bcc333a8a317c9b312a3d" +source = "git+https://github.com/paradigmxyz/revmc?branch=main#68835cf19661307d293c26dd06bd1e59ccae5144" dependencies = [ "alloy-primitives", "cc", @@ -11124,7 +11124,7 @@ dependencies = [ [[package]] name = "revmc-runtime" version = "0.1.0" -source = "git+https://github.com/paradigmxyz/revmc?branch=main#cf68a87f627299a9c49bcc333a8a317c9b312a3d" +source = "git+https://github.com/paradigmxyz/revmc?branch=main#68835cf19661307d293c26dd06bd1e59ccae5144" dependencies = [ "alloy-evm", "alloy-primitives", @@ -11373,7 +11373,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.52.0", + "windows-sys 0.60.2", ] [[package]] @@ -11453,7 +11453,7 @@ dependencies = [ "security-framework", "security-framework-sys", "webpki-root-certs 1.0.7", - "windows-sys 0.52.0", + "windows-sys 0.60.2", ] [[package]] @@ -12345,7 +12345,7 @@ dependencies = [ "getrandom 0.4.2", "once_cell", "rustix", - "windows-sys 0.52.0", + "windows-sys 0.60.2", ] [[package]] @@ -13608,7 +13608,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.52.0", + "windows-sys 0.60.2", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 283f9b8d4c8..6c7cc6aaff7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -324,8 +324,8 @@ reth-cli = { path = "crates/cli/cli" } reth-cli-commands = { path = "crates/cli/commands" } reth-cli-runner = { path = "crates/cli/runner" } reth-cli-util = { path = "crates/cli/util" } -reth-codecs = { version = "0.6.0", default-features = false } -reth-codecs-derive = "0.6.0" +reth-codecs = { version = "0.7.0", default-features = false } +reth-codecs-derive = "0.7.0" reth-config = { path = "crates/config", default-features = false } reth-consensus = { path = "crates/consensus/consensus", default-features = false } reth-consensus-common = { path = "crates/consensus/common", default-features = false } @@ -393,7 +393,7 @@ reth-payload-builder-primitives = { path = "crates/payload/builder-primitives" } reth-payload-primitives = { path = "crates/payload/primitives" } reth-payload-validator = { path = "crates/payload/validator" } reth-payload-util = { path = "crates/payload/util" } -reth-primitives-traits = { version = "0.6.0", default-features = false } +reth-primitives-traits = { version = "0.7.0", default-features = false } reth-provider = { path = "crates/storage/provider" } reth-prune = { path = "crates/prune/prune" } reth-prune-types = { path = "crates/prune/types", default-features = false } @@ -408,7 +408,7 @@ reth-rpc-eth-types = { path = "crates/rpc/rpc-eth-types", default-features = fal reth-rpc-layer = { path = "crates/rpc/rpc-layer" } reth-rpc-server-types = { path = "crates/rpc/rpc-server-types" } reth-rpc-convert = { path = "crates/rpc/rpc-convert" } -reth-rpc-traits = { version = "0.6.0", default-features = false } +reth-rpc-traits = { version = "0.7.0", default-features = false } reth-stages = { path = "crates/stages/stages" } reth-stages-api = { path = "crates/stages/api" } reth-stages-types = { path = "crates/stages/types", default-features = false } @@ -428,12 +428,12 @@ reth-trie-common = { path = "crates/trie/common", default-features = false } reth-trie-db = { path = "crates/trie/db" } reth-trie-parallel = { path = "crates/trie/parallel" } reth-trie-sparse = { path = "crates/trie/sparse", default-features = false } -reth-zstd-compressors = { version = "0.6.0", default-features = false } +reth-zstd-compressors = { version = "0.7.0", default-features = false } # revm -revm = { version = "42.0.1", default-features = false } +revm = { version = "43.0.0", default-features = false } revmc = { git = "https://github.com/paradigmxyz/revmc", branch = "main", default-features = false, features = ["llvm-prefer-static"] } -revm-inspectors = "0.42.2" +revm-inspectors = "0.43.0" # eth alloy-dyn-abi = "1.6.1" @@ -443,7 +443,7 @@ alloy-sol-types = { version = "1.6.1", default-features = false } alloy-chains = { version = "0.2.33", default-features = false } alloy-eip2124 = { version = "0.2.0", default-features = false } alloy-eip7928 = { version = "0.4.8", default-features = false, features = ["rlp"] } -alloy-evm = { version = "0.38.0", default-features = false } +alloy-evm = { version = "0.39.0", default-features = false } alloy-rlp = { version = "0.3.16", default-features = false, features = ["core-net"] } alloy-trie = { version = "0.9.4", default-features = false } diff --git a/examples/custom-beacon-withdrawals/src/main.rs b/examples/custom-beacon-withdrawals/src/main.rs index efd6b0e5d97..9ab3e6bbf6a 100644 --- a/examples/custom-beacon-withdrawals/src/main.rs +++ b/examples/custom-beacon-withdrawals/src/main.rs @@ -191,7 +191,7 @@ pub struct CustomBlockExecutor<'a, Evm> { impl BlockExecutor for CustomBlockExecutor<'_, E> where - E: Evm, + E: Evm + Clone, Tx = TxEnv>, { type Transaction = TransactionSigned; type Receipt = Receipt; From 701b5a175a1aada7acb2f914327f481f8ab90a48 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Thu, 27 Aug 2026 13:46:27 +0000 Subject: [PATCH 084/294] feat(rpc): use cached BALs to position transaction replay state (#26835) Co-authored-by: Karl --- crates/rpc/rpc-eth-api/src/helpers/block.rs | 41 ++++++++ crates/rpc/rpc-eth-api/src/helpers/call.rs | 32 +++++-- crates/rpc/rpc-eth-api/src/helpers/trace.rs | 77 ++++++++++----- .../rpc-eth-api/src/helpers/transaction.rs | 34 ++++++- crates/rpc/rpc-eth-types/src/cache/db.rs | 93 ++++++++++++++++++- crates/rpc/rpc/src/debug.rs | 28 ++++-- 6 files changed, 260 insertions(+), 45 deletions(-) diff --git a/crates/rpc/rpc-eth-api/src/helpers/block.rs b/crates/rpc/rpc-eth-api/src/helpers/block.rs index 479b45623c4..755be4eda00 100644 --- a/crates/rpc/rpc-eth-api/src/helpers/block.rs +++ b/crates/rpc/rpc-eth-api/src/helpers/block.rs @@ -6,6 +6,7 @@ use crate::{ RpcReceipt, }; use alloy_consensus::{transaction::TxHashRef, TxReceipt}; +use alloy_eip7928::bal::DecodedBal; use alloy_eips::BlockId; use alloy_rlp::Encodable; use alloy_rpc_types_eth::{Block, BlockTransactions, Index}; @@ -15,6 +16,7 @@ use reth_primitives_traits::{AlloyBlockHeader, RecoveredBlock, SealedHeader, Tra use reth_rpc_convert::{transaction::ConvertReceiptInput, RpcConvert, RpcHeader}; use reth_storage_api::{BlockIdReader, BlockReader, ProviderHeader, ProviderReceipt, ProviderTx}; use reth_transaction_pool::{PoolTransaction, TransactionPool}; +use revm::state::bal::Bal as RevmBal; use std::sync::Arc; /// Result type of the fetched block receipts. @@ -294,4 +296,43 @@ pub trait LoadBlock: LoadPendingBlock + SpawnBlocking + RpcNodeCoreExt { self.cache().get_recovered_block(block_hash).await.map_err(Self::Error::from_eth_err) } } + + /// Returns the block for the given block id, together with the block's cached block access + /// list, if any. + /// + /// The BAL is only returned if it is already cached, it is never fetched from the BAL store. + /// Pending blocks never have a BAL. + #[expect(clippy::type_complexity)] + fn recovered_block_and_maybe_bal( + &self, + block_id: BlockId, + ) -> impl Future< + Output = Result< + Option<( + Arc::Block>>, + Option>>>, + )>, + Self::Error, + >, + > + Send { + async move { + if block_id.is_pending() { + return Ok(self.recovered_block(block_id).await?.map(|block| (block, None))); + } + + let block_hash = match self + .provider() + .block_hash_for_id(block_id) + .map_err(Self::Error::from_eth_err)? + { + Some(block_hash) => block_hash, + None => return Ok(None), + }; + + self.cache() + .get_recovered_block_and_maybe_bal(block_hash) + .await + .map_err(Self::Error::from_eth_err) + } + } } diff --git a/crates/rpc/rpc-eth-api/src/helpers/call.rs b/crates/rpc/rpc-eth-api/src/helpers/call.rs index b6e6c6cf410..e9bf0b16da8 100644 --- a/crates/rpc/rpc-eth-api/src/helpers/call.rs +++ b/crates/rpc/rpc-eth-api/src/helpers/call.rs @@ -33,6 +33,7 @@ use reth_revm::{ }; use reth_rpc_convert::{RpcConvert, RpcTxReq}; use reth_rpc_eth_types::{ + cache::db::attach_bal_before_tx, error::{AsEthApiError, FromEthApiError}, simulate::{self, EthSimulateError}, EthApiError, StateCacheDb, @@ -356,7 +357,9 @@ pub trait EthCall: EstimateCall + Call + LoadPendingBlock + LoadBlock + FullEthA let mut all_results = Vec::with_capacity(bundles.len()); if replay_block_txs { - this.replay_block_until(&mut db, &block, num_txs)?; + // no BAL positioning here: bundle transactions commit state on top, and an + // attached BAL would take read precedence over the committed changes + this.replay_block_until(&mut db, &block, num_txs, None)?; } // transact all bundles @@ -663,10 +666,13 @@ pub trait Call: /// Retrieves the transaction if it exists and executes it. /// - /// Before the transaction is executed, all previous transaction in the block are applied to the - /// state by executing them first. + /// Before the transaction is executed, the state is positioned right before the transaction, + /// either by attaching the block's cached BAL or by executing all previous transactions in the + /// block. /// The callback `f` is invoked with the [`ResultAndState`] after the transaction was executed - /// and the database that points to the beginning of the transaction. + /// and the database that points to the beginning of the transaction. The database may have + /// the block's BAL attached and must only be used for reads, because an attached BAL takes + /// read precedence over state committed on top, see [`attach_bal_before_tx`]. /// /// Note: Implementers should use a threadpool where blocking is allowed, such as /// [`BlockingTaskPool`](reth_tasks::pool::BlockingTaskPool). @@ -687,10 +693,11 @@ pub trait Call: R: Send + 'static, { async move { - let (transaction, block) = match self.transaction_and_block(hash).await? { - None => return Ok(None), - Some(res) => res, - }; + let (transaction, block, bal) = + match self.transaction_and_block_and_maybe_bal(hash).await? { + None => return Ok(None), + Some(res) => res, + }; let (tx, tx_info) = transaction.split(); // we need to get the state of the parent block because we're essentially replaying the @@ -698,6 +705,15 @@ pub trait Call: let parent_block = block.parent_hash(); self.spawn_with_state_at_block(parent_block, move |this, mut db| { + if let Some((bal, tx_index)) = bal.zip(tx_info.index) { + attach_bal_before_tx(&mut db, &bal, tx_index as usize); + + let evm_env = this.evm_env_for_header(block.sealed_block().sealed_header())?; + let tx_env = RpcNodeCore::evm_config(&this).tx_env(tx); + let res = this.transact(&mut db, evm_env, tx_env)?; + return f(tx_info, res, db) + } + let block_txs = block.transactions_recovered(); let mut executor = RpcNodeCore::evm_config(&this) diff --git a/crates/rpc/rpc-eth-api/src/helpers/trace.rs b/crates/rpc/rpc-eth-api/src/helpers/trace.rs index a2662147624..d51921713e5 100644 --- a/crates/rpc/rpc-eth-api/src/helpers/trace.rs +++ b/crates/rpc/rpc-eth-api/src/helpers/trace.rs @@ -3,6 +3,7 @@ use super::{Call, LoadBlock, LoadState, LoadTransaction}; use crate::{FromEthApiError, FromEvmError}; use alloy_consensus::{transaction::TxHashRef, BlockHeader}; +use alloy_eip7928::bal::DecodedBal; use alloy_primitives::B256; use alloy_rpc_types_eth::{BlockId, TransactionInfo}; use futures::Future; @@ -12,9 +13,9 @@ use reth_evm::{ EvmFor, HaltReasonFor, InspectorFor, IntoTxEnv, TxEnvFor, }; use reth_primitives_traits::{BlockBody, BlockTy, Recovered, RecoveredBlock}; -use reth_rpc_eth_types::cache::db::StateCacheDb; +use reth_rpc_eth_types::cache::db::{attach_bal_before_tx, StateCacheDb}; use reth_storage_api::{ProviderBlock, ProviderTx}; -use revm::{context::Block, context_interface::result::ResultAndState}; +use revm::{context::Block, context_interface::result::ResultAndState, state::bal::Bal as RevmBal}; use revm_inspectors::tracing::{TracingInspector, TracingInspectorConfig}; use std::sync::Arc; @@ -37,10 +38,13 @@ pub trait Trace: LoadState> + Call { /// Retrieves the transaction if it exists and returns its trace. /// - /// Before the transaction is traced, all previous transaction in the block are applied to the - /// state by executing them first. + /// Before the transaction is traced, the state is positioned right before the transaction, + /// either by attaching the block's cached BAL or by executing all previous transactions in + /// the block. /// The callback `f` is invoked with the [`ResultAndState`] after the transaction was executed - /// and the database that points to the beginning of the transaction. + /// and the database that points to the beginning of the transaction. The database may have + /// the block's BAL attached and must only be used for reads, because an attached BAL takes + /// read precedence over state committed on top, see [`attach_bal_before_tx`]. /// /// Note: Implementers should use a threadpool where blocking is allowed, such as /// [`BlockingTaskPool`](reth_tasks::pool::BlockingTaskPool). @@ -67,10 +71,13 @@ pub trait Trace: LoadState> + Call { /// Retrieves the transaction if it exists and returns its trace. /// - /// Before the transaction is traced, all previous transaction in the block are applied to the - /// state by executing them first. + /// Before the transaction is traced, the state is positioned right before the transaction, + /// either by attaching the block's cached BAL or by executing all previous transactions in + /// the block. /// The callback `f` is invoked with the [`ResultAndState`] after the transaction was executed - /// and the database that points to the beginning of the transaction. + /// and the database that points to the beginning of the transaction. The database may have + /// the block's BAL attached and must only be used for reads, because an attached BAL takes + /// read precedence over state committed on top, see [`attach_bal_before_tx`]. /// /// Note: Implementers should use a threadpool where blocking is allowed, such as /// [`BlockingTaskPool`](reth_tasks::pool::BlockingTaskPool). @@ -94,10 +101,11 @@ pub trait Trace: LoadState> + Call { R: Send + 'static, { async move { - let (transaction, block) = match self.transaction_and_block(hash).await? { - None => return Ok(None), - Some(res) => res, - }; + let (transaction, block, bal) = + match self.transaction_and_block_and_maybe_bal(hash).await? { + None => return Ok(None), + Some(res) => res, + }; let (tx, tx_info) = transaction.split(); // we need to get the state of the parent block because we're essentially replaying the @@ -114,6 +122,7 @@ pub trait Trace: LoadState> + Call { tx_info.index.expect("transaction_and_block only returns block transactions") as usize, tx, + bal.as_deref(), )?; f(tx_info, inspector, res, db) }) @@ -122,10 +131,12 @@ pub trait Trace: LoadState> + Call { } } - /// Replays all transactions before the target transaction index. + /// Positions the state of `db` right before the transaction at the target index. /// - /// All transactions before the target transaction are executed and their changes are written to - /// the _runtime_ db ([`StateCacheDb`]). + /// If the block's cached BAL is given, it is attached to the database at the target index and + /// no transactions are executed, see [`attach_bal_before_tx`]. Otherwise all transactions + /// before the target transaction are executed and their changes are written to the + /// _runtime_ db ([`StateCacheDb`]). /// /// If the target index is greater than or equal to the block's transaction count, all /// transactions are replayed. @@ -134,7 +145,13 @@ pub trait Trace: LoadState> + Call { db: &mut StateCacheDb, block: &RecoveredBlock>, target_tx_index: usize, + bal: Option<&DecodedBal>>, ) -> Result<(), Self::Error> { + if let Some(bal) = bal { + attach_bal_before_tx(db, bal, target_tx_index); + return Ok(()) + } + self.apply_pre_execution_changes(block, db)?; let evm_env = self.evm_env_for_header(block.sealed_block().sealed_header())?; @@ -146,8 +163,12 @@ pub trait Trace: LoadState> + Call { ) } - /// Replays all transactions before the target transaction without inspection, then executes - /// the target transaction with the configured inspector, all on the given EVM. + /// Executes the target transaction with the configured inspector on the state right before + /// the transaction. + /// + /// If the block's cached BAL is given, the state is positioned by attaching the BAL at the + /// target index, see [`attach_bal_before_tx`]. Otherwise all transactions before the target + /// transaction are replayed without inspection first. #[expect(clippy::type_complexity)] fn inspect_transaction_in_block<'a>( &self, @@ -156,17 +177,27 @@ pub trait Trace: LoadState> + Call { inspector: impl InspectorFor, target_tx_index: usize, target_tx_env: impl IntoTxEnv>, + bal: Option<&DecodedBal>>, ) -> Result<(ResultAndState>, EvmEnvFor), Self::Error> { - let block_txs = block.transactions_recovered(); - - self.apply_pre_execution_changes(block, db)?; + if let Some(bal) = bal { + // the BAL also covers the block's pre-execution changes + attach_bal_before_tx(db, bal, target_tx_index); + } else { + self.apply_pre_execution_changes(block, db)?; + } let evm_env = self.evm_env_for_header(block.sealed_block().sealed_header())?; let mut evm = self.evm_config().evm_with_env_and_inspector(db, evm_env, inspector); - evm.disable_inspector(); - self.replay_transactions_until_with_evm(&mut evm, block_txs, target_tx_index)?; - evm.enable_inspector(); + if bal.is_none() { + evm.disable_inspector(); + self.replay_transactions_until_with_evm( + &mut evm, + block.transactions_recovered(), + target_tx_index, + )?; + evm.enable_inspector(); + } let res = evm.transact(target_tx_env).map_err(Self::Error::from_evm_err)?; diff --git a/crates/rpc/rpc-eth-api/src/helpers/transaction.rs b/crates/rpc/rpc-eth-api/src/helpers/transaction.rs index c34a6fe2955..674cc825fd5 100644 --- a/crates/rpc/rpc-eth-api/src/helpers/transaction.rs +++ b/crates/rpc/rpc-eth-api/src/helpers/transaction.rs @@ -12,6 +12,7 @@ use alloy_consensus::{ BlockHeader, Transaction, }; use alloy_dyn_abi::TypedData; +use alloy_eip7928::bal::DecodedBal; use alloy_eips::{eip2718::Encodable2718, BlockId}; use alloy_network::{TransactionBuilder, TransactionBuilder4844}; use alloy_primitives::{Address, Bytes, TxHash, B256, U256}; @@ -36,6 +37,7 @@ use reth_transaction_pool::{ AddedTransactionOutcome, PoolPooledTx, PoolTransaction, PoolTx, TransactionOrigin, TransactionPool, }; +use revm::state::bal::Bal as RevmBal; use std::{sync::Arc, time::Duration}; /// Transaction related functions for the [`EthApiServer`](crate::EthApiServer) trait in @@ -806,6 +808,32 @@ pub trait LoadTransaction: SpawnBlocking + FullEthApiTypes + RpcNodeCoreExt { )>, Self::Error, >, + > + Send { + async move { + Ok(self + .transaction_and_block_and_maybe_bal(hash) + .await? + .map(|(transaction, block, _)| (transaction, block))) + } + } + + /// Fetches the transaction and the transaction's block, together with the block's cached + /// block access list, if any. + /// + /// The BAL is only returned if it is already cached, it is never fetched from the BAL store. + #[expect(clippy::type_complexity)] + fn transaction_and_block_and_maybe_bal( + &self, + hash: B256, + ) -> impl Future< + Output = Result< + Option<( + TransactionSource>, + Arc>>, + Option>>>, + )>, + Self::Error, + >, > + Send { async move { let (transaction, at) = match self.transaction_by_hash_at(hash).await? { @@ -818,12 +846,12 @@ pub trait LoadTransaction: SpawnBlocking + FullEthApiTypes + RpcNodeCoreExt { BlockId::Hash(hash) => hash.block_hash, _ => return Ok(None), }; - let block = self + let block_and_bal = self .cache() - .get_recovered_block(block_hash) + .get_recovered_block_and_maybe_bal(block_hash) .await .map_err(Self::Error::from_eth_err)?; - Ok(block.map(|block| (transaction, block))) + Ok(block_and_bal.map(|(block, bal)| (transaction, block, bal))) } } } diff --git a/crates/rpc/rpc-eth-types/src/cache/db.rs b/crates/rpc/rpc-eth-types/src/cache/db.rs index 00bb6737c0a..0a2220e5204 100644 --- a/crates/rpc/rpc-eth-types/src/cache/db.rs +++ b/crates/rpc/rpc-eth-types/src/cache/db.rs @@ -1,8 +1,99 @@ //! EVM database types used by the RPC state cache. +use alloy_eip7928::{bal::DecodedBal, BlockAccessIndex}; use reth_revm::database::StateProviderDatabase; use reth_storage_api::StateProviderBox; -use revm::database::State; +use revm::{database::State, state::bal::Bal as RevmBal, Database}; +use std::sync::Arc; /// Helper alias type for the state's [`State`] pub type StateCacheDb = State>; + +/// Attaches `bal` to the database, positioned at the state right before the transaction at +/// `tx_index`. +/// +/// Reads served by the attached BAL reflect all writes prior to the transaction, including the +/// block's pre-execution system calls. Reads not covered by the BAL fall back to the underlying +/// database, which holds the correct values for all state the block does not touch. +/// +/// Note: changes must not be committed to the database afterwards, because the attached BAL takes +/// precedence over committed state when serving reads. +#[inline] +pub fn attach_bal_before_tx( + db: &mut State, + bal: &DecodedBal>, + tx_index: usize, +) { + db.set_bal(Some(bal.as_bal().clone())); + db.set_allow_bal_db_fallback(true); + db.set_bal_index(BlockAccessIndex::from_tx_index(tx_index as u64)); +} + +#[cfg(test)] +mod tests { + use super::*; + use alloy_primitives::{address, Bytes, U256}; + use revm::{ + database::{CacheDB, EmptyDB}, + state::{ + bal::{AccountBal, Bal, BalWrites, BlockAccessIndex}, + AccountInfo, + }, + }; + + #[test] + fn attach_bal_before_tx_serves_positioned_reads() { + let covered = address!("0x0000000000000000000000000000000000000001"); + let uncovered = address!("0x0000000000000000000000000000000000000002"); + let written_slot = U256::from(1); + let read_slot = U256::from(2); + + // pre-block state + let mut db = CacheDB::new(EmptyDB::default()); + db.insert_account_info( + covered, + AccountInfo { balance: U256::from(7), nonce: 5, ..Default::default() }, + ); + db.insert_account_storage(covered, written_slot, U256::from(11)).unwrap(); + db.insert_account_storage(covered, read_slot, U256::from(99)).unwrap(); + db.insert_account_info( + uncovered, + AccountInfo { balance: U256::from(3), ..Default::default() }, + ); + + // tx 1 writes the slot, tx 2 changes the balance + let mut account = AccountBal::default(); + account.storage.storage.insert( + written_slot, + BalWrites::new(vec![(BlockAccessIndex::from_tx_index(1), U256::from(42))]), + ); + account.account_info.balance = + BalWrites::new(vec![(BlockAccessIndex::from_tx_index(2), U256::from(1000))]); + let mut bal = Bal::default(); + bal.accounts.insert(covered, account); + let bal = DecodedBal::new(Arc::new(bal), Bytes::new()); + + let mut state = State::builder().with_database(db).build(); + + // before tx 0, none of the block's writes are visible + attach_bal_before_tx(&mut state, &bal, 0); + assert_eq!(Database::storage(&mut state, covered, written_slot).unwrap(), U256::from(11)); + assert_eq!(Database::basic(&mut state, covered).unwrap().unwrap().balance, U256::from(7)); + + // before tx 2, the storage write of tx 1 is visible, the balance change of tx 2 is not + attach_bal_before_tx(&mut state, &bal, 2); + assert_eq!(Database::storage(&mut state, covered, written_slot).unwrap(), U256::from(42)); + assert_eq!(Database::basic(&mut state, covered).unwrap().unwrap().balance, U256::from(7)); + + // before tx 3, the balance change of tx 2 is visible + attach_bal_before_tx(&mut state, &bal, 3); + assert_eq!( + Database::basic(&mut state, covered).unwrap().unwrap().balance, + U256::from(1000) + ); + + // reads not covered by the BAL fall back to the underlying database + assert_eq!(Database::storage(&mut state, covered, read_slot).unwrap(), U256::from(99)); + assert_eq!(Database::basic(&mut state, uncovered).unwrap().unwrap().balance, U256::from(3)); + } +} diff --git a/crates/rpc/rpc/src/debug.rs b/crates/rpc/rpc/src/debug.rs index 0ff27fdecf8..c0739108cab 100644 --- a/crates/rpc/rpc/src/debug.rs +++ b/crates/rpc/rpc/src/debug.rs @@ -316,10 +316,11 @@ where tx_hash: B256, opts: GethDebugTracingOptions, ) -> Result { - let (transaction, block) = match self.eth_api().transaction_and_block(tx_hash).await? { - None => return Err(EthApiError::TracingTransactionNotFound.into()), - Some(res) => res, - }; + let (transaction, block, bal) = + match self.eth_api().transaction_and_block_and_maybe_bal(tx_hash).await? { + None => return Err(EthApiError::TracingTransactionNotFound.into()), + Some(res) => res, + }; self.eth_api() .spawn_with_state_at_block(block.parent_hash(), move |eth_api, mut db| { @@ -340,6 +341,7 @@ where &mut inspector, index, tx_env.clone(), + bal.as_deref(), )?; let trace = inspector @@ -423,9 +425,9 @@ where overrides: EvmOverrides, ) -> Result { // Get the target block to check transaction count - let block = self + let (block, bal) = self .eth_api() - .recovered_block(block_id) + .recovered_block_and_maybe_bal(block_id) .await? .ok_or(EthApiError::HeaderNotFound(block_id))?; @@ -439,12 +441,16 @@ where .into()) } + // state overrides commit changes to the database, which an attached BAL would take read + // precedence over, so only position via BAL if there are none + let bal = bal.filter(|_| !overrides.has_state()); + let evm_env = self.eth_api().evm_env_for_header(block.sealed_block().sealed_header())?; self.eth_api() .spawn_with_state_at_block(block.parent_hash(), move |eth_api, mut db| { - // 1. replay the required number of transactions - eth_api.replay_block_until(&mut db, &block, tx_index)?; + // 1. position the state before the transaction at the index + eth_api.replay_block_until(&mut db, &block, tx_index, bal.as_deref())?; // 2. now execute the trace call on this state let (evm_env, tx_env) = @@ -515,8 +521,10 @@ where if replay_block_txs { // only need to replay the transactions in the block if not all transactions are // to be replayed - // Execute all transactions until index - eth_api.replay_block_until(&mut db, &block, num_txs)?; + // Execute all transactions until index. No BAL positioning here: bundle + // transactions commit state on top, and an attached BAL would take read + // precedence over the committed changes + eth_api.replay_block_until(&mut db, &block, num_txs, None)?; } // Trace all bundles From 66a08aba2274d3446caf5d8849fda9b6a0e2f770 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Fri, 28 Aug 2026 11:09:43 +0200 Subject: [PATCH 085/294] revert: "chore(deps): bump revm to 43.0.0" (#26858) --- Cargo.lock | 108 +++++++++--------- Cargo.toml | 16 +-- .../custom-beacon-withdrawals/src/main.rs | 2 +- 3 files changed, 63 insertions(+), 63 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e58fb0b55ed..687f2df1b90 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -293,9 +293,9 @@ dependencies = [ [[package]] name = "alloy-evm" -version = "0.39.0" +version = "0.38.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b6bd5f7e5ce9858bac99275e3f548c3b63bb3e5806da0142dc0b2f518d3ec4a" +checksum = "6b307a52f91f7f3aad56b5cb4f54e5eb17dd0b3f4b2c82823301499fe1c6fdaf" dependencies = [ "alloy-consensus", "alloy-eips", @@ -1967,7 +1967,7 @@ version = "3.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6dee98b0db6a962de883bf5d20362dee4d7ca0d12fe39a7c6c73c844e1cd7c1f" dependencies = [ - "darling 0.23.0", + "darling 0.21.3", "ident_case", "prettyplease", "proc-macro2", @@ -3429,7 +3429,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.60.2", + "windows-sys 0.52.0", ] [[package]] @@ -5173,7 +5173,7 @@ checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" dependencies = [ "hermit-abi", "libc", - "windows-sys 0.60.2", + "windows-sys 0.52.0", ] [[package]] @@ -6241,7 +6241,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.60.2", + "windows-sys 0.59.0", ] [[package]] @@ -7962,9 +7962,9 @@ dependencies = [ [[package]] name = "reth-codecs" -version = "0.7.0" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d34b0b409ecf930ed3850e92bd9b4d4f8a8c397eef23e8ba5b1a1a02682a94bd" +checksum = "1f0dd7f542fbcae0e490ebe13f3eababcc05547375246ff7f62700b533cf3f01" dependencies = [ "alloy-consensus", "alloy-eips", @@ -7983,9 +7983,9 @@ dependencies = [ [[package]] name = "reth-codecs-derive" -version = "0.7.0" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebcb49ab084e764e7d20f710e2c9daa439380d0fc33f8bfee8c372bfdbc1f437" +checksum = "e7e508fcbb8cb775639dbbb56b81b62756d4322f28e76c60a52aa32575f993fe" dependencies = [ "proc-macro2", "quote", @@ -9704,9 +9704,9 @@ dependencies = [ [[package]] name = "reth-primitives-traits" -version = "0.7.0" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9060738a07ca1b0d8ede3ce21d8187c57cc22ee9e2991b29cbbc663280f843cd" +checksum = "4666a8d5b246650b23e6767a45e46334d352557f473e91c2ea7211cc29291cca" dependencies = [ "alloy-consensus", "alloy-eips", @@ -10235,9 +10235,9 @@ dependencies = [ [[package]] name = "reth-rpc-traits" -version = "0.7.0" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f410d6bff1fd059ebaec2edeff1326f103f61162158411325d78158f94b2b4a9" +checksum = "b417a50d3fa9b58bb1a22e46fb47e0866006f24abffa1fbe75ae60116b5b2efa" dependencies = [ "alloy-consensus", "alloy-network", @@ -10789,18 +10789,18 @@ dependencies = [ [[package]] name = "reth-zstd-compressors" -version = "0.7.0" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08f1ebb37e81a596ce16e81bec6a0c9accbdb914cf19b9190eff69ed00395900" +checksum = "80addacffc5df9398c55919f8993922d2b688f2aa5753708fdd658ca3cc63e39" dependencies = [ "zstd", ] [[package]] name = "revm" -version = "43.0.0" +version = "42.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0d1681d52ee61171b838db4924827f63d659d4c3ad559aa3a57003f3dda3f7c" +checksum = "b7c7fcd426beab3b91300a6d0d5eb8d583c0fad0fba451126e19fd51443eca62" dependencies = [ "revm-bytecode", "revm-context", @@ -10817,9 +10817,9 @@ dependencies = [ [[package]] name = "revm-bytecode" -version = "43.0.0" +version = "42.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8d38e16b72ec1c836ec8490562bfb21003165ea8bc496086bf345680a6e6c41" +checksum = "537356bdec80dbc04e4e6bb688b3441ff7cf18eecdcf3cac23150633c46467f1" dependencies = [ "bitvec", "paste", @@ -10830,9 +10830,9 @@ dependencies = [ [[package]] name = "revm-context" -version = "43.0.0" +version = "42.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67822069c254eb83e95fa5912a0d1324dd176f6ea05e8f110671ad242e27d61b" +checksum = "cc6512f05f83378e29b2b016fa5bd7f5a533e3625548be4113732e1a536c2564" dependencies = [ "bitvec", "cfg-if", @@ -10847,9 +10847,9 @@ dependencies = [ [[package]] name = "revm-context-interface" -version = "43.0.0" +version = "42.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d97d0c7f8bdf65601d35ed27cf6b0f0bd436087f0a97a52aa767a49bbcea8373" +checksum = "103fff818b10311034c0c11ff8760294ee0e3c320daab3c5717990fe8ee2617b" dependencies = [ "alloy-eip2930", "alloy-eip7702", @@ -10863,9 +10863,9 @@ dependencies = [ [[package]] name = "revm-database" -version = "43.0.0" +version = "42.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31d764afa9a1c9278c05d40461b07319d21ccb39c1bdf0e9dde7e5ea7d9be53d" +checksum = "df72760eab0852100e748c7cb03ce0221626d60ac14534118c84c5cc894fb4c9" dependencies = [ "alloy-eips", "derive_more", @@ -10879,9 +10879,9 @@ dependencies = [ [[package]] name = "revm-database-interface" -version = "43.0.0" +version = "42.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2a27e4440c2ef4932afdaa28a1d1ac8601504c7030c1a387b5e307d07d864d1" +checksum = "d9286c059a11be1dee814cafdc3a3ade57521c8caa050785a52c19a9d894f400" dependencies = [ "auto_impl", "either", @@ -10893,9 +10893,9 @@ dependencies = [ [[package]] name = "revm-handler" -version = "43.0.0" +version = "42.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab0b5923be8784704d119be5381d76a602f65e4a9991650bb87291565bd0d8ec" +checksum = "09126175c3fa482e71e80cc28c6bb17e681911e472b94d4f7f73dd6780411798" dependencies = [ "auto_impl", "derive-where", @@ -10912,9 +10912,9 @@ dependencies = [ [[package]] name = "revm-inspector" -version = "43.0.0" +version = "42.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a988ad0879bd81b269cad53c235a8ce87cd44a7c1de7d21a6ff3dd9e9f3e70a9" +checksum = "4aa55b208b52f0c083ea68147c78866eeecadd7c0071a614f477de316b60648e" dependencies = [ "auto_impl", "either", @@ -10930,9 +10930,9 @@ dependencies = [ [[package]] name = "revm-inspectors" -version = "0.43.0" +version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79b609cb78fcb3a8c10f3190847c9bb0223423cf46e2de30a61ea9e37a3e6736" +checksum = "90c136a072540987ee442e9824631835520b13e5756f20780a637ba2e11f9c28" dependencies = [ "alloy-primitives", "alloy-rpc-types-eth", @@ -10950,9 +10950,9 @@ dependencies = [ [[package]] name = "revm-interpreter" -version = "43.0.0" +version = "42.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ddc442e3c5006ebb65f479e9e8fded7fc10f300c040276e825eee81e00c94cc" +checksum = "fd310927437752ce4bac6997b385772823b84843f04c82e151993bec1265a5ed" dependencies = [ "revm-bytecode", "revm-context-interface", @@ -10963,9 +10963,9 @@ dependencies = [ [[package]] name = "revm-precompile" -version = "43.0.0" +version = "42.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d85f7a8906482f8d18f2a2c793bc4daf254b29465af4285268384505ce6ee308" +checksum = "f2431091420999795df6a200b9f7933defe3d654710560a895e6bcabbd9954a5" dependencies = [ "ark-bls12-381", "ark-bn254", @@ -10990,9 +10990,9 @@ dependencies = [ [[package]] name = "revm-primitives" -version = "43.0.0" +version = "42.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1f4ba1a5757fe4398262290422c371556fcebf236f762f7ed7ff2c56768a63c" +checksum = "2ef5564c1bce219fd08beca43d4f39fdcbc19193fa4f604c0c6b58950a13194a" dependencies = [ "alloy-primitives", "once_cell", @@ -11001,9 +11001,9 @@ dependencies = [ [[package]] name = "revm-state" -version = "43.0.0" +version = "42.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7cd02082cf1baca01e8bc17456c45136bdc66632395a98db1c1a2afe4c6ea87f" +checksum = "2c4ae66db3abfcb565854dbfc95513bfa440b3dc390ddf8e50b0363d12a6fa79" dependencies = [ "alloy-eip7928", "bitflags 2.13.0", @@ -11016,7 +11016,7 @@ dependencies = [ [[package]] name = "revmc" version = "0.1.0" -source = "git+https://github.com/paradigmxyz/revmc?branch=main#68835cf19661307d293c26dd06bd1e59ccae5144" +source = "git+https://github.com/paradigmxyz/revmc?branch=main#cf68a87f627299a9c49bcc333a8a317c9b312a3d" dependencies = [ "revm-bytecode", "revm-context-interface", @@ -11036,7 +11036,7 @@ dependencies = [ [[package]] name = "revmc-backend" version = "0.1.0" -source = "git+https://github.com/paradigmxyz/revmc?branch=main#68835cf19661307d293c26dd06bd1e59ccae5144" +source = "git+https://github.com/paradigmxyz/revmc?branch=main#cf68a87f627299a9c49bcc333a8a317c9b312a3d" dependencies = [ "eyre", "ruint", @@ -11045,12 +11045,12 @@ dependencies = [ [[package]] name = "revmc-build" version = "0.1.0" -source = "git+https://github.com/paradigmxyz/revmc?branch=main#68835cf19661307d293c26dd06bd1e59ccae5144" +source = "git+https://github.com/paradigmxyz/revmc?branch=main#cf68a87f627299a9c49bcc333a8a317c9b312a3d" [[package]] name = "revmc-builtins" version = "0.1.0" -source = "git+https://github.com/paradigmxyz/revmc?branch=main#68835cf19661307d293c26dd06bd1e59ccae5144" +source = "git+https://github.com/paradigmxyz/revmc?branch=main#cf68a87f627299a9c49bcc333a8a317c9b312a3d" dependencies = [ "paste", "revm-bytecode", @@ -11065,7 +11065,7 @@ dependencies = [ [[package]] name = "revmc-codegen" version = "0.1.0" -source = "git+https://github.com/paradigmxyz/revmc?branch=main#68835cf19661307d293c26dd06bd1e59ccae5144" +source = "git+https://github.com/paradigmxyz/revmc?branch=main#cf68a87f627299a9c49bcc333a8a317c9b312a3d" dependencies = [ "alloy-primitives", "bitflags 2.13.0", @@ -11096,7 +11096,7 @@ dependencies = [ [[package]] name = "revmc-context" version = "0.1.0" -source = "git+https://github.com/paradigmxyz/revmc?branch=main#68835cf19661307d293c26dd06bd1e59ccae5144" +source = "git+https://github.com/paradigmxyz/revmc?branch=main#cf68a87f627299a9c49bcc333a8a317c9b312a3d" dependencies = [ "revm-context", "revm-context-interface", @@ -11111,7 +11111,7 @@ dependencies = [ [[package]] name = "revmc-llvm" version = "0.1.0" -source = "git+https://github.com/paradigmxyz/revmc?branch=main#68835cf19661307d293c26dd06bd1e59ccae5144" +source = "git+https://github.com/paradigmxyz/revmc?branch=main#cf68a87f627299a9c49bcc333a8a317c9b312a3d" dependencies = [ "alloy-primitives", "cc", @@ -11124,7 +11124,7 @@ dependencies = [ [[package]] name = "revmc-runtime" version = "0.1.0" -source = "git+https://github.com/paradigmxyz/revmc?branch=main#68835cf19661307d293c26dd06bd1e59ccae5144" +source = "git+https://github.com/paradigmxyz/revmc?branch=main#cf68a87f627299a9c49bcc333a8a317c9b312a3d" dependencies = [ "alloy-evm", "alloy-primitives", @@ -11373,7 +11373,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.60.2", + "windows-sys 0.52.0", ] [[package]] @@ -11453,7 +11453,7 @@ dependencies = [ "security-framework", "security-framework-sys", "webpki-root-certs 1.0.7", - "windows-sys 0.60.2", + "windows-sys 0.52.0", ] [[package]] @@ -12345,7 +12345,7 @@ dependencies = [ "getrandom 0.4.2", "once_cell", "rustix", - "windows-sys 0.60.2", + "windows-sys 0.52.0", ] [[package]] @@ -13608,7 +13608,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.60.2", + "windows-sys 0.52.0", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 6c7cc6aaff7..283f9b8d4c8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -324,8 +324,8 @@ reth-cli = { path = "crates/cli/cli" } reth-cli-commands = { path = "crates/cli/commands" } reth-cli-runner = { path = "crates/cli/runner" } reth-cli-util = { path = "crates/cli/util" } -reth-codecs = { version = "0.7.0", default-features = false } -reth-codecs-derive = "0.7.0" +reth-codecs = { version = "0.6.0", default-features = false } +reth-codecs-derive = "0.6.0" reth-config = { path = "crates/config", default-features = false } reth-consensus = { path = "crates/consensus/consensus", default-features = false } reth-consensus-common = { path = "crates/consensus/common", default-features = false } @@ -393,7 +393,7 @@ reth-payload-builder-primitives = { path = "crates/payload/builder-primitives" } reth-payload-primitives = { path = "crates/payload/primitives" } reth-payload-validator = { path = "crates/payload/validator" } reth-payload-util = { path = "crates/payload/util" } -reth-primitives-traits = { version = "0.7.0", default-features = false } +reth-primitives-traits = { version = "0.6.0", default-features = false } reth-provider = { path = "crates/storage/provider" } reth-prune = { path = "crates/prune/prune" } reth-prune-types = { path = "crates/prune/types", default-features = false } @@ -408,7 +408,7 @@ reth-rpc-eth-types = { path = "crates/rpc/rpc-eth-types", default-features = fal reth-rpc-layer = { path = "crates/rpc/rpc-layer" } reth-rpc-server-types = { path = "crates/rpc/rpc-server-types" } reth-rpc-convert = { path = "crates/rpc/rpc-convert" } -reth-rpc-traits = { version = "0.7.0", default-features = false } +reth-rpc-traits = { version = "0.6.0", default-features = false } reth-stages = { path = "crates/stages/stages" } reth-stages-api = { path = "crates/stages/api" } reth-stages-types = { path = "crates/stages/types", default-features = false } @@ -428,12 +428,12 @@ reth-trie-common = { path = "crates/trie/common", default-features = false } reth-trie-db = { path = "crates/trie/db" } reth-trie-parallel = { path = "crates/trie/parallel" } reth-trie-sparse = { path = "crates/trie/sparse", default-features = false } -reth-zstd-compressors = { version = "0.7.0", default-features = false } +reth-zstd-compressors = { version = "0.6.0", default-features = false } # revm -revm = { version = "43.0.0", default-features = false } +revm = { version = "42.0.1", default-features = false } revmc = { git = "https://github.com/paradigmxyz/revmc", branch = "main", default-features = false, features = ["llvm-prefer-static"] } -revm-inspectors = "0.43.0" +revm-inspectors = "0.42.2" # eth alloy-dyn-abi = "1.6.1" @@ -443,7 +443,7 @@ alloy-sol-types = { version = "1.6.1", default-features = false } alloy-chains = { version = "0.2.33", default-features = false } alloy-eip2124 = { version = "0.2.0", default-features = false } alloy-eip7928 = { version = "0.4.8", default-features = false, features = ["rlp"] } -alloy-evm = { version = "0.39.0", default-features = false } +alloy-evm = { version = "0.38.0", default-features = false } alloy-rlp = { version = "0.3.16", default-features = false, features = ["core-net"] } alloy-trie = { version = "0.9.4", default-features = false } diff --git a/examples/custom-beacon-withdrawals/src/main.rs b/examples/custom-beacon-withdrawals/src/main.rs index 9ab3e6bbf6a..efd6b0e5d97 100644 --- a/examples/custom-beacon-withdrawals/src/main.rs +++ b/examples/custom-beacon-withdrawals/src/main.rs @@ -191,7 +191,7 @@ pub struct CustomBlockExecutor<'a, Evm> { impl BlockExecutor for CustomBlockExecutor<'_, E> where - E: Evm + Clone, Tx = TxEnv>, + E: Evm, { type Transaction = TransactionSigned; type Receipt = Receipt; From 692fc6999c44d251c93d86317bb095b41534c218 Mon Sep 17 00:00:00 2001 From: Brian Picciano Date: Fri, 28 Aug 2026 10:30:38 +0000 Subject: [PATCH 086/294] feat(overlay): cache execution overlays in manager (#26834) --- Cargo.lock | 1 + .../src/providers/state/historical.rs | 6 +- crates/storage/storage-overlay/Cargo.toml | 1 + crates/storage/storage-overlay/src/builder.rs | 110 +++- .../storage-overlay/src/changeset_cache.rs | 8 +- crates/storage/storage-overlay/src/lib.rs | 2 + crates/storage/storage-overlay/src/manager.rs | 609 +++++++++++++++--- .../storage-overlay/src/manager_metrics.rs | 54 ++ .../storage/storage-overlay/src/provider.rs | 34 +- 9 files changed, 676 insertions(+), 149 deletions(-) create mode 100644 crates/storage/storage-overlay/src/manager_metrics.rs diff --git a/Cargo.lock b/Cargo.lock index 687f2df1b90..1ce04d65684 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -10471,6 +10471,7 @@ dependencies = [ "reth-trie", "reth-trie-common", "reth-trie-db", + "revm", "tracing", ] diff --git a/crates/storage/provider/src/providers/state/historical.rs b/crates/storage/provider/src/providers/state/historical.rs index 348e295497c..864acfad2ad 100644 --- a/crates/storage/provider/src/providers/state/historical.rs +++ b/crates/storage/provider/src/providers/state/historical.rs @@ -18,7 +18,7 @@ use reth_storage_api::{ StorageSettingsCache, }; use reth_storage_errors::provider::ProviderResult; -use reth_storage_overlay::{Overlay, OverlayManager}; +use reth_storage_overlay::{OverlayManager, StateTrieOverlay}; use reth_trie::{ hashed_cursor::{zero_destroyed_account_storage, HashedPostStateCursorFactory}, proof::{Proof, StorageProof}, @@ -302,8 +302,8 @@ where .overlay_manager .overlay_builder(anchor_hash) .with_immediate_state_trie_overlay(state, nodes); - let Overlay { trie_updates, hashed_post_state } = - overlay_builder.build_overlay(self.provider)?; + let StateTrieOverlay { trie_updates, hashed_post_state } = + overlay_builder.build_state_trie_overlay(self.provider)?; Ok(TrieInputSorted::new(trie_updates, hashed_post_state, prefix_sets)) } diff --git a/crates/storage/storage-overlay/Cargo.toml b/crates/storage/storage-overlay/Cargo.toml index 08a1d77a7c8..0fc018be217 100644 --- a/crates/storage/storage-overlay/Cargo.toml +++ b/crates/storage/storage-overlay/Cargo.toml @@ -39,6 +39,7 @@ metrics.workspace = true parking_lot.workspace = true rayon = { workspace = true, optional = true } tracing.workspace = true +revm.workspace = true [dev-dependencies] reth-chain-state = { workspace = true, features = ["test-utils"] } diff --git a/crates/storage/storage-overlay/src/builder.rs b/crates/storage/storage-overlay/src/builder.rs index ae2a9088c42..6d3c921fb3a 100644 --- a/crates/storage/storage-overlay/src/builder.rs +++ b/crates/storage/storage-overlay/src/builder.rs @@ -1,6 +1,9 @@ use crate::OverlayManager; use alloy_eips::BlockNumHash; -use alloy_primitives::{BlockHash, B256}; +use alloy_primitives::{ + map::{AddressMap, B256Map, U256Map}, + BlockHash, B256, U256, +}; use metrics::{Counter, Histogram}; use reth_chain_state::ExecutedBlock; use reth_errors::{ProviderError, ProviderResult}; @@ -15,6 +18,7 @@ use reth_storage_api::{ }; use reth_trie::{updates::TrieUpdatesSorted, HashedPostStateSorted}; use reth_trie_db::DatabaseHashedPostState; +use revm::{bytecode::Bytecode, state::AccountInfo}; use std::{ sync::Arc, time::{Duration, Instant}, @@ -23,14 +27,14 @@ use tracing::{debug, debug_span, instrument}; /// Contains the trie and hashed-state data required to initialize an overlay state provider. #[derive(Debug, Clone)] -pub struct Overlay { +pub struct StateTrieOverlay { /// Trie updates overlay. pub trie_updates: Arc, /// Hashed state overlay. pub hashed_post_state: Arc, } -impl Overlay { +impl StateTrieOverlay { fn empty() -> Self { Self { trie_updates: Arc::new(TrieUpdatesSorted::default()), @@ -39,6 +43,59 @@ impl Overlay { } } +/// Execution state required to initialize an overlay state provider. +/// +/// Account entries preserve known non-existence, while storage and code entries contain only data +/// explicitly observed during execution. This intentionally does not retain account status or +/// known-storage wipes, so it cannot represent pre-Dencun `SELFDESTRUCT` of existing accounts. +/// The execution overlay is used only for post-Dencun engine execution, where that operation does +/// not clear existing storage; pipeline sync does not use it. Extend this representation before +/// using it for pre-Dencun execution. +#[derive(Clone, Debug, Default)] +pub struct ExecutionOverlay { + /// In-memory block hashes in ascending block-number order. + pub block_hashes: Vec, + /// Account state by address. + pub accounts: AddressMap>, + /// Storage values by address and slot. + pub storage: AddressMap>, + /// Bytecode by code hash. + pub code_hashes: B256Map, +} + +impl ExecutionOverlay { + pub(crate) fn extend(&mut self, block: &ExecutedBlock) { + self.block_hashes.push(block.recovered_block().num_hash()); + let state = &block.execution_output.state; + let (accounts, storage, code_hashes) = + (&mut self.accounts, &mut self.storage, &mut self.code_hashes); + + #[allow(unused_mut)] + let mut extend_state = || { + for (address, account) in state.state() { + accounts.insert(*address, account.info.clone()); + let account_storage = storage.entry(*address).or_default(); + for (slot, value) in &account.storage { + account_storage.insert(*slot, value.present_value); + } + } + }; + #[allow(unused_mut)] + let mut extend_code_hashes = || { + code_hashes.extend(state.contracts.iter().map(|(hash, code)| (*hash, code.clone()))); + }; + + #[cfg(feature = "rayon")] + rayon::join(extend_state, extend_code_hashes); + + #[cfg(not(feature = "rayon"))] + { + extend_state(); + extend_code_hashes(); + } + } +} + /// Source of data to apply on top of the durable database state. #[derive(Debug, Clone)] pub enum OverlaySource { @@ -160,9 +217,12 @@ impl OverlayBuilder { } } - /// Builds the effective overlay for the given provider. + /// Builds the effective state trie overlay for the given provider. #[instrument(level = "debug", target = "storage::overlay", skip_all)] - pub fn build_overlay(&self, provider: &Provider) -> ProviderResult + pub fn build_state_trie_overlay( + &self, + provider: &Provider, + ) -> ProviderResult where Provider: StageCheckpointReader + PruneCheckpointReader @@ -173,10 +233,10 @@ impl OverlayBuilder { + StorageSettingsCache, { let (state_trie_tip_block, finish_tip_block) = database_state_frontiers(provider)?; - self.build_overlay_at_frontiers(provider, state_trie_tip_block, finish_tip_block) + self.build_state_trie_overlay_at_frontiers(provider, state_trie_tip_block, finish_tip_block) } - /// Builds the effective overlay using frontiers already read from the provider. + /// Builds the effective state trie overlay using frontiers already read from the provider. /// /// This is useful for callers that key an overlay cache by the durable frontiers. #[instrument( @@ -185,12 +245,12 @@ impl OverlayBuilder { skip_all, fields(?state_trie_tip_block, ?finish_tip_block, parent_hash = ?self.parent_hash) )] - pub fn build_overlay_at_frontiers( + pub fn build_state_trie_overlay_at_frontiers( &self, provider: &Provider, state_trie_tip_block: BlockNumHash, finish_tip_block: BlockNumHash, - ) -> ProviderResult + ) -> ProviderResult where Provider: ChangeSetReader + StorageChangeSetReader @@ -253,7 +313,8 @@ impl OverlayBuilder { // Resolve overlays and extend reverts with them. If reverts are empty, use overlays // directly to avoid cloning. - let (overlay_trie, overlay_state) = self.resolve_overlays(anchor.hash)?; + let (overlay_trie, overlay_state) = + self.resolve_state_trie_overlays(anchor.hash)?; let trie_updates = if trie_reverts.is_empty() { overlay_trie @@ -306,10 +367,11 @@ impl OverlayBuilder { self.metrics.sparse_trie_overlay_skips.increment(1); - return Ok(Overlay::empty()) + return Ok(StateTrieOverlay::empty()) } - let (trie_updates, hashed_post_state) = self.resolve_overlays(anchor.hash)?; + let (trie_updates, hashed_post_state) = + self.resolve_state_trie_overlays(anchor.hash)?; retrieve_trie_reverts_duration = Duration::ZERO; retrieve_hashed_state_reverts_duration = Duration::ZERO; @@ -337,11 +399,11 @@ impl OverlayBuilder { self.metrics.trie_updates_size.record(trie_updates_total_len as f64); self.metrics.hashed_state_size.record(hashed_state_updates_total_len as f64); - Ok(Overlay { trie_updates, hashed_post_state }) + Ok(StateTrieOverlay { trie_updates, hashed_post_state }) } - /// Resolves the effective overlay (trie updates, hashed state). - fn resolve_overlays( + /// Resolves the effective state trie overlay (trie updates, hashed state). + fn resolve_state_trie_overlays( &self, anchor_hash: BlockHash, ) -> ProviderResult<(Arc, Arc)> { @@ -678,11 +740,11 @@ mod tests { (factory, blocks) } - fn account_keys(overlay: &Overlay) -> Vec { + fn account_keys(overlay: &StateTrieOverlay) -> Vec { overlay.hashed_post_state.accounts.iter().map(|(key, _)| *key).collect() } - fn account_node_paths(overlay: &Overlay) -> Vec { + fn account_node_paths(overlay: &StateTrieOverlay) -> Vec { overlay.trie_updates.account_nodes_ref().iter().map(|(path, _)| *path).collect() } @@ -698,7 +760,7 @@ mod tests { for (parent_index, expected_ids) in [(3, vec![3, 4]), (4, vec![3, 4, 5])] { let overlay = manager .overlay_builder(blocks[parent_index].recovered_block().hash()) - .build_overlay(&provider) + .build_state_trie_overlay(&provider) .unwrap(); assert_eq!( @@ -726,7 +788,7 @@ mod tests { let overlay = manager .overlay_builder(blocks[4].recovered_block().hash()) .with_skip_overlay_for_reused_sparse_trie(blocks[3].recovered_block().hash()) - .build_overlay(&provider) + .build_state_trie_overlay(&provider) .unwrap(); assert!(overlay.hashed_post_state.is_empty()); @@ -741,7 +803,7 @@ mod tests { let error = OverlayManager::::default() .overlay_builder(blocks[1].recovered_block().hash()) .with_no_reverts() - .build_overlay(&provider) + .build_state_trie_overlay(&provider) .unwrap_err(); assert!(error.to_string().contains("reverts are disabled")); @@ -773,7 +835,7 @@ mod tests { let error = OverlayManager::::default() .overlay_builder(blocks[3].recovered_block().hash()) .with_overlay_source(None) - .build_overlay(&provider) + .build_state_trie_overlay(&provider) .unwrap_err(); assert!( @@ -789,7 +851,7 @@ mod tests { let parent_hash = blocks[3].recovered_block().hash(); let error = OverlayManager::::default() .overlay_builder(parent_hash) - .build_overlay(&provider) + .build_state_trie_overlay(&provider) .unwrap_err(); assert!(error.to_string().contains("is after partial state trie frontier")); @@ -800,7 +862,7 @@ mod tests { let parent_hash = B256::with_last_byte(1); let builder = OverlayManager::::default().overlay_builder(parent_hash); - let (trie, state) = builder.resolve_overlays(parent_hash).unwrap(); + let (trie, state) = builder.resolve_state_trie_overlays(parent_hash).unwrap(); assert!(trie.is_empty()); assert!(state.is_empty()); } @@ -811,7 +873,7 @@ mod tests { let anchor_hash = B256::with_last_byte(2); let builder = OverlayManager::::default().overlay_builder(parent_hash); - let err = builder.resolve_overlays(anchor_hash).unwrap_err(); + let err = builder.resolve_state_trie_overlays(anchor_hash).unwrap_err(); assert!(err.to_string().contains("cannot be anchored")); } diff --git a/crates/storage/storage-overlay/src/changeset_cache.rs b/crates/storage/storage-overlay/src/changeset_cache.rs index 54f713d8d54..240e6236fdb 100644 --- a/crates/storage/storage-overlay/src/changeset_cache.rs +++ b/crates/storage/storage-overlay/src/changeset_cache.rs @@ -401,7 +401,7 @@ impl ChangesetCache { let overlay = overlay_manager .overlay_builder(finish.hash) .with_no_reverts() - .build_overlay_at_frontiers(provider, partial_state_trie, finish)?; + .build_state_trie_overlay_at_frontiers(provider, partial_state_trie, finish)?; let state_trie_provider = OverlayStateProvider::new( provider, overlay, @@ -615,7 +615,7 @@ impl ChangesetCacheInner { #[cfg(test)] mod tests { use super::*; - use crate::Overlay; + use crate::StateTrieOverlay; use alloy_consensus::Header; use alloy_primitives::{ keccak256, @@ -641,8 +641,8 @@ mod tests { Arc::new(TrieUpdatesSorted::new(vec![], B256Map::default())) } - fn empty_overlay() -> Overlay { - Overlay { trie_updates: Arc::default(), hashed_post_state: Arc::default() } + fn empty_overlay() -> StateTrieOverlay { + StateTrieOverlay { trie_updates: Arc::default(), hashed_post_state: Arc::default() } } fn insert_test_changesets( diff --git a/crates/storage/storage-overlay/src/lib.rs b/crates/storage/storage-overlay/src/lib.rs index 426d3cc9e30..2594c8c3e37 100644 --- a/crates/storage/storage-overlay/src/lib.rs +++ b/crates/storage/storage-overlay/src/lib.rs @@ -11,5 +11,7 @@ pub(crate) use changeset_cache::ChangesetCache; mod manager; pub use manager::*; +mod manager_metrics; + mod provider; pub use provider::*; diff --git a/crates/storage/storage-overlay/src/manager.rs b/crates/storage/storage-overlay/src/manager.rs index 466d9625703..a56a59143da 100644 --- a/crates/storage/storage-overlay/src/manager.rs +++ b/crates/storage/storage-overlay/src/manager.rs @@ -1,12 +1,14 @@ -//! Flattened state trie overlays for in-memory blocks. +//! State trie and execution overlays for in-memory blocks. //! //! Payload validation needs a view of the state trie as of an in-memory parent block even when that -//! parent has not been persisted yet. [`OverlayManager`] tracks those in-memory blocks and -//! builds reusable flattened state trie overlays on demand. +//! parent has not been persisted yet. [`OverlayManager`] tracks those in-memory blocks and builds +//! reusable state trie and execution overlays on demand. use crate::{ - changeset_cache::compute_block_trie_updates, database_state_frontiers, ChangesetCache, - OverlayBuilder, + changeset_cache::compute_block_trie_updates, + database_state_frontiers, + manager_metrics::{ExecutionOverlayMetrics, OverlayCacheMetrics, StateTrieOverlayMetrics}, + ChangesetCache, ExecutionOverlay, OverlayBuilder, }; use alloy_eips::BlockNumHash; use alloy_primitives::{BlockNumber, B256}; @@ -14,10 +16,6 @@ use parking_lot::Mutex; use reth_chain_state::{ExecutedBlock, PreservedSparseTrie}; use reth_errors::ProviderResult; use reth_ethereum_primitives::EthPrimitives; -use reth_metrics::{ - metrics::{Counter, Histogram}, - Metrics, -}; use reth_primitives_traits::{ dashmap::{mapref::entry::Entry, DashMap}, AlloyBlockHeader, FastInstant, NodePrimitives, @@ -37,43 +35,35 @@ use std::{ }; use tracing::{debug, trace}; -/// Manages flattened state trie overlays for in-memory blocks. +/// Manages state trie and execution overlays for in-memory blocks. /// -/// The manager owns the in-memory block graph, changeset cache, and a cache of flattened state trie -/// overlays keyed by `(anchor_hash, tip_hash)`. +/// The manager owns the in-memory block graph, changeset cache, and caches keyed by +/// `(anchor_hash, tip_hash)`. #[derive(Clone)] pub struct OverlayManager { blocks: Arc>>, - overlays: Arc>, + state_trie_overlays: OverlayCache, + execution_overlays: OverlayCache, changeset_cache: ChangesetCache, preserved_sparse_trie: Arc>>, #[cfg(feature = "rayon")] worker_pool: Option>, metrics: StateTrieOverlayMetrics, -} - -/// Metrics for state trie overlay management. -#[derive(Clone, Metrics)] -#[metrics(scope = "sync.block_validation.state_trie_overlay")] -struct StateTrieOverlayMetrics { - /// Duration of overlay computation in seconds. - overlay_computation_duration_seconds: Histogram, - /// Number of requests satisfied by an existing overlay cache entry. - overlay_cache_reuses: Counter, - /// Number of overlay cache entries populated by computing an overlay. - overlay_cache_fills: Counter, + execution_metrics: ExecutionOverlayMetrics, } impl Default for OverlayManager { fn default() -> Self { Self { blocks: Default::default(), - overlays: Default::default(), + state_trie_overlays: Default::default(), + execution_overlays: Default::default(), changeset_cache: Default::default(), preserved_sparse_trie: Default::default(), #[cfg(feature = "rayon")] worker_pool: None, metrics: Default::default(), + execution_metrics: Default::default(), } } } @@ -82,7 +72,8 @@ impl std::fmt::Debug for OverlayManager { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("OverlayManager") .field("blocks", &self.blocks.len()) - .field("overlays", &self.overlays.len()) + .field("state_trie_overlays", &self.state_trie_overlays.len()) + .field("execution_overlays", &self.execution_overlays.len()) .finish() } } @@ -93,11 +84,13 @@ impl OverlayManager { pub fn new(worker_pool: Arc) -> Self { Self { blocks: Default::default(), - overlays: Default::default(), + state_trie_overlays: Default::default(), + execution_overlays: Default::default(), changeset_cache: Default::default(), preserved_sparse_trie: Default::default(), worker_pool: Some(worker_pool), metrics: Default::default(), + execution_metrics: Default::default(), } } @@ -241,12 +234,57 @@ impl OverlayManager { } } + // Snapshot matching parent overlays before spawning so DashMap iteration guards are + // dropped. + let cached_parent_overlays = self + .execution_overlays + .entries + .iter() + .filter_map(|entry| { + let key = *entry.key(); + (key.tip_hash == parent_hash).then_some(key.anchor_hash) + }) + .collect::>(); + debug!( target: "storage::overlay::manager", %hash, %parent_hash, "inserted block into state trie overlay manager" ); + if cached_parent_overlays.is_empty() { + return + } + + #[cfg(feature = "rayon")] + let Some(worker_pool) = self.worker_pool.clone() else { + return + }; + + #[cfg(not(feature = "rayon"))] + let _ = cached_parent_overlays; + + // When a new block is inserted we optimistically and asynchronously flatten an execution + // overlay for it + #[cfg(feature = "rayon")] + { + let parent_span = span; + for anchor_hash in cached_parent_overlays { + let manager = self.clone(); + let parent_span = parent_span.clone(); + worker_pool.spawn(move || { + let _span = tracing::trace_span!( + target: "storage::overlay::manager", + parent: parent_span, + "precompute_execution_overlay", + tip_hash = %hash, + anchor_hash = %anchor_hash, + ) + .entered(); + let _ = manager.precompute_execution_overlay_for_parent(hash, anchor_hash); + }); + } + } } /// Removes blocks from the live block graph and prunes cached overlays that can no longer be @@ -276,11 +314,15 @@ impl OverlayManager { span.record("removed_blocks", removed_blocks); if removed_blocks > 0 { - let overlays_before = self.overlays.len(); - self.overlays.retain(|key, _| { + let overlays_before = self.state_trie_overlays.len() + self.execution_overlays.len(); + self.state_trie_overlays.retain(|key, _| { + self.contains_hash(key.tip_hash, key.anchor_hash, key.anchor_hash) + }); + self.execution_overlays.retain(|key, _| { self.contains_hash(key.tip_hash, key.anchor_hash, key.anchor_hash) }); - pruned_overlays = overlays_before.saturating_sub(self.overlays.len()); + pruned_overlays = overlays_before + .saturating_sub(self.state_trie_overlays.len() + self.execution_overlays.len()); span.record("pruned_overlays", pruned_overlays); } debug!( @@ -310,10 +352,65 @@ impl OverlayManager { %anchor_hash, "loading state trie overlay for parent" ); - let input = self.get_overlay(parent_hash, anchor_hash)?; + let input = self + .get_or_compute_overlay( + &self.state_trie_overlays, + &self.metrics, + parent_hash, + anchor_hash, + true, + |input, span| self.compute_state_trie_overlay(input, anchor_hash, span), + )? + .expect("required overlay lookup cannot skip an in-progress computation"); Ok((Arc::clone(&input.nodes), Arc::clone(&input.state))) } + /// Returns execution data for the in-memory chain from `anchor_hash` to `parent_hash`. + #[tracing::instrument( + level = "trace", + target = "storage::overlay::manager", + skip_all, + fields(tip_hash = %parent_hash, anchor_hash = %anchor_hash) + )] + pub fn execution_overlay_for_parent( + &self, + parent_hash: B256, + anchor_hash: B256, + ) -> Result, StateTrieOverlayError> { + Ok(self + .execution_overlay_for_parent_inner(parent_hash, anchor_hash, true)? + .expect("required overlay lookup cannot skip an in-progress computation")) + } + + #[cfg(feature = "rayon")] + fn precompute_execution_overlay_for_parent( + &self, + parent_hash: B256, + anchor_hash: B256, + ) -> Result<(), StateTrieOverlayError> { + self.execution_overlay_for_parent_inner(parent_hash, anchor_hash, false).map(drop) + } + + fn execution_overlay_for_parent_inner( + &self, + parent_hash: B256, + anchor_hash: B256, + wait_for_pending: bool, + ) -> Result>, StateTrieOverlayError> { + if parent_hash == anchor_hash { + return Ok(Some(Arc::new(ExecutionOverlay::default()))) + } + + self.get_or_compute_overlay( + &self.execution_overlays, + &self.execution_metrics, + parent_hash, + anchor_hash, + wait_for_pending, + |input, span| self.compute_execution_overlay(input, anchor_hash, span), + ) + } + #[tracing::instrument( level = "trace", target = "storage::overlay::manager", @@ -326,20 +423,29 @@ impl OverlayManager { parent_overlay_reused = tracing::field::Empty, ) )] - fn get_overlay( + fn get_or_compute_overlay( &self, + cache: &OverlayCache, + metrics: &M, tip_hash: B256, anchor_hash: B256, - ) -> Result, StateTrieOverlayError> { + wait_for_pending: bool, + compute: impl FnOnce(ComputeOverlayInput, tracing::Span) -> T, + ) -> Result>, StateTrieOverlayError> + where + M: OverlayCacheMetrics, + { let key = OverlayCacheKey { anchor_hash, tip_hash }; let span = tracing::Span::current(); - if let Some(entry) = self.overlays.get(&key).map(|entry| entry.value().clone()) { - self.record_overlay_cache_reuse(&span); - return Ok(match entry { - OverlayCacheEntry::Ready(input) => input, - OverlayCacheEntry::Computing(waiter) => waiter.wait(), - }) + if let Some(entry) = cache.entries.get(&key).map(|entry| entry.value().clone()) { + metrics.record_cache_reuse(); + span.record("cache_reused", true); + return match entry { + OverlayCacheEntry::Ready(input) => Ok(Some(input)), + OverlayCacheEntry::Computing(waiter) if wait_for_pending => Ok(Some(waiter.wait())), + OverlayCacheEntry::Computing(_) => Ok(None), + } } span.record("cache_reused", false); @@ -358,41 +464,25 @@ impl OverlayManager { hash = parent_hash; } span.record("block_count", blocks.len()); - let parent_input = blocks.first().and_then(|block| { - let parent_hash = block.recovered_block().parent_hash(); - (parent_hash != anchor_hash) - .then(|| { - self.overlays - .get(&OverlayCacheKey { anchor_hash, tip_hash: parent_hash }) - .and_then(|entry| entry.value().ready()) - }) - .flatten() - }); - span.record("parent_overlay_reused", parent_input.is_some()); - let compute_input = match parent_input { - Some(parent_input) => { - ComputeOverlayInput::ExtendCached { block: blocks.swap_remove(0), parent_input } - } - None => ComputeOverlayInput::MergeBlocks(blocks), - }; - - enum CacheAction { - Ready(Arc), - Wait(Arc), - Compute(Arc), + enum CacheAction { + Ready(Arc), + Wait(Arc>), + Compute(Arc>), } - let action = match self.overlays.entry(key) { + let action = match cache.entries.entry(key) { Entry::Occupied(entry) => { let entry = entry.get().clone(); - self.record_overlay_cache_reuse(&span); + metrics.record_cache_reuse(); + span.record("cache_reused", true); match entry { OverlayCacheEntry::Ready(input) => CacheAction::Ready(input), + OverlayCacheEntry::Computing(_) if !wait_for_pending => return Ok(None), OverlayCacheEntry::Computing(waiter) => CacheAction::Wait(waiter), } } Entry::Vacant(entry) => { - self.metrics.overlay_cache_fills.increment(1); + metrics.record_cache_fill(); let waiter = Arc::new(OverlayWaiter::new()); entry.insert(OverlayCacheEntry::Computing(Arc::clone(&waiter))); CacheAction::Compute(waiter) @@ -400,13 +490,30 @@ impl OverlayManager { }; match action { - CacheAction::Ready(input) => Ok(input), - CacheAction::Wait(waiter) => Ok(waiter.wait()), + CacheAction::Ready(input) => Ok(Some(input)), + CacheAction::Wait(waiter) => Ok(Some(waiter.wait())), CacheAction::Compute(waiter) => { - let input = self.compute_overlay(compute_input, anchor_hash, span); + let parent_input = blocks.first().and_then(|block| { + let parent_hash = block.recovered_block().parent_hash(); + (parent_hash != anchor_hash) + .then(|| { + cache + .take_ready(&OverlayCacheKey { anchor_hash, tip_hash: parent_hash }) + }) + .flatten() + }); + span.record("parent_overlay_reused", parent_input.is_some()); + let compute_input = match parent_input { + Some(parent_input) => ComputeOverlayInput::ExtendCached { + block: blocks.swap_remove(0), + parent_input, + }, + None => ComputeOverlayInput::MergeBlocks(blocks), + }; + let input = Arc::new(compute(compute_input, span)); waiter.finish(Arc::clone(&input)); - if let Entry::Occupied(mut entry) = self.overlays.entry(key) { + if let Entry::Occupied(mut entry) = cache.entries.entry(key) { // The entry may have been pruned while the overlay was computing. Only cache // the result if the map still points at the waiter installed by this task. let should_publish = match entry.get() { @@ -418,16 +525,11 @@ impl OverlayManager { } } - Ok(input) + Ok(Some(input)) } } } - fn record_overlay_cache_reuse(&self, span: &tracing::Span) { - self.metrics.overlay_cache_reuses.increment(1); - span.record("cache_reused", true); - } - /// Returns every in-memory block in the chain whose tip is `parent_hash`. pub(crate) fn parent_chain( &self, @@ -459,25 +561,46 @@ impl OverlayManager { } } - fn compute_overlay( + fn compute_state_trie_overlay( &self, - compute_input: ComputeOverlayInput, + compute_input: ComputeOverlayInput, anchor_hash: B256, _span: tracing::Span, - ) -> Arc { + ) -> TrieInputSorted { #[cfg(feature = "rayon")] { if let Some(worker_pool) = &self.worker_pool { let compute_span = _span; let metrics = self.metrics.clone(); - return Arc::new(worker_pool.install_fn(move || { + return worker_pool.install_fn(move || { let _guard = compute_span.enter(); compute_overlay(compute_input, anchor_hash, &metrics) - })) + }) + } + } + + compute_overlay(compute_input, anchor_hash, &self.metrics) + } + + fn compute_execution_overlay( + &self, + compute_input: ComputeOverlayInput, + anchor_hash: B256, + _span: tracing::Span, + ) -> ExecutionOverlay { + #[cfg(feature = "rayon")] + { + if let Some(worker_pool) = &self.worker_pool { + let compute_span = _span; + let metrics = self.execution_metrics.clone(); + return worker_pool.install_fn(move || { + let _guard = compute_span.enter(); + compute_execution_overlay_inner(compute_input, anchor_hash, &metrics) + }) } } - Arc::new(compute_overlay(compute_input, anchor_hash, &self.metrics)) + compute_execution_overlay_inner(compute_input, anchor_hash, &self.execution_metrics) } } @@ -508,41 +631,77 @@ struct OverlayCacheKey { tip_hash: B256, } -#[derive(Clone)] -enum OverlayCacheEntry { - Ready(Arc), - Computing(Arc), +struct OverlayCache { + entries: Arc>>, } -impl OverlayCacheEntry { - fn ready(&self) -> Option> { +impl Default for OverlayCache { + fn default() -> Self { + Self { entries: Default::default() } + } +} + +impl Clone for OverlayCache { + fn clone(&self) -> Self { + Self { entries: Arc::clone(&self.entries) } + } +} + +impl OverlayCache { + fn len(&self) -> usize { + self.entries.len() + } + + fn retain(&self, mut keep: impl FnMut(&OverlayCacheKey, &OverlayCacheEntry) -> bool) { + self.entries.retain(|key, entry| keep(key, entry)) + } + + /// Removes and returns a ready entry. + /// + /// Transferring a parent entry lets `Arc::make_mut` extend it in place when no caller retains + /// it. Keeping the cache entry would otherwise guarantee a clone. + fn take_ready(&self, key: &OverlayCacheKey) -> Option> { + let (_, entry) = + self.entries.remove_if(key, |_, entry| matches!(entry, OverlayCacheEntry::Ready(_)))?; + let OverlayCacheEntry::Ready(input) = entry else { unreachable!() }; + Some(input) + } +} + +enum OverlayCacheEntry { + Ready(Arc), + Computing(Arc>), +} + +impl Clone for OverlayCacheEntry { + fn clone(&self) -> Self { match self { - Self::Ready(input) => Some(Arc::clone(input)), - Self::Computing(_) => None, + Self::Ready(input) => Self::Ready(Arc::clone(input)), + Self::Computing(waiter) => Self::Computing(Arc::clone(waiter)), } } } -struct OverlayWaiter { - input: OnceLock>, +struct OverlayWaiter { + input: OnceLock>, } -impl OverlayWaiter { +impl OverlayWaiter { const fn new() -> Self { Self { input: OnceLock::new() } } - fn wait(&self) -> Arc { + fn wait(&self) -> Arc { Arc::clone(self.input.wait()) } - fn finish(&self, computed: Arc) { + fn finish(&self, computed: Arc) { let _ = self.input.set(computed); } } -enum ComputeOverlayInput { - ExtendCached { block: ExecutedBlock, parent_input: Arc }, +enum ComputeOverlayInput { + ExtendCached { block: ExecutedBlock, parent_input: Arc }, MergeBlocks(Vec>), } @@ -558,7 +717,7 @@ enum ComputeOverlayInput { ) )] fn compute_overlay( - input: ComputeOverlayInput, + input: ComputeOverlayInput, anchor_hash: B256, metrics: &StateTrieOverlayMetrics, ) -> TrieInputSorted { @@ -582,13 +741,13 @@ fn compute_overlay( "extending cached parent state trie overlay" ); - let mut overlay = parent_input.as_ref().clone(); + let mut parent_input = parent_input; extend_overlay( - &mut overlay, + Arc::make_mut(&mut parent_input), &trie_data.sorted.hashed_state, &trie_data.sorted.trie_updates, ); - overlay + Arc::try_unwrap(parent_input).expect("Arc::make_mut leaves the child overlay unique") } ComputeOverlayInput::MergeBlocks(blocks) => merge_blocks(blocks), }; @@ -670,14 +829,61 @@ fn extend_overlay( } } +fn compute_execution_overlay_inner( + input: ComputeOverlayInput, + anchor_hash: B256, + metrics: &ExecutionOverlayMetrics, +) -> ExecutionOverlay { + let started_at = Instant::now(); + let block_count = match &input { + ComputeOverlayInput::ExtendCached { .. } => 1, + ComputeOverlayInput::MergeBlocks(blocks) => blocks.len(), + }; + let parent_overlay = matches!(&input, ComputeOverlayInput::ExtendCached { .. }); + tracing::Span::current().record("block_count", block_count); + tracing::Span::current().record("parent_overlay", parent_overlay); + + let overlay = match input { + ComputeOverlayInput::ExtendCached { block, parent_input } => { + let mut parent_input = parent_input; + Arc::make_mut(&mut parent_input).extend(&block); + Arc::try_unwrap(parent_input).expect("Arc::make_mut leaves the child overlay unique") + } + ComputeOverlayInput::MergeBlocks(blocks) => { + let mut overlay = ExecutionOverlay::default(); + for block in blocks.iter().rev() { + overlay.extend(block); + } + overlay + } + }; + + let elapsed = started_at.elapsed(); + metrics.overlay_computation_duration_seconds.record(elapsed.as_secs_f64()); + tracing::Span::current().record("elapsed_us", elapsed.as_micros() as u64); + debug!( + target: "storage::overlay::manager", + %anchor_hash, + block_count, + parent_overlay, + ?elapsed, + "computed execution overlay" + ); + + overlay +} + #[cfg(test)] mod tests { use super::*; - use alloy_primitives::U256; + use alloy_primitives::{map::HashMap, Address, U256}; use reth_chain_state::{test_utils::TestBlockBuilder, ExecutedBlock, SparseTrie}; use reth_ethereum_primitives::EthPrimitives; use reth_primitives_traits::Account; + #[cfg(feature = "rayon")] + use reth_tasks::WorkerPool; use reth_trie::{updates::TrieUpdatesSorted, ComputedTrieData, HashedPostState, HashedStorage}; + use revm::{bytecode::Bytecode, database::BundleState, state::AccountInfo}; use std::{ sync::{mpsc, Arc}, thread, @@ -697,10 +903,23 @@ mod tests { HashedStorage::from_iter([(hashed_slot, U256::from(id))]), )]) .into_sorted(); + let address = Address::with_last_byte(id); + let slot = U256::from(id); + let code_hash = B256::with_last_byte(id.saturating_add(64)); + let state = BundleState::builder(block.block_number()..=block.block_number()) + .state_present_account_info( + address, + AccountInfo { nonce: id as u64, balance: U256::from(id), ..Default::default() }, + ) + .state_storage(address, HashMap::from_iter([(slot, (U256::ZERO, U256::from(id)))])) + .contract(code_hash, Bytecode::new_raw(vec![id].into())) + .build(); + let mut execution_output = (*block.execution_output).clone(); + execution_output.state = state; ExecutedBlock::new( Arc::clone(&block.recovered_block), - Arc::clone(&block.execution_output), + Arc::new(execution_output), ComputedTrieData::new(Arc::new(hashed_state), Arc::new(TrieUpdatesSorted::default())), ) } @@ -748,6 +967,177 @@ mod tests { assert!(Arc::ptr_eq(&short, &cached_short)); } + #[test] + fn builds_execution_overlay_for_inserted_blocks() { + let manager = OverlayManager::default(); + let blocks = test_blocks(); + for block in &blocks { + manager.insert_block(block.clone()); + } + + let anchor_hash = blocks[0].recovered_block().parent_hash(); + let overlay = manager + .execution_overlay_for_parent(blocks[2].recovered_block().hash(), anchor_hash) + .unwrap(); + + for id in 1..=3 { + let address = Address::with_last_byte(id); + let code_hash = B256::with_last_byte(id + 64); + assert_eq!(overlay.accounts[&address].as_ref().unwrap().nonce, id as u64); + assert_eq!(overlay.storage[&address][&U256::from(id)], U256::from(id)); + assert_eq!(overlay.code_hashes[&code_hash], Bytecode::new_raw(vec![id].into())); + } + assert_eq!( + overlay.block_hashes, + blocks[..=2].iter().map(|block| block.recovered_block().num_hash()).collect::>(), + ); + + let cached = manager + .execution_overlay_for_parent(blocks[2].recovered_block().hash(), anchor_hash) + .unwrap(); + assert!(Arc::ptr_eq(&overlay, &cached)); + + let short_anchor = blocks[1].recovered_block().hash(); + let short = manager + .execution_overlay_for_parent(blocks[2].recovered_block().hash(), short_anchor) + .unwrap(); + assert_eq!(short.accounts.len(), 1); + } + + #[test] + fn execution_overlay_for_parent_at_anchor_is_empty() { + let manager = OverlayManager::::default(); + let anchor_hash = B256::with_last_byte(1); + + let overlay = manager.execution_overlay_for_parent(anchor_hash, anchor_hash).unwrap(); + + assert!(overlay.accounts.is_empty()); + assert!(overlay.storage.is_empty()); + assert!(overlay.code_hashes.is_empty()); + assert!(overlay.block_hashes.is_empty()); + } + + #[test] + fn promotes_ready_parent_overlays_to_the_child() { + let manager = OverlayManager::default(); + let blocks = test_blocks(); + for block in &blocks { + manager.insert_block(block.clone()); + } + + let anchor_hash = blocks[0].recovered_block().parent_hash(); + let parent_hash = blocks[1].recovered_block().hash(); + let child_hash = blocks[2].recovered_block().hash(); + let parent_key = OverlayCacheKey { anchor_hash, tip_hash: parent_hash }; + let child_key = OverlayCacheKey { anchor_hash, tip_hash: child_hash }; + + manager.overlay_for_parent(parent_hash, anchor_hash).unwrap(); + manager.execution_overlay_for_parent(parent_hash, anchor_hash).unwrap(); + + manager.overlay_for_parent(child_hash, anchor_hash).unwrap(); + manager.execution_overlay_for_parent(child_hash, anchor_hash).unwrap(); + + assert!(!manager.state_trie_overlays.entries.contains_key(&parent_key)); + assert!(manager.state_trie_overlays.entries.contains_key(&child_key)); + assert!(!manager.execution_overlays.entries.contains_key(&parent_key)); + assert!(manager.execution_overlays.entries.contains_key(&child_key)); + } + + #[test] + fn promotes_parent_overlays_held_by_callers() { + let manager = OverlayManager::default(); + let blocks = test_blocks(); + for block in &blocks { + manager.insert_block(block.clone()); + } + + let anchor_hash = blocks[0].recovered_block().parent_hash(); + let parent_hash = blocks[1].recovered_block().hash(); + let child_hash = blocks[2].recovered_block().hash(); + let parent_key = OverlayCacheKey { anchor_hash, tip_hash: parent_hash }; + + manager.overlay_for_parent(parent_hash, anchor_hash).unwrap(); + let state_parent = manager + .state_trie_overlays + .entries + .get(&parent_key) + .and_then(|entry| match entry.value() { + OverlayCacheEntry::Ready(input) => Some(Arc::clone(input)), + OverlayCacheEntry::Computing(_) => None, + }) + .unwrap(); + let execution_parent = + manager.execution_overlay_for_parent(parent_hash, anchor_hash).unwrap(); + + let (_, child_state) = manager.overlay_for_parent(child_hash, anchor_hash).unwrap(); + let child_execution = + manager.execution_overlay_for_parent(child_hash, anchor_hash).unwrap(); + + assert!(!manager.state_trie_overlays.entries.contains_key(&parent_key)); + assert!(!manager.execution_overlays.entries.contains_key(&parent_key)); + assert_eq!(state_parent.state.accounts.len(), 2); + assert_eq!(execution_parent.accounts.len(), 2); + assert_eq!(child_state.accounts.len(), 3); + assert_eq!(child_execution.accounts.len(), 3); + } + + #[cfg(feature = "rayon")] + #[test] + fn precomputes_execution_overlay_for_cached_parent() { + let manager = OverlayManager::new(Arc::new(WorkerPool::new(1, "execution-overlay-test"))); + let blocks = test_blocks(); + let anchor_hash = blocks[0].recovered_block().parent_hash(); + + manager.insert_block(blocks[0].clone()); + manager + .execution_overlay_for_parent(blocks[0].recovered_block().hash(), anchor_hash) + .unwrap(); + + manager.insert_block(blocks[1].clone()); + let key = OverlayCacheKey { anchor_hash, tip_hash: blocks[1].recovered_block().hash() }; + let deadline = std::time::Instant::now() + Duration::from_secs(1); + while !manager + .execution_overlays + .entries + .get(&key) + .is_some_and(|entry| matches!(entry.value(), OverlayCacheEntry::Ready(_))) + { + assert!(std::time::Instant::now() < deadline, "execution overlay was not precomputed"); + thread::sleep(Duration::from_millis(10)); + } + assert!(!manager.execution_overlays.entries.contains_key(&OverlayCacheKey { + anchor_hash, + tip_hash: blocks[0].recovered_block().hash(), + })); + } + + #[cfg(feature = "rayon")] + #[test] + fn execution_overlay_precompute_does_not_wait_for_pending_entry() { + let worker_pool = Arc::new(WorkerPool::new(1, "execution-overlay-pending-test")); + let manager = OverlayManager::new(Arc::clone(&worker_pool)); + let block = test_blocks().remove(0); + let anchor_hash = block.recovered_block().parent_hash(); + let tip_hash = block.recovered_block().hash(); + manager.insert_block(block); + + let waiter = Arc::new(OverlayWaiter::new()); + manager.execution_overlays.entries.insert( + OverlayCacheKey { anchor_hash, tip_hash }, + OverlayCacheEntry::Computing(Arc::clone(&waiter)), + ); + + let (tx, rx) = mpsc::channel(); + worker_pool.spawn(move || { + manager.precompute_execution_overlay_for_parent(tip_hash, anchor_hash).unwrap(); + tx.send(()).unwrap(); + }); + + let completed = rx.recv_timeout(Duration::from_millis(100)); + waiter.finish(Arc::new(ExecutionOverlay::default())); + assert!(completed.is_ok(), "execution overlay precompute waited for pending entry"); + } + #[test] fn contains_hash_detects_hashes_from_anchor_to_parent() { let manager = OverlayManager::default(); @@ -829,7 +1219,10 @@ mod tests { tip_hash: B256::with_last_byte(2), }; let waiter = Arc::new(OverlayWaiter::new()); - manager.overlays.insert(key, OverlayCacheEntry::Computing(Arc::clone(&waiter))); + manager + .state_trie_overlays + .entries + .insert(key, OverlayCacheEntry::Computing(Arc::clone(&waiter))); let (tx, rx) = mpsc::channel(); thread::spawn(move || { @@ -859,6 +1252,9 @@ mod tests { let original_anchor = blocks[0].recovered_block().parent_hash(); manager.overlay_for_parent(blocks[2].recovered_block().hash(), original_anchor).unwrap(); + manager + .execution_overlay_for_parent(blocks[2].recovered_block().hash(), original_anchor) + .unwrap(); manager.remove_blocks([ blocks[0].recovered_block().hash(), @@ -869,9 +1265,16 @@ mod tests { assert!(manager .overlay_for_parent(blocks[2].recovered_block().hash(), original_anchor) .is_err()); + assert!(manager + .execution_overlay_for_parent(blocks[2].recovered_block().hash(), original_anchor) + .is_err()); let (_, state) = manager.overlay_for_parent(blocks[2].recovered_block().hash(), anchor_hash).unwrap(); assert_eq!(state.accounts.len(), 1); + let execution = manager + .execution_overlay_for_parent(blocks[2].recovered_block().hash(), anchor_hash) + .unwrap(); + assert_eq!(execution.accounts.len(), 1); } } diff --git a/crates/storage/storage-overlay/src/manager_metrics.rs b/crates/storage/storage-overlay/src/manager_metrics.rs new file mode 100644 index 00000000000..ea0cb2cfbf8 --- /dev/null +++ b/crates/storage/storage-overlay/src/manager_metrics.rs @@ -0,0 +1,54 @@ +use reth_metrics::{ + metrics::{Counter, Histogram}, + Metrics, +}; + +/// Metrics for state trie overlay management. +#[derive(Clone, Metrics)] +#[metrics(scope = "sync.block_validation.state_trie_overlay")] +pub(crate) struct StateTrieOverlayMetrics { + /// Duration of overlay computation in seconds. + pub(crate) overlay_computation_duration_seconds: Histogram, + /// Number of requests satisfied by an existing overlay cache entry. + pub(crate) overlay_cache_reuses: Counter, + /// Number of overlay cache entries populated by computing an overlay. + pub(crate) overlay_cache_fills: Counter, +} + +/// Metrics for execution overlay management. +#[derive(Clone, Metrics)] +#[metrics(scope = "sync.block_validation.execution_overlay")] +pub(crate) struct ExecutionOverlayMetrics { + /// Duration of overlay computation in seconds. + pub(crate) overlay_computation_duration_seconds: Histogram, + /// Number of requests satisfied by an existing overlay cache entry. + pub(crate) overlay_cache_reuses: Counter, + /// Number of overlay cache entries populated by computing an overlay. + pub(crate) overlay_cache_fills: Counter, +} + +pub(crate) trait OverlayCacheMetrics { + fn record_cache_reuse(&self); + + fn record_cache_fill(&self); +} + +impl OverlayCacheMetrics for StateTrieOverlayMetrics { + fn record_cache_reuse(&self) { + self.overlay_cache_reuses.increment(1); + } + + fn record_cache_fill(&self) { + self.overlay_cache_fills.increment(1); + } +} + +impl OverlayCacheMetrics for ExecutionOverlayMetrics { + fn record_cache_reuse(&self) { + self.overlay_cache_reuses.increment(1); + } + + fn record_cache_fill(&self) { + self.overlay_cache_fills.increment(1); + } +} diff --git a/crates/storage/storage-overlay/src/provider.rs b/crates/storage/storage-overlay/src/provider.rs index 8f6e57728ce..29736a442c8 100644 --- a/crates/storage/storage-overlay/src/provider.rs +++ b/crates/storage/storage-overlay/src/provider.rs @@ -1,4 +1,4 @@ -use crate::{database_state_frontiers, Overlay, OverlayBuilder}; +use crate::{database_state_frontiers, OverlayBuilder, StateTrieOverlay}; use alloy_primitives::{BlockHash, B256}; use metrics::{Counter, Histogram}; use reth_db_api::{tables, transaction::DbTx, DatabaseError}; @@ -34,7 +34,7 @@ pub(crate) struct OverlayStateProviderFactoryMetrics { create_provider_duration: Histogram, /// Overall duration of the [`OverlayStateProviderFactory::database_provider_ro`] call. database_provider_ro_duration: Histogram, - /// Number of cache misses when fetching [`Overlay`]s from the overlay cache. + /// Number of cache misses when fetching state trie overlays from the overlay cache. overlay_cache_misses: Counter, } @@ -48,11 +48,11 @@ pub struct OverlayStateProviderFactory { factory: F, /// Overlay builder containing the configuration and overlay calculation logic. overlay_builder: OverlayBuilder, - /// A cache which maps `(state_trie_tip, finish_tip) -> Overlay`. + /// A cache mapping `(state_trie_tip, finish_tip)` to [`StateTrieOverlay`]. /// /// Under partial persistence the overlay depends on both durable frontiers, so both hashes are /// part of the cache key. - overlay_cache: Arc>, + overlay_cache: Arc>, /// Metrics for provider factory operations. metrics: OverlayStateProviderFactoryMetrics, } @@ -77,10 +77,14 @@ impl OverlayStateProviderFactory { self } - /// Fetches an [`Overlay`] from the cache based on the current durable frontiers. If there is no - /// cached value then this calculates the [`Overlay`] and populates the cache. + /// Fetches a [`StateTrieOverlay`] from the cache based on the current durable frontiers. If + /// there is no cached value then this calculates the [`StateTrieOverlay`] and populates the + /// cache. #[instrument(level = "debug", target = "providers::state::overlay", skip_all)] - fn get_overlay(&self, provider: &Provider) -> ProviderResult + fn get_state_trie_overlay( + &self, + provider: &Provider, + ) -> ProviderResult where Provider: StageCheckpointReader + PruneCheckpointReader @@ -97,7 +101,7 @@ impl OverlayStateProviderFactory { dashmap::Entry::Occupied(entry) => entry.get().clone(), dashmap::Entry::Vacant(entry) => { self.metrics.overlay_cache_misses.increment(1); - let overlay = self.overlay_builder.build_overlay_at_frontiers( + let overlay = self.overlay_builder.build_state_trie_overlay_at_frontiers( provider, state_trie_tip_block, finish_tip_block, @@ -137,7 +141,7 @@ where res }; - let overlay = self.get_overlay(&provider)?; + let overlay = self.get_state_trie_overlay(&provider)?; let is_v2 = provider.cached_storage_settings().is_v2(); self.metrics.database_provider_ro_duration.record(overall_start.elapsed()); @@ -153,13 +157,13 @@ where #[derive(Debug)] pub struct OverlayStateProvider { provider: Provider, - overlay: Overlay, + overlay: StateTrieOverlay, is_v2: bool, } impl OverlayStateProvider { /// Creates a new overlay state provider. - pub const fn new(provider: Provider, overlay: Overlay, is_v2: bool) -> Self { + pub const fn new(provider: Provider, overlay: StateTrieOverlay, is_v2: bool) -> Self { Self { provider, overlay, is_v2 } } } @@ -328,11 +332,11 @@ mod tests { (factory, blocks) } - fn account_keys(overlay: &Overlay) -> Vec { + fn account_keys(overlay: &StateTrieOverlay) -> Vec { overlay.hashed_post_state.accounts.iter().map(|(key, _)| *key).collect() } - fn account_node_paths(overlay: &Overlay) -> Vec { + fn account_node_paths(overlay: &StateTrieOverlay) -> Vec { overlay.trie_updates.account_nodes_ref().iter().map(|(path, _)| *path).collect() } @@ -349,7 +353,7 @@ mod tests { ); let provider = factory.provider().unwrap(); - let first = overlay_factory.get_overlay(&provider).unwrap(); + let first = overlay_factory.get_state_trie_overlay(&provider).unwrap(); assert_eq!(account_keys(&first), vec![B256::with_last_byte(3), B256::with_last_byte(4)]); drop(provider); @@ -365,7 +369,7 @@ mod tests { provider_rw.commit().unwrap(); let provider = factory.provider().unwrap(); - let second = overlay_factory.get_overlay(&provider).unwrap(); + let second = overlay_factory.get_state_trie_overlay(&provider).unwrap(); assert_eq!(account_keys(&second), vec![B256::with_last_byte(4)]); assert_eq!(account_node_paths(&second), vec![Nibbles::from_nibbles([4])]); assert_eq!(overlay_factory.overlay_cache.len(), 2); From 74ce7f1b2be659f4deb9ec8946e9e606368209f5 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Fri, 28 Aug 2026 10:50:08 +0000 Subject: [PATCH 087/294] fix(txpool): enforce minimum priority fee for local transactions (#26861) --- crates/ethereum/node/tests/e2e/pool.rs | 66 ++++++++++++++++++++- crates/transaction-pool/src/lib.rs | 2 +- crates/transaction-pool/src/validate/eth.rs | 38 +++++------- 3 files changed, 80 insertions(+), 26 deletions(-) diff --git a/crates/ethereum/node/tests/e2e/pool.rs b/crates/ethereum/node/tests/e2e/pool.rs index 6ad25bfeaa5..82cde6a9998 100644 --- a/crates/ethereum/node/tests/e2e/pool.rs +++ b/crates/ethereum/node/tests/e2e/pool.rs @@ -2,13 +2,18 @@ use crate::utils::eth_payload_attributes; use alloy_consensus::{EthereumTxEnvelope, TxEip4844}; use alloy_eips::{eip1559::ETHEREUM_BLOCK_GAS_LIMIT_30M, Encodable2718}; use alloy_genesis::Genesis; -use alloy_primitives::B256; +use alloy_primitives::{Address, TxKind, B256, U256}; +use alloy_provider::{Provider, ProviderBuilder}; +use alloy_rpc_types_eth::TransactionRequest; use reth_chainspec::{ChainSpecBuilder, MAINNET}; use reth_e2e_test_utils::{ - node::NodeTestContext, transaction::TransactionTestContext, wallet::Wallet, + node::NodeTestContext, transaction::TransactionTestContext, wallet::Wallet, E2ETestSetupBuilder, }; use reth_node_builder::{NodeBuilder, NodeHandle}; -use reth_node_core::{args::RpcServerArgs, node_config::NodeConfig}; +use reth_node_core::{ + args::{RpcServerArgs, TxPoolArgs}, + node_config::NodeConfig, +}; use reth_node_ethereum::EthereumNode; use reth_primitives_traits::Recovered; use reth_provider::CanonStateSubscriptions; @@ -20,6 +25,61 @@ use reth_transaction_pool::{ }; use std::{sync::Arc, time::Duration}; +#[tokio::test] +async fn rpc_enforces_minimum_priority_fee() -> eyre::Result<()> { + reth_tracing::init_test_tracing(); + + const MINIMUM_PRIORITY_FEE: u128 = 1; + + let genesis: Genesis = serde_json::from_str(include_str!("../assets/genesis.json"))?; + let chain_spec = Arc::new( + ChainSpecBuilder::default() + .chain(MAINNET.chain) + .genesis(genesis) + .cancun_activated() + .build(), + ); + let (mut nodes, wallet) = + E2ETestSetupBuilder::::new(1, chain_spec, eth_payload_attributes) + .with_node_config_modifier(|config| { + config.with_txpool(TxPoolArgs { + minimum_priority_fee: Some(MINIMUM_PRIORITY_FEE), + ..Default::default() + }) + }) + .build() + .await?; + let node = nodes.pop().unwrap(); + let provider = ProviderBuilder::new().connect_http(node.rpc_url()); + + let transaction = |max_priority_fee_per_gas| TransactionRequest { + nonce: Some(0), + value: Some(U256::from(100)), + to: Some(TxKind::Call(Address::ZERO)), + gas: Some(21_000), + max_fee_per_gas: Some(20_000_000_000), + max_priority_fee_per_gas: Some(max_priority_fee_per_gas), + chain_id: Some(1), + ..Default::default() + }; + + let below_minimum = TransactionTestContext::sign_tx(wallet.inner.clone(), transaction(0)).await; + let err = provider.send_raw_transaction(&below_minimum.encoded_2718()).await.unwrap_err(); + assert!( + err.to_string().contains("transaction priority fee below minimum required priority fee 1"), + "{err}" + ); + assert!(node.inner.pool.is_empty()); + + let at_minimum = + TransactionTestContext::sign_tx(wallet.inner, transaction(MINIMUM_PRIORITY_FEE)).await; + let pending = provider.send_raw_transaction(&at_minimum.encoded_2718()).await?; + assert_eq!(node.inner.pool.len(), 1); + assert!(node.inner.pool.contains(pending.tx_hash())); + + Ok(()) +} + // Test that stale transactions could be correctly evicted. #[tokio::test] async fn maintain_txpool_stale_eviction() -> eyre::Result<()> { diff --git a/crates/transaction-pool/src/lib.rs b/crates/transaction-pool/src/lib.rs index f61f3895c56..f1c99b1571a 100644 --- a/crates/transaction-pool/src/lib.rs +++ b/crates/transaction-pool/src/lib.rs @@ -71,7 +71,7 @@ //! activation) //! - **Size**: Input data ≤ 128KB (default) //! - **Gas**: Limit ≤ block gas limit -//! - **Fees**: Priority fee ≤ max fee; local tx fee cap; external minimum priority fee +//! - **Fees**: Priority fee ≤ max fee; local tx fee cap; minimum priority fee //! - **Chain ID**: Must match current chain //! - **Intrinsic Gas**: Sufficient for data and access lists //! - **Blobs** (EIP-4844): Valid count, KZG proofs diff --git a/crates/transaction-pool/src/validate/eth.rs b/crates/transaction-pool/src/validate/eth.rs index 79ab0afa54f..e8037b1fb3f 100644 --- a/crates/transaction-pool/src/validate/eth.rs +++ b/crates/transaction-pool/src/validate/eth.rs @@ -573,10 +573,9 @@ where } } - // Drop non-local transactions with a fee lower than the configured fee for acceptance into - // the pool. - if !is_local && - transaction.is_dynamic_fee() && + // Drop dynamic fee transactions with a fee lower than the configured fee for acceptance + // into the pool. + if transaction.is_dynamic_fee() && transaction.max_priority_fee_per_gas() < self.minimum_priority_fee { return Err(InvalidPoolTransactionError::PriorityFeeBelowMinimum { @@ -1951,24 +1950,27 @@ mod tests { let validator = create_validator_with_minimum_fee(provider, Some(minimum_priority_fee), None); - // External transaction should be rejected due to low priority fee - let outcome = validator.validate_one(TransactionOrigin::External, transaction.clone()); - assert!(outcome.is_invalid()); - - if let TransactionValidationOutcome::Invalid(_, err) = outcome { + for origin in + [TransactionOrigin::External, TransactionOrigin::Local, TransactionOrigin::Private] + { + let outcome = validator.validate_one(origin, transaction.clone()); assert!(matches!( - err, - InvalidPoolTransactionError::PriorityFeeBelowMinimum { minimum_priority_fee: min_fee } - if min_fee == minimum_priority_fee + outcome, + TransactionValidationOutcome::Invalid( + _, + InvalidPoolTransactionError::PriorityFeeBelowMinimum { + minimum_priority_fee: min_fee + } + ) if min_fee == minimum_priority_fee )); } - // Test pool integration + // Local submission is the path used by `eth_sendRawTransaction`. let blob_store = InMemoryBlobStore::default(); let pool = Pool::new(validator, CoinbaseTipOrdering::default(), blob_store, Default::default()); - let res = pool.add_external_transaction(transaction.clone()).await; + let res = pool.add_transaction(TransactionOrigin::Local, transaction.clone()).await; assert!(res.is_err()); assert!(matches!( res.unwrap_err().kind, @@ -1978,14 +1980,6 @@ mod tests { )); let tx = pool.get(transaction.hash()); assert!(tx.is_none()); - - // Local transactions should still be accepted regardless of minimum priority fee - let (_, local_provider) = setup_priority_fee_test(); - let validator_local = - create_validator_with_minimum_fee(local_provider, Some(minimum_priority_fee), None); - - let local_outcome = validator_local.validate_one(TransactionOrigin::Local, transaction); - assert!(local_outcome.is_valid()); } #[tokio::test] From 7f23df763425668b898084ec3bb49bda15ee98ad Mon Sep 17 00:00:00 2001 From: Brian Picciano Date: Fri, 28 Aug 2026 11:33:33 +0000 Subject: [PATCH 088/294] feat(overlay): build execution overlays (#26838) Co-authored-by: Matthias Seitz --- crates/storage/storage-overlay/src/builder.rs | 544 +++++++++++++++++- crates/storage/storage-overlay/src/manager.rs | 47 +- 2 files changed, 544 insertions(+), 47 deletions(-) diff --git a/crates/storage/storage-overlay/src/builder.rs b/crates/storage/storage-overlay/src/builder.rs index 6d3c921fb3a..9218d6385e4 100644 --- a/crates/storage/storage-overlay/src/builder.rs +++ b/crates/storage/storage-overlay/src/builder.rs @@ -2,7 +2,7 @@ use crate::OverlayManager; use alloy_eips::BlockNumHash; use alloy_primitives::{ map::{AddressMap, B256Map, U256Map}, - BlockHash, B256, U256, + BlockHash, BlockNumber, B256, U256, }; use metrics::{Counter, Histogram}; use reth_chain_state::ExecutedBlock; @@ -18,8 +18,10 @@ use reth_storage_api::{ }; use reth_trie::{updates::TrieUpdatesSorted, HashedPostStateSorted}; use reth_trie_db::DatabaseHashedPostState; -use revm::{bytecode::Bytecode, state::AccountInfo}; +use revm::{bytecode::Bytecode, database::BundleState, state::AccountInfo}; use std::{ + collections::HashSet, + ops::RangeInclusive, sync::Arc, time::{Duration, Instant}, }; @@ -46,7 +48,8 @@ impl StateTrieOverlay { /// Execution state required to initialize an overlay state provider. /// /// Account entries preserve known non-existence, while storage and code entries contain only data -/// explicitly observed during execution. This intentionally does not retain account status or +/// explicitly observed during execution. Accounts never retain database-context-local lookup IDs. +/// This intentionally does not retain account status or /// known-storage wipes, so it cannot represent pre-Dencun `SELFDESTRUCT` of existing accounts. /// The execution overlay is used only for post-Dencun engine execution, where that operation does /// not clear existing storage; pipeline sync does not use it. Extend this representation before @@ -54,26 +57,55 @@ impl StateTrieOverlay { #[derive(Clone, Debug, Default)] pub struct ExecutionOverlay { /// In-memory block hashes in ascending block-number order. - pub block_hashes: Vec, - /// Account state by address. - pub accounts: AddressMap>, + block_hashes: Vec, + /// Account state by address, without database-context-local [`AccountInfo::account_id`] hints. + accounts: AddressMap>, /// Storage values by address and slot. - pub storage: AddressMap>, + storage: AddressMap>, /// Bytecode by code hash. - pub code_hashes: B256Map, + code_hashes: B256Map, } impl ExecutionOverlay { - pub(crate) fn extend(&mut self, block: &ExecutedBlock) { + /// Returns the in-memory block hashes in ascending block-number order. + pub const fn block_hashes(&self) -> &[BlockNumHash] { + self.block_hashes.as_slice() + } + + /// Returns the account state by address. + pub const fn accounts(&self) -> &AddressMap> { + &self.accounts + } + + /// Returns the storage values by address and slot. + pub const fn storage(&self) -> &AddressMap> { + &self.storage + } + + /// Returns the bytecode by code hash. + pub const fn code_hashes(&self) -> &B256Map { + &self.code_hashes + } + + /// Extends this overlay with the execution state of a later block. + pub(crate) fn extend_block(&mut self, block: &ExecutedBlock) { self.block_hashes.push(block.recovered_block().num_hash()); - let state = &block.execution_output.state; + self.extend_state(&block.execution_output.state); + } + + /// Extends this overlay with a later bundle state. + /// + /// [`AccountInfo::account_id`] is a lookup hint owned by the database context that assigned it + /// and cannot be reused by the overlay's database context. All other account fields are + /// preserved. + fn extend_state(&mut self, state: &BundleState) { let (accounts, storage, code_hashes) = (&mut self.accounts, &mut self.storage, &mut self.code_hashes); #[allow(unused_mut)] - let mut extend_state = || { + let mut extend_accounts_and_storage = || { for (address, account) in state.state() { - accounts.insert(*address, account.info.clone()); + accounts.insert(*address, Self::normalized_account_info(account.info.clone())); let account_storage = storage.entry(*address).or_default(); for (slot, value) in &account.storage { account_storage.insert(*slot, value.present_value); @@ -86,14 +118,47 @@ impl ExecutionOverlay { }; #[cfg(feature = "rayon")] - rayon::join(extend_state, extend_code_hashes); + rayon::join(extend_accounts_and_storage, extend_code_hashes); #[cfg(not(feature = "rayon"))] { - extend_state(); + extend_accounts_and_storage(); extend_code_hashes(); } } + + /// Extends this overlay with another, later overlay. + /// + /// Entries in `other` take precedence when both overlays contain the same account, storage + /// slot, or bytecode hash. + fn extend_overlay(&mut self, other: &Self) { + self.block_hashes.extend_from_slice(&other.block_hashes); + self.accounts.extend( + other + .accounts + .iter() + .map(|(address, info)| (*address, Self::normalized_account_info(info.clone()))), + ); + for (address, slots) in &other.storage { + self.storage + .entry(*address) + .or_default() + .extend(slots.iter().map(|(slot, value)| (*slot, *value))); + } + self.code_hashes.extend(other.code_hashes.iter().map(|(hash, code)| (*hash, code.clone()))); + } + + /// Removes the database-local account lookup hint before caching account state. + /// + /// `account_id` indexes the database or BAL context that produced the [`AccountInfo`]. A later + /// execution context can assign that ID to a different account, so it must not cross the + /// execution-overlay boundary. + const fn normalized_account_info(mut info: Option) -> Option { + if let Some(info) = &mut info { + info.account_id = None; + } + info + } } /// Source of data to apply on top of the durable database state. @@ -268,16 +333,10 @@ impl OverlayBuilder { self.anchor_at_parent_with_frontiers(provider, state_trie_tip_block, finish_tip_block)?; // Collect any reverts which are required to bring the DB view back to the anchor hash. - let (trie_updates, hashed_post_state) = match anchor_for_parent { - AnchorForParent::RevertsRequired { anchor, finish, .. } => { - if self.no_reverts { - return Err(ProviderError::other(std::io::Error::other(format!( - "reverts are disabled, but overlay for parent {} requires reverting Finish #{} ({}) to anchor #{} ({})", - self.parent_hash, finish.number, finish.hash, anchor.number, anchor.hash, - )))) - } - - let revert_blocks = anchor.number + 1..=finish.number; + let (trie_updates, hashed_post_state) = match &anchor_for_parent { + AnchorForParent::RevertsRequired { anchor, .. } => { + let revert_blocks = + self.revert_blocks(&anchor_for_parent)?.expect("reverts are required"); debug!( target: "storage::overlay", @@ -402,7 +461,66 @@ impl OverlayBuilder { Ok(StateTrieOverlay { trie_updates, hashed_post_state }) } - /// Resolves the effective state trie overlay (trie updates, hashed state). + /// Builds the effective execution overlay for the given provider. + #[instrument(level = "debug", target = "storage::overlay", skip_all)] + pub fn build_execution_overlay( + &self, + provider: &Provider, + ) -> ProviderResult> + where + Provider: StageCheckpointReader + + PruneCheckpointReader + + ChangeSetReader + + StorageChangeSetReader + + DBProvider + + BlockNumReader, + { + let (state_trie_tip_block, finish_tip_block) = database_state_frontiers(provider)?; + self.build_execution_overlay_at_frontiers(provider, state_trie_tip_block, finish_tip_block) + } + + /// Builds the effective execution overlay using frontiers already read from the provider. + #[instrument( + level = "debug", + target = "storage::overlay", + skip_all, + fields(?state_trie_tip_block, ?finish_tip_block, parent_hash = ?self.parent_hash) + )] + pub fn build_execution_overlay_at_frontiers( + &self, + provider: &Provider, + state_trie_tip_block: BlockNumHash, + finish_tip_block: BlockNumHash, + ) -> ProviderResult> + where + Provider: ChangeSetReader + + StorageChangeSetReader + + DBProvider + + BlockNumReader + + PruneCheckpointReader, + { + let anchor_for_parent = + self.anchor_at_parent_with_frontiers(provider, state_trie_tip_block, finish_tip_block)?; + let anchor = anchor_for_parent.anchor(); + let mut overlay = match self.revert_blocks(&anchor_for_parent)? { + Some(revert_blocks) => { + debug!( + target: "storage::overlay", + ?revert_blocks, + ?anchor, + "Collecting execution reverts for overlay state provider" + ); + execution_reverts(provider, revert_blocks)? + } + None => return self.resolve_execution_overlay(anchor.hash), + }; + + let managed_overlay = self.resolve_execution_overlay(anchor.hash)?; + overlay.extend_overlay(&managed_overlay); + Ok(Arc::new(overlay)) + } + + /// Resolves the effective overlay (trie updates, hashed state). fn resolve_state_trie_overlays( &self, anchor_hash: BlockHash, @@ -436,6 +554,42 @@ impl OverlayBuilder { } } + /// Resolves the execution overlay for the configured in-memory source. + fn resolve_execution_overlay( + &self, + anchor_hash: BlockHash, + ) -> ProviderResult> { + match &self.overlay_source { + Some(OverlaySource::Managed) if anchor_hash != self.parent_hash => self + .overlay_manager + .execution_overlay_for_parent(self.parent_hash, anchor_hash) + .map_err(ProviderError::other), + Some(OverlaySource::Managed) | None => Ok(Arc::new(ExecutionOverlay::default())), + Some(OverlaySource::Immediate { .. }) => Err(ProviderError::other( + std::io::Error::other("immediate state trie overlay has no execution state"), + )), + } + } + + /// Returns the blocks to revert from Finish to the selected anchor, if any. + fn revert_blocks( + &self, + anchor_for_parent: &AnchorForParent, + ) -> ProviderResult>> { + match anchor_for_parent { + AnchorForParent::NoReverts { .. } => Ok(None), + AnchorForParent::RevertsRequired { anchor, finish, .. } => { + if self.no_reverts { + return Err(ProviderError::other(std::io::Error::other(format!( + "reverts are disabled, but overlay for parent {} requires reverting Finish #{} ({}) to anchor #{} ({})", + self.parent_hash, finish.number, finish.hash, anchor.number, anchor.hash, + )))) + } + Ok(Some(anchor.number + 1..=finish.number)) + } + } + } + /// Returns true if managed overlay resolution can be skipped for this builder. fn should_skip_overlay_for_reused_sparse_trie( &self, @@ -550,6 +704,48 @@ pub enum AnchorForParent { }, } +impl AnchorForParent { + /// Returns the durable block anchoring this overlay. + pub const fn anchor(&self) -> BlockNumHash { + match self { + Self::NoReverts { anchor, .. } | Self::RevertsRequired { anchor, .. } => *anchor, + } + } +} + +/// Collects the execution data needed to revert the database to `range`'s first block. +fn execution_reverts( + provider: &Provider, + range: RangeInclusive, +) -> ProviderResult +where + Provider: ChangeSetReader + StorageChangeSetReader, +{ + let mut overlay = ExecutionOverlay::default(); + let mut seen_accounts = HashSet::new(); + for (_, account) in provider.account_changesets_range(range.clone())? { + if seen_accounts.insert(account.address) { + overlay.accounts.insert(account.address, account.info.map(Into::into)); + } + } + + let mut seen_storage = HashSet::new(); + for (block_address, storage) in provider.storage_changesets_range(range)? { + let address = block_address.address(); + if seen_storage.insert((address, storage.key)) { + overlay + .storage + .entry(address) + .or_default() + .insert(U256::from_be_bytes(storage.key.0), storage.value); + } + } + + // Bytecode rows are append-only. Restoring an account's previous code hash therefore makes + // its bytecode directly available from the database without an additional revert overlay. + Ok(overlay) +} + /// Returns the anchor block to use for the target parent and a chain of in-memory blocks. /// /// # Arguments @@ -669,9 +865,14 @@ where #[cfg(test)] mod tests { use super::*; - use alloy_primitives::U256; + use alloy_primitives::{map::HashMap, Address, U256}; use reth_chain_state::{test_utils::TestBlockBuilder, ExecutedBlock}; - use reth_primitives_traits::Account; + use reth_db::{ + models::{AccountBeforeTx, BlockNumberAddress}, + tables, + transaction::DbTxMut, + }; + use reth_primitives_traits::{Account, StorageEntry}; use reth_provider::{ test_utils::{create_test_provider_factory, MockNodeTypesWithDB}, BlockWriter, ProviderFactory, @@ -679,6 +880,11 @@ mod tests { use reth_stages_types::{FinishCheckpoint, StageCheckpoint}; use reth_storage_api::StageCheckpointWriter; use reth_trie::{BranchNodeCompact, ComputedTrieData, HashedPostState, HashedStorage, Nibbles}; + use revm::{ + bytecode::Bytecode, + database::BundleState, + state::{AccountId, AccountInfo}, + }; fn with_unique_trie_data( block: &ExecutedBlock, @@ -700,10 +906,28 @@ mod tests { )], Default::default(), ); + let address = Address::with_last_byte(id); + let slot = U256::from(id); + let code_hash = B256::with_last_byte(id.saturating_add(64)); + let state = BundleState::builder(block.block_number()..=block.block_number()) + .state_present_account_info( + address, + AccountInfo { + nonce: id as u64, + balance: U256::from(id), + account_id: AccountId::new(id as usize), + ..Default::default() + }, + ) + .state_storage(address, HashMap::from_iter([(slot, (U256::ZERO, U256::from(id)))])) + .contract(code_hash, Bytecode::new_raw(vec![id].into())) + .build(); + let mut execution_output = (*block.execution_output).clone(); + execution_output.state = state; ExecutedBlock::new( Arc::clone(&block.recovered_block), - Arc::clone(&block.execution_output), + Arc::new(execution_output), ComputedTrieData::new(Arc::new(hashed_state), Arc::new(trie_updates)), ) } @@ -748,6 +972,86 @@ mod tests { overlay.trie_updates.account_nodes_ref().iter().map(|(path, _)| *path).collect() } + #[test] + fn execution_overlay_extends_bundle_state_without_account_ids() { + let address = Address::with_last_byte(1); + let slot = U256::from(2); + let value = U256::from(3); + let code = Bytecode::new_raw(vec![0x60, 0x00].into()); + let code_hash = code.hash_slow(); + let account = AccountInfo { + nonce: 4, + balance: U256::from(5), + code_hash, + code: Some(code.clone()), + account_id: AccountId::new(6), + }; + let state = BundleState::builder(0..=0) + .state_present_account_info(address, account.clone()) + .state_storage(address, HashMap::from_iter([(slot, (U256::ZERO, value))])) + .contract(code_hash, code.clone()) + .build(); + assert!(state.state()[&address].info.as_ref().unwrap().account_id.is_some()); + + let mut overlay = ExecutionOverlay::default(); + overlay.extend_state(&state); + + let stored_account = overlay.accounts[&address].as_ref().unwrap(); + assert_eq!(stored_account.account_id, None); + assert_eq!( + stored_account, + &AccountInfo { account_id: None, ..account }, + "normalization must preserve durable account fields" + ); + assert_eq!(stored_account.code, Some(code.clone())); + assert_eq!(overlay.storage[&address][&slot], value); + assert_eq!(overlay.code_hashes[&code_hash], code); + } + + #[test] + fn execution_overlay_composition_uses_later_values_and_normalizes_accounts() { + let address = Address::with_last_byte(1); + let retained_address = Address::with_last_byte(2); + let slot = U256::from(3); + let retained_slot = U256::from(4); + let first_code_hash = B256::with_last_byte(5); + let later_code_hash = B256::with_last_byte(6); + let first_block = BlockNumHash::new(1, B256::with_last_byte(7)); + let later_block = BlockNumHash::new(2, B256::with_last_byte(8)); + + let mut overlay = ExecutionOverlay::default(); + overlay.block_hashes.push(first_block); + overlay.accounts.insert( + address, + Some(AccountInfo { nonce: 1, account_id: None, ..Default::default() }), + ); + overlay.accounts.insert(retained_address, Some(AccountInfo::default())); + overlay.storage.entry(address).or_default().insert(slot, U256::from(9)); + overlay.storage.entry(address).or_default().insert(retained_slot, U256::from(10)); + overlay.code_hashes.insert(first_code_hash, Bytecode::new_raw(vec![1].into())); + + let mut later = ExecutionOverlay::default(); + later.block_hashes.push(later_block); + later.accounts.insert( + address, + Some(AccountInfo { nonce: 11, account_id: AccountId::new(12), ..Default::default() }), + ); + later.storage.entry(address).or_default().insert(slot, U256::from(13)); + later.code_hashes.insert(later_code_hash, Bytecode::new_raw(vec![2].into())); + + overlay.extend_overlay(&later); + + assert!(later.accounts[&address].as_ref().unwrap().account_id.is_some()); + assert_eq!(overlay.block_hashes, vec![first_block, later_block]); + assert_eq!(overlay.accounts[&address].as_ref().unwrap().nonce, 11); + assert_eq!(overlay.accounts[&address].as_ref().unwrap().account_id, None); + assert!(overlay.accounts.contains_key(&retained_address)); + assert_eq!(overlay.storage[&address][&slot], U256::from(13)); + assert_eq!(overlay.storage[&address][&retained_slot], U256::from(10)); + assert!(overlay.code_hashes.contains_key(&first_code_hash)); + assert!(overlay.code_hashes.contains_key(&later_code_hash)); + } + #[test] fn managed_overlay_starts_at_state_trie_frontier() { let (factory, blocks) = setup_frontiers(1, 3); @@ -800,13 +1104,191 @@ mod tests { let (factory, blocks) = setup_frontiers(2, 3); let provider = factory.provider().unwrap(); + let builder = OverlayManager::::default() + .overlay_builder(blocks[1].recovered_block().hash()) + .with_no_reverts(); + let error = builder.build_state_trie_overlay(&provider).unwrap_err(); + + assert!(error.to_string().contains("reverts are disabled")); + let error = builder.build_execution_overlay(&provider).unwrap_err(); + assert!(error.to_string().contains("reverts are disabled")); + } + + #[test] + fn execution_overlay_reverts_to_anchor_state() { + let (factory, blocks) = setup_frontiers(1, 3); + let provider_rw = factory.provider_rw().unwrap(); + let address = Address::with_last_byte(1); + let slot = U256::from(5); + + provider_rw + .tx_ref() + .put::( + 2, + AccountBeforeTx { + address, + info: Some(Account { balance: U256::from(10), ..Default::default() }), + }, + ) + .unwrap(); + provider_rw + .tx_ref() + .put::( + 3, + AccountBeforeTx { + address, + info: Some(Account { balance: U256::from(20), ..Default::default() }), + }, + ) + .unwrap(); + for (block_number, value) in [(2, 10), (3, 15)] { + provider_rw + .tx_ref() + .put::( + BlockNumberAddress((block_number, address)), + StorageEntry { key: B256::from(slot), value: U256::from(value) }, + ) + .unwrap(); + } + provider_rw.commit().unwrap(); + + let provider = factory.provider().unwrap(); + let overlay = OverlayManager::::default() + .overlay_builder(blocks[1].recovered_block().hash()) + .build_execution_overlay(&provider) + .unwrap(); + + assert_eq!(overlay.accounts[&address].as_ref().unwrap().balance, U256::from(10)); + assert_eq!(overlay.storage[&address][&slot], U256::from(10)); + assert!(overlay.code_hashes.is_empty()); + } + + #[test] + fn execution_overlay_rejects_immediate_state_trie_overlay() { + let (factory, blocks) = setup_frontiers(1, 1); + let provider = factory.provider().unwrap(); let error = OverlayManager::::default() .overlay_builder(blocks[1].recovered_block().hash()) - .with_no_reverts() - .build_state_trie_overlay(&provider) + .with_immediate_state_trie_overlay(Default::default(), Default::default()) + .build_execution_overlay(&provider) .unwrap_err(); - assert!(error.to_string().contains("reverts are disabled")); + assert!(error.to_string().contains("immediate state trie overlay has no execution state")); + } + + #[test] + fn execution_overlay_uses_managed_blocks_after_the_anchor() { + let (factory, blocks) = setup_frontiers(1, 3); + let manager = OverlayManager::default(); + for block in &blocks[2..=4] { + manager.insert_block(block.clone()); + } + let provider = factory.provider().unwrap(); + + let overlay = manager + .overlay_builder(blocks[3].recovered_block().hash()) + .build_execution_overlay(&provider) + .unwrap(); + + for id in [3, 4] { + let address = Address::with_last_byte(id); + let slot = U256::from(id); + assert_eq!(overlay.accounts[&address].as_ref().unwrap().balance, U256::from(id)); + assert_eq!(overlay.accounts[&address].as_ref().unwrap().account_id, None); + assert_eq!(overlay.storage[&address][&slot], U256::from(id)); + assert!(overlay.code_hashes.contains_key(&B256::with_last_byte(id + 64))); + } + assert_eq!( + overlay.block_hashes, + blocks[2..=3] + .iter() + .map(|block| block.recovered_block().num_hash()) + .collect::>() + ); + } + + #[test] + fn execution_overlay_merges_reverts_with_a_managed_fork() { + let (factory, blocks) = setup_frontiers(1, 3); + let address = Address::with_last_byte(1); + let slot = U256::from(1); + let provider_rw = factory.provider_rw().unwrap(); + for (block_number, balance, storage_value) in [(2u64, 10u64, 10u64), (3u64, 20u64, 15u64)] { + provider_rw + .tx_ref() + .put::( + block_number, + AccountBeforeTx { + address, + info: Some(Account { balance: U256::from(balance), ..Default::default() }), + }, + ) + .unwrap(); + provider_rw + .tx_ref() + .put::( + BlockNumberAddress((block_number, address)), + StorageEntry { key: B256::from(slot), value: U256::from(storage_value) }, + ) + .unwrap(); + } + provider_rw.commit().unwrap(); + + let mut side_chain_builder = TestBlockBuilder::eth(); + let side_block_two = side_chain_builder.get_executed_block_with_number( + blocks[2].block_number(), + blocks[1].recovered_block().hash(), + ); + let side_block_two = with_unique_trie_data(&side_block_two, 1); + let side_block_three = side_chain_builder.get_executed_block_with_number( + blocks[3].block_number(), + side_block_two.recovered_block().hash(), + ); + let side_block_three = with_unique_trie_data(&side_block_three, 1); + assert_ne!( + side_block_three.recovered_block().hash(), + blocks[3].recovered_block().hash(), + "the managed chain must not contain the durable Finish block" + ); + + let manager = OverlayManager::default(); + manager.insert_block(side_block_two.clone()); + manager.insert_block(side_block_three.clone()); + let provider = factory.provider().unwrap(); + + let overlay = manager + .overlay_builder(side_block_three.recovered_block().hash()) + .build_execution_overlay(&provider) + .unwrap(); + + assert_eq!(overlay.accounts[&address].as_ref().unwrap().balance, U256::from(1)); + assert_eq!(overlay.accounts[&address].as_ref().unwrap().account_id, None); + assert_eq!(overlay.storage[&address][&slot], U256::from(1)); + assert_eq!( + overlay.block_hashes, + [side_block_two, side_block_three] + .iter() + .map(|block| block.recovered_block().num_hash()) + .collect::>() + ); + } + + #[test] + fn execution_overlay_no_revert_path_discards_account_ids() { + let (factory, blocks) = setup_frontiers(1, 1); + let manager = OverlayManager::default(); + for block in &blocks[2..=3] { + manager.insert_block(block.clone()); + } + let provider = factory.provider().unwrap(); + + let overlay = manager + .overlay_builder(blocks[3].recovered_block().hash()) + .build_execution_overlay(&provider) + .unwrap(); + + assert_eq!(overlay.accounts.len(), 2); + assert!(overlay.accounts.values().flatten().all(|account| account.account_id.is_none())); } #[test] diff --git a/crates/storage/storage-overlay/src/manager.rs b/crates/storage/storage-overlay/src/manager.rs index a56a59143da..5f3908473cb 100644 --- a/crates/storage/storage-overlay/src/manager.rs +++ b/crates/storage/storage-overlay/src/manager.rs @@ -846,13 +846,13 @@ fn compute_execution_overlay_inner( let overlay = match input { ComputeOverlayInput::ExtendCached { block, parent_input } => { let mut parent_input = parent_input; - Arc::make_mut(&mut parent_input).extend(&block); + Arc::make_mut(&mut parent_input).extend_block(&block); Arc::try_unwrap(parent_input).expect("Arc::make_mut leaves the child overlay unique") } ComputeOverlayInput::MergeBlocks(blocks) => { let mut overlay = ExecutionOverlay::default(); for block in blocks.iter().rev() { - overlay.extend(block); + overlay.extend_block(block); } overlay } @@ -883,7 +883,11 @@ mod tests { #[cfg(feature = "rayon")] use reth_tasks::WorkerPool; use reth_trie::{updates::TrieUpdatesSorted, ComputedTrieData, HashedPostState, HashedStorage}; - use revm::{bytecode::Bytecode, database::BundleState, state::AccountInfo}; + use revm::{ + bytecode::Bytecode, + database::BundleState, + state::{AccountId, AccountInfo}, + }; use std::{ sync::{mpsc, Arc}, thread, @@ -909,7 +913,12 @@ mod tests { let state = BundleState::builder(block.block_number()..=block.block_number()) .state_present_account_info( address, - AccountInfo { nonce: id as u64, balance: U256::from(id), ..Default::default() }, + AccountInfo { + nonce: id as u64, + balance: U256::from(id), + account_id: AccountId::new(id as usize), + ..Default::default() + }, ) .state_storage(address, HashMap::from_iter([(slot, (U256::ZERO, U256::from(id)))])) .contract(code_hash, Bytecode::new_raw(vec![id].into())) @@ -983,12 +992,13 @@ mod tests { for id in 1..=3 { let address = Address::with_last_byte(id); let code_hash = B256::with_last_byte(id + 64); - assert_eq!(overlay.accounts[&address].as_ref().unwrap().nonce, id as u64); - assert_eq!(overlay.storage[&address][&U256::from(id)], U256::from(id)); - assert_eq!(overlay.code_hashes[&code_hash], Bytecode::new_raw(vec![id].into())); + assert_eq!(overlay.accounts()[&address].as_ref().unwrap().nonce, id as u64); + assert_eq!(overlay.accounts()[&address].as_ref().unwrap().account_id, None); + assert_eq!(overlay.storage()[&address][&U256::from(id)], U256::from(id)); + assert_eq!(overlay.code_hashes()[&code_hash], Bytecode::new_raw(vec![id].into())); } assert_eq!( - overlay.block_hashes, + overlay.block_hashes(), blocks[..=2].iter().map(|block| block.recovered_block().num_hash()).collect::>(), ); @@ -1001,7 +1011,7 @@ mod tests { let short = manager .execution_overlay_for_parent(blocks[2].recovered_block().hash(), short_anchor) .unwrap(); - assert_eq!(short.accounts.len(), 1); + assert_eq!(short.accounts().len(), 1); } #[test] @@ -1011,10 +1021,10 @@ mod tests { let overlay = manager.execution_overlay_for_parent(anchor_hash, anchor_hash).unwrap(); - assert!(overlay.accounts.is_empty()); - assert!(overlay.storage.is_empty()); - assert!(overlay.code_hashes.is_empty()); - assert!(overlay.block_hashes.is_empty()); + assert!(overlay.accounts().is_empty()); + assert!(overlay.storage().is_empty()); + assert!(overlay.code_hashes().is_empty()); + assert!(overlay.block_hashes().is_empty()); } #[test] @@ -1076,9 +1086,14 @@ mod tests { assert!(!manager.state_trie_overlays.entries.contains_key(&parent_key)); assert!(!manager.execution_overlays.entries.contains_key(&parent_key)); assert_eq!(state_parent.state.accounts.len(), 2); - assert_eq!(execution_parent.accounts.len(), 2); + assert_eq!(execution_parent.accounts().len(), 2); assert_eq!(child_state.accounts.len(), 3); - assert_eq!(child_execution.accounts.len(), 3); + assert_eq!(child_execution.accounts().len(), 3); + assert!(child_execution + .accounts() + .values() + .flatten() + .all(|account| account.account_id.is_none())); } #[cfg(feature = "rayon")] @@ -1275,6 +1290,6 @@ mod tests { let execution = manager .execution_overlay_for_parent(blocks[2].recovered_block().hash(), anchor_hash) .unwrap(); - assert_eq!(execution.accounts.len(), 1); + assert_eq!(execution.accounts().len(), 1); } } From 7bd50eaa4dc0966b8f7a40e77efdea9ac9a9f621 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A9a=20Narzis?= <78718413+lean-apple@users.noreply.github.com> Date: Fri, 28 Aug 2026 13:32:33 +0000 Subject: [PATCH 089/294] feat(downloader): select accounts with storage for snap storage requests (#26852) --- crates/net/downloaders/src/snap/mod.rs | 94 +++++++++++++++++----- crates/net/downloaders/src/snap/storage.rs | 45 ++++++----- 2 files changed, 99 insertions(+), 40 deletions(-) diff --git a/crates/net/downloaders/src/snap/mod.rs b/crates/net/downloaders/src/snap/mod.rs index dc76caff111..80d4d3c4b38 100644 --- a/crates/net/downloaders/src/snap/mod.rs +++ b/crates/net/downloaders/src/snap/mod.rs @@ -119,30 +119,40 @@ impl VerifiedAccountRange { /// Borrows the accounts together with the root that authenticated them. pub fn batch(&self) -> VerifiedAccountBatch<'_> { - VerifiedAccountBatch { state_root: self.state_root, accounts: &self.accounts } + VerifiedAccountBatch { + state_root: self.state_root, + accounts: self.accounts.iter().map(|(hash, account)| (*hash, account)).collect(), + } } - /// Borrows a positional subrange of the accounts, so a caller can request storage in bounded - /// chunks without losing the root that authenticated them. + /// Borrows only the accounts that have storage, together with the root that authenticated + /// them. /// - /// `None` when the range falls outside the response. - pub fn batch_range(&self, range: Range) -> Option> { - self.accounts - .get(range) - .map(|accounts| VerifiedAccountBatch { state_root: self.state_root, accounts }) + /// Accounts without storage are omitted because snap storage responses do not preserve an + /// outer-list position for them. Empty when no account in the range has storage. + pub fn storage_batch(&self) -> VerifiedAccountBatch<'_> { + VerifiedAccountBatch { + state_root: self.state_root, + accounts: self + .accounts + .iter() + .filter(|(_, account)| account.storage_root != EMPTY_ROOT_HASH) + .map(|(hash, account)| (*hash, account)) + .collect(), + } } } /// Accounts and the state root they were authenticated against. /// -/// Only obtainable from [`VerifiedAccountRange::batch`], so requests built from it can always be -/// checked against the root the accounts came from. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] +/// Only obtainable from [`VerifiedAccountRange`], so requests built from it can always be checked +/// against the root the accounts came from. +#[derive(Clone, Debug, PartialEq, Eq)] pub struct VerifiedAccountBatch<'a> { // Root and accounts travel together, so neither can be swapped for another generation's. state_root: B256, // Accounts the root authenticated, in response order. - accounts: &'a [(B256, TrieAccount)], + accounts: Vec<(B256, &'a TrieAccount)>, } impl<'a> VerifiedAccountBatch<'a> { @@ -152,8 +162,18 @@ impl<'a> VerifiedAccountBatch<'a> { } /// Accounts in the order the range returned them. - pub const fn accounts(&self) -> &'a [(B256, TrieAccount)] { + pub fn accounts(&self) -> &[(B256, &'a TrieAccount)] { + &self.accounts + } + + /// Borrows a positional subrange of the batch, so storage can be requested in bounded chunks + /// without losing the root that authenticated the accounts. + /// + /// `None` when the range falls outside the batch. + pub fn range(&self, range: Range) -> Option { self.accounts + .get(range) + .map(|accounts| Self { state_root: self.state_root, accounts: accounts.to_vec() }) } // Confirms the batch is the one `request` was built from, so every returned range is checked @@ -175,7 +195,7 @@ impl<'a> VerifiedAccountBatch<'a> { }) } for (index, (requested, (supplied, _))) in - request.account_hashes.iter().zip(self.accounts).enumerate() + request.account_hashes.iter().zip(&self.accounts).enumerate() { if requested != supplied { return Err(InvalidStorageRangeRequest::AccountMismatch { @@ -189,8 +209,11 @@ impl<'a> VerifiedAccountBatch<'a> { } // The accounts from `from` onwards under the same state root, or none if the batch is shorter. - pub(super) fn slice(&self, from: usize) -> Option { - self.accounts.get(from..).map(|accounts| Self { state_root: self.state_root, accounts }) + pub(super) fn slice(mut self, from: usize) -> Option { + (from <= self.accounts.len()).then(|| { + self.accounts.drain(..from); + self + }) } } @@ -430,7 +453,7 @@ mod tests { } #[test] - fn batch_range_narrows_the_accounts_and_keeps_their_root() { + fn a_subrange_narrows_the_accounts_and_keeps_their_root() { let accounts = vec![(key(1), account(7)), (key(2), account(8)), (key(3), account(9))]; let root_hash = root(&accounts); let range = VerifiedAccountRange { @@ -440,11 +463,42 @@ mod tests { next: None, }; - let chunk = range.batch_range(1..3).expect("chunk is inside the range"); - assert_eq!(chunk.accounts(), &accounts[1..3]); + let batch = range.batch(); + let chunk = batch.range(1..3).expect("chunk is inside the batch"); + let expected = + accounts[1..3].iter().map(|(hash, account)| (*hash, account)).collect::>(); + assert_eq!(chunk.accounts(), expected); assert_eq!(chunk.state_root(), root_hash); - assert_eq!(range.batch_range(2..4), None); + assert_eq!(batch.range(2..4), None); + } + + #[test] + fn storage_batch_omits_interleaved_accounts_without_storage() { + let mut first = account(1); + first.storage_root = B256::repeat_byte(0x11); + let empty = account(2); + let mut third = account(3); + third.storage_root = B256::repeat_byte(0x33); + let accounts = vec![(key(1), first), (key(2), empty), (key(3), third)]; + let root_hash = root(&accounts); + let range = + VerifiedAccountRange { state_root: root_hash, accounts, has_more: false, next: None }; + + let batch = range.storage_batch(); + + assert_eq!(batch.state_root(), root_hash); + assert_eq!( + batch.accounts().iter().map(|(hash, _)| *hash).collect::>(), + vec![key(1), key(3)] + ); + + let chunk = batch.range(1..2).expect("chunk is inside the batch"); + assert_eq!( + chunk.accounts().iter().map(|(hash, _)| *hash).collect::>(), + vec![key(3)] + ); + assert_eq!(chunk.state_root(), root_hash); } #[test] diff --git a/crates/net/downloaders/src/snap/storage.rs b/crates/net/downloaders/src/snap/storage.rs index 9c17865f56f..fcf117f4e40 100644 --- a/crates/net/downloaders/src/snap/storage.rs +++ b/crates/net/downloaders/src/snap/storage.rs @@ -39,13 +39,13 @@ impl StorageRangeDownloader { /// root; their storage roots authenticate the returned ranges. Pairing a batch with a request /// for another root would penalize a peer that answered honestly. /// - /// Accounts with empty storage roots must be omitted because Snap responses do not preserve - /// an outer-list position for them. Use [`super::VerifiedAccountRange::batch_range`] to select - /// the accounts included in the request without losing their state-root provenance. + /// Accounts without storage must be omitted, since snap responses do not preserve an + /// outer-list position for them. [`super::VerifiedAccountRange::storage_batch`] selects the + /// accounts to request, and [`VerifiedAccountBatch::range`] splits them into bounded chunks. pub fn new( client: C, request: GetStorageRangesMessage, - batch: VerifiedAccountBatch<'_>, + batch: &VerifiedAccountBatch<'_>, runtime: Runtime, ) -> Result { let origin = request.starting_hash.unwrap_or(B256::ZERO); @@ -506,6 +506,10 @@ mod tests { slots.iter().map(|(hash, value)| StorageData::from_value(*hash, *value)).collect() } + fn account_refs(accounts: &[(B256, TrieAccount)]) -> Vec<(B256, &TrieAccount)> { + accounts.iter().map(|(hash, account)| (*hash, account)).collect() + } + fn request(accounts: &[(B256, TrieAccount)]) -> GetStorageRangesMessage { GetStorageRangesMessage { request_id: 1, @@ -542,7 +546,7 @@ mod tests { accounts: &[(B256, TrieAccount)], ) -> Result, InvalidStorageRangeRequest> { let range = verified_range(accounts); - StorageRangeDownloader::new(client, request, range.batch(), Runtime::test()) + StorageRangeDownloader::new(client, request, &range.batch(), Runtime::test()) } #[tokio::test] @@ -626,27 +630,28 @@ mod tests { } #[test] - fn a_mixed_batch_with_an_empty_storage_root_is_refused_before_submission() { + fn empty_storage_roots_must_be_filtered_before_submission() { let accounts = vec![ - (key(100), account(EMPTY_ROOT_HASH)), - (key(200), account(B256::repeat_byte(0xbb))), + (key(100), account(B256::repeat_byte(0xaa))), + (key(200), account(EMPTY_ROOT_HASH)), + (key(300), account(B256::repeat_byte(0xbb))), ]; let client = TestSnapClient::new([response(PeerId::random(), 1, Vec::new(), Vec::new())]); assert_eq!( downloader(&client, request(&accounts), &accounts).unwrap_err(), - InvalidStorageRangeRequest::EmptyStorageRoot { index: 0, account_hash: key(100) } + InvalidStorageRangeRequest::EmptyStorageRoot { index: 1, account_hash: key(200) } ); assert!(client.priorities().is_empty()); let range = verified_range(&accounts); - assert!(StorageRangeDownloader::new( - &client, - request(&accounts[1..]), - range.batch_range(1..2).unwrap(), - Runtime::test(), - ) - .is_ok()); + let batch = range.storage_batch(); + let mut storage_request = request(&accounts); + storage_request.account_hashes = batch.accounts().iter().map(|(hash, _)| *hash).collect(); + assert_eq!(storage_request.account_hashes, vec![key(100), key(300)]); + assert!( + StorageRangeDownloader::new(&client, storage_request, &batch, Runtime::test()).is_ok() + ); assert_eq!(*client.priorities(), [Priority::Normal]); } @@ -820,7 +825,7 @@ mod tests { let StorageRangeOutcome::Verified(verified) = outcome else { panic!("verified ranges") }; let range = verified_range(&accounts); let (follow_up, narrowed) = verified.follow_up(2, range.batch()).unwrap().unwrap(); - assert_eq!(narrowed.accounts(), &accounts[..]); + assert_eq!(narrowed.accounts(), account_refs(&accounts)); assert_eq!(follow_up.request_id, 2); assert_eq!(follow_up.root_hash, bounded.root_hash); assert_eq!(follow_up.account_hashes, bounded.account_hashes); @@ -859,7 +864,7 @@ mod tests { let range = verified_range(&accounts); let (follow_up, narrowed) = verified.follow_up(2, range.batch()).unwrap().unwrap(); - assert_eq!(narrowed.accounts(), &accounts[1..]); + assert_eq!(narrowed.accounts(), account_refs(&accounts[1..])); assert_eq!(follow_up.account_hashes, vec![key(200)]); assert_eq!(follow_up.starting_hash, key(2).into()); assert_eq!(follow_up.limit_hash, RangeBound::default()); @@ -952,7 +957,7 @@ mod tests { let outcome = StorageRangeDownloader::new( Arc::clone(&client), request.clone(), - batch, + &batch, Runtime::test(), ) .unwrap() @@ -965,7 +970,7 @@ mod tests { let (follow_up, narrowed) = verified.follow_up(request.request_id + 1, batch).unwrap().unwrap(); assert_eq!(follow_up.account_hashes.len(), remaining); - assert_eq!(narrowed.accounts(), &accounts[accounts.len() - remaining..]); + assert_eq!(narrowed.accounts(), account_refs(&accounts[accounts.len() - remaining..])); request = follow_up; batch = narrowed; } From 8215b28f42eee01233e116f4c1cc5b685bbf6062 Mon Sep 17 00:00:00 2001 From: Brian Picciano Date: Fri, 28 Aug 2026 13:42:10 +0000 Subject: [PATCH 090/294] feat(overlay): add overlay state providers (#26862) --- .../src/providers/blockchain_provider.rs | 6 +- .../src/providers/state/historical.rs | 2 +- crates/storage/storage-overlay/src/builder.rs | 37 +- .../storage-overlay/src/changeset_cache.rs | 28 +- crates/storage/storage-overlay/src/manager.rs | 3 +- .../storage/storage-overlay/src/provider.rs | 942 ++++++++++++++++-- 6 files changed, 895 insertions(+), 123 deletions(-) diff --git a/crates/storage/provider/src/providers/blockchain_provider.rs b/crates/storage/provider/src/providers/blockchain_provider.rs index 3b861024584..3dbdd0cfbf0 100644 --- a/crates/storage/provider/src/providers/blockchain_provider.rs +++ b/crates/storage/provider/src/providers/blockchain_provider.rs @@ -23,7 +23,7 @@ use reth_chain_state::{ use reth_chainspec::ChainInfo; use reth_db_api::models::{AccountBeforeTx, BlockNumberAddress, StoredBlockBodyIndices}; use reth_execution_types::ExecutionOutcome; -use reth_node_types::{BlockTy, HeaderTy, NodeTypesWithDB, ReceiptTy, TxTy}; +use reth_node_types::{BlockTy, HeaderTy, NodeTypes, NodeTypesWithDB, ReceiptTy, TxTy}; use reth_primitives_traits::{ Account, RecoveredBlock, SealedHeader, SealedOrRecoveredBlock, StorageEntry, }; @@ -38,6 +38,7 @@ use reth_storage_api::{ use reth_storage_errors::provider::ProviderResult; use reth_storage_overlay::{ anchor_for_parent, AnchorForParent, OverlayStateProvider, OverlayStateProviderFactory, + OwnedProvider, }; use reth_trie::{ hashed_cursor::{HashedCursor, HashedCursorFactory}, @@ -57,7 +58,8 @@ use tracing::trace; pub const SNAPSHOT_STATE_RETENTION: u64 = 128; type StateRangeDbProvider = as DatabaseProviderFactory>::Provider; -type HistoricalStateRangeProvider = OverlayStateProvider>; +type HistoricalStateRangeProvider = + OverlayStateProvider>, ::Primitives>; /// The main type for interacting with the blockchain. /// diff --git a/crates/storage/provider/src/providers/state/historical.rs b/crates/storage/provider/src/providers/state/historical.rs index 864acfad2ad..a45b77ba3c3 100644 --- a/crates/storage/provider/src/providers/state/historical.rs +++ b/crates/storage/provider/src/providers/state/historical.rs @@ -302,7 +302,7 @@ where .overlay_manager .overlay_builder(anchor_hash) .with_immediate_state_trie_overlay(state, nodes); - let StateTrieOverlay { trie_updates, hashed_post_state } = + let StateTrieOverlay { trie_updates, hashed_post_state, .. } = overlay_builder.build_state_trie_overlay(self.provider)?; Ok(TrieInputSorted::new(trie_updates, hashed_post_state, prefix_sets)) diff --git a/crates/storage/storage-overlay/src/builder.rs b/crates/storage/storage-overlay/src/builder.rs index 9218d6385e4..eeb93a3be62 100644 --- a/crates/storage/storage-overlay/src/builder.rs +++ b/crates/storage/storage-overlay/src/builder.rs @@ -34,15 +34,29 @@ pub struct StateTrieOverlay { pub trie_updates: Arc, /// Hashed state overlay. pub hashed_post_state: Arc, + /// Whether construction was skipped because a reused sparse trie covers this range. + skipped_for_reused_sparse_trie: bool, } impl StateTrieOverlay { + pub(crate) const fn new( + trie_updates: Arc, + hashed_post_state: Arc, + ) -> Self { + Self { trie_updates, hashed_post_state, skipped_for_reused_sparse_trie: false } + } + fn empty() -> Self { Self { trie_updates: Arc::new(TrieUpdatesSorted::default()), hashed_post_state: Arc::new(HashedPostStateSorted::default()), + skipped_for_reused_sparse_trie: true, } } + + pub(crate) const fn skipped_for_reused_sparse_trie(&self) -> bool { + self.skipped_for_reused_sparse_trie + } } /// Execution state required to initialize an overlay state provider. @@ -87,6 +101,26 @@ impl ExecutionOverlay { &self.code_hashes } + #[cfg(test)] + pub(crate) const fn block_hashes_mut(&mut self) -> &mut Vec { + &mut self.block_hashes + } + + #[cfg(test)] + pub(crate) const fn accounts_mut(&mut self) -> &mut AddressMap> { + &mut self.accounts + } + + #[cfg(test)] + pub(crate) const fn storage_mut(&mut self) -> &mut AddressMap> { + &mut self.storage + } + + #[cfg(test)] + pub(crate) const fn code_hashes_mut(&mut self) -> &mut B256Map { + &mut self.code_hashes + } + /// Extends this overlay with the execution state of a later block. pub(crate) fn extend_block(&mut self, block: &ExecutedBlock) { self.block_hashes.push(block.recovered_block().num_hash()); @@ -321,6 +355,7 @@ impl OverlayBuilder { + StorageChangeSetReader + DBProvider + BlockNumReader + + StageCheckpointReader + PruneCheckpointReader + StorageSettingsCache, { @@ -458,7 +493,7 @@ impl OverlayBuilder { self.metrics.trie_updates_size.record(trie_updates_total_len as f64); self.metrics.hashed_state_size.record(hashed_state_updates_total_len as f64); - Ok(StateTrieOverlay { trie_updates, hashed_post_state }) + Ok(StateTrieOverlay::new(trie_updates, hashed_post_state)) } /// Builds the effective execution overlay for the given provider. diff --git a/crates/storage/storage-overlay/src/changeset_cache.rs b/crates/storage/storage-overlay/src/changeset_cache.rs index 240e6236fdb..e91d6717e68 100644 --- a/crates/storage/storage-overlay/src/changeset_cache.rs +++ b/crates/storage/storage-overlay/src/changeset_cache.rs @@ -224,6 +224,7 @@ impl ChangesetCache { P: DBProvider + ChangeSetReader + StorageChangeSetReader + + StageCheckpointReader + PruneCheckpointReader + BlockNumReader + StorageSettingsCache, @@ -276,6 +277,7 @@ impl ChangesetCache { P: DBProvider + ChangeSetReader + StorageChangeSetReader + + StageCheckpointReader + PruneCheckpointReader + BlockNumReader + StorageSettingsCache, @@ -402,7 +404,7 @@ impl ChangesetCache { .overlay_builder(finish.hash) .with_no_reverts() .build_state_trie_overlay_at_frontiers(provider, partial_state_trie, finish)?; - let state_trie_provider = OverlayStateProvider::new( + let state_trie_provider = OverlayStateProvider::<&P, N>::new_with_state_trie( provider, overlay, provider.cached_storage_settings().is_v2(), @@ -642,7 +644,7 @@ mod tests { } fn empty_overlay() -> StateTrieOverlay { - StateTrieOverlay { trie_updates: Arc::default(), hashed_post_state: Arc::default() } + StateTrieOverlay::new(Arc::default(), Arc::default()) } fn insert_test_changesets( @@ -904,11 +906,12 @@ mod tests { reth_trie_db::with_adapter!(provider, |A| seed_tip_trie_tables::<_, A>(&*provider)); let overlay = empty_overlay(); - let state_trie_provider = OverlayStateProvider::new( - &*provider, - overlay, - provider.cached_storage_settings().is_v2(), - ); + let state_trie_provider = + OverlayStateProvider::<&_, reth_ethereum_primitives::EthPrimitives>::new_with_state_trie( + &*provider, + overlay, + provider.cached_storage_settings().is_v2(), + ); let actual = reth_trie_db::compute_range_trie_changesets(&*provider, &state_trie_provider, 1..=3, 3) .unwrap(); @@ -990,11 +993,12 @@ mod tests { let expected = legacy_compute_range_trie_changesets(&*provider, 2..=3); let overlay = empty_overlay(); - let state_trie_provider = OverlayStateProvider::new( - &*provider, - overlay, - provider.cached_storage_settings().is_v2(), - ); + let state_trie_provider = + OverlayStateProvider::<&_, reth_ethereum_primitives::EthPrimitives>::new_with_state_trie( + &*provider, + overlay, + provider.cached_storage_settings().is_v2(), + ); let actual = reth_trie_db::compute_range_trie_changesets(&*provider, &state_trie_provider, 2..=3, 3) .unwrap(); diff --git a/crates/storage/storage-overlay/src/manager.rs b/crates/storage/storage-overlay/src/manager.rs index 5f3908473cb..7fb8e0839c6 100644 --- a/crates/storage/storage-overlay/src/manager.rs +++ b/crates/storage/storage-overlay/src/manager.rs @@ -113,8 +113,8 @@ impl OverlayManager { P: DBProvider + ChangeSetReader + StorageChangeSetReader - + PruneCheckpointReader + StageCheckpointReader + + PruneCheckpointReader + BlockNumReader + StorageSettingsCache, { @@ -138,6 +138,7 @@ impl OverlayManager { P: DBProvider + ChangeSetReader + StorageChangeSetReader + + StageCheckpointReader + PruneCheckpointReader + BlockNumReader + StorageSettingsCache, diff --git a/crates/storage/storage-overlay/src/provider.rs b/crates/storage/storage-overlay/src/provider.rs index 29736a442c8..f1294c37e5c 100644 --- a/crates/storage/storage-overlay/src/provider.rs +++ b/crates/storage/storage-overlay/src/provider.rs @@ -1,43 +1,44 @@ -use crate::{database_state_frontiers, OverlayBuilder, StateTrieOverlay}; -use alloy_primitives::{BlockHash, B256}; +use crate::{database_state_frontiers, ExecutionOverlay, OverlayBuilder, StateTrieOverlay}; +use alloy_primitives::{Address, BlockHash, BlockNumber, B256, U256}; use metrics::{Counter, Histogram}; -use reth_db_api::{tables, transaction::DbTx, DatabaseError}; -use reth_errors::ProviderResult; +use reth_db_api::{cursor::DbDupCursorRO, tables, transaction::DbTx, DatabaseError}; +use reth_errors::{ProviderError, ProviderResult}; use reth_ethereum_primitives::EthPrimitives; use reth_metrics::Metrics; use reth_primitives_traits::{ dashmap::{self, DashMap}, - NodePrimitives, + Account, NodePrimitives, }; use reth_storage_api::{ - BlockNumReader, ChangeSetReader, DBProvider, DatabaseProviderFactory, - DatabaseProviderROFactory, DbTxProvider, PruneCheckpointReader, StageCheckpointReader, - StorageChangeSetReader, StorageSettingsCache, + AccountReader, BlockHashReader, BlockNumReader, BytecodeReader, ChangeSetReader, DBProvider, + DatabaseProviderFactory, DatabaseProviderROFactory, DbTxProvider, HashedPostStateProvider, + PruneCheckpointReader, StageCheckpointReader, StateProofProvider, StateProvider, + StateRootProvider, StorageChangeSetReader, StorageRootProvider, StorageSettingsCache, }; use reth_trie::{ - hashed_cursor::{HashedCursorFactory, HashedPostStateCursorFactory}, - trie_cursor::{InMemoryTrieCursor, TrieCursor, TrieCursorFactory, TrieStorageCursor}, - HashedPostStateSorted, + hashed_cursor::{ + zero_destroyed_account_storage, HashedCursorFactory, HashedPostStateCursorFactory, + }, + proof::{Proof, StorageProof as TrieStorageProof}, + trie_cursor::{ + InMemoryTrieCursor, InMemoryTrieCursorFactory, TrieCursor, TrieCursorFactory, + TrieStorageCursor, + }, + updates::TrieUpdates, + witness::TrieWitness, + AccountProof, ExecutionWitnessMode, HashedPostState, HashedPostStateSorted, HashedStorage, + KeccakKeyHasher, MultiProof, MultiProofTargets, StateRoot, StorageMultiProof, StorageProof, + StorageRoot, TrieInput, TrieInputSorted, }; use reth_trie_db::{ - DatabaseAccountTrieCursor, DatabaseHashedCursorFactory, DatabaseStorageTrieCursor, - LegacyKeyAdapter, PackedAccountsTrie, PackedKeyAdapter, PackedStoragesTrie, + DatabaseAccountTrieCursor, DatabaseHashedCursorFactory, DatabaseProof, DatabaseStateRoot, + DatabaseStorageProof, DatabaseStorageRoot, DatabaseStorageTrieCursor, + DatabaseTrieCursorFactory, LegacyKeyAdapter, PackedAccountsTrie, PackedKeyAdapter, + PackedStoragesTrie, }; -use std::{sync::Arc, time::Instant}; +use std::{cell::OnceCell, fmt, ops::Deref, sync::Arc, time::Instant}; use tracing::instrument; -/// Metrics for overlay state provider factory operations. -#[derive(Clone, Metrics)] -#[metrics(scope = "storage.providers.overlay")] -pub(crate) struct OverlayStateProviderFactoryMetrics { - /// Duration of creating the database provider transaction. - create_provider_duration: Histogram, - /// Overall duration of the [`OverlayStateProviderFactory::database_provider_ro`] call. - database_provider_ro_duration: Histogram, - /// Number of cache misses when fetching state trie overlays from the overlay cache. - overlay_cache_misses: Counter, -} - /// Factory for creating overlay state providers with optional reverts and overlays. /// /// This factory allows building an `OverlayStateProvider` whose DB state has been reverted to a @@ -52,18 +53,21 @@ pub struct OverlayStateProviderFactory { /// /// Under partial persistence the overlay depends on both durable frontiers, so both hashes are /// part of the cache key. - overlay_cache: Arc>, + state_trie_overlay_cache: StateTrieOverlayCache, + /// A cache which maps durable frontier pairs to [`ExecutionOverlay`]s. + execution_overlay_cache: ExecutionOverlayCache, /// Metrics for provider factory operations. metrics: OverlayStateProviderFactoryMetrics, } impl OverlayStateProviderFactory { - /// Create a new overlay state provider factory + /// Create a new overlay state provider factory. pub fn new(factory: F, overlay_builder: OverlayBuilder) -> Self { Self { factory, overlay_builder, - overlay_cache: Default::default(), + state_trie_overlay_cache: Default::default(), + execution_overlay_cache: Default::default(), metrics: Default::default(), } } @@ -73,20 +77,136 @@ impl OverlayStateProviderFactory { pub fn with_skip_overlay_for_reused_sparse_trie(mut self, anchor_hash: B256) -> Self { self.overlay_builder = self.overlay_builder.with_skip_overlay_for_reused_sparse_trie(anchor_hash); - self.overlay_cache = Default::default(); + self.state_trie_overlay_cache = Default::default(); + self.execution_overlay_cache = Default::default(); self } +} - /// Fetches a [`StateTrieOverlay`] from the cache based on the current durable frontiers. If - /// there is no cached value then this calculates the [`StateTrieOverlay`] and populates the - /// cache. +impl DatabaseProviderROFactory for OverlayStateProviderFactory +where + N: NodePrimitives, + F: DatabaseProviderFactory, + F::Provider: StageCheckpointReader + + PruneCheckpointReader + + BlockNumReader + + ChangeSetReader + + StorageChangeSetReader + + StorageSettingsCache, +{ + type Provider = OverlayStateProvider, N>; + + /// Create a read-only [`OverlayStateProvider`]. #[instrument(level = "debug", target = "providers::state::overlay", skip_all)] - fn get_state_trie_overlay( + fn database_provider_ro( &self, - provider: &Provider, - ) -> ProviderResult + ) -> ProviderResult, N>> { + let overall_start = Instant::now(); + + // Get a read-only provider + let provider = { + let start = Instant::now(); + let res = self.factory.database_provider_ro()?; + self.metrics.create_provider_duration.record(start.elapsed()); + res + }; + + let is_v2 = provider.cached_storage_settings().is_v2(); + self.metrics.database_provider_ro_duration.record(overall_start.elapsed()); + Ok(OverlayStateProvider::new( + provider, + self.overlay_builder.clone(), + Arc::clone(&self.state_trie_overlay_cache), + Arc::clone(&self.execution_overlay_cache), + self.metrics.clone(), + is_v2, + )) + } +} + +/// State provider with lazily resolved state trie and execution overlays. +pub struct OverlayStateProvider { + provider: Provider, + overlay_builder: Option>, + state_trie_overlay_cache: StateTrieOverlayCache, + execution_overlay_cache: ExecutionOverlayCache, + metrics: OverlayStateProviderFactoryMetrics, + state_trie_overlay: OnceCell, + execution_overlay: OnceCell>, + is_v2: bool, +} + +impl OverlayStateProvider, N> { + const fn new( + provider: Provider, + overlay_builder: OverlayBuilder, + state_trie_overlay_cache: StateTrieOverlayCache, + execution_overlay_cache: ExecutionOverlayCache, + metrics: OverlayStateProviderFactoryMetrics, + is_v2: bool, + ) -> Self { + Self { + provider: OwnedProvider(provider), + overlay_builder: Some(overlay_builder), + state_trie_overlay_cache, + execution_overlay_cache, + metrics, + state_trie_overlay: OnceCell::new(), + execution_overlay: OnceCell::new(), + is_v2, + } + } + + #[cfg(test)] + fn new_with_execution( + provider: Provider, + execution_overlay: Arc, + is_v2: bool, + ) -> Self { + Self { + provider: OwnedProvider(provider), + overlay_builder: None, + state_trie_overlay_cache: Default::default(), + execution_overlay_cache: Default::default(), + metrics: Default::default(), + state_trie_overlay: OnceCell::new(), + execution_overlay: OnceCell::from(execution_overlay), + is_v2, + } + } +} + +impl<'a, Provider, N: NodePrimitives> OverlayStateProvider<&'a Provider, N> { + pub(crate) fn new_with_state_trie( + provider: &'a Provider, + state_trie_overlay: StateTrieOverlay, + is_v2: bool, + ) -> Self { + Self { + provider, + overlay_builder: None, + state_trie_overlay_cache: Default::default(), + execution_overlay_cache: Default::default(), + metrics: Default::default(), + state_trie_overlay: OnceCell::from(state_trie_overlay), + execution_overlay: OnceCell::new(), + is_v2, + } + } +} + +impl OverlayStateProvider +where + Provider: Deref, + Provider::Target: Sized, +{ + fn provider(&self) -> &Provider::Target { + &self.provider + } + + fn state_trie_overlay(&self) -> ProviderResult<&StateTrieOverlay> where - Provider: StageCheckpointReader + Provider::Target: StageCheckpointReader + PruneCheckpointReader + ChangeSetReader + StorageChangeSetReader @@ -94,83 +214,541 @@ impl OverlayStateProviderFactory { + BlockNumReader + StorageSettingsCache, { - let (state_trie_tip_block, finish_tip_block) = database_state_frontiers(provider)?; - - let overlay = - match self.overlay_cache.entry((state_trie_tip_block.hash, finish_tip_block.hash)) { - dashmap::Entry::Occupied(entry) => entry.get().clone(), - dashmap::Entry::Vacant(entry) => { - self.metrics.overlay_cache_misses.increment(1); - let overlay = self.overlay_builder.build_state_trie_overlay_at_frontiers( - provider, + if let Some(overlay) = self.state_trie_overlay.get() { + return Ok(overlay) + } + + let (state_trie_tip_block, finish_tip_block) = database_state_frontiers(self.provider())?; + let overlay = match self + .state_trie_overlay_cache + .entry((state_trie_tip_block.hash, finish_tip_block.hash)) + { + dashmap::Entry::Occupied(entry) => entry.get().clone(), + dashmap::Entry::Vacant(entry) => { + self.metrics.state_trie_overlay_cache_misses.increment(1); + let overlay = self + .overlay_builder + .as_ref() + .expect("state trie overlay must be initialized or lazily resolvable") + .build_state_trie_overlay_at_frontiers( + self.provider(), state_trie_tip_block, finish_tip_block, )?; + if !overlay.skipped_for_reused_sparse_trie() { entry.insert(overlay.clone()); - overlay } - }; + overlay + } + }; + let _ = self.state_trie_overlay.set(overlay); + Ok(self.state_trie_overlay.get().expect("state trie overlay was just initialized")) + } + + fn build_overlay(&self, input: TrieInputSorted) -> ProviderResult + where + Provider::Target: StageCheckpointReader + + PruneCheckpointReader + + ChangeSetReader + + StorageChangeSetReader + + DBProvider + + BlockNumReader + + StorageSettingsCache, + { + let overlay = self.state_trie_overlay()?; + if overlay.skipped_for_reused_sparse_trie() { + return Err(ProviderError::UnsupportedProvider) + } + let TrieInputSorted { nodes: input_nodes, state: input_state, prefix_sets } = input; + let mut nodes = Arc::clone(&overlay.trie_updates); + let mut state = Arc::clone(&overlay.hashed_post_state); + + if !input_nodes.is_empty() { + Arc::make_mut(&mut nodes).extend_ref_and_sort(&input_nodes); + } + if !input_state.is_empty() { + Arc::make_mut(&mut state).extend_ref_and_sort(&input_state); + } - Ok(overlay) + Ok(TrieInputSorted::new(nodes, state, prefix_sets)) + } + + fn execution_overlay(&self) -> ProviderResult<&Arc> + where + Provider::Target: StageCheckpointReader + + PruneCheckpointReader + + ChangeSetReader + + StorageChangeSetReader + + DBProvider + + BlockNumReader, + { + if let Some(overlay) = self.execution_overlay.get() { + return Ok(overlay) + } + + let (state_trie_tip_block, finish_tip_block) = database_state_frontiers(self.provider())?; + let overlay = match self + .execution_overlay_cache + .entry((state_trie_tip_block.hash, finish_tip_block.hash)) + { + dashmap::Entry::Occupied(entry) => entry.get().clone(), + dashmap::Entry::Vacant(entry) => { + self.metrics.execution_overlay_cache_misses.increment(1); + let overlay = self + .overlay_builder + .as_ref() + .expect("execution overlay must be initialized or lazily resolvable") + .build_execution_overlay_at_frontiers( + self.provider(), + state_trie_tip_block, + finish_tip_block, + )?; + entry.insert(overlay.clone()); + overlay + } + }; + let _ = self.execution_overlay.set(overlay); + Ok(self.execution_overlay.get().expect("execution overlay was just initialized")) } } -impl DatabaseProviderROFactory for OverlayStateProviderFactory +impl fmt::Debug for OverlayStateProvider where - N: NodePrimitives, - F: DatabaseProviderFactory, - F::Provider: StageCheckpointReader + Provider: Deref, + Provider::Target: fmt::Debug + Sized, +{ + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("OverlayStateProvider") + .field("provider", self.provider()) + .field("state_trie_overlay", &self.state_trie_overlay.get()) + .field("execution_overlay", &self.execution_overlay.get()) + .field("is_v2", &self.is_v2) + .finish() + } +} + +impl AccountReader for OverlayStateProvider +where + Provider: Deref, + Provider::Target: DBProvider + + StorageSettingsCache + + StageCheckpointReader + + PruneCheckpointReader + + ChangeSetReader + + StorageChangeSetReader + + BlockNumReader, +{ + fn basic_account(&self, address: &Address) -> ProviderResult> { + let overlay = self.execution_overlay()?; + if let Some(account) = overlay.accounts().get(address) { + return Ok(account.as_ref().map(Account::from)) + } + if self.provider().cached_storage_settings().use_hashed_state() { + let hashed_address = alloy_primitives::keccak256(address); + self.provider() + .tx() + .get_by_encoded_key::(&hashed_address) + .map_err(Into::into) + } else { + self.provider() + .tx() + .get_by_encoded_key::(address) + .map_err(Into::into) + } + } +} + +impl BlockHashReader for OverlayStateProvider +where + Provider: Deref, + Provider::Target: BlockHashReader + + DBProvider + + Sized + + StageCheckpointReader + PruneCheckpointReader + + ChangeSetReader + + StorageChangeSetReader + + BlockNumReader, +{ + fn block_hash(&self, number: BlockNumber) -> ProviderResult> { + let overlay = self.execution_overlay()?; + if let Some(block) = overlay.block_hashes().iter().find(|block| block.number == number) { + return Ok(Some(block.hash)) + } + self.provider().block_hash(number) + } + + fn canonical_hashes_range( + &self, + start: BlockNumber, + end: BlockNumber, + ) -> ProviderResult> { + let overlay = self.execution_overlay()?; + let mut block_hashes = + overlay.block_hashes().iter().filter(|block| (start..end).contains(&block.number)); + let Some(first_block) = block_hashes.next() else { + return self.provider().canonical_hashes_range(start, end) + }; + + let mut hashes = self.provider().canonical_hashes_range(start, first_block.number)?; + hashes.push(first_block.hash); + hashes.extend(block_hashes.map(|block| block.hash)); + Ok(hashes) + } +} + +impl BytecodeReader for OverlayStateProvider +where + Provider: Deref, + Provider::Target: DBProvider + + StageCheckpointReader + + PruneCheckpointReader + + ChangeSetReader + + StorageChangeSetReader + + BlockNumReader, +{ + fn bytecode_by_hash( + &self, + code_hash: &B256, + ) -> ProviderResult> { + let overlay = self.execution_overlay()?; + if let Some(bytecode) = overlay.code_hashes().get(code_hash) { + return Ok(Some(reth_primitives_traits::Bytecode(bytecode.clone()))); + } + self.provider().tx().get_by_encoded_key::(code_hash).map_err(Into::into) + } +} + +impl StateRootProvider for OverlayStateProvider +where + Provider: Deref, + Provider::Target: DBProvider + + StageCheckpointReader + + PruneCheckpointReader + + ChangeSetReader + + StorageChangeSetReader + BlockNumReader + + StorageSettingsCache, +{ + fn state_root(&self, hashed_state: HashedPostState) -> ProviderResult { + reth_trie_db::with_adapter!(self.provider(), |A| { + let input = self.build_overlay(TrieInputSorted::from_unsorted( + TrieInput::from_state(hashed_state), + ))?; + Ok(>::overlay_root_from_nodes(self.provider().tx(), input)?) + }) + } + + fn state_root_from_nodes(&self, input: TrieInput) -> ProviderResult { + reth_trie_db::with_adapter!(self.provider(), |A| { + let input = self.build_overlay(TrieInputSorted::from_unsorted(input))?; + Ok( as DatabaseStateRoot<_>>::overlay_root_from_nodes( + self.provider().tx(), + input, + )?) + }) + } + + fn state_root_with_updates( + &self, + hashed_state: HashedPostState, + ) -> ProviderResult<(B256, TrieUpdates)> { + reth_trie_db::with_adapter!(self.provider(), |A| { + let input = self.build_overlay(TrieInputSorted::from_unsorted( + TrieInput::from_state(hashed_state), + ))?; + Ok(>::overlay_root_from_nodes_with_updates( + self.provider().tx(), + input, + )?) + }) + } + + fn state_root_from_nodes_with_updates( + &self, + input: TrieInput, + ) -> ProviderResult<(B256, TrieUpdates)> { + reth_trie_db::with_adapter!(self.provider(), |A| { + let input = self.build_overlay(TrieInputSorted::from_unsorted(input))?; + Ok( + as DatabaseStateRoot<_>>::overlay_root_from_nodes_with_updates( + self.provider().tx(), + input, + )?, + ) + }) + } +} + +impl StorageRootProvider for OverlayStateProvider +where + Provider: Deref, + Provider::Target: DBProvider + + StageCheckpointReader + + PruneCheckpointReader + ChangeSetReader + StorageChangeSetReader + + BlockNumReader + StorageSettingsCache, { - type Provider = OverlayStateProvider; + fn storage_root( + &self, + address: Address, + hashed_storage: HashedStorage, + ) -> ProviderResult { + reth_trie_db::with_adapter!(self.provider(), |A| { + let input = self.build_overlay(TrieInputSorted::from_unsorted( + TrieInput::from_state(HashedPostState::from_hashed_storage( + alloy_primitives::keccak256(address), + hashed_storage, + )), + ))?; + let hashed_storage = input + .state + .account_storages() + .get(&alloy_primitives::keccak256(address)) + .cloned() + .unwrap_or_default() + .into(); + >::overlay_root(self.provider().tx(), address, hashed_storage) + .map_err(|err| ProviderError::Database(err.into())) + }) + } - /// Create a read-only [`OverlayStateProvider`]. - #[instrument(level = "debug", target = "providers::state::overlay", skip_all)] - fn database_provider_ro(&self) -> ProviderResult> { - let overall_start = Instant::now(); + fn storage_proof( + &self, + address: Address, + slot: B256, + hashed_storage: HashedStorage, + ) -> ProviderResult { + reth_trie_db::with_adapter!(self.provider(), |A| { + let input = self.build_overlay(TrieInputSorted::from_unsorted( + TrieInput::from_state(HashedPostState::from_hashed_storage( + alloy_primitives::keccak256(address), + hashed_storage, + )), + ))?; + let hashed_storage = input + .state + .account_storages() + .get(&alloy_primitives::keccak256(address)) + .cloned() + .unwrap_or_default() + .into(); + >::overlay_storage_proof( + self.provider().tx(), + address, + slot, + hashed_storage, + ) + .map_err(ProviderError::from) + }) + } - // Get a read-only provider - let provider = { - let start = Instant::now(); - let res = self.factory.database_provider_ro()?; - self.metrics.create_provider_duration.record(start.elapsed()); - res - }; + fn storage_multiproof( + &self, + address: Address, + slots: &[B256], + hashed_storage: HashedStorage, + ) -> ProviderResult { + reth_trie_db::with_adapter!(self.provider(), |A| { + let input = self.build_overlay(TrieInputSorted::from_unsorted( + TrieInput::from_state(HashedPostState::from_hashed_storage( + alloy_primitives::keccak256(address), + hashed_storage, + )), + ))?; + let hashed_storage = input + .state + .account_storages() + .get(&alloy_primitives::keccak256(address)) + .cloned() + .unwrap_or_default() + .into(); + >::overlay_storage_multiproof( + self.provider().tx(), + address, + slots, + hashed_storage, + ) + .map_err(ProviderError::from) + }) + } +} - let overlay = self.get_state_trie_overlay(&provider)?; +impl StateProofProvider for OverlayStateProvider +where + Provider: Deref, + Provider::Target: DBProvider + + StageCheckpointReader + + PruneCheckpointReader + + ChangeSetReader + + StorageChangeSetReader + + BlockNumReader + + StorageSettingsCache, +{ + fn proof( + &self, + input: TrieInput, + address: Address, + slots: &[B256], + ) -> ProviderResult { + reth_trie_db::with_adapter!(self.provider(), |A| { + let TrieInputSorted { nodes, state, prefix_sets } = + self.build_overlay(TrieInputSorted::from_unsorted(input))?; + let input = TrieInput::new( + Arc::unwrap_or_clone(nodes).into(), + Arc::unwrap_or_clone(state).into(), + prefix_sets, + ); + let proof = as DatabaseProof>::from_tx(self.provider().tx()); + proof.overlay_account_proof(input, address, slots).map_err(ProviderError::from) + }) + } - let is_v2 = provider.cached_storage_settings().is_v2(); - self.metrics.database_provider_ro_duration.record(overall_start.elapsed()); - Ok(OverlayStateProvider::new(provider, overlay, is_v2)) + fn multiproof( + &self, + input: TrieInput, + targets: MultiProofTargets, + ) -> ProviderResult { + reth_trie_db::with_adapter!(self.provider(), |A| { + let TrieInputSorted { nodes, state, prefix_sets } = + self.build_overlay(TrieInputSorted::from_unsorted(input))?; + let input = TrieInput::new( + Arc::unwrap_or_clone(nodes).into(), + Arc::unwrap_or_clone(state).into(), + prefix_sets, + ); + let proof = as DatabaseProof>::from_tx(self.provider().tx()); + proof.overlay_multiproof(input, targets).map_err(ProviderError::from) + }) + } + + fn witness( + &self, + input: TrieInput, + target: HashedPostState, + mode: ExecutionWitnessMode, + ) -> ProviderResult> { + reth_trie_db::with_adapter!(self.provider(), |A| { + let TrieInputSorted { nodes, state, prefix_sets } = + self.build_overlay(TrieInputSorted::from_unsorted(input))?; + let witness = TrieWitness::new( + InMemoryTrieCursorFactory::new( + DatabaseTrieCursorFactory::<_, A>::new(self.provider().tx()), + nodes.as_ref(), + ), + HashedPostStateCursorFactory::new( + DatabaseHashedCursorFactory::new(self.provider().tx()), + state.as_ref(), + ), + ) + .with_prefix_sets_mut(prefix_sets) + .with_execution_witness_mode(mode); + let witness = + if mode.is_canonical() { witness } else { witness.always_include_root_node() }; + let mut values: Vec<_> = witness.compute(target)?.into_values().collect(); + if mode.is_canonical() { + values.sort_unstable(); + } + Ok(values) + }) } } -/// State provider with in-memory overlay from trie updates and hashed post state. -/// -/// This provider uses in-memory trie updates and hashed post state as an overlay -/// on top of a database provider, implementing [`TrieCursorFactory`] and [`HashedCursorFactory`] -/// using the in-memory overlay factories. -#[derive(Debug)] -pub struct OverlayStateProvider { - provider: Provider, - overlay: StateTrieOverlay, - is_v2: bool, +impl HashedPostStateProvider for OverlayStateProvider +where + Provider: Deref, + Provider::Target: DBProvider + + StageCheckpointReader + + PruneCheckpointReader + + ChangeSetReader + + StorageChangeSetReader + + BlockNumReader + + StorageSettingsCache, +{ + fn hashed_post_state( + &self, + bundle_state: &revm::database::BundleState, + ) -> ProviderResult { + let mut hashed_state = + HashedPostState::from_bundle_state::(bundle_state.state()); + if !bundle_state + .state() + .values() + .any(|account| account.was_destroyed() && account.original_info.is_some()) + { + return Ok(hashed_state) + } + + let overlay_state = self.build_overlay(TrieInputSorted::default())?.state; + zero_destroyed_account_storage( + &HashedPostStateCursorFactory::new( + DatabaseHashedCursorFactory::new(self.provider().tx()), + overlay_state.as_ref(), + ), + bundle_state.state(), + &mut hashed_state, + )?; + Ok(hashed_state) + } } -impl OverlayStateProvider { - /// Creates a new overlay state provider. - pub const fn new(provider: Provider, overlay: StateTrieOverlay, is_v2: bool) -> Self { - Self { provider, overlay, is_v2 } +impl StateProvider for OverlayStateProvider +where + Provider: Deref, + Provider::Target: DBProvider + + BlockHashReader + + StorageSettingsCache + + StageCheckpointReader + + PruneCheckpointReader + + ChangeSetReader + + StorageChangeSetReader + + BlockNumReader, +{ + fn storage( + &self, + address: Address, + storage_key: alloy_primitives::StorageKey, + ) -> ProviderResult> { + let overlay = self.execution_overlay()?; + if let Some(value) = overlay + .storage() + .get(&address) + .and_then(|storage| storage.get(&U256::from_be_bytes(storage_key.0))) + { + return Ok(Some(*value)); + } + if self.provider().cached_storage_settings().use_hashed_state() { + let hashed_address = alloy_primitives::keccak256(address); + let hashed_slot = alloy_primitives::keccak256(storage_key); + let mut cursor = self.provider().tx().cursor_dup_read::()?; + Ok(cursor + .seek_by_key_subkey(hashed_address, hashed_slot)? + .filter(|entry| entry.key == hashed_slot) + .map(|entry| entry.value)) + } else { + let mut cursor = self.provider().tx().cursor_dup_read::()?; + if let Some(entry) = cursor.seek_by_key_subkey(address, storage_key)? && + entry.key == storage_key + { + return Ok(Some(entry.value)) + } + Ok(None) + } } } -impl TrieCursorFactory for OverlayStateProvider +impl TrieCursorFactory for OverlayStateProvider where - Provider: DbTxProvider, + Provider: Deref, + Provider::Target: DBProvider + + StageCheckpointReader + + PruneCheckpointReader + + ChangeSetReader + + StorageChangeSetReader + + BlockNumReader + + StorageSettingsCache, { type AccountTrieCursor<'a> = InMemoryTrieCursor<'a, Box> @@ -183,44 +761,53 @@ where Self: 'a; fn account_trie_cursor(&self) -> Result, DatabaseError> { + let overlay = self.state_trie_overlay().map_err(into_database_error)?; let cursor: Box = if self.is_v2 { Box::new(DatabaseAccountTrieCursor::<_, PackedKeyAdapter>::new( - self.provider.tx().cursor_read::()?, + self.provider().tx().cursor_read::()?, )) } else { Box::new(DatabaseAccountTrieCursor::<_, LegacyKeyAdapter>::new( - self.provider.tx().cursor_read::()?, + self.provider().tx().cursor_read::()?, )) }; - Ok(InMemoryTrieCursor::new_account(cursor, &self.overlay.trie_updates)) + Ok(InMemoryTrieCursor::new_account(cursor, &overlay.trie_updates)) } fn storage_trie_cursor( &self, hashed_address: B256, ) -> Result, DatabaseError> { + let overlay = self.state_trie_overlay().map_err(into_database_error)?; let cursor: Box = if self.is_v2 { Box::new(DatabaseStorageTrieCursor::<_, PackedKeyAdapter>::new( - self.provider.tx().cursor_dup_read::()?, + self.provider().tx().cursor_dup_read::()?, hashed_address, )) } else { Box::new(DatabaseStorageTrieCursor::<_, LegacyKeyAdapter>::new( - self.provider.tx().cursor_dup_read::()?, + self.provider().tx().cursor_dup_read::()?, hashed_address, )) }; - Ok(InMemoryTrieCursor::new_storage(cursor, &self.overlay.trie_updates, hashed_address)) + Ok(InMemoryTrieCursor::new_storage(cursor, &overlay.trie_updates, hashed_address)) } } -impl HashedCursorFactory for OverlayStateProvider +impl HashedCursorFactory for OverlayStateProvider where - Provider: DbTxProvider, + Provider: Deref, + Provider::Target: DBProvider + + StageCheckpointReader + + PruneCheckpointReader + + ChangeSetReader + + StorageChangeSetReader + + BlockNumReader + + StorageSettingsCache, { type AccountCursor<'a> = , + DatabaseHashedCursorFactory<&'a ::Tx>, &'a Arc, > as HashedCursorFactory>::AccountCursor<'a> where @@ -228,16 +815,17 @@ where type StorageCursor<'a> = , + DatabaseHashedCursorFactory<&'a ::Tx>, &'a Arc, > as HashedCursorFactory>::StorageCursor<'a> where Self: 'a; fn hashed_account_cursor(&self) -> Result, DatabaseError> { + let overlay = self.state_trie_overlay().map_err(into_database_error)?; HashedPostStateCursorFactory::new( - DatabaseHashedCursorFactory::new(self.provider.tx()), - &self.overlay.hashed_post_state, + DatabaseHashedCursorFactory::new(self.provider().tx()), + &overlay.hashed_post_state, ) .hashed_account_cursor() } @@ -246,19 +834,69 @@ where &self, hashed_address: B256, ) -> Result, DatabaseError> { + let overlay = self.state_trie_overlay().map_err(into_database_error)?; HashedPostStateCursorFactory::new( - DatabaseHashedCursorFactory::new(self.provider.tx()), - &self.overlay.hashed_post_state, + DatabaseHashedCursorFactory::new(self.provider().tx()), + &overlay.hashed_post_state, ) .hashed_storage_cursor(hashed_address) } } +/// Metrics for overlay state provider factory operations. +#[derive(Clone, Metrics)] +#[metrics(scope = "storage.providers.overlay")] +pub(crate) struct OverlayStateProviderFactoryMetrics { + /// Duration of creating the database provider transaction. + create_provider_duration: Histogram, + /// Overall duration of the [`OverlayStateProviderFactory::database_provider_ro`] call. + database_provider_ro_duration: Histogram, + /// Number of cache misses when fetching state trie overlays. + state_trie_overlay_cache_misses: Counter, + /// Number of cache misses when fetching execution overlays. + execution_overlay_cache_misses: Counter, +} + +type StateTrieOverlayCache = Arc>; +type ExecutionOverlayCache = Arc>>; + +type DbStateRoot<'a, TX, A> = + StateRoot, DatabaseHashedCursorFactory<&'a TX>>; +type DbStorageRoot<'a, TX, A> = + StorageRoot, DatabaseHashedCursorFactory<&'a TX>>; +type DbStorageProof<'a, TX, A> = TrieStorageProof< + 'static, + DatabaseTrieCursorFactory<&'a TX, A>, + DatabaseHashedCursorFactory<&'a TX>, +>; +type DbProof<'a, TX, A> = + Proof, DatabaseHashedCursorFactory<&'a TX>>; + +#[doc(hidden)] +#[derive(Debug)] +pub struct OwnedProvider(Provider); + +impl Deref for OwnedProvider { + type Target = Provider; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +fn into_database_error(error: ProviderError) -> DatabaseError { + match error { + ProviderError::Database(error) => error, + error => DatabaseError::Other(error.to_string()), + } +} + #[cfg(test)] mod tests { use super::*; - use crate::OverlayManager; - use alloy_primitives::U256; + use crate::{ExecutionOverlay, OverlayManager}; + use alloy_eips::BlockNumHash; + use alloy_primitives::{Address, U256}; use reth_chain_state::{test_utils::TestBlockBuilder, ExecutedBlock}; use reth_primitives_traits::Account; use reth_provider::{ @@ -271,6 +909,7 @@ mod tests { updates::TrieUpdatesSorted, BranchNodeCompact, ComputedTrieData, HashedPostState, HashedStorage, Nibbles, }; + use revm::{bytecode::Bytecode as RevmBytecode, state::AccountInfo}; fn with_unique_trie_data( block: &ExecutedBlock, @@ -352,8 +991,8 @@ mod tests { manager.overlay_builder(blocks[3].recovered_block().hash()), ); - let provider = factory.provider().unwrap(); - let first = overlay_factory.get_state_trie_overlay(&provider).unwrap(); + let provider = overlay_factory.database_provider_ro().unwrap(); + let first = provider.state_trie_overlay().unwrap().clone(); assert_eq!(account_keys(&first), vec![B256::with_last_byte(3), B256::with_last_byte(4)]); drop(provider); @@ -368,10 +1007,101 @@ mod tests { .unwrap(); provider_rw.commit().unwrap(); - let provider = factory.provider().unwrap(); - let second = overlay_factory.get_state_trie_overlay(&provider).unwrap(); + let provider = overlay_factory.database_provider_ro().unwrap(); + let second = provider.state_trie_overlay().unwrap().clone(); assert_eq!(account_keys(&second), vec![B256::with_last_byte(4)]); assert_eq!(account_node_paths(&second), vec![Nibbles::from_nibbles([4])]); - assert_eq!(overlay_factory.overlay_cache.len(), 2); + assert_eq!(overlay_factory.state_trie_overlay_cache.len(), 2); + } + + #[test] + fn overlays_are_computed_lazily() { + let (factory, blocks) = setup_frontiers(1, 3); + let manager = OverlayManager::default(); + for block in &blocks[2..=3] { + manager.insert_block(block.clone()); + } + let overlay_factory = OverlayStateProviderFactory::new( + factory, + manager.overlay_builder(blocks[3].recovered_block().hash()), + ); + + let provider = overlay_factory.database_provider_ro().unwrap(); + + assert!(provider.state_trie_overlay.get().is_none()); + assert!(provider.execution_overlay.get().is_none()); + assert!(overlay_factory.state_trie_overlay_cache.is_empty()); + assert!(overlay_factory.execution_overlay_cache.is_empty()); + + provider.basic_account(&Address::ZERO).unwrap(); + let execution_overlay = Arc::clone(provider.execution_overlay.get().unwrap()); + assert!(provider.state_trie_overlay.get().is_none()); + assert!(provider.execution_overlay.get().is_some()); + assert!(overlay_factory.state_trie_overlay_cache.is_empty()); + assert_eq!(overlay_factory.execution_overlay_cache.len(), 1); + let cached_overlay = overlay_factory.execution_overlay_cache.iter().next().unwrap(); + assert!(Arc::ptr_eq(&execution_overlay, cached_overlay.value())); + + provider.account_trie_cursor().unwrap(); + assert_eq!(overlay_factory.state_trie_overlay_cache.len(), 1); + assert_eq!(overlay_factory.execution_overlay_cache.len(), 1); + } + + #[test] + fn skipped_state_trie_overlay_is_not_cached_or_used_for_state_roots() { + let (factory, blocks) = setup_frontiers(3, 3); + let manager = OverlayManager::default(); + manager.insert_block(blocks[4].clone()); + let state_provider_factory = OverlayStateProviderFactory::new( + factory, + manager + .overlay_builder(blocks[4].recovered_block().hash()) + .with_skip_overlay_for_reused_sparse_trie(blocks[3].recovered_block().hash()), + ); + + let provider = state_provider_factory.database_provider_ro().unwrap(); + assert!(provider.state_trie_overlay().unwrap().skipped_for_reused_sparse_trie()); + assert!(state_provider_factory.state_trie_overlay_cache.is_empty()); + assert!(matches!( + provider.state_root(HashedPostState::default()), + Err(ProviderError::UnsupportedProvider) + )); + assert!(state_provider_factory.state_trie_overlay_cache.is_empty()); + } + + #[test] + fn execution_overlay_readers_use_overlay_first() { + let (factory, _) = setup_frontiers(1, 3); + let address = Address::with_last_byte(1); + let account_info = AccountInfo { nonce: 1, balance: U256::from(2), ..Default::default() }; + let block_hash = B256::with_last_byte(3); + let storage_key = B256::with_last_byte(4); + let storage_value = U256::from(5); + let code_hash = B256::with_last_byte(6); + let bytecode = RevmBytecode::new_raw([0x60, 0x01].into()); + let mut execution_overlay = ExecutionOverlay::default(); + execution_overlay.accounts_mut().insert(address, Some(account_info.clone())); + execution_overlay.block_hashes_mut().push(BlockNumHash::new(1, block_hash)); + execution_overlay + .storage_mut() + .entry(address) + .or_default() + .insert(U256::from_be_bytes(storage_key.0), storage_value); + execution_overlay.code_hashes_mut().insert(code_hash, bytecode.clone()); + let provider = OverlayStateProvider::<_, EthPrimitives>::new_with_execution( + factory.provider().unwrap(), + Arc::new(execution_overlay), + false, + ); + + assert_eq!(provider.basic_account(&address).unwrap(), Some(Account::from(account_info))); + assert!(provider.basic_account(&Address::with_last_byte(2)).unwrap().is_none()); + assert_eq!(provider.block_hash(1).unwrap(), Some(block_hash)); + assert_eq!(provider.canonical_hashes_range(1, 2).unwrap(), vec![block_hash]); + assert_eq!(provider.storage(address, storage_key).unwrap(), Some(storage_value)); + assert_eq!( + provider.bytecode_by_hash(&code_hash).unwrap(), + Some(reth_primitives_traits::Bytecode(bytecode)) + ); } } From 2ac1b597500c8fd58d5ace78232dddeea40f8bba Mon Sep 17 00:00:00 2001 From: Brian Picciano Date: Fri, 28 Aug 2026 14:37:01 +0000 Subject: [PATCH 091/294] fix(overlay): run computations on dedicated workers (#26871) --- crates/storage/storage-overlay/src/manager.rs | 4 +- crates/tasks/src/pool.rs | 39 +++++++++++++++++++ 2 files changed, 41 insertions(+), 2 deletions(-) diff --git a/crates/storage/storage-overlay/src/manager.rs b/crates/storage/storage-overlay/src/manager.rs index 7fb8e0839c6..7afbebc9d39 100644 --- a/crates/storage/storage-overlay/src/manager.rs +++ b/crates/storage/storage-overlay/src/manager.rs @@ -573,7 +573,7 @@ impl OverlayManager { if let Some(worker_pool) = &self.worker_pool { let compute_span = _span; let metrics = self.metrics.clone(); - return worker_pool.install_fn(move || { + return worker_pool.spawn_and_wait(move || { let _guard = compute_span.enter(); compute_overlay(compute_input, anchor_hash, &metrics) }) @@ -594,7 +594,7 @@ impl OverlayManager { if let Some(worker_pool) = &self.worker_pool { let compute_span = _span; let metrics = self.execution_metrics.clone(); - return worker_pool.install_fn(move || { + return worker_pool.spawn_and_wait(move || { let _guard = compute_span.enter(); compute_execution_overlay_inner(compute_input, anchor_hash, &metrics) }) diff --git a/crates/tasks/src/pool.rs b/crates/tasks/src/pool.rs index b1f632bb37b..e44593cd6b4 100644 --- a/crates/tasks/src/pool.rs +++ b/crates/tasks/src/pool.rs @@ -301,6 +301,28 @@ impl WorkerPool { }) } + /// Runs a closure on this pool, waiting for its result. + /// + /// Unlike [`install_fn`](Self::install_fn), this always queues the closure onto this pool + /// when called from another rayon pool. This avoids Rayon running the closure on the caller's + /// worker through its cross-pool install path. + pub fn spawn_and_wait(&self, f: impl FnOnce() -> R + Send + 'static) -> R { + let pool = self.pool(); + if pool.current_thread_index().is_some() { + return f() + } + + let (tx, rx) = std::sync::mpsc::sync_channel(1); + self.spawn(move || { + let _ = tx.send(catch_unwind(AssertUnwindSafe(f))); + }); + + match rx.recv().expect("worker pool exited before completing task") { + Ok(result) => result, + Err(panic) => std::panic::resume_unwind(panic), + } + } + /// Spawns a closure on the pool. pub fn spawn(&self, f: impl FnOnce() + Send + 'static) { let pool = self.pool(); @@ -549,4 +571,21 @@ mod tests { pool.clear(); } + + #[test] + fn worker_pool_spawn_and_wait_runs_on_target_pool() { + let caller_pool = WorkerPool::new(1, "caller"); + let target_pool = Arc::new(WorkerPool::new(1, "target")); + + let thread_name = caller_pool.install_fn({ + let target_pool = Arc::clone(&target_pool); + move || { + WorkerPool::with_worker_mut(|_| { + target_pool.spawn_and_wait(|| thread::current().name().unwrap().to_owned()) + }) + } + }); + + assert_eq!(thread_name, "target-00"); + } } From f55ba955cd9ed40f6d1e69479cc6ed10c03545a9 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Fri, 28 Aug 2026 14:49:46 +0000 Subject: [PATCH 092/294] docs(tasks): document worker access reentrancy (#26873) --- crates/tasks/src/pool.rs | 39 +++++++++++++++++++++++++++++++++++++-- 1 file changed, 37 insertions(+), 2 deletions(-) diff --git a/crates/tasks/src/pool.rs b/crates/tasks/src/pool.rs index e44593cd6b4..c6c718fb39c 100644 --- a/crates/tasks/src/pool.rs +++ b/crates/tasks/src/pool.rs @@ -168,6 +168,10 @@ thread_local! { /// [`Worker::init`]. The state is thread-local and accessible during [`install`](Self::install) /// calls. /// +/// Worker access is backed by a thread-local [`RefCell`]. Keep [`with_worker`](Self::with_worker) +/// and [`with_worker_mut`](Self::with_worker_mut) closures short and non-yielding: if Rayon runs +/// another job on the same thread while a borrow is active, re-entrant worker access can panic. +/// /// The pool supports multiple init/clear cycles, allowing reuse of the same threads with /// different state configurations. /// @@ -344,15 +348,46 @@ impl WorkerPool { self.pool().in_place_scope(f) } - /// Access the current thread's [`Worker`] from within an [`install`](Self::install) closure. + /// Accesses the current thread's [`Worker`] from within a pool closure. /// /// This is useful for accessing the worker from inside `par_iter` where the initial `&Worker` /// reference from `install` belongs to a different thread. + /// + /// This borrows the thread-local worker for the entire duration of `f`. Do not yield to Rayon + /// from inside `f` if another job could call [`with_worker_mut`](Self::with_worker_mut) on the + /// same thread. Yield points include parallel iterators, `rayon::join`, scopes, and waiting in + /// `ThreadPool::install` on a different Rayon pool. A cross-pool `install` may run another job + /// from the caller's pool on the same thread while it waits. + /// + /// Prefer copying or cloning the required worker state in `f`, returning from this method to + /// release the borrow, and only then performing work that may yield. + /// + /// # Panics + /// + /// Panics if the current thread's worker is already mutably borrowed, including by a + /// re-entrant [`with_worker_mut`](Self::with_worker_mut) call. pub fn with_worker(f: impl FnOnce(&Worker) -> R) -> R { WORKER.with_borrow(|worker| f(worker)) } - /// Mutably access the current thread's [`Worker`] from within a pool closure. + /// Mutably accesses the current thread's [`Worker`] from within a pool closure. + /// + /// This exclusively borrows the thread-local worker for the entire duration of `f`. The borrow + /// is not re-entrant: if Rayon runs another job on the same thread before `f` returns, any call + /// to [`with_worker`](Self::with_worker) or `with_worker_mut` from that job will panic. + /// + /// Do not call operations that can yield to Rayon from inside `f` when re-entrant worker access + /// is possible. This includes parallel iterators, `rayon::join`, scopes, and waiting in + /// `ThreadPool::install` on a different Rayon pool. In particular, a cross-pool `install` + /// cooperatively runs jobs from the caller's pool while waiting for the target pool. + /// + /// Prefer computing updates before entering `with_worker_mut`, then use this closure only to + /// apply the update to the worker state. + /// + /// # Panics + /// + /// Panics if the current thread's worker is already borrowed, including by a re-entrant + /// `with_worker` or `with_worker_mut` call. pub fn with_worker_mut(f: impl FnOnce(&mut Worker) -> R) -> R { WORKER.with_borrow_mut(|worker| f(worker)) } From fa9635290b9232ab5d2ebc6ae6abd70b68caec88 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Fri, 28 Aug 2026 16:09:12 +0000 Subject: [PATCH 093/294] fix(rpc): track batch entries in a dedicated call counter (#26866) --- crates/rpc/rpc-builder/src/metrics.rs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/crates/rpc/rpc-builder/src/metrics.rs b/crates/rpc/rpc-builder/src/metrics.rs index 5bd2ab23f2f..f0d7f26c5f2 100644 --- a/crates/rpc/rpc-builder/src/metrics.rs +++ b/crates/rpc/rpc-builder/src/metrics.rs @@ -132,7 +132,7 @@ where for batch_entry in req.iter().flatten() { let method_name = batch_entry.method_name(); if let Some(call_metrics) = self.metrics.inner.call_metrics.get(method_name) { - call_metrics.started_total.increment(1); + call_metrics.batched_total.increment(1); } } @@ -299,6 +299,12 @@ struct RpcServerCallMetrics { successful_total: Counter, /// The number of failed calls failed_total: Counter, + /// The number of calls received as batch entries. + /// + /// jsonrpsee dispatches batch entries internally without invoking this middleware's `call`, + /// so only their per-method volume can be tracked; batched calls are excluded from + /// `started_total`, `successful_total`, `failed_total` and `time_seconds`. + batched_total: Counter, /// Response for a single call time_seconds: Histogram, } From 96512668ffd875c381bfe075643b7dc34e7972df Mon Sep 17 00:00:00 2001 From: Brian Picciano Date: Fri, 28 Aug 2026 18:04:28 +0000 Subject: [PATCH 094/294] fix(overlay): preserve storage wipes in execution overlays (#26872) --- crates/storage/storage-overlay/src/builder.rs | 62 +++++++++++++++---- .../storage/storage-overlay/src/provider.rs | 8 +-- 2 files changed, 53 insertions(+), 17 deletions(-) diff --git a/crates/storage/storage-overlay/src/builder.rs b/crates/storage/storage-overlay/src/builder.rs index eeb93a3be62..403b6800dff 100644 --- a/crates/storage/storage-overlay/src/builder.rs +++ b/crates/storage/storage-overlay/src/builder.rs @@ -1,8 +1,8 @@ use crate::OverlayManager; use alloy_eips::BlockNumHash; use alloy_primitives::{ - map::{AddressMap, B256Map, U256Map}, - BlockHash, BlockNumber, B256, U256, + map::{AddressMap, AddressSet, B256Map, U256Map}, + Address, BlockHash, BlockNumber, B256, U256, }; use metrics::{Counter, Histogram}; use reth_chain_state::ExecutedBlock; @@ -63,11 +63,6 @@ impl StateTrieOverlay { /// /// Account entries preserve known non-existence, while storage and code entries contain only data /// explicitly observed during execution. Accounts never retain database-context-local lookup IDs. -/// This intentionally does not retain account status or -/// known-storage wipes, so it cannot represent pre-Dencun `SELFDESTRUCT` of existing accounts. -/// The execution overlay is used only for post-Dencun engine execution, where that operation does -/// not clear existing storage; pipeline sync does not use it. Extend this representation before -/// using it for pre-Dencun execution. #[derive(Clone, Debug, Default)] pub struct ExecutionOverlay { /// In-memory block hashes in ascending block-number order. @@ -76,6 +71,11 @@ pub struct ExecutionOverlay { accounts: AddressMap>, /// Storage values by address and slot. storage: AddressMap>, + /// Accounts whose storage was wiped during execution. + /// + /// An absent slot for one of these accounts is known to be zero, rather than falling back to + /// the durable database state. + storage_wipes: AddressSet, /// Bytecode by code hash. code_hashes: B256Map, } @@ -96,6 +96,16 @@ impl ExecutionOverlay { &self.storage } + /// Returns an explicitly observed storage value, or zero when the account's storage was + /// wiped. + pub(crate) fn storage_value(&self, address: Address, slot: U256) -> Option { + self.storage + .get(&address) + .and_then(|storage| storage.get(&slot)) + .copied() + .or_else(|| self.storage_wipes.contains(&address).then_some(U256::ZERO)) + } + /// Returns the bytecode by code hash. pub const fn code_hashes(&self) -> &B256Map { &self.code_hashes @@ -133,13 +143,17 @@ impl ExecutionOverlay { /// and cannot be reused by the overlay's database context. All other account fields are /// preserved. fn extend_state(&mut self, state: &BundleState) { - let (accounts, storage, code_hashes) = - (&mut self.accounts, &mut self.storage, &mut self.code_hashes); + let (accounts, storage, storage_wipes, code_hashes) = + (&mut self.accounts, &mut self.storage, &mut self.storage_wipes, &mut self.code_hashes); #[allow(unused_mut)] let mut extend_accounts_and_storage = || { for (address, account) in state.state() { accounts.insert(*address, Self::normalized_account_info(account.info.clone())); + if account.was_destroyed() { + storage_wipes.insert(*address); + storage.remove(address); + } let account_storage = storage.entry(*address).or_default(); for (slot, value) in &account.storage { account_storage.insert(*slot, value.present_value); @@ -173,12 +187,16 @@ impl ExecutionOverlay { .iter() .map(|(address, info)| (*address, Self::normalized_account_info(info.clone()))), ); + for address in &other.storage_wipes { + self.storage.remove(address); + } for (address, slots) in &other.storage { self.storage .entry(*address) .or_default() .extend(slots.iter().map(|(slot, value)| (*slot, *value))); } + self.storage_wipes.extend(other.storage_wipes.iter().copied()); self.code_hashes.extend(other.code_hashes.iter().map(|(hash, code)| (*hash, code.clone()))); } @@ -917,7 +935,7 @@ mod tests { use reth_trie::{BranchNodeCompact, ComputedTrieData, HashedPostState, HashedStorage, Nibbles}; use revm::{ bytecode::Bytecode, - database::BundleState, + database::{AccountStatus, BundleAccount, BundleState}, state::{AccountId, AccountInfo}, }; @@ -1043,6 +1061,26 @@ mod tests { assert_eq!(overlay.code_hashes[&code_hash], code); } + #[test] + fn execution_overlay_zeroes_unobserved_storage_for_destroyed_accounts() { + let address = Address::with_last_byte(1); + let mut state = BundleState::default(); + state.state.insert( + address, + BundleAccount::new( + Some(AccountInfo::default()), + None, + Default::default(), + AccountStatus::Destroyed, + ), + ); + + let mut overlay = ExecutionOverlay::default(); + overlay.extend_state(&state); + + assert_eq!(overlay.storage_value(address, U256::ZERO), Some(U256::ZERO)); + } + #[test] fn execution_overlay_composition_uses_later_values_and_normalizes_accounts() { let address = Address::with_last_byte(1); @@ -1072,6 +1110,7 @@ mod tests { Some(AccountInfo { nonce: 11, account_id: AccountId::new(12), ..Default::default() }), ); later.storage.entry(address).or_default().insert(slot, U256::from(13)); + later.storage_wipes.insert(address); later.code_hashes.insert(later_code_hash, Bytecode::new_raw(vec![2].into())); overlay.extend_overlay(&later); @@ -1082,7 +1121,8 @@ mod tests { assert_eq!(overlay.accounts[&address].as_ref().unwrap().account_id, None); assert!(overlay.accounts.contains_key(&retained_address)); assert_eq!(overlay.storage[&address][&slot], U256::from(13)); - assert_eq!(overlay.storage[&address][&retained_slot], U256::from(10)); + assert!(!overlay.storage[&address].contains_key(&retained_slot)); + assert_eq!(overlay.storage_value(address, U256::from(14)), Some(U256::ZERO)); assert!(overlay.code_hashes.contains_key(&first_code_hash)); assert!(overlay.code_hashes.contains_key(&later_code_hash)); } diff --git a/crates/storage/storage-overlay/src/provider.rs b/crates/storage/storage-overlay/src/provider.rs index f1294c37e5c..428aa735197 100644 --- a/crates/storage/storage-overlay/src/provider.rs +++ b/crates/storage/storage-overlay/src/provider.rs @@ -712,12 +712,8 @@ where storage_key: alloy_primitives::StorageKey, ) -> ProviderResult> { let overlay = self.execution_overlay()?; - if let Some(value) = overlay - .storage() - .get(&address) - .and_then(|storage| storage.get(&U256::from_be_bytes(storage_key.0))) - { - return Ok(Some(*value)); + if let Some(value) = overlay.storage_value(address, U256::from_be_bytes(storage_key.0)) { + return Ok(Some(value)); } if self.provider().cached_storage_settings().use_hashed_state() { let hashed_address = alloy_primitives::keccak256(address); From 3bc71d43f7101f772bbb4f9e15d3cdd58f60e958 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Fri, 28 Aug 2026 22:48:30 +0000 Subject: [PATCH 095/294] test: add post-Cancun selfdestruct e2e suite across hardforks (#26865) --- Cargo.lock | 2 + crates/e2e-test-utils/Cargo.toml | 2 + crates/e2e-test-utils/src/lib.rs | 28 +- crates/e2e-test-utils/src/trie.rs | 96 +++ crates/ethereum/node/tests/e2e/eip6780.rs | 705 ++++++++++++++++++++++ crates/ethereum/node/tests/e2e/main.rs | 1 + crates/ethereum/node/tests/e2e/utils.rs | 41 +- 7 files changed, 839 insertions(+), 36 deletions(-) create mode 100644 crates/e2e-test-utils/src/trie.rs create mode 100644 crates/ethereum/node/tests/e2e/eip6780.rs diff --git a/Cargo.lock b/Cargo.lock index 1ce04d65684..089af079131 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8338,6 +8338,8 @@ dependencies = [ "reth-tasks", "reth-tokio-util", "reth-tracing", + "reth-trie", + "reth-trie-db", "revm", "serde_json", "tempfile", diff --git a/crates/e2e-test-utils/Cargo.toml b/crates/e2e-test-utils/Cargo.toml index 3998a54d937..b337377f522 100644 --- a/crates/e2e-test-utils/Cargo.toml +++ b/crates/e2e-test-utils/Cargo.toml @@ -40,6 +40,8 @@ reth-config.workspace = true reth-consensus.workspace = true reth-db-common.workspace = true reth-primitives-traits.workspace = true +reth-trie.workspace = true +reth-trie-db.workspace = true revm.workspace = true tempfile.workspace = true diff --git a/crates/e2e-test-utils/src/lib.rs b/crates/e2e-test-utils/src/lib.rs index 919df05321b..4507c295188 100644 --- a/crates/e2e-test-utils/src/lib.rs +++ b/crates/e2e-test-utils/src/lib.rs @@ -1,8 +1,9 @@ //! Utilities for end-to-end tests. +use alloy_primitives::{Address, B256}; use alloy_rpc_types_engine::PayloadAttributes; use node::NodeTestContext; -use reth_chainspec::ChainSpec; +use reth_chainspec::{ChainSpec, EthereumHardfork}; use reth_db::{test_utils::TempDatabase, DatabaseEnv}; use reth_network_api::test_utils::PeersHandleProvider; use reth_node_builder::{ @@ -40,6 +41,9 @@ mod rpc; /// Utilities for creating and writing RLP test data pub mod test_rlp_utils; +/// Helpers for verifying the persisted state and trie representation +pub mod trie; + /// Builder for configuring test node setups mod setup_builder; pub use setup_builder::E2ETestSetupBuilder; @@ -123,6 +127,28 @@ where .await } +/// Creates Ethereum [`PayloadAttributes`] valid for the given hardfork, allowing the same test +/// to be run against multiple hardfork targets. +/// +/// Withdrawals are set once Shanghai is active, the parent beacon block root once Cancun is +/// active, and a slot number once Amsterdam is active. The payload builder requires a slot number +/// for EIP-7843; tests use the timestamp as a deterministic dummy slot because the exact beacon +/// slot is irrelevant for local e2e payloads. +pub fn eth_payload_attributes_for_fork( + fork: EthereumHardfork, + timestamp: u64, +) -> PayloadAttributes { + PayloadAttributes { + timestamp, + prev_randao: B256::ZERO, + suggested_fee_recipient: Address::ZERO, + withdrawals: (fork >= EthereumHardfork::Shanghai).then(Vec::new), + parent_beacon_block_root: (fork >= EthereumHardfork::Cancun).then_some(B256::ZERO), + slot_number: (fork >= EthereumHardfork::Amsterdam).then_some(timestamp), + ..Default::default() + } +} + // Type aliases /// Testing database diff --git a/crates/e2e-test-utils/src/trie.rs b/crates/e2e-test-utils/src/trie.rs new file mode 100644 index 00000000000..5e091cf76e0 --- /dev/null +++ b/crates/e2e-test-utils/src/trie.rs @@ -0,0 +1,96 @@ +//! Helpers for verifying the persisted state and trie representation of test nodes. + +use alloy_consensus::BlockHeader; +use eyre::{ensure, eyre, Result}; +use reth_provider::{ + BlockNumReader, DBProvider, DatabaseProviderFactory, HeaderProvider, StorageSettingsCache, +}; +use reth_trie::{ + prefix_set::{PrefixSetMut, TriePrefixSets}, + verify::{Output, Verifier}, + StateRoot, +}; +use reth_trie_db::{DatabaseHashedCursorFactory, DatabaseTrieCursorFactory}; +use std::time::{Duration, Instant}; + +/// Waits until the node has persisted at least the given block number to disk. +/// +/// The engine keeps the most recent blocks in memory, so tests that want to inspect the +/// database, e.g. via [`assert_trie_consistency`], must first advance the chain far enough and +/// wait for the persistence task to catch up. Unlike `NodeTestContext::wait_block` this only +/// needs the block number and fails after `timeout` instead of polling indefinitely. +pub async fn wait_for_persisted_block(factory: &F, number: u64, timeout: Duration) -> Result<()> +where + F: DatabaseProviderFactory, +{ + let start = Instant::now(); + loop { + // the finish checkpoint is committed in the same transaction as the block's state + if factory.database_provider_ro()?.best_block_number()? >= number { + return Ok(()) + } + ensure!(start.elapsed() < timeout, "timed out waiting for block {number} to be persisted"); + tokio::time::sleep(Duration::from_millis(50)).await; + } +} + +/// Asserts that the persisted state and trie representation is internally consistent and matches +/// the state root of the latest persisted block. +/// +/// This performs two checks against a single database snapshot: +/// - the state root recomputed from the persisted hashed state matches the state root of the latest +/// persisted header, and +/// - the persisted trie nodes match a recomputation from the hashed state (the library equivalent +/// of a `reth db repair-trie --dry-run`). +/// +/// Only on-disk data is read, so callers must ensure the blocks of interest have been persisted, +/// e.g. via [`wait_for_persisted_block`]. +pub fn assert_trie_consistency(factory: &F) -> Result<()> +where + F: DatabaseProviderFactory< + Provider: DBProvider + StorageSettingsCache + BlockNumReader + HeaderProvider, + >, +{ + let provider = factory.database_provider_ro()?; + let tip = provider.best_block_number()?; + let header = provider + .header_by_number(tip)? + .ok_or_else(|| eyre!("missing persisted header for block {tip}"))?; + let tx = provider.tx_ref(); + + reth_trie_db::with_adapter!(provider, |A| { + // Recompute all account leaves from the persisted hashed state. Storage roots are taken + // from the persisted storage tries, which the verifier below checks against the hashed + // storage tables. + let recomputed = StateRoot::new( + DatabaseTrieCursorFactory::<_, A>::new(tx), + DatabaseHashedCursorFactory::new(tx), + ) + .with_prefix_sets(TriePrefixSets { + account_prefix_set: PrefixSetMut::all().freeze(), + ..Default::default() + }) + .root()?; + ensure!( + recomputed == header.state_root(), + "state root {recomputed} recomputed from the persisted state does not match the state root of persisted block {tip}: {}", + header.state_root() + ); + + let trie_cursor_factory = DatabaseTrieCursorFactory::<_, A>::new(tx); + let verifier = Verifier::new(&trie_cursor_factory, DatabaseHashedCursorFactory::new(tx))?; + let mut inconsistencies = Vec::new(); + for output in verifier { + match output? { + Output::Progress(_) => {} + inconsistency => inconsistencies.push(inconsistency), + } + } + ensure!( + inconsistencies.is_empty(), + "persisted trie is inconsistent with the persisted hashed state: {inconsistencies:?}" + ); + + Ok(()) + }) +} diff --git a/crates/ethereum/node/tests/e2e/eip6780.rs b/crates/ethereum/node/tests/e2e/eip6780.rs new file mode 100644 index 00000000000..c68e15c7a49 --- /dev/null +++ b/crates/ethereum/node/tests/e2e/eip6780.rs @@ -0,0 +1,705 @@ +//! Post-Cancun SELFDESTRUCT (EIP-6780) scenario suite driven through the engine API. +//! +//! The same scenario battery runs against multiple hardfork targets (Cancun, Osaka, Amsterdam) +//! on a single node per fork: every block goes through payload building, `newPayload` and +//! forkchoice via the consensus engine, and assertions cover receipts, RPC state and the +//! committed execution outcome. At the end of a suite the persisted database is checked with +//! [`assert_trie_consistency`], which recomputes the state root from the on-disk hashed state +//! and verifies the stored trie nodes against it, so incorrect persistence of selfdestructed +//! accounts (such as the prefunded `CREATE2` divergence introduced by +//! ) is caught even when the live, in-memory state +//! root is correct. +//! +//! The scenarios cover the EIP-6780 equivalence classes that affect reth's state +//! representation: +//! - prefunded `CREATE2` target destroyed in its creation transaction (prior block and same block +//! prefunding), +//! - creation and destruction in the same transaction, both directly from initcode and via a call +//! after the contract returned its runtime code, +//! - creation and destruction in different transactions of the same block (must be treated as +//! pre-existing), +//! - selfdestruct to self for pre-existing (balance kept) and same-transaction created (balance +//! burned, or kept as a balance-only account once EIP-8246 is active) contracts, and to a +//! contract beneficiary (code must not run), +//! - a reverted nested selfdestruct while the outer frame commits, +//! - destroy/recreate cycles at the same `CREATE2` address across transactions and blocks, +//! including the collision of a recreate attempt with a persisting destroyed contract. + +use alloy_network::{EthereumWallet, TransactionBuilder}; +use alloy_primitives::{Address, Bytes, TxKind, B256, U256}; +use alloy_provider::{Provider, ProviderBuilder}; +use alloy_rpc_types_eth::{TransactionReceipt, TransactionRequest}; +use futures::StreamExt; +use reth_chainspec::{ChainSpec, ChainSpecBuilder, EthereumHardfork, MAINNET}; +use reth_e2e_test_utils::{ + eth_payload_attributes_for_fork, setup_engine, + trie::{assert_trie_consistency, wait_for_persisted_block}, + NodeHelperType, +}; +use reth_node_api::TreeConfig; +use reth_node_ethereum::EthereumNode; +use reth_provider::Chain; +use reth_revm::db::BundleAccount; +use std::{sync::Arc, time::Duration}; + +const MAX_FEE_PER_GAS: u128 = 20_000_000_000; +const MAX_PRIORITY_FEE_PER_GAS: u128 = 1_000_000_000; + +const ETH: u128 = 1_000_000_000_000_000_000; + +#[tokio::test] +async fn test_eip6780_selfdestruct_cancun() -> eyre::Result<()> { + run_selfdestruct_suite(EthereumHardfork::Cancun).await +} + +#[tokio::test] +async fn test_eip6780_selfdestruct_osaka() -> eyre::Result<()> { + run_selfdestruct_suite(EthereumHardfork::Osaka).await +} + +#[tokio::test] +async fn test_eip6780_selfdestruct_amsterdam() -> eyre::Result<()> { + run_selfdestruct_suite(EthereumHardfork::Amsterdam).await +} + +/// Runs all selfdestruct scenarios back to back against a single node with the given hardfork +/// active at genesis, then verifies the persisted trie representation. +async fn run_selfdestruct_suite(fork: EthereumHardfork) -> eyre::Result<()> { + reth_tracing::init_test_tracing(); + + // low persistence thresholds so the scenario blocks reach the database and the persisted + // trie representation can be verified at the end of the suite + let tree_config = + TreeConfig::default().with_persistence_threshold(2).with_memory_block_buffer_target(1); + let (mut nodes, wallet) = + setup_engine::(1, fork_spec(fork), false, tree_config, move |timestamp| { + eth_payload_attributes_for_fork(fork, timestamp) + }) + .await?; + let node = nodes.pop().unwrap(); + let signer = wallet.inner.clone(); + let provider = ProviderBuilder::new() + .wallet(EthereumWallet::new(signer.clone())) + .connect_http(node.rpc_url()); + + let mut ctx = SuiteCtx { + node, + provider, + fork, + signer: signer.address(), + nonce: 0, + factory: Address::ZERO, + last_committed: None, + }; + + let fixtures = deploy_fixtures(&mut ctx).await?; + + prefunded_create2_selfdestruct(&mut ctx).await?; + same_block_prefunded_create2_selfdestruct(&mut ctx, &fixtures).await?; + create_then_destroy_in_same_tx(&mut ctx).await?; + create_and_destroy_in_different_txs_of_same_block(&mut ctx).await?; + selfdestruct_to_self(&mut ctx, &fixtures).await?; + reverted_selfdestruct(&mut ctx, &fixtures).await?; + destroy_and_recreate(&mut ctx).await?; + destroy_and_recreate_in_same_block(&mut ctx).await?; + + // leave the scenario blocks behind the in-memory window so they are persisted, then verify + // the on-disk state and trie representation against the persisted tip + let scenario_tip = ctx.provider.get_block_number().await?; + for _ in 0..4 { + ctx.mine_block(vec![]).await?; + } + wait_for_persisted_block(&ctx.node.inner.provider, scenario_tip, Duration::from_secs(30)) + .await?; + assert_trie_consistency(&ctx.node.inner.provider)?; + + Ok(()) +} + +/// Test context that drives a single node and tracks the signer nonce across scenarios. +struct SuiteCtx

{ + node: NodeHelperType, + provider: P, + fork: EthereumHardfork, + signer: Address, + nonce: u64, + /// The mode-switched `CREATE2` factory all scenarios deploy through. + factory: Address, + /// The chain segment committed by the most recently mined block. + last_committed: Option>, +} + +impl SuiteCtx

{ + /// Sends the given transactions from the signer and mines a block containing exactly them. + /// + /// Nonces and fees are filled in automatically in transaction order. + async fn mine_block( + &mut self, + txs: Vec, + ) -> eyre::Result> { + let expected = txs.len(); + let mut pending = Vec::with_capacity(expected); + for tx in txs { + let nonce = self.nonce; + self.nonce += 1; + let tx = tx + .with_from(self.signer) + .with_nonce(nonce) + .with_max_fee_per_gas(MAX_FEE_PER_GAS) + .with_max_priority_fee_per_gas(MAX_PRIORITY_FEE_PER_GAS); + pending.push(self.provider.send_transaction(tx).await?); + } + + let payload = self.node.advance_block().await?; + let included = payload.block().body().transactions().count(); + assert_eq!(included, expected, "block should contain exactly the sent transactions"); + + let notification = self + .node + .canonical_stream + .next() + .await + .ok_or_else(|| eyre::eyre!("canonical stream ended"))?; + self.last_committed = Some(notification.committed()); + + let mut receipts = Vec::with_capacity(expected); + for tx in pending { + receipts.push(tx.get_receipt().await?); + } + for (index, receipt) in receipts.iter().enumerate() { + assert!( + receipt.status(), + "transaction {index} in block {:?} reverted: gas used {}", + receipt.block_number, + receipt.gas_used, + ); + } + Ok(receipts) + } + + /// Returns the account entry of the most recently committed block's bundle state. + fn bundle_account(&self, address: Address) -> Option<&BundleAccount> { + self.last_committed.as_ref().unwrap().execution_outcome().bundle.account(&address) + } + + async fn balance(&self, address: Address) -> eyre::Result { + Ok(self.provider.get_balance(address).await?) + } + + async fn slot0(&self, address: Address) -> eyre::Result { + Ok(self.provider.get_storage_at(address, U256::ZERO).await?) + } + + /// Asserts that the account is fully absent: no balance, code, nonce or storage. + async fn assert_destroyed(&self, address: Address, scenario: &str) -> eyre::Result<()> { + assert_eq!(self.balance(address).await?, U256::ZERO, "{scenario}: balance not cleared"); + assert!( + self.provider.get_code_at(address).await?.is_empty(), + "{scenario}: code not cleared" + ); + assert_eq!( + self.provider.get_transaction_count(address).await?, + 0, + "{scenario}: nonce not cleared" + ); + assert_eq!(self.slot0(address).await?, U256::ZERO, "{scenario}: storage not cleared"); + Ok(()) + } + + /// Asserts that a live contract has the expected runtime code, slot 0 value and balance. + async fn assert_contract( + &self, + address: Address, + code: &Bytes, + slot0: U256, + balance: U256, + scenario: &str, + ) -> eyre::Result<()> { + assert_eq!(&self.provider.get_code_at(address).await?, code, "{scenario}: wrong code"); + assert_eq!(self.slot0(address).await?, slot0, "{scenario}: wrong storage"); + assert_eq!(self.balance(address).await?, balance, "{scenario}: wrong balance"); + Ok(()) + } + + /// Predicts the address the factory deploys the given initcode to. + fn create2_address(&self, salt: B256, initcode: &Bytes) -> Address { + self.factory.create2_from_code(salt, initcode) + } +} + +/// Contracts shared between scenarios, deployed in the first block of the suite. +struct Fixtures { + /// Pre-existing contract whose runtime selfdestructs to itself, funded with 1 ETH. + self_destructing_to_self: Address, + /// Pre-existing contract whose runtime selfdestructs to `beneficiary(7)`, funded with 1 ETH. + revert_child: Address, + /// Contract that calls `revert_child` and then stores to prove its own frame committed, + /// swallowing the revert of the intermediate [`call_then_revert_runtime`] contract. + /// + /// Also serves as the contract beneficiary in `same_block_prefunded_create2_selfdestruct`, + /// which asserts that its slot 0 is still zero and therefore must run before + /// `reverted_selfdestruct` permanently sets it. + revert_wrapper: Address, +} + +/// Deploys the `CREATE2` factory and the fixture contracts for the pre-existing and revert +/// scenarios. +async fn deploy_fixtures(ctx: &mut SuiteCtx

) -> eyre::Result { + let base_nonce = ctx.nonce; + ctx.factory = ctx.signer.create(base_nonce); + let self_destructing_to_self = ctx.signer.create(base_nonce + 1); + let revert_child = ctx.signer.create(base_nonce + 2); + let reverter = ctx.signer.create(base_nonce + 3); + let revert_wrapper = ctx.signer.create(base_nonce + 4); + + let receipts = ctx + .mine_block(vec![ + create_tx(deploying_initcode(&factory_runtime()), U256::ZERO), + create_tx(deploying_initcode(&selfdestruct_to_self_runtime()), eth_tenths(10)), + create_tx(deploying_initcode(&selfdestruct_runtime(beneficiary(7))), eth_tenths(10)), + create_tx(deploying_initcode(&call_then_revert_runtime(revert_child)), U256::ZERO), + create_tx(deploying_initcode(&call_then_store_runtime(reverter)), U256::ZERO), + ]) + .await?; + assert_eq!(receipts[0].contract_address, Some(ctx.factory)); + + Ok(Fixtures { self_destructing_to_self, revert_child, revert_wrapper }) +} + +/// The incident reproducer: an address is prefunded in an earlier block, then a +/// contract is `CREATE2`-deployed there whose initcode selfdestructs. The account had prior +/// state, so its deletion must be represented correctly all the way into the persisted trie. +async fn prefunded_create2_selfdestruct(ctx: &mut SuiteCtx

) -> eyre::Result<()> { + let initcode = selfdestruct_initcode(beneficiary(1)); + let destroyed = ctx.create2_address(salt(1), &initcode); + + let receipts = ctx.mine_block(vec![transfer_tx(destroyed, eth_tenths(10))]).await?; + assert_eq!(ctx.balance(destroyed).await?, eth_tenths(10)); + // EIP-7708: plain transfers emit a transfer log once Amsterdam is active + assert_eq!( + !receipts[0].inner.logs().is_empty(), + ctx.fork >= EthereumHardfork::Amsterdam, + "transfer log emission must match EIP-7708 activation" + ); + + ctx.mine_block(vec![factory_create_tx(ctx.factory, false, salt(1), &initcode, eth_tenths(5))]) + .await?; + + assert_eq!(ctx.slot0(ctx.factory).await?, address_word(destroyed), "creation should succeed"); + ctx.assert_destroyed(destroyed, "prefunded create2").await?; + assert_eq!(ctx.balance(beneficiary(1)).await?, eth_tenths(15)); + // the account existed in the database before this block, so the bundle must mark it as + // destroyed with no post-block info + let account = ctx.bundle_account(destroyed).expect("prefunded account must be in the bundle"); + assert!(account.was_destroyed(), "prefunded create2: bundle must mark account destroyed"); + assert!(account.info.is_none(), "prefunded create2: destroyed account must have no info"); + + // touching the address again must start from a clean account + ctx.mine_block(vec![transfer_tx(destroyed, eth_tenths(3))]).await?; + assert_eq!(ctx.balance(destroyed).await?, eth_tenths(3), "old balance must not resurface"); + assert!(ctx.provider.get_code_at(destroyed).await?.is_empty()); + + Ok(()) +} + +/// Same as [`prefunded_create2_selfdestruct`], but the prefunding happens in an earlier +/// transaction of the same block, and the selfdestruct beneficiary is a contract whose code +/// must not be executed by the balance transfer. +async fn same_block_prefunded_create2_selfdestruct( + ctx: &mut SuiteCtx

, + fixtures: &Fixtures, +) -> eyre::Result<()> { + let contract_beneficiary = fixtures.revert_wrapper; + let initcode = selfdestruct_initcode(contract_beneficiary); + let destroyed = ctx.create2_address(salt(2), &initcode); + + ctx.mine_block(vec![ + transfer_tx(destroyed, eth_tenths(10)), + factory_create_tx(ctx.factory, false, salt(2), &initcode, eth_tenths(3)), + ]) + .await?; + + assert_eq!(ctx.slot0(ctx.factory).await?, address_word(destroyed), "creation should succeed"); + ctx.assert_destroyed(destroyed, "same-block prefunded create2").await?; + // the beneficiary contract receives the funds without its code running + assert_eq!(ctx.balance(contract_beneficiary).await?, eth_tenths(13)); + assert_eq!(ctx.slot0(contract_beneficiary).await?, U256::ZERO, "beneficiary code must not run"); + + Ok(()) +} + +/// A contract is `CREATE2`-deployed, returns its runtime code and is then called to +/// selfdestruct later in the same transaction: it still counts as created in this transaction +/// and must be deleted. +async fn create_then_destroy_in_same_tx(ctx: &mut SuiteCtx

) -> eyre::Result<()> { + let initcode = deploying_initcode(&selfdestruct_runtime(beneficiary(3))); + let destroyed = ctx.create2_address(salt(3), &initcode); + + ctx.mine_block(vec![factory_create_tx(ctx.factory, true, salt(3), &initcode, eth_tenths(2))]) + .await?; + + assert_eq!(ctx.slot0(ctx.factory).await?, address_word(destroyed), "creation should succeed"); + ctx.assert_destroyed(destroyed, "create then destroy in same tx").await?; + assert_eq!(ctx.balance(beneficiary(3)).await?, eth_tenths(2)); + + Ok(()) +} + +/// A contract created in an earlier transaction of the block selfdestructs in a later +/// transaction of the same block: it must be treated as pre-existing, transferring only the +/// balance while account, code and storage persist. A repeated selfdestruct with zero balance +/// in the next block must leave it untouched as well. +async fn create_and_destroy_in_different_txs_of_same_block( + ctx: &mut SuiteCtx

, +) -> eyre::Result<()> { + let runtime = selfdestruct_runtime(beneficiary(4)); + let initcode = deploying_initcode(&runtime); + let contract = ctx.create2_address(salt(4), &initcode); + + ctx.mine_block(vec![ + factory_create_tx(ctx.factory, false, salt(4), &initcode, eth_tenths(2)), + call_tx(contract), + ]) + .await?; + + ctx.assert_contract(contract, &runtime, eth_tenths(2), U256::ZERO, "same-block create").await?; + assert_eq!(ctx.provider.get_transaction_count(contract).await?, 1, "nonce must persist"); + assert_eq!(ctx.balance(beneficiary(4)).await?, eth_tenths(2)); + let account = ctx.bundle_account(contract).expect("contract must be in the bundle"); + assert!(!account.was_destroyed(), "same-block create: account must not be destroyed"); + assert!(account.info.is_some(), "same-block create: account must persist"); + + // repeated selfdestruct of the now pre-existing contract with zero balance + ctx.mine_block(vec![call_tx(contract)]).await?; + ctx.assert_contract(contract, &runtime, eth_tenths(2), U256::ZERO, "repeated selfdestruct") + .await?; + assert_eq!(ctx.balance(beneficiary(4)).await?, eth_tenths(2), "no double transfer"); + + Ok(()) +} + +/// Selfdestruct with the beneficiary being the destructing account itself: a pre-existing +/// contract keeps its balance, while a contract created in the same transaction burns it. +async fn selfdestruct_to_self( + ctx: &mut SuiteCtx

, + fixtures: &Fixtures, +) -> eyre::Result<()> { + let initcode = selfdestruct_to_self_initcode(); + let destroyed = ctx.create2_address(salt(6), &initcode); + + ctx.mine_block(vec![ + call_tx(fixtures.self_destructing_to_self), + factory_create_tx(ctx.factory, false, salt(6), &initcode, eth_tenths(4)), + ]) + .await?; + + // the pre-existing contract keeps its funds: the transfer to itself is a no-op + ctx.assert_contract( + fixtures.self_destructing_to_self, + &selfdestruct_to_self_runtime(), + eth_tenths(10), + eth_tenths(10), + "pre-existing selfdestruct to self", + ) + .await?; + + assert_eq!(ctx.slot0(ctx.factory).await?, address_word(destroyed), "creation should succeed"); + if ctx.fork >= EthereumHardfork::Amsterdam { + // EIP-8246: instead of burning the funds, the account survives as a balance-only + // account without nonce, code or storage + ctx.assert_contract( + destroyed, + &Bytes::new(), + U256::ZERO, + eth_tenths(4), + "same-tx selfdestruct to self", + ) + .await?; + assert_eq!(ctx.provider.get_transaction_count(destroyed).await?, 0, "nonce must be reset"); + } else { + // the same-transaction created contract is deleted and its funds are burned + ctx.assert_destroyed(destroyed, "same-tx selfdestruct to self").await?; + } + + Ok(()) +} + +/// An inner frame selfdestructs a pre-existing contract, but the intermediate frame reverts +/// while the outer frame commits: the selfdestruct must be rolled back completely. +async fn reverted_selfdestruct( + ctx: &mut SuiteCtx

, + fixtures: &Fixtures, +) -> eyre::Result<()> { + ctx.mine_block(vec![call_tx(fixtures.revert_wrapper)]).await?; + + // the outer frame committed + assert_eq!( + ctx.slot0(fixtures.revert_wrapper).await?, + U256::ONE, + "reverted selfdestruct: outer frame must commit" + ); + // the reverted selfdestruct left the child completely untouched + ctx.assert_contract( + fixtures.revert_child, + &selfdestruct_runtime(beneficiary(7)), + eth_tenths(10), + eth_tenths(10), + "reverted selfdestruct", + ) + .await?; + assert_eq!(ctx.balance(beneficiary(7)).await?, U256::ZERO, "transfer must be reverted"); + if let Some(account) = ctx.bundle_account(fixtures.revert_child) { + assert!(!account.was_destroyed(), "reverted selfdestruct: account must not be destroyed"); + } + + Ok(()) +} + +/// Destroy/recreate cycle at the same `CREATE2` address across blocks: an address freed by a +/// same-transaction destruction can be recreated with fresh storage, the recreated contract is +/// pre-existing for a later selfdestruct, and afterwards the address keeps colliding with +/// creation attempts. +async fn destroy_and_recreate(ctx: &mut SuiteCtx

) -> eyre::Result<()> { + let runtime = selfdestruct_runtime(beneficiary(8)); + let initcode = deploying_initcode(&runtime); + let contract = ctx.create2_address(salt(8), &initcode); + let factory_balance_before = ctx.balance(ctx.factory).await?; + + // create and destroy in the same transaction to free the address again + ctx.mine_block(vec![factory_create_tx(ctx.factory, true, salt(8), &initcode, eth_tenths(1))]) + .await?; + ctx.assert_destroyed(contract, "destroy before recreate").await?; + + // recreate at the same address; the initcode stores the callvalue, so a value distinct + // from the first deployment proves the storage is fresh + ctx.mine_block(vec![factory_create_tx(ctx.factory, false, salt(8), &initcode, eth_tenths(2))]) + .await?; + ctx.assert_contract(contract, &runtime, eth_tenths(2), eth_tenths(2), "recreate").await?; + + // the recreated contract is pre-existing now: selfdestruct keeps code and storage + ctx.mine_block(vec![call_tx(contract)]).await?; + ctx.assert_contract(contract, &runtime, eth_tenths(2), U256::ZERO, "destroy recreated").await?; + assert_eq!(ctx.balance(beneficiary(8)).await?, eth_tenths(3)); + + // the destroyed but persisting contract keeps colliding with creation attempts; the failed + // `CREATE2` leaves the value with the factory + ctx.mine_block(vec![factory_create_tx(ctx.factory, false, salt(8), &initcode, eth_tenths(1))]) + .await?; + assert_eq!(ctx.slot0(ctx.factory).await?, U256::ZERO, "creation must collide"); + assert_eq!(ctx.balance(ctx.factory).await?, factory_balance_before + eth_tenths(1)); + ctx.assert_contract( + contract, + &runtime, + eth_tenths(2), + U256::ZERO, + "collision with destroyed contract", + ) + .await?; + + Ok(()) +} + +/// Destroy and recreate at the same `CREATE2` address within one block: the first transaction +/// creates and destroys the contract, the second recreates it at the now free address. +async fn destroy_and_recreate_in_same_block( + ctx: &mut SuiteCtx

, +) -> eyre::Result<()> { + let runtime = selfdestruct_runtime(beneficiary(9)); + let initcode = deploying_initcode(&runtime); + let contract = ctx.create2_address(salt(9), &initcode); + + ctx.mine_block(vec![ + factory_create_tx(ctx.factory, true, salt(9), &initcode, eth_tenths(1)), + factory_create_tx(ctx.factory, false, salt(9), &initcode, eth_tenths(3)), + ]) + .await?; + + assert_eq!(ctx.slot0(ctx.factory).await?, address_word(contract), "recreation should succeed"); + ctx.assert_contract( + contract, + &runtime, + eth_tenths(3), + eth_tenths(3), + "same-block destroy and recreate", + ) + .await?; + assert_eq!(ctx.balance(beneficiary(9)).await?, eth_tenths(1)); + + Ok(()) +} + +// Transaction builders +// +// All transactions use a generous fixed gas limit: Amsterdam repricing raises costs well above +// the historic values (a plain transfer no longer fits into 21k gas), and gas estimation cannot +// be used because some transactions depend on contracts created earlier in the same block. + +const TX_GAS_LIMIT: u64 = 1_000_000; + +fn create_tx(initcode: Bytes, value: U256) -> TransactionRequest { + TransactionRequest::default() + .with_kind(TxKind::Create) + .with_input(initcode) + .with_value(value) + .with_gas_limit(TX_GAS_LIMIT) +} + +fn call_tx(to: Address) -> TransactionRequest { + TransactionRequest::default().with_to(to).with_gas_limit(TX_GAS_LIMIT) +} + +fn transfer_tx(to: Address, value: U256) -> TransactionRequest { + TransactionRequest::default().with_to(to).with_value(value).with_gas_limit(TX_GAS_LIMIT) +} + +/// Calls the factory to `CREATE2`-deploy the given initcode, optionally calling the deployed +/// contract afterwards in the same transaction. +fn factory_create_tx( + factory: Address, + call_after_create: bool, + salt: B256, + initcode: &Bytes, + value: U256, +) -> TransactionRequest { + let mut input = Vec::with_capacity(64 + initcode.len()); + input.extend_from_slice(&U256::from(call_after_create as u8).to_be_bytes::<32>()); + input.extend_from_slice(salt.as_slice()); + input.extend_from_slice(initcode); + TransactionRequest::default() + .with_to(factory) + .with_input(input) + .with_value(value) + .with_gas_limit(TX_GAS_LIMIT) +} + +// Scenario parameters and small conversion helpers + +fn eth_tenths(tenths: u128) -> U256 { + U256::from(tenths * ETH / 10) +} + +const fn salt(id: u8) -> B256 { + B256::with_last_byte(id) +} + +/// Returns a distinct EOA beneficiary address per scenario. +const fn beneficiary(id: u8) -> Address { + let mut bytes = [0xbe; 20]; + bytes[19] = id; + Address::new(bytes) +} + +/// The address as it appears in a storage slot written by the factory. +fn address_word(address: Address) -> U256 { + U256::from_be_slice(address.as_slice()) +} + +// Contract bytecode, hand-assembled to keep the scenarios self-contained + +/// Initcode that stores a marker at slot 0 and selfdestructs to `target` during initialization. +fn selfdestruct_initcode(target: Address) -> Bytes { + let mut code = vec![0x60, 0x42, 0x60, 0x00, 0x55]; // PUSH1 0x42 PUSH1 0 SSTORE + code.push(0x73); // PUSH20 + code.extend_from_slice(target.as_slice()); + code.push(0xff); // SELFDESTRUCT + code.into() +} + +/// Initcode that stores a marker at slot 0 and selfdestructs to its own address during +/// initialization. +const fn selfdestruct_to_self_initcode() -> Bytes { + // PUSH1 0x42 PUSH1 0 SSTORE ADDRESS SELFDESTRUCT + Bytes::from_static(&[0x60, 0x42, 0x60, 0x00, 0x55, 0x30, 0xff]) +} + +/// Runtime code that selfdestructs to `target` on any call. +fn selfdestruct_runtime(target: Address) -> Bytes { + let mut code = vec![0x73]; // PUSH20 + code.extend_from_slice(target.as_slice()); + code.push(0xff); // SELFDESTRUCT + code.into() +} + +/// Runtime code that selfdestructs to its own address on any call. +const fn selfdestruct_to_self_runtime() -> Bytes { + // ADDRESS SELFDESTRUCT + Bytes::from_static(&[0x30, 0xff]) +} + +/// Initcode that stores the callvalue at slot 0 and returns the given runtime code. +/// +/// Storing the callvalue makes redeployments distinguishable: the same initcode deployed with a +/// different value must yield different storage, proving that no stale storage survived. +fn deploying_initcode(runtime: &Bytes) -> Bytes { + // runtime starts after the 16 initcode bytes below + let offset = 0x10_u8; + let len = u8::try_from(runtime.len()).unwrap(); + let mut code = vec![ + 0x34, 0x60, 0x00, 0x55, // CALLVALUE PUSH1 0 SSTORE + 0x60, len, 0x60, offset, 0x60, 0x00, 0x39, // PUSH1 len PUSH1 offset PUSH1 0 CODECOPY + 0x60, len, 0x60, 0x00, 0xf3, // PUSH1 len PUSH1 0 RETURN + ]; + code.extend_from_slice(runtime); + code.into() +} + +/// Runtime of the `CREATE2` factory. +/// +/// Calldata layout: `[mode word | salt | initcode]`. The factory forwards its callvalue to +/// `CREATE2` and stores the resulting address (zero on failure) at slot 0. A non-zero mode word +/// additionally calls the deployed contract with empty calldata, which lets a single +/// transaction create a contract and trigger its selfdestruct after initialization. +const fn factory_runtime() -> Bytes { + const CODE: [u8; 44] = [ + 0x60, 0x40, 0x36, 0x03, // PUSH1 0x40 CALLDATASIZE SUB -> initcode length + 0x80, 0x60, 0x40, 0x60, 0x00, 0x37, // DUP1 PUSH1 0x40 PUSH1 0 CALLDATACOPY + 0x60, 0x20, 0x35, 0x90, // PUSH1 0x20 CALLDATALOAD SWAP1 -> salt below length + 0x60, 0x00, 0x34, 0xf5, // PUSH1 0 CALLVALUE CREATE2 -> deployed address + 0x80, 0x60, 0x00, 0x55, // DUP1 PUSH1 0 SSTORE -> record address at slot 0 + 0x60, 0x00, 0x35, // PUSH1 0 CALLDATALOAD -> mode word + 0x60, 0x1d, 0x57, // PUSH1 0x1d JUMPI -> call section if mode is non-zero + 0x00, // STOP + 0x5b, // JUMPDEST (offset 0x1d) + 0x60, 0x00, 0x60, 0x00, 0x60, 0x00, 0x60, 0x00, 0x60, 0x00, // ret/args/value zeros + 0x85, 0x5a, 0xf1, // DUP6 GAS CALL -> call the deployed address + 0x00, // STOP + ]; + const _: () = assert!(CODE[0x1d] == 0x5b, "jump destination must point at the JUMPDEST"); + Bytes::from_static(&CODE) +} + +/// Runtime that calls `target` (whose runtime selfdestructs) and then reverts, undoing the +/// selfdestruct. +fn call_then_revert_runtime(target: Address) -> Bytes { + let mut code = vec![0x60, 0x00, 0x60, 0x00, 0x60, 0x00, 0x60, 0x00, 0x60, 0x00]; + code.push(0x73); // PUSH20 + code.extend_from_slice(target.as_slice()); + code.extend_from_slice(&[0x5a, 0xf1, 0x50]); // GAS CALL POP + code.extend_from_slice(&[0x60, 0x00, 0x60, 0x00, 0xfd]); // PUSH1 0 PUSH1 0 REVERT + code.into() +} + +/// Runtime that calls `target`, ignores the result and stores 1 at slot 0 to prove that its own +/// frame committed. +fn call_then_store_runtime(target: Address) -> Bytes { + let mut code = vec![0x60, 0x00, 0x60, 0x00, 0x60, 0x00, 0x60, 0x00, 0x60, 0x00]; + code.push(0x73); // PUSH20 + code.extend_from_slice(target.as_slice()); + code.extend_from_slice(&[0x5a, 0xf1, 0x50]); // GAS CALL POP + code.extend_from_slice(&[0x60, 0x01, 0x60, 0x00, 0x55]); // PUSH1 1 PUSH1 0 SSTORE + code.push(0x00); // STOP + code.into() +} + +fn fork_spec(fork: EthereumHardfork) -> Arc { + let builder = ChainSpecBuilder::default() + .chain(MAINNET.chain) + .genesis(serde_json::from_str(include_str!("../assets/genesis.json")).unwrap()); + let builder = match fork { + EthereumHardfork::Cancun => builder.cancun_activated(), + EthereumHardfork::Osaka => builder.osaka_activated(), + EthereumHardfork::Amsterdam => builder.amsterdam_activated(), + fork => unimplemented!("no activation configured for {fork}"), + }; + Arc::new(builder.build()) +} diff --git a/crates/ethereum/node/tests/e2e/main.rs b/crates/ethereum/node/tests/e2e/main.rs index b26565592b9..045998ffedc 100644 --- a/crates/ethereum/node/tests/e2e/main.rs +++ b/crates/ethereum/node/tests/e2e/main.rs @@ -3,6 +3,7 @@ mod blobs; mod custom_genesis; mod dev; +mod eip6780; mod eth; mod invalid_payload; mod p2p; diff --git a/crates/ethereum/node/tests/e2e/utils.rs b/crates/ethereum/node/tests/e2e/utils.rs index 863731ba36c..1c92e5dc5cc 100644 --- a/crates/ethereum/node/tests/e2e/utils.rs +++ b/crates/ethereum/node/tests/e2e/utils.rs @@ -1,5 +1,5 @@ use alloy_eips::{eip2930::AccessListItem, eip7702::Authorization, BlockId, BlockNumberOrTag}; -use alloy_primitives::{bytes, Address, B256, U256}; +use alloy_primitives::{bytes, U256}; use alloy_provider::{ network::{ Ethereum, EthereumWallet, NetworkWallet, TransactionBuilder, TransactionBuilder7702, @@ -10,7 +10,8 @@ use alloy_rpc_types_engine::PayloadAttributes; use alloy_rpc_types_eth::TransactionRequest; use alloy_signer::SignerSync; use rand::{seq::IndexedRandom, Rng}; -use reth_e2e_test_utils::{wallet::Wallet, NodeHelperType, TmpDB}; +use reth_chainspec::EthereumHardfork; +use reth_e2e_test_utils::{eth_payload_attributes_for_fork, wallet::Wallet, NodeHelperType, TmpDB}; use reth_ethereum_primitives::TxType; use reth_node_api::NodeTypesWithDBAdapter; use reth_node_ethereum::EthereumNode; @@ -18,47 +19,17 @@ use reth_provider::FullProvider; /// Helper function to create a new eth payload attributes pub(crate) fn eth_payload_attributes(timestamp: u64) -> PayloadAttributes { - PayloadAttributes { - timestamp, - prev_randao: B256::ZERO, - suggested_fee_recipient: Address::ZERO, - withdrawals: Some(vec![]), - parent_beacon_block_root: Some(B256::ZERO), - slot_number: None, - ..Default::default() - } + eth_payload_attributes_for_fork(EthereumHardfork::Cancun, timestamp) } /// Helper function to create pre-Cancun (Shanghai) payload attributes. -/// No `parent_beacon_block_root` field. pub(crate) fn eth_payload_attributes_shanghai(timestamp: u64) -> PayloadAttributes { - PayloadAttributes { - timestamp, - prev_randao: B256::ZERO, - suggested_fee_recipient: Address::ZERO, - withdrawals: Some(vec![]), - parent_beacon_block_root: None, - slot_number: None, - ..Default::default() - } + eth_payload_attributes_for_fork(EthereumHardfork::Shanghai, timestamp) } /// Helper function to create Amsterdam payload attributes. -/// -/// Amsterdam payloads include block access lists, and the payload builder expects a -/// `slot_number` in the attributes once Amsterdam is active. Tests use the timestamp as a -/// deterministic dummy slot because the exact beacon slot is irrelevant for these local e2e -/// payloads. pub(crate) fn eth_payload_attributes_amsterdam(timestamp: u64) -> PayloadAttributes { - PayloadAttributes { - timestamp, - prev_randao: B256::ZERO, - suggested_fee_recipient: Address::ZERO, - withdrawals: Some(vec![]), - parent_beacon_block_root: Some(B256::ZERO), - slot_number: Some(timestamp), - ..Default::default() - } + eth_payload_attributes_for_fork(EthereumHardfork::Amsterdam, timestamp) } /// Advances node by producing blocks with random transactions. From 8decdbeaeee054b7855fefe83ae9bac6a16d0146 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Mon, 31 Aug 2026 12:02:05 +0200 Subject: [PATCH 096/294] chore: apply latest nightly rustfmt (#26887) --- .../engine/tree/src/tree/payload_processor/bal/execute.rs | 3 ++- crates/engine/tree/src/tree/state.rs | 3 ++- crates/era-downloader/tests/it/stream.rs | 4 +++- crates/net/eth-wire-types/src/capability.rs | 3 ++- crates/net/network/tests/it/requests.rs | 3 ++- crates/transaction-pool/src/pool/txpool.rs | 3 ++- crates/trie/trie/src/proof_v2/mod.rs | 3 ++- crates/trie/trie/src/trie_cursor/depth_first.rs | 3 ++- crates/trie/trie/src/verify.rs | 6 ++++-- examples/bsc-p2p/src/chainspec.rs | 3 ++- 10 files changed, 23 insertions(+), 11 deletions(-) diff --git a/crates/engine/tree/src/tree/payload_processor/bal/execute.rs b/crates/engine/tree/src/tree/payload_processor/bal/execute.rs index 4b1bc14d331..887539cd3d8 100644 --- a/crates/engine/tree/src/tree/payload_processor/bal/execute.rs +++ b/crates/engine/tree/src/tree/payload_processor/bal/execute.rs @@ -1212,7 +1212,8 @@ mod tests { let block_gas_limit = 1_000_000u64; let first_tx_gas = 600_000u64; - let second_tx_gas_limit = 500_000u64; // fits in total limit but not after cumulative deduction + let second_tx_gas_limit = 500_000u64; // fits in total limit but not after cumulative + // deduction let gas = ResultGas::new_with_state_gas(first_tx_gas, 0, 0, first_tx_gas / 2); let fake_result: ResultAndState = diff --git a/crates/engine/tree/src/tree/state.rs b/crates/engine/tree/src/tree/state.rs index a879ffd1b51..e94a3b9c244 100644 --- a/crates/engine/tree/src/tree/state.rs +++ b/crates/engine/tree/src/tree/state.rs @@ -480,7 +480,8 @@ mod tests { tree_state.insert_executed(fork_block_5.clone()); assert_eq!(tree_state.blocks_by_hash.len(), 8); - assert_eq!(tree_state.blocks_by_number[&3].len(), 2); // two blocks at height 3 (original and fork) + assert_eq!(tree_state.blocks_by_number[&3].len(), 2); // two blocks at height 3 (original + // and fork) assert_eq!(tree_state.parent_to_child[&blocks[1].recovered_block().hash()].len(), 2); // block 2 should have two children // verify that we can insert the same block again without issues diff --git a/crates/era-downloader/tests/it/stream.rs b/crates/era-downloader/tests/it/stream.rs index d901004ce66..291389b0277 100644 --- a/crates/era-downloader/tests/it/stream.rs +++ b/crates/era-downloader/tests/it/stream.rs @@ -74,7 +74,9 @@ async fn test_streaming_era1_files_after_fetching_file_list_into_missing_folder_ #[tokio::test] async fn test_streaming_era_files_after_fetching_file_list_into_missing_folder_fails() { - let base_url = Url::from_str("https://mainnet.era.nimbus.team").unwrap(); //TODO: change once ithaca host era files + let base_url = Url::from_str("https://mainnet.era.nimbus.team").unwrap(); //TODO: change once + // ithaca host era + // files let folder = tempdir().unwrap().path().to_owned(); let client = EraClient::new(StubClient, base_url, folder); diff --git a/crates/net/eth-wire-types/src/capability.rs b/crates/net/eth-wire-types/src/capability.rs index 1e4c191ab3d..6d7fb9890eb 100644 --- a/crates/net/eth-wire-types/src/capability.rs +++ b/crates/net/eth-wire-types/src/capability.rs @@ -207,7 +207,8 @@ impl From for Capability { impl<'a> arbitrary::Arbitrary<'a> for Capability { fn arbitrary(u: &mut arbitrary::Unstructured<'a>) -> arbitrary::Result { let version = u.int_in_range(66..=71)?; // Valid eth protocol versions are 66-71 - // Only generate valid eth protocol name for now since it's the only supported protocol + // Only generate valid eth protocol name for now + // since it's the only supported protocol Ok(Self::new_static("eth", version)) } } diff --git a/crates/net/network/tests/it/requests.rs b/crates/net/network/tests/it/requests.rs index 6e7e79ab0ef..81f6adfb612 100644 --- a/crates/net/network/tests/it/requests.rs +++ b/crates/net/network/tests/it/requests.rs @@ -269,7 +269,8 @@ async fn test_get_header_range_falling() { // ensure we can fetch the correct headers in falling direction // start from the last header and work backwards for idx in (0..100).rev() { - let count = std::cmp::min(idx + 1, 100); // Can't fetch more than idx+1 headers when going backwards + let count = std::cmp::min(idx + 1, 100); // Can't fetch more than idx+1 headers when going + // backwards let header = &all_headers[idx]; let req = HeadersRequest { start: header.hash().into(), diff --git a/crates/transaction-pool/src/pool/txpool.rs b/crates/transaction-pool/src/pool/txpool.rs index 0fd49c7ad35..1b87888a129 100644 --- a/crates/transaction-pool/src/pool/txpool.rs +++ b/crates/transaction-pool/src/pool/txpool.rs @@ -3515,7 +3515,8 @@ mod tests { // Now decrease basefee to trigger the zero-allocation optimization let mut block_info = pool.block_info(); - block_info.pending_basefee = 450; // tx1 (500) and tx2 (600) can now afford it, tx3 (400) cannot + block_info.pending_basefee = 450; // tx1 (500) and tx2 (600) can now afford it, tx3 (400) + // cannot pool.set_block_info(block_info); // Verify the optimization worked correctly: diff --git a/crates/trie/trie/src/proof_v2/mod.rs b/crates/trie/trie/src/proof_v2/mod.rs index 07db154868f..38390efaedc 100644 --- a/crates/trie/trie/src/proof_v2/mod.rs +++ b/crates/trie/trie/src/proof_v2/mod.rs @@ -3109,7 +3109,8 @@ mod tests { // Keys whose first bytes directly set the nibble paths we need. let key_a0 = B256::right_padding_from(&[0x6a, 0x80]); // nibbles: 6,a,8,0,... let key_a1 = B256::right_padding_from(&[0x6a, 0x81]); // nibbles: 6,a,8,1,... - let key_c = B256::right_padding_from(&[0x6a, 0x30]); // nibbles: 6,a,3,0,... (BEFORE [6,a,8]) + let key_c = B256::right_padding_from(&[0x6a, 0x30]); // nibbles: 6,a,3,0,... (BEFORE + // [6,a,8]) let key_d = B256::right_padding_from(&[0x6b, 0x00]); // nibbles: 6,b,0,0,... let key_e = B256::right_padding_from(&[0x6c, 0x00]); // nibbles: 6,c,0,0,... diff --git a/crates/trie/trie/src/trie_cursor/depth_first.rs b/crates/trie/trie/src/trie_cursor/depth_first.rs index d0856a4eeeb..525940754b0 100644 --- a/crates/trie/trie/src/trie_cursor/depth_first.rs +++ b/crates/trie/trie/src/trie_cursor/depth_first.rs @@ -267,7 +267,8 @@ mod tests { // Expected depth-first order: // All descendants come before ancestors // Within the same level, lexicographical order - assert_eq!(paths[0], Nibbles::from_nibbles([0x1, 0x1, 0x1])); // 0x111 (deepest in 0x1 branch) + assert_eq!(paths[0], Nibbles::from_nibbles([0x1, 0x1, 0x1])); // 0x111 (deepest in 0x1 + // branch) assert_eq!(paths[1], Nibbles::from_nibbles([0x1, 0x1, 0x2])); // 0x112 (sibling of 0x111) assert_eq!(paths[2], Nibbles::from_nibbles([0x1, 0x1])); // 0x11 (parent of 0x111, 0x112) assert_eq!(paths[3], Nibbles::from_nibbles([0x1, 0x2])); // 0x12 (sibling of 0x11) diff --git a/crates/trie/trie/src/verify.rs b/crates/trie/trie/src/verify.rs index fc371058ce1..b8aeb276eeb 100644 --- a/crates/trie/trie/src/verify.rs +++ b/crates/trie/trie/src/verify.rs @@ -959,9 +959,11 @@ mod tests { fn test_single_verifier_complex_depth_first() { // Test a complex tree structure with depth-first ordering // Build a tree structure with proper parent-child relationships - let node_root = test_branch_node(0b0110, 0, 0b0110, vec![]); // root: children at nibbles 1 and 2 + let node_root = test_branch_node(0b0110, 0, 0b0110, vec![]); // root: children at nibbles 1 + // and 2 let node1 = test_branch_node(0b0110, 0, 0b0110, vec![]); // 0x1: children at nibbles 1 and 2 - let node11 = test_branch_node(0b0110, 0, 0b0110, vec![]); // 0x11: children at nibbles 1 and 2 + let node11 = test_branch_node(0b0110, 0, 0b0110, vec![]); // 0x11: children at nibbles 1 and + // 2 let node111 = test_branch_node(0b0001, 0, 0b0001, vec![]); // 0x111: leaf let node112 = test_branch_node(0b0010, 0, 0b0010, vec![]); // 0x112: leaf let node12 = test_branch_node(0b0100, 0, 0b0100, vec![]); // 0x12: leaf diff --git a/examples/bsc-p2p/src/chainspec.rs b/examples/bsc-p2p/src/chainspec.rs index b820a06fe93..1896593b039 100644 --- a/examples/bsc-p2p/src/chainspec.rs +++ b/examples/bsc-p2p/src/chainspec.rs @@ -12,7 +12,8 @@ use std::{str::FromStr, sync::Arc}; hardfork!( /// The name of a bsc hardfork. /// - /// When building a list of hardforks for a chain, it's still expected to mix with [`EthereumHardfork`]. + /// When building a list of hardforks for a chain, it's still expected to mix with + /// [`EthereumHardfork`]. BscHardfork { /// BSC `Ramanujan` hardfork Ramanujan, From 0efb81fb4a2d9936dc0468d4db72bffe8fdd5788 Mon Sep 17 00:00:00 2001 From: Derek Cofausper <256792747+decofe@users.noreply.github.com> Date: Mon, 31 Aug 2026 10:16:00 +0000 Subject: [PATCH 097/294] chore(docker): configure nightly persistence thresholds (#26888) Co-authored-by: Brian Picciano <933154+mediocregopher@users.noreply.github.com> --- .github/workflows/docker.yml | 2 ++ Dockerfile.depot | 4 ++++ crates/node/core/src/args/engine.rs | 7 ++++++- docs/vocs/docs/pages/cli/reth/node.mdx | 2 ++ 4 files changed, 14 insertions(+), 1 deletion(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 5441c81cfd9..f0d3425e4ed 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -96,6 +96,8 @@ jobs: echo "ethereum.tags=${REGISTRY}/reth:nightly" echo "ethereum.args.RETH_ENGINE_TXPOOL_PREWARMING=true" echo "ethereum.args.RETH_ENGINE_SENDER_RECOVERY_CACHE=true" + echo "ethereum.args.RETH_ENGINE_PERSISTENCE_THRESHOLD=50" + echo "ethereum.args.RETH_ENGINE_NUM_STATE_MASKING_BLOCKS=30" echo "EOF" } >> "$GITHUB_OUTPUT" diff --git a/Dockerfile.depot b/Dockerfile.depot index 0ff3f2b736a..7602205e7f3 100644 --- a/Dockerfile.depot +++ b/Dockerfile.depot @@ -79,6 +79,10 @@ ARG RETH_ENGINE_TXPOOL_PREWARMING=false ENV RETH_ENGINE_TXPOOL_PREWARMING=$RETH_ENGINE_TXPOOL_PREWARMING ARG RETH_ENGINE_SENDER_RECOVERY_CACHE=false ENV RETH_ENGINE_SENDER_RECOVERY_CACHE=$RETH_ENGINE_SENDER_RECOVERY_CACHE +ARG RETH_ENGINE_PERSISTENCE_THRESHOLD=7 +ENV RETH_ENGINE_PERSISTENCE_THRESHOLD=$RETH_ENGINE_PERSISTENCE_THRESHOLD +ARG RETH_ENGINE_NUM_STATE_MASKING_BLOCKS=0 +ENV RETH_ENGINE_NUM_STATE_MASKING_BLOCKS=$RETH_ENGINE_NUM_STATE_MASKING_BLOCKS # Install runtime dependencies RUN apt-get update && \ diff --git a/crates/node/core/src/args/engine.rs b/crates/node/core/src/args/engine.rs index 50bb3977a93..f6aba3bdeee 100644 --- a/crates/node/core/src/args/engine.rs +++ b/crates/node/core/src/args/engine.rs @@ -313,7 +313,11 @@ pub struct EngineArgs { /// /// To persist blocks as fast as the node receives them, set this value to zero. This will /// cause more frequent DB writes. - #[arg(long = "engine.persistence-threshold", default_value_t = DefaultEngineValues::get_global().persistence_threshold)] + #[arg( + long = "engine.persistence-threshold", + env = "RETH_ENGINE_PERSISTENCE_THRESHOLD", + default_value_t = DefaultEngineValues::get_global().persistence_threshold + )] pub persistence_threshold: u64, /// Configure the maximum number of blocks beyond the in-memory buffer target that may await @@ -330,6 +334,7 @@ pub struct EngineArgs { /// writes instead of durably persisting their state/trie updates in the current cycle. #[arg( long = "engine.num-state-masking-blocks", + env = "RETH_ENGINE_NUM_STATE_MASKING_BLOCKS", default_value_t = DefaultEngineValues::get_global().num_state_masking_blocks )] pub num_state_masking_blocks: u64, diff --git a/docs/vocs/docs/pages/cli/reth/node.mdx b/docs/vocs/docs/pages/cli/reth/node.mdx index b234af4b03e..318fb7463d2 100644 --- a/docs/vocs/docs/pages/cli/reth/node.mdx +++ b/docs/vocs/docs/pages/cli/reth/node.mdx @@ -997,6 +997,7 @@ Engine: To persist blocks as fast as the node receives them, set this value to zero. This will cause more frequent DB writes. + [env: RETH_ENGINE_PERSISTENCE_THRESHOLD=] [default: 7] --engine.persistence-backpressure-threshold @@ -1009,6 +1010,7 @@ Engine: --engine.num-state-masking-blocks EXPERIMENTAL: Configure how many of the blocks being persisted should only mask state/trie writes instead of durably persisting their state/trie updates in the current cycle + [env: RETH_ENGINE_NUM_STATE_MASKING_BLOCKS=] [default: 0] --engine.memory-block-buffer-target From 8d4556ed54805dbd7115deca6190b4d277a30b7a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A9a=20Narzis?= <78718413+lean-apple@users.noreply.github.com> Date: Mon, 31 Aug 2026 11:01:05 +0000 Subject: [PATCH 098/294] fix(rpc): align block access list getter semantics (#26882) Co-authored-by: Matthias Seitz --- crates/ethereum/node/tests/e2e/rpc.rs | 45 ++++++++++++++++++++++- crates/rpc/rpc-eth-api/src/helpers/bal.rs | 4 ++ crates/rpc/rpc-eth-types/src/error/mod.rs | 7 ++-- 3 files changed, 50 insertions(+), 6 deletions(-) diff --git a/crates/ethereum/node/tests/e2e/rpc.rs b/crates/ethereum/node/tests/e2e/rpc.rs index d4ec107410f..82f949e5d2e 100644 --- a/crates/ethereum/node/tests/e2e/rpc.rs +++ b/crates/ethereum/node/tests/e2e/rpc.rs @@ -14,9 +14,9 @@ use alloy_rpc_types_engine::{ BlobsBundleV1, CancunPayloadFields, ExecutionPayload, ExecutionPayloadSidecar, ExecutionPayloadV3, PraguePayloadFields, }; -use alloy_rpc_types_eth::TransactionRequest; +use alloy_rpc_types_eth::{error::EthRpcErrorCode, TransactionRequest}; use alloy_rpc_types_trace::geth::{ChainBlockTraceResult, GethDebugTracingOptions}; -use jsonrpsee::core::client::{Subscription, SubscriptionClientT}; +use jsonrpsee::core::client::{ClientT, Subscription, SubscriptionClientT}; use rand::{rngs::StdRng, Rng, SeedableRng}; use reth_chainspec::{ChainSpecBuilder, EthChainSpec, MAINNET}; use reth_e2e_test_utils::{setup_engine, E2ETestSetupBuilder}; @@ -52,6 +52,47 @@ alloy_sol_types::sol! { } } +#[tokio::test] +async fn test_block_access_list_lookup_semantics() -> eyre::Result<()> { + reth_tracing::init_test_tracing(); + + let chain_spec = Arc::new( + ChainSpecBuilder::default() + .chain(MAINNET.chain) + .genesis(serde_json::from_str(include_str!("../assets/genesis.json")).unwrap()) + .cancun_activated() + .build(), + ); + + let (mut nodes, _) = setup_engine::( + 1, + chain_spec, + false, + Default::default(), + eth_payload_attributes, + ) + .await?; + let node = nodes.pop().unwrap(); + let client = node.rpc_client().unwrap(); + + let pending: Option = + client.request("eth_getBlockAccessList", (BlockNumberOrTag::Pending,)).await?; + assert_eq!(pending, None); + + for method in ["eth_getBlockAccessList", "debug_getRawBlockAccessList"] { + let error = client + .request::(method, (BlockNumberOrTag::Latest,)) + .await + .unwrap_err(); + let jsonrpsee::core::client::Error::Call(error) = error else { + panic!("expected a resource not found error, got {error:?}") + }; + assert_eq!(error.code(), EthRpcErrorCode::ResourceNotFound.code()); + } + + Ok(()) +} + #[tokio::test] async fn test_fee_history() -> eyre::Result<()> { reth_tracing::init_test_tracing(); diff --git a/crates/rpc/rpc-eth-api/src/helpers/bal.rs b/crates/rpc/rpc-eth-api/src/helpers/bal.rs index b3c37b7606c..d00c997a5fd 100644 --- a/crates/rpc/rpc-eth-api/src/helpers/bal.rs +++ b/crates/rpc/rpc-eth-api/src/helpers/bal.rs @@ -22,6 +22,10 @@ pub trait GetBlockAccessList: Trace + Call + LoadBlock + RpcNodeCoreExt { block_id: BlockId, ) -> impl Future, Self::Error>> + Send { async move { + if block_id.is_pending() { + return Ok(None) + } + let Some(block) = self.recovered_block(block_id).await? else { return Ok(None); }; diff --git a/crates/rpc/rpc-eth-types/src/error/mod.rs b/crates/rpc/rpc-eth-types/src/error/mod.rs index 94dcb7c5615..5cbf7dd96ad 100644 --- a/crates/rpc/rpc-eth-types/src/error/mod.rs +++ b/crates/rpc/rpc-eth-types/src/error/mod.rs @@ -313,7 +313,9 @@ impl From for jsonrpsee_types::error::ErrorObject<'static> { EthApiError::InvalidBlockData(_) | EthApiError::Internal(_) | EthApiError::EvmCustom(_) => internal_rpc_err(error.to_string()), - EthApiError::UnknownBlockOrTxIndex | EthApiError::TransactionNotFound => { + EthApiError::UnknownBlockOrTxIndex | + EthApiError::TransactionNotFound | + EthApiError::BlockAccessListNotAvailablePreAmsterdam => { rpc_error_with_code(EthRpcErrorCode::ResourceNotFound.code(), error.to_string()) } EthApiError::TracingTransactionNotFound | EthApiError::GenesisNotTraceable => { @@ -380,9 +382,6 @@ impl From for jsonrpsee_types::error::ErrorObject<'static> { error.data(), ) } - EthApiError::BlockAccessListNotAvailablePreAmsterdam => { - rpc_error_with_code(4445, error.to_string()) - } } } } From f0166c32e235bc1e94667c7760818e345f2e1189 Mon Sep 17 00:00:00 2001 From: Brian Picciano Date: Mon, 31 Aug 2026 11:08:13 +0000 Subject: [PATCH 099/294] feat(overlay): Use OverlayStateProvider for most StateProviders in hot paths (#26616) Co-authored-by: Matthias Seitz Co-authored-by: Brian Picciano <933154+mediocregopher@users.noreply.github.com> Co-authored-by: Derek Cofausper <256792747+decofe@users.noreply.github.com> --- crates/engine/tree/src/tree/mod.rs | 5 +- .../tree/src/tree/payload_processor/mod.rs | 21 +-- .../src/tree/payload_processor/prewarm.rs | 38 +++-- .../engine/tree/src/tree/payload_validator.rs | 98 ++++++------ .../tree/src/tree/state_root_strategy/mod.rs | 151 +++++++++--------- .../tree/state_root_strategy/sparse_trie.rs | 16 +- .../tree/src/tree/txpool_prewarm/mod.rs | 18 ++- .../tree/src/tree/txpool_prewarm/worker.rs | 18 ++- .../src/providers/blockchain_provider.rs | 60 +++---- .../storage-overlay/src/changeset_cache.rs | 4 +- .../storage/storage-overlay/src/provider.rs | 26 +-- 11 files changed, 233 insertions(+), 222 deletions(-) diff --git a/crates/engine/tree/src/tree/mod.rs b/crates/engine/tree/src/tree/mod.rs index 75a7bec431f..a85c050d151 100644 --- a/crates/engine/tree/src/tree/mod.rs +++ b/crates/engine/tree/src/tree/mod.rs @@ -2277,8 +2277,9 @@ where return None } - if self.canonical_in_memory_state.canonical_chain().count() <= - self.config.persistence_threshold() as usize + let persistence_threshold = + usize::try_from(self.config.persistence_threshold()).unwrap_or(usize::MAX); + if self.canonical_in_memory_state.canonical_chain().count() <= persistence_threshold { return None } diff --git a/crates/engine/tree/src/tree/payload_processor/mod.rs b/crates/engine/tree/src/tree/payload_processor/mod.rs index 628155f3ca5..551b0adf4df 100644 --- a/crates/engine/tree/src/tree/payload_processor/mod.rs +++ b/crates/engine/tree/src/tree/payload_processor/mod.rs @@ -4,7 +4,7 @@ use super::precompile_cache::PrecompileCacheMap; use crate::tree::{ payload_processor::prewarm::{PrewarmCacheTask, PrewarmContext, PrewarmMode, PrewarmTaskEvent}, CachedStateCacheMetrics, CachedStateMetrics, CachedStateMetricsSource, ExecutionCache, - ExecutionEnv, PayloadExecutionCache, SavedCache, StateProviderBuilder, TreeConfig, + ExecutionEnv, PayloadExecutionCache, SavedCache, TreeConfig, }; use alloy_eips::eip1898::BlockWithParent; use alloy_primitives::B256; @@ -18,10 +18,11 @@ use reth_evm::{ }; use reth_primitives_traits::{FastInstant as Instant, NodePrimitives}; use reth_provider::{ - BlockExecutionOutput, BlockNumReader, DatabaseProviderFactory, PruneCheckpointReader, - StageCheckpointReader, StorageSettingsCache, TryIntoHistoricalStateProvider, + BlockExecutionOutput, BlockNumReader, ChangeSetReader, DatabaseProviderFactory, + PruneCheckpointReader, StageCheckpointReader, StorageChangeSetReader, StorageSettingsCache, }; use reth_revm::db::BundleState; +use reth_storage_overlay::OverlayStateProviderFactory; use reth_tasks::Runtime; pub use reth_trie_parallel::{ error::StateRootTaskError, @@ -169,7 +170,7 @@ where &self, env: ExecutionEnv, transactions: I, - provider_builder: StateProviderBuilder, + state_provider_factory: OverlayStateProviderFactory, hint_stream: Option, hashed_update_stream: Option, parallel_bal_execution: bool, @@ -179,8 +180,9 @@ where P::Provider: BlockNumReader + PruneCheckpointReader + StageCheckpointReader + + ChangeSetReader + + StorageChangeSetReader + StorageSettingsCache - + TryIntoHistoricalStateProvider + 'static, { let (prewarm_rx, execution_rx) = @@ -188,7 +190,7 @@ where let prewarm_handle = self.spawn_caching_with( env, prewarm_rx, - provider_builder, + state_provider_factory, hint_stream, hashed_update_stream, parallel_bal_execution, @@ -339,7 +341,7 @@ where &self, env: ExecutionEnv, transactions: mpsc::Receiver<(usize, impl ExecutableTxFor + Clone + Send + 'static)>, - provider_builder: StateProviderBuilder, + state_provider_factory: OverlayStateProviderFactory, hint_stream: Option, hashed_update_stream: Option, parallel_bal_execution: bool, @@ -349,8 +351,9 @@ where P::Provider: BlockNumReader + PruneCheckpointReader + StageCheckpointReader + + ChangeSetReader + + StorageChangeSetReader + StorageSettingsCache - + TryIntoHistoricalStateProvider + 'static, { // Each mode carries the capability its producers use; the rest is dropped here, so @@ -375,7 +378,7 @@ where env, evm_config: self.evm_config.clone(), saved_cache: saved_cache.clone(), - provider: provider_builder, + provider: state_provider_factory, bal_prewarm_pool: parallel_bal_execution.then(|| self.bal_prewarm_pool()), metrics: PrewarmMetrics::default(), cache_metrics: self.cache_metrics.clone(), diff --git a/crates/engine/tree/src/tree/payload_processor/prewarm.rs b/crates/engine/tree/src/tree/payload_processor/prewarm.rs index ed795224604..dba966aadd6 100644 --- a/crates/engine/tree/src/tree/payload_processor/prewarm.rs +++ b/crates/engine/tree/src/tree/payload_processor/prewarm.rs @@ -15,7 +15,7 @@ use super::{bal_prewarm_pool::BalPrewarmPool, StateRootHintStream, StateRootUpda use crate::tree::{ precompile_cache::{CachedPrecompile, PrecompileCacheMap}, CachedStateCacheMetrics, CachedStateMetrics, CachedStateProvider, ExecutionEnv, - PayloadExecutionCache, SavedCache, StateProviderBuilder, + PayloadExecutionCache, SavedCache, }; use alloy_consensus::transaction::TxHashRef; use alloy_eip7928::bal::DecodedBal; @@ -27,11 +27,12 @@ use reth_evm::{execute::ExecutableTxFor, ConfigureEvm, Evm, EvmFor, RecoveredTx, use reth_metrics::Metrics; use reth_primitives_traits::{Account, FastInstant as Instant, NodePrimitives}; use reth_provider::{ - AccountReader, BlockExecutionOutput, BlockNumReader, DatabaseProviderFactory, - PruneCheckpointReader, StageCheckpointReader, StorageSettingsCache, - TryIntoHistoricalStateProvider, + AccountReader, BlockExecutionOutput, BlockNumReader, ChangeSetReader, DatabaseProviderFactory, + DatabaseProviderROFactory, PruneCheckpointReader, StageCheckpointReader, StateProviderBox, + StorageChangeSetReader, StorageSettingsCache, }; use reth_revm::database::StateProviderDatabase; +use reth_storage_overlay::OverlayStateProviderFactory; use reth_tasks::{pool::WorkerPool, Runtime}; use reth_trie_common::MultiProofTargetsV2; use std::sync::{ @@ -96,8 +97,9 @@ where P::Provider: BlockNumReader + PruneCheckpointReader + StageCheckpointReader + + ChangeSetReader + + StorageChangeSetReader + StorageSettingsCache - + TryIntoHistoricalStateProvider + 'static, Evm: ConfigureEvm + 'static, { @@ -418,8 +420,12 @@ where // This runs side-by-side with the parallel transaction execution reducing the time it // spends blocking on the data. let caches = saved_cache.cache().clone(); - let provider_builder = ctx.provider.clone(); - let build = Arc::new(move || provider_builder.build()); + let state_provider_factory = ctx.provider.clone(); + let build = Arc::new(move || { + state_provider_factory + .database_provider_ro() + .map(|provider| Box::new(provider) as _) + }); pool.begin_block(build, caches, ctx.env.txpool_snapshot.clone()); for account in prefetch_bal.as_bal() { @@ -536,7 +542,7 @@ where /// The saved cache. pub saved_cache: Option, /// Provider to obtain the state - pub provider: StateProviderBuilder, + pub provider: OverlayStateProviderFactory, /// Dedicated blocking pool for warming the BAL read-set. `Some` only on the BAL parallel /// execution path; the pool is owned by the [`PayloadProcessor`](super::PayloadProcessor). pub(crate) bal_prewarm_pool: Option>, @@ -576,16 +582,17 @@ where P::Provider: BlockNumReader + PruneCheckpointReader + StageCheckpointReader + + ChangeSetReader + + StorageChangeSetReader + StorageSettingsCache - + TryIntoHistoricalStateProvider + 'static, Evm: ConfigureEvm + 'static, { /// Creates a per-thread EVM for prewarming. #[instrument(level = "debug", target = "engine::tree::payload_processor::prewarm", skip_all)] fn evm_for_ctx(&self) -> PrewarmEvmState { - let mut state_provider = match self.provider.build() { - Ok(provider) => provider, + let mut state_provider: StateProviderBox = match self.provider.database_provider_ro() { + Ok(provider) => Box::new(provider), Err(err) => { trace!( target: "engine::tree::payload_processor::prewarm", @@ -704,7 +711,7 @@ where ) .entered(); - let inner = match self.provider.build() { + let inner = match self.provider.database_provider_ro() { Ok(p) => p, Err(err) => { warn!( @@ -834,7 +841,7 @@ mod tests { }; use alloy_primitives::{address, bytes}; use reth_chainspec::ChainSpec; - use reth_ethereum_primitives::{EthPrimitives, TransactionSigned}; + use reth_ethereum_primitives::TransactionSigned; use reth_evm::{execute::WithTxEnv, TxEnvFor}; use reth_evm_ethereum::EthEvmConfig; use reth_provider::test_utils::MockEthProvider; @@ -847,10 +854,9 @@ mod tests { env: ExecutionEnv::test_default(), evm_config: EthEvmConfig::new(Arc::new(ChainSpec::default())), saved_cache: None, - provider: StateProviderBuilder::::new( + provider: OverlayStateProviderFactory::new( MockEthProvider::default(), - B256::ZERO, - OverlayManager::default(), + OverlayManager::default().overlay_builder(B256::ZERO), ), bal_prewarm_pool: None, metrics: PrewarmMetrics::default(), diff --git a/crates/engine/tree/src/tree/payload_validator.rs b/crates/engine/tree/src/tree/payload_validator.rs index 4b72a2b7847..e586c23e8ff 100644 --- a/crates/engine/tree/src/tree/payload_validator.rs +++ b/crates/engine/tree/src/tree/payload_validator.rs @@ -105,7 +105,7 @@ use crate::tree::{ txpool_prewarm, types::{InsertPayloadResult, ValidationOutput}, CacheWaitDurations, CachedStateProvider, EngineApiMetrics, EngineApiTreeState, ExecutionEnv, - PayloadHandle, StateProviderBuilder, StateProviderDatabase, TreeConfig, WaitForCaches, + PayloadHandle, StateProviderDatabase, TreeConfig, WaitForCaches, }; use alloy_consensus::transaction::{Either, TxHashRef}; use alloy_eip7928::{ @@ -151,10 +151,10 @@ use reth_primitives_traits::{ RecoveredBlock, SealedBlock, SealedHeader, SignerRecoverable, }; use reth_provider::{ - BlockExecutionOutput, BlockReader, ChangeSetReader, DatabaseProviderFactory, - DatabaseProviderROFactory, ProviderError, PruneCheckpointReader, StageCheckpointReader, - StateProvider, StateProviderBox, StateProviderFactory, StateReader, StorageChangeSetReader, - StorageSettingsCache, TryIntoHistoricalStateProvider, + BlockExecutionOutput, BlockHashReader, BlockReader, ChangeSetReader, DatabaseProviderFactory, + DatabaseProviderROFactory, HashedPostStateProvider, ProviderError, PruneCheckpointReader, + StageCheckpointReader, StateProvider, StateProviderBox, StateProviderFactory, StateReader, + StateRootProvider, StorageChangeSetReader, StorageSettingsCache, }; use reth_revm::db::{states::bundle_state::BundleRetention, BundleAccount, State}; use reth_storage_overlay::{OverlayManager, OverlayStateProviderFactory}; @@ -309,12 +309,12 @@ where N: NodePrimitives, P: DatabaseProviderFactory< Provider: BlockReader + + BlockHashReader + StageCheckpointReader + PruneCheckpointReader + ChangeSetReader + StorageChangeSetReader + StorageSettingsCache - + TryIntoHistoricalStateProvider + 'static, > + BlockReader

+ ChangeSetReader @@ -322,8 +322,14 @@ where + StateReader + Clone + 'static, - OverlayStateProviderFactory: DatabaseProviderROFactory - + Clone + OverlayStateProviderFactory: DatabaseProviderROFactory< + Provider: TrieCursorFactory + + HashedCursorFactory + + HashedPostStateProvider + + StateRootProvider + + StateProvider + + Send, + > + Clone + 'static, Evm: ConfigureEvm + 'static, { @@ -554,8 +560,8 @@ where trace!(target: "engine::tree::payload_validator", "Fetching block state provider"); let _enter = debug_span!(target: "engine::tree::payload_validator", "state_provider").entered(); - let Some(provider_builder) = - ensure_ok!(self.state_provider_builder(parent_hash, ctx.state())) + let Some(state_provider_factory) = + ensure_ok!(self.overlay_state_provider_factory(parent_hash, ctx.state())) else { // this is pre-validated in the tree return Err(InsertBlockError::new( @@ -599,11 +605,6 @@ where // Get an iterator over the transactions in the payload let txs = self.tx_iterator_for(&input)?; - // Create overlay factory for state-root tasks that need multiproofs. - let provider_factory = self.provider.clone(); - let overlay_builder = ctx.state().tree_state.overlay_manager.overlay_builder(parent_hash); - let overlay_factory = OverlayStateProviderFactory::new(provider_factory, overlay_builder); - let parallel_bal_execution = ensure_ok!(self.bal_path_eligible(env.decoded_bal.as_deref())); // Prepare the state-root job before execution so it can provide streaming hooks. @@ -613,8 +614,7 @@ where &self.overlay_manager, &env, &parent_block, - provider_builder.clone(), - overlay_factory, + state_provider_factory.clone(), &self.config, parallel_bal_execution, ctx.state_mut(), @@ -639,7 +639,7 @@ where let mut handle = ensure_ok!(self.spawn_payload_processor( env.clone(), txs, - provider_builder.clone(), + state_provider_factory.clone(), hint_stream, hashed_update_stream, parallel_bal_execution, @@ -675,7 +675,7 @@ where // The second parameter `instrument_state_provider` controls whether we should // instrument the state provider with metrics. let make_state_provider = |fill_on_miss: bool| -> ProviderResult { - let provider = provider_builder.build()?; + let provider = state_provider_factory.database_provider_ro()?; let mut provider = if let Some((caches, cache_metrics)) = &execution_cache { let fill_mode = if fill_on_miss { CacheFillMode::FillOnMiss @@ -693,7 +693,7 @@ where .with_txpool_snapshot(txpool_snapshot.clone()), ) as StateProviderBox } else { - provider + Box::new(provider) as StateProviderBox }; if instrument_state_provider { @@ -814,7 +814,11 @@ where || hashed_state.get(), &block, &parent_block, - || provider_builder.build(), + || { + state_provider_factory + .database_provider_ro() + .map(|provider| Box::new(provider) as _) + }, ) }); @@ -850,7 +854,11 @@ where || hashed_state.get(), &block, &parent_block, - || provider_builder.build(), + || { + state_provider_factory + .database_provider_ro() + .map(|provider| Box::new(provider) as _) + }, ) }); } @@ -1381,7 +1389,7 @@ where &self, env: ExecutionEnv, txs: T, - provider_builder: StateProviderBuilder, + state_provider_factory: OverlayStateProviderFactory, hint_stream: Option, hashed_update_stream: Option, parallel_bal_execution: bool, @@ -1397,7 +1405,7 @@ where let handle = self.payload_processor.spawn_with_state_root_streams( env, txs, - provider_builder, + state_provider_factory, hint_stream, hashed_update_stream, parallel_bal_execution, @@ -1408,23 +1416,22 @@ where Ok(handle) } - /// Creates a `StateProviderBuilder` for the given parent hash. + /// Creates an overlay state provider factory for the given parent hash. /// /// Returns `None` when the parent is neither in memory nor persisted. - fn state_provider_builder( + fn overlay_state_provider_factory( &self, hash: B256, state: &EngineApiTreeState, - ) -> ProviderResult>> { + ) -> ProviderResult>> { if !state.tree_state.contains_hash(&hash) && self.provider.header(hash)?.is_none() { debug!(target: "engine::tree::payload_validator", %hash, "no canonical state found for block"); return Ok(None) } - Ok(Some(StateProviderBuilder::new( + Ok(Some(OverlayStateProviderFactory::new( self.provider.clone(), - hash, - state.tree_state.overlay_manager.clone(), + state.tree_state.overlay_manager.overlay_builder(hash), ))) } @@ -1453,8 +1460,8 @@ where timestamp: u64, state: &mut EngineApiTreeState, ) -> Option { - let provider_builder = match self.state_provider_builder(parent_hash, state) { - Ok(Some(provider_builder)) => provider_builder, + let state_provider_factory = match self.overlay_state_provider_factory(parent_hash, state) { + Ok(Some(state_provider_factory)) => state_provider_factory, Ok(None) => return None, Err(err) => { warn!( @@ -1466,11 +1473,6 @@ where return None } }; - let overlay_factory = OverlayStateProviderFactory::new( - self.provider.clone(), - state.tree_state.overlay_manager.overlay_builder(parent_hash), - ); - match self.state_root_strategy.prepare_payload_builder(PayloadStateRootJobContext::new( &self.runtime, &self.overlay_manager, @@ -1478,8 +1480,7 @@ where parent_header, timestamp, state, - provider_builder, - overlay_factory, + state_provider_factory, &self.config, )) { Ok(handle) => handle, @@ -1792,12 +1793,12 @@ impl EngineValidator for BasicEngineValidator + BlockReader
+ StateProviderFactory @@ -1805,8 +1806,14 @@ where + ChangeSetReader + Clone + 'static, - OverlayStateProviderFactory: DatabaseProviderROFactory - + Clone + OverlayStateProviderFactory: DatabaseProviderROFactory< + Provider: TrieCursorFactory + + HashedCursorFactory + + HashedPostStateProvider + + StateRootProvider + + StateProvider + + Send, + > + Clone + 'static, N: NodePrimitives, V: PayloadValidator + Clone, @@ -1896,8 +1903,9 @@ where } }; - let provider_builder = match self.state_provider_builder(parent.hash(), state) { - Ok(Some(provider_builder)) => provider_builder, + let state_provider_factory = match self.overlay_state_provider_factory(parent.hash(), state) + { + Ok(Some(state_provider_factory)) => state_provider_factory, Ok(None) => return, Err(err) => { trace!( @@ -1909,7 +1917,7 @@ where return } }; - txpool_prewarm.start(parent.hash(), evm_env, provider_builder) + txpool_prewarm.start(parent.hash(), evm_env, state_provider_factory) } fn payload_builder_resources( diff --git a/crates/engine/tree/src/tree/state_root_strategy/mod.rs b/crates/engine/tree/src/tree/state_root_strategy/mod.rs index 8edc4a0052e..c6519c2795d 100644 --- a/crates/engine/tree/src/tree/state_root_strategy/mod.rs +++ b/crates/engine/tree/src/tree/state_root_strategy/mod.rs @@ -56,10 +56,7 @@ mod sparse_trie; use self::sparse_trie::{SparseTrieCacheTask, SparseTrieTaskMetrics}; -use crate::tree::{ - metrics::BlockValidationMetrics, EngineApiTreeState, ExecutionEnv, StateProviderBuilder, - TreeConfig, -}; +use crate::tree::{metrics::BlockValidationMetrics, EngineApiTreeState, ExecutionEnv, TreeConfig}; use alloy_primitives::B256; use crossbeam_channel::{Receiver as CrossbeamReceiver, Sender as CrossbeamSender}; use reth_chain_state::{ExecutedBlock, PreservedSparseTrie}; @@ -69,9 +66,9 @@ use reth_primitives_traits::{ AlloyBlockHeader, FastInstant as Instant, NodePrimitives, RecoveredBlock, SealedHeader, }; use reth_provider::{ - BlockExecutionOutput, BlockNumReader, DatabaseProviderFactory, DatabaseProviderROFactory, - HashedPostStateProvider, ProviderError, PruneCheckpointReader, StageCheckpointReader, - StateRootProvider, StorageSettingsCache, TryIntoHistoricalStateProvider, + BlockExecutionOutput, BlockNumReader, ChangeSetReader, DatabaseProviderFactory, + DatabaseProviderROFactory, HashedPostStateProvider, ProviderError, PruneCheckpointReader, + StageCheckpointReader, StateRootProvider, StorageChangeSetReader, StorageSettingsCache, }; use reth_storage_overlay::{OverlayManager, OverlayStateProviderFactory}; use reth_tasks::utils::increase_thread_priority; @@ -146,8 +143,7 @@ where parent_header: &'a N::BlockHeader, timestamp: u64, state: &'a mut EngineApiTreeState, - provider_builder: StateProviderBuilder, - overlay_factory: OverlayStateProviderFactory, + state_provider_factory: OverlayStateProviderFactory, config: &'a TreeConfig, } @@ -178,8 +174,7 @@ where parent_header: &'a N::BlockHeader, timestamp: u64, state: &'a mut EngineApiTreeState, - provider_builder: StateProviderBuilder, - overlay_factory: OverlayStateProviderFactory, + state_provider_factory: OverlayStateProviderFactory, config: &'a TreeConfig, ) -> Self { Self { @@ -189,8 +184,7 @@ where parent_header, timestamp, state, - provider_builder, - overlay_factory, + state_provider_factory, config, } } @@ -225,14 +219,6 @@ where self.executor } - /// Returns a clone of the state provider builder. - pub fn provider_builder(&self) -> StateProviderBuilder - where - P: Clone, - { - self.provider_builder.clone() - } - /// Consumes the pending sparse trie prune request as in-memory parent-chain blocks, if any. /// /// Custom strategies that maintain a reusable sparse trie should call this when starting the @@ -252,8 +238,7 @@ where overlay_manager: &'a OverlayManager, env: &'a ExecutionEnv, parent_header: &'a SealedHeader, - provider_builder: StateProviderBuilder, - overlay_factory: OverlayStateProviderFactory, + state_provider_factory: OverlayStateProviderFactory, config: &'a TreeConfig, parallel_bal_execution: bool, state: &'a mut EngineApiTreeState, @@ -284,8 +269,7 @@ where overlay_manager: &'a OverlayManager, env: &'a ExecutionEnv, parent_header: &'a SealedHeader, - provider_builder: StateProviderBuilder, - overlay_factory: OverlayStateProviderFactory, + state_provider_factory: OverlayStateProviderFactory, config: &'a TreeConfig, parallel_bal_execution: bool, state: &'a mut EngineApiTreeState, @@ -295,8 +279,7 @@ where overlay_manager, env, parent_header, - provider_builder, - overlay_factory, + state_provider_factory, config, parallel_bal_execution, state, @@ -323,14 +306,6 @@ where self.parallel_bal_execution } - /// Returns a clone of the state provider builder. - pub fn provider_builder(&self) -> StateProviderBuilder - where - P: Clone, - { - self.provider_builder.clone() - } - /// Consumes the pending sparse trie prune request as in-memory parent-chain blocks, if any. /// /// Custom strategies that maintain a reusable sparse trie should call this when starting the @@ -823,11 +798,17 @@ where P::Provider: BlockNumReader + PruneCheckpointReader + StageCheckpointReader + + ChangeSetReader + + StorageChangeSetReader + StorageSettingsCache - + TryIntoHistoricalStateProvider + 'static, - OverlayStateProviderFactory: DatabaseProviderROFactory - + Clone + OverlayStateProviderFactory: DatabaseProviderROFactory< + Provider: TrieCursorFactory + + HashedCursorFactory + + HashedPostStateProvider + + StateRootProvider + + Send, + > + Clone + 'static, Evm: ConfigureEvm + 'static, { @@ -841,7 +822,9 @@ where if !ctx.config.use_state_root_task() { return Ok(PreparedStateRootJob::new( - Box::new(SynchronousStateRootJob { provider_builder: ctx.provider_builder }), + Box::new(SynchronousStateRootJob { + state_provider_factory: ctx.state_provider_factory, + }), None, )) } @@ -852,28 +835,27 @@ where overlay_manager, env, parent_header, - provider_builder, - overlay_factory, + state_provider_factory, config, parallel_bal_execution, state: _, } = ctx; let preserved_sparse_trie = overlay_manager.take_sparse_trie(); - let overlay_factory = if let Some(anchor_hash) = preserved_sparse_trie + let proof_state_provider_factory = if let Some(anchor_hash) = preserved_sparse_trie .as_ref() .filter(|trie| trie.state_root() == env.parent_state_root) .map(|trie| trie.anchor_hash()) { - overlay_factory.with_skip_overlay_for_reused_sparse_trie(anchor_hash) + state_provider_factory.clone().with_skip_overlay_for_reused_sparse_trie(anchor_hash) } else { - overlay_factory + state_provider_factory.clone() }; let mut handle = self.spawn_state_root( executor, overlay_manager, - overlay_factory.clone(), + proof_state_provider_factory, StateRootTaskOptions { parent_header: parent_header.clone(), preserved_sparse_trie, @@ -900,8 +882,7 @@ where let mut prepared = PreparedStateRootJob::new( Box::new(SparseTrieStateRootJob { handle, - provider_builder, - overlay_factory, + state_provider_factory, executor: executor.clone(), timeout: config.state_root_task_timeout(), compare_trie_updates: config.always_compare_trie_updates(), @@ -936,20 +917,20 @@ where let parent_state_root = ctx.parent_state_root(); let parent_header = SealedHeader::new(ctx.parent_header().clone(), ctx.parent_hash()); let preserved_sparse_trie = ctx.overlay_manager.take_sparse_trie(); - let overlay_factory = if let Some(anchor_hash) = preserved_sparse_trie + let proof_state_provider_factory = if let Some(anchor_hash) = preserved_sparse_trie .as_ref() .filter(|trie| trie.state_root() == parent_state_root) .map(|trie| trie.anchor_hash()) { - ctx.overlay_factory.clone().with_skip_overlay_for_reused_sparse_trie(anchor_hash) + ctx.state_provider_factory.clone().with_skip_overlay_for_reused_sparse_trie(anchor_hash) } else { - ctx.overlay_factory.clone() + ctx.state_provider_factory.clone() }; Ok(Some( self.spawn_state_root( ctx.executor, ctx.overlay_manager, - overlay_factory, + proof_state_provider_factory, StateRootTaskOptions { parent_header, preserved_sparse_trie, @@ -984,7 +965,7 @@ impl StateRootJob for SkippedStateRootJob { #[derive(Debug)] struct SynchronousStateRootJob { - provider_builder: StateProviderBuilder, + state_provider_factory: OverlayStateProviderFactory, } impl StateRootJob for SynchronousStateRootJob @@ -994,9 +975,11 @@ where P::Provider: BlockNumReader + PruneCheckpointReader + StageCheckpointReader + + ChangeSetReader + + StorageChangeSetReader + StorageSettingsCache - + TryIntoHistoricalStateProvider + 'static, + OverlayStateProviderFactory: DatabaseProviderROFactory, { fn name(&self) -> &'static str { "synchronous" @@ -1008,7 +991,7 @@ where _output: Arc>, hashed_state: &LazyHashedPostState, ) -> ProviderResult { - let provider = self.provider_builder.clone().build()?; + let provider = self.state_provider_factory.database_provider_ro()?; let (state_root, trie_updates) = provider.state_root_with_updates(hashed_state.get().as_ref().clone())?; Ok(StateRootJobOutcome::new(state_root, Arc::new(trie_updates))) @@ -1018,8 +1001,7 @@ where #[derive(Debug)] struct SparseTrieStateRootJob { handle: StateRootHandle, - provider_builder: StateProviderBuilder, - overlay_factory: OverlayStateProviderFactory, + state_provider_factory: OverlayStateProviderFactory, executor: reth_tasks::Runtime, timeout: Option, compare_trie_updates: bool, @@ -1033,19 +1015,25 @@ where P::Provider: BlockNumReader + PruneCheckpointReader + StageCheckpointReader + + ChangeSetReader + + StorageChangeSetReader + StorageSettingsCache - + TryIntoHistoricalStateProvider + 'static, - OverlayStateProviderFactory: DatabaseProviderROFactory - + Clone + OverlayStateProviderFactory: DatabaseProviderROFactory< + Provider: TrieCursorFactory + + HashedCursorFactory + + HashedPostStateProvider + + StateRootProvider + + Send, + > + Clone + 'static, { fn serial_fallback( executor: &reth_tasks::Runtime, - provider_builder: StateProviderBuilder, + state_provider_factory: OverlayStateProviderFactory, output: Arc>, ) -> ProviderResult { - let provider = provider_builder.build()?; + let provider = state_provider_factory.database_provider_ro()?; let (fallback_tx, fallback_rx) = mpsc::channel(); executor.spawn_blocking_named("serial-root", move || { let result = (|| { @@ -1068,7 +1056,7 @@ where &self, output: &BlockExecutionOutput, ) -> ProviderResult { - let provider = self.provider_builder.clone().build()?; + let provider = self.state_provider_factory.database_provider_ro()?; let hashed_state = Arc::new(provider.hashed_post_state(&output.state)?); let (state_root, trie_updates) = provider.state_root_with_updates(hashed_state.as_ref().clone())?; @@ -1116,8 +1104,7 @@ where if self.compare_trie_updates { let _has_diff = compare_trie_updates_with_serial( - self.provider_builder.clone(), - self.overlay_factory.clone(), + self.state_provider_factory.clone(), output, trie_updates.as_ref().clone(), ); @@ -1143,11 +1130,17 @@ where P::Provider: BlockNumReader + PruneCheckpointReader + StageCheckpointReader + + ChangeSetReader + + StorageChangeSetReader + StorageSettingsCache - + TryIntoHistoricalStateProvider + 'static, - OverlayStateProviderFactory: DatabaseProviderROFactory - + Clone + OverlayStateProviderFactory: DatabaseProviderROFactory< + Provider: TrieCursorFactory + + HashedCursorFactory + + HashedPostStateProvider + + StateRootProvider + + Send, + > + Clone + 'static, { fn name(&self) -> &'static str { @@ -1178,7 +1171,7 @@ where debug!(target: "engine::tree::state_root_strategy", %err, "State root task failed, falling back to serial root"); Self::serial_fallback( &self.executor, - self.provider_builder.clone(), + self.state_provider_factory.clone(), output.clone(), )? } @@ -1187,7 +1180,7 @@ where self.metrics.state_root_task_timeout_total.increment(1); Self::serial_fallback( &self.executor, - self.provider_builder.clone(), + self.state_provider_factory.clone(), output.clone(), )? } @@ -1195,7 +1188,7 @@ where debug!(target: "engine::tree::state_root_strategy", "State root task dropped, falling back to serial root"); Self::serial_fallback( &self.executor, - self.provider_builder.clone(), + self.state_provider_factory.clone(), output.clone(), )? } @@ -1237,8 +1230,7 @@ where } fn compare_trie_updates_with_serial( - state_provider_builder: StateProviderBuilder, - overlay_factory: OverlayStateProviderFactory, + state_provider_factory: OverlayStateProviderFactory, output: &BlockExecutionOutput, task_trie_updates: TrieUpdates, ) -> bool @@ -1248,15 +1240,20 @@ where P::Provider: BlockNumReader + PruneCheckpointReader + StageCheckpointReader + + ChangeSetReader + + StorageChangeSetReader + StorageSettingsCache - + TryIntoHistoricalStateProvider + 'static, - OverlayStateProviderFactory: - DatabaseProviderROFactory, + OverlayStateProviderFactory: DatabaseProviderROFactory< + Provider: TrieCursorFactory + + HashedCursorFactory + + HashedPostStateProvider + + StateRootProvider, + >, { debug!(target: "engine::tree::state_root_strategy", "Comparing trie updates with serial computation"); - match state_provider_builder.build().and_then(|provider| { + match state_provider_factory.database_provider_ro().and_then(|provider| { let hashed_state = provider.hashed_post_state(&output.state)?; provider.state_root_with_updates(hashed_state) }) { @@ -1267,7 +1264,7 @@ where "Serial state root computation finished for comparison" ); - match overlay_factory.database_provider_ro() { + match state_provider_factory.database_provider_ro() { Ok(provider) => match super::trie_updates::compare_trie_updates( &provider, task_trie_updates, diff --git a/crates/engine/tree/src/tree/state_root_strategy/sparse_trie.rs b/crates/engine/tree/src/tree/state_root_strategy/sparse_trie.rs index 3f9cd9fe79b..a8622f7c687 100644 --- a/crates/engine/tree/src/tree/state_root_strategy/sparse_trie.rs +++ b/crates/engine/tree/src/tree/state_root_strategy/sparse_trie.rs @@ -1230,7 +1230,7 @@ mod tests { let runtime = reth_tasks::Runtime::test(); let provider_factory = create_test_provider_factory(); let anchor_hash = init_genesis(&provider_factory).expect("failed to initialize genesis"); - let overlay_factory = OverlayStateProviderFactory::new( + let state_provider_factory = OverlayStateProviderFactory::new( provider_factory, OverlayManager::::default() .overlay_builder(anchor_hash), @@ -1238,7 +1238,7 @@ mod tests { let (proof_result_tx, proof_result_rx) = crossbeam_channel::unbounded(); let proof_worker_handle = ProofWorkerHandle::new( &runtime, - ProofTaskCtx::new(overlay_factory), + ProofTaskCtx::new(state_provider_factory), false, proof_result_tx.clone(), ); @@ -1285,7 +1285,7 @@ mod tests { let runtime = reth_tasks::Runtime::test(); let provider_factory = create_test_provider_factory(); let anchor_hash = init_genesis(&provider_factory).expect("failed to initialize genesis"); - let overlay_factory = OverlayStateProviderFactory::new( + let state_provider_factory = OverlayStateProviderFactory::new( provider_factory, OverlayManager::::default() .overlay_builder(anchor_hash), @@ -1293,7 +1293,7 @@ mod tests { let (proof_result_tx, proof_result_rx) = crossbeam_channel::unbounded(); let proof_worker_handle = ProofWorkerHandle::new( &runtime, - ProofTaskCtx::new(overlay_factory), + ProofTaskCtx::new(state_provider_factory), false, proof_result_tx.clone(), ); @@ -1373,7 +1373,7 @@ mod tests { let runtime = reth_tasks::Runtime::test(); let provider_factory = create_test_provider_factory(); let anchor_hash = init_genesis(&provider_factory).expect("failed to initialize genesis"); - let overlay_factory = OverlayStateProviderFactory::new( + let state_provider_factory = OverlayStateProviderFactory::new( provider_factory, OverlayManager::::default() .overlay_builder(anchor_hash), @@ -1381,7 +1381,7 @@ mod tests { let (proof_result_tx, proof_result_rx) = crossbeam_channel::unbounded(); let proof_worker_handle = ProofWorkerHandle::new( &runtime, - ProofTaskCtx::new(overlay_factory), + ProofTaskCtx::new(state_provider_factory), false, proof_result_tx.clone(), ); @@ -1426,7 +1426,7 @@ mod tests { let runtime = reth_tasks::Runtime::test(); let provider_factory = create_test_provider_factory(); let anchor_hash = init_genesis(&provider_factory).expect("failed to initialize genesis"); - let overlay_factory = OverlayStateProviderFactory::new( + let state_provider_factory = OverlayStateProviderFactory::new( provider_factory, OverlayManager::::default() .overlay_builder(anchor_hash), @@ -1434,7 +1434,7 @@ mod tests { let (proof_result_tx, proof_result_rx) = crossbeam_channel::unbounded(); let proof_worker_handle = ProofWorkerHandle::new( &runtime, - ProofTaskCtx::new(overlay_factory), + ProofTaskCtx::new(state_provider_factory), false, proof_result_tx.clone(), ); diff --git a/crates/engine/tree/src/tree/txpool_prewarm/mod.rs b/crates/engine/tree/src/tree/txpool_prewarm/mod.rs index 091058b0c74..5032d7851c4 100644 --- a/crates/engine/tree/src/tree/txpool_prewarm/mod.rs +++ b/crates/engine/tree/src/tree/txpool_prewarm/mod.rs @@ -4,15 +4,17 @@ mod control; mod worker; use self::control::Control; -use crate::tree::{StateProviderBuilder, TxPoolPrewarmCacheSnapshot}; +use crate::tree::TxPoolPrewarmCacheSnapshot; use alloy_consensus::transaction::Recovered; use alloy_primitives::{Address, B256}; use reth_evm::{ConfigureEvm, EvmEnvFor}; use reth_primitives_traits::{NodePrimitives, TxTy}; use reth_provider::{ - BlockNumReader, DatabaseProviderFactory, PruneCheckpointReader, StageCheckpointReader, - StorageSettingsCache, TryIntoHistoricalStateProvider, + BlockNumReader, ChangeSetReader, DatabaseProviderFactory, DatabaseProviderROFactory, + PruneCheckpointReader, StageCheckpointReader, StateProvider, StorageChangeSetReader, + StorageSettingsCache, }; +use reth_storage_overlay::OverlayStateProviderFactory; use std::{fmt::Debug, sync::Arc}; /// Coordinates a long-lived worker and the latest completed immutable snapshot. @@ -41,9 +43,11 @@ where P::Provider: BlockNumReader + PruneCheckpointReader + StageCheckpointReader + + ChangeSetReader + + StorageChangeSetReader + StorageSettingsCache - + TryIntoHistoricalStateProvider + 'static, + OverlayStateProviderFactory: DatabaseProviderROFactory + Send, Evm: ConfigureEvm + 'static, { /// Spawns the long-lived worker, which owns its mutable read cache and starts a fresh one for @@ -84,9 +88,9 @@ where &self, parent_hash: B256, evm_env: EvmEnvFor, - provider_builder: StateProviderBuilder, + state_provider_factory: OverlayStateProviderFactory, ) { - self.control.start(parent_hash, Job { evm_env, provider_builder }); + self.control.start(parent_hash, Job { evm_env, state_provider_factory }); } } @@ -120,5 +124,5 @@ pub trait Source: Send + Sync + Debug { /// A request to warm txpool transactions against one fully validated parent state. struct Job> { evm_env: EvmEnvFor, - provider_builder: StateProviderBuilder, + state_provider_factory: OverlayStateProviderFactory, } diff --git a/crates/engine/tree/src/tree/txpool_prewarm/worker.rs b/crates/engine/tree/src/tree/txpool_prewarm/worker.rs index 1400b68ae72..71a6efd9bea 100644 --- a/crates/engine/tree/src/tree/txpool_prewarm/worker.rs +++ b/crates/engine/tree/src/tree/txpool_prewarm/worker.rs @@ -9,10 +9,12 @@ use crossbeam_channel::{Receiver, RecvTimeoutError, TryRecvError}; use reth_evm::ConfigureEvm; use reth_primitives_traits::NodePrimitives; use reth_provider::{ - BlockNumReader, DatabaseProviderFactory, PruneCheckpointReader, StageCheckpointReader, - StorageSettingsCache, TryIntoHistoricalStateProvider, + BlockNumReader, ChangeSetReader, DatabaseProviderFactory, DatabaseProviderROFactory, + PruneCheckpointReader, StageCheckpointReader, StateProvider, StorageChangeSetReader, + StorageSettingsCache, }; use reth_revm::{cached::CachedReads, db::State}; +use reth_storage_overlay::OverlayStateProviderFactory; use std::{ sync::Arc, time::{Duration, Instant}, @@ -70,9 +72,11 @@ where P::Provider: BlockNumReader + PruneCheckpointReader + StageCheckpointReader + + ChangeSetReader + + StorageChangeSetReader + StorageSettingsCache - + TryIntoHistoricalStateProvider + 'static, + OverlayStateProviderFactory: DatabaseProviderROFactory + Send, Evm: ConfigureEvm, { pub(super) fn new( @@ -184,7 +188,7 @@ where return BatchEnd::GoAgain } - let state_provider = match job.provider_builder.build() { + let state_provider = match job.state_provider_factory.database_provider_ro() { Ok(provider) => provider, Err(err) => { trace!( @@ -316,7 +320,6 @@ fn entry_counts(reads: &CachedReads) -> (usize, usize, usize) { #[cfg(test)] mod tests { use super::{super::Transaction as PoolTransaction, *}; - use crate::tree::StateProviderBuilder; use alloy_consensus::{transaction::Recovered, Signed, TxLegacy}; use alloy_primitives::{Address, Signature, TxKind, U256}; use crossbeam_channel::{unbounded, Sender}; @@ -376,10 +379,9 @@ mod tests { provider.add_stage_checkpoint(StageId::Finish, StageCheckpoint::new(0)); let job = Job { evm_env: Default::default(), - provider_builder: StateProviderBuilder::new( + state_provider_factory: OverlayStateProviderFactory::new( provider, - parent_hash, - reth_storage_overlay::OverlayManager::default(), + reth_storage_overlay::OverlayManager::default().overlay_builder(parent_hash), ), }; self.commands.send(Command::Start { parent_hash, job }).unwrap(); diff --git a/crates/storage/provider/src/providers/blockchain_provider.rs b/crates/storage/provider/src/providers/blockchain_provider.rs index 3dbdd0cfbf0..952c9915005 100644 --- a/crates/storage/provider/src/providers/blockchain_provider.rs +++ b/crates/storage/provider/src/providers/blockchain_provider.rs @@ -1,4 +1,3 @@ -use super::state::latest::LatestStateProvider; use crate::{ providers::{ ConsistentProvider, ProviderNodeTypes, RocksDBProvider, StaticFileProvider, @@ -18,7 +17,7 @@ use alloy_primitives::{Address, BlockHash, BlockNumber, Bytes, TxHash, TxNumber, use alloy_rpc_types_engine::ForkchoiceState; use reth_chain_state::{ BlockState, CanonicalInMemoryState, ForkChoiceNotifications, ForkChoiceSubscriptions, - MemoryOverlayStateProvider, PersistedBlockNotifications, PersistedBlockSubscriptions, + PersistedBlockNotifications, PersistedBlockSubscriptions, }; use reth_chainspec::ChainInfo; use reth_db_api::models::{AccountBeforeTx, BlockNumberAddress, StoredBlockBodyIndices}; @@ -31,15 +30,12 @@ use reth_prune_types::{PruneCheckpoint, PruneSegment}; use reth_stages_types::{StageCheckpoint, StageId}; use reth_static_file_types::StaticFileSegment; use reth_storage_api::{ - BlockBodyIndicesProvider, NodePrimitivesProvider, RangeEnd, RangeResponse, RangeResult, - StateRangeProvider, StateRangeProviderFactory, StateRangeView, StorageChangeSetReader, - StorageRangeResult, TryIntoHistoricalStateProvider, + BlockBodyIndicesProvider, DatabaseProviderROFactory, NodePrimitivesProvider, RangeEnd, + RangeResponse, RangeResult, StateRangeProvider, StateRangeProviderFactory, StateRangeView, + StorageChangeSetReader, StorageRangeResult, }; use reth_storage_errors::provider::ProviderResult; -use reth_storage_overlay::{ - anchor_for_parent, AnchorForParent, OverlayStateProvider, OverlayStateProviderFactory, - OwnedProvider, -}; +use reth_storage_overlay::{OverlayStateProvider, OverlayStateProviderFactory, OwnedProvider}; use reth_trie::{ hashed_cursor::{HashedCursor, HashedCursorFactory}, metrics::TrieRootMetrics, @@ -157,21 +153,12 @@ impl BlockchainProvider { fn block_state_provider( &self, state: &BlockState, - ) -> ProviderResult> { - let provider = self.database.provider()?; - let anchor = - anchor_for_parent(state.hash(), state.chain().map(|state| state.block()), &provider)?; - - let (historical, overlay): (StateProviderBox, _) = match anchor { - AnchorForParent::NoReverts { overlay, .. } => { - (Box::new(LatestStateProvider::new(provider)), overlay) - } - AnchorForParent::RevertsRequired { anchor, overlay, .. } => { - (provider.try_into_history_at_block(anchor.number)?, overlay) - } - }; - - Ok(MemoryOverlayStateProvider::new(historical, overlay)) + ) -> ProviderResult { + let state_provider_factory = OverlayStateProviderFactory::new( + self.database.clone(), + self.database.overlay_manager().overlay_builder(state.hash()), + ); + Ok(Box::new(state_provider_factory.database_provider_ro()?)) } /// Returns a cursor-backed state view for a state root still only in canonical in-memory @@ -199,15 +186,14 @@ impl BlockchainProvider { // Anchor at the persisted block; the overlay reverts any db-tip advancement past it // via changesets, then the merged in-memory delta applies on top. - let overlay_factory = OverlayStateProviderFactory::new( + let state_provider_factory = OverlayStateProviderFactory::new( self.database.clone(), self.database .overlay_manager() .overlay_builder(matched.anchor().hash) .with_immediate_state_trie_overlay(merged.state, merged.nodes), ); - reth_storage_api::DatabaseProviderROFactory::database_provider_ro(&overlay_factory) - .map(Some) + state_provider_factory.database_provider_ro().map(Some) } /// Returns a cursor-backed state view for a retained canonical state root. @@ -230,12 +216,11 @@ impl BlockchainProvider { drop(provider); let Some(block_hash) = block_hash else { return Ok(None) }; - let overlay_factory = OverlayStateProviderFactory::new( + let state_provider_factory = OverlayStateProviderFactory::new( self.database.clone(), self.database.overlay_manager().overlay_builder(block_hash), ); - reth_storage_api::DatabaseProviderROFactory::database_provider_ro(&overlay_factory) - .map(Some) + state_provider_factory.database_provider_ro().map(Some) } } @@ -750,7 +735,7 @@ impl StateProviderFactory for BlockchainProvider { // use latest state provider if the head state exists if let Some(state) = self.canonical_in_memory_state.head_state() { trace!(target: "providers::blockchain", "Using head state for latest state provider"); - Ok(self.block_state_provider(&state)?.boxed()) + self.block_state_provider(&state) } else { trace!(target: "providers::blockchain", "Using database state for latest state provider"); self.database.latest() @@ -807,7 +792,9 @@ impl StateProviderFactory for BlockchainProvider { fn state_by_block_hash(&self, hash: BlockHash) -> ProviderResult { trace!(target: "providers::blockchain", ?hash, "Getting state by block hash"); - if let Ok(state) = self.history_by_block_hash(hash) { + if let Some(state) = self.canonical_in_memory_state.state_by_hash(hash) { + self.block_state_provider(&state) + } else if let Ok(state) = self.history_by_block_hash(hash) { // This could be tracked by a historical block Ok(state) } else if let Ok(Some(pending)) = self.pending_state_by_hash(hash) { @@ -828,7 +815,7 @@ impl StateProviderFactory for BlockchainProvider { if let Some(pending) = self.canonical_in_memory_state.pending_state() { // we have a pending block - return Ok(Box::new(self.block_state_provider(&pending)?)); + return self.block_state_provider(&pending); } // fallback to latest state if the pending block is not available @@ -839,14 +826,14 @@ impl StateProviderFactory for BlockchainProvider { if let Some(pending) = self.canonical_in_memory_state.pending_state() && pending.hash() == block_hash { - return Ok(Some(Box::new(self.block_state_provider(&pending)?))); + return self.block_state_provider(&pending).map(Some); } Ok(None) } fn maybe_pending(&self) -> ProviderResult> { if let Some(pending) = self.canonical_in_memory_state.pending_state() { - return Ok(Some(Box::new(self.block_state_provider(&pending)?))) + return self.block_state_provider(&pending).map(Some) } Ok(None) @@ -1181,6 +1168,9 @@ mod tests { .collect(), }; provider.canonical_in_memory_state.update_chain(chain); + for state in provider.canonical_in_memory_state.canonical_chain() { + provider.database.overlay_manager().insert_block(state.block()); + } // Get canonical, safe, and finalized blocks let blocks = database_blocks.iter().chain(in_memory_blocks.iter()).collect::>(); diff --git a/crates/storage/storage-overlay/src/changeset_cache.rs b/crates/storage/storage-overlay/src/changeset_cache.rs index e91d6717e68..c2e070fd4b1 100644 --- a/crates/storage/storage-overlay/src/changeset_cache.rs +++ b/crates/storage/storage-overlay/src/changeset_cache.rs @@ -774,10 +774,10 @@ mod tests { .into_sorted(); let db_cursor_factory = DatabaseTrieCursorFactory::<_, A>::new(provider.tx_ref()); - let overlay_factory = + let state_provider_factory = InMemoryTrieCursorFactory::new(db_cursor_factory, &cumulative_trie_updates_prev); - compute_trie_changesets(&overlay_factory, &trie_updates).unwrap() + compute_trie_changesets(&state_provider_factory, &trie_updates).unwrap() } fn seed_tip_trie_tables(provider: &Provider) diff --git a/crates/storage/storage-overlay/src/provider.rs b/crates/storage/storage-overlay/src/provider.rs index 428aa735197..7403544cebb 100644 --- a/crates/storage/storage-overlay/src/provider.rs +++ b/crates/storage/storage-overlay/src/provider.rs @@ -982,12 +982,12 @@ mod tests { for block in &blocks[2..=3] { manager.insert_block(block.clone()); } - let overlay_factory = OverlayStateProviderFactory::new( + let state_provider_factory = OverlayStateProviderFactory::new( factory.clone(), manager.overlay_builder(blocks[3].recovered_block().hash()), ); - let provider = overlay_factory.database_provider_ro().unwrap(); + let provider = state_provider_factory.database_provider_ro().unwrap(); let first = provider.state_trie_overlay().unwrap().clone(); assert_eq!(account_keys(&first), vec![B256::with_last_byte(3), B256::with_last_byte(4)]); drop(provider); @@ -1003,11 +1003,11 @@ mod tests { .unwrap(); provider_rw.commit().unwrap(); - let provider = overlay_factory.database_provider_ro().unwrap(); + let provider = state_provider_factory.database_provider_ro().unwrap(); let second = provider.state_trie_overlay().unwrap().clone(); assert_eq!(account_keys(&second), vec![B256::with_last_byte(4)]); assert_eq!(account_node_paths(&second), vec![Nibbles::from_nibbles([4])]); - assert_eq!(overlay_factory.state_trie_overlay_cache.len(), 2); + assert_eq!(state_provider_factory.state_trie_overlay_cache.len(), 2); } #[test] @@ -1017,30 +1017,30 @@ mod tests { for block in &blocks[2..=3] { manager.insert_block(block.clone()); } - let overlay_factory = OverlayStateProviderFactory::new( + let state_provider_factory = OverlayStateProviderFactory::new( factory, manager.overlay_builder(blocks[3].recovered_block().hash()), ); - let provider = overlay_factory.database_provider_ro().unwrap(); + let provider = state_provider_factory.database_provider_ro().unwrap(); assert!(provider.state_trie_overlay.get().is_none()); assert!(provider.execution_overlay.get().is_none()); - assert!(overlay_factory.state_trie_overlay_cache.is_empty()); - assert!(overlay_factory.execution_overlay_cache.is_empty()); + assert!(state_provider_factory.state_trie_overlay_cache.is_empty()); + assert!(state_provider_factory.execution_overlay_cache.is_empty()); provider.basic_account(&Address::ZERO).unwrap(); let execution_overlay = Arc::clone(provider.execution_overlay.get().unwrap()); assert!(provider.state_trie_overlay.get().is_none()); assert!(provider.execution_overlay.get().is_some()); - assert!(overlay_factory.state_trie_overlay_cache.is_empty()); - assert_eq!(overlay_factory.execution_overlay_cache.len(), 1); - let cached_overlay = overlay_factory.execution_overlay_cache.iter().next().unwrap(); + assert!(state_provider_factory.state_trie_overlay_cache.is_empty()); + assert_eq!(state_provider_factory.execution_overlay_cache.len(), 1); + let cached_overlay = state_provider_factory.execution_overlay_cache.iter().next().unwrap(); assert!(Arc::ptr_eq(&execution_overlay, cached_overlay.value())); provider.account_trie_cursor().unwrap(); - assert_eq!(overlay_factory.state_trie_overlay_cache.len(), 1); - assert_eq!(overlay_factory.execution_overlay_cache.len(), 1); + assert_eq!(state_provider_factory.state_trie_overlay_cache.len(), 1); + assert_eq!(state_provider_factory.execution_overlay_cache.len(), 1); } #[test] From 1e9a9438addd454e6675265e5190914c9810e43d Mon Sep 17 00:00:00 2001 From: Chase Wright Date: Mon, 31 Aug 2026 14:39:34 +0000 Subject: [PATCH 100/294] fix(rpc): return invalid params error for eth_getLogs fromBlock beyond head (#26891) --- crates/rpc/rpc/src/eth/filter.rs | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/crates/rpc/rpc/src/eth/filter.rs b/crates/rpc/rpc/src/eth/filter.rs index da5a5d5da66..d811c650e88 100644 --- a/crates/rpc/rpc/src/eth/filter.rs +++ b/crates/rpc/rpc/src/eth/filter.rs @@ -583,13 +583,6 @@ where }); } - if let Some(f) = from && - f > info.best_number - { - // start block higher than local head, can return empty - return Ok(Vec::new()); - } - let (from_block_number, to_block_number) = logs_utils::get_filter_block_range(from, to, start_block, info)?; @@ -1428,6 +1421,20 @@ mod tests { .build() } + #[tokio::test] + async fn test_logs_for_filter_from_block_beyond_head() { + let provider = MockEthProvider::default(); + provider.add_header(FixedBytes::random(), alloy_consensus::Header::default()); + let eth_api = build_test_eth_api(provider); + + let eth_filter = + super::EthFilter::new(eth_api, EthFilterConfig::default(), Runtime::test()); + + let filter = Filter::new().from_block(100u64).to_block(BlockNumberOrTag::Latest); + let result = eth_filter.inner.clone().logs_for_filter(filter, QueryLimits::default()).await; + assert!(matches!(result, Err(EthFilterError::InvalidBlockRangeParams)), "{result:?}"); + } + #[tokio::test] async fn test_range_block_mode_empty_range() { let provider = MockEthProvider::default(); From 931c9c71c692c3de2e7529fb55ae1acab05e0a64 Mon Sep 17 00:00:00 2001 From: Chase Wright Date: Mon, 31 Aug 2026 15:41:14 +0000 Subject: [PATCH 101/294] fix(rpc): return used gas from the basic transfer estimate shortcut (#26875) Co-authored-by: Matthias Seitz --- crates/ethereum/node/tests/e2e/rpc.rs | 97 ++++++++++++++++++- .../rpc/rpc-eth-api/src/helpers/estimate.rs | 27 +++--- 2 files changed, 107 insertions(+), 17 deletions(-) diff --git a/crates/ethereum/node/tests/e2e/rpc.rs b/crates/ethereum/node/tests/e2e/rpc.rs index 82f949e5d2e..93b27320e30 100644 --- a/crates/ethereum/node/tests/e2e/rpc.rs +++ b/crates/ethereum/node/tests/e2e/rpc.rs @@ -3,7 +3,9 @@ use alloy_eips::{eip2718::Encodable2718, eip7910::EthConfig, BlockNumberOrTag}; use alloy_genesis::Genesis; use alloy_primitives::{Address, Bytes, B256, U256}; use alloy_provider::{ - ext::DebugApi, network::EthereumWallet, Provider, ProviderBuilder, SendableTx, + ext::DebugApi, + network::{EthereumWallet, TransactionBuilder}, + Provider, ProviderBuilder, SendableTx, }; use alloy_rpc_types_beacon::relay::{ BidTrace, BuilderBlockValidationRequestV3, BuilderBlockValidationRequestV4, @@ -14,12 +16,16 @@ use alloy_rpc_types_engine::{ BlobsBundleV1, CancunPayloadFields, ExecutionPayload, ExecutionPayloadSidecar, ExecutionPayloadV3, PraguePayloadFields, }; -use alloy_rpc_types_eth::{error::EthRpcErrorCode, TransactionRequest}; +use alloy_rpc_types_eth::{ + error::EthRpcErrorCode, + state::{AccountOverride, StateOverride}, + TransactionRequest, +}; use alloy_rpc_types_trace::geth::{ChainBlockTraceResult, GethDebugTracingOptions}; use jsonrpsee::core::client::{ClientT, Subscription, SubscriptionClientT}; use rand::{rngs::StdRng, Rng, SeedableRng}; use reth_chainspec::{ChainSpecBuilder, EthChainSpec, MAINNET}; -use reth_e2e_test_utils::{setup_engine, E2ETestSetupBuilder}; +use reth_e2e_test_utils::{setup_engine, wallet::Wallet, E2ETestSetupBuilder}; use reth_network::{types::NatResolver, PeersInfo}; use reth_node_builder::{NodeBuilder, NodeHandle}; use reth_node_core::{ @@ -540,6 +546,91 @@ async fn test_flashbots_validate_v6() -> eyre::Result<()> { Ok(()) } +#[tokio::test] +async fn test_estimate_gas_basic_transfers_post_amsterdam() -> eyre::Result<()> { + reth_tracing::init_test_tracing(); + + let chain_spec = Arc::new( + ChainSpecBuilder::default() + .chain(MAINNET.chain) + .genesis(serde_json::from_str(include_str!("../assets/genesis.json")).unwrap()) + .amsterdam_activated() + .build(), + ); + + let (mut nodes, _) = setup_engine::( + 1, + chain_spec, + false, + Default::default(), + eth_payload_attributes_amsterdam, + ) + .await?; + let node = nodes.pop().unwrap(); + let provider = ProviderBuilder::new().connect_http(node.rpc_url()); + + let mut signers = Wallet::new(2).wallet_gen(); + let from = signers.remove(0).address(); + let existing_recipient = signers.remove(0).address(); + + let self_send_gas = provider + .estimate_gas( + TransactionRequest::default().with_from(from).with_to(from).with_value(U256::from(1)), + ) + .await?; + assert_eq!(self_send_gas, 12_000); + + let zero_value_existing_gas = provider + .estimate_gas(TransactionRequest::default().with_from(from).with_to(existing_recipient)) + .await?; + assert_eq!(zero_value_existing_gas, 15_000); + + let value_existing_gas = provider + .estimate_gas( + TransactionRequest::default() + .with_from(from) + .with_to(existing_recipient) + .with_value(U256::from(1)), + ) + .await?; + assert_eq!(value_existing_gas, 21_000); + + let fresh_recipient = Address::repeat_byte(0xfe); + let zero_value_fresh_gas = provider + .estimate_gas(TransactionRequest::default().with_from(from).with_to(fresh_recipient)) + .await?; + assert_eq!(zero_value_fresh_gas, 15_000); + + // Creating the recipient requires additional state gas, so the 21_000-gas run fails and + // estimation falls back to binary search. + let value_fresh_gas = provider + .estimate_gas( + TransactionRequest::default() + .with_from(from) + .with_to(fresh_recipient) + .with_value(U256::from(1)), + ) + .await?; + assert!(value_fresh_gas > 21_000); + + // The override installs code that needs more than 4_000 gas at entry. If the estimator + // mistakes this for a basic transfer, it returns too little gas for the code to run. + let gas_gate = Address::repeat_byte(0xaa); + let mut overrides = StateOverride::default(); + overrides.insert( + gas_gate, + AccountOverride { + code: Some("0x5a610fa010600957fe5b00".parse::()?), + ..Default::default() + }, + ); + let gated_tx = TransactionRequest::default().with_from(from).with_to(gas_gate); + let gated_gas = provider.estimate_gas(gated_tx.clone()).overrides(overrides.clone()).await?; + provider.call(gated_tx.with_gas_limit(gated_gas)).overrides(overrides).await?; + + Ok(()) +} + /// Recomputes the block hash of the request after its execution payload has been modified and /// updates it in the payload and the bid trace. fn update_block_hash_v6(request: &mut BuilderBlockValidationRequestV6) -> eyre::Result<()> { diff --git a/crates/rpc/rpc-eth-api/src/helpers/estimate.rs b/crates/rpc/rpc-eth-api/src/helpers/estimate.rs index 0990ab2c52a..8f66619d4b9 100644 --- a/crates/rpc/rpc-eth-api/src/helpers/estimate.rs +++ b/crates/rpc/rpc-eth-api/src/helpers/estimate.rs @@ -29,7 +29,7 @@ use reth_rpc_server_types::constants::gas_oracle::{CALL_STIPEND_GAS, ESTIMATE_GA use revm::{ context::Block, context_interface::{result::ExecutionResult, Cfg, Transaction}, - primitives::KECCAK_EMPTY, + Database as _, }; use tracing::trace; @@ -116,15 +116,16 @@ pub trait EstimateCall: Call { let mut tx_env = self.create_txn_env(&evm_env, request, &mut db)?; - // Check if this is a basic transfer (no input data to account with no code) + // Check whether this is a basic transfer: empty input to an account without bytecode. let is_basic_transfer = if tx_env.input().is_empty() && let TxKind::Call(to) = tx_env.kind() { - match db.database.basic_account(&to) { - Ok(Some(account)) => { - account.bytecode_hash.is_none() || account.bytecode_hash == Some(KECCAK_EMPTY) - } - _ => true, + // Fetch the account through `Database::basic` so the state overrides applied above + // are visible. + match db.basic(to) { + Ok(Some(account)) => account.is_empty_code_hash(), + Ok(None) => true, + Err(_) => false, } } else { false @@ -145,13 +146,11 @@ pub trait EstimateCall: Call { // Create EVM instance once and reuse it throughout the entire estimation process let mut evm = self.evm_config().evm_with_env(&mut db, evm_env); - // For basic transfers, try using minimum gas before running full binary search + // For basic transfers, try 21_000 gas before running the full binary search. if is_basic_transfer { - // If the tx is a simple transfer (call to an account with no code) we can - // shortcircuit. But simply returning - // `MIN_TRANSACTION_GAS` is dangerous because there might be additional - // field combos that bump the price up, so we try executing the function - // with the minimum gas limit to make sure. + // A basic transfer executes no bytecode and receives no refunds, so the amount + // consumed by a successful run is the exact gas required. EIP-2780 can make that less + // than 21_000. let mut min_tx_env = tx_env.clone(); min_tx_env.set_gas_limit(MIN_TRANSACTION_GAS); @@ -159,7 +158,7 @@ pub trait EstimateCall: Call { if let Ok(res) = evm.transact(min_tx_env).map_err(Self::Error::from_evm_err) && res.result.is_success() { - return Ok(U256::from(MIN_TRANSACTION_GAS)) + return Ok(U256::from(res.result.tx_gas_used())) } } From 3a1cc31f02060e8689f06b5247eeaac296a55aeb Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Tue, 1 Sep 2026 00:15:28 +0200 Subject: [PATCH 102/294] fix(rpc): guard block access list replay (#26895) --- crates/rpc/rpc-eth-api/src/helpers/bal.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/crates/rpc/rpc-eth-api/src/helpers/bal.rs b/crates/rpc/rpc-eth-api/src/helpers/bal.rs index d00c997a5fd..1f7492b8b0a 100644 --- a/crates/rpc/rpc-eth-api/src/helpers/bal.rs +++ b/crates/rpc/rpc-eth-api/src/helpers/bal.rs @@ -40,6 +40,8 @@ pub trait GetBlockAccessList: Trace + Call + LoadBlock + RpcNodeCoreExt { return Ok(Some(Vec::from(bal))) } + let _permit = self.acquire_owned_blocking_io().await; + self.spawn_blocking_io(move |eth_api| { let state = eth_api .provider() From 26b9ee3a6045907ea9ae942b189e1b9dfb65bc5d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=96mer=20Faruk=20Irmak?= Date: Tue, 1 Sep 2026 08:58:45 +0000 Subject: [PATCH 103/294] feat(engine): expose BalPrewarmPool (#26769) --- .../tree/payload_processor/bal_prewarm_pool.rs | 18 ++++++++++-------- .../tree/src/tree/payload_processor/mod.rs | 2 +- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/crates/engine/tree/src/tree/payload_processor/bal_prewarm_pool.rs b/crates/engine/tree/src/tree/payload_processor/bal_prewarm_pool.rs index 734fa57c229..d76c4ff8058 100644 --- a/crates/engine/tree/src/tree/payload_processor/bal_prewarm_pool.rs +++ b/crates/engine/tree/src/tree/payload_processor/bal_prewarm_pool.rs @@ -1,3 +1,5 @@ +//! BAL read-set prewarming pool. + use alloy_primitives::{Address, StorageKey}; use reth_execution_cache::{CachedStateProvider, ExecutionCache, TxPoolPrewarmCacheSnapshot}; use reth_provider::{ @@ -15,7 +17,7 @@ use tracing::trace; /// Builds a fresh `StateProviderBox` over the block's parent state. Type-erased so the pool is not /// generic over the provider factory; each worker builds its own per block. -type BuildProviderFn = dyn Fn() -> ProviderResult + Send + Sync; +pub type BuildProviderFn = dyn Fn() -> ProviderResult + Send + Sync; /// A single warm request: a whole account (basic account + its bytecode) or one storage slot. enum PrewarmTarget { @@ -40,7 +42,7 @@ enum PrewarmMsg { /// Long-lived pool of blocking threads that warm the BAL read-set into the shared execution cache. #[derive(Debug)] -pub(crate) struct BalPrewarmPool { +pub struct BalPrewarmPool { /// One queue per worker. `BeginBlock`/`EndBlock` are broadcast to all; `Warm`s round-robin. workers: Vec>, /// Round-robin cursor for distributing warm requests across workers. @@ -51,7 +53,7 @@ pub(crate) struct BalPrewarmPool { impl BalPrewarmPool { /// Spawns `num_threads` long-lived blocking worker threads. Owned by the /// [`PayloadProcessor`](super::PayloadProcessor); the threads exit when the pool is dropped. - pub(crate) fn new(num_threads: usize) -> Arc { + pub fn new(num_threads: usize) -> Arc { let mut workers = Vec::with_capacity(num_threads); let mut handles = Vec::with_capacity(num_threads); for i in 0..num_threads { @@ -70,7 +72,7 @@ impl BalPrewarmPool { /// Begins a block: hands every worker the provider builder and shared cache so each opens its /// own read txn over the parent state. Pair with [`end_block`](Self::end_block). - pub(crate) fn begin_block( + pub fn begin_block( &self, build: Arc, caches: ExecutionCache, @@ -86,12 +88,12 @@ impl BalPrewarmPool { } /// Fire-and-forget: warm an account (basic account + bytecode) on some worker. - pub(crate) fn warm_account(&self, addr: Address) { + pub fn warm_account(&self, addr: Address) { self.send_warm(PrewarmTarget::Account(addr)); } /// Fire-and-forget: warm one storage slot on some worker. - pub(crate) fn warm_storage(&self, addr: Address, slot: StorageKey) { + pub fn warm_storage(&self, addr: Address, slot: StorageKey) { self.send_warm(PrewarmTarget::Storage(addr, slot)); } @@ -99,7 +101,7 @@ impl BalPrewarmPool { /// requests queued ahead of this message. /// /// Blocks until all workers processed the end block message. - pub(crate) fn end_block(&self) { + pub fn end_block(&self) { let (tx, rx) = oneshot::channel(); let tx = Arc::new(SendOnDrop { sender: Some(tx) }); @@ -137,7 +139,7 @@ impl BalPrewarmPool { /// time to finish is 312.5ms at QD=32 and 156.26ms at QD=64. /// /// This should explain why this particular value is picked. -pub(crate) const DEFAULT_BAL_PREWARM_THREADS: usize = 128; +pub const DEFAULT_BAL_PREWARM_THREADS: usize = 128; fn prewarm_loop(rx: crossbeam_channel::Receiver) { // The provider (and its MDBX read txn) held for the current block, between `BeginBlock` and diff --git a/crates/engine/tree/src/tree/payload_processor/mod.rs b/crates/engine/tree/src/tree/payload_processor/mod.rs index 551b0adf4df..88af766be67 100644 --- a/crates/engine/tree/src/tree/payload_processor/mod.rs +++ b/crates/engine/tree/src/tree/payload_processor/mod.rs @@ -42,7 +42,7 @@ use std::{ use tracing::{debug, instrument, trace, warn, Span}; pub mod bal; -pub(crate) mod bal_prewarm_pool; +pub mod bal_prewarm_pool; pub mod prewarm; pub mod receipt_root_task; From 8ead4dc94857a5f3d152ee6f7611e755dde3947e Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Tue, 1 Sep 2026 09:22:17 +0000 Subject: [PATCH 104/294] fix(engine): return invalid for undecodable block access lists (#26889) --- crates/engine/primitives/src/error.rs | 29 ++++++++++++------- crates/engine/tree/src/tree/mod.rs | 12 ++++++-- .../src/tree/payload_processor/bal/execute.rs | 6 ++-- .../engine/tree/src/tree/payload_validator.rs | 4 +-- crates/engine/tree/src/tree/tests.rs | 12 ++++---- .../node/tests/e2e/invalid_payload.rs | 19 +++++------- crates/rpc/rpc-engine-api/src/error.rs | 6 ++-- 7 files changed, 48 insertions(+), 40 deletions(-) diff --git a/crates/engine/primitives/src/error.rs b/crates/engine/primitives/src/error.rs index 939a9ac7f89..0062d47c751 100644 --- a/crates/engine/primitives/src/error.rs +++ b/crates/engine/primitives/src/error.rs @@ -90,23 +90,28 @@ pub enum InsertBlockErrorKind { } impl InsertBlockErrorKind { - /// Returns whether the error was caused by an invalid block. + /// Returns whether the error must be reported as an invalid payload. pub const fn is_validation_error(&self) -> bool { - matches!(self, Self::Consensus(_) | Self::Execution(BlockExecutionError::Validation(_))) + matches!( + self, + Self::Consensus(_) | + Self::BlockAccessListDecode(_) | + Self::Execution(BlockExecutionError::Validation(_)) + ) } - /// Returns an [`InsertBlockValidationError`] if the failure invalidates the block, or an - /// [`InsertBlockProcessingError`] if the failure is not attributable to the block itself. + /// Returns an [`InsertBlockValidationError`] if the failure must be reported as an invalid + /// payload, or an [`InsertBlockProcessingError`] if block processing itself failed. /// - /// This distinction controls whether the block may be returned as `INVALID` and its hash cached - /// as invalid. Because `INVALID` has consensus meaning, malformed supplemental input and - /// internal failures must remain processing errors instead. + /// This distinction controls whether the payload may be returned as `INVALID`. Errors + /// classified as invalid payloads by the Engine API are validation errors; malformed request + /// parameters and internal failures remain processing errors instead. pub fn ensure_validation_error( self, ) -> Result { match self { Self::BlockAccessListDecode(error) => { - Err(InsertBlockProcessingError::MalformedInput(Box::new(error))) + Ok(InsertBlockValidationError::BlockAccessListDecode(error)) } Self::Consensus(err) => Ok(InsertBlockValidationError::Consensus(err)), Self::Execution(err) => match err { @@ -176,6 +181,9 @@ pub enum InsertBlockValidationError { /// Block violated consensus rules. #[error(transparent)] Consensus(#[from] ConsensusError), + /// Block access list bytes could not be decoded. + #[error(transparent)] + BlockAccessListDecode(#[from] BlockAccessListDecodeError), /// Validation error, transparently wrapping [`BlockValidationError`]. #[error(transparent)] Validation(#[from] BlockValidationError), @@ -185,16 +193,15 @@ pub enum InsertBlockValidationError { mod tests { use super::*; - // Undecodable block access list bytes are malformed request params and must not be treated - // as a block validation error. #[test] fn ensure_insert_block_validation_error() { let err = InsertBlockErrorKind::BlockAccessListDecode(BlockAccessListDecodeError::new( alloy_rlp::Error::UnexpectedString, )); + assert!(err.is_validation_error()); assert!(matches!( err.ensure_validation_error(), - Err(InsertBlockProcessingError::MalformedInput(_)) + Ok(InsertBlockValidationError::BlockAccessListDecode(_)) )); assert!(matches!( diff --git a/crates/engine/tree/src/tree/mod.rs b/crates/engine/tree/src/tree/mod.rs index a85c050d151..265e375d498 100644 --- a/crates/engine/tree/src/tree/mod.rs +++ b/crates/engine/tree/src/tree/mod.rs @@ -3314,9 +3314,15 @@ where %validation_err, "Invalid block error on new payload", ); - let latest_valid_hash = self - .latest_valid_hash_for_invalid_payload(block.parent_hash()) - .map_err(InsertBlockFatalError::from)?; + // The Amsterdam Engine API requires `latestValidHash: null` for an undecodable BAL. + // + let latest_valid_hash = + if matches!(&validation_err, InsertBlockValidationError::BlockAccessListDecode(_)) { + None + } else { + self.latest_valid_hash_for_invalid_payload(block.parent_hash()) + .map_err(InsertBlockFatalError::from)? + }; // keep track of the invalid header unless the consensus impl considers it transient let is_transient = match &validation_err { diff --git a/crates/engine/tree/src/tree/payload_processor/bal/execute.rs b/crates/engine/tree/src/tree/payload_processor/bal/execute.rs index 887539cd3d8..11d155bdb8d 100644 --- a/crates/engine/tree/src/tree/payload_processor/bal/execute.rs +++ b/crates/engine/tree/src/tree/payload_processor/bal/execute.rs @@ -326,7 +326,7 @@ impl BlockGasTracker { #[cfg(test)] mod tests { use super::*; - use crate::tree::error::{InsertBlockErrorKind, InsertBlockProcessingError}; + use crate::tree::error::{InsertBlockErrorKind, InsertBlockValidationError}; use alloy_consensus::{BlockHeader, Header}; use alloy_eip7928::{ bal::Bal as AlloyBal, AccountChanges, BlockAccessIndex, BlockAccessList, CodeChange, @@ -451,7 +451,7 @@ mod tests { } #[test] - fn invalid_bal_bytecode_is_malformed_input() { + fn invalid_bal_bytecode_is_validation_error() { let alloy_bal = vec![AccountChanges { address: Address::ZERO, code_changes: vec![CodeChange::new( @@ -468,7 +468,7 @@ mod tests { assert!(matches!(&error, InsertBlockErrorKind::BlockAccessListDecode(_))); assert!(matches!( error.ensure_validation_error(), - Err(InsertBlockProcessingError::MalformedInput(_)) + Ok(InsertBlockValidationError::BlockAccessListDecode(_)) )); } diff --git a/crates/engine/tree/src/tree/payload_validator.rs b/crates/engine/tree/src/tree/payload_validator.rs index e586c23e8ff..2472a5511aa 100644 --- a/crates/engine/tree/src/tree/payload_validator.rs +++ b/crates/engine/tree/src/tree/payload_validator.rs @@ -576,8 +576,8 @@ where .in_scope(|| self.evm_env_for(&input)) .map_err(NewPayloadError::other)?; - // Extract the decoded BAL, if present. Undecodable block access list bytes are malformed - // request params, not an invalid block. + // Extract the decoded BAL, if present. Undecodable block access list bytes invalidate the + // payload. let decoded_bal = ensure_ok!(input.try_decoded_access_list().map_err(BlockAccessListDecodeError::new)) .map(Arc::new); diff --git a/crates/engine/tree/src/tree/tests.rs b/crates/engine/tree/src/tree/tests.rs index fa1b0130324..df998ccac14 100644 --- a/crates/engine/tree/src/tree/tests.rs +++ b/crates/engine/tree/src/tree/tests.rs @@ -565,19 +565,19 @@ fn test_tree_persist_block_batch() { } #[test] -fn malformed_input_is_non_fatal_insert_outcome() { +fn block_access_list_decode_returns_invalid_with_null_latest_valid_hash() { let mut test_harness = TestHarness::new(MAINNET.clone()); - let block = test_harness.block_builder.generate_random_block(1, B256::ZERO); + let parent_hash = test_harness.tree.state.tree_state.canonical_block_hash(); + let block = test_harness.block_builder.generate_random_block(1, parent_hash); let error = InsertBlockError::new( block, InsertBlockErrorKind::BlockAccessListDecode(BlockAccessListDecodeError::new( alloy_rlp::Error::UnexpectedString, )), ); - assert_matches!( - test_harness.tree.on_insert_block_error(error), - Err(InsertBlockProcessingError::MalformedInput(_)) - ); + let status = test_harness.tree.on_insert_block_error(error).unwrap(); + assert!(status.is_invalid()); + assert_eq!(status.latest_valid_hash, None); } #[tokio::test] diff --git a/crates/ethereum/node/tests/e2e/invalid_payload.rs b/crates/ethereum/node/tests/e2e/invalid_payload.rs index 7d9cc1110e7..42d9fa376c5 100644 --- a/crates/ethereum/node/tests/e2e/invalid_payload.rs +++ b/crates/ethereum/node/tests/e2e/invalid_payload.rs @@ -7,7 +7,6 @@ use crate::utils::{eth_payload_attributes, eth_payload_attributes_amsterdam}; use alloy_eips::eip7685::RequestsOrHash; use alloy_primitives::{keccak256, Bytes, B256}; use alloy_rpc_types_engine::{ExecutionPayloadV3, PayloadStatusEnum}; -use jsonrpsee::types::error::INVALID_PARAMS_CODE; use rand::{rngs::StdRng, Rng, SeedableRng}; use reth_chainspec::{ChainSpecBuilder, MAINNET}; use reth_e2e_test_utils::{setup_engine, transaction::TransactionTestContext}; @@ -358,10 +357,10 @@ async fn can_handle_invalid_payload_with_transactions() -> eyre::Result<()> { Ok(()) } -/// Tests that `engine_newPayloadV5` rejects undecodable block access list bytes with an invalid -/// params error (`-32602`) instead of an `INVALID` payload status. +/// Tests that `engine_newPayloadV5` returns `INVALID` with no latest valid hash for undecodable +/// block access list bytes. #[tokio::test] -async fn undecodable_bal_is_rejected_as_invalid_params() -> eyre::Result<()> { +async fn undecodable_bal_is_invalid_payload() -> eyre::Result<()> { reth_tracing::init_test_tracing(); let chain_spec = Arc::new( @@ -402,20 +401,16 @@ async fn undecodable_bal_is_rejected_as_invalid_params() -> eyre::Result<()> { let engine = node.auth_server_handle().http_client(); let parent_beacon_block_root = block.header().parent_beacon_block_root.unwrap(); - let err = EngineApiClient::::new_payload_v5( + let invalid_status = EngineApiClient::::new_payload_v5( &engine, corrupted, vec![], parent_beacon_block_root, RequestsOrHash::Requests(envelope.execution_requests.clone()), ) - .await - .unwrap_err(); - - let jsonrpsee::core::client::Error::Call(err) = err else { - panic!("expected an invalid params error object, got {err:?}") - }; - assert_eq!(err.code(), INVALID_PARAMS_CODE); + .await?; + assert!(matches!(invalid_status.status, PayloadStatusEnum::Invalid { .. })); + assert_eq!(invalid_status.latest_valid_hash, None); // The same block with well-formed block access list bytes is processed normally. let status = EngineApiClient::::new_payload_v5( diff --git a/crates/rpc/rpc-engine-api/src/error.rs b/crates/rpc/rpc-engine-api/src/error.rs index 9b5709109ea..7c7dbb3b1ea 100644 --- a/crates/rpc/rpc-engine-api/src/error.rs +++ b/crates/rpc/rpc-engine-api/src/error.rs @@ -302,13 +302,13 @@ mod tests { EngineApiError::UnknownPayload, ); - // Malformed payload params, e.g. undecodable block access list bytes, are rejected with - // an invalid params error instead of an `INVALID` payload status. + // Processing errors explicitly classified as malformed request parameters are rejected + // with an invalid params error instead of an `INVALID` payload status. ensure_engine_rpc_error( INVALID_PARAMS_CODE, INVALID_PARAMS_MSG, EngineApiError::NewPayload(BeaconOnNewPayloadError::InvalidParams( - "undecodable block access list".into(), + "malformed payload parameters".into(), )), ); From 498847cb2e2847c8740d2e9f4a35ea4c67f09a5c Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Tue, 1 Sep 2026 11:33:14 +0200 Subject: [PATCH 105/294] chore(rpc): fix unused ChainBlockTraceResult import warning (#26869) --- crates/rpc/rpc-api/src/debug.rs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/crates/rpc/rpc-api/src/debug.rs b/crates/rpc/rpc-api/src/debug.rs index 571da6f8b15..7b2a6eaf353 100644 --- a/crates/rpc/rpc-api/src/debug.rs +++ b/crates/rpc/rpc-api/src/debug.rs @@ -5,8 +5,7 @@ use alloy_primitives::{Address, Bytes, B256, U64}; use alloy_rpc_types_debug::ExecutionWitness; use alloy_rpc_types_eth::{Account, AccountInfo, Bundle, Index, StateContext}; use alloy_rpc_types_trace::geth::{ - ChainBlockTraceResult, GethDebugTracingCallOptions, GethDebugTracingOptions, GethTrace, - TraceResult, + GethDebugTracingCallOptions, GethDebugTracingOptions, GethTrace, TraceResult, }; use jsonrpsee::{core::RpcResult, proc_macros::rpc}; use reth_trie_common::{updates::TrieUpdates, ExecutionWitnessMode, HashedPostState}; @@ -54,7 +53,7 @@ pub trait DebugApi { #[subscription( name = "subscribe" => "subscription", unsubscribe = "unsubscribe", - item = ChainBlockTraceResult + item = alloy_rpc_types_trace::geth::ChainBlockTraceResult )] async fn debug_subscribe( &self, From 78579e61d54eeec72c69e6b4142e4abe42d7ec9a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A9a=20Narzis?= <78718413+lean-apple@users.noreply.github.com> Date: Tue, 1 Sep 2026 11:00:18 +0000 Subject: [PATCH 106/294] feat(net): add authenticated snap block access list downloader (#26885) --- .../downloaders/src/snap/block_access_list.rs | 656 ++++++++++++++++++ crates/net/downloaders/src/snap/mod.rs | 2 + crates/net/downloaders/src/snap/request.rs | 10 +- crates/net/downloaders/src/snap/test_utils.rs | 7 +- 4 files changed, 670 insertions(+), 5 deletions(-) create mode 100644 crates/net/downloaders/src/snap/block_access_list.rs diff --git a/crates/net/downloaders/src/snap/block_access_list.rs b/crates/net/downloaders/src/snap/block_access_list.rs new file mode 100644 index 00000000000..020efdd691b --- /dev/null +++ b/crates/net/downloaders/src/snap/block_access_list.rs @@ -0,0 +1,656 @@ +//! Downloads block access lists and authenticates them against header commitments. +//! +//! Responses are positional: entry `i` answers the `i`th requested block hash, and is accepted +//! only if it hashes to the commitment carried by that block's header, as defined by +//! [EIP-8189](https://eips.ethereum.org/EIPS/eip-8189). + +use super::request::{SnapVerifier, VerifyingRequest}; +use alloy_consensus::BlockHeader; +use alloy_eips::eip7928::bal::{DecodedBal, RawBal}; +use alloy_primitives::{Bytes, Sealable, B256}; +use futures::Future; +use reth_eth_wire_types::snap::{BlockAccessListsMessage, GetBlockAccessListsMessage}; +use reth_network_p2p::{ + error::RequestError, + snap::client::{SnapClient, SnapResponse}, +}; +use reth_network_peers::PeerId; +use reth_primitives_traits::SealedHeader; +use reth_tasks::Runtime; +use std::{ + pin::Pin, + task::{Context, Poll}, +}; +use tracing::debug; + +/// Downloads block access lists and authenticates each against its header commitment. +/// +/// Invalid responses penalize their peer and retry. Decoding and hashing run on the blocking +/// pool. +#[derive(Debug)] +pub struct BlockAccessListDownloader(VerifyingRequest); + +impl BlockAccessListDownloader { + /// Creates a downloader that verifies responses against `headers`. + /// + /// Headers must match the requested block hashes in order and carry their block-access-list + /// commitments. + pub fn new( + client: C, + request: GetBlockAccessListsMessage, + headers: &[SealedHeader], + runtime: Runtime, + ) -> Result { + if request.block_hashes.is_empty() { + return Err(InvalidBlockAccessListRequest::NoBlocks) + } + if request.block_hashes.len() != headers.len() { + return Err(InvalidBlockAccessListRequest::HeaderCount { + requested: request.block_hashes.len(), + supplied: headers.len(), + }) + } + + // Only authenticated block identities cross into blocking work, so the verifier stays + // free of the caller's header type. + let mut blocks = Vec::with_capacity(headers.len()); + for (index, (requested, header)) in request.block_hashes.iter().zip(headers).enumerate() { + if *requested != header.hash() { + return Err(InvalidBlockAccessListRequest::HashMismatch { + index, + requested: *requested, + supplied: header.hash(), + }) + } + let Some(commitment) = header.block_access_list_hash() else { + return Err(InvalidBlockAccessListRequest::MissingCommitment { + index, + block_hash: *requested, + }) + }; + blocks.push((*requested, commitment)); + } + + let verifier = BlockAccessListVerifier { + request_id: request.request_id, + response_bytes: request.response_bytes, + blocks, + }; + Ok(Self(VerifyingRequest::new(client, request, verifier, runtime))) + } +} + +impl Future for BlockAccessListDownloader +where + C: SnapClient + Unpin, +{ + type Output = Result; + + fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + self.get_mut().0.poll_verified(cx) + } +} + +/// Result of an authenticated block-access-lists request. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum BlockAccessListOutcome { + /// The peer serves snap but holds none of the requested lists, and was not penalized. + Unavailable { + /// The peer that answered. + peer_id: PeerId, + }, + /// Lists authenticated against their header commitments. + Verified(VerifiedBlockAccessLists), +} + +/// Positional block access lists authenticated against their header commitments. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct VerifiedBlockAccessLists { + // Needed to avoid retrying unavailable entries against the peer that omitted them. + peer_id: PeerId, + // Each list stays bound to the block whose header commitment authenticated it. + block_access_lists: Vec<(B256, Option)>, + // Requested blocks the response left unanswered, in request order. + missing: Vec, + // Soft byte limit of the answered request, so a follow-up is bounded the same way. + response_bytes: u64, +} + +impl VerifiedBlockAccessLists { + /// Peer that returned these lists. + pub const fn peer_id(&self) -> PeerId { + self.peer_id + } + + /// Block hashes and lists in requested order, with `None` where the peer had none. + pub fn block_access_lists(&self) -> &[(B256, Option)] { + &self.block_access_lists + } + + /// Consumes the result without separating lists from their authenticated block hashes. + pub fn into_block_access_lists(self) -> Vec<(B256, Option)> { + self.block_access_lists + } + + /// Request the blocks left unanswered, or `None` once every list is authenticated. + /// The returned request contains only unanswered hashes, preserving authenticated lists. + pub fn follow_up(&self, request_id: u64) -> Option { + (!self.missing.is_empty()).then(|| GetBlockAccessListsMessage { + request_id, + block_hashes: self.missing.clone(), + response_bytes: self.response_bytes, + }) + } + + /// Requested blocks this response left unanswered, in request order. + /// This includes in-place omissions and any truncated suffix. + pub fn missing(&self) -> &[B256] { + &self.missing + } +} + +/// A block-access-lists request that cannot be authenticated by the headers supplied with it. +#[derive(Clone, Copy, Debug, PartialEq, Eq, thiserror::Error)] +pub enum InvalidBlockAccessListRequest { + /// The request asks for no blocks. + #[error("block access list request has no block hashes")] + NoBlocks, + /// A different number of headers was supplied than blocks requested. + #[error("requested {requested} block access lists but supplied {supplied} headers")] + HeaderCount { + /// Blocks the request asks for. + requested: usize, + /// Headers supplied to authenticate them. + supplied: usize, + }, + /// A requested hash does not match the header at the same position. + #[error("requested block {requested} at index {index} but header is for {supplied}")] + HashMismatch { + /// Position the mismatch was found at. + index: usize, + /// Hash the request asks for. + requested: B256, + /// Hash of the header supplied for it. + supplied: B256, + }, + /// A supplied header carries no commitment, so its list could never be authenticated. + #[error("header for block {block_hash} at index {index} has no block access list commitment")] + MissingCommitment { + /// Position of the header without a commitment. + index: usize, + /// Block the header belongs to. + block_hash: B256, + }, +} + +// Authenticates each returned list against the commitment of the block it answers. +// +// Keeps only response identity and authenticated block pairs, avoiding the caller's generic +// header type on the blocking pool. +#[derive(Clone, Debug)] +struct BlockAccessListVerifier { + // Matches the response to the request that asked for it. + request_id: u64, + // Soft byte limit the request was sent with, carried into any follow-up. + response_bytes: u64, + // Requested hashes paired with their header commitments, in wire order. + blocks: Vec<(B256, B256)>, +} + +impl BlockAccessListVerifier { + // Checks that the response can be paired with this request before its entries are decoded. + fn validate_response( + &self, + response: SnapResponse, + ) -> Result { + let SnapResponse::BlockAccessLists(response) = response else { + debug!(target: "downloaders::snap", "Expected block access lists response"); + return Err(RequestError::BadResponse) + }; + if response.request_id != self.request_id { + debug!( + target: "downloaders::snap", + expected = self.request_id, + got = response.request_id, + "Block access lists response id mismatch" + ); + return Err(RequestError::BadResponse) + } + if response.block_access_lists.0.len() > self.blocks.len() { + debug!( + target: "downloaders::snap", + requested = self.blocks.len(), + got = response.block_access_lists.0.len(), + "Block access lists response is longer than the request" + ); + return Err(RequestError::BadResponse) + } + + Ok(response) + } + + // Keeps each decoded entry tied to the commitment at the same request position. + fn authenticate_entries( + &self, + entries: Vec>, + ) -> Result)>, RequestError> { + let mut block_access_lists = Vec::with_capacity(entries.len()); + for (index, entry) in entries.into_iter().enumerate() { + let (block_hash, commitment) = self.blocks[index]; + // An omitted list stays in place, so every later entry keeps the commitment it + // answers. + let Some(raw) = entry else { + block_access_lists.push((block_hash, None)); + continue + }; + // Hashing the raw bytes settles authenticity without decoding, so a peer cannot + // charge us the decode of a list it was never able to serve. + let raw = RawBal::new(raw); + if raw.hash() != commitment { + debug!( + target: "downloaders::snap", + %block_hash, + expected = %commitment, + got = %raw.hash(), + "Block access list does not match its header commitment" + ); + return Err(RequestError::BadResponse) + } + let decoded = DecodedBal::from_raw_bal(raw).map_err(|error| { + debug!(target: "downloaders::snap", %block_hash, %error, "Invalid block access list"); + RequestError::BadResponse + })?; + block_access_lists.push((block_hash, Some(decoded))); + } + + Ok(block_access_lists) + } +} + +impl SnapVerifier for BlockAccessListVerifier { + type Request = GetBlockAccessListsMessage; + type Output = BlockAccessListOutcome; + + // Validates the response identity and authenticates every supplied block access list. + fn verify(self, peer_id: PeerId, response: SnapResponse) -> Result { + let entries = self.validate_response(response)?.block_access_lists.0; + // An empty response is the peer's explicit statement that it has none of these lists. + if entries.is_empty() { + return Ok(BlockAccessListOutcome::Unavailable { peer_id }) + } + + let block_access_lists = self.authenticate_entries(entries)?; + // Omitted entries and a cut at the peer's soft byte limit both leave blocks unanswered, + // and neither invalidates the entries around them. + let missing = block_access_lists + .iter() + .filter_map(|(block_hash, list)| list.is_none().then_some(*block_hash)) + .chain( + self.blocks[block_access_lists.len()..].iter().map(|(block_hash, _)| *block_hash), + ) + .collect::>(); + // A full response of omitted entries is unavailable, a shorter one can be byte-limited. + // Preserve the latter's omitted suffix for resumption. + if block_access_lists.len() == self.blocks.len() && missing.len() == self.blocks.len() { + return Ok(BlockAccessListOutcome::Unavailable { peer_id }) + } + + Ok(BlockAccessListOutcome::Verified(VerifiedBlockAccessLists { + peer_id, + block_access_lists, + missing, + response_bytes: self.response_bytes, + })) + } +} + +#[cfg(test)] +mod tests { + use super::{ + super::{request::MAX_RETRIES, test_utils::TestSnapClient}, + *, + }; + use alloy_consensus::Header; + use alloy_eips::eip7928::bal::Bal; + use alloy_primitives::Bytes; + use reth_eth_wire_types::{ + snap::{AccountRangeMessage, BlockAccessListsMessage, ByteCodesMessage}, + BlockAccessLists, + }; + use reth_network_p2p::{error::PeerRequestResult, priority::Priority}; + use reth_network_peers::WithPeerId; + use std::sync::Arc; + + fn bal() -> Bytes { + Bytes::from(alloy_rlp::encode(Bal::default())) + } + + fn commitment(raw: Bytes) -> B256 { + DecodedBal::from_rlp_bytes(raw).expect("test bal decodes").hash() + } + + // Headers committing to `entries`, sealed with distinct block hashes. A `None` entry gets a + // commitment no list can match, since nothing authenticates against it. + fn headers(entries: &[Option]) -> Vec> { + entries + .iter() + .enumerate() + .map(|(index, raw)| { + let header = Header { + block_access_list_hash: Some( + raw.clone().map_or(B256::repeat_byte(0xee), commitment), + ), + ..Default::default() + }; + SealedHeader::new(header, B256::repeat_byte(index as u8 + 1)) + }) + .collect() + } + + fn request(headers: &[SealedHeader
]) -> GetBlockAccessListsMessage { + GetBlockAccessListsMessage { + request_id: 1, + block_hashes: headers.iter().map(SealedHeader::hash).collect(), + response_bytes: 512 * 1024, + } + } + + fn response( + peer: PeerId, + request_id: u64, + entries: Vec>, + ) -> PeerRequestResult { + Ok(WithPeerId::new( + peer, + SnapResponse::BlockAccessLists(BlockAccessListsMessage { + request_id, + block_access_lists: BlockAccessLists(entries), + }), + )) + } + + // A response that can never authenticate a block access list. + fn unverifiable(peer: PeerId) -> PeerRequestResult { + Ok(WithPeerId::new( + peer, + SnapResponse::ByteCodes(ByteCodesMessage { request_id: 1, codes: Vec::new() }), + )) + } + + // Every attempt gets the same answer, so a rejected response exhausts the retry budget. + fn always( + peer: PeerId, + request_id: u64, + entries: Vec>, + ) -> impl Iterator> { + std::iter::repeat_with(move || response(peer, request_id, entries.clone())) + .take(usize::from(MAX_RETRIES) + 1) + } + + fn downloader( + client: Arc, + request: GetBlockAccessListsMessage, + headers: &[SealedHeader
], + ) -> Result>, InvalidBlockAccessListRequest> { + BlockAccessListDownloader::new(client, request, headers, Runtime::test()) + } + + fn verified(outcome: BlockAccessListOutcome) -> VerifiedBlockAccessLists { + match outcome { + BlockAccessListOutcome::Verified(verified) => verified, + BlockAccessListOutcome::Unavailable { .. } => panic!("expected verified lists"), + } + } + + #[tokio::test] + async fn present_and_omitted_entries_keep_their_positions() { + let entries = vec![Some(bal()), None, Some(bal())]; + let headers = headers(&entries); + let peer = PeerId::random(); + let client = Arc::new(TestSnapClient::new([response(peer, 1, entries)])); + + let outcome = + downloader(Arc::clone(&client), request(&headers), &headers).unwrap().await.unwrap(); + + let verified = verified(outcome); + assert_eq!(verified.peer_id(), peer); + assert_eq!( + verified + .block_access_lists() + .iter() + .map(|(block_hash, entry)| (*block_hash, entry.as_ref().map(DecodedBal::hash))) + .collect::>(), + [ + (headers[0].hash(), Some(commitment(bal()))), + (headers[1].hash(), None), + (headers[2].hash(), Some(commitment(bal()))), + ] + ); + // The list after the omission stays authenticated, so only the gap is asked for again. + assert_eq!(verified.missing(), [headers[1].hash()]); + assert_eq!( + verified.follow_up(2), + Some(GetBlockAccessListsMessage { + request_id: 2, + block_hashes: vec![headers[1].hash()], + response_bytes: request(&headers).response_bytes, + }) + ); + assert!(client.reported().is_empty()); + } + + #[tokio::test] + async fn a_complete_response_needs_no_follow_up() { + let entries = vec![Some(bal()), Some(bal())]; + let headers = headers(&entries); + let client = Arc::new(TestSnapClient::new([response(PeerId::random(), 1, entries)])); + + let outcome = + downloader(Arc::clone(&client), request(&headers), &headers).unwrap().await.unwrap(); + + let verified = verified(outcome); + assert!(verified.missing().is_empty()); + assert_eq!(verified.follow_up(2), None); + } + + #[tokio::test] + async fn a_truncated_response_leaves_the_rest_of_the_request_unanswered() { + let entries = vec![Some(bal()), Some(bal()), Some(bal())]; + let headers = headers(&entries); + let client = + Arc::new(TestSnapClient::new([response(PeerId::random(), 1, entries[..2].to_vec())])); + + let outcome = + downloader(Arc::clone(&client), request(&headers), &headers).unwrap().await.unwrap(); + + let verified = verified(outcome); + assert_eq!(verified.missing(), [headers[2].hash()]); + assert_eq!(verified.block_access_lists().len(), 2); + assert_eq!(verified.block_access_lists()[0].0, headers[0].hash()); + assert_eq!(verified.block_access_lists()[1].0, headers[1].hash()); + assert!(client.reported().is_empty()); + } + + #[tokio::test] + async fn an_empty_or_complete_response_holding_no_lists_is_unavailable() { + let headers = headers(&[Some(bal()), Some(bal())]); + let peer = PeerId::random(); + for entries in [Vec::new(), vec![None, None]] { + let client = Arc::new(TestSnapClient::new([response(peer, 1, entries)])); + + let outcome = downloader(Arc::clone(&client), request(&headers), &headers) + .unwrap() + .await + .unwrap(); + + assert_eq!(outcome, BlockAccessListOutcome::Unavailable { peer_id: peer }); + assert!(client.reported().is_empty()); + } + } + + #[tokio::test] + async fn a_truncated_response_after_an_omission_stays_resumable() { + let headers = headers(&[Some(bal()), Some(bal())]); + let peer = PeerId::random(); + let client = Arc::new(TestSnapClient::new([response(peer, 1, vec![None])])); + + let outcome = + downloader(Arc::clone(&client), request(&headers), &headers).unwrap().await.unwrap(); + + let verified = verified(outcome); + assert_eq!(verified.missing(), [headers[0].hash(), headers[1].hash()]); + assert_eq!( + verified.follow_up(2), + Some(GetBlockAccessListsMessage { + request_id: 2, + block_hashes: headers.iter().map(SealedHeader::hash).collect(), + response_bytes: request(&headers).response_bytes, + }) + ); + assert!(client.reported().is_empty()); + } + + #[tokio::test] + async fn an_invalid_list_is_reported_and_retried() { + let entries = vec![Some(bal())]; + let headers = headers(&entries); + let bad_peer = PeerId::random(); + let client = Arc::new(TestSnapClient::new([ + response(bad_peer, 1, vec![Some(Bytes::from_static(&[0xff, 0xff]))]), + response(PeerId::random(), 1, entries), + ])); + + let outcome = + downloader(Arc::clone(&client), request(&headers), &headers).unwrap().await.unwrap(); + + assert_eq!(verified(outcome).block_access_lists().len(), 1); + assert_eq!(*client.reported(), [bad_peer]); + assert_eq!(*client.priorities(), [Priority::Normal, Priority::High]); + } + + #[tokio::test] + async fn a_list_that_misses_its_header_commitment_is_rejected() { + // The header commits to something this list cannot hash to. + let headers = vec![SealedHeader::new( + Header { block_access_list_hash: Some(B256::repeat_byte(0xab)), ..Default::default() }, + B256::repeat_byte(1), + )]; + let peer = PeerId::random(); + let client = Arc::new(TestSnapClient::new(always(peer, 1, vec![Some(bal())]))); + + let error = downloader(Arc::clone(&client), request(&headers), &headers) + .unwrap() + .await + .unwrap_err(); + + assert_eq!(error, RequestError::BadResponse); + assert_eq!(client.reported().len(), usize::from(MAX_RETRIES) + 1); + } + + #[tokio::test] + async fn dropping_an_entry_instead_of_omitting_it_in_place_is_rejected() { + // The peer holds no list for the first block and drops the slot rather than sending + // `None`, so the second block's list lands on the first block's commitment. + let headers = headers(&[None, Some(bal())]); + let peer = PeerId::random(); + let client = Arc::new(TestSnapClient::new(always(peer, 1, vec![Some(bal())]))); + + let error = downloader(Arc::clone(&client), request(&headers), &headers) + .unwrap() + .await + .unwrap_err(); + + assert_eq!(error, RequestError::BadResponse); + assert_eq!(client.reported().len(), usize::from(MAX_RETRIES) + 1); + } + + #[tokio::test] + async fn malformed_wrong_id_and_oversized_responses_are_rejected() { + let headers = headers(&[Some(bal())]); + let peer = PeerId::random(); + for (request_id, entries) in [ + // undecodable payload + (1, vec![Some(Bytes::from_static(&[0xff, 0xff]))]), + // answers a different request + (7, vec![Some(bal())]), + // more entries than blocks requested + (1, vec![Some(bal()), Some(bal())]), + ] { + let client = Arc::new(TestSnapClient::new(always(peer, request_id, entries))); + + let error = downloader(Arc::clone(&client), request(&headers), &headers) + .unwrap() + .await + .unwrap_err(); + + assert_eq!(error, RequestError::BadResponse); + assert_eq!(client.reported().len(), usize::from(MAX_RETRIES) + 1); + } + } + + #[tokio::test] + async fn a_wrong_response_type_exhausts_the_retry_budget() { + let headers = headers(&[Some(bal())]); + let peers = [PeerId::random(), PeerId::random(), PeerId::random()]; + let client = Arc::new(TestSnapClient::new(peers.map(unverifiable))); + + let error = downloader(Arc::clone(&client), request(&headers), &headers) + .unwrap() + .await + .unwrap_err(); + + assert_eq!(error, RequestError::BadResponse); + assert_eq!(*client.reported(), peers); + assert_eq!(*client.priorities(), [Priority::Normal, Priority::High, Priority::High]); + } + + #[test] + fn requests_that_cannot_be_authenticated_are_rejected_before_submission() { + let headers = headers(&[Some(bal())]); + let client = Arc::new(TestSnapClient::new(std::iter::empty())); + + let mut empty = request(&headers); + empty.block_hashes.clear(); + assert_eq!( + downloader(Arc::clone(&client), empty, &[]).unwrap_err(), + InvalidBlockAccessListRequest::NoBlocks + ); + + assert!(matches!( + downloader(Arc::clone(&client), request(&headers), &[]).unwrap_err(), + InvalidBlockAccessListRequest::HeaderCount { requested: 1, supplied: 0 } + )); + + let mut wrong_block = request(&headers); + wrong_block.block_hashes[0] = B256::repeat_byte(0x99); + assert!(matches!( + downloader(Arc::clone(&client), wrong_block, &headers).unwrap_err(), + InvalidBlockAccessListRequest::HashMismatch { index: 0, .. } + )); + + let uncommitted = vec![SealedHeader::new(Header::default(), headers[0].hash())]; + assert!(matches!( + downloader(Arc::clone(&client), request(&headers), &uncommitted).unwrap_err(), + InvalidBlockAccessListRequest::MissingCommitment { index: 0, .. } + )); + + // Nothing reached the network. + assert!(client.priorities().is_empty()); + } + + #[test] + fn a_response_of_another_kind_is_rejected() { + let verifier = BlockAccessListVerifier { + request_id: 1, + response_bytes: 512 * 1024, + blocks: vec![(B256::repeat_byte(1), commitment(bal()))], + }; + let wrong = SnapResponse::AccountRange(AccountRangeMessage { + request_id: 1, + accounts: Vec::new(), + proof: Vec::new(), + }); + + assert_eq!(verifier.verify(PeerId::random(), wrong), Err(RequestError::BadResponse)); + } +} diff --git a/crates/net/downloaders/src/snap/mod.rs b/crates/net/downloaders/src/snap/mod.rs index 80d4d3c4b38..dfa0d7fc4ca 100644 --- a/crates/net/downloaders/src/snap/mod.rs +++ b/crates/net/downloaders/src/snap/mod.rs @@ -22,11 +22,13 @@ use std::{ }; use tracing::debug; +mod block_access_list; mod request; mod storage; #[cfg(test)] mod test_utils; +pub use block_access_list::*; use request::{SnapVerifier, VerifyingRequest}; pub use storage::*; diff --git a/crates/net/downloaders/src/snap/request.rs b/crates/net/downloaders/src/snap/request.rs index 6ef539c5270..6108a871457 100644 --- a/crates/net/downloaders/src/snap/request.rs +++ b/crates/net/downloaders/src/snap/request.rs @@ -4,7 +4,9 @@ //! penalizes and reissues in exactly the same way. use futures::FutureExt; -use reth_eth_wire_types::snap::{GetAccountRangeMessage, GetStorageRangesMessage}; +use reth_eth_wire_types::snap::{ + GetAccountRangeMessage, GetBlockAccessListsMessage, GetStorageRangesMessage, +}; use reth_network_p2p::{ error::RequestError, priority::Priority, @@ -159,6 +161,12 @@ impl SnapRequest for GetStorageRangesMessage { } } +impl SnapRequest for GetBlockAccessListsMessage { + fn send(&self, client: &C, priority: Priority) -> C::Output { + client.get_block_access_lists_with_priority(self.clone(), priority) + } +} + /// Authenticates a snap response against the request that asked for it. pub(super) trait SnapVerifier: Clone + Send + 'static { /// Request type this verifier authenticates responses for. diff --git a/crates/net/downloaders/src/snap/test_utils.rs b/crates/net/downloaders/src/snap/test_utils.rs index ace9e52d388..1e134eb5b62 100644 --- a/crates/net/downloaders/src/snap/test_utils.rs +++ b/crates/net/downloaders/src/snap/test_utils.rs @@ -17,8 +17,7 @@ use std::{ sync::{Mutex, MutexGuard}, }; -/// Answers account and storage requests from one queue, so retries across request kinds stay -/// observable in submission order. +/// Serves queued responses so retries remain observable in submission order. #[derive(Debug)] pub(super) struct TestSnapClient { responses: Mutex>>, @@ -102,9 +101,9 @@ impl SnapClient for TestSnapClient { fn get_block_access_lists_with_priority( &self, _request: GetBlockAccessListsMessage, - _priority: Priority, + priority: Priority, ) -> Self::Output { - unsupported() + self.next(priority) } } From c9cf89ffc5ec0a51c061ce97672906145594b31f Mon Sep 17 00:00:00 2001 From: Derek Cofausper <256792747+decofe@users.noreply.github.com> Date: Tue, 1 Sep 2026 11:30:59 +0000 Subject: [PATCH 107/294] feat(rpc): make V2 multiproofs the default (#26851) Co-authored-by: Alexey Shekhirin <5773434+shekhirin@users.noreply.github.com> Co-authored-by: Brian Picciano Co-authored-by: Alexey Shekhirin --- crates/chain-state/src/in_memory.rs | 8 ++ crates/chain-state/src/memory_overlay.rs | 13 +- .../execution-cache/src/cached_state.rs | 8 ++ .../tree/src/tree/instrumented_state.rs | 8 ++ crates/revm/src/test_utils.rs | 8 ++ crates/rpc/rpc-eth-api/src/helpers/state.rs | 15 ++- .../src/providers/state/historical.rs | 24 +++- .../provider/src/providers/state/latest.rs | 17 ++- .../storage/provider/src/test_utils/mock.rs | 8 ++ crates/storage/rpc-provider/src/lib.rs | 8 ++ crates/storage/storage-api/src/macros.rs | 1 + crates/storage/storage-api/src/noop.rs | 13 +- crates/storage/storage-api/src/trie.rs | 15 ++- crates/trie/common/src/proofs.rs | 121 +++++++++++++++++- crates/trie/db/src/proof.rs | 26 +++- crates/trie/db/tests/proof.rs | 48 ++++++- 16 files changed, 316 insertions(+), 25 deletions(-) diff --git a/crates/chain-state/src/in_memory.rs b/crates/chain-state/src/in_memory.rs index d58da478321..ef0fa4fabb3 100644 --- a/crates/chain-state/src/in_memory.rs +++ b/crates/chain-state/src/in_memory.rs @@ -1176,6 +1176,14 @@ mod tests { Ok(MultiProof::default()) } + fn multiproof_v2( + &self, + _input: TrieInput, + _targets: reth_trie::MultiProofTargetsV2, + ) -> ProviderResult { + Ok(reth_trie::DecodedMultiProofV2::default()) + } + fn witness( &self, _input: TrieInput, diff --git a/crates/chain-state/src/memory_overlay.rs b/crates/chain-state/src/memory_overlay.rs index b61cd2880da..5173a16b06b 100644 --- a/crates/chain-state/src/memory_overlay.rs +++ b/crates/chain-state/src/memory_overlay.rs @@ -10,8 +10,8 @@ use reth_storage_api::{ StateProvider, StateProviderBox, StateRootProvider, StorageRootProvider, }; use reth_trie::{ - updates::TrieUpdates, AccountProof, HashedPostState, HashedStorage, MultiProof, - MultiProofTargets, StorageMultiProof, TrieInput, + updates::TrieUpdates, AccountProof, DecodedMultiProofV2, HashedPostState, HashedStorage, + MultiProof, MultiProofTargets, MultiProofTargetsV2, StorageMultiProof, TrieInput, }; use revm::database::BundleState; use std::{borrow::Cow, sync::OnceLock}; @@ -199,6 +199,15 @@ impl StateProofProvider for MemoryOverlayStateProviderRef<'_, self.historical.multiproof(input, targets) } + fn multiproof_v2( + &self, + mut input: TrieInput, + targets: MultiProofTargetsV2, + ) -> ProviderResult { + input.prepend_self(self.trie_input().clone()); + self.historical.multiproof_v2(input, targets) + } + fn witness( &self, mut input: TrieInput, diff --git a/crates/engine/execution-cache/src/cached_state.rs b/crates/engine/execution-cache/src/cached_state.rs index 9be40f13b56..ab1dc80fcb0 100644 --- a/crates/engine/execution-cache/src/cached_state.rs +++ b/crates/engine/execution-cache/src/cached_state.rs @@ -998,6 +998,14 @@ impl StateProofProvider for CachedStateProvider { self.state_provider.multiproof(input, targets) } + fn multiproof_v2( + &self, + input: TrieInput, + targets: reth_trie::MultiProofTargetsV2, + ) -> ProviderResult { + self.state_provider.multiproof_v2(input, targets) + } + fn witness( &self, input: TrieInput, diff --git a/crates/engine/tree/src/tree/instrumented_state.rs b/crates/engine/tree/src/tree/instrumented_state.rs index cdc1f7dfb3b..e60ee6110b2 100644 --- a/crates/engine/tree/src/tree/instrumented_state.rs +++ b/crates/engine/tree/src/tree/instrumented_state.rs @@ -244,6 +244,14 @@ impl StateProofProvider for InstrumentedStateProvider self.state_provider.multiproof(input, targets) } + fn multiproof_v2( + &self, + input: TrieInput, + targets: reth_trie::MultiProofTargetsV2, + ) -> ProviderResult { + self.state_provider.multiproof_v2(input, targets) + } + fn witness( &self, input: TrieInput, diff --git a/crates/revm/src/test_utils.rs b/crates/revm/src/test_utils.rs index de906dd4ac0..b1f0ecb631a 100644 --- a/crates/revm/src/test_utils.rs +++ b/crates/revm/src/test_utils.rs @@ -141,6 +141,14 @@ impl StateProofProvider for StateProviderTest { unimplemented!("proof generation is not supported") } + fn multiproof_v2( + &self, + _input: TrieInput, + _targets: reth_trie::MultiProofTargetsV2, + ) -> ProviderResult { + unimplemented!("proof generation is not supported") + } + fn witness( &self, _input: TrieInput, diff --git a/crates/rpc/rpc-eth-api/src/helpers/state.rs b/crates/rpc/rpc-eth-api/src/helpers/state.rs index 4dc41576c98..209f1d5926b 100644 --- a/crates/rpc/rpc-eth-api/src/helpers/state.rs +++ b/crates/rpc/rpc-eth-api/src/helpers/state.rs @@ -22,7 +22,7 @@ use reth_storage_api::{ BlockIdReader, BlockReaderIdExt, StateProvider, StateProviderBox, StateProviderFactory, }; use reth_transaction_pool::TransactionPool; -use reth_trie_common::MultiProofTargets; +use reth_trie_common::{MultiProofTargetsV2, ProofV2Target}; use std::{collections::HashMap, sync::Arc}; /// Helper methods for `eth_` methods relating to state (accounts). @@ -211,16 +211,21 @@ pub trait EthState: LoadState + SpawnBlocking { self.spawn_blocking_io_fut(async move |this| { let state = this.state_at_block_id(block_id).await?; - let mut proof_targets = MultiProofTargets::with_capacity(targets.len()); + let mut proof_targets = MultiProofTargetsV2::default(); + proof_targets.account_targets.reserve(targets.len()); + proof_targets.storage_targets.reserve(targets.len()); for (address, slots) in &targets { + let hashed_address = keccak256(address); + proof_targets.account_targets.push(ProofV2Target::new(hashed_address)); proof_targets - .entry(keccak256(address)) + .storage_targets + .entry(hashed_address) .or_default() - .extend(slots.iter().map(keccak256)); + .extend(slots.iter().map(|slot| ProofV2Target::new(keccak256(slot)))); } let multiproof = state - .multiproof(Default::default(), proof_targets) + .multiproof_v2(Default::default(), proof_targets) .map_err(Self::Error::from_eth_err)?; targets diff --git a/crates/storage/provider/src/providers/state/historical.rs b/crates/storage/provider/src/providers/state/historical.rs index a45b77ba3c3..118f6308c63 100644 --- a/crates/storage/provider/src/providers/state/historical.rs +++ b/crates/storage/provider/src/providers/state/historical.rs @@ -25,9 +25,9 @@ use reth_trie::{ trie_cursor::InMemoryTrieCursorFactory, updates::TrieUpdates, witness::TrieWitness, - AccountProof, ExecutionWitnessMode, HashedPostState, HashedStorage, KeccakKeyHasher, - MultiProof, MultiProofTargets, StateRoot, StorageMultiProof, StorageRoot, TrieInput, - TrieInputSorted, + AccountProof, DecodedMultiProofV2, ExecutionWitnessMode, HashedPostState, HashedStorage, + KeccakKeyHasher, MultiProof, MultiProofTargets, MultiProofTargetsV2, StateRoot, + StorageMultiProof, StorageRoot, TrieInput, TrieInputSorted, }; use reth_trie_db::{DatabaseProof, DatabaseStateRoot, DatabaseStorageProof, DatabaseStorageRoot}; @@ -595,6 +595,24 @@ where }) } + fn multiproof_v2( + &self, + input: TrieInput, + targets: MultiProofTargetsV2, + ) -> ProviderResult { + reth_trie_db::with_adapter!(self.provider, |A| { + let TrieInputSorted { nodes, state, prefix_sets } = + self.build_overlay(TrieInputSorted::from_unsorted(input))?; + let input = TrieInput::new( + Arc::unwrap_or_clone(nodes).into(), + Arc::unwrap_or_clone(state).into(), + prefix_sets, + ); + let proof = as DatabaseProof>::from_tx(self.tx()); + proof.overlay_multiproof_v2(input, targets).map_err(ProviderError::from) + }) + } + fn witness( &self, input: TrieInput, diff --git a/crates/storage/provider/src/providers/state/latest.rs b/crates/storage/provider/src/providers/state/latest.rs index 9d402ef9b4c..f85b371ea72 100644 --- a/crates/storage/provider/src/providers/state/latest.rs +++ b/crates/storage/provider/src/providers/state/latest.rs @@ -14,9 +14,9 @@ use reth_trie::{ trie_cursor::InMemoryTrieCursorFactory, updates::TrieUpdates, witness::TrieWitness, - AccountProof, ExecutionWitnessMode, HashedPostState, HashedStorage, KeccakKeyHasher, - MultiProof, MultiProofTargets, StateRoot, StorageMultiProof, StorageRoot, TrieInput, - TrieInputSorted, + AccountProof, DecodedMultiProofV2, ExecutionWitnessMode, HashedPostState, HashedStorage, + KeccakKeyHasher, MultiProof, MultiProofTargets, MultiProofTargetsV2, StateRoot, + StorageMultiProof, StorageRoot, TrieInput, TrieInputSorted, }; use reth_trie_db::{DatabaseProof, DatabaseStateRoot, DatabaseStorageProof, DatabaseStorageRoot}; @@ -219,6 +219,17 @@ impl StateProofProvider }) } + fn multiproof_v2( + &self, + input: TrieInput, + targets: MultiProofTargetsV2, + ) -> ProviderResult { + reth_trie_db::with_adapter!(self.0, |A| { + let proof = as DatabaseProof>::from_tx(self.tx()); + proof.overlay_multiproof_v2(input, targets).map_err(ProviderError::from) + }) + } + fn witness( &self, input: TrieInput, diff --git a/crates/storage/provider/src/test_utils/mock.rs b/crates/storage/provider/src/test_utils/mock.rs index 24264c60ffc..f299836aba9 100644 --- a/crates/storage/provider/src/test_utils/mock.rs +++ b/crates/storage/provider/src/test_utils/mock.rs @@ -1113,6 +1113,14 @@ where Ok(MultiProof::default()) } + fn multiproof_v2( + &self, + _input: TrieInput, + _targets: reth_trie::MultiProofTargetsV2, + ) -> ProviderResult { + Ok(reth_trie::DecodedMultiProofV2::default()) + } + fn witness( &self, _input: TrieInput, diff --git a/crates/storage/rpc-provider/src/lib.rs b/crates/storage/rpc-provider/src/lib.rs index 8569b55e11d..60e9b0f3099 100644 --- a/crates/storage/rpc-provider/src/lib.rs +++ b/crates/storage/rpc-provider/src/lib.rs @@ -1299,6 +1299,14 @@ where Err(ProviderError::UnsupportedProvider) } + fn multiproof_v2( + &self, + _input: TrieInput, + _targets: reth_trie::MultiProofTargetsV2, + ) -> Result { + Err(ProviderError::UnsupportedProvider) + } + fn witness( &self, _input: TrieInput, diff --git a/crates/storage/storage-api/src/macros.rs b/crates/storage/storage-api/src/macros.rs index 7ef599a011c..961e34d1435 100644 --- a/crates/storage/storage-api/src/macros.rs +++ b/crates/storage/storage-api/src/macros.rs @@ -59,6 +59,7 @@ macro_rules! delegate_provider_impls { StateProofProvider $(where [$($generics)*])? { fn proof(&self, input: reth_trie::TrieInput, address: alloy_primitives::Address, slots: &[alloy_primitives::B256]) -> reth_storage_api::errors::provider::ProviderResult; fn multiproof(&self, input: reth_trie::TrieInput, targets: reth_trie::MultiProofTargets) -> reth_storage_api::errors::provider::ProviderResult; + fn multiproof_v2(&self, input: reth_trie::TrieInput, targets: reth_trie::MultiProofTargetsV2) -> reth_storage_api::errors::provider::ProviderResult; fn witness(&self, input: reth_trie::TrieInput, target: reth_trie::HashedPostState, mode: reth_trie::ExecutionWitnessMode) -> reth_storage_api::errors::provider::ProviderResult>; } HashedPostStateProvider $(where [$($generics)*])? { diff --git a/crates/storage/storage-api/src/noop.rs b/crates/storage/storage-api/src/noop.rs index 2d8f57c3ad4..e1612bf64f4 100644 --- a/crates/storage/storage-api/src/noop.rs +++ b/crates/storage/storage-api/src/noop.rs @@ -40,8 +40,9 @@ use reth_prune_types::{PruneCheckpoint, PruneSegment}; use reth_stages_types::{StageCheckpoint, StageId}; use reth_storage_errors::provider::{ProviderError, ProviderResult}; use reth_trie_common::{ - updates::TrieUpdates, AccountProof, ExecutionWitnessMode, HashedPostState, HashedStorage, - MultiProof, MultiProofTargets, StorageMultiProof, StorageProof, TrieInput, + updates::TrieUpdates, AccountProof, DecodedMultiProofV2, ExecutionWitnessMode, HashedPostState, + HashedStorage, MultiProof, MultiProofTargets, MultiProofTargetsV2, StorageMultiProof, + StorageProof, TrieInput, }; /// Supports various api interfaces for testing purposes. @@ -525,6 +526,14 @@ impl StateProofProvider for NoopProvider ProviderResult { + Ok(DecodedMultiProofV2::default()) + } + fn witness( &self, _input: TrieInput, diff --git a/crates/storage/storage-api/src/trie.rs b/crates/storage/storage-api/src/trie.rs index a031873e800..119de13d8b3 100644 --- a/crates/storage/storage-api/src/trie.rs +++ b/crates/storage/storage-api/src/trie.rs @@ -1,11 +1,11 @@ use alloc::{boxed::Box, vec::Vec}; use alloy_primitives::{Address, Bytes, B256, U256}; use reth_primitives_traits::Account; -use reth_storage_errors::provider::ProviderResult; +use reth_storage_errors::provider::{ProviderError, ProviderResult}; use reth_trie_common::{ updates::{StorageTrieUpdatesSorted, TrieUpdates, TrieUpdatesSorted}, - AccountProof, ExecutionWitnessMode, HashedPostState, HashedStorage, MultiProof, - MultiProofTargets, StorageMultiProof, StorageProof, TrieInput, + AccountProof, DecodedMultiProofV2, ExecutionWitnessMode, HashedPostState, HashedStorage, + MultiProof, MultiProofTargets, MultiProofTargetsV2, StorageMultiProof, StorageProof, TrieInput, }; /// A type that can compute the state root of a given post state. @@ -162,6 +162,15 @@ pub trait StateProofProvider { targets: MultiProofTargets, ) -> ProviderResult; + /// Generate a V2 decoded multiproof for target hashed accounts and storage slots. + fn multiproof_v2( + &self, + _input: TrieInput, + _targets: MultiProofTargetsV2, + ) -> ProviderResult { + Err(ProviderError::UnsupportedProvider) + } + /// Get trie witness for provided state using the given witness generation mode. fn witness( &self, diff --git a/crates/trie/common/src/proofs.rs b/crates/trie/common/src/proofs.rs index bb8f17094a7..3610ae5ec68 100644 --- a/crates/trie/common/src/proofs.rs +++ b/crates/trie/common/src/proofs.rs @@ -1,6 +1,8 @@ //! Merkle trie proofs. -use crate::{BranchNodeMasks, BranchNodeMasksMap, Nibbles, ProofTrieNodeV2, TrieAccount}; +use crate::{ + BranchNodeMasks, BranchNodeMasksMap, Nibbles, ProofTrieNodeV2, TrieAccount, TrieNodeV2, +}; use alloc::{borrow::Cow, collections::VecDeque, vec::Vec}; use alloy_consensus::constants::KECCAK_EMPTY; use alloy_primitives::{ @@ -8,9 +10,9 @@ use alloy_primitives::{ map::{hash_map, B256Map, B256Set}, Address, Bytes, B256, U256, }; -use alloy_rlp::{encode_fixed_size, Decodable, EMPTY_STRING_CODE}; +use alloy_rlp::{encode_fixed_size, Decodable, Encodable, EMPTY_STRING_CODE}; use alloy_trie::{ - nodes::TrieNode, + nodes::{BranchNodeRef, TrieNode}, proof::{verify_proof, DecodedProofNodes, ProofNodes, ProofVerificationError}, EMPTY_ROOT_HASH, }; @@ -466,6 +468,62 @@ impl DecodedMultiProofV2 { self.account_proofs.is_empty() && self.storage_proofs.is_empty() } + /// Construct an account proof from the V2 multiproof. + pub fn account_proof( + &self, + address: Address, + slots: &[B256], + ) -> Result { + let hashed_address = keccak256(address); + let nibbles = Nibbles::unpack(hashed_address); + let account_nodes = matching_v2_proof_nodes(&self.account_proofs, &nibbles); + + let (info, storage_root) = 'account: { + if let Some(ProofTrieNodeV2 { node: TrieNodeV2::Leaf(leaf), .. }) = + account_nodes.clone().last() && + nibbles.ends_with(&leaf.key) + { + let account = TrieAccount::decode(&mut &leaf.value[..])?; + break 'account ( + Some(Account { + balance: account.balance, + nonce: account.nonce, + bytecode_hash: (account.code_hash != KECCAK_EMPTY) + .then_some(account.code_hash), + }), + account.storage_root, + ) + } + (None, EMPTY_ROOT_HASH) + }; + + let proof = encode_v2_proof_nodes(account_nodes); + let storage_nodes = self.storage_proofs.get(&hashed_address); + let mut storage_proofs = Vec::with_capacity(slots.len()); + for slot in slots { + let nibbles = Nibbles::unpack(keccak256(slot)); + let nodes = + storage_nodes.iter().flat_map(|nodes| matching_v2_proof_nodes(nodes, &nibbles)); + let value = 'value: { + if let Some(ProofTrieNodeV2 { node: TrieNodeV2::Leaf(leaf), .. }) = + nodes.clone().last() && + nibbles.ends_with(&leaf.key) + { + break 'value U256::decode(&mut &leaf.value[..])? + } + U256::ZERO + }; + storage_proofs.push(StorageProof { + key: *slot, + nibbles, + value, + proof: encode_v2_proof_nodes(nodes), + }); + } + + Ok(AccountProof { address, info, proof, storage_root, storage_proofs }) + } + /// Builds a `DecodedMultiProofV2` from a flat witness map (hash → RLP-encoded trie node). /// /// This performs a BFS traversal starting from `state_root`, decoding each witness entry @@ -561,6 +619,36 @@ impl DecodedMultiProofV2 { } } +/// Returns proof nodes matching `path` in root-to-leaf order. +fn matching_v2_proof_nodes<'a>( + nodes: &'a [ProofTrieNodeV2], + path: &'a Nibbles, +) -> impl Iterator + Clone { + nodes.iter().rev().filter(move |node| path.starts_with(&node.path)) +} + +/// Encodes V2 proof nodes as standard MPT proof nodes. +/// +/// V2 combines an extension and its child branch in one node. Both nodes need to be emitted for +/// consumers that reconstruct a sparse trie from the returned proof. +fn encode_v2_proof_nodes<'a>(nodes: impl Iterator) -> Vec { + let mut proof = Vec::new(); + for proof_node in nodes { + let mut encoded = Vec::new(); + proof_node.node.encode(&mut encoded); + proof.push(Bytes::from(encoded)); + + if let TrieNodeV2::Branch(branch) = &proof_node.node && + !branch.key.is_empty() + { + let mut encoded = Vec::new(); + BranchNodeRef::new(&branch.stack, branch.state_mask).encode(&mut encoded); + proof.push(Bytes::from(encoded)); + } + } + proof +} + impl From for DecodedMultiProofV2 { fn from(proof: DecodedMultiProof) -> Self { let account_proofs = @@ -1067,6 +1155,33 @@ mod tests { TrieMask, }; + #[test] + fn v2_account_proof_expands_extension_branch() { + let branch = + BranchNode::new(vec![RlpNode::word_rlp(&B256::repeat_byte(0x11))], TrieMask::from(1)); + let branch_rlp = alloy_rlp::encode(&branch); + let combined = TrieNodeV2::Branch(crate::BranchNodeV2::new( + Nibbles::from_nibbles([0xa, 0xb]), + branch.stack, + branch.state_mask, + Some(RlpNode::from_rlp(&branch_rlp)), + )); + let extension_rlp = alloy_rlp::encode(&combined); + let multiproof = DecodedMultiProofV2 { + account_proofs: vec![ProofTrieNodeV2 { + path: Nibbles::default(), + node: combined, + masks: None, + }], + ..Default::default() + }; + + let proof = multiproof.account_proof(Address::ZERO, &[]).unwrap(); + + assert_eq!(proof.proof, vec![Bytes::from(extension_rlp), Bytes::from(branch_rlp)]); + assert!(proof.info.is_none()); + } + #[test] fn witness_nodes_are_depth_first_ordered() { fn insert_node(witness: &mut B256Map, node: impl alloy_rlp::Encodable) -> RlpNode { diff --git a/crates/trie/db/src/proof.rs b/crates/trie/db/src/proof.rs index b7db69f50eb..f362692963d 100644 --- a/crates/trie/db/src/proof.rs +++ b/crates/trie/db/src/proof.rs @@ -6,8 +6,8 @@ use reth_trie::{ hashed_cursor::HashedPostStateCursorFactory, proof::{Proof, StorageProof}, trie_cursor::InMemoryTrieCursorFactory, - AccountProof, HashedPostStateSorted, HashedStorage, MultiProof, MultiProofTargets, - StorageMultiProof, TrieInput, + AccountProof, DecodedMultiProofV2, HashedPostStateSorted, HashedStorage, MultiProof, + MultiProofTargets, MultiProofTargetsV2, StorageMultiProof, TrieInput, }; /// Extends [`Proof`] with operations specific for working with a database transaction. @@ -32,6 +32,13 @@ pub trait DatabaseProof<'a> { input: TrieInput, targets: MultiProofTargets, ) -> Result; + + /// Generates a V2 decoded state multiproof for target hashed accounts and storage keys. + fn overlay_multiproof_v2( + &self, + input: TrieInput, + targets: MultiProofTargetsV2, + ) -> Result; } impl<'a, TX: DbTx, A: TrieTableAdapter> DatabaseProof<'a> @@ -72,6 +79,21 @@ impl<'a, TX: DbTx, A: TrieTableAdapter> DatabaseProof<'a> .with_prefix_sets_mut(input.prefix_sets) .multiproof(targets) } + + fn overlay_multiproof_v2( + &self, + input: TrieInput, + targets: MultiProofTargetsV2, + ) -> Result { + let nodes_sorted = input.nodes.into_sorted(); + let state_sorted = input.state.into_sorted(); + Proof::new( + InMemoryTrieCursorFactory::new(self.trie_cursor_factory().clone(), &nodes_sorted), + HashedPostStateCursorFactory::new(self.hashed_cursor_factory().clone(), &state_sorted), + ) + .with_prefix_sets_mut(input.prefix_sets) + .multiproof_v2(targets) + } } /// Extends [`StorageProof`] with operations specific for working with a database transaction. diff --git a/crates/trie/db/tests/proof.rs b/crates/trie/db/tests/proof.rs index e2bc6f0cf6a..de03f30f6db 100644 --- a/crates/trie/db/tests/proof.rs +++ b/crates/trie/db/tests/proof.rs @@ -2,12 +2,15 @@ use alloy_consensus::EMPTY_ROOT_HASH; use alloy_primitives::{address, b256, keccak256, Address, Bytes, B256, U256}; -use alloy_rlp::EMPTY_STRING_CODE; +use alloy_rlp::{Decodable, EMPTY_STRING_CODE}; use reth_chainspec::{Chain, ChainSpec, HOLESKY, MAINNET}; use reth_primitives_traits::Account; use reth_provider::test_utils::{create_test_provider_factory, insert_genesis}; use reth_storage_api::StorageSettingsCache; -use reth_trie::{proof::Proof, AccountProof, Nibbles, StorageProof}; +use reth_trie::{ + proof::Proof, AccountProof, MultiProofTargetsV2, Nibbles, ProofV2Target, StorageProof, + TrieInput, TrieNode, +}; use reth_trie_db::{DatabaseHashedCursorFactory, DatabaseProof, DatabaseTrieCursorFactory}; use std::{ str::FromStr, @@ -134,6 +137,47 @@ fn testspec_empty_storage_proof() { }); } +#[test] +fn multiproof_v2_includes_branch_after_terminal_extension() { + let factory = create_test_provider_factory(); + let root = insert_genesis(&factory, TEST_SPEC.clone()).unwrap(); + let target = Address::ZERO; + let provider = factory.provider().unwrap(); + + reth_trie_db::with_adapter!(provider, |A| { + let proof = as DatabaseProof>::from_tx(provider.tx_ref()); + let legacy = proof.account_proof(target, &[]).unwrap(); + assert_eq!(legacy.proof.len(), 1); + assert!(matches!( + TrieNode::decode(&mut &legacy.proof[0][..]).unwrap(), + TrieNode::Extension(_) + )); + + let proof = as DatabaseProof>::from_tx(provider.tx_ref()); + let multiproof = proof + .overlay_multiproof_v2( + TrieInput::default(), + MultiProofTargetsV2 { + account_targets: vec![ProofV2Target::new(keccak256(target))], + ..Default::default() + }, + ) + .unwrap(); + let account_proof = multiproof.account_proof(target, &[]).unwrap(); + + assert_eq!(account_proof.proof.len(), 2); + assert!(matches!( + TrieNode::decode(&mut &account_proof.proof[0][..]).unwrap(), + TrieNode::Extension(_) + )); + assert!(matches!( + TrieNode::decode(&mut &account_proof.proof[1][..]).unwrap(), + TrieNode::Branch(_) + )); + assert_eq!(account_proof.verify(root), Ok(())); + }); +} + #[test] fn empty_state_trie_account_proof() { // Empty database => empty account (state) trie, root == EMPTY_ROOT_HASH. From 3c31377d6533f4298739dbb4ab6c371a8d5b3eb6 Mon Sep 17 00:00:00 2001 From: Brian Picciano Date: Tue, 1 Sep 2026 11:40:52 +0000 Subject: [PATCH 108/294] refactor(engine): remove unused state provider builder (#26902) --- crates/engine/tree/src/tree/mod.rs | 115 ++++------------------------- 1 file changed, 15 insertions(+), 100 deletions(-) diff --git a/crates/engine/tree/src/tree/mod.rs b/crates/engine/tree/src/tree/mod.rs index 265e375d498..9e44704e170 100644 --- a/crates/engine/tree/src/tree/mod.rs +++ b/crates/engine/tree/src/tree/mod.rs @@ -15,8 +15,7 @@ use error::{ InsertBlockError, InsertBlockFatalError, InsertBlockProcessingError, InsertBlockValidationError, }; use reth_chain_state::{ - CanonicalInMemoryState, ExecutedBlock, ExecutionTimingStats, MemoryOverlayStateProvider, - NewCanonicalChain, + CanonicalInMemoryState, ExecutedBlock, ExecutionTimingStats, NewCanonicalChain, }; use reth_consensus::{Consensus, FullConsensus}; use reth_engine_primitives::{ @@ -31,11 +30,10 @@ use reth_primitives_traits::{ FastInstant as Instant, NodePrimitives, RecoveredBlock, SealedBlock, SealedHeader, }; use reth_provider::{ - BalProvider, BlockExecutionOutput, BlockExecutionResult, BlockNumReader, BlockReader, - ChangeSetReader, DatabaseProviderFactory, LatestStateProvider, ProviderError, - PruneCheckpointReader, SaveBlocksInput, StageCheckpointReader, StateProviderBox, - StateProviderFactory, StateReader, StorageChangeSetReader, StorageSettingsCache, - TransactionVariant, TryIntoHistoricalStateProvider, + BalProvider, BlockExecutionOutput, BlockExecutionResult, BlockReader, ChangeSetReader, + DatabaseProviderFactory, ProviderError, PruneCheckpointReader, SaveBlocksInput, + StageCheckpointReader, StateProviderFactory, StateReader, StorageChangeSetReader, + StorageSettingsCache, TransactionVariant, }; use reth_revm::database::StateProviderDatabase; use reth_stages_api::ControlFlow; @@ -114,67 +112,6 @@ pub(crate) const MIN_BLOCKS_FOR_PIPELINE_RUN: u64 = EPOCH_SLOTS; /// even when the finalized block is not set (e.g., on L2s like Optimism). const CHANGESET_CACHE_RETENTION_BLOCKS: u64 = 64; -/// A builder for creating state providers that can be used across threads. -#[derive(Clone, Debug)] -pub struct StateProviderBuilder { - /// The provider factory used to create providers. - provider_factory: P, - /// Hash of the block whose state to provide. - parent_hash: B256, - /// Tracks the in-memory parent chain and its overlays. - overlay_manager: OverlayManager, -} - -impl StateProviderBuilder { - /// Creates a new state provider builder for `parent_hash`. - pub const fn new( - provider_factory: P, - parent_hash: B256, - overlay_manager: OverlayManager, - ) -> Self { - Self { provider_factory, parent_hash, overlay_manager } - } -} - -impl StateProviderBuilder -where - P: DatabaseProviderFactory, - P::Provider: BlockNumReader - + PruneCheckpointReader - + StageCheckpointReader - + StorageSettingsCache - + TryIntoHistoricalStateProvider - + 'static, -{ - /// Creates a new state provider from this builder. - pub fn build(&self) -> ProviderResult { - let overlay_builder = self.overlay_manager.overlay_builder(self.parent_hash); - let provider = self.provider_factory.database_provider_ro()?; - let anchor = overlay_builder.anchor_at_parent(&provider)?; - let (provider, overlay): (StateProviderBox, _) = match anchor { - reth_storage_overlay::AnchorForParent::NoReverts { anchor, overlay } => { - debug!( - target: "engine::tree", - parent_hash = %self.parent_hash, - ?anchor, - "creating state provider from latest state" - ); - (Box::new(LatestStateProvider::new(provider)), overlay) - } - reth_storage_overlay::AnchorForParent::RevertsRequired { anchor, overlay, .. } => { - debug!( - target: "engine::tree", - parent_hash = %self.parent_hash, - ?anchor, - "creating state provider from historical state" - ); - (provider.try_into_history_at_block(anchor.number)?, overlay) - } - }; - Ok(Box::new(MemoryOverlayStateProvider::new(provider, overlay))) - } -} - /// Tracks the state of the engine api internals. /// /// This type is not shareable. @@ -445,7 +382,6 @@ where + ChangeSetReader + StorageChangeSetReader + StorageSettingsCache - + TryIntoHistoricalStateProvider + 'static, C: ConfigureEvm + 'static, T: PayloadTypes>, @@ -3193,14 +3129,17 @@ where } // Ensure that the parent state is available. - match self.state_provider_builder(block_id.parent) { - Err(err) => { - let block = convert_to_block(self, input)?; - return Err(InsertBlockError::new(block, err.into()).into()); - } - Ok(None) => { - let block = convert_to_block(self, input)?; + if !self.state.tree_state.contains_hash(&block_id.parent) { + let parent_exists = match self.provider.header(block_id.parent) { + Ok(header) => header.is_some(), + Err(err) => { + let block = convert_to_block(self, input)?; + return Err(InsertBlockError::new(block, err.into()).into()); + } + }; + if !parent_exists { + let block = convert_to_block(self, input)?; // we don't have the state required to execute this block, buffering it and find the // missing parent block let missing_ancestor = self @@ -3217,7 +3156,6 @@ where missing_ancestor, })) } - Ok(Some(_)) => {} } // determine whether we are on a fork chain by comparing the block number with the @@ -3589,29 +3527,6 @@ where ); Ok(()) } - - /// Returns a builder for creating state providers for the given hash. - /// - /// This is an optimization for parallel execution contexts where we want to avoid - /// creating state providers in the critical path. - pub fn state_provider_builder( - &self, - hash: B256, - ) -> ProviderResult>> - where - P: BlockReader + StateProviderFactory + StateReader + Clone, - { - if !self.state.tree_state.contains_hash(&hash) && self.provider.header(hash)?.is_none() { - debug!(target: "engine::tree", %hash, "no canonical state found for block"); - return Ok(None) - } - - Ok(Some(StateProviderBuilder::new( - self.provider.clone(), - hash, - self.state.tree_state.overlay_manager.clone(), - ))) - } } /// Events received in the main engine loop. From 5f57d640b32950eb7a03bea628163c57309ac5ce Mon Sep 17 00:00:00 2001 From: Brian Picciano Date: Tue, 1 Sep 2026 11:56:49 +0000 Subject: [PATCH 109/294] refactor(provider): remove historical state factory methods (#26903) --- crates/cli/commands/src/db/state.rs | 10 +++++--- crates/engine/tree/src/persistence.rs | 12 ++++++--- .../provider/src/providers/database/mod.rs | 25 +------------------ examples/db-access/src/main.rs | 11 +++++--- 4 files changed, 24 insertions(+), 34 deletions(-) diff --git a/crates/cli/commands/src/db/state.rs b/crates/cli/commands/src/db/state.rs index 94efe18d770..abac9fe0b08 100644 --- a/crates/cli/commands/src/db/state.rs +++ b/crates/cli/commands/src/db/state.rs @@ -9,8 +9,11 @@ use reth_db_api::{ }; use reth_db_common::DbTool; use reth_node_builder::NodeTypesWithDB; -use reth_provider::{providers::ProviderNodeTypes, StaticFileProviderFactory}; -use reth_storage_api::{BlockNumReader, StateProvider, StorageSettingsCache}; +use reth_provider::{ + providers::{BlockchainProvider, ProviderNodeTypes}, + StaticFileProviderFactory, +}; +use reth_storage_api::{BlockNumReader, StateProvider, StateProviderFactory, StorageSettingsCache}; use reth_tasks::spawn_scoped_os_thread; use std::{ collections::BTreeSet, @@ -138,7 +141,8 @@ impl Command { block: BlockNumber, limit: usize, ) -> eyre::Result<()> { - let provider = tool.provider_factory.history_by_block_number(block)?; + let provider = BlockchainProvider::new(tool.provider_factory.clone())? + .history_by_block_number(block)?; // Get account info at that block let account = provider.basic_account(&address)?; diff --git a/crates/engine/tree/src/persistence.rs b/crates/engine/tree/src/persistence.rs index feef9a6e368..e2b58f76810 100644 --- a/crates/engine/tree/src/persistence.rs +++ b/crates/engine/tree/src/persistence.rs @@ -701,14 +701,16 @@ mod tests { // Check the primary can read its own historical state. { - let primary_state_at_1 = provider_factory.history_by_block_number(1).unwrap(); + let provider = provider_factory.provider().unwrap(); + let primary_state_at_1 = provider.history_by_block_number(1).unwrap(); let primary_account = primary_state_at_1.basic_account(&signer).unwrap(); assert!(primary_account.is_some(), "primary: signer must exist at block 1"); } // Verify historical state at block 1 is accessible via changesets on the secondary. { - let state_at_1 = secondary.history_by_block_number(1).unwrap(); + let provider = secondary.provider().unwrap(); + let state_at_1 = provider.history_by_block_number(1).unwrap(); let account_at_1 = state_at_1.basic_account(&signer).unwrap(); assert!(account_at_1.is_some(), "signer account must exist at block 1"); let account_at_1 = account_at_1.unwrap(); @@ -803,7 +805,8 @@ mod tests { ); // Verify historical state at block 1 is still accessible after the reorg. - let state_at_1 = secondary.history_by_block_number(1).unwrap(); + let provider = secondary.provider().unwrap(); + let state_at_1 = provider.history_by_block_number(1).unwrap(); let account_at_1 = state_at_1.basic_account(&signer).unwrap(); assert!(account_at_1.is_some(), "signer account must exist at block 1 after reorg"); let account_at_1 = account_at_1.unwrap(); @@ -817,7 +820,8 @@ mod tests { ); // Verify the latest state (at block 2) reflects the reorged execution. - let state_at_2 = secondary.history_by_block_number(2).unwrap(); + let provider = secondary.provider().unwrap(); + let state_at_2 = provider.history_by_block_number(2).unwrap(); let account_at_2 = state_at_2.basic_account(&signer).unwrap(); assert!(account_at_2.is_some(), "signer account must exist at block 2 after reorg"); let account_at_2 = account_at_2.unwrap(); diff --git a/crates/storage/provider/src/providers/database/mod.rs b/crates/storage/provider/src/providers/database/mod.rs index 80c47356b37..25cdf18884e 100644 --- a/crates/storage/provider/src/providers/database/mod.rs +++ b/crates/storage/provider/src/providers/database/mod.rs @@ -30,7 +30,7 @@ use reth_stages_types::{PipelineTarget, StageCheckpoint, StageId}; use reth_static_file_types::StaticFileSegment; use reth_storage_api::{ BlockBodyIndicesProvider, ChainStateBlockReader, ChainStateBlockWriter, DBProvider, - NodePrimitivesProvider, StorageSettings, StorageSettingsCache, TryIntoHistoricalStateProvider, + NodePrimitivesProvider, StorageSettings, StorageSettingsCache, }; use reth_storage_errors::provider::ProviderResult; use reth_storage_overlay::OverlayManager; @@ -463,29 +463,6 @@ impl ProviderFactory { Ok(Box::new(LatestStateProvider::new(self.database_provider_ro()?))) } - /// Storage provider for state at that given block - pub fn history_by_block_number( - &self, - block_number: BlockNumber, - ) -> ProviderResult { - let state_provider = self.provider()?.try_into_history_at_block(block_number)?; - trace!(target: "providers::db", ?block_number, "Returning historical state provider for block number"); - Ok(state_provider) - } - - /// Storage provider for state at that given block hash - pub fn history_by_block_hash(&self, block_hash: BlockHash) -> ProviderResult { - let provider = self.provider()?; - - let block_number = provider - .block_number(block_hash)? - .ok_or(ProviderError::BlockHashNotFound(block_hash))?; - - let state_provider = provider.try_into_history_at_block(block_number)?; - trace!(target: "providers::db", ?block_number, %block_hash, "Returning historical state provider for block hash"); - Ok(state_provider) - } - /// Asserts that the static files and database are consistent. If not, /// returns [`ProviderError::MustUnwind`] with the appropriate unwind /// target. May also return any [`ProviderError`] that diff --git a/examples/db-access/src/main.rs b/examples/db-access/src/main.rs index 25a8d3eabf3..0984273f014 100644 --- a/examples/db-access/src/main.rs +++ b/examples/db-access/src/main.rs @@ -6,8 +6,9 @@ use reth_ethereum::{ node::EthereumNode, primitives::{AlloyBlockHeader, SealedBlock, SealedHeader}, provider::{ - providers::ReadOnlyConfig, AccountReader, BlockNumReader, BlockReader, BlockSource, - HeaderProvider, ReceiptProvider, StateProvider, TransactionVariant, TransactionsProvider, + providers::{BlockchainProvider, ReadOnlyConfig}, + AccountReader, BlockNumReader, BlockReader, BlockSource, HeaderProvider, ReceiptProvider, + StateProvider, StateProviderFactory, TransactionVariant, TransactionsProvider, }, rpc::eth::primitives::Filter, TransactionSigned, @@ -44,7 +45,11 @@ fn main() -> eyre::Result<()> { receipts_provider_example(&provider)?; state_provider_example(factory.latest()?, &provider, provider.best_block_number()?)?; - state_provider_example(factory.history_by_block_number(block_num)?, &provider, block_num)?; + state_provider_example( + BlockchainProvider::new(factory.clone())?.history_by_block_number(block_num)?, + &provider, + block_num, + )?; // Closes the RO transaction opened in the `factory.provider()` call. This is optional and // would happen anyway at the end of the function scope. From 00d9e9e1cf654c8aa5cdf4acc5be3ea549a45b4b Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Tue, 1 Sep 2026 12:31:01 +0000 Subject: [PATCH 110/294] fix(engine): keep tree alive on invalid downloads (#26901) --- crates/engine/tree/src/tree/mod.rs | 21 ++++++------------- .../src/tree/payload_processor/bal/execute.rs | 8 +++---- .../src/tree/payload_processor/bal/worker.rs | 6 ++++-- .../engine/tree/src/tree/payload_validator.rs | 4 ++-- 4 files changed, 16 insertions(+), 23 deletions(-) diff --git a/crates/engine/tree/src/tree/mod.rs b/crates/engine/tree/src/tree/mod.rs index 9e44704e170..dadeb923ed9 100644 --- a/crates/engine/tree/src/tree/mod.rs +++ b/crates/engine/tree/src/tree/mod.rs @@ -2658,9 +2658,10 @@ where Err(err) => { if let InsertPayloadError::Block(err) = err { debug!(target: "engine::tree", ?err, "failed to connect buffered block to tree"); - if let Err(fatal) = self.on_internal_insert_block_error(err) { + if let Err(InsertBlockProcessingError::Fatal(fatal)) = + self.on_insert_block_error(err) + { warn!(target: "engine::tree", %fatal, "fatal error occurred while connecting buffered blocks"); - return Err(fatal) } } } @@ -3033,9 +3034,10 @@ where Err(err) => { if let InsertPayloadError::Block(err) = err { debug!(target: "engine::tree", err=%err.kind(), "failed to insert downloaded block"); - if let Err(fatal) = self.on_internal_insert_block_error(err) { + if let Err(InsertBlockProcessingError::Fatal(fatal)) = + self.on_insert_block_error(err) + { warn!(target: "engine::tree", %fatal, "fatal error occurred while inserting downloaded block"); - return Err(fatal) } } } @@ -3289,17 +3291,6 @@ where )) } - /// Handles an insertion error from a non-RPC source, where malformed input can be discarded. - fn on_internal_insert_block_error( - &mut self, - error: InsertBlockError, - ) -> Result<(), InsertBlockFatalError> { - match self.on_insert_block_error(error) { - Ok(_) | Err(InsertBlockProcessingError::MalformedInput(_)) => Ok(()), - Err(InsertBlockProcessingError::Fatal(error)) => Err(error), - } - } - /// Handles a [`NewPayloadError`] by converting it to a [`PayloadStatus`]. fn on_new_payload_error( &mut self, diff --git a/crates/engine/tree/src/tree/payload_processor/bal/execute.rs b/crates/engine/tree/src/tree/payload_processor/bal/execute.rs index 11d155bdb8d..4f69147d24b 100644 --- a/crates/engine/tree/src/tree/payload_processor/bal/execute.rs +++ b/crates/engine/tree/src/tree/payload_processor/bal/execute.rs @@ -1159,9 +1159,9 @@ mod tests { } #[test] - fn worker_tx_recovery_error_becomes_other_error() { + fn worker_tx_recovery_error_becomes_validation_error() { // A tx recovery failure fed into the worker channel must surface as - // BalExecutionError::Other. Uses execute_block directly since tx_stream hardcodes + // a block validation error. Uses execute_block directly since tx_stream hardcodes // Infallible and cannot inject errors. let evm_config = EthEvmConfig::mainnet(); let block = empty_amsterdam_block(B256::ZERO); @@ -1192,8 +1192,8 @@ mod tests { ); assert!( - matches!(result, Err(BalExecutionError::Other(_))), - "expected Other error from tx recovery failure, got {result:?}", + matches!(result, Err(BalExecutionError::Execution(BlockExecutionError::Validation(_)))), + "expected validation error from tx recovery failure, got {result:?}", ); } diff --git a/crates/engine/tree/src/tree/payload_processor/bal/worker.rs b/crates/engine/tree/src/tree/payload_processor/bal/worker.rs index a9a9ee5be2d..b8fa1a86248 100644 --- a/crates/engine/tree/src/tree/payload_processor/bal/worker.rs +++ b/crates/engine/tree/src/tree/payload_processor/bal/worker.rs @@ -2,7 +2,7 @@ use super::BalExecutionError; use alloy_consensus::Transaction; use alloy_eip7928::BlockAccessIndex; use alloy_evm::{ - block::{BlockExecutionError, BlockExecutor, BlockExecutorFactory}, + block::{BlockExecutionError, BlockExecutor, BlockExecutorFactory, BlockValidationError}, Evm, }; use alloy_primitives::Address; @@ -28,7 +28,9 @@ impl From for BalExecutionError { fn from(err: BalWorkerError) -> Self { match err { BalWorkerError::Setup(err) => err, - BalWorkerError::Transaction(err) => Self::Other(err), + BalWorkerError::Transaction(err) => { + Self::Execution(BlockValidationError::Other(err).into()) + } BalWorkerError::Execution(err) => Self::Execution(err), } } diff --git a/crates/engine/tree/src/tree/payload_validator.rs b/crates/engine/tree/src/tree/payload_validator.rs index 2472a5511aa..ecf4632527f 100644 --- a/crates/engine/tree/src/tree/payload_validator.rs +++ b/crates/engine/tree/src/tree/payload_validator.rs @@ -135,7 +135,7 @@ use reth_consensus::{ConsensusError, FullConsensus, ReceiptRootBloom}; use reth_engine_primitives::{ ConfigureEngineEvm, ExecutableTxIterator, ExecutionPayload, InvalidBlockHook, PayloadValidator, }; -use reth_errors::{BlockExecutionError, ProviderResult}; +use reth_errors::{BlockExecutionError, BlockValidationError, ProviderResult}; use reth_evm::{ block::BlockExecutor, execute::ExecutableTxFor, ConfigureEvm, EvmEnvFor, ExecutionCtxFor, OnStateHook, SpecFor, @@ -1262,7 +1262,7 @@ where let Some(tx_result) = transactions.next() else { break }; self.metrics.record_transaction_wait(wait_start.elapsed()); - let tx = tx_result.map_err(BlockExecutionError::other)?; + let tx = tx_result.map_err(BlockValidationError::other)?; let tx_signer = *>::signer(&tx); senders.push(tx_signer); From 9d315f28dcb5142e26c2b7d623f3bab390157d67 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Tue, 1 Sep 2026 13:58:21 +0000 Subject: [PATCH 111/294] feat(engine): fetch BALs for downloaded blocks (#26823) Co-authored-by: Karl --- crates/engine/tree/src/download.rs | 244 +++++++++++++++--- crates/engine/tree/src/engine.rs | 46 +++- crates/engine/tree/src/launch.rs | 6 +- crates/engine/tree/src/tree/block_buffer.rs | 160 ++++++++---- crates/engine/tree/src/tree/mod.rs | 61 +++-- .../engine/tree/src/tree/payload_validator.rs | 17 +- crates/engine/tree/src/tree/tests.rs | 46 +++- 7 files changed, 440 insertions(+), 140 deletions(-) diff --git a/crates/engine/tree/src/download.rs b/crates/engine/tree/src/download.rs index 31eb2417964..7dd2611f93b 100644 --- a/crates/engine/tree/src/download.rs +++ b/crates/engine/tree/src/download.rs @@ -6,10 +6,13 @@ use alloy_primitives::{map::B256Set, B256}; use futures::FutureExt; use reth_consensus::Consensus; use reth_network_p2p::{ - full_block::{FetchFullBlockFuture, FetchFullBlockRangeFuture, FullBlockClient}, - BlockClient, + full_block::{ + FetchFullBlockFuture, FetchFullBlockRangeFuture, FetchFullBlockRangeWithBalFuture, + FetchFullBlockWithBalFuture, FullBlockClient, SealedBlockWithAccessList, + }, + BlockAccessListsClient, BlockClient, }; -use reth_primitives_traits::{Block, SealedBlock}; +use reth_primitives_traits::{Block, SealedBlockWith}; use std::{ cmp::{Ordering, Reverse}, collections::{binary_heap::PeekMut, BinaryHeap, VecDeque}, @@ -43,8 +46,8 @@ pub enum DownloadAction { /// Outcome of downloaded blocks. #[derive(Debug)] pub enum DownloadOutcome { - /// Downloaded blocks. - Blocks(Vec>), + /// Downloaded blocks with optional block access list data. + Blocks(Vec>), /// New download started. NewDownloadStarted { /// How many blocks are pending in this download. @@ -58,17 +61,17 @@ pub enum DownloadOutcome { #[expect(missing_debug_implementations)] pub struct BasicBlockDownloader where - Client: BlockClient + 'static, + Client: BlockClient + BlockAccessListsClient + 'static, { /// A downloader that can download full blocks from the network. full_block_client: FullBlockClient, /// In-flight full block requests in progress. - inflight_full_block_requests: Vec>, + inflight_full_block_requests: Vec>, /// In-flight full block _range_ requests in progress. - inflight_block_range_requests: Vec>, + inflight_block_range_requests: Vec>, /// Buffered blocks from downloads - this is a min-heap of blocks, using the block number for /// ordering. This means the blocks will be popped from the heap with ascending block numbers. - set_buffered_blocks: BinaryHeap>>, + set_buffered_blocks: BinaryHeap>>, /// Engine download metrics. metrics: BlockDownloaderMetrics, /// Pending events to be emitted. @@ -77,7 +80,7 @@ where impl BasicBlockDownloader where - Client: BlockClient + 'static, + Client: BlockClient + BlockAccessListsClient + 'static, B: Block, { /// Create a new instance @@ -103,31 +106,45 @@ where /// Processes a download request. fn download(&mut self, request: DownloadRequest) { match request { - DownloadRequest::BlockSet(hashes) => self.download_block_set(hashes), - DownloadRequest::BlockRange(hash, count) => self.download_block_range(hash, count), + DownloadRequest::BlockSet { hashes, access_lists } => { + self.download_block_set(hashes, access_lists) + } + DownloadRequest::BlockRange { hash, count, access_lists } => { + self.download_block_range(hash, count, access_lists) + } } } /// Processes a block set download request. - fn download_block_set(&mut self, hashes: B256Set) { + fn download_block_set(&mut self, hashes: B256Set, access_lists: bool) { for hash in hashes { - self.download_full_block(hash); + self.download_full_block(hash, access_lists); } } /// Processes a block range download request. - fn download_block_range(&mut self, hash: B256, count: u64) { + fn download_block_range(&mut self, hash: B256, count: u64, access_lists: bool) { if count == 1 { - self.download_full_block(hash); + self.download_full_block(hash, access_lists); } else { trace!( target: "engine::download", ?hash, ?count, + access_lists, "start downloading full block range." ); - let request = self.full_block_client.get_full_block_range(hash, count); + let request = if access_lists { + FullBlockRangeDownload::WithAccessLists( + self.full_block_client + .get_full_block_range_with_optional_access_lists(hash, count), + ) + } else { + FullBlockRangeDownload::Blocks( + self.full_block_client.get_full_block_range(hash, count), + ) + }; self.push_pending_event(DownloadOutcome::NewDownloadStarted { remaining_blocks: request.count(), target: request.start_hash(), @@ -142,7 +159,7 @@ where /// /// Returns `true` if the request was started, `false` if there's already a request for the /// given hash. - fn download_full_block(&mut self, hash: B256) -> bool { + fn download_full_block(&mut self, hash: B256, access_lists: bool) -> bool { if self.is_inflight_request(hash) { return false } @@ -154,10 +171,17 @@ where trace!( target: "engine::download", ?hash, + access_lists, "Start downloading full block" ); - let request = self.full_block_client.get_full_block(hash); + let request = if access_lists { + FullBlockDownload::WithAccessList( + self.full_block_client.get_full_block_with_access_lists(hash), + ) + } else { + FullBlockDownload::Block(self.full_block_client.get_full_block(hash)) + }; self.inflight_full_block_requests.push(request); self.update_block_download_metrics(); @@ -190,7 +214,7 @@ where impl BlockDownloader for BasicBlockDownloader where - Client: BlockClient, + Client: BlockClient + BlockAccessListsClient, B: Block, { type Block = B; @@ -212,7 +236,7 @@ where // advance all full block requests for idx in (0..self.inflight_full_block_requests.len()).rev() { let mut request = self.inflight_full_block_requests.swap_remove(idx); - if let Poll::Ready(block) = request.poll_unpin(cx) { + if let Poll::Ready(block) = request.poll(cx) { trace!(target: "engine::download", block=?block.num_hash(), "Received single full block, buffering"); self.set_buffered_blocks.push(Reverse(block.into())); } else { @@ -224,10 +248,10 @@ where // advance all full block range requests for idx in (0..self.inflight_block_range_requests.len()).rev() { let mut request = self.inflight_block_range_requests.swap_remove(idx); - if let Poll::Ready(blocks) = request.poll_unpin(cx) { + if let Poll::Ready(blocks) = request.poll(cx) { trace!(target: "engine::download", len=?blocks.len(), first=?blocks.first().map(|b| b.num_hash()), last=?blocks.last().map(|b| b.num_hash()), "Received full block range, buffering"); self.set_buffered_blocks - .extend(blocks.into_iter().map(OrderedSealedBlock).map(Reverse)); + .extend(blocks.into_iter().map(OrderedDownloadedBlock).map(Reverse)); } else { // still pending self.inflight_block_range_requests.push(request); @@ -243,46 +267,121 @@ where // drain all unique element of the block buffer if there are any let mut downloaded_blocks = Vec::with_capacity(self.set_buffered_blocks.len()); while let Some(block) = self.set_buffered_blocks.pop() { - // peek ahead and pop duplicates + let mut block = block.0 .0; + // peek ahead and pop duplicates, keeping the copy that includes access list data while let Some(peek) = self.set_buffered_blocks.peek_mut() { - if peek.0 .0.hash() == block.0 .0.hash() { - PeekMut::pop(peek); + if peek.0 .0.hash() == block.hash() { + let duplicate = PeekMut::pop(peek).0 .0; + if block.data().is_none() && duplicate.data().is_some() { + block = duplicate; + } } else { break } } - downloaded_blocks.push(block.0.into()); + downloaded_blocks.push(block); } Poll::Ready(DownloadOutcome::Blocks(downloaded_blocks)) } } -/// A wrapper type around [`SealedBlock`] that implements the [Ord] +/// A wrapper type around [`SealedBlockWithAccessList`] that implements the [Ord] /// trait by block number. #[derive(Debug, Clone, PartialEq, Eq)] -struct OrderedSealedBlock(SealedBlock); +struct OrderedDownloadedBlock(SealedBlockWithAccessList); -impl PartialOrd for OrderedSealedBlock { +impl PartialOrd for OrderedDownloadedBlock { fn partial_cmp(&self, other: &Self) -> Option { Some(self.cmp(other)) } } -impl Ord for OrderedSealedBlock { +impl Ord for OrderedDownloadedBlock { fn cmp(&self, other: &Self) -> Ordering { self.0.number().cmp(&other.0.number()) } } -impl From> for OrderedSealedBlock { - fn from(block: SealedBlock) -> Self { +impl From> for OrderedDownloadedBlock { + fn from(block: SealedBlockWithAccessList) -> Self { Self(block) } } -impl From> for SealedBlock { - fn from(value: OrderedSealedBlock) -> Self { - value.0 +/// An in-flight full block request that optionally also fetches the block's access list. +enum FullBlockDownload +where + Client: BlockClient + BlockAccessListsClient, +{ + /// Fetches the block only. + Block(FetchFullBlockFuture), + /// Fetches the block and attempts to fetch its access list. + WithAccessList(FetchFullBlockWithBalFuture), +} + +impl FullBlockDownload +where + Client: BlockClient + BlockAccessListsClient + 'static, +{ + /// Returns the hash of the block being requested. + const fn hash(&self) -> &B256 { + match self { + Self::Block(req) => req.hash(), + Self::WithAccessList(req) => req.hash(), + } + } + + /// Advances the download. + fn poll(&mut self, cx: &mut Context<'_>) -> Poll> { + match self { + Self::Block(req) => req.poll_unpin(cx).map(SealedBlockWith::from_block), + Self::WithAccessList(req) => req.poll_unpin(cx), + } + } +} + +/// An in-flight full block range request that optionally also fetches the blocks' access lists. +enum FullBlockRangeDownload +where + Client: BlockClient + BlockAccessListsClient, +{ + /// Fetches the block range only. + Blocks(FetchFullBlockRangeFuture), + /// Fetches the block range and attempts to fetch the blocks' access lists. + WithAccessLists(FetchFullBlockRangeWithBalFuture), +} + +impl FullBlockRangeDownload +where + Client: BlockClient + BlockAccessListsClient + 'static, +{ + /// Returns the block hash the requested range starts at (inclusive). + const fn start_hash(&self) -> B256 { + match self { + Self::Blocks(req) => req.start_hash(), + Self::WithAccessLists(req) => req.start_hash(), + } + } + + /// Returns the number of requested blocks. + const fn count(&self) -> u64 { + match self { + Self::Blocks(req) => req.count(), + Self::WithAccessLists(req) => req.count(), + } + } + + /// Advances the download. + fn poll( + &mut self, + cx: &mut Context<'_>, + ) -> Poll>> { + match self { + Self::Blocks(req) => req + .poll_unpin(cx) + .map(|blocks| blocks.into_iter().map(SealedBlockWith::from_block).collect()), + Self::WithAccessLists(req) => req.poll_unpin(cx), + } } } @@ -353,7 +452,7 @@ mod tests { let tip = client.highest_block().expect("there should be blocks here"); // send block range download request - block_downloader.on_action(DownloadAction::Download(DownloadRequest::BlockRange( + block_downloader.on_action(DownloadAction::Download(DownloadRequest::block_range( tip.hash(), tip.number, ))); @@ -383,7 +482,7 @@ mod tests { // ensure they are in ascending order for num in 1..=TOTAL_BLOCKS { - assert_eq!(blocks[num-1].number(), num as u64); + assert_eq!(blocks[num - 1].number(), num as u64); } }); } @@ -396,7 +495,7 @@ mod tests { let tip = client.highest_block().expect("there should be blocks here"); // send block set download request - block_downloader.on_action(DownloadAction::Download(DownloadRequest::BlockSet( + block_downloader.on_action(DownloadAction::Download(DownloadRequest::block_set( B256Set::from_iter([tip.hash(), tip.parent_hash]), ))); @@ -421,7 +520,68 @@ mod tests { // ensure they are in ascending order for num in 1..=TOTAL_BLOCKS { - assert_eq!(blocks[num-1].number(), num as u64); + assert_eq!(blocks[num - 1].number(), num as u64); + } + }); + } + + #[tokio::test] + async fn block_downloader_range_request_with_access_lists() { + const TOTAL_BLOCKS: usize = 4; + let chain_spec = Arc::new( + ChainSpecBuilder::default() + .chain(MAINNET.chain) + .genesis(MAINNET.genesis.clone()) + .paris_activated() + .build(), + ); + + let client = TestFullBlockClient::default(); + // empty RLP list + let access_list = alloy_primitives::Bytes::from_static(&[0xc0]); + let header = Header { + base_fee_per_gas: Some(7), + gas_limit: ETHEREUM_BLOCK_GAS_LIMIT_30M, + block_access_list_hash: Some(alloy_primitives::keccak256(access_list.as_ref())), + ..Default::default() + }; + let mut sealed_header = SealedHeader::seal_slow(header); + let body = reth_ethereum_primitives::BlockBody::default(); + for _ in 0..TOTAL_BLOCKS { + let (mut header, hash) = sealed_header.split(); + header.parent_hash = hash; + header.number += 1; + header.timestamp += 1; + sealed_header = SealedHeader::seal_slow(header); + client.insert(sealed_header.clone(), body.clone()); + client.insert_access_list(sealed_header.hash(), access_list.clone()); + } + + let consensus = Arc::new(EthBeaconConsensus::new(chain_spec).with_allow_bal_hashes(true)); + let mut block_downloader = BasicBlockDownloader::new(client.clone(), consensus); + + let tip = client.highest_block().expect("there should be blocks here"); + + block_downloader.on_action(DownloadAction::Download( + DownloadRequest::block_range(tip.hash(), tip.number).with_access_lists(true), + )); + + let sync_future = poll_fn(|cx| block_downloader.poll(cx)); + let next_ready = sync_future.await; + + assert_matches!(next_ready, DownloadOutcome::NewDownloadStarted { remaining_blocks, .. } => { + assert_eq!(remaining_blocks, TOTAL_BLOCKS as u64); + }); + + let sync_future = poll_fn(|cx| block_downloader.poll(cx)); + let next_ready = sync_future.await; + + assert_matches!(next_ready, DownloadOutcome::Blocks(blocks) => { + assert_eq!(blocks.len(), TOTAL_BLOCKS); + + // every block carries its validated access list data + for block in &blocks { + assert!(block.data().is_some()); } }); } @@ -434,7 +594,7 @@ mod tests { let tip = client.highest_block().expect("there should be blocks here"); // send block range download request - block_downloader.on_action(DownloadAction::Download(DownloadRequest::BlockRange( + block_downloader.on_action(DownloadAction::Download(DownloadRequest::block_range( tip.hash(), tip.number, ))); @@ -442,7 +602,7 @@ mod tests { // send block set download request let download_set = B256Set::from_iter([tip.hash(), tip.parent_hash]); block_downloader - .on_action(DownloadAction::Download(DownloadRequest::BlockSet(download_set.clone()))); + .on_action(DownloadAction::Download(DownloadRequest::block_set(download_set.clone()))); // ensure we have one in flight range request assert_eq!(block_downloader.inflight_block_range_requests.len(), 1); diff --git a/crates/engine/tree/src/engine.rs b/crates/engine/tree/src/engine.rs index f7951426da1..4217cf447f7 100644 --- a/crates/engine/tree/src/engine.rs +++ b/crates/engine/tree/src/engine.rs @@ -10,8 +10,9 @@ use crossbeam_channel::Sender; use futures::{Stream, StreamExt}; use reth_engine_primitives::{BeaconEngineMessage, ConsensusEngineEvent}; use reth_ethereum_primitives::EthPrimitives; +use reth_network_p2p::full_block::SealedBlockWithAccessList; use reth_payload_primitives::{BuiltPayloadExecutedBlock, PayloadTypes}; -use reth_primitives_traits::{Block, NodePrimitives, SealedBlock}; +use reth_primitives_traits::{Block, NodePrimitives}; use std::{ fmt::Display, task::{ready, Context, Poll}, @@ -305,7 +306,7 @@ pub enum FromEngine { /// Request from the engine. Request(Req), /// Downloaded blocks from the network. - DownloadedBlocks(Vec>), + DownloadedBlocks(Vec>), } impl Display for FromEngine { @@ -339,14 +340,49 @@ pub enum RequestHandlerEvent { #[derive(Debug)] pub enum DownloadRequest { /// Download the given set of blocks. - BlockSet(B256Set), + BlockSet { + /// The hashes of the blocks to download. + hashes: B256Set, + /// Whether to also attempt to download the blocks' access lists. + access_lists: bool, + }, /// Download the given range of blocks. - BlockRange(B256, u64), + BlockRange { + /// The hash of the highest block of the range. + hash: B256, + /// The number of blocks to download. + count: u64, + /// Whether to also attempt to download the blocks' access lists. + access_lists: bool, + }, } impl DownloadRequest { /// Returns a [`DownloadRequest`] for a single block. pub fn single_block(hash: B256) -> Self { - Self::BlockSet(B256Set::from_iter([hash])) + Self::block_set(B256Set::from_iter([hash])) + } + + /// Returns a [`DownloadRequest`] for the given set of blocks. + pub const fn block_set(hashes: B256Set) -> Self { + Self::BlockSet { hashes, access_lists: false } + } + + /// Returns a [`DownloadRequest`] for the given range of blocks. + pub const fn block_range(hash: B256, count: u64) -> Self { + Self::BlockRange { hash, count, access_lists: false } + } + + /// Configures whether the download should also attempt to fetch the blocks' access lists. + /// + /// Access list downloads are best-effort: blocks are returned without access list data if no + /// peer can serve them. + pub const fn with_access_lists(mut self, access_lists: bool) -> Self { + match &mut self { + Self::BlockSet { access_lists: al, .. } | Self::BlockRange { access_lists: al, .. } => { + *al = access_lists + } + } + self } } diff --git a/crates/engine/tree/src/launch.rs b/crates/engine/tree/src/launch.rs index 2028cd8de4a..9d524c8c35e 100644 --- a/crates/engine/tree/src/launch.rs +++ b/crates/engine/tree/src/launch.rs @@ -16,7 +16,7 @@ use futures::Stream; use reth_consensus::FullConsensus; use reth_engine_primitives::BeaconEngineMessage; use reth_evm::ConfigureEvm; -use reth_network_p2p::BlockClient; +use reth_network_p2p::{BlockAccessListsClient, BlockClient}; use reth_payload_builder::PayloadBuilderHandle; use reth_primitives_traits::NodePrimitives; use reth_provider::{ @@ -75,7 +75,9 @@ pub fn build_engine_orchestrator( > where N: ProviderNodeTypes, - Client: BlockClient::Block> + 'static, + Client: BlockClient::Block> + + BlockAccessListsClient + + 'static, S: Stream> + Send + Sync + Unpin + 'static, V: EngineValidator + WaitForCaches, C: ConfigureEvm + 'static, diff --git a/crates/engine/tree/src/tree/block_buffer.rs b/crates/engine/tree/src/tree/block_buffer.rs index 28c9c445547..c0e917eb613 100644 --- a/crates/engine/tree/src/tree/block_buffer.rs +++ b/crates/engine/tree/src/tree/block_buffer.rs @@ -2,6 +2,7 @@ use crate::tree::metrics::BlockBufferMetrics; use alloy_consensus::BlockHeader; use alloy_primitives::{BlockHash, BlockNumber}; use indexmap::IndexSet; +use reth_network_p2p::full_block::SealedBlockWithAccessList; use reth_primitives_traits::{Block, SealedBlock}; use std::collections::{BTreeMap, HashMap, HashSet, VecDeque}; @@ -19,7 +20,7 @@ use std::collections::{BTreeMap, HashMap, HashSet, VecDeque}; #[derive(Debug)] pub struct BlockBuffer { /// All blocks in the buffer stored by their block hash. - pub(crate) blocks: HashMap>, + pub(crate) blocks: HashMap>, /// Map of any parent block hash (even the ones not currently in the buffer) /// to the buffered children. /// Allows connecting buffered blocks by parent. @@ -51,7 +52,7 @@ impl BlockBuffer { /// Return reference to the requested block. pub fn block(&self, hash: &BlockHash) -> Option<&SealedBlock> { - self.blocks.get(hash) + self.blocks.get(hash).map(|block| &**block) } /// Return a reference to the lowest ancestor of the given block in the buffer. @@ -64,11 +65,17 @@ impl BlockBuffer { } /// Insert a correct block inside the buffer. - pub fn insert_block(&mut self, block: SealedBlock) { + pub fn insert_block(&mut self, block: SealedBlockWithAccessList) { let hash = block.hash(); match self.blocks.entry(hash) { - std::collections::hash_map::Entry::Occupied(_) => return, + std::collections::hash_map::Entry::Occupied(mut entry) => { + // a duplicate that includes access list data is preferred over one without + if entry.get().data().is_none() && block.data().is_some() { + entry.insert(block); + } + return + } std::collections::hash_map::Entry::Vacant(entry) => { self.parent_to_child.entry(block.parent_hash()).or_default().insert(hash); self.earliest_blocks.entry(block.number()).or_default().insert(hash); @@ -93,7 +100,10 @@ impl BlockBuffer { /// /// Note: that order of returned blocks is important and the blocks with lower block number /// in the chain will come first so that they can be executed in the correct order. - pub fn remove_block_with_children(&mut self, parent_hash: &BlockHash) -> Vec> { + pub fn remove_block_with_children( + &mut self, + parent_hash: &BlockHash, + ) -> Vec> { let removed = self .remove_block(parent_hash) .into_iter() @@ -152,7 +162,7 @@ impl BlockBuffer { /// This method will only remove the block if it's present inside `self.blocks`. /// The block might be missing from other collections, the method will only ensure that it has /// been removed. - fn remove_block(&mut self, hash: &BlockHash) -> Option> { + fn remove_block(&mut self, hash: &BlockHash) -> Option> { let block = self.blocks.remove(hash)?; self.remove_from_earliest_blocks(block.number(), hash); self.remove_from_parent(block.parent_hash(), hash); @@ -161,7 +171,10 @@ impl BlockBuffer { } /// Remove all children and their descendants for the given blocks and return them. - fn remove_children(&mut self, parent_hashes: Vec) -> Vec> { + fn remove_children( + &mut self, + parent_hashes: Vec, + ) -> Vec> { // remove all parent child connection and all the child children blocks that are connected // to the discarded parent blocks. let mut remove_parent_children = parent_hashes; @@ -185,8 +198,9 @@ impl BlockBuffer { #[cfg(test)] mod tests { use super::*; + use alloy_eip7928::bal::RawBal; use alloy_eips::BlockNumHash; - use alloy_primitives::BlockHash; + use alloy_primitives::{BlockHash, Bytes}; use reth_testing_utils::generators::{self, random_block, BlockParams, Rng}; use std::collections::HashMap; @@ -238,11 +252,53 @@ mod tests { let block1 = create_block(&mut rng, 10, parent); let mut buffer = BlockBuffer::new(3); - buffer.insert_block(block1.clone()); + buffer.insert_block(block1.clone().into()); assert_buffer_lengths(&buffer, 1); assert_eq!(buffer.block(&block1.hash()), Some(&block1)); } + /// Creates a raw access list holding an empty RLP list. + fn raw_bal() -> RawBal { + RawBal::from(Bytes::from_static(&[alloy_rlp::EMPTY_LIST_CODE])) + } + + #[test] + fn preserves_access_list_for_buffered_blocks() { + let mut rng = generators::rng(); + + let access_list = raw_bal(); + let parent = rng.random(); + let block = create_block(&mut rng, 10, parent); + + let mut buffer = BlockBuffer::new(1); + buffer + .insert_block(SealedBlockWithAccessList::new(block.clone(), Some(access_list.clone()))); + + let blocks = buffer.remove_block_with_children(&parent); + assert_eq!(blocks.len(), 1); + assert_eq!(&*blocks[0], &block); + assert_eq!(blocks[0].data().as_ref(), Some(&access_list)); + } + + #[test] + fn updates_buffered_duplicate_with_access_list() { + let mut rng = generators::rng(); + + let access_list = raw_bal(); + let parent = rng.random(); + let block = create_block(&mut rng, 10, parent); + + let mut buffer = BlockBuffer::new(1); + buffer.insert_block(block.clone().into()); + buffer + .insert_block(SealedBlockWithAccessList::new(block.clone(), Some(access_list.clone()))); + + let blocks = buffer.remove_block_with_children(&parent); + assert_eq!(blocks.len(), 1); + assert_eq!(&*blocks[0], &block); + assert_eq!(blocks[0].data().as_ref(), Some(&access_list)); + } + #[test] fn take_entire_chain_of_children() { let mut rng = generators::rng(); @@ -256,10 +312,10 @@ mod tests { let mut buffer = BlockBuffer::new(5); - buffer.insert_block(block1.clone()); - buffer.insert_block(block2.clone()); - buffer.insert_block(block3.clone()); - buffer.insert_block(block4.clone()); + buffer.insert_block(block1.clone().into()); + buffer.insert_block(block2.clone().into()); + buffer.insert_block(block3.clone().into()); + buffer.insert_block(block4.clone().into()); assert_buffer_lengths(&buffer, 4); assert_eq!(buffer.block(&block4.hash()), Some(&block4)); @@ -270,7 +326,11 @@ mod tests { assert_eq!(buffer.lowest_ancestor(&block3.hash()), Some(&block1)); assert_eq!(buffer.lowest_ancestor(&block1.hash()), Some(&block1)); assert_eq!( - buffer.remove_block_with_children(&main_parent_hash), + buffer + .remove_block_with_children(&main_parent_hash) + .into_iter() + .map(|b| b.split().0) + .collect::>(), vec![block1, block2, block3] ); assert_buffer_lengths(&buffer, 1); @@ -288,17 +348,17 @@ mod tests { let mut buffer = BlockBuffer::new(5); - buffer.insert_block(block1.clone()); - buffer.insert_block(block2.clone()); - buffer.insert_block(block3.clone()); - buffer.insert_block(block4.clone()); + buffer.insert_block(block1.clone().into()); + buffer.insert_block(block2.clone().into()); + buffer.insert_block(block3.clone().into()); + buffer.insert_block(block4.clone().into()); assert_buffer_lengths(&buffer, 4); assert_eq!( buffer .remove_block_with_children(&main_parent_hash) .into_iter() - .map(|b| (b.hash(), b)) + .map(|b| (b.hash(), b.split().0)) .collect::>(), HashMap::from([ (block1.hash(), block1), @@ -322,17 +382,17 @@ mod tests { let mut buffer = BlockBuffer::new(5); - buffer.insert_block(block1.clone()); - buffer.insert_block(block2.clone()); - buffer.insert_block(block3.clone()); - buffer.insert_block(block4.clone()); + buffer.insert_block(block1.clone().into()); + buffer.insert_block(block2.clone().into()); + buffer.insert_block(block3.clone().into()); + buffer.insert_block(block4.clone().into()); assert_buffer_lengths(&buffer, 4); assert_eq!( buffer .remove_block_with_children(&block1.hash()) .into_iter() - .map(|b| (b.hash(), b)) + .map(|b| (b.hash(), b.split().0)) .collect::>(), HashMap::from([ (block1.hash(), block1), @@ -357,10 +417,10 @@ mod tests { let mut buffer = BlockBuffer::new(5); - buffer.insert_block(block1.clone()); - buffer.insert_block(block2); - buffer.insert_block(block3); - buffer.insert_block(block4); + buffer.insert_block(block1.clone().into()); + buffer.insert_block(block2.into()); + buffer.insert_block(block3.into()); + buffer.insert_block(block4.into()); assert_buffer_lengths(&buffer, 4); buffer.remove_old_blocks(block1.number); @@ -379,10 +439,10 @@ mod tests { let mut buffer = BlockBuffer::new(5); - buffer.insert_block(block1.clone()); - buffer.insert_block(block2); - buffer.insert_block(block3); - buffer.insert_block(block4); + buffer.insert_block(block1.clone().into()); + buffer.insert_block(block2.into()); + buffer.insert_block(block3.into()); + buffer.insert_block(block4.into()); assert_buffer_lengths(&buffer, 4); buffer.remove_old_blocks(block1.number); @@ -407,13 +467,13 @@ mod tests { let mut buffer = BlockBuffer::new(10); - buffer.insert_block(block1.clone()); - buffer.insert_block(block1a.clone()); - buffer.insert_block(block2.clone()); - buffer.insert_block(block2a.clone()); - buffer.insert_block(random_block1.clone()); - buffer.insert_block(random_block2.clone()); - buffer.insert_block(random_block3.clone()); + buffer.insert_block(block1.clone().into()); + buffer.insert_block(block1a.clone().into()); + buffer.insert_block(block2.clone().into()); + buffer.insert_block(block2a.clone().into()); + buffer.insert_block(random_block1.clone().into()); + buffer.insert_block(random_block2.clone().into()); + buffer.insert_block(random_block3.clone().into()); // check that random blocks are their own ancestor, and that chains have proper ancestors assert_eq!(buffer.lowest_ancestor(&random_block1.hash()), Some(&random_block1)); @@ -446,16 +506,16 @@ mod tests { let mut buffer = BlockBuffer::new(3); - buffer.insert_block(block1.clone()); - buffer.insert_block(block2.clone()); - buffer.insert_block(block3.clone()); + buffer.insert_block(block1.clone().into()); + buffer.insert_block(block2.clone().into()); + buffer.insert_block(block3.clone().into()); // pre-eviction block1 is the root assert_eq!(buffer.lowest_ancestor(&block3.hash()), Some(&block1)); assert_eq!(buffer.lowest_ancestor(&block2.hash()), Some(&block1)); assert_eq!(buffer.lowest_ancestor(&block1.hash()), Some(&block1)); - buffer.insert_block(block4.clone()); + buffer.insert_block(block4.clone().into()); assert_eq!(buffer.lowest_ancestor(&block4.hash()), Some(&block4)); @@ -483,10 +543,10 @@ mod tests { let mut buffer = BlockBuffer::new(3); - buffer.insert_block(block1.clone()); - buffer.insert_block(block2); - buffer.insert_block(block3); - buffer.insert_block(block4); + buffer.insert_block(block1.clone().into()); + buffer.insert_block(block2.into()); + buffer.insert_block(block3.into()); + buffer.insert_block(block4.into()); // block3 gets evicted assert_block_removal(&buffer, &block1); @@ -508,8 +568,8 @@ mod tests { // Capacity 2 so third insert evicts the oldest (block1) let mut buffer = BlockBuffer::new(2); - buffer.insert_block(block1.clone()); - buffer.insert_block(block2.clone()); + buffer.insert_block(block1.clone().into()); + buffer.insert_block(block2.clone().into()); // Pre-eviction: parent_to_child contains main_parent -> {block1}, block1 -> {block2} assert!(buffer @@ -524,7 +584,7 @@ mod tests { .is_some()); // Insert unrelated block to evict block1 - buffer.insert_block(unrelated_block); + buffer.insert_block(unrelated_block.into()); // Evicted block1 should be fully removed from collections assert_block_removal(&buffer, &block1); diff --git a/crates/engine/tree/src/tree/mod.rs b/crates/engine/tree/src/tree/mod.rs index dadeb923ed9..35fd1da056c 100644 --- a/crates/engine/tree/src/tree/mod.rs +++ b/crates/engine/tree/src/tree/mod.rs @@ -24,6 +24,7 @@ use reth_engine_primitives::{ }; use reth_errors::{ConsensusError, ProviderResult}; use reth_evm::ConfigureEvm; +use reth_network_p2p::full_block::SealedBlockWithAccessList; use reth_payload_builder::{BuildNewPayload, PayloadBuilderHandle, PayloadBuilderLease}; use reth_payload_primitives::{BuiltPayload, NewPayloadError, PayloadAttributes, PayloadTypes}; use reth_primitives_traits::{ @@ -724,7 +725,7 @@ where /// block request processing isn't blocked for a long time. fn on_downloaded( &mut self, - mut blocks: Vec>, + mut blocks: Vec>, ) -> Result, InsertBlockFatalError> { if blocks.is_empty() { // nothing to execute @@ -1429,7 +1430,10 @@ where Ok(TreeOutcome::new(OnForkChoiceUpdated::valid(PayloadStatus::from_status( PayloadStatusEnum::Syncing, ))) - .with_event(TreeEvent::Download(DownloadRequest::single_block(target)))) + .with_event(TreeEvent::Download( + DownloadRequest::single_block(target) + .with_access_lists(self.should_download_access_lists()), + ))) } /// Helper method to remove blocks and set the persistence state. This ensures we keep track of @@ -2028,10 +2032,10 @@ where "Backfill complete, downloading remaining blocks to reach FCU target" ); - self.emit_event(EngineApiEvent::Download(DownloadRequest::BlockRange( - lowest_buffered.parent_hash(), - distance, - ))); + self.emit_event(EngineApiEvent::Download( + DownloadRequest::block_range(lowest_buffered.parent_hash(), distance) + .with_access_lists(self.should_download_access_lists()), + )); return Ok(()); } } else { @@ -2042,9 +2046,10 @@ where head_hash = %sync_target_state.head_block_hash, "Backfill complete but head block not buffered, requesting download" ); - self.emit_event(EngineApiEvent::Download(DownloadRequest::single_block( - sync_target_state.head_block_hash, - ))); + self.emit_event(EngineApiEvent::Download( + DownloadRequest::single_block(sync_target_state.head_block_hash) + .with_access_lists(self.should_download_access_lists()), + )); return Ok(()); } @@ -2421,6 +2426,18 @@ where .unwrap_or_else(|| hash) } + /// Returns whether block downloads should also attempt to fetch the blocks' access lists. + /// + /// This is the case once Amsterdam is active at the current canonical head, indicated by the + /// head header carrying a block access list hash. + /// + /// This is a coarse gate: a download issued while the head is still pre-Amsterdam won't ask + /// for access lists of blocks past the fork. Access list downloads are best-effort anyway, so + /// those blocks simply take the regular execution path. + fn should_download_access_lists(&self) -> bool { + self.canonical_in_memory_state.get_canonical_head().block_access_list_hash().is_some() + } + /// If validation fails, the response MUST contain the latest valid hash: /// /// - The block hash of the ancestor of the invalid payload satisfying the following two @@ -2680,7 +2697,7 @@ where if let Err(err) = self.validate_block(&block) { return Err(InsertBlockError::consensus_error(err, block)) } - self.state.buffer.insert_block(block); + self.state.buffer.insert_block(block.into()); Ok(()) } @@ -2938,7 +2955,7 @@ where self.distance_from_local_tip(head.number, missing_parent.number) { trace!(target: "engine::tree", %distance, missing=?missing_parent, "downloading missing parent block range"); - DownloadRequest::BlockRange(missing_parent.hash, distance) + DownloadRequest::block_range(missing_parent.hash, distance) } else { trace!(target: "engine::tree", missing=?missing_parent, "downloading missing parent block"); // This happens when the missing parent is on an outdated @@ -2946,7 +2963,7 @@ where DownloadRequest::single_block(missing_parent.hash) }; - Some(TreeEvent::Download(request)) + Some(TreeEvent::Download(request.with_access_lists(self.should_download_access_lists()))) } /// Handles a downloaded block that was successfully inserted as valid. @@ -2984,7 +3001,10 @@ where if self.state.tree_state.canonical_block_hash() != sync_target.head_block_hash { let target = self.lowest_buffered_ancestor_or(sync_target.head_block_hash); trace!(target: "engine::tree", %target, "sync target head not yet reached, downloading head block"); - return Ok(Some(TreeEvent::Download(DownloadRequest::single_block(target)))) + return Ok(Some(TreeEvent::Download( + DownloadRequest::single_block(target) + .with_access_lists(self.should_download_access_lists()), + ))) } return Ok(None) @@ -3002,7 +3022,7 @@ where #[instrument(level = "debug", target = "engine::tree", skip_all, fields(block_hash = %block.hash(), block_num = %block.number()))] fn on_downloaded_block( &mut self, - block: SealedBlock, + block: SealedBlockWithAccessList, ) -> Result, InsertBlockFatalError> { let block_num_hash = block.num_hash(); let lowest_buffered_ancestor = self.lowest_buffered_ancestor_or(block_num_hash.hash); @@ -3061,13 +3081,13 @@ where payload.block_with_parent(), payload, |validator, payload, ctx| validator.validate_payload(payload, ctx), - |this, payload| Ok(this.payload_validator.convert_payload_to_block(payload)?), + |this, payload| Ok(this.payload_validator.convert_payload_to_block(payload)?.into()), ) } fn insert_block( &mut self, - block: SealedBlock, + block: SealedBlockWithAccessList, ) -> Result> { self.insert_block_or_payload( block.block_with_parent(), @@ -3099,7 +3119,10 @@ where block_id: BlockWithParent, input: Input, execute: impl FnOnce(&mut V, Input, TreeCtx<'_, N>) -> Result, Err>, - convert_to_block: impl FnOnce(&mut Self, Input) -> Result, Err>, + convert_to_block: impl FnOnce( + &mut Self, + Input, + ) -> Result, Err>, ) -> Result where Err: From>, @@ -3120,7 +3143,7 @@ where match self.provider.sealed_header_by_hash(block_num_hash.hash) { Err(err) => { let block = convert_to_block(self, input)?; - return Err(InsertBlockError::new(block, err.into()).into()); + return Err(InsertBlockError::new(block.split().0, err.into()).into()); } Ok(Some(_)) => { convert_to_block(self, input)?; @@ -3136,7 +3159,7 @@ where Ok(header) => header.is_some(), Err(err) => { let block = convert_to_block(self, input)?; - return Err(InsertBlockError::new(block, err.into()).into()); + return Err(InsertBlockError::new(block.split().0, err.into()).into()); } }; diff --git a/crates/engine/tree/src/tree/payload_validator.rs b/crates/engine/tree/src/tree/payload_validator.rs index ecf4632527f..631a692ac2a 100644 --- a/crates/engine/tree/src/tree/payload_validator.rs +++ b/crates/engine/tree/src/tree/payload_validator.rs @@ -141,6 +141,7 @@ use reth_evm::{ OnStateHook, SpecFor, }; use reth_execution_cache::{CacheFillMode, CacheStats}; +use reth_network_p2p::full_block::SealedBlockWithAccessList; use reth_payload_builder::{PayloadBuilderLease, PayloadBuilderResources}; use reth_payload_primitives::{ BuiltPayload, BuiltPayloadExecutedBlock, InvalidPayloadAttributesError, NewPayloadError, @@ -404,7 +405,7 @@ where { match input { BlockOrPayload::Payload(payload) => self.validator.convert_payload_to_block(payload), - BlockOrPayload::Block(block) => Ok(block), + BlockOrPayload::Block(block) => Ok(block.split().0), } } @@ -941,7 +942,7 @@ where ) .entered(); let block = match input { - BlockOrPayload::Block(block) => block, + BlockOrPayload::Block(block) => block.split().0, BlockOrPayload::Payload(payload) => { validator.convert_payload_to_block(payload)? } @@ -1756,10 +1757,10 @@ pub trait EngineValidator< ctx: TreeCtx<'_, N>, ) -> ValidationOutcome; - /// Validates a block downloaded from the network. + /// Validates a block downloaded from the network, optionally carrying its access list data. fn validate_block( &mut self, - block: SealedBlock, + block: SealedBlockWithAccessList, ctx: TreeCtx<'_, N>, ) -> ValidationOutcome; @@ -1846,7 +1847,7 @@ where fn validate_block( &mut self, - block: SealedBlock, + block: SealedBlockWithAccessList, ctx: TreeCtx<'_, N>, ) -> ValidationOutcome { self.validate_block_with_state(BlockOrPayload::Block(block), ctx) @@ -1984,8 +1985,8 @@ where pub enum BlockOrPayload { /// Payload. Payload(T::ExecutionData), - /// Block. - Block(SealedBlock::Primitives>>), + /// Block with optional access list data, e.g. downloaded from the network. + Block(SealedBlockWithAccessList::Primitives>>), } impl BlockOrPayload { @@ -2046,7 +2047,7 @@ impl BlockOrPayload { .block_access_list() .map(|block_access_list| DecodedBal::from_rlp_bytes(block_access_list.clone())) .transpose(), - Self::Block(_) => Ok(None), + Self::Block(block) => block.data().clone().map(DecodedBal::from_raw_bal).transpose(), } } diff --git a/crates/engine/tree/src/tree/tests.rs b/crates/engine/tree/src/tree/tests.rs index df998ccac14..f23a55a6bf4 100644 --- a/crates/engine/tree/src/tree/tests.rs +++ b/crates/engine/tree/src/tree/tests.rs @@ -44,6 +44,13 @@ use std::{ }; use tokio::sync::oneshot; +/// Wraps blocks as if they had been downloaded without any access list data. +fn downloaded_blocks( + blocks: Vec>, +) -> Vec> { + blocks.into_iter().map(SealedBlockWithAccessList::from_block).collect() +} + /// Mock engine validator for tests #[derive(Debug, Clone)] struct MockEngineValidator; @@ -58,7 +65,8 @@ impl reth_engine_primitives::PayloadValidator for MockEngineVali reth_primitives_traits::SealedBlock, reth_payload_primitives::NewPayloadError, > { - let block = reth_ethereum_primitives::Block::try_from(payload.payload).map_err(|e| { + let ExecutionData { payload, sidecar } = payload; + let block = payload.try_into_block_with_sidecar(&sidecar).map_err(|e| { reth_payload_primitives::NewPayloadError::Other(format!("{e:?}").into()) })?; Ok(block.seal_slow()) @@ -475,7 +483,8 @@ impl ValidatorTestHarness { &mut self.harness.tree.state, &self.harness.tree.canonical_in_memory_state, ); - let result = self.validator.validate_block(block, ctx); + let result = + self.validator.validate_block(SealedBlockWithAccessList::from_block(block), ctx); self.metrics.record_validation(result.is_ok()); result } @@ -542,7 +551,7 @@ fn test_tree_persist_block_batch() { blocks.push(test_block_builder.generate_random_block(idx as u64, B256::random())); } - test_harness.to_tree_tx.send(FromEngine::DownloadedBlocks(blocks)).unwrap(); + test_harness.to_tree_tx.send(FromEngine::DownloadedBlocks(downloaded_blocks(blocks))).unwrap(); // process the message let msg = match test_harness.tree.wait_for_event() { @@ -1011,7 +1020,10 @@ fn test_disconnected_block() { let mut test_harness = TestHarness::new(HOLESKY.clone()); - let outcome = test_harness.tree.insert_block(sealed.clone()).unwrap(); + let outcome = test_harness + .tree + .insert_block(SealedBlockWithAccessList::from_block(sealed.clone())) + .unwrap(); assert_eq!( outcome, InsertPayloadOk::Inserted(BlockStatus::Disconnected { @@ -1045,7 +1057,7 @@ fn test_validated_payload_bal_is_inserted_into_store() { .with_raw_bal(Some(RawBal::from(raw_bal.clone()))), ) }, - |_, executed| Ok(executed.recovered_block().clone_sealed_block()), + |_, executed| Ok(executed.recovered_block().clone_sealed_block().into()), ) .unwrap(); @@ -1520,7 +1532,9 @@ async fn test_engine_tree_fcu_missing_head() { // after FCU we receive an EngineApiEvent::Download event to get the missing block. let event = test_harness.from_tree_rx.recv().await.unwrap(); match event { - EngineApiEvent::Download(DownloadRequest::BlockSet(actual_block_set)) => { + EngineApiEvent::Download(DownloadRequest::BlockSet { + hashes: actual_block_set, .. + }) => { let expected_block_set = B256Set::from_iter([missing_block.hash()]); assert_eq!(actual_block_set, expected_block_set); } @@ -1565,7 +1579,7 @@ async fn test_engine_tree_live_sync_transition_required_blocks_requested() { let event = test_harness.from_tree_rx.recv().await.unwrap(); match event { - EngineApiEvent::Download(DownloadRequest::BlockSet(hash_set)) => { + EngineApiEvent::Download(DownloadRequest::BlockSet { hashes: hash_set, .. }) => { assert_eq!(hash_set, B256Set::from_iter([main_chain_last_hash])); } _ => panic!("Unexpected event: {event:#?}"), @@ -1574,7 +1588,7 @@ async fn test_engine_tree_live_sync_transition_required_blocks_requested() { // After backfill completes with head not buffered, we also request head download let event = test_harness.from_tree_rx.recv().await.unwrap(); match event { - EngineApiEvent::Download(DownloadRequest::BlockSet(hash_set)) => { + EngineApiEvent::Download(DownloadRequest::BlockSet { hashes: hash_set, .. }) => { assert_eq!(hash_set, B256Set::from_iter([main_chain_last_hash])); } _ => panic!("Unexpected event: {event:#?}"), @@ -1582,15 +1596,19 @@ async fn test_engine_tree_live_sync_transition_required_blocks_requested() { let _ = test_harness .tree - .on_engine_message(FromEngine::DownloadedBlocks(vec![main_chain + .on_engine_message(FromEngine::DownloadedBlocks(downloaded_blocks(vec![main_chain .last() .unwrap() - .clone_sealed_block()])) + .clone_sealed_block()]))) .unwrap(); let event = test_harness.from_tree_rx.recv().await.unwrap(); match event { - EngineApiEvent::Download(DownloadRequest::BlockRange(initial_hash, total_blocks)) => { + EngineApiEvent::Download(DownloadRequest::BlockRange { + hash: initial_hash, + count: total_blocks, + .. + }) => { assert_eq!( total_blocks, (main_chain.len() - backfill_finished_block_number as usize - 1) as u64 @@ -2560,7 +2578,7 @@ mod forkchoice_updated_tests { if let Some(TreeEvent::Download(download_request)) = result.event { match download_request { - DownloadRequest::BlockSet(block_set) => { + DownloadRequest::BlockSet { hashes: block_set, .. } => { assert_eq!(block_set.len(), 1); } _ => panic!("Expected single block download request"), @@ -2834,7 +2852,7 @@ fn test_on_valid_downloaded_non_head_sync_target_continues_to_head() { // With the fix: the engine makes safe canonical inline, then emits Download for head. // Without the fix: it would return MakeCanonical{safe_hash} and never download head. match result { - Some(TreeEvent::Download(DownloadRequest::BlockSet(hashes))) => { + Some(TreeEvent::Download(DownloadRequest::BlockSet { hashes, .. })) => { assert!( hashes.contains(&head_hash), "Expected download for head block {head_hash}, got {hashes:?}" @@ -3172,7 +3190,7 @@ async fn assert_post_backfill_recheck_retriggers_to_buffered_target(engine_kind: // Buffer the target block — the recheck looks up the helper's resolved hash in the buffer // to decide whether to retrigger. - test_harness.tree.state.buffer.insert_block(target_block.clone_sealed_block()); + test_harness.tree.state.buffer.insert_block(target_block.clone_sealed_block().into()); // Place the buffered hash in the FCU slot the helper picks for this chain type, and put // an unrelated hash (not in buffer) in the other slot to keep the two slots distinct. From e72d761cb0428d0eb0cba517dbb256d931951ec3 Mon Sep 17 00:00:00 2001 From: nk_ysg Date: Tue, 1 Sep 2026 14:15:13 +0000 Subject: [PATCH 112/294] fix(txpool): hold sidecar permit until conversion completes (#26847) --- crates/transaction-pool/src/blobstore/converter.rs | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/crates/transaction-pool/src/blobstore/converter.rs b/crates/transaction-pool/src/blobstore/converter.rs index 3f6abc56bff..2e64071603c 100644 --- a/crates/transaction-pool/src/blobstore/converter.rs +++ b/crates/transaction-pool/src/blobstore/converter.rs @@ -21,10 +21,13 @@ impl BlobSidecarConverter { &self, sidecar: BlobTransactionSidecar, ) -> Option { - let _permit = SEMAPHORE.acquire().await.ok()?; - tokio::task::spawn_blocking(move || sidecar.try_into_7594(EnvKzgSettings::Default.get())) - .await - .ok()? - .ok() + let permit = SEMAPHORE.acquire().await.ok()?; + tokio::task::spawn_blocking(move || { + let _permit = permit; + sidecar.try_into_7594(EnvKzgSettings::Default.get()) + }) + .await + .ok()? + .ok() } } From 98120568b6f1c133fbffefd0371e1af6413271ee Mon Sep 17 00:00:00 2001 From: Brian Picciano Date: Tue, 1 Sep 2026 14:51:27 +0000 Subject: [PATCH 113/294] feat(provider): implement multiproof_v2 on OverlayStateProvider (#26906) Co-authored-by: Brian Picciano <933154+mediocregopher@users.noreply.github.com> Co-authored-by: Derek Cofausper <256792747+decofe@users.noreply.github.com> --- crates/storage/storage-api/src/trie.rs | 10 +++----- .../storage/storage-overlay/src/provider.rs | 25 ++++++++++++++++--- 2 files changed, 26 insertions(+), 9 deletions(-) diff --git a/crates/storage/storage-api/src/trie.rs b/crates/storage/storage-api/src/trie.rs index 119de13d8b3..ffff6e09de1 100644 --- a/crates/storage/storage-api/src/trie.rs +++ b/crates/storage/storage-api/src/trie.rs @@ -1,7 +1,7 @@ use alloc::{boxed::Box, vec::Vec}; use alloy_primitives::{Address, Bytes, B256, U256}; use reth_primitives_traits::Account; -use reth_storage_errors::provider::{ProviderError, ProviderResult}; +use reth_storage_errors::provider::ProviderResult; use reth_trie_common::{ updates::{StorageTrieUpdatesSorted, TrieUpdates, TrieUpdatesSorted}, AccountProof, DecodedMultiProofV2, ExecutionWitnessMode, HashedPostState, HashedStorage, @@ -165,11 +165,9 @@ pub trait StateProofProvider { /// Generate a V2 decoded multiproof for target hashed accounts and storage slots. fn multiproof_v2( &self, - _input: TrieInput, - _targets: MultiProofTargetsV2, - ) -> ProviderResult { - Err(ProviderError::UnsupportedProvider) - } + input: TrieInput, + targets: MultiProofTargetsV2, + ) -> ProviderResult; /// Get trie witness for provided state using the given witness generation mode. fn witness( diff --git a/crates/storage/storage-overlay/src/provider.rs b/crates/storage/storage-overlay/src/provider.rs index 7403544cebb..b1157a7c240 100644 --- a/crates/storage/storage-overlay/src/provider.rs +++ b/crates/storage/storage-overlay/src/provider.rs @@ -26,9 +26,10 @@ use reth_trie::{ }, updates::TrieUpdates, witness::TrieWitness, - AccountProof, ExecutionWitnessMode, HashedPostState, HashedPostStateSorted, HashedStorage, - KeccakKeyHasher, MultiProof, MultiProofTargets, StateRoot, StorageMultiProof, StorageProof, - StorageRoot, TrieInput, TrieInputSorted, + AccountProof, DecodedMultiProofV2, ExecutionWitnessMode, HashedPostState, + HashedPostStateSorted, HashedStorage, KeccakKeyHasher, MultiProof, MultiProofTargets, + MultiProofTargetsV2, StateRoot, StorageMultiProof, StorageProof, StorageRoot, TrieInput, + TrieInputSorted, }; use reth_trie_db::{ DatabaseAccountTrieCursor, DatabaseHashedCursorFactory, DatabaseProof, DatabaseStateRoot, @@ -624,6 +625,24 @@ where }) } + fn multiproof_v2( + &self, + input: TrieInput, + targets: MultiProofTargetsV2, + ) -> ProviderResult { + reth_trie_db::with_adapter!(self.provider(), |A| { + let TrieInputSorted { nodes, state, prefix_sets } = + self.build_overlay(TrieInputSorted::from_unsorted(input))?; + let input = TrieInput::new( + Arc::unwrap_or_clone(nodes).into(), + Arc::unwrap_or_clone(state).into(), + prefix_sets, + ); + let proof = as DatabaseProof>::from_tx(self.provider().tx()); + proof.overlay_multiproof_v2(input, targets).map_err(ProviderError::from) + }) + } + fn witness( &self, input: TrieInput, From 0b3475a83e0712beb3d1f639ea467c55c5117412 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Tue, 1 Sep 2026 18:18:14 +0000 Subject: [PATCH 114/294] fix(stages): log bad block hash (#26909) Co-authored-by: DaniPopes <57450786+DaniPopes@users.noreply.github.com> --- crates/engine/tree/src/tree/precompile_cache.rs | 9 +++++++-- crates/stages/api/src/pipeline/mod.rs | 2 ++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/crates/engine/tree/src/tree/precompile_cache.rs b/crates/engine/tree/src/tree/precompile_cache.rs index e868cca8485..23146ca3914 100644 --- a/crates/engine/tree/src/tree/precompile_cache.rs +++ b/crates/engine/tree/src/tree/precompile_cache.rs @@ -14,6 +14,9 @@ use tracing::error; /// Default max cache size for [`PrecompileCache`] const MAX_CACHE_SIZE: u32 = 1024 * 1024; +/// Maximum calldata size to cache for a precompile. +const MAX_PRECOMPILE_CACHE_INPUT_SIZE: usize = 2 * 1024; + /// Stores caches for each precompile. #[derive(Debug, Clone, Default)] pub struct PrecompileCacheMap(Arc, FbBuildHasher<20>>>) @@ -179,7 +182,9 @@ where } fn call(&self, input: PrecompileInput<'_>) -> PrecompileResult { - if let Some(entry) = &self.cache.get(input.data, self.spec_id.clone()) && + let cacheable_input = input.data.len() <= MAX_PRECOMPILE_CACHE_INPUT_SIZE; + if cacheable_input && + let Some(entry) = &self.cache.get(input.data, self.spec_id.clone()) && input.gas >= entry.gas_used() { self.increment_by_one_precompile_cache_hits(); @@ -193,7 +198,7 @@ where match &result { // Only successful outputs are cacheable. Non-success statuses and errors must execute // again instead of poisoning the cache for subsequent calls. - Ok(output) if output.is_success() => { + Ok(output) if cacheable_input && output.is_success() => { // Sanity-check precompile output to ensure that it does not affect state gas in any // way. // diff --git a/crates/stages/api/src/pipeline/mod.rs b/crates/stages/api/src/pipeline/mod.rs index 0c2f3b668d6..36a8a6f3c3d 100644 --- a/crates/stages/api/src/pipeline/mod.rs +++ b/crates/stages/api/src/pipeline/mod.rs @@ -585,6 +585,7 @@ impl Pipeline { target: "sync::pipeline", stage = %stage_id, bad_block = %block.block.number, + bad_block_hash = %block.block.hash, "Stage encountered a validation error: {validation_error}" ); @@ -609,6 +610,7 @@ impl Pipeline { target: "sync::pipeline", stage = %stage_id, bad_block = %block.block.number, + bad_block_hash = %block.block.hash, "Stage encountered an execution error: {execution_error}" ); } From 8ba930bface320822936cf7c04e59d6351e5c4bd Mon Sep 17 00:00:00 2001 From: Derek Cofausper <256792747+decofe@users.noreply.github.com> Date: Tue, 1 Sep 2026 22:33:36 +0000 Subject: [PATCH 115/294] fix(bench): suppress storage overlay OTLP logs (#26908) Co-authored-by: Tim <12827757+laibe@users.noreply.github.com> Co-authored-by: Matthias Seitz --- .github/scripts/bench-txgen-run.sh | 2 +- .../engine/tree/src/tree/precompile_cache.rs | 53 +++++++++++++++++++ 2 files changed, 54 insertions(+), 1 deletion(-) diff --git a/.github/scripts/bench-txgen-run.sh b/.github/scripts/bench-txgen-run.sh index b709cf7c544..6aaf9a113d9 100755 --- a/.github/scripts/bench-txgen-run.sh +++ b/.github/scripts/bench-txgen-run.sh @@ -298,7 +298,7 @@ if [ "${BENCH_OTLP_DISABLED:-false}" != "true" ]; then RETH_ARGS+=(--tracing-otlp="${BENCH_OTLP_TRACES_ENDPOINT}" --tracing-otlp.service-name=reth-bench --tracing-otlp.service-version="${LABEL}") fi if [ -n "${BENCH_OTLP_LOGS_ENDPOINT:-}" ]; then - RETH_ARGS+=(--logs-otlp="${BENCH_OTLP_LOGS_ENDPOINT}" --logs-otlp.filter=debug) + RETH_ARGS+=(--logs-otlp="${BENCH_OTLP_LOGS_ENDPOINT}" --logs-otlp.filter=debug,storage::overlay=off) fi fi diff --git a/crates/engine/tree/src/tree/precompile_cache.rs b/crates/engine/tree/src/tree/precompile_cache.rs index 23146ca3914..e4b570043ab 100644 --- a/crates/engine/tree/src/tree/precompile_cache.rs +++ b/crates/engine/tree/src/tree/precompile_cache.rs @@ -217,6 +217,8 @@ where self.increment_by_one_precompile_cache_misses(); } } + // Oversized successful inputs execute normally but are not cacheable. + Ok(output) if output.is_success() => {} _ => { self.increment_by_one_precompile_errors(); } @@ -255,6 +257,7 @@ impl CachedPrecompileMetrics { #[cfg(test)] mod tests { use super::*; + use metrics_util::debugging::{DebugValue, DebuggingRecorder}; use reth_evm::{EthEvmFactory, Evm, EvmEnv, EvmFactory}; use reth_revm::db::EmptyDB; use revm::{ @@ -412,4 +415,54 @@ mod tests { .unwrap(); assert_eq!(result3.as_ref(), b"output_from_precompile_1"); } + + #[test] + fn test_oversized_successful_input_is_not_an_error() { + let recorder = DebuggingRecorder::new(); + let snapshotter = recorder.snapshotter(); + let cache = PrecompileCache::default(); + let input_data = Bytes::from(vec![0; MAX_PRECOMPILE_CACHE_INPUT_SIZE + 1]); + let address = Address::with_last_byte(1); + + metrics::with_local_recorder(&recorder, || { + let precompile: DynPrecompile = (|_input: PrecompileInput<'_>| { + Ok(PrecompileOutput { + status: PrecompileStatus::Success, + gas_used: 0, + state_gas_used: 0, + state_gas_spilled: 0, + reservoir: 0, + gas_refunded: 0, + bytes: Bytes::default(), + }) + }) + .into(); + let wrapped = CachedPrecompile::wrap( + precompile, + cache.clone(), + SpecId::PRAGUE, + Some(CachedPrecompileMetrics::new_with_address(address)), + ); + let mut evm = + EthEvmFactory::default().create_evm(EmptyDB::default(), EvmEnv::default()); + evm.precompiles_mut().apply_precompile(&address, |_| Some(wrapped)); + + evm.transact_raw(TxEnv { + caller: Address::ZERO, + gas_limit: 100_000, + data: input_data.clone(), + kind: address.into(), + ..Default::default() + }) + .unwrap(); + }); + + assert!(cache.get(&input_data, SpecId::PRAGUE).is_none()); + let error_count = snapshotter.snapshot().into_vec().into_iter().find_map( + |(key, _unit, _description, value)| { + (key.key().name() == "sync.caching.precompile_errors").then_some(value) + }, + ); + assert_eq!(error_count, Some(DebugValue::Counter(0))); + } } From 9571dec6ea3c1ad7a857214ec24ceaa0b0c9d89b Mon Sep 17 00:00:00 2001 From: rakita Date: Wed, 2 Sep 2026 07:35:04 +0000 Subject: [PATCH 116/294] chore(deps): reapply revm 43 update (#26859) --- Cargo.lock | 108 +++++++++--------- Cargo.toml | 16 +-- .../custom-beacon-withdrawals/src/main.rs | 2 +- 3 files changed, 63 insertions(+), 63 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 089af079131..cbab24bf9d7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -293,9 +293,9 @@ dependencies = [ [[package]] name = "alloy-evm" -version = "0.38.0" +version = "0.39.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b307a52f91f7f3aad56b5cb4f54e5eb17dd0b3f4b2c82823301499fe1c6fdaf" +checksum = "7b6bd5f7e5ce9858bac99275e3f548c3b63bb3e5806da0142dc0b2f518d3ec4a" dependencies = [ "alloy-consensus", "alloy-eips", @@ -1967,7 +1967,7 @@ version = "3.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6dee98b0db6a962de883bf5d20362dee4d7ca0d12fe39a7c6c73c844e1cd7c1f" dependencies = [ - "darling 0.21.3", + "darling 0.23.0", "ident_case", "prettyplease", "proc-macro2", @@ -3429,7 +3429,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.52.0", + "windows-sys 0.60.2", ] [[package]] @@ -5173,7 +5173,7 @@ checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" dependencies = [ "hermit-abi", "libc", - "windows-sys 0.52.0", + "windows-sys 0.60.2", ] [[package]] @@ -6241,7 +6241,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.60.2", ] [[package]] @@ -7962,9 +7962,9 @@ dependencies = [ [[package]] name = "reth-codecs" -version = "0.6.0" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f0dd7f542fbcae0e490ebe13f3eababcc05547375246ff7f62700b533cf3f01" +checksum = "d34b0b409ecf930ed3850e92bd9b4d4f8a8c397eef23e8ba5b1a1a02682a94bd" dependencies = [ "alloy-consensus", "alloy-eips", @@ -7983,9 +7983,9 @@ dependencies = [ [[package]] name = "reth-codecs-derive" -version = "0.6.0" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7e508fcbb8cb775639dbbb56b81b62756d4322f28e76c60a52aa32575f993fe" +checksum = "ebcb49ab084e764e7d20f710e2c9daa439380d0fc33f8bfee8c372bfdbc1f437" dependencies = [ "proc-macro2", "quote", @@ -9706,9 +9706,9 @@ dependencies = [ [[package]] name = "reth-primitives-traits" -version = "0.6.0" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4666a8d5b246650b23e6767a45e46334d352557f473e91c2ea7211cc29291cca" +checksum = "9060738a07ca1b0d8ede3ce21d8187c57cc22ee9e2991b29cbbc663280f843cd" dependencies = [ "alloy-consensus", "alloy-eips", @@ -10237,9 +10237,9 @@ dependencies = [ [[package]] name = "reth-rpc-traits" -version = "0.6.0" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b417a50d3fa9b58bb1a22e46fb47e0866006f24abffa1fbe75ae60116b5b2efa" +checksum = "f410d6bff1fd059ebaec2edeff1326f103f61162158411325d78158f94b2b4a9" dependencies = [ "alloy-consensus", "alloy-network", @@ -10792,18 +10792,18 @@ dependencies = [ [[package]] name = "reth-zstd-compressors" -version = "0.6.0" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "80addacffc5df9398c55919f8993922d2b688f2aa5753708fdd658ca3cc63e39" +checksum = "08f1ebb37e81a596ce16e81bec6a0c9accbdb914cf19b9190eff69ed00395900" dependencies = [ "zstd", ] [[package]] name = "revm" -version = "42.0.1" +version = "43.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7c7fcd426beab3b91300a6d0d5eb8d583c0fad0fba451126e19fd51443eca62" +checksum = "b0d1681d52ee61171b838db4924827f63d659d4c3ad559aa3a57003f3dda3f7c" dependencies = [ "revm-bytecode", "revm-context", @@ -10820,9 +10820,9 @@ dependencies = [ [[package]] name = "revm-bytecode" -version = "42.0.0" +version = "43.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "537356bdec80dbc04e4e6bb688b3441ff7cf18eecdcf3cac23150633c46467f1" +checksum = "c8d38e16b72ec1c836ec8490562bfb21003165ea8bc496086bf345680a6e6c41" dependencies = [ "bitvec", "paste", @@ -10833,9 +10833,9 @@ dependencies = [ [[package]] name = "revm-context" -version = "42.0.0" +version = "43.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc6512f05f83378e29b2b016fa5bd7f5a533e3625548be4113732e1a536c2564" +checksum = "f85bce65cc8969955309d5c4c37a63612209c720d040f1982dda6b59e39ab059" dependencies = [ "bitvec", "cfg-if", @@ -10850,9 +10850,9 @@ dependencies = [ [[package]] name = "revm-context-interface" -version = "42.0.0" +version = "43.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "103fff818b10311034c0c11ff8760294ee0e3c320daab3c5717990fe8ee2617b" +checksum = "d97d0c7f8bdf65601d35ed27cf6b0f0bd436087f0a97a52aa767a49bbcea8373" dependencies = [ "alloy-eip2930", "alloy-eip7702", @@ -10866,9 +10866,9 @@ dependencies = [ [[package]] name = "revm-database" -version = "42.0.0" +version = "43.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df72760eab0852100e748c7cb03ce0221626d60ac14534118c84c5cc894fb4c9" +checksum = "31d764afa9a1c9278c05d40461b07319d21ccb39c1bdf0e9dde7e5ea7d9be53d" dependencies = [ "alloy-eips", "derive_more", @@ -10882,9 +10882,9 @@ dependencies = [ [[package]] name = "revm-database-interface" -version = "42.0.0" +version = "43.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9286c059a11be1dee814cafdc3a3ade57521c8caa050785a52c19a9d894f400" +checksum = "d2a27e4440c2ef4932afdaa28a1d1ac8601504c7030c1a387b5e307d07d864d1" dependencies = [ "auto_impl", "either", @@ -10896,9 +10896,9 @@ dependencies = [ [[package]] name = "revm-handler" -version = "42.0.1" +version = "43.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09126175c3fa482e71e80cc28c6bb17e681911e472b94d4f7f73dd6780411798" +checksum = "ab0b5923be8784704d119be5381d76a602f65e4a9991650bb87291565bd0d8ec" dependencies = [ "auto_impl", "derive-where", @@ -10915,9 +10915,9 @@ dependencies = [ [[package]] name = "revm-inspector" -version = "42.0.1" +version = "43.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4aa55b208b52f0c083ea68147c78866eeecadd7c0071a614f477de316b60648e" +checksum = "a988ad0879bd81b269cad53c235a8ce87cd44a7c1de7d21a6ff3dd9e9f3e70a9" dependencies = [ "auto_impl", "either", @@ -10933,9 +10933,9 @@ dependencies = [ [[package]] name = "revm-inspectors" -version = "0.42.2" +version = "0.43.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90c136a072540987ee442e9824631835520b13e5756f20780a637ba2e11f9c28" +checksum = "79b609cb78fcb3a8c10f3190847c9bb0223423cf46e2de30a61ea9e37a3e6736" dependencies = [ "alloy-primitives", "alloy-rpc-types-eth", @@ -10953,9 +10953,9 @@ dependencies = [ [[package]] name = "revm-interpreter" -version = "42.0.0" +version = "43.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd310927437752ce4bac6997b385772823b84843f04c82e151993bec1265a5ed" +checksum = "7ddc442e3c5006ebb65f479e9e8fded7fc10f300c040276e825eee81e00c94cc" dependencies = [ "revm-bytecode", "revm-context-interface", @@ -10966,9 +10966,9 @@ dependencies = [ [[package]] name = "revm-precompile" -version = "42.0.1" +version = "43.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2431091420999795df6a200b9f7933defe3d654710560a895e6bcabbd9954a5" +checksum = "d85f7a8906482f8d18f2a2c793bc4daf254b29465af4285268384505ce6ee308" dependencies = [ "ark-bls12-381", "ark-bn254", @@ -10993,9 +10993,9 @@ dependencies = [ [[package]] name = "revm-primitives" -version = "42.0.0" +version = "43.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ef5564c1bce219fd08beca43d4f39fdcbc19193fa4f604c0c6b58950a13194a" +checksum = "f1f4ba1a5757fe4398262290422c371556fcebf236f762f7ed7ff2c56768a63c" dependencies = [ "alloy-primitives", "once_cell", @@ -11004,9 +11004,9 @@ dependencies = [ [[package]] name = "revm-state" -version = "42.0.0" +version = "43.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c4ae66db3abfcb565854dbfc95513bfa440b3dc390ddf8e50b0363d12a6fa79" +checksum = "7cd02082cf1baca01e8bc17456c45136bdc66632395a98db1c1a2afe4c6ea87f" dependencies = [ "alloy-eip7928", "bitflags 2.13.0", @@ -11019,7 +11019,7 @@ dependencies = [ [[package]] name = "revmc" version = "0.1.0" -source = "git+https://github.com/paradigmxyz/revmc?branch=main#cf68a87f627299a9c49bcc333a8a317c9b312a3d" +source = "git+https://github.com/paradigmxyz/revmc?branch=main#68835cf19661307d293c26dd06bd1e59ccae5144" dependencies = [ "revm-bytecode", "revm-context-interface", @@ -11039,7 +11039,7 @@ dependencies = [ [[package]] name = "revmc-backend" version = "0.1.0" -source = "git+https://github.com/paradigmxyz/revmc?branch=main#cf68a87f627299a9c49bcc333a8a317c9b312a3d" +source = "git+https://github.com/paradigmxyz/revmc?branch=main#68835cf19661307d293c26dd06bd1e59ccae5144" dependencies = [ "eyre", "ruint", @@ -11048,12 +11048,12 @@ dependencies = [ [[package]] name = "revmc-build" version = "0.1.0" -source = "git+https://github.com/paradigmxyz/revmc?branch=main#cf68a87f627299a9c49bcc333a8a317c9b312a3d" +source = "git+https://github.com/paradigmxyz/revmc?branch=main#68835cf19661307d293c26dd06bd1e59ccae5144" [[package]] name = "revmc-builtins" version = "0.1.0" -source = "git+https://github.com/paradigmxyz/revmc?branch=main#cf68a87f627299a9c49bcc333a8a317c9b312a3d" +source = "git+https://github.com/paradigmxyz/revmc?branch=main#68835cf19661307d293c26dd06bd1e59ccae5144" dependencies = [ "paste", "revm-bytecode", @@ -11068,7 +11068,7 @@ dependencies = [ [[package]] name = "revmc-codegen" version = "0.1.0" -source = "git+https://github.com/paradigmxyz/revmc?branch=main#cf68a87f627299a9c49bcc333a8a317c9b312a3d" +source = "git+https://github.com/paradigmxyz/revmc?branch=main#68835cf19661307d293c26dd06bd1e59ccae5144" dependencies = [ "alloy-primitives", "bitflags 2.13.0", @@ -11099,7 +11099,7 @@ dependencies = [ [[package]] name = "revmc-context" version = "0.1.0" -source = "git+https://github.com/paradigmxyz/revmc?branch=main#cf68a87f627299a9c49bcc333a8a317c9b312a3d" +source = "git+https://github.com/paradigmxyz/revmc?branch=main#68835cf19661307d293c26dd06bd1e59ccae5144" dependencies = [ "revm-context", "revm-context-interface", @@ -11114,7 +11114,7 @@ dependencies = [ [[package]] name = "revmc-llvm" version = "0.1.0" -source = "git+https://github.com/paradigmxyz/revmc?branch=main#cf68a87f627299a9c49bcc333a8a317c9b312a3d" +source = "git+https://github.com/paradigmxyz/revmc?branch=main#68835cf19661307d293c26dd06bd1e59ccae5144" dependencies = [ "alloy-primitives", "cc", @@ -11127,7 +11127,7 @@ dependencies = [ [[package]] name = "revmc-runtime" version = "0.1.0" -source = "git+https://github.com/paradigmxyz/revmc?branch=main#cf68a87f627299a9c49bcc333a8a317c9b312a3d" +source = "git+https://github.com/paradigmxyz/revmc?branch=main#68835cf19661307d293c26dd06bd1e59ccae5144" dependencies = [ "alloy-evm", "alloy-primitives", @@ -11376,7 +11376,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.52.0", + "windows-sys 0.60.2", ] [[package]] @@ -11456,7 +11456,7 @@ dependencies = [ "security-framework", "security-framework-sys", "webpki-root-certs 1.0.7", - "windows-sys 0.52.0", + "windows-sys 0.60.2", ] [[package]] @@ -12348,7 +12348,7 @@ dependencies = [ "getrandom 0.4.2", "once_cell", "rustix", - "windows-sys 0.52.0", + "windows-sys 0.60.2", ] [[package]] @@ -13611,7 +13611,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.52.0", + "windows-sys 0.60.2", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 283f9b8d4c8..6c7cc6aaff7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -324,8 +324,8 @@ reth-cli = { path = "crates/cli/cli" } reth-cli-commands = { path = "crates/cli/commands" } reth-cli-runner = { path = "crates/cli/runner" } reth-cli-util = { path = "crates/cli/util" } -reth-codecs = { version = "0.6.0", default-features = false } -reth-codecs-derive = "0.6.0" +reth-codecs = { version = "0.7.0", default-features = false } +reth-codecs-derive = "0.7.0" reth-config = { path = "crates/config", default-features = false } reth-consensus = { path = "crates/consensus/consensus", default-features = false } reth-consensus-common = { path = "crates/consensus/common", default-features = false } @@ -393,7 +393,7 @@ reth-payload-builder-primitives = { path = "crates/payload/builder-primitives" } reth-payload-primitives = { path = "crates/payload/primitives" } reth-payload-validator = { path = "crates/payload/validator" } reth-payload-util = { path = "crates/payload/util" } -reth-primitives-traits = { version = "0.6.0", default-features = false } +reth-primitives-traits = { version = "0.7.0", default-features = false } reth-provider = { path = "crates/storage/provider" } reth-prune = { path = "crates/prune/prune" } reth-prune-types = { path = "crates/prune/types", default-features = false } @@ -408,7 +408,7 @@ reth-rpc-eth-types = { path = "crates/rpc/rpc-eth-types", default-features = fal reth-rpc-layer = { path = "crates/rpc/rpc-layer" } reth-rpc-server-types = { path = "crates/rpc/rpc-server-types" } reth-rpc-convert = { path = "crates/rpc/rpc-convert" } -reth-rpc-traits = { version = "0.6.0", default-features = false } +reth-rpc-traits = { version = "0.7.0", default-features = false } reth-stages = { path = "crates/stages/stages" } reth-stages-api = { path = "crates/stages/api" } reth-stages-types = { path = "crates/stages/types", default-features = false } @@ -428,12 +428,12 @@ reth-trie-common = { path = "crates/trie/common", default-features = false } reth-trie-db = { path = "crates/trie/db" } reth-trie-parallel = { path = "crates/trie/parallel" } reth-trie-sparse = { path = "crates/trie/sparse", default-features = false } -reth-zstd-compressors = { version = "0.6.0", default-features = false } +reth-zstd-compressors = { version = "0.7.0", default-features = false } # revm -revm = { version = "42.0.1", default-features = false } +revm = { version = "43.0.0", default-features = false } revmc = { git = "https://github.com/paradigmxyz/revmc", branch = "main", default-features = false, features = ["llvm-prefer-static"] } -revm-inspectors = "0.42.2" +revm-inspectors = "0.43.0" # eth alloy-dyn-abi = "1.6.1" @@ -443,7 +443,7 @@ alloy-sol-types = { version = "1.6.1", default-features = false } alloy-chains = { version = "0.2.33", default-features = false } alloy-eip2124 = { version = "0.2.0", default-features = false } alloy-eip7928 = { version = "0.4.8", default-features = false, features = ["rlp"] } -alloy-evm = { version = "0.38.0", default-features = false } +alloy-evm = { version = "0.39.0", default-features = false } alloy-rlp = { version = "0.3.16", default-features = false, features = ["core-net"] } alloy-trie = { version = "0.9.4", default-features = false } diff --git a/examples/custom-beacon-withdrawals/src/main.rs b/examples/custom-beacon-withdrawals/src/main.rs index efd6b0e5d97..9ab3e6bbf6a 100644 --- a/examples/custom-beacon-withdrawals/src/main.rs +++ b/examples/custom-beacon-withdrawals/src/main.rs @@ -191,7 +191,7 @@ pub struct CustomBlockExecutor<'a, Evm> { impl BlockExecutor for CustomBlockExecutor<'_, E> where - E: Evm, + E: Evm + Clone, Tx = TxEnv>, { type Transaction = TransactionSigned; type Receipt = Receipt; From 66a7229e66ef868047381a1cab0da024f093b34e Mon Sep 17 00:00:00 2001 From: Brian Picciano Date: Wed, 2 Sep 2026 08:25:29 +0000 Subject: [PATCH 117/294] fix(overlay): avoid materializing historical execution state (#26907) --- .../tree/src/tree/payload_processor/mod.rs | 4 +- .../src/tree/payload_processor/prewarm.rs | 6 +- .../engine/tree/src/tree/payload_validator.rs | 9 +- .../src/providers/database/provider.rs | 51 +++- .../src/providers/state/historical.rs | 73 +++--- .../storage/provider/src/test_utils/mock.rs | 29 ++- crates/storage/provider/src/traits/full.rs | 4 +- crates/storage/storage-api/src/history.rs | 34 +++ crates/storage/storage-api/src/noop.rs | 25 +- crates/storage/storage-overlay/src/builder.rs | 110 +++------ .../storage/storage-overlay/src/provider.rs | 232 ++++++++++++++++-- 11 files changed, 437 insertions(+), 140 deletions(-) diff --git a/crates/engine/tree/src/tree/payload_processor/mod.rs b/crates/engine/tree/src/tree/payload_processor/mod.rs index 88af766be67..c12855aa07e 100644 --- a/crates/engine/tree/src/tree/payload_processor/mod.rs +++ b/crates/engine/tree/src/tree/payload_processor/mod.rs @@ -18,7 +18,7 @@ use reth_evm::{ }; use reth_primitives_traits::{FastInstant as Instant, NodePrimitives}; use reth_provider::{ - BlockExecutionOutput, BlockNumReader, ChangeSetReader, DatabaseProviderFactory, + BlockExecutionOutput, BlockNumReader, ChangeSetReader, DatabaseProviderFactory, HistoryReader, PruneCheckpointReader, StageCheckpointReader, StorageChangeSetReader, StorageSettingsCache, }; use reth_revm::db::BundleState; @@ -183,6 +183,7 @@ where + ChangeSetReader + StorageChangeSetReader + StorageSettingsCache + + HistoryReader + 'static, { let (prewarm_rx, execution_rx) = @@ -354,6 +355,7 @@ where + ChangeSetReader + StorageChangeSetReader + StorageSettingsCache + + HistoryReader + 'static, { // Each mode carries the capability its producers use; the rest is dropped here, so diff --git a/crates/engine/tree/src/tree/payload_processor/prewarm.rs b/crates/engine/tree/src/tree/payload_processor/prewarm.rs index dba966aadd6..21ccee961fb 100644 --- a/crates/engine/tree/src/tree/payload_processor/prewarm.rs +++ b/crates/engine/tree/src/tree/payload_processor/prewarm.rs @@ -28,8 +28,8 @@ use reth_metrics::Metrics; use reth_primitives_traits::{Account, FastInstant as Instant, NodePrimitives}; use reth_provider::{ AccountReader, BlockExecutionOutput, BlockNumReader, ChangeSetReader, DatabaseProviderFactory, - DatabaseProviderROFactory, PruneCheckpointReader, StageCheckpointReader, StateProviderBox, - StorageChangeSetReader, StorageSettingsCache, + DatabaseProviderROFactory, HistoryReader, PruneCheckpointReader, StageCheckpointReader, + StateProviderBox, StorageChangeSetReader, StorageSettingsCache, }; use reth_revm::database::StateProviderDatabase; use reth_storage_overlay::OverlayStateProviderFactory; @@ -100,6 +100,7 @@ where + ChangeSetReader + StorageChangeSetReader + StorageSettingsCache + + HistoryReader + 'static, Evm: ConfigureEvm + 'static, { @@ -585,6 +586,7 @@ where + ChangeSetReader + StorageChangeSetReader + StorageSettingsCache + + HistoryReader + 'static, Evm: ConfigureEvm + 'static, { diff --git a/crates/engine/tree/src/tree/payload_validator.rs b/crates/engine/tree/src/tree/payload_validator.rs index 631a692ac2a..4655ab978a0 100644 --- a/crates/engine/tree/src/tree/payload_validator.rs +++ b/crates/engine/tree/src/tree/payload_validator.rs @@ -153,9 +153,10 @@ use reth_primitives_traits::{ }; use reth_provider::{ BlockExecutionOutput, BlockHashReader, BlockReader, ChangeSetReader, DatabaseProviderFactory, - DatabaseProviderROFactory, HashedPostStateProvider, ProviderError, PruneCheckpointReader, - StageCheckpointReader, StateProvider, StateProviderBox, StateProviderFactory, StateReader, - StateRootProvider, StorageChangeSetReader, StorageSettingsCache, + DatabaseProviderROFactory, HashedPostStateProvider, HistoryReader, ProviderError, + PruneCheckpointReader, StageCheckpointReader, StateProvider, StateProviderBox, + StateProviderFactory, StateReader, StateRootProvider, StorageChangeSetReader, + StorageSettingsCache, }; use reth_revm::db::{states::bundle_state::BundleRetention, BundleAccount, State}; use reth_storage_overlay::{OverlayManager, OverlayStateProviderFactory}; @@ -316,6 +317,7 @@ where + ChangeSetReader + StorageChangeSetReader + StorageSettingsCache + + HistoryReader + 'static, > + BlockReader
+ ChangeSetReader @@ -1800,6 +1802,7 @@ where + ChangeSetReader + StorageChangeSetReader + StorageSettingsCache + + HistoryReader + 'static, > + BlockReader
+ StateProviderFactory diff --git a/crates/storage/provider/src/providers/database/provider.rs b/crates/storage/provider/src/providers/database/provider.rs index 6ced4f2624b..ff5d9b2e513 100644 --- a/crates/storage/provider/src/providers/database/provider.rs +++ b/crates/storage/provider/src/providers/database/provider.rs @@ -62,9 +62,10 @@ use reth_prune_types::{ use reth_stages_types::{FinishCheckpoint, StageCheckpoint, StageId}; use reth_static_file_types::StaticFileSegment; use reth_storage_api::{ - BlockBodyIndicesProvider, BlockBodyReader, MetadataProvider, MetadataWriter, - NodePrimitivesProvider, StateProvider, StateReader, StateWriteConfig, StorageChangeSetReader, - StoragePath, StorageSettingsCache, TryIntoHistoricalStateProvider, WriteStateInput, + BlockBodyIndicesProvider, BlockBodyReader, HistoryInfo, HistoryReader, MetadataProvider, + MetadataWriter, NodePrimitivesProvider, StateProvider, StateReader, StateWriteConfig, + StorageChangeSetReader, StoragePath, StorageSettingsCache, TryIntoHistoricalStateProvider, + WriteStateInput, }; use reth_storage_errors::provider::{ProviderResult, StaticFileWriterError}; use reth_storage_overlay::OverlayManager; @@ -1732,6 +1733,50 @@ impl ChangeSetReader for DatabaseProvider { } } +impl HistoryReader for DatabaseProvider { + fn account_history_info( + &self, + address: Address, + block_number: BlockNumber, + lowest_available_block_number: Option, + ) -> ProviderResult { + let visible_tip = self.best_block_number()?; + self.with_rocksdb_snapshot(|rocksdb_ref| { + let mut reader = EitherReader::new_accounts_history(self, rocksdb_ref)?; + reader + .account_history_info( + address, + block_number, + lowest_available_block_number, + visible_tip, + ) + .map(Into::into) + }) + } + + fn storage_history_info( + &self, + address: Address, + storage_key: B256, + block_number: BlockNumber, + lowest_available_block_number: Option, + ) -> ProviderResult { + let visible_tip = self.best_block_number()?; + self.with_rocksdb_snapshot(|rocksdb_ref| { + let mut reader = EitherReader::new_storages_history(self, rocksdb_ref)?; + reader + .storage_history_info( + address, + storage_key, + block_number, + lowest_available_block_number, + visible_tip, + ) + .map(Into::into) + }) + } +} + impl StateReader for DatabaseProvider { type Receipt = ReceiptTy; diff --git a/crates/storage/provider/src/providers/state/historical.rs b/crates/storage/provider/src/providers/state/historical.rs index 118f6308c63..2e4d3d69b96 100644 --- a/crates/storage/provider/src/providers/state/historical.rs +++ b/crates/storage/provider/src/providers/state/historical.rs @@ -1,6 +1,6 @@ use crate::{ - AccountReader, BlockHashReader, ChangeSetReader, EitherReader, HashedPostStateProvider, - ProviderError, RocksDBProviderFactory, StateProvider, StateRootProvider, + AccountReader, BlockHashReader, ChangeSetReader, HashedPostStateProvider, ProviderError, + StateProvider, StateRootProvider, }; use alloy_eips::merge::EPOCH_SLOTS; use alloy_primitives::{Address, BlockNumber, Bytes, StorageKey, StorageValue, B256}; @@ -13,9 +13,9 @@ use reth_db_api::{ }; use reth_primitives_traits::{Account, Bytecode, NodePrimitives}; use reth_storage_api::{ - BlockNumReader, BytecodeReader, DBProvider, NodePrimitivesProvider, PruneCheckpointReader, - StageCheckpointReader, StateProofProvider, StorageChangeSetReader, StorageRootProvider, - StorageSettingsCache, + BlockNumReader, BytecodeReader, DBProvider, HistoryInfo as ReaderHistoryInfo, HistoryReader, + NodePrimitivesProvider, PruneCheckpointReader, StageCheckpointReader, StateProofProvider, + StorageChangeSetReader, StorageRootProvider, StorageSettingsCache, }; use reth_storage_errors::provider::ProviderResult; use reth_storage_overlay::{OverlayManager, StateTrieOverlay}; @@ -107,6 +107,28 @@ impl HistoryInfo { } } +impl From for HistoryInfo { + fn from(info: ReaderHistoryInfo) -> Self { + match info { + ReaderHistoryInfo::NotYetWritten => Self::NotYetWritten, + ReaderHistoryInfo::InChangeset(block_number) => Self::InChangeset(block_number), + ReaderHistoryInfo::InPlainState => Self::InPlainState, + ReaderHistoryInfo::MaybeInPlainState => Self::MaybeInPlainState, + } + } +} + +impl From for ReaderHistoryInfo { + fn from(info: HistoryInfo) -> Self { + match info { + HistoryInfo::NotYetWritten => Self::NotYetWritten, + HistoryInfo::InChangeset(block_number) => Self::InChangeset(block_number), + HistoryInfo::InPlainState => Self::InPlainState, + HistoryInfo::MaybeInPlainState => Self::MaybeInPlainState, + } + } +} + /// State provider for a given block number which takes a tx reference. /// /// Historical state provider accesses the state at the start of the provided block number. @@ -170,29 +192,25 @@ where Self { provider, overlay_manager, block_number, lowest_available_blocks } } - /// Lookup an account in the `AccountsHistory` table using `EitherReader`. + /// Lookup an account in the `AccountsHistory` table using `HistoryReader`. pub fn account_history_lookup(&self, address: Address) -> ProviderResult where - Provider: StorageSettingsCache + RocksDBProviderFactory + NodePrimitivesProvider, + Provider: HistoryReader, { if !self.lowest_available_blocks.is_account_history_available(self.block_number) { return Err(ProviderError::StateAtBlockPruned(self.block_number)) } - let visible_tip = self.provider.best_block_number()?; - - self.provider.with_rocksdb_snapshot(|rocksdb_ref| { - let mut reader = EitherReader::new_accounts_history(self.provider, rocksdb_ref)?; - reader.account_history_info( + self.provider + .account_history_info( address, self.block_number, self.lowest_available_blocks.account_history_block_number, - visible_tip, ) - }) + .map(Into::into) } - /// Lookup a storage key in the `StoragesHistory` table using `EitherReader`. + /// Lookup a storage key in the `StoragesHistory` table using `HistoryReader`. /// /// `lookup_key` is always a plain (unhashed) storage key. pub fn storage_history_lookup( @@ -201,24 +219,20 @@ where lookup_key: B256, ) -> ProviderResult where - Provider: StorageSettingsCache + RocksDBProviderFactory + NodePrimitivesProvider, + Provider: HistoryReader, { if !self.lowest_available_blocks.is_storage_history_available(self.block_number) { return Err(ProviderError::StateAtBlockPruned(self.block_number)) } - let visible_tip = self.provider.best_block_number()?; - - self.provider.with_rocksdb_snapshot(|rocksdb_ref| { - let mut reader = EitherReader::new_storages_history(self.provider, rocksdb_ref)?; - reader.storage_history_info( + self.provider + .storage_history_info( address, lookup_key, self.block_number, self.lowest_available_blocks.storage_history_block_number, - visible_tip, ) - }) + .map(Into::into) } /// Resolves a storage value by looking up the given key in history, changesets, or @@ -231,7 +245,7 @@ where lookup_key: B256, ) -> ProviderResult> where - Provider: StorageSettingsCache + RocksDBProviderFactory + NodePrimitivesProvider, + Provider: StorageSettingsCache + HistoryReader, { match self.storage_history_lookup(address, lookup_key)? { HistoryInfo::NotYetWritten => Ok(None), @@ -344,7 +358,7 @@ where + ChangeSetReader + StorageChangeSetReader + StorageSettingsCache - + RocksDBProviderFactory + + HistoryReader + NodePrimitivesProvider, N: NodePrimitives, { @@ -697,7 +711,7 @@ where + PruneCheckpointReader + StageCheckpointReader + StorageSettingsCache - + RocksDBProviderFactory + + HistoryReader + NodePrimitivesProvider, N: NodePrimitives, { @@ -798,7 +812,7 @@ impl< } // Delegates all provider impls to [HistoricalStateProviderRef] -reth_storage_api::macros::delegate_provider_impls!(HistoricalStateProvider where [Provider: DBProvider + BlockNumReader + BlockHashReader + ChangeSetReader + StorageChangeSetReader + PruneCheckpointReader + StageCheckpointReader + StorageSettingsCache + RocksDBProviderFactory + NodePrimitivesProvider]); +reth_storage_api::macros::delegate_provider_impls!(HistoricalStateProvider where [Provider: DBProvider + BlockNumReader + BlockHashReader + ChangeSetReader + StorageChangeSetReader + PruneCheckpointReader + StageCheckpointReader + StorageSettingsCache + HistoryReader + NodePrimitivesProvider]); /// Lowest blocks at which different parts of the state are available. /// They may be [Some] if pruning is enabled. @@ -896,7 +910,6 @@ where // table. let is_before_first_write = needs_prev_shard_check(rank, found_block, block_number) && !cursor.prev()?.is_some_and(|(k, _)| key_filter(&k)); - Ok(HistoryInfo::from_lookup( found_block, is_before_first_write, @@ -918,7 +931,7 @@ mod tests { use crate::{ providers::state::historical::{HistoryInfo, LowestAvailableBlocks}, test_utils::create_test_provider_factory, - AccountReader, HistoricalStateProvider, HistoricalStateProviderRef, RocksDBProviderFactory, + AccountReader, HistoricalStateProvider, HistoricalStateProviderRef, HistoryReader, StateProvider, }; use alloy_primitives::{address, b256, Address, B256, U256}; @@ -953,7 +966,7 @@ mod tests { + PruneCheckpointReader + StageCheckpointReader + StorageSettingsCache - + RocksDBProviderFactory + + HistoryReader + NodePrimitivesProvider, >() { assert_state_provider::>(); diff --git a/crates/storage/provider/src/test_utils/mock.rs b/crates/storage/provider/src/test_utils/mock.rs index f299836aba9..b75fefcfbbc 100644 --- a/crates/storage/provider/src/test_utils/mock.rs +++ b/crates/storage/provider/src/test_utils/mock.rs @@ -36,9 +36,9 @@ use reth_prune_types::{PruneCheckpoint, PruneModes, PruneSegment}; use reth_stages_types::{StageCheckpoint, StageId}; use reth_storage_api::{ BlockBodyIndicesProvider, BytecodeReader, DBProvider, DatabaseProviderFactory, DbTxProvider, - HashedPostStateProvider, NodePrimitivesProvider, StageCheckpointReader, StateProofProvider, - StorageChangeSetReader, StorageRootProvider, StorageSettingsCache, - TryIntoHistoricalStateProvider, + HashedPostStateProvider, HistoryInfo, HistoryReader, NodePrimitivesProvider, + StageCheckpointReader, StateProofProvider, StorageChangeSetReader, StorageRootProvider, + StorageSettingsCache, TryIntoHistoricalStateProvider, }; use reth_storage_errors::provider::{ConsistentViewError, ProviderError, ProviderResult}; use reth_trie::{ @@ -512,6 +512,29 @@ impl DatabaseProvi } } +impl HistoryReader + for MockEthProvider +{ + fn account_history_info( + &self, + _address: Address, + _block_number: BlockNumber, + _lowest_available_block_number: Option, + ) -> ProviderResult { + Ok(HistoryInfo::InPlainState) + } + + fn storage_history_info( + &self, + _address: Address, + _storage_key: B256, + _block_number: BlockNumber, + _lowest_available_block_number: Option, + ) -> ProviderResult { + Ok(HistoryInfo::InPlainState) + } +} + impl DbTxProvider for MockEthProvider { diff --git a/crates/storage/provider/src/traits/full.rs b/crates/storage/provider/src/traits/full.rs index 1f4bdcd7760..49b87029a22 100644 --- a/crates/storage/provider/src/traits/full.rs +++ b/crates/storage/provider/src/traits/full.rs @@ -10,7 +10,7 @@ use reth_chain_state::{ }; use reth_node_types::{BlockTy, HeaderTy, NodeTypesWithDB, ReceiptTy, TxTy}; use reth_storage_api::{ - NodePrimitivesProvider, StorageChangeSetReader, StorageSettingsCache, + HistoryReader, NodePrimitivesProvider, StorageChangeSetReader, StorageSettingsCache, TryIntoHistoricalStateProvider, }; use std::fmt::Debug; @@ -25,6 +25,7 @@ pub trait FullProvider: + ChangeSetReader + StorageChangeSetReader + StorageSettingsCache + + HistoryReader + TryIntoHistoricalStateProvider + 'static, > + NodePrimitivesProvider @@ -63,6 +64,7 @@ impl FullProvider for T where + ChangeSetReader + StorageChangeSetReader + StorageSettingsCache + + HistoryReader + TryIntoHistoricalStateProvider + 'static, > + NodePrimitivesProvider diff --git a/crates/storage/storage-api/src/history.rs b/crates/storage/storage-api/src/history.rs index d3a21c183d0..90b123a7145 100644 --- a/crates/storage/storage-api/src/history.rs +++ b/crates/storage/storage-api/src/history.rs @@ -6,6 +6,40 @@ use reth_db_models::AccountBeforeTx; use reth_primitives_traits::StorageEntry; use reth_storage_errors::provider::ProviderResult; +/// Location of a historical account or storage value. +#[derive(Debug, Eq, PartialEq)] +pub enum HistoryInfo { + /// The key had not been written at the requested block. + NotYetWritten, + /// The value is in this block's changeset. + InChangeset(BlockNumber), + /// The value is in plain state. + InPlainState, + /// Pruning requires a plain-state fallback. + MaybeInPlainState, +} + +/// Reads account and storage history indices. +#[auto_impl(&, Arc, Box)] +pub trait HistoryReader: Send { + /// Looks up an account's historical storage location. + fn account_history_info( + &self, + address: Address, + block_number: BlockNumber, + lowest_available_block_number: Option, + ) -> ProviderResult; + + /// Looks up a storage slot's historical storage location. + fn storage_history_info( + &self, + address: Address, + storage_key: B256, + block_number: BlockNumber, + lowest_available_block_number: Option, + ) -> ProviderResult; +} + /// History Writer #[auto_impl(&, Arc, Box)] pub trait HistoryWriter: Send { diff --git a/crates/storage/storage-api/src/noop.rs b/crates/storage/storage-api/src/noop.rs index e1612bf64f4..49178ae983b 100644 --- a/crates/storage/storage-api/src/noop.rs +++ b/crates/storage/storage-api/src/noop.rs @@ -14,7 +14,8 @@ use crate::{ #[cfg(feature = "db-api")] use crate::{ - DBProvider, DatabaseProviderFactory, DbTxProvider, StorageChangeSetReader, StorageSettingsCache, + DBProvider, DatabaseProviderFactory, DbTxProvider, HistoryInfo, HistoryReader, + StorageChangeSetReader, StorageSettingsCache, }; use alloc::{boxed::Box, string::String, sync::Arc, vec::Vec}; use alloy_consensus::transaction::TransactionMeta; @@ -426,6 +427,28 @@ impl ChangeSetReader for NoopProvider { } } +#[cfg(feature = "db-api")] +impl HistoryReader for NoopProvider { + fn account_history_info( + &self, + _address: Address, + _block_number: BlockNumber, + _lowest_available_block_number: Option, + ) -> ProviderResult { + Ok(HistoryInfo::NotYetWritten) + } + + fn storage_history_info( + &self, + _address: Address, + _storage_key: B256, + _block_number: BlockNumber, + _lowest_available_block_number: Option, + ) -> ProviderResult { + Ok(HistoryInfo::NotYetWritten) + } +} + #[cfg(feature = "db-api")] impl StorageChangeSetReader for NoopProvider { fn storage_changeset( diff --git a/crates/storage/storage-overlay/src/builder.rs b/crates/storage/storage-overlay/src/builder.rs index 403b6800dff..c157ac12d2a 100644 --- a/crates/storage/storage-overlay/src/builder.rs +++ b/crates/storage/storage-overlay/src/builder.rs @@ -20,7 +20,6 @@ use reth_trie::{updates::TrieUpdatesSorted, HashedPostStateSorted}; use reth_trie_db::DatabaseHashedPostState; use revm::{bytecode::Bytecode, database::BundleState, state::AccountInfo}; use std::{ - collections::HashSet, ops::RangeInclusive, sync::Arc, time::{Duration, Instant}, @@ -175,10 +174,7 @@ impl ExecutionOverlay { } } - /// Extends this overlay with another, later overlay. - /// - /// Entries in `other` take precedence when both overlays contain the same account, storage - /// slot, or bytecode hash. + #[cfg(test)] fn extend_overlay(&mut self, other: &Self) { self.block_hashes.extend_from_slice(&other.block_hashes); self.accounts.extend( @@ -514,12 +510,12 @@ impl OverlayBuilder { Ok(StateTrieOverlay::new(trie_updates, hashed_post_state)) } - /// Builds the effective execution overlay for the given provider. + /// Returns the in-memory execution overlay and the block for historical fallback reads. #[instrument(level = "debug", target = "storage::overlay", skip_all)] - pub fn build_execution_overlay( + pub fn execution_overlay( &self, provider: &Provider, - ) -> ProviderResult> + ) -> ProviderResult<(Arc, Option)> where Provider: StageCheckpointReader + PruneCheckpointReader @@ -529,22 +525,22 @@ impl OverlayBuilder { + BlockNumReader, { let (state_trie_tip_block, finish_tip_block) = database_state_frontiers(provider)?; - self.build_execution_overlay_at_frontiers(provider, state_trie_tip_block, finish_tip_block) + self.execution_overlay_at_frontiers(provider, state_trie_tip_block, finish_tip_block) } - /// Builds the effective execution overlay using frontiers already read from the provider. + /// Returns the in-memory execution overlay using frontiers already read from the provider. #[instrument( level = "debug", target = "storage::overlay", skip_all, fields(?state_trie_tip_block, ?finish_tip_block, parent_hash = ?self.parent_hash) )] - pub fn build_execution_overlay_at_frontiers( + pub fn execution_overlay_at_frontiers( &self, provider: &Provider, state_trie_tip_block: BlockNumHash, finish_tip_block: BlockNumHash, - ) -> ProviderResult> + ) -> ProviderResult<(Arc, Option)> where Provider: ChangeSetReader + StorageChangeSetReader @@ -554,23 +550,14 @@ impl OverlayBuilder { { let anchor_for_parent = self.anchor_at_parent_with_frontiers(provider, state_trie_tip_block, finish_tip_block)?; - let anchor = anchor_for_parent.anchor(); - let mut overlay = match self.revert_blocks(&anchor_for_parent)? { - Some(revert_blocks) => { - debug!( - target: "storage::overlay", - ?revert_blocks, - ?anchor, - "Collecting execution reverts for overlay state provider" - ); - execution_reverts(provider, revert_blocks)? - } - None => return self.resolve_execution_overlay(anchor.hash), + let fallback_block_number = match &anchor_for_parent { + AnchorForParent::RevertsRequired { anchor, .. } => Some(anchor.number + 1), + AnchorForParent::NoReverts { .. } => None, }; - - let managed_overlay = self.resolve_execution_overlay(anchor.hash)?; - overlay.extend_overlay(&managed_overlay); - Ok(Arc::new(overlay)) + Ok(( + self.resolve_execution_overlay(anchor_for_parent.anchor().hash)?, + fallback_block_number, + )) } /// Resolves the effective overlay (trie updates, hashed state). @@ -766,39 +753,6 @@ impl AnchorForParent { } } -/// Collects the execution data needed to revert the database to `range`'s first block. -fn execution_reverts( - provider: &Provider, - range: RangeInclusive, -) -> ProviderResult -where - Provider: ChangeSetReader + StorageChangeSetReader, -{ - let mut overlay = ExecutionOverlay::default(); - let mut seen_accounts = HashSet::new(); - for (_, account) in provider.account_changesets_range(range.clone())? { - if seen_accounts.insert(account.address) { - overlay.accounts.insert(account.address, account.info.map(Into::into)); - } - } - - let mut seen_storage = HashSet::new(); - for (block_address, storage) in provider.storage_changesets_range(range)? { - let address = block_address.address(); - if seen_storage.insert((address, storage.key)) { - overlay - .storage - .entry(address) - .or_default() - .insert(U256::from_be_bytes(storage.key.0), storage.value); - } - } - - // Bytecode rows are append-only. Restoring an account's previous code hash therefore makes - // its bytecode directly available from the database without an additional revert overlay. - Ok(overlay) -} - /// Returns the anchor block to use for the target parent and a chain of in-memory blocks. /// /// # Arguments @@ -1185,12 +1139,10 @@ mod tests { let error = builder.build_state_trie_overlay(&provider).unwrap_err(); assert!(error.to_string().contains("reverts are disabled")); - let error = builder.build_execution_overlay(&provider).unwrap_err(); - assert!(error.to_string().contains("reverts are disabled")); } #[test] - fn execution_overlay_reverts_to_anchor_state() { + fn execution_overlay_marks_historical_fallback() { let (factory, blocks) = setup_frontiers(1, 3); let provider_rw = factory.provider_rw().unwrap(); let address = Address::with_last_byte(1); @@ -1228,13 +1180,14 @@ mod tests { provider_rw.commit().unwrap(); let provider = factory.provider().unwrap(); - let overlay = OverlayManager::::default() + let (overlay, fallback_block_number) = OverlayManager::::default() .overlay_builder(blocks[1].recovered_block().hash()) - .build_execution_overlay(&provider) + .execution_overlay(&provider) .unwrap(); - assert_eq!(overlay.accounts[&address].as_ref().unwrap().balance, U256::from(10)); - assert_eq!(overlay.storage[&address][&slot], U256::from(10)); + assert_eq!(fallback_block_number, Some(2)); + assert!(overlay.accounts.is_empty()); + assert!(overlay.storage.is_empty()); assert!(overlay.code_hashes.is_empty()); } @@ -1245,7 +1198,7 @@ mod tests { let error = OverlayManager::::default() .overlay_builder(blocks[1].recovered_block().hash()) .with_immediate_state_trie_overlay(Default::default(), Default::default()) - .build_execution_overlay(&provider) + .execution_overlay(&provider) .unwrap_err(); assert!(error.to_string().contains("immediate state trie overlay has no execution state")); @@ -1260,11 +1213,13 @@ mod tests { } let provider = factory.provider().unwrap(); - let overlay = manager + let (overlay, fallback_block_number) = manager .overlay_builder(blocks[3].recovered_block().hash()) - .build_execution_overlay(&provider) + .execution_overlay(&provider) .unwrap(); + assert_eq!(fallback_block_number, None); + for id in [3, 4] { let address = Address::with_last_byte(id); let slot = U256::from(id); @@ -1283,7 +1238,7 @@ mod tests { } #[test] - fn execution_overlay_merges_reverts_with_a_managed_fork() { + fn execution_overlay_marks_historical_fallback_for_managed_fork() { let (factory, blocks) = setup_frontiers(1, 3); let address = Address::with_last_byte(1); let slot = U256::from(1); @@ -1331,11 +1286,13 @@ mod tests { manager.insert_block(side_block_three.clone()); let provider = factory.provider().unwrap(); - let overlay = manager + let (overlay, fallback_block_number) = manager .overlay_builder(side_block_three.recovered_block().hash()) - .build_execution_overlay(&provider) + .execution_overlay(&provider) .unwrap(); + assert_eq!(fallback_block_number, Some(2)); + assert_eq!(overlay.accounts[&address].as_ref().unwrap().balance, U256::from(1)); assert_eq!(overlay.accounts[&address].as_ref().unwrap().account_id, None); assert_eq!(overlay.storage[&address][&slot], U256::from(1)); @@ -1357,11 +1314,12 @@ mod tests { } let provider = factory.provider().unwrap(); - let overlay = manager + let (overlay, fallback_block_number) = manager .overlay_builder(blocks[3].recovered_block().hash()) - .build_execution_overlay(&provider) + .execution_overlay(&provider) .unwrap(); + assert_eq!(fallback_block_number, None); assert_eq!(overlay.accounts.len(), 2); assert!(overlay.accounts.values().flatten().all(|account| account.account_id.is_none())); } diff --git a/crates/storage/storage-overlay/src/provider.rs b/crates/storage/storage-overlay/src/provider.rs index b1157a7c240..104f6cbc620 100644 --- a/crates/storage/storage-overlay/src/provider.rs +++ b/crates/storage/storage-overlay/src/provider.rs @@ -9,11 +9,13 @@ use reth_primitives_traits::{ dashmap::{self, DashMap}, Account, NodePrimitives, }; +use reth_prune_types::PruneSegment; use reth_storage_api::{ AccountReader, BlockHashReader, BlockNumReader, BytecodeReader, ChangeSetReader, DBProvider, DatabaseProviderFactory, DatabaseProviderROFactory, DbTxProvider, HashedPostStateProvider, - PruneCheckpointReader, StageCheckpointReader, StateProofProvider, StateProvider, - StateRootProvider, StorageChangeSetReader, StorageRootProvider, StorageSettingsCache, + HistoryInfo, HistoryReader, PruneCheckpointReader, StageCheckpointReader, StateProofProvider, + StateProvider, StateRootProvider, StorageChangeSetReader, StorageRootProvider, + StorageSettingsCache, }; use reth_trie::{ hashed_cursor::{ @@ -133,7 +135,7 @@ pub struct OverlayStateProvider { execution_overlay_cache: ExecutionOverlayCache, metrics: OverlayStateProviderFactoryMetrics, state_trie_overlay: OnceCell, - execution_overlay: OnceCell>, + execution_overlay: OnceCell, is_v2: bool, } @@ -171,7 +173,10 @@ impl OverlayStateProvider, execution_overlay_cache: Default::default(), metrics: Default::default(), state_trie_overlay: OnceCell::new(), - execution_overlay: OnceCell::from(execution_overlay), + execution_overlay: OnceCell::from(CachedExecutionOverlay { + overlay: execution_overlay, + historical_fallback: None, + }), is_v2, } } @@ -274,7 +279,9 @@ where Ok(TrieInputSorted::new(nodes, state, prefix_sets)) } - fn execution_overlay(&self) -> ProviderResult<&Arc> + fn execution_overlay( + &self, + ) -> ProviderResult<(&Arc, Option<&HistoricalFallback>)> where Provider::Target: StageCheckpointReader + PruneCheckpointReader @@ -284,7 +291,7 @@ where + BlockNumReader, { if let Some(overlay) = self.execution_overlay.get() { - return Ok(overlay) + return Ok((&overlay.overlay, overlay.historical_fallback.as_ref())) } let (state_trie_tip_block, finish_tip_block) = database_state_frontiers(self.provider())?; @@ -295,21 +302,52 @@ where dashmap::Entry::Occupied(entry) => entry.get().clone(), dashmap::Entry::Vacant(entry) => { self.metrics.execution_overlay_cache_misses.increment(1); - let overlay = self + let (overlay, fallback_block_number) = self .overlay_builder .as_ref() .expect("execution overlay must be initialized or lazily resolvable") - .build_execution_overlay_at_frontiers( + .execution_overlay_at_frontiers( self.provider(), state_trie_tip_block, finish_tip_block, )?; + let historical_fallback = fallback_block_number + .map(|block_number| { + let account_history_block_number = self + .provider() + .get_prune_checkpoint(PruneSegment::AccountHistory)? + .and_then(|checkpoint| checkpoint.block_number) + .map(|block_number| block_number + 1); + if account_history_block_number.is_some_and(|lowest| block_number < lowest) + { + return Err(ProviderError::StateAtBlockPruned(block_number)) + } + + let storage_history_block_number = self + .provider() + .get_prune_checkpoint(PruneSegment::StorageHistory)? + .and_then(|checkpoint| checkpoint.block_number) + .map(|block_number| block_number + 1); + if storage_history_block_number.is_some_and(|lowest| block_number < lowest) + { + return Err(ProviderError::StateAtBlockPruned(block_number)) + } + + Ok(HistoricalFallback { + block_number, + account_history_block_number, + storage_history_block_number, + }) + }) + .transpose()?; + let overlay = CachedExecutionOverlay { overlay, historical_fallback }; entry.insert(overlay.clone()); overlay } }; let _ = self.execution_overlay.set(overlay); - Ok(self.execution_overlay.get().expect("execution overlay was just initialized")) + let overlay = self.execution_overlay.get().expect("execution overlay was just initialized"); + Ok((&overlay.overlay, overlay.historical_fallback.as_ref())) } } @@ -332,6 +370,7 @@ impl AccountReader for OverlayStateProvider ProviderResult> { - let overlay = self.execution_overlay()?; + let (overlay, historical_fallback) = self.execution_overlay()?; if let Some(account) = overlay.accounts().get(address) { return Ok(account.as_ref().map(Account::from)) } + if let Some(historical_fallback) = historical_fallback { + return match self.provider().account_history_info( + *address, + historical_fallback.block_number, + historical_fallback.account_history_block_number, + )? { + HistoryInfo::NotYetWritten => Ok(None), + HistoryInfo::InChangeset(changeset_block_number) => self + .provider() + .get_account_before_block(changeset_block_number, *address)? + .ok_or(ProviderError::AccountChangesetNotFound { + block_number: changeset_block_number, + address: *address, + }) + .map(|account_before| account_before.info), + HistoryInfo::InPlainState | HistoryInfo::MaybeInPlainState => { + self.basic_account_from_db(address) + } + } + } + self.basic_account_from_db(address) + } +} + +impl OverlayStateProvider +where + Provider: Deref, + Provider::Target: DBProvider + StorageSettingsCache, +{ + fn basic_account_from_db(&self, address: &Address) -> ProviderResult> { if self.provider().cached_storage_settings().use_hashed_state() { let hashed_address = alloy_primitives::keccak256(address); self.provider() @@ -372,7 +441,7 @@ where + BlockNumReader, { fn block_hash(&self, number: BlockNumber) -> ProviderResult> { - let overlay = self.execution_overlay()?; + let (overlay, _) = self.execution_overlay()?; if let Some(block) = overlay.block_hashes().iter().find(|block| block.number == number) { return Ok(Some(block.hash)) } @@ -384,7 +453,7 @@ where start: BlockNumber, end: BlockNumber, ) -> ProviderResult> { - let overlay = self.execution_overlay()?; + let (overlay, _) = self.execution_overlay()?; let mut block_hashes = overlay.block_hashes().iter().filter(|block| (start..end).contains(&block.number)); let Some(first_block) = block_hashes.next() else { @@ -412,7 +481,7 @@ where &self, code_hash: &B256, ) -> ProviderResult> { - let overlay = self.execution_overlay()?; + let (overlay, _) = self.execution_overlay()?; if let Some(bytecode) = overlay.code_hashes().get(code_hash) { return Ok(Some(reth_primitives_traits::Bytecode(bytecode.clone()))); } @@ -717,6 +786,7 @@ impl StateProvider for OverlayStateProvider ProviderResult> { - let overlay = self.execution_overlay()?; + let (overlay, historical_fallback) = self.execution_overlay()?; if let Some(value) = overlay.storage_value(address, U256::from_be_bytes(storage_key.0)) { return Ok(Some(value)); } + if let Some(historical_fallback) = historical_fallback { + return match self.provider().storage_history_info( + address, + storage_key, + historical_fallback.block_number, + historical_fallback.storage_history_block_number, + )? { + HistoryInfo::NotYetWritten => Ok(None), + HistoryInfo::InChangeset(changeset_block_number) => self + .provider() + .get_storage_before_block(changeset_block_number, address, storage_key)? + .ok_or_else(|| ProviderError::StorageChangesetNotFound { + block_number: changeset_block_number, + address, + storage_key: Box::new(storage_key), + }) + .map(|entry| Some(entry.value)), + HistoryInfo::InPlainState | HistoryInfo::MaybeInPlainState => { + self.storage_from_db(address, storage_key, true) + } + } + } + self.storage_from_db(address, storage_key, false) + } +} + +impl OverlayStateProvider +where + Provider: Deref, + Provider::Target: DBProvider + StorageSettingsCache, +{ + fn storage_from_db( + &self, + address: Address, + storage_key: alloy_primitives::StorageKey, + zero_if_missing: bool, + ) -> ProviderResult> { if self.provider().cached_storage_settings().use_hashed_state() { let hashed_address = alloy_primitives::keccak256(address); let hashed_slot = alloy_primitives::keccak256(storage_key); let mut cursor = self.provider().tx().cursor_dup_read::()?; - Ok(cursor + let value = cursor .seek_by_key_subkey(hashed_address, hashed_slot)? .filter(|entry| entry.key == hashed_slot) - .map(|entry| entry.value)) + .map(|entry| entry.value); + Ok(value.or_else(|| zero_if_missing.then_some(U256::ZERO))) } else { let mut cursor = self.provider().tx().cursor_dup_read::()?; if let Some(entry) = cursor.seek_by_key_subkey(address, storage_key)? && @@ -749,7 +857,7 @@ where { return Ok(Some(entry.value)) } - Ok(None) + Ok(zero_if_missing.then_some(U256::ZERO)) } } } @@ -873,7 +981,20 @@ pub(crate) struct OverlayStateProviderFactoryMetrics { } type StateTrieOverlayCache = Arc>; -type ExecutionOverlayCache = Arc>>; +type ExecutionOverlayCache = Arc>; + +#[derive(Clone, Debug)] +struct CachedExecutionOverlay { + overlay: Arc, + historical_fallback: Option, +} + +#[derive(Clone, Copy, Debug)] +struct HistoricalFallback { + block_number: BlockNumber, + account_history_block_number: Option, + storage_history_block_number: Option, +} type DbStateRoot<'a, TX, A> = StateRoot, DatabaseHashedCursorFactory<&'a TX>>; @@ -913,6 +1034,14 @@ mod tests { use alloy_eips::BlockNumHash; use alloy_primitives::{Address, U256}; use reth_chain_state::{test_utils::TestBlockBuilder, ExecutedBlock}; + use reth_db_api::{ + models::{ + storage_sharded_key::StorageShardedKey, AccountBeforeTx, BlockNumberAddress, ShardedKey, + }, + tables, + transaction::DbTxMut, + BlockNumberList, + }; use reth_primitives_traits::Account; use reth_provider::{ test_utils::{create_test_provider_factory, MockNodeTypesWithDB}, @@ -1049,13 +1178,13 @@ mod tests { assert!(state_provider_factory.execution_overlay_cache.is_empty()); provider.basic_account(&Address::ZERO).unwrap(); - let execution_overlay = Arc::clone(provider.execution_overlay.get().unwrap()); + let execution_overlay = Arc::clone(&provider.execution_overlay.get().unwrap().overlay); assert!(provider.state_trie_overlay.get().is_none()); assert!(provider.execution_overlay.get().is_some()); assert!(state_provider_factory.state_trie_overlay_cache.is_empty()); assert_eq!(state_provider_factory.execution_overlay_cache.len(), 1); let cached_overlay = state_provider_factory.execution_overlay_cache.iter().next().unwrap(); - assert!(Arc::ptr_eq(&execution_overlay, cached_overlay.value())); + assert!(Arc::ptr_eq(&execution_overlay, &cached_overlay.value().overlay)); provider.account_trie_cursor().unwrap(); assert_eq!(state_provider_factory.state_trie_overlay_cache.len(), 1); @@ -1119,4 +1248,67 @@ mod tests { Some(reth_primitives_traits::Bytecode(bytecode)) ); } + + #[test] + fn historical_execution_reads_use_history_indexes() { + let (factory, blocks) = setup_frontiers(1, 3); + let address = Address::with_last_byte(1); + let storage_key = B256::with_last_byte(2); + let account = Account { balance: U256::from(10), ..Default::default() }; + let storage = U256::from(10); + let provider_rw = factory.provider_rw().unwrap(); + + provider_rw + .tx_ref() + .put::( + ShardedKey { key: address, highest_block_number: u64::MAX }, + BlockNumberList::new([2]).unwrap(), + ) + .unwrap(); + provider_rw + .tx_ref() + .put::(2, AccountBeforeTx { address, info: Some(account) }) + .unwrap(); + provider_rw + .tx_ref() + .put::( + address, + Account { balance: U256::from(20), ..Default::default() }, + ) + .unwrap(); + provider_rw + .tx_ref() + .put::( + StorageShardedKey { + address, + sharded_key: ShardedKey { key: storage_key, highest_block_number: u64::MAX }, + }, + BlockNumberList::new([2]).unwrap(), + ) + .unwrap(); + provider_rw + .tx_ref() + .put::( + BlockNumberAddress((2, address)), + reth_primitives_traits::StorageEntry { key: storage_key, value: storage }, + ) + .unwrap(); + provider_rw + .tx_ref() + .put::( + address, + reth_primitives_traits::StorageEntry { key: storage_key, value: U256::from(20) }, + ) + .unwrap(); + provider_rw.commit().unwrap(); + + let state_provider_factory = OverlayStateProviderFactory::<_, EthPrimitives>::new( + factory, + OverlayManager::default().overlay_builder(blocks[1].recovered_block().hash()), + ); + let provider = state_provider_factory.database_provider_ro().unwrap(); + + assert_eq!(provider.basic_account(&address).unwrap(), Some(account)); + assert_eq!(provider.storage(address, storage_key).unwrap(), Some(storage)); + } } From 7786a607522cde8e6cc1e8abefb8a275f4398f67 Mon Sep 17 00:00:00 2001 From: Tim Date: Wed, 2 Sep 2026 09:13:23 +0000 Subject: [PATCH 118/294] chore(logging): reduce noisy overlay and static file logs (#26916) --- .../provider/src/providers/static_file/manager.rs | 8 ++++---- crates/storage/storage-overlay/src/builder.rs | 9 --------- 2 files changed, 4 insertions(+), 13 deletions(-) diff --git a/crates/storage/provider/src/providers/static_file/manager.rs b/crates/storage/provider/src/providers/static_file/manager.rs index 0213cd96fb0..937ec9f8b1c 100644 --- a/crates/storage/provider/src/providers/static_file/manager.rs +++ b/crates/storage/provider/src/providers/static_file/manager.rs @@ -1046,7 +1046,7 @@ impl StaticFileProvider { segment: StaticFileSegment, segment_max_block: Option, ) -> ProviderResult<()> { - debug!( + trace!( target: "providers::static_file", ?segment, ?segment_max_block, @@ -1159,11 +1159,11 @@ impl StaticFileProvider { } // Update the cached provider. - debug!(target: "providers::static_file", ?segment, "Inserting updated jar into cache"); + trace!(target: "providers::static_file", ?segment, "Inserting updated jar into cache"); self.map.insert((fixed_range.end(), segment), LoadedJar::new(jar)?); // Delete any cached provider that no longer has an associated jar. - debug!(target: "providers::static_file", ?segment, "Cleaning up jar map"); + trace!(target: "providers::static_file", ?segment, "Cleaning up jar map"); self.map.retain(|(end, seg), _| !(*seg == segment && *end > fixed_range.end())); } None => { @@ -1172,7 +1172,7 @@ impl StaticFileProvider { } }; - debug!(target: "providers::static_file", ?segment, "Updated provider index"); + trace!(target: "providers::static_file", ?segment, "Updated provider index"); Ok(()) } diff --git a/crates/storage/storage-overlay/src/builder.rs b/crates/storage/storage-overlay/src/builder.rs index c157ac12d2a..7a834cc56ea 100644 --- a/crates/storage/storage-overlay/src/builder.rs +++ b/crates/storage/storage-overlay/src/builder.rs @@ -464,15 +464,6 @@ impl OverlayBuilder { state_trie_tip_block.hash, finish_tip_block.hash, ) { - debug!( - target: "storage::overlay", - parent_hash = %self.parent_hash, - state_trie_tip_hash = %state_trie_tip_block.hash, - finish_tip_hash = %finish_tip_block.hash, - sparse_trie_anchor_hash = ?self.reused_sparse_trie_anchor_hash, - "Skipping overlay construction because reused sparse trie covers durable frontiers to parent" - ); - self.metrics.sparse_trie_overlay_skips.increment(1); return Ok(StateTrieOverlay::empty()) From eb03c3f71c2e11354f745de5d1063a043ecf4c48 Mon Sep 17 00:00:00 2001 From: Derek Cofausper <256792747+decofe@users.noreply.github.com> Date: Wed, 2 Sep 2026 09:56:29 +0000 Subject: [PATCH 119/294] feat(consensus): expose receipt validation helpers (#26919) Co-authored-by: Alexey Shekhirin <5773434+shekhirin@users.noreply.github.com> --- crates/ethereum/consensus/src/lib.rs | 4 +++- crates/ethereum/consensus/src/validation.rs | 4 ++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/crates/ethereum/consensus/src/lib.rs b/crates/ethereum/consensus/src/lib.rs index 1973babb2fc..66e8118a651 100644 --- a/crates/ethereum/consensus/src/lib.rs +++ b/crates/ethereum/consensus/src/lib.rs @@ -33,7 +33,9 @@ use reth_primitives_traits::{ }; mod validation; -pub use validation::validate_block_post_execution; +pub use validation::{ + compare_receipts_root_and_logs_bloom, validate_block_post_execution, verify_receipts, +}; /// Ethereum beacon consensus /// diff --git a/crates/ethereum/consensus/src/validation.rs b/crates/ethereum/consensus/src/validation.rs index d0c11ff1b94..a144e50289f 100644 --- a/crates/ethereum/consensus/src/validation.rs +++ b/crates/ethereum/consensus/src/validation.rs @@ -131,7 +131,7 @@ where /// Calculate the receipts root, and compare it against the expected receipts root and logs /// bloom. -fn verify_receipts( +pub fn verify_receipts( expected_receipts_root: B256, expected_logs_bloom: Bloom, receipts: &[R], @@ -153,7 +153,7 @@ fn verify_receipts( /// Compare the calculated receipts root with the expected receipts root, also compare /// the calculated logs bloom with the expected logs bloom. -fn compare_receipts_root_and_logs_bloom( +pub fn compare_receipts_root_and_logs_bloom( calculated_receipts_root: B256, calculated_logs_bloom: Bloom, expected_receipts_root: B256, From caafc1d903246dfd027ce35851619e3bae107d8c Mon Sep 17 00:00:00 2001 From: Brian Picciano Date: Wed, 2 Sep 2026 10:07:57 +0000 Subject: [PATCH 120/294] refactor(trie): remove sparse trie recorder (#26918) --- Cargo.lock | 3 - crates/engine/tree/Cargo.toml | 3 - .../tree/src/tree/state_root_strategy/mod.rs | 56 +---- .../tree/state_root_strategy/sparse_trie.rs | 5 - crates/trie/parallel/Cargo.toml | 4 - crates/trie/parallel/src/state_root_task.rs | 6 - crates/trie/sparse/Cargo.toml | 5 - crates/trie/sparse/src/arena/mod.rs | 215 ------------------ crates/trie/sparse/src/debug_recorder.rs | 207 ----------------- crates/trie/sparse/src/lib.rs | 5 - crates/trie/sparse/src/state.rs | 22 -- crates/trie/sparse/src/traits.rs | 12 - 12 files changed, 3 insertions(+), 540 deletions(-) delete mode 100644 crates/trie/sparse/src/debug_recorder.rs diff --git a/Cargo.lock b/Cargo.lock index cbab24bf9d7..a70d99f5623 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -10752,7 +10752,6 @@ dependencies = [ "reth-storage-overlay", "reth-tasks", "reth-trie", - "reth-trie-sparse", "revm", "thiserror 2.0.18", "tokio", @@ -10782,8 +10781,6 @@ dependencies = [ "reth-tracing", "reth-trie", "reth-trie-common", - "serde", - "serde_json", "slotmap", "smallvec", "strum 0.27.2", diff --git a/crates/engine/tree/Cargo.toml b/crates/engine/tree/Cargo.toml index f3e605bb2e3..c621ebaf6b8 100644 --- a/crates/engine/tree/Cargo.toml +++ b/crates/engine/tree/Cargo.toml @@ -72,7 +72,6 @@ reth-prune-types = { workspace = true, optional = true } reth-stages = { workspace = true, optional = true } reth-static-file = { workspace = true, optional = true } reth-tracing = { workspace = true, optional = true } -serde_json = { workspace = true, optional = true } [dev-dependencies] # reth @@ -132,10 +131,8 @@ test-utils = [ "reth-evm-ethereum/test-utils", ] trie-debug = [ - "reth-trie-sparse/trie-debug", "reth-trie-parallel/trie-debug", "reth-engine-primitives/trie-debug", - "dep:serde_json", ] [[test]] diff --git a/crates/engine/tree/src/tree/state_root_strategy/mod.rs b/crates/engine/tree/src/tree/state_root_strategy/mod.rs index c6519c2795d..cb9be5c5516 100644 --- a/crates/engine/tree/src/tree/state_root_strategy/mod.rs +++ b/crates/engine/tree/src/tree/state_root_strategy/mod.rs @@ -85,8 +85,6 @@ pub use reth_trie_parallel::{ StateRootSink, StateRootTaskCancelGuard, StateRootUpdateHook, StateRootUpdateStream, }, }; -#[cfg(feature = "trie-debug")] -use reth_trie_sparse::debug_recorder::TrieDebugRecorder; use reth_trie_sparse::{ ArenaParallelSparseTrie, RevealableSparseTrie, SparseStateTrie, TrieNodeEpoch, }; @@ -1094,29 +1092,15 @@ where output: &BlockExecutionOutput, outcome: StateRootComputeOutcome, ) -> StateRootJobOutcome { - let StateRootComputeOutcome { - state_root, - trie_updates, - hashed_state: _hashed_state, - #[cfg(feature = "trie-debug")] - debug_recorders, - } = outcome; + let StateRootComputeOutcome { state_root, trie_updates, hashed_state: _hashed_state } = + outcome; if self.compare_trie_updates { - let _has_diff = compare_trie_updates_with_serial( + compare_trie_updates_with_serial( self.state_provider_factory.clone(), output, trie_updates.as_ref().clone(), ); - #[cfg(feature = "trie-debug")] - if _has_diff { - write_trie_debug_recorders(_block.header().number(), &debug_recorders); - } - } - - #[cfg(feature = "trie-debug")] - if state_root != _block.header().state_root() { - write_trie_debug_recorders(_block.header().number(), &debug_recorders); } StateRootJobOutcome::new(state_root, trie_updates) @@ -1300,40 +1284,6 @@ where false } -/// Writes trie debug recorders to a JSON file for the given block number. -/// -/// The file is written to the current working directory as `trie_debug_block_{block_number}.json`. -#[cfg(feature = "trie-debug")] -fn write_trie_debug_recorders(block_number: u64, recorders: &[(Option, TrieDebugRecorder)]) { - let path = format!("trie_debug_block_{block_number}.json"); - match serde_json::to_string_pretty(recorders) { - Ok(json) => match std::fs::write(&path, json) { - Ok(()) => { - warn!( - target: "engine::tree::state_root_strategy", - %path, - "Wrote trie debug recorders to file" - ); - } - Err(err) => { - warn!( - target: "engine::tree::state_root_strategy", - %err, - %path, - "Failed to write trie debug recorders" - ); - } - }, - Err(err) => { - warn!( - target: "engine::tree::state_root_strategy", - %err, - "Failed to serialize trie debug recorders" - ); - } - } -} - #[cfg(test)] mod tests { use super::*; diff --git a/crates/engine/tree/src/tree/state_root_strategy/sparse_trie.rs b/crates/engine/tree/src/tree/state_root_strategy/sparse_trie.rs index a8622f7c687..96528edb3f2 100644 --- a/crates/engine/tree/src/tree/state_root_strategy/sparse_trie.rs +++ b/crates/engine/tree/src/tree/state_root_strategy/sparse_trie.rs @@ -359,9 +359,6 @@ where } }; - #[cfg(feature = "trie-debug")] - let debug_recorders = self.trie.take_debug_recorders(); - let end = Instant::now(); self.metrics.sparse_trie_final_update_duration_histogram.record(end.duration_since(start)); self.metrics.sparse_trie_total_duration_histogram.record(end.duration_since(now)); @@ -380,8 +377,6 @@ where trie_updates: Arc::new(trie_updates), hashed_state: finalized_hashed_state .expect("finished state updates publish the hashed post state"), - #[cfg(feature = "trie-debug")] - debug_recorders, }) } diff --git a/crates/trie/parallel/Cargo.toml b/crates/trie/parallel/Cargo.toml index b9bc03c7513..7a67d2fffa8 100644 --- a/crates/trie/parallel/Cargo.toml +++ b/crates/trie/parallel/Cargo.toml @@ -41,7 +41,6 @@ reth-metrics = { workspace = true, optional = true } metrics = { workspace = true, optional = true } # `trie-debug` feature -reth-trie-sparse = { workspace = true, optional = true, features = ["trie-debug"] } rand = { workspace = true, optional = true } [dev-dependencies] @@ -62,8 +61,6 @@ default = ["metrics"] metrics = ["reth-metrics", "dep:metrics", "reth-trie/metrics"] trie-debug = [ "dep:rand", - "dep:reth-trie-sparse", - "reth-trie-sparse?/trie-debug", ] test-utils = [ "reth-primitives-traits/test-utils", @@ -71,6 +68,5 @@ test-utils = [ "reth-trie/test-utils", "reth-tasks/test-utils", "reth-chainspec/test-utils", - "reth-trie-sparse?/test-utils", "reth-ethereum-primitives/test-utils", ] diff --git a/crates/trie/parallel/src/state_root_task.rs b/crates/trie/parallel/src/state_root_task.rs index eabc3770649..5eb920317a9 100644 --- a/crates/trie/parallel/src/state_root_task.rs +++ b/crates/trie/parallel/src/state_root_task.rs @@ -43,10 +43,6 @@ pub struct StateRootComputeOutcome { pub trie_updates: Arc, /// Hashed post state produced while computing the state root. pub hashed_state: Arc, - /// Debug recorders taken from the sparse tries, keyed by `None` for account trie - /// and `Some(address)` for storage tries. - #[cfg(feature = "trie-debug")] - pub debug_recorders: Vec<(Option, reth_trie_sparse::debug_recorder::TrieDebugRecorder)>, } /// Handle to a background sparse trie state root computation. @@ -730,8 +726,6 @@ mod tests { state_root: B256::repeat_byte(0x42), trie_updates: Arc::new(TrieUpdates::default()), hashed_state: Arc::new(HashedPostState::default()), - #[cfg(feature = "trie-debug")] - debug_recorders: Vec::new(), })) .unwrap(); let outcome = handle.state_root().expect("outcome is delivered"); diff --git a/crates/trie/sparse/Cargo.toml b/crates/trie/sparse/Cargo.toml index 59f2e1b13bf..0888117dd74 100644 --- a/crates/trie/sparse/Cargo.toml +++ b/crates/trie/sparse/Cargo.toml @@ -25,8 +25,6 @@ alloy-primitives.workspace = true # misc either = { workspace = true, optional = true } rayon = { workspace = true, optional = true } -serde = { workspace = true, features = ["derive"], optional = true } -serde_json = { workspace = true, optional = true } smallvec = { workspace = true, optional = true } slotmap = { workspace = true, optional = true } strum = { workspace = true, features = ["derive"] } @@ -67,14 +65,11 @@ std = [ "reth-primitives-traits/rayon", "reth-trie-common/std", "tracing/std", - "serde?/std", - "serde_json?/std", "strum/std", "reth-tracing/std", "either/std", ] metrics = ["dep:reth-metrics", "dep:metrics", "std"] -trie-debug = ["std", "dep:serde", "dep:serde_json", "alloy-primitives/serde", "alloy-trie/serde"] test-utils = [ "std", "reth-primitives-traits/test-utils", diff --git a/crates/trie/sparse/src/arena/mod.rs b/crates/trie/sparse/src/arena/mod.rs index 26ae9f62710..2eb4f97b3af 100644 --- a/crates/trie/sparse/src/arena/mod.rs +++ b/crates/trie/sparse/src/arena/mod.rs @@ -24,9 +24,6 @@ use slotmap::{DefaultKey, SlotMap}; use smallvec::SmallVec; use tracing::{instrument, trace}; -#[cfg(feature = "trie-debug")] -use crate::debug_recorder::{LeafUpdateRecord, ProofTrieNodeRecord, RecordedOp, TrieDebugRecorder}; - /// Alias for the slotmap key type used as node references throughout the arena trie. type Index = DefaultKey; /// Alias for the slotmap used as the node arena throughout the arena trie. @@ -607,9 +604,6 @@ pub struct ArenaParallelSparseTrie { buffers: ArenaTrieBuffers, /// Thresholds controlling when parallelism is enabled for different operations. parallelism_thresholds: ArenaParallelismThresholds, - /// Debug recorder for tracking mutating operations. - #[cfg(feature = "trie-debug")] - debug_recorder: TrieDebugRecorder, } impl ArenaParallelSparseTrie { @@ -622,154 +616,6 @@ impl ArenaParallelSparseTrie { self } - /// Resets the debug recorder and records the current trie state as `SetRoot` + `RevealNodes` - /// ops, representing the initial state at the beginning of a block (after pruning). - /// - /// Walks the upper arena and all subtries depth-first using the cursor, converting each - /// node into a [`crate::debug_recorder::ProofTrieNodeRecord`]. - #[cfg(feature = "trie-debug")] - fn record_initial_state(&mut self) { - use crate::debug_recorder::{NodeStateRecord, TrieNodeRecord}; - use alloy_primitives::hex; - use alloy_trie::nodes::{BranchNode, TrieNode}; - - fn state_to_record(state: &ArenaSparseNodeState) -> NodeStateRecord { - match state { - ArenaSparseNodeState::Revealed => NodeStateRecord::Revealed, - ArenaSparseNodeState::Cached { rlp_node, .. } => { - NodeStateRecord::Cached { rlp_node: hex::encode(rlp_node.as_ref()) } - } - ArenaSparseNodeState::Dirty => NodeStateRecord::Dirty, - } - } - - /// Converts an [`ArenaSparseNode`] into a [`ProofTrieNodeRecord`] at the given path. - /// For branch children, resolves revealed children's cached RLP from `arena`. - /// Returns `None` for subtrie/taken-subtrie nodes (handled separately). - fn node_to_record( - arena: &NodeArena, - idx: Index, - path: Nibbles, - ) -> Option { - match &arena[idx] { - ArenaSparseNode::EmptyRoot { .. } => Some(ProofTrieNodeRecord { - path, - node: TrieNodeRecord(TrieNode::EmptyRoot), - masks: None, - short_key: None, - state: None, - }), - ArenaSparseNode::Branch(b) => { - let stack = b - .children - .iter() - .map(|child| match child { - ArenaSparseNodeBranchChild::Blinded(rlp) => rlp.clone(), - ArenaSparseNodeBranchChild::Revealed(child_idx) => { - // After pruning / root(), all nodes have cached RLP. - arena[*child_idx] - .state_ref() - .and_then(|s| s.cached_rlp_node()) - .cloned() - .unwrap_or_default() - } - }) - .collect(); - Some(ProofTrieNodeRecord { - path, - node: TrieNodeRecord(TrieNode::Branch(BranchNode::new( - stack, - b.state_mask, - ))), - masks: Some(( - b.branch_masks.hash_mask.get(), - b.branch_masks.tree_mask.get(), - )), - short_key: (!b.short_key.is_empty()).then_some(b.short_key), - state: Some(state_to_record(&b.state)), - }) - } - ArenaSparseNode::Leaf { key, value, state, .. } => Some(ProofTrieNodeRecord { - path, - node: TrieNodeRecord(TrieNode::Leaf(alloy_trie::nodes::LeafNode::new( - *key, - value.clone(), - ))), - masks: None, - short_key: None, - state: Some(state_to_record(state)), - }), - ArenaSparseNode::Subtrie(_) | ArenaSparseNode::TakenSubtrie => None, - } - } - - /// Walks an arena depth-first using `cursor` and collects all nodes as records. - fn collect_records( - arena: &mut NodeArena, - root: Index, - root_path: Nibbles, - cursor: &mut ArenaCursor, - result: &mut Vec, - ) { - cursor.reset(arena, root, root_path); - - // The cursor starts with root on the stack but `next` only yields children. - if let Some(record) = node_to_record(arena, root, root_path) { - result.push(record); - } - - loop { - match cursor.next(arena, |_, node| { - matches!(node, ArenaSparseNode::Branch(_) | ArenaSparseNode::Leaf { .. }) - }) { - NextResult::Done => break, - NextResult::Branch | NextResult::NonBranch => { - let head = cursor.head().expect("cursor is non-empty"); - if let Some(record) = node_to_record(arena, head.index, head.path) { - result.push(record); - } - } - } - } - } - - let mut nodes = Vec::new(); - - // Collect from the upper arena. - collect_records( - &mut self.upper_arena, - self.root, - Nibbles::default(), - &mut self.buffers.cursor, - &mut nodes, - ); - - // Collect from all subtries. - for (_, node) in &mut self.upper_arena { - if let ArenaSparseNode::Subtrie(subtrie) = node { - collect_records( - &mut subtrie.arena, - subtrie.root, - subtrie.path, - &mut self.buffers.cursor, - &mut nodes, - ); - } - } - - // Reset the recorder and record that we pruned, then the initial state. - self.debug_recorder.reset(); - self.debug_recorder.record(RecordedOp::Prune); - - // First node is the root → SetRoot, remaining → RevealNodes. - if let Some(root_record) = nodes.first() { - self.debug_recorder.record(RecordedOp::SetRoot { node: root_record.clone() }); - } - if nodes.len() > 1 { - self.debug_recorder.record(RecordedOp::RevealNodes { nodes: nodes[1..].to_vec() }); - } - } - /// Returns `true` if a node at the given path length should be placed in a subtrie rather /// than the upper arena. const fn should_be_subtrie(path_len: usize) -> bool { @@ -2230,8 +2076,6 @@ impl Default for ArenaParallelSparseTrie { root, buffers: ArenaTrieBuffers::default(), parallelism_thresholds: ArenaParallelismThresholds::default(), - #[cfg(feature = "trie-debug")] - debug_recorder: Default::default(), } } } @@ -2259,15 +2103,6 @@ impl SparseTrie for ArenaParallelSparseTrie { masks: Option, retain_updates: bool, ) -> SparseTrieResult<()> { - #[cfg(feature = "trie-debug")] - self.debug_recorder.record(RecordedOp::SetRoot { - node: ProofTrieNodeRecord::from_proof_trie_node_v2(&ProofTrieNodeV2 { - path: Nibbles::default(), - node: root.clone(), - masks, - }), - }); - debug_assert!( matches!(self.upper_arena[self.root], ArenaSparseNode::EmptyRoot { .. }), "set_root called on a trie that already has revealed nodes" @@ -2327,11 +2162,6 @@ impl SparseTrie for ArenaParallelSparseTrie { return Ok(()); } - #[cfg(feature = "trie-debug")] - self.debug_recorder.record(RecordedOp::RevealNodes { - nodes: nodes.iter().map(ProofTrieNodeRecord::from_proof_trie_node_v2).collect(), - }); - if matches!(self.upper_arena[self.root], ArenaSparseNode::EmptyRoot { .. }) { trace!(target: TRACE_TARGET, "Skipping reveal_nodes on empty root"); return Ok(()); @@ -2467,9 +2297,6 @@ impl SparseTrie for ArenaParallelSparseTrie { #[instrument(level = "trace", target = TRACE_TARGET, skip_all, ret)] fn root(&mut self, new_epoch: TrieNodeEpoch) -> B256 { - #[cfg(feature = "trie-debug")] - self.debug_recorder.record(RecordedOp::Root); - self.update_subtrie_hashes(new_epoch); let rlp_node = Self::update_cached_rlp( @@ -2497,9 +2324,6 @@ impl SparseTrie for ArenaParallelSparseTrie { #[instrument(level = "trace", target = TRACE_TARGET, skip_all)] fn update_subtrie_hashes(&mut self, new_epoch: TrieNodeEpoch) { - #[cfg(feature = "trie-debug")] - self.debug_recorder.record(RecordedOp::UpdateSubtrieHashes); - trace!(target: TRACE_TARGET, "Updating subtrie hashes"); // Only descend if the root is a branch; otherwise there are no subtries. @@ -2623,9 +2447,6 @@ impl SparseTrie for ArenaParallelSparseTrie { #[instrument(level = "trace", target = TRACE_TARGET, skip_all)] fn clear(&mut self) { - #[cfg(feature = "trie-debug")] - self.debug_recorder.reset(); - self.upper_arena = SlotMap::new(); self.root = self .upper_arena @@ -2776,9 +2597,6 @@ impl SparseTrie for ArenaParallelSparseTrie { compact_arena(&mut self.upper_arena, &mut self.root); } - #[cfg(feature = "trie-debug")] - self.record_initial_state(); - pruned } @@ -2797,12 +2615,6 @@ impl SparseTrie for ArenaParallelSparseTrie { return Ok(()); } - #[cfg(feature = "trie-debug")] - let recorded_updates: Vec<_> = - updates.iter().map(|(k, v)| (*k, LeafUpdateRecord::from(v))).collect(); - #[cfg(feature = "trie-debug")] - let mut recorded_proof_targets: Vec<(B256, Option)> = Vec::new(); - // Drain and sort updates lexicographically by nibbles path. let mut sorted: Vec<_> = updates.drain().map(|(key, update)| (key, Nibbles::unpack(key), update)).collect(); @@ -2830,8 +2642,6 @@ impl SparseTrie for ArenaParallelSparseTrie { let parent = ProofV2TargetParent::new(logical_len); trace!(target: TRACE_TARGET, ?key, ?parent, "Update hit blinded node, requesting proof"); proof_required_fn(key, parent); - #[cfg(feature = "trie-debug")] - recorded_proof_targets.push((key, parent.path_len())); updates.insert(key, update.clone()); } // Subtrie — forward all consecutive updates under this subtrie's prefix. @@ -2859,8 +2669,6 @@ impl SparseTrie for ArenaParallelSparseTrie { ) { trace!(target: TRACE_TARGET, proof_key = ?proof.key, proof_parent = ?proof.parent, "Subtrie collapse would need blinded sibling, requesting proof"); proof_required_fn(proof.key, proof.parent); - #[cfg(feature = "trie-debug")] - recorded_proof_targets.push((proof.key, proof.parent.path_len())); for &(key, _, ref update) in subtrie_updates { updates.insert(key, update.clone()); } @@ -2911,8 +2719,6 @@ impl SparseTrie for ArenaParallelSparseTrie { for (target_idx, proof) in subtrie.required_proofs.drain(..) { proof_required_fn(proof.key, proof.parent); - #[cfg(feature = "trie-debug")] - recorded_proof_targets.push((proof.key, proof.parent.path_len())); let (key, _, ref update) = subtrie_updates[target_idx]; updates.insert(key, update.clone()); } @@ -2973,8 +2779,6 @@ impl SparseTrie for ArenaParallelSparseTrie { match result { RemoveLeafResult::NeedsProof { key, proof_key, parent } => { proof_required_fn(proof_key, parent); - #[cfg(feature = "trie-debug")] - recorded_proof_targets.push((proof_key, parent.path_len())); let update = mem::replace(&mut sorted[update_idx].2, LeafUpdate::Touched); updates.insert(key, update); @@ -3011,12 +2815,6 @@ impl SparseTrie for ArenaParallelSparseTrie { #[cfg(debug_assertions)] self.debug_assert_subtrie_structure(); - #[cfg(feature = "trie-debug")] - self.debug_recorder.record(RecordedOp::UpdateLeaves { - updates: recorded_updates, - proof_targets: recorded_proof_targets, - }); - return Ok(()); } @@ -3041,8 +2839,6 @@ impl SparseTrie for ArenaParallelSparseTrie { let subtrie_updates = &sorted[range]; for (target_idx, proof) in subtrie.required_proofs.drain(..) { proof_required_fn(proof.key, proof.parent); - #[cfg(feature = "trie-debug")] - recorded_proof_targets.push((proof.key, proof.parent.path_len())); let (key, _, ref update) = subtrie_updates[target_idx]; updates.insert(key, update.clone()); } @@ -3096,19 +2892,8 @@ impl SparseTrie for ArenaParallelSparseTrie { #[cfg(debug_assertions)] self.debug_assert_subtrie_structure(); - #[cfg(feature = "trie-debug")] - self.debug_recorder.record(RecordedOp::UpdateLeaves { - updates: recorded_updates, - proof_targets: recorded_proof_targets, - }); - Ok(()) } - - #[cfg(feature = "trie-debug")] - fn take_debug_recorder(&mut self) -> TrieDebugRecorder { - core::mem::take(&mut self.debug_recorder) - } } #[cfg(test)] diff --git a/crates/trie/sparse/src/debug_recorder.rs b/crates/trie/sparse/src/debug_recorder.rs deleted file mode 100644 index dd99f3f3e7e..00000000000 --- a/crates/trie/sparse/src/debug_recorder.rs +++ /dev/null @@ -1,207 +0,0 @@ -//! Debug recorder for tracking mutating operations on sparse tries. -//! -//! This module is only available with the `trie-debug` feature and provides -//! infrastructure for recording all mutations to a [`crate::ArenaParallelSparseTrie`] -//! for post-hoc debugging of state root mismatches. - -use alloc::{string::String, vec::Vec}; -use alloy_primitives::{hex, Bytes, B256}; -use alloy_trie::nodes::TrieNode; -use reth_trie_common::Nibbles; -use serde::Serialize; - -/// Records mutating operations performed on a sparse trie in the order they occurred. -#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize)] -pub struct TrieDebugRecorder { - ops: Vec, -} - -impl TrieDebugRecorder { - /// Creates a new empty recorder. - pub fn new() -> Self { - Self::default() - } - - /// Clears all recorded operations. - pub fn reset(&mut self) { - self.ops.clear(); - } - - /// Records a single operation. - pub fn record(&mut self, op: RecordedOp) { - self.ops.push(op); - } - - /// Returns a reference to the recorded operations. - pub fn ops(&self) -> &[RecordedOp] { - &self.ops - } - - /// Takes and returns the recorded operations, leaving the recorder empty. - pub fn take_ops(&mut self) -> Vec { - core::mem::take(&mut self.ops) - } - - /// Returns `true` if no operations have been recorded. - pub const fn is_empty(&self) -> bool { - self.ops.is_empty() - } -} - -/// A mutating operation recorded from a sparse trie. -#[derive(Debug, Clone, PartialEq, Eq, Serialize)] -pub enum RecordedOp { - /// Records a `reveal_nodes` call with the nodes that were revealed. - RevealNodes { - /// The proof trie nodes that were revealed. - nodes: Vec, - }, - /// Records an `update_leaves` call with the leaf updates. - UpdateLeaves { - /// The leaf updates that were applied. - updates: Vec<(B256, LeafUpdateRecord)>, - /// Proof targets returned via the callback as `(key, optional parent path length)` pairs. - proof_targets: Vec<(B256, Option)>, - }, - /// Records an `update_subtrie_hashes` call. - UpdateSubtrieHashes, - /// Records a `root()` call. - Root, - /// Records a `set_root` call with the root node that was set. - SetRoot { - /// The root trie node that was set. - node: ProofTrieNodeRecord, - }, - /// Records a `prune` call. Emitted before the post-prune `SetRoot`/`RevealNodes` - /// so consumers know the following initial state is the result of pruning. - Prune, -} - -/// A serializable record of a proof trie node. -#[derive(Debug, Clone, PartialEq, Eq, Serialize)] -pub struct ProofTrieNodeRecord { - /// The nibble path of the node. - pub path: Nibbles, - /// The trie node. - pub node: TrieNodeRecord, - /// The branch node masks `(hash_mask, tree_mask)` stored as raw `u16` values, if present. - pub masks: Option<(u16, u16)>, - /// The short key (extension key) of a branch node. When present, the branch's logical path - /// is `path` extended by this key. The arena trie merges extension nodes into their child - /// branch, so this replaces separate `Extension` node records. - #[serde(skip_serializing_if = "Option::is_none")] - pub short_key: Option, - /// The node's state (`Revealed`, `Cached`, or `Dirty`), if known. - #[serde(skip_serializing_if = "Option::is_none")] - pub state: Option, -} - -/// A serializable record of a node's state, mirroring the arena trie's `ArenaSparseNodeState`. -#[derive(Debug, Clone, PartialEq, Eq, Serialize)] -pub enum NodeStateRecord { - /// The node has been revealed but its RLP encoding is not cached. - Revealed, - /// The node has a cached RLP encoding that is still valid. - Cached { - /// The cached RLP-encoded node, hex-encoded. - rlp_node: String, - }, - /// The node has been modified and its RLP encoding needs recomputation. - Dirty, -} - -impl ProofTrieNodeRecord { - /// Creates a record from a [`reth_trie_common::ProofTrieNode`]. - pub fn from_proof_trie_node(node: &reth_trie_common::ProofTrieNode) -> Self { - Self { - path: node.path, - node: TrieNodeRecord(node.node.clone()), - masks: node.masks.map(|masks| (masks.hash_mask.get(), masks.tree_mask.get())), - short_key: None, - state: None, - } - } - - /// Creates a record from a [`reth_trie_common::ProofTrieNodeV2`]. - pub fn from_proof_trie_node_v2(node: &reth_trie_common::ProofTrieNodeV2) -> Self { - use reth_trie_common::TrieNodeV2; - let (trie_node, short_key) = match &node.node { - TrieNodeV2::EmptyRoot => (TrieNode::EmptyRoot, None), - TrieNodeV2::Leaf(leaf) => (TrieNode::Leaf(leaf.clone()), None), - TrieNodeV2::Extension(ext) => (TrieNode::Extension(ext.clone()), None), - TrieNodeV2::Branch(branch) => ( - TrieNode::Branch(alloy_trie::nodes::BranchNode::new( - branch.stack.clone(), - branch.state_mask, - )), - (!branch.key.is_empty()).then_some(branch.key), - ), - }; - Self { - path: node.path, - node: TrieNodeRecord(trie_node), - masks: node.masks.map(|masks| (masks.hash_mask.get(), masks.tree_mask.get())), - short_key, - state: None, - } - } -} - -/// A newtype wrapper around [`TrieNode`] with custom serialization that hex-encodes byte fields -/// (leaf values, branch stack entries, extension child pointers) instead of serializing them as -/// raw integer arrays. -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct TrieNodeRecord(pub TrieNode); - -impl From for TrieNodeRecord { - fn from(node: TrieNode) -> Self { - Self(node) - } -} - -impl Serialize for TrieNodeRecord { - fn serialize(&self, serializer: S) -> Result { - use serde::ser::SerializeStructVariant; - match &self.0 { - TrieNode::EmptyRoot => serializer.serialize_unit_variant("TrieNode", 0, "EmptyRoot"), - TrieNode::Branch(branch) => { - let stack_hex: Vec = - branch.stack.iter().map(|n| hex::encode(n.as_ref())).collect(); - let mut sv = serializer.serialize_struct_variant("TrieNode", 1, "Branch", 2)?; - sv.serialize_field("stack", &stack_hex)?; - sv.serialize_field("state_mask", &branch.state_mask.get())?; - sv.end() - } - TrieNode::Extension(ext) => { - let mut sv = serializer.serialize_struct_variant("TrieNode", 2, "Extension", 2)?; - sv.serialize_field("key", &ext.key)?; - sv.serialize_field("child", &hex::encode(ext.child.as_ref()))?; - sv.end() - } - TrieNode::Leaf(leaf) => { - let mut sv = serializer.serialize_struct_variant("TrieNode", 3, "Leaf", 2)?; - sv.serialize_field("key", &leaf.key)?; - sv.serialize_field("value", &hex::encode(&leaf.value))?; - sv.end() - } - } - } -} - -/// A serializable record of a leaf update, mirroring [`crate::LeafUpdate`]. -#[derive(Debug, Clone, PartialEq, Eq, Serialize)] -pub enum LeafUpdateRecord { - /// The leaf value was changed to the given RLP-encoded value. - Changed(Bytes), - /// The leaf value was touched but the new value is not yet known. - Touched, -} - -impl From<&crate::LeafUpdate> for LeafUpdateRecord { - fn from(update: &crate::LeafUpdate) -> Self { - match update { - crate::LeafUpdate::Changed(value) => Self::Changed(value.clone().into()), - crate::LeafUpdate::Touched => Self::Touched, - } - } -} diff --git a/crates/trie/sparse/src/lib.rs b/crates/trie/sparse/src/lib.rs index 36e7868dca3..2e4bef5a415 100644 --- a/crates/trie/sparse/src/lib.rs +++ b/crates/trie/sparse/src/lib.rs @@ -26,11 +26,6 @@ pub use arena::*; #[cfg(feature = "metrics")] mod metrics; -#[cfg(feature = "trie-debug")] -pub mod debug_recorder; -#[cfg(feature = "trie-debug")] -use serde_json as _; - /// Re-export sparse trie error types. pub mod errors { pub use reth_execution_errors::{ diff --git a/crates/trie/sparse/src/state.rs b/crates/trie/sparse/src/state.rs index a0ed6c92acf..fed07f8e53f 100644 --- a/crates/trie/sparse/src/state.rs +++ b/crates/trie/sparse/src/state.rs @@ -1,5 +1,3 @@ -#[cfg(feature = "trie-debug")] -use crate::debug_recorder::TrieDebugRecorder; use crate::{ traits::SparseTrie as SparseTrieTrait, ArenaParallelSparseTrie, RevealableSparseTrie, TrieNodeEpoch, @@ -122,26 +120,6 @@ impl SparseStateTrie { } } -impl SparseStateTrie { - /// Takes all debug recorders from the account trie and all revealed storage tries. - /// - /// Returns a vec of `(Option, TrieDebugRecorder)` where `None` is the account trie - /// key, and `Some(address)` are storage trie keys. - #[cfg(feature = "trie-debug")] - pub fn take_debug_recorders(&mut self) -> alloc::vec::Vec<(Option, TrieDebugRecorder)> { - let mut recorders = alloc::vec::Vec::new(); - if let Some(trie) = self.state.as_revealed_mut() { - recorders.push((None, trie.take_debug_recorder())); - } - for (address, trie) in &mut self.storage.tries { - if let Some(trie) = trie.as_revealed_mut() { - recorders.push((Some(*address), trie.take_debug_recorder())); - } - } - recorders - } -} - impl SparseStateTrie where A: SparseTrieTrait + Default, diff --git a/crates/trie/sparse/src/traits.rs b/crates/trie/sparse/src/traits.rs index 2c0fd0f9e88..2f8ffc08b25 100644 --- a/crates/trie/sparse/src/traits.rs +++ b/crates/trie/sparse/src/traits.rs @@ -13,9 +13,6 @@ use reth_trie_common::{ BranchNodeMasks, Nibbles, ProofTrieNodeV2, ProofV2TargetParent, TrieNodeV2, }; -#[cfg(feature = "trie-debug")] -use crate::debug_recorder::TrieDebugRecorder; - /// Modification epoch assigned to cached sparse trie nodes. /// /// Epochs must increase monotonically. Nodes materialized from the parent state without being @@ -224,15 +221,6 @@ pub trait SparseTrie: Sized + Debug + Send + Sync { /// The number of nodes converted to hash stubs. fn prune(&mut self, prune_before: TrieNodeEpoch) -> usize; - /// Takes the debug recorder out of this trie, replacing it with an empty one. - /// - /// Returns the recorder containing all recorded mutations since the last reset. - /// The default implementation returns an empty recorder. - #[cfg(feature = "trie-debug")] - fn take_debug_recorder(&mut self) -> TrieDebugRecorder { - TrieDebugRecorder::default() - } - /// Applies leaf updates to the sparse trie. /// /// When a [`LeafUpdate::Changed`] is successfully applied, it is removed from the From 4145879102d6473ea1fe5a9a8b6187c3b996c3a1 Mon Sep 17 00:00:00 2001 From: Derek Cofausper <256792747+decofe@users.noreply.github.com> Date: Wed, 2 Sep 2026 12:20:54 +0000 Subject: [PATCH 121/294] chore: bump versions to 2.5.2 (#26921) Co-authored-by: Emma Jamieson-Hoare <21029500+emmajam@users.noreply.github.com> --- Cargo.lock | 228 +++++++++++++++++++-------------------- Cargo.toml | 2 +- docs/vocs/vocs.config.ts | 2 +- 3 files changed, 116 insertions(+), 116 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a70d99f5623..d97840c0201 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3271,7 +3271,7 @@ dependencies = [ [[package]] name = "ef-test-runner" -version = "2.5.1" +version = "2.5.2" dependencies = [ "clap", "ef-tests", @@ -3279,7 +3279,7 @@ dependencies = [ [[package]] name = "ef-tests" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-consensus", "alloy-eips", @@ -3732,7 +3732,7 @@ dependencies = [ [[package]] name = "example-full-contract-state" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-primitives", "eyre", @@ -3850,7 +3850,7 @@ dependencies = [ [[package]] name = "exex-subscription" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-primitives", "clap", @@ -7654,7 +7654,7 @@ checksum = "1e061d1b48cb8d38042de4ae0a7a6401009d6143dc80d2e2d6f31f0bdd6470c7" [[package]] name = "reth" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-node-bindings", "alloy-primitives", @@ -7704,7 +7704,7 @@ dependencies = [ [[package]] name = "reth-basic-payload-builder" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-consensus", "alloy-eips", @@ -7730,7 +7730,7 @@ dependencies = [ [[package]] name = "reth-bb" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-consensus", "alloy-eips", @@ -7777,7 +7777,7 @@ dependencies = [ [[package]] name = "reth-chain-state" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-consensus", "alloy-eips", @@ -7808,7 +7808,7 @@ dependencies = [ [[package]] name = "reth-chainspec" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-chains", "alloy-consensus", @@ -7828,7 +7828,7 @@ dependencies = [ [[package]] name = "reth-cli" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-genesis", "clap", @@ -7840,7 +7840,7 @@ dependencies = [ [[package]] name = "reth-cli-commands" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-chains", "alloy-consensus", @@ -7931,7 +7931,7 @@ dependencies = [ [[package]] name = "reth-cli-runner" -version = "2.5.1" +version = "2.5.2" dependencies = [ "reth-tasks", "tokio", @@ -7940,7 +7940,7 @@ dependencies = [ [[package]] name = "reth-cli-util" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-eips", "alloy-primitives", @@ -7994,7 +7994,7 @@ dependencies = [ [[package]] name = "reth-config" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-primitives", "eyre", @@ -8012,7 +8012,7 @@ dependencies = [ [[package]] name = "reth-consensus" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-consensus", "alloy-eip7928", @@ -8025,7 +8025,7 @@ dependencies = [ [[package]] name = "reth-consensus-common" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-consensus", "alloy-eips", @@ -8039,7 +8039,7 @@ dependencies = [ [[package]] name = "reth-consensus-debug-client" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-consensus", "alloy-eips", @@ -8063,7 +8063,7 @@ dependencies = [ [[package]] name = "reth-db" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-consensus", "alloy-primitives", @@ -8099,7 +8099,7 @@ dependencies = [ [[package]] name = "reth-db-api" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-consensus", "alloy-primitives", @@ -8127,7 +8127,7 @@ dependencies = [ [[package]] name = "reth-db-common" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-consensus", "alloy-genesis", @@ -8158,7 +8158,7 @@ dependencies = [ [[package]] name = "reth-db-models" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-eips", "alloy-primitives", @@ -8174,7 +8174,7 @@ dependencies = [ [[package]] name = "reth-discv4" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-primitives", "alloy-rlp", @@ -8200,7 +8200,7 @@ dependencies = [ [[package]] name = "reth-discv5" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-primitives", "alloy-rlp", @@ -8225,7 +8225,7 @@ dependencies = [ [[package]] name = "reth-dns-discovery" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-chains", "alloy-primitives", @@ -8253,7 +8253,7 @@ dependencies = [ [[package]] name = "reth-downloaders" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-consensus", "alloy-eips", @@ -8293,7 +8293,7 @@ dependencies = [ [[package]] name = "reth-e2e-test-utils" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-consensus", "alloy-eips", @@ -8351,7 +8351,7 @@ dependencies = [ [[package]] name = "reth-ecies" -version = "2.5.1" +version = "2.5.2" dependencies = [ "aes", "alloy-primitives", @@ -8377,7 +8377,7 @@ dependencies = [ [[package]] name = "reth-engine-local" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-consensus", "alloy-primitives", @@ -8399,7 +8399,7 @@ dependencies = [ [[package]] name = "reth-engine-primitives" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-consensus", "alloy-eips", @@ -8426,7 +8426,7 @@ dependencies = [ [[package]] name = "reth-engine-tree" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-consensus", "alloy-eip7928", @@ -8496,7 +8496,7 @@ dependencies = [ [[package]] name = "reth-engine-util" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-consensus", "alloy-primitives", @@ -8525,7 +8525,7 @@ dependencies = [ [[package]] name = "reth-era" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-consensus", "alloy-eips", @@ -8550,7 +8550,7 @@ dependencies = [ [[package]] name = "reth-era-downloader" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-primitives", "bytes", @@ -8568,7 +8568,7 @@ dependencies = [ [[package]] name = "reth-era-utils" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-consensus", "alloy-primitives", @@ -8599,7 +8599,7 @@ dependencies = [ [[package]] name = "reth-errors" -version = "2.5.1" +version = "2.5.2" dependencies = [ "reth-consensus", "reth-execution-errors", @@ -8609,7 +8609,7 @@ dependencies = [ [[package]] name = "reth-eth-wire" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-chains", "alloy-consensus", @@ -8647,7 +8647,7 @@ dependencies = [ [[package]] name = "reth-eth-wire-types" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-chains", "alloy-consensus", @@ -8675,7 +8675,7 @@ dependencies = [ [[package]] name = "reth-ethereum" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-rpc-types-engine", "alloy-rpc-types-eth", @@ -8715,7 +8715,7 @@ dependencies = [ [[package]] name = "reth-ethereum-cli" -version = "2.5.1" +version = "2.5.2" dependencies = [ "clap", "eyre", @@ -8738,7 +8738,7 @@ dependencies = [ [[package]] name = "reth-ethereum-consensus" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-consensus", "alloy-eips", @@ -8754,7 +8754,7 @@ dependencies = [ [[package]] name = "reth-ethereum-engine-primitives" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-eips", "alloy-primitives", @@ -8770,7 +8770,7 @@ dependencies = [ [[package]] name = "reth-ethereum-forks" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-eip2124", "alloy-hardforks 0.4.8", @@ -8783,7 +8783,7 @@ dependencies = [ [[package]] name = "reth-ethereum-payload-builder" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-consensus", "alloy-eips", @@ -8812,7 +8812,7 @@ dependencies = [ [[package]] name = "reth-ethereum-primitives" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-consensus", "alloy-eips", @@ -8833,7 +8833,7 @@ dependencies = [ [[package]] name = "reth-etl" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-primitives", "rayon", @@ -8843,7 +8843,7 @@ dependencies = [ [[package]] name = "reth-evm" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-consensus", "alloy-eip7928", @@ -8869,7 +8869,7 @@ dependencies = [ [[package]] name = "reth-evm-ethereum" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-consensus", "alloy-eips", @@ -8894,7 +8894,7 @@ dependencies = [ [[package]] name = "reth-execution-cache" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-primitives", "fixed-cache", @@ -8912,7 +8912,7 @@ dependencies = [ [[package]] name = "reth-execution-errors" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-evm", "alloy-primitives", @@ -8924,7 +8924,7 @@ dependencies = [ [[package]] name = "reth-execution-types" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-consensus", "alloy-eips", @@ -8945,7 +8945,7 @@ dependencies = [ [[package]] name = "reth-exex" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-consensus", "alloy-eips", @@ -8990,7 +8990,7 @@ dependencies = [ [[package]] name = "reth-exex-test-utils" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-eips", "eyre", @@ -9021,7 +9021,7 @@ dependencies = [ [[package]] name = "reth-exex-types" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-eips", "alloy-primitives", @@ -9038,7 +9038,7 @@ dependencies = [ [[package]] name = "reth-fs-util" -version = "2.5.1" +version = "2.5.2" dependencies = [ "serde", "serde_json", @@ -9047,7 +9047,7 @@ dependencies = [ [[package]] name = "reth-invalid-block-hooks" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-consensus", "alloy-eips", @@ -9078,7 +9078,7 @@ dependencies = [ [[package]] name = "reth-ipc" -version = "2.5.1" +version = "2.5.2" dependencies = [ "bytes", "futures", @@ -9100,7 +9100,7 @@ dependencies = [ [[package]] name = "reth-libmdbx" -version = "2.5.1" +version = "2.5.2" dependencies = [ "bitflags 2.13.0", "byteorder", @@ -9117,7 +9117,7 @@ dependencies = [ [[package]] name = "reth-mdbx-sys" -version = "2.5.1" +version = "2.5.2" dependencies = [ "bindgen", "cc", @@ -9125,7 +9125,7 @@ dependencies = [ [[package]] name = "reth-metrics" -version = "2.5.1" +version = "2.5.2" dependencies = [ "futures", "libc", @@ -9138,7 +9138,7 @@ dependencies = [ [[package]] name = "reth-net-banlist" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-primitives", "ipnet", @@ -9146,7 +9146,7 @@ dependencies = [ [[package]] name = "reth-net-nat" -version = "2.5.1" +version = "2.5.2" dependencies = [ "futures-util", "if-addrs", @@ -9160,7 +9160,7 @@ dependencies = [ [[package]] name = "reth-network" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-consensus", "alloy-eip7928", @@ -9229,7 +9229,7 @@ dependencies = [ [[package]] name = "reth-network-api" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-consensus", "alloy-primitives", @@ -9253,7 +9253,7 @@ dependencies = [ [[package]] name = "reth-network-p2p" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-consensus", "alloy-eip7928", @@ -9277,7 +9277,7 @@ dependencies = [ [[package]] name = "reth-network-peers" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-primitives", "alloy-rlp", @@ -9294,7 +9294,7 @@ dependencies = [ [[package]] name = "reth-network-types" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-eip2124", "humantime-serde", @@ -9307,7 +9307,7 @@ dependencies = [ [[package]] name = "reth-nippy-jar" -version = "2.5.1" +version = "2.5.2" dependencies = [ "anyhow", "bincode", @@ -9325,7 +9325,7 @@ dependencies = [ [[package]] name = "reth-node-api" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-rpc-types-engine", "eyre", @@ -9348,7 +9348,7 @@ dependencies = [ [[package]] name = "reth-node-builder" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-consensus", "alloy-eips", @@ -9419,7 +9419,7 @@ dependencies = [ [[package]] name = "reth-node-core" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-consensus", "alloy-eips", @@ -9475,7 +9475,7 @@ dependencies = [ [[package]] name = "reth-node-ethereum" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-consensus", "alloy-contract", @@ -9544,7 +9544,7 @@ dependencies = [ [[package]] name = "reth-node-ethstats" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-consensus", "alloy-primitives", @@ -9567,7 +9567,7 @@ dependencies = [ [[package]] name = "reth-node-events" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-consensus", "alloy-eips", @@ -9590,7 +9590,7 @@ dependencies = [ [[package]] name = "reth-node-metrics" -version = "2.5.1" +version = "2.5.2" dependencies = [ "bytes", "eyre", @@ -9619,7 +9619,7 @@ dependencies = [ [[package]] name = "reth-node-types" -version = "2.5.1" +version = "2.5.2" dependencies = [ "reth-chainspec", "reth-db-api", @@ -9630,7 +9630,7 @@ dependencies = [ [[package]] name = "reth-payload-builder" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-consensus", "alloy-primitives", @@ -9653,7 +9653,7 @@ dependencies = [ [[package]] name = "reth-payload-builder-primitives" -version = "2.5.1" +version = "2.5.2" dependencies = [ "pin-project", "reth-payload-primitives", @@ -9664,7 +9664,7 @@ dependencies = [ [[package]] name = "reth-payload-primitives" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-consensus", "alloy-eips", @@ -9688,7 +9688,7 @@ dependencies = [ [[package]] name = "reth-payload-util" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-consensus", "alloy-primitives", @@ -9697,7 +9697,7 @@ dependencies = [ [[package]] name = "reth-payload-validator" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-consensus", "alloy-rpc-types-engine", @@ -9739,7 +9739,7 @@ dependencies = [ [[package]] name = "reth-provider" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-consensus", "alloy-eip7928", @@ -9794,7 +9794,7 @@ dependencies = [ [[package]] name = "reth-prune" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-consensus", "alloy-primitives", @@ -9825,7 +9825,7 @@ dependencies = [ [[package]] name = "reth-prune-types" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-primitives", "arbitrary", @@ -9845,7 +9845,7 @@ dependencies = [ [[package]] name = "reth-revm" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-consensus", "alloy-primitives", @@ -9861,7 +9861,7 @@ dependencies = [ [[package]] name = "reth-rpc" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-consensus", "alloy-dyn-abi", @@ -9941,7 +9941,7 @@ dependencies = [ [[package]] name = "reth-rpc-api" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-eips", "alloy-genesis", @@ -9971,7 +9971,7 @@ dependencies = [ [[package]] name = "reth-rpc-builder" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-eips", "alloy-network", @@ -10029,7 +10029,7 @@ dependencies = [ [[package]] name = "reth-rpc-convert" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-consensus", "alloy-evm", @@ -10049,7 +10049,7 @@ dependencies = [ [[package]] name = "reth-rpc-e2e-tests" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-genesis", "alloy-rpc-types-engine", @@ -10069,7 +10069,7 @@ dependencies = [ [[package]] name = "reth-rpc-engine-api" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-eips", "alloy-primitives", @@ -10105,7 +10105,7 @@ dependencies = [ [[package]] name = "reth-rpc-eth-api" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-consensus", "alloy-dyn-abi", @@ -10151,7 +10151,7 @@ dependencies = [ [[package]] name = "reth-rpc-eth-types" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-chains", "alloy-consensus", @@ -10202,7 +10202,7 @@ dependencies = [ [[package]] name = "reth-rpc-layer" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-rpc-types-engine", "brotli", @@ -10222,7 +10222,7 @@ dependencies = [ [[package]] name = "reth-rpc-server-types" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-eips", "alloy-primitives", @@ -10252,7 +10252,7 @@ dependencies = [ [[package]] name = "reth-stages" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-consensus", "alloy-eip7928", @@ -10315,7 +10315,7 @@ dependencies = [ [[package]] name = "reth-stages-api" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-eips", "alloy-primitives", @@ -10349,7 +10349,7 @@ dependencies = [ [[package]] name = "reth-stages-types" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-primitives", "arbitrary", @@ -10365,7 +10365,7 @@ dependencies = [ [[package]] name = "reth-static-file" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-primitives", "assert_matches", @@ -10388,7 +10388,7 @@ dependencies = [ [[package]] name = "reth-static-file-types" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-primitives", "clap", @@ -10406,7 +10406,7 @@ dependencies = [ [[package]] name = "reth-storage-api" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-consensus", "alloy-eip7928", @@ -10433,7 +10433,7 @@ dependencies = [ [[package]] name = "reth-storage-errors" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-eips", "alloy-primitives", @@ -10449,7 +10449,7 @@ dependencies = [ [[package]] name = "reth-storage-overlay" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-consensus", "alloy-eips", @@ -10479,7 +10479,7 @@ dependencies = [ [[package]] name = "reth-storage-rpc-provider" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-consensus", "alloy-eips", @@ -10508,7 +10508,7 @@ dependencies = [ [[package]] name = "reth-tasks" -version = "2.5.1" +version = "2.5.2" dependencies = [ "crossbeam-utils", "dashmap", @@ -10528,7 +10528,7 @@ dependencies = [ [[package]] name = "reth-testing-utils" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-consensus", "alloy-eips", @@ -10544,7 +10544,7 @@ dependencies = [ [[package]] name = "reth-tokio-util" -version = "2.5.1" +version = "2.5.2" dependencies = [ "tokio", "tokio-stream", @@ -10553,7 +10553,7 @@ dependencies = [ [[package]] name = "reth-tracing" -version = "2.5.1" +version = "2.5.2" dependencies = [ "clap", "eyre", @@ -10572,7 +10572,7 @@ dependencies = [ [[package]] name = "reth-tracing-otlp" -version = "2.5.1" +version = "2.5.2" dependencies = [ "base64 0.22.1", "clap", @@ -10590,7 +10590,7 @@ dependencies = [ [[package]] name = "reth-transaction-pool" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-consensus", "alloy-eips", @@ -10641,7 +10641,7 @@ dependencies = [ [[package]] name = "reth-trie" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-consensus", "alloy-eips", @@ -10673,7 +10673,7 @@ dependencies = [ [[package]] name = "reth-trie-common" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-consensus", "alloy-eips", @@ -10706,7 +10706,7 @@ dependencies = [ [[package]] name = "reth-trie-db" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-consensus", "alloy-primitives", @@ -10733,7 +10733,7 @@ dependencies = [ [[package]] name = "reth-trie-parallel" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-evm", "alloy-primitives", @@ -10760,7 +10760,7 @@ dependencies = [ [[package]] name = "reth-trie-sparse" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-primitives", "alloy-rlp", diff --git a/Cargo.toml b/Cargo.toml index 6c7cc6aaff7..3b2206ec662 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,5 +1,5 @@ [workspace.package] -version = "2.5.1" +version = "2.5.2" edition = "2024" rust-version = "1.95" license = "MIT OR Apache-2.0" diff --git a/docs/vocs/vocs.config.ts b/docs/vocs/vocs.config.ts index 941f6e83ed6..6aa891ac973 100644 --- a/docs/vocs/vocs.config.ts +++ b/docs/vocs/vocs.config.ts @@ -23,7 +23,7 @@ export default defineConfig({ { text: 'Rustdocs', link: '/docs' }, { text: 'GitHub', link: 'https://github.com/paradigmxyz/reth' }, { - text: 'v2.5.1', + text: 'v2.5.2', items: [ { text: 'Releases', From 8e6ed15ff84852980e3752e6fa1c9ca2f02c63f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A9a=20Narzis?= <78718413+lean-apple@users.noreply.github.com> Date: Wed, 2 Sep 2026 12:27:43 +0000 Subject: [PATCH 122/294] feat(net): add authenticated snap bytecode downloader (#26898) --- crates/net/downloaders/src/snap/bytecode.rs | 362 ++++++++++++++++++ crates/net/downloaders/src/snap/mod.rs | 2 + crates/net/downloaders/src/snap/request.rs | 9 +- crates/net/downloaders/src/snap/test_utils.rs | 4 +- 4 files changed, 374 insertions(+), 3 deletions(-) create mode 100644 crates/net/downloaders/src/snap/bytecode.rs diff --git a/crates/net/downloaders/src/snap/bytecode.rs b/crates/net/downloaders/src/snap/bytecode.rs new file mode 100644 index 00000000000..9207818090e --- /dev/null +++ b/crates/net/downloaders/src/snap/bytecode.rs @@ -0,0 +1,362 @@ +//! Downloads contract bytecode and authenticates it by hash. +//! +//! A peer may omit code it does not have, so a response is an ordered subsequence of the +//! requested hashes, as defined by +//! [snap](https://github.com/ethereum/devp2p/blob/master/caps/snap.md#bytecodes-0x05). + +use super::request::{SnapVerifier, VerifyingRequest}; +use alloy_primitives::{keccak256, Bytes, B256}; +use futures::Future; +use reth_eth_wire_types::snap::GetByteCodesMessage; +use reth_network_p2p::{ + error::RequestError, + snap::client::{SnapClient, SnapResponse}, +}; +use reth_network_peers::PeerId; +use reth_tasks::Runtime; +use std::{ + pin::Pin, + task::{Context, Poll}, +}; +use tracing::debug; + +/// Downloads contract bytecode and authenticates each blob against a requested hash. +/// +/// Invalid responses penalize their peer and retry. Hashing runs on the blocking pool. +#[derive(Debug)] +pub struct BytecodeDownloader(VerifyingRequest); + +impl BytecodeDownloader { + /// Submits `request`, rejecting one that asks for no code. + pub fn new( + client: C, + request: GetByteCodesMessage, + runtime: Runtime, + ) -> Result { + if request.hashes.is_empty() { + return Err(InvalidBytecodeRequest::NoHashes) + } + + let verifier = + BytecodeVerifier { request_id: request.request_id, hashes: request.hashes.clone() }; + Ok(Self(VerifyingRequest::new(client, request, verifier, runtime))) + } +} + +impl Future for BytecodeDownloader +where + C: SnapClient + Unpin, +{ + type Output = Result; + + fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + self.get_mut().0.poll_verified(cx) + } +} + +/// Result of an authenticated bytecode request. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum BytecodeOutcome { + /// The peer holds none of the requested code, and was not penalized. + Unavailable { + /// The peer that answered. + peer_id: PeerId, + }, + /// Code authenticated against the requested hashes. + Verified(VerifiedBytecode), +} + +/// Contract code authenticated against the hashes that were requested for it. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct VerifiedBytecode { + // Identifies the peer that omitted any missing code. + peer_id: PeerId, + // Code paired with the requested hash it answers, in requested order. Private so a blob + // cannot be relabelled with a hash that did not authenticate it. + codes: Vec<(B256, Bytes)>, +} + +impl VerifiedBytecode { + /// Peer that returned this code. + pub const fn peer_id(&self) -> PeerId { + self.peer_id + } + + /// Code with the hash it authenticated against, in requested order. + pub fn codes(&self) -> &[(B256, Bytes)] { + &self.codes + } + + /// Consumes the result and returns the authenticated code. + pub fn into_codes(self) -> Vec<(B256, Bytes)> { + self.codes + } + + /// Requested hashes the response did not answer, in requested order. + /// + /// Omissions are not a fault: a peer serves what it has, and cuts the response at its own + /// soft byte limit. These hashes can be asked for again. + pub fn missing<'a>(&'a self, requested: &'a [B256]) -> impl Iterator + 'a { + requested.iter().copied().filter(|hash| !self.codes.iter().any(|(got, _)| got == hash)) + } +} + +/// A bytecode request that can never be answered. +#[derive(Clone, Copy, Debug, PartialEq, Eq, thiserror::Error)] +pub enum InvalidBytecodeRequest { + /// The request asks for no code hashes. + #[error("bytecode request has no code hashes")] + NoHashes, +} + +// Authenticates returned code by hashing it against the requested hashes. +// +// Holds only what blocking verification needs, so nothing else crosses onto the blocking pool. +#[derive(Clone, Debug)] +struct BytecodeVerifier { + // Matches the response to the request that asked for it. + request_id: u64, + // Requested hashes, in the order a response must follow. + hashes: Vec, +} + +impl SnapVerifier for BytecodeVerifier { + type Request = GetByteCodesMessage; + type Output = BytecodeOutcome; + + fn verify(self, peer_id: PeerId, response: SnapResponse) -> Result { + let SnapResponse::ByteCodes(response) = response else { + debug!(target: "downloaders::snap", "Expected byte codes response"); + return Err(RequestError::BadResponse) + }; + if response.request_id != self.request_id { + debug!( + target: "downloaders::snap", + expected = self.request_id, + got = response.request_id, + "Byte codes response id mismatch" + ); + return Err(RequestError::BadResponse) + } + if response.codes.len() > self.hashes.len() { + debug!( + target: "downloaders::snap", + requested = self.hashes.len(), + got = response.codes.len(), + "Byte codes response is longer than the request" + ); + return Err(RequestError::BadResponse) + } + // Serving nothing is a valid answer from a peer that has none of this code. + if response.codes.is_empty() { + return Ok(BytecodeOutcome::Unavailable { peer_id }) + } + + // The cursor only moves forward, so code that repeats, reorders, or was never requested + // finds no hash left to match. + let mut remaining = self.hashes.as_slice(); + let mut codes = Vec::with_capacity(response.codes.len()); + for code in response.codes { + let hash = keccak256(&code); + let Some(offset) = remaining.iter().position(|requested| *requested == hash) else { + debug!(target: "downloaders::snap", %hash, "Unrequested or out-of-order bytecode"); + return Err(RequestError::BadResponse) + }; + remaining = &remaining[offset + 1..]; + codes.push((hash, code)); + } + + Ok(BytecodeOutcome::Verified(VerifiedBytecode { peer_id, codes })) + } +} + +#[cfg(test)] +mod tests { + use super::{ + super::{request::MAX_RETRIES, test_utils::TestSnapClient}, + *, + }; + use reth_eth_wire_types::snap::{AccountRangeMessage, ByteCodesMessage}; + use reth_network_p2p::{error::PeerRequestResult, priority::Priority}; + use reth_network_peers::WithPeerId; + use std::sync::Arc; + + fn code(byte: u8) -> Bytes { + Bytes::from(vec![byte; 4]) + } + + fn request(codes: &[Bytes]) -> GetByteCodesMessage { + GetByteCodesMessage { + request_id: 1, + hashes: codes.iter().map(keccak256).collect(), + response_bytes: 512 * 1024, + } + } + + fn response( + peer: PeerId, + request_id: u64, + codes: Vec, + ) -> PeerRequestResult { + Ok(WithPeerId::new(peer, SnapResponse::ByteCodes(ByteCodesMessage { request_id, codes }))) + } + + // Every attempt gets the same answer, so a rejected response exhausts the retry budget. + fn always( + peer: PeerId, + request_id: u64, + codes: Vec, + ) -> impl Iterator> { + std::iter::repeat_with(move || response(peer, request_id, codes.clone())) + .take(usize::from(MAX_RETRIES) + 1) + } + + fn downloader( + client: Arc, + request: GetByteCodesMessage, + ) -> Result>, InvalidBytecodeRequest> { + BytecodeDownloader::new(client, request, Runtime::test()) + } + + fn verified(outcome: BytecodeOutcome) -> VerifiedBytecode { + match outcome { + BytecodeOutcome::Verified(verified) => verified, + BytecodeOutcome::Unavailable { .. } => panic!("expected verified code"), + } + } + + #[tokio::test] + async fn every_requested_code_is_authenticated_by_its_hash() { + let codes = vec![code(1), code(2), code(3)]; + let client = Arc::new(TestSnapClient::new([response(PeerId::random(), 1, codes.clone())])); + + let outcome = downloader(Arc::clone(&client), request(&codes)).unwrap().await.unwrap(); + + let verified = verified(outcome); + assert_eq!( + verified.codes(), + codes.iter().map(|code| (keccak256(code), code.clone())).collect::>() + ); + assert!(verified.missing(&request(&codes).hashes).next().is_none()); + assert!(client.reported().is_empty()); + } + + #[tokio::test] + async fn an_ordered_subsequence_reports_the_omitted_hashes() { + let codes = vec![code(1), code(2), code(3)]; + let request = request(&codes); + // The peer serves only the first and last of the three. + let served = vec![codes[0].clone(), codes[2].clone()]; + let peer = PeerId::random(); + let client = Arc::new(TestSnapClient::new([response(peer, 1, served)])); + + let outcome = downloader(Arc::clone(&client), request.clone()).unwrap().await.unwrap(); + + let verified = verified(outcome); + assert_eq!(verified.peer_id(), peer); + assert_eq!( + verified.codes().iter().map(|(hash, _)| *hash).collect::>(), + [keccak256(&codes[0]), keccak256(&codes[2])] + ); + assert_eq!(verified.missing(&request.hashes).collect::>(), [keccak256(&codes[1])]); + assert!(client.reported().is_empty()); + } + + #[tokio::test] + async fn an_empty_response_is_unavailable_and_not_a_peer_fault() { + let codes = vec![code(1)]; + let peer = PeerId::random(); + let client = Arc::new(TestSnapClient::new([response(peer, 1, Vec::new())])); + + let outcome = downloader(Arc::clone(&client), request(&codes)).unwrap().await.unwrap(); + + assert_eq!(outcome, BytecodeOutcome::Unavailable { peer_id: peer }); + assert!(client.reported().is_empty()); + } + + #[tokio::test] + async fn unrequested_code_is_reported_and_retried_against_another_peer() { + let codes = vec![code(1)]; + let bad_peer = PeerId::random(); + let client = Arc::new(TestSnapClient::new([ + response(bad_peer, 1, vec![code(9)]), + response(PeerId::random(), 1, codes.clone()), + ])); + + let outcome = downloader(Arc::clone(&client), request(&codes)).unwrap().await.unwrap(); + + assert_eq!(verified(outcome).codes().len(), 1); + assert_eq!(*client.reported(), [bad_peer]); + assert_eq!(*client.priorities(), [Priority::Normal, Priority::High]); + } + + #[tokio::test] + async fn out_of_order_repeated_and_oversized_responses_are_rejected() { + let codes = vec![code(1), code(2)]; + let peer = PeerId::random(); + for served in [ + // reversed, so the second hash is already behind the cursor + vec![codes[1].clone(), codes[0].clone()], + // the same code twice, which only one hash was requested for + vec![codes[0].clone(), codes[0].clone()], + // more blobs than hashes requested + vec![codes[0].clone(), codes[1].clone(), code(3)], + ] { + let client = Arc::new(TestSnapClient::new(always(peer, 1, served))); + + let error = + downloader(Arc::clone(&client), request(&codes)).unwrap().await.unwrap_err(); + + assert_eq!(error, RequestError::BadResponse); + assert_eq!(client.reported().len(), usize::from(MAX_RETRIES) + 1); + } + } + + #[tokio::test] + async fn a_response_for_another_request_id_is_rejected() { + let codes = vec![code(1)]; + let peer = PeerId::random(); + let client = Arc::new(TestSnapClient::new(always(peer, 7, codes.clone()))); + + let error = downloader(Arc::clone(&client), request(&codes)).unwrap().await.unwrap_err(); + + assert_eq!(error, RequestError::BadResponse); + assert_eq!(client.reported().len(), usize::from(MAX_RETRIES) + 1); + } + + #[tokio::test] + async fn a_wrong_response_type_exhausts_the_retry_budget() { + let codes = vec![code(1)]; + let peers = [PeerId::random(), PeerId::random(), PeerId::random()]; + let responses = peers.map(|peer| { + Ok(WithPeerId::new( + peer, + SnapResponse::AccountRange(AccountRangeMessage { + request_id: 1, + accounts: Vec::new(), + proof: Vec::new(), + }), + )) + }); + let client = Arc::new(TestSnapClient::new(responses)); + + let error = downloader(Arc::clone(&client), request(&codes)).unwrap().await.unwrap_err(); + + assert_eq!(error, RequestError::BadResponse); + assert_eq!(*client.reported(), peers); + assert_eq!(*client.priorities(), [Priority::Normal, Priority::High, Priority::High]); + } + + #[test] + fn a_request_without_hashes_is_rejected_before_submission() { + let client = Arc::new(TestSnapClient::new(std::iter::empty())); + let mut empty = request(&[code(1)]); + empty.hashes.clear(); + + assert_eq!( + downloader(Arc::clone(&client), empty).unwrap_err(), + InvalidBytecodeRequest::NoHashes + ); + assert!(client.priorities().is_empty()); + } +} diff --git a/crates/net/downloaders/src/snap/mod.rs b/crates/net/downloaders/src/snap/mod.rs index dfa0d7fc4ca..7466448fb6b 100644 --- a/crates/net/downloaders/src/snap/mod.rs +++ b/crates/net/downloaders/src/snap/mod.rs @@ -23,12 +23,14 @@ use std::{ use tracing::debug; mod block_access_list; +mod bytecode; mod request; mod storage; #[cfg(test)] mod test_utils; pub use block_access_list::*; +pub use bytecode::*; use request::{SnapVerifier, VerifyingRequest}; pub use storage::*; diff --git a/crates/net/downloaders/src/snap/request.rs b/crates/net/downloaders/src/snap/request.rs index 6108a871457..7bb4a09c55d 100644 --- a/crates/net/downloaders/src/snap/request.rs +++ b/crates/net/downloaders/src/snap/request.rs @@ -5,7 +5,8 @@ use futures::FutureExt; use reth_eth_wire_types::snap::{ - GetAccountRangeMessage, GetBlockAccessListsMessage, GetStorageRangesMessage, + GetAccountRangeMessage, GetBlockAccessListsMessage, GetByteCodesMessage, + GetStorageRangesMessage, }; use reth_network_p2p::{ error::RequestError, @@ -167,6 +168,12 @@ impl SnapRequest for GetBlockAccessListsMessage { } } +impl SnapRequest for GetByteCodesMessage { + fn send(&self, client: &C, priority: Priority) -> C::Output { + client.get_byte_codes_with_priority(self.clone(), priority) + } +} + /// Authenticates a snap response against the request that asked for it. pub(super) trait SnapVerifier: Clone + Send + 'static { /// Request type this verifier authenticates responses for. diff --git a/crates/net/downloaders/src/snap/test_utils.rs b/crates/net/downloaders/src/snap/test_utils.rs index 1e134eb5b62..6c352663504 100644 --- a/crates/net/downloaders/src/snap/test_utils.rs +++ b/crates/net/downloaders/src/snap/test_utils.rs @@ -93,9 +93,9 @@ impl SnapClient for TestSnapClient { fn get_byte_codes_with_priority( &self, _request: GetByteCodesMessage, - _priority: Priority, + priority: Priority, ) -> Self::Output { - unsupported() + self.next(priority) } fn get_block_access_lists_with_priority( From 16611a2a1a73b2c6acd5805c3f8c5c0ccb150172 Mon Sep 17 00:00:00 2001 From: figtracer Date: Wed, 2 Sep 2026 12:54:00 +0000 Subject: [PATCH 123/294] feat(rpc): support EIP-8037 tracing (#26804) Co-authored-by: Matthias Seitz --- crates/ethereum/node/tests/e2e/rpc.rs | 109 +++++++++++++++++++++++++- 1 file changed, 108 insertions(+), 1 deletion(-) diff --git a/crates/ethereum/node/tests/e2e/rpc.rs b/crates/ethereum/node/tests/e2e/rpc.rs index 93b27320e30..e0d20896221 100644 --- a/crates/ethereum/node/tests/e2e/rpc.rs +++ b/crates/ethereum/node/tests/e2e/rpc.rs @@ -21,7 +21,9 @@ use alloy_rpc_types_eth::{ state::{AccountOverride, StateOverride}, TransactionRequest, }; -use alloy_rpc_types_trace::geth::{ChainBlockTraceResult, GethDebugTracingOptions}; +use alloy_rpc_types_trace::geth::{ + CallConfig, ChainBlockTraceResult, GethDebugTracingOptions, GethTrace, +}; use jsonrpsee::core::client::{ClientT, Subscription, SubscriptionClientT}; use rand::{rngs::StdRng, Rng, SeedableRng}; use reth_chainspec::{ChainSpecBuilder, EthChainSpec, MAINNET}; @@ -267,6 +269,111 @@ async fn test_debug_trace_chain_subscription() -> eyre::Result<()> { Ok(()) } +#[tokio::test] +async fn test_debug_trace_eip8037_gas() -> eyre::Result<()> { + reth_tracing::init_test_tracing(); + + // The test payload clock starts at 1_710_338_135, so this activates Amsterdam for the second + // block and lets the same node exercise both sides of the fork. + const AMSTERDAM_TIMESTAMP: u64 = 1_710_338_137; + let chain_spec = Arc::new( + ChainSpecBuilder::default() + .chain(MAINNET.chain) + .genesis(serde_json::from_str(include_str!("../assets/genesis.json")).unwrap()) + .osaka_activated() + .with_amsterdam_at(AMSTERDAM_TIMESTAMP) + .build(), + ); + let payload_attributes = |timestamp| { + if timestamp >= AMSTERDAM_TIMESTAMP { + eth_payload_attributes_amsterdam(timestamp) + } else { + eth_payload_attributes(timestamp) + } + }; + let (mut nodes, wallet) = + setup_engine::(1, chain_spec, false, Default::default(), payload_attributes) + .await?; + let mut node = nodes.pop().unwrap(); + let provider = ProviderBuilder::new() + .wallet(EthereumWallet::new(wallet.wallet_gen().swap_remove(0))) + .connect_http(node.rpc_url()); + + let pre_amsterdam = GasWaster::deploy_builder(&provider, U256::from(1)).send().await?; + let pre_amsterdam_hash = *pre_amsterdam.tx_hash(); + node.advance_block().await?; + assert!(pre_amsterdam.get_receipt().await?.status()); + + let GethTrace::Default(frame) = provider + .debug_trace_transaction(pre_amsterdam_hash, GethDebugTracingOptions::default()) + .await? + else { + panic!("expected default trace") + }; + assert_eq!(frame.execution_gas_used, None); + assert_eq!(frame.state_gas_used, None); + assert_eq!(frame.gas_refund, None); + assert!(frame + .struct_logs + .iter() + .all(|log| log.state_gas_cost.is_none() && log.state_gas_reservoir.is_none())); + + let GethTrace::CallTracer(frame) = provider + .debug_trace_transaction( + pre_amsterdam_hash, + GethDebugTracingOptions::call_tracer(CallConfig::default()), + ) + .await? + else { + panic!("expected call trace") + }; + assert_eq!(frame.execution_gas_used, None); + assert_eq!(frame.state_gas_used, None); + assert_eq!(frame.gas_refund, None); + + let amsterdam = GasWaster::deploy_builder(&provider, U256::from(1)).send().await?; + let amsterdam_hash = *amsterdam.tx_hash(); + node.advance_block().await?; + assert!(amsterdam.get_receipt().await?.status()); + + let GethTrace::StateGasTracer(state_gas) = provider + .debug_trace_transaction(amsterdam_hash, GethDebugTracingOptions::state_gas_tracer()) + .await? + else { + panic!("expected state gas trace") + }; + assert!(state_gas.state_gas_used > 0); + + let GethTrace::Default(frame) = provider + .debug_trace_transaction(amsterdam_hash, GethDebugTracingOptions::default()) + .await? + else { + panic!("expected default trace") + }; + assert_eq!(frame.gas, state_gas.gas_used); + assert_eq!(frame.execution_gas_used, Some(state_gas.execution_gas_used)); + assert_eq!(frame.state_gas_used, Some(state_gas.state_gas_used)); + assert_eq!(frame.gas_refund, Some(state_gas.gas_refund)); + assert!(frame.struct_logs.iter().any(|log| log.state_gas_cost.is_some())); + assert!(frame.struct_logs.iter().all(|log| log.state_gas_reservoir.is_some())); + + let GethTrace::CallTracer(frame) = provider + .debug_trace_transaction( + amsterdam_hash, + GethDebugTracingOptions::call_tracer(CallConfig::default()), + ) + .await? + else { + panic!("expected call trace") + }; + assert_eq!(frame.gas_used, U256::from(state_gas.gas_used)); + assert_eq!(frame.execution_gas_used, Some(U256::from(state_gas.execution_gas_used))); + assert_eq!(frame.state_gas_used, Some(U256::from(state_gas.state_gas_used))); + assert_eq!(frame.gas_refund, Some(U256::from(state_gas.gas_refund))); + + Ok(()) +} + #[tokio::test] async fn test_flashbots_validate_v3() -> eyre::Result<()> { reth_tracing::init_test_tracing(); From 68877d5fd7d146ef070bbe7dbabbe1097ea1da62 Mon Sep 17 00:00:00 2001 From: Brian Picciano Date: Wed, 2 Sep 2026 14:45:26 +0000 Subject: [PATCH 124/294] fix(overlay): snapshot parent blocks for lazy providers (#26917) --- crates/engine/tree/src/tree/mod.rs | 133 +++------------- crates/engine/tree/src/tree/tests.rs | 137 +---------------- crates/storage/storage-overlay/src/builder.rs | 105 +++++++------ crates/storage/storage-overlay/src/manager.rs | 145 ++++++++++++------ .../storage/storage-overlay/src/provider.rs | 26 ++++ 5 files changed, 211 insertions(+), 335 deletions(-) diff --git a/crates/engine/tree/src/tree/mod.rs b/crates/engine/tree/src/tree/mod.rs index 35fd1da056c..3a8b77046f8 100644 --- a/crates/engine/tree/src/tree/mod.rs +++ b/crates/engine/tree/src/tree/mod.rs @@ -330,9 +330,6 @@ where payload_builds: PayloadBuildTracker, /// Notifies the engine when the final active payload job finishes. payload_build_finished: Receiver<()>, - /// A durable persistence completion whose destructive in-memory handoff is deferred until - /// payload jobs finish. - pending_persisted_handoff: Option, /// Task runtime for spawning blocking work on named, reusable threads. runtime: reth_tasks::Runtime, } @@ -361,7 +358,6 @@ where .field("evm_config", &self.evm_config) .field("execution_timing_stats", &self.execution_timing_stats.len()) .field("payload_builds_active", &self.payload_builds.is_active()) - .field("pending_persisted_handoff", &self.pending_persisted_handoff) .field("runtime", &self.runtime) .finish() } @@ -429,7 +425,6 @@ where execution_timing_stats: B256Map::default(), payload_builds, payload_build_finished, - pending_persisted_handoff: None, runtime, } } @@ -606,12 +601,7 @@ where return } } - LoopEvent::PayloadBuildFinished => { - if let Err(err) = self.on_payload_build_finished() { - error!(target: "engine::tree", %err, "Payload build completion handling failed"); - return - } - } + LoopEvent::PayloadBuildFinished => {} LoopEvent::Disconnected => { error!(target: "engine::tree", "Channel disconnected"); return @@ -647,28 +637,11 @@ where } } - /// Blocks until the next event that can safely be processed is ready. + /// Blocks until the next event is ready. /// - /// A pending persisted handoff keeps processing engine messages while prioritizing the - /// notification that all active payload builds have finished. This keeps the engine responsive - /// without reclaiming the in-memory overlay while a payload job may still access it. Otherwise, - /// uses biased selection to prioritize persistence completion to update in-memory state and - /// unblock further writes. + /// Uses biased selection to prioritize persistence completion so in-memory state is updated and + /// further writes are unblocked. fn wait_for_event(&mut self) -> LoopEvent { - if self.pending_persisted_handoff.is_some() { - self.metrics.engine.backpressure_active.set(0.0); - return crossbeam_channel::select_biased! { - recv(self.payload_build_finished) -> result => match result { - Ok(()) => LoopEvent::PayloadBuildFinished, - Err(_) => LoopEvent::Disconnected, - }, - recv(self.incoming) -> msg => match msg { - Ok(m) => LoopEvent::EngineMessage(m), - Err(_) => LoopEvent::Disconnected, - }, - } - } - // Take ownership of persistence rx if present let maybe_persistence = self.persistence_state.rx.take(); @@ -1466,12 +1439,6 @@ where /// This checks if we need to remove blocks (disk reorg) or save new blocks to disk. /// Persistence completion is handled separately via the `wait_for_event` method. fn advance_persistence(&mut self) -> Result<(), AdvancePersistenceError> { - // A second persistence completion would overwrite the frontier needed by the deferred - // handoff. Wait until the payload jobs using the old overlay have finished. - if self.pending_persisted_handoff.is_some() { - return Ok(()) - } - if !self.persistence_state.in_progress() { if let Some(new_tip_num) = self.find_disk_reorg()? { self.remove_blocks(new_tip_num) @@ -1504,7 +1471,7 @@ where if let Some((rx, start_time, action)) = self.persistence_state.rx.take() { debug!(target: "engine::tree", ?action, "waiting for in-flight persistence"); let result = rx.recv().map_err(|_| AdvancePersistenceError::ChannelClosed)?; - self.finish_persistence(result, start_time); + self.on_persistence_complete(result, start_time)?; continue } @@ -1556,31 +1523,9 @@ where result: PersistenceResult, start_time: Instant, ) -> Result<(), AdvancePersistenceError> { - let handoff = self.finish_persistence(result, start_time); - - if self.payload_builds.is_active() { - debug_assert!( - self.pending_persisted_handoff.is_none(), - "a new persistence task must not start while its predecessor handoff is pending" - ); - debug!(target: "engine::tree", "Deferring persisted in-memory handoff until payload jobs finish"); - self.pending_persisted_handoff = Some(handoff); - return Ok(()) - } - - self.on_persisted_handoff(handoff) - } - - /// Records a completed persistence operation and returns its deferred in-memory handoff. - fn finish_persistence( - &mut self, - result: PersistenceResult, - start_time: Instant, - ) -> PersistenceResult { self.metrics.engine.persistence_duration.record(start_time.elapsed()); - let last_block = result.last_block; - let last_state_trie_block = result.last_state_trie_block; + let PersistenceResult { last_block, last_state_trie_block, commit_duration } = result; debug_assert!( last_state_trie_block.number <= last_block.number, "state/trie frontier cannot exceed the last persisted block" @@ -1589,28 +1534,6 @@ where debug!(target: "engine::tree", ?last_block, ?last_state_trie_block, elapsed=?start_time.elapsed(), "Finished persisting, calling finish"); self.persistence_state.finish(last_block, last_state_trie_block); - result - } - - /// Applies the destructive in-memory portion of a completed persistence operation. - fn on_persisted_handoff( - &mut self, - handoff: PersistenceResult, - ) -> Result<(), AdvancePersistenceError> { - if handoff.last_block != self.persistence_state.last_persisted_block || - handoff.last_state_trie_block != - self.persistence_state.last_state_trie_persisted_block - { - debug!( - target: "engine::tree", - handoff_last_block = ?handoff.last_block, - current_last_block = ?self.persistence_state.last_persisted_block, - "Discarding stale persisted handoff" - ); - return Ok(()) - } - - let PersistenceResult { last_block, commit_duration, .. } = handoff; let last_block_number = last_block.number; // Evict cached changesets for blocks below the eviction threshold. @@ -1641,19 +1564,6 @@ where Ok(()) } - /// Releases work deferred until all payload jobs have finished. - fn on_payload_build_finished(&mut self) -> Result<(), AdvancePersistenceError> { - if self.payload_builds.is_active() { - return Ok(()) - } - - if let Some(handoff) = self.pending_persisted_handoff.take() { - self.on_persisted_handoff(handoff)?; - } - - Ok(()) - } - /// Handles a message from the engine. /// /// Returns `ControlFlow::Break(())` if the engine should terminate. @@ -1704,16 +1614,15 @@ where } }; - // if the parent is the canonical head, we can insert the block as the - // pending block - if self.state.tree_state.canonical_block_hash() == - block.recovered_block().parent_hash() - { + let is_pending = self.state.tree_state.canonical_block_hash() == + block.recovered_block().parent_hash(); + self.state.tree_state.insert_executed(block.clone()); + + if is_pending { debug!(target: "engine::tree", pending=?block_num_hash, "updating pending block"); self.canonical_in_memory_state.set_pending_block(block.clone()); } - self.state.tree_state.insert_executed(block.clone()); self.metrics.engine.inserted_already_executed_blocks.increment(1); self.emit_event(EngineApiEvent::BeaconConsensus( ConsensusEngineEvent::CanonicalBlockAdded(block, now.elapsed()), @@ -2174,12 +2083,9 @@ where "backfill action should only be emitted when backfill is idle" ); - if self.payload_builds.is_active() || - self.persistence_state.in_progress() || - self.pending_persisted_handoff.is_some() - { - // Backfill can remove the same in-memory blocks as an active payload job or a - // persisted handoff, so it must not start until the next sync trigger. + if self.payload_builds.is_active() || self.persistence_state.in_progress() { + // Backfill can remove the same in-memory blocks as an active payload job or + // persistence task, so it must not start until the next sync trigger. debug!(target: "engine::tree", "skipping backfill while in-memory overlay is in use"); return } @@ -3227,14 +3133,15 @@ where self.execution_timing_stats.insert(executed.recovered_block().hash(), stats); } - // if the parent is the canonical head, we can insert the block as the pending block - if self.state.tree_state.canonical_block_hash() == executed.recovered_block().parent_hash() - { + let is_pending = self.state.tree_state.canonical_block_hash() == + executed.recovered_block().parent_hash(); + self.state.tree_state.insert_executed(executed.clone()); + + if is_pending { debug!(target: "engine::tree", pending=?block_num_hash, "updating pending block"); self.canonical_in_memory_state.set_pending_block(executed.clone()); } - self.state.tree_state.insert_executed(executed.clone()); self.metrics.engine.executed_blocks.set(self.state.tree_state.block_count() as f64); // emit insert event @@ -3559,7 +3466,7 @@ where /// When the persistence operation started. start_time: Instant, }, - /// The last active payload job has finished, so deferred in-memory work may proceed. + /// The last active payload job has finished, so suppressed persistence may resume. PayloadBuildFinished, /// A channel was disconnected. Disconnected, diff --git a/crates/engine/tree/src/tree/tests.rs b/crates/engine/tree/src/tree/tests.rs index f23a55a6bf4..96550260468 100644 --- a/crates/engine/tree/src/tree/tests.rs +++ b/crates/engine/tree/src/tree/tests.rs @@ -683,7 +683,7 @@ fn payload_build_tracker_notifies_after_last_lease_drops() { } #[test] -fn persistence_handoff_waits_for_active_payload_jobs() { +fn persistence_completion_does_not_wait_for_active_payload_jobs() { let config = TreeConfig::default() .with_has_enough_parallelism(true) .with_persistence_threshold(0) @@ -693,8 +693,7 @@ fn persistence_handoff_waits_for_active_payload_jobs() { TestHarness::with_config(MAINNET.clone(), config).with_blocks(blocks.clone()); let persisted = blocks[0].recovered_block().num_hash(); let persisted_hash = persisted.hash; - let first = test_harness.tree.payload_builds.acquire(); - let second = test_harness.tree.payload_builds.acquire(); + let _payload_build = test_harness.tree.payload_builds.acquire(); test_harness .tree @@ -708,104 +707,13 @@ fn persistence_handoff_waits_for_active_payload_jobs() { ) .unwrap(); - // Durability advances immediately, but the in-memory overlay remains available to jobs. + // Payload jobs own an overlay snapshot, so persistence can immediately reclaim the manager's + // copy. assert_eq!(test_harness.tree.persistence_state.last_persisted_block, persisted); - assert!(test_harness.tree.pending_persisted_handoff.is_some()); - assert!(test_harness.tree.state.tree_state.contains_hash(&persisted_hash)); - assert!(test_harness.tree.canonical_in_memory_state.state_by_hash(persisted_hash).is_some()); - - // Do not let another persistence action overwrite the deferred handoff's frontier. - test_harness.tree.advance_persistence().unwrap(); - assert!(!test_harness.tree.persistence_state.in_progress()); - - drop(first); - assert!(test_harness.tree.payload_build_finished.try_recv().is_err()); - assert!(test_harness.tree.pending_persisted_handoff.is_some()); - - drop(second); - assert!(matches!(test_harness.tree.wait_for_event(), super::LoopEvent::PayloadBuildFinished)); - test_harness.tree.on_payload_build_finished().unwrap(); - - assert!(test_harness.tree.pending_persisted_handoff.is_none()); assert!(!test_harness.tree.state.tree_state.contains_hash(&persisted_hash)); assert!(test_harness.tree.canonical_in_memory_state.state_by_hash(persisted_hash).is_none()); } -#[test] -fn pending_persisted_handoff_keeps_engine_messages_responsive() { - let blocks: Vec<_> = TestBlockBuilder::eth().get_executed_blocks(1..4).collect(); - let mut test_harness = TestHarness::with_config( - MAINNET.clone(), - TreeConfig::default().with_has_enough_parallelism(true), - ) - .with_blocks(blocks.clone()); - let persisted = blocks[0].recovered_block().num_hash(); - let payload_build = test_harness.tree.payload_builds.acquire(); - - test_harness - .tree - .on_persistence_complete( - PersistenceResult { - last_block: persisted, - last_state_trie_block: persisted, - commit_duration: Some(Duration::ZERO), - }, - Instant::now(), - ) - .unwrap(); - assert!(test_harness.tree.pending_persisted_handoff.is_some()); - - // Engine messages remain processable while the handoff waits for the active payload build. - test_harness.to_tree_tx.send(FromEngine::Event(FromOrchestrator::BackfillSyncStarted)).unwrap(); - assert_matches!( - test_harness.tree.wait_for_event(), - super::LoopEvent::EngineMessage(FromEngine::Event(FromOrchestrator::BackfillSyncStarted)) - ); - assert!(test_harness.tree.pending_persisted_handoff.is_some()); - assert!(test_harness.tree.state.tree_state.contains_hash(&persisted.hash)); - - // Once the final lease drops, its completion is prioritized over queued engine messages so the - // handoff gets the first opportunity to reclaim the overlay. - test_harness.to_tree_tx.send(FromEngine::Event(FromOrchestrator::BackfillSyncStarted)).unwrap(); - drop(payload_build); - assert_matches!(test_harness.tree.wait_for_event(), super::LoopEvent::PayloadBuildFinished); - test_harness.tree.on_payload_build_finished().unwrap(); - assert!(test_harness.tree.pending_persisted_handoff.is_none()); - assert!(!test_harness.tree.state.tree_state.contains_hash(&persisted.hash)); - assert_matches!( - test_harness.tree.wait_for_event(), - super::LoopEvent::EngineMessage(FromEngine::Event(FromOrchestrator::BackfillSyncStarted)) - ); -} - -#[test] -fn persist_until_complete_updates_frontiers_without_in_memory_handoff() { - let blocks: Vec<_> = TestBlockBuilder::eth().get_executed_blocks(1..4).collect(); - let mut test_harness = TestHarness::new(MAINNET.clone()).with_blocks(blocks.clone()); - let persisted = blocks.last().unwrap().recovered_block().num_hash(); - let persisted_hash = persisted.hash; - let (tx, rx) = crossbeam_channel::bounded(1); - - tx.send(PersistenceResult { - last_block: persisted, - last_state_trie_block: persisted, - commit_duration: Some(Duration::ZERO), - }) - .unwrap(); - test_harness.tree.persistence_state.start_save(persisted, rx); - - test_harness.tree.persist_until_complete().unwrap(); - - // Shutdown waits for the durable write, but process teardown does not need the destructive - // in-memory handoff that normally follows a completed persistence task. - assert_eq!(test_harness.tree.persistence_state.last_persisted_block, persisted); - assert_eq!(test_harness.tree.persistence_state.last_state_trie_persisted_block, persisted); - assert!(!test_harness.tree.persistence_state.in_progress()); - assert!(test_harness.tree.pending_persisted_handoff.is_none()); - assert!(test_harness.tree.state.tree_state.contains_hash(&persisted_hash)); - assert!(test_harness.tree.canonical_in_memory_state.state_by_hash(persisted_hash).is_some()); -} - #[test] fn backfill_action_skips_while_payload_build_is_active() { let mut test_harness = TestHarness::new(MAINNET.clone()); @@ -813,54 +721,17 @@ fn backfill_action_skips_while_payload_build_is_active() { let action = BackfillAction::Start(B256::random().into()); test_harness.tree.emit_event(EngineApiEvent::BackfillAction(action)); - assert!(test_harness.tree.pending_persisted_handoff.is_none()); assert!(test_harness.tree.backfill_sync_state.is_idle()); assert!(test_harness.from_tree_rx.try_recv().is_err()); drop(payload_build); assert!(matches!(test_harness.tree.wait_for_event(), super::LoopEvent::PayloadBuildFinished)); - test_harness.tree.on_payload_build_finished().unwrap(); // The skipped action is not queued for a later replay. assert!(test_harness.tree.backfill_sync_state.is_idle()); assert!(test_harness.from_tree_rx.try_recv().is_err()); } -#[test] -fn backfill_action_skips_while_persisted_handoff_is_pending() { - let blocks: Vec<_> = TestBlockBuilder::eth().get_executed_blocks(1..4).collect(); - let mut test_harness = TestHarness::with_config( - MAINNET.clone(), - TreeConfig::default().with_has_enough_parallelism(true), - ) - .with_blocks(blocks.clone()); - let persisted = blocks[0].recovered_block().num_hash(); - let payload_build = test_harness.tree.payload_builds.acquire(); - - test_harness - .tree - .on_persistence_complete( - PersistenceResult { - last_block: persisted, - last_state_trie_block: persisted, - commit_duration: Some(Duration::ZERO), - }, - Instant::now(), - ) - .unwrap(); - - drop(payload_build); - assert!(!test_harness.tree.payload_builds.is_active()); - assert!(test_harness.tree.pending_persisted_handoff.is_some()); - - test_harness - .tree - .emit_event(EngineApiEvent::BackfillAction(BackfillAction::Start(B256::random().into()))); - - assert!(test_harness.tree.backfill_sync_state.is_idle()); - assert!(test_harness.from_tree_rx.try_recv().is_err()); -} - #[test] fn configured_persistence_suppression_tracks_payload_job_lifetime() { let config = TreeConfig::default() diff --git a/crates/storage/storage-overlay/src/builder.rs b/crates/storage/storage-overlay/src/builder.rs index 7a834cc56ea..c1ae9e60d71 100644 --- a/crates/storage/storage-overlay/src/builder.rs +++ b/crates/storage/storage-overlay/src/builder.rs @@ -5,7 +5,7 @@ use alloy_primitives::{ Address, BlockHash, BlockNumber, B256, U256, }; use metrics::{Counter, Histogram}; -use reth_chain_state::ExecutedBlock; +use reth_chain_state::{BlockState, ExecutedBlock}; use reth_errors::{ProviderError, ProviderResult}; use reth_ethereum_primitives::EthPrimitives; use reth_metrics::Metrics; @@ -236,8 +236,10 @@ pub struct OverlayBuilder { parent_hash: B256, /// Optional overlay source. overlay_source: Option, - /// Manager used for cached changesets and in-memory parent state. + /// Manager used for cached changesets and overlays. overlay_manager: OverlayManager, + /// Snapshot of the in-memory chain ending at the requested parent. + parent_state: Option>, /// Anchor hash of the reused sparse trie, if this task reused one. reused_sparse_trie_anchor_hash: Option, /// Whether building the overlay may query revert changesets. @@ -248,11 +250,16 @@ pub struct OverlayBuilder { impl OverlayBuilder { /// Create a new manager-backed overlay builder. - pub(crate) fn new(parent_hash: B256, overlay_manager: OverlayManager) -> Self { + pub(crate) fn new( + parent_hash: B256, + parent_state: Option>, + overlay_manager: OverlayManager, + ) -> Self { Self { parent_hash, overlay_source: Some(OverlaySource::Managed), overlay_manager, + parent_state, reused_sparse_trie_anchor_hash: None, no_reverts: false, metrics: OverlayBuilderMetrics::default(), @@ -291,10 +298,7 @@ impl OverlayBuilder { } /// Returns the durable anchor to use for this builder's parent. - pub fn anchor_at_parent( - &self, - provider: &Provider, - ) -> ProviderResult> + pub fn anchor_at_parent(&self, provider: &Provider) -> ProviderResult where Provider: StageCheckpointReader + BlockNumReader + PruneCheckpointReader, { @@ -308,21 +312,21 @@ impl OverlayBuilder { provider: &Provider, partial_state_trie: BlockNumHash, finish: BlockNumHash, - ) -> ProviderResult> + ) -> ProviderResult where Provider: BlockNumReader + PruneCheckpointReader, { match &self.overlay_source { Some(OverlaySource::Managed) => anchor_for_parent_with_frontiers( self.parent_hash, - self.overlay_manager.parent_chain(self.parent_hash), + self.parent_state.iter().flat_map(|state| state.chain()).map(BlockState::block), partial_state_trie, finish, provider, ), _ => anchor_for_parent_with_frontiers( self.parent_hash, - std::iter::empty(), + std::iter::empty::>(), partial_state_trie, finish, provider, @@ -456,7 +460,7 @@ impl OverlayBuilder { (trie_updates, hashed_state_updates) } - AnchorForParent::NoReverts { anchor, .. } => { + AnchorForParent::NoReverts { anchor } => { // If no reverts are needed, use the manager overlay directly unless the reused // sparse trie already covers both durable frontiers through the // requested parent. @@ -564,8 +568,13 @@ impl OverlayBuilder { Arc::new(HashedPostStateSorted::default()), )) } else { + let parent_state = self.parent_state.as_ref().ok_or_else(|| { + ProviderError::other(std::io::Error::other( + "state trie overlay cannot be anchored without in-memory parent state", + )) + })?; self.overlay_manager - .overlay_for_parent(self.parent_hash, anchor_hash) + .overlay_for_parent(parent_state, anchor_hash) .map_err(ProviderError::other) } } @@ -591,10 +600,14 @@ impl OverlayBuilder { anchor_hash: BlockHash, ) -> ProviderResult> { match &self.overlay_source { - Some(OverlaySource::Managed) if anchor_hash != self.parent_hash => self - .overlay_manager - .execution_overlay_for_parent(self.parent_hash, anchor_hash) - .map_err(ProviderError::other), + Some(OverlaySource::Managed) if anchor_hash != self.parent_hash => { + let parent_state = self.parent_state.as_ref().ok_or_else(|| { + ProviderError::other(std::io::Error::other("missing in-memory parent state")) + })?; + self.overlay_manager + .execution_overlay_for_block_state(parent_state, anchor_hash) + .map_err(ProviderError::other) + } Some(OverlaySource::Managed) | None => Ok(Arc::new(ExecutionOverlay::default())), Some(OverlaySource::Immediate { .. }) => Err(ProviderError::other( std::io::Error::other("immediate state trie overlay has no execution state"), @@ -605,7 +618,7 @@ impl OverlayBuilder { /// Returns the blocks to revert from Finish to the selected anchor, if any. fn revert_blocks( &self, - anchor_for_parent: &AnchorForParent, + anchor_for_parent: &AnchorForParent, ) -> ProviderResult>> { match anchor_for_parent { AnchorForParent::NoReverts { .. } => Ok(None), @@ -631,19 +644,29 @@ impl OverlayBuilder { match &self.overlay_source { Some(OverlaySource::Managed) => { - self.overlay_manager.contains_hash( - self.parent_hash, - anchor_hash, - state_trie_tip_hash, - ) && self.overlay_manager.contains_hash( - self.parent_hash, - anchor_hash, - finish_tip_hash, - ) + self.contains_hash(anchor_hash, state_trie_tip_hash) && + self.contains_hash(anchor_hash, finish_tip_hash) } _ => false, } } + + fn contains_hash(&self, anchor_hash: B256, hash: B256) -> bool { + let mut current_hash = self.parent_hash; + let mut blocks = self.parent_state.iter().flat_map(|state| state.chain()); + + loop { + if current_hash == hash { + return true + } + if current_hash == anchor_hash { + return false + } + + let Some(block) = blocks.next() else { return false }; + current_hash = block.block_ref().recovered_block().parent_hash(); + } + } } /// Returns the highest blocks whose state/trie data and non-state/trie data are durably @@ -716,13 +739,11 @@ fn anchor_for_parent_in( /// Describes whether an overlay must revert the database before using its anchor. #[derive(Debug)] -pub enum AnchorForParent { +pub enum AnchorForParent { /// The in-memory chain covers the durable frontiers through this anchor. NoReverts { /// Block to anchor the overlay to. anchor: BlockNumHash, - /// In-memory blocks from `parent_hash` through, but excluding, `anchor`. - overlay: Vec>, }, /// The database must be reverted from `finish` to `anchor` first. RevertsRequired { @@ -730,12 +751,10 @@ pub enum AnchorForParent { anchor: BlockNumHash, /// Current Finish frontier. finish: BlockNumHash, - /// In-memory blocks from `parent_hash` through, but excluding, `anchor`. - overlay: Vec>, }, } -impl AnchorForParent { +impl AnchorForParent { /// Returns the durable block anchoring this overlay. pub const fn anchor(&self) -> BlockNumHash { match self { @@ -754,7 +773,7 @@ pub fn anchor_for_parent( parent_hash: B256, in_mem_chain: impl Iterator>, provider: &Provider, -) -> ProviderResult> +) -> ProviderResult where N: NodePrimitives, Provider: StageCheckpointReader + BlockNumReader + PruneCheckpointReader, @@ -784,7 +803,7 @@ pub fn anchor_for_parent_with_frontiers( partial_state_trie: BlockNumHash, finish: BlockNumHash, provider: &Provider, -) -> ProviderResult> +) -> ProviderResult where N: NodePrimitives, Provider: BlockNumReader + PruneCheckpointReader, @@ -796,26 +815,23 @@ where .filter(|&parent_number| parent_number <= partial_state_trie.number); let mut finish_seen = parent_hash == finish.hash; - let (anchor, overlay) = if let Some(parent_number) = persisted_parent { - (BlockNumHash::new(parent_number, parent_hash), Vec::new()) + let anchor = if let Some(parent_number) = persisted_parent { + BlockNumHash::new(parent_number, parent_hash) } else { - let mut overlay = Vec::new(); let mut in_mem_chain = in_mem_chain.inspect(|block| { finish_seen |= block.recovered_block().hash() == finish.hash; - overlay.push(block.clone()); }); let anchor_hash = anchor_for_parent_in(parent_hash, &mut in_mem_chain, partial_state_trie.hash); - let anchor = if anchor_hash == partial_state_trie.hash { + if anchor_hash == partial_state_trie.hash { BlockNumHash::new(partial_state_trie.number, anchor_hash) } else { let anchor_number = provider .convert_hash_or_number(anchor_hash.into())? .ok_or(ProviderError::BlockHashNotFound(anchor_hash))?; BlockNumHash::new(anchor_number, anchor_hash) - }; - (anchor, overlay) + } }; finish_seen |= anchor.hash == finish.hash; @@ -837,7 +853,7 @@ where // be sure that the in-memory chain is a superset of partial_state_trie+1..finish, and therefore // can be used without reverts. if finish_seen { - return Ok(AnchorForParent::NoReverts { anchor, overlay }) + return Ok(AnchorForParent::NoReverts { anchor }) } // Otherwise reverts are required; we check the changesets to make sure they are actually @@ -857,7 +873,7 @@ where }) } - Ok(AnchorForParent::RevertsRequired { anchor, finish, overlay }) + Ok(AnchorForParent::RevertsRequired { anchor, finish }) } #[cfg(test)] @@ -1323,10 +1339,9 @@ mod tests { let provider = factory.provider().unwrap(); let builder = manager.overlay_builder(blocks[1].recovered_block().hash()); match builder.anchor_at_parent(&provider).unwrap() { - AnchorForParent::RevertsRequired { anchor, finish, overlay } => { + AnchorForParent::RevertsRequired { anchor, finish } => { assert_eq!(anchor, blocks[1].recovered_block().num_hash()); assert_eq!(finish, blocks[3].recovered_block().num_hash()); - assert!(overlay.is_empty()); } AnchorForParent::NoReverts { .. } => { panic!("persisted parent below Finish must require reverts") diff --git a/crates/storage/storage-overlay/src/manager.rs b/crates/storage/storage-overlay/src/manager.rs index 7afbebc9d39..1db1ea8db10 100644 --- a/crates/storage/storage-overlay/src/manager.rs +++ b/crates/storage/storage-overlay/src/manager.rs @@ -13,7 +13,7 @@ use crate::{ use alloy_eips::BlockNumHash; use alloy_primitives::{BlockNumber, B256}; use parking_lot::Mutex; -use reth_chain_state::{ExecutedBlock, PreservedSparseTrie}; +use reth_chain_state::{BlockState, ExecutedBlock, PreservedSparseTrie}; use reth_errors::ProviderResult; use reth_ethereum_primitives::EthPrimitives; use reth_primitives_traits::{ @@ -96,7 +96,16 @@ impl OverlayManager { /// Creates an overlay builder for `parent_hash`. pub fn overlay_builder(&self, parent_hash: B256) -> OverlayBuilder { - OverlayBuilder::new(parent_hash, self.clone()) + OverlayBuilder::new(parent_hash, self.block_state(parent_hash), self.clone()) + } + + fn block_state(&self, parent_hash: B256) -> Option> { + let mut blocks = self.parent_chain(parent_hash).collect::>(); + blocks.pop().map(|oldest| { + blocks.into_iter().rev().fold(BlockState::new(oldest), |parent, block| { + BlockState::with_parent(block, Some(Arc::new(parent))) + }) + }) } pub(crate) const fn changeset_cache(&self) -> &ChangesetCache { @@ -340,13 +349,20 @@ impl OverlayManager { level = "trace", target = "storage::overlay::manager", skip_all, - fields(tip_hash = %parent_hash, anchor_hash = %anchor_hash) + fields(tip_hash = %parent_state.hash(), anchor_hash = %anchor_hash) )] pub(crate) fn overlay_for_parent( &self, - parent_hash: B256, + parent_state: &BlockState, anchor_hash: B256, ) -> Result<(Arc, Arc), StateTrieOverlayError> { + let parent_hash = parent_state.hash(); + if parent_hash == anchor_hash { + return Ok(( + Arc::new(TrieUpdatesSorted::default()), + Arc::new(HashedPostStateSorted::default()), + )) + } debug!( target: "storage::overlay::manager", tip_hash = %parent_hash, @@ -357,8 +373,8 @@ impl OverlayManager { .get_or_compute_overlay( &self.state_trie_overlays, &self.metrics, - parent_hash, anchor_hash, + parent_state, true, |input, span| self.compute_state_trie_overlay(input, anchor_hash, span), )? @@ -371,15 +387,15 @@ impl OverlayManager { level = "trace", target = "storage::overlay::manager", skip_all, - fields(tip_hash = %parent_hash, anchor_hash = %anchor_hash) + fields(tip_hash = %parent_state.hash(), anchor_hash = %anchor_hash) )] - pub fn execution_overlay_for_parent( + pub(crate) fn execution_overlay_for_block_state( &self, - parent_hash: B256, + parent_state: &BlockState, anchor_hash: B256, ) -> Result, StateTrieOverlayError> { Ok(self - .execution_overlay_for_parent_inner(parent_hash, anchor_hash, true)? + .execution_overlay_for_parent_inner(parent_state, anchor_hash, true)? .expect("required overlay lookup cannot skip an in-progress computation")) } @@ -389,15 +405,19 @@ impl OverlayManager { parent_hash: B256, anchor_hash: B256, ) -> Result<(), StateTrieOverlayError> { - self.execution_overlay_for_parent_inner(parent_hash, anchor_hash, false).map(drop) + let parent_state = self + .block_state(parent_hash) + .ok_or(StateTrieOverlayError { tip_hash: parent_hash, anchor_hash })?; + self.execution_overlay_for_parent_inner(&parent_state, anchor_hash, false).map(drop) } fn execution_overlay_for_parent_inner( &self, - parent_hash: B256, + parent_state: &BlockState, anchor_hash: B256, wait_for_pending: bool, ) -> Result>, StateTrieOverlayError> { + let parent_hash = parent_state.hash(); if parent_hash == anchor_hash { return Ok(Some(Arc::new(ExecutionOverlay::default()))) } @@ -405,8 +425,8 @@ impl OverlayManager { self.get_or_compute_overlay( &self.execution_overlays, &self.execution_metrics, - parent_hash, anchor_hash, + parent_state, wait_for_pending, |input, span| self.compute_execution_overlay(input, anchor_hash, span), ) @@ -417,7 +437,7 @@ impl OverlayManager { target = "storage::overlay::manager", skip_all, fields( - tip_hash = %tip_hash, + tip_hash = %parent_state.hash(), anchor_hash = %anchor_hash, cache_reused = tracing::field::Empty, block_count = tracing::field::Empty, @@ -428,17 +448,17 @@ impl OverlayManager { &self, cache: &OverlayCache, metrics: &M, - tip_hash: B256, anchor_hash: B256, + parent_state: &BlockState, wait_for_pending: bool, compute: impl FnOnce(ComputeOverlayInput, tracing::Span) -> T, ) -> Result>, StateTrieOverlayError> where M: OverlayCacheMetrics, { + let tip_hash = parent_state.hash(); let key = OverlayCacheKey { anchor_hash, tip_hash }; let span = tracing::Span::current(); - if let Some(entry) = cache.entries.get(&key).map(|entry| entry.value().clone()) { metrics.record_cache_reuse(); span.record("cache_reused", true); @@ -451,19 +471,7 @@ impl OverlayManager { span.record("cache_reused", false); // Resolve the block path and any cached parent overlay before locking the child entry. - let mut hash = tip_hash; - let mut blocks = Vec::new(); - loop { - let block = - self.blocks.get(&hash).ok_or(StateTrieOverlayError { tip_hash, anchor_hash })?; - let parent_hash = block.recovered_block().parent_hash(); - blocks.push(block.clone()); - - if parent_hash == anchor_hash { - break - } - hash = parent_hash; - } + let mut blocks = Self::blocks_from_parent_state(parent_state, anchor_hash)?; span.record("block_count", blocks.len()); enum CacheAction { Ready(Arc), @@ -531,6 +539,27 @@ impl OverlayManager { } } + fn blocks_from_parent_state( + parent_state: &BlockState, + anchor_hash: B256, + ) -> Result>, StateTrieOverlayError> { + let tip_hash = parent_state.hash(); + let mut hash = tip_hash; + let mut blocks = Vec::new(); + for state in parent_state.chain() { + let block = state.block(); + if block.recovered_block().hash() != hash { + return Err(StateTrieOverlayError { tip_hash, anchor_hash }) + } + hash = block.recovered_block().parent_hash(); + blocks.push(block); + if hash == anchor_hash { + return Ok(blocks) + } + } + Err(StateTrieOverlayError { tip_hash, anchor_hash }) + } + /// Returns every in-memory block in the chain whose tip is `parent_hash`. pub(crate) fn parent_chain( &self, @@ -546,7 +575,7 @@ impl OverlayManager { /// Returns true if `hash` is in the parent chain segment from `anchor_hash` inclusive to /// `parent_hash` inclusive. - pub fn contains_hash(&self, parent_hash: B256, anchor_hash: B256, hash: B256) -> bool { + fn contains_hash(&self, parent_hash: B256, anchor_hash: B256, hash: B256) -> bool { let mut current_hash = parent_hash; loop { @@ -942,13 +971,40 @@ mod tests { .collect() } + impl OverlayManager { + fn execution_overlay_for_parent( + &self, + parent_hash: B256, + anchor_hash: B256, + ) -> Result, StateTrieOverlayError> { + if parent_hash == anchor_hash { + return Ok(Arc::new(ExecutionOverlay::default())) + } + let parent_state = self + .block_state(parent_hash) + .ok_or(StateTrieOverlayError { tip_hash: parent_hash, anchor_hash })?; + self.execution_overlay_for_block_state(&parent_state, anchor_hash) + } + } + + fn overlay_for_parent( + manager: &OverlayManager, + parent_hash: B256, + anchor_hash: B256, + ) -> Result<(Arc, Arc), StateTrieOverlayError> { + let parent_state = manager + .block_state(parent_hash) + .ok_or(StateTrieOverlayError { tip_hash: parent_hash, anchor_hash })?; + manager.overlay_for_parent(&parent_state, anchor_hash) + } + #[test] fn errors_for_unknown_parent() { let manager = OverlayManager::::default(); let parent = B256::random(); let anchor = B256::random(); - let err = manager.overlay_for_parent(parent, anchor).unwrap_err(); + let err = overlay_for_parent(&manager, parent, anchor).unwrap_err(); assert_eq!(err.tip_hash, parent); assert_eq!(err.anchor_hash, anchor); @@ -965,15 +1021,15 @@ mod tests { let anchor_hash = blocks[0].recovered_block().parent_hash(); let (_, state) = - manager.overlay_for_parent(blocks[2].recovered_block().hash(), anchor_hash).unwrap(); + overlay_for_parent(&manager, blocks[2].recovered_block().hash(), anchor_hash).unwrap(); assert_eq!(state.accounts.len(), 3); let short_anchor = blocks[1].recovered_block().hash(); let (_, short) = - manager.overlay_for_parent(blocks[2].recovered_block().hash(), short_anchor).unwrap(); + overlay_for_parent(&manager, blocks[2].recovered_block().hash(), short_anchor).unwrap(); assert_eq!(short.accounts.len(), 1); let (_, cached_short) = - manager.overlay_for_parent(blocks[2].recovered_block().hash(), short_anchor).unwrap(); + overlay_for_parent(&manager, blocks[2].recovered_block().hash(), short_anchor).unwrap(); assert!(Arc::ptr_eq(&short, &cached_short)); } @@ -1042,10 +1098,10 @@ mod tests { let parent_key = OverlayCacheKey { anchor_hash, tip_hash: parent_hash }; let child_key = OverlayCacheKey { anchor_hash, tip_hash: child_hash }; - manager.overlay_for_parent(parent_hash, anchor_hash).unwrap(); + overlay_for_parent(&manager, parent_hash, anchor_hash).unwrap(); manager.execution_overlay_for_parent(parent_hash, anchor_hash).unwrap(); - manager.overlay_for_parent(child_hash, anchor_hash).unwrap(); + overlay_for_parent(&manager, child_hash, anchor_hash).unwrap(); manager.execution_overlay_for_parent(child_hash, anchor_hash).unwrap(); assert!(!manager.state_trie_overlays.entries.contains_key(&parent_key)); @@ -1067,7 +1123,7 @@ mod tests { let child_hash = blocks[2].recovered_block().hash(); let parent_key = OverlayCacheKey { anchor_hash, tip_hash: parent_hash }; - manager.overlay_for_parent(parent_hash, anchor_hash).unwrap(); + overlay_for_parent(&manager, parent_hash, anchor_hash).unwrap(); let state_parent = manager .state_trie_overlays .entries @@ -1080,7 +1136,7 @@ mod tests { let execution_parent = manager.execution_overlay_for_parent(parent_hash, anchor_hash).unwrap(); - let (_, child_state) = manager.overlay_for_parent(child_hash, anchor_hash).unwrap(); + let (_, child_state) = overlay_for_parent(&manager, child_hash, anchor_hash).unwrap(); let child_execution = manager.execution_overlay_for_parent(child_hash, anchor_hash).unwrap(); @@ -1230,9 +1286,11 @@ mod tests { #[test] fn required_lookup_waits_for_in_progress_overlay() { let manager = OverlayManager::::default(); + let block = test_blocks().remove(0); + let parent_state = BlockState::new(block); let key = OverlayCacheKey { - anchor_hash: B256::with_last_byte(1), - tip_hash: B256::with_last_byte(2), + anchor_hash: parent_state.block_ref().recovered_block().parent_hash(), + tip_hash: parent_state.hash(), }; let waiter = Arc::new(OverlayWaiter::new()); manager @@ -1243,7 +1301,7 @@ mod tests { let (tx, rx) = mpsc::channel(); thread::spawn(move || { let res = - manager.overlay_for_parent(key.tip_hash, key.anchor_hash).map(|(_, state)| state); + manager.overlay_for_parent(&parent_state, key.anchor_hash).map(|(_, state)| state); tx.send(res).unwrap(); }); @@ -1267,7 +1325,7 @@ mod tests { } let original_anchor = blocks[0].recovered_block().parent_hash(); - manager.overlay_for_parent(blocks[2].recovered_block().hash(), original_anchor).unwrap(); + overlay_for_parent(&manager, blocks[2].recovered_block().hash(), original_anchor).unwrap(); manager .execution_overlay_for_parent(blocks[2].recovered_block().hash(), original_anchor) .unwrap(); @@ -1278,15 +1336,14 @@ mod tests { ]); let anchor_hash = blocks[1].recovered_block().hash(); - assert!(manager - .overlay_for_parent(blocks[2].recovered_block().hash(), original_anchor) + assert!(overlay_for_parent(&manager, blocks[2].recovered_block().hash(), original_anchor) .is_err()); assert!(manager .execution_overlay_for_parent(blocks[2].recovered_block().hash(), original_anchor) .is_err()); let (_, state) = - manager.overlay_for_parent(blocks[2].recovered_block().hash(), anchor_hash).unwrap(); + overlay_for_parent(&manager, blocks[2].recovered_block().hash(), anchor_hash).unwrap(); assert_eq!(state.accounts.len(), 1); let execution = manager .execution_overlay_for_parent(blocks[2].recovered_block().hash(), anchor_hash) diff --git a/crates/storage/storage-overlay/src/provider.rs b/crates/storage/storage-overlay/src/provider.rs index 104f6cbc620..c7c2cd9858f 100644 --- a/crates/storage/storage-overlay/src/provider.rs +++ b/crates/storage/storage-overlay/src/provider.rs @@ -1191,6 +1191,32 @@ mod tests { assert_eq!(state_provider_factory.execution_overlay_cache.len(), 1); } + #[test] + fn lazy_overlays_survive_manager_block_removal() { + let (factory, blocks) = setup_frontiers(1, 1); + let manager = OverlayManager::default(); + for block in &blocks[2..=3] { + manager.insert_block(block.clone()); + } + let state_provider_factory = OverlayStateProviderFactory::new( + factory, + manager.overlay_builder(blocks[3].recovered_block().hash()), + ); + let provider = state_provider_factory.database_provider_ro().unwrap(); + + manager.remove_blocks(blocks[2..=3].iter().map(|block| block.recovered_block().hash())); + + let (execution_overlay, _) = provider.execution_overlay().unwrap(); + assert_eq!( + execution_overlay.block_hashes(), + [blocks[2].recovered_block().num_hash(), blocks[3].recovered_block().num_hash()] + ); + assert_eq!( + account_keys(provider.state_trie_overlay().unwrap()), + vec![B256::with_last_byte(3), B256::with_last_byte(4)] + ); + } + #[test] fn skipped_state_trie_overlay_is_not_cached_or_used_for_state_roots() { let (factory, blocks) = setup_frontiers(3, 3); From ca6b1cb89bbff701d68f4be945c5170677cd38af Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Thu, 3 Sep 2026 11:30:24 +0200 Subject: [PATCH 125/294] perf(rpc): update eth cache gauges once per drained batch (#26968) --- crates/rpc/rpc-eth-types/src/cache/mod.rs | 6 ++- .../rpc-eth-types/src/cache/multi_consumer.rs | 48 +++++++++++++++++-- 2 files changed, 47 insertions(+), 7 deletions(-) diff --git a/crates/rpc/rpc-eth-types/src/cache/mod.rs b/crates/rpc/rpc-eth-types/src/cache/mod.rs index f8b3461eaa6..3d06e7548a8 100644 --- a/crates/rpc/rpc-eth-types/src/cache/mod.rs +++ b/crates/rpc/rpc-eth-types/src/cache/mod.rs @@ -429,7 +429,7 @@ where self.bal_cache.shrink_to(min_capacity); } - fn update_cached_metrics(&self) { + fn update_cached_metrics(&mut self) { self.full_block_cache.update_cached_metrics(); self.receipts_cache.update_cached_metrics(); self.bal_cache.update_cached_metrics(); @@ -449,6 +449,9 @@ where let Poll::Ready(action) = this.action_rx.poll_next_unpin(cx) else { // shrink queues if we don't have any work to do this.shrink_queues(); + // gauges only need to be accurate once the batch of messages is drained, and only + // caches that changed during the batch republish theirs + this.update_cached_metrics(); return Poll::Pending; }; @@ -603,7 +606,6 @@ where let _ = response_tx.send(result); } }; - this.update_cached_metrics(); } } } diff --git a/crates/rpc/rpc-eth-types/src/cache/multi_consumer.rs b/crates/rpc/rpc-eth-types/src/cache/multi_consumer.rs index 3ce52ee5a4b..2ac7d37b814 100644 --- a/crates/rpc/rpc-eth-types/src/cache/multi_consumer.rs +++ b/crates/rpc/rpc-eth-types/src/cache/multi_consumer.rs @@ -24,6 +24,8 @@ where metrics: CacheMetrics, // Tracked heap usage memory_usage: usize, + /// Whether the cached count or memory usage changed since the gauges were last published. + metrics_dirty: bool, } impl Debug for MultiConsumerLruCache @@ -68,9 +70,10 @@ where where V: InMemorySize, { - self.cache - .remove(key) - .inspect(|value| self.memory_usage = self.memory_usage.saturating_sub(value.size())); + self.cache.remove(key).inspect(|value| { + self.memory_usage = self.memory_usage.saturating_sub(value.size()); + self.metrics_dirty = true; + }); self.queued .remove(key) .inspect(|removed| self.metrics.queued_consumers_count.decrement(removed.len() as f64)) @@ -105,10 +108,12 @@ where { // update tracked memory with the evicted value self.memory_usage = self.memory_usage.saturating_sub(evicted.size()); + self.metrics_dirty = true; } if self.cache.insert(key, value) { self.memory_usage = self.memory_usage.saturating_add(size); + self.metrics_dirty = true; true } else { false @@ -121,11 +126,18 @@ where self.queued.shrink_to(min_capacity); } - /// Update metrics for the inner cache. + /// Publishes the cached count and memory usage gauges if either changed since the last call. + /// + /// Returns whether the gauges were written, so lookups that only hit the cache cost nothing. #[inline] - pub fn update_cached_metrics(&self) { + pub fn update_cached_metrics(&mut self) -> bool { + if !self.metrics_dirty { + return false + } + self.metrics_dirty = false; self.metrics.cached_count.set(self.cache.len() as f64); self.metrics.memory_usage.set(self.memory_usage as f64); + true } } @@ -140,6 +152,32 @@ where queued: Default::default(), metrics: CacheMetrics::new_with_labels(&[("cache", cache_id.to_string())]), memory_usage: 0, + metrics_dirty: false, } } } + +#[cfg(test)] +mod tests { + use super::*; + use alloy_primitives::B256; + + #[test] + fn gauges_only_republished_after_cache_changes() { + let mut cache: MultiConsumerLruCache = + MultiConsumerLruCache::new(2, "test"); + assert!(!cache.update_cached_metrics()); + + assert!(cache.insert(B256::ZERO, B256::ZERO)); + assert!(cache.update_cached_metrics()); + assert!(!cache.update_cached_metrics()); + + // hits do not touch the gauges + assert!(cache.get(&B256::ZERO).is_some()); + assert!(cache.get(&B256::with_last_byte(1)).is_none()); + assert!(!cache.update_cached_metrics()); + + assert!(cache.remove(&B256::ZERO).is_none()); + assert!(cache.update_cached_metrics()); + } +} From fac89a4a723f77e7575f5374f3213678a61ab543 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Thu, 3 Sep 2026 12:00:27 +0200 Subject: [PATCH 126/294] chore: fix latest nightly clippy lints (#26985) --- crates/etl/src/lib.rs | 5 +---- crates/net/network/src/peers.rs | 2 +- crates/stages/api/src/pipeline/mod.rs | 2 +- crates/stages/stages/src/stages/execution/mod.rs | 2 +- crates/stages/stages/src/stages/sender_recovery.rs | 2 +- crates/stages/stages/tests/pipeline.rs | 4 ++-- .../storage/provider/src/providers/static_file/mod.rs | 2 +- crates/transaction-pool/src/pool/txpool.rs | 11 +++++------ crates/trie/db/tests/trie.rs | 4 ++-- crates/trie/sparse/src/arena/mod.rs | 3 +-- testing/ef-tests/src/cases/blockchain_test.rs | 2 +- 11 files changed, 17 insertions(+), 22 deletions(-) diff --git a/crates/etl/src/lib.rs b/crates/etl/src/lib.rs index d32905ea043..e1f48e4ea28 100644 --- a/crates/etl/src/lib.rs +++ b/crates/etl/src/lib.rs @@ -299,10 +299,7 @@ mod tests { for (id, entry) in collector.iter().unwrap().enumerate() { let expected = entries[id]; - assert_eq!( - entry.unwrap(), - (expected.0.encode().to_vec(), expected.1.compress().clone()) - ); + assert_eq!(entry.unwrap(), (expected.0.encode().to_vec(), expected.1.compress())); } let temp_dir_path = collector.dir.as_ref().unwrap().path().to_path_buf(); diff --git a/crates/net/network/src/peers.rs b/crates/net/network/src/peers.rs index 818f7cdf073..419d503829d 100644 --- a/crates/net/network/src/peers.rs +++ b/crates/net/network/src/peers.rs @@ -27,7 +27,7 @@ use reth_network_types::{ use std::{ collections::VecDeque, fmt::Display, - io::{self}, + io, net::{IpAddr, SocketAddr}, pin::Pin, task::{Context, Poll}, diff --git a/crates/stages/api/src/pipeline/mod.rs b/crates/stages/api/src/pipeline/mod.rs index 36a8a6f3c3d..7ae0ecb9ac8 100644 --- a/crates/stages/api/src/pipeline/mod.rs +++ b/crates/stages/api/src/pipeline/mod.rs @@ -512,7 +512,7 @@ impl Pipeline { total: total_stages, }, stage_id, - result: out.clone(), + result: out, }); if let Some(metrics_tx) = &mut self.metrics_tx { let _ = metrics_tx.send(MetricEvent::StageCheckpoint { diff --git a/crates/stages/stages/src/stages/execution/mod.rs b/crates/stages/stages/src/stages/execution/mod.rs index e3e8121d11d..0b76fdf01d1 100644 --- a/crates/stages/stages/src/stages/execution/mod.rs +++ b/crates/stages/stages/src/stages/execution/mod.rs @@ -1210,7 +1210,7 @@ mod tests { // Test Unwind provider = factory.database_provider_rw().unwrap(); let mut stage = stage(); - provider.set_prune_modes(mode.clone().unwrap_or_default()); + provider.set_prune_modes(mode.unwrap_or_default()); let result = stage .unwind( diff --git a/crates/stages/stages/src/stages/sender_recovery.rs b/crates/stages/stages/src/stages/sender_recovery.rs index 1d44de77271..2aa99804145 100644 --- a/crates/stages/stages/src/stages/sender_recovery.rs +++ b/crates/stages/stages/src/stages/sender_recovery.rs @@ -350,7 +350,7 @@ where // Read the raw value, and let the rayon worker to decompress & decode. let chunk = match static_file_provider.fetch_range_with_predicate( StaticFileSegment::Transactions, - chunk_range.clone(), + chunk_range, |cursor, number| { Ok(cursor .get_one:: = SealedBlock::seal_parts( - header.clone(), + header, BlockBody { transactions, ommers: Vec::new(), withdrawals: None }, ); diff --git a/crates/storage/provider/src/providers/static_file/mod.rs b/crates/storage/provider/src/providers/static_file/mod.rs index 21c98d3bf86..d06393f1f1c 100644 --- a/crates/storage/provider/src/providers/static_file/mod.rs +++ b/crates/storage/provider/src/providers/static_file/mod.rs @@ -752,7 +752,7 @@ mod tests { }) .collect(); - let changeset = generate_test_changesets(block_num, addresses.clone()); + let changeset = generate_test_changesets(block_num, addresses); writer.append_account_changeset(changeset, block_num).unwrap(); } diff --git a/crates/transaction-pool/src/pool/txpool.rs b/crates/transaction-pool/src/pool/txpool.rs index 1b87888a129..c7227c73879 100644 --- a/crates/transaction-pool/src/pool/txpool.rs +++ b/crates/transaction-pool/src/pool/txpool.rs @@ -3925,7 +3925,7 @@ mod tests { // insert a bunch of transactions into the queued pool for _ in 0..queued_limit.max_txs { let tx = MockTransaction::eip1559().inc_price_by(10).inc_nonce(); - let validated = f.validated(tx.clone()); + let validated = f.validated(tx); let _id = *validated.id(); pool.add_transaction(validated, U256::from(1_000), 0, None).unwrap(); } @@ -3935,7 +3935,7 @@ mod tests { for _ in 0..queued_limit.max_txs { let tx = MockTransaction::eip1559().inc_price_by(10).inc_nonce(); - let validated = f.validated(tx.clone()); + let validated = f.validated(tx); let _id = *validated.id(); pool.add_transaction(validated, U256::from(1_000), 0, None).unwrap(); @@ -3955,7 +3955,7 @@ mod tests { // insert a bunch of transactions into the queued pool for _ in 0..blob_limit.max_txs { let tx = MockTransaction::eip4844().inc_price_by(100).with_blob_fee(100); - let validated = f.validated(tx.clone()); + let validated = f.validated(tx); let _id = *validated.id(); pool.add_transaction(validated, U256::from(1_000), 0, None).unwrap(); } @@ -3965,7 +3965,7 @@ mod tests { for _ in 0..blob_limit.max_txs { let tx = MockTransaction::eip4844().inc_price_by(100).with_blob_fee(100); - let validated = f.validated(tx.clone()); + let validated = f.validated(tx); let _id = *validated.id(); pool.add_transaction(validated, U256::from(1_000), 0, None).unwrap(); @@ -4408,8 +4408,7 @@ mod tests { tx6.clone(), tx7.clone(), ] { - pool.add_transaction(f.validated(tx.clone()), on_chain_balance, on_chain_nonce, None) - .unwrap(); + pool.add_transaction(f.validated(tx), on_chain_balance, on_chain_nonce, None).unwrap(); } let base_fee = base_fee as u64; diff --git a/crates/trie/db/tests/trie.rs b/crates/trie/db/tests/trie.rs index 903b6e2b0c1..75af2b41ca8 100644 --- a/crates/trie/db/tests/trie.rs +++ b/crates/trie/db/tests/trie.rs @@ -837,7 +837,7 @@ proptest! { let mut hashed_account_cursor = tx.tx_ref().cursor_write::().unwrap(); let mut state = BTreeMap::default(); - for accounts in account_changes { + for mut accounts in account_changes { let should_generate_changeset = !state.is_empty(); let mut changes = PrefixSetMut::default(); for (hashed_address, balance) in accounts.clone() { @@ -854,7 +854,7 @@ proptest! { .unwrap() }); - state.append(&mut accounts.clone()); + state.append(&mut accounts); let expected_root = state_root_prehashed( state.iter().map(|(&key, &balance)| (key, (Account { balance, ..Default::default() }, std::iter::empty()))) ); diff --git a/crates/trie/sparse/src/arena/mod.rs b/crates/trie/sparse/src/arena/mod.rs index 2eb4f97b3af..d2e96357641 100644 --- a/crates/trie/sparse/src/arena/mod.rs +++ b/crates/trie/sparse/src/arena/mod.rs @@ -1124,8 +1124,7 @@ impl ArenaParallelSparseTrie { ); let branch = arena[head_idx].branch_mut(); - branch.state = - ArenaSparseNodeState::Cached { rlp_node: rlp_node.clone(), epoch: node_epoch }; + branch.state = ArenaSparseNodeState::Cached { rlp_node, epoch: node_epoch }; branch.branch_masks = new_branch_masks; // Record trie updates for dirty branches only. diff --git a/testing/ef-tests/src/cases/blockchain_test.rs b/testing/ef-tests/src/cases/blockchain_test.rs index 2c54e3870e3..f48ccab2283 100644 --- a/testing/ef-tests/src/cases/blockchain_test.rs +++ b/testing/ef-tests/src/cases/blockchain_test.rs @@ -332,7 +332,7 @@ fn decode_blocks( .map_err(|err| Error::block_failed(block_number, err))?; let recovered_block = - decoded.clone().try_recover().map_err(|err| Error::block_failed(block_number, err))?; + decoded.try_recover().map_err(|err| Error::block_failed(block_number, err))?; blocks.push(recovered_block); } From 5124a584a0c5ac324c9a35ce3dbbe1befad4b479 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Thu, 3 Sep 2026 09:52:08 +0000 Subject: [PATCH 127/294] perf(discv5): don't block startup on boot node ENR requests (#26929) --- crates/net/discv5/src/lib.rs | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/crates/net/discv5/src/lib.rs b/crates/net/discv5/src/lib.rs index 34d2d66e0c1..06b9a14c88a 100644 --- a/crates/net/discv5/src/lib.rs +++ b/crates/net/discv5/src/lib.rs @@ -540,6 +540,13 @@ pub fn build_local_enr( } /// Bootstraps underlying [`discv5::Discv5`] node with configured peers. +/// +/// Boot nodes given as an ENR are added to the kbuckets right away. Boot nodes given as an enode +/// need their ENR requested over the wire first, which is driven in the background: an unreachable +/// boot node only answers after the discv5 request timeout, and waiting for that would stall node +/// startup. +/// +/// Stays `async` because spawning those requests requires a tokio runtime context. pub async fn bootstrap( bootstrap_nodes: HashSet, discv5: &Arc, @@ -573,7 +580,11 @@ pub async fn bootstrap( } // If a session is established, the ENR is added straight away to discv5 kbuckets - Ok(_ = join_all(enr_requests).await) + if !enr_requests.is_empty() { + task::spawn(join_all(enr_requests)); + } + + Ok(()) } /// Backgrounds regular look up queries, in order to keep kbuckets populated. From ac411120c3f2d93c1599401931d112a4c66ebcb3 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Thu, 3 Sep 2026 09:52:49 +0000 Subject: [PATCH 128/294] fix(dns): keep rechecking a tree whose root did not change (#26937) --- crates/net/dns/src/lib.rs | 138 ++++++++++++++++++++++++++++++++++++- crates/net/dns/src/sync.rs | 94 ++++++++++++++++++++++++- 2 files changed, 229 insertions(+), 3 deletions(-) diff --git a/crates/net/dns/src/lib.rs b/crates/net/dns/src/lib.rs index bac285524d6..6d7c581d940 100644 --- a/crates/net/dns/src/lib.rs +++ b/crates/net/dns/src/lib.rs @@ -41,6 +41,7 @@ use tokio::{ oneshot, }, task::JoinHandle, + time::{Interval, MissedTickBehavior}, }; use tokio_stream::{ wrappers::{ReceiverStream, UnboundedReceiverStream}, @@ -107,6 +108,12 @@ pub struct DnsDiscoveryService { queued_events: VecDeque, /// The rate at which trees should be updated. recheck_interval: Duration, + /// Wakes the service up so due trees are rechecked. + /// + /// [`SyncTree::poll`] decides that a recheck is due by comparing timestamps, which registers + /// no waker, and once the initial walk is done there is nothing else to poll. Without this the + /// spawned service would sleep until an unrelated command or query happened to wake it. + recheck_tick: Interval, /// Links to the DNS networks to bootstrap. bootstrap_dns_networks: HashSet, } @@ -145,6 +152,7 @@ impl DnsDiscoveryService { dns_record_cache: LruMap::new(ByLength::new(dns_record_cache_limit.get())), queued_events: Default::default(), recheck_interval, + recheck_tick: recheck_tick(recheck_interval), bootstrap_dns_networks: bootstrap_dns_networks.unwrap_or_default(), } } @@ -235,7 +243,12 @@ impl DnsDiscoveryService { } }, Err((err, link)) => { - debug!(target: "disc::dns",%err, ?link, "Failed to lookup root") + debug!(target: "disc::dns",%err, ?link, "Failed to lookup root"); + // an already synced tree asked for this lookup and is waiting on it, so it has to + // be released even though the lookup failed + if let Some(tree) = self.trees.get_mut(&link) { + tree.root_update_failed(); + } } } } @@ -346,6 +359,11 @@ impl DnsDiscoveryService { self.sync_tree_with_link(link) } + // Register the wake-up for the next recheck. Polled to `Pending` so the waker is + // always armed; a ready tick needs no extra work because the trees were already polled + // with a current timestamp above. + while self.recheck_tick.poll_tick(cx).is_ready() {} + if !progress && self.queued_events.is_empty() { return Poll::Pending } @@ -353,6 +371,14 @@ impl DnsDiscoveryService { } } +/// Builds the interval that wakes the service to look for trees due a recheck. +fn recheck_tick(recheck_interval: Duration) -> Interval { + let mut tick = tokio::time::interval(recheck_interval); + // a service that was starved must not then fire a burst of catch-up ticks + tick.set_missed_tick_behavior(MissedTickBehavior::Delay); + tick +} + /// A Stream events, mainly used for debugging impl Stream for DnsDiscoveryService { type Item = DnsDiscoveryEvent; @@ -413,6 +439,7 @@ mod tests { use std::{ future::poll_fn, net::{IpAddr, Ipv4Addr}, + num::NonZeroUsize, }; fn entry_hash(entry_txt: &str) -> String { @@ -611,6 +638,115 @@ mod tests { .await; } + #[tokio::test] + async fn spawned_service_rechecks_without_outside_wakeups() { + reth_tracing::init_test_tracing(); + + let config = DnsDiscoveryConfig { + recheck_interval: Duration::from_millis(100), + max_requests_per_sec: NonZeroUsize::new(50).unwrap(), + ..Default::default() + }; + + let secret_key = SecretKey::new(&mut thread_rng()); + let resolver = Arc::new(MapResolver::default()); + let s = "enrtree-root:v1 e=QFT4PBCRX4XQCV3VUYJ6BTCEPU l=JGUFMSAGI7KZYB3P7IZW4S5Y3A seq=3 sig=3FmXuVwpa8Y7OstZTx9PIb1mt8FrW7VpDOFv4AaGCsZ2EIHmhraWhe4NxYhQDlw5MjeFXYMbJjsPeKlHzmJREQE"; + let mut root: TreeRootEntry = s.parse().unwrap(); + root.sign(&secret_key).unwrap(); + + let link = + LinkEntry { domain: "nodes.example.org".to_string(), pubkey: secret_key.public() }; + resolver.insert(link.domain.clone(), root.to_string()); + + let mut service = DnsDiscoveryService::new(Arc::clone(&resolver), config.clone()); + let mut node_records = service.node_record_stream(); + service.sync_tree_with_link(link.clone()); + + // drive it the way the production path does: nothing polls the service except its own + // event stream, so only a wake-up the service arms itself can start a recheck + let handle = service.spawn(); + + // let the initial walk finish and the service go idle + tokio::time::sleep(config.recheck_interval * 3).await; + + // publish a change that only a self-scheduled recheck can pick up + let mut new_root = root.clone(); + new_root.sequence_number = new_root.sequence_number.saturating_add(1); + // needs an address, an ENR without one yields no node record + let enr = Enr::builder() + .ip4(Ipv4Addr::LOCALHOST) + .udp4(30303) + .tcp4(30303) + .build(&secret_key) + .unwrap(); + let enr_txt = enr.to_base64(); + new_root.enr_root = entry_hash(&enr_txt); + new_root.sign(&secret_key).unwrap(); + resolver.insert(link.domain.clone(), new_root.to_string()); + resolver.insert(format!("{}.{}", new_root.enr_root.clone(), link.domain), enr_txt); + + let update = tokio::time::timeout(Duration::from_secs(10), node_records.next()) + .await + .expect("service never woke up to recheck the tree") + .expect("record stream closed"); + assert_eq!(update.enr, enr); + + handle.abort(); + } + + #[tokio::test] + async fn test_recheck_tree_survives_unchanged_root() { + reth_tracing::init_test_tracing(); + + let config = DnsDiscoveryConfig { + recheck_interval: Duration::from_millis(100), + // the default 3/s would leave the recheck lookup queued behind the initial tree walk + max_requests_per_sec: NonZeroUsize::new(50).unwrap(), + ..Default::default() + }; + + let secret_key = SecretKey::new(&mut thread_rng()); + let resolver = Arc::new(MapResolver::default()); + let s = "enrtree-root:v1 e=QFT4PBCRX4XQCV3VUYJ6BTCEPU l=JGUFMSAGI7KZYB3P7IZW4S5Y3A seq=3 sig=3FmXuVwpa8Y7OstZTx9PIb1mt8FrW7VpDOFv4AaGCsZ2EIHmhraWhe4NxYhQDlw5MjeFXYMbJjsPeKlHzmJREQE"; + let mut root: TreeRootEntry = s.parse().unwrap(); + root.sign(&secret_key).unwrap(); + + let link = + LinkEntry { domain: "nodes.example.org".to_string(), pubkey: secret_key.public() }; + resolver.insert(link.domain.clone(), root.to_string()); + + let mut service = DnsDiscoveryService::new(Arc::clone(&resolver), config.clone()); + service.sync_tree_with_link(link.clone()); + + // first recheck sees the very same root, which must not take the tree out of rotation + for _ in 0..10 { + poll_fn(|cx| { + let _ = service.poll(cx); + Poll::Ready(()) + }) + .await; + tokio::time::sleep(config.recheck_interval / 2).await; + } + + // now publish a change and expect the tree to still pick it up + let mut new_root = root.clone(); + new_root.sequence_number = new_root.sequence_number.saturating_add(1); + let enr = Enr::empty(&secret_key).unwrap(); + let enr_txt = enr.to_base64(); + new_root.enr_root = entry_hash(&enr_txt); + new_root.sign(&secret_key).unwrap(); + resolver.insert(link.domain.clone(), new_root.to_string()); + resolver.insert(format!("{}.{}", new_root.enr_root.clone(), link.domain), enr_txt); + + let event = tokio::time::timeout(Duration::from_secs(10), poll_fn(|cx| service.poll(cx))) + .await + .expect("tree stopped rechecking after an unchanged root"); + + match event { + DnsDiscoveryEvent::Enr(discovered) => assert_eq!(discovered, enr), + } + } + #[tokio::test] async fn test_hash_mismatch_is_not_cached_and_does_not_poison_same_hash() { let secret_key = SecretKey::new(&mut thread_rng()); diff --git a/crates/net/dns/src/sync.rs b/crates/net/dns/src/sync.rs index 43e7e9886ad..d2f2c04c071 100644 --- a/crates/net/dns/src/sync.rs +++ b/crates/net/dns/src/sync.rs @@ -90,6 +90,22 @@ impl SyncTree { Some(SyncAction::Enr(enr)) } + /// Records that the scheduled root lookup failed. + /// + /// Without this the tree would stay in [`SyncState::RootUpdate`] and stop polling entirely, + /// so one failed lookup would take the tree out of rotation permanently. It retries at the + /// next recheck instead. + pub(crate) fn root_update_failed(&mut self) { + // Only a lookup this tree scheduled releases it. Root lookups are also started from + // bootstrap, from the public sync command and from link entries found in other trees, and + // those outcomes must not overwrite a walk that is still recorded in `Pending`/`Enr`/ + // `Link`, or the tree would go quiet with its subtree unvisited. + if matches!(self.sync_state, SyncState::RootUpdate) { + self.root_updated = Instant::now(); + self.sync_state = SyncState::Active; + } + } + /// Updates the root and returns what changed pub(crate) fn update_root(&mut self, root: TreeRootEntry) { let enr_unchanged = root.enr_root == self.root.enr_root; @@ -99,8 +115,15 @@ impl SyncTree { self.root_updated = Instant::now(); let state = match (enr_unchanged, link_unchanged) { - // both unchanged — no resync needed - (true, true) => return, + // Nothing to resync. The tree still has to leave `RootUpdate`, which would otherwise + // park `poll` on `None` forever, but only when this answers its own recheck: an + // unsolicited lookup of an unchanged root must leave a walk in progress alone. + (true, true) => { + if !matches!(self.sync_state, SyncState::RootUpdate) { + return + } + SyncState::Active + } // only ENR changed (false, true) => { self.unresolved_nodes.clear(); @@ -193,6 +216,73 @@ mod tests { assert!(tree.poll(now, timeout).is_none()); } + #[test] + fn unsolicited_root_outcome_does_not_cancel_a_walk() { + let timeout = Duration::from_secs(60); + + // a root lookup this tree did not schedule must not discard the walk it still owes. Root + // lookups are also started from bootstrap, the public sync command and link entries in + // other trees, and the query pool does not de-duplicate them. + for unsolicited in [true, false] { + let mut tree = make_tree(); + // Pending -> emits Link, leaving the enr root still to walk + assert!(matches!(tree.poll(Instant::now(), timeout), Some(SyncAction::Link(_)))); + + if unsolicited { + tree.update_root(base_root()); + } else { + tree.root_update_failed(); + } + + assert!( + matches!(tree.poll(Instant::now(), timeout), Some(SyncAction::Enr(_))), + "walk was cancelled by an unsolicited root outcome" + ); + } + } + + #[test] + fn update_root_unchanged_still_rechecks_later() { + let mut tree = make_tree(); + let timeout = Duration::from_secs(60); + advance_to_active(&mut tree); + + // the recheck interval elapses and the root is looked up again + let due = Instant::now() + timeout * 2; + assert!(matches!(tree.poll(due, timeout), Some(SyncAction::UpdateRoot))); + + // it comes back unchanged, so there is nothing to resync right now + tree.update_root(base_root()); + assert!(tree.poll(Instant::now(), timeout).is_none()); + + // but the tree has to keep rechecking, otherwise it never picks up later changes + let later = Instant::now() + timeout * 4; + assert!( + matches!(tree.poll(later, timeout), Some(SyncAction::UpdateRoot)), + "tree stopped rechecking its root after an unchanged update" + ); + } + + #[test] + fn failed_root_update_still_rechecks_later() { + let mut tree = make_tree(); + let timeout = Duration::from_secs(60); + advance_to_active(&mut tree); + + let due = Instant::now() + timeout * 2; + assert!(matches!(tree.poll(due, timeout), Some(SyncAction::UpdateRoot))); + + // the lookup failed, which must not take the tree out of rotation for good + tree.root_update_failed(); + assert!(tree.poll(Instant::now(), timeout).is_none()); + + let later = Instant::now() + timeout * 4; + assert!( + matches!(tree.poll(later, timeout), Some(SyncAction::UpdateRoot)), + "tree stopped rechecking its root after a failed lookup" + ); + } + #[test] fn update_root_only_enr_changed_triggers_enr() { let mut tree = make_tree(); From 5475a47a718b32a15cf3386e76f6d25a08f6e943 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Thu, 3 Sep 2026 09:53:48 +0000 Subject: [PATCH 129/294] perf(rpc): bound getLogs range header capacity by remaining headers (#26971) --- crates/rpc/rpc/src/eth/filter.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/rpc/rpc/src/eth/filter.rs b/crates/rpc/rpc/src/eth/filter.rs index d811c650e88..46cb1d53d9a 100644 --- a/crates/rpc/rpc/src/eth/filter.rs +++ b/crates/rpc/rpc/src/eth/filter.rs @@ -1220,7 +1220,7 @@ impl< return Ok(None); }; - let mut range_headers = Vec::with_capacity(self.max_range); + let mut range_headers = Vec::with_capacity(self.max_range.min(self.iter.len() + 1)); range_headers.push(next_header); // Collect consecutive blocks up to max_range size From d71913364273b092e33f8da52f5994ba5f4674b6 Mon Sep 17 00:00:00 2001 From: Arsenii Kulikov Date: Thu, 3 Sep 2026 10:04:24 +0000 Subject: [PATCH 130/294] fix(engine): finalize pipeline sync target after backfill (#26983) --- crates/engine/tree/src/tree/mod.rs | 7 +++ crates/engine/tree/src/tree/tests.rs | 45 ++++++++++++++++++ crates/ethereum/node/tests/e2e/p2p.rs | 66 ++++++++++++++++++++++++++- 3 files changed, 117 insertions(+), 1 deletion(-) diff --git a/crates/engine/tree/src/tree/mod.rs b/crates/engine/tree/src/tree/mod.rs index 3a8b77046f8..084a087129a 100644 --- a/crates/engine/tree/src/tree/mod.rs +++ b/crates/engine/tree/src/tree/mod.rs @@ -1885,6 +1885,13 @@ where // update the tracked canonical head self.canonical_in_memory_state.set_canonical_head(new_head); + + // If the pipeline reached the head of the syncing FCU, apply its safe and finalized + // blocks now that the head is canonical. An unwind must wait for a new FCU because the + // supplied sync target may have been invalidated. + if !ctrl.is_unwind() { + self.on_canonicalized_sync_target(backfill_num_hash.hash); + } } // check if we need to run backfill again by comparing the most recent backfill target diff --git a/crates/engine/tree/src/tree/tests.rs b/crates/engine/tree/src/tree/tests.rs index 96550260468..c767f6f1311 100644 --- a/crates/engine/tree/src/tree/tests.rs +++ b/crates/engine/tree/src/tree/tests.rs @@ -2853,6 +2853,51 @@ fn test_canonicalizing_downloaded_sync_target_head_updates_finalized() { assert!(test_harness.tree.state.forkchoice_state_tracker.sync_target_state().is_none()); } +/// Tests that pipeline backfill reaching a syncing FCU with `head == finalized` applies the +/// finalized block without waiting for another FCU. +#[test] +fn test_backfill_reaching_sync_target_head_updates_finalized() { + reth_tracing::init_test_tracing(); + + let chain_spec = MAINNET.clone(); + let mut test_harness = TestHarness::new(chain_spec); + + let blocks: Vec<_> = test_harness.block_builder.get_executed_blocks(0..3).collect(); + let initial_head = blocks[0].recovered_block().num_hash(); + let sync_target = blocks[2].recovered_block().num_hash(); + test_harness = test_harness.with_blocks(blocks); + test_harness.tree.state.tree_state.set_canonical_head(initial_head); + + let fcu_state = ForkchoiceState { + head_block_hash: sync_target.hash, + safe_block_hash: sync_target.hash, + finalized_block_hash: sync_target.hash, + }; + test_harness + .tree + .state + .forkchoice_state_tracker + .set_latest(fcu_state, ForkchoiceStatus::Syncing); + + test_harness + .tree + .on_backfill_sync_finished(ControlFlow::Continue { block_number: sync_target.number }) + .unwrap(); + + assert_eq!(test_harness.tree.state.tree_state.current_canonical_head, sync_target); + assert_eq!( + test_harness.tree.canonical_in_memory_state.get_finalized_num_hash(), + Some(sync_target) + ); + assert_eq!(test_harness.tree.canonical_in_memory_state.get_safe_num_hash(), Some(sync_target)); + assert_eq!( + test_harness.tree.state.forkchoice_state_tracker.last_valid_state(), + Some(fcu_state) + ); + assert!(test_harness.tree.state.forkchoice_state_tracker.sync_target_state().is_none()); + assert!(test_harness.from_tree_rx.try_recv().is_err()); +} + // --- Backfill target selection tests --- // // Cover `backfill_target_hash` and its consumer `backfill_sync_target`, exercised end-to-end diff --git a/crates/ethereum/node/tests/e2e/p2p.rs b/crates/ethereum/node/tests/e2e/p2p.rs index e77d9e316e7..8d139639ed9 100644 --- a/crates/ethereum/node/tests/e2e/p2p.rs +++ b/crates/ethereum/node/tests/e2e/p2p.rs @@ -2,7 +2,7 @@ use crate::utils::{ advance_with_random_transactions, eth_payload_attributes, eth_payload_attributes_amsterdam, }; use alloy_consensus::{SignableTransaction, TxEip1559, TxEnvelope}; -use alloy_eips::Encodable2718; +use alloy_eips::{BlockNumberOrTag, Encodable2718}; use alloy_network::TxSignerSync; use alloy_primitives::B256; use alloy_provider::{Provider, ProviderBuilder}; @@ -331,6 +331,70 @@ async fn test_long_reorg() -> eyre::Result<()> { Ok(()) } +#[tokio::test] +async fn test_pipeline_sync_target_head_becomes_finalized() -> eyre::Result<()> { + reth_tracing::init_test_tracing(); + + const TARGET_BLOCK: u64 = 100; + + let seed: [u8; 32] = rand::rng().random(); + let mut rng = StdRng::from_seed(seed); + println!("Seed: {seed:?}"); + + let chain_spec = Arc::new( + ChainSpecBuilder::default() + .chain(MAINNET.chain) + .genesis(serde_json::from_str(include_str!("../assets/genesis.json")).unwrap()) + .cancun_activated() + .prague_activated() + .build(), + ); + + let (mut nodes, _) = setup_engine::( + 2, + chain_spec.clone(), + false, + Default::default(), + eth_payload_attributes, + ) + .await?; + + let mut first_node = nodes.pop().unwrap(); + let second_node = nodes.pop().unwrap(); + + let first_provider = ProviderBuilder::new().connect_http(first_node.rpc_url()); + let second_provider = ProviderBuilder::new().connect_http(second_node.rpc_url()); + + advance_with_random_transactions(&mut first_node, TARGET_BLOCK as usize, &mut rng, false) + .await?; + + assert_eq!(second_provider.get_block_number().await?, 0); + + let target = first_provider.get_block_by_number(TARGET_BLOCK.into()).await?.unwrap(); + + // Send exactly one FCU with an unknown head == safe == finalized. The node must promote this + // FCU when pipeline sync completes, without relying on another FCU. + second_node.update_forkchoice(target.header.hash, target.header.hash).await?; + + tokio::time::timeout(Duration::from_secs(40), async { + while second_provider.get_block_number().await? != TARGET_BLOCK { + tokio::time::sleep(Duration::from_millis(100)).await; + } + eyre::Ok(()) + }) + .await + .map_err(|_| eyre::eyre!("timed out waiting for pipeline sync to block {TARGET_BLOCK}"))??; + + let finalized = second_provider + .get_block_by_number(BlockNumberOrTag::Finalized) + .await? + .ok_or_else(|| eyre::eyre!("finalized block not found"))?; + assert_eq!(finalized.header.number, TARGET_BLOCK); + assert_eq!(finalized.header.hash, target.header.hash); + + Ok(()) +} + #[tokio::test] async fn test_reorg_through_backfill() -> eyre::Result<()> { reth_tracing::init_test_tracing(); From 6d708031eb7fd3d31b83c5a03a1bb336cb7c9b56 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Thu, 3 Sep 2026 10:54:05 +0000 Subject: [PATCH 131/294] chore(trie): remove redundant clone flagged by nightly clippy (#26979) From 3fba6ad70a18f870669a8cc5078b65d3ffc01251 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Thu, 3 Sep 2026 13:24:49 +0200 Subject: [PATCH 132/294] perf(trie): avoid formatting storage root for disabled span (#26938) --- crates/trie/trie/src/trie.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/trie/trie/src/trie.rs b/crates/trie/trie/src/trie.rs index 65efe86a27a..bf3144046de 100644 --- a/crates/trie/trie/src/trie.rs +++ b/crates/trie/trie/src/trie.rs @@ -791,7 +791,7 @@ where (!retain_updates || prefix_set.is_empty()) && hashed_storage_cursor.is_storage_empty()? { - Span::current().record("storage_root", format!("{EMPTY_ROOT_HASH:?}")); + Span::current().record("storage_root", tracing::field::debug(EMPTY_ROOT_HASH)); return Ok(StorageRootProgress::Complete( EMPTY_ROOT_HASH, 0, @@ -869,7 +869,7 @@ where } let root = hash_builder.root(); - Span::current().record("storage_root", format!("{root:?}")); + Span::current().record("storage_root", tracing::field::debug(root)); let removed_keys = storage_node_iter.walker.take_removed_keys(); trie_updates.finalize(hash_builder, removed_keys); From 89964d320896f1b609572f8be2162b8b5461bc79 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Thu, 3 Sep 2026 11:29:00 +0000 Subject: [PATCH 133/294] perf(stages): use fast hasher for history index caches (#26950) --- crates/stages/stages/src/stages/utils.rs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/crates/stages/stages/src/stages/utils.rs b/crates/stages/stages/src/stages/utils.rs index 4c07f4120d3..ab6f19fc73e 100644 --- a/crates/stages/stages/src/stages/utils.rs +++ b/crates/stages/stages/src/stages/utils.rs @@ -1,5 +1,8 @@ //! Utils for `stages`. -use alloy_primitives::{map::AddressMap, Address, BlockNumber, TxNumber, B256}; +use alloy_primitives::{ + map::{AddressMap, HashMap}, + Address, BlockNumber, TxNumber, B256, +}; use reth_config::config::EtlConfig; use reth_db_api::{ cursor::{DbCursorRO, DbCursorRW}, @@ -20,7 +23,7 @@ use reth_provider::{ use reth_stages_api::StageError; use reth_static_file_types::StaticFileSegment; use reth_storage_api::{ChangeSetReader, StorageChangeSetReader}; -use std::{collections::HashMap, hash::Hash, ops::RangeBounds}; +use std::{hash::Hash, ops::RangeBounds}; use tracing::info; /// Number of blocks before pushing indices from cache to [`Collector`] From 70e776bfe588056b582077acf466bf57570842fd Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Thu, 3 Sep 2026 11:32:22 +0000 Subject: [PATCH 134/294] perf(static-files): use btree range lookup for block index (#26943) --- .../storage/provider/src/providers/static_file/manager.rs | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/crates/storage/provider/src/providers/static_file/manager.rs b/crates/storage/provider/src/providers/static_file/manager.rs index 937ec9f8b1c..a0e49fbce84 100644 --- a/crates/storage/provider/src/providers/static_file/manager.rs +++ b/crates/storage/provider/src/providers/static_file/manager.rs @@ -356,16 +356,15 @@ impl StaticFileProviderInner { if let Some(block_index) = block_index { // Find first block range that contains the requested block - if let Some((_, range)) = block_index.iter().find(|(max_block, _)| block <= **max_block) - { + if let Some((_, range)) = block_index.range(block..).next() { // Found matching range for an existing file using block index return *range; } else if let Some((_, range)) = block_index.last_key_value() { // Didn't find matching range for an existing file, derive a new range from the end // of the last existing file range. // - // `block` is always higher than `range.end()` here, because we iterated over all - // `block_index` ranges above and didn't find one that contains our block + // `block` is always higher than `range.end()` here, because `block_index` holds no + // range with a `max_block` greater than or equal to `block` let blocks_after_last_range = block - range.end(); let segments_to_skip = (blocks_after_last_range - 1) / blocks_per_file; let start = range.end() + 1 + segments_to_skip * blocks_per_file; From c50390456733fbac617ec7b201a02393a39ec6b1 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Thu, 3 Sep 2026 11:41:20 +0000 Subject: [PATCH 135/294] perf(ipc): avoid copying IPC subscription payloads (#26972) --- crates/rpc/ipc/src/server/mod.rs | 2 +- crates/rpc/ipc/src/stream_codec.rs | 13 ++++++++----- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/crates/rpc/ipc/src/server/mod.rs b/crates/rpc/ipc/src/server/mod.rs index 64dbfb27128..718561db871 100644 --- a/crates/rpc/ipc/src/server/mod.rs +++ b/crates/rpc/ipc/src/server/mod.rs @@ -536,7 +536,7 @@ async fn to_ipc_service( } item = rx_item.next() => { let Some(item) = item else { break }; - conn.push_back(item.to_string()); + conn.push_back(String::from(Box::::from(item))); } _ = &mut stopped => { // shutdown diff --git a/crates/rpc/ipc/src/stream_codec.rs b/crates/rpc/ipc/src/stream_codec.rs index f01a455a93c..8db8fc19ce0 100644 --- a/crates/rpc/ipc/src/stream_codec.rs +++ b/crates/rpc/ipc/src/stream_codec.rs @@ -27,7 +27,7 @@ // This basis of this file has been taken from the deprecated jsonrpc codebase: // https://github.com/paritytech/jsonrpc -use bytes::{Buf, BytesMut}; +use bytes::{Buf, BufMut, BytesMut}; use std::{io, str}; /// Separator for enveloping messages in streaming codecs @@ -162,11 +162,14 @@ impl tokio_util::codec::Encoder for StreamCodec { type Error = io::Error; fn encode(&mut self, msg: String, buf: &mut BytesMut) -> io::Result<()> { - let mut payload = msg.into_bytes(); - if let Separator::Byte(separator) = self.outgoing_separator { - payload.push(separator); + match self.outgoing_separator { + Separator::Byte(separator) => { + buf.reserve(msg.len() + 1); + buf.extend_from_slice(msg.as_bytes()); + buf.put_u8(separator); + } + Separator::Empty => buf.extend_from_slice(msg.as_bytes()), } - buf.extend_from_slice(&payload); Ok(()) } } From ddf3fee040f7752dde730a096a42833c31d7e984 Mon Sep 17 00:00:00 2001 From: Derek Cofausper <256792747+decofe@users.noreply.github.com> Date: Thu, 3 Sep 2026 12:03:57 +0000 Subject: [PATCH 136/294] perf(trie): avoid allocations for empty prefix sets (#26987) Co-authored-by: Matthias Seitz <19890894+mattsse@users.noreply.github.com> Co-authored-by: DaniPopes <57450786+DaniPopes@users.noreply.github.com> --- crates/trie/common/src/prefix_set.rs | 61 ++++++++++++++++++---------- 1 file changed, 40 insertions(+), 21 deletions(-) diff --git a/crates/trie/common/src/prefix_set.rs b/crates/trie/common/src/prefix_set.rs index cacaf768e5c..f662be1fda4 100644 --- a/crates/trie/common/src/prefix_set.rs +++ b/crates/trie/common/src/prefix_set.rs @@ -193,15 +193,13 @@ impl PrefixSetMut { /// /// If not yet sorted, the elements will be sorted and deduplicated. pub fn freeze(mut self) -> PrefixSet { - if self.all { - PrefixSet { index: 0, all: true, keys: Arc::new(Vec::new()) } - } else { + if !self.all { self.keys.sort_unstable(); self.keys.dedup(); // Shrink after deduplication to release unused capacity. self.keys.shrink_to_fit(); - PrefixSet { index: 0, all: false, keys: Arc::new(self.keys) } } + PrefixSet::new(self.all, self.keys) } } @@ -216,12 +214,19 @@ impl<'a> IntoIterator for &'a PrefixSetMut { /// A sorted prefix set that has an immutable _sorted_ list of unique keys. /// /// See also [`PrefixSetMut::freeze`]. -#[derive(Debug, Default, Clone)] +#[derive(Debug, Clone)] pub struct PrefixSet { /// Flag indicating that any entry should be considered changed. all: bool, index: usize, - keys: Arc>, + /// `None` for an empty set to avoid allocating an empty vector. + keys: Option>>, +} + +impl Default for PrefixSet { + fn default() -> Self { + Self::new(false, Vec::new()) + } } impl From for PrefixSet @@ -244,14 +249,18 @@ where previous = Some(key); } - Self { index: 0, all: false, keys: Arc::new(unpacked) } + Self::new(false, unpacked) } } impl PrefixSet { + fn new(all: bool, keys: Vec) -> Self { + Self { index: 0, all, keys: (!all && !keys.is_empty()).then(|| Arc::new(keys)) } + } + /// Creates a prefix set that considers every path changed. pub fn all_paths() -> Self { - Self { index: 0, all: true, keys: Arc::new(Vec::new()) } + Self::new(true, Vec::new()) } /// Returns `true` if any of the keys in the set has the given prefix @@ -274,11 +283,12 @@ impl PrefixSet { return true } - while self.index > 0 && &self.keys[self.index] > prefix { + let keys = self.keys.as_deref().map(Vec::as_slice).unwrap_or_default(); + while self.index > 0 && &keys[self.index] > prefix { self.index -= 1; } - for (idx, key) in self.keys[self.index..].iter().enumerate() { + for (idx, key) in keys[self.index..].iter().enumerate() { if key.starts_with(prefix) { self.index += idx; return true @@ -304,11 +314,12 @@ impl PrefixSet { return true } - while self.index > 0 && &self.keys[self.index] >= range.end { + let keys = self.keys.as_deref().map(Vec::as_slice).unwrap_or_default(); + while self.index > 0 && &keys[self.index] >= range.end { self.index -= 1; } - for (idx, key) in self.keys[self.index..].iter().enumerate() { + for (idx, key) in keys[self.index..].iter().enumerate() { if key >= range.start && key < range.end { self.index += idx; return true @@ -325,12 +336,12 @@ impl PrefixSet { /// Returns an iterator over reference to _all_ nibbles regardless of cursor position. pub fn iter(&self) -> core::slice::Iter<'_, Nibbles> { - self.keys.iter() + self.slice().iter() } /// Returns the underlying sorted prefix slice. pub fn slice(&self) -> &[Nibbles] { - self.keys.as_slice() + self.keys.as_deref().map(Vec::as_slice).unwrap_or_default() } /// Returns true if every entry should be considered changed. @@ -340,12 +351,12 @@ impl PrefixSet { /// Returns the number of elements in the set. pub fn len(&self) -> usize { - self.keys.len() + self.slice().len() } /// Returns `true` if the set is empty and `all` flag is not set. - pub fn is_empty(&self) -> bool { - !self.all && self.keys.is_empty() + pub const fn is_empty(&self) -> bool { + !self.all && self.keys.is_none() } } @@ -417,8 +428,8 @@ mod tests { assert!(prefix_set.contains(&Nibbles::from_nibbles_unchecked([1, 2]))); assert!(prefix_set.contains(&Nibbles::from_nibbles_unchecked([4, 5]))); assert!(!prefix_set.contains(&Nibbles::from_nibbles_unchecked([7, 8]))); - assert_eq!(prefix_set.keys.len(), 3); // Length should be 3 (excluding duplicate) - assert_eq!(prefix_set.keys.capacity(), 3); // Capacity should be 3 after shrinking + assert_eq!(prefix_set.slice().len(), 3); // Length should be 3 (excluding duplicate) + assert_eq!(prefix_set.keys.as_ref().unwrap().capacity(), 3); // Capacity after shrinking } #[test] @@ -437,8 +448,16 @@ mod tests { assert!(prefix_set.contains(&Nibbles::from_nibbles_unchecked([1, 2]))); assert!(prefix_set.contains(&Nibbles::from_nibbles_unchecked([4, 5]))); assert!(!prefix_set.contains(&Nibbles::from_nibbles_unchecked([7, 8]))); - assert_eq!(prefix_set.keys.len(), 3); // Length should be 3 (excluding duplicate) - assert_eq!(prefix_set.keys.capacity(), 3); // Capacity should be 3 after shrinking + assert_eq!(prefix_set.slice().len(), 3); // Length should be 3 (excluding duplicate) + assert_eq!(prefix_set.keys.as_ref().unwrap().capacity(), 3); // Capacity after shrinking + } + + #[test] + fn test_empty_prefix_sets_do_not_allocate() { + assert!(PrefixSet::default().keys.is_none()); + assert!(PrefixSetMut::default().freeze().keys.is_none()); + assert!(PrefixSet::from(core::iter::empty()).keys.is_none()); + assert!(PrefixSet::all_paths().keys.is_none()); } #[test] From 41717ed7b3df8235be5417b43b73c6f75518a9c7 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Thu, 3 Sep 2026 13:10:52 +0000 Subject: [PATCH 137/294] perf(txpool): use next_back to find the highest remaining nonce (#26964) --- crates/transaction-pool/src/pool/pending.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/transaction-pool/src/pool/pending.rs b/crates/transaction-pool/src/pool/pending.rs index b273758b86a..cf44baf3ea5 100644 --- a/crates/transaction-pool/src/pool/pending.rs +++ b/crates/transaction-pool/src/pool/pending.rs @@ -340,7 +340,7 @@ impl PendingPool { id.sender.start_bound(), std::ops::Bound::Included(TransactionId::new(id.sender, u64::MAX)), )) - .last() + .next_back() { // insert the new highest nonce for this sender entry.insert(new_highest.clone()); From d9762c8d6bd45ce25ba83012f9b989663a84a59a Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Thu, 3 Sep 2026 13:11:33 +0000 Subject: [PATCH 138/294] perf(trie): preallocate branch node hashes vec (#26961) --- crates/trie/sparse/src/arena/nodes.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/trie/sparse/src/arena/nodes.rs b/crates/trie/sparse/src/arena/nodes.rs index a00d271ebf8..df226831057 100644 --- a/crates/trie/sparse/src/arena/nodes.rs +++ b/crates/trie/sparse/src/arena/nodes.rs @@ -139,7 +139,7 @@ impl ArenaSparseNodeBranch { /// Returns a [`BranchNodeCompact`] from this branch's masks and children hashes. pub(super) fn branch_node_compact(&self, arena: &NodeArena) -> BranchNodeCompact { - let mut hashes = Vec::new(); + let mut hashes = Vec::with_capacity(self.branch_masks.hash_mask.count_bits() as usize); for (nibble, child) in self.child_iter() { if self.branch_masks.hash_mask.is_bit_set(nibble) { let hash = match child { From dfd976930ce5805f6046300294393bf6dfc49fa0 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Thu, 3 Sep 2026 13:13:39 +0000 Subject: [PATCH 139/294] perf(db): use FxHashMap for cached MDBX dbi lookups (#26967) --- crates/storage/db/src/implementation/mdbx/mod.rs | 4 ++-- crates/storage/db/src/implementation/mdbx/tx.rs | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/crates/storage/db/src/implementation/mdbx/mod.rs b/crates/storage/db/src/implementation/mdbx/mod.rs index b10ad2ff6be..6585cd1f9bf 100644 --- a/crates/storage/db/src/implementation/mdbx/mod.rs +++ b/crates/storage/db/src/implementation/mdbx/mod.rs @@ -22,8 +22,8 @@ use reth_libmdbx::{ }; use reth_storage_errors::db::LogLevel; use reth_tracing::tracing::error; +use rustc_hash::FxHashMap; use std::{ - collections::HashMap, ops::{Deref, Range}, path::{Path, PathBuf}, sync::Arc, @@ -251,7 +251,7 @@ pub struct DatabaseEnv { /// More generally, do not dynamically create, re-open, or drop tables at /// runtime. It's better to perform table creation and migration only once /// at startup. - dbis: Arc>, + dbis: Arc>, /// Cache for metric handles. If `None`, metrics are not recorded. metrics: Option>, /// Write lock for when dealing with a read-write environment. diff --git a/crates/storage/db/src/implementation/mdbx/tx.rs b/crates/storage/db/src/implementation/mdbx/tx.rs index 10b4ae3039c..23935907410 100644 --- a/crates/storage/db/src/implementation/mdbx/tx.rs +++ b/crates/storage/db/src/implementation/mdbx/tx.rs @@ -12,9 +12,9 @@ use reth_db_api::{ use reth_libmdbx::{ffi::MDBX_dbi, CommitLatency, Transaction, TransactionKind, WriteFlags, RW}; use reth_storage_errors::db::{DatabaseWriteError, DatabaseWriteOperation}; use reth_tracing::tracing::{debug, instrument, trace, warn}; +use rustc_hash::FxHashMap; use std::{ backtrace::Backtrace, - collections::HashMap, marker::PhantomData, sync::{ atomic::{AtomicBool, Ordering}, @@ -33,7 +33,7 @@ pub struct Tx { inner: Transaction, /// Cached MDBX DBIs for reuse. - dbis: Arc>, + dbis: Arc>, /// Handler for metrics with its own [Drop] implementation for cases when the transaction isn't /// closed by [`Tx::commit`] or [`Tx::abort`], but we still need to report it in the metrics. @@ -48,7 +48,7 @@ impl Tx { #[track_caller] pub(crate) fn new( inner: Transaction, - dbis: Arc>, + dbis: Arc>, env_metrics: Option>, ) -> reth_libmdbx::Result { let metrics_handler = env_metrics From 053703f7e27899dbc4a4363eec82a5bc1e1bc5e8 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Thu, 3 Sep 2026 13:14:22 +0000 Subject: [PATCH 140/294] perf(nippy-jar): allocate cursor scratch buffer lazily (#26975) --- crates/storage/nippy-jar/src/cursor.rs | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/crates/storage/nippy-jar/src/cursor.rs b/crates/storage/nippy-jar/src/cursor.rs index 505a00260b4..ffb65652280 100644 --- a/crates/storage/nippy-jar/src/cursor.rs +++ b/crates/storage/nippy-jar/src/cursor.rs @@ -12,7 +12,8 @@ pub struct NippyJarCursor<'a, H = ()> { jar: &'a NippyJar, /// Data and offset reader. reader: Arc, - /// Internal buffer to unload data to without reallocating memory on each retrieval. + /// Internal buffer to unload data to without reallocating memory on each retrieval. Only + /// compressed jars decompress into it, so it is sized on first use. internal_buffer: Vec, /// Cursor row position. row: u64, @@ -27,30 +28,21 @@ impl std::fmt::Debug for NippyJarCursor<'_, H> { impl<'a, H: NippyJarHeader> NippyJarCursor<'a, H> { /// Creates a new instance of [`NippyJarCursor`] for the given [`NippyJar`]. pub fn new(jar: &'a NippyJar) -> Result { - let max_row_size = jar.max_row_size; Ok(Self { jar, reader: Arc::new(jar.open_data_reader()?), - // Makes sure that we have enough buffer capacity to decompress any row of data. - internal_buffer: Vec::with_capacity(max_row_size), + internal_buffer: Vec::new(), row: 0, }) } /// Creates a new instance of [`NippyJarCursor`] with the specified [`NippyJar`] and data /// reader. - pub fn with_reader( + pub const fn with_reader( jar: &'a NippyJar, reader: Arc, ) -> Result { - let max_row_size = jar.max_row_size; - Ok(Self { - jar, - reader, - // Makes sure that we have enough buffer capacity to decompress any row of data. - internal_buffer: Vec::with_capacity(max_row_size), - row: 0, - }) + Ok(Self { jar, reader, internal_buffer: Vec::new(), row: 0 }) } /// Returns a reference to the related [`NippyJar`] @@ -162,6 +154,12 @@ impl<'a, H: NippyJarHeader> NippyJarCursor<'a, H> { }; if let Some(compression) = self.jar.compressor() { + // The decompressors write into the spare capacity of the buffer, so it has to fit any + // row of data. The buffer is only cleared between rows, so this reserves once. + if self.internal_buffer.capacity() < self.jar.max_row_size { + self.internal_buffer.reserve(self.jar.max_row_size - self.internal_buffer.len()); + } + let from = self.internal_buffer.len(); match compression { Compressors::Zstd(z) if z.use_dict => { From acc1197b75c1c0130154f2204f1cf02c87e837cf Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Thu, 3 Sep 2026 13:46:18 +0000 Subject: [PATCH 141/294] perf(txpool): track highest nonce per sender on fee rebuild (#26978) --- crates/transaction-pool/src/pool/pending.rs | 64 ++++++++++++++++++++- 1 file changed, 62 insertions(+), 2 deletions(-) diff --git a/crates/transaction-pool/src/pool/pending.rs b/crates/transaction-pool/src/pool/pending.rs index cf44baf3ea5..bbb92039306 100644 --- a/crates/transaction-pool/src/pool/pending.rs +++ b/crates/transaction-pool/src/pool/pending.rs @@ -178,7 +178,9 @@ impl PendingPool { // Drain and iterate over all transactions. let mut transactions_iter = self.clear_transactions().into_iter().peekable(); + let mut highest: Option = None; while let Some((id, tx)) = transactions_iter.next() { + self.flush_highest_nonce(&mut highest, Some(id.sender)); if tx.transaction.is_eip4844() && tx.transaction.max_fee_per_blob_gas() < Some(blob_fee) { // Add this tx to the removed collection since it no longer satisfies the blob fee @@ -195,10 +197,15 @@ impl PendingPool { } } else { self.size_of += tx.transaction.size(); - self.update_independents_and_highest_nonces(&tx); + if highest.is_none() { + // first kept transaction of this sender, hence the independent one + self.independent_transactions.insert(id.sender, tx.clone()); + } + highest = Some(id); self.by_id.insert(id, tx); } } + self.flush_highest_nonce(&mut highest, None); removed } @@ -221,7 +228,9 @@ impl PendingPool { // Drain and iterate over all transactions. let mut transactions_iter = self.clear_transactions().into_iter().peekable(); + let mut highest: Option = None; while let Some((id, mut tx)) = transactions_iter.next() { + self.flush_highest_nonce(&mut highest, Some(id.sender)); if tx.transaction.max_fee_per_gas() < base_fee as u128 { // Add this tx to the removed collection since it no longer satisfies the base fee // condition. Decrease the total pool size. @@ -240,14 +249,36 @@ impl PendingPool { tx.priority = self.ordering.priority(&tx.transaction.transaction, base_fee); self.size_of += tx.transaction.size(); - self.update_independents_and_highest_nonces(&tx); + if highest.is_none() { + // first kept transaction of this sender, hence the independent one + self.independent_transactions.insert(id.sender, tx.clone()); + } + highest = Some(id); self.by_id.insert(id, tx); } } + self.flush_highest_nonce(&mut highest, None); removed } + /// Writes the transaction tracked by a rebuild loop (`update_base_fee`, `update_blob_fee`) to + /// `highest_nonces` if the loop has moved on to `next_sender`. + /// + /// The pool is drained in ascending `(sender, nonce)` order, so the highest kept nonce of a + /// sender is only known once the drain reaches the next sender or the end of the pool. + fn flush_highest_nonce( + &mut self, + tracked: &mut Option, + next_sender: Option, + ) { + if let Some(id) = tracked.take_if(|id| Some(id.sender) != next_sender) && + let Some(tx) = self.by_id.get(&id).cloned() + { + self.highest_nonces.insert(id.sender, tx); + } + } + /// Updates the independent transaction and highest nonces set, assuming the given transaction /// is being _added_ to the pool. fn update_independents_and_highest_nonces(&mut self, tx: &PendingTransaction) { @@ -747,6 +778,35 @@ mod tests { pool.assert_invariants(); } + #[test] + fn test_enforce_basefee_tracks_highest_nonce_of_partially_removed_sender() { + let mut f = MockTransactionFactory::default(); + let mut pool = PendingPool::new(MockOrdering::default()); + + // sender a keeps its first transaction but its descendant is priced out + let a = MockTransaction::eip1559().inc_price_by(20); + let a0 = f.validated_arc(a.clone()); + let a1 = f.validated_arc(a.inc_nonce().rng_hash().decr_price_by(15)); + // sender b keeps both of its transactions + let b = MockTransaction::eip1559().inc_price_by(20); + let b0 = f.validated_arc(b.clone()); + let b1 = f.validated_arc(b.inc_nonce().rng_hash()); + + for tx in [&a0, &a1, &b0, &b1] { + pool.add_transaction(tx.clone(), 0); + } + + let removed = pool.update_base_fee((a1.max_fee_per_gas() + 1) as u64); + assert_eq!(removed.len(), 1); + assert_eq!(removed[0].hash(), a1.hash()); + + pool.assert_invariants(); + assert_eq!(pool.highest_nonces[&a0.sender_id()].transaction.nonce(), a0.nonce()); + assert_eq!(pool.independent_transactions[&a0.sender_id()].transaction.nonce(), a0.nonce()); + assert_eq!(pool.highest_nonces[&b1.sender_id()].transaction.nonce(), b1.nonce()); + assert_eq!(pool.independent_transactions[&b0.sender_id()].transaction.nonce(), b0.nonce()); + } + #[test] fn evict_worst() { let mut f = MockTransactionFactory::default(); From cfd72a3a25994801c07630a3604c8fe376252125 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Thu, 3 Sep 2026 15:25:58 +0000 Subject: [PATCH 142/294] perf(evm): reuse receipt blooms for the block logs bloom (#26939) --- crates/ethereum/evm/src/build.rs | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/crates/ethereum/evm/src/build.rs b/crates/ethereum/evm/src/build.rs index 263cf4e0b35..295f6880e72 100644 --- a/crates/ethereum/evm/src/build.rs +++ b/crates/ethereum/evm/src/build.rs @@ -9,7 +9,7 @@ use alloy_primitives::{Bloom, B256}; use reth_chainspec::{EthChainSpec, EthereumHardforks}; use reth_evm::execute::{BlockAssembler, BlockAssemblerInput, BlockExecutionError}; use reth_execution_types::BlockExecutionResult; -use reth_primitives_traits::{logs_bloom as calculate_logs_bloom, Receipt, SignedTransaction}; +use reth_primitives_traits::{Receipt, SignedTransaction}; use revm::context::Block as _; /// Block builder for Ethereum. @@ -58,11 +58,21 @@ impl EthBlockAssembler { let transactions_root = transactions_root.unwrap_or_else(|| proofs::calculate_transaction_root(&transactions)); - let receipts_root = receipts_root.unwrap_or_else(|| { - calculate_receipt_root(&receipts.iter().map(|r| r.with_bloom_ref()).collect::>()) - }); - let logs_bloom = logs_bloom - .unwrap_or_else(|| calculate_logs_bloom(receipts.iter().flat_map(|r| r.logs()))); + let (receipts_root, logs_bloom) = match (receipts_root, logs_bloom) { + (Some(receipts_root), Some(logs_bloom)) => (receipts_root, logs_bloom), + (receipts_root, logs_bloom) => { + // The block bloom is the OR of the receipt blooms, so the blooms computed for the + // receipts root are reused instead of hashing every log a second time. + let receipts_with_bloom = + receipts.iter().map(TxReceipt::with_bloom_ref).collect::>(); + ( + receipts_root.unwrap_or_else(|| calculate_receipt_root(&receipts_with_bloom)), + logs_bloom.unwrap_or_else(|| { + receipts_with_bloom.iter().fold(Bloom::ZERO, |acc, r| acc | r.bloom_ref()) + }), + ) + } + }; let withdrawals = self .chain_spec From 46474854ab6a86d76a8119d70d1dee23dcd0811a Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Thu, 3 Sep 2026 15:26:12 +0000 Subject: [PATCH 143/294] perf(engine): skip prewarm tx clones when nothing consumes them (#26944) --- .../tree/src/tree/payload_processor/mod.rs | 74 ++++++++++++++----- 1 file changed, 57 insertions(+), 17 deletions(-) diff --git a/crates/engine/tree/src/tree/payload_processor/mod.rs b/crates/engine/tree/src/tree/payload_processor/mod.rs index c12855aa07e..73dcdb1db9d 100644 --- a/crates/engine/tree/src/tree/payload_processor/mod.rs +++ b/crates/engine/tree/src/tree/payload_processor/mod.rs @@ -186,8 +186,14 @@ where + HistoryReader + 'static, { - let (prewarm_rx, execution_rx) = - self.spawn_tx_iterator(transactions, env.transaction_count, parallel_bal_execution); + let prewarm_transactions = + self.prewarms_transactions(env.transaction_count, parallel_bal_execution); + let (prewarm_rx, execution_rx) = self.spawn_tx_iterator( + transactions, + env.transaction_count, + parallel_bal_execution, + prewarm_transactions, + ); let prewarm_handle = self.spawn_caching_with( env, prewarm_rx, @@ -199,6 +205,22 @@ where PayloadHandle { prewarm_handle, transactions: execution_rx, _span: Span::current() } } + /// Whether the prewarm task will consume converted transactions, i.e. whether + /// [`Self::spawn_caching_with`] ends up in [`PrewarmMode::Transactions`]. + /// + /// This is the only place the decision is made: the tx iterator uses it to skip creating the + /// prewarm channel and cloning every transaction into it, and the resulting `Option` receiver + /// then selects the mode, so the two cannot disagree. + const fn prewarms_transactions( + &self, + transaction_count: usize, + parallel_bal_execution: bool, + ) -> bool { + !parallel_bal_execution && + !self.disable_transaction_prewarming && + transaction_count >= SMALL_BLOCK_TX_THRESHOLD + } + /// Transaction count threshold below which sequential conversion is used. /// /// For blocks with fewer than this many transactions, the rayon parallel iterator overhead @@ -227,14 +249,19 @@ where /// /// When `parallel_bal_execution` is disabled, preserves the original transaction order. /// Otherwise, streams results as they become available. + /// + /// The prewarm channel is only created when `prewarm_transactions` is set, see + /// [`Self::prewarms_transactions`]; otherwise no transaction is cloned into it. #[instrument(level = "debug", target = "engine::tree::payload_processor", skip_all)] fn spawn_tx_iterator>( &self, transactions: I, transaction_count: usize, parallel_bal_execution: bool, - ) -> (IteratorPrewarmTxReceiver, IteratorExecuteTxReceiver) { - let (prewarm_tx, prewarm_rx) = mpsc::sync_channel(transaction_count); + prewarm_transactions: bool, + ) -> (Option>, IteratorExecuteTxReceiver) { + let (prewarm_tx, prewarm_rx) = + prewarm_transactions.then(|| mpsc::sync_channel(transaction_count)).unzip(); let (execute_tx, execute_rx) = crossbeam_channel::bounded(transaction_count); if transaction_count == 0 { @@ -249,7 +276,12 @@ where ); self.executor.spawn_blocking_named("tx-iterator", move || { let (transactions, convert) = transactions.into_parts(); - convert_serial(transactions.into_iter(), &convert, &prewarm_tx, &execute_tx); + convert_serial( + transactions.into_iter(), + &convert, + prewarm_tx.as_ref(), + &execute_tx, + ); }); } else { // Parallel path — recover signatures in parallel on rayon, stream results @@ -271,7 +303,9 @@ where .for_each(|(idx, tx)| { let tx = tx.map(|tx| { let tx = WithTxEnv::new(tx); - let _ = prewarm_tx.send((idx, tx.clone())); + if let Some(prewarm_tx) = &prewarm_tx { + let _ = prewarm_tx.send((idx, tx.clone())); + } tx }); let _ = execute_tx.send((idx, tx)); @@ -287,7 +321,12 @@ where // Convert the first few transactions sequentially so execution can // start immediately without waiting for rayon work-stealing. - convert_serial(iter.by_ref().take(prefetch), &convert, &prewarm_tx, &execute_tx); + convert_serial( + iter.by_ref().take(prefetch), + &convert, + prewarm_tx.as_ref(), + &execute_tx, + ); let mut iter = iter.enumerate(); @@ -317,7 +356,7 @@ where .collect::>(); for (idx, tx) in chunk { - if let Ok(tx) = &tx { + if let (Some(prewarm_tx), Ok(tx)) = (&prewarm_tx, &tx) { let _ = prewarm_tx.send((idx, tx.clone())); } let _ = execute_tx.send((idx, tx)); @@ -341,7 +380,9 @@ where fn spawn_caching_with

( &self, env: ExecutionEnv, - transactions: mpsc::Receiver<(usize, impl ExecutableTxFor + Clone + Send + 'static)>, + transactions: Option< + mpsc::Receiver<(usize, impl ExecutableTxFor + Clone + Send + 'static)>, + >, state_provider_factory: OverlayStateProviderFactory, hint_stream: Option, hashed_update_stream: Option, @@ -365,12 +406,10 @@ where bal: env.decoded_bal.clone().expect("BAL dispatch implies decoded BAL"), updates: hashed_update_stream, } - } else if self.disable_transaction_prewarming || - env.transaction_count < SMALL_BLOCK_TX_THRESHOLD - { - PrewarmMode::Skipped + } else if let Some(pending) = transactions { + PrewarmMode::Transactions { pending, hints: hint_stream } } else { - PrewarmMode::Transactions { pending: transactions, hints: hint_stream } + PrewarmMode::Skipped }; let saved_cache = self.disable_state_cache.not().then(|| self.cache_for(env.parent_hash)); @@ -485,11 +524,12 @@ where } } -/// Converts transactions sequentially and sends them to the prewarm and execute channels. +/// Converts transactions sequentially and sends them to the execute channel, and to the prewarm +/// channel if there is one. fn convert_serial( iter: impl Iterator, convert: &C, - prewarm_tx: &mpsc::SyncSender<(usize, WithTxEnv)>, + prewarm_tx: Option<&mpsc::SyncSender<(usize, WithTxEnv)>>, execute_tx: &ExecuteTxSender, ) where Tx: ExecutableTxParts, @@ -499,7 +539,7 @@ fn convert_serial( for (idx, raw_tx) in iter.enumerate() { let tx = convert.convert(raw_tx); let tx = tx.map(|tx| WithTxEnv::new(tx)); - if let Ok(tx) = &tx { + if let (Some(prewarm_tx), Ok(tx)) = (prewarm_tx, &tx) { let _ = prewarm_tx.send((idx, tx.clone())); } let _ = execute_tx.send((idx, tx)); From c05b6fbf47be50d6968d38570b34bc789b503ee6 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Thu, 3 Sep 2026 15:54:47 +0000 Subject: [PATCH 144/294] perf(static-files): return jar rows without allocating (#26951) --- Cargo.lock | 1 + crates/storage/db/src/static_file/cursor.rs | 4 ++-- crates/storage/nippy-jar/Cargo.toml | 1 + crates/storage/nippy-jar/src/cursor.rs | 19 ++++++++++++------- crates/storage/nippy-jar/src/lib.rs | 6 +++++- 5 files changed, 21 insertions(+), 10 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d97840c0201..8a5c05d173e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9317,6 +9317,7 @@ dependencies = [ "rand 0.9.4", "reth-fs-util", "serde", + "smallvec", "tempfile", "thiserror 2.0.18", "tracing", diff --git a/crates/storage/db/src/static_file/cursor.rs b/crates/storage/db/src/static_file/cursor.rs index fc29892b37b..3a058ba0d07 100644 --- a/crates/storage/db/src/static_file/cursor.rs +++ b/crates/storage/db/src/static_file/cursor.rs @@ -2,7 +2,7 @@ use super::mask::{ColumnSelectorOne, ColumnSelectorThree, ColumnSelectorTwo}; use alloy_primitives::B256; use derive_more::{Deref, DerefMut}; use reth_db_api::table::Decompress; -use reth_nippy_jar::{DataReader, NippyJar, NippyJarCursor}; +use reth_nippy_jar::{DataReader, NippyJar, NippyJarCursor, RefRow}; use reth_static_file_types::SegmentHeader; use reth_storage_errors::provider::{ProviderError, ProviderResult}; use std::sync::Arc; @@ -31,7 +31,7 @@ impl<'a> StaticFileCursor<'a> { &mut self, key_or_num: KeyOrNumber<'_>, mask: usize, - ) -> ProviderResult>> { + ) -> ProviderResult>> { if self.jar().rows() == 0 { return Ok(None) } diff --git a/crates/storage/nippy-jar/Cargo.toml b/crates/storage/nippy-jar/Cargo.toml index be6699d19c3..c7c8e83e025 100644 --- a/crates/storage/nippy-jar/Cargo.toml +++ b/crates/storage/nippy-jar/Cargo.toml @@ -24,6 +24,7 @@ lz4_flex.workspace = true memmap2.workspace = true bincode.workspace = true +smallvec.workspace = true serde = { workspace = true, features = ["derive"] } tracing.workspace = true anyhow.workspace = true diff --git a/crates/storage/nippy-jar/src/cursor.rs b/crates/storage/nippy-jar/src/cursor.rs index ffb65652280..4fc74a57264 100644 --- a/crates/storage/nippy-jar/src/cursor.rs +++ b/crates/storage/nippy-jar/src/cursor.rs @@ -2,9 +2,18 @@ use crate::{ compression::{Compression, Compressors, Zstd}, DataReader, NippyJar, NippyJarError, NippyJarHeader, RefRow, }; +use smallvec::SmallVec; use std::{ops::Range, sync::Arc}; use zstd::bulk::Decompressor; +/// The column value ranges of a single row, mirroring the inline capacity of [`RefRow`]. +/// +/// The ranges are collected before they are resolved into slices because [`read_value`] borrows the +/// internal buffer mutably while filling it. +/// +/// [`read_value`]: NippyJarCursor::read_value +type ValueRanges = SmallVec<[ValueRange; 4]>; + /// Simple cursor implementation to retrieve data from [`NippyJar`]. #[derive(Clone)] pub struct NippyJarCursor<'a, H = ()> { @@ -75,7 +84,7 @@ impl<'a, H: NippyJarHeader> NippyJarCursor<'a, H> { return Ok(None) } - let mut row = Vec::with_capacity(self.jar.columns); + let mut row = ValueRanges::with_capacity(self.jar.columns); // Retrieve all column values from the row for column in 0..self.jar.columns { @@ -116,7 +125,7 @@ impl<'a, H: NippyJarHeader> NippyJarCursor<'a, H> { } let columns = self.jar.columns; - let mut row = Vec::with_capacity(columns); + let mut row = ValueRanges::with_capacity(columns); for column in 0..columns { if mask & (1 << column) != 0 { @@ -136,11 +145,7 @@ impl<'a, H: NippyJarHeader> NippyJarCursor<'a, H> { } /// Takes the column index and reads the range value for the corresponding column. - fn read_value( - &mut self, - column: usize, - row: &mut Vec, - ) -> Result<(), NippyJarError> { + fn read_value(&mut self, column: usize, row: &mut ValueRanges) -> Result<(), NippyJarError> { // Find out the offset of the column value let offset_pos = self.row as usize * self.jar.columns + column; let value_offset = self.reader.offset(offset_pos)? as usize; diff --git a/crates/storage/nippy-jar/src/lib.rs b/crates/storage/nippy-jar/src/lib.rs index 3c579651458..c1d6cefb316 100644 --- a/crates/storage/nippy-jar/src/lib.rs +++ b/crates/storage/nippy-jar/src/lib.rs @@ -14,6 +14,7 @@ use memmap2::Mmap; use serde::{Deserialize, Serialize}; +use smallvec::SmallVec; use std::{ error::Error as StdError, fs::File, @@ -64,7 +65,10 @@ pub const CHANGESET_OFFSETS_FILE_EXTENSION: &str = "csoff"; /// A [`RefRow`] is a list of column value slices pointing to either an internal buffer or a /// memory-mapped file. -type RefRow<'a> = Vec<&'a [u8]>; +/// +/// The inline capacity covers every segment used by reth (at most three columns), so reading a row +/// does not allocate. +pub type RefRow<'a> = SmallVec<[&'a [u8]; 4]>; /// Alias type for a column value wrapped in `Result`. pub type ColumnResult = Result>; From 2e6e838b938f636409bcbb88454d2b1c3e0b06a1 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Thu, 3 Sep 2026 18:09:51 +0200 Subject: [PATCH 145/294] test(txpool): assert per-sender side map contents in pending pool (#26989) --- crates/transaction-pool/src/pool/pending.rs | 90 ++++++++++++++++++--- 1 file changed, 77 insertions(+), 13 deletions(-) diff --git a/crates/transaction-pool/src/pool/pending.rs b/crates/transaction-pool/src/pool/pending.rs index bbb92039306..272ede0546f 100644 --- a/crates/transaction-pool/src/pool/pending.rs +++ b/crates/transaction-pool/src/pool/pending.rs @@ -187,7 +187,8 @@ impl PendingPool { // condition. Decrease the total pool size. removed.push(Arc::clone(&tx.transaction)); - // Remove all dependent transactions. + // Remove all dependent transactions. Draining the rest of the sender here is what + // keeps the tracked `highest` correct, see `flush_highest_nonce`. 'this: while let Some((next_id, next_tx)) = transactions_iter.peek() { if next_id.sender != id.sender { break 'this @@ -236,7 +237,8 @@ impl PendingPool { // condition. Decrease the total pool size. removed.push(Arc::clone(&tx.transaction)); - // Remove all dependent transactions. + // Remove all dependent transactions. Draining the rest of the sender here is what + // keeps the tracked `highest` correct, see `flush_highest_nonce`. 'this: while let Some((next_id, next_tx)) = transactions_iter.peek() { if next_id.sender != id.sender { break 'this @@ -266,7 +268,10 @@ impl PendingPool { /// `highest_nonces` if the loop has moved on to `next_sender`. /// /// The pool is drained in ascending `(sender, nonce)` order, so the highest kept nonce of a - /// sender is only known once the drain reaches the next sender or the end of the pool. + /// sender is only known once the drain reaches the next sender or the end of the pool. This + /// relies on the loops removing all remaining transactions of a sender once one of them no + /// longer satisfies the fee, otherwise the tracked transaction would not be the highest kept + /// one. fn flush_highest_nonce( &mut self, tracked: &mut Option, @@ -637,21 +642,56 @@ impl PendingPool { &self.independent_transactions } - /// Asserts that the bijection between `by_id` and `all` is valid. + /// Asserts that the per-sender side maps are consistent with `by_id`. + /// + /// Every sender in `by_id` must be tracked with its lowest nonce transaction in + /// `independent_transactions` and its highest nonce transaction in `highest_nonces`, no other + /// sender may be tracked, and the tracked size must match the size of all transactions. #[cfg(any(test, feature = "test-utils"))] pub(crate) fn assert_invariants(&self) { - assert!( - self.independent_transactions.len() <= self.by_id.len(), - "independent_transactions.len() > by_id.len()" - ); - assert!( - self.highest_nonces.len() <= self.by_id.len(), - "highest_nonces.len() > by_id.len()" + let mut senders = 0; + let mut txs = self.by_id.iter().peekable(); + while let Some((lowest_id, lowest)) = txs.next() { + let sender = lowest_id.sender; + senders += 1; + + let mut highest = lowest; + while let Some((_, tx)) = txs.next_if(|(id, _)| id.sender == sender) { + highest = tx; + } + + let independent = self + .independent_transactions + .get(&sender) + .unwrap_or_else(|| panic!("no independent transaction tracked for {sender:?}")); + assert_eq!( + independent, lowest, + "independent transaction of {sender:?} does not match its lowest nonce pool entry" + ); + let tracked_highest = self + .highest_nonces + .get(&sender) + .unwrap_or_else(|| panic!("no highest nonce tracked for {sender:?}")); + assert_eq!( + tracked_highest, highest, + "highest nonce of {sender:?} does not match its highest nonce pool entry" + ); + } + + assert_eq!( + self.independent_transactions.len(), + senders, + "independent_transactions tracks senders without transactions" ); assert_eq!( self.highest_nonces.len(), - self.independent_transactions.len(), - "highest_nonces.len() != independent_transactions.len()" + senders, + "highest_nonces tracks senders without transactions" + ); + assert_eq!( + self.size(), + self.by_id.values().map(|tx| tx.transaction.size()).sum::(), + "size_of does not match the size of all transactions" ); } } @@ -807,6 +847,30 @@ mod tests { assert_eq!(pool.independent_transactions[&b0.sender_id()].transaction.nonce(), b0.nonce()); } + #[test] + fn test_enforce_blobfee_tracks_highest_nonce_of_partially_removed_sender() { + let mut f = MockTransactionFactory::default(); + let mut pool = PendingPool::new(MockOrdering::default()); + + // sender a keeps its first transaction but its descendant is priced out + let a = MockTransaction::eip4844().with_blob_fee(150); + let a0 = f.validated_arc(a.clone()); + let a1 = f.validated_arc(a.inc_nonce().rng_hash().with_blob_fee(50)); + // sender b keeps both of its transactions + let b = MockTransaction::eip4844().with_blob_fee(150); + let b0 = f.validated_arc(b.clone()); + let b1 = f.validated_arc(b.inc_nonce().rng_hash()); + + for tx in [&a0, &a1, &b0, &b1] { + pool.add_transaction(tx.clone(), 0); + } + + let removed = pool.update_blob_fee(100); + assert_eq!(removed.len(), 1); + assert_eq!(removed[0].hash(), a1.hash()); + pool.assert_invariants(); + } + #[test] fn evict_worst() { let mut f = MockTransactionFactory::default(); From 1ca1201167f61ea7eec099c7d1ea59e828d0d515 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Thu, 3 Sep 2026 16:20:12 +0000 Subject: [PATCH 146/294] perf(rpc): bloom-filter blocks in log subscriptions and getLogs by hash (#26946) --- crates/rpc/rpc-eth-types/src/logs_utils.rs | 6 ++++++ crates/rpc/rpc/src/eth/filter.rs | 4 ++++ 2 files changed, 10 insertions(+) diff --git a/crates/rpc/rpc-eth-types/src/logs_utils.rs b/crates/rpc/rpc-eth-types/src/logs_utils.rs index 495e4e15673..062c7a0db9a 100644 --- a/crates/rpc/rpc-eth-types/src/logs_utils.rs +++ b/crates/rpc/rpc-eth-types/src/logs_utils.rs @@ -36,6 +36,12 @@ where return Ok(vec![]) } + // An empty address/topic set matches any bloom, so this can only skip blocks that have no + // matching logs. + if !filter.matches_bloom(header.logs_bloom()) { + return Ok(vec![]) + } + let mut all_logs = Vec::new(); // Tracks the index of a log in the entire block. let mut log_index: u64 = 0; diff --git a/crates/rpc/rpc/src/eth/filter.rs b/crates/rpc/rpc/src/eth/filter.rs index 46cb1d53d9a..5a9b0e7492c 100644 --- a/crates/rpc/rpc/src/eth/filter.rs +++ b/crates/rpc/rpc/src/eth/filter.rs @@ -512,6 +512,10 @@ where .into()); } + if !filter.matches_bloom(header.logs_bloom()) { + return Ok(Vec::new()) + } + let mut all_logs = Vec::new(); append_matching_block_logs( &mut all_logs, From 8a8a0b75e7fa566e8f0357cc4b5a98211c6c9b7a Mon Sep 17 00:00:00 2001 From: Derek Cofausper <256792747+decofe@users.noreply.github.com> Date: Thu, 3 Sep 2026 16:54:51 +0000 Subject: [PATCH 147/294] refactor(db): use FastInstant alias for metrics (#26992) Co-authored-by: DaniPopes <57450786+DaniPopes@users.noreply.github.com> --- Cargo.lock | 1 - Cargo.toml | 1 - crates/storage/db/Cargo.toml | 4 ++-- crates/storage/db/src/metrics.rs | 2 +- 4 files changed, 3 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8a5c05d173e..ac5deab64f2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8076,7 +8076,6 @@ dependencies = [ "page_size", "parking_lot", "proptest", - "quanta", "reth-db-api", "reth-fs-util", "reth-libmdbx", diff --git a/Cargo.toml b/Cargo.toml index 3b2206ec662..282a2ac8faa 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -513,7 +513,6 @@ notify = { version = "8.0.0", default-features = false, features = ["macos_fseve nybbles = { version = "0.4.8", default-features = false } once_cell = { version = "1.19", default-features = false, features = ["critical-section"] } parking_lot = "0.12" -quanta = "0.12" paste = "1.0" rand = "0.9" rayon = "1.7" diff --git a/crates/storage/db/Cargo.toml b/crates/storage/db/Cargo.toml index a72b2a4ca3a..6c01c4b62f7 100644 --- a/crates/storage/db/Cargo.toml +++ b/crates/storage/db/Cargo.toml @@ -20,6 +20,7 @@ reth-nippy-jar.workspace = true reth-tracing.workspace = true tracing.workspace = true reth-static-file-types.workspace = true +reth-primitives-traits = { workspace = true, optional = true, features = ["quanta"] } # ethereum alloy-primitives.workspace = true @@ -31,7 +32,6 @@ eyre = { workspace = true, optional = true } # metrics reth-metrics = { workspace = true, optional = true } metrics = { workspace = true, optional = true } -quanta = { workspace = true, optional = true } # misc page_size = { workspace = true, optional = true } @@ -73,7 +73,7 @@ mdbx = [ "dep:reth-libmdbx", "dep:eyre", "dep:page_size", - "dep:quanta", + "dep:reth-primitives-traits", "reth-metrics", "dep:metrics", "dep:strum", diff --git a/crates/storage/db/src/metrics.rs b/crates/storage/db/src/metrics.rs index 84f7b52d518..8bd2daa4524 100644 --- a/crates/storage/db/src/metrics.rs +++ b/crates/storage/db/src/metrics.rs @@ -1,7 +1,7 @@ use crate::Tables; use metrics::Histogram; -use quanta::Instant; use reth_metrics::{metrics::Counter, Metrics}; +use reth_primitives_traits::FastInstant as Instant; use rustc_hash::FxHashMap; use std::{array, sync::Arc, time::Duration}; use strum::{EnumCount, EnumIter, IntoEnumIterator}; From 972ad6194e63527c0eaa0edf393a23110ce50349 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Thu, 3 Sep 2026 17:04:46 +0000 Subject: [PATCH 148/294] perf(tracing): avoid allocating formatted log fields (#26953) --- crates/era-utils/src/history.rs | 2 +- crates/node/events/src/node.rs | 4 ++-- crates/stages/stages/src/stages/hashing_account.rs | 2 +- crates/stages/stages/src/stages/hashing_storage.rs | 2 +- crates/stages/stages/src/stages/headers.rs | 4 ++-- crates/stages/stages/src/stages/tx_lookup.rs | 2 +- crates/stages/stages/src/stages/utils.rs | 6 +++--- 7 files changed, 11 insertions(+), 11 deletions(-) diff --git a/crates/era-utils/src/history.rs b/crates/era-utils/src/history.rs index 364335a120e..618e88cf35a 100644 --- a/crates/era-utils/src/history.rs +++ b/crates/era-utils/src/history.rs @@ -826,7 +826,7 @@ where let (hash, number) = hash_to_number?; if index != 0 && index.is_multiple_of(interval) { - info!(target: "era::history::import", progress = %format!("{:.2}%", (index as f64 / total_headers as f64) * 100.0), "Writing headers hash index"); + info!(target: "era::history::import", progress = %format_args!("{:.2}%", (index as f64 / total_headers as f64) * 100.0), "Writing headers hash index"); } let hash = RawKey::::from_vec(hash); diff --git a/crates/node/events/src/node.rs b/crates/node/events/src/node.rs index 7ee9ef5660f..7806316f56e 100644 --- a/crates/node/events/src/node.rs +++ b/crates/node/events/src/node.rs @@ -252,8 +252,8 @@ impl NodeState { gas_used=%format_gas(block.gas_used()), gas_throughput=%format_gas_throughput(block.gas_used(), elapsed), gas_limit=%format_gas(block.gas_limit()), - full=%format!("{:.1}%", full), - base_fee=%format!("{:.2}Gwei", block.base_fee_per_gas().unwrap_or(0) as f64 / GWEI_TO_WEI as f64), + full=%format_args!("{:.1}%", full), + base_fee=%format_args!("{:.2}Gwei", block.base_fee_per_gas().unwrap_or(0) as f64 / GWEI_TO_WEI as f64), blobs=block.blob_gas_used().unwrap_or(0) / alloy_eips::eip4844::DATA_GAS_PER_BLOB, excess_blobs=block.excess_blob_gas().unwrap_or(0) / alloy_eips::eip4844::DATA_GAS_PER_BLOB, ?elapsed, diff --git a/crates/stages/stages/src/stages/hashing_account.rs b/crates/stages/stages/src/stages/hashing_account.rs index 2410e8131fe..d060a67b3ea 100644 --- a/crates/stages/stages/src/stages/hashing_account.rs +++ b/crates/stages/stages/src/stages/hashing_account.rs @@ -219,7 +219,7 @@ where if index > 0 && index.is_multiple_of(interval) { info!( target: "sync::stages::hashing_account", - progress = %format!("{:.2}%", (index as f64 / total_hashes as f64) * 100.0), + progress = %format_args!("{:.2}%", (index as f64 / total_hashes as f64) * 100.0), "Inserting hashes" ); } diff --git a/crates/stages/stages/src/stages/hashing_storage.rs b/crates/stages/stages/src/stages/hashing_storage.rs index 59913bb452b..e2680636917 100644 --- a/crates/stages/stages/src/stages/hashing_storage.rs +++ b/crates/stages/stages/src/stages/hashing_storage.rs @@ -154,7 +154,7 @@ where if index > 0 && index.is_multiple_of(interval) { info!( target: "sync::stages::hashing_storage", - progress = %format!("{:.2}%", (index as f64 / total_hashes as f64) * 100.0), + progress = %format_args!("{:.2}%", (index as f64 / total_hashes as f64) * 100.0), "Inserting hashes" ); } diff --git a/crates/stages/stages/src/stages/headers.rs b/crates/stages/stages/src/stages/headers.rs index f9ca2a86f3a..6b8a8c9d361 100644 --- a/crates/stages/stages/src/stages/headers.rs +++ b/crates/stages/stages/src/stages/headers.rs @@ -123,7 +123,7 @@ where let (_, header_buf) = header?; if index > 0 && index.is_multiple_of(interval) && total_headers > 100 { - info!(target: "sync::stages::headers", progress = %format!("{:.2}%", (index as f64 / total_headers as f64) * 100.0), "Writing headers"); + info!(target: "sync::stages::headers", progress = %format_args!("{:.2}%", (index as f64 / total_headers as f64) * 100.0), "Writing headers"); } let sealed_header: SealedHeader = SealedHeader::new_unhashed( @@ -164,7 +164,7 @@ where let (hash, number) = hash_to_number?; if index > 0 && index.is_multiple_of(interval) && total_headers > 100 { - info!(target: "sync::stages::headers", progress = %format!("{:.2}%", (index as f64 / total_headers as f64) * 100.0), "Writing headers hash index"); + info!(target: "sync::stages::headers", progress = %format_args!("{:.2}%", (index as f64 / total_headers as f64) * 100.0), "Writing headers hash index"); } if first_sync { diff --git a/crates/stages/stages/src/stages/tx_lookup.rs b/crates/stages/stages/src/stages/tx_lookup.rs index 9eae2084e3d..ae808a0e7c8 100644 --- a/crates/stages/stages/src/stages/tx_lookup.rs +++ b/crates/stages/stages/src/stages/tx_lookup.rs @@ -175,7 +175,7 @@ where info!( target: "sync::stages::transaction_lookup", ?append_only, - progress = %format!("{:.2}%", (index as f64 / total_hashes as f64) * 100.0), + progress = %format_args!("{:.2}%", (index as f64 / total_hashes as f64) * 100.0), "Inserting hashes" ); } diff --git a/crates/stages/stages/src/stages/utils.rs b/crates/stages/stages/src/stages/utils.rs index ab6f19fc73e..da8d3f1fe1b 100644 --- a/crates/stages/stages/src/stages/utils.rs +++ b/crates/stages/stages/src/stages/utils.rs @@ -84,7 +84,7 @@ where cache.entry(key).or_default().push(block_number); if idx > 0 && idx.is_multiple_of(interval) && total_changesets > 1000 { - info!(target: "sync::stages::index_history", progress = %format!("{:.4}%", (idx as f64 / total_changesets as f64) * 100.0), "Collecting indices"); + info!(target: "sync::stages::index_history", progress = %format_args!("{:.4}%", (idx as f64 / total_changesets as f64) * 100.0), "Collecting indices"); } // Make sure we only flush the cache every DEFAULT_CACHE_THRESHOLD blocks. @@ -262,7 +262,7 @@ where let new_list = BlockNumberList::decompress_owned(v)?; if index > 0 && index.is_multiple_of(interval) && total_entries > 10 { - info!(target: "sync::stages::index_history", progress = %format!("{:.2}%", (index as f64 / total_entries as f64) * 100.0), "Writing indices"); + info!(target: "sync::stages::index_history", progress = %format_args!("{:.2}%", (index as f64 / total_entries as f64) * 100.0), "Writing indices"); } let address = sharded_key.key; @@ -470,7 +470,7 @@ where let new_list = BlockNumberList::decompress_owned(v)?; if index > 0 && index.is_multiple_of(interval) && total_entries > 10 { - info!(target: "sync::stages::index_history", progress = %format!("{:.2}%", (index as f64 / total_entries as f64) * 100.0), "Writing indices"); + info!(target: "sync::stages::index_history", progress = %format_args!("{:.2}%", (index as f64 / total_entries as f64) * 100.0), "Writing indices"); } let partial_key = (sharded_key.address, sharded_key.sharded_key.key); From c29ab4319891d14aa1800eb4759fd05da9a6a9ee Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Thu, 3 Sep 2026 17:52:30 +0000 Subject: [PATCH 149/294] perf(db): use TSC-backed clock for transaction start time (#26941) Co-authored-by: Matthias Seitz <19890894+mattsse@users.noreply.github.com> Co-authored-by: Derek Cofausper <256792747+decofe@users.noreply.github.com> --- crates/storage/db/src/implementation/mdbx/tx.rs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/crates/storage/db/src/implementation/mdbx/tx.rs b/crates/storage/db/src/implementation/mdbx/tx.rs index 23935907410..396233508f8 100644 --- a/crates/storage/db/src/implementation/mdbx/tx.rs +++ b/crates/storage/db/src/implementation/mdbx/tx.rs @@ -10,6 +10,7 @@ use reth_db_api::{ transaction::{DbTx, DbTxMut}, }; use reth_libmdbx::{ffi::MDBX_dbi, CommitLatency, Transaction, TransactionKind, WriteFlags, RW}; +use reth_primitives_traits::FastInstant as Instant; use reth_storage_errors::db::{DatabaseWriteError, DatabaseWriteOperation}; use reth_tracing::tracing::{debug, instrument, trace, warn}; use rustc_hash::FxHashMap; @@ -20,7 +21,7 @@ use std::{ atomic::{AtomicBool, Ordering}, Arc, }, - time::{Duration, Instant}, + time::Duration, }; /// Duration after which we emit the log about long-lived database transactions. @@ -177,6 +178,11 @@ struct MetricsHandler { /// Cached internal transaction ID provided by libmdbx. txn_id: u64, /// The time when transaction has started. + /// + /// This is a TSC-backed [`reth_primitives_traits::FastInstant`] rather than + /// [`std::time::Instant`] because + /// [`MetricsHandler::log_backtrace_on_long_read_transaction`] reads it on every database + /// operation. start: Instant, /// Duration after which we emit the log about long-lived database transactions. long_transaction_duration: Duration, From 032cafc63fae3e1253825954634b21c60ff679f6 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Thu, 3 Sep 2026 23:22:49 +0200 Subject: [PATCH 150/294] perf(net): avoid quadratic cells sizing and presize responses (#26965) --- crates/net/network/src/eth_requests.rs | 52 ++++++++++++++++++++++++-- 1 file changed, 48 insertions(+), 4 deletions(-) diff --git a/crates/net/network/src/eth_requests.rs b/crates/net/network/src/eth_requests.rs index 5d1172e31f8..9098a352913 100644 --- a/crates/net/network/src/eth_requests.rs +++ b/crates/net/network/src/eth_requests.rs @@ -80,6 +80,20 @@ pub const MAX_STORAGE_RANGE_ACCOUNTS_SERVE: usize = 1024; /// Maximum size of replies to data retrievals: 2MB pub const SOFT_RESPONSE_LIMIT: usize = 2 * 1024 * 1024; +/// Upper bound on the headers reserved once the first requested header resolves. +/// +/// Header responses are bound by [`MAX_HEADERS_SERVE`] rather than by size, so a full batch reaches +/// its final capacity within a few doublings while a batch that runs past the local tip wastes at +/// most this many slots. +const MAX_HEADERS_RESERVE: usize = 64; + +/// Upper bound on the block bodies or receipt lists reserved once the first requested block +/// resolves. +/// +/// These responses are bound by [`SOFT_RESPONSE_LIMIT`], so the first block's size predicts how +/// many fit and this only caps the estimate for tiny blocks. +const MAX_BLOCKS_RESERVE: usize = 16; + /// Manages eth related requests on top of the p2p network. /// /// This can be spawned to another task and is supposed to be run as background service. @@ -148,8 +162,14 @@ where if let Some(header) = self.client.header_by_hash_or_number(block).unwrap_or_default() { let number = header.number(); let parent_hash = header.parent_hash(); + let len = header.length(); - total_bytes += header.length(); + if headers.is_empty() { + let count = limit.min(MAX_HEADERS_SERVE as u64) as usize; + headers.reserve(response_reserve_hint(count, len, MAX_HEADERS_RESERVE)); + } + + total_bytes += len; headers.push(header); if headers.len() >= MAX_HEADERS_SERVE || total_bytes > SOFT_RESPONSE_LIMIT { @@ -207,6 +227,7 @@ where response: oneshot::Sender::Body>>>, ) { self.metrics.eth_bodies_requests_received_total.increment(1); + let count = request.0.len(); let mut bodies = Vec::new(); let mut total_bytes = 0; @@ -214,7 +235,11 @@ where for hash in request { if let Some(block) = self.client.block_by_hash(hash).unwrap_or_default() { let body = block.into_body(); - total_bytes += body.length(); + let len = body.length(); + if bodies.is_empty() { + bodies.reserve(response_reserve_hint(count, len, MAX_BLOCKS_RESERVE)); + } + total_bytes += len; bodies.push(body); if bodies.len() >= MAX_BODIES_SERVE || total_bytes > SOFT_RESPONSE_LIMIT { @@ -287,6 +312,7 @@ where let GetReceipts70 { first_block_receipt_index, block_hashes } = request; + let count = block_hashes.len(); let mut receipts = Vec::new(); let mut total_bytes = 0usize; let mut last_block_incomplete = false; @@ -312,6 +338,9 @@ where } let block_size = block_receipts.length(); + if receipts.is_empty() { + receipts.reserve(response_reserve_hint(count, block_size, MAX_BLOCKS_RESERVE)); + } if total_bytes + block_size <= SOFT_RESPONSE_LIMIT { total_bytes += block_size; @@ -343,6 +372,7 @@ where F: Fn(Vec) -> Vec, T: Encodable, { + let count = request.0.len(); let mut receipts = Vec::new(); let mut total_bytes = 0; @@ -351,7 +381,11 @@ where self.client.receipts_by_block(BlockHashOrNumber::Hash(hash)).unwrap_or_default() { let transformed_receipts = transform_fn(receipts_by_block); - total_bytes += transformed_receipts.length(); + let len = transformed_receipts.length(); + if receipts.is_empty() { + receipts.reserve(response_reserve_hint(count, len, MAX_BLOCKS_RESERVE)); + } + total_bytes += len; receipts.push(transformed_receipts); if receipts.len() >= MAX_RECEIPTS_SERVE || total_bytes > SOFT_RESPONSE_LIMIT { @@ -372,6 +406,7 @@ where response: oneshot::Sender>, ) { let mut cells_response = Cells { cell_mask: request.cell_mask, ..Default::default() }; + let mut total_bytes = 0; for hash in request.hashes.into_iter().take(MAX_CELLS_SERVE) { let Some(cells) = @@ -380,10 +415,11 @@ where continue; }; + total_bytes += hash.length() + cells.length(); cells_response.hashes.push(hash); cells_response.cells.push(cells); - if cells_response.length() > SOFT_RESPONSE_LIMIT { + if total_bytes > SOFT_RESPONSE_LIMIT { break } } @@ -392,6 +428,14 @@ where } } +/// Number of response items to reserve once the first requested item has resolved. +/// +/// Assumes the remaining items are about as large as the first one and reserves as many as fit into +/// [`SOFT_RESPONSE_LIMIT`], bounded by the requested `count` and `max`. +fn response_reserve_hint(count: usize, first_len: usize, max: usize) -> usize { + SOFT_RESPONSE_LIMIT.div_ceil(first_len.max(1)).min(count).min(max) +} + impl EthRequestHandler where N: NetworkPrimitives, From 6005ebc9796e8eea5358b9d403a32de378f41e9c Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Thu, 3 Sep 2026 23:44:16 +0200 Subject: [PATCH 151/294] perf(rpc): store fee history entries behind Arc (#26942) --- crates/rpc/rpc-eth-types/src/fee_history.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/crates/rpc/rpc-eth-types/src/fee_history.rs b/crates/rpc/rpc-eth-types/src/fee_history.rs index ba8d7dffec5..8b14633ece7 100644 --- a/crates/rpc/rpc-eth-types/src/fee_history.rs +++ b/crates/rpc/rpc-eth-types/src/fee_history.rs @@ -98,7 +98,7 @@ where receipts, ) .unwrap_or_default(); - entries.insert(block.number(), fee_history_entry); + entries.insert(block.number(), Arc::new(fee_history_entry)); } // enforce bounds by popping the oldest entries @@ -145,7 +145,7 @@ where &self, start_block: u64, end_block: u64, - ) -> Option>> { + ) -> Option>>> { if end_block < start_block { // invalid range, return None return None @@ -210,7 +210,7 @@ struct FeeHistoryCacheInner { /// and max number of blocks config: FeeHistoryCacheConfig, /// Stores the entries of the cache - entries: tokio::sync::RwLock>>, + entries: tokio::sync::RwLock>>>, } /// Awaits for new chain events and directly inserts them into the cache so they're available From fcaed49efbcb465a9d62712e847d8465c69f7bd9 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Thu, 3 Sep 2026 21:32:43 +0000 Subject: [PATCH 152/294] perf(engine): batch BAL prewarm slot dispatch (#26955) Co-authored-by: Derek <256792747+decofe@users.noreply.github.com> --- .../payload_processor/bal_prewarm_pool.rs | 51 +++++++++++++------ .../src/tree/payload_processor/prewarm.rs | 17 ++++--- 2 files changed, 46 insertions(+), 22 deletions(-) diff --git a/crates/engine/tree/src/tree/payload_processor/bal_prewarm_pool.rs b/crates/engine/tree/src/tree/payload_processor/bal_prewarm_pool.rs index d76c4ff8058..fcec9395040 100644 --- a/crates/engine/tree/src/tree/payload_processor/bal_prewarm_pool.rs +++ b/crates/engine/tree/src/tree/payload_processor/bal_prewarm_pool.rs @@ -19,10 +19,11 @@ use tracing::trace; /// generic over the provider factory; each worker builds its own per block. pub type BuildProviderFn = dyn Fn() -> ProviderResult + Send + Sync; -/// A single warm request: a whole account (basic account + its bytecode) or one storage slot. +/// A single warm request: a whole account (basic account + its bytecode) followed by a batch of +/// its storage slots, or a batch of storage slots on their own. enum PrewarmTarget { - Account(Address), - Storage(Address, StorageKey), + Account(Address, Box<[StorageKey]>), + Storage(Address, Box<[StorageKey]>), } /// A message in a worker's queue. The per-block lifecycle is explicit and ordered (the queue is @@ -87,14 +88,23 @@ impl BalPrewarmPool { } } - /// Fire-and-forget: warm an account (basic account + bytecode) on some worker. - pub fn warm_account(&self, addr: Address) { - self.send_warm(PrewarmTarget::Account(addr)); - } - - /// Fire-and-forget: warm one storage slot on some worker. - pub fn warm_storage(&self, addr: Address, slot: StorageKey) { - self.send_warm(PrewarmTarget::Storage(addr, slot)); + /// Fire-and-forget: warm an account (basic account + bytecode) and its storage slots. + /// + /// The slots are dispatched in `WARM_BATCH_SIZE` chunks that are distributed independently, + /// so a single account with a large read-set does not serialize onto one worker; + /// [`end_block`](Self::end_block) waits for the slowest queue. + pub fn warm_account(&self, addr: Address, slots: impl IntoIterator) { + let mut slots = slots.into_iter(); + let mut batch: Box<[StorageKey]> = slots.by_ref().take(WARM_BATCH_SIZE).collect(); + self.send_warm(PrewarmTarget::Account(addr, batch)); + + loop { + batch = slots.by_ref().take(WARM_BATCH_SIZE).collect(); + if batch.is_empty() { + break + } + self.send_warm(PrewarmTarget::Storage(addr, batch)); + } } /// Ends the block: every worker drops its provider (and read txn) once it has drained the warm @@ -141,6 +151,13 @@ impl BalPrewarmPool { /// This should explain why this particular value is picked. pub const DEFAULT_BAL_PREWARM_THREADS: usize = 128; +/// Number of storage slots carried by one warm message. +/// +/// Batching amortizes the send over many slots and hands the worker a run of slots that live +/// close together in the storage table, while the cap keeps enough messages in flight to saturate +/// the workers on blocks whose read-set is concentrated in a few accounts. +const WARM_BATCH_SIZE: usize = 8; + fn prewarm_loop(rx: crossbeam_channel::Receiver) { // The provider (and its MDBX read txn) held for the current block, between `BeginBlock` and // `EndBlock`. `None` while idle, so no read txn is pinned across the inter-block gap. @@ -164,16 +181,21 @@ fn prewarm_loop(rx: crossbeam_channel::Receiver) { PrewarmMsg::Warm(target) => { let Some(provider) = provider.as_ref() else { continue }; match target { - PrewarmTarget::Account(addr) => { + PrewarmTarget::Account(addr, slots) => { if let Ok(Some(account)) = provider.basic_account(&addr) && let Some(code_hash) = account.bytecode_hash && code_hash != alloy_consensus::constants::KECCAK_EMPTY { let _ = provider.bytecode_by_hash(&code_hash); } + for &slot in &slots { + let _ = provider.storage(addr, slot); + } } - PrewarmTarget::Storage(addr, slot) => { - let _ = provider.storage(addr, slot); + PrewarmTarget::Storage(addr, slots) => { + for &slot in &slots { + let _ = provider.storage(addr, slot); + } } } } @@ -184,7 +206,6 @@ fn prewarm_loop(rx: crossbeam_channel::Receiver) { } } } - struct SendOnDrop { sender: Option>, } diff --git a/crates/engine/tree/src/tree/payload_processor/prewarm.rs b/crates/engine/tree/src/tree/payload_processor/prewarm.rs index 21ccee961fb..a2dbc4866ea 100644 --- a/crates/engine/tree/src/tree/payload_processor/prewarm.rs +++ b/crates/engine/tree/src/tree/payload_processor/prewarm.rs @@ -429,15 +429,18 @@ where }); pool.begin_block(build, caches, ctx.env.txpool_snapshot.clone()); + let dispatch_start = Instant::now(); for account in prefetch_bal.as_bal() { - pool.warm_account(account.address); - for change in &account.storage_changes { - pool.warm_storage(account.address, change.slot.into()); - } - for &slot in &account.storage_reads { - pool.warm_storage(account.address, slot.into()); - } + pool.warm_account( + account.address, + account + .storage_changes + .iter() + .map(|change| change.slot.into()) + .chain(account.storage_reads.iter().map(|&slot| slot.into())), + ); } + ctx.metrics.bal_slot_iteration_duration.record(dispatch_start.elapsed()); pool.end_block(); } From bcb9d9c612fdbeafbb4294eb4545718ee47f3a30 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Thu, 3 Sep 2026 21:33:31 +0000 Subject: [PATCH 153/294] perf(exex): buffer WAL notification serialization (#26949) --- crates/exex/exex/src/wal/storage.rs | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/crates/exex/exex/src/wal/storage.rs b/crates/exex/exex/src/wal/storage.rs index 6a23f86b3a9..a69509c05bc 100644 --- a/crates/exex/exex/src/wal/storage.rs +++ b/crates/exex/exex/src/wal/storage.rs @@ -1,5 +1,6 @@ use std::{ fs::File, + io::{BufReader, BufWriter, Write}, path::{Path, PathBuf}, }; @@ -138,7 +139,7 @@ where // Deserialize using the bincode- and msgpack-compatible serde wrapper let notification: reth_exex_types::serde_bincode_compat::ExExNotification<'_, N> = - rmp_serde::decode::from_read(&mut file) + rmp_serde::decode::from_read(BufReader::new(&mut file)) .map_err(|err| WalError::Decode(file_id, file_path, err))?; Ok(Some((notification.into(), size))) @@ -163,7 +164,12 @@ where reth_exex_types::serde_bincode_compat::ExExNotification::::from(notification); reth_fs_util::atomic_write_file(&file_path, |file| { - rmp_serde::encode::write(file, ¬ification) + let mut writer = BufWriter::new(file); + rmp_serde::encode::write(&mut writer, ¬ification)?; + // a `BufWriter` dropped without an explicit flush discards write errors, and + // `atomic_write_file` fsyncs as soon as this returns + writer.flush()?; + Ok::<_, Box>(()) })?; Ok(file_path.metadata().map_err(|err| WalError::FileMetadata(file_id, err))?.len()) From b4cdc0e4fde16a5351d02769c046b90f1b068da0 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Thu, 3 Sep 2026 21:50:04 +0000 Subject: [PATCH 154/294] fix(metrics): collect storage metrics off the scrape path (#26734) --- Cargo.lock | 1 + crates/node/builder/src/launch/common.rs | 28 +-- crates/node/metrics/Cargo.toml | 1 + crates/node/metrics/src/hooks.rs | 223 ++++++++++++++++++++++- crates/node/metrics/src/server.rs | 74 +++++++- crates/tracing/src/throttle.rs | 4 + 6 files changed, 312 insertions(+), 19 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ac5deab64f2..1cb020af96f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9603,6 +9603,7 @@ dependencies = [ "metrics-exporter-prometheus", "metrics-process", "metrics-util", + "parking_lot", "pprof_util", "procfs", "reqwest", diff --git a/crates/node/builder/src/launch/common.rs b/crates/node/builder/src/launch/common.rs index 498c65b3fab..0ec7f90efba 100644 --- a/crates/node/builder/src/launch/common.rs +++ b/crates/node/builder/src/launch/common.rs @@ -87,10 +87,7 @@ use reth_stages::{ use reth_static_file::{blocks_per_file_for_prune_distance, StaticFileProducer, StaticFileSegment}; use reth_storage_overlay::OverlayManager; use reth_tasks::TaskExecutor; -use reth_tracing::{ - throttle, - tracing::{debug, error, info, warn}, -}; +use reth_tracing::tracing::{debug, error, info, warn}; use reth_transaction_pool::TransactionPool; use std::{num::NonZeroUsize, sync::Arc, thread::available_parallelism, time::Duration}; use tokio::sync::{ @@ -1375,25 +1372,28 @@ where } /// Returns the metrics hooks for the node. +/// +/// The storage hooks walk their backing store, which scales with the dataset, so they are +/// registered as background hooks: metrics collection still refreshes them at most every 5 +/// minutes, but renders the values of the previous refresh instead of waiting for the walk. pub fn metrics_hooks(provider_factory: &ProviderFactory) -> Hooks { Hooks::builder() - .with_hook({ + .with_background_interval(Duration::from_secs(5 * 60)) + .with_background_hook({ let db = provider_factory.db_ref().clone(); - move || throttle!(Duration::from_secs(5 * 60), || db.report_metrics()) + move || db.report_metrics() }) - .with_hook({ + .with_background_hook({ let sfp = provider_factory.static_file_provider(); move || { - throttle!(Duration::from_secs(5 * 60), || { - if let Err(error) = sfp.report_metrics() { - error!(%error, "Failed to report metrics from static file provider"); - } - }) + if let Err(error) = sfp.report_metrics() { + error!(%error, "Failed to report metrics from static file provider"); + } } }) - .with_hook({ + .with_background_hook({ let rocksdb = provider_factory.rocksdb_provider(); - move || throttle!(Duration::from_secs(5 * 60), || rocksdb.report_metrics()) + move || rocksdb.report_metrics() }) .build() } diff --git a/crates/node/metrics/Cargo.toml b/crates/node/metrics/Cargo.toml index b972e7f21df..edfca935fb5 100644 --- a/crates/node/metrics/Cargo.toml +++ b/crates/node/metrics/Cargo.toml @@ -27,6 +27,7 @@ reqwest.workspace = true tracing.workspace = true eyre.workspace = true +parking_lot.workspace = true [target.'cfg(unix)'.dependencies] jemalloc_pprof = { workspace = true, optional = true } diff --git a/crates/node/metrics/src/hooks.rs b/crates/node/metrics/src/hooks.rs index d8ddce0a4d6..26519e279fc 100644 --- a/crates/node/metrics/src/hooks.rs +++ b/crates/node/metrics/src/hooks.rs @@ -1,5 +1,12 @@ use metrics_process::Collector; -use std::{fmt, sync::Arc}; +use parking_lot::Mutex; +use reth_tasks::TaskExecutor; +use std::{ + fmt, + panic::{catch_unwind, AssertUnwindSafe}, + sync::Arc, + time::{Duration, Instant}, +}; /// The simple alias for function types that are `'static`, `Send`, and `Sync`. pub trait Hook: Fn() + Send + Sync + 'static {} @@ -8,10 +15,18 @@ impl Hook for T {} /// A builder-like type to create a new [`Hooks`] instance. pub struct HooksBuilder { hooks: Vec>>, + background_hooks: Vec>>, + background_interval: Duration, } impl HooksBuilder { - /// Registers a [`Hook`]. + /// Default interval at which background hooks are refreshed. + pub const DEFAULT_BACKGROUND_INTERVAL: Duration = Duration::from_secs(5 * 60); + + /// Registers a [`Hook`] that runs while metrics are collected. + /// + /// Only suitable for cheap collection; anything that can take longer than a scrape timeout + /// belongs in [`with_background_hook`](Self::with_background_hook). pub fn with_hook(self, hook: impl Hook) -> Self { self.with_boxed_hook(Box::new(hook)) } @@ -32,9 +47,39 @@ impl HooksBuilder { self } + /// Registers a [`Hook`] whose collection is refreshed out of band: metrics collection kicks it + /// off, at most once per [`background_interval`](Self::with_background_interval) and never + /// while a previous refresh is still running, but never waits for it. + /// + /// Collection that walks a backing store (e.g. every static file jar) scales with the dataset + /// and can take seconds, which would stall a scrape for its entire duration. Such hooks only + /// set gauges, and the interval already means most scrapes render values collected by an + /// earlier one, so serving the previous values while the refresh runs costs at most one scrape + /// worth of freshness. + /// + /// Background hooks are collected sequentially in registration order. A hook that panics is + /// logged and does not prevent the hooks registered after it from being collected. + pub fn with_background_hook(mut self, hook: impl Hook) -> Self { + self.background_hooks.push(Box::new(hook)); + self + } + + /// Sets the minimum interval between the starts of two background collections. + pub const fn with_background_interval(mut self, interval: Duration) -> Self { + self.background_interval = interval; + self + } + /// Builds the [`Hooks`] collection from the registered hooks. pub fn build(self) -> Hooks { - Hooks { inner: Arc::new(self.hooks) } + Hooks { + inner: Arc::new(self.hooks), + background: Arc::new(BackgroundHooks { + hooks: self.background_hooks, + interval: self.background_interval, + state: Mutex::default(), + }), + } } } @@ -46,6 +91,8 @@ impl Default for HooksBuilder { Box::new(collect_memory_stats), Box::new(collect_io_stats), ], + background_hooks: Vec::new(), + background_interval: Self::DEFAULT_BACKGROUND_INTERVAL, } } } @@ -54,6 +101,8 @@ impl std::fmt::Debug for HooksBuilder { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("HooksBuilder") .field("hooks", &format_args!("Vec>, len: {}", self.hooks.len())) + .field("background_hooks", &self.background_hooks.len()) + .field("background_interval", &self.background_interval) .finish() } } @@ -62,6 +111,7 @@ impl std::fmt::Debug for HooksBuilder { #[derive(Clone)] pub struct Hooks { inner: Arc>>>, + background: Arc, } impl Hooks { @@ -74,6 +124,20 @@ impl Hooks { pub(crate) fn iter(&self) -> impl Iterator>> { self.inner.iter() } + + /// Refreshes the background hooks unless a refresh is already in flight or the previous one + /// started less than [`with_background_interval`](HooksBuilder::with_background_interval) ago. + /// + /// Returns without waiting for the collection to finish: the caller renders the values of the + /// previous refresh. + pub(crate) fn refresh_background(&self, executor: &TaskExecutor) { + if !self.background.claim() { + return + } + + let background = self.background.clone(); + executor.spawn_blocking(move || background.collect()); + } } impl fmt::Debug for Hooks { @@ -81,10 +145,85 @@ impl fmt::Debug for Hooks { let hooks_len = self.inner.len(); f.debug_struct("Hooks") .field("inner", &format_args!("Arc>>, len: {hooks_len}")) + .field("background", &self.background) + .finish() + } +} + +/// The [`Hook`]s that are collected out of band, see +/// [`with_background_hook`](HooksBuilder::with_background_hook). +struct BackgroundHooks { + hooks: Vec>>, + interval: Duration, + state: Mutex, +} + +impl BackgroundHooks { + /// Marks a collection as started if none is in flight and the previous one started at least + /// `interval` ago, so that a concurrent caller can not start a second one. + /// + /// The interval is claimed upfront, while the in-flight flag is released once + /// [`collect`](Self::collect) returns or unwinds, so a panicking hook delays the next + /// collection by one interval instead of blocking it forever. + fn claim(&self) -> bool { + if self.hooks.is_empty() { + return false + } + + let mut state = self.state.lock(); + if state.in_flight || + state.last_collected.is_some_and(|last| last.elapsed() < self.interval) + { + return false + } + state.last_collected = Some(Instant::now()); + state.in_flight = true; + + true + } + + /// Collects a claimed refresh and releases the in-flight flag afterwards. + /// + /// Each hook is collected on its own so that one that panics is logged instead of skipping the + /// hooks registered after it. + fn collect(&self) { + let _in_flight = InFlightGuard(&self.state); + for (idx, hook) in self.hooks.iter().enumerate() { + if catch_unwind(AssertUnwindSafe(hook)).is_err() { + tracing::error!(hook = idx, "Background metrics hook panicked"); + } + } + } +} + +impl fmt::Debug for BackgroundHooks { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("BackgroundHooks") + .field("hooks", &format_args!("Vec>, len: {}", self.hooks.len())) + .field("interval", &self.interval) .finish() } } +/// Refresh state of [`BackgroundHooks`], kept under one lock so that the interval and the +/// in-flight condition are checked and claimed atomically. +#[derive(Default)] +struct BackgroundHooksState { + /// When the last collection was started. + last_collected: Option, + /// Whether a collection is currently running. + in_flight: bool, +} + +/// Releases the in-flight flag when the collection finishes, also by unwinding. +struct InFlightGuard<'a>(&'a Mutex); + +impl Drop for InFlightGuard<'_> { + fn drop(&mut self) { + self.0.lock().in_flight = false; + } +} + #[cfg(all(feature = "jemalloc", unix))] fn collect_memory_stats() { use metrics::gauge; @@ -164,3 +303,81 @@ fn collect_io_stats() { #[cfg(not(target_os = "linux"))] const fn collect_io_stats() {} + +#[cfg(test)] +mod tests { + use super::*; + use reth_tasks::Runtime; + use std::sync::atomic::{AtomicUsize, Ordering}; + + /// Keeps kicking off refreshes until `done` holds, so that a refresh that was skipped because + /// the previous one was still in flight is retried. + fn refresh_until(hooks: &Hooks, runtime: &Runtime, done: impl Fn() -> bool) { + let deadline = Instant::now() + Duration::from_secs(10); + while !done() { + assert!(Instant::now() < deadline, "background collection did not make progress"); + hooks.refresh_background(runtime); + std::thread::sleep(Duration::from_millis(5)); + } + } + + #[test] + fn background_collections_do_not_overlap() { + let runtime = Runtime::test(); + let active = Arc::new(AtomicUsize::new(0)); + let max_active = Arc::new(AtomicUsize::new(0)); + let collections = Arc::new(AtomicUsize::new(0)); + let hooks = Hooks::builder() + .with_background_interval(Duration::from_millis(20)) + .with_background_hook({ + let active = active.clone(); + let max_active = max_active.clone(); + let collections = collections.clone(); + move || { + let now_active = active.fetch_add(1, Ordering::SeqCst) + 1; + max_active.fetch_max(now_active, Ordering::SeqCst); + // outlive the interval, so that refreshes are attempted while this runs + std::thread::sleep(Duration::from_millis(100)); + collections.fetch_add(1, Ordering::SeqCst); + active.fetch_sub(1, Ordering::SeqCst); + } + }) + .build(); + + let started = Instant::now(); + while started.elapsed() < Duration::from_millis(350) { + hooks.refresh_background(&runtime); + std::thread::sleep(Duration::from_millis(5)); + } + while active.load(Ordering::SeqCst) > 0 { + std::thread::sleep(Duration::from_millis(5)); + } + + assert_eq!(max_active.load(Ordering::SeqCst), 1, "collections overlapped"); + assert!( + collections.load(Ordering::SeqCst) >= 2, + "collection did not resume once the previous one finished" + ); + } + + #[test] + fn panicking_background_hook_does_not_block_collection() { + let runtime = Runtime::test(); + let collections = Arc::new(AtomicUsize::new(0)); + let hooks = Hooks::builder() + .with_background_interval(Duration::ZERO) + .with_background_hook(|| panic!("hook panicked")) + .with_background_hook({ + let collections = collections.clone(); + move || { + collections.fetch_add(1, Ordering::SeqCst); + } + }) + .build(); + + // the hook registered after the panicking one is still collected + refresh_until(&hooks, &runtime, || collections.load(Ordering::SeqCst) >= 1); + // and the panic does not leave the collection marked as in flight + refresh_until(&hooks, &runtime, || collections.load(Ordering::SeqCst) >= 2); + } +} diff --git a/crates/node/metrics/src/server.rs b/crates/node/metrics/src/server.rs index 1cd7d83c8c7..d8c623eba73 100644 --- a/crates/node/metrics/src/server.rs +++ b/crates/node/metrics/src/server.rs @@ -95,9 +95,13 @@ impl MetricServer { } = &self.config; let hooks_for_endpoint = hooks.clone(); + let executor_for_hooks = task_executor.clone(); self.start_endpoint( *listen_addr, - Arc::new(move || hooks_for_endpoint.iter().for_each(|hook| hook())), + Arc::new(move || { + hooks_for_endpoint.refresh_background(&executor_for_hooks); + hooks_for_endpoint.iter().for_each(|hook| hook()); + }), task_executor.clone(), pprof_dump_dir.clone(), ) @@ -209,6 +213,7 @@ impl MetricServer { break; } _ = tokio::time::sleep(interval) => { + hooks.refresh_background(&executor); let hooks = hooks.clone(); let metrics_handle = handle.handle().clone(); let metrics = match executor.spawn_blocking(move || { @@ -487,7 +492,10 @@ mod tests { use reqwest::Client; use reth_tasks::Runtime; use socket2::{Domain, Socket, Type}; - use std::net::{SocketAddr, TcpListener}; + use std::{ + net::{SocketAddr, TcpListener}, + sync::atomic::{AtomicUsize, Ordering}, + }; fn get_random_available_addr() -> SocketAddr { let addr = &"127.0.0.1:0".parse::().unwrap().into(); @@ -555,4 +563,66 @@ mod tests { // Make sure the runtime is dropped after the test runs. drop(runtime); } + + #[tokio::test(flavor = "multi_thread")] + async fn test_background_hooks_do_not_block_collection() { + install_prometheus_recorder(); + + let collections = Arc::new(AtomicUsize::new(0)); + let hooks = Hooks::builder() + .with_background_interval(Duration::from_secs(60)) + .with_background_hook({ + let collections = collections.clone(); + move || { + std::thread::sleep(Duration::from_secs(2)); + let collections = collections.fetch_add(1, Ordering::Relaxed) + 1; + metrics::gauge!("test_background_hook_collections").set(collections as f64); + } + }) + .build(); + + let runtime = Runtime::test(); + let listen_addr = get_random_available_addr(); + let config = MetricServerConfig::new( + listen_addr, + VersionInfo { + version: "test", + build_timestamp: "test", + cargo_features: "test", + git_sha: "test", + target_triple: "test", + build_profile: "test", + }, + ChainSpecInfo { name: "test".to_string() }, + runtime.clone(), + hooks, + std::env::temp_dir(), + ); + + MetricServer::new(config).serve().await.unwrap(); + + // the first scrape kicks off the background hook but must not wait for it + let url = format!("http://{listen_addr}"); + let started = std::time::Instant::now(); + let response = + Client::new().get(&url).timeout(Duration::from_millis(500)).send().await.unwrap(); + assert!(response.status().is_success()); + assert!(started.elapsed() < Duration::from_secs(1), "scrape waited for the hook"); + assert_eq!(collections.load(Ordering::Relaxed), 0); + + // once it completes, its values are rendered by the following scrape + while collections.load(Ordering::Relaxed) == 0 { + tokio::time::sleep(Duration::from_millis(10)).await; + } + let body = Client::new().get(&url).send().await.unwrap().text().await.unwrap(); + assert!( + body.contains("test_background_hook_collections"), + "background hook metric missing: {body}" + ); + + // and scrapes within the interval do not collect again + assert_eq!(collections.load(Ordering::Relaxed), 1); + + drop(runtime); + } } diff --git a/crates/tracing/src/throttle.rs b/crates/tracing/src/throttle.rs index a2ce2c74b78..0764570dfcb 100644 --- a/crates/tracing/src/throttle.rs +++ b/crates/tracing/src/throttle.rs @@ -47,6 +47,9 @@ pub fn should_run(start: &LazyLock, last: &AtomicU64, duration_millis: /// tracing::info!("This message is throttled"); /// }); /// ``` +#[deprecated( + note = "no longer used by reth and scheduled for removal, copy the macro if you depend on it" +)] #[macro_export] macro_rules! throttle { ($duration:expr, || $expr:expr) => {{ @@ -62,6 +65,7 @@ macro_rules! throttle { } #[cfg(test)] +#[allow(deprecated)] mod tests { use std::sync::atomic::{AtomicUsize, Ordering}; From eadb887f1c1698e7bd932f60a0c654dccccd081f Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Fri, 4 Sep 2026 07:08:33 +0000 Subject: [PATCH 155/294] perf(trie): reuse address hash across storage changeset rows (#26940) --- crates/trie/db/src/prefix_set.rs | 17 ++++++++++++++--- crates/trie/db/src/state.rs | 14 ++++++++++++-- 2 files changed, 26 insertions(+), 5 deletions(-) diff --git a/crates/trie/db/src/prefix_set.rs b/crates/trie/db/src/prefix_set.rs index 6d5bc1bdf08..4335e2bd3f1 100644 --- a/crates/trie/db/src/prefix_set.rs +++ b/crates/trie/db/src/prefix_set.rs @@ -1,7 +1,7 @@ use alloy_primitives::{ keccak256, map::{HashMap, HashSet}, - BlockNumber, B256, + Address, BlockNumber, B256, }; use core::ops::RangeInclusive; use reth_db_api::{ @@ -50,9 +50,20 @@ where // Walk storage changesets using the provider (handles static files + database) let storage_changesets = provider.storage_changesets_range(range)?; + // Rows are ordered by `BlockNumberAddress`, so all slots of one account arrive consecutively. + // Memoizing the last address avoids re-hashing it once per slot; the repeated account prefix + // insert can be skipped because `PrefixSetMut::freeze` deduplicates anyway. + let mut last_address: Option<(Address, B256)> = None; for (BlockNumberAddress((_, address)), storage_entry) in storage_changesets { - let hashed_address = keccak256(address); - account_prefix_set.insert(Nibbles::unpack(hashed_address)); + let hashed_address = match last_address { + Some((last, hashed)) if last == address => hashed, + _ => { + let hashed = keccak256(address); + account_prefix_set.insert(Nibbles::unpack(hashed)); + last_address = Some((address, hashed)); + hashed + } + }; storage_prefix_sets .entry(hashed_address) .or_default() diff --git a/crates/trie/db/src/state.rs b/crates/trie/db/src/state.rs index 512dc96ccd3..d35c9b7d008 100644 --- a/crates/trie/db/src/state.rs +++ b/crates/trie/db/src/state.rs @@ -1,5 +1,5 @@ use crate::{DatabaseHashedCursorFactory, DatabaseTrieCursorFactory}; -use alloy_primitives::{keccak256, map::B256Map, BlockNumber, B256}; +use alloy_primitives::{keccak256, map::B256Map, Address, BlockNumber, B256}; use reth_db_api::{ models::{AccountBeforeTx, BlockNumberAddress}, transaction::DbTx, @@ -307,11 +307,21 @@ impl DatabaseHashedPostState for HashedPostStateSorted { if start < end { let end_inclusive = end.saturating_sub(1); + // Rows are ordered by `BlockNumberAddress`, so all slots of one account arrive + // consecutively and the address hash can be reused across them. + let mut last_address: Option<(Address, B256)> = None; for (BlockNumberAddress((_, address)), storage) in provider.storage_changesets_range(start..=end_inclusive)? { if seen_storage_keys.insert((address, storage.key)) { - let hashed_address = keccak256(address); + let hashed_address = match last_address { + Some((last, hashed)) if last == address => hashed, + _ => { + let hashed = keccak256(address); + last_address = Some((address, hashed)); + hashed + } + }; storages .entry(hashed_address) .or_default() From 049cfd1af8f53e9b4495a525715fcfd944ec27c9 Mon Sep 17 00:00:00 2001 From: Derek Cofausper <256792747+decofe@users.noreply.github.com> Date: Fri, 4 Sep 2026 11:15:56 +0000 Subject: [PATCH 156/294] refactor(overlay): reduce execution overlay setup overhead (#26990) Co-authored-by: Brian Picciano <933154+mediocregopher@users.noreply.github.com> Co-authored-by: Brian Picciano --- crates/storage/storage-overlay/src/builder.rs | 104 ++++++++++++----- .../storage/storage-overlay/src/provider.rs | 106 ++++++------------ 2 files changed, 111 insertions(+), 99 deletions(-) diff --git a/crates/storage/storage-overlay/src/builder.rs b/crates/storage/storage-overlay/src/builder.rs index c1ae9e60d71..ccc779ec7d3 100644 --- a/crates/storage/storage-overlay/src/builder.rs +++ b/crates/storage/storage-overlay/src/builder.rs @@ -319,14 +319,14 @@ impl OverlayBuilder { match &self.overlay_source { Some(OverlaySource::Managed) => anchor_for_parent_with_frontiers( self.parent_hash, - self.parent_state.iter().flat_map(|state| state.chain()).map(BlockState::block), + self.parent_state.iter().flat_map(|state| state.chain()).map(BlockState::block_ref), partial_state_trie, finish, provider, ), _ => anchor_for_parent_with_frontiers( self.parent_hash, - std::iter::empty::>(), + std::iter::empty::<&ExecutedBlock>(), partial_state_trie, finish, provider, @@ -506,7 +506,7 @@ impl OverlayBuilder { } /// Returns the in-memory execution overlay and the block for historical fallback reads. - #[instrument(level = "debug", target = "storage::overlay", skip_all)] + #[instrument(level = "trace", target = "storage::overlay", skip_all)] pub fn execution_overlay( &self, provider: &Provider, @@ -525,7 +525,7 @@ impl OverlayBuilder { /// Returns the in-memory execution overlay using frontiers already read from the provider. #[instrument( - level = "debug", + level = "trace", target = "storage::overlay", skip_all, fields(?state_trie_tip_block, ?finish_tip_block, parent_hash = ?self.parent_hash) @@ -715,26 +715,27 @@ struct OverlayBuilderMetrics { sparse_trie_overlay_skips: Counter, } -fn anchor_for_parent_in( +fn anchor_for_parent_in<'a, N: NodePrimitives + 'a>( parent_hash: B256, - mut in_mem_chain: impl Iterator>, - preferred_anchor: B256, -) -> B256 { - if parent_hash == preferred_anchor { - return parent_hash + in_mem_chain: impl Iterator>, + preferred_anchor: BlockNumHash, +) -> Option { + if parent_hash == preferred_anchor.hash { + return Some(preferred_anchor) } - let mut hash = parent_hash; + let mut anchor = None; - loop { - let Some(block) = in_mem_chain.next() else { return hash }; - let block_parent_hash = block.recovered_block().parent_hash(); + for block in in_mem_chain { + let block_parent = block.recovered_block().parent_num_hash(); - if block_parent_hash == preferred_anchor { - return block_parent_hash + if block_parent.hash == preferred_anchor.hash { + return Some(preferred_anchor) } - hash = block_parent_hash; + anchor = Some(block_parent); } + + anchor } /// Describes whether an overlay must revert the database before using its anchor. @@ -769,13 +770,13 @@ impl AnchorForParent { /// * `parent`: The block whose post-state is being targeted. /// * `in_mem_chain`: Yields the in-memory blocks in the chain, starting at `parent_hash`. /// * `provider`: Used to resolve the durable frontiers and check changeset availability. -pub fn anchor_for_parent( +pub fn anchor_for_parent<'a, N, Provider>( parent_hash: B256, - in_mem_chain: impl Iterator>, + in_mem_chain: impl Iterator>, provider: &Provider, ) -> ProviderResult where - N: NodePrimitives, + N: NodePrimitives + 'a, Provider: StageCheckpointReader + BlockNumReader + PruneCheckpointReader, { let (partial_state_trie, finish) = database_state_frontiers(provider)?; @@ -797,22 +798,26 @@ where /// * `partial_state_trie`: The durable state/trie frontier. /// * `finish`: The durable Finish frontier. /// * `provider`: Used to resolve the parent and check changeset availability. -pub fn anchor_for_parent_with_frontiers( +pub fn anchor_for_parent_with_frontiers<'a, N, Provider>( parent_hash: B256, - in_mem_chain: impl Iterator>, + in_mem_chain: impl Iterator>, partial_state_trie: BlockNumHash, finish: BlockNumHash, provider: &Provider, ) -> ProviderResult where - N: NodePrimitives, + N: NodePrimitives + 'a, Provider: BlockNumReader + PruneCheckpointReader, { use std::io::Error; - let persisted_parent = provider - .block_number(parent_hash)? - .filter(|&parent_number| parent_number <= partial_state_trie.number); + let mut in_mem_chain = in_mem_chain.peekable(); + let persisted_parent = persisted_parent_number( + parent_hash, + in_mem_chain.peek().copied(), + partial_state_trie, + provider, + )?; let mut finish_seen = parent_hash == finish.hash; let anchor = if let Some(parent_number) = persisted_parent { @@ -822,11 +827,12 @@ where finish_seen |= block.recovered_block().hash() == finish.hash; }); - let anchor_hash = - anchor_for_parent_in(parent_hash, &mut in_mem_chain, partial_state_trie.hash); - if anchor_hash == partial_state_trie.hash { - BlockNumHash::new(partial_state_trie.number, anchor_hash) + if let Some(anchor) = + anchor_for_parent_in(parent_hash, &mut in_mem_chain, partial_state_trie) + { + anchor } else { + let anchor_hash = parent_hash; let anchor_number = provider .convert_hash_or_number(anchor_hash.into())? .ok_or(ProviderError::BlockHashNotFound(anchor_hash))?; @@ -876,6 +882,44 @@ where Ok(AnchorForParent::RevertsRequired { anchor, finish }) } +/// Returns the parent number when `parent_hash` is a durable anchor. +fn persisted_parent_number( + parent_hash: B256, + managed_parent: Option<&ExecutedBlock>, + partial_state_trie: BlockNumHash, + provider: &Provider, +) -> ProviderResult> +where + N: NodePrimitives, + Provider: BlockNumReader, +{ + let managed_parent_number = managed_parent + .filter(|block| block.recovered_block().hash() == parent_hash) + .map(|block| block.recovered_block().number()); + + // Managed blocks already carry their number. Blocks above the state trie frontier cannot be a + // persisted anchor; for older blocks, verify canonicality with a number-to-hash lookup. + if let Some(parent_number) = managed_parent_number { + if parent_number > partial_state_trie.number { + return Ok(None) + } + if parent_number == partial_state_trie.number && parent_hash == partial_state_trie.hash { + return Ok(Some(parent_number)) + } + return Ok( + (provider.block_hash(parent_number)? == Some(parent_hash)).then_some(parent_number) + ) + } + + if parent_hash == partial_state_trie.hash { + Ok(Some(partial_state_trie.number)) + } else { + provider.block_number(parent_hash).map(|parent_number| { + parent_number.filter(|&number| number <= partial_state_trie.number) + }) + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/crates/storage/storage-overlay/src/provider.rs b/crates/storage/storage-overlay/src/provider.rs index c7c2cd9858f..9d9c1bf928d 100644 --- a/crates/storage/storage-overlay/src/provider.rs +++ b/crates/storage/storage-overlay/src/provider.rs @@ -57,8 +57,6 @@ pub struct OverlayStateProviderFactory { /// Under partial persistence the overlay depends on both durable frontiers, so both hashes are /// part of the cache key. state_trie_overlay_cache: StateTrieOverlayCache, - /// A cache which maps durable frontier pairs to [`ExecutionOverlay`]s. - execution_overlay_cache: ExecutionOverlayCache, /// Metrics for provider factory operations. metrics: OverlayStateProviderFactoryMetrics, } @@ -70,7 +68,6 @@ impl OverlayStateProviderFactory { factory, overlay_builder, state_trie_overlay_cache: Default::default(), - execution_overlay_cache: Default::default(), metrics: Default::default(), } } @@ -81,7 +78,6 @@ impl OverlayStateProviderFactory { self.overlay_builder = self.overlay_builder.with_skip_overlay_for_reused_sparse_trie(anchor_hash); self.state_trie_overlay_cache = Default::default(); - self.execution_overlay_cache = Default::default(); self } } @@ -120,7 +116,6 @@ where provider, self.overlay_builder.clone(), Arc::clone(&self.state_trie_overlay_cache), - Arc::clone(&self.execution_overlay_cache), self.metrics.clone(), is_v2, )) @@ -132,7 +127,6 @@ pub struct OverlayStateProvider { provider: Provider, overlay_builder: Option>, state_trie_overlay_cache: StateTrieOverlayCache, - execution_overlay_cache: ExecutionOverlayCache, metrics: OverlayStateProviderFactoryMetrics, state_trie_overlay: OnceCell, execution_overlay: OnceCell, @@ -144,7 +138,6 @@ impl OverlayStateProvider, provider: Provider, overlay_builder: OverlayBuilder, state_trie_overlay_cache: StateTrieOverlayCache, - execution_overlay_cache: ExecutionOverlayCache, metrics: OverlayStateProviderFactoryMetrics, is_v2: bool, ) -> Self { @@ -152,7 +145,6 @@ impl OverlayStateProvider, provider: OwnedProvider(provider), overlay_builder: Some(overlay_builder), state_trie_overlay_cache, - execution_overlay_cache, metrics, state_trie_overlay: OnceCell::new(), execution_overlay: OnceCell::new(), @@ -170,7 +162,6 @@ impl OverlayStateProvider, provider: OwnedProvider(provider), overlay_builder: None, state_trie_overlay_cache: Default::default(), - execution_overlay_cache: Default::default(), metrics: Default::default(), state_trie_overlay: OnceCell::new(), execution_overlay: OnceCell::from(CachedExecutionOverlay { @@ -192,7 +183,6 @@ impl<'a, Provider, N: NodePrimitives> OverlayStateProvider<&'a Provider, N> { provider, overlay_builder: None, state_trie_overlay_cache: Default::default(), - execution_overlay_cache: Default::default(), metrics: Default::default(), state_trie_overlay: OnceCell::from(state_trie_overlay), execution_overlay: OnceCell::new(), @@ -295,56 +285,43 @@ where } let (state_trie_tip_block, finish_tip_block) = database_state_frontiers(self.provider())?; - let overlay = match self - .execution_overlay_cache - .entry((state_trie_tip_block.hash, finish_tip_block.hash)) - { - dashmap::Entry::Occupied(entry) => entry.get().clone(), - dashmap::Entry::Vacant(entry) => { - self.metrics.execution_overlay_cache_misses.increment(1); - let (overlay, fallback_block_number) = self - .overlay_builder - .as_ref() - .expect("execution overlay must be initialized or lazily resolvable") - .execution_overlay_at_frontiers( - self.provider(), - state_trie_tip_block, - finish_tip_block, - )?; - let historical_fallback = fallback_block_number - .map(|block_number| { - let account_history_block_number = self - .provider() - .get_prune_checkpoint(PruneSegment::AccountHistory)? - .and_then(|checkpoint| checkpoint.block_number) - .map(|block_number| block_number + 1); - if account_history_block_number.is_some_and(|lowest| block_number < lowest) - { - return Err(ProviderError::StateAtBlockPruned(block_number)) - } - - let storage_history_block_number = self - .provider() - .get_prune_checkpoint(PruneSegment::StorageHistory)? - .and_then(|checkpoint| checkpoint.block_number) - .map(|block_number| block_number + 1); - if storage_history_block_number.is_some_and(|lowest| block_number < lowest) - { - return Err(ProviderError::StateAtBlockPruned(block_number)) - } - - Ok(HistoricalFallback { - block_number, - account_history_block_number, - storage_history_block_number, - }) - }) - .transpose()?; - let overlay = CachedExecutionOverlay { overlay, historical_fallback }; - entry.insert(overlay.clone()); - overlay - } - }; + let (overlay, fallback_block_number) = self + .overlay_builder + .as_ref() + .expect("execution overlay must be initialized or lazily resolvable") + .execution_overlay_at_frontiers( + self.provider(), + state_trie_tip_block, + finish_tip_block, + )?; + let historical_fallback = fallback_block_number + .map(|block_number| { + let account_history_block_number = self + .provider() + .get_prune_checkpoint(PruneSegment::AccountHistory)? + .and_then(|checkpoint| checkpoint.block_number) + .map(|block_number| block_number + 1); + if account_history_block_number.is_some_and(|lowest| block_number < lowest) { + return Err(ProviderError::StateAtBlockPruned(block_number)) + } + + let storage_history_block_number = self + .provider() + .get_prune_checkpoint(PruneSegment::StorageHistory)? + .and_then(|checkpoint| checkpoint.block_number) + .map(|block_number| block_number + 1); + if storage_history_block_number.is_some_and(|lowest| block_number < lowest) { + return Err(ProviderError::StateAtBlockPruned(block_number)) + } + + Ok(HistoricalFallback { + block_number, + account_history_block_number, + storage_history_block_number, + }) + }) + .transpose()?; + let overlay = CachedExecutionOverlay { overlay, historical_fallback }; let _ = self.execution_overlay.set(overlay); let overlay = self.execution_overlay.get().expect("execution overlay was just initialized"); Ok((&overlay.overlay, overlay.historical_fallback.as_ref())) @@ -976,12 +953,9 @@ pub(crate) struct OverlayStateProviderFactoryMetrics { database_provider_ro_duration: Histogram, /// Number of cache misses when fetching state trie overlays. state_trie_overlay_cache_misses: Counter, - /// Number of cache misses when fetching execution overlays. - execution_overlay_cache_misses: Counter, } type StateTrieOverlayCache = Arc>; -type ExecutionOverlayCache = Arc>; #[derive(Clone, Debug)] struct CachedExecutionOverlay { @@ -1175,20 +1149,14 @@ mod tests { assert!(provider.state_trie_overlay.get().is_none()); assert!(provider.execution_overlay.get().is_none()); assert!(state_provider_factory.state_trie_overlay_cache.is_empty()); - assert!(state_provider_factory.execution_overlay_cache.is_empty()); provider.basic_account(&Address::ZERO).unwrap(); - let execution_overlay = Arc::clone(&provider.execution_overlay.get().unwrap().overlay); assert!(provider.state_trie_overlay.get().is_none()); assert!(provider.execution_overlay.get().is_some()); assert!(state_provider_factory.state_trie_overlay_cache.is_empty()); - assert_eq!(state_provider_factory.execution_overlay_cache.len(), 1); - let cached_overlay = state_provider_factory.execution_overlay_cache.iter().next().unwrap(); - assert!(Arc::ptr_eq(&execution_overlay, &cached_overlay.value().overlay)); provider.account_trie_cursor().unwrap(); assert_eq!(state_provider_factory.state_trie_overlay_cache.len(), 1); - assert_eq!(state_provider_factory.execution_overlay_cache.len(), 1); } #[test] From e423a23d8132d02997f73a2a55c197807688f301 Mon Sep 17 00:00:00 2001 From: Dan Cline <6798349+Rjected@users.noreply.github.com> Date: Fri, 4 Sep 2026 11:45:33 +0000 Subject: [PATCH 157/294] fix(cli): use configured minimal download defaults (#26934) Co-authored-by: Arsenii Kulikov <62447812+klkvr@users.noreply.github.com> Co-authored-by: Derek Cofausper <256792747+decofe@users.noreply.github.com> --- .../cli/commands/src/download/config_gen.rs | 72 ++++--- crates/cli/commands/src/download/manifest.rs | 22 --- crates/cli/commands/src/download/mod.rs | 176 ++++++++++++------ crates/cli/commands/src/download/tui.rs | 125 ++++++++++--- 4 files changed, 267 insertions(+), 128 deletions(-) diff --git a/crates/cli/commands/src/download/config_gen.rs b/crates/cli/commands/src/download/config_gen.rs index 34f70b10af2..2eda1df907b 100644 --- a/crates/cli/commands/src/download/config_gen.rs +++ b/crates/cli/commands/src/download/config_gen.rs @@ -7,18 +7,13 @@ use reth_config::config::{BlocksPerFileConfig, Config, PruneConfig, StaticFilesC use reth_db::tables; use reth_db_api::transaction::{DbTx, DbTxMut}; use reth_node_core::args::DefaultPruningValues; -use reth_prune_types::{PruneCheckpoint, PruneMode, PruneSegment}; +use reth_prune_types::{ + PruneCheckpoint, PruneMode, PruneSegment, MINIMUM_DISTANCE, MINIMUM_UNWIND_SAFE_DISTANCE, +}; use reth_stages_types::StageCheckpoint; use std::{collections::BTreeMap, path::Path}; use tracing::info; -/// Minimum blocks to keep for receipts, matching `--minimal` prune settings. -const MINIMUM_RECEIPTS_DISTANCE: u64 = 64; - -/// Minimum blocks to keep for history/bodies, matching `--minimal` prune settings -/// (`MINIMUM_UNWIND_SAFE_DISTANCE`). -const MINIMUM_HISTORY_DISTANCE: u64 = 10064; - /// Writes a [`Config`] as TOML to `/reth.toml`. /// /// If the file already exists, it is not overwritten. Returns `true` if the file was written. @@ -181,10 +176,21 @@ pub(crate) fn config_for_selections( }, }; - if is_archive || matches!(preset, Some(SelectionPreset::Archive)) { + if matches!(preset, Some(SelectionPreset::Archive)) { return Config { static_files, ..Default::default() }; } + if matches!(preset, Some(SelectionPreset::Minimal)) { + return Config { + prune: PruneConfig { + segments: DefaultPruningValues::get_global().minimal_prune_modes.clone(), + ..Default::default() + }, + static_files, + ..Default::default() + }; + } + if matches!(preset, Some(SelectionPreset::Full)) { let defaults = DefaultPruningValues::get_global(); let mut segments = defaults.full_prune_modes.clone(); @@ -205,6 +211,10 @@ pub(crate) fn config_for_selections( }; } + if is_archive { + return Config { static_files, ..Default::default() }; + } + let mut config = Config::default(); let mut prune = PruneConfig::default(); @@ -213,19 +223,21 @@ pub(crate) fn config_for_selections( } prune.segments.transaction_lookup = Some(PruneMode::Full); - if let Some(mode) = selection_to_prune_mode(tx_sel, Some(MINIMUM_HISTORY_DISTANCE)) { + if let Some(mode) = selection_to_prune_mode(tx_sel, Some(MINIMUM_UNWIND_SAFE_DISTANCE)) { prune.segments.bodies_history = Some(mode); } - if let Some(mode) = selection_to_prune_mode(receipt_sel, Some(MINIMUM_RECEIPTS_DISTANCE)) { + if let Some(mode) = selection_to_prune_mode(receipt_sel, Some(MINIMUM_DISTANCE)) { prune.segments.receipts = Some(mode); } - if let Some(mode) = selection_to_prune_mode(account_cs_sel, Some(MINIMUM_HISTORY_DISTANCE)) { + if let Some(mode) = selection_to_prune_mode(account_cs_sel, Some(MINIMUM_UNWIND_SAFE_DISTANCE)) + { prune.segments.account_history = Some(mode); } - if let Some(mode) = selection_to_prune_mode(storage_cs_sel, Some(MINIMUM_HISTORY_DISTANCE)) { + if let Some(mode) = selection_to_prune_mode(storage_cs_sel, Some(MINIMUM_UNWIND_SAFE_DISTANCE)) + { prune.segments.storage_history = Some(mode); } @@ -409,19 +421,16 @@ mod tests { // All segments clamped to their minimum distances assert_eq!( config.prune.segments.bodies_history, - Some(PruneMode::Distance(MINIMUM_HISTORY_DISTANCE)) - ); - assert_eq!( - config.prune.segments.receipts, - Some(PruneMode::Distance(MINIMUM_RECEIPTS_DISTANCE)) + Some(PruneMode::Distance(MINIMUM_UNWIND_SAFE_DISTANCE)) ); + assert_eq!(config.prune.segments.receipts, Some(PruneMode::Distance(MINIMUM_DISTANCE))); assert_eq!( config.prune.segments.account_history, - Some(PruneMode::Distance(MINIMUM_HISTORY_DISTANCE)) + Some(PruneMode::Distance(MINIMUM_UNWIND_SAFE_DISTANCE)) ); assert_eq!( config.prune.segments.storage_history, - Some(PruneMode::Distance(MINIMUM_HISTORY_DISTANCE)) + Some(PruneMode::Distance(MINIMUM_UNWIND_SAFE_DISTANCE)) ); } @@ -448,10 +457,7 @@ mod tests { assert_eq!(config.prune.segments.sender_recovery, Some(PruneMode::Full)); // Bodies follows tx selection assert_eq!(config.prune.segments.bodies_history, Some(PruneMode::Distance(10_064))); - assert_eq!( - config.prune.segments.receipts, - Some(PruneMode::Distance(MINIMUM_RECEIPTS_DISTANCE)) - ); + assert_eq!(config.prune.segments.receipts, Some(PruneMode::Distance(MINIMUM_DISTANCE))); assert_eq!(config.prune.segments.account_history, Some(PruneMode::Distance(10_064))); assert_eq!(config.prune.segments.storage_history, Some(PruneMode::Distance(10_064))); } @@ -507,15 +513,15 @@ mod tests { assert_eq!(config.prune.segments.transaction_lookup, None); assert_eq!( config.prune.segments.receipts, - Some(PruneMode::Distance(MINIMUM_HISTORY_DISTANCE)) + Some(PruneMode::Distance(MINIMUM_UNWIND_SAFE_DISTANCE)) ); assert_eq!( config.prune.segments.account_history, - Some(PruneMode::Distance(MINIMUM_HISTORY_DISTANCE)) + Some(PruneMode::Distance(MINIMUM_UNWIND_SAFE_DISTANCE)) ); assert_eq!( config.prune.segments.storage_history, - Some(PruneMode::Distance(MINIMUM_HISTORY_DISTANCE)) + Some(PruneMode::Distance(MINIMUM_UNWIND_SAFE_DISTANCE)) ); let paris_block = chain_spec @@ -525,6 +531,18 @@ mod tests { assert_eq!(config.prune.segments.bodies_history, Some(PruneMode::Before(paris_block))); } + #[test] + fn minimal_preset_matches_default_minimal_prune_config() { + let config = config_for_selections( + &BTreeMap::new(), + &empty_manifest(), + Some(SelectionPreset::Minimal), + None::<&reth_chainspec::ChainSpec>, + ); + + assert_eq!(&config.prune.segments, &DefaultPruningValues::get_global().minimal_prune_modes); + } + #[test] fn describe_selections_all_no_pruning() { let mut selections = BTreeMap::new(); diff --git a/crates/cli/commands/src/download/manifest.rs b/crates/cli/commands/src/download/manifest.rs index a0329024953..15c5cf1db57 100644 --- a/crates/cli/commands/src/download/manifest.rs +++ b/crates/cli/commands/src/download/manifest.rs @@ -257,28 +257,6 @@ impl SnapshotComponentType { matches!(self, Self::State | Self::Headers) } - /// Returns the default selection for this component in the minimal download preset. - /// - /// Matches the `--minimal` prune configuration: - /// - State/Headers: always All (required) - /// - Transactions/Changesets: Distance(10_064) (`MINIMUM_UNWIND_SAFE_DISTANCE`) - /// - Receipts: Distance(64) (`MINIMUM_DISTANCE`) - /// - TransactionSenders: None (only downloaded for archive nodes) - /// - RocksdbIndices: None (only downloaded for archive nodes) - /// - /// `tx_lookup` and `sender_recovery` are always pruned full regardless. - pub const fn minimal_selection(&self) -> ComponentSelection { - match self { - Self::State | Self::Headers => ComponentSelection::All, - Self::Transactions | Self::AccountChangesets | Self::StorageChangesets => { - ComponentSelection::Distance(10_064) - } - Self::Receipts => ComponentSelection::Distance(64), - Self::TransactionSenders => ComponentSelection::None, - Self::RocksdbIndices => ComponentSelection::None, - } - } - /// Whether this component type uses chunked archives. pub const fn is_chunked(&self) -> bool { !matches!(self, Self::State | Self::RocksdbIndices) diff --git a/crates/cli/commands/src/download/mod.rs b/crates/cli/commands/src/download/mod.rs index d7279afda21..7cd8de7f42d 100644 --- a/crates/cli/commands/src/download/mod.rs +++ b/crates/cli/commands/src/download/mod.rs @@ -108,7 +108,7 @@ use reth_db::{init_db, Database}; use reth_db_api::transaction::DbTx; use reth_fs_util as fs; use reth_node_core::args::DefaultPruningValues; -use reth_prune_types::PruneMode; +use reth_prune_types::{PruneMode, PruneModes}; use source::{ discover_manifest_url, fetch_manifest_from_source, fetch_snapshot_api_entries, print_snapshot_listing, resolve_manifest_base_url, @@ -741,8 +741,10 @@ impl> DownloadCo } // Interactive TUI + let minimal_preset = self.minimal_preset_selections(manifest); let full_preset = self.full_preset_selections(manifest); - let SelectorOutput { selections, preset } = run_selector(manifest.clone(), &full_preset)?; + let SelectorOutput { selections, preset } = + run_selector(manifest.clone(), &minimal_preset, &full_preset)?; let selected = selections.into_iter().filter(|(_, sel)| *sel != ComponentSelection::None).collect(); @@ -754,12 +756,7 @@ impl> DownloadCo &self, manifest: &SnapshotManifest, ) -> BTreeMap { - SnapshotComponentType::ALL - .iter() - .copied() - .filter(|ty| manifest.component(*ty).is_some()) - .map(|ty| (ty, ty.minimal_selection())) - .collect() + self.pruning_preset_selections(manifest, SelectionPreset::Minimal) } /// Builds the default full-node component selection for the manifest. @@ -767,23 +764,36 @@ impl> DownloadCo &self, manifest: &SnapshotManifest, ) -> BTreeMap { + self.pruning_preset_selections(manifest, SelectionPreset::Full) + } + + /// Builds component selections from the configured pruning preset. + fn pruning_preset_selections( + &self, + manifest: &SnapshotManifest, + preset: SelectionPreset, + ) -> BTreeMap { + let defaults = DefaultPruningValues::get_global(); + let (prune_modes, bodies_history_use_pre_merge) = match preset { + SelectionPreset::Minimal => (&defaults.minimal_prune_modes, false), + SelectionPreset::Full => { + (&defaults.full_prune_modes, defaults.full_bodies_history_use_pre_merge) + } + SelectionPreset::Archive => unreachable!("archive selects every component"), + }; let mut selections = BTreeMap::new(); - for ty in [ - SnapshotComponentType::State, - SnapshotComponentType::Headers, - SnapshotComponentType::Transactions, - SnapshotComponentType::Receipts, - SnapshotComponentType::AccountChangesets, - SnapshotComponentType::StorageChangesets, - SnapshotComponentType::TransactionSenders, - SnapshotComponentType::RocksdbIndices, - ] { + for &ty in &SnapshotComponentType::ALL { if manifest.component(ty).is_none() { continue; } - let selection = self.full_selection_for_component(ty, manifest.block); + let selection = self.pruning_selection_for_component( + ty, + manifest.block, + prune_modes, + bodies_history_use_pre_merge, + ); if selection != ComponentSelection::None { selections.insert(ty, selection); } @@ -792,51 +802,40 @@ impl> DownloadCo selections } - /// Returns the full preset selection for one component type. - fn full_selection_for_component( + /// Returns the component selection for one configured pruning preset. + fn pruning_selection_for_component( &self, ty: SnapshotComponentType, snapshot_block: u64, + prune_modes: &PruneModes, + bodies_history_use_pre_merge: bool, ) -> ComponentSelection { - let defaults = DefaultPruningValues::get_global(); - match ty { - SnapshotComponentType::State | SnapshotComponentType::Headers => { - ComponentSelection::All - } - SnapshotComponentType::Transactions => { - if defaults.full_bodies_history_use_pre_merge { - match self - .env - .chain - .ethereum_fork_activation(EthereumHardfork::Paris) - .block_number() - { - Some(paris) if snapshot_block >= paris => ComponentSelection::Since(paris), - Some(_) => ComponentSelection::None, - None => ComponentSelection::All, - } - } else { - selection_from_prune_mode( - defaults.full_prune_modes.bodies_history, - snapshot_block, - ) - } - } - SnapshotComponentType::Receipts => { - selection_from_prune_mode(defaults.full_prune_modes.receipts, snapshot_block) - } - SnapshotComponentType::AccountChangesets => { - selection_from_prune_mode(defaults.full_prune_modes.account_history, snapshot_block) - } - SnapshotComponentType::StorageChangesets => { - selection_from_prune_mode(defaults.full_prune_modes.storage_history, snapshot_block) - } - SnapshotComponentType::TransactionSenders => { - selection_from_prune_mode(defaults.full_prune_modes.sender_recovery, snapshot_block) + if ty == SnapshotComponentType::Transactions && bodies_history_use_pre_merge { + return match self + .env + .chain + .ethereum_fork_activation(EthereumHardfork::Paris) + .block_number() + { + Some(paris) if snapshot_block >= paris => ComponentSelection::Since(paris), + Some(_) => ComponentSelection::None, + None => ComponentSelection::All, } - // Keep hidden by default in full mode; if users want indices they can use archive. - SnapshotComponentType::RocksdbIndices => ComponentSelection::None, } + + let mode = match ty { + SnapshotComponentType::State | SnapshotComponentType::Headers => { + return ComponentSelection::All + } + SnapshotComponentType::Transactions => prune_modes.bodies_history, + SnapshotComponentType::TransactionSenders => prune_modes.sender_recovery, + SnapshotComponentType::Receipts => prune_modes.receipts, + SnapshotComponentType::AccountChangesets => prune_modes.account_history, + SnapshotComponentType::StorageChangesets => prune_modes.storage_history, + SnapshotComponentType::RocksdbIndices => return ComponentSelection::None, + }; + + selection_from_prune_mode(mode, snapshot_block) } /// Resolves the manifest source from CLI input or snapshot discovery. @@ -1280,6 +1279,67 @@ mod tests { assert!(result.is_err()); } + #[test] + fn minimal_component_selection_uses_configured_prune_modes() { + let args = + CommandParser::>::parse_from(["reth"]).args; + let history_distance = 64_864; + let modes = PruneModes { + sender_recovery: Some(PruneMode::Full), + receipts: Some(PruneMode::Distance(128)), + account_history: Some(PruneMode::Distance(history_distance)), + storage_history: Some(PruneMode::Distance(history_distance)), + bodies_history: Some(PruneMode::Distance(history_distance)), + ..Default::default() + }; + + assert_eq!( + args.pruning_selection_for_component( + SnapshotComponentType::Transactions, + 1_000_000, + &modes, + false, + ), + ComponentSelection::Distance(history_distance) + ); + assert_eq!( + args.pruning_selection_for_component( + SnapshotComponentType::Receipts, + 1_000_000, + &modes, + false, + ), + ComponentSelection::Distance(128) + ); + assert_eq!( + args.pruning_selection_for_component( + SnapshotComponentType::AccountChangesets, + 1_000_000, + &modes, + false, + ), + ComponentSelection::Distance(history_distance) + ); + assert_eq!( + args.pruning_selection_for_component( + SnapshotComponentType::StorageChangesets, + 1_000_000, + &modes, + false, + ), + ComponentSelection::Distance(history_distance) + ); + assert_eq!( + args.pruning_selection_for_component( + SnapshotComponentType::TransactionSenders, + 1_000_000, + &modes, + false, + ), + ComponentSelection::None + ); + } + #[test] fn resolve_manifest_source_requires_explicit_source_for_non_mainnet_defaults() { let args = CommandParser::>::parse_from([ diff --git a/crates/cli/commands/src/download/tui.rs b/crates/cli/commands/src/download/tui.rs index 712ce3bee6a..08319e978f9 100644 --- a/crates/cli/commands/src/download/tui.rs +++ b/crates/cli/commands/src/download/tui.rs @@ -16,6 +16,7 @@ use ratatui::{ widgets::{Block, Borders, List, ListItem, ListState, Paragraph}, Frame, Terminal, }; +use reth_prune_types::{MINIMUM_DISTANCE, MINIMUM_UNWIND_SAFE_DISTANCE}; use std::{ collections::BTreeMap, io, @@ -33,25 +34,25 @@ pub struct SelectorOutput { /// All distance presets. Groups filter this to only valid options. const DISTANCE_PRESETS: [ComponentSelection; 6] = [ ComponentSelection::None, - ComponentSelection::Distance(64), - ComponentSelection::Distance(10_064), + ComponentSelection::Distance(MINIMUM_DISTANCE), + ComponentSelection::Distance(MINIMUM_UNWIND_SAFE_DISTANCE), ComponentSelection::Distance(100_000), ComponentSelection::Distance(1_000_000), ComponentSelection::All, ]; -/// Presets for components that require at least 64 blocks (receipts). +/// Presets for components that require the minimum pruning distance (receipts). const RECEIPTS_PRESETS: [ComponentSelection; 5] = [ - ComponentSelection::Distance(64), - ComponentSelection::Distance(10_064), + ComponentSelection::Distance(MINIMUM_DISTANCE), + ComponentSelection::Distance(MINIMUM_UNWIND_SAFE_DISTANCE), ComponentSelection::Distance(100_000), ComponentSelection::Distance(1_000_000), ComponentSelection::All, ]; -/// Presets for components that require at least 10064 blocks (account/storage history). +/// Presets for components that require the minimum unwind-safe history. const HISTORY_PRESETS: [ComponentSelection; 4] = [ - ComponentSelection::Distance(10_064), + ComponentSelection::Distance(MINIMUM_UNWIND_SAFE_DISTANCE), ComponentSelection::Distance(100_000), ComponentSelection::Distance(1_000_000), ComponentSelection::All, @@ -67,11 +68,43 @@ struct DisplayGroup { required: bool, /// Valid presets for this group. Components with minimum distance requirements /// exclude presets that would produce invalid prune configs. - presets: &'static [ComponentSelection], + presets: Vec, +} + +/// Adds a configured preset to the ordered list of generic TUI choices. +fn presets_with_configured( + presets: &[ComponentSelection], + configured: ComponentSelection, +) -> Vec { + let mut presets = presets.to_vec(); + if presets.contains(&configured) { + return presets; + } + + let insert_at = match configured { + ComponentSelection::None => 0, + ComponentSelection::Distance(distance) => presets + .iter() + .position(|selection| { + matches!(selection, ComponentSelection::Distance(other) if *other > distance) || + matches!(selection, ComponentSelection::All) + }) + .unwrap_or(presets.len()), + ComponentSelection::Since(_) => presets + .iter() + .position(|selection| matches!(selection, ComponentSelection::All)) + .unwrap_or(presets.len()), + ComponentSelection::All => presets.len(), + }; + presets.insert(insert_at, configured); + presets } /// Build the display groups from available components in the manifest. -fn build_groups(manifest: &SnapshotManifest) -> Vec { +fn build_groups( + manifest: &SnapshotManifest, + minimal_preset: &BTreeMap, +) -> Vec { let has = |ty: SnapshotComponentType| manifest.component(ty).is_some(); let mut groups = Vec::new(); @@ -81,7 +114,7 @@ fn build_groups(manifest: &SnapshotManifest) -> Vec { name: "State (mdbx)", types: vec![SnapshotComponentType::State], required: true, - presets: &DISTANCE_PRESETS, + presets: DISTANCE_PRESETS.to_vec(), }); } @@ -90,7 +123,7 @@ fn build_groups(manifest: &SnapshotManifest) -> Vec { name: "Headers", types: vec![SnapshotComponentType::Headers], required: true, - presets: &DISTANCE_PRESETS, + presets: DISTANCE_PRESETS.to_vec(), }); } @@ -99,7 +132,13 @@ fn build_groups(manifest: &SnapshotManifest) -> Vec { name: "Transactions", types: vec![SnapshotComponentType::Transactions], required: false, - presets: &HISTORY_PRESETS, + presets: presets_with_configured( + &HISTORY_PRESETS, + minimal_preset + .get(&SnapshotComponentType::Transactions) + .copied() + .unwrap_or(ComponentSelection::None), + ), }); } @@ -108,7 +147,13 @@ fn build_groups(manifest: &SnapshotManifest) -> Vec { name: "Receipts", types: vec![SnapshotComponentType::Receipts], required: false, - presets: &RECEIPTS_PRESETS, + presets: presets_with_configured( + &RECEIPTS_PRESETS, + minimal_preset + .get(&SnapshotComponentType::Receipts) + .copied() + .unwrap_or(ComponentSelection::None), + ), }); } @@ -123,11 +168,12 @@ fn build_groups(manifest: &SnapshotManifest) -> Vec { if has_stor { types.push(SnapshotComponentType::StorageChangesets); } + let configured = minimal_preset.get(&types[0]).copied().unwrap_or(ComponentSelection::None); groups.push(DisplayGroup { name: "State History", types, required: false, - presets: &HISTORY_PRESETS, + presets: presets_with_configured(&HISTORY_PRESETS, configured), }); } @@ -136,6 +182,7 @@ fn build_groups(manifest: &SnapshotManifest) -> Vec { struct SelectorApp { manifest: SnapshotManifest, + minimal_preset: BTreeMap, full_preset: BTreeMap, /// Display groups shown in the TUI. groups: Vec, @@ -152,18 +199,23 @@ struct SelectorApp { impl SelectorApp { fn new( manifest: SnapshotManifest, + minimal_preset: BTreeMap, full_preset: BTreeMap, ) -> Self { - let groups = build_groups(&manifest); + let groups = build_groups(&manifest, &minimal_preset); // Default to the minimal preset (matches --minimal prune config) - let selections = groups.iter().map(|g| g.types[0].minimal_selection()).collect(); + let selections = groups + .iter() + .map(|g| minimal_preset.get(&g.types[0]).copied().unwrap_or(ComponentSelection::None)) + .collect(); let mut list_state = ListState::default(); list_state.select(Some(0)); Self { manifest, + minimal_preset, full_preset, groups, selections, @@ -178,7 +230,7 @@ impl SelectorApp { if group.required { return; } - let presets = group.presets; + let presets = &group.presets; let current = self.selections[self.cursor]; let idx = presets.iter().position(|p| *p == current).unwrap_or(0); self.selections[self.cursor] = presets[(idx + 1) % presets.len()]; @@ -191,7 +243,7 @@ impl SelectorApp { if group.required { return; } - let presets = group.presets; + let presets = &group.presets; let current = self.selections[self.cursor]; let idx = presets.iter().position(|p| *p == current).unwrap_or(0); self.selections[self.cursor] = presets[(idx + presets.len() - 1) % presets.len()]; @@ -206,14 +258,22 @@ impl SelectorApp { fn select_minimal(&mut self) { for (i, group) in self.groups.iter().enumerate() { - self.selections[i] = group.types[0].minimal_selection(); + self.selections[i] = self + .minimal_preset + .get(&group.types[0]) + .copied() + .unwrap_or(ComponentSelection::None); } self.preset = Some(SelectionPreset::Minimal); } fn select_full(&mut self) { for (i, group) in self.groups.iter().enumerate() { - let mut selection = group.types[0].minimal_selection(); + let mut selection = self + .minimal_preset + .get(&group.types[0]) + .copied() + .unwrap_or(ComponentSelection::None); for ty in &group.types { if let Some(sel) = self.full_preset.get(ty).copied() { selection = sel; @@ -278,6 +338,7 @@ impl SelectorApp { /// Runs the interactive component selector TUI. pub fn run_selector( manifest: SnapshotManifest, + minimal_preset: &BTreeMap, full_preset: &BTreeMap, ) -> eyre::Result { enable_raw_mode()?; @@ -286,7 +347,7 @@ pub fn run_selector( let backend = CrosstermBackend::new(stdout); let mut terminal = Terminal::new(backend)?; - let mut app = SelectorApp::new(manifest, full_preset.clone()); + let mut app = SelectorApp::new(manifest, minimal_preset.clone(), full_preset.clone()); let result = event_loop(&mut terminal, &mut app); disable_raw_mode()?; @@ -436,3 +497,25 @@ fn render(f: &mut Frame<'_>, app: &mut SelectorApp) { .block(Block::default().borders(Borders::ALL)); f.render_widget(footer, chunks[2]); } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn configured_distance_is_an_ordered_tui_choice() { + let configured = ComponentSelection::Distance(64_864); + let presets = presets_with_configured(&HISTORY_PRESETS, configured); + + assert_eq!( + presets, + vec![ + ComponentSelection::Distance(MINIMUM_UNWIND_SAFE_DISTANCE), + configured, + ComponentSelection::Distance(100_000), + ComponentSelection::Distance(1_000_000), + ComponentSelection::All, + ] + ); + } +} From 6165a9994c587883d4340396582fcb6cf57bd0fd Mon Sep 17 00:00:00 2001 From: Brian Picciano Date: Fri, 4 Sep 2026 12:01:58 +0000 Subject: [PATCH 158/294] refactor(provider): remove historical state conversion (#26904) --- crates/cli/commands/src/re_execute.rs | 15 +- crates/engine/tree/src/persistence.rs | 35 +- crates/storage/db-api/src/tables/mod.rs | 4 - crates/storage/provider/src/either_writer.rs | 6 +- crates/storage/provider/src/lib.rs | 7 +- .../src/providers/blockchain_provider.rs | 85 +- .../provider/src/providers/consistent.rs | 87 +- .../src/providers/database/provider.rs | 217 +-- crates/storage/provider/src/providers/mod.rs | 5 +- .../src/providers/rocksdb/provider.rs | 5 +- .../src/providers/state/historical.rs | 1421 +---------------- .../storage/provider/src/test_utils/mock.rs | 13 +- crates/storage/provider/src/traits/full.rs | 3 - crates/storage/storage-api/src/noop.rs | 13 +- crates/storage/storage-api/src/state.rs | 9 - .../storage/storage-overlay/src/provider.rs | 36 +- 16 files changed, 166 insertions(+), 1795 deletions(-) diff --git a/crates/cli/commands/src/re_execute.rs b/crates/cli/commands/src/re_execute.rs index 2cd537d042e..a2e134b73ce 100644 --- a/crates/cli/commands/src/re_execute.rs +++ b/crates/cli/commands/src/re_execute.rs @@ -17,8 +17,8 @@ use reth_evm::{execute::Executor, ConfigureEvm}; use reth_node_core::args::JitArgs; use reth_primitives_traits::{format_gas_throughput, Account, BlockBody, GotExpected}; use reth_provider::{ - BlockNumReader, BlockReader, ChainSpecProvider, DatabaseProviderFactory, ReceiptProvider, - StaticFileProviderFactory, TransactionVariant, + providers::BlockchainProvider, BlockHashReader, BlockNumReader, BlockReader, ChainSpecProvider, + DatabaseProviderFactory, ReceiptProvider, StaticFileProviderFactory, TransactionVariant, }; use reth_revm::{ database::StateProviderDatabase, @@ -150,15 +150,20 @@ impl let evm_config = evm_config.with_jit_support(); let executor_lifetime = Duration::from_secs(600); let provider = provider_factory.database_provider_ro()?.disable_long_read_transaction_safety(); + let state_provider_factory = BlockchainProvider::new(provider_factory.clone())?; // Reused across blocks for BAL hash encoding. let mut bal_buf = Vec::new(); let db_at = { |block_number: u64| { + let provider = provider_factory + .database_provider_ro() + .unwrap() + .disable_long_read_transaction_safety(); + let hash = provider.block_hash(block_number).unwrap().unwrap(); StateProviderDatabase( - provider - .history_by_block_number(block_number) - .unwrap(), + state_provider_factory + .state_provider_from_database(provider, hash), ) } }; diff --git a/crates/engine/tree/src/persistence.rs b/crates/engine/tree/src/persistence.rs index e2b58f76810..8db05e6c0e8 100644 --- a/crates/engine/tree/src/persistence.rs +++ b/crates/engine/tree/src/persistence.rs @@ -418,11 +418,11 @@ mod tests { use reth_db_common::init::init_genesis; use reth_exex_types::FinishedExExHeight; use reth_provider::{ - providers::{ProviderFactoryBuilder, ReadOnlyConfig}, + providers::{BlockchainProvider, ProviderFactoryBuilder, ReadOnlyConfig}, test_utils::{create_test_provider_factory, MockNodeTypes}, AccountReader, BalConfig, BalNotificationStream, BalStore, BalStoreHandle, ChainSpecProvider, HeaderProvider, InMemoryBalStore, ProviderError, ProviderResult, RawBal, - StorageSettingsCache, TryIntoHistoricalStateProvider, + StateProviderFactory, StorageSettingsCache, }; use reth_prune::Pruner; use reth_prune_types::PruneMode; @@ -690,6 +690,9 @@ mod tests { provider_rw.save_blocks(&input).unwrap(); provider_rw.commit().unwrap(); + let primary = BlockchainProvider::new(provider_factory.clone()).unwrap(); + let secondary_blockchain = BlockchainProvider::new(secondary.clone()).unwrap(); + // Secondary catches up and sees all 3 blocks. // Hold this provider (and its MDBX RO tx) across the reorg to test snapshot isolation. let pre_reorg_provider = secondary.provider().unwrap(); @@ -701,16 +704,14 @@ mod tests { // Check the primary can read its own historical state. { - let provider = provider_factory.provider().unwrap(); - let primary_state_at_1 = provider.history_by_block_number(1).unwrap(); + let primary_state_at_1 = primary.history_by_block_number(1).unwrap(); let primary_account = primary_state_at_1.basic_account(&signer).unwrap(); assert!(primary_account.is_some(), "primary: signer must exist at block 1"); } // Verify historical state at block 1 is accessible via changesets on the secondary. { - let provider = secondary.provider().unwrap(); - let state_at_1 = provider.history_by_block_number(1).unwrap(); + let state_at_1 = secondary_blockchain.history_by_block_number(1).unwrap(); let account_at_1 = state_at_1.basic_account(&signer).unwrap(); assert!(account_at_1.is_some(), "signer account must exist at block 1"); let account_at_1 = account_at_1.unwrap(); @@ -731,6 +732,8 @@ mod tests { balance_after_block1 - single_cost * U256::from(txs_in_block_b2); let nonce_after_reorg_block2 = nonce_after_block1 + txs_in_block_b2; + let state_at_1 = secondary_blockchain.history_by_block_number(1).unwrap(); + // Spawn the reorg on a background thread because `commit_unwind` calls // `wait_for_pre_commit_readers()` which blocks until the secondary's held // RO tx is dropped. @@ -767,25 +770,24 @@ mod tests { "pre-reorg provider must still see block 1" ); - // The held RO tx must still be able to read historical state at block 1 via - // changesets, even though the reorg thread is about to rewrite block 2's data. - // Consuming pre_reorg_provider here also unblocks the reorg commit. - let state_at_1 = pre_reorg_provider.try_into_history_at_block(1).unwrap(); + // The held overlay-backed provider must still read historical state at block 1, even + // though the reorg thread is about to rewrite block 2's data. let account = state_at_1.basic_account(&signer).unwrap(); assert!( account.is_some(), - "pre-reorg RO tx must still read signer at block 1 during reorg" + "pre-reorg state provider must still read signer at block 1 during reorg" ); let account = account.unwrap(); assert_eq!( account.balance, balance_after_block1, - "pre-reorg RO tx: signer balance at block 1 during reorg" + "pre-reorg state provider: signer balance at block 1 during reorg" ); assert_eq!( account.nonce, nonce_after_block1, - "pre-reorg RO tx: signer nonce at block 1 during reorg" + "pre-reorg state provider: signer nonce at block 1 during reorg" ); drop(state_at_1); + drop(pre_reorg_provider); reorg_handle.join().expect("reorg thread panicked"); // A new provider catches up and sees the reorged chain. @@ -805,8 +807,8 @@ mod tests { ); // Verify historical state at block 1 is still accessible after the reorg. - let provider = secondary.provider().unwrap(); - let state_at_1 = provider.history_by_block_number(1).unwrap(); + let secondary_blockchain = BlockchainProvider::new(secondary).unwrap(); + let state_at_1 = secondary_blockchain.history_by_block_number(1).unwrap(); let account_at_1 = state_at_1.basic_account(&signer).unwrap(); assert!(account_at_1.is_some(), "signer account must exist at block 1 after reorg"); let account_at_1 = account_at_1.unwrap(); @@ -820,8 +822,7 @@ mod tests { ); // Verify the latest state (at block 2) reflects the reorged execution. - let provider = secondary.provider().unwrap(); - let state_at_2 = provider.history_by_block_number(2).unwrap(); + let state_at_2 = secondary_blockchain.history_by_block_number(2).unwrap(); let account_at_2 = state_at_2.basic_account(&signer).unwrap(); assert!(account_at_2.is_some(), "signer account must exist at block 2 after reorg"); let account_at_2 = account_at_2.unwrap(); diff --git a/crates/storage/db-api/src/tables/mod.rs b/crates/storage/db-api/src/tables/mod.rs index 407e2f9ad69..a857968ef8a 100644 --- a/crates/storage/db-api/src/tables/mod.rs +++ b/crates/storage/db-api/src/tables/mod.rs @@ -427,8 +427,6 @@ tables! { /// * for N=150 we would get second shard. /// * If max block number is 200 and we ask for N=250 we would fetch last shard and know that needed entry is in `AccountPlainState`. /// * If there were no shard we would get `None` entry or entry of different storage key. - /// - /// Code example can be found in `reth_provider::HistoricalStateProviderRef` table AccountsHistory { type Key = ShardedKey

; type Value = BlockNumberList; @@ -449,8 +447,6 @@ tables! { /// * for N=150 we would get second shard. /// * If max block number is 200 and we ask for N=250 we would fetch last shard and know that needed entry is in `StoragePlainState`. /// * If there were no shard we would get `None` entry or entry of different storage key. - /// - /// Code example can be found in `reth_provider::HistoricalStateProviderRef` table StoragesHistory { type Key = StorageShardedKey; type Value = BlockNumberList; diff --git a/crates/storage/provider/src/either_writer.rs b/crates/storage/provider/src/either_writer.rs index b12390259e6..ff90f98bc68 100644 --- a/crates/storage/provider/src/either_writer.rs +++ b/crates/storage/provider/src/either_writer.rs @@ -1684,9 +1684,9 @@ mod rocksdb_tests { ); // Scenario 4: Query at pruning boundary - // Note: We test block >= lowest_available because HistoricalStateProviderRef - // errors on blocks below the pruning boundary before doing the lookup. - // The RocksDB implementation doesn't have this check at the same level. + // We test block >= lowest_available because state queries reject blocks below the + // pruning boundary before doing the lookup. The RocksDB implementation doesn't have + // this check at the same level. // This tests that when pruning IS available, both backends agree. run_account_history_scenario( "with_pruning_boundary", diff --git a/crates/storage/provider/src/lib.rs b/crates/storage/provider/src/lib.rs index edb06faadb9..4460bbc8736 100644 --- a/crates/storage/provider/src/lib.rs +++ b/crates/storage/provider/src/lib.rs @@ -22,10 +22,9 @@ pub use traits::*; /// Provider trait implementations. pub mod providers; pub use providers::{ - DatabaseProvider, DatabaseProviderRO, DatabaseProviderRW, HistoricalStateProvider, - HistoricalStateProviderRef, LatestStateProvider, LatestStateProviderRef, ProviderFactory, - PruneShardOutcome, PrunedIndices, SaveBlocksInput, StaticFileAccess, StaticFileProviderBuilder, - StaticFileWriteCtx, StaticFileWriter, + DatabaseProvider, DatabaseProviderRO, DatabaseProviderRW, LatestStateProvider, + LatestStateProviderRef, ProviderFactory, PruneShardOutcome, PrunedIndices, SaveBlocksInput, + StaticFileAccess, StaticFileProviderBuilder, StaticFileWriteCtx, StaticFileWriter, }; pub mod changeset_walker; diff --git a/crates/storage/provider/src/providers/blockchain_provider.rs b/crates/storage/provider/src/providers/blockchain_provider.rs index 952c9915005..b8afdcfbc6f 100644 --- a/crates/storage/provider/src/providers/blockchain_provider.rs +++ b/crates/storage/provider/src/providers/blockchain_provider.rs @@ -3,12 +3,12 @@ use crate::{ ConsistentProvider, ProviderNodeTypes, RocksDBProvider, StaticFileProvider, StaticFileProviderRWRefMut, }, - BalProvider, BalStoreHandle, BlockHashReader, BlockIdReader, BlockNumReader, BlockReader, - BlockReaderIdExt, BlockSource, CanonChainTracker, CanonStateNotifications, + AccountReader, BalProvider, BalStoreHandle, BlockHashReader, BlockIdReader, BlockNumReader, + BlockReader, BlockReaderIdExt, BlockSource, CanonChainTracker, CanonStateNotifications, CanonStateSubscriptions, ChainSpecProvider, ChainStateBlockReader, ChangeSetReader, DatabaseProviderFactory, HeaderProvider, ProviderError, ProviderFactory, PruneCheckpointReader, ReceiptProvider, ReceiptProviderIdExt, RocksDBProviderFactory, StageCheckpointReader, - StateProviderBox, StateProviderFactory, StateReader, StaticFileProviderFactory, + StateProvider, StateProviderBox, StateProviderFactory, StateReader, StaticFileProviderFactory, TransactionVariant, TransactionsProvider, }; use alloy_consensus::{transaction::TransactionMeta, BlockHeader}; @@ -16,7 +16,7 @@ use alloy_eips::{BlockHashOrNumber, BlockId, BlockNumHash, BlockNumberOrTag}; use alloy_primitives::{Address, BlockHash, BlockNumber, Bytes, TxHash, TxNumber, B256}; use alloy_rpc_types_engine::ForkchoiceState; use reth_chain_state::{ - BlockState, CanonicalInMemoryState, ForkChoiceNotifications, ForkChoiceSubscriptions, + CanonicalInMemoryState, ForkChoiceNotifications, ForkChoiceSubscriptions, PersistedBlockNotifications, PersistedBlockSubscriptions, }; use reth_chainspec::ChainInfo; @@ -149,18 +149,27 @@ impl BlockchainProvider { ConsistentProvider::new(self.database.clone(), self.canonical_in_memory_state()) } - /// This uses a given [`BlockState`] to initialize a state provider for that block. - fn block_state_provider( - &self, - state: &BlockState, - ) -> ProviderResult { + /// Returns a state provider for the post-state of `block_hash`. + fn state_provider_at_block_hash(&self, block_hash: B256) -> ProviderResult { let state_provider_factory = OverlayStateProviderFactory::new( self.database.clone(), - self.database.overlay_manager().overlay_builder(state.hash()), + self.database.overlay_manager().overlay_builder(block_hash), ); Ok(Box::new(state_provider_factory.database_provider_ro()?)) } + /// Returns a historical state provider using an existing database snapshot. + pub fn state_provider_from_database( + &self, + provider: StateRangeDbProvider, + block_hash: B256, + ) -> StateProviderBox { + Box::new(OverlayStateProvider::new( + provider, + self.database.overlay_manager().overlay_builder(block_hash), + )) + } + /// Returns a cursor-backed state view for a state root still only in canonical in-memory /// blocks, overlaying their merged trie state on the persisted anchor. fn block_state_range_provider( @@ -735,7 +744,7 @@ impl StateProviderFactory for BlockchainProvider { // use latest state provider if the head state exists if let Some(state) = self.canonical_in_memory_state.head_state() { trace!(target: "providers::blockchain", "Using head state for latest state provider"); - self.block_state_provider(&state) + self.state_provider_at_block_hash(state.hash()) } else { trace!(target: "providers::blockchain", "Using database state for latest state provider"); self.database.latest() @@ -782,18 +791,20 @@ impl StateProviderFactory for BlockchainProvider { let hash = provider .block_hash(block_number)? .ok_or_else(|| ProviderError::HeaderNotFound(block_number.into()))?; - provider.into_state_provider_at_block_hash(hash) + Ok(self.state_provider_from_database(provider.into_database_provider(), hash)) } fn history_by_block_hash(&self, block_hash: BlockHash) -> ProviderResult { trace!(target: "providers::blockchain", ?block_hash, "Getting history by block hash"); - self.consistent_provider()?.into_state_provider_at_block_hash(block_hash) + let provider = self.consistent_provider()?; + provider.block_number(block_hash)?.ok_or(ProviderError::BlockHashNotFound(block_hash))?; + Ok(self.state_provider_from_database(provider.into_database_provider(), block_hash)) } fn state_by_block_hash(&self, hash: BlockHash) -> ProviderResult { trace!(target: "providers::blockchain", ?hash, "Getting state by block hash"); if let Some(state) = self.canonical_in_memory_state.state_by_hash(hash) { - self.block_state_provider(&state) + self.state_provider_at_block_hash(state.hash()) } else if let Ok(state) = self.history_by_block_hash(hash) { // This could be tracked by a historical block Ok(state) @@ -815,7 +826,7 @@ impl StateProviderFactory for BlockchainProvider { if let Some(pending) = self.canonical_in_memory_state.pending_state() { // we have a pending block - return self.block_state_provider(&pending); + return self.state_provider_at_block_hash(pending.hash()); } // fallback to latest state if the pending block is not available @@ -826,14 +837,14 @@ impl StateProviderFactory for BlockchainProvider { if let Some(pending) = self.canonical_in_memory_state.pending_state() && pending.hash() == block_hash { - return self.block_state_provider(&pending).map(Some); + return self.state_provider_at_block_hash(pending.hash()).map(Some); } Ok(None) } fn maybe_pending(&self) -> ProviderResult> { if let Some(pending) = self.canonical_in_memory_state.pending_state() { - return self.block_state_provider(&pending).map(Some) + return self.state_provider_at_block_hash(pending.hash()).map(Some) } Ok(None) @@ -991,7 +1002,45 @@ impl StateReader for BlockchainProvider { &self, block: BlockNumber, ) -> ProviderResult>> { - self.consistent_provider()?.get_state(block) + if let Some(head) = self.canonical_in_memory_state.head_state() && + let Some(state) = head.block_on_chain(block.into()) + { + return Ok(Some(ExecutionOutcome::from(( + state.block_ref().execution_outcome().clone(), + block, + )))) + } + + let provider = self.database.provider()?; + let Some(block_body) = provider.block_body_indices(block)? else { return Ok(None) }; + + let from_transaction_num = block_body.first_tx_num(); + let to_transaction_num = block_body.last_tx_num(); + let account_changeset = provider.account_changesets_range(block..=block)?; + let storage_changeset = provider.storage_changeset(block)?; + + let Some(block_hash) = provider.block_hash(block)? else { return Ok(None) }; + let state_provider = OverlayStateProvider::<&_, N::Primitives>::new_ref( + &provider, + self.database.overlay_manager().overlay_builder(block_hash), + ); + let (state, reverts) = provider.populate_bundle_state( + account_changeset, + storage_changeset, + |address| state_provider.basic_account(&address), + |address, storage_key| state_provider.storage(address, storage_key), + )?; + let receipts = provider.receipts_by_tx_range(from_transaction_num..=to_transaction_num)?; + + Ok(Some(ExecutionOutcome::new_init( + state, + reverts, + // We skip new contracts since we never delete them from the database + Vec::new(), + vec![receipts], + block, + Vec::new(), + ))) } } diff --git a/crates/storage/provider/src/providers/consistent.rs b/crates/storage/provider/src/providers/consistent.rs index d10ac824c21..9fe14d05317 100644 --- a/crates/storage/provider/src/providers/consistent.rs +++ b/crates/storage/provider/src/providers/consistent.rs @@ -4,7 +4,7 @@ use crate::{ to_range, BlockHashReader, BlockIdReader, BlockNumReader, BlockReader, BlockReaderIdExt, BlockSource, ChainSpecProvider, ChangeSetReader, HeaderProvider, ProviderError, PruneCheckpointReader, ReceiptProvider, ReceiptProviderIdExt, StageCheckpointReader, - StateReader, StaticFileProviderFactory, TransactionVariant, TransactionsProvider, + StaticFileProviderFactory, TransactionVariant, TransactionsProvider, }; use alloy_consensus::{ transaction::{TransactionMeta, TxHashRef}, @@ -15,7 +15,6 @@ use alloy_primitives::{Address, BlockHash, BlockNumber, TxHash, TxNumber, B256}; use reth_chain_state::{BlockState, CanonicalInMemoryState}; use reth_chainspec::ChainInfo; use reth_db_api::models::{AccountBeforeTx, BlockNumberAddress, StoredBlockBodyIndices}; -use reth_execution_types::ExecutionOutcome; use reth_node_types::{BlockTy, HeaderTy, ReceiptTy, TxTy}; use reth_primitives_traits::{ BlockBody, RecoveredBlock, SealedHeader, SealedOrRecoveredBlock, StorageEntry, @@ -24,8 +23,8 @@ use reth_prune_types::{PruneCheckpoint, PruneSegment}; use reth_stages_types::{StageCheckpoint, StageId}; use reth_static_file_types::StaticFileSegment; use reth_storage_api::{ - BlockBodyIndicesProvider, DatabaseProviderFactory, NodePrimitivesProvider, StateProviderBox, - StorageChangeSetReader, TryIntoHistoricalStateProvider, + BlockBodyIndicesProvider, DatabaseProviderFactory, NodePrimitivesProvider, + StorageChangeSetReader, }; use reth_storage_errors::provider::ProviderResult; use revm::database::states::PlainStorageRevert; @@ -73,6 +72,13 @@ impl ConsistentProvider { Ok(Self { storage_provider, head_block, canonical_in_memory_state: state }) } + /// Consumes this consistent view and returns its database provider snapshot. + pub(crate) fn into_database_provider( + self, + ) -> as DatabaseProviderFactory>::Provider { + self.storage_provider + } + // Helper function to convert range bounds fn convert_range_bounds( &self, @@ -397,50 +403,6 @@ impl ConsistentProvider { } fetch_from_db(&self.storage_provider) } - - /// Consumes the provider and returns a state provider for the specific block hash. - pub(crate) fn into_state_provider_at_block_hash( - self, - block_hash: BlockHash, - ) -> ProviderResult { - // Resolve block number and verify it's canonical before destructuring self - let block_number = - self.block_number(block_hash)?.ok_or(ProviderError::BlockHashNotFound(block_hash))?; - self.ensure_canonical_block(block_number)?; - - let Self { storage_provider, head_block, .. } = self; - if let Some(Some(block_state)) = - head_block.as_ref().map(|b| b.block_on_chain(block_hash.into())) - { - let anchor_hash = block_state.anchor().hash; - let block_number = storage_provider - .block_number(anchor_hash)? - .ok_or(ProviderError::BlockHashNotFound(anchor_hash))?; - let latest_historical = storage_provider.try_into_history_at_block(block_number)?; - return Ok(Box::new(block_state.state_provider(latest_historical))); - } - storage_provider.try_into_history_at_block(block_number) - } -} - -impl ConsistentProvider { - /// Ensures that the given block number is canonical (synced) - /// - /// This is a helper for guarding the `HistoricalStateProvider` against block numbers that are - /// out of range and would lead to invalid results, mainly during initial sync. - /// - /// Verifying the `block_number` would be expensive since we need to lookup sync table - /// Instead, we ensure that the `block_number` is within the range of the - /// [`Self::best_block_number`] which is updated when a block is synced. - #[inline] - pub(crate) fn ensure_canonical_block(&self, block_number: BlockNumber) -> ProviderResult<()> { - let latest = self.best_block_number()?; - if block_number > latest { - Err(ProviderError::HeaderNotFound(block_number.into())) - } else { - Ok(()) - } - } } impl NodePrimitivesProvider for ConsistentProvider { @@ -1448,31 +1410,6 @@ impl ChangeSetReader for ConsistentProvider { } } -impl StateReader for ConsistentProvider { - type Receipt = ReceiptTy; - - /// Re-constructs the [`ExecutionOutcome`] from in-memory and database state, if necessary. - /// - /// If data for the block does not exist, this will return [`None`]. - /// - /// NOTE: This cannot be called safely in a loop outside of the blockchain tree thread. This is - /// because the [`CanonicalInMemoryState`] could change during a reorg, causing results to be - /// inconsistent. Currently this can safely be called within the blockchain tree thread, - /// because the tree thread is responsible for modifying the [`CanonicalInMemoryState`] in the - /// first place. - fn get_state( - &self, - block: BlockNumber, - ) -> ProviderResult>> { - if let Some(state) = self.head_block.as_ref().and_then(|b| b.block_on_chain(block.into())) { - let state = state.block_ref().execution_outcome().clone(); - Ok(Some(ExecutionOutcome::from((state, block)))) - } else { - self.storage_provider.get_state(block) - } - } -} - #[cfg(test)] mod tests { use crate::{ @@ -1937,9 +1874,7 @@ mod tests { provider_rw.commit()?; let provider = BlockchainProvider::new(factory)?; - let consistent_provider = provider.consistent_provider()?; - - let outcome = consistent_provider.get_state(1)?.expect("should return execution outcome"); + let outcome = provider.get_state(1)?.expect("should return execution outcome"); let state = &outcome.bundle.state; let account_state = state.get(&address).expect("should have account in bundle state"); diff --git a/crates/storage/provider/src/providers/database/provider.rs b/crates/storage/provider/src/providers/database/provider.rs index ff5d9b2e513..461dee4c452 100644 --- a/crates/storage/provider/src/providers/database/provider.rs +++ b/crates/storage/provider/src/providers/database/provider.rs @@ -14,11 +14,10 @@ use crate::{ AccountReader, BlockBodyWriter, BlockExecutionWriter, BlockHashReader, BlockNumReader, BlockReader, BlockWriter, BundleStateInit, ChainStateBlockReader, ChainStateBlockWriter, DBProvider, DbTxProvider, EitherReader, EitherWriter, EitherWriterDestination, HashingWriter, - HeaderProvider, HeaderSyncGapProvider, HistoricalStateProvider, HistoricalStateProviderRef, - HistoryWriter, LatestStateProvider, LatestStateProviderRef, OriginalValuesKnown, - PersistenceFrontiers, ProviderError, PruneCheckpointReader, PruneCheckpointWriter, - RawRocksDBBatch, RevertsInit, RocksBatchArg, RocksDBProviderFactory, StageCheckpointReader, - StateProviderBox, StateWriter, StaticFileProviderFactory, StatsReader, StorageReader, + HeaderProvider, HeaderSyncGapProvider, HistoryWriter, LatestStateProviderRef, + OriginalValuesKnown, PersistenceFrontiers, ProviderError, PruneCheckpointReader, + PruneCheckpointWriter, RawRocksDBBatch, RevertsInit, RocksBatchArg, RocksDBProviderFactory, + StageCheckpointReader, StateWriter, StaticFileProviderFactory, StatsReader, StorageReader, StorageTrieWriter, TransactionVariant, TransactionsProvider, TransactionsProviderExt, TrieWriter, }; @@ -63,9 +62,8 @@ use reth_stages_types::{FinishCheckpoint, StageCheckpoint, StageId}; use reth_static_file_types::StaticFileSegment; use reth_storage_api::{ BlockBodyIndicesProvider, BlockBodyReader, HistoryInfo, HistoryReader, MetadataProvider, - MetadataWriter, NodePrimitivesProvider, StateProvider, StateReader, StateWriteConfig, - StorageChangeSetReader, StoragePath, StorageSettingsCache, TryIntoHistoricalStateProvider, - WriteStateInput, + MetadataWriter, NodePrimitivesProvider, StateProvider, StateWriteConfig, + StorageChangeSetReader, StoragePath, StorageSettingsCache, WriteStateInput, }; use reth_storage_errors::provider::{ProviderResult, StaticFileWriterError}; use reth_storage_overlay::OverlayManager; @@ -300,57 +298,6 @@ impl DatabaseProvider { Box::new(LatestStateProviderRef::new(self)) } - /// Storage provider for state at that given block hash - pub fn history_by_block_hash<'a>( - &'a self, - block_hash: BlockHash, - ) -> ProviderResult> { - let block_number = - self.block_number(block_hash)?.ok_or(ProviderError::BlockHashNotFound(block_hash))?; - self.history_by_block_number(block_number) - } - - /// Storage provider for state at that given block number - pub fn history_by_block_number<'a>( - &'a self, - mut block_number: BlockNumber, - ) -> ProviderResult> { - if block_number == self.best_block_number().unwrap_or_default() && - block_number == self.last_block_number().unwrap_or_default() - { - return Ok(Box::new(LatestStateProviderRef::new(self))) - } - - // +1 as the changeset that we want is the one that was applied after this block. - block_number += 1; - - let account_history_prune_checkpoint = - self.get_prune_checkpoint(PruneSegment::AccountHistory)?; - let storage_history_prune_checkpoint = - self.get_prune_checkpoint(PruneSegment::StorageHistory)?; - - let mut state_provider = - HistoricalStateProviderRef::new(self, block_number, self.overlay_manager.clone()); - // If we pruned account or storage history, we can't return state on every historical block. - // Instead, we should cap it at the latest prune checkpoint for corresponding prune segment. - if let Some(prune_checkpoint_block_number) = - account_history_prune_checkpoint.and_then(|checkpoint| checkpoint.block_number) - { - state_provider = state_provider.with_lowest_available_account_history_block_number( - prune_checkpoint_block_number + 1, - ); - } - if let Some(prune_checkpoint_block_number) = - storage_history_prune_checkpoint.and_then(|checkpoint| checkpoint.block_number) - { - state_provider = state_provider.with_lowest_available_storage_history_block_number( - prune_checkpoint_block_number + 1, - ); - } - - Ok(Box::new(state_provider)) - } - #[cfg(feature = "test-utils")] /// Sets the prune modes for provider. pub fn set_prune_modes(&mut self, prune_modes: PruneModes) { @@ -992,58 +939,6 @@ impl DatabaseProvider TryIntoHistoricalStateProvider for DatabaseProvider { - fn try_into_history_at_block( - self, - mut block_number: BlockNumber, - ) -> ProviderResult { - let best_block = self.best_block_number().unwrap_or_default(); - - // Reject requests for blocks beyond the best block - if block_number > best_block { - return Err(ProviderError::BlockNotExecuted { - requested: block_number, - executed: best_block, - }); - } - - // If requesting state at the best block, use the latest state provider - if block_number == best_block { - return Ok(Box::new(LatestStateProvider::new(self))); - } - - // +1 as the changeset that we want is the one that was applied after this block. - block_number += 1; - - let account_history_prune_checkpoint = - self.get_prune_checkpoint(PruneSegment::AccountHistory)?; - let storage_history_prune_checkpoint = - self.get_prune_checkpoint(PruneSegment::StorageHistory)?; - let overlay_manager = self.overlay_manager.clone(); - - let mut state_provider = HistoricalStateProvider::new(self, block_number, overlay_manager); - - // If we pruned account or storage history, we can't return state on every historical block. - // Instead, we should cap it at the latest prune checkpoint for corresponding prune segment. - if let Some(prune_checkpoint_block_number) = - account_history_prune_checkpoint.and_then(|checkpoint| checkpoint.block_number) - { - state_provider = state_provider.with_lowest_available_account_history_block_number( - prune_checkpoint_block_number + 1, - ); - } - if let Some(prune_checkpoint_block_number) = - storage_history_prune_checkpoint.and_then(|checkpoint| checkpoint.block_number) - { - state_provider = state_provider.with_lowest_available_storage_history_block_number( - prune_checkpoint_block_number + 1, - ); - } - - Ok(Box::new(state_provider)) - } -} - /// For a given key, unwind all history shards that contain block numbers at or above the given /// block number. /// @@ -1339,7 +1234,7 @@ impl DatabaseProvider { /// Populate a [`BundleStateInit`] and [`RevertsInit`] using cursors over the /// [`tables::PlainAccountState`] and [`tables::PlainStorageState`] tables, based on the given /// storage and account changesets. - fn populate_bundle_state( + pub(crate) fn populate_bundle_state( &self, account_changeset: Vec<(u64, AccountBeforeTx)>, storage_changeset: Vec<(BlockNumberAddress, StorageEntry)>, @@ -1456,20 +1351,6 @@ impl DatabaseProvider { }, ) } - - fn populate_bundle_state_with_provider( - &self, - account_changeset: Vec<(u64, AccountBeforeTx)>, - storage_changeset: Vec<(BlockNumberAddress, StorageEntry)>, - state_provider: impl StateProvider, - ) -> ProviderResult<(BundleStateInit, RevertsInit)> { - self.populate_bundle_state( - account_changeset, - storage_changeset, - |address| state_provider.basic_account(&address), - |address, storage_key| state_provider.storage(address, storage_key), - ) - } } impl DatabaseProvider { @@ -1777,43 +1658,6 @@ impl HistoryReader for DatabaseProvider } } -impl StateReader for DatabaseProvider { - type Receipt = ReceiptTy; - - fn get_state( - &self, - block: BlockNumber, - ) -> ProviderResult>> { - let Some(block_body) = self.block_body_indices(block)? else { return Ok(None) }; - - let from_transaction_num = block_body.first_tx_num(); - let to_transaction_num = block_body.last_tx_num(); - - let account_changeset = self.account_changesets_range(block..=block)?; - let storage_changeset = self.storage_changeset(block)?; - - let Some(block_hash) = self.block_hash(block)? else { return Ok(None) }; - let state_provider = self.history_by_block_hash(block_hash)?; - let (state, reverts) = self.populate_bundle_state_with_provider( - account_changeset, - storage_changeset, - state_provider, - )?; - - let receipts = self.receipts_by_tx_range(from_transaction_num..=to_transaction_num)?; - - Ok(Some(ExecutionOutcome::new_init( - state, - reverts, - // We skip new contracts since we never delete them from the database - Vec::new(), - vec![receipts], - block, - Vec::new(), - ))) - } -} - impl HeaderSyncGapProvider for DatabaseProvider { @@ -5081,44 +4925,6 @@ mod tests { } } - #[test] - fn test_try_into_history_rejects_unexecuted_blocks() { - use reth_storage_api::TryIntoHistoricalStateProvider; - - let factory = create_test_provider_factory(); - - // Insert genesis block to have some data - let data = BlockchainTestData::default(); - let provider_rw = factory.provider_rw().unwrap(); - provider_rw.insert_block(&data.genesis.try_recover().unwrap()).unwrap(); - provider_rw - .write_state( - &ExecutionOutcome { first_block: 0, receipts: vec![vec![]], ..Default::default() }, - crate::OriginalValuesKnown::No, - StateWriteConfig::default(), - ) - .unwrap(); - provider_rw.commit().unwrap(); - - // Get a fresh provider - Execution checkpoint is 0, no receipts written beyond genesis - let provider = factory.provider().unwrap(); - - // Requesting historical state for block 0 (executed) should succeed - let result = provider.try_into_history_at_block(0); - assert!(result.is_ok(), "Block 0 should be available"); - - // Get another provider and request state for block 100 (not executed) - let provider = factory.provider().unwrap(); - let result = provider.try_into_history_at_block(100); - - // Should fail with BlockNotExecuted error - match result { - Err(ProviderError::BlockNotExecuted { requested: 100, .. }) => {} - Err(e) => panic!("Expected BlockNotExecuted error, got: {e:?}"), - Ok(_) => panic!("Expected error, got Ok"), - } - } - #[test] fn test_unwind_storage_hashing_with_hashed_state() { let factory = create_test_provider_factory(); @@ -5375,8 +5181,7 @@ mod tests { } #[test] - fn test_write_state_and_historical_read_hashed() { - use reth_storage_api::StateProvider; + fn test_write_state_hashed() { use reth_trie::{HashedPostState, KeccakKeyHasher}; use revm::{database::BundleState, state::AccountInfo}; @@ -5475,12 +5280,6 @@ mod tests { let account_cs = sf.account_block_changeset(1).unwrap(); assert!(!account_cs.is_empty()); assert_eq!(account_cs[0].address, address); - - let historical_value = - HistoricalStateProviderRef::new(&*provider_rw, 0, OverlayManager::default()) - .storage(address, slot_key) - .unwrap(); - assert_eq!(historical_value, None); } #[derive(Debug, Clone, Copy, PartialEq, Eq)] diff --git a/crates/storage/provider/src/providers/mod.rs b/crates/storage/provider/src/providers/mod.rs index ee2bc34adb5..b51aa898aef 100644 --- a/crates/storage/provider/src/providers/mod.rs +++ b/crates/storage/provider/src/providers/mod.rs @@ -15,10 +15,7 @@ pub use static_file::{ mod state; pub use state::{ - historical::{ - compute_history_rank, history_info, needs_prev_shard_check, HistoricalStateProvider, - HistoricalStateProviderRef, HistoryInfo, LowestAvailableBlocks, - }, + historical::{compute_history_rank, history_info, needs_prev_shard_check, HistoryInfo}, latest::{LatestStateProvider, LatestStateProviderRef}, }; diff --git a/crates/storage/provider/src/providers/rocksdb/provider.rs b/crates/storage/provider/src/providers/rocksdb/provider.rs index 9fae9514ebd..9bf75bb8256 100644 --- a/crates/storage/provider/src/providers/rocksdb/provider.rs +++ b/crates/storage/provider/src/providers/rocksdb/provider.rs @@ -3366,8 +3366,9 @@ mod tests { } /// Tests the edge case where block < `lowest_available_block_number`. - /// This case cannot be tested via `HistoricalStateProviderRef` (which errors before lookup), - /// so we keep this RocksDB-specific test to verify the low-level behavior. + /// + /// State queries reject this before the `RocksDB` lookup, so this verifies the low-level + /// behavior directly. #[test] fn test_account_history_info_pruned_before_first_entry() { let temp_dir = TempDir::new().unwrap(); diff --git a/crates/storage/provider/src/providers/state/historical.rs b/crates/storage/provider/src/providers/state/historical.rs index 2e4d3d69b96..3b27301a6d4 100644 --- a/crates/storage/provider/src/providers/state/historical.rs +++ b/crates/storage/provider/src/providers/state/historical.rs @@ -1,55 +1,7 @@ -use crate::{ - AccountReader, BlockHashReader, ChangeSetReader, HashedPostStateProvider, ProviderError, - StateProvider, StateRootProvider, -}; -use alloy_eips::merge::EPOCH_SLOTS; -use alloy_primitives::{Address, BlockNumber, Bytes, StorageKey, StorageValue, B256}; -use reth_db_api::{ - cursor::{DbCursorRO, DbDupCursorRO}, - table::Table, - tables, - transaction::DbTx, - BlockNumberList, -}; -use reth_primitives_traits::{Account, Bytecode, NodePrimitives}; -use reth_storage_api::{ - BlockNumReader, BytecodeReader, DBProvider, HistoryInfo as ReaderHistoryInfo, HistoryReader, - NodePrimitivesProvider, PruneCheckpointReader, StageCheckpointReader, StateProofProvider, - StorageChangeSetReader, StorageRootProvider, StorageSettingsCache, -}; +use alloy_primitives::BlockNumber; +use reth_db_api::{cursor::DbCursorRO, table::Table, BlockNumberList}; +use reth_storage_api::HistoryInfo as ReaderHistoryInfo; use reth_storage_errors::provider::ProviderResult; -use reth_storage_overlay::{OverlayManager, StateTrieOverlay}; -use reth_trie::{ - hashed_cursor::{zero_destroyed_account_storage, HashedPostStateCursorFactory}, - proof::{Proof, StorageProof}, - trie_cursor::InMemoryTrieCursorFactory, - updates::TrieUpdates, - witness::TrieWitness, - AccountProof, DecodedMultiProofV2, ExecutionWitnessMode, HashedPostState, HashedStorage, - KeccakKeyHasher, MultiProof, MultiProofTargets, MultiProofTargetsV2, StateRoot, - StorageMultiProof, StorageRoot, TrieInput, TrieInputSorted, -}; -use reth_trie_db::{DatabaseProof, DatabaseStateRoot, DatabaseStorageProof, DatabaseStorageRoot}; - -use std::{fmt::Debug, sync::Arc}; - -type DbStateRoot<'a, TX, A> = StateRoot< - reth_trie_db::DatabaseTrieCursorFactory<&'a TX, A>, - reth_trie_db::DatabaseHashedCursorFactory<&'a TX>, ->; -type DbStorageRoot<'a, TX, A> = StorageRoot< - reth_trie_db::DatabaseTrieCursorFactory<&'a TX, A>, - reth_trie_db::DatabaseHashedCursorFactory<&'a TX>, ->; -type DbStorageProof<'a, TX, A> = StorageProof< - 'static, - reth_trie_db::DatabaseTrieCursorFactory<&'a TX, A>, - reth_trie_db::DatabaseHashedCursorFactory<&'a TX>, ->; -type DbProof<'a, TX, A> = Proof< - reth_trie_db::DatabaseTrieCursorFactory<&'a TX, A>, - reth_trie_db::DatabaseHashedCursorFactory<&'a TX>, ->; /// Result of a history lookup for an account or storage slot. /// @@ -129,719 +81,6 @@ impl From for ReaderHistoryInfo { } } -/// State provider for a given block number which takes a tx reference. -/// -/// Historical state provider accesses the state at the start of the provided block number. -/// It means that all changes made in the provided block number are not included. -/// -/// Historical state provider reads the following tables: -/// - [`tables::AccountsHistory`] -/// - [`tables::Bytecodes`] -/// - [`tables::StoragesHistory`] -/// - [`tables::AccountChangeSets`] -/// - [`tables::StorageChangeSets`] -#[derive(Debug)] -pub struct HistoricalStateProviderRef< - 'b, - Provider, - N: NodePrimitives = ::Primitives, -> where - Provider: NodePrimitivesProvider, -{ - /// Database provider - provider: &'b Provider, - /// Manager for state trie overlays and cached changesets. - overlay_manager: OverlayManager, - /// Block number is main index for the history state of accounts and storages. - block_number: BlockNumber, - /// Lowest blocks at which different parts of the state are available. - lowest_available_blocks: LowestAvailableBlocks, -} - -impl<'b, Provider, N> HistoricalStateProviderRef<'b, Provider, N> -where - Provider: DBProvider - + ChangeSetReader - + StorageChangeSetReader - + BlockNumReader - + NodePrimitivesProvider, - N: NodePrimitives, -{ - /// Create new `StateProvider` for historical block number - pub fn new( - provider: &'b Provider, - block_number: BlockNumber, - overlay_manager: OverlayManager, - ) -> Self { - Self { - provider, - overlay_manager, - block_number, - lowest_available_blocks: Default::default(), - } - } - - /// Create new `StateProvider` for historical block number and lowest block numbers at which - /// account & storage histories are available. - pub const fn new_with_lowest_available_blocks( - provider: &'b Provider, - block_number: BlockNumber, - lowest_available_blocks: LowestAvailableBlocks, - overlay_manager: OverlayManager, - ) -> Self { - Self { provider, overlay_manager, block_number, lowest_available_blocks } - } - - /// Lookup an account in the `AccountsHistory` table using `HistoryReader`. - pub fn account_history_lookup(&self, address: Address) -> ProviderResult - where - Provider: HistoryReader, - { - if !self.lowest_available_blocks.is_account_history_available(self.block_number) { - return Err(ProviderError::StateAtBlockPruned(self.block_number)) - } - - self.provider - .account_history_info( - address, - self.block_number, - self.lowest_available_blocks.account_history_block_number, - ) - .map(Into::into) - } - - /// Lookup a storage key in the `StoragesHistory` table using `HistoryReader`. - /// - /// `lookup_key` is always a plain (unhashed) storage key. - pub fn storage_history_lookup( - &self, - address: Address, - lookup_key: B256, - ) -> ProviderResult - where - Provider: HistoryReader, - { - if !self.lowest_available_blocks.is_storage_history_available(self.block_number) { - return Err(ProviderError::StateAtBlockPruned(self.block_number)) - } - - self.provider - .storage_history_info( - address, - lookup_key, - self.block_number, - self.lowest_available_blocks.storage_history_block_number, - ) - .map(Into::into) - } - - /// Resolves a storage value by looking up the given key in history, changesets, or - /// plain state. - /// - /// `lookup_key` is always a plain (unhashed) storage key. - fn storage_by_lookup_key( - &self, - address: Address, - lookup_key: B256, - ) -> ProviderResult> - where - Provider: StorageSettingsCache + HistoryReader, - { - match self.storage_history_lookup(address, lookup_key)? { - HistoryInfo::NotYetWritten => Ok(None), - HistoryInfo::InChangeset(changeset_block_number) => self - .provider - .get_storage_before_block(changeset_block_number, address, lookup_key)? - .ok_or_else(|| ProviderError::StorageChangesetNotFound { - block_number: changeset_block_number, - address, - storage_key: Box::new(lookup_key), - }) - .map(|entry| entry.value) - .map(Some), - HistoryInfo::InPlainState | HistoryInfo::MaybeInPlainState => { - if self.provider.cached_storage_settings().use_hashed_state() { - let hashed_address = alloy_primitives::keccak256(address); - let hashed_slot = alloy_primitives::keccak256(lookup_key); - Ok(self - .tx() - .cursor_dup_read::()? - .seek_by_key_subkey(hashed_address, hashed_slot)? - .filter(|entry| entry.key == hashed_slot) - .map(|entry| entry.value) - .or(Some(StorageValue::ZERO))) - } else { - Ok(self - .tx() - .cursor_dup_read::()? - .seek_by_key_subkey(address, lookup_key)? - .filter(|entry| entry.key == lookup_key) - .map(|entry| entry.value) - .or(Some(StorageValue::ZERO))) - } - } - } - } - - /// Checks and returns `true` if distance to historical block exceeds the provided limit. - fn check_distance_against_limit(&self, limit: u64) -> ProviderResult { - let tip = self.provider.last_block_number()?; - - Ok(tip.saturating_sub(self.block_number) > limit) - } - - fn build_overlay(&self, input: TrieInputSorted) -> ProviderResult - where - Provider: - BlockHashReader + PruneCheckpointReader + StageCheckpointReader + StorageSettingsCache, - { - if self.check_distance_against_limit(EPOCH_SLOTS)? { - tracing::warn!( - target: "providers::historical_sp", - target = self.block_number, - "Attempt to calculate state root for an old block might result in OOM" - ); - } - - // Historical providers expose state at the start of `self.block_number`, so the overlay - // builder needs the previous canonical block hash to preserve those semantics. - let target_block = self.block_number.saturating_sub(1); - let anchor_hash = self - .provider - .block_hash(target_block)? - .ok_or_else(|| ProviderError::HeaderNotFound(target_block.into()))?; - - let TrieInputSorted { nodes, state, prefix_sets } = input; - let overlay_builder = self - .overlay_manager - .overlay_builder(anchor_hash) - .with_immediate_state_trie_overlay(state, nodes); - let StateTrieOverlay { trie_updates, hashed_post_state, .. } = - overlay_builder.build_state_trie_overlay(self.provider)?; - - Ok(TrieInputSorted::new(trie_updates, hashed_post_state, prefix_sets)) - } - - /// Set the lowest block number at which the account history is available. - pub const fn with_lowest_available_account_history_block_number( - mut self, - block_number: BlockNumber, - ) -> Self { - self.lowest_available_blocks.account_history_block_number = Some(block_number); - self - } - - /// Set the lowest block number at which the storage history is available. - pub const fn with_lowest_available_storage_history_block_number( - mut self, - block_number: BlockNumber, - ) -> Self { - self.lowest_available_blocks.storage_history_block_number = Some(block_number); - self - } -} - -impl HistoricalStateProviderRef<'_, Provider, N> -where - Provider: DBProvider + BlockNumReader + NodePrimitivesProvider, - N: NodePrimitives, -{ - fn tx(&self) -> &Provider::Tx { - self.provider.tx_ref() - } -} - -impl AccountReader for HistoricalStateProviderRef<'_, Provider, N> -where - Provider: DBProvider - + BlockNumReader - + ChangeSetReader - + StorageChangeSetReader - + StorageSettingsCache - + HistoryReader - + NodePrimitivesProvider, - N: NodePrimitives, -{ - /// Get basic account information. - fn basic_account(&self, address: &Address) -> ProviderResult> { - match self.account_history_lookup(*address)? { - HistoryInfo::NotYetWritten => Ok(None), - HistoryInfo::InChangeset(changeset_block_number) => { - // Use ChangeSetReader trait method to get the account from changesets - self.provider - .get_account_before_block(changeset_block_number, *address)? - .ok_or(ProviderError::AccountChangesetNotFound { - block_number: changeset_block_number, - address: *address, - }) - .map(|account_before| account_before.info) - } - HistoryInfo::InPlainState | HistoryInfo::MaybeInPlainState => { - if self.provider.cached_storage_settings().use_hashed_state() { - let hashed_address = alloy_primitives::keccak256(address); - Ok(self.tx().get_by_encoded_key::(&hashed_address)?) - } else { - Ok(self.tx().get_by_encoded_key::(address)?) - } - } - } - } -} - -impl BlockHashReader for HistoricalStateProviderRef<'_, Provider, N> -where - Provider: - DBProvider + BlockNumReader + BlockHashReader + NodePrimitivesProvider, - N: NodePrimitives, -{ - /// Get block hash by number. - fn block_hash(&self, number: u64) -> ProviderResult> { - self.provider.block_hash(number) - } - - fn canonical_hashes_range( - &self, - start: BlockNumber, - end: BlockNumber, - ) -> ProviderResult> { - self.provider.canonical_hashes_range(start, end) - } -} - -impl StateRootProvider for HistoricalStateProviderRef<'_, Provider, N> -where - Provider: DBProvider - + ChangeSetReader - + StorageChangeSetReader - + BlockNumReader - + BlockHashReader - + PruneCheckpointReader - + StageCheckpointReader - + StorageSettingsCache - + NodePrimitivesProvider, - N: NodePrimitives, -{ - fn state_root(&self, hashed_state: HashedPostState) -> ProviderResult { - reth_trie_db::with_adapter!(self.provider, |A| { - let input = self.build_overlay(TrieInputSorted::from_unsorted( - TrieInput::from_state(hashed_state), - ))?; - Ok(>::overlay_root_from_nodes(self.tx(), input)?) - }) - } - - fn state_root_from_nodes(&self, input: TrieInput) -> ProviderResult { - reth_trie_db::with_adapter!(self.provider, |A| { - let input = self.build_overlay(TrieInputSorted::from_unsorted(input))?; - Ok(>::overlay_root_from_nodes(self.tx(), input)?) - }) - } - - fn state_root_with_updates( - &self, - hashed_state: HashedPostState, - ) -> ProviderResult<(B256, TrieUpdates)> { - reth_trie_db::with_adapter!(self.provider, |A| { - let input = self.build_overlay(TrieInputSorted::from_unsorted( - TrieInput::from_state(hashed_state), - ))?; - Ok(>::overlay_root_from_nodes_with_updates(self.tx(), input)?) - }) - } - - fn state_root_from_nodes_with_updates( - &self, - input: TrieInput, - ) -> ProviderResult<(B256, TrieUpdates)> { - reth_trie_db::with_adapter!(self.provider, |A| { - let input = self.build_overlay(TrieInputSorted::from_unsorted(input))?; - Ok(>::overlay_root_from_nodes_with_updates(self.tx(), input)?) - }) - } -} - -impl StorageRootProvider for HistoricalStateProviderRef<'_, Provider, N> -where - Provider: DBProvider - + ChangeSetReader - + StorageChangeSetReader - + BlockNumReader - + BlockHashReader - + PruneCheckpointReader - + StageCheckpointReader - + StorageSettingsCache - + NodePrimitivesProvider, - N: NodePrimitives, -{ - fn storage_root( - &self, - address: Address, - hashed_storage: HashedStorage, - ) -> ProviderResult { - reth_trie_db::with_adapter!(self.provider, |A| { - let input = self.build_overlay(TrieInputSorted::from_unsorted( - TrieInput::from_state(HashedPostState::from_hashed_storage( - alloy_primitives::keccak256(address), - hashed_storage, - )), - ))?; - let hashed_storage = input - .state - .account_storages() - .get(&alloy_primitives::keccak256(address)) - .cloned() - .unwrap_or_default() - .into(); - >::overlay_root(self.tx(), address, hashed_storage) - .map_err(|err| ProviderError::Database(err.into())) - }) - } - - fn storage_proof( - &self, - address: Address, - slot: B256, - hashed_storage: HashedStorage, - ) -> ProviderResult { - reth_trie_db::with_adapter!(self.provider, |A| { - let input = self.build_overlay(TrieInputSorted::from_unsorted( - TrieInput::from_state(HashedPostState::from_hashed_storage( - alloy_primitives::keccak256(address), - hashed_storage, - )), - ))?; - let hashed_storage = input - .state - .account_storages() - .get(&alloy_primitives::keccak256(address)) - .cloned() - .unwrap_or_default() - .into(); - >::overlay_storage_proof( - self.tx(), - address, - slot, - hashed_storage, - ) - .map_err(ProviderError::from) - }) - } - - fn storage_multiproof( - &self, - address: Address, - slots: &[B256], - hashed_storage: HashedStorage, - ) -> ProviderResult { - reth_trie_db::with_adapter!(self.provider, |A| { - let input = self.build_overlay(TrieInputSorted::from_unsorted( - TrieInput::from_state(HashedPostState::from_hashed_storage( - alloy_primitives::keccak256(address), - hashed_storage, - )), - ))?; - let hashed_storage = input - .state - .account_storages() - .get(&alloy_primitives::keccak256(address)) - .cloned() - .unwrap_or_default() - .into(); - >::overlay_storage_multiproof( - self.tx(), - address, - slots, - hashed_storage, - ) - .map_err(ProviderError::from) - }) - } -} - -impl StateProofProvider for HistoricalStateProviderRef<'_, Provider, N> -where - Provider: DBProvider - + ChangeSetReader - + StorageChangeSetReader - + BlockNumReader - + BlockHashReader - + PruneCheckpointReader - + StageCheckpointReader - + StorageSettingsCache - + NodePrimitivesProvider, - N: NodePrimitives, -{ - /// Get account and storage proofs. - fn proof( - &self, - input: TrieInput, - address: Address, - slots: &[B256], - ) -> ProviderResult { - reth_trie_db::with_adapter!(self.provider, |A| { - let TrieInputSorted { nodes, state, prefix_sets } = - self.build_overlay(TrieInputSorted::from_unsorted(input))?; - let input = TrieInput::new( - Arc::unwrap_or_clone(nodes).into(), - Arc::unwrap_or_clone(state).into(), - prefix_sets, - ); - let proof = as DatabaseProof>::from_tx(self.tx()); - proof.overlay_account_proof(input, address, slots).map_err(ProviderError::from) - }) - } - - fn multiproof( - &self, - input: TrieInput, - targets: MultiProofTargets, - ) -> ProviderResult { - reth_trie_db::with_adapter!(self.provider, |A| { - let TrieInputSorted { nodes, state, prefix_sets } = - self.build_overlay(TrieInputSorted::from_unsorted(input))?; - let input = TrieInput::new( - Arc::unwrap_or_clone(nodes).into(), - Arc::unwrap_or_clone(state).into(), - prefix_sets, - ); - let proof = as DatabaseProof>::from_tx(self.tx()); - proof.overlay_multiproof(input, targets).map_err(ProviderError::from) - }) - } - - fn multiproof_v2( - &self, - input: TrieInput, - targets: MultiProofTargetsV2, - ) -> ProviderResult { - reth_trie_db::with_adapter!(self.provider, |A| { - let TrieInputSorted { nodes, state, prefix_sets } = - self.build_overlay(TrieInputSorted::from_unsorted(input))?; - let input = TrieInput::new( - Arc::unwrap_or_clone(nodes).into(), - Arc::unwrap_or_clone(state).into(), - prefix_sets, - ); - let proof = as DatabaseProof>::from_tx(self.tx()); - proof.overlay_multiproof_v2(input, targets).map_err(ProviderError::from) - }) - } - - fn witness( - &self, - input: TrieInput, - target: HashedPostState, - mode: ExecutionWitnessMode, - ) -> ProviderResult> { - reth_trie_db::with_adapter!(self.provider, |A| { - let TrieInputSorted { nodes, state, prefix_sets } = - self.build_overlay(TrieInputSorted::from_unsorted(input))?; - let witness = TrieWitness::new( - InMemoryTrieCursorFactory::new( - reth_trie_db::DatabaseTrieCursorFactory::<_, A>::new(self.tx()), - nodes.as_ref(), - ), - HashedPostStateCursorFactory::new( - reth_trie_db::DatabaseHashedCursorFactory::new(self.tx()), - state.as_ref(), - ), - ) - .with_prefix_sets_mut(prefix_sets) - .with_execution_witness_mode(mode); - let witness = - if mode.is_canonical() { witness } else { witness.always_include_root_node() }; - witness.compute(target).map_err(ProviderError::from).map(|hm| { - let mut values: Vec<_> = hm.into_values().collect(); - if mode.is_canonical() { - values.sort_unstable(); - } - values - }) - }) - } -} - -impl HashedPostStateProvider for HistoricalStateProviderRef<'_, Provider, N> -where - Provider: DBProvider - + ChangeSetReader - + StorageChangeSetReader - + BlockNumReader - + BlockHashReader - + PruneCheckpointReader - + StageCheckpointReader - + StorageSettingsCache - + NodePrimitivesProvider, - N: NodePrimitives, -{ - fn hashed_post_state( - &self, - bundle_state: &revm::database::BundleState, - ) -> ProviderResult { - let mut hashed_state = - HashedPostState::from_bundle_state::(bundle_state.state()); - if !bundle_state - .state() - .values() - .any(|account| account.was_destroyed() && account.original_info.is_some()) - { - return Ok(hashed_state) - } - - let historical = self.build_overlay(TrieInputSorted::default())?.state; - zero_destroyed_account_storage( - &HashedPostStateCursorFactory::new( - reth_trie_db::DatabaseHashedCursorFactory::new(self.tx()), - historical.as_ref(), - ), - bundle_state.state(), - &mut hashed_state, - )?; - Ok(hashed_state) - } -} - -impl StateProvider for HistoricalStateProviderRef<'_, Provider, N> -where - Provider: DBProvider - + BlockNumReader - + BlockHashReader - + ChangeSetReader - + StorageChangeSetReader - + PruneCheckpointReader - + StageCheckpointReader - + StorageSettingsCache - + HistoryReader - + NodePrimitivesProvider, - N: NodePrimitives, -{ - /// Expects a plain (unhashed) storage key slot. - fn storage( - &self, - address: Address, - storage_key: StorageKey, - ) -> ProviderResult> { - self.storage_by_lookup_key(address, storage_key) - } -} - -impl BytecodeReader for HistoricalStateProviderRef<'_, Provider, N> -where - Provider: DBProvider + BlockNumReader + NodePrimitivesProvider, - N: NodePrimitives, -{ - /// Get account code by its hash - fn bytecode_by_hash(&self, code_hash: &B256) -> ProviderResult> { - self.tx().get_by_encoded_key::(code_hash).map_err(Into::into) - } -} - -/// State provider for a given block number. -/// For more detailed description, see [`HistoricalStateProviderRef`]. -#[derive(Debug)] -pub struct HistoricalStateProvider { - /// Database provider. - provider: Provider, - /// Manager for state trie overlays and cached changesets. - overlay_manager: OverlayManager, - /// State at the block number is the main indexer of the state. - block_number: BlockNumber, - /// Lowest blocks at which different parts of the state are available. - lowest_available_blocks: LowestAvailableBlocks, -} - -impl< - Provider: DBProvider - + ChangeSetReader - + StorageChangeSetReader - + BlockNumReader - + NodePrimitivesProvider, - > HistoricalStateProvider -{ - /// Create new `StateProvider` for historical block number - pub fn new( - provider: Provider, - block_number: BlockNumber, - overlay_manager: OverlayManager, - ) -> Self { - Self { - provider, - overlay_manager, - block_number, - lowest_available_blocks: Default::default(), - } - } - - /// Set the lowest block number at which the account history is available. - pub const fn with_lowest_available_account_history_block_number( - mut self, - block_number: BlockNumber, - ) -> Self { - self.lowest_available_blocks.account_history_block_number = Some(block_number); - self - } - - /// Set the lowest block number at which the storage history is available. - pub const fn with_lowest_available_storage_history_block_number( - mut self, - block_number: BlockNumber, - ) -> Self { - self.lowest_available_blocks.storage_history_block_number = Some(block_number); - self - } -} - -impl< - Provider: DBProvider - + ChangeSetReader - + StorageChangeSetReader - + BlockNumReader - + NodePrimitivesProvider, - > HistoricalStateProvider -{ - /// Returns a new provider that takes the `TX` as reference - #[inline(always)] - fn as_ref(&self) -> HistoricalStateProviderRef<'_, Provider> { - HistoricalStateProviderRef::new_with_lowest_available_blocks( - &self.provider, - self.block_number, - self.lowest_available_blocks, - self.overlay_manager.clone(), - ) - } -} - -// Delegates all provider impls to [HistoricalStateProviderRef] -reth_storage_api::macros::delegate_provider_impls!(HistoricalStateProvider where [Provider: DBProvider + BlockNumReader + BlockHashReader + ChangeSetReader + StorageChangeSetReader + PruneCheckpointReader + StageCheckpointReader + StorageSettingsCache + HistoryReader + NodePrimitivesProvider]); - -/// Lowest blocks at which different parts of the state are available. -/// They may be [Some] if pruning is enabled. -#[derive(Clone, Copy, Debug, Default)] -pub struct LowestAvailableBlocks { - /// Lowest block number at which the account history is available. It may not be available if - /// [`reth_prune_types::PruneSegment::AccountHistory`] was pruned. - /// [`Option::None`] means all history is available. - pub account_history_block_number: Option, - /// Lowest block number at which the storage history is available. It may not be available if - /// [`reth_prune_types::PruneSegment::StorageHistory`] was pruned. - /// [`Option::None`] means all history is available. - pub storage_history_block_number: Option, -} - -impl LowestAvailableBlocks { - /// Check if account history is available at the provided block number, i.e. lowest available - /// block number for account history is less than or equal to the provided block number. - pub fn is_account_history_available(&self, at: BlockNumber) -> bool { - self.account_history_block_number.map(|block_number| block_number <= at).unwrap_or(true) - } - - /// Check if storage history is available at the provided block number, i.e. lowest available - /// block number for storage history is less than or equal to the provided block number. - pub fn is_storage_history_available(&self, at: BlockNumber) -> bool { - self.storage_history_block_number.map(|block_number| block_number <= at).unwrap_or(true) - } -} - /// Computes the rank and finds the next modification block in a history shard. /// /// Given a `block_number`, this function returns: @@ -855,7 +94,7 @@ impl LowestAvailableBlocks { /// This logic is shared between MDBX cursor-based lookups and `RocksDB` iterator lookups. #[inline] pub fn compute_history_rank( - chunk: &reth_db_api::BlockNumberList, + chunk: &BlockNumberList, block_number: BlockNumber, ) -> (u64, Option) { let mut rank = chunk.rank(block_number); @@ -910,6 +149,7 @@ where // table. let is_before_first_write = needs_prev_shard_check(rank, found_block, block_number) && !cursor.prev()?.is_some_and(|(k, _)| key_filter(&k)); + Ok(HistoryInfo::from_lookup( found_block, is_before_first_write, @@ -927,329 +167,7 @@ where #[cfg(test)] mod tests { - use super::needs_prev_shard_check; - use crate::{ - providers::state::historical::{HistoryInfo, LowestAvailableBlocks}, - test_utils::create_test_provider_factory, - AccountReader, HistoricalStateProvider, HistoricalStateProviderRef, HistoryReader, - StateProvider, - }; - use alloy_primitives::{address, b256, Address, B256, U256}; - use reth_db_api::{ - models::{storage_sharded_key::StorageShardedKey, AccountBeforeTx, ShardedKey}, - tables, - transaction::{DbTx, DbTxMut}, - BlockNumberList, - }; - use reth_primitives_traits::{Account, StorageEntry}; - use reth_storage_api::{ - BlockHashReader, BlockNumReader, ChangeSetReader, DBProvider, DatabaseProviderFactory, - NodePrimitivesProvider, PruneCheckpointReader, StageCheckpointReader, - StorageChangeSetReader, StorageSettingsCache, - }; - use reth_storage_errors::provider::ProviderError; - use reth_storage_overlay::OverlayManager; - - const ADDRESS: Address = address!("0x0000000000000000000000000000000000000001"); - const HIGHER_ADDRESS: Address = address!("0x0000000000000000000000000000000000000005"); - const STORAGE: B256 = - b256!("0x0000000000000000000000000000000000000000000000000000000000000001"); - - const fn assert_state_provider() {} - #[expect(dead_code)] - const fn assert_historical_state_provider< - T: DBProvider - + BlockNumReader - + BlockHashReader - + ChangeSetReader - + StorageChangeSetReader - + PruneCheckpointReader - + StageCheckpointReader - + StorageSettingsCache - + HistoryReader - + NodePrimitivesProvider, - >() { - assert_state_provider::>(); - } - - #[test] - fn history_provider_get_account() { - let factory = create_test_provider_factory(); - let tx = factory.provider_rw().unwrap().into_tx(); - - tx.put::( - ShardedKey { key: ADDRESS, highest_block_number: 7 }, - BlockNumberList::new([1, 3, 7]).unwrap(), - ) - .unwrap(); - tx.put::( - ShardedKey { key: ADDRESS, highest_block_number: u64::MAX }, - BlockNumberList::new([10, 15]).unwrap(), - ) - .unwrap(); - tx.put::( - ShardedKey { key: HIGHER_ADDRESS, highest_block_number: u64::MAX }, - BlockNumberList::new([4]).unwrap(), - ) - .unwrap(); - - let acc_plain = Account { nonce: 100, balance: U256::ZERO, bytecode_hash: None }; - let acc_at15 = Account { nonce: 15, balance: U256::ZERO, bytecode_hash: None }; - let acc_at10 = Account { nonce: 10, balance: U256::ZERO, bytecode_hash: None }; - let acc_at7 = Account { nonce: 7, balance: U256::ZERO, bytecode_hash: None }; - let acc_at3 = Account { nonce: 3, balance: U256::ZERO, bytecode_hash: None }; - - let higher_acc_plain = Account { nonce: 4, balance: U256::ZERO, bytecode_hash: None }; - - // setup - tx.put::(1, AccountBeforeTx { address: ADDRESS, info: None }) - .unwrap(); - tx.put::( - 3, - AccountBeforeTx { address: ADDRESS, info: Some(acc_at3) }, - ) - .unwrap(); - tx.put::( - 4, - AccountBeforeTx { address: HIGHER_ADDRESS, info: None }, - ) - .unwrap(); - tx.put::( - 7, - AccountBeforeTx { address: ADDRESS, info: Some(acc_at7) }, - ) - .unwrap(); - tx.put::( - 10, - AccountBeforeTx { address: ADDRESS, info: Some(acc_at10) }, - ) - .unwrap(); - tx.put::( - 15, - AccountBeforeTx { address: ADDRESS, info: Some(acc_at15) }, - ) - .unwrap(); - - // setup plain state - tx.put::(ADDRESS, acc_plain).unwrap(); - tx.put::(HIGHER_ADDRESS, higher_acc_plain).unwrap(); - tx.commit().unwrap(); - - let db = factory.provider().unwrap(); - - // run - assert!(matches!( - HistoricalStateProviderRef::new(&db, 1, OverlayManager::default()) - .basic_account(&ADDRESS), - Ok(None) - )); - assert!(matches!( - HistoricalStateProviderRef::new(&db, 2, OverlayManager::default()).basic_account(&ADDRESS), - Ok(Some(acc)) if acc == acc_at3 - )); - assert!(matches!( - HistoricalStateProviderRef::new(&db, 3, OverlayManager::default()).basic_account(&ADDRESS), - Ok(Some(acc)) if acc == acc_at3 - )); - assert!(matches!( - HistoricalStateProviderRef::new(&db, 4, OverlayManager::default()).basic_account(&ADDRESS), - Ok(Some(acc)) if acc == acc_at7 - )); - assert!(matches!( - HistoricalStateProviderRef::new(&db, 7, OverlayManager::default()).basic_account(&ADDRESS), - Ok(Some(acc)) if acc == acc_at7 - )); - assert!(matches!( - HistoricalStateProviderRef::new(&db, 9, OverlayManager::default()).basic_account(&ADDRESS), - Ok(Some(acc)) if acc == acc_at10 - )); - assert!(matches!( - HistoricalStateProviderRef::new(&db, 10, OverlayManager::default()).basic_account(&ADDRESS), - Ok(Some(acc)) if acc == acc_at10 - )); - assert!(matches!( - HistoricalStateProviderRef::new(&db, 11, OverlayManager::default()).basic_account(&ADDRESS), - Ok(Some(acc)) if acc == acc_at15 - )); - assert!(matches!( - HistoricalStateProviderRef::new(&db, 16, OverlayManager::default()).basic_account(&ADDRESS), - Ok(Some(acc)) if acc == acc_plain - )); - - assert!(matches!( - HistoricalStateProviderRef::new(&db, 1, OverlayManager::default()) - .basic_account(&HIGHER_ADDRESS), - Ok(None) - )); - assert!(matches!( - HistoricalStateProviderRef::new(&db, 1000, OverlayManager::default()).basic_account(&HIGHER_ADDRESS), - Ok(Some(acc)) if acc == higher_acc_plain - )); - } - - #[test] - fn history_provider_get_storage() { - let factory = create_test_provider_factory(); - let tx = factory.provider_rw().unwrap().into_tx(); - - tx.put::( - StorageShardedKey { - address: ADDRESS, - sharded_key: ShardedKey { key: STORAGE, highest_block_number: 7 }, - }, - BlockNumberList::new([3, 7]).unwrap(), - ) - .unwrap(); - tx.put::( - StorageShardedKey { - address: ADDRESS, - sharded_key: ShardedKey { key: STORAGE, highest_block_number: u64::MAX }, - }, - BlockNumberList::new([10, 15]).unwrap(), - ) - .unwrap(); - tx.put::( - StorageShardedKey { - address: HIGHER_ADDRESS, - sharded_key: ShardedKey { key: STORAGE, highest_block_number: u64::MAX }, - }, - BlockNumberList::new([4]).unwrap(), - ) - .unwrap(); - - let higher_entry_plain = StorageEntry { key: STORAGE, value: U256::from(1000) }; - let higher_entry_at4 = StorageEntry { key: STORAGE, value: U256::from(0) }; - let entry_plain = StorageEntry { key: STORAGE, value: U256::from(100) }; - let entry_at15 = StorageEntry { key: STORAGE, value: U256::from(15) }; - let entry_at10 = StorageEntry { key: STORAGE, value: U256::from(10) }; - let entry_at7 = StorageEntry { key: STORAGE, value: U256::from(7) }; - let entry_at3 = StorageEntry { key: STORAGE, value: U256::from(0) }; - - // setup - tx.put::((3, ADDRESS).into(), entry_at3).unwrap(); - tx.put::((4, HIGHER_ADDRESS).into(), higher_entry_at4).unwrap(); - tx.put::((7, ADDRESS).into(), entry_at7).unwrap(); - tx.put::((10, ADDRESS).into(), entry_at10).unwrap(); - tx.put::((15, ADDRESS).into(), entry_at15).unwrap(); - - // setup plain state - tx.put::(ADDRESS, entry_plain).unwrap(); - tx.put::(HIGHER_ADDRESS, higher_entry_plain).unwrap(); - tx.commit().unwrap(); - - let db = factory.provider().unwrap(); - - // run - assert!(matches!( - HistoricalStateProviderRef::new(&db, 0, OverlayManager::default()) - .storage(ADDRESS, STORAGE), - Ok(None) - )); - assert!(matches!( - HistoricalStateProviderRef::new(&db, 3, OverlayManager::default()) - .storage(ADDRESS, STORAGE), - Ok(Some(U256::ZERO)) - )); - assert!(matches!( - HistoricalStateProviderRef::new(&db, 4, OverlayManager::default()).storage(ADDRESS, STORAGE), - Ok(Some(expected_value)) if expected_value == entry_at7.value - )); - assert!(matches!( - HistoricalStateProviderRef::new(&db, 7, OverlayManager::default()).storage(ADDRESS, STORAGE), - Ok(Some(expected_value)) if expected_value == entry_at7.value - )); - assert!(matches!( - HistoricalStateProviderRef::new(&db, 9, OverlayManager::default()).storage(ADDRESS, STORAGE), - Ok(Some(expected_value)) if expected_value == entry_at10.value - )); - assert!(matches!( - HistoricalStateProviderRef::new(&db, 10, OverlayManager::default()).storage(ADDRESS, STORAGE), - Ok(Some(expected_value)) if expected_value == entry_at10.value - )); - assert!(matches!( - HistoricalStateProviderRef::new(&db, 11, OverlayManager::default()).storage(ADDRESS, STORAGE), - Ok(Some(expected_value)) if expected_value == entry_at15.value - )); - assert!(matches!( - HistoricalStateProviderRef::new(&db, 16, OverlayManager::default()).storage(ADDRESS, STORAGE), - Ok(Some(expected_value)) if expected_value == entry_plain.value - )); - assert!(matches!( - HistoricalStateProviderRef::new(&db, 1, OverlayManager::default()) - .storage(HIGHER_ADDRESS, STORAGE), - Ok(None) - )); - assert!(matches!( - HistoricalStateProviderRef::new(&db, 1000, OverlayManager::default()).storage(HIGHER_ADDRESS, STORAGE), - Ok(Some(expected_value)) if expected_value == higher_entry_plain.value - )); - } - - #[test] - fn history_provider_unavailable() { - let factory = create_test_provider_factory(); - let db = factory.database_provider_rw().unwrap(); - - // provider block_number < lowest available block number, - // i.e. state at provider block is pruned - let provider = HistoricalStateProviderRef::new_with_lowest_available_blocks( - &db, - 2, - LowestAvailableBlocks { - account_history_block_number: Some(3), - storage_history_block_number: Some(3), - }, - OverlayManager::default(), - ); - assert!(matches!( - provider.account_history_lookup(ADDRESS), - Err(ProviderError::StateAtBlockPruned(number)) if number == provider.block_number - )); - assert!(matches!( - provider.storage_history_lookup(ADDRESS, STORAGE), - Err(ProviderError::StateAtBlockPruned(number)) if number == provider.block_number - )); - - // provider block_number == lowest available block number, - // i.e. state at provider block is available - let provider = HistoricalStateProviderRef::new_with_lowest_available_blocks( - &db, - 2, - LowestAvailableBlocks { - account_history_block_number: Some(2), - storage_history_block_number: Some(2), - }, - OverlayManager::default(), - ); - assert!(matches!( - provider.account_history_lookup(ADDRESS), - Ok(HistoryInfo::MaybeInPlainState) - )); - assert!(matches!( - provider.storage_history_lookup(ADDRESS, STORAGE), - Ok(HistoryInfo::MaybeInPlainState) - )); - - // provider block_number == lowest available block number, - // i.e. state at provider block is available - let provider = HistoricalStateProviderRef::new_with_lowest_available_blocks( - &db, - 2, - LowestAvailableBlocks { - account_history_block_number: Some(1), - storage_history_block_number: Some(1), - }, - OverlayManager::default(), - ); - assert!(matches!( - provider.account_history_lookup(ADDRESS), - Ok(HistoryInfo::MaybeInPlainState) - )); - assert!(matches!( - provider.storage_history_lookup(ADDRESS, STORAGE), - Ok(HistoryInfo::MaybeInPlainState) - )); - } + use super::{needs_prev_shard_check, HistoryInfo}; #[test] fn test_history_info_from_lookup() { @@ -1266,333 +184,6 @@ mod tests { assert_eq!(HistoryInfo::from_lookup(None, false, None), HistoryInfo::InPlainState); } - #[test] - fn history_provider_get_storage_legacy() { - let factory = create_test_provider_factory(); - - assert!(!factory.provider().unwrap().cached_storage_settings().use_hashed_state()); - - let tx = factory.provider_rw().unwrap().into_tx(); - - tx.put::( - StorageShardedKey { - address: ADDRESS, - sharded_key: ShardedKey { key: STORAGE, highest_block_number: 7 }, - }, - BlockNumberList::new([3, 7]).unwrap(), - ) - .unwrap(); - tx.put::( - StorageShardedKey { - address: ADDRESS, - sharded_key: ShardedKey { key: STORAGE, highest_block_number: u64::MAX }, - }, - BlockNumberList::new([10, 15]).unwrap(), - ) - .unwrap(); - tx.put::( - StorageShardedKey { - address: HIGHER_ADDRESS, - sharded_key: ShardedKey { key: STORAGE, highest_block_number: u64::MAX }, - }, - BlockNumberList::new([4]).unwrap(), - ) - .unwrap(); - - let higher_entry_plain = StorageEntry { key: STORAGE, value: U256::from(1000) }; - let higher_entry_at4 = StorageEntry { key: STORAGE, value: U256::from(0) }; - let entry_plain = StorageEntry { key: STORAGE, value: U256::from(100) }; - let entry_at15 = StorageEntry { key: STORAGE, value: U256::from(15) }; - let entry_at10 = StorageEntry { key: STORAGE, value: U256::from(10) }; - let entry_at7 = StorageEntry { key: STORAGE, value: U256::from(7) }; - let entry_at3 = StorageEntry { key: STORAGE, value: U256::from(0) }; - - tx.put::((3, ADDRESS).into(), entry_at3).unwrap(); - tx.put::((4, HIGHER_ADDRESS).into(), higher_entry_at4).unwrap(); - tx.put::((7, ADDRESS).into(), entry_at7).unwrap(); - tx.put::((10, ADDRESS).into(), entry_at10).unwrap(); - tx.put::((15, ADDRESS).into(), entry_at15).unwrap(); - - tx.put::(ADDRESS, entry_plain).unwrap(); - tx.put::(HIGHER_ADDRESS, higher_entry_plain).unwrap(); - tx.commit().unwrap(); - - let db = factory.provider().unwrap(); - - assert!(matches!( - HistoricalStateProviderRef::new(&db, 0, OverlayManager::default()) - .storage(ADDRESS, STORAGE), - Ok(None) - )); - assert!(matches!( - HistoricalStateProviderRef::new(&db, 3, OverlayManager::default()) - .storage(ADDRESS, STORAGE), - Ok(Some(U256::ZERO)) - )); - assert!(matches!( - HistoricalStateProviderRef::new(&db, 4, OverlayManager::default()).storage(ADDRESS, STORAGE), - Ok(Some(expected_value)) if expected_value == entry_at7.value - )); - assert!(matches!( - HistoricalStateProviderRef::new(&db, 7, OverlayManager::default()).storage(ADDRESS, STORAGE), - Ok(Some(expected_value)) if expected_value == entry_at7.value - )); - assert!(matches!( - HistoricalStateProviderRef::new(&db, 9, OverlayManager::default()).storage(ADDRESS, STORAGE), - Ok(Some(expected_value)) if expected_value == entry_at10.value - )); - assert!(matches!( - HistoricalStateProviderRef::new(&db, 10, OverlayManager::default()).storage(ADDRESS, STORAGE), - Ok(Some(expected_value)) if expected_value == entry_at10.value - )); - assert!(matches!( - HistoricalStateProviderRef::new(&db, 11, OverlayManager::default()).storage(ADDRESS, STORAGE), - Ok(Some(expected_value)) if expected_value == entry_at15.value - )); - assert!(matches!( - HistoricalStateProviderRef::new(&db, 16, OverlayManager::default()).storage(ADDRESS, STORAGE), - Ok(Some(expected_value)) if expected_value == entry_plain.value - )); - assert!(matches!( - HistoricalStateProviderRef::new(&db, 1, OverlayManager::default()) - .storage(HIGHER_ADDRESS, STORAGE), - Ok(None) - )); - assert!(matches!( - HistoricalStateProviderRef::new(&db, 1000, OverlayManager::default()).storage(HIGHER_ADDRESS, STORAGE), - Ok(Some(expected_value)) if expected_value == higher_entry_plain.value - )); - } - - #[test] - fn history_provider_get_storage_hashed_state() { - use crate::BlockWriter; - use alloy_primitives::keccak256; - use reth_db_api::models::StorageSettings; - use reth_execution_types::ExecutionOutcome; - use reth_testing_utils::generators::{self, random_block_range, BlockRangeParams}; - use revm::database::BundleState; - use std::collections::HashMap; - - let factory = create_test_provider_factory(); - factory.set_storage_settings_cache(StorageSettings::v2()); - - let slot = U256::from_be_bytes(*STORAGE); - let account: revm::state::AccountInfo = - Account { nonce: 1, balance: U256::from(1000), bytecode_hash: None }.into(); - let higher_account: revm::state::AccountInfo = - Account { nonce: 1, balance: U256::from(2000), bytecode_hash: None }.into(); - - let mut rng = generators::rng(); - let blocks = random_block_range( - &mut rng, - 0..=15, - BlockRangeParams { parent: Some(B256::ZERO), tx_count: 0..1, ..Default::default() }, - ); - - let mut addr_storage = HashMap::default(); - addr_storage.insert(slot, (U256::ZERO, U256::from(100))); - let mut higher_storage = HashMap::default(); - higher_storage.insert(slot, (U256::ZERO, U256::from(1000))); - - type Revert = Vec<(Address, Option>, Vec<(U256, U256)>)>; - let mut reverts: Vec = vec![Vec::new(); 16]; - - reverts[3] = vec![(ADDRESS, Some(Some(account.clone())), vec![(slot, U256::ZERO)])]; - reverts[4] = - vec![(HIGHER_ADDRESS, Some(Some(higher_account.clone())), vec![(slot, U256::ZERO)])]; - reverts[7] = vec![(ADDRESS, Some(Some(account.clone())), vec![(slot, U256::from(7))])]; - reverts[10] = vec![(ADDRESS, Some(Some(account.clone())), vec![(slot, U256::from(10))])]; - reverts[15] = vec![(ADDRESS, Some(Some(account.clone())), vec![(slot, U256::from(15))])]; - - let bundle = BundleState::new( - [ - (ADDRESS, None, Some(account), addr_storage), - (HIGHER_ADDRESS, None, Some(higher_account), higher_storage), - ], - reverts, - [], - ); - - let provider_rw = factory.provider_rw().unwrap(); - provider_rw - .append_blocks_with_state( - blocks - .into_iter() - .map(|b| b.try_recover().expect("failed to seal block with senders")) - .collect(), - &ExecutionOutcome { bundle, first_block: 0, ..Default::default() }, - Default::default(), - ) - .unwrap(); - - let hashed_address = keccak256(ADDRESS); - let hashed_higher_address = keccak256(HIGHER_ADDRESS); - let hashed_storage = keccak256(STORAGE); - - provider_rw - .tx_ref() - .put::( - hashed_address, - StorageEntry { key: hashed_storage, value: U256::from(100) }, - ) - .unwrap(); - provider_rw - .tx_ref() - .put::( - hashed_higher_address, - StorageEntry { key: hashed_storage, value: U256::from(1000) }, - ) - .unwrap(); - provider_rw - .tx_ref() - .put::( - hashed_address, - Account { nonce: 1, balance: U256::from(1000), bytecode_hash: None }, - ) - .unwrap(); - provider_rw - .tx_ref() - .put::( - hashed_higher_address, - Account { nonce: 1, balance: U256::from(2000), bytecode_hash: None }, - ) - .unwrap(); - provider_rw.commit().unwrap(); - - let db = factory.provider().unwrap(); - - assert!(matches!( - HistoricalStateProviderRef::new(&db, 0, OverlayManager::default()) - .storage(ADDRESS, STORAGE), - Ok(None) - )); - assert!(matches!( - HistoricalStateProviderRef::new(&db, 3, OverlayManager::default()) - .storage(ADDRESS, STORAGE), - Ok(Some(U256::ZERO)) - )); - assert!(matches!( - HistoricalStateProviderRef::new(&db, 4, OverlayManager::default()).storage(ADDRESS, STORAGE), - Ok(Some(v)) if v == U256::from(7) - )); - assert!(matches!( - HistoricalStateProviderRef::new(&db, 7, OverlayManager::default()).storage(ADDRESS, STORAGE), - Ok(Some(v)) if v == U256::from(7) - )); - assert!(matches!( - HistoricalStateProviderRef::new(&db, 9, OverlayManager::default()).storage(ADDRESS, STORAGE), - Ok(Some(v)) if v == U256::from(10) - )); - assert!(matches!( - HistoricalStateProviderRef::new(&db, 10, OverlayManager::default()).storage(ADDRESS, STORAGE), - Ok(Some(v)) if v == U256::from(10) - )); - assert!(matches!( - HistoricalStateProviderRef::new(&db, 11, OverlayManager::default()).storage(ADDRESS, STORAGE), - Ok(Some(v)) if v == U256::from(15) - )); - assert!(matches!( - HistoricalStateProviderRef::new(&db, 16, OverlayManager::default()).storage(ADDRESS, STORAGE), - Ok(Some(v)) if v == U256::from(100) - )); - assert!(matches!( - HistoricalStateProviderRef::new(&db, 1, OverlayManager::default()) - .storage(HIGHER_ADDRESS, STORAGE), - Ok(None) - )); - assert!(matches!( - HistoricalStateProviderRef::new(&db, 1000, OverlayManager::default()).storage(HIGHER_ADDRESS, STORAGE), - Ok(Some(v)) if v == U256::from(1000) - )); - } - - #[test] - fn destroyed_storage_zeros_use_historical_state() { - use crate::BlockWriter; - use alloy_primitives::{keccak256, map::HashMap}; - use reth_execution_types::ExecutionOutcome; - use reth_stages_types::{StageCheckpoint, StageId}; - use reth_storage_api::{HashedPostStateProvider, StageCheckpointWriter}; - use reth_testing_utils::generators::{self, random_block_range, BlockRangeParams}; - use revm::{ - database::{AccountStatus, BundleAccount, BundleState}, - state::AccountInfo, - }; - - let factory = create_test_provider_factory(); - let slot = U256::from(1); - let old_value = U256::from(2); - let account = AccountInfo::default(); - let blocks = random_block_range( - &mut generators::rng(), - 0..=1, - BlockRangeParams { parent: Some(B256::ZERO), tx_count: 0..1, ..Default::default() }, - ); - let mut reverts = vec![Vec::new(); 2]; - reverts[1] = vec![(ADDRESS, None, vec![(slot, old_value)])]; - let bundle = BundleState::new( - [( - ADDRESS, - Some(account.clone()), - Some(account.clone()), - HashMap::from_iter([(slot, (old_value, U256::ZERO))]), - )], - reverts, - [], - ); - - let provider_rw = factory.provider_rw().unwrap(); - provider_rw - .append_blocks_with_state( - blocks - .into_iter() - .map(|block| block.try_recover().expect("failed to seal block with senders")) - .collect(), - &ExecutionOutcome { bundle, first_block: 0, ..Default::default() }, - Default::default(), - ) - .unwrap(); - provider_rw.save_stage_checkpoint(StageId::Finish, StageCheckpoint::new(1)).unwrap(); - provider_rw.commit().unwrap(); - - let db = factory.provider().unwrap(); - let hashed_address = keccak256(ADDRESS); - assert!(db.tx_ref().get::(hashed_address).unwrap().is_none()); - - let mut destroyed_bundle = BundleState::default(); - destroyed_bundle.state.insert( - ADDRESS, - BundleAccount::new(Some(account), None, Default::default(), AccountStatus::Destroyed), - ); - let provider = HistoricalStateProviderRef::new(&db, 1, OverlayManager::default()); - let hashed_state = provider.hashed_post_state(&destroyed_bundle).unwrap(); - - assert_eq!( - hashed_state.storages[&hashed_address].storage[&keccak256(B256::from(slot))], - U256::ZERO - ); - } - - #[test] - fn newly_created_destroyed_account_skips_historical_overlay() { - use reth_storage_api::HashedPostStateProvider; - use revm::database::{AccountStatus, BundleAccount, BundleState}; - - let factory = create_test_provider_factory(); - let db = factory.provider().unwrap(); - let mut bundle = BundleState::default(); - bundle.state.insert( - ADDRESS, - BundleAccount::new(None, None, Default::default(), AccountStatus::Destroyed), - ); - - let provider = HistoricalStateProviderRef::new(&db, 1, OverlayManager::default()); - let hashed_state = provider.hashed_post_state(&bundle).unwrap(); - - assert!(hashed_state.storages.is_empty()); - } - #[test] fn test_needs_prev_shard_check() { // Only needs check when rank == 0 and found_block != block_number diff --git a/crates/storage/provider/src/test_utils/mock.rs b/crates/storage/provider/src/test_utils/mock.rs index b75fefcfbbc..e24928c51af 100644 --- a/crates/storage/provider/src/test_utils/mock.rs +++ b/crates/storage/provider/src/test_utils/mock.rs @@ -38,7 +38,7 @@ use reth_storage_api::{ BlockBodyIndicesProvider, BytecodeReader, DBProvider, DatabaseProviderFactory, DbTxProvider, HashedPostStateProvider, HistoryInfo, HistoryReader, NodePrimitivesProvider, StageCheckpointReader, StateProofProvider, StorageChangeSetReader, StorageRootProvider, - StorageSettingsCache, TryIntoHistoricalStateProvider, + StorageSettingsCache, }; use reth_storage_errors::provider::{ConsistentViewError, ProviderError, ProviderResult}; use reth_trie::{ @@ -1269,17 +1269,6 @@ impl StatePr } } -impl - TryIntoHistoricalStateProvider for MockEthProvider -{ - fn try_into_history_at_block( - self, - block_number: BlockNumber, - ) -> ProviderResult { - self.history_by_block_number(block_number) - } -} - impl BlockBodyIndicesProvider for MockEthProvider { diff --git a/crates/storage/provider/src/traits/full.rs b/crates/storage/provider/src/traits/full.rs index 49b87029a22..04af272ce5b 100644 --- a/crates/storage/provider/src/traits/full.rs +++ b/crates/storage/provider/src/traits/full.rs @@ -11,7 +11,6 @@ use reth_chain_state::{ use reth_node_types::{BlockTy, HeaderTy, NodeTypesWithDB, ReceiptTy, TxTy}; use reth_storage_api::{ HistoryReader, NodePrimitivesProvider, StorageChangeSetReader, StorageSettingsCache, - TryIntoHistoricalStateProvider, }; use std::fmt::Debug; @@ -26,7 +25,6 @@ pub trait FullProvider: + StorageChangeSetReader + StorageSettingsCache + HistoryReader - + TryIntoHistoricalStateProvider + 'static, > + NodePrimitivesProvider + StaticFileProviderFactory @@ -65,7 +63,6 @@ impl FullProvider for T where + StorageChangeSetReader + StorageSettingsCache + HistoryReader - + TryIntoHistoricalStateProvider + 'static, > + NodePrimitivesProvider + StaticFileProviderFactory diff --git a/crates/storage/storage-api/src/noop.rs b/crates/storage/storage-api/src/noop.rs index 49178ae983b..5eb5a01e3c7 100644 --- a/crates/storage/storage-api/src/noop.rs +++ b/crates/storage/storage-api/src/noop.rs @@ -9,7 +9,7 @@ use crate::{ PruneCheckpointReader, ReceiptProvider, ReceiptProviderIdExt, StageCheckpointReader, StateProofProvider, StateProvider, StateProviderBox, StateProviderFactory, StateRangeProviderFactory, StateRangeView, StateReader, StateRootProvider, StorageRootProvider, - TransactionVariant, TransactionsProvider, TryIntoHistoricalStateProvider, + TransactionVariant, TransactionsProvider, }; #[cfg(feature = "db-api")] @@ -661,17 +661,6 @@ impl StateProviderFactory for NoopP } } -impl TryIntoHistoricalStateProvider - for NoopProvider -{ - fn try_into_history_at_block( - self, - block_number: BlockNumber, - ) -> ProviderResult { - self.history_by_block_number(block_number) - } -} - impl StageCheckpointReader for NoopProvider { fn get_stage_checkpoint(&self, _id: StageId) -> ProviderResult> { Ok(None) diff --git a/crates/storage/storage-api/src/state.rs b/crates/storage/storage-api/src/state.rs index 5c2617022ca..d448671c8ea 100644 --- a/crates/storage/storage-api/src/state.rs +++ b/crates/storage/storage-api/src/state.rs @@ -112,15 +112,6 @@ pub trait BytecodeReader { fn bytecode_by_hash(&self, code_hash: &B256) -> ProviderResult>; } -/// Trait implemented for database providers that can be converted into a historical state provider. -pub trait TryIntoHistoricalStateProvider { - /// Returns a historical [`StateProvider`] indexed by the given historic block number. - fn try_into_history_at_block( - self, - block_number: BlockNumber, - ) -> ProviderResult; -} - /// Light wrapper that returns `StateProvider` implementations that correspond to the given /// `BlockNumber`, the latest state, or the pending state. /// diff --git a/crates/storage/storage-overlay/src/provider.rs b/crates/storage/storage-overlay/src/provider.rs index 9d9c1bf928d..578e4453560 100644 --- a/crates/storage/storage-overlay/src/provider.rs +++ b/crates/storage/storage-overlay/src/provider.rs @@ -112,7 +112,7 @@ where let is_v2 = provider.cached_storage_settings().is_v2(); self.metrics.database_provider_ro_duration.record(overall_start.elapsed()); - Ok(OverlayStateProvider::new( + Ok(OverlayStateProvider::new_with_caches( provider, self.overlay_builder.clone(), Arc::clone(&self.state_trie_overlay_cache), @@ -134,7 +134,22 @@ pub struct OverlayStateProvider { } impl OverlayStateProvider, N> { - const fn new( + /// Creates an overlay state provider over an already-open database provider. + pub fn new(provider: Provider, overlay_builder: OverlayBuilder) -> Self + where + Provider: StorageSettingsCache, + { + let is_v2 = provider.cached_storage_settings().is_v2(); + Self::new_with_caches( + provider, + overlay_builder, + Default::default(), + Default::default(), + is_v2, + ) + } + + const fn new_with_caches( provider: Provider, overlay_builder: OverlayBuilder, state_trie_overlay_cache: StateTrieOverlayCache, @@ -174,6 +189,23 @@ impl OverlayStateProvider, } impl<'a, Provider, N: NodePrimitives> OverlayStateProvider<&'a Provider, N> { + /// Creates an overlay state provider over a borrowed database provider. + pub fn new_ref(provider: &'a Provider, overlay_builder: OverlayBuilder) -> Self + where + Provider: StorageSettingsCache, + { + let is_v2 = provider.cached_storage_settings().is_v2(); + Self { + provider, + overlay_builder: Some(overlay_builder), + state_trie_overlay_cache: Default::default(), + metrics: Default::default(), + state_trie_overlay: OnceCell::new(), + execution_overlay: OnceCell::new(), + is_v2, + } + } + pub(crate) fn new_with_state_trie( provider: &'a Provider, state_trie_overlay: StateTrieOverlay, From 3ff3d1757af255496d3da85eaf30f16556a11880 Mon Sep 17 00:00:00 2001 From: Brian Picciano Date: Fri, 4 Sep 2026 12:02:50 +0000 Subject: [PATCH 159/294] refactor(overlay): remove immediate overlay source (#26922) --- .../src/providers/blockchain_provider.rs | 23 +- crates/storage/storage-overlay/src/builder.rs | 433 ++++++------------ crates/storage/storage-overlay/src/manager.rs | 7 +- 3 files changed, 139 insertions(+), 324 deletions(-) diff --git a/crates/storage/provider/src/providers/blockchain_provider.rs b/crates/storage/provider/src/providers/blockchain_provider.rs index b8afdcfbc6f..ac11538c2fd 100644 --- a/crates/storage/provider/src/providers/blockchain_provider.rs +++ b/crates/storage/provider/src/providers/blockchain_provider.rs @@ -40,7 +40,7 @@ use reth_trie::{ hashed_cursor::{HashedCursor, HashedCursorFactory}, metrics::TrieRootMetrics, proof::{Proof, StorageProof}, - MultiProofTargets, StorageRoot, TrieInput, TrieInputSorted, TrieType, + MultiProofTargets, StorageRoot, TrieType, }; use std::{ ops::{RangeBounds, RangeInclusive}, @@ -170,8 +170,7 @@ impl BlockchainProvider { )) } - /// Returns a cursor-backed state view for a state root still only in canonical in-memory - /// blocks, overlaying their merged trie state on the persisted anchor. + /// Returns a cursor-backed state view for a state root still in canonical in-memory blocks. fn block_state_range_provider( &self, state_root: B256, @@ -184,23 +183,9 @@ impl BlockchainProvider { return Ok(None) }; - // Merge each in-memory block's trie delta, anchor to `matched`, oldest to newest. - let blocks: Vec<_> = matched.chain().map(|state| state.block()).collect(); - let sorted: Vec<_> = - blocks.iter().rev().map(|block| (block.hashed_state(), block.trie_updates())).collect(); - let input = TrieInput::from_blocks_sorted( - sorted.iter().map(|(state, nodes)| (state.as_ref(), nodes.as_ref())), - ); - let merged = TrieInputSorted::from_unsorted(input); - - // Anchor at the persisted block; the overlay reverts any db-tip advancement past it - // via changesets, then the merged in-memory delta applies on top. let state_provider_factory = OverlayStateProviderFactory::new( self.database.clone(), - self.database - .overlay_manager() - .overlay_builder(matched.anchor().hash) - .with_immediate_state_trie_overlay(merged.state, merged.nodes), + self.database.overlay_manager().overlay_builder(matched.hash()), ); state_provider_factory.database_provider_ro().map(Some) } @@ -3132,6 +3117,7 @@ mod tests { state: Default::default(), }; let executed = ExecutedBlock::new(Arc::new(block), Arc::new(execution_output), trie_data); + provider.database.overlay_manager().insert_block(executed.clone()); provider .canonical_in_memory_state .update_chain(NewCanonicalChain::Commit { new: vec![executed] }); @@ -3180,6 +3166,7 @@ mod tests { state: Default::default(), }; let executed = ExecutedBlock::new(Arc::new(block), Arc::new(execution_output), trie_data); + provider.database.overlay_manager().insert_block(executed.clone()); provider .canonical_in_memory_state .update_chain(NewCanonicalChain::Commit { new: vec![executed] }); diff --git a/crates/storage/storage-overlay/src/builder.rs b/crates/storage/storage-overlay/src/builder.rs index ccc779ec7d3..bf2a27f15e3 100644 --- a/crates/storage/storage-overlay/src/builder.rs +++ b/crates/storage/storage-overlay/src/builder.rs @@ -209,23 +209,6 @@ impl ExecutionOverlay { } } -/// Source of data to apply on top of the durable database state. -#[derive(Debug, Clone)] -pub enum OverlaySource { - /// Immediate overlay with already-computed data. - Immediate { - /// Trie updates overlay. - /// - /// This can be non-empty when a caller starts with an explicit `TrieInputSorted`, such - /// as historical providers. - trie: Arc, - /// Hashed state overlay. - state: Arc, - }, - /// Manager-backed overlay for in-memory state. - Managed, -} - /// Builder for calculating trie and hashed-state overlays. /// /// This stores the overlay manager, overlay configuration, and the logic for resolving overlays @@ -234,8 +217,6 @@ pub enum OverlaySource { pub struct OverlayBuilder { /// Parent hash requested by the caller. parent_hash: B256, - /// Optional overlay source. - overlay_source: Option, /// Manager used for cached changesets and overlays. overlay_manager: OverlayManager, /// Snapshot of the in-memory chain ending at the requested parent. @@ -257,7 +238,6 @@ impl OverlayBuilder { ) -> Self { Self { parent_hash, - overlay_source: Some(OverlaySource::Managed), overlay_manager, parent_state, reused_sparse_trie_anchor_hash: None, @@ -266,39 +246,25 @@ impl OverlayBuilder { } } - /// Set the overlay source. - /// - /// This overlay will be applied on top of any reverts. - pub fn with_overlay_source(mut self, source: Option) -> Self { - self.overlay_source = source; - self - } - /// Skips managed overlay construction when the sparse trie was reused and the DB tip is /// already covered by its anchor-to-parent range. - pub const fn with_skip_overlay_for_reused_sparse_trie(mut self, anchor_hash: B256) -> Self { + pub(crate) const fn with_skip_overlay_for_reused_sparse_trie( + mut self, + anchor_hash: B256, + ) -> Self { self.reused_sparse_trie_anchor_hash = Some(anchor_hash); self } /// Returns an error instead of querying revert changesets when reverts are required. - pub const fn with_no_reverts(mut self) -> Self { + pub(crate) const fn with_no_reverts(mut self) -> Self { self.no_reverts = true; self } - /// Sets an immediate hashed-state and trie-updates overlay. - pub fn with_immediate_state_trie_overlay( - mut self, - state: Arc, - trie: Arc, - ) -> Self { - self.overlay_source = Some(OverlaySource::Immediate { trie, state }); - self - } - /// Returns the durable anchor to use for this builder's parent. - pub fn anchor_at_parent(&self, provider: &Provider) -> ProviderResult + #[cfg(test)] + fn anchor_at_parent(&self, provider: &Provider) -> ProviderResult where Provider: StageCheckpointReader + BlockNumReader + PruneCheckpointReader, { @@ -307,7 +273,7 @@ impl OverlayBuilder { } /// Returns the durable anchor to use for this builder's parent using known frontiers. - pub fn anchor_at_parent_with_frontiers( + fn anchor_at_parent_with_frontiers( &self, provider: &Provider, partial_state_trie: BlockNumHash, @@ -316,27 +282,100 @@ impl OverlayBuilder { where Provider: BlockNumReader + PruneCheckpointReader, { - match &self.overlay_source { - Some(OverlaySource::Managed) => anchor_for_parent_with_frontiers( - self.parent_hash, - self.parent_state.iter().flat_map(|state| state.chain()).map(BlockState::block_ref), - partial_state_trie, - finish, - provider, - ), - _ => anchor_for_parent_with_frontiers( - self.parent_hash, - std::iter::empty::<&ExecutedBlock>(), - partial_state_trie, - finish, - provider, - ), + use std::io::Error; + + let mut in_mem_chain = self + .parent_state + .iter() + .flat_map(|state| state.chain()) + .map(BlockState::block_ref) + .peekable(); + let persisted_parent = match in_mem_chain + .peek() + .filter(|block| block.recovered_block().hash() == self.parent_hash) + .map(|block| block.recovered_block().number()) + { + Some(parent_number) if parent_number > partial_state_trie.number => None, + Some(parent_number) + if parent_number == partial_state_trie.number && + self.parent_hash == partial_state_trie.hash => + { + Some(parent_number) + } + Some(parent_number) => (provider.block_hash(parent_number)? == Some(self.parent_hash)) + .then_some(parent_number), + None if self.parent_hash == partial_state_trie.hash => Some(partial_state_trie.number), + None => provider + .block_number(self.parent_hash)? + .filter(|&number| number <= partial_state_trie.number), + }; + + let mut finish_seen = self.parent_hash == finish.hash; + let anchor = if let Some(parent_number) = persisted_parent { + BlockNumHash::new(parent_number, self.parent_hash) + } else { + let mut in_mem_chain = in_mem_chain.inspect(|block| { + finish_seen |= block.recovered_block().hash() == finish.hash; + }); + + if let Some(anchor) = + anchor_for_parent_in(self.parent_hash, &mut in_mem_chain, partial_state_trie) + { + anchor + } else { + let anchor_number = provider + .convert_hash_or_number(self.parent_hash.into())? + .ok_or(ProviderError::BlockHashNotFound(self.parent_hash))?; + BlockNumHash::new(anchor_number, self.parent_hash) + } + }; + + finish_seen |= anchor.hash == finish.hash; + + if anchor.number > partial_state_trie.number { + return Err(ProviderError::other(Error::other(format!( + "overlay anchor #{} ({}) is after partial state trie frontier #{} ({}); missing trie updates for blocks #{}..=#{}", + anchor.number, + anchor.hash, + partial_state_trie.number, + partial_state_trie.hash, + partial_state_trie.number + 1, + anchor.number, + )))) + } + + // If the Finish block (db tip) was seen in the in-memory chain then we know that anchor is + // on the same chain as partial_state_trie as well. Given that anchor <= partial_state_trie, + // we can be sure that the in-memory chain is a superset of partial_state_trie+1..finish, + // and therefore can be used without reverts. + if finish_seen { + return Ok(AnchorForParent::NoReverts { anchor }) } + + // Otherwise reverts are required; we check the changesets to make sure they are actually + // available before signaling that they are required. + let account_history = provider + .get_prune_checkpoint(PruneSegment::AccountHistory)? + .and_then(|checkpoint| checkpoint.block_number); + let storage_history = provider + .get_prune_checkpoint(PruneSegment::StorageHistory)? + .and_then(|checkpoint| checkpoint.block_number); + let lower_bound = account_history.max(storage_history).unwrap_or_default(); + let available_range = lower_bound..=finish.number; + if !available_range.contains(&anchor.number) { + return Err(ProviderError::InsufficientChangesets { + requested: anchor.number, + available: available_range, + }) + } + + Ok(AnchorForParent::RevertsRequired { anchor, finish }) } /// Builds the effective state trie overlay for the given provider. + #[cfg(test)] #[instrument(level = "debug", target = "storage::overlay", skip_all)] - pub fn build_state_trie_overlay( + fn build_state_trie_overlay( &self, provider: &Provider, ) -> ProviderResult @@ -362,7 +401,7 @@ impl OverlayBuilder { skip_all, fields(?state_trie_tip_block, ?finish_tip_block, parent_hash = ?self.parent_hash) )] - pub fn build_state_trie_overlay_at_frontiers( + pub(crate) fn build_state_trie_overlay_at_frontiers( &self, provider: &Provider, state_trie_tip_block: BlockNumHash, @@ -506,8 +545,9 @@ impl OverlayBuilder { } /// Returns the in-memory execution overlay and the block for historical fallback reads. - #[instrument(level = "trace", target = "storage::overlay", skip_all)] - pub fn execution_overlay( + #[cfg(test)] + #[instrument(level = "debug", target = "storage::overlay", skip_all)] + fn execution_overlay( &self, provider: &Provider, ) -> ProviderResult<(Arc, Option)> @@ -530,7 +570,7 @@ impl OverlayBuilder { skip_all, fields(?state_trie_tip_block, ?finish_tip_block, parent_hash = ?self.parent_hash) )] - pub fn execution_overlay_at_frontiers( + pub(crate) fn execution_overlay_at_frontiers( &self, provider: &Provider, state_trie_tip_block: BlockNumHash, @@ -545,14 +585,13 @@ impl OverlayBuilder { { let anchor_for_parent = self.anchor_at_parent_with_frontiers(provider, state_trie_tip_block, finish_tip_block)?; - let fallback_block_number = match &anchor_for_parent { - AnchorForParent::RevertsRequired { anchor, .. } => Some(anchor.number + 1), - AnchorForParent::NoReverts { .. } => None, + let (anchor_hash, fallback_block_number) = match anchor_for_parent { + AnchorForParent::RevertsRequired { anchor, .. } => { + (anchor.hash, Some(anchor.number + 1)) + } + AnchorForParent::NoReverts { anchor } => (anchor.hash, None), }; - Ok(( - self.resolve_execution_overlay(anchor_for_parent.anchor().hash)?, - fallback_block_number, - )) + Ok((self.resolve_execution_overlay(anchor_hash)?, fallback_block_number)) } /// Resolves the effective overlay (trie updates, hashed state). @@ -560,37 +599,17 @@ impl OverlayBuilder { &self, anchor_hash: BlockHash, ) -> ProviderResult<(Arc, Arc)> { - match &self.overlay_source { - Some(OverlaySource::Managed) => { - if anchor_hash == self.parent_hash { - Ok(( - Arc::new(TrieUpdatesSorted::default()), - Arc::new(HashedPostStateSorted::default()), - )) - } else { - let parent_state = self.parent_state.as_ref().ok_or_else(|| { - ProviderError::other(std::io::Error::other( - "state trie overlay cannot be anchored without in-memory parent state", - )) - })?; - self.overlay_manager - .overlay_for_parent(parent_state, anchor_hash) - .map_err(ProviderError::other) - } - } - Some(OverlaySource::Immediate { trie, state }) => { - if anchor_hash != self.parent_hash { - return Err(ProviderError::other(std::io::Error::other(format!( - "anchor_hash {anchor_hash} doesn't match OverlayBuilder's configured parent ({})", - self.parent_hash - )))) - } - Ok((Arc::clone(trie), Arc::clone(state))) - } - None => Ok(( - Arc::new(TrieUpdatesSorted::default()), - Arc::new(HashedPostStateSorted::default()), - )), + if anchor_hash == self.parent_hash { + Ok((Arc::new(TrieUpdatesSorted::default()), Arc::new(HashedPostStateSorted::default()))) + } else { + let parent_state = self.parent_state.as_ref().ok_or_else(|| { + ProviderError::other(std::io::Error::other( + "state trie overlay cannot be anchored without in-memory parent state", + )) + })?; + self.overlay_manager + .overlay_for_parent(parent_state, anchor_hash) + .map_err(ProviderError::other) } } @@ -599,19 +618,15 @@ impl OverlayBuilder { &self, anchor_hash: BlockHash, ) -> ProviderResult> { - match &self.overlay_source { - Some(OverlaySource::Managed) if anchor_hash != self.parent_hash => { - let parent_state = self.parent_state.as_ref().ok_or_else(|| { - ProviderError::other(std::io::Error::other("missing in-memory parent state")) - })?; - self.overlay_manager - .execution_overlay_for_block_state(parent_state, anchor_hash) - .map_err(ProviderError::other) - } - Some(OverlaySource::Managed) | None => Ok(Arc::new(ExecutionOverlay::default())), - Some(OverlaySource::Immediate { .. }) => Err(ProviderError::other( - std::io::Error::other("immediate state trie overlay has no execution state"), - )), + if anchor_hash == self.parent_hash { + Ok(Arc::new(ExecutionOverlay::default())) + } else { + let parent_state = self.parent_state.as_ref().ok_or_else(|| { + ProviderError::other(std::io::Error::other("missing in-memory parent state")) + })?; + self.overlay_manager + .execution_overlay_for_block_state(parent_state, anchor_hash) + .map_err(ProviderError::other) } } @@ -642,13 +657,8 @@ impl OverlayBuilder { ) -> bool { let Some(anchor_hash) = self.reused_sparse_trie_anchor_hash else { return false }; - match &self.overlay_source { - Some(OverlaySource::Managed) => { - self.contains_hash(anchor_hash, state_trie_tip_hash) && - self.contains_hash(anchor_hash, finish_tip_hash) - } - _ => false, - } + self.contains_hash(anchor_hash, state_trie_tip_hash) && + self.contains_hash(anchor_hash, finish_tip_hash) } fn contains_hash(&self, anchor_hash: B256, hash: B256) -> bool { @@ -671,7 +681,7 @@ impl OverlayBuilder { /// Returns the highest blocks whose state/trie data and non-state/trie data are durably /// available in the database. -pub fn database_state_frontiers( +pub(crate) fn database_state_frontiers( provider: &Provider, ) -> ProviderResult<(BlockNumHash, BlockNumHash)> where @@ -740,7 +750,7 @@ fn anchor_for_parent_in<'a, N: NodePrimitives + 'a>( /// Describes whether an overlay must revert the database before using its anchor. #[derive(Debug)] -pub enum AnchorForParent { +enum AnchorForParent { /// The in-memory chain covers the durable frontiers through this anchor. NoReverts { /// Block to anchor the overlay to. @@ -755,171 +765,6 @@ pub enum AnchorForParent { }, } -impl AnchorForParent { - /// Returns the durable block anchoring this overlay. - pub const fn anchor(&self) -> BlockNumHash { - match self { - Self::NoReverts { anchor, .. } | Self::RevertsRequired { anchor, .. } => *anchor, - } - } -} - -/// Returns the anchor block to use for the target parent and a chain of in-memory blocks. -/// -/// # Arguments -/// * `parent`: The block whose post-state is being targeted. -/// * `in_mem_chain`: Yields the in-memory blocks in the chain, starting at `parent_hash`. -/// * `provider`: Used to resolve the durable frontiers and check changeset availability. -pub fn anchor_for_parent<'a, N, Provider>( - parent_hash: B256, - in_mem_chain: impl Iterator>, - provider: &Provider, -) -> ProviderResult -where - N: NodePrimitives + 'a, - Provider: StageCheckpointReader + BlockNumReader + PruneCheckpointReader, -{ - let (partial_state_trie, finish) = database_state_frontiers(provider)?; - anchor_for_parent_with_frontiers( - parent_hash, - in_mem_chain, - partial_state_trie, - finish, - provider, - ) -} - -/// Returns the anchor block to use for the target parent and a chain of in-memory blocks using -/// known durable frontiers. -/// -/// # Arguments -/// * `parent`: The block whose post-state is being targeted. -/// * `in_mem_chain`: Yields the in-memory blocks in the chain, starting at `parent_hash`. -/// * `partial_state_trie`: The durable state/trie frontier. -/// * `finish`: The durable Finish frontier. -/// * `provider`: Used to resolve the parent and check changeset availability. -pub fn anchor_for_parent_with_frontiers<'a, N, Provider>( - parent_hash: B256, - in_mem_chain: impl Iterator>, - partial_state_trie: BlockNumHash, - finish: BlockNumHash, - provider: &Provider, -) -> ProviderResult -where - N: NodePrimitives + 'a, - Provider: BlockNumReader + PruneCheckpointReader, -{ - use std::io::Error; - - let mut in_mem_chain = in_mem_chain.peekable(); - let persisted_parent = persisted_parent_number( - parent_hash, - in_mem_chain.peek().copied(), - partial_state_trie, - provider, - )?; - - let mut finish_seen = parent_hash == finish.hash; - let anchor = if let Some(parent_number) = persisted_parent { - BlockNumHash::new(parent_number, parent_hash) - } else { - let mut in_mem_chain = in_mem_chain.inspect(|block| { - finish_seen |= block.recovered_block().hash() == finish.hash; - }); - - if let Some(anchor) = - anchor_for_parent_in(parent_hash, &mut in_mem_chain, partial_state_trie) - { - anchor - } else { - let anchor_hash = parent_hash; - let anchor_number = provider - .convert_hash_or_number(anchor_hash.into())? - .ok_or(ProviderError::BlockHashNotFound(anchor_hash))?; - BlockNumHash::new(anchor_number, anchor_hash) - } - }; - - finish_seen |= anchor.hash == finish.hash; - - if anchor.number > partial_state_trie.number { - return Err(ProviderError::other(Error::other(format!( - "overlay anchor #{} ({}) is after partial state trie frontier #{} ({}); missing trie updates for blocks #{}..=#{}", - anchor.number, - anchor.hash, - partial_state_trie.number, - partial_state_trie.hash, - partial_state_trie.number + 1, - anchor.number, - )))) - } - - // If the Finish block (db tip) was seen in the in-memory chain then we know that anchor is on - // the same chain as partial_state_trie as well. Given that anchor <= partial_state_trie, we can - // be sure that the in-memory chain is a superset of partial_state_trie+1..finish, and therefore - // can be used without reverts. - if finish_seen { - return Ok(AnchorForParent::NoReverts { anchor }) - } - - // Otherwise reverts are required; we check the changesets to make sure they are actually - // available before signaling that they are required. - let account_history = provider - .get_prune_checkpoint(PruneSegment::AccountHistory)? - .and_then(|checkpoint| checkpoint.block_number); - let storage_history = provider - .get_prune_checkpoint(PruneSegment::StorageHistory)? - .and_then(|checkpoint| checkpoint.block_number); - let lower_bound = account_history.max(storage_history).unwrap_or_default(); - let available_range = lower_bound..=finish.number; - if !available_range.contains(&anchor.number) { - return Err(ProviderError::InsufficientChangesets { - requested: anchor.number, - available: available_range, - }) - } - - Ok(AnchorForParent::RevertsRequired { anchor, finish }) -} - -/// Returns the parent number when `parent_hash` is a durable anchor. -fn persisted_parent_number( - parent_hash: B256, - managed_parent: Option<&ExecutedBlock>, - partial_state_trie: BlockNumHash, - provider: &Provider, -) -> ProviderResult> -where - N: NodePrimitives, - Provider: BlockNumReader, -{ - let managed_parent_number = managed_parent - .filter(|block| block.recovered_block().hash() == parent_hash) - .map(|block| block.recovered_block().number()); - - // Managed blocks already carry their number. Blocks above the state trie frontier cannot be a - // persisted anchor; for older blocks, verify canonicality with a number-to-hash lookup. - if let Some(parent_number) = managed_parent_number { - if parent_number > partial_state_trie.number { - return Ok(None) - } - if parent_number == partial_state_trie.number && parent_hash == partial_state_trie.hash { - return Ok(Some(parent_number)) - } - return Ok( - (provider.block_hash(parent_number)? == Some(parent_hash)).then_some(parent_number) - ) - } - - if parent_hash == partial_state_trie.hash { - Ok(Some(partial_state_trie.number)) - } else { - provider.block_number(parent_hash).map(|parent_number| { - parent_number.filter(|&number| number <= partial_state_trie.number) - }) - } -} - #[cfg(test)] mod tests { use super::*; @@ -1242,19 +1087,6 @@ mod tests { assert!(overlay.code_hashes.is_empty()); } - #[test] - fn execution_overlay_rejects_immediate_state_trie_overlay() { - let (factory, blocks) = setup_frontiers(1, 1); - let provider = factory.provider().unwrap(); - let error = OverlayManager::::default() - .overlay_builder(blocks[1].recovered_block().hash()) - .with_immediate_state_trie_overlay(Default::default(), Default::default()) - .execution_overlay(&provider) - .unwrap_err(); - - assert!(error.to_string().contains("immediate state trie overlay has no execution state")); - } - #[test] fn execution_overlay_uses_managed_blocks_after_the_anchor() { let (factory, blocks) = setup_frontiers(1, 3); @@ -1399,7 +1231,6 @@ mod tests { let provider = factory.provider().unwrap(); let error = OverlayManager::::default() .overlay_builder(blocks[3].recovered_block().hash()) - .with_overlay_source(None) .build_state_trie_overlay(&provider) .unwrap_err(); diff --git a/crates/storage/storage-overlay/src/manager.rs b/crates/storage/storage-overlay/src/manager.rs index 1db1ea8db10..dfcef3f9508 100644 --- a/crates/storage/storage-overlay/src/manager.rs +++ b/crates/storage/storage-overlay/src/manager.rs @@ -561,10 +561,7 @@ impl OverlayManager { } /// Returns every in-memory block in the chain whose tip is `parent_hash`. - pub(crate) fn parent_chain( - &self, - parent_hash: B256, - ) -> impl Iterator> + '_ { + fn parent_chain(&self, parent_hash: B256) -> impl Iterator> + '_ { let mut hash = parent_hash; std::iter::from_fn(move || { let block = self.blocks.get(&hash)?; @@ -636,7 +633,7 @@ impl OverlayManager { /// Error returned when a state trie overlay cannot be built from the manager's current block set. #[derive(Debug)] -pub struct StateTrieOverlayError { +pub(crate) struct StateTrieOverlayError { /// Requested in-memory tip hash. tip_hash: B256, /// Requested anchor hash. From 91b82195300e452fec6bde5f58ed2d96ac0374d7 Mon Sep 17 00:00:00 2001 From: Brian Picciano Date: Fri, 4 Sep 2026 12:10:54 +0000 Subject: [PATCH 160/294] fix(trie): process dirty ranges around cached branches (#26842) --- crates/trie/common/src/prefix_set.rs | 35 +++++ crates/trie/sparse/src/arena/mod.rs | 21 ++- crates/trie/trie/src/proof_v2/mod.rs | 197 +++++++++++++++++++++------ 3 files changed, 211 insertions(+), 42 deletions(-) diff --git a/crates/trie/common/src/prefix_set.rs b/crates/trie/common/src/prefix_set.rs index f662be1fda4..e3de9ba77bf 100644 --- a/crates/trie/common/src/prefix_set.rs +++ b/crates/trie/common/src/prefix_set.rs @@ -334,6 +334,28 @@ impl PrefixSet { false } + /// Returns `true` if any key in the set is at or after `start`. + #[inline] + pub fn contains_from(&mut self, start: &Nibbles) -> bool { + if self.all { + return true + } + + let keys = self.keys.as_deref().map(Vec::as_slice).unwrap_or_default(); + while self.index > 0 && &keys[self.index] > start { + self.index -= 1; + } + + for (idx, key) in keys[self.index..].iter().enumerate() { + if key >= start { + self.index += idx; + return true + } + } + + false + } + /// Returns an iterator over reference to _all_ nibbles regardless of cursor position. pub fn iter(&self) -> core::slice::Iter<'_, Nibbles> { self.slice().iter() @@ -388,6 +410,19 @@ mod tests { assert_eq!(prefix_set.len(), 3); // Length should be 3 (excluding duplicate) } + #[test] + fn test_contains_from() { + let first = Nibbles::from_nibbles([1, 2, 3]); + let middle = Nibbles::from_nibbles([1, 2, 4]); + let last = Nibbles::from_nibbles([4, 5, 6]); + let mut prefix_set = PrefixSetMut::from([first, middle, last]).freeze(); + + assert!(prefix_set.contains_range(&first..&middle)); + assert!(prefix_set.contains_from(&middle)); + assert!(prefix_set.contains_from(&last)); + assert!(!prefix_set.contains_from(&Nibbles::from_nibbles([5]))); + } + #[test] fn test_from_sorted_b256_iterator() { let first = B256::with_last_byte(1); diff --git a/crates/trie/sparse/src/arena/mod.rs b/crates/trie/sparse/src/arena/mod.rs index d2e96357641..c0c4091a591 100644 --- a/crates/trie/sparse/src/arena/mod.rs +++ b/crates/trie/sparse/src/arena/mod.rs @@ -3001,14 +3001,27 @@ mod tests { .removed_nodes .retain(|path| self.storage_trie_updates().storage_nodes.contains_key(path)); + let mut expected_updated_nodes = + expected_trie_updates.storage_nodes.into_iter().collect::>(); + let mut actual_updated_nodes = + actual_updates.updated_nodes.into_iter().collect::>(); + expected_updated_nodes.sort(); + actual_updated_nodes.sort(); pretty_assertions::assert_eq!( - expected_trie_updates.storage_nodes.into_iter().collect::>().sort(), - actual_updates.updated_nodes.into_iter().collect::>().sort(), + expected_updated_nodes, + actual_updated_nodes, "updated nodes mismatch" ); + + let mut expected_removed_nodes = + expected_trie_updates.removed_nodes.into_iter().collect::>(); + let mut actual_removed_nodes = + actual_updates.removed_nodes.into_iter().collect::>(); + expected_removed_nodes.sort(); + actual_removed_nodes.sort(); pretty_assertions::assert_eq!( - expected_trie_updates.removed_nodes.into_iter().collect::>().sort(), - actual_updates.removed_nodes.into_iter().collect::>().sort(), + expected_removed_nodes, + actual_removed_nodes, "removed nodes mismatch" ); assert_eq!(expected_root, actual_root, "storage root mismatch"); diff --git a/crates/trie/trie/src/proof_v2/mod.rs b/crates/trie/trie/src/proof_v2/mod.rs index 38390efaedc..c713596fe45 100644 --- a/crates/trie/trie/src/proof_v2/mod.rs +++ b/crates/trie/trie/src/proof_v2/mod.rs @@ -974,19 +974,71 @@ where Ok(PopCachedBranchOutcome::Popped(cached)) } - // Pop any under-construction branches that are now complete. Assumes that all trie data prior - // to `next_path`, if any, has been computed. Any branches which were under-construction - // previously, and which do not share a prefix with `next_path`, can be assumed to be completed; - // they will not have any further keys added to them. + /// Pop any under-construction branches that are now complete. Assumes that all trie data prior + /// to `next_path`, if any, has been computed. Any branches which were under-construction + /// previously, and which do not share a prefix with `next_path`, can be assumed to be + /// completed; they will not have any further keys added to them. + /// + /// Returns a range to calculate if a branch still has dirty keys to process, or popping it + /// exposes dirty keys which could split its extension. A missing lower bound disables these + /// checks when the caller has already scheduled the remaining range. fn commit_branches<'a>( &mut self, targets: &mut Option>, next_path: &Nibbles, - ) -> Result<(), StateProofError> { + uncalculated_lower_bound: Option<&Nibbles>, + ) -> Result)>, StateProofError> { + let dirty_range = |prefix_set: &mut PrefixSet, upper_bound: Option| { + let uncalculated_lower_bound = uncalculated_lower_bound?; + + if upper_bound.as_ref().is_some_and(|upper| uncalculated_lower_bound >= upper) { + return None + } + + match upper_bound { + Some(upper_bound) => prefix_set + .contains_range(uncalculated_lower_bound..&upper_bound) + .then_some((*uncalculated_lower_bound, Some(upper_bound))), + None => prefix_set + .contains_from(uncalculated_lower_bound) + .then_some((*uncalculated_lower_bound, None)), + } + }; + + let mut popped_child_path_upper = None; while !next_path.starts_with(&self.branch_path) { + // If the lower bound is still within this branch, process any remaining dirty keys + // before popping it so they can be added directly to the branch. + if uncalculated_lower_bound.is_some_and(|lower| lower.starts_with(&self.branch_path)) && + let Some(range) = + dirty_range(&mut self.prefix_set, self.branch_path.next_without_prefix()) + { + return Ok(Some(range)) + } + + let branch = self.branch_stack.last().expect("branch_stack cannot be empty"); + // Once popped, this branch becomes a child at this path. Its upper bound therefore + // covers any keys which could split the branch's extension on the right. + popped_child_path_upper = Some( + self.branch_path + .slice_unchecked(0, self.branch_path.len() - branch.ext_len as usize) + .next_without_prefix(), + ); + self.pop_branch(targets)?; } - Ok(()) + + // An empty branch_stack is skipped because a popped local root does not need this check: + // any gap before `next_path` was already returned by `try_pop_cached_branch`, and forward + // traversal will split its extension and process later dirty keys as needed. + if !self.branch_stack.is_empty() && + let Some(upper_bound) = popped_child_path_upper && + let Some(range) = dirty_range(&mut self.prefix_set, upper_bound) + { + return Ok(Some(range)) + } + + Ok(None) } /// Accepts the current state of both hashed and trie cursors, and determines the next range of @@ -1040,13 +1092,13 @@ where // used to return that range. trace!(target: TRACE_TARGET, ?uncalculated_lower_bound, "Exhausted cached trie nodes"); if let Some(lower) = uncalculated_lower_bound { - self.commit_branches(targets, &lower)?; + self.commit_branches(targets, &lower, None)?; return Ok(Some((lower, traversal_upper_bound.copied()))); } return Ok(None) } PopCachedBranchOutcome::CalculateLeaves(range) => { - self.commit_branches(targets, &range.0)?; + self.commit_branches(targets, &range.0, None)?; return Ok(Some(range)); } }; @@ -1065,7 +1117,12 @@ where "loop", ); - self.commit_branches(targets, &cached_path)?; + if let Some(range) = + self.commit_branches(targets, &cached_path, Some(uncalculated_lower_bound_ref))? + { + self.cached_branch_stack.push((cached_path, cached_branch)); + return Ok(Some(range)) + } // Since we've popped all branches which don't start with cached_path, branch_path at // this point must be equal to or shorter than cached_path. @@ -1079,6 +1136,16 @@ where // top branch is the parent of this cached branch. Either way we push a branch // corresponding to the cached one onto the stack, so we can begin constructing it. if self.branch_path != cached_path { + // If the prefix set contains any entries from the lower bound up until the new + // cached path it means that there might be a new node(s) which split the extension + // node between cached_path and its parent (self.branch_path). + if uncalculated_lower_bound_ref < &cached_path && + self.prefix_set.contains_range(uncalculated_lower_bound_ref..&cached_path) + { + self.cached_branch_stack.push((cached_path, cached_branch)); + return Ok(Some((*uncalculated_lower_bound_ref, Some(cached_path)))) + } + self.push_cached_branch(targets, cached_path, &cached_branch)?; } @@ -1149,24 +1216,22 @@ where ); } - // If there are no further children to construct for this branch then pop it off both - // stacks and loop using the parent branch. + // Defer popping this completed branch to `commit_branches`, which checks for dirty + // keys around it before looping with the parent branch. if next_child_nibbles.is_empty() { trace!( target: TRACE_TARGET, path=?cached_path, ?curr_branch, ?cached_branch, - "No further children, popping branch", + "No further children", ); - self.pop_branch(targets)?; // no need to pop from `cached_branch_stack`, the current cached branch is already // popped (see note at the top of the loop). - // The just-popped branch is completely processed; we know there can be no more keys - // with that prefix. Set the lower bound which can be returned from this method to - // be the next possible prefix, if any. + // The completed branch has no more keys with its prefix. Set the lower bound which + // can be returned from this method to be the next possible prefix, if any. uncalculated_lower_bound = cached_path.next_without_prefix(); continue @@ -1177,18 +1242,6 @@ where let child_nibble = next_child_nibbles.trailing_zeros() as u8; let child_path = self.child_path_at(child_nibble); - // If the previous child was a cached branch with a short key (extension), then the new - // uncalculated_lower_bound will be the increment of that branch's path. If there are - // any dirty leaves between that path and this child, it indicates there may be leaves - // which would split that extension node. In that case we return the range to process - // the leaves. - if uncalculated_lower_bound_ref < &child_path && - self.prefix_set.contains_range(uncalculated_lower_bound_ref..&child_path) - { - self.cached_branch_stack.push((cached_path, cached_branch)); - return Ok(Some((*uncalculated_lower_bound_ref, Some(child_path)))); - } - // If the `hash_mask` bit is set for the next child it means the child's hash is cached // in the `cached_branch`. We can use that instead of re-calculating the hash of the // entire sub-trie. @@ -1264,16 +1317,6 @@ where // Push the current cached branch back on before pushing its child and then looping self.cached_branch_stack.push((cached_path, cached_branch)); - // If the next cached branch's path is in the prefix set, it could indicate that - // there are dirty leaves which would split the cached branch's extension node (if - // there is one). In that case we return the range those leaves would potentially be - // in to calculate them. - if self.prefix_set.contains(&child_path) { - let gap_upper = Some(*next_cached_path); - self.cached_branch_stack.push(trie_cursor_state.take()); - return Ok(Some((*uncalculated_lower_bound_ref, gap_upper))); - } - trace!( target: TRACE_TARGET, ?child_path, @@ -2090,6 +2133,20 @@ mod tests { Self { inner: TrieTestHarness::new(storage) } } + /// Computes the storage root while treating the supplied prefixes as dirty. + fn root_with_prefix_set(&self, prefix_set: PrefixSet) -> Option { + let trie_cursor = + self.trie_cursor_factory().storage_trie_cursor(self.hashed_address()).unwrap(); + let hashed_cursor = + self.hashed_cursor_factory().hashed_storage_cursor(self.hashed_address()).unwrap(); + let mut calculator = StorageProofCalculator::new_storage(trie_cursor, hashed_cursor) + .with_prefix_set(prefix_set); + + let mut targets = [ProofV2Target::new(B256::ZERO)]; + let proof = calculator.storage_proof(self.hashed_address(), &mut targets).unwrap(); + calculator.compute_root_hash(&proof).unwrap() + } + /// Asserts that `StorageProofCalculator` and legacy `StorageProof` produce equivalent /// results for storage proofs. fn assert_proof( @@ -2792,6 +2849,70 @@ mod tests { ); } + fn b256(s: &str) -> B256 { + B256::from_slice(&alloy_primitives::hex::decode(s).expect("valid hex string")) + } + + #[test] + fn test_prefix_set_root_proof_processes_sibling_after_cached_descendant() { + reth_tracing::init_test_tracing(); + + let storage = [ + ("1022c69e9d900e40775cd387c134899f465f291dbc3c97899ff6bfb8dc972b37", 45u64), + ("1111ad8083c8a3a398b2b781217b989ff4d1ed182f46cc765eda49a7b316139d", 60), + ("12012d20943649899b2fc0f87b9840b70ef68e93613aac17c269bf8c5a78a712", 17), + ("12014b57b9a162c03d072eb6acd4e936f1c4bc23b803a054347c5ee9a9bcfb9a", 49), + ("1203f800840af3f898ab4572f2750106a7c4bd2b3e844b6e7fa72704673cc2c6", 76), + ("12208f18fbcd6971c92808721392acbf11d5af58e9143a374cc86e70bdd1f097", 10), + ] + .into_iter() + .map(|(key, value)| (b256(key), U256::from(value))) + .collect(); + + let dirty = b256("12208f18fbcd6971c92808721392acbf11d5af58e9143a374cc86e70bdd1f097"); + let harness = ProofTestHarness::new(storage); + let expected_root = harness.original_root(); + + let mut prefix_set = PrefixSetMut::default(); + prefix_set.insert(Nibbles::unpack(dirty)); + + pretty_assertions::assert_eq!( + Some(expected_root), + harness.root_with_prefix_set(prefix_set.freeze()), + "root proof must process a prefix-set sibling after a cached descendant", + ); + } + + #[test] + fn test_prefix_set_root_proof_processes_trailing_dirty_sibling() { + reth_tracing::init_test_tracing(); + + let keys = [ + "0022001020000000000000000000000000000000000000000000000000000000", + "0110212112000000000000000000000000000000000000000000000000000000", + "0202210210000000000000000000000000000000000000000000000000000000", + "0211020211000000000000000000000000000000000000000000000000000000", + "0211211002000000000000000000000000000000000000000000000000000000", + "0212221010000000000000000000000000000000000000000000000000000000", + "0222011102000000000000000000000000000000000000000000000000000000", + ]; + let storage = + keys.iter().enumerate().map(|(i, key)| (b256(key), U256::from(i as u64 + 1))).collect(); + let harness = ProofTestHarness::new(storage); + let expected_root = harness.original_root(); + + // The dirty children straddle a clean cached descendant under branch 0x02. Traversal + // must resume at the trailing dirty sibling after using the cached descendant. + let mut prefix_set = PrefixSetMut::default(); + prefix_set.insert(Nibbles::unpack(b256(keys[2]))); + prefix_set.insert(Nibbles::unpack(b256(keys[6]))); + + pretty_assertions::assert_eq!( + Some(expected_root), + harness.root_with_prefix_set(prefix_set.freeze()), + ); + } + /// Helper to compute the keccak256 hash of a storage leaf node. The `short_key` is the /// leaf's key after trimming all branch/extension nibbles consumed by ancestor nodes. fn storage_leaf_hash(short_key: &Nibbles, value: &U256) -> B256 { From 1f94488d1e42866d3f8280702a546d42a7b288af Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Fri, 4 Sep 2026 12:55:06 +0000 Subject: [PATCH 161/294] fix(rpc): hold a blocking IO permit for eth_getLogs range scans (#26999) --- .../rpc-eth-api/src/helpers/blocking_task.rs | 6 +-- crates/rpc/rpc/src/eth/filter.rs | 40 +++++++++++++++++++ 2 files changed, 43 insertions(+), 3 deletions(-) diff --git a/crates/rpc/rpc-eth-api/src/helpers/blocking_task.rs b/crates/rpc/rpc-eth-api/src/helpers/blocking_task.rs index 5336d016455..f6918948cf0 100644 --- a/crates/rpc/rpc-eth-api/src/helpers/blocking_task.rs +++ b/crates/rpc/rpc-eth-api/src/helpers/blocking_task.rs @@ -40,7 +40,7 @@ pub trait SpawnBlocking: EthApiTypes + Clone + Send + Sync + 'static { /// Returns handle to semaphore for blocking IO tasks. /// /// This semaphore is used to limit concurrent blocking IO operations like `eth_call`, - /// `eth_estimateGas`, and similar methods that require EVM execution. + /// `eth_estimateGas` and the `eth_getLogs` range scans. fn blocking_io_task_guard(&self) -> &Arc; /// Acquires a permit from the tracing task semaphore. @@ -80,8 +80,8 @@ pub trait SpawnBlocking: EthApiTypes + Clone + Send + Sync + 'static { /// Acquires a permit from the blocking IO request semaphore. /// - /// This should be used for operations like `eth_call`, `eth_estimateGas`, and similar methods - /// that require EVM execution and are spawned as blocking tasks. + /// This should be used for operations like `eth_call`, `eth_estimateGas` and the `eth_getLogs` + /// range scans, which are spawned as blocking tasks. /// /// See also [`Semaphore::acquire_owned`](`tokio::sync::Semaphore::acquire_owned`). fn acquire_owned_blocking_io( diff --git a/crates/rpc/rpc/src/eth/filter.rs b/crates/rpc/rpc/src/eth/filter.rs index 5a9b0e7492c..c7433fafc15 100644 --- a/crates/rpc/rpc/src/eth/filter.rs +++ b/crates/rpc/rpc/src/eth/filter.rs @@ -655,9 +655,19 @@ where return Err(EthFilterError::QueryExceedsMaxBlocks(max_blocks_per_filter)) } + // The scan occupies a blocking thread until it completes, so it shares the budget for + // blocking IO requests with `eth_call` and friends instead of pinning an unbounded number + // of pool threads. + let permit = self + .eth_api + .acquire_owned_blocking_io() + .await + .map_err(|_| EthFilterError::InternalError)?; + let (mut tx, rx) = oneshot::channel(); let this = self.clone(); self.task_spawner.spawn_blocking_task(async move { + let _permit = permit; let fut = this.get_logs_in_block_range_inner(&filter, from_block, to_block, limits); tokio::pin!(fut); let res = tokio::select! { @@ -2040,4 +2050,34 @@ mod tests { assert_eq!(logs[0].block_hash, Some(expected_hashes[0])); // block 100 assert_eq!(logs[1].block_hash, Some(expected_hashes[2])); // block 102 } + + #[tokio::test] + async fn test_range_scan_waits_for_blocking_io_permit() { + use reth_rpc_eth_api::helpers::SpawnBlocking; + + let provider = MockEthProvider::default(); + provider.add_header(FixedBytes::random(), alloy_consensus::Header::default()); + let eth_api = build_test_eth_api(provider); + + // take every permit so the scan has to wait for one + let guard = eth_api.blocking_io_task_guard().clone(); + let permits = + guard.clone().acquire_many_owned(guard.available_permits() as u32).await.unwrap(); + + let eth_filter = EthFilter::new(eth_api, EthFilterConfig::default(), Runtime::test()); + let scan = eth_filter.inner.clone().get_logs_in_block_range( + Filter::default(), + 0, + 0, + QueryLimits::default(), + ); + tokio::pin!(scan); + assert!( + tokio::time::timeout(Duration::from_millis(100), &mut scan).await.is_err(), + "scan must wait for a blocking IO permit" + ); + + drop(permits); + assert!(scan.await.unwrap().is_empty()); + } } From 055fafc723d1df096e551a14a80e76d2483e97fa Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Fri, 4 Sep 2026 14:34:50 +0000 Subject: [PATCH 162/294] perf(rpc): scan eth_getLogs ranges window by window (#27000) --- crates/rpc/rpc/src/eth/filter.rs | 351 ++++++++++++++++++++++--------- 1 file changed, 248 insertions(+), 103 deletions(-) diff --git a/crates/rpc/rpc/src/eth/filter.rs b/crates/rpc/rpc/src/eth/filter.rs index c7433fafc15..11663d204f2 100644 --- a/crates/rpc/rpc/src/eth/filter.rs +++ b/crates/rpc/rpc/src/eth/filter.rs @@ -81,12 +81,20 @@ const BLOOM_ADJUSTMENT_MIN_BLOCKS: u64 = 100; /// The maximum number of headers we read at once when handling a range filter. const MAX_HEADERS_RANGE: u64 = 1_000; // with ~530bytes per header this is ~500kb -/// Threshold for enabling parallel processing in range mode +// Cached mode is only reachable for ranges that fit into a single header window, which is what +// keeps the mode decision independent of how a range is split. +const _: () = assert!(CACHED_MODE_BLOCK_THRESHOLD <= MAX_HEADERS_RANGE); + +/// Minimum number of bloom matching blocks in a header window for fetching their receipts in +/// parallel const PARALLEL_PROCESSING_THRESHOLD: usize = 1000; /// Default concurrency for parallel processing const DEFAULT_PARALLEL_CONCURRENCY: usize = 4; +/// Maximum number of blocks whose receipts the parallel fetching holds in memory at once +const MAX_PARALLEL_BATCH_SIZE: usize = 256; + /// `Eth` filter RPC implementation. /// /// This type handles `eth_` rpc requests related to filters (`eth_getLogs`). @@ -700,15 +708,21 @@ where limits: QueryLimits, ) -> Result>, EthFilterError> { let mut all_logs = Vec::new(); - let mut matching_headers = Vec::new(); // get current chain tip to determine processing mode let chain_tip = self.provider().best_block_number()?; - // first collect all headers that match the bloom filter for cached mode decision + // Scan the range window by window so that receipts are fetched while headers are still + // being read: the log limit can end the query after the first window, memory is bounded by + // one window, and a cancelled query stops at the next window. for (from, to) in BlockRangeInclusiveIter::new(from_block..=to_block, self.max_headers_range) { + // reading headers is blocking, this gives the cancellation check a chance to run + tokio::task::yield_now().await; + + // collect the headers of this window that match the bloom filter + let mut matching_headers = Vec::new(); let headers = self.provider().headers_range(from..=to)?; let mut headers_iter = headers.into_iter().peekable(); @@ -733,58 +747,61 @@ where matching_headers.push(SealedHeader::new(header, block_hash)); } - } - // initialize the appropriate range mode based on collected headers - let mut range_mode = RangeMode::new( - self.clone(), - matching_headers, - from_block, - to_block, - self.max_headers_range, - chain_tip, - ); + // initialize the appropriate range mode based on collected headers + let mut range_mode = RangeMode::new( + self.clone(), + matching_headers, + from_block, + to_block, + self.max_headers_range, + chain_tip, + ); - // iterate through the range mode to get receipts and blocks - while let Some(ReceiptBlockResult { receipts, recovered_block, header }) = - range_mode.next().await? - { - let num_hash = header.num_hash(); - append_matching_block_logs( - &mut all_logs, - self.eth_api.converter(), - recovered_block - .map(ProviderOrBlock::Block) - .unwrap_or_else(|| ProviderOrBlock::Provider(self.provider())), - filter, - &header, - &receipts, - false, - )?; - - // size check but only if range is multiple blocks, so we always return all - // logs of a single block - let is_multi_block_range = from_block != to_block; - if let Some(max_logs_per_response) = limits.max_logs_per_response && - is_multi_block_range && - all_logs.len() > max_logs_per_response + // iterate through the range mode to get receipts and blocks + while let Some(ReceiptBlockResult { receipts, recovered_block, header }) = + range_mode.next().await? { - let retry_to_block = - if num_hash.number == from_block { from_block } else { num_hash.number - 1 }; - - debug!( - target: "rpc::eth::filter", - logs_found = all_logs.len(), - max_logs_per_response, - from_block, - to_block = retry_to_block, - "Query exceeded max logs per response limit" - ); - return Err(EthFilterError::QueryExceedsMaxResults { - max_logs: max_logs_per_response, - from_block, - to_block: retry_to_block, - }); + let num_hash = header.num_hash(); + append_matching_block_logs( + &mut all_logs, + self.eth_api.converter(), + recovered_block + .map(ProviderOrBlock::Block) + .unwrap_or_else(|| ProviderOrBlock::Provider(self.provider())), + filter, + &header, + &receipts, + false, + )?; + + // size check but only if range is multiple blocks, so we always return all + // logs of a single block + let is_multi_block_range = from_block != to_block; + if let Some(max_logs_per_response) = limits.max_logs_per_response && + is_multi_block_range && + all_logs.len() > max_logs_per_response + { + let retry_to_block = if num_hash.number == from_block { + from_block + } else { + num_hash.number - 1 + }; + + debug!( + target: "rpc::eth::filter", + logs_found = all_logs.len(), + max_logs_per_response, + from_block, + to_block = retry_to_block, + "Query exceeded max logs per response limit" + ); + return Err(EthFilterError::QueryExceedsMaxResults { + max_logs: max_logs_per_response, + from_block, + to_block: retry_to_block, + }); + } } } @@ -1098,6 +1115,10 @@ impl< let use_cached_mode = Self::should_use_cached_mode(&sealed_headers, block_count, distance_from_tip); + // Fetching receipts in parallel is only worth the extra tasks when most of the window has + // them to read; the sequential path serves the rest from the receipt cache where it can + let parallel = sealed_headers.len() >= PARALLEL_PROCESSING_THRESHOLD; + if use_cached_mode && !sealed_headers.is_empty() { Self::Cached(CachedMode { filter_inner, headers_iter: sealed_headers.into_iter() }) } else { @@ -1105,7 +1126,8 @@ impl< filter_inner, iter: sealed_headers.into_iter().peekable(), next: VecDeque::new(), - max_range: max_headers_range as usize, + max_range: (max_headers_range as usize).min(MAX_PARALLEL_BATCH_SIZE), + parallel, pending_tasks: FuturesOrdered::new(), }) } @@ -1202,7 +1224,10 @@ struct RangeBlockMode< filter_inner: Arc>, iter: Peekable::Header>>>, next: VecDeque>, + /// Maximum number of consecutive blocks fetched by one batch of parallel tasks max_range: usize, + /// Whether receipts are fetched in parallel batches of consecutive blocks + parallel: bool, // Stream of ongoing receipt fetching tasks pending_tasks: FuturesOrdered>, } @@ -1234,6 +1259,15 @@ impl< return Ok(None); }; + if !self.parallel { + // Process the block on its own so that only one block's receipts are held + if let Some(result) = self.process_small_range(vec![next_header]).await? { + return Ok(Some(result)); + } + // Continue loop to check for more work + continue; + } + let mut range_headers = Vec::with_capacity(self.max_range.min(self.iter.len() + 1)); range_headers.push(next_header); @@ -1259,24 +1293,14 @@ impl< range_headers.push(next_header); } - // Check if we should use parallel processing for large ranges - let remaining_headers = self.iter.len() + range_headers.len(); - if remaining_headers >= PARALLEL_PROCESSING_THRESHOLD { - self.spawn_parallel_tasks(range_headers); - // Continue loop to await the spawned tasks - } else { - // Process small range sequentially and add results to buffer - if let Some(result) = self.process_small_range(range_headers).await? { - return Ok(Some(result)); - } - // Continue loop to check for more work - } + self.spawn_parallel_tasks(range_headers); + // Continue loop to await the spawned tasks } } /// Process a small range of headers sequentially /// - /// This is used when the remaining headers count is below [`PARALLEL_PROCESSING_THRESHOLD`]. + /// This is used for ranges below [`PARALLEL_PROCESSING_THRESHOLD`], one block at a time. async fn process_small_range( &mut self, range_headers: Vec::Header>>, @@ -1315,14 +1339,13 @@ impl< /// Spawn parallel tasks for processing a large range of headers /// - /// This is used when the remaining headers count is at or above - /// [`PARALLEL_PROCESSING_THRESHOLD`]. + /// This is used for ranges of at least [`PARALLEL_PROCESSING_THRESHOLD`] blocks. fn spawn_parallel_tasks( &mut self, range_headers: Vec::Header>>, ) { // Split headers into chunks - let chunk_size = std::cmp::max(range_headers.len() / DEFAULT_PARALLEL_CONCURRENCY, 1); + let chunk_size = range_headers.len().div_ceil(DEFAULT_PARALLEL_CONCURRENCY).max(1); let header_chunks = range_headers .into_iter() .chunks(chunk_size) @@ -1333,47 +1356,61 @@ impl< // Spawn each chunk as a separate task directly into the FuturesOrdered stream for chunk_headers in header_chunks { let filter_inner = self.filter_inner.clone(); - let chunk_task = Box::pin(async move { - let chunk_task = tokio::task::spawn_blocking(move || { - let mut chunk_results = Vec::with_capacity(chunk_headers.len()); - - for header in chunk_headers { - // Fetch directly from provider - RangeMode is used for older blocks - // unlikely to be cached - let receipts = match filter_inner - .provider() - .receipts_by_block(header.hash().into())? - { - Some(receipts) => Arc::new(receipts), - None => continue, // No receipts found - }; - - if !receipts.is_empty() { - chunk_results.push(ReceiptBlockResult { - receipts, - recovered_block: None, - header, - }); - } - } + let fetch = move || Self::fetch_chunk_receipts(&filter_inner, chunk_headers); - Ok(chunk_results) - }); + // A parallel task occupies an additional blocking thread, so it needs its own share + // of the blocking IO budget. Without one the chunk is fetched on this task instead. + let chunk_task: ReceiptFetchFuture = match self + .filter_inner + .eth_api + .blocking_io_task_guard() + .clone() + .try_acquire_owned() + { + Ok(permit) => Box::pin(async move { + let chunk_task = tokio::task::spawn_blocking(move || { + let _permit = permit; + fetch() + }); - // Await the blocking task and handle the result - match chunk_task.await { - Ok(Ok(chunk_results)) => Ok(chunk_results), - Ok(Err(e)) => Err(e), - Err(join_err) => { - trace!(target: "rpc::eth::filter", error = ?join_err, "Task join error"); - Err(EthFilterError::InternalError) + // Await the blocking task and handle the result + match chunk_task.await { + Ok(chunk_results) => chunk_results, + Err(join_err) => { + trace!(target: "rpc::eth::filter", error = ?join_err, "Task join error"); + Err(EthFilterError::InternalError) + } } - } - }); + }), + Err(_) => Box::pin(async move { fetch() }), + }; self.pending_tasks.push_back(chunk_task); } } + + /// Fetches the receipts of the given blocks from the provider. + fn fetch_chunk_receipts( + filter_inner: &EthFilterInner, + chunk_headers: Vec::Header>>, + ) -> Result>, EthFilterError> { + let mut chunk_results = Vec::with_capacity(chunk_headers.len()); + + for header in chunk_headers { + // Fetch directly from provider - RangeMode is used for older blocks + // unlikely to be cached + let receipts = match filter_inner.provider().receipts_by_block(header.hash().into())? { + Some(receipts) => Arc::new(receipts), + None => continue, // No receipts found + }; + + if !receipts.is_empty() { + chunk_results.push(ReceiptBlockResult { receipts, recovered_block: None, header }); + } + } + + Ok(chunk_results) + } } #[cfg(test)] @@ -1466,6 +1503,8 @@ mod tests { iter: headers.into_iter().peekable(), next: VecDeque::new(), max_range, + parallel: false, + pending_tasks: FuturesOrdered::new(), }; @@ -1541,6 +1580,8 @@ mod tests { iter: headers.into_iter().peekable(), next: VecDeque::from([mock_result_1, mock_result_2]), // Queue two results max_range: 100, + parallel: false, + pending_tasks: FuturesOrdered::new(), }; @@ -1608,6 +1649,8 @@ mod tests { iter: headers.into_iter().peekable(), next: VecDeque::new(), max_range: 100, + parallel: false, + pending_tasks: FuturesOrdered::new(), }; @@ -1676,6 +1719,8 @@ mod tests { iter: headers.into_iter().peekable(), next: VecDeque::new(), max_range: 3, // include the 3 blocks in the first queried results + parallel: false, + pending_tasks: FuturesOrdered::new(), }; @@ -1766,6 +1811,8 @@ mod tests { iter: headers.into_iter().peekable(), next: VecDeque::new(), max_range: 1, + parallel: false, + pending_tasks: FuturesOrdered::new(), }; @@ -2080,4 +2127,102 @@ mod tests { drop(permits); assert!(scan.await.unwrap().is_empty()); } + + #[tokio::test] + async fn test_range_scan_across_windows() { + use alloy_consensus::TxLegacy; + use alloy_primitives::{Address, Bloom, Bytes, Log, LogData, Signature}; + use reth_db_api::models::StoredBlockBodyIndices; + use reth_ethereum_primitives::{Block, BlockBody, Receipt, TransactionSigned}; + + let provider = MockEthProvider::default(); + let tx = TransactionSigned::new_unhashed( + TxLegacy { + chain_id: Some(1), + gas_price: 21_000, + gas_limit: 21_000, + ..Default::default() + } + .into(), + Signature::test_signature(), + ); + let receipt = Receipt { + tx_type: TxType::Legacy, + cumulative_gas_used: 21_000, + logs: vec![Log { + address: Address::ZERO, + data: LogData::new_unchecked(vec![], Bytes::new()), + }], + success: true, + }; + + // an empty filter matches every header, so every window is fetched in parallel while + // only three blocks have logs to return + let matching_blocks = [5u64, 1_200, 2_400]; + let mut parent_hash = FixedBytes::default(); + for number in 0..=2_500u64 { + let matches = matching_blocks.contains(&number); + let header = alloy_consensus::Header { + number, + parent_hash, + logs_bloom: if matches { Bloom::from([1u8; 256]) } else { Bloom::default() }, + ..Default::default() + }; + parent_hash = header.hash_slow(); + let transactions = if matches { vec![tx.clone()] } else { Vec::new() }; + provider.add_block( + parent_hash, + Block { header, body: BlockBody { transactions, ..Default::default() } }, + ); + if matches { + let tx_num = matching_blocks.iter().position(|b| *b == number).unwrap() as u64; + provider.add_receipts(number, vec![receipt.clone()]); + provider.add_block_body_indices( + number, + StoredBlockBodyIndices { first_tx_num: tx_num, tx_count: 1 }, + ); + } else { + provider.add_receipts(number, vec![]); + } + } + + let eth_filter = EthFilter::new( + build_test_eth_api(provider), + EthFilterConfig::default(), + Runtime::test(), + ); + let logs = eth_filter + .inner + .clone() + .get_logs_in_block_range(Filter::default(), 0, 2_500, QueryLimits::default()) + .await + .unwrap(); + let blocks = logs.iter().map(|log| log.block_number.unwrap()).collect::>(); + assert_eq!(blocks, matching_blocks); + + // the log limit ends the scan in the window that exceeds it and names the last block that + // fit as the range to retry with + let err = eth_filter + .inner + .clone() + .get_logs_in_block_range( + Filter::default(), + 0, + 2_500, + QueryLimits { max_blocks_per_filter: None, max_logs_per_response: Some(1) }, + ) + .await + .unwrap_err(); + assert!( + matches!( + err, + EthFilterError::QueryExceedsMaxResults { + max_logs: 1, + from_block: 0, + to_block: 1_199 + } + ), + "{err:?}" + ); + } } From 8fce12d1ee3ffabac19cdc702c0df486e6754607 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Fri, 4 Sep 2026 14:43:58 +0000 Subject: [PATCH 163/294] perf(prune): only read history shards that pruning can affect (#26900) Co-authored-by: ControlCplusControlV <44706811+ControlCplusControlV@users.noreply.github.com> --- .../storage/db-api/src/models/integer_list.rs | 30 +- .../src/providers/rocksdb/provider.rs | 303 ++++++++++++++++-- 2 files changed, 311 insertions(+), 22 deletions(-) diff --git a/crates/storage/db-api/src/models/integer_list.rs b/crates/storage/db-api/src/models/integer_list.rs index 3a6d1738a82..06a993e75e8 100644 --- a/crates/storage/db-api/src/models/integer_list.rs +++ b/crates/storage/db-api/src/models/integer_list.rs @@ -2,7 +2,7 @@ use crate::table::{Compress, Decompress}; use bytes::BufMut; -use core::fmt; +use core::{fmt, ops::RangeBounds}; use derive_more::Deref; use reth_codecs::DecompressError; use roaring::RoaringTreemap; @@ -68,6 +68,11 @@ impl IntegerList { self.0.clear(); } + /// Removes the integers in the given range, returning how many were removed. + pub fn remove_range>(&mut self, range: R) -> u64 { + self.0.remove_range(range) + } + /// Serializes an [`IntegerList`] into a sequence of bytes. pub fn to_bytes(&self) -> Vec { let mut vec = Vec::with_capacity(self.0.serialized_size()); @@ -199,4 +204,27 @@ mod tests { let blist = ef_list.to_bytes(); assert_eq!(IntegerList::from_bytes(&blist).unwrap(), ef_list) } + + #[test] + fn remove_range_matches_filtering() { + // Spans more than one 2^16 roaring container so multi-container removal is covered. + let values = [1u64, 2, 100, 65_535, 65_536, 70_000, 200_000]; + + for to_block in [0u64, 1, 99, 100, 65_535, 65_536, 199_999, 200_000, 200_001] { + let mut list = IntegerList::new(values).unwrap(); + let removed = list.remove_range(0..=to_block); + + let expected = values.into_iter().filter(|value| *value > to_block).collect::>(); + assert_eq!(list.iter().collect::>(), expected, "to_block {to_block}"); + assert_eq!(removed, (values.len() - expected.len()) as u64, "to_block {to_block}"); + assert_eq!(list.is_empty(), expected.is_empty(), "to_block {to_block}"); + } + } + + #[test] + fn remove_range_on_empty_list_removes_nothing() { + let mut list = IntegerList::empty(); + assert_eq!(list.remove_range(0..=100), 0); + assert!(list.is_empty()); + } } diff --git a/crates/storage/provider/src/providers/rocksdb/provider.rs b/crates/storage/provider/src/providers/rocksdb/provider.rs index 9bf75bb8256..c13977e0cc5 100644 --- a/crates/storage/provider/src/providers/rocksdb/provider.rs +++ b/crates/storage/provider/src/providers/rocksdb/provider.rs @@ -2123,10 +2123,14 @@ impl<'a> RocksDBBatch<'a> { /// Generic implementation for both account and storage history pruning. /// Mirrors MDBX `prune_shard` semantics. After pruning, the last remaining shard /// (if any) will have the sentinel key (`u64::MAX`). + /// + /// `shards_complete` must be `false` when `shards` stops short of the key's last shard, so the + /// survivor is not re-keyed over a sentinel that is still on disk. #[expect(clippy::too_many_arguments)] fn prune_history_shards_inner( &mut self, shards: Vec<(K, BlockNumberList)>, + shards_complete: bool, to_block: BlockNumber, get_highest: impl Fn(&K) -> u64, is_sentinel: impl Fn(&K) -> bool, @@ -2145,21 +2149,19 @@ impl<'a> RocksDBBatch<'a> { let mut updated = false; let mut last_remaining: Option<(K, BlockNumberList)> = None; - for (key, block_list) in shards { + for (key, mut block_list) in shards { if !is_sentinel(&key) && get_highest(&key) <= to_block { delete_shard(self, key)?; deleted = true; } else { - let original_len = block_list.len(); - let filtered = - BlockNumberList::new_pre_sorted(block_list.iter().filter(|&b| b > to_block)); + let removed = block_list.remove_range(0..=to_block); - if filtered.is_empty() { + if block_list.is_empty() { delete_shard(self, key)?; deleted = true; - } else if filtered.len() < original_len { - put_shard(self, key.clone(), &filtered)?; - last_remaining = Some((key, filtered)); + } else if removed > 0 { + put_shard(self, key.clone(), &block_list)?; + last_remaining = Some((key, block_list)); updated = true; } else { last_remaining = Some((key, block_list)); @@ -2167,7 +2169,8 @@ impl<'a> RocksDBBatch<'a> { } } - if let Some((last_key, last_value)) = last_remaining && + if shards_complete && + let Some((last_key, last_value)) = last_remaining && !is_sentinel(&last_key) { delete_shard(self, last_key)?; @@ -2196,6 +2199,7 @@ impl<'a> RocksDBBatch<'a> { let shards = self.provider.account_history_shards(address)?; self.prune_history_shards_inner( shards, + true, to_block, |key| key.highest_block_number, |key| key.highest_block_number == u64::MAX, @@ -2211,8 +2215,8 @@ impl<'a> RocksDBBatch<'a> { /// because it reuses a single raw iterator and skips seeks when the iterator is already /// positioned correctly (which happens when targets are sorted and adjacent in key order). /// - /// `targets` MUST be sorted by address for correctness and optimal performance - /// (matches on-disk key order). + /// `targets` MUST be sorted by address and contain each address at most once, for + /// correctness and optimal performance (matches on-disk key order). pub fn prune_account_history_batch( &mut self, targets: &[(Address, BlockNumber)], @@ -2222,8 +2226,8 @@ impl<'a> RocksDBBatch<'a> { } debug_assert!( - targets.windows(2).all(|w| w[0].0 <= w[1].0), - "prune_account_history_batch: targets must be sorted by address" + targets.windows(2).all(|w| w[0].0 < w[1].0), + "prune_account_history_batch: targets must be sorted and unique" ); // ShardedKey
layout: [address: 20][block: 8] = 28 bytes @@ -2242,7 +2246,9 @@ impl<'a> RocksDBBatch<'a> { // Check if we need to seek or if the iterator is already positioned correctly. // After processing the previous target, the iterator is either: // 1. Positioned at a key with a different prefix (we iterated past our shards) - // 2. Invalid (no more keys) + // 2. Positioned on a later shard of the previous target (we stopped early), whose + // prefix is below ours because targets are sorted and unique + // 3. Invalid (no more keys) // If the current key's prefix >= our target prefix, we may be able to skip the seek. let needs_seek = if iter.valid() { if let Some(current_key) = iter.key() { @@ -2267,8 +2273,10 @@ impl<'a> RocksDBBatch<'a> { })?; } - // Collect all shards for this address using raw prefix comparison + // Collect the shards for this address that pruning can touch, using raw prefix + // comparison let mut shards = Vec::new(); + let mut shards_complete = true; while iter.valid() { let Some(key_bytes) = iter.key() else { break }; @@ -2286,12 +2294,36 @@ impl<'a> RocksDBBatch<'a> { let value = BlockNumberList::decompress(value_bytes) .map_err(|_| ProviderError::Database(DatabaseError::Decode))?; + let highest = key.highest_block_number; shards.push((key, value)); + iter.next(); + + // Shards are ordered by their highest block and their contents partition the + // address's history, so this is the last shard holding anything at or below the + // target. Peek past it only to tell whether it was the address's last shard, + // which decides whether a survivor may be re-keyed to the sentinel. + if highest > *to_block { + shards_complete = iter.key().and_then(|next_key| next_key.get(..PREFIX_LEN)) != + Some(target_prefix); + break; + } + } + + // The iterator also goes invalid on a read error, which would otherwise pass a + // truncated shard list off as the key's complete one. + if !iter.valid() { + iter.status().map_err(|e| { + ProviderError::Database(DatabaseError::Read(DatabaseErrorInfo { + message: e.to_string().into(), + code: -1, + })) + })?; } match self.prune_history_shards_inner( shards, + shards_complete, *to_block, |key| key.highest_block_number, |key| key.highest_block_number == u64::MAX, @@ -2322,6 +2354,7 @@ impl<'a> RocksDBBatch<'a> { let shards = self.provider.storage_history_shards(address, storage_key)?; self.prune_history_shards_inner( shards, + true, to_block, |key| key.sharded_key.highest_block_number, |key| key.sharded_key.highest_block_number == u64::MAX, @@ -2338,8 +2371,8 @@ impl<'a> RocksDBBatch<'a> { /// because it reuses a single raw iterator and skips seeks when the iterator is already /// positioned correctly (which happens when targets are sorted and adjacent in key order). /// - /// `targets` MUST be sorted by (address, `storage_key`) for correctness and optimal - /// performance (matches on-disk key order). + /// `targets` MUST be sorted by (address, `storage_key`) and contain each pair at most once, + /// for correctness and optimal performance (matches on-disk key order). pub fn prune_storage_history_batch( &mut self, targets: &[((Address, B256), BlockNumber)], @@ -2349,8 +2382,8 @@ impl<'a> RocksDBBatch<'a> { } debug_assert!( - targets.windows(2).all(|w| w[0].0 <= w[1].0), - "prune_storage_history_batch: targets must be sorted by (address, storage_key)" + targets.windows(2).all(|w| w[0].0 < w[1].0), + "prune_storage_history_batch: targets must be sorted and unique" ); // StorageShardedKey layout: [address: 20][storage_key: 32][block: 8] = 60 bytes @@ -2369,7 +2402,9 @@ impl<'a> RocksDBBatch<'a> { // Check if we need to seek or if the iterator is already positioned correctly. // After processing the previous target, the iterator is either: // 1. Positioned at a key with a different prefix (we iterated past our shards) - // 2. Invalid (no more keys) + // 2. Positioned on a later shard of the previous target (we stopped early), whose + // prefix is below ours because targets are sorted and unique + // 3. Invalid (no more keys) // If the current key's prefix >= our target prefix, we may be able to skip the seek. let needs_seek = if iter.valid() { if let Some(current_key) = iter.key() { @@ -2394,8 +2429,10 @@ impl<'a> RocksDBBatch<'a> { })?; } - // Collect all shards for this (address, storage_key) pair using prefix comparison + // Collect the shards for this (address, storage_key) pair that pruning can touch, + // using prefix comparison let mut shards = Vec::new(); + let mut shards_complete = true; while iter.valid() { let Some(key_bytes) = iter.key() else { break }; @@ -2413,13 +2450,37 @@ impl<'a> RocksDBBatch<'a> { let value = BlockNumberList::decompress(value_bytes) .map_err(|_| ProviderError::Database(DatabaseError::Decode))?; + let highest = key.sharded_key.highest_block_number; shards.push((key, value)); + iter.next(); + + // Shards are ordered by their highest block and their contents partition the + // key's history, so this is the last shard holding anything at or below the + // target. Peek past it only to tell whether it was the key's last shard, which + // decides whether a survivor may be re-keyed to the sentinel. + if highest > *to_block { + shards_complete = iter.key().and_then(|next_key| next_key.get(..PREFIX_LEN)) != + Some(target_prefix); + break; + } + } + + // The iterator also goes invalid on a read error, which would otherwise pass a + // truncated shard list off as the key's complete one. + if !iter.valid() { + iter.status().map_err(|e| { + ProviderError::Database(DatabaseError::Read(DatabaseErrorInfo { + message: e.to_string().into(), + code: -1, + })) + })?; } // Use existing prune_history_shards_inner logic match self.prune_history_shards_inner( shards, + shards_complete, *to_block, |key| key.sharded_key.highest_block_number, |key| key.sharded_key.highest_block_number == u64::MAX, @@ -4613,4 +4674,204 @@ mod tests { let shards2 = provider.storage_history_shards(addr, slot2).unwrap(); assert_eq!(shards2[0].1.iter().collect::>(), vec![15, 25]); } + + /// Shards for one address, keyed by highest block, as `(highest, blocks)`. + fn account_shard_layout(provider: &RocksDBProvider, address: Address) -> Vec<(u64, Vec)> { + provider + .account_history_shards(address) + .unwrap() + .into_iter() + .map(|(key, list)| (key.highest_block_number, list.iter().collect::>())) + .collect() + } + + /// Shards for one storage slot, keyed by highest block, as `(highest, blocks)`. + fn storage_shard_layout( + provider: &RocksDBProvider, + address: Address, + storage_key: B256, + ) -> Vec<(u64, Vec)> { + provider + .storage_history_shards(address, storage_key) + .unwrap() + .into_iter() + .map(|(key, list)| { + (key.sharded_key.highest_block_number, list.iter().collect::>()) + }) + .collect() + } + + fn seed_three_storage_shards(provider: &RocksDBProvider, address: Address, storage_key: B256) { + let mut batch = provider.batch(); + batch + .put::( + StorageShardedKey::new(address, storage_key, 100), + &BlockNumberList::new_pre_sorted([10, 50, 100]), + ) + .unwrap(); + batch + .put::( + StorageShardedKey::new(address, storage_key, 200), + &BlockNumberList::new_pre_sorted([150, 200]), + ) + .unwrap(); + batch + .put::( + StorageShardedKey::last(address, storage_key), + &BlockNumberList::new_pre_sorted([250, 300]), + ) + .unwrap(); + batch.commit().unwrap(); + } + + #[test] + fn test_prune_storage_history_batch_leaves_shards_above_target_untouched() { + let temp_dir = TempDir::new().unwrap(); + let provider = RocksDBBuilder::new(temp_dir.path()).with_default_tables().build().unwrap(); + + let addr = Address::from([0x42; 20]); + let slot = B256::from([0x01; 32]); + seed_three_storage_shards(&provider, addr, slot); + + // Only the oldest shard holds blocks at or below the target. + let mut batch = provider.batch(); + let outcomes = batch.prune_storage_history_batch(&[((addr, slot), 50)]).unwrap(); + batch.commit().unwrap(); + + assert_eq!(outcomes.updated, 1); + // The trimmed shard keeps its own key. Re-keying it to the sentinel here would overwrite + // the sentinel's blocks. + assert_eq!( + storage_shard_layout(&provider, addr, slot), + vec![(100, vec![100]), (200, vec![150, 200]), (u64::MAX, vec![250, 300])] + ); + } + + #[test] + fn test_prune_storage_history_batch_trims_sentinel_once_earlier_shards_expire() { + let temp_dir = TempDir::new().unwrap(); + let provider = RocksDBBuilder::new(temp_dir.path()).with_default_tables().build().unwrap(); + + let addr = Address::from([0x42; 20]); + let slot = B256::from([0x01; 32]); + seed_three_storage_shards(&provider, addr, slot); + + // Every non-sentinel shard expires whole and the sentinel loses its lowest block. + let mut batch = provider.batch(); + let outcomes = batch.prune_storage_history_batch(&[((addr, slot), 250)]).unwrap(); + batch.commit().unwrap(); + + assert_eq!(outcomes.deleted, 1); + assert_eq!(storage_shard_layout(&provider, addr, slot), vec![(u64::MAX, vec![300])]); + } + + #[test] + fn test_prune_account_history_batch_leaves_shards_above_target_untouched() { + let temp_dir = TempDir::new().unwrap(); + let provider = RocksDBBuilder::new(temp_dir.path()).with_default_tables().build().unwrap(); + + let addr = Address::from([0x42; 20]); + + let mut batch = provider.batch(); + batch + .put::( + ShardedKey::new(addr, 100), + &BlockNumberList::new_pre_sorted([10, 50, 100]), + ) + .unwrap(); + batch + .put::( + ShardedKey::new(addr, u64::MAX), + &BlockNumberList::new_pre_sorted([250, 300]), + ) + .unwrap(); + batch.commit().unwrap(); + + let mut batch = provider.batch(); + let outcomes = batch.prune_account_history_batch(&[(addr, 50)]).unwrap(); + batch.commit().unwrap(); + + assert_eq!(outcomes.updated, 1); + assert_eq!( + account_shard_layout(&provider, addr), + vec![(100, vec![100]), (u64::MAX, vec![250, 300])] + ); + } + + #[test] + fn test_prune_account_history_batch_seeks_after_stopping_early() { + let temp_dir = TempDir::new().unwrap(); + let provider = RocksDBBuilder::new(temp_dir.path()).with_default_tables().build().unwrap(); + + let addr1 = Address::from([0x01; 20]); + let addr2 = Address::from([0x02; 20]); + + let mut batch = provider.batch(); + batch + .put::( + ShardedKey::new(addr1, 100), + &BlockNumberList::new_pre_sorted([10, 50, 100]), + ) + .unwrap(); + batch + .put::( + ShardedKey::new(addr1, u64::MAX), + &BlockNumberList::new_pre_sorted([250, 300]), + ) + .unwrap(); + batch + .put::( + ShardedKey::new(addr2, u64::MAX), + &BlockNumberList::new_pre_sorted([5, 10, 15]), + ) + .unwrap(); + batch.commit().unwrap(); + + // The first target stops on addr1's oldest shard, leaving the iterator on addr1's + // sentinel. The second target must seek past it instead of skipping addr2. + let mut batch = provider.batch(); + let outcomes = batch.prune_account_history_batch(&[(addr1, 50), (addr2, 10)]).unwrap(); + batch.commit().unwrap(); + + assert_eq!(outcomes.updated, 2); + assert_eq!( + account_shard_layout(&provider, addr1), + vec![(100, vec![100]), (u64::MAX, vec![250, 300])] + ); + assert_eq!(account_shard_layout(&provider, addr2), vec![(u64::MAX, vec![15])]); + } + + #[test] + fn test_prune_storage_history_batch_seeks_after_stopping_early() { + let temp_dir = TempDir::new().unwrap(); + let provider = RocksDBBuilder::new(temp_dir.path()).with_default_tables().build().unwrap(); + + let addr = Address::from([0x42; 20]); + let slot1 = B256::from([0x01; 32]); + let slot2 = B256::from([0x02; 32]); + seed_three_storage_shards(&provider, addr, slot1); + + let mut batch = provider.batch(); + batch + .put::( + StorageShardedKey::last(addr, slot2), + &BlockNumberList::new_pre_sorted([20, 40]), + ) + .unwrap(); + batch.commit().unwrap(); + + // The first target stops on slot1's oldest shard, leaving the iterator on slot1's next + // shard. The second target must seek past it instead of skipping slot2. + let mut batch = provider.batch(); + let outcomes = + batch.prune_storage_history_batch(&[((addr, slot1), 50), ((addr, slot2), 30)]).unwrap(); + batch.commit().unwrap(); + + assert_eq!(outcomes.updated, 2); + assert_eq!( + storage_shard_layout(&provider, addr, slot1), + vec![(100, vec![100]), (200, vec![150, 200]), (u64::MAX, vec![250, 300])] + ); + assert_eq!(storage_shard_layout(&provider, addr, slot2), vec![(u64::MAX, vec![40])]); + } } From 67d41c900bf37d3f5c146ecfa31196ca6844f2ff Mon Sep 17 00:00:00 2001 From: Brian Picciano Date: Fri, 4 Sep 2026 15:16:33 +0000 Subject: [PATCH 164/294] perf(trie): avoid changesets for read-only overlays (#26986) --- crates/storage/storage-overlay/src/builder.rs | 111 ++++++++--- .../storage-overlay/src/changeset_cache.rs | 4 +- .../storage/storage-overlay/src/provider.rs | 182 ++++++++++++------ 3 files changed, 207 insertions(+), 90 deletions(-) diff --git a/crates/storage/storage-overlay/src/builder.rs b/crates/storage/storage-overlay/src/builder.rs index bf2a27f15e3..bf6e7376468 100644 --- a/crates/storage/storage-overlay/src/builder.rs +++ b/crates/storage/storage-overlay/src/builder.rs @@ -16,7 +16,7 @@ use reth_storage_api::{ BlockNumReader, ChangeSetReader, DBProvider, PruneCheckpointReader, StageCheckpointReader, StorageChangeSetReader, StorageSettingsCache, }; -use reth_trie::{updates::TrieUpdatesSorted, HashedPostStateSorted}; +use reth_trie::{updates::TrieUpdatesSorted, HashedPostStateSorted, TrieInputSorted}; use reth_trie_db::DatabaseHashedPostState; use revm::{bytecode::Bytecode, database::BundleState, state::AccountInfo}; use std::{ @@ -29,28 +29,28 @@ use tracing::{debug, debug_span, instrument}; /// Contains the trie and hashed-state data required to initialize an overlay state provider. #[derive(Debug, Clone)] pub struct StateTrieOverlay { - /// Trie updates overlay. - pub trie_updates: Arc, - /// Hashed state overlay. - pub hashed_post_state: Arc, + input: TrieInputSorted, /// Whether construction was skipped because a reused sparse trie covers this range. skipped_for_reused_sparse_trie: bool, } impl StateTrieOverlay { - pub(crate) const fn new( - trie_updates: Arc, - hashed_post_state: Arc, - ) -> Self { - Self { trie_updates, hashed_post_state, skipped_for_reused_sparse_trie: false } + pub(crate) const fn new(input: TrieInputSorted) -> Self { + Self { input, skipped_for_reused_sparse_trie: false } } fn empty() -> Self { - Self { - trie_updates: Arc::new(TrieUpdatesSorted::default()), - hashed_post_state: Arc::new(HashedPostStateSorted::default()), - skipped_for_reused_sparse_trie: true, - } + Self { input: TrieInputSorted::default(), skipped_for_reused_sparse_trie: true } + } + + /// Returns the trie input represented by this overlay. + pub const fn input(&self) -> &TrieInputSorted { + &self.input + } + + /// Consumes the overlay and returns its trie input. + pub fn into_input(self) -> TrieInputSorted { + self.input } pub(crate) const fn skipped_for_reused_sparse_trie(&self) -> bool { @@ -373,11 +373,17 @@ impl OverlayBuilder { } /// Builds the effective state trie overlay for the given provider. + /// + /// Set `trie_changesets` only for consumers that produce [`TrieUpdates`], such as + /// [`StateRootProvider::state_root_with_updates`]. Other consumers, including roots, proofs, + /// multiproofs, and witnesses, should leave it false: reverts are then represented by + /// hashed-state prefix sets instead of querying trie changesets. #[cfg(test)] #[instrument(level = "debug", target = "storage::overlay", skip_all)] fn build_state_trie_overlay( &self, provider: &Provider, + trie_changesets: bool, ) -> ProviderResult where Provider: StageCheckpointReader @@ -389,7 +395,12 @@ impl OverlayBuilder { + StorageSettingsCache, { let (state_trie_tip_block, finish_tip_block) = database_state_frontiers(provider)?; - self.build_state_trie_overlay_at_frontiers(provider, state_trie_tip_block, finish_tip_block) + self.build_state_trie_overlay_at_frontiers( + provider, + state_trie_tip_block, + finish_tip_block, + trie_changesets, + ) } /// Builds the effective state trie overlay using frontiers already read from the provider. @@ -406,6 +417,7 @@ impl OverlayBuilder { provider: &Provider, state_trie_tip_block: BlockNumHash, finish_tip_block: BlockNumHash, + trie_changesets: bool, ) -> ProviderResult where Provider: ChangeSetReader @@ -425,7 +437,7 @@ impl OverlayBuilder { self.anchor_at_parent_with_frontiers(provider, state_trie_tip_block, finish_tip_block)?; // Collect any reverts which are required to bring the DB view back to the anchor hash. - let (trie_updates, hashed_post_state) = match &anchor_for_parent { + let (trie_updates, hashed_post_state, prefix_sets) = match &anchor_for_parent { AnchorForParent::RevertsRequired { anchor, .. } => { let revert_blocks = self.revert_blocks(&anchor_for_parent)?.expect("reverts are required"); @@ -437,7 +449,7 @@ impl OverlayBuilder { "Collecting trie reverts for overlay state provider" ); - let trie_reverts = { + let trie_reverts = if trie_changesets { let _guard = debug_span!(target: "storage::overlay", "retrieving_trie_reverts") .entered(); let start = Instant::now(); @@ -450,6 +462,9 @@ impl OverlayBuilder { )?; retrieve_trie_reverts_duration = start.elapsed(); accumulated_reverts + } else { + retrieve_trie_reverts_duration = Duration::ZERO; + Arc::default() }; let mut hashed_state_reverts = { @@ -462,6 +477,12 @@ impl OverlayBuilder { res }; + let prefix_sets = if trie_changesets { + Default::default() + } else { + hashed_state_reverts.construct_prefix_sets() + }; + // Resolve overlays and extend reverts with them. If reverts are empty, use overlays // directly to avoid cloning. let (overlay_trie, overlay_state) = @@ -497,7 +518,7 @@ impl OverlayBuilder { "Reverted to anchor block", ); - (trie_updates, hashed_state_updates) + (trie_updates, hashed_state_updates, prefix_sets) } AnchorForParent::NoReverts { anchor } => { // If no reverts are needed, use the manager overlay directly unless the reused @@ -528,7 +549,7 @@ impl OverlayBuilder { "Built overlay directly from durable frontier" ); - (trie_updates, hashed_post_state) + (trie_updates, hashed_post_state, Default::default()) } }; @@ -541,7 +562,11 @@ impl OverlayBuilder { self.metrics.trie_updates_size.record(trie_updates_total_len as f64); self.metrics.hashed_state_size.record(hashed_state_updates_total_len as f64); - Ok(StateTrieOverlay::new(trie_updates, hashed_post_state)) + Ok(StateTrieOverlay::new(TrieInputSorted::new( + trie_updates, + hashed_post_state, + prefix_sets, + ))) } /// Returns the in-memory execution overlay and the block for historical fallback reads. @@ -868,11 +893,11 @@ mod tests { } fn account_keys(overlay: &StateTrieOverlay) -> Vec { - overlay.hashed_post_state.accounts.iter().map(|(key, _)| *key).collect() + overlay.input().state.accounts.iter().map(|(key, _)| *key).collect() } fn account_node_paths(overlay: &StateTrieOverlay) -> Vec { - overlay.trie_updates.account_nodes_ref().iter().map(|(path, _)| *path).collect() + overlay.input().nodes.account_nodes_ref().iter().map(|(path, _)| *path).collect() } #[test] @@ -989,7 +1014,7 @@ mod tests { for (parent_index, expected_ids) in [(3, vec![3, 4]), (4, vec![3, 4, 5])] { let overlay = manager .overlay_builder(blocks[parent_index].recovered_block().hash()) - .build_state_trie_overlay(&provider) + .build_state_trie_overlay(&provider, true) .unwrap(); assert_eq!( @@ -1017,11 +1042,11 @@ mod tests { let overlay = manager .overlay_builder(blocks[4].recovered_block().hash()) .with_skip_overlay_for_reused_sparse_trie(blocks[3].recovered_block().hash()) - .build_state_trie_overlay(&provider) + .build_state_trie_overlay(&provider, true) .unwrap(); - assert!(overlay.hashed_post_state.is_empty()); - assert!(overlay.trie_updates.is_empty()); + assert!(overlay.input().state.is_empty()); + assert!(overlay.input().nodes.is_empty()); } #[test] @@ -1032,11 +1057,37 @@ mod tests { let builder = OverlayManager::::default() .overlay_builder(blocks[1].recovered_block().hash()) .with_no_reverts(); - let error = builder.build_state_trie_overlay(&provider).unwrap_err(); + let error = builder.build_state_trie_overlay(&provider, true).unwrap_err(); assert!(error.to_string().contains("reverts are disabled")); } + #[test] + fn state_trie_overlay_uses_revert_prefix_sets_without_trie_changesets() { + let (factory, blocks) = setup_frontiers(2, 3); + let provider_rw = factory.provider_rw().unwrap(); + provider_rw + .tx_ref() + .put::( + 3, + AccountBeforeTx { + address: Address::with_last_byte(1), + info: Some(Account::default()), + }, + ) + .unwrap(); + provider_rw.commit().unwrap(); + + let provider = factory.provider().unwrap(); + let overlay = OverlayManager::::default() + .overlay_builder(blocks[1].recovered_block().hash()) + .build_state_trie_overlay(&provider, false) + .unwrap(); + + assert!(overlay.input().nodes.is_empty()); + assert!(!overlay.input().prefix_sets.is_empty()); + } + #[test] fn execution_overlay_marks_historical_fallback() { let (factory, blocks) = setup_frontiers(1, 3); @@ -1231,7 +1282,7 @@ mod tests { let provider = factory.provider().unwrap(); let error = OverlayManager::::default() .overlay_builder(blocks[3].recovered_block().hash()) - .build_state_trie_overlay(&provider) + .build_state_trie_overlay(&provider, true) .unwrap_err(); assert!( @@ -1247,7 +1298,7 @@ mod tests { let parent_hash = blocks[3].recovered_block().hash(); let error = OverlayManager::::default() .overlay_builder(parent_hash) - .build_state_trie_overlay(&provider) + .build_state_trie_overlay(&provider, true) .unwrap_err(); assert!(error.to_string().contains("is after partial state trie frontier")); diff --git a/crates/storage/storage-overlay/src/changeset_cache.rs b/crates/storage/storage-overlay/src/changeset_cache.rs index c2e070fd4b1..69191d65ae9 100644 --- a/crates/storage/storage-overlay/src/changeset_cache.rs +++ b/crates/storage/storage-overlay/src/changeset_cache.rs @@ -403,7 +403,7 @@ impl ChangesetCache { let overlay = overlay_manager .overlay_builder(finish.hash) .with_no_reverts() - .build_state_trie_overlay_at_frontiers(provider, partial_state_trie, finish)?; + .build_state_trie_overlay_at_frontiers(provider, partial_state_trie, finish, true)?; let state_trie_provider = OverlayStateProvider::<&P, N>::new_with_state_trie( provider, overlay, @@ -644,7 +644,7 @@ mod tests { } fn empty_overlay() -> StateTrieOverlay { - StateTrieOverlay::new(Arc::default(), Arc::default()) + StateTrieOverlay::new(TrieInputSorted::default()) } fn insert_test_changesets( diff --git a/crates/storage/storage-overlay/src/provider.rs b/crates/storage/storage-overlay/src/provider.rs index 578e4453560..d40677ae1b8 100644 --- a/crates/storage/storage-overlay/src/provider.rs +++ b/crates/storage/storage-overlay/src/provider.rs @@ -52,7 +52,7 @@ pub struct OverlayStateProviderFactory { factory: F, /// Overlay builder containing the configuration and overlay calculation logic. overlay_builder: OverlayBuilder, - /// A cache mapping `(state_trie_tip, finish_tip)` to [`StateTrieOverlay`]. + /// A cache mapping `(state_trie_tip, finish_tip, trie_changesets)` to [`StateTrieOverlay`]. /// /// Under partial persistence the overlay depends on both durable frontiers, so both hashes are /// part of the cache key. @@ -129,6 +129,7 @@ pub struct OverlayStateProvider { state_trie_overlay_cache: StateTrieOverlayCache, metrics: OverlayStateProviderFactoryMetrics, state_trie_overlay: OnceCell, + state_trie_overlay_with_trie_changesets: OnceCell, execution_overlay: OnceCell, is_v2: bool, } @@ -162,6 +163,7 @@ impl OverlayStateProvider, state_trie_overlay_cache, metrics, state_trie_overlay: OnceCell::new(), + state_trie_overlay_with_trie_changesets: OnceCell::new(), execution_overlay: OnceCell::new(), is_v2, } @@ -179,6 +181,7 @@ impl OverlayStateProvider, state_trie_overlay_cache: Default::default(), metrics: Default::default(), state_trie_overlay: OnceCell::new(), + state_trie_overlay_with_trie_changesets: OnceCell::new(), execution_overlay: OnceCell::from(CachedExecutionOverlay { overlay: execution_overlay, historical_fallback: None, @@ -201,6 +204,7 @@ impl<'a, Provider, N: NodePrimitives> OverlayStateProvider<&'a Provider, N> { state_trie_overlay_cache: Default::default(), metrics: Default::default(), state_trie_overlay: OnceCell::new(), + state_trie_overlay_with_trie_changesets: OnceCell::new(), execution_overlay: OnceCell::new(), is_v2, } @@ -216,7 +220,8 @@ impl<'a, Provider, N: NodePrimitives> OverlayStateProvider<&'a Provider, N> { overlay_builder: None, state_trie_overlay_cache: Default::default(), metrics: Default::default(), - state_trie_overlay: OnceCell::from(state_trie_overlay), + state_trie_overlay: OnceCell::from(state_trie_overlay.clone()), + state_trie_overlay_with_trie_changesets: OnceCell::from(state_trie_overlay), execution_overlay: OnceCell::new(), is_v2, } @@ -232,7 +237,7 @@ where &self.provider } - fn state_trie_overlay(&self) -> ProviderResult<&StateTrieOverlay> + fn state_trie_overlay(&self, trie_changesets: bool) -> ProviderResult<&StateTrieOverlay> where Provider::Target: StageCheckpointReader + PruneCheckpointReader @@ -242,15 +247,17 @@ where + BlockNumReader + StorageSettingsCache, { - if let Some(overlay) = self.state_trie_overlay.get() { + let state_trie_overlay = self.state_trie_overlay_mut(trie_changesets); + if let Some(overlay) = state_trie_overlay.get() { return Ok(overlay) } let (state_trie_tip_block, finish_tip_block) = database_state_frontiers(self.provider())?; - let overlay = match self - .state_trie_overlay_cache - .entry((state_trie_tip_block.hash, finish_tip_block.hash)) - { + let overlay = match self.state_trie_overlay_cache.entry(( + state_trie_tip_block.hash, + finish_tip_block.hash, + trie_changesets, + )) { dashmap::Entry::Occupied(entry) => entry.get().clone(), dashmap::Entry::Vacant(entry) => { self.metrics.state_trie_overlay_cache_misses.increment(1); @@ -262,6 +269,7 @@ where self.provider(), state_trie_tip_block, finish_tip_block, + trie_changesets, )?; if !overlay.skipped_for_reused_sparse_trie() { entry.insert(overlay.clone()); @@ -269,11 +277,23 @@ where overlay } }; - let _ = self.state_trie_overlay.set(overlay); - Ok(self.state_trie_overlay.get().expect("state trie overlay was just initialized")) + let _ = state_trie_overlay.set(overlay); + Ok(state_trie_overlay.get().expect("state trie overlay was just initialized")) + } + + const fn state_trie_overlay_mut(&self, trie_changesets: bool) -> &OnceCell { + if trie_changesets { + &self.state_trie_overlay_with_trie_changesets + } else { + &self.state_trie_overlay + } } - fn build_overlay(&self, input: TrieInputSorted) -> ProviderResult + fn build_overlay( + &self, + input: TrieInputSorted, + trie_changesets: bool, + ) -> ProviderResult where Provider::Target: StageCheckpointReader + PruneCheckpointReader @@ -283,13 +303,14 @@ where + BlockNumReader + StorageSettingsCache, { - let overlay = self.state_trie_overlay()?; + let overlay = self.state_trie_overlay(trie_changesets)?; if overlay.skipped_for_reused_sparse_trie() { return Err(ProviderError::UnsupportedProvider) } - let TrieInputSorted { nodes: input_nodes, state: input_state, prefix_sets } = input; - let mut nodes = Arc::clone(&overlay.trie_updates); - let mut state = Arc::clone(&overlay.hashed_post_state); + let TrieInputSorted { nodes: input_nodes, state: input_state, mut prefix_sets } = input; + let overlay_input = overlay.input(); + let mut nodes = Arc::clone(&overlay_input.nodes); + let mut state = Arc::clone(&overlay_input.state); if !input_nodes.is_empty() { Arc::make_mut(&mut nodes).extend_ref_and_sort(&input_nodes); @@ -298,6 +319,7 @@ where Arc::make_mut(&mut state).extend_ref_and_sort(&input_state); } + prefix_sets.extend_ref(&overlay_input.prefix_sets); Ok(TrieInputSorted::new(nodes, state, prefix_sets)) } @@ -511,16 +533,17 @@ where { fn state_root(&self, hashed_state: HashedPostState) -> ProviderResult { reth_trie_db::with_adapter!(self.provider(), |A| { - let input = self.build_overlay(TrieInputSorted::from_unsorted( - TrieInput::from_state(hashed_state), - ))?; + let input = self.build_overlay( + TrieInputSorted::from_unsorted(TrieInput::from_state(hashed_state)), + false, + )?; Ok(>::overlay_root_from_nodes(self.provider().tx(), input)?) }) } fn state_root_from_nodes(&self, input: TrieInput) -> ProviderResult { reth_trie_db::with_adapter!(self.provider(), |A| { - let input = self.build_overlay(TrieInputSorted::from_unsorted(input))?; + let input = self.build_overlay(TrieInputSorted::from_unsorted(input), false)?; Ok( as DatabaseStateRoot<_>>::overlay_root_from_nodes( self.provider().tx(), input, @@ -533,9 +556,10 @@ where hashed_state: HashedPostState, ) -> ProviderResult<(B256, TrieUpdates)> { reth_trie_db::with_adapter!(self.provider(), |A| { - let input = self.build_overlay(TrieInputSorted::from_unsorted( - TrieInput::from_state(hashed_state), - ))?; + let input = self.build_overlay( + TrieInputSorted::from_unsorted(TrieInput::from_state(hashed_state)), + true, + )?; Ok(>::overlay_root_from_nodes_with_updates( self.provider().tx(), input, @@ -548,7 +572,7 @@ where input: TrieInput, ) -> ProviderResult<(B256, TrieUpdates)> { reth_trie_db::with_adapter!(self.provider(), |A| { - let input = self.build_overlay(TrieInputSorted::from_unsorted(input))?; + let input = self.build_overlay(TrieInputSorted::from_unsorted(input), true)?; Ok( as DatabaseStateRoot<_>>::overlay_root_from_nodes_with_updates( self.provider().tx(), @@ -576,12 +600,15 @@ where hashed_storage: HashedStorage, ) -> ProviderResult { reth_trie_db::with_adapter!(self.provider(), |A| { - let input = self.build_overlay(TrieInputSorted::from_unsorted( - TrieInput::from_state(HashedPostState::from_hashed_storage( - alloy_primitives::keccak256(address), - hashed_storage, + let input = self.build_overlay( + TrieInputSorted::from_unsorted(TrieInput::from_state( + HashedPostState::from_hashed_storage( + alloy_primitives::keccak256(address), + hashed_storage, + ), )), - ))?; + false, + )?; let hashed_storage = input .state .account_storages() @@ -601,12 +628,15 @@ where hashed_storage: HashedStorage, ) -> ProviderResult { reth_trie_db::with_adapter!(self.provider(), |A| { - let input = self.build_overlay(TrieInputSorted::from_unsorted( - TrieInput::from_state(HashedPostState::from_hashed_storage( - alloy_primitives::keccak256(address), - hashed_storage, + let input = self.build_overlay( + TrieInputSorted::from_unsorted(TrieInput::from_state( + HashedPostState::from_hashed_storage( + alloy_primitives::keccak256(address), + hashed_storage, + ), )), - ))?; + false, + )?; let hashed_storage = input .state .account_storages() @@ -631,12 +661,15 @@ where hashed_storage: HashedStorage, ) -> ProviderResult { reth_trie_db::with_adapter!(self.provider(), |A| { - let input = self.build_overlay(TrieInputSorted::from_unsorted( - TrieInput::from_state(HashedPostState::from_hashed_storage( - alloy_primitives::keccak256(address), - hashed_storage, + let input = self.build_overlay( + TrieInputSorted::from_unsorted(TrieInput::from_state( + HashedPostState::from_hashed_storage( + alloy_primitives::keccak256(address), + hashed_storage, + ), )), - ))?; + false, + )?; let hashed_storage = input .state .account_storages() @@ -674,7 +707,7 @@ where ) -> ProviderResult { reth_trie_db::with_adapter!(self.provider(), |A| { let TrieInputSorted { nodes, state, prefix_sets } = - self.build_overlay(TrieInputSorted::from_unsorted(input))?; + self.build_overlay(TrieInputSorted::from_unsorted(input), false)?; let input = TrieInput::new( Arc::unwrap_or_clone(nodes).into(), Arc::unwrap_or_clone(state).into(), @@ -692,7 +725,7 @@ where ) -> ProviderResult { reth_trie_db::with_adapter!(self.provider(), |A| { let TrieInputSorted { nodes, state, prefix_sets } = - self.build_overlay(TrieInputSorted::from_unsorted(input))?; + self.build_overlay(TrieInputSorted::from_unsorted(input), false)?; let input = TrieInput::new( Arc::unwrap_or_clone(nodes).into(), Arc::unwrap_or_clone(state).into(), @@ -710,7 +743,7 @@ where ) -> ProviderResult { reth_trie_db::with_adapter!(self.provider(), |A| { let TrieInputSorted { nodes, state, prefix_sets } = - self.build_overlay(TrieInputSorted::from_unsorted(input))?; + self.build_overlay(TrieInputSorted::from_unsorted(input), false)?; let input = TrieInput::new( Arc::unwrap_or_clone(nodes).into(), Arc::unwrap_or_clone(state).into(), @@ -729,7 +762,7 @@ where ) -> ProviderResult> { reth_trie_db::with_adapter!(self.provider(), |A| { let TrieInputSorted { nodes, state, prefix_sets } = - self.build_overlay(TrieInputSorted::from_unsorted(input))?; + self.build_overlay(TrieInputSorted::from_unsorted(input), false)?; let witness = TrieWitness::new( InMemoryTrieCursorFactory::new( DatabaseTrieCursorFactory::<_, A>::new(self.provider().tx()), @@ -778,7 +811,7 @@ where return Ok(hashed_state) } - let overlay_state = self.build_overlay(TrieInputSorted::default())?.state; + let overlay_state = self.build_overlay(TrieInputSorted::default(), false)?.state; zero_destroyed_account_storage( &HashedPostStateCursorFactory::new( DatabaseHashedCursorFactory::new(self.provider().tx()), @@ -893,7 +926,7 @@ where Self: 'a; fn account_trie_cursor(&self) -> Result, DatabaseError> { - let overlay = self.state_trie_overlay().map_err(into_database_error)?; + let overlay = self.state_trie_overlay(true).map_err(into_database_error)?; let cursor: Box = if self.is_v2 { Box::new(DatabaseAccountTrieCursor::<_, PackedKeyAdapter>::new( self.provider().tx().cursor_read::()?, @@ -903,14 +936,14 @@ where self.provider().tx().cursor_read::()?, )) }; - Ok(InMemoryTrieCursor::new_account(cursor, &overlay.trie_updates)) + Ok(InMemoryTrieCursor::new_account(cursor, &overlay.input().nodes)) } fn storage_trie_cursor( &self, hashed_address: B256, ) -> Result, DatabaseError> { - let overlay = self.state_trie_overlay().map_err(into_database_error)?; + let overlay = self.state_trie_overlay(true).map_err(into_database_error)?; let cursor: Box = if self.is_v2 { Box::new(DatabaseStorageTrieCursor::<_, PackedKeyAdapter>::new( self.provider().tx().cursor_dup_read::()?, @@ -922,7 +955,7 @@ where hashed_address, )) }; - Ok(InMemoryTrieCursor::new_storage(cursor, &overlay.trie_updates, hashed_address)) + Ok(InMemoryTrieCursor::new_storage(cursor, &overlay.input().nodes, hashed_address)) } } @@ -954,10 +987,10 @@ where Self: 'a; fn hashed_account_cursor(&self) -> Result, DatabaseError> { - let overlay = self.state_trie_overlay().map_err(into_database_error)?; + let overlay = self.state_trie_overlay(true).map_err(into_database_error)?; HashedPostStateCursorFactory::new( DatabaseHashedCursorFactory::new(self.provider().tx()), - &overlay.hashed_post_state, + &overlay.input().state, ) .hashed_account_cursor() } @@ -966,10 +999,10 @@ where &self, hashed_address: B256, ) -> Result, DatabaseError> { - let overlay = self.state_trie_overlay().map_err(into_database_error)?; + let overlay = self.state_trie_overlay(true).map_err(into_database_error)?; HashedPostStateCursorFactory::new( DatabaseHashedCursorFactory::new(self.provider().tx()), - &overlay.hashed_post_state, + &overlay.input().state, ) .hashed_storage_cursor(hashed_address) } @@ -987,7 +1020,7 @@ pub(crate) struct OverlayStateProviderFactoryMetrics { state_trie_overlay_cache_misses: Counter, } -type StateTrieOverlayCache = Arc>; +type StateTrieOverlayCache = Arc>; #[derive(Clone, Debug)] struct CachedExecutionOverlay { @@ -1122,11 +1155,11 @@ mod tests { } fn account_keys(overlay: &StateTrieOverlay) -> Vec { - overlay.hashed_post_state.accounts.iter().map(|(key, _)| *key).collect() + overlay.input().state.accounts.iter().map(|(key, _)| *key).collect() } fn account_node_paths(overlay: &StateTrieOverlay) -> Vec { - overlay.trie_updates.account_nodes_ref().iter().map(|(path, _)| *path).collect() + overlay.input().nodes.account_nodes_ref().iter().map(|(path, _)| *path).collect() } #[test] @@ -1142,7 +1175,7 @@ mod tests { ); let provider = state_provider_factory.database_provider_ro().unwrap(); - let first = provider.state_trie_overlay().unwrap().clone(); + let first = provider.state_trie_overlay(false).unwrap().clone(); assert_eq!(account_keys(&first), vec![B256::with_last_byte(3), B256::with_last_byte(4)]); drop(provider); @@ -1158,7 +1191,7 @@ mod tests { provider_rw.commit().unwrap(); let provider = state_provider_factory.database_provider_ro().unwrap(); - let second = provider.state_trie_overlay().unwrap().clone(); + let second = provider.state_trie_overlay(false).unwrap().clone(); assert_eq!(account_keys(&second), vec![B256::with_last_byte(4)]); assert_eq!(account_node_paths(&second), vec![Nibbles::from_nibbles([4])]); assert_eq!(state_provider_factory.state_trie_overlay_cache.len(), 2); @@ -1191,6 +1224,39 @@ mod tests { assert_eq!(state_provider_factory.state_trie_overlay_cache.len(), 1); } + #[test] + fn state_trie_overlay_cache_is_keyed_by_trie_changesets() { + let (factory, blocks) = setup_frontiers(1, 3); + let manager = OverlayManager::default(); + for block in &blocks[2..=3] { + manager.insert_block(block.clone()); + } + let state_provider_factory = OverlayStateProviderFactory::new( + factory, + manager.overlay_builder(blocks[3].recovered_block().hash()), + ); + let provider = state_provider_factory.database_provider_ro().unwrap(); + + provider.state_trie_overlay(false).unwrap(); + provider.state_trie_overlay(true).unwrap(); + + assert_eq!(state_provider_factory.state_trie_overlay_cache.len(), 2); + } + + #[test] + fn supplied_state_trie_overlay_is_available_in_both_modes() { + let (factory, _) = setup_frontiers(1, 1); + let provider = factory.provider().unwrap(); + let provider = OverlayStateProvider::<&_, EthPrimitives>::new_with_state_trie( + &provider, + StateTrieOverlay::new(TrieInputSorted::default()), + false, + ); + + assert!(provider.state_trie_overlay(false).is_ok()); + assert!(provider.state_trie_overlay(true).is_ok()); + } + #[test] fn lazy_overlays_survive_manager_block_removal() { let (factory, blocks) = setup_frontiers(1, 1); @@ -1212,7 +1278,7 @@ mod tests { [blocks[2].recovered_block().num_hash(), blocks[3].recovered_block().num_hash()] ); assert_eq!( - account_keys(provider.state_trie_overlay().unwrap()), + account_keys(provider.state_trie_overlay(false).unwrap()), vec![B256::with_last_byte(3), B256::with_last_byte(4)] ); } @@ -1230,7 +1296,7 @@ mod tests { ); let provider = state_provider_factory.database_provider_ro().unwrap(); - assert!(provider.state_trie_overlay().unwrap().skipped_for_reused_sparse_trie()); + assert!(provider.state_trie_overlay(false).unwrap().skipped_for_reused_sparse_trie()); assert!(state_provider_factory.state_trie_overlay_cache.is_empty()); assert!(matches!( provider.state_root(HashedPostState::default()), From f8740008d618aa3ee5d88ea145cfc6989b5c2339 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Fri, 4 Sep 2026 17:50:56 +0200 Subject: [PATCH 165/294] fix(rpc): resolve a pending toBlock in eth_getLogs to the head (#27001) --- crates/rpc/rpc/src/eth/filter.rs | 33 +++++++++++++++++++ .../storage/provider/src/test_utils/mock.rs | 14 ++++++-- 2 files changed, 45 insertions(+), 2 deletions(-) diff --git a/crates/rpc/rpc/src/eth/filter.rs b/crates/rpc/rpc/src/eth/filter.rs index 11663d204f2..d64efc561b8 100644 --- a/crates/rpc/rpc/src/eth/filter.rs +++ b/crates/rpc/rpc/src/eth/filter.rs @@ -576,11 +576,15 @@ where let info = self.provider().chain_info()?; let start_block = info.best_number; + // Without a pending block to serve, a `pending` bound resolves to the head on both + // ends instead of to whatever payload the engine currently holds let from = from_block + .filter(|num| !num.is_pending()) .map(|num| self.provider().convert_block_number(num)) .transpose()? .flatten(); let to = to_block + .filter(|num| !num.is_pending()) .map(|num| self.provider().convert_block_number(num)) .transpose()? .flatten(); @@ -2225,4 +2229,33 @@ mod tests { "{err:?}" ); } + + #[tokio::test] + async fn test_logs_for_filter_pending_to_block_ends_at_head() { + let provider = MockEthProvider::default(); + let header = alloy_consensus::Header { number: 2, ..Default::default() }; + let hash = header.hash_slow(); + provider.add_header(hash, header); + provider.add_receipts(2, vec![]); + // the engine holds a payload that is not canonical yet + provider.set_pending_block_num_hash(Some(alloy_eips::BlockNumHash::new(3, hash))); + + let eth_filter = EthFilter::new( + build_test_eth_api(provider), + EthFilterConfig::default(), + Runtime::test(), + ); + for filter in [ + Filter::new().from_block(0u64).to_block(BlockNumberOrTag::Pending), + Filter::new().select(BlockNumberOrTag::Pending..), + ] { + let logs = eth_filter + .inner + .clone() + .logs_for_filter(filter, QueryLimits::default()) + .await + .unwrap(); + assert!(logs.is_empty()); + } + } } diff --git a/crates/storage/provider/src/test_utils/mock.rs b/crates/storage/provider/src/test_utils/mock.rs index e24928c51af..ae1eb506362 100644 --- a/crates/storage/provider/src/test_utils/mock.rs +++ b/crates/storage/provider/src/test_utils/mock.rs @@ -12,7 +12,7 @@ use alloy_consensus::{ transaction::{TransactionMeta, TxHashRef}, BlockHeader, }; -use alloy_eips::{BlockHashOrNumber, BlockId, BlockNumberOrTag}; +use alloy_eips::{BlockHashOrNumber, BlockId, BlockNumHash, BlockNumberOrTag}; use alloy_primitives::{ keccak256, map::{AddressMap, B256Map, HashMap}, @@ -76,6 +76,8 @@ pub struct MockEthProvider>>, /// Local stage checkpoints stage_checkpoints: Arc>>, + /// The engine's pending block, if any + pending_block_num_hash: Arc>>, /// Local BAL store handle pub bal_store: BalStoreHandle, /// Whether database provider creation succeeds. @@ -131,6 +133,7 @@ where state_roots: self.state_roots.clone(), block_body_indices: self.block_body_indices.clone(), stage_checkpoints: self.stage_checkpoints.clone(), + pending_block_num_hash: self.pending_block_num_hash.clone(), bal_store: self.bal_store.clone(), database_provider_available: self.database_provider_available.clone(), snap_state_reads_fail: self.snap_state_reads_fail.clone(), @@ -160,6 +163,7 @@ impl MockEthProvider { state_roots: Default::default(), block_body_indices: Default::default(), stage_checkpoints: Default::default(), + pending_block_num_hash: Default::default(), bal_store: Default::default(), database_provider_available: Default::default(), snap_state_reads_fail: Default::default(), @@ -315,6 +319,11 @@ impl MockEthProvider { self.stage_checkpoints.lock().insert(id, checkpoint); } + /// Sets the pending block the engine holds + pub fn set_pending_block_num_hash(&self, num_hash: Option) { + *self.pending_block_num_hash.lock() = num_hash; + } + /// Add state root to local state root store pub fn add_state_root(&self, state_root: B256) { self.state_roots.lock().push(state_root); @@ -331,6 +340,7 @@ impl MockEthProvider { state_roots: self.state_roots, block_body_indices: self.block_body_indices, stage_checkpoints: self.stage_checkpoints, + pending_block_num_hash: self.pending_block_num_hash, bal_store: self.bal_store, database_provider_available: self.database_provider_available, snap_state_reads_fail: self.snap_state_reads_fail, @@ -889,7 +899,7 @@ impl BlockId for MockEthProvider { fn pending_block_num_hash(&self) -> ProviderResult> { - Ok(None) + Ok(*self.pending_block_num_hash.lock()) } fn safe_block_num_hash(&self) -> ProviderResult> { From 95e1aa0858f9b7d470f04c7c77cc37059751f0ed Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Fri, 4 Sep 2026 18:02:22 +0200 Subject: [PATCH 166/294] test(node): wait for dev finality updates (#27014) --- crates/ethereum/node/tests/e2e/dev.rs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/crates/ethereum/node/tests/e2e/dev.rs b/crates/ethereum/node/tests/e2e/dev.rs index c6e15ce4189..03a0629787d 100644 --- a/crates/ethereum/node/tests/e2e/dev.rs +++ b/crates/ethereum/node/tests/e2e/dev.rs @@ -32,8 +32,17 @@ async fn can_run_dev_node() -> eyre::Result<()> { .launch_with_debug_capabilities() .await?; + let canon_state = node.provider.canonical_in_memory_state(); + let mut safe_block = canon_state.subscribe_safe_block(); + let mut finalized_block = canon_state.subscribe_finalized_block(); + assert_chain_advances(&node).await; + let (safe_changed, finalized_changed) = + tokio::join!(safe_block.changed(), finalized_block.changed()); + safe_changed?; + finalized_changed?; + let chain_info = node.provider.chain_info()?; assert_eq!(node.provider.safe_block_num_hash()?, Some(chain_info.into())); assert_eq!(node.provider.finalized_block_num_hash()?, Some(chain_info.into())); From d5cd94331e030f7dbd40ce9d42afa5b1951fe132 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Fri, 4 Sep 2026 16:11:08 +0000 Subject: [PATCH 167/294] fix(rpc): reject eth_getLogs queries over pruned receipts (#27002) --- crates/rpc/rpc/src/eth/filter.rs | 133 +++++++++++++++++++++++++------ 1 file changed, 108 insertions(+), 25 deletions(-) diff --git a/crates/rpc/rpc/src/eth/filter.rs b/crates/rpc/rpc/src/eth/filter.rs index d64efc561b8..025e9d247cf 100644 --- a/crates/rpc/rpc/src/eth/filter.rs +++ b/crates/rpc/rpc/src/eth/filter.rs @@ -497,7 +497,22 @@ where let Some((receipts, maybe_block)) = self.eth_cache().get_receipts_and_maybe_block(block_hash).await? else { - return Err(ProviderError::HeaderNotFound(block_hash.into()).into()) + // the block itself may still exist with its receipts pruned + return Err(match self.provider().block_number(block_hash)? { + Some(number) => { + let earliest_available = self.provider().earliest_block_number()?; + if number < earliest_available { + EthApiError::PrunedHistoryUnavailable { + requested: number, + earliest_available, + } + .into() + } else { + EthFilterError::ReceiptsUnavailable(number) + } + } + None => ProviderError::HeaderNotFound(block_hash.into()).into(), + }) }; let header = if let Some(block) = &maybe_block { @@ -1011,6 +1026,9 @@ pub enum EthFilterError { /// Query scope is too broad. #[error("query exceeds max block range {0}")] QueryExceedsMaxBlocks(u64), + /// Receipts of a block the filter matched are gone, most likely pruned. + #[error("pruned history unavailable")] + ReceiptsUnavailable(u64), /// Query result is too large. #[error("query exceeds max results {max_logs}, retry with the range {from_block}-{to_block}")] QueryExceedsMaxResults { @@ -1040,6 +1058,9 @@ impl From for jsonrpsee::types::error::ErrorObject<'static> { rpc_error_with_code(jsonrpsee::types::error::INTERNAL_ERROR_CODE, err.to_string()) } EthFilterError::EthAPIError(err) => err.into(), + err @ EthFilterError::ReceiptsUnavailable(_) => { + rpc_error_with_code(4444, err.to_string()) + } err @ (EthFilterError::InvalidBlockRangeParams | EthFilterError::QueryExceedsMaxBlocks(_) | EthFilterError::QueryExceedsMaxResults { .. } | @@ -1196,20 +1217,16 @@ impl< > CachedMode { async fn next(&mut self) -> Result>, EthFilterError> { - for header in self.headers_iter.by_ref() { - // Use get_receipts_and_maybe_block which has automatic fallback to provider - if let Some((receipts, maybe_block)) = - self.filter_inner.eth_cache().get_receipts_and_maybe_block(header.hash()).await? - { - return Ok(Some(ReceiptBlockResult { - receipts, - recovered_block: maybe_block, - header, - })); - } - } + let Some(header) = self.headers_iter.next() else { return Ok(None) }; - Ok(None) // No more headers + // Use get_receipts_and_maybe_block which has automatic fallback to provider + let Some((receipts, maybe_block)) = + self.filter_inner.eth_cache().get_receipts_and_maybe_block(header.hash()).await? + else { + return Err(EthFilterError::ReceiptsUnavailable(header.number())) + }; + + Ok(Some(ReceiptBlockResult { receipts, recovered_block: maybe_block, header })) } } @@ -1324,7 +1341,7 @@ impl< // Not cached - fetch directly from provider match self.filter_inner.provider().receipts_by_block(header.hash().into())? { Some(receipts) => Arc::new(receipts), - None => continue, // No receipts found + None => return Err(EthFilterError::ReceiptsUnavailable(header.number())), } } }; @@ -1405,7 +1422,7 @@ impl< // unlikely to be cached let receipts = match filter_inner.provider().receipts_by_block(header.hash().into())? { Some(receipts) => Arc::new(receipts), - None => continue, // No receipts found + None => return Err(EthFilterError::ReceiptsUnavailable(header.number())), }; if !receipts.is_empty() { @@ -1437,6 +1454,14 @@ mod tests { use reth_transaction_pool::test_utils::{testing_pool, TestPool}; use std::{collections::VecDeque, sync::Arc}; + #[test] + fn receipts_unavailable_error_matches_geth() { + let err: jsonrpsee::types::error::ErrorObject<'static> = + EthFilterError::ReceiptsUnavailable(100).into(); + assert_eq!(err.code(), 4444); + assert_eq!(err.message(), "pruned history unavailable"); + } + #[test] fn test_block_range_iter() { let mut rng = generators::rng(); @@ -1520,11 +1545,6 @@ mod tests { #[tokio::test] async fn test_range_block_mode_queued_results_priority() { let provider = MockEthProvider::default(); - let eth_api = build_test_eth_api(provider); - - let eth_filter = - super::EthFilter::new(eth_api, EthFilterConfig::default(), Runtime::test()); - let filter_inner = eth_filter.inner; let headers = vec![ SealedHeader::new( @@ -1536,6 +1556,16 @@ mod tests { FixedBytes::random(), ), ]; + for header in &headers { + provider.add_header(header.hash(), header.header().clone()); + provider.add_receipts(header.number(), vec![]); + } + + let eth_api = build_test_eth_api(provider); + + let eth_filter = + super::EthFilter::new(eth_api, EthFilterConfig::default(), Runtime::test()); + let filter_inner = eth_filter.inner; // create specific mock results to test ordering let expected_block_hash_1 = FixedBytes::from([1u8; 32]); @@ -1635,7 +1665,7 @@ mod tests { } #[tokio::test] - async fn test_range_block_mode_single_block_no_receipts() { + async fn test_range_block_mode_single_block_missing_receipts() { let provider = MockEthProvider::default(); let eth_api = build_test_eth_api(provider); @@ -1658,8 +1688,10 @@ mod tests { pending_tasks: FuturesOrdered::new(), }; - let result = range_mode.next().await; - assert!(result.is_ok()); + // a block whose header matched the filter but whose receipts are gone must not be + // silently skipped + let Err(err) = range_mode.next().await else { panic!("missing receipts must be an error") }; + assert!(matches!(err, EthFilterError::ReceiptsUnavailable(100)), "{err:?}"); } #[tokio::test] @@ -1705,6 +1737,8 @@ mod tests { provider.add_receipts(100, vec![receipt_100_1.clone(), receipt_100_2.clone()]); provider.add_receipts(101, vec![receipt_101_1.clone()]); + // a block without transactions, which a provider reports as an empty list + provider.add_receipts(102, vec![]); let eth_api = build_test_eth_api(provider); @@ -2107,7 +2141,9 @@ mod tests { use reth_rpc_eth_api::helpers::SpawnBlocking; let provider = MockEthProvider::default(); - provider.add_header(FixedBytes::random(), alloy_consensus::Header::default()); + let header = alloy_consensus::Header::default(); + provider.add_header(header.hash_slow(), header); + provider.add_receipts(0, vec![]); let eth_api = build_test_eth_api(provider); // take every permit so the scan has to wait for one @@ -2258,4 +2294,51 @@ mod tests { assert!(logs.is_empty()); } } + + #[tokio::test] + async fn test_logs_for_filter_over_pruned_receipts() { + let provider = MockEthProvider::default(); + let mut parent_hash = FixedBytes::default(); + for number in 0..=2u64 { + let header = alloy_consensus::Header { + number, + parent_hash, + logs_bloom: alloy_primitives::Bloom::from([1u8; 256]), + ..Default::default() + }; + parent_hash = header.hash_slow(); + provider.add_block( + parent_hash, + reth_ethereum_primitives::Block { header, body: Default::default() }, + ); + // the receipts of block 1 were pruned + if number != 1 { + provider.add_receipts(number, vec![]); + } + } + + let eth_filter = EthFilter::new( + build_test_eth_api(provider), + EthFilterConfig::default(), + Runtime::test(), + ); + + // a range that reaches the pruned block is rejected instead of served incompletely + let err = eth_filter + .inner + .clone() + .logs_for_filter(Filter::new().from_block(0u64).to_block(2u64), QueryLimits::default()) + .await + .unwrap_err(); + assert!(matches!(err, EthFilterError::ReceiptsUnavailable(1)), "{err:?}"); + + // a range that avoids it is served + let logs = eth_filter + .inner + .clone() + .logs_for_filter(Filter::new().from_block(2u64).to_block(2u64), QueryLimits::default()) + .await + .unwrap(); + assert!(logs.is_empty()); + } } From bf1db83ae0e771a436c7e9bc4c06415157438e84 Mon Sep 17 00:00:00 2001 From: Derek Cofausper <256792747+decofe@users.noreply.github.com> Date: Fri, 4 Sep 2026 16:24:30 +0000 Subject: [PATCH 168/294] fix(transaction-pool): release receiver lock before validating (#27015) Co-authored-by: Alexey Shekhirin <5773434+shekhirin@users.noreply.github.com> Co-authored-by: Emma Jamieson-Hoare --- crates/transaction-pool/src/validate/task.rs | 47 +++++++++++++++++++- 1 file changed, 46 insertions(+), 1 deletion(-) diff --git a/crates/transaction-pool/src/validate/task.rs b/crates/transaction-pool/src/validate/task.rs index bfd5b29bbf8..a3718cdce91 100644 --- a/crates/transaction-pool/src/validate/task.rs +++ b/crates/transaction-pool/src/validate/task.rs @@ -60,7 +60,11 @@ impl ValidationTask { /// /// This will run as long as the channel is alive and is expected to be spawned as a task. pub async fn run(self) { - while let Some(task) = self.validation_jobs.lock().await.next().await { + loop { + // Release the shared receiver before running the job, so other workers + // can dequeue validations while this worker is busy. + let task = { self.validation_jobs.lock().await.next().await }; + let Some(task) = task else { break }; task.await; } } @@ -350,6 +354,47 @@ mod tests { }; use alloy_primitives::{Address, U256}; + #[tokio::test] + async fn cloned_workers_validate_while_another_job_is_blocked() { + let (sender, task) = ValidationTask::new(); + let first_worker = tokio::spawn(task.clone().run()); + let second_worker = tokio::spawn(task.run()); + let (started_tx, started_rx) = oneshot::channel(); + let (release_tx, release_rx) = oneshot::channel(); + sender + .send(Box::pin(async move { + started_tx.send(()).unwrap(); + release_rx.await.unwrap(); + })) + .await + .unwrap(); + started_rx.await.unwrap(); + + // The first validation remains blocked. A second configured worker must + // independently dequeue and finish this job without releasing the first. + let (completed_tx, completed_rx) = oneshot::channel(); + sender + .send(Box::pin(async move { + completed_tx.send(()).unwrap(); + })) + .await + .unwrap(); + tokio::time::timeout(std::time::Duration::from_secs(1), completed_rx) + .await + .expect("second worker cannot dequeue while first validation holds receiver lock") + .unwrap(); + + release_tx.send(()).unwrap(); + drop(sender); + // Closing the bounded channel still shuts every worker down cleanly. + tokio::time::timeout(std::time::Duration::from_secs(1), async { + first_worker.await.unwrap(); + second_worker.await.unwrap(); + }) + .await + .unwrap(); + } + #[derive(Debug)] struct NoopValidator; From 279b758f4d91f09b9cb10686cca787b6e2d4103e Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Fri, 4 Sep 2026 16:39:35 +0000 Subject: [PATCH 169/294] perf(stages): avoid cloning ExecutionOutcome for ExEx notification (#26947) --- .../stages/stages/src/stages/execution/mod.rs | 52 ++++++++++++------- 1 file changed, 33 insertions(+), 19 deletions(-) diff --git a/crates/stages/stages/src/stages/execution/mod.rs b/crates/stages/stages/src/stages/execution/mod.rs index 0b76fdf01d1..f185df2db23 100644 --- a/crates/stages/stages/src/stages/execution/mod.rs +++ b/crates/stages/stages/src/stages/execution/mod.rs @@ -438,31 +438,22 @@ where "Finished executing block range" ); - // Prepare the input for post execute commit hook, where an `ExExNotification` will be sent. - // - // Note: Since we only write to `blocks` if there are any ExExes, we don't need to perform - // the `has_exexs` check here as well - if !blocks.is_empty() { - let previous_input = self.post_execute_commit_input.replace(Chain::new( - blocks, - state.clone(), - BTreeMap::new(), - )); - - if previous_input.is_some() { - // Not processing the previous post execute commit input is a critical error, as it - // means that we didn't send the notification to ExExes - return Err(StageError::PostExecuteCommit( - "Previous post execute commit input wasn't processed", - )) - } - } + // The ExEx notification must carry the outcome as executed, so a copy is only taken if the + // state is modified below before it is written. + let mut exex_state = None; let time = Instant::now(); if self.can_prune_changesets(provider, start_block, max_block)? { let prune_modes = provider.prune_modes_ref(); + if !blocks.is_empty() && + prune_modes.account_history.is_some() && + prune_modes.storage_history.is_some() + { + exex_state = Some(state.clone()); + } + // Iterate over all reverts and clear them if pruning is configured. for block_number in start_block..=max_block { let Some(reverts) = @@ -498,6 +489,9 @@ where let path = provider.storage_path().join("preimage"); if !provider.chain_spec().is_cancun_active_at_timestamp(start_header.timestamp()) { + if !blocks.is_empty() && exex_state.is_none() { + exex_state = Some(state.clone()); + } slot_preimages::inject_plain_wipe_slots(&path, provider, &mut state)?; } else if path.exists() { // Post-Cancun: no more self-destructs, preimage db is no longer needed. @@ -526,6 +520,26 @@ where "Execution time" ); + // Prepare the input for post execute commit hook, where an `ExExNotification` will be sent. + // + // Note: Since we only write to `blocks` if there are any ExExes, we don't need to perform + // the `has_exexs` check here as well + if !blocks.is_empty() { + let previous_input = self.post_execute_commit_input.replace(Chain::new( + blocks, + exex_state.unwrap_or(state), + BTreeMap::new(), + )); + + if previous_input.is_some() { + // Not processing the previous post execute commit input is a critical error, as it + // means that we didn't send the notification to ExExes + return Err(StageError::PostExecuteCommit( + "Previous post execute commit input wasn't processed", + )) + } + } + let done = stage_progress == max_block; Ok(ExecOutput { checkpoint: StageCheckpoint::new(stage_progress) From b4c4775946edcf3943399e39c2ef5e8879550af3 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Fri, 4 Sep 2026 17:06:22 +0000 Subject: [PATCH 170/294] perf(rpc): build txpool_contentFrom from the sender's transactions (#26969) --- crates/rpc/rpc/src/txpool.rs | 105 ++++++++++++++++++------ crates/transaction-pool/src/lib.rs | 7 ++ crates/transaction-pool/src/noop.rs | 7 ++ crates/transaction-pool/src/pool/mod.rs | 55 ++++++++++++- crates/transaction-pool/src/traits.rs | 10 +++ 5 files changed, 154 insertions(+), 30 deletions(-) diff --git a/crates/rpc/rpc/src/txpool.rs b/crates/rpc/rpc/src/txpool.rs index 51355dc1c12..850b4797bcf 100644 --- a/crates/rpc/rpc/src/txpool.rs +++ b/crates/rpc/rpc/src/txpool.rs @@ -10,7 +10,7 @@ use async_trait::async_trait; use jsonrpsee::core::RpcResult; use reth_primitives_traits::NodePrimitives; use reth_rpc_api::TxPoolApiServer; -use reth_rpc_convert::{RpcConvert, RpcTypes}; +use reth_rpc_convert::RpcConvert; use reth_rpc_eth_api::RpcTransaction; use reth_transaction_pool::{ AllPoolTransactions, PoolConsensusTx, PoolTransaction, TransactionPool, @@ -40,39 +40,50 @@ where Eth: RpcConvert>>, { fn content(&self) -> Result>, Eth::Error> { - #[inline] - fn insert( - tx: &Tx, - content: &mut BTreeMap< - Address, - BTreeMap::TransactionResponse>, - >, - resp_builder: &RpcTxB, - ) -> Result<(), RpcTxB::Error> - where - Tx: PoolTransaction, - RpcTxB: RpcConvert>, - { - content.entry(tx.sender()).or_default().insert( - tx.nonce().to_string(), - resp_builder.fill_pending(tx.clone_into_consensus())?, - ); - - Ok(()) - } - let AllPoolTransactions { pending, queued } = self.pool.all_transactions(); let mut content = TxpoolContent::default(); - for pending in pending { - insert::<_, Eth>(&pending.transaction, &mut content.pending, &self.converter)?; + for tx in pending { + let sender = tx.transaction.sender(); + self.insert_by_nonce(&tx.transaction, content.pending.entry(sender).or_default())?; + } + for tx in queued { + let sender = tx.transaction.sender(); + self.insert_by_nonce(&tx.transaction, content.queued.entry(sender).or_default())?; + } + + Ok(content) + } + + fn content_from( + &self, + from: Address, + ) -> Result>, Eth::Error> { + let mut content = TxpoolContentFrom::default(); + // one snapshot for both sides, so a transaction moving between sub-pools meanwhile is + // reported on exactly one of them + let AllPoolTransactions { pending, queued } = self.pool.all_transactions_by_sender(from); + for tx in pending { + self.insert_by_nonce(&tx.transaction, &mut content.pending)?; } - for queued in queued { - insert::<_, Eth>(&queued.transaction, &mut content.queued, &self.converter)?; + for tx in queued { + self.insert_by_nonce(&tx.transaction, &mut content.queued)?; } Ok(content) } + + /// Converts the pool transaction and inserts it into the given map, keyed by its nonce. + #[inline] + fn insert_by_nonce( + &self, + tx: &Pool::Transaction, + txs: &mut BTreeMap>, + ) -> Result<(), Eth::Error> { + txs.insert(tx.nonce().to_string(), self.converter.fill_pending(tx.clone_into_consensus())?); + + Ok(()) + } } #[async_trait] @@ -135,7 +146,7 @@ where from: Address, ) -> RpcResult>> { trace!(target: "rpc::eth", ?from, "Serving txpool_contentFrom"); - Ok(self.content().map_err(Into::into)?.remove_from(&from)) + Ok(self.content_from(from).map_err(Into::into)?) } /// Returns the details of all transactions currently pending for inclusion in the next @@ -154,3 +165,43 @@ impl fmt::Debug for TxPoolApi { f.debug_struct("TxpoolApi").finish_non_exhaustive() } } + +#[cfg(test)] +mod tests { + use super::*; + use crate::eth::helpers::types::EthRpcConverter; + use reth_chainspec::MAINNET; + use reth_rpc_eth_types::receipt::EthReceiptConverter; + use reth_transaction_pool::{ + test_utils::{testing_pool, MockTransaction}, + TransactionOrigin, + }; + + #[tokio::test] + async fn content_from_matches_content() { + let senders = [Address::with_last_byte(1), Address::with_last_byte(2)]; + + let pool = testing_pool(); + for sender in senders { + // nonces 0 and 1 end up in the pending sub-pool, the gapped nonce 9 in a parked one + for nonce in [0, 1, 9] { + let tx = MockTransaction::legacy() + .with_sender(sender) + .with_nonce(nonce) + .with_gas_price(100); + pool.add_transaction(TransactionOrigin::External, tx).await.unwrap(); + } + } + + let api = + TxPoolApi::new(pool, EthRpcConverter::new(EthReceiptConverter::new(MAINNET.clone()))); + let mut content = api.content().unwrap(); + // guards the fixture: both sub-pools must be populated for the comparison to mean anything + assert_eq!((content.pending.len(), content.queued.len()), (senders.len(), senders.len())); + + // the unknown sender must yield the same empty result as the whole-pool path + for sender in senders.into_iter().chain([Address::with_last_byte(3)]) { + assert_eq!(api.content_from(sender).unwrap(), content.remove_from(&sender)); + } + } +} diff --git a/crates/transaction-pool/src/lib.rs b/crates/transaction-pool/src/lib.rs index f1c99b1571a..90ef4a14029 100644 --- a/crates/transaction-pool/src/lib.rs +++ b/crates/transaction-pool/src/lib.rs @@ -641,6 +641,13 @@ where self.pool.all_transactions() } + fn all_transactions_by_sender( + &self, + sender: Address, + ) -> AllPoolTransactions { + self.pool.all_transactions_by_sender(sender) + } + fn all_transaction_hashes(&self) -> Vec { self.pool.all_transaction_hashes() } diff --git a/crates/transaction-pool/src/noop.rs b/crates/transaction-pool/src/noop.rs index cca97f611b4..10341ee5117 100644 --- a/crates/transaction-pool/src/noop.rs +++ b/crates/transaction-pool/src/noop.rs @@ -220,6 +220,13 @@ impl TransactionPool for NoopTransactionPool { AllPoolTransactions::default() } + fn all_transactions_by_sender( + &self, + _sender: Address, + ) -> AllPoolTransactions { + AllPoolTransactions::default() + } + fn all_transaction_hashes(&self) -> Vec { vec![] } diff --git a/crates/transaction-pool/src/pool/mod.rs b/crates/transaction-pool/src/pool/mod.rs index 6d1139a4c18..b4a3897436d 100644 --- a/crates/transaction-pool/src/pool/mod.rs +++ b/crates/transaction-pool/src/pool/mod.rs @@ -1055,6 +1055,20 @@ where } } + /// Returns all transactions of the given sender, collected under a single read guard so that + /// both sides reflect the same pool state. + pub fn all_transactions_by_sender( + &self, + sender: Address, + ) -> AllPoolTransactions { + let Some(sender_id) = self.sender_id(&sender) else { return Default::default() }; + let pool = self.get_pool_data(); + AllPoolTransactions { + pending: pool.pending_txs_by_sender(sender_id), + queued: pool.queued_txs_by_sender(sender_id), + } + } + /// Returns _all_ transactions in the pool pub fn all_transaction_hashes(&self) -> Vec { self.get_pool_data().all().transactions_iter().map(|tx| *tx.hash()).collect() @@ -1668,13 +1682,48 @@ mod tests { use crate::{ blobstore::{BlobStore, InMemoryBlobStore, PooledBlobSidecar}, identifier::SenderId, - test_utils::{MockTransaction, TestPoolBuilder}, + test_utils::{testing_pool, MockTransaction, TestPoolBuilder}, validate::ValidTransaction, - BlockInfo, PoolConfig, SubPoolLimit, TransactionOrigin, TransactionValidationOutcome, U256, + BlockInfo, PoolConfig, SubPoolLimit, TransactionOrigin, TransactionPool, + TransactionPoolExt, TransactionValidationOutcome, ValidPoolTransaction, U256, }; + use alloy_consensus::Transaction; use alloy_eips::{eip4844::BlobTransactionSidecar, eip7594::BlobTransactionSidecarVariant}; use alloy_primitives::Address; - use std::{fs, path::PathBuf}; + use std::{fs, path::PathBuf, sync::Arc}; + + #[tokio::test] + async fn all_transactions_by_sender_across_reclassification() { + let pool = testing_pool(); + let sender = Address::with_last_byte(1); + // nonces 0 and 1 are pending, the gapped nonce 9 is queued + for nonce in [0, 1, 9] { + let tx = + MockTransaction::legacy().with_sender(sender).with_nonce(nonce).with_gas_price(100); + pool.add_transaction(TransactionOrigin::External, tx).await.unwrap(); + } + let nonces = |txs: &[Arc>]| { + let mut nonces: Vec<_> = txs.iter().map(|tx| tx.transaction.nonce()).collect(); + nonces.sort_unstable(); + nonces + }; + + let txs = pool.all_transactions_by_sender(sender); + assert_eq!(nonces(&txs.pending), [0, 1]); + assert_eq!(nonces(&txs.queued), [9]); + + // a higher base fee reclassifies the pending transactions as queued; the snapshot must + // report each of them on exactly one side + pool.set_block_info(BlockInfo { + pending_basefee: 200, + block_gas_limit: 30_000_000, + ..Default::default() + }); + let txs = pool.all_transactions_by_sender(sender); + assert!(txs.pending.is_empty()); + assert_eq!(nonces(&txs.queued), [0, 1, 9]); + assert_eq!(nonces(&pool.get_transactions_by_sender(sender)), [0, 1, 9]); + } #[test] fn test_discard_blobs_on_blob_tx_eviction() { diff --git a/crates/transaction-pool/src/traits.rs b/crates/transaction-pool/src/traits.rs index ca20e7fa4d7..4a994829575 100644 --- a/crates/transaction-pool/src/traits.rs +++ b/crates/transaction-pool/src/traits.rs @@ -458,6 +458,16 @@ pub trait TransactionPool: Clone + Debug + Send + Sync { /// Consumer: RPC fn all_transactions(&self) -> AllPoolTransactions; + /// Returns all transactions of the given sender that are currently in the pool, grouped by + /// whether they are ready for inclusion in the next block or not. + /// + /// Both groups are collected from one snapshot of the pool, so a transaction that is moved + /// between sub-pools concurrently shows up in exactly one of them. + /// + /// Consumer: RPC + fn all_transactions_by_sender(&self, sender: Address) + -> AllPoolTransactions; + /// Returns the _hashes_ of all transactions regardless of whether they can be propagated or /// not. /// From bef4922b92be2185dc0b010608f5a5ece735d112 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Fri, 4 Sep 2026 19:27:28 +0200 Subject: [PATCH 171/294] perf(discv4): reject duplicate and unknown packets before recovering the key (#26935) --- Cargo.lock | 1 + crates/net/discv4/Cargo.toml | 5 ++ crates/net/discv4/benches/codec.rs | 71 ++++++++++++++++++++++++ crates/net/discv4/src/lib.rs | 88 ++++++++++++++++++++++++++++-- crates/net/discv4/src/proto.rs | 8 ++- 5 files changed, 165 insertions(+), 8 deletions(-) create mode 100644 crates/net/discv4/benches/codec.rs diff --git a/Cargo.lock b/Cargo.lock index 1cb020af96f..68f7b4d62c4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8178,6 +8178,7 @@ dependencies = [ "alloy-primitives", "alloy-rlp", "assert_matches", + "codspeed-criterion-compat", "discv5", "enr", "itertools 0.14.0", diff --git a/crates/net/discv4/Cargo.toml b/crates/net/discv4/Cargo.toml index fadda2b6348..dd2fb6f8496 100644 --- a/crates/net/discv4/Cargo.toml +++ b/crates/net/discv4/Cargo.toml @@ -40,6 +40,7 @@ itertools.workspace = true [dev-dependencies] secp256k1 = { workspace = true, features = ["rand"] } +criterion.workspace = true assert_matches.workspace = true rand_08.workspace = true tokio = { workspace = true, features = ["macros", "rt-multi-thread"] } @@ -58,3 +59,7 @@ serde = [ "reth-ethereum-forks/serde", ] test-utils = ["dep:rand_08"] + +[[bench]] +name = "codec" +harness = false diff --git a/crates/net/discv4/benches/codec.rs b/crates/net/discv4/benches/codec.rs new file mode 100644 index 00000000000..03db2187a9d --- /dev/null +++ b/crates/net/discv4/benches/codec.rs @@ -0,0 +1,71 @@ +//! Benchmarks for the discv4 wire codec. +//! +//! The receive path decodes every inbound datagram before any of the service level checks run, so +//! the cost measured here is paid once per packet, including for packets that are later dropped. + +#![allow(missing_docs)] + +use criterion::{criterion_group, criterion_main, Criterion}; +use reth_discv4::proto::{Message, Neighbours, NodeEndpoint, Ping}; +use reth_network_peers::NodeRecord; +use secp256k1::{SecretKey, SECP256K1}; +use std::{ + hint::black_box, + net::{IpAddr, Ipv4Addr}, +}; + +const fn endpoint(port: u16) -> NodeEndpoint { + NodeEndpoint { address: IpAddr::V4(Ipv4Addr::new(10, 0, 0, 1)), udp_port: port, tcp_port: port } +} + +const fn record(port: u16) -> NodeRecord { + NodeRecord { + address: IpAddr::V4(Ipv4Addr::new(10, 0, 0, 1)), + udp_port: port, + tcp_port: port, + id: alloy_primitives::FixedBytes::<64>::repeat_byte(port as u8), + } +} + +const fn ping() -> Message { + Message::Ping(Ping { + from: endpoint(30303), + to: endpoint(30304), + expire: u64::MAX, + enr_sq: Some(1), + }) +} + +/// A full neighbours response, the largest message on the wire in normal operation. +fn neighbours(len: usize) -> Message { + Message::Neighbours(Neighbours { + nodes: (0..len).map(|i| record(i as u16)).collect(), + expire: u64::MAX, + }) +} + +fn codec(c: &mut Criterion) { + let secret_key = SecretKey::new(&mut rand_08::thread_rng()); + let _ = SECP256K1; + + let mut group = c.benchmark_group("discv4_codec"); + + for (name, msg) in + [("ping", ping()), ("neighbours_4", neighbours(4)), ("neighbours_12", neighbours(12))] + { + let (encoded, _) = msg.encode(&secret_key); + + group.bench_function(format!("encode/{name}"), |b| { + b.iter(|| black_box(black_box(&msg).encode(black_box(&secret_key)))) + }); + + group.bench_function(format!("decode/{name}"), |b| { + b.iter(|| black_box(Message::decode(black_box(&encoded)).unwrap())) + }); + } + + group.finish(); +} + +criterion_group!(benches, codec); +criterion_main!(benches); diff --git a/crates/net/discv4/src/lib.rs b/crates/net/discv4/src/lib.rs index 3ef421ea628..aaebf273c2c 100644 --- a/crates/net/discv4/src/lib.rs +++ b/crates/net/discv4/src/lib.rs @@ -2086,6 +2086,18 @@ impl IngressHandler { return } + // A packet starts with the hash of everything that follows it, and `Message::decode` + // rejects any packet whose contents do not hash to it. A repeat of a packet we already + // accepted can therefore be recognised from those 32 bytes alone, without decoding. + // Decoding runs an ECDSA recovery, so checking here keeps a replayed packet from costing + // a signature verification. + if data.len() >= MIN_PACKET_SIZE && + self.cache.contains_packet(B256::from_slice(&data[..32])) + { + debug!(target: "discv4", ?src, "Received duplicate packet."); + return + } + let event = match Message::decode(data) { Ok(packet) => { if packet.node_id == self.local_id { @@ -2093,10 +2105,9 @@ impl IngressHandler { return } - if self.cache.contains_packet(packet.hash) { - debug!(target: "discv4", ?src, "Received duplicate packet."); - return - } + // Only packets that decoded are remembered, so a peer cannot suppress a packet we + // have not seen yet by guessing its hash. + self.cache.insert_packet(packet.hash); IngressEvent::Packet(src, packet) } @@ -2147,9 +2158,16 @@ impl ReceiveCache { *ctn } - /// Returns true if we previously received the packet + /// Returns true if we previously received the packet. + /// + /// A hit refreshes the entry so that a packet being replayed repeatedly stays cached. fn contains_packet(&mut self, hash: B256) -> bool { - !self.unique_packets.insert(hash, ()) + self.unique_packets.get(&hash).is_some() + } + + /// Remembers a packet we accepted. + fn insert_packet(&mut self, hash: B256) { + self.unique_packets.insert(hash, ()); } } @@ -2571,8 +2589,66 @@ mod tests { use rand_08::Rng; use reth_ethereum_forks::{EnrForkIdEntry, ForkHash}; use reth_network_peers::mainnet_nodes; + use secp256k1::SECP256K1; use std::future::poll_fn; + #[tokio::test] + async fn test_duplicate_packet_rejected_without_decoding() { + let secret_key = SecretKey::new(&mut rand_08::thread_rng()); + let local_id = pk2id(&secret_key.public_key(SECP256K1)); + let (tx, mut rx) = mpsc::channel(16); + let mut handler = IngressHandler::new(tx, local_id); + + let remote_key = SecretKey::new(&mut rand_08::thread_rng()); + let msg = Message::Ping(Ping { + from: rng_endpoint(&mut rand_08::thread_rng()), + to: rng_endpoint(&mut rand_08::thread_rng()), + expire: u64::MAX, + enr_sq: None, + }); + let (packet, hash) = msg.encode(&remote_key); + let src = "10.0.0.1:30303".parse().unwrap(); + + handler.handle_packet(&packet, src).await; + assert!(matches!(rx.try_recv(), Ok(IngressEvent::Packet(_, _)))); + + // the replay is dropped on the hash prefix alone + handler.handle_packet(&packet, src).await; + assert!(rx.try_recv().is_err()); + assert!(handler.cache.contains_packet(hash)); + } + + #[tokio::test] + async fn test_undecodable_packet_does_not_poison_cache() { + let secret_key = SecretKey::new(&mut rand_08::thread_rng()); + let local_id = pk2id(&secret_key.public_key(SECP256K1)); + let (tx, mut rx) = mpsc::channel(16); + let mut handler = IngressHandler::new(tx, local_id); + + let remote_key = SecretKey::new(&mut rand_08::thread_rng()); + let msg = Message::Ping(Ping { + from: rng_endpoint(&mut rand_08::thread_rng()), + to: rng_endpoint(&mut rand_08::thread_rng()), + expire: u64::MAX, + enr_sq: None, + }); + let (packet, hash) = msg.encode(&remote_key); + let src = "10.0.0.1:30303".parse().unwrap(); + + // a packet carrying the right hash prefix but a corrupt body must not be remembered, + // otherwise it could be used to suppress the real packet + let mut forged = packet.to_vec(); + let last = forged.len() - 1; + forged[last] ^= 0xff; + handler.handle_packet(&forged, src).await; + assert!(matches!(rx.try_recv(), Ok(IngressEvent::BadPacket(..)))); + assert!(!handler.cache.contains_packet(hash)); + + // so the genuine packet still gets through + handler.handle_packet(&packet, src).await; + assert!(matches!(rx.try_recv(), Ok(IngressEvent::Packet(_, _)))); + } + #[tokio::test] async fn test_configured_enr_forkid_entry() { let fork: ForkId = ForkId { hash: ForkHash([220, 233, 108, 45]), next: 0u64 }; diff --git a/crates/net/discv4/src/proto.rs b/crates/net/discv4/src/proto.rs index 2aabd45d99f..dff1601d97a 100644 --- a/crates/net/discv4/src/proto.rs +++ b/crates/net/discv4/src/proto.rs @@ -155,6 +155,11 @@ impl Message { return Err(DecodePacketError::HashMismatch) } + // Resolve the message type before recovering the public key: recovery is by far the most + // expensive step of decoding, and a packet we have no handler for is rejected either way. + let message_id = + MessageId::from_u8(packet[97]).map_err(DecodePacketError::UnknownMessage)?; + let signature = &packet[32..96]; let recovery_id = RecoveryId::try_from(packet[96] as i32)?; let recoverable_sig = RecoverableSignature::from_compact(signature, recovery_id)?; @@ -165,10 +170,9 @@ impl Message { let pk = SECP256K1.recover_ecdsa(&msg, &recoverable_sig)?; let node_id = pk2id(&pk); - let msg_type = packet[97]; let payload = &mut &packet[98..]; - let msg = match MessageId::from_u8(msg_type).map_err(DecodePacketError::UnknownMessage)? { + let msg = match message_id { MessageId::Ping => Self::Ping(Ping::decode(payload)?), MessageId::Pong => Self::Pong(Pong::decode(payload)?), MessageId::FindNode => Self::FindNode(FindNode::decode(payload)?), From dfd1505d8cb18e28ed2ea3b61dadda26e6747587 Mon Sep 17 00:00:00 2001 From: Derek Cofausper <256792747+decofe@users.noreply.github.com> Date: Fri, 4 Sep 2026 18:07:57 +0000 Subject: [PATCH 172/294] chore(net): trace duplicate discovery packets (#26933) Co-authored-by: Matthias Seitz <19890894+mattsse@users.noreply.github.com> --- crates/net/discv4/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/net/discv4/src/lib.rs b/crates/net/discv4/src/lib.rs index aaebf273c2c..d54ac82d4e0 100644 --- a/crates/net/discv4/src/lib.rs +++ b/crates/net/discv4/src/lib.rs @@ -2094,7 +2094,7 @@ impl IngressHandler { if data.len() >= MIN_PACKET_SIZE && self.cache.contains_packet(B256::from_slice(&data[..32])) { - debug!(target: "discv4", ?src, "Received duplicate packet."); + trace!(target: "discv4", ?src, "Received duplicate packet."); return } From 9c4573f0963c99b5b13f5c0f1b9750b46f10aed5 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Fri, 4 Sep 2026 18:54:15 +0000 Subject: [PATCH 173/294] test(cli): cover snapshot downloads with a mock HTTP server (#27009) Co-authored-by: Matthias Seitz <19890894+mattsse@users.noreply.github.com> --- Cargo.lock | 4 + bin/reth/Cargo.toml | 4 + bin/reth/tests/it/download/mod.rs | 213 ++++++++++++++++++++++++ bin/reth/tests/it/download/server.rs | 98 +++++++++++ bin/reth/tests/it/main.rs | 2 + crates/cli/commands/src/download/mod.rs | 5 + 6 files changed, 326 insertions(+) create mode 100644 bin/reth/tests/it/download/mod.rs create mode 100644 bin/reth/tests/it/download/server.rs diff --git a/Cargo.lock b/Cargo.lock index 68f7b4d62c4..f8cd5633ed9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7663,7 +7663,11 @@ dependencies = [ "alloy-rpc-types-eth", "aquamarine", "backon", + "blake3", "clap", + "http-body-util", + "hyper", + "hyper-util", "reth-chainspec", "reth-cli-runner", "reth-cli-util", diff --git a/bin/reth/Cargo.toml b/bin/reth/Cargo.toml index cc83d1b4265..320dbddfb1a 100644 --- a/bin/reth/Cargo.toml +++ b/bin/reth/Cargo.toml @@ -75,6 +75,10 @@ alloy-node-bindings.workspace = true alloy-provider = { workspace = true, features = ["reqwest"] } alloy-rpc-types-eth.workspace = true backon.workspace = true +blake3.workspace = true +http-body-util.workspace = true +hyper = { workspace = true, features = ["server", "http1"] } +hyper-util = { workspace = true, features = ["tokio"] } serde_json.workspace = true tar.workspace = true tempfile.workspace = true diff --git a/bin/reth/tests/it/download/mod.rs b/bin/reth/tests/it/download/mod.rs new file mode 100644 index 00000000000..a29d799f7de --- /dev/null +++ b/bin/reth/tests/it/download/mod.rs @@ -0,0 +1,213 @@ +mod server; + +use self::server::SnapshotServer; +use super::{reth_ok, RETH}; +use serde_json::{json, Value}; +use std::{ + collections::BTreeMap, + fs, + path::Path, + process::{Command, Output}, +}; + +/// Tiny transport fixtures: real tar/zstd archives and checksums, with distinct payloads +/// standing in for database/static files. These test download preparation, not node sync. +struct Snapshot { + files: BTreeMap>, + manifest: Value, +} + +impl Snapshot { + fn new() -> Self { + let mut files = BTreeMap::new(); + let mut entry = |archive: &str, path: &str| { + let data = archive.as_bytes(); + let bytes = archive_bytes(path, data); + let metadata = json!({ + "file": archive, "size": bytes.len(), + "output_files": [{"path": path, "size": data.len(), "blake3": blake3::hash(data).to_hex().to_string()}] + }); + files.insert(format!("/snapshot/{archive}"), bytes); + metadata + }; + let state = entry("state.tar.zst", "db/snapshot"); + let headers = entry("headers.tar.zst", "static_files/headers"); + let chunks = [ + entry("txs-0.tar.zst", "static_files/txs-0"), + entry("txs-1.tar.zst", "static_files/txs-1"), + ]; + let manifest = json!({ + "block": 19, "chain_id": 1, "storage_version": 2, "timestamp": 0, + "components": { + "state": state, "headers": headers, + "transactions": { + "blocks_per_file": 10, "total_blocks": 20, + "chunk_files": ["txs-0.tar.zst", "txs-1.tar.zst"], + "chunk_sizes": [chunks[0]["size"], chunks[1]["size"]], + "chunk_output_files": [chunks[0]["output_files"], chunks[1]["output_files"]] + } + } + }); + Self { files, manifest } + } + + async fn serve(mut self, ranges: bool) -> SnapshotServer { + self.files + .insert("/snapshot/manifest.json".into(), serde_json::to_vec(&self.manifest).unwrap()); + SnapshotServer::start(self.files, ranges).await + } +} + +fn archive_bytes(path: &str, data: &[u8]) -> Vec { + let mut archive = tar::Builder::new(zstd::Encoder::new(Vec::new(), 0).unwrap()); + let mut header = tar::Header::new_gnu(); + header.set_size(data.len() as u64); + header.set_mode(0o644); + header.set_cksum(); + archive.append_data(&mut header, path, data).unwrap(); + archive.into_inner().unwrap().finish().unwrap() +} + +fn download(server: &SnapshotServer, datadir: &Path, args: &[&str]) -> Output { + Command::new(RETH) + .env("RUST_LOG", "off") + .args(["download", "--datadir"]) + .arg(datadir) + .args([ + "--manifest-url", + &format!("{}/snapshot/manifest.json", server.url), + "--non-interactive", + ]) + .args(args) + .output() + .unwrap() +} + +#[track_caller] +fn success(output: Output) -> String { + assert!( + output.status.success(), + "stdout: {}\nstderr: {}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + String::from_utf8(output.stdout).unwrap() +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn manifest_selection_reuse_and_repair() { + let snapshot = Snapshot::new(); + let state = &snapshot.files["/snapshot/state.tar.zst"]; + let offset = state.len() / 2; + let dir = tempfile::tempdir().unwrap(); + fs::create_dir(dir.path().join(".download-cache")).unwrap(); + fs::write(dir.path().join(".download-cache/state.tar.zst.part"), &state[..offset]).unwrap(); + let server = snapshot.serve(true).await; + let args = ["--with-txs-since", "10"]; + success(download(&server, dir.path(), &args)); + for (path, expected) in [ + ("db/snapshot", "state.tar.zst"), + ("static_files/headers", "headers.tar.zst"), + ("static_files/txs-1", "txs-1.tar.zst"), + ] { + assert_eq!(fs::read(dir.path().join(path)).unwrap(), expected.as_bytes()); + } + assert!(server.requests().iter().any(|(path, range)| { + path.ends_with("state.tar.zst") && range.as_deref() == Some(&format!("bytes={offset}-")) + })); + assert!(!dir.path().join("static_files/txs-0").exists()); + assert!(!server.requests().iter().any(|(path, _)| path.ends_with("txs-0.tar.zst"))); + let config: toml::Value = + toml::from_str(&fs::read_to_string(dir.path().join("reth.toml")).unwrap()).unwrap(); + assert_eq!(config["prune"]["segments"]["bodies_history"]["before"].as_integer(), Some(10)); + + server.clear_requests(); + success(download(&server, dir.path(), &args)); + assert_eq!(server.requests(), vec![("/snapshot/manifest.json".into(), None)]); + + // Same-size corruption must be detected by checksum, not just the file length. + fs::write(dir.path().join("static_files/txs-1"), b"bad-1.tar.zst").unwrap(); + server.clear_requests(); + success(download(&server, dir.path(), &args)); + assert_eq!(fs::read(dir.path().join("static_files/txs-1")).unwrap(), b"txs-1.tar.zst"); + assert!(server.requests().iter().any(|(path, _)| path.ends_with("txs-1.tar.zst"))); + assert!(server + .requests() + .iter() + .all(|(path, _)| path.ends_with("manifest.json") || path.ends_with("txs-1.tar.zst"))); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn plan_does_not_download_or_create_datadir() { + let server = Snapshot::new().serve(true).await; + let dir = tempfile::tempdir().unwrap(); + let target = dir.path().join("new"); + let output = success(download(&server, &target, &["--archive", "--print-plan-json"])); + let plan: Value = serde_json::from_str(&output).unwrap(); + assert_eq!(plan["archives"].as_array().unwrap().len(), 4); + assert!(!target.exists()); + assert_eq!(server.requests(), vec![("/snapshot/manifest.json".into(), None)]); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn legacy_streaming_and_resume() { + for (resumable, ranges) in [(false, false), (true, true), (true, false)] { + let bytes = archive_bytes("db/snapshot", b"snapshot contents"); + let offset = bytes.len() / 2; + let server = SnapshotServer::start( + BTreeMap::from([("/state.tar.zst".into(), bytes.clone())]), + ranges, + ) + .await; + let dir = tempfile::tempdir().unwrap(); + if resumable { + fs::write(dir.path().join("state.tar.zst.part"), &bytes[..offset]).unwrap(); + } + reth_ok(&[ + "download", + "--url", + &format!("{}/state.tar.zst", server.url), + "--datadir", + dir.path().to_str().unwrap(), + if resumable { "--resumable=true" } else { "--resumable=false" }, + ]); + assert_eq!(fs::read(dir.path().join("db/snapshot")).unwrap(), b"snapshot contents"); + assert!(!dir.path().join("state.tar.zst.part").exists()); + assert!(!dir.path().join("state.tar.zst").exists()); + if resumable { + assert!(server + .requests() + .iter() + .any(|(_, range)| range.as_deref() == Some(&format!("bytes={offset}-")))); + } + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn wrong_chain_is_rejected_before_force_removes_data() { + let mut snapshot = Snapshot::new(); + snapshot.manifest["chain_id"] = json!(11155111); + let server = snapshot.serve(true).await; + let dir = tempfile::tempdir().unwrap(); + fs::create_dir(dir.path().join("db")).unwrap(); + fs::write(dir.path().join("db/existing"), b"keep").unwrap(); + let output = download(&server, dir.path(), &["--force"]); + assert!(!output.status.success(), "a Sepolia snapshot must not be installed for mainnet"); + assert!(String::from_utf8_lossy(&output.stderr).contains("chain ID")); + assert_eq!(fs::read(dir.path().join("db/existing")).unwrap(), b"keep"); + assert_eq!(server.requests(), vec![("/snapshot/manifest.json".into(), None)]); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn bad_output_checksum_prevents_finalization() { + let mut snapshot = Snapshot::new(); + snapshot.manifest["components"]["state"]["output_files"][0]["blake3"] = + json!(blake3::hash(b"wrong").to_hex().to_string()); + let server = snapshot.serve(true).await; + let dir = tempfile::tempdir().unwrap(); + let output = download(&server, dir.path(), &["--with-txs-since", "20"]); + assert!(!output.status.success()); + assert!(String::from_utf8_lossy(&output.stderr).contains("Failed integrity validation")); + assert!(!dir.path().join("reth.toml").exists()); + assert!(!dir.path().join("db/mdbx.dat").exists()); +} diff --git a/bin/reth/tests/it/download/server.rs b/bin/reth/tests/it/download/server.rs new file mode 100644 index 00000000000..e2a60d22306 --- /dev/null +++ b/bin/reth/tests/it/download/server.rs @@ -0,0 +1,98 @@ +use http_body_util::Full; +use hyper::{body::Bytes, server::conn::http1, service::service_fn, Request, Response, StatusCode}; +use hyper_util::rt::TokioIo; +use std::{ + collections::BTreeMap, + convert::Infallible, + sync::{Arc, Mutex}, +}; +use tokio::{net::TcpListener, task::JoinHandle}; + +/// Loopback snapshot server with optional byte-range support and a request log. +/// Each test owns its server, so ports and responses are isolated across parallel runs. +pub(super) struct SnapshotServer { + pub(super) url: String, + requests: Arc>>, + task: JoinHandle<()>, +} + +impl SnapshotServer { + pub(super) async fn start(files: BTreeMap>, ranges: bool) -> Self { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let url = format!("http://{}", listener.local_addr().unwrap()); + let requests = Arc::new(Mutex::new(Vec::new())); + let log = Arc::clone(&requests); + let files = Arc::new(files); + let task = tokio::spawn(async move { + while let Ok((stream, _)) = listener.accept().await { + let files = Arc::clone(&files); + let log = Arc::clone(&log); + tokio::spawn(async move { + let service = service_fn(move |request: Request| { + let path = request.uri().path().to_owned(); + let range = + request.headers().get("range").map(|v| v.to_str().unwrap().to_owned()); + log.lock().unwrap().push((path.clone(), range.clone())); + let mut response = Response::builder(); + let body = if let Some(data) = files.get(&path) { + let mut bytes = data.as_slice(); + if let Some(range) = range.filter(|_| ranges) { + let (start, end) = + range.strip_prefix("bytes=").unwrap().split_once('-').unwrap(); + let start: usize = start.parse().unwrap(); + let end = end + .parse::() + .unwrap_or(data.len() - 1) + .min(data.len() - 1); + if start >= data.len() { + response = response + .status(StatusCode::RANGE_NOT_SATISFIABLE) + .header("content-range", format!("bytes */{}", data.len())); + bytes = &[]; + } else { + response = response.status(StatusCode::PARTIAL_CONTENT).header( + "content-range", + format!("bytes {start}-{end}/{}", data.len()), + ); + bytes = &data[start..=end]; + } + } + response = response.header("content-length", bytes.len()); + if request.method() == hyper::Method::HEAD { + Bytes::new() + } else { + Bytes::copy_from_slice(bytes) + } + } else { + response = response.status(StatusCode::NOT_FOUND); + Bytes::new() + }; + std::future::ready(Ok::<_, Infallible>( + response.body(Full::new(body)).unwrap(), + )) + }); + // Range probes may close the connection without consuming the body. + let _ = + http1::Builder::new().serve_connection(TokioIo::new(stream), service).await; + }); + } + }); + Self { url, requests, task } + } + + pub(super) fn requests(&self) -> Vec { + self.requests.lock().unwrap().clone() + } + + pub(super) fn clear_requests(&self) { + self.requests.lock().unwrap().clear(); + } +} + +impl Drop for SnapshotServer { + fn drop(&mut self) { + self.task.abort(); + } +} + +type RecordedRequest = (String, Option); diff --git a/bin/reth/tests/it/main.rs b/bin/reth/tests/it/main.rs index d71390501d6..68e0520d2c4 100644 --- a/bin/reth/tests/it/main.rs +++ b/bin/reth/tests/it/main.rs @@ -2,6 +2,8 @@ use std::{fs, path::Path, process::Command}; +mod download; + const RETH: &str = env!("CARGO_BIN_EXE_reth"); // ── Helpers ────────────────────────────────────────────────────────────────── diff --git a/crates/cli/commands/src/download/mod.rs b/crates/cli/commands/src/download/mod.rs index 7cd8de7f42d..aeab3e04d45 100644 --- a/crates/cli/commands/src/download/mod.rs +++ b/crates/cli/commands/src/download/mod.rs @@ -579,6 +579,11 @@ impl> DownloadCo info!(target: "reth::cli", source = %manifest_source, "Fetching snapshot manifest"); let mut manifest = fetch_manifest_from_source(&manifest_source).await?; + eyre::ensure!( + manifest.chain_id == chain_id, + "Snapshot chain ID {} does not match selected chain ID {chain_id}", + manifest.chain_id + ); manifest.base_url = Some(resolve_manifest_base_url(&manifest, &manifest_source)?); info!(target: "reth::cli", From f5648cfd953c1d4d8a48552e292c361f0f41f3f5 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Sat, 5 Sep 2026 04:03:17 +0200 Subject: [PATCH 174/294] fix(rpc): return -32000 for unknown filter ids (#27020) --- crates/rpc/rpc/src/eth/filter.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/crates/rpc/rpc/src/eth/filter.rs b/crates/rpc/rpc/src/eth/filter.rs index 025e9d247cf..7a60f2ee743 100644 --- a/crates/rpc/rpc/src/eth/filter.rs +++ b/crates/rpc/rpc/src/eth/filter.rs @@ -1050,8 +1050,9 @@ pub enum EthFilterError { impl From for jsonrpsee::types::error::ErrorObject<'static> { fn from(err: EthFilterError) -> Self { match err { + // geth and Nethermind answer -32000 for unknown filter ids EthFilterError::FilterNotFound(_) => rpc_error_with_code( - jsonrpsee::types::error::INVALID_PARAMS_CODE, + jsonrpsee::types::error::CALL_EXECUTION_FAILED_CODE, "filter not found", ), err @ EthFilterError::InternalError => { From 5d6634a6f1aef5f770ce9ccba7009be111efdfb8 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Sat, 5 Sep 2026 15:12:46 +0200 Subject: [PATCH 175/294] fix(rpc): return tracing error for missing trace call blocks (#27022) --- crates/rpc/rpc/src/debug.rs | 57 +++++++++++++++++++++++++++++++++++-- 1 file changed, 54 insertions(+), 3 deletions(-) diff --git a/crates/rpc/rpc/src/debug.rs b/crates/rpc/rpc/src/debug.rs index c0739108cab..1154f793839 100644 --- a/crates/rpc/rpc/src/debug.rs +++ b/crates/rpc/rpc/src/debug.rs @@ -29,7 +29,7 @@ use reth_rpc_api::DebugApiServer; use reth_rpc_convert::RpcTxReq; use reth_rpc_eth_api::{ helpers::{EthTransactions, TraceExt}, - FromEthApiError, FromEvmError, RpcConvert, RpcNodeCore, + AsEthApiError, FromEthApiError, FromEvmError, RpcConvert, RpcNodeCore, }; use reth_rpc_eth_types::{EthApiError, StateCacheDb}; use reth_rpc_server_types::{result::internal_rpc_err, ToRpcResult}; @@ -411,6 +411,13 @@ where Ok(trace) }) .await + .map_err(|err| match err.as_err() { + Some(EthApiError::HeaderNotFound(id)) if *id == at => { + // Unknown blocks use -32000: https://github.com/ethereum/execution-apis/pull/855 + EthApiError::TracingBlockNotFound(at).into() + } + _ => err, + }) } /// Helper method to execute `debug_trace_call` at a specific transaction index within a block. @@ -429,7 +436,7 @@ where .eth_api() .recovered_block_and_maybe_bal(block_id) .await? - .ok_or(EthApiError::HeaderNotFound(block_id))?; + .ok_or(EthApiError::TracingBlockNotFound(block_id))?; if tx_index >= block.transaction_count() { // tx_index out of bounds @@ -1495,15 +1502,59 @@ impl Default for BadBlockStore { #[cfg(test)] mod tests { use super::*; + use crate::{eth::helpers::types::EthRpcConverter, EthApi}; use alloy_primitives::{keccak256, U256}; + use reth_chainspec::ChainSpec; use reth_db_api::{tables, transaction::DbTxMut}; + use reth_evm_ethereum::EthEvmConfig; + use reth_network_api::noop::NoopNetwork; use reth_primitives_traits::StorageEntry; - use reth_provider::test_utils::create_test_provider_factory; + use reth_provider::test_utils::{create_test_provider_factory, NoopProvider}; + use reth_rpc_eth_api::EthApiServer; + use reth_transaction_pool::test_utils::testing_pool; use revm::{ database::{states::StorageSlot, AccountStatus, BundleAccount, BundleState}, state::AccountInfo as RevmAccountInfo, }; + #[tokio::test] + async fn trace_call_out_of_range_block_error() { + let eth_api = EthApi::<_, EthRpcConverter>::builder( + NoopProvider::default(), + testing_pool(), + NoopNetwork::default(), + EthEvmConfig::mainnet(), + ) + .build(); + let debug_api = DebugApi::new( + eth_api.clone(), + BlockingTaskGuard::new(1), + &Runtime::test(), + futures::stream::empty(), + ); + let block_id = BlockId::number(0xfffffffff); + for tx_index in [None, Some(0)] { + let mut opts: GethDebugTracingCallOptions = + serde_json::from_value(serde_json::json!({ "tracer": "callTracer" })).unwrap(); + opts.tx_index = tx_index; + let err = DebugApiServer::debug_trace_call( + &debug_api, + Default::default(), + Some(block_id), + Some(opts), + ) + .await + .unwrap_err(); + assert_eq!(err.code(), -32000); + assert!(err.message().contains("not found")); + } + + let err = EthApiServer::call(ð_api, Default::default(), Some(block_id), None, None) + .await + .unwrap_err(); + assert_eq!(err.code(), -32001); + } + #[test] fn hashed_post_state_zeroes_destroyed_account_parent_storage() { let factory = create_test_provider_factory(); From 4f99d25508c3c3b8164e832a4cf1e664ff88c480 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Sat, 5 Sep 2026 15:30:49 +0200 Subject: [PATCH 176/294] fix(rpc): include simulated block BAL hash (#27023) --- crates/ethereum/node/tests/e2e/simulate.rs | 60 ++++++++++++++++++++++ crates/rpc/rpc-eth-api/src/helpers/call.rs | 12 ++++- 2 files changed, 71 insertions(+), 1 deletion(-) diff --git a/crates/ethereum/node/tests/e2e/simulate.rs b/crates/ethereum/node/tests/e2e/simulate.rs index 7e6b04af72d..19f0d7290dd 100644 --- a/crates/ethereum/node/tests/e2e/simulate.rs +++ b/crates/ethereum/node/tests/e2e/simulate.rs @@ -12,6 +12,66 @@ use reth_e2e_test_utils::setup_engine; use reth_node_ethereum::EthereumNode; use std::sync::Arc; +#[tokio::test] +async fn test_simulate_v1_block_access_list_hash_across_amsterdam() -> eyre::Result<()> { + reth_tracing::init_test_tracing(); + let chain_spec = Arc::new( + ChainSpecBuilder::default() + .chain(MAINNET.chain) + .genesis(serde_json::from_str(include_str!("../assets/genesis.json"))?) + .osaka_activated() + .with_amsterdam_at(24) + .build(), + ); + let (mut nodes, _) = setup_engine::( + 1, + chain_spec, + false, + Default::default(), + eth_payload_attributes_amsterdam, + ) + .await?; + let node = nodes.pop().unwrap(); + let provider = ProviderBuilder::new().connect_http(node.rpc_url()); + let from = Address::with_last_byte(0x41); + let contract = Address::with_last_byte(0x42); + let state_overrides = StateOverride::from_iter([ + (from, AccountOverride::default().with_nonce(0)), + (contract, AccountOverride::default().with_code(bytes!("0x5f545f5260205ff3"))), + ]); + let call_block = SimBlock::default() + .with_state_overrides(state_overrides) + .call(TransactionRequest::default().from(from).to(contract)); + + for trace_transfers in [false, true] { + let mut payload = SimulatePayload::default() + .extend(SimBlock::default()) + .extend(call_block.clone()) + .extend(call_block.clone()) + .extend(SimBlock::default()); + payload.trace_transfers = trace_transfers; + let result: Vec = + provider.raw_request("eth_simulateV1".into(), (&payload, "latest")).await?; + assert_eq!(result.len(), 4); + assert_eq!(result[0].inner.header.block_access_list_hash, None); + let bal_hash = result[1].inner.header.block_access_list_hash.expect("Amsterdam BAL hash"); + assert!(result[1].calls[0].status); + assert!(result[2].calls[0].status); + // Identical calls with the same initial state must reset their BAL indices each block. + assert_eq!(result[2].inner.header.block_access_list_hash, Some(bal_hash)); + let empty_hash = + result[3].inner.header.block_access_list_hash.expect("empty block BAL hash"); + assert_ne!(bal_hash, empty_hash); + for pair in result.windows(2) { + assert_eq!(pair[1].inner.header.parent_hash, pair[0].inner.header.hash); + } + for block in &result { + assert_eq!(block.inner.header.hash, block.inner.header.inner.hash_slow()); + } + } + Ok(()) +} + #[tokio::test] async fn test_simulate_v1_explicit_gas_uses_remaining_block_gas() -> eyre::Result<()> { reth_tracing::init_test_tracing(); diff --git a/crates/rpc/rpc-eth-api/src/helpers/call.rs b/crates/rpc/rpc-eth-api/src/helpers/call.rs index e9bf0b16da8..3cbe6c5e3d5 100644 --- a/crates/rpc/rpc-eth-api/src/helpers/call.rs +++ b/crates/rpc/rpc-eth-api/src/helpers/call.rs @@ -29,7 +29,10 @@ use reth_primitives_traits::Recovered; use reth_revm::{ cancelled::CancelOnDrop, database::StateProviderDatabase, - db::{bal::EvmDatabaseError, State}, + db::{ + bal::{BalState, EvmDatabaseError}, + State, + }, }; use reth_rpc_convert::{RpcConvert, RpcTxReq}; use reth_rpc_eth_types::{ @@ -190,6 +193,13 @@ pub trait EthCall: EstimateCall + Call + LoadPendingBlock + LoadBlock + FullEthA let chain_id = evm_env.cfg_env.chain_id; + // Each simulated block needs its own BAL, including when crossing Amsterdam. + if this.provider().chain_spec().is_amsterdam_active_at_timestamp( + evm_env.block_env.timestamp().saturating_to(), + ) { + db.bal_state = BalState::new().with_bal_builder(); + } + let ctx = this .evm_config() .context_for_next_block(&parent, attributes) From c216789600c814cc6864e5098a4bfe5263f1fd57 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Sat, 5 Sep 2026 16:19:00 +0200 Subject: [PATCH 177/294] fix(engine): count inclusion list transaction bytes without RLP framing (#27024) --- crates/rpc/rpc-engine-api/src/engine_api.rs | 67 +++++++++++++++++++-- 1 file changed, 62 insertions(+), 5 deletions(-) diff --git a/crates/rpc/rpc-engine-api/src/engine_api.rs b/crates/rpc/rpc-engine-api/src/engine_api.rs index a012836b3a3..4f300225d7b 100644 --- a/crates/rpc/rpc-engine-api/src/engine_api.rs +++ b/crates/rpc/rpc-engine-api/src/engine_api.rs @@ -485,10 +485,8 @@ where for pool_tx in self.inner.tx_pool.best_transactions().without_blobs().without_updates() { let encoded = pool_tx.encoded_2718_consensus(); - let new_size = total_size + alloy_rlp::Encodable::length(&encoded); - if new_size + alloy_rlp::length_of_length(new_size) > - MAX_BYTES_PER_INCLUSION_LIST as usize - { + let new_size = total_size + encoded.len(); + if new_size > MAX_BYTES_PER_INCLUSION_LIST as usize { break } @@ -1911,10 +1909,69 @@ mod tests { let res = EngineApiServer::get_inclusion_list_v1(&api).await.unwrap(); assert_eq!(res, vec![expected]); assert!( - alloy_rlp::list_length::(&res) <= MAX_BYTES_PER_INCLUSION_LIST as usize + res.iter().map(|tx| tx.len()).sum::() <= MAX_BYTES_PER_INCLUSION_LIST as usize ); } + /// Signs candidate transactions because signature lengths affect the final encoded size. + fn inclusion_list_transaction(size: usize, typed: bool, fee: u128) -> EthPooledTransaction { + for input_size in size.saturating_sub(256)..size { + let builder = TransactionBuilder::default() + .signer(B256::with_last_byte(if typed { 2 } else { 1 })) + .max_fee_per_gas(fee) + .max_priority_fee_per_gas(fee) + .input(vec![0; input_size]); + let transaction = if typed { builder.into_eip1559() } else { builder.into_legacy() }; + if transaction.encode_2718_len() == size { + return pooled_transaction(transaction) + } + } + panic!("could not construct a transaction of {size} bytes") + } + + #[tokio::test] + async fn get_inclusion_list_v1_transaction_byte_boundaries() { + let limit = MAX_BYTES_PER_INCLUSION_LIST as usize; + for typed in [false, true] { + for size in [limit - 3, limit, limit + 1] { + let pool = eth_test_pool(); + let transaction = inclusion_list_transaction(size, typed, 3_000_000_000); + let encoded = transaction.encoded_2718_consensus(); + assert_eq!(encoded.len(), size); + pool.add_transaction(TransactionOrigin::External, transaction).await.unwrap(); + let (_, api) = setup_engine_api_with_pool(pool); + + let res = EngineApiServer::get_inclusion_list_v1(&api).await.unwrap(); + let expected = if size <= limit { vec![encoded] } else { vec![] }; + assert_eq!(res, expected, "typed={typed}, size={size}"); + assert!(res.iter().all(|tx| !tx.is_empty())); + } + } + } + + #[tokio::test] + async fn get_inclusion_list_v1_cumulative_transaction_byte_boundaries() { + let limit = MAX_BYTES_PER_INCLUSION_LIST as usize; + for extra in [0, 1] { + let pool = eth_test_pool(); + let first = inclusion_list_transaction(limit / 2, false, 3_000_000_000); + let second = inclusion_list_transaction(limit / 2 + extra, true, 2_000_000_000); + let mut expected = + vec![first.encoded_2718_consensus(), second.encoded_2718_consensus()]; + assert_eq!(expected.iter().map(|tx| tx.len()).sum::(), limit + extra); + if extra != 0 { + expected.pop(); + } + pool.add_transaction(TransactionOrigin::External, first).await.unwrap(); + pool.add_transaction(TransactionOrigin::External, second).await.unwrap(); + let (_, api) = setup_engine_api_with_pool(pool); + + let res = EngineApiServer::get_inclusion_list_v1(&api).await.unwrap(); + assert_eq!(res, expected); + assert!(res.iter().all(|tx| !tx.is_empty())); + } + } + #[tokio::test] async fn get_inclusion_list_v1_excludes_blob_transactions() { let pool = eth_test_pool(); From f98dd7843f25e9b6b1b67e09e938da83f2f07ca0 Mon Sep 17 00:00:00 2001 From: Soubhik Singha Mahapatra <160333583+Soubhik-10@users.noreply.github.com> Date: Sat, 5 Sep 2026 14:13:26 +0000 Subject: [PATCH 178/294] feat(engine): add SSZ payload bodies endpoints (#26394) Co-authored-by: Matthias Seitz --- crates/ethereum/node/src/engine_ssz_proxy.rs | 394 ++++++++++++++++++- crates/ethereum/node/tests/e2e/eth.rs | 29 +- crates/rpc/rpc-engine-api/src/engine_api.rs | 190 +++++++-- 3 files changed, 565 insertions(+), 48 deletions(-) diff --git a/crates/ethereum/node/src/engine_ssz_proxy.rs b/crates/ethereum/node/src/engine_ssz_proxy.rs index 8071df39611..7f867d98aa1 100644 --- a/crates/ethereum/node/src/engine_ssz_proxy.rs +++ b/crates/ethereum/node/src/engine_ssz_proxy.rs @@ -5,24 +5,26 @@ //! [EIP-8178]: https://eips.ethereum.org/EIPS/eip-8178 use crate::engine_ssz_containers::{ - BuiltPayloadAmsterdam, BuiltPayloadCancun, BuiltPayloadOsaka, BuiltPayloadParis, - BuiltPayloadPrague, BuiltPayloadShanghai, ExecutionPayloadEnvelopeAmsterdam, - ExecutionPayloadEnvelopeCancun, ExecutionPayloadEnvelopeOsaka, ExecutionPayloadEnvelopeParis, - ExecutionPayloadEnvelopePrague, ExecutionPayloadEnvelopeShanghai, ForkchoiceUpdateAmsterdam, - ForkchoiceUpdateCancun, ForkchoiceUpdateOsaka, ForkchoiceUpdateParis, ForkchoiceUpdatePrague, - ForkchoiceUpdateResponse, ForkchoiceUpdateShanghai, Optional, - PayloadStatus as EngineSszPayloadStatus, + BodiesByHashRequest, BodiesResponse, BuiltPayloadAmsterdam, BuiltPayloadCancun, + BuiltPayloadOsaka, BuiltPayloadParis, BuiltPayloadPrague, BuiltPayloadShanghai, + ExecutionPayloadBodyAmsterdam, ExecutionPayloadBodyParis, ExecutionPayloadBodyShanghai, + ExecutionPayloadEnvelopeAmsterdam, ExecutionPayloadEnvelopeCancun, + ExecutionPayloadEnvelopeOsaka, ExecutionPayloadEnvelopeParis, ExecutionPayloadEnvelopePrague, + ExecutionPayloadEnvelopeShanghai, ForkchoiceUpdateAmsterdam, ForkchoiceUpdateCancun, + ForkchoiceUpdateOsaka, ForkchoiceUpdateParis, ForkchoiceUpdatePrague, ForkchoiceUpdateResponse, + ForkchoiceUpdateShanghai, Optional, PayloadStatus as EngineSszPayloadStatus, }; use alloy_consensus::{Transaction, TxEnvelope}; use alloy_eips::{eip2718::Decodable2718, eip7685::RequestsOrHash}; use alloy_primitives::{Bytes, B128, B256}; use alloy_rpc_types_engine::{ - CancunPayloadFields, ExecutionData, ExecutionPayload, ExecutionPayloadFieldV2, - ExecutionPayloadSidecar, ForkchoiceState, PayloadAttributes, PayloadId, PraguePayloadFields, + CancunPayloadFields, ExecutionData, ExecutionPayload, ExecutionPayloadBodyV1, + ExecutionPayloadFieldV2, ExecutionPayloadSidecar, ForkchoiceState, PayloadAttributes, + PayloadId, PraguePayloadFields, }; -use http_body_util::BodyExt; +use http_body_util::{BodyExt, LengthLimitError, Limited}; use jsonrpsee::server::{HttpBody, HttpRequest, HttpResponse}; -use reth_chainspec::EthereumHardforks; +use reth_chainspec::{EthereumHardfork, EthereumHardforks}; use reth_engine_primitives::EngineApiValidator; use reth_ethereum_engine_primitives::EthEngineTypes; use reth_provider::{BalProvider, BlockReader, HeaderProvider, StateProviderFactory}; @@ -50,11 +52,16 @@ const STATUS_OK: u16 = 200; const STATUS_BAD_REQUEST: u16 = 400; const STATUS_NOT_FOUND: u16 = 404; const STATUS_METHOD_NOT_ALLOWED: u16 = 405; +const STATUS_PAYLOAD_TOO_LARGE: u16 = 413; const STATUS_INTERNAL_SERVER_ERROR: u16 = 500; const STATUS_SERVICE_UNAVAILABLE: u16 = 503; +const STATUS_UNSUPPORTED_MEDIA_TYPE: u16 = 415; const MAX_BLOB_LIMIT: usize = 128; -const MAX_PAYLOAD_BYTES: u64 = 64 * 1024 * 1024; +const MAX_BODIES_REQUEST: usize = crate::engine_ssz_containers::MAX_BODIES_REQUEST; +const MAX_BODIES_REQUEST_BYTES: usize = 4 + MAX_BODIES_REQUEST * 32; +const MAX_PAYLOAD_BYTES: usize = 64 * 1024 * 1024; +const PROBLEM_JSON: &str = "application/problem+json"; type EthEngineApi = EngineApi; @@ -267,6 +274,53 @@ where }; handle_forkchoice_updated(engine_api, fork, &body).await } + EngineSszEndpoint::PayloadBodiesByHash => { + if method != "POST" { + return problem_response(STATUS_METHOD_NOT_ALLOWED, "method-not-allowed", None) + } + let Some(fork) = request_fork(&request) else { + return problem_response(STATUS_BAD_REQUEST, "unsupported-fork", None) + }; + let body = match read_bodies_hash_body(request).await { + Ok(body) => body, + Err(response) => return response, + }; + let request = match BodiesByHashRequest::from_ssz_bytes(&body) { + Ok(request) if request.block_hashes.len() <= MAX_BODIES_REQUEST => { + PayloadBodiesRequest::Hash(request.block_hashes) + } + Ok(_) => { + return problem_response(STATUS_PAYLOAD_TOO_LARGE, "request-too-large", None) + } + Err(_) => return problem_response(STATUS_BAD_REQUEST, "ssz-decode-error", None), + }; + let Some(engine_api) = handle.engine_api().await else { + return problem_response(STATUS_SERVICE_UNAVAILABLE, "service-unavailable", None) + }; + handle_get_payload_bodies(engine_api, fork, request).await + } + EngineSszEndpoint::PayloadBodiesByRange => { + if method != "GET" { + return problem_response(STATUS_METHOD_NOT_ALLOWED, "method-not-allowed", None) + } + let Some(fork) = request_fork(&request) else { + return problem_response(STATUS_BAD_REQUEST, "unsupported-fork", None) + }; + let (start, count) = + match parse_bodies_range_query(request.uri().query().unwrap_or_default()) { + Ok(range) => range, + Err(response) => return response, + }; + let Some(engine_api) = handle.engine_api().await else { + return problem_response(STATUS_SERVICE_UNAVAILABLE, "service-unavailable", None) + }; + handle_get_payload_bodies( + engine_api, + fork, + PayloadBodiesRequest::Range { start, count }, + ) + .await + } EngineSszEndpoint::Blobs(version) => { if method != "POST" { return text_response(STATUS_METHOD_NOT_ALLOWED, "method not allowed") @@ -308,6 +362,12 @@ fn parse_engine_path(path: &str) -> Option { (Some("engine"), Some("v1"), Some("blobs"), version, None) => { Some(EngineSszEndpoint::Blobs(parse_method_version(version?)?)) } + (Some("engine"), Some("v1"), Some("bodies"), Some("hash"), None) => { + Some(EngineSszEndpoint::PayloadBodiesByHash) + } + (Some("engine"), Some("v1"), Some("bodies"), None, None) => { + Some(EngineSszEndpoint::PayloadBodiesByRange) + } _ => None, } } @@ -319,6 +379,8 @@ enum EngineSszEndpoint { NewPayload, GetPayload(Result::Err>), Forkchoice, + PayloadBodiesByHash, + PayloadBodiesByRange, Blobs(u8), } @@ -382,7 +444,7 @@ fn handle_capabilities() -> HttpResponse { }, "unscoped_endpoints": ["capabilities", "identity"], "limits": { - "bodies.max_count": 128, + "bodies.max_count": MAX_BODIES_REQUEST, "blobs.max_versioned_hashes": MAX_BLOB_LIMIT, "payload.max_bytes": MAX_PAYLOAD_BYTES, }, @@ -531,6 +593,173 @@ where } } +enum PayloadBodiesRequest { + Hash(Vec), + Range { start: u64, count: u64 }, +} + +async fn handle_get_payload_bodies( + engine_api: EthEngineApi, + fork: EngineSszFork, + request: PayloadBodiesRequest, +) -> HttpResponse +where + Provider: HeaderProvider + BlockReader + StateProviderFactory + BalProvider + 'static, + Pool: TransactionPool + 'static, + Validator: EngineApiValidator, + ChainSpec: EthereumHardforks + Send + Sync + 'static, +{ + let include_bal = fork == EngineSszFork::Amsterdam; + let response = match request { + PayloadBodiesRequest::Hash(hashes) => { + engine_api.get_payload_bodies_by_hash_with_timestamps_metered(hashes, include_bal).await + } + PayloadBodiesRequest::Range { start, count } => { + engine_api + .get_payload_bodies_by_range_with_timestamps_metered(start, count, include_bal) + .await + } + }; + let chain_spec = engine_api.chain_spec().as_ref(); + match fork { + EngineSszFork::Amsterdam => payload_bodies_http_response( + response, + |body| ExecutionPayloadBodyAmsterdam::try_from(body).ok(), + fork, + chain_spec, + ), + EngineSszFork::Paris => payload_bodies_http_response( + response, + |body| ExecutionPayloadBodyParis::try_from(ExecutionPayloadBodyV1::from(body)).ok(), + fork, + chain_spec, + ), + _ => payload_bodies_http_response( + response, + |body| ExecutionPayloadBodyShanghai::try_from(ExecutionPayloadBodyV1::from(body)).ok(), + fork, + chain_spec, + ), + } +} + +fn parse_bodies_range_query(query: &str) -> Result<(u64, u64), HttpResponse> { + let invalid = || problem_response(STATUS_BAD_REQUEST, "invalid-request", None); + let mut start = None; + let mut count = None; + for pair in query.split('&') { + let (key, value) = pair.split_once('=').ok_or_else(invalid)?; + let field = match key { + "from" if start.is_none() => &mut start, + "count" if count.is_none() => &mut count, + _ => return Err(invalid()), + }; + *field = Some(value.parse::().map_err(|_| invalid())?); + } + let range = (start.ok_or_else(invalid)?, count.ok_or_else(invalid)?); + if range.1 > MAX_BODIES_REQUEST as u64 { + return Err(problem_response(STATUS_PAYLOAD_TOO_LARGE, "request-too-large", None)) + } + Ok(range) +} + +/// Amsterdam bodies with a missing or pruned BAL are unavailable, just like pruned blocks. +fn payload_bodies_response( + response: Result>, EngineApiError>, + convert: impl Fn(LegacyBody) -> Option, + fork: EngineSszFork, + chain_spec: &impl EthereumHardforks, +) -> Result, EngineApiError> +where + ForkBody: Default + ssz::Encode + ssz::Decode, +{ + let bodies = response?; + let bodies = bodies + .into_iter() + .map(|body| { + body.filter(|(timestamp, _)| body_matches_fork(chain_spec, fork, *timestamp)) + .map(|(_, body)| body) + }) + .collect(); + Ok(BodiesResponse::from_optional_bodies(bodies, convert)) +} + +fn payload_bodies_http_response( + response: Result>, EngineApiError>, + convert: impl Fn(LegacyBody) -> Option, + fork: EngineSszFork, + chain_spec: &impl EthereumHardforks, +) -> HttpResponse +where + ForkBody: Default + ssz::Encode + ssz::Decode, +{ + match payload_bodies_response(response, convert, fork, chain_spec) { + Ok(response) => ssz_response(response), + Err(err) => payload_bodies_error_response(err), + } +} + +fn body_matches_fork( + chain_spec: &ChainSpec, + fork: EngineSszFork, + timestamp: u64, +) -> bool { + let active = |fork| chain_spec.is_ethereum_fork_active_at_timestamp(fork, timestamp); + match fork { + EngineSszFork::Paris => !active(EthereumHardfork::Shanghai), + EngineSszFork::Shanghai => { + active(EthereumHardfork::Shanghai) && !active(EthereumHardfork::Cancun) + } + EngineSszFork::Cancun => { + active(EthereumHardfork::Cancun) && !active(EthereumHardfork::Prague) + } + EngineSszFork::Prague => { + active(EthereumHardfork::Prague) && !active(EthereumHardfork::Osaka) + } + EngineSszFork::Osaka => { + active(EthereumHardfork::Osaka) && !active(EthereumHardfork::Amsterdam) + } + EngineSszFork::Amsterdam => active(EthereumHardfork::Amsterdam), + } +} + +fn payload_bodies_error_response(err: EngineApiError) -> HttpResponse { + let (status, problem_type) = match &err { + EngineApiError::PayloadRequestTooLarge { .. } => { + (STATUS_PAYLOAD_TOO_LARGE, "request-too-large") + } + EngineApiError::InvalidBodiesRange { .. } => (422, "invalid-body"), + _ => (STATUS_INTERNAL_SERVER_ERROR, "internal"), + }; + problem_response(status, problem_type, Some(err.to_string())) +} + +async fn read_bodies_hash_body(request: HttpRequest) -> Result, HttpResponse> { + let content_type = request.headers().get(CONTENT_TYPE).and_then(|value| value.to_str().ok()); + if content_type != Some(OCTET_STREAM) { + return Err(problem_response(STATUS_UNSUPPORTED_MEDIA_TYPE, "unsupported-media-type", None)) + } + + if let Some(content_length) = request.headers().get("content-length") { + let Some(content_length) = + content_length.to_str().ok().and_then(|value| value.parse::().ok()) + else { + return Err(problem_response(STATUS_BAD_REQUEST, "invalid-request", None)) + }; + if content_length > MAX_BODIES_REQUEST_BYTES { + return Err(problem_response(STATUS_PAYLOAD_TOO_LARGE, "request-too-large", None)) + } + } + + match Limited::new(request.into_body(), MAX_BODIES_REQUEST_BYTES).collect().await { + Ok(body) => Ok(body.to_bytes().to_vec()), + Err(err) if err.downcast_ref::().is_some() => { + Err(problem_response(STATUS_PAYLOAD_TOO_LARGE, "request-too-large", None)) + } + Err(_) => Err(problem_response(STATUS_BAD_REQUEST, "invalid-request", None)), + } +} + /// Handles SSZ `engine_getBlobsV*` requests with the node's blob store. async fn handle_get_blobs( engine_api: EthEngineApi, @@ -783,11 +1012,138 @@ fn text_response(status: u16, body: impl Into) -> HttpResponse { .expect("valid response") } +fn problem_response( + status: u16, + problem_type: &'static str, + detail: Option, +) -> HttpResponse { + let problem_type = format!("/engine-api/errors/{problem_type}"); + let body = match detail { + Some(detail) => serde_json::json!({ "type": problem_type, "detail": detail }), + None => serde_json::json!({ "type": problem_type }), + }; + + HttpResponse::builder() + .status(status) + .header(CONTENT_TYPE, PROBLEM_JSON) + .body(HttpBody::from(body.to_string())) + .expect("valid response") +} + #[cfg(test)] mod tests { use super::*; use ssz::Encode; + #[test] + fn bodies_range_query_validation() { + assert_eq!(parse_bodies_range_query("from=1&count=32").unwrap(), (1, 32)); + assert_eq!(parse_bodies_range_query("count=2&from=3").unwrap(), (3, 2)); + for query in + ["", "from=1", "from=x&count=1", "from=1&count=1&count=2", "from=1&count=1&other=0"] + { + assert_eq!(parse_bodies_range_query(query).unwrap_err().status(), 400); + } + assert_eq!(parse_bodies_range_query("from=1&count=33").unwrap_err().status(), 413); + } + + #[test] + fn payload_bodies_are_filtered_at_fork_boundaries() { + use reth_chainspec::{ChainSpecBuilder, ForkCondition}; + let chain_spec = ChainSpecBuilder::default() + .chain(1.into()) + .genesis(Default::default()) + .with_fork(EthereumHardfork::Shanghai, ForkCondition::Timestamp(10)) + .with_fork(EthereumHardfork::Cancun, ForkCondition::Timestamp(20)) + .with_fork(EthereumHardfork::Prague, ForkCondition::Timestamp(30)) + .with_fork(EthereumHardfork::Osaka, ForkCondition::Timestamp(40)) + .with_fork(EthereumHardfork::Amsterdam, ForkCondition::Timestamp(50)) + .build(); + for (fork, start, end) in [ + (EngineSszFork::Shanghai, 10, 20), + (EngineSszFork::Cancun, 20, 30), + (EngineSszFork::Prague, 30, 40), + (EngineSszFork::Osaka, 40, 50), + ] { + let body = ExecutionPayloadBodyV1 { + transactions: vec![Bytes::from_static(&[1, 2, 3])], + withdrawals: Some(vec![]), + }; + let response = payload_bodies_response( + Ok(vec![ + Some((start - 1, body.clone())), + Some((start, body.clone())), + Some((end - 1, body.clone())), + Some((end, body)), + None, + ]), + |body| ExecutionPayloadBodyShanghai::try_from(body).ok(), + fork, + &chain_spec, + ) + .unwrap(); + assert_eq!( + response.entries.iter().map(|entry| entry.available).collect::>(), + [false, true, true, false, false] + ); + for index in [0, 3, 4] { + assert_eq!(response.entries[index].body, ExecutionPayloadBodyShanghai::default()); + } + } + assert!(body_matches_fork(&chain_spec, EngineSszFork::Paris, 9)); + assert!(!body_matches_fork(&chain_spec, EngineSszFork::Paris, 10)); + assert!(!body_matches_fork(&chain_spec, EngineSszFork::Amsterdam, 49)); + assert!(body_matches_fork(&chain_spec, EngineSszFork::Amsterdam, 50)); + } + + #[tokio::test] + async fn payload_bodies_hash_request_limits_and_media_type() { + for content_length in [false, true] { + for count in [32, 33] { + let bytes = + BodiesByHashRequest { block_hashes: vec![B256::ZERO; count] }.as_ssz_bytes(); + let mut request = HttpRequest::builder().header(CONTENT_TYPE, OCTET_STREAM); + if content_length { + request = request.header("content-length", bytes.len()); + } + let result = + read_bodies_hash_body(request.body(HttpBody::from(bytes)).unwrap()).await; + if count == 32 { + assert_eq!( + BodiesByHashRequest::from_ssz_bytes(&result.unwrap()) + .unwrap() + .block_hashes + .len(), + 32 + ); + } else { + let response = result.unwrap_err(); + assert_eq!(response.status(), STATUS_PAYLOAD_TOO_LARGE); + assert_eq!(response.headers()[CONTENT_TYPE], PROBLEM_JSON); + } + } + } + let response = + read_bodies_hash_body(HttpRequest::new(HttpBody::empty())).await.unwrap_err(); + assert_eq!(response.status(), STATUS_UNSUPPORTED_MEDIA_TYPE); + } + + #[tokio::test] + async fn payload_bodies_errors_and_capabilities_match_rest_spec() { + let response = payload_bodies_error_response(EngineApiError::InvalidBodiesRange { + start: 0, + count: 1, + }); + assert_eq!(response.status(), 422); + assert_eq!(response.headers()[CONTENT_TYPE], PROBLEM_JSON); + let body = response.into_body().collect().await.unwrap().to_bytes(); + let problem: serde_json::Value = serde_json::from_slice(&body).unwrap(); + assert_eq!(problem["type"], "/engine-api/errors/invalid-body"); + let body = handle_capabilities().into_body().collect().await.unwrap().to_bytes(); + let capabilities: serde_json::Value = serde_json::from_slice(&body).unwrap(); + assert_eq!(capabilities["limits"]["bodies.max_count"], 32); + } + #[test] fn parses_capabilities_endpoint() { let endpoint = parse_engine_path("/engine/v1/capabilities").unwrap(); @@ -829,6 +1185,18 @@ mod tests { assert_eq!(endpoint, EngineSszEndpoint::Forkchoice); } + #[test] + fn parses_payload_bodies_endpoints() { + assert_eq!( + parse_engine_path("/engine/v1/bodies/hash").unwrap(), + EngineSszEndpoint::PayloadBodiesByHash + ); + assert_eq!( + parse_engine_path("/engine/v1/bodies").unwrap(), + EngineSszEndpoint::PayloadBodiesByRange + ); + } + #[test] fn rejects_legacy_version_scoped_endpoint() { assert!(parse_engine_path("/engine/v4/payloads").is_none()); diff --git a/crates/ethereum/node/tests/e2e/eth.rs b/crates/ethereum/node/tests/e2e/eth.rs index a44e6b27190..0413f7187d6 100644 --- a/crates/ethereum/node/tests/e2e/eth.rs +++ b/crates/ethereum/node/tests/e2e/eth.rs @@ -19,6 +19,7 @@ use reth_node_core::{ }; use reth_node_ethereum::{ engine_ssz_containers::{ + BodiesByHashRequest, BodiesResponsePrague, ForkchoiceUpdateResponse as SszForkchoiceUpdateResponse, PayloadStatus as SszPayloadStatus, }, engine_ssz_proxy::EngineSszProxyLayer, @@ -378,7 +379,7 @@ async fn test_engine_ssz_proxy_can_mine_block() -> eyre::Result<()> { }, "unscoped_endpoints": ["capabilities", "identity"], "limits": { - "bodies.max_count": 128, + "bodies.max_count": 32, "blobs.max_versioned_hashes": 128, "payload.max_bytes": 67108864, }, @@ -471,6 +472,32 @@ async fn test_engine_ssz_proxy_can_mine_block() -> eyre::Result<()> { node.wait_block(1, block_hash, false).await?; + for (fork, available) in [("prague", true), ("cancun", false)] { + let response = client + .post(format!("{auth_url}/engine/v1/bodies/hash")) + .header(reqwest::header::AUTHORIZATION, auth_header.to_str()?) + .header(ENGINE_EXECUTION_VERSION_HEADER, fork) + .header(reqwest::header::CONTENT_TYPE, "application/octet-stream") + .body(BodiesByHashRequest { block_hashes: vec![block_hash, B256::ZERO] }.as_ssz_bytes()) + .send() + .await?; + assert_eq!(response.status(), reqwest::StatusCode::OK); + let bodies = BodiesResponsePrague::from_ssz_bytes(&response.bytes().await?).unwrap(); + assert_eq!(bodies.entries.len(), 2); + assert_eq!(bodies.entries[0].available, available); + assert!(!bodies.entries[1].available); + } + let response = client + .get(format!("{auth_url}/engine/v1/bodies?from=1&count=32")) + .header(reqwest::header::AUTHORIZATION, auth_header.to_str()?) + .header(ENGINE_EXECUTION_VERSION_HEADER, "prague") + .send() + .await?; + assert_eq!(response.status(), reqwest::StatusCode::OK); + let bodies = BodiesResponsePrague::from_ssz_bytes(&response.bytes().await?).unwrap(); + assert_eq!(bodies.entries.len(), 1); + assert!(bodies.entries[0].available); + Ok(()) } diff --git a/crates/rpc/rpc-engine-api/src/engine_api.rs b/crates/rpc/rpc-engine-api/src/engine_api.rs index 4f300225d7b..3624c48091c 100644 --- a/crates/rpc/rpc-engine-api/src/engine_api.rs +++ b/crates/rpc/rpc-engine-api/src/engine_api.rs @@ -26,7 +26,7 @@ use reth_payload_primitives::{ validate_payload_timestamp, EngineApiMessageVersion, MessageValidationKind, PayloadOrAttributes, PayloadTypes, }; -use reth_primitives_traits::{Block, BlockBody}; +use reth_primitives_traits::{AlloyBlockHeader, Block, BlockBody}; use reth_rpc_api::{EngineApiServer, IntoEngineApiRpcModule}; use reth_storage_api::{BalProvider, BlockReader, HeaderProvider, StateProviderFactory}; use reth_tasks::Runtime; @@ -766,6 +766,41 @@ where rx.await.map_err(|err| EngineApiError::Internal(Box::new(err)))? } + /// Returns payload bodies and their timestamps from the same block read. + /// + /// SSZ transports use the timestamp to select the block's fork schema. Reading it + /// separately by block number could pair a body with another block after a reorg. + pub async fn get_payload_bodies_by_range_with_timestamps( + &self, + start: BlockNumber, + count: u64, + include_bal: bool, + ) -> EngineApiResult>> { + let bodies = self + .get_payload_bodies_by_range_with(start, count, Self::payload_body_with_timestamp) + .await?; + self.attach_payload_body_bals(bodies, include_bal).await + } + + /// Records SSZ body requests in the corresponding JSON-RPC V1/V2 latency histogram. + pub async fn get_payload_bodies_by_range_with_timestamps_metered( + &self, + start: BlockNumber, + count: u64, + include_bal: bool, + ) -> EngineApiResult>> { + let start_time = Instant::now(); + let result = + self.get_payload_bodies_by_range_with_timestamps(start, count, include_bal).await; + let latency = &self.inner.metrics.latency; + if include_bal { + latency.get_payload_bodies_by_range_v2.record(start_time.elapsed()); + } else { + latency.get_payload_bodies_by_range_v1.record(start_time.elapsed()); + } + result + } + /// Returns the execution payload bodies by the range starting at `start`, containing `count` /// blocks. /// @@ -808,39 +843,11 @@ where start: BlockNumber, count: u64, ) -> EngineApiResult { - let mut payload_bodies = self - .get_payload_bodies_by_range_with(start, count, |block| { - let block_hash = block.header().hash_slow(); - ( - block_hash, - ExecutionPayloadBodyV2 { - transactions: block.body().encoded_2718_transactions(), - withdrawals: block - .body() - .withdrawals() - .cloned() - .map(Withdrawals::into_inner), - block_access_list: None, - }, - ) - }) - .await?; - - let block_hashes = payload_bodies - .iter() - .filter_map(|payload_body| payload_body.as_ref().map(|(block_hash, _)| *block_hash)) - .collect::>(); - let block_access_lists = self.get_block_access_lists_by_hashes(block_hashes).await?; - - for (payload_body, block_access_list) in - payload_bodies.iter_mut().filter_map(Option::as_mut).zip(block_access_lists) - { - payload_body.1.block_access_list = block_access_list; - } - - Ok(payload_bodies + Ok(self + .get_payload_bodies_by_range_with_timestamps(start, count, true) + .await? .into_iter() - .map(|payload_body| payload_body.map(|(_, payload_body)| payload_body)) + .map(|body| body.map(|(_, body)| body)) .collect()) } @@ -898,6 +905,66 @@ where rx.await.map_err(|err| EngineApiError::Internal(Box::new(err)))? } + /// Returns payload bodies and their timestamps from the same block read. + pub async fn get_payload_bodies_by_hash_with_timestamps( + &self, + hashes: Vec, + include_bal: bool, + ) -> EngineApiResult>> { + let bodies = + self.get_payload_bodies_by_hash_with(hashes, Self::payload_body_with_timestamp).await?; + self.attach_payload_body_bals(bodies, include_bal).await + } + + /// Records SSZ body requests in the corresponding JSON-RPC V1/V2 latency histogram. + pub async fn get_payload_bodies_by_hash_with_timestamps_metered( + &self, + hashes: Vec, + include_bal: bool, + ) -> EngineApiResult>> { + let start = Instant::now(); + let result = self.get_payload_bodies_by_hash_with_timestamps(hashes, include_bal).await; + let latency = &self.inner.metrics.latency; + if include_bal { + latency.get_payload_bodies_by_hash_v2.record(start.elapsed()); + } else { + latency.get_payload_bodies_by_hash_v1.record(start.elapsed()); + } + result + } + + fn payload_body_with_timestamp( + block: Provider::Block, + ) -> (BlockHash, u64, ExecutionPayloadBodyV2) { + ( + block.header().hash_slow(), + block.header().timestamp(), + ExecutionPayloadBodyV2 { + transactions: block.body().encoded_2718_transactions(), + withdrawals: block.body().withdrawals().cloned().map(Withdrawals::into_inner), + block_access_list: None, + }, + ) + } + + async fn attach_payload_body_bals( + &self, + mut bodies: Vec>, + include_bal: bool, + ) -> EngineApiResult>> { + if include_bal { + let hashes = bodies.iter().flatten().map(|(hash, _, _)| *hash).collect(); + let bals = self.get_block_access_lists_by_hashes(hashes).await?; + for ((_, _, body), bal) in bodies.iter_mut().flatten().zip(bals) { + body.block_access_list = bal; + } + } + Ok(bodies + .into_iter() + .map(|body| body.map(|(_, timestamp, body)| (timestamp, body))) + .collect()) + } + async fn get_block_access_lists_by_hashes( &self, hashes: Vec, @@ -1828,6 +1895,19 @@ mod tests { EngineApiTestHandle, EngineApi, EthEngineTypes, Pool, EthereumEngineValidator, ChainSpec>, ) + where + Pool: TransactionPool + 'static, + { + setup_engine_api_with_provider(tx_pool, MockEthProvider::default()) + } + + fn setup_engine_api_with_provider( + tx_pool: Pool, + provider: MockEthProvider, + ) -> ( + EngineApiTestHandle, + EngineApi, EthEngineTypes, Pool, EthereumEngineValidator, ChainSpec>, + ) where Pool: TransactionPool + 'static, { @@ -1839,7 +1919,7 @@ mod tests { }; let chain_spec: Arc = MAINNET.clone(); - let provider = Arc::new(MockEthProvider::default()); + let provider = Arc::new(provider); let payload_store = spawn_test_payload_service(); let (to_engine, engine_rx) = unbounded_channel(); let task_executor = Runtime::test(); @@ -2016,6 +2096,48 @@ mod tests { ); } + #[tokio::test] + async fn payload_bodies_with_timestamps_preserve_missing_entries_and_head_truncation() { + let mut provider = MockEthProvider::default(); + provider.bal_store = BalStoreHandle::new(InMemoryBalStore::default()); + let (handle, api) = + setup_engine_api_with_provider(NoopTransactionPool::default(), provider); + let mut block = Block::default(); + block.header.number = 1; + block.header.timestamp = 123; + let hash = block.header.hash_slow(); + handle.provider.add_block(hash, block); + let raw_bal = Bytes::from_static(&[alloy_rlp::EMPTY_LIST_CODE]); + handle + .provider + .bal_store + .insert(NumHash::new(1, hash), RawBal::new(raw_bal.clone())) + .unwrap(); + for include_bal in [false, true] { + let by_hash = api + .get_payload_bodies_by_hash_with_timestamps( + vec![B256::ZERO, hash, hash], + include_bal, + ) + .await + .unwrap(); + assert_eq!(by_hash.len(), 3); + assert!(by_hash[0].is_none()); + assert_eq!(by_hash[1], by_hash[2]); + let (timestamp, body) = by_hash[1].as_ref().unwrap(); + assert_eq!(*timestamp, 123); + assert_eq!(body.block_access_list, include_bal.then(|| raw_bal.clone())); + let by_range = + api.get_payload_bodies_by_range_with_timestamps(1, 3, include_bal).await.unwrap(); + assert_eq!(by_range, vec![by_hash[1].clone()]); + assert!(api + .get_payload_bodies_by_range_with_timestamps(2, 3, include_bal) + .await + .unwrap() + .is_empty()); + } + } + #[tokio::test] async fn get_payload_bodies_by_hash_v2_returns_block_access_list_from_store() { let bal_store = BalStoreHandle::new(InMemoryBalStore::default()); From 475a872db510c1dee8a77be10781e6b6a92133f6 Mon Sep 17 00:00:00 2001 From: Soubhik Singha Mahapatra <160333583+Soubhik-10@users.noreply.github.com> Date: Sat, 5 Sep 2026 14:50:18 +0000 Subject: [PATCH 179/294] feat(engine): install SSZ proxy by default (#26254) Co-authored-by: Matthias Seitz --- crates/ethereum/node/Cargo.toml | 2 +- crates/ethereum/node/src/engine_ssz_proxy.rs | 1217 +++++++++++------- crates/ethereum/node/src/node.rs | 19 +- crates/ethereum/node/tests/e2e/eth.rs | 323 ++++- crates/node/builder/src/rpc.rs | 39 +- 5 files changed, 1095 insertions(+), 505 deletions(-) diff --git a/crates/ethereum/node/Cargo.toml b/crates/ethereum/node/Cargo.toml index ba86215d317..82ce27c96ae 100644 --- a/crates/ethereum/node/Cargo.toml +++ b/crates/ethereum/node/Cargo.toml @@ -59,6 +59,7 @@ revm = { workspace = true, features = ["secp256k1", "blst", "c-kzg", "memory_lim eyre.workspace = true ethereum_ssz.workspace = true ethereum_ssz_derive.workspace = true +futures.workspace = true http-body-util.workspace = true serde.workspace = true serde_json.workspace = true @@ -82,7 +83,6 @@ alloy-sol-types.workspace = true alloy-contract.workspace = true alloy-rpc-types-beacon = { workspace = true, features = ["ssz"] } -futures.workspace = true tokio.workspace = true serde_json.workspace = true rand.workspace = true diff --git a/crates/ethereum/node/src/engine_ssz_proxy.rs b/crates/ethereum/node/src/engine_ssz_proxy.rs index 7f867d98aa1..915568fd1de 100644 --- a/crates/ethereum/node/src/engine_ssz_proxy.rs +++ b/crates/ethereum/node/src/engine_ssz_proxy.rs @@ -5,23 +5,26 @@ //! [EIP-8178]: https://eips.ethereum.org/EIPS/eip-8178 use crate::engine_ssz_containers::{ - BodiesByHashRequest, BodiesResponse, BuiltPayloadAmsterdam, BuiltPayloadCancun, - BuiltPayloadOsaka, BuiltPayloadParis, BuiltPayloadPrague, BuiltPayloadShanghai, - ExecutionPayloadBodyAmsterdam, ExecutionPayloadBodyParis, ExecutionPayloadBodyShanghai, - ExecutionPayloadEnvelopeAmsterdam, ExecutionPayloadEnvelopeCancun, - ExecutionPayloadEnvelopeOsaka, ExecutionPayloadEnvelopeParis, ExecutionPayloadEnvelopePrague, - ExecutionPayloadEnvelopeShanghai, ForkchoiceUpdateAmsterdam, ForkchoiceUpdateCancun, - ForkchoiceUpdateOsaka, ForkchoiceUpdateParis, ForkchoiceUpdatePrague, ForkchoiceUpdateResponse, - ForkchoiceUpdateShanghai, Optional, PayloadStatus as EngineSszPayloadStatus, + BlobsV1Request, BlobsV1Response, BlobsV2Response, BlobsV3Response, BlobsV4Request, + BlobsV4Response, BodiesByHashRequest, BodiesResponse, BuiltPayloadAmsterdam, + BuiltPayloadCancun, BuiltPayloadOsaka, BuiltPayloadParis, BuiltPayloadPrague, + BuiltPayloadShanghai, ExecutionPayloadBodyAmsterdam, ExecutionPayloadBodyParis, + ExecutionPayloadBodyShanghai, ExecutionPayloadEnvelopeAmsterdam, + ExecutionPayloadEnvelopeCancun, ExecutionPayloadEnvelopeOsaka, ExecutionPayloadEnvelopeParis, + ExecutionPayloadEnvelopePrague, ExecutionPayloadEnvelopeShanghai, ForkchoiceUpdateAmsterdam, + ForkchoiceUpdateCancun, ForkchoiceUpdateOsaka, ForkchoiceUpdateParis, ForkchoiceUpdatePrague, + ForkchoiceUpdateResponse, ForkchoiceUpdateShanghai, Optional, + PayloadStatus as EngineSszPayloadStatus, }; use alloy_consensus::{Transaction, TxEnvelope}; -use alloy_eips::{eip2718::Decodable2718, eip7685::RequestsOrHash}; +use alloy_eips::{eip2718::Decodable2718, eip7685::Requests}; use alloy_primitives::{Bytes, B128, B256}; use alloy_rpc_types_engine::{ CancunPayloadFields, ExecutionData, ExecutionPayload, ExecutionPayloadBodyV1, ExecutionPayloadFieldV2, ExecutionPayloadSidecar, ForkchoiceState, PayloadAttributes, PayloadId, PraguePayloadFields, }; +use futures::future::{BoxFuture, Either}; use http_body_util::{BodyExt, LengthLimitError, Limited}; use jsonrpsee::server::{HttpBody, HttpRequest, HttpResponse}; use reth_chainspec::{EthereumHardfork, EthereumHardforks}; @@ -34,7 +37,6 @@ use reth_transaction_pool::TransactionPool; use ssz::Decode; use std::{ future::Future, - pin::Pin, sync::Arc, task::{Context, Poll}, }; @@ -43,13 +45,14 @@ use tower::{BoxError, Layer, Service}; const OCTET_STREAM: &str = "application/octet-stream"; const APPLICATION_JSON: &str = "application/json"; -const TEXT_PLAIN: &str = "text/plain"; const CONTENT_TYPE: &str = "content-type"; const CACHE_CONTROL: &str = "cache-control"; const ETH_EXECUTION_VERSION: &str = "eth-execution-version"; const STATUS_OK: u16 = 200; const STATUS_BAD_REQUEST: u16 = 400; +const STATUS_CONFLICT: u16 = 409; +const STATUS_UNPROCESSABLE_ENTITY: u16 = 422; const STATUS_NOT_FOUND: u16 = 404; const STATUS_METHOD_NOT_ALLOWED: u16 = 405; const STATUS_PAYLOAD_TOO_LARGE: u16 = 413; @@ -57,7 +60,8 @@ const STATUS_INTERNAL_SERVER_ERROR: u16 = 500; const STATUS_SERVICE_UNAVAILABLE: u16 = 503; const STATUS_UNSUPPORTED_MEDIA_TYPE: u16 = 415; -const MAX_BLOB_LIMIT: usize = 128; +const MAX_BLOB_LIMIT: usize = crate::engine_ssz_containers::MAX_BLOBS_REQUEST; +const MAX_BLOB_REQUEST_BYTES: usize = 4 + 16 + MAX_BLOB_LIMIT * 32; const MAX_BODIES_REQUEST: usize = crate::engine_ssz_containers::MAX_BODIES_REQUEST; const MAX_BODIES_REQUEST_BYTES: usize = 4 + MAX_BODIES_REQUEST * 32; const MAX_PAYLOAD_BYTES: usize = 64 * 1024 * 1024; @@ -65,52 +69,41 @@ const PROBLEM_JSON: &str = "application/problem+json"; type EthEngineApi = EngineApi; -type SharedEthEngineApi = - Arc>>>; +type SharedEngineApi = Arc>>; /// Shared handle used by [`EngineSszProxyLayer`]. -pub struct EngineSszProxyHandle { - engine_api: SharedEthEngineApi, +pub struct EngineSszProxyHandle { + engine_api: SharedEngineApi, } -impl Clone for EngineSszProxyHandle { +impl Clone for EngineSszProxyHandle { fn clone(&self) -> Self { Self { engine_api: self.engine_api.clone() } } } -impl std::fmt::Debug - for EngineSszProxyHandle -{ +impl std::fmt::Debug for EngineSszProxyHandle { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("EngineSszProxyHandle").finish_non_exhaustive() } } -impl - EngineSszProxyHandle -{ +impl EngineSszProxyHandle { fn new() -> Self { Self { engine_api: Default::default() } } - fn with_engine_api(engine_api: EthEngineApi) -> Self { + fn with_engine_api(engine_api: Api) -> Self { Self { engine_api: Arc::new(RwLock::new(Some(engine_api))) } } /// Sets the Engine API implementation used by the proxy. - pub async fn set_engine_api( - &self, - engine_api: EthEngineApi, - ) { + pub async fn set_engine_api(&self, engine_api: Api) { *self.engine_api.write().await = Some(engine_api); } /// Sets the Engine API implementation during synchronous launch wiring. - pub fn set_engine_api_sync( - &self, - engine_api: EthEngineApi, - ) { + pub fn set_engine_api_sync(&self, engine_api: Api) { *self .engine_api .try_write() @@ -118,43 +111,35 @@ impl } } -impl - EngineSszProxyHandle -{ +impl EngineSszProxyHandle { /// Returns the Engine API implementation used by the proxy. - pub async fn engine_api(&self) -> Option> { + pub async fn engine_api(&self) -> Option { self.engine_api.read().await.clone() } } /// A tower layer that intercepts SSZ Engine API routes under `/engine/v1`. #[derive(Clone, Debug)] -pub struct EngineSszProxyLayer { - handle: EngineSszProxyHandle, +pub struct EngineSszProxyLayer { + handle: EngineSszProxyHandle, } -impl - EngineSszProxyLayer -{ +impl EngineSszProxyLayer { /// Creates a new proxy layer and a handle for setting the engine after node launch. - pub fn new() -> (Self, EngineSszProxyHandle) { + pub fn new() -> (Self, EngineSszProxyHandle) { let handle = EngineSszProxyHandle::new(); (Self { handle: handle.clone() }, handle) } /// Creates a new proxy layer with an Engine API implementation. - pub fn with_engine_api( - engine_api: EthEngineApi, - ) -> (Self, EngineSszProxyHandle) { + pub fn with_engine_api(engine_api: Api) -> (Self, EngineSszProxyHandle) { let handle = EngineSszProxyHandle::with_engine_api(engine_api); (Self { handle: handle.clone() }, handle) } } -impl Layer - for EngineSszProxyLayer -{ - type Service = EngineSszProxyService; +impl Layer for EngineSszProxyLayer { + type Service = EngineSszProxyService; fn layer(&self, inner: S) -> Self::Service { EngineSszProxyService { inner, handle: self.handle.clone() } @@ -163,24 +148,20 @@ impl Layer /// The service produced by [`EngineSszProxyLayer`]. #[derive(Clone, Debug)] -pub struct EngineSszProxyService { +pub struct EngineSszProxyService { inner: S, - handle: EngineSszProxyHandle, + handle: EngineSszProxyHandle, } -impl Service - for EngineSszProxyService +impl Service for EngineSszProxyService where S: Service + Send + Clone, - S::Future: Send + 'static, - Provider: HeaderProvider + BlockReader + StateProviderFactory + BalProvider + 'static, - Pool: TransactionPool + 'static, - Validator: EngineApiValidator, - ChainSpec: EthereumHardforks + Send + Sync + 'static, + S::Future: Send, + Api: EngineSszApi, { type Response = HttpResponse; type Error = BoxError; - type Future = Pin> + Send>>; + type Future = Either>>; fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll> { self.inner.poll_ready(cx) @@ -188,91 +169,386 @@ where fn call(&mut self, request: HttpRequest) -> Self::Future { if !request.uri().path().starts_with("/engine/") { - let fut = self.inner.call(request); - return Box::pin(fut) + return Either::Left(self.inner.call(request)) } let handle = self.handle.clone(); - Box::pin(async move { Ok(handle_engine_ssz_request(handle, request).await) }) + Either::Right(Box::pin(async move { Ok(handle_engine_ssz_request(handle, request).await) })) } } -async fn handle_engine_ssz_request( - handle: EngineSszProxyHandle, - request: HttpRequest, -) -> HttpResponse +/// Fork selector used by SSZ Engine API request handling. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum EngineSszFork { + /// Paris fork. + Paris, + /// Shanghai fork. + Shanghai, + /// Cancun fork. + Cancun, + /// Prague fork. + Prague, + /// Osaka fork. + Osaka, + /// Amsterdam fork. + Amsterdam, +} + +impl EngineSszFork { + const fn payloads_version(self) -> u8 { + match self { + Self::Paris => 1, + Self::Shanghai => 2, + Self::Cancun => 3, + Self::Prague | Self::Osaka => 4, + Self::Amsterdam => 5, + } + } + + const fn forkchoice_version(self) -> u8 { + match self { + Self::Paris => 1, + Self::Shanghai => 2, + Self::Cancun | Self::Prague | Self::Osaka => 3, + Self::Amsterdam => 4, + } + } +} + +impl std::str::FromStr for EngineSszFork { + type Err = (); + + fn from_str(value: &str) -> Result { + match value { + "paris" => Ok(Self::Paris), + "shanghai" => Ok(Self::Shanghai), + "cancun" => Ok(Self::Cancun), + "prague" => Ok(Self::Prague), + "osaka" => Ok(Self::Osaka), + "amsterdam" => Ok(Self::Amsterdam), + _ => Err(()), + } + } +} + +/// API surface required by the SSZ Engine API proxy. +/// +/// Custom APIs can opt in with a marker implementation; every route, including +/// `/capabilities`, then answers 404 so clients fall back to the JSON-RPC Engine API. +pub trait EngineSszApi: Clone + Send + Sync + 'static { + /// Returns the capabilities advertisement. + fn capabilities(&self) -> HttpResponse { + problem_response(STATUS_NOT_FOUND, "method-not-found", None) + } + + /// Returns the Engine API identity response. + fn identity(&self) -> HttpResponse { + problem_response(STATUS_NOT_FOUND, "method-not-found", None) + } + + /// Handles a new payload request. + fn new_payload( + &self, + _fork: EngineSszFork, + _body: Bytes, + ) -> impl Future + Send { + async { problem_response(STATUS_NOT_FOUND, "method-not-found", None) } + } + + /// Handles a getPayload request. + fn get_payload( + &self, + _fork: EngineSszFork, + _payload_id: PayloadId, + ) -> impl Future + Send { + async { problem_response(STATUS_NOT_FOUND, "method-not-found", None) } + } + + /// Handles a forkchoice update request. + fn forkchoice_updated( + &self, + _fork: EngineSszFork, + _body: Bytes, + ) -> impl Future + Send { + async { problem_response(STATUS_NOT_FOUND, "method-not-found", None) } + } + + /// Handles a getPayloadBodiesByHash request. + fn get_payload_bodies_by_hash( + &self, + _fork: EngineSszFork, + _hashes: Vec, + ) -> impl Future + Send { + async { problem_response(STATUS_NOT_FOUND, "method-not-found", None) } + } + + /// Handles a getPayloadBodiesByRange request. + fn get_payload_bodies_by_range( + &self, + _fork: EngineSszFork, + _start: u64, + _count: u64, + ) -> impl Future + Send { + async { problem_response(STATUS_NOT_FOUND, "method-not-found", None) } + } + + /// Handles a getBlobs request. + fn get_blobs(&self, _version: u8, _body: Bytes) -> impl Future + Send { + async { problem_response(STATUS_NOT_FOUND, "method-not-found", None) } + } +} + +impl EngineSszApi for reth_node_builder::rpc::NoopEngineApi {} + +impl EngineSszApi + for EthEngineApi where Provider: HeaderProvider + BlockReader + StateProviderFactory + BalProvider + 'static, Pool: TransactionPool + 'static, Validator: EngineApiValidator, ChainSpec: EthereumHardforks + Send + Sync + 'static, +{ + fn capabilities(&self) -> HttpResponse { + handle_capabilities() + } + + fn identity(&self) -> HttpResponse { + json_response(vec![self.client_version().clone()]) + } + + async fn new_payload(&self, fork: EngineSszFork, body: Bytes) -> HttpResponse { + let payload = match decode_new_payload_request(fork, &body) { + Ok(payload) => payload, + Err(err) => return err.into_response(), + }; + + let response = match fork.payloads_version() { + 1 => self.new_payload_v1(payload).await, + 2 => self.new_payload_v2(payload).await, + 3 => self.new_payload_v3(payload).await, + 4 => self.new_payload_v4(payload).await, + 5 => self.new_payload_v5(payload).await, + _ => return problem_response(STATUS_BAD_REQUEST, "unsupported-fork", None), + }; + + match response { + Ok(status) => match EngineSszPayloadStatus::try_from(status) { + Ok(status) => ssz_response(status), + Err(err) => problem_response( + STATUS_INTERNAL_SERVER_ERROR, + "internal", + Some(err.to_string()), + ), + }, + Err(err) => engine_error_response(err), + } + } + + async fn get_payload(&self, fork: EngineSszFork, payload_id: PayloadId) -> HttpResponse { + match fork { + EngineSszFork::Paris => match self.get_payload_v2_metered(payload_id).await { + Ok(payload) => { + let block_value = payload.block_value; + match payload.execution_payload { + ExecutionPayloadFieldV2::V1(payload) => { + get_payload_response(BuiltPayloadParis { payload, block_value }) + } + ExecutionPayloadFieldV2::V2(_) => { + problem_response(STATUS_BAD_REQUEST, "unsupported-fork", None) + } + } + } + Err(err) => engine_error_response(err), + }, + EngineSszFork::Shanghai => match self.get_payload_v2_metered(payload_id).await { + Ok(payload) => match BuiltPayloadShanghai::try_from(payload) { + Ok(payload) => get_payload_response(payload), + Err(err) => problem_response( + STATUS_UNPROCESSABLE_ENTITY, + "invalid-body", + Some(err.to_string()), + ), + }, + Err(err) => engine_error_response(err), + }, + EngineSszFork::Cancun => match self.get_payload_v3_metered(payload_id).await { + Ok(payload) => get_payload_response(BuiltPayloadCancun::from(payload)), + Err(err) => engine_error_response(err), + }, + EngineSszFork::Prague => match self.get_payload_v4_metered(payload_id).await { + Ok(payload) => get_payload_response(BuiltPayloadPrague::from(payload)), + Err(err) => engine_error_response(err), + }, + EngineSszFork::Osaka => match self.get_payload_v5_metered(payload_id).await { + Ok(payload) => get_payload_response(BuiltPayloadOsaka::from(payload)), + Err(err) => engine_error_response(err), + }, + EngineSszFork::Amsterdam => match self.get_payload_v6_metered(payload_id).await { + Ok(payload) => get_payload_response(BuiltPayloadAmsterdam::from(payload)), + Err(err) => engine_error_response(err), + }, + } + } + + async fn forkchoice_updated(&self, fork: EngineSszFork, body: Bytes) -> HttpResponse { + let (state, attrs, custody_columns) = match decode_forkchoice_request(fork, &body) { + Ok(request) => request, + Err(_) => return problem_response(STATUS_BAD_REQUEST, "ssz-decode-error", None), + }; + + if attrs.as_ref().is_some_and(|attrs| { + !timestamp_matches_fork(self.chain_spec().as_ref(), fork, attrs.timestamp) + }) { + return problem_response(STATUS_BAD_REQUEST, "unsupported-fork", None) + } + + let response = match fork.forkchoice_version() { + 1 => self.fork_choice_updated_v1_metered(state, attrs).await, + 2 => self.fork_choice_updated_v2_metered(state, attrs).await, + 3 => self.fork_choice_updated_v3_metered(state, attrs).await, + 4 => self.fork_choice_updated_v4_metered(state, attrs, custody_columns).await, + _ => return problem_response(STATUS_BAD_REQUEST, "unsupported-fork", None), + }; + + match response { + Ok(updated) => match ForkchoiceUpdateResponse::try_from(updated) { + Ok(updated) => ssz_response(updated), + Err(err) => problem_response( + STATUS_INTERNAL_SERVER_ERROR, + "internal", + Some(err.to_string()), + ), + }, + Err(err) => engine_error_response(err), + } + } + + async fn get_payload_bodies_by_hash( + &self, + fork: EngineSszFork, + hashes: Vec, + ) -> HttpResponse { + handle_get_payload_bodies(self.clone(), fork, PayloadBodiesRequest::Hash(hashes)).await + } + + async fn get_payload_bodies_by_range( + &self, + fork: EngineSszFork, + start: u64, + count: u64, + ) -> HttpResponse { + handle_get_payload_bodies(self.clone(), fork, PayloadBodiesRequest::Range { start, count }) + .await + } + + async fn get_blobs(&self, version: u8, body: Bytes) -> HttpResponse { + if version == 4 { + let request = match BlobsV4Request::from_ssz_bytes(&body) { + Ok(request) => request, + Err(_) => return problem_response(STATUS_BAD_REQUEST, "ssz-decode-error", None), + }; + if request.versioned_hashes.len() > MAX_BLOB_LIMIT { + return problem_response(STATUS_PAYLOAD_TOO_LARGE, "request-too-large", None) + } + return blob_response::( + self.get_blobs_v4_metered(request.versioned_hashes, request.indices_bitarray), + ) + } + let request = match BlobsV1Request::from_ssz_bytes(&body) { + Ok(request) => request, + Err(_) => return problem_response(STATUS_BAD_REQUEST, "ssz-decode-error", None), + }; + if request.versioned_hashes.len() > MAX_BLOB_LIMIT { + return problem_response(STATUS_PAYLOAD_TOO_LARGE, "request-too-large", None) + } + let hashes = request.versioned_hashes; + match version { + 1 => blob_response::(self.get_blobs_v1_metered(hashes).map(Some)), + 2 => blob_response::(self.get_blobs_v2_metered(hashes)), + 3 => blob_response::(self.get_blobs_v3_metered(hashes)), + _ => problem_response(STATUS_NOT_FOUND, "method-not-found", None), + } + } +} + +async fn handle_engine_ssz_request( + handle: EngineSszProxyHandle, + request: HttpRequest, +) -> HttpResponse +where + Api: EngineSszApi, { let method = request.method().as_str().to_owned(); let path = request.uri().path().to_owned(); let Some(endpoint) = parse_engine_path(&path) else { - return text_response(STATUS_NOT_FOUND, "unknown engine ssz endpoint") + return problem_response(STATUS_NOT_FOUND, "method-not-found", None) }; match endpoint { EngineSszEndpoint::Capabilities => { if method != "GET" { - return text_response(STATUS_METHOD_NOT_ALLOWED, "method not allowed") + return problem_response(STATUS_METHOD_NOT_ALLOWED, "method-not-allowed", None) } - handle_capabilities() + let Some(engine_api) = handle.engine_api().await else { + return problem_response(STATUS_SERVICE_UNAVAILABLE, "service-unavailable", None) + }; + engine_api.capabilities() } EngineSszEndpoint::Identity => { if method != "GET" { - return text_response(STATUS_METHOD_NOT_ALLOWED, "method not allowed") + return problem_response(STATUS_METHOD_NOT_ALLOWED, "method-not-allowed", None) } let Some(engine_api) = handle.engine_api().await else { - return text_response(STATUS_SERVICE_UNAVAILABLE, "engine api unavailable") + return problem_response(STATUS_SERVICE_UNAVAILABLE, "service-unavailable", None) }; - handle_identity(engine_api) + engine_api.identity() } EngineSszEndpoint::NewPayload => { if method != "POST" { - return text_response(STATUS_METHOD_NOT_ALLOWED, "method not allowed") + return problem_response(STATUS_METHOD_NOT_ALLOWED, "method-not-allowed", None) } let Some(fork) = request_fork(&request) else { - return text_response(STATUS_BAD_REQUEST, "unsupported fork") + return problem_response(STATUS_BAD_REQUEST, "unsupported-fork", None) }; - let Ok(body) = request.into_body().collect().await.map(|body| body.to_bytes()) else { - return text_response(STATUS_BAD_REQUEST, "failed to read request body") + let body = match read_ssz_body(request, MAX_PAYLOAD_BYTES).await { + Ok(body) => body, + Err(response) => return response, }; let Some(engine_api) = handle.engine_api().await else { - return text_response(STATUS_SERVICE_UNAVAILABLE, "engine api unavailable") + return problem_response(STATUS_SERVICE_UNAVAILABLE, "service-unavailable", None) }; - handle_new_payload(engine_api, fork, &body).await + engine_api.new_payload(fork, body).await } EngineSszEndpoint::GetPayload(payload_id) => { if method != "GET" { - return text_response(STATUS_METHOD_NOT_ALLOWED, "method not allowed") + return problem_response(STATUS_METHOD_NOT_ALLOWED, "method-not-allowed", None) } let Ok(payload_id) = payload_id else { - return text_response(STATUS_BAD_REQUEST, "invalid payload id") + return problem_response(STATUS_BAD_REQUEST, "invalid-request", None) }; let Some(fork) = request_fork(&request) else { - return text_response(STATUS_BAD_REQUEST, "unsupported fork") + return problem_response(STATUS_BAD_REQUEST, "unsupported-fork", None) }; let Some(engine_api) = handle.engine_api().await else { - return text_response(STATUS_SERVICE_UNAVAILABLE, "engine api unavailable") + return problem_response(STATUS_SERVICE_UNAVAILABLE, "service-unavailable", None) }; - handle_get_payload(engine_api, fork, payload_id).await + engine_api.get_payload(fork, payload_id).await } EngineSszEndpoint::Forkchoice => { if method != "POST" { - return text_response(STATUS_METHOD_NOT_ALLOWED, "method not allowed") + return problem_response(STATUS_METHOD_NOT_ALLOWED, "method-not-allowed", None) } let Some(fork) = request_fork(&request) else { - return text_response(STATUS_BAD_REQUEST, "unsupported fork") + return problem_response(STATUS_BAD_REQUEST, "unsupported-fork", None) }; - let Ok(body) = request.into_body().collect().await.map(|body| body.to_bytes()) else { - return text_response(STATUS_BAD_REQUEST, "failed to read request body") + let body = match read_ssz_body(request, MAX_PAYLOAD_BYTES).await { + Ok(body) => body, + Err(response) => return response, }; let Some(engine_api) = handle.engine_api().await else { - return text_response(STATUS_SERVICE_UNAVAILABLE, "engine api unavailable") + return problem_response(STATUS_SERVICE_UNAVAILABLE, "service-unavailable", None) }; - handle_forkchoice_updated(engine_api, fork, &body).await + engine_api.forkchoice_updated(fork, body).await } EngineSszEndpoint::PayloadBodiesByHash => { if method != "POST" { @@ -281,13 +557,13 @@ where let Some(fork) = request_fork(&request) else { return problem_response(STATUS_BAD_REQUEST, "unsupported-fork", None) }; - let body = match read_bodies_hash_body(request).await { + let body = match read_ssz_body(request, MAX_BODIES_REQUEST_BYTES).await { Ok(body) => body, Err(response) => return response, }; let request = match BodiesByHashRequest::from_ssz_bytes(&body) { Ok(request) if request.block_hashes.len() <= MAX_BODIES_REQUEST => { - PayloadBodiesRequest::Hash(request.block_hashes) + request.block_hashes } Ok(_) => { return problem_response(STATUS_PAYLOAD_TOO_LARGE, "request-too-large", None) @@ -297,7 +573,7 @@ where let Some(engine_api) = handle.engine_api().await else { return problem_response(STATUS_SERVICE_UNAVAILABLE, "service-unavailable", None) }; - handle_get_payload_bodies(engine_api, fork, request).await + engine_api.get_payload_bodies_by_hash(fork, request).await } EngineSszEndpoint::PayloadBodiesByRange => { if method != "GET" { @@ -314,28 +590,28 @@ where let Some(engine_api) = handle.engine_api().await else { return problem_response(STATUS_SERVICE_UNAVAILABLE, "service-unavailable", None) }; - handle_get_payload_bodies( - engine_api, - fork, - PayloadBodiesRequest::Range { start, count }, - ) - .await + engine_api.get_payload_bodies_by_range(fork, start, count).await } EngineSszEndpoint::Blobs(version) => { if method != "POST" { - return text_response(STATUS_METHOD_NOT_ALLOWED, "method not allowed") + return problem_response(STATUS_METHOD_NOT_ALLOWED, "method-not-allowed", None) } - let Ok(body) = request.into_body().collect().await.map(|body| body.to_bytes()) else { - return text_response(STATUS_BAD_REQUEST, "failed to read request body") + let body = match read_ssz_body(request, MAX_BLOB_REQUEST_BYTES).await { + Ok(body) => body, + Err(response) => return response, }; let Some(engine_api) = handle.engine_api().await else { - return text_response(STATUS_SERVICE_UNAVAILABLE, "engine api unavailable") + return problem_response(STATUS_SERVICE_UNAVAILABLE, "service-unavailable", None) }; - handle_get_blobs(engine_api, version, &body).await + engine_api.get_blobs(version, body).await } } } +fn parse_method_version(version: &str) -> Option { + version.strip_prefix('v')?.parse().ok().filter(|version| (1..=4).contains(version)) +} + fn request_fork(request: &HttpRequest) -> Option { request.headers().get(ETH_EXECUTION_VERSION)?.to_str().ok()?.parse().ok() } @@ -384,57 +660,6 @@ enum EngineSszEndpoint { Blobs(u8), } -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -enum EngineSszFork { - Paris, - Shanghai, - Cancun, - Prague, - Osaka, - Amsterdam, -} - -impl EngineSszFork { - const fn payloads_version(self) -> u8 { - match self { - Self::Paris => 1, - Self::Shanghai => 2, - Self::Cancun => 3, - Self::Prague | Self::Osaka => 4, - Self::Amsterdam => 5, - } - } - - const fn forkchoice_version(self) -> u8 { - match self { - Self::Paris => 1, - Self::Shanghai => 2, - Self::Cancun | Self::Prague | Self::Osaka => 3, - Self::Amsterdam => 4, - } - } -} - -impl std::str::FromStr for EngineSszFork { - type Err = (); - - fn from_str(value: &str) -> Result { - match value { - "paris" => Ok(Self::Paris), - "shanghai" => Ok(Self::Shanghai), - "cancun" => Ok(Self::Cancun), - "prague" => Ok(Self::Prague), - "osaka" => Ok(Self::Osaka), - "amsterdam" => Ok(Self::Amsterdam), - _ => Err(()), - } - } -} - -fn parse_method_version(version: &str) -> Option { - version.strip_prefix('v')?.parse().ok().filter(|version| (1..=4).contains(version)) -} - fn handle_capabilities() -> HttpResponse { json_response(serde_json::json!({ "supported_forks": ["paris", "shanghai", "cancun", "prague", "osaka", "amsterdam"], @@ -451,146 +676,23 @@ fn handle_capabilities() -> HttpResponse { })) } -fn handle_identity( - engine_api: EthEngineApi, -) -> HttpResponse -where - Provider: HeaderProvider + BlockReader + StateProviderFactory + BalProvider + 'static, - Pool: TransactionPool + 'static, - Validator: EngineApiValidator, - ChainSpec: EthereumHardforks + Send + Sync + 'static, -{ - json_response(vec![engine_api.client_version().clone()]) -} - -async fn handle_get_payload( - engine_api: EthEngineApi, - fork: EngineSszFork, - payload_id: PayloadId, -) -> HttpResponse -where - Provider: HeaderProvider + BlockReader + StateProviderFactory + BalProvider + 'static, - Pool: TransactionPool + 'static, - Validator: EngineApiValidator, - ChainSpec: EthereumHardforks + Send + Sync + 'static, -{ - match fork { - EngineSszFork::Paris => match engine_api.get_payload_v2_metered(payload_id).await { - Ok(payload) => { - let block_value = payload.block_value; - match payload.execution_payload { - ExecutionPayloadFieldV2::V1(payload) => { - get_payload_response(BuiltPayloadParis { payload, block_value }) - } - ExecutionPayloadFieldV2::V2(_) => { - text_response(STATUS_BAD_REQUEST, "unsupported fork") - } - } - } - Err(err) => get_payload_error_response(err), - }, - EngineSszFork::Shanghai => match engine_api.get_payload_v2_metered(payload_id).await { - Ok(payload) => match BuiltPayloadShanghai::try_from(payload) { - Ok(payload) => get_payload_response(payload), - Err(err) => text_response(STATUS_BAD_REQUEST, err.to_string()), - }, - Err(err) => get_payload_error_response(err), - }, - EngineSszFork::Cancun => match engine_api.get_payload_v3_metered(payload_id).await { - Ok(payload) => get_payload_response(BuiltPayloadCancun::from(payload)), - Err(err) => get_payload_error_response(err), - }, - EngineSszFork::Prague => match engine_api.get_payload_v4_metered(payload_id).await { - Ok(payload) => get_payload_response(BuiltPayloadPrague::from(payload)), - Err(err) => get_payload_error_response(err), - }, - EngineSszFork::Osaka => match engine_api.get_payload_v5_metered(payload_id).await { - Ok(payload) => get_payload_response(BuiltPayloadOsaka::from(payload)), - Err(err) => get_payload_error_response(err), - }, - EngineSszFork::Amsterdam => match engine_api.get_payload_v6_metered(payload_id).await { - Ok(payload) => get_payload_response(BuiltPayloadAmsterdam::from(payload)), - Err(err) => get_payload_error_response(err), - }, - } -} - -fn get_payload_error_response(err: EngineApiError) -> HttpResponse { - let status = match &err { - EngineApiError::UnknownPayload => STATUS_NOT_FOUND, - EngineApiError::EngineObjectValidationError( - reth_payload_primitives::EngineObjectValidationError::UnsupportedFork, - ) => STATUS_BAD_REQUEST, - _ => STATUS_INTERNAL_SERVER_ERROR, - }; - text_response(status, err.to_string()) -} - -async fn handle_new_payload( - engine_api: EthEngineApi, - fork: EngineSszFork, - body: &[u8], -) -> HttpResponse -where - Provider: HeaderProvider + BlockReader + StateProviderFactory + BalProvider + 'static, - Pool: TransactionPool + 'static, - Validator: EngineApiValidator, - ChainSpec: EthereumHardforks + Send + Sync + 'static, -{ - let payload = match decode_new_payload_request(fork, body) { - Ok(payload) => payload, - Err(err) => return text_response(STATUS_BAD_REQUEST, err), - }; - - let response = match fork.payloads_version() { - 1 => engine_api.new_payload_v1(payload).await, - 2 => engine_api.new_payload_v2(payload).await, - 3 => engine_api.new_payload_v3(payload).await, - 4 => engine_api.new_payload_v4(payload).await, - 5 => engine_api.new_payload_v5(payload).await, - _ => return text_response(STATUS_BAD_REQUEST, "unsupported payload endpoint version"), - }; - - match response { - Ok(status) => match EngineSszPayloadStatus::try_from(status) { - Ok(status) => ssz_response(status), - Err(err) => text_response(STATUS_INTERNAL_SERVER_ERROR, err.to_string()), - }, - Err(err) => text_response(STATUS_INTERNAL_SERVER_ERROR, err.to_string()), - } -} - -async fn handle_forkchoice_updated( - engine_api: EthEngineApi, - fork: EngineSszFork, - body: &[u8], -) -> HttpResponse -where - Provider: HeaderProvider + BlockReader + StateProviderFactory + BalProvider + 'static, - Pool: TransactionPool + 'static, - Validator: EngineApiValidator, - ChainSpec: EthereumHardforks + Send + Sync + 'static, -{ - let (state, attrs, custody_columns) = match decode_forkchoice_request(fork, body) { - Ok(request) => request, - Err(err) => return text_response(STATUS_BAD_REQUEST, err), - }; - - let response = match fork.forkchoice_version() { - 1 => engine_api.fork_choice_updated_v1_metered(state, attrs).await, - 2 => engine_api.fork_choice_updated_v2_metered(state, attrs).await, - 3 => engine_api.fork_choice_updated_v3_metered(state, attrs).await, - 4 => engine_api.fork_choice_updated_v4_metered(state, attrs, custody_columns).await, - _ => return text_response(STATUS_BAD_REQUEST, "unsupported forkchoice endpoint version"), +fn engine_error_response(err: EngineApiError) -> HttpResponse { + let detail = err.to_string(); + let error: jsonrpsee::types::ErrorObjectOwned = err.into(); + let (status, problem_type) = match error.code() { + -32700 => (STATUS_BAD_REQUEST, "parse-error"), + -32600 => (STATUS_BAD_REQUEST, "invalid-request"), + -32601 => (STATUS_NOT_FOUND, "method-not-found"), + -32602 => (STATUS_UNPROCESSABLE_ENTITY, "invalid-body"), + -38001 => (STATUS_NOT_FOUND, "unknown-payload"), + -38002 => (STATUS_CONFLICT, "invalid-forkchoice"), + -38003 => (STATUS_UNPROCESSABLE_ENTITY, "invalid-attributes"), + -38004 => (STATUS_PAYLOAD_TOO_LARGE, "request-too-large"), + -38005 => (STATUS_BAD_REQUEST, "unsupported-fork"), + -38006 => (STATUS_CONFLICT, "reorg-too-deep"), + _ => (STATUS_INTERNAL_SERVER_ERROR, "internal"), }; - - match response { - Ok(updated) => match ForkchoiceUpdateResponse::try_from(updated) { - Ok(updated) => ssz_response(updated), - Err(err) => text_response(STATUS_INTERNAL_SERVER_ERROR, err.to_string()), - }, - Err(err) => text_response(STATUS_INTERNAL_SERVER_ERROR, err.to_string()), - } + problem_response(status, problem_type, Some(detail)) } enum PayloadBodiesRequest { @@ -677,7 +779,7 @@ where let bodies = bodies .into_iter() .map(|body| { - body.filter(|(timestamp, _)| body_matches_fork(chain_spec, fork, *timestamp)) + body.filter(|(timestamp, _)| timestamp_matches_fork(chain_spec, fork, *timestamp)) .map(|(_, body)| body) }) .collect(); @@ -695,11 +797,11 @@ where { match payload_bodies_response(response, convert, fork, chain_spec) { Ok(response) => ssz_response(response), - Err(err) => payload_bodies_error_response(err), + Err(err) => engine_error_response(err), } } -fn body_matches_fork( +fn timestamp_matches_fork( chain_spec: &ChainSpec, fork: EngineSszFork, timestamp: u64, @@ -723,18 +825,7 @@ fn body_matches_fork( } } -fn payload_bodies_error_response(err: EngineApiError) -> HttpResponse { - let (status, problem_type) = match &err { - EngineApiError::PayloadRequestTooLarge { .. } => { - (STATUS_PAYLOAD_TOO_LARGE, "request-too-large") - } - EngineApiError::InvalidBodiesRange { .. } => (422, "invalid-body"), - _ => (STATUS_INTERNAL_SERVER_ERROR, "internal"), - }; - problem_response(status, problem_type, Some(err.to_string())) -} - -async fn read_bodies_hash_body(request: HttpRequest) -> Result, HttpResponse> { +async fn read_ssz_body(request: HttpRequest, max_bytes: usize) -> Result { let content_type = request.headers().get(CONTENT_TYPE).and_then(|value| value.to_str().ok()); if content_type != Some(OCTET_STREAM) { return Err(problem_response(STATUS_UNSUPPORTED_MEDIA_TYPE, "unsupported-media-type", None)) @@ -746,13 +837,13 @@ async fn read_bodies_hash_body(request: HttpRequest) -> Result, HttpResp else { return Err(problem_response(STATUS_BAD_REQUEST, "invalid-request", None)) }; - if content_length > MAX_BODIES_REQUEST_BYTES { + if content_length > max_bytes { return Err(problem_response(STATUS_PAYLOAD_TOO_LARGE, "request-too-large", None)) } } - match Limited::new(request.into_body(), MAX_BODIES_REQUEST_BYTES).collect().await { - Ok(body) => Ok(body.to_bytes().to_vec()), + match Limited::new(request.into_body(), max_bytes).collect().await { + Ok(body) => Ok(body.to_bytes().into()), Err(err) if err.downcast_ref::().is_some() => { Err(problem_response(STATUS_PAYLOAD_TOO_LARGE, "request-too-large", None)) } @@ -760,208 +851,194 @@ async fn read_bodies_hash_body(request: HttpRequest) -> Result, HttpResp } } -/// Handles SSZ `engine_getBlobsV*` requests with the node's blob store. -async fn handle_get_blobs( - engine_api: EthEngineApi, - version: u8, - body: &[u8], -) -> HttpResponse +fn blob_response(response: Result, EngineApiError>) -> HttpResponse where - Provider: HeaderProvider + BlockReader + StateProviderFactory + BalProvider + 'static, - Pool: TransactionPool + 'static, - Validator: EngineApiValidator, - ChainSpec: EthereumHardforks + Send + Sync + 'static, + Ssz: TryFrom + ssz::Encode, + Ssz::Error: std::fmt::Display, { - match version { - 1 => { - let hashes = match decode_blob_hashes_request(body) { - Ok(hashes) => hashes, - Err(err) => return text_response(STATUS_BAD_REQUEST, err), - }; - match engine_api.get_blobs_v1_metered(hashes) { - Ok(response) => ssz_response(response), - Err(err) => text_response(STATUS_INTERNAL_SERVER_ERROR, err.to_string()), - } - } - 2 => { - let hashes = match decode_blob_hashes_request(body) { - Ok(hashes) => hashes, - Err(err) => return text_response(STATUS_BAD_REQUEST, err), - }; - match engine_api.get_blobs_v2_metered(hashes) { - Ok(Some(response)) => ssz_response(response), - Ok(None) => no_content_response(), - Err(err) => text_response(STATUS_INTERNAL_SERVER_ERROR, err.to_string()), - } - } - 3 => { - let hashes = match decode_blob_hashes_request(body) { - Ok(hashes) => hashes, - Err(err) => return text_response(STATUS_BAD_REQUEST, err), - }; - match engine_api.get_blobs_v3_metered(hashes) { - Ok(Some(response)) => ssz_response(response), - Ok(None) => no_content_response(), - Err(err) => text_response(STATUS_INTERNAL_SERVER_ERROR, err.to_string()), - } - } - 4 => { - let (hashes, indices_bitarray) = match decode_blob_cells_request(body) { - Ok(request) => request, - Err(err) => return text_response(STATUS_BAD_REQUEST, err), - }; - match engine_api.get_blobs_v4_metered(hashes, indices_bitarray) { - Ok(Some(response)) => ssz_response(response), - Ok(None) => no_content_response(), - Err(err) => text_response(STATUS_INTERNAL_SERVER_ERROR, err.to_string()), + match response { + Ok(Some(response)) => match Ssz::try_from(response) { + Ok(response) => ssz_response(response), + Err(err) => { + problem_response(STATUS_INTERNAL_SERVER_ERROR, "internal", Some(err.to_string())) } - } - _ => text_response(STATUS_NOT_FOUND, "unsupported blobs endpoint version"), + }, + Ok(None) => no_content_response(), + Err(err) => engine_error_response(err), + } +} + +/// Structural SSZ errors and invalid values have distinct REST error codes. +#[derive(Debug)] +enum PayloadDecodeError { + Ssz(ssz::DecodeError), + InvalidTransaction(alloy_eips::eip2718::Eip2718Error), +} + +impl From for PayloadDecodeError { + fn from(error: ssz::DecodeError) -> Self { + Self::Ssz(error) } } -/// Decodes the common getBlobs request container with only versioned hashes. -fn decode_blob_hashes_request(body: &[u8]) -> Result, &'static str> { - Vec::::from_ssz_bytes(body).map_err(|_| "invalid ssz") +impl PayloadDecodeError { + fn into_response(self) -> HttpResponse { + match self { + Self::Ssz(error) => { + problem_response(STATUS_BAD_REQUEST, "ssz-decode-error", Some(format!("{error:?}"))) + } + Self::InvalidTransaction(error) => problem_response( + STATUS_UNPROCESSABLE_ENTITY, + "invalid-body", + Some(error.to_string()), + ), + } + } } -/// Decodes the Amsterdam getBlobs request container with hashes and a cell index mask. -fn decode_blob_cells_request(body: &[u8]) -> Result<(Vec, B128), &'static str> { - <(Vec, B128) as ssz::Decode>::from_ssz_bytes(body).map_err(|_| "invalid ssz") +fn check_ssz_bound(actual: usize, max: usize, field: &str) -> Result<(), ssz::DecodeError> { + if actual > max { + return Err(ssz::DecodeError::BytesInvalid(format!( + "{field} exceeds SSZ bound: {actual} > {max}" + ))) + } + Ok(()) } fn decode_new_payload_request( fork: EngineSszFork, body: &[u8], -) -> Result { - match fork { +) -> Result { + let (payload, parent_root, requests): (ExecutionPayload, _, Option) = match fork { EngineSszFork::Paris => { - let ExecutionPayloadEnvelopeParis { payload: execution_payload } = - ExecutionPayloadEnvelopeParis::from_ssz_bytes(body).map_err(|_| "invalid ssz")?; - Ok(ExecutionData::new(execution_payload.into(), ExecutionPayloadSidecar::none())) + let envelope = ExecutionPayloadEnvelopeParis::from_ssz_bytes(body)?; + (envelope.payload.into(), None, None) } EngineSszFork::Shanghai => { - let ExecutionPayloadEnvelopeShanghai { payload: execution_payload } = - ExecutionPayloadEnvelopeShanghai::from_ssz_bytes(body) - .map_err(|_| "invalid ssz")?; - Ok(ExecutionData::new(execution_payload.into(), ExecutionPayloadSidecar::none())) + let envelope = ExecutionPayloadEnvelopeShanghai::from_ssz_bytes(body)?; + (envelope.payload.into(), None, None) } EngineSszFork::Cancun => { - let ExecutionPayloadEnvelopeCancun { - payload: execution_payload, - parent_beacon_block_root, - } = ExecutionPayloadEnvelopeCancun::from_ssz_bytes(body).map_err(|_| "invalid ssz")?; - let versioned_hashes = calculate_versioned_hashes( - &execution_payload.payload_inner.payload_inner.transactions, - )?; - let sidecar = ExecutionPayloadSidecar::v3(CancunPayloadFields { - parent_beacon_block_root, - versioned_hashes, - }); - Ok(ExecutionData::new(execution_payload.into(), sidecar)) + let envelope = ExecutionPayloadEnvelopeCancun::from_ssz_bytes(body)?; + (envelope.payload.into(), Some(envelope.parent_beacon_block_root), None) } EngineSszFork::Prague => { - let ExecutionPayloadEnvelopePrague { - payload: execution_payload, - parent_beacon_block_root, - execution_requests, - } = ExecutionPayloadEnvelopePrague::from_ssz_bytes(body).map_err(|_| "invalid ssz")?; - let versioned_hashes = calculate_versioned_hashes( - &execution_payload.payload_inner.payload_inner.transactions, - )?; - let sidecar = ExecutionPayloadSidecar::v4( - CancunPayloadFields { parent_beacon_block_root, versioned_hashes }, - PraguePayloadFields::new(RequestsOrHash::Requests(execution_requests)), - ); - Ok(ExecutionData::new(execution_payload.into(), sidecar)) + let envelope = ExecutionPayloadEnvelopePrague::from_ssz_bytes(body)?; + ( + envelope.payload.into(), + Some(envelope.parent_beacon_block_root), + Some(envelope.execution_requests), + ) } EngineSszFork::Osaka => { - let ExecutionPayloadEnvelopeOsaka { - payload: execution_payload, - parent_beacon_block_root, - execution_requests, - } = ExecutionPayloadEnvelopeOsaka::from_ssz_bytes(body).map_err(|_| "invalid ssz")?; - let versioned_hashes = calculate_versioned_hashes( - &execution_payload.payload_inner.payload_inner.transactions, - )?; - let sidecar = ExecutionPayloadSidecar::v4( - CancunPayloadFields { parent_beacon_block_root, versioned_hashes }, - PraguePayloadFields::new(RequestsOrHash::Requests(execution_requests)), - ); - Ok(ExecutionData::new(execution_payload.into(), sidecar)) + let envelope = ExecutionPayloadEnvelopeOsaka::from_ssz_bytes(body)?; + ( + envelope.payload.into(), + Some(envelope.parent_beacon_block_root), + Some(envelope.execution_requests), + ) } EngineSszFork::Amsterdam => { - let ExecutionPayloadEnvelopeAmsterdam { - payload: execution_payload, - parent_beacon_block_root, - execution_requests, - } = ExecutionPayloadEnvelopeAmsterdam::from_ssz_bytes(body) - .map_err(|_| "invalid ssz")?; - let versioned_hashes = calculate_versioned_hashes( - &execution_payload.payload_inner.payload_inner.payload_inner.transactions, - )?; - let sidecar = ExecutionPayloadSidecar::v4( - CancunPayloadFields { parent_beacon_block_root, versioned_hashes }, - PraguePayloadFields::new(RequestsOrHash::Requests(execution_requests)), - ); - Ok(ExecutionData::new(ExecutionPayload::V4(execution_payload), sidecar)) + let envelope = ExecutionPayloadEnvelopeAmsterdam::from_ssz_bytes(body)?; + ( + ExecutionPayload::V4(envelope.payload), + Some(envelope.parent_beacon_block_root), + Some(envelope.execution_requests), + ) } + }; + + // Alloy's codecs reproduce the wire layout but do not enforce the SSZ list bounds. + // Check those independently of the smaller, advertised total HTTP request limit. + let inner = payload.as_v1(); + check_ssz_bound(inner.extra_data.len(), 32, "extra_data")?; + check_ssz_bound(inner.transactions.len(), 1 << 20, "transactions")?; + for transaction in &inner.transactions { + check_ssz_bound(transaction.len(), 1 << 30, "transaction")?; } + if let Some(withdrawals) = payload.withdrawals() { + check_ssz_bound(withdrawals.len(), 16, "withdrawals")?; + } + if let Some(bal) = payload.block_access_list() { + check_ssz_bound(bal.len(), 1 << 30, "block_access_list")?; + } + if let Some(requests) = &requests { + check_ssz_bound(requests.len(), 256, "execution_requests")?; + for request in requests.iter() { + check_ssz_bound(request.len(), 1 << 30, "execution_request")?; + } + } + + let versioned_hashes = calculate_versioned_hashes(&inner.transactions)?; + let sidecar = match parent_root { + Some(parent_beacon_block_root) => { + let cancun = CancunPayloadFields { parent_beacon_block_root, versioned_hashes }; + match requests { + Some(requests) => { + ExecutionPayloadSidecar::v4(cancun, PraguePayloadFields::new(requests)) + } + None => ExecutionPayloadSidecar::v3(cancun), + } + } + None => ExecutionPayloadSidecar::none(), + }; + Ok(ExecutionData::new(payload, sidecar)) } -fn calculate_versioned_hashes(transactions: &[Bytes]) -> Result, &'static str> { +fn calculate_versioned_hashes(transactions: &[Bytes]) -> Result, PayloadDecodeError> { let mut versioned_hashes = Vec::new(); for transaction in transactions { - let transaction = - TxEnvelope::decode_2718_exact(transaction.as_ref()).map_err(|_| "invalid tx")?; + let transaction = TxEnvelope::decode_2718_exact(transaction.as_ref()) + .map_err(PayloadDecodeError::InvalidTransaction)?; if let Some(hashes) = transaction.blob_versioned_hashes() { versioned_hashes.extend_from_slice(hashes); } } - Ok(versioned_hashes) } fn decode_forkchoice_request( fork: EngineSszFork, body: &[u8], -) -> Result<(ForkchoiceState, Option, Option), &'static str> { - match fork { +) -> Result<(ForkchoiceState, Option, Option), ssz::DecodeError> { + let (state, attrs, custody) = match fork { EngineSszFork::Paris => { let ForkchoiceUpdateParis { forkchoice_state, payload_attributes } = - ForkchoiceUpdateParis::from_ssz_bytes(body).map_err(|_| "invalid ssz")?; - Ok((forkchoice_state, optional_attrs(payload_attributes), None)) + ForkchoiceUpdateParis::from_ssz_bytes(body)?; + (forkchoice_state, optional_attrs(payload_attributes), None) } EngineSszFork::Shanghai => { let ForkchoiceUpdateShanghai { forkchoice_state, payload_attributes } = - ForkchoiceUpdateShanghai::from_ssz_bytes(body).map_err(|_| "invalid ssz")?; - Ok((forkchoice_state, optional_attrs(payload_attributes), None)) + ForkchoiceUpdateShanghai::from_ssz_bytes(body)?; + (forkchoice_state, optional_attrs(payload_attributes), None) } EngineSszFork::Cancun => { let ForkchoiceUpdateCancun { forkchoice_state, payload_attributes } = - ForkchoiceUpdateCancun::from_ssz_bytes(body).map_err(|_| "invalid ssz")?; - Ok((forkchoice_state, optional_attrs(payload_attributes), None)) + ForkchoiceUpdateCancun::from_ssz_bytes(body)?; + (forkchoice_state, optional_attrs(payload_attributes), None) } EngineSszFork::Prague => { let ForkchoiceUpdatePrague { forkchoice_state, payload_attributes } = - ForkchoiceUpdatePrague::from_ssz_bytes(body).map_err(|_| "invalid ssz")?; - Ok((forkchoice_state, optional_attrs(payload_attributes), None)) + ForkchoiceUpdatePrague::from_ssz_bytes(body)?; + (forkchoice_state, optional_attrs(payload_attributes), None) } EngineSszFork::Osaka => { let ForkchoiceUpdateOsaka { forkchoice_state, payload_attributes } = - ForkchoiceUpdateOsaka::from_ssz_bytes(body).map_err(|_| "invalid ssz")?; - Ok((forkchoice_state, optional_attrs(payload_attributes), None)) + ForkchoiceUpdateOsaka::from_ssz_bytes(body)?; + (forkchoice_state, optional_attrs(payload_attributes), None) } EngineSszFork::Amsterdam => { let ForkchoiceUpdateAmsterdam { forkchoice_state, payload_attributes, custody_columns } = - ForkchoiceUpdateAmsterdam::from_ssz_bytes(body).map_err(|_| "invalid ssz")?; - Ok(( - forkchoice_state, - optional_attrs(payload_attributes), - custody_columns.into_option(), - )) + ForkchoiceUpdateAmsterdam::from_ssz_bytes(body)?; + (forkchoice_state, optional_attrs(payload_attributes), custody_columns.into_option()) } + }; + if let Some(withdrawals) = + attrs.as_ref().and_then(|attrs: &PayloadAttributes| attrs.withdrawals.as_ref()) + { + check_ssz_bound(withdrawals.len(), 16, "withdrawals")?; } + Ok((state, attrs, custody)) } fn optional_attrs(attrs: Optional) -> Option @@ -990,7 +1067,11 @@ fn get_payload_response(value: T) -> HttpResponse { fn json_response(value: T) -> HttpResponse { let Ok(body) = serde_json::to_string(&value) else { - return text_response(STATUS_INTERNAL_SERVER_ERROR, "failed to encode json") + return problem_response( + STATUS_INTERNAL_SERVER_ERROR, + "internal", + Some("failed to encode json".to_string()), + ) }; HttpResponse::builder() @@ -1004,14 +1085,6 @@ fn no_content_response() -> HttpResponse { HttpResponse::builder().status(204).body(HttpBody::empty()).expect("valid response") } -fn text_response(status: u16, body: impl Into) -> HttpResponse { - HttpResponse::builder() - .status(status) - .header(CONTENT_TYPE, TEXT_PLAIN) - .body(HttpBody::from(body.into())) - .expect("valid response") -} - fn problem_response( status: u16, problem_type: &'static str, @@ -1035,6 +1108,190 @@ mod tests { use super::*; use ssz::Encode; + #[test] + fn payload_schema_bounds() { + use alloy_rpc_types_engine::{ExecutionPayloadV1, ExecutionPayloadV2, ExecutionPayloadV3}; + let mut payload = ExecutionPayloadV3 { + payload_inner: ExecutionPayloadV2 { + payload_inner: ExecutionPayloadV1::from_block_unchecked( + B256::ZERO, + &reth_ethereum_primitives::Block::default(), + ), + withdrawals: vec![], + }, + blob_gas_used: 0, + excess_blob_gas: 0, + }; + payload.payload_inner.payload_inner.extra_data = vec![0; 32].into(); + payload.payload_inner.withdrawals = vec![Default::default(); 16]; + let mut envelope = ExecutionPayloadEnvelopePrague { + payload, + parent_beacon_block_root: B256::ZERO, + execution_requests: Requests::new(vec![Bytes::new(); 256]), + }; + assert!(decode_new_payload_request(EngineSszFork::Prague, &envelope.as_ssz_bytes()).is_ok()); + envelope.execution_requests = Requests::new(vec![Bytes::new(); 257]); + assert!(matches!( + decode_new_payload_request(EngineSszFork::Prague, &envelope.as_ssz_bytes()), + Err(PayloadDecodeError::Ssz(_)) + )); + envelope.execution_requests = Requests::default(); + envelope.payload.payload_inner.withdrawals.push(Default::default()); + assert!(matches!( + decode_new_payload_request(EngineSszFork::Prague, &envelope.as_ssz_bytes()), + Err(PayloadDecodeError::Ssz(_)) + )); + envelope.payload.payload_inner.withdrawals.clear(); + envelope.payload.payload_inner.payload_inner.extra_data = vec![0; 33].into(); + assert!(matches!( + decode_new_payload_request(EngineSszFork::Prague, &envelope.as_ssz_bytes()), + Err(PayloadDecodeError::Ssz(_)) + )); + let shanghai = ExecutionPayloadEnvelopeShanghai { + payload: ExecutionPayloadV2 { + payload_inner: ExecutionPayloadV1::from_block_unchecked( + B256::ZERO, + &reth_ethereum_primitives::Block::default(), + ), + withdrawals: vec![Default::default(); 17], + }, + }; + assert!(matches!( + decode_new_payload_request(EngineSszFork::Shanghai, &shanghai.as_ssz_bytes()), + Err(PayloadDecodeError::Ssz(_)) + )); + let paris = ExecutionPayloadEnvelopeParis { + payload: ExecutionPayloadV1 { + transactions: vec![Bytes::new(); (1 << 20) + 1], + ..ExecutionPayloadV1::from_block_unchecked( + B256::ZERO, + &reth_ethereum_primitives::Block::default(), + ) + }, + }; + assert!(matches!( + decode_new_payload_request(EngineSszFork::Paris, &paris.as_ssz_bytes()), + Err(PayloadDecodeError::Ssz(_)) + )); + } + + #[test] + fn forkchoice_withdrawal_bounds() { + use crate::engine_ssz_containers::{ + PayloadAttributesAmsterdam, PayloadAttributesCancun, PayloadAttributesShanghai, + }; + for count in [16, 17] { + let withdrawals = vec![Default::default(); count]; + let state = ForkchoiceState::default(); + let shanghai = ForkchoiceUpdateShanghai { + forkchoice_state: state, + payload_attributes: Optional::some(PayloadAttributesShanghai { + withdrawals: withdrawals.clone(), + ..Default::default() + }), + }; + let cancun = ForkchoiceUpdateCancun { + forkchoice_state: state, + payload_attributes: Optional::some(PayloadAttributesCancun { + withdrawals: withdrawals.clone(), + ..Default::default() + }), + }; + let amsterdam = ForkchoiceUpdateAmsterdam { + forkchoice_state: state, + payload_attributes: Optional::some(PayloadAttributesAmsterdam { + withdrawals, + ..Default::default() + }), + custody_columns: Optional::none(), + }; + for (fork, bytes) in [ + (EngineSszFork::Shanghai, shanghai.as_ssz_bytes()), + (EngineSszFork::Cancun, cancun.as_ssz_bytes()), + (EngineSszFork::Prague, cancun.as_ssz_bytes()), + (EngineSszFork::Osaka, cancun.as_ssz_bytes()), + (EngineSszFork::Amsterdam, amsterdam.as_ssz_bytes()), + ] { + assert_eq!( + decode_forkchoice_request(fork, &bytes).is_ok(), + count == 16, + "{fork:?}" + ); + } + } + } + + #[test] + fn paris_forkchoice_uses_fixed_size_optional_attributes() { + use crate::engine_ssz_containers::PayloadAttributesParis; + let request = ForkchoiceUpdateParis { + forkchoice_state: ForkchoiceState::default(), + payload_attributes: Optional::some(PayloadAttributesParis { + timestamp: 1_700_000_000, + ..Default::default() + }), + }; + let bytes = request.as_ssz_bytes(); + assert_eq!(bytes.len(), 160); + assert_eq!(&bytes[96..100], &100u32.to_le_bytes()); + assert_eq!(&bytes[100..108], &1_700_000_000u64.to_le_bytes()); + assert_eq!( + decode_forkchoice_request(EngineSszFork::Paris, &bytes).unwrap().1.unwrap().timestamp, + 1_700_000_000 + ); + } + + #[tokio::test] + async fn post_body_limits_apply_without_content_length() { + for size in [16, 17] { + let request = HttpRequest::builder() + .header(CONTENT_TYPE, OCTET_STREAM) + .body(HttpBody::from(vec![0; size])) + .unwrap(); + let response = read_ssz_body(request, 16).await; + if size == 16 { + assert_eq!(response.unwrap().len(), size); + } else { + assert_eq!(response.unwrap_err().status(), STATUS_PAYLOAD_TOO_LARGE); + } + } + } + + #[tokio::test] + async fn engine_errors_preserve_validation_semantics() { + use alloy_rpc_types_engine::ForkchoiceUpdateError; + use reth_engine_primitives::BeaconForkChoiceUpdateError; + for (error, status, kind) in [ + (EngineApiError::UnknownPayload, 404, "unknown-payload"), + (EngineApiError::BlobRequestTooLarge { len: 129 }, 413, "request-too-large"), + ( + EngineApiError::ForkChoiceUpdate( + BeaconForkChoiceUpdateError::ForkchoiceUpdateError( + ForkchoiceUpdateError::InvalidState, + ), + ), + 409, + "invalid-forkchoice", + ), + ( + EngineApiError::ForkChoiceUpdate( + BeaconForkChoiceUpdateError::ForkchoiceUpdateError( + ForkchoiceUpdateError::UpdatedInvalidPayloadAttributes, + ), + ), + 422, + "invalid-attributes", + ), + ] { + let response = engine_error_response(error); + assert_eq!(response.status(), status); + assert_eq!(response.headers()[CONTENT_TYPE], PROBLEM_JSON); + let body = response.into_body().collect().await.unwrap().to_bytes(); + let problem: serde_json::Value = serde_json::from_slice(&body).unwrap(); + assert_eq!(problem["type"], format!("/engine-api/errors/{kind}")); + } + } + #[test] fn bodies_range_query_validation() { assert_eq!(parse_bodies_range_query("from=1&count=32").unwrap(), (1, 32)); @@ -1090,10 +1347,10 @@ mod tests { assert_eq!(response.entries[index].body, ExecutionPayloadBodyShanghai::default()); } } - assert!(body_matches_fork(&chain_spec, EngineSszFork::Paris, 9)); - assert!(!body_matches_fork(&chain_spec, EngineSszFork::Paris, 10)); - assert!(!body_matches_fork(&chain_spec, EngineSszFork::Amsterdam, 49)); - assert!(body_matches_fork(&chain_spec, EngineSszFork::Amsterdam, 50)); + assert!(timestamp_matches_fork(&chain_spec, EngineSszFork::Paris, 9)); + assert!(!timestamp_matches_fork(&chain_spec, EngineSszFork::Paris, 10)); + assert!(!timestamp_matches_fork(&chain_spec, EngineSszFork::Amsterdam, 49)); + assert!(timestamp_matches_fork(&chain_spec, EngineSszFork::Amsterdam, 50)); } #[tokio::test] @@ -1106,8 +1363,11 @@ mod tests { if content_length { request = request.header("content-length", bytes.len()); } - let result = - read_bodies_hash_body(request.body(HttpBody::from(bytes)).unwrap()).await; + let result = read_ssz_body( + request.body(HttpBody::from(bytes)).unwrap(), + MAX_BODIES_REQUEST_BYTES, + ) + .await; if count == 32 { assert_eq!( BodiesByHashRequest::from_ssz_bytes(&result.unwrap()) @@ -1123,17 +1383,16 @@ mod tests { } } } - let response = - read_bodies_hash_body(HttpRequest::new(HttpBody::empty())).await.unwrap_err(); + let response = read_ssz_body(HttpRequest::new(HttpBody::empty()), MAX_BODIES_REQUEST_BYTES) + .await + .unwrap_err(); assert_eq!(response.status(), STATUS_UNSUPPORTED_MEDIA_TYPE); } #[tokio::test] async fn payload_bodies_errors_and_capabilities_match_rest_spec() { - let response = payload_bodies_error_response(EngineApiError::InvalidBodiesRange { - start: 0, - count: 1, - }); + let response = + engine_error_response(EngineApiError::InvalidBodiesRange { start: 0, count: 1 }); assert_eq!(response.status(), 422); assert_eq!(response.headers()[CONTENT_TYPE], PROBLEM_JSON); let body = response.into_body().collect().await.unwrap().to_bytes(); @@ -1203,9 +1462,13 @@ mod tests { } #[test] - fn decodes_top_level_blob_hashes_request() { + fn decodes_blob_request_container() { let hashes = vec![B256::ZERO, B256::with_last_byte(1)]; - let decoded = decode_blob_hashes_request(&hashes.as_ssz_bytes()).unwrap(); + let decoded = BlobsV1Request::from_ssz_bytes( + &BlobsV1Request { versioned_hashes: hashes.clone() }.as_ssz_bytes(), + ) + .unwrap() + .versioned_hashes; assert_eq!(decoded, hashes); } diff --git a/crates/ethereum/node/src/node.rs b/crates/ethereum/node/src/node.rs index bc1b49ab7cf..442332553ee 100644 --- a/crates/ethereum/node/src/node.rs +++ b/crates/ethereum/node/src/node.rs @@ -1,6 +1,9 @@ //! Ethereum Node types config. -use crate::{EthEngineTypes, EthEvmConfig}; +use crate::{ + engine_ssz_proxy::{EngineSszApi, EngineSszProxyLayer}, + EthEngineTypes, EthEvmConfig, +}; use alloy_eips::{eip7840::BlobParams, merge::EPOCH_SLOTS}; use alloy_network::Ethereum; use alloy_rpc_types_engine::ExecutionData; @@ -33,7 +36,7 @@ use reth_node_builder::{ PayloadValidatorBuilder, RethAuthHttpMiddleware, RethRpcAddOns, RethRpcMiddleware, RpcAddOns, RpcHandle, Stack, }, - BuilderContext, DebugNode, Node, NodeAdapter, PayloadBuilderConfig, + BuilderContext, DebugNode, EngineApiExt, Node, NodeAdapter, PayloadBuilderConfig, }; use reth_node_core::args::JitArgs; use reth_payload_primitives::PayloadTypes; @@ -317,11 +320,13 @@ where EthB: EthApiBuilder, PVB: Send, EB: EngineApiBuilder, + EB::EngineApi: EngineSszApi, EVB: EngineValidatorBuilder, EthApiError: FromEvmError, EvmFactoryFor: EvmFactory, RpcMiddleware: RethRpcMiddleware, AuthHttpMiddleware: RethAuthHttpMiddleware, + Stack, AuthHttpMiddleware>: RethAuthHttpMiddleware, { type Handle = RpcHandle; @@ -346,7 +351,15 @@ where let testing_desired_gas_limit = ctx.config.builder.gas_limit_for(ctx.config.chain.chain()); let testing_engine_handle = ctx.beacon_engine_handle.clone(); + let (ssz_proxy_layer, ssz_proxy_handle) = EngineSszProxyLayer::new(); + self.inner + .map_engine_api(|engine_api_builder| { + EngineApiExt::new(engine_api_builder, move |engine_api| { + ssz_proxy_handle.set_engine_api_sync(engine_api); + }) + }) + .map_auth_http_middleware(|middleware| Stack::new(ssz_proxy_layer, middleware)) .launch_add_ons_with(ctx, move |container| { container.modules.merge_if_module_configured( RethRpcModule::Flashbots, @@ -395,11 +408,13 @@ where EthB: EthApiBuilder, PVB: PayloadValidatorBuilder, EB: EngineApiBuilder, + EB::EngineApi: EngineSszApi, EVB: EngineValidatorBuilder, EthApiError: FromEvmError, EvmFactoryFor: EvmFactory, RpcMiddleware: RethRpcMiddleware, AuthHttpMiddleware: RethAuthHttpMiddleware, + Stack, AuthHttpMiddleware>: RethAuthHttpMiddleware, { type EthApi = EthB::EthApi; diff --git a/crates/ethereum/node/tests/e2e/eth.rs b/crates/ethereum/node/tests/e2e/eth.rs index 0413f7187d6..a25d9738694 100644 --- a/crates/ethereum/node/tests/e2e/eth.rs +++ b/crates/ethereum/node/tests/e2e/eth.rs @@ -1,5 +1,5 @@ use crate::utils::{advance_with_random_transactions, eth_payload_attributes}; -use alloy_eips::{eip4844::BlobAndProofV1, eip7685::RequestsOrHash}; +use alloy_eips::eip7685::RequestsOrHash; use alloy_genesis::Genesis; use alloy_primitives::{Address, B256}; use alloy_rpc_types_engine::{ @@ -11,7 +11,7 @@ use reth_e2e_test_utils::{ node::NodeTestContext, setup, setup_engine, transaction::TransactionTestContext, wallet::Wallet, }; use reth_node_api::TreeConfig; -use reth_node_builder::{rpc::BasicEngineApiBuilder, EngineApiExt, NodeBuilder, NodeHandle}; +use reth_node_builder::{NodeBuilder, NodeHandle}; use reth_node_core::{ args::RpcServerArgs, node_config::NodeConfig, @@ -19,11 +19,11 @@ use reth_node_core::{ }; use reth_node_ethereum::{ engine_ssz_containers::{ - BodiesByHashRequest, BodiesResponsePrague, + BlobsV1Request, BlobsV1Response, BlobsV2Response, BlobsV3Response, BlobsV4Request, + BlobsV4Response, BodiesByHashRequest, BodiesResponsePrague, ForkchoiceUpdateResponse as SszForkchoiceUpdateResponse, PayloadStatus as SszPayloadStatus, }, - engine_ssz_proxy::EngineSszProxyLayer, - EthereumAddOns, EthereumEngineValidatorBuilder, EthereumNode, + EthereumAddOns, EthereumNode, }; use reth_provider::BlockNumReader; use reth_rpc_api::TestingBuildBlockRequestV1; @@ -302,23 +302,11 @@ async fn test_engine_ssz_proxy_can_mine_block() -> eyre::Result<()> { .with_http_api(reth_rpc_server_types::RpcModuleSelection::All), ); - let (ssz_layer, ssz_handle) = EngineSszProxyLayer::new(); - let engine_api_handle = ssz_handle.clone(); - let engine_api_builder = EngineApiExt::new( - BasicEngineApiBuilder::::default(), - move |engine_api| { - engine_api_handle.set_engine_api_sync(engine_api); - }, - ); let NodeHandle { node, node_exit_future: _ } = NodeBuilder::new(node_config) .testing_node(runtime) .with_types::() .with_components(EthereumNode::components()) - .with_add_ons( - EthereumAddOns::default() - .with_engine_api(engine_api_builder) - .with_auth_http_middleware(ssz_layer), - ) + .with_add_ons(EthereumAddOns::default()) .launch() .await?; @@ -353,6 +341,11 @@ async fn test_engine_ssz_proxy_can_mine_block() -> eyre::Result<()> { let auth_url = auth_server.http_url(); let auth_header = secret_to_bearer_header(auth_server.jwt_secret()); + for route in [ENGINE_CAPABILITIES_ROUTE, ENGINE_PAYLOADS_ROUTE] { + let response = client.get(format!("{auth_url}{route}")).send().await?; + assert_eq!(response.status(), reqwest::StatusCode::UNAUTHORIZED); + } + let capabilities_response = client .get(format!("{auth_url}{ENGINE_CAPABILITIES_ROUTE}")) .header(reqwest::header::AUTHORIZATION, auth_header.to_str()?) @@ -452,20 +445,22 @@ async fn test_engine_ssz_proxy_can_mine_block() -> eyre::Result<()> { let envelope = node.rpc.envelope_by_hash(blob_tx_hash).await?; let versioned_hashes = TransactionTestContext::validate_sidecar(envelope); + let mut requested_hashes = versioned_hashes.clone(); + requested_hashes.push(B256::ZERO); let blobs_response = client .post(format!("{auth_url}{ENGINE_V1_BLOBS_ROUTE}")) .header(reqwest::header::AUTHORIZATION, auth_header.to_str()?) .header(reqwest::header::CONTENT_TYPE, "application/octet-stream") .header(reqwest::header::ACCEPT, "application/octet-stream") - .body(versioned_hashes.as_ssz_bytes()) + .body(BlobsV1Request { versioned_hashes: requested_hashes }.as_ssz_bytes()) .send() .await?; assert_eq!(blobs_response.status(), reqwest::StatusCode::OK); - let blobs = - Vec::>::from_ssz_bytes(&blobs_response.bytes().await?).unwrap(); - assert_eq!(blobs.len(), versioned_hashes.len()); - assert!(blobs.iter().all(Option::is_some)); + let blobs = BlobsV1Response::from_ssz_bytes(&blobs_response.bytes().await?).unwrap(); + assert_eq!(blobs.entries.len(), versioned_hashes.len() + 1); + assert!(blobs.entries[..versioned_hashes.len()].iter().all(|entry| entry.available)); + assert!(!blobs.entries.last().unwrap().available); let fcu = SszForkchoiceUpdateResponse::from_ssz_bytes(&fcu_response.bytes().await?).unwrap(); assert_eq!(fcu.payload_status.status, PayloadStatusEnum::Valid); @@ -501,6 +496,97 @@ async fn test_engine_ssz_proxy_can_mine_block() -> eyre::Result<()> { Ok(()) } +#[tokio::test] +async fn test_engine_ssz_proxy_blob_revisions() -> eyre::Result<()> { + let chain_spec = Arc::new( + ChainSpecBuilder::default() + .chain(MAINNET.chain) + .genesis(serde_json::from_str(include_str!("../assets/genesis.json"))?) + .osaka_activated() + .build(), + ); + let (mut nodes, _) = + setup::(1, chain_spec, false, eth_payload_attributes).await?; + let mut node = nodes.pop().unwrap(); + node.advance_block().await?; + let client = reqwest::Client::new(); + let auth_server = node.auth_server_handle(); + let auth_url = auth_server.http_url(); + let auth_header = secret_to_bearer_header(auth_server.jwt_secret()); + // Missing blobs retain one unavailable entry, except V2's all-or-nothing response. + for version in 2..=4 { + let body = if version == 4 { + BlobsV4Request { + versioned_hashes: vec![B256::ZERO], + indices_bitarray: alloy_primitives::B128::ZERO, + } + .as_ssz_bytes() + } else { + BlobsV1Request { versioned_hashes: vec![B256::ZERO] }.as_ssz_bytes() + }; + let response = client + .post(format!("{auth_url}/engine/v1/blobs/v{version}")) + .header(reqwest::header::AUTHORIZATION, auth_header.to_str()?) + .header(reqwest::header::CONTENT_TYPE, "application/octet-stream") + .body(body) + .send() + .await?; + if version == 2 { + assert_eq!(response.status(), reqwest::StatusCode::NO_CONTENT); + assert!(response.bytes().await?.is_empty()); + continue + } + assert_eq!(response.status(), reqwest::StatusCode::OK); + let bytes = response.bytes().await?; + let availability = match version { + 3 => BlobsV3Response::from_ssz_bytes(&bytes) + .unwrap() + .entries + .into_iter() + .map(|entry| entry.available) + .collect::>(), + 4 => BlobsV4Response::from_ssz_bytes(&bytes) + .unwrap() + .entries + .into_iter() + .map(|entry| entry.available) + .collect(), + _ => unreachable!(), + }; + assert_eq!(availability, [false]); + } + // A container with no requested hashes is a successful empty V2 response. + let response = client + .post(format!("{auth_url}/engine/v1/blobs/v2")) + .header(reqwest::header::AUTHORIZATION, auth_header.to_str()?) + .header(reqwest::header::CONTENT_TYPE, "application/octet-stream") + .body(BlobsV1Request { versioned_hashes: vec![] }.as_ssz_bytes()) + .send() + .await?; + assert_eq!(response.status(), reqwest::StatusCode::OK); + assert!(BlobsV2Response::from_ssz_bytes(&response.bytes().await?).unwrap().entries.is_empty()); + + for (body, status) in [ + (vec![0; 32], reqwest::StatusCode::BAD_REQUEST), + ( + BlobsV1Request { versioned_hashes: vec![B256::ZERO; 129] }.as_ssz_bytes(), + reqwest::StatusCode::PAYLOAD_TOO_LARGE, + ), + ] { + let response = client + .post(format!("{auth_url}/engine/v1/blobs/v1")) + .header(reqwest::header::AUTHORIZATION, auth_header.to_str()?) + .header(reqwest::header::CONTENT_TYPE, "application/octet-stream") + .body(body) + .send() + .await?; + assert_eq!(response.status(), status); + assert_eq!(response.headers()[reqwest::header::CONTENT_TYPE], "application/problem+json"); + } + + Ok(()) +} + /// Tests that the sparse trie pipeline can be shared with the payload builder. /// /// Enables both `share_execution_cache_with_payload_builder` and @@ -596,3 +682,194 @@ async fn test_sparse_trie_reuse_across_blocks() -> eyre::Result<()> { Ok(()) } + +#[tokio::test] +async fn test_engine_ssz_request_validation() -> eyre::Result<()> { + use alloy_primitives::Bytes; + use reth_node_ethereum::engine_ssz_containers::{ + BuiltPayloadPrague, ExecutionPayloadEnvelopePrague, ExecutionPayloadPrague, + ForkchoiceUpdateCancun, Optional, PayloadAttributesCancun, + }; + + let chain = Arc::new( + ChainSpecBuilder::default() + .chain(MAINNET.chain) + .genesis(serde_json::from_str(include_str!("../assets/genesis.json"))?) + .prague_activated() + .build(), + ); + let (mut nodes, _) = + setup::(1, chain.clone(), false, eth_payload_attributes).await?; + let node = nodes.pop().unwrap(); + let auth = node.auth_server_handle(); + let url = auth.http_url(); + let jwt = secret_to_bearer_header(auth.jwt_secret()); + let client = reqwest::Client::new(); + let state = ForkchoiceState { + head_block_hash: chain.genesis_hash(), + safe_block_hash: chain.genesis_hash(), + finalized_block_hash: chain.genesis_hash(), + }; + for (fork, withdrawals, expected_error) in [ + ("cancun", 0, Some("unsupported-fork")), + ("osaka", 0, Some("unsupported-fork")), + ("prague", 17, Some("ssz-decode-error")), + ("prague", 16, None), + ] { + let attrs = PayloadAttributesCancun { + timestamp: chain.genesis().timestamp + 1, + withdrawals: vec![Default::default(); withdrawals], + ..Default::default() + }; + let response = client + .post(format!("{url}{ENGINE_FORKCHOICE_ROUTE}")) + .header("Authorization", jwt.to_str()?) + .header("Content-Type", "application/octet-stream") + .header(ENGINE_EXECUTION_VERSION_HEADER, fork) + .body( + ForkchoiceUpdateCancun { + forkchoice_state: state, + payload_attributes: Optional::some(attrs), + } + .as_ssz_bytes(), + ) + .send() + .await?; + if let Some(error) = expected_error { + assert_eq!(response.status(), reqwest::StatusCode::BAD_REQUEST); + assert_eq!( + response.json::().await?["type"], + format!("/engine-api/errors/{error}") + ); + } else { + assert_eq!(response.status(), reqwest::StatusCode::OK); + let fcu = + SszForkchoiceUpdateResponse::from_ssz_bytes(&response.bytes().await?).unwrap(); + assert!(matches!(fcu.payload_status.status, PayloadStatusEnum::Valid)); + let id = fcu.payload_id.into_option().unwrap(); + let response = client + .get(format!("{url}{ENGINE_PAYLOADS_ROUTE}/{id}")) + .header("Authorization", jwt.to_str()?) + .header(ENGINE_EXECUTION_VERSION_HEADER, fork) + .send() + .await?; + assert_eq!(response.status(), reqwest::StatusCode::OK); + let built = BuiltPayloadPrague::from_ssz_bytes(&response.bytes().await?).unwrap(); + assert_eq!(built.payload.payload_inner.withdrawals.len(), 16); + } + } + + // The timestamp restriction only applies to builds, not historical head updates. + let response = client + .post(format!("{url}{ENGINE_FORKCHOICE_ROUTE}")) + .header("Authorization", jwt.to_str()?) + .header("Content-Type", "application/octet-stream") + .header(ENGINE_EXECUTION_VERSION_HEADER, "cancun") + .body( + ForkchoiceUpdateCancun { + forkchoice_state: state, + payload_attributes: Optional::none(), + } + .as_ssz_bytes(), + ) + .send() + .await?; + assert_eq!(response.status(), reqwest::StatusCode::OK); + + for (extra_data_len, transactions, status, error) in [ + (33, vec![], reqwest::StatusCode::BAD_REQUEST, "ssz-decode-error"), + ( + 0, + vec![Bytes::from_static(&[2])], + reqwest::StatusCode::UNPROCESSABLE_ENTITY, + "invalid-body", + ), + ] { + let mut payload = ExecutionPayloadPrague { + payload_inner: alloy_rpc_types_engine::ExecutionPayloadV2 { + payload_inner: alloy_rpc_types_engine::ExecutionPayloadV1::from_block_unchecked( + B256::ZERO, + &reth_ethereum_primitives::Block::default(), + ), + withdrawals: vec![], + }, + blob_gas_used: 0, + excess_blob_gas: 0, + }; + payload.payload_inner.payload_inner.extra_data = vec![0; extra_data_len].into(); + payload.payload_inner.payload_inner.transactions = transactions; + let response = client + .post(format!("{url}{ENGINE_PAYLOADS_ROUTE}")) + .header("Authorization", jwt.to_str()?) + .header("Content-Type", "application/octet-stream") + .header(ENGINE_EXECUTION_VERSION_HEADER, "prague") + .body( + ExecutionPayloadEnvelopePrague { + payload, + parent_beacon_block_root: B256::ZERO, + execution_requests: Default::default(), + } + .as_ssz_bytes(), + ) + .send() + .await?; + assert_eq!(response.status(), status); + assert_eq!(response.headers()[reqwest::header::CONTENT_TYPE], "application/problem+json"); + assert_eq!( + response.json::().await?["type"], + format!("/engine-api/errors/{error}") + ); + } + Ok(()) +} + +#[tokio::test] +async fn test_engine_ssz_custom_engine_and_middleware() -> eyre::Result<()> { + use std::sync::atomic::{AtomicUsize, Ordering}; + let runtime = Runtime::test(); + let requests = Arc::new(AtomicUsize::new(0)); + let observed = requests.clone(); + let middleware = + tower::util::MapRequestLayer::new(move |request: jsonrpsee::server::HttpRequest| { + if request.uri().path().starts_with("/engine/") { + observed.fetch_add(1, Ordering::Relaxed); + } + request + }); + let chain = Arc::new( + ChainSpecBuilder::default() + .chain(MAINNET.chain) + .genesis(serde_json::from_str(include_str!("../assets/genesis.json"))?) + .prague_activated() + .build(), + ); + let NodeHandle { node, .. } = + NodeBuilder::new(NodeConfig::test().with_chain(chain).with_unused_ports()) + .testing_node(runtime) + .with_types::() + .with_components(EthereumNode::components()) + .with_add_ons( + EthereumAddOns::default() + .with_engine_api(reth_node_builder::rpc::NoopEngineApiBuilder::default()) + .layer_auth_http_middleware(middleware), + ) + .launch() + .await?; + let auth = &node.add_ons_handle.rpc_server_handles.auth; + let jwt = secret_to_bearer_header(auth.jwt_secret()); + let client = reqwest::Client::new(); + for route in [ENGINE_CAPABILITIES_ROUTE, ENGINE_IDENTITY_ROUTE] { + let response = client + .get(format!("{}{route}", auth.http_url())) + .header("Authorization", jwt.to_str()?) + .send() + .await?; + assert_eq!(response.status(), reqwest::StatusCode::NOT_FOUND); + assert_eq!( + response.json::().await?["type"], + "/engine-api/errors/method-not-found" + ); + } + assert_eq!(requests.load(Ordering::Relaxed), 2); + Ok(()) +} diff --git a/crates/node/builder/src/rpc.rs b/crates/node/builder/src/rpc.rs index 47af03d871d..d76fed14791 100644 --- a/crates/node/builder/src/rpc.rs +++ b/crates/node/builder/src/rpc.rs @@ -601,22 +601,30 @@ where pub fn with_engine_api( self, engine_api_builder: T, + ) -> RpcAddOns { + self.map_engine_api(|_| engine_api_builder) + } + + /// Maps the existing [`EngineApiBuilder`] builder value. + pub fn map_engine_api( + self, + f: impl FnOnce(EB) -> T, ) -> RpcAddOns { let Self { hooks, eth_api_builder, payload_validator_builder, + engine_api_builder, engine_validator_builder, rpc_middleware, auth_http_middleware, tokio_runtime, - .. } = self; RpcAddOns { hooks, eth_api_builder, payload_validator_builder, - engine_api_builder, + engine_api_builder: f(engine_api_builder), engine_validator_builder, rpc_middleware, auth_http_middleware, @@ -800,6 +808,33 @@ where } } + /// Maps the existing auth HTTP middleware, preserving its configuration. + pub fn map_auth_http_middleware( + self, + f: impl FnOnce(AuthHttpMiddleware) -> T, + ) -> RpcAddOns { + let Self { + hooks, + eth_api_builder, + payload_validator_builder, + engine_api_builder, + engine_validator_builder, + rpc_middleware, + auth_http_middleware, + tokio_runtime, + } = self; + RpcAddOns { + hooks, + eth_api_builder, + payload_validator_builder, + engine_api_builder, + engine_validator_builder, + rpc_middleware, + auth_http_middleware: f(auth_http_middleware), + tokio_runtime, + } + } + /// Conditionally stacks an HTTP transport middleware layer for the auth / Engine API server. #[expect(clippy::type_complexity)] pub fn option_layer_auth_http_middleware( From daa2d88233bcf8fe8aff9ddee718de116a96a0d5 Mon Sep 17 00:00:00 2001 From: Soubhik Singha Mahapatra <160333583+Soubhik-10@users.noreply.github.com> Date: Sat, 5 Sep 2026 15:10:51 +0000 Subject: [PATCH 180/294] feat(engine): add SSZ payload witness endpoint (#26406) Co-authored-by: Matthias Seitz --- Cargo.lock | 2 + crates/ethereum/node/Cargo.toml | 6 +- .../node/src/engine_ssz_containers.rs | 6 +- crates/ethereum/node/src/engine_ssz_proxy.rs | 305 +++++++++++++++--- .../ethereum/node/src/engine_ssz_witness.rs | 173 ++++++++++ crates/ethereum/node/src/lib.rs | 1 + crates/ethereum/node/src/node.rs | 6 + crates/ethereum/node/tests/e2e/eth.rs | 142 +++++++- crates/revm/src/witness.rs | 53 +-- 9 files changed, 619 insertions(+), 75 deletions(-) create mode 100644 crates/ethereum/node/src/engine_ssz_witness.rs diff --git a/Cargo.lock b/Cargo.lock index f8cd5633ed9..2369ed53aeb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9489,6 +9489,7 @@ dependencies = [ "alloy-network", "alloy-primitives", "alloy-provider", + "alloy-rlp", "alloy-rpc-types-beacon", "alloy-rpc-types-engine", "alloy-rpc-types-eth", @@ -9538,6 +9539,7 @@ dependencies = [ "reth-testing-utils", "reth-tracing", "reth-transaction-pool", + "reth-trie-common", "revm", "serde", "serde_json", diff --git a/crates/ethereum/node/Cargo.toml b/crates/ethereum/node/Cargo.toml index 82ce27c96ae..37de01f4c68 100644 --- a/crates/ethereum/node/Cargo.toml +++ b/crates/ethereum/node/Cargo.toml @@ -34,14 +34,17 @@ reth-rpc-server-types.workspace = true reth-node-api.workspace = true reth-node-core.workspace = true reth-chainspec.workspace = true -reth-revm = { workspace = true, features = ["std"] } +reth-revm = { workspace = true, features = ["std", "witness"] } reth-rpc-eth-types.workspace = true reth-engine-local.workspace = true reth-engine-primitives = { workspace = true, features = ["std"] } reth-payload-primitives.workspace = true +reth-tasks.workspace = true +reth-trie-common.workspace = true # ethereum alloy-consensus.workspace = true alloy-eips.workspace = true +alloy-rlp.workspace = true alloy-primitives.workspace = true jsonrpsee.workspace = true alloy-network.workspace = true @@ -126,4 +129,5 @@ test-utils = [ "reth-evm-ethereum/test-utils", "reth-stages-types/test-utils", "reth-tasks/test-utils", + "reth-trie-common/test-utils", ] diff --git a/crates/ethereum/node/src/engine_ssz_containers.rs b/crates/ethereum/node/src/engine_ssz_containers.rs index be042493a0c..ea5b0337b7d 100644 --- a/crates/ethereum/node/src/engine_ssz_containers.rs +++ b/crates/ethereum/node/src/engine_ssz_containers.rs @@ -1402,7 +1402,7 @@ pub type WitnessHeaderV1 = Vec; pub struct ExecutionWitnessV1 { /// Hashed trie-node preimages required during execution and state-root recomputation. pub state: Vec, - /// Contract bytecode preimages created or accessed during execution. + /// Contract bytecode preimages required from the pre-state. pub codes: Vec, /// RLP-encoded ancestor headers used for pre-state and `BLOCKHASH` correctness proofs. pub headers: Vec, @@ -1414,7 +1414,9 @@ pub type ExecutionWitness = ExecutionWitnessV1; /// REST-SSZ response for `POST /payloads/witness`. /// /// The witness uses the Engine REST-SSZ `Optional[T]` encoding from execution-apis and is present -/// only when the payload status is `VALID`. +/// only when the payload status is `VALID`. A `VALID` status without a witness means the parent +/// state was not yet available through the provider (the parent is only known to the engine +/// tree); resubmitting the payload once forkchoice has made the parent canonical yields it. #[derive(Clone, Debug, PartialEq, Eq, ssz_derive::Encode)] pub struct PayloadStatusWithWitness { /// Result of processing the submitted payload. diff --git a/crates/ethereum/node/src/engine_ssz_proxy.rs b/crates/ethereum/node/src/engine_ssz_proxy.rs index 915568fd1de..d79bcc0aaf7 100644 --- a/crates/ethereum/node/src/engine_ssz_proxy.rs +++ b/crates/ethereum/node/src/engine_ssz_proxy.rs @@ -4,17 +4,21 @@ //! //! [EIP-8178]: https://eips.ethereum.org/EIPS/eip-8178 -use crate::engine_ssz_containers::{ - BlobsV1Request, BlobsV1Response, BlobsV2Response, BlobsV3Response, BlobsV4Request, - BlobsV4Response, BodiesByHashRequest, BodiesResponse, BuiltPayloadAmsterdam, - BuiltPayloadCancun, BuiltPayloadOsaka, BuiltPayloadParis, BuiltPayloadPrague, - BuiltPayloadShanghai, ExecutionPayloadBodyAmsterdam, ExecutionPayloadBodyParis, - ExecutionPayloadBodyShanghai, ExecutionPayloadEnvelopeAmsterdam, - ExecutionPayloadEnvelopeCancun, ExecutionPayloadEnvelopeOsaka, ExecutionPayloadEnvelopeParis, - ExecutionPayloadEnvelopePrague, ExecutionPayloadEnvelopeShanghai, ForkchoiceUpdateAmsterdam, - ForkchoiceUpdateCancun, ForkchoiceUpdateOsaka, ForkchoiceUpdateParis, ForkchoiceUpdatePrague, - ForkchoiceUpdateResponse, ForkchoiceUpdateShanghai, Optional, - PayloadStatus as EngineSszPayloadStatus, +use crate::{ + engine_ssz_containers::{ + BlobsV1Request, BlobsV1Response, BlobsV2Response, BlobsV3Response, BlobsV4Request, + BlobsV4Response, BodiesByHashRequest, BodiesResponse, BuiltPayloadAmsterdam, + BuiltPayloadCancun, BuiltPayloadOsaka, BuiltPayloadParis, BuiltPayloadPrague, + BuiltPayloadShanghai, ExecutionPayloadBodyAmsterdam, ExecutionPayloadBodyParis, + ExecutionPayloadBodyShanghai, ExecutionPayloadEnvelopeAmsterdam, + ExecutionPayloadEnvelopeCancun, ExecutionPayloadEnvelopeOsaka, + ExecutionPayloadEnvelopeParis, ExecutionPayloadEnvelopePrague, + ExecutionPayloadEnvelopeShanghai, ForkchoiceUpdateAmsterdam, ForkchoiceUpdateCancun, + ForkchoiceUpdateOsaka, ForkchoiceUpdateParis, ForkchoiceUpdatePrague, + ForkchoiceUpdateResponse, ForkchoiceUpdateShanghai, Optional, + PayloadStatus as EngineSszPayloadStatus, PayloadStatusWithWitness, + }, + engine_ssz_witness::{EngineSszWitness, EngineSszWitnessError}, }; use alloy_consensus::{Transaction, TxEnvelope}; use alloy_eips::{eip2718::Decodable2718, eip7685::Requests}; @@ -22,7 +26,7 @@ use alloy_primitives::{Bytes, B128, B256}; use alloy_rpc_types_engine::{ CancunPayloadFields, ExecutionData, ExecutionPayload, ExecutionPayloadBodyV1, ExecutionPayloadFieldV2, ExecutionPayloadSidecar, ForkchoiceState, PayloadAttributes, - PayloadId, PraguePayloadFields, + PayloadId, PayloadStatusEnum, PraguePayloadFields, }; use futures::future::{BoxFuture, Either}; use http_body_util::{BodyExt, LengthLimitError, Limited}; @@ -33,6 +37,7 @@ use reth_ethereum_engine_primitives::EthEngineTypes; use reth_provider::{BalProvider, BlockReader, HeaderProvider, StateProviderFactory}; use reth_rpc::EngineApi; use reth_rpc_engine_api::EngineApiError; +use reth_tracing::tracing::debug; use reth_transaction_pool::TransactionPool; use ssz::Decode; use std::{ @@ -69,16 +74,14 @@ const PROBLEM_JSON: &str = "application/problem+json"; type EthEngineApi = EngineApi; -type SharedEngineApi = Arc>>; - /// Shared handle used by [`EngineSszProxyLayer`]. pub struct EngineSszProxyHandle { - engine_api: SharedEngineApi, + state: Arc>>, } impl Clone for EngineSszProxyHandle { fn clone(&self) -> Self { - Self { engine_api: self.engine_api.clone() } + Self { state: self.state.clone() } } } @@ -90,31 +93,72 @@ impl std::fmt::Debug for EngineSszProxyHandle { impl EngineSszProxyHandle { fn new() -> Self { - Self { engine_api: Default::default() } + Self { + state: Arc::new(RwLock::new(EngineSszState { + engine_api: None, + witness_handler: None, + witness_enabled: false, + })), + } } fn with_engine_api(engine_api: Api) -> Self { - Self { engine_api: Arc::new(RwLock::new(Some(engine_api))) } + Self { + state: Arc::new(RwLock::new(EngineSszState { + engine_api: Some(engine_api), + witness_handler: None, + witness_enabled: false, + })), + } + } + + /// Returns whether both the API and generator support the witness extension. + pub async fn witness_enabled(&self) -> bool { + self.state.read().await.witness_enabled + } + + /// Returns the configured witness generator. + pub async fn witness_handler(&self) -> Option> { + self.state.read().await.witness_handler.clone() } +} +impl EngineSszProxyHandle { /// Sets the Engine API implementation used by the proxy. pub async fn set_engine_api(&self, engine_api: Api) { - *self.engine_api.write().await = Some(engine_api); + let mut state = self.state.write().await; + state.engine_api = Some(engine_api); + state.update_witness_support(); } /// Sets the Engine API implementation during synchronous launch wiring. pub fn set_engine_api_sync(&self, engine_api: Api) { - *self - .engine_api - .try_write() - .expect("engine api handle should not be locked during launch") = Some(engine_api); + let mut state = + self.state.try_write().expect("engine api handle should not be locked during launch"); + state.engine_api = Some(engine_api); + state.update_witness_support(); + } + + /// Sets the witness generator used by `/payloads/witness`. + pub async fn set_witness_handler(&self, witness_handler: Arc) { + let mut state = self.state.write().await; + state.witness_handler = Some(witness_handler); + state.update_witness_support(); + } + + /// Sets the witness generator during synchronous launch wiring. + pub fn set_witness_handler_sync(&self, witness_handler: Arc) { + let mut state = + self.state.try_write().expect("witness handle should not be locked during launch"); + state.witness_handler = Some(witness_handler); + state.update_witness_support(); } } impl EngineSszProxyHandle { /// Returns the Engine API implementation used by the proxy. pub async fn engine_api(&self) -> Option { - self.engine_api.read().await.clone() + self.state.read().await.engine_api.clone() } } @@ -237,10 +281,27 @@ impl std::str::FromStr for EngineSszFork { /// `/capabilities`, then answers 404 so clients fall back to the JSON-RPC Engine API. pub trait EngineSszApi: Clone + Send + Sync + 'static { /// Returns the capabilities advertisement. - fn capabilities(&self) -> HttpResponse { + /// + /// `witness_enabled` is true when a witness generator is configured and + /// [`Self::supports_witness`] holds, so the advertisement can include the extension. + fn capabilities(&self, _witness_enabled: bool) -> HttpResponse { problem_response(STATUS_NOT_FOUND, "method-not-found", None) } + /// Whether the implementation supports the witness extension. + fn supports_witness(&self) -> bool { + false + } + + /// Handles the experimental Amsterdam payload submission with a witness. + fn new_payload_with_witness( + &self, + _body: Bytes, + _witness_handler: Arc, + ) -> impl Future + Send { + async { problem_response(STATUS_NOT_FOUND, "method-not-found", None) } + } + /// Returns the Engine API identity response. fn identity(&self) -> HttpResponse { problem_response(STATUS_NOT_FOUND, "method-not-found", None) @@ -308,8 +369,8 @@ where Validator: EngineApiValidator, ChainSpec: EthereumHardforks + Send + Sync + 'static, { - fn capabilities(&self) -> HttpResponse { - handle_capabilities() + fn capabilities(&self, witness_enabled: bool) -> HttpResponse { + handle_capabilities(witness_enabled) } fn identity(&self) -> HttpResponse { @@ -322,26 +383,55 @@ where Err(err) => return err.into_response(), }; - let response = match fork.payloads_version() { - 1 => self.new_payload_v1(payload).await, - 2 => self.new_payload_v2(payload).await, - 3 => self.new_payload_v3(payload).await, - 4 => self.new_payload_v4(payload).await, - 5 => self.new_payload_v5(payload).await, - _ => return problem_response(STATUS_BAD_REQUEST, "unsupported-fork", None), - }; + match submit_payload(self, fork, payload).await { + Ok(status) => ssz_response(status), + Err(response) => response, + } + } - match response { - Ok(status) => match EngineSszPayloadStatus::try_from(status) { - Ok(status) => ssz_response(status), - Err(err) => problem_response( - STATUS_INTERNAL_SERVER_ERROR, - "internal", - Some(err.to_string()), - ), + fn supports_witness(&self) -> bool { + true + } + + async fn new_payload_with_witness( + &self, + body: Bytes, + witness_handler: Arc, + ) -> HttpResponse { + let payload = match decode_new_payload_request(EngineSszFork::Amsterdam, &body) { + Ok(payload) => payload, + Err(err) => return err.into_response(), + }; + let status = match submit_payload(self, EngineSszFork::Amsterdam, payload.clone()).await { + Ok(status) => status, + Err(response) => return response, + }; + let witness = match status.status { + PayloadStatusEnum::Valid => match witness_handler.generate_witness(payload).await { + Ok(witness) => Some(witness), + // The block is valid but its parent is only known to the engine tree. The + // status stays authoritative; resubmitting once forkchoice has made the parent + // canonical yields the witness. + Err(EngineSszWitnessError::ParentStateUnavailable { parent, source }) => { + debug!( + target: "engine::ssz", + %parent, + %source, + "witness omitted for valid payload" + ); + None + } + Err(err) => { + return problem_response( + STATUS_INTERNAL_SERVER_ERROR, + "internal", + Some(err.to_string()), + ) + } }, - Err(err) => engine_error_response(err), - } + _ => None, + }; + ssz_response(PayloadStatusWithWitness::new(status, witness)) } async fn get_payload(&self, fork: EngineSszFork, payload_id: PayloadId) -> HttpResponse { @@ -471,6 +561,19 @@ where } } +struct EngineSszState { + engine_api: Option, + witness_handler: Option>, + witness_enabled: bool, +} + +impl EngineSszState { + fn update_witness_support(&mut self) { + self.witness_enabled = self.witness_handler.is_some() && + self.engine_api.as_ref().is_some_and(EngineSszApi::supports_witness); + } +} + async fn handle_engine_ssz_request( handle: EngineSszProxyHandle, request: HttpRequest, @@ -492,7 +595,7 @@ where let Some(engine_api) = handle.engine_api().await else { return problem_response(STATUS_SERVICE_UNAVAILABLE, "service-unavailable", None) }; - engine_api.capabilities() + engine_api.capabilities(handle.witness_enabled().await) } EngineSszEndpoint::Identity => { if method != "GET" { @@ -519,6 +622,28 @@ where }; engine_api.new_payload(fork, body).await } + EngineSszEndpoint::PayloadsWithWitness => { + if method != "POST" { + return problem_response(STATUS_METHOD_NOT_ALLOWED, "method-not-allowed", None) + } + let Some(fork) = request_fork(&request) else { + return problem_response(STATUS_BAD_REQUEST, "unsupported-fork", None) + }; + if fork != EngineSszFork::Amsterdam { + return problem_response(STATUS_BAD_REQUEST, "unsupported-fork", None) + } + let body = match read_ssz_body(request, MAX_PAYLOAD_BYTES).await { + Ok(body) => body, + Err(response) => return response, + }; + let Some(engine_api) = handle.engine_api().await else { + return problem_response(STATUS_SERVICE_UNAVAILABLE, "service-unavailable", None) + }; + let Some(witness_handler) = handle.witness_handler().await else { + return problem_response(STATUS_SERVICE_UNAVAILABLE, "service-unavailable", None) + }; + engine_api.new_payload_with_witness(body, witness_handler).await + } EngineSszEndpoint::GetPayload(payload_id) => { if method != "GET" { return problem_response(STATUS_METHOD_NOT_ALLOWED, "method-not-allowed", None) @@ -628,6 +753,9 @@ fn parse_engine_path(path: &str) -> Option { (Some("engine"), Some("v1"), Some("payloads"), None, None) => { Some(EngineSszEndpoint::NewPayload) } + (Some("engine"), Some("v1"), Some("payloads"), Some("witness"), None) => { + Some(EngineSszEndpoint::PayloadsWithWitness) + } (Some("engine"), Some("v1"), Some("payloads"), Some(payload_id), None) => { let payload_id = payload_id.parse::(); Some(EngineSszEndpoint::GetPayload(payload_id)) @@ -653,6 +781,7 @@ enum EngineSszEndpoint { Capabilities, Identity, NewPayload, + PayloadsWithWitness, GetPayload(Result::Err>), Forkchoice, PayloadBodiesByHash, @@ -660,10 +789,14 @@ enum EngineSszEndpoint { Blobs(u8), } -fn handle_capabilities() -> HttpResponse { +fn handle_capabilities(witness_enabled: bool) -> HttpResponse { + let mut fork_scoped_endpoints = vec!["payloads", "forkchoice", "bodies"]; + if witness_enabled { + fork_scoped_endpoints.push("payloads/witness"); + } json_response(serde_json::json!({ "supported_forks": ["paris", "shanghai", "cancun", "prague", "osaka", "amsterdam"], - "fork_scoped_endpoints": ["payloads", "forkchoice", "bodies"], + "fork_scoped_endpoints": fork_scoped_endpoints, "independently_versioned": { "blobs": ["v1", "v2", "v3", "v4"], }, @@ -676,6 +809,31 @@ fn handle_capabilities() -> HttpResponse { })) } +async fn submit_payload( + engine_api: &EthEngineApi, + fork: EngineSszFork, + payload: ExecutionData, +) -> Result +where + Provider: HeaderProvider + BlockReader + StateProviderFactory + BalProvider + 'static, + Pool: TransactionPool + 'static, + Validator: EngineApiValidator, + ChainSpec: EthereumHardforks + Send + Sync + 'static, +{ + let status = match fork.payloads_version() { + 1 => engine_api.new_payload_v1(payload).await, + 2 => engine_api.new_payload_v2(payload).await, + 3 => engine_api.new_payload_v3(payload).await, + 4 => engine_api.new_payload_v4(payload).await, + 5 => engine_api.new_payload_v5(payload).await, + _ => return Err(problem_response(STATUS_BAD_REQUEST, "unsupported-fork", None)), + } + .map_err(engine_error_response)?; + EngineSszPayloadStatus::try_from(status).map_err(|error| { + problem_response(STATUS_INTERNAL_SERVER_ERROR, "internal", Some(error.to_string())) + }) +} + fn engine_error_response(err: EngineApiError) -> HttpResponse { let detail = err.to_string(); let error: jsonrpsee::types::ErrorObjectOwned = err.into(); @@ -1108,6 +1266,55 @@ mod tests { use super::*; use ssz::Encode; + #[tokio::test] + async fn witness_capabilities_follow_both_wiring_orders() { + #[derive(Clone)] + struct Api(bool); + impl EngineSszApi for Api { + fn supports_witness(&self) -> bool { + self.0 + } + } + struct Witness; + impl EngineSszWitness for Witness { + fn generate_witness( + &self, + _: ExecutionData, + ) -> BoxFuture< + 'static, + Result, + > { + Box::pin(async { Ok(Default::default()) }) + } + } + let handle = EngineSszProxyHandle::new(); + handle.set_witness_handler_sync(Arc::new(Witness)); + assert!(!handle.witness_enabled().await); + handle.set_engine_api_sync(Api(true)); + assert!(handle.witness_enabled().await); + handle.set_engine_api(Api(false)).await; + assert!(!handle.witness_enabled().await); + + let handle = EngineSszProxyHandle::with_engine_api(Api(true)); + assert!(!handle.witness_enabled().await); + handle.set_witness_handler(Arc::new(Witness)).await; + assert!(handle.witness_enabled().await); + } + + #[tokio::test] + async fn witness_is_only_advertised_when_configured() { + for enabled in [false, true] { + let body = handle_capabilities(enabled).into_body().collect().await.unwrap().to_bytes(); + let capabilities: serde_json::Value = serde_json::from_slice(&body).unwrap(); + let endpoints = capabilities["fork_scoped_endpoints"].as_array().unwrap(); + assert_eq!(endpoints.iter().any(|endpoint| endpoint == "payloads/witness"), enabled); + } + assert_eq!( + parse_engine_path("/engine/v1/payloads/witness"), + Some(EngineSszEndpoint::PayloadsWithWitness) + ); + } + #[test] fn payload_schema_bounds() { use alloy_rpc_types_engine::{ExecutionPayloadV1, ExecutionPayloadV2, ExecutionPayloadV3}; @@ -1398,7 +1605,7 @@ mod tests { let body = response.into_body().collect().await.unwrap().to_bytes(); let problem: serde_json::Value = serde_json::from_slice(&body).unwrap(); assert_eq!(problem["type"], "/engine-api/errors/invalid-body"); - let body = handle_capabilities().into_body().collect().await.unwrap().to_bytes(); + let body = handle_capabilities(false).into_body().collect().await.unwrap().to_bytes(); let capabilities: serde_json::Value = serde_json::from_slice(&body).unwrap(); assert_eq!(capabilities["limits"]["bodies.max_count"], 32); } diff --git a/crates/ethereum/node/src/engine_ssz_witness.rs b/crates/ethereum/node/src/engine_ssz_witness.rs new file mode 100644 index 00000000000..ae245142e53 --- /dev/null +++ b/crates/ethereum/node/src/engine_ssz_witness.rs @@ -0,0 +1,173 @@ +//! Execution witness generation for the SSZ Engine API extension. + +use crate::engine_ssz_containers::ExecutionWitnessV1; +use alloy_primitives::B256; +use alloy_rpc_types_engine::ExecutionData; +use reth_ethereum_primitives::{EthPrimitives, TransactionSigned}; +use reth_evm::{execute::Executor, ConfigureEvm}; +use reth_primitives_traits::{AlloyBlockHeader, Block}; +use reth_provider::{HeaderProvider, StateProviderFactory}; +use reth_revm::{database::StateProviderDatabase, witness::ExecutionWitnessRecord}; +use reth_tasks::Runtime; +use reth_trie_common::ExecutionWitnessMode; +use std::{future::Future, pin::Pin}; + +/// Re-executes validated payloads against their parent state for `/payloads/witness`. +/// +/// The parent state must be available through the provider (persisted, canonical in-memory, +/// or pending). Parents present only in the engine tree are reported as +/// [`EngineSszWitnessError::ParentStateUnavailable`]; the route then answers with the payload +/// status alone. +#[derive(Clone, Debug)] +pub struct EngineSszWitnessGenerator { + provider: Provider, + evm_config: Evm, + task_spawner: Runtime, +} + +impl EngineSszWitnessGenerator { + /// Creates a new witness generator. + pub const fn new(provider: Provider, evm_config: Evm, task_spawner: Runtime) -> Self { + Self { provider, evm_config, task_spawner } + } +} + +impl EngineSszWitness for EngineSszWitnessGenerator +where + Provider: HeaderProvider + StateProviderFactory + Clone + Send + Sync + 'static, + Provider::Header: alloy_rlp::Encodable, + Evm: ConfigureEvm + 'static, +{ + fn generate_witness( + &self, + payload: ExecutionData, + ) -> Pin< + Box< + dyn Future> + Send + 'static, + >, + > { + let provider = self.provider.clone(); + let evm_config = self.evm_config.clone(); + let task_spawner = self.task_spawner.clone(); + + Box::pin(async move { + task_spawner + .spawn_blocking(move || { + // A VALID newPayload need not be canonical or visible through the provider. + let block = payload + .payload + .try_into_block_with_sidecar::(&payload.sidecar) + .map_err(eyre::Report::new)? + .try_into_recovered() + .map_err(eyre::Report::new)?; + + let block_number = block.header().number; + let parent_hash = block.header().parent_hash; + let state_provider = + provider.state_by_block_hash(parent_hash).map_err(|source| { + EngineSszWitnessError::ParentStateUnavailable { + parent: parent_hash, + source: eyre::Report::new(source), + } + })?; + let block_executor = + evm_config.executor(StateProviderDatabase::new(state_provider)); + let mut witness = None; + let mut first_header = block_number.saturating_sub(1); + block_executor + .execute_with_state_closure(&block, |statedb: &reth_revm::State<_>| { + if let Some((number, _)) = statedb.block_hashes.lowest() { + first_header = number; + } + witness = Some( + ExecutionWitnessRecord::new(statedb) + .into_execution_witness_without_headers( + &statedb.database.0, + ExecutionWitnessMode::Canonical, + ), + ); + }) + .map_err(eyre::Report::new)?; + + let witness = witness + .expect("state closure is called after successful execution") + .map_err(eyre::Report::new)?; + + // Header numbers may refer to a different canonical ancestor. + let mut headers = Vec::new(); + let mut hash = parent_hash; + for _ in first_header..block_number { + let header = provider + .header(hash) + .map_err(eyre::Report::new)? + .ok_or_else(|| eyre::eyre!("ancestor {hash} not found for witness"))?; + hash = header.parent_hash(); + headers.push(alloy_rlp::encode(&header)); + } + headers.reverse(); + + Ok(ExecutionWitnessV1 { + state: witness.state.into_iter().map(|bytes| bytes.to_vec()).collect(), + codes: witness.codes.into_iter().map(|bytes| bytes.to_vec()).collect(), + headers, + }) + }) + .await + .map_err(eyre::Report::new)? + }) + } +} + +/// Generates an execution witness for a valid payload. +pub trait EngineSszWitness: Send + Sync + 'static { + /// Generates a REST-SSZ execution witness after the submitted payload has been validated. + fn generate_witness( + &self, + payload: ExecutionData, + ) -> Pin< + Box< + dyn Future> + Send + 'static, + >, + >; +} + +/// Failure to produce a witness for a validated payload. +#[derive(Debug)] +pub enum EngineSszWitnessError { + /// The parent state is not available through the provider yet. + ParentStateUnavailable { + /// Parent block whose state is required. + parent: B256, + /// Provider failure while accessing the state. + source: eyre::Report, + }, + /// Witness execution or proof generation failed. + Internal(eyre::Report), +} + +impl From for EngineSszWitnessError { + fn from(error: eyre::Report) -> Self { + Self::Internal(error) + } +} + +impl std::fmt::Display for EngineSszWitnessError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::ParentStateUnavailable { parent, source } => { + write!(f, "parent state {parent} is unavailable through the provider: {source}") + } + Self::Internal(error) => std::fmt::Display::fmt(error, f), + } + } +} + +impl std::error::Error for EngineSszWitnessError { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + match self { + Self::ParentStateUnavailable { source, .. } | Self::Internal(source) => { + Some(source.as_ref()) + } + } + } +} diff --git a/crates/ethereum/node/src/lib.rs b/crates/ethereum/node/src/lib.rs index c02fb86b1bf..be55029642d 100644 --- a/crates/ethereum/node/src/lib.rs +++ b/crates/ethereum/node/src/lib.rs @@ -33,3 +33,4 @@ pub use engine::EthereumEngineValidator; pub mod engine_ssz_containers; pub mod engine_ssz_proxy; +pub mod engine_ssz_witness; diff --git a/crates/ethereum/node/src/node.rs b/crates/ethereum/node/src/node.rs index 442332553ee..4907343e441 100644 --- a/crates/ethereum/node/src/node.rs +++ b/crates/ethereum/node/src/node.rs @@ -2,6 +2,7 @@ use crate::{ engine_ssz_proxy::{EngineSszApi, EngineSszProxyLayer}, + engine_ssz_witness::EngineSszWitnessGenerator, EthEngineTypes, EthEvmConfig, }; use alloy_eips::{eip7840::BlobParams, merge::EPOCH_SLOTS}; @@ -352,6 +353,11 @@ where let testing_engine_handle = ctx.beacon_engine_handle.clone(); let (ssz_proxy_layer, ssz_proxy_handle) = EngineSszProxyLayer::new(); + ssz_proxy_handle.set_witness_handler_sync(Arc::new(EngineSszWitnessGenerator::new( + ctx.node.provider().clone(), + ctx.node.evm_config().clone(), + ctx.node.task_executor().clone(), + ))); self.inner .map_engine_api(|engine_api_builder| { diff --git a/crates/ethereum/node/tests/e2e/eth.rs b/crates/ethereum/node/tests/e2e/eth.rs index a25d9738694..a246eafb2c5 100644 --- a/crates/ethereum/node/tests/e2e/eth.rs +++ b/crates/ethereum/node/tests/e2e/eth.rs @@ -1,4 +1,6 @@ -use crate::utils::{advance_with_random_transactions, eth_payload_attributes}; +use crate::utils::{ + advance_with_random_transactions, eth_payload_attributes, eth_payload_attributes_amsterdam, +}; use alloy_eips::eip7685::RequestsOrHash; use alloy_genesis::Genesis; use alloy_primitives::{Address, B256}; @@ -21,11 +23,12 @@ use reth_node_ethereum::{ engine_ssz_containers::{ BlobsV1Request, BlobsV1Response, BlobsV2Response, BlobsV3Response, BlobsV4Request, BlobsV4Response, BodiesByHashRequest, BodiesResponsePrague, - ForkchoiceUpdateResponse as SszForkchoiceUpdateResponse, PayloadStatus as SszPayloadStatus, + ExecutionPayloadEnvelopeAmsterdam, ForkchoiceUpdateResponse as SszForkchoiceUpdateResponse, + PayloadStatus as SszPayloadStatus, PayloadStatusWithWitness, }, EthereumAddOns, EthereumNode, }; -use reth_provider::BlockNumReader; +use reth_provider::{BlockNumReader, StateProviderFactory}; use reth_rpc_api::TestingBuildBlockRequestV1; use reth_rpc_layer::secret_to_bearer_header; use reth_tasks::Runtime; @@ -366,7 +369,7 @@ async fn test_engine_ssz_proxy_can_mine_block() -> eyre::Result<()> { capabilities, serde_json::json!({ "supported_forks": ["paris", "shanghai", "cancun", "prague", "osaka", "amsterdam"], - "fork_scoped_endpoints": ["payloads", "forkchoice", "bodies"], + "fork_scoped_endpoints": ["payloads", "forkchoice", "bodies", "payloads/witness"], "independently_versioned": { "blobs": ["v1", "v2", "v3", "v4"], }, @@ -587,6 +590,96 @@ async fn test_engine_ssz_proxy_blob_revisions() -> eyre::Result<()> { Ok(()) } +#[tokio::test] +async fn test_engine_ssz_proxy_returns_canonical_witness() -> eyre::Result<()> { + let chain_spec = Arc::new( + ChainSpecBuilder::default() + .chain(MAINNET.chain) + .genesis(serde_json::from_str(include_str!("../assets/genesis.json"))?) + .amsterdam_activated() + .build(), + ); + let genesis_hash = chain_spec.genesis_hash(); + let (mut nodes, _) = + setup::(1, chain_spec, false, eth_payload_attributes_amsterdam).await?; + let mut node = nodes.pop().unwrap(); + let payload = node.new_payload().await?; + let envelope = payload.try_into_v6()?; + let request = ExecutionPayloadEnvelopeAmsterdam { + payload: envelope.execution_payload, + parent_beacon_block_root: B256::ZERO, + execution_requests: envelope.execution_requests, + }; + let client = reqwest::Client::new(); + let auth_server = node.auth_server_handle(); + let auth_url = auth_server.http_url(); + let auth_header = secret_to_bearer_header(auth_server.jwt_secret()); + let url = format!("{auth_url}/engine/v1/payloads/witness"); + let response = client + .post(&url) + .header(reqwest::header::CONTENT_TYPE, "application/octet-stream") + .header(ENGINE_EXECUTION_VERSION_HEADER, "amsterdam") + .body(request.as_ssz_bytes()) + .send() + .await?; + assert_eq!(response.status(), reqwest::StatusCode::UNAUTHORIZED); + for fork in ["prague", "amsterdam"] { + let response = client + .post(&url) + .header(reqwest::header::AUTHORIZATION, auth_header.to_str()?) + .header(reqwest::header::CONTENT_TYPE, "application/octet-stream") + .header(ENGINE_EXECUTION_VERSION_HEADER, fork) + .body(request.as_ssz_bytes()) + .send() + .await?; + if fork == "prague" { + assert_eq!(response.status(), reqwest::StatusCode::BAD_REQUEST); + continue + } + let status = response.status(); + let bytes = response.bytes().await?; + assert_eq!(status, reqwest::StatusCode::OK, "{}", String::from_utf8_lossy(&bytes)); + let response = PayloadStatusWithWitness::from_ssz_bytes(&bytes).unwrap(); + assert_eq!(response.payload_status.status, PayloadStatusEnum::Valid); + let witness = response.witness.as_ref().expect("valid payload includes a witness"); + assert!(!witness.state.is_empty()); + assert!(witness.state.windows(2).all(|pair| pair[0] < pair[1])); + assert!(witness.codes.windows(2).all(|pair| pair[0] < pair[1])); + let parent: alloy_consensus::Header = alloy_rlp::decode_exact(&witness.headers[0])?; + assert_eq!(parent.hash_slow(), genesis_hash); + } + let mut invalid = request; + invalid.payload.payload_inner.payload_inner.payload_inner.block_hash = B256::ZERO; + let response = client + .post(&url) + .header(reqwest::header::AUTHORIZATION, auth_header.to_str()?) + .header(reqwest::header::CONTENT_TYPE, "application/octet-stream") + .header(ENGINE_EXECUTION_VERSION_HEADER, "amsterdam") + .body(invalid.as_ssz_bytes()) + .send() + .await?; + assert_eq!(response.status(), reqwest::StatusCode::OK); + let response = PayloadStatusWithWitness::from_ssz_bytes(&response.bytes().await?).unwrap(); + assert!(matches!(response.payload_status.status, PayloadStatusEnum::Invalid { .. })); + assert!(response.witness.is_none()); + invalid.payload.payload_inner.payload_inner.payload_inner.transactions = + vec![alloy_primitives::Bytes::from_static(&[2])]; + let response = client + .post(&url) + .header(reqwest::header::AUTHORIZATION, auth_header.to_str()?) + .header(reqwest::header::CONTENT_TYPE, "application/octet-stream") + .header(ENGINE_EXECUTION_VERSION_HEADER, "amsterdam") + .body(invalid.as_ssz_bytes()) + .send() + .await?; + assert_eq!(response.status(), reqwest::StatusCode::UNPROCESSABLE_ENTITY); + assert_eq!( + response.json::().await?["type"], + "/engine-api/errors/invalid-body" + ); + Ok(()) +} + /// Tests that the sparse trie pipeline can be shared with the payload builder. /// /// Enables both `share_execution_cache_with_payload_builder` and @@ -873,3 +966,44 @@ async fn test_engine_ssz_custom_engine_and_middleware() -> eyre::Result<()> { assert_eq!(requests.load(Ordering::Relaxed), 2); Ok(()) } + +#[tokio::test] +async fn test_engine_ssz_witness_omitted_without_provider_parent_state() -> eyre::Result<()> { + let chain = Arc::new( + ChainSpecBuilder::default() + .chain(MAINNET.chain) + .genesis(serde_json::from_str(include_str!("../assets/genesis.json"))?) + .amsterdam_activated() + .build(), + ); + let (mut nodes, _) = + setup::(2, chain, false, eth_payload_attributes_amsterdam).await?; + let target = nodes.pop().unwrap(); + let mut source = nodes.pop().unwrap(); + let first = source.advance_block().await?; + target.submit_payload(first).await?; + let parent = source.advance_block().await?; + let parent_hash = target.submit_payload(parent).await?; + assert!(target.inner.provider.state_by_block_hash(parent_hash).is_err()); + let child = source.new_payload().await?.try_into_v6()?; + let request = ExecutionPayloadEnvelopeAmsterdam { + payload: child.execution_payload, + parent_beacon_block_root: B256::ZERO, + execution_requests: child.execution_requests, + }; + let auth = target.auth_server_handle(); + let jwt = secret_to_bearer_header(auth.jwt_secret()); + let response = reqwest::Client::new() + .post(format!("{}/engine/v1/payloads/witness", auth.http_url())) + .header("Authorization", jwt.to_str()?) + .header("Content-Type", "application/octet-stream") + .header(ENGINE_EXECUTION_VERSION_HEADER, "amsterdam") + .body(request.as_ssz_bytes()) + .send() + .await?; + assert_eq!(response.status(), reqwest::StatusCode::OK); + let response = PayloadStatusWithWitness::from_ssz_bytes(&response.bytes().await?).unwrap(); + assert_eq!(response.payload_status.status, PayloadStatusEnum::Valid); + assert!(response.witness.is_none()); + Ok(()) +} diff --git a/crates/revm/src/witness.rs b/crates/revm/src/witness.rs index e22efd2b4c3..21a6980a333 100644 --- a/crates/revm/src/witness.rs +++ b/crates/revm/src/witness.rs @@ -47,6 +47,39 @@ impl<'a, DB> ExecutionWitnessRecord<'a, DB> { + ?Sized, HP: reth_storage_api::HeaderProvider + ?Sized, HP::Header: alloy_rlp::Encodable, + { + let lowest_block_number = self.state.block_hashes.lowest().map(|(number, _)| number); + let mut exec_witness = self.into_execution_witness_without_headers(state_provider, mode)?; + + let smallest = lowest_block_number.unwrap_or_else(|| block_number.saturating_sub(1)); + let range = smallest..block_number; + + exec_witness.headers = headers_provider + .headers_range(range)? + .into_iter() + .map(|header| { + let mut buf = Vec::new(); + alloy_rlp::Encodable::encode(&header, &mut buf); + buf.into() + }) + .collect(); + + Ok(exec_witness) + } + + /// Generates state proofs and codes without fetching ancestor headers. + /// + /// Callers witnessing non-canonical blocks can supply headers by following parent hashes. + #[cfg(feature = "witness")] + pub fn into_execution_witness_without_headers( + self, + state_provider: &SP, + mode: ExecutionWitnessMode, + ) -> reth_storage_errors::provider::ProviderResult + where + SP: reth_storage_api::HashedPostStateProvider + + reth_storage_api::StateProofProvider + + ?Sized, { let codes = match mode { ExecutionWitnessMode::Legacy => self @@ -77,28 +110,10 @@ impl<'a, DB> ExecutionWitnessRecord<'a, DB> { } }; - let lowest_block_number = - self.state.block_hashes.lowest().map(|(block_number, _)| block_number); let (hashed_state, keys) = self.hashed_post_state(state_provider)?; let state = state_provider.witness(Default::default(), hashed_state, mode)?; - let mut exec_witness = - alloy_rpc_types_debug::ExecutionWitness { state, codes, keys, ..Default::default() }; - - let smallest = lowest_block_number.unwrap_or_else(|| block_number.saturating_sub(1)); - let range = smallest..block_number; - - exec_witness.headers = headers_provider - .headers_range(range)? - .into_iter() - .map(|header| { - let mut buf = Vec::new(); - alloy_rlp::Encodable::encode(&header, &mut buf); - buf.into() - }) - .collect(); - - Ok(exec_witness) + Ok(alloy_rpc_types_debug::ExecutionWitness { state, codes, keys, ..Default::default() }) } #[cfg(feature = "witness")] From b9cd22bae84a5e53e7c4706d358c8d8cb6ac3242 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Sat, 5 Sep 2026 18:26:49 +0200 Subject: [PATCH 181/294] refactor(engine): simplify SSZ transport and reuse witness buffers (#27028) --- .../node/src/engine_ssz_containers.rs | 12 +- crates/ethereum/node/src/engine_ssz_proxy.rs | 113 ++++++++---------- .../ethereum/node/src/engine_ssz_witness.rs | 8 +- 3 files changed, 60 insertions(+), 73 deletions(-) diff --git a/crates/ethereum/node/src/engine_ssz_containers.rs b/crates/ethereum/node/src/engine_ssz_containers.rs index ea5b0337b7d..54fb83c8912 100644 --- a/crates/ethereum/node/src/engine_ssz_containers.rs +++ b/crates/ethereum/node/src/engine_ssz_containers.rs @@ -1383,13 +1383,13 @@ impl TryFrom>> for BlobsV4Response { } /// A trie-node byte list in an [`ExecutionWitnessV1`]. -pub type WitnessNodeV1 = Vec; +pub type WitnessNodeV1 = Bytes; /// A contract-code byte list in an [`ExecutionWitnessV1`]. -pub type WitnessCodeV1 = Vec; +pub type WitnessCodeV1 = Bytes; /// An RLP-encoded header byte list in an [`ExecutionWitnessV1`]. -pub type WitnessHeaderV1 = Vec; +pub type WitnessHeaderV1 = Bytes; /// Canonical execution witness for `POST /payloads/witness`. /// @@ -1943,9 +1943,9 @@ mod tests { validation_error: Optional::none(), }; let witness = ExecutionWitnessV1 { - state: vec![vec![1, 2, 3]], - codes: vec![vec![4, 5]], - headers: vec![vec![6]], + state: vec![vec![1, 2, 3].into()], + codes: vec![vec![4, 5].into()], + headers: vec![vec![6].into()], }; let response = PayloadStatusWithWitness::new(payload_status, Some(witness)); diff --git a/crates/ethereum/node/src/engine_ssz_proxy.rs b/crates/ethereum/node/src/engine_ssz_proxy.rs index d79bcc0aaf7..26260293a0d 100644 --- a/crates/ethereum/node/src/engine_ssz_proxy.rs +++ b/crates/ethereum/node/src/engine_ssz_proxy.rs @@ -8,15 +8,15 @@ use crate::{ engine_ssz_containers::{ BlobsV1Request, BlobsV1Response, BlobsV2Response, BlobsV3Response, BlobsV4Request, BlobsV4Response, BodiesByHashRequest, BodiesResponse, BuiltPayloadAmsterdam, - BuiltPayloadCancun, BuiltPayloadOsaka, BuiltPayloadParis, BuiltPayloadPrague, - BuiltPayloadShanghai, ExecutionPayloadBodyAmsterdam, ExecutionPayloadBodyParis, - ExecutionPayloadBodyShanghai, ExecutionPayloadEnvelopeAmsterdam, - ExecutionPayloadEnvelopeCancun, ExecutionPayloadEnvelopeOsaka, - ExecutionPayloadEnvelopeParis, ExecutionPayloadEnvelopePrague, - ExecutionPayloadEnvelopeShanghai, ForkchoiceUpdateAmsterdam, ForkchoiceUpdateCancun, - ForkchoiceUpdateOsaka, ForkchoiceUpdateParis, ForkchoiceUpdatePrague, - ForkchoiceUpdateResponse, ForkchoiceUpdateShanghai, Optional, - PayloadStatus as EngineSszPayloadStatus, PayloadStatusWithWitness, + BuiltPayloadOsaka, BuiltPayloadParis, BuiltPayloadPrague, BuiltPayloadShanghai, + ExecutionPayloadBodyAmsterdam, ExecutionPayloadBodyParis, ExecutionPayloadBodyShanghai, + ExecutionPayloadEnvelopeAmsterdam, ExecutionPayloadEnvelopeCancun, + ExecutionPayloadEnvelopeOsaka, ExecutionPayloadEnvelopeParis, + ExecutionPayloadEnvelopePrague, ExecutionPayloadEnvelopeShanghai, + ForkchoiceUpdateAmsterdam, ForkchoiceUpdateCancun, ForkchoiceUpdateOsaka, + ForkchoiceUpdateParis, ForkchoiceUpdatePrague, ForkchoiceUpdateResponse, + ForkchoiceUpdateShanghai, Optional, PayloadStatus as EngineSszPayloadStatus, + PayloadStatusWithWitness, }, engine_ssz_witness::{EngineSszWitness, EngineSszWitnessError}, }; @@ -461,22 +461,25 @@ where }, Err(err) => engine_error_response(err), }, - EngineSszFork::Cancun => match self.get_payload_v3_metered(payload_id).await { - Ok(payload) => get_payload_response(BuiltPayloadCancun::from(payload)), - Err(err) => engine_error_response(err), - }, - EngineSszFork::Prague => match self.get_payload_v4_metered(payload_id).await { - Ok(payload) => get_payload_response(BuiltPayloadPrague::from(payload)), - Err(err) => engine_error_response(err), - }, - EngineSszFork::Osaka => match self.get_payload_v5_metered(payload_id).await { - Ok(payload) => get_payload_response(BuiltPayloadOsaka::from(payload)), - Err(err) => engine_error_response(err), - }, - EngineSszFork::Amsterdam => match self.get_payload_v6_metered(payload_id).await { - Ok(payload) => get_payload_response(BuiltPayloadAmsterdam::from(payload)), - Err(err) => engine_error_response(err), - }, + EngineSszFork::Cancun => self + .get_payload_v3_metered(payload_id) + .await + .map_or_else(engine_error_response, get_payload_response), + EngineSszFork::Prague => self + .get_payload_v4_metered(payload_id) + .await + .map(BuiltPayloadPrague::from) + .map_or_else(engine_error_response, get_payload_response), + EngineSszFork::Osaka => self + .get_payload_v5_metered(payload_id) + .await + .map(BuiltPayloadOsaka::from) + .map_or_else(engine_error_response, get_payload_response), + EngineSszFork::Amsterdam => self + .get_payload_v6_metered(payload_id) + .await + .map(BuiltPayloadAmsterdam::from) + .map_or_else(engine_error_response, get_payload_response), } } @@ -581,35 +584,28 @@ async fn handle_engine_ssz_request( where Api: EngineSszApi, { - let method = request.method().as_str().to_owned(); - let path = request.uri().path().to_owned(); - let Some(endpoint) = parse_engine_path(&path) else { + let Some(endpoint) = parse_engine_path(request.uri().path()) else { return problem_response(STATUS_NOT_FOUND, "method-not-found", None) }; + if request.method() != endpoint.method() { + return problem_response(STATUS_METHOD_NOT_ALLOWED, "method-not-allowed", None) + } + match endpoint { EngineSszEndpoint::Capabilities => { - if method != "GET" { - return problem_response(STATUS_METHOD_NOT_ALLOWED, "method-not-allowed", None) - } let Some(engine_api) = handle.engine_api().await else { return problem_response(STATUS_SERVICE_UNAVAILABLE, "service-unavailable", None) }; engine_api.capabilities(handle.witness_enabled().await) } EngineSszEndpoint::Identity => { - if method != "GET" { - return problem_response(STATUS_METHOD_NOT_ALLOWED, "method-not-allowed", None) - } let Some(engine_api) = handle.engine_api().await else { return problem_response(STATUS_SERVICE_UNAVAILABLE, "service-unavailable", None) }; engine_api.identity() } EngineSszEndpoint::NewPayload => { - if method != "POST" { - return problem_response(STATUS_METHOD_NOT_ALLOWED, "method-not-allowed", None) - } let Some(fork) = request_fork(&request) else { return problem_response(STATUS_BAD_REQUEST, "unsupported-fork", None) }; @@ -623,9 +619,6 @@ where engine_api.new_payload(fork, body).await } EngineSszEndpoint::PayloadsWithWitness => { - if method != "POST" { - return problem_response(STATUS_METHOD_NOT_ALLOWED, "method-not-allowed", None) - } let Some(fork) = request_fork(&request) else { return problem_response(STATUS_BAD_REQUEST, "unsupported-fork", None) }; @@ -645,9 +638,6 @@ where engine_api.new_payload_with_witness(body, witness_handler).await } EngineSszEndpoint::GetPayload(payload_id) => { - if method != "GET" { - return problem_response(STATUS_METHOD_NOT_ALLOWED, "method-not-allowed", None) - } let Ok(payload_id) = payload_id else { return problem_response(STATUS_BAD_REQUEST, "invalid-request", None) }; @@ -660,9 +650,6 @@ where engine_api.get_payload(fork, payload_id).await } EngineSszEndpoint::Forkchoice => { - if method != "POST" { - return problem_response(STATUS_METHOD_NOT_ALLOWED, "method-not-allowed", None) - } let Some(fork) = request_fork(&request) else { return problem_response(STATUS_BAD_REQUEST, "unsupported-fork", None) }; @@ -676,9 +663,6 @@ where engine_api.forkchoice_updated(fork, body).await } EngineSszEndpoint::PayloadBodiesByHash => { - if method != "POST" { - return problem_response(STATUS_METHOD_NOT_ALLOWED, "method-not-allowed", None) - } let Some(fork) = request_fork(&request) else { return problem_response(STATUS_BAD_REQUEST, "unsupported-fork", None) }; @@ -701,9 +685,6 @@ where engine_api.get_payload_bodies_by_hash(fork, request).await } EngineSszEndpoint::PayloadBodiesByRange => { - if method != "GET" { - return problem_response(STATUS_METHOD_NOT_ALLOWED, "method-not-allowed", None) - } let Some(fork) = request_fork(&request) else { return problem_response(STATUS_BAD_REQUEST, "unsupported-fork", None) }; @@ -718,9 +699,6 @@ where engine_api.get_payload_bodies_by_range(fork, start, count).await } EngineSszEndpoint::Blobs(version) => { - if method != "POST" { - return problem_response(STATUS_METHOD_NOT_ALLOWED, "method-not-allowed", None) - } let body = match read_ssz_body(request, MAX_BLOB_REQUEST_BYTES).await { Ok(body) => body, Err(response) => return response, @@ -789,6 +767,22 @@ enum EngineSszEndpoint { Blobs(u8), } +impl EngineSszEndpoint { + const fn method(&self) -> &'static str { + match self { + Self::Capabilities | + Self::Identity | + Self::GetPayload(_) | + Self::PayloadBodiesByRange => "GET", + Self::NewPayload | + Self::PayloadsWithWitness | + Self::Forkchoice | + Self::PayloadBodiesByHash | + Self::Blobs(_) => "POST", + } + } +} + fn handle_capabilities(witness_enabled: bool) -> HttpResponse { let mut fork_scoped_endpoints = vec!["payloads", "forkchoice", "bodies"]; if witness_enabled { @@ -1215,12 +1209,9 @@ fn ssz_response(value: T) -> HttpResponse { } fn get_payload_response(value: T) -> HttpResponse { - HttpResponse::builder() - .status(STATUS_OK) - .header(CONTENT_TYPE, OCTET_STREAM) - .header(CACHE_CONTROL, "no-store") - .body(HttpBody::from(value.as_ssz_bytes())) - .expect("valid response") + let mut response = ssz_response(value); + response.headers_mut().insert(CACHE_CONTROL, "no-store".parse().expect("valid cache control")); + response } fn json_response(value: T) -> HttpResponse { diff --git a/crates/ethereum/node/src/engine_ssz_witness.rs b/crates/ethereum/node/src/engine_ssz_witness.rs index ae245142e53..79c98146d0e 100644 --- a/crates/ethereum/node/src/engine_ssz_witness.rs +++ b/crates/ethereum/node/src/engine_ssz_witness.rs @@ -102,15 +102,11 @@ where .map_err(eyre::Report::new)? .ok_or_else(|| eyre::eyre!("ancestor {hash} not found for witness"))?; hash = header.parent_hash(); - headers.push(alloy_rlp::encode(&header)); + headers.push(alloy_rlp::encode(&header).into()); } headers.reverse(); - Ok(ExecutionWitnessV1 { - state: witness.state.into_iter().map(|bytes| bytes.to_vec()).collect(), - codes: witness.codes.into_iter().map(|bytes| bytes.to_vec()).collect(), - headers, - }) + Ok(ExecutionWitnessV1 { state: witness.state, codes: witness.codes, headers }) }) .await .map_err(eyre::Report::new)? From b3dbb065d31a5db941cf4b803d412f73cdae36a6 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Sun, 6 Sep 2026 10:52:04 +0200 Subject: [PATCH 182/294] perf(rpc): convert trace_get results lazily (#27031) --- crates/rpc/rpc/src/trace.rs | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/crates/rpc/rpc/src/trace.rs b/crates/rpc/rpc/src/trace.rs index 9ab456b253a..c79b087b169 100644 --- a/crates/rpc/rpc/src/trace.rs +++ b/crates/rpc/rpc/src/trace.rs @@ -237,7 +237,19 @@ where hash: B256, index: usize, ) -> Result, Eth::Error> { - Ok(self.trace_transaction(hash).await?.and_then(|traces| traces.into_iter().nth(index))) + self.eth_api() + .spawn_trace_transaction_in_block( + hash, + TracingInspectorConfig::default_parity(), + move |tx_info, inspector, _, _| { + Ok(inspector + .into_parity_builder() + .into_localized_transaction_traces_iter(tx_info) + .nth(index)) + }, + ) + .await + .map(Option::flatten) } /// Returns all traces for the given transaction hash From 4934f6e0606f49903111047e67bd73597c3ef78d Mon Sep 17 00:00:00 2001 From: cui Date: Sun, 6 Sep 2026 11:58:46 +0000 Subject: [PATCH 183/294] feat(cli): add --authrpc.jwtsecret-hex (#27034) --- crates/node/core/src/args/rpc_server.rs | 54 ++++++++++++++++++++++++- crates/rpc/rpc-builder/src/config.rs | 29 ++++++++++--- docs/vocs/docs/pages/cli/reth/node.mdx | 5 +++ 3 files changed, 81 insertions(+), 7 deletions(-) diff --git a/crates/node/core/src/args/rpc_server.rs b/crates/node/core/src/args/rpc_server.rs index eb0f2432adc..22f5ceba6d8 100644 --- a/crates/node/core/src/args/rpc_server.rs +++ b/crates/node/core/src/args/rpc_server.rs @@ -545,9 +545,22 @@ pub struct RpcServerArgs { /// If no path is provided, a secret will be generated and stored in the datadir under /// `//jwt.hex`. For mainnet this would be `~/.local/share/reth/mainnet/jwt.hex` /// by default. - #[arg(long = "authrpc.jwtsecret", value_name = "PATH", global = true, required = false, default_value = Resettable::from(DefaultRpcServerArgs::get_global().auth_jwtsecret.as_ref().map(|v| v.to_string_lossy().into())))] + #[arg(long = "authrpc.jwtsecret", value_name = "PATH", global = true, required = false, conflicts_with = "auth_jwtsecret_hex", default_value = Resettable::from(DefaultRpcServerArgs::get_global().auth_jwtsecret.as_ref().map(|v| v.to_string_lossy().into())))] pub auth_jwtsecret: Option, + /// Hex encoded JWT secret to use for the authenticated engine-API RPC server. + /// + /// This will enforce JWT authentication for all requests coming from the consensus layer. + /// Cannot be used together with `--authrpc.jwtsecret`. + #[arg( + long = "authrpc.jwtsecret-hex", + value_name = "HEX", + global = true, + required = false, + conflicts_with = "auth_jwtsecret" + )] + pub auth_jwtsecret_hex: Option, + /// Enable auth engine API over IPC #[arg(long, default_value_t = DefaultRpcServerArgs::get_global().auth_ipc)] pub auth_ipc: bool, @@ -949,6 +962,7 @@ impl Default for RpcServerArgs { auth_addr, auth_port, auth_jwtsecret, + auth_jwtsecret_hex: None, auth_ipc, auth_ipc_path, disable_auth_server, @@ -1172,6 +1186,7 @@ mod tests { auth_addr: "127.0.0.1".parse().unwrap(), auth_port: 8551, auth_jwtsecret: Some(std::path::PathBuf::from("/tmp/jwt.hex")), + auth_jwtsecret_hex: None, auth_ipc: false, auth_ipc_path: "engine.ipc".to_string(), disable_auth_server: false, @@ -1316,4 +1331,41 @@ mod tests { assert_eq!(parsed_args, args); } + + #[test] + fn parse_auth_jwtsecret_hex() { + let hex = "1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef"; + let args = + CommandParser::::parse_from(["reth", "--authrpc.jwtsecret-hex", hex]) + .args; + + let expected = JwtSecret::from_hex(hex).unwrap(); + assert_eq!(args.auth_jwtsecret_hex, Some(expected)); + assert_eq!(args.auth_jwtsecret, None); + } + + #[test] + fn parse_auth_jwtsecret_hex_with_0x_prefix() { + let hex = "0x1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef"; + let args = + CommandParser::::parse_from(["reth", "--authrpc.jwtsecret-hex", hex]) + .args; + + let expected = JwtSecret::from_hex(hex).unwrap(); + assert_eq!(args.auth_jwtsecret_hex, Some(expected)); + assert_eq!(args.auth_jwtsecret, None); + } + + #[test] + fn test_auth_jwtsecret_and_hex_are_mutually_exclusive() { + let result = CommandParser::::try_parse_from([ + "reth", + "--authrpc.jwtsecret", + "/tmp/jwt.hex", + "--authrpc.jwtsecret-hex", + "1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef", + ]); + + assert!(result.is_err()); + } } diff --git a/crates/rpc/rpc-builder/src/config.rs b/crates/rpc/rpc-builder/src/config.rs index 93496950290..6a6b8f1351a 100644 --- a/crates/rpc/rpc-builder/src/config.rs +++ b/crates/rpc/rpc-builder/src/config.rs @@ -75,7 +75,7 @@ pub trait RethRpcServerConfig { /// the filesystem. This file can then be used to provision the counterpart client. /// /// The `default_jwt_path` provided as an argument will be used as the default location for the - /// jwt secret in case the `auth_jwtsecret` argument is not provided. + /// jwt secret in case neither `--authrpc.jwtsecret` nor `--authrpc.jwtsecret-hex` is provided. fn auth_jwt_secret(&self, default_jwt_path: PathBuf) -> Result; /// Returns the configured jwt secret key for the regular rpc servers, if any. @@ -254,12 +254,17 @@ impl RethRpcServerConfig for RpcServerArgs { } fn auth_jwt_secret(&self, default_jwt_path: PathBuf) -> Result { - match self.auth_jwtsecret.as_ref() { - Some(fpath) => { - debug!(target: "reth::cli", user_path=?fpath, "Reading JWT auth secret file"); - JwtSecret::from_file(fpath) + if let Some(secret) = self.auth_jwtsecret_hex { + debug!(target: "reth::cli", "Using JWT auth secret from hex"); + Ok(secret) + } else { + match self.auth_jwtsecret.as_ref() { + Some(fpath) => { + debug!(target: "reth::cli", user_path=?fpath, "Reading JWT auth secret file"); + JwtSecret::from_file(fpath) + } + None => get_or_create_jwt_secret_from_path(&default_jwt_path), } - None => get_or_create_jwt_secret_from_path(&default_jwt_path), } } @@ -273,6 +278,7 @@ mod tests { use clap::{Args, Parser}; use reth_node_core::args::RpcServerArgs; use reth_rpc_eth_types::RPC_DEFAULT_GAS_CAP; + use reth_rpc_layer::JwtSecret; use reth_rpc_server_types::{constants, RethRpcModule, RpcModuleSelection}; use std::net::{Ipv4Addr, SocketAddr, SocketAddrV4}; @@ -426,4 +432,15 @@ mod tests { assert_eq!(config.max_blocks_per_filter, Some(100)); assert_eq!(config.max_logs_per_response, Some(200)); } + + #[test] + fn test_auth_jwt_secret_from_hex() { + let hex = "1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef"; + let args = + CommandParser::::parse_from(["reth", "--authrpc.jwtsecret-hex", hex]) + .args; + + let secret = args.auth_jwt_secret(std::env::temp_dir().join("unused.jwt")).unwrap(); + assert_eq!(secret, JwtSecret::from_hex(hex).unwrap()); + } } diff --git a/docs/vocs/docs/pages/cli/reth/node.mdx b/docs/vocs/docs/pages/cli/reth/node.mdx index 318fb7463d2..2d82db99149 100644 --- a/docs/vocs/docs/pages/cli/reth/node.mdx +++ b/docs/vocs/docs/pages/cli/reth/node.mdx @@ -407,6 +407,11 @@ RPC: If no path is provided, a secret will be generated and stored in the datadir under `//jwt.hex`. For mainnet this would be `~/.local/share/reth/mainnet/jwt.hex` by default. + --authrpc.jwtsecret-hex + Hex encoded JWT secret to use for the authenticated engine-API RPC server. + + This will enforce JWT authentication for all requests coming from the consensus layer. Cannot be used together with `--authrpc.jwtsecret`. + --auth-ipc Enable auth engine API over IPC From 88815425c194309c934144d77dee6cd65633346e Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Sun, 6 Sep 2026 17:37:18 +0200 Subject: [PATCH 184/294] test(e2e): await payload builds instead of sleeping (#27041) Co-authored-by: Matthias Seitz <19890894+mattsse@users.noreply.github.com> --- crates/e2e-test-utils/src/node.rs | 5 ++++- .../src/testsuite/actions/produce_blocks.rs | 15 +++++++++++++-- crates/e2e-test-utils/src/testsuite/mod.rs | 14 +++++++++++--- crates/e2e-test-utils/tests/e2e-testsuite/main.rs | 3 ++- 4 files changed, 30 insertions(+), 7 deletions(-) diff --git a/crates/e2e-test-utils/src/node.rs b/crates/e2e-test-utils/src/node.rs index 3375583769b..21cf7b90c1b 100644 --- a/crates/e2e-test-utils/src/node.rs +++ b/crates/e2e-test-utils/src/node.rs @@ -343,7 +343,10 @@ where let url = self.rpc_url(); let beacon_handle = self.inner.add_ons_handle.beacon_engine_handle.clone(); - Ok(crate::testsuite::NodeClient::new_with_beacon_engine(rpc, auth, url, beacon_handle)) + let mut client = + crate::testsuite::NodeClient::new_with_beacon_engine(rpc, auth, url, beacon_handle); + client.payload_builder = Some(self.inner.payload_builder_handle.clone()); + Ok(client) } /// Calls the `testing_buildBlockV1` RPC on this node. diff --git a/crates/e2e-test-utils/src/testsuite/actions/produce_blocks.rs b/crates/e2e-test-utils/src/testsuite/actions/produce_blocks.rs index 5915ee03101..02a19982e7d 100644 --- a/crates/e2e-test-utils/src/testsuite/actions/produce_blocks.rs +++ b/crates/e2e-test-utils/src/testsuite/actions/produce_blocks.rs @@ -12,7 +12,7 @@ use alloy_rpc_types_eth::{Block, Header, Receipt, Transaction, TransactionReques use eyre::Result; use futures_util::future::BoxFuture; use reth_ethereum_primitives::TransactionSigned; -use reth_node_api::{EngineTypes, PayloadTypes}; +use reth_node_api::{EngineTypes, PayloadKind, PayloadTypes}; use reth_rpc_api::clients::{EngineApiClient, EthApiClient}; use std::{collections::HashSet, marker::PhantomData, time::Duration}; use tokio::time::sleep; @@ -330,7 +330,18 @@ where env.active_node_state_mut()?.next_payload_id = Some(payload_id); - sleep(Duration::from_secs(1)).await; + if let Some(builder) = &env.node_clients[producer_idx].payload_builder { + // Wait for the pending build rather than racing it with an empty fallback payload. + tokio::time::timeout( + Duration::from_secs(30), + builder.resolve_kind(payload_id, PayloadKind::WaitForPending), + ) + .await? + .ok_or_else(|| eyre::eyre!("Unknown payload {payload_id}"))??; + } else { + // RPC-only clients do not expose the local payload builder. + sleep(Duration::from_secs(1)).await; + } let built_payload_envelope = EngineApiClient::::get_payload_v3( &env.node_clients[producer_idx].engine.http_client(), diff --git a/crates/e2e-test-utils/src/testsuite/mod.rs b/crates/e2e-test-utils/src/testsuite/mod.rs index 17d04f64eaa..208c24be1ff 100644 --- a/crates/e2e-test-utils/src/testsuite/mod.rs +++ b/crates/e2e-test-utils/src/testsuite/mod.rs @@ -8,7 +8,7 @@ use alloy_primitives::B256; use eyre::Result; use jsonrpsee::http_client::HttpClient; use reth_node_api::{EngineTypes, PayloadTypes}; -use reth_payload_builder::PayloadId; +use reth_payload_builder::{PayloadBuilderHandle, PayloadId}; use std::{collections::HashMap, marker::PhantomData}; pub mod actions; pub mod setup; @@ -32,6 +32,8 @@ where pub engine: AuthServerHandle, /// Beacon consensus engine handle for direct interaction with the consensus engine pub beacon_engine_handle: Option>, + /// Local payload builder used to wait for an in-progress build before requesting it over RPC. + pub(crate) payload_builder: Option>, /// Alloy provider for interacting with the node provider: Arc, } @@ -44,7 +46,7 @@ where pub fn new(rpc: HttpClient, engine: AuthServerHandle, url: Url) -> Self { let provider = Arc::new(ProviderBuilder::new().connect_http(url)) as Arc; - Self { rpc, engine, beacon_engine_handle: None, provider } + Self { rpc, engine, beacon_engine_handle: None, payload_builder: None, provider } } /// Instantiates a new [`NodeClient`] with the given handles, RPC URL, and beacon engine handle @@ -56,7 +58,13 @@ where ) -> Self { let provider = Arc::new(ProviderBuilder::new().connect_http(url)) as Arc; - Self { rpc, engine, beacon_engine_handle: Some(beacon_engine_handle), provider } + Self { + rpc, + engine, + beacon_engine_handle: Some(beacon_engine_handle), + payload_builder: None, + provider, + } } /// Get a block by number using the alloy provider diff --git a/crates/e2e-test-utils/tests/e2e-testsuite/main.rs b/crates/e2e-test-utils/tests/e2e-testsuite/main.rs index 023a50b343c..01367f0f7ce 100644 --- a/crates/e2e-test-utils/tests/e2e-testsuite/main.rs +++ b/crates/e2e-test-utils/tests/e2e-testsuite/main.rs @@ -192,7 +192,8 @@ async fn test_testsuite_produce_blocks() -> Result<()> { let test = TestBuilder::new() .with_setup(setup) .with_action(ProduceBlocks::::new(5)) - .with_action(MakeCanonical::new()); + .with_action(MakeCanonical::new()) + .with_action(AssertChainTip::new(5)); test.run::().await?; From 315d04c0b4bf7644bffe06db21cdccb887c9339d Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Mon, 7 Sep 2026 00:09:37 +0000 Subject: [PATCH 185/294] chore(ci): cancel superseded PR lint runs (#27042) Co-authored-by: Matthias Seitz <19890894+mattsse@users.noreply.github.com> --- .github/workflows/lint.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 30fc52850ed..d8b935eea3c 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -10,6 +10,11 @@ env: CARGO_TERM_COLOR: always RUSTC_WRAPPER: "sccache" +# Supersede PR runs without cancelling independent main or merge-queue runs. +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }} + cancel-in-progress: true + permissions: {} jobs: From d98f4475ed1057d05d57d04b5f000368876a5337 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Mon, 7 Sep 2026 00:09:52 +0000 Subject: [PATCH 186/294] chore(ci): cache test dependency artifacts (#27040) Co-authored-by: Matthias Seitz <19890894+mattsse@users.noreply.github.com> --- .github/workflows/e2e.yml | 3 ++- .github/workflows/integration.yml | 3 ++- .github/workflows/unit.yml | 3 ++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 40ae0fb086c..aa9cd861881 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -41,7 +41,8 @@ jobs: tool: nextest - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 with: - cache-targets: false + prefix-key: v1-rust-e2e-targets + cache-bin: false cache-on-failure: true - name: Run e2e tests run: | diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index dde997fa0e6..0f8f3d4670a 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -50,7 +50,8 @@ jobs: - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 with: - cache-targets: false + prefix-key: v1-rust-integration-targets + cache-bin: false cache-on-failure: true - name: Run tests run: | diff --git a/.github/workflows/unit.yml b/.github/workflows/unit.yml index f844ddfeb75..42f83bc87e8 100644 --- a/.github/workflows/unit.yml +++ b/.github/workflows/unit.yml @@ -45,7 +45,8 @@ jobs: - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 with: - cache-targets: false + prefix-key: v1-rust-unit-targets + cache-bin: false cache-on-failure: true - uses: taiki-e/install-action@b6b84cf49ebfe0176417bdce007c624f0db37f20 # v2.86.2 with: From ed4fee7e54a301bdc055748ecf70b395d28dff03 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Mon, 7 Sep 2026 00:10:02 +0000 Subject: [PATCH 187/294] chore(ci): build only e2e test targets (#27043) Co-authored-by: Matthias Seitz <19890894+mattsse@users.noreply.github.com> --- .github/workflows/e2e.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index aa9cd861881..fa8ad62f75f 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -49,7 +49,7 @@ jobs: cargo nextest run \ --no-fail-fast \ --locked --features "asm-keccak" \ - --workspace \ + --workspace --test e2e_testsuite \ --exclude 'example-*' \ --exclude 'exex-subscription' \ --exclude 'ef-tests' \ From 2117377448154812b935d53deaf1a80783c06195 Mon Sep 17 00:00:00 2001 From: Brian Picciano Date: Mon, 7 Sep 2026 07:48:38 +0000 Subject: [PATCH 188/294] fix(trie): correctly handle branch collapse in proof v2 (#26843) --- crates/trie/common/src/trie_node_v2.rs | 13 + crates/trie/trie/src/proof_v2/mod.rs | 1186 +++++++++++------------- crates/trie/trie/src/proof_v2/node.rs | 74 +- 3 files changed, 586 insertions(+), 687 deletions(-) diff --git a/crates/trie/common/src/trie_node_v2.rs b/crates/trie/common/src/trie_node_v2.rs index 46c03811b72..bbf1d0daefc 100644 --- a/crates/trie/common/src/trie_node_v2.rs +++ b/crates/trie/common/src/trie_node_v2.rs @@ -222,4 +222,17 @@ impl Encodable for BranchNodeV2 { ExtensionNodeRef::new(&self.key, branch_rlp_node.as_slice()).encode(out); } + + fn length(&self) -> usize { + if self.key.is_empty() { + return BranchNodeRef::new(&self.stack, self.state_mask).length() + } + + let branch_rlp_node = self + .branch_rlp_node + .as_ref() + .expect("branch_rlp_node must always be present for extension nodes"); + + ExtensionNodeRef::new(&self.key, branch_rlp_node.as_slice()).length() + } } diff --git a/crates/trie/trie/src/proof_v2/mod.rs b/crates/trie/trie/src/proof_v2/mod.rs index c713596fe45..392af474cd6 100644 --- a/crates/trie/trie/src/proof_v2/mod.rs +++ b/crates/trie/trie/src/proof_v2/mod.rs @@ -19,7 +19,7 @@ use reth_trie_common::{ prefix_set::PrefixSet, BranchNodeMasks, BranchNodeRef, BranchNodeV2, Nibbles, ProofTrieNodeV2, ProofV2Target, RlpNode, TrieNodeV2, }; -use std::cmp::Ordering; +use std::{cmp::Ordering, sync::Arc}; use tracing::{error, instrument, trace}; mod value; @@ -110,11 +110,8 @@ impl ProofCalculator { /// Sets the prefix set and returns `self`. /// - /// When given, all cached hashes matching the [`PrefixSet`] will be invalidated. When all but - /// one of a branch's children match the prefix set then that remaining child's cached hash, if - /// any, will also be invalidated. This allows for properly handling branch collapse situations, - /// where all but one child of a branch is deleted and the remaining child is required to be - /// unrevealed in order to collapse the branch. + /// When given, all cached hashes matching the [`PrefixSet`] will be invalidated and their + /// subtries recalculated. pub fn with_prefix_set(mut self, prefix_set: PrefixSet) -> Self { self.prefix_set = prefix_set; self @@ -266,9 +263,10 @@ where child_path: Nibbles, child: ProofTrieBranchChild, ) -> Result { - // If the child is already an `RlpNode` then there is nothing to do. - if let ProofTrieBranchChild::RlpNode(rlp_node) = child { - return Ok(rlp_node) + // An already encoded child only needs an extension if it has been rebased upward. + if matches!(&child, ProofTrieBranchChild::RlpNode { .. }) { + self.rlp_encode_buf.clear(); + return child.into_rlp(&mut self.rlp_encode_buf).map(|(node, _)| node) } // If we should retain the child then do so. @@ -357,7 +355,7 @@ where &mut self, targets: &mut Option>, ) -> Result<(), StateProofError> { - if matches!(self.child_stack.last(), Some(ProofTrieBranchChild::RlpNode(_))) { + if matches!(self.child_stack.last(), Some(ProofTrieBranchChild::RlpNode { .. })) { trace!(target: TRACE_TARGET, "Last child already committed, leaving stack unchanged"); return Ok(()) } @@ -369,9 +367,15 @@ where // Only commit immediately if retained for the proof. Otherwise, defer conversion // to pop_branch() to give DeferredEncoder time for async work. if self.should_retain(targets, &child_path, true) { + let (hash_mask_bit, tree_mask_bit) = child.mask_bits(); let child_rlp_node = self.commit_child(targets, child_path, child)?; trace!(target: TRACE_TARGET, ?child_rlp_node, "Pushing committed child RlpNode onto stack"); - self.child_stack.push(ProofTrieBranchChild::RlpNode(child_rlp_node)); + self.child_stack.push(ProofTrieBranchChild::RlpNode { + node: child_rlp_node, + short_key: Nibbles::new(), + hash_mask_bit, + tree_mask_bit, + }); } else { trace!(target: TRACE_TARGET, "Pushing uncommitted child onto stack"); self.child_stack.push(child); @@ -380,35 +384,84 @@ where Ok(()) } - /// Creates a new leaf node on a branch, setting its `state_mask` bit and pushing the leaf onto - /// the `child_stack`. - /// - /// # Panics - /// - /// - If `branch_stack` is empty - /// - If the leaf's nibble is already set in the branch's `state_mask`. - fn push_new_leaf<'a>( + /// Adds a child at its full path, possibly collapsing an existing branch and/or creating a new + /// one depending on the path. + fn push_child<'a>( &mut self, targets: &mut Option>, - leaf_nibble: u8, - leaf_short_key: Nibbles, - leaf_val: VE::DeferredEncoder, + mut child: ProofTrieBranchChild, ) -> Result<(), StateProofError> { - // Before pushing the new leaf onto the `child_stack` we need to commit the previous last - // child, so that only `child_stack`'s final child is a non-RlpNode. - self.commit_last_child(targets)?; + let path = *child.short_key(); - // Once the last child is committed we set the new child's bit on the top branch's - // `state_mask` and push that new child. - let branch = self.branch_stack.last_mut().expect("branch_stack cannot be empty"); + loop { + trace!( + target: TRACE_TARGET, + ?path, + branch_stack_len = ?self.branch_stack.len(), + branch_path = ?self.branch_path, + child_stack_len = ?self.child_stack.len(), + "push_child: loop", + ); - debug_assert!(!branch.state_mask.is_bit_set(leaf_nibble)); - branch.state_mask.set_bit(leaf_nibble); + // Get the `state_mask` of the branch currently being built. If there are no branches + // on the stack then the trie is either empty or contains only a single child. + let (nibble, short_key) = match self.branch_stack.last().map(|branch| branch.state_mask) + { + None if self.child_stack.is_empty() => { + // The first child is the root and already has the correct short key. + self.child_stack.push(child); + return Ok(()) + } + None => { + // Split the existing root child from the new child by their common prefix. + debug_assert_eq!(self.child_stack.len(), 1); + debug_assert!(!self + .child_stack + .last() + .expect("already checked for emptiness") + .short_key() + .is_empty()); + self.push_new_branch(path) + } + Some(state_mask) => { + // Find the number of nibbles shared by the current branch and the new child. + let common_prefix_len = self.branch_path.common_prefix_length(&path); + + // A branch which is not a prefix of the child cannot receive any more + // children, so finish it and try again with its parent. + if common_prefix_len < self.branch_path.len() { + self.pop_branch(targets)?; + continue + } - self.child_stack - .push(ProofTrieBranchChild::Leaf { short_key: leaf_short_key, value: leaf_val }); + // If this nibble is already occupied, split the existing child from the new + // child. Otherwise the new child can be inserted directly into this branch. + let nibble = path.get_unchecked(common_prefix_len); + if state_mask.is_bit_set(nibble) { + self.push_new_branch(path) + } else { + (nibble, trim_nibbles_prefix(&path, common_prefix_len + 1)) + } + } + }; - Ok(()) + // Store the child key relative to its new parent branch. + child.trim_short_key_prefix(path.len() - short_key.len()); + + // Commit the preceding child before pushing this one, so only the final child on the + // stack can remain uncommitted. + self.commit_last_child(targets)?; + + let branch = self.branch_stack.last_mut().expect("branch_stack cannot be empty"); + debug_assert!(!branch.state_mask.is_bit_set(nibble)); + + // Mark the nibble occupied now that the preceding child has been committed. + branch.state_mask.set_bit(nibble); + + // The parent now contains this child at `nibble`. + self.child_stack.push(child); + return Ok(()) + } } /// Pushes a new branch onto the `branch_stack` based on the path and short key of the last @@ -416,69 +469,57 @@ where /// stack after this call. /// /// Returns the nibble of the branch's `state_mask` which should be set for the new child, and - /// short key that the next child should use. + /// the short key that child should use. fn push_new_branch(&mut self, new_child_path: Nibbles) -> (u8, Nibbles) { - // First determine the new child's shortkey relative to the current branch. If there is no - // current branch then the short key is the full path. - let new_child_short_key = if self.branch_stack.is_empty() { - new_child_path - } else { - // When there is a current branch then trim off its path as well as the nibble that it - // has set for this leaf. - trim_nibbles_prefix(&new_child_path, self.branch_path.len() + 1) - }; - - // Get the new branch's first child, which is the child on the top of the stack with which - // the new child shares the same nibble on the current branch. - let first_child = self + // Get the full path to the root of the existing child. The trie root is used when there is + // no parent branch. + let first_child_path = self + .last_child_path() + .expect("push_new_branch requires the current branch to have a child"); + + // Put both children in the same coordinate system by trimming the existing child's root + // path from the new child's full path. + let new_child_short_key = trim_nibbles_prefix(&new_child_path, first_child_path.len()); + let first_child_short_key = *self .child_stack - .last_mut() - .expect("push_new_branch can't be called with empty child_stack"); + .last() + .expect("push_new_branch can't be called with empty child_stack") + .short_key(); + debug_assert!(!first_child_short_key.is_empty()); - let first_child_short_key = first_child.short_key(); - debug_assert!( - !first_child_short_key.is_empty(), - "push_new_branch called when top child on stack is not a leaf or extension with a short key", - ); - - // Determine how many nibbles are shared between the new branch's first child and the new - // child. This common prefix will be the extension of the new branch + // Their shared prefix becomes the new branch's extension. The first differing nibble from + // each key becomes that child's position in the branch. let common_prefix_len = first_child_short_key.common_prefix_length(&new_child_short_key); - - // Trim off the common prefix from the first child's short key, plus one nibble which will - // stored by the new branch itself in its state mask. let first_child_nibble = first_child_short_key.get_unchecked(common_prefix_len); - first_child.trim_short_key_prefix(common_prefix_len + 1); - - // Similarly, trim off the common prefix, plus one nibble for the new branch, from the new - // child's short key. let new_child_nibble = new_child_short_key.get_unchecked(common_prefix_len); + + // Remove the extension and branch nibble from the new child's remaining short key. let new_child_short_key = trim_nibbles_prefix(&new_child_short_key, common_prefix_len + 1); - // Update the branch path to reflect the new branch about to be pushed. Its path will be - // the path of the previous branch, plus the nibble shared by each child, plus the parent - // extension (denoted by a non-zero `ext_len`). Since the new branch's path is a prefix of - // the original new_child_path we can just slice that. - // - // If the new branch is the first branch then we do not add the extra 1, as there is no - // nibble in a parent branch to account for. - let branch_path_len = - self.branch_path.len() + common_prefix_len + self.maybe_parent_nibble(); + // The new branch starts after the existing child's parent path and their shared extension. + let branch_path_len = first_child_path.len() + common_prefix_len; self.branch_path = new_child_path.slice_unchecked(0, branch_path_len); + // Rebase the existing child beneath the new branch by removing the extension and its + // branch nibble from the front of its short key. + let first_child = self + .child_stack + .last_mut() + .expect("push_new_branch can't be called with empty child_stack"); + first_child.trim_short_key_prefix(common_prefix_len + 1); + // Push the new branch onto the `branch_stack`. We do not yet set the `state_mask` bit of // the new child; whatever actually pushes the child onto the `child_stack` is expected to // do that. self.branch_stack.push(ProofTrieBranch { ext_len: common_prefix_len as u8, state_mask: TrieMask::new(1 << first_child_nibble), - masks: None, }); trace!( target: TRACE_TARGET, ?new_child_path, - ?common_prefix_len, + ext_len = ?common_prefix_len, ?first_child_nibble, branch_path = ?self.branch_path, "Pushed new branch", @@ -512,6 +553,7 @@ where self.commit_last_child(targets)?; let mut rlp_nodes_buf = self.take_rlp_nodes_buf(); + let mut masks = BranchNodeMasks::default(); let branch = self.branch_stack.pop().expect("branch_stack cannot be empty"); // Take the branch's children off the stack, using the state mask to determine how many @@ -528,20 +570,19 @@ where // Collect children into RlpNode Vec. Children are in lexicographic order. rlp_nodes_buf.reserve(num_children); - for child in self.child_stack.drain(self.child_stack.len() - num_children..) { - let child_rlp_node = match child { - ProofTrieBranchChild::RlpNode(rlp_node) => rlp_node, - uncommitted_child => { - // Convert uncommitted child (not retained for proof) to RlpNode now. - self.rlp_encode_buf.clear(); - let (rlp_node, freed_buf) = - uncommitted_child.into_rlp(&mut self.rlp_encode_buf)?; - if let Some(buf) = freed_buf { - self.rlp_nodes_bufs.push(buf); - } - rlp_node - } - }; + for (nibble, child) in branch + .state_mask + .iter() + .zip(self.child_stack.drain(self.child_stack.len() - num_children..)) + { + let (hash_mask_bit, tree_mask_bit) = child.mask_bits(); + masks.set_child_bits(nibble, hash_mask_bit, tree_mask_bit); + + self.rlp_encode_buf.clear(); + let (child_rlp_node, freed_buf) = child.into_rlp(&mut self.rlp_encode_buf)?; + if let Some(buf) = freed_buf { + self.rlp_nodes_bufs.push(buf); + } rlp_nodes_buf.push(child_rlp_node); } @@ -567,100 +608,23 @@ where Some(RlpNode::from_rlp(&self.rlp_encode_buf)) }; - // Wrap the `BranchNodeV2` so it can be pushed onto the child stack. - let branch_as_child = ProofTrieBranchChild::Branch { - node: BranchNodeV2::new(short_key, rlp_nodes_buf, branch.state_mask, rlp_node), - masks: branch.masks, - }; - - self.child_stack.push(branch_as_child); - // Update the branch_path. If this branch is the only branch then only its extension needs // to be trimmed, otherwise we also need to remove its nibble from its parent. let new_path_len = self.branch_path.len() - branch.ext_len as usize - self.maybe_parent_nibble(); + // Wrap the `BranchNodeV2` so it can be pushed onto the child stack. + let branch_as_child = ProofTrieBranchChild::Branch { + node: BranchNodeV2::new(short_key, rlp_nodes_buf, branch.state_mask, rlp_node), + masks: (!masks.is_empty()).then_some(masks), + }; + debug_assert!(self.branch_path.len() >= new_path_len); self.branch_path = self.branch_path.slice_unchecked(0, new_path_len); - Ok(()) - } - - /// Adds a single leaf for a key to the stack, possibly collapsing an existing branch and/or - /// creating a new one depending on the path of the key. - fn push_leaf<'a>( - &mut self, - targets: &mut Option>, - key: Nibbles, - val: VE::DeferredEncoder, - ) -> Result<(), StateProofError> { - loop { - trace!( - target: TRACE_TARGET, - ?key, - branch_stack_len = ?self.branch_stack.len(), - branch_path = ?self.branch_path, - child_stack_len = ?self.child_stack.len(), - "push_leaf: loop", - ); - - // Get the `state_mask` of the branch currently being built. If there are no branches - // on the stack then it means either the trie is empty or only a single leaf has been - // added previously. - let curr_branch_state_mask = match self.branch_stack.last() { - Some(curr_branch) => curr_branch.state_mask, - None if self.child_stack.is_empty() => { - // If the child stack is empty then this is the first leaf, push it and be done - self.child_stack - .push(ProofTrieBranchChild::Leaf { short_key: key, value: val }); - return Ok(()) - } - None => { - // If the child stack is not empty then it must only have a single other child - // which is either a leaf or extension with a non-zero short key. - debug_assert_eq!(self.child_stack.len(), 1); - debug_assert!(!self - .child_stack - .last() - .expect("already checked for emptiness") - .short_key() - .is_empty()); - let (nibble, short_key) = self.push_new_branch(key); - self.push_new_leaf(targets, nibble, short_key, val)?; - return Ok(()) - } - }; - - // Find the common prefix length, which is the number of nibbles shared between the - // current branch and the key. - let common_prefix_len = self.branch_path.common_prefix_length(&key); - - // If the current branch does not share all of its nibbles with the new key then it is - // not the parent of the new key. In this case the current branch will have no more - // children. We can pop it and loop back to the top to try again with its parent branch. - if common_prefix_len < self.branch_path.len() { - self.pop_branch(targets)?; - continue - } - - // If the current branch is a prefix of the new key then the leaf is a child of the - // branch. If the branch doesn't have the leaf's nibble set then the leaf can be added - // directly, otherwise a new branch must be created in-between this branch and that - // existing child. - let nibble = key.get_unchecked(common_prefix_len); - if curr_branch_state_mask.is_bit_set(nibble) { - // Push a new branch which splits the short key of the existing child at this - // nibble. - let (nibble, short_key) = self.push_new_branch(key); - // Push the new leaf onto the new branch. - self.push_new_leaf(targets, nibble, short_key, val)?; - } else { - let short_key = key.slice_unchecked(common_prefix_len + 1, key.len()); - self.push_new_leaf(targets, nibble, short_key, val)?; - } + self.child_stack.push(branch_as_child); - return Ok(()) - } + Ok(()) } /// Given the lower and upper bounds (exclusive) of a range of keys, iterates over the @@ -709,13 +673,13 @@ where ); } - // Loop over all keys in the range, calling `push_leaf` on each. + // Loop over all keys in the range, pushing each leaf onto the stack. while hashed_cursor_state .path() .is_some_and(|key| upper_bound.is_none_or(|upper_bound| key < &upper_bound)) { let (key, val) = hashed_cursor_state.take(); - self.push_leaf(targets, key, val)?; + self.push_child(targets, ProofTrieBranchChild::Leaf { short_key: key, value: val })?; *hashed_cursor_state = HashedCursorState::seeked( key, self.hashed_cursor.next()?.map(&mut map_hashed_cursor_entry), @@ -726,165 +690,42 @@ where Ok(()) } - /// Constructs and returns a new [`ProofTrieBranch`] based on an existing [`BranchNodeCompact`]. - #[inline] - const fn new_from_cached_branch( - cached_branch: &BranchNodeCompact, - ext_len: u8, - ) -> ProofTrieBranch { - ProofTrieBranch { - ext_len, - state_mask: TrieMask::new(0), - masks: Some(BranchNodeMasks { - tree_mask: cached_branch.tree_mask, - hash_mask: cached_branch.hash_mask, - }), - } - } - - /// Pushes a new branch onto the `branch_stack` which is based on a cached branch obtained via - /// the trie cursor. - /// - /// If there is already a child at the top branch of `branch_stack` occupying this new branch's - /// nibble then that child will have its short-key split with another new branch, and this - /// cached branch will be a child of that splitting branch. - fn push_cached_branch<'a>( + /// Takes a cached branch from the trie cursor and invalidates its hashes if it may collapse. + fn take_cached_branch( &mut self, - targets: &mut Option>, - cached_path: Nibbles, - cached_branch: &BranchNodeCompact, - ) -> Result<(), StateProofError> { - debug_assert!( - cached_path.starts_with(&self.branch_path), - "push_cached_branch called with path {cached_path:?} which is not a child of current branch {:?}", - self.branch_path, - ); - - let parent_branch = self.branch_stack.last(); - - // If both stacks are empty then there were no leaves before this cached branch, push it and - // be done; the extension of the branch will be its full path. - if self.child_stack.is_empty() && parent_branch.is_none() { - self.branch_path = cached_path; - self.branch_stack - .push(Self::new_from_cached_branch(cached_branch, cached_path.len() as u8)); - return Ok(()) - } - - // Get the nibble which should be set in the parent branch's `state_mask` for this new - // branch. - let cached_branch_nibble = cached_path.get_unchecked(self.branch_path.len()); - - // We calculate the `ext_len` of the new branch, and potentially update its nibble if a new - // parent branch is inserted here, based on the state of the parent branch. - let (cached_branch_nibble, ext_len) = if parent_branch - .is_none_or(|parent_branch| parent_branch.state_mask.is_bit_set(cached_branch_nibble)) - { - // If the `child_stack` is not empty but the `branch_stack` is then it implies that - // there must be a leaf or extension at the root of the trie whose short-key will get - // split by a new branch, which will become the parent of both that leaf/extension and - // this new branch. - // - // Similarly, if there is a branch on the `branch_stack` but its `state_mask` bit for - // this new branch is already set, then there must be a leaf/extension with a short-key - // to be split. - debug_assert!(!self - .child_stack - .last() - .expect("already checked for emptiness") - .short_key() - .is_empty()); - - // Split that leaf/extension's short key with a new branch. - let (nibble, short_key) = self.push_new_branch(cached_path); - (nibble, short_key.len()) - } else { - // If there is a parent branch but its `state_mask` bit for this branch is not set - // then we can simply calculate the `ext_len` based on the difference of each, minus - // 1 to account for the nibble in the `state_mask`. - (cached_branch_nibble, cached_path.len() - self.branch_path.len() - 1) - }; - - // `commit_last_child` relies on the last set bit of the parent branch's `state_mask` to - // determine the path of the last child on the `child_stack`. Since we are about to - // change that mask we need to commit that last child first. - self.commit_last_child(targets)?; - - // When pushing a new branch we need to set its child nibble in the `state_mask` of - // its parent, if there is one. - if let Some(parent_branch) = self.branch_stack.last_mut() { - parent_branch.state_mask.set_bit(cached_branch_nibble); - } - - // Finally update the `branch_path` and push the new branch. - self.branch_path = cached_path; - self.branch_stack.push(Self::new_from_cached_branch(cached_branch, ext_len as u8)); - - trace!( - target: TRACE_TARGET, - branch=?self.branch_stack.last(), - branch_path=?self.branch_path, - "Pushed cached branch", - ); - - Ok(()) - } - - /// Wraps [`TrieCursor::seek`], skipping cached branches whose sub-tries must be recalculated - /// from leaves. - /// - /// A cached branch is skipped when all but at most one of its children match the prefix set. - /// In that case those children might all be deleted, leaving a branch with a single child. - /// A single-child branch must be collapsed, but collapsing requires the child to be a full - /// node (not a cached hash). Skipping the branch avoids this by forcing recalculation. - fn trie_cursor_seek( - &mut self, - key: Nibbles, - ) -> Result, StateProofError> { - let mut entry = self.trie_cursor.seek(key)?; - while let Some((ref path, ref branch)) = entry { - if !self.should_skip_cached_branch(path, branch) { - break - } - entry = self.trie_cursor.next()?; - } - Ok(entry) - } - - /// Returns true if the cached branch should be skipped entirely and its sub-trie recalculated - /// from leaves. - fn should_skip_cached_branch( - &mut self, - cached_path: &Nibbles, - cached_branch: &BranchNodeCompact, - ) -> bool { - if !self.prefix_set.contains(cached_path) { - return false - } + trie_cursor_state: &mut TrieCursorState, + ) -> (Nibbles, BranchNodeCompact) { + let (cached_path, mut cached_branch) = trie_cursor_state.take(); - let mut num_unmatched = 0u32; - let mut child_path = *cached_path; - for nibble in 0u8..16 { - if cached_branch.state_mask.is_bit_set(nibble) { + if self.prefix_set.contains(&cached_path) { + let mut unchanged_children = 0; + let mut child_path = cached_path; + for nibble in cached_branch.state_mask.iter() { child_path.truncate(cached_path.len()); child_path.push_unchecked(nibble); if !self.prefix_set.contains(&child_path) { - num_unmatched += 1; + unchanged_children += 1; + if unchanged_children > 1 { + break + } } } - } - if num_unmatched <= 1 { - trace!( - target: TRACE_TARGET, - ?cached_path, - ?num_unmatched, - "Skipping cached branch: all but <=1 children match prefix set, branch may collapse", - ); - true - } else { - false + // If every dirty child is deleted, a lone unchanged child must be revealed so the + // branch can collapse into it. Keep the masks as structural hints, but prevent any of + // this branch's cached hashes from hiding that child. + if unchanged_children <= 1 { + Arc::make_mut(&mut cached_branch.hashes).fill(B256::ZERO); + trace!( + target: TRACE_TARGET, + ?cached_path, + ?unchanged_children, + "Invalidated cached hashes because branch may collapse", + ); + } } + + (cached_path, cached_branch) } /// Attempts to pop off the top branch of the `cached_branch_stack`, returning @@ -906,8 +747,17 @@ where return Ok(PopCachedBranchOutcome::Exhausted) }; - // If there is a branch on top of the stack we use that. - if let Some(cached) = self.cached_branch_stack.pop() { + // Use the branch on top of the stack unless a previously calculated range has already + // covered its entire subtrie. + while let Some(cached) = self.cached_branch_stack.pop() { + if cached + .0 + .next_without_prefix() + .is_some_and(|upper_bound| upper_bound <= *uncalculated_lower_bound) + { + trace!(target: TRACE_TARGET, cached_path=?cached.0, ?uncalculated_lower_bound, "Skipping covered cached branch"); + continue + } return Ok(PopCachedBranchOutcome::Popped(cached)); } @@ -926,7 +776,7 @@ where if trie_cursor_path < uncalculated_lower_bound { *trie_cursor_state = TrieCursorState::seeked( *uncalculated_lower_bound, - self.trie_cursor_seek(*uncalculated_lower_bound)?, + self.trie_cursor.seek(*uncalculated_lower_bound)?, ); // Having just seeked forward we need to check if the cursor is now exhausted, @@ -950,7 +800,7 @@ where // stack is empty. // // We will use this `Available` cached branch as our next branch. - let cached = trie_cursor_state.take(); + let cached = self.take_cached_branch(trie_cursor_state); trace!(target: TRACE_TARGET, cached=?cached, "Pushed next trie node onto cached_branch_stack"); // If the calculated range is not caught up to the next cached branch it means there @@ -1041,8 +891,73 @@ where Ok(None) } + // Returns the next child nibble on the current branch to process, or None if there are no + // further nibbles to process. + fn next_uncached_child_nibble( + prefix_set: &mut PrefixSet, + branch_path: &Nibbles, + uncalculated_lower_bound_ref: &Nibbles, + cached_state_mask: TrieMask, + ) -> Option { + let mut next_child_nibbles = cached_state_mask; + + // Also include child nibbles indicated by the prefix set. The prefix set can + // indicate children that need recalculation from leaves (e.g. new keys inserted + // under this branch). + if prefix_set.contains(branch_path) { + let branch_path_len = branch_path.len(); + let mut child_path = *branch_path; + for nibble in 0u8..16 { + child_path.truncate(branch_path_len); + child_path.push_unchecked(nibble); + if prefix_set.contains(&child_path) { + next_child_nibbles.set_bit(nibble); + } + } + } + + let _orig_next_child_nibbles = next_child_nibbles; + + // Mask out any child nibbles whose ranges have already been fully processed. + // This can happen when `calculate_key_range` finds no keys for a child's range, + // leaving the child's bit unset in `state_mask`. Without this, re-entering this + // function would select the same child again. + if uncalculated_lower_bound_ref.starts_with(branch_path) && + uncalculated_lower_bound_ref.len() > branch_path.len() + { + let lower_nibble = uncalculated_lower_bound_ref.get_unchecked(branch_path.len()); + // Clear all nibbles strictly below `lower_nibble`. If the lower bound is within the + // current child, the remainder of that child may still need to be processed. + let already_processed_mask = TrieMask::new((1u16 << lower_nibble) - 1); + next_child_nibbles &= !already_processed_mask; + trace!( + target: TRACE_TARGET, + ?branch_path, + ?_orig_next_child_nibbles, + ?already_processed_mask, + ?next_child_nibbles, + "Unset already processed key nibbles from next_child_nibbles", + ); + } else if !uncalculated_lower_bound_ref.starts_with(branch_path) && + uncalculated_lower_bound_ref > branch_path + { + // The lower bound has moved entirely past this branch (e.g. branch is 0x6 but + // lower is 0x7). All remaining children have been processed. + next_child_nibbles = TrieMask::default(); + trace!( + target: TRACE_TARGET, + ?branch_path, + ?_orig_next_child_nibbles, + ?next_child_nibbles, + "Unset all nibbles from next_child_nibbles due to branch_path being outside this subtrie", + ); + } + + next_child_nibbles.first_set_bit_index() + } + /// Accepts the current state of both hashed and trie cursors, and determines the next range of - /// hashed keys which need to be processed using [`Self::push_leaf`]. + /// hashed keys which need to be processed using [`Self::calculate_key_range`]. /// /// This method will use cached branch node data from the trie cursor to skip over all possible /// ranges of keys, to reduce computation as much as possible. @@ -1051,11 +966,11 @@ where /// /// - `None`: No more data to process, finish computation /// - /// - `Some(lower, None)`: Indicates to call `push_leaf` on all keys starting at `lower`, with - /// no upper bound. This method won't be called again after this. + /// - `Some(lower, None)`: Indicates to process all keys starting at `lower`, with no upper + /// bound. This method won't be called again after this. /// - /// - `Some(lower, Some(upper))`: Indicates to call `push_leaf` on all keys starting at `lower`, - /// up to but excluding `upper`, and then call this method once done. + /// - `Some(lower, Some(upper))`: Indicates to process all keys starting at `lower`, up to but + /// excluding `upper`, and then call this method once done. /// /// Once returned the `branch_stack` will be in the correct state to start calculating leaves /// for the given range, if any. @@ -1124,107 +1039,36 @@ where return Ok(Some(range)) } - // Since we've popped all branches which don't start with cached_path, branch_path at - // this point must be equal to or shorter than cached_path. + // Since we've popped all constructed branches which don't contain `cached_path`, the + // remaining branch path must be its prefix. debug_assert!( self.branch_path.len() < cached_path.len() || self.branch_path == cached_path, "branch_path {:?} is different-or-longer-than cached_path {cached_path:?}", self.branch_path ); - // If the branch_path != cached_path it means the branch_stack is either empty, or the - // top branch is the parent of this cached branch. Either way we push a branch - // corresponding to the cached one onto the stack, so we can begin constructing it. - if self.branch_path != cached_path { - // If the prefix set contains any entries from the lower bound up until the new - // cached path it means that there might be a new node(s) which split the extension - // node between cached_path and its parent (self.branch_path). - if uncalculated_lower_bound_ref < &cached_path && - self.prefix_set.contains_range(uncalculated_lower_bound_ref..&cached_path) - { - self.cached_branch_stack.push((cached_path, cached_branch)); - return Ok(Some((*uncalculated_lower_bound_ref, Some(cached_path)))) - } - - self.push_cached_branch(targets, cached_path, &cached_branch)?; - } - - // At this point the top of the branch stack is the same branch which was found in the - // cache. - let curr_branch = - self.branch_stack.last().expect("top of branch_stack corresponds to cached branch"); - - let cached_state_mask = cached_branch.state_mask; - let curr_state_mask = curr_branch.state_mask; - - // Determine all child nibbles which are set in the cached branch but not the - // under-construction branch. - let mut next_child_nibbles = curr_state_mask ^ cached_state_mask; - - // Also include child nibbles indicated by the prefix set. The prefix set can - // indicate children that need recalculation from leaves (e.g. new keys inserted - // under this branch). Skip nibbles already set in `curr_state_mask` since those - // children have already been constructed. - if self.prefix_set.contains(&self.branch_path) { - let branch_path_len = self.branch_path.len(); - let mut child_path = self.branch_path; - for nibble in 0u8..16 { - if !curr_state_mask.is_bit_set(nibble) { - child_path.truncate(branch_path_len); - child_path.push_unchecked(nibble); - if self.prefix_set.contains(&child_path) { - next_child_nibbles.set_bit(nibble); - } - } - } - } - - let _orig_next_child_nibbles = next_child_nibbles; - - // Mask out any child nibbles whose ranges have already been fully processed. - // This can happen when `calculate_key_range` finds no keys for a child's range, - // leaving the child's bit unset in `state_mask`. Without this, re-entering this - // function would select the same child again. - if uncalculated_lower_bound_ref.starts_with(&self.branch_path) && - uncalculated_lower_bound_ref.len() > self.branch_path.len() + // Dirty keys before this cached path may split the extension leading to it. Process + // them before using the cached branch as a hint. + if uncalculated_lower_bound_ref < &cached_path && + self.prefix_set.contains_range(uncalculated_lower_bound_ref..&cached_path) { - let lower_nibble = - uncalculated_lower_bound_ref.get_unchecked(self.branch_path.len()); - // Clear all nibbles strictly below `lower_nibble` since they've been processed. - let already_processed_mask = TrieMask::new((1u16 << lower_nibble) - 1); - next_child_nibbles &= !already_processed_mask; - trace!( - target: TRACE_TARGET, - branch_path = ?self.branch_path, - ?_orig_next_child_nibbles, - ?already_processed_mask, - ?next_child_nibbles, - "Unset already processed key nibbles from next_child_nibbles", - ); - } else if !uncalculated_lower_bound_ref.starts_with(&self.branch_path) && - uncalculated_lower_bound_ref > &self.branch_path - { - // The lower bound has moved entirely past this branch (e.g. branch is 0x6 but - // lower is 0x7). All remaining children have been processed. - next_child_nibbles = TrieMask::default(); - trace!( - target: TRACE_TARGET, - branch_path = ?self.branch_path, - ?_orig_next_child_nibbles, - ?next_child_nibbles, - "Unset all nibbles from next_child_nibbles due to branch_path being outside this subtrie", - ); + self.cached_branch_stack.push((cached_path, cached_branch)); + return Ok(Some((*uncalculated_lower_bound_ref, Some(cached_path)))) } - // Defer popping this completed branch to `commit_branches`, which checks for dirty - // keys around it before looping with the parent branch. - if next_child_nibbles.is_empty() { + let child_nibble = Self::next_uncached_child_nibble( + &mut self.prefix_set, + &cached_path, + uncalculated_lower_bound_ref, + cached_branch.state_mask, + ); + + let Some(child_nibble) = child_nibble else { trace!( target: TRACE_TARGET, path=?cached_path, - ?curr_branch, ?cached_branch, - "No further children", + "No further cached children", ); // no need to pop from `cached_branch_stack`, the current cached branch is already @@ -1235,12 +1079,11 @@ where uncalculated_lower_bound = cached_path.next_without_prefix(); continue - } + }; - // Determine the next nibble of the branch which has not yet been constructed, and - // determine the child's full path. - let child_nibble = next_child_nibbles.trailing_zeros() as u8; - let child_path = self.child_path_at(child_nibble); + let mut child_path = cached_path; + child_path.push_unchecked(child_nibble); + let child_lower_bound = (*uncalculated_lower_bound_ref).max(child_path); // If the `hash_mask` bit is set for the next child it means the child's hash is cached // in the `cached_branch`. We can use that instead of re-calculating the hash of the @@ -1252,46 +1095,54 @@ where // If the child's path is in the prefix set then the cached hash is stale and must // not be used. if cached_branch.hash_mask.is_bit_set(child_nibble) && + child_lower_bound == child_path && !self.prefix_set.contains(&child_path) { - // Commit the last child. We do this here for two reasons: - // - `commit_last_child` will check if the last child needs to be retained. We need - // to check that before the subsequent `should_retain` call here to prevent - // `targets` from being moved beyond the last child before it is checked. - // - If we do end up using the cached hash value, then we will need to commit the - // last child before pushing a new one onto the stack anyway. - self.commit_last_child(targets)?; - - if !self.should_retain(targets, &child_path, false) { - // Pull this child's hash out of the cached branch node. The hash index - // is the number of hash_mask bits set below this child's nibble. - let lower_bits = TrieMask::new((1u16 << child_nibble) - 1); - let hash_idx = (cached_branch.hash_mask & lower_bits).count_ones() as usize; - let hash = cached_branch.hashes[hash_idx]; - - trace!( - target: TRACE_TARGET, - ?child_path, - ?hash_idx, - ?hash, - "Using cached hash for child", - ); - - self.child_stack.push(ProofTrieBranchChild::RlpNode(RlpNode::word_rlp(&hash))); - self.branch_stack - .last_mut() - .expect("already asserted there is a last branch") - .state_mask - .set_bit(child_nibble); - - // Update the `uncalculated_lower_bound` to indicate that the child whose bit - // was just set is completely processed. - uncalculated_lower_bound = child_path.next_without_prefix(); + // Pull this child's hash out of the cached branch node. The hash index is the + // number of hash_mask bits set below this child's nibble. + let lower_bits = TrieMask::new((1u16 << child_nibble) - 1); + let hash_idx = (cached_branch.hash_mask & lower_bits).count_ones() as usize; + let hash = cached_branch.hashes[hash_idx]; + + // `take_cached_branch` replaces hashes with zero when their nodes must be + // revealed to support a possible branch collapse. + if hash != B256::ZERO { + let mut probed_targets = targets.clone(); + if !self.should_retain(&mut probed_targets, &child_path, false) { + trace!( + target: TRACE_TARGET, + ?child_path, + ?hash_idx, + ?hash, + "Using cached hash for child", + ); + + // Keep this hint available while inserting the child so a branch which is + // naturally materialized at `cached_path` can inherit its masks. + let tree_mask_bit = cached_branch.tree_mask.is_bit_set(child_nibble); + self.cached_branch_stack.push((cached_path, cached_branch)); + self.push_child( + targets, + ProofTrieBranchChild::RlpNode { + node: RlpNode::word_rlp(&hash), + short_key: child_path, + hash_mask_bit: true, + tree_mask_bit, + }, + )?; + + if let (Some(targets), Some(probed_targets)) = + (targets.as_mut(), probed_targets) + { + targets.i = targets.i.max(probed_targets.i); + } - // Push the current cached branch back onto the stack before looping. - self.cached_branch_stack.push((cached_path, cached_branch)); + // Update the `uncalculated_lower_bound` to indicate that the child whose + // bit was just set is completely processed. + uncalculated_lower_bound = child_path.next_without_prefix(); - continue + continue + } } } @@ -1299,12 +1150,14 @@ where // branch node may be the node at this child directly, or this child may be an // extension and the cached branch is the child of that extension. - // All trie nodes prior to `child_path` will not be modified further, so we can seek the - // trie cursor to the next cached node at-or-after `child_path`. - if trie_cursor_state.path().is_some_and(|path| path < &child_path) { - trace!(target: TRACE_TARGET, ?child_path, "Seeking trie cursor to child path"); - *trie_cursor_state = - TrieCursorState::seeked(child_path, self.trie_cursor_seek(child_path)?); + // All trie nodes prior to `child_lower_bound` have been processed, so seek the trie + // cursor forward if necessary. + if trie_cursor_state.path().is_some_and(|path| path < &child_lower_bound) { + trace!(target: TRACE_TARGET, ?child_lower_bound, "Seeking trie cursor to child lower bound"); + *trie_cursor_state = TrieCursorState::seeked( + child_lower_bound, + self.trie_cursor.seek(child_lower_bound)?, + ); } // If the next cached branch node is a child of `child_path` then we can assume it is @@ -1324,25 +1177,26 @@ where ?next_cached_branch, "Pushing cached branch for child", ); - self.cached_branch_stack.push(trie_cursor_state.take()); + let cached = self.take_cached_branch(trie_cursor_state); + self.cached_branch_stack.push(cached); continue; } // There is no cached data for the sub-trie at this child, we must recalculate the // sub-trie root (this child) using the leaves. Return the range of keys based on the // child path. - let child_path_upper = child_path.next_without_prefix(); + let child_upper_bound = child_path.next_without_prefix(); trace!( target: TRACE_TARGET, - lower=?child_path, - upper=?child_path_upper, + lower=?child_lower_bound, + upper=?child_upper_bound, "Returning sub-trie's key range to calculate", ); // Push the current cached branch back onto the stack before returning. self.cached_branch_stack.push((cached_path, cached_branch)); - return Ok(Some((child_path, child_path_upper))); + return Ok(Some((child_lower_bound, child_upper_bound))); } } @@ -1392,7 +1246,7 @@ where trace!(target: TRACE_TARGET, "Doing initial seek of trie cursor"); *trie_cursor_state = TrieCursorState::seeked( traversal_lower_bound, - self.trie_cursor_seek(traversal_lower_bound)?, + self.trie_cursor.seek(traversal_lower_bound)?, ); } @@ -1490,25 +1344,24 @@ where child_stack_empty = self.child_stack.is_empty(), "Maybe retaining local root", ); - // Either there was only a single leaf node placed on the child stack, or the final branch - // was popped off the branch stack and placed unencoded on the child stack. Either way the - // child stack will not have an RlpNode at this point. - let root_node = match self.child_stack.pop() { - Some(ProofTrieBranchChild::RlpNode(_)) => { - unreachable!("local root cannot be an encoded RLP node") - } - root_node => root_node, - }; + let root_node = self.child_stack.pop(); // A full-trie calculation always retains a root, using an empty root when traversal // produced no root node. let Some(parent_prefix) = sub_trie_targets.parent_prefix else { - let root_node = if let Some(root_node) = root_node { + let mut root_node = if let Some(root_node) = root_node { self.rlp_encode_buf.clear(); root_node.into_proof_trie_node(Nibbles::new(), &mut self.rlp_encode_buf)? } else { ProofTrieNodeV2::empty() }; + + // Direct root branches do not have an entry in the branch node table. A root extension + // still carries the masks of the child branch embedded within it. + if matches!(&root_node.node, TrieNodeV2::Branch(branch) if branch.key.is_empty()) { + root_node.masks = None; + } + self.retained_proofs.push(root_node); return Ok(()) }; @@ -1516,11 +1369,11 @@ where // If there's no root node then the subtrie has no keys, return nothing. let Some(mut root_node) = root_node else { return Ok(()) }; - let root_short_key = *root_node.short_key(); + let root_full_path = *root_node.short_key(); // An exact match reconstructed the already-revealed parent; its targeted children were // retained while that parent branch was popped. - if root_short_key == parent_prefix { + if root_full_path == parent_prefix { return Ok(()) } @@ -1529,9 +1382,9 @@ where // which this root should be rebased onto. // The local root of a partial calculation must be at or below its known parent. - if !root_short_key.starts_with(&parent_prefix) { + if !root_full_path.starts_with(&parent_prefix) { return Err(StateProofError::TrieInconsistency(format!( - "local root short key {root_short_key:?} does not start with parent prefix \ + "local root path {root_full_path:?} does not start with parent prefix \ {parent_prefix:?}", ))) } @@ -1539,7 +1392,7 @@ where // Keep the parent branch's child nibble in the proof path so the local root attaches // directly below that parent. let child_path_len = parent_prefix.len() + 1; - let child_path = root_short_key.slice_unchecked(0, child_path_len); + let child_path = root_full_path.slice_unchecked(0, child_path_len); // It's possible that the local root lies on a child which is not targeted. if !sub_trie_targets @@ -1827,6 +1680,7 @@ where /// /// It is assumed that the underlying slice is never empty, and that the iterator is never /// exhausted. +#[derive(Clone)] struct TargetsCursor<'a> { targets: &'a [ProofV2Target], i: usize, @@ -1985,7 +1839,7 @@ impl HashedCursorState { match self { Self::Unseeked => false, Self::Available(path, _) => path > key, - Self::Exhausted(exhausted_at) => exhausted_at > key, + Self::Exhausted(exhausted_at) => exhausted_at >= key, } } @@ -2023,7 +1877,8 @@ mod tests { use alloy_trie::proof::AddedRemovedKeys; use itertools::Itertools; use reth_trie_common::{ - prefix_set::PrefixSetMut, ProofTrieNode, ProofV2TargetParent, TrieNode, EMPTY_ROOT_HASH, + prefix_set::{PrefixSet, PrefixSetMut}, + ProofTrieNode, ProofV2TargetParent, TrieNode, EMPTY_ROOT_HASH, }; use std::collections::BTreeMap; @@ -2241,19 +2096,18 @@ mod tests { // Simulate stale state left by a mid-computation error: push fake entries onto internal // stacks and set a non-empty branch_path. - proof_calculator.branch_stack.push(ProofTrieBranch { - ext_len: 2, - state_mask: TrieMask::new(0b1111), - masks: None, - }); - proof_calculator.branch_stack.push(ProofTrieBranch { - ext_len: 0, - state_mask: TrieMask::new(0b11), - masks: None, - }); proof_calculator - .child_stack - .push(ProofTrieBranchChild::RlpNode(RlpNode::word_rlp(&B256::ZERO))); + .branch_stack + .push(ProofTrieBranch { ext_len: 2, state_mask: TrieMask::new(0b1111) }); + proof_calculator + .branch_stack + .push(ProofTrieBranch { ext_len: 0, state_mask: TrieMask::new(0b11) }); + proof_calculator.child_stack.push(ProofTrieBranchChild::RlpNode { + node: RlpNode::word_rlp(&B256::ZERO), + short_key: Nibbles::new(), + hash_mask_bit: false, + tree_mask_bit: false, + }); proof_calculator.branch_path = Nibbles::from_nibbles([0x1, 0x2, 0x3]); // clear_computation_state should reset everything so a subsequent call works. @@ -2462,6 +2316,29 @@ mod tests { assert_eq!(child_leaf.key, slot_nibbles.slice(1..)); } + #[test] + fn test_exhausted_cursor_resets_at_equal_target_boundary() { + let first = B256::ZERO; + let last = B256::with_last_byte(1); + let last_nibbles = Nibbles::unpack(last); + let harness = + ProofTestHarness::new(BTreeMap::from([(first, U256::from(1)), (last, U256::from(2))])); + let mut targets = [ + ProofV2Target::new(first), + ProofV2Target::new(last).with_parent(ProofV2TargetParent::new(63)), + ]; + + let (proof, root) = harness.proof_v2(&mut targets); + + assert_eq!(root, Some(harness.original_root())); + let child = proof + .iter() + .find(|node| node.path == last_nibbles) + .expect("depth-63 target child proof"); + let TrieNodeV2::Leaf(leaf) = &child.node else { panic!("target child should be a leaf") }; + assert!(leaf.key.is_empty()); + } + #[test] fn test_rebases_compressed_branch_subtrie_root() { let slot_a = B256::right_padding_from(&[0xae, 0xd4, 0x00]); @@ -2922,187 +2799,143 @@ mod tests { keccak256(&buf) } - /// Tests branch collapse when the removed child comes BEFORE the remaining child. - /// - /// Trie structure (3 hashed storage keys): - /// `key_a` = 0x20... (root nibble 2, sub-nibble 0) - /// `key_b` = 0x21... (root nibble 2, sub-nibble 1) - /// `key_c` = 0xb0... (root nibble b) - /// - /// This creates: - /// root branch at nibbles {2, b} - /// sub-branch at path [2] at nibbles {0, 1} - /// - /// `key_a` is removed (prefix set marks it dirty, cursor has no value for it). - /// The sub-branch at [2] collapses into its remaining child (`key_b`). The removed child - /// (nibble 0) comes before the remaining child (nibble 1). - #[test] - fn test_branch_collapse_removed_child_before_remaining() { + /// Checks collapsing a cached branch after removing one of its two direct branch children. + fn assert_branch_collapse(remaining_nibble: u8, removed_nibble: u8) { reth_tracing::init_test_tracing(); let val = U256::from(1u64); - - let key_a = B256::right_padding_from(&[0x20]); // root nibble 2, sub-nibble 0 - let key_b = B256::right_padding_from(&[0x21]); // root nibble 2, sub-nibble 1 - let key_c = B256::right_padding_from(&[0xb0]); // root nibble b - - // Compute leaf hashes for the sub-branch's children. - // The sub-branch at path [2] consumes 2 nibbles from each key (root nibble + sub-nibble). - let leaf_hash_a = storage_leaf_hash(&Nibbles::unpack(key_a).slice(2..), &val); - let leaf_hash_b = storage_leaf_hash(&Nibbles::unpack(key_b).slice(2..), &val); - - // Only cache the sub-branch at path [2] — the root will be built from leaves. - // The sub-branch has children at nibbles 0 and 1, both with cached hashes. - let sub_branch_state_mask = TrieMask::new((1 << 0) | (1 << 1)); - let cached_sub_branch = BranchNodeCompact::new( - sub_branch_state_mask, - TrieMask::new(0), - sub_branch_state_mask, - vec![leaf_hash_a, leaf_hash_b], - None, + let child_keys = |nibble| { + [ + B256::right_padding_from(&[0x20 | nibble, 0x00]), + B256::right_padding_from(&[0x20 | nibble, 0x10]), + ] + }; + let [remaining_a, remaining_b] = child_keys(remaining_nibble); + let [removed_a, removed_b] = child_keys(removed_nibble); + + // Build the cached trie through the production HashBuilder path. Each child at + // `remaining_nibble` and `removed_nibble` is a direct branch with two leaf children. + let initial_storage = BTreeMap::from([ + (remaining_a, val), + (remaining_b, val), + (removed_a, val), + (removed_b, val), + ]); + let harness = TrieTestHarness::new(initial_storage); + + let cached_branch = &harness + .storage_trie_updates() + .storage_nodes + .get(&Nibbles::from_nibbles([0x2])) + .expect("branch at 0x2"); + let child_mask = + TrieMask::from_nibble(remaining_nibble) | TrieMask::from_nibble(removed_nibble); + assert_eq!(cached_branch.state_mask, child_mask); + assert_eq!(cached_branch.hash_mask, child_mask); + assert!(cached_branch.tree_mask.is_empty()); + + let final_storage = BTreeMap::from([(remaining_a, val), (remaining_b, val)]); + let expected_root = TrieTestHarness::new(final_storage.clone()).original_root(); + let updated_hashed = MockHashedCursorFactory::new( + BTreeMap::new(), + std::iter::once((harness.hashed_address(), final_storage)).collect(), ); - let storage_nodes: BTreeMap = - std::iter::once((Nibbles::from_nibbles([0x2]), cached_sub_branch)).collect(); - - // The hashed cursor contains key_b and key_c (the root's other child). key_a was removed - // (not in cursor) - let mut harness = TrieTestHarness::new([(key_b, val), (key_c, val)].into_iter().collect()); - harness.set_trie_nodes(storage_nodes); - - // Prefix set marks key_a as dirty (removed). - let mut prefix_set_mut = PrefixSetMut::default(); - prefix_set_mut.insert(Nibbles::unpack(key_a)); - let prefix_set = prefix_set_mut.freeze(); + // The prefix set marks the entire removed child subtree dirty. + let mut prefix_set = PrefixSetMut::default(); + prefix_set.insert(Nibbles::unpack(removed_a)); + prefix_set.insert(Nibbles::unpack(removed_b)); - // Compute root with cached branches + prefix set — triggers sub-branch collapse. - let storage_trie_cursor = + let trie_cursor = harness.trie_cursor_factory().storage_trie_cursor(harness.hashed_address()).unwrap(); - let hashed_storage_cursor = harness - .hashed_cursor_factory() - .hashed_storage_cursor(harness.hashed_address()) - .unwrap(); - let mut calculator = - StorageProofCalculator::new_storage(storage_trie_cursor, hashed_storage_cursor) - .with_prefix_set(prefix_set); + let hashed_cursor = updated_hashed.hashed_storage_cursor(harness.hashed_address()).unwrap(); + let mut calculator = StorageProofCalculator::new_storage(trie_cursor, hashed_cursor) + .with_prefix_set(prefix_set.freeze()); let root_node = calculator .storage_root_node(harness.hashed_address()) .expect("storage_root_node should succeed after branch collapse"); - let root_with_collapse = + let root = calculator.compute_root_hash(core::slice::from_ref(&root_node)).unwrap().unwrap(); - // Compute reference root from scratch (no cached branches) using the full final state. - let mut fresh_harness = - TrieTestHarness::new([(key_b, val), (key_c, val)].into_iter().collect()); - fresh_harness.set_trie_nodes(BTreeMap::new()); - let storage_trie_cursor = fresh_harness - .trie_cursor_factory() - .storage_trie_cursor(fresh_harness.hashed_address()) - .unwrap(); - let hashed_storage_cursor = fresh_harness - .hashed_cursor_factory() - .hashed_storage_cursor(fresh_harness.hashed_address()) - .unwrap(); - let mut fresh_calculator = - StorageProofCalculator::new_storage(storage_trie_cursor, hashed_storage_cursor); - let fresh_root_node = fresh_calculator - .storage_root_node(fresh_harness.hashed_address()) - .expect("fresh storage_root_node should succeed"); - let expected_root = fresh_calculator - .compute_root_hash(core::slice::from_ref(&fresh_root_node)) - .unwrap() - .unwrap(); + pretty_assertions::assert_eq!(expected_root, root); + } - pretty_assertions::assert_eq!( - expected_root, - root_with_collapse, - "Root hash after collapsing branch (removed child before remaining) should match fresh computation" - ); + #[test] + fn test_branch_collapse_removed_child_before_remaining() { + assert_branch_collapse(1, 0); } - /// Tests branch collapse when the removed child comes AFTER the remaining child. - /// - /// Same trie structure as "before" test, but with nibbles 4 and 9 instead of 0 and 1 for - /// the sub-branch, and nibble 9 is removed. The removed child (nibble 9) comes after the - /// remaining child (nibble 4). #[test] fn test_branch_collapse_removed_child_after_remaining() { + assert_branch_collapse(4, 9); + } + + #[test] + fn test_prefix_set_root_proof_preserves_clean_sibling_after_cached_branch_collapse() { reth_tracing::init_test_tracing(); - let val = U256::from(1u64); + let dirty = B256::right_padding_from(&[0x10, 0x00, 0x10]); + let clean_sibling = B256::right_padding_from(&[0x10, 0x10]); + let storage = [ + B256::right_padding_from(&[0x10]), + dirty, + B256::right_padding_from(&[0x10, 0x01]), + B256::right_padding_from(&[0x10, 0x02]), + clean_sibling, + B256::right_padding_from(&[0x11]), + B256::right_padding_from(&[0x12]), + ] + .into_iter() + .map(|key| (key, U256::from(1u64))) + .collect(); + + let harness = ProofTestHarness::new(storage); + let expected_root = harness.original_root(); - // key_a at sub-nibble 4, key_b at sub-nibble 9 (under root nibble 2). - let key_a = B256::right_padding_from(&[0x24]); // root nibble 2, sub-nibble 4 - let key_b = B256::right_padding_from(&[0x29]); // root nibble 2, sub-nibble 9 - let key_c = B256::right_padding_from(&[0xb0]); // root nibble b + let mut prefix_set = PrefixSetMut::default(); + prefix_set.insert(Nibbles::unpack(dirty)); - let leaf_hash_a = storage_leaf_hash(&Nibbles::unpack(key_a).slice(2..), &val); - let leaf_hash_b = storage_leaf_hash(&Nibbles::unpack(key_b).slice(2..), &val); + let mut prefix_set_with_sibling = PrefixSetMut::default(); + prefix_set_with_sibling.insert(Nibbles::unpack(dirty)); + prefix_set_with_sibling.insert(Nibbles::unpack(clean_sibling)); - // Only cache the sub-branch at path [2] — the root will be built from leaves. - let sub_branch_state_mask = TrieMask::new((1 << 4) | (1 << 9)); - let cached_sub_branch = BranchNodeCompact::new( - sub_branch_state_mask, - TrieMask::new(0), - sub_branch_state_mask, - vec![leaf_hash_a, leaf_hash_b], - None, + pretty_assertions::assert_eq!( + Some(expected_root), + harness.root_with_prefix_set(prefix_set_with_sibling.freeze()), ); + pretty_assertions::assert_eq!( + Some(expected_root), + harness.root_with_prefix_set(prefix_set.freeze()), + "a dirty prefix must not omit a clean sibling after collapsing a cached branch", + ); + } - let storage_nodes: BTreeMap = - std::iter::once((Nibbles::from_nibbles([0x2]), cached_sub_branch)).collect(); - - // The hashed cursor contains key_a and key_c. key_b was removed (not in cursor) - let mut harness = TrieTestHarness::new([(key_a, val), (key_c, val)].into_iter().collect()); - harness.set_trie_nodes(storage_nodes); + #[test] + fn test_prefix_set_range_skips_covered_cached_branch() { + reth_tracing::init_test_tracing(); - // Prefix set marks key_b as dirty (removed). - let mut prefix_set_mut = PrefixSetMut::default(); - prefix_set_mut.insert(Nibbles::unpack(key_b)); - let prefix_set = prefix_set_mut.freeze(); + let before = B256::right_padding_from(&[0x30]); + let cached_a = B256::right_padding_from(&[0x80, 0x10]); + let cached_b = B256::right_padding_from(&[0x80, 0x15]); + let dirty = B256::right_padding_from(&[0x80, 0xc0]); + let after = B256::right_padding_from(&[0x90]); - // Compute root with cached branches + prefix set — triggers sub-branch collapse. - let storage_trie_cursor = - harness.trie_cursor_factory().storage_trie_cursor(harness.hashed_address()).unwrap(); - let hashed_storage_cursor = harness - .hashed_cursor_factory() - .hashed_storage_cursor(harness.hashed_address()) - .unwrap(); - let mut calculator = - StorageProofCalculator::new_storage(storage_trie_cursor, hashed_storage_cursor) - .with_prefix_set(prefix_set); - let root_node = calculator - .storage_root_node(harness.hashed_address()) - .expect("storage_root_node should succeed after branch collapse"); - let root_with_collapse = - calculator.compute_root_hash(core::slice::from_ref(&root_node)).unwrap().unwrap(); + // The cached branch at 0x80 remains parked while its children are rebuilt. Processing the + // dirty child advances the lower bound to 0x9, past the cached branch's entire range. + let storage = [before, cached_a, cached_b, dirty, after] + .into_iter() + .enumerate() + .map(|(i, key)| (key, U256::from(i + 1))) + .collect(); - // Compute reference root from scratch (no cached branches) using the full final state. - let mut fresh_harness = - TrieTestHarness::new([(key_a, val), (key_c, val)].into_iter().collect()); - fresh_harness.set_trie_nodes(BTreeMap::new()); - let storage_trie_cursor = fresh_harness - .trie_cursor_factory() - .storage_trie_cursor(fresh_harness.hashed_address()) - .unwrap(); - let hashed_storage_cursor = fresh_harness - .hashed_cursor_factory() - .hashed_storage_cursor(fresh_harness.hashed_address()) - .unwrap(); - let mut fresh_calculator = - StorageProofCalculator::new_storage(storage_trie_cursor, hashed_storage_cursor); - let fresh_root_node = fresh_calculator - .storage_root_node(fresh_harness.hashed_address()) - .expect("fresh storage_root_node should succeed"); - let expected_root = fresh_calculator - .compute_root_hash(core::slice::from_ref(&fresh_root_node)) - .unwrap() - .unwrap(); + let harness = ProofTestHarness::new(storage); + let expected_root = harness.original_root(); + let mut prefix_set = PrefixSetMut::default(); + prefix_set.insert(Nibbles::unpack(dirty)); pretty_assertions::assert_eq!( - expected_root, - root_with_collapse, - "Root hash after collapsing branch (removed child after remaining) should match fresh computation" + Some(expected_root), + harness.root_with_prefix_set(prefix_set.freeze()), ); } @@ -3138,8 +2971,7 @@ mod tests { // hash_mask: bits b and c — both have cached leaf hashes. Bit a has no hash, so the // calculator will seek the trie cursor to find a deeper cached branch. // - // Having three children with two (b, c) NOT in the prefix set ensures - // `should_skip_cached_branch` does NOT skip this branch (num_unmatched >= 2). + // Children b and c remain clean and can still use their cached hashes. let branch_6_state_mask = TrieMask::new((1 << 0xa) | (1 << 0xb) | (1 << 0xc)); let branch_6_hash_mask = TrieMask::new((1 << 0xb) | (1 << 0xc)); let branch_6 = BranchNodeCompact::new( @@ -3174,11 +3006,8 @@ mod tests { let mut harness = TrieTestHarness::new(all_storage); harness.set_trie_nodes(inconsistent_nodes); - // Mark key_c as dirty — in the real scenario the leaf was touched by execution. - // The prefix set contains only key_c's full path. `should_skip_cached_branch` will - // NOT skip branch [6] because two of its three children (b, c) are not in the set - // (num_unmatched = 2 > 1). It also will not skip branch [6,a,3] because - // `contains([6,a,3])` is false (key_c's nibbles 6,a,8,... do not start with 6,a,3). + // Mark key_c as dirty — in the real scenario the leaf was touched by execution. The + // nested branch [6,a,3] remains clean because key_c diverges at [6,a,8]. let mut prefix_set = PrefixSetMut::default(); prefix_set.insert(Nibbles::unpack(key_c)); @@ -3254,8 +3083,7 @@ mod tests { // hash_mask: bits b and c — both have cached leaf hashes. Bit a has no hash, so the // calculator will seek the trie cursor to find a deeper cached branch. // - // Having three children with two (b, c) NOT in the prefix set ensures - // `should_skip_cached_branch` does NOT skip this branch (num_unmatched >= 2). + // Children b and c remain clean and can still use their cached hashes. let branch_6_state_mask = TrieMask::new((1 << 0xa) | (1 << 0xb) | (1 << 0xc)); let branch_6_hash_mask = TrieMask::new((1 << 0xb) | (1 << 0xc)); let branch_6 = BranchNodeCompact::new( @@ -3401,6 +3229,32 @@ mod tests { pretty_assertions::assert_eq!(expected_root, got_root); } + #[test] + fn test_blinded_local_root_returns_trie_inconsistency() { + let key = B256::right_padding_from(&[0x63, 0xaa]); + let value = U256::from(1); + let hash = storage_leaf_hash(&Nibbles::unpack(key).slice(2..), &value); + let mask = TrieMask::from_nibble(3); + let cached_branch = + BranchNodeCompact::new(mask, TrieMask::default(), mask, vec![hash], None); + + let mut harness = TrieTestHarness::new(BTreeMap::from([(key, value)])); + harness.set_trie_nodes(BTreeMap::from([(Nibbles::from_nibbles([6]), cached_branch)])); + + let trie_cursor = + harness.trie_cursor_factory().storage_trie_cursor(harness.hashed_address()).unwrap(); + let hashed_cursor = harness + .hashed_cursor_factory() + .hashed_storage_cursor(harness.hashed_address()) + .unwrap(); + let mut calculator = StorageProofCalculator::new_storage(trie_cursor, hashed_cursor); + + assert!(matches!( + calculator.storage_root_node(harness.hashed_address()), + Err(StateProofError::TrieInconsistency(_)) + )); + } + #[test] fn test_cached_hash_with_deleted_leaf() { reth_tracing::init_test_tracing(); diff --git a/crates/trie/trie/src/proof_v2/node.rs b/crates/trie/trie/src/proof_v2/node.rs index 070deda7565..fb66d97ce28 100644 --- a/crates/trie/trie/src/proof_v2/node.rs +++ b/crates/trie/trie/src/proof_v2/node.rs @@ -2,8 +2,8 @@ use crate::proof_v2::DeferredValueEncoder; use alloy_rlp::Encodable; use reth_execution_errors::trie::StateProofError; use reth_trie_common::{ - BranchNodeMasks, BranchNodeV2, LeafNode, LeafNodeRef, Nibbles, ProofTrieNodeV2, RlpNode, - TrieMask, TrieNodeV2, + BranchNodeMasks, BranchNodeV2, ExtensionNodeRef, LeafNode, LeafNodeRef, Nibbles, + ProofTrieNodeV2, RlpNode, TrieMask, TrieNodeV2, }; /// A trie node which is the child of a branch in the trie. @@ -24,7 +24,18 @@ pub(crate) enum ProofTrieBranchChild { masks: Option, }, /// A node whose type is not known, as it has already been converted to an [`RlpNode`]. - RlpNode(RlpNode), + RlpNode { + /// The RLP-encoded node. + node: RlpNode, + /// The path from the parent branch's child nibble to the encoded node. This field can only + /// be set if `node` was sourced from the hashes of a cached branch, and therefore we know + /// that it is a blinded branch. + short_key: Nibbles, + /// Whether this node contributes to its parent's hash mask when it is a direct child. + hash_mask_bit: bool, + /// Whether this node contributes to its parent's tree mask. + tree_mask_bit: bool, + }, } impl ProofTrieBranchChild { @@ -63,15 +74,25 @@ impl ProofTrieBranchChild { branch_node.encode(buf); Ok((RlpNode::from_rlp(buf), Some(branch_node.stack))) } - Self::RlpNode(rlp_node) => Ok((rlp_node, None)), + Self::RlpNode { node, short_key, hash_mask_bit, .. } => { + if short_key.is_empty() { + return Ok((node, None)) + } + + // Only branch hashes sourced from a cached hash mask can have an external short + // key. Other committed nodes already encode their key internally. + debug_assert!(hash_mask_bit); + ExtensionNodeRef::new(&short_key, node.as_slice()).encode(buf); + Ok((RlpNode::from_rlp(buf), None)) + } } } /// Converts this child into a [`ProofTrieNodeV2`] having the given path. /// - /// # Panics + /// # Errors /// - /// If called on a [`Self::RlpNode`]. + /// Returns [`StateProofError::TrieInconsistency`] if called on a [`Self::RlpNode`]. pub(crate) fn into_proof_trie_node( self, path: Nibbles, @@ -91,21 +112,38 @@ impl ProofTrieBranchChild { (TrieNodeV2::Leaf(LeafNode::new(short_key, rlp_val)), None) } Self::Branch { node, masks } => (TrieNodeV2::Branch(node), masks), - Self::RlpNode(_) => panic!("Cannot call `into_proof_trie_node` on RlpNode"), + // Cached hashes cannot be retained as proof nodes: targeted children are recalculated, + // while untargeted children are either combined into a branch or discarded. Reaching + // this arm means inconsistent cached trie data left a blinded node as the local root. + Self::RlpNode { .. } => { + return Err(StateProofError::TrieInconsistency( + "cannot convert RLP node to proof node".to_string(), + )) + } }; Ok(ProofTrieNodeV2 { node, path, masks }) } - /// Returns the short key of the child, if it is a leaf or branch, or empty if its a - /// [`Self::RlpNode`]. - pub(crate) fn short_key(&self) -> &Nibbles { + /// Returns the child's short key. + pub(crate) const fn short_key(&self) -> &Nibbles { match self { Self::Leaf { short_key, .. } | - Self::Branch { node: BranchNodeV2 { key: short_key, .. }, .. } => short_key, - Self::RlpNode(_) => { - static EMPTY_NIBBLES: Nibbles = Nibbles::new(); - &EMPTY_NIBBLES + Self::Branch { node: BranchNodeV2 { key: short_key, .. }, .. } | + Self::RlpNode { short_key, .. } => short_key, + } + } + + /// Returns this child's hash and tree mask contributions. + pub(crate) fn mask_bits(&self) -> (bool, bool) { + match self { + Self::Leaf { .. } => (false, false), + Self::Branch { node, masks } => ( + node.key.is_empty() && node.length() >= 32, + masks.is_some_and(|masks| !masks.is_empty()), + ), + Self::RlpNode { short_key, hash_mask_bit, tree_mask_bit, .. } => { + (*hash_mask_bit && short_key.is_empty(), *tree_mask_bit) } } } @@ -115,10 +153,9 @@ impl ProofTrieBranchChild { /// # Panics /// /// - If the given len is longer than the short key - /// - If the node is a [`Self::RlpNode`] pub(crate) fn trim_short_key_prefix(&mut self, len: usize) { match self { - Self::Leaf { short_key, .. } => { + Self::Leaf { short_key, .. } | Self::RlpNode { short_key, .. } => { *short_key = trim_nibbles_prefix(short_key, len); } Self::Branch { node: BranchNodeV2 { key, branch_rlp_node, .. }, .. } => { @@ -127,9 +164,6 @@ impl ProofTrieBranchChild { *branch_rlp_node = None; } } - Self::RlpNode(_) => { - panic!("Cannot call `trim_short_key_prefix` on RlpNode") - } } } } @@ -144,8 +178,6 @@ pub(crate) struct ProofTrieBranch { /// A mask tracking which child nibbles are set on the branch so far. There will be a single /// child on the stack for each set bit. pub(crate) state_mask: TrieMask, - /// Bitmasks which are subsets of `state_mask`. - pub(crate) masks: Option, } /// Trims the first `len` nibbles from the head of the given `Nibbles`. From abba04f89c706c5363e2ea24267f8625d2bfe59a Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Mon, 7 Sep 2026 12:49:51 +0200 Subject: [PATCH 189/294] test: signal background metrics hook completion (#27047) --- crates/node/metrics/src/server.rs | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/crates/node/metrics/src/server.rs b/crates/node/metrics/src/server.rs index d8c623eba73..be13514a034 100644 --- a/crates/node/metrics/src/server.rs +++ b/crates/node/metrics/src/server.rs @@ -494,7 +494,10 @@ mod tests { use socket2::{Domain, Socket, Type}; use std::{ net::{SocketAddr, TcpListener}, - sync::atomic::{AtomicUsize, Ordering}, + sync::{ + atomic::{AtomicUsize, Ordering}, + mpsc, Mutex, + }, }; fn get_random_available_addr() -> SocketAddr { @@ -569,14 +572,24 @@ mod tests { install_prometheus_recorder(); let collections = Arc::new(AtomicUsize::new(0)); + let (release, wait_for_release) = mpsc::channel(); + let wait_for_release = Mutex::new(Some(wait_for_release)); + let completed = Arc::new(tokio::sync::Notify::new()); let hooks = Hooks::builder() .with_background_interval(Duration::from_secs(60)) .with_background_hook({ let collections = collections.clone(); + let completed = completed.clone(); move || { - std::thread::sleep(Duration::from_secs(2)); + // Dropping the sender also unblocks the hook if a scrape assertion fails. + if let Some(wait_for_release) = wait_for_release.lock().unwrap().take() && + wait_for_release.recv().is_err() + { + return + } let collections = collections.fetch_add(1, Ordering::Relaxed) + 1; metrics::gauge!("test_background_hook_collections").set(collections as f64); + completed.notify_one(); } }) .build(); @@ -611,9 +624,8 @@ mod tests { assert_eq!(collections.load(Ordering::Relaxed), 0); // once it completes, its values are rendered by the following scrape - while collections.load(Ordering::Relaxed) == 0 { - tokio::time::sleep(Duration::from_millis(10)).await; - } + release.send(()).unwrap(); + tokio::time::timeout(Duration::from_secs(10), completed.notified()).await.unwrap(); let body = Client::new().get(&url).send().await.unwrap().text().await.unwrap(); assert!( body.contains("test_background_hook_collections"), From ad3929fdf10f399055a4e9340fff697b70687195 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Mon, 7 Sep 2026 12:55:37 +0200 Subject: [PATCH 190/294] test: skip wall-clock waits in peer tick test (#27045) --- crates/net/network/src/peers.rs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/crates/net/network/src/peers.rs b/crates/net/network/src/peers.rs index 419d503829d..39c0d090a10 100644 --- a/crates/net/network/src/peers.rs +++ b/crates/net/network/src/peers.rs @@ -2912,7 +2912,8 @@ mod tests { let peer_id = PeerId::random(); peer_manager.add_peer(peer_id, PeerAddr::from_tcp(socket_addr), None); - tokio::time::sleep(Duration::from_secs(1)).await; + // Age the last tick directly so each reputation update sees a full elapsed second. + peer_manager.last_tick -= Duration::from_secs(1); peer_manager.tick(); // still unconnected @@ -2921,7 +2922,7 @@ mod tests { // mark as connected peer_manager.peers.get_mut(&peer_id).unwrap().state = PeerConnectionState::Out; - tokio::time::sleep(Duration::from_secs(1)).await; + peer_manager.last_tick -= Duration::from_secs(1); peer_manager.tick(); // still at default reputation @@ -2929,7 +2930,7 @@ mod tests { peer_manager.peers.get_mut(&peer_id).unwrap().reputation -= 1; - tokio::time::sleep(Duration::from_secs(1)).await; + peer_manager.last_tick -= Duration::from_secs(1); peer_manager.tick(); // tick applied From 0297d4f9d076d7edc7e330020da6825928c85a9d Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Mon, 7 Sep 2026 10:55:41 +0000 Subject: [PATCH 191/294] feat(download): allow retry delay overrides (#27048) Co-authored-by: Matthias Seitz <19890894+mattsse@users.noreply.github.com> --- bin/reth/tests/it/download/mod.rs | 13 +++++++- crates/cli/commands/src/download/archive.rs | 8 +++-- crates/cli/commands/src/download/extract.rs | 23 ++++++++++---- crates/cli/commands/src/download/fetch.rs | 23 +++++++++++--- crates/cli/commands/src/download/mod.rs | 31 ++++++++++++++++++ crates/cli/commands/src/download/session.rs | 35 ++++++++++++++++++++- docs/vocs/docs/pages/cli/reth/download.mdx | 5 +++ 7 files changed, 123 insertions(+), 15 deletions(-) diff --git a/bin/reth/tests/it/download/mod.rs b/bin/reth/tests/it/download/mod.rs index a29d799f7de..db794e30102 100644 --- a/bin/reth/tests/it/download/mod.rs +++ b/bin/reth/tests/it/download/mod.rs @@ -205,9 +205,20 @@ async fn bad_output_checksum_prevents_finalization() { json!(blake3::hash(b"wrong").to_hex().to_string()); let server = snapshot.serve(true).await; let dir = tempfile::tempdir().unwrap(); - let output = download(&server, dir.path(), &["--with-txs-since", "20"]); + let output = + download(&server, dir.path(), &["--with-txs-since", "20", "--retry-backoff", "0ms"]); assert!(!output.status.success()); assert!(String::from_utf8_lossy(&output.stderr).contains("Failed integrity validation")); + assert!(String::from_utf8_lossy(&output.stderr).contains("after 10 attempts")); + assert_eq!( + server + .requests() + .iter() + .filter(|(path, range)| { path == "/snapshot/state.tar.zst" && range.is_none() }) + .count(), + 10, + "each checksum failure must fetch a fresh archive" + ); assert!(!dir.path().join("reth.toml").exists()); assert!(!dir.path().join("db/mdbx.dat").exists()); } diff --git a/crates/cli/commands/src/download/archive.rs b/crates/cli/commands/src/download/archive.rs index f1189486f29..65573eec561 100644 --- a/crates/cli/commands/src/download/archive.rs +++ b/crates/cli/commands/src/download/archive.rs @@ -31,6 +31,7 @@ pub(crate) async fn run_modular_downloads( target_dir: &Path, download_concurrency: usize, cancel_token: CancellationToken, + retry_backoff: Option, ) -> Result<()> { let download_cache_dir = target_dir.join(DOWNLOAD_CACHE_DIR); fs::create_dir_all(&download_cache_dir)?; @@ -45,7 +46,8 @@ pub(crate) async fn run_modular_downloads( Some(Arc::clone(&shared)), Some(DownloadRequestLimiter::new(download_concurrency)), cancel_token, - ); + ) + .with_retry_backoff(retry_backoff); let ctx = ArchiveProcessContext::new(target_dir.to_path_buf(), Some(download_cache_dir), session); @@ -185,7 +187,9 @@ impl ArchiveProcessor { if attempt >= MAX_DOWNLOAD_RETRIES { state = ArchiveAttemptState::Fail; } else { - std::thread::sleep(Duration::from_secs(RETRY_BACKOFF_SECS)); + std::thread::sleep( + self.ctx.session().retry_delay(Duration::from_secs(RETRY_BACKOFF_SECS)), + ); attempt += 1; state = ArchiveAttemptState::RunAttempt; } diff --git a/crates/cli/commands/src/download/extract.rs b/crates/cli/commands/src/download/extract.rs index 29bb49089d4..bc5722cb19d 100644 --- a/crates/cli/commands/src/download/extract.rs +++ b/crates/cli/commands/src/download/extract.rs @@ -300,7 +300,9 @@ pub(crate) fn streaming_download_and_extract( } last_error = Some(err); if attempt < MAX_DOWNLOAD_RETRIES { - std::thread::sleep(Duration::from_secs(RETRY_BACKOFF_SECS)); + std::thread::sleep( + session.retry_delay(Duration::from_secs(RETRY_BACKOFF_SECS)), + ); } continue; } @@ -342,7 +344,9 @@ pub(crate) fn streaming_download_and_extract( } last_error = Some(error); if attempt < MAX_DOWNLOAD_RETRIES { - std::thread::sleep(Duration::from_secs(RETRY_BACKOFF_SECS)); + std::thread::sleep( + session.retry_delay(Duration::from_secs(RETRY_BACKOFF_SECS)), + ); } } } @@ -417,13 +421,15 @@ fn blocking_download_and_extract( resumable: bool, request_limiter: Option>, cancel_token: CancellationToken, + retry_backoff: Option, ) -> Result<()> { let format = CompressionFormat::from_url(url)?; if let Ok(parsed_url) = Url::parse(url) && parsed_url.scheme() == "file" { - let session = DownloadSession::new(shared, request_limiter, cancel_token); + let session = DownloadSession::new(shared, request_limiter, cancel_token) + .with_retry_backoff(retry_backoff); let file_path = parsed_url .to_file_path() .map_err(|_| eyre::eyre!("Invalid file:// URL path: {}", url))?; @@ -437,14 +443,17 @@ fn blocking_download_and_extract( url, format, target_dir, - DownloadSession::new(shared, Some(request_limiter), cancel_token), + DownloadSession::new(shared, Some(request_limiter), cancel_token) + .with_retry_backoff(retry_backoff), ) } else if resumable { let session = - DownloadSession::new(shared, Some(DownloadRequestLimiter::new(1)), cancel_token); + DownloadSession::new(shared, Some(DownloadRequestLimiter::new(1)), cancel_token) + .with_retry_backoff(retry_backoff); download_and_extract(url, format, target_dir, session) } else { - let session = DownloadSession::new(shared, None, cancel_token); + let session = + DownloadSession::new(shared, None, cancel_token).with_retry_backoff(retry_backoff); let result = streaming_download_and_extract(url, format, target_dir, &session); if result.is_ok() { session.record_archive_output_complete(0); @@ -465,6 +474,7 @@ pub(crate) async fn stream_and_extract( resumable: bool, request_limiter: Option>, cancel_token: CancellationToken, + retry_backoff: Option, ) -> Result<()> { let target_dir = target_dir.to_path_buf(); let url = url.to_string(); @@ -476,6 +486,7 @@ pub(crate) async fn stream_and_extract( resumable, request_limiter, cancel_token, + retry_backoff, ) }) .await??; diff --git a/crates/cli/commands/src/download/fetch.rs b/crates/cli/commands/src/download/fetch.rs index 83eea6ebbcd..db0300344dd 100644 --- a/crates/cli/commands/src/download/fetch.rs +++ b/crates/cli/commands/src/download/fetch.rs @@ -247,9 +247,12 @@ impl ArchiveFetcher { if attempt < max_download_retries { info!(target: "reth::cli", file = %self.paths.file_name(), - "Download failed, retrying in {RETRY_BACKOFF_SECS}s..." + retry_delay = ?self.session.retry_delay(Duration::from_secs(RETRY_BACKOFF_SECS)), + "Download failed, retrying" + ); + std::thread::sleep( + self.session.retry_delay(Duration::from_secs(RETRY_BACKOFF_SECS)), ); - std::thread::sleep(Duration::from_secs(RETRY_BACKOFF_SECS)); } continue; } @@ -328,9 +331,12 @@ impl ArchiveFetcher { if attempt < max_download_retries { info!(target: "reth::cli", file = %self.paths.file_name(), - "Download interrupted, retrying in {RETRY_BACKOFF_SECS}s..." + retry_delay = ?self.session.retry_delay(Duration::from_secs(RETRY_BACKOFF_SECS)), + "Download interrupted, retrying" + ); + std::thread::sleep( + self.session.retry_delay(Duration::from_secs(RETRY_BACKOFF_SECS)), ); - std::thread::sleep(Duration::from_secs(RETRY_BACKOFF_SECS)); } continue; } @@ -550,6 +556,8 @@ struct SegmentedWorkerContext<'a> { request_limiter: &'a DownloadRequestLimiter, /// Cancellation token shared by the whole command. cancel_token: &'a CancellationToken, + /// Session carrying the retry-delay override. + session: &'a DownloadSession, } impl SegmentedDownload { @@ -588,6 +596,7 @@ impl SegmentedDownload { let worker_context = SegmentedWorkerContext { url, part_path: paths.part_path(), + session: &session, shared, request_limiter: request_limiter.as_ref(), cancel_token, @@ -669,6 +678,7 @@ impl SegmentedDownload { &piece_progress_bytes, context.request_limiter, context.cancel_token, + context.session, ) { state.note_terminal_failure(); terminal_failure.record(error); @@ -691,6 +701,7 @@ impl SegmentedDownload { piece_progress_bytes: &AtomicU64, request_limiter: &DownloadRequestLimiter, cancel_token: &CancellationToken, + session: &DownloadSession, ) -> Result<()> { for attempt in 1..=SEGMENT_RETRY_ATTEMPTS { if cancel_token.is_cancelled() { @@ -711,7 +722,9 @@ impl SegmentedDownload { Err(PieceAttemptFailure::Retryable { error: _, throttled }) if attempt < SEGMENT_RETRY_ATTEMPTS => { - std::thread::sleep(piece_retry_backoff(attempt, throttled)); + std::thread::sleep( + session.retry_delay(piece_retry_backoff(attempt, throttled)), + ); } Err(PieceAttemptFailure::Retryable { error, .. }) => return Err(error), Err(PieceAttemptFailure::Terminal(error)) => return Err(error), diff --git a/crates/cli/commands/src/download/mod.rs b/crates/cli/commands/src/download/mod.rs index aeab3e04d45..411fa824cba 100644 --- a/crates/cli/commands/src/download/mod.rs +++ b/crates/cli/commands/src/download/mod.rs @@ -118,6 +118,7 @@ use std::{ collections::BTreeMap, path::{Path, PathBuf}, sync::{Arc, OnceLock}, + time::Duration, }; use tracing::info; use tui::{run_selector, SelectorOutput}; @@ -442,6 +443,14 @@ pub struct DownloadCommand { #[arg(long, default_value_t = MAX_CONCURRENT_DOWNLOADS)] download_concurrency: usize, + /// Override the delay between retry attempts (for example, 500ms or 5s). + /// + /// Applies to requests, extraction, output verification, and segmented downloads. + /// By default, retries wait five seconds; segmented requests use adaptive backoff. + /// This does not change the number of attempts. + #[arg(long, value_name = "DURATION", value_parser = reth_cli_util::parse_duration_from_secs_or_ms)] + retry_backoff: Option, + /// List available snapshots and exit. /// /// Queries the snapshots API and prints all available snapshots for the selected chain, @@ -491,6 +500,7 @@ impl> DownloadCo self.resumable, Some(request_limiter), cancel_token.clone(), + self.retry_backoff, ) .await?; info!(target: "reth::cli", "Snapshot downloaded and extracted successfully"); @@ -534,6 +544,7 @@ impl> DownloadCo target_dir, self.download_concurrency.max(1), cancel_token.clone(), + self.retry_backoff, ) .await?; @@ -1228,6 +1239,26 @@ mod tests { assert_eq!(defaults.long_help, Some("Custom help for snapshots".to_string())); } + #[test] + fn test_download_retry_backoff() { + let parse = |args: Vec<&str>| { + CommandParser::>::try_parse_from(args) + }; + assert_eq!(parse(vec!["reth"]).unwrap().args.retry_backoff, None); + for (value, expected) in [ + ("0ms", Duration::ZERO), + ("250ms", Duration::from_millis(250)), + ("2s", Duration::from_secs(2)), + ] { + assert_eq!( + parse(vec!["reth", "--retry-backoff", value]).unwrap().args.retry_backoff, + Some(expected) + ); + } + assert!(parse(vec!["reth", "--retry-backoff=-1s"]).is_err()); + assert!(parse(vec!["reth", "--retry-backoff", "invalid"]).is_err()); + } + #[test] fn test_download_resumable_defaults_to_true() { let args = diff --git a/crates/cli/commands/src/download/session.rs b/crates/cli/commands/src/download/session.rs index 23412e23d4a..bc1e5a941af 100644 --- a/crates/cli/commands/src/download/session.rs +++ b/crates/cli/commands/src/download/session.rs @@ -4,6 +4,7 @@ use reth_cli_util::cancellation::CancellationToken; use std::{ path::{Path, PathBuf}, sync::Arc, + time::Duration, }; /// Shared state for one run of `reth download`. @@ -15,6 +16,8 @@ pub(crate) struct DownloadSession { request_limiter: Option>, /// Cancellation token shared by the whole command. cancel_token: CancellationToken, + /// Optional fixed delay overriding the default retry backoff. + retry_backoff: Option, } impl DownloadSession { @@ -24,7 +27,18 @@ impl DownloadSession { request_limiter: Option>, cancel_token: CancellationToken, ) -> Self { - Self { progress, request_limiter, cancel_token } + Self { progress, request_limiter, cancel_token, retry_backoff: None } + } + + /// Overrides the delay between retry attempts for this download. + pub(crate) const fn with_retry_backoff(mut self, retry_backoff: Option) -> Self { + self.retry_backoff = retry_backoff; + self + } + + /// Returns the configured delay, preserving the caller's default when no override is set. + pub(crate) fn retry_delay(&self, default: Duration) -> Duration { + self.retry_backoff.unwrap_or(default) } /// Returns the shared progress tracker, if this flow uses one. @@ -98,3 +112,22 @@ impl ArchiveProcessContext { &self.session } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn retry_delay_preserves_defaults_or_uses_override() { + let session = DownloadSession::new(None, None, CancellationToken::new()); + for default in [Duration::from_secs(2), Duration::from_secs(5), Duration::from_secs(40)] { + assert_eq!(session.retry_delay(default), default); + for delay in [Duration::ZERO, Duration::from_millis(250)] { + assert_eq!( + session.clone().with_retry_backoff(Some(delay)).retry_delay(default), + delay + ); + } + } + } +} diff --git a/docs/vocs/docs/pages/cli/reth/download.mdx b/docs/vocs/docs/pages/cli/reth/download.mdx index 2dc38158f09..6817a3c413b 100644 --- a/docs/vocs/docs/pages/cli/reth/download.mdx +++ b/docs/vocs/docs/pages/cli/reth/download.mdx @@ -223,6 +223,11 @@ Storage: [default: 8] + --retry-backoff + Override the delay between retry attempts (for example, 500ms or 5s). + + Applies to requests, extraction, output verification, and segmented downloads. By default, retries wait five seconds; segmented requests use adaptive backoff. This does not change the number of attempts. + --list List available snapshots and exit. From 0b30d58c3471be5cf40d4c6afd67efa40a328d38 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Mon, 7 Sep 2026 11:01:38 +0000 Subject: [PATCH 192/294] fix(engine): flush state before backfill sync (#26911) Co-authored-by: Brian Picciano --- crates/e2e-test-utils/src/testsuite/mod.rs | 12 +- crates/engine/tree/src/backfill.rs | 10 +- crates/engine/tree/src/tree/error.rs | 3 + crates/engine/tree/src/tree/mod.rs | 121 ++++++++- crates/engine/tree/src/tree/tests.rs | 200 ++++++++++++++- .../engine/tree/tests/e2e-testsuite/main.rs | 231 +++++++++++++++++- 6 files changed, 549 insertions(+), 28 deletions(-) diff --git a/crates/e2e-test-utils/src/testsuite/mod.rs b/crates/e2e-test-utils/src/testsuite/mod.rs index 208c24be1ff..59041a0ec81 100644 --- a/crates/e2e-test-utils/src/testsuite/mod.rs +++ b/crates/e2e-test-utils/src/testsuite/mod.rs @@ -4,7 +4,7 @@ use crate::{ testsuite::actions::{Action, ActionBox}, NodeBuilderHelper, }; -use alloy_primitives::B256; +use alloy_primitives::{Bytes, B256}; use eyre::Result; use jsonrpsee::http_client::HttpClient; use reth_node_api::{EngineTypes, PayloadTypes}; @@ -78,6 +78,16 @@ where .map_err(|e| eyre::eyre!("Failed to get block by number: {}", e)) } + /// Submit a raw transaction using the alloy provider. + pub async fn send_raw_transaction(&self, raw_tx: Bytes) -> Result { + let pending = self + .provider + .send_raw_transaction(&raw_tx) + .await + .map_err(|e| eyre::eyre!("Failed to send raw transaction: {}", e))?; + Ok(*pending.tx_hash()) + } + /// Check if the node is ready by attempting to get the latest block pub async fn is_ready(&self) -> bool { self.get_block_by_number(alloy_eips::BlockNumberOrTag::Latest).await.is_ok() diff --git a/crates/engine/tree/src/backfill.rs b/crates/engine/tree/src/backfill.rs index 61513827d3c..b96f9e25220 100644 --- a/crates/engine/tree/src/backfill.rs +++ b/crates/engine/tree/src/backfill.rs @@ -22,9 +22,10 @@ pub enum BackfillSyncState { /// This is the initial or default state. #[default] Idle, - /// A backfill synchronization has been requested or planned, but processing has not started - /// yet. + /// A backfill action has been sent to the pipeline, but processing has not started yet. Pending, + /// A backfill synchronization must be re-evaluated after persistence catches up. + PendingRevalidation, /// The node is actively engaged in backfill synchronization. Active, } @@ -40,6 +41,11 @@ impl BackfillSyncState { matches!(self, Self::Pending) } + /// Returns true if backfill must be re-evaluated before it can start. + pub const fn is_pending_revalidation(&self) -> bool { + matches!(self, Self::PendingRevalidation) + } + /// Returns true if the state is active. pub const fn is_active(&self) -> bool { matches!(self, Self::Active) diff --git a/crates/engine/tree/src/tree/error.rs b/crates/engine/tree/src/tree/error.rs index d2a823e0227..2c9de6392ce 100644 --- a/crates/engine/tree/src/tree/error.rs +++ b/crates/engine/tree/src/tree/error.rs @@ -17,6 +17,9 @@ pub enum AdvancePersistenceError { /// The persistence channel was closed unexpectedly #[error("persistence channel closed")] ChannelClosed, + /// State/trie catch-up could not construct an input despite split persistence frontiers. + #[error("state/trie catch-up input unavailable while persistence frontiers are split")] + StateTrieCatchupUnavailable, /// A provider error #[error(transparent)] Provider(#[from] ProviderError), diff --git a/crates/engine/tree/src/tree/mod.rs b/crates/engine/tree/src/tree/mod.rs index 084a087129a..5aa4e4e5699 100644 --- a/crates/engine/tree/src/tree/mod.rs +++ b/crates/engine/tree/src/tree/mod.rs @@ -1440,8 +1440,20 @@ where /// Persistence completion is handled separately via the `wait_for_event` method. fn advance_persistence(&mut self) -> Result<(), AdvancePersistenceError> { if !self.persistence_state.in_progress() { + let payload_build_active = self.payload_builds.is_active(); if let Some(new_tip_num) = self.find_disk_reorg()? { self.remove_blocks(new_tip_num) + } else if self.backfill_sync_state.is_pending_revalidation() && + !payload_build_active && + self.persistence_state.last_state_trie_persisted_block != + self.persistence_state.last_persisted_block + { + let Some(input) = self.get_save_blocks_input(PersistTarget::Persisted) else { + return Err(AdvancePersistenceError::StateTrieCatchupUnavailable) + }; + self.persist_blocks(input); + } else if self.backfill_sync_state.is_pending_revalidation() && !payload_build_active { + self.revalidate_pending_backfill()?; } else if let Some(input) = self.get_save_blocks_input(PersistTarget::Threshold) { self.persist_blocks(input); } @@ -2079,29 +2091,108 @@ where } } + /// Re-evaluates whether a deferred backfill is still required after persistence catches up. + fn revalidate_pending_backfill(&mut self) -> ProviderResult<()> { + debug_assert!(self.backfill_sync_state.is_pending_revalidation()); + + let sync_target_state = self.state.forkchoice_state_tracker.sync_target_state(); + let backfill_target = if let Some(state) = sync_target_state { + let configured_target = self.backfill_target_hash(state); + let target_hash = + if configured_target.is_zero() { state.head_block_hash } else { configured_target }; + let target_number = if let Some(block) = self.state.buffer.block(&target_hash) { + Some(block.number()) + } else { + self.sealed_header_by_hash(target_hash)?.map(|header| header.number()) + }; + + target_number.and_then(|target_number| { + self.backfill_sync_target( + self.state.tree_state.canonical_block_number(), + target_number, + None, + ) + }) + } else { + None + }; + + if let Some(target) = backfill_target { + self.dispatch_backfill_action(BackfillAction::Start(target.into())); + return Ok(()) + } + + self.backfill_sync_state = BackfillSyncState::Idle; + debug!(target: "engine::tree", "dropping deferred backfill after re-evaluation"); + + // The target may have changed while persistence was draining. Resume the live-sync + // download flow so a newer target can produce a fresh backfill decision. + if let Some(state) = sync_target_state && + state.head_block_hash != self.state.tree_state.canonical_block_hash() + { + let target = self.lowest_buffered_ancestor_or(state.head_block_hash); + self.send_event(EngineApiEvent::Download(DownloadRequest::single_block(target))); + } + + Ok(()) + } + /// Emits an outgoing event to the engine. fn emit_event(&mut self, event: impl Into>) { let event = event.into(); - if event.is_backfill_action() { + if let EngineApiEvent::BackfillAction(action) = event { debug_assert_eq!( self.backfill_sync_state, BackfillSyncState::Idle, "backfill action should only be emitted when backfill is idle" ); - if self.payload_builds.is_active() || self.persistence_state.in_progress() { - // Backfill can remove the same in-memory blocks as an active payload job or - // persistence task, so it must not start until the next sync trigger. - debug!(target: "engine::tree", "skipping backfill while in-memory overlay is in use"); + let persistence_in_progress = self.persistence_state.in_progress(); + let state_trie_needs_catchup = self.persistence_state.last_state_trie_persisted_block != + self.persistence_state.last_persisted_block; + if self.payload_builds.is_active() || + persistence_in_progress || + state_trie_needs_catchup + { + // Backfill can remove the same in-memory blocks as an active payload job or a + // persistence task. Enter pending mode to prevent new payload jobs and + // re-evaluate the current sync target after all readers and writes drain. + debug!( + target: "engine::tree", + last_persisted_block = self.persistence_state.last_persisted_block.number, + last_state_trie_persisted_block = self + .persistence_state + .last_state_trie_persisted_block + .number, + "deferring backfill until persistence and payload jobs drain" + ); + self.backfill_sync_state = BackfillSyncState::PendingRevalidation; return } - self.backfill_sync_state = BackfillSyncState::Pending; - self.metrics.engine.pipeline_runs.increment(1); - debug!(target: "engine::tree", "emitting backfill action event"); + self.dispatch_backfill_action(action); + return } + self.send_event(event); + } + + /// Dispatches a validated backfill action to the orchestrator. + fn dispatch_backfill_action(&mut self, action: BackfillAction) { + debug_assert!( + self.backfill_sync_state.is_idle() || + self.backfill_sync_state.is_pending_revalidation(), + "backfill action can only be dispatched while idle or pending revalidation" + ); + self.backfill_sync_state = BackfillSyncState::Pending; + self.metrics.engine.pipeline_runs.increment(1); + debug!(target: "engine::tree", "emitting backfill action event"); + self.send_event(EngineApiEvent::BackfillAction(action)); + } + + /// Sends an event to the orchestrator. + fn send_event(&self, event: EngineApiEvent) { let _ = self.outgoing.send(event).inspect_err( |err| error!(target: "engine::tree", "Failed to send internal event: {err:?}"), ); @@ -2110,8 +2201,9 @@ where /// Returns the blocks and frontiers for the next persistence cycle, if one should start. /// /// Threshold persistence honors the normal scheduling gates and retains the configured - /// in-memory block buffer. Head persistence bypasses those gates during shutdown and returns - /// `None` once both persistence frontiers have reached the canonical head. + /// in-memory block buffer. Persisted-target persistence catches the state/trie frontier up to + /// the existing database tip. Head persistence bypasses those gates during shutdown and + /// returns `None` once both persistence frontiers have reached the canonical head. fn get_save_blocks_input(&self, target: PersistTarget) -> Option> { // We will calculate the state root using the database, so we need to be sure there are no // changes @@ -2123,6 +2215,13 @@ where let (new_db_tip, new_partial_state_trie) = match target { PersistTarget::Head => (canonical_head_number, canonical_head_number), + PersistTarget::Persisted => { + // Catch-up persistence is the transition into pipeline sync, so it deliberately + // runs while backfill is pending and bypasses the normal threshold gates. + debug_assert!(self.backfill_sync_state.is_pending_revalidation()); + debug_assert!(!self.payload_builds.is_active()); + (prev_db_tip, prev_db_tip) + } PersistTarget::Threshold => { if (self.config.suppress_persistence_during_build() && self.payload_builds.is_active()) || @@ -3569,6 +3668,8 @@ enum PersistTarget { Threshold, /// Persist all blocks up to and including the canonical head. Head, + /// Persist state/trie updates through the persisted block frontier. + Persisted, } /// Result of waiting for caches to become available. diff --git a/crates/engine/tree/src/tree/tests.rs b/crates/engine/tree/src/tree/tests.rs index c767f6f1311..0680bf4afb8 100644 --- a/crates/engine/tree/src/tree/tests.rs +++ b/crates/engine/tree/src/tree/tests.rs @@ -715,23 +715,211 @@ fn persistence_completion_does_not_wait_for_active_payload_jobs() { } #[test] -fn backfill_action_skips_while_payload_build_is_active() { - let mut test_harness = TestHarness::new(MAINNET.clone()); +fn backfill_action_waits_while_payload_build_is_active() { + let (mut test_harness, _, action) = deferred_backfill_harness(); let payload_build = test_harness.tree.payload_builds.acquire(); - let action = BackfillAction::Start(B256::random().into()); - test_harness.tree.emit_event(EngineApiEvent::BackfillAction(action)); - assert!(test_harness.tree.backfill_sync_state.is_idle()); + test_harness.tree.emit_event(EngineApiEvent::BackfillAction(action.clone())); + assert!(test_harness.tree.backfill_sync_state.is_pending_revalidation()); assert!(test_harness.from_tree_rx.try_recv().is_err()); drop(payload_build); assert!(matches!(test_harness.tree.wait_for_event(), super::LoopEvent::PayloadBuildFinished)); + test_harness.tree.advance_persistence().unwrap(); + + assert!(test_harness.tree.backfill_sync_state.is_pending()); + let EngineApiEvent::BackfillAction(emitted_action) = + test_harness.from_tree_rx.try_recv().unwrap() + else { + panic!("expected backfill action") + }; + assert_eq!(emitted_action, action); +} + +fn deferred_backfill_harness() -> (TestHarness, Vec, BackfillAction) { + let all_blocks: Vec<_> = + TestBlockBuilder::eth().get_executed_blocks(1..MIN_BLOCKS_FOR_PIPELINE_RUN + 10).collect(); + let canonical_blocks = all_blocks[..6].to_vec(); + let target = all_blocks.last().unwrap().recovered_block().clone_sealed_block(); + let target_hash = target.hash(); + let mut test_harness = TestHarness::new(MAINNET.clone()).with_blocks(canonical_blocks.clone()); + test_harness.tree.state.buffer.insert_block(target.into()); + test_harness.tree.state.forkchoice_state_tracker.set_latest( + ForkchoiceState { + head_block_hash: target_hash, + safe_block_hash: target_hash, + finalized_block_hash: target_hash, + }, + ForkchoiceStatus::Syncing, + ); + + (test_harness, canonical_blocks, BackfillAction::Start(target_hash.into())) +} + +#[test] +fn backfill_action_catches_up_state_trie_before_starting_pipeline() { + let (mut test_harness, blocks, action) = deferred_backfill_harness(); + let state_trie_tip = blocks[2].recovered_block().num_hash(); + let database_tip = blocks[4].recovered_block().num_hash(); + test_harness.tree.persistence_state.last_state_trie_persisted_block = state_trie_tip; + test_harness.tree.persistence_state.last_persisted_block = database_tip; + + test_harness.tree.emit_event(EngineApiEvent::BackfillAction(action.clone())); + + assert!(test_harness.tree.backfill_sync_state.is_pending_revalidation()); + assert!(test_harness.from_tree_rx.try_recv().is_err()); + + test_harness.tree.advance_persistence().unwrap(); + let persistence_action = test_harness.action_rx.recv().unwrap(); + let PersistenceAction::SaveBlocks(input, sender) = persistence_action else { + panic!("expected state/trie catch-up save, got {persistence_action:?}") + }; + assert_eq!(input.prev_db_tip(), database_tip.number); + assert_eq!(input.new_db_tip(), database_tip.number); + assert_eq!(input.prev_partial_state_trie(), state_trie_tip.number); + assert_eq!(input.new_partial_state_trie(), database_tip.number); + assert!(input.persist_rest_blocks().is_empty()); + assert_eq!( + input + .state_trie_blocks() + .iter() + .map(|block| block.recovered_block().number()) + .collect::>(), + (state_trie_tip.number + 1..=database_tip.number).collect::>() + ); + assert!(input.state_trie_masking_blocks().is_empty()); + + sender + .send(PersistenceResult { + last_block: database_tip, + last_state_trie_block: database_tip, + commit_duration: Some(Duration::ZERO), + }) + .unwrap(); + assert!(test_harness.tree.try_poll_persistence().unwrap()); + assert!(test_harness.from_tree_rx.try_recv().is_err()); + + test_harness.tree.advance_persistence().unwrap(); + + assert!(test_harness.tree.backfill_sync_state.is_pending()); + let emitted = test_harness.from_tree_rx.try_recv().unwrap(); + let EngineApiEvent::BackfillAction(emitted_action) = emitted else { + panic!("expected backfill action, got {emitted:?}") + }; + assert_eq!(emitted_action, action); +} + +#[test] +fn deferred_backfill_is_dropped_when_target_becomes_local() { + let (mut test_harness, blocks, action) = deferred_backfill_harness(); + let state_trie_tip = blocks[2].recovered_block().num_hash(); + let database_tip = blocks[4].recovered_block().num_hash(); + test_harness.tree.persistence_state.last_state_trie_persisted_block = state_trie_tip; + test_harness.tree.persistence_state.last_persisted_block = database_tip; + + test_harness.tree.emit_event(EngineApiEvent::BackfillAction(action)); + assert!(test_harness.tree.backfill_sync_state.is_pending_revalidation()); + + // A newer FCU points at the local head while persistence is draining. Re-evaluation must use + // this current target instead of replaying the original backfill action. + let local_head = blocks.last().unwrap().recovered_block().num_hash(); + test_harness.tree.state.forkchoice_state_tracker.set_latest( + ForkchoiceState { + head_block_hash: local_head.hash, + safe_block_hash: local_head.hash, + finalized_block_hash: local_head.hash, + }, + ForkchoiceStatus::Syncing, + ); + test_harness.tree.persistence_state.last_state_trie_persisted_block = database_tip; + + test_harness.tree.advance_persistence().unwrap(); - // The skipped action is not queued for a later replay. assert!(test_harness.tree.backfill_sync_state.is_idle()); assert!(test_harness.from_tree_rx.try_recv().is_err()); } +#[test] +fn deferred_backfill_uses_latest_sync_target() { + let (mut test_harness, blocks, original_action) = deferred_backfill_harness(); + let database_tip = blocks[4].recovered_block().num_hash(); + test_harness.tree.persistence_state.last_state_trie_persisted_block = + blocks[2].recovered_block().num_hash(); + test_harness.tree.persistence_state.last_persisted_block = database_tip; + test_harness.tree.emit_event(EngineApiEvent::BackfillAction(original_action.clone())); + + let newer_chain = test_harness + .block_builder + .create_fork(blocks[0].recovered_block(), MIN_BLOCKS_FOR_PIPELINE_RUN + 20); + let newer_target = newer_chain.last().unwrap().clone_sealed_block(); + let newer_target_hash = newer_target.hash(); + test_harness.tree.state.buffer.insert_block(newer_target.into()); + test_harness.tree.state.forkchoice_state_tracker.set_latest( + ForkchoiceState { + head_block_hash: newer_target_hash, + safe_block_hash: newer_target_hash, + finalized_block_hash: newer_target_hash, + }, + ForkchoiceStatus::Syncing, + ); + test_harness.tree.persistence_state.last_state_trie_persisted_block = database_tip; + + test_harness.tree.advance_persistence().unwrap(); + + let emitted = test_harness.from_tree_rx.try_recv().unwrap(); + let EngineApiEvent::BackfillAction(BackfillAction::Start(target)) = emitted else { + panic!("expected backfill action, got {emitted:?}") + }; + assert_eq!(target.sync_target(), Some(newer_target_hash)); + assert_ne!(BackfillAction::Start(target), original_action); +} + +#[test] +fn backfill_request_is_preserved_while_persistence_is_in_flight() { + let (mut test_harness, blocks, action) = deferred_backfill_harness(); + let state_trie_tip = blocks[2].recovered_block().num_hash(); + let database_tip = blocks[4].recovered_block().num_hash(); + let (persistence_tx, persistence_rx) = crossbeam_channel::bounded(1); + test_harness.tree.persistence_state.start_save(database_tip, persistence_rx); + + test_harness.tree.emit_event(EngineApiEvent::BackfillAction(action.clone())); + + assert!(test_harness.tree.backfill_sync_state.is_pending_revalidation()); + assert!(test_harness.from_tree_rx.try_recv().is_err()); + + persistence_tx + .send(PersistenceResult { + last_block: database_tip, + last_state_trie_block: state_trie_tip, + commit_duration: Some(Duration::ZERO), + }) + .unwrap(); + assert!(test_harness.tree.try_poll_persistence().unwrap()); + + test_harness.tree.advance_persistence().unwrap(); + let persistence_action = test_harness.action_rx.recv().unwrap(); + let PersistenceAction::SaveBlocks(_, sender) = persistence_action else { + panic!("expected state/trie catch-up save, got {persistence_action:?}") + }; + sender + .send(PersistenceResult { + last_block: database_tip, + last_state_trie_block: database_tip, + commit_duration: Some(Duration::ZERO), + }) + .unwrap(); + assert!(test_harness.tree.try_poll_persistence().unwrap()); + + test_harness.tree.advance_persistence().unwrap(); + + assert!(test_harness.tree.backfill_sync_state.is_pending()); + let emitted = test_harness.from_tree_rx.try_recv().unwrap(); + let EngineApiEvent::BackfillAction(emitted_action) = emitted else { + panic!("expected backfill action, got {emitted:?}") + }; + assert_eq!(emitted_action, action); +} + #[test] fn configured_persistence_suppression_tracks_payload_job_lifetime() { let config = TreeConfig::default() diff --git a/crates/engine/tree/tests/e2e-testsuite/main.rs b/crates/engine/tree/tests/e2e-testsuite/main.rs index e83a2fa040e..dda1b19884c 100644 --- a/crates/engine/tree/tests/e2e-testsuite/main.rs +++ b/crates/engine/tree/tests/e2e-testsuite/main.rs @@ -2,24 +2,159 @@ mod fcu_finalized_blocks; -use alloy_rpc_types_engine::PayloadStatusEnum; +use alloy_rpc_types_engine::{ + CancunPayloadFields, ExecutionData, ExecutionPayloadEnvelopeV3, ExecutionPayloadSidecar, + PayloadStatusEnum, +}; use eyre::Result; +use futures::future::BoxFuture; use reth_chainspec::{ChainSpecBuilder, MAINNET}; -use reth_e2e_test_utils::testsuite::{ - actions::{ - AssertChainTip, BlockReference, CaptureBlock, CompareNodeChainTips, CreateFork, - ExpectFcuStatus, FinalizeBlock, MakeCanonical, ProduceBlocks, ProduceBlocksLocally, - ProduceInvalidBlocks, ReorgTo, SelectActiveNode, SendForkchoiceUpdate, SendNewPayloads, - SetForkBase, UpdateBlockInfo, ValidateCanonicalTag, WaitForSync, +use reth_e2e_test_utils::{ + testsuite::{ + actions::{ + Action, AssertChainTip, BlockReference, CaptureBlock, CompareNodeChainTips, CreateFork, + ExpectFcuStatus, FinalizeBlock, MakeCanonical, ProduceBlocks, ProduceBlocksLocally, + ProduceInvalidBlocks, ReorgTo, SelectActiveNode, SendForkchoiceUpdate, SendNewPayloads, + SetForkBase, UpdateBlockInfo, ValidateCanonicalTag, WaitForSync, + }, + setup::{NetworkSetup, Setup}, + Environment, TestBuilder, }, - setup::{NetworkSetup, Setup}, - TestBuilder, + transaction::TransactionTestContext, + wallet::Wallet, }; use reth_engine_tree::tree::TreeConfig; use reth_ethereum_engine_primitives::EthEngineTypes; use reth_node_ethereum::EthereumNode; use std::sync::Arc; +/// Waits for in-flight persistence by resubmitting the latest built payload to a node. +#[derive(Debug)] +struct WaitForPersistence { + node_idx: usize, +} + +impl WaitForPersistence { + const fn new(node_idx: usize) -> Self { + Self { node_idx } + } +} + +impl Action for WaitForPersistence { + fn execute<'a>( + &'a mut self, + env: &'a mut Environment, + ) -> BoxFuture<'a, Result<()>> { + Box::pin(async move { + let node = env + .node_clients + .get(self.node_idx) + .ok_or_else(|| eyre::eyre!("Node index {} out of bounds", self.node_idx))?; + let engine = node + .beacon_engine_handle + .clone() + .ok_or_else(|| eyre::eyre!("Node {} has no beacon engine handle", self.node_idx))?; + let state = env.active_node_state()?; + let envelope: ExecutionPayloadEnvelopeV3 = state + .latest_payload_envelope + .clone() + .ok_or_else(|| eyre::eyre!("No latest payload envelope available"))?; + let parent_beacon_block_root = state + .latest_payload_built + .as_ref() + .and_then(|attrs| attrs.parent_beacon_block_root) + .ok_or_else(|| eyre::eyre!("No parent beacon block root available"))?; + let versioned_hashes = envelope.blobs_bundle.versioned_hashes(); + let payload = ExecutionData::new( + envelope.execution_payload.into(), + ExecutionPayloadSidecar::v3(CancunPayloadFields { + parent_beacon_block_root, + versioned_hashes, + }), + ); + + let (status, _) = engine.reth_new_payload(payload, true, false).await?; + eyre::ensure!( + status.is_valid(), + "Persistence barrier payload was not valid: {status:?}" + ); + Ok(()) + }) + } +} + +/// Submits a funded transfer to the node that will produce the next shared block. +#[derive(Debug)] +struct SubmitTransfer { + node_idx: usize, +} + +impl SubmitTransfer { + const fn new(node_idx: usize) -> Self { + Self { node_idx } + } +} + +impl Action for SubmitTransfer { + fn execute<'a>( + &'a mut self, + env: &'a mut Environment, + ) -> BoxFuture<'a, Result<()>> { + Box::pin(async move { + let node = env + .node_clients + .get(self.node_idx) + .ok_or_else(|| eyre::eyre!("Node index {} out of bounds", self.node_idx))?; + let signer = Wallet::default() + .with_chain_id(1) + .wallet_gen() + .into_iter() + .next() + .expect("default wallet has one signer"); + let raw_tx = TransactionTestContext::transfer_tx_bytes_with_nonce(1, signer, 0).await; + node.send_raw_transaction(raw_tx).await?; + Ok(()) + }) + } +} + +/// Verifies that the masked persistence suffix contains a state-changing transaction. +#[derive(Debug)] +struct AssertBlockHasTransactions { + node_idx: usize, + block_number: u64, +} + +impl AssertBlockHasTransactions { + const fn new(node_idx: usize, block_number: u64) -> Self { + Self { node_idx, block_number } + } +} + +impl Action for AssertBlockHasTransactions { + fn execute<'a>( + &'a mut self, + env: &'a mut Environment, + ) -> BoxFuture<'a, Result<()>> { + Box::pin(async move { + let node = env + .node_clients + .get(self.node_idx) + .ok_or_else(|| eyre::eyre!("Node index {} out of bounds", self.node_idx))?; + let block = node + .get_block_by_number(alloy_eips::BlockNumberOrTag::Number(self.block_number)) + .await? + .ok_or_else(|| eyre::eyre!("Block {} not found", self.block_number))?; + eyre::ensure!( + !block.transactions.is_empty(), + "Block {} contains no transactions", + self.block_number + ); + Ok(()) + }) + } +} + /// Creates the standard setup for engine tree e2e tests. fn default_engine_tree_setup() -> Setup { Setup::default() @@ -361,6 +496,84 @@ async fn test_engine_tree_live_sync_transition_eventually_canonical_e2e() -> Res Ok(()) } +/// Verifies that pipeline sync starts only after masked state/trie updates reach the persisted +/// block frontier. +#[tokio::test] +async fn test_engine_tree_pipeline_sync_catches_up_masked_state_e2e() -> Result<()> { + reth_tracing::init_test_tracing(); + + const PERSISTENCE_THRESHOLD: u64 = 4; + const MEMORY_BLOCK_BUFFER_TARGET: u64 = 1; + const NUM_STATE_MASKING_BLOCKS: u64 = 2; + const MASKED_TRANSACTION_BLOCK: u64 = 3; + const SHARED_BLOCKS: u64 = 5; + const MIN_BLOCKS_FOR_PIPELINE_RUN: u64 = 32; // EPOCH_SLOTS from alloy-eips + const BLOCKS_AFTER_FINALIZED: u64 = 5; + + let test = TestBuilder::new() + .with_setup( + Setup::default() + .with_chain_spec(Arc::new( + ChainSpecBuilder::default() + .chain(MAINNET.chain) + .genesis( + serde_json::from_str(include_str!( + "../../../../e2e-test-utils/src/testsuite/assets/genesis.json" + )) + .unwrap(), + ) + .cancun_activated() + .build(), + )) + .with_network(NetworkSetup::multi_node(2)) + .with_storage_v2() + .with_tree_config( + TreeConfig::default() + .with_persistence_threshold(PERSISTENCE_THRESHOLD) + .with_memory_block_buffer_target(MEMORY_BLOCK_BUFFER_TARGET) + .with_num_state_masking_blocks(NUM_STATE_MASKING_BLOCKS) + .with_has_enough_parallelism(true), + ), + ) + // Put a state transition in block 3, then canonicalize five shared blocks. This starts a + // persistence cycle on node 1 with block data through block 4 but state/trie updates only + // through block 2, leaving the transaction's state changes in the masked suffix. + .with_action(ProduceBlocks::::new(MASKED_TRANSACTION_BLOCK - 1)) + .with_action(SubmitTransfer::new(1)) + .with_action(ProduceBlocks::::new( + SHARED_BLOCKS - (MASKED_TRANSACTION_BLOCK - 1), + )) + .with_action(MakeCanonical::new()) + .with_action(AssertBlockHasTransactions::new(1, MASKED_TRANSACTION_BLOCK)) + .with_action(WaitForPersistence::new(1)) + // Node 0 privately extends the shared chain. The finalized target is strictly more than + // EPOCH_SLOTS ahead of node 1, so receiving it triggers pipeline sync. + .with_action(SelectActiveNode::new(0)) + .with_action(ProduceBlocksLocally::::new(MIN_BLOCKS_FOR_PIPELINE_RUN + 1)) + .with_action(MakeCanonical::with_active_node()) + .with_action(CaptureBlock::new("finalized_target")) + .with_action(ProduceBlocksLocally::::new(BLOCKS_AFTER_FINALIZED)) + .with_action(MakeCanonical::with_active_node()) + .with_action(CaptureBlock::new("source_head")) + .with_action(SelectActiveNode::new(1)) + .with_action(CompareNodeChainTips::expect_different(0, 1)) + .with_action( + SendForkchoiceUpdate::::new( + BlockReference::Tag("finalized_target".into()), + BlockReference::Tag("source_head".into()), + BlockReference::Tag("source_head".into()), + ) + .with_expected_status(PayloadStatusEnum::Syncing) + .with_node_idx(1), + ) + .with_action(WaitForSync::new(0, 1).with_timeout(60)) + .with_action(CompareNodeChainTips::expect_same(0, 1)); + + test.run::().await?; + + Ok(()) +} + // ==================== v2 storage mode variants ==================== /// v2 variant: Verifies forkchoice update and canonical chain insertion in v2 storage mode. From d65679771dfcd76b07ea20f9ae333664ed3efef8 Mon Sep 17 00:00:00 2001 From: Oliver Tipton Date: Mon, 7 Sep 2026 12:10:57 +0000 Subject: [PATCH 193/294] perf(rpc): compute oversized block RLP length once (#27037) --- crates/rpc/rpc/src/validation.rs | 30 ++++++++++++++++-------------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/crates/rpc/rpc/src/validation.rs b/crates/rpc/rpc/src/validation.rs index 33751dec024..90b994a2ebe 100644 --- a/crates/rpc/rpc/src/validation.rs +++ b/crates/rpc/rpc/src/validation.rs @@ -445,13 +445,14 @@ where // Check block size as per EIP-7934 (only applies when Osaka hardfork is active) let chain_spec = self.provider.chain_spec(); - if chain_spec.is_osaka_active_at_timestamp(block.timestamp()) && - block.rlp_length() > MAX_RLP_BLOCK_SIZE - { - return Err(ValidationApiError::Consensus(ConsensusError::BlockTooLarge { - rlp_length: block.rlp_length(), - max_rlp_length: MAX_RLP_BLOCK_SIZE, - })); + if chain_spec.is_osaka_active_at_timestamp(block.timestamp()) { + let rlp_length = block.rlp_length(); + if rlp_length > MAX_RLP_BLOCK_SIZE { + return Err(ValidationApiError::Consensus(ConsensusError::BlockTooLarge { + rlp_length, + max_rlp_length: MAX_RLP_BLOCK_SIZE, + })); + } } self.validate_message_against_block( @@ -489,13 +490,14 @@ where })?; let chain_spec = self.provider.chain_spec(); - if chain_spec.is_osaka_active_at_timestamp(block.timestamp()) && - block.rlp_length() > MAX_RLP_BLOCK_SIZE - { - return Err(ValidationApiError::Consensus(ConsensusError::BlockTooLarge { - rlp_length: block.rlp_length(), - max_rlp_length: MAX_RLP_BLOCK_SIZE, - })); + if chain_spec.is_osaka_active_at_timestamp(block.timestamp()) { + let rlp_length = block.rlp_length(); + if rlp_length > MAX_RLP_BLOCK_SIZE { + return Err(ValidationApiError::Consensus(ConsensusError::BlockTooLarge { + rlp_length, + max_rlp_length: MAX_RLP_BLOCK_SIZE, + })); + } } self.validate_message_against_block( From 842a0fbf2d6186c6186d259f7122a8cf436107be Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Mon, 7 Sep 2026 13:04:16 +0000 Subject: [PATCH 194/294] perf(txpool): cache pooled transaction in-memory size (#27061) --- crates/transaction-pool/src/traits.rs | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/crates/transaction-pool/src/traits.rs b/crates/transaction-pool/src/traits.rs index 4a994829575..b3400ba7a0c 100644 --- a/crates/transaction-pool/src/traits.rs +++ b/crates/transaction-pool/src/traits.rs @@ -1544,6 +1544,7 @@ pub trait EthPoolTransaction: PoolTransaction { /// /// - `cost`: Pre-calculated max cost (gas * price + value + blob costs) /// - `encoded_length`: Cached RLP encoding length for size limits +/// - `in_memory_size`: Cached transaction size for subpool memory accounting /// - `blob_sidecar`: Blob data state (None/Missing/Present) /// - `blob_cell_availability`: Cached blob cell availability for eth/72 announcements /// @@ -1563,6 +1564,11 @@ pub struct EthPooledTransaction { /// pool. pub encoded_length: usize, + /// Cached in-memory size of `transaction`, excluding the blob sidecar. + /// + /// Must be updated if `transaction` is modified or replaced. + pub in_memory_size: usize, + /// The blob side car for this transaction pub blob_sidecar: EthBlobTransactionSidecar, @@ -1601,7 +1607,15 @@ impl EthPooledTransaction { blob_cell_availability = Some(BlobCellAvailability::full()); } - Self { transaction, cost, encoded_length, blob_sidecar, blob_cell_availability } + let in_memory_size = transaction.size(); + Self { + transaction, + cost, + encoded_length, + in_memory_size, + blob_sidecar, + blob_cell_availability, + } } /// Return the reference to the underlying transaction. @@ -1699,8 +1713,9 @@ impl Typed2718 for EthPooledTransaction { } impl InMemorySize for EthPooledTransaction { + #[inline] fn size(&self) -> usize { - self.transaction.size() + self.in_memory_size } } From fc272b49bca40cb46c85db6df2fc58a5e67599a0 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Mon, 7 Sep 2026 14:07:48 +0000 Subject: [PATCH 195/294] perf(txpool): process insertion updates with one iterator (#27064) --- crates/transaction-pool/src/pool/txpool.rs | 50 ++++++++-------------- 1 file changed, 17 insertions(+), 33 deletions(-) diff --git a/crates/transaction-pool/src/pool/txpool.rs b/crates/transaction-pool/src/pool/txpool.rs index c7227c73879..f60f1a00e60 100644 --- a/crates/transaction-pool/src/pool/txpool.rs +++ b/crates/transaction-pool/src/pool/txpool.rs @@ -650,10 +650,11 @@ impl TxPool { self.all_transactions.sender_info.extend(changed_senders); // Process the sub-pool updates - let update = self.process_updates(updates); + let mut outcome = UpdateOutcome::default(); + self.process_updates(updates, &mut outcome); // update the metrics after the update self.update_size_metrics(); - update + outcome } /// Updates the entire pool after a new block was mined. @@ -756,7 +757,7 @@ impl TxPool { .update(on_chain_nonce, on_chain_balance); match self.all_transactions.insert_tx(tx, on_chain_balance, on_chain_nonce) { - Ok(InsertOk { transaction, move_to, replaced_tx, mut updates, state }) => { + Ok(InsertOk { transaction, move_to, replaced_tx, updates, state }) => { // Interleave update processing and new-tx insertion so that live // `BestTransactions` iterators always receive transactions in nonce order. // Updates are already in nonce-ascending order, so we split them around @@ -765,31 +766,13 @@ impl TxPool { // 2. Add the new transaction // 3. Promote higher-nonce txs last (e.g. gap-fill scenario) let new_nonce = transaction.id().nonce; - let split = updates.iter().position(|u| u.id.nonce >= new_nonce); - let (promoted, discarded) = match split { - // All updates are lower-nonce — promote them first, then add new tx - None => { - let UpdateOutcome { promoted, discarded } = self.process_updates(updates); - self.add_new_transaction(transaction.clone(), replaced_tx.clone(), move_to); - (promoted, discarded) - } - // All updates are higher-nonce — add new tx first, then promote - Some(0) => { - self.add_new_transaction(transaction.clone(), replaced_tx.clone(), move_to); - let UpdateOutcome { promoted, discarded } = self.process_updates(updates); - (promoted, discarded) - } - // Mixed — split and interleave - Some(i) => { - let after = updates.split_off(i); - let mut outcome = self.process_updates(updates); - self.add_new_transaction(transaction.clone(), replaced_tx.clone(), move_to); - let after_outcome = self.process_updates(after); - outcome.promoted.extend(after_outcome.promoted); - outcome.discarded.extend(after_outcome.discarded); - (outcome.promoted, outcome.discarded) - } - }; + let split = updates.partition_point(|u| u.id.nonce < new_nonce); + let mut outcome = UpdateOutcome::default(); + let mut updates = updates.into_iter(); + self.process_updates(updates.by_ref().take(split), &mut outcome); + self.add_new_transaction(transaction.clone(), replaced_tx.clone(), move_to); + self.process_updates(updates, &mut outcome); + let UpdateOutcome { promoted, discarded } = outcome; self.metrics.inserted_transactions.increment(1); let replaced = replaced_tx.map(|(tx, _)| tx); @@ -974,8 +957,11 @@ impl TxPool { /// Maintenance task to apply a series of updates. /// /// This will move/discard the given transaction according to the `PoolUpdate` - fn process_updates(&mut self, updates: Vec) -> UpdateOutcome { - let mut outcome = UpdateOutcome::default(); + fn process_updates( + &mut self, + updates: impl IntoIterator, + outcome: &mut UpdateOutcome, + ) { let mut removed = 0; for PoolUpdate { id, current, destination } in updates { match destination { @@ -1002,8 +988,6 @@ impl TxPool { if removed > 0 { self.metrics.removed_transactions.increment(removed); } - - outcome } /// Moves a transaction from one sub pool to another. @@ -1106,7 +1090,7 @@ impl TxPool { // After a tx is removed, its descendants must become parked due to the nonce gap let updates = self.all_transactions.park_descendant_transactions(tx.id()); - self.process_updates(updates); + self.process_updates(updates, &mut UpdateOutcome::default()); self.remove_from_subpool(pool, tx.id()) } From 14ab693f534a478d3249ef40245c383e65d8fa43 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Mon, 7 Sep 2026 14:07:49 +0000 Subject: [PATCH 196/294] perf(txpool): reuse subpool update buffer (#27060) --- crates/transaction-pool/src/pool/txpool.rs | 101 ++++++++++++++++++--- 1 file changed, 89 insertions(+), 12 deletions(-) diff --git a/crates/transaction-pool/src/pool/txpool.rs b/crates/transaction-pool/src/pool/txpool.rs index f60f1a00e60..a5d1ce052d2 100644 --- a/crates/transaction-pool/src/pool/txpool.rs +++ b/crates/transaction-pool/src/pool/txpool.rs @@ -644,14 +644,16 @@ impl TxPool { changed_senders: FxHashMap, ) -> UpdateOutcome { // Apply the state changes to the total set of transactions which triggers sub-pool updates. - let updates = self.all_transactions.update(&changed_senders); + let mut updates = self.all_transactions.update(&changed_senders); // track changed accounts self.all_transactions.sender_info.extend(changed_senders); // Process the sub-pool updates let mut outcome = UpdateOutcome::default(); - self.process_updates(updates, &mut outcome); + #[expect(clippy::iter_with_drain, reason = "retain the scratch buffer allocation")] + self.process_updates(updates.drain(..), &mut outcome); + self.all_transactions.update_buffer = updates; // update the metrics after the update self.update_size_metrics(); outcome @@ -757,7 +759,7 @@ impl TxPool { .update(on_chain_nonce, on_chain_balance); match self.all_transactions.insert_tx(tx, on_chain_balance, on_chain_nonce) { - Ok(InsertOk { transaction, move_to, replaced_tx, updates, state }) => { + Ok(InsertOk { transaction, move_to, replaced_tx, mut updates, state }) => { // Interleave update processing and new-tx insertion so that live // `BestTransactions` iterators always receive transactions in nonce order. // Updates are already in nonce-ascending order, so we split them around @@ -768,10 +770,12 @@ impl TxPool { let new_nonce = transaction.id().nonce; let split = updates.partition_point(|u| u.id.nonce < new_nonce); let mut outcome = UpdateOutcome::default(); - let mut updates = updates.into_iter(); - self.process_updates(updates.by_ref().take(split), &mut outcome); + #[expect(clippy::iter_with_drain, reason = "retain the scratch buffer allocation")] + let mut drain = updates.drain(..); + self.process_updates(drain.by_ref().take(split), &mut outcome); self.add_new_transaction(transaction.clone(), replaced_tx.clone(), move_to); - self.process_updates(updates, &mut outcome); + self.process_updates(drain, &mut outcome); + self.all_transactions.update_buffer = updates; let UpdateOutcome { promoted, discarded } = outcome; self.metrics.inserted_transactions.increment(1); @@ -1089,8 +1093,10 @@ impl TxPool { let (tx, pool) = self.all_transactions.remove_transaction_by_hash(tx_hash)?; // After a tx is removed, its descendants must become parked due to the nonce gap - let updates = self.all_transactions.park_descendant_transactions(tx.id()); - self.process_updates(updates, &mut UpdateOutcome::default()); + let mut updates = self.all_transactions.park_descendant_transactions(tx.id()); + #[expect(clippy::iter_with_drain, reason = "retain the scratch buffer allocation")] + self.process_updates(updates.drain(..), &mut UpdateOutcome::default()); + self.all_transactions.update_buffer = updates; self.remove_from_subpool(pool, tx.id()) } @@ -1385,6 +1391,9 @@ pub(crate) struct AllTransactions { /// Number of transactions in the pool by transaction type, tracked incrementally so metrics /// updates don't require iterating the entire pool. tx_type_counts: TxTypeCounts, + /// Scratch space for sub-pool changes. Update methods take this buffer and return it to + /// `TxPool`, which drains the updates and restores the empty buffer for reuse. + update_buffer: Vec, /// All Transactions metrics metrics: AllTransactionsMetrics, } @@ -1496,8 +1505,7 @@ impl AllTransactions { &mut self, changed_accounts: &FxHashMap, ) -> Vec { - // pre-allocate a few updates - let mut updates = Vec::with_capacity(64); + let mut updates = std::mem::take(&mut self.update_buffer); let mut iter = self.txs.iter_mut().peekable(); @@ -1750,7 +1758,7 @@ impl AllTransactions { &mut self, tx_id: &TransactionId, ) -> Vec { - let mut updates = Vec::new(); + let mut updates = std::mem::take(&mut self.update_buffer); for (id, tx) in self.descendant_txs_mut(tx_id) { let current_pool = tx.subpool; @@ -1970,7 +1978,6 @@ impl AllTransactions { let inserted_tx_id = *transaction.id(); let mut state = TxState::default(); let mut cumulative_cost = U256::ZERO; - let mut updates = Vec::new(); // Current tx does not exceed block gas limit after ensure_valid check state.insert(TxState::NOT_TOO_MUCH_GAS); @@ -2069,6 +2076,9 @@ impl AllTransactions { } } + // Take the scratch buffer only after all fallible checks so rejected inserts retain it. + let mut updates = std::mem::take(&mut self.update_buffer); + // The next transaction of this sender let on_chain_id = TransactionId::new(transaction.sender_id(), on_chain_nonce); { @@ -2189,6 +2199,7 @@ impl Default for AllTransactions { local_transactions_config: Default::default(), auths: Default::default(), tx_type_counts: Default::default(), + update_buffer: Default::default(), metrics: Default::default(), } } @@ -3959,6 +3970,72 @@ mod tests { } } + #[test] + fn reuse_update_buffer_across_pool_operations() { + let mut f = MockTransactionFactory::default(); + let mut pool = TxPool::new( + MockOrdering::default(), + PoolConfig { max_account_slots: 128, ..Default::default() }, + ); + let mut tx = MockTransaction::eip1559().set_gas_price(100).inc_limit(); + let first = f.validated(tx.clone()); + for _ in 0..80 { + pool.add_transaction(f.validated(tx.clone()), U256::MAX, 0, None).unwrap(); + tx = tx.next(); + } + assert_eq!(pool.all_transactions.update_buffer.capacity(), 0); + + let changed = |balance, state_nonce| { + FxHashMap::from_iter([(first.sender_id(), SenderInfo { balance, state_nonce })]) + }; + let outcome = pool.update_accounts(changed(U256::ZERO, 0)); + assert!(outcome.promoted.is_empty()); + assert!(outcome.discarded.is_empty()); + assert_eq!(pool.queued_pool.len(), 80); + let capacity = pool.all_transactions.update_buffer.capacity(); + assert!(capacity >= 80); + let buffer = pool.all_transactions.update_buffer.as_ptr(); + + for _ in 0..2 { + let outcome = pool.update_accounts(changed(U256::MAX, 0)); + assert_eq!(outcome.promoted.len(), 80); + assert!(outcome.discarded.is_empty()); + assert_eq!(pool.pending_pool.len(), 80); + + let outcome = pool.update_accounts(FxHashMap::default()); + assert!(outcome.promoted.is_empty()); + assert!(outcome.discarded.is_empty()); + + pool.remove_transaction_by_hash(first.hash()).unwrap(); + assert_eq!(pool.queued_pool.len(), 79); + + // A rejected replacement must leave the reusable allocation available. + let replacement = f.validated(first.transaction.next()); + assert!(pool.add_transaction(replacement, U256::MAX, 0, None).is_err()); + assert_eq!(pool.all_transactions.update_buffer.as_ptr(), buffer); + + let added = pool.add_transaction(first.clone(), U256::MAX, 0, None).unwrap(); + let AddedTransaction::Pending(added) = added else { panic!("expected pending") }; + assert_eq!(added.promoted.len(), 79); + assert!(added.discarded.is_empty()); + assert_eq!(pool.pending_pool.len(), 80); + + pool.update_accounts(changed(U256::ZERO, 0)); + assert_eq!(pool.queued_pool.len(), 80); + assert!(pool.all_transactions.update_buffer.is_empty()); + assert_eq!(pool.all_transactions.update_buffer.capacity(), capacity); + assert_eq!(pool.all_transactions.update_buffer.as_ptr(), buffer); + pool.assert_invariants(); + } + + let outcome = pool.update_accounts(changed(U256::MAX, 80)); + assert_eq!(outcome.discarded.len(), 80); + assert!(outcome.promoted.is_empty()); + assert!(pool.is_empty()); + assert!(pool.all_transactions.update_buffer.is_empty()); + assert_eq!(pool.all_transactions.update_buffer.as_ptr(), buffer); + } + #[test] fn account_updates_sender_balance() { let mut on_chain_balance = U256::from(100); From 280a76318d7ab933f4a1536d1942ffeb4c742d1a Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Mon, 7 Sep 2026 15:27:37 +0000 Subject: [PATCH 197/294] perf(txpool): only visit changed senders when the fees did not move (#26956) Co-authored-by: Matthias Seitz <19890894+mattsse@users.noreply.github.com> --- .../benches/saturated_pool.rs | 140 +++++- crates/transaction-pool/src/pool/blob.rs | 4 +- crates/transaction-pool/src/pool/txpool.rs | 413 +++++++++++++++++- 3 files changed, 538 insertions(+), 19 deletions(-) diff --git a/crates/transaction-pool/benches/saturated_pool.rs b/crates/transaction-pool/benches/saturated_pool.rs index 0d212583409..c87612b8bd8 100644 --- a/crates/transaction-pool/benches/saturated_pool.rs +++ b/crates/transaction-pool/benches/saturated_pool.rs @@ -4,7 +4,7 @@ //! continuously trigger evictions, mimicking a saturated mainnet mempool. use alloy_primitives::{Address, B256, U256}; -use criterion::{criterion_group, criterion_main, BatchSize, Criterion}; +use criterion::{criterion_group, criterion_main, BatchSize, BenchmarkId, Criterion}; use reth_execution_types::ChangedAccount; use reth_primitives_traits::SealedBlock; use reth_transaction_pool::{ @@ -13,7 +13,7 @@ use reth_transaction_pool::{ pool::PoolInner, test_utils::{MockOrdering, MockTransaction}, validate::ValidTransaction, - CanonicalStateUpdate, PoolConfig, PoolUpdateKind, TransactionOrigin, + CanonicalStateUpdate, PoolConfig, PoolUpdateKind, SubPoolLimit, TransactionOrigin, TransactionValidationOutcome, }; @@ -143,6 +143,44 @@ fn build_saturated_pool() -> (BenchPool, u64) { (pool, counter) } +/// Builds an uncapped pending pool with one transaction per sender. +fn build_single_tx_sender_pool(senders: usize) -> (BenchPool, u64) { + let pool = PoolInner::new( + MockTransactionValidator::default(), + MockOrdering::default(), + InMemoryBlobStore::default(), + PoolConfig { pending_limit: SubPoolLimit::max(), ..Default::default() }, + ); + + let block = tip_block(); + pool.on_canonical_state_change(CanonicalStateUpdate { + new_tip: &block, + pending_block_base_fee: BASE_FEE, + pending_block_blob_fee: None, + changed_accounts: Vec::new(), + mined_transactions: Vec::new(), + update_kind: PoolUpdateKind::Commit, + }); + + let mut counter = 0; + pool.add_transactions_with_origins(pending_batch(&mut counter, senders, 1)); + (pool, counter) +} + +/// Returns all live senders followed by an additional 25% of tracked senders without +/// transactions. +fn senders_for_update_bench(pool: &BenchPool, counter: &mut u64) -> (Vec
, usize) { + let mut senders = pool.unique_senders().into_iter().collect::>(); + senders.sort_unstable(); + let pool_senders = senders.len(); + + let extra_senders = (0..pool_senders / 4).map(|_| sender_address(counter)).collect::>(); + let _ = pool.get_sender_ids(extra_senders.iter().copied()); + senders.extend(extra_senders); + + (senders, pool_senders) +} + /// Benchmarks adding a batch of transactions to a pool at capacity, which forces eviction. fn bench_add_transactions(c: &mut Criterion) { let mut group = c.benchmark_group("saturated_pool"); @@ -174,6 +212,96 @@ fn bench_add_transactions(c: &mut Criterion) { group.finish(); } +/// Benchmarks an account-only update against a pool at capacity: sender nonces and balances move +/// without any fee change, as happens on the reorg account-reload path and via +/// `Pool::update_accounts`. +fn bench_update_accounts(c: &mut Criterion) { + let mut group = c.benchmark_group("saturated_pool"); + group.sample_size(30); + + let (pool, mut counter) = build_saturated_pool(); + let size = pool.size(); + assert!(size.pending >= 9_000, "pending pool not saturated: {size:?}"); + + let (senders, pool_senders) = senders_for_update_bench(&pool, &mut counter); + assert_eq!(pool_senders, 4_000); + + // the reported nonce stays put so the transactions are re-evaluated rather than discarded, + // which keeps every iteration doing the same amount of work + let mut flip = false; + for ratio_percent in [0.25, 1.0, 10.0, 25.0, 50.0, 100.0, 125.0] { + let changed_senders = + ((pool_senders as f64 * ratio_percent / 100.0).round() as usize).max(1); + group.bench_with_input( + BenchmarkId::new("update_accounts", format!("{ratio_percent}%")), + &changed_senders, + |b, &changed_senders| { + b.iter_batched( + || { + flip = !flip; + let balance = + if flip { U256::from(u128::MAX) } else { U256::from(u128::MAX - 1) }; + senders[..changed_senders] + .iter() + .map(|address| ChangedAccount { address: *address, nonce: 0, balance }) + .collect::>() + }, + |changed| pool.update_accounts(changed), + BatchSize::PerIteration, + ) + }, + ); + } + + group.finish(); +} + +/// Benchmarks account-only updates by changed-sender ratio in pools where every sender has a +/// single transaction. +fn bench_update_accounts_single_tx_senders(c: &mut Criterion) { + let mut group = c.benchmark_group("single_tx_sender_pool"); + group.sample_size(30); + + for pool_senders in [100, 500, 1_000, 4_000] { + let (pool, mut counter) = build_single_tx_sender_pool(pool_senders); + let (senders, actual_pool_senders) = senders_for_update_bench(&pool, &mut counter); + assert_eq!(actual_pool_senders, pool_senders); + + let mut flip = false; + for ratio_percent in [25usize, 100, 125] { + let changed_senders = pool_senders * ratio_percent / 100; + group.bench_with_input( + BenchmarkId::new(format!("{pool_senders}_senders"), format!("{ratio_percent}%")), + &changed_senders, + |b, &changed_senders| { + b.iter_batched( + || { + flip = !flip; + let balance = if flip { + U256::from(u128::MAX) + } else { + U256::from(u128::MAX - 1) + }; + senders[..changed_senders] + .iter() + .map(|address| ChangedAccount { + address: *address, + nonce: 0, + balance, + }) + .collect::>() + }, + |changed| pool.update_accounts(changed), + BatchSize::PerIteration, + ) + }, + ); + } + } + + group.finish(); +} + /// Benchmarks a canonical state update against a pool at capacity: mined transactions are /// removed, sender states change and the base fee moves. fn bench_canonical_state_change(c: &mut Criterion) { @@ -233,5 +361,11 @@ fn bench_canonical_state_change(c: &mut Criterion) { group.finish(); } -criterion_group!(benches, bench_add_transactions, bench_canonical_state_change); +criterion_group!( + benches, + bench_add_transactions, + bench_canonical_state_change, + bench_update_accounts, + bench_update_accounts_single_tx_senders +); criterion_main!(benches); diff --git a/crates/transaction-pool/src/pool/blob.rs b/crates/transaction-pool/src/pool/blob.rs index 01dde166d72..587f192bbd0 100644 --- a/crates/transaction-pool/src/pool/blob.rs +++ b/crates/transaction-pool/src/pool/blob.rs @@ -225,7 +225,7 @@ impl BlobTransactions { .collect(); // Update pending fees and reprioritize - self.pending_fees = pending_fees.clone(); + self.pending_fees = *pending_fees; self.reprioritize(); removed @@ -662,7 +662,7 @@ mod tests { } // update fees and resort the pool - pool.pending_fees = ordering.network_fees.clone(); + pool.pending_fees = ordering.network_fees; pool.reprioritize(); // now iterate through the pool and make sure they're in the same order as the original diff --git a/crates/transaction-pool/src/pool/txpool.rs b/crates/transaction-pool/src/pool/txpool.rs index a5d1ce052d2..0cf8e8127a2 100644 --- a/crates/transaction-pool/src/pool/txpool.rs +++ b/crates/transaction-pool/src/pool/txpool.rs @@ -1355,6 +1355,15 @@ impl fmt::Debug for TxPool { } } +/// Minimum number of live senders before the changed-sender ratio can trigger a full update. +/// +/// Account-update benchmarks put the paths near parity at 500 senders and favor a full traversal +/// from 1,000 senders at the ratio below. +const FULL_UPDATE_MIN_SENDERS: usize = 1_000; + +/// Run a full update when at least one in this many live senders changed. +const FULL_UPDATE_SENDER_RATIO: usize = 4; + /// Container for _all_ transaction in the pool. /// /// This is the sole entrypoint that's guarding all sub-pools, all sub-pool actions are always @@ -1382,6 +1391,14 @@ pub(crate) struct AllTransactions { last_seen_block_hash: B256, /// Expected blob and base fee for the pending block. pending_fees: PendingFees, + /// Snapshot of [`Self::pending_fees`] recorded after the most recent all-transactions pass in + /// [`Self::update`]. + /// + /// The all-transactions pass calls [`Self::update_txs`] over every entry in [`Self::txs`]. + /// This field is initialized to the default fees because an empty pool trivially reflects + /// them. It is not necessarily the immediately preceding fee value: partial sender updates + /// and fee changes do not modify it. + last_full_update_fees: PendingFees, /// Configured price bump settings for replacements price_bumps: PriceBumpConfig, /// How to handle [`TransactionOrigin::Local`](crate::TransactionOrigin) transactions. @@ -1506,8 +1523,53 @@ impl AllTransactions { changed_accounts: &FxHashMap, ) -> Vec { let mut updates = std::mem::take(&mut self.update_buffer); + let pending_fees = self.pending_fees; + + let update_all = self.last_full_update_fees != self.pending_fees || + self.should_update_all_senders(changed_accounts.len()); + + if update_all { + Self::update_txs(pending_fees, changed_accounts, &mut updates, self.txs.iter_mut()); + self.last_full_update_fees = self.pending_fees; + } else { + // Fee eligibility is unchanged, while nonce gaps, ancestors, and cumulative cost are + // sender-local; only transactions from changed accounts can require updates. + for sender in changed_accounts.keys() { + let range = TransactionId::new(*sender, 0)..=TransactionId::new(*sender, u64::MAX); + Self::update_txs( + pending_fees, + changed_accounts, + &mut updates, + self.txs.range_mut(range), + ); + } + } - let mut iter = self.txs.iter_mut().peekable(); + updates + } + + /// Returns whether one full traversal is preferable to a range lookup per changed sender. + fn should_update_all_senders(&self, changed_sender_count: usize) -> bool { + let pool_sender_count = self.tx_counter.len(); + changed_sender_count >= pool_sender_count || + (pool_sender_count >= FULL_UPDATE_MIN_SENDERS && + changed_sender_count >= pool_sender_count.div_ceil(FULL_UPDATE_SENDER_RATIO)) + } + + /// Updates the given transactions, which must be ordered by [`TransactionId`] and must start + /// at the lowest tracked nonce of the first sender they contain. + /// + /// Records a [`PoolUpdate`] for every transaction whose sub-pool changed. + fn update_txs<'a, I>( + pending_fees: PendingFees, + changed_accounts: &FxHashMap, + updates: &mut Vec, + txs: I, + ) where + T: 'a, + I: Iterator)>, + { + let mut iter = txs.peekable(); // Loop over all individual senders and update all affected transactions. // One sender may have up to `max_account_slots` transactions here, which means, worst case @@ -1572,9 +1634,9 @@ impl AllTransactions { tx.state.insert(TxState::NO_PARKED_ANCESTORS); // Update the first transaction of this sender. - Self::update_tx_base_fee(self.pending_fees.base_fee, tx); + Self::update_tx_fees(pending_fees, tx); // Track if the transaction's sub-pool changed. - Self::record_subpool_update(&mut updates, tx); + Self::record_subpool_update(updates, tx); // Track blocking transactions. let mut has_parked_ancestor = !tx.state.is_pending(); @@ -1624,18 +1686,16 @@ impl AllTransactions { } else { tx.state.insert(TxState::NO_PARKED_ANCESTORS); } - has_parked_ancestor = !tx.state.is_pending(); // Update and record sub-pool changes. - Self::update_tx_base_fee(self.pending_fees.base_fee, tx); - Self::record_subpool_update(&mut updates, tx); + Self::update_tx_fees(pending_fees, tx); + Self::record_subpool_update(updates, tx); + has_parked_ancestor = !tx.state.is_pending(); // Advance iterator iter.next(); } } - - updates } /// This will update the transaction's `subpool` based on its state. @@ -1654,10 +1714,9 @@ impl AllTransactions { } } - /// Rechecks the transaction's dynamic fee condition. - fn update_tx_base_fee(pending_block_base_fee: u64, tx: &mut PoolInternalTransaction) { - // Recheck dynamic fee condition. - match tx.transaction.max_fee_per_gas().cmp(&(pending_block_base_fee as u128)) { + /// Rechecks the transaction's dynamic fee conditions. + fn update_tx_fees(pending_fees: PendingFees, tx: &mut PoolInternalTransaction) { + match tx.transaction.max_fee_per_gas().cmp(&(pending_fees.base_fee as u128)) { Ordering::Greater | Ordering::Equal => { tx.state.insert(TxState::ENOUGH_FEE_CAP_BLOCK); } @@ -1665,6 +1724,15 @@ impl AllTransactions { tx.state.remove(TxState::ENOUGH_FEE_CAP_BLOCK); } } + + match tx.transaction.max_fee_per_blob_gas() { + Some(blob_fee_cap) if blob_fee_cap < pending_fees.blob_fee => { + tx.state.remove(TxState::ENOUGH_BLOB_FEE_CAP_BLOCK); + } + _ => { + tx.state.insert(TxState::ENOUGH_BLOB_FEE_CAP_BLOCK); + } + } } /// Returns an iterator over all transactions for the given sender, starting with the lowest @@ -2081,6 +2149,7 @@ impl AllTransactions { // The next transaction of this sender let on_chain_id = TransactionId::new(transaction.sender_id(), on_chain_nonce); + let pending_fees = self.pending_fees; { // Tracks the next nonce we expect if the transactions are gapless let mut next_nonce = on_chain_id.nonce; @@ -2122,10 +2191,12 @@ impl AllTransactions { } else { tx.state.insert(TxState::NO_PARKED_ANCESTORS); } - has_parked_ancestor = !tx.state.is_pending(); + + Self::update_tx_fees(pending_fees, tx); // update the pool based on the state tx.subpool = tx.state.into(); + has_parked_ancestor = !tx.state.is_pending(); if inserted_tx_id.eq(id) { // if it is the new transaction, track its updated state @@ -2195,6 +2266,8 @@ impl Default for AllTransactions { last_seen_block_number: Default::default(), last_seen_block_hash: Default::default(), pending_fees: Default::default(), + // an empty pool trivially reflects the initial fees + last_full_update_fees: Default::default(), price_bumps: Default::default(), local_transactions_config: Default::default(), auths: Default::default(), @@ -2244,7 +2317,7 @@ impl TxTypeCounts { } /// Represents updated fees for the pending block. -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] pub(crate) struct PendingFees { /// The pending base fee pub(crate) base_fee: u64, @@ -3404,6 +3477,318 @@ mod tests { assert!(state.contains(TxState::NOT_TOO_MUCH_GAS)); } + #[test] + fn full_update_sender_heuristic() { + let mut pool = AllTransactions::::default(); + for sender in 0..(FULL_UPDATE_MIN_SENDERS - 1) as u64 { + pool.tx_counter.insert(sender.into(), 1); + } + + // Below the pool-size floor, only covering every live sender triggers a full update. + assert!(!pool.should_update_all_senders(250)); + assert!(pool.should_update_all_senders(FULL_UPDATE_MIN_SENDERS - 1)); + + pool.tx_counter.insert((FULL_UPDATE_MIN_SENDERS as u64 - 1).into(), 1); + + // At the floor, one quarter of the live senders is the crossover. + assert!( + !pool.should_update_all_senders(FULL_UPDATE_MIN_SENDERS / FULL_UPDATE_SENDER_RATIO - 1) + ); + assert!(pool.should_update_all_senders(FULL_UPDATE_MIN_SENDERS / FULL_UPDATE_SENDER_RATIO)); + assert!(pool.should_update_all_senders(FULL_UPDATE_MIN_SENDERS + 1)); + } + + #[test] + fn update_only_visits_changed_senders_when_fees_are_unchanged() { + let mut f = MockTransactionFactory::default(); + let mut pool = TxPool::new(MockOrdering::default(), Default::default()); + + // two senders, each with one transaction + let a = f.validated(MockTransaction::eip1559().inc_price_by(10)); + let b = f.validated(MockTransaction::eip1559().inc_price_by(10)); + let (a_id, b_id) = (*a.id(), *b.id()); + assert_ne!(a_id.sender, b_id.sender); + pool.add_transaction(a, U256::from(1_000_000), 0, None).unwrap(); + pool.add_transaction(b, U256::from(1_000_000), 0, None).unwrap(); + + // a full update applies the current fees to every transaction and records them + pool.all_transactions.update(&Default::default()); + assert_eq!(pool.all_transactions.last_full_update_fees, pool.all_transactions.pending_fees); + + // sender `a` moved past its transaction on chain, the fees did not move. Only `a` should + // be evaluated, and `b` must be left exactly as it was. + let b_state_before = pool.all_transactions.txs.get(&b_id).unwrap().state; + let mut changed = FxHashMap::default(); + changed.insert(a_id.sender, SenderInfo { state_nonce: 1, balance: U256::from(1_000_000) }); + + let produced = pool.all_transactions.update(&changed); + + assert_eq!(produced.len(), 1, "expected exactly the changed sender's update"); + assert_eq!(produced[0].id, a_id); + assert!(matches!(produced[0].destination, Destination::Discard)); + assert_eq!( + pool.all_transactions.txs.get(&b_id).unwrap().state, + b_state_before, + "unchanged sender was modified" + ); + } + + #[test] + fn changed_sender_update_matches_full_update() { + let senders = [ + address!("0x000000000000000000000000000000000000000a"), + address!("0x000000000000000000000000000000000000000b"), + address!("0x000000000000000000000000000000000000000c"), + ]; + let starting_nonces = [5, 11, 17]; + + let build_pool = || { + let mut f = MockTransactionFactory::default(); + let mut pool = AllTransactions::default(); + pool.pending_fees.base_fee = 1; + + for (sender, starting_nonce) in senders.into_iter().zip(starting_nonces) { + for nonce in starting_nonce..starting_nonce + 3 { + let tx = MockTransaction::eip1559() + .with_sender(sender) + .with_nonce(nonce) + .inc_price_by(10) + .rng_hash(); + pool.insert_tx(f.validated(tx), U256::from(1_000_000), starting_nonce).unwrap(); + } + } + + // The fee differs from the initial marker, so this takes the all-transactions path. + pool.update(&Default::default()); + let sender_ids = senders.map(|sender| f.ids.sender_id(&sender).unwrap()); + (pool, sender_ids) + }; + + let (mut changed_senders_only, sender_ids) = build_pool(); + let (mut full_update, full_update_sender_ids) = build_pool(); + assert_eq!(sender_ids, full_update_sender_ids); + + let mut changed = FxHashMap::default(); + changed.insert( + sender_ids[0], + SenderInfo { state_nonce: starting_nonces[0] + 1, balance: U256::from(1_000_000) }, + ); + changed.insert( + sender_ids[1], + SenderInfo { state_nonce: starting_nonces[1], balance: U256::ZERO }, + ); + + let mut changed_sender_updates = changed_senders_only.update(&changed); + // Force the reference pool through the all-transactions path with the same pending fees. + full_update.last_full_update_fees.base_fee = + full_update.last_full_update_fees.base_fee.saturating_add(1); + let mut full_updates = full_update.update(&changed); + + let update_key = |update: &PoolUpdate| { + let destination = match &update.destination { + Destination::Discard => None, + Destination::Pool(pool) => Some(*pool), + }; + (update.id, update.current, destination) + }; + changed_sender_updates.sort_unstable_by_key(|update| update.id); + full_updates.sort_unstable_by_key(|update| update.id); + assert_eq!( + changed_sender_updates.iter().map(update_key).collect::>(), + full_updates.iter().map(update_key).collect::>() + ); + + let metadata = |pool: &AllTransactions| { + pool.txs + .iter() + .map(|(id, tx)| (*id, tx.state, tx.subpool, tx.cumulative_cost)) + .collect::>() + }; + assert_eq!(metadata(&changed_senders_only), metadata(&full_update)); + } + + #[test] + fn update_visits_every_sender_when_the_base_fee_moved() { + let mut f = MockTransactionFactory::default(); + let mut pool = TxPool::new(MockOrdering::default(), Default::default()); + + let tx = MockTransaction::eip1559().inc_price_by(10); + let validated = f.validated(tx.clone()); + let id = *validated.id(); + pool.add_transaction(validated, U256::from(1_000_000), 0, None).unwrap(); + + pool.all_transactions.update(&Default::default()); + assert!(pool + .all_transactions + .txs + .get(&id) + .unwrap() + .state + .contains(TxState::ENOUGH_FEE_CAP_BLOCK)); + + // raising the base fee past the tx must still reach it even though no account changed, + // otherwise the fast path would leave a stale fee bit behind + pool.all_transactions.pending_fees.base_fee = (tx.max_fee_per_gas() + 1) as u64; + pool.all_transactions.update(&Default::default()); + + assert!( + !pool + .all_transactions + .txs + .get(&id) + .unwrap() + .state + .contains(TxState::ENOUGH_FEE_CAP_BLOCK), + "fee change was not applied to an unchanged sender" + ); + } + + #[test] + fn blob_fee_change_records_full_update() { + let mut pool = AllTransactions::::default(); + pool.pending_fees.blob_fee += 1; + + pool.update(&Default::default()); + + assert_eq!(pool.last_full_update_fees, pool.pending_fees); + } + + #[test] + fn gap_fill_rechecks_descendant_fee_eligibility() { + let mut f = MockTransactionFactory::default(); + let mut pool = AllTransactions::default(); + let sender = address!("0x000000000000000000000000000000000000000d"); + let balance = U256::MAX; + + pool.pending_fees.base_fee = 100; + let descendant = MockTransaction::eip1559() + .with_sender(sender) + .with_nonce(1) + .with_gas_limit(21_000) + .with_max_fee(150) + .with_priority_fee(1) + .rng_hash(); + let descendant = f.validated(descendant); + let descendant_id = *descendant.id(); + pool.insert_tx(descendant, balance, 0).unwrap(); + pool.update(&Default::default()); + + // The fee increase cannot affect a nonce-gapped transaction yet, but closing its gap must + // evaluate it against the current fee. + pool.pending_fees.base_fee = 200; + pool.update(&Default::default()); + assert!(pool.get(&descendant_id).unwrap().state.contains(TxState::ENOUGH_FEE_CAP_BLOCK)); + + let predecessor = MockTransaction::eip1559() + .with_sender(sender) + .with_nonce(0) + .with_gas_limit(21_000) + .with_max_fee(250) + .with_priority_fee(1) + .rng_hash(); + let InsertOk { move_to, .. } = + pool.insert_tx(f.validated(predecessor), balance, 0).unwrap(); + + assert_eq!(move_to, SubPool::Pending); + let descendant = pool.get(&descendant_id).unwrap(); + assert_eq!(descendant.subpool, SubPool::BaseFee); + assert!(!descendant.state.contains(TxState::ENOUGH_FEE_CAP_BLOCK)); + assert!(descendant.state.contains(TxState::ENOUGH_BLOB_FEE_CAP_BLOCK)); + } + + #[test] + fn base_fee_update_unparks_all_descendants() { + let mut f = MockTransactionFactory::default(); + let mut pool = AllTransactions::default(); + let sender = address!("0x000000000000000000000000000000000000000e"); + let mut ids = Vec::new(); + + pool.pending_fees.base_fee = 200; + for nonce in 0..3 { + let tx = MockTransaction::eip1559() + .with_sender(sender) + .with_nonce(nonce) + .with_gas_limit(21_000) + .with_max_fee(150) + .with_priority_fee(1) + .rng_hash(); + let tx = f.validated(tx); + ids.push(*tx.id()); + pool.insert_tx(tx, U256::MAX, 0).unwrap(); + } + + pool.pending_fees.base_fee = 100; + pool.update(&Default::default()); + + for id in ids { + assert_eq!(pool.get(&id).unwrap().subpool, SubPool::Pending); + } + } + + #[test] + fn fee_update_keeps_descendants_of_underpriced_transaction_parked() { + let mut f = MockTransactionFactory::default(); + let mut pool = AllTransactions::default(); + let sender = address!("0x0000000000000000000000000000000000000010"); + let fee_caps = [150, 50, 150]; + let mut ids = Vec::new(); + + pool.pending_fees.base_fee = 200; + for (nonce, fee_cap) in fee_caps.into_iter().enumerate() { + let tx = MockTransaction::eip1559() + .with_sender(sender) + .with_nonce(nonce as u64) + .with_gas_limit(21_000) + .with_max_fee(fee_cap) + .with_priority_fee(1) + .rng_hash(); + let tx = f.validated(tx); + ids.push(*tx.id()); + pool.insert_tx(tx, U256::MAX, 0).unwrap(); + } + + pool.pending_fees.base_fee = 100; + pool.update(&Default::default()); + + assert_eq!(pool.get(&ids[0]).unwrap().subpool, SubPool::Pending); + assert_eq!(pool.get(&ids[1]).unwrap().subpool, SubPool::BaseFee); + assert_eq!(pool.get(&ids[2]).unwrap().subpool, SubPool::Queued); + } + + #[test] + fn blob_fee_update_unparks_all_descendants() { + let mut f = MockTransactionFactory::default(); + let mut pool = TxPool::new(MockOrdering::default(), Default::default()); + let sender = address!("0x000000000000000000000000000000000000000f"); + let mut block_info = pool.block_info(); + block_info.pending_blob_fee = Some(200); + pool.set_block_info(block_info); + + for nonce in 0..3 { + let tx = MockTransaction::eip4844() + .with_sender(sender) + .with_nonce(nonce) + .with_gas_limit(21_000) + .with_max_fee(1_000) + .with_priority_fee(1) + .with_blob_fee(150) + .rng_hash(); + pool.add_transaction(f.validated(tx), U256::MAX, 0, None).unwrap(); + } + assert_eq!(pool.blob_pool.len(), 3); + + block_info.pending_blob_fee = Some(100); + pool.on_canonical_state_change( + block_info, + Vec::new(), + FxHashMap::default(), + PoolUpdateKind::Commit, + ); + + let nonces = pool.best_transactions().map(|tx| tx.nonce()).collect::>(); + assert_eq!(nonces, vec![0, 1, 2]); + } + #[test] fn update_basefee_subpools() { let mut f = MockTransactionFactory::default(); From 8439ef17d10a15bbeb22c3e24b664eddc7c58f41 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A9a=20Narzis?= <78718413+lean-apple@users.noreply.github.com> Date: Tue, 8 Sep 2026 06:46:13 +0000 Subject: [PATCH 198/294] refactor(net): reuse alloy bal hash validation (#27051) --- .../downloaders/src/snap/block_access_list.rs | 10 ++++---- crates/net/p2p/src/full_block.rs | 24 +++++++++---------- 2 files changed, 16 insertions(+), 18 deletions(-) diff --git a/crates/net/downloaders/src/snap/block_access_list.rs b/crates/net/downloaders/src/snap/block_access_list.rs index 020efdd691b..d1d56110456 100644 --- a/crates/net/downloaders/src/snap/block_access_list.rs +++ b/crates/net/downloaders/src/snap/block_access_list.rs @@ -246,16 +246,16 @@ impl BlockAccessListVerifier { // Hashing the raw bytes settles authenticity without decoding, so a peer cannot // charge us the decode of a list it was never able to serve. let raw = RawBal::new(raw); - if raw.hash() != commitment { + raw.ensure_hash(commitment).map_err(|error| { debug!( target: "downloaders::snap", %block_hash, - expected = %commitment, - got = %raw.hash(), + expected = %error.expected, + got = %error.computed, "Block access list does not match its header commitment" ); - return Err(RequestError::BadResponse) - } + RequestError::BadResponse + })?; let decoded = DecodedBal::from_raw_bal(raw).map_err(|error| { debug!(target: "downloaders::snap", %block_hash, %error, "Invalid block access list"); RequestError::BadResponse diff --git a/crates/net/p2p/src/full_block.rs b/crates/net/p2p/src/full_block.rs index 7affef53cd8..df8d3f8fba2 100644 --- a/crates/net/p2p/src/full_block.rs +++ b/crates/net/p2p/src/full_block.rs @@ -1144,19 +1144,17 @@ fn seal_block_access_list_for_block( let (peer, bal) = bal.split(); let Some(bal) = bal else { return Ok(None) }; let raw_bal = RawBal::new(bal); - let computed = raw_bal.hash(); - if computed == expected { - return Ok(Some(raw_bal)) - } - - debug!( - target: "downloaders", - block_hash = ?block.hash(), - ?computed, - ?expected, - "Received block access list with wrong hash", - ); - Err(peer) + raw_bal.ensure_hash(expected).map_err(|error| { + debug!( + target: "downloaders", + block_hash = ?block.hash(), + computed = ?error.computed, + expected = ?error.expected, + "Received block access list with wrong hash", + ); + peer + })?; + Ok(Some(raw_bal)) } /// Wraps a block range with validated block access-list entries. From 067dd8e4d383017b7066727c1fc9cbff20efe7d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A9a=20Narzis?= <78718413+lean-apple@users.noreply.github.com> Date: Tue, 8 Sep 2026 07:00:14 +0000 Subject: [PATCH 199/294] refactor(engine): reuse alloy bal state and index helpers (#27053) --- .../tree/src/tree/payload_processor/bal/worker.rs | 5 ++++- .../tree/src/tree/payload_processor/prewarm.rs | 13 +++++-------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/crates/engine/tree/src/tree/payload_processor/bal/worker.rs b/crates/engine/tree/src/tree/payload_processor/bal/worker.rs index b8fa1a86248..09d7cfdf67a 100644 --- a/crates/engine/tree/src/tree/payload_processor/bal/worker.rs +++ b/crates/engine/tree/src/tree/payload_processor/bal/worker.rs @@ -92,7 +92,10 @@ pub(super) fn spawn_worker<'scope, Evm, Tx, Err, DB, MakeDb>( let signer = *tx.signer(); let tx_gas_limit = tx.tx().gas_limit(); - executor.evm_mut().db_mut().set_bal_index(BlockAccessIndex::new(index as u64 + 1)); + executor + .evm_mut() + .db_mut() + .set_bal_index(BlockAccessIndex::from_tx_index(index as u64)); let result = executor .execute_transaction_without_commit(tx) .map_err(BalWorkerError::Execution)?; diff --git a/crates/engine/tree/src/tree/payload_processor/prewarm.rs b/crates/engine/tree/src/tree/payload_processor/prewarm.rs index a2dbc4866ea..0041615f4b6 100644 --- a/crates/engine/tree/src/tree/payload_processor/prewarm.rs +++ b/crates/engine/tree/src/tree/payload_processor/prewarm.rs @@ -692,14 +692,11 @@ where // changes to start processing them before potentially hitting the db in the next step. if !account_changes.storage_changes.is_empty() { let hashed_address = *hashed_address.get_or_insert_with(|| keccak256(address)); - let mut storage_map = reth_trie::HashedStorage::default(); - - for slot_changes in &account_changes.storage_changes { - let hashed_slot = keccak256(slot_changes.slot.to_be_bytes::<32>()); - if let Some(last_change) = slot_changes.changes.last() { - storage_map.storage.insert(hashed_slot, last_change.new_value); - } - } + let storage_map = reth_trie::HashedStorage::from_iter( + account_changes + .storage_post_states() + .map(|(slot, value)| (keccak256(slot.to_be_bytes::<32>()), value)), + ); let mut hashed_state = reth_trie::HashedPostState::default(); hashed_state.storages.insert(hashed_address, storage_map); From 92dd2108b50784188f10bc8669678641f9401465 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A9a=20Narzis?= <78718413+lean-apple@users.noreply.github.com> Date: Tue, 8 Sep 2026 07:08:15 +0000 Subject: [PATCH 200/294] feat: scalffold new `snap-sync` crate with minimal pivot module (#27054) --- Cargo.lock | 16 ++ Cargo.toml | 2 + crates/snap-sync/Cargo.toml | 31 ++++ crates/snap-sync/src/error.rs | 11 ++ crates/snap-sync/src/generation.rs | 75 +++++++++ crates/snap-sync/src/lib.rs | 44 +++++ crates/snap-sync/src/pivot.rs | 256 +++++++++++++++++++++++++++++ crates/snap-sync/src/session.rs | 213 ++++++++++++++++++++++++ crates/snap-sync/src/test_utils.rs | 46 ++++++ 9 files changed, 694 insertions(+) create mode 100644 crates/snap-sync/Cargo.toml create mode 100644 crates/snap-sync/src/error.rs create mode 100644 crates/snap-sync/src/generation.rs create mode 100644 crates/snap-sync/src/lib.rs create mode 100644 crates/snap-sync/src/pivot.rs create mode 100644 crates/snap-sync/src/session.rs create mode 100644 crates/snap-sync/src/test_utils.rs diff --git a/Cargo.lock b/Cargo.lock index 2369ed53aeb..2e56824c40f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -10258,6 +10258,22 @@ dependencies = [ "thiserror 2.0.18", ] +[[package]] +name = "reth-snap-sync" +version = "2.5.2" +dependencies = [ + "alloy-consensus", + "alloy-eip7928", + "alloy-eips", + "alloy-primitives", + "reth-primitives-traits", + "reth-provider", + "reth-storage-api", + "reth-storage-errors", + "thiserror 2.0.18", + "tokio-util", +] + [[package]] name = "reth-stages" version = "2.5.2" diff --git a/Cargo.toml b/Cargo.toml index 282a2ac8faa..e511ef9442d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -91,6 +91,7 @@ members = [ "crates/rpc/rpc-e2e-tests/", "crates/rpc/rpc-convert/", "crates/rpc/rpc/", + "crates/snap-sync/", "crates/stages/api/", "crates/stages/stages/", "crates/stages/types/", @@ -409,6 +410,7 @@ reth-rpc-layer = { path = "crates/rpc/rpc-layer" } reth-rpc-server-types = { path = "crates/rpc/rpc-server-types" } reth-rpc-convert = { path = "crates/rpc/rpc-convert" } reth-rpc-traits = { version = "0.7.0", default-features = false } +reth-snap-sync = { path = "crates/snap-sync" } reth-stages = { path = "crates/stages/stages" } reth-stages-api = { path = "crates/stages/api" } reth-stages-types = { path = "crates/stages/types", default-features = false } diff --git a/crates/snap-sync/Cargo.toml b/crates/snap-sync/Cargo.toml new file mode 100644 index 00000000000..0df1d88842d --- /dev/null +++ b/crates/snap-sync/Cargo.toml @@ -0,0 +1,31 @@ +[package] +name = "reth-snap-sync" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +homepage.workspace = true +repository.workspace = true +description = "snap/2 state synchronization" + +[lints] +workspace = true + +[dependencies] +# reth +reth-primitives-traits.workspace = true +reth-storage-api.workspace = true +reth-storage-errors.workspace = true + +# alloy +alloy-eip7928.workspace = true +alloy-eips.workspace = true +alloy-primitives.workspace = true + +# misc +thiserror.workspace = true +tokio-util.workspace = true + +[dev-dependencies] +alloy-consensus.workspace = true +reth-provider = { workspace = true, features = ["test-utils"] } diff --git a/crates/snap-sync/src/error.rs b/crates/snap-sync/src/error.rs new file mode 100644 index 00000000000..e78bd7a328d --- /dev/null +++ b/crates/snap-sync/src/error.rs @@ -0,0 +1,11 @@ +//! Failures raised while assembling a snap state generation. + +use reth_storage_errors::provider::ProviderError; + +/// Error returned while assembling a snap state generation. +#[derive(Debug, thiserror::Error)] +pub enum SnapSyncError { + /// A header lookup failed. + #[error(transparent)] + Provider(#[from] ProviderError), +} diff --git a/crates/snap-sync/src/generation.rs b/crates/snap-sync/src/generation.rs new file mode 100644 index 00000000000..defc67e34db --- /dev/null +++ b/crates/snap-sync/src/generation.rs @@ -0,0 +1,75 @@ +//! Identifies one attempt at downloading state, and how far it has progressed. + +use crate::SnapSyncError; +use alloy_eips::BlockNumHash; +use alloy_primitives::B256; +use reth_storage_api::HeaderProvider; + +/// One attempt at downloading state, anchored to the pivot block it targets. +/// +/// The anchor is kept as both a number-hash pair and a state root: the hash decides whether the +/// attempt is still on the canonical chain, and the root is what downloaded ranges authenticate +/// against. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct SnapGeneration { + // Pivot block this attempt is anchored to. + target: BlockNumHash, + // Root downloaded ranges authenticate against. + state_root: B256, + // Stage reached so far. + phase: SnapPhase, +} + +impl SnapGeneration { + /// Creates a generation anchored to the given pivot, before any range is downloaded. + pub const fn new(target: BlockNumHash, state_root: B256) -> Self { + Self { target, state_root, phase: SnapPhase::Accounts } + } + + /// Pivot block this generation is anchored to. + pub const fn target(&self) -> BlockNumHash { + self.target + } + + /// State root that downloaded ranges authenticate against. + pub const fn state_root(&self) -> B256 { + self.state_root + } + + /// Stage this generation has reached. + pub const fn phase(&self) -> SnapPhase { + self.phase + } + + /// Returns how far the canonical head has moved past this generation's anchor. + pub const fn lag(&self, head: u64) -> u64 { + head.saturating_sub(self.target.number) + } + + /// Returns whether the block this generation is anchored to is still canonical. + /// + /// Separate from whether it is worth finishing: an orphaned anchor is recoverable from the + /// abandoned branch's lists. + pub fn is_canonical(&self, provider: &impl HeaderProvider) -> Result { + let header = provider.sealed_header(self.target.number)?; + Ok(header.is_some_and(|header| header.hash() == self.target.hash)) + } + + /// Returns this generation moved to `phase`. + #[cfg(test)] + pub(crate) const fn with_phase(mut self, phase: SnapPhase) -> Self { + self.phase = phase; + self + } +} + +/// The stage a [`SnapGeneration`] has reached. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum SnapPhase { + /// Account, storage and bytecode ranges are being downloaded. + Accounts, + /// Authenticated block access lists are being applied. + BlockAccessLists, + /// The final state trie is being rebuilt and checked. + Trie, +} diff --git a/crates/snap-sync/src/lib.rs b/crates/snap-sync/src/lib.rs new file mode 100644 index 00000000000..b37157c2031 --- /dev/null +++ b/crates/snap-sync/src/lib.rs @@ -0,0 +1,44 @@ +//! snap/2 state synchronization for [EIP-8189](https://eips.ethereum.org/EIPS/eip-8189). +//! +//! Coordinates a state bootstrap that starts from a recent pivot block, downloads accounts, storage +//! and bytecode authenticated against that pivot's state root, and advances the pivot with +//! [EIP-7928 block access lists](https://eips.ethereum.org/EIPS/eip-7928) as the chain moves past +//! it. +//! +//! This crate owns download progress only. Authenticated downloads come from +//! `reth-downloaders`, and verified state is handed back to node integration once its trie root +//! matches the target header. +//! +//! ``` +//! use reth_snap_sync::SnapPivotPolicy; +//! +//! let policy = SnapPivotPolicy::default(); +//! // Without a finalized block, anchor at the EIP's example distance. +//! assert_eq!(policy.pivot_block(1_000, None), Some(936)); +//! // A recent finalized block is anchored to directly. +//! assert_eq!(policy.pivot_block(1_000, Some(950)), Some(950)); +//! // Stalled finality falls back to the example distance. +//! assert_eq!(policy.pivot_block(1_000, Some(500)), Some(936)); +//! // A chain shorter than the head distance has no pivot yet. +//! assert_eq!(policy.pivot_block(4, None), None); +//! ``` + +#![doc( + html_logo_url = "https://raw.githubusercontent.com/paradigmxyz/reth/main/assets/reth-docs.png", + html_favicon_url = "https://avatars0.githubusercontent.com/u/97369466?s=256", + issue_tracker_base_url = "https://github.com/paradigmxyz/reth/issues/" +)] +#![cfg_attr(not(test), warn(unused_crate_dependencies))] + +mod error; +mod generation; +mod pivot; +mod session; + +#[cfg(test)] +mod test_utils; + +pub use error::SnapSyncError; +pub use generation::{SnapGeneration, SnapPhase}; +pub use pivot::SnapPivotPolicy; +pub use session::{SnapSyncSession, SnapSyncSessionState}; diff --git a/crates/snap-sync/src/pivot.rs b/crates/snap-sync/src/pivot.rs new file mode 100644 index 00000000000..8409a498fdf --- /dev/null +++ b/crates/snap-sync/src/pivot.rs @@ -0,0 +1,256 @@ +//! Chooses the canonical block a snap generation is anchored to. +//! +//! [EIP-8189](https://eips.ethereum.org/EIPS/eip-8189#synchronization-algorithm) pivot selection +//! anchors synchronization at a block "sufficiently behind the chain head [...] to reduce the +//! likelihood of P being reorged while remaining recent enough that serving peers still hold its +//! state in memory". Those two pressures are what this policy balances: too close to the head and +//! the anchor is reorged, too far and no peer will serve its state. +//! +//! Two choices depart from the EIP's example: a finalized block is preferred as the anchor when one +//! is available, and re-anchoring starts once a pivot lags by 96 blocks rather than at the edge of +//! the window peers still serve state for. + +use crate::{SnapGeneration, SnapPhase, SnapSyncError}; +use alloy_eip7928::BAL_RETENTION_PERIOD_SLOTS; +use reth_primitives_traits::AlloyBlockHeader; +use reth_storage_api::HeaderProvider; + +// EIP-8189's example anchor, matching go-ethereum's `fsMinFullBlocks`. +const DEFAULT_HEAD_DISTANCE: u64 = 64; + +// Blocks of state history a serving peer is assumed to still hold, mirroring reth's own +// `SNAPSHOT_STATE_RETENTION`. +const SERVED_STATE_WINDOW: u64 = 128; + +// Re-anchor before the pivot reaches the edge of that window, so ranges in flight do not fail +// against a root peers just dropped. +const DEFAULT_ADVANCE_AFTER: u64 = SERVED_STATE_WINDOW - DEFAULT_HEAD_DISTANCE / 2; + +/// Distance and history bounds that decide where a generation is anchored. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct SnapPivotPolicy { + // Blocks behind the head to anchor at when no finalized block is available. + head_distance: u64, + // Pivot lag that triggers re-anchoring while ranges are still downloading. + advance_after: u64, + // Blocks of block access list history a peer is assumed to still serve. + history: u64, +} + +impl Default for SnapPivotPolicy { + fn default() -> Self { + Self { + head_distance: DEFAULT_HEAD_DISTANCE, + advance_after: DEFAULT_ADVANCE_AFTER, + history: BAL_RETENTION_PERIOD_SLOTS, + } + } +} + +impl SnapPivotPolicy { + /// Returns this policy anchoring `head_distance` blocks behind the head. + pub const fn with_head_distance(mut self, head_distance: u64) -> Self { + self.head_distance = head_distance; + self + } + + /// Returns this policy re-anchoring once a pivot lags by `advance_after` blocks. + pub const fn with_advance_after(mut self, advance_after: u64) -> Self { + self.advance_after = advance_after; + self + } + + /// Returns this policy assuming `history` blocks of block access lists remain servable. + /// + /// Defaults to the full EIP-7928 retention period, since applying lists beats downloading the + /// state again. + pub const fn with_history(mut self, history: u64) -> Self { + self.history = history; + self + } + + /// Returns the block a pivot anchored under `head` targets. + /// + /// Prefers a finalized block that peers still serve state for, since it cannot be reorged, and + /// falls back to the head distance. + pub const fn pivot_block(&self, head: u64, finalized: Option) -> Option { + if let Some(finalized) = finalized && + head.saturating_sub(finalized) <= self.advance_after + { + return Some(finalized) + } + head.checked_sub(self.head_distance) + } + + /// Returns whether `generation` should be re-anchored under `head`. + /// + /// Advancing stays far cheaper than restarting, so this triggers well before peers stop + /// serving the old root. + pub const fn needs_advance(&self, generation: SnapGeneration, head: u64) -> bool { + generation.lag(head) > self.advance_after + } + + /// Returns whether the block access lists `generation` still needs remain servable. + /// + /// Once they are not, its state cannot be carried forward and the attempt has to restart. + pub const fn is_catchable(&self, generation: SnapGeneration, head: u64) -> bool { + generation.lag(head) <= self.history + } + + /// Returns a fresh generation for the canonical pivot under `head`. + /// + /// A candidate that is not eligible falls back to the head distance; `None` means no candidate + /// can anchor a sync yet. + pub fn select( + &self, + provider: &impl HeaderProvider, + head: u64, + finalized: Option, + ) -> Result, SnapSyncError> { + let preferred = self.pivot_block(head, finalized); + let fallback = + head.checked_sub(self.head_distance).filter(|block| Some(*block) != preferred); + for block_number in preferred.into_iter().chain(fallback) { + let Some(header) = provider.sealed_header(block_number)? else { continue }; + if header.block_access_list_hash().is_some() { + return Ok(Some(SnapGeneration::new(header.num_hash(), header.state_root()))) + } + } + Ok(None) + } + + /// Returns whether an interrupted generation is still worth finishing under `head`. + /// + /// A fully downloaded generation only needs its trie rebuilt, so it always is. + pub const fn is_finishable(&self, generation: SnapGeneration, head: u64) -> bool { + matches!(generation.phase(), SnapPhase::Trie) || self.is_catchable(generation, head) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::test_utils::{chain, policy, provider_with}; + use alloy_eips::BlockNumHash; + use alloy_primitives::B256; + + #[test] + fn selects_the_bal_capable_pivot_behind_the_head() { + let headers = chain(Some(0)); + let expected = headers[2].clone(); + let provider = provider_with(headers); + + let generation = policy().select(&provider, 3, None).unwrap().unwrap(); + + assert_eq!(generation.target().number, 2); + assert_eq!(generation.target().hash, expected.hash_slow()); + assert_eq!(generation.state_root(), expected.state_root); + assert_eq!(generation.phase(), SnapPhase::Accounts); + } + + #[test] + fn a_recent_finalized_block_is_anchored_to_instead_of_the_head_distance() { + let headers = chain(Some(0)); + let expected = headers[1].clone(); + let provider = provider_with(headers); + + let generation = policy().select(&provider, 3, Some(1)).unwrap().unwrap(); + + assert_eq!(generation.target().number, 1); + assert_eq!(generation.target().hash, expected.hash_slow()); + } + + #[test] + fn finality_stalled_outside_the_advance_window_falls_back_to_the_head_distance() { + let headers = chain(Some(0)); + let fallback = headers[2].clone(); + let provider = provider_with(headers); + // A finalized block two behind the head is outside this policy's advance window. + let policy = policy().with_advance_after(1); + + let generation = policy.select(&provider, 3, Some(1)).unwrap().unwrap(); + + // HEAD-1, not the stale finalized block 1. + assert_eq!(generation.target().number, 2); + assert_eq!(generation.target().hash, fallback.hash_slow()); + } + + #[test] + fn an_ineligible_finalized_pivot_falls_back_to_the_head_distance() { + // Block access lists only start at block 2, so the finalized block predates activation. + let headers = chain(Some(2)); + let fallback = headers[2].clone(); + let provider = provider_with(headers); + + let generation = policy().select(&provider, 3, Some(1)).unwrap().unwrap(); + + // HEAD-1, rather than waiting for finality to reach activation. + assert_eq!(generation.target().number, 2); + assert_eq!(generation.target().hash, fallback.hash_slow()); + } + + #[test] + fn pivot_without_a_bal_commitment_is_not_selectable() { + let provider = provider_with(chain(Some(3))); + + assert_eq!(policy().select(&provider, 3, None).unwrap(), None); + // Neither the finalized anchor nor the fallback carries a commitment. + assert_eq!(policy().select(&provider, 3, Some(1)).unwrap(), None); + } + + #[test] + fn pivot_beyond_downloaded_headers_is_not_selectable() { + let provider = provider_with(chain(Some(0))); + + assert_eq!(policy().select(&provider, 9, None).unwrap(), None); + } + + #[test] + fn chain_shorter_than_the_head_distance_has_no_pivot() { + let provider = provider_with(chain(Some(0))); + + assert_eq!(policy().with_head_distance(4).select(&provider, 0, None).unwrap(), None); + } + + #[test] + fn a_pivot_lagging_past_the_advance_window_is_re_anchored() { + let policy = policy(); + let generation = SnapGeneration::new(BlockNumHash::new(0, B256::ZERO), B256::ZERO); + + assert!(!policy.needs_advance(generation, 4)); + assert!(policy.needs_advance(generation, 5)); + } + + #[test] + fn generation_outside_the_bal_window_is_not_finishable() { + let headers = chain(Some(0)); + let anchor = headers[1].clone(); + let provider = provider_with(headers); + let generation = + SnapGeneration::new(BlockNumHash::new(1, anchor.hash_slow()), anchor.state_root); + let policy = policy(); + + assert!(generation.is_canonical(&provider).unwrap()); + assert!(policy.is_finishable(generation, 9)); + assert!(!policy.is_finishable(generation, 10)); + } + + #[test] + fn downloaded_state_finishes_outside_the_bal_window() { + let anchor = chain(Some(0))[1].clone(); + let generation = + SnapGeneration::new(BlockNumHash::new(1, anchor.hash_slow()), anchor.state_root) + .with_phase(SnapPhase::Trie); + + assert!(policy().is_finishable(generation, 1_000)); + } + + #[test] + fn reorged_anchor_is_not_canonical() { + let provider = provider_with(chain(Some(0))); + let generation = + SnapGeneration::new(BlockNumHash::new(1, B256::repeat_byte(0xff)), B256::ZERO); + + assert!(!generation.is_canonical(&provider).unwrap()); + } +} diff --git a/crates/snap-sync/src/session.rs b/crates/snap-sync/src/session.rs new file mode 100644 index 00000000000..9a51d486901 --- /dev/null +++ b/crates/snap-sync/src/session.rs @@ -0,0 +1,213 @@ +//! Drives one snap synchronization attempt: what it targets, and when it stops. + +use crate::{SnapGeneration, SnapPivotPolicy, SnapSyncError}; +use reth_storage_api::HeaderProvider; +use tokio_util::sync::CancellationToken; + +/// One snap synchronization attempt, from the pivot it targets to the work it owns. +/// +/// Targets come from the local chain, never from a peer: peers only supply state, which is +/// authenticated against the target's state root. +#[derive(Debug)] +pub struct SnapSyncSession { + // Decides which blocks are eligible targets. + policy: SnapPivotPolicy, + // How far the attempt has got. + state: SnapSyncSessionState, + // Cancelled once, watched by whatever took the target. + cancellation: CancellationToken, +} + +impl SnapSyncSession { + /// Creates a session waiting for its first eligible target. + pub fn new(policy: SnapPivotPolicy) -> Self { + Self { + policy, + state: SnapSyncSessionState::Waiting, + cancellation: CancellationToken::new(), + } + } + + /// What the session is doing. + pub const fn state(&self) -> &SnapSyncSessionState { + &self.state + } + + /// Pivot this session is anchored to, if it has one. + pub const fn target(&self) -> Option<&SnapGeneration> { + self.state.target() + } + + /// Returns whether this session has been cancelled. + pub fn is_cancelled(&self) -> bool { + self.cancellation.is_cancelled() + } + + /// Selects a target under `head`, or waits while no block is eligible. + /// + /// A target no work has taken yet is replaced by a newer eligible one and dropped when none is + /// eligible, since nothing authenticates against its root so far. Moving a target that work + /// has taken is instead pivot advancement, which has to carry the downloaded state forward. + pub fn select( + &mut self, + provider: &impl HeaderProvider, + head: u64, + finalized: Option, + ) -> Result<&SnapSyncSessionState, SnapSyncError> { + if matches!(self.state, SnapSyncSessionState::Waiting | SnapSyncSessionState::Selected(_)) { + self.state = match self.policy.select(provider, head, finalized)? { + Some(generation) => SnapSyncSessionState::Selected(generation), + None => SnapSyncSessionState::Waiting, + }; + } + Ok(&self.state) + } + + /// Hands the selected target, and the token to watch, to the work downloading against it. + /// + /// Taking the target is what starts it, so only the first caller gets one: a target already + /// being downloaded has an owner, and a waiting or cancelled session has nothing to hand out. + pub fn start(&mut self) -> Option<(SnapGeneration, CancellationToken)> { + let SnapSyncSessionState::Selected(generation) = self.state else { return None }; + self.state = SnapSyncSessionState::Downloading(generation); + Some((generation, self.cancellation.clone())) + } + + /// Signals outstanding work to stop and ends the session. + /// + /// Terminal: a later attempt needs a new session. + pub fn cancel(&mut self) { + self.cancellation.cancel(); + self.state = SnapSyncSessionState::Cancelled; + } +} + +/// What a [`SnapSyncSession`] is doing. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum SnapSyncSessionState { + /// No block is eligible yet, so the session holds no target. + Waiting, + /// A target is selected, but no work has taken it yet. + Selected(SnapGeneration), + /// Work is outstanding against the target. + Downloading(SnapGeneration), + /// The session was cancelled and its outstanding work signalled to stop. + Cancelled, +} + +impl SnapSyncSessionState { + /// Target of this state, if it has one. + pub const fn target(&self) -> Option<&SnapGeneration> { + match self { + Self::Selected(generation) | Self::Downloading(generation) => Some(generation), + Self::Waiting | Self::Cancelled => None, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::test_utils::{chain, policy, provider_with}; + + fn session() -> SnapSyncSession { + SnapSyncSession::new(policy()) + } + + #[test] + fn waits_while_no_block_is_eligible() { + // The only block access list commitment is at the head, past the head distance. + let provider = provider_with(chain(Some(3))); + let mut session = session(); + + assert_eq!(session.select(&provider, 3, None).unwrap(), &SnapSyncSessionState::Waiting); + assert_eq!(session.target(), None); + assert!(session.start().is_none()); + } + + #[test] + fn selects_an_eligible_target() { + let headers = chain(Some(0)); + let expected = headers[2].clone(); + let provider = provider_with(headers); + let mut session = session(); + + session.select(&provider, 3, None).unwrap(); + + let target = session.target().unwrap(); + assert_eq!(target.target().number, 2); + assert_eq!(target.target().hash, expected.hash_slow()); + } + + #[test] + fn a_target_no_work_took_is_replaced_as_the_head_advances() { + let provider = provider_with(chain(Some(0))); + let mut session = session(); + + session.select(&provider, 2, None).unwrap(); + assert_eq!(session.target().unwrap().target().number, 1); + + session.select(&provider, 3, None).unwrap(); + assert_eq!(session.target().unwrap().target().number, 2); + } + + #[test] + fn a_target_no_work_took_is_dropped_once_it_is_no_longer_eligible() { + let provider = provider_with(chain(Some(0))); + let mut session = session(); + session.select(&provider, 3, None).unwrap(); + + // The head has run past the downloaded headers, so no candidate confirms the old target. + session.select(&provider, 9, None).unwrap(); + + assert_eq!(session.state(), &SnapSyncSessionState::Waiting); + assert!(session.start().is_none()); + } + + #[test] + fn a_target_work_took_is_kept() { + let provider = provider_with(chain(Some(0))); + let mut session = session(); + session.select(&provider, 2, None).unwrap(); + let (started, _) = session.start().unwrap(); + + session.select(&provider, 3, None).unwrap(); + + assert_eq!(session.state(), &SnapSyncSessionState::Downloading(started)); + } + + #[test] + fn only_one_worker_takes_a_target() { + let provider = provider_with(chain(Some(0))); + let mut session = session(); + session.select(&provider, 3, None).unwrap(); + + assert!(session.start().is_some()); + assert!(session.start().is_none()); + } + + #[test] + fn cancellation_stops_outstanding_work() { + let provider = provider_with(chain(Some(0))); + let mut session = session(); + session.select(&provider, 3, None).unwrap(); + let (_, outstanding) = session.start().unwrap(); + + session.cancel(); + + assert!(outstanding.is_cancelled()); + assert!(session.is_cancelled()); + assert_eq!(session.state(), &SnapSyncSessionState::Cancelled); + assert_eq!(session.target(), None); + } + + #[test] + fn a_cancelled_session_selects_nothing() { + let provider = provider_with(chain(Some(0))); + let mut session = session(); + session.cancel(); + + assert_eq!(session.select(&provider, 3, None).unwrap(), &SnapSyncSessionState::Cancelled); + assert!(session.start().is_none()); + } +} diff --git a/crates/snap-sync/src/test_utils.rs b/crates/snap-sync/src/test_utils.rs new file mode 100644 index 00000000000..7df948657c4 --- /dev/null +++ b/crates/snap-sync/src/test_utils.rs @@ -0,0 +1,46 @@ +//! Header fixtures shared by the crate's tests. + +use crate::SnapPivotPolicy; +use alloy_consensus::Header; +use alloy_primitives::B256; +use reth_provider::test_utils::MockEthProvider; + +/// Small bounds keep header fixtures short without changing the policy's decisions. +pub(crate) fn policy() -> SnapPivotPolicy { + SnapPivotPolicy::default().with_head_distance(1).with_advance_after(4).with_history(8) +} + +/// A header with a state root distinctive to its number, and a commitment when one is given. +pub(crate) fn header( + number: u64, + parent_hash: B256, + block_access_list_hash: Option, +) -> Header { + Header { + number, + parent_hash, + state_root: B256::repeat_byte(number as u8), + block_access_list_hash, + ..Default::default() + } +} + +/// Blocks `0..=3`, carrying a block access list commitment from `bal_from` onwards. +pub(crate) fn chain(bal_from: Option) -> Vec
{ + let mut headers = Vec::new(); + let mut parent = B256::ZERO; + for number in 0..=3 { + let bal = + bal_from.filter(|from| number >= *from).map(|_| B256::with_last_byte(number as u8)); + let header = header(number, parent, bal); + parent = header.hash_slow(); + headers.push(header); + } + headers +} + +pub(crate) fn provider_with(headers: impl IntoIterator) -> MockEthProvider { + let provider = MockEthProvider::default(); + provider.extend_headers(headers.into_iter().map(|header| (header.hash_slow(), header))); + provider +} From 7b882d315b035565a76023b1730fe90dd2790a26 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A9a=20Narzis?= <78718413+lean-apple@users.noreply.github.com> Date: Tue, 8 Sep 2026 07:25:30 +0000 Subject: [PATCH 201/294] refactor(engine): reuse alloy helpers for bal prewarming (#27052) Co-authored-by: Matthias Seitz --- Cargo.lock | 4 ++-- Cargo.toml | 2 +- .../src/tree/payload_processor/prewarm.rs | 19 ++++++------------- 3 files changed, 9 insertions(+), 16 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2e56824c40f..653a5fd9186 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -252,9 +252,9 @@ dependencies = [ [[package]] name = "alloy-eip7928" -version = "0.4.8" +version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fb59cca9e58f5c624becc3cffb32772dc278f31eeae4606844a88592e8d2672" +checksum = "fdd59d7221d384c46771ffbf6f18f70aa7753326351277857016bed5127b888f" dependencies = [ "alloy-primitives", "alloy-rlp", diff --git a/Cargo.toml b/Cargo.toml index e511ef9442d..6dee546ab1a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -444,7 +444,7 @@ alloy-sol-types = { version = "1.6.1", default-features = false } alloy-chains = { version = "0.2.33", default-features = false } alloy-eip2124 = { version = "0.2.0", default-features = false } -alloy-eip7928 = { version = "0.4.8", default-features = false, features = ["rlp"] } +alloy-eip7928 = { version = "0.4.9", default-features = false, features = ["rlp"] } alloy-evm = { version = "0.39.0", default-features = false } alloy-rlp = { version = "0.3.16", default-features = false, features = ["core-net"] } alloy-trie = { version = "0.9.4", default-features = false } diff --git a/crates/engine/tree/src/tree/payload_processor/prewarm.rs b/crates/engine/tree/src/tree/payload_processor/prewarm.rs index 0041615f4b6..a3a45623a17 100644 --- a/crates/engine/tree/src/tree/payload_processor/prewarm.rs +++ b/crates/engine/tree/src/tree/payload_processor/prewarm.rs @@ -431,14 +431,7 @@ where pool.begin_block(build, caches, ctx.env.txpool_snapshot.clone()); let dispatch_start = Instant::now(); for account in prefetch_bal.as_bal() { - pool.warm_account( - account.address, - account - .storage_changes - .iter() - .map(|change| change.slot.into()) - .chain(account.storage_reads.iter().map(|&slot| slot.into())), - ); + pool.warm_account(account.address, account.storage_slots().map(Into::into)); } ctx.metrics.bal_slot_iteration_duration.record(dispatch_start.elapsed()); pool.end_block(); @@ -775,13 +768,13 @@ struct BalAccountStateFields { impl BalAccountStateFields { fn from_changes(account_changes: &alloy_eip7928::AccountChanges) -> Self { Self { - balance: account_changes.balance_changes.last().map(|change| change.post_balance), - nonce: account_changes.nonce_changes.last().map(|change| change.new_nonce), - code_hash: account_changes.code_changes.last().map(|code_change| { - if code_change.new_code.is_empty() { + balance: account_changes.balance_post_state(), + nonce: account_changes.nonce_post_state(), + code_hash: account_changes.code_post_state().map(|code| { + if code.is_empty() { alloy_consensus::constants::KECCAK_EMPTY } else { - keccak256(&code_change.new_code) + keccak256(code) } }), } From 3d62abaa172092552adf04720dfa701968429b2c Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Tue, 8 Sep 2026 09:54:22 +0200 Subject: [PATCH 202/294] chore(deps): bump alloy to 2.4.2 (#27073) --- Cargo.lock | 126 +++++++++++++++++++++++++++-------------------------- Cargo.toml | 56 ++++++++++++------------ 2 files changed, 92 insertions(+), 90 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 653a5fd9186..322d23e806a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -121,9 +121,9 @@ dependencies = [ [[package]] name = "alloy-consensus" -version = "2.4.1" +version = "2.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12a542fe1e6a48cfd010a9e8eba1235436de58ee0f1415e588c27d5d556e98df" +checksum = "0b836b372d3b742b7c083d2617d6e68563cd5dadbd8752c461a935a756551e69" dependencies = [ "alloy-eips", "alloy-primitives", @@ -149,9 +149,9 @@ dependencies = [ [[package]] name = "alloy-consensus-any" -version = "2.4.1" +version = "2.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05ac1559726a5e71d6dbab59cbd8ad0988c762105d2ae41ae4f3241a3166262d" +checksum = "b1befb37485ea71e8599e32ffd76c98af7c3720a34741d30abd87f41c3148407" dependencies = [ "alloy-consensus", "alloy-eips", @@ -164,9 +164,9 @@ dependencies = [ [[package]] name = "alloy-contract" -version = "2.4.1" +version = "2.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b86722084d6d07822a6c67e39ae794d437fb9de12fa4127d4f9fd93f89651121" +checksum = "b432cf09e307820b39ec5b856b25f396d8fde6089358f0c1d96d9654ad1e0b7d" dependencies = [ "alloy-consensus", "alloy-dyn-abi", @@ -267,9 +267,9 @@ dependencies = [ [[package]] name = "alloy-eips" -version = "2.4.1" +version = "2.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d40246adf7468b430fce87945ece8a54ef27fa760dd9f670f5162a412007bdb2" +checksum = "ca3e76d47e8de277cb55e7e77fa38adde835162fc01cddbe1b258b5e0a86035b" dependencies = [ "alloy-eip2124", "alloy-eip2930", @@ -313,9 +313,9 @@ dependencies = [ [[package]] name = "alloy-genesis" -version = "2.4.1" +version = "2.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4951473e6ff25cdf82eb87b6a1a5fbb4af9c97528398a4a62fbf516198ec5028" +checksum = "0680a2f7d4bbc6f48c0cb43368fa544918a1fa291e772bdb47c834ade338f363" dependencies = [ "alloy-eips", "alloy-primitives", @@ -367,9 +367,9 @@ dependencies = [ [[package]] name = "alloy-json-rpc" -version = "2.4.1" +version = "2.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e5722ea29a9a85ecdba2516c58f50d71e77cd25f8ce7e9f4ee8570ddeee6df1" +checksum = "2a523ca087dda941e53f6ecde5bf5df78b9dbe760f3f77488e13dbf355de4a84" dependencies = [ "alloy-primitives", "alloy-sol-types", @@ -382,9 +382,9 @@ dependencies = [ [[package]] name = "alloy-network" -version = "2.4.1" +version = "2.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2e4d278913f00ab611f5367448a3a800fed173ab791c3cc67b7a1b0d7afd305" +checksum = "f263c8d752b15e19b7c2d3cbad00338ec2d7cc923a46cd1ff98aea2fa333166b" dependencies = [ "alloy-consensus", "alloy-consensus-any", @@ -408,9 +408,9 @@ dependencies = [ [[package]] name = "alloy-network-primitives" -version = "2.4.1" +version = "2.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e4e52ae56d1acaa6b46e9d7015b8ae6c10f3d1c213a32ae70b17426a03ae6729" +checksum = "8795eecf50374a0dd942caff57922c59333c9991ead9eccbbc1709f55d73c7c2" dependencies = [ "alloy-consensus", "alloy-eips", @@ -421,9 +421,9 @@ dependencies = [ [[package]] name = "alloy-node-bindings" -version = "2.4.1" +version = "2.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cdbb4a1bc03b411a3880d410ccc48d866b72d753ab1080605631890955b9caa2" +checksum = "3a58b30703aa2afe3b3ba66b0e9422393a9e869c6b84bff3b0a2f7e23f6fe5d1" dependencies = [ "alloy-genesis", "alloy-hardforks 0.2.13", @@ -475,9 +475,9 @@ dependencies = [ [[package]] name = "alloy-provider" -version = "2.4.1" +version = "2.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "386124639a5c386329b9f2ef2b42969badbf7bba40422b513e3e5739c516dc34" +checksum = "ef487e44dfae9c87cb732989d3f43682f65aefb11a0f56045b26be357f671fcf" dependencies = [ "alloy-chains", "alloy-consensus", @@ -504,6 +504,7 @@ dependencies = [ "either", "futures", "futures-utils-wasm", + "http", "lru", "parking_lot", "pin-project", @@ -519,9 +520,9 @@ dependencies = [ [[package]] name = "alloy-pubsub" -version = "2.4.1" +version = "2.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca2c32b323d8001b6e883444b7433804be0e6d55e4dbfd64e3bd2ba025282b77" +checksum = "18662bde9be7d5218bed6e6fcb472caac3a65ea31730654678b6bba7c1b0553a" dependencies = [ "alloy-json-rpc", "alloy-primitives", @@ -563,9 +564,9 @@ dependencies = [ [[package]] name = "alloy-rpc-client" -version = "2.4.1" +version = "2.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3dfd3ebc1756426a6a9e17979d4467632aff01f359136c50e5320ac6e32d6b8d" +checksum = "98267c2d47f9c185b575bac3f772a8ca66bcbd712693535e50980c12eab80a42" dependencies = [ "alloy-json-rpc", "alloy-primitives", @@ -589,9 +590,9 @@ dependencies = [ [[package]] name = "alloy-rpc-types" -version = "2.4.1" +version = "2.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "417dc664965e36fc43a4f9c8eb8b2d14067a29c430e23470d2c923c1c977bac7" +checksum = "637fed69eda26e19fb42314006570519003c1bfc338c54165e610a657e3b46ff" dependencies = [ "alloy-primitives", "alloy-rpc-types-engine", @@ -602,9 +603,9 @@ dependencies = [ [[package]] name = "alloy-rpc-types-admin" -version = "2.4.1" +version = "2.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8bfdc2a2cde178ad3a79594b4fabbc0f2f11be5a98a3363083e91575fc95335c" +checksum = "f91f1a1fa4d3c95ea5dba30c02d8987f0c052137dd7f1d55086cf36270517cbc" dependencies = [ "alloy-genesis", "alloy-primitives", @@ -614,9 +615,9 @@ dependencies = [ [[package]] name = "alloy-rpc-types-anvil" -version = "2.4.1" +version = "2.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a07bb602ee07767e393b5862f719a1d061deca15e617fbb1f7af49cd49576e4" +checksum = "848bfe2315338ec605863073aae54a90fcea9cec7e6855a9e93d988bf42e9740" dependencies = [ "alloy-primitives", "alloy-rpc-types-eth", @@ -626,9 +627,9 @@ dependencies = [ [[package]] name = "alloy-rpc-types-any" -version = "2.4.1" +version = "2.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd0a6cce3f97e5d9c0048081b5d173e0e406c2056c52c012e578a1d451ff0cdc" +checksum = "1176430031ba27abfb51b95516cf71e8ce3ff4172c72f3136627f0f89813a739" dependencies = [ "alloy-consensus-any", "alloy-network-primitives", @@ -641,9 +642,9 @@ dependencies = [ [[package]] name = "alloy-rpc-types-beacon" -version = "2.4.1" +version = "2.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "905e705ef91d2ed6cf08afbec0f9d79318175785eb146656441e8c644786ee5f" +checksum = "c20f226662436bbdc221c9156a1f445c257da5e695b39a26878aa8b2644f710e" dependencies = [ "alloy-eips", "alloy-primitives", @@ -661,9 +662,9 @@ dependencies = [ [[package]] name = "alloy-rpc-types-debug" -version = "2.4.1" +version = "2.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32f6d7c9462a3cf3bb046d0b870b86f1ce8c6892157491abebba62913e6f53cb" +checksum = "2a97960aa11b9984a18580b8dfbfc3eacfce1befa0d588b03ccc0bc5563d01f8" dependencies = [ "alloy-primitives", "alloy-rlp", @@ -674,9 +675,9 @@ dependencies = [ [[package]] name = "alloy-rpc-types-engine" -version = "2.4.1" +version = "2.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6e23b5864b3ed3479285ae9af1924266d60cac839dc559d50da868a0734c1d3" +checksum = "0a43d7b6b86be4885c7a76f1dfcde9651af2d419ff571e8ee7257de47101203b" dependencies = [ "alloy-consensus", "alloy-eips", @@ -695,9 +696,9 @@ dependencies = [ [[package]] name = "alloy-rpc-types-eth" -version = "2.4.1" +version = "2.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1eb4c87dfdde83cd97e03a8c0c5e986b52e14075e02266e322ac6c7eb02e7399" +checksum = "756eb5803e70985be4066b636194ccb5f6629e5d141817c1f0c50cea8350bf5c" dependencies = [ "alloy-consensus", "alloy-consensus-any", @@ -717,9 +718,9 @@ dependencies = [ [[package]] name = "alloy-rpc-types-mev" -version = "2.4.1" +version = "2.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e97bfdac1ec57f53ab2a0dacfedf382aeba1245ecb4ad7ecd8aeb4438a6c3b9b" +checksum = "c833a9b946d827d2b2aa8cad8e92279a4340ec1b7cf666f3e1ee6ef4684e0159" dependencies = [ "alloy-consensus", "alloy-eips", @@ -732,9 +733,9 @@ dependencies = [ [[package]] name = "alloy-rpc-types-trace" -version = "2.4.1" +version = "2.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d45ae3eadb97bf7933086205b4be0fef82c208bfc3007759e9afdcb6efa74c5" +checksum = "1933d60c2f51b8906103484ea2ca2218357264dbf16f948f7c2dd4ac5818e1d8" dependencies = [ "alloy-primitives", "alloy-rpc-types-eth", @@ -746,9 +747,9 @@ dependencies = [ [[package]] name = "alloy-rpc-types-txpool" -version = "2.4.1" +version = "2.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce0d7a9ab748d011daac12902ab9aedf8232d9b070bec6344cadd8509d8a6798" +checksum = "2eb5d708e45799bd963d7e3bd8d04110e90983833ed02f91a9099811ed39ce6d" dependencies = [ "alloy-primitives", "alloy-rpc-types-eth", @@ -758,9 +759,9 @@ dependencies = [ [[package]] name = "alloy-serde" -version = "2.4.1" +version = "2.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ff89f75b8a9d00f659750e9220c31b42d67c69ffc43b2ac8911dfd0506eb4a6" +checksum = "cc142e40df0c3eb5983b1637a68ab33471d8d22b8f402c9ed8260411a4e89d86" dependencies = [ "alloy-primitives", "arbitrary", @@ -770,9 +771,9 @@ dependencies = [ [[package]] name = "alloy-signer" -version = "2.4.1" +version = "2.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69e1f6fae3ade809daa4c32bee35a30a439bb307209d7b42f3995988b5cc74b8" +checksum = "b4c84f88f8a29d2607f98993687ed55a5943ba06819bc2126bb078dca54be943" dependencies = [ "alloy-primitives", "async-trait", @@ -785,9 +786,9 @@ dependencies = [ [[package]] name = "alloy-signer-local" -version = "2.4.1" +version = "2.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "621cc39c34b85875858ef149598f3d5f9797dff9a3247f3606fcc06f9b20acc8" +checksum = "58c81472fb0b73cbb18106e323e33e3f249f56318a330a8e0f6aa6e913e3519a" dependencies = [ "alloy-consensus", "alloy-network", @@ -874,9 +875,9 @@ dependencies = [ [[package]] name = "alloy-transport" -version = "2.4.1" +version = "2.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06f4ca8484d6295d5165f67de8fdb00ae630ba585efb606f003171286b56441c" +checksum = "6576826236d84590b037e333bfc3643ae0a90cb325ca9da565419102047b3b81" dependencies = [ "alloy-json-rpc", "auto_impl", @@ -897,9 +898,9 @@ dependencies = [ [[package]] name = "alloy-transport-http" -version = "2.4.1" +version = "2.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8342aa5a7f8dee6d606307eebbc847d9799a823c7e4f37b9c1e32b08715f73c8" +checksum = "e38c841f63141cad3f1a4378cedf121914b32932cf09b66c6006d4a906112136" dependencies = [ "alloy-json-rpc", "alloy-transport", @@ -913,9 +914,9 @@ dependencies = [ [[package]] name = "alloy-transport-ipc" -version = "2.4.1" +version = "2.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "184e03e1845edd5534f309d3169d93969a819ad1609108d9378fd33059de7547" +checksum = "03044aa44ec1da9468adf6a6e8c27d2bbaafd7a854ae14ac3e093b214139816d" dependencies = [ "alloy-json-rpc", "alloy-pubsub", @@ -923,6 +924,7 @@ dependencies = [ "bytes", "futures", "interprocess", + "memchr", "pin-project", "serde", "serde_json", @@ -933,9 +935,9 @@ dependencies = [ [[package]] name = "alloy-transport-ws" -version = "2.4.1" +version = "2.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee146fb5859e612e95570642a29def87c060c7ce47d14a0caa5ab7ee8570ec2c" +checksum = "55691c488ab3c88f664b6ffcaf9027f106b4642e6ff220502b6f645543a9efe8" dependencies = [ "alloy-pubsub", "alloy-transport", @@ -972,9 +974,9 @@ dependencies = [ [[package]] name = "alloy-tx-macros" -version = "2.4.1" +version = "2.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d0626c4f3b2028f7e8db32f53c22d9ecd5939f772317b7c4b6fe5219cb0589a" +checksum = "25484345e4d89c259235e200d4f895eba33e70f125f4e8bb37a7aa69a97bd656" dependencies = [ "darling 0.23.0", "proc-macro2", diff --git a/Cargo.toml b/Cargo.toml index 6dee546ab1a..80708157382 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -451,34 +451,34 @@ alloy-trie = { version = "0.9.4", default-features = false } alloy-hardforks = "0.4.8" -alloy-consensus = { version = "2.4.1", default-features = false } -alloy-contract = { version = "2.4.1", default-features = false } -alloy-eips = { version = "2.4.1", default-features = false } -alloy-genesis = { version = "2.4.1", default-features = false } -alloy-json-rpc = { version = "2.4.1", default-features = false } -alloy-network = { version = "2.4.1", default-features = false } -alloy-network-primitives = { version = "2.4.1", default-features = false } -alloy-node-bindings = "2.4.1" -alloy-provider = { version = "2.4.1", features = ["reqwest", "debug-api"], default-features = false } -alloy-pubsub = { version = "2.4.1", default-features = false } -alloy-rpc-client = { version = "2.4.1", default-features = false } -alloy-rpc-types = { version = "2.4.1", features = ["eth"], default-features = false } -alloy-rpc-types-admin = { version = "2.4.1", default-features = false } -alloy-rpc-types-anvil = { version = "2.4.1", default-features = false } -alloy-rpc-types-beacon = { version = "2.4.1", default-features = false } -alloy-rpc-types-debug = { version = "2.4.1", default-features = false } -alloy-rpc-types-engine = { version = "2.4.1", default-features = false } -alloy-rpc-types-eth = { version = "2.4.1", default-features = false } -alloy-rpc-types-mev = { version = "2.4.1", default-features = false } -alloy-rpc-types-trace = { version = "2.4.1", default-features = false } -alloy-rpc-types-txpool = { version = "2.4.1", default-features = false } -alloy-serde = { version = "2.4.1", default-features = false } -alloy-signer = { version = "2.4.1", default-features = false } -alloy-signer-local = { version = "2.4.1", default-features = false } -alloy-transport = { version = "2.4.1" } -alloy-transport-http = { version = "2.4.1", features = ["reqwest-rustls-tls"], default-features = false } -alloy-transport-ipc = { version = "2.4.1", default-features = false } -alloy-transport-ws = { version = "2.4.1", default-features = false } +alloy-consensus = { version = "2.4.2", default-features = false } +alloy-contract = { version = "2.4.2", default-features = false } +alloy-eips = { version = "2.4.2", default-features = false } +alloy-genesis = { version = "2.4.2", default-features = false } +alloy-json-rpc = { version = "2.4.2", default-features = false } +alloy-network = { version = "2.4.2", default-features = false } +alloy-network-primitives = { version = "2.4.2", default-features = false } +alloy-node-bindings = "2.4.2" +alloy-provider = { version = "2.4.2", features = ["reqwest", "debug-api"], default-features = false } +alloy-pubsub = { version = "2.4.2", default-features = false } +alloy-rpc-client = { version = "2.4.2", default-features = false } +alloy-rpc-types = { version = "2.4.2", features = ["eth"], default-features = false } +alloy-rpc-types-admin = { version = "2.4.2", default-features = false } +alloy-rpc-types-anvil = { version = "2.4.2", default-features = false } +alloy-rpc-types-beacon = { version = "2.4.2", default-features = false } +alloy-rpc-types-debug = { version = "2.4.2", default-features = false } +alloy-rpc-types-engine = { version = "2.4.2", default-features = false } +alloy-rpc-types-eth = { version = "2.4.2", default-features = false } +alloy-rpc-types-mev = { version = "2.4.2", default-features = false } +alloy-rpc-types-trace = { version = "2.4.2", default-features = false } +alloy-rpc-types-txpool = { version = "2.4.2", default-features = false } +alloy-serde = { version = "2.4.2", default-features = false } +alloy-signer = { version = "2.4.2", default-features = false } +alloy-signer-local = { version = "2.4.2", default-features = false } +alloy-transport = { version = "2.4.2" } +alloy-transport-http = { version = "2.4.2", features = ["reqwest-rustls-tls"], default-features = false } +alloy-transport-ipc = { version = "2.4.2", default-features = false } +alloy-transport-ws = { version = "2.4.2", default-features = false } # misc either = { version = "1.16.0", default-features = false } From c28aaffaf5a08eeadf559d0ce11d08b4aba3f2b3 Mon Sep 17 00:00:00 2001 From: Oliver Tipton Date: Tue, 8 Sep 2026 08:29:18 +0000 Subject: [PATCH 203/294] test(rpc): add non-empty V2 blob validation fixtures (#26997) Co-authored-by: Matthias Seitz --- crates/ethereum/node/tests/e2e/rpc.rs | 128 +++++++++++++++++++++++--- 1 file changed, 113 insertions(+), 15 deletions(-) diff --git a/crates/ethereum/node/tests/e2e/rpc.rs b/crates/ethereum/node/tests/e2e/rpc.rs index e0d20896221..b818fc88d0e 100644 --- a/crates/ethereum/node/tests/e2e/rpc.rs +++ b/crates/ethereum/node/tests/e2e/rpc.rs @@ -9,8 +9,8 @@ use alloy_provider::{ }; use alloy_rpc_types_beacon::relay::{ BidTrace, BuilderBlockValidationRequestV3, BuilderBlockValidationRequestV4, - BuilderBlockValidationRequestV6, SignedBidSubmissionV3, SignedBidSubmissionV4, - SignedBidSubmissionV6, + BuilderBlockValidationRequestV5, BuilderBlockValidationRequestV6, SignedBidSubmissionV3, + SignedBidSubmissionV4, SignedBidSubmissionV5, SignedBidSubmissionV6, }; use alloy_rpc_types_engine::{ BlobsBundleV1, CancunPayloadFields, ExecutionPayload, ExecutionPayloadSidecar, @@ -27,7 +27,10 @@ use alloy_rpc_types_trace::geth::{ use jsonrpsee::core::client::{ClientT, Subscription, SubscriptionClientT}; use rand::{rngs::StdRng, Rng, SeedableRng}; use reth_chainspec::{ChainSpecBuilder, EthChainSpec, MAINNET}; -use reth_e2e_test_utils::{setup_engine, wallet::Wallet, E2ETestSetupBuilder}; +use reth_e2e_test_utils::{ + setup_engine, transaction::TransactionTestContext, wallet::Wallet, E2ETestSetupBuilder, + NodeHelperType, +}; use reth_network::{types::NatResolver, PeersInfo}; use reth_node_builder::{NodeBuilder, NodeHandle}; use reth_node_core::{ @@ -60,6 +63,16 @@ alloy_sol_types::sol! { } } +async fn inject_blob_transaction( + node: &NodeHelperType, + wallet: &Wallet, +) -> eyre::Result<()> { + let blob_wallet = wallet.wallet_gen().swap_remove(0); + let blob_tx = TransactionTestContext::tx_with_blobs_bytes(1, blob_wallet).await?; + node.rpc.inject_tx(blob_tx).await?; + Ok(()) +} + #[tokio::test] async fn test_block_access_list_lookup_semantics() -> eyre::Result<()> { reth_tracing::init_test_tracing(); @@ -539,7 +552,77 @@ async fn test_flashbots_validate_v4() -> eyre::Result<()> { Ok(()) } -#[tokio::test] +// Keep blob JSON decoding off the test thread's deeper debug stack. +#[tokio::test(flavor = "multi_thread")] +async fn test_flashbots_validate_v5() -> eyre::Result<()> { + reth_tracing::init_test_tracing(); + + let chain_spec = Arc::new( + ChainSpecBuilder::default() + .chain(MAINNET.chain) + .genesis(serde_json::from_str(include_str!("../assets/genesis.json")).unwrap()) + .osaka_activated() + .build(), + ); + + let (mut nodes, wallet) = + E2ETestSetupBuilder::::new(1, chain_spec, eth_payload_attributes) + .with_node_config_modifier(|config| { + config.with_rpc( + RpcServerArgs::default() + .with_unused_ports() + .with_http() + .with_http_api(RpcModuleSelection::All) + .with_force_blob_sidecar_upcasting(), + ) + }) + .build() + .await?; + let mut node = nodes.pop().unwrap(); + let provider = ProviderBuilder::new() + .wallet(EthereumWallet::new(wallet.wallet_gen().swap_remove(0))) + .connect_http(node.rpc_url()); + + inject_blob_transaction(&node, &wallet).await?; + let payload = node.new_payload().await?; + let envelope = payload.clone().try_into_v5()?; + assert!(!envelope.blobs_bundle.blobs.is_empty()); + + let mut request = BuilderBlockValidationRequestV5 { + request: SignedBidSubmissionV5 { + message: BidTrace { + parent_hash: payload.block().parent_hash, + block_hash: payload.block().hash(), + gas_used: payload.block().gas_used, + gas_limit: payload.block().gas_limit, + ..Default::default() + }, + execution_payload: envelope.execution_payload, + blobs_bundle: envelope.blobs_bundle, + execution_requests: envelope.execution_requests.try_into().unwrap(), + signature: Default::default(), + }, + parent_beacon_block_root: payload.block().parent_beacon_block_root.unwrap(), + registered_gas_limit: payload.block().gas_limit, + }; + + provider + .raw_request::<_, ()>("flashbots_validateBuilderSubmissionV5".into(), (&request,)) + .await + .expect("request should validate"); + + request.request.blobs_bundle.proofs[0] = request.request.blobs_bundle.proofs[1]; + let err = provider + .raw_request::<_, ()>("flashbots_validateBuilderSubmissionV5".into(), (&request,)) + .await + .unwrap_err(); + assert!(err.to_string().contains("invalid KZG proof"), "{err}"); + + Ok(()) +} + +// Keep blob JSON decoding off the test thread's deeper debug stack. +#[tokio::test(flavor = "multi_thread")] async fn test_flashbots_validate_v6() -> eyre::Result<()> { reth_tracing::init_test_tracing(); @@ -551,30 +634,33 @@ async fn test_flashbots_validate_v6() -> eyre::Result<()> { .build(), ); - let (mut nodes, wallet) = setup_engine::( + let (mut nodes, wallet) = E2ETestSetupBuilder::::new( 1, - chain_spec.clone(), - false, - Default::default(), + chain_spec, eth_payload_attributes_amsterdam, ) + .with_node_config_modifier(|config| { + config.with_rpc( + RpcServerArgs::default() + .with_unused_ports() + .with_http() + .with_http_api(RpcModuleSelection::All) + .with_force_blob_sidecar_upcasting(), + ) + }) + .build() .await?; let mut node = nodes.pop().unwrap(); let provider = ProviderBuilder::new() .wallet(EthereumWallet::new(wallet.wallet_gen().swap_remove(0))) .connect_http(node.rpc_url()); - for nonce in 0..3 { - let _ = provider - .send_transaction(TransactionRequest::default().to(Address::ZERO).nonce(nonce)) - .await?; - } - + inject_blob_transaction(&node, &wallet).await?; let payload = node.new_payload().await?; assert!(payload.block_access_list().is_some()); - assert!(payload.block().body().transactions().count() >= 3); let envelope = payload.clone().try_into_v6()?; + assert!(!envelope.blobs_bundle.blobs.is_empty()); let mut request = BuilderBlockValidationRequestV6 { request: SignedBidSubmissionV6 { message: BidTrace { @@ -598,6 +684,18 @@ async fn test_flashbots_validate_v6() -> eyre::Result<()> { .await .expect("request should validate"); + let mut invalid_proof_request = request.clone(); + invalid_proof_request.request.blobs_bundle.proofs[0] = + invalid_proof_request.request.blobs_bundle.proofs[1]; + let err = provider + .raw_request::<_, ()>( + "flashbots_validateBuilderSubmissionV6".into(), + (&invalid_proof_request,), + ) + .await + .unwrap_err(); + assert!(err.to_string().contains("invalid KZG proof"), "{err}"); + request.registered_gas_limit -= 1; assert!(provider .raw_request::<_, ()>("flashbots_validateBuilderSubmissionV6".into(), (&request,)) From bb05b1d69821a1a19aa04c54a4eeeb6c1aa3fd51 Mon Sep 17 00:00:00 2001 From: Sergei Shulepov Date: Tue, 8 Sep 2026 08:30:51 +0000 Subject: [PATCH 204/294] fix(trie): reset cursors on root_node (#27019) --- crates/trie/trie/src/proof_v2/mod.rs | 51 ++++++++++++++++++++++++++-- 1 file changed, 48 insertions(+), 3 deletions(-) diff --git a/crates/trie/trie/src/proof_v2/mod.rs b/crates/trie/trie/src/proof_v2/mod.rs index 392af474cd6..178aa1060b9 100644 --- a/crates/trie/trie/src/proof_v2/mod.rs +++ b/crates/trie/trie/src/proof_v2/mod.rs @@ -1549,6 +1549,9 @@ where &mut self, value_encoder: &mut VE, ) -> Result { + self.trie_cursor.reset(); + self.hashed_cursor.reset(); + // Initialize the variables which track the state of the two cursors. Both indicate the // cursors are unseeked. let mut trie_cursor_state = TrieCursorState::unseeked(); @@ -1867,10 +1870,15 @@ enum PopCachedBranchOutcome { mod tests { use super::*; use crate::{ - hashed_cursor::{mock::MockHashedCursorFactory, HashedCursorFactory}, + hashed_cursor::{ + mock::MockHashedCursorFactory, noop::NoopHashedCursor, HashedCursorFactory, + HashedPostStateCursor, + }, proof::StorageProof as LegacyStorageProof, test_utils::TrieTestHarness, - trie_cursor::{depth_first, TrieCursorFactory}, + trie_cursor::{ + depth_first, noop::NoopStorageTrieCursor, InMemoryTrieCursor, TrieCursorFactory, + }, }; use alloy_primitives::map::B256Set; use alloy_rlp::Decodable; @@ -1878,7 +1886,9 @@ mod tests { use itertools::Itertools; use reth_trie_common::{ prefix_set::{PrefixSet, PrefixSetMut}, - ProofTrieNode, ProofV2TargetParent, TrieNode, EMPTY_ROOT_HASH, + updates::TrieUpdatesSorted, + HashedPostState, HashedStorage, ProofTrieNode, ProofV2TargetParent, TrieNode, + EMPTY_ROOT_HASH, }; use std::collections::BTreeMap; @@ -2129,6 +2139,41 @@ mod tests { pretty_assertions::assert_eq!(fresh_result, result); } + #[test] + fn test_root_node_reuse_with_overlay() { + let storage = BTreeMap::from([ + (B256::right_padding_from(&[0x10]), U256::from(1)), + (B256::right_padding_from(&[0x20]), U256::from(2)), + ]); + let harness = ProofTestHarness::new(storage.clone()); + let hashed_address = harness.hashed_address(); + let post_state = + HashedPostState::from_hashed_storage(hashed_address, HashedStorage::from_iter(storage)) + .into_sorted(); + let trie_updates = TrieUpdatesSorted::default(); + let trie_cursor = InMemoryTrieCursor::new_storage( + NoopStorageTrieCursor::default(), + &trie_updates, + hashed_address, + ); + let hashed_cursor = HashedPostStateCursor::new_storage( + NoopHashedCursor::default(), + &post_state, + hashed_address, + ); + let mut calculator = StorageProofCalculator::new_storage(trie_cursor, hashed_cursor); + + let first_root = calculator.root_node(&mut StorageValueEncoder).unwrap(); + assert!(matches!(first_root.node, TrieNodeV2::Branch(_))); + assert_eq!( + calculator.compute_root_hash(core::slice::from_ref(&first_root)).unwrap(), + Some(harness.original_root()) + ); + + let second_root = calculator.root_node(&mut StorageValueEncoder).unwrap(); + pretty_assertions::assert_eq!(first_root, second_root); + } + #[test] fn test_partial_storage_proof_after_root_calculation() { let slot_a = B256::right_padding_from(&[0xae, 0xd4, 0x00]); From a01f93dee0c9e984a144fb28e10e8022164268c4 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Tue, 8 Sep 2026 08:50:13 +0000 Subject: [PATCH 205/294] perf(trie): evaluate should_retain once per committed proof child (#26973) Co-authored-by: Brian Picciano --- crates/trie/trie/src/proof_v2/mod.rs | 82 ++++++++++++---------------- 1 file changed, 35 insertions(+), 47 deletions(-) diff --git a/crates/trie/trie/src/proof_v2/mod.rs b/crates/trie/trie/src/proof_v2/mod.rs index 178aa1060b9..f3dc4a0f61e 100644 --- a/crates/trie/trie/src/proof_v2/mod.rs +++ b/crates/trie/trie/src/proof_v2/mod.rs @@ -65,12 +65,12 @@ pub struct ProofCalculator { /// and so on. When a branch is removed from `branch_stack` its children are removed from this /// one, and the branch is pushed onto this stack in their place (see [`Self::pop_branch`]. /// - /// Children on the `child_stack` are converted to [`ProofTrieBranchChild::RlpNode`]s via the - /// [`Self::commit_child`] method. Committing a child indicates that no further changes are - /// expected to happen to it (e.g. splitting its short key when inserting a new branch). Given - /// that keys are consumed in lexicographical order, only the last child on the stack can - /// ever be modified, and therefore all children besides the last are expected to be - /// [`ProofTrieBranchChild::RlpNode`]s. + /// Given that keys are consumed in lexicographical order, only the last child on the stack + /// can still require structural changes (e.g. splitting its short key when inserting a new + /// branch). Once a child is finalized, [`Self::commit_last_child`] converts it to a + /// [`ProofTrieBranchChild::RlpNode`] if retained for the proof. Unretained children can remain + /// unencoded anywhere on the stack until [`Self::pop_branch`], giving deferred encoders more + /// time for async work. child_stack: Vec>, /// Cached branch data pulled from the `trie_cursor`. The calculator will use the cached /// [`BranchNodeCompact::hashes`] to skip over the calculation of sub-tries in the overall @@ -253,13 +253,15 @@ where } /// Takes a child which has been removed from the `child_stack` and converts it to an - /// [`RlpNode`]. + /// [`RlpNode`], retaining it as a proof node. /// - /// Calling this method indicates that the child will not undergo any further modifications, and - /// therefore can be retained as a proof node if applicable. - fn commit_child<'a>( + /// Calling this method indicates that the child will not undergo any further modifications. + /// + /// The caller has already determined via [`Self::should_retain`] that this child is retained. + /// That decision must not be re-evaluated here, because `should_retain` advances the `targets` + /// cursor. Children which are not retained are converted later, in [`Self::pop_branch`]. + fn commit_child( &mut self, - targets: &mut Option>, child_path: Nibbles, child: ProofTrieBranchChild, ) -> Result { @@ -269,38 +271,22 @@ where return child.into_rlp(&mut self.rlp_encode_buf).map(|(node, _)| node) } - // If we should retain the child then do so. - if self.should_retain(targets, &child_path, true) { - trace!(target: TRACE_TARGET, ?child_path, "Retaining child"); + trace!(target: TRACE_TARGET, ?child_path, "Retaining child"); - // Convert to `ProofTrieNodeV2`, which will be what is retained. - // - // If this node is a branch then its `rlp_nodes_buf` will be taken and not returned to - // the `rlp_nodes_bufs` free-list. - self.rlp_encode_buf.clear(); - let proof_node = child.into_proof_trie_node(child_path, &mut self.rlp_encode_buf)?; - - // Use the `ProofTrieNodeV2` to encode the `RlpNode`, and then push it onto retained - // nodes before returning. - self.rlp_encode_buf.clear(); - proof_node.node.encode(&mut self.rlp_encode_buf); - - self.retained_proofs.push(proof_node); - return Ok(RlpNode::from_rlp(&self.rlp_encode_buf)); - } - - // If the child path is not being retained then we convert directly to an `RlpNode` - // using `into_rlp`. Since we are not retaining the node we can recover any `RlpNode` - // buffers for the free-list here, hence why we do this as a separate logical branch. + // Convert to `ProofTrieNodeV2`, which will be what is retained. + // + // If this node is a branch then its `rlp_nodes_buf` will be taken and not returned to the + // `rlp_nodes_bufs` free-list. self.rlp_encode_buf.clear(); - let (child_rlp_node, freed_rlp_nodes_buf) = child.into_rlp(&mut self.rlp_encode_buf)?; + let proof_node = child.into_proof_trie_node(child_path, &mut self.rlp_encode_buf)?; - // If there is an `RlpNode` buffer which can be re-used then push it onto the free-list. - if let Some(buf) = freed_rlp_nodes_buf { - self.rlp_nodes_bufs.push(buf); - } + // Use the `ProofTrieNodeV2` to encode the `RlpNode`, and then push it onto retained nodes + // before returning. + self.rlp_encode_buf.clear(); + proof_node.node.encode(&mut self.rlp_encode_buf); - Ok(child_rlp_node) + self.retained_proofs.push(proof_node); + Ok(RlpNode::from_rlp(&self.rlp_encode_buf)) } /// Returns the path of the child of the currently under-construction branch at the given @@ -336,8 +322,9 @@ where .then(|| self.child_path_at(Self::highest_set_nibble(branch.state_mask))) } - /// Calls [`Self::commit_child`] on the last child of `child_stack`, replacing it with a - /// [`ProofTrieBranchChild::RlpNode`]. + /// If the last child of `child_stack` is retained for the proof, calls [`Self::commit_child`] + /// to replace it with a [`ProofTrieBranchChild::RlpNode`]. Otherwise, leaves it unencoded until + /// [`Self::pop_branch`]. /// /// If `child_stack` is empty then this is a no-op. /// @@ -368,7 +355,7 @@ where // to pop_branch() to give DeferredEncoder time for async work. if self.should_retain(targets, &child_path, true) { let (hash_mask_bit, tree_mask_bit) = child.mask_bits(); - let child_rlp_node = self.commit_child(targets, child_path, child)?; + let child_rlp_node = self.commit_child(child_path, child)?; trace!(target: TRACE_TARGET, ?child_rlp_node, "Pushing committed child RlpNode onto stack"); self.child_stack.push(ProofTrieBranchChild::RlpNode { node: child_rlp_node, @@ -448,14 +435,15 @@ where // Store the child key relative to its new parent branch. child.trim_short_key_prefix(path.len() - short_key.len()); - // Commit the preceding child before pushing this one, so only the final child on the - // stack can remain uncommitted. + // The preceding child is structurally final. Commit it if retained for the proof; + // otherwise its encoding can wait until pop_branch. self.commit_last_child(targets)?; let branch = self.branch_stack.last_mut().expect("branch_stack cannot be empty"); debug_assert!(!branch.state_mask.is_bit_set(nibble)); - // Mark the nibble occupied now that the preceding child has been committed. + // Set the new child's bit after commit_last_child, which uses the mask to find the + // preceding child's path. branch.state_mask.set_bit(nibble); // The parent now contains this child at `nibble`. @@ -548,8 +536,8 @@ where "called", ); - // Ensure the final child on the child stack has been committed, as this method expects all - // children of the branch to have been committed. + // Retain the final child if needed before draining the branch's children. Unretained + // children are encoded below. self.commit_last_child(targets)?; let mut rlp_nodes_buf = self.take_rlp_nodes_buf(); From 66508a6d3fd1b1ca33ee7b954643a4579829bd4a Mon Sep 17 00:00:00 2001 From: Brian Picciano Date: Tue, 8 Sep 2026 09:22:16 +0000 Subject: [PATCH 206/294] feat(overlay): support overlaying a pending block (#27056) --- crates/storage/storage-overlay/src/builder.rs | 51 +++++++- crates/storage/storage-overlay/src/manager.rs | 113 ++++++++++++++++-- 2 files changed, 148 insertions(+), 16 deletions(-) diff --git a/crates/storage/storage-overlay/src/builder.rs b/crates/storage/storage-overlay/src/builder.rs index bf6e7376468..44a1500dfd7 100644 --- a/crates/storage/storage-overlay/src/builder.rs +++ b/crates/storage/storage-overlay/src/builder.rs @@ -1,4 +1,4 @@ -use crate::OverlayManager; +use crate::{manager::OverlayCacheConfig, OverlayManager}; use alloy_eips::BlockNumHash; use alloy_primitives::{ map::{AddressMap, AddressSet, B256Map, U256Map}, @@ -225,6 +225,8 @@ pub struct OverlayBuilder { reused_sparse_trie_anchor_hash: Option, /// Whether building the overlay may query revert changesets. no_reverts: bool, + /// Cache behavior for the overlays this builder resolves. + overlay_cache_config: OverlayCacheConfig, /// Metrics for overlay construction. metrics: OverlayBuilderMetrics, } @@ -242,6 +244,7 @@ impl OverlayBuilder { parent_state, reused_sparse_trie_anchor_hash: None, no_reverts: false, + overlay_cache_config: OverlayCacheConfig::default(), metrics: OverlayBuilderMetrics::default(), } } @@ -262,6 +265,17 @@ impl OverlayBuilder { self } + /// Appends an executed block to this builder's in-memory parent state. + pub fn with_appended_block(mut self, block: ExecutedBlock) -> Self { + debug_assert_eq!(block.recovered_block().parent_hash(), self.parent_hash); + self.parent_hash = block.recovered_block().hash(); + self.parent_state = + Some(BlockState::with_parent(block, self.parent_state.take().map(Arc::new))); + self.reused_sparse_trie_anchor_hash = None; + self.overlay_cache_config.write_to_cache = false; + self + } + /// Returns the durable anchor to use for this builder's parent. #[cfg(test)] fn anchor_at_parent(&self, provider: &Provider) -> ProviderResult @@ -352,6 +366,12 @@ impl OverlayBuilder { return Ok(AnchorForParent::NoReverts { anchor }) } + // Reverts reconstruct canonical state by block number, so they cannot recover an anchor + // from another fork even if its number is covered by the available changesets. + if provider.block_hash(anchor.number)? != Some(anchor.hash) { + return Err(ProviderError::BlockHashNotFound(anchor.hash)) + } + // Otherwise reverts are required; we check the changesets to make sure they are actually // available before signaling that they are required. let account_history = provider @@ -633,7 +653,7 @@ impl OverlayBuilder { )) })?; self.overlay_manager - .overlay_for_parent(parent_state, anchor_hash) + .overlay_for_parent(parent_state, anchor_hash, self.overlay_cache_config) .map_err(ProviderError::other) } } @@ -650,7 +670,11 @@ impl OverlayBuilder { ProviderError::other(std::io::Error::other("missing in-memory parent state")) })?; self.overlay_manager - .execution_overlay_for_block_state(parent_state, anchor_hash) + .execution_overlay_for_block_state( + parent_state, + anchor_hash, + self.overlay_cache_config, + ) .map_err(ProviderError::other) } } @@ -1062,6 +1086,27 @@ mod tests { assert!(error.to_string().contains("reverts are disabled")); } + #[test] + fn appended_overlay_rejects_noncanonical_anchor_when_reverts_are_required() { + let (factory, blocks) = setup_frontiers(1, 3); + let provider = factory.provider().unwrap(); + let parent_hash = B256::with_last_byte(100); + assert_ne!(parent_hash, blocks[1].recovered_block().hash()); + let block = TestBlockBuilder::eth() + .get_executed_block_with_number(blocks[2].block_number(), parent_hash); + let builder = + OverlayManager::default().overlay_builder(parent_hash).with_appended_block(block); + + assert!(matches!( + builder.execution_overlay(&provider), + Err(ProviderError::BlockHashNotFound(hash)) if hash == parent_hash + )); + assert!(matches!( + builder.build_state_trie_overlay(&provider, true), + Err(ProviderError::BlockHashNotFound(hash)) if hash == parent_hash + )); + } + #[test] fn state_trie_overlay_uses_revert_prefix_sets_without_trie_changesets() { let (factory, blocks) = setup_frontiers(2, 3); diff --git a/crates/storage/storage-overlay/src/manager.rs b/crates/storage/storage-overlay/src/manager.rs index dfcef3f9508..e019486ab9d 100644 --- a/crates/storage/storage-overlay/src/manager.rs +++ b/crates/storage/storage-overlay/src/manager.rs @@ -355,6 +355,7 @@ impl OverlayManager { &self, parent_state: &BlockState, anchor_hash: B256, + cache_config: OverlayCacheConfig, ) -> Result<(Arc, Arc), StateTrieOverlayError> { let parent_hash = parent_state.hash(); if parent_hash == anchor_hash { @@ -375,7 +376,7 @@ impl OverlayManager { &self.metrics, anchor_hash, parent_state, - true, + cache_config, |input, span| self.compute_state_trie_overlay(input, anchor_hash, span), )? .expect("required overlay lookup cannot skip an in-progress computation"); @@ -393,9 +394,10 @@ impl OverlayManager { &self, parent_state: &BlockState, anchor_hash: B256, + cache_config: OverlayCacheConfig, ) -> Result, StateTrieOverlayError> { Ok(self - .execution_overlay_for_parent_inner(parent_state, anchor_hash, true)? + .execution_overlay_for_parent_inner(parent_state, anchor_hash, cache_config)? .expect("required overlay lookup cannot skip an in-progress computation")) } @@ -408,14 +410,19 @@ impl OverlayManager { let parent_state = self .block_state(parent_hash) .ok_or(StateTrieOverlayError { tip_hash: parent_hash, anchor_hash })?; - self.execution_overlay_for_parent_inner(&parent_state, anchor_hash, false).map(drop) + self.execution_overlay_for_parent_inner( + &parent_state, + anchor_hash, + OverlayCacheConfig { precompute: true, write_to_cache: true }, + ) + .map(drop) } fn execution_overlay_for_parent_inner( &self, parent_state: &BlockState, anchor_hash: B256, - wait_for_pending: bool, + cache_config: OverlayCacheConfig, ) -> Result>, StateTrieOverlayError> { let parent_hash = parent_state.hash(); if parent_hash == anchor_hash { @@ -427,7 +434,7 @@ impl OverlayManager { &self.execution_metrics, anchor_hash, parent_state, - wait_for_pending, + cache_config, |input, span| self.compute_execution_overlay(input, anchor_hash, span), ) } @@ -450,7 +457,7 @@ impl OverlayManager { metrics: &M, anchor_hash: B256, parent_state: &BlockState, - wait_for_pending: bool, + cache_config: OverlayCacheConfig, compute: impl FnOnce(ComputeOverlayInput, tracing::Span) -> T, ) -> Result>, StateTrieOverlayError> where @@ -464,8 +471,8 @@ impl OverlayManager { span.record("cache_reused", true); return match entry { OverlayCacheEntry::Ready(input) => Ok(Some(input)), - OverlayCacheEntry::Computing(waiter) if wait_for_pending => Ok(Some(waiter.wait())), - OverlayCacheEntry::Computing(_) => Ok(None), + OverlayCacheEntry::Computing(_) if cache_config.precompute => Ok(None), + OverlayCacheEntry::Computing(waiter) => Ok(Some(waiter.wait())), } } span.record("cache_reused", false); @@ -473,6 +480,24 @@ impl OverlayManager { // Resolve the block path and any cached parent overlay before locking the child entry. let mut blocks = Self::blocks_from_parent_state(parent_state, anchor_hash)?; span.record("block_count", blocks.len()); + + if !cache_config.write_to_cache { + let parent_input = blocks.first().and_then(|block| { + let parent_hash = block.recovered_block().parent_hash(); + (parent_hash != anchor_hash) + .then(|| cache.ready(&OverlayCacheKey { anchor_hash, tip_hash: parent_hash })) + .flatten() + }); + span.record("parent_overlay_reused", parent_input.is_some()); + let compute_input = match parent_input { + Some(parent_input) => { + ComputeOverlayInput::ExtendCached { block: blocks.swap_remove(0), parent_input } + } + None => ComputeOverlayInput::MergeBlocks(blocks), + }; + return Ok(Some(Arc::new(compute(compute_input, span)))) + } + enum CacheAction { Ready(Arc), Wait(Arc>), @@ -486,7 +511,7 @@ impl OverlayManager { span.record("cache_reused", true); match entry { OverlayCacheEntry::Ready(input) => CacheAction::Ready(input), - OverlayCacheEntry::Computing(_) if !wait_for_pending => return Ok(None), + OverlayCacheEntry::Computing(_) if cache_config.precompute => return Ok(None), OverlayCacheEntry::Computing(waiter) => CacheAction::Wait(waiter), } } @@ -631,6 +656,21 @@ impl OverlayManager { } } +/// Controls how an overlay computation interacts with the manager cache. +#[derive(Clone, Copy, Debug)] +pub(crate) struct OverlayCacheConfig { + /// Whether this is a best-effort cache fill that must not wait for an existing computation. + pub(crate) precompute: bool, + /// Whether to retain the computed overlay in the manager cache. + pub(crate) write_to_cache: bool, +} + +impl Default for OverlayCacheConfig { + fn default() -> Self { + Self { precompute: false, write_to_cache: true } + } +} + /// Error returned when a state trie overlay cannot be built from the manager's current block set. #[derive(Debug)] pub(crate) struct StateTrieOverlayError { @@ -683,6 +723,14 @@ impl OverlayCache { self.entries.retain(|key, entry| keep(key, entry)) } + /// Returns a ready entry without removing it from the cache. + fn ready(&self, key: &OverlayCacheKey) -> Option> { + self.entries.get(key).and_then(|entry| match entry.value() { + OverlayCacheEntry::Ready(input) => Some(Arc::clone(input)), + OverlayCacheEntry::Computing(_) => None, + }) + } + /// Removes and returns a ready entry. /// /// Transferring a parent entry lets `Arc::make_mut` extend it in place when no caller retains @@ -980,7 +1028,11 @@ mod tests { let parent_state = self .block_state(parent_hash) .ok_or(StateTrieOverlayError { tip_hash: parent_hash, anchor_hash })?; - self.execution_overlay_for_block_state(&parent_state, anchor_hash) + self.execution_overlay_for_block_state( + &parent_state, + anchor_hash, + OverlayCacheConfig::default(), + ) } } @@ -992,7 +1044,7 @@ mod tests { let parent_state = manager .block_state(parent_hash) .ok_or(StateTrieOverlayError { tip_hash: parent_hash, anchor_hash })?; - manager.overlay_for_parent(&parent_state, anchor_hash) + manager.overlay_for_parent(&parent_state, anchor_hash, OverlayCacheConfig::default()) } #[test] @@ -1150,6 +1202,40 @@ mod tests { .all(|account| account.account_id.is_none())); } + #[test] + fn does_not_cache_or_take_parent_overlays_for_unmanaged_blocks() { + let manager = OverlayManager::default(); + let blocks = test_blocks(); + for block in &blocks[..2] { + manager.insert_block(block.clone()); + } + + let anchor_hash = blocks[0].recovered_block().parent_hash(); + let parent_hash = blocks[1].recovered_block().hash(); + let child_hash = blocks[2].recovered_block().hash(); + let parent_key = OverlayCacheKey { anchor_hash, tip_hash: parent_hash }; + let child_key = OverlayCacheKey { anchor_hash, tip_hash: child_hash }; + let parent_state = manager.block_state(parent_hash).unwrap(); + let child_state = BlockState::with_parent(blocks[2].clone(), Some(Arc::new(parent_state))); + let cache_config = OverlayCacheConfig { precompute: false, write_to_cache: false }; + + overlay_for_parent(&manager, parent_hash, anchor_hash).unwrap(); + manager.execution_overlay_for_parent(parent_hash, anchor_hash).unwrap(); + + let (_, state) = + manager.overlay_for_parent(&child_state, anchor_hash, cache_config).unwrap(); + let execution = manager + .execution_overlay_for_block_state(&child_state, anchor_hash, cache_config) + .unwrap(); + + assert_eq!(state.accounts.len(), 3); + assert_eq!(execution.accounts().len(), 3); + assert!(manager.state_trie_overlays.entries.contains_key(&parent_key)); + assert!(!manager.state_trie_overlays.entries.contains_key(&child_key)); + assert!(manager.execution_overlays.entries.contains_key(&parent_key)); + assert!(!manager.execution_overlays.entries.contains_key(&child_key)); + } + #[cfg(feature = "rayon")] #[test] fn precomputes_execution_overlay_for_cached_parent() { @@ -1297,8 +1383,9 @@ mod tests { let (tx, rx) = mpsc::channel(); thread::spawn(move || { - let res = - manager.overlay_for_parent(&parent_state, key.anchor_hash).map(|(_, state)| state); + let res = manager + .overlay_for_parent(&parent_state, key.anchor_hash, OverlayCacheConfig::default()) + .map(|(_, state)| state); tx.send(res).unwrap(); }); From fb659d7c5ed3d3bb11f2be74cc74f176bb9532e1 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Tue, 8 Sep 2026 10:52:37 +0000 Subject: [PATCH 207/294] perf(trie): avoid staging drained storage addresses (#27080) Co-authored-by: Matthias Seitz <19890894+mattsse@users.noreply.github.com> --- .../tree/state_root_strategy/sparse_trie.rs | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/crates/engine/tree/src/tree/state_root_strategy/sparse_trie.rs b/crates/engine/tree/src/tree/state_root_strategy/sparse_trie.rs index 96528edb3f2..d1d0ee1b24a 100644 --- a/crates/engine/tree/src/tree/state_root_strategy/sparse_trie.rs +++ b/crates/engine/tree/src/tree/state_root_strategy/sparse_trie.rs @@ -713,24 +713,18 @@ where /// /// we trigger state root computation on a rayon pool. fn compute_drained_storage_roots(&mut self) { - let addresses_to_compute_roots: Vec<_> = self - .storage_updates - .iter() - .filter_map(|(address, updates)| updates.is_empty().then_some(*address)) - .collect(); - struct SendStorageTriePtr(*mut RevealableSparseTrie); // SAFETY: this wrapper only forwards the pointer across rayon; deref invariants are // documented at the use site below. unsafe impl Send for SendStorageTriePtr {} - let mut tries_to_compute_roots: Vec<(B256, SendStorageTriePtr)> = - Vec::with_capacity(addresses_to_compute_roots.len()); - for address in addresses_to_compute_roots { - if let Some(trie) = self.trie.storage_tries_mut().get_mut(&address) && + let mut tries_to_compute_roots: Vec<(B256, SendStorageTriePtr)> = Vec::new(); + for (address, updates) in &self.storage_updates { + if updates.is_empty() && + let Some(trie) = self.trie.storage_tries_mut().get_mut(address) && !trie.is_root_cached() { - tries_to_compute_roots.push((address, SendStorageTriePtr(trie))); + tries_to_compute_roots.push((*address, SendStorageTriePtr(trie))); } } @@ -759,7 +753,7 @@ where let _enter = span.entered(); // SAFETY: // - pointers are created from `storage_tries_mut().get_mut(address)` above; - // - `addresses_to_compute_roots` comes from map iteration, so addresses are unique; + // - `storage_updates` is a map, so addresses are unique; // - we do not insert/remove entries between pointer collection and use, so pointers // stay valid and map reallocation cannot occur; // - each pointer is consumed by at most one rayon task, so no aliasing mutable access. From 8be978802474e643d1a154c184da3deaeb8175f5 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Tue, 8 Sep 2026 11:49:08 +0000 Subject: [PATCH 208/294] perf(engine): reserve canonical BAL account capacity (#27079) Co-authored-by: Matthias Seitz <19890894+mattsse@users.noreply.github.com> --- .../engine/tree/src/tree/payload_processor/bal/execute.rs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/crates/engine/tree/src/tree/payload_processor/bal/execute.rs b/crates/engine/tree/src/tree/payload_processor/bal/execute.rs index 4f69147d24b..aa88f7c7510 100644 --- a/crates/engine/tree/src/tree/payload_processor/bal/execute.rs +++ b/crates/engine/tree/src/tree/payload_processor/bal/execute.rs @@ -117,6 +117,13 @@ where .with_bundle_update() .with_bal_builder() .build(); + canonical_state + .bal_state + .bal_builder + .as_mut() + .expect("with_bal_builder set") + .accounts + .reserve(bal.len()); let (block_result, senders) = { let (result_tx, result_rx) = crossbeam_channel::unbounded(); From a8e0ae87961f759a1a0b78c82324156e53c3f4be Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Tue, 8 Sep 2026 13:19:19 +0000 Subject: [PATCH 209/294] perf(engine): skip unused pool cleanup on BAL path (#27074) Co-authored-by: Matthias Seitz <19890894+mattsse@users.noreply.github.com> --- crates/engine/tree/src/tree/payload_processor/prewarm.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/crates/engine/tree/src/tree/payload_processor/prewarm.rs b/crates/engine/tree/src/tree/payload_processor/prewarm.rs index a3a45623a17..003697d4c64 100644 --- a/crates/engine/tree/src/tree/payload_processor/prewarm.rs +++ b/crates/engine/tree/src/tree/payload_processor/prewarm.rs @@ -443,7 +443,6 @@ where // Drop the per-thread providers executor.bal_streaming_pool().clear(); - executor.prewarming_pool().clear(); let _ = actions_tx.send(PrewarmTaskEvent::FinishedTxExecution { executed_transactions: 0 }); } From 2e812318c81253449fe5cd417476eefd399f25c9 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Tue, 8 Sep 2026 15:47:07 +0200 Subject: [PATCH 210/294] fix(test): stabilize gossip disconnect and testing RPC tests (#27087) --- crates/ethereum/node/tests/it/testing.rs | 6 ++-- crates/net/network/tests/it/txgossip.rs | 36 +++++++++--------------- 2 files changed, 18 insertions(+), 24 deletions(-) diff --git a/crates/ethereum/node/tests/it/testing.rs b/crates/ethereum/node/tests/it/testing.rs index c86080ab403..6ddc532ab6c 100644 --- a/crates/ethereum/node/tests/it/testing.rs +++ b/crates/ethereum/node/tests/it/testing.rs @@ -24,7 +24,8 @@ use tokio::sync::oneshot; #[tokio::test(flavor = "multi_thread")] async fn testing_rpc_build_block_works() -> eyre::Result<()> { let runtime = Runtime::test(); - let mut rpc_args = reth_node_core::args::RpcServerArgs::default().with_http(); + let mut rpc_args = + reth_node_core::args::RpcServerArgs::default().with_http().with_unused_ports(); rpc_args.http_api = Some(RpcModuleSelection::from_iter([RethRpcModule::Testing])); let tempdir = tempdir().expect("temp datadir"); let datadir_args = DatadirArgs { @@ -95,7 +96,8 @@ async fn testing_rpc_build_block_works() -> eyre::Result<()> { #[tokio::test(flavor = "multi_thread")] async fn testing_rpc_commit_block_works() -> eyre::Result<()> { let runtime = Runtime::test(); - let mut rpc_args = reth_node_core::args::RpcServerArgs::default().with_http(); + let mut rpc_args = + reth_node_core::args::RpcServerArgs::default().with_http().with_unused_ports(); rpc_args.http_api = Some(RpcModuleSelection::from_iter([RethRpcModule::Eth, RethRpcModule::Testing])); let tempdir = tempdir().expect("temp datadir"); diff --git a/crates/net/network/tests/it/txgossip.rs b/crates/net/network/tests/it/txgossip.rs index 1b518c50e9c..ba5d92b824b 100644 --- a/crates/net/network/tests/it/txgossip.rs +++ b/crates/net/network/tests/it/txgossip.rs @@ -1,16 +1,15 @@ //! Testing gossiping of transactions. use alloy_consensus::TxLegacy; use alloy_primitives::{Signature, U256}; -use futures::StreamExt; use reth_ethereum_primitives::TransactionSigned; use reth_network::{ test_utils::{NetworkEventStream, Testnet}, transactions::config::{ TransactionIngressPolicy, TransactionPropagationKind, TransactionsManagerConfig, }, - NetworkEvent, NetworkEventListenerProvider, Peers, + NetworkEventListenerProvider, Peers, }; -use reth_network_api::{events::PeerEvent, PeerKind, PeersInfo}; +use reth_network_api::{PeerKind, PeersInfo}; use reth_provider::test_utils::{ExtendedAccount, MockEthProvider}; use reth_transaction_pool::{ test_utils::TransactionGenerator, AddedTransactionOutcome, PoolTransaction, TransactionPool, @@ -256,11 +255,12 @@ async fn test_sending_invalid_transactions() { let peer0 = &handle.peers()[0]; let peer1 = &handle.peers()[1]; + let mut peer1_events = NetworkEventStream::new(peer1.network().event_listener()); + // connect all the peers handle.connect_peers().await; assert_eq!(peer0.network().num_connected_peers(), 1); - let mut peer1_events = peer1.network().event_listener(); let mut tx_listener = peer1.pool().unwrap().new_transactions_listener(); for idx in 0..10 { @@ -278,24 +278,16 @@ async fn test_sending_invalid_transactions() { peer0.network().send_transactions(*peer1.peer_id(), vec![Arc::new(tx)]); } - // await disconnect for bad tx spam - if let Some(ev) = peer1_events.next().await { - match ev { - NetworkEvent::Peer(PeerEvent::SessionClosed { peer_id, .. }) => { - assert_eq!(peer_id, *peer0.peer_id()); - } - NetworkEvent::ActivePeerSession { .. } | - NetworkEvent::Peer(PeerEvent::SessionEstablished { .. }) => { - panic!("unexpected SessionEstablished event") - } - NetworkEvent::Peer(PeerEvent::PeerAdded(_)) => { - panic!("unexpected PeerAdded event") - } - NetworkEvent::Peer(PeerEvent::PeerRemoved(_)) => { - panic!("unexpected PeerRemoved event") - } - } - } + // The listener also receives connection events, and PeerAdded can still be queued after + // connect_peers returns. Wait specifically for the disconnect after bad transaction spam. + let (peer_id, _) = tokio::time::timeout( + std::time::Duration::from_secs(10), + peer1_events.next_session_closed(), + ) + .await + .expect("peer did not disconnect after invalid transaction spam") + .expect("network event stream ended before disconnect"); + assert_eq!(peer_id, *peer0.peer_id()); // ensure txs never made it to the pool assert!(tx_listener.try_recv().is_err()); From ea10653e2ca33216bdc0ee044fa7d44a6ee7c69a Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Tue, 8 Sep 2026 16:39:59 +0200 Subject: [PATCH 211/294] fix(net): decode wire bitvectors into typed blob cell masks (#27089) --- crates/net/eth-wire-types/src/transactions.rs | 9 ++- crates/net/network/src/eth_requests.rs | 62 ++++++++++++++++--- crates/rpc/rpc-engine-api/src/engine_api.rs | 25 +++----- crates/transaction-pool/src/blobstore/disk.rs | 27 ++++---- crates/transaction-pool/src/blobstore/mem.rs | 23 ++++--- crates/transaction-pool/src/blobstore/mod.rs | 8 +-- crates/transaction-pool/src/blobstore/noop.rs | 10 +-- crates/transaction-pool/src/lib.rs | 8 +-- crates/transaction-pool/src/noop.rs | 6 +- crates/transaction-pool/src/traits.rs | 8 +-- 10 files changed, 117 insertions(+), 69 deletions(-) diff --git a/crates/net/eth-wire-types/src/transactions.rs b/crates/net/eth-wire-types/src/transactions.rs index ebfe4f4e4ad..c2f8e1c1074 100644 --- a/crates/net/eth-wire-types/src/transactions.rs +++ b/crates/net/eth-wire-types/src/transactions.rs @@ -3,7 +3,7 @@ use crate::broadcast::decode_list_with_memory_budget; use alloc::vec::Vec; use alloy_consensus::transaction::{PooledTransaction, TxHashRef}; -use alloy_eips::eip7594::Cell; +use alloy_eips::eip7594::{BlobCellMask, Cell}; use alloy_primitives::{B128, B256}; use alloy_rlp::{Decodable, RlpDecodable, RlpDecodableWrapper, RlpEncodable, RlpEncodableWrapper}; use derive_more::{Constructor, Deref, IntoIterator}; @@ -127,6 +127,13 @@ pub struct GetCells { pub cell_mask: B128, } +impl GetCells { + /// Returns the requested cell indices, decoded from the little-endian wire bitvector. + pub fn cell_mask(&self) -> BlobCellMask { + BlobCellMask::from_bits(u128::from_le_bytes(self.cell_mask.into())) + } +} + impl InMemorySize for GetCells { fn size(&self) -> usize { self.hashes.len() * core::mem::size_of::() + core::mem::size_of::() diff --git a/crates/net/network/src/eth_requests.rs b/crates/net/network/src/eth_requests.rs index 9098a352913..ae1957ba56e 100644 --- a/crates/net/network/src/eth_requests.rs +++ b/crates/net/network/src/eth_requests.rs @@ -407,11 +407,10 @@ where ) { let mut cells_response = Cells { cell_mask: request.cell_mask, ..Default::default() }; let mut total_bytes = 0; + let cell_mask = request.cell_mask(); for hash in request.hashes.into_iter().take(MAX_CELLS_SERVE) { - let Some(cells) = - self.blob_store.get_cells(hash, request.cell_mask).unwrap_or_default() - else { + let Some(cells) = self.blob_store.get_cells(hash, cell_mask).unwrap_or_default() else { continue; }; @@ -859,7 +858,7 @@ mod tests { use alloy_consensus::constants::EMPTY_ROOT_HASH; use alloy_eips::{ eip4844::{BlobAndProofV1, BlobAndProofV2, BlobCellsAndProofsV1}, - eip7594::{BlobTransactionSidecarVariant, Cell}, + eip7594::{BlobCellMask, BlobTransactionSidecarVariant, Cell}, }; use alloy_primitives::{keccak256, Address, TxHash, B128, U256}; use reth_network_api::test_utils::PeersHandle; @@ -879,6 +878,7 @@ mod tests { #[derive(Debug, Default)] struct CountingBlobStore { get_cells_calls: Arc, + expected_cell_mask: Option, } impl BlobStore for CountingBlobStore { @@ -955,7 +955,7 @@ mod tests { fn get_by_versioned_hashes_v4( &self, versioned_hashes: &[B256], - _indices_bitarray: B128, + _cell_mask: BlobCellMask, ) -> Result>, BlobStoreError> { Ok(vec![None; versioned_hashes.len()]) } @@ -970,9 +970,13 @@ mod tests { fn get_cells( &self, _tx_hash: TxHash, - _indices_bitarray: B128, + cell_mask: BlobCellMask, ) -> Result>, BlobStoreError> { self.get_cells_calls.fetch_add(1, Ordering::Relaxed); + if let Some(expected) = self.expected_cell_mask { + assert_eq!(cell_mask, expected); + return Ok(Some(vec![Cell::default()])) + } Ok(None) } @@ -990,7 +994,10 @@ mod tests { let (peers_tx, _) = mpsc::unbounded_channel(); let (_incoming_tx, incoming_rx) = mpsc::channel(1); let get_cells_calls = Arc::new(AtomicUsize::new(0)); - let blob_store = CountingBlobStore { get_cells_calls: Arc::clone(&get_cells_calls) }; + let blob_store = CountingBlobStore { + get_cells_calls: Arc::clone(&get_cells_calls), + ..Default::default() + }; let handler = EthRequestHandler::::new( NoopProvider::default(), PeersHandle::new(peers_tx), @@ -1008,6 +1015,47 @@ mod tests { assert_eq!(get_cells_calls.load(Ordering::Relaxed), MAX_CELLS_SERVE); } + #[test_case(0)] + #[test_case(7)] + #[test_case(8)] + #[test_case(63)] + #[test_case(64)] + #[test_case(127)] + #[tokio::test] + async fn get_cells_request_uses_little_endian_wire_mask(index: u32) { + use alloy_rlp::{Decodable, Encodable}; + + let (peers_tx, _) = mpsc::unbounded_channel(); + let (_incoming_tx, incoming_rx) = mpsc::channel(1); + let numeric_mask = 1u128 << index; + let blob_store = CountingBlobStore { + expected_cell_mask: Some(BlobCellMask::from_bits(numeric_mask)), + ..Default::default() + }; + let handler = EthRequestHandler::::new( + NoopProvider::default(), + PeersHandle::new(peers_tx), + incoming_rx, + ) + .with_blob_store(Box::new(blob_store)); + let wire_mask = B128::from(numeric_mask.to_le_bytes()); + let request = GetCells { hashes: vec![B256::ZERO], cell_mask: wire_mask }; + let mut encoded = Vec::new(); + request.encode(&mut encoded); + let request = GetCells::decode(&mut encoded.as_slice()).unwrap(); + let (response, rx) = oneshot::channel(); + + handler.on_cells_request(PeerId::default(), request, response); + + let cells = rx.await.unwrap().unwrap(); + assert_eq!(cells.hashes, vec![B256::ZERO]); + assert_eq!(cells.cells, vec![vec![Cell::default()]]); + assert_eq!(cells.cell_mask, wire_mask); + encoded.clear(); + cells.encode(&mut encoded); + assert_eq!(Cells::decode(&mut encoded.as_slice()).unwrap(), cells); + } + #[tokio::test] async fn get_node_data_responds_with_empty_payload() { let (peers_tx, _) = mpsc::unbounded_channel(); diff --git a/crates/rpc/rpc-engine-api/src/engine_api.rs b/crates/rpc/rpc-engine-api/src/engine_api.rs index 3624c48091c..161bd58f155 100644 --- a/crates/rpc/rpc-engine-api/src/engine_api.rs +++ b/crates/rpc/rpc-engine-api/src/engine_api.rs @@ -5,6 +5,7 @@ use alloy_eips::{ eip1898::BlockHashOrNumber, eip4844::{BlobAndProofV1, BlobAndProofV2, BlobCellsAndProofsV1}, eip4895::Withdrawals, + eip7594::BlobCellMask, eip7685::RequestsOrHash, }; use alloy_primitives::{BlockHash, BlockNumber, Bytes, Sealable, B128, B256, U64}; @@ -1223,9 +1224,8 @@ where versioned_hashes: Vec, indices_bitarray: B128, ) -> EngineApiResult>>> { - // Engine API bitvectors are little-endian, while B128 integer conversions are - // big-endian. The transaction pool uses the latter representation as a numeric cell mask. - let indices_bitarray = B128::from(u128::from_le_bytes(indices_bitarray.into())); + // Engine API bitvectors encode the lowest cell indices in the first byte. + let cell_mask = BlobCellMask::from_bits(u128::from_le_bytes(indices_bitarray.into())); let current_timestamp = SystemTime::now().duration_since(SystemTime::UNIX_EPOCH).unwrap_or_default().as_secs(); if !self.inner.chain_spec.is_osaka_active_at_timestamp(current_timestamp) { @@ -1245,7 +1245,7 @@ where self.inner .tx_pool - .get_blobs_for_versioned_hashes_v4(&versioned_hashes, indices_bitarray) + .get_blobs_for_versioned_hashes_v4(&versioned_hashes, cell_mask) .map(Some) .map_err(|err| EngineApiError::Internal(Box::new(err))) } @@ -2432,18 +2432,11 @@ mod tests { #[test] fn engine_bitvector_uses_little_endian_cell_indices() { - assert_eq!( - B128::from(u128::from_le_bytes(B128::from(1u128.to_le_bytes()).into())), - B128::from(1u128) - ); - assert_eq!( - B128::from(u128::from_le_bytes(B128::from((1u128 << 127).to_le_bytes()).into())), - B128::from(1u128 << 127) - ); - assert_eq!( - B128::from(u128::from_le_bytes(B128::from(((1u128 << 64) - 1).to_le_bytes()).into(),)), - B128::from((1u128 << 64) - 1) - ); + for index in [0, 7, 8, 63, 64, 127] { + let wire_mask = B128::from((1u128 << index).to_le_bytes()); + let mask = BlobCellMask::from_bits(u128::from_le_bytes(wire_mask.into())); + assert_eq!(mask.selected_indices().collect::>(), vec![index]); + } } #[tokio::test] diff --git a/crates/transaction-pool/src/blobstore/disk.rs b/crates/transaction-pool/src/blobstore/disk.rs index ed3ef441b3f..01d1d9085cd 100644 --- a/crates/transaction-pool/src/blobstore/disk.rs +++ b/crates/transaction-pool/src/blobstore/disk.rs @@ -9,7 +9,7 @@ use alloy_eips::{ eip7840::BlobParams, merge::EPOCH_SLOTS, }; -use alloy_primitives::{map::B256Set, TxHash, B128, B256}; +use alloy_primitives::{map::B256Set, TxHash, B256}; use parking_lot::{Mutex, RwLock}; use schnellru::{ByLength, LruMap}; use std::{fmt, fs, io, path::PathBuf, sync::Arc}; @@ -143,9 +143,8 @@ impl DiskFileBlobStore { fn get_by_versioned_hashes_cells_eip7594( &self, versioned_hashes: &[B256], - indices_bitarray: B128, + cell_mask: BlobCellMask, ) -> Result>, BlobStoreError> { - let cell_mask = BlobCellMask::new(indices_bitarray); let mut result = vec![None; versioned_hashes.len()]; let mut missing_count = result.len(); @@ -383,9 +382,9 @@ impl BlobStore for DiskFileBlobStore { fn get_by_versioned_hashes_v4( &self, versioned_hashes: &[B256], - indices_bitarray: B128, + cell_mask: BlobCellMask, ) -> Result>, BlobStoreError> { - self.get_by_versioned_hashes_cells_eip7594(versioned_hashes, indices_bitarray) + self.get_by_versioned_hashes_cells_eip7594(versioned_hashes, cell_mask) } fn has_versioned_hashes(&self, versioned_hashes: &[B256]) -> Result, BlobStoreError> { @@ -428,7 +427,7 @@ impl BlobStore for DiskFileBlobStore { fn get_cells( &self, tx: B256, - indices_bitarray: B128, + cell_mask: BlobCellMask, ) -> Result>, BlobStoreError> { let Some(sidecar) = self.get(tx)? else { return Ok(None); @@ -439,7 +438,7 @@ impl BlobStore for DiskFileBlobStore { }; sidecar - .compute_matching_cells(BlobCellMask::new(indices_bitarray)) + .compute_matching_cells(cell_mask) .map(Some) .map_err(|err| BlobStoreError::Other(Box::new(err))) } @@ -1090,10 +1089,10 @@ mod tests { let (sidecar, versioned_hash, _) = eip7594_single_blob_sidecar(); store.insert(TxHash::random(), sidecar.into()).unwrap(); - let indices_bitarray = B128::from((1u128 << 0) | (1u128 << 7)); + let cell_mask = BlobCellMask::from_bits((1u128 << 0) | (1u128 << 7)); let request = vec![versioned_hash, B256::ZERO]; - let v4 = store.get_by_versioned_hashes_v4(&request, indices_bitarray).unwrap(); + let v4 = store.get_by_versioned_hashes_v4(&request, cell_mask).unwrap(); assert_eq!(v4.len(), request.len()); assert!(v4[1].is_none()); @@ -1150,7 +1149,9 @@ mod tests { store.insert(TxHash::random(), sidecar.into()).unwrap(); store.clear_cache(); - let v4 = store.get_by_versioned_hashes_v4(&[versioned_hash], B128::from(1u128)).unwrap(); + let v4 = store + .get_by_versioned_hashes_v4(&[versioned_hash], BlobCellMask::from_bits(1u128)) + .unwrap(); let cells_and_proofs = v4[0].as_ref().unwrap(); assert_eq!(cells_and_proofs.blob_cells.len(), 1); assert_eq!(cells_and_proofs.proofs, vec![Some(Bytes48::default())]); @@ -1164,9 +1165,9 @@ mod tests { let (sidecar, versioned_hash, _) = eip7594_single_blob_sidecar(); store.insert(tx_hash, sidecar.into()).unwrap(); - let indices_bitarray = B128::from((1u128 << 0) | (1u128 << 7)); + let cell_mask = BlobCellMask::from_bits((1u128 << 0) | (1u128 << 7)); let expected = store - .get_by_versioned_hashes_v4(&[versioned_hash], indices_bitarray) + .get_by_versioned_hashes_v4(&[versioned_hash], cell_mask) .unwrap() .pop() .unwrap() @@ -1178,7 +1179,7 @@ mod tests { store.clear_cache(); - assert_eq!(store.get_cells(tx_hash, indices_bitarray).unwrap(), Some(expected)); + assert_eq!(store.get_cells(tx_hash, cell_mask).unwrap(), Some(expected)); } #[test] diff --git a/crates/transaction-pool/src/blobstore/mem.rs b/crates/transaction-pool/src/blobstore/mem.rs index 78637605c8e..634c3afe1b2 100644 --- a/crates/transaction-pool/src/blobstore/mem.rs +++ b/crates/transaction-pool/src/blobstore/mem.rs @@ -5,7 +5,7 @@ use alloy_eips::{ eip4844::{BlobAndProofV1, BlobAndProofV2, BlobCellsAndProofsV1}, eip7594::{BlobCellMask, BlobTransactionSidecarVariant, Cell}, }; -use alloy_primitives::{map::B256Map, B128, B256}; +use alloy_primitives::{map::B256Map, B256}; use parking_lot::RwLock; use std::sync::Arc; @@ -55,9 +55,8 @@ impl InMemoryBlobStore { fn get_by_versioned_hashes_cells_eip7594( &self, versioned_hashes: &[B256], - indices_bitarray: B128, + cell_mask: BlobCellMask, ) -> Result>, BlobStoreError> { - let cell_mask = BlobCellMask::new(indices_bitarray); let mut result = vec![None; versioned_hashes.len()]; let mut missing_count = result.len(); let blob_sidecars = self.inner.store.read().values().cloned().collect::>(); @@ -219,9 +218,9 @@ impl BlobStore for InMemoryBlobStore { fn get_by_versioned_hashes_v4( &self, versioned_hashes: &[B256], - indices_bitarray: B128, + cell_mask: BlobCellMask, ) -> Result>, BlobStoreError> { - self.get_by_versioned_hashes_cells_eip7594(versioned_hashes, indices_bitarray) + self.get_by_versioned_hashes_cells_eip7594(versioned_hashes, cell_mask) } fn has_versioned_hashes(&self, versioned_hashes: &[B256]) -> Result, BlobStoreError> { @@ -245,7 +244,7 @@ impl BlobStore for InMemoryBlobStore { fn get_cells( &self, tx: B256, - indices_bitarray: B128, + cell_mask: BlobCellMask, ) -> Result>, BlobStoreError> { let Some(sidecar) = self.get(tx)? else { return Ok(None); @@ -256,7 +255,7 @@ impl BlobStore for InMemoryBlobStore { }; sidecar - .compute_matching_cells(BlobCellMask::new(indices_bitarray)) + .compute_matching_cells(cell_mask) .map(Some) .map_err(|err| BlobStoreError::Other(Box::new(err))) } @@ -366,10 +365,10 @@ mod tests { let (sidecar, versioned_hash, _) = eip7594_single_blob_sidecar(); store.insert(B256::random(), sidecar.into()).unwrap(); - let indices_bitarray = B128::from((1u128 << 0) | (1u128 << 7)); + let cell_mask = BlobCellMask::from_bits((1u128 << 0) | (1u128 << 7)); let request = vec![versioned_hash, B256::ZERO]; - let v4 = store.get_by_versioned_hashes_v4(&request, indices_bitarray).unwrap(); + let v4 = store.get_by_versioned_hashes_v4(&request, cell_mask).unwrap(); assert_eq!(v4.len(), request.len()); assert!(v4[1].is_none()); @@ -388,9 +387,9 @@ mod tests { let (sidecar, versioned_hash, _) = eip7594_single_blob_sidecar(); store.insert(tx_hash, sidecar.into()).unwrap(); - let indices_bitarray = B128::from((1u128 << 0) | (1u128 << 7)); + let cell_mask = BlobCellMask::from_bits((1u128 << 0) | (1u128 << 7)); let expected = store - .get_by_versioned_hashes_v4(&[versioned_hash], indices_bitarray) + .get_by_versioned_hashes_v4(&[versioned_hash], cell_mask) .unwrap() .pop() .unwrap() @@ -400,6 +399,6 @@ mod tests { .collect::>>() .unwrap(); - assert_eq!(store.get_cells(tx_hash, indices_bitarray).unwrap(), Some(expected)); + assert_eq!(store.get_cells(tx_hash, cell_mask).unwrap(), Some(expected)); } } diff --git a/crates/transaction-pool/src/blobstore/mod.rs b/crates/transaction-pool/src/blobstore/mod.rs index eaec381933d..caad2a9854f 100644 --- a/crates/transaction-pool/src/blobstore/mod.rs +++ b/crates/transaction-pool/src/blobstore/mod.rs @@ -4,7 +4,7 @@ use alloy_eips::{ eip4844::{BlobAndProofV1, BlobAndProofV2, BlobCellsAndProofsV1}, eip7594::{BlobCellMask, BlobTransactionSidecarVariant, Cell}, }; -use alloy_primitives::{TxHash, B128, B256}; +use alloy_primitives::{TxHash, B256}; pub use converter::BlobSidecarConverter; pub use disk::{DiskFileBlobStore, DiskFileBlobStoreConfig, OpenDiskFileBlobStore}; pub use mem::InMemoryBlobStore; @@ -207,7 +207,7 @@ pub trait BlobStore: fmt::Debug + Send + Sync + 'static { fn get_by_versioned_hashes_v4( &self, versioned_hashes: &[B256], - indices_bitarray: B128, + cell_mask: BlobCellMask, ) -> Result>, BlobStoreError>; /// Return whether each requested blob versioned hash is available. @@ -217,7 +217,7 @@ pub trait BlobStore: fmt::Debug + Send + Sync + 'static { /// Returns all requested cells for all blobs belonging to the transaction. /// - /// The `indices_bitarray` is applied independently to every blob in the tx. + /// The `cell_mask` is applied independently to every blob in the tx. /// /// Returned cells are flattened in blob order, then cell-index order. /// @@ -238,7 +238,7 @@ pub trait BlobStore: fmt::Debug + Send + Sync + 'static { fn get_cells( &self, tx_hash: TxHash, - indices_bitarray: B128, + cell_mask: BlobCellMask, ) -> Result>, BlobStoreError>; /// Data size of all transactions in the blob store. diff --git a/crates/transaction-pool/src/blobstore/noop.rs b/crates/transaction-pool/src/blobstore/noop.rs index 80e9ae412fa..3a6f5dd2f62 100644 --- a/crates/transaction-pool/src/blobstore/noop.rs +++ b/crates/transaction-pool/src/blobstore/noop.rs @@ -1,9 +1,9 @@ use crate::blobstore::{BlobStore, BlobStoreCleanupStat, BlobStoreError, PooledBlobSidecar}; use alloy_eips::{ eip4844::{BlobAndProofV1, BlobAndProofV2, BlobCellsAndProofsV1}, - eip7594::{BlobTransactionSidecarVariant, Cell}, + eip7594::{BlobCellMask, BlobTransactionSidecarVariant, Cell}, }; -use alloy_primitives::{TxHash, B128, B256}; +use alloy_primitives::{TxHash, B256}; use std::sync::Arc; /// A blobstore implementation that does nothing @@ -81,7 +81,7 @@ impl BlobStore for NoopBlobStore { fn get_by_versioned_hashes_v4( &self, versioned_hashes: &[B256], - _indices_bitarray: B128, + _cell_mask: BlobCellMask, ) -> Result>, BlobStoreError> { Ok(vec![None; versioned_hashes.len()]) } @@ -93,9 +93,9 @@ impl BlobStore for NoopBlobStore { fn get_cells( &self, tx_hash: TxHash, - indices_bitarray: B128, + cell_mask: BlobCellMask, ) -> Result>, BlobStoreError> { - let _ = (tx_hash, indices_bitarray); + let _ = (tx_hash, cell_mask); Ok(None) } diff --git a/crates/transaction-pool/src/lib.rs b/crates/transaction-pool/src/lib.rs index 90ef4a14029..233689ccff5 100644 --- a/crates/transaction-pool/src/lib.rs +++ b/crates/transaction-pool/src/lib.rs @@ -305,9 +305,9 @@ pub use crate::{ use crate::{identifier::TransactionId, pool::PoolInner}; use alloy_eips::{ eip4844::{BlobAndProofV1, BlobAndProofV2, BlobCellsAndProofsV1}, - eip7594::BlobTransactionSidecarVariant, + eip7594::{BlobCellMask, BlobTransactionSidecarVariant}, }; -use alloy_primitives::{map::AddressSet, Address, TxHash, B128, B256, U256}; +use alloy_primitives::{map::AddressSet, Address, TxHash, B256, U256}; use aquamarine as _; use reth_chainspec::{ChainSpecProvider, EthereumHardforks}; use reth_eth_wire_types::HandleMempoolData; @@ -824,9 +824,9 @@ where fn get_blobs_for_versioned_hashes_v4( &self, versioned_hashes: &[B256], - indices_bitarray: B128, + cell_mask: BlobCellMask, ) -> Result>, BlobStoreError> { - self.pool.blob_store().get_by_versioned_hashes_v4(versioned_hashes, indices_bitarray) + self.pool.blob_store().get_by_versioned_hashes_v4(versioned_hashes, cell_mask) } fn has_blobs_for_versioned_hashes( diff --git a/crates/transaction-pool/src/noop.rs b/crates/transaction-pool/src/noop.rs index 10341ee5117..226dd07a6b1 100644 --- a/crates/transaction-pool/src/noop.rs +++ b/crates/transaction-pool/src/noop.rs @@ -18,9 +18,9 @@ use crate::{ use alloy_eips::{ eip1559::ETHEREUM_BLOCK_GAS_LIMIT_30M, eip4844::{BlobAndProofV1, BlobAndProofV2, BlobCellsAndProofsV1}, - eip7594::BlobTransactionSidecarVariant, + eip7594::{BlobCellMask, BlobTransactionSidecarVariant}, }; -use alloy_primitives::{map::AddressSet, Address, TxHash, B128, B256, U256}; +use alloy_primitives::{map::AddressSet, Address, TxHash, B256, U256}; use reth_eth_wire_types::HandleMempoolData; use reth_primitives_traits::Recovered; use std::{marker::PhantomData, sync::Arc}; @@ -398,7 +398,7 @@ impl TransactionPool for NoopTransactionPool { fn get_blobs_for_versioned_hashes_v4( &self, versioned_hashes: &[B256], - _indices_bitarray: B128, + _cell_mask: BlobCellMask, ) -> Result>, BlobStoreError> { Ok(vec![None; versioned_hashes.len()]) } diff --git a/crates/transaction-pool/src/traits.rs b/crates/transaction-pool/src/traits.rs index b3400ba7a0c..bb16e8ae72c 100644 --- a/crates/transaction-pool/src/traits.rs +++ b/crates/transaction-pool/src/traits.rs @@ -68,12 +68,12 @@ use alloy_eips::{ env_settings::KzgSettings, BlobAndProofV1, BlobAndProofV2, BlobCellsAndProofsV1, BlobTransactionValidationError, }, - eip7594::BlobTransactionSidecarVariant, + eip7594::{BlobCellMask, BlobTransactionSidecarVariant}, eip7702::SignedAuthorization, }; use alloy_primitives::{ map::{AddressSet, B256Map}, - Address, Bytes, TxHash, TxKind, B128, B256, U256, + Address, Bytes, TxHash, TxKind, B256, U256, }; use futures_util::{ready, Stream}; use reth_eth_wire_types::HandleMempoolData; @@ -755,7 +755,7 @@ pub trait TransactionPool: Clone + Debug + Send + Sync { fn get_blobs_for_versioned_hashes_v4( &self, versioned_hashes: &[B256], - indices_bitarray: B128, + cell_mask: BlobCellMask, ) -> Result>, BlobStoreError>; /// Return whether each requested blob versioned hash is available. @@ -2005,7 +2005,7 @@ mod tests { EthereumTxEnvelope, SignableTransaction, TxEip1559, TxEip2930, TxEip4844, TxEip7702, TxEnvelope, TxLegacy, }; - use alloy_eips::{eip4844::DATA_GAS_PER_BLOB, eip7594::BlobCellMask}; + use alloy_eips::eip4844::DATA_GAS_PER_BLOB; use alloy_primitives::Signature; #[test] From fc8c4fc6b84e5e2812fa89ec5f589b9ea8a28be1 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Tue, 8 Sep 2026 16:23:39 +0000 Subject: [PATCH 212/294] fix(engine): stop transaction recovery on channel disconnection (#27092) Co-authored-by: Matthias Seitz <19890894+mattsse@users.noreply.github.com> --- .../tree/src/tree/payload_processor/mod.rs | 191 ++++++++++++++++-- 1 file changed, 169 insertions(+), 22 deletions(-) diff --git a/crates/engine/tree/src/tree/payload_processor/mod.rs b/crates/engine/tree/src/tree/payload_processor/mod.rs index 73dcdb1db9d..9ef57678448 100644 --- a/crates/engine/tree/src/tree/payload_processor/mod.rs +++ b/crates/engine/tree/src/tree/payload_processor/mod.rs @@ -293,23 +293,22 @@ where // With BALs, we don't care about the order of transactions in execution and // prewarming, so we don't have to use `for_each_ordered_in`. executor.cpu_pool().install(|| { - transactions + let _ = transactions .into_par_iter() .enumerate() - .map(|(i, tx)| { - let tx = convert.convert(tx); - (i, tx) - }) - .for_each(|(idx, tx)| { - let tx = tx.map(|tx| { - let tx = WithTxEnv::new(tx); - if let Some(prewarm_tx) = &prewarm_tx { - let _ = prewarm_tx.send((idx, tx.clone())); - } - tx - }); - let _ = execute_tx.send((idx, tx)); + .try_for_each(|(idx, tx)| { + let tx = convert.convert(tx).map(WithTxEnv::new); + let failed = tx.is_err(); + if let (Some(prewarm_tx), Ok(tx)) = (&prewarm_tx, &tx) { + let _ = prewarm_tx.send((idx, tx.clone())); + } + let disconnected = execute_tx.send((idx, tx)).is_err(); trace!(target: "engine::tree::payload_processor", idx, "yielded transaction"); + if failed || disconnected { + Err(()) + } else { + Ok(()) + } }); }); } else { @@ -321,12 +320,14 @@ where // Convert the first few transactions sequentially so execution can // start immediately without waiting for rayon work-stealing. - convert_serial( + if !convert_serial( iter.by_ref().take(prefetch), &convert, prewarm_tx.as_ref(), &execute_tx, - ); + ) { + return + } let mut iter = iter.enumerate(); @@ -356,10 +357,13 @@ where .collect::>(); for (idx, tx) in chunk { + let failed = tx.is_err(); if let (Some(prewarm_tx), Ok(tx)) = (&prewarm_tx, &tx) { let _ = prewarm_tx.send((idx, tx.clone())); } - let _ = execute_tx.send((idx, tx)); + if execute_tx.send((idx, tx)).is_err() || failed { + return + } trace!(target: "engine::tree::payload_processor", idx, "yielded transaction"); } } @@ -525,26 +529,31 @@ where } /// Converts transactions sequentially and sends them to the execute channel, and to the prewarm -/// channel if there is one. +/// channel if there is one. Returns false on conversion failure or disconnection. fn convert_serial( iter: impl Iterator, convert: &C, prewarm_tx: Option<&mpsc::SyncSender<(usize, WithTxEnv)>>, execute_tx: &ExecuteTxSender, -) where +) -> bool +where Tx: ExecutableTxParts, TxEnv: Clone, C: ConvertTx, { for (idx, raw_tx) in iter.enumerate() { let tx = convert.convert(raw_tx); - let tx = tx.map(|tx| WithTxEnv::new(tx)); + let failed = tx.is_err(); + let tx = tx.map(WithTxEnv::new); if let (Some(prewarm_tx), Ok(tx)) = (prewarm_tx, &tx) { let _ = prewarm_tx.send((idx, tx.clone())); } - let _ = execute_tx.send((idx, tx)); + if execute_tx.send((idx, tx)).is_err() || failed { + return false + } trace!(target: "engine::tree::payload_processor", idx, "yielded transaction"); } + true } /// Handle to all the spawned tasks. @@ -685,7 +694,145 @@ mod tests { use reth_execution_cache::CachedStatus; use reth_revm::db::BundleState; use revm::state::AccountInfo; - use std::sync::Arc; + use std::sync::{atomic::Ordering, Arc}; + + type TestTx = reth_evm::execute::WithTxEnv< + reth_evm::TxEnvFor, + reth_primitives_traits::Recovered, + >; + + fn converted_tx() -> TestTx { + TestTx { + tx_env: Default::default(), + tx: Arc::new(reth_primitives_traits::Recovered::new_unchecked( + reth_ethereum_primitives::TransactionSigned::Legacy( + alloy_consensus::Signed::new_unchecked( + alloy_consensus::TxLegacy::default(), + alloy_primitives::Signature::test_signature(), + B256::ZERO, + ), + ), + Address::ZERO, + )), + } + } + + fn test_processor() -> PayloadProcessor { + PayloadProcessor::new( + reth_tasks::Runtime::test(), + EthEvmConfig::new(Arc::new(ChainSpec::default())), + &TreeConfig::default(), + PrecompileCacheMap::default(), + ) + } + + #[test] + fn transaction_conversion_preserves_results() { + for (count, bal) in [(10, false), (200, false), (200, true)] { + let processor = test_processor(); + let (prewarm, receiver) = processor.spawn_tx_iterator( + ((0..count).collect::>(), |_| Ok::<_, std::io::Error>(converted_tx())), + count, + bal, + true, + ); + let mut indices = Vec::new(); + for _ in 0..count { + let (idx, tx) = receiver.recv_timeout(std::time::Duration::from_secs(10)).unwrap(); + assert!(tx.is_ok()); + indices.push(idx); + } + if bal { + indices.sort_unstable(); + } + assert_eq!(indices, (0..count).collect::>()); + assert_eq!(prewarm.unwrap().iter().count(), count); + } + } + + #[test] + fn transaction_conversion_stops_on_error() { + for (count, bal, fail_at) in + [(10, false, 0), (200, false, 0), (200, false, 4), (200, false, 20), (200, true, 0)] + { + let processor = test_processor(); + let (_, receiver) = processor.spawn_tx_iterator( + ((0..count).collect::>(), move |idx| { + if idx >= fail_at { + Err(std::io::Error::other("invalid transaction")) + } else { + Ok(converted_tx()) + } + }), + count, + bal, + false, + ); + let mut results = Vec::new(); + loop { + match receiver.recv_timeout(std::time::Duration::from_secs(10)) { + Ok(tx) => results.push(tx), + Err(crossbeam_channel::RecvTimeoutError::Disconnected) => break, + Err(err) => panic!("conversion did not terminate: {err}"), + } + } + assert!(results.iter().any(|(_, tx)| tx.is_err())); + assert!(results.len() < count); + if !bal { + assert_eq!(results.len(), fail_at + 1); + assert!(results.last().unwrap().1.is_err()); + } + } + } + + #[test] + fn dropping_payload_handle_stops_transaction_conversion() { + for (count, bal, pause_at) in + [(10, false, 0), (1000, false, 0), (1000, false, 4), (1000, true, 0)] + { + let processor = test_processor(); + let calls = Arc::new(super::AtomicUsize::new(0)); + let converted = calls.clone(); + let (started_tx, started_rx) = crossbeam_channel::unbounded(); + let (release_tx, release_rx) = crossbeam_channel::bounded::<()>(0); + let (_, receiver) = processor.spawn_tx_iterator( + ((0..count).collect::>(), move |idx| { + converted.fetch_add(1, Ordering::Relaxed); + if idx >= pause_at { + started_tx.send(()).unwrap(); + let _ = release_rx.recv_timeout(std::time::Duration::from_secs(10)); + } + Ok::<_, std::io::Error>(converted_tx()) + }), + count, + bal, + false, + ); + let handle = super::PayloadHandle { + prewarm_handle: super::CacheTaskHandle::<()> { + saved_cache: None, + to_prewarm_task: None, + executed_tx_index: Default::default(), + cache_metrics: None, + }, + transactions: receiver, + _span: tracing::Span::none(), + }; + started_rx.recv_timeout(std::time::Duration::from_secs(10)).unwrap(); + drop(handle); + drop(release_tx); + + // The converter owns the last sender, so disconnection confirms recovery exited. + loop { + match started_rx.recv_timeout(std::time::Duration::from_secs(10)) { + Ok(()) => {} + Err(crossbeam_channel::RecvTimeoutError::Disconnected) => break, + Err(err) => panic!("conversion did not terminate: {err}"), + } + } + assert!(calls.load(Ordering::Relaxed) < count); + } + } fn make_saved_cache(hash: B256) -> SavedCache { let execution_cache = ExecutionCache::new(1_000); From db702b511a9dfe5a80ef76685dd93652ffe37cc5 Mon Sep 17 00:00:00 2001 From: rakita Date: Tue, 8 Sep 2026 17:06:25 +0000 Subject: [PATCH 213/294] chore(deps): bump revm to 43.0.1 (#27096) --- Cargo.lock | 48 ++++++++++++++++++++++++------------------------ Cargo.toml | 2 +- 2 files changed, 25 insertions(+), 25 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 322d23e806a..bdd49fbff3e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1035,7 +1035,7 @@ version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" dependencies = [ - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -1046,7 +1046,7 @@ checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" dependencies = [ "anstyle", "once_cell_polyfill", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -3431,7 +3431,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -5175,7 +5175,7 @@ checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" dependencies = [ "hermit-abi", "libc", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -6243,7 +6243,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -10824,9 +10824,9 @@ dependencies = [ [[package]] name = "revm" -version = "43.0.0" +version = "43.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0d1681d52ee61171b838db4924827f63d659d4c3ad559aa3a57003f3dda3f7c" +checksum = "fc210dfb3b45482a9f7c8806f737f7848fb7054b04235e6816480e515c1fe2d5" dependencies = [ "revm-bytecode", "revm-context", @@ -10856,9 +10856,9 @@ dependencies = [ [[package]] name = "revm-context" -version = "43.0.1" +version = "43.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f85bce65cc8969955309d5c4c37a63612209c720d040f1982dda6b59e39ab059" +checksum = "c16835bc2ead203219a18fcc84c2c71879ad6b890e8a71150cf7f10e4e355dc4" dependencies = [ "bitvec", "cfg-if", @@ -10873,9 +10873,9 @@ dependencies = [ [[package]] name = "revm-context-interface" -version = "43.0.0" +version = "43.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d97d0c7f8bdf65601d35ed27cf6b0f0bd436087f0a97a52aa767a49bbcea8373" +checksum = "e3b4109bdceaca5ee6e27bf41f6b156f2b15892dc5bb47d3530bc4d3b3060077" dependencies = [ "alloy-eip2930", "alloy-eip7702", @@ -10919,9 +10919,9 @@ dependencies = [ [[package]] name = "revm-handler" -version = "43.0.0" +version = "43.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab0b5923be8784704d119be5381d76a602f65e4a9991650bb87291565bd0d8ec" +checksum = "7b8d988f896e22b19962658bc1afcd0989f727d48b2860a8bd7ceaecbf72ce34" dependencies = [ "auto_impl", "derive-where", @@ -10938,9 +10938,9 @@ dependencies = [ [[package]] name = "revm-inspector" -version = "43.0.0" +version = "43.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a988ad0879bd81b269cad53c235a8ce87cd44a7c1de7d21a6ff3dd9e9f3e70a9" +checksum = "e2584990403728b2e13fb5522510b61e4bb217c6a4c8f4d063447a6ddd7dd7cc" dependencies = [ "auto_impl", "either", @@ -10976,9 +10976,9 @@ dependencies = [ [[package]] name = "revm-interpreter" -version = "43.0.0" +version = "43.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ddc442e3c5006ebb65f479e9e8fded7fc10f300c040276e825eee81e00c94cc" +checksum = "932f7fe05a2cdeb08195dd7e88a09a4da7c3660cdc6e62720839c77ba77f65aa" dependencies = [ "revm-bytecode", "revm-context-interface", @@ -10989,9 +10989,9 @@ dependencies = [ [[package]] name = "revm-precompile" -version = "43.0.0" +version = "43.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d85f7a8906482f8d18f2a2c793bc4daf254b29465af4285268384505ce6ee308" +checksum = "ac4dc0279f2a44770faa8c93d008df684440cf573c91b8e0e39d0d581abf7b3d" dependencies = [ "ark-bls12-381", "ark-bn254", @@ -11399,7 +11399,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -11479,7 +11479,7 @@ dependencies = [ "security-framework", "security-framework-sys", "webpki-root-certs 1.0.7", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -12112,7 +12112,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "52d1cfed4120b4d927bf7c0f86d2087a4a7d6027c906d9f9d525a80573b9be51" dependencies = [ "libc", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -12371,7 +12371,7 @@ dependencies = [ "getrandom 0.4.2", "once_cell", "rustix", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -13634,7 +13634,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 80708157382..15b9024e28b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -433,7 +433,7 @@ reth-trie-sparse = { path = "crates/trie/sparse", default-features = false } reth-zstd-compressors = { version = "0.7.0", default-features = false } # revm -revm = { version = "43.0.0", default-features = false } +revm = { version = "43.0.1", default-features = false } revmc = { git = "https://github.com/paradigmxyz/revmc", branch = "main", default-features = false, features = ["llvm-prefer-static"] } revm-inspectors = "0.43.0" From cee96c6f6e4f107609afaa0d0852206618d2983f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 8 Sep 2026 18:02:29 +0000 Subject: [PATCH 214/294] chore(deps): bump the cargo-weekly group across 1 directory with 42 updates (#27027) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Matthias Seitz --- Cargo.lock | 740 +++++++++++++++---------- docs/vocs/docs/pages/cli/reth/node.mdx | 2 +- 2 files changed, 447 insertions(+), 295 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index bdd49fbff3e..fcbee8dfa26 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -106,17 +106,17 @@ checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" [[package]] name = "alloy-chains" -version = "0.2.34" +version = "0.2.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84e0378e959aa6a885897522080a990e80eb317f1e9a222a604492ea50e13096" +checksum = "e5fdcfed8f106be3df944054aaa42bc13ae103a3ac8a9f4b08d4f053e3a743f8" dependencies = [ "alloy-primitives", "alloy-rlp", "arbitrary", "num_enum", + "phf 0.14.0", "proptest", "serde", - "strum 0.27.2", ] [[package]] @@ -144,7 +144,7 @@ dependencies = [ "serde", "serde_json", "serde_with", - "thiserror 2.0.18", + "thiserror 2.0.20", ] [[package]] @@ -181,7 +181,7 @@ dependencies = [ "futures", "futures-util", "serde_json", - "thiserror 2.0.18", + "thiserror 2.0.20", "tokio", "tracing", "wasmtimer", @@ -189,9 +189,9 @@ dependencies = [ [[package]] name = "alloy-dyn-abi" -version = "1.6.1" +version = "1.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a04eb4abc2b5074a18e687ee63918f407cc7990083cba9b999445f839796060" +checksum = "d9f1a3f2206f2ba4206fdeeddce6640eed3e26b8a13ac41444adb66b76d8e650" dependencies = [ "alloy-json-abi", "alloy-primitives", @@ -216,7 +216,7 @@ dependencies = [ "crc", "rand 0.8.6", "serde", - "thiserror 2.0.18", + "thiserror 2.0.20", ] [[package]] @@ -247,7 +247,7 @@ dependencies = [ "rand 0.8.6", "serde", "serde_with", - "thiserror 2.0.18", + "thiserror 2.0.20", ] [[package]] @@ -262,7 +262,7 @@ dependencies = [ "borsh", "once_cell", "serde", - "thiserror 2.0.18", + "thiserror 2.0.20", ] [[package]] @@ -307,7 +307,7 @@ dependencies = [ "auto_impl", "derive_more", "revm", - "thiserror 2.0.18", + "thiserror 2.0.20", "tracing", ] @@ -355,9 +355,9 @@ dependencies = [ [[package]] name = "alloy-json-abi" -version = "1.6.1" +version = "1.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6cee30dd4c2f4b23f434fdf675e7bf9681b86768141277266c6f548ef25cba0a" +checksum = "1dba4e59c3581a39e03e0b0b4a46ee9c41315b5d843b1e903271952b32bedb7c" dependencies = [ "alloy-primitives", "alloy-sol-type-parser", @@ -376,7 +376,7 @@ dependencies = [ "http", "serde", "serde_json", - "thiserror 2.0.18", + "thiserror 2.0.20", "tracing", ] @@ -403,7 +403,7 @@ dependencies = [ "futures-utils-wasm", "serde", "serde_json", - "thiserror 2.0.18", + "thiserror 2.0.20", ] [[package]] @@ -436,16 +436,16 @@ dependencies = [ "rand 0.8.6", "serde_json", "tempfile", - "thiserror 2.0.18", + "thiserror 2.0.20", "tracing", "url", ] [[package]] name = "alloy-primitives" -version = "1.6.1" +version = "1.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f007e257069855bdf21d27762fd3f3705a613f805c9a08309bf353503f081d71" +checksum = "ce7b00f0cb42c66ec353076ded1dff1fbf818f6e0e26c40c8a8456c04483fca4" dependencies = [ "alloy-rlp", "arbitrary", @@ -457,7 +457,7 @@ dependencies = [ "foldhash 0.2.0", "getrandom 0.4.2", "hashbrown 0.17.0", - "indexmap 2.14.0", + "indexmap 2.14.2", "itoa", "k256", "keccak-asm", @@ -511,7 +511,7 @@ dependencies = [ "reqwest", "serde", "serde_json", - "thiserror 2.0.18", + "thiserror 2.0.20", "tokio", "tracing", "url", @@ -655,7 +655,7 @@ dependencies = [ "serde", "serde_json", "serde_with", - "thiserror 2.0.18", + "thiserror 2.0.20", "tree_hash", "tree_hash_derive", ] @@ -713,7 +713,7 @@ dependencies = [ "serde", "serde_json", "serde_with", - "thiserror 2.0.18", + "thiserror 2.0.20", ] [[package]] @@ -742,7 +742,7 @@ dependencies = [ "alloy-serde", "serde", "serde_json", - "thiserror 2.0.18", + "thiserror 2.0.20", ] [[package]] @@ -781,7 +781,7 @@ dependencies = [ "either", "elliptic-curve", "k256", - "thiserror 2.0.18", + "thiserror 2.0.20", ] [[package]] @@ -799,15 +799,15 @@ dependencies = [ "coins-bip39", "k256", "rand 0.8.6", - "thiserror 2.0.18", + "thiserror 2.0.20", "zeroize", ] [[package]] name = "alloy-sol-macro" -version = "1.6.1" +version = "1.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5655c38d5f84955bf727b2eeb62fddd91ebb98fd1d7ae6eb77f73ea88f9b9cf" +checksum = "c64558980fb038cd34b4285ec2b36a2a8bd8d4ddd13b3f6e42d97cb5ee29938e" dependencies = [ "alloy-sol-macro-expander", "alloy-sol-macro-input", @@ -819,14 +819,14 @@ dependencies = [ [[package]] name = "alloy-sol-macro-expander" -version = "1.6.1" +version = "1.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6277c780e07b76951e09a59788dde230d1582612324177d11a43a61e21a6bb83" +checksum = "1ffb0e793abdbaea9d01259493c8272c3af295d03389e70a2acdf53b57ad1edf" dependencies = [ "alloy-sol-macro-input", "const-hex", "heck", - "indexmap 2.14.0", + "indexmap 2.14.2", "proc-macro-error3", "proc-macro2", "quote", @@ -837,9 +837,9 @@ dependencies = [ [[package]] name = "alloy-sol-macro-input" -version = "1.6.1" +version = "1.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9762b2ad3e5a0c09886de54fe549ab0056681df843cb082e2df7e1c0eb270d30" +checksum = "32c2c0ec8425d9663dac939ba75750f17d2933d2f020814b4f8fde4a60da6d26" dependencies = [ "const-hex", "dunce", @@ -853,9 +853,9 @@ dependencies = [ [[package]] name = "alloy-sol-type-parser" -version = "1.6.1" +version = "1.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da4c7130f0f01f4719678bda3db3bc7267fc2f7f9d0565e3bd964cd2bb45050d" +checksum = "96b37db6a7ad8170596345f864522f789cc7af093f516d6cdf34bbdcfba8adab" dependencies = [ "serde", "winnow 1.0.1", @@ -863,9 +863,9 @@ dependencies = [ [[package]] name = "alloy-sol-types" -version = "1.6.1" +version = "1.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d96e74d6213180f78dbdccddce8af02a639c160c94b0a543fa35c77c58b8a7fc" +checksum = "1f40e33a0f588dde548c3b6767a71e61b593421a8c1b253b9cf1441dc7cf7ab5" dependencies = [ "alloy-json-abi", "alloy-primitives", @@ -888,7 +888,7 @@ dependencies = [ "parking_lot", "serde", "serde_json", - "thiserror 2.0.18", + "thiserror 2.0.20", "tokio", "tower", "tracing", @@ -968,7 +968,7 @@ dependencies = [ "proptest-derive 0.7.0", "serde", "smallvec", - "thiserror 2.0.18", + "thiserror 2.0.20", "tracing", ] @@ -1051,9 +1051,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.103" +version = "1.0.104" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a4385e2e34eb35d6b3efe798b9eb88096925d87726c0798709bf56d9ed84af3" +checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" [[package]] name = "approx" @@ -1348,7 +1348,7 @@ dependencies = [ "ark-serialize 0.6.0", "ark-std 0.6.0", "foldhash 0.1.5", - "indexmap 2.14.0", + "indexmap 2.14.2", "tracing", "tracing-subscriber", ] @@ -1479,9 +1479,9 @@ checksum = "9b34d609dfbaf33d6889b2b7106d3ca345eacad44200913df5ba02bfd31d2ba9" [[package]] name = "async-compression" -version = "0.4.42" +version = "0.4.44" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e79b3f8a79cccc2898f31920fc69f304859b3bd567490f75ebf51ae1c792a9ac" +checksum = "515a1f282e33d55983c499d7e9e87082e81cbc32974825bf9032f928392d5844" dependencies = [ "compression-codecs", "compression-core", @@ -1617,7 +1617,7 @@ dependencies = [ "addr2line", "cfg-if", "libc", - "miniz_oxide", + "miniz_oxide 0.8.9", "object 0.37.3", "rustc-demangle", "windows-link", @@ -1657,6 +1657,12 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "base64" +version = "0.23.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5" + [[package]] name = "base64ct" version = "1.8.3" @@ -1690,7 +1696,7 @@ version = "0.72.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "993776b509cfb49c750f11b8f07a46fa23e0a1386ffc01fb1e7d343efc387895" dependencies = [ - "bitflags 2.13.0", + "bitflags 2.13.1", "cexpr", "clang-sys", "itertools 0.13.0", @@ -1741,9 +1747,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.13.0" +version = "2.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4388bee8683e3d04af747c73422af53102d2bd24d9eadb6cbc100baef4b43f8" +checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" dependencies = [ "arbitrary", "serde_core", @@ -1770,11 +1776,10 @@ dependencies = [ [[package]] name = "blake3" -version = "1.8.5" +version = "1.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0aa83c34e62843d924f905e0f5c866eb1dd6545fc4d719e803d9ba6030371fce" +checksum = "6d9e454fc11f76977dc803893aff6304ed33d6a26efae8696573bea74baa27ae" dependencies = [ - "arrayref", "arrayvec", "cc", "cfg-if", @@ -1818,11 +1823,11 @@ version = "0.21.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6339a700715bda376f5ea65c76e8fe8fc880930d8b0638cea68e7f3da6538e0a" dependencies = [ - "bitflags 2.13.0", + "bitflags 2.13.1", "boa_interner", "boa_macros", "boa_string", - "indexmap 2.14.0", + "indexmap 2.14.2", "num-bigint", "rustc-hash", ] @@ -1835,7 +1840,7 @@ checksum = "1521be326f8a5c8887e95d4ce7f002917a002a23f7b93b9a6a2bf50ed4157824" dependencies = [ "aligned-vec", "arrayvec", - "bitflags 2.13.0", + "bitflags 2.13.1", "boa_ast", "boa_gc", "boa_interner", @@ -1854,7 +1859,7 @@ dependencies = [ "futures-lite", "hashbrown 0.16.1", "icu_normalizer", - "indexmap 2.14.0", + "indexmap 2.14.2", "intrusive-collections", "itertools 0.14.0", "num-bigint", @@ -1874,7 +1879,7 @@ dependencies = [ "tag_ptr", "tap", "thin-vec", - "thiserror 2.0.18", + "thiserror 2.0.20", "time", "xsum", ] @@ -1900,7 +1905,7 @@ dependencies = [ "boa_gc", "boa_macros", "hashbrown 0.16.1", - "indexmap 2.14.0", + "indexmap 2.14.2", "once_cell", "phf 0.13.1", "rustc-hash", @@ -1927,7 +1932,7 @@ version = "0.21.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "35bd957fa9fa93e3a001a8aba5a5cd40c2bbfde486378be4c4b472fd304aaddb" dependencies = [ - "bitflags 2.13.0", + "bitflags 2.13.1", "boa_ast", "boa_interner", "boa_macros", @@ -1969,7 +1974,7 @@ version = "3.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6dee98b0db6a962de883bf5d20362dee4d7ca0d12fe39a7c6c73c844e1cd7c1f" dependencies = [ - "darling 0.23.0", + "darling 0.21.3", "ident_case", "prettyplease", "proc-macro2", @@ -2073,9 +2078,9 @@ checksum = "7575182f7272186991736b70173b0ea045398f984bf5ebbb3804736ce1330c9d" [[package]] name = "bytemuck" -version = "1.25.0" +version = "1.25.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8efb64bd706a16a1bdde310ae86b351e4d21550d98d056f22f8a7f7a2183fec" +checksum = "95832e849adfb21180ccb6826a99da14e5d266ae5c2e668e1602cf234f153797" dependencies = [ "bytemuck_derive", ] @@ -2161,7 +2166,7 @@ dependencies = [ "semver 1.0.28", "serde", "serde_json", - "thiserror 2.0.18", + "thiserror 2.0.20", ] [[package]] @@ -2181,9 +2186,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.67" +version = "1.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e17dd265a7d0f31ef544e1b20e03add05d3b45b491b633b10d67145d2acc1a38" +checksum = "005ec2760ca554fae18df7a11195552ec576cd665632a881bc011d5bb2fd4d80" dependencies = [ "find-msvc-tools", "jobserver", @@ -2293,9 +2298,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.6.1" +version = "4.6.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ddb117e43bbf7dacf0a4190fef4d345b9bad68dfc649cb349e7d17d28428e51" +checksum = "473c7e07f409a8d772161724aa8db6a765a2532a70f9667eeb7b49d3d02fbdca" dependencies = [ "clap_builder", "clap_derive", @@ -2303,9 +2308,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.6.0" +version = "4.6.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "714a53001bf66416adb0e2ef5ac857140e7dc3a0c48fb28b2f10762fc4b5069f" +checksum = "7b48fea5a88e9ae728a2dcbedbfc0e730f7d60da42e1cb049a83c9fb8b789889" dependencies = [ "anstream", "anstyle", @@ -2315,14 +2320,14 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.6.1" +version = "4.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2ce8604710f6733aa641a2b3731eaa1e8b3d9973d5e3565da11800813f997a9" +checksum = "d012d2b9d65aca7f18f4d9878a045bc17899bba951561ba5ec3c2ba1eed9a061" dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.118", + "syn 3.0.3", ] [[package]] @@ -2346,7 +2351,7 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0fa961b519f0b462e3a3b4a34b64d119eeaca1d59af726fe450bbba07a9fc0a1" dependencies = [ - "thiserror 2.0.18", + "thiserror 2.0.20", ] [[package]] @@ -2511,9 +2516,9 @@ dependencies = [ [[package]] name = "compression-codecs" -version = "0.4.38" +version = "0.4.39" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce2548391e9c1929c21bf6aa2680af86fe4c1b33e6cea9ac1cfeec0bd11218cf" +checksum = "2fe67f2944eef52fc7b106b8c9450d243a88701a0c065f7f57235e76abaed7df" dependencies = [ "brotli", "compression-core", @@ -2525,9 +2530,9 @@ dependencies = [ [[package]] name = "compression-core" -version = "0.4.32" +version = "0.4.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc14f565cf027a105f7a44ccf9e5b424348421a1d8952a8fc9d499d313107789" +checksum = "6e8ccc4ea9f6acc32d102c0f6d471d11d913ad15f20c04de743374861fa1d414" [[package]] name = "concat-kdf" @@ -2713,9 +2718,9 @@ checksum = "790eea4361631c5e7d22598ecd5723ff611904e3344ce8720784c93e3d83d40b" [[package]] name = "crossbeam-channel" -version = "0.5.15" +version = "0.5.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82b8f8f868b36967f9606790d1903570de9ceaf870a7bf9fbbd3016d636a2cb2" +checksum = "98b0cc327b5bc766e7fda9c9260cc0fa81b43a8e240440422dff70788e3f9ef1" dependencies = [ "crossbeam-utils", ] @@ -2741,18 +2746,18 @@ dependencies = [ [[package]] name = "crossbeam-queue" -version = "0.3.12" +version = "0.3.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f58bbc28f91df819d0aa2a2c00cd19754769c2fad90579b3592b1c9ba7a3115" +checksum = "03e8bd762f7479489c70ed6c768ddca99d7296857de437a68dcb2a94365b3fae" dependencies = [ "crossbeam-utils", ] [[package]] name = "crossbeam-utils" -version = "0.8.21" +version = "0.8.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" +checksum = "a31eee39dddec8330830986fcd7625edb5a24ec90ea038215273bbc3adb08ac6" [[package]] name = "crossterm" @@ -2760,7 +2765,7 @@ version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d8b9f2e4c67f833b660cdb0a3523065869fb35570177239812ed4c905aeff87b" dependencies = [ - "bitflags 2.13.0", + "bitflags 2.13.1", "crossterm_winapi", "derive_more", "document-features", @@ -2888,6 +2893,16 @@ dependencies = [ "darling_macro 0.23.0", ] +[[package]] +name = "darling" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed17f5901b6630b993ca003def43f2f8ef4014fc13b047b57aad617ff32bc2ec" +dependencies = [ + "darling_core 0.24.1", + "darling_macro 0.24.1", +] + [[package]] name = "darling_core" version = "0.21.3" @@ -2916,6 +2931,19 @@ dependencies = [ "syn 2.0.118", ] +[[package]] +name = "darling_core" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6837e2cf7485aaae18f86181d2f0e9a7ed297a025e220aeabf63fdebd3a2ddff" +dependencies = [ + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn 3.0.3", +] + [[package]] name = "darling_macro" version = "0.21.3" @@ -2938,6 +2966,17 @@ dependencies = [ "syn 2.0.118", ] +[[package]] +name = "darling_macro" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ac7135c3ef02b2f7833bbeb1be5ba7f966dcde8a87c6b87f65a778d71a02785" +dependencies = [ + "darling_core 0.24.1", + "quote", + "syn 3.0.3", +] + [[package]] name = "dashmap" version = "6.2.1" @@ -2956,9 +2995,9 @@ dependencies = [ [[package]] name = "data-encoding" -version = "2.11.0" +version = "2.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4ae5f15dda3c708c0ade84bfee31ccab44a3da4f88015ed22f63732abe300c8" +checksum = "4583a4551df46e2792f82ceeac45e850d2e2d5debba0b91f102385cda5b11f06" [[package]] name = "data-encoding-macro" @@ -2977,7 +3016,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ab67060fc6b8ef687992d439ca0fa36e7ed17e9a0b16b25b601e8757df720de" dependencies = [ "data-encoding", - "syn 2.0.118", + "syn 1.0.109", ] [[package]] @@ -2986,6 +3025,37 @@ version = "0.3.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f578e8e2c440e7297e008bb5486a3a8a194775224bbc23729b0dbdfaeebf162e" +[[package]] +name = "defmt" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2953bfe4f93bbd20cc71198842756f77d161884c99ebbabc41d80231ded88d1" +dependencies = [ + "bitflags 1.3.2", + "defmt-macros", +] + +[[package]] +name = "defmt-macros" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bad9c72e7ca2137e0dc3813245a0d282fd6daad32fd800af018306a9169b5fe8" +dependencies = [ + "defmt-parser", + "proc-macro2", + "quote", + "syn 2.0.118", +] + +[[package]] +name = "defmt-parser" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10d60334b3b2e7c9d91ef8150abfb6fa4c1c39ebbcf4a81c2e346aad939fee3e" +dependencies = [ + "thiserror 2.0.20", +] + [[package]] name = "delay_map" version = "0.4.1" @@ -3307,15 +3377,15 @@ dependencies = [ "revm", "serde", "serde_json", - "thiserror 2.0.18", + "thiserror 2.0.20", "walkdir", ] [[package]] name = "either" -version = "1.16.0" +version = "1.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91622ff5e7162018101f2fea40d6ebf4a78bbe5a49736a2020649edf9693679e" +checksum = "252afb9ae5eaa683babdc6a068b3f5726eb19e05070c731f9b2a23a7c3e8ed34" dependencies = [ "serde", ] @@ -3511,7 +3581,7 @@ dependencies = [ "reth-ethereum", "serde", "serde_json", - "thiserror 2.0.18", + "thiserror 2.0.20", ] [[package]] @@ -3557,7 +3627,7 @@ dependencies = [ "secp256k1 0.30.0", "serde", "serde_json", - "thiserror 2.0.18", + "thiserror 2.0.20", "tokio", "tokio-stream", "tracing", @@ -3614,7 +3684,7 @@ dependencies = [ "reth-payload-builder", "reth-tracing", "serde", - "thiserror 2.0.18", + "thiserror 2.0.20", "tokio", ] @@ -3868,10 +3938,11 @@ dependencies = [ [[package]] name = "eyre" -version = "0.6.12" +version = "0.6.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7cd915d99f24784cdc19fd37ef22b97e3ff0ae756c7e492e9fbfe897d61e2aec" +checksum = "c08309dbcc659c5549a24ddb9b27027640641b282ef5768267c7e675558986a3" dependencies = [ + "autocfg", "indenter", "once_cell", ] @@ -3955,9 +4026,9 @@ dependencies = [ [[package]] name = "find-msvc-tools" -version = "0.1.9" +version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +checksum = "3e0f1c7c3a72c66fd80abe965175f7523475c0489a87d3ff9d6e8c87d87a9d2d" [[package]] name = "fixed-cache" @@ -4011,12 +4082,13 @@ checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99" [[package]] name = "flate2" -version = "1.1.9" +version = "1.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" +checksum = "6e634e2e0ebac1ee034020da1ca582e17ffe4e0f5e985823721e168928136dcb" dependencies = [ "crc32fast", - "miniz_oxide", + "miniz_oxide 0.9.1", + "zlib-rs", ] [[package]] @@ -4079,9 +4151,9 @@ checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" [[package]] name = "futures" -version = "0.3.32" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b147ee9d1f6d097cef9ce628cd2ee62288d963e16fb287bd9286455b241382d" +checksum = "9a31d2a3fbaaeb2af2368bbdd904aa8e812d3c04a1ee10d3171f52d556e5d0a3" dependencies = [ "futures-channel", "futures-core", @@ -4094,9 +4166,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.32" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07bbe89c50d7a535e539b8c17bc0b49bdb77747034daa8087407d655f3f7cc1d" +checksum = "b1f9e3d69d39e4862ffed03ed071a76f9a13ba1d9109d355b0f0aa6b15e393c4" dependencies = [ "futures-core", "futures-sink", @@ -4117,15 +4189,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.32" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" +checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e" [[package]] name = "futures-executor" -version = "0.3.32" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "baf29c38818342a3b26b5b923639e7b1f4a61fc5e76102d4b1981c6dc7a7579d" +checksum = "031b47cf1a3c6cc8bc2fc76cd437f521619387907d469316e7c0bc278f1f5432" dependencies = [ "futures-core", "futures-task", @@ -4134,9 +4206,9 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.32" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cecba35d7ad927e23624b22ad55235f2239cfa44fd10428eecbeba6d6a717718" +checksum = "53c0fa8157de1303bfffdaa1cc2a673bfffb60102f76b0ef4441659124373fed" [[package]] name = "futures-lite" @@ -4153,26 +4225,26 @@ dependencies = [ [[package]] name = "futures-macro" -version = "0.3.32" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" +checksum = "9fb9654ba8355388abeb8dcb4fc62f511300867002afc858860463bdd9fe0c44" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 3.0.3", ] [[package]] name = "futures-sink" -version = "0.3.32" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c39754e157331b013978ec91992bde1ac089843443c49cbc7f46150b0fad0893" +checksum = "1944426bf7d03f1d14f708785e4b33efd750b36d48a157b836b3efc15ede8e1d" [[package]] name = "futures-task" -version = "0.3.32" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" +checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd" [[package]] name = "futures-timer" @@ -4186,9 +4258,9 @@ dependencies = [ [[package]] name = "futures-util" -version = "0.3.32" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" +checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc" dependencies = [ "futures-channel", "futures-core", @@ -4296,7 +4368,7 @@ version = "0.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ddddbf932745a6be37109b6112d3ee09696106f848449069d3a57bba937ab82e" dependencies = [ - "bitflags 2.13.0", + "bitflags 2.13.1", "libc", "libgit2-sys", "log", @@ -4387,7 +4459,7 @@ dependencies = [ "futures-core", "futures-sink", "http", - "indexmap 2.14.0", + "indexmap 2.14.2", "slab", "tokio", "tokio-util", @@ -4554,7 +4626,7 @@ dependencies = [ "ipnet", "jni 0.22.4", "rand 0.10.1", - "thiserror 2.0.18", + "thiserror 2.0.20", "tinyvec", "tokio", "tracing", @@ -4576,7 +4648,7 @@ dependencies = [ "rand 0.10.1", "ring", "serde", - "thiserror 2.0.18", + "thiserror 2.0.20", "tinyvec", "tracing", "url", @@ -4604,7 +4676,7 @@ dependencies = [ "serde", "smallvec", "system-configuration", - "thiserror 2.0.18", + "thiserror 2.0.20", "tokio", "tracing", ] @@ -4629,9 +4701,9 @@ dependencies = [ [[package]] name = "http" -version = "1.4.2" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6970f50e31d6fc17d3fa27329444bfa74e196cf62e95052a3f6fee181dba6425" +checksum = "918d3568bebf352712bc2ef3d46a8bcf1a75b373be6539de198e9105cbbf9ce0" dependencies = [ "bytes", "itoa", @@ -4639,9 +4711,9 @@ dependencies = [ [[package]] name = "http-body" -version = "1.0.1" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" +checksum = "ca2a8f2913ee65f60facd6a5905613afaa448497a0230cc41ce022d93290bc2c" dependencies = [ "bytes", "http", @@ -4649,9 +4721,9 @@ dependencies = [ [[package]] name = "http-body-util" -version = "0.1.3" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" +checksum = "23169fe34a5fbcdd3f3862e78fb9b6fccd5f02a6dc6f732547005d45631ce71c" dependencies = [ "bytes", "futures-core", @@ -4711,9 +4783,9 @@ dependencies = [ [[package]] name = "hyper" -version = "1.10.1" +version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "55281c53a1894c864990125767da440a4e630446785086f52523b20033b74498" +checksum = "27b501faa50e7a26c3d3560ca625132f4078a17771f4810baf70475ae48cbe43" dependencies = [ "atomic-waker", "bytes", @@ -4941,13 +5013,14 @@ dependencies = [ [[package]] name = "imbl" -version = "7.0.0" +version = "7.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e525189e5f603908d0c6e0d402cb5de9c4b2c8866151fabc4ebd771ed2630a2e" +checksum = "43ea8d4c37ee560727e824d62804183624d371e632019b0e9e3532bce64a33e5" dependencies = [ "arbitrary", "archery", "bitmaps", + "equivalent", "imbl-sized-chunks", "rand_core 0.9.5", "rand_xoshiro", @@ -5023,9 +5096,9 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.14.0" +version = "2.14.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" +checksum = "cc4e190f5d26ca7051642629da2c52fc03bde85a03197c99408dcd291734c855" dependencies = [ "arbitrary", "equivalent", @@ -5049,11 +5122,11 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7decbc9dfa45a4a827a6ff7b822c113b1285678a937e84213417d4ca8a095782" dependencies = [ - "bitflags 2.13.0", + "bitflags 2.13.1", "inkwell_internals", "libc", "llvm-sys", - "thiserror 2.0.18", + "thiserror 2.0.20", ] [[package]] @@ -5073,7 +5146,7 @@ version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bd5b3eaf1a28b758ac0faa5a4254e8ab2705605496f1b1f3fbbc3988ad73d199" dependencies = [ - "bitflags 2.13.0", + "bitflags 2.13.1", "inotify-sys", "libc", ] @@ -5123,9 +5196,9 @@ dependencies = [ [[package]] name = "interprocess" -version = "2.4.2" +version = "2.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "069323743400cb7ab06a8fe5c1ed911d36b6919ec531661d034c89083629595b" +checksum = "6cb02ffbf25e8ef0f4a95bff054eef75f6d1bf16919474cf2caa63b21997478a" dependencies = [ "doctest-file", "futures-core", @@ -5160,9 +5233,9 @@ dependencies = [ [[package]] name = "ipnet" -version = "2.12.0" +version = "2.12.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2" +checksum = "791930b43c0d5973160d90a8f3894509f2b273430f5c5c73b668636d0287c5c0" dependencies = [ "serde", ] @@ -5234,6 +5307,59 @@ dependencies = [ "tracing", ] +[[package]] +name = "jiff" +version = "0.2.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "668b7183bd07af9a4885f5c35b0cc5c83c4607a913c16b7e17291832910d2dcc" +dependencies = [ + "defmt", + "jiff-core", + "jiff-static", + "jiff-tzdb-platform", + "log", + "portable-atomic", + "portable-atomic-util", + "serde_core", + "windows-link", +] + +[[package]] +name = "jiff-core" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7feca88439efe53da3754500c1851dedf3cb36c524dd5cf8225cc0794de95d09" +dependencies = [ + "defmt", +] + +[[package]] +name = "jiff-static" +version = "0.2.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a69dcb3a21cfb32ce1cd056169337ca284af0766dd766e7878819b251a49204" +dependencies = [ + "jiff-core", + "proc-macro2", + "quote", + "syn 2.0.118", +] + +[[package]] +name = "jiff-tzdb" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "142bd39932ad231f10513df9ab62661fead8719872150b7ad02a2df79f4e141e" + +[[package]] +name = "jiff-tzdb-platform" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "875a5a69ac2bab1a891711cf5eccbec1ce0341ea805560dcd90b7a2e925132e8" +dependencies = [ + "jiff-tzdb", +] + [[package]] name = "jni" version = "0.21.1" @@ -5262,7 +5388,7 @@ dependencies = [ "jni-sys 0.4.1", "log", "simd_cesu8", - "thiserror 2.0.18", + "thiserror 2.0.20", "walkdir", "windows-link", ] @@ -5365,7 +5491,7 @@ dependencies = [ "rustls-pki-types", "rustls-platform-verifier 0.5.3", "soketto", - "thiserror 2.0.18", + "thiserror 2.0.20", "tokio", "tokio-rustls", "tokio-util", @@ -5393,7 +5519,7 @@ dependencies = [ "rustc-hash", "serde", "serde_json", - "thiserror 2.0.18", + "thiserror 2.0.20", "tokio", "tokio-stream", "tower", @@ -5418,7 +5544,7 @@ dependencies = [ "rustls-platform-verifier 0.5.3", "serde", "serde_json", - "thiserror 2.0.18", + "thiserror 2.0.20", "tokio", "tower", "url", @@ -5456,7 +5582,7 @@ dependencies = [ "serde", "serde_json", "soketto", - "thiserror 2.0.18", + "thiserror 2.0.20", "tokio", "tokio-stream", "tokio-util", @@ -5473,7 +5599,7 @@ dependencies = [ "http", "serde", "serde_json", - "thiserror 2.0.18", + "thiserror 2.0.20", ] [[package]] @@ -5542,7 +5668,7 @@ checksum = "bde5057d6143cc94e861d90f591b9303d6716c6b9602309150bd068853c10899" dependencies = [ "hashbrown 0.16.1", "portable-atomic", - "thiserror 2.0.18", + "thiserror 2.0.20", ] [[package]] @@ -5634,9 +5760,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.186" +version = "0.2.189" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" [[package]] name = "libgit2-sys" @@ -5690,7 +5816,7 @@ dependencies = [ "multihash", "quick-protobuf", "sha2 0.10.9", - "thiserror 2.0.18", + "thiserror 2.0.20", "tracing", "zeroize", ] @@ -5712,7 +5838,7 @@ version = "0.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e02f3bb43d335493c96bf3fd3a321600bf6bd07ed34bc64118e9293bdffea46c" dependencies = [ - "bitflags 2.13.0", + "bitflags 2.13.1", "libc", "plain", "redox_syscall 0.7.4", @@ -5752,7 +5878,7 @@ version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f50e8f47623268b5407192d26876c4d7f89d686ca130fdc53bced4814cd29f8" dependencies = [ - "bitflags 2.13.0", + "bitflags 2.13.1", ] [[package]] @@ -5984,11 +6110,11 @@ checksum = "1db0d8f1fc9e62caebd0319e11eaec5822b0186c171568f0480b46a0137f9108" dependencies = [ "base64 0.22.1", "evmap", - "indexmap 2.14.0", + "indexmap 2.14.2", "metrics", "metrics-util", "quanta", - "thiserror 2.0.18", + "thiserror 2.0.20", ] [[package]] @@ -6016,7 +6142,7 @@ dependencies = [ "crossbeam-epoch", "crossbeam-utils", "hashbrown 0.16.1", - "indexmap 2.14.0", + "indexmap 2.14.2", "metrics", "ordered-float", "quanta", @@ -6053,6 +6179,15 @@ name = "miniz_oxide" version = "0.8.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", +] + +[[package]] +name = "miniz_oxide" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b63fbc4a50860e98e7b2aa7804ded1db5cbc3aff9193adaff57a6931bf7c4b4c" dependencies = [ "adler2", "simd-adler32", @@ -6093,9 +6228,9 @@ dependencies = [ [[package]] name = "moka" -version = "0.12.15" +version = "0.12.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "957228ad12042ee839f93c8f257b62b4c0ab5eaae1d4fa60de53b27c9d7c5046" +checksum = "4293f18e7567a1caf3c584855554377025c65e0aa445344d04171f5ad63d19b9" dependencies = [ "crossbeam-channel", "crossbeam-epoch", @@ -6167,7 +6302,7 @@ version = "0.31.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5d6d0705320c1e6ba1d912b5e37cf18071b6c2e9b7fa8215a1e8a7651966f5d3" dependencies = [ - "bitflags 2.13.0", + "bitflags 2.13.1", "cfg-if", "cfg_aliases", "libc", @@ -6207,7 +6342,7 @@ version = "8.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4d3d07927151ff8575b7087f245456e549fea62edf0ec4e565a5ee50c8402bc3" dependencies = [ - "bitflags 2.13.0", + "bitflags 2.13.1", "fsevent-sys", "inotify", "kqueue", @@ -6225,7 +6360,7 @@ version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42b8cfee0e339a0337359f3c88165702ac6e600dc01c0cc9579a92d62b08477a" dependencies = [ - "bitflags 2.13.0", + "bitflags 2.13.1", ] [[package]] @@ -6389,7 +6524,7 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2a180dd8642fa45cdb7dd721cd4c11b1cadd4929ce112ebd8b9f5803cc79d536" dependencies = [ - "bitflags 2.13.0", + "bitflags 2.13.1", ] [[package]] @@ -6464,7 +6599,7 @@ dependencies = [ "futures-sink", "js-sys", "pin-project-lite", - "thiserror 2.0.18", + "thiserror 2.0.20", "tracing", ] @@ -6506,7 +6641,7 @@ dependencies = [ "opentelemetry_sdk", "prost", "reqwest", - "thiserror 2.0.18", + "thiserror 2.0.20", "tokio", "tonic", "tonic-types", @@ -6544,7 +6679,7 @@ dependencies = [ "percent-encoding", "portable-atomic", "rand 0.9.4", - "thiserror 2.0.18", + "thiserror 2.0.20", ] [[package]] @@ -6921,6 +7056,15 @@ version = "1.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49" +[[package]] +name = "portable-atomic-util" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10ab3eb7f3becc3a1cbc4f2c6f20267996cfc1a6467a873763411b136a122715" +dependencies = [ + "portable-atomic", +] + [[package]] name = "postcard" version = "1.1.3" @@ -7077,9 +7221,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.106" +version = "1.0.107" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" dependencies = [ "unicode-ident", ] @@ -7090,7 +7234,7 @@ version = "0.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "25485360a54d6861439d60facef26de713b1e126bf015ec8f98239467a2b82f7" dependencies = [ - "bitflags 2.13.0", + "bitflags 2.13.1", "chrono", "flate2", "procfs-core", @@ -7103,7 +7247,7 @@ version = "0.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6401bf7b6af22f78b563665d15a22e9aef27775b79b149a66ca022468a4e405" dependencies = [ - "bitflags 2.13.0", + "bitflags 2.13.1", "chrono", "hex", ] @@ -7116,7 +7260,7 @@ checksum = "4b45fcc2344c680f5025fe57779faef368840d0bd1f42f216291f0dc4ace4744" dependencies = [ "bit-set", "bit-vec", - "bitflags 2.13.0", + "bitflags 2.13.1", "num-traits", "rand 0.9.4", "rand_chacha 0.9.0", @@ -7235,7 +7379,7 @@ dependencies = [ "rustc-hash", "rustls", "socket2", - "thiserror 2.0.18", + "thiserror 2.0.20", "tokio", "tracing", "web-time", @@ -7257,7 +7401,7 @@ dependencies = [ "rustls", "rustls-pki-types", "slab", - "thiserror 2.0.18", + "thiserror 2.0.20", "tinyvec", "tracing", "web-time", @@ -7274,14 +7418,14 @@ dependencies = [ "once_cell", "socket2", "tracing", - "windows-sys 0.60.2", + "windows-sys 0.59.0", ] [[package]] name = "quote" -version = "1.0.46" +version = "1.0.47" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dfbc457d0c7a0759a614551b11a6409e5951f6c7537be1f1b7682b9ae9230368" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" dependencies = [ "proc-macro2", ] @@ -7430,7 +7574,7 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cbb175c433c8e28a809d1f5773a2ae96e68c0ce40db865cbab1020bf33ae479c" dependencies = [ - "bitflags 2.13.0", + "bitflags 2.13.1", "compact_str", "hashbrown 0.17.0", "itertools 0.14.0", @@ -7439,7 +7583,7 @@ dependencies = [ "palette", "serde", "strum 0.28.0", - "thiserror 2.0.18", + "thiserror 2.0.20", "unicode-segmentation", "unicode-truncate", "unicode-width", @@ -7463,7 +7607,7 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "66e3d19bcc9130ca376277d93b60767ff121ace3be06f5f95f81dd68956407d1" dependencies = [ - "bitflags 2.13.0", + "bitflags 2.13.1", "hashbrown 0.17.0", "indoc", "instability", @@ -7483,7 +7627,7 @@ version = "11.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "498cd0dc59d73224351ee52a95fee0f1a617a2eae0e7d9d720cc622c73a54186" dependencies = [ - "bitflags 2.13.0", + "bitflags 2.13.1", ] [[package]] @@ -7518,7 +7662,7 @@ version = "0.5.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" dependencies = [ - "bitflags 2.13.0", + "bitflags 2.13.1", ] [[package]] @@ -7527,7 +7671,7 @@ version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f450ad9c3b1da563fb6948a8e0fb0fb9269711c9c73d9ea1de5058c79c8d643a" dependencies = [ - "bitflags 2.13.0", + "bitflags 2.13.1", ] [[package]] @@ -7960,7 +8104,7 @@ dependencies = [ "secp256k1 0.30.0", "serde", "snmalloc-rs", - "thiserror 2.0.18", + "thiserror 2.0.20", "tikv-jemalloc-sys", "tikv-jemallocator", "tracy-client", @@ -8026,7 +8170,7 @@ dependencies = [ "auto_impl", "reth-execution-types", "reth-primitives-traits", - "thiserror 2.0.18", + "thiserror 2.0.20", ] [[package]] @@ -8098,7 +8242,7 @@ dependencies = [ "strum 0.27.2", "sysinfo", "tempfile", - "thiserror 2.0.18", + "thiserror 2.0.20", "tracing", ] @@ -8157,7 +8301,7 @@ dependencies = [ "reth-trie-db", "serde", "serde_json", - "thiserror 2.0.18", + "thiserror 2.0.20", "tracing", ] @@ -8198,7 +8342,7 @@ dependencies = [ "schnellru", "secp256k1 0.30.0", "serde", - "thiserror 2.0.18", + "thiserror 2.0.20", "tokio", "tokio-stream", "tracing", @@ -8224,7 +8368,7 @@ dependencies = [ "reth-network-peers", "reth-tracing", "secp256k1 0.30.0", - "thiserror 2.0.18", + "thiserror 2.0.20", "tokio", "tracing", ] @@ -8251,7 +8395,7 @@ dependencies = [ "secp256k1 0.30.0", "serde", "serde_with", - "thiserror 2.0.18", + "thiserror 2.0.20", "tokio", "tokio-stream", "tracing", @@ -8290,7 +8434,7 @@ dependencies = [ "reth-tracing", "reth-trie-common", "tempfile", - "thiserror 2.0.18", + "thiserror 2.0.20", "tokio", "tokio-stream", "tokio-util", @@ -8374,7 +8518,7 @@ dependencies = [ "reth-network-peers", "secp256k1 0.30.0", "sha2 0.10.9", - "thiserror 2.0.18", + "thiserror 2.0.20", "tokio", "tokio-stream", "tokio-util", @@ -8426,7 +8570,7 @@ dependencies = [ "reth-storage-api", "reth-trie-common", "serde", - "thiserror 2.0.18", + "thiserror 2.0.20", "tokio", ] @@ -8446,7 +8590,7 @@ dependencies = [ "derive_more", "eyre", "futures", - "indexmap 2.14.0", + "indexmap 2.14.2", "metrics", "metrics-util", "moka", @@ -8495,7 +8639,7 @@ dependencies = [ "revm", "schnellru", "serde_json", - "thiserror 2.0.18", + "thiserror 2.0.20", "tokio", "tracing", ] @@ -8550,7 +8694,7 @@ dependencies = [ "snap", "tempfile", "test-case", - "thiserror 2.0.18", + "thiserror 2.0.20", "tokio", ] @@ -8610,7 +8754,7 @@ dependencies = [ "reth-consensus", "reth-execution-errors", "reth-storage-errors", - "thiserror 2.0.18", + "thiserror 2.0.20", ] [[package]] @@ -8644,7 +8788,7 @@ dependencies = [ "serde", "snap", "test-fuzz", - "thiserror 2.0.18", + "thiserror 2.0.20", "tokio", "tokio-stream", "tokio-util", @@ -8676,7 +8820,7 @@ dependencies = [ "reth-primitives-traits", "serde", "test-case", - "thiserror 2.0.18", + "thiserror 2.0.20", ] [[package]] @@ -8771,7 +8915,7 @@ dependencies = [ "reth-primitives-traits", "serde", "serde_json", - "thiserror 2.0.18", + "thiserror 2.0.20", ] [[package]] @@ -8925,7 +9069,7 @@ dependencies = [ "alloy-rlp", "nybbles", "reth-storage-errors", - "thiserror 2.0.18", + "thiserror 2.0.20", ] [[package]] @@ -8988,7 +9132,7 @@ dependencies = [ "rmp-serde", "secp256k1 0.30.0", "tempfile", - "thiserror 2.0.18", + "thiserror 2.0.20", "tokio", "tokio-util", "tracing", @@ -9021,7 +9165,7 @@ dependencies = [ "reth-tasks", "reth-transaction-pool", "tempfile", - "thiserror 2.0.18", + "thiserror 2.0.20", "tokio", ] @@ -9048,7 +9192,7 @@ version = "2.5.2" dependencies = [ "serde", "serde_json", - "thiserror 2.0.18", + "thiserror 2.0.20", ] [[package]] @@ -9096,7 +9240,7 @@ dependencies = [ "reth-tracing", "serde", "serde_json", - "thiserror 2.0.18", + "thiserror 2.0.20", "tokio", "tokio-stream", "tokio-util", @@ -9108,7 +9252,7 @@ dependencies = [ name = "reth-libmdbx" version = "2.5.2" dependencies = [ - "bitflags 2.13.0", + "bitflags 2.13.1", "byteorder", "crossbeam-queue", "dashmap", @@ -9117,7 +9261,7 @@ dependencies = [ "reth-mdbx-sys", "smallvec", "tempfile", - "thiserror 2.0.18", + "thiserror 2.0.20", "tracing", ] @@ -9159,7 +9303,7 @@ dependencies = [ "reqwest", "reth-tracing", "serde_with", - "thiserror 2.0.18", + "thiserror 2.0.20", "tokio", "tracing", ] @@ -9225,7 +9369,7 @@ dependencies = [ "smallvec", "socket2", "test-case", - "thiserror 2.0.18", + "thiserror 2.0.20", "tokio", "tokio-stream", "tokio-util", @@ -9252,7 +9396,7 @@ dependencies = [ "reth-network-types", "reth-tokio-util", "serde", - "thiserror 2.0.18", + "thiserror 2.0.20", "tokio", "tokio-stream", ] @@ -9293,7 +9437,7 @@ dependencies = [ "secp256k1 0.30.0", "serde_json", "serde_with", - "thiserror 2.0.18", + "thiserror 2.0.20", "tokio", "url", ] @@ -9325,7 +9469,7 @@ dependencies = [ "serde", "smallvec", "tempfile", - "thiserror 2.0.18", + "thiserror 2.0.20", "tracing", "zstd", ] @@ -9471,7 +9615,7 @@ dependencies = [ "secp256k1 0.30.0", "serde", "strum 0.27.2", - "thiserror 2.0.18", + "thiserror 2.0.20", "tokio", "toml", "tracing", @@ -9566,7 +9710,7 @@ dependencies = [ "reth-transaction-pool", "serde", "serde_json", - "thiserror 2.0.18", + "thiserror 2.0.20", "tokio", "tokio-stream", "tokio-tungstenite", @@ -9692,7 +9836,7 @@ dependencies = [ "serde", "serde_json", "sha2 0.10.9", - "thiserror 2.0.18", + "thiserror 2.0.20", "tokio", ] @@ -9744,7 +9888,7 @@ dependencies = [ "revm-state", "secp256k1 0.30.0", "serde", - "thiserror 2.0.18", + "thiserror 2.0.20", ] [[package]] @@ -9828,7 +9972,7 @@ dependencies = [ "reth-tokio-util", "reth-tracing", "rustc-hash", - "thiserror 2.0.18", + "thiserror 2.0.20", "tokio", "tracing", ] @@ -9848,7 +9992,7 @@ dependencies = [ "serde", "serde_json", "strum 0.27.2", - "thiserror 2.0.18", + "thiserror 2.0.20", "toml", "tracing", ] @@ -9942,7 +10086,7 @@ dependencies = [ "serde", "serde_json", "sha2 0.10.9", - "thiserror 2.0.18", + "thiserror 2.0.20", "tokio", "tokio-stream", "tracing", @@ -10029,7 +10173,7 @@ dependencies = [ "reth-transaction-pool", "serde", "serde_json", - "thiserror 2.0.18", + "thiserror 2.0.20", "tokio", "tokio-util", "tower", @@ -10054,7 +10198,7 @@ dependencies = [ "reth-primitives-traits", "reth-rpc-traits", "serde_json", - "thiserror 2.0.18", + "thiserror 2.0.20", ] [[package]] @@ -10108,7 +10252,7 @@ dependencies = [ "reth-testing-utils", "reth-transaction-pool", "serde", - "thiserror 2.0.18", + "thiserror 2.0.20", "tokio", "tracing", ] @@ -10203,7 +10347,7 @@ dependencies = [ "schnellru", "serde", "serde_json", - "thiserror 2.0.18", + "thiserror 2.0.20", "tokio", "tokio-stream", "tracing", @@ -10257,7 +10401,7 @@ dependencies = [ "alloy-rpc-types-eth", "alloy-signer", "reth-primitives-traits", - "thiserror 2.0.18", + "thiserror 2.0.20", ] [[package]] @@ -10272,7 +10416,7 @@ dependencies = [ "reth-provider", "reth-storage-api", "reth-storage-errors", - "thiserror 2.0.18", + "thiserror 2.0.20", "tokio-util", ] @@ -10334,7 +10478,7 @@ dependencies = [ "reth-trie", "reth-trie-db", "tempfile", - "thiserror 2.0.18", + "thiserror 2.0.20", "tokio", "tracing", ] @@ -10367,7 +10511,7 @@ dependencies = [ "reth-tasks", "reth-testing-utils", "reth-tokio-util", - "thiserror 2.0.18", + "thiserror 2.0.20", "tokio", "tokio-stream", "tracing", @@ -10470,7 +10614,7 @@ dependencies = [ "reth-prune-types", "reth-static-file-types", "revm", - "thiserror 2.0.18", + "thiserror 2.0.20", ] [[package]] @@ -10545,7 +10689,7 @@ dependencies = [ "pin-project", "rayon", "reth-metrics", - "thiserror 2.0.18", + "thiserror 2.0.20", "thread-priority", "tokio", "tracing", @@ -10625,7 +10769,7 @@ dependencies = [ "aquamarine", "assert_matches", "auto_impl", - "bitflags 2.13.0", + "bitflags 2.13.1", "codspeed-criterion-compat", "futures", "futures-util", @@ -10659,7 +10803,7 @@ dependencies = [ "serde_json", "smallvec", "tempfile", - "thiserror 2.0.18", + "thiserror 2.0.20", "tokio", "tokio-stream", "tracing", @@ -10727,7 +10871,7 @@ dependencies = [ "serde", "serde_json", "serde_with", - "thiserror 2.0.18", + "thiserror 2.0.20", ] [[package]] @@ -10779,7 +10923,7 @@ dependencies = [ "reth-tasks", "reth-trie", "revm", - "thiserror 2.0.18", + "thiserror 2.0.20", "tokio", "tracing", ] @@ -10914,7 +11058,7 @@ dependencies = [ "revm-primitives", "revm-state", "serde", - "thiserror 2.0.18", + "thiserror 2.0.20", ] [[package]] @@ -10971,7 +11115,7 @@ dependencies = [ "revm", "serde", "serde_json", - "thiserror 2.0.18", + "thiserror 2.0.20", ] [[package]] @@ -11032,7 +11176,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7cd02082cf1baca01e8bc17456c45136bdc66632395a98db1c1a2afe4c6ea87f" dependencies = [ "alloy-eip7928", - "bitflags 2.13.0", + "bitflags 2.13.1", "nonmax", "revm-bytecode", "revm-primitives", @@ -11094,11 +11238,11 @@ version = "0.1.0" source = "git+https://github.com/paradigmxyz/revmc?branch=main#68835cf19661307d293c26dd06bd1e59ccae5144" dependencies = [ "alloy-primitives", - "bitflags 2.13.0", + "bitflags 2.13.1", "bitvec", "derive_more", "either", - "indexmap 2.14.0", + "indexmap 2.14.2", "oxc_index", "revm-bytecode", "revm-context", @@ -11272,9 +11416,9 @@ dependencies = [ [[package]] name = "roaring" -version = "0.11.4" +version = "0.11.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1dedc5658c6ecb3bdb5ef5f3295bb9253f42dcf3fd1402c03f6b1f7659c3c4a9" +checksum = "18bd8a37d17a58532776dcdf6041ce64929adca78e8489d5cacbafe99229d3e1" dependencies = [ "bytemuck", "byteorder", @@ -11395,7 +11539,7 @@ version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" dependencies = [ - "bitflags 2.13.0", + "bitflags 2.13.1", "errno", "libc", "linux-raw-sys", @@ -11666,7 +11810,7 @@ version = "3.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" dependencies = [ - "bitflags 2.13.0", + "bitflags 2.13.1", "core-foundation 0.10.1", "core-foundation-sys", "libc", @@ -11740,9 +11884,9 @@ checksum = "cd0b0ec5f1c1ca621c432a25813d8d60c88abe6d3e08a3eb9cf37d97a0fe3d73" [[package]] name = "serde" -version = "1.0.228" +version = "1.0.229" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" dependencies = [ "serde_core", "serde_derive", @@ -11759,31 +11903,31 @@ dependencies = [ [[package]] name = "serde_core" -version = "1.0.228" +version = "1.0.229" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.228" +version = "1.0.229" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 3.0.3", ] [[package]] name = "serde_json" -version = "1.0.150" +version = "1.0.151" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" dependencies = [ - "indexmap 2.14.0", + "indexmap 2.14.2", "itoa", "memchr", "serde", @@ -11814,16 +11958,17 @@ dependencies = [ [[package]] name = "serde_with" -version = "3.21.0" +version = "3.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76a5c54c7310e7b8b9577c286d7e399ddd876c3e12b3ed917a8aabc4b96e9e8c" +checksum = "935177bb8c0cd8ca1a4e6d1a2ac8988bea69cab4f9d3a31311e012ad27868ea4" dependencies = [ - "base64 0.22.1", + "base64 0.23.1", "bs58", "chrono", "hex", "indexmap 1.9.3", - "indexmap 2.14.0", + "indexmap 2.14.2", + "jiff", "schemars 0.9.0", "schemars 1.2.1", "serde_core", @@ -11834,14 +11979,14 @@ dependencies = [ [[package]] name = "serde_with_macros" -version = "3.21.0" +version = "3.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84d57bc0c8b9a17920c178daa6bb924850d54a9c97ab45194bb8c17ad66bb660" +checksum = "1d607aa01a3cb0ad757d6fd216136910db3c97b102fe686585689615a02dbcdc" dependencies = [ - "darling 0.23.0", + "darling 0.24.1", "proc-macro2", "quote", - "syn 2.0.118", + "syn 3.0.3", ] [[package]] @@ -12030,7 +12175,7 @@ checksum = "0d585997b0ac10be3c5ee635f1bab02d512760d14b7c468801ac8a01d9ae5f1d" dependencies = [ "num-bigint", "num-traits", - "thiserror 2.0.18", + "thiserror 2.0.20", "time", ] @@ -12082,9 +12227,9 @@ dependencies = [ [[package]] name = "snap" -version = "1.1.1" +version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b6b67fb9a61334225b5b790716f609cd58395f895b3fe8b328786812a40bc3b" +checksum = "199905e6153d6405f9728fe44daace35f8f837bbf830bb6e85fbd5828709a886" [[package]] name = "snmalloc-rs" @@ -12107,9 +12252,9 @@ dependencies = [ [[package]] name = "socket2" -version = "0.6.4" +version = "0.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52d1cfed4120b4d927bf7c0f86d2087a4a7d6027c906d9f9d525a80573b9be51" +checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4" dependencies = [ "libc", "windows-sys 0.61.2", @@ -12267,9 +12412,9 @@ dependencies = [ [[package]] name = "syn-solidity" -version = "1.6.1" +version = "1.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "083be3061e64d362cbe6ef12cfe1307ba3884326d8856448fe8a120fa2c44ebf" +checksum = "4c6415502cd1e9ed58b3ceb415164b812d5572757b1a6f0e280ae806723c1fab" dependencies = [ "paste", "proc-macro2", @@ -12317,7 +12462,7 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a13f3d0daba03132c0aa9767f98351b3488edc2c100cda2d2ec2b04f3d8d3c8b" dependencies = [ - "bitflags 2.13.0", + "bitflags 2.13.1", "core-foundation 0.9.4", "system-configuration-sys", ] @@ -12476,11 +12621,11 @@ dependencies = [ [[package]] name = "thiserror" -version = "2.0.18" +version = "2.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" +checksum = "ec86235f5fcc2a73650310756d2ac5b138a5780bbbdfae3eeccec992c435ba4f" dependencies = [ - "thiserror-impl 2.0.18", + "thiserror-impl 2.0.20", ] [[package]] @@ -12496,13 +12641,13 @@ dependencies = [ [[package]] name = "thiserror-impl" -version = "2.0.18" +version = "2.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" +checksum = "bc04cd3e1236dd4a98afca4569f2deb3f120e5422a4023be2cb683f8486292af" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 3.0.3", ] [[package]] @@ -12511,7 +12656,7 @@ version = "3.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8d2e834949be5111506bb252643498af1514f600d9e1dceedaa42afae155b67f" dependencies = [ - "bitflags 2.13.0", + "bitflags 2.13.1", "cfg-if", "libc", "log", @@ -12638,9 +12783,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.52.3" +version = "1.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fc7f01b389ac15039e4dc9531aa973a135d7a4135281b12d7c1bc79fd57fffe" +checksum = "202caea871b69668250d242070849eb495be178ed697a3e98aebce5bc81a0bed" dependencies = [ "bytes", "libc", @@ -12676,9 +12821,9 @@ dependencies = [ [[package]] name = "tokio-stream" -version = "0.1.18" +version = "0.1.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32da49809aab5c3bc678af03902d4ccddea2a87d028d86392a4b1560c6906c70" +checksum = "a3d06f0b082ba57c26b79407372e57cf2a1e28124f78e9479fe80322cf53420b" dependencies = [ "futures-core", "pin-project-lite", @@ -12705,14 +12850,15 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.18" +version = "0.7.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ae9cec805b01e8fc3fd2fe289f89149a9b66dd16786abd8b19cfa7b48cb0098" +checksum = "494815d09bf52b5548659851081238f0ca39ff638363907596da739561c62c52" dependencies = [ "bytes", "futures-core", "futures-io", "futures-sink", + "libc", "pin-project-lite", "slab", "tokio", @@ -12724,7 +12870,7 @@ version = "0.9.12+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf92845e79fc2e2def6a5d828f0801e29a2f8acc037becc5ab08595c7d5e9863" dependencies = [ - "indexmap 2.14.0", + "indexmap 2.14.2", "serde_core", "serde_spanned", "toml_datetime 0.7.5+spec-1.1.0", @@ -12757,7 +12903,7 @@ version = "0.25.11+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b59c4d22ed448339746c59b905d24568fcbb3ab65a500494f7b8c3e97739f2b" dependencies = [ - "indexmap 2.14.0", + "indexmap 2.14.2", "toml_datetime 1.1.1+spec-1.1.0", "toml_parser", "winnow 1.0.1", @@ -12835,7 +12981,7 @@ dependencies = [ "futures-core", "futures-util", "hdrhistogram", - "indexmap 2.14.0", + "indexmap 2.14.2", "pin-project-lite", "slab", "sync_wrapper", @@ -12854,7 +13000,7 @@ checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" dependencies = [ "async-compression", "base64 0.22.1", - "bitflags 2.13.0", + "bitflags 2.13.1", "bytes", "futures-core", "futures-util", @@ -12909,7 +13055,7 @@ checksum = "050686193eb999b4bb3bc2acfa891a13da00f79734704c4b8b4ef1a10b368a3c" dependencies = [ "crossbeam-channel", "symlink", - "thiserror 2.0.18", + "thiserror 2.0.20", "time", "tracing-subscriber", ] @@ -13142,7 +13288,7 @@ dependencies = [ "rustls", "rustls-pki-types", "sha1", - "thiserror 2.0.18", + "thiserror 2.0.20", ] [[package]] @@ -13496,7 +13642,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bb0e353e6a2fbdc176932bbaab493762eb1255a7900fe0fea1a2f96c296cc909" dependencies = [ "anyhow", - "indexmap 2.14.0", + "indexmap 2.14.2", "wasm-encoder", "wasmparser", ] @@ -13520,9 +13666,9 @@ version = "0.244.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe" dependencies = [ - "bitflags 2.13.0", + "bitflags 2.13.1", "hashbrown 0.15.5", - "indexmap 2.14.0", + "indexmap 2.14.2", "semver 1.0.28", ] @@ -13652,7 +13798,7 @@ dependencies = [ "pastey", "proc-macro2", "quote", - "thiserror 2.0.18", + "thiserror 2.0.20", "wincode-derive", ] @@ -14069,7 +14215,7 @@ checksum = "b7c566e0f4b284dd6561c786d9cb0142da491f46a9fbed79ea69cdad5db17f21" dependencies = [ "anyhow", "heck", - "indexmap 2.14.0", + "indexmap 2.14.2", "prettyplease", "syn 2.0.118", "wasm-metadata", @@ -14099,8 +14245,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2" dependencies = [ "anyhow", - "bitflags 2.13.0", - "indexmap 2.14.0", + "bitflags 2.13.1", + "indexmap 2.14.2", "log", "serde", "serde_derive", @@ -14119,7 +14265,7 @@ checksum = "ecc8ac4bc1dc3381b7f59c34f00b67e18f910c2c0f50015669dde7def656a736" dependencies = [ "anyhow", "id-arena", - "indexmap 2.14.0", + "indexmap 2.14.2", "log", "semver 1.0.28", "serde", @@ -14154,7 +14300,7 @@ dependencies = [ "pharos", "rustc_version 0.4.1", "send_wrapper 0.6.0", - "thiserror 2.0.18", + "thiserror 2.0.20", "wasm-bindgen", "wasm-bindgen-futures", "web-sys", @@ -14310,6 +14456,12 @@ dependencies = [ "syn 2.0.118", ] +[[package]] +name = "zlib-rs" +version = "0.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34b31d188d9d685a4f9c7b46d6e36631b07058d2cfe190267adce54dc230bf12" + [[package]] name = "zmij" version = "1.0.21" diff --git a/docs/vocs/docs/pages/cli/reth/node.mdx b/docs/vocs/docs/pages/cli/reth/node.mdx index 2d82db99149..c88b071e9c9 100644 --- a/docs/vocs/docs/pages/cli/reth/node.mdx +++ b/docs/vocs/docs/pages/cli/reth/node.mdx @@ -444,7 +444,7 @@ RPC: Set the maximum RPC response payload size for both HTTP and WS in megabytes [default: 160] - [aliases: --rpc.returndata.limit] + [alias: --rpc.returndata.limit] --rpc.max-subscriptions-per-connection Set the maximum concurrent subscriptions per connection From b7331716b1f9f8bc32cfdd8465a875a87b127ad1 Mon Sep 17 00:00:00 2001 From: Oliver Tipton Date: Tue, 8 Sep 2026 20:23:59 +0000 Subject: [PATCH 215/294] fix(rpc): reject raw BidTrace mismatches before KZG (#27026) --- crates/ethereum/node/tests/e2e/rpc.rs | 21 +++- crates/rpc/rpc/src/validation.rs | 138 +++++++++++++++++++++++++- 2 files changed, 151 insertions(+), 8 deletions(-) diff --git a/crates/ethereum/node/tests/e2e/rpc.rs b/crates/ethereum/node/tests/e2e/rpc.rs index b818fc88d0e..0182d842ce8 100644 --- a/crates/ethereum/node/tests/e2e/rpc.rs +++ b/crates/ethereum/node/tests/e2e/rpc.rs @@ -588,7 +588,7 @@ async fn test_flashbots_validate_v5() -> eyre::Result<()> { let envelope = payload.clone().try_into_v5()?; assert!(!envelope.blobs_bundle.blobs.is_empty()); - let mut request = BuilderBlockValidationRequestV5 { + let request = BuilderBlockValidationRequestV5 { request: SignedBidSubmissionV5 { message: BidTrace { parent_hash: payload.block().parent_hash, @@ -611,13 +611,28 @@ async fn test_flashbots_validate_v5() -> eyre::Result<()> { .await .expect("request should validate"); - request.request.blobs_bundle.proofs[0] = request.request.blobs_bundle.proofs[1]; + let mut invalid_proof_request = request.clone(); + invalid_proof_request.request.blobs_bundle.proofs[0] = + invalid_proof_request.request.blobs_bundle.proofs[1]; let err = provider - .raw_request::<_, ()>("flashbots_validateBuilderSubmissionV5".into(), (&request,)) + .raw_request::<_, ()>( + "flashbots_validateBuilderSubmissionV5".into(), + (&invalid_proof_request,), + ) .await .unwrap_err(); assert!(err.to_string().contains("invalid KZG proof"), "{err}"); + invalid_proof_request.request.message.block_hash = B256::ZERO; + let err = provider + .raw_request::<_, ()>( + "flashbots_validateBuilderSubmissionV5".into(), + (&invalid_proof_request,), + ) + .await + .unwrap_err(); + assert!(err.to_string().contains("block hash mismatch"), "{err}"); + Ok(()) } diff --git a/crates/rpc/rpc/src/validation.rs b/crates/rpc/rpc/src/validation.rs index 90b994a2ebe..7468a994011 100644 --- a/crates/rpc/rpc/src/validation.rs +++ b/crates/rpc/rpc/src/validation.rs @@ -427,8 +427,11 @@ where &self, request: BuilderBlockValidationRequestV5, ) -> Result<(), ValidationApiError> { + let payload = ExecutionPayload::V3(request.request.execution_payload); + validate_message_against_payload(&request.request.message, &payload)?; + let block = self.payload_validator.ensure_well_formed_payload(ExecutionData { - payload: ExecutionPayload::V3(request.request.execution_payload), + payload, sidecar: ExecutionPayloadSidecar::v4( CancunPayloadFields { parent_beacon_block_root: request.parent_beacon_block_root, @@ -469,12 +472,15 @@ where &self, request: BuilderBlockValidationRequestV6, ) -> Result<(), ValidationApiError> { + let payload = ExecutionPayload::V4(request.request.execution_payload); + validate_message_against_payload(&request.request.message, &payload)?; + let decoded_bal = - DecodedBal::from_rlp_bytes(request.request.execution_payload.block_access_list.clone()) + DecodedBal::from_rlp_bytes(payload.as_v4().unwrap().block_access_list.clone()) .map_err(ValidationApiError::InvalidBlockAccessList)?; let block = self.payload_validator.ensure_well_formed_payload(ExecutionData { - payload: ExecutionPayload::V4(request.request.execution_payload), + payload, sidecar: ExecutionPayloadSidecar::v4( CancunPayloadFields { parent_beacon_block_root: request.parent_beacon_block_root, @@ -635,6 +641,38 @@ pub struct ValidationApiInner { metrics: ValidationMetrics, } +/// Ensures that the raw execution payload fields match the corresponding [`BidTrace`] fields. +fn validate_message_against_payload( + message: &BidTrace, + payload: &ExecutionPayload, +) -> Result<(), ValidationApiError> { + let payload = payload.as_v1(); + + if payload.block_hash != message.block_hash { + Err(ValidationApiError::BlockHashMismatch(GotExpected { + got: message.block_hash, + expected: payload.block_hash, + })) + } else if payload.parent_hash != message.parent_hash { + Err(ValidationApiError::ParentHashMismatch(GotExpected { + got: message.parent_hash, + expected: payload.parent_hash, + })) + } else if payload.gas_limit != message.gas_limit { + Err(ValidationApiError::GasLimitMismatch(GotExpected { + got: message.gas_limit, + expected: payload.gas_limit, + })) + } else if payload.gas_used != message.gas_used { + Err(ValidationApiError::GasUsedMismatch(GotExpected { + got: message.gas_used, + expected: payload.gas_used, + })) + } else { + Ok(()) + } +} + /// Calculates a deterministic hash of the blocklist for change detection. /// /// This function sorts addresses to ensure deterministic output regardless of @@ -760,8 +798,98 @@ pub(crate) struct ValidationMetrics { #[cfg(test)] mod tests { - use super::{hash_disallow_list, AddressSet}; - use alloy_primitives::Address; + use super::{ + hash_disallow_list, validate_message_against_payload, AddressSet, ValidationApiError, + }; + use alloy_primitives::{Address, B256}; + use alloy_rpc_types_beacon::relay::BidTrace; + use alloy_rpc_types_engine::{ExecutionPayload, ExecutionPayloadV1}; + + fn test_execution_payload() -> ExecutionPayload { + ExecutionPayload::V1(ExecutionPayloadV1 { + parent_hash: B256::repeat_byte(0x11), + fee_recipient: Address::ZERO, + state_root: B256::ZERO, + receipts_root: B256::ZERO, + logs_bloom: Default::default(), + prev_randao: B256::ZERO, + block_number: 1, + gas_limit: 30_000_000, + gas_used: 15_000_000, + timestamp: 1, + extra_data: Default::default(), + base_fee_per_gas: Default::default(), + block_hash: B256::repeat_byte(0x22), + transactions: Default::default(), + }) + } + + fn matching_bid_trace(payload: &ExecutionPayload) -> BidTrace { + let payload = payload.as_v1(); + BidTrace { + parent_hash: payload.parent_hash, + block_hash: payload.block_hash, + gas_limit: payload.gas_limit, + gas_used: payload.gas_used, + ..Default::default() + } + } + + #[test] + fn test_validate_message_against_payload_block_hash_mismatch() { + let payload = test_execution_payload(); + let mut message = matching_bid_trace(&payload); + message.block_hash = B256::repeat_byte(0x33); + + let err = validate_message_against_payload(&message, &payload).unwrap_err(); + let ValidationApiError::BlockHashMismatch(mismatch) = err else { + panic!("unexpected error: {err}") + }; + assert_eq!(mismatch.got, message.block_hash); + assert_eq!(mismatch.expected, payload.block_hash()); + } + + #[test] + fn test_validate_message_against_payload_parent_hash_mismatch() { + let payload = test_execution_payload(); + let mut message = matching_bid_trace(&payload); + message.parent_hash = B256::repeat_byte(0x33); + + let err = validate_message_against_payload(&message, &payload).unwrap_err(); + let ValidationApiError::ParentHashMismatch(mismatch) = err else { + panic!("unexpected error: {err}") + }; + assert_eq!(mismatch.got, message.parent_hash); + assert_eq!(mismatch.expected, payload.parent_hash()); + } + + #[test] + fn test_validate_message_against_payload_gas_limit_mismatch() { + let payload = test_execution_payload(); + let mut message = matching_bid_trace(&payload); + message.gas_limit += 1; + + let err = validate_message_against_payload(&message, &payload).unwrap_err(); + let ValidationApiError::GasLimitMismatch(mismatch) = err else { + panic!("unexpected error: {err}") + }; + assert_eq!(mismatch.got, message.gas_limit); + assert_eq!(mismatch.expected, payload.gas_limit()); + } + + #[test] + fn test_validate_message_against_payload_gas_used_mismatch() { + let payload = test_execution_payload(); + let mut message = matching_bid_trace(&payload); + message.gas_used += 1; + + let err = validate_message_against_payload(&message, &payload).unwrap_err(); + let ValidationApiError::GasUsedMismatch(mismatch) = err else { + panic!("unexpected error: {err}") + }; + assert_eq!(mismatch.got, message.gas_used); + assert_eq!(mismatch.expected, payload.as_v1().gas_used); + } #[test] fn test_hash_disallow_list_deterministic() { From a70cde4c76d8ed4c1c904e00d52533ed5a157559 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Wed, 9 Sep 2026 11:19:02 +0200 Subject: [PATCH 216/294] fix(rpc): suppress synthetic transfer logs when EIP-7708 is active (#27110) --- crates/ethereum/node/tests/e2e/simulate.rs | 77 ++++++++++++++++++++++ crates/rpc/rpc-eth-api/src/helpers/call.rs | 6 ++ 2 files changed, 83 insertions(+) diff --git a/crates/ethereum/node/tests/e2e/simulate.rs b/crates/ethereum/node/tests/e2e/simulate.rs index 19f0d7290dd..49367310c77 100644 --- a/crates/ethereum/node/tests/e2e/simulate.rs +++ b/crates/ethereum/node/tests/e2e/simulate.rs @@ -12,6 +12,83 @@ use reth_e2e_test_utils::setup_engine; use reth_node_ethereum::EthereumNode; use std::sync::Arc; +#[tokio::test] +async fn test_simulate_v1_transfer_logs_across_amsterdam() -> eyre::Result<()> { + let chain_spec = Arc::new( + ChainSpecBuilder::default() + .chain(MAINNET.chain) + .genesis(serde_json::from_str(include_str!("../assets/genesis.json"))?) + .osaka_activated() + .with_amsterdam_at(24) + .build(), + ); + let (mut nodes, _) = setup_engine::( + 1, + chain_spec, + false, + Default::default(), + eth_payload_attributes_amsterdam, + ) + .await?; + let node = nodes.pop().unwrap(); + let provider = ProviderBuilder::new().connect_http(node.rpc_url()); + let from = Address::with_last_byte(0x41); + let to = Address::with_last_byte(0x42); + let reverting = Address::with_last_byte(0x43); + let call_block = SimBlock::default() + .with_state_overrides(StateOverride::from_iter([ + (from, AccountOverride::default().with_balance(U256::from(100)).with_nonce(0)), + (reverting, AccountOverride::default().with_code(bytes!("5f5ffd"))), + ])) + .call(TransactionRequest::default().from(from).to(to).value(U256::from(1))) + .call(TransactionRequest::default().from(from).to(reverting).value(U256::from(1))); + let mut untraced: Option> = None; + for trace_transfers in [false, true] { + let mut payload = SimulatePayload::default() + .extend(call_block.clone()) + .extend(call_block.clone()) + .extend(call_block.clone()); + payload.trace_transfers = trace_transfers; + let result: Vec = + provider.raw_request("eth_simulateV1".into(), (&payload, "latest")).await?; + assert_eq!(result.len(), 3); + for (index, block) in result.iter().enumerate() { + assert_eq!(block.inner.header.timestamp, (index as u64 + 1) * 12); + assert!(block.calls[0].status); + let logs = &block.calls[0].logs; + if index == 0 && !trace_transfers { + assert!(logs.is_empty()); + } else { + assert_eq!(logs.len(), 1); + let emitter = if index == 0 { + Address::repeat_byte(0xee) + } else { + revm::primitives::eip7708::ETH_TRANSFER_LOG_ADDRESS + }; + assert_eq!(logs[0].address(), emitter); + } + assert!(!block.calls[1].status); + assert!(block.calls[1].logs.is_empty()); + if index > 0 { + assert!(!block.inner.header.logs_bloom.is_zero()); + if let Some(untraced) = &untraced { + assert_eq!(logs[0].inner, untraced[index].calls[0].logs[0].inner); + assert_eq!( + block.inner.header.logs_bloom, + untraced[index].inner.header.logs_bloom + ); + assert_eq!( + block.inner.header.receipts_root, + untraced[index].inner.header.receipts_root + ); + } + } + } + untraced = Some(result); + } + Ok(()) +} + #[tokio::test] async fn test_simulate_v1_block_access_list_hash_across_amsterdam() -> eyre::Result<()> { reth_tracing::init_test_tracing(); diff --git a/crates/rpc/rpc-eth-api/src/helpers/call.rs b/crates/rpc/rpc-eth-api/src/helpers/call.rs index 3cbe6c5e3d5..2cf99fe6bd5 100644 --- a/crates/rpc/rpc-eth-api/src/helpers/call.rs +++ b/crates/rpc/rpc-eth-api/src/helpers/call.rs @@ -212,6 +212,12 @@ pub trait EthCall: EstimateCall + Call + LoadPendingBlock + LoadBlock + FullEthA } }; + // EIP-7708 already emits transfer logs: https://eips.ethereum.org/EIPS/eip-7708 + let trace_transfers = trace_transfers && + (!Cfg::spec(&evm_env.cfg_env) + .into() + .is_enabled_in(revm::primitives::hardfork::SpecId::AMSTERDAM) || + evm_env.cfg_env.is_eip7708_disabled()); let (result, results) = if trace_transfers { // prepare inspector to capture transfer inside the evm so they are recorded // and included in logs From a7ee2a5659dc33f8abb6c9f971a09946c66da384 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A9a=20Narzis?= <78718413+lean-apple@users.noreply.github.com> Date: Wed, 9 Sep 2026 11:30:40 +0000 Subject: [PATCH 217/294] feat(snap-sync): track download attempts and prevent premature completion (#27108) Co-authored-by: Matthias Seitz --- Cargo.lock | 4 + crates/snap-sync/Cargo.toml | 5 +- crates/snap-sync/src/attempt.rs | 339 ++++++++++++++++++ crates/snap-sync/src/error.rs | 15 + crates/snap-sync/src/lib.rs | 5 + crates/storage/db-api/Cargo.toml | 2 + crates/storage/db-api/src/models/mod.rs | 2 + crates/storage/db-api/src/models/snap.rs | 135 +++++++ crates/storage/errors/src/provider.rs | 16 + .../src/providers/database/provider.rs | 152 +++++++- crates/storage/storage-api/src/lib.rs | 2 +- crates/storage/storage-api/src/metadata.rs | 35 +- 12 files changed, 707 insertions(+), 5 deletions(-) create mode 100644 crates/snap-sync/src/attempt.rs create mode 100644 crates/storage/db-api/src/models/snap.rs diff --git a/Cargo.lock b/Cargo.lock index fcbee8dfa26..133785dd1cd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8251,6 +8251,7 @@ name = "reth-db-api" version = "2.5.2" dependencies = [ "alloy-consensus", + "alloy-eips", "alloy-primitives", "arbitrary", "arrayvec", @@ -10412,10 +10413,13 @@ dependencies = [ "alloy-eip7928", "alloy-eips", "alloy-primitives", + "reth-db-api", "reth-primitives-traits", "reth-provider", "reth-storage-api", "reth-storage-errors", + "reth-trie-common", + "revm", "thiserror 2.0.20", "tokio-util", ] diff --git a/crates/snap-sync/Cargo.toml b/crates/snap-sync/Cargo.toml index 0df1d88842d..93585aac909 100644 --- a/crates/snap-sync/Cargo.toml +++ b/crates/snap-sync/Cargo.toml @@ -14,7 +14,7 @@ workspace = true [dependencies] # reth reth-primitives-traits.workspace = true -reth-storage-api.workspace = true +reth-storage-api = { workspace = true, features = ["db-api"] } reth-storage-errors.workspace = true # alloy @@ -28,4 +28,7 @@ tokio-util.workspace = true [dev-dependencies] alloy-consensus.workspace = true +reth-db-api.workspace = true reth-provider = { workspace = true, features = ["test-utils"] } +reth-trie-common.workspace = true +revm.workspace = true diff --git a/crates/snap-sync/src/attempt.rs b/crates/snap-sync/src/attempt.rs new file mode 100644 index 00000000000..270e4e1bfe1 --- /dev/null +++ b/crates/snap-sync/src/attempt.rs @@ -0,0 +1,339 @@ +//! Persistence for the attempt that owns downloaded snap state. +//! +//! Snap writes land in the canonical hashed state tables, so nothing in the data says which +//! attempt produced it. Every write presents a [`SnapWrite`]; ones that no longer match are +//! refused. + +use crate::{SnapGeneration, SnapSyncError}; +use reth_storage_api::{ + MetadataProvider, MetadataWriter, SnapAttempt, SnapAttemptId, StorageSettings, +}; + +/// Persistence for the attempt that owns downloaded snap state. +/// +/// Blanket-implemented over node metadata access, so these writes join the caller's transaction: +/// state, bytecode and the attempt record commit together or not at all. +pub trait SnapAttemptStore { + /// Starts an attempt anchored to `generation`, superseding any already recorded. + fn start_snap_attempt(&self, generation: SnapGeneration) -> Result; + + /// Returns the write an unfinished attempt accepts, if one owns the persisted state. + fn active_snap_write(&self) -> Result, SnapSyncError>; + + /// Returns the attempt when `write` still owns the persisted state, rejecting it otherwise. + fn authorize_snap_write(&self, write: SnapWrite) -> Result; + + /// Re-anchors the attempt to `generation`, refusing writes proved against the previous root. + fn advance_snap_pivot( + &self, + write: SnapWrite, + generation: SnapGeneration, + ) -> Result; + + /// Marks the attempt's downloaded state verified. + fn verify_snap_attempt(&self, write: SnapWrite) -> Result<(), SnapSyncError>; + + /// Gives up on an unfinished attempt, refusing its outstanding writes. + fn abandon_snap_attempt(&self) -> Result<(), SnapSyncError>; +} + +/// What a write presents to prove it belongs to the attempt owning the persisted state. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct SnapWrite { + // Attempt this write belongs to. + attempt: SnapAttemptId, + // Pivot generation within that attempt. + state_version: u64, +} + +impl SnapWrite { + // What `attempt` currently accepts. + const fn of(attempt: &SnapAttempt) -> Self { + Self { attempt: attempt.id(), state_version: attempt.state_version() } + } + + /// Attempt this write belongs to. + pub const fn attempt(&self) -> SnapAttemptId { + self.attempt + } + + /// Pivot generation this write was proved against. + pub const fn state_version(&self) -> u64 { + self.state_version + } +} + +impl SnapAttemptStore for T +where + T: MetadataProvider + MetadataWriter, +{ + fn start_snap_attempt(&self, generation: SnapGeneration) -> Result { + // Absent settings mean the legacy layout. + if !self.storage_settings()?.unwrap_or_else(StorageSettings::v1).use_hashed_state() { + return Err(SnapSyncError::UnsupportedStorage) + } + + let attempt = + SnapAttempt::start(self.snap_attempt()?, generation.target(), generation.state_root()); + self.write_snap_attempt(&attempt)?; + Ok(SnapWrite::of(&attempt)) + } + + fn active_snap_write(&self) -> Result, SnapSyncError> { + Ok(self.snap_attempt()?.filter(SnapAttempt::is_unfinished).as_ref().map(SnapWrite::of)) + } + + fn authorize_snap_write(&self, write: SnapWrite) -> Result { + let attempt = self.snap_attempt()?.ok_or(SnapSyncError::NoAttempt)?; + if !attempt.is_unfinished() || SnapWrite::of(&attempt) != write { + return Err(SnapSyncError::StaleWrite { + attempt: write.attempt, + state_version: write.state_version, + }) + } + Ok(attempt) + } + + fn advance_snap_pivot( + &self, + write: SnapWrite, + generation: SnapGeneration, + ) -> Result { + let mut attempt = self.authorize_snap_write(write)?; + attempt.re_anchor(generation.target(), generation.state_root()); + self.write_snap_attempt(&attempt)?; + Ok(SnapWrite::of(&attempt)) + } + + fn verify_snap_attempt(&self, write: SnapWrite) -> Result<(), SnapSyncError> { + let mut attempt = self.authorize_snap_write(write)?; + attempt.verify(); + self.write_snap_attempt(&attempt)?; + Ok(()) + } + + fn abandon_snap_attempt(&self) -> Result<(), SnapSyncError> { + if let Some(mut attempt) = self.snap_attempt()? && + attempt.is_unfinished() + { + attempt.abandon(); + self.write_snap_attempt(&attempt)?; + } + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use alloy_eips::BlockNumHash; + use alloy_primitives::{Bytes, B256}; + use reth_db_api::{tables, transaction::DbTx}; + use reth_primitives_traits::Account; + use reth_provider::{ + test_utils::{create_test_provider_factory, MockNodeTypesWithDB}, + DBProvider, DatabaseProviderFactory, ProviderFactory, + }; + use reth_storage_api::{metadata::keys, StateWriter}; + use reth_trie_common::HashedPostState; + use revm::{bytecode::Bytecode, database::states::StateChangeset}; + + const HASHED_ADDRESS: B256 = B256::repeat_byte(0xbb); + + fn code() -> Bytecode { + Bytecode::new_raw(Bytes::from_static(&[0x60, 0x00])) + } + + fn generation(block: u64) -> SnapGeneration { + SnapGeneration::new( + BlockNumHash::new(block, B256::repeat_byte(block as u8)), + B256::repeat_byte(0xaa), + ) + } + + // A database using the hashed state layout snap writes into. + fn factory() -> ProviderFactory { + let factory = create_test_provider_factory(); + let provider = factory.database_provider_rw().unwrap(); + provider.write_storage_settings(StorageSettings::v2()).unwrap(); + provider.commit().unwrap(); + factory + } + + // Persists one account and its bytecode through the writers snap shares with execution. + fn download(provider: &impl StateWriter) { + let mut state = HashedPostState::default(); + state.accounts.insert(HASHED_ADDRESS, Some(Account::default())); + provider.write_hashed_state(&state.into_sorted()).unwrap(); + provider + .write_state_changes(StateChangeset { + contracts: vec![(code().hash_slow(), code())], + ..Default::default() + }) + .unwrap(); + } + + fn downloaded(provider: &impl DBProvider) -> (bool, bool) { + let tx = provider.tx_ref(); + ( + tx.get::(HASHED_ADDRESS).unwrap().is_some(), + tx.get::(code().hash_slow()).unwrap().is_some(), + ) + } + + #[test] + fn nothing_owns_the_state_before_an_attempt_starts() { + let factory = factory(); + let provider = factory.database_provider_rw().unwrap(); + + assert_eq!(provider.active_snap_write().unwrap(), None); + let write = SnapWrite { attempt: SnapAttemptId::FIRST, state_version: 0 }; + assert!(matches!(provider.authorize_snap_write(write), Err(SnapSyncError::NoAttempt))); + } + + #[test] + fn address_keyed_state_cannot_host_an_attempt() { + let factory = create_test_provider_factory(); + let provider = factory.database_provider_rw().unwrap(); + + // Nothing recorded means the legacy layout. + assert!(matches!( + provider.start_snap_attempt(generation(1)), + Err(SnapSyncError::UnsupportedStorage) + )); + provider.write_storage_settings(StorageSettings::v1()).unwrap(); + assert!(matches!( + provider.start_snap_attempt(generation(1)), + Err(SnapSyncError::UnsupportedStorage) + )); + assert_eq!(provider.snap_attempt().unwrap(), None); + } + + #[test] + fn restarting_at_the_same_pivot_takes_a_new_identity() { + let factory = factory(); + let provider = factory.database_provider_rw().unwrap(); + + let first = provider.start_snap_attempt(generation(1)).unwrap(); + let second = provider.start_snap_attempt(generation(1)).unwrap(); + + assert_ne!(first.attempt(), second.attempt()); + // The superseded attempt's outstanding downloads no longer own the state. + assert!(matches!( + provider.authorize_snap_write(first), + Err(SnapSyncError::StaleWrite { .. }) + )); + provider.authorize_snap_write(second).unwrap(); + } + + #[test] + fn abandoning_an_attempt_keeps_its_identity_taken() { + let factory = factory(); + let provider = factory.database_provider_rw().unwrap(); + + let abandoned = provider.start_snap_attempt(generation(1)).unwrap(); + provider.abandon_snap_attempt().unwrap(); + assert_eq!(provider.active_snap_write().unwrap(), None); + let started = provider.start_snap_attempt(generation(2)).unwrap(); + + assert_ne!(abandoned.attempt(), started.attempt()); + // The abandoned attempt's outstanding downloads cannot pass as the new attempt's work. + assert!(matches!( + provider.authorize_snap_write(abandoned), + Err(SnapSyncError::StaleWrite { .. }) + )); + } + + #[test] + fn the_attempt_survives_reopening_the_database() { + let factory = factory(); + let provider = factory.database_provider_rw().unwrap(); + let write = provider.start_snap_attempt(generation(7)).unwrap(); + provider.commit().unwrap(); + + let reopened = factory.database_provider_rw().unwrap(); + + assert_eq!(reopened.active_snap_write().unwrap(), Some(write)); + let attempt = reopened.snap_attempt().unwrap().unwrap(); + assert_eq!(attempt.pivot(), BlockNumHash::new(7, B256::repeat_byte(7))); + assert_eq!(attempt.state_root(), B256::repeat_byte(0xaa)); + assert!(attempt.is_unfinished()); + } + + #[test] + fn committing_keeps_downloaded_state_and_progress_together() { + let factory = factory(); + let provider = factory.database_provider_rw().unwrap(); + let write = provider.start_snap_attempt(generation(1)).unwrap(); + download(&provider); + provider.commit().unwrap(); + + let provider = factory.database_provider_rw().unwrap(); + assert_eq!(provider.active_snap_write().unwrap(), Some(write)); + assert_eq!(downloaded(&provider), (true, true)); + } + + #[test] + fn rolling_back_drops_downloaded_state_and_progress_together() { + let factory = factory(); + let provider = factory.database_provider_rw().unwrap(); + provider.start_snap_attempt(generation(1)).unwrap(); + download(&provider); + // Dropping without committing is the interrupted-commit case. + drop(provider); + + let provider = factory.database_provider_rw().unwrap(); + assert_eq!(provider.active_snap_write().unwrap(), None); + assert_eq!(downloaded(&provider), (false, false)); + } + + #[test] + fn advancing_the_pivot_rejects_writes_proved_against_the_old_root() { + let factory = factory(); + let provider = factory.database_provider_rw().unwrap(); + + let before = provider.start_snap_attempt(generation(1)).unwrap(); + let after = provider.advance_snap_pivot(before, generation(2)).unwrap(); + + assert_eq!(after.attempt(), before.attempt()); + assert_eq!(after.state_version(), before.state_version() + 1); + assert!(matches!( + provider.authorize_snap_write(before), + Err(SnapSyncError::StaleWrite { .. }) + )); + provider.authorize_snap_write(after).unwrap(); + } + + #[test] + fn a_verified_attempt_accepts_no_further_writes() { + let factory = factory(); + let provider = factory.database_provider_rw().unwrap(); + let write = provider.start_snap_attempt(generation(1)).unwrap(); + + provider.verify_snap_attempt(write).unwrap(); + + assert!(!provider.snap_attempt().unwrap().unwrap().is_unfinished()); + assert_eq!(provider.active_snap_write().unwrap(), None); + assert!(matches!( + provider.authorize_snap_write(write), + Err(SnapSyncError::StaleWrite { .. }) + )); + // Verified state is complete, so abandoning cannot turn it into leftovers. + provider.abandon_snap_attempt().unwrap(); + assert!(provider.snap_attempt().unwrap().unwrap().is_verified()); + } + + #[test] + fn a_record_this_build_cannot_read_is_reported_rather_than_ignored() { + let factory = factory(); + + for record in [br#"{"version":999}"#.to_vec(), b"{}".to_vec(), b"not json".to_vec()] { + let provider = factory.database_provider_rw().unwrap(); + provider.write_metadata(keys::SNAP_ATTEMPT, record).unwrap(); + + assert!(provider.snap_attempt().is_err()); + // Starting a new attempt cannot guarantee a distinct identity over an unreadable one. + assert!(provider.start_snap_attempt(generation(1)).is_err()); + } + } +} diff --git a/crates/snap-sync/src/error.rs b/crates/snap-sync/src/error.rs index e78bd7a328d..3234b1239c1 100644 --- a/crates/snap-sync/src/error.rs +++ b/crates/snap-sync/src/error.rs @@ -1,5 +1,6 @@ //! Failures raised while assembling a snap state generation. +use reth_storage_api::SnapAttemptId; use reth_storage_errors::provider::ProviderError; /// Error returned while assembling a snap state generation. @@ -8,4 +9,18 @@ pub enum SnapSyncError { /// A header lookup failed. #[error(transparent)] Provider(#[from] ProviderError), + /// The storage layout keys state by address, which snap cannot fill in without preimages. + #[error("snap synchronization requires the hashed state layout")] + UnsupportedStorage, + /// No attempt owns the persisted state. + #[error("no snap attempt owns the persisted state")] + NoAttempt, + /// A write was presented for an attempt or pivot that no longer owns the persisted state. + #[error("stale snap write for attempt {attempt} at state version {state_version}")] + StaleWrite { + /// Attempt the rejected write claims. + attempt: SnapAttemptId, + /// State version the rejected write was proved against. + state_version: u64, + }, } diff --git a/crates/snap-sync/src/lib.rs b/crates/snap-sync/src/lib.rs index b37157c2031..9b652e3df60 100644 --- a/crates/snap-sync/src/lib.rs +++ b/crates/snap-sync/src/lib.rs @@ -9,6 +9,9 @@ //! `reth-downloaders`, and verified state is handed back to node integration once its trie root //! matches the target header. //! +//! Downloaded state goes into the hashed state tables, owned by an attempt record that commits +//! with it. Scheduling, in-flight requests and cancellation stay in memory. +//! //! ``` //! use reth_snap_sync::SnapPivotPolicy; //! @@ -30,6 +33,7 @@ )] #![cfg_attr(not(test), warn(unused_crate_dependencies))] +mod attempt; mod error; mod generation; mod pivot; @@ -38,6 +42,7 @@ mod session; #[cfg(test)] mod test_utils; +pub use attempt::{SnapAttemptStore, SnapWrite}; pub use error::SnapSyncError; pub use generation::{SnapGeneration, SnapPhase}; pub use pivot::SnapPivotPolicy; diff --git a/crates/storage/db-api/Cargo.toml b/crates/storage/db-api/Cargo.toml index 2f40f9aeeaa..c3b4130e274 100644 --- a/crates/storage/db-api/Cargo.toml +++ b/crates/storage/db-api/Cargo.toml @@ -25,6 +25,7 @@ reth-trie-common.workspace = true # ethereum alloy-primitives.workspace = true alloy-consensus.workspace = true +alloy-eips = { workspace = true, features = ["serde"] } # codecs modular-bitfield.workspace = true @@ -80,4 +81,5 @@ arbitrary = [ "reth-stages-types/arbitrary", "alloy-consensus/arbitrary", "reth-ethereum-primitives/arbitrary", + "alloy-eips/arbitrary", ] diff --git a/crates/storage/db-api/src/models/mod.rs b/crates/storage/db-api/src/models/mod.rs index aa7717252b1..677ad4000c6 100644 --- a/crates/storage/db-api/src/models/mod.rs +++ b/crates/storage/db-api/src/models/mod.rs @@ -16,6 +16,7 @@ pub mod blocks; pub mod integer_list; pub mod metadata; pub mod sharded_key; +pub mod snap; pub mod storage_sharded_key; pub use accounts::*; @@ -28,6 +29,7 @@ pub use reth_db_models::{ StoredBlockBodyIndices, StoredBlockWithdrawals, }; pub use sharded_key::ShardedKey; +pub use snap::*; /// Macro that implements [`Encode`] and [`Decode`] for uint types. macro_rules! impl_uints { diff --git a/crates/storage/db-api/src/models/snap.rs b/crates/storage/db-api/src/models/snap.rs new file mode 100644 index 00000000000..b536f40af74 --- /dev/null +++ b/crates/storage/db-api/src/models/snap.rs @@ -0,0 +1,135 @@ +//! Snap synchronization models. + +use alloy_eips::BlockNumHash; +use alloy_primitives::B256; +use core::fmt; +use serde::{Deserialize, Serialize}; + +/// Encoding version of [`SnapAttempt`] written by this build. +pub const SNAP_ATTEMPT_VERSION: u32 = 1; + +/// The snap synchronization attempt that owns the downloaded state. +/// +/// Snap writes land in the canonical hashed state tables, so this record is what separates state a +/// live attempt is filling in from what an abandoned one left behind. It is never deleted: an +/// abandoned attempt keeps its identity, so a later attempt can never take it. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub struct SnapAttempt { + // Encoding version of this record. + version: u32, + // Identity of this attempt. + id: SnapAttemptId, + // Pivot block the downloaded state is anchored to. + pivot: BlockNumHash, + // Root downloaded ranges authenticate against. + state_root: B256, + // Bumped whenever the pivot moves, so writes proved against a superseded root are refused. + state_version: u64, + // Whether the downloaded state has been verified. + status: SnapBootstrapStatus, +} + +impl SnapAttempt { + /// Creates the record for an attempt superseding `previous`. + pub const fn start(previous: Option, pivot: BlockNumHash, state_root: B256) -> Self { + let id = match previous { + Some(previous) => previous.id.next(), + None => SnapAttemptId::FIRST, + }; + Self { + version: SNAP_ATTEMPT_VERSION, + id, + pivot, + state_root, + state_version: 0, + status: SnapBootstrapStatus::Unfinished, + } + } + + /// Identity of this attempt. + pub const fn id(&self) -> SnapAttemptId { + self.id + } + + /// Pivot block the downloaded state is anchored to. + pub const fn pivot(&self) -> BlockNumHash { + self.pivot + } + + /// Root downloaded ranges authenticate against. + pub const fn state_root(&self) -> B256 { + self.state_root + } + + /// Generation of the pivot this attempt is anchored to. + pub const fn state_version(&self) -> u64 { + self.state_version + } + + /// Returns whether the downloaded state is still incomplete. + pub const fn is_unfinished(&self) -> bool { + matches!(self.status, SnapBootstrapStatus::Unfinished) + } + + /// Returns whether the reconstructed trie root matched the target header. + pub const fn is_verified(&self) -> bool { + matches!(self.status, SnapBootstrapStatus::Verified) + } + + /// Re-anchors this attempt, superseding writes proved against the previous root. + pub const fn re_anchor(&mut self, pivot: BlockNumHash, state_root: B256) { + self.pivot = pivot; + self.state_root = state_root; + self.state_version = self.state_version.saturating_add(1); + } + + /// Marks the downloaded state verified. + pub const fn verify(&mut self) { + self.status = SnapBootstrapStatus::Verified; + } + + /// Gives up on the downloaded state, keeping this identity taken. + pub const fn abandon(&mut self) { + self.status = SnapBootstrapStatus::Abandoned; + } +} + +/// Identity of one snap synchronization attempt. +/// +/// Only ever handed out once, so an abandoned attempt's leftover state is never mistaken for the +/// current attempt's work. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] +pub struct SnapAttemptId(u64); + +impl SnapAttemptId { + /// Identity of a node's first attempt. + pub const FIRST: Self = Self(0); + + /// Returns the identity superseding this one. + pub const fn next(self) -> Self { + Self(self.0.saturating_add(1)) + } +} + +impl From for u64 { + fn from(id: SnapAttemptId) -> Self { + id.0 + } +} + +impl fmt::Display for SnapAttemptId { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}", self.0) + } +} + +// Whether a snap attempt's downloaded state has been verified. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +enum SnapBootstrapStatus { + // Downloads are outstanding, so the state is incomplete and is not canonical yet. + Unfinished, + // The reconstructed trie root matched the target header. + Verified, + // Downloads were given up, leaving incomplete state behind for cleanup. + Abandoned, +} diff --git a/crates/storage/errors/src/provider.rs b/crates/storage/errors/src/provider.rs index acc3e60396c..8c61f4b95b2 100644 --- a/crates/storage/errors/src/provider.rs +++ b/crates/storage/errors/src/provider.rs @@ -190,6 +190,22 @@ pub enum ProviderError { /// The block number to which the database must be unwound. unwind_to: BlockNumber, }, + /// A persisted snap attempt record this build cannot read. + #[error( + "snap attempt record version {found:?} is not supported, this build writes {supported}" + )] + UnsupportedSnapAttemptVersion { + /// Version found on disk, absent when the record carries no numeric version. + found: Option, + /// Version this build writes. + supported: u32, + }, + /// State a snap attempt is still downloading was about to be marked complete. + #[error("snap attempt {attempt} has not verified the downloaded state")] + UnverifiedSnapState { + /// Attempt that owns the unverified state. + attempt: u64, + }, /// Any other error type wrapped into a cloneable [`AnyError`]. #[error(transparent)] Other(#[from] AnyError), diff --git a/crates/storage/provider/src/providers/database/provider.rs b/crates/storage/provider/src/providers/database/provider.rs index 461dee4c452..7c968a44a1b 100644 --- a/crates/storage/provider/src/providers/database/provider.rs +++ b/crates/storage/provider/src/providers/database/provider.rs @@ -1412,6 +1412,33 @@ impl DatabaseProvider { } } +impl DatabaseProvider { + /// Rejects advances of either the Finish block number or its state/trie frontier while a + /// persisted snap attempt is unverified. A missing partial frontier equals the block + /// number, so clearing it can advance state even when the block number stays put or rewinds. + /// Updates that advance neither frontier are allowed. + fn ensure_finish_may_advance(&self, proposed: &StageCheckpoint) -> ProviderResult<()> { + let current = self.get_stage_checkpoint(StageId::Finish)?.unwrap_or_default(); + let state_trie_frontier = |checkpoint: &StageCheckpoint| { + checkpoint + .finish_stage_checkpoint() + .and_then(|finish| finish.partial_state_trie()) + .unwrap_or(checkpoint.block_number) + }; + if proposed.block_number <= current.block_number && + state_trie_frontier(proposed) <= state_trie_frontier(¤t) + { + return Ok(()) + } + match self.snap_attempt()? { + Some(attempt) if !attempt.is_verified() => { + Err(ProviderError::UnverifiedSnapState { attempt: attempt.id().into() }) + } + _ => Ok(()), + } + } +} + impl AccountReader for DatabaseProvider { fn basic_account(&self, address: &Address) -> ProviderResult> { if self.cached_storage_settings().use_hashed_state() { @@ -2230,13 +2257,16 @@ impl StageCheckpointReader for DatabaseProvider { } } -impl StageCheckpointWriter for DatabaseProvider { +impl StageCheckpointWriter for DatabaseProvider { /// Save stage checkpoint. fn save_stage_checkpoint( &self, id: StageId, checkpoint: StageCheckpoint, ) -> ProviderResult<()> { + if id == StageId::Finish { + self.ensure_finish_may_advance(&checkpoint)?; + } Ok(self.tx.put::(id.to_string(), checkpoint)?) } @@ -2255,6 +2285,13 @@ impl StageCheckpointWriter for DatabaseProvider ProviderResult<()> { + let current = self.get_stage_checkpoint(StageId::Finish)?.unwrap_or_default(); + let proposed = StageCheckpoint { + block_number, + ..if drop_stage_checkpoint { Default::default() } else { current } + }; + self.ensure_finish_may_advance(&proposed)?; + // iterate over all existing stages in the table and update its progress. let mut cursor = self.tx.cursor_write::()?; for stage_id in StageId::ALL { @@ -3962,7 +3999,7 @@ mod tests { use reth_ethereum_primitives::Receipt; use reth_execution_types::{AccountRevertInit, BlockExecutionOutput, BlockExecutionResult}; use reth_primitives_traits::SealedBlock; - use reth_storage_api::{MetadataProvider, MetadataWriter}; + use reth_storage_api::{DatabaseProviderFactory, MetadataProvider, MetadataWriter}; use reth_testing_utils::generators::{self, random_block, BlockParams}; use reth_trie::{ HashedPostState, KeccakKeyHasher, Nibbles, SortedTrieData, StoredNibbles, @@ -3991,6 +4028,117 @@ mod tests { ) } + #[test] + fn snap_attempt_guards_finish_checkpoint_writers() { + use alloy_eips::BlockNumHash; + use reth_db_api::models::SnapAttempt; + + let factory = create_test_provider_factory(); + let provider = factory.database_provider_rw().unwrap(); + provider.update_pipeline_stages(5, false).unwrap(); + let mut attempt = SnapAttempt::start( + None, + BlockNumHash::new(10, B256::repeat_byte(1)), + B256::repeat_byte(2), + ); + provider.write_snap_attempt(&attempt).unwrap(); + provider.commit().unwrap(); + + let provider = factory.database_provider_rw().unwrap(); + let refused = provider.save_stage_checkpoint(StageId::Finish, StageCheckpoint::new(10)); + assert!(matches!(refused, Err(ProviderError::UnverifiedSnapState { attempt: 0 }))); + let refused = provider.update_pipeline_stages(10, false); + assert!(matches!(refused, Err(ProviderError::UnverifiedSnapState { attempt: 0 }))); + for stage in [StageId::Finish, StageId::Headers] { + assert_eq!(provider.get_stage_checkpoint(stage).unwrap().unwrap().block_number, 5); + } + + // Rewinding claims nothing about the downloaded state, through either writer. + provider.update_pipeline_stages(4, true).unwrap(); + provider.save_stage_checkpoint(StageId::Finish, StageCheckpoint::new(3)).unwrap(); + assert_eq!( + provider.get_stage_checkpoint(StageId::Finish).unwrap().unwrap().block_number, + 3 + ); + + // Abandoned leftovers are still not complete state. + attempt.abandon(); + provider.write_snap_attempt(&attempt).unwrap(); + let refused = provider.save_stage_checkpoint(StageId::Finish, StageCheckpoint::new(10)); + assert!(matches!(refused, Err(ProviderError::UnverifiedSnapState { attempt: 0 }))); + + // Header progress does not claim the downloaded state is complete. + provider.save_stage_checkpoint(StageId::Headers, StageCheckpoint::new(10)).unwrap(); + attempt.verify(); + provider.write_snap_attempt(&attempt).unwrap(); + provider.save_stage_checkpoint(StageId::Finish, StageCheckpoint::new(10)).unwrap(); + provider.update_pipeline_stages(11, false).unwrap(); + provider.commit().unwrap(); + + let provider = factory.database_provider_ro().unwrap(); + assert_eq!( + provider.get_stage_checkpoint(StageId::Finish).unwrap().unwrap().block_number, + 11 + ); + } + + #[test] + fn snap_attempt_guards_partial_state_trie_advances() { + use alloy_eips::BlockNumHash; + use reth_db_api::models::SnapAttempt; + + for abandoned in [false, true] { + let factory = create_test_provider_factory(); + let provider = factory.database_provider_rw().unwrap(); + provider.update_pipeline_stages(100, false).unwrap(); + let checkpoint = StageCheckpoint::new(100) + .with_finish_stage_checkpoint(FinishCheckpoint { partial_state_trie: Some(50) }); + provider.save_stage_checkpoint(StageId::Finish, checkpoint).unwrap(); + let mut attempt = SnapAttempt::start(None, BlockNumHash::default(), B256::ZERO); + if abandoned { + attempt.abandon(); + } + provider.write_snap_attempt(&attempt).unwrap(); + + for block in [100, 90] { + let result = + provider.save_stage_checkpoint(StageId::Finish, StageCheckpoint::new(block)); + assert!(matches!(result, Err(ProviderError::UnverifiedSnapState { .. }))); + let result = provider.update_pipeline_stages(block, true); + assert!(matches!(result, Err(ProviderError::UnverifiedSnapState { .. }))); + for stage in [StageId::Finish, StageId::Headers] { + assert_eq!( + provider.get_stage_checkpoint(stage).unwrap().unwrap().block_number, + 100 + ); + } + assert_eq!( + provider.get_stage_checkpoint(StageId::Finish).unwrap(), + Some(checkpoint) + ); + } + + let advanced = StageCheckpoint::new(100) + .with_finish_stage_checkpoint(FinishCheckpoint { partial_state_trie: Some(60) }); + assert!(matches!( + provider.save_stage_checkpoint(StageId::Finish, advanced), + Err(ProviderError::UnverifiedSnapState { .. }) + )); + + // Retaining the partial frontier does not claim additional state progress. + provider.update_pipeline_stages(90, false).unwrap(); + let rewind = StageCheckpoint::new(80) + .with_finish_stage_checkpoint(FinishCheckpoint { partial_state_trie: Some(40) }); + provider.save_stage_checkpoint(StageId::Finish, rewind).unwrap(); + provider.update_pipeline_stages(30, true).unwrap(); + + attempt.verify(); + provider.write_snap_attempt(&attempt).unwrap(); + provider.save_stage_checkpoint(StageId::Finish, advanced).unwrap(); + provider.update_pipeline_stages(100, true).unwrap(); + } + } + #[test] fn test_receipts_by_block_range_empty_range() { let factory = create_test_provider_factory(); diff --git a/crates/storage/storage-api/src/lib.rs b/crates/storage/storage-api/src/lib.rs index 3ae54256d02..9651b870dcf 100644 --- a/crates/storage/storage-api/src/lib.rs +++ b/crates/storage/storage-api/src/lib.rs @@ -106,7 +106,7 @@ pub use metadata::StoragePath; #[cfg(feature = "db-api")] pub use metadata::{MetadataProvider, MetadataWriter, StorageSettingsCache}; #[cfg(feature = "db-api")] -pub use reth_db_api::models::StorageSettings; +pub use reth_db_api::models::{SnapAttempt, SnapAttemptId, StorageSettings, SNAP_ATTEMPT_VERSION}; mod full; pub use full::*; diff --git a/crates/storage/storage-api/src/metadata.rs b/crates/storage/storage-api/src/metadata.rs index 8bfee7e76f2..2b1875f4258 100644 --- a/crates/storage/storage-api/src/metadata.rs +++ b/crates/storage/storage-api/src/metadata.rs @@ -1,13 +1,16 @@ //! Metadata provider trait for reading and writing node metadata. use alloc::vec::Vec; -use reth_db_api::models::StorageSettings; +use reth_db_api::models::{SnapAttempt, StorageSettings, SNAP_ATTEMPT_VERSION}; use reth_storage_errors::provider::{ProviderError, ProviderResult}; /// Metadata keys. pub mod keys { /// Storage configuration settings for this node. pub const STORAGE_SETTINGS: &str = "storage_settings"; + + /// The snap synchronization attempt that owns downloaded state. + pub const SNAP_ATTEMPT: &str = "snap_attempt"; } /// Client trait for reading node metadata from the database. @@ -26,6 +29,28 @@ pub trait MetadataProvider: Send { .get_metadata(keys::STORAGE_SETTINGS)? .and_then(|bytes| serde_json::from_slice(&bytes).ok())) } + + /// Returns the snap synchronization attempt that owns the downloaded state. + /// + /// Unlike [`Self::storage_settings`], an unreadable record is an error: its state is already + /// in the canonical tables, so reporting it absent would let the node adopt it. + fn snap_attempt(&self) -> ProviderResult> { + let Some(bytes) = self.get_metadata(keys::SNAP_ATTEMPT)? else { return Ok(None) }; + + // Read the version first, so a future build's record is reported as unsupported rather + // than as a decode failure. + let value: serde_json::Value = + serde_json::from_slice(&bytes).map_err(ProviderError::other)?; + let found = value.get("version").and_then(serde_json::Value::as_u64); + if found != Some(SNAP_ATTEMPT_VERSION as u64) { + return Err(ProviderError::UnsupportedSnapAttemptVersion { + found, + supported: SNAP_ATTEMPT_VERSION, + }) + } + + serde_json::from_slice(&bytes).map(Some).map_err(ProviderError::other) + } } /// Client trait for writing node metadata to the database. @@ -48,6 +73,14 @@ pub trait MetadataWriter: Send { serde_json::to_vec(&settings).map_err(ProviderError::other)?, ) } + + /// Writes the snap synchronization attempt that owns the downloaded state. + fn write_snap_attempt(&self, attempt: &SnapAttempt) -> ProviderResult<()> { + self.write_metadata( + keys::SNAP_ATTEMPT, + serde_json::to_vec(attempt).map_err(ProviderError::other)?, + ) + } } /// Trait for caching storage settings on a provider factory. From 038c782fecb1bec0896df7af1408588a6eeb9a01 Mon Sep 17 00:00:00 2001 From: Derek Cofausper <256792747+decofe@users.noreply.github.com> Date: Wed, 9 Sep 2026 11:32:33 +0000 Subject: [PATCH 218/294] fix(trie): revert changeset avoidance for read-only overlays (#27114) Co-authored-by: Alexey Shekhirin <5773434+shekhirin@users.noreply.github.com> --- crates/storage/storage-overlay/src/builder.rs | 113 +++-------- .../storage-overlay/src/changeset_cache.rs | 4 +- .../storage/storage-overlay/src/provider.rs | 182 ++++++------------ 3 files changed, 91 insertions(+), 208 deletions(-) diff --git a/crates/storage/storage-overlay/src/builder.rs b/crates/storage/storage-overlay/src/builder.rs index 44a1500dfd7..d56fc0b4d7f 100644 --- a/crates/storage/storage-overlay/src/builder.rs +++ b/crates/storage/storage-overlay/src/builder.rs @@ -16,7 +16,7 @@ use reth_storage_api::{ BlockNumReader, ChangeSetReader, DBProvider, PruneCheckpointReader, StageCheckpointReader, StorageChangeSetReader, StorageSettingsCache, }; -use reth_trie::{updates::TrieUpdatesSorted, HashedPostStateSorted, TrieInputSorted}; +use reth_trie::{updates::TrieUpdatesSorted, HashedPostStateSorted}; use reth_trie_db::DatabaseHashedPostState; use revm::{bytecode::Bytecode, database::BundleState, state::AccountInfo}; use std::{ @@ -29,28 +29,28 @@ use tracing::{debug, debug_span, instrument}; /// Contains the trie and hashed-state data required to initialize an overlay state provider. #[derive(Debug, Clone)] pub struct StateTrieOverlay { - input: TrieInputSorted, + /// Trie updates overlay. + pub trie_updates: Arc, + /// Hashed state overlay. + pub hashed_post_state: Arc, /// Whether construction was skipped because a reused sparse trie covers this range. skipped_for_reused_sparse_trie: bool, } impl StateTrieOverlay { - pub(crate) const fn new(input: TrieInputSorted) -> Self { - Self { input, skipped_for_reused_sparse_trie: false } + pub(crate) const fn new( + trie_updates: Arc, + hashed_post_state: Arc, + ) -> Self { + Self { trie_updates, hashed_post_state, skipped_for_reused_sparse_trie: false } } fn empty() -> Self { - Self { input: TrieInputSorted::default(), skipped_for_reused_sparse_trie: true } - } - - /// Returns the trie input represented by this overlay. - pub const fn input(&self) -> &TrieInputSorted { - &self.input - } - - /// Consumes the overlay and returns its trie input. - pub fn into_input(self) -> TrieInputSorted { - self.input + Self { + trie_updates: Arc::new(TrieUpdatesSorted::default()), + hashed_post_state: Arc::new(HashedPostStateSorted::default()), + skipped_for_reused_sparse_trie: true, + } } pub(crate) const fn skipped_for_reused_sparse_trie(&self) -> bool { @@ -393,17 +393,11 @@ impl OverlayBuilder { } /// Builds the effective state trie overlay for the given provider. - /// - /// Set `trie_changesets` only for consumers that produce [`TrieUpdates`], such as - /// [`StateRootProvider::state_root_with_updates`]. Other consumers, including roots, proofs, - /// multiproofs, and witnesses, should leave it false: reverts are then represented by - /// hashed-state prefix sets instead of querying trie changesets. #[cfg(test)] #[instrument(level = "debug", target = "storage::overlay", skip_all)] fn build_state_trie_overlay( &self, provider: &Provider, - trie_changesets: bool, ) -> ProviderResult where Provider: StageCheckpointReader @@ -415,12 +409,7 @@ impl OverlayBuilder { + StorageSettingsCache, { let (state_trie_tip_block, finish_tip_block) = database_state_frontiers(provider)?; - self.build_state_trie_overlay_at_frontiers( - provider, - state_trie_tip_block, - finish_tip_block, - trie_changesets, - ) + self.build_state_trie_overlay_at_frontiers(provider, state_trie_tip_block, finish_tip_block) } /// Builds the effective state trie overlay using frontiers already read from the provider. @@ -437,7 +426,6 @@ impl OverlayBuilder { provider: &Provider, state_trie_tip_block: BlockNumHash, finish_tip_block: BlockNumHash, - trie_changesets: bool, ) -> ProviderResult where Provider: ChangeSetReader @@ -457,7 +445,7 @@ impl OverlayBuilder { self.anchor_at_parent_with_frontiers(provider, state_trie_tip_block, finish_tip_block)?; // Collect any reverts which are required to bring the DB view back to the anchor hash. - let (trie_updates, hashed_post_state, prefix_sets) = match &anchor_for_parent { + let (trie_updates, hashed_post_state) = match &anchor_for_parent { AnchorForParent::RevertsRequired { anchor, .. } => { let revert_blocks = self.revert_blocks(&anchor_for_parent)?.expect("reverts are required"); @@ -469,7 +457,7 @@ impl OverlayBuilder { "Collecting trie reverts for overlay state provider" ); - let trie_reverts = if trie_changesets { + let trie_reverts = { let _guard = debug_span!(target: "storage::overlay", "retrieving_trie_reverts") .entered(); let start = Instant::now(); @@ -482,9 +470,6 @@ impl OverlayBuilder { )?; retrieve_trie_reverts_duration = start.elapsed(); accumulated_reverts - } else { - retrieve_trie_reverts_duration = Duration::ZERO; - Arc::default() }; let mut hashed_state_reverts = { @@ -497,12 +482,6 @@ impl OverlayBuilder { res }; - let prefix_sets = if trie_changesets { - Default::default() - } else { - hashed_state_reverts.construct_prefix_sets() - }; - // Resolve overlays and extend reverts with them. If reverts are empty, use overlays // directly to avoid cloning. let (overlay_trie, overlay_state) = @@ -538,7 +517,7 @@ impl OverlayBuilder { "Reverted to anchor block", ); - (trie_updates, hashed_state_updates, prefix_sets) + (trie_updates, hashed_state_updates) } AnchorForParent::NoReverts { anchor } => { // If no reverts are needed, use the manager overlay directly unless the reused @@ -569,7 +548,7 @@ impl OverlayBuilder { "Built overlay directly from durable frontier" ); - (trie_updates, hashed_post_state, Default::default()) + (trie_updates, hashed_post_state) } }; @@ -582,11 +561,7 @@ impl OverlayBuilder { self.metrics.trie_updates_size.record(trie_updates_total_len as f64); self.metrics.hashed_state_size.record(hashed_state_updates_total_len as f64); - Ok(StateTrieOverlay::new(TrieInputSorted::new( - trie_updates, - hashed_post_state, - prefix_sets, - ))) + Ok(StateTrieOverlay::new(trie_updates, hashed_post_state)) } /// Returns the in-memory execution overlay and the block for historical fallback reads. @@ -917,11 +892,11 @@ mod tests { } fn account_keys(overlay: &StateTrieOverlay) -> Vec { - overlay.input().state.accounts.iter().map(|(key, _)| *key).collect() + overlay.hashed_post_state.accounts.iter().map(|(key, _)| *key).collect() } fn account_node_paths(overlay: &StateTrieOverlay) -> Vec { - overlay.input().nodes.account_nodes_ref().iter().map(|(path, _)| *path).collect() + overlay.trie_updates.account_nodes_ref().iter().map(|(path, _)| *path).collect() } #[test] @@ -1038,7 +1013,7 @@ mod tests { for (parent_index, expected_ids) in [(3, vec![3, 4]), (4, vec![3, 4, 5])] { let overlay = manager .overlay_builder(blocks[parent_index].recovered_block().hash()) - .build_state_trie_overlay(&provider, true) + .build_state_trie_overlay(&provider) .unwrap(); assert_eq!( @@ -1066,11 +1041,11 @@ mod tests { let overlay = manager .overlay_builder(blocks[4].recovered_block().hash()) .with_skip_overlay_for_reused_sparse_trie(blocks[3].recovered_block().hash()) - .build_state_trie_overlay(&provider, true) + .build_state_trie_overlay(&provider) .unwrap(); - assert!(overlay.input().state.is_empty()); - assert!(overlay.input().nodes.is_empty()); + assert!(overlay.hashed_post_state.is_empty()); + assert!(overlay.trie_updates.is_empty()); } #[test] @@ -1081,7 +1056,7 @@ mod tests { let builder = OverlayManager::::default() .overlay_builder(blocks[1].recovered_block().hash()) .with_no_reverts(); - let error = builder.build_state_trie_overlay(&provider, true).unwrap_err(); + let error = builder.build_state_trie_overlay(&provider).unwrap_err(); assert!(error.to_string().contains("reverts are disabled")); } @@ -1102,37 +1077,11 @@ mod tests { Err(ProviderError::BlockHashNotFound(hash)) if hash == parent_hash )); assert!(matches!( - builder.build_state_trie_overlay(&provider, true), + builder.build_state_trie_overlay(&provider), Err(ProviderError::BlockHashNotFound(hash)) if hash == parent_hash )); } - #[test] - fn state_trie_overlay_uses_revert_prefix_sets_without_trie_changesets() { - let (factory, blocks) = setup_frontiers(2, 3); - let provider_rw = factory.provider_rw().unwrap(); - provider_rw - .tx_ref() - .put::( - 3, - AccountBeforeTx { - address: Address::with_last_byte(1), - info: Some(Account::default()), - }, - ) - .unwrap(); - provider_rw.commit().unwrap(); - - let provider = factory.provider().unwrap(); - let overlay = OverlayManager::::default() - .overlay_builder(blocks[1].recovered_block().hash()) - .build_state_trie_overlay(&provider, false) - .unwrap(); - - assert!(overlay.input().nodes.is_empty()); - assert!(!overlay.input().prefix_sets.is_empty()); - } - #[test] fn execution_overlay_marks_historical_fallback() { let (factory, blocks) = setup_frontiers(1, 3); @@ -1327,7 +1276,7 @@ mod tests { let provider = factory.provider().unwrap(); let error = OverlayManager::::default() .overlay_builder(blocks[3].recovered_block().hash()) - .build_state_trie_overlay(&provider, true) + .build_state_trie_overlay(&provider) .unwrap_err(); assert!( @@ -1343,7 +1292,7 @@ mod tests { let parent_hash = blocks[3].recovered_block().hash(); let error = OverlayManager::::default() .overlay_builder(parent_hash) - .build_state_trie_overlay(&provider, true) + .build_state_trie_overlay(&provider) .unwrap_err(); assert!(error.to_string().contains("is after partial state trie frontier")); diff --git a/crates/storage/storage-overlay/src/changeset_cache.rs b/crates/storage/storage-overlay/src/changeset_cache.rs index 69191d65ae9..c2e070fd4b1 100644 --- a/crates/storage/storage-overlay/src/changeset_cache.rs +++ b/crates/storage/storage-overlay/src/changeset_cache.rs @@ -403,7 +403,7 @@ impl ChangesetCache { let overlay = overlay_manager .overlay_builder(finish.hash) .with_no_reverts() - .build_state_trie_overlay_at_frontiers(provider, partial_state_trie, finish, true)?; + .build_state_trie_overlay_at_frontiers(provider, partial_state_trie, finish)?; let state_trie_provider = OverlayStateProvider::<&P, N>::new_with_state_trie( provider, overlay, @@ -644,7 +644,7 @@ mod tests { } fn empty_overlay() -> StateTrieOverlay { - StateTrieOverlay::new(TrieInputSorted::default()) + StateTrieOverlay::new(Arc::default(), Arc::default()) } fn insert_test_changesets( diff --git a/crates/storage/storage-overlay/src/provider.rs b/crates/storage/storage-overlay/src/provider.rs index d40677ae1b8..578e4453560 100644 --- a/crates/storage/storage-overlay/src/provider.rs +++ b/crates/storage/storage-overlay/src/provider.rs @@ -52,7 +52,7 @@ pub struct OverlayStateProviderFactory { factory: F, /// Overlay builder containing the configuration and overlay calculation logic. overlay_builder: OverlayBuilder, - /// A cache mapping `(state_trie_tip, finish_tip, trie_changesets)` to [`StateTrieOverlay`]. + /// A cache mapping `(state_trie_tip, finish_tip)` to [`StateTrieOverlay`]. /// /// Under partial persistence the overlay depends on both durable frontiers, so both hashes are /// part of the cache key. @@ -129,7 +129,6 @@ pub struct OverlayStateProvider { state_trie_overlay_cache: StateTrieOverlayCache, metrics: OverlayStateProviderFactoryMetrics, state_trie_overlay: OnceCell, - state_trie_overlay_with_trie_changesets: OnceCell, execution_overlay: OnceCell, is_v2: bool, } @@ -163,7 +162,6 @@ impl OverlayStateProvider, state_trie_overlay_cache, metrics, state_trie_overlay: OnceCell::new(), - state_trie_overlay_with_trie_changesets: OnceCell::new(), execution_overlay: OnceCell::new(), is_v2, } @@ -181,7 +179,6 @@ impl OverlayStateProvider, state_trie_overlay_cache: Default::default(), metrics: Default::default(), state_trie_overlay: OnceCell::new(), - state_trie_overlay_with_trie_changesets: OnceCell::new(), execution_overlay: OnceCell::from(CachedExecutionOverlay { overlay: execution_overlay, historical_fallback: None, @@ -204,7 +201,6 @@ impl<'a, Provider, N: NodePrimitives> OverlayStateProvider<&'a Provider, N> { state_trie_overlay_cache: Default::default(), metrics: Default::default(), state_trie_overlay: OnceCell::new(), - state_trie_overlay_with_trie_changesets: OnceCell::new(), execution_overlay: OnceCell::new(), is_v2, } @@ -220,8 +216,7 @@ impl<'a, Provider, N: NodePrimitives> OverlayStateProvider<&'a Provider, N> { overlay_builder: None, state_trie_overlay_cache: Default::default(), metrics: Default::default(), - state_trie_overlay: OnceCell::from(state_trie_overlay.clone()), - state_trie_overlay_with_trie_changesets: OnceCell::from(state_trie_overlay), + state_trie_overlay: OnceCell::from(state_trie_overlay), execution_overlay: OnceCell::new(), is_v2, } @@ -237,7 +232,7 @@ where &self.provider } - fn state_trie_overlay(&self, trie_changesets: bool) -> ProviderResult<&StateTrieOverlay> + fn state_trie_overlay(&self) -> ProviderResult<&StateTrieOverlay> where Provider::Target: StageCheckpointReader + PruneCheckpointReader @@ -247,17 +242,15 @@ where + BlockNumReader + StorageSettingsCache, { - let state_trie_overlay = self.state_trie_overlay_mut(trie_changesets); - if let Some(overlay) = state_trie_overlay.get() { + if let Some(overlay) = self.state_trie_overlay.get() { return Ok(overlay) } let (state_trie_tip_block, finish_tip_block) = database_state_frontiers(self.provider())?; - let overlay = match self.state_trie_overlay_cache.entry(( - state_trie_tip_block.hash, - finish_tip_block.hash, - trie_changesets, - )) { + let overlay = match self + .state_trie_overlay_cache + .entry((state_trie_tip_block.hash, finish_tip_block.hash)) + { dashmap::Entry::Occupied(entry) => entry.get().clone(), dashmap::Entry::Vacant(entry) => { self.metrics.state_trie_overlay_cache_misses.increment(1); @@ -269,7 +262,6 @@ where self.provider(), state_trie_tip_block, finish_tip_block, - trie_changesets, )?; if !overlay.skipped_for_reused_sparse_trie() { entry.insert(overlay.clone()); @@ -277,23 +269,11 @@ where overlay } }; - let _ = state_trie_overlay.set(overlay); - Ok(state_trie_overlay.get().expect("state trie overlay was just initialized")) - } - - const fn state_trie_overlay_mut(&self, trie_changesets: bool) -> &OnceCell { - if trie_changesets { - &self.state_trie_overlay_with_trie_changesets - } else { - &self.state_trie_overlay - } + let _ = self.state_trie_overlay.set(overlay); + Ok(self.state_trie_overlay.get().expect("state trie overlay was just initialized")) } - fn build_overlay( - &self, - input: TrieInputSorted, - trie_changesets: bool, - ) -> ProviderResult + fn build_overlay(&self, input: TrieInputSorted) -> ProviderResult where Provider::Target: StageCheckpointReader + PruneCheckpointReader @@ -303,14 +283,13 @@ where + BlockNumReader + StorageSettingsCache, { - let overlay = self.state_trie_overlay(trie_changesets)?; + let overlay = self.state_trie_overlay()?; if overlay.skipped_for_reused_sparse_trie() { return Err(ProviderError::UnsupportedProvider) } - let TrieInputSorted { nodes: input_nodes, state: input_state, mut prefix_sets } = input; - let overlay_input = overlay.input(); - let mut nodes = Arc::clone(&overlay_input.nodes); - let mut state = Arc::clone(&overlay_input.state); + let TrieInputSorted { nodes: input_nodes, state: input_state, prefix_sets } = input; + let mut nodes = Arc::clone(&overlay.trie_updates); + let mut state = Arc::clone(&overlay.hashed_post_state); if !input_nodes.is_empty() { Arc::make_mut(&mut nodes).extend_ref_and_sort(&input_nodes); @@ -319,7 +298,6 @@ where Arc::make_mut(&mut state).extend_ref_and_sort(&input_state); } - prefix_sets.extend_ref(&overlay_input.prefix_sets); Ok(TrieInputSorted::new(nodes, state, prefix_sets)) } @@ -533,17 +511,16 @@ where { fn state_root(&self, hashed_state: HashedPostState) -> ProviderResult { reth_trie_db::with_adapter!(self.provider(), |A| { - let input = self.build_overlay( - TrieInputSorted::from_unsorted(TrieInput::from_state(hashed_state)), - false, - )?; + let input = self.build_overlay(TrieInputSorted::from_unsorted( + TrieInput::from_state(hashed_state), + ))?; Ok(>::overlay_root_from_nodes(self.provider().tx(), input)?) }) } fn state_root_from_nodes(&self, input: TrieInput) -> ProviderResult { reth_trie_db::with_adapter!(self.provider(), |A| { - let input = self.build_overlay(TrieInputSorted::from_unsorted(input), false)?; + let input = self.build_overlay(TrieInputSorted::from_unsorted(input))?; Ok( as DatabaseStateRoot<_>>::overlay_root_from_nodes( self.provider().tx(), input, @@ -556,10 +533,9 @@ where hashed_state: HashedPostState, ) -> ProviderResult<(B256, TrieUpdates)> { reth_trie_db::with_adapter!(self.provider(), |A| { - let input = self.build_overlay( - TrieInputSorted::from_unsorted(TrieInput::from_state(hashed_state)), - true, - )?; + let input = self.build_overlay(TrieInputSorted::from_unsorted( + TrieInput::from_state(hashed_state), + ))?; Ok(>::overlay_root_from_nodes_with_updates( self.provider().tx(), input, @@ -572,7 +548,7 @@ where input: TrieInput, ) -> ProviderResult<(B256, TrieUpdates)> { reth_trie_db::with_adapter!(self.provider(), |A| { - let input = self.build_overlay(TrieInputSorted::from_unsorted(input), true)?; + let input = self.build_overlay(TrieInputSorted::from_unsorted(input))?; Ok( as DatabaseStateRoot<_>>::overlay_root_from_nodes_with_updates( self.provider().tx(), @@ -600,15 +576,12 @@ where hashed_storage: HashedStorage, ) -> ProviderResult { reth_trie_db::with_adapter!(self.provider(), |A| { - let input = self.build_overlay( - TrieInputSorted::from_unsorted(TrieInput::from_state( - HashedPostState::from_hashed_storage( - alloy_primitives::keccak256(address), - hashed_storage, - ), + let input = self.build_overlay(TrieInputSorted::from_unsorted( + TrieInput::from_state(HashedPostState::from_hashed_storage( + alloy_primitives::keccak256(address), + hashed_storage, )), - false, - )?; + ))?; let hashed_storage = input .state .account_storages() @@ -628,15 +601,12 @@ where hashed_storage: HashedStorage, ) -> ProviderResult { reth_trie_db::with_adapter!(self.provider(), |A| { - let input = self.build_overlay( - TrieInputSorted::from_unsorted(TrieInput::from_state( - HashedPostState::from_hashed_storage( - alloy_primitives::keccak256(address), - hashed_storage, - ), + let input = self.build_overlay(TrieInputSorted::from_unsorted( + TrieInput::from_state(HashedPostState::from_hashed_storage( + alloy_primitives::keccak256(address), + hashed_storage, )), - false, - )?; + ))?; let hashed_storage = input .state .account_storages() @@ -661,15 +631,12 @@ where hashed_storage: HashedStorage, ) -> ProviderResult { reth_trie_db::with_adapter!(self.provider(), |A| { - let input = self.build_overlay( - TrieInputSorted::from_unsorted(TrieInput::from_state( - HashedPostState::from_hashed_storage( - alloy_primitives::keccak256(address), - hashed_storage, - ), + let input = self.build_overlay(TrieInputSorted::from_unsorted( + TrieInput::from_state(HashedPostState::from_hashed_storage( + alloy_primitives::keccak256(address), + hashed_storage, )), - false, - )?; + ))?; let hashed_storage = input .state .account_storages() @@ -707,7 +674,7 @@ where ) -> ProviderResult { reth_trie_db::with_adapter!(self.provider(), |A| { let TrieInputSorted { nodes, state, prefix_sets } = - self.build_overlay(TrieInputSorted::from_unsorted(input), false)?; + self.build_overlay(TrieInputSorted::from_unsorted(input))?; let input = TrieInput::new( Arc::unwrap_or_clone(nodes).into(), Arc::unwrap_or_clone(state).into(), @@ -725,7 +692,7 @@ where ) -> ProviderResult { reth_trie_db::with_adapter!(self.provider(), |A| { let TrieInputSorted { nodes, state, prefix_sets } = - self.build_overlay(TrieInputSorted::from_unsorted(input), false)?; + self.build_overlay(TrieInputSorted::from_unsorted(input))?; let input = TrieInput::new( Arc::unwrap_or_clone(nodes).into(), Arc::unwrap_or_clone(state).into(), @@ -743,7 +710,7 @@ where ) -> ProviderResult { reth_trie_db::with_adapter!(self.provider(), |A| { let TrieInputSorted { nodes, state, prefix_sets } = - self.build_overlay(TrieInputSorted::from_unsorted(input), false)?; + self.build_overlay(TrieInputSorted::from_unsorted(input))?; let input = TrieInput::new( Arc::unwrap_or_clone(nodes).into(), Arc::unwrap_or_clone(state).into(), @@ -762,7 +729,7 @@ where ) -> ProviderResult> { reth_trie_db::with_adapter!(self.provider(), |A| { let TrieInputSorted { nodes, state, prefix_sets } = - self.build_overlay(TrieInputSorted::from_unsorted(input), false)?; + self.build_overlay(TrieInputSorted::from_unsorted(input))?; let witness = TrieWitness::new( InMemoryTrieCursorFactory::new( DatabaseTrieCursorFactory::<_, A>::new(self.provider().tx()), @@ -811,7 +778,7 @@ where return Ok(hashed_state) } - let overlay_state = self.build_overlay(TrieInputSorted::default(), false)?.state; + let overlay_state = self.build_overlay(TrieInputSorted::default())?.state; zero_destroyed_account_storage( &HashedPostStateCursorFactory::new( DatabaseHashedCursorFactory::new(self.provider().tx()), @@ -926,7 +893,7 @@ where Self: 'a; fn account_trie_cursor(&self) -> Result, DatabaseError> { - let overlay = self.state_trie_overlay(true).map_err(into_database_error)?; + let overlay = self.state_trie_overlay().map_err(into_database_error)?; let cursor: Box = if self.is_v2 { Box::new(DatabaseAccountTrieCursor::<_, PackedKeyAdapter>::new( self.provider().tx().cursor_read::()?, @@ -936,14 +903,14 @@ where self.provider().tx().cursor_read::()?, )) }; - Ok(InMemoryTrieCursor::new_account(cursor, &overlay.input().nodes)) + Ok(InMemoryTrieCursor::new_account(cursor, &overlay.trie_updates)) } fn storage_trie_cursor( &self, hashed_address: B256, ) -> Result, DatabaseError> { - let overlay = self.state_trie_overlay(true).map_err(into_database_error)?; + let overlay = self.state_trie_overlay().map_err(into_database_error)?; let cursor: Box = if self.is_v2 { Box::new(DatabaseStorageTrieCursor::<_, PackedKeyAdapter>::new( self.provider().tx().cursor_dup_read::()?, @@ -955,7 +922,7 @@ where hashed_address, )) }; - Ok(InMemoryTrieCursor::new_storage(cursor, &overlay.input().nodes, hashed_address)) + Ok(InMemoryTrieCursor::new_storage(cursor, &overlay.trie_updates, hashed_address)) } } @@ -987,10 +954,10 @@ where Self: 'a; fn hashed_account_cursor(&self) -> Result, DatabaseError> { - let overlay = self.state_trie_overlay(true).map_err(into_database_error)?; + let overlay = self.state_trie_overlay().map_err(into_database_error)?; HashedPostStateCursorFactory::new( DatabaseHashedCursorFactory::new(self.provider().tx()), - &overlay.input().state, + &overlay.hashed_post_state, ) .hashed_account_cursor() } @@ -999,10 +966,10 @@ where &self, hashed_address: B256, ) -> Result, DatabaseError> { - let overlay = self.state_trie_overlay(true).map_err(into_database_error)?; + let overlay = self.state_trie_overlay().map_err(into_database_error)?; HashedPostStateCursorFactory::new( DatabaseHashedCursorFactory::new(self.provider().tx()), - &overlay.input().state, + &overlay.hashed_post_state, ) .hashed_storage_cursor(hashed_address) } @@ -1020,7 +987,7 @@ pub(crate) struct OverlayStateProviderFactoryMetrics { state_trie_overlay_cache_misses: Counter, } -type StateTrieOverlayCache = Arc>; +type StateTrieOverlayCache = Arc>; #[derive(Clone, Debug)] struct CachedExecutionOverlay { @@ -1155,11 +1122,11 @@ mod tests { } fn account_keys(overlay: &StateTrieOverlay) -> Vec { - overlay.input().state.accounts.iter().map(|(key, _)| *key).collect() + overlay.hashed_post_state.accounts.iter().map(|(key, _)| *key).collect() } fn account_node_paths(overlay: &StateTrieOverlay) -> Vec { - overlay.input().nodes.account_nodes_ref().iter().map(|(path, _)| *path).collect() + overlay.trie_updates.account_nodes_ref().iter().map(|(path, _)| *path).collect() } #[test] @@ -1175,7 +1142,7 @@ mod tests { ); let provider = state_provider_factory.database_provider_ro().unwrap(); - let first = provider.state_trie_overlay(false).unwrap().clone(); + let first = provider.state_trie_overlay().unwrap().clone(); assert_eq!(account_keys(&first), vec![B256::with_last_byte(3), B256::with_last_byte(4)]); drop(provider); @@ -1191,7 +1158,7 @@ mod tests { provider_rw.commit().unwrap(); let provider = state_provider_factory.database_provider_ro().unwrap(); - let second = provider.state_trie_overlay(false).unwrap().clone(); + let second = provider.state_trie_overlay().unwrap().clone(); assert_eq!(account_keys(&second), vec![B256::with_last_byte(4)]); assert_eq!(account_node_paths(&second), vec![Nibbles::from_nibbles([4])]); assert_eq!(state_provider_factory.state_trie_overlay_cache.len(), 2); @@ -1224,39 +1191,6 @@ mod tests { assert_eq!(state_provider_factory.state_trie_overlay_cache.len(), 1); } - #[test] - fn state_trie_overlay_cache_is_keyed_by_trie_changesets() { - let (factory, blocks) = setup_frontiers(1, 3); - let manager = OverlayManager::default(); - for block in &blocks[2..=3] { - manager.insert_block(block.clone()); - } - let state_provider_factory = OverlayStateProviderFactory::new( - factory, - manager.overlay_builder(blocks[3].recovered_block().hash()), - ); - let provider = state_provider_factory.database_provider_ro().unwrap(); - - provider.state_trie_overlay(false).unwrap(); - provider.state_trie_overlay(true).unwrap(); - - assert_eq!(state_provider_factory.state_trie_overlay_cache.len(), 2); - } - - #[test] - fn supplied_state_trie_overlay_is_available_in_both_modes() { - let (factory, _) = setup_frontiers(1, 1); - let provider = factory.provider().unwrap(); - let provider = OverlayStateProvider::<&_, EthPrimitives>::new_with_state_trie( - &provider, - StateTrieOverlay::new(TrieInputSorted::default()), - false, - ); - - assert!(provider.state_trie_overlay(false).is_ok()); - assert!(provider.state_trie_overlay(true).is_ok()); - } - #[test] fn lazy_overlays_survive_manager_block_removal() { let (factory, blocks) = setup_frontiers(1, 1); @@ -1278,7 +1212,7 @@ mod tests { [blocks[2].recovered_block().num_hash(), blocks[3].recovered_block().num_hash()] ); assert_eq!( - account_keys(provider.state_trie_overlay(false).unwrap()), + account_keys(provider.state_trie_overlay().unwrap()), vec![B256::with_last_byte(3), B256::with_last_byte(4)] ); } @@ -1296,7 +1230,7 @@ mod tests { ); let provider = state_provider_factory.database_provider_ro().unwrap(); - assert!(provider.state_trie_overlay(false).unwrap().skipped_for_reused_sparse_trie()); + assert!(provider.state_trie_overlay().unwrap().skipped_for_reused_sparse_trie()); assert!(state_provider_factory.state_trie_overlay_cache.is_empty()); assert!(matches!( provider.state_root(HashedPostState::default()), From 8e55cc50c95cfd1675149aab7281fd87d751b3c8 Mon Sep 17 00:00:00 2001 From: rakita Date: Wed, 9 Sep 2026 11:41:56 +0000 Subject: [PATCH 219/294] chore(deps): bump revm to 43.0.2 (#27115) --- Cargo.lock | 12 ++++++------ Cargo.toml | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 133785dd1cd..c655ecf4ae1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -10972,9 +10972,9 @@ dependencies = [ [[package]] name = "revm" -version = "43.0.1" +version = "43.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc210dfb3b45482a9f7c8806f737f7848fb7054b04235e6816480e515c1fe2d5" +checksum = "55f2322f5ea197fac72c2cdc88cb501b2a7db0b0009911133fd3dda45bfceedd" dependencies = [ "revm-bytecode", "revm-context", @@ -11067,9 +11067,9 @@ dependencies = [ [[package]] name = "revm-handler" -version = "43.0.1" +version = "43.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b8d988f896e22b19962658bc1afcd0989f727d48b2860a8bd7ceaecbf72ce34" +checksum = "73a7409031d87ec281b74efff65d7a4dbea9d80727a55dbc7e4b3fd77b5e03fb" dependencies = [ "auto_impl", "derive-where", @@ -11086,9 +11086,9 @@ dependencies = [ [[package]] name = "revm-inspector" -version = "43.0.1" +version = "43.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2584990403728b2e13fb5522510b61e4bb217c6a4c8f4d063447a6ddd7dd7cc" +checksum = "7a56f37224afd628f7244aa6965609ada04b440ed8ab7a19f55e4291df5a0112" dependencies = [ "auto_impl", "either", diff --git a/Cargo.toml b/Cargo.toml index 15b9024e28b..2d8bc4bf58d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -433,7 +433,7 @@ reth-trie-sparse = { path = "crates/trie/sparse", default-features = false } reth-zstd-compressors = { version = "0.7.0", default-features = false } # revm -revm = { version = "43.0.1", default-features = false } +revm = { version = "43.0.2", default-features = false } revmc = { git = "https://github.com/paradigmxyz/revmc", branch = "main", default-features = false, features = ["llvm-prefer-static"] } revm-inspectors = "0.43.0" From b668a317a7282b295dbdf7736fe6e1226cc34a9b Mon Sep 17 00:00:00 2001 From: onbjerg Date: Wed, 9 Sep 2026 12:41:46 +0000 Subject: [PATCH 220/294] chore: remove mdbook leftovers (#27117) --- .dockerignore | 1 - .github/workflows/book.yml | 2 +- .github/workflows/lint.yml | 4 ---- .github/workflows/unit.yml | 4 ---- book.toml | 27 --------------------------- 5 files changed, 1 insertion(+), 37 deletions(-) delete mode 100644 book.toml diff --git a/.dockerignore b/.dockerignore index f4baa18a38b..e9d3ac4167b 100644 --- a/.dockerignore +++ b/.dockerignore @@ -7,7 +7,6 @@ !/pkg !/testing !/.github/scripts/install_llvm_ubuntu.sh -!book.toml !Cargo.lock !Cargo.toml !Cross.toml diff --git a/.github/workflows/book.yml b/.github/workflows/book.yml index c8cea3e8832..ad4f40bc6c9 100644 --- a/.github/workflows/book.yml +++ b/.github/workflows/book.yml @@ -1,4 +1,4 @@ -# Documentation and mdbook related jobs. +# Documentation jobs. name: book diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index d8b935eea3c..47b2b30d370 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -43,10 +43,6 @@ jobs: - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 with: cache-on-failure: true - - if: "${{ matrix.type == 'book' }}" - uses: arduino/setup-protoc@c65c819552d16ad3c9b72d9dfd5ba5237b9c906b # v3.0.0 - with: - repo-token: ${{ secrets.GITHUB_TOKEN }} - name: Run clippy on binaries run: cargo clippy ${{ matrix.args }} --features "${{ matrix.features }}" env: diff --git a/.github/workflows/unit.yml b/.github/workflows/unit.yml index 42f83bc87e8..1c211cc92aa 100644 --- a/.github/workflows/unit.yml +++ b/.github/workflows/unit.yml @@ -51,10 +51,6 @@ jobs: - uses: taiki-e/install-action@b6b84cf49ebfe0176417bdce007c624f0db37f20 # v2.86.2 with: tool: nextest - - if: "${{ matrix.type == 'book' }}" - uses: arduino/setup-protoc@c65c819552d16ad3c9b72d9dfd5ba5237b9c906b # v3.0.0 - with: - repo-token: ${{ secrets.GITHUB_TOKEN }} - name: Run tests run: | cargo nextest run \ diff --git a/book.toml b/book.toml deleted file mode 100644 index 00838b0010d..00000000000 --- a/book.toml +++ /dev/null @@ -1,27 +0,0 @@ -[book] -authors = ["reth Core Contributors"] -language = "en" -multilingual = false -src = "book" -title = "reth Book" -description = "A book on all things Reth" - -[output.html] -theme = "book/theme" -git-repository-url = "https://github.com/paradigmxyz/reth" -default-theme = "ayu" -no-section-label = true - -[output.html.fold] -enable = true -level = 1 - -[build] -build-dir = "target/book" - -[preprocessor.template] -before = ["links"] - -[preprocessor.index] - -[preprocessor.links] From 8e773f0c6bf7c9ac16f4b603e654e1c1efddd1ad Mon Sep 17 00:00:00 2001 From: onbjerg Date: Wed, 9 Sep 2026 12:44:18 +0000 Subject: [PATCH 221/294] chore: prune unused zepter features (#27118) --- .config/zepter.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.config/zepter.yaml b/.config/zepter.yaml index 7aa1bd725ef..d2d5dfb5255 100644 --- a/.config/zepter.yaml +++ b/.config/zepter.yaml @@ -12,7 +12,7 @@ workflows: # Check that `A` activates the features of `B`. "propagate-feature", # These are the features to check: - "--features=std,op,dev,asm-keccak,jemalloc,jemalloc-prof,tracy-allocator,tracy,serde-bincode-compat,serde,test-utils,arbitrary,bench,alloy-compat,min-error-logs,min-warn-logs,min-info-logs,min-debug-logs,min-trace-logs,otlp,otlp-logs,js-tracer,portable,keccak-cache-global,trie-debug,secp256k1", + "--features=std,dev,asm-keccak,jemalloc,jemalloc-prof,tracy-allocator,tracy,serde-bincode-compat,serde,test-utils,arbitrary,min-error-logs,min-warn-logs,min-info-logs,min-debug-logs,min-trace-logs,otlp,otlp-logs,js-tracer,portable,keccak-cache-global,trie-debug,secp256k1", # Do not try to add a new section to `[features]` of `A` only because `B` exposes that feature. There are edge-cases where this is still needed, but we can add them manually. "--left-side-feature-missing=ignore", # Ignore the case that `A` it outside of the workspace. Otherwise it will report errors in external dependencies that we have no influence on. From 1533195359e583c964666ab24c606aaa82d70ab2 Mon Sep 17 00:00:00 2001 From: onbjerg Date: Wed, 9 Sep 2026 15:07:14 +0200 Subject: [PATCH 222/294] refactor(rpc): use Alloy BlockId display (#27120) --- Cargo.lock | 1 - crates/rpc/rpc-eth-types/src/error/mod.rs | 10 +++------- crates/rpc/rpc-eth-types/src/simulate.rs | 4 ++-- crates/rpc/rpc-server-types/Cargo.toml | 1 - crates/rpc/rpc-server-types/src/result.rs | 15 --------------- 5 files changed, 5 insertions(+), 26 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c655ecf4ae1..3bdb911bf35 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -10379,7 +10379,6 @@ dependencies = [ name = "reth-rpc-server-types" version = "2.5.2" dependencies = [ - "alloy-eips", "alloy-primitives", "alloy-rpc-types-engine", "jsonrpsee-core", diff --git a/crates/rpc/rpc-eth-types/src/error/mod.rs b/crates/rpc/rpc-eth-types/src/error/mod.rs index 5cbf7dd96ad..7374de36678 100644 --- a/crates/rpc/rpc-eth-types/src/error/mod.rs +++ b/crates/rpc/rpc-eth-types/src/error/mod.rs @@ -14,7 +14,7 @@ use reth_primitives_traits::transaction::{error::InvalidTransactionError, signed use reth_revm::db::bal::EvmDatabaseError; use reth_rpc_convert::{CallFeesError, EthTxEnvError, TransactionConversionError}; use reth_rpc_server_types::result::{ - block_id_to_str, internal_rpc_err, invalid_params_rpc_err, rpc_err, rpc_error_with_code, + internal_rpc_err, invalid_params_rpc_err, rpc_err, rpc_error_with_code, }; use reth_transaction_pool::error::{ Eip4844PoolTransactionError, Eip7702PoolTransactionError, InvalidPoolTransactionError, @@ -337,16 +337,12 @@ impl From for jsonrpsee_types::error::ErrorObject<'static> { EthApiError::HeaderNotFound(id) | EthApiError::ReceiptsNotFound(id) => { rpc_error_with_code( EthRpcErrorCode::ResourceNotFound.code(), - format!("block not found: {}", block_id_to_str(id)), + format!("block not found: {id}"), ) } EthApiError::HeaderRangeNotFound(start_id, end_id) => rpc_error_with_code( EthRpcErrorCode::ResourceNotFound.code(), - format!( - "{error}: start block: {}, end block: {}", - block_id_to_str(start_id), - block_id_to_str(end_id), - ), + format!("{error}: start block: {start_id}, end block: {end_id}"), ), err @ EthApiError::TransactionConfirmationTimeout { .. } => rpc_error_with_code( EthRpcErrorCode::TransactionConfirmationTimeout.code(), diff --git a/crates/rpc/rpc-eth-types/src/simulate.rs b/crates/rpc/rpc-eth-types/src/simulate.rs index 90e88592c6f..903eda2d872 100644 --- a/crates/rpc/rpc-eth-types/src/simulate.rs +++ b/crates/rpc/rpc-eth-types/src/simulate.rs @@ -23,7 +23,7 @@ use reth_primitives_traits::{ BlockBody as _, BlockTy, NodePrimitives, Recovered, RecoveredBlock, SealedHeader, }; use reth_rpc_convert::{RpcBlock, RpcConvert, RpcTxReq}; -use reth_rpc_server_types::result::{block_id_to_str, rpc_err}; +use reth_rpc_server_types::result::rpc_err; use reth_storage_api::{noop::NoopProvider, StateProvider}; use revm::{ context::Block, @@ -61,7 +61,7 @@ pub enum EthSimulateError { #[error("Client adjustable limit reached")] GasLimitReached, /// Base block for the simulation was not found. - #[error("block not found: {}", block_id_to_str(*block))] + #[error("block not found: {block}")] BlockNotFound { /// The block id that was requested. block: BlockId, diff --git a/crates/rpc/rpc-server-types/Cargo.toml b/crates/rpc/rpc-server-types/Cargo.toml index 275d8ea561b..c656c970829 100644 --- a/crates/rpc/rpc-server-types/Cargo.toml +++ b/crates/rpc/rpc-server-types/Cargo.toml @@ -18,7 +18,6 @@ reth-network-api.workspace = true # ethereum alloy-primitives.workspace = true alloy-rpc-types-engine.workspace = true -alloy-eips.workspace = true # rpc jsonrpsee-core.workspace = true diff --git a/crates/rpc/rpc-server-types/src/result.rs b/crates/rpc/rpc-server-types/src/result.rs index e2170dcfe04..7c518a5213f 100644 --- a/crates/rpc/rpc-server-types/src/result.rs +++ b/crates/rpc/rpc-server-types/src/result.rs @@ -2,7 +2,6 @@ use std::fmt; -use alloy_eips::BlockId; use alloy_rpc_types_engine::PayloadError; use jsonrpsee_core::RpcResult; use reth_errors::ConsensusError; @@ -152,20 +151,6 @@ pub fn rpc_err( ) } -/// Formats a [`BlockId`] into an error message. -pub fn block_id_to_str(id: BlockId) -> String { - match id { - BlockId::Hash(h) => { - if h.require_canonical == Some(true) { - format!("canonical hash {}", h.block_hash) - } else { - format!("hash {}", h.block_hash) - } - } - BlockId::Number(n) => format!("{n}"), - } -} - #[cfg(test)] mod tests { use super::*; From 1e8a570db3ff27f463063c6ff638c5328e427b71 Mon Sep 17 00:00:00 2001 From: onbjerg Date: Wed, 9 Sep 2026 13:04:34 +0000 Subject: [PATCH 223/294] refactor(primitives): use Alloy receipt root calculation (#27119) --- crates/ethereum/primitives/Cargo.toml | 2 +- crates/ethereum/primitives/src/receipt.rs | 12 ------------ crates/evm/execution-types/src/execution_outcome.rs | 2 +- 3 files changed, 2 insertions(+), 14 deletions(-) diff --git a/crates/ethereum/primitives/Cargo.toml b/crates/ethereum/primitives/Cargo.toml index bdd8a2c2f15..6c7a01ec3ad 100644 --- a/crates/ethereum/primitives/Cargo.toml +++ b/crates/ethereum/primitives/Cargo.toml @@ -18,7 +18,6 @@ reth-primitives-traits.workspace = true # ethereum alloy-eips = { workspace = true, features = ["k256"] } -alloy-primitives.workspace = true alloy-consensus = { workspace = true, features = ["serde"] } alloy-rpc-types-eth = { workspace = true, optional = true } @@ -26,6 +25,7 @@ alloy-rpc-types-eth = { workspace = true, optional = true } serde = { workspace = true, optional = true } [dev-dependencies] +alloy-primitives.workspace = true arbitrary.workspace = true proptest.workspace = true proptest-arbitrary-interop.workspace = true diff --git a/crates/ethereum/primitives/src/receipt.rs b/crates/ethereum/primitives/src/receipt.rs index f916620f6fc..cb7adb88562 100644 --- a/crates/ethereum/primitives/src/receipt.rs +++ b/crates/ethereum/primitives/src/receipt.rs @@ -1,8 +1,5 @@ use alloy_consensus::TxType; pub use alloy_consensus::{EthereumReceipt, TxTy}; -use alloy_eips::eip2718::Encodable2718; -use alloy_primitives::B256; -use reth_primitives_traits::proofs::ordered_trie_root_with_encoder; /// Raw ethereum receipt. pub type Receipt = EthereumReceipt; @@ -11,15 +8,6 @@ pub type Receipt = EthereumReceipt; /// Receipt representation for RPC. pub type RpcReceipt = EthereumReceipt; -/// Calculates the receipt root for a header for the reference type of [`Receipt`]. -/// -/// NOTE: Prefer `proofs::calculate_receipt_root` if you have log blooms memoized. -pub fn calculate_receipt_root_no_memo(receipts: &[Receipt]) -> B256 { - ordered_trie_root_with_encoder(receipts, |r, buf| { - alloy_consensus::TxReceipt::with_bloom_ref(r).encode_2718(buf) - }) -} - #[cfg(test)] mod tests { use super::*; diff --git a/crates/evm/execution-types/src/execution_outcome.rs b/crates/evm/execution-types/src/execution_outcome.rs index 8a11e21bc2a..3c7fc2c642b 100644 --- a/crates/evm/execution-types/src/execution_outcome.rs +++ b/crates/evm/execution-types/src/execution_outcome.rs @@ -411,7 +411,7 @@ impl ExecutionOutcome { pub fn ethereum_receipts_root(&self, block_number: BlockNumber) -> Option { self.generic_receipts_root_slow( block_number, - reth_ethereum_primitives::calculate_receipt_root_no_memo, + reth_ethereum_primitives::Receipt::calculate_receipt_root_no_memo, ) } } From 94eee5fc841237cbb0f00664336db90af321e879 Mon Sep 17 00:00:00 2001 From: onbjerg Date: Wed, 9 Sep 2026 14:25:09 +0000 Subject: [PATCH 224/294] refactor: delegate fee calculations to Alloy (#27123) --- crates/chainspec/src/api.rs | 11 +++-------- crates/rpc/rpc-eth-types/src/fee_history.rs | 11 ++--------- 2 files changed, 5 insertions(+), 17 deletions(-) diff --git a/crates/chainspec/src/api.rs b/crates/chainspec/src/api.rs index ee63132973c..553935b991b 100644 --- a/crates/chainspec/src/api.rs +++ b/crates/chainspec/src/api.rs @@ -1,7 +1,7 @@ use crate::{ChainSpec, DepositContract}; use alloc::{boxed::Box, vec::Vec}; use alloy_chains::Chain; -use alloy_eips::{calc_next_block_base_fee, eip1559::BaseFeeParams, eip7840::BlobParams}; +use alloy_eips::{eip1559::BaseFeeParams, eip7840::BlobParams}; use alloy_genesis::Genesis; use alloy_primitives::{B256, U256}; use core::fmt::{Debug, Display}; @@ -63,14 +63,9 @@ pub trait EthChainSpec: Send + Sync + Unpin + Debug { /// Returns the final total difficulty if the Paris hardfork is known. fn final_paris_total_difficulty(&self) -> Option; - /// See [`calc_next_block_base_fee`]. + /// See [`AlloyBlockHeader::next_block_base_fee`]. fn next_block_base_fee(&self, parent: &Self::Header, target_timestamp: u64) -> Option { - Some(calc_next_block_base_fee( - parent.gas_used(), - parent.gas_limit(), - parent.base_fee_per_gas()?, - self.base_fee_params_at_timestamp(target_timestamp), - )) + parent.next_block_base_fee(self.base_fee_params_at_timestamp(target_timestamp)) } } diff --git a/crates/rpc/rpc-eth-types/src/fee_history.rs b/crates/rpc/rpc-eth-types/src/fee_history.rs index 8b14633ece7..fd200236930 100644 --- a/crates/rpc/rpc-eth-types/src/fee_history.rs +++ b/crates/rpc/rpc-eth-types/src/fee_history.rs @@ -396,21 +396,14 @@ where /// /// See also [`Self::next_block_excess_blob_gas`] pub fn next_block_blob_fee(&self) -> Option { - self.next_block_excess_blob_gas() - .and_then(|excess_blob_gas| Some(self.blob_params?.calc_blob_fee(excess_blob_gas))) + self.header.maybe_next_block_blob_fee(self.blob_params) } /// Calculate excess blob gas for the next block according to the EIP-4844 spec. /// /// Returns a `None` if no excess blob gas is set, no EIP-4844 support pub fn next_block_excess_blob_gas(&self) -> Option { - self.header.excess_blob_gas().and_then(|excess_blob_gas| { - Some(self.blob_params?.next_block_excess_blob_gas_osaka( - excess_blob_gas, - self.header.blob_gas_used()?, - self.header.base_fee_per_gas()?, - )) - }) + self.header.maybe_next_block_excess_blob_gas(self.blob_params) } } From e88a669b1be89ef335e457d5093f8be61fc293ac Mon Sep 17 00:00:00 2001 From: nk_ysg Date: Wed, 9 Sep 2026 15:39:31 +0000 Subject: [PATCH 225/294] fix(rpc): hold tracing permits for proof tasks (#27107) --- crates/rpc/rpc-eth-api/src/helpers/state.rs | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/crates/rpc/rpc-eth-api/src/helpers/state.rs b/crates/rpc/rpc-eth-api/src/helpers/state.rs index 209f1d5926b..3fcfd25353d 100644 --- a/crates/rpc/rpc-eth-api/src/helpers/state.rs +++ b/crates/rpc/rpc-eth-api/src/helpers/state.rs @@ -166,7 +166,7 @@ pub trait EthState: LoadState + SpawnBlocking { Self: EthApiSpec, { Ok(async move { - let _permit = self + let permit = self .acquire_owned_tracing() .await .map_err(RethError::other) @@ -175,7 +175,8 @@ pub trait EthState: LoadState + SpawnBlocking { let block_id = block_id.unwrap_or_default(); self.ensure_within_proof_window(block_id)?; - self.spawn_blocking_io_fut(async move |this| { + self.spawn_blocking_io_fut(move |this| async move { + let _permit = permit; let state = this.state_at_block_id(block_id).await?; let storage_keys = keys.iter().map(|key| key.as_b256()).collect::>(); let proof = state @@ -200,7 +201,7 @@ pub trait EthState: LoadState + SpawnBlocking { Self: EthApiSpec, { Ok(async move { - let _permit = self + let permit = self .acquire_owned_tracing() .await .map_err(RethError::other) @@ -209,7 +210,8 @@ pub trait EthState: LoadState + SpawnBlocking { let block_id = block_id.unwrap_or_default(); self.ensure_within_proof_window(block_id)?; - self.spawn_blocking_io_fut(async move |this| { + self.spawn_blocking_io_fut(move |this| async move { + let _permit = permit; let state = this.state_at_block_id(block_id).await?; let mut proof_targets = MultiProofTargetsV2::default(); proof_targets.account_targets.reserve(targets.len()); From 99414e0912865fddf94aa609f1b1ee9d31a9ea81 Mon Sep 17 00:00:00 2001 From: Chase Wright Date: Wed, 9 Sep 2026 16:17:08 +0000 Subject: [PATCH 226/294] chore(peers): replace EF bootnodes with NodeOps fleet (#27130) --- crates/net/peers/src/bootnodes.rs | 31 +++++++++++++++++-------------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/crates/net/peers/src/bootnodes.rs b/crates/net/peers/src/bootnodes.rs index 81e4991dc92..9faba462612 100644 --- a/crates/net/peers/src/bootnodes.rs +++ b/crates/net/peers/src/bootnodes.rs @@ -6,20 +6,21 @@ use crate::NodeRecord; use alloc::vec::Vec; /// Ethereum Foundation Go Bootnodes -pub static MAINNET_BOOTNODES: [&str; 4] = [ - "enode://d860a01f9722d78051619d1e2351aba3f43f943f6f00718d1b9baa4101932a1f5011f16bb2b1bb35db20d6fe28fa0bf09636d26a87d31de9ec6203eeedb1f666@18.138.108.67:30303", // bootnode-aws-ap-southeast-1-001 - "enode://22a8232c3abc76a16ae9d6c3b164f98775fe226f0917b0ca871128a74a8e9630b458460865bab457221f1d448dd9791d24c4e5d88786180ac185df813a68d4de@3.209.45.79:30303", // bootnode-aws-us-east-1-001 - "enode://2b252ab6a1d0f971d9722cb839a42cb81db019ba44c08754628ab4a823487071b5695317c8ccd085219c3a03af063495b2f1da8d18218da2d6a82981b45e6ffc@65.108.70.101:30303", // bootnode-hetzner-hel - "enode://4aeb4ab6c14b23e2c4cfdce879c04b0748a20d8e9b59e25ded2a08143e265c6c25936e74cbc8e641e3312ca288673d91f2f93f8e277de3cfa444ecdaaf982052@157.90.35.166:30303", // bootnode-hetzner-fsn +pub static MAINNET_BOOTNODES: [&str; 5] = [ + "enode://ca967418ba165105303cfbb733dfb92bfcab80d65009d5e5f158c8e9e5f2c90795ae396a28d2114d66b4001e123e8c2c0465b018aed619fff41faca2ab4d2e64@212.99.218.66:0?discport=20151", // nodeops-bootnode-dcl1-01 + "enode://de4265bc38cba8508a14226356fabf480b88da880c19da49df57806a5b242f2ada08c25e54725c5ce1a762436c8f89e91781787c6b153ef9da72d00b62048170@129.212.166.61:0?discport=30303", // nodeops-bootnode-sfo3-01 + "enode://0b75ce940a50f9b3f37cfa162120642d16a4584774317fdaa02d0eec8f8aaf11fac79ba9d4213f33da9d4c512a3e0716a286b1b8f715c0a2cf5ac630030e0672@144.126.252.24:0?discport=30303", // nodeops-bootnode-blr1-01 + "enode://01472729e328876c6ccea993cf326b8a1ac969d4da4eac57ffa6b79d2d8338fa21012ff6c05b7ed78811a2f91768944219d9a3f0d33cb97ee0c128a8ed26386b@178.156.215.140:0?discport=30303", // nodeops-bootnode-ash-01 + "enode://0c949a7bc8d71b95ab9889c11e481a4743314093d5a0de5871de27bd8294e884b8e3b230b7cecfaf421fa6617aeff897e414444d7b4bf5b249dc9604716077ba@5.223.94.81:0?discport=30303", // nodeops-bootnode-sin-01 ]; /// Ethereum Foundation Sepolia Bootnodes pub static SEPOLIA_BOOTNODES: [&str; 5] = [ - "enode://4e5e92199ee224a01932a377160aa432f31d0b351f84ab413a8e0a42f4f36476f8fb1cbe914af0d9aef0d51665c214cf653c651c4bbd9d5550a934f241f1682b@138.197.51.181:30303", // sepolia-bootnode-1-nyc3 - "enode://143e11fb766781d22d92a2e33f8f104cddae4411a122295ed1fdb6638de96a6ce65f5b7c964ba3763bba27961738fef7d3ecc739268f3e5e771fb4c87b6234ba@146.190.1.103:30303", // sepolia-bootnode-1-sfo3 - "enode://8b61dc2d06c3f96fddcbebb0efb29d60d3598650275dc469c22229d3e5620369b0d3dedafd929835fe7f489618f19f456fe7c0df572bf2d914a9f4e006f783a9@170.64.250.88:30303", // sepolia-bootnode-1-syd1 - "enode://10d62eff032205fcef19497f35ca8477bea0eadfff6d769a147e895d8b2b8f8ae6341630c645c30f5df6e67547c03494ced3d9c5764e8622a26587b083b028e8@139.59.49.206:30303", // sepolia-bootnode-1-blr1 - "enode://9e9492e2e8836114cc75f5b929784f4f46c324ad01daf87d956f98b3b6c5fcba95524d6e5cf9861dc96a2c8a171ea7105bb554a197455058de185fa870970c7c@138.68.123.152:30303", // sepolia-bootnode-1-ams3 + "enode://4aff27bd8f1f667a56be304fcab797b4d7b630bf78581ffe6bc0d84852bb73362361ab2884ad396418abe25e7da621be0cadf5b02b6709b49d76b404813c9ddc@212.99.218.66:0?discport=20152", // nodeops-bootnode-dcl1-01 + "enode://665565ef7b9734bafb27fda8234ca43ca51ea097d0f29d9c9340daee0437c9e1d409c8283d1ce135eedccc2850f94c39c3cc63c641ad5fb0bef9dd37bd0fa6c1@129.212.166.61:0?discport=30403", // nodeops-bootnode-sfo3-01 + "enode://8e41eb6b03ef7b4c42d4cee19e8150f5fdc1ca28d9e33a627d09875e493a2bedfdea0bfe8c5bab778929a5334a311ca8d37e9016928fc6141c8fe52e708fbd98@144.126.252.24:0?discport=30403", // nodeops-bootnode-blr1-01 + "enode://b1e27df0cb42adc27b990879a5c4c99ce1bd15bdf0b829afdfdec50fbe352ad602075607f488ca805781e7f127d9709117a8bf8763ebac6e769f7cf50ded3806@178.156.215.140:0?discport=30403", // nodeops-bootnode-ash-01 + "enode://02dc5303f128bd0c8055a1fb126c9929bd74d401fca2f2aeba5d74208d1854b08cd5e7932d2db39e21891bf0347599b2cb63bedf14c96fac2bd92c77def82352@5.223.94.81:0?discport=30403", // nodeops-bootnode-sin-01 ]; /// Ethereum Foundation Holesky Bootnodes @@ -30,10 +31,12 @@ pub static HOLESKY_BOOTNODES: [&str; 2] = [ /// Ethereum Foundation Hoodi Bootnodes /// From: -pub static HOODI_BOOTNODES: [&str; 3] = [ - "enode://2112dd3839dd752813d4df7f40936f06829fc54c0e051a93967c26e5f5d27d99d886b57b4ffcc3c475e930ec9e79c56ef1dbb7d86ca5ee83a9d2ccf36e5c240c@134.209.138.84:30303", - "enode://60203fcb3524e07c5df60a14ae1c9c5b24023ea5d47463dfae051d2c9f3219f309657537576090ca0ae641f73d419f53d8e8000d7a464319d4784acd7d2abc41@209.38.124.160:30303", - "enode://8ae4a48101b2299597341263da0deb47cc38aa4d3ef4b7430b897d49bfa10eb1ccfe1655679b1ed46928ef177fbf21b86837bd724400196c508427a6f41602cd@134.199.184.23:30303", +pub static HOODI_BOOTNODES: [&str; 5] = [ + "enode://70bab91175f9bbcebbfbf155644f46c03ed44ef4927d816e3601ba5fd32f7a240ffb8953e737dd8595cf3773a62230ca27dd5390d268570f980f7d47a40dd1a3@212.99.218.66:0?discport=20153", // nodeops-bootnode-dcl1-01 + "enode://afd50407db4562c5049f3af489743a24b732cba6af2195e62205be59ae645aa0a4179da81515bfcfd20d3fd1f12ffdfcf5d4853ff1ad39189c3bb97132cd0986@129.212.166.61:0?discport=30503", // nodeops-bootnode-sfo3-01 + "enode://f787af01c154fb0fe82798d5c72ca81e3f2a789798a476a6c93da7acbb3f58002e7494c916680035a5beed158092c10bb08b10917bbc3ac41784e7b01017b4a4@144.126.252.24:0?discport=30503", // nodeops-bootnode-blr1-01 + "enode://71c400e99baa31e91d8ea7bda5850ef143b9da94b07413dbd4c1c84c4b5fc474806234c123bc5e1a97b7ae3162115cad9ccdf290319b0710f9d40d81b967153e@178.156.215.140:0?discport=30503", // nodeops-bootnode-ash-01 + "enode://81aabd7223f345e713b7f355cc76680ba8ec3a7b098f100655a467e4049113586fb5ce052a6ef90033ec62b91ed63457c87aa49dd0f1338af963b85522176db7@5.223.94.81:0?discport=30503", // nodeops-bootnode-sin-01 ]; /// Returns parsed mainnet nodes From b629142d0680d17600593a4d13add3848ce59956 Mon Sep 17 00:00:00 2001 From: ControlCplusControlV <44706811+ControlCplusControlV@users.noreply.github.com> Date: Wed, 9 Sep 2026 16:36:03 +0000 Subject: [PATCH 227/294] fix(provider): preserve refreshed static file cache entries (#27131) Co-authored-by: Matthias Seitz --- .../storage/provider/src/providers/static_file/manager.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/crates/storage/provider/src/providers/static_file/manager.rs b/crates/storage/provider/src/providers/static_file/manager.rs index a0e49fbce84..46bbff3d800 100644 --- a/crates/storage/provider/src/providers/static_file/manager.rs +++ b/crates/storage/provider/src/providers/static_file/manager.rs @@ -974,7 +974,11 @@ impl StaticFileProvider { trace!(target: "providers::static_file", ?segment, ?fixed_block_range, "Creating jar from scratch"); let path = self.path.join(segment.filename(fixed_block_range)); let jar = NippyJar::load(&path).map_err(ProviderError::other)?; - self.map.entry(key).insert(LoadedJar::new(jar)?).downgrade().into() + let loaded = LoadedJar::new(jar)?; + // The cache may have been populated since the initial miss, including by + // `update_index` publishing a newer snapshot while we loaded this jar without a lock. + // Preserve that entry instead of overwriting it with our potentially stale snapshot. + self.map.entry(key).or_insert(loaded).downgrade().into() }; if let Some(metrics) = &self.metrics { From 1da73cf352aab75e36df320b88b8099fb0245065 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Wed, 9 Sep 2026 17:16:09 +0000 Subject: [PATCH 228/294] perf(engine): start trie proofs with an early update batch (#27093) Co-authored-by: Matthias Seitz <19890894+mattsse@users.noreply.github.com> --- .../tree/state_root_strategy/sparse_trie.rs | 99 +++++++++++++++++++ 1 file changed, 99 insertions(+) diff --git a/crates/engine/tree/src/tree/state_root_strategy/sparse_trie.rs b/crates/engine/tree/src/tree/state_root_strategy/sparse_trie.rs index d1d0ee1b24a..c9647b57122 100644 --- a/crates/engine/tree/src/tree/state_root_strategy/sparse_trie.rs +++ b/crates/engine/tree/src/tree/state_root_strategy/sparse_trie.rs @@ -113,6 +113,8 @@ pub(super) struct SparseTrieCacheTask= INITIAL_UPDATE_BATCH_SIZE + { + // Start proof fetching before a continuously arriving state stream drains. Later + // batches retain the usual coalescing policy to avoid repeatedly sorting small maps. + let t = Instant::now(); + self.process_new_updates()?; + self.metrics.sparse_trie_process_updates_duration_histogram.record(t.elapsed()); + self.dispatch_pending_targets()?; } else if self.pending_targets.len() > self.chunk_size { // Make sure to dispatch targets if we've accumulated a lot of them. self.dispatch_pending_targets()?; @@ -570,6 +581,7 @@ where let _span = debug_span!("process_new_updates").entered(); self.pending_updates = 0; + self.initial_updates_applied = true; // Firstly apply all new storage and account updates to the tries. self.process_leaf_updates(true)?; @@ -1007,6 +1019,9 @@ pub(super) struct SparseTrieTaskMetrics { /// fetched by a single worker. If exceeded, chunking is forced regardless of worker availability. const DEFAULT_MAX_TARGETS_FOR_CHUNKING: usize = 300; +/// Start proof fetching while the first state-update batch is still arriving. +const INITIAL_UPDATE_BATCH_SIZE: usize = 64; + /// Dispatches work items as a single unit or in chunks based on target size and worker /// availability. #[expect(clippy::too_many_arguments)] @@ -1214,6 +1229,90 @@ mod tests { assert_eq!(account_rlp_buf, encoded); } + #[test] + fn first_leaf_batch_starts_proofs_before_input_queue_drains() { + let runtime = reth_tasks::Runtime::test(); + let provider_factory = create_test_provider_factory(); + let anchor_hash = init_genesis(&provider_factory).expect("failed to initialize genesis"); + let state_provider_factory = OverlayStateProviderFactory::new( + provider_factory, + OverlayManager::::default() + .overlay_builder(anchor_hash), + ); + let (proof_result_tx, proof_result_rx) = crossbeam_channel::unbounded(); + let proof_worker_handle = ProofWorkerHandle::new( + &runtime, + ProofTaskCtx::new(state_provider_factory), + false, + proof_result_tx.clone(), + ); + + let default_trie = RevealableSparseTrie::blind_from(ArenaParallelSparseTrie::default()); + let trie = SparseStateTrie::default() + .with_accounts_trie(default_trie.clone()) + .with_default_storage_trie(default_trie) + .with_updates(true); + + let parent_state_root = B256::from([0x55; 32]); + let (updates_tx, updates_rx) = crossbeam_channel::unbounded(); + let (_cancel_guard, cancel_rx) = crossbeam_channel::bounded::<()>(0); + let mut task = SparseTrieCacheTask::new_with_trie( + &runtime, + updates_rx, + cancel_rx, + std::sync::mpsc::channel().0, + proof_worker_handle, + proof_result_tx, + proof_result_rx, + SparseTrieTaskMetrics::default(), + trie, + parent_state_root, + TrieNodeEpoch::UNMODIFIED, + 1, + ); + + // Keep an input queued so progress cannot use its normal queue-empty flush. + updates_tx.send(StateRootMessage::PrefetchProofs(Default::default())).unwrap(); + let deadline = std::time::Instant::now(); + while task.updates.is_empty() { + assert!(deadline.elapsed() < std::time::Duration::from_secs(1)); + std::thread::yield_now(); + } + for index in 0..INITIAL_UPDATE_BATCH_SIZE { + let mut state = HashedPostState::default(); + state.accounts.insert( + B256::repeat_byte(index as u8), + Some(Account { nonce: 1, ..Default::default() }), + ); + task.on_hashed_state_update(state); + task.pending_updates += 1; + assert!(!task.make_progress().unwrap()); + if index + 1 < INITIAL_UPDATE_BATCH_SIZE { + assert_eq!(task.in_flight_proof_batches, 0); + } + } + assert!(task.in_flight_proof_batches > 0, "proof work must start before the queue drains"); + assert_eq!(task.pending_updates, 0); + + // A second batch remains buffered; the early flush must not become a permanent small + // batch policy that repeatedly scans and sorts pending leaves. + for index in INITIAL_UPDATE_BATCH_SIZE..INITIAL_UPDATE_BATCH_SIZE * 2 { + let mut state = HashedPostState::default(); + state.accounts.insert( + B256::repeat_byte(index as u8), + Some(Account { nonce: 1, ..Default::default() }), + ); + task.on_hashed_state_update(state); + task.pending_updates += 1; + assert!(!task.make_progress().unwrap()); + } + assert_eq!(task.pending_updates, INITIAL_UPDATE_BATCH_SIZE); + assert_eq!(task.new_account_updates.len(), INITIAL_UPDATE_BATCH_SIZE); + drop(updates_tx); + drop(task); + drain_sparse_trie_tasks(&runtime); + } + #[test] fn run_returns_parent_root_without_revealing_blind_trie_when_no_state_updates() { let runtime = reth_tasks::Runtime::test(); From e63ec720ac2d11d9ff8694e54b22ac88b9714381 Mon Sep 17 00:00:00 2001 From: nk_ysg Date: Wed, 9 Sep 2026 18:31:45 +0000 Subject: [PATCH 229/294] fix(rpc): hold blocking IO permits for call tasks (#27106) Co-authored-by: Matthias Seitz --- crates/rpc/rpc-eth-api/src/helpers/bal.rs | 6 +++++- crates/rpc/rpc-eth-api/src/helpers/call.rs | 18 +++++++++++++++--- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/crates/rpc/rpc-eth-api/src/helpers/bal.rs b/crates/rpc/rpc-eth-api/src/helpers/bal.rs index 1f7492b8b0a..b6640ff5777 100644 --- a/crates/rpc/rpc-eth-api/src/helpers/bal.rs +++ b/crates/rpc/rpc-eth-api/src/helpers/bal.rs @@ -40,9 +40,13 @@ pub trait GetBlockAccessList: Trace + Call + LoadBlock + RpcNodeCoreExt { return Ok(Some(Vec::from(bal))) } - let _permit = self.acquire_owned_blocking_io().await; + let permit = self + .acquire_owned_blocking_io() + .await + .map_err(|_| EthApiError::InternalEthError)?; self.spawn_blocking_io(move |eth_api| { + let _permit = permit; let state = eth_api .provider() .state_by_block_id(block.parent_hash().into()) diff --git a/crates/rpc/rpc-eth-api/src/helpers/call.rs b/crates/rpc/rpc-eth-api/src/helpers/call.rs index 2cf99fe6bd5..2cc96dfbdcf 100644 --- a/crates/rpc/rpc-eth-api/src/helpers/call.rs +++ b/crates/rpc/rpc-eth-api/src/helpers/call.rs @@ -94,7 +94,10 @@ pub trait EthCall: EstimateCall + Call + LoadPendingBlock + LoadBlock + FullEthA return Err(EthApiError::InvalidParams(String::from("calls are empty.")).into()) } - let _permit = self.acquire_owned_blocking_io().await; + let permit = self + .acquire_owned_blocking_io() + .await + .map_err(|_| EthApiError::InternalEthError)?; let base_block = self .recovered_block(block) @@ -104,6 +107,7 @@ pub trait EthCall: EstimateCall + Call + LoadPendingBlock + LoadBlock + FullEthA let max_simulate_blocks = self.max_simulate_blocks(); self.spawn_with_state_at_block(block, move |this, db| { + let _permit = permit; let state_provider = db.database.0; let mut db = State::builder() .with_database(StateProviderDatabase::new(&state_provider)) @@ -300,7 +304,6 @@ pub trait EthCall: EstimateCall + Call + LoadPendingBlock + LoadBlock + FullEthA overrides: EvmOverrides, ) -> impl Future> + Send { async move { - let _permit = self.acquire_owned_blocking_io().await; let res = self.transact_call_at(request, block_number.unwrap_or_default(), overrides).await?; @@ -325,7 +328,10 @@ pub trait EthCall: EstimateCall + Call + LoadPendingBlock + LoadBlock + FullEthA return Err(EthApiError::InvalidParams(String::from("bundles are empty.")).into()); } - let _permit = self.acquire_owned_blocking_io().await; + let permit = self + .acquire_owned_blocking_io() + .await + .map_err(|_| EthApiError::InternalEthError)?; let StateContext { transaction_index, block_number } = state_context.unwrap_or_default(); @@ -370,6 +376,7 @@ pub trait EthCall: EstimateCall + Call + LoadPendingBlock + LoadBlock + FullEthA } self.spawn_with_state_at_block(at, move |this, mut db| { + let _permit = permit; let mut all_results = Vec::with_capacity(bundles.len()); if replay_block_txs { @@ -599,12 +606,17 @@ pub trait Call: Self: LoadPendingBlock, { async move { + let permit = self + .acquire_owned_blocking_io() + .await + .map_err(|_| EthApiError::InternalEthError)?; let guard = CancelOnDrop::default(); let cancel = guard.clone(); let this = self.clone(); let res = self .spawn_with_call_at(request, at, overrides, move |db, evm_env, tx_env| { + let _permit = permit; if cancel.is_cancelled() { // callsite dropped the guard return Err(EthApiError::InternalEthError.into()) From 4bf2ff217a0a80912424d7cdf19a2efe32b47039 Mon Sep 17 00:00:00 2001 From: onbjerg Date: Wed, 9 Sep 2026 21:40:21 +0000 Subject: [PATCH 230/294] refactor(rpc): use Alloy message signing helper (#27132) --- crates/rpc/rpc/src/eth/helpers/signer.rs | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/crates/rpc/rpc/src/eth/helpers/signer.rs b/crates/rpc/rpc/src/eth/helpers/signer.rs index 214d8438349..086d8334e28 100644 --- a/crates/rpc/rpc/src/eth/helpers/signer.rs +++ b/crates/rpc/rpc/src/eth/helpers/signer.rs @@ -2,7 +2,7 @@ use alloy_dyn_abi::TypedData; use alloy_eips::eip2718::Decodable2718; -use alloy_primitives::{eip191_hash_message, map::AddressMap, Address, Signature, B256}; +use alloy_primitives::{map::AddressMap, Address, Signature, B256}; use alloy_signer::SignerSync; use alloy_signer_local::{coins_bip39::English, MnemonicBuilder, PrivateKeySigner}; use reth_rpc_convert::SignableTxRequest; @@ -82,10 +82,7 @@ impl> EthSigner for DevS } async fn sign(&self, address: Address, message: &[u8]) -> Result { - // Hash message according to EIP 191: - // https://ethereum.org/es/developers/docs/apis/json-rpc/#eth_sign - let hash = eip191_hash_message(message); - self.sign_hash(hash, address) + self.get_key(address)?.sign_message_sync(message).map_err(|_| SignError::CouldNotSign) } async fn sign_transaction(&self, request: TxReq, address: &Address) -> Result { From 51faa4581408bd5fde806652269f6abaf235827c Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Thu, 10 Sep 2026 13:38:47 +0000 Subject: [PATCH 231/294] chore(rpc): increase default transaction hash cache to 100k (#27137) --- crates/node/core/src/args/rpc_server.rs | 2 +- crates/rpc/rpc-server-types/src/constants.rs | 2 +- docs/vocs/docs/pages/cli/reth/node.mdx | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/crates/node/core/src/args/rpc_server.rs b/crates/node/core/src/args/rpc_server.rs index 22f5ceba6d8..74514481335 100644 --- a/crates/node/core/src/args/rpc_server.rs +++ b/crates/node/core/src/args/rpc_server.rs @@ -1222,7 +1222,7 @@ mod tests { max_headers: 1000, max_bals: 1000, max_concurrent_db_requests: 512, - max_cached_tx_hashes: 30_000, + max_cached_tx_hashes: 100_000, }, gas_price_oracle: GasPriceOracleArgs { blocks: 20, diff --git a/crates/rpc/rpc-server-types/src/constants.rs b/crates/rpc/rpc-server-types/src/constants.rs index 042768b67c9..e2ad2e6671d 100644 --- a/crates/rpc/rpc-server-types/src/constants.rs +++ b/crates/rpc/rpc-server-types/src/constants.rs @@ -130,5 +130,5 @@ pub mod cache { pub const DEFAULT_CONCURRENT_DB_REQUESTS: usize = 512; /// Default maximum number of transaction hashes to cache for lookups. - pub const DEFAULT_MAX_CACHED_TX_HASHES: u32 = 30_000; + pub const DEFAULT_MAX_CACHED_TX_HASHES: u32 = 100_000; } diff --git a/docs/vocs/docs/pages/cli/reth/node.mdx b/docs/vocs/docs/pages/cli/reth/node.mdx index c88b071e9c9..8e14dcff292 100644 --- a/docs/vocs/docs/pages/cli/reth/node.mdx +++ b/docs/vocs/docs/pages/cli/reth/node.mdx @@ -557,7 +557,7 @@ RPC State Cache: --rpc-cache.max-cached-tx-hashes Maximum number of transaction hashes to cache for transaction lookups - [default: 30000] + [default: 100000] Gas Price Oracle: --gpo.blocks From 040bef7b319c7ec145689e03fe78710e11986db6 Mon Sep 17 00:00:00 2001 From: imitater-dou Date: Thu, 10 Sep 2026 13:47:05 +0000 Subject: [PATCH 232/294] docs: point layout.md at reth-core for moved crates (#27135) Signed-off-by: ADou --- docs/repo/layout.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/repo/layout.md b/docs/repo/layout.md index 6ecc21e1076..04d55023d9f 100644 --- a/docs/repo/layout.md +++ b/docs/repo/layout.md @@ -39,8 +39,8 @@ All binaries are stored in [`bin`](../../bin). These crates are related to the database. -- [`storage/codecs`](../../crates/storage/codecs): Different storage codecs. -- [`storage/codecs/derive`](../../crates/storage/codecs/derive): Derive macros for storage codecs. +- [`storage/codecs`](https://github.com/paradigmxyz/reth-core/tree/main/crates/codecs): Different storage codecs (lives in [reth-core](https://github.com/paradigmxyz/reth-core)). +- [`storage/codecs/derive`](https://github.com/paradigmxyz/reth-core/tree/main/crates/codecs-derive): Derive macros for storage codecs (lives in [reth-core](https://github.com/paradigmxyz/reth-core)). - [`storage/libmdbx-rs`](../../crates/storage/libmdbx-rs): Rust bindings for [libmdbx](https://github.com/erthink/libmdbx). A fork of an earlier Apache-licensed version of [libmdbx-rs][libmdbx-rs]. - [`storage/db`](../../crates/storage/db): Strongly typed Database abstractions (transactions, cursors, tables) over lower level database backends. - Implemented backends: mdbx @@ -52,7 +52,7 @@ These crates are related to the database. - [`storage/rpc-provider`](../../crates/storage/rpc-provider): Storage provider implementations tailored for RPC access patterns. - [`storage/errors`](../../crates/storage/errors): Common error types used by storage crates. - [`storage/nippy-jar`](../../crates/storage/nippy-jar): Compressed columnar storage for historical data. -- [`storage/zstd-compressors`](../../crates/storage/zstd-compressors): Zstandard-based compressors used by storage components. +- [`storage/zstd-compressors`](https://github.com/paradigmxyz/reth-core/tree/main/crates/zstd-compressors): Zstandard-based compressors used by storage components (lives in [reth-core](https://github.com/paradigmxyz/reth-core)). ### Networking @@ -172,8 +172,8 @@ Crates related to building and validating payloads (blocks). These crates define primitive types or algorithms. -- [`primitives`](../../crates/primitives): Commonly used types in Reth. -- [`primitives-traits`](../../crates/primitives-traits/): Common abstracted types in reth. +- [`ethereum/primitives`](../../crates/ethereum/primitives): Commonly used Ethereum primitive types in Reth (successor to the removed top-level `crates/primitives`). +- [`primitives-traits`](https://github.com/paradigmxyz/reth-core/tree/main/crates/primitives-traits): Common abstracted types in reth (lives in [reth-core](https://github.com/paradigmxyz/reth-core)). - [`trie`](../../crates/trie): An implementation of a Merkle Patricia Trie used for various roots (e.g. the state root) in Ethereum. ### Ethereum-Specific Crates From a3eea4364185db4df75f9ea0c1661e18cc8f8a01 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Fri, 11 Sep 2026 03:44:48 -0400 Subject: [PATCH 233/294] fix(net): preserve cell masks when coalescing ETH/72 announcements (#27090) --- crates/net/network/src/session/active.rs | 46 +++++++++++++++++++++++- 1 file changed, 45 insertions(+), 1 deletion(-) diff --git a/crates/net/network/src/session/active.rs b/crates/net/network/src/session/active.rs index 34e22aee46e..8f7b38e0684 100644 --- a/crates/net/network/src/session/active.rs +++ b/crates/net/network/src/session/active.rs @@ -1191,6 +1191,10 @@ impl OutgoingMessage { EthMessage::NewPooledTransactionHashes72(existing), NewPooledTransactionHashes::Eth72(inc), ) => { + // One mask describes every blob transaction in the message. + if existing.cell_mask != inc.cell_mask { + return Some(inc.into()) + } existing.hashes.extend(inc.hashes); existing.sizes.extend(inc.sizes); existing.types.extend(inc.types); @@ -1282,7 +1286,7 @@ impl QueuedOutgoingMessages { } /// Pushes a pooled transaction hash announcement, merging into the last queued message if - /// it is the same variant (eth66, eth68, or eth72). + /// it is the same variant (eth66, eth68, or eth72) and has a compatible cell mask. pub(crate) fn push_pooled_hashes(&mut self, msg: NewPooledTransactionHashes) { let msg = if let Some(last) = self.messages.back_mut() { match last.try_merge_hashes(msg) { @@ -2057,6 +2061,46 @@ mod tests { )); } + #[test] + fn eth72_coalescing_preserves_cell_masks() { + use alloy_primitives::B128; + + let masks = + [None, Some(B128::repeat_byte(0xff)), Some(B128::from(1u128)), Some(B128::from(2u128))]; + for existing_mask in masks { + for incoming_mask in masks { + let announcement = |mask: Option, byte| NewPooledTransactionHashes72 { + types: vec![if mask.is_some() { 3 } else { 2 }], + sizes: vec![100], + hashes: vec![B256::repeat_byte(byte)], + cell_mask: mask, + }; + let existing = announcement(existing_mask, 1); + let incoming = announcement(incoming_mask, 2); + let mut message: OutgoingMessage = + EthMessage::NewPooledTransactionHashes72(existing.clone()).into(); + + let remainder = message.try_merge_hashes(incoming.clone().into()); + + let OutgoingMessage::Eth(EthMessage::NewPooledTransactionHashes72(merged)) = + message + else { + panic!("expected eth72 announcement"); + }; + if existing_mask == incoming_mask { + assert!(remainder.is_none()); + assert_eq!(merged.cell_mask, existing_mask); + assert_eq!(merged.hashes, vec![B256::repeat_byte(1), B256::repeat_byte(2)]); + assert_eq!(merged.types, [existing.types, incoming.types].concat()); + assert_eq!(merged.sizes, vec![100, 100]); + } else { + assert_eq!(remainder, Some(incoming.into())); + assert_eq!(merged, existing); + } + } + } + } + #[test] fn eth72_pooled_hashes_count_broadcast_items() { let hashes = From 6763b2a2d07ec7f2c51df5bcc3c4f94d741b124b Mon Sep 17 00:00:00 2001 From: Soubhik Singha Mahapatra <160333583+Soubhik-10@users.noreply.github.com> Date: Fri, 11 Sep 2026 11:08:10 +0000 Subject: [PATCH 234/294] feat(payload): validate amsterdam payload fields (#27173) --- crates/ethereum/payload/src/validator.rs | 7 ++- crates/payload/validator/src/amsterdam.rs | 71 +++++++++++++++++++++++ crates/payload/validator/src/lib.rs | 1 + 3 files changed, 78 insertions(+), 1 deletion(-) create mode 100644 crates/payload/validator/src/amsterdam.rs diff --git a/crates/ethereum/payload/src/validator.rs b/crates/ethereum/payload/src/validator.rs index ccace26ef80..84f8ccecf73 100644 --- a/crates/ethereum/payload/src/validator.rs +++ b/crates/ethereum/payload/src/validator.rs @@ -3,7 +3,7 @@ use alloy_consensus::Block; use alloy_rpc_types_engine::{ExecutionData, PayloadError}; use reth_chainspec::EthereumHardforks; -use reth_payload_validator::{cancun, prague, shanghai}; +use reth_payload_validator::{amsterdam, cancun, prague, shanghai}; use reth_primitives_traits::{Block as _, SealedBlock, SignedTransaction}; use std::sync::Arc; @@ -103,5 +103,10 @@ where chain_spec.is_prague_active_at_timestamp(sealed_block.timestamp), )?; + amsterdam::ensure_well_formed_fields( + &sealed_block, + chain_spec.is_amsterdam_active_at_timestamp(sealed_block.timestamp), + )?; + Ok(sealed_block) } diff --git a/crates/payload/validator/src/amsterdam.rs b/crates/payload/validator/src/amsterdam.rs new file mode 100644 index 00000000000..60bf50dd336 --- /dev/null +++ b/crates/payload/validator/src/amsterdam.rs @@ -0,0 +1,71 @@ +//! Amsterdam rules for new payloads. + +use alloy_consensus::BlockHeader; +use alloy_rpc_types_engine::PayloadError; +use reth_primitives_traits::{Block, SealedBlock}; + +/// Checks that Amsterdam header fields are present if Amsterdam is active and absent otherwise. +#[inline] +pub fn ensure_well_formed_fields( + block: &SealedBlock, + is_amsterdam_active: bool, +) -> Result<(), PayloadError> { + if is_amsterdam_active { + if block.block_access_list_hash().is_none() { + return Err(PayloadError::PostAmsterdamBlockWithoutBlockAccessList) + } + if block.slot_number().is_none() { + return Err(PayloadError::PostAmsterdamBlockWithoutSlotNumber) + } + } else { + if block.block_access_list_hash().is_some() { + return Err(PayloadError::PreAmsterdamBlockWithBlockAccessList) + } + if block.slot_number().is_some() { + return Err(PayloadError::PreAmsterdamBlockWithSlotNumber) + } + } + + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use alloy_consensus::{Block as AlloyBlock, Header, TxEnvelope}; + + fn block( + has_block_access_list: bool, + has_slot_number: bool, + ) -> SealedBlock> { + let header = Header { + block_access_list_hash: has_block_access_list.then_some(Default::default()), + slot_number: has_slot_number.then_some(0), + ..Default::default() + }; + AlloyBlock { header, body: Default::default() }.seal_slow() + } + + #[test] + fn validates_amsterdam_fields() { + assert!(ensure_well_formed_fields(&block(true, true), true).is_ok()); + assert!(ensure_well_formed_fields(&block(false, false), false).is_ok()); + + assert!(matches!( + ensure_well_formed_fields(&block(false, true), true), + Err(PayloadError::PostAmsterdamBlockWithoutBlockAccessList) + )); + assert!(matches!( + ensure_well_formed_fields(&block(true, false), true), + Err(PayloadError::PostAmsterdamBlockWithoutSlotNumber) + )); + assert!(matches!( + ensure_well_formed_fields(&block(true, false), false), + Err(PayloadError::PreAmsterdamBlockWithBlockAccessList) + )); + assert!(matches!( + ensure_well_formed_fields(&block(false, true), false), + Err(PayloadError::PreAmsterdamBlockWithSlotNumber) + )); + } +} diff --git a/crates/payload/validator/src/lib.rs b/crates/payload/validator/src/lib.rs index d7853ffa3b0..90d8b704459 100644 --- a/crates/payload/validator/src/lib.rs +++ b/crates/payload/validator/src/lib.rs @@ -9,6 +9,7 @@ #![cfg_attr(docsrs, feature(doc_cfg))] #![cfg_attr(not(feature = "std"), no_std)] +pub mod amsterdam; pub mod cancun; pub mod prague; pub mod shanghai; From 2aaf3ce09a8cd2233d7ffd1e2fb508fd6445403b Mon Sep 17 00:00:00 2001 From: imitater-dou Date: Fri, 11 Sep 2026 07:05:19 -0700 Subject: [PATCH 235/294] docs: point eth-wire TransactionSigned at primitives lib.rs (#27176) Signed-off-by: ADou --- docs/crates/eth-wire.md | 34 ++++++---------------------------- 1 file changed, 6 insertions(+), 28 deletions(-) diff --git a/docs/crates/eth-wire.md b/docs/crates/eth-wire.md index 01ef767981f..c76091f764b 100644 --- a/docs/crates/eth-wire.md +++ b/docs/crates/eth-wire.md @@ -128,35 +128,13 @@ pub struct Transactions( And the corresponding transaction type is defined here: -[File: crates/ethereum/primitives/src/transaction.rs](../../crates/ethereum/primitives/src/transaction.rs) -```rust, ignore -#[reth_codec] -#[derive(Debug, Clone, PartialEq, Eq, Hash, AsRef, Deref, Default, Serialize, Deserialize)] -pub struct TransactionSigned { - pub hash: TxHash, - pub signature: Signature, - #[deref] - #[as_ref] - pub transaction: Transaction, -} - -impl Encodable for TransactionSigned { - fn encode(&self, out: &mut dyn bytes::BufMut) { - self.encode_inner(out, true); - } - - fn length(&self) -> usize { - let len = self.payload_len(); - len + length_of_length(len) - } -} +[File: crates/ethereum/primitives/src/lib.rs](../../crates/ethereum/primitives/src/lib.rs) +```rust,ignore +/// Typed Transaction type without a signature +pub type Transaction = alloy_consensus::EthereumTypedTransaction; -impl Decodable for TransactionSigned { - fn decode(buf: &mut &[u8]) -> alloy_rlp::Result { - // Implementation omitted for brevity - //... - } -} +/// Signed transaction. +pub type TransactionSigned = alloy_consensus::EthereumTxEnvelope; ``` Now that we know how the types work, let's take a look at how these are utilized in the network. From d779aae046271b62357ef1bbf3c619b9c31eb97c Mon Sep 17 00:00:00 2001 From: Kanan <93033289+kriss39@users.noreply.github.com> Date: Fri, 11 Sep 2026 14:23:02 +0000 Subject: [PATCH 236/294] test(net): cover eth/72 announcements in the rlp roundtrip fuzz (#27180) --- crates/net/eth-wire-types/src/broadcast.rs | 25 ++++++++++++++++++++- crates/net/eth-wire/tests/fuzz_roundtrip.rs | 5 +++-- 2 files changed, 27 insertions(+), 3 deletions(-) diff --git a/crates/net/eth-wire-types/src/broadcast.rs b/crates/net/eth-wire-types/src/broadcast.rs index 2c3921a7487..d6f9d46bf8c 100644 --- a/crates/net/eth-wire-types/src/broadcast.rs +++ b/crates/net/eth-wire-types/src/broadcast.rs @@ -824,7 +824,10 @@ impl proptest::prelude::Arbitrary for NewPooledTransactionHashes72 { // Map the usize values to the range 0..131072(0x20000) let sizes_vec = vec(proptest::num::usize::ANY.prop_map(|x| x % 131072), len..=len); let hashes_vec = vec(any::(), len..=len); - let cell_mask = any::>(); + // A zero mask is spelled `None`, so generating `Some(ZERO)` would produce a + // value that cannot survive its own encoding. + let cell_mask = + any::>().prop_map(|mask| mask.filter(|mask| !mask.is_zero())); (types_vec, sizes_vec, hashes_vec, cell_mask) }) @@ -1669,6 +1672,26 @@ mod tests { } } + #[test] + fn eth_72_zero_cell_mask_decodes_as_none() { + // The wire cannot tell a zero mask from an absent one, so `None` is the only + // representation that survives a round trip. + let zero = NewPooledTransactionHashes72 { + types: vec![], + sizes: vec![], + hashes: vec![], + cell_mask: Some(B128::ZERO), + }; + + let mut encoded = Vec::new(); + zero.encode(&mut encoded); + + let decoded = NewPooledTransactionHashes72::decode(&mut &encoded[..]).unwrap(); + + assert_eq!(decoded.cell_mask, None); + assert_eq!(encoded, hex!("d480c0c09000000000000000000000000000000000")); + } + #[test] fn eth_72_tx_hash_roundtrip() { let vectors = vec![ diff --git a/crates/net/eth-wire/tests/fuzz_roundtrip.rs b/crates/net/eth-wire/tests/fuzz_roundtrip.rs index 9cd9194ab1f..91799251fa7 100644 --- a/crates/net/eth-wire/tests/fuzz_roundtrip.rs +++ b/crates/net/eth-wire/tests/fuzz_roundtrip.rs @@ -54,8 +54,8 @@ pub mod fuzz_rlp { use reth_eth_wire::{ BlockBodies, BlockHeaders, DisconnectReason, GetBlockBodies, GetBlockHeaders, GetNodeData, GetPooledTransactions, GetReceipts, HelloMessage, NewBlock, NewBlockHashes, - NewPooledTransactionHashes66, NewPooledTransactionHashes68, NodeData, P2PMessage, - PooledTransactions, Receipts, Status, Transactions, + NewPooledTransactionHashes66, NewPooledTransactionHashes68, NewPooledTransactionHashes72, + NodeData, P2PMessage, PooledTransactions, Receipts, Status, Transactions, }; use serde::{Deserialize, Serialize}; use test_fuzz::test_fuzz; @@ -154,6 +154,7 @@ pub mod fuzz_rlp { fuzz_type_and_name!(NewBlock, fuzz_NewBlock); fuzz_type_and_name!(NewPooledTransactionHashes66, fuzz_NewPooledTransactionHashes66); fuzz_type_and_name!(NewPooledTransactionHashes68, fuzz_NewPooledTransactionHashes68); + fuzz_type_and_name!(NewPooledTransactionHashes72, fuzz_NewPooledTransactionHashes72); fuzz_type_and_name!(GetPooledTransactions, fuzz_GetPooledTransactions); fuzz_type_and_name!(PooledTransactions, fuzz_PooledTransactions); fuzz_type_and_name!(GetNodeData, fuzz_GetNodeData); From 844c104c8e5231ab2692e91a99d98b4d27e4796c Mon Sep 17 00:00:00 2001 From: Mattis Date: Sat, 12 Sep 2026 10:19:18 +0200 Subject: [PATCH 237/294] fix nix build process (#27202) Co-authored-by: Mattis Hasler --- flake.nix | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/flake.nix b/flake.nix index 9237003f6ae..bbc5c90ce11 100644 --- a/flake.nix +++ b/flake.nix @@ -13,6 +13,7 @@ outputs = { + self, nixpkgs, nixpkgs-llvm22, utils, @@ -55,6 +56,13 @@ pkgs.perl ]; + buildInputs = [ + pkgs.libffi + pkgs.libxml2 + pkgs.ncurses + pkgs.zlib + ]; + withClang = prev: { buildInputs = prev.buildInputs or [] ++ [ pkgs.clang @@ -96,11 +104,19 @@ ]; }; + withVergenBypass = prev: { + preBuild = '' + export VERGEN_GIT_SHA=${self.rev or self.dirtyRev} + export VERGEN_GIT_DIRTY=${if self ? rev then "false" else "true"} + export VERGEN_GIT_DESCRIBE=${self.shortRev or self.dirtyRev} + ''; + }; + mkReth = overrides: craneLib.buildPackage (composeAttrOverrides { pname = "reth"; version = packageVersion; src = ./.; - inherit nativeBuildInputs; + inherit nativeBuildInputs buildInputs; doCheck = false; } overrides); @@ -112,6 +128,7 @@ withClang withLlvm22 withMaxPerf + withVergenBypass ] ++ pkgs.lib.optionals pkgs.stdenv.isLinux [ withMold ]); From 087050198739b2f6610429aa30c777ddf1cfa5ba Mon Sep 17 00:00:00 2001 From: Derek Cofausper <256792747+decofe@users.noreply.github.com> Date: Sat, 12 Sep 2026 08:23:19 +0000 Subject: [PATCH 238/294] feat(txpool): add nonce bound check hook (#27185) Co-authored-by: Alexey Shekhirin <5773434+shekhirin@users.noreply.github.com> --- crates/transaction-pool/src/traits.rs | 8 ++++++ crates/transaction-pool/src/validate/eth.rs | 31 +++++++++++++++++++-- 2 files changed, 37 insertions(+), 2 deletions(-) diff --git a/crates/transaction-pool/src/traits.rs b/crates/transaction-pool/src/traits.rs index bb16e8ae72c..6c087fb11da 100644 --- a/crates/transaction-pool/src/traits.rs +++ b/crates/transaction-pool/src/traits.rs @@ -1491,6 +1491,14 @@ pub trait PoolTransaction: } } + /// Whether the transaction's nonce must be below [`u64::MAX`] according to EIP-2681. + /// + /// Defaults to `true`. Transactions with alternative nonce semantics can override this + /// independently of the sender nonce check in [`Self::requires_nonce_check`]. + fn requires_nonce_bound_check(&self) -> bool { + true + } + /// Allows to communicate to the pool that the transaction doesn't require a nonce check. fn requires_nonce_check(&self) -> bool { true diff --git a/crates/transaction-pool/src/validate/eth.rs b/crates/transaction-pool/src/validate/eth.rs index e8037b1fb3f..25339f13b48 100644 --- a/crates/transaction-pool/src/validate/eth.rs +++ b/crates/transaction-pool/src/validate/eth.rs @@ -487,8 +487,7 @@ where }; // Reject transactions with a nonce equal to U64::max according to EIP-2681 - let tx_nonce = transaction.nonce(); - if tx_nonce == u64::MAX { + if transaction.requires_nonce_bound_check() && transaction.nonce() == u64::MAX { return Err(InvalidPoolTransactionError::Eip2681) } @@ -1702,6 +1701,34 @@ mod tests { assert!(outcome.is_valid()); } + #[test] + fn validates_nonce_bound() { + let provider = MockEthProvider::default().with_genesis_block(); + let validator = EthTransactionValidatorBuilder::new(provider, test_evm_config()) + .build(InMemoryBlobStore::default()); + let transaction = |nonce| { + EthPooledTransaction::try_from_consensus( + TransactionBuilder::default() + .chain_id(validator.chain_id()) + .nonce(nonce) + .gas_limit(21_000) + .to(Address::ZERO) + .into_eip1559() + .try_into_recovered() + .unwrap(), + ) + .unwrap() + }; + + assert!(validator + .validate_stateless(TransactionOrigin::External, &transaction(u64::MAX - 1)) + .is_ok()); + assert!(matches!( + validator.validate_stateless(TransactionOrigin::External, &transaction(u64::MAX)), + Err(InvalidPoolTransactionError::Eip2681) + )); + } + #[test] fn validates_configured_chain_id() { let provider = MockEthProvider::default().with_genesis_block(); From 95823365b9f0787a676de38c044b54830e3fb29d Mon Sep 17 00:00:00 2001 From: Brian Bland Date: Sat, 12 Sep 2026 08:56:52 +0000 Subject: [PATCH 239/294] feat(txpool): make replacement policy extensible (#27201) Co-authored-by: Matthias Seitz --- crates/transaction-pool/src/config.rs | 103 +++++++++++++++++++- crates/transaction-pool/src/pool/txpool.rs | 4 +- crates/transaction-pool/src/traits.rs | 15 ++- crates/transaction-pool/src/validate/mod.rs | 52 ++-------- 4 files changed, 126 insertions(+), 48 deletions(-) diff --git a/crates/transaction-pool/src/config.rs b/crates/transaction-pool/src/config.rs index 0b22ac7970c..0a47dee12f2 100644 --- a/crates/transaction-pool/src/config.rs +++ b/crates/transaction-pool/src/config.rs @@ -3,7 +3,7 @@ use crate::{ pool::{NEW_TX_LISTENER_BUFFER_SIZE, PENDING_TX_LISTENER_BUFFER_SIZE}, PoolSize, TransactionOrigin, }; -use alloy_consensus::constants::EIP4844_TX_TYPE_ID; +use alloy_consensus::{constants::EIP4844_TX_TYPE_ID, Transaction}; use alloy_eips::eip1559::{ETHEREUM_BLOCK_GAS_LIMIT_30M, MIN_PROTOCOL_BASE_FEE}; use alloy_primitives::{map::AddressSet, Address}; use std::{ops::Mul, time::Duration}; @@ -202,6 +202,58 @@ impl PriceBumpConfig { } self.default_price_bump } + + /// Determines whether a candidate transaction (`maybe_replacement`) is underpriced compared to + /// an existing transaction in the pool. + /// + /// A transaction is considered underpriced if it doesn't meet the required fee bump threshold. + /// This applies to both standard gas fees and, for blob-carrying transactions (EIP-4844), + /// the blob-specific fees. + #[inline] + pub fn is_replacement_underpriced( + &self, + existing: &T, + maybe_replacement: &T, + ) -> bool { + // Retrieve the required price bump percentage for this type of transaction. + // + // The bump is different for EIP-4844 and other transactions. See `PriceBumpConfig`. + let price_bump = self.price_bump(existing.ty()); + let required_bumped_fee = + |existing_fee: u128| existing_fee.saturating_mul(100 + price_bump).div_ceil(100); + + // Check if the max fee per gas is underpriced. + if maybe_replacement.max_fee_per_gas() < required_bumped_fee(existing.max_fee_per_gas()) { + return true + } + + let existing_max_priority_fee_per_gas = + existing.max_priority_fee_per_gas().unwrap_or_default(); + let replacement_max_priority_fee_per_gas = + maybe_replacement.max_priority_fee_per_gas().unwrap_or_default(); + + // Check max priority fee per gas (relevant for EIP-1559 transactions only) + if existing_max_priority_fee_per_gas != 0 && + replacement_max_priority_fee_per_gas != 0 && + replacement_max_priority_fee_per_gas < + required_bumped_fee(existing_max_priority_fee_per_gas) + { + return true + } + + // Check max blob fee per gas + if let Some(existing_max_blob_fee_per_gas) = existing.max_fee_per_blob_gas() { + // This enforces that blob txs can only be replaced by blob txs + let replacement_max_blob_fee_per_gas = + maybe_replacement.max_fee_per_blob_gas().unwrap_or_default(); + if replacement_max_blob_fee_per_gas < required_bumped_fee(existing_max_blob_fee_per_gas) + { + return true + } + } + + false + } } impl Default for PriceBumpConfig { @@ -278,8 +330,57 @@ impl LocalTransactionConfig { #[cfg(test)] mod tests { + use alloy_consensus::{TxEip1559, TxEip4844}; + use super::*; + #[test] + fn replacement_uses_configured_price_bump() { + let config = PriceBumpConfig { default_price_bump: 25, ..Default::default() }; + let existing = + TxEip1559 { max_fee_per_gas: 100, max_priority_fee_per_gas: 10, ..Default::default() }; + let mut replacement = existing.clone(); + replacement.max_fee_per_gas = 125; + replacement.max_priority_fee_per_gas = 12; + assert!(config.is_replacement_underpriced(&existing, &replacement)); + + replacement.max_priority_fee_per_gas = 13; + assert!(!config.is_replacement_underpriced(&existing, &replacement)); + + replacement.max_fee_per_gas = 124; + assert!(config.is_replacement_underpriced(&existing, &replacement)); + } + + #[test] + fn blob_replacement_requires_all_fee_bumps() { + let config = PriceBumpConfig::default(); + let existing = TxEip4844 { + max_fee_per_gas: 100, + max_priority_fee_per_gas: 10, + max_fee_per_blob_gas: 50, + ..Default::default() + }; + let replacement = TxEip4844 { + max_fee_per_gas: 200, + max_priority_fee_per_gas: 20, + max_fee_per_blob_gas: 100, + ..existing.clone() + }; + assert!(!config.is_replacement_underpriced(&existing, &replacement)); + + let mut underpriced = replacement.clone(); + underpriced.max_fee_per_gas -= 1; + assert!(config.is_replacement_underpriced(&existing, &underpriced)); + + let mut underpriced = replacement.clone(); + underpriced.max_priority_fee_per_gas -= 1; + assert!(config.is_replacement_underpriced(&existing, &underpriced)); + + let mut underpriced = replacement; + underpriced.max_fee_per_blob_gas -= 1; + assert!(config.is_replacement_underpriced(&existing, &underpriced)); + } + #[test] fn test_pool_size_sanity() { let pool_size = PoolSize { diff --git a/crates/transaction-pool/src/pool/txpool.rs b/crates/transaction-pool/src/pool/txpool.rs index 0cf8e8127a2..b9ccc9c7992 100644 --- a/crates/transaction-pool/src/pool/txpool.rs +++ b/crates/transaction-pool/src/pool/txpool.rs @@ -2116,7 +2116,9 @@ impl AllTransactions { let maybe_replacement = transaction.as_ref(); // Ensure the new transaction is not underpriced - if existing_transaction.is_underpriced(maybe_replacement, &self.price_bumps) { + if existing_transaction + .is_replacement_underpriced(maybe_replacement, &self.price_bumps) + { return Err(InsertErr::Underpriced { transaction: pool_tx.transaction, existing: *entry.get().transaction.hash(), diff --git a/crates/transaction-pool/src/traits.rs b/crates/transaction-pool/src/traits.rs index 6c087fb11da..a41152ad8cb 100644 --- a/crates/transaction-pool/src/traits.rs +++ b/crates/transaction-pool/src/traits.rs @@ -58,7 +58,7 @@ use crate::{ TransactionListenerKind, }, validate::{TransactionValidationOutcome, TransactionValidator, ValidPoolTransaction}, - AddedTransactionOutcome, AllTransactionsEvents, + AddedTransactionOutcome, AllTransactionsEvents, PriceBumpConfig, }; use alloy_consensus::{error::ValueError, transaction::TxHashRef, BlockHeader, Signed, Typed2718}; use alloy_eips::{ @@ -1491,6 +1491,19 @@ pub trait PoolTransaction: } } + /// Returns whether `replacement` is underpriced relative to this transaction. + /// + /// Called on the existing transaction when another transaction would replace it. + /// By default, delegates to [`PriceBumpConfig::is_replacement_underpriced`]. + /// Implementations may override this to define transaction-specific replacement semantics. + fn is_replacement_underpriced( + &self, + replacement: &Self, + price_bumps: &PriceBumpConfig, + ) -> bool { + price_bumps.is_replacement_underpriced(self, replacement) + } + /// Whether the transaction's nonce must be below [`u64::MAX`] according to EIP-2681. /// /// Defaults to `true`. Transactions with alternative nonce semantics can override this diff --git a/crates/transaction-pool/src/validate/mod.rs b/crates/transaction-pool/src/validate/mod.rs index 9c941035c84..a6a8ef34f5d 100644 --- a/crates/transaction-pool/src/validate/mod.rs +++ b/crates/transaction-pool/src/validate/mod.rs @@ -460,52 +460,14 @@ impl ValidPoolTransaction { self.transaction.encoded_2718_consensus() } - /// Determines whether a candidate transaction (`maybe_replacement`) is underpriced compared to - /// an existing transaction in the pool. - /// - /// A transaction is considered underpriced if it doesn't meet the required fee bump threshold. - /// This applies to both standard gas fees and, for blob-carrying transactions (EIP-4844), - /// the blob-specific fees. + /// Returns whether `maybe_replacement` is underpriced relative to this transaction. #[inline] - pub fn is_underpriced(&self, maybe_replacement: &Self, price_bumps: &PriceBumpConfig) -> bool { - // Retrieve the required price bump percentage for this type of transaction. - // - // The bump is different for EIP-4844 and other transactions. See `PriceBumpConfig`. - let price_bump = price_bumps.price_bump(self.tx_type()); - let required_bumped_fee = - |existing_fee: u128| existing_fee.saturating_mul(100 + price_bump).div_ceil(100); - - // Check if the max fee per gas is underpriced. - if maybe_replacement.max_fee_per_gas() < required_bumped_fee(self.max_fee_per_gas()) { - return true - } - - let existing_max_priority_fee_per_gas = - self.transaction.max_priority_fee_per_gas().unwrap_or_default(); - let replacement_max_priority_fee_per_gas = - maybe_replacement.transaction.max_priority_fee_per_gas().unwrap_or_default(); - - // Check max priority fee per gas (relevant for EIP-1559 transactions only) - if existing_max_priority_fee_per_gas != 0 && - replacement_max_priority_fee_per_gas != 0 && - replacement_max_priority_fee_per_gas < - required_bumped_fee(existing_max_priority_fee_per_gas) - { - return true - } - - // Check max blob fee per gas - if let Some(existing_max_blob_fee_per_gas) = self.transaction.max_fee_per_blob_gas() { - // This enforces that blob txs can only be replaced by blob txs - let replacement_max_blob_fee_per_gas = - maybe_replacement.transaction.max_fee_per_blob_gas().unwrap_or_default(); - if replacement_max_blob_fee_per_gas < required_bumped_fee(existing_max_blob_fee_per_gas) - { - return true - } - } - - false + pub fn is_replacement_underpriced( + &self, + maybe_replacement: &Self, + price_bumps: &PriceBumpConfig, + ) -> bool { + self.transaction.is_replacement_underpriced(&maybe_replacement.transaction, price_bumps) } } From b2d7bd7f3fb04077f89f3368c6df9affdc8af095 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Mon, 14 Sep 2026 09:39:18 +0000 Subject: [PATCH 240/294] refactor(net): rewrite transaction fetcher (#26924) Co-authored-by: figtracer Co-authored-by: Matthias Seitz <19890894+mattsse@users.noreply.github.com> --- Cargo.lock | 1 + crates/net/network/Cargo.toml | 6 + crates/net/network/benches/tx_fetcher.rs | 262 ++ crates/net/network/src/metrics.rs | 24 +- .../network/src/test_utils/transactions.rs | 54 +- crates/net/network/src/transactions/config.rs | 65 +- .../net/network/src/transactions/constants.rs | 202 +- .../net/network/src/transactions/fetcher.rs | 4074 +++++++++++------ crates/net/network/src/transactions/mod.rs | 455 +- .../network/src/transactions/test_harness.rs | 464 ++ .../tests/it/transaction_hash_fetching.rs | 142 +- crates/node/core/src/args/network.rs | 2 +- docs/vocs/docs/pages/cli/reth/node.mdx | 2 +- docs/vocs/docs/pages/cli/reth/p2p/body.mdx | 2 +- docs/vocs/docs/pages/cli/reth/p2p/header.mdx | 2 +- docs/vocs/docs/pages/cli/reth/stage/run.mdx | 2 +- etc/grafana/dashboards/reth-mempool.json | 54 +- 17 files changed, 4007 insertions(+), 1806 deletions(-) create mode 100644 crates/net/network/benches/tx_fetcher.rs create mode 100644 crates/net/network/src/transactions/test_harness.rs diff --git a/Cargo.lock b/Cargo.lock index 3bdb911bf35..ce3c6ebd134 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9322,6 +9322,7 @@ dependencies = [ "alloy-trie", "aquamarine", "auto_impl", + "codspeed-criterion-compat", "derive_more", "discv5", "enr", diff --git a/crates/net/network/Cargo.toml b/crates/net/network/Cargo.toml index af752ef3f29..a8fca7bcd8f 100644 --- a/crates/net/network/Cargo.toml +++ b/crates/net/network/Cargo.toml @@ -101,6 +101,7 @@ alloy-genesis.workspace = true alloy-trie.workspace = true # misc +criterion.workspace = true test-case.workspace = true url.workspace = true secp256k1 = { workspace = true, features = ["rand"] } @@ -153,3 +154,8 @@ test-utils = [ "reth-trie/test-utils", "reth-evm/test-utils", ] + +[[bench]] +name = "tx_fetcher" +harness = false +required-features = ["test-utils"] diff --git a/crates/net/network/benches/tx_fetcher.rs b/crates/net/network/benches/tx_fetcher.rs new file mode 100644 index 00000000000..b97e59dfec0 --- /dev/null +++ b/crates/net/network/benches/tx_fetcher.rs @@ -0,0 +1,262 @@ +#![allow(missing_docs)] + +//! Benchmarks for the `TransactionFetcher`: recording announcements from many peers, packing +//! `GetPooledTransactions` requests for idle peers and processing their responses. + +use alloy_consensus::transaction::Recovered; +use alloy_primitives::{ + map::{B256Map, FbBuildHasher, HashMap}, + TxHash, +}; +use criterion::{criterion_group, criterion_main, BatchSize, Criterion, Throughput}; +use futures::{task::noop_waker_ref, StreamExt}; +use reth_eth_wire::{Eth68TxMetadata, EthVersion, PooledTransactions}; +use reth_ethereum_primitives::{PooledTransactionVariant, TransactionSigned}; +use reth_network::{ + test_utils::transactions::new_mock_session_with_capacity, + transactions::{ + constants::tx_fetcher::MAX_COUNT_EAGER_CANDIDATE_PEERS_PER_HASH, + fetcher::TransactionFetcher, PeerMetadata, + }, +}; +use reth_network_api::PeerRequest; +use reth_network_peers::PeerId; +use reth_transaction_pool::test_utils::MockTransactionFactory; +use std::task::{Context, Poll}; +use tokio::sync::mpsc; + +/// Number of connected peers. +const PEERS: usize = 64; +/// Number of transactions announced in the gossip scenarios, one full announcement. +const TXS: usize = 4096; +/// Announced size of every transaction. +const TX_SIZE: usize = 512; +/// Transaction type announced for every transaction. +const TX_TYPE: u8 = 2; + +/// A fetcher with mock peer sessions. +struct Rig { + fetcher: TransactionFetcher, + peers: HashMap>, + sessions: Vec<(PeerId, mpsc::Receiver)>, +} + +impl Rig { + fn new(num_peers: usize) -> Self { + let mut peers = HashMap::default(); + let mut sessions = Vec::with_capacity(num_peers); + for i in 0..num_peers { + let peer_id = peer_id(i); + let (peer, rx) = new_mock_session_with_capacity(peer_id, EthVersion::Eth68, 4); + peers.insert(peer_id, peer); + sessions.push((peer_id, rx)); + } + Self { fetcher: TransactionFetcher::default(), peers, sessions } + } + + /// Every peer announces the same hashes, like a gossiped batch of transactions. + fn announce_gossip(&mut self, announcement: &[(TxHash, Eth68TxMetadata)]) { + for (peer_id, _) in &self.sessions { + self.fetcher.on_announcement(*peer_id, announcement.iter().copied()); + } + } + + /// Every peer announces its own slice of the hashes. + fn announce_disjoint(&mut self, announcement: &[(TxHash, Eth68TxMetadata)]) { + let per_peer = announcement.len() / self.sessions.len(); + for (i, (peer_id, _)) in self.sessions.iter().enumerate() { + let slice = &announcement[i * per_peer..(i + 1) * per_peer]; + self.fetcher.on_announcement(*peer_id, slice.iter().copied()); + } + } + + fn dispatch(&mut self) -> usize { + self.fetcher.dispatch(&self.peers, usize::MAX) + } + + /// Answers every queued request with the given handler and processes the resulting events. + fn respond_all( + &mut self, + mut respond: impl FnMut(&[TxHash]) -> PooledTransactions, + ) { + for (_, rx) in &mut self.sessions { + while let Ok(PeerRequest::GetPooledTransactions { request, response }) = rx.try_recv() { + let _ = response.send(Ok(respond(&request.0))); + } + } + let mut cx = Context::from_waker(noop_waker_ref()); + while let Poll::Ready(Some(_)) = self.fetcher.poll_next_unpin(&mut cx) {} + } + + /// Runs dispatch and respond rounds until nothing is left to fetch. Returns the number of + /// requests that were sent. + fn run_to_completion( + &mut self, + mut respond: impl FnMut(&[TxHash]) -> PooledTransactions, + ) -> usize { + let mut requests = 0; + loop { + let sent = self.dispatch(); + if sent == 0 { + break + } + requests += sent; + self.respond_all(&mut respond); + } + assert_eq!(self.fetcher.num_hashes(), 0, "all hashes must be fetched or given up on"); + requests + } +} + +fn peer_id(index: usize) -> PeerId { + let mut bytes = [0u8; 64]; + bytes[..8].copy_from_slice(&(index as u64 + 1).to_be_bytes()); + PeerId::new(bytes) +} + +fn pooled_txs(count: usize) -> Vec { + let mut factory = MockTransactionFactory::default(); + (0..count) + .map(|_| { + let recovered: Recovered = + factory.create_eip1559().transaction.into(); + PooledTransactionVariant::try_from(recovered.into_inner()).unwrap() + }) + .collect() +} + +fn announcement_of(txs: &[PooledTransactionVariant]) -> Vec<(TxHash, Eth68TxMetadata)> { + txs.iter().map(|tx| (*tx.tx_hash(), Some((TX_TYPE, TX_SIZE)))).collect() +} + +/// Shared fixtures. +struct Fixtures { + announcement: Vec<(TxHash, Eth68TxMetadata)>, + by_hash: B256Map, +} + +impl Fixtures { + fn new() -> Self { + let txs = pooled_txs(TXS); + let announcement = announcement_of(&txs); + let by_hash = txs.into_iter().map(|tx| (*tx.tx_hash(), tx)).collect(); + Self { announcement, by_hash } + } + + /// Delivers every requested transaction. + fn deliver(&self, hashes: &[TxHash]) -> PooledTransactions { + PooledTransactions(hashes.iter().map(|hash| self.by_hash[hash].clone()).collect()) + } +} + +fn bench_announce(c: &mut Criterion) { + let fixtures = Fixtures::new(); + let mut group = c.benchmark_group("tx_fetcher/announce"); + + group.throughput(Throughput::Elements((PEERS * TXS) as u64)); + group.bench_function("gossip", |b| { + b.iter_batched_ref( + || Rig::new(PEERS), + |rig| rig.announce_gossip(&fixtures.announcement), + BatchSize::PerIteration, + ) + }); + + group.throughput(Throughput::Elements(TXS as u64)); + group.bench_function("disjoint", |b| { + b.iter_batched_ref( + || Rig::new(PEERS), + |rig| rig.announce_disjoint(&fixtures.announcement), + BatchSize::PerIteration, + ) + }); + + group.finish(); +} + +fn bench_dispatch(c: &mut Criterion) { + let fixtures = Fixtures::new(); + let mut group = c.benchmark_group("tx_fetcher/dispatch"); + group.throughput(Throughput::Elements((MAX_COUNT_EAGER_CANDIDATE_PEERS_PER_HASH * 256) as u64)); + + // sanity check the workload: a hash is only queued for the first peers that announced it, so + // one dispatch sends a request of 256 hashes to each of them + let mut rig = Rig::new(PEERS); + rig.announce_gossip(&fixtures.announcement); + assert_eq!(rig.dispatch(), MAX_COUNT_EAGER_CANDIDATE_PEERS_PER_HASH); + + group.bench_function("gossip", |b| { + b.iter_batched_ref( + || { + let mut rig = Rig::new(PEERS); + rig.announce_gossip(&fixtures.announcement); + rig + }, + |rig| rig.dispatch(), + BatchSize::PerIteration, + ) + }); + + group.finish(); +} + +fn bench_fetch(c: &mut Criterion) { + let fixtures = Fixtures::new(); + let mut group = c.benchmark_group("tx_fetcher/fetch"); + group.throughput(Throughput::Elements(TXS as u64)); + + // every request is answered in full: one request per 256 hashes + let mut rig = Rig::new(PEERS); + rig.announce_gossip(&fixtures.announcement); + assert_eq!(rig.run_to_completion(|hashes| fixtures.deliver(hashes)), TXS / 256); + + group.bench_function("gossip_delivered", |b| { + b.iter_batched_ref( + || { + let mut rig = Rig::new(PEERS); + rig.announce_gossip(&fixtures.announcement); + // Bodies and response vectors are fixture work, outside the measured path. + let responses = fixtures + .announcement + .chunks(256) + .map(|chunk| { + let hashes = chunk.iter().map(|(hash, _)| *hash).collect::>(); + (hashes[0], fixtures.deliver(&hashes)) + }) + .collect::>(); + (rig, responses) + }, + |(rig, responses)| { + rig.run_to_completion(|hashes| { + let response = responses.remove(&hashes[0]).expect("prepared response"); + assert_eq!(response.len(), hashes.len()); + response + }) + }, + BatchSize::PerIteration, + ) + }); + + // every peer fails to deliver, so every hash is retried with all 8 candidates before it is + // given up on + let mut rig = Rig::new(8); + rig.announce_gossip(&fixtures.announcement); + assert_eq!(rig.run_to_completion(|_| PooledTransactions::default()), 8 * TXS / 256); + + group.bench_function("gossip_empty_responses", |b| { + b.iter_batched_ref( + || { + let mut rig = Rig::new(8); + rig.announce_gossip(&fixtures.announcement); + rig + }, + |rig| rig.run_to_completion(|_| PooledTransactions::default()), + BatchSize::PerIteration, + ) + }); + + group.finish(); +} + +criterion_group!(benches, bench_announce, bench_dispatch, bench_fetch); +criterion_main!(benches); diff --git a/crates/net/network/src/metrics.rs b/crates/net/network/src/metrics.rs index c94c9ebc544..adba83f365b 100644 --- a/crates/net/network/src/metrics.rs +++ b/crates/net/network/src/metrics.rs @@ -277,18 +277,18 @@ pub struct TransactionFetcherMetrics { /// [`PooledTransactions`](reth_eth_wire::PooledTransactions) responses, that weren't /// requested. pub(crate) unsolicited_transactions: Counter, - /* ================ SEARCH DURATION ================ */ - /// Time spent searching for an idle peer in call to - /// [`TransactionFetcher::find_any_idle_fallback_peer_for_any_pending_hash`](crate::transactions::TransactionFetcher::find_any_idle_fallback_peer_for_any_pending_hash). - /// - /// Duration in seconds. - pub(crate) duration_find_idle_fallback_peer_for_any_pending_hash: Gauge, - - /// Time spent searching for hashes pending fetch, announced by a given peer in - /// [`TransactionFetcher::fill_request_from_hashes_pending_fetch`](crate::transactions::TransactionFetcher::fill_request_from_hashes_pending_fetch). - /// - /// Duration in seconds. - pub(crate) duration_fill_request_from_hashes_pending_fetch: Gauge, + /// Total number of pending hashes that were evicted for newly announced hashes because the + /// fetcher already tracks the maximum number of hashes. + pub(crate) hashes_evicted_at_capacity: Counter, + /// Total number of announced hashes that were dropped because the fetcher already tracks the + /// maximum number of hashes and no pending eviction victim was found within the search budget. + pub(crate) announced_hashes_dropped_at_capacity: Counter, + /// Total number of announced hashes that were dropped because the announcing peer is already + /// a candidate for the maximum number of hashes. + pub(crate) announced_hashes_dropped_peer_limit: Counter, + /// Total number of tracked hashes dropped because no candidate remains or the bounded + /// fetch-attempt budget was exhausted. + pub(crate) hashes_dropped_no_candidate_peers: Counter, } /// Measures the duration of executing the given code block. The duration is added to the given diff --git a/crates/net/network/src/test_utils/transactions.rs b/crates/net/network/src/test_utils/transactions.rs index bb51cd3c56a..804d2993fbb 100644 --- a/crates/net/network/src/test_utils/transactions.rs +++ b/crates/net/network/src/test_utils/transactions.rs @@ -3,14 +3,9 @@ #![allow(dead_code)] use crate::{ - cache::LruCache, transactions::{ - constants::{ - tx_fetcher::DEFAULT_MAX_COUNT_FALLBACK_PEERS, - tx_manager::DEFAULT_MAX_COUNT_TRANSACTIONS_SEEN_BY_PEER, - }, - fetcher::{TransactionFetcher, TxFetchMetadata}, - PeerMetadata, TransactionsManager, + constants::tx_manager::DEFAULT_MAX_COUNT_TRANSACTIONS_SEEN_BY_PEER, + fetcher::TransactionFetcher, PeerMetadata, TransactionsManager, TransactionsManagerConfig, }, NetworkConfigBuilder, NetworkManager, }; @@ -25,10 +20,16 @@ use reth_transaction_pool::test_utils::{testing_pool, TestPool}; use secp256k1::SecretKey; use std::sync::Arc; use tokio::sync::mpsc; -use tracing::trace; /// A new tx manager for testing. pub async fn new_tx_manager( +) -> (TransactionsManager, NetworkManager) { + new_tx_manager_with_config(TransactionsManagerConfig::default()).await +} + +/// A new tx manager for testing with the given config. +pub async fn new_tx_manager_with_config( + transactions_manager_config: TransactionsManagerConfig, ) -> (TransactionsManager, NetworkManager) { let secret_key = SecretKey::new(&mut rand_08::thread_rng()); let client = NoopProvider::default(); @@ -41,7 +42,6 @@ pub async fn new_tx_manager( let pool = testing_pool(); - let transactions_manager_config = config.transactions_manager_config.clone(); let (_network_handle, network, transactions, _) = NetworkManager::new(config) .await .unwrap() @@ -52,34 +52,14 @@ pub async fn new_tx_manager( (transactions, network) } -/// Directly buffer hash into tx fetcher for testing. +/// Announces a hash to the tx fetcher on behalf of the given peer, without sending a request. pub fn buffer_hash_to_tx_fetcher( tx_fetcher: &mut TransactionFetcher, hash: TxHash, peer_id: PeerId, - retries: u8, tx_encoded_length: Option, ) { - match tx_fetcher.hashes_fetch_inflight_and_pending_fetch.get_or_insert(hash, || { - TxFetchMetadata::new( - retries, - LruCache::with_hasher(DEFAULT_MAX_COUNT_FALLBACK_PEERS as u32, Default::default()), - tx_encoded_length, - ) - }) { - Some(metadata) => { - metadata.fallback_peers_mut().insert(peer_id); - } - None => { - trace!(target: "net::tx", - peer_id=format!("{peer_id:#}"), - %hash, - "failed to insert hash from peer in schnellru::LruMap, dropping hash" - ) - } - } - - tx_fetcher.hashes_pending_fetch.insert(hash); + tx_fetcher.on_announcement(peer_id, [(hash, tx_encoded_length.map(|size| (0u8, size)))]); } /// Mock a new session, returns (peer, channel-to-send-get-pooled-tx-response-on). @@ -87,7 +67,17 @@ pub fn new_mock_session( peer_id: PeerId, version: EthVersion, ) -> (PeerMetadata, mpsc::Receiver) { - let (to_mock_session_tx, to_mock_session_rx) = mpsc::channel(1); + new_mock_session_with_capacity(peer_id, version, 1) +} + +/// Mock a new session whose request channel buffers up to `capacity` requests, returns (peer, +/// channel-to-send-get-pooled-tx-response-on). +pub fn new_mock_session_with_capacity( + peer_id: PeerId, + version: EthVersion, + capacity: usize, +) -> (PeerMetadata, mpsc::Receiver) { + let (to_mock_session_tx, to_mock_session_rx) = mpsc::channel(capacity); ( PeerMetadata::new( diff --git a/crates/net/network/src/transactions/config.rs b/crates/net/network/src/transactions/config.rs index fb2c0cbbddf..c8ec184830f 100644 --- a/crates/net/network/src/transactions/config.rs +++ b/crates/net/network/src/transactions/config.rs @@ -8,8 +8,8 @@ use super::{ }; use crate::transactions::constants::{ tx_fetcher::{ - DEFAULT_MAX_CAPACITY_CACHE_PENDING_FETCH, DEFAULT_MAX_COUNT_CONCURRENT_REQUESTS, - DEFAULT_MAX_COUNT_CONCURRENT_REQUESTS_PER_PEER, + DEFAULT_MAX_CAPACITY_CACHE_PENDING_FETCH, DEFAULT_MAX_COUNT_ANNOUNCED_HASHES_PER_PEER, + DEFAULT_MAX_COUNT_CONCURRENT_REQUESTS, DEFAULT_MAX_COUNT_CONCURRENT_REQUESTS_PER_PEER, }, tx_manager::{ DEFAULT_MAX_COUNT_PENDING_POOL_IMPORTS, DEFAULT_TX_MANAGER_CHANNEL_MEMORY_LIMIT_BYTES, @@ -17,7 +17,7 @@ use crate::transactions::constants::{ }; use alloy_eips::eip2718::IsTyped2718; use alloy_primitives::B256; -use derive_more::{Constructor, Display}; +use derive_more::Display; use reth_eth_wire::NetworkPrimitives; use reth_network_types::peers::kind::PeerKind; @@ -29,7 +29,8 @@ pub struct TransactionsManagerConfig { pub transaction_fetcher_config: TransactionFetcherConfig, /// Max number of seen transactions to store for each peer. pub max_transactions_seen_by_peer_history: u32, - /// Max number of transactions allowed to be imported concurrently. + /// Soft limit on concurrent transaction imports. A fetched response admitted while capacity + /// remains may exceed this limit by at most 255 transactions; broadcasts are truncated to fit. #[cfg_attr(feature = "serde", serde(default = "default_max_pending_pool_imports"))] pub max_pending_pool_imports: usize, /// How new pending transactions are propagated. @@ -124,7 +125,7 @@ impl FromStr for TransactionPropagationMode { } /// Configuration for fetching transactions. -#[derive(Debug, Constructor, Clone)] +#[derive(Debug, Clone)] #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] pub struct TransactionFetcherConfig { /// Max inflight [`GetPooledTransactions`](reth_eth_wire::GetPooledTransactions) requests. @@ -141,27 +142,55 @@ pub struct TransactionFetcherConfig { /// [`PooledTransactions`](reth_eth_wire::PooledTransactions) response on packing a /// [`GetPooledTransactions`](reth_eth_wire::GetPooledTransactions) request with hashes. pub soft_limit_byte_size_pooled_transactions_response_on_pack_request: usize, - /// Max capacity of the cache of transaction hashes, for transactions that weren't yet fetched. - /// A transaction is pending fetch if its hash didn't fit into a - /// [`GetPooledTransactions`](reth_eth_wire::GetPooledTransactions) yet, or it wasn't returned - /// upon request to peers. + /// Max number of announced transaction hashes to keep track of, i.e. hashes that were + /// announced but not fetched yet, both pending and inflight. Once reached, the peer tracking + /// the most hashes gives up its oldest unshared pending hash for a newly announced one. + /// If none is found, the oldest pending hash (including shared hashes) is evicted instead. + /// Inflight hashes are preserved; if no victim is found, the announcement is dropped. pub max_capacity_cache_txns_pending_fetch: u32, + /// Max number of tracked hashes a single peer can be a candidate for. Announcements from a + /// peer that exceed this limit are dropped. + #[cfg_attr(feature = "serde", serde(default = "default_max_announced_hashes_per_peer"))] + pub max_announced_hashes_per_peer: u32, } -impl Default for TransactionFetcherConfig { - fn default() -> Self { +#[cfg(feature = "serde")] +const fn default_max_announced_hashes_per_peer() -> u32 { + DEFAULT_MAX_COUNT_ANNOUNCED_HASHES_PER_PEER +} + +impl TransactionFetcherConfig { + /// Creates a new config with the default limit on announced hashes per peer. + pub const fn new( + max_inflight_requests: u32, + max_inflight_requests_per_peer: u8, + soft_limit_byte_size_pooled_transactions_response: usize, + soft_limit_byte_size_pooled_transactions_response_on_pack_request: usize, + max_capacity_cache_txns_pending_fetch: u32, + ) -> Self { Self { - max_inflight_requests: DEFAULT_MAX_COUNT_CONCURRENT_REQUESTS, - max_inflight_requests_per_peer: DEFAULT_MAX_COUNT_CONCURRENT_REQUESTS_PER_PEER, - soft_limit_byte_size_pooled_transactions_response: - SOFT_LIMIT_BYTE_SIZE_POOLED_TRANSACTIONS_RESPONSE, - soft_limit_byte_size_pooled_transactions_response_on_pack_request: - DEFAULT_SOFT_LIMIT_BYTE_SIZE_POOLED_TRANSACTIONS_RESP_ON_PACK_GET_POOLED_TRANSACTIONS_REQ, - max_capacity_cache_txns_pending_fetch: DEFAULT_MAX_CAPACITY_CACHE_PENDING_FETCH, + max_inflight_requests, + max_inflight_requests_per_peer, + soft_limit_byte_size_pooled_transactions_response, + soft_limit_byte_size_pooled_transactions_response_on_pack_request, + max_capacity_cache_txns_pending_fetch, + max_announced_hashes_per_peer: DEFAULT_MAX_COUNT_ANNOUNCED_HASHES_PER_PEER, } } } +impl Default for TransactionFetcherConfig { + fn default() -> Self { + Self::new( + DEFAULT_MAX_COUNT_CONCURRENT_REQUESTS, + DEFAULT_MAX_COUNT_CONCURRENT_REQUESTS_PER_PEER, + SOFT_LIMIT_BYTE_SIZE_POOLED_TRANSACTIONS_RESPONSE, + DEFAULT_SOFT_LIMIT_BYTE_SIZE_POOLED_TRANSACTIONS_RESP_ON_PACK_GET_POOLED_TRANSACTIONS_REQ, + DEFAULT_MAX_CAPACITY_CACHE_PENDING_FETCH, + ) + } +} + /// A policy defining which peers pending transactions are gossiped to. pub trait TransactionPropagationPolicy: Send + Sync + Unpin + fmt::Debug + 'static diff --git a/crates/net/network/src/transactions/constants.rs b/crates/net/network/src/transactions/constants.rs index 99c18c47402..66fe973b80f 100644 --- a/crates/net/network/src/transactions/constants.rs +++ b/crates/net/network/src/transactions/constants.rs @@ -60,204 +60,82 @@ pub mod tx_manager { /// Constants used by [`TransactionFetcher`](super::TransactionFetcher). pub mod tx_fetcher { - use reth_network_types::peers::config::{ - DEFAULT_MAX_COUNT_PEERS_INBOUND, DEFAULT_MAX_COUNT_PEERS_OUTBOUND, - }; - use super::{ SOFT_LIMIT_BYTE_SIZE_POOLED_TRANSACTIONS_RESPONSE, SOFT_LIMIT_COUNT_HASHES_IN_GET_POOLED_TRANSACTIONS_REQUEST, SOFT_LIMIT_COUNT_HASHES_IN_NEW_POOLED_TRANSACTIONS_BROADCAST_MESSAGE, }; + use reth_network_types::peers::config::{ + DEFAULT_MAX_COUNT_PEERS_INBOUND, DEFAULT_MAX_COUNT_PEERS_OUTBOUND, + }; - /* ============== SCALARS OF MESSAGES ============== */ - - /// Default soft limit for the byte size of a - /// [`PooledTransactions`](reth_eth_wire::PooledTransactions) response on assembling a - /// [`GetPooledTransactions`](reth_eth_wire::PooledTransactions) request. This defaults to less - /// than the [`SOFT_LIMIT_BYTE_SIZE_POOLED_TRANSACTIONS_RESPONSE`], at 2 MiB, used when - /// assembling a [`PooledTransactions`](reth_eth_wire::PooledTransactions) response. + /// Default soft limit for the byte size of the expected + /// [`PooledTransactions`](reth_eth_wire::PooledTransactions) response when packing a + /// [`GetPooledTransactions`](reth_eth_wire::GetPooledTransactions) request. This is much + /// smaller than the 2 MiB [`SOFT_LIMIT_BYTE_SIZE_POOLED_TRANSACTIONS_RESPONSE`] applied when + /// assembling a response, so that a single request fetches at most about one blob transaction + /// and doesn't hog the connection to a peer. /// /// Default is 128 KiB. pub const DEFAULT_SOFT_LIMIT_BYTE_SIZE_POOLED_TRANSACTIONS_RESP_ON_PACK_GET_POOLED_TRANSACTIONS_REQ: usize = 128 * 1024; - /* ==================== RETRIES ==================== */ - - /// Default maximum request retires per [`TxHash`](alloy_primitives::TxHash). Note, this is - /// reset should the [`TxHash`](alloy_primitives::TxHash) re-appear in an announcement after it - /// has been evicted from the hashes pending fetch cache, i.e. the counter is restarted. If - /// this happens, it is likely a very popular transaction, that should and can indeed be - /// fetched hence this behaviour is favourable. - /// - /// Default is 2 retries. - pub const DEFAULT_MAX_RETRIES: u8 = 2; - - /// Default number of alternative peers to keep track of for each transaction pending fetch. At - /// most [`DEFAULT_MAX_RETRIES`], which defaults to 2 peers, can ever be needed per peer. - /// - /// Default is the sum of [`DEFAULT_MAX_RETRIES`] an - /// [`DEFAULT_MARGINAL_COUNT_FALLBACK_PEERS`], which defaults to 1 peer, so 3 peers. - pub const DEFAULT_MAX_COUNT_FALLBACK_PEERS: u8 = - DEFAULT_MAX_RETRIES + DEFAULT_MARGINAL_COUNT_FALLBACK_PEERS; - - /// Default marginal on fallback peers. This is the case, since a transaction is only requested - /// once from each individual peer. - /// - /// Default is 1 peer. - pub const DEFAULT_MARGINAL_COUNT_FALLBACK_PEERS: u8 = 1; - - /* ==================== CONCURRENCY ==================== */ - - /// Default maximum concurrent [`GetPooledTransactions`](reth_eth_wire::GetPooledTransactions) - /// requests. + /// Default maximum number of concurrent + /// [`GetPooledTransactions`](reth_eth_wire::GetPooledTransactions) requests. /// - /// Default is the product of [`DEFAULT_MAX_COUNT_CONCURRENT_REQUESTS_PER_PEER`], which - /// defaults to 1 request, and the sum of [`DEFAULT_MAX_COUNT_PEERS_INBOUND`] and + /// Default is the sum of [`DEFAULT_MAX_COUNT_PEERS_INBOUND`] and /// [`DEFAULT_MAX_COUNT_PEERS_OUTBOUND`], which default to 30 and 100 peers respectively, so /// 130 requests. pub const DEFAULT_MAX_COUNT_CONCURRENT_REQUESTS: u32 = DEFAULT_MAX_COUNT_PEERS_INBOUND + DEFAULT_MAX_COUNT_PEERS_OUTBOUND; /// Default maximum number of concurrent - /// [`GetPooledTransactions`](reth_eth_wire::GetPooledTransactions)s to allow per peer. This - /// number reflects concurrent requests for different hashes. + /// [`GetPooledTransactions`](reth_eth_wire::GetPooledTransactions) requests per peer. /// - /// Default is 1 request. + /// Default is 1 request. With a single request at a time, the hashes a peer failed to deliver + /// are known as soon as its response arrives and can be rescheduled immediately. pub const DEFAULT_MAX_COUNT_CONCURRENT_REQUESTS_PER_PEER: u8 = 1; - /* =============== HASHES PENDING FETCH ================ */ - - /// Default limit for number of transactions waiting for an idle peer to be fetched from. + /// Default maximum number of announced transaction hashes to keep track of, pending and + /// inflight. Once reached, the peer tracking the most hashes gives up its oldest pending + /// hash for a newly announced one. /// /// Default is 100 times the [`SOFT_LIMIT_COUNT_HASHES_IN_GET_POOLED_TRANSACTIONS_REQUEST`], - /// which defaults to 256 hashes, so 25 600 hashes. + /// which is 256 hashes, so 25 600 hashes. pub const DEFAULT_MAX_CAPACITY_CACHE_PENDING_FETCH: u32 = 100 * SOFT_LIMIT_COUNT_HASHES_IN_GET_POOLED_TRANSACTIONS_REQUEST as u32; - /// Default max size for cache of inflight and pending transactions fetch. - /// - /// Default is [`DEFAULT_MAX_CAPACITY_CACHE_PENDING_FETCH`] + - /// [`DEFAULT_MAX_COUNT_INFLIGHT_REQUESTS_ON_FETCH_PENDING_HASHES`], which is 25600 hashes and - /// 65 requests, so it is 25665 hashes. - pub const DEFAULT_MAX_CAPACITY_CACHE_INFLIGHT_AND_PENDING_FETCH: u32 = - DEFAULT_MAX_CAPACITY_CACHE_PENDING_FETCH + - DEFAULT_MAX_COUNT_INFLIGHT_REQUESTS_ON_FETCH_PENDING_HASHES as u32; - - /// Default maximum number of hashes pending fetch to tolerate at any time. + /// Default maximum number of tracked hashes a single peer can be a candidate for. Bounds the + /// memory a single peer can occupy in the fetcher with its announcements. /// - /// Default is half of [`DEFAULT_MAX_CAPACITY_CACHE_PENDING_FETCH`], which defaults to 25 600 - /// hashes, so 12 800 hashes. - pub const DEFAULT_MAX_COUNT_PENDING_FETCH: usize = - DEFAULT_MAX_CAPACITY_CACHE_PENDING_FETCH as usize / 2; + /// Default is one full announcement, + /// [`SOFT_LIMIT_COUNT_HASHES_IN_NEW_POOLED_TRANSACTIONS_BROADCAST_MESSAGE`], so 4096 hashes. + pub const DEFAULT_MAX_COUNT_ANNOUNCED_HASHES_PER_PEER: u32 = + SOFT_LIMIT_COUNT_HASHES_IN_NEW_POOLED_TRANSACTIONS_BROADCAST_MESSAGE as u32; - /* ====== LIMITED CAPACITY ON FETCH PENDING HASHES ====== */ - - /// Default budget for finding an idle fallback peer for any hash pending fetch, when said - /// search is budget constrained. - /// - /// Default is a sixth of [`DEFAULT_MAX_COUNT_PENDING_FETCH`], which defaults to 12 800 hashes - /// (the ideal max number of hashes pending fetch), divided by - /// [`DEFAULT_MAX_COUNT_FALLBACK_PEERS`], which defaults to 3 peers (the depth of the search), - /// so a search breadth of 711 lru hashes in the pending hashes cache. - pub const DEFAULT_BUDGET_FIND_IDLE_FALLBACK_PEER: usize = - DEFAULT_MAX_COUNT_PENDING_FETCH / 6 / DEFAULT_MAX_COUNT_FALLBACK_PEERS as usize; - - /// Default budget for finding hashes in the intersection of transactions announced by a peer - /// and in the cache of hashes pending fetch, when said search is budget constrained. - /// - /// Default is an eight of [`DEFAULT_MAX_COUNT_PENDING_FETCH`], which defaults to 12 800 hashes - /// (the ideal max number of hashes pending fetch), so a search breadth of 1 600 lru hashes in - /// the pending hashes cache. - pub const DEFAULT_BUDGET_FIND_INTERSECTION_ANNOUNCED_BY_PEER_AND_PENDING_FETCH: usize = - DEFAULT_MAX_COUNT_PENDING_FETCH / 8; - - /* ====== SCALARS FOR USE ON FETCH PENDING HASHES ====== */ - - /// Default soft limit for the number of hashes in a - /// [`GetPooledTransactions`](reth_eth_wire::GetPooledTransactions) request, when it is filled - /// from hashes pending fetch. + /// Maximum number of peers remembered as candidates for a single hash. The first eager + /// sources are retained while later announcements replace the oldest fallback sources. /// - /// Default is half of the [`SOFT_LIMIT_COUNT_HASHES_IN_GET_POOLED_TRANSACTIONS_REQUEST`] - /// which by spec is 256 hashes, so 128 hashes. - pub const DEFAULT_SOFT_LIMIT_COUNT_HASHES_IN_GET_POOLED_TRANSACTIONS_REQUEST_ON_FETCH_PENDING_HASHES: - usize = SOFT_LIMIT_COUNT_HASHES_IN_GET_POOLED_TRANSACTIONS_REQUEST / 2; + /// Default is 8 peers. + pub const MAX_COUNT_CANDIDATE_PEERS_PER_HASH: usize = 8; - /// Default soft limit for a [`PooledTransactions`](reth_eth_wire::PooledTransactions) response - /// when it's used as expected response in calibrating the filling of a - /// [`GetPooledTransactions`](reth_eth_wire::GetPooledTransactions) request, when the request - /// is filled from hashes pending fetch. - /// - /// Default is half of - /// [`DEFAULT_SOFT_LIMIT_BYTE_SIZE_POOLED_TRANSACTIONS_RESP_ON_PACK_GET_POOLED_TRANSACTIONS_REQ`], - /// which defaults to 128 KiB, so 64 KiB. - pub const DEFAULT_SOFT_LIMIT_BYTE_SIZE_POOLED_TRANSACTIONS_RESPONSE_ON_FETCH_PENDING_HASHES: - usize = - DEFAULT_SOFT_LIMIT_BYTE_SIZE_POOLED_TRANSACTIONS_RESP_ON_PACK_GET_POOLED_TRANSACTIONS_REQ / - 2; - - /// Default max inflight request when fetching pending hashes. - /// - /// Default is half of [`DEFAULT_MAX_COUNT_CONCURRENT_REQUESTS`], which defaults to 130 - /// requests, so 65 requests. - pub const DEFAULT_MAX_COUNT_INFLIGHT_REQUESTS_ON_FETCH_PENDING_HASHES: usize = - DEFAULT_MAX_COUNT_CONCURRENT_REQUESTS as usize / 2; - - /// Default divisor of the max inflight request when calculating search breadth of the search - /// for any idle peer to which to send a request filled with hashes pending fetch. The max - /// inflight requests is configured in - /// [`TransactionFetcherInfo`](crate::transactions::fetcher::TransactionFetcherInfo). - /// - /// Default is 3 requests. - pub const DEFAULT_DIVISOR_MAX_COUNT_INFLIGHT_REQUESTS_ON_FIND_IDLE_PEER: usize = 3; + /// Maximum fetch attempts for a single hash, independent of candidate storage capacity. + pub const MAX_FETCH_ATTEMPTS_PER_HASH: usize = 16; - /// Default divisor of the max inflight request when calculating search breadth of the search - /// for the intersection of hashes announced by a peer and hashes pending fetch. The max - /// inflight requests is configured in - /// [`TransactionFetcherInfo`](crate::transactions::fetcher::TransactionFetcherInfo). + /// Number of candidates a hash is queued for right away. The remaining candidates only get + /// the hash queued once one of these failed to deliver it, which keeps late announcements + /// cheap while a hash is not given up on when its first announcers fail. /// - /// Default is 3 requests. - pub const DEFAULT_DIVISOR_MAX_COUNT_INFLIGHT_REQUESTS_ON_FIND_INTERSECTION: usize = 3; + /// Default is half of [`MAX_COUNT_CANDIDATE_PEERS_PER_HASH`], so 4 peers. + pub const MAX_COUNT_EAGER_CANDIDATE_PEERS_PER_HASH: usize = + MAX_COUNT_CANDIDATE_PEERS_PER_HASH / 2; - // Default divisor to the max pending pool imports when calculating search breadth of the - /// search for any idle peer to which to send a request filled with hashes pending fetch. - /// The max pending pool imports is configured in - /// [`PendingPoolImportsInfo`](crate::transactions::PendingPoolImportsInfo). - /// - /// Default is 4 requests. - pub const DEFAULT_DIVISOR_MAX_COUNT_PENDING_POOL_IMPORTS_ON_FIND_IDLE_PEER: usize = 4; - - /// Default divisor to the max pending pool imports when calculating search breadth of the - /// search for any idle peer to which to send a request filled with hashes pending fetch. - /// The max pending pool imports is configured in - /// [`PendingPoolImportsInfo`](crate::transactions::PendingPoolImportsInfo). - /// - /// Default is 4 requests. - pub const DEFAULT_DIVISOR_MAX_COUNT_PENDING_POOL_IMPORTS_ON_FIND_INTERSECTION: usize = 4; - - /* ================== ROUGH MEASURES ================== */ - - /// Average byte size of an encoded transaction. + /// Assumed byte size of a transaction whose size wasn't announced, used when packing + /// requests. /// /// Default is [`SOFT_LIMIT_BYTE_SIZE_POOLED_TRANSACTIONS_RESPONSE`], which defaults to 2 MiB, /// divided by [`SOFT_LIMIT_COUNT_HASHES_IN_NEW_POOLED_TRANSACTIONS_BROADCAST_MESSAGE`], which - /// is spec'd at 4096 hashes, so 521 bytes. + /// is spec'd at 4096 hashes, so 512 bytes. pub const AVERAGE_BYTE_SIZE_TX_ENCODED: usize = SOFT_LIMIT_BYTE_SIZE_POOLED_TRANSACTIONS_RESPONSE / SOFT_LIMIT_COUNT_HASHES_IN_NEW_POOLED_TRANSACTIONS_BROADCAST_MESSAGE; - - /// Median observed size in bytes of a small encoded legacy transaction. - /// - /// Default is 120 bytes. - pub const MEDIAN_BYTE_SIZE_SMALL_LEGACY_TX_ENCODED: usize = 120; - - /// Marginal on the number of hashes to preallocate memory for in a - /// [`GetPooledTransactions`](reth_eth_wire::GetPooledTransactions) request, when packed - /// according to the [`Eth68`](reth_eth_wire::EthVersion::Eth68) protocol version. To make - /// sure enough memory is preallocated in most cases, it's sensible to use a margin. This, - /// since the capacity is calculated based on median value - /// [`MEDIAN_BYTE_SIZE_SMALL_LEGACY_TX_ENCODED`]. There may otherwise be a noteworthy number of - /// cases where just 1 or 2 bytes too little memory is preallocated. - /// - /// Default is 8 hashes. - pub const DEFAULT_MARGINAL_COUNT_HASHES_GET_POOLED_TRANSACTIONS_REQUEST: usize = 8; } diff --git a/crates/net/network/src/transactions/fetcher.rs b/crates/net/network/src/transactions/fetcher.rs index fee72ce45d7..7771c72170c 100644 --- a/crates/net/network/src/transactions/fetcher.rs +++ b/crates/net/network/src/transactions/fetcher.rs @@ -1,1051 +1,1147 @@ -//! `TransactionFetcher` is responsible for rate limiting and retry logic for fetching -//! transactions. Upon receiving an announcement, functionality of the `TransactionFetcher` is -//! used for filtering out hashes 1) for which the tx is already known and 2) unknown but the hash -//! is already seen in a previous announcement. The hashes that remain from an announcement are -//! then packed into a request with respect to the [`EthVersion`] of the announcement. Any hashes -//! that don't fit into the request, are buffered in the `TransactionFetcher`. If on the other -//! hand, space remains, hashes that the peer has previously announced are taken out of buffered -//! hashes to fill the request up. The [`GetPooledTransactions`] request is then sent to the -//! peer's session, this marks the peer as active with respect to -//! `MAX_CONCURRENT_TX_REQUESTS_PER_PEER`. +//! Fetches transactions that peers announced with `NewPooledTransactionHashes`. //! -//! When a peer buffers hashes in the `TransactionsManager::on_new_pooled_transaction_hashes` -//! pipeline, it is stored as fallback peer for those hashes. When [`TransactionsManager`] is -//! polled, it checks if any of fallback peer is idle. If so, it packs a request for that peer, -//! filling it from the buffered hashes. It does so until there are no more idle peers or until -//! the hashes buffer is empty. +//! The [`TransactionFetcher`] tracks every announced hash that is not known yet, together with the +//! peers that announced it, and turns those announcements into `GetPooledTransactions` requests. //! -//! If a [`GetPooledTransactions`] request resolves with an error, the hashes in the request are -//! buffered with respect to `MAX_REQUEST_RETRIES_PER_TX_HASH`. So is the case if the request -//! resolves with partial success, that is some of the requested hashes are not in the response, -//! these are then buffered. +//! # Model //! -//! Most healthy peers will send the same hashes in their announcements, as RLPx is a gossip -//! protocol. This means it's unlikely, that a valid hash, will be buffered for very long -//! before it's re-tried. Nonetheless, the capacity of the buffered hashes cache must be large -//! enough to buffer many hashes during network failure, to allow for recovery. +//! Every tracked hash is in exactly one of two states: +//! +//! - _pending_: waiting for one of the peers that announced it, its _candidates_, to become idle +//! - _fetching_: part of exactly one inflight request +//! +//! A hash remembers the peers that announced it as candidates. Each peer keeps a FIFO queue of +//! hashes to request, and the first peers to announce a hash get it queued right away. Later +//! announcers are only remembered, so that a hash is not given up on when the first announcers +//! fail to deliver it, without costing a queue entry per announcement. A request for an idle peer +//! is built by draining its queue in the order the announcements were processed, skipping hashes +//! that are being fetched from another peer or are not tracked anymore, until the request is +//! full. Packing a request is therefore proportional to the request size and independent of the +//! total number of pending hashes. FIFO refers to the fetcher's input order; callers may +//! reorder or deduplicate hashes within a wire announcement. +//! +//! When a request resolves, delivered hashes are dropped from tracking. Undelivered hashes go back +//! to pending and are queued for their remaining candidates, most recent announcers first, or are +//! dropped if none remain. The responding peer is dropped as a candidate for every undelivered +//! hash that precedes the last delivered hash of the request, treating it as skipped, and for +//! all requested hashes if the response was empty or the request failed. +//! Undelivered hashes after the last delivered hash keep the peer as a candidate only if it +//! delivered at least half of the request, treating that tail as potentially truncated. This is +//! a retry heuristic: transaction counts do not establish whether the response reached a byte +//! limit. Requiring half the request limits repeated low-progress retries. +//! +//! Request timeouts are enforced by the peer's session, which resolves the request with +//! [`RequestError::Timeout`], so the fetcher does not run any timers. A first timeout is retried +//! once when the responding peer is the only remaining source; otherwise another source is tried. +//! +//! # Bounds +//! +//! - a configurable number of inflight requests per peer, one by default, and a global inflight +//! request limit +//! - a per peer limit on the number of tracked hashes it is a candidate for, so a single peer +//! cannot flood the fetcher with announcements +//! - a global limit on the number of tracked hashes, at which the peer tracking the most hashes +//! gives up its oldest pending hash, so a group of peers flooding the fetcher evicts its own +//! hashes rather than everyone else's +//! - a fixed number of candidates per hash and a separate fetch-attempt limit, so remembering +//! recent fallback sources does not allow unlimited retries use super::{ config::TransactionFetcherConfig, - constants::{tx_fetcher::*, SOFT_LIMIT_COUNT_HASHES_IN_GET_POOLED_TRANSACTIONS_REQUEST}, - PeerMetadata, PooledTransactions, SOFT_LIMIT_BYTE_SIZE_POOLED_TRANSACTIONS_RESPONSE, -}; -use crate::{ - cache::{LruCache, LruMap}, - duration_metered_exec, - metrics::TransactionFetcherMetrics, + constants::{ + tx_fetcher::{ + AVERAGE_BYTE_SIZE_TX_ENCODED, MAX_COUNT_CANDIDATE_PEERS_PER_HASH, + MAX_COUNT_EAGER_CANDIDATE_PEERS_PER_HASH, MAX_FETCH_ATTEMPTS_PER_HASH, + }, + SOFT_LIMIT_BYTE_SIZE_POOLED_TRANSACTIONS_RESPONSE, + SOFT_LIMIT_COUNT_HASHES_IN_GET_POOLED_TRANSACTIONS_REQUEST, + }, + PeerMetadata, }; +use crate::metrics::TransactionFetcherMetrics; use alloy_consensus::transaction::PooledTransaction; use alloy_primitives::{ - map::{FbBuildHasher, HashMap}, + map::{B256Map, B256Set, Entry, FbBuildHasher, HashMap}, TxHash, }; -use derive_more::{Constructor, Deref}; use futures::{stream::FuturesUnordered, Future, FutureExt, Stream, StreamExt}; -use pin_project::pin_project; -use reth_eth_wire::{ - DedupPayload, GetPooledTransactions, HandleMempoolData, HandleVersionedMempoolData, - PartiallyValidData, RequestTxHashes, ValidAnnouncementData, -}; +use reth_eth_wire::{Eth68TxMetadata, EthVersion, GetPooledTransactions, PooledTransactions}; use reth_eth_wire_types::{EthNetworkPrimitives, NetworkPrimitives}; use reth_network_api::PeerRequest; use reth_network_p2p::error::{RequestError, RequestResult}; use reth_network_peers::PeerId; use reth_primitives_traits::SignedTransaction; -use schnellru::ByLength; +use smallvec::SmallVec; use std::{ + collections::{BinaryHeap, VecDeque}, pin::Pin, + sync::Arc, task::{ready, Context, Poll}, - time::Duration, }; -use tokio::sync::{mpsc::error::TrySendError, oneshot, oneshot::error::RecvError}; +use tokio::sync::{mpsc::error::TrySendError, oneshot}; use tracing::trace; -/// The type responsible for fetching missing transactions from peers. +/// Maximum live entries inspected in a global fallback eviction search. +const MAX_EVICTION_ATTEMPTS: usize = 8; + +/// Fetches transactions that peers announced but that are not in the pool yet. /// -/// This will keep track of unique transaction hashes that are currently being fetched and submits -/// new requests on announced hashes. +/// Announcements are recorded with [`Self::on_announcement`], requests are sent with +/// [`Self::dispatch`] and resolved requests are yielded as [`FetchEvent`]s by the [`Stream`] +/// implementation. See the [module docs](self) for how requests are scheduled. #[derive(Debug)] -#[pin_project] pub struct TransactionFetcher { - /// All peers with to which a [`GetPooledTransactions`] request is inflight. - pub active_peers: LruMap>, - /// All currently active [`GetPooledTransactions`] requests. - /// - /// The set of hashes encompassed by these requests are a subset of all hashes in the fetcher. - /// It's disjoint from the set of hashes which are awaiting an idle fallback peer in order to - /// be fetched. - #[pin] - pub inflight_requests: FuturesUnordered>, - /// Hashes that are awaiting an idle fallback peer so they can be fetched. - /// - /// This is a subset of all hashes in the fetcher, and is disjoint from the set of hashes for - /// which a [`GetPooledTransactions`] request is inflight. - pub hashes_pending_fetch: LruCache>, - /// Tracks all hashes in the transaction fetcher. - pub hashes_fetch_inflight_and_pending_fetch: - LruMap>, - /// Info on capacity of the transaction fetcher. - pub info: TransactionFetcherInfo, - #[doc(hidden)] + /// All tracked hashes with their candidate peers and fetch state. + hashes: B256Map, + /// Tracked hashes in the order they were added, used to evict the oldest pending hash once + /// the fetcher is at capacity and the peer tracking the most hashes has none pending. Entries + /// are removed lazily, so it may contain hashes that are not tracked anymore. + order: VecDeque<(TxHash, u64)>, + /// Fetch state of all peers that announced tracked hashes. + peers: HashMap, + /// Maps peer ids to their compact key. + peer_keys: HashMap>, + /// Key assigned to the next new peer. Keys are never reused, so the candidates of a hash can + /// safely refer to peers that disconnected in the meantime. + next_peer_key: u32, + /// Distinguishes overlapping requests, including repeat requests for the same hash. + next_request_id: u64, + /// Distinguishes a re-announcement from stale queue entries for an earlier tracked hash. + next_generation: u64, + /// Idle peers with queued hashes, in the order they became ready. + ready: VecDeque, + /// All inflight `GetPooledTransactions` requests. + inflight: FuturesUnordered>, + /// Number of tracked hashes that are part of an inflight request. + num_fetching: usize, + /// Reused when verifying responses, so no sets are allocated per response. + scratch_requested: B256Set, + /// Reused when verifying responses, so no sets are allocated per response. + scratch_delivered: B256Set, + /// Reused when processing announcements, so no vector is allocated per announcement. + scratch_queue: Vec<(TxHash, u64)>, + /// Configured limits. + config: TransactionFetcherConfig, metrics: TransactionFetcherMetrics, } impl TransactionFetcher { - /// Removes the peer from the active set. - pub(crate) fn remove_peer(&mut self, peer_id: &PeerId) { - self.active_peers.remove(peer_id); - } - - /// Updates metrics. - #[inline] - pub fn update_metrics(&self) { - let metrics = &self.metrics; - - metrics.inflight_transaction_requests.set(self.inflight_requests.len() as f64); - - let hashes_pending_fetch = self.hashes_pending_fetch.len() as f64; - let total_hashes = self.hashes_fetch_inflight_and_pending_fetch.len() as f64; - - metrics.hashes_pending_fetch.set(hashes_pending_fetch); - metrics.hashes_inflight_transaction_requests.set(total_hashes - hashes_pending_fetch); - } - - #[inline] - fn update_pending_fetch_cache_search_metrics(&self, durations: TxFetcherSearchDurations) { - let metrics = &self.metrics; - - let TxFetcherSearchDurations { find_idle_peer, fill_request } = durations; - metrics - .duration_find_idle_fallback_peer_for_any_pending_hash - .set(find_idle_peer.as_secs_f64()); - metrics.duration_fill_request_from_hashes_pending_fetch.set(fill_request.as_secs_f64()); - } - - /// Sets up transaction fetcher with config - pub fn with_transaction_fetcher_config(config: &TransactionFetcherConfig) -> Self { - let TransactionFetcherConfig { - max_inflight_requests, - max_capacity_cache_txns_pending_fetch, - .. - } = *config; - - let info = config.clone().into(); - + /// Creates a new fetcher with the given config. + pub fn new(config: TransactionFetcherConfig) -> Self { let metrics = TransactionFetcherMetrics::default(); - metrics.capacity_inflight_requests.increment(max_inflight_requests as u64); + metrics.capacity_inflight_requests.increment(config.max_inflight_requests as u64); Self { - active_peers: LruMap::with_hasher(max_inflight_requests, Default::default()), - hashes_pending_fetch: LruCache::with_hasher( - max_capacity_cache_txns_pending_fetch, - Default::default(), - ), - hashes_fetch_inflight_and_pending_fetch: LruMap::with_hasher( - max_inflight_requests + max_capacity_cache_txns_pending_fetch, - Default::default(), - ), - info, + hashes: Default::default(), + order: Default::default(), + peers: Default::default(), + peer_keys: Default::default(), + next_peer_key: 0, + next_request_id: 0, + next_generation: 0, + ready: Default::default(), + inflight: Default::default(), + num_fetching: 0, + scratch_requested: Default::default(), + scratch_delivered: Default::default(), + scratch_queue: Default::default(), + config, metrics, - ..Default::default() } } - /// Removes the specified hashes from inflight tracking. - #[inline] - pub fn remove_hashes_from_transaction_fetcher<'a, I>(&mut self, hashes: I) - where - I: IntoIterator, - { - for hash in hashes { - self.hashes_fetch_inflight_and_pending_fetch.remove(hash); - self.hashes_pending_fetch.remove(hash); - } + /// Returns the fetcher's config. + pub const fn config(&self) -> &TransactionFetcherConfig { + &self.config } - /// Updates peer's activity status upon a resolved [`GetPooledTxRequest`]. - fn decrement_inflight_request_count_for(&mut self, peer_id: &PeerId) { - let remove = || -> bool { - if let Some(inflight_count) = self.active_peers.get(peer_id) { - *inflight_count = inflight_count.saturating_sub(1); - if *inflight_count == 0 { - return true - } - } - false - }(); + /// Returns the number of tracked hashes, pending and fetching. + pub fn num_hashes(&self) -> usize { + self.hashes.len() + } - if remove { - self.active_peers.remove(peer_id); - } + /// Returns the number of hashes that are waiting for an idle candidate peer. + pub fn num_pending_hashes(&self) -> usize { + self.hashes.len().saturating_sub(self.num_fetching) } - /// Returns `true` if peer is idle with respect to `self.inflight_requests`. - #[inline] - pub fn is_idle(&self, peer_id: &PeerId) -> bool { - let Some(inflight_count) = self.active_peers.peek(peer_id) else { return true }; - *inflight_count < self.info.max_inflight_requests_per_peer + /// Returns the number of hashes that are part of an inflight request. + pub const fn num_fetching_hashes(&self) -> usize { + self.num_fetching } - /// Returns any idle peer for the given hash. - pub fn get_idle_peer_for(&self, hash: TxHash) -> Option<&PeerId> { - let TxFetchMetadata { fallback_peers, .. } = - self.hashes_fetch_inflight_and_pending_fetch.peek(&hash)?; + /// Returns the number of inflight requests. + pub fn num_inflight_requests(&self) -> usize { + self.inflight.len() + } - fallback_peers.iter().find(|peer_id| self.is_idle(peer_id)) + /// Returns `true` if there is no inflight request to the peer. + pub fn is_idle(&self, peer_id: &PeerId) -> bool { + self.peer_keys + .get(peer_id) + .and_then(|key| self.peers.get(key)) + .is_none_or(|peer| peer.inflight == 0) } - /// Returns any idle peer for any hash pending fetch. If one is found, the corresponding - /// hash is written to the request buffer that is passed as parameter. + /// Records hashes announced by the peer and queues them for fetching. + /// + /// The metadata of a hash is the announced transaction type and size, if the announcement + /// carried it. The caller is expected to have filtered out hashes that are already known. /// - /// Loops through the hashes pending fetch in lru order until one is found with an idle - /// fallback peer, or the budget passed as parameter is depleted, whatever happens first. - pub fn find_any_idle_fallback_peer_for_any_pending_hash( + /// Requests are only sent by [`Self::dispatch`]. + pub fn on_announcement( &mut self, - hashes_to_request: &mut RequestTxHashes, - mut budget: Option, // search fallback peers for max `budget` lru pending hashes - ) -> Option { - let mut hashes_pending_fetch_iter = self.hashes_pending_fetch.iter(); - - let idle_peer = loop { - let &hash = hashes_pending_fetch_iter.next()?; - - let idle_peer = self.get_idle_peer_for(hash); + peer_id: PeerId, + announcement: impl IntoIterator, + ) { + let key = self.peer_key(peer_id); + let max_per_peer = self.config.max_announced_hashes_per_peer as usize; + let max_total = self.config.max_capacity_cache_txns_pending_fetch as usize; + + let mut dropped_peer_limit = 0u64; + let mut evicted_at_capacity = 0u64; + let mut dropped_at_capacity = 0u64; + + // The peer's count of tracked hashes and the hashes to queue for it are kept locally and + // applied once at the end, so every announced hash costs a single map lookup. + let Some(peer) = self.peers.get(&key) else { return }; + let mut tracked = peer.tracked; + let mut queue = std::mem::take(&mut self.scratch_queue); + queue.clear(); + // Entries at the front of `queue` already evicted or moved into the peer queue. + let mut queue_start = 0; + // Other peers can only lose tracked hashes during this announcement. Refresh their + // heap entries lazily instead of scanning every peer for each hash at capacity. + let mut eviction = None; + // Gossip commonly replaces the same fallback peer for every hash in a batch. Apply + // that peer's accounting once, flushing before eviction needs up-to-date counts. + let mut retired_candidates = DeferredCandidateRemovals::default(); + + for (hash, metadata) in announcement { + let size = announced_size(metadata); + let at_capacity = self.hashes.len() >= max_total; + + // whether the hash is queued for the peer right away + let (eager, generation) = match self.hashes.entry(hash) { + Entry::Occupied(mut occupied) => { + let entry = occupied.get_mut(); + if let Some(candidate) = entry.candidate_mut(key) { + // announced before by this peer, keep the latest size + candidate.set_size(size); + continue + } + if tracked >= max_per_peer { + dropped_peer_limit += 1; + continue + } + if entry.candidates.len() == MAX_COUNT_CANDIDATE_PEERS_PER_HASH { + // Keep the first eager sources and the most recent fallback sources. + // A fixed first-N list lets early co-announcers exclude every later source. + let replace = (MAX_COUNT_EAGER_CANDIDATE_PEERS_PER_HASH.. + entry.candidates.len()) + .find(|&idx| { + entry.fetching_by.is_none_or(|(fetching, _)| { + entry.candidates[idx].peer != fetching + }) + }) + .expect("only one candidate can be fetching"); + let removed = entry.candidates.remove(replace); + retired_candidates.record_removed(removed.peer, &mut self.peers); + } + // The first announcers get the hash queued right away, later ones are only + // remembered and asked once the earlier ones failed to deliver. + let eager = entry.candidates.len() < MAX_COUNT_EAGER_CANDIDATE_PEERS_PER_HASH || + (entry.fetching_by.is_none() && + !entry.candidates.iter().any(Candidate::is_queued)); + let candidate = if eager { + Candidate::queued(key, size) + } else { + Candidate::unqueued(key, size) + }; + entry.candidates.push(candidate); + (eager, entry.generation) + } + Entry::Vacant(vacant) => { + let generation = self.next_generation; + self.next_generation += 1; + if tracked >= max_per_peer { + dropped_peer_limit += 1; + continue + } + if at_capacity { + retired_candidates.flush(&mut self.peers); + // the evicted hash may be one of this peer's, so its count is synced + if let Some(peer) = self.peers.get_mut(&key) { + peer.tracked = tracked; + } + if self.evict_pending(key, &queue, &mut queue_start, &mut eviction) { + evicted_at_capacity += 1; + tracked = self.peers.get(&key).map_or(tracked, |peer| peer.tracked); + } else { + dropped_at_capacity += 1; + continue + } + self.hashes.insert(hash, TxEntry::new(key, size, generation)); + } else { + vacant.insert(TxEntry::new(key, size, generation)); + } + self.record_order(hash, generation); + (true, generation) + } + }; - if idle_peer.is_some() { - hashes_to_request.insert(hash); - break idle_peer.copied() + tracked += 1; + if eager { + queue.push((hash, generation)); } + } - if let Some(ref mut bud) = budget { - *bud = bud.saturating_sub(1); - if *bud == 0 { - return None - } + retired_candidates.flush(&mut self.peers); + let queued = queue.len() - queue_start; + if let Some(peer) = self.peers.get_mut(&key) { + peer.tracked = tracked; + for &(hash, generation) in &queue[queue_start..] { + peer.push_queue( + &self.hashes, + key, + (hash, generation), + QueuePosition::Back, + max_per_peer, + ); } - }; - let hash = hashes_to_request.iter().next()?; + } + queue.clear(); + self.scratch_queue = queue; - // pop hash that is loaded in request buffer from cache of hashes pending fetch - drop(hashes_pending_fetch_iter); - _ = self.hashes_pending_fetch.remove(hash); + if dropped_peer_limit > 0 { + self.metrics.announced_hashes_dropped_peer_limit.increment(dropped_peer_limit); + } + if evicted_at_capacity > 0 { + self.metrics.hashes_evicted_at_capacity.increment(evicted_at_capacity); + } + if dropped_at_capacity > 0 { + self.metrics.announced_hashes_dropped_at_capacity.increment(dropped_at_capacity); + } - idle_peer + if queued > 0 { + trace!(target: "net::tx", + peer_id=format!("{peer_id:#}"), + queued, + dropped_peer_limit, + evicted_at_capacity, + dropped_at_capacity, + "queued announced hashes" + ); + } + if queued > 0 || queue_start > 0 { + self.mark_ready(key, QueuePosition::Back); + } } - /// Packages hashes for a [`GetPooledTxRequest`] up to limit. Returns left over hashes. Takes - /// a [`RequestTxHashes`] buffer as parameter for filling with hashes to request. - /// - /// Returns left over hashes. - pub fn pack_request( - &self, - hashes_to_request: &mut RequestTxHashes, - hashes_from_announcement: ValidAnnouncementData, - ) -> RequestTxHashes { - if hashes_from_announcement.msg_version().has_eth68_metadata() { - return self.pack_request_eth68(hashes_to_request, hashes_from_announcement) - } - self.pack_request_eth66(hashes_to_request, hashes_from_announcement) - } - - /// Packages hashes for a [`GetPooledTxRequest`] from an - /// [`Eth68`](reth_eth_wire::EthVersion::Eth68) announcement up to limit as defined by protocol - /// version 68. Takes a [`RequestTxHashes`] buffer as parameter for filling with hashes to - /// request. + /// Sends `GetPooledTransactions` requests to idle peers that have queued hashes. /// - /// Returns left over hashes. + /// Stops when the global inflight request limit is reached. `max_hashes_per_request` caps + /// each request independently of hashes inflight to other peers, so stalled peers cannot + /// shrink requests to responsive peers. Zero sends nothing. The caller must enforce its + /// import backpressure when consuming responses; decoded responses remain bounded by + /// the configured inflight request limit. /// - /// Loops through hashes passed as parameter and checks if a hash fits in the expected - /// response. If no, it's added to surplus hashes. If yes, it's added to hashes to the request - /// and expected response size is accumulated. - pub fn pack_request_eth68( - &self, - hashes_to_request: &mut RequestTxHashes, - hashes_from_announcement: impl HandleMempoolData - + IntoIterator)>, - ) -> RequestTxHashes { - let mut acc_size_response = 0; - - let mut hashes_from_announcement_iter = hashes_from_announcement.into_iter(); - - if let Some((hash, Some((_ty, size)))) = hashes_from_announcement_iter.next() { - hashes_to_request.insert(hash); - - // tx is really big, pack request with single tx - if size >= self.info.soft_limit_byte_size_pooled_transactions_response_on_pack_request { - return hashes_from_announcement_iter.collect() - } - acc_size_response = size; + /// Returns the number of requests sent. New requests are only polled by the next call to + /// [`Stream::poll_next`], so the caller must poll the fetcher again if any were sent. + pub fn dispatch( + &mut self, + peers: &HashMap, FbBuildHasher<64>>, + max_hashes_per_request: usize, + ) -> usize { + if max_hashes_per_request == 0 { + return 0 } - - let mut surplus_hashes = RequestTxHashes::default(); - - // folds size based on expected response size and adds selected hashes to the request - // list and the other hashes to the surplus list - for (hash, metadata) in hashes_from_announcement_iter.by_ref() { - let Some((_ty, size)) = metadata else { - unreachable!("this method is called upon reception of an eth68 announcement") - }; - - let next_acc_size = acc_size_response.checked_add(size).filter(|next_acc_size| { - *next_acc_size <= - self.info.soft_limit_byte_size_pooled_transactions_response_on_pack_request - }); - - if let Some(next_acc_size) = next_acc_size { - // only update accumulated size of tx response if tx will fit in without exceeding - // soft limit - acc_size_response = next_acc_size; - _ = hashes_to_request.insert(hash) - } else { - _ = surplus_hashes.insert(hash) + let max_inflight = self.config.max_inflight_requests as usize; + let mut sent = 0; + // peers whose session channel is full, they get another chance on the next dispatch + let mut retry = SmallVec::<[PeerKey; 4]>::new(); + + while self.inflight.len() < max_inflight { + let Some(key) = self.ready.pop_front() else { break }; + let Some(peer) = self.peers.get_mut(&key) else { continue }; + peer.ready = false; + if peer.inflight >= self.config.max_inflight_requests_per_peer { + continue + } + let peer_id = peer.peer_id; + // the session is gone if the manager doesn't know the peer anymore + let Some(session) = peers.get(&peer_id) else { continue }; + + let limit = max_hashes_per_request + .min(SOFT_LIMIT_COUNT_HASHES_IN_GET_POOLED_TRANSACTIONS_REQUEST); + let request_id = self.next_request_id; + self.next_request_id += 1; + let hashes = self.pack_request(key, request_id, limit); + if hashes.is_empty() { + continue } - let free_space = - self.info.soft_limit_byte_size_pooled_transactions_response_on_pack_request - - acc_size_response; + let (response, rx) = oneshot::channel(); + let request = PeerRequest::GetPooledTransactions { + request: GetPooledTransactions(hashes.clone()), + response, + }; - if free_space < MEDIAN_BYTE_SIZE_SMALL_LEGACY_TX_ENCODED { - break + match session.request_tx().try_send(request) { + Ok(()) => { + trace!(target: "net::tx", + peer_id=format!("{peer_id:#}"), + hashes=hashes.len(), + "sending `GetPooledTransactions` request to peer's session" + ); + if let Some(peer) = self.peers.get_mut(&key) { + peer.inflight += 1; + } + self.inflight.push(InflightRequest { + peer: key, + request_id, + peer_id, + version: session.version(), + client_version: session.client_version.clone(), + hashes, + response: rx, + }); + sent += 1; + // the peer may be allowed more than one inflight request + self.mark_ready(key, QueuePosition::Back); + } + Err(err) => { + self.unpack_request(key, request_id, hashes); + match err { + TrySendError::Full(_) => { + self.metrics.egress_peer_channel_full.increment(1); + retry.push(key); + } + TrySendError::Closed(_) => self.on_peer_disconnected(&peer_id), + } + } } } - surplus_hashes.extend(hashes_from_announcement_iter.map(|(hash, _metadata)| hash)); + for key in retry { + self.mark_ready(key, QueuePosition::Back); + } - surplus_hashes + sent } - /// Packages hashes for a [`GetPooledTxRequest`] from an - /// [`Eth66`](reth_eth_wire::EthVersion::Eth66) announcement up to limit as defined by - /// protocol version 66. Takes a [`RequestTxHashes`] buffer as parameter for filling with - /// hashes to request. + /// Stops tracking the given hashes because the transactions were received, e.g. over + /// broadcast. /// - /// Returns left over hashes. - pub fn pack_request_eth66( - &self, - hashes_to_request: &mut RequestTxHashes, - hashes_from_announcement: ValidAnnouncementData, - ) -> RequestTxHashes { - let (mut hashes, _version) = hashes_from_announcement.into_request_hashes(); - if hashes.len() <= SOFT_LIMIT_COUNT_HASHES_IN_GET_POOLED_TRANSACTIONS_REQUEST { - *hashes_to_request = hashes; - hashes_to_request.shrink_to_fit(); - - RequestTxHashes::default() - } else { - let surplus_hashes = - hashes.retain_count(SOFT_LIMIT_COUNT_HASHES_IN_GET_POOLED_TRANSACTIONS_REQUEST); - *hashes_to_request = hashes; - hashes_to_request.shrink_to_fit(); - - surplus_hashes + /// Hashes that are part of an inflight request are simply not rescheduled when that request + /// resolves. + pub fn on_transactions_received<'a>(&mut self, hashes: impl IntoIterator) { + for hash in hashes { + self.remove_hash(hash); } } - /// Tries to buffer hashes for retry. - pub fn try_buffer_hashes_for_retry( - &mut self, - mut hashes: RequestTxHashes, - peer_failed_to_serve: &PeerId, - ) { - // It could be that the txns have been received over broadcast in the time being. Remove - // the peer as fallback peer so it isn't request again for these hashes. - hashes.retain(|hash| { - if let Some(entry) = self.hashes_fetch_inflight_and_pending_fetch.get(hash) { - entry.fallback_peers_mut().remove(peer_failed_to_serve); + /// Removes the peer as a candidate for all hashes it announced and drops pending hashes that + /// have no candidate left. + /// + /// An inflight request to the peer resolves with an error once its session is gone, which + /// reschedules the requested hashes. + pub fn on_peer_disconnected(&mut self, peer_id: &PeerId) { + let Some(key) = self.peer_keys.remove(peer_id) else { return }; + if self.peers.remove(&key).is_none() { + return + } + + // Every tracked hash is visited, since the peer is a candidate of hashes that are not in + // its queue as well. Disconnects are rare compared to announcements, so this is cheaper + // overall than checking for gone peers whenever candidates are counted. + let mut dropped = 0u64; + let mut requeue = Vec::new(); + self.hashes.retain(|hash, entry| { + let before = entry.candidates.len(); + entry.candidates.retain(|candidate| candidate.peer != key); + if entry.candidates.len() == before || entry.fetching_by.is_some() { return true } - // tx has been seen over broadcast in the time it took for the request to resolve - false + if entry.candidates.is_empty() { + dropped += 1; + return false + } + // the hash must stay queued for at least one candidate + if !entry.candidates.iter().any(|candidate| candidate.is_queued()) { + requeue.push((entry.generation, *hash, entry.unqueued_candidates())); + } + true }); - self.buffer_hashes(hashes, None) - } + let mut requeued = SmallVec::<[PeerKey; MAX_COUNT_CANDIDATE_PEERS_PER_HASH]>::new(); + // Requeue at the front in reverse announcement order. + requeue.sort_unstable_by_key(|(generation, _, _)| std::cmp::Reverse(*generation)); + for (_, hash, targets) in requeue { + self.requeue(hash, targets, &mut requeued); + } - /// Number of hashes pending fetch. - pub fn num_pending_hashes(&self) -> usize { - self.hashes_pending_fetch.len() + if dropped > 0 { + self.metrics.hashes_dropped_no_candidate_peers.increment(dropped); + } + // retried hashes go to the most recent announcers first + for key in requeued { + self.mark_ready(key, QueuePosition::Front); + } } - /// Number of all transaction hashes in the fetcher. - pub fn num_all_hashes(&self) -> usize { - self.hashes_fetch_inflight_and_pending_fetch.len() + /// Updates the fetcher's gauges. + pub fn update_metrics(&self) { + self.metrics.inflight_transaction_requests.set(self.inflight.len() as f64); + self.metrics.hashes_inflight_transaction_requests.set(self.num_fetching as f64); + self.metrics.hashes_pending_fetch.set(self.num_pending_hashes() as f64); } - /// Buffers hashes. Note: Only peers that haven't yet tried to request the hashes should be - /// passed as `fallback_peer` parameter! For re-buffering hashes on failed request, use - /// [`TransactionFetcher::try_buffer_hashes_for_retry`]. Hashes that have been re-requested - /// [`DEFAULT_MAX_RETRIES`], are dropped. - pub fn buffer_hashes(&mut self, hashes: RequestTxHashes, fallback_peer: Option) { - for hash in hashes { - // hash could have been evicted from bounded lru map - let Some(TxFetchMetadata { retries, fallback_peers, .. }) = - self.hashes_fetch_inflight_and_pending_fetch.get(&hash) - else { - continue - }; - - if let Some(peer_id) = fallback_peer { - // peer has not yet requested hash - fallback_peers.insert(peer_id); - } else { - if *retries >= DEFAULT_MAX_RETRIES { - trace!(target: "net::tx", - %hash, - retries, - "retry limit for `GetPooledTransactions` requests reached for hash, dropping hash" - ); + /// Returns the key of the peer, registering the peer if it isn't known yet. + fn peer_key(&mut self, peer_id: PeerId) -> PeerKey { + *self.peer_keys.entry(peer_id).or_insert_with(|| { + let key = PeerKey(self.next_peer_key); + self.next_peer_key += 1; + self.peers.insert(key, PeerState::new(peer_id)); + key + }) + } - self.hashes_fetch_inflight_and_pending_fetch.remove(&hash); - self.hashes_pending_fetch.remove(&hash); - continue - } - *retries += 1; + /// Marks the peer as ready for a request if it is idle and has hashes queued, either behind + /// the other ready peers or ahead of them. + fn mark_ready(&mut self, key: PeerKey, position: QueuePosition) { + if let Some(peer) = self.peers.get_mut(&key) && + !peer.ready && + !peer.queue.is_empty() && + peer.inflight < self.config.max_inflight_requests_per_peer + { + peer.ready = true; + match position { + QueuePosition::Front => self.ready.push_front(key), + QueuePosition::Back => self.ready.push_back(key), } + } + } - if let (_, Some(evicted_hash)) = self.hashes_pending_fetch.insert_and_get_evicted(hash) - { - self.hashes_fetch_inflight_and_pending_fetch.remove(&evicted_hash); - } + /// Queues the hash for the peer, at the back or the front of its queue, and marks it as + /// queued in the hash's candidate entry. + fn enqueue(&mut self, key: PeerKey, hash: TxHash, position: QueuePosition) { + let max_per_peer = self.config.max_announced_hashes_per_peer as usize; + let Some(peer) = self.peers.get_mut(&key) else { return }; + let Some(entry) = self.hashes.get_mut(&hash) else { return }; + let generation = entry.generation; + if let Some(candidate) = entry.candidate_mut(key) { + candidate.set_queued(true); + } + peer.push_queue(&self.hashes, key, (hash, generation), position, max_per_peer); + } + + /// Records a newly tracked hash in the eviction order. + /// + /// Stale lifetimes are removed lazily. Once the order reaches twice the capacity it is + /// compacted in place, retaining only the current generation of every tracked hash. + fn record_order(&mut self, hash: TxHash, generation: u64) { + let max_len = 2 * self.config.max_capacity_cache_txns_pending_fetch as usize; + if self.order.len() >= max_len { + self.order.retain(|(hash, generation)| { + self.hashes.get(hash).is_some_and(|entry| entry.generation == *generation) + }); } + self.order.push_back((hash, generation)); } - /// Tries to request hashes pending fetch. + /// Evicts a pending hash to make room for one announced by `announcer` and returns whether + /// one was found. /// - /// Finds the first buffered hash with a fallback peer that is idle, if any. Fills the rest of - /// the request by checking the transactions seen by the peer against the buffer. - pub fn on_fetch_pending_hashes( + /// Prefer the oldest exclusively announced pending hash of the peer tracking the most + /// hashes, so a group of peers flooding the fetcher gives up its own hashes when possible. The + /// announcer gives way when it tracks as many hashes as the busiest peer. If the + /// chosen peer has no exclusively announced pending hash, the oldest pending hash overall + /// is evicted. Preferring exclusive hashes protects other peers' work when possible, while + /// the fallback prevents co-announcements from making the entire cache unevictable. + /// + /// `queued` are the hashes of the current announcement that are not in the announcer's queue + /// yet; `queued_start` marks entries already evicted or moved into the peer queue. + fn evict_pending( &mut self, - peers: &HashMap, FbBuildHasher<64>>, - has_capacity_wrt_pending_pool_imports: impl Fn(usize) -> bool, + announcer: PeerKey, + queued: &[(TxHash, u64)], + queued_start: &mut usize, + eviction: &mut Option, ) -> bool { - let mut hashes_to_request = RequestTxHashes::with_capacity( - DEFAULT_MARGINAL_COUNT_HASHES_GET_POOLED_TRANSACTIONS_REQUEST, - ); - let mut search_durations = TxFetcherSearchDurations::default(); - - // budget to look for an idle peer before giving up - let budget_find_idle_fallback_peer = self - .search_breadth_budget_find_idle_fallback_peer(&has_capacity_wrt_pending_pool_imports); - - let peer_id = duration_metered_exec!( - { - let Some(peer_id) = self.find_any_idle_fallback_peer_for_any_pending_hash( - &mut hashes_to_request, - budget_find_idle_fallback_peer, - ) else { - // no peers are idle or budget is depleted - return false - }; - - peer_id - }, - search_durations.find_idle_peer - ); - - // peer may have disconnected between idle check and here, re-buffer hashes so they - // aren't lost from the pending fetch cache - let Some(peer) = peers.get(&peer_id) else { - self.buffer_hashes(hashes_to_request, None); - return false + let eviction = eviction.get_or_insert_with(|| EvictionState { + peers: self + .peers + .iter() + .filter(|(key, peer)| **key != announcer && peer.tracked > 0) + .map(|(key, peer)| (peer.tracked, *key)) + .collect(), + exclusive: Default::default(), + }); + let heap = &mut eviction.peers; + let (mut tracked, mut key) = loop { + let Some(&(previous, key)) = heap.peek() else { break (0, announcer) }; + let current = self.peers.get(&key).map_or(0, |peer| peer.tracked); + if current == previous { + break (current, key) + } + heap.pop(); + if current > 0 { + heap.push((current, key)); + } }; - let conn_eth_version = peer.version; - - // fill the request with more hashes pending fetch that have been announced by the peer. - // the search for more hashes is done with respect to the given budget, which determines - // how many hashes to loop through before giving up. if no more hashes are found wrt to - // the budget, the single hash that was taken out of the cache above is sent in a request. - let budget_fill_request = self - .search_breadth_budget_find_intersection_pending_hashes_and_hashes_seen_by_peer( - &has_capacity_wrt_pending_pool_imports, - ); - - duration_metered_exec!( - { - self.fill_request_from_hashes_pending_fetch( - &mut hashes_to_request, - &peer.seen_transactions, - budget_fill_request, - ) - }, - search_durations.fill_request - ); - - self.update_pending_fetch_cache_search_metrics(search_durations); - - trace!(target: "net::tx", - peer_id=format!("{peer_id:#}"), - hashes=?*hashes_to_request, - %conn_eth_version, - "requesting hashes that were stored pending fetch from peer" - ); - - // request the buffered missing transactions - if let Some(failed_to_request_hashes) = - self.request_transactions_from_peer(hashes_to_request, peer) + if let Some(peer) = self.peers.get(&announcer) && + peer.tracked >= tracked { - trace!(target: "net::tx", - peer_id=format!("{peer_id:#}"), - ?failed_to_request_hashes, - %conn_eth_version, - "failed sending request to peer's session, buffering hashes" - ); - - self.buffer_hashes(failed_to_request_hashes, Some(peer_id)); - return false + key = announcer; + tracked = peer.tracked; } - - true - } - - /// Filters out hashes that have been seen before. For hashes that have already been seen, the - /// peer is added as fallback peer. - pub fn filter_unseen_and_pending_hashes( - &mut self, - new_announced_hashes: &mut ValidAnnouncementData, - is_tx_bad_import: impl Fn(&TxHash) -> bool, - peer_id: &PeerId, - client_version: &str, - ) { - let mut previously_unseen_hashes_count = 0; - - let msg_version = new_announced_hashes.msg_version(); - - // filter out inflight hashes, and register the peer as fallback for all inflight hashes - new_announced_hashes.retain(|hash, metadata| { - - // occupied entry - if let Some(TxFetchMetadata{ tx_encoded_length: previously_seen_size, ..}) = self.hashes_fetch_inflight_and_pending_fetch.peek_mut(hash) { - // update size metadata if available - if let Some((_ty, size)) = metadata { - if let Some(prev_size) = previously_seen_size { - // check if this peer is announcing a different size than a previous peer - if size != prev_size { - trace!(target: "net::tx", - peer_id=format!("{peer_id:#}"), - %hash, - size, - previously_seen_size, - %client_version, - "peer announced a different size for tx, this is especially worrying if one size is much bigger..." - ); - } - } - // believe the most recent peer to announce tx - *previously_seen_size = Some(*size); - } - - // hash has been seen but is not inflight - if self.hashes_pending_fetch.remove(hash) { + if tracked == 0 { + return self.evict_oldest_pending() + } + if self.evict_oldest_pending_of(key, &mut eviction.exclusive) { + return true + } + if key == announcer { + while *queued_start < queued.len() { + let (hash, generation) = queued[*queued_start]; + *queued_start += 1; + let Some(entry) = + self.hashes.get(&hash).filter(|entry| entry.generation == generation) + else { + continue + }; + if entry.candidates.len() == 1 && entry.fetching_by.is_none() { + self.remove_hash(&hash); return true } - - return false - } - - // vacant entry - - if is_tx_bad_import(hash) { - return false + // Preserve shared entries skipped by the cursor in this announcement. + self.enqueue(key, hash, QueuePosition::Back); } + } + self.evict_oldest_pending() + } - previously_unseen_hashes_count += 1; - - if self - .hashes_fetch_inflight_and_pending_fetch - .get_or_insert(*hash, || TxFetchMetadata { - retries: 0, - fallback_peers: LruCache::with_hasher( - DEFAULT_MAX_COUNT_FALLBACK_PEERS as u32, - Default::default(), - ), - tx_encoded_length: None, + /// Evicts the oldest pending hash announced only by this peer, if one exists. + /// + /// Scan each selected peer's queue once per announcement. A bounded prefix can hide + /// exclusive victims behind shared hashes; caching avoids rescanning that prefix for every + /// eviction. Newly tracked hashes from the current announcement are handled by its cursor. + fn evict_oldest_pending_of( + &mut self, + key: PeerKey, + exclusive: &mut HashMap>, + ) -> bool { + let candidates = exclusive.entry(key).or_insert_with(|| { + self.peers + .get(&key) + .into_iter() + .flat_map(|peer| &peer.queue) + .filter(|(hash, generation)| { + self.hashes.get(hash).is_some_and(|entry| { + entry.generation == *generation && + entry.candidates.len() == 1 && + entry.has_candidate(key) && + entry.fetching_by.is_none() + }) }) - .is_none() - { - - trace!(target: "net::tx", - peer_id=format!("{peer_id:#}"), - %hash, - ?msg_version, - %client_version, - "failed to cache new announced hash from peer in schnellru::LruMap, dropping hash" - ); - - return false - } - true + .copied() + .collect() }); - - trace!(target: "net::tx", - peer_id=format!("{peer_id:#}"), - previously_unseen_hashes_count=previously_unseen_hashes_count, - msg_version=?msg_version, - client_version=%client_version, - "received previously unseen hashes in announcement from peer" - ); + while let Some((hash, generation)) = candidates.pop_front() { + if self.hashes.get(&hash).is_some_and(|entry| { + entry.generation == generation && + entry.candidates.len() == 1 && + entry.has_candidate(key) && + entry.fetching_by.is_none() + }) { + self.remove_hash(&hash); + return true + } + } + false } - /// Requests the missing transactions from the previously unseen announced hashes of the peer. - /// Returns the requested hashes if the request concurrency limit is reached or if the request - /// fails to send over the channel to the peer's session task. + /// Evicts the oldest pending hash, including shared hashes, as a last resort. /// - /// This filters all announced hashes that are already in flight, and requests the missing, - /// while marking the given peer as an alternative peer for the hashes that are already in - /// flight. - pub fn request_transactions_from_peer( - &mut self, - new_announced_hashes: RequestTxHashes, - peer: &PeerMetadata, - ) -> Option { - let peer_id: PeerId = peer.request_tx.peer_id; - let conn_eth_version = peer.version; - - if self.active_peers.len() >= self.info.max_inflight_requests { - trace!(target: "net::tx", - peer_id=format!("{peer_id:#}"), - hashes=?*new_announced_hashes, - %conn_eth_version, - max_inflight_transaction_requests=self.info.max_inflight_requests, - "limit for concurrent `GetPooledTransactions` requests reached, dropping request for hashes to peer" - ); - return Some(new_announced_hashes) + /// Shared hashes cannot be exempt from eviction: co-announcements could pin every slot. + /// Returns `false` if only fetching hashes were found within the search budget. + fn evict_oldest_pending(&mut self) -> bool { + let mut index = 0; + let mut attempts = 0; + while attempts < MAX_EVICTION_ATTEMPTS { + let Some(&(hash, generation)) = self.order.get(index) else { break }; + let Some(entry) = self.hashes.get(&hash).filter(|entry| entry.generation == generation) + else { + self.order.remove(index); + continue + }; + attempts += 1; + if entry.fetching_by.is_some() { + index += 1; + continue + } + self.order.remove(index); + self.remove_hash(&hash); + return true } + false + } - let Some(inflight_count) = self.active_peers.get_or_insert(peer_id, || 0) else { - trace!(target: "net::tx", - peer_id=format!("{peer_id:#}"), - hashes=?*new_announced_hashes, - conn_eth_version=%conn_eth_version, - "failed to cache active peer in schnellru::LruMap, dropping request to peer" - ); - return Some(new_announced_hashes) - }; + /// Drains the peer's queue into a request, in announcement order, until the request holds + /// `max_hashes` hashes or the expected response size reaches the configured soft limit. A + /// transaction that on its own exceeds the size limit is requested alone. + /// + /// The returned hashes are marked as being fetched by the peer. + fn pack_request(&mut self, key: PeerKey, request_id: u64, max_hashes: usize) -> Vec { + let max_bytes = + self.config.soft_limit_byte_size_pooled_transactions_response_on_pack_request; + let Some(peer) = self.peers.get_mut(&key) else { return Vec::new() }; + + let mut hashes = Vec::with_capacity(peer.queue.len().min(max_hashes)); + let mut bytes = 0usize; + + while let Some((hash, generation)) = peer.queue.pop_front() { + // skip hashes that were delivered in the meantime, or that this peer is no longer a + // candidate for + let Some(entry) = + self.hashes.get_mut(&hash).filter(|entry| entry.generation == generation) + else { + continue + }; + let Some(candidate) = entry.candidate_mut(key) else { continue }; + candidate.set_queued(false); + let size = candidate.request_size(); - if *inflight_count >= self.info.max_inflight_requests_per_peer { - trace!(target: "net::tx", - peer_id=format!("{peer_id:#}"), - hashes=?*new_announced_hashes, - %conn_eth_version, - max_concurrent_tx_reqs_per_peer=self.info.max_inflight_requests_per_peer, - "limit for concurrent `GetPooledTransactions` requests per peer reached" - ); - return Some(new_announced_hashes) - } + // hashes that are being fetched elsewhere are queued again if that fetch fails + if entry.fetching_by.is_some() { + continue + } - #[cfg(debug_assertions)] - { - for hash in &new_announced_hashes { - if self.hashes_pending_fetch.contains(hash) { - tracing::debug!(target: "net::tx", "`{}` should have been taken out of buffer before packing in a request, breaks invariant `@hashes_pending_fetch` and `@inflight_requests`, `@hashes_fetch_inflight_and_pending_fetch` for `{}`: {:?}", - format!("{:?}", new_announced_hashes), // Assuming new_announced_hashes can be debug-printed directly - format!("{:?}", new_announced_hashes), - new_announced_hashes.iter().map(|hash| { - let metadata = self.hashes_fetch_inflight_and_pending_fetch.get(hash); - // Assuming you only need `retries` and `tx_encoded_length` for debugging - (*hash, metadata.map(|m| (m.retries, m.tx_encoded_length))) - }).collect::)>)>>()) + if !hashes.is_empty() && bytes.saturating_add(size) > max_bytes { + if let Some(candidate) = entry.candidate_mut(key) { + candidate.set_queued(true); } + peer.queue.push_front((hash, generation)); + break } - } - let (response, rx) = oneshot::channel(); - let req = PeerRequest::GetPooledTransactions { - request: GetPooledTransactions(new_announced_hashes.iter().copied().collect()), - response, - }; + entry.fetching_by = Some((key, request_id)); + entry.attempts += 1; + self.num_fetching += 1; + bytes = bytes.saturating_add(size); + hashes.push(hash); - // try to send the request to the peer - if let Err(err) = peer.request_tx.try_send(req) { - // peer channel is full - return match err { - TrySendError::Full(_) | TrySendError::Closed(_) => { - self.metrics.egress_peer_channel_full.increment(1); - Some(new_announced_hashes) - } + if hashes.len() >= max_hashes { + break } } - *inflight_count += 1; - // stores a new request future for the request - self.inflight_requests.push(GetPooledTxRequestFut::new(peer_id, new_announced_hashes, rx)); + hashes + } - None + /// Reverts [`Self::pack_request`] for a request that could not be sent: the hashes are pending + /// again and are queued at the front of the peer's queue in their original order. + fn unpack_request(&mut self, key: PeerKey, request_id: u64, hashes: Vec) { + for hash in hashes.into_iter().rev() { + if let Some(entry) = self.hashes.get_mut(&hash) && + entry.fetching_by == Some((key, request_id)) + { + entry.fetching_by = None; + entry.attempts -= 1; + self.num_fetching -= 1; + self.enqueue(key, hash, QueuePosition::Front); + } + } } - /// Tries to fill request with hashes pending fetch so that the expected [`PooledTransactions`] - /// response is full enough. A mutable reference to a list of hashes to request is passed as - /// parameter. A budget is passed as parameter, this ensures that the node stops searching - /// for more hashes after the budget is depleted. Under bad network conditions, the cache of - /// hashes pending fetch may become very full for a while. As the node recovers, the hashes - /// pending fetch cache should get smaller. The budget should aim to be big enough to loop - /// through all buffered hashes in good network conditions. - /// - /// The request hashes buffer is filled as if it's an eth68 request, i.e. smartly assemble - /// the request based on expected response size. For any hash missing size metadata, it is - /// guessed at [`AVERAGE_BYTE_SIZE_TX_ENCODED`]. - /// - /// Loops through hashes pending fetch and does: - /// - /// 1. Check if a hash pending fetch is seen by peer. - /// 2. Optimistically include the hash in the request. - /// 3. Accumulate expected total response size. - /// 4. Check if acc size and hashes count is at limit, if so stop looping. - /// 5. Remove hashes to request from cache of hashes pending fetch. - pub fn fill_request_from_hashes_pending_fetch( + /// Queues a pending hash for the given candidates, the ones that don't have it queued, and + /// records those peers in `requeued`. + fn requeue( &mut self, - hashes_to_request: &mut RequestTxHashes, - seen_hashes: &LruCache>, - mut budget_fill_request: Option, // check max `budget` lru pending hashes + hash: TxHash, + targets: SmallVec<[PeerKey; MAX_COUNT_CANDIDATE_PEERS_PER_HASH]>, + requeued: &mut SmallVec<[PeerKey; MAX_COUNT_CANDIDATE_PEERS_PER_HASH]>, ) { - let Some(hash) = hashes_to_request.iter().next() else { return }; - - let mut acc_size_response = self - .hashes_fetch_inflight_and_pending_fetch - .get(hash) - .and_then(|entry| entry.tx_encoded_len()) - .unwrap_or(AVERAGE_BYTE_SIZE_TX_ENCODED); - - // if request full enough already, we're satisfied, send request for single tx - if acc_size_response >= - DEFAULT_SOFT_LIMIT_BYTE_SIZE_POOLED_TRANSACTIONS_RESPONSE_ON_FETCH_PENDING_HASHES - { - return - } - - // try to fill request by checking if any other hashes pending fetch (in lru order) are - // also seen by peer - for hash in self.hashes_pending_fetch.iter() { - // 1. Check if a hash pending fetch is seen by peer. - if !seen_hashes.contains(hash) { - continue - }; - - // 2. Optimistically include the hash in the request. - hashes_to_request.insert(*hash); - - // 3. Accumulate expected total response size. - let size = self - .hashes_fetch_inflight_and_pending_fetch - .get(hash) - .and_then(|entry| entry.tx_encoded_len()) - .unwrap_or(AVERAGE_BYTE_SIZE_TX_ENCODED); - - acc_size_response = acc_size_response.saturating_add(size); - - // 4. Check if acc size or hashes count is at limit, if so stop looping. - // if expected response is full enough or the number of hashes in the request is - // enough, we're satisfied - if acc_size_response >= - DEFAULT_SOFT_LIMIT_BYTE_SIZE_POOLED_TRANSACTIONS_RESPONSE_ON_FETCH_PENDING_HASHES || - hashes_to_request.len() > - DEFAULT_SOFT_LIMIT_COUNT_HASHES_IN_GET_POOLED_TRANSACTIONS_REQUEST_ON_FETCH_PENDING_HASHES - { - break + for key in targets { + self.enqueue(key, hash, QueuePosition::Front); + if !requeued.contains(&key) { + requeued.push(key); } + } + } - if let Some(ref mut bud) = budget_fill_request { - *bud -= 1; - if *bud == 0 { - break - } + /// Stops tracking the hash. + fn remove_hash(&mut self, hash: &TxHash) { + let Some(entry) = self.hashes.remove(hash) else { return }; + if entry.fetching_by.is_some() { + self.num_fetching -= 1; + } + for candidate in &entry.candidates { + if let Some(peer) = self.peers.get_mut(&candidate.peer) { + peer.tracked = peer.tracked.saturating_sub(1); } } + } - // 5. Remove hashes to request from cache of hashes pending fetch. - for hash in hashes_to_request.iter() { - self.hashes_pending_fetch.remove(hash); + /// Processes a resolved request and returns the corresponding event. + fn on_resolved( + &mut self, + resolved: ResolvedRequest, + ) -> FetchEvent { + let ResolvedRequest { + peer: key, + request_id, + peer_id, + version, + client_version, + hashes: requested, + result, + } = resolved; + + if let Some(peer) = self.peers.get_mut(&key) { + peer.inflight = peer.inflight.saturating_sub(1); } - } - /// Returns `true` if [`TransactionFetcher`] has capacity to request pending hashes. Returns - /// `false` if [`TransactionFetcher`] is operating close to full capacity. - pub fn has_capacity_for_fetching_pending_hashes(&self) -> bool { - let info = &self.info; + let mut delivered = std::mem::take(&mut self.scratch_delivered); + delivered.clear(); + + let outcome = match result { + Ok(mut transactions) => { + let mut requested_set = std::mem::take(&mut self.scratch_requested); + requested_set.clear(); + requested_set.extend(requested.iter().copied()); + let unsolicited = + verify_response(&mut transactions, &requested_set, &mut delivered); + self.scratch_requested = requested_set; + Ok((transactions, unsolicited)) + } + Err(error) => Err(error), + }; - self.has_capacity(info.max_inflight_requests) - } + let timed_out = matches!(&outcome, Err(RequestError::Timeout)); + self.on_delivery(key, request_id, &requested, &delivered, timed_out); + self.scratch_delivered = delivered; + self.mark_ready(key, QueuePosition::Back); - /// Returns `true` if the number of inflight requests are under a given tolerated max. - fn has_capacity(&self, max_inflight_requests: usize) -> bool { - self.inflight_requests.len() <= max_inflight_requests - } - - /// Returns the limit to enforce when looking for any pending hash with an idle fallback peer. - /// - /// Returns `Some(limit)` if [`TransactionFetcher`] and the - /// [`TransactionPool`](reth_transaction_pool::TransactionPool) are operating close to full - /// capacity. Returns `None`, unlimited, if they are not that busy. - pub fn search_breadth_budget_find_idle_fallback_peer( - &self, - has_capacity_wrt_pending_pool_imports: impl Fn(usize) -> bool, - ) -> Option { - let info = &self.info; - - let tx_fetcher_has_capacity = self.has_capacity( - info.max_inflight_requests / - DEFAULT_DIVISOR_MAX_COUNT_INFLIGHT_REQUESTS_ON_FIND_IDLE_PEER, - ); - let tx_pool_has_capacity = has_capacity_wrt_pending_pool_imports( - DEFAULT_DIVISOR_MAX_COUNT_PENDING_POOL_IMPORTS_ON_FIND_IDLE_PEER, - ); - - if tx_fetcher_has_capacity && tx_pool_has_capacity { - // unlimited search breadth - None - } else { - // limited breadth of search for idle peer - let limit = DEFAULT_BUDGET_FIND_IDLE_FALLBACK_PEER; - - trace!(target: "net::tx", - inflight_requests=self.inflight_requests.len(), - max_inflight_transaction_requests=info.max_inflight_requests, - hashes_pending_fetch=self.hashes_pending_fetch.len(), - limit, - "search breadth limited in search for idle fallback peer for some hash pending fetch" - ); - - Some(limit) + match outcome { + Ok((transactions, unsolicited)) => { + if unsolicited > 0 { + self.metrics.unsolicited_transactions.increment(unsolicited as u64); + trace!(target: "net::tx", + peer_id=format!("{peer_id:#}"), + unsolicited, + "received transactions in `PooledTransactions` response that weren't requested" + ); + } + if !transactions.is_empty() { + self.metrics.fetched_transactions.increment(transactions.len() as u64); + FetchEvent::TransactionsFetched { + peer_id, + version, + client_version, + transactions, + report_peer: unsolicited > 0, + } + } else if unsolicited > 0 { + // the peer only sent transactions we didn't ask for + FetchEvent::FetchError { peer_id, error: RequestError::BadResponse } + } else { + trace!(target: "net::tx", + peer_id=format!("{peer_id:#}"), + requested=requested.len(), + "received empty `PooledTransactions` response, peer failed to serve hashes it announced" + ); + FetchEvent::EmptyResponse { peer_id } + } + } + Err(error) => FetchEvent::FetchError { peer_id, error }, } } - /// Returns the limit to enforce when looking for the intersection between hashes announced by - /// peer and hashes pending fetch. - /// - /// Returns `Some(limit)` if [`TransactionFetcher`] and the - /// [`TransactionPool`](reth_transaction_pool::TransactionPool) are operating close to full - /// capacity. Returns `None`, unlimited, if they are not that busy. - pub fn search_breadth_budget_find_intersection_pending_hashes_and_hashes_seen_by_peer( - &self, - has_capacity_wrt_pending_pool_imports: impl Fn(usize) -> bool, - ) -> Option { - let info = &self.info; - - let tx_fetcher_has_capacity = self.has_capacity( - info.max_inflight_requests / - DEFAULT_DIVISOR_MAX_COUNT_INFLIGHT_REQUESTS_ON_FIND_INTERSECTION, - ); - let tx_pool_has_capacity = has_capacity_wrt_pending_pool_imports( - DEFAULT_DIVISOR_MAX_COUNT_PENDING_POOL_IMPORTS_ON_FIND_INTERSECTION, - ); - - if tx_fetcher_has_capacity && tx_pool_has_capacity { - // unlimited search breadth - None + /// Settles the requested hashes of a resolved request: delivered hashes are dropped and + /// undelivered hashes are rescheduled for their remaining candidates. + fn on_delivery( + &mut self, + key: PeerKey, + request_id: u64, + requested: &[TxHash], + delivered: &B256Set, + timed_out: bool, + ) { + // Position right after the last delivered hash. Drop the responder as a candidate for + // missing hashes before it. Keep it for the undelivered tail only if at least half of + // the requested hashes were delivered, treating that tail as potentially truncated. + // This is a retry heuristic: transaction counts do not establish whether the response + // reached a byte limit. Requiring half the request limits repeated low-progress retries. + let cutoff = if 2 * delivered.len() >= requested.len() { + requested + .iter() + .rposition(|hash| delivered.contains(hash)) + .map_or(requested.len(), |idx| idx + 1) } else { - // limited breadth of search for idle peer - let limit = DEFAULT_BUDGET_FIND_INTERSECTION_ANNOUNCED_BY_PEER_AND_PENDING_FETCH; - - trace!(target: "net::tx", - inflight_requests=self.inflight_requests.len(), - max_inflight_transaction_requests=self.info.max_inflight_requests, - hashes_pending_fetch=self.hashes_pending_fetch.len(), - limit=limit, - "search breadth limited in search for intersection of hashes announced by peer and hashes pending fetch" - ); + requested.len() + }; - Some(limit) - } - } + let mut dropped = 0u64; + let mut requeued = SmallVec::<[PeerKey; MAX_COUNT_CANDIDATE_PEERS_PER_HASH]>::new(); - /// Processes a resolved [`GetPooledTransactions`] request. Queues the outcome as a - /// [`FetchEvent`], which will then be streamed by - /// [`TransactionsManager`](super::TransactionsManager). - pub fn on_resolved_get_pooled_transactions_request_fut( - &mut self, - response: GetPooledTxResponse, - ) -> FetchEvent { - // update peer activity, requests for buffered hashes can only be made to idle - // fallback peers - let GetPooledTxResponse { peer_id, mut requested_hashes, result } = response; - - self.decrement_inflight_request_count_for(&peer_id); - - match result { - Ok(Ok(transactions)) => { - // - // 1. peer has failed to serve any of the hashes it has announced to us that we, - // as a follow, have requested - // - if transactions.is_empty() { - trace!(target: "net::tx", - peer_id=format!("{peer_id:#}"), - requested_hashes_len=requested_hashes.len(), - "received empty `PooledTransactions` response from peer, peer failed to serve hashes it announced" - ); + // iterate in reverse so that queueing at the front of a queue preserves the request order + for (idx, hash) in requested.iter().enumerate().rev() { + if delivered.contains(hash) { + self.remove_hash(hash); + continue + } - return FetchEvent::EmptyResponse { peer_id } + // the hash was received elsewhere in the meantime, or was even announced and assigned + // to another peer again + let Some(entry) = self.hashes.get_mut(hash) else { continue }; + if entry.fetching_by != Some((key, request_id)) { + continue + } + entry.fetching_by = None; + self.num_fetching -= 1; + + // A short adaptive session timeout can expire before the first response from the + // only source arrives. Allow one retry; prefer another source whenever available. + let retry_only_source = timed_out && entry.attempts == 1 && entry.candidates.len() == 1; + let peers = &mut self.peers; + entry.candidates.retain(|candidate| { + // disconnected peers are pruned as well + let Some(peer) = peers.get_mut(&candidate.peer) else { return false }; + if idx < cutoff && candidate.peer == key && !retry_only_source { + peer.tracked = peer.tracked.saturating_sub(1); + return false } + true + }); - // - // 2. filter out hashes that we didn't request - // - let payload = UnverifiedPooledTransactions::new(transactions); + if entry.candidates.is_empty() || entry.attempts >= MAX_FETCH_ATTEMPTS_PER_HASH { + // Rotating fallback candidates must not allow unlimited retries for one hash. + self.remove_hash(hash); + dropped += 1; + continue + } - let unverified_len = payload.len(); - let (verification_outcome, verified_payload) = - payload.verify(&requested_hashes, &peer_id); + let targets = entry.unqueued_candidates(); + self.requeue(*hash, targets, &mut requeued); + } - let unsolicited = unverified_len - verified_payload.len(); - if unsolicited > 0 { - self.metrics.unsolicited_transactions.increment(unsolicited as u64); - } + if dropped > 0 { + self.metrics.hashes_dropped_no_candidate_peers.increment(dropped); + trace!(target: "net::tx", dropped, "dropped hashes with no candidates or exhausted fetch attempts"); + } - let report_peer = if verification_outcome == VerificationOutcome::ReportPeer { - trace!(target: "net::tx", - peer_id=format!("{peer_id:#}"), - unverified_len, - verified_payload_len=verified_payload.len(), - "received `PooledTransactions` response from peer with entries that didn't verify against request, filtered out transactions" - ); - true - } else { - false - }; + // retried hashes go to the most recent announcers first + for key in requeued { + self.mark_ready(key, QueuePosition::Front); + } + } +} - // peer has only sent hashes that we didn't request - if verified_payload.is_empty() { - return FetchEvent::FetchError { peer_id, error: RequestError::BadResponse } - } +#[cfg(test)] +impl TransactionFetcher { + /// Returns the connected peers that are candidates for the hash, in announcement order. + pub(super) fn candidate_peers(&self, hash: &TxHash) -> Vec { + self.hashes + .get(hash) + .map(|entry| { + entry + .candidates + .iter() + .filter_map(|candidate| self.peers.get(&candidate.peer)) + .map(|peer| peer.peer_id) + .collect() + }) + .unwrap_or_default() + } - // - // 3. stateless validation of payload, e.g. dedup - // - let unvalidated_payload_len = verified_payload.len(); + /// Returns the peer the hash is currently fetched from, if any. + fn fetching_peer(&self, hash: &TxHash) -> Option { + let (key, _) = self.hashes.get(hash)?.fetching_by?; + self.peers.get(&key).map(|peer| peer.peer_id) + } - let valid_payload = verified_payload.dedup(); + /// Returns the hashes queued for the peer, oldest first. May include hashes that are not + /// tracked anymore or are being fetched elsewhere. + pub(super) fn queued_hashes(&self, peer_id: &PeerId) -> Vec { + self.peer_keys + .get(peer_id) + .and_then(|key| self.peers.get(key)) + .map(|peer| peer.queue.iter().map(|(hash, _)| *hash).collect()) + .unwrap_or_default() + } - // todo: validate based on announced tx size/type and report peer for sending - // invalid response . requires - // passing the rlp encoded length down from active session along with the decoded - // tx. + /// Returns the number of peers the fetcher tracks. + fn num_peers(&self) -> usize { + self.peers.len() + } - if valid_payload.len() != unvalidated_payload_len { - trace!(target: "net::tx", - peer_id=format!("{peer_id:#}"), - unvalidated_payload_len, - valid_payload_len=valid_payload.len(), - "received `PooledTransactions` response from peer with duplicate entries, filtered them out" - ); - } - // valid payload will have at least one transaction at this point. even if the tx - // size/type announced by the peer is different to the actual tx size/type, pass on - // to pending pool imports pipeline for validation. - - // - // 4. clear received hashes - // - let requested_hashes_len = requested_hashes.len(); - let mut fetched = Vec::with_capacity(valid_payload.len()); - requested_hashes.retain(|requested_hash| { - if valid_payload.contains_key(requested_hash) { - // hash is now known, stop tracking - fetched.push(*requested_hash); - return false - } - true - }); - fetched.shrink_to_fit(); - self.metrics.fetched_transactions.increment(fetched.len() as u64); + /// Panics if the internal bookkeeping is inconsistent. + fn assert_invariants(&self) { + let fetching = self.hashes.values().filter(|entry| entry.fetching_by.is_some()).count(); + assert_eq!(fetching, self.num_fetching, "fetching counter out of sync"); + + let ordered = self + .order + .iter() + .filter(|(hash, generation)| { + self.hashes.get(hash).is_some_and(|entry| entry.generation == *generation) + }) + .map(|(hash, _)| *hash) + .collect::(); + assert!( + self.order.len() <= 2 * self.config.max_capacity_cache_txns_pending_fetch as usize, + "eviction order grew beyond its bound" + ); - if fetched.len() < requested_hashes_len { - trace!(target: "net::tx", - peer_id=format!("{peer_id:#}"), - requested_hashes_len=requested_hashes_len, - fetched_len=fetched.len(), - "peer failed to serve hashes it announced" + for (hash, entry) in &self.hashes { + assert!( + !entry.candidates.is_empty() || entry.fetching_by.is_some(), + "{hash} is pending without candidates" + ); + assert!( + entry.candidates.len() <= MAX_COUNT_CANDIDATE_PEERS_PER_HASH, + "{hash} has too many candidates" + ); + let mut unique = entry.candidates.iter().map(|c| c.peer).collect::>(); + unique.sort_unstable(); + unique.dedup(); + assert_eq!(unique.len(), entry.candidates.len(), "{hash} has duplicate candidates"); + assert!(ordered.contains(hash), "{hash} is missing from the eviction order"); + if let Some((peer, request_id)) = entry.fetching_by { + assert!( + self.inflight.iter().any(|request| request.peer == peer && + request.request_id == request_id && + request.hashes.contains(hash)), + "fetching hash has no matching request" + ); + } + assert!(entry.attempts <= MAX_FETCH_ATTEMPTS_PER_HASH); + } + + // a hash flagged as queued is in the peer's queue, and a pending hash is queued for at + // least one connected candidate, otherwise it could starve + let queued = self + .peers + .iter() + .map(|(key, peer)| { + assert!( + peer.queue.len() <= 2 * self.config.max_announced_hashes_per_peer as usize, + "queue of {:#} grew beyond its bound", + peer.peer_id + ); + ( + *key, + peer.queue + .iter() + .filter(|(hash, generation)| { + self.hashes + .get(hash) + .is_some_and(|entry| entry.generation == *generation) + }) + .map(|(hash, _)| *hash) + .collect::(), + ) + }) + .collect::>(); + for (hash, entry) in &self.hashes { + let mut fetchable = entry.fetching_by.is_some(); + for candidate in &entry.candidates { + let queue = + queued.get(&candidate.peer).expect("candidate refers to a disconnected peer"); + if candidate.is_queued() { + assert!( + queue.contains(hash), + "{hash} is flagged but not queued for {:?}", + candidate.peer ); + fetchable = true; } - - // - // 5. buffer left over hashes - // - self.try_buffer_hashes_for_retry(requested_hashes, &peer_id); - - let transactions = valid_payload.into_data().into_values().collect(); - - FetchEvent::TransactionsFetched { peer_id, transactions, report_peer } - } - Ok(Err(req_err)) => { - self.try_buffer_hashes_for_retry(requested_hashes, &peer_id); - FetchEvent::FetchError { peer_id, error: req_err } - } - Err(_) => { - self.try_buffer_hashes_for_retry(requested_hashes, &peer_id); - // request channel closed/dropped - FetchEvent::FetchError { peer_id, error: RequestError::ChannelClosed } } + assert!(fetchable, "pending {hash} is not queued for any connected candidate"); } - } -} - -impl Stream for TransactionFetcher { - type Item = FetchEvent; - /// Advances all inflight requests and returns the next event. - fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { - // `FuturesUnordered` doesn't close when `None` is returned. so just return pending. - // - if self.inflight_requests.is_empty() { - return Poll::Pending + for (key, peer) in &self.peers { + assert_eq!( + peer.inflight as usize, + self.inflight.iter().filter(|request| request.peer == *key).count() + ); + assert!(peer.inflight <= self.config.max_inflight_requests_per_peer); + if peer.inflight < self.config.max_inflight_requests_per_peer && + peer.queue.iter().any(|(hash, generation)| { + self.hashes.get(hash).is_some_and(|entry| { + entry.generation == *generation && + entry.fetching_by.is_none() && + entry + .candidates + .iter() + .any(|candidate| candidate.peer == *key && candidate.is_queued()) + }) + }) + { + assert!(peer.ready, "idle peer has queued work but is not ready"); + } + let tracked = self.hashes.values().filter(|entry| entry.has_candidate(*key)).count(); + assert_eq!(tracked, peer.tracked, "tracked counter of {:#} out of sync", peer.peer_id); + assert!( + peer.tracked <= self.config.max_announced_hashes_per_peer as usize, + "{:#} exceeds the per peer limit", + peer.peer_id + ); + assert_eq!( + peer.ready, + self.ready.contains(key), + "ready flag of {:#} out of sync", + peer.peer_id + ); + if peer.ready { + assert!( + !peer.queue.is_empty(), + "{:#} is ready without queued hashes", + peer.peer_id + ); + assert!( + peer.inflight < self.config.max_inflight_requests_per_peer, + "{:#} is ready but busy", + peer.peer_id + ); + } + assert_eq!( + self.peer_keys.get(&peer.peer_id), + Some(key), + "peer key mapping out of sync" + ); } - if let Some(resp) = ready!(self.inflight_requests.poll_next_unpin(cx)) { - return Poll::Ready(Some(self.on_resolved_get_pooled_transactions_request_fut(resp))) + for (peer_id, key) in &self.peer_keys { + assert!(self.peers.contains_key(key), "{peer_id:#} maps to an unknown key"); } - Poll::Pending + let inflight = self.peers.values().map(|peer| peer.inflight as usize).sum::(); + assert!(inflight <= self.inflight.len(), "peers claim more inflight requests than exist"); } } -impl Default for TransactionFetcher { +impl Default for TransactionFetcher { fn default() -> Self { - Self { - active_peers: LruMap::with_hasher( - DEFAULT_MAX_COUNT_CONCURRENT_REQUESTS, - Default::default(), - ), - inflight_requests: Default::default(), - hashes_pending_fetch: LruCache::with_hasher( - DEFAULT_MAX_CAPACITY_CACHE_PENDING_FETCH, - Default::default(), - ), - hashes_fetch_inflight_and_pending_fetch: LruMap::with_hasher( - DEFAULT_MAX_CAPACITY_CACHE_INFLIGHT_AND_PENDING_FETCH, - Default::default(), - ), - info: TransactionFetcherInfo::default(), - metrics: Default::default(), - } + Self::new(TransactionFetcherConfig::default()) } } -/// Metadata of a transaction hash that is yet to be fetched. -#[derive(Debug, Constructor)] -pub struct TxFetchMetadata { - /// The number of times a request attempt has been made for the hash. - retries: u8, - /// Peers that have announced the hash, but to which a request attempt has not yet been made. - fallback_peers: LruCache>, - /// Size metadata of the transaction if it has been seen in an eth68 announcement. - // todo: store all seen sizes as a `(size, peer_id)` tuple to catch peers that respond with - // another size tx than they announced. alt enter in request (won't catch peers announcing - // wrong size for requests assembled from hashes pending fetch if stored in request fut) - tx_encoded_length: Option, -} - -impl TxFetchMetadata { - /// Returns a mutable reference to the fallback peers cache for this transaction hash. - pub const fn fallback_peers_mut(&mut self) -> &mut LruCache> { - &mut self.fallback_peers - } +impl Stream for TransactionFetcher { + type Item = FetchEvent; - /// Returns the size of the transaction, if its hash has been received in any - /// [`Eth68`](reth_eth_wire::EthVersion::Eth68) announcement. If the transaction hash has only - /// been seen in [`Eth66`](reth_eth_wire::EthVersion::Eth66) announcements so far, this will - /// return `None`. - pub const fn tx_encoded_len(&self) -> Option { - self.tx_encoded_length + /// Advances all inflight requests and yields the next resolved request as an event. + /// + /// Never terminates, returns [`Poll::Pending`] while no request is inflight. + fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + // `FuturesUnordered` yields `None` while empty but keeps working once new requests are + // pushed, so this is mapped to pending + match self.inflight.poll_next_unpin(cx) { + Poll::Ready(Some(resolved)) => Poll::Ready(Some(self.on_resolved(resolved))), + Poll::Ready(None) | Poll::Pending => Poll::Pending, + } } } @@ -1056,6 +1152,10 @@ pub enum FetchEvent { TransactionsFetched { /// The ID of the peer from which transactions were fetched. peer_id: PeerId, + /// The negotiated protocol of the session that served the response. + version: EthVersion, + /// The client version of the session that served the response. + client_version: Arc, /// The transactions that were fetched, if available. transactions: PooledTransactions, /// Whether the peer should be penalized for sending unsolicited transactions or for @@ -1076,525 +1176,1907 @@ pub enum FetchEvent { }, } -/// An inflight request for [`PooledTransactions`] from a peer. -#[derive(Debug)] -pub struct GetPooledTxRequest { - peer_id: PeerId, - /// Transaction hashes that were requested, for cleanup purposes - requested_hashes: RequestTxHashes, - response: oneshot::Receiver>>, -} +/// Compact identifier of a peer within the fetcher. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +struct PeerKey(u32); -/// Upon reception of a response, a [`GetPooledTxRequest`] is deconstructed to form a -/// [`GetPooledTxResponse`]. +/// Cached eviction work for one announcement. Other peers can only lose candidates; newly +/// tracked hashes from this announcer are handled by the announcement's scratch cursor. #[derive(Debug)] -pub struct GetPooledTxResponse { - peer_id: PeerId, - /// Transaction hashes that were requested, for cleanup purposes, since peer may only return a - /// subset of requested hashes. - requested_hashes: RequestTxHashes, - result: Result>, RecvError>, +struct EvictionState { + peers: BinaryHeap<(usize, PeerKey)>, + exclusive: HashMap>, } -/// Stores the response receiver made by sending a [`GetPooledTransactions`] request to a peer's -/// session. -#[must_use = "futures do nothing unless polled"] -#[pin_project::pin_project] -#[derive(Debug)] -pub struct GetPooledTxRequestFut { - #[pin] - inner: Option>, +/// Position for newly queued hashes or ready peers. +#[derive(Debug, Clone, Copy)] +enum QueuePosition { + Front, + Back, } -impl GetPooledTxRequestFut { - #[inline] - const fn new( - peer_id: PeerId, - requested_hashes: RequestTxHashes, - response: oneshot::Receiver>>, - ) -> Self { - Self { inner: Some(GetPooledTxRequest { peer_id, requested_hashes, response }) } - } +/// Batches consecutive removals from the same candidate peer during an announcement. +#[derive(Debug, Default)] +struct DeferredCandidateRemovals { + pending: Option<(PeerKey, usize)>, } -impl Future for GetPooledTxRequestFut { - type Output = GetPooledTxResponse; - - fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { - let mut req = self.as_mut().project().inner.take().expect("polled after completion"); - match req.response.poll_unpin(cx) { - Poll::Ready(result) => Poll::Ready(GetPooledTxResponse { - peer_id: req.peer_id, - requested_hashes: req.requested_hashes, - result, - }), - Poll::Pending => { - self.project().inner.set(Some(req)); - Poll::Pending - } +impl DeferredCandidateRemovals { + fn record_removed(&mut self, key: PeerKey, peers: &mut HashMap) { + if let Some((pending_key, count)) = &mut self.pending && + *pending_key == key + { + *count += 1; + } else { + self.flush(peers); + self.pending = Some((key, 1)); } } -} -/// Wrapper of unverified [`PooledTransactions`]. -#[derive(Debug, Constructor, Deref)] -pub struct UnverifiedPooledTransactions { - txns: PooledTransactions, + /// Synchronizes tracked counts before capacity eviction or the end of an announcement. + fn flush(&mut self, peers: &mut HashMap) { + if let Some((key, count)) = self.pending.take() { + peers.get_mut(&key).expect("candidate peer is connected").tracked -= count; + } + } } -/// [`PooledTransactions`] that have been successfully verified. -#[derive(Debug, Constructor, Deref)] -pub struct VerifiedPooledTransactions { - txns: PooledTransactions, +/// State of a tracked hash. +#[derive(Debug)] +struct TxEntry { + /// Peers that announced the hash and may be asked for it, oldest first. + candidates: SmallVec<[Candidate; MAX_COUNT_CANDIDATE_PEERS_PER_HASH]>, + /// Identity of this tracking lifetime, also stored in lazy queue entries. + generation: u64, + /// The peer and request currently fetching the transaction, if any. + fetching_by: Option<(PeerKey, u64)>, + /// Sent requests for this entry, bounded independently of candidate replacement. + attempts: usize, } -impl DedupPayload for VerifiedPooledTransactions { - type Value = T; +impl TxEntry { + /// A new entry for a hash that is queued for the announcing peer. + fn new(peer: PeerKey, size: u32, generation: u64) -> Self { + let mut candidates = SmallVec::new(); + candidates.push(Candidate::queued(peer, size)); + Self { candidates, generation, fetching_by: None, attempts: 0 } + } - fn is_empty(&self) -> bool { - self.txns.is_empty() + fn has_candidate(&self, key: PeerKey) -> bool { + self.candidates.iter().any(|candidate| candidate.peer == key) } - fn len(&self) -> usize { - self.txns.len() + /// Returns the candidates that don't have the hash queued, in the order they announced it. + fn unqueued_candidates(&self) -> SmallVec<[PeerKey; MAX_COUNT_CANDIDATE_PEERS_PER_HASH]> { + self.candidates + .iter() + .filter(|candidate| !candidate.is_queued()) + .map(|candidate| candidate.peer) + .collect() } - fn dedup(self) -> PartiallyValidData { - PartiallyValidData::from_raw_data( - self.txns.into_iter().map(|tx| (*tx.tx_hash(), tx)).collect(), - None, - ) + fn candidate_mut(&mut self, key: PeerKey) -> Option<&mut Candidate> { + self.candidates.iter_mut().find(|candidate| candidate.peer == key) } } -trait VerifyPooledTransactionsResponse { - type Transaction: SignedTransaction; - - fn verify( - self, - requested_hashes: &RequestTxHashes, - peer_id: &PeerId, - ) -> (VerificationOutcome, VerifiedPooledTransactions); +/// A peer that announced a hash. +#[derive(Debug, Clone, Copy)] +struct Candidate { + peer: PeerKey, + /// The size the peer announced for the transaction, 0 if unknown, with + /// [`Self::QUEUED`] set while the hash is in the peer's queue. + size_and_queued: u32, } -impl VerifyPooledTransactionsResponse for UnverifiedPooledTransactions { - type Transaction = T; +impl Candidate { + /// Marks the hash as queued for the peer. Announced sizes are capped well below this bit. + const QUEUED: u32 = 1 << 31; - fn verify( - self, - requested_hashes: &RequestTxHashes, - _peer_id: &PeerId, - ) -> (VerificationOutcome, VerifiedPooledTransactions) { - let mut verification_outcome = VerificationOutcome::Ok; + /// A candidate that has the hash queued. + const fn queued(peer: PeerKey, size: u32) -> Self { + Self { peer, size_and_queued: size | Self::QUEUED } + } - let Self { mut txns } = self; + /// A candidate that doesn't have the hash queued. + const fn unqueued(peer: PeerKey, size: u32) -> Self { + Self { peer, size_and_queued: size } + } - #[cfg(debug_assertions)] - let mut tx_hashes_not_requested: smallvec::SmallVec<[TxHash; 16]> = smallvec::smallvec!(); - #[cfg(not(debug_assertions))] - let mut tx_hashes_not_requested_count = 0; + const fn is_queued(&self) -> bool { + self.size_and_queued & Self::QUEUED != 0 + } - txns.0.retain(|tx| { - if !requested_hashes.contains(tx.tx_hash()) { - verification_outcome = VerificationOutcome::ReportPeer; + const fn set_queued(&mut self, queued: bool) { + if queued { + self.size_and_queued |= Self::QUEUED; + } else { + self.size_and_queued &= !Self::QUEUED; + } + } - #[cfg(debug_assertions)] - tx_hashes_not_requested.push(*tx.tx_hash()); - #[cfg(not(debug_assertions))] - { - tx_hashes_not_requested_count += 1; - } + const fn set_size(&mut self, size: u32) { + self.size_and_queued = (self.size_and_queued & Self::QUEUED) | size; + } - return false - } - true - }); + /// Returns the size to account for when packing the hash into a request to this peer. + const fn request_size(&self) -> usize { + let size = self.size_and_queued & !Self::QUEUED; + if size == 0 { + AVERAGE_BYTE_SIZE_TX_ENCODED + } else { + size as usize + } + } +} - #[cfg(debug_assertions)] - if !tx_hashes_not_requested.is_empty() { - trace!(target: "net::tx", - peer_id=format!("{_peer_id:#}"), - ?tx_hashes_not_requested, - "transactions in `PooledTransactions` response from peer were not requested" - ); +/// Fetch state of a peer. +#[derive(Debug)] +struct PeerState { + peer_id: PeerId, + /// Hashes the peer announced, oldest first. Entries are removed lazily, so the queue may + /// contain hashes that are not tracked anymore or that the peer is no longer a candidate for. + queue: VecDeque<(TxHash, u64)>, + /// Number of tracked hashes that list this peer as a candidate. + tracked: usize, + /// Number of inflight requests to this peer. + inflight: u8, + /// Whether the peer is queued in the ready list. + ready: bool, +} + +impl PeerState { + const fn new(peer_id: PeerId) -> Self { + Self { peer_id, queue: VecDeque::new(), tracked: 0, inflight: 0, ready: false } + } + + /// Queues the hash at the back or the front of the queue. + /// + /// Queues are bounded: once a queue holds twice `max_tracked` hashes, entries the peer is no + /// longer a candidate for and duplicates are removed. A duplicate can occur when a hash is + /// tracked again after it was delivered or evicted, while its old entry still lingers in the + /// queue. Generations prevent those stale entries from selecting a newer tracking lifetime. + fn push_queue( + &mut self, + hashes: &B256Map, + key: PeerKey, + queued: (TxHash, u64), + position: QueuePosition, + max_tracked: usize, + ) { + if self.queue.len() >= 2 * max_tracked { + let mut seen = B256Set::with_capacity_and_hasher(self.tracked, Default::default()); + self.queue.retain(|(hash, generation)| { + hashes.get(hash).is_some_and(|entry| { + entry.generation == *generation && entry.has_candidate(key) + }) && seen.insert(*hash) + }); } - #[cfg(not(debug_assertions))] - if tx_hashes_not_requested_count != 0 { - trace!(target: "net::tx", - peer_id=format!("{_peer_id:#}"), - tx_hashes_not_requested_count, - "transactions in `PooledTransactions` response from peer were not requested" - ); + match position { + QueuePosition::Front => self.queue.push_front(queued), + QueuePosition::Back => self.queue.push_back(queued), } - - (verification_outcome, VerifiedPooledTransactions::new(txns)) } } -/// Outcome from verifying a [`PooledTransactions`] response. Signals to caller whether to penalize -/// the sender of the response or not. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum VerificationOutcome { - /// Peer behaves appropriately. - Ok, - /// A penalty should be flagged for the peer. Peer sent a response with unacceptably - /// invalid entries. - ReportPeer, +/// An inflight `GetPooledTransactions` request. +#[derive(Debug)] +struct InflightRequest { + peer: PeerKey, + request_id: u64, + peer_id: PeerId, + version: EthVersion, + client_version: Arc, + /// The requested hashes, in request order. + hashes: Vec, + response: oneshot::Receiver>>, } -/// Tracks stats about the [`TransactionFetcher`]. -#[derive(Debug, Constructor)] -pub struct TransactionFetcherInfo { - /// Max inflight [`GetPooledTransactions`] requests. - pub max_inflight_requests: usize, - /// Max inflight [`GetPooledTransactions`] requests per peer. - pub max_inflight_requests_per_peer: u8, - /// Soft limit for the byte size of the expected [`PooledTransactions`] response, upon packing - /// a [`GetPooledTransactions`] request with hashes (by default less than 2 MiB worth of - /// transactions is requested). - pub soft_limit_byte_size_pooled_transactions_response_on_pack_request: usize, - /// Soft limit for the byte size of a [`PooledTransactions`] response, upon assembling the - /// response. Spec'd at 2 MiB, but can be adjusted for research purpose. - pub soft_limit_byte_size_pooled_transactions_response: usize, - /// Max capacity of the cache of transaction hashes, for transactions that weren't yet fetched. - /// A transaction is pending fetch if its hash didn't fit into a [`GetPooledTransactions`] yet, - /// or it wasn't returned upon request to peers. - pub max_capacity_cache_txns_pending_fetch: u32, +impl Future for InflightRequest { + type Output = ResolvedRequest; + + fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + let this = self.get_mut(); + let result = + ready!(this.response.poll_unpin(cx)).unwrap_or(Err(RequestError::ChannelClosed)); + Poll::Ready(ResolvedRequest { + peer: this.peer, + request_id: this.request_id, + peer_id: this.peer_id, + version: this.version, + client_version: this.client_version.clone(), + hashes: std::mem::take(&mut this.hashes), + result, + }) + } } -impl Default for TransactionFetcherInfo { - fn default() -> Self { - Self::new( - DEFAULT_MAX_COUNT_CONCURRENT_REQUESTS as usize, - DEFAULT_MAX_COUNT_CONCURRENT_REQUESTS_PER_PEER, - DEFAULT_SOFT_LIMIT_BYTE_SIZE_POOLED_TRANSACTIONS_RESP_ON_PACK_GET_POOLED_TRANSACTIONS_REQ, - SOFT_LIMIT_BYTE_SIZE_POOLED_TRANSACTIONS_RESPONSE, - DEFAULT_MAX_CAPACITY_CACHE_PENDING_FETCH, - ) - } +/// A resolved `GetPooledTransactions` request. +#[derive(Debug)] +struct ResolvedRequest { + peer: PeerKey, + request_id: u64, + peer_id: PeerId, + version: EthVersion, + client_version: Arc, + hashes: Vec, + result: RequestResult>, } -impl From for TransactionFetcherInfo { - fn from(config: TransactionFetcherConfig) -> Self { - let TransactionFetcherConfig { - max_inflight_requests, - max_inflight_requests_per_peer, - soft_limit_byte_size_pooled_transactions_response, - soft_limit_byte_size_pooled_transactions_response_on_pack_request, - max_capacity_cache_txns_pending_fetch, - } = config; - - Self::new( - max_inflight_requests as usize, - max_inflight_requests_per_peer, - soft_limit_byte_size_pooled_transactions_response_on_pack_request, - soft_limit_byte_size_pooled_transactions_response, - max_capacity_cache_txns_pending_fetch, - ) - } +/// Returns the announced transaction size used for request packing, capped at the response soft +/// limit, or 0 if the announcement carried no size. +fn announced_size(metadata: Eth68TxMetadata) -> u32 { + metadata + .map_or(0, |(_, size)| size.min(SOFT_LIMIT_BYTE_SIZE_POOLED_TRANSACTIONS_RESPONSE) as u32) } -#[derive(Debug, Default)] -struct TxFetcherSearchDurations { - find_idle_peer: Duration, - fill_request: Duration, +/// Filters a response down to the transactions that were requested, dropping duplicates. +/// +/// Records the delivered hashes in `delivered` and returns the number of unsolicited transactions +/// that were dropped. +fn verify_response( + transactions: &mut PooledTransactions, + requested: &B256Set, + delivered: &mut B256Set, +) -> usize { + let mut unsolicited = 0; + transactions.0.retain(|tx| { + let hash = *tx.tx_hash(); + if !requested.contains(&hash) { + unsolicited += 1; + return false + } + delivered.insert(hash) + }); + unsolicited } #[cfg(test)] -mod test { +mod tests { use super::*; - use crate::test_utils::transactions::{buffer_hash_to_tx_fetcher, new_mock_session}; - use alloy_primitives::{ - hex, - map::{B256Map, B256Set, HashMap}, - B256, + use crate::test_utils::transactions::new_mock_session_with_capacity; + use alloy_consensus::transaction::Recovered; + use alloy_primitives::B256; + use futures::task::{noop_waker_ref, waker, ArcWake}; + use rand::{rngs::StdRng, seq::IndexedRandom, Rng, SeedableRng}; + use reth_eth_wire::EthVersion; + use reth_ethereum_primitives::{PooledTransactionVariant, TransactionSigned}; + use reth_transaction_pool::test_utils::MockTransactionFactory; + use std::sync::{ + atomic::{AtomicUsize, Ordering}, + Arc, }; - use alloy_rlp::Decodable; - use derive_more::IntoIterator; - use reth_eth_wire_types::EthVersion; - use reth_ethereum_primitives::TransactionSigned; - use std::str::FromStr; + use tokio::sync::mpsc; - #[derive(IntoIterator)] - struct TestValidAnnouncementData(Vec<(TxHash, Option<(u8, usize)>)>); + type Fetcher = TransactionFetcher; + type ResponseSender = + oneshot::Sender>>; - impl HandleMempoolData for TestValidAnnouncementData { - fn is_empty(&self) -> bool { - self.0.is_empty() - } + const KIB: usize = 1024; + + fn peer(n: u8) -> PeerId { + PeerId::new([n; 64]) + } + + fn hash(n: u64) -> TxHash { + let mut bytes = [0u8; 32]; + bytes[24..].copy_from_slice(&n.to_be_bytes()); + B256::from(bytes) + } + + fn hashes(range: std::ops::Range) -> Vec { + range.map(hash).collect() + } - fn len(&self) -> usize { - self.0.len() + /// Counts how often a task is woken. + #[derive(Default)] + struct WakeCounter(AtomicUsize); + + impl ArcWake for WakeCounter { + fn wake_by_ref(arc_self: &Arc) { + arc_self.0.fetch_add(1, Ordering::SeqCst); } + } - fn retain_by_hash(&mut self, mut f: impl FnMut(&TxHash) -> bool) { - self.0.retain(|(hash, _)| f(hash)) + impl WakeCounter { + fn wakes(&self) -> usize { + self.0.load(Ordering::SeqCst) } } - impl HandleVersionedMempoolData for TestValidAnnouncementData { - fn msg_version(&self) -> EthVersion { - EthVersion::Eth68 + fn pooled_txs(count: usize) -> Vec { + let mut factory = MockTransactionFactory::default(); + (0..count) + .map(|_| { + let recovered: Recovered = + factory.create_eip1559().transaction.into(); + PooledTransactionVariant::try_from(recovered.into_inner()) + .expect("eip1559 transaction converts to pooled transaction") + }) + .collect() + } + + fn hashes_of(txs: &[PooledTransactionVariant]) -> Vec { + txs.iter().map(|tx| *tx.tx_hash()).collect() + } + + /// A fetcher with mock peer sessions. + struct Rig { + fetcher: Fetcher, + peers: HashMap, FbBuildHasher<64>>, + sessions: HashMap, FbBuildHasher<64>>, + } + + impl Rig { + fn new() -> Self { + Self::with_config(TransactionFetcherConfig::default()) + } + + fn with_config(config: TransactionFetcherConfig) -> Self { + Self { + fetcher: Fetcher::new(config), + peers: Default::default(), + sessions: Default::default(), + } + } + + fn verify(&self) { + self.fetcher.assert_invariants(); + } + + fn add_peer(&mut self, peer_id: PeerId) { + self.add_peer_with_capacity(peer_id, 8); + } + + fn add_peer_with_capacity(&mut self, peer_id: PeerId, capacity: usize) { + let (peer, rx) = new_mock_session_with_capacity(peer_id, EthVersion::Eth68, capacity); + self.peers.insert(peer_id, peer); + self.sessions.insert(peer_id, rx); + } + + fn announce(&mut self, peer_id: PeerId, hashes: &[TxHash]) { + self.announce_with_sizes(peer_id, hashes.iter().map(|hash| (*hash, 512))); + } + + fn announce_with_sizes( + &mut self, + peer_id: PeerId, + entries: impl IntoIterator, + ) { + self.fetcher.on_announcement( + peer_id, + entries.into_iter().map(|(h, size)| (h, Some((2, size)))), + ); + self.verify(); + } + + fn announce_unsized(&mut self, peer_id: PeerId, hashes: &[TxHash]) { + self.fetcher.on_announcement(peer_id, hashes.iter().map(|hash| (*hash, None))); + self.verify(); + } + + fn dispatch(&mut self) -> usize { + self.dispatch_with_budget(usize::MAX) + } + + fn dispatch_with_budget(&mut self, max_hashes_per_request: usize) -> usize { + let sent = self.fetcher.dispatch(&self.peers, max_hashes_per_request); + self.verify(); + sent + } + + /// Takes the next request the peer's session received. + fn take_request(&mut self, peer_id: PeerId) -> Option<(Vec, ResponseSender)> { + match self.sessions.get_mut(&peer_id)?.try_recv().ok()? { + PeerRequest::GetPooledTransactions { request, response } => { + Some((request.0, response)) + } + _ => unreachable!("the fetcher only sends `GetPooledTransactions` requests"), + } + } + + fn next_event(&mut self) -> Option> { + let mut cx = Context::from_waker(noop_waker_ref()); + let event = match self.fetcher.poll_next_unpin(&mut cx) { + Poll::Ready(event) => event, + Poll::Pending => None, + }; + self.verify(); + event + } + + fn drain_events(&mut self) -> Vec> { + std::iter::from_fn(|| self.next_event()).collect() + } + + fn respond( + &mut self, + peer_id: PeerId, + txs: Vec, + ) -> FetchEvent { + let (_, response) = self.take_request(peer_id).expect("request inflight"); + response.send(Ok(PooledTransactions(txs))).unwrap(); + self.next_event().expect("response yields an event") + } + + fn fail( + &mut self, + peer_id: PeerId, + error: RequestError, + ) -> FetchEvent { + let (_, response) = self.take_request(peer_id).expect("request inflight"); + response.send(Err(error)).unwrap(); + self.next_event().expect("error yields an event") + } + + fn disconnect(&mut self, peer_id: PeerId) { + self.peers.remove(&peer_id); + self.sessions.remove(&peer_id); + self.fetcher.on_peer_disconnected(&peer_id); + self.verify(); } } #[test] - fn pack_eth68_request() { - reth_tracing::init_test_tracing(); - - // RIG TEST - - let tx_fetcher = &mut TransactionFetcher::::default(); - - let eth68_hashes = [ - B256::from_slice(&[1; 32]), - B256::from_slice(&[2; 32]), - B256::from_slice(&[3; 32]), - B256::from_slice(&[4; 32]), - B256::from_slice(&[5; 32]), - ]; - let eth68_sizes = [ - DEFAULT_SOFT_LIMIT_BYTE_SIZE_POOLED_TRANSACTIONS_RESP_ON_PACK_GET_POOLED_TRANSACTIONS_REQ - MEDIAN_BYTE_SIZE_SMALL_LEGACY_TX_ENCODED - 1, // first will fit - DEFAULT_SOFT_LIMIT_BYTE_SIZE_POOLED_TRANSACTIONS_RESP_ON_PACK_GET_POOLED_TRANSACTIONS_REQ, // second won't - 2, // free space > `MEDIAN_BYTE_SIZE_SMALL_LEGACY_TX_ENCODED`, third will fit, no more after this - 9, - 0, - ]; - - let expected_request_hashes = - [eth68_hashes[0], eth68_hashes[2]].into_iter().collect::(); - - let expected_surplus_hashes = - [eth68_hashes[1], eth68_hashes[3], eth68_hashes[4]].into_iter().collect::(); - - let mut eth68_hashes_to_request = RequestTxHashes::with_capacity(3); - - let valid_announcement_data = TestValidAnnouncementData( - eth68_hashes - .into_iter() - .zip(eth68_sizes) - .map(|(hash, size)| (hash, Some((0u8, size)))) - .collect::>(), + fn announced_hashes_are_requested_from_announcing_peer() { + let mut rig = Rig::new(); + let peer_a = peer(1); + rig.add_peer(peer_a); + let mut txs = pooled_txs(2); + txs.sort_unstable_by_key(|tx| std::cmp::Reverse(*tx.tx_hash())); + let hashes = hashes_of(&txs); + + let mut cx = Context::from_waker(noop_waker_ref()); + assert!(rig.fetcher.poll_next_unpin(&mut cx).is_pending()); + rig.announce(peer_a, &hashes[..1]); + rig.announce(peer_a, &hashes[1..]); + rig.announce(peer_a, &hashes); + assert_eq!(rig.fetcher.queued_hashes(&peer_a), hashes); + assert_eq!(rig.fetcher.candidate_peers(&hashes[0]), vec![peer_a]); + assert_eq!(rig.fetcher.num_pending_hashes(), 2); + assert!(rig.fetcher.is_idle(&peer_a)); + + assert_eq!(rig.dispatch(), 1); + assert!(!rig.fetcher.is_idle(&peer_a)); + assert_eq!(rig.fetcher.num_inflight_requests(), 1); + assert_eq!(rig.fetcher.num_fetching_hashes(), 2); + assert_eq!(rig.fetcher.num_pending_hashes(), 0); + assert_eq!(rig.fetcher.fetching_peer(&hashes[0]), Some(peer_a)); + + let (requested, response) = rig.take_request(peer_a).unwrap(); + assert_eq!(requested, hashes); + response.send(Ok(PooledTransactions(txs))).unwrap(); + + let FetchEvent::TransactionsFetched { peer_id, transactions, report_peer, .. } = + rig.next_event().unwrap() + else { + panic!("expected fetched transactions") + }; + assert_eq!(peer_id, peer_a); + assert_eq!(transactions.len(), 2); + assert!(!report_peer); + + assert_eq!(rig.fetcher.num_hashes(), 0); + assert_eq!(rig.fetcher.num_inflight_requests(), 0); + assert!(rig.fetcher.is_idle(&peer_a)); + assert!(rig.fetcher.poll_next_unpin(&mut cx).is_pending()); + } + + #[test] + fn request_is_capped_by_hash_count() { + let mut rig = Rig::new(); + let peer_a = peer(1); + rig.add_peer(peer_a); + let hashes = hashes(0..300); + + // no size metadata, so the count limit is what bounds the request + rig.announce_unsized(peer_a, &hashes); + assert_eq!(rig.dispatch(), 1); + + let (requested, response) = rig.take_request(peer_a).unwrap(); + assert_eq!(requested.len(), SOFT_LIMIT_COUNT_HASHES_IN_GET_POOLED_TRANSACTIONS_REQUEST); + assert_eq!(requested, hashes[..256]); + assert_eq!(rig.fetcher.num_pending_hashes(), 44); + + // the peer is busy until the request resolves + assert_eq!(rig.dispatch(), 0); + response.send(Err(RequestError::BadResponse)).unwrap(); + rig.next_event().unwrap(); + assert_eq!(rig.fetcher.num_pending_hashes(), 44); + + assert_eq!(rig.dispatch(), 1); + let (requested, _) = rig.take_request(peer_a).unwrap(); + assert_eq!(requested, hashes[256..]); + } + + #[test] + fn request_is_capped_by_expected_response_size() { + let mut rig = Rig::new(); + let peer_a = peer(1); + rig.add_peer(peer_a); + let hashes = hashes(0..3); + + rig.announce_with_sizes( + peer_a, + [(hashes[0], 100 * KIB), (hashes[1], 100 * KIB), (hashes[2], 100)], ); - // TEST + rig.dispatch(); + let (requested, response) = rig.take_request(peer_a).unwrap(); + assert_eq!(requested, hashes[..1], "second transaction doesn't fit in 128 KiB"); - let surplus_eth68_hashes = - tx_fetcher.pack_request_eth68(&mut eth68_hashes_to_request, valid_announcement_data); + response.send(Ok(PooledTransactions(vec![]))).unwrap(); + rig.next_event().unwrap(); + // the empty response dropped the peer as candidate for the first hash + assert_eq!(rig.fetcher.num_hashes(), 2); - let eth68_hashes_to_request = eth68_hashes_to_request.into_iter().collect::(); - let surplus_eth68_hashes = surplus_eth68_hashes.into_iter().collect::(); + rig.dispatch(); + let (requested, _) = rig.take_request(peer_a).unwrap(); + assert_eq!(requested, hashes[1..]); + } - assert_eq!(expected_request_hashes, eth68_hashes_to_request); - assert_eq!(expected_surplus_hashes, surplus_eth68_hashes); + #[test] + fn oversized_transaction_is_requested_alone() { + let mut rig = Rig::new(); + let peer_a = peer(1); + rig.add_peer(peer_a); + let hashes = hashes(0..3); + + rig.announce_with_sizes( + peer_a, + [(hashes[0], 100), (hashes[1], 1024 * KIB), (hashes[2], 100)], + ); + + let mut requests = Vec::new(); + for _ in 0..3 { + assert_eq!(rig.dispatch(), 1); + let (requested, response) = rig.take_request(peer_a).unwrap(); + requests.push(requested); + // A bad response keeps other hashes pending but drops the requested ones. + response.send(Err(RequestError::BadResponse)).unwrap(); + rig.next_event().unwrap(); + } + assert_eq!( + requests, + vec![hashes[..1].to_vec(), hashes[1..2].to_vec(), hashes[2..].to_vec()] + ); + assert_eq!(rig.fetcher.num_hashes(), 0); } #[test] - fn pack_eth68_request_does_not_overflow_announced_size() { - reth_tracing::init_test_tracing(); + fn announced_size_is_per_peer() { + let mut rig = Rig::new(); + let peer_a = peer(1); + let peer_b = peer(2); + rig.add_peer(peer_a); + rig.add_peer(peer_b); + let hashes = hashes(0..2); + + // peer_a doesn't know the sizes, peer_b announces a huge first transaction + rig.announce_unsized(peer_a, &hashes); + rig.announce_with_sizes(peer_b, [(hashes[0], 1024 * KIB), (hashes[1], 100)]); + + // peer_a's request is packed with the size estimate + rig.dispatch(); + let (requested, response) = rig.take_request(peer_a).unwrap(); + assert_eq!(requested, hashes); + response.send(Ok(PooledTransactions(vec![]))).unwrap(); + rig.next_event().unwrap(); + + // peer_b's request honors the size it announced + rig.dispatch(); + let (requested, _) = rig.take_request(peer_b).unwrap(); + assert_eq!(requested, hashes[..1]); + } - let tx_fetcher = &mut TransactionFetcher::::default(); + #[test] + fn announced_sizes_are_capped() { + let mut rig = Rig::new(); + let peer_a = peer(1); + rig.add_peer(peer_a); + let hashes = hashes(0..3); + + // absurd sizes must not break the size accounting + rig.announce_with_sizes( + peer_a, + [(hashes[0], usize::MAX - 1), (hashes[1], usize::MAX), (hashes[2], 100)], + ); - let eth68_hashes = - [B256::from_slice(&[1; 32]), B256::from_slice(&[2; 32]), B256::from_slice(&[3; 32])]; - let eth68_sizes = [ - DEFAULT_SOFT_LIMIT_BYTE_SIZE_POOLED_TRANSACTIONS_RESP_ON_PACK_GET_POOLED_TRANSACTIONS_REQ - MEDIAN_BYTE_SIZE_SMALL_LEGACY_TX_ENCODED - 1, - usize::MAX, - 2, - ]; + rig.dispatch(); + let (requested, response) = rig.take_request(peer_a).unwrap(); + assert_eq!(requested, hashes[..1], "an oversized transaction is requested alone"); + response.send(Err(RequestError::BadResponse)).unwrap(); + rig.next_event().unwrap(); - let expected_request_hashes = - [eth68_hashes[0], eth68_hashes[2]].into_iter().collect::(); - let expected_surplus_hashes = std::iter::once(eth68_hashes[1]).collect::(); + rig.dispatch(); + let (requested, _) = rig.take_request(peer_a).unwrap(); + assert_eq!(requested, hashes[1..2]); + } - let mut eth68_hashes_to_request = RequestTxHashes::with_capacity(3); - let valid_announcement_data = TestValidAnnouncementData( - eth68_hashes - .into_iter() - .zip(eth68_sizes) - .map(|(hash, size)| (hash, Some((0u8, size)))) - .collect::>(), + #[test] + fn hash_is_fetched_from_one_peer_at_a_time() { + let mut rig = Rig::new(); + let peer_a = peer(1); + let peer_b = peer(2); + rig.add_peer(peer_a); + rig.add_peer(peer_b); + let txs = pooled_txs(1); + let hashes = hashes_of(&txs); + + rig.announce(peer_a, &hashes); + rig.announce(peer_b, &hashes); + assert_eq!(rig.fetcher.candidate_peers(&hashes[0]), vec![peer_a, peer_b]); + + assert_eq!(rig.dispatch(), 1); + assert_eq!(rig.fetcher.fetching_peer(&hashes[0]), Some(peer_a)); + assert!(rig.take_request(peer_b).is_none()); + // peer_b's queue entry was consumed without a request + assert!(rig.fetcher.queued_hashes(&peer_b).is_empty()); + assert_eq!(rig.fetcher.candidate_peers(&hashes[0]), vec![peer_a, peer_b]); + + rig.respond(peer_a, txs); + assert_eq!(rig.fetcher.num_hashes(), 0); + assert_eq!(rig.dispatch(), 0); + } + + #[test] + fn failed_request_is_retried_with_alternate_peer() { + let mut rig = Rig::new(); + let peer_a = peer(1); + let peer_b = peer(2); + rig.add_peer(peer_a); + rig.add_peer(peer_b); + let txs = pooled_txs(2); + let hashes = hashes_of(&txs); + + rig.announce(peer_a, &hashes); + rig.announce(peer_b, &hashes); + rig.dispatch(); + // peer_b skipped the hashes while they were inflight to peer_a + assert!(rig.fetcher.queued_hashes(&peer_b).is_empty()); + + let FetchEvent::FetchError { peer_id, error } = rig.fail(peer_a, RequestError::Timeout) + else { + panic!("expected fetch error") + }; + assert_eq!(peer_id, peer_a); + assert!(matches!(error, RequestError::Timeout)); + + // the hashes are pending again for peer_b only and queued in request order + assert_eq!(rig.fetcher.num_pending_hashes(), 2); + assert_eq!(rig.fetcher.candidate_peers(&hashes[0]), vec![peer_b]); + assert_eq!(rig.fetcher.queued_hashes(&peer_b), hashes); + + assert_eq!(rig.dispatch(), 1); + assert!(rig.take_request(peer_a).is_none()); + let event = rig.respond(peer_b, txs); + assert!( + matches!(event, FetchEvent::TransactionsFetched { peer_id, .. } if peer_id == peer_b) ); + assert_eq!(rig.fetcher.num_hashes(), 0); + } - let surplus_eth68_hashes = - tx_fetcher.pack_request_eth68(&mut eth68_hashes_to_request, valid_announcement_data); + #[test] + fn empty_response_drops_peer_as_candidate() { + let mut rig = Rig::new(); + let peer_a = peer(1); + rig.add_peer(peer_a); + let hashes = hashes(0..2); + + rig.announce(peer_a, &hashes); + rig.dispatch(); + let event = rig.respond(peer_a, vec![]); + assert!(matches!(event, FetchEvent::EmptyResponse { peer_id } if peer_id == peer_a)); + + // no other peer announced the hashes, so they are dropped + assert_eq!(rig.fetcher.num_hashes(), 0); + assert_eq!(rig.dispatch(), 0); + } + + #[test] + fn partial_response_keeps_peer_for_truncated_tail() { + let mut rig = Rig::new(); + let peer_a = peer(1); + rig.add_peer(peer_a); + let txs = pooled_txs(4); + let hashes = hashes_of(&txs); + + rig.announce(peer_a, &hashes); + rig.dispatch(); + + // the first half is delivered, the rest looks like truncation + rig.respond(peer_a, txs[..2].to_vec()); + assert_eq!(rig.fetcher.num_pending_hashes(), 2); + assert_eq!(rig.fetcher.candidate_peers(&hashes[2]), vec![peer_a]); + assert_eq!(rig.fetcher.queued_hashes(&peer_a), hashes[2..]); + + rig.dispatch(); + let (requested, _) = rig.take_request(peer_a).unwrap(); + assert_eq!(requested, hashes[2..]); + } + + #[test] + fn small_partial_response_drops_peer_for_all_undelivered_hashes() { + let mut rig = Rig::new(); + let peer_a = peer(1); + let peer_b = peer(2); + rig.add_peer(peer_a); + rig.add_peer(peer_b); + let txs = pooled_txs(10); + let hashes = hashes_of(&txs); + + rig.announce(peer_a, &hashes); + rig.announce(peer_b, &hashes[..5]); + rig.dispatch(); + assert_eq!(rig.fetcher.fetching_peer(&hashes[0]), Some(peer_a)); + + // Delivering fewer than half of the requested hashes drops peer_a for the rest, + // limiting repeated low-progress retries regardless of transaction sizes. + rig.respond(peer_a, txs[..1].to_vec()); + assert_eq!(rig.fetcher.num_hashes(), 4, "hashes without another candidate are dropped"); + for hash in &hashes[1..5] { + assert_eq!(rig.fetcher.candidate_peers(hash), vec![peer_b]); + } + assert!(rig.fetcher.queued_hashes(&peer_a).is_empty()); + } + + #[test] + fn skipped_hashes_without_alternates_are_dropped() { + let mut rig = Rig::new(); + let peer_a = peer(1); + rig.add_peer(peer_a); + let txs = pooled_txs(4); + let hashes = hashes_of(&txs); + + rig.announce(peer_a, &hashes); + rig.dispatch(); + rig.respond(peer_a, txs[1..3].to_vec()); + + // the first hash was skipped and has no other candidate, the last one is retried + assert_eq!(rig.fetcher.num_hashes(), 1); + assert_eq!(rig.fetcher.candidate_peers(&hashes[3]), vec![peer_a]); + assert!(rig.fetcher.candidate_peers(&hashes[0]).is_empty()); + } + + #[test] + fn unsolicited_and_duplicate_transactions_are_filtered_and_reported() { + let mut rig = Rig::new(); + let peer_a = peer(1); + rig.add_peer(peer_a); + let txs = pooled_txs(2); + let hashes = hashes_of(&txs); + + rig.announce(peer_a, &hashes[..1]); + rig.dispatch(); + + let FetchEvent::TransactionsFetched { transactions, report_peer, .. } = + rig.respond(peer_a, vec![txs[0].clone(), txs[1].clone(), txs[0].clone()]) + else { + panic!("expected fetched transactions") + }; + assert!(report_peer); + assert_eq!(transactions.0, txs[..1]); + assert_eq!(rig.fetcher.num_hashes(), 0); + } + + #[test] + fn response_with_only_unsolicited_transactions_is_a_bad_response() { + let mut rig = Rig::new(); + let peer_a = peer(1); + let peer_b = peer(2); + rig.add_peer(peer_a); + rig.add_peer(peer_b); + let txs = pooled_txs(2); + let hashes = hashes_of(&txs); + + rig.announce(peer_a, &hashes[..1]); + rig.announce(peer_b, &hashes[..1]); + rig.dispatch(); + + let event = rig.respond(peer_a, txs[1..].to_vec()); + assert!(matches!( + event, + FetchEvent::FetchError { peer_id, error: RequestError::BadResponse } if peer_id == peer_a + )); + // treated like a failed request, peer_b gets to try + assert_eq!(rig.fetcher.candidate_peers(&hashes[0]), vec![peer_b]); + rig.dispatch(); + assert!(rig.take_request(peer_b).is_some()); + } + + #[test] + fn duplicate_transactions_in_response_are_deduplicated() { + let mut rig = Rig::new(); + let peer_a = peer(1); + rig.add_peer(peer_a); + let txs = pooled_txs(1); + + rig.announce(peer_a, &hashes_of(&txs)); + rig.dispatch(); + + let FetchEvent::TransactionsFetched { transactions, report_peer, .. } = + rig.respond(peer_a, vec![txs[0].clone(), txs[0].clone()]) + else { + panic!("expected fetched transactions") + }; + assert_eq!(transactions.len(), 1); + assert!(!report_peer); + } - let eth68_hashes_to_request = eth68_hashes_to_request.into_iter().collect::(); - let surplus_eth68_hashes = surplus_eth68_hashes.into_iter().collect::(); + #[test] + fn received_transactions_stop_tracking() { + let mut rig = Rig::new(); + let peer_a = peer(1); + rig.add_peer(peer_a); + let txs = pooled_txs(3); + let hashes = hashes_of(&txs); + + rig.announce(peer_a, &hashes[..2]); + rig.dispatch(); + rig.announce(peer_a, &hashes[2..]); + assert_eq!(rig.fetcher.num_fetching_hashes(), 2); + assert_eq!(rig.fetcher.num_pending_hashes(), 1); + + // one inflight and the pending hash arrive over broadcast + rig.fetcher.on_transactions_received([&hashes[0], &hashes[2]]); + rig.fetcher.assert_invariants(); + assert_eq!(rig.fetcher.num_hashes(), 1); + assert_eq!(rig.fetcher.num_fetching_hashes(), 1); + + // the response doesn't include the hash received over broadcast, which must not be + // rescheduled, but includes the other one + let FetchEvent::TransactionsFetched { transactions, .. } = + rig.respond(peer_a, txs[1..2].to_vec()) + else { + panic!("expected fetched transactions") + }; + assert_eq!(transactions.len(), 1); + assert_eq!(rig.fetcher.num_hashes(), 0); + assert_eq!(rig.dispatch(), 0); + } - assert_eq!(expected_request_hashes, eth68_hashes_to_request); - assert_eq!(expected_surplus_hashes, surplus_eth68_hashes); + #[test] + fn response_delivering_hash_received_over_broadcast_is_still_returned() { + let mut rig = Rig::new(); + let peer_a = peer(1); + rig.add_peer(peer_a); + let txs = pooled_txs(1); + let hashes = hashes_of(&txs); + + rig.announce(peer_a, &hashes); + rig.dispatch(); + rig.fetcher.on_transactions_received(&hashes); + + let FetchEvent::TransactionsFetched { transactions, report_peer, .. } = + rig.respond(peer_a, txs) + else { + panic!("expected fetched transactions") + }; + assert_eq!(transactions.len(), 1); + assert!(!report_peer, "the transaction was requested, even if it arrived elsewhere first"); + assert_eq!(rig.fetcher.num_hashes(), 0); } #[test] - fn pack_eth72_request_uses_metadata_size_limit() { - reth_tracing::init_test_tracing(); - - let tx_fetcher = &mut TransactionFetcher::::default(); - - let hashes = - [B256::from_slice(&[1; 32]), B256::from_slice(&[2; 32]), B256::from_slice(&[3; 32])]; - let announced_size = - DEFAULT_SOFT_LIMIT_BYTE_SIZE_POOLED_TRANSACTIONS_RESP_ON_PACK_GET_POOLED_TRANSACTIONS_REQ; - let announcement_data = hashes - .into_iter() - .map(|hash| (hash, Some((0u8, announced_size)))) - .collect::>(); - let valid_announcement_data = ValidAnnouncementData::from_partially_valid_data( - PartiallyValidData::from_raw_data_eth72(announcement_data), + fn peer_disconnect_drops_hashes_without_alternates() { + let mut rig = Rig::new(); + let peer_a = peer(1); + let peer_b = peer(2); + rig.add_peer(peer_a); + rig.add_peer(peer_b); + let hashes = hashes(0..2); + + rig.announce(peer_a, &hashes); + rig.announce(peer_b, &hashes[1..]); + + rig.disconnect(peer_a); + assert_eq!(rig.fetcher.num_hashes(), 1); + assert_eq!(rig.fetcher.candidate_peers(&hashes[1]), vec![peer_b]); + assert!(rig.fetcher.is_idle(&peer_a)); + + rig.dispatch(); + let (requested, _) = rig.take_request(peer_b).unwrap(); + assert_eq!(requested, hashes[1..]); + } + + #[test] + fn inflight_request_of_disconnected_peer_is_rescheduled() { + let mut rig = Rig::new(); + let peer_a = peer(1); + let peer_b = peer(2); + rig.add_peer(peer_a); + rig.add_peer(peer_b); + let txs = pooled_txs(2); + let hashes = hashes_of(&txs); + + rig.announce(peer_a, &hashes); + rig.announce(peer_b, &hashes); + rig.dispatch(); + assert_eq!(rig.fetcher.fetching_peer(&hashes[0]), Some(peer_a)); + + // dropping the session drops the pending request, which resolves it with an error + rig.disconnect(peer_a); + assert_eq!( + rig.fetcher.num_fetching_hashes(), + 2, + "hashes stay inflight until the request resolves" ); - let mut hashes_to_request = RequestTxHashes::with_capacity(3); - let surplus_hashes = - tx_fetcher.pack_request(&mut hashes_to_request, valid_announcement_data); - - assert_eq!(1, hashes_to_request.len()); - assert_eq!(2, surplus_hashes.len()); - } - - #[tokio::test] - async fn test_on_fetch_pending_hashes() { - reth_tracing::init_test_tracing(); - - let tx_fetcher = &mut TransactionFetcher::default(); - - // RIG TEST - - // hashes that will be fetched because they are stored as pending fetch - let seen_hashes = [ - B256::from_slice(&[1; 32]), - B256::from_slice(&[2; 32]), - B256::from_slice(&[3; 32]), - B256::from_slice(&[4; 32]), - ]; - // - // txns 1-3 are small, all will fit in request. no metadata has been made available for - // hash 4, it has only been seen over eth66 conn, so average tx size will be assumed in - // filling request. - let seen_eth68_hashes_sizes = [120, 158, 116]; - - // peer that will fetch seen hashes because they are pending fetch - let peer_1 = PeerId::new([1; 64]); - // second peer, won't do anything in this test - let peer_2 = PeerId::new([2; 64]); - - // add seen hashes to peers seen transactions - // - // get handle for peer_1's session to receive request for pending hashes - let (mut peer_1_data, mut peer_1_mock_session_rx) = - new_mock_session(peer_1, EthVersion::Eth66); - for hash in &seen_hashes { - peer_1_data.seen_transactions.insert(*hash); - } - let (mut peer_2_data, _) = new_mock_session(peer_2, EthVersion::Eth66); - for hash in &seen_hashes { - peer_2_data.seen_transactions.insert(*hash); - } - let mut peers: HashMap> = HashMap::default(); - peers.insert(peer_1, peer_1_data); - peers.insert(peer_2, peer_2_data); - - // insert seen_hashes into tx fetcher - for i in 0..3 { - // insert peer_2 as fallback peer for seen_hashes - buffer_hash_to_tx_fetcher( - tx_fetcher, - seen_hashes[i], - peer_2, - 0, - Some(seen_eth68_hashes_sizes[i]), - ); + let event = rig.next_event().unwrap(); + assert!(matches!( + event, + FetchEvent::FetchError { peer_id, error: RequestError::ChannelClosed } if peer_id == peer_a + )); + assert_eq!(rig.fetcher.num_pending_hashes(), 2); + assert_eq!(rig.fetcher.candidate_peers(&hashes[0]), vec![peer_b]); + + rig.dispatch(); + let event = rig.respond(peer_b, txs); + assert!(matches!(event, FetchEvent::TransactionsFetched { .. })); + assert_eq!(rig.fetcher.num_hashes(), 0); + } + + #[test] + fn per_peer_announcement_limit_is_enforced() { + let config = + TransactionFetcherConfig { max_announced_hashes_per_peer: 3, ..Default::default() }; + let mut rig = Rig::with_config(config); + let peer_a = peer(1); + let peer_b = peer(2); + rig.add_peer(peer_a); + rig.add_peer(peer_b); + let txs = pooled_txs(5); + let hashes = hashes_of(&txs); + + rig.announce(peer_a, &hashes); + assert_eq!(rig.fetcher.num_hashes(), 3); + assert_eq!(rig.fetcher.queued_hashes(&peer_a), hashes[..3]); + + // the limit is per peer, another peer can still announce the dropped hashes + rig.announce(peer_b, &hashes[3..]); + assert_eq!(rig.fetcher.num_hashes(), 5); + assert_eq!(rig.fetcher.candidate_peers(&hashes[4]), vec![peer_b]); + + // delivering frees up the budget of the peer + rig.dispatch(); + rig.respond(peer_a, txs[..3].to_vec()); + rig.announce(peer_a, &hashes[3..]); + assert_eq!(rig.fetcher.candidate_peers(&hashes[3]), vec![peer_b, peer_a]); + } + + #[test] + fn global_capacity_evicts_oldest_pending_hash() { + let config = TransactionFetcherConfig { + max_capacity_cache_txns_pending_fetch: 2, + ..Default::default() + }; + let mut rig = Rig::with_config(config); + let peer_a = peer(1); + let peer_b = peer(2); + rig.add_peer(peer_a); + rig.add_peer(peer_b); + let txs = pooled_txs(5); + let hashes = hashes_of(&txs); + + // the oldest pending hash makes room for a new one + rig.announce(peer_a, &hashes[..3]); + assert_eq!(rig.fetcher.num_hashes(), 2); + assert!(rig.fetcher.candidate_peers(&hashes[0]).is_empty()); + assert_eq!(rig.fetcher.candidate_peers(&hashes[2]), vec![peer_a]); + + // hashes that are being fetched are not evicted, the announcement is dropped instead + rig.dispatch(); + let (requested, response) = rig.take_request(peer_a).unwrap(); + assert_eq!(requested, hashes[1..3]); + rig.announce(peer_b, &hashes[3..4]); + assert_eq!(rig.fetcher.num_hashes(), 2); + assert!(rig.fetcher.candidate_peers(&hashes[3]).is_empty()); + + // the second hash is delivered, the third is pending again. Both peers then track one + // hash and the announcing peer gives way, so its own older hash makes room. + response.send(Ok(PooledTransactions(txs[1..2].to_vec()))).unwrap(); + rig.next_event().unwrap(); + assert_eq!(rig.fetcher.num_hashes(), 1); + rig.announce(peer_b, &hashes[3..4]); + assert_eq!(rig.fetcher.num_hashes(), 2); + rig.announce(peer_b, &hashes[4..5]); + assert_eq!(rig.fetcher.num_hashes(), 2); + assert_eq!(rig.fetcher.candidate_peers(&hashes[2]), vec![peer_a]); + assert!(rig.fetcher.candidate_peers(&hashes[3]).is_empty()); + assert_eq!(rig.fetcher.candidate_peers(&hashes[4]), vec![peer_b]); + } + + #[test] + fn later_announcers_are_asked_after_the_first_ones_failed() { + let mut rig = Rig::new(); + let batch = hashes(0..3); + let hash = batch[0]; + let peers = + (1..=MAX_COUNT_CANDIDATE_PEERS_PER_HASH as u8 + 2).map(peer).collect::>(); + for peer_id in &peers { + rig.add_peer(*peer_id); + rig.announce(*peer_id, &batch); + } + + // Keep the eager sources and rotate the oldest fallback sources out for later ones. + let candidates = peers[..MAX_COUNT_EAGER_CANDIDATE_PEERS_PER_HASH] + .iter() + .chain(&peers[MAX_COUNT_EAGER_CANDIDATE_PEERS_PER_HASH + 2..]) + .copied() + .collect::>(); + assert_eq!(rig.fetcher.candidate_peers(&hash), candidates); + for (i, peer_id) in candidates.iter().enumerate() { + let eager = i < MAX_COUNT_EAGER_CANDIDATE_PEERS_PER_HASH; + assert_eq!(rig.fetcher.queued_hashes(peer_id).contains(&hash), eager, "peer {i}"); + } + assert!(rig.fetcher.queued_hashes(&peers[MAX_COUNT_CANDIDATE_PEERS_PER_HASH]).is_empty()); + + // the first announcer fetches the hash, retries go to the most recent announcers first + assert_eq!(rig.dispatch(), 1); + assert_eq!(rig.fetcher.fetching_peer(&hash), Some(peers[0])); + rig.fail(peers[0], RequestError::Timeout); + assert_eq!(rig.dispatch(), 1); + assert_eq!(rig.fetcher.fetching_peer(&hash), Some(*candidates.last().unwrap())); + + // the hash is given up on once all candidates failed + let mut order = vec![peers[0]]; + while let Some(fetching) = rig.fetcher.fetching_peer(&hash) { + rig.fail(fetching, RequestError::Timeout); + order.push(fetching); + rig.dispatch(); + } + let expected = std::iter::once(candidates[0]) + .chain(candidates[1..].iter().rev().copied()) + .collect::>(); + assert_eq!(order, expected); + assert_eq!(rig.fetcher.num_hashes(), 0); + } + + #[test] + fn old_request_error_does_not_settle_a_new_request_to_the_same_peer() { + let config = + TransactionFetcherConfig { max_inflight_requests_per_peer: 2, ..Default::default() }; + let mut rig = Rig::with_config(config); + let p = peer(1); + let h = hash(1); + rig.add_peer(p); + rig.announce(p, &[h]); + rig.dispatch(); + let (_, first) = rig.take_request(p).unwrap(); + rig.fetcher.on_transactions_received([&h]); + rig.announce(p, &[h]); + rig.dispatch(); + let (_, second) = rig.take_request(p).unwrap(); + first.send(Err(RequestError::Timeout)).unwrap(); + rig.next_event().unwrap(); + assert_eq!(rig.fetcher.num_fetching_hashes(), 1); + assert_eq!(rig.fetcher.fetching_peer(&h), Some(p)); + assert_eq!(rig.fetcher.candidate_peers(&h), vec![p]); + second.send(Ok(PooledTransactions(vec![]))).unwrap(); + rig.next_event().unwrap(); + assert_eq!(rig.fetcher.num_hashes(), 0); + } + + #[test] + fn first_timeout_retries_the_only_source_once() { + let mut rig = Rig::new(); + let p = peer(1); + let h = hash(1); + rig.add_peer(p); + rig.announce(p, &[h]); + rig.dispatch(); + rig.fail(p, RequestError::Timeout); + assert_eq!(rig.fetcher.candidate_peers(&h), vec![p]); + assert_eq!(rig.dispatch(), 1); + rig.fail(p, RequestError::Timeout); + assert_eq!(rig.fetcher.num_hashes(), 0); + assert_eq!(rig.dispatch(), 0); + } + + #[test] + fn replacing_candidates_cannot_extend_the_fetch_attempt_budget() { + let mut rig = Rig::new(); + let h = hash(1); + for n in 1..=MAX_COUNT_CANDIDATE_PEERS_PER_HASH as u8 { + rig.add_peer(peer(n)); + rig.announce(peer(n), &[h]); + } + for n in 0..MAX_FETCH_ATTEMPTS_PER_HASH { + let newcomer = peer((n + MAX_COUNT_CANDIDATE_PEERS_PER_HASH + 1) as u8); + rig.add_peer(newcomer); + rig.announce(newcomer, &[h]); + assert_eq!(rig.dispatch(), 1); + let fetching = rig.fetcher.fetching_peer(&h).unwrap(); + rig.fail(fetching, RequestError::Timeout); + } + assert_eq!(rig.fetcher.num_hashes(), 0); + } + + #[test] + fn self_eviction_skips_shared_and_stale_announcement_entries() { + let mut rig = Rig::with_config(TransactionFetcherConfig { + max_capacity_cache_txns_pending_fetch: 4, + ..Default::default() + }); + let honest = peer(1); + let flooder = peer(2); + for p in [honest, flooder] { + rig.add_peer(p); } - buffer_hash_to_tx_fetcher(tx_fetcher, seen_hashes[3], peer_2, 0, None); + rig.announce(honest, &hashes(0..2)); + rig.announce(flooder, &hashes(10..12)); + // The shared cursor entry must not hide newly queued exclusive victims. + rig.announce(flooder, &[hash(0), hash(12), hash(13), hash(14), hash(15), hash(16)]); + assert_eq!(rig.fetcher.candidate_peers(&hash(1)), vec![honest]); + assert_eq!(rig.fetcher.candidate_peers(&hash(0)), vec![honest, flooder]); + assert_eq!(rig.fetcher.num_hashes(), 4); + } - // insert pending hash without peer_1 as fallback peer, only with peer_2 as fallback peer - let hash_other = B256::from_slice(&[5; 32]); - buffer_hash_to_tx_fetcher(tx_fetcher, hash_other, peer_2, 0, None); + #[test] + fn stale_queue_entries_do_not_hide_exclusive_eviction_victims() { + let mut rig = Rig::with_config(TransactionFetcherConfig { + max_capacity_cache_txns_pending_fetch: 16, + ..Default::default() + }); + let honest = peer(1); + let flooder = peer(2); + let newcomer = peer(3); + for p in [honest, flooder, newcomer] { + rig.add_peer(p); + } + rig.announce(honest, &[hash(0)]); + rig.announce(flooder, &hashes(1..16)); + rig.fetcher.on_transactions_received(hashes(1..9).iter()); + rig.announce(flooder, &hashes(16..24)); + rig.announce(newcomer, &[hash(24)]); + assert_eq!(rig.fetcher.candidate_peers(&hash(0)), vec![honest]); + assert!(rig.fetcher.candidate_peers(&hash(9)).is_empty()); + } - // add peer_1 as lru fallback peer for seen hashes - for hash in &seen_hashes { - buffer_hash_to_tx_fetcher(tx_fetcher, *hash, peer_1, 0, None); + #[test] + fn capacity_eviction_targets_the_peer_with_the_most_hashes() { + let config = TransactionFetcherConfig { + max_capacity_cache_txns_pending_fetch: 100, + max_announced_hashes_per_peer: 60, + ..Default::default() + }; + let mut rig = Rig::with_config(config); + let honest = peer(1); + let flooders = [peer(2), peer(3)]; + rig.add_peer(honest); + for peer_id in flooders { + rig.add_peer(peer_id); } - // seen hashes and the random hash from peer_2 are pending fetch - assert_eq!(tx_fetcher.num_pending_hashes(), 5); + // the honest peer's hashes are the oldest when the flood fills the fetcher + let honest_hashes = hashes(0..10); + rig.announce(honest, &honest_hashes); + rig.announce(flooders[0], &hashes(100..160)); + rig.announce(flooders[1], &hashes(200..260)); - // TEST + assert_eq!(rig.fetcher.num_hashes(), 100); + for hash in &honest_hashes { + assert_eq!(rig.fetcher.candidate_peers(hash), vec![honest], "{hash} was evicted"); + } + assert_eq!(rig.fetcher.queued_hashes(&honest), honest_hashes); - tx_fetcher.on_fetch_pending_hashes(&peers, |_| true); + // the flooders gave up their oldest hashes + assert!(rig.fetcher.candidate_peers(&hashes(100..101)[0]).is_empty()); + assert_eq!(rig.fetcher.candidate_peers(&hashes(259..260)[0]), vec![flooders[1]]); + } - // mock session of peer_1 receives request - let req = peer_1_mock_session_rx - .recv() - .await - .expect("peer session should receive request with buffered hashes"); - let PeerRequest::GetPooledTransactions { request, .. } = req else { unreachable!() }; - let GetPooledTransactions(requested_hashes) = request; + #[test] + fn shared_queue_prefix_does_not_hide_exclusive_eviction_victim() { + let mut rig = Rig::with_config(TransactionFetcherConfig { + max_capacity_cache_txns_pending_fetch: 10, + ..Default::default() + }); + let honest = peer(1); + let flooder = peer(2); + let newcomer = peer(3); + for p in [honest, flooder, newcomer] { + rig.add_peer(p); + } + rig.announce(honest, &hashes(0..9)); + rig.announce(flooder, &hashes(0..8)); + rig.announce(flooder, &[hash(9)]); + rig.announce(newcomer, &[hash(10)]); + for h in hashes(0..8) { + assert_eq!(rig.fetcher.candidate_peers(&h), vec![honest, flooder]); + } + assert!(rig.fetcher.candidate_peers(&hash(9)).is_empty()); + assert_eq!(rig.fetcher.candidate_peers(&hash(10)), vec![newcomer]); + } - assert_eq!( - requested_hashes.into_iter().collect::(), - seen_hashes.into_iter().collect::() - ) + #[test] + fn capacity_fallback_keeps_the_announcer_ready() { + let mut rig = Rig::with_config(TransactionFetcherConfig { + max_capacity_cache_txns_pending_fetch: 9, + ..Default::default() + }); + let stalled = peer(1); + let source = peer(2); + let announcer = peer(3); + for p in [stalled, source, announcer] { + rig.add_peer(p); + } + rig.announce(stalled, &hashes(0..8)); + rig.dispatch(); + rig.announce(source, &[hash(8)]); + // The global scan sees only fetching hashes. The last new hash is dropped after + // the shared scratch entries have been moved into the announcer's real queue. + rig.announce(announcer, &hashes(0..10)); + rig.disconnect(source); + assert_eq!(rig.dispatch(), 1); + assert_eq!(rig.take_request(announcer).unwrap().0, vec![hash(8)]); } #[test] - fn on_fetch_pending_hashes_rebuffers_on_disconnected_peer() { - let tx_fetcher = &mut TransactionFetcher::default(); - let peer_1 = PeerId::new([1; 64]); - let peer_2 = PeerId::new([2; 64]); - let hash_1 = B256::from_slice(&[1; 32]); + fn capacity_eviction_preserves_coannounced_hashes() { + let config = TransactionFetcherConfig { + max_capacity_cache_txns_pending_fetch: 4, + ..Default::default() + }; + let mut rig = Rig::with_config(config); + let honest = peer(1); + let flooder = peer(2); + let newcomer = peer(3); + for peer_id in [honest, flooder, newcomer] { + rig.add_peer(peer_id); + } + let shared = hash(0); + rig.announce(honest, &[shared]); + rig.announce(flooder, &hashes(0..4)); + rig.announce(newcomer, &[hash(4)]); + assert_eq!(rig.fetcher.num_hashes(), 4); + assert_eq!(rig.fetcher.candidate_peers(&shared), vec![honest, flooder]); + assert!(rig.fetcher.candidate_peers(&hash(1)).is_empty()); + assert_eq!(rig.fetcher.candidate_peers(&hash(4)), vec![newcomer]); + rig.dispatch(); + assert_eq!(rig.fetcher.fetching_peer(&shared), Some(honest)); + assert_eq!(rig.take_request(flooder).unwrap().0, vec![hash(2), hash(3)]); + } - buffer_hash_to_tx_fetcher(tx_fetcher, hash_1, peer_1, 0, Some(128)); - buffer_hash_to_tx_fetcher(tx_fetcher, hash_1, peer_2, 0, Some(128)); + #[test] + fn capacity_with_only_shared_hashes_admits_new_announcements() { + let config = TransactionFetcherConfig { + max_capacity_cache_txns_pending_fetch: 2, + ..Default::default() + }; + let mut rig = Rig::with_config(config); + for peer_id in [peer(1), peer(2), peer(3)] { + rig.add_peer(peer_id); + } + for peer_id in [peer(1), peer(2)] { + rig.announce(peer_id, &hashes(0..2)); + } + rig.announce(peer(3), &[hash(2)]); + assert_eq!(rig.fetcher.num_hashes(), 2); + assert_eq!(rig.fetcher.candidate_peers(&hash(2)), vec![peer(3)]); + assert!(rig.fetcher.candidate_peers(&hash(0)).is_empty()); + assert_eq!(rig.fetcher.candidate_peers(&hash(1)), vec![peer(1), peer(2)]); + rig.dispatch(); + assert_eq!(rig.fetcher.num_fetching_hashes(), 2); + } - assert_eq!(tx_fetcher.num_pending_hashes(), 1); + #[test] + fn retracked_hash_is_not_evicted_at_its_stale_queue_position() { + let mut rig = Rig::with_config(TransactionFetcherConfig { + max_capacity_cache_txns_pending_fetch: 3, + ..Default::default() + }); + let p = peer(1); + rig.add_peer(p); + rig.announce(p, &hashes(0..3)); + rig.fetcher.on_transactions_received([&hash(0)]); + rig.announce(p, &[hash(0)]); + rig.announce(p, &[hash(3)]); + assert_eq!(rig.fetcher.candidate_peers(&hash(0)), vec![p]); + assert!(rig.fetcher.candidate_peers(&hash(1)).is_empty()); + rig.dispatch(); + assert_eq!(rig.take_request(p).unwrap().0, vec![hash(2), hash(0), hash(3)]); + } + + #[test] + fn shared_fallback_ignores_stale_tracking_generations() { + let mut rig = Rig::with_config(TransactionFetcherConfig { + max_capacity_cache_txns_pending_fetch: 3, + ..Default::default() + }); + for p in [peer(1), peer(2), peer(3)] { + rig.add_peer(p); + } + for p in [peer(1), peer(2)] { + rig.announce(p, &hashes(0..3)); + } + rig.fetcher.on_transactions_received([&hash(0)]); + for p in [peer(1), peer(2)] { + rig.announce(p, &[hash(0)]); + } + rig.announce(peer(3), &[hash(3)]); + assert_eq!(rig.fetcher.candidate_peers(&hash(0)), vec![peer(1), peer(2)]); + assert!(rig.fetcher.candidate_peers(&hash(1)).is_empty()); + } - // pass empty peers map — both peers are "disconnected" - let peers: HashMap> = HashMap::default(); - tx_fetcher.on_fetch_pending_hashes(&peers, |_| true); + #[test] + fn eviction_order_stays_bounded_when_hashes_are_tracked_again() { + let config = TransactionFetcherConfig { + max_capacity_cache_txns_pending_fetch: 8, + ..Default::default() + }; + let mut rig = Rig::with_config(config); + let peer_a = peer(1); + rig.add_peer(peer_a); + let batch = hashes(0..8); + + // the same hashes are given up on and announced again over and over, every round leaves + // stale entries in the eviction order behind + for _ in 0..10 { + rig.announce(peer_a, &batch); + assert_eq!(rig.fetcher.num_hashes(), 8); + rig.dispatch(); + rig.fail(peer_a, RequestError::BadResponse); + assert_eq!(rig.fetcher.num_hashes(), 0); + } + rig.announce(peer_a, &batch); + assert_eq!(rig.fetcher.num_hashes(), 8); - // hash should be re-buffered, not lost - assert_eq!(tx_fetcher.num_pending_hashes(), 1); + // a new hash still evicts the oldest one rather than getting dropped + rig.announce(peer_a, &hashes(8..9)); + assert_eq!(rig.fetcher.num_hashes(), 8); + assert!(rig.fetcher.candidate_peers(&batch[0]).is_empty()); } #[test] - fn verify_response_hashes() { - let input = hex!( - "02f871018302a90f808504890aef60826b6c94ddf4c5025d1a5742cf12f74eec246d4432c295e487e09c3bbcc12b2b80c080a0f21a4eacd0bf8fea9c5105c543be5a1d8c796516875710fafafdf16d16d8ee23a001280915021bb446d1973501a67f93d2b38894a514b976e7b46dc2fe54598daa" - ); - let signed_tx_1: PooledTransaction = - TransactionSigned::decode(&mut &input[..]).unwrap().try_into().unwrap(); - let input = hex!( - "02f871018302a90f808504890aef60826b6c94ddf4c5025d1a5742cf12f74eec246d4432c295e487e09c3bbcc12b2b80c080a0f21a4eacd0bf8fea9c5105c543be5a1d8c796516875710fafafdf16d16d8ee23a001280915021bb446d1973501a67f93d2b38894a514b976e7b46dc2fe54598d76" - ); - let signed_tx_2: PooledTransaction = - TransactionSigned::decode(&mut &input[..]).unwrap().try_into().unwrap(); + fn disconnected_remembered_candidates_are_forgotten() { + let mut rig = Rig::new(); + let hash = hash(1); + let peers = (1..=MAX_COUNT_CANDIDATE_PEERS_PER_HASH as u8).map(peer).collect::>(); + for peer_id in &peers { + rig.add_peer(*peer_id); + rig.announce(*peer_id, &[hash]); + } + let (queued, remembered) = peers.split_at(MAX_COUNT_EAGER_CANDIDATE_PEERS_PER_HASH); + + // remembered candidates have no queue entry, they are forgotten nonetheless + for peer_id in remembered { + rig.disconnect(*peer_id); + } + assert_eq!(rig.fetcher.candidate_peers(&hash), queued); + + // which frees their slots for later announcers + let newcomer = peer(42); + rig.add_peer(newcomer); + rig.announce(newcomer, &[hash]); + assert!(rig.fetcher.candidate_peers(&hash).contains(&newcomer)); + + // and a hash whose last connected candidate leaves is dropped + for peer_id in queued { + rig.disconnect(*peer_id); + } + rig.disconnect(newcomer); + assert_eq!(rig.fetcher.num_hashes(), 0); + } + + #[test] + fn remembered_candidates_take_over_when_queued_ones_disconnect() { + let mut rig = Rig::new(); + let hash = hash(1); + let peers = + (1..=MAX_COUNT_EAGER_CANDIDATE_PEERS_PER_HASH as u8 + 1).map(peer).collect::>(); + for peer_id in &peers { + rig.add_peer(*peer_id); + rig.announce(*peer_id, &[hash]); + } + let last = *peers.last().unwrap(); + assert!(rig.fetcher.queued_hashes(&last).is_empty()); + + // all peers that had the hash queued disconnect before fetching it + for peer_id in &peers[..peers.len() - 1] { + rig.disconnect(*peer_id); + } + assert_eq!(rig.fetcher.candidate_peers(&hash), vec![last]); + assert_eq!(rig.fetcher.queued_hashes(&last), vec![hash]); + assert_eq!(rig.dispatch(), 1); + assert_eq!(rig.fetcher.fetching_peer(&hash), Some(last)); + } - // only tx 1 is requested - let request_hashes = [ - B256::from_str("0x3b9aca00f0671c9a2a1b817a0a78d3fe0c0f776cccb2a8c3c1b412a4f4e67890") - .unwrap(), - *signed_tx_1.hash(), - B256::from_str("0x3b9aca00f0671c9a2a1b817a0a78d3fe0c0f776cccb2a8c3c1b412a4f4e12345") - .unwrap(), - B256::from_str("0x3b9aca00f0671c9a2a1b817a0a78d3fe0c0f776cccb2a8c3c1b412a4f4edabe3") - .unwrap(), - ]; + #[test] + fn global_inflight_limit_defers_ready_peers() { + let config = TransactionFetcherConfig { max_inflight_requests: 1, ..Default::default() }; + let mut rig = Rig::with_config(config); + let peer_a = peer(1); + let peer_b = peer(2); + rig.add_peer(peer_a); + rig.add_peer(peer_b); + + rig.announce(peer_a, &hashes(0..2)); + rig.announce(peer_b, &hashes(2..4)); + + assert_eq!(rig.dispatch(), 1); + let (_, response) = rig.take_request(peer_a).unwrap(); + assert!(rig.take_request(peer_b).is_none()); + assert_eq!(rig.dispatch(), 0); + + // the deferred peer is served once a request slot frees up + rig.announce(peer_a, &hashes(4..5)); + drop(response); + rig.next_event().unwrap(); + assert_eq!(rig.dispatch(), 1); + assert!(rig.take_request(peer_b).is_some()); + assert!(rig.take_request(peer_a).is_none(), "peer_b was ready first"); + } - for hash in &request_hashes { - assert_ne!(hash, signed_tx_2.hash()) + #[test] + fn dispatch_applies_the_request_limit_independently_to_each_peer() { + let mut rig = Rig::new(); + let peer_a = peer(1); + let peer_b = peer(2); + rig.add_peer(peer_a); + rig.add_peer(peer_b); + + rig.announce(peer_a, &hashes(0..100)); + rig.announce(peer_b, &hashes(100..200)); + + assert_eq!(rig.dispatch_with_budget(0), 0); + + // Each peer can use the full request limit regardless of other inflight requests. + assert_eq!(rig.dispatch_with_budget(40), 2); + let (requested, response_a) = rig.take_request(peer_a).unwrap(); + assert_eq!(requested, hashes(0..40)); + let (requested, _) = rig.take_request(peer_b).unwrap(); + assert_eq!(requested, hashes(100..140)); + assert_eq!(rig.fetcher.num_fetching_hashes(), 80); + + // A smaller request limit is respected too. + response_a.send(Err(RequestError::BadResponse)).unwrap(); + rig.next_event().unwrap(); + assert_eq!(rig.dispatch_with_budget(10), 1); + let (requested, _) = rig.take_request(peer_a).unwrap(); + assert_eq!(requested, hashes(40..50)); + } + + #[test] + fn stalled_peers_do_not_shrink_requests_to_an_idle_peer() { + let mut rig = Rig::new(); + for n in 0..16 { + let p = peer(n + 1); + rig.add_peer(p); + rig.announce(p, &hashes(u64::from(n) * 256..u64::from(n + 1) * 256)); } + assert_eq!(rig.dispatch_with_budget(4096), 16); + assert_eq!(rig.fetcher.num_fetching_hashes(), 4096); + // All earlier peers keep their requests open. The newly ready peer still gets a + // full request instead of being throttled by their inflight hashes. + let honest = peer(17); + rig.add_peer(honest); + rig.announce(honest, &hashes(4096..4352)); + assert_eq!(rig.dispatch_with_budget(4096), 1); + assert_eq!(rig.take_request(honest).unwrap().0, hashes(4096..4352)); + } - let request_hashes = RequestTxHashes::new(request_hashes.into_iter().collect()); + #[test] + fn full_session_channel_rolls_back_request() { + let mut rig = Rig::new(); + let peer_a = peer(1); + rig.add_peer_with_capacity(peer_a, 1); + let hashes = hashes(0..3); + + // occupy the only slot of the session channel + let (blocker, _rx) = oneshot::channel(); + rig.peers[&peer_a] + .request_tx() + .try_send(PeerRequest::GetPooledTransactions { + request: GetPooledTransactions(vec![]), + response: blocker, + }) + .unwrap(); + + rig.announce(peer_a, &hashes); + assert_eq!(rig.dispatch(), 0); + assert_eq!(rig.fetcher.num_pending_hashes(), 3); + assert_eq!(rig.fetcher.num_inflight_requests(), 0); + assert!(rig.fetcher.is_idle(&peer_a)); + assert_eq!(rig.fetcher.queued_hashes(&peer_a), hashes); + + // drain the blocker, the peer is retried on the next dispatch + let (blocked, _) = rig.take_request(peer_a).unwrap(); + assert!(blocked.is_empty()); + assert_eq!(rig.dispatch(), 1); + let (requested, _) = rig.take_request(peer_a).unwrap(); + assert_eq!(requested, hashes); + } - // but response contains tx 1 + another tx - let response_txns = PooledTransactions(vec![signed_tx_1.clone(), signed_tx_2]); - let payload = UnverifiedPooledTransactions::new(response_txns); + #[test] + fn closed_session_channel_disconnects_peer() { + let mut rig = Rig::new(); + let peer_a = peer(1); + let peer_b = peer(2); + rig.add_peer(peer_a); + rig.add_peer(peer_b); + let hashes = hashes(0..2); + + rig.announce(peer_a, &hashes); + rig.announce(peer_b, &hashes[..1]); + // the session task is gone, but the manager didn't process the disconnect yet + rig.sessions.remove(&peer_a); + + assert_eq!(rig.dispatch(), 1); + assert!(rig.fetcher.candidate_peers(&hashes[0]).contains(&peer_b)); + assert_eq!(rig.fetcher.candidate_peers(&hashes[0]), vec![peer_b]); + assert_eq!(rig.fetcher.num_hashes(), 1, "hash only announced by the gone peer is dropped"); + assert_eq!(rig.fetcher.fetching_peer(&hashes[0]), Some(peer_b)); + } - let (outcome, verified_payload) = payload.verify(&request_hashes, &PeerId::ZERO); + #[test] + fn busy_peer_queue_is_compacted() { + let config = + TransactionFetcherConfig { max_announced_hashes_per_peer: 4, ..Default::default() }; + let mut rig = Rig::with_config(config); + let peer_a = peer(1); + let peer_b = peer(2); + rig.add_peer(peer_a); + rig.add_peer(peer_b); + + // keep peer_a busy + rig.announce(peer_a, &hashes(0..1)); + rig.dispatch(); + + // hashes announced by the busy peer are delivered by others over and over + for round in 1..20u64 { + let hashes = hashes(round * 10..round * 10 + 4); + rig.announce(peer_a, &hashes); + rig.fetcher.on_transactions_received(&hashes); + rig.fetcher.assert_invariants(); + } + assert!(rig.fetcher.queued_hashes(&peer_a).len() <= 8); + } - assert_eq!(VerificationOutcome::ReportPeer, outcome); - assert_eq!(1, verified_payload.len()); - assert!(verified_payload.contains(&signed_tx_1)); + #[test] + fn late_response_for_reassigned_hash_is_ignored() { + let mut rig = Rig::new(); + let peer_a = peer(1); + let peer_b = peer(2); + rig.add_peer(peer_a); + rig.add_peer(peer_b); + let txs = pooled_txs(1); + let hashes = hashes_of(&txs); + + rig.announce(peer_a, &hashes); + rig.dispatch(); + // arrives over broadcast, gets announced again and is assigned to another peer + rig.fetcher.on_transactions_received(&hashes); + rig.announce(peer_b, &hashes); + rig.dispatch(); + assert_eq!(rig.fetcher.fetching_peer(&hashes[0]), Some(peer_b)); + + // the original request resolves without the hash, which must not touch the new fetch + rig.respond(peer_a, vec![]); + assert_eq!(rig.fetcher.fetching_peer(&hashes[0]), Some(peer_b)); + assert_eq!(rig.fetcher.candidate_peers(&hashes[0]), vec![peer_b]); + + rig.respond(peer_b, txs); + assert_eq!(rig.fetcher.num_hashes(), 0); + } + + #[test] + fn random_operations_keep_invariants() { + let mut rng = StdRng::seed_from_u64(0x5eed); + let txs = pooled_txs(150); + let all_hashes = hashes_of(&txs); + let by_hash = txs.iter().map(|tx| (*tx.tx_hash(), tx.clone())).collect::>(); + // more peers than a hash has candidate slots, so announcers get remembered and rejected + let peer_ids = (1..=24).map(peer).collect::>(); + + let config = TransactionFetcherConfig { + max_inflight_requests: 8, + max_inflight_requests_per_peer: 2, + max_capacity_cache_txns_pending_fetch: 120, + max_announced_hashes_per_peer: 40, + ..Default::default() + }; + let mut rig = Rig::with_config(config); + for peer_id in &peer_ids { + rig.add_peer_with_capacity(*peer_id, 4); + } + let mut connected = peer_ids.clone(); + // requests taken from the sessions that weren't answered yet + let mut outstanding: Vec<(PeerId, Vec, ResponseSender)> = Vec::new(); + + for _ in 0..5000 { + match rng.random_range(0..100u32) { + 0..=39 => { + let Some(&peer_id) = connected.choose(&mut rng) else { continue }; + let count = rng.random_range(1..=30); + let entries = all_hashes + .choose_multiple(&mut rng, count) + .map(|hash| { + let size = match rng.random_range(0..10) { + 0 => 0, + 1 => 200 * KIB, + _ => rng.random_range(100..1500), + }; + (*hash, size) + }) + .collect::>(); + rig.announce_with_sizes(peer_id, entries); + } + 40..=59 => { + let budget = + if rng.random_bool(0.2) { rng.random_range(0..60) } else { usize::MAX }; + rig.dispatch_with_budget(budget); + for peer_id in &connected { + while let Some((requested, response)) = rig.take_request(*peer_id) { + assert!(!requested.is_empty()); + assert!( + requested.len() <= + SOFT_LIMIT_COUNT_HASHES_IN_GET_POOLED_TRANSACTIONS_REQUEST + ); + let unique = requested.iter().copied().collect::(); + assert_eq!(unique.len(), requested.len(), "request has duplicates"); + outstanding.push((*peer_id, requested, response)); + } + } + } + 60..=84 => { + if outstanding.is_empty() { + continue + } + let (_, requested, response) = + outstanding.swap_remove(rng.random_range(0..outstanding.len())); + match rng.random_range(0..10) { + 0..=5 => { + let mut delivered = requested + .iter() + .filter(|_| rng.random_bool(0.7)) + .map(|hash| by_hash[hash].clone()) + .collect::>(); + if rng.random_bool(0.1) && + let Some(duplicate) = delivered.first().cloned() + { + delivered.push(duplicate); + } + if rng.random_bool(0.1) { + delivered.push(txs.choose(&mut rng).unwrap().clone()); + } + let _ = response.send(Ok(PooledTransactions(delivered))); + } + 6..=7 => { + let _ = response.send(Ok(PooledTransactions(vec![]))); + } + 8 => { + let _ = response.send(Err(RequestError::Timeout)); + } + _ => drop(response), + } + rig.drain_events(); + } + 85..=89 => { + let count = rng.random_range(1..=10); + let received = + all_hashes.choose_multiple(&mut rng, count).copied().collect::>(); + rig.fetcher.on_transactions_received(&received); + rig.fetcher.assert_invariants(); + } + 90..=94 => { + if connected.len() > 1 { + let peer_id = connected.swap_remove(rng.random_range(0..connected.len())); + rig.disconnect(peer_id); + // drops the response senders of the peer's outstanding requests + outstanding.retain(|(id, ..)| *id != peer_id); + rig.drain_events(); + } + } + _ => { + if let Some(&peer_id) = peer_ids.iter().find(|id| !connected.contains(id)) { + rig.add_peer_with_capacity(peer_id, 4); + connected.push(peer_id); + } + } + } + } + + // settle everything that is still inflight + drop(outstanding); + for peer_id in &connected { + rig.disconnect(*peer_id); + } + rig.drain_events(); + assert_eq!(rig.fetcher.num_inflight_requests(), 0); + assert_eq!(rig.fetcher.num_fetching_hashes(), 0); + assert_eq!(rig.fetcher.num_hashes(), 0); + assert_eq!(rig.fetcher.num_peers(), 0); + } + #[test] + fn resolved_requests_are_yielded_one_per_poll() { + let mut rig = Rig::new(); + let peers = (1..=3).map(peer).collect::>(); + for (i, peer_id) in peers.iter().enumerate() { + rig.add_peer(*peer_id); + rig.announce(*peer_id, &hashes(i as u64..i as u64 + 1)); + } + assert_eq!(rig.dispatch(), 3); + for peer_id in &peers { + let (_, response) = rig.take_request(*peer_id).unwrap(); + response.send(Err(RequestError::Timeout)).unwrap(); + } + + let events = rig.drain_events(); + assert_eq!(events.len(), 3); + assert!(events.iter().all(|event| matches!(event, FetchEvent::FetchError { .. }))); + assert!(rig.next_event().is_none()); + assert_eq!(rig.fetcher.num_inflight_requests(), 0); + } + + #[test] + fn requests_sent_after_a_poll_register_wakers_on_the_next_poll() { + let mut rig = Rig::new(); + let peer_a = peer(1); + rig.add_peer(peer_a); + let counter = Arc::new(WakeCounter::default()); + let waker = waker(counter.clone()); + let mut cx = Context::from_waker(&waker); + + assert!(rig.fetcher.poll_next_unpin(&mut cx).is_pending()); + rig.announce(peer_a, &hashes(0..1)); + assert_eq!(rig.dispatch(), 1); + + // the caller has to poll again after dispatching, only then the request is polled and + // registers the waker that its response wakes + assert!(rig.fetcher.poll_next_unpin(&mut cx).is_pending()); + let wakes = counter.wakes(); + let (_, response) = rig.take_request(peer_a).unwrap(); + response.send(Err(RequestError::Timeout)).unwrap(); + assert_eq!(counter.wakes(), wakes + 1); + assert!(matches!( + rig.fetcher.poll_next_unpin(&mut cx), + Poll::Ready(Some(FetchEvent::FetchError { .. })) + )); + } + + #[test] + fn global_capacity_bounds_tracked_hashes_across_peers() { + let config = TransactionFetcherConfig { + max_capacity_cache_txns_pending_fetch: 40, + ..Default::default() + }; + let mut rig = Rig::with_config(config); + let peers = (1..=4).map(peer).collect::>(); + for (i, peer_id) in peers.iter().enumerate() { + rig.add_peer(*peer_id); + rig.announce_unsized(*peer_id, &hashes(i as u64 * 16..(i as u64 + 1) * 16)); + } + + // 24 of the 64 announced hashes were evicted, spread over the peers so that every + // peer lost its oldest hashes and kept its newest + assert_eq!(rig.fetcher.num_hashes(), 40); + for (i, peer_id) in peers.iter().enumerate() { + let first = i as u64 * 16; + assert!(rig.fetcher.candidate_peers(&hash(first)).is_empty()); + assert!(rig.fetcher.candidate_peers(&hash(first + 3)).is_empty()); + assert_eq!(rig.fetcher.candidate_peers(&hash(first + 8)), vec![*peer_id]); + assert_eq!(rig.fetcher.candidate_peers(&hash(first + 15)), vec![*peer_id]); + } + rig.fetcher.assert_invariants(); + + // every peer still has hashes to request, the evicted ones are skipped + assert_eq!(rig.dispatch(), 4); + let mut requested_total = 0; + for peer_id in &peers { + let (requested, _) = rig.take_request(*peer_id).unwrap(); + assert!(requested.len() <= 256); + requested_total += requested.len(); + } + assert_eq!(requested_total, 40, "every remaining hash is requested"); + rig.fetcher.assert_invariants(); + } + + #[test] + fn delivery_order_does_not_matter() { + let mut rig = Rig::new(); + let peer_a = peer(1); + let peer_b = peer(2); + rig.add_peer(peer_a); + rig.add_peer(peer_b); + let txs = pooled_txs(4); + let hashes = hashes_of(&txs); + + rig.announce(peer_a, &hashes); + rig.announce(peer_b, &hashes); + rig.dispatch(); + + // the last and the first hash are delivered in reverse order, the two in between were + // skipped by peer_a + let event = rig.respond(peer_a, vec![txs[3].clone(), txs[0].clone()]); + let FetchEvent::TransactionsFetched { transactions, .. } = event else { panic!() }; + assert_eq!(transactions.len(), 2); + assert_eq!(rig.fetcher.num_hashes(), 2); + assert_eq!(rig.fetcher.candidate_peers(&hashes[1]), vec![peer_b]); + assert_eq!(rig.fetcher.candidate_peers(&hashes[2]), vec![peer_b]); + assert_eq!(rig.fetcher.queued_hashes(&peer_b), hashes[1..3]); + } + + #[test] + fn partial_delivery_with_concurrent_requests_per_peer() { + let config = + TransactionFetcherConfig { max_inflight_requests_per_peer: 2, ..Default::default() }; + let mut rig = Rig::with_config(config); + let peer_a = peer(1); + rig.add_peer(peer_a); + let txs = pooled_txs(300); + let hashes = hashes_of(&txs); + + rig.announce(peer_a, &hashes); + assert_eq!(rig.dispatch(), 2); + let (first, response_first) = rig.take_request(peer_a).unwrap(); + let (second, response_second) = rig.take_request(peer_a).unwrap(); + assert_eq!(first, hashes[..256]); + assert_eq!(second, hashes[256..]); + + // only the last hash of the first request is delivered, the rest was skipped and has no + // other candidate + response_first.send(Ok(PooledTransactions(txs[255..256].to_vec()))).unwrap(); + rig.next_event().unwrap(); + assert_eq!(rig.fetcher.num_hashes(), 44); + assert_eq!(rig.fetcher.num_fetching_hashes(), 44); + assert!(!rig.fetcher.is_idle(&peer_a)); + + response_second.send(Ok(PooledTransactions(txs[256..].to_vec()))).unwrap(); + rig.next_event().unwrap(); + assert_eq!(rig.fetcher.num_hashes(), 0); + assert!(rig.fetcher.is_idle(&peer_a)); + } + + #[test] + fn old_request_resolves_after_peer_reconnected_and_reannounced() { + let mut rig = Rig::new(); + let peer_a = peer(1); + rig.add_peer(peer_a); + let txs = pooled_txs(2); + let hashes = hashes_of(&txs); + + rig.announce(peer_a, &hashes); + rig.dispatch(); + let (_, response) = rig.take_request(peer_a).unwrap(); + + // the peer reconnects and announces the hashes again while the old request is pending + rig.disconnect(peer_a); + rig.add_peer(peer_a); + rig.announce(peer_a, &hashes); + assert_eq!(rig.fetcher.candidate_peers(&hashes[0]), vec![peer_a]); + assert_eq!(rig.fetcher.num_fetching_hashes(), 2); + assert_eq!(rig.dispatch(), 0, "the hashes are still being fetched"); + + // the old request delivers one hash and skips the other, which is retried with the new + // session + response.send(Ok(PooledTransactions(txs[1..].to_vec()))).unwrap(); + rig.next_event().unwrap(); + assert_eq!(rig.fetcher.num_hashes(), 1); + assert_eq!(rig.fetcher.candidate_peers(&hashes[0]), vec![peer_a]); + assert_eq!(rig.dispatch(), 1); + let (requested, _) = rig.take_request(peer_a).unwrap(); + assert_eq!(requested, hashes[..1]); + } + + #[test] + fn requeued_hashes_are_not_queued_twice() { + let mut rig = Rig::new(); + let peer_a = peer(1); + let peer_b = peer(2); + rig.add_peer(peer_a); + rig.add_peer(peer_b); + let hashes = hashes(0..3); + + // peer_b is busy while both announce the hashes, so they stay in its queue + rig.announce(peer_b, &hashes[..1]); + rig.dispatch(); + rig.announce(peer_a, &hashes[1..]); + rig.announce(peer_b, &hashes[1..]); + rig.dispatch(); + assert_eq!(rig.fetcher.fetching_peer(&hashes[1]), Some(peer_a)); + assert_eq!(rig.fetcher.queued_hashes(&peer_b), hashes[1..]); + + rig.fail(peer_a, RequestError::Timeout); + assert_eq!(rig.fetcher.queued_hashes(&peer_b), hashes[1..], "no duplicates"); } } diff --git a/crates/net/network/src/transactions/mod.rs b/crates/net/network/src/transactions/mod.rs index fdb6bf64125..a507409b124 100644 --- a/crates/net/network/src/transactions/mod.rs +++ b/crates/net/network/src/transactions/mod.rs @@ -12,6 +12,8 @@ pub mod constants; pub mod fetcher; /// Defines the traits for transaction-related policies. pub mod policy; +#[cfg(test)] +mod test_harness; pub use self::constants::{ tx_fetcher::DEFAULT_SOFT_LIMIT_BYTE_SIZE_POOLED_TRANSACTIONS_RESP_ON_PACK_GET_POOLED_TRANSACTIONS_REQ, @@ -26,7 +28,10 @@ use policy::NetworkPolicies; pub(crate) use fetcher::{FetchEvent, TransactionFetcher}; -use self::constants::{tx_manager::*, DEFAULT_SOFT_LIMIT_BYTE_SIZE_TRANSACTIONS_BROADCAST_MESSAGE}; +use self::constants::{ + tx_manager::*, DEFAULT_SOFT_LIMIT_BYTE_SIZE_TRANSACTIONS_BROADCAST_MESSAGE, + SOFT_LIMIT_COUNT_HASHES_IN_GET_POOLED_TRANSACTIONS_REQUEST, +}; use crate::{ budget::{ DEFAULT_BUDGET_TRY_DRAIN_NETWORK_TRANSACTION_EVENTS, @@ -52,7 +57,7 @@ use reth_eth_wire::{ GetPooledTransactions, HandleMempoolData, HandleVersionedMempoolData, LazyEncoded, LazyEncodedTransaction, NetworkPrimitives, NewPooledTransactionHashes, NewPooledTransactionHashes66, NewPooledTransactionHashes68, NewPooledTransactionHashes72, - PooledTransactions, RequestTxHashes, Transactions, ValidAnnouncementData, + PooledTransactions, Transactions, ValidAnnouncementData, }; use reth_ethereum_primitives::TxType; use reth_evm::SenderRecoveryCache; @@ -414,9 +419,8 @@ impl TransactionsManager { let (command_tx, command_rx) = mpsc::unbounded_channel(); - let transaction_fetcher = TransactionFetcher::with_transaction_fetcher_config( - &transactions_manager_config.transaction_fetcher_config, - ); + let transaction_fetcher = + TransactionFetcher::new(transactions_manager_config.transaction_fetcher_config.clone()); // install a listener for new __pending__ transactions that are allowed to be propagated // over the network @@ -461,25 +465,41 @@ impl TransactionsManager { self } - /// Returns `true` if [`TransactionsManager`] has capacity to request pending hashes. Returns - /// `false` if [`TransactionsManager`] is operating close to full capacity. - fn has_capacity_for_fetching_pending_hashes(&self) -> bool { - self.has_capacity_for_pending_pool_imports() && - self.transaction_fetcher.has_capacity_for_fetching_pending_hashes() - } - - /// Returns `true` if [`TransactionsManager`] has capacity for more pending pool imports. + /// Returns whether pending pool imports are below the soft limit. + /// + /// Fetch responses are admitted in full when capacity remains, so one response can exceed + /// the limit by at most 255 transactions (the 256-hash request cap minus one free slot). + /// Further responses and requests wait until imports fall below the limit again. fn has_capacity_for_pending_pool_imports(&self) -> bool { self.remaining_pool_import_capacity() > 0 } - /// Returns the remaining capacity for pending pool imports. + /// Returns the remaining capacity below the soft limit for pending pool imports. + /// + /// Outstanding requests do not reserve capacity. A fetched response is admitted in full + /// if any capacity remains; while imports exceed the limit, this returns zero. fn remaining_pool_import_capacity(&self) -> usize { self.pending_pool_imports_info.max_pending_pool_imports.saturating_sub( self.pending_pool_imports_info.pending_pool_imports.load(Ordering::Relaxed), ) } + /// Returns import capacity available to broadcasts after reserving space for fetched hashes. + fn remaining_broadcast_import_capacity(&self) -> usize { + let capacity = self.remaining_pool_import_capacity(); + if self.transaction_fetcher.num_hashes() == 0 { + return capacity + } + // Reserve one batch (or half a small import budget) for fetched transactions. + // Broadcasts are processed first and must not consume every newly freed slot. + let reserved = self + .pending_pool_imports_info + .max_pending_pool_imports + .div_ceil(2) + .min(SOFT_LIMIT_COUNT_HASHES_IN_GET_POOLED_TRANSACTIONS_REQUEST); + capacity.saturating_sub(reserved) + } + fn report_peer_bad_transactions(&self, peer_id: PeerId) { self.report_peer(peer_id, ReputationChangeKind::BadTransactions); self.metrics.reported_bad_transactions.increment(1); @@ -500,7 +520,7 @@ impl TransactionsManager { if let Some(mut peer) = self.peers.remove(peer_id) { self.policies.propagation_policy_mut().on_session_closed(&mut peer); } - self.transaction_fetcher.remove_peer(peer_id); + self.transaction_fetcher.on_peer_disconnected(peer_id); } /// Clear the transaction @@ -571,20 +591,6 @@ impl TransactionsManager { self.bad_imports.insert(err.hash); } - /// Runs an operation to fetch hashes that are cached in [`TransactionFetcher`]. - /// - /// Returns `true` if a request was sent. - fn on_fetch_hashes_pending_fetch(&mut self) -> bool { - // try drain transaction hashes pending fetch - let info = &self.pending_pool_imports_info; - let max_pending_pool_imports = info.max_pending_pool_imports; - let has_capacity_wrt_pending_pool_imports = - |divisor| info.has_capacity(max_pending_pool_imports / divisor); - - self.transaction_fetcher - .on_fetch_pending_hashes(&self.peers, has_capacity_wrt_pending_pool_imports) - } - fn on_request_error(&self, peer_id: PeerId, req_err: RequestError) { let kind = match req_err { RequestError::UnsupportedCapability => ReputationChangeKind::BadProtocol, @@ -706,8 +712,10 @@ impl TransactionsManager { self.report_peer(peer_id, ReputationChangeKind::BadAnnouncement); } - // 2. filter out transactions pending import to pool - partially_valid_msg.retain_by_hash(|hash| !self.transactions_by_peers.contains_key(hash)); + // 2. filter out transactions pending import to pool and known bad transactions + partially_valid_msg.retain_by_hash(|hash| { + !self.transactions_by_peers.contains_key(hash) && !self.bad_imports.contains(hash) + }); // 3. filter out invalid entries (spam) // @@ -794,103 +802,19 @@ impl TransactionsManager { return } - let mut valid_announcement_data = + let valid_announcement_data = ValidAnnouncementData::from_partially_valid_data(partially_valid_msg); - if valid_announcement_data.is_empty() { - // no valid announcement data - return - } - - // 5. filter out already seen unknown hashes - // - // seen hashes are already in the tx fetcher, pending fetch. - // - // for any seen hashes add the peer as fallback. unseen hashes are loaded into the tx - // fetcher, hence they should be valid at this point. - let bad_imports = &self.bad_imports; - self.transaction_fetcher.filter_unseen_and_pending_hashes( - &mut valid_announcement_data, - |hash| bad_imports.contains(hash), - &peer_id, - &client, - ); - - if valid_announcement_data.is_empty() { - // nothing to request - return - } - trace!(target: "net::tx::propagation", peer_id=format!("{peer_id:#}"), hashes_len=valid_announcement_data.len(), - hashes=?valid_announcement_data.keys(), msg_version=%valid_announcement_data.msg_version(), client_version=%client, - "received previously unseen and pending hashes in announcement from peer" + "received unknown hashes in announcement from peer" ); - // only send request for hashes to idle peer, otherwise buffer hashes storing peer as - // fallback - if !self.transaction_fetcher.is_idle(&peer_id) { - // load message version before announcement data is destructed in packing - let msg_version = valid_announcement_data.msg_version(); - let (hashes, _version) = valid_announcement_data.into_request_hashes(); - - trace!(target: "net::tx", - peer_id=format!("{peer_id:#}"), - hashes=?*hashes, - %msg_version, - %client, - "buffering hashes announced by busy peer" - ); - - self.transaction_fetcher.buffer_hashes(hashes, Some(peer_id)); - - return - } - - let mut hashes_to_request = - RequestTxHashes::with_capacity(valid_announcement_data.len() / 4); - let surplus_hashes = - self.transaction_fetcher.pack_request(&mut hashes_to_request, valid_announcement_data); - - if !surplus_hashes.is_empty() { - trace!(target: "net::tx", - peer_id=format!("{peer_id:#}"), - surplus_hashes=?*surplus_hashes, - %client, - "some hashes in announcement from peer didn't fit in `GetPooledTransactions` request, buffering surplus hashes" - ); - - self.transaction_fetcher.buffer_hashes(surplus_hashes, Some(peer_id)); - } - - trace!(target: "net::tx", - peer_id=format!("{peer_id:#}"), - hashes=?*hashes_to_request, - %client, - "sending hashes in `GetPooledTransactions` request to peer's session" - ); - - // request the missing transactions - // - // get handle to peer's session again, at this point we know it exists - let Some(peer) = self.peers.get_mut(&peer_id) else { return }; - if let Some(failed_to_request_hashes) = - self.transaction_fetcher.request_transactions_from_peer(hashes_to_request, peer) - { - let conn_eth_version = peer.version; - - trace!(target: "net::tx", - peer_id=format!("{peer_id:#}"), - failed_to_request_hashes=?*failed_to_request_hashes, - %conn_eth_version, - %client, - "sending `GetPooledTransactions` request to peer's session failed, buffering hashes" - ); - self.transaction_fetcher.buffer_hashes(failed_to_request_hashes, Some(peer_id)); - } + // 5. queue the hashes for fetching, requests are sent when the manager is polled + self.transaction_fetcher.on_announcement(peer_id, valid_announcement_data); } } @@ -1207,7 +1131,9 @@ where let transactions = self.pool.get_pooled_transaction_elements( request.0, GetPooledTransactionLimit::ResponseSizeSoftLimit( - self.transaction_fetcher.info.soft_limit_byte_size_pooled_transactions_response, + self.config + .transaction_fetcher_config + .soft_limit_byte_size_pooled_transactions_response, ), ); trace!(target: "net::tx::propagation", sent_txs=?transactions.iter().map(|tx| tx.tx_hash()), "Sending requested transactions to peer"); @@ -1403,6 +1329,10 @@ where } /// Starts the import process for the given transactions. + /// + /// Callers must check import capacity before consuming a fetch response. Responses are + /// admitted in full and may overshoot the soft limit by one bounded response; broadcasts + /// are truncated to their available capacity. fn import_transactions( &mut self, peer_id: PeerId, @@ -1417,43 +1347,50 @@ where return } - // Early return if we don't have capacity for any imports - if !self.has_capacity_for_pending_pool_imports() { - return - } - + let (version, client_version) = match &source { + TransactionSource::Broadcast => { + let Some(peer) = self.peers.get(&peer_id) else { return }; + (peer.version(), peer.client_version.clone()) + } + TransactionSource::Response { version, client_version } => { + (*version, client_version.clone()) + } + }; + let is_broadcast = source.is_broadcast(); let mut transactions = transactions.0; - // Truncate to remaining capacity early to bound work on all subsequent processing. - // Well-behaved peers follow the 4096 soft limit, so oversized payloads are likely - // malicious and we avoid wasting CPU on them. - let capacity = self.remaining_pool_import_capacity(); - if transactions.len() > capacity { - let skipped = transactions.len() - capacity; - transactions.truncate(capacity); - self.metrics - .skipped_transactions_pending_pool_imports_at_capacity - .increment(skipped as u64); - trace!(target: "net::tx", skipped, capacity, "Truncated transactions batch to capacity"); + // Fetched batches may exceed the soft limit: their request size is bounded, and the + // caller stops consuming responses until capacity becomes available again. + if is_broadcast { + let capacity = self.remaining_broadcast_import_capacity(); + if transactions.len() > capacity { + self.metrics + .skipped_transactions_pending_pool_imports_at_capacity + .increment((transactions.len() - capacity) as u64); + transactions.truncate(capacity); + } } - let Some(peer) = self.peers.get_mut(&peer_id) else { return }; - let client_version = peer.client_version.clone(); + if transactions.is_empty() { + return + } let start = Instant::now(); - // mark the transactions as received + // Stop fetching transactions received through either broadcasts or responses. self.transaction_fetcher - .remove_hashes_from_transaction_fetcher(transactions.iter().map(|tx| tx.tx_hash())); + .on_transactions_received(transactions.iter().map(|tx| tx.tx_hash())); // track that the peer knows these transaction, but only if this is a new broadcast. // If we received the transactions as the response to our `GetPooledTransactions`` // requests (based on received `NewPooledTransactionHashes`) then we already // recorded the hashes as seen by this peer in `Self::on_new_pooled_transaction_hashes`. let mut num_already_seen_by_peer = 0; - for tx in &transactions { - if source.is_broadcast() && !peer.seen_transactions.insert(*tx.tx_hash()) { - num_already_seen_by_peer += 1; + if is_broadcast && let Some(peer) = self.peers.get_mut(&peer_id) { + for tx in &transactions { + if !peer.seen_transactions.insert(*tx.tx_hash()) { + num_already_seen_by_peer += 1; + } } } @@ -1461,7 +1398,7 @@ where // EIP-8070, so their sidecars can never validate. Drop them before touching the pool; // geth equivalently diverts these bodies into a buffer that is completed with cells // fetched via `GetCells`, which is not implemented yet. - if peer.version() == EthVersion::Eth72 { + if version == EthVersion::Eth72 { let len_before = transactions.len(); transactions.retain(|tx| !tx.is_eip4844()); let dropped = len_before - transactions.len(); @@ -1595,8 +1532,18 @@ where /// Processes a [`FetchEvent`]. fn on_fetch_event(&mut self, fetch_event: FetchEvent) { match fetch_event { - FetchEvent::TransactionsFetched { peer_id, transactions, report_peer } => { - self.import_transactions(peer_id, transactions, TransactionSource::Response); + FetchEvent::TransactionsFetched { + peer_id, + transactions, + report_peer, + version, + client_version, + } => { + self.import_transactions( + peer_id, + transactions, + TransactionSource::Response { version, client_version }, + ); if report_peer { self.report_peer(peer_id, ReputationChangeKind::BadTransactions); } @@ -1655,17 +1602,11 @@ where // Advance incoming transaction events (stream new txns/announcements from // network manager and queue for import to pool/fetch txns). // - // This will potentially remove hashes from hashes pending fetch, it the event - // is an announcement (if same hashes are announced that didn't fit into a - // previous request). + // Announcements are queued in the transaction fetcher, requests for them are sent + // further below. // - // The smallest decodable transaction is an empty legacy transaction, 10 bytes - // (128 KiB / 10 bytes > 13k transactions). - // - // If this is an event with `Transactions` message, since transactions aren't - // validated until they are inserted into the pool, this can potentially queue - // >13k transactions for insertion to pool. More if the message size is bigger - // than the soft limit on a `Transactions` broadcast message, which is 128 KiB. + // Decoded broadcasts may exceed the available import capacity. Admission truncates + // them before filtering and sender recovery, reserving capacity for fetched transactions. let maybe_more_tx_events = metered_poll_nested_stream_with_budget!( poll_durations.acc_tx_events, "net::tx", @@ -1678,36 +1619,29 @@ where // Advance inflight fetch requests (flush transaction fetcher and queue for // import to pool). // - // The smallest decodable transaction is an empty legacy transaction, 10 bytes - // (2 MiB / 10 bytes > 200k transactions). - // - // Since transactions aren't validated until they are inserted into the pool, - // this can potentially queue >200k transactions for insertion to pool. More - // if the message size is bigger than the soft limit on a `PooledTransactions` - // response which is 2 MiB. + // Pace response consumption with the soft import limit. Dispatch checks capacity but + // does not reserve it for outstanding requests, so concurrent responses can exceed it. + // Admit one complete response while capacity remains (at most 256 transactions), then + // pause here if it fills or overshoots the limit. Existing requests continue in flight; + // completed responses wait in their channels until pool imports free capacity. let mut maybe_more_tx_fetch_events = metered_poll_nested_stream_with_budget!( poll_durations.acc_fetch_events, "net::tx", "Transaction fetch events stream", DEFAULT_BUDGET_TRY_DRAIN_STREAM, - this.transaction_fetcher.poll_next_unpin(cx), + if this.has_capacity_for_pending_pool_imports() { + this.transaction_fetcher.poll_next_unpin(cx) + } else { + Poll::Pending + }, |event| this.on_fetch_event(event), ); - // Advance pool imports (flush txns to pool). - // - // Note, this is done in batches. A batch is filled from one `Transactions` - // broadcast messages or one `PooledTransactions` response at a time. The - // minimum batch size is 1 transaction (and might often be the case with blob - // transactions). - // - // The smallest decodable transaction is an empty legacy transaction, 10 bytes - // (2 MiB / 10 bytes > 200k transactions). - // - // Since transactions aren't validated until they are inserted into the pool, - // this can potentially validate >200k transactions. More if the message size - // is bigger than the soft limit on a `PooledTransactions` response which is - // 2 MiB (`Transactions` broadcast messages is smaller, 128 KiB). + // Remember whether response polling stopped at capacity so imports completing below + // can trigger another poll even if the waiting response has not registered a waker yet. + let fetcher_paused_at_capacity = !this.has_capacity_for_pending_pool_imports(); + + // Advance admitted batches and free capacity for more responses. let maybe_more_pool_imports = metered_poll_nested_stream_with_budget!( poll_durations.acc_pending_imports, "net::tx", @@ -1756,15 +1690,15 @@ where this.on_new_pending_transactions(new_txs); } - // Tries to drain hashes pending fetch cache if the tx manager currently has - // capacity for this (fetch txns). - // - // Sends at most one request. + // Dispatch only below the soft import limit. Each request is capped independently; + // outstanding requests do not reserve capacity against other peers. duration_metered_exec!( { - if this.has_capacity_for_fetching_pending_hashes() && - this.on_fetch_hashes_pending_fetch() - { + // Peers whose session channel was full are retried on the next poll, which any + // other event triggers. + let budget = this.remaining_pool_import_capacity(); + if budget > 0 && this.transaction_fetcher.dispatch(&this.peers, budget) > 0 { + // poll the fetcher again so the new inflight requests register their wakers maybe_more_tx_fetch_events = true; } }, @@ -1783,13 +1717,18 @@ where this.transaction_fetcher.update_metrics(); + // Revisit the fetcher if imports freed capacity after response polling was paused. + let resume_fetcher = + fetcher_paused_at_capacity && this.has_capacity_for_pending_pool_imports(); + // all channels are fully drained and import futures pending if maybe_more_network_events || maybe_more_commands || maybe_more_tx_events || maybe_more_tx_fetch_events || maybe_more_pool_imports || - maybe_more_pending_txns + maybe_more_pending_txns || + resume_fetcher { // make sure we're woken up again cx.waker().wake_by_ref(); @@ -2049,7 +1988,7 @@ impl FullTransactionsBuilder { /// exceed the soft maximum target byte size. The limit is soft, meaning if one single /// transaction goes over the limit, it will be broadcasted in its own [`Transactions`] /// message. The same pattern is followed in filling a [`GetPooledTransactions`] request in - /// [`TransactionFetcher::fill_request_from_hashes_pending_fetch`]. + /// [`TransactionFetcher::dispatch`]. /// /// If the transaction is unsuitable for broadcast or would exceed the softlimit, it is appended /// to list of pooled transactions, (e.g. 4844 transactions). @@ -2206,11 +2145,16 @@ impl PooledTransactionsHashesBuilder { } /// How we received the transactions. +#[derive(Debug)] enum TransactionSource { /// Transactions were broadcast to us via [`Transactions`] message. Broadcast, - /// Transactions were sent as the response of [`fetcher::GetPooledTxRequest`] issued by us. - Response, + /// Transactions were sent as the response to a `GetPooledTransactions` request issued by us. + Response { + /// Session metadata remains available after the responding peer disconnects. + version: EthVersion, + client_version: Arc, + }, } // === impl TransactionSource === @@ -2719,6 +2663,13 @@ mod tests { )), }); + // advance the transaction manager future to send the request + poll_fn(|cx| { + let _ = tx_manager.poll_unpin(cx); + Poll::Ready(()) + }) + .await; + // mock session of peer_1 receives request let req = to_mock_session_rx .recv() @@ -2836,37 +2787,35 @@ mod tests { let peer_id = PeerId::new([1; 64]); let fallback_peer = PeerId::new([2; 64]); let (peer, _) = new_mock_session(peer_id, EthVersion::Eth66); + let (fallback, _) = new_mock_session(fallback_peer, EthVersion::Eth66); let hash_shared = B256::from_slice(&[1; 32]); tx_manager.peers.insert(peer_id, peer); - buffer_hash_to_tx_fetcher( - &mut tx_manager.transaction_fetcher, - hash_shared, - peer_id, - 0, - None, - ); + tx_manager.peers.insert(fallback_peer, fallback); + buffer_hash_to_tx_fetcher(&mut tx_manager.transaction_fetcher, hash_shared, peer_id, None); buffer_hash_to_tx_fetcher( &mut tx_manager.transaction_fetcher, hash_shared, fallback_peer, - 0, None, ); - tx_manager.transaction_fetcher.active_peers.insert(peer_id, 1); + assert_eq!( + tx_manager.transaction_fetcher.candidate_peers(&hash_shared), + vec![peer_id, fallback_peer] + ); tx_manager.on_network_event(NetworkEvent::Peer(PeerEvent::SessionClosed { peer_id, reason: None, })); - // peer removed from peers map and active_peers + // peer removed from peers map and from the fetcher assert!(!tx_manager.peers.contains_key(&peer_id)); - assert!(tx_manager.transaction_fetcher.active_peers.peek(&peer_id).is_none()); + assert!(tx_manager.transaction_fetcher.queued_hashes(&peer_id).is_empty()); // fallback peer is still available for the hash assert_eq!( - tx_manager.transaction_fetcher.get_idle_peer_for(hash_shared), - Some(&fallback_peer) + tx_manager.transaction_fetcher.candidate_peers(&hash_shared), + vec![fallback_peer] ); } @@ -3081,20 +3030,20 @@ mod tests { peer_1.seen_transactions.insert(txs_hashes[1]); tx_manager.peers.insert(peer_id_1, peer_1); - buffer_hash_to_tx_fetcher(tx_fetcher, txs_hashes[0], peer_id_1, 0, None); - buffer_hash_to_tx_fetcher(tx_fetcher, txs_hashes[1], peer_id_1, 0, None); + buffer_hash_to_tx_fetcher(tx_fetcher, txs_hashes[0], peer_id_1, None); + buffer_hash_to_tx_fetcher(tx_fetcher, txs_hashes[1], peer_id_1, None); // peer_1 is idle assert!(tx_fetcher.is_idle(&peer_id_1)); - assert_eq!(tx_fetcher.active_peers.len(), 0); + assert_eq!(tx_fetcher.num_inflight_requests(), 0); // sends requests for buffered hashes to peer_1 - tx_fetcher.on_fetch_pending_hashes(&tx_manager.peers, |_| true); + assert_eq!(tx_fetcher.dispatch(&tx_manager.peers, usize::MAX), 1); assert_eq!(tx_fetcher.num_pending_hashes(), 0); // as long as request is in flight peer_1 is not idle assert!(!tx_fetcher.is_idle(&peer_id_1)); - assert_eq!(tx_fetcher.active_peers.len(), 1); + assert_eq!(tx_fetcher.num_inflight_requests(), 1); // mock session of peer_1 receives request let req = to_mock_session_rx @@ -3121,13 +3070,16 @@ mod tests { // request has resolved, peer_1 is idle again assert!(tx_fetcher.is_idle(&peer_id)); - assert_eq!(tx_fetcher.active_peers.len(), 0); - // failing peer_1's request buffers requested hashes for retry. + assert_eq!(tx_fetcher.num_inflight_requests(), 0); + // the undelivered hash at the end of the request is retried with peer_1 assert_eq!(tx_fetcher.num_pending_hashes(), 1); + assert_eq!(tx_fetcher.candidate_peers(&txs_hashes[1]), vec![peer_id_1]); } + /// Tests that hashes of a failed request are retried with the alternate peer that announced + /// them, and are given up on when no peer is left to fetch them from. #[tokio::test] - async fn test_max_retries_tx_request() { + async fn test_failed_request_retries_on_alternate_peer() { reth_tracing::init_test_tracing(); let mut tx_manager = new_tx_manager().await.0; @@ -3138,44 +3090,35 @@ mod tests { let eth_version = EthVersion::Eth66; let seen_hashes = [B256::from_slice(&[1; 32]), B256::from_slice(&[2; 32])]; - let (mut peer_1, mut to_mock_session_rx) = new_mock_session(peer_id_1, eth_version); - // mark hashes as seen by peer so it can fish them out from the cache for hashes pending - // fetch - peer_1.seen_transactions.insert(seen_hashes[0]); - peer_1.seen_transactions.insert(seen_hashes[1]); + let (peer_1, mut to_mock_session_rx_1) = new_mock_session(peer_id_1, eth_version); + let (peer_2, mut to_mock_session_rx_2) = new_mock_session(peer_id_2, eth_version); tx_manager.peers.insert(peer_id_1, peer_1); + tx_manager.peers.insert(peer_id_2, peer_2); - // hashes are seen and currently not inflight, with one fallback peer, and are buffered - // for first retry in reverse order to make index 0 lru - let retries = 1; - buffer_hash_to_tx_fetcher(tx_fetcher, seen_hashes[1], peer_id_1, retries, None); - buffer_hash_to_tx_fetcher(tx_fetcher, seen_hashes[0], peer_id_1, retries, None); - - // peer_1 is idle + // both peers announce the hashes, peer_1 first + for hash in seen_hashes { + buffer_hash_to_tx_fetcher(tx_fetcher, hash, peer_id_1, None); + buffer_hash_to_tx_fetcher(tx_fetcher, hash, peer_id_2, None); + } assert!(tx_fetcher.is_idle(&peer_id_1)); - assert_eq!(tx_fetcher.active_peers.len(), 0); - - // sends request for buffered hashes to peer_1 - tx_fetcher.on_fetch_pending_hashes(&tx_manager.peers, |_| true); - - let tx_fetcher = &mut tx_manager.transaction_fetcher; + assert!(tx_fetcher.is_idle(&peer_id_2)); + // the hashes are requested from peer_1 only + assert_eq!(tx_fetcher.dispatch(&tx_manager.peers, usize::MAX), 1); assert_eq!(tx_fetcher.num_pending_hashes(), 0); - // as long as request is in inflight peer_1 is not idle assert!(!tx_fetcher.is_idle(&peer_id_1)); - assert_eq!(tx_fetcher.active_peers.len(), 1); + assert!(tx_fetcher.is_idle(&peer_id_2)); - // mock session of peer_1 receives request - let req = to_mock_session_rx + let req = to_mock_session_rx_1 .recv() .await .expect("peer_1 session should receive request with buffered hashes"); let PeerRequest::GetPooledTransactions { request, response } = req else { unreachable!() }; let GetPooledTransactions(hashes) = request; - - let hashes = hashes.into_iter().collect::(); - - assert_eq!(hashes, seen_hashes.into_iter().collect::()); + assert_eq!( + hashes.into_iter().collect::(), + seen_hashes.into_iter().collect::() + ); // fail request to peer_1 response @@ -3184,48 +3127,32 @@ mod tests { let Some(FetchEvent::FetchError { peer_id, .. }) = tx_fetcher.next().await else { unreachable!() }; + assert_eq!(peer_id, peer_id_1); - // request has resolved, peer_1 is idle again - assert!(tx_fetcher.is_idle(&peer_id)); - assert_eq!(tx_fetcher.active_peers.len(), 0); - // failing peer_1's request buffers requested hashes for retry + // request has resolved, peer_1 is idle again and the hashes are pending for peer_2 only + assert!(tx_fetcher.is_idle(&peer_id_1)); + assert_eq!(tx_fetcher.num_inflight_requests(), 0); assert_eq!(tx_fetcher.num_pending_hashes(), 2); + assert_eq!(tx_fetcher.candidate_peers(&seen_hashes[0]), vec![peer_id_2]); - let (peer_2, mut to_mock_session_rx) = new_mock_session(peer_id_2, eth_version); - tx_manager.peers.insert(peer_id_2, peer_2); - - // peer_2 announces same hashes as peer_1 - let msg = - NewPooledTransactionHashes::Eth66(NewPooledTransactionHashes66(seen_hashes.to_vec())); - tx_manager.on_new_pooled_transaction_hashes(peer_id_2, msg); - - let tx_fetcher = &mut tx_manager.transaction_fetcher; - - // peer_2 should be in active_peers. - assert_eq!(tx_fetcher.active_peers.len(), 1); - - // since hashes are already seen, no changes to length of unknown hashes - assert_eq!(tx_fetcher.num_all_hashes(), 2); - // but hashes are taken out of buffer and packed into request to peer_2 + // the hashes are retried with peer_2 + assert_eq!(tx_fetcher.dispatch(&tx_manager.peers, usize::MAX), 1); assert_eq!(tx_fetcher.num_pending_hashes(), 0); - - // mock session of peer_2 receives request - let req = to_mock_session_rx + let req = to_mock_session_rx_2 .recv() .await .expect("peer_2 session should receive request with buffered hashes"); let PeerRequest::GetPooledTransactions { response, .. } = req else { unreachable!() }; - // report failed request to tx manager + // fail request to peer_2 as well response .send(Err(RequestError::BadResponse)) .expect("should send peer_2 response to tx manager"); let Some(FetchEvent::FetchError { .. }) = tx_fetcher.next().await else { unreachable!() }; - // `MAX_REQUEST_RETRIES_PER_TX_HASH`, 2, for hashes reached so this time won't be buffered - // for retry - assert_eq!(tx_fetcher.num_pending_hashes(), 0); - assert_eq!(tx_fetcher.active_peers.len(), 0); + // no peer is left to fetch the hashes from, they are dropped + assert_eq!(tx_fetcher.num_hashes(), 0); + assert_eq!(tx_fetcher.num_inflight_requests(), 0); } #[test] diff --git a/crates/net/network/src/transactions/test_harness.rs b/crates/net/network/src/transactions/test_harness.rs new file mode 100644 index 00000000000..bad9cf7653c --- /dev/null +++ b/crates/net/network/src/transactions/test_harness.rs @@ -0,0 +1,464 @@ +//! Test harness driving the transaction fetching pipeline of a [`TransactionsManager`] with mock +//! peer sessions. +//! +//! Announcements are fed to the manager directly and every `GetPooledTransactions` request the +//! manager sends ends up in a mock session channel, from where tests answer it. + +use super::{ + NetworkTransactionEvent, TransactionPropagationMode, TransactionsManager, + TransactionsManagerConfig, +}; +use crate::{ + test_utils::transactions::{new_mock_session_with_capacity, new_tx_manager_with_config}, + NetworkManager, +}; +use alloy_primitives::TxHash; +use reth_eth_wire::{ + EthNetworkPrimitives, EthVersion, GetPooledTransactions, NewPooledTransactionHashes, + PooledTransactions, +}; +use reth_ethereum_primitives::PooledTransactionVariant; +use reth_network_api::PeerRequest; +use reth_network_p2p::{ + error::RequestResult, + sync::{NetworkSyncUpdater, SyncState}, +}; +use reth_network_peers::PeerId; +use reth_transaction_pool::test_utils::TestPool; +use std::{ + future::{poll_fn, Future}, + pin::Pin, + sync::{ + atomic::{AtomicBool, Ordering}, + Arc, + }, + task::{Context, Wake, Waker}, +}; +use tokio::sync::{mpsc, oneshot}; + +/// Number of requests a mock session buffers before the manager fails to send to it. +const SESSION_CHANNEL_CAPACITY: usize = 16; + +/// Drives a [`TransactionsManager`] whose peers are mock sessions. +struct TxFetchHarness { + manager: TransactionsManager, + _network: NetworkManager, + sessions: Vec<(PeerId, mpsc::Receiver)>, + /// Keeps the manager's detached pending transaction listener open. + _pending_transactions: mpsc::Sender, + wake_flag: Arc, + waker: Waker, +} + +impl TxFetchHarness { + /// Creates a manager with a mock session for every peer. + async fn new(peers: impl IntoIterator) -> Self { + Self::with_config(TransactionsManagerConfig::default(), peers).await + } + + /// Creates a manager with the given config and a mock session for every peer. + async fn with_config( + config: TransactionsManagerConfig, + peers: impl IntoIterator, + ) -> Self { + let config = TransactionsManagerConfig { + propagation_mode: TransactionPropagationMode::Max(0), + ..config + }; + let (mut manager, network) = new_tx_manager_with_config(config).await; + manager.network.update_sync_state(SyncState::Idle); + + // Propagating imported transactions is not part of fetching them and the mock pool can't + // serve them for propagation, so the manager never learns about pending transactions. + let (pending_transactions, pending_rx) = mpsc::channel(1); + manager.pending_transactions = pending_rx; + + let sessions = peers + .into_iter() + .map(|peer_id| { + let (peer, rx) = new_mock_session_with_capacity( + peer_id, + EthVersion::Eth68, + SESSION_CHANNEL_CAPACITY, + ); + manager.peers.insert(peer_id, peer); + (peer_id, rx) + }) + .collect(); + + let wake_flag = Arc::new(WakeFlag::default()); + let waker = Waker::from(wake_flag.clone()); + + Self { + manager, + _network: network, + sessions, + _pending_transactions: pending_transactions, + wake_flag, + waker, + } + } + + /// Delivers an announcement from the peer to the manager. + fn announce(&mut self, peer_id: PeerId, msg: NewPooledTransactionHashes) { + self.manager.on_network_tx_event( + NetworkTransactionEvent::IncomingPooledTransactionHashes { peer_id, msg }, + ); + } + + /// Polls the manager until a poll completes without asking to be woken again, i.e. until all + /// buffered work is processed. + /// + /// Returns the number of polls. + fn poll_until_idle(&mut self) -> usize { + // Polled outside of tokio's cooperative budget: inside a tokio task the manager's channels + // stop making progress after a number of polls and defer a wake that this loop would + // never see, leaving responses unprocessed. + let manager = &mut self.manager; + let mut manager = + tokio::task::unconstrained(poll_fn(|cx| Pin::new(&mut *manager).poll(cx))); + let mut polls = 0; + loop { + self.wake_flag.0.store(false, Ordering::Relaxed); + let mut cx = Context::from_waker(&self.waker); + let _ = Pin::new(&mut manager).poll(&mut cx); + polls += 1; + if !self.wake_flag.0.load(Ordering::Relaxed) { + return polls + } + } + } + + /// Returns `true` if the manager asked to be polled again since the last poll, e.g. because + /// a response arrived. + fn was_woken(&self) -> bool { + self.wake_flag.0.load(Ordering::Relaxed) + } + + /// Returns the number of hashes the transaction fetcher is tracking. + fn num_tracked_hashes(&self) -> usize { + self.manager.transaction_fetcher.num_hashes() + } + + /// Takes all `GetPooledTransactions` requests that are queued for the mock sessions. + fn take_requests(&mut self) -> Vec { + let mut requests = Vec::new(); + for (peer_id, rx) in &mut self.sessions { + while let Ok(request) = rx.try_recv() { + if let PeerRequest::GetPooledTransactions { request, response } = request { + requests.push(MockRequest { peer_id: *peer_id, request, response }); + } + } + } + requests + } + + /// Returns the manager's transaction pool. + const fn pool(&self) -> &TestPool { + &self.manager.pool + } +} + +/// A `GetPooledTransactions` request the manager sent to a mock session. +#[derive(Debug)] +struct MockRequest { + /// The peer the request was sent to. + peer_id: PeerId, + /// The requested hashes. + request: GetPooledTransactions, + /// Sends the response to the manager. + response: oneshot::Sender>>, +} + +/// Records whether the manager asked to be polled again. +#[derive(Debug, Default)] +struct WakeFlag(AtomicBool); + +impl Wake for WakeFlag { + fn wake(self: Arc) { + self.wake_by_ref(); + } + + fn wake_by_ref(self: &Arc) { + self.0.store(true, Ordering::Relaxed); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use alloy_primitives::map::{B256Map, B256Set}; + use futures::StreamExt; + use reth_eth_wire::NewPooledTransactionHashes68; + use reth_network_p2p::error::RequestError; + use reth_transaction_pool::{test_utils::TransactionGenerator, TransactionPool}; + + const PEER_A: PeerId = PeerId::new([1; 64]); + const PEER_B: PeerId = PeerId::new([2; 64]); + + fn peer(n: u8) -> PeerId { + PeerId::new([n; 64]) + } + + fn hash(n: u64) -> TxHash { + let mut bytes = [0u8; 32]; + bytes[24..].copy_from_slice(&n.to_be_bytes()); + TxHash::from(bytes) + } + + /// Signed transactions from distinct senders that can be imported into the pool. + fn pooled_txs(count: usize) -> Vec { + let mut generator = TransactionGenerator::with_num_signers(rand::rng(), count); + generator + .signer_keys + .clone() + .into_iter() + .map(|signer| { + let mut tx = generator.transaction(); + tx.signer = signer; + PooledTransactionVariant::try_from(tx.into_eip1559()).unwrap() + }) + .collect() + } + + fn announcement(hashes: &[TxHash]) -> NewPooledTransactionHashes { + NewPooledTransactionHashes::Eth68(NewPooledTransactionHashes68 { + types: vec![2; hashes.len()], + sizes: vec![512; hashes.len()], + hashes: hashes.to_vec(), + }) + } + + #[tokio::test] + async fn failed_request_is_retried_from_alternate_peer() { + let txs = pooled_txs(3); + let hashes = txs.iter().map(|tx| *tx.tx_hash()).collect::>(); + let mut harness = TxFetchHarness::new([PEER_A, PEER_B]).await; + + harness.announce(PEER_A, announcement(&hashes)); + harness.announce(PEER_B, announcement(&hashes)); + harness.poll_until_idle(); + + let mut requests = harness.take_requests(); + assert_eq!(requests.len(), 1, "a hash is only requested from one peer at a time"); + let request = requests.pop().unwrap(); + assert_eq!(request.peer_id, PEER_A); + let expected = hashes.iter().copied().collect::(); + assert_eq!(request.request.0.iter().copied().collect::(), expected); + + request.response.send(Err(RequestError::Timeout)).unwrap(); + harness.poll_until_idle(); + + let mut requests = harness.take_requests(); + assert_eq!(requests.len(), 1); + let request = requests.pop().unwrap(); + assert_eq!(request.peer_id, PEER_B, "the alternate peer is asked after the failure"); + assert_eq!(request.request.0.iter().copied().collect::(), expected); + + assert!(!harness.was_woken(), "nothing happened since the last poll"); + request.response.send(Ok(PooledTransactions(txs))).unwrap(); + assert!(harness.was_woken(), "the response must wake the manager"); + harness.poll_until_idle(); + assert!(harness.take_requests().is_empty()); + assert_eq!(harness.pool().get_all(hashes).len(), 3, "delivered transactions are imported"); + } + + #[tokio::test] + async fn responses_are_processed_across_polls() { + // one request per peer, more than the manager processes per poll iteration + let peers = (1..=64).map(peer).collect::>(); + let mut harness = TxFetchHarness::new(peers.iter().copied()).await; + for (i, peer_id) in peers.iter().enumerate() { + let hashes = (i as u64 * 64..(i as u64 + 1) * 64).map(hash).collect::>(); + harness.announce(*peer_id, announcement(&hashes)); + } + harness.poll_until_idle(); + let requests = harness.take_requests(); + assert_eq!(requests.len(), 64); + assert_eq!(harness.num_tracked_hashes(), 64 * 64); + + for request in requests { + request.response.send(Err(RequestError::Timeout)).unwrap(); + } + let polls = harness.poll_until_idle(); + assert!( + polls > 1, + "the fetch events exceed the budget of a single poll, got {polls} polls" + ); + // Each sole source gets one timeout retry, still processed across multiple polls. + let retries = harness.take_requests(); + assert_eq!(retries.len(), 64); + for request in retries { + request.response.send(Err(RequestError::Timeout)).unwrap(); + } + harness.poll_until_idle(); + assert_eq!(harness.num_tracked_hashes(), 0); + assert!(harness.take_requests().is_empty()); + } + + #[tokio::test] + async fn responses_are_processed_beyond_the_cooperative_budget() { + // tokio's channels stop making progress after 128 successful polls within one task poll, + // the harness must not be fooled by that into reporting an idle manager + let txs = pooled_txs(200); + let hashes = txs.iter().map(|tx| *tx.tx_hash()).collect::>(); + let by_hash = txs.iter().map(|tx| (*tx.tx_hash(), tx.clone())).collect::>(); + let peers = (1..=200).map(peer).collect::>(); + let mut harness = TxFetchHarness::new(peers.iter().copied()).await; + + // every peer announces one hash, so every hash needs its own response + for (peer_id, hash) in peers.iter().zip(&hashes) { + harness.announce(*peer_id, announcement(std::slice::from_ref(hash))); + } + harness.poll_until_idle(); + let mut responses = 0; + loop { + let requests = harness.take_requests(); + if requests.is_empty() { + break + } + for request in requests { + let txs = request.request.0.iter().map(|hash| by_hash[hash].clone()).collect(); + request.response.send(Ok(PooledTransactions(txs))).unwrap(); + responses += 1; + harness.poll_until_idle(); + } + } + + assert_eq!(responses, 200); + assert_eq!(harness.num_tracked_hashes(), 0); + assert_eq!(harness.pool().get_all(hashes).len(), 200, "all transactions are imported"); + } + + #[tokio::test] + async fn concurrent_responses_wait_for_pool_import_capacity() { + let txs = pooled_txs(2); + let hashes = txs.iter().map(|tx| *tx.tx_hash()).collect::>(); + let config = + TransactionsManagerConfig { max_pending_pool_imports: 1, ..Default::default() }; + let mut harness = TxFetchHarness::with_config(config, [PEER_A, PEER_B]).await; + harness.announce(PEER_A, announcement(&hashes[..1])); + harness.announce(PEER_B, announcement(&hashes[1..])); + harness.poll_until_idle(); + let requests = harness.take_requests(); + assert_eq!(requests.len(), 2); + for request in requests { + let tx = txs.iter().find(|tx| request.request.0.contains(tx.tx_hash())).unwrap(); + request.response.send(Ok(PooledTransactions(vec![tx.clone()]))).unwrap(); + } + + // Only one response can be consumed before imports are polled. The second remains + // inflight, and freeing capacity must wake the manager to consume it on the next poll. + harness.wake_flag.0.store(false, Ordering::Relaxed); + let mut cx = Context::from_waker(&harness.waker); + let _ = Pin::new(&mut harness.manager).poll(&mut cx); + assert_eq!(harness.manager.transaction_fetcher.num_inflight_requests(), 1); + assert!(harness.was_woken()); + harness.poll_until_idle(); + assert_eq!(harness.pool().get_all(hashes).len(), 2); + } + + #[tokio::test] + async fn fetched_response_can_overshoot_soft_import_limit() { + let txs = pooled_txs(257); + let hashes = txs.iter().map(|tx| *tx.tx_hash()).collect::>(); + let config = + TransactionsManagerConfig { max_pending_pool_imports: 256, ..Default::default() }; + let mut harness = TxFetchHarness::with_config(config, [PEER_A, PEER_B]).await; + harness.announce(PEER_A, announcement(&hashes[..256])); + harness.poll_until_idle(); + let request = harness.take_requests().pop().unwrap(); + assert_eq!(request.request.0.len(), 256); + request.response.send(Ok(PooledTransactions(txs[..256].to_vec()))).unwrap(); + let event = harness.manager.transaction_fetcher.next().await.unwrap(); + + // Competing imports leave one slot before the response is admitted. A full response + // is still imported, exercising the maximum overshoot of 255 transactions. + harness + .manager + .pending_pool_imports_info + .pending_pool_imports + .store(255, Ordering::Relaxed); + assert!(harness.manager.has_capacity_for_pending_pool_imports()); + harness.manager.on_fetch_event(event); + assert_eq!( + harness.manager.pending_pool_imports_info.pending_pool_imports.load(Ordering::Relaxed), + 511, + ); + assert_eq!(harness.manager.remaining_pool_import_capacity(), 0); + assert!(!harness.manager.has_capacity_for_pending_pool_imports()); + harness.manager.import_transactions( + PEER_B, + PooledTransactions(txs[256..].to_vec()), + crate::transactions::TransactionSource::Broadcast, + ); + assert!(!harness.manager.transactions_by_peers.contains_key(&hashes[256])); + + // Completing the competing imports and disconnecting the responder must not lose any + // admitted bodies or cause them to be requested again. + harness + .manager + .pending_pool_imports_info + .pending_pool_imports + .fetch_sub(255, Ordering::Relaxed); + harness.manager.peers.remove(&PEER_A); + harness.manager.transaction_fetcher.on_peer_disconnected(&PEER_A); + harness.announce(PEER_B, announcement(&hashes[..256])); + harness.poll_until_idle(); + assert!(harness.take_requests().is_empty()); + assert_eq!(harness.pool().get_all(hashes[..256].to_vec()).len(), 256); + assert_eq!(harness.manager.remaining_pool_import_capacity(), 256); + } + + #[tokio::test] + async fn stalled_fetches_cannot_starve_broadcasts() { + let txs = pooled_txs(1); + let tx_hash = *txs[0].tx_hash(); + let config = + TransactionsManagerConfig { max_pending_pool_imports: 2, ..Default::default() }; + let mut harness = TxFetchHarness::with_config(config, [PEER_A, PEER_B]).await; + harness.announce(PEER_A, announcement(&[hash(0), hash(1)])); + harness.poll_until_idle(); + let requests = harness.take_requests(); + assert_eq!(requests.len(), 1); + assert_eq!(harness.manager.transaction_fetcher.num_fetching_hashes(), 2); + harness.manager.import_transactions( + PEER_B, + PooledTransactions(txs), + crate::transactions::TransactionSource::Broadcast, + ); + harness.poll_until_idle(); + assert!( + harness.pool().get(&tx_hash).is_some(), + "stalled requests must leave broadcast capacity" + ); + drop(requests); + } + + #[tokio::test] + async fn broadcasts_cannot_starve_announced_transactions() { + let txs = pooled_txs(3); + let hashes = txs.iter().map(|tx| *tx.tx_hash()).collect::>(); + let config = + TransactionsManagerConfig { max_pending_pool_imports: 2, ..Default::default() }; + let mut harness = TxFetchHarness::with_config(config, [PEER_A, PEER_B]).await; + harness.announce(PEER_A, announcement(&hashes[..1])); + for _ in 0..10 { + harness.manager.import_transactions( + PEER_B, + PooledTransactions(txs[1..].to_vec()), + crate::transactions::TransactionSource::Broadcast, + ); + assert!(harness.manager.remaining_pool_import_capacity() >= 1); + } + harness.poll_until_idle(); + let request = + harness.take_requests().pop().expect("broadcast pressure leaves fetch capacity"); + request.response.send(Ok(PooledTransactions(txs[..1].to_vec()))).unwrap(); + // The response retains its original session metadata even if disconnect is processed first. + harness.manager.peers.remove(&PEER_A); + harness.manager.transaction_fetcher.on_peer_disconnected(&PEER_A); + harness.poll_until_idle(); + assert_eq!(harness.pool().get_all(vec![hashes[0]]).len(), 1); + } +} diff --git a/crates/net/network/tests/it/transaction_hash_fetching.rs b/crates/net/network/tests/it/transaction_hash_fetching.rs index fc4af420e49..2fb6b7ddc92 100644 --- a/crates/net/network/tests/it/transaction_hash_fetching.rs +++ b/crates/net/network/tests/it/transaction_hash_fetching.rs @@ -1,22 +1,150 @@ -use alloy_primitives::U256; +//! Tests fetching transactions that peers announce by hash over real sessions. + +use alloy_primitives::{TxHash, U256}; use reth_network::{ test_utils::Testnet, - transactions::{TransactionPropagationMode::Max, TransactionsManagerConfig}, + transactions::{TransactionPropagationMode, TransactionsManagerConfig}, }; +use reth_primitives_traits::SignedTransaction; use reth_provider::test_utils::{ExtendedAccount, MockEthProvider}; use reth_tracing::init_test_tracing; -use reth_transaction_pool::{test_utils::TransactionGenerator, PoolTransaction, TransactionPool}; -use tokio::time::Duration; +use reth_transaction_pool::{ + test_utils::TransactionGenerator, EthPooledTransaction, PoolTransaction, TransactionPool, +}; +use std::time::Duration; + +/// How long a peer may take to fetch and import the announced transactions. +const FETCH_TIMEOUT: Duration = Duration::from_secs(30); + +/// Generates pooled transactions from distinct senders that are funded in the provider. +fn funded_transactions(provider: &MockEthProvider, count: usize) -> Vec { + let mut generator = TransactionGenerator::with_num_signers(rand::rng(), count); + generator + .signer_keys + .clone() + .into_iter() + .map(|signer| { + let mut tx = generator.transaction(); + tx.signer = signer; + let tx = tx.into_eip1559().try_into_recovered().unwrap(); + let tx = EthPooledTransaction::try_from_consensus(tx).unwrap(); + provider.add_account(tx.sender(), ExtendedAccount::new(0, U256::from(100_000_000))); + tx + }) + .collect() +} + +/// Waits until the pool contains all the given transactions. +async fn wait_for_transactions(pool: &impl TransactionPool, hashes: &[TxHash]) { + tokio::time::timeout(FETCH_TIMEOUT, async { + loop { + if pool.get_all(hashes.to_vec()).len() == hashes.len() { + return + } + tokio::time::sleep(Duration::from_millis(50)).await; + } + }) + .await + .unwrap_or_else(|_| { + panic!( + "timed out, pool has {} of {} transactions", + pool.get_all(hashes.to_vec()).len(), + hashes.len() + ) + }); +} + +/// Hash-only propagation, so peers must request every transaction they learn about. +fn hashes_only() -> TransactionsManagerConfig { + TransactionsManagerConfig { + propagation_mode: TransactionPropagationMode::Max(0), + ..Default::default() + } +} + +#[tokio::test(flavor = "multi_thread")] +async fn announced_transactions_are_fetched_from_announcing_peer() { + init_test_tracing(); + + let provider = MockEthProvider::default().with_genesis_block(); + let net = Testnet::create_with(2, provider.clone()).await; + let net = net.with_eth_pool_config(hashes_only()); + let handle = net.spawn(); + handle.connect_peers().await; + + // more transactions than fit into a single request + let txs = funded_transactions(&provider, 300); + let hashes = txs.iter().map(|tx| *tx.hash()).collect::>(); + + let peer0_pool = handle.peers()[0].pool().unwrap(); + for outcome in peer0_pool.add_external_transactions(txs).await { + outcome.unwrap(); + } + + wait_for_transactions(handle.peers()[1].pool().unwrap(), &hashes).await; +} + +#[tokio::test(flavor = "multi_thread")] +async fn transactions_reach_all_peers_by_broadcast_and_fetching() { + init_test_tracing(); + + // with the default propagation mode only some peers receive the transactions in full, the + // others are announced the hashes and fetch them + let provider = MockEthProvider::default().with_genesis_block(); + let num_peers = 5; + let net = Testnet::create_with(num_peers, provider.clone()).await; + let net = net.with_eth_pool(); + let handle = net.spawn(); + handle.connect_peers().await; + + let txs = funded_transactions(&provider, 100); + let hashes = txs.iter().map(|tx| *tx.hash()).collect::>(); + + let peer0_pool = handle.peers()[0].pool().unwrap(); + for outcome in peer0_pool.add_external_transactions(txs).await { + outcome.unwrap(); + } + + for peer in &handle.peers()[1..] { + wait_for_transactions(peer.pool().unwrap(), &hashes).await; + } +} + +#[tokio::test(flavor = "multi_thread")] +async fn transactions_announced_by_many_peers_are_fetched() { + init_test_tracing(); + + // every peer announces the same transactions to the last peer, which spreads its requests + // over them + let provider = MockEthProvider::default().with_genesis_block(); + let num_peers = 6; + let net = Testnet::create_with(num_peers, provider.clone()).await; + let net = net.with_eth_pool_config(hashes_only()); + let handle = net.spawn(); + + let txs = funded_transactions(&provider, 400); + let hashes = txs.iter().map(|tx| *tx.hash()).collect::>(); + + for peer in &handle.peers()[..num_peers - 1] { + for outcome in peer.pool().unwrap().add_external_transactions(txs.clone()).await { + outcome.unwrap(); + } + } + + handle.connect_peers().await; + + let listening_peer = &handle.peers()[num_peers - 1]; + wait_for_transactions(listening_peer.pool().unwrap(), &hashes).await; +} #[tokio::test(flavor = "multi_thread")] -#[ignore] async fn transaction_hash_fetching() { init_test_tracing(); - let mut config = TransactionsManagerConfig { propagation_mode: Max(0), ..Default::default() }; + let mut config = hashes_only(); config.transaction_fetcher_config.max_inflight_requests = 1; - let provider = MockEthProvider::default(); + let provider = MockEthProvider::default().with_genesis_block(); let num_peers = 10; let net = Testnet::create_with(num_peers, provider.clone()).await; diff --git a/crates/node/core/src/args/network.rs b/crates/node/core/src/args/network.rs index 6f212ce5499..d381fd7576a 100644 --- a/crates/node/core/src/args/network.rs +++ b/crates/node/core/src/args/network.rs @@ -359,7 +359,7 @@ pub struct NetworkArgs { #[arg(long = "pooled-tx-pack-soft-limit", value_name = "BYTES", default_value_t = DefaultNetworkArgs::get_global().soft_limit_byte_size_pooled_transactions_response_on_pack_request, verbatim_doc_comment)] pub soft_limit_byte_size_pooled_transactions_response_on_pack_request: usize, - /// Max capacity of cache of hashes for transactions pending fetch. + /// Maximum number of announced transaction hashes to track, including inflight requests. #[arg(long = "max-tx-pending-fetch", value_name = "COUNT", default_value_t = DefaultNetworkArgs::get_global().max_capacity_cache_txns_pending_fetch, verbatim_doc_comment)] pub max_capacity_cache_txns_pending_fetch: u32, diff --git a/docs/vocs/docs/pages/cli/reth/node.mdx b/docs/vocs/docs/pages/cli/reth/node.mdx index 8e14dcff292..baf3f155a3a 100644 --- a/docs/vocs/docs/pages/cli/reth/node.mdx +++ b/docs/vocs/docs/pages/cli/reth/node.mdx @@ -249,7 +249,7 @@ Networking: [default: 131072] --max-tx-pending-fetch - Max capacity of cache of hashes for transactions pending fetch. + Maximum number of announced transaction hashes to track, including inflight requests. [default: 25600] diff --git a/docs/vocs/docs/pages/cli/reth/p2p/body.mdx b/docs/vocs/docs/pages/cli/reth/p2p/body.mdx index 400ffe78c88..cac838b76cf 100644 --- a/docs/vocs/docs/pages/cli/reth/p2p/body.mdx +++ b/docs/vocs/docs/pages/cli/reth/p2p/body.mdx @@ -189,7 +189,7 @@ Networking: [default: 131072] --max-tx-pending-fetch - Max capacity of cache of hashes for transactions pending fetch. + Maximum number of announced transaction hashes to track, including inflight requests. [default: 25600] diff --git a/docs/vocs/docs/pages/cli/reth/p2p/header.mdx b/docs/vocs/docs/pages/cli/reth/p2p/header.mdx index 673dcca6b6b..6519ff37604 100644 --- a/docs/vocs/docs/pages/cli/reth/p2p/header.mdx +++ b/docs/vocs/docs/pages/cli/reth/p2p/header.mdx @@ -189,7 +189,7 @@ Networking: [default: 131072] --max-tx-pending-fetch - Max capacity of cache of hashes for transactions pending fetch. + Maximum number of announced transaction hashes to track, including inflight requests. [default: 25600] diff --git a/docs/vocs/docs/pages/cli/reth/stage/run.mdx b/docs/vocs/docs/pages/cli/reth/stage/run.mdx index 2e6846de20d..7a0b858f4b9 100644 --- a/docs/vocs/docs/pages/cli/reth/stage/run.mdx +++ b/docs/vocs/docs/pages/cli/reth/stage/run.mdx @@ -348,7 +348,7 @@ Networking: [default: 131072] --max-tx-pending-fetch - Max capacity of cache of hashes for transactions pending fetch. + Maximum number of announced transaction hashes to track, including inflight requests. [default: 25600] diff --git a/etc/grafana/dashboards/reth-mempool.json b/etc/grafana/dashboards/reth-mempool.json index e738011e8b2..559738948e8 100644 --- a/etc/grafana/dashboards/reth-mempool.json +++ b/etc/grafana/dashboards/reth-mempool.json @@ -2853,7 +2853,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "description": "Duration of one call to `TransactionFetcher::on_fetch_pending_hashes`.\n\nFind Peer - find an idle fallback peer for a hash pending fetch.\n\nFill Request - fill `GetPooledTransactions` request, for the found peer, with more hashes from cache of hashes pending fetch. ", + "description": "Rate of announced transaction hashes the fetcher drops or evicts.\n\nAt Capacity - a new hash was dropped because the fetcher tracks the maximum number of hashes and no pending eviction victim was found within the search budget.\n\nEvicted At Capacity - a pending hash was evicted for a newly announced one, preferring an unshared hash from the busiest peer before falling back to the oldest pending hash.\n\nPeer Limit - the announcing peer is already a candidate for the maximum number of hashes.\n\nNo Peer Left - no candidate remains or the bounded fetch-attempt budget was exhausted.", "fieldConfig": { "defaults": { "color": { @@ -2906,7 +2906,7 @@ } ] }, - "unit": "s" + "unit": "short" }, "overrides": [] }, @@ -2936,15 +2936,15 @@ "uid": "${DS_PROMETHEUS}" }, "disableTextWrap": false, - "editorMode": "builder", - "expr": "reth_network_duration_find_idle_fallback_peer_for_any_pending_hash{$instance_label=\"$instance\"}", + "editorMode": "code", + "expr": "rate(reth_network_announced_hashes_dropped_at_capacity{$instance_label=\"$instance\"}[$__rate_interval])", "fullMetaSearch": false, "hide": false, "includeNullMetadata": true, "instant": false, - "legendFormat": "Find Idle Peer", + "legendFormat": "At Capacity", "range": true, - "refId": "C", + "refId": "A", "useBackend": false }, { @@ -2953,19 +2953,53 @@ "uid": "${DS_PROMETHEUS}" }, "disableTextWrap": false, - "editorMode": "builder", - "expr": "reth_network_duration_fill_request_from_hashes_pending_fetch{$instance_label=\"$instance\"}", + "editorMode": "code", + "expr": "rate(reth_network_announced_hashes_dropped_peer_limit{$instance_label=\"$instance\"}[$__rate_interval])", "fullMetaSearch": false, "hide": false, "includeNullMetadata": true, "instant": false, - "legendFormat": "Fill Request", + "legendFormat": "Peer Limit", "range": true, "refId": "B", "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "rate(reth_network_hashes_dropped_no_candidate_peers{$instance_label=\"$instance\"}[$__rate_interval])", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "No Peer Left", + "range": true, + "refId": "C", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "rate(reth_network_hashes_evicted_at_capacity{$instance_label=\"$instance\"}[$__rate_interval])", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "Evicted At Capacity", + "range": true, + "refId": "D", + "useBackend": false } ], - "title": "Fetch Hashes Pending Fetch Duration", + "title": "Transaction Fetcher Dropped Hashes", "type": "timeseries" }, { From a3e1beb2fc2da1a5ecda8740c785f1d35c7d30ba Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Mon, 14 Sep 2026 10:07:11 +0000 Subject: [PATCH 241/294] fix(db): apply --db.rocksdb-block-cache-size on node startup (#27211) --- crates/node/builder/src/launch/common.rs | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/crates/node/builder/src/launch/common.rs b/crates/node/builder/src/launch/common.rs index 0ec7f90efba..494bf3982e4 100644 --- a/crates/node/builder/src/launch/common.rs +++ b/crates/node/builder/src/launch/common.rs @@ -514,11 +514,14 @@ where let rocksdb_provider = if let Some(provider) = rocksdb_provider { provider } else { - RocksDBProvider::builder(self.data_dir().rocksdb()) + let mut builder = RocksDBProvider::builder(self.data_dir().rocksdb()) .with_default_tables() .with_metrics() - .with_statistics() - .build()? + .with_statistics(); + if let Some(cache_size) = self.node_config().db.rocksdb_block_cache_size { + builder = builder.with_block_cache_size(cache_size); + } + builder.build()? }; let balstore_cache_size = self From 0c8340d66b7fff434ee038600f3cace591b6ca66 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Mon, 14 Sep 2026 10:19:35 +0000 Subject: [PATCH 242/294] perf(rpc): avoid redundant copies of IPC responses (#27209) --- crates/rpc/ipc/src/server/ipc.rs | 29 +++++++++++++++++++++-------- 1 file changed, 21 insertions(+), 8 deletions(-) diff --git a/crates/rpc/ipc/src/server/ipc.rs b/crates/rpc/ipc/src/server/ipc.rs index fda19c7cb31..f3d5f4383f1 100644 --- a/crates/rpc/ipc/src/server/ipc.rs +++ b/crates/rpc/ipc/src/server/ipc.rs @@ -66,7 +66,7 @@ where while let Some(response) = pending_calls.next().await { if let Err(too_large) = batch_response.append(response) { - return Some(too_large.to_json().to_string()) + return Some(raw_response_into_string(too_large.into_json())) } } @@ -74,10 +74,13 @@ where None } else { let batch_resp = batch_response.finish(); - Some(MethodResponse::from_batch(batch_resp).to_json().to_string()) + Some(raw_response_into_string(MethodResponse::from_batch(batch_resp).into_json())) } } else { - Some(batch_response_error(Id::Null, ErrorObject::from(ErrorCode::ParseError)).to_string()) + Some(raw_response_into_string(batch_response_error( + Id::Null, + ErrorObject::from(ErrorCode::ParseError), + ))) } } @@ -135,17 +138,19 @@ where let data = request.into_bytes(); if data.len() > max_request_body_size { - return Some( - batch_response_error(Id::Null, reject_too_big_request(max_request_body_size as u32)) - .to_string(), - ) + return Some(raw_response_into_string(batch_response_error( + Id::Null, + reject_too_big_request(max_request_body_size as u32), + ))) } // Single request or notification let res = if matches!(request_kind, Kind::Single) { let response = process_single_request(data, &rpc_service).await; match response { - Some(response) if response.is_method_call() => Some(response.to_json().to_string()), + Some(response) if response.is_method_call() => { + Some(raw_response_into_string(response.into_json())) + } _ => { // subscription responses are sent directly over the sink, return a response here // would lead to duplicate responses for the subscription response @@ -160,3 +165,11 @@ where res } + +/// Converts an already serialized JSON-RPC response into the `String` the connection writes. +/// +/// `Box` is a `Box` under the hood, so this reuses the existing allocation instead +/// of copying the response body. +fn raw_response_into_string(raw: Box) -> String { + String::from(Box::::from(raw)) +} From 75d5872086604ff3d84889ef3792df48e6955188 Mon Sep 17 00:00:00 2001 From: Rez Date: Mon, 14 Sep 2026 10:54:25 +0000 Subject: [PATCH 243/294] fix(txpool): reject transactions below tracked nonce (#26981) Co-authored-by: Amp Co-authored-by: Matthias Seitz --- crates/node/core/src/args/txpool.rs | 1 + crates/transaction-pool/src/config.rs | 15 ++ crates/transaction-pool/src/pool/txpool.rs | 203 ++++++++++++++++++++- 3 files changed, 212 insertions(+), 7 deletions(-) diff --git a/crates/node/core/src/args/txpool.rs b/crates/node/core/src/args/txpool.rs index 1843bf88996..9b7fea97baf 100644 --- a/crates/node/core/src/args/txpool.rs +++ b/crates/node/core/src/args/txpool.rs @@ -543,6 +543,7 @@ impl RethTransactionPoolConfig for TxPoolArgs { max_new_pending_txs_notifications: self.max_new_pending_txs_notifications, max_queued_lifetime: self.max_queued_lifetime, max_inflight_delegated_slot_limit: default_config.max_inflight_delegated_slot_limit, + enforce_tracked_nonce: default_config.enforce_tracked_nonce, } } diff --git a/crates/transaction-pool/src/config.rs b/crates/transaction-pool/src/config.rs index 0a47dee12f2..c3938f7fa0f 100644 --- a/crates/transaction-pool/src/config.rs +++ b/crates/transaction-pool/src/config.rs @@ -72,6 +72,13 @@ pub struct PoolConfig { /// /// This restricts how many executable transaction a delegated sender can stack. pub max_inflight_delegated_slot_limit: usize, + /// Whether to enforce the sender nonce tracked from canonical updates over the nonce a + /// transaction was validated against, rejecting transactions below it. + /// + /// Closes the window in which a validation result that predates a block inserts an already + /// mined nonce as pending. Assumes sender nonces only move forward, so this is primarily + /// recommended for chains without reorgs and very low block times. Disabled by default. + pub enforce_tracked_nonce: bool, } impl PoolConfig { @@ -100,6 +107,13 @@ impl PoolConfig { self } + /// Configures whether the sender nonce tracked from canonical updates is enforced on + /// insertion, see [`Self::enforce_tracked_nonce`]. + pub const fn with_enforce_tracked_nonce(mut self, enforce: bool) -> Self { + self.enforce_tracked_nonce = enforce; + self + } + /// Returns whether the size and amount constraints in any sub-pools are exceeded. #[inline] pub const fn is_exceeded(&self, pool_size: PoolSize) -> bool { @@ -129,6 +143,7 @@ impl Default for PoolConfig { max_new_pending_txs_notifications: MAX_NEW_PENDING_TXS_NOTIFICATIONS, max_queued_lifetime: MAX_QUEUED_TRANSACTION_LIFETIME, max_inflight_delegated_slot_limit: DEFAULT_MAX_INFLIGHT_DELEGATED_SLOTS, + enforce_tracked_nonce: false, } } } diff --git a/crates/transaction-pool/src/pool/txpool.rs b/crates/transaction-pool/src/pool/txpool.rs index b9ccc9c7992..92a5252674f 100644 --- a/crates/transaction-pool/src/pool/txpool.rs +++ b/crates/transaction-pool/src/pool/txpool.rs @@ -35,6 +35,7 @@ use alloy_primitives::{ map::{AddressSet, B256Map, B256Set}, TxHash, B256, }; +use reth_primitives_traits::transaction::error::InvalidTransactionError; use rustc_hash::FxHashMap; use smallvec::SmallVec; #[cfg(test)] @@ -646,13 +647,20 @@ impl TxPool { // Apply the state changes to the total set of transactions which triggers sub-pool updates. let mut updates = self.all_transactions.update(&changed_senders); - // track changed accounts - self.all_transactions.sender_info.extend(changed_senders); - // Process the sub-pool updates let mut outcome = UpdateOutcome::default(); - #[expect(clippy::iter_with_drain, reason = "retain the scratch buffer allocation")] - self.process_updates(updates.drain(..), &mut outcome); + if self.config.enforce_tracked_nonce { + // Track the changed accounts only after the discards: a discard that removes the + // sender's last transaction also removes its info, and the tracked nonce must survive + // for `add_transaction` to reject a stale validation result. + #[expect(clippy::iter_with_drain, reason = "retain the scratch buffer allocation")] + self.process_updates(updates.drain(..), &mut outcome); + self.all_transactions.sender_info.extend(changed_senders); + } else { + self.all_transactions.sender_info.extend(changed_senders); + #[expect(clippy::iter_with_drain, reason = "retain the scratch buffer allocation")] + self.process_updates(updates.drain(..), &mut outcome); + } self.all_transactions.update_buffer = updates; // update the metrics after the update self.update_size_metrics(); @@ -741,14 +749,40 @@ impl TxPool { pub(crate) fn add_transaction( &mut self, tx: ValidPoolTransaction, - on_chain_balance: U256, - on_chain_nonce: u64, + mut on_chain_balance: U256, + mut on_chain_nonce: u64, on_chain_code_hash: Option, ) -> PoolResult> { if self.contains(tx.hash()) { return Err(PoolError::new(*tx.hash(), PoolErrorKind::AlreadyImported)) } + // Validation reads state outside the pool lock, so its snapshot can predate the last + // canonical update. Opt-in because it assumes nonces only move forward, which a reorg + // breaks. Transactions exempt from the nonce check keep the validator's verdict. + if self.config.enforce_tracked_nonce && + tx.transaction.requires_nonce_check() && + let Some(info) = self.all_transactions.sender_info.get(&tx.sender_id()) && + info.state_nonce > on_chain_nonce + { + // Below the tracked nonce the transaction was validated against outdated state; + // inserted as pending it would shadow the sender's executable transactions in the + // payload builder. + if tx.nonce() < info.state_nonce { + return Err(PoolError::new( + *tx.hash(), + InvalidPoolTransactionError::Consensus( + InvalidTransactionError::NonceNotConsistent { + tx: tx.nonce(), + state: info.state_nonce, + }, + ), + )) + } + on_chain_nonce = info.state_nonce; + on_chain_balance = info.balance; + } + self.validate_auth(&tx, on_chain_nonce, on_chain_code_hash)?; // Update sender info with balance and nonce @@ -1382,6 +1416,10 @@ pub(crate) struct AllTransactions { /// _All_ transaction in the pool sorted by their sender and nonce pair. txs: BTreeMap>, /// Contains the currently known information about the senders. + /// + /// With `PoolConfig::enforce_tracked_nonce` entries outlive a sender's transactions when an + /// account update discards them, so a validation result computed against older state cannot + /// regress the tracked nonce. sender_info: FxHashMap, /// Tracks the number of transactions by sender that are currently in the pool. tx_counter: FxHashMap, @@ -4217,6 +4255,157 @@ mod tests { assert_eq!(pool.pending_pool.len(), 1); } + /// Pool with `enforce_tracked_nonce` enabled. + fn stale_validation_pool() -> TxPool { + TxPool::new( + MockOrdering::default(), + PoolConfig { enforce_tracked_nonce: true, ..Default::default() }, + ) + } + + #[test] + fn stale_validation_accepted_by_default() { + let mut f = MockTransactionFactory::default(); + let mut pool = TxPool::new(MockOrdering::default(), Default::default()); + + let template = MockTransaction::eip1559(); + let first = f.validated(template.clone().with_nonce(0).rng_hash()); + let sender = first.sender_id(); + pool.add_transaction(first, U256::from(1_000), 0, None).unwrap(); + + let changed = FxHashMap::from_iter([( + sender, + SenderInfo { state_nonce: 1, balance: U256::from(1_000) }, + )]); + let outcome = pool.update_accounts(changed); + assert_eq!(outcome.discarded.len(), 1); + // discarding the sender's last transaction drops its info + assert!(!pool.all_transactions.sender_info.contains_key(&sender)); + + // the validation snapshot is trusted as is + let stale = f.validated(template.with_nonce(0).rng_hash()); + pool.add_transaction(stale, U256::from(1_000), 0, None).unwrap(); + assert_eq!(pool.pending_pool.len(), 1); + assert_eq!(pool.all_transactions.sender_info[&sender].state_nonce, 0); + } + + #[test] + fn stale_validation_does_not_regress_sender_state() { + let mut f = MockTransactionFactory::default(); + let mut pool = stale_validation_pool(); + + let template = MockTransaction::eip1559(); + let first = f.validated(template.clone().with_nonce(0).rng_hash()); + let sender = first.sender_id(); + pool.add_transaction(first, U256::from(1_000), 0, None).unwrap(); + + let mut changed_senders = HashMap::default(); + changed_senders.insert(sender, SenderInfo { state_nonce: 1, balance: U256::from(1_000) }); + let outcome = pool.update_accounts(changed_senders); + assert_eq!(outcome.discarded.len(), 1); + assert_eq!(pool.all_transactions.sender_info[&sender].state_nonce, 1); + + // Stale validation for the next nonce remains pending. + let current = f.validated(template.clone().with_nonce(1).rng_hash()); + pool.add_transaction(current, U256::from(1_000), 0, None).unwrap(); + assert_eq!(pool.pending_pool.len(), 1); + assert_eq!(pool.queued_pool.len(), 0); + assert_eq!(pool.all_transactions.sender_info[&sender].state_nonce, 1); + + let stale = f.validated(template.with_nonce(0).rng_hash()); + let stale_hash = *stale.hash(); + let err = pool.add_transaction(stale, U256::from(1_000), 0, None).unwrap_err(); + assert!(matches!( + err.kind, + PoolErrorKind::InvalidTransaction(InvalidPoolTransactionError::Consensus( + InvalidTransactionError::NonceNotConsistent { tx: 0, state: 1 } + )) + )); + assert!(!pool.contains(&stale_hash)); + assert_eq!(pool.all_transactions.sender_info[&sender].state_nonce, 1); + } + + #[test] + fn stale_validation_resubmission_after_commit() { + let mut f = MockTransactionFactory::default(); + let mut pool = stale_validation_pool(); + let block_info = pool.block_info(); + + let template = MockTransaction::eip1559(); + let tx0 = template.clone().with_nonce(0).rng_hash(); + let tx1 = template.clone().with_nonce(1).rng_hash(); + let first = f.validated(tx0.clone()); + let sender = first.sender_id(); + pool.add_transaction(first, U256::from(1_000), 0, None).unwrap(); + pool.add_transaction(f.validated(tx1.clone()), U256::from(1_000), 0, None).unwrap(); + assert_eq!(pool.pending_pool.len(), 2); + + // Nonce 0 is mined while resubmissions of both nonces are still being validated. + let changed = FxHashMap::from_iter([( + sender, + SenderInfo { state_nonce: 1, balance: U256::from(1_000) }, + )]); + pool.on_canonical_state_change( + block_info, + vec![*tx0.get_hash()], + changed, + PoolUpdateKind::Commit, + ); + assert_eq!(pool.pending_pool.len(), 1); + + // The resubmitted mined nonce is rejected instead of shadowing nonce 1. + let retry = f.validated(template.with_nonce(0).rng_hash()); + let err = pool.add_transaction(retry, U256::from(1_000), 0, None).unwrap_err(); + assert!(matches!( + err.kind, + PoolErrorKind::InvalidTransaction(InvalidPoolTransactionError::Consensus( + InvalidTransactionError::NonceNotConsistent { tx: 0, state: 1 } + )) + )); + + // The replacement of the executable nonce is accepted using the tracked nonce. + let replacement = f.validated(tx1.inc_price_by(10).rng_hash()); + let added = pool.add_transaction(replacement, U256::from(1_000), 0, None).unwrap(); + assert!(added.as_pending().is_some()); + assert!(added.replaced().is_some()); + assert_eq!(pool.pending_pool.len(), 1); + assert_eq!(pool.all_transactions.sender_info[&sender].state_nonce, 1); + let next = + pool.get_highest_consecutive_transaction_by_sender(sender.into_transaction_id(1)); + assert_eq!(next.map(|tx| tx.nonce()), Some(1)); + } + + #[test] + fn stale_validation_uses_tracked_balance() { + let mut f = MockTransactionFactory::default(); + let mut pool = stale_validation_pool(); + + // each transaction costs 1_500_000 + let template = MockTransaction::eip1559().with_gas_price(50).with_gas_limit(30_000); + let first = f.validated(template.clone().with_nonce(0).rng_hash()); + let sender = first.sender_id(); + pool.add_transaction(first, U256::from(10_000_000), 0, None).unwrap(); + assert_eq!(pool.pending_pool.len(), 1); + + // The commit mines nonce 0 and drains the balance below the cost of the next transaction. + let changed = FxHashMap::from_iter([( + sender, + SenderInfo { state_nonce: 1, balance: U256::from(1_000) }, + )]); + pool.update_accounts(changed); + + // The stale snapshot still reports the old balance, the tracked balance applies. + let next = f.validated(template.with_nonce(1).rng_hash()); + let id = *next.id(); + pool.add_transaction(next, U256::from(10_000_000), 0, None).unwrap(); + assert_eq!(pool.pending_pool.len(), 0); + assert_eq!(pool.queued_pool.len(), 1); + let state = pool.all_transactions.get(&id).unwrap().state; + assert!(state.contains(TxState::NO_NONCE_GAPS)); + assert!(!state.contains(TxState::ENOUGH_BALANCE)); + assert_eq!(pool.all_transactions.sender_info[&sender].balance, U256::from(1_000)); + } + #[test] fn discard_with_large_blob_txs() { // init tracing From 8cb75f0a9389ecbbce1b0795117965adaf27a57b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 14 Sep 2026 11:34:38 +0000 Subject: [PATCH 244/294] chore(deps): bump the ci-weekly group across 1 directory with 5 updates (#27215) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/book.yml | 2 +- .github/workflows/docker-test.yml | 2 +- .github/workflows/docker.yml | 2 +- .github/workflows/e2e.yml | 4 ++-- .github/workflows/integration.yml | 6 +++--- .github/workflows/lint.yml | 12 ++++++------ .github/workflows/unit.yml | 6 +++--- 7 files changed, 17 insertions(+), 17 deletions(-) diff --git a/.github/workflows/book.yml b/.github/workflows/book.yml index ad4f40bc6c9..07f26676b62 100644 --- a/.github/workflows/book.yml +++ b/.github/workflows/book.yml @@ -86,4 +86,4 @@ jobs: steps: - name: Deploy to GitHub Pages id: deployment - uses: actions/deploy-pages@cd2ce8fcbc39b97be8ca5fce6e763baed58fa128 # v5.0.0 + uses: actions/deploy-pages@368f82528645a54fb793d4d04e342629a3f51346 # v5.0.1 diff --git a/.github/workflows/docker-test.yml b/.github/workflows/docker-test.yml index bda764e0c6f..31edbe19fa0 100644 --- a/.github/workflows/docker-test.yml +++ b/.github/workflows/docker-test.yml @@ -50,7 +50,7 @@ jobs: # Depot build (upstream only) - name: Set up Depot CLI if: steps.fork.outputs.is_fork == 'false' - uses: depot/setup-action@15c09a5f77a0840ad4bce955686522a257853461 # v1.7.1 + uses: depot/setup-action@91bc8495a33ebfc504ffc89e5674379ccf23c29c # v1.7.2 - name: Build reth image (Depot) if: steps.fork.outputs.is_fork == 'false' diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index f0d3425e4ed..78a2457beae 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -46,7 +46,7 @@ jobs: persist-credentials: false - name: Set up Depot CLI - uses: depot/setup-action@15c09a5f77a0840ad4bce955686522a257853461 # v1.7.1 + uses: depot/setup-action@91bc8495a33ebfc504ffc89e5674379ccf23c29c # v1.7.2 - name: Log in to GHCR uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0 diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index fa8ad62f75f..6031cd278a5 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -36,7 +36,7 @@ jobs: - run: .github/scripts/install_llvm.sh ubuntu - uses: dtolnay/rust-toolchain@stable - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 - - uses: taiki-e/install-action@b6b84cf49ebfe0176417bdce007c624f0db37f20 # v2.86.2 + - uses: taiki-e/install-action@d438492cf8a250514fa2d34b30bc3c0dc37c65ff # v2.87.8 with: tool: nextest - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 @@ -72,7 +72,7 @@ jobs: - run: .github/scripts/install_llvm.sh ubuntu - uses: dtolnay/rust-toolchain@stable - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 - - uses: taiki-e/install-action@b6b84cf49ebfe0176417bdce007c624f0db37f20 # v2.86.2 + - uses: taiki-e/install-action@d438492cf8a250514fa2d34b30bc3c0dc37c65ff # v2.87.8 with: tool: nextest - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 0f8f3d4670a..3d3ff5abf6d 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -44,7 +44,7 @@ jobs: - uses: dtolnay/rust-toolchain@stable - name: Install Geth run: .github/scripts/install_geth.sh - - uses: taiki-e/install-action@b6b84cf49ebfe0176417bdce007c624f0db37f20 # v2.86.2 + - uses: taiki-e/install-action@d438492cf8a250514fa2d34b30bc3c0dc37c65ff # v2.87.8 with: tool: nextest - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 @@ -69,7 +69,7 @@ jobs: timeout-minutes: 30 steps: - name: Decide whether the needed jobs succeeded or failed - uses: re-actors/alls-green@05ac9388f0aebcb5727afa17fcccfecd6f8ec5fe # release/v1 + uses: re-actors/alls-green@b5b5b37504aa4183270bd3d855c52a67f212be35 # release/v1 with: jobs: ${{ toJSON(needs) }} @@ -86,7 +86,7 @@ jobs: - uses: rui314/setup-mold@7e4f20ad28a2e8ca6fd0892ccf72e2abb706b9c3 # v1 - run: .github/scripts/install_llvm.sh ubuntu - uses: dtolnay/rust-toolchain@stable - - uses: taiki-e/install-action@b6b84cf49ebfe0176417bdce007c624f0db37f20 # v2.86.2 + - uses: taiki-e/install-action@d438492cf8a250514fa2d34b30bc3c0dc37c65ff # v2.87.8 with: tool: nextest - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 47b2b30d370..b52ca66799c 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -84,7 +84,7 @@ jobs: - uses: dtolnay/rust-toolchain@stable with: target: wasm32-wasip1 - - uses: taiki-e/install-action@b6b84cf49ebfe0176417bdce007c624f0db37f20 # v2.86.2 + - uses: taiki-e/install-action@d438492cf8a250514fa2d34b30bc3c0dc37c65ff # v2.87.8 with: tool: cargo-hack - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 @@ -110,7 +110,7 @@ jobs: - uses: dtolnay/rust-toolchain@stable with: target: riscv32imac-unknown-none-elf - - uses: taiki-e/install-action@b6b84cf49ebfe0176417bdce007c624f0db37f20 # v2.86.2 + - uses: taiki-e/install-action@d438492cf8a250514fa2d34b30bc3c0dc37c65ff # v2.87.8 with: tool: cargo-hack - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 @@ -138,7 +138,7 @@ jobs: - uses: rui314/setup-mold@7e4f20ad28a2e8ca6fd0892ccf72e2abb706b9c3 # v1 - run: .github/scripts/install_llvm.sh ubuntu - uses: dtolnay/rust-toolchain@stable - - uses: taiki-e/install-action@b6b84cf49ebfe0176417bdce007c624f0db37f20 # v2.86.2 + - uses: taiki-e/install-action@d438492cf8a250514fa2d34b30bc3c0dc37c65ff # v2.87.8 with: tool: cargo-hack - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 @@ -228,7 +228,7 @@ jobs: - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 with: cache-on-failure: true - - uses: taiki-e/install-action@b6b84cf49ebfe0176417bdce007c624f0db37f20 # v2.86.2 + - uses: taiki-e/install-action@d438492cf8a250514fa2d34b30bc3c0dc37c65ff # v2.87.8 with: tool: cargo-udeps - run: cargo udeps --workspace --lib --examples --tests --benches --all-features --locked @@ -279,7 +279,7 @@ jobs: with: persist-credentials: false - name: Run dprint - uses: dprint/check@9cb3a2b17a8e606d37aae341e49df3654933fc23 # v2.3 + uses: dprint/check@7dc032d8874778fc024f6f9b4a7e6d896782c330 # v2.5 with: config-path: dprint.json @@ -358,6 +358,6 @@ jobs: timeout-minutes: 30 steps: - name: Decide whether the needed jobs succeeded or failed - uses: re-actors/alls-green@05ac9388f0aebcb5727afa17fcccfecd6f8ec5fe # release/v1 + uses: re-actors/alls-green@b5b5b37504aa4183270bd3d855c52a67f212be35 # release/v1 with: jobs: ${{ toJSON(needs) }} diff --git a/.github/workflows/unit.yml b/.github/workflows/unit.yml index 1c211cc92aa..9e59205b8d8 100644 --- a/.github/workflows/unit.yml +++ b/.github/workflows/unit.yml @@ -48,7 +48,7 @@ jobs: prefix-key: v1-rust-unit-targets cache-bin: false cache-on-failure: true - - uses: taiki-e/install-action@b6b84cf49ebfe0176417bdce007c624f0db37f20 # v2.86.2 + - uses: taiki-e/install-action@d438492cf8a250514fa2d34b30bc3c0dc37c65ff # v2.87.8 with: tool: nextest - name: Run tests @@ -94,7 +94,7 @@ jobs: - uses: rui314/setup-mold@7e4f20ad28a2e8ca6fd0892ccf72e2abb706b9c3 # v1 - run: .github/scripts/install_llvm.sh ubuntu - uses: dtolnay/rust-toolchain@stable - - uses: taiki-e/install-action@b6b84cf49ebfe0176417bdce007c624f0db37f20 # v2.86.2 + - uses: taiki-e/install-action@d438492cf8a250514fa2d34b30bc3c0dc37c65ff # v2.87.8 with: tool: nextest - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 @@ -133,6 +133,6 @@ jobs: timeout-minutes: 30 steps: - name: Decide whether the needed jobs succeeded or failed - uses: re-actors/alls-green@05ac9388f0aebcb5727afa17fcccfecd6f8ec5fe # release/v1 + uses: re-actors/alls-green@b5b5b37504aa4183270bd3d855c52a67f212be35 # release/v1 with: jobs: ${{ toJSON(needs) }} From 4608b3a40f25bd5ee8cd7eaceff03ed13fac12e3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 14 Sep 2026 11:34:42 +0000 Subject: [PATCH 245/294] chore(deps): bump the cargo-weekly group with 3 updates (#27216) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 38 +++++++++++++++++++------------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ce3c6ebd134..fd44e504aa9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -189,9 +189,9 @@ dependencies = [ [[package]] name = "alloy-dyn-abi" -version = "1.7.1" +version = "1.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9f1a3f2206f2ba4206fdeeddce6640eed3e26b8a13ac41444adb66b76d8e650" +checksum = "201b9e973fe90b2effd9ab356d4f2a46ab56046ba9d46f163367553e72c045b0" dependencies = [ "alloy-json-abi", "alloy-primitives", @@ -3501,7 +3501,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -4611,9 +4611,9 @@ dependencies = [ [[package]] name = "hickory-net" -version = "0.26.1" +version = "0.26.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2295ed2f9c31e471e1428a8f88a3f0e1f4b27c15049592138d1eebe9c35b183" +checksum = "c480823ed7c2c5d0f09c41020cb6b7c28029ce60ec42dc942158dcf22f8e0a4d" dependencies = [ "async-trait", "cfg-if", @@ -4635,9 +4635,9 @@ dependencies = [ [[package]] name = "hickory-proto" -version = "0.26.1" +version = "0.26.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0bab31817bfb44672a252e97fe81cd0c18d1b2cf892108922f6818820df8c643" +checksum = "12b92608f679a6fa515dd1d15c1ff89443026e391200a2c840c7afcba482893d" dependencies = [ "data-encoding", "idna", @@ -4656,9 +4656,9 @@ dependencies = [ [[package]] name = "hickory-resolver" -version = "0.26.1" +version = "0.26.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0d58d28879ceecde6607729660c2667a081ccdc082e082675042793960f178c" +checksum = "3f3da5255c95d5a716857d54b5b8f4e8d67c3484d3beaaaae2ce25063b3ba981" dependencies = [ "cfg-if", "futures-util", @@ -5248,7 +5248,7 @@ checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" dependencies = [ "hermit-abi", "libc", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -6378,7 +6378,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -7118,9 +7118,9 @@ dependencies = [ [[package]] name = "prefix-trie" -version = "0.8.2" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23370be78b7e5bcbb0cab4a02047eb040279a693c78daad04c2c5f1c24a83503" +checksum = "4cf6e3177f0684016a5c209b00882e15f8bdd3f3bb48f0491df10cd102d0c6e7" dependencies = [ "either", "ipnet", @@ -11547,7 +11547,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -11627,7 +11627,7 @@ dependencies = [ "security-framework", "security-framework-sys", "webpki-root-certs 1.0.7", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -12221,9 +12221,9 @@ dependencies = [ [[package]] name = "smallvec" -version = "1.15.2" +version = "1.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" +checksum = "ba467056f1b547ed52077911161fc86985becbc60e8e1857c8a144dab0def891" dependencies = [ "arbitrary", "serde", @@ -12520,7 +12520,7 @@ dependencies = [ "getrandom 0.4.2", "once_cell", "rustix", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -13784,7 +13784,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] From ee845e317f903f14d7cb9ec47a7e3f387d95c8ec Mon Sep 17 00:00:00 2001 From: grandizzy <38490174+grandizzy@users.noreply.github.com> Date: Mon, 14 Sep 2026 11:40:00 +0000 Subject: [PATCH 246/294] fix(engine): avoid panic on payload stream termination (#27218) --- crates/node/builder/src/launch/engine.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/node/builder/src/launch/engine.rs b/crates/node/builder/src/launch/engine.rs index c47b5ed6bb3..b4dbf9111d2 100644 --- a/crates/node/builder/src/launch/engine.rs +++ b/crates/node/builder/src/launch/engine.rs @@ -376,7 +376,7 @@ impl EngineNodeLauncher { } } } - payload = built_payloads.select_next_some(), if !built_payloads.is_terminated() => { + Some(payload) = built_payloads.next(), if !built_payloads.is_terminated() => { if let Some(executed_block) = payload.executed_block() { debug!(target: "reth::cli", block=?executed_block.recovered_block.num_hash(), "inserting built payload"); orchestrator.handler_mut().handler_mut().on_event(EngineApiRequest::InsertExecutedBlock(executed_block).into()); From 64eb623128deef31659a52e009654a4e83856fe3 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Mon, 14 Sep 2026 11:59:01 +0000 Subject: [PATCH 247/294] chore(db): skip unread RocksDB timer histograms (#27212) --- .../src/providers/rocksdb/provider.rs | 35 +++++++++++++++++-- 1 file changed, 33 insertions(+), 2 deletions(-) diff --git a/crates/storage/provider/src/providers/rocksdb/provider.rs b/crates/storage/provider/src/providers/rocksdb/provider.rs index c13977e0cc5..471b1ef7b7e 100644 --- a/crates/storage/provider/src/providers/rocksdb/provider.rs +++ b/crates/storage/provider/src/providers/rocksdb/provider.rs @@ -26,8 +26,8 @@ use reth_storage_errors::{ provider::{ProviderError, ProviderResult}, }; use rocksdb::{ - BlockBasedOptions, Cache, ColumnFamilyDescriptor, CompactionPri, DBCompressionType, - DBRawIteratorWithThreadMode, IteratorMode, OptimisticTransactionDB, + statistics::StatsLevel, BlockBasedOptions, Cache, ColumnFamilyDescriptor, CompactionPri, + DBCompressionType, DBRawIteratorWithThreadMode, IteratorMode, OptimisticTransactionDB, OptimisticTransactionOptions, Options, SnapshotWithThreadMode, Transaction, WriteBatchWithTransaction, WriteBufferManager, WriteOptions, DB, DEFAULT_COLUMN_FAMILY_NAME, }; @@ -252,6 +252,16 @@ impl RocksDBBuilder { // Statistics can view from RocksDB log file if enable_statistics { options.enable_statistics(); + // Nothing reads the statistics programmatically, they only end up in the periodic + // LOG dump, so collect the counters but not the timer histograms: at the default + // level every `Get`, `Seek` and write wraps itself in a `StopWatch` that reads the + // clock twice, while the tickers are plain atomic increments. + // + // Note: the discriminants of `rocksdb::StatsLevel` are one higher than the levels the + // C API defines, so this variant selects "except timers". Should that ever be + // corrected upstream it selects "except histogram or timers", which also keeps the + // tickers and drops the timers. + options.set_statistics_level(StatsLevel::ExceptHistogramOrTimers); } options @@ -3263,6 +3273,27 @@ mod tests { } } + /// Guards the statistics level: the counters must keep working, the per-operation timer + /// histograms must stay empty. `rocksdb`'s `StatsLevel` discriminants do not line up with the + /// levels the C API defines, so the variant name alone does not tell us what was selected. + #[test] + fn test_statistics_level_skips_timers() { + use rocksdb::statistics::{Histogram, Ticker}; + + let temp_dir = TempDir::new().unwrap(); + let cache = Cache::new_lru_cache(1 << 20); + let options = RocksDBBuilder::default_options(rocksdb::LogLevel::Info, &cache, true); + + let db = DB::open(&options, temp_dir.path()).unwrap(); + for i in 0..10u8 { + db.put([i], [i]).unwrap(); + assert_eq!(db.get([i]).unwrap(), Some(vec![i])); + } + + assert!(options.get_ticker_count(Ticker::NumberKeysRead) > 0); + assert_eq!(options.get_histogram_data(Histogram::DbGet).count(), 0); + } + #[test] fn test_data_persistence() { let temp_dir = TempDir::new().unwrap(); From 4d6ccfbd5cb8929e2d631f98501e587728e692e0 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Mon, 14 Sep 2026 08:25:10 -0400 Subject: [PATCH 248/294] refactor(engine): use upstream BalAccountInfo (#27204) --- Cargo.lock | 27 ++-- Cargo.toml | 4 +- crates/engine/tree/Cargo.toml | 2 +- .../src/tree/payload_processor/prewarm.rs | 122 +++--------------- 4 files changed, 37 insertions(+), 118 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index fd44e504aa9..a3124534f97 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -252,9 +252,9 @@ dependencies = [ [[package]] name = "alloy-eip7928" -version = "0.4.9" +version = "0.4.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fdd59d7221d384c46771ffbf6f18f70aa7753326351277857016bed5127b888f" +checksum = "1cb19c9d53b507960f649e5689f7098e556696023fe1b57b67f2181ed54bc5b7" dependencies = [ "alloy-primitives", "alloy-rlp", @@ -1974,7 +1974,7 @@ version = "3.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6dee98b0db6a962de883bf5d20362dee4d7ca0d12fe39a7c6c73c844e1cd7c1f" dependencies = [ - "darling 0.21.3", + "darling 0.23.0", "ident_case", "prettyplease", "proc-macro2", @@ -3016,7 +3016,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ab67060fc6b8ef687992d439ca0fa36e7ed17e9a0b16b25b601e8757df720de" dependencies = [ "data-encoding", - "syn 1.0.109", + "syn 2.0.118", ] [[package]] @@ -7418,7 +7418,7 @@ dependencies = [ "once_cell", "socket2", "tracing", - "windows-sys 0.59.0", + "windows-sys 0.60.2", ] [[package]] @@ -8112,9 +8112,9 @@ dependencies = [ [[package]] name = "reth-codecs" -version = "0.7.0" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d34b0b409ecf930ed3850e92bd9b4d4f8a8c397eef23e8ba5b1a1a02682a94bd" +checksum = "a6bac09d9d84a412ae151149ec870d6eb7770335c5eb63f209453d81deff8d39" dependencies = [ "alloy-consensus", "alloy-eips", @@ -8133,9 +8133,9 @@ dependencies = [ [[package]] name = "reth-codecs-derive" -version = "0.7.0" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebcb49ab084e764e7d20f710e2c9daa439380d0fc33f8bfee8c372bfdbc1f437" +checksum = "0101b1f474563c4c48c04fc2477199279a53d9c19b65301c6fd2131cf6e4a330" dependencies = [ "proc-macro2", "quote", @@ -9862,11 +9862,12 @@ dependencies = [ [[package]] name = "reth-primitives-traits" -version = "0.7.0" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9060738a07ca1b0d8ede3ce21d8187c57cc22ee9e2991b29cbbc663280f843cd" +checksum = "18341fba74d004c7fad32f784488baaea9adebfde7a4f231a4d8d445aea70347" dependencies = [ "alloy-consensus", + "alloy-eip7928", "alloy-eips", "alloy-genesis", "alloy-primitives", @@ -10963,9 +10964,9 @@ dependencies = [ [[package]] name = "reth-zstd-compressors" -version = "0.7.0" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08f1ebb37e81a596ce16e81bec6a0c9accbdb914cf19b9190eff69ed00395900" +checksum = "dbdee2f0ad09dc32b43ab52641f8239766a66de4345834bad27d5cfa2f62864d" dependencies = [ "zstd", ] diff --git a/Cargo.toml b/Cargo.toml index 2d8bc4bf58d..8a54e15dbe9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -394,7 +394,7 @@ reth-payload-builder-primitives = { path = "crates/payload/builder-primitives" } reth-payload-primitives = { path = "crates/payload/primitives" } reth-payload-validator = { path = "crates/payload/validator" } reth-payload-util = { path = "crates/payload/util" } -reth-primitives-traits = { version = "0.7.0", default-features = false } +reth-primitives-traits = { version = "0.7.1", default-features = false } reth-provider = { path = "crates/storage/provider" } reth-prune = { path = "crates/prune/prune" } reth-prune-types = { path = "crates/prune/types", default-features = false } @@ -444,7 +444,7 @@ alloy-sol-types = { version = "1.6.1", default-features = false } alloy-chains = { version = "0.2.33", default-features = false } alloy-eip2124 = { version = "0.2.0", default-features = false } -alloy-eip7928 = { version = "0.4.9", default-features = false, features = ["rlp"] } +alloy-eip7928 = { version = "0.4.10", default-features = false, features = ["rlp"] } alloy-evm = { version = "0.39.0", default-features = false } alloy-rlp = { version = "0.3.16", default-features = false, features = ["core-net"] } alloy-trie = { version = "0.9.4", default-features = false } diff --git a/crates/engine/tree/Cargo.toml b/crates/engine/tree/Cargo.toml index c621ebaf6b8..d121dfa2df7 100644 --- a/crates/engine/tree/Cargo.toml +++ b/crates/engine/tree/Cargo.toml @@ -24,7 +24,7 @@ reth-evm = { workspace = true, features = ["metrics"] } reth-network-p2p.workspace = true reth-payload-builder.workspace = true reth-payload-primitives.workspace = true -reth-primitives-traits = { workspace = true, features = ["rayon", "dashmap"] } +reth-primitives-traits = { workspace = true, features = ["rayon", "dashmap", "eip7928"] } reth-ethereum-primitives.workspace = true reth-provider.workspace = true reth-prune.workspace = true diff --git a/crates/engine/tree/src/tree/payload_processor/prewarm.rs b/crates/engine/tree/src/tree/payload_processor/prewarm.rs index 003697d4c64..5fa6a461a84 100644 --- a/crates/engine/tree/src/tree/payload_processor/prewarm.rs +++ b/crates/engine/tree/src/tree/payload_processor/prewarm.rs @@ -20,12 +20,12 @@ use crate::tree::{ use alloy_consensus::transaction::TxHashRef; use alloy_eip7928::bal::DecodedBal; use alloy_eips::eip4895::Withdrawal; -use alloy_primitives::{keccak256, B256, U256}; +use alloy_primitives::keccak256; use metrics::{Counter, Gauge, Histogram}; use rayon::prelude::*; use reth_evm::{execute::ExecutableTxFor, ConfigureEvm, Evm, EvmFor, RecoveredTx, SpecFor}; use reth_metrics::Metrics; -use reth_primitives_traits::{Account, FastInstant as Instant, NodePrimitives}; +use reth_primitives_traits::{FastInstant as Instant, NodePrimitives}; use reth_provider::{ AccountReader, BlockExecutionOutput, BlockNumReader, ChangeSetReader, DatabaseProviderFactory, DatabaseProviderROFactory, HistoryReader, PruneCheckpointReader, StageCheckpointReader, @@ -673,16 +673,16 @@ where } let address = account_changes.address; let mut hashed_address = None; - let account_fields = BalAccountStateFields::from_changes(account_changes); + let account_info = account_changes.account_info(); - if !bal_account_changes_state_root(account_changes, account_fields) { + if !account_info.changes_state_root(account_changes) { return; } // If there are any storage changes we can assume that the resulting account info will be // non-empty, so the account will exist, and therefore we can pre-emptively send out storage // changes to start processing them before potentially hitting the db in the next step. - if !account_changes.storage_changes.is_empty() { + if account_changes.has_storage_changes() { let hashed_address = *hashed_address.get_or_insert_with(|| keccak256(address)); let storage_map = reth_trie::HashedStorage::from_iter( account_changes @@ -695,7 +695,9 @@ where hashed_update_stream.on_hashed_state_update(hashed_state); } - let existing_account = if account_fields.needs_parent_account() { + let existing_account = if account_info.is_complete() { + None + } else { if provider.is_none() { let _span = debug_span!( target: "engine::tree::payload_processor::prewarm", @@ -731,11 +733,10 @@ where } let account_reader = provider.as_ref().expect("provider just initialized"); account_reader.basic_account(&address).ok().flatten() - } else { - None }; - let account = account_fields.into_account(existing_account); + let mut account = existing_account.unwrap_or_default(); + account.apply_bal_info(account_info); let hashed_address = hashed_address.unwrap_or_else(|| keccak256(address)); // It is possible for the resulting account info to be empty. This can happen when, in the @@ -757,63 +758,6 @@ where } } -#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] -struct BalAccountStateFields { - balance: Option, - nonce: Option, - code_hash: Option, -} - -impl BalAccountStateFields { - fn from_changes(account_changes: &alloy_eip7928::AccountChanges) -> Self { - Self { - balance: account_changes.balance_post_state(), - nonce: account_changes.nonce_post_state(), - code_hash: account_changes.code_post_state().map(|code| { - if code.is_empty() { - alloy_consensus::constants::KECCAK_EMPTY - } else { - keccak256(code) - } - }), - } - } - - const fn is_empty(self) -> bool { - self.balance.is_none() && self.nonce.is_none() && self.code_hash.is_none() - } - - const fn needs_parent_account(self) -> bool { - self.balance.is_none() || self.nonce.is_none() || self.code_hash.is_none() - } - - fn into_account(self, existing_account: Option) -> Account { - let existing_account = existing_account.as_ref(); - Account { - balance: self.balance.unwrap_or_else(|| { - existing_account - .map(|account| account.balance) - .unwrap_or(alloy_primitives::U256::ZERO) - }), - nonce: self - .nonce - .unwrap_or_else(|| existing_account.map(|account| account.nonce).unwrap_or(0)), - bytecode_hash: self.code_hash.or_else(|| { - existing_account - .and_then(|account| account.bytecode_hash) - .or(Some(alloy_consensus::constants::KECCAK_EMPTY)) - }), - } - } -} - -const fn bal_account_changes_state_root( - account_changes: &alloy_eip7928::AccountChanges, - account_fields: BalAccountStateFields, -) -> bool { - !account_fields.is_empty() || !account_changes.storage_changes.is_empty() -} - /// Returns [`MultiProofTargetsV2`] for withdrawal addresses. /// /// Withdrawals only modify account balances (no storage), so the targets contain @@ -829,15 +773,13 @@ fn multiproof_targets_from_withdrawals(withdrawals: &[Withdrawal]) -> MultiProof mod tests { use super::*; use alloy_consensus::transaction::Recovered; - use alloy_eip7928::{ - AccountChanges, BalanceChange, BlockAccessIndex, CodeChange, NonceChange, SlotChanges, - StorageChange, - }; - use alloy_primitives::{address, bytes}; + use alloy_eip7928::{AccountChanges, BalanceChange, BlockAccessIndex}; + use alloy_primitives::{address, B256, U256}; use reth_chainspec::ChainSpec; use reth_ethereum_primitives::TransactionSigned; use reth_evm::{execute::WithTxEnv, TxEnvFor}; use reth_evm_ethereum::EthEvmConfig; + use reth_primitives_traits::Account; use reth_provider::test_utils::MockEthProvider; use reth_storage_overlay::OverlayManager; @@ -884,47 +826,23 @@ mod tests { fn bal_read_only_account_does_not_change_state_root() { let changes = AccountChanges::new(address!("0000000000000000000000000000000000000001")) .with_storage_read(U256::from(1)); - let fields = BalAccountStateFields::from_changes(&changes); - - assert!(fields.is_empty()); - assert!(!bal_account_changes_state_root(&changes, fields)); - } - - #[test] - fn bal_account_with_all_leaf_fields_does_not_need_parent_account() { - let changes = AccountChanges::new(address!("0000000000000000000000000000000000000001")) - .with_balance_change(BalanceChange::new(BlockAccessIndex::new(1), U256::from(10))) - .with_nonce_change(NonceChange::new(BlockAccessIndex::new(1), 7)) - .with_code_change(CodeChange::new(BlockAccessIndex::new(1), bytes!("6001600155"))); - let fields = BalAccountStateFields::from_changes(&changes); - - assert!(bal_account_changes_state_root(&changes, fields)); - assert!(!fields.needs_parent_account()); - } - #[test] - fn bal_storage_change_needs_parent_account_when_leaf_fields_missing() { - let changes = AccountChanges::new(address!("0000000000000000000000000000000000000001")) - .with_storage_change(SlotChanges::new( - U256::from(1), - vec![StorageChange::new(BlockAccessIndex::new(1), U256::from(2))], - )); - let fields = BalAccountStateFields::from_changes(&changes); - - assert!(bal_account_changes_state_root(&changes, fields)); - assert!(fields.needs_parent_account()); + assert!(!changes.account_info().changes_state_root(&changes)); } #[test] fn bal_account_uses_existing_fields_only_when_missing() { let changes = AccountChanges::new(address!("0000000000000000000000000000000000000001")) .with_balance_change(BalanceChange::new(BlockAccessIndex::new(1), U256::from(10))); - let fields = BalAccountStateFields::from_changes(&changes); - let account = fields.into_account(Some(Account { + let info = changes.account_info(); + + assert!(!info.is_complete()); + let mut account = Account { balance: U256::from(1), nonce: 3, bytecode_hash: Some(B256::repeat_byte(0xaa)), - })); + }; + account.apply_bal_info(info); assert_eq!(account.balance, U256::from(10)); assert_eq!(account.nonce, 3); From b8d127b8feb0f0b5102f8f6b070240c75299ffdd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A9a=20Narzis?= <78718413+lean-apple@users.noreply.github.com> Date: Mon, 14 Sep 2026 12:44:50 +0000 Subject: [PATCH 249/294] feat(snap-sync): download and persist account ranges under the attempt record (#27111) --- .github/scripts/check_wasm.sh | 1 + Cargo.lock | 10 + crates/net/downloaders/src/snap/mod.rs | 44 +- crates/net/downloaders/src/snap/storage.rs | 1 + crates/snap-sync/Cargo.toml | 18 +- crates/snap-sync/src/account/download.rs | 406 +++++++++++++ crates/snap-sync/src/account/mod.rs | 9 + crates/snap-sync/src/account/store.rs | 642 +++++++++++++++++++++ crates/snap-sync/src/attempt.rs | 42 +- crates/snap-sync/src/error.rs | 80 ++- crates/snap-sync/src/lib.rs | 9 +- crates/snap-sync/src/test_utils.rs | 219 ++++++- 12 files changed, 1452 insertions(+), 29 deletions(-) create mode 100644 crates/snap-sync/src/account/download.rs create mode 100644 crates/snap-sync/src/account/mod.rs create mode 100644 crates/snap-sync/src/account/store.rs diff --git a/.github/scripts/check_wasm.sh b/.github/scripts/check_wasm.sh index 13bea47cfea..15a8b8122ff 100755 --- a/.github/scripts/check_wasm.sh +++ b/.github/scripts/check_wasm.sh @@ -63,6 +63,7 @@ exclude_crates=( reth-provider # tokio reth-prune # tokio reth-prune-static-files # reth-provider + reth-snap-sync # reth-tasks, reth-downloaders reth-tasks # tokio rt-multi-thread reth-stages-api # reth-provider, reth-prune reth-static-file # tokio diff --git a/Cargo.lock b/Cargo.lock index a3124534f97..71dc24ac381 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -10414,14 +10414,24 @@ dependencies = [ "alloy-eip7928", "alloy-eips", "alloy-primitives", + "alloy-rlp", + "futures", "reth-db-api", + "reth-downloaders", + "reth-eth-wire-types", + "reth-network-p2p", + "reth-network-peers", "reth-primitives-traits", "reth-provider", "reth-storage-api", "reth-storage-errors", + "reth-tasks", "reth-trie-common", "revm", + "serde", + "serde_json", "thiserror 2.0.20", + "tokio", "tokio-util", ] diff --git a/crates/net/downloaders/src/snap/mod.rs b/crates/net/downloaders/src/snap/mod.rs index 7466448fb6b..9c490184e09 100644 --- a/crates/net/downloaders/src/snap/mod.rs +++ b/crates/net/downloaders/src/snap/mod.rs @@ -89,6 +89,8 @@ pub struct VerifiedAccountRange { // Root the accounts were proven against. Private so a range cannot be relabelled with a root // that did not authenticate it. state_root: B256, + // Inclusive key the proof was verified from. Private for the same reason as the root. + origin: B256, // Accounts as the response returned them, in the order every positional check assumes. accounts: Vec<(B256, TrieAccount)>, // Whether the requested interval may continue past this response. @@ -103,6 +105,11 @@ impl VerifiedAccountRange { self.state_root } + /// Key the range was requested from, proven to have nothing before the first account. + pub const fn origin(&self) -> B256 { + self.origin + } + /// Accounts in strictly increasing hashed-key order. pub fn accounts(&self) -> &[(B256, TrieAccount)] { &self.accounts @@ -254,6 +261,7 @@ impl SnapVerifier for GetAccountRangeMessage { return if self.root_hash == EMPTY_ROOT_HASH { Ok(AccountRangeOutcome::Verified(VerifiedAccountRange { state_root: self.root_hash, + origin: self.starting_hash, accounts: Vec::new(), has_more: false, next: None, @@ -291,11 +299,20 @@ fn verify_account_range( .collect::, _>>()?; let next = verify_proof(request, &accounts, &response.proof)?; - // Authenticate the boundary account before removing it from the requested range. - accounts.truncate(accounts.partition_point(|(hash, _)| *hash <= request.limit_hash)); + // Authenticate the boundary account before removing it from the requested range. Once + // removed, it is the first key after the response, so resuming from it cannot skip it. + let kept = accounts.partition_point(|(hash, _)| *hash <= request.limit_hash); + let next = accounts.get(kept).map(|(hash, _)| *hash).or(next); + accounts.truncate(kept); let has_more = next.is_some_and(|next| next <= request.limit_hash); - Ok(VerifiedAccountRange { state_root: request.root_hash, accounts, has_more, next }) + Ok(VerifiedAccountRange { + state_root: request.root_hash, + origin: request.starting_hash, + accounts, + has_more, + next, + }) } // Re-encodes decoded accounts so the proof authenticates their canonical trie values. @@ -404,6 +421,7 @@ mod tests { outcome, AccountRangeOutcome::Verified(VerifiedAccountRange { state_root: root_hash, + origin: B256::ZERO, accounts, has_more: false, next: None, @@ -462,6 +480,7 @@ mod tests { let root_hash = root(&accounts); let range = VerifiedAccountRange { state_root: root_hash, + origin: B256::ZERO, accounts: accounts.clone(), has_more: false, next: None, @@ -486,8 +505,13 @@ mod tests { third.storage_root = B256::repeat_byte(0x33); let accounts = vec![(key(1), first), (key(2), empty), (key(3), third)]; let root_hash = root(&accounts); - let range = - VerifiedAccountRange { state_root: root_hash, accounts, has_more: false, next: None }; + let range = VerifiedAccountRange { + state_root: root_hash, + origin: B256::ZERO, + accounts, + has_more: false, + next: None, + }; let batch = range.storage_batch(); @@ -542,9 +566,10 @@ mod tests { outcome, AccountRangeOutcome::Verified(VerifiedAccountRange { state_root: root_hash, + origin: B256::ZERO, accounts: vec![accounts[0]], has_more: false, - next: Some(key(4)), + next: Some(key(3)), }) ); assert!(client.reported().is_empty()); @@ -599,6 +624,7 @@ mod tests { outcome, AccountRangeOutcome::Verified(VerifiedAccountRange { state_root: root_hash, + origin: B256::ZERO, accounts: accounts[..2].to_vec(), has_more: false, next: Some(key(3)), @@ -629,9 +655,10 @@ mod tests { outcome, AccountRangeOutcome::Verified(VerifiedAccountRange { state_root: root_hash, + origin: key(3), accounts: Vec::new(), has_more: false, - next: None, + next: Some(key(9)), }) ); assert!(client.reported().is_empty()); @@ -654,6 +681,7 @@ mod tests { outcome, AccountRangeOutcome::Verified(VerifiedAccountRange { state_root: root_hash, + origin: key(3), accounts: Vec::new(), has_more: false, next: Some(key(9)), @@ -683,6 +711,7 @@ mod tests { outcome, AccountRangeOutcome::Verified(VerifiedAccountRange { state_root: root_hash, + origin: B256::ZERO, accounts: vec![accounts[0]], has_more: false, next: Some(key(9)), @@ -710,6 +739,7 @@ mod tests { outcome, AccountRangeOutcome::Verified(VerifiedAccountRange { state_root: root_hash, + origin: B256::ZERO, accounts: vec![accounts[0]], has_more: true, next: Some(key(3)), diff --git a/crates/net/downloaders/src/snap/storage.rs b/crates/net/downloaders/src/snap/storage.rs index fcf117f4e40..50fd1f9528b 100644 --- a/crates/net/downloaders/src/snap/storage.rs +++ b/crates/net/downloaders/src/snap/storage.rs @@ -492,6 +492,7 @@ mod tests { fn verified_range(accounts: &[(B256, TrieAccount)]) -> VerifiedAccountRange { VerifiedAccountRange { state_root: STATE_ROOT, + origin: B256::ZERO, accounts: accounts.to_vec(), has_more: false, next: None, diff --git a/crates/snap-sync/Cargo.toml b/crates/snap-sync/Cargo.toml index 93585aac909..cfe053cab7a 100644 --- a/crates/snap-sync/Cargo.toml +++ b/crates/snap-sync/Cargo.toml @@ -13,22 +13,34 @@ workspace = true [dependencies] # reth +reth-db-api.workspace = true +reth-downloaders.workspace = true +reth-eth-wire-types.workspace = true +reth-network-p2p.workspace = true +reth-network-peers.workspace = true reth-primitives-traits.workspace = true reth-storage-api = { workspace = true, features = ["db-api"] } reth-storage-errors.workspace = true +reth-tasks.workspace = true +reth-trie-common.workspace = true # alloy alloy-eip7928.workspace = true alloy-eips.workspace = true alloy-primitives.workspace = true +# revm +revm.workspace = true + # misc +serde = { workspace = true, features = ["derive"] } +serde_json.workspace = true thiserror.workspace = true tokio-util.workspace = true [dev-dependencies] alloy-consensus.workspace = true -reth-db-api.workspace = true +alloy-rlp.workspace = true +futures.workspace = true reth-provider = { workspace = true, features = ["test-utils"] } -reth-trie-common.workspace = true -revm.workspace = true +tokio = { workspace = true, features = ["macros", "rt-multi-thread"] } diff --git a/crates/snap-sync/src/account/download.rs b/crates/snap-sync/src/account/download.rs new file mode 100644 index 00000000000..04f81c9bf29 --- /dev/null +++ b/crates/snap-sync/src/account/download.rs @@ -0,0 +1,406 @@ +//! Downloads account ranges in key order and commits each under the write it was fetched with. + +use crate::{AccountCoverage, SnapAccountStore, SnapAttemptStore, SnapSyncError, SnapWrite}; +use alloy_primitives::{map::B256Map, B256}; +use reth_db_api::transaction::DbTxMut; +use reth_downloaders::snap::{AccountRangeDownloader, AccountRangeOutcome, VerifiedAccountRange}; +use reth_eth_wire_types::snap::GetAccountRangeMessage; +use reth_network_p2p::snap::client::SnapClient; +use reth_network_peers::PeerId; +use reth_storage_api::{ + DBProvider, DatabaseProviderFactory, MetadataProvider, MetadataWriter, StateWriter, +}; +use reth_storage_errors::provider::ProviderError; +use reth_tasks::Runtime; +use reth_trie_common::HashedStorage; +use revm::bytecode::Bytecode; +use std::fmt; + +/// Default soft response limit for account-range requests, matching common peer limits. +pub const DEFAULT_RESPONSE_BYTES: u64 = 512 * 1024; + +/// Inclusive upper bound covering the full account trie keyspace. +pub const MAX_HASH: B256 = B256::new([0xff; B256::len_bytes()]); + +/// Downloads the account ranges an attempt still needs, one at a time in key order. +/// +/// [`Self::next`] fetches the range at the coverage cursor the store records; the caller resolves +/// its storage and code, then [`Self::commit`] persists everything and moves the cursor. +pub struct AccountRangeDownload { + client: C, + factory: F, + // Proof verification and commits run on the blocking pool. + runtime: Runtime, + // Coverage as last read from the store; none before the first request. + coverage: Option, + response_bytes: u64, + // Distinguishes responses to reissued requests. + request_id: u64, +} + +impl AccountRangeDownload { + /// Creates a download that continues from the coverage the store records. + pub const fn new(client: C, factory: F, runtime: Runtime) -> Self { + Self { + client, + factory, + runtime, + coverage: None, + response_bytes: DEFAULT_RESPONSE_BYTES, + request_id: 0, + } + } + + /// Returns this download asking peers for at most `response_bytes` per response. + pub const fn with_response_bytes(mut self, response_bytes: u64) -> Self { + self.response_bytes = response_bytes; + self + } + + /// How far the download has got, as last read from the store. + pub const fn coverage(&self) -> Option { + self.coverage + } +} + +impl AccountRangeDownload +where + C: SnapClient + Clone + Unpin, + F: DatabaseProviderFactory + Clone + 'static, + F::Provider: MetadataProvider, + F::ProviderRW: MetadataProvider + MetadataWriter + StateWriter + DBProvider, +{ + /// Fetches the range at the coverage cursor. + /// + /// `Ok(None)` once every account is downloaded. A failed request leaves the cursor where it + /// was, so the download can be resumed from the persisted coverage. + pub async fn next(&mut self) -> Result, SnapSyncError> { + let (write, root_hash, coverage) = self.active_write()?; + self.coverage = Some(coverage); + let Some(origin) = coverage.next() else { return Ok(None) }; + + self.request_id = self.request_id.wrapping_add(1); + let request = GetAccountRangeMessage { + request_id: self.request_id, + root_hash, + starting_hash: origin, + limit_hash: MAX_HASH, + response_bytes: self.response_bytes, + }; + let downloader = + AccountRangeDownloader::new(self.client.clone(), request, self.runtime.clone()) + .expect("origin never exceeds the maximum hash"); + + Ok(Some(match downloader.await? { + AccountRangeOutcome::Verified(range) => { + AccountRangeStep::Verified(VerifiedRange { write, range }) + } + AccountRangeOutcome::Unavailable { peer_id } => { + AccountRangeStep::Unavailable { origin, peer_id } + } + })) + } + + /// Commits `verified` with the storage and code resolved for it, moving the cursor past it. + /// + /// Refused, with the coverage unchanged, when the attempt no longer accepts the write the + /// range was fetched under or a dependency is missing. + pub async fn commit( + &mut self, + verified: VerifiedRange, + storages: B256Map, + bytecodes: Vec<(B256, Bytecode)>, + ) -> Result { + let factory = self.factory.clone(); + let coverage = self + .runtime + .spawn_blocking(move || -> Result { + let VerifiedRange { write, range } = verified; + let provider = factory.database_provider_rw()?; + let coverage = provider.commit_account_range(write, &range, storages, bytecodes)?; + provider.commit()?; + Ok(coverage) + }) + .await + .map_err(|error| SnapSyncError::Provider(ProviderError::other(error)))??; + self.coverage = Some(coverage); + Ok(coverage) + } + + // The write the attempt accepts right now, the root to request against, and the coverage the + // store records for it. + fn active_write(&self) -> Result<(SnapWrite, B256, AccountCoverage), SnapSyncError> { + let provider = self.factory.database_provider_ro()?; + let write = provider.active_snap_write()?.ok_or(SnapSyncError::NoAttempt)?; + let root = provider.authorize_snap_write(write)?.state_root(); + let coverage = provider.account_coverage(write)?.ok_or(SnapSyncError::NoCoverage)?; + Ok((write, root, coverage)) + } +} + +impl fmt::Debug for AccountRangeDownload { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("AccountRangeDownload") + .field("coverage", &self.coverage) + .field("response_bytes", &self.response_bytes) + .field("request_id", &self.request_id) + .finish_non_exhaustive() + } +} + +/// What one request of an [`AccountRangeDownload`] produced. +#[derive(Debug)] +pub enum AccountRangeStep { + /// A range authenticated against the pivot root, waiting for its dependencies. + Verified(VerifiedRange), + /// The peer did not serve the range, so the cursor stays at `origin`. + Unavailable { + /// Key the range was requested from. + origin: B256, + /// Peer that answered without the state, so the retry can go elsewhere. + peer_id: PeerId, + }, +} + +/// A verified range with the write it was fetched under. +/// +/// Only [`AccountRangeDownload::commit`] consumes it, so the range is committed under the +/// attempt that was active when it was requested, never a later one. +#[derive(Debug)] +pub struct VerifiedRange { + // Write the attempt accepted when the range was requested. + write: SnapWrite, + // Accounts authenticated against that attempt's root. + range: VerifiedAccountRange, +} + +impl VerifiedRange { + /// Key the range was requested from. + pub const fn origin(&self) -> B256 { + self.range.origin() + } + + /// The accounts, whose storage and code the commit needs. + pub const fn range(&self) -> &VerifiedAccountRange { + &self.range + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::test_utils::{ + account, account_range, generation, hashed_factory, key, state_root, verified_range, + ScriptedSnapClient, + }; + use reth_db_api::{cursor::DbCursorRO, tables, transaction::DbTx}; + use reth_eth_wire_types::snap::AccountRangeMessage; + use reth_network_p2p::{ + error::{PeerRequestResult, RequestError}, + snap::client::SnapResponse, + }; + use reth_network_peers::WithPeerId; + use reth_provider::{test_utils::MockNodeTypesWithDB, ProviderFactory}; + use reth_trie_common::TrieAccount; + use std::sync::Arc; + + const FAR: B256 = B256::repeat_byte(0xaa); + + fn accounts() -> Vec<(B256, TrieAccount)> { + vec![(key(1), account(1)), (key(2), account(2)), (FAR, account(3))] + } + + fn started(accounts: &[(B256, TrieAccount)]) -> ProviderFactory { + let factory = hashed_factory(); + let provider = factory.database_provider_rw().unwrap(); + let write = provider.start_snap_attempt(generation(1, state_root(accounts))).unwrap(); + provider.start_account_coverage(write).unwrap(); + provider.commit().unwrap(); + factory + } + + type Download = + AccountRangeDownload, ProviderFactory>; + + fn download( + responses: impl IntoIterator>, + factory: ProviderFactory, + ) -> (Arc, Download) { + let client = Arc::new(ScriptedSnapClient::new(responses)); + let download = AccountRangeDownload::new(Arc::clone(&client), factory, Runtime::test()); + (client, download) + } + + fn stored_accounts(factory: &ProviderFactory) -> Vec { + let provider = factory.database_provider_ro().unwrap(); + let mut cursor = provider.tx_ref().cursor_read::().unwrap(); + cursor.walk(None).unwrap().map(|entry| entry.unwrap().0).collect() + } + + async fn verified(download: &mut Download) -> VerifiedRange { + match download.next().await.unwrap().unwrap() { + AccountRangeStep::Verified(verified) => verified, + AccountRangeStep::Unavailable { .. } => panic!("fixture serves the range"), + } + } + + #[tokio::test] + async fn downloads_the_trie_in_key_order_and_stops() { + let accounts = accounts(); + let factory = started(&accounts); + let responses = [ + // The first account only; the proof shows key 2 follows. + account_range(1, &accounts, 0..1, &[key(1)]), + // Everything from key 2 on. + account_range(2, &accounts, 1..3, &[key(2), FAR]), + ]; + let (client, mut download) = download(responses, factory.clone()); + + let first = verified(&mut download).await; + assert_eq!(first.origin(), B256::ZERO); + assert_eq!(first.range().accounts().len(), 1); + let coverage = download.commit(first, Default::default(), Vec::new()).await.unwrap(); + assert_eq!(coverage.next(), Some(key(2))); + + let second = verified(&mut download).await; + assert_eq!(second.origin(), key(2)); + let coverage = download.commit(second, Default::default(), Vec::new()).await.unwrap(); + assert!(coverage.is_complete()); + + assert!(download.next().await.unwrap().is_none()); + assert_eq!(*client.origins(), [B256::ZERO, key(2)]); + assert_eq!(stored_accounts(&factory), [key(1), key(2), FAR]); + } + + #[tokio::test] + async fn an_unavailable_response_names_the_peer_and_leaves_the_cursor_in_place() { + let accounts = accounts(); + let factory = started(&accounts); + let peer = PeerId::random(); + let empty = AccountRangeMessage { request_id: 1, accounts: Vec::new(), proof: Vec::new() }; + let responses = [ + Ok(WithPeerId::new(peer, SnapResponse::AccountRange(empty))), + account_range(2, &accounts, 0..3, &[]), + ]; + let (client, mut download) = download(responses, factory.clone()); + + let step = download.next().await.unwrap().unwrap(); + + assert!(matches!( + step, + AccountRangeStep::Unavailable { origin, peer_id } + if origin == B256::ZERO && peer_id == peer + )); + assert_eq!(download.coverage(), Some(AccountCoverage::START)); + + let range = verified(&mut download).await; + download.commit(range, Default::default(), Vec::new()).await.unwrap(); + assert!(download.coverage().unwrap().is_complete()); + assert_eq!(*client.origins(), [B256::ZERO, B256::ZERO]); + } + + #[tokio::test] + async fn a_failed_request_leaves_the_cursor_in_place() { + let factory = started(&accounts()); + let (_, mut download) = + download([Err(RequestError::UnsupportedCapability)], factory.clone()); + + let error = download.next().await.unwrap_err(); + + assert!(matches!(error, SnapSyncError::Request(RequestError::UnsupportedCapability))); + assert_eq!(download.coverage(), Some(AccountCoverage::START)); + assert!(stored_accounts(&factory).is_empty()); + } + + #[tokio::test] + async fn the_download_continues_from_the_coverage_the_store_records() { + let accounts = accounts(); + let factory = started(&accounts); + // A previous run committed the first account. + let provider = factory.database_provider_rw().unwrap(); + let write = provider.active_snap_write().unwrap().unwrap(); + let head = verified_range(&accounts, 0..1, B256::ZERO, &[key(1)]); + provider.commit_account_range(write, &head, Default::default(), Vec::new()).unwrap(); + provider.commit().unwrap(); + let (client, mut download) = + download([account_range(1, &accounts, 1..3, &[key(2), FAR])], factory.clone()); + + let rest = verified(&mut download).await; + + assert_eq!(rest.origin(), key(2)); + assert_eq!(*client.origins(), [key(2)]); + download.commit(rest, Default::default(), Vec::new()).await.unwrap(); + assert!(download.coverage().unwrap().is_complete()); + assert_eq!(stored_accounts(&factory), [key(1), key(2), FAR]); + } + + #[tokio::test] + async fn a_range_fetched_under_a_replaced_attempt_is_refused_at_commit() { + let accounts = accounts(); + let factory = started(&accounts); + let (_, mut download) = download([account_range(1, &accounts, 0..3, &[])], factory.clone()); + let range = verified(&mut download).await; + // The attempt is replaced after the range was fetched. + let provider = factory.database_provider_rw().unwrap(); + provider.start_snap_attempt(generation(1, state_root(&accounts))).unwrap(); + provider.commit().unwrap(); + + let error = download.commit(range, Default::default(), Vec::new()).await.unwrap_err(); + + assert!(matches!(error, SnapSyncError::StaleWrite { .. })); + assert!(stored_accounts(&factory).is_empty()); + assert_eq!(download.coverage(), Some(AccountCoverage::START)); + } + + #[tokio::test] + async fn a_range_missing_its_dependencies_is_refused_at_commit() { + let mut accounts = accounts(); + accounts[1].1.code_hash = B256::repeat_byte(0x33); + let factory = started(&accounts); + let (_, mut download) = download([account_range(1, &accounts, 0..3, &[])], factory.clone()); + let range = verified(&mut download).await; + + let error = download.commit(range, Default::default(), Vec::new()).await.unwrap_err(); + + assert!(matches!(error, SnapSyncError::MissingCode { .. })); + assert!(stored_accounts(&factory).is_empty()); + assert_eq!(download.coverage(), Some(AccountCoverage::START)); + } + + #[tokio::test] + async fn nothing_is_requested_without_an_active_attempt() { + let (client, mut download) = download([], hashed_factory()); + + assert!(matches!(download.next().await, Err(SnapSyncError::NoAttempt))); + assert!(client.origins().is_empty()); + assert_eq!(download.coverage(), None); + } + + #[tokio::test] + async fn nothing_is_requested_without_recorded_coverage() { + let factory = hashed_factory(); + let provider = factory.database_provider_rw().unwrap(); + provider.start_snap_attempt(generation(1, state_root(&accounts()))).unwrap(); + provider.commit().unwrap(); + let (client, mut download) = download([], factory); + + assert!(matches!(download.next().await, Err(SnapSyncError::NoCoverage))); + assert!(client.origins().is_empty()); + } + + #[tokio::test] + async fn a_complete_coverage_requests_nothing() { + let accounts = accounts(); + let factory = started(&accounts); + let provider = factory.database_provider_rw().unwrap(); + let write = provider.active_snap_write().unwrap().unwrap(); + let whole = verified_range(&accounts, 0..3, B256::ZERO, &[]); + provider.commit_account_range(write, &whole, Default::default(), Vec::new()).unwrap(); + provider.commit().unwrap(); + let (client, mut download) = download([], factory); + + assert!(download.next().await.unwrap().is_none()); + assert_eq!(download.coverage(), Some(AccountCoverage::COMPLETE)); + assert!(client.origins().is_empty()); + } +} diff --git a/crates/snap-sync/src/account/mod.rs b/crates/snap-sync/src/account/mod.rs new file mode 100644 index 00000000000..a92238fbef9 --- /dev/null +++ b/crates/snap-sync/src/account/mod.rs @@ -0,0 +1,9 @@ +//! Account-range download coordination and persistence. + +mod download; +mod store; + +pub use download::{ + AccountRangeDownload, AccountRangeStep, VerifiedRange, DEFAULT_RESPONSE_BYTES, MAX_HASH, +}; +pub use store::{AccountCoverage, SnapAccountStore}; diff --git a/crates/snap-sync/src/account/store.rs b/crates/snap-sync/src/account/store.rs new file mode 100644 index 00000000000..d6df887edf9 --- /dev/null +++ b/crates/snap-sync/src/account/store.rs @@ -0,0 +1,642 @@ +//! Persists account ranges, their storage and code, and the coverage cursor in one transaction. +//! +//! Each range replaces its key interval, and ranges commit in key order. + +use crate::{SnapAttemptStore, SnapSyncError, SnapWrite}; +use alloy_primitives::{ + map::{B256Map, B256Set}, + B256, KECCAK256_EMPTY, +}; +use reth_db_api::{ + tables, + transaction::{DbTx, DbTxMut}, + RawKey, RawTable, +}; +use reth_downloaders::snap::VerifiedAccountRange; +use reth_primitives_traits::Account; +use reth_storage_api::{DBProvider, MetadataProvider, MetadataWriter, SnapAttemptId, StateWriter}; +use reth_storage_errors::provider::ProviderError; +use reth_trie_common::{ + root::storage_root, HashedPostState, HashedPostStateSorted, HashedStorage, TrieAccount, + EMPTY_ROOT_HASH, +}; +use revm::{bytecode::Bytecode, database::states::StateChangeset}; +use serde::{Deserialize, Serialize}; +use std::ops::Bound; + +// Metadata key of the coverage record. +const COVERAGE_KEY: &str = "snap_account_coverage"; + +// Encoding version of the coverage record this build writes. +const COVERAGE_VERSION: u32 = 1; + +/// Persistence for the account ranges an attempt downloads. +/// +/// Blanket-implemented over the node's writers, so accounts, their dependencies and the coverage +/// join the caller's transaction and commit together or not at all. +pub trait SnapAccountStore { + /// Returns the coverage recorded for the attempt `write` belongs to, recording that no + /// account has been downloaded yet when there is none. + fn start_account_coverage(&self, write: SnapWrite) -> Result + where + Self: MetadataWriter; + + /// Returns the coverage recorded for the attempt `write` belongs to, if any. + fn account_coverage(&self, write: SnapWrite) -> Result, SnapSyncError>; + + /// Persists `range` with its storage and code, replacing its key interval. + /// + /// Storage must match each account's root, and code must be supplied or already stored. + fn commit_account_range( + &self, + write: SnapWrite, + range: &VerifiedAccountRange, + storages: B256Map, + bytecodes: Vec<(B256, Bytecode)>, + ) -> Result + where + Self: MetadataWriter + StateWriter + DBProvider; +} + +/// How far the account key space has been downloaded. +#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize, Deserialize)] +pub struct AccountCoverage { + // Next key to request, or none once the trie is exhausted. + next: Option, +} + +impl AccountCoverage { + /// Nothing downloaded yet. + pub const START: Self = Self { next: Some(B256::ZERO) }; + + /// Every account downloaded. + pub const COMPLETE: Self = Self { next: None }; + + /// Key the next range is requested from, or `None` once every account is downloaded. + pub const fn next(&self) -> Option { + self.next + } + + /// Returns whether every account is downloaded. + pub const fn is_complete(&self) -> bool { + self.next.is_none() + } + + // The coverage after `range`, which must start at this cursor and end past it. + fn advance(&self, range: &VerifiedAccountRange) -> Result { + let origin = range.origin(); + if self.next != Some(origin) { + return Err(SnapSyncError::OutOfOrderRange { expected: self.next, got: origin }) + } + if range.next().is_some_and(|next| next <= origin) { + return Err(SnapSyncError::NoProgress { origin }) + } + Ok(Self { next: range.next() }) + } + + // Keys from this cursor up to `to`, which a range proved hold only its own accounts. + fn interval_to(&self, to: &Self) -> (Bound, Bound) { + ( + self.next.map_or(Bound::Unbounded, Bound::Included), + to.next.map_or(Bound::Unbounded, Bound::Excluded), + ) + } +} + +// The coverage record as persisted, tied to the attempt that recorded it. +#[derive(Serialize, Deserialize)] +struct StoredCoverage { + // Encoding version, checked before the rest is decoded. + version: u32, + // Attempt the coverage belongs to. + attempt: SnapAttemptId, + // How far that attempt has downloaded. + coverage: AccountCoverage, +} + +impl StoredCoverage { + // Serializes `coverage` for `attempt` at this build's version. + fn encode(attempt: SnapAttemptId, coverage: AccountCoverage) -> Result, SnapSyncError> { + let stored = Self { version: COVERAGE_VERSION, attempt, coverage }; + Ok(serde_json::to_vec(&stored).map_err(ProviderError::other)?) + } + + // Checks the version first, so a record from another build is reported rather than misread. + fn decode(bytes: &[u8]) -> Result { + let value: serde_json::Value = + serde_json::from_slice(bytes).map_err(ProviderError::other)?; + let version = value.get("version").and_then(serde_json::Value::as_u64); + if version != Some(COVERAGE_VERSION as u64) { + return Err(SnapSyncError::UnsupportedCoverage { version }) + } + Ok(serde_json::from_value(value).map_err(ProviderError::other)?) + } +} + +impl SnapAccountStore for T { + // Resumes the recorded coverage, so restarting an attempt keeps its progress. + fn start_account_coverage(&self, write: SnapWrite) -> Result + where + Self: MetadataWriter, + { + if let Some(coverage) = self.account_coverage(write)? { + return Ok(coverage) + } + let start = AccountCoverage::START; + self.write_metadata(COVERAGE_KEY, StoredCoverage::encode(write.attempt(), start)?)?; + Ok(start) + } + + // A record left by another attempt reads as no coverage. + fn account_coverage(&self, write: SnapWrite) -> Result, SnapSyncError> { + self.authorize_snap_write(write)?; + let Some(bytes) = self.get_metadata(COVERAGE_KEY)? else { return Ok(None) }; + let stored = StoredCoverage::decode(&bytes)?; + Ok((stored.attempt == write.attempt()).then_some(stored.coverage)) + } + + // Every check runs before the first write, so a refused range changes nothing. + fn commit_account_range( + &self, + write: SnapWrite, + range: &VerifiedAccountRange, + storages: B256Map, + bytecodes: Vec<(B256, Bytecode)>, + ) -> Result + where + Self: MetadataWriter + StateWriter + DBProvider, + { + let attempt = self.authorize_snap_write(write)?; + if range.state_root() != attempt.state_root() { + return Err(SnapSyncError::RootMismatch { + expected: attempt.state_root(), + got: range.state_root(), + }) + } + let coverage = self.account_coverage(write)?.ok_or(SnapSyncError::NoCoverage)?; + let advanced = coverage.advance(range)?; + let dependencies = RangeDependencies::new(range.accounts(), storages, bytecodes); + dependencies.verify(range.accounts(), self.tx_ref())?; + + let interval = coverage.interval_to(&advanced); + self.remove::(interval)?; + self.remove::(interval)?; + dependencies.write(self)?; + self.write_metadata(COVERAGE_KEY, StoredCoverage::encode(write.attempt(), advanced)?)?; + Ok(advanced) + } +} + +// Accounts of a range with the storage and code supplied for them. +struct RangeDependencies { + // Accounts and storage, sorted once for both the root checks and the write. + state: HashedPostStateSorted, + // Code keyed by the hash it was supplied under. + bytecodes: Vec<(B256, Bytecode)>, +} + +impl RangeDependencies { + fn new( + accounts: &[(B256, TrieAccount)], + storages: B256Map, + bytecodes: Vec<(B256, Bytecode)>, + ) -> Self { + let state = HashedPostState::default() + .with_accounts( + accounts.iter().map(|(hash, account)| (*hash, Some(Account::from(*account)))), + ) + .with_storages(storages) + .into_sorted(); + Self { state, bytecodes } + } + + // Checks storage against each account's root, and code against its hash or the table. + fn verify( + &self, + accounts: &[(B256, TrieAccount)], + tx: &impl DbTx, + ) -> Result<(), SnapSyncError> { + let mut available = self.supplied_code()?; + let mut contracts = B256Set::default(); + for (hash, account) in accounts { + if self.verify_storage(*hash, account)? { + contracts.insert(*hash); + } + // Only presence matters, so stored code is not decoded. + if account.code_hash != KECCAK256_EMPTY && + available.insert(account.code_hash) && + tx.get::>(RawKey::new(account.code_hash))?.is_none() + { + return Err(SnapSyncError::MissingCode { hash: account.code_hash }) + } + } + if let Some(account) = + self.state.account_storages().keys().find(|hash| !contracts.contains(*hash)) + { + return Err(SnapSyncError::UnexpectedStorage { account: *account }) + } + Ok(()) + } + + // Hashes of the supplied code, refusing code filed under a hash it does not hash to. + fn supplied_code(&self) -> Result { + let mut hashes = B256Set::default(); + for (hash, code) in &self.bytecodes { + let got = code.hash_slow(); + if got != *hash { + return Err(SnapSyncError::CodeMismatch { expected: *hash, got }) + } + hashes.insert(*hash); + } + Ok(hashes) + } + + // Whether `account` has storage, which must be supplied and hash to its root. + fn verify_storage(&self, hash: B256, account: &TrieAccount) -> Result { + if account.storage_root == EMPTY_ROOT_HASH { + return Ok(false) + } + let storage = self + .state + .account_storages() + .get(&hash) + .ok_or(SnapSyncError::MissingStorage { account: hash })?; + // Zero slots are deletions, which the trie does not hold. + let got = storage_root( + storage.storage_slots_ref().iter().filter(|(_, value)| !value.is_zero()).copied(), + ); + if got != account.storage_root { + return Err(SnapSyncError::StorageRootMismatch { + account: hash, + expected: account.storage_root, + got, + }) + } + Ok(true) + } + + // Writes the accounts with their storage and code. + fn write(self, writer: &impl StateWriter) -> Result<(), SnapSyncError> { + writer.write_hashed_state(&self.state)?; + writer.write_state_changes(StateChangeset { + contracts: self.bytecodes, + ..Default::default() + })?; + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{ + test_utils::{account, generation, hashed_factory, key, state_root, verified_range}, + SnapGeneration, + }; + use alloy_primitives::{Bytes, U256}; + use reth_db_api::cursor::DbCursorRO; + use reth_provider::{ + test_utils::MockNodeTypesWithDB, DatabaseProviderFactory, ProviderFactory, + }; + use reth_trie_common::root::storage_root_unsorted; + + const FAR: B256 = B256::repeat_byte(0xaa); + const SLOT: B256 = B256::repeat_byte(0x55); + + fn code() -> Bytecode { + Bytecode::new_raw(Bytes::from_static(&[0x60, 0x00])) + } + + // Storage holding one slot, with the root the contract fixture commits to. + fn storage() -> (B256, HashedStorage) { + let storage = HashedStorage::from_iter([(SLOT, U256::from(7))]); + ( + storage_root_unsorted(storage.storage.iter().map(|(slot, value)| (*slot, *value))), + storage, + ) + } + + // A plain account, a contract with storage and code, and a far account. + fn accounts() -> Vec<(B256, TrieAccount)> { + let mut contract = account(2); + contract.storage_root = storage().0; + contract.code_hash = code().hash_slow(); + vec![(key(1), account(1)), (key(2), contract), (FAR, account(3))] + } + + // Storage and code that satisfy the contract in `accounts`. + fn dependencies() -> (B256Map, Vec<(B256, Bytecode)>) { + (B256Map::from_iter([(key(2), storage().1)]), vec![(code().hash_slow(), code())]) + } + + // An attempt anchored to the trie's root, with its coverage recorded. + fn started( + accounts: &[(B256, TrieAccount)], + ) -> (ProviderFactory, SnapWrite, SnapGeneration) { + let factory = hashed_factory(); + let generation = generation(1, state_root(accounts)); + let provider = factory.database_provider_rw().unwrap(); + let write = provider.start_snap_attempt(generation).unwrap(); + provider.start_account_coverage(write).unwrap(); + provider.commit().unwrap(); + (factory, write, generation) + } + + fn stored_accounts(provider: &impl DBProvider) -> Vec { + let mut cursor = provider.tx_ref().cursor_read::().unwrap(); + cursor.walk(None).unwrap().map(|entry| entry.unwrap().0).collect() + } + + fn stored(provider: &impl DBProvider) -> (Vec, bool, bool) { + let tx = provider.tx_ref(); + ( + stored_accounts(provider), + tx.get::(key(2)).unwrap().is_some(), + tx.get::(code().hash_slow()).unwrap().is_some(), + ) + } + + #[test] + fn accounts_dependencies_and_coverage_commit_together() { + let accounts = accounts(); + let (factory, write, _) = started(&accounts); + let range = verified_range(&accounts, 0..3, B256::ZERO, &[]); + let (storages, bytecodes) = dependencies(); + + let provider = factory.database_provider_rw().unwrap(); + let coverage = provider.commit_account_range(write, &range, storages, bytecodes).unwrap(); + provider.commit().unwrap(); + + assert!(coverage.is_complete()); + let provider = factory.database_provider_rw().unwrap(); + assert_eq!(stored(&provider), (vec![key(1), key(2), FAR], true, true)); + assert_eq!(provider.account_coverage(write).unwrap(), Some(coverage)); + } + + #[test] + fn an_interrupted_commit_leaves_nothing_behind() { + let accounts = accounts(); + let (factory, write, _) = started(&accounts); + let range = verified_range(&accounts, 0..3, B256::ZERO, &[]); + let (storages, bytecodes) = dependencies(); + + let provider = factory.database_provider_rw().unwrap(); + provider.commit_account_range(write, &range, storages, bytecodes).unwrap(); + drop(provider); + + let provider = factory.database_provider_rw().unwrap(); + assert_eq!(stored(&provider), (Vec::new(), false, false)); + assert_eq!(provider.account_coverage(write).unwrap(), Some(AccountCoverage::START)); + } + + #[test] + fn a_partial_range_moves_the_coverage_to_its_next_key() { + let accounts = accounts(); + let (factory, write, _) = started(&accounts); + // Only the first account, with a proof placing the next one at key 2. + let range = verified_range(&accounts, 0..1, B256::ZERO, &[key(1)]); + + let provider = factory.database_provider_rw().unwrap(); + let coverage = + provider.commit_account_range(write, &range, Default::default(), Vec::new()).unwrap(); + + assert_eq!(coverage.next(), Some(key(2))); + assert_eq!(stored_accounts(&provider), [key(1)]); + } + + #[test] + fn a_proven_empty_tail_completes_the_coverage() { + let accounts = vec![(key(1), account(1)), (key(2), account(2))]; + let (factory, write, _) = started(&accounts); + let tail = verified_range(&accounts, 0..0, key(3), &[key(3)]); + assert!(tail.accounts().is_empty()); + let provider = factory.database_provider_rw().unwrap(); + let coverage = AccountCoverage { next: Some(key(3)) }; + provider + .write_metadata( + COVERAGE_KEY, + StoredCoverage::encode(write.attempt(), coverage).unwrap(), + ) + .unwrap(); + + let coverage = + provider.commit_account_range(write, &tail, Default::default(), Vec::new()).unwrap(); + + assert!(coverage.is_complete()); + assert!(stored_accounts(&provider).is_empty()); + } + + #[test] + fn a_range_from_another_origin_changes_nothing() { + let accounts = accounts(); + let (factory, write, _) = started(&accounts); + let provider = factory.database_provider_rw().unwrap(); + let first = verified_range(&accounts, 0..1, B256::ZERO, &[key(1)]); + provider.commit_account_range(write, &first, Default::default(), Vec::new()).unwrap(); + + // The same range again, and one skipping ahead of the cursor. + let duplicate = + provider.commit_account_range(write, &first, Default::default(), Vec::new()); + let skipped = verified_range(&accounts, 2..3, FAR, &[FAR]); + let ahead = provider.commit_account_range(write, &skipped, Default::default(), Vec::new()); + + assert!(matches!(duplicate, Err(SnapSyncError::OutOfOrderRange { .. }))); + assert!(matches!(ahead, Err(SnapSyncError::OutOfOrderRange { .. }))); + assert_eq!(provider.account_coverage(write).unwrap().unwrap().next(), Some(key(2))); + assert_eq!(stored_accounts(&provider), [key(1)]); + } + + #[test] + fn a_contract_without_its_storage_or_code_is_refused() { + let accounts = accounts(); + let (factory, write, _) = started(&accounts); + let range = verified_range(&accounts, 0..3, B256::ZERO, &[]); + let (storages, bytecodes) = dependencies(); + let provider = factory.database_provider_rw().unwrap(); + + let no_storage = + provider.commit_account_range(write, &range, Default::default(), bytecodes.clone()); + assert!( + matches!(no_storage, Err(SnapSyncError::MissingStorage { account }) if account == key(2)) + ); + + let no_code = provider.commit_account_range(write, &range, storages, Vec::new()); + assert!(matches!(no_code, Err(SnapSyncError::MissingCode { .. }))); + + let stray = B256Map::from_iter([(key(1), storage().1), (key(2), storage().1)]); + let unexpected = provider.commit_account_range(write, &range, stray, bytecodes); + assert!( + matches!(unexpected, Err(SnapSyncError::UnexpectedStorage { account }) if account == key(1)) + ); + + assert_eq!(stored(&provider), (Vec::new(), false, false)); + assert_eq!(provider.account_coverage(write).unwrap(), Some(AccountCoverage::START)); + } + + #[test] + fn storage_and_code_are_checked_against_what_the_account_commits_to() { + let accounts = accounts(); + let (factory, write, _) = started(&accounts); + let range = verified_range(&accounts, 0..3, B256::ZERO, &[]); + let (storages, bytecodes) = dependencies(); + let provider = factory.database_provider_rw().unwrap(); + + // A slot short of the storage root, as a partial download would be. + let partial = B256Map::from_iter([(key(2), HashedStorage::default())]); + let short = provider.commit_account_range(write, &range, partial, bytecodes); + assert!(matches!( + short, + Err(SnapSyncError::StorageRootMismatch { account, expected, .. }) + if account == key(2) && expected == storage().0 + )); + + // Code filed under a hash it does not hash to. + let other = Bytecode::new_raw(Bytes::from_static(&[0x60, 0x01])); + let relabelled = vec![(code().hash_slow(), other)]; + let wrong = provider.commit_account_range(write, &range, storages, relabelled); + assert!(matches!( + wrong, + Err(SnapSyncError::CodeMismatch { expected, .. }) if expected == code().hash_slow() + )); + + assert_eq!(stored(&provider), (Vec::new(), false, false)); + assert_eq!(provider.account_coverage(write).unwrap(), Some(AccountCoverage::START)); + } + + #[test] + fn code_already_stored_need_not_be_supplied() { + let accounts = accounts(); + let (factory, write, _) = started(&accounts); + let range = verified_range(&accounts, 0..3, B256::ZERO, &[]); + let (storages, _) = dependencies(); + let provider = factory.database_provider_rw().unwrap(); + provider + .write_state_changes(StateChangeset { + contracts: vec![(code().hash_slow(), code())], + ..Default::default() + }) + .unwrap(); + + let coverage = provider.commit_account_range(write, &range, storages, Vec::new()).unwrap(); + + assert!(coverage.is_complete()); + } + + #[test] + fn starting_coverage_again_resumes_where_the_attempt_left_off() { + let accounts = accounts(); + let (factory, write, _) = started(&accounts); + let range = verified_range(&accounts, 0..1, B256::ZERO, &[key(1)]); + let provider = factory.database_provider_rw().unwrap(); + let committed = + provider.commit_account_range(write, &range, Default::default(), Vec::new()).unwrap(); + + assert_eq!(provider.start_account_coverage(write).unwrap(), committed); + assert_eq!(provider.account_coverage(write).unwrap(), Some(committed)); + } + + #[test] + fn a_write_from_a_replaced_attempt_changes_nothing() { + let accounts = accounts(); + let (factory, replaced, generation) = started(&accounts); + let range = verified_range(&accounts, 0..3, B256::ZERO, &[]); + let (storages, bytecodes) = dependencies(); + let provider = factory.database_provider_rw().unwrap(); + let current = provider.start_snap_attempt(generation).unwrap(); + + let refused = provider.commit_account_range(replaced, &range, storages, bytecodes); + + assert!(matches!(refused, Err(SnapSyncError::StaleWrite { .. }))); + assert_eq!(stored(&provider), (Vec::new(), false, false)); + // The replaced attempt's coverage is not the new attempt's either. + assert_eq!(provider.account_coverage(current).unwrap(), None); + assert!(matches!( + provider.account_coverage(replaced), + Err(SnapSyncError::StaleWrite { .. }) + )); + } + + #[test] + fn a_new_attempt_replaces_what_an_earlier_one_left_in_the_interval() { + let earlier = accounts(); + let (factory, write, _) = started(&earlier); + let (storages, bytecodes) = dependencies(); + let provider = factory.database_provider_rw().unwrap(); + let range = verified_range(&earlier, 0..3, B256::ZERO, &[]); + provider.commit_account_range(write, &range, storages, bytecodes).unwrap(); + provider.commit().unwrap(); + + // At the new root the contract is gone and the first account changed. + let current = vec![(key(1), account(9)), (FAR, account(3))]; + let provider = factory.database_provider_rw().unwrap(); + let write = provider.start_snap_attempt(generation(2, state_root(¤t))).unwrap(); + provider.start_account_coverage(write).unwrap(); + let range = verified_range(¤t, 0..2, B256::ZERO, &[]); + let coverage = + provider.commit_account_range(write, &range, Default::default(), Vec::new()).unwrap(); + + assert!(coverage.is_complete()); + assert_eq!(stored(&provider), (vec![key(1), FAR], false, true)); + let first = provider.tx_ref().get::(key(1)).unwrap().unwrap(); + assert_eq!(first.nonce, 9); + } + + #[test] + fn a_range_proved_against_another_root_is_refused() { + let accounts = accounts(); + let (factory, write, _) = started(&accounts); + let provider = factory.database_provider_rw().unwrap(); + let other = vec![(key(7), account(7))]; + let range = verified_range(&other, 0..1, B256::ZERO, &[]); + + let refused = provider.commit_account_range(write, &range, Default::default(), Vec::new()); + + assert!(matches!(refused, Err(SnapSyncError::RootMismatch { .. }))); + assert!(stored_accounts(&provider).is_empty()); + } + + #[test] + fn a_range_without_recorded_coverage_is_refused() { + let accounts = accounts(); + let factory = hashed_factory(); + let provider = factory.database_provider_rw().unwrap(); + let write = provider.start_snap_attempt(generation(1, state_root(&accounts))).unwrap(); + let range = verified_range(&accounts, 0..3, B256::ZERO, &[]); + + assert!(matches!( + provider.commit_account_range(write, &range, Default::default(), Vec::new()), + Err(SnapSyncError::NoCoverage) + )); + } + + #[test] + fn coverage_survives_reopening_the_database() { + let accounts = accounts(); + let (factory, write, _) = started(&accounts); + let range = verified_range(&accounts, 0..1, B256::ZERO, &[key(1)]); + let provider = factory.database_provider_rw().unwrap(); + let coverage = + provider.commit_account_range(write, &range, Default::default(), Vec::new()).unwrap(); + provider.commit().unwrap(); + + let reopened = factory.database_provider_rw().unwrap(); + + assert_eq!(reopened.account_coverage(write).unwrap(), Some(coverage)); + } + + #[test] + fn a_record_this_build_cannot_read_is_reported() { + let accounts = accounts(); + let (factory, write, _) = started(&accounts); + + for record in [br#"{"version":999}"#.to_vec(), b"{}".to_vec()] { + let provider = factory.database_provider_rw().unwrap(); + provider.write_metadata(COVERAGE_KEY, record).unwrap(); + + assert!(matches!( + provider.account_coverage(write), + Err(SnapSyncError::UnsupportedCoverage { .. }) + )); + } + } +} diff --git a/crates/snap-sync/src/attempt.rs b/crates/snap-sync/src/attempt.rs index 270e4e1bfe1..fec3d65158d 100644 --- a/crates/snap-sync/src/attempt.rs +++ b/crates/snap-sync/src/attempt.rs @@ -15,7 +15,9 @@ use reth_storage_api::{ /// state, bytecode and the attempt record commit together or not at all. pub trait SnapAttemptStore { /// Starts an attempt anchored to `generation`, superseding any already recorded. - fn start_snap_attempt(&self, generation: SnapGeneration) -> Result; + fn start_snap_attempt(&self, generation: SnapGeneration) -> Result + where + Self: MetadataWriter; /// Returns the write an unfinished attempt accepts, if one owns the persisted state. fn active_snap_write(&self) -> Result, SnapSyncError>; @@ -28,13 +30,19 @@ pub trait SnapAttemptStore { &self, write: SnapWrite, generation: SnapGeneration, - ) -> Result; + ) -> Result + where + Self: MetadataWriter; /// Marks the attempt's downloaded state verified. - fn verify_snap_attempt(&self, write: SnapWrite) -> Result<(), SnapSyncError>; + fn verify_snap_attempt(&self, write: SnapWrite) -> Result<(), SnapSyncError> + where + Self: MetadataWriter; /// Gives up on an unfinished attempt, refusing its outstanding writes. - fn abandon_snap_attempt(&self) -> Result<(), SnapSyncError>; + fn abandon_snap_attempt(&self) -> Result<(), SnapSyncError> + where + Self: MetadataWriter; } /// What a write presents to prove it belongs to the attempt owning the persisted state. @@ -63,11 +71,11 @@ impl SnapWrite { } } -impl SnapAttemptStore for T -where - T: MetadataProvider + MetadataWriter, -{ - fn start_snap_attempt(&self, generation: SnapGeneration) -> Result { +impl SnapAttemptStore for T { + fn start_snap_attempt(&self, generation: SnapGeneration) -> Result + where + Self: MetadataWriter, + { // Absent settings mean the legacy layout. if !self.storage_settings()?.unwrap_or_else(StorageSettings::v1).use_hashed_state() { return Err(SnapSyncError::UnsupportedStorage) @@ -93,26 +101,34 @@ where } Ok(attempt) } - fn advance_snap_pivot( &self, write: SnapWrite, generation: SnapGeneration, - ) -> Result { + ) -> Result + where + Self: MetadataWriter, + { let mut attempt = self.authorize_snap_write(write)?; attempt.re_anchor(generation.target(), generation.state_root()); self.write_snap_attempt(&attempt)?; Ok(SnapWrite::of(&attempt)) } - fn verify_snap_attempt(&self, write: SnapWrite) -> Result<(), SnapSyncError> { + fn verify_snap_attempt(&self, write: SnapWrite) -> Result<(), SnapSyncError> + where + Self: MetadataWriter, + { let mut attempt = self.authorize_snap_write(write)?; attempt.verify(); self.write_snap_attempt(&attempt)?; Ok(()) } - fn abandon_snap_attempt(&self) -> Result<(), SnapSyncError> { + fn abandon_snap_attempt(&self) -> Result<(), SnapSyncError> + where + Self: MetadataWriter, + { if let Some(mut attempt) = self.snap_attempt()? && attempt.is_unfinished() { diff --git a/crates/snap-sync/src/error.rs b/crates/snap-sync/src/error.rs index 3234b1239c1..f99cb412d78 100644 --- a/crates/snap-sync/src/error.rs +++ b/crates/snap-sync/src/error.rs @@ -1,7 +1,9 @@ //! Failures raised while assembling a snap state generation. +use alloy_primitives::B256; +use reth_network_p2p::error::RequestError; use reth_storage_api::SnapAttemptId; -use reth_storage_errors::provider::ProviderError; +use reth_storage_errors::{db::DatabaseError, provider::ProviderError}; /// Error returned while assembling a snap state generation. #[derive(Debug, thiserror::Error)] @@ -9,6 +11,9 @@ pub enum SnapSyncError { /// A header lookup failed. #[error(transparent)] Provider(#[from] ProviderError), + /// A request for state failed. + #[error(transparent)] + Request(#[from] RequestError), /// The storage layout keys state by address, which snap cannot fill in without preimages. #[error("snap synchronization requires the hashed state layout")] UnsupportedStorage, @@ -23,4 +28,77 @@ pub enum SnapSyncError { /// State version the rejected write was proved against. state_version: u64, }, + /// A range was proved against a root other than the one the attempt downloads. + #[error("range proved against {got} while the attempt downloads {expected}")] + RootMismatch { + /// Root the attempt currently downloads. + expected: B256, + /// Root the range was proved against. + got: B256, + }, + /// No account coverage is recorded for the attempt. + #[error("no account coverage is recorded for the attempt")] + NoCoverage, + /// A persisted coverage record this build cannot read. + #[error("account coverage record version {version:?} is not supported")] + UnsupportedCoverage { + /// Version found on disk, absent when the record carries no numeric version. + version: Option, + }, + /// A range was downloaded from somewhere other than the next key to cover. + #[error("account range starts at {got}, coverage continues at {expected:?}")] + OutOfOrderRange { + /// Key the coverage continues at, or none once it is complete. + expected: Option, + /// Key the range was requested from. + got: B256, + }, + /// A verified range proved nothing past the key it was requested from. + #[error("account range made no progress past {origin}")] + NoProgress { + /// Key the range was requested from. + origin: B256, + }, + /// An account with storage was committed without it. + #[error("account {account} has storage that was not downloaded")] + MissingStorage { + /// Hashed address of the account. + account: B256, + }, + /// The storage supplied for an account does not hash to the root the account commits to. + #[error("storage supplied for {account} hashes to {got}, its account commits to {expected}")] + StorageRootMismatch { + /// Hashed address of the account. + account: B256, + /// Storage root the account commits to. + expected: B256, + /// Root of the supplied storage. + got: B256, + }, + /// Storage was supplied for an account that is not in the range or has none. + #[error("storage supplied for {account}, which is not a contract in the range")] + UnexpectedStorage { + /// Hashed address the storage was supplied for. + account: B256, + }, + /// An account's code is neither supplied nor stored. + #[error("code {hash} is neither supplied nor stored")] + MissingCode { + /// Hash of the missing code. + hash: B256, + }, + /// Code was supplied under a hash it does not hash to. + #[error("code supplied as {expected} hashes to {got}")] + CodeMismatch { + /// Hash the code was supplied under. + expected: B256, + /// Hash of the supplied code. + got: B256, + }, +} + +impl From for SnapSyncError { + fn from(error: DatabaseError) -> Self { + Self::Provider(error.into()) + } } diff --git a/crates/snap-sync/src/lib.rs b/crates/snap-sync/src/lib.rs index 9b652e3df60..8f0cc6654d2 100644 --- a/crates/snap-sync/src/lib.rs +++ b/crates/snap-sync/src/lib.rs @@ -10,7 +10,9 @@ //! matches the target header. //! //! Downloaded state goes into the hashed state tables, owned by an attempt record that commits -//! with it. Scheduling, in-flight requests and cancellation stay in memory. +//! with it, along with how far the account key space has been downloaded. An account range only +//! commits with storage matching its accounts' roots and code matching their hashes, so committed +//! progress never depends on work still pending. //! //! ``` //! use reth_snap_sync::SnapPivotPolicy; @@ -33,6 +35,7 @@ )] #![cfg_attr(not(test), warn(unused_crate_dependencies))] +mod account; mod attempt; mod error; mod generation; @@ -42,6 +45,10 @@ mod session; #[cfg(test)] mod test_utils; +pub use account::{ + AccountCoverage, AccountRangeDownload, AccountRangeStep, SnapAccountStore, VerifiedRange, + DEFAULT_RESPONSE_BYTES, MAX_HASH, +}; pub use attempt::{SnapAttemptStore, SnapWrite}; pub use error::SnapSyncError; pub use generation::{SnapGeneration, SnapPhase}; diff --git a/crates/snap-sync/src/test_utils.rs b/crates/snap-sync/src/test_utils.rs index 7df948657c4..5934dd8bbb2 100644 --- a/crates/snap-sync/src/test_utils.rs +++ b/crates/snap-sync/src/test_utils.rs @@ -1,9 +1,37 @@ -//! Header fixtures shared by the crate's tests. +//! Fixtures shared by the crate's tests: headers, an account trie and a scripted snap client. -use crate::SnapPivotPolicy; +use crate::{ + account::{DEFAULT_RESPONSE_BYTES, MAX_HASH}, + SnapGeneration, SnapPivotPolicy, +}; use alloy_consensus::Header; -use alloy_primitives::B256; -use reth_provider::test_utils::MockEthProvider; +use alloy_eips::BlockNumHash; +use alloy_primitives::{Bytes, B256, KECCAK256_EMPTY, U256}; +use futures::future::{ready, Ready}; +use reth_downloaders::snap::{AccountRangeDownloader, AccountRangeOutcome, VerifiedAccountRange}; +use reth_eth_wire_types::snap::{ + AccountData, AccountRangeMessage, GetAccountRangeMessage, GetBlockAccessListsMessage, + GetByteCodesMessage, GetStorageRangesMessage, +}; +use reth_network_p2p::{ + download::DownloadClient, + error::{PeerRequestResult, RequestError}, + priority::Priority, + snap::client::{SnapClient, SnapResponse}, +}; +use reth_network_peers::{PeerId, WithPeerId}; +use reth_provider::{ + test_utils::{create_test_provider_factory, MockEthProvider, MockNodeTypesWithDB}, + DatabaseProviderFactory, ProviderFactory, +}; +use reth_storage_api::{DBProvider, MetadataWriter, StorageSettings, StorageSettingsCache}; +use reth_tasks::Runtime; +use reth_trie_common::{proof::ProofRetainer, HashBuilder, Nibbles, TrieAccount, EMPTY_ROOT_HASH}; +use std::{ + collections::VecDeque, + ops::Range, + sync::{Mutex, MutexGuard}, +}; /// Small bounds keep header fixtures short without changing the policy's decisions. pub(crate) fn policy() -> SnapPivotPolicy { @@ -44,3 +72,186 @@ pub(crate) fn provider_with(headers: impl IntoIterator) -> MockEt provider.extend_headers(headers.into_iter().map(|header| (header.hash_slow(), header))); provider } + +/// A generation anchored to `block`, downloading against `state_root`. +pub(crate) fn generation(block: u64, state_root: B256) -> SnapGeneration { + SnapGeneration::new(BlockNumHash::new(block, B256::repeat_byte(block as u8)), state_root) +} + +/// A database using the hashed state layout snap writes into. +pub(crate) fn hashed_factory() -> ProviderFactory { + let factory = create_test_provider_factory(); + let provider = factory.database_provider_rw().unwrap(); + provider.write_storage_settings(StorageSettings::v2()).unwrap(); + provider.commit().unwrap(); + // Writers consult the cache, not the table. + factory.set_storage_settings_cache(StorageSettings::v2()); + factory +} + +/// A hashed account key in the lowest part of the key space. +pub(crate) fn key(value: u64) -> B256 { + B256::left_padding_from(&value.to_be_bytes()) +} + +/// An account without storage or code, distinguished by its nonce. +pub(crate) fn account(nonce: u64) -> TrieAccount { + TrieAccount { + nonce, + balance: U256::from(1), + storage_root: EMPTY_ROOT_HASH, + code_hash: KECCAK256_EMPTY, + } +} + +/// Root of the account trie holding `accounts`. +pub(crate) fn state_root(accounts: &[(B256, TrieAccount)]) -> B256 { + root_and_proof(accounts, &[]).0 +} + +// Root of the trie, and the proof nodes on the paths to `targets`. +fn root_and_proof(accounts: &[(B256, TrieAccount)], targets: &[B256]) -> (B256, Vec) { + let targets = targets.iter().copied().map(Nibbles::unpack).collect(); + let mut builder = HashBuilder::default().with_proof_retainer(ProofRetainer::new(targets)); + for (key, account) in accounts { + builder.add_leaf(Nibbles::unpack(*key), &alloy_rlp::encode(account)); + } + let root = builder.root(); + let proof = + builder.take_proof_nodes().into_nodes_sorted().into_iter().map(|(_, node)| node).collect(); + (root, proof) +} + +/// A peer's answer serving `accounts[served]` out of the trie holding `accounts`, proven along +/// the paths to `proof_targets`. No targets means the whole trie is served without a proof. +pub(crate) fn account_range( + request_id: u64, + accounts: &[(B256, TrieAccount)], + served: Range, + proof_targets: &[B256], +) -> PeerRequestResult { + // A complete trie needs no proof; the retainer would still emit the root node. + let proof = if proof_targets.is_empty() { + Vec::new() + } else { + root_and_proof(accounts, proof_targets).1 + }; + let message = AccountRangeMessage { + request_id, + accounts: accounts[served] + .iter() + .map(|(key, account)| AccountData::from_trie_account(*key, account)) + .collect(), + proof, + }; + Ok(WithPeerId::new(PeerId::random(), SnapResponse::AccountRange(message))) +} + +/// Runs the same verification production uses on [`account_range`]'s answer to a request from +/// `origin` through the end of the key space. +pub(crate) fn verified_range( + accounts: &[(B256, TrieAccount)], + served: Range, + origin: B256, + proof_targets: &[B256], +) -> VerifiedAccountRange { + let client = ScriptedSnapClient::new([account_range(1, accounts, served, proof_targets)]); + let request = GetAccountRangeMessage { + request_id: 1, + root_hash: state_root(accounts), + starting_hash: origin, + limit_hash: MAX_HASH, + response_bytes: DEFAULT_RESPONSE_BYTES, + }; + let downloader = AccountRangeDownloader::new(client, request, Runtime::test()).unwrap(); + match futures::executor::block_on(downloader).unwrap() { + AccountRangeOutcome::Verified(range) => range, + AccountRangeOutcome::Unavailable { .. } => panic!("fixture serves the requested root"), + } +} + +/// Serves scripted answers in request order, recording what each request asked for. +pub(crate) struct ScriptedSnapClient { + responses: Mutex>>, + origins: Mutex>, +} + +impl ScriptedSnapClient { + pub(crate) fn new( + responses: impl IntoIterator>, + ) -> Self { + Self { + responses: Mutex::new(responses.into_iter().collect()), + origins: Mutex::new(Vec::new()), + } + } + + /// Origins of the account range requests sent so far. + pub(crate) fn origins(&self) -> MutexGuard<'_, Vec> { + self.origins.lock().unwrap() + } +} + +impl std::fmt::Debug for ScriptedSnapClient { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("ScriptedSnapClient").finish_non_exhaustive() + } +} + +impl DownloadClient for ScriptedSnapClient { + fn report_bad_message(&self, _peer_id: PeerId) {} + + fn num_connected_peers(&self) -> usize { + 1 + } +} + +impl SnapClient for ScriptedSnapClient { + type Output = Ready>; + + fn get_account_range_with_priority( + &self, + request: GetAccountRangeMessage, + _priority: Priority, + ) -> Self::Output { + self.origins.lock().unwrap().push(request.starting_hash); + let response = self.responses.lock().unwrap().pop_front(); + ready(response.unwrap_or(Err(RequestError::ChannelClosed))) + } + + fn get_storage_ranges(&self, _request: GetStorageRangesMessage) -> Self::Output { + unsupported() + } + + fn get_storage_ranges_with_priority( + &self, + _request: GetStorageRangesMessage, + _priority: Priority, + ) -> Self::Output { + unsupported() + } + + fn get_byte_codes(&self, _request: GetByteCodesMessage) -> Self::Output { + unsupported() + } + + fn get_byte_codes_with_priority( + &self, + _request: GetByteCodesMessage, + _priority: Priority, + ) -> Self::Output { + unsupported() + } + + fn get_block_access_lists_with_priority( + &self, + _request: GetBlockAccessListsMessage, + _priority: Priority, + ) -> Self::Output { + unsupported() + } +} + +fn unsupported() -> Ready> { + ready(Err(RequestError::UnsupportedCapability)) +} From 4b01c866c8743cb1a26d0786ea858a196a8d15f2 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Mon, 14 Sep 2026 14:02:31 +0000 Subject: [PATCH 250/294] fix(download): restore custom static files directory handling (#27065) --- bin/reth/tests/it/download/mod.rs | 77 +++++- crates/cli/commands/src/download/archive.rs | 13 +- crates/cli/commands/src/download/extract.rs | 264 ++++++++++++++++--- crates/cli/commands/src/download/mod.rs | 27 +- crates/cli/commands/src/download/planning.rs | 5 +- crates/cli/commands/src/download/session.rs | 10 +- crates/cli/commands/src/download/verify.rs | 47 +++- 7 files changed, 384 insertions(+), 59 deletions(-) diff --git a/bin/reth/tests/it/download/mod.rs b/bin/reth/tests/it/download/mod.rs index db794e30102..d76377e4093 100644 --- a/bin/reth/tests/it/download/mod.rs +++ b/bin/reth/tests/it/download/mod.rs @@ -96,6 +96,15 @@ fn success(output: Output) -> String { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn manifest_selection_reuse_and_repair() { + check_manifest_selection_reuse_and_repair(false).await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn custom_static_files_selection_reuse_and_repair() { + check_manifest_selection_reuse_and_repair(true).await; +} + +async fn check_manifest_selection_reuse_and_repair(custom_static_files: bool) { let snapshot = Snapshot::new(); let state = &snapshot.files["/snapshot/state.tar.zst"]; let offset = state.len() / 2; @@ -103,19 +112,28 @@ async fn manifest_selection_reuse_and_repair() { fs::create_dir(dir.path().join(".download-cache")).unwrap(); fs::write(dir.path().join(".download-cache/state.tar.zst.part"), &state[..offset]).unwrap(); let server = snapshot.serve(true).await; - let args = ["--with-txs-since", "10"]; + let custom_dir = tempfile::tempdir().unwrap(); + let static_files = if custom_static_files { + custom_dir.path().join("nested/static") + } else { + dir.path().join("static_files") + }; + let mut args = vec!["--with-txs-since", "10"]; + if custom_static_files { + args.extend(["--datadir.static-files", static_files.to_str().unwrap()]); + } success(download(&server, dir.path(), &args)); for (path, expected) in [ - ("db/snapshot", "state.tar.zst"), - ("static_files/headers", "headers.tar.zst"), - ("static_files/txs-1", "txs-1.tar.zst"), + (dir.path().join("db/snapshot"), "state.tar.zst"), + (static_files.join("headers"), "headers.tar.zst"), + (static_files.join("txs-1"), "txs-1.tar.zst"), ] { - assert_eq!(fs::read(dir.path().join(path)).unwrap(), expected.as_bytes()); + assert_eq!(fs::read(path).unwrap(), expected.as_bytes()); } assert!(server.requests().iter().any(|(path, range)| { path.ends_with("state.tar.zst") && range.as_deref() == Some(&format!("bytes={offset}-")) })); - assert!(!dir.path().join("static_files/txs-0").exists()); + assert!(!static_files.join("txs-0").exists()); assert!(!server.requests().iter().any(|(path, _)| path.ends_with("txs-0.tar.zst"))); let config: toml::Value = toml::from_str(&fs::read_to_string(dir.path().join("reth.toml")).unwrap()).unwrap(); @@ -126,15 +144,23 @@ async fn manifest_selection_reuse_and_repair() { assert_eq!(server.requests(), vec![("/snapshot/manifest.json".into(), None)]); // Same-size corruption must be detected by checksum, not just the file length. - fs::write(dir.path().join("static_files/txs-1"), b"bad-1.tar.zst").unwrap(); + fs::write(static_files.join("txs-1"), b"bad-1.tar.zst").unwrap(); server.clear_requests(); success(download(&server, dir.path(), &args)); - assert_eq!(fs::read(dir.path().join("static_files/txs-1")).unwrap(), b"txs-1.tar.zst"); + assert_eq!(fs::read(static_files.join("txs-1")).unwrap(), b"txs-1.tar.zst"); assert!(server.requests().iter().any(|(path, _)| path.ends_with("txs-1.tar.zst"))); assert!(server .requests() .iter() .all(|(path, _)| path.ends_with("manifest.json") || path.ends_with("txs-1.tar.zst"))); + if custom_static_files { + assert!(!dir.path().join("static_files").exists()); + } + fs::write(static_files.join("stale"), b"stale").unwrap(); + args.push("--force"); + success(download(&server, dir.path(), &args)); + assert!(!static_files.join("stale").exists()); + assert_eq!(fs::read(static_files.join("headers")).unwrap(), b"headers.tar.zst"); } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] @@ -183,6 +209,41 @@ async fn legacy_streaming_and_resume() { } } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn custom_static_files_legacy_http_and_local_archives() { + for local in [false, true] { + let bytes = archive_bytes("./static_files/nested/headers", b"snapshot contents"); + let archive_name = "headers.tar.zst"; + let source = tempfile::tempdir().unwrap(); + let archive_path = source.path().join(archive_name); + fs::write(&archive_path, &bytes).unwrap(); + let server = + SnapshotServer::start(BTreeMap::from([(format!("/{archive_name}"), bytes)]), true) + .await; + let url = if local { + format!("file://{}", archive_path.display()) + } else { + format!("{}/{archive_name}", server.url) + }; + let dir = tempfile::tempdir().unwrap(); + let static_dir = tempfile::tempdir().unwrap(); + reth_ok(&[ + "download", + "--url", + &url, + "--datadir", + dir.path().to_str().unwrap(), + "--datadir.static-files", + static_dir.path().to_str().unwrap(), + ]); + assert_eq!( + fs::read(static_dir.path().join("nested/headers")).unwrap(), + b"snapshot contents" + ); + assert!(!dir.path().join("static_files").exists()); + } +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn wrong_chain_is_rejected_before_force_removes_data() { let mut snapshot = Snapshot::new(); diff --git a/crates/cli/commands/src/download/archive.rs b/crates/cli/commands/src/download/archive.rs index 65573eec561..c4e46e1ddbf 100644 --- a/crates/cli/commands/src/download/archive.rs +++ b/crates/cli/commands/src/download/archive.rs @@ -29,6 +29,7 @@ const DOWNLOAD_CACHE_DIR: &str = ".download-cache"; pub(crate) async fn run_modular_downloads( planned_downloads: PlannedDownloads, target_dir: &Path, + static_files_dir: Option<&Path>, download_concurrency: usize, cancel_token: CancellationToken, retry_backoff: Option, @@ -48,8 +49,12 @@ pub(crate) async fn run_modular_downloads( cancel_token, ) .with_retry_backoff(retry_backoff); - let ctx = - ArchiveProcessContext::new(target_dir.to_path_buf(), Some(download_cache_dir), session); + let ctx = ArchiveProcessContext::new( + target_dir.to_path_buf(), + static_files_dir.map(Path::to_path_buf), + Some(download_cache_dir), + session, + ); ModularDownloadJob::new(ctx, download_concurrency).run(planned_downloads).await } @@ -220,7 +225,7 @@ impl ArchiveProcessor { /// Returns the verifier for this archive's output files. fn output_verifier(&self) -> OutputVerifier<'_> { - OutputVerifier::new(self.ctx.target_dir()) + OutputVerifier::new(self.ctx.target_dir(), self.ctx.static_files_dir()) } /// Returns `true` if this archive can be reused from existing verified outputs. @@ -304,6 +309,7 @@ impl ArchiveProcessor { &self.archive().url, format, self.ctx.target_dir(), + self.ctx.static_files_dir(), self.ctx.session(), ) } @@ -316,6 +322,7 @@ impl ArchiveProcessor { file, format, self.ctx.target_dir(), + self.ctx.static_files_dir(), Some(&mut extraction_progress), ); extraction_progress.finish(); diff --git a/crates/cli/commands/src/download/extract.rs b/crates/cli/commands/src/download/extract.rs index bc5722cb19d..ba405ff6fed 100644 --- a/crates/cli/commands/src/download/extract.rs +++ b/crates/cli/commands/src/download/extract.rs @@ -2,7 +2,7 @@ use super::{ fetch::{ArchiveFetcher, DownloadedArchive}, progress::{ ArchiveExtractionProgress, ArchiveExtractionProgressHandle, DownloadProgress, - DownloadRequestLimiter, ProgressReader, SharedProgress, SharedProgressReader, + DownloadRequestLimiter, ProgressReader, SharedProgressReader, }, session::DownloadSession, MAX_DOWNLOAD_RETRIES, RETRY_BACKOFF_SECS, @@ -67,6 +67,7 @@ fn extract_archive( total_size: u64, format: CompressionFormat, target_dir: &Path, + static_files_dir: Option<&Path>, cancel_token: CancellationToken, ) -> Result<()> { let progress_reader = ProgressReader::new(reader, total_size, cancel_token); @@ -74,11 +75,11 @@ fn extract_archive( match format { CompressionFormat::Lz4 => { let decoder = Decoder::new(progress_reader)?; - Archive::new(decoder).unpack(target_dir)?; + unpack_archive(Archive::new(decoder), target_dir, static_files_dir, None)?; } CompressionFormat::Zstd => { let decoder = ZstdDecoder::new(progress_reader)?; - Archive::new(decoder).unpack(target_dir)?; + unpack_archive(Archive::new(decoder), target_dir, static_files_dir, None)?; } } @@ -91,14 +92,25 @@ pub(crate) fn extract_archive_raw( reader: R, format: CompressionFormat, target_dir: &Path, + static_files_dir: Option<&Path>, progress: Option<&mut ArchiveExtractionProgress>, ) -> Result<()> { match format { CompressionFormat::Lz4 => { - unpack_archive(Archive::new(Decoder::new(reader)?), target_dir, progress)?; + unpack_archive( + Archive::new(Decoder::new(reader)?), + target_dir, + static_files_dir, + progress, + )?; } CompressionFormat::Zstd => { - unpack_archive(Archive::new(ZstdDecoder::new(reader)?), target_dir, progress)?; + unpack_archive( + Archive::new(ZstdDecoder::new(reader)?), + target_dir, + static_files_dir, + progress, + )?; } } @@ -108,8 +120,13 @@ pub(crate) fn extract_archive_raw( fn unpack_archive( mut archive: Archive, target_dir: &Path, + static_files_dir: Option<&Path>, mut progress: Option<&mut ArchiveExtractionProgress>, ) -> Result<()> { + if static_files_dir.is_none() && progress.is_none() { + archive.unpack(target_dir)?; + return Ok(()) + } let entries = archive.entries().wrap_err_with(|| { format!("failed to read archive entries for `{}`", target_dir.display()) })?; @@ -118,29 +135,105 @@ fn unpack_archive( let mut entry = entry.wrap_err_with(|| { format!("failed to read archive entry for `{}`", target_dir.display()) })?; - extract_entry_with_progress(&mut entry, target_dir, progress.as_deref_mut())?; + extract_entry_with_progress( + &mut entry, + target_dir, + static_files_dir, + progress.as_deref_mut(), + )?; } Ok(()) } +/// Returns the path within the static files directory, accepting tar's optional `./` prefix. +pub(crate) fn static_file_relative_path(path: &Path) -> Option<&Path> { + path.strip_prefix(".").unwrap_or(path).strip_prefix("static_files").ok() +} + +/// Extracts static files beneath their configured root without allowing archive paths or links +/// to escape it. +fn unpack_static_file( + entry: &mut tar::Entry<'_, R>, + static_files_dir: &Path, + relative_path: &Path, +) -> Result<()> { + eyre::ensure!( + relative_path + .components() + .all(|part| matches!(part, Component::Normal(_) | Component::CurDir)), + "Invalid static file archive path: {}", + relative_path.display() + ); + let entry_type = entry.header().entry_type(); + eyre::ensure!( + entry_type.is_file() || entry_type.is_dir(), + "Unsupported static file archive entry" + ); + fs::create_dir_all(static_files_dir)?; + let root = static_files_dir.to_path_buf(); + let dest = root.join(relative_path); + // Check each existing ancestor before creating directories or replacing an output. + let mut current = root.clone(); + for part in relative_path.components() { + current.push(part); + if let Ok(metadata) = std::fs::symlink_metadata(¤t) { + eyre::ensure!( + !metadata.file_type().is_symlink(), + "Static file archive path contains a symlink" + ); + } + } + if relative_path.as_os_str().is_empty() { + return Ok(()) + } + if let Some(parent) = dest.parent() { + fs::create_dir_all(parent)?; + } + entry.unpack(dest)?; + Ok(()) +} + +fn unpack_entry( + entry: &mut tar::Entry<'_, R>, + target_dir: &Path, + static_files_dir: Option<&Path>, +) -> Result<()> { + // A non-static entry could otherwise plant a symlink at a nested custom root before + // extraction or retry cleanup uses it. + eyre::ensure!( + static_files_dir.is_none() || !entry.header().entry_type().is_symlink(), + "Archive symlinks are unsupported with a custom static files directory" + ); + let path = entry.path()?.into_owned(); + if let Some(static_files_dir) = static_files_dir && + let Some(relative_path) = static_file_relative_path(&path) + { + unpack_static_file(entry, static_files_dir, relative_path) + } else { + entry.unpack_in(target_dir)?; + Ok(()) + } +} + fn extract_entry_with_progress( entry: &mut tar::Entry<'_, R>, target_dir: &Path, + static_files_dir: Option<&Path>, progress: Option<&mut ArchiveExtractionProgress>, ) -> Result<()> { let size = entry.header().entry_size().unwrap_or(0); let entry_type = entry.header().entry_type(); if !entry_type.is_file() || size == 0 { - entry.unpack_in(target_dir).wrap_err_with(|| { + unpack_entry(entry, target_dir, static_files_dir).wrap_err_with(|| { format!("failed to extract archive into `{}`", target_dir.display()) })?; return Ok(()) } if size < STREAMING_EXTRACTION_PROGRESS_MIN_FILE_SIZE { - entry.unpack_in(target_dir).wrap_err_with(|| { + unpack_entry(entry, target_dir, static_files_dir).wrap_err_with(|| { format!("failed to extract archive into `{}`", target_dir.display()) })?; if let Some(progress) = progress { @@ -150,14 +243,14 @@ fn extract_entry_with_progress( } let Some(progress_handle) = progress.as_ref().and_then(|progress| progress.handle()) else { - entry.unpack_in(target_dir).wrap_err_with(|| { + unpack_entry(entry, target_dir, static_files_dir).wrap_err_with(|| { format!("failed to extract archive into `{}`", target_dir.display()) })?; return Ok(()) }; - let Some(entry_path) = entry_destination_path(entry, target_dir)? else { - entry.unpack_in(target_dir).wrap_err_with(|| { + let Some(entry_path) = entry_destination_path(entry, target_dir, static_files_dir)? else { + unpack_entry(entry, target_dir, static_files_dir).wrap_err_with(|| { format!("failed to extract archive into `{}`", target_dir.display()) })?; return Ok(()) @@ -165,8 +258,7 @@ fn extract_entry_with_progress( let stop = Arc::new(AtomicBool::new(false)); let monitor = spawn_extraction_progress_monitor(entry_path, progress_handle, Arc::clone(&stop)); - let unpack_result = entry - .unpack_in(target_dir) + let unpack_result = unpack_entry(entry, target_dir, static_files_dir) .wrap_err_with(|| format!("failed to extract archive into `{}`", target_dir.display())); stop.store(true, Ordering::Relaxed); @@ -180,9 +272,18 @@ fn extract_entry_with_progress( fn entry_destination_path( entry: &tar::Entry<'_, R>, target_dir: &Path, + static_files_dir: Option<&Path>, ) -> Result> { let mut file_dst = target_dir.to_path_buf(); let path = entry.path().wrap_err("invalid path in archive entry")?; + let path = if let Some(static_files_dir) = static_files_dir && + let Some(relative_path) = static_file_relative_path(&path) + { + file_dst = static_files_dir.to_path_buf(); + relative_path + } else { + path.as_ref() + }; for part in path.components() { match part { @@ -231,7 +332,12 @@ fn record_extracted_file_bytes( } /// Extracts a snapshot from a local file. -fn extract_from_file(path: &Path, format: CompressionFormat, target_dir: &Path) -> Result<()> { +fn extract_from_file( + path: &Path, + format: CompressionFormat, + target_dir: &Path, + static_files_dir: Option<&Path>, +) -> Result<()> { let file = std::fs::File::open(path)?; let total_size = file.metadata()?.len(); info!(target: "reth::cli", @@ -240,7 +346,14 @@ fn extract_from_file(path: &Path, format: CompressionFormat, target_dir: &Path) "Extracting local archive" ); let start = Instant::now(); - extract_archive(file, total_size, format, target_dir, CancellationToken::new())?; + extract_archive( + file, + total_size, + format, + target_dir, + static_files_dir, + CancellationToken::new(), + )?; info!(target: "reth::cli", file = %path.display(), elapsed = %DownloadProgress::format_duration(start.elapsed()), @@ -256,11 +369,12 @@ pub(crate) fn streaming_download_and_extract( url: &str, format: CompressionFormat, target_dir: &Path, + static_files_dir: Option<&Path>, session: &DownloadSession, ) -> Result<()> { if let Some(path) = archive_file_url_path(url)? { let size = path.metadata()?.len(); - extract_from_file(&path, format, target_dir)?; + extract_from_file(&path, format, target_dir, static_files_dir)?; session.record_archive_output_complete(size); return Ok(()) } @@ -318,7 +432,7 @@ pub(crate) fn streaming_download_and_extract( let result = if let Some(progress) = shared { let reader = SharedProgressReader { inner: response, progress: Arc::clone(progress) }; - extract_archive_raw(reader, format, target_dir, None) + extract_archive_raw(reader, format, target_dir, static_files_dir, None) } else { let total_size = response.content_length().unwrap_or(0); extract_archive( @@ -326,6 +440,7 @@ pub(crate) fn streaming_download_and_extract( total_size, format, target_dir, + static_files_dir, session.cancel_token().clone(), ) }; @@ -375,6 +490,7 @@ fn download_and_extract( url: &str, format: CompressionFormat, target_dir: &Path, + static_files_dir: Option<&Path>, session: DownloadSession, ) -> Result<()> { let quiet = session.progress().is_some(); @@ -394,9 +510,16 @@ fn download_and_extract( let file = fs::open(&downloaded_path)?; if quiet { - extract_archive_raw(file, format, target_dir, None)?; + extract_archive_raw(file, format, target_dir, static_files_dir, None)?; } else { - extract_archive(file, total_size, format, target_dir, session.cancel_token().clone())?; + extract_archive( + file, + total_size, + format, + target_dir, + static_files_dir, + session.cancel_token().clone(), + )?; info!(target: "reth::cli", file = %file_name, "Extraction complete" @@ -417,7 +540,7 @@ fn download_and_extract( fn blocking_download_and_extract( url: &str, target_dir: &Path, - shared: Option>, + static_files_dir: Option<&Path>, resumable: bool, request_limiter: Option>, cancel_token: CancellationToken, @@ -428,12 +551,12 @@ fn blocking_download_and_extract( if let Ok(parsed_url) = Url::parse(url) && parsed_url.scheme() == "file" { - let session = DownloadSession::new(shared, request_limiter, cancel_token) + let session = DownloadSession::new(None, request_limiter, cancel_token) .with_retry_backoff(retry_backoff); let file_path = parsed_url .to_file_path() .map_err(|_| eyre::eyre!("Invalid file:// URL path: {}", url))?; - let result = extract_from_file(&file_path, format, target_dir); + let result = extract_from_file(&file_path, format, target_dir, static_files_dir); if result.is_ok() { session.record_archive_output_complete(file_path.metadata()?.len()); } @@ -443,18 +566,20 @@ fn blocking_download_and_extract( url, format, target_dir, - DownloadSession::new(shared, Some(request_limiter), cancel_token) + static_files_dir, + DownloadSession::new(None, Some(request_limiter), cancel_token) .with_retry_backoff(retry_backoff), ) } else if resumable { let session = - DownloadSession::new(shared, Some(DownloadRequestLimiter::new(1)), cancel_token) + DownloadSession::new(None, Some(DownloadRequestLimiter::new(1)), cancel_token) .with_retry_backoff(retry_backoff); - download_and_extract(url, format, target_dir, session) + download_and_extract(url, format, target_dir, static_files_dir, session) } else { let session = - DownloadSession::new(shared, None, cancel_token).with_retry_backoff(retry_backoff); - let result = streaming_download_and_extract(url, format, target_dir, &session); + DownloadSession::new(None, None, cancel_token).with_retry_backoff(retry_backoff); + let result = + streaming_download_and_extract(url, format, target_dir, static_files_dir, &session); if result.is_ok() { session.record_archive_output_complete(0); } @@ -464,25 +589,25 @@ fn blocking_download_and_extract( /// Downloads and extracts a snapshot archive asynchronously. /// -/// When `shared` is provided, download progress is reported to the shared -/// counter for aggregated display. Otherwise uses a local progress bar. +/// Download progress is reported using a local progress bar. /// When `resumable` is true, uses two-phase download with `.part` files. pub(crate) async fn stream_and_extract( url: &str, target_dir: &Path, - shared: Option>, + static_files_dir: Option<&Path>, resumable: bool, request_limiter: Option>, cancel_token: CancellationToken, retry_backoff: Option, ) -> Result<()> { let target_dir = target_dir.to_path_buf(); + let static_files_dir = static_files_dir.map(Path::to_path_buf); let url = url.to_string(); task::spawn_blocking(move || { blocking_download_and_extract( &url, &target_dir, - shared, + static_files_dir.as_deref(), resumable, request_limiter, cancel_token, @@ -498,6 +623,83 @@ pub(crate) async fn stream_and_extract( mod tests { use super::*; + #[test] + fn custom_static_root_cannot_be_redirected_by_archive_symlinks() { + for custom_path in ["custom", "custom/nested"] { + let target = tempfile::tempdir().unwrap(); + let outside = tempfile::tempdir().unwrap(); + fs::create_dir_all(outside.path().join("nested")).unwrap(); + for path in ["headers", "nested/headers"] { + fs::write(outside.path().join(path), b"keep").unwrap(); + } + let custom = target.path().join(custom_path); + let mut archive = tar::Builder::new(Vec::new()); + let mut link = tar::Header::new_gnu(); + link.set_entry_type(tar::EntryType::Symlink); + link.set_size(0); + link.set_mode(0o777); + archive.append_link(&mut link, "custom", outside.path()).unwrap(); + let mut file = tar::Header::new_gnu(); + file.set_size(4); + file.set_mode(0o644); + file.set_cksum(); + archive.append_data(&mut file, "static_files/headers", b"data".as_slice()).unwrap(); + let tar = archive.into_inner().unwrap(); + let err = + unpack_archive(Archive::new(tar.as_slice()), target.path(), Some(&custom), None) + .unwrap_err(); + assert!(format!("{err:#}").contains("Archive symlinks are unsupported")); + let outputs = [super::super::manifest::OutputFileChecksum { + path: "static_files/headers".into(), + size: 4, + blake3: String::new(), + }]; + super::super::verify::OutputVerifier::new(target.path(), Some(&custom)) + .cleanup(&outputs); + for path in ["headers", "nested/headers"] { + assert_eq!(fs::read(outside.path().join(path)).unwrap(), b"keep"); + } + } + } + + #[test] + fn remap_static_files_in_both_compression_formats() { + let mut archive = tar::Builder::new(Vec::new()); + let mut directory = tar::Header::new_gnu(); + directory.set_entry_type(tar::EntryType::Directory); + directory.set_size(0); + directory.set_mode(0o755); + directory.set_cksum(); + archive.append_data(&mut directory, "./static_files/", std::io::empty()).unwrap(); + for path in ["./static_files/nested/headers", "db/data"] { + let mut header = tar::Header::new_gnu(); + header.set_size(4); + header.set_mode(0o644); + header.set_cksum(); + archive.append_data(&mut header, path, b"data".as_slice()).unwrap(); + } + let tar = archive.into_inner().unwrap(); + for format in [CompressionFormat::Lz4, CompressionFormat::Zstd] { + let bytes = match format { + CompressionFormat::Lz4 => { + let mut encoder = lz4::EncoderBuilder::new().build(Vec::new()).unwrap(); + std::io::copy(&mut tar.as_slice(), &mut encoder).unwrap(); + let (bytes, result) = encoder.finish(); + result.unwrap(); + bytes + } + CompressionFormat::Zstd => zstd::encode_all(tar.as_slice(), 0).unwrap(), + }; + let target = tempfile::tempdir().unwrap(); + let custom = tempfile::tempdir().unwrap(); + extract_archive_raw(bytes.as_slice(), format, target.path(), Some(custom.path()), None) + .unwrap(); + assert_eq!(fs::read(custom.path().join("nested/headers")).unwrap(), b"data"); + assert_eq!(fs::read(target.path().join("db/data")).unwrap(), b"data"); + assert!(!target.path().join("static_files").exists()); + } + } + #[test] fn test_compression_format_detection() { assert!(matches!( diff --git a/crates/cli/commands/src/download/mod.rs b/crates/cli/commands/src/download/mod.rs index 411fa824cba..3726743f6bc 100644 --- a/crates/cli/commands/src/download/mod.rs +++ b/crates/cli/commands/src/download/mod.rs @@ -475,6 +475,11 @@ impl> DownloadCo return Ok(None); } + let data_dir = self.env.datadir.clone().resolve_datadir(chain); + let static_files_dir = data_dir.static_files(); + let static_files_dir = (static_files_dir != data_dir.data_dir().join("static_files")) + .then_some(static_files_dir); + // Legacy single-URL mode: download one archive and extract it if let Some(ref url) = self.url { let cancel_token = CancellationToken::new(); @@ -482,7 +487,7 @@ impl> DownloadCo let data_dir = self.env.datadir.clone().resolve_datadir(chain); let target_dir = data_dir.data_dir(); if self.force { - clear_existing_datadir(target_dir)?; + clear_existing_datadir(target_dir, static_files_dir.as_deref())?; } fs::create_dir_all(target_dir)?; @@ -496,7 +501,7 @@ impl> DownloadCo stream_and_extract( url, data_dir.data_dir(), - None, + static_files_dir.as_deref(), self.resumable, Some(request_limiter), cancel_token.clone(), @@ -522,10 +527,11 @@ impl> DownloadCo let cancel_token = CancellationToken::new(); let _cancel_guard = cancel_token.drop_guard(); if self.force { - clear_existing_datadir(target_dir)?; + clear_existing_datadir(target_dir, static_files_dir.as_deref())?; } fs::create_dir_all(target_dir)?; - let startup_summary = summarize_download_startup(&planned.archives, target_dir)?; + let startup_summary = + summarize_download_startup(&planned.archives, target_dir, static_files_dir.as_deref())?; info!(target: "reth::cli", reusable = startup_summary.reusable, needs_download = startup_summary.needs_download, @@ -542,6 +548,7 @@ impl> DownloadCo run_modular_downloads( planned, target_dir, + static_files_dir.as_deref(), self.download_concurrency.max(1), cancel_token.clone(), self.retry_backoff, @@ -916,14 +923,14 @@ fn selection_from_prune_mode(mode: Option, snapshot_block: u64) -> Co } /// Removes existing snapshot data that is managed by `reth download`. -fn clear_existing_datadir(target_dir: &Path) -> Result<()> { - if !target_dir.try_exists()? { - return Ok(()); - } - +fn clear_existing_datadir(target_dir: &Path, static_files_dir: Option<&Path>) -> Result<()> { info!(target: "reth::cli", dir = ?target_dir, "Clearing existing snapshot data"); for entry in FORCE_REMOVED_DATADIR_PATHS { - let path = target_dir.join(entry); + let path = if *entry == "static_files" { + static_files_dir.map_or_else(|| target_dir.join(entry), Path::to_path_buf) + } else { + target_dir.join(entry) + }; if !path.try_exists()? { continue; } diff --git a/crates/cli/commands/src/download/planning.rs b/crates/cli/commands/src/download/planning.rs index 22bb8b96bc9..6a3f8457986 100644 --- a/crates/cli/commands/src/download/planning.rs +++ b/crates/cli/commands/src/download/planning.rs @@ -153,9 +153,10 @@ pub(crate) struct DownloadStartupSummary { pub(crate) fn summarize_download_startup( all_downloads: &[PlannedArchive], target_dir: &Path, + static_files_dir: Option<&Path>, ) -> Result { let mut summary = DownloadStartupSummary::default(); - let verifier = OutputVerifier::new(target_dir); + let verifier = OutputVerifier::new(target_dir, static_files_dir); for planned in all_downloads { if verifier.verify(&planned.archive.output_files)? { @@ -303,7 +304,7 @@ mod tests { }, ]; - let summary = summarize_download_startup(&planned, target_dir).unwrap(); + let summary = summarize_download_startup(&planned, target_dir, None).unwrap(); assert_eq!(summary.reusable, 1); assert_eq!(summary.needs_download, 2); } diff --git a/crates/cli/commands/src/download/session.rs b/crates/cli/commands/src/download/session.rs index bc1e5a941af..9aa898e66b5 100644 --- a/crates/cli/commands/src/download/session.rs +++ b/crates/cli/commands/src/download/session.rs @@ -81,6 +81,8 @@ impl DownloadSession { pub(crate) struct ArchiveProcessContext { /// Directory where extracted output files are written. target_dir: PathBuf, + /// Custom location of static file outputs. + static_files_dir: Option, /// Directory used for cached archive downloads, when enabled. cache_dir: Option, /// Shared command-scoped download state. @@ -91,10 +93,11 @@ impl ArchiveProcessContext { /// Creates the context used while processing modular archives. pub(crate) fn new( target_dir: PathBuf, + static_files_dir: Option, cache_dir: Option, session: DownloadSession, ) -> Self { - Self { target_dir, cache_dir, session } + Self { target_dir, static_files_dir, cache_dir, session } } /// Returns the directory where extracted outputs should be written. @@ -102,6 +105,11 @@ impl ArchiveProcessContext { &self.target_dir } + /// Returns the custom static files directory, if configured. + pub(crate) fn static_files_dir(&self) -> Option<&Path> { + self.static_files_dir.as_deref() + } + /// Returns the cache directory for two-phase downloads, if enabled. pub(crate) fn cache_dir(&self) -> Option<&Path> { self.cache_dir.as_deref() diff --git a/crates/cli/commands/src/download/verify.rs b/crates/cli/commands/src/download/verify.rs index a4b84b992eb..d87c7bf6fef 100644 --- a/crates/cli/commands/src/download/verify.rs +++ b/crates/cli/commands/src/download/verify.rs @@ -8,12 +8,14 @@ use std::{io::Read, path::Path}; pub(crate) struct OutputVerifier<'a> { /// Directory containing the output files declared by the manifest. target_dir: &'a Path, + /// Custom location of static file outputs. + static_files_dir: Option<&'a Path>, } impl<'a> OutputVerifier<'a> { /// Creates a verifier for one extraction target directory. - pub(crate) const fn new(target_dir: &'a Path) -> Self { - Self { target_dir } + pub(crate) const fn new(target_dir: &'a Path, static_files_dir: Option<&'a Path>) -> Self { + Self { target_dir, static_files_dir } } /// Returns `true` only when every declared output file exists and matches size and BLAKE3. @@ -34,7 +36,7 @@ impl<'a> OutputVerifier<'a> { } for expected in output_files { - let output_path = self.target_dir.join(&expected.path); + let output_path = self.output_path(&expected.path); let meta = match fs::metadata(&output_path) { Ok(meta) => meta, Err(_) => return Ok(false), @@ -55,10 +57,20 @@ impl<'a> OutputVerifier<'a> { /// Removes any declared output files so a fresh archive attempt can restart cleanly. pub(crate) fn cleanup(&self, output_files: &[OutputFileChecksum]) { for output in output_files { - let _ = fs::remove_file(self.target_dir.join(&output.path)); + let _ = fs::remove_file(self.output_path(&output.path)); } } + /// Resolves archive paths consistently for verification and retry cleanup. + fn output_path(&self, path: &str) -> std::path::PathBuf { + if let Some(static_files_dir) = self.static_files_dir && + let Some(relative_path) = super::extract::static_file_relative_path(Path::new(path)) + { + return static_files_dir.join(relative_path) + } + self.target_dir.join(path) + } + /// Computes the hex-encoded BLAKE3 checksum for one plain output file. fn file_blake3_hex( path: &Path, @@ -82,3 +94,30 @@ impl<'a> OutputVerifier<'a> { Ok(hasher.finalize().to_hex().to_string()) } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn custom_static_files_verification_and_cleanup() { + let datadir = tempfile::tempdir().unwrap(); + let static_dir = tempfile::tempdir().unwrap(); + let default_file = datadir.path().join("static_files/headers"); + fs::create_dir_all(default_file.parent().unwrap()).unwrap(); + fs::write(&default_file, b"headers").unwrap(); + let outputs = [OutputFileChecksum { + path: "./static_files/headers".into(), + size: 7, + blake3: blake3::hash(b"headers").to_hex().to_string(), + }]; + let verifier = OutputVerifier::new(datadir.path(), Some(static_dir.path())); + assert!(!verifier.verify(&outputs).unwrap()); + let custom_file = static_dir.path().join("headers"); + fs::write(&custom_file, b"headers").unwrap(); + assert!(verifier.verify(&outputs).unwrap()); + verifier.cleanup(&outputs); + assert!(!custom_file.exists()); + assert_eq!(fs::read(default_file).unwrap(), b"headers"); + } +} From 6fe745484b5d5b0bdf98de94ab0ea7bfb15f386c Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Mon, 14 Sep 2026 14:16:50 +0000 Subject: [PATCH 251/294] feat(ci): add RPC replay mode to derek bench (#27207) --- .github/scripts/bench-call-summary.py | 1097 ++++++++++++++++++++ .github/scripts/bench-job-summary.js | 9 +- .github/scripts/bench-txgen-extract.sh | 331 +++++- .github/scripts/bench-txgen-install.sh | 13 +- .github/scripts/bench-txgen-run.sh | 198 +++- .github/scripts/bench-utils.js | 26 + .github/scripts/test_bench_call_summary.py | 424 ++++++++ .github/workflows/bench.yml | 467 ++++++++- 8 files changed, 2494 insertions(+), 71 deletions(-) create mode 100755 .github/scripts/bench-call-summary.py create mode 100644 .github/scripts/test_bench_call_summary.py diff --git a/.github/scripts/bench-call-summary.py b/.github/scripts/bench-call-summary.py new file mode 100755 index 00000000000..49eeba43069 --- /dev/null +++ b/.github/scripts/bench-call-summary.py @@ -0,0 +1,1097 @@ +#!/usr/bin/env python3 +"""Summarize an RPC replay benchmark and generate summary JSON + markdown. + +Usage: + bench-call-summary.py \ + --output-summary \ + --output-markdown \ + --baseline-dir [ ...] \ + --feature-dir [ ...] + +Each run directory holds the output of one `bench call` measured phase: + + report.json run metadata, identity and per-method aggregates + requests.csv open-loop, one row per request + record_timings.csv closed-loop, one row per record and pass + responses.ndjson one response digest per corpus record + cpu.json CPU time the node's cgroup spent on the measured phase + +Latency, throughput and CPU per request are compared with whole-run cluster +bootstrap confidence intervals, and every response digest is compared against +the first baseline run. Any digest that differs between the arms fails the +verdict regardless of the timings; digests that already differ between two +baseline runs make the comparison inconclusive instead. +""" + +from __future__ import annotations + +import argparse +import csv +import importlib.util +import json +import os +from pathlib import Path +import random +import sys + +# Latency floors below which a delta is treated as noise. These are starting +# values for the first runs on the bench runner and are meant to be tightened +# once the A/A spread of each class is known. +PRACTICAL_FLOOR_PCT = { + "mean": 2.5, + "p50": 2.5, + "p90": 2.5, + "p99": 5.0, + "record_median": 2.5, + "closed_loop_rps": 2.5, + "cpu_per_request": 2.5, +} +MAX_DIVERGENT_REPORTED = 40 +OK_STATUS = "ok" + + +def _load_engine_summary() -> object: + """Import the engine summary script for its statistics helpers.""" + path = Path(__file__).resolve().with_name("bench-reth-summary.py") + spec = importlib.util.spec_from_file_location("bench_reth_summary", path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +_engine = _load_engine_summary() +BOOTSTRAP_ITERATIONS = _engine.BOOTSTRAP_ITERATIONS +_bootstrap_ci = _engine._bootstrap_ci +_ci_half_width = _engine._ci_half_width +_mean = _engine._mean +build_grafana_logs_url = _engine.build_grafana_logs_url +build_grafana_traces_url = _engine.build_grafana_traces_url +fmt_ms = _engine.fmt_ms +generate_observability_section = _engine.generate_observability_section +percentile = _engine.percentile +resolve_observability_range = _engine.resolve_observability_range +significance = _engine.significance + + +def load_run(directory: Path) -> dict: + """Load one run directory into the per-run shape used by the comparison.""" + report_path = directory / "report.json" + if not report_path.is_file(): + raise SystemExit(f"Missing {report_path}") + with report_path.open() as f: + report = json.load(f) + + responses_path = directory / "responses.ndjson" + if not responses_path.is_file(): + raise SystemExit(f"Missing {responses_path}") + + requests = parse_requests_csv(directory / "requests.csv") + records = parse_record_timings_csv(directory / "record_timings.csv") + if not requests and not records: + raise SystemExit(f"No open-loop or closed-loop timings in {directory}") + + cpu = {} + cpu_path = directory / "cpu.json" + if cpu_path.is_file(): + with cpu_path.open() as f: + cpu = json.load(f) + + return { + "name": directory.name, + "report": report, + "requests": requests, + "records": records, + "responses": parse_responses(responses_path), + "cpu": cpu, + } + + +def parse_requests_csv(path: Path) -> list[dict]: + """Parse the open-loop per-request CSV.""" + if not path.is_file(): + return [] + rows = [] + with path.open() as f: + for row in csv.DictReader(f): + rows.append( + { + "offset_ms": float(row["offset_ms"]), + "record_index": int(row["record_index"]), + "method": row["method"], + "latency_us": int(row["latency_us"]), + "status": row["status"], + } + ) + return rows + + +def parse_record_timings_csv(path: Path) -> list[dict]: + """Parse the closed-loop per-record CSV.""" + if not path.is_file(): + return [] + rows = [] + with path.open() as f: + for row in csv.DictReader(f): + rows.append( + { + "record_index": int(row["record_index"]), + "method": row["method"], + "pass": int(row["pass"]), + "latency_us": int(row["latency_us"]), + "status": row["status"], + } + ) + return rows + + +def parse_responses(path: Path) -> dict[int, dict]: + """Parse response digests, keyed by record index.""" + responses = {} + with path.open() as f: + for line in f: + line = line.strip() + if not line: + continue + entry = json.loads(line) + responses[int(entry["record_index"])] = { + "method": entry.get("method", "unknown"), + "kind": entry.get("kind", "unknown"), + "digest": entry.get("digest", ""), + "len": entry.get("len"), + } + return responses + + +def call_section(report: dict) -> dict: + """The replay section of a report; `bench call` nests its fields under `call`.""" + section = report.get("call") + return section if isinstance(section, dict) else report + + +def report_field(report: dict, key: str): + """Look a field up in the report, tolerating the nested `call` and identity blocks.""" + for scope in (report, call_section(report)): + if key in scope: + return scope[key] + for nested in ("identity", "metadata", "meta"): + block = scope.get(nested) + if isinstance(block, dict) and key in block: + return block[key] + return None + + +def median(values: list[float]) -> float: + if not values: + return 0.0 + ordered = sorted(values) + mid = len(ordered) // 2 + if len(ordered) % 2: + return ordered[mid] + return (ordered[mid - 1] + ordered[mid]) / 2.0 + + +def open_loop_stats(rows: list[dict]) -> dict: + """Latency distribution, achieved rate and error rate of one open-loop run.""" + if not rows: + return {} + ok_rows = [row for row in rows if row["status"] == OK_STATUS] + latencies_ms = sorted(row["latency_us"] / 1_000 for row in ok_rows) + offsets = [row["offset_ms"] for row in rows] + span_s = (max(offsets) - min(offsets)) / 1_000 if len(rows) > 1 else 0.0 + return { + "requests": len(rows), + "ok": len(ok_rows), + "errors": len(rows) - len(ok_rows), + "error_rate_pct": (len(rows) - len(ok_rows)) / len(rows) * 100.0, + "mean_ms": _mean(latencies_ms), + "p50_ms": percentile(latencies_ms, 50), + "p90_ms": percentile(latencies_ms, 90), + "p99_ms": percentile(latencies_ms, 99), + "open_loop_rps": len(rows) / span_s if span_s > 0 else 0.0, + } + + +def record_medians(rows: list[dict]) -> dict[int, float]: + """Median closed-loop latency in ms per record, over the passes of one run.""" + by_record: dict[int, list[float]] = {} + for row in rows: + if row["status"] != OK_STATUS: + continue + by_record.setdefault(row["record_index"], []).append(row["latency_us"] / 1_000) + return {index: median(values) for index, values in by_record.items()} + + +def arm_record_medians(runs: list[dict], method: str | None = None) -> dict[int, float]: + """Median closed-loop latency per record, over all passes and runs of one arm.""" + by_record: dict[int, list[float]] = {} + for run in runs: + for row in run["records"]: + if row["status"] != OK_STATUS: + continue + if method is not None and row["method"] != method: + continue + by_record.setdefault(row["record_index"], []).append(row["latency_us"] / 1_000) + return {index: median(values) for index, values in by_record.items()} + + +def run_stats(run: dict, method: str | None = None) -> dict: + """Every displayed metric for one run, optionally restricted to one method.""" + requests = run["requests"] + records = run["records"] + if method is not None: + requests = [row for row in requests if row["method"] == method] + records = [row for row in records if row["method"] == method] + + stats = open_loop_stats(requests) + medians = record_medians(records) + if medians: + stats["record_median_ms"] = median(list(medians.values())) + stats["records"] = len(medians) + + report = run["report"] + if method is None: + closed_loop_rps = report_field(report, "closed_loop_rps") + dropped = report_field(report, "dropped") + else: + methods = call_section(report).get("methods") or {} + entry = methods.get(method) or {} + closed_loop_rps = entry.get("closed_loop_rps") + if closed_loop_rps is None and isinstance(entry.get("closed_loop"), dict): + closed_loop_rps = entry["closed_loop"].get("rps") + dropped = entry.get("dropped") + if isinstance(closed_loop_rps, (int, float)): + stats["closed_loop_rps"] = float(closed_loop_rps) + if isinstance(dropped, int): + stats["dropped"] = dropped + + if method is None: + cpu_ms = cpu_ms_per_request(run) + if cpu_ms is not None: + stats["cpu_ms_per_request"] = cpu_ms + + lengths = [ + entry["len"] + for entry in run["responses"].values() + if isinstance(entry.get("len"), int) and (method is None or entry["method"] == method) + ] + if lengths: + stats["response_bytes_median"] = median([float(v) for v in lengths]) + return stats + + +def cpu_ms_per_request(run: dict) -> float | None: + """CPU milliseconds the node spent per successful request of one run.""" + cpu = run["cpu"] + usage_usec = cpu.get("usage_usec_delta") + if not isinstance(usage_usec, (int, float)) or usage_usec <= 0: + return None + requests_ok = cpu.get("requests_ok") + if not isinstance(requests_ok, int) or requests_ok <= 0: + requests_ok = sum( + 1 for row in run["requests"] + run["records"] if row["status"] == OK_STATUS + ) + if requests_ok <= 0: + return None + return usage_usec / 1_000 / requests_ok + + +def arm_stats(runs: list[dict], method: str | None = None) -> dict: + """Point estimates for one arm, averaged over its runs.""" + per_run = [run_stats(run, method) for run in runs] + per_run = [stats for stats in per_run if stats] + if not per_run: + return {} + + stats = {"runs": len(per_run)} + for key in ( + "mean_ms", + "p50_ms", + "p90_ms", + "p99_ms", + "open_loop_rps", + "closed_loop_rps", + "record_median_ms", + "cpu_ms_per_request", + "error_rate_pct", + "response_bytes_median", + ): + values = [run[key] for run in per_run if key in run] + if values: + stats[key] = _mean(values) + for key in ("requests", "ok", "errors", "dropped"): + values = [run[key] for run in per_run if key in run] + if values: + stats[key] = sum(values) + records = [run["records"] for run in per_run if "records" in run] + if records: + stats["records"] = max(records) + return stats + + +def per_run_values(runs: list[dict], method: str | None = None) -> dict[str, list[float]]: + """One value per run and metric, the input of the cluster bootstrap.""" + values: dict[str, list[float]] = {} + for run in runs: + stats = run_stats(run, method) + for key, value in stats.items(): + if isinstance(value, (int, float)): + values.setdefault(key, []).append(float(value)) + return values + + +def cluster_bootstrap_ci( + rng: random.Random, + baseline_values: dict[str, list[float]], + feature_values: dict[str, list[float]], + metrics: tuple[str, ...], + n_iter: int = BOOTSTRAP_ITERATIONS, +) -> dict[str, float]: + """Resample whole runs to estimate run-to-run noise per metric. + + Mirrors the engine summary's cluster bootstrap, which resamples block rows + and cannot be reused for request shaped runs. + """ + empty = {metric: 0.0 for metric in metrics} + usable = [ + metric + for metric in metrics + if len(baseline_values.get(metric, [])) >= 2 and len(feature_values.get(metric, [])) >= 2 + ] + if not usable: + return empty + + samples: dict[str, list[float]] = {metric: [] for metric in usable} + for metric in usable: + baseline_metric = baseline_values[metric] + feature_metric = feature_values[metric] + baseline_count = len(baseline_metric) + feature_count = len(feature_metric) + for _ in range(n_iter): + baseline_sample = [baseline_metric[rng.randrange(baseline_count)] for _ in range(baseline_count)] + feature_sample = [feature_metric[rng.randrange(feature_count)] for _ in range(feature_count)] + samples[metric].append(_mean(feature_sample) - _mean(baseline_sample)) + + result = dict(empty) + for metric in usable: + result[metric] = _ci_half_width(samples[metric]) + return result + + +def paired_record_delta( + rng: random.Random, + baseline_runs: list[dict], + feature_runs: list[dict], + method: str | None = None, +) -> dict | None: + """Per-record closed-loop delta with a bootstrap CI over records.""" + baseline_medians = arm_record_medians(baseline_runs, method) + feature_medians = arm_record_medians(feature_runs, method) + common = sorted(set(baseline_medians) & set(feature_medians)) + if not common: + return None + diffs = [feature_medians[index] - baseline_medians[index] for index in common] + base = _mean([baseline_medians[index] for index in common]) + return { + "records": len(common), + "baseline_ms": base, + "feature_ms": _mean([feature_medians[index] for index in common]), + "delta_ms": _mean(diffs), + "ci_ms": _bootstrap_ci(rng, diffs), + } + + +def make_change( + baseline_value: float | None, + feature_value: float | None, + ci_value: float, + floor_pct: float, + lower_is_better: bool, + aa_pct: float | None = None, +) -> dict | None: + """Build one `changes` entry in the shape the comment renderers consume.""" + if baseline_value is None or feature_value is None or baseline_value <= 0: + return None + pct = (feature_value - baseline_value) / baseline_value * 100.0 + ci_pct = ci_value / baseline_value * 100.0 if ci_value else 0.0 + change = { + "pct": round(pct, 4), + "ci_pct": round(ci_pct, 4), + "floor_pct": round(floor_pct, 4), + "sig": significance(pct, ci_pct, floor_pct, lower_is_better), + "baseline": baseline_value, + "feature": feature_value, + } + if aa_pct is not None: + change["aa_pct"] = round(aa_pct, 4) + return change + + +def aa_spread(runs: list[dict], method: str | None = None) -> dict[str, float]: + """Percent difference between the first two baseline runs, the noise floor.""" + if len(runs) < 2: + return {} + first = run_stats(runs[0], method) + second = run_stats(runs[1], method) + spread = {} + for key, value in first.items(): + if not isinstance(value, (int, float)) or value <= 0: + continue + other = second.get(key) + if isinstance(other, (int, float)): + spread[key] = (other - value) / value * 100.0 + return spread + + +def compute_changes( + rng: random.Random, + baseline_runs: list[dict], + feature_runs: list[dict], + baseline_stats: dict, + feature_stats: dict, + method: str | None = None, +) -> dict: + """Compare one corpus or one method between the two arms.""" + metrics = ( + ("mean", "mean_ms", True), + ("p50", "p50_ms", True), + ("p90", "p90_ms", True), + ("p99", "p99_ms", True), + ("closed_loop_rps", "closed_loop_rps", False), + ("cpu_per_request", "cpu_ms_per_request", True), + ) + ci = cluster_bootstrap_ci( + rng, + per_run_values(baseline_runs, method), + per_run_values(feature_runs, method), + tuple(stat_key for _, stat_key, _ in metrics) + ("record_median_ms",), + ) + spread = aa_spread(baseline_runs, method) + + changes = {} + for name, stat_key, lower_is_better in metrics: + change = make_change( + baseline_stats.get(stat_key), + feature_stats.get(stat_key), + ci.get(stat_key, 0.0), + PRACTICAL_FLOOR_PCT.get(name, 0.0), + lower_is_better, + spread.get(stat_key), + ) + if change: + changes[name] = change + + # The point estimate is the per-run record median averaged over runs, so its + # interval comes from the same whole-run bootstrap as the other metrics; the + # per-record pairing is kept as supplementary detail only. + change = make_change( + baseline_stats.get("record_median_ms"), + feature_stats.get("record_median_ms"), + ci.get("record_median_ms", 0.0), + PRACTICAL_FLOOR_PCT["record_median"], + True, + spread.get("record_median_ms"), + ) + if change: + paired = paired_record_delta(rng, baseline_runs, feature_runs, method) + if paired: + change["records"] = paired["records"] + change["paired_delta_ms"] = round(paired["delta_ms"], 6) + changes["record_median"] = change + return changes + + +def parity_entry() -> dict: + return { + "matched": 0, + "content_mismatch": 0, + "kind_mismatch": 0, + "missing": 0, + "nondeterministic": 0, + "excluded": 0, + "divergent_records": [], + } + + +def compare_responses( + reference: dict[int, dict], + other: dict[int, dict], + exclude: frozenset[int] = frozenset(), +) -> dict: + """Compare one run's digests against the reference run, per method. + + Records in `exclude` are unstable in the baseline itself and are only counted, + never judged. + """ + per_method: dict[str, dict] = {} + for index, expected in reference.items(): + method = expected["method"] + entry = per_method.setdefault(method, parity_entry()) + if index in exclude: + entry["excluded"] += 1 + continue + actual = other.get(index) + if actual is None: + entry["missing"] += 1 + elif actual["kind"] != expected["kind"]: + entry["kind_mismatch"] += 1 + elif actual["digest"] != expected["digest"]: + entry["content_mismatch"] += 1 + else: + entry["matched"] += 1 + continue + if len(entry["divergent_records"]) < MAX_DIVERGENT_REPORTED: + entry["divergent_records"].append(index) + # A record the reference run never answered but this run did is just as divergent. + for index, actual in other.items(): + if index in reference: + continue + entry = per_method.setdefault(actual["method"], parity_entry()) + if index in exclude: + entry["excluded"] += 1 + continue + entry["missing"] += 1 + if len(entry["divergent_records"]) < MAX_DIVERGENT_REPORTED: + entry["divergent_records"].append(index) + return per_method + + +def merge_parity(target: dict, addition: dict) -> None: + for method, counts in addition.items(): + entry = target.setdefault(method, parity_entry()) + for key in ("matched", "content_mismatch", "kind_mismatch", "missing", "nondeterministic", "excluded"): + entry[key] += counts.get(key, 0) + for index in counts["divergent_records"]: + if index not in entry["divergent_records"] and len(entry["divergent_records"]) < MAX_DIVERGENT_REPORTED: + entry["divergent_records"].append(index) + + +def parity_totals(per_method: dict) -> dict: + totals = { + "matched": 0, + "content_mismatch": 0, + "kind_mismatch": 0, + "missing": 0, + "nondeterministic": 0, + "excluded": 0, + } + for counts in per_method.values(): + for key in totals: + totals[key] += counts.get(key, 0) + # A record answered in only one of the two runs, or answered differently + # within the feature runs alone, counts as a divergence. + totals["mismatched"] = ( + totals["content_mismatch"] + totals["kind_mismatch"] + totals["missing"] + totals["nondeterministic"] + ) + return totals + + +def nondeterministic_count(runs: list[dict]) -> int: + total = 0 + for run in runs: + entries = report_field(run["report"], "nondeterministic") + if isinstance(entries, list): + total += len(entries) + elif isinstance(entries, int): + total += entries + return total + + +def nondeterministic_records(runs: list[dict]) -> tuple[set[int], bool]: + """Record indexes a run answered differently within itself. + + The second value is true when a run only reported a count, so the records + cannot be identified. + """ + indexes: set[int] = set() + unknown = False + for run in runs: + entries = report_field(run["report"], "nondeterministic") + if isinstance(entries, list): + indexes.update(int(index) for index in entries) + total = report_field(run["report"], "nondeterministic_total") + if isinstance(total, int) and total > len(entries): + unknown = True + elif isinstance(entries, int) and entries > 0: + unknown = True + return indexes, unknown + + +def compute_parity(baseline_runs: list[dict], feature_runs: list[dict]) -> dict: + """Compare every run's digests against the first baseline run. + + Records that are unstable in the baseline itself, answered differently + between baseline runs or within one, are excluded from the decision and + reported; the feature is judged on the remaining stable records, where its + own nondeterminism counts as a divergence. + """ + reference = baseline_runs[0]["responses"] + + baseline_parity: dict[str, dict] = {} + for run in baseline_runs[1:]: + merge_parity(baseline_parity, compare_responses(reference, run["responses"])) + baseline_nondeterministic, baseline_unknown = nondeterministic_records(baseline_runs) + unstable = set(baseline_nondeterministic) + for counts in baseline_parity.values(): + unstable.update(counts["divergent_records"]) + baseline_totals = parity_totals(baseline_parity) + if baseline_totals["mismatched"] > sum(len(c["divergent_records"]) for c in baseline_parity.values()): + # More baseline divergences than we could list: the unstable set is incomplete. + baseline_unknown = True + + feature_nondeterministic, feature_unknown = nondeterministic_records(feature_runs) + excluded = frozenset(unstable) + feature_parity: dict[str, dict] = {} + for run in feature_runs: + merge_parity(feature_parity, compare_responses(reference, run["responses"], excluded)) + for index in sorted(feature_nondeterministic - unstable): + expected = reference.get(index) + if expected is None: + continue + entry = feature_parity.setdefault(expected["method"], parity_entry()) + entry["nondeterministic"] += 1 + if index not in entry["divergent_records"] and len(entry["divergent_records"]) < MAX_DIVERGENT_REPORTED: + entry["divergent_records"].append(index) + feature_totals = parity_totals(feature_parity) + nondeterministic = nondeterministic_count(baseline_runs + feature_runs) + + records = len(reference) + stable_records = records - len(unstable & set(reference)) + if baseline_unknown or feature_unknown or (unstable and stable_records == 0): + status = "inconclusive" + elif feature_totals["mismatched"]: + status = "mismatch" + else: + status = "matched" + + excluded_list = sorted(unstable) + return { + "status": status, + "records": records, + "feature": {"totals": feature_totals, "methods": feature_parity}, + "baseline": {"totals": baseline_totals, "methods": baseline_parity}, + "nondeterministic": nondeterministic, + "excluded": excluded_list[:MAX_DIVERGENT_REPORTED], + "excluded_count": len(excluded_list), + "line": parity_line( + status, records, feature_totals, baseline_totals, nondeterministic, feature_parity, excluded_list + ), + } + + +def parity_line( + status: str, + records: int, + feature_totals: dict, + baseline_totals: dict, + nondeterministic: int, + feature_parity: dict, + excluded: list[int] | None = None, +) -> str: + excluded = excluded or [] + excluded_note = "" + if excluded: + shown = ", ".join(str(index) for index in excluded[:MAX_DIVERGENT_REPORTED]) + excluded_note = f"; {len(excluded)} unstable in the baseline, excluded (records {shown})" + checked = feature_totals["matched"] + feature_totals["mismatched"] + if status == "matched": + return ( + f"✅ matched {feature_totals['matched']}/{checked} responses " + f"over {records} records{excluded_note}" + ) + + details = [] + for method, counts in sorted(feature_parity.items()): + mismatched = ( + counts["content_mismatch"] + counts["kind_mismatch"] + counts["missing"] + counts.get("nondeterministic", 0) + ) + if not mismatched: + continue + indexes = ", ".join(str(index) for index in counts["divergent_records"]) + details.append(f"`{method}`: {mismatched} (records {indexes})") + + if status == "mismatch": + return "❌ {} content, {} kind mismatches, {} answered in one arm only, {} nondeterministic in the feature only — {}{}".format( + feature_totals["content_mismatch"], + feature_totals["kind_mismatch"], + feature_totals["missing"], + feature_totals.get("nondeterministic", 0), + "; ".join(details) if details else "no per-method detail", + excluded_note, + ) + + reasons = [] + if baseline_totals["mismatched"]: + reasons.append(f"{baseline_totals['mismatched']} baseline-vs-baseline mismatches") + if nondeterministic: + reasons.append(f"{nondeterministic} records answered differently within a run") + if not reasons: + reasons.append("no stable records to judge") + suffix = f"; feature mismatches: {feature_totals['mismatched']}" if feature_totals["mismatched"] else "" + return "⚠️ inconclusive — " + ", ".join(reasons) + suffix + + +def parity_change(parity: dict) -> dict: + """Fold parity into `changes` so the shared verdict helper sees it.""" + totals = parity["feature"]["totals"] + change = { + "pct": 0.0, + "ci_pct": 0.0, + "floor_pct": 0.0, + "sig": "bad" if parity["status"] == "mismatch" else "neutral", + "mismatched": totals["mismatched"], + "excluded": parity.get("excluded_count", 0), + } + if parity["status"] == "inconclusive": + change["informational"] = True + change["informational_reason"] = "informational, responses are not deterministic" + return change + + +def corpus_methods(baseline_runs: list[dict], feature_runs: list[dict]) -> list[str]: + methods = set() + for run in baseline_runs + feature_runs: + for entry in run["responses"].values(): + methods.add(entry["method"]) + for row in run["requests"]: + methods.add(row["method"]) + return sorted(methods) + + +def assert_same_head(runs: list[dict]) -> str | None: + """Refuse to compare runs that replayed against different chain tips.""" + hashes = {} + for run in runs: + head_hash = report_field(run["report"], "head_hash") + if head_hash: + hashes[run["name"]] = head_hash + distinct = set(hashes.values()) + if len(distinct) > 1: + detail = ", ".join(f"{name}={value}" for name, value in sorted(hashes.items())) + raise SystemExit(f"Runs replayed against different chain tips: {detail}") + return next(iter(distinct), None) + + +def fmt_opt_ms(value: float | None) -> str: + return fmt_ms(value) if isinstance(value, (int, float)) else "n/a" + + +def fmt_opt_num(value: float | None, digits: int = 2) -> str: + return f"{value:.{digits}f}" if isinstance(value, (int, float)) else "n/a" + + +def change_str(change: dict | None) -> str: + if not change: + return "n/a" + emoji = {"good": "✅", "bad": "❌", "neutral": "⚪"}[ + "neutral" if change.get("informational") else change["sig"] + ] + details = [f"±{change['ci_pct']:.2f}%", f"floor {change['floor_pct']:.2f}%"] + if "aa_pct" in change: + details.append(f"A/A {change['aa_pct']:+.2f}%") + if change.get("informational"): + details.append("informational") + return f"{change['pct']:+.2f}% {emoji} ({', '.join(details)})" + + +def generate_metric_table(summary: dict, baseline_label: str, feature_label: str) -> str: + baseline = summary["baseline"]["stats"] + feature = summary["feature"]["stats"] + changes = summary["changes"] + rows = [ + ("Mean", "mean_ms", "mean", fmt_opt_ms), + ("P50", "p50_ms", "p50", fmt_opt_ms), + ("P90", "p90_ms", "p90", fmt_opt_ms), + ("P99", "p99_ms", "p99", fmt_opt_ms), + ("Record median", "record_median_ms", "record_median", fmt_opt_ms), + ("Closed-loop rps", "closed_loop_rps", "closed_loop_rps", fmt_opt_num), + ("CPU / request", "cpu_ms_per_request", "cpu_per_request", fmt_opt_ms), + ] + lines = [ + f"| Metric | {baseline_label} | {feature_label} | Change |", + "|--------|------|--------|--------|", + ] + for label, stat_key, change_key, formatter in rows: + if stat_key not in baseline and stat_key not in feature: + continue + lines.append( + f"| {label} | {formatter(baseline.get(stat_key))} | " + f"{formatter(feature.get(stat_key))} | {change_str(changes.get(change_key))} |" + ) + lines.append( + f"| Error rate | {fmt_opt_num(baseline.get('error_rate_pct'))}% | " + f"{fmt_opt_num(feature.get('error_rate_pct'))}% | |" + ) + lines.append( + f"| Open-loop rps | {fmt_opt_num(baseline.get('open_loop_rps'))} | " + f"{fmt_opt_num(feature.get('open_loop_rps'))} | |" + ) + lines.append("") + return "\n".join(lines) + + +def generate_method_table(summary: dict) -> str: + methods = summary.get("methods") or {} + if len(methods) < 2: + return "" + lines = [ + "### Per-method", + "", + "| Method | Records | Baseline mean | Feature mean | Change | Response bytes | Parity |", + "|--------|---------|---------------|--------------|--------|----------------|--------|", + ] + for method in sorted(methods): + entry = methods[method] + baseline = entry["baseline"] + feature = entry["feature"] + totals = entry["parity"] + mismatched = totals["content_mismatch"] + totals["kind_mismatch"] + parity = f"{totals['matched']} matched" if not mismatched else f"❌ {mismatched} mismatched" + lines.append( + "| `{}` | {} | {} | {} | {} | {} | {} |".format( + method, + baseline.get("records", baseline.get("requests", 0)), + fmt_opt_ms(baseline.get("mean_ms")), + fmt_opt_ms(feature.get("mean_ms")), + change_str(entry["changes"].get("mean")), + fmt_opt_num(baseline.get("response_bytes_median"), 0), + parity, + ) + ) + lines.append("") + return "\n".join(lines) + + +def generate_markdown( + summary: dict, + baseline_label: str, + feature_label: str, + behind_baseline: int = 0, + repo: str = "", + baseline_ref: str = "", + baseline_name: str = "", + derek_command: str | None = None, +) -> str: + corpus = summary["corpus"] + lines = ["## Benchmark Results", "", "## Configuration"] + if derek_command: + lines.append(f"- Derek command: `{derek_command}`") + methods = ", ".join(corpus.get("methods") or summary.get("replayed_methods") or []) or "all" + top_gas_note = f" (top-gas {corpus['top_gas']})" if corpus.get("top_gas") else "" + tracer_notes = [] + if corpus.get("tracer"): + tracer_notes.append("tracer " + ", ".join(corpus["tracer"])) + for key in ("tracer_config", "trace_options"): + if corpus.get(key): + tracer_notes.append(f"{key.replace('_', '-')} `{json.dumps(corpus[key], separators=(',', ':'))}`") + tracer_note = f", {'; '.join(tracer_notes)}" if tracer_notes else "" + blocks_note = "" + if corpus.get("blocks_source"): + blocks_note = f", blocks {corpus.get('corpus_from')}..{corpus.get('corpus_to')} from {corpus['blocks_source']}" + if corpus.get("block_param") == "parent": + blocks_note += " at parent-block state" + lines.append( + f"- Corpus: `{corpus.get('name', 'static')}` ({corpus.get('source', 'static')}), " + f"class `{corpus.get('class', 'call')}`{top_gas_note}{tracer_note}{blocks_note}, {corpus.get('records', summary['records'])} records, " + f"methods: {methods}" + ) + lines.append( + f"- Replay: {summary['rps']} rps for {summary['duration']}, " + f"{summary['passes']} passes x {summary['concurrency']} workers, " + f"{summary['warmup_seconds']}s warmup, " + f"max tracing requests: {summary.get('max_tracing_requests') or 'default'}" + ) + if corpus.get("tip"): + lines.append(f"- Tip: {corpus['tip']} (`{corpus.get('tip_hash', 'unknown')}`)") + lines.append("") + if behind_baseline > 0: + plural = "s" if behind_baseline > 1 else "" + diff_link = f"https://github.com/{repo}/compare/{baseline_ref[:12]}...{baseline_name}" + lines.append( + f"> ⚠️ Feature is [**{behind_baseline} commit{plural} behind `{baseline_name}`**]({diff_link}). " + "Consider rebasing for accurate results." + ) + lines.append("") + lines.append(generate_metric_table(summary, baseline_label, feature_label)) + meta_parts = [ + f"{summary['records']} records", + "mode: call", + f"{summary['run_pairs']} run pairs" if summary.get("run_pairs") else "", + ] + lines.append(f"*{', '.join(part for part in meta_parts if part)}*") + lines.append("") + lines.append(f"**Parity:** {summary['parity']['line']}") + method_table = generate_method_table(summary) + if method_table: + lines.append("") + lines.append(method_table) + lines.extend(generate_observability_section(summary)) + return "\n".join(lines) + + +def build_summary(args, baseline_runs: list[dict], feature_runs: list[dict]) -> dict: + rng = random.Random(42) + head_hash = assert_same_head(baseline_runs + feature_runs) + + baseline_stats = arm_stats(baseline_runs) + feature_stats = arm_stats(feature_runs) + changes = compute_changes(rng, baseline_runs, feature_runs, baseline_stats, feature_stats) + parity = compute_parity(baseline_runs, feature_runs) + changes["parity"] = parity_change(parity) + + corpus = {"name": "static", "source": "static", "class": args.klass} + if args.corpus_meta: + meta_path = Path(args.corpus_meta) + if meta_path.is_file(): + with meta_path.open() as f: + corpus.update(json.load(f)) + if head_hash and not corpus.get("tip_hash"): + corpus["tip_hash"] = head_hash + + methods = corpus_methods(baseline_runs, feature_runs) + method_entries = {} + if len(methods) > 1: + for method in methods: + method_baseline = arm_stats(baseline_runs, method) + method_feature = arm_stats(feature_runs, method) + if not method_baseline or not method_feature: + continue + method_entries[method] = { + "baseline": method_baseline, + "feature": method_feature, + "changes": compute_changes( + rng, baseline_runs, feature_runs, method_baseline, method_feature, method + ), + "parity": parity["feature"]["methods"].get( + method, + {"matched": 0, "content_mismatch": 0, "kind_mismatch": 0, "missing": 0, "divergent_records": []}, + ), + } + + records = parity["records"] or corpus.get("records") or 0 + summary = { + "mode": "call", + "blocks": records, + "records": records, + "corpus": corpus, + "replayed_methods": methods, + "rps": args.rps, + "duration": args.duration, + "passes": args.passes, + "concurrency": args.concurrency, + "warmup_seconds": args.warmup_seconds, + "run_pairs": args.run_pairs, + "baseline": { + "name": args.baseline_name or "baseline", + "ref": args.baseline_ref or "main", + "stats": baseline_stats, + }, + "feature": { + "name": args.feature_name or "feature", + "ref": args.feature_ref or "unknown", + "stats": feature_stats, + }, + "changes": changes, + "methods": method_entries, + "parity": parity, + } + if args.max_tracing_requests: + summary["max_tracing_requests"] = args.max_tracing_requests + if args.benchmark_id: + summary["benchmark_id"] = args.benchmark_id + + from_ms, to_ms = resolve_observability_range( + args.observability_from_ms, args.observability_to_ms + ) + logs_url = args.logs_url + traces_url = args.traces_url + if args.benchmark_id and from_ms and to_ms: + logs_url = logs_url or build_grafana_logs_url(args.benchmark_id, from_ms, to_ms) + traces_url = traces_url or build_grafana_traces_url(args.benchmark_id, from_ms, to_ms) + observability = { + key: value + for key, value in { + "benchmark_id": args.benchmark_id, + "from_ms": from_ms, + "to_ms": to_ms, + "grafana_url": args.grafana_url, + "logs_url": logs_url, + "traces_url": traces_url, + }.items() + if value + } + if observability: + summary["observability"] = observability + return summary + + +def parse_args(argv: list[str] | None = None): + parser = argparse.ArgumentParser(description="Summarize an RPC replay benchmark") + parser.add_argument("--baseline-dir", nargs="+", required=True, help="Baseline run directories") + parser.add_argument("--feature-dir", nargs="+", required=True, help="Feature run directories") + parser.add_argument("--output-summary", required=True, help="Output JSON summary path") + parser.add_argument("--output-markdown", required=True, help="Output markdown path") + parser.add_argument("--repo", default="paradigmxyz/reth", help="GitHub repo (owner/name)") + parser.add_argument("--baseline-ref", default=None, help="Baseline commit SHA") + parser.add_argument("--baseline-name", default=None, help="Baseline display name") + parser.add_argument("--feature-name", default=None, help="Feature branch name") + parser.add_argument("--feature-ref", default=None, help="Feature commit SHA") + parser.add_argument("--behind-baseline", type=int, default=0, help="Commits behind baseline") + parser.add_argument("--corpus-meta", default=None, help="corpus.meta.json written by the extract step") + parser.add_argument("--class", dest="klass", default="call", help="Replayed method class") + parser.add_argument("--rps", default="", help="Configured open-loop request rate") + parser.add_argument("--duration", default="", help="Configured open-loop duration") + parser.add_argument("--passes", default="", help="Configured closed-loop passes") + parser.add_argument("--concurrency", default="", help="Configured closed-loop workers") + parser.add_argument("--warmup-seconds", default="", help="Warmup duration in seconds") + parser.add_argument("--run-pairs", type=int, default=None, help="Configured number of run pairs") + parser.add_argument("--max-tracing-requests", default=None, help="Node tracing request limit") + parser.add_argument("--benchmark-id", default=None, help="Benchmark ID used for OTLP labels") + parser.add_argument("--grafana-url", default=None, help="Grafana dashboard URL") + parser.add_argument("--logs-url", default=None, help="Grafana Explore URL for benchmark logs") + parser.add_argument("--traces-url", default=None, help="Grafana Explore URL for benchmark traces") + parser.add_argument("--observability-from-ms", type=int, default=None, help="Observability range start") + parser.add_argument("--observability-to-ms", type=int, default=None, help="Observability range end") + parser.add_argument( + "--derek-command", + default=os.environ.get("DEREK_BENCH_COMMAND"), + help="Full derek bench command", + ) + return parser.parse_args(argv) + + +def main(argv: list[str] | None = None) -> int: + args = parse_args(argv) + if args.run_pairs is not None and args.run_pairs < 1: + print("--run-pairs must be greater than zero", file=sys.stderr) + return 1 + baseline_runs = [load_run(Path(path)) for path in args.baseline_dir] + feature_runs = [load_run(Path(path)) for path in args.feature_dir] + + summary = build_summary(args, baseline_runs, feature_runs) + + base_url = f"https://github.com/{args.repo}/commit" + baseline_label = f"[`{summary['baseline']['name']}`]({base_url}/{summary['baseline']['ref']})" + feature_label = f"[`{summary['feature']['name']}`]({base_url}/{summary['feature']['ref']})" + markdown = generate_markdown( + summary, + baseline_label, + feature_label, + behind_baseline=args.behind_baseline, + repo=args.repo, + baseline_ref=summary["baseline"]["ref"], + baseline_name=summary["baseline"]["name"], + derek_command=args.derek_command, + ) + + with open(args.output_summary, "w") as f: + json.dump(summary, f, indent=2) + print(f"Summary written to {args.output_summary}") + with open(args.output_markdown, "w") as f: + f.write(markdown) + print(f"Markdown written to {args.output_markdown}") + print(f"Parity: {summary['parity']['line']}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/scripts/bench-job-summary.js b/.github/scripts/bench-job-summary.js index e79085eb2fd..129507b7afa 100644 --- a/.github/scripts/bench-job-summary.js +++ b/.github/scripts/bench-job-summary.js @@ -82,6 +82,11 @@ module.exports = async function ({ core, context, chartSha, grafanaUrl, logsUrl, } md += '\n'; + // Response parity (call mode) + if (summary.parity) { + md += `**Parity:** ${summary.parity.line}\n\n`; + } + // Wait time breakdown const wtRows = waitTimeRows(summary); if (wtRows.length > 0) { @@ -111,8 +116,8 @@ module.exports = async function ({ core, context, chartSha, grafanaUrl, logsUrl, md += '\n'; } - // Charts - if (chartSha) { + // Charts (call mode does not generate any) + if (chartSha && summary.mode !== 'call') { const prNum = prNumber || '0'; const baseUrl = `https://raw.githubusercontent.com/decofe/reth-bench-charts/${chartSha}/pr/${prNum}/${runId}`; const charts = [ diff --git a/.github/scripts/bench-txgen-extract.sh b/.github/scripts/bench-txgen-extract.sh index a513dcfd1d4..8d1c9953cbc 100755 --- a/.github/scripts/bench-txgen-extract.sh +++ b/.github/scripts/bench-txgen-extract.sh @@ -15,8 +15,28 @@ # /benchmark-blocks.ndjson # In RPC mode, the corresponding names end in `-transactions.ndjson`. # +# In call mode it instead writes a request corpus replayed by `bench call`: +# /corpus.jsonl[.gz] one JSON-RPC request per line +# /corpus.meta.json record counts and the tip it was built for +# +# Corpus sources +# -------------- +# `corpus=static` generates the corpus from the chain around the snapshot tip +# for the requested class. `corpus=` replays a corpus staged on the runner +# under BENCH_CORPUS_DIR (default /reth-bench/corpora), which holds: +# +# .jsonl.gz the corpus itself, gzip optional +# .sha256 optional checksum, verified before the corpus is used +# tracers/.js JavaScript tracers selectable as tracer=js: +# +# Provisioning that directory is manual (copy the file onto the runner). Names +# must not contain path separators. Corpus contents are never logged and never +# uploaded as build artifacts; only the record counts in corpus.meta.json are. +# # Required env: SCHELK_MOUNT, BENCH_RPC_URL, BENCH_BLOCKS, BENCH_WARMUP_BLOCKS -# Optional env: BENCH_EXECUTION_MODE, BENCH_BIG_BLOCKS, BENCH_BIG_BLOCKS_TARGET_GAS, BENCH_BAL +# Optional env: BENCH_EXECUTION_MODE, BENCH_BIG_BLOCKS, BENCH_BIG_BLOCKS_TARGET_GAS, BENCH_BAL, +# BENCH_CORPUS, BENCH_CORPUS_DIR, BENCH_CALL_CLASS, BENCH_CALL_METHODS, BENCH_CALL_TOP_GAS, +# BENCH_CALL_TRACER, BENCH_CALL_TRACER_CONFIG, BENCH_CALL_TRACE_OPTIONS, BENCH_CALL_NAMESPACE set -euxo pipefail BINARY="$1" @@ -69,6 +89,11 @@ RETH_ARGS=( --no-persist-peers ) +# Call mode probes the snapshot's tracing range before the node is stopped. +if [ "$EXECUTION_MODE" = "call" ]; then + RETH_ARGS+=(--http.api "eth,net,web3,debug,trace") +fi + if "$BINARY" node --help 2>/dev/null | grep -qF -- '--debug.startup-sync-state-idle'; then RETH_ARGS+=(--debug.startup-sync-state-idle) fi @@ -104,12 +129,304 @@ HEAD_HEX=$(jq -r '.result' <<< "$HEAD_JSON") HEAD_DEC=$((16#${HEAD_HEX#0x})) echo "Snapshot chain tip: ${HEAD_DEC}" -# Stop the throwaway node and recover snapshot before extraction. -sudo systemctl stop "$RETH_SCOPE" 2>/dev/null || true -sudo systemctl reset-failed "$RETH_SCOPE" 2>/dev/null || true -kill "${TAIL_PID:-}" 2>/dev/null || true -TAIL_PID= -sudo schelk recover -y --kill || true +HEAD_HASH=$(curl -sf http://127.0.0.1:8545 -X POST \ + -H 'Content-Type: application/json' \ + -d '{"jsonrpc":"2.0","method":"eth_getBlockByNumber","params":["latest",false],"id":1}' \ + | jq -r '.result.hash') + +# Returns non-zero when the snapshot can no longer trace the given block, which +# happens once the block falls out of the retained account and storage history. +# The archive node serves the blocks after the snapshot tip for the call +# classes; probe it once so an outage falls back instead of failing the run. +archive_rpc_reachable() { + local response + response=$(curl -sf -m 15 "$BENCH_RPC_URL" -X POST -H 'Content-Type: application/json' \ + --data '{"jsonrpc":"2.0","id":1,"method":"eth_chainId","params":[]}' 2>/dev/null) || return 1 + [[ "$response" == *'"result"'* ]] +} + +block_is_traceable() { + local block="$1" + local block_hex tx_hash response + block_hex=$(printf '0x%x' "$block") + tx_hash=$(curl -sf http://127.0.0.1:8545 -X POST \ + -H 'Content-Type: application/json' \ + -d "{\"jsonrpc\":\"2.0\",\"method\":\"eth_getBlockByNumber\",\"params\":[\"${block_hex}\",false],\"id\":1}" \ + | jq -r '.result.transactions[0] // empty') + if [ -z "$tx_hash" ]; then + echo "Block ${block} has no transactions, skipping traceability probe" + return 0 + fi + # The block class never resolves hashes, so it probes the block-addressed + # method it will replay; the transaction class probes by hash, which also + # detects a pruned transaction lookup. + local probe + if [ "$CALL_CLASS" = "traceblock" ]; then + probe="{\"jsonrpc\":\"2.0\",\"method\":\"debug_traceBlockByNumber\",\"params\":[\"${block_hex}\",{\"tracer\":\"callTracer\",\"tracerConfig\":{\"onlyTopCall\":true}}],\"id\":1}" + else + probe="{\"jsonrpc\":\"2.0\",\"method\":\"debug_traceTransaction\",\"params\":[\"${tx_hash}\",{\"tracer\":\"callTracer\"}],\"id\":1}" + fi + if ! response=$(curl -sf http://127.0.0.1:8545 -X POST \ + -H 'Content-Type: application/json' \ + -d "$probe"); then + echo "traceability probe request for block ${block} failed" + return 1 + fi + if jq -e 'has("error")' <<< "$response" > /dev/null 2>&1; then + local message + message="$(jq -r '.error.message' <<< "$response")" + # A transaction the node just listed in the block but cannot find by hash + # means the transaction lookup index is pruned, so no block in the range + # will do any better; report that instead of shrinking the range. + if [[ "$message" == *"transaction not found"* ]]; then + echo "::error::Snapshot has no transaction hash index (transaction lookup is pruned); hash-addressed classes cannot run on it. Use class=tracecall or class=traceblock, or a snapshot that keeps the transaction lookup." + exit 1 + fi + echo "Block ${block} is not traceable: ${message}" + return 1 + fi + return 0 +} + +CALL_BLOCKS="${BENCH_BLOCKS:-20}" +CALL_CLASS="${BENCH_CALL_CLASS:-call}" +CORPUS_SOURCE="${BENCH_CORPUS:-static}" +if [ "$EXECUTION_MODE" = "call" ] && [ "$CORPUS_SOURCE" = "static" ]; then + case "$CALL_CLASS" in + tracetx|traceblock) + # The snapshot is a pruned full node whose persisted state can lag its + # header tip, so the newest blocks may trace against stale state ("nonce + # too high"). Walk down from the tip to the newest block that actually + # traces, then window below it; also shrink the window if its oldest + # block has fallen past the retained history floor. + CORPUS_TOP="$HEAD_DEC" + probe_budget="${BENCH_TRACE_PROBE_DEPTH:-1024}" + while ! block_is_traceable "$CORPUS_TOP"; do + probe_budget=$(( probe_budget - 1 )) + if [ "$probe_budget" -le 0 ] || [ "$CORPUS_TOP" -le 1 ]; then + echo "::error::Snapshot has no traceable block near its tip ${HEAD_DEC} (searched down to ${CORPUS_TOP}); its persisted state lags too far behind the header for class ${CALL_CLASS}." + exit 1 + fi + CORPUS_TOP=$(( CORPUS_TOP - 1 )) + done + if [ "$CORPUS_TOP" -ne "$HEAD_DEC" ]; then + echo "Newest traceable block is ${CORPUS_TOP}, $(( HEAD_DEC - CORPUS_TOP )) below the tip" + fi + while [ "$CALL_BLOCKS" -gt 1 ] && ! block_is_traceable $(( CORPUS_TOP - CALL_BLOCKS + 1 )); do + CALL_BLOCKS=$(( CALL_BLOCKS / 2 )) + echo "Retrying traceability probe with ${CALL_BLOCKS} source blocks" + done + echo "Snapshot traces blocks $(( CORPUS_TOP - CALL_BLOCKS + 1 ))..${CORPUS_TOP}" + ;; + esac +fi + +# Stop the throwaway node and recover the snapshot before extraction. Call mode +# keeps it up: the trace classes read their blocks from it, and the exit trap +# stops it once the corpus is staged. +if [ "$EXECUTION_MODE" != "call" ]; then + sudo systemctl stop "$RETH_SCOPE" 2>/dev/null || true + sudo systemctl reset-failed "$RETH_SCOPE" 2>/dev/null || true + kill "${TAIL_PID:-}" 2>/dev/null || true + TAIL_PID= + sudo schelk recover -y --kill || true +fi + +# --- Stage the call corpus --- + +if [ "$EXECUTION_MODE" = "call" ]; then + CALL_METHODS="${BENCH_CALL_METHODS:-}" + CORPUS_FILE="$OUTPUT_DIR/corpus.jsonl" + + read_corpus() { + case "$1" in + *.gz) gzip -dc "$1" ;; + *) cat "$1" ;; + esac + } + + if [ "$CORPUS_SOURCE" = "static" ]; then + TXGEN_ETHEREUM="$(which txgen-ethereum)" + if [ "$CALL_BLOCKS" -le 0 ]; then + echo "::error::BENCH_BLOCKS must be greater than 0" + exit 1 + fi + case "$CALL_CLASS" in + call|tracecall) + # Transactions of the blocks after the tip are exactly the calls a + # client would have served against the tip state; only the archive + # node has those blocks. When it is unreachable, the blocks just below + # the tip on the snapshot stand in: their calls replay against the tip + # state too, only with more reverts. + CORPUS_FORMAT=calls + if archive_rpc_reachable; then + CORPUS_FROM=$(( HEAD_DEC + 1 )) + CORPUS_TO=$(( HEAD_DEC + CALL_BLOCKS )) + CORPUS_RPC="$BENCH_RPC_URL" + CORPUS_BLOCKS_SOURCE=archive + else + echo "::warning::Archive node ${BENCH_RPC_URL} is unreachable; building the ${CALL_CLASS} corpus from snapshot blocks below the tip" + CORPUS_FROM=$(( HEAD_DEC - CALL_BLOCKS + 1 )) + CORPUS_TO="$HEAD_DEC" + CORPUS_RPC="http://127.0.0.1:8545" + CORPUS_BLOCKS_SOURCE=snapshot + # These transactions already executed, so replay each against the + # state before its block instead of the tip, where its preconditions + # no longer hold. + CORPUS_BLOCK_PARAM=parent + fi + ;; + tracetx|traceblock) + # These blocks are on the snapshot, so the throwaway node serves them + # and the archive node is not needed. CORPUS_TOP is the newest block the + # probe found traceable, which may sit below the header tip. + CORPUS_FROM=$(( CORPUS_TOP - CALL_BLOCKS + 1 )) + CORPUS_TO="$CORPUS_TOP" + CORPUS_FORMAT=traces + CORPUS_RPC="http://127.0.0.1:8545" + CORPUS_BLOCKS_SOURCE=snapshot + ;; + *) + echo "::error::Unknown call class: ${CALL_CLASS}" + exit 1 + ;; + esac + CORPUS_METHOD_ARGS=() + if [ -n "$CALL_METHODS" ]; then + CORPUS_METHOD_ARGS+=(--methods "$CALL_METHODS") + fi + if [ -n "${BENCH_CALL_TOP_GAS:-}" ]; then + CORPUS_METHOD_ARGS+=(--top-gas "$BENCH_CALL_TOP_GAS") + fi + if [ -n "${BENCH_CALL_TRACER:-}" ]; then + IFS=',' read -r -a tracer_specs <<< "$BENCH_CALL_TRACER" + for spec in "${tracer_specs[@]}"; do + spec="${spec// /}" + [ -z "$spec" ] && continue + case "$spec" in + js:*) + tracer_name="${spec#js:}" + case "$tracer_name" in + */*|.*|"") + echo "::error::Invalid tracer name: ${spec}" + exit 1 + ;; + esac + tracer_file="${BENCH_CORPUS_DIR:-/reth-bench/corpora}/tracers/${tracer_name}.js" + if [ ! -f "$tracer_file" ]; then + echo "::error::JS tracer ${tracer_name} not found at ${tracer_file}" + exit 1 + fi + CORPUS_METHOD_ARGS+=(--tracer "js:${tracer_file}") + ;; + *) + CORPUS_METHOD_ARGS+=(--tracer "$spec") + ;; + esac + done + fi + if [ -n "${BENCH_CALL_TRACER_CONFIG:-}" ]; then + CORPUS_METHOD_ARGS+=(--tracer-config "$BENCH_CALL_TRACER_CONFIG") + fi + if [ -n "${CORPUS_BLOCK_PARAM:-}" ]; then + CORPUS_METHOD_ARGS+=(--block-param "$CORPUS_BLOCK_PARAM") + fi + if [ -n "${BENCH_CALL_TRACE_OPTIONS:-}" ]; then + CORPUS_METHOD_ARGS+=(--trace-options "$BENCH_CALL_TRACE_OPTIONS") + fi + echo "Generating ${CALL_CLASS} corpus from blocks ${CORPUS_FROM}..${CORPUS_TO} (methods: ${CALL_METHODS:-default})" + "$TXGEN_ETHEREUM" extract \ + --rpc "$CORPUS_RPC" \ + --from "$CORPUS_FROM" \ + --to "$CORPUS_TO" \ + --format "$CORPUS_FORMAT" \ + "${CORPUS_METHOD_ARGS[@]}" \ + -o "$CORPUS_FILE" + else + case "$CORPUS_SOURCE" in + */*|.*) + echo "::error::Invalid corpus name: ${CORPUS_SOURCE}" + exit 1 + ;; + esac + CORPUS_DIR="${BENCH_CORPUS_DIR:-/reth-bench/corpora}" + CORPUS_SRC="" + for candidate in "$CORPUS_DIR/$CORPUS_SOURCE.jsonl.gz" "$CORPUS_DIR/$CORPUS_SOURCE.jsonl"; do + if [ -f "$candidate" ]; then + CORPUS_SRC="$candidate" + break + fi + done + if [ -z "$CORPUS_SRC" ]; then + echo "::error::Corpus ${CORPUS_SOURCE} not found in ${CORPUS_DIR} (expected ${CORPUS_SOURCE}.jsonl.gz or ${CORPUS_SOURCE}.jsonl)" + exit 1 + fi + CORPUS_SHA256="$CORPUS_DIR/$CORPUS_SOURCE.sha256" + if [ -f "$CORPUS_SHA256" ]; then + EXPECTED_SHA="$(awk '{print $1; exit}' "$CORPUS_SHA256")" + ACTUAL_SHA="$(sha256sum "$CORPUS_SRC" | awk '{print $1}')" + if [ "$EXPECTED_SHA" != "$ACTUAL_SHA" ]; then + echo "::error::Checksum mismatch for corpus ${CORPUS_SOURCE}" + exit 1 + fi + echo "Corpus ${CORPUS_SOURCE} checksum verified" + fi + case "$CORPUS_SRC" in + *.gz) CORPUS_FILE="$OUTPUT_DIR/corpus.jsonl.gz" ;; + esac + cp "$CORPUS_SRC" "$CORPUS_FILE" + fi + + CORPUS_RECORDS=$(read_corpus "$CORPUS_FILE" | wc -l | tr -d ' ') + if [ "$CORPUS_RECORDS" -le 0 ]; then + echo "::error::Corpus ${CORPUS_SOURCE} is empty" + exit 1 + fi + CORPUS_RECORDS_PER_METHOD=$(read_corpus "$CORPUS_FILE" | jq -r '.method' | sort | uniq -c \ + | jq -R -s 'split("\n") + | map(select(length > 0) | capture("^\\s*(?\\d+)\\s+(?.+)$") | {(.method): (.count | tonumber)}) + | add // {}') + jq -n \ + --arg source "$CORPUS_SOURCE" \ + --arg name "$CORPUS_SOURCE" \ + --arg class "$CALL_CLASS" \ + --arg methods "$CALL_METHODS" \ + --argjson records "$CORPUS_RECORDS" \ + --argjson records_per_method "$CORPUS_RECORDS_PER_METHOD" \ + --argjson tip "$HEAD_DEC" \ + --arg tip_hash "$HEAD_HASH" \ + --arg blocks_source "${CORPUS_BLOCKS_SOURCE:-}" \ + --arg block_param "${CORPUS_BLOCK_PARAM:-}" \ + --argjson corpus_from "${CORPUS_FROM:-null}" \ + --argjson corpus_to "${CORPUS_TO:-null}" \ + --argjson top_gas "${BENCH_CALL_TOP_GAS:-null}" \ + --arg tracer "${BENCH_CALL_TRACER:-}" \ + --arg namespace "${BENCH_CALL_NAMESPACE:-}" \ + --argjson tracer_config "${BENCH_CALL_TRACER_CONFIG:-null}" \ + --argjson trace_options "${BENCH_CALL_TRACE_OPTIONS:-null}" \ + '{ + source: (if $source == "static" then "static" else "custom" end), + name: $name, + class: $class, + methods: (if $methods == "" then [] else ($methods | split(",")) end), + records: $records, + records_per_method: $records_per_method, + tip: $tip, + tip_hash: $tip_hash, + blocks_source: (if $blocks_source == "" then null else $blocks_source end), + block_param: (if $block_param == "" then null else $block_param end), + corpus_from: $corpus_from, + corpus_to: $corpus_to, + top_gas: $top_gas, + tracer: (if $tracer == "" then null else ($tracer | split(",")) end), + namespace: (if $namespace == "" then null else $namespace end), + tracer_config: $tracer_config, + trace_options: $trace_options, + }' > "$OUTPUT_DIR/corpus.meta.json" + + echo "Corpus staged: ${CORPUS_RECORDS} records in ${CORPUS_FILE}" + exit 0 +fi # --- Extract payloads from the remote RPC --- diff --git a/.github/scripts/bench-txgen-install.sh b/.github/scripts/bench-txgen-install.sh index 6ac1d31897c..366a01d01eb 100755 --- a/.github/scripts/bench-txgen-install.sh +++ b/.github/scripts/bench-txgen-install.sh @@ -4,6 +4,7 @@ # # Optional env: # TXGEN_REPO – txgen repository URL (default: https://github.com/tempoxyz/txgen) +# TXGEN_REF – branch or commit to install instead of the default branch set -euxo pipefail TXGEN_REPO="${TXGEN_REPO:-https://github.com/tempoxyz/txgen}" @@ -31,4 +32,14 @@ elif [ -n "${TXGEN_TOKEN:-${GH_PROJECT_TOKEN:-${DEREK_PAT:-${DEREK_TOKEN:-}}}}" fi export CARGO_NET_GIT_FETCH_WITH_CLI=true -cargo install --git "$TXGEN_REPO" --locked txgen-ethereum bench-cli +INSTALL_ARGS=() +if [ -n "${TXGEN_REF:-}" ]; then + if [[ "$TXGEN_REF" =~ ^[0-9a-f]{7,40}$ ]]; then + INSTALL_ARGS+=(--rev "$TXGEN_REF") + else + INSTALL_ARGS+=(--branch "$TXGEN_REF") + fi + # A pinned ref must replace whatever build of the same version is installed. + INSTALL_ARGS+=(--force) +fi +cargo install --git "$TXGEN_REPO" ${INSTALL_ARGS[@]+"${INSTALL_ARGS[@]}"} --locked txgen-ethereum bench-cli diff --git a/.github/scripts/bench-txgen-run.sh b/.github/scripts/bench-txgen-run.sh index 6aaf9a113d9..902f4ebb174 100755 --- a/.github/scripts/bench-txgen-run.sh +++ b/.github/scripts/bench-txgen-run.sh @@ -4,10 +4,16 @@ # mount snapshot → start node → extract source blocks → warmup → replay → # convert txgen JSON report into legacy benchmark CSVs. # +# In call mode it replays the request corpus staged by bench-txgen-extract.sh +# instead, and writes responses.ndjson, record_timings.csv, requests.csv, +# report.json and cpu.json for bench-call-summary.py. +# # Usage: bench-txgen-run.sh

( provider: &P, - ) -> ProviderResult> + ) -> ProviderResult> where P: DBProvider + NodePrimitivesProvider + StorageSettingsCache + StaticFileProviderFactory, P::Tx: DbTx, @@ -782,7 +784,7 @@ impl<'a> EitherReader<'a, (), ()> { } } -impl EitherReader<'_, CURSOR, N> +impl EitherReader<'_, '_, CURSOR, N> where CURSOR: DbCursorRO, { @@ -813,7 +815,7 @@ where } } -impl EitherReader<'_, CURSOR, N> +impl EitherReader<'_, '_, CURSOR, N> where CURSOR: DbCursorRO, { @@ -830,7 +832,7 @@ where } } -impl EitherReader<'_, CURSOR, N> +impl EitherReader<'_, '_, CURSOR, N> where CURSOR: DbCursorRO, { @@ -878,7 +880,7 @@ where } } -impl EitherReader<'_, CURSOR, N> +impl EitherReader<'_, '_, CURSOR, N> where CURSOR: DbCursorRO, { @@ -924,7 +926,7 @@ where } } -impl EitherReader<'_, CURSOR, N> +impl EitherReader<'_, '_, CURSOR, N> where CURSOR: DbCursorRO, { @@ -1443,7 +1445,7 @@ mod rocksdb_tests { for (i, query) in queries.iter().enumerate() { // MDBX query via EitherReader - let mut mdbx_reader: EitherReader<'_, AccountsHistoryReadCursor, EthPrimitives> = + let mut mdbx_reader: EitherReader<'_, '_, AccountsHistoryReadCursor, EthPrimitives> = EitherReader::Database( mdbx_ro.tx_ref().cursor_read::().unwrap(), PhantomData, @@ -1532,7 +1534,7 @@ mod rocksdb_tests { for (i, query) in queries.iter().enumerate() { // MDBX query via EitherReader - let mut mdbx_reader: EitherReader<'_, StoragesHistoryReadCursor, EthPrimitives> = + let mut mdbx_reader: EitherReader<'_, '_, StoragesHistoryReadCursor, EthPrimitives> = EitherReader::Database( mdbx_ro.tx_ref().cursor_read::().unwrap(), PhantomData, diff --git a/crates/storage/provider/src/providers/database/provider.rs b/crates/storage/provider/src/providers/database/provider.rs index 7c968a44a1b..f6e9ef97034 100644 --- a/crates/storage/provider/src/providers/database/provider.rs +++ b/crates/storage/provider/src/providers/database/provider.rs @@ -3,7 +3,10 @@ use crate::{ changesets_utils::StorageRevertsIter, providers::{ database::{chain::ChainStorage, metrics, DatabaseProviderMetrics}, - rocksdb::{PendingRocksDBBatches, RocksDBProvider, RocksDBWriteCtx}, + rocksdb::{ + OwnedRocksReadSnapshot, PendingRocksDBBatches, RocksDBProvider, RocksDBWriteCtx, + RocksReadSnapshot, + }, static_file::{StaticFileWriteCtx, StaticFileWriter}, NodeTypesForProvider, StaticFileProvider, }, @@ -82,7 +85,7 @@ use std::{ fmt::Debug, ops::{Deref, DerefMut, Range, RangeBounds, RangeInclusive}, path::PathBuf, - sync::Arc, + sync::{Arc, OnceLock}, }; use tracing::{debug, instrument, trace}; @@ -201,6 +204,12 @@ pub struct DatabaseProvider { storage_settings: Arc>, /// `RocksDB` provider rocksdb_provider: RocksDBProvider, + /// `RocksDB` snapshot shared by all history lookups made through this provider. + /// + /// It lives as long as the provider, next to the MDBX read transaction, and pins the + /// `RocksDB` versions its cached iterators were created on for that long; see + /// [`Self::history_rocksdb_snapshot`]. + rocksdb_history_snapshot: OnceLock>, /// Manager for state trie overlays and cached changesets. overlay_manager: OverlayManager, /// Task runtime for spawning parallel I/O work. @@ -381,6 +390,7 @@ impl DatabaseProvider { overlay_manager, runtime, db_path, + rocksdb_history_snapshot: OnceLock::new(), pending_rocksdb_batches: Default::default(), commit_order, minimum_pruning_distance: MINIMUM_UNWIND_SAFE_DISTANCE, @@ -1029,6 +1039,7 @@ impl DatabaseProvider { overlay_manager, runtime, db_path, + rocksdb_history_snapshot: OnceLock::new(), pending_rocksdb_batches: Default::default(), commit_order: CommitOrder::Normal, minimum_pruning_distance: MINIMUM_UNWIND_SAFE_DISTANCE, @@ -1641,6 +1652,31 @@ impl ChangeSetReader for DatabaseProvider { } } +impl DatabaseProvider { + /// Returns the `RocksDB` snapshot used for history lookups, creating it on first use. + /// + /// A reading provider serves a single request, so one snapshot is created for all of its + /// history lookups instead of one per lookup. The snapshot also caches the raw iterator of + /// each history column family, which is what makes repeated lookups cheap. The storage + /// settings are read once here as well, so a lookup does not take the settings lock. + /// + /// The snapshot and its iterators pin the `RocksDB` snapshot and the SST files and memtables + /// they were created on until the provider is dropped, the same way the provider's MDBX read + /// transaction pins its pages. Provider lifetimes are request or job scoped, and a provider + /// whose MDBX transaction hits the read-transaction timeout errors on every later read and is + /// dropped by its caller, so that timeout bounds the `RocksDB` retention as well. + fn history_rocksdb_snapshot(&self) -> Option<&RocksReadSnapshot<'_>> { + self.rocksdb_history_snapshot + .get_or_init(|| { + self.cached_storage_settings() + .storage_v2 + .then(|| self.rocksdb_provider.owned_snapshot()) + }) + .as_ref() + .map(OwnedRocksReadSnapshot::as_snapshot) + } +} + impl HistoryReader for DatabaseProvider { fn account_history_info( &self, @@ -1649,17 +1685,10 @@ impl HistoryReader for DatabaseProvider lowest_available_block_number: Option, ) -> ProviderResult { let visible_tip = self.best_block_number()?; - self.with_rocksdb_snapshot(|rocksdb_ref| { - let mut reader = EitherReader::new_accounts_history(self, rocksdb_ref)?; - reader - .account_history_info( - address, - block_number, - lowest_available_block_number, - visible_tip, - ) - .map(Into::into) - }) + let mut reader = EitherReader::new_accounts_history(self, self.history_rocksdb_snapshot())?; + reader + .account_history_info(address, block_number, lowest_available_block_number, visible_tip) + .map(Into::into) } fn storage_history_info( @@ -1670,18 +1699,16 @@ impl HistoryReader for DatabaseProvider lowest_available_block_number: Option, ) -> ProviderResult { let visible_tip = self.best_block_number()?; - self.with_rocksdb_snapshot(|rocksdb_ref| { - let mut reader = EitherReader::new_storages_history(self, rocksdb_ref)?; - reader - .storage_history_info( - address, - storage_key, - block_number, - lowest_available_block_number, - visible_tip, - ) - .map(Into::into) - }) + let mut reader = EitherReader::new_storages_history(self, self.history_rocksdb_snapshot())?; + reader + .storage_history_info( + address, + storage_key, + block_number, + lowest_available_block_number, + visible_tip, + ) + .map(Into::into) } } diff --git a/crates/storage/provider/src/providers/mod.rs b/crates/storage/provider/src/providers/mod.rs index b51aa898aef..75624586368 100644 --- a/crates/storage/provider/src/providers/mod.rs +++ b/crates/storage/provider/src/providers/mod.rs @@ -28,8 +28,9 @@ pub use consistent::ConsistentProvider; pub(crate) mod rocksdb; pub use rocksdb::{ - PruneShardOutcome, PrunedIndices, RocksDBBatch, RocksDBBuilder, RocksDBIter, RocksDBProvider, - RocksDBRawIter, RocksDBStats, RocksDBTableStats, RocksReadSnapshot, RocksTx, + OwnedRocksReadSnapshot, PruneShardOutcome, PrunedIndices, RocksDBBatch, RocksDBBuilder, + RocksDBIter, RocksDBProvider, RocksDBRawIter, RocksDBStats, RocksDBTableStats, + RocksReadSnapshot, RocksTx, }; /// Helper trait to bound [`NodeTypes`] so that combined with database they satisfy diff --git a/crates/storage/provider/src/providers/rocksdb/mod.rs b/crates/storage/provider/src/providers/rocksdb/mod.rs index 46bb9724a5c..7a5c4f93ea8 100644 --- a/crates/storage/provider/src/providers/rocksdb/mod.rs +++ b/crates/storage/provider/src/providers/rocksdb/mod.rs @@ -4,8 +4,9 @@ mod invariants; mod metrics; mod provider; -pub(crate) use provider::{PendingRocksDBBatches, RocksDBWriteCtx}; pub use provider::{ - PruneShardOutcome, PrunedIndices, RocksDBBatch, RocksDBBuilder, RocksDBIter, RocksDBProvider, - RocksDBRawIter, RocksDBStats, RocksDBTableStats, RocksReadSnapshot, RocksTx, + OwnedRocksReadSnapshot, PruneShardOutcome, PrunedIndices, RocksDBBatch, RocksDBBuilder, + RocksDBIter, RocksDBProvider, RocksDBRawIter, RocksDBStats, RocksDBTableStats, + RocksReadSnapshot, RocksTx, }; +pub(crate) use provider::{PendingRocksDBBatches, RocksDBWriteCtx}; diff --git a/crates/storage/provider/src/providers/rocksdb/provider.rs b/crates/storage/provider/src/providers/rocksdb/provider.rs index 471b1ef7b7e..170d0619a49 100644 --- a/crates/storage/provider/src/providers/rocksdb/provider.rs +++ b/crates/storage/provider/src/providers/rocksdb/provider.rs @@ -28,7 +28,7 @@ use reth_storage_errors::{ use rocksdb::{ statistics::StatsLevel, BlockBasedOptions, Cache, ColumnFamilyDescriptor, CompactionPri, DBCompressionType, DBRawIteratorWithThreadMode, IteratorMode, OptimisticTransactionDB, - OptimisticTransactionOptions, Options, SnapshotWithThreadMode, Transaction, + OptimisticTransactionOptions, Options, ReadOptions, SnapshotWithThreadMode, Transaction, WriteBatchWithTransaction, WriteBufferManager, WriteOptions, DB, DEFAULT_COLUMN_FAMILY_NAME, }; use std::{ @@ -840,7 +840,20 @@ impl RocksDBProvider { /// /// Lighter weight than [`RocksTx`] — no write-conflict tracking, and `Send + Sync`. pub fn snapshot(&self) -> RocksReadSnapshot<'_> { - RocksReadSnapshot { inner: self.0.snapshot(), provider: self } + RocksReadSnapshot { + accounts_history_iter: Mutex::new(None), + storages_history_iter: Mutex::new(None), + inner: self.0.snapshot(), + provider: &self.0, + } + } + + /// Returns a read-only, point-in-time snapshot that owns the provider handle it reads through. + /// + /// Lets a reader keep one snapshot (and the iterators cached in it) alive for its whole + /// lifetime, instead of creating a snapshot per lookup. + pub fn owned_snapshot(&self) -> OwnedRocksReadSnapshot { + OwnedRocksReadSnapshot::new(self) } /// Creates a new transaction with MDBX-like semantics (read-your-writes, rollback). @@ -1606,8 +1619,16 @@ impl RocksDBProvider { /// /// Lighter weight than [`RocksTx`] — no transaction overhead, no write support. pub struct RocksReadSnapshot<'db> { + /// Raw iterator reused across account history lookups. + /// + /// Declared before `inner` so it is dropped before the snapshot it reads from. + accounts_history_iter: Mutex>>, + /// Raw iterator reused across storage history lookups. + /// + /// Declared before `inner` so it is dropped before the snapshot it reads from. + storages_history_iter: Mutex>>, inner: RocksReadSnapshotInner<'db>, - provider: &'db RocksDBProvider, + provider: &'db RocksDBProviderInner, } /// Inner enum to hold the snapshot for either read-write or secondary mode. @@ -1618,16 +1639,6 @@ enum RocksReadSnapshotInner<'db> { Secondary(&'db DB), } -impl<'db> RocksReadSnapshotInner<'db> { - /// Returns a raw iterator over a column family. - fn raw_iterator_cf(&self, cf: &rocksdb::ColumnFamily) -> RocksDBRawIterEnum<'_> { - match self { - Self::ReadWrite(snap) => RocksDBRawIterEnum::ReadWrite(snap.raw_iterator_cf(cf)), - Self::Secondary(db) => RocksDBRawIterEnum::ReadOnly(db.raw_iterator_cf(cf)), - } - } -} - impl fmt::Debug for RocksReadSnapshot<'_> { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_struct("RocksReadSnapshot") @@ -1639,7 +1650,28 @@ impl fmt::Debug for RocksReadSnapshot<'_> { impl<'db> RocksReadSnapshot<'db> { /// Gets the column family handle for a table. fn cf_handle(&self) -> Result<&'db rocksdb::ColumnFamily, DatabaseError> { - self.provider.get_cf_handle::() + self.provider.cf_handle::() + } + + /// Creates a raw iterator over `cf` that observes this snapshot. + /// + /// The iterator is created from the database handle rather than from the snapshot value, so + /// its lifetime is tied to the database and it can be cached in `self`. The snapshot is + /// attached through [`ReadOptions`], giving the same point-in-time view as reading through the + /// snapshot directly. + fn new_raw_iterator_cf(&self, cf: &rocksdb::ColumnFamily) -> RocksDBRawIterEnum<'db> { + match &self.inner { + RocksReadSnapshotInner::ReadWrite(snap) => { + let mut readopts = ReadOptions::default(); + readopts.set_snapshot(snap); + RocksDBRawIterEnum::ReadWrite( + self.provider.db_rw().raw_iterator_cf_opt(cf, readopts), + ) + } + RocksReadSnapshotInner::Secondary(db) => { + RocksDBRawIterEnum::ReadOnly((*db).raw_iterator_cf(cf)) + } + } } /// Gets a value from the specified table. @@ -1673,6 +1705,7 @@ impl<'db> RocksReadSnapshot<'db> { ) -> ProviderResult { let key = ShardedKey::new(address, block_number); self.history_info::( + &self.accounts_history_iter, key.encode().as_ref(), block_number, lowest_available_block_number, @@ -1700,6 +1733,7 @@ impl<'db> RocksReadSnapshot<'db> { ) -> ProviderResult { let key = StorageShardedKey::new(address, storage_key, block_number); self.history_info::( + &self.storages_history_iter, key.encode().as_ref(), block_number, lowest_available_block_number, @@ -1721,8 +1755,15 @@ impl<'db> RocksReadSnapshot<'db> { /// The result is derived from the history that is visible through `visible_tip`, not from the /// full contents of `RocksDB`. This lets a reader combine an older MDBX snapshot with a newer /// Rocks snapshot without routing through history entries that MDBX cannot see yet. + /// + /// `iter_cache` holds the raw iterator for `T`'s column family. Seeking an existing iterator + /// is much cheaper than constructing one, so the iterator is created on the first lookup and + /// reused by every later lookup through this snapshot. A lookup that finds the cache held by + /// another thread uses a private iterator instead of waiting. + #[expect(clippy::too_many_arguments)] fn history_info( &self, + iter_cache: &Mutex>>, encoded_key: &[u8], block_number: BlockNumber, lowest_available_block_number: Option, @@ -1743,7 +1784,17 @@ impl<'db> RocksReadSnapshot<'db> { }; let cf = self.cf_handle::()?; - let mut iter = self.inner.raw_iterator_cf(cf); + // A lookup on another thread may be holding the cached iterator; a private iterator costs + // what every lookup used to cost, whereas waiting would serialize the two lookups. + let mut guard = iter_cache.try_lock(); + let mut private_iter; + let iter = match guard.as_mut() { + Some(cached) => cached.get_or_insert_with(|| self.new_raw_iterator_cf(cf)), + None => { + private_iter = self.new_raw_iterator_cf(cf); + &mut private_iter + } + }; iter.seek(encoded_key); iter.status().map_err(|e| { @@ -1803,6 +1854,51 @@ impl<'db> RocksReadSnapshot<'db> { } } +/// A [`RocksReadSnapshot`] that owns the [`RocksDBProvider`] handle it reads through. +/// +/// [`RocksDBProvider::snapshot`] borrows the provider, so a reader that wants to keep a single +/// snapshot alive has to keep the provider handle next to it. This type does that, which lets the +/// snapshot cache its history iterators across lookups. +pub struct OwnedRocksReadSnapshot { + /// Borrows the allocation retained by `provider`. Declared first so the snapshot and its + /// iterators are released before the owning database handle. Boxing keeps its internal + /// references out of the movable owner, including when the owner is passed by value. + snapshot: Box>, + provider: RocksDBProvider, +} + +impl fmt::Debug for OwnedRocksReadSnapshot { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("OwnedRocksReadSnapshot").field("provider", &self.provider).finish() + } +} + +impl OwnedRocksReadSnapshot { + fn new(provider: &RocksDBProvider) -> Self { + let provider = provider.clone(); + let snapshot = provider.snapshot(); + // SAFETY: Every reference in `snapshot` points into the Arc allocation retained by + // `provider`, not into the movable provider handle. The handle is private and never + // replaced, and `snapshot` is dropped before it. `as_snapshot` restricts access to the + // lifetime of a borrow of this owner. + let snapshot = unsafe { + std::mem::transmute::, RocksReadSnapshot<'static>>(snapshot) + }; + Self { snapshot: Box::new(snapshot), provider } + } + + /// Returns the borrowed snapshot. + pub fn as_snapshot(&self) -> &RocksReadSnapshot<'_> { + // SAFETY: shortening the snapshot's lifetime to this borrow of `self` is sound; the owned + // provider handle keeps the database open for at least that long. + unsafe { + std::mem::transmute::<&RocksReadSnapshot<'static>, &RocksReadSnapshot<'_>>( + &self.snapshot, + ) + } + } +} + /// Outcome of pruning a history shard in `RocksDB`. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum PruneShardOutcome { @@ -3457,6 +3553,94 @@ mod tests { assert_eq!(last, Some((20, b"value_20".to_vec()))); } + #[test] + fn test_owned_history_snapshot_outlives_provider() { + let temp_dir = TempDir::new().unwrap(); + let provider = RocksDBBuilder::new(temp_dir.path()).with_default_tables().build().unwrap(); + let address = Address::repeat_byte(0x42); + let slot = B256::repeat_byte(0x43); + let chunk = IntegerList::new([100, 200, 300]).unwrap(); + provider + .put::(ShardedKey::new(address, u64::MAX), &chunk) + .unwrap(); + provider + .put::(StorageShardedKey::new(address, slot, u64::MAX), &chunk) + .unwrap(); + + let owned = provider.owned_snapshot(); + let snapshot = owned.as_snapshot(); + assert_eq!( + snapshot.account_history_info(address, 200, None, u64::MAX).unwrap(), + HistoryInfo::InChangeset(200) + ); + assert_eq!( + snapshot.storage_history_info(address, slot, 200, None, u64::MAX).unwrap(), + HistoryInfo::InChangeset(200) + ); + drop(provider); + + // Move the owner with populated iterator caches, then seek both forwards and backwards. + std::thread::spawn(move || { + let snapshot = owned.as_snapshot(); + for (block, expected) in [ + (400, HistoryInfo::InPlainState), + (50, HistoryInfo::NotYetWritten), + (200, HistoryInfo::InChangeset(200)), + ] { + assert_eq!( + snapshot.account_history_info(address, block, None, u64::MAX).unwrap(), + expected + ); + assert_eq!( + snapshot.storage_history_info(address, slot, block, None, u64::MAX).unwrap(), + expected + ); + } + }) + .join() + .unwrap(); + } + + #[test] + fn test_history_snapshot_cached_and_private_iterators_keep_same_view() { + let temp_dir = TempDir::new().unwrap(); + let provider = RocksDBBuilder::new(temp_dir.path()).with_default_tables().build().unwrap(); + let address = Address::repeat_byte(0x42); + let slot = B256::repeat_byte(0x43); + let account_key = ShardedKey::new(address, u64::MAX); + let storage_key = StorageShardedKey::new(address, slot, u64::MAX); + let chunk = IntegerList::new([100, 200]).unwrap(); + provider.put::(account_key.clone(), &chunk).unwrap(); + provider.put::(storage_key.clone(), &chunk).unwrap(); + + let owned = provider.owned_snapshot(); + let snapshot = owned.as_snapshot(); + let check = |snapshot: &RocksReadSnapshot<'_>, expected| { + assert_eq!( + snapshot.account_history_info(address, 125, None, u64::MAX).unwrap(), + expected + ); + assert_eq!( + snapshot.storage_history_info(address, slot, 125, None, u64::MAX).unwrap(), + expected + ); + }; + check(snapshot, HistoryInfo::InChangeset(200)); + + let updated = IntegerList::new([100, 150, 200]).unwrap(); + provider.put::(account_key, &updated).unwrap(); + provider.put::(storage_key, &updated).unwrap(); + check(&provider.snapshot(), HistoryInfo::InChangeset(150)); + check(snapshot, HistoryInfo::InChangeset(200)); + + // Force another thread to use private iterators after the database has changed. + let _accounts = snapshot.accounts_history_iter.lock(); + let _storages = snapshot.storages_history_iter.lock(); + std::thread::scope(|scope| { + scope.spawn(|| check(snapshot, HistoryInfo::InChangeset(200))).join().unwrap(); + }); + } + /// Tests the edge case where block < `lowest_available_block_number`. /// /// State queries reject this before the `RocksDB` lookup, so this verifies the low-level diff --git a/crates/storage/provider/src/traits/rocksdb_provider.rs b/crates/storage/provider/src/traits/rocksdb_provider.rs index 6ddbbc2bfe3..deacc142394 100644 --- a/crates/storage/provider/src/traits/rocksdb_provider.rs +++ b/crates/storage/provider/src/traits/rocksdb_provider.rs @@ -37,12 +37,12 @@ pub trait RocksDBProviderFactory { fn with_rocksdb_snapshot(&self, f: F) -> ProviderResult where Self: StorageSettingsCache, - F: FnOnce(RocksDBRefArg<'_>) -> ProviderResult, + F: FnOnce(RocksDBRefArg<'_, '_>) -> ProviderResult, { if self.cached_storage_settings().storage_v2 { let rocksdb = self.rocksdb_provider(); let snapshot = rocksdb.snapshot(); - return f(Some(snapshot)); + return f(Some(&snapshot)); } f(None) } From 5ecd9939437edb9fb4fd44f8dd60bc8250e82151 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A9a=20Narzis?= <78718413+lean-apple@users.noreply.github.com> Date: Mon, 14 Sep 2026 18:31:08 +0200 Subject: [PATCH 254/294] feat(snap-sync): build downloaded state updates from verified BALs (#27175) --- Cargo.lock | 4 + crates/snap-sync/Cargo.toml | 6 +- crates/snap-sync/src/bal.rs | 447 ++++++++++++++++++++++++++++++++++++ crates/snap-sync/src/lib.rs | 2 + 4 files changed, 458 insertions(+), 1 deletion(-) create mode 100644 crates/snap-sync/src/bal.rs diff --git a/Cargo.lock b/Cargo.lock index 71dc24ac381..dc937330fce 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -10416,9 +10416,13 @@ dependencies = [ "alloy-primitives", "alloy-rlp", "futures", + "reth-chainspec", "reth-db-api", "reth-downloaders", "reth-eth-wire-types", + "reth-ethereum-primitives", + "reth-evm", + "reth-evm-ethereum", "reth-network-p2p", "reth-network-peers", "reth-primitives-traits", diff --git a/crates/snap-sync/Cargo.toml b/crates/snap-sync/Cargo.toml index cfe053cab7a..56e660796e4 100644 --- a/crates/snap-sync/Cargo.toml +++ b/crates/snap-sync/Cargo.toml @@ -18,7 +18,7 @@ reth-downloaders.workspace = true reth-eth-wire-types.workspace = true reth-network-p2p.workspace = true reth-network-peers.workspace = true -reth-primitives-traits.workspace = true +reth-primitives-traits = { workspace = true, features = ["eip7928"] } reth-storage-api = { workspace = true, features = ["db-api"] } reth-storage-errors.workspace = true reth-tasks.workspace = true @@ -42,5 +42,9 @@ tokio-util.workspace = true alloy-consensus.workspace = true alloy-rlp.workspace = true futures.workspace = true +reth-chainspec.workspace = true +reth-ethereum-primitives.workspace = true +reth-evm.workspace = true +reth-evm-ethereum.workspace = true reth-provider = { workspace = true, features = ["test-utils"] } tokio = { workspace = true, features = ["macros", "rt-multi-thread"] } diff --git a/crates/snap-sync/src/bal.rs b/crates/snap-sync/src/bal.rs new file mode 100644 index 00000000000..a0063b8d83c --- /dev/null +++ b/crates/snap-sync/src/bal.rs @@ -0,0 +1,447 @@ +//! Carries downloaded state forward through verified block access lists. +//! +//! [EIP-8189](https://eips.ethereum.org/EIPS/eip-8189#synchronization-algorithm) advances the +//! pivot by applying later blocks' lists to the state downloaded so far. Each list records the +//! final value of every field it changes, so untouched fields come from the downloaded account. + +use crate::SnapSyncError; +use alloy_eip7928::AccountChanges; +use alloy_primitives::{keccak256, map::B256Map, Bytes, B256, KECCAK256_EMPTY}; +use reth_primitives_traits::Account; +use reth_trie_common::{HashedPostState, HashedStorage}; + +/// Changes one block access list makes to the downloaded state. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct BalStateUpdate { + // Post-state of every entry whose account is downloaded. + state: HashedPostState, + // Final code of accounts whose code changed, keyed by code hash. + bytecodes: B256Map, + // Entries left out because their account is not downloaded yet. + unresolved: Vec, +} + +impl BalStateUpdate { + /// Applies `bal` on top of the downloaded state, resolved by hashed address through `base`. + /// + /// The list must already be verified against its header. + pub fn from_block_access_list( + bal: &[AccountChanges], + mut base: impl FnMut(B256) -> Result, + ) -> Result { + let mut update = Self::default(); + for account_changes in bal { + let account_info = account_changes.account_info(); + // Read-only entries record accesses, not changes. + if !account_info.changes_state_root(account_changes) { + continue + } + + let hashed_address = keccak256(account_changes.address()); + let mut account = match base(hashed_address)? { + // Its range is downloaded against a later root, which includes this change. + DownloadedAccount::Unknown => { + update.unresolved.push(hashed_address); + continue + } + DownloadedAccount::Absent => Account::default(), + DownloadedAccount::Present(account) => account, + }; + + account.apply_bal_info(account_info); + // Stored accounts represent empty code with no code hash. + account.bytecode_hash = account.bytecode_hash.filter(|hash| *hash != KECCAK256_EMPTY); + // Execution removes accounts a block leaves empty, see EIP-161. + update.state.accounts.insert(hashed_address, (!account.is_empty()).then_some(account)); + if account_changes.has_storage_changes() { + update.state.storages.insert( + hashed_address, + HashedStorage::from_iter( + account_changes + .storage_post_states() + .map(|(slot, value)| (keccak256(B256::from(slot)), value)), + ), + ); + } + if let Some((code_hash, code)) = account_info + .code_hash + .zip(account_changes.code_post_state().filter(|code| !code.is_empty())) + { + update.bytecodes.insert(code_hash, code.clone()); + } + } + Ok(update) + } + + /// Post-state of every entry whose account is downloaded, keyed by hashed address. + pub const fn state(&self) -> &HashedPostState { + &self.state + } + + /// Final code of accounts whose code changed, keyed by code hash. + pub const fn bytecodes(&self) -> &B256Map { + &self.bytecodes + } + + /// Hashed addresses of entries left out because their account is not downloaded yet. + pub fn unresolved(&self) -> &[B256] { + &self.unresolved + } +} + +/// What the downloaded state holds for an account a list changes. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum DownloadedAccount { + /// The account's range is not downloaded yet. + Unknown, + /// The account's range is downloaded and does not hold it. + Absent, + /// The downloaded account. + Present(Account), +} + +#[cfg(test)] +mod tests { + use super::*; + use alloy_consensus::{Header, TxLegacy}; + use alloy_eip7928::{ + BalanceChange, BlockAccessIndex, CodeChange, NonceChange, SlotChanges, StorageChange, + }; + use alloy_eips::{ + eip2935::{HISTORY_STORAGE_ADDRESS, HISTORY_STORAGE_CODE}, + eip4788::{BEACON_ROOTS_ADDRESS, BEACON_ROOTS_CODE}, + eip4895::Withdrawal, + eip7002::{WITHDRAWAL_REQUEST_PREDEPLOY_ADDRESS, WITHDRAWAL_REQUEST_PREDEPLOY_CODE}, + }; + use alloy_primitives::{bytes, Address, Signature, TxKind, U256}; + use reth_chainspec::ChainSpecBuilder; + use reth_ethereum_primitives::{Block, BlockBody, Transaction, TransactionSigned}; + use reth_evm::{execute::BlockExecutor, ConfigureEvm, Evm}; + use reth_evm_ethereum::EthEvmConfig; + use reth_primitives_traits::{Block as _, Recovered}; + use reth_trie_common::KeccakKeyHasher; + use revm::{ + database::{states::bundle_state::BundleRetention, CacheDB, EmptyDB, State}, + state::{AccountInfo, Bytecode}, + }; + use std::{collections::BTreeMap, sync::Arc}; + + const ACCOUNT: Address = Address::repeat_byte(0xaa); + const SENDER: Address = Address::repeat_byte(0x11); + + fn index(value: u64) -> BlockAccessIndex { + BlockAccessIndex::new(value) + } + + fn apply(changes: &AccountChanges, base: DownloadedAccount) -> BalStateUpdate { + BalStateUpdate::from_block_access_list(std::slice::from_ref(changes), |_| Ok(base)).unwrap() + } + + #[test] + fn read_only_entries_write_nothing() { + let changes = AccountChanges::new(ACCOUNT).with_storage_read(U256::from(1)); + + let update = BalStateUpdate::from_block_access_list(&[changes], |_| { + panic!("read-only entries need no downloaded account") + }) + .unwrap(); + + assert_eq!(update, BalStateUpdate::default()); + } + + #[test] + fn empty_slot_entries_write_nothing() { + let changes = AccountChanges::new(ACCOUNT) + .with_storage_change(SlotChanges::new(U256::from(1), vec![])); + + let update = BalStateUpdate::from_block_access_list(&[changes], |_| { + panic!("empty slot entries need no downloaded account") + }) + .unwrap(); + + assert_eq!(update, BalStateUpdate::default()); + } + + #[test] + fn account_changes_with_empty_slots_write_no_storage() { + let changes = AccountChanges::new(ACCOUNT) + .with_balance_change(BalanceChange::new(index(1), U256::from(10))) + .with_storage_change(SlotChanges::new(U256::from(1), vec![])); + + let update = apply(&changes, DownloadedAccount::Absent); + + assert_eq!(update.state.accounts[&keccak256(ACCOUNT)].unwrap().balance, U256::from(10)); + assert!(update.state.storages.is_empty()); + } + + #[test] + fn accounts_not_downloaded_yet_are_left_unresolved() { + // Even a fully determined account is left to the download against the later root. + let changes = AccountChanges::new(ACCOUNT) + .with_balance_change(BalanceChange::new(index(1), U256::from(10))) + .with_nonce_change(NonceChange::new(index(1), 1)) + .with_code_change(CodeChange::new(index(1), bytes!("6001"))); + + let update = apply(&changes, DownloadedAccount::Unknown); + + assert_eq!(update.unresolved, vec![keccak256(ACCOUNT)]); + assert!(update.state.is_empty()); + assert!(update.bytecodes.is_empty()); + } + + #[test] + fn untouched_fields_keep_their_downloaded_values() { + let existing = + Account { balance: U256::from(9), nonce: 4, bytecode_hash: Some(B256::repeat_byte(1)) }; + let changes = AccountChanges::new(ACCOUNT) + .with_balance_change(BalanceChange::new(index(1), U256::from(10))) + .with_balance_change(BalanceChange::new(index(2), U256::from(20))); + + let update = apply(&changes, DownloadedAccount::Present(existing)); + assert_eq!( + update.state.accounts[&keccak256(ACCOUNT)], + Some(Account { balance: U256::from(20), ..existing }) + ); + + // An absent account starts from the empty one. + let update = apply(&changes, DownloadedAccount::Absent); + assert_eq!( + update.state.accounts[&keccak256(ACCOUNT)], + Some(Account { balance: U256::from(20), nonce: 0, bytecode_hash: None }) + ); + } + + #[test] + fn zeroed_slots_and_cleared_code_are_written() { + let existing = + Account { balance: U256::from(1), nonce: 1, bytecode_hash: Some(B256::repeat_byte(1)) }; + let changes = AccountChanges::new(ACCOUNT) + .with_code_change(CodeChange::new(index(1), bytes!("6001"))) + .with_code_change(CodeChange::new(index(2), Bytes::new())) + .with_storage_change(SlotChanges::new( + U256::from(1), + vec![ + StorageChange::new(index(1), U256::from(5)), + StorageChange::new(index(2), U256::ZERO), + ], + )); + + let update = apply(&changes, DownloadedAccount::Present(existing)); + + let hashed_address = keccak256(ACCOUNT); + assert_eq!(update.state.accounts[&hashed_address].unwrap().bytecode_hash, None); + assert_eq!( + update.state.storages[&hashed_address], + HashedStorage::from_iter([(keccak256(B256::from(U256::from(1))), U256::ZERO)]) + ); + assert!(update.bytecodes.is_empty()); + } + + #[test] + fn an_account_the_block_empties_is_removed() { + let changes = AccountChanges::new(ACCOUNT) + .with_balance_change(BalanceChange::new(index(1), U256::from(100))) + .with_balance_change(BalanceChange::new(index(2), U256::ZERO)); + + let update = apply(&changes, DownloadedAccount::Absent); + + assert_eq!(update.state.accounts[&keccak256(ACCOUNT)], None); + } + + // Final state as a flat map: accounts, and non-zero slots by hashed address and slot. + type FlatState = (BTreeMap, BTreeMap<(B256, B256), U256>); + + fn flatten(db: &CacheDB) -> FlatState { + let mut state = FlatState::default(); + for (address, account) in &db.cache.accounts { + let hashed_address = keccak256(address); + state.0.insert(hashed_address, Account::from(&account.info)); + for (slot, value) in &account.storage { + if !value.is_zero() { + state.1.insert((hashed_address, keccak256(B256::from(*slot))), *value); + } + } + } + state + } + + fn fold(mut state: FlatState, update: &HashedPostState) -> FlatState { + for (hashed_address, storage) in &update.storages { + for (slot, value) in &storage.storage { + if value.is_zero() { + state.1.remove(&(*hashed_address, *slot)); + } else { + state.1.insert((*hashed_address, *slot), *value); + } + } + } + for (hashed_address, account) in &update.accounts { + match account { + Some(account) => { + state.0.insert(*hashed_address, *account); + } + None => { + state.0.remove(hashed_address); + state.1.retain(|(address, _), _| address != hashed_address); + } + } + } + state + } + + #[test] + fn flat_state_excludes_zero_slots_and_deleted_account_storage() { + let mut db = CacheDB::::default(); + for address in [ACCOUNT, SENDER] { + db.insert_account_info(address, AccountInfo::from_balance(U256::from(1))); + db.insert_account_storage(address, U256::from(1), U256::from(5)).unwrap(); + db.insert_account_storage(address, U256::from(2), U256::ZERO).unwrap(); + } + let pre = flatten(&db); + assert_eq!(pre.1.len(), 2); + + let mut update = HashedPostState::default(); + update.accounts.insert(keccak256(ACCOUNT), None); + let post = fold(pre, &update); + + assert_eq!( + post.0, + BTreeMap::from([( + keccak256(SENDER), + Account { balance: U256::from(1), ..Default::default() }, + )]) + ); + assert_eq!( + post.1, + BTreeMap::from([( + (keccak256(SENDER), keccak256(B256::from(U256::from(1)))), + U256::from(5) + )]) + ); + } + + fn insert(db: &mut CacheDB, address: Address, nonce: u64, code: Bytes) { + let code = Bytecode::new_raw(code); + let info = AccountInfo { + nonce, + code_hash: code.hash_slow(), + code: Some(code), + ..Default::default() + }; + db.insert_account_info(address, info); + } + + fn tx(nonce: u64, to: TxKind, value: u64, input: Bytes) -> Recovered { + let tx = Transaction::Legacy(TxLegacy { + nonce, + gas_price: 1, + gas_limit: 1_000_000, + to, + value: U256::from(value), + input, + ..Default::default() + }); + Recovered::new_unchecked( + TransactionSigned::new_unhashed(tx, Signature::test_signature()), + SENDER, + ) + } + + #[test] + fn applying_the_list_matches_execution() { + let contract = Address::repeat_byte(0xc0); + let beneficiary = Address::repeat_byte(0xbe); + + let mut db = CacheDB::::new(Default::default()); + insert(&mut db, BEACON_ROOTS_ADDRESS, 1, BEACON_ROOTS_CODE.clone()); + insert(&mut db, HISTORY_STORAGE_ADDRESS, 1, HISTORY_STORAGE_CODE.clone()); + insert( + &mut db, + WITHDRAWAL_REQUEST_PREDEPLOY_ADDRESS, + 1, + WITHDRAWAL_REQUEST_PREDEPLOY_CODE.clone(), + ); + db.insert_account_info(SENDER, AccountInfo::from_balance(U256::from(u64::MAX))); + // Zeroes slot 1, reads slot 3, stores the block number in slot 2 and the call value in 4. + insert(&mut db, contract, 1, bytes!("6000600155600354504360025534600455")); + db.insert_account_storage(contract, U256::from(1), U256::from(5)).unwrap(); + db.insert_account_storage(contract, U256::from(3), U256::from(7)).unwrap(); + let pre = flatten(&db); + + let txs = [ + // Repeated writes, the second of slots 1 and 2 being no-ops. + tx(0, TxKind::Call(contract), 1, Bytes::new()), + tx(1, TxKind::Call(contract), 2, Bytes::new()), + // Deploys `INVALID` as runtime code. + tx(2, TxKind::Create, 0, bytes!("60fe60005360016000f3")), + // Funds an address, then creates and destroys a contract there in one transaction. + tx(3, TxKind::Call(SENDER.create(4)), 100, Bytes::new()), + tx( + 4, + TxKind::Create, + 0, + [&[0x73][..], beneficiary.as_slice(), &[0xff]].concat().into(), + ), + ]; + let header = Header { + number: 1, + timestamp: 1, + gas_limit: 30_000_000, + parent_beacon_block_root: Some(B256::repeat_byte(1)), + excess_blob_gas: Some(0), + blob_gas_used: Some(0), + block_access_list_hash: Some(B256::ZERO), + ..Default::default() + }; + let withdrawal = Withdrawal { address: beneficiary, amount: 1, ..Default::default() }; + let block = Block { + header, + body: BlockBody { withdrawals: Some(vec![withdrawal].into()), ..Default::default() }, + } + .seal_slow(); + + let evm_config = + EthEvmConfig::new(Arc::new(ChainSpecBuilder::mainnet().amsterdam_activated().build())); + let mut state = + State::builder().with_database(&mut db).with_bundle_update().with_bal_builder().build(); + { + let mut executor = evm_config.executor_for_block(&mut state, &block).unwrap(); + executor.apply_pre_execution_changes().unwrap(); + for tx in txs { + executor.evm_mut().db_mut().bump_bal_index(); + executor.execute_transaction(tx).unwrap(); + } + executor.evm_mut().db_mut().bump_bal_index(); + executor.apply_post_execution_changes().unwrap(); + } + let bal = state.take_built_alloy_bal().unwrap(); + state.merge_transitions(BundleRetention::PlainState); + let bundle = state.take_bundle(); + + let update = BalStateUpdate::from_block_access_list(&bal, |hashed_address| { + Ok(pre + .0 + .get(&hashed_address) + .map_or(DownloadedAccount::Absent, |account| DownloadedAccount::Present(*account))) + }) + .unwrap(); + let executed = HashedPostState::from_bundle_state::(bundle.state()); + + let post = fold(pre.clone(), &update.state); + assert!(update.unresolved.is_empty()); + assert_eq!(post, fold(pre, &executed)); + + // The block did what the cases need. + let slot = |address: Address, slot: u64| { + post.1.get(&(keccak256(address), keccak256(B256::from(U256::from(slot))))) + }; + assert_eq!((slot(contract, 1), slot(contract, 4)), (None, Some(&U256::from(2)))); + assert!(!post.0.contains_key(&keccak256(SENDER.create(4)))); + assert_eq!(post.0[&keccak256(beneficiary)].balance, U256::from(100 + 1_000_000_000u64)); + assert!(post.1.keys().any(|(address, _)| *address == keccak256(BEACON_ROOTS_ADDRESS))); + assert_eq!( + update.bytecodes, + B256Map::from_iter([(keccak256([0xfe]), Bytes::from_static(&[0xfe]))]) + ); + } +} diff --git a/crates/snap-sync/src/lib.rs b/crates/snap-sync/src/lib.rs index 8f0cc6654d2..95ce4bcae03 100644 --- a/crates/snap-sync/src/lib.rs +++ b/crates/snap-sync/src/lib.rs @@ -37,6 +37,7 @@ mod account; mod attempt; +mod bal; mod error; mod generation; mod pivot; @@ -50,6 +51,7 @@ pub use account::{ DEFAULT_RESPONSE_BYTES, MAX_HASH, }; pub use attempt::{SnapAttemptStore, SnapWrite}; +pub use bal::{BalStateUpdate, DownloadedAccount}; pub use error::SnapSyncError; pub use generation::{SnapGeneration, SnapPhase}; pub use pivot::SnapPivotPolicy; From ef7014f6b46500c9f99cc4d71d242a0144248c47 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Mon, 14 Sep 2026 14:02:14 -0400 Subject: [PATCH 255/294] test: wait for mined block finality in dev node test (#27227) --- crates/ethereum/node/tests/e2e/dev.rs | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/crates/ethereum/node/tests/e2e/dev.rs b/crates/ethereum/node/tests/e2e/dev.rs index 03a0629787d..ba413cae5bd 100644 --- a/crates/ethereum/node/tests/e2e/dev.rs +++ b/crates/ethereum/node/tests/e2e/dev.rs @@ -12,7 +12,7 @@ use reth_provider::{ }; use reth_rpc_eth_api::{helpers::EthTransactions, EthApiServer}; use reth_tasks::Runtime; -use std::{num::NonZeroUsize, sync::Arc}; +use std::{num::NonZeroUsize, sync::Arc, time::Duration}; #[tokio::test] async fn can_run_dev_node() -> eyre::Result<()> { @@ -38,12 +38,22 @@ async fn can_run_dev_node() -> eyre::Result<()> { assert_chain_advances(&node).await; - let (safe_changed, finalized_changed) = - tokio::join!(safe_block.changed(), finalized_block.changed()); - safe_changed?; - finalized_changed?; - let chain_info = node.provider.chain_info()?; + // Startup can leave an unread genesis notification, and the canonical head notification + // precedes the safe/finalized updates. Wait for the mined block itself on both channels. + tokio::time::timeout(Duration::from_secs(10), async { + tokio::try_join!( + safe_block.wait_for(|header| { + header.as_ref().is_some_and(|header| header.num_hash() == chain_info.into()) + }), + finalized_block.wait_for(|header| { + header.as_ref().is_some_and(|header| header.num_hash() == chain_info.into()) + }), + ) + .map(|_| ()) + }) + .await??; + assert_eq!(node.provider.safe_block_num_hash()?, Some(chain_info.into())); assert_eq!(node.provider.finalized_block_num_hash()?, Some(chain_info.into())); From f9efb88530c54f1191f9a4d34b143bb7de97aaf6 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Mon, 14 Sep 2026 14:15:50 -0400 Subject: [PATCH 256/294] refactor(net): simplify transaction announcement handling (#27021) --- crates/net/eth-wire-types/src/broadcast.rs | 430 +----------------- crates/net/network/benches/tx_fetcher.rs | 23 +- .../network/src/test_utils/transactions.rs | 12 +- .../network/src/transactions/announcement.rs | 244 ++++++++++ .../net/network/src/transactions/fetcher.rs | 24 +- crates/net/network/src/transactions/mod.rs | 185 +++++--- 6 files changed, 419 insertions(+), 499 deletions(-) create mode 100644 crates/net/network/src/transactions/announcement.rs diff --git a/crates/net/eth-wire-types/src/broadcast.rs b/crates/net/eth-wire-types/src/broadcast.rs index d6f9d46bf8c..7e2e9e30874 100644 --- a/crates/net/eth-wire-types/src/broadcast.rs +++ b/crates/net/eth-wire-types/src/broadcast.rs @@ -4,17 +4,13 @@ use crate::{EthMessage, EthVersion, NetworkPrimitives}; use alloc::{sync::Arc, vec::Vec}; use alloy_consensus::transaction::TxHashRef; use alloy_eips::eip2718::Typed2718; -use alloy_primitives::{ - bytes::BufMut, - map::{B256Map, B256Set}, - Bytes, TxHash, B128, B256, U128, -}; +use alloy_primitives::{bytes::BufMut, Bytes, TxHash, B128, B256, U128}; use alloy_rlp::{ decode_append, Decodable, Encodable, Header, RlpDecodable, RlpDecodableWrapper, RlpEncodable, RlpEncodableWrapper, }; use core::{fmt::Debug, mem}; -use derive_more::{Constructor, Deref, DerefMut, From, IntoIterator}; +use derive_more::{Deref, DerefMut, IntoIterator}; use reth_codecs_derive::{add_arbitrary_tests, generate_tests}; use reth_ethereum_primitives::TransactionSigned; use reth_primitives_traits::{sync::OnceLock, Block, InMemorySize, SignedTransaction}; @@ -657,6 +653,16 @@ impl proptest::prelude::Arbitrary for NewPooledTransactionHashes68 { } impl NewPooledTransactionHashes68 { + /// Returns the number of announced hashes. + pub const fn len(&self) -> usize { + self.hashes.len() + } + + /// Returns whether there are no announced hashes. + pub const fn is_empty(&self) -> bool { + self.hashes.is_empty() + } + /// Returns a new instance with capacity for `capacity` entries. pub fn with_capacity(capacity: usize) -> Self { Self { @@ -839,6 +845,16 @@ impl proptest::prelude::Arbitrary for NewPooledTransactionHashes72 { } impl NewPooledTransactionHashes72 { + /// Returns the number of announced hashes. + pub const fn len(&self) -> usize { + self.hashes.len() + } + + /// Returns whether there are no announced hashes. + pub const fn is_empty(&self) -> bool { + self.hashes.is_empty() + } + /// Cell mask advertising availability of every cell. /// /// Used when announcing blob transactions whose full sidecar is available locally, since @@ -1001,122 +1017,6 @@ const fn ensure_pooled_transaction_hashes_lengths( Ok(()) } -/// Validation pass that checks for unique transaction hashes. -pub trait DedupPayload { - /// Value type in [`PartiallyValidData`] map. - type Value; - - /// The payload contains no entries. - fn is_empty(&self) -> bool; - - /// Returns the number of entries. - fn len(&self) -> usize; - - /// Consumes self, returning an iterator over hashes in payload. - fn dedup(self) -> PartiallyValidData; -} - -/// Value in [`PartiallyValidData`] map obtained from an announcement. -pub type Eth68TxMetadata = Option<(u8, usize)>; - -impl DedupPayload for NewPooledTransactionHashes { - type Value = Eth68TxMetadata; - - fn is_empty(&self) -> bool { - self.is_empty() - } - - fn len(&self) -> usize { - self.len() - } - - fn dedup(self) -> PartiallyValidData { - match self { - Self::Eth66(msg) => msg.dedup(), - Self::Eth68(msg) => msg.dedup(), - Self::Eth72(msg) => msg.dedup(), - } - } -} - -impl DedupPayload for NewPooledTransactionHashes72 { - type Value = Eth68TxMetadata; - - fn is_empty(&self) -> bool { - self.hashes.is_empty() - } - - fn len(&self) -> usize { - self.hashes.len() - } - - fn dedup(self) -> PartiallyValidData { - let Self { hashes, mut sizes, mut types, cell_mask } = self; - - let mut deduped_data = B256Map::with_capacity_and_hasher(hashes.len(), Default::default()); - - for hash in hashes.into_iter().rev() { - if let (Some(ty), Some(size)) = (types.pop(), sizes.pop()) { - deduped_data.insert(hash, Some((ty, size))); - } - } - - PartiallyValidData::from_raw_data_eth72_with_cell_mask(deduped_data, cell_mask) - } -} - -impl DedupPayload for NewPooledTransactionHashes68 { - type Value = Eth68TxMetadata; - - fn is_empty(&self) -> bool { - self.hashes.is_empty() - } - - fn len(&self) -> usize { - self.hashes.len() - } - - fn dedup(self) -> PartiallyValidData { - let Self { hashes, mut sizes, mut types } = self; - - let mut deduped_data = B256Map::with_capacity_and_hasher(hashes.len(), Default::default()); - - for hash in hashes.into_iter().rev() { - if let (Some(ty), Some(size)) = (types.pop(), sizes.pop()) { - deduped_data.insert(hash, Some((ty, size))); - } - } - - PartiallyValidData::from_raw_data_eth68(deduped_data) - } -} - -impl DedupPayload for NewPooledTransactionHashes66 { - type Value = Eth68TxMetadata; - - fn is_empty(&self) -> bool { - self.0.is_empty() - } - - fn len(&self) -> usize { - self.0.len() - } - - fn dedup(self) -> PartiallyValidData { - let Self(hashes) = self; - - let mut deduped_data = B256Map::with_capacity_and_hasher(hashes.len(), Default::default()); - - let noop_value: Eth68TxMetadata = None; - - for hash in hashes.into_iter().rev() { - deduped_data.insert(hash, noop_value); - } - - PartiallyValidData::from_raw_data_eth66(deduped_data) - } -} - /// Interface for handling mempool message data. Used in various filters in pipelines in /// `TransactionsManager` and in queries to `TransactionPool`. pub trait HandleMempoolData { @@ -1130,13 +1030,6 @@ pub trait HandleMempoolData { fn retain_by_hash(&mut self, f: impl FnMut(&TxHash) -> bool); } -/// Extension of [`HandleMempoolData`] interface, for mempool messages that are versioned. -pub trait HandleVersionedMempoolData { - /// Returns the announcement version, either [`Eth66`](EthVersion::Eth66) or - /// [`Eth68`](EthVersion::Eth68). - fn msg_version(&self) -> EthVersion; -} - impl HandleMempoolData for Vec { fn is_empty(&self) -> bool { self.is_empty() @@ -1151,207 +1044,6 @@ impl HandleMempoolData for Vec { } } -macro_rules! handle_mempool_data_map_impl { - ($data_ty:ty, $(<$generic:ident>)?) => { - impl$(<$generic>)? HandleMempoolData for $data_ty { - fn is_empty(&self) -> bool { - self.data.is_empty() - } - - fn len(&self) -> usize { - self.data.len() - } - - fn retain_by_hash(&mut self, mut f: impl FnMut(&TxHash) -> bool) { - self.data.retain(|hash, _| f(hash)); - } - } - }; -} - -/// Data that has passed an initial validation pass that is not specific to any mempool message -/// type. -#[derive(Debug, Deref, DerefMut, IntoIterator)] -pub struct PartiallyValidData { - #[deref] - #[deref_mut] - #[into_iterator] - data: B256Map, - version: Option, - /// The eth/72 message-level cell mask, if present. - cell_mask: Option, -} - -handle_mempool_data_map_impl!(PartiallyValidData, ); - -impl PartiallyValidData { - /// Wraps raw data. - pub const fn from_raw_data(data: B256Map, version: Option) -> Self { - Self { data, version, cell_mask: None } - } - - /// Wraps raw data with version [`EthVersion::Eth72`]. - pub const fn from_raw_data_eth72(data: B256Map) -> Self { - Self::from_raw_data(data, Some(EthVersion::Eth72)) - } - - /// Wraps raw data with an eth/72 message-level cell mask. - pub const fn from_raw_data_eth72_with_cell_mask( - data: B256Map, - cell_mask: Option, - ) -> Self { - Self { data, version: Some(EthVersion::Eth72), cell_mask } - } - - /// Wraps raw data with version [`EthVersion::Eth68`]. - pub const fn from_raw_data_eth68(data: B256Map) -> Self { - Self::from_raw_data(data, Some(EthVersion::Eth68)) - } - - /// Wraps raw data with version [`EthVersion::Eth66`]. - pub const fn from_raw_data_eth66(data: B256Map) -> Self { - Self::from_raw_data(data, Some(EthVersion::Eth66)) - } - - /// Returns a new [`PartiallyValidData`] with empty data from an [`Eth72`](EthVersion::Eth72) - /// announcement. - pub fn empty_eth72() -> Self { - Self::from_raw_data_eth72(B256Map::default()) - } - - /// Returns a new [`PartiallyValidData`] with empty data from an [`Eth68`](EthVersion::Eth68) - /// announcement. - pub fn empty_eth68() -> Self { - Self::from_raw_data_eth68(B256Map::default()) - } - - /// Returns a new [`PartiallyValidData`] with empty data from an [`Eth66`](EthVersion::Eth66) - /// announcement. - pub fn empty_eth66() -> Self { - Self::from_raw_data_eth66(B256Map::default()) - } - - /// Returns the version of the message this data was received in if different versions of the - /// message exist. - pub const fn msg_version(&self) -> Option { - self.version - } - - /// Returns the eth/72 message-level cell mask, if present. - pub const fn eth72_cell_mask(&self) -> Option { - self.cell_mask - } - - /// Destructs returning the validated data. - pub fn into_data(self) -> B256Map { - self.data - } -} - -/// Partially validated data from an announcement or a -/// [`PooledTransactions`](crate::PooledTransactions) response. -#[derive(Debug, Deref, DerefMut, IntoIterator, From)] -pub struct ValidAnnouncementData { - #[deref] - #[deref_mut] - #[into_iterator] - data: B256Map, - version: EthVersion, - /// The eth/72 message-level cell mask, if present. - cell_mask: Option, -} - -handle_mempool_data_map_impl!(ValidAnnouncementData,); - -impl ValidAnnouncementData { - /// Destructs returning only the valid hashes and the announcement message version. Caution! If - /// this is [`Eth68`](EthVersion::Eth68) announcement data, this drops the metadata. - pub fn into_request_hashes(self) -> (RequestTxHashes, EthVersion) { - let hashes = self.data.into_keys().collect::(); - - (RequestTxHashes::new(hashes), self.version) - } - - /// Conversion from [`PartiallyValidData`] from an announcement. Note! [`PartiallyValidData`] - /// from an announcement, should have some [`EthVersion`]. Panics if [`PartiallyValidData`] has - /// version set to `None`. - pub fn from_partially_valid_data(data: PartiallyValidData) -> Self { - let PartiallyValidData { data, version, cell_mask } = data; - - let version = version.expect("should have eth version for conversion"); - - Self { data, version, cell_mask } - } - - /// Returns the eth/72 message-level cell mask, if present. - pub const fn eth72_cell_mask(&self) -> Option { - self.cell_mask - } - - /// Destructs returning the validated data. - pub fn into_data(self) -> B256Map { - self.data - } -} - -impl HandleVersionedMempoolData for ValidAnnouncementData { - fn msg_version(&self) -> EthVersion { - self.version - } -} - -/// Hashes to request from a peer. -#[derive(Debug, Default, Deref, DerefMut, IntoIterator, Constructor)] -pub struct RequestTxHashes { - #[deref] - #[deref_mut] - #[into_iterator(owned, ref)] - hashes: B256Set, -} - -impl RequestTxHashes { - /// Returns a new [`RequestTxHashes`] with given capacity for hashes. Caution! Make sure to - /// call `shrink_to_fit` on [`RequestTxHashes`] when full, especially where it will - /// be stored in its entirety like in the future waiting for a - /// [`GetPooledTransactions`](crate::GetPooledTransactions) request to resolve. - pub fn with_capacity(capacity: usize) -> Self { - Self::new(B256Set::with_capacity_and_hasher(capacity, Default::default())) - } - - /// Returns a new empty instance. - fn empty() -> Self { - Self::new(B256Set::default()) - } - - /// Retains the given number of elements, returning an iterator over the rest. - pub fn retain_count(&mut self, count: usize) -> Self { - let rest_capacity = self.hashes.len().saturating_sub(count); - if rest_capacity == 0 { - return Self::empty() - } - let mut rest = Self::with_capacity(rest_capacity); - - let mut i = 0; - self.hashes.retain(|hash| { - if i >= count { - rest.insert(*hash); - return false - } - i += 1; - - true - }); - - rest - } -} - -impl FromIterator<(TxHash, Eth68TxMetadata)> for RequestTxHashes { - fn from_iter>(iter: I) -> Self { - Self::new(iter.into_iter().map(|(hash, _)| hash).collect()) - } -} - /// The earliest block, the latest block and hash of the latest block which can be provided. /// See [BlockRangeUpdate](https://github.com/ethereum/devp2p/blob/master/caps/eth.md#blockrangeupdate-0x11). #[derive(Clone, Debug, PartialEq, Eq, Default, RlpEncodable, RlpDecodable)] @@ -1390,7 +1082,7 @@ mod tests { use super::*; use alloy_consensus::{transaction::TxHashRef, Typed2718}; use alloy_eips::eip2718::Encodable2718; - use alloy_primitives::{b256, hex, Bytes, Signature, U256}; + use alloy_primitives::{hex, Bytes, Signature, U256}; use alloy_rlp::{RlpDecodable, RlpEncodable}; use proptest::prelude::*; use reth_ethereum_primitives::{Transaction, TransactionSigned}; @@ -1742,82 +1434,6 @@ mod tests { assert!(matches!(result, Err(alloy_rlp::Error::InputTooShort))); } - #[test] - fn eth_72_dedup_preserves_message_cell_mask() { - let cell_mask = Some(B128::repeat_byte(0x11)); - let announcement = NewPooledTransactionHashes72 { - types: vec![3], - sizes: vec![128], - hashes: vec![B256::from([1u8; 32])], - cell_mask, - }; - - let partially_valid = announcement.dedup(); - assert_eq!(partially_valid.eth72_cell_mask(), cell_mask); - - let valid = ValidAnnouncementData::from_partially_valid_data(partially_valid); - assert_eq!(valid.eth72_cell_mask(), cell_mask); - } - - #[test] - fn request_hashes_retain_count_keep_subset() { - let mut hashes = RequestTxHashes::new( - [ - b256!("0x0000000000000000000000000000000000000000000000000000000000000001"), - b256!("0x0000000000000000000000000000000000000000000000000000000000000002"), - b256!("0x0000000000000000000000000000000000000000000000000000000000000003"), - b256!("0x0000000000000000000000000000000000000000000000000000000000000004"), - b256!("0x0000000000000000000000000000000000000000000000000000000000000005"), - ] - .into_iter() - .collect::(), - ); - - let rest = hashes.retain_count(3); - - assert_eq!(3, hashes.len()); - assert_eq!(2, rest.len()); - } - - #[test] - fn request_hashes_retain_count_keep_all() { - let mut hashes = RequestTxHashes::new( - [ - b256!("0x0000000000000000000000000000000000000000000000000000000000000001"), - b256!("0x0000000000000000000000000000000000000000000000000000000000000002"), - b256!("0x0000000000000000000000000000000000000000000000000000000000000003"), - b256!("0x0000000000000000000000000000000000000000000000000000000000000004"), - b256!("0x0000000000000000000000000000000000000000000000000000000000000005"), - ] - .into_iter() - .collect::(), - ); - - let _ = hashes.retain_count(6); - - assert_eq!(5, hashes.len()); - } - - #[test] - fn split_request_hashes_keep_none() { - let mut hashes = RequestTxHashes::new( - [ - b256!("0x0000000000000000000000000000000000000000000000000000000000000001"), - b256!("0x0000000000000000000000000000000000000000000000000000000000000002"), - b256!("0x0000000000000000000000000000000000000000000000000000000000000003"), - b256!("0x0000000000000000000000000000000000000000000000000000000000000004"), - b256!("0x0000000000000000000000000000000000000000000000000000000000000005"), - ] - .into_iter() - .collect::(), - ); - - let rest = hashes.retain_count(0); - - assert_eq!(0, hashes.len()); - assert_eq!(5, rest.len()); - } - fn signed_transaction() -> impl SignedTransaction { TransactionSigned::new_unhashed( Transaction::Legacy(Default::default()), diff --git a/crates/net/network/benches/tx_fetcher.rs b/crates/net/network/benches/tx_fetcher.rs index b97e59dfec0..1976d22dd28 100644 --- a/crates/net/network/benches/tx_fetcher.rs +++ b/crates/net/network/benches/tx_fetcher.rs @@ -10,13 +10,15 @@ use alloy_primitives::{ }; use criterion::{criterion_group, criterion_main, BatchSize, Criterion, Throughput}; use futures::{task::noop_waker_ref, StreamExt}; -use reth_eth_wire::{Eth68TxMetadata, EthVersion, PooledTransactions}; +use reth_eth_wire::{EthVersion, PooledTransactions}; use reth_ethereum_primitives::{PooledTransactionVariant, TransactionSigned}; use reth_network::{ test_utils::transactions::new_mock_session_with_capacity, transactions::{ + announcement::{AnnouncedTransaction, TransactionMetadata}, constants::tx_fetcher::MAX_COUNT_EAGER_CANDIDATE_PEERS_PER_HASH, - fetcher::TransactionFetcher, PeerMetadata, + fetcher::TransactionFetcher, + PeerMetadata, }, }; use reth_network_api::PeerRequest; @@ -55,14 +57,14 @@ impl Rig { } /// Every peer announces the same hashes, like a gossiped batch of transactions. - fn announce_gossip(&mut self, announcement: &[(TxHash, Eth68TxMetadata)]) { + fn announce_gossip(&mut self, announcement: &[AnnouncedTransaction]) { for (peer_id, _) in &self.sessions { self.fetcher.on_announcement(*peer_id, announcement.iter().copied()); } } /// Every peer announces its own slice of the hashes. - fn announce_disjoint(&mut self, announcement: &[(TxHash, Eth68TxMetadata)]) { + fn announce_disjoint(&mut self, announcement: &[AnnouncedTransaction]) { let per_peer = announcement.len() / self.sessions.len(); for (i, (peer_id, _)) in self.sessions.iter().enumerate() { let slice = &announcement[i * per_peer..(i + 1) * per_peer]; @@ -125,13 +127,18 @@ fn pooled_txs(count: usize) -> Vec { .collect() } -fn announcement_of(txs: &[PooledTransactionVariant]) -> Vec<(TxHash, Eth68TxMetadata)> { - txs.iter().map(|tx| (*tx.tx_hash(), Some((TX_TYPE, TX_SIZE)))).collect() +fn announcement_of(txs: &[PooledTransactionVariant]) -> Vec { + txs.iter() + .map(|tx| AnnouncedTransaction { + hash: *tx.tx_hash(), + metadata: Some(TransactionMetadata { tx_type: TX_TYPE, size: TX_SIZE }), + }) + .collect() } /// Shared fixtures. struct Fixtures { - announcement: Vec<(TxHash, Eth68TxMetadata)>, + announcement: Vec, by_hash: B256Map, } @@ -220,7 +227,7 @@ fn bench_fetch(c: &mut Criterion) { .announcement .chunks(256) .map(|chunk| { - let hashes = chunk.iter().map(|(hash, _)| *hash).collect::>(); + let hashes = chunk.iter().map(|tx| tx.hash).collect::>(); (hashes[0], fixtures.deliver(&hashes)) }) .collect::>(); diff --git a/crates/net/network/src/test_utils/transactions.rs b/crates/net/network/src/test_utils/transactions.rs index 804d2993fbb..50b7266486c 100644 --- a/crates/net/network/src/test_utils/transactions.rs +++ b/crates/net/network/src/test_utils/transactions.rs @@ -4,8 +4,10 @@ use crate::{ transactions::{ + announcement::{AnnouncedTransaction, TransactionMetadata}, constants::tx_manager::DEFAULT_MAX_COUNT_TRANSACTIONS_SEEN_BY_PEER, - fetcher::TransactionFetcher, PeerMetadata, TransactionsManager, TransactionsManagerConfig, + fetcher::TransactionFetcher, + PeerMetadata, TransactionsManager, TransactionsManagerConfig, }, NetworkConfigBuilder, NetworkManager, }; @@ -59,7 +61,13 @@ pub fn buffer_hash_to_tx_fetcher( peer_id: PeerId, tx_encoded_length: Option, ) { - tx_fetcher.on_announcement(peer_id, [(hash, tx_encoded_length.map(|size| (0u8, size)))]); + tx_fetcher.on_announcement( + peer_id, + [AnnouncedTransaction { + hash, + metadata: tx_encoded_length.map(|size| TransactionMetadata { tx_type: 0, size }), + }], + ); } /// Mock a new session, returns (peer, channel-to-send-get-pooled-tx-response-on). diff --git a/crates/net/network/src/transactions/announcement.rs b/crates/net/network/src/transactions/announcement.rs new file mode 100644 index 00000000000..bec79dce9db --- /dev/null +++ b/crates/net/network/src/transactions/announcement.rs @@ -0,0 +1,244 @@ +//! Ordered transaction announcements used by the transaction manager and fetcher. + +use super::constants::SOFT_LIMIT_COUNT_HASHES_IN_NEW_POOLED_TRANSACTIONS_BROADCAST_MESSAGE; +use alloy_primitives::{map::B256Set, TxHash, B128}; +use derive_more::IntoIterator; +use reth_eth_wire::{EthVersion, HandleMempoolData, NewPooledTransactionHashes}; + +/// An announcement with unique hashes in the order supplied by the peer. +/// +/// Metadata comes from the wire message. Network-specific validation is performed by the +/// transaction manager's announcement policy. +#[derive(Debug, IntoIterator)] +pub struct TransactionAnnouncement { + #[into_iterator(owned, ref)] + entries: Vec, + version: EthVersion, + cell_mask: Option, +} + +impl TransactionAnnouncement { + /// Normalizes a wire announcement, keeping the first occurrence of each hash and its metadata. + /// The scratch set is cleared before use and its allocation is kept for subsequent messages, + /// unless an oversized announcement grew it past twice the announcement soft limit. + /// + /// Returns an error if the hash, type and size arrays have different lengths. + pub fn from_message( + msg: &NewPooledTransactionHashes, + seen: &mut B256Set, + ) -> alloy_rlp::Result { + let (metadata, cell_mask) = match msg { + NewPooledTransactionHashes::Eth66(_) => (None, None), + NewPooledTransactionHashes::Eth68(msg) => { + (Some((msg.types.as_slice(), msg.sizes.as_slice())), None) + } + NewPooledTransactionHashes::Eth72(msg) => { + (Some((msg.types.as_slice(), msg.sizes.as_slice())), msg.cell_mask) + } + }; + if let Some((types, sizes)) = metadata { + for len in [types.len(), sizes.len()] { + if len != msg.len() { + return Err(alloy_rlp::Error::ListLengthMismatch { + expected: msg.len(), + got: len, + }) + } + } + } + + seen.clear(); + seen.reserve(msg.len()); + let mut entries = Vec::with_capacity(msg.len()); + entries.extend(msg.iter_hashes().enumerate().filter(|(_, hash)| seen.insert(**hash)).map( + |(index, &hash)| AnnouncedTransaction { + hash, + metadata: metadata.map(|(types, sizes)| TransactionMetadata { + tx_type: types[index], + size: sizes[index], + }), + }, + )); + if seen.capacity() > MAX_RETAINED_SCRATCH_CAPACITY { + *seen = B256Set::default(); + } + Ok(Self { entries, version: msg.version(), cell_mask }) + } + + /// Returns the wire message version. + pub const fn version(&self) -> EthVersion { + self.version + } + + /// Returns the eth/72 message-level cell mask, if present. + pub const fn cell_mask(&self) -> Option { + self.cell_mask + } + + /// Returns the number of entries. + pub const fn len(&self) -> usize { + self.entries.len() + } + + /// Returns whether there are no entries. + pub const fn is_empty(&self) -> bool { + self.entries.is_empty() + } + + /// Iterates over the entries in announcement order. + pub fn iter(&self) -> impl ExactSizeIterator { + self.entries.iter() + } + + /// Retains entries satisfying the predicate without changing their order. + pub fn retain(&mut self, f: impl FnMut(&AnnouncedTransaction) -> bool) { + self.entries.retain(f); + } +} + +impl HandleMempoolData for TransactionAnnouncement { + fn is_empty(&self) -> bool { + self.is_empty() + } + + fn len(&self) -> usize { + self.len() + } + + fn retain_by_hash(&mut self, mut f: impl FnMut(&TxHash) -> bool) { + self.retain(|tx| f(&tx.hash)); + } +} + +/// A transaction hash and the metadata supplied by its announcing peer. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct AnnouncedTransaction { + /// The announced transaction hash. + pub hash: TxHash, + /// Type and size for eth/68 and later; absent for eth/66. + pub metadata: Option, +} + +/// Transaction metadata supplied in eth/68 and later announcements. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct TransactionMetadata { + /// The transaction type byte, interpreted by the network's announcement policy. + pub tx_type: u8, + /// The announced encoded transaction size in bytes. + pub size: usize, +} + +/// Scratch capacity kept between messages. Only oversized announcements grow the set past this +/// bound, and dropping it avoids pinning their allocation for the caller's lifetime. +const MAX_RETAINED_SCRATCH_CAPACITY: usize = + 2 * SOFT_LIMIT_COUNT_HASHES_IN_NEW_POOLED_TRANSACTIONS_BROADCAST_MESSAGE; + +#[cfg(test)] +mod tests { + use super::*; + use alloy_primitives::B256; + use reth_eth_wire::{NewPooledTransactionHashes68, NewPooledTransactionHashes72}; + + #[test] + fn dedup_preserves_first_metadata_order_and_version() { + let hashes = [ + B256::repeat_byte(3), + B256::repeat_byte(1), + B256::repeat_byte(3), + B256::repeat_byte(2), + ]; + let types = vec![1, 2, 3, 4]; + let sizes = vec![100, 200, 300, 400]; + let mask = Some(B128::repeat_byte(0x11)); + let messages = [ + NewPooledTransactionHashes::Eth66(hashes.to_vec().into()), + NewPooledTransactionHashes68 { + hashes: hashes.to_vec(), + types: types.clone(), + sizes: sizes.clone(), + } + .into(), + NewPooledTransactionHashes72 { hashes: hashes.to_vec(), types, sizes, cell_mask: mask } + .into(), + ]; + let mut seen = B256Set::default(); + for msg in messages { + let mut announcement = TransactionAnnouncement::from_message(&msg, &mut seen).unwrap(); + assert_eq!(announcement.version(), msg.version()); + assert_eq!( + announcement.iter().map(|tx| tx.hash).collect::>(), + [hashes[0], hashes[1], hashes[3]] + ); + let metadata = announcement.iter().map(|tx| tx.metadata).collect::>(); + if msg.version().has_eth68_metadata() { + assert_eq!( + metadata, + [(1, 100), (2, 200), (4, 400)] + .map(|(tx_type, size)| Some(TransactionMetadata { tx_type, size })) + ); + } else { + assert_eq!(metadata, [None; 3]); + } + + // Both hash-only and entry filters must keep the remaining metadata aligned. + announcement.retain_by_hash(|hash| *hash != hashes[1]); + announcement.retain(|tx| tx.hash != hashes[0]); + assert_eq!(announcement.iter().next().unwrap().hash, hashes[3]); + assert_eq!( + announcement.cell_mask(), + if msg.version() == EthVersion::Eth72 { mask } else { None } + ); + } + } + + #[test] + fn rejects_misaligned_metadata_in_locally_constructed_messages() { + for (hashes_len, types_len, sizes_len) in + [(2, 1, 2), (2, 2, 1), (1, 2, 1), (1, 1, 2), (0, 1, 1)] + { + let hashes = vec![B256::ZERO; hashes_len]; + let types = vec![2; types_len]; + let sizes = vec![100; sizes_len]; + for msg in [ + NewPooledTransactionHashes68 { + hashes: hashes.clone(), + types: types.clone(), + sizes: sizes.clone(), + } + .into(), + NewPooledTransactionHashes72 { hashes, types, sizes, cell_mask: None }.into(), + ] { + assert!( + TransactionAnnouncement::from_message(&msg, &mut B256Set::default()).is_err() + ); + } + } + } + + #[test] + fn scratch_is_reused_between_messages_and_dropped_when_oversized() { + let mut seen = B256Set::default(); + let msg = NewPooledTransactionHashes::Eth66(vec![B256::ZERO; 64].into()); + TransactionAnnouncement::from_message(&msg, &mut seen).unwrap(); + let capacity = seen.capacity(); + assert!(capacity >= 64); + let second = TransactionAnnouncement::from_message(&msg, &mut seen).unwrap(); + assert_eq!(second.len(), 1); + assert_eq!(seen.capacity(), capacity); + + // A full-size announcement keeps its allocation for later messages. + let full_len = SOFT_LIMIT_COUNT_HASHES_IN_NEW_POOLED_TRANSACTIONS_BROADCAST_MESSAGE; + let full = NewPooledTransactionHashes::Eth66(vec![B256::repeat_byte(1); full_len].into()); + assert_eq!(TransactionAnnouncement::from_message(&full, &mut seen).unwrap().len(), 1); + let capacity = seen.capacity(); + assert!(capacity >= full_len && capacity <= MAX_RETAINED_SCRATCH_CAPACITY); + assert_eq!(TransactionAnnouncement::from_message(&msg, &mut seen).unwrap().len(), 1); + assert_eq!(seen.capacity(), capacity); + + // An oversized announcement must not pin its allocation. + let oversized = + NewPooledTransactionHashes::Eth66(vec![B256::repeat_byte(2); 4 * full_len].into()); + assert_eq!(TransactionAnnouncement::from_message(&oversized, &mut seen).unwrap().len(), 1); + assert_eq!(seen.capacity(), 0); + } +} diff --git a/crates/net/network/src/transactions/fetcher.rs b/crates/net/network/src/transactions/fetcher.rs index 7771c72170c..d32d738ed7c 100644 --- a/crates/net/network/src/transactions/fetcher.rs +++ b/crates/net/network/src/transactions/fetcher.rs @@ -47,6 +47,7 @@ //! recent fallback sources does not allow unlimited retries use super::{ + announcement::{AnnouncedTransaction, TransactionMetadata}, config::TransactionFetcherConfig, constants::{ tx_fetcher::{ @@ -65,7 +66,7 @@ use alloy_primitives::{ TxHash, }; use futures::{stream::FuturesUnordered, Future, FutureExt, Stream, StreamExt}; -use reth_eth_wire::{Eth68TxMetadata, EthVersion, GetPooledTransactions, PooledTransactions}; +use reth_eth_wire::{EthVersion, GetPooledTransactions, PooledTransactions}; use reth_eth_wire_types::{EthNetworkPrimitives, NetworkPrimitives}; use reth_network_api::PeerRequest; use reth_network_p2p::error::{RequestError, RequestResult}; @@ -192,7 +193,7 @@ impl TransactionFetcher { pub fn on_announcement( &mut self, peer_id: PeerId, - announcement: impl IntoIterator, + announcement: impl IntoIterator, ) { let key = self.peer_key(peer_id); let max_per_peer = self.config.max_announced_hashes_per_peer as usize; @@ -217,7 +218,7 @@ impl TransactionFetcher { // that peer's accounting once, flushing before eviction needs up-to-date counts. let mut retired_candidates = DeferredCandidateRemovals::default(); - for (hash, metadata) in announcement { + for AnnouncedTransaction { hash, metadata } in announcement { let size = announced_size(metadata); let at_capacity = self.hashes.len() >= max_total; @@ -1405,9 +1406,10 @@ struct ResolvedRequest { /// Returns the announced transaction size used for request packing, capped at the response soft /// limit, or 0 if the announcement carried no size. -fn announced_size(metadata: Eth68TxMetadata) -> u32 { - metadata - .map_or(0, |(_, size)| size.min(SOFT_LIMIT_BYTE_SIZE_POOLED_TRANSACTIONS_RESPONSE) as u32) +fn announced_size(metadata: Option) -> u32 { + metadata.map_or(0, |metadata| { + metadata.size.min(SOFT_LIMIT_BYTE_SIZE_POOLED_TRANSACTIONS_RESPONSE) as u32 + }) } /// Filters a response down to the transactions that were requested, dropping duplicates. @@ -1545,13 +1547,19 @@ mod tests { ) { self.fetcher.on_announcement( peer_id, - entries.into_iter().map(|(h, size)| (h, Some((2, size)))), + entries.into_iter().map(|(hash, size)| AnnouncedTransaction { + hash, + metadata: Some(TransactionMetadata { tx_type: 2, size }), + }), ); self.verify(); } fn announce_unsized(&mut self, peer_id: PeerId, hashes: &[TxHash]) { - self.fetcher.on_announcement(peer_id, hashes.iter().map(|hash| (*hash, None))); + self.fetcher.on_announcement( + peer_id, + hashes.iter().map(|&hash| AnnouncedTransaction { hash, metadata: None }), + ); self.verify(); } diff --git a/crates/net/network/src/transactions/mod.rs b/crates/net/network/src/transactions/mod.rs index a507409b124..3bf96181d8b 100644 --- a/crates/net/network/src/transactions/mod.rs +++ b/crates/net/network/src/transactions/mod.rs @@ -4,6 +4,9 @@ use alloy_consensus::{constants::EIP4844_TX_TYPE_ID, transaction::TxHashRef}; use rayon::iter::{IntoParallelIterator, ParallelIterator}; use smallvec::SmallVec; +/// Normalized transaction announcements. +pub mod announcement; + /// Aggregation on configurable parameters for [`TransactionsManager`]. pub mod config; /// Default and spec'd bounds. @@ -19,6 +22,7 @@ pub use self::constants::{ tx_fetcher::DEFAULT_SOFT_LIMIT_BYTE_SIZE_POOLED_TRANSACTIONS_RESP_ON_PACK_GET_POOLED_TRANSACTIONS_REQ, SOFT_LIMIT_BYTE_SIZE_POOLED_TRANSACTIONS_RESPONSE, }; +use announcement::TransactionAnnouncement; use config::AnnouncementAcceptance; pub use config::{ AnnouncementFilteringPolicy, TransactionFetcherConfig, TransactionIngressPolicy, @@ -53,11 +57,10 @@ use alloy_rlp::Encodable; use constants::SOFT_LIMIT_COUNT_HASHES_IN_NEW_POOLED_TRANSACTIONS_BROADCAST_MESSAGE; use futures::{stream::FuturesUnordered, Future, StreamExt}; use reth_eth_wire::{ - BroadcastPoolTransactions, Cells, DedupPayload, EthNetworkPrimitives, EthVersion, GetCells, - GetPooledTransactions, HandleMempoolData, HandleVersionedMempoolData, LazyEncoded, - LazyEncodedTransaction, NetworkPrimitives, NewPooledTransactionHashes, - NewPooledTransactionHashes66, NewPooledTransactionHashes68, NewPooledTransactionHashes72, - PooledTransactions, Transactions, ValidAnnouncementData, + BroadcastPoolTransactions, Cells, EthNetworkPrimitives, EthVersion, GetCells, + GetPooledTransactions, LazyEncoded, LazyEncodedTransaction, NetworkPrimitives, + NewPooledTransactionHashes, NewPooledTransactionHashes66, NewPooledTransactionHashes68, + NewPooledTransactionHashes72, PooledTransactions, Transactions, }; use reth_ethereum_primitives::TxType; use reth_evm::SenderRecoveryCache; @@ -333,6 +336,8 @@ pub struct TransactionsManager>, + /// Scratch hashes reused when deduplicating announcements. + announcement_hashes: B256Set, /// Transactions that are currently imported into the `Pool`. /// /// The import process includes: @@ -439,6 +444,7 @@ impl TransactionsManager { network_events, transaction_fetcher, transactions_by_peers: Default::default(), + announcement_hashes: B256Set::default(), pool_imports: Default::default(), pending_pool_imports_info, bad_imports: LruCache::with_hasher(DEFAULT_MAX_COUNT_BAD_IMPORTS, Default::default()), @@ -672,13 +678,23 @@ impl TransactionsManager { }; let client = peer.client_version.clone(); - // keep track of the transactions the peer knows - let mut count_txns_already_seen_by_peer = 0; - for tx in msg.iter_hashes().copied() { - if !peer.seen_transactions.insert(tx) { - count_txns_already_seen_by_peer += 1; - } + if msg.is_empty() { + self.report_peer(peer_id, ReputationChangeKind::BadAnnouncement); + return } + + let Ok(mut announcement) = + TransactionAnnouncement::from_message(&msg, &mut self.announcement_hashes) + else { + self.report_peer(peer_id, ReputationChangeKind::BadAnnouncement); + return + }; + let has_duplicates = announcement.len() != msg.len(); + + // keep track of the transactions the peer knows + let count_txns_already_seen_by_peer = + msg.iter_hashes().filter(|hash| !peer.seen_transactions.insert(**hash)).count(); + drop(msg); if count_txns_already_seen_by_peer > 0 { // this may occur if transactions are sent or announced to a peer, at the same time as // the peer sends/announces those hashes to us. this is because, marking @@ -687,7 +703,7 @@ impl TransactionsManager { self.metrics.messages_with_hashes_already_seen_by_peer.increment(1); self.metrics .occurrences_hash_already_seen_by_peer - .increment(count_txns_already_seen_by_peer); + .increment(count_txns_already_seen_by_peer as u64); trace!(target: "net::tx", %count_txns_already_seen_by_peer, @@ -699,66 +715,34 @@ impl TransactionsManager { self.report_already_seen(peer_id); } - // 1. filter out spam - if msg.is_empty() { - self.report_peer(peer_id, ReputationChangeKind::BadAnnouncement); - return; - } - - let original_len = msg.len(); - let mut partially_valid_msg = msg.dedup(); - - if partially_valid_msg.len() != original_len { + if has_duplicates { self.report_peer(peer_id, ReputationChangeKind::BadAnnouncement); } - // 2. filter out transactions pending import to pool and known bad transactions - partially_valid_msg.retain_by_hash(|hash| { - !self.transactions_by_peers.contains_key(hash) && !self.bad_imports.contains(hash) - }); - - // 3. filter out invalid entries (spam) - // - // validates messages with respect to the given network, e.g. allowed tx types. - // done before the pool lookup since these are cheap in-memory checks that shrink - // the set before acquiring the pool lock. - // + // Exclude pending imports and known bad transactions before counting types and applying + // policy. These cheap checks shrink the batch before acquiring the pool lock. let mut should_report_peer = false; let mut tx_types_counter = TxTypesCounter::default(); + let has_eth68_metadata = announcement.version().has_eth68_metadata(); - let has_eth68_metadata = partially_valid_msg - .msg_version() - .expect("partially valid announcement should have a version") - .has_eth68_metadata(); - - partially_valid_msg.retain(|tx_hash, metadata_ref_mut| { - let (ty_byte, size_val) = match *metadata_ref_mut { - Some((ty, size)) => { - if !has_eth68_metadata { - should_report_peer = true; - } - (ty, size) - } - None => { - if has_eth68_metadata { - should_report_peer = true; - return false; - } - (0u8, 0) - } - }; - - if has_eth68_metadata && let Some((actual_ty_byte, _)) = *metadata_ref_mut { - match TxType::try_from(actual_ty_byte) { - Ok(parsed_tx_type) => tx_types_counter.increase_by_tx_type(parsed_tx_type), + announcement.retain(|tx| { + if self.transactions_by_peers.contains_key(&tx.hash) || + self.bad_imports.contains(&tx.hash) + { + return false + } + let (ty_byte, size_val) = tx.metadata.map_or((0, 0), |metadata| { + match TxType::try_from(metadata.tx_type) { + Ok(tx_type) => tx_types_counter.increase_by_tx_type(tx_type), Err(_) => tx_types_counter.increase_other(), } - } + (metadata.tx_type, metadata.size) + }); let decision = self .policies .announcement_filter() - .decide_on_announcement(ty_byte, tx_hash, size_val); + .decide_on_announcement(ty_byte, &tx.hash, size_val); match decision { AnnouncementAcceptance::Accept => true, @@ -780,41 +764,37 @@ impl TransactionsManager { self.report_peer(peer_id, ReputationChangeKind::BadAnnouncement); } - // 4. filter out known hashes + // Filter out hashes already in the pool. // // known txns have already been successfully fetched or received over gossip. // // most hashes will be filtered out here since the mempool protocol is a gossip // protocol, healthy peers will send many of the same hashes. // - let hashes_count_pre_pool_filter = partially_valid_msg.len(); - self.pool.retain_unknown(&mut partially_valid_msg); - if hashes_count_pre_pool_filter > partially_valid_msg.len() { - let already_known_hashes_count = - hashes_count_pre_pool_filter - partially_valid_msg.len(); + let hashes_count_pre_pool_filter = announcement.len(); + self.pool.retain_unknown(&mut announcement); + if hashes_count_pre_pool_filter > announcement.len() { + let already_known_hashes_count = hashes_count_pre_pool_filter - announcement.len(); self.metrics .occurrences_hashes_already_in_pool .increment(already_known_hashes_count as u64); } - if partially_valid_msg.is_empty() { + if announcement.is_empty() { // nothing to request return } - let valid_announcement_data = - ValidAnnouncementData::from_partially_valid_data(partially_valid_msg); - trace!(target: "net::tx::propagation", peer_id=format!("{peer_id:#}"), - hashes_len=valid_announcement_data.len(), - msg_version=%valid_announcement_data.msg_version(), + hashes_len=announcement.len(), + msg_version=%announcement.version(), client_version=%client, "received unknown hashes in announcement from peer" ); - // 5. queue the hashes for fetching, requests are sent when the manager is polled - self.transaction_fetcher.on_announcement(peer_id, valid_announcement_data); + // Queue hashes for fetching; requests are sent when the manager is polled. + self.transaction_fetcher.on_announcement(peer_id, announcement); } } @@ -2396,6 +2376,63 @@ mod tests { InMemoryBlobStore, >; + #[tokio::test] + async fn announcement_policy_preserves_order_and_skips_pending_and_bad_imports() { + #[derive(Debug, Clone, Default)] + struct RecordingPolicy(Arc>>); + + impl AnnouncementFilteringPolicy for RecordingPolicy { + fn decide_on_announcement( + &self, + tx_type: u8, + hash: &TxHash, + size: usize, + ) -> AnnouncementAcceptance { + self.0.lock().push((tx_type, *hash, size)); + match tx_type { + 0xff => AnnouncementAcceptance::Reject { penalize_peer: true }, + 1 => AnnouncementAcceptance::Ignore, + _ => AnnouncementAcceptance::Accept, + } + } + } + + let (mut manager, _network) = new_tx_manager().await; + let policy = RecordingPolicy::default(); + manager.policies = + NetworkPolicies::new(TransactionPropagationKind::default(), policy.clone()); + let peer_id = PeerId::new([1; 64]); + let (peer, mut rx) = new_mock_session(peer_id, EthVersion::Eth68); + manager.peers.insert(peer_id, peer); + let [pending, accepted, ignored, rejected, later, bad] = + [1, 2, 3, 4, 5, 6].map(B256::repeat_byte); + manager.transactions_by_peers.insert(pending, Default::default()); + manager.bad_imports.insert(bad); + manager.on_new_pooled_transaction_hashes( + peer_id, + NewPooledTransactionHashes68 { + hashes: vec![pending, accepted, accepted, ignored, rejected, later, bad], + types: vec![0xff, 2, 0xff, 1, 0xff, 2, 0xff], + sizes: vec![100, 200, 300, 400, 500, 600, 700], + } + .into(), + ); + + assert_eq!( + *policy.0.lock(), + [(2, accepted, 200), (1, ignored, 400), (0xff, rejected, 500), (2, later, 600)] + ); + poll_fn(|cx| { + let _ = manager.poll_unpin(cx); + Poll::Ready(()) + }) + .await; + let PeerRequest::GetPooledTransactions { request, .. } = rx.try_recv().unwrap() else { + panic!("expected transaction request") + }; + assert_eq!(request.0, [accepted, later]); + } + async fn new_eth_tx_manager() -> ( TransactionsManager, NetworkManager, From ba0c656e251933767627f80ee0e2e62c4d2e8229 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A9a=20Narzis?= <78718413+lean-apple@users.noreply.github.com> Date: Mon, 14 Sep 2026 21:04:57 +0200 Subject: [PATCH 257/294] chore: update rustls to 0.23.45 (#27233) --- Cargo.lock | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index dc937330fce..fad0103d913 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1577,9 +1577,9 @@ checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" [[package]] name = "aws-lc-rs" -version = "1.16.3" +version = "1.18.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ec6fb3fe69024a75fa7e1bfb48aa6cf59706a101658ea01bfd33b2b248a038f" +checksum = "b281d307588d634de920874890732659e2e7672f72b5e10e81badc1a8a83621e" dependencies = [ "aws-lc-sys", "untrusted 0.7.1", @@ -1588,14 +1588,15 @@ dependencies = [ [[package]] name = "aws-lc-sys" -version = "0.40.0" +version = "0.45.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f50037ee5e1e41e7b8f9d161680a725bd1626cb6f8c7e901f91f942850852fe7" +checksum = "9bff6c3b54fad79a2e60b8102caf565819711497c1f5f092f49508e2f5c31b27" dependencies = [ "cc", "cmake", "dunce", "fs_extra", + "pkg-config", ] [[package]] @@ -11567,9 +11568,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.38" +version = "0.23.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69f9466fb2c14ea04357e91413efb882e2a6d4a406e625449bc0a5d360d53a21" +checksum = "0d41d731c7d2f962d1ccc364cec258de3c0e93b38c2fb3ba97ac74513048d634" dependencies = [ "aws-lc-rs", "log", @@ -11653,9 +11654,9 @@ checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f" [[package]] name = "rustls-webpki" -version = "0.103.13" +version = "0.103.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e" +checksum = "f3c3cf1d8b1e7d4927e2d154c3fcb02979afb9939629c62cd9048d4f07b60ac2" dependencies = [ "aws-lc-rs", "ring", From 1f5446b01c2aa5acd41e6fb66eef232634dceb59 Mon Sep 17 00:00:00 2001 From: Derek Cofausper <256792747+decofe@users.noreply.github.com> Date: Mon, 14 Sep 2026 19:44:50 +0000 Subject: [PATCH 258/294] fix(engine): consume saved cache during handoff (#27234) Co-authored-by: Matthias Seitz <19890894+mattsse@users.noreply.github.com> Co-authored-by: Matthias Seitz --- crates/engine/execution-cache/src/cached_state.rs | 5 +++++ crates/engine/tree/src/tree/payload_processor/prewarm.rs | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/crates/engine/execution-cache/src/cached_state.rs b/crates/engine/execution-cache/src/cached_state.rs index ab1dc80fcb0..d7ebd201d5f 100644 --- a/crates/engine/execution-cache/src/cached_state.rs +++ b/crates/engine/execution-cache/src/cached_state.rs @@ -1394,6 +1394,11 @@ impl SavedCache { &self.caches } + /// Consumes the saved cache and returns its [`ExecutionCache`]. + pub fn into_cache(self) -> ExecutionCache { + self.caches + } + /// Updates the cache metrics (size/capacity/collisions) from the stats handlers. pub fn update_metrics(&self, metrics: Option<&CachedStateCacheMetrics>) { if let Some(metrics) = metrics { diff --git a/crates/engine/tree/src/tree/payload_processor/prewarm.rs b/crates/engine/tree/src/tree/payload_processor/prewarm.rs index 5fa6a461a84..4d428291920 100644 --- a/crates/engine/tree/src/tree/payload_processor/prewarm.rs +++ b/crates/engine/tree/src/tree/payload_processor/prewarm.rs @@ -301,7 +301,7 @@ where execution_cache.update_with_guard(|cached| { // consumes the `SavedCache` held by the prewarming task, which releases its cache // handle - let caches = saved_cache.cache().clone(); + let caches = saved_cache.into_cache(); let new_cache = SavedCache::new(hash, caches); // Insert state into cache while holding the lock From 8194fcedefcfa1cbfe11d57f30cae45a21fe136f Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Mon, 14 Sep 2026 17:06:23 -0400 Subject: [PATCH 259/294] refactor(db): keep optional block cache size in builder chain (#27237) --- crates/node/builder/src/launch/common.rs | 10 ++++------ .../storage/provider/src/providers/rocksdb/provider.rs | 9 +++++++++ 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/crates/node/builder/src/launch/common.rs b/crates/node/builder/src/launch/common.rs index 494bf3982e4..70287d252f2 100644 --- a/crates/node/builder/src/launch/common.rs +++ b/crates/node/builder/src/launch/common.rs @@ -514,14 +514,12 @@ where let rocksdb_provider = if let Some(provider) = rocksdb_provider { provider } else { - let mut builder = RocksDBProvider::builder(self.data_dir().rocksdb()) + RocksDBProvider::builder(self.data_dir().rocksdb()) .with_default_tables() .with_metrics() - .with_statistics(); - if let Some(cache_size) = self.node_config().db.rocksdb_block_cache_size { - builder = builder.with_block_cache_size(cache_size); - } - builder.build()? + .with_statistics() + .with_block_cache_size_opt(self.node_config().db.rocksdb_block_cache_size) + .build()? }; let balstore_cache_size = self diff --git a/crates/storage/provider/src/providers/rocksdb/provider.rs b/crates/storage/provider/src/providers/rocksdb/provider.rs index 170d0619a49..3fd66f00067 100644 --- a/crates/storage/provider/src/providers/rocksdb/provider.rs +++ b/crates/storage/provider/src/providers/rocksdb/provider.rs @@ -365,6 +365,15 @@ impl RocksDBBuilder { self } + /// Sets a custom block cache size if provided, otherwise keeps the current cache. + pub fn with_block_cache_size_opt(self, capacity_bytes: Option) -> Self { + if let Some(capacity_bytes) = capacity_bytes { + self.with_block_cache_size(capacity_bytes) + } else { + self + } + } + /// Sets read-only mode. /// /// Opens the database as a secondary instance, which supports catching up From 32a801ff86510faed96a574965be5b8f1e9393b4 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Tue, 15 Sep 2026 00:03:48 +0000 Subject: [PATCH 260/294] fix(txpool): park pending blob txs when the blob fee rises and the base fee falls (#26962) --- crates/transaction-pool/src/pool/txpool.rs | 119 ++++++++++++++++++++- 1 file changed, 118 insertions(+), 1 deletion(-) diff --git a/crates/transaction-pool/src/pool/txpool.rs b/crates/transaction-pool/src/pool/txpool.rs index 92a5252674f..01608d93fda 100644 --- a/crates/transaction-pool/src/pool/txpool.rs +++ b/crates/transaction-pool/src/pool/txpool.rs @@ -253,7 +253,45 @@ impl TxPool { self.add_transaction_to_subpool(to, tx); } } - (Ordering::Less, _) | (_, Ordering::Less) => { + (Ordering::Greater, Ordering::Less) => { + // The blob fee increased while the base fee decreased, so this update must both + // park transactions that no longer cover the blob fee and promote transactions + // that now cover both fees. Demote first so the promotion check sees the final + // fee state. + let removed = + self.pending_pool.update_blob_fee(self.all_transactions.pending_fees.blob_fee); + for tx in removed { + let to = { + let tx = + self.all_transactions.txs.get_mut(tx.id()).expect("tx exists in set"); + + tx.state.remove(TxState::ENOUGH_BLOB_FEE_CAP_BLOCK); + tx.subpool = tx.state.into(); + tx.subpool + }; + self.add_transaction_to_subpool(to, tx); + } + + let removed = + self.blob_pool.enforce_pending_fees(&self.all_transactions.pending_fees); + for tx in removed { + let subpool = { + let tx_meta = + self.all_transactions.txs.get_mut(tx.id()).expect("tx exists in set"); + tx_meta.state.insert(TxState::ENOUGH_BLOB_FEE_CAP_BLOCK); + tx_meta.state.insert(TxState::ENOUGH_FEE_CAP_BLOCK); + tx_meta.subpool = tx_meta.state.into(); + tx_meta.subpool + }; + + if subpool == SubPool::Pending { + on_promoted(&tx); + } + + self.add_transaction_to_subpool(subpool, tx); + } + } + (Ordering::Less, _) | (Ordering::Equal, Ordering::Less) => { // decreased blob/base fee: recheck blob pool and promote all that are now valid let removed = self.blob_pool.enforce_pending_fees(&self.all_transactions.pending_fees); @@ -4005,6 +4043,85 @@ mod tests { assert!(tx_meta.state.contains(TxState::ENOUGH_FEE_CAP_BLOCK)); } + #[test] + fn update_blob_fee_parks_pending_when_base_fee_falls_in_the_same_block() { + let mut f = MockTransactionFactory::default(); + let mut pool = TxPool::new(MockOrdering::default(), Default::default()); + + let initial_blob_fee = pool.all_transactions.pending_fees.blob_fee; + let initial_base_fee = 100u64; + pool.all_transactions.pending_fees.base_fee = initial_base_fee; + + // comfortably satisfies both fees, so it starts out pending + let tx = MockTransaction::eip4844() + .with_max_fee(500) + .with_priority_fee(1) + .with_blob_fee(initial_blob_fee + 100); + let validated = f.validated(tx.clone()); + let id = *validated.id(); + pool.add_transaction(validated, U256::from(1_000_000), 0, None).unwrap(); + assert_eq!(pool.pending_pool.len(), 1); + + // the blob fee rises past its cap while the base fee falls, which is what a blob heavy + // but gas light block produces. The rise still has to park it. + let raised_blob_fee = tx.max_fee_per_blob_gas().unwrap() + 1; + pool.all_transactions.pending_fees.base_fee = initial_base_fee - 1; + pool.update_blob_fee(raised_blob_fee, Ordering::Less, |_| {}); + + assert!(pool.pending_pool.is_empty(), "transaction was left in the pending pool"); + assert_eq!(pool.blob_pool.len(), 1); + + let tx_meta = pool.all_transactions.txs.get(&id).unwrap(); + assert!( + !tx_meta.state.contains(TxState::ENOUGH_BLOB_FEE_CAP_BLOCK), + "blob fee cap flag was not cleared" + ); + assert_eq!(tx_meta.subpool, SubPool::Blob); + } + + #[test] + fn update_blob_fee_demotes_and_promotes_when_base_fee_falls() { + let mut f = MockTransactionFactory::default(); + let mut pool = TxPool::new(MockOrdering::default(), Default::default()); + + let initial_blob_fee = pool.all_transactions.pending_fees.blob_fee; + let initial_base_fee = 600; + pool.all_transactions.pending_fees.base_fee = initial_base_fee; + + let tx_to_demote = MockTransaction::eip4844() + .with_max_fee(700) + .with_priority_fee(1) + .with_blob_fee(initial_blob_fee + 100); + let validated = f.validated(tx_to_demote); + let demoted_id = *validated.id(); + pool.add_transaction(validated, U256::from(1_000_000), 0, None).unwrap(); + + let tx_to_promote = MockTransaction::eip4844() + .with_max_fee(500) + .with_priority_fee(1) + .with_blob_fee(initial_blob_fee + 300); + let validated = f.validated(tx_to_promote); + let promoted_id = *validated.id(); + pool.add_transaction(validated, U256::from(1_000_000), 0, None).unwrap(); + + assert_eq!(pool.pending_pool.len(), 1); + assert_eq!(pool.blob_pool.len(), 1); + + // The lower base fee makes one transaction affordable while the higher blob fee makes the + // other unaffordable, so the same update must both promote and demote. + let raised_blob_fee = initial_blob_fee + 200; + pool.all_transactions.pending_fees.base_fee = 400; + let mut promoted = Vec::new(); + pool.update_blob_fee(raised_blob_fee, Ordering::Less, |tx| promoted.push(*tx.id())); + + assert_eq!(pool.pending_pool.len(), 1); + assert_eq!(pool.blob_pool.len(), 1); + assert_eq!(promoted, vec![promoted_id]); + + assert_eq!(pool.all_transactions.txs.get(&demoted_id).unwrap().subpool, SubPool::Blob); + assert_eq!(pool.all_transactions.txs.get(&promoted_id).unwrap().subpool, SubPool::Pending); + } + #[test] fn apply_fee_updates_records_promotions_after_blob_fee_drop() { let mut f = MockTransactionFactory::default(); From c3f0746bd837610aa1eaeb0c2ab5ca9cf4337eff Mon Sep 17 00:00:00 2001 From: joshieDo <93316087+joshieDo@users.noreply.github.com> Date: Tue, 15 Sep 2026 09:11:58 +0000 Subject: [PATCH 261/294] fix(engine): avoid cache handoff misses and locked drops (#27230) Co-authored-by: Codex Co-authored-by: Matthias Seitz <19890894+mattsse@users.noreply.github.com> Co-authored-by: Derek Cofausper <256792747+decofe@users.noreply.github.com> --- .../execution-cache/src/cached_state.rs | 6 + crates/engine/execution-cache/src/lib.rs | 24 +- crates/engine/primitives/src/message.rs | 3 +- crates/engine/tree/src/tree/mod.rs | 6 +- .../src/tree/payload_processor/prewarm.rs | 399 ++++++++++++++++-- crates/rpc/rpc-api/src/reth_engine.rs | 5 +- 6 files changed, 400 insertions(+), 43 deletions(-) diff --git a/crates/engine/execution-cache/src/cached_state.rs b/crates/engine/execution-cache/src/cached_state.rs index d7ebd201d5f..6c500bbad96 100644 --- a/crates/engine/execution-cache/src/cached_state.rs +++ b/crates/engine/execution-cache/src/cached_state.rs @@ -1399,6 +1399,12 @@ impl SavedCache { self.caches } + /// Returns whether `self` and `other` refer to the same [`ExecutionCache`] data, + /// regardless of their block hashes. + pub fn shares_cache_with(&self, other: &Self) -> bool { + Arc::ptr_eq(&self.caches.0, &other.caches.0) + } + /// Updates the cache metrics (size/capacity/collisions) from the stats handlers. pub fn update_metrics(&self, metrics: Option<&CachedStateCacheMetrics>) { if let Some(metrics) = metrics { diff --git a/crates/engine/execution-cache/src/lib.rs b/crates/engine/execution-cache/src/lib.rs index f9cd0cf5f1b..a419b9e3167 100644 --- a/crates/engine/execution-cache/src/lib.rs +++ b/crates/engine/execution-cache/src/lib.rs @@ -104,12 +104,13 @@ impl PayloadExecutionCache { None } - /// Waits until the execution cache becomes available for use. + /// Waits for the mutex protecting the stored `Option` to be released. /// - /// This acquires a write lock to ensure exclusive access, then immediately releases it. - /// This is useful for synchronization before starting payload processing. + /// This does not wait for other users to drop their [`ExecutionCache`] clones or for removed + /// caches to finish dropping after unlocking. A subsequent [`Self::get_cache_for`] can still + /// return `None`, causing its caller to allocate a new cache while those drops run. /// - /// Returns the time spent waiting for the lock. + /// Returns only the time spent waiting for the mutex, excluding post-unlock cleanup. pub fn wait_for_availability(&self) -> Duration { let start = Instant::now(); // Acquire lock to wait for any current holders to finish @@ -125,8 +126,13 @@ impl PayloadExecutionCache { elapsed } - /// Updates the cache with a closure that has exclusive access to the guard. - /// This ensures that all cache operations happen atomically. + /// Runs `update_fn` with mutable access to the stored `Option` under the mutex. + /// Returns the closure's result after releasing the mutex, allowing removed caches to be + /// dropped outside the lock. + /// + /// Drop extra [`SavedCache`] or [`ExecutionCache`] clones of the stored cache before the + /// closure returns: [`Self::get_cache_for`] requires that cache's Arc strong reference + /// count to be one. /// /// ## CRITICAL SAFETY REQUIREMENT /// @@ -138,12 +144,12 @@ impl PayloadExecutionCache { /// /// Violating this requirement can result in cache corruption, incorrect state data, /// and potential consensus failures. - pub fn update_with_guard(&self, update_fn: F) + pub fn update_with_guard(&self, update_fn: F) -> R where - F: FnOnce(&mut Option), + F: FnOnce(&mut Option) -> R, { let mut guard = self.inner.lock(); - update_fn(&mut guard); + update_fn(&mut guard) } } diff --git a/crates/engine/primitives/src/message.rs b/crates/engine/primitives/src/message.rs index 9e7c70fab76..951f3196421 100644 --- a/crates/engine/primitives/src/message.rs +++ b/crates/engine/primitives/src/message.rs @@ -351,7 +351,8 @@ where /// Sends a new payload message used by `reth_newPayload` endpoint. /// /// `wait_for_persistence`: waits for in-flight persistence to complete. - /// `wait_for_caches`: waits for execution cache and sparse trie locks. + /// `wait_for_caches`: waits for execution cache and sparse trie locks, excluding destruction + /// of removed execution-cache allocations after unlocking. /// /// Returns detailed timing breakdown alongside the payload status. pub async fn reth_new_payload( diff --git a/crates/engine/tree/src/tree/mod.rs b/crates/engine/tree/src/tree/mod.rs index 799c62a4d41..414ad1834cf 100644 --- a/crates/engine/tree/src/tree/mod.rs +++ b/crates/engine/tree/src/tree/mod.rs @@ -3681,7 +3681,7 @@ enum PersistTarget { /// Result of waiting for caches to become available. #[derive(Debug, Clone, Copy, Default)] pub struct CacheWaitDurations { - /// Time spent waiting for the execution cache lock. + /// Time spent waiting for the execution cache lock, excluding post-unlock cache destruction. pub execution_cache: Duration, /// Time spent waiting for the sparse trie lock. pub sparse_trie: Duration, @@ -3689,8 +3689,8 @@ pub struct CacheWaitDurations { /// Trait for types that can wait for caches to become available. /// -/// This is used by `reth_newPayload` endpoint to ensure that payload processing -/// waits for any ongoing operations to complete before starting. +/// Used by `reth_newPayload` to wait for cache updates before starting payload processing. +/// Removed execution-cache allocations may still be destroyed concurrently after unlocking. pub trait WaitForCaches { /// Waits for cache updates to complete. /// diff --git a/crates/engine/tree/src/tree/payload_processor/prewarm.rs b/crates/engine/tree/src/tree/payload_processor/prewarm.rs index 4d428291920..b16ed3f2a1f 100644 --- a/crates/engine/tree/src/tree/payload_processor/prewarm.rs +++ b/crates/engine/tree/src/tree/payload_processor/prewarm.rs @@ -270,17 +270,21 @@ where }); } - /// This method calls `ExecutionCache::update_with_guard` which requires exclusive access. - /// It should only be called after ensuring that: + /// Saves the warmed cache in `self.execution_cache` after prewarming completes. + /// + /// This method calls [`PayloadExecutionCache::update_with_guard`], which requires exclusive + /// access. It should only be called after ensuring that: /// 1. All prewarming tasks have completed execution /// 2. No other concurrent operations are accessing the cache /// - /// Saves the warmed caches back into the shared slot after prewarming completes. - /// - /// This consumes the `SavedCache` held by the task, which releases its cache handle and allows - /// the new, warmed cache to be inserted. - /// + /// This moves the task's `ExecutionCache` into `self.execution_cache` when the block is valid, + /// without retaining an extra Arc reference that would prevent reuse after unlocking. /// This method is called from `run()` only after all execution tasks are complete. + /// + /// State insertion and block validation run under the mutex because the cache being updated + /// may also be stored in `self.execution_cache`. Removed `SavedCache` values are dropped after + /// unlocking, before the next prewarm task. Their contents are freed only when the last + /// `ExecutionCache` clone is dropped, which may happen later on another thread. #[instrument(level = "debug", target = "engine::tree::payload_processor::prewarm", skip_all)] fn save_cache( self, @@ -298,35 +302,45 @@ where if let Some(saved_cache) = saved_cache { debug!(target: "engine::caching", parent_hash=?hash, "Updating execution cache"); - execution_cache.update_with_guard(|cached| { - // consumes the `SavedCache` held by the prewarming task, which releases its cache - // handle - let caches = saved_cache.into_cache(); - let new_cache = SavedCache::new(hash, caches); - - // Insert state into cache while holding the lock - // Access the BundleState through the shared ExecutionOutcome + let (previous, rejected) = execution_cache.update_with_guard(|cached| { + let new_cache = SavedCache::new(hash, saved_cache.into_cache()); + + // Update under the mutex so no checkout can observe partially updated state. if new_cache.cache().insert_state(&execution_outcome.state).is_err() { - // Clear the cache on error to prevent having a polluted cache - *cached = None; debug!(target: "engine::caching", "cleared execution cache on update error"); - return; + return (cached.take(), Some(new_cache)); } new_cache.update_metrics(cache_state_metrics.as_ref()); - if valid_block_rx.recv().is_ok() { - // Replace the shared cache with the new one; the previous cache (if any) is - // dropped. - *cached = Some(new_cache); - } else { - // Block was invalid; caches were already mutated by insert_state above, - // so we must clear to prevent using polluted state - *cached = None; + // `cached` and `new_cache` can point to the same cache. + // insert_state has already applied this block's changes to it, so keep the mutex + // locked until validation succeeds or we clear `cached` on failure. + if valid_block_rx.recv().is_err() { debug!(target: "engine::caching", "cleared execution cache on invalid block"); + return (cached.take(), Some(new_cache)); + } + + let reused = + cached.as_ref().is_some_and(|previous| previous.shares_cache_with(&new_cache)); + let previous = cached.replace(new_cache); + if reused { + // `cached` and `previous` hold the same cache. Drop the extra Arc reference + // before unlocking so get_cache_for can reuse it. + drop(previous); + (None, None) + } else { + // Drop the old cache after unlocking; freeing it may be expensive. + (previous, None) } }); + // Drop these SavedCache values on this worker, without a background drop queue. + // This frees their contents only if no other ExecutionCache clones remain. + // Otherwise, the thread dropping the last ExecutionCache clone frees them later. + // Another payload may access or allocate a cache while these drops run. + drop((previous, rejected)); + let elapsed = start.elapsed(); debug!(target: "engine::caching", parent_hash=?hash, elapsed=?elapsed, "Updated execution cache"); @@ -776,7 +790,7 @@ mod tests { use alloy_eip7928::{AccountChanges, BalanceChange, BlockAccessIndex}; use alloy_primitives::{address, B256, U256}; use reth_chainspec::ChainSpec; - use reth_ethereum_primitives::TransactionSigned; + use reth_ethereum_primitives::{EthPrimitives, TransactionSigned}; use reth_evm::{execute::WithTxEnv, TxEnvFor}; use reth_evm_ethereum::EthEvmConfig; use reth_primitives_traits::Account; @@ -822,6 +836,334 @@ mod tests { assert!(terminate_execution.load(Ordering::Relaxed)); } + fn test_prewarm_context( + saved_cache: SavedCache, + saving_duration: Gauge, + ) -> PrewarmContext { + PrewarmContext { + env: ExecutionEnv { hash: B256::repeat_byte(2), ..ExecutionEnv::test_default() }, + evm_config: EthEvmConfig::new(Arc::new(ChainSpec::default())), + saved_cache: Some(saved_cache), + provider: OverlayStateProviderFactory::new( + MockEthProvider::default(), + OverlayManager::default().overlay_builder(B256::ZERO), + ), + bal_prewarm_pool: None, + metrics: PrewarmMetrics { + cache_saving_duration: saving_duration, + ..Default::default() + }, + cache_metrics: None, + cache_state_metrics: None, + terminate_execution: Arc::new(AtomicBool::new(false)), + executed_tx_index: Arc::new(AtomicUsize::new(0)), + precompile_cache_disabled: false, + precompile_cache_map: PrecompileCacheMap::default(), + disable_bal_parallel_state_root: false, + disable_bal_batch_io: false, + } + } + + fn save_test_cache( + runtime: &Runtime, + execution_cache: &PayloadExecutionCache, + saved_cache: SavedCache, + state: reth_revm::db::BundleState, + valid: bool, + saving_duration: Gauge, + ) { + let ctx = test_prewarm_context(saved_cache, saving_duration); + let (task, _) = PrewarmCacheTask::new(runtime.clone(), execution_cache.clone(), ctx); + let (valid_tx, valid_rx) = mpsc::channel(); + if valid { + valid_tx.send(()).unwrap(); + } + drop(valid_tx); + task.save_cache( + Arc::new(BlockExecutionOutput { state, result: Default::default() }), + valid_rx, + ); + } + + // Observe the handoff before save_cache returns and drops its local variables. A check after + // return would miss the window where the saving task still owns an extra reference. + struct CacheSaveObserver { + cache: PayloadExecutionCache, + observed: Arc, + } + + impl metrics::GaugeFn for CacheSaveObserver { + fn increment(&self, _: f64) {} + fn decrement(&self, _: f64) {} + fn set(&self, _: f64) { + assert!( + self.cache.get_cache_for(B256::repeat_byte(2)).is_some(), + "saved cache must be available before the saving task returns" + ); + self.observed.store(true, Ordering::Relaxed); + } + } + + #[test] + fn save_cache_releases_warm_cache_before_duration_metric() { + let runtime = Runtime::test(); + let execution_cache = PayloadExecutionCache::default(); + let saved = SavedCache::new(B256::repeat_byte(1), crate::tree::ExecutionCache::new(1_000)); + let address = address!("0000000000000000000000000000000000000001"); + saved.cache().insert_storage(address, B256::ZERO, Some(U256::from(7))); + execution_cache.update_with_guard(|slot| *slot = Some(saved.clone())); + // Keep the drop worker occupied: a queued SavedCache would delay cache reuse. + let (release_tx, release_rx) = mpsc::channel::<()>(); + runtime.spawn_blocking_named("drop", move || { + let _ = release_rx.recv(); + }); + let observed = Arc::new(AtomicBool::new(false)); + let observer = Arc::new(CacheSaveObserver { + cache: execution_cache.clone(), + observed: observed.clone(), + }); + save_test_cache( + &runtime, + &execution_cache, + saved, + Default::default(), + true, + Gauge::from_arc(observer), + ); + assert!(observed.load(Ordering::Relaxed), "save duration was not recorded"); + let saved = execution_cache.get_cache_for(B256::repeat_byte(2)).unwrap(); + assert_eq!( + saved.cache().get_or_try_insert_storage_with(address, B256::ZERO, || Err(())), + Ok(reth_execution_cache::CachedStatus::Cached(U256::from(7))), + "handoff must preserve the warmed contents", + ); + drop(release_tx); + } + + #[test] + fn save_cache_blocks_reuse_while_execution_cache_is_cloned() { + let runtime = Runtime::test(); + let execution_cache = PayloadExecutionCache::default(); + let saved = SavedCache::new(B256::repeat_byte(1), crate::tree::ExecutionCache::new(1_000)); + execution_cache.update_with_guard(|slot| *slot = Some(saved.clone())); + let other_cache_clone = saved.cache().clone(); + + save_test_cache(&runtime, &execution_cache, saved, Default::default(), true, Gauge::noop()); + + assert!(execution_cache.get_cache_for(B256::repeat_byte(2)).is_none()); + drop(other_cache_clone); + assert!(execution_cache.get_cache_for(B256::repeat_byte(2)).is_some()); + } + + #[derive(Clone, Copy, Debug)] + enum CacheSlot { + Empty, + Shared, + Distinct, + } + + // EIP-7702 bytecode preserves its owned buffer, letting us observe actual cache destruction. + struct CacheDropProbe { + started: Sender, + inspected: Receiver<()>, + result: Sender, + } + + impl AsRef<[u8]> for CacheDropProbe { + fn as_ref(&self) -> &[u8] { + &[0xef, 0x01, 0x00, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] + } + } + + impl Drop for CacheDropProbe { + fn drop(&mut self) { + let _ = self.started.send(std::thread::current().id()); + // A timeout turns destruction under the mutex into a failure instead of a deadlock. + let unlocked = self.inspected.recv_timeout(std::time::Duration::from_secs(5)).is_ok(); + let _ = self.result.send(unlocked); + } + } + + fn observe_cache_drop( + saved: &SavedCache, + execution_cache: &PayloadExecutionCache, + expect_saved_cache: bool, + ) -> (Receiver, std::thread::JoinHandle) { + let (started_tx, started_rx) = mpsc::channel(); + let (inspected_tx, inspected_rx) = mpsc::channel(); + let (result_tx, result_rx) = mpsc::channel(); + let cache = execution_cache.clone(); + let reader = std::thread::spawn(move || { + let drop_thread = started_rx.recv().unwrap(); + // Waiting for the mutex must return while the removed cache is still being dropped. + cache.wait_for_availability(); + assert_eq!(cache.get_cache_for(B256::repeat_byte(2)).is_some(), expect_saved_cache); + cache.update_with_guard(|slot| assert_eq!(slot.is_some(), expect_saved_cache)); + let _ = inspected_tx.send(()); + drop_thread + }); + let bytes = alloy_primitives::bytes::Bytes::from_owner(CacheDropProbe { + started: started_tx, + inspected: inspected_rx, + result: result_tx, + }); + let code = reth_revm::bytecode::Bytecode::new_eip7702_raw(bytes.into()).unwrap(); + saved + .cache() + .insert_code(B256::repeat_byte(3), Some(reth_primitives_traits::Bytecode(code))); + (result_rx, reader) + } + + #[test] + fn save_cache_freeing_waits_for_validator_cache_references() { + use crate::tree::payload_processor::{CacheTaskHandle, PayloadHandle}; + + let runtime = Runtime::test(); + let execution_cache = PayloadExecutionCache::default(); + let saved = SavedCache::new(B256::repeat_byte(1), crate::tree::ExecutionCache::new(1_000)); + execution_cache.update_with_guard(|cached| *cached = Some(saved.clone())); + let (dropped, reader) = observe_cache_drop(&saved, &execution_cache, false); + let ctx = test_prewarm_context(saved, Gauge::noop()); + let (task, actions_tx) = + PrewarmCacheTask::new(runtime.clone(), execution_cache.clone(), ctx); + let mut payload = PayloadHandle { + prewarm_handle: CacheTaskHandle { + saved_cache: task.ctx.saved_cache.clone(), + to_prewarm_task: Some(actions_tx.clone()), + executed_tx_index: task.ctx.executed_tx_index.clone(), + cache_metrics: None, + }, + transactions: crossbeam_channel::never::<(usize, Result<(), ()>)>(), + _span: Span::none(), + }; + // The validator keeps this ExecutionCache clone after calling terminate_caching. + let validator_cache = payload.caches().unwrap(); + let valid_block_tx = payload.terminate_caching(Some(Arc::new(BlockExecutionOutput { + state: Default::default(), + result: Default::default(), + }))); + drop(valid_block_tx); + let prewarm = runtime.spawn_blocking_named("prewarm", move || { + task.run::, Recovered>>( + PrewarmMode::Skipped, + actions_tx, + ); + std::thread::current().id() + }); + let prewarm_thread = *prewarm.get(); + + execution_cache.update_with_guard(|cached| assert!(cached.is_none())); + assert_eq!(payload.prewarm_handle.saved_cache.as_ref().unwrap().usage_count(), 2); + assert!(matches!(dropped.try_recv(), Err(mpsc::TryRecvError::Empty))); + drop(payload); + assert!(matches!(dropped.try_recv(), Err(mpsc::TryRecvError::Empty))); + + drop(validator_cache); + assert!(dropped.try_recv().expect("last ExecutionCache drop must free the cache")); + let drop_thread = reader.join().unwrap(); + assert_eq!(drop_thread, std::thread::current().id()); + assert_ne!(drop_thread, prewarm_thread); + } + + fn assert_save_cache_drops_removed_caches(slot: CacheSlot, valid: bool, insert_error: bool) { + use reth_revm::db::{AccountStatus, BundleAccount, BundleState}; + + let runtime = Runtime::test(); + let execution_cache = PayloadExecutionCache::default(); + let cache_to_save = + SavedCache::new(B256::repeat_byte(1), crate::tree::ExecutionCache::new(1_000)); + let distinct_previous = matches!(slot, CacheSlot::Distinct).then(|| { + // The same block hash does not imply the same allocation. + SavedCache::new(B256::repeat_byte(2), crate::tree::ExecutionCache::new(1_000)) + }); + execution_cache.update_with_guard(|cached| { + *cached = match slot { + CacheSlot::Empty => None, + CacheSlot::Shared => Some(cache_to_save.clone()), + CacheSlot::Distinct => distinct_previous.clone(), + }; + }); + let expect_saved_cache = valid && !insert_error; + let mut drops = Vec::new(); + if let Some(previous) = &distinct_previous { + drops.push(observe_cache_drop(previous, &execution_cache, expect_saved_cache)); + } + if !expect_saved_cache { + drops.push(observe_cache_drop(&cache_to_save, &execution_cache, expect_saved_cache)); + } + // Retaining this SavedCache would prevent save_cache from freeing the old cache. + drop(distinct_previous); + + // Cleanup must finish even when the shared background drop worker is occupied. + let (release_tx, release_rx) = mpsc::channel::<()>(); + runtime.spawn_blocking_named("drop", move || { + let _ = release_rx.recv(); + }); + + let mut state = BundleState::default(); + if insert_error { + // Modified accounts without current info are rejected by insert_state. + state.state.insert( + address!("0000000000000000000000000000000000000001"), + BundleAccount::new(None, None, Default::default(), AccountStatus::Changed), + ); + } + save_test_cache(&runtime, &execution_cache, cache_to_save, state, valid, Gauge::noop()); + + for (result, reader) in drops { + assert!( + result.try_recv().expect("removed cache must be destroyed before save returns"), + "cache mutex must be unlocked during destruction" + ); + assert_eq!(reader.join().unwrap(), std::thread::current().id()); + } + execution_cache.update_with_guard(|slot| { + if expect_saved_cache { + let saved = slot.as_ref().expect("valid cache saved"); + assert_eq!(saved.executed_block_hash(), B256::repeat_byte(2)); + } else { + assert!(slot.is_none(), "polluted cache must be removed"); + } + }); + assert_eq!( + execution_cache.get_cache_for(B256::repeat_byte(2)).is_some(), + expect_saved_cache + ); + drop(release_tx); + } + + #[test] + fn save_cache_drops_replaced_allocation_after_unlock() { + assert_save_cache_drops_removed_caches(CacheSlot::Distinct, true, false); + } + + #[test] + fn save_cache_drops_invalid_allocations_after_unlock() { + assert_save_cache_drops_removed_caches(CacheSlot::Distinct, false, false); + } + + #[test] + fn save_cache_drops_allocations_after_unlock_on_insert_error() { + assert_save_cache_drops_removed_caches(CacheSlot::Distinct, true, true); + } + + #[test] + fn save_cache_drops_shared_allocation_after_unlock_on_invalid_block() { + assert_save_cache_drops_removed_caches(CacheSlot::Shared, false, false); + } + + #[test] + fn save_cache_drops_shared_allocation_after_unlock_on_insert_error() { + assert_save_cache_drops_removed_caches(CacheSlot::Shared, true, true); + } + + #[test] + fn save_cache_handles_empty_slot() { + for (valid, insert_error) in [(true, false), (false, false), (true, true)] { + assert_save_cache_drops_removed_caches(CacheSlot::Empty, valid, insert_error); + } + } + #[test] fn bal_read_only_account_does_not_change_state_root() { let changes = AccountChanges::new(address!("0000000000000000000000000000000000000001")) @@ -902,7 +1244,8 @@ pub struct PrewarmMetrics { pub(crate) execution_duration: Histogram, /// A histogram for prefetch targets per transaction prewarming pub(crate) prefetch_storage_targets: Histogram, - /// A histogram of duration for cache saving + /// Time spent in `save_cache`, including dropping its removed `SavedCache` values. + /// Excludes any later freeing of cache contents by other `ExecutionCache` clones. pub(crate) cache_saving_duration: Gauge, /// Counter for transaction execution errors during prewarming pub(crate) transaction_errors: Counter, diff --git a/crates/rpc/rpc-api/src/reth_engine.rs b/crates/rpc/rpc-api/src/reth_engine.rs index b8f52a2f8fb..d95f67e6647 100644 --- a/crates/rpc/rpc-api/src/reth_engine.rs +++ b/crates/rpc/rpc-api/src/reth_engine.rs @@ -100,8 +100,9 @@ where /// waiting for persistence and cache locks before processing. /// /// By default, the endpoint waits for both in-flight persistence and cache updates to complete -/// before executing the payload, providing unbiased timing measurements. Each can be independently -/// disabled via `wait_for_persistence` and `wait_for_caches`. +/// before executing the payload. Execution-cache allocations removed by those updates may still +/// be destroyed concurrently after unlocking; their cleanup is excluded from cache wait timings. +/// Each wait can be independently disabled via `wait_for_persistence` and `wait_for_caches`. /// /// Responses include timing breakdowns with server-measured execution latency. #[cfg_attr(not(feature = "client"), rpc(server, namespace = "reth"))] From d64c403e732fd0589b1a34544269e1a994e30f83 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A9a=20Narzis?= <78718413+lean-apple@users.noreply.github.com> Date: Tue, 15 Sep 2026 09:19:19 +0000 Subject: [PATCH 262/294] feat(snap-sync): download and persist contract storage incrementally (#27236) --- crates/snap-sync/src/account/download.rs | 10 + crates/snap-sync/src/account/store.rs | 155 ++++++-- crates/snap-sync/src/error.rs | 20 +- crates/snap-sync/src/lib.rs | 8 +- crates/snap-sync/src/storage/download.rs | 408 +++++++++++++++++++++ crates/snap-sync/src/storage/mod.rs | 8 + crates/snap-sync/src/storage/store.rs | 430 +++++++++++++++++++++++ crates/snap-sync/src/test_utils.rs | 90 ++++- 8 files changed, 1091 insertions(+), 38 deletions(-) create mode 100644 crates/snap-sync/src/storage/download.rs create mode 100644 crates/snap-sync/src/storage/mod.rs create mode 100644 crates/snap-sync/src/storage/store.rs diff --git a/crates/snap-sync/src/account/download.rs b/crates/snap-sync/src/account/download.rs index 04f81c9bf29..ab8233f6914 100644 --- a/crates/snap-sync/src/account/download.rs +++ b/crates/snap-sync/src/account/download.rs @@ -175,6 +175,16 @@ pub struct VerifiedRange { } impl VerifiedRange { + #[cfg(test)] + pub(crate) const fn new(write: SnapWrite, range: VerifiedAccountRange) -> Self { + Self { write, range } + } + + /// Write the attempt accepted when the range was requested. + pub(crate) const fn write(&self) -> SnapWrite { + self.write + } + /// Key the range was requested from. pub const fn origin(&self) -> B256 { self.range.origin() diff --git a/crates/snap-sync/src/account/store.rs b/crates/snap-sync/src/account/store.rs index d6df887edf9..3caa812ed50 100644 --- a/crates/snap-sync/src/account/store.rs +++ b/crates/snap-sync/src/account/store.rs @@ -2,10 +2,13 @@ //! //! Each range replaces its key interval, and ranges commit in key order. -use crate::{SnapAttemptStore, SnapSyncError, SnapWrite}; +use crate::{ + storage::persisted_storage_root, SnapAttemptStore, SnapStorageStore, SnapSyncError, SnapWrite, + StorageProgress, +}; use alloy_primitives::{ map::{B256Map, B256Set}, - B256, KECCAK256_EMPTY, + B256, KECCAK256_EMPTY, U256, }; use reth_db_api::{ tables, @@ -46,7 +49,8 @@ pub trait SnapAccountStore { /// Persists `range` with its storage and code, replacing its key interval. /// - /// Storage must match each account's root, and code must be supplied or already stored. + /// Storage must match each account's root, supplied or persisted ahead of the range by + /// [`SnapStorageStore`], and code must be supplied or already stored. fn commit_account_range( &self, write: SnapWrite, @@ -56,6 +60,16 @@ pub trait SnapAccountStore { ) -> Result where Self: MetadataWriter + StateWriter + DBProvider; + + /// Removes storage in `interval` except the contracts in `kept`, which are sorted and inside + /// it. + fn remove_storages_except( + &self, + interval: (Bound, Bound), + kept: &[B256], + ) -> Result<(), SnapSyncError> + where + Self: DBProvider; } /// How far the account key space has been downloaded. @@ -175,16 +189,37 @@ impl SnapAccountStore for T { } let coverage = self.account_coverage(write)?.ok_or(SnapSyncError::NoCoverage)?; let advanced = coverage.advance(range)?; + let progress = self.storage_progress(write, range.origin())?; let dependencies = RangeDependencies::new(range.accounts(), storages, bytecodes); - dependencies.verify(range.accounts(), self.tx_ref())?; + let persisted = dependencies.verify(range.accounts(), &progress, self.tx_ref())?; let interval = coverage.interval_to(&advanced); self.remove::(interval)?; - self.remove::(interval)?; + self.remove_storages_except(interval, &persisted)?; dependencies.write(self)?; self.write_metadata(COVERAGE_KEY, StoredCoverage::encode(write.attempt(), advanced)?)?; Ok(advanced) } + + fn remove_storages_except( + &self, + (mut start, end): (Bound, Bound), + kept: &[B256], + ) -> Result<(), SnapSyncError> + where + Self: DBProvider, + { + for account in kept { + self.remove::((start, Bound::Excluded(*account)))?; + // Walks cannot start at an excluded key, so the next piece starts one key past it. + let Some(after) = U256::from_be_bytes(account.0).checked_add(U256::from(1)) else { + return Ok(()) + }; + start = Bound::Included(after.into()); + } + self.remove::((start, end))?; + Ok(()) + } } // Accounts of a range with the storage and code supplied for them. @@ -211,16 +246,22 @@ impl RangeDependencies { } // Checks storage against each account's root, and code against its hash or the table. + // Returns the contracts whose storage was persisted ahead of the range, in key order. fn verify( &self, accounts: &[(B256, TrieAccount)], + progress: &StorageProgress, tx: &impl DbTx, - ) -> Result<(), SnapSyncError> { + ) -> Result, SnapSyncError> { let mut available = self.supplied_code()?; let mut contracts = B256Set::default(); + let mut persisted = Vec::new(); for (hash, account) in accounts { - if self.verify_storage(*hash, account)? { + if account.storage_root != EMPTY_ROOT_HASH { contracts.insert(*hash); + if self.verify_storage(*hash, account, progress, tx)? { + persisted.push(*hash); + } } // Only presence matters, so stored code is not decoded. if account.code_hash != KECCAK256_EMPTY && @@ -235,7 +276,7 @@ impl RangeDependencies { { return Err(SnapSyncError::UnexpectedStorage { account: *account }) } - Ok(()) + Ok(persisted) } // Hashes of the supplied code, refusing code filed under a hash it does not hash to. @@ -251,20 +292,30 @@ impl RangeDependencies { Ok(hashes) } - // Whether `account` has storage, which must be supplied and hash to its root. - fn verify_storage(&self, hash: B256, account: &TrieAccount) -> Result { - if account.storage_root == EMPTY_ROOT_HASH { - return Ok(false) - } - let storage = self - .state - .account_storages() - .get(&hash) - .ok_or(SnapSyncError::MissingStorage { account: hash })?; - // Zero slots are deletions, which the trie does not hold. - let got = storage_root( - storage.storage_slots_ref().iter().filter(|(_, value)| !value.is_zero()).copied(), - ); + // Checks the contract's storage, supplied or persisted, against its root. Returns whether it + // was persisted. + fn verify_storage( + &self, + hash: B256, + account: &TrieAccount, + progress: &StorageProgress, + tx: &impl DbTx, + ) -> Result { + let (got, persisted) = match self.state.account_storages().get(&hash) { + // Zero slots are deletions, which the trie does not hold. + Some(storage) => ( + storage_root( + storage + .storage_slots_ref() + .iter() + .filter(|(_, value)| !value.is_zero()) + .copied(), + ), + false, + ), + None if progress.is_complete(hash) => (persisted_storage_root(tx, hash)?, true), + None => return Err(SnapSyncError::MissingStorage { account: hash }), + }; if got != account.storage_root { return Err(SnapSyncError::StorageRootMismatch { account: hash, @@ -272,7 +323,7 @@ impl RangeDependencies { got, }) } - Ok(true) + Ok(persisted) } // Writes the accounts with their storage and code. @@ -291,7 +342,7 @@ mod tests { use super::*; use crate::{ test_utils::{account, generation, hashed_factory, key, state_root, verified_range}, - SnapGeneration, + SnapGeneration, StorageChunk, }; use alloy_primitives::{Bytes, U256}; use reth_db_api::cursor::DbCursorRO; @@ -624,6 +675,62 @@ mod tests { assert_eq!(reopened.account_coverage(write).unwrap(), Some(coverage)); } + #[test] + fn storage_persisted_ahead_of_the_range_commits_with_it() { + let accounts = accounts(); + let (factory, write, _) = started(&accounts); + let range = verified_range(&accounts, 0..3, B256::ZERO, &[]); + let (_, bytecodes) = dependencies(); + let provider = factory.database_provider_rw().unwrap(); + // Left behind for an account that has no storage at this root. + let leftover = HashedPostState::default().with_storages([(key(1), storage().1)]); + provider.write_hashed_state(&leftover.into_sorted()).unwrap(); + let slots = vec![(SLOT, U256::from(7))]; + let chunk = StorageChunk::new(key(2), storage().0, B256::ZERO, slots, None); + provider.commit_storage_chunk(write, B256::ZERO, chunk).unwrap(); + + let coverage = + provider.commit_account_range(write, &range, Default::default(), bytecodes).unwrap(); + + assert!(coverage.is_complete()); + assert_eq!(stored(&provider), (vec![key(1), key(2), FAR], true, true)); + assert!(provider.tx_ref().get::(key(1)).unwrap().is_none()); + } + + #[test] + fn persisted_storage_must_be_complete_and_match_the_account_root() { + let accounts = accounts(); + let (factory, write, _) = started(&accounts); + let range = verified_range(&accounts, 0..3, B256::ZERO, &[]); + let (_, bytecodes) = dependencies(); + + // Part way through the contract. + let provider = factory.database_provider_rw().unwrap(); + let slots = vec![(SLOT, U256::from(7))]; + let next = Some(B256::repeat_byte(0x66)); + let partial = StorageChunk::new(key(2), storage().0, B256::ZERO, slots, next); + provider.commit_storage_chunk(write, B256::ZERO, partial).unwrap(); + let incomplete = + provider.commit_account_range(write, &range, Default::default(), bytecodes.clone()); + assert!( + matches!(incomplete, Err(SnapSyncError::MissingStorage { account }) if account == key(2)) + ); + drop(provider); + + // Complete, but not the storage the account commits to. + let provider = factory.database_provider_rw().unwrap(); + let slots = vec![(SLOT, U256::from(8))]; + let wrong = StorageChunk::new(key(2), storage().0, B256::ZERO, slots, None); + provider.commit_storage_chunk(write, B256::ZERO, wrong).unwrap(); + let mismatched = + provider.commit_account_range(write, &range, Default::default(), bytecodes); + assert!(matches!( + mismatched, + Err(SnapSyncError::StorageRootMismatch { account, .. }) if account == key(2) + )); + assert_eq!(provider.account_coverage(write).unwrap(), Some(AccountCoverage::START)); + } + #[test] fn a_record_this_build_cannot_read_is_reported() { let accounts = accounts(); diff --git a/crates/snap-sync/src/error.rs b/crates/snap-sync/src/error.rs index f99cb412d78..5c61947bcb5 100644 --- a/crates/snap-sync/src/error.rs +++ b/crates/snap-sync/src/error.rs @@ -1,6 +1,7 @@ //! Failures raised while assembling a snap state generation. use alloy_primitives::B256; +use reth_downloaders::snap::InvalidStorageRangeRequest; use reth_network_p2p::error::RequestError; use reth_storage_api::SnapAttemptId; use reth_storage_errors::{db::DatabaseError, provider::ProviderError}; @@ -14,6 +15,9 @@ pub enum SnapSyncError { /// A request for state failed. #[error(transparent)] Request(#[from] RequestError), + /// A storage request did not match the accounts it was built from. + #[error(transparent)] + StorageRequest(#[from] InvalidStorageRangeRequest), /// The storage layout keys state by address, which snap cannot fill in without preimages. #[error("snap synchronization requires the hashed state layout")] UnsupportedStorage, @@ -54,11 +58,25 @@ pub enum SnapSyncError { got: B256, }, /// A verified range proved nothing past the key it was requested from. - #[error("account range made no progress past {origin}")] + #[error("range made no progress past {origin}")] NoProgress { /// Key the range was requested from. origin: B256, }, + /// Storage was downloaded from somewhere other than where its contract's progress continues. + #[error("storage for {account} from {from} does not continue its persisted progress")] + OutOfOrderStorage { + /// Hashed address of the contract. + account: B256, + /// Slot the storage was requested from. + from: B256, + }, + /// A persisted storage progress record this build cannot read. + #[error("storage progress record version {version:?} is not supported")] + UnsupportedStorageProgress { + /// Version found on disk, absent when the record carries no numeric version. + version: Option, + }, /// An account with storage was committed without it. #[error("account {account} has storage that was not downloaded")] MissingStorage { diff --git a/crates/snap-sync/src/lib.rs b/crates/snap-sync/src/lib.rs index 95ce4bcae03..8d0f70e23a3 100644 --- a/crates/snap-sync/src/lib.rs +++ b/crates/snap-sync/src/lib.rs @@ -12,7 +12,8 @@ //! Downloaded state goes into the hashed state tables, owned by an attempt record that commits //! with it, along with how far the account key space has been downloaded. An account range only //! commits with storage matching its accounts' roots and code matching their hashes, so committed -//! progress never depends on work still pending. +//! progress never depends on work still pending. Storage too large for one response is persisted +//! ahead of its range, under progress tied to the attempt, pivot and range. //! //! ``` //! use reth_snap_sync::SnapPivotPolicy; @@ -42,6 +43,7 @@ mod error; mod generation; mod pivot; mod session; +mod storage; #[cfg(test)] mod test_utils; @@ -56,3 +58,7 @@ pub use error::SnapSyncError; pub use generation::{SnapGeneration, SnapPhase}; pub use pivot::SnapPivotPolicy; pub use session::{SnapSyncSession, SnapSyncSessionState}; +pub use storage::{ + SnapStorageStore, StorageChunk, StorageProgress, StorageRangeDownload, StorageRangeStep, + DEFAULT_STORAGE_ACCOUNTS, +}; diff --git a/crates/snap-sync/src/storage/download.rs b/crates/snap-sync/src/storage/download.rs new file mode 100644 index 00000000000..e3b092dbb05 --- /dev/null +++ b/crates/snap-sync/src/storage/download.rs @@ -0,0 +1,408 @@ +//! Downloads the storage of an account range's contracts, committing every verified response. + +use crate::{ + SnapStorageStore, SnapSyncError, SnapWrite, StorageChunk, StorageProgress, VerifiedRange, + DEFAULT_RESPONSE_BYTES, MAX_HASH, +}; +use alloy_primitives::B256; +use reth_db_api::transaction::DbTxMut; +use reth_downloaders::snap::{ + StorageRangeDownloader, StorageRangeOutcome, VerifiedAccountBatch, VerifiedStorageRanges, +}; +use reth_eth_wire_types::snap::GetStorageRangesMessage; +use reth_network_p2p::snap::client::SnapClient; +use reth_network_peers::PeerId; +use reth_storage_api::{ + DBProvider, DatabaseProviderFactory, MetadataProvider, MetadataWriter, StateWriter, +}; +use reth_storage_errors::provider::ProviderError; +use reth_tasks::Runtime; +use std::fmt; + +/// Default number of contracts asked for per storage request. +pub const DEFAULT_STORAGE_ACCOUNTS: usize = 128; + +/// Downloads the storage an account range's contracts still need, one request at a time. +/// +/// Every verified response is committed before the next request, so at most one response is held +/// however large a contract is, and a download resumes from the persisted progress. +pub struct StorageRangeDownload { + client: C, + factory: F, + // Proof verification and commits run on the blocking pool. + runtime: Runtime, + response_bytes: u64, + // Contracts asked for per request. + max_accounts: usize, + // Distinguishes responses to reissued requests. + request_id: u64, +} + +impl StorageRangeDownload { + /// Creates a download that continues from the progress the store records. + pub const fn new(client: C, factory: F, runtime: Runtime) -> Self { + Self { + client, + factory, + runtime, + response_bytes: DEFAULT_RESPONSE_BYTES, + max_accounts: DEFAULT_STORAGE_ACCOUNTS, + request_id: 0, + } + } + + /// Returns this download asking peers for at most `response_bytes` per response. + pub const fn with_response_bytes(mut self, response_bytes: u64) -> Self { + self.response_bytes = response_bytes; + self + } + + /// Returns this download asking for at most `max_accounts` contracts per request, at least one. + pub const fn with_max_accounts(mut self, max_accounts: usize) -> Self { + self.max_accounts = if max_accounts == 0 { 1 } else { max_accounts }; + self + } +} + +impl StorageRangeDownload +where + C: SnapClient + Clone + Unpin, + F: DatabaseProviderFactory + Clone + 'static, + F::Provider: MetadataProvider, + F::ProviderRW: MetadataProvider + MetadataWriter + StateWriter + DBProvider, +{ + /// Requests storage `range` still needs and commits the verified response. + /// + /// [`StorageRangeStep::Complete`] once every contract in the range is persisted, so the range + /// can commit without supplying their storage. A failed request leaves the progress in place. + pub async fn next(&mut self, range: &VerifiedRange) -> Result { + let (write, origin) = (range.write(), range.origin()); + let progress = self.factory.database_provider_ro()?.storage_progress(write, origin)?; + let contracts = range.range().storage_batch(); + let Some(first) = + contracts.accounts().iter().position(|(account, _)| !progress.is_complete(*account)) + else { + return Ok(StorageRangeStep::Complete) + }; + let end = contracts.accounts().len().min(first.saturating_add(self.max_accounts)); + let batch = contracts.range(first..end).expect("positions are inside the batch"); + let from = progress.resume_at(batch.accounts()[0].0).expect("first contract is incomplete"); + + self.request_id = self.request_id.wrapping_add(1); + let request = GetStorageRangesMessage { + request_id: self.request_id, + root_hash: batch.state_root(), + account_hashes: batch.accounts().iter().map(|(account, _)| *account).collect(), + starting_hash: from.into(), + limit_hash: MAX_HASH.into(), + response_bytes: self.response_bytes, + }; + let downloader = StorageRangeDownloader::new( + self.client.clone(), + request, + &batch, + self.runtime.clone(), + )?; + let ranges = match downloader.await? { + StorageRangeOutcome::Verified(ranges) => ranges, + StorageRangeOutcome::Unavailable { peer_id } => { + return Ok(StorageRangeStep::Unavailable { peer_id }) + } + }; + let chunks = chunks(ranges, batch, from)?; + self.commit(write, origin, chunks).await.map(StorageRangeStep::Committed) + } + + // Commits one response's chunks together. + async fn commit( + &self, + write: SnapWrite, + origin: B256, + chunks: Vec, + ) -> Result { + let factory = self.factory.clone(); + self.runtime + .spawn_blocking(move || -> Result { + let provider = factory.database_provider_rw()?; + let mut progress = StorageProgress::START; + for chunk in chunks { + progress = provider.commit_storage_chunk(write, origin, chunk)?; + } + provider.commit()?; + Ok(progress) + }) + .await + .map_err(|error| SnapSyncError::Provider(ProviderError::other(error)))? + } +} + +impl fmt::Debug for StorageRangeDownload { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("StorageRangeDownload") + .field("response_bytes", &self.response_bytes) + .field("max_accounts", &self.max_accounts) + .field("request_id", &self.request_id) + .finish_non_exhaustive() + } +} + +/// What one request of a [`StorageRangeDownload`] produced. +#[derive(Debug)] +pub enum StorageRangeStep { + /// A response was committed, reaching this progress. + Committed(StorageProgress), + /// The peer did not serve the storage, so the progress stays where it was. + Unavailable { + /// Peer that answered without the state, so the retry can go elsewhere. + peer_id: PeerId, + }, + /// Every contract in the range has its storage persisted. + Complete, +} + +// Splits a response into per-contract chunks. Only the last contract returned can be part way +// through, and it continues where a follow-up request would resume. +fn chunks( + ranges: VerifiedStorageRanges, + batch: VerifiedAccountBatch<'_>, + from: B256, +) -> Result, SnapSyncError> { + let roots: Vec = + batch.accounts().iter().map(|(_, account)| account.storage_root).collect(); + let resume = ranges.follow_up(0, batch)?.map(|(request, _)| { + (request.account_hashes[0], request.starting_hash.unwrap_or(B256::ZERO)) + }); + Ok(ranges + .into_ranges() + .into_iter() + .zip(roots) + .enumerate() + .map(|(index, (range, storage_root))| { + // Contracts after the first are requested whole. + let from = if index == 0 { from } else { B256::ZERO }; + let next = + resume.filter(|(account, _)| *account == range.account_hash).map(|(_, slot)| slot); + StorageChunk::new(range.account_hash, storage_root, from, range.slots, next) + }) + .collect()) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{ + test_utils::{ + account, generation, hashed_factory, key, state_root, storage_ranges, storage_root_of, + stored_slots, verified_range, ScriptedSnapClient, + }, + SnapAccountStore, SnapAttemptStore, + }; + use alloy_primitives::U256; + use reth_eth_wire_types::snap::StorageRangesMessage; + use reth_network_p2p::{error::PeerRequestResult, snap::client::SnapResponse}; + use reth_network_peers::WithPeerId; + use reth_provider::{test_utils::MockNodeTypesWithDB, ProviderFactory}; + use reth_trie_common::TrieAccount; + use std::sync::Arc; + + const FAR: B256 = B256::repeat_byte(0xaa); + + type Factory = ProviderFactory; + type Download = StorageRangeDownload, Factory>; + + // Storage served over several responses. + fn large() -> Vec<(B256, U256)> { + vec![(key(1), U256::from(11)), (key(2), U256::from(12)), (key(3), U256::from(13))] + } + + fn small() -> Vec<(B256, U256)> { + vec![(key(9), U256::from(19))] + } + + fn contract(nonce: u64, slots: &[(B256, U256)]) -> TrieAccount { + let mut contract = account(nonce); + contract.storage_root = storage_root_of(slots); + contract + } + + // A plain account, a large and a small contract, and a far plain account. + fn accounts() -> Vec<(B256, TrieAccount)> { + vec![ + (key(1), account(1)), + (key(2), contract(2, &large())), + (key(3), contract(3, &small())), + (FAR, account(4)), + ] + } + + // An attempt that fetched all of `accounts` as one range, not yet committed. + fn started(accounts: &[(B256, TrieAccount)]) -> (Factory, VerifiedRange) { + let factory = hashed_factory(); + let provider = factory.database_provider_rw().unwrap(); + let write = provider.start_snap_attempt(generation(1, state_root(accounts))).unwrap(); + provider.start_account_coverage(write).unwrap(); + provider.commit().unwrap(); + let range = verified_range(accounts, 0..accounts.len(), B256::ZERO, &[]); + (factory, VerifiedRange::new(write, range)) + } + + fn download( + responses: impl IntoIterator>, + factory: Factory, + ) -> (Arc, Download) { + let client = Arc::new(ScriptedSnapClient::new(responses)); + (Arc::clone(&client), StorageRangeDownload::new(client, factory, Runtime::test())) + } + + async fn committed(download: &mut Download, range: &VerifiedRange) -> StorageProgress { + match download.next(range).await.unwrap() { + StorageRangeStep::Committed(progress) => progress, + step => panic!("expected a commit, got {step:?}"), + } + } + + fn slots_of(factory: &Factory, account: B256) -> Vec<(B256, U256)> { + stored_slots(&factory.database_provider_ro().unwrap(), account) + } + + #[tokio::test] + async fn a_contract_spanning_responses_is_committed_response_by_response() { + let accounts = accounts(); + let (factory, range) = started(&accounts); + let (large, small) = (large(), small()); + let responses = [ + // The large contract's first slot; the proof shows more follow. + storage_ranges(1, &[&large[..1]], &large, &[B256::ZERO, key(1)]), + // The rest of it, without the small contract. + storage_ranges(2, &[&large[1..]], &large, &[key(2), key(3)]), + storage_ranges(3, &[&small[..]], &small, &[]), + ]; + let (client, mut download) = download(responses, factory.clone()); + + let progress = committed(&mut download, &range).await; + assert_eq!(progress.resume_at(key(2)), Some(key(2))); + assert_eq!(slots_of(&factory, key(2)), large[..1]); + + assert!(committed(&mut download, &range).await.is_complete(key(2))); + assert!(committed(&mut download, &range).await.is_complete(key(3))); + assert!(matches!(download.next(&range).await.unwrap(), StorageRangeStep::Complete)); + assert_eq!( + *client.storage_requests(), + [ + (vec![key(2), key(3)], B256::ZERO), + (vec![key(2), key(3)], key(2)), + (vec![key(3)], B256::ZERO), + ] + ); + + // The range commits against the persisted storage, none of it supplied in memory. + let provider = factory.database_provider_rw().unwrap(); + let coverage = provider + .commit_account_range(range.write(), range.range(), Default::default(), Vec::new()) + .unwrap(); + provider.commit().unwrap(); + assert!(coverage.is_complete()); + assert_eq!(slots_of(&factory, key(2)), large); + assert_eq!(slots_of(&factory, key(3)), small); + } + + #[tokio::test] + async fn an_unavailable_response_leaves_the_progress_in_place() { + let accounts = accounts(); + let (factory, range) = started(&accounts); + let large = large(); + let peer = PeerId::random(); + let empty = StorageRangesMessage { request_id: 2, slots: Vec::new(), proof: Vec::new() }; + let responses = [ + storage_ranges(1, &[&large[..1]], &large, &[B256::ZERO, key(1)]), + Ok(WithPeerId::new(peer, SnapResponse::StorageRanges(empty))), + storage_ranges(3, &[&large[1..]], &large, &[key(2), key(3)]), + ]; + let (client, mut download) = download(responses, factory.clone()); + let progress = committed(&mut download, &range).await; + + let step = download.next(&range).await.unwrap(); + + assert!(matches!(step, StorageRangeStep::Unavailable { peer_id } if peer_id == peer)); + let provider = factory.database_provider_ro().unwrap(); + assert_eq!(provider.storage_progress(range.write(), range.origin()).unwrap(), progress); + drop(provider); + assert!(committed(&mut download, &range).await.is_complete(key(2))); + let origins: Vec<_> = client.storage_requests().iter().map(|(_, from)| *from).collect(); + assert_eq!(origins, [B256::ZERO, key(2), key(2)]); + } + + #[tokio::test] + async fn a_new_download_resumes_from_the_last_committed_response() { + let accounts = accounts(); + let (factory, range) = started(&accounts); + let (large, small) = (large(), small()); + let first = storage_ranges(1, &[&large[..1]], &large, &[B256::ZERO, key(1)]); + let (_, mut interrupted) = download([first], factory.clone()); + committed(&mut interrupted, &range).await; + drop(interrupted); + + let responses = [ + storage_ranges(1, &[&large[1..]], &large, &[key(2), key(3)]), + storage_ranges(2, &[&small[..]], &small, &[]), + ]; + let (client, mut resumed) = download(responses, factory.clone()); + committed(&mut resumed, &range).await; + committed(&mut resumed, &range).await; + + assert!(matches!(resumed.next(&range).await.unwrap(), StorageRangeStep::Complete)); + assert_eq!( + *client.storage_requests(), + [(vec![key(2), key(3)], key(2)), (vec![key(3)], B256::ZERO)] + ); + assert_eq!(slots_of(&factory, key(2)), large); + } + + #[tokio::test] + async fn requests_ask_for_at_most_the_configured_contracts() { + let accounts = accounts(); + let (factory, range) = started(&accounts); + let (large, small) = (large(), small()); + let responses = [ + storage_ranges(1, &[&large[..]], &large, &[]), + storage_ranges(2, &[&small[..]], &small, &[]), + ]; + let (client, download) = download(responses, factory); + let mut download = download.with_max_accounts(1); + + committed(&mut download, &range).await; + committed(&mut download, &range).await; + + assert!(matches!(download.next(&range).await.unwrap(), StorageRangeStep::Complete)); + assert_eq!( + *client.storage_requests(), + [(vec![key(2)], B256::ZERO), (vec![key(3)], B256::ZERO)] + ); + } + + #[tokio::test] + async fn a_range_without_contracts_requests_no_storage() { + let accounts = vec![(key(1), account(1)), (FAR, account(2))]; + let (factory, range) = started(&accounts); + let (client, mut download) = download([], factory); + + assert!(matches!(download.next(&range).await.unwrap(), StorageRangeStep::Complete)); + assert!(client.storage_requests().is_empty()); + } + + #[tokio::test] + async fn storage_fetched_before_the_pivot_moved_is_not_resumed() { + let accounts = accounts(); + let (factory, range) = started(&accounts); + let large = large(); + let first = storage_ranges(1, &[&large[..1]], &large, &[B256::ZERO, key(1)]); + let (client, mut download) = download([first], factory.clone()); + committed(&mut download, &range).await; + let provider = factory.database_provider_rw().unwrap(); + provider.advance_snap_pivot(range.write(), generation(2, state_root(&accounts))).unwrap(); + provider.commit().unwrap(); + + assert!(matches!(download.next(&range).await, Err(SnapSyncError::StaleWrite { .. }))); + assert_eq!(client.storage_requests().len(), 1); + } +} diff --git a/crates/snap-sync/src/storage/mod.rs b/crates/snap-sync/src/storage/mod.rs new file mode 100644 index 00000000000..976641b934f --- /dev/null +++ b/crates/snap-sync/src/storage/mod.rs @@ -0,0 +1,8 @@ +//! Contract storage download coordination and persistence. + +mod download; +mod store; + +pub use download::{StorageRangeDownload, StorageRangeStep, DEFAULT_STORAGE_ACCOUNTS}; +pub(crate) use store::persisted_storage_root; +pub use store::{SnapStorageStore, StorageChunk, StorageProgress}; diff --git a/crates/snap-sync/src/storage/store.rs b/crates/snap-sync/src/storage/store.rs new file mode 100644 index 00000000000..cd2b8311d20 --- /dev/null +++ b/crates/snap-sync/src/storage/store.rs @@ -0,0 +1,430 @@ +//! Persists contract storage response by response, ahead of the account range committing to it. +//! +//! Progress is tied to the attempt, its pivot and the range, so slots proved against a superseded +//! root are never resumed. + +use crate::{SnapAccountStore, SnapAttemptStore, SnapSyncError, SnapWrite}; +use alloy_primitives::{B256, U256}; +use reth_db_api::{ + cursor::DbDupCursorRO, + tables, + transaction::{DbTx, DbTxMut}, +}; +use reth_storage_api::{DBProvider, MetadataProvider, MetadataWriter, SnapAttemptId, StateWriter}; +use reth_storage_errors::provider::ProviderError; +use reth_trie_common::{root::storage_root, HashedPostState, HashedStorage, EMPTY_ROOT_HASH}; +use serde::{Deserialize, Serialize}; + +// Metadata key of the storage progress record. +const PROGRESS_KEY: &str = "snap_storage_progress"; + +// Encoding version of the storage progress record this build writes. +const PROGRESS_VERSION: u32 = 1; + +/// Persistence for contract storage downloaded ahead of its account range. +/// +/// Blanket-implemented over the node's writers, so slots and progress join the caller's +/// transaction and commit together or not at all. +pub trait SnapStorageStore { + /// Returns how far the storage of the range requested from `origin` is persisted. + fn storage_progress( + &self, + write: SnapWrite, + origin: B256, + ) -> Result; + + /// Persists `chunk` for the range requested from `origin`, where the coverage must continue. + /// + /// A contract starts at the zero slot, dropping whatever an earlier attempt left for it, and + /// otherwise continues only where its last chunk ended. + fn commit_storage_chunk( + &self, + write: SnapWrite, + origin: B256, + chunk: StorageChunk, + ) -> Result + where + Self: MetadataWriter + StateWriter + DBProvider; +} + +/// Verified slots of one contract, from the slot they were requested at. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct StorageChunk { + // Hashed address of the contract. + account: B256, + // Root the slots were proved against. + storage_root: B256, + // Slot the chunk was requested from. + from: B256, + // Non-zero slots in key order. + slots: Vec<(B256, U256)>, + // Slot the storage continues at, or none once the trie is exhausted. + next: Option, +} + +impl StorageChunk { + /// Creates a chunk of `account`'s storage requested from `from` and continuing at `next`. + pub const fn new( + account: B256, + storage_root: B256, + from: B256, + slots: Vec<(B256, U256)>, + next: Option, + ) -> Self { + Self { account, storage_root, from, slots, next } + } +} + +/// How far the contracts of the account range being downloaded have their storage persisted. +/// +/// Contracts complete in key order, so only one is ever part way through. +#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize, Deserialize)] +pub struct StorageProgress { + // Contracts up to this key have all their storage persisted. + complete: Option, + // The contract part way through, if any. + partial: Option, +} + +impl StorageProgress { + /// No storage persisted yet. + pub const START: Self = Self { complete: None, partial: None }; + + /// Returns whether all of `account`'s storage is persisted. + pub fn is_complete(&self, account: B256) -> bool { + self.complete.is_some_and(|complete| account <= complete) + } + + /// Slot `account`'s storage resumes at: zero before it starts, `None` once it is complete. + pub fn resume_at(&self, account: B256) -> Option { + if self.is_complete(account) { + return None + } + let partial = self.partial.filter(|partial| partial.account == account); + Some(partial.map_or(B256::ZERO, |partial| partial.next)) + } + + // The progress after `chunk`, which must continue its contract or start the next one. + fn advance(&self, chunk: &StorageChunk) -> Result { + let another = self.partial.is_some_and(|partial| partial.account != chunk.account); + if another || self.resume_at(chunk.account) != Some(chunk.from) { + return Err(SnapSyncError::OutOfOrderStorage { + account: chunk.account, + from: chunk.from, + }) + } + if let Some(partial) = self.partial && + partial.storage_root != chunk.storage_root + { + return Err(SnapSyncError::StorageRootMismatch { + account: chunk.account, + expected: partial.storage_root, + got: chunk.storage_root, + }) + } + if chunk.next.is_some_and(|next| next <= chunk.from) { + return Err(SnapSyncError::NoProgress { origin: chunk.from }) + } + Ok(match chunk.next { + Some(next) => Self { + complete: self.complete, + partial: Some(PartialStorage { + account: chunk.account, + storage_root: chunk.storage_root, + next, + }), + }, + None => Self { complete: Some(chunk.account), partial: None }, + }) + } +} + +// A contract whose storage is persisted up to, but not including, `next`. +#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize, Deserialize)] +struct PartialStorage { + // Hashed address of the contract. + account: B256, + // Root its slots were proved against. + storage_root: B256, + // Slot its storage resumes at. + next: B256, +} + +// The progress record as persisted, tied to the write and range that recorded it. +#[derive(Serialize, Deserialize)] +struct StoredProgress { + // Encoding version, checked before the rest is decoded. + version: u32, + // Attempt the slots belong to. + attempt: SnapAttemptId, + // Pivot generation the slots were proved against. + state_version: u64, + // Key the account range was requested from. + origin: B256, + // How far that range's contracts have got. + progress: StorageProgress, +} + +impl StoredProgress { + // Serializes `progress` for `write`'s range at `origin` at this build's version. + fn encode( + write: SnapWrite, + origin: B256, + progress: StorageProgress, + ) -> Result, SnapSyncError> { + let stored = Self { + version: PROGRESS_VERSION, + attempt: write.attempt(), + state_version: write.state_version(), + origin, + progress, + }; + Ok(serde_json::to_vec(&stored).map_err(ProviderError::other)?) + } + + // Checks the version first, so a record from another build is reported rather than misread. + fn decode(bytes: &[u8]) -> Result { + let value: serde_json::Value = + serde_json::from_slice(bytes).map_err(ProviderError::other)?; + let version = value.get("version").and_then(serde_json::Value::as_u64); + if version != Some(PROGRESS_VERSION as u64) { + return Err(SnapSyncError::UnsupportedStorageProgress { version }) + } + Ok(serde_json::from_value(value).map_err(ProviderError::other)?) + } + + fn belongs_to(&self, write: SnapWrite, origin: B256) -> bool { + self.attempt == write.attempt() && + self.state_version == write.state_version() && + self.origin == origin + } +} + +impl SnapStorageStore for T { + // A record left by another attempt, pivot or range reads as nothing persisted. + fn storage_progress( + &self, + write: SnapWrite, + origin: B256, + ) -> Result { + self.authorize_snap_write(write)?; + let Some(bytes) = self.get_metadata(PROGRESS_KEY)? else { + return Ok(StorageProgress::START) + }; + let stored = StoredProgress::decode(&bytes)?; + Ok(if stored.belongs_to(write, origin) { stored.progress } else { StorageProgress::START }) + } + + // Every check runs before the first write, so a refused chunk changes nothing. + fn commit_storage_chunk( + &self, + write: SnapWrite, + origin: B256, + chunk: StorageChunk, + ) -> Result + where + Self: MetadataWriter + StateWriter + DBProvider, + { + let coverage = self.account_coverage(write)?.ok_or(SnapSyncError::NoCoverage)?; + if coverage.next() != Some(origin) { + return Err(SnapSyncError::OutOfOrderRange { expected: coverage.next(), got: origin }) + } + if chunk.account < origin || chunk.storage_root == EMPTY_ROOT_HASH { + return Err(SnapSyncError::UnexpectedStorage { account: chunk.account }) + } + let progress = self.storage_progress(write, origin)?.advance(&chunk)?; + + if chunk.from == B256::ZERO { + self.remove::(chunk.account..=chunk.account)?; + } + let state = HashedPostState::default() + .with_storages([(chunk.account, HashedStorage::from_iter(chunk.slots))]) + .into_sorted(); + self.write_hashed_state(&state)?; + self.write_metadata(PROGRESS_KEY, StoredProgress::encode(write, origin, progress)?)?; + Ok(progress) + } +} + +/// Root of the storage persisted for `account`, streamed so a large contract is never held in +/// memory. +pub(crate) fn persisted_storage_root(tx: &impl DbTx, account: B256) -> Result { + let mut cursor = tx.cursor_dup_read::()?; + let mut failed = None; + let slots = cursor.walk_dup(Some(account), None)?.map_while(|entry| { + entry.map(|(_, slot)| (slot.key, slot.value)).map_err(|error| failed = Some(error)).ok() + }); + let root = storage_root(slots); + failed.map_or(Ok(root), |error| Err(error.into())) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::test_utils::{ + account, generation, hashed_factory, state_root, storage_root_of, stored_slots, + }; + use reth_provider::{ + test_utils::MockNodeTypesWithDB, DatabaseProviderFactory, ProviderFactory, + }; + use std::ops::Range; + + const CONTRACT: B256 = B256::repeat_byte(0x22); + + fn slot(value: u8) -> B256 { + B256::with_last_byte(value) + } + + fn slots() -> Vec<(B256, U256)> { + vec![(slot(1), U256::from(11)), (slot(2), U256::from(12)), (slot(3), U256::from(13))] + } + + fn root() -> B256 { + storage_root_of(&slots()) + } + + // `slots()[served]` requested from `from`, continuing at `next`. + fn chunk(served: Range, from: B256, next: Option) -> StorageChunk { + StorageChunk::new(CONTRACT, root(), from, slots()[served].to_vec(), next) + } + + // An attempt whose account coverage has not reached the contract yet. + fn started() -> (ProviderFactory, SnapWrite) { + let factory = hashed_factory(); + let provider = factory.database_provider_rw().unwrap(); + let mut contract = account(1); + contract.storage_root = root(); + let generation = generation(1, state_root(&[(CONTRACT, contract)])); + let write = provider.start_snap_attempt(generation).unwrap(); + provider.start_account_coverage(write).unwrap(); + provider.commit().unwrap(); + (factory, write) + } + + #[test] + fn an_interrupted_chunk_commit_keeps_the_last_committed_progress() { + let (factory, write) = started(); + let provider = factory.database_provider_rw().unwrap(); + let first = chunk(0..1, B256::ZERO, Some(slot(2))); + let progress = provider.commit_storage_chunk(write, B256::ZERO, first).unwrap(); + provider.commit().unwrap(); + assert_eq!(progress.resume_at(CONTRACT), Some(slot(2))); + + let provider = factory.database_provider_rw().unwrap(); + let rest = chunk(1..3, slot(2), None); + assert!(provider + .commit_storage_chunk(write, B256::ZERO, rest) + .unwrap() + .is_complete(CONTRACT)); + drop(provider); + + let provider = factory.database_provider_rw().unwrap(); + assert_eq!(provider.storage_progress(write, B256::ZERO).unwrap(), progress); + assert_eq!(stored_slots(&provider, CONTRACT), slots()[..1]); + } + + #[test] + fn a_chunk_must_continue_its_contract_before_another_starts() { + let (factory, write) = started(); + let provider = factory.database_provider_rw().unwrap(); + let first = chunk(0..1, B256::ZERO, Some(slot(2))); + let progress = provider.commit_storage_chunk(write, B256::ZERO, first).unwrap(); + + let refused = [ + // Restarted, skipping ahead, or another contract while this one is part way through. + chunk(0..1, B256::ZERO, Some(slot(2))), + chunk(2..3, slot(3), None), + StorageChunk::new(B256::repeat_byte(0x44), root(), B256::ZERO, slots(), None), + ]; + for chunk in refused { + assert!(matches!( + provider.commit_storage_chunk(write, B256::ZERO, chunk), + Err(SnapSyncError::OutOfOrderStorage { .. }) + )); + } + let stalled = chunk(1..1, slot(2), Some(slot(2))); + assert!(matches!( + provider.commit_storage_chunk(write, B256::ZERO, stalled), + Err(SnapSyncError::NoProgress { .. }) + )); + let other_root = StorageChunk::new( + CONTRACT, + B256::repeat_byte(0x33), + slot(2), + slots()[1..].to_vec(), + None, + ); + assert!(matches!( + provider.commit_storage_chunk(write, B256::ZERO, other_root), + Err(SnapSyncError::StorageRootMismatch { .. }) + )); + + assert_eq!(provider.storage_progress(write, B256::ZERO).unwrap(), progress); + assert_eq!(stored_slots(&provider, CONTRACT), slots()[..1]); + } + + #[test] + fn starting_a_contract_replaces_what_an_earlier_attempt_left() { + let (factory, write) = started(); + let provider = factory.database_provider_rw().unwrap(); + let leftover = HashedStorage::from_iter([(slot(7), U256::from(77))]); + let leftover = HashedPostState::default().with_storages([(CONTRACT, leftover)]); + provider.write_hashed_state(&leftover.into_sorted()).unwrap(); + + provider.commit_storage_chunk(write, B256::ZERO, chunk(0..3, B256::ZERO, None)).unwrap(); + + assert_eq!(stored_slots(&provider, CONTRACT), slots()); + assert_eq!(persisted_storage_root(provider.tx_ref(), CONTRACT).unwrap(), root()); + } + + #[test] + fn progress_belongs_to_the_pivot_and_range_it_was_recorded_for() { + let (factory, write) = started(); + let provider = factory.database_provider_rw().unwrap(); + let first = chunk(0..1, B256::ZERO, Some(slot(2))); + provider.commit_storage_chunk(write, B256::ZERO, first).unwrap(); + + assert_eq!(provider.storage_progress(write, slot(9)).unwrap(), StorageProgress::START); + + // Slots proved against the previous root are not resumed once the pivot moves. + let moved = generation(2, B256::repeat_byte(0xcc)); + let advanced = provider.advance_snap_pivot(write, moved).unwrap(); + assert_eq!( + provider.storage_progress(advanced, B256::ZERO).unwrap(), + StorageProgress::START + ); + assert!(matches!( + provider.commit_storage_chunk(write, B256::ZERO, chunk(1..3, slot(2), None)), + Err(SnapSyncError::StaleWrite { .. }) + )); + } + + #[test] + fn storage_outside_the_range_being_downloaded_is_refused() { + let (factory, write) = started(); + let provider = factory.database_provider_rw().unwrap(); + + let ahead = provider.commit_storage_chunk(write, slot(9), chunk(0..3, B256::ZERO, None)); + let empty = StorageChunk::new(CONTRACT, EMPTY_ROOT_HASH, B256::ZERO, Vec::new(), None); + let empty = provider.commit_storage_chunk(write, B256::ZERO, empty); + + assert!(matches!(ahead, Err(SnapSyncError::OutOfOrderRange { .. }))); + assert!(matches!(empty, Err(SnapSyncError::UnexpectedStorage { .. }))); + assert!(stored_slots(&provider, CONTRACT).is_empty()); + } + + #[test] + fn a_record_this_build_cannot_read_is_reported() { + let (factory, write) = started(); + + for record in [br#"{"version":999}"#.to_vec(), b"{}".to_vec()] { + let provider = factory.database_provider_rw().unwrap(); + provider.write_metadata(PROGRESS_KEY, record).unwrap(); + + assert!(matches!( + provider.storage_progress(write, B256::ZERO), + Err(SnapSyncError::UnsupportedStorageProgress { .. }) + )); + } + } +} diff --git a/crates/snap-sync/src/test_utils.rs b/crates/snap-sync/src/test_utils.rs index 5934dd8bbb2..5c7376d2058 100644 --- a/crates/snap-sync/src/test_utils.rs +++ b/crates/snap-sync/src/test_utils.rs @@ -8,10 +8,11 @@ use alloy_consensus::Header; use alloy_eips::BlockNumHash; use alloy_primitives::{Bytes, B256, KECCAK256_EMPTY, U256}; use futures::future::{ready, Ready}; +use reth_db_api::{cursor::DbDupCursorRO, tables, transaction::DbTx}; use reth_downloaders::snap::{AccountRangeDownloader, AccountRangeOutcome, VerifiedAccountRange}; use reth_eth_wire_types::snap::{ AccountData, AccountRangeMessage, GetAccountRangeMessage, GetBlockAccessListsMessage, - GetByteCodesMessage, GetStorageRangesMessage, + GetByteCodesMessage, GetStorageRangesMessage, StorageData, StorageRangesMessage, }; use reth_network_p2p::{ download::DownloadClient, @@ -26,7 +27,11 @@ use reth_provider::{ }; use reth_storage_api::{DBProvider, MetadataWriter, StorageSettings, StorageSettingsCache}; use reth_tasks::Runtime; -use reth_trie_common::{proof::ProofRetainer, HashBuilder, Nibbles, TrieAccount, EMPTY_ROOT_HASH}; +use reth_trie_common::{ + proof::ProofRetainer, + root::{state_root_unsorted, storage_root_unsorted}, + HashBuilder, Nibbles, TrieAccount, EMPTY_ROOT_HASH, +}; use std::{ collections::VecDeque, ops::Range, @@ -106,15 +111,30 @@ pub(crate) fn account(nonce: u64) -> TrieAccount { /// Root of the account trie holding `accounts`. pub(crate) fn state_root(accounts: &[(B256, TrieAccount)]) -> B256 { - root_and_proof(accounts, &[]).0 + state_root_unsorted(accounts.iter().copied()) } -// Root of the trie, and the proof nodes on the paths to `targets`. +// Root of the account trie, and the proof nodes on the paths to `targets`. fn root_and_proof(accounts: &[(B256, TrieAccount)], targets: &[B256]) -> (B256, Vec) { + trie(accounts.iter().map(|(key, account)| (*key, alloy_rlp::encode(account))), targets) +} + +/// Root of the storage trie holding `slots`. +pub(crate) fn storage_root_of(slots: &[(B256, U256)]) -> B256 { + storage_root_unsorted(slots.iter().copied()) +} + +// Root of the storage trie, and the proof nodes on the paths to `targets`. +fn storage_trie(slots: &[(B256, U256)], targets: &[B256]) -> (B256, Vec) { + trie(slots.iter().map(|(slot, value)| (*slot, alloy_rlp::encode(value))), targets) +} + +// Root of the trie holding `leaves` in key order, and the proof nodes on the paths to `targets`. +fn trie(leaves: impl IntoIterator)>, targets: &[B256]) -> (B256, Vec) { let targets = targets.iter().copied().map(Nibbles::unpack).collect(); let mut builder = HashBuilder::default().with_proof_retainer(ProofRetainer::new(targets)); - for (key, account) in accounts { - builder.add_leaf(Nibbles::unpack(*key), &alloy_rlp::encode(account)); + for (key, leaf) in leaves { + builder.add_leaf(Nibbles::unpack(key), &leaf); } let root = builder.root(); let proof = @@ -170,10 +190,44 @@ pub(crate) fn verified_range( } } +/// A peer's answer serving `ranges`, the last of which belongs to the storage trie holding `last` +/// and is proven along the paths to `proof_targets`. No targets means no proof. +pub(crate) fn storage_ranges( + request_id: u64, + ranges: &[&[(B256, U256)]], + last: &[(B256, U256)], + proof_targets: &[B256], +) -> PeerRequestResult { + let proof = + if proof_targets.is_empty() { Vec::new() } else { storage_trie(last, proof_targets).1 }; + let slots = ranges + .iter() + .map(|range| { + range.iter().map(|(slot, value)| StorageData::from_value(*slot, *value)).collect() + }) + .collect(); + let message = StorageRangesMessage { request_id, slots, proof }; + Ok(WithPeerId::new(PeerId::random(), SnapResponse::StorageRanges(message))) +} + +/// Slots persisted for `account`, in key order. +pub(crate) fn stored_slots(provider: &impl DBProvider, account: B256) -> Vec<(B256, U256)> { + let mut cursor = provider.tx_ref().cursor_dup_read::().unwrap(); + cursor + .walk_dup(Some(account), None) + .unwrap() + .map(|entry| { + let (_, slot) = entry.unwrap(); + (slot.key, slot.value) + }) + .collect() +} + /// Serves scripted answers in request order, recording what each request asked for. pub(crate) struct ScriptedSnapClient { responses: Mutex>>, origins: Mutex>, + storage_requests: Mutex, B256)>>, } impl ScriptedSnapClient { @@ -183,6 +237,7 @@ impl ScriptedSnapClient { Self { responses: Mutex::new(responses.into_iter().collect()), origins: Mutex::new(Vec::new()), + storage_requests: Mutex::new(Vec::new()), } } @@ -190,6 +245,16 @@ impl ScriptedSnapClient { pub(crate) fn origins(&self) -> MutexGuard<'_, Vec> { self.origins.lock().unwrap() } + + /// Accounts and starting slot of the storage range requests sent so far. + pub(crate) fn storage_requests(&self) -> MutexGuard<'_, Vec<(Vec, B256)>> { + self.storage_requests.lock().unwrap() + } + + fn next_response(&self) -> Ready> { + let response = self.responses.lock().unwrap().pop_front(); + ready(response.unwrap_or(Err(RequestError::ChannelClosed))) + } } impl std::fmt::Debug for ScriptedSnapClient { @@ -215,20 +280,21 @@ impl SnapClient for ScriptedSnapClient { _priority: Priority, ) -> Self::Output { self.origins.lock().unwrap().push(request.starting_hash); - let response = self.responses.lock().unwrap().pop_front(); - ready(response.unwrap_or(Err(RequestError::ChannelClosed))) + self.next_response() } - fn get_storage_ranges(&self, _request: GetStorageRangesMessage) -> Self::Output { - unsupported() + fn get_storage_ranges(&self, request: GetStorageRangesMessage) -> Self::Output { + self.get_storage_ranges_with_priority(request, Priority::Normal) } fn get_storage_ranges_with_priority( &self, - _request: GetStorageRangesMessage, + request: GetStorageRangesMessage, _priority: Priority, ) -> Self::Output { - unsupported() + let from = request.starting_hash.unwrap_or(B256::ZERO); + self.storage_requests.lock().unwrap().push((request.account_hashes, from)); + self.next_response() } fn get_byte_codes(&self, _request: GetByteCodesMessage) -> Self::Output { From 01d039b1f1a0f761a8217cf899e4d8f0781f7372 Mon Sep 17 00:00:00 2001 From: joshieDo <93316087+joshieDo@users.noreply.github.com> Date: Tue, 15 Sep 2026 09:35:55 +0000 Subject: [PATCH 263/294] fix(provider): serialize static file cache publication (#27223) Co-authored-by: Arsenii Kulikov <62447812+klkvr@users.noreply.github.com> Co-authored-by: Derek Cofausper <256792747+decofe@users.noreply.github.com> Co-authored-by: Codex --- .../src/providers/static_file/manager.rs | 209 +++++++++++++++++- 1 file changed, 202 insertions(+), 7 deletions(-) diff --git a/crates/storage/provider/src/providers/static_file/manager.rs b/crates/storage/provider/src/providers/static_file/manager.rs index 46bbff3d800..bef0e1421f4 100644 --- a/crates/storage/provider/src/providers/static_file/manager.rs +++ b/crates/storage/provider/src/providers/static_file/manager.rs @@ -55,7 +55,10 @@ use std::{ fmt::Debug, ops::{Bound, Deref, Range, RangeBounds, RangeInclusive}, path::{Path, PathBuf}, - sync::{atomic::AtomicU64, mpsc, Arc}, + sync::{ + atomic::{AtomicU64, Ordering}, + mpsc, Arc, + }, }; use tracing::{debug, info, info_span, instrument, trace, warn}; @@ -268,12 +271,43 @@ impl Deref for StaticFileProvider { } } +#[cfg(test)] +type BeforeCacheInsert = Box; + +#[cfg(test)] +#[derive(Default)] +struct BeforeCacheInsertHook(parking_lot::Mutex>); + +#[cfg(test)] +impl Debug for BeforeCacheInsertHook { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("BeforeCacheInsertHook").finish_non_exhaustive() + } +} + +#[cfg(test)] +impl BeforeCacheInsertHook { + fn set(&self, hook: impl FnOnce(StaticFileSegment) + Send + 'static) { + let previous = self.0.lock().replace(Box::new(hook)); + assert!(previous.is_none(), "cache fill hook already installed"); + } + + fn fire(&self, segment: StaticFileSegment) { + if let Some(hook) = self.0.lock().take() { + hook(segment); + } + } +} + /// [`StaticFileProviderInner`] manages all existing [`StaticFileJarProvider`]. #[derive(Debug)] pub struct StaticFileProviderInner { /// Maintains a map which allows for concurrent access to different `NippyJars`, over different /// segments and ranges. map: DashMap<(BlockNumber, StaticFileSegment), LoadedJar>, + /// Incremented before clearing the jar cache so in-flight loads cannot repopulate it with a + /// snapshot from before the invalidation. + cache_generation: AtomicU64, /// Indexes per segment. indexes: RwLock>, /// This is an additional index that tracks the expired height, this will track the highest @@ -301,6 +335,10 @@ pub struct StaticFileProviderInner { _lock_file: Option, /// Genesis block number, default is 0; genesis_block_number: u64, + #[cfg(test)] + before_cache_insert: BeforeCacheInsertHook, + #[cfg(test)] + after_cache_validation: BeforeCacheInsertHook, } impl StaticFileProviderInner { @@ -319,6 +357,7 @@ impl StaticFileProviderInner { let provider = Self { map: Default::default(), + cache_generation: Default::default(), indexes: Default::default(), writers: Default::default(), earliest_history_height: Default::default(), @@ -328,6 +367,10 @@ impl StaticFileProviderInner { blocks_per_file, _lock_file, genesis_block_number: 0, + #[cfg(test)] + before_cache_insert: Default::default(), + #[cfg(test)] + after_cache_validation: Default::default(), }; Ok(provider) @@ -971,14 +1014,36 @@ impl StaticFileProvider { trace!(target: "providers::static_file", ?segment, ?fixed_block_range, "Jar found in cache"); jar.into() } else { - trace!(target: "providers::static_file", ?segment, ?fixed_block_range, "Creating jar from scratch"); + let generation = self.cache_generation.load(Ordering::Acquire); + trace!(target: "providers::static_file", ?segment, ?fixed_block_range, generation, "Creating jar from scratch"); let path = self.path.join(segment.filename(fixed_block_range)); let jar = NippyJar::load(&path).map_err(ProviderError::other)?; let loaded = LoadedJar::new(jar)?; + #[cfg(test)] + self.before_cache_insert.fire(segment); + // The cache may have been populated since the initial miss, including by // `update_index` publishing a newer snapshot while we loaded this jar without a lock. // Preserve that entry instead of overwriting it with our potentially stale snapshot. - self.map.entry(key).or_insert(loaded).downgrade().into() + self.map + .entry(key) + .or_try_insert_with(|| -> ProviderResult<_> { + // Validate while holding the entry's shard lock so clearing the cache cannot + // slip between validation and insertion. If invalidated, reload once under + // this lock rather than retrying unboundedly during frequent pruning. + let loaded = if self.cache_generation.load(Ordering::Acquire) == generation { + loaded + } else { + trace!(target: "providers::static_file", ?segment, ?fixed_block_range, generation, "Reloading jar after cache invalidation"); + drop(loaded); + LoadedJar::new(NippyJar::load(&path).map_err(ProviderError::other)?)? + }; + #[cfg(test)] + self.after_cache_validation.fire(segment); + Ok(loaded) + })? + .downgrade() + .into() }; if let Some(metrics) = &self.metrics { @@ -1221,7 +1286,10 @@ impl StaticFileProvider { ); } - // If this is a re-initialization, we need to clear this as well + // If this is a re-initialization, invalidate in-flight cache fills before clearing the + // cache. A fill that started before this point either reloads under its shard lock or + // publishes before the clear reaches that shard and is removed by it. + self.cache_generation.fetch_add(1, Ordering::AcqRel); self.map.clear(); // initialize the expired history height to the lowest static file block @@ -3064,13 +3132,140 @@ where #[cfg(test)] mod tests { - use std::collections::BTreeMap; - + use std::{ + collections::BTreeMap, + sync::{atomic::Ordering, mpsc}, + thread, + time::{Duration, Instant}, + }; + + use alloy_consensus::Header; + use alloy_primitives::B256; use reth_chain_state::EthPrimitives; use reth_db::test_utils::create_test_static_files_dir; use reth_static_file_types::{SegmentRangeInclusive, StaticFileSegment}; - use crate::{providers::StaticFileProvider, StaticFileProviderBuilder}; + use super::StaticFileWriter; + use crate::{providers::StaticFileProvider, BlockHashReader, StaticFileProviderBuilder}; + + #[test] + fn stale_cache_fill_does_not_survive_index_reinitialization() -> eyre::Result<()> { + let (static_dir, _) = create_test_static_files_dir(); + let static_files: StaticFileProvider = + StaticFileProviderBuilder::read_write(&static_dir) + .with_blocks_per_file_for_segment(StaticFileSegment::Headers, 10) + .with_blocks_per_file_for_segment(StaticFileSegment::Receipts, 2) + .build()?; + + let hash_0 = B256::from([0x10; 32]); + let header_0 = Header { number: 0, ..Default::default() }; + { + let mut writer = static_files.latest_writer(StaticFileSegment::Headers)?; + writer.append_header(&header_0, &hash_0)?; + writer.commit()?; + } + + // Create two receipt jars. Deleting either one reinitializes every segment index and + // invalidates the shared jar cache, matching the pruning path from the incidents. + { + let mut writer = static_files.latest_writer(StaticFileSegment::Receipts)?; + for block in 0..=3 { + writer.increment_block(block)?; + } + writer.commit()?; + } + static_files.delete_jar(StaticFileSegment::Receipts, 0)?; + + let (loaded_tx, loaded_rx) = mpsc::sync_channel(0); + let (resume_tx, resume_rx) = mpsc::sync_channel(0); + static_files.before_cache_insert.set(move |segment| { + assert_eq!(segment, StaticFileSegment::Headers); + loaded_tx.send(()).expect("test controller dropped"); + resume_rx + .recv_timeout(Duration::from_secs(10)) + .expect("test controller did not resume"); + }); + + // Pause a cache fill after it loads the header jar containing only block 0. + let reader_static_files = static_files.clone(); + let reader = thread::spawn(move || reader_static_files.block_hash(0)); + loaded_rx + .recv_timeout(Duration::from_secs(10)) + .expect("reader did not reach cache-fill hook"); + + // Publish a newer snapshot of the same header jar, then simulate unrelated segment + // pruning invalidating the whole cache while the old load remains in flight. + let hash_1 = B256::from([0x11; 32]); + let header_1 = Header { number: 1, ..Default::default() }; + { + let mut writer = static_files.latest_writer(StaticFileSegment::Headers)?; + writer.append_header(&header_1, &hash_1)?; + writer.commit()?; + } + static_files.delete_jar(StaticFileSegment::Receipts, 2)?; + + resume_tx.send(()).expect("reader dropped before cache publication"); + assert_eq!(reader.join().expect("reader panicked")?, Some(hash_0)); + assert_eq!( + static_files.block_hash(1)?, + Some(hash_1), + "cache fill started before reinitialization repopulated an invalidated jar" + ); + + Ok(()) + } + + #[test] + fn cache_fill_validation_is_serialized_with_index_reinitialization() -> eyre::Result<()> { + let (static_dir, _) = create_test_static_files_dir(); + let static_files: StaticFileProvider = + StaticFileProviderBuilder::read_write(&static_dir).with_blocks_per_file(10).build()?; + let hash = B256::from([0x10; 32]); + { + let mut writer = static_files.latest_writer(StaticFileSegment::Headers)?; + writer.append_header(&Header::default(), &hash)?; + writer.commit()?; + } + static_files.remove_cached_provider(StaticFileSegment::Headers, 9); + + let generation = static_files.cache_generation.load(Ordering::Acquire); + let (validated_tx, validated_rx) = mpsc::sync_channel(0); + let (resume_tx, resume_rx) = mpsc::sync_channel(0); + static_files.after_cache_validation.set(move |segment| { + assert_eq!(segment, StaticFileSegment::Headers); + validated_tx.send(()).expect("test controller dropped"); + resume_rx + .recv_timeout(Duration::from_secs(10)) + .expect("test controller did not resume"); + }); + + let reader_static_files = static_files.clone(); + let reader = thread::spawn(move || reader_static_files.block_hash(0)); + validated_rx + .recv_timeout(Duration::from_secs(10)) + .expect("reader did not reach cache validation"); + + // The shard must remain locked between checking the generation and inserting the jar. + // Otherwise a clear can finish here and the reader can repopulate the cache afterward. + assert!(static_files.map.try_get(&(9, StaticFileSegment::Headers)).is_locked()); + let invalidator_static_files = static_files.clone(); + let invalidator = thread::spawn(move || invalidator_static_files.initialize_index()); + let deadline = Instant::now() + Duration::from_secs(10); + while static_files.cache_generation.load(Ordering::Acquire) == generation { + assert!(Instant::now() < deadline, "reinitialization did not invalidate the cache"); + thread::sleep(Duration::from_millis(1)); + } + + // Invalidation has started, but clearing this shard must wait for publication and the + // reader's guard to be released. The published snapshot must not survive that clear. + resume_tx.send(()).expect("reader dropped before cache publication"); + assert_eq!(reader.join().expect("reader panicked")?, Some(hash)); + invalidator.join().expect("invalidator panicked")?; + assert!(static_files.map.is_empty()); + assert_eq!(static_files.block_hash(0)?, Some(hash)); + + Ok(()) + } #[test] fn test_find_fixed_range_with_block_index() -> eyre::Result<()> { From bd61f3919e702032ee44849e6326c2110ead58af Mon Sep 17 00:00:00 2001 From: Derek Cofausper <256792747+decofe@users.noreply.github.com> Date: Tue, 15 Sep 2026 09:46:52 +0000 Subject: [PATCH 264/294] perf(cli): raise persistence and state masking defaults (#27220) Co-authored-by: Brian Picciano <933154+mediocregopher@users.noreply.github.com> --- .github/workflows/docker.yml | 2 - Dockerfile.depot | 4 -- crates/engine/primitives/src/config.rs | 41 ++++++++--- crates/engine/tree/src/tree/tests.rs | 16 ++++- .../engine/tree/tests/e2e-testsuite/main.rs | 8 ++- crates/ethereum/node/tests/e2e/eip6780.rs | 6 +- crates/node/core/src/args/engine.rs | 72 +++++++++++++++---- docs/vocs/docs/pages/cli/reth/node.mdx | 8 +-- 8 files changed, 118 insertions(+), 39 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 78a2457beae..1b6932b7a4c 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -96,8 +96,6 @@ jobs: echo "ethereum.tags=${REGISTRY}/reth:nightly" echo "ethereum.args.RETH_ENGINE_TXPOOL_PREWARMING=true" echo "ethereum.args.RETH_ENGINE_SENDER_RECOVERY_CACHE=true" - echo "ethereum.args.RETH_ENGINE_PERSISTENCE_THRESHOLD=50" - echo "ethereum.args.RETH_ENGINE_NUM_STATE_MASKING_BLOCKS=30" echo "EOF" } >> "$GITHUB_OUTPUT" diff --git a/Dockerfile.depot b/Dockerfile.depot index 7602205e7f3..0ff3f2b736a 100644 --- a/Dockerfile.depot +++ b/Dockerfile.depot @@ -79,10 +79,6 @@ ARG RETH_ENGINE_TXPOOL_PREWARMING=false ENV RETH_ENGINE_TXPOOL_PREWARMING=$RETH_ENGINE_TXPOOL_PREWARMING ARG RETH_ENGINE_SENDER_RECOVERY_CACHE=false ENV RETH_ENGINE_SENDER_RECOVERY_CACHE=$RETH_ENGINE_SENDER_RECOVERY_CACHE -ARG RETH_ENGINE_PERSISTENCE_THRESHOLD=7 -ENV RETH_ENGINE_PERSISTENCE_THRESHOLD=$RETH_ENGINE_PERSISTENCE_THRESHOLD -ARG RETH_ENGINE_NUM_STATE_MASKING_BLOCKS=0 -ENV RETH_ENGINE_NUM_STATE_MASKING_BLOCKS=$RETH_ENGINE_NUM_STATE_MASKING_BLOCKS # Install runtime dependencies RUN apt-get update && \ diff --git a/crates/engine/primitives/src/config.rs b/crates/engine/primitives/src/config.rs index 0bec4746106..763f56cc16a 100644 --- a/crates/engine/primitives/src/config.rs +++ b/crates/engine/primitives/src/config.rs @@ -4,14 +4,14 @@ use alloy_eips::merge::EPOCH_SLOTS; use core::time::Duration; /// Triggers persistence when the number of canonical blocks in memory exceeds this threshold. -pub const DEFAULT_PERSISTENCE_THRESHOLD: u64 = 7; +pub const DEFAULT_PERSISTENCE_THRESHOLD: u64 = 50; /// Number of persisted blocks whose state/trie writes are masked by an in-memory suffix. -pub const DEFAULT_NUM_STATE_MASKING_BLOCKS: u64 = 0; +pub const DEFAULT_NUM_STATE_MASKING_BLOCKS: u64 = 30; /// Maximum number of blocks beyond the in-memory buffer target awaiting persistence before engine /// API processing is stalled. -pub const DEFAULT_PERSISTENCE_BACKPRESSURE_THRESHOLD: u64 = 16; +pub const DEFAULT_PERSISTENCE_BACKPRESSURE_THRESHOLD: u64 = DEFAULT_PERSISTENCE_THRESHOLD * 2; /// How close to the canonical head we persist blocks. pub const DEFAULT_MEMORY_BLOCK_BUFFER_TARGET: u64 = 5; @@ -303,6 +303,8 @@ impl TreeConfig { share_execution_cache_with_payload_builder: bool, share_sparse_trie_with_payload_builder: bool, ) -> Self { + let num_state_masking_blocks = + if persistence_threshold == 0 { 0 } else { num_state_masking_blocks }; assert_backpressure_threshold_invariant( persistence_threshold, persistence_backpressure_threshold, @@ -471,9 +473,12 @@ impl TreeConfig { self.allow_unwind_canonical_header } - /// Setter for persistence threshold. + /// Setter for persistence threshold. Setting this to zero disables state masking. pub const fn with_persistence_threshold(mut self, persistence_threshold: u64) -> Self { self.persistence_threshold = persistence_threshold; + if persistence_threshold == 0 { + self.num_state_masking_blocks = 0; + } assert_backpressure_threshold_invariant( self.persistence_threshold, self.persistence_backpressure_threshold, @@ -487,8 +492,10 @@ impl TreeConfig { } /// Setter for the number of persisted blocks whose state/trie writes are masked. + /// State masking is disabled when the persistence threshold is zero. pub const fn with_num_state_masking_blocks(mut self, num_state_masking_blocks: u64) -> Self { - self.num_state_masking_blocks = num_state_masking_blocks; + self.num_state_masking_blocks = + if self.persistence_threshold == 0 { 0 } else { num_state_masking_blocks }; assert_state_masking_invariant( self.persistence_threshold, self.num_state_masking_blocks, @@ -814,7 +821,7 @@ impl TreeConfig { #[cfg(test)] mod tests { - use super::{TreeConfig, DEFAULT_NUM_STATE_MASKING_BLOCKS}; + use super::TreeConfig; #[test] fn txpool_prewarming_is_disabled_by_default_and_can_be_enabled() { @@ -842,16 +849,27 @@ mod tests { )] fn rejects_backpressure_threshold_at_or_below_persistence_threshold() { let _ = TreeConfig::default() + .with_num_state_masking_blocks(0) .with_persistence_threshold(4) .with_persistence_backpressure_threshold(4); } #[test] - fn state_masking_is_disabled_by_default() { - assert_eq!( - TreeConfig::default().num_state_masking_blocks(), - DEFAULT_NUM_STATE_MASKING_BLOCKS - ); + fn default_persistence_settings() { + let config = TreeConfig::default(); + assert_eq!(config.persistence_threshold(), 50); + assert_eq!(config.num_state_masking_blocks(), 30); + assert_eq!(config.persistence_backpressure_threshold(), 100); + } + + #[test] + fn zero_persistence_threshold_disables_state_masking() { + let config = TreeConfig::default().with_persistence_threshold(0); + assert_eq!(config.num_state_masking_blocks(), 0); + let config = config.with_num_state_masking_blocks(u64::MAX); + assert_eq!(config.num_state_masking_blocks(), 0); + let config = config.with_memory_block_buffer_target(0); + assert_eq!(config.num_state_masking_blocks(), 0); } #[test] @@ -860,6 +878,7 @@ mod tests { )] fn rejects_state_masking_window_at_or_above_persistence_threshold() { let _ = TreeConfig::default() + .with_num_state_masking_blocks(0) .with_persistence_threshold(4) .with_memory_block_buffer_target(2) .with_num_state_masking_blocks(2); diff --git a/crates/engine/tree/src/tree/tests.rs b/crates/engine/tree/src/tree/tests.rs index 0680bf4afb8..6830500088a 100644 --- a/crates/engine/tree/src/tree/tests.rs +++ b/crates/engine/tree/src/tree/tests.rs @@ -615,7 +615,10 @@ async fn test_tree_persist_blocks() { assert_eq!(input.persist_rest_blocks().len(), expected_persist_len); assert_eq!(input.persist_rest_blocks(), &blocks[..expected_persist_len]); assert_eq!(input.prev_db_tip(), input.prev_partial_state_trie()); - assert_eq!(input.new_db_tip(), input.new_partial_state_trie()); + assert_eq!( + input.new_db_tip() - tree_config.num_state_masking_blocks(), + input.new_partial_state_trie() + ); } else { panic!("unexpected action received {received_action:?}"); } @@ -1254,8 +1257,12 @@ async fn test_tree_state_on_new_head_reorg() { // Set persistence_threshold to 1 let mut test_harness = TestHarness::new(chain_spec); - test_harness.tree.config = - test_harness.tree.config.with_persistence_threshold(1).with_memory_block_buffer_target(1); + test_harness.tree.config = test_harness + .tree + .config + .with_num_state_masking_blocks(0) + .with_persistence_threshold(1) + .with_memory_block_buffer_target(1); let mut test_block_builder = TestBlockBuilder::eth(); let blocks: Vec<_> = test_block_builder.get_executed_blocks(1..6).collect(); @@ -1460,6 +1467,7 @@ async fn test_get_canonical_blocks_to_persist() { let persistence_threshold = 4; let memory_block_buffer_target = 3; test_harness.tree.config = TreeConfig::default() + .with_num_state_masking_blocks(0) .with_persistence_threshold(persistence_threshold) .with_memory_block_buffer_target(memory_block_buffer_target); @@ -1509,6 +1517,7 @@ fn threshold_persistence_uses_canonical_in_memory_chain_length() { let blocks: Vec<_> = TestBlockBuilder::eth().get_executed_blocks(0..10).collect(); let mut test_harness = TestHarness::new(MAINNET.clone()).with_blocks(blocks.clone()); test_harness.tree.config = TreeConfig::default() + .with_num_state_masking_blocks(0) .with_persistence_threshold(3) .with_memory_block_buffer_target(0) .with_num_state_masking_blocks(2); @@ -1535,6 +1544,7 @@ fn test_threshold_persistence_with_state_masking_blocks() { test_harness.tree.persistence_state.last_persisted_block = blocks[3].recovered_block().num_hash(); test_harness.tree.config = TreeConfig::default() + .with_num_state_masking_blocks(0) .with_persistence_threshold(4) .with_memory_block_buffer_target(1) .with_num_state_masking_blocks(2); diff --git a/crates/engine/tree/tests/e2e-testsuite/main.rs b/crates/engine/tree/tests/e2e-testsuite/main.rs index dda1b19884c..6ffd2624802 100644 --- a/crates/engine/tree/tests/e2e-testsuite/main.rs +++ b/crates/engine/tree/tests/e2e-testsuite/main.rs @@ -529,6 +529,7 @@ async fn test_engine_tree_pipeline_sync_catches_up_masked_state_e2e() -> Result< .with_storage_v2() .with_tree_config( TreeConfig::default() + .with_num_state_masking_blocks(0) .with_persistence_threshold(PERSISTENCE_THRESHOLD) .with_memory_block_buffer_target(MEMORY_BLOCK_BUFFER_TARGET) .with_num_state_masking_blocks(NUM_STATE_MASKING_BLOCKS) @@ -663,7 +664,12 @@ fn disk_reorg_setup(storage_v2: bool) -> Setup { .build(), )) .with_network(NetworkSetup::multi_node_unconnected(2)) - .with_tree_config(TreeConfig::default().with_has_enough_parallelism(true)); + .with_tree_config( + TreeConfig::default() + .with_num_state_masking_blocks(0) + .with_persistence_threshold(7) + .with_has_enough_parallelism(true), + ); if storage_v2 { setup = setup.with_storage_v2(); } diff --git a/crates/ethereum/node/tests/e2e/eip6780.rs b/crates/ethereum/node/tests/e2e/eip6780.rs index c68e15c7a49..157d3e96f6b 100644 --- a/crates/ethereum/node/tests/e2e/eip6780.rs +++ b/crates/ethereum/node/tests/e2e/eip6780.rs @@ -69,8 +69,10 @@ async fn run_selfdestruct_suite(fork: EthereumHardfork) -> eyre::Result<()> { // low persistence thresholds so the scenario blocks reach the database and the persisted // trie representation can be verified at the end of the suite - let tree_config = - TreeConfig::default().with_persistence_threshold(2).with_memory_block_buffer_target(1); + let tree_config = TreeConfig::default() + .with_num_state_masking_blocks(0) + .with_persistence_threshold(2) + .with_memory_block_buffer_target(1); let (mut nodes, wallet) = setup_engine::(1, fork_spec(fork), false, tree_config, move |timestamp| { eth_payload_attributes_for_fork(fork, timestamp) diff --git a/crates/node/core/src/args/engine.rs b/crates/node/core/src/args/engine.rs index f6aba3bdeee..f93a4236031 100644 --- a/crates/node/core/src/args/engine.rs +++ b/crates/node/core/src/args/engine.rs @@ -311,8 +311,8 @@ pub struct EngineArgs { /// must be in-memory, ahead of the last persisted block, before flushing canonical blocks to /// disk again. /// - /// To persist blocks as fast as the node receives them, set this value to zero. This will - /// cause more frequent DB writes. + /// To persist blocks as fast as the node receives them, set this value to zero. This disables + /// state masking and causes more frequent DB writes. #[arg( long = "engine.persistence-threshold", env = "RETH_ENGINE_PERSISTENCE_THRESHOLD", @@ -330,7 +330,7 @@ pub struct EngineArgs { #[arg(long = "engine.persistence-backpressure-threshold")] pub persistence_backpressure_threshold: Option, - /// EXPERIMENTAL: Configure how many of the blocks being persisted should only mask state/trie + /// Configure how many of the blocks being persisted should only mask state/trie /// writes instead of durably persisting their state/trie updates in the current cycle. #[arg( long = "engine.num-state-masking-blocks", @@ -661,6 +661,15 @@ impl Default for EngineArgs { } impl EngineArgs { + /// Returns the effective state masking window, disabled when persistence is immediate. + pub const fn num_state_masking_blocks(&self) -> u64 { + if self.persistence_threshold == 0 { + 0 + } else { + self.num_state_masking_blocks + } + } + /// Returns the effective memory block buffer target. pub fn memory_block_buffer_target(&self) -> u64 { self.memory_block_buffer_target.unwrap_or_else(|| { @@ -680,6 +689,7 @@ impl EngineArgs { pub fn validate(&self) -> eyre::Result<()> { let persistence_backpressure_threshold = self.persistence_backpressure_threshold(); let memory_block_buffer_target = self.memory_block_buffer_target(); + let num_state_masking_blocks = self.num_state_masking_blocks(); ensure!( persistence_backpressure_threshold > self.persistence_threshold, "--engine.persistence-backpressure-threshold ({}) must be greater than --engine.persistence-threshold ({})", @@ -693,13 +703,13 @@ impl EngineArgs { self.persistence_threshold, ); ensure!( - self.num_state_masking_blocks == 0 || + num_state_masking_blocks == 0 || matches!( - self.num_state_masking_blocks.checked_add(memory_block_buffer_target), + num_state_masking_blocks.checked_add(memory_block_buffer_target), Some(window) if window < self.persistence_threshold ), "--engine.num-state-masking-blocks ({}) + --engine.memory-block-buffer-target ({}) must be less than --engine.persistence-threshold ({})", - self.num_state_masking_blocks, + num_state_masking_blocks, memory_block_buffer_target, self.persistence_threshold, ); @@ -721,10 +731,12 @@ impl EngineArgs { tracing::warn!(target: "reth::cli", "--engine.legacy-state-root has no effect anymore, use --engine.state-root-fallback to force synchronous state root computation"); } let config = TreeConfig::default() + // Clear the default window before applying overrides that may be smaller than it. + .with_persistence_threshold(0) .with_persistence_backpressure_threshold(self.persistence_backpressure_threshold()) .with_persistence_threshold(self.persistence_threshold) .with_memory_block_buffer_target(self.memory_block_buffer_target()) - .with_num_state_masking_blocks(self.num_state_masking_blocks) + .with_num_state_masking_blocks(self.num_state_masking_blocks()) .with_invalid_header_hit_eviction_threshold(self.invalid_header_hit_eviction_threshold) .without_state_cache(self.state_cache_disabled) .without_prewarming(self.prewarming_disabled) @@ -777,13 +789,16 @@ mod tests { let default_args = EngineArgs::default(); let args = CommandParser::::parse_from(["reth"]).args; assert_eq!(args, default_args); - assert_eq!(args.persistence_threshold, 7); + assert_eq!(args.persistence_threshold, 50); + assert_eq!(args.num_state_masking_blocks, 30); assert_eq!(args.memory_block_buffer_target, None); assert_eq!(args.memory_block_buffer_target(), 5); - assert_eq!( - args.persistence_backpressure_threshold(), - DefaultEngineValues::get_global().persistence_backpressure_threshold - ); + assert_eq!(args.persistence_backpressure_threshold(), 100); + args.validate().unwrap(); + let config = args.tree_config(); + assert_eq!(config.persistence_threshold(), 50); + assert_eq!(config.num_state_masking_blocks(), 30); + assert_eq!(config.persistence_backpressure_threshold(), 100); } #[test] @@ -981,6 +996,7 @@ mod tests { fn validate_memory_block_buffer_target() { let args = EngineArgs { persistence_threshold: 4, + num_state_masking_blocks: 0, memory_block_buffer_target: Some(4), ..EngineArgs::default() }; @@ -1037,6 +1053,8 @@ mod tests { "reth", "--engine.persistence-threshold", "4", + "--engine.num-state-masking-blocks", + "0", ]) .args; @@ -1046,6 +1064,36 @@ mod tests { args.validate().unwrap(); } + #[test] + fn explicit_persistence_settings_can_be_lower_than_defaults() { + let args = CommandParser::::parse_from([ + "reth", + "--engine.persistence-threshold", + "0", + "--engine.persistence-backpressure-threshold", + "1", + ]) + .args; + + args.validate().unwrap(); + let config = args.tree_config(); + assert_eq!(config.persistence_threshold(), 0); + assert_eq!(config.num_state_masking_blocks(), 0); + assert_eq!(config.memory_block_buffer_target(), 0); + assert_eq!(config.persistence_backpressure_threshold(), 1); + } + + #[test] + fn zero_persistence_threshold_disables_explicit_state_masking() { + let args = EngineArgs { + persistence_threshold: 0, + num_state_masking_blocks: u64::MAX, + ..EngineArgs::default() + }; + args.validate().unwrap(); + assert_eq!(args.tree_config().num_state_masking_blocks(), 0); + } + #[test] fn parse_rejects_zero_multiproof_chunk_size() { let result = CommandParser::::try_parse_from([ diff --git a/docs/vocs/docs/pages/cli/reth/node.mdx b/docs/vocs/docs/pages/cli/reth/node.mdx index baf3f155a3a..a0c77f80ee0 100644 --- a/docs/vocs/docs/pages/cli/reth/node.mdx +++ b/docs/vocs/docs/pages/cli/reth/node.mdx @@ -1000,10 +1000,10 @@ Engine: --engine.persistence-threshold Configure persistence threshold for the engine. This determines how many canonical blocks must be in-memory, ahead of the last persisted block, before flushing canonical blocks to disk again. - To persist blocks as fast as the node receives them, set this value to zero. This will cause more frequent DB writes. + To persist blocks as fast as the node receives them, set this value to zero. This disables state masking and causes more frequent DB writes. [env: RETH_ENGINE_PERSISTENCE_THRESHOLD=] - [default: 7] + [default: 50] --engine.persistence-backpressure-threshold Configure the maximum number of blocks beyond the in-memory buffer target that may await persistence before engine API processing stalls. @@ -1013,10 +1013,10 @@ Engine: This value must be greater than `--engine.persistence-threshold`. --engine.num-state-masking-blocks - EXPERIMENTAL: Configure how many of the blocks being persisted should only mask state/trie writes instead of durably persisting their state/trie updates in the current cycle + Configure how many of the blocks being persisted should only mask state/trie writes instead of durably persisting their state/trie updates in the current cycle [env: RETH_ENGINE_NUM_STATE_MASKING_BLOCKS=] - [default: 0] + [default: 30] --engine.memory-block-buffer-target Configure the target number of blocks to keep in memory. From 03e374b05caaade4521ea82f50b5d953cdf99ae3 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Tue, 15 Sep 2026 10:33:12 +0000 Subject: [PATCH 265/294] test(engine): cover unrecoverable payload signatures (#27243) Co-authored-by: Matthias Seitz <19890894+mattsse@users.noreply.github.com> --- .../node/tests/e2e/invalid_payload.rs | 79 +++++++++++++++++-- 1 file changed, 74 insertions(+), 5 deletions(-) diff --git a/crates/ethereum/node/tests/e2e/invalid_payload.rs b/crates/ethereum/node/tests/e2e/invalid_payload.rs index 42d9fa376c5..bb5eec4e627 100644 --- a/crates/ethereum/node/tests/e2e/invalid_payload.rs +++ b/crates/ethereum/node/tests/e2e/invalid_payload.rs @@ -4,13 +4,18 @@ //! state roots) before receiving valid ones, ensuring the node can recover and continue. use crate::utils::{eth_payload_attributes, eth_payload_attributes_amsterdam}; -use alloy_eips::eip7685::RequestsOrHash; -use alloy_primitives::{keccak256, Bytes, B256}; -use alloy_rpc_types_engine::{ExecutionPayloadV3, PayloadStatusEnum}; +use alloy_consensus::proofs::calculate_transaction_root; +use alloy_eips::{eip2718::Decodable2718, eip7685::RequestsOrHash}; +use alloy_primitives::{bytes, keccak256, Bytes, B256}; +use alloy_rpc_types_engine::{ExecutionPayloadV1, ExecutionPayloadV3, PayloadStatusEnum}; use rand::{rngs::StdRng, Rng, SeedableRng}; -use reth_chainspec::{ChainSpecBuilder, MAINNET}; -use reth_e2e_test_utils::{setup_engine, transaction::TransactionTestContext}; +use reth_chainspec::{ChainSpecBuilder, EthereumHardfork, MAINNET}; +use reth_e2e_test_utils::{ + eth_payload_attributes_for_fork, setup_engine, transaction::TransactionTestContext, +}; +use reth_ethereum_primitives::TransactionSigned; use reth_node_ethereum::EthereumNode; +use reth_primitives_traits::SignedTransaction; use reth_rpc_api::EngineApiClient; use std::sync::Arc; @@ -357,6 +362,70 @@ async fn can_handle_invalid_payload_with_transactions() -> eyre::Result<()> { Ok(()) } +/// Tests that `engine_newPayloadV1` returns `INVALID` for an in-range but unrecoverable +/// transaction signature, and still accepts the valid payload afterwards. +#[tokio::test] +async fn unrecoverable_signature_is_invalid_payload() -> eyre::Result<()> { + reth_tracing::init_test_tracing(); + + let chain_spec = Arc::new( + ChainSpecBuilder::default() + .chain(MAINNET.chain) + .genesis(serde_json::from_str(include_str!("../assets/genesis.json")).unwrap()) + .paris_activated() + .build(), + ); + let (mut nodes, wallet) = + setup_engine::(1, chain_spec, false, Default::default(), |timestamp| { + eth_payload_attributes_for_fork(EthereumHardfork::Paris, timestamp) + }) + .await?; + let mut node = nodes.pop().unwrap(); + + let raw_tx = TransactionTestContext::transfer_tx_bytes(1, wallet.inner).await; + node.rpc.inject_tx(raw_tx).await?; + let payload = node.new_payload().await?; + let block = payload.block().clone(); + let valid_payload = + ExecutionPayloadV1::from_block_unchecked(block.hash(), &block.clone().into_block()); + + // r = 5, s = 1, v = 27 are in range, but r = 5 is not a secp256k1 x-coordinate. + let raw_tx = + bytes!("e48085174876e8008252089400000000000000000000000000000000000000ee01801b0501"); + let tx = TransactionSigned::decode_2718_exact(&raw_tx)?; + let recovery_error = tx.try_recover().unwrap_err(); + + // Recompute the transaction root and block hash so validation reaches sender recovery. + let mut invalid_block = block.clone().into_block(); + invalid_block.body.transactions = vec![tx]; + invalid_block.header.transactions_root = + calculate_transaction_root(&invalid_block.body.transactions); + let invalid_payload = + ExecutionPayloadV1::from_block_unchecked(invalid_block.header.hash_slow(), &invalid_block); + + let engine = node.auth_server_handle().http_client(); + let status = EngineApiClient::::new_payload_v1( + &engine, + invalid_payload, + ) + .await?; + let PayloadStatusEnum::Invalid { validation_error } = status.status else { + panic!("Expected INVALID for an unrecoverable signature, got {status:?}"); + }; + assert!(validation_error.contains(&recovery_error.to_string()), "{validation_error}"); + assert_eq!(status.latest_valid_hash, Some(block.parent_hash)); + + let status = EngineApiClient::::new_payload_v1( + &engine, + valid_payload, + ) + .await?; + assert_eq!(status.status, PayloadStatusEnum::Valid); + assert_eq!(status.latest_valid_hash, Some(block.hash())); + + Ok(()) +} + /// Tests that `engine_newPayloadV5` returns `INVALID` with no latest valid hash for undecodable /// block access list bytes. #[tokio::test] From aec876e579a1644bc7afa2dfa255dccf2d63d27c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A9a=20Narzis?= <78718413+lean-apple@users.noreply.github.com> Date: Tue, 15 Sep 2026 13:17:37 +0000 Subject: [PATCH 266/294] refactor(snap-sync): share download context and progress records (#27238) --- crates/snap-sync/src/account/download.rs | 65 ++++++----------- crates/snap-sync/src/account/mod.rs | 4 +- crates/snap-sync/src/account/store.rs | 52 +++++--------- crates/snap-sync/src/common/download.rs | 92 ++++++++++++++++++++++++ crates/snap-sync/src/common/mod.rs | 11 +++ crates/snap-sync/src/common/record.rs | 39 ++++++++++ crates/snap-sync/src/error.rs | 14 ++-- crates/snap-sync/src/lib.rs | 3 +- crates/snap-sync/src/storage/download.rs | 61 +++++----------- crates/snap-sync/src/storage/store.rs | 49 ++++--------- crates/snap-sync/src/test_utils.rs | 2 +- 11 files changed, 222 insertions(+), 170 deletions(-) create mode 100644 crates/snap-sync/src/common/download.rs create mode 100644 crates/snap-sync/src/common/mod.rs create mode 100644 crates/snap-sync/src/common/record.rs diff --git a/crates/snap-sync/src/account/download.rs b/crates/snap-sync/src/account/download.rs index ab8233f6914..6c01563a341 100644 --- a/crates/snap-sync/src/account/download.rs +++ b/crates/snap-sync/src/account/download.rs @@ -1,6 +1,9 @@ //! Downloads account ranges in key order and commits each under the write it was fetched with. -use crate::{AccountCoverage, SnapAccountStore, SnapAttemptStore, SnapSyncError, SnapWrite}; +use crate::{ + common::DownloadContext, AccountCoverage, SnapAccountStore, SnapAttemptStore, SnapSyncError, + SnapWrite, MAX_HASH, +}; use alloy_primitives::{map::B256Map, B256}; use reth_db_api::transaction::DbTxMut; use reth_downloaders::snap::{AccountRangeDownloader, AccountRangeOutcome, VerifiedAccountRange}; @@ -10,50 +13,30 @@ use reth_network_peers::PeerId; use reth_storage_api::{ DBProvider, DatabaseProviderFactory, MetadataProvider, MetadataWriter, StateWriter, }; -use reth_storage_errors::provider::ProviderError; use reth_tasks::Runtime; use reth_trie_common::HashedStorage; use revm::bytecode::Bytecode; use std::fmt; -/// Default soft response limit for account-range requests, matching common peer limits. -pub const DEFAULT_RESPONSE_BYTES: u64 = 512 * 1024; - -/// Inclusive upper bound covering the full account trie keyspace. -pub const MAX_HASH: B256 = B256::new([0xff; B256::len_bytes()]); - /// Downloads the account ranges an attempt still needs, one at a time in key order. /// /// [`Self::next`] fetches the range at the coverage cursor the store records; the caller resolves /// its storage and code, then [`Self::commit`] persists everything and moves the cursor. pub struct AccountRangeDownload { - client: C, - factory: F, - // Proof verification and commits run on the blocking pool. - runtime: Runtime, + context: DownloadContext, // Coverage as last read from the store; none before the first request. coverage: Option, - response_bytes: u64, - // Distinguishes responses to reissued requests. - request_id: u64, } impl AccountRangeDownload { /// Creates a download that continues from the coverage the store records. pub const fn new(client: C, factory: F, runtime: Runtime) -> Self { - Self { - client, - factory, - runtime, - coverage: None, - response_bytes: DEFAULT_RESPONSE_BYTES, - request_id: 0, - } + Self { context: DownloadContext::new(client, factory, runtime), coverage: None } } /// Returns this download asking peers for at most `response_bytes` per response. pub const fn with_response_bytes(mut self, response_bytes: u64) -> Self { - self.response_bytes = response_bytes; + self.context.set_response_bytes(response_bytes); self } @@ -79,17 +62,19 @@ where self.coverage = Some(coverage); let Some(origin) = coverage.next() else { return Ok(None) }; - self.request_id = self.request_id.wrapping_add(1); let request = GetAccountRangeMessage { - request_id: self.request_id, + request_id: self.context.next_request_id(), root_hash, starting_hash: origin, limit_hash: MAX_HASH, - response_bytes: self.response_bytes, + response_bytes: self.context.response_bytes(), }; - let downloader = - AccountRangeDownloader::new(self.client.clone(), request, self.runtime.clone()) - .expect("origin never exceeds the maximum hash"); + let downloader = AccountRangeDownloader::new( + self.context.client().clone(), + request, + self.context.runtime().clone(), + ) + .expect("origin never exceeds the maximum hash"); Ok(Some(match downloader.await? { AccountRangeOutcome::Verified(range) => { @@ -111,18 +96,13 @@ where storages: B256Map, bytecodes: Vec<(B256, Bytecode)>, ) -> Result { - let factory = self.factory.clone(); let coverage = self - .runtime - .spawn_blocking(move || -> Result { + .context + .commit(move |provider| { let VerifiedRange { write, range } = verified; - let provider = factory.database_provider_rw()?; - let coverage = provider.commit_account_range(write, &range, storages, bytecodes)?; - provider.commit()?; - Ok(coverage) + provider.commit_account_range(write, &range, storages, bytecodes) }) - .await - .map_err(|error| SnapSyncError::Provider(ProviderError::other(error)))??; + .await?; self.coverage = Some(coverage); Ok(coverage) } @@ -130,7 +110,7 @@ where // The write the attempt accepts right now, the root to request against, and the coverage the // store records for it. fn active_write(&self) -> Result<(SnapWrite, B256, AccountCoverage), SnapSyncError> { - let provider = self.factory.database_provider_ro()?; + let provider = self.context.factory().database_provider_ro()?; let write = provider.active_snap_write()?.ok_or(SnapSyncError::NoAttempt)?; let root = provider.authorize_snap_write(write)?.state_root(); let coverage = provider.account_coverage(write)?.ok_or(SnapSyncError::NoCoverage)?; @@ -141,10 +121,9 @@ where impl fmt::Debug for AccountRangeDownload { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_struct("AccountRangeDownload") + .field("context", &self.context) .field("coverage", &self.coverage) - .field("response_bytes", &self.response_bytes) - .field("request_id", &self.request_id) - .finish_non_exhaustive() + .finish() } } diff --git a/crates/snap-sync/src/account/mod.rs b/crates/snap-sync/src/account/mod.rs index a92238fbef9..765ab7cba08 100644 --- a/crates/snap-sync/src/account/mod.rs +++ b/crates/snap-sync/src/account/mod.rs @@ -3,7 +3,5 @@ mod download; mod store; -pub use download::{ - AccountRangeDownload, AccountRangeStep, VerifiedRange, DEFAULT_RESPONSE_BYTES, MAX_HASH, -}; +pub use download::{AccountRangeDownload, AccountRangeStep, VerifiedRange}; pub use store::{AccountCoverage, SnapAccountStore}; diff --git a/crates/snap-sync/src/account/store.rs b/crates/snap-sync/src/account/store.rs index 3caa812ed50..ff2ee065501 100644 --- a/crates/snap-sync/src/account/store.rs +++ b/crates/snap-sync/src/account/store.rs @@ -3,8 +3,8 @@ //! Each range replaces its key interval, and ranges commit in key order. use crate::{ - storage::persisted_storage_root, SnapAttemptStore, SnapStorageStore, SnapSyncError, SnapWrite, - StorageProgress, + common::SnapRecord, storage::persisted_storage_root, SnapAttemptStore, SnapStorageStore, + SnapSyncError, SnapWrite, StorageProgress, }; use alloy_primitives::{ map::{B256Map, B256Set}, @@ -18,7 +18,6 @@ use reth_db_api::{ use reth_downloaders::snap::VerifiedAccountRange; use reth_primitives_traits::Account; use reth_storage_api::{DBProvider, MetadataProvider, MetadataWriter, SnapAttemptId, StateWriter}; -use reth_storage_errors::provider::ProviderError; use reth_trie_common::{ root::storage_root, HashedPostState, HashedPostStateSorted, HashedStorage, TrieAccount, EMPTY_ROOT_HASH, @@ -27,12 +26,6 @@ use revm::{bytecode::Bytecode, database::states::StateChangeset}; use serde::{Deserialize, Serialize}; use std::ops::Bound; -// Metadata key of the coverage record. -const COVERAGE_KEY: &str = "snap_account_coverage"; - -// Encoding version of the coverage record this build writes. -const COVERAGE_VERSION: u32 = 1; - /// Persistence for the account ranges an attempt downloads. /// /// Blanket-implemented over the node's writers, so accounts, their dependencies and the coverage @@ -128,22 +121,15 @@ struct StoredCoverage { coverage: AccountCoverage, } +impl SnapRecord for StoredCoverage { + const KEY: &'static str = "snap_account_coverage"; + const VERSION: u32 = 1; +} + impl StoredCoverage { - // Serializes `coverage` for `attempt` at this build's version. - fn encode(attempt: SnapAttemptId, coverage: AccountCoverage) -> Result, SnapSyncError> { - let stored = Self { version: COVERAGE_VERSION, attempt, coverage }; - Ok(serde_json::to_vec(&stored).map_err(ProviderError::other)?) - } - - // Checks the version first, so a record from another build is reported rather than misread. - fn decode(bytes: &[u8]) -> Result { - let value: serde_json::Value = - serde_json::from_slice(bytes).map_err(ProviderError::other)?; - let version = value.get("version").and_then(serde_json::Value::as_u64); - if version != Some(COVERAGE_VERSION as u64) { - return Err(SnapSyncError::UnsupportedCoverage { version }) - } - Ok(serde_json::from_value(value).map_err(ProviderError::other)?) + // `coverage` for `attempt` at this build's version. + const fn new(attempt: SnapAttemptId, coverage: AccountCoverage) -> Self { + Self { version: Self::VERSION, attempt, coverage } } } @@ -157,15 +143,14 @@ impl SnapAccountStore for T { return Ok(coverage) } let start = AccountCoverage::START; - self.write_metadata(COVERAGE_KEY, StoredCoverage::encode(write.attempt(), start)?)?; + StoredCoverage::new(write.attempt(), start).write(self)?; Ok(start) } // A record left by another attempt reads as no coverage. fn account_coverage(&self, write: SnapWrite) -> Result, SnapSyncError> { self.authorize_snap_write(write)?; - let Some(bytes) = self.get_metadata(COVERAGE_KEY)? else { return Ok(None) }; - let stored = StoredCoverage::decode(&bytes)?; + let Some(stored) = StoredCoverage::read(self)? else { return Ok(None) }; Ok((stored.attempt == write.attempt()).then_some(stored.coverage)) } @@ -197,7 +182,7 @@ impl SnapAccountStore for T { self.remove::(interval)?; self.remove_storages_except(interval, &persisted)?; dependencies.write(self)?; - self.write_metadata(COVERAGE_KEY, StoredCoverage::encode(write.attempt(), advanced)?)?; + StoredCoverage::new(write.attempt(), advanced).write(self)?; Ok(advanced) } @@ -463,12 +448,7 @@ mod tests { assert!(tail.accounts().is_empty()); let provider = factory.database_provider_rw().unwrap(); let coverage = AccountCoverage { next: Some(key(3)) }; - provider - .write_metadata( - COVERAGE_KEY, - StoredCoverage::encode(write.attempt(), coverage).unwrap(), - ) - .unwrap(); + StoredCoverage::new(write.attempt(), coverage).write(&provider).unwrap(); let coverage = provider.commit_account_range(write, &tail, Default::default(), Vec::new()).unwrap(); @@ -738,11 +718,11 @@ mod tests { for record in [br#"{"version":999}"#.to_vec(), b"{}".to_vec()] { let provider = factory.database_provider_rw().unwrap(); - provider.write_metadata(COVERAGE_KEY, record).unwrap(); + provider.write_metadata(StoredCoverage::KEY, record).unwrap(); assert!(matches!( provider.account_coverage(write), - Err(SnapSyncError::UnsupportedCoverage { .. }) + Err(SnapSyncError::UnsupportedRecord { .. }) )); } } diff --git a/crates/snap-sync/src/common/download.rs b/crates/snap-sync/src/common/download.rs new file mode 100644 index 00000000000..85c1d342bd8 --- /dev/null +++ b/crates/snap-sync/src/common/download.rs @@ -0,0 +1,92 @@ +//! Client, database and request settings shared by the account, storage and bytecode downloads. +//! +//! Each download holds a [`DownloadContext`] to number its requests and commit verified responses +//! in one transaction on the blocking pool. + +use crate::SnapSyncError; +use alloy_primitives::B256; +use reth_storage_api::{DBProvider, DatabaseProviderFactory}; +use reth_storage_errors::provider::ProviderError; +use reth_tasks::Runtime; +use std::fmt; + +/// Default soft response limit for snap requests, matching common peer limits. +pub const DEFAULT_RESPONSE_BYTES: u64 = 512 * 1024; + +/// Inclusive upper bound covering the full trie keyspace. +pub const MAX_HASH: B256 = B256::new([0xff; B256::len_bytes()]); + +/// Client, database and request settings a domain download sends and commits through. +pub(crate) struct DownloadContext { + client: C, + factory: F, + // Proof verification and commits run on the blocking pool. + runtime: Runtime, + response_bytes: u64, + // Distinguishes responses to reissued requests. + request_id: u64, +} + +impl DownloadContext { + pub(crate) const fn new(client: C, factory: F, runtime: Runtime) -> Self { + Self { client, factory, runtime, response_bytes: DEFAULT_RESPONSE_BYTES, request_id: 0 } + } + + pub(crate) const fn client(&self) -> &C { + &self.client + } + + pub(crate) const fn factory(&self) -> &F { + &self.factory + } + + pub(crate) const fn runtime(&self) -> &Runtime { + &self.runtime + } + + pub(crate) const fn response_bytes(&self) -> u64 { + self.response_bytes + } + + pub(crate) const fn set_response_bytes(&mut self, response_bytes: u64) { + self.response_bytes = response_bytes; + } + + /// Returns the id for the next request. + pub(crate) const fn next_request_id(&mut self) -> u64 { + self.request_id = self.request_id.wrapping_add(1); + self.request_id + } +} + +impl DownloadContext +where + F: DatabaseProviderFactory + Clone + 'static, + F::ProviderRW: DBProvider, +{ + /// Runs `write` in one transaction on the blocking pool, committing only if it succeeds. + pub(crate) async fn commit( + &self, + write: impl FnOnce(&F::ProviderRW) -> Result + Send + 'static, + ) -> Result { + let factory = self.factory.clone(); + self.runtime + .spawn_blocking(move || -> Result { + let provider = factory.database_provider_rw()?; + let output = write(&provider)?; + provider.commit()?; + Ok(output) + }) + .await + .map_err(|error| SnapSyncError::Provider(ProviderError::other(error)))? + } +} + +impl fmt::Debug for DownloadContext { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("DownloadContext") + .field("response_bytes", &self.response_bytes) + .field("request_id", &self.request_id) + .finish_non_exhaustive() + } +} diff --git a/crates/snap-sync/src/common/mod.rs b/crates/snap-sync/src/common/mod.rs new file mode 100644 index 00000000000..2eb77c41a48 --- /dev/null +++ b/crates/snap-sync/src/common/mod.rs @@ -0,0 +1,11 @@ +//! Plumbing shared by the account, storage and bytecode downloads. +//! +//! [`DownloadContext`] carries the client, database and request settings each download sends +//! and commits through; [`SnapRecord`] stores their progress as versioned metadata records. + +mod download; +mod record; + +pub(crate) use download::DownloadContext; +pub use download::{DEFAULT_RESPONSE_BYTES, MAX_HASH}; +pub(crate) use record::SnapRecord; diff --git a/crates/snap-sync/src/common/record.rs b/crates/snap-sync/src/common/record.rs new file mode 100644 index 00000000000..b34744395b2 --- /dev/null +++ b/crates/snap-sync/src/common/record.rs @@ -0,0 +1,39 @@ +//! Versioned progress records kept in the metadata table. +//! +//! Account coverage and storage progress each implement [`SnapRecord`], so a record written by +//! another build version is reported instead of misread when a download resumes. + +use crate::SnapSyncError; +use reth_storage_api::{MetadataProvider, MetadataWriter}; +use reth_storage_errors::provider::ProviderError; +use serde::{de::DeserializeOwned, Serialize}; + +/// A progress record stored as JSON under its own metadata key. +/// +/// The serialized record carries a `version` field, checked before the rest is decoded. +pub(crate) trait SnapRecord: Serialize + DeserializeOwned { + /// Metadata key the record is stored under. + const KEY: &'static str; + + /// Encoding version this build writes. + const VERSION: u32; + + /// Reads the record, reporting one written at another version instead of misreading it. + fn read(provider: &impl MetadataProvider) -> Result, SnapSyncError> { + let Some(bytes) = provider.get_metadata(Self::KEY)? else { return Ok(None) }; + let value: serde_json::Value = + serde_json::from_slice(&bytes).map_err(ProviderError::other)?; + let version = value.get("version").and_then(serde_json::Value::as_u64); + if version != Some(u64::from(Self::VERSION)) { + return Err(SnapSyncError::UnsupportedRecord { key: Self::KEY, version }) + } + Ok(Some(serde_json::from_value(value).map_err(ProviderError::other)?)) + } + + /// Writes this record under its key. + fn write(&self, provider: &impl MetadataWriter) -> Result<(), SnapSyncError> { + provider + .write_metadata(Self::KEY, serde_json::to_vec(self).map_err(ProviderError::other)?)?; + Ok(()) + } +} diff --git a/crates/snap-sync/src/error.rs b/crates/snap-sync/src/error.rs index 5c61947bcb5..22ba20c9d55 100644 --- a/crates/snap-sync/src/error.rs +++ b/crates/snap-sync/src/error.rs @@ -43,9 +43,11 @@ pub enum SnapSyncError { /// No account coverage is recorded for the attempt. #[error("no account coverage is recorded for the attempt")] NoCoverage, - /// A persisted coverage record this build cannot read. - #[error("account coverage record version {version:?} is not supported")] - UnsupportedCoverage { + /// A persisted progress record this build cannot read. + #[error("{key} record version {version:?} is not supported")] + UnsupportedRecord { + /// Metadata key the record is stored under. + key: &'static str, /// Version found on disk, absent when the record carries no numeric version. version: Option, }, @@ -71,12 +73,6 @@ pub enum SnapSyncError { /// Slot the storage was requested from. from: B256, }, - /// A persisted storage progress record this build cannot read. - #[error("storage progress record version {version:?} is not supported")] - UnsupportedStorageProgress { - /// Version found on disk, absent when the record carries no numeric version. - version: Option, - }, /// An account with storage was committed without it. #[error("account {account} has storage that was not downloaded")] MissingStorage { diff --git a/crates/snap-sync/src/lib.rs b/crates/snap-sync/src/lib.rs index 8d0f70e23a3..c9157628063 100644 --- a/crates/snap-sync/src/lib.rs +++ b/crates/snap-sync/src/lib.rs @@ -39,6 +39,7 @@ mod account; mod attempt; mod bal; +mod common; mod error; mod generation; mod pivot; @@ -50,10 +51,10 @@ mod test_utils; pub use account::{ AccountCoverage, AccountRangeDownload, AccountRangeStep, SnapAccountStore, VerifiedRange, - DEFAULT_RESPONSE_BYTES, MAX_HASH, }; pub use attempt::{SnapAttemptStore, SnapWrite}; pub use bal::{BalStateUpdate, DownloadedAccount}; +pub use common::{DEFAULT_RESPONSE_BYTES, MAX_HASH}; pub use error::SnapSyncError; pub use generation::{SnapGeneration, SnapPhase}; pub use pivot::SnapPivotPolicy; diff --git a/crates/snap-sync/src/storage/download.rs b/crates/snap-sync/src/storage/download.rs index e3b092dbb05..f140ada9a05 100644 --- a/crates/snap-sync/src/storage/download.rs +++ b/crates/snap-sync/src/storage/download.rs @@ -1,8 +1,8 @@ //! Downloads the storage of an account range's contracts, committing every verified response. use crate::{ - SnapStorageStore, SnapSyncError, SnapWrite, StorageChunk, StorageProgress, VerifiedRange, - DEFAULT_RESPONSE_BYTES, MAX_HASH, + common::DownloadContext, SnapStorageStore, SnapSyncError, StorageChunk, StorageProgress, + VerifiedRange, MAX_HASH, }; use alloy_primitives::B256; use reth_db_api::transaction::DbTxMut; @@ -15,7 +15,6 @@ use reth_network_peers::PeerId; use reth_storage_api::{ DBProvider, DatabaseProviderFactory, MetadataProvider, MetadataWriter, StateWriter, }; -use reth_storage_errors::provider::ProviderError; use reth_tasks::Runtime; use std::fmt; @@ -27,33 +26,23 @@ pub const DEFAULT_STORAGE_ACCOUNTS: usize = 128; /// Every verified response is committed before the next request, so at most one response is held /// however large a contract is, and a download resumes from the persisted progress. pub struct StorageRangeDownload { - client: C, - factory: F, - // Proof verification and commits run on the blocking pool. - runtime: Runtime, - response_bytes: u64, + context: DownloadContext, // Contracts asked for per request. max_accounts: usize, - // Distinguishes responses to reissued requests. - request_id: u64, } impl StorageRangeDownload { /// Creates a download that continues from the progress the store records. pub const fn new(client: C, factory: F, runtime: Runtime) -> Self { Self { - client, - factory, - runtime, - response_bytes: DEFAULT_RESPONSE_BYTES, + context: DownloadContext::new(client, factory, runtime), max_accounts: DEFAULT_STORAGE_ACCOUNTS, - request_id: 0, } } /// Returns this download asking peers for at most `response_bytes` per response. pub const fn with_response_bytes(mut self, response_bytes: u64) -> Self { - self.response_bytes = response_bytes; + self.context.set_response_bytes(response_bytes); self } @@ -77,7 +66,8 @@ where /// can commit without supplying their storage. A failed request leaves the progress in place. pub async fn next(&mut self, range: &VerifiedRange) -> Result { let (write, origin) = (range.write(), range.origin()); - let progress = self.factory.database_provider_ro()?.storage_progress(write, origin)?; + let progress = + self.context.factory().database_provider_ro()?.storage_progress(write, origin)?; let contracts = range.range().storage_batch(); let Some(first) = contracts.accounts().iter().position(|(account, _)| !progress.is_complete(*account)) @@ -88,20 +78,19 @@ where let batch = contracts.range(first..end).expect("positions are inside the batch"); let from = progress.resume_at(batch.accounts()[0].0).expect("first contract is incomplete"); - self.request_id = self.request_id.wrapping_add(1); let request = GetStorageRangesMessage { - request_id: self.request_id, + request_id: self.context.next_request_id(), root_hash: batch.state_root(), account_hashes: batch.accounts().iter().map(|(account, _)| *account).collect(), starting_hash: from.into(), limit_hash: MAX_HASH.into(), - response_bytes: self.response_bytes, + response_bytes: self.context.response_bytes(), }; let downloader = StorageRangeDownloader::new( - self.client.clone(), + self.context.client().clone(), request, &batch, - self.runtime.clone(), + self.context.runtime().clone(), )?; let ranges = match downloader.await? { StorageRangeOutcome::Verified(ranges) => ranges, @@ -110,39 +99,27 @@ where } }; let chunks = chunks(ranges, batch, from)?; - self.commit(write, origin, chunks).await.map(StorageRangeStep::Committed) - } - - // Commits one response's chunks together. - async fn commit( - &self, - write: SnapWrite, - origin: B256, - chunks: Vec, - ) -> Result { - let factory = self.factory.clone(); - self.runtime - .spawn_blocking(move || -> Result { - let provider = factory.database_provider_rw()?; + // One response's chunks commit together. + let committed = self + .context + .commit(move |provider| { let mut progress = StorageProgress::START; for chunk in chunks { progress = provider.commit_storage_chunk(write, origin, chunk)?; } - provider.commit()?; Ok(progress) }) - .await - .map_err(|error| SnapSyncError::Provider(ProviderError::other(error)))? + .await?; + Ok(StorageRangeStep::Committed(committed)) } } impl fmt::Debug for StorageRangeDownload { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_struct("StorageRangeDownload") - .field("response_bytes", &self.response_bytes) + .field("context", &self.context) .field("max_accounts", &self.max_accounts) - .field("request_id", &self.request_id) - .finish_non_exhaustive() + .finish() } } diff --git a/crates/snap-sync/src/storage/store.rs b/crates/snap-sync/src/storage/store.rs index cd2b8311d20..0bf957d62e3 100644 --- a/crates/snap-sync/src/storage/store.rs +++ b/crates/snap-sync/src/storage/store.rs @@ -3,7 +3,7 @@ //! Progress is tied to the attempt, its pivot and the range, so slots proved against a superseded //! root are never resumed. -use crate::{SnapAccountStore, SnapAttemptStore, SnapSyncError, SnapWrite}; +use crate::{common::SnapRecord, SnapAccountStore, SnapAttemptStore, SnapSyncError, SnapWrite}; use alloy_primitives::{B256, U256}; use reth_db_api::{ cursor::DbDupCursorRO, @@ -11,16 +11,9 @@ use reth_db_api::{ transaction::{DbTx, DbTxMut}, }; use reth_storage_api::{DBProvider, MetadataProvider, MetadataWriter, SnapAttemptId, StateWriter}; -use reth_storage_errors::provider::ProviderError; use reth_trie_common::{root::storage_root, HashedPostState, HashedStorage, EMPTY_ROOT_HASH}; use serde::{Deserialize, Serialize}; -// Metadata key of the storage progress record. -const PROGRESS_KEY: &str = "snap_storage_progress"; - -// Encoding version of the storage progress record this build writes. -const PROGRESS_VERSION: u32 = 1; - /// Persistence for contract storage downloaded ahead of its account range. /// /// Blanket-implemented over the node's writers, so slots and progress join the caller's @@ -165,32 +158,21 @@ struct StoredProgress { progress: StorageProgress, } +impl SnapRecord for StoredProgress { + const KEY: &'static str = "snap_storage_progress"; + const VERSION: u32 = 1; +} + impl StoredProgress { - // Serializes `progress` for `write`'s range at `origin` at this build's version. - fn encode( - write: SnapWrite, - origin: B256, - progress: StorageProgress, - ) -> Result, SnapSyncError> { - let stored = Self { - version: PROGRESS_VERSION, + // `progress` for `write`'s range at `origin` at this build's version. + const fn new(write: SnapWrite, origin: B256, progress: StorageProgress) -> Self { + Self { + version: Self::VERSION, attempt: write.attempt(), state_version: write.state_version(), origin, progress, - }; - Ok(serde_json::to_vec(&stored).map_err(ProviderError::other)?) - } - - // Checks the version first, so a record from another build is reported rather than misread. - fn decode(bytes: &[u8]) -> Result { - let value: serde_json::Value = - serde_json::from_slice(bytes).map_err(ProviderError::other)?; - let version = value.get("version").and_then(serde_json::Value::as_u64); - if version != Some(PROGRESS_VERSION as u64) { - return Err(SnapSyncError::UnsupportedStorageProgress { version }) } - Ok(serde_json::from_value(value).map_err(ProviderError::other)?) } fn belongs_to(&self, write: SnapWrite, origin: B256) -> bool { @@ -208,10 +190,7 @@ impl SnapStorageStore for T { origin: B256, ) -> Result { self.authorize_snap_write(write)?; - let Some(bytes) = self.get_metadata(PROGRESS_KEY)? else { - return Ok(StorageProgress::START) - }; - let stored = StoredProgress::decode(&bytes)?; + let Some(stored) = StoredProgress::read(self)? else { return Ok(StorageProgress::START) }; Ok(if stored.belongs_to(write, origin) { stored.progress } else { StorageProgress::START }) } @@ -241,7 +220,7 @@ impl SnapStorageStore for T { .with_storages([(chunk.account, HashedStorage::from_iter(chunk.slots))]) .into_sorted(); self.write_hashed_state(&state)?; - self.write_metadata(PROGRESS_KEY, StoredProgress::encode(write, origin, progress)?)?; + StoredProgress::new(write, origin, progress).write(self)?; Ok(progress) } } @@ -419,11 +398,11 @@ mod tests { for record in [br#"{"version":999}"#.to_vec(), b"{}".to_vec()] { let provider = factory.database_provider_rw().unwrap(); - provider.write_metadata(PROGRESS_KEY, record).unwrap(); + provider.write_metadata(StoredProgress::KEY, record).unwrap(); assert!(matches!( provider.storage_progress(write, B256::ZERO), - Err(SnapSyncError::UnsupportedStorageProgress { .. }) + Err(SnapSyncError::UnsupportedRecord { .. }) )); } } diff --git a/crates/snap-sync/src/test_utils.rs b/crates/snap-sync/src/test_utils.rs index 5c7376d2058..66d0c6ac48f 100644 --- a/crates/snap-sync/src/test_utils.rs +++ b/crates/snap-sync/src/test_utils.rs @@ -1,7 +1,7 @@ //! Fixtures shared by the crate's tests: headers, an account trie and a scripted snap client. use crate::{ - account::{DEFAULT_RESPONSE_BYTES, MAX_HASH}, + common::{DEFAULT_RESPONSE_BYTES, MAX_HASH}, SnapGeneration, SnapPivotPolicy, }; use alloy_consensus::Header; From d932ef2938637bbaca595a4675dbe2e6bab30136 Mon Sep 17 00:00:00 2001 From: joshieDo <93316087+joshieDo@users.noreply.github.com> Date: Tue, 15 Sep 2026 15:01:59 +0000 Subject: [PATCH 267/294] fix(prune): handle empty ranges in receipt log filtering (#27232) Co-authored-by: Codex Co-authored-by: Matthias Seitz --- .../src/segments/user/receipts_by_logs.rs | 120 +++++++++++++++++- 1 file changed, 113 insertions(+), 7 deletions(-) diff --git a/crates/prune/prune/src/segments/user/receipts_by_logs.rs b/crates/prune/prune/src/segments/user/receipts_by_logs.rs index 3b4955053e1..299f5804673 100644 --- a/crates/prune/prune/src/segments/user/receipts_by_logs.rs +++ b/crates/prune/prune/src/segments/user/receipts_by_logs.rs @@ -67,10 +67,9 @@ where let initial_last_pruned_block = last_pruned_block; let mut from_tx_number = match initial_last_pruned_block { - Some(block) => provider - .block_body_indices(block)? - .map(|block| block.last_tx_num() + 1) - .unwrap_or(0), + Some(block) => { + provider.block_body_indices(block)?.map(|block| block.next_tx_num()).unwrap_or(0) + } None => 0, }; @@ -139,7 +138,7 @@ where // Calculate the transaction range from this block range let tx_range_end = match provider.block_body_indices(end_block)? { - Some(body) => body.last_tx_num(), + Some(body) => body.next_tx_num(), None => { trace!( target: "pruner", @@ -149,7 +148,12 @@ where continue } }; - let tx_range = from_tx_number..=tx_range_end; + let tx_range = from_tx_number..tx_range_end; + if tx_range.is_empty() { + // Empty blocks advance the block checkpoint without consuming a transaction. + last_pruned_block = Some(end_block); + continue + } // Delete receipts, except the ones in the inclusion list let mut last_skipped_transaction = 0; @@ -238,13 +242,115 @@ mod tests { use reth_db_api::{cursor::DbCursorRO, tables, transaction::DbTx}; use reth_primitives_traits::InMemorySize; use reth_provider::{BlockReader, DBProvider, DatabaseProviderFactory, PruneCheckpointReader}; - use reth_prune_types::{PruneMode, PruneSegment, ReceiptsLogPruneConfig}; + use reth_prune_types::{ + PruneCheckpoint, PruneMode, PruneSegment, ReceiptsLogPruneConfig, + MINIMUM_UNWIND_SAFE_DISTANCE, + }; use reth_stages::test_utils::{StorageKind, TestStageDB}; use reth_testing_utils::generators::{ self, random_block_range, random_eoa_account, random_log, random_receipt, BlockRangeParams, }; use std::collections::BTreeMap; + #[test] + fn prune_receipts_by_logs_empty_prefix() { + for previous_block in [None, Some(1)] { + for retain_first in [false, true] { + let db = TestStageDB::default(); + let mut rng = generators::rng(); + let blocks = [ + random_block_range( + &mut rng, + 0..=1, + BlockRangeParams { tx_count: 0..1, ..Default::default() }, + ), + random_block_range( + &mut rng, + 2..=2, + BlockRangeParams { tx_count: 2..3, ..Default::default() }, + ), + ] + .concat(); + db.insert_blocks(blocks.iter(), StorageKind::Database(None)).unwrap(); + + let (address, _) = random_eoa_account(&mut rng); + let receipts = blocks[2] + .body() + .transactions + .iter() + .enumerate() + .map(|(id, tx)| { + let mut receipt = random_receipt(&mut rng, tx, Some(0), None); + if id == 0 && retain_first { + receipt.logs.push(random_log(&mut rng, Some(address), Some(1))); + } + (id as u64, receipt) + }) + .collect::>(); + db.insert_receipts(receipts.clone()).unwrap(); + + let provider = db.factory.database_provider_rw().unwrap(); + let result = ReceiptsByLogs::new(ReceiptsLogPruneConfig(BTreeMap::from([( + address, + PruneMode::Before(2), + )]))) + .prune( + &provider, + PruneInput { + previous_checkpoint: previous_block.map(|block| PruneCheckpoint { + block_number: Some(block), + tx_number: None, + prune_mode: PruneMode::Before(2), + }), + to_block: MINIMUM_UNWIND_SAFE_DISTANCE + 2, + limiter: PruneLimiter::default(), + }, + ) + .unwrap(); + assert!(result.progress.is_finished()); + assert_eq!(result.pruned, if retain_first { 1 } else { 2 }); + provider.commit().unwrap(); + let expected = if retain_first { vec![receipts[0].clone()] } else { vec![] }; + assert_eq!(db.table::().unwrap(), expected); + } + } + } + + #[test] + fn prune_receipts_by_logs_empty_chain() { + let db = TestStageDB::default(); + let blocks = random_block_range( + &mut generators::rng(), + 0..=2, + BlockRangeParams { tx_count: 0..1, ..Default::default() }, + ); + db.insert_blocks(blocks.iter(), StorageKind::Database(None)).unwrap(); + let provider = db.factory.database_provider_rw().unwrap(); + let result = ReceiptsByLogs::new(ReceiptsLogPruneConfig(BTreeMap::from([( + alloy_primitives::Address::ZERO, + PruneMode::Before(2), + )]))) + .prune( + &provider, + PruneInput { + previous_checkpoint: None, + to_block: MINIMUM_UNWIND_SAFE_DISTANCE + 2, + limiter: PruneLimiter::default(), + }, + ) + .unwrap(); + assert!(result.progress.is_finished()); + assert_eq!(result.pruned, 0); + assert_eq!( + provider.get_prune_checkpoint(PruneSegment::ContractLogs).unwrap(), + Some(PruneCheckpoint { + block_number: Some(2), + tx_number: None, + prune_mode: PruneMode::Before(2), + }) + ); + } + #[test] fn prune_receipts_by_logs() { reth_tracing::init_test_tracing(); From 5d0ea55c5246b883fcefda2ebe9ba03a240d4d03 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Tue, 15 Sep 2026 13:11:38 -0400 Subject: [PATCH 268/294] perf(engine): run storage trie updates inside checked-out jobs (#27156) Co-authored-by: Matthias Seitz <19890894+mattsse@users.noreply.github.com> --- .../tree/state_root_strategy/sparse_trie.rs | 1392 +++++++++++++++-- crates/trie/sparse/src/state.rs | 35 + 2 files changed, 1263 insertions(+), 164 deletions(-) diff --git a/crates/engine/tree/src/tree/state_root_strategy/sparse_trie.rs b/crates/engine/tree/src/tree/state_root_strategy/sparse_trie.rs index c9647b57122..566df3e58be 100644 --- a/crates/engine/tree/src/tree/state_root_strategy/sparse_trie.rs +++ b/crates/engine/tree/src/tree/state_root_strategy/sparse_trie.rs @@ -1,6 +1,10 @@ //! Sparse Trie task related functionality. -use std::sync::Arc; +use std::{ + any::Any, + panic::{self, AssertUnwindSafe}, + sync::Arc, +}; use super::{evm_state_to_hashed_post_state, StateRootComputeOutcome, StateRootMessage}; use alloy_primitives::{ @@ -10,7 +14,6 @@ use alloy_primitives::{ use alloy_rlp::{Decodable, Encodable}; use crossbeam_channel::{Receiver as CrossbeamReceiver, Sender as CrossbeamSender}; use metrics::{Gauge, Histogram}; -use rayon::iter::{IntoParallelIterator, ParallelIterator}; use reth_metrics::Metrics; use reth_primitives_traits::{Account, FastInstant as Instant}; use reth_tasks::Runtime; @@ -18,7 +21,7 @@ use reth_trie::{ updates::TrieUpdates, DecodedMultiProofV2, HashedPostState, TrieAccount, EMPTY_ROOT_HASH, TRIE_ACCOUNT_RLP_MAX_SIZE, }; -use reth_trie_common::{MultiProofTargetsV2, ProofV2Target, ProofV2TargetParent}; +use reth_trie_common::{MultiProofTargetsV2, ProofTrieNodeV2, ProofV2Target, ProofV2TargetParent}; use reth_trie_parallel::{ error::StateRootTaskError, proof_task::{ @@ -27,13 +30,95 @@ use reth_trie_parallel::{ }, }; use reth_trie_sparse::{ - errors::{SparseStateTrieErrorKind, SparseTrieErrorKind, SparseTrieResult}, + errors::{ + SparseStateTrieErrorKind, SparseStateTrieResult, SparseTrieErrorKind, SparseTrieResult, + }, ArenaParallelSparseTrie, DeferredDrops, LeafUpdate, RevealableSparseTrie, SparseStateTrie, SparseTrie, TrieNodeEpoch, }; use tracing::{debug, debug_span, error, instrument, trace_span}; -/// Sparse trie task implementation that uses in-memory sparse trie data to schedule proof fetching. +/// Computes a block's state root by applying streamed updates to an in-memory [`SparseStateTrie`]. +/// Updates that reach a blinded node stay pending while the task fetches the missing proof. The +/// account trie stays on this task's thread; storage tries can be moved into independent rayon +/// jobs. +/// +/// # Message flow +/// +/// The task coordinates three incoming streams: +/// +/// 1. **Execution and prewarming:** [`Self::run_hashing_task`] receives [`StateRootMessage`]s, +/// hashes raw state updates, and forwards [`SparseTrieTaskMessage`]s to [`Self::updates`] in +/// receive order. `HashedState` supplies changed values; `PrefetchProofs` supplies +/// [`LeafUpdate::Touched`] hints that must not overwrite changed values. The task buffers these +/// leaf updates before applying them in batches. +/// 2. **Proof workers:** blinded paths become [`MultiProofTargetsV2`] requests dispatched through +/// [`ProofWorkerHandle`]. Each request carries a [`ProofResultContext`] with a sender for +/// [`Self::proof_result_rx`]. The task coalesces returned [`ProofResultMessage`]s, reveals +/// account proofs locally, and queues storage proofs on the corresponding address' +/// [`StorageTrieState`]. +/// 3. **Storage jobs:** [`Self::spawn_storage_jobs`] moves each [`StorageTrieWork`] into a rayon +/// closure. A pass reveals queued proofs, applies unblocked leaves, and hashes the storage trie +/// once its pending leaves are drained. It sends [`StorageJobMessage::Done`] via +/// [`Self::storage_done_tx`] to [`Self::storage_done_rx`], returning ownership of the payload +/// along with newly discovered proof targets, cache counters, and any error. Small or +/// target-only passes run inline and apply the same output directly. +/// +/// [`Self::run`] selects between these streams and cancellation. After handling a message, +/// [`Self::make_progress`] collects queued job returns, applies buffered updates, dispatches proof +/// requests and storage jobs, and promotes account updates whose storage roots are available. +/// Proof workers fetch nodes; storage jobs reveal and modify the owned storage trie. Neither +/// mutates the account trie. +/// +/// # Storage ownership +/// +/// On first use, an address' storage trie moves out of [`SparseStateTrie`] into an idle entry. +/// There is at most one job per address, and no shared mutable access to its trie: +/// +/// ```text +/// dispatch: move StorageTrieWork to rayon +/// Idle --------------------------------------------------------> InFlight +/// (task owns trie) (job owns trie) +/// ^ | +/// +--------------------------------------------------------------+ +/// Done: restore payload and merge buffered arrivals +/// ``` +/// +/// In `Idle`, new proofs and leaf updates make the payload ready for another pass. An inline pass +/// keeps it in `Idle`. In `InFlight`, the task holds only [`InFlightStorage`] buffers for new +/// arrivals; the job owns the trie, earlier pending leaves, proofs, and fetched-target cache. +/// When `Done` arrives, newer changed values and deletions override older pending values, while +/// touch hints preserve them. Buffered work can then trigger another pass. +/// +/// `Idle` describes ownership, not completion: a payload can still have leaves blocked on a proof. +/// Blocked leaves alone do not trigger another pass; new updates or that address' own proof nodes +/// make it ready again. Account updates wait while their storage trie is in flight or still has +/// pending leaves, so they cannot use an incomplete storage root. +/// +/// # Task lifecycle +/// +/// ```text +/// Streaming -- FinishedStateUpdates --> Draining -- no pending trie work --> Finalize +/// | | +/// +------------- cancellation ---------+--> Canceled +/// ``` +/// +/// The finish marker publishes the accumulated [`HashedPostState`] through +/// [`Self::final_hashed_state_tx`] and ends consumption of the update stream. Draining continues +/// to receive proof results and storage-job returns; late prefetch hints are ignored. Closing +/// the update channel before the marker is an error, while closing it afterward is harmless. +/// +/// Finalization waits for all leaf updates, account promotions, and checked-out storage tries. +/// It restores the storage tries to [`SparseStateTrie`], computes the final root and trie updates, +/// and returns [`StateRootComputeOutcome`]. The caller publishes that result before consuming +/// the task with [`Self::into_trie_for_reuse`], which also releases the retained job buffers. +/// +/// Dropping the consumer's cancel guard makes [`Self::run`] return +/// [`StateRootTaskError::Canceled`]. Teardown through [`Self::into_cleared_trie`] disconnects the +/// storage return channel, so late jobs drop their payloads when sending fails. Proof and trie +/// errors fail the calculation; [`StorageJobMessage::Panicked`] resumes the worker's panic on +/// this task's thread. During draining, pending updates with no remaining source of progress +/// produce [`StateRootTaskError::Stalled`]. pub(super) struct SparseTrieCacheTask { /// Sender for proof results. proof_result_tx: ProofResultSender, @@ -66,8 +151,6 @@ pub(super) struct SparseTrieCacheTask, - /// Storage trie updates. hashed address -> slot -> update. - storage_updates: B256Map>, /// Account updates that are buffered but were not yet applied to the trie. new_account_updates: B256Map, @@ -91,9 +174,6 @@ pub(super) struct SparseTrieCacheTask, - /// Cache of storage proof targets that have already been fetched/requested from the proof - /// workers. Account to slot to the broadest requested parent context. - fetched_storage_targets: B256Map>, /// Reusable buffer for RLP encoding of accounts. account_rlp_buf: Vec, /// Whether the last state update has been received. @@ -110,6 +190,22 @@ pub(super) struct SparseTrieCacheTask>, + /// Storage job buffers kept alive after returning their tries, until the result is sent. + storage_to_drop: B256Map>, + /// Number of storage tries currently owned by a job running off this thread. + storage_in_flight: usize, + /// Sender handed to storage jobs. Kept alive by the task so the receiver never disconnects. + storage_done_tx: CrossbeamSender>, + /// Receives storage tries coming back from jobs spawned by this task. + storage_done_rx: CrossbeamReceiver>, /// Number of pending execution/prewarming updates received but not yet passed to /// `update_leaves`. pending_updates: usize, @@ -129,7 +225,7 @@ pub(super) struct SparseTrieCacheTask SparseTrieCacheTask where A: SparseTrie + Default, - S: SparseTrie + Default + Clone, + S: SparseTrie + Default + Clone + 'static, { /// Creates a new sparse trie, pre-populating with an existing [`SparseStateTrie`]. #[expect(clippy::too_many_arguments)] @@ -148,6 +244,7 @@ where chunk_size: usize, ) -> Self { let (hashed_state_tx, hashed_state_rx) = crossbeam_channel::unbounded(); + let (storage_done_tx, storage_done_rx) = crossbeam_channel::unbounded(); let parent_span = tracing::Span::current(); let hashing_metrics = metrics.clone(); @@ -169,12 +266,10 @@ where chunk_size, max_targets_for_chunking: DEFAULT_MAX_TARGETS_FOR_CHUNKING, account_updates: Default::default(), - storage_updates: Default::default(), new_account_updates: Default::default(), new_storage_updates: Default::default(), pending_account_updates: Default::default(), fetched_account_targets: Default::default(), - fetched_storage_targets: Default::default(), account_rlp_buf: Vec::with_capacity(TRIE_ACCOUNT_RLP_MAX_SIZE), finished_state_updates: Default::default(), account_cache_hits: 0, @@ -183,6 +278,11 @@ where storage_cache_misses: 0, pending_targets: Default::default(), in_flight_proof_batches: 0, + storage: Default::default(), + storage_to_drop: Default::default(), + storage_in_flight: 0, + storage_done_tx, + storage_done_rx, pending_updates: Default::default(), initial_updates_applied: false, final_hashed_state: Default::default(), @@ -233,6 +333,10 @@ where /// /// Should be called after the state root result has been sent. pub(super) fn into_trie_for_reuse(self) -> (SparseStateTrie, DeferredDrops) { + debug_assert!( + self.storage.is_empty(), + "storage tries must be back before the trie is preserved" + ); let Self { mut trie, .. } = self; let deferred = trie.take_deferred_drops(); (trie, deferred) @@ -243,7 +347,14 @@ where /// Use this when the payload was invalid or cancelled - we don't want to preserve /// potentially invalid trie state, but we keep the allocations for reuse. pub(super) fn into_cleared_trie(self) -> (SparseStateTrie, DeferredDrops) { - let Self { mut trie, .. } = self; + let Self { mut trie, storage_done_rx, storage, .. } = self; + // Disconnect first so tries still out with a job are dropped by that job instead of here. + drop(storage_done_rx); + for (address, state) in storage { + if let StorageTrieState::Idle(work) = state { + trie.insert_storage_trie(address, work.trie); + } + } trie.clear(); let deferred = trie.take_deferred_drops(); (trie, deferred) @@ -254,7 +365,9 @@ where /// This waits for new incoming [`SparseTrieTaskMessage`]s, applies updates /// to the trie and schedules proof fetching when needed. /// - /// This concludes once the last state update has been received and processed. + /// After the finish marker, drains the remaining proof and storage work until all trie updates + /// are applied and all checked-out storage tries have returned. See the [task + /// lifecycle](Self#task-lifecycle). #[instrument( name = "SparseTrieCacheTask::run", level = "debug", @@ -303,6 +416,18 @@ where }; self.on_proof_results(result, &mut t)?; }, + recv(self.storage_done_rx) -> message => { + let wake = Instant::now(); + total_idle_time += wake.duration_since(idle_start); + self.metrics + .sparse_trie_channel_wait_duration_histogram + .record(wake.duration_since(t)); + + let Ok(returned) = message else { + unreachable!("we own the sender half") + }; + self.on_storage_job_message(returned)?; + }, recv(self.cancel_rx) -> _ => return Err(StateRootTaskError::Canceled), } @@ -311,9 +436,9 @@ where } // Draining phase: the marker is the last message read from the updates channel, so - // after it only proof results and cancellation can occur. The channel closing when - // the producers drop their senders is not observed here, and late best-effort hints - // are ignored: with all updates known, prefetching has nothing left to help. + // after it only proof results, storage-job returns, and cancellation can occur. The + // update channel closing when producers drop their senders is not observed here, and late + // hints are ignored: with all updates known, prefetching has nothing left to help. while !done { let mut t = Instant::now(); crossbeam_channel::select_biased! { @@ -330,6 +455,18 @@ where }; self.on_proof_results(result, &mut t)?; }, + recv(self.storage_done_rx) -> message => { + let wake = Instant::now(); + total_idle_time += wake.duration_since(idle_start); + self.metrics + .sparse_trie_channel_wait_duration_histogram + .record(wake.duration_since(t)); + + let Ok(returned) = message else { + unreachable!("we own the sender half") + }; + self.on_storage_job_message(returned)?; + }, recv(self.cancel_rx) -> _ => return Err(StateRootTaskError::Canceled), } @@ -337,11 +474,16 @@ where idle_start = Instant::now(); } + debug_assert_eq!( + self.storage_in_flight, 0, + "completion must wait for every checked out storage trie" + ); self.metrics.sparse_trie_idle_time_seconds.record(total_idle_time.as_secs_f64()); debug!(target: "engine::root", "All proofs processed, ending calculation"); let start = Instant::now(); + self.return_storage_tries(); let (state_root, trie_updates) = match self.trie.root_with_updates(self.new_epoch) { Ok(result) => result, Err(err) @@ -412,6 +554,10 @@ where /// Messages queued after the finish marker are best-effort hints and are not actionable. /// Returns `true` once the finish marker was received and all pending trie work is done. fn make_progress(&mut self) -> Result { + // Absorb a whole burst of returns before the scans below, so one batch of storage jobs + // costs one promotion pass rather than one per trie. + self.drain_returned_storage_tries()?; + let updates_queued = !self.finished_state_updates && !self.updates.is_empty(); if !updates_queued && self.proof_result_rx.is_empty() { @@ -512,9 +658,8 @@ where fn on_hashed_state_update(&mut self, hashed_state_update: HashedPostState) { for (&address, storage) in &hashed_state_update.storages { if !storage.storage.is_empty() { - // Look up outer maps once per address instead of once per slot. + // Look up the outer map once per address instead of once per slot. let new_updates = self.new_storage_updates.entry(address).or_default(); - let mut existing_updates = self.storage_updates.get_mut(&address); for (&slot, &value) in &storage.storage { let encoded = if value.is_zero() { @@ -523,11 +668,6 @@ where alloy_rlp::encode_fixed_size(&value).to_vec() }; new_updates.insert(slot, LeafUpdate::Changed(encoded)); - - // Remove an existing storage update if it exists. - if let Some(ref mut existing) = existing_updates { - existing.remove(&slot); - } } } @@ -556,11 +696,52 @@ where } fn on_proof_result(&mut self, result: DecodedMultiProofV2) -> Result<(), StateRootTaskError> { - self.trie - .reveal_decoded_multiproof_v2(result) + self.reveal_proof_result(result) .map_err(|e| StateRootTaskError::Other(format!("could not reveal multiproof: {e:?}"))) } + /// Reveals a proof batch. + /// + /// The account trie is revealed here because it can never be checked out. Storage proof nodes + /// are queued on their address' entry and revealed by the job that runs for it next. + fn reveal_proof_result(&mut self, result: DecodedMultiProofV2) -> SparseStateTrieResult<()> { + let DecodedMultiProofV2 { account_proofs, storage_proofs } = result; + + self.queue_storage_proofs(storage_proofs); + // Get the workers going before spending this thread on the account trie. + self.run_ready_storage_work()?; + + self.trie.reveal_account_proof_nodes(account_proofs) + } + + /// Queues storage proof nodes on the entry for the address they belong to. + fn queue_storage_proofs(&mut self, storage_proofs: B256Map>) { + let mut revealed_nodes = 0; + for (address, mut nodes) in storage_proofs { + if nodes.is_empty() { + continue; + } + revealed_nodes += nodes.len(); + + match self.storage_trie_state_mut(address) { + StorageTrieState::Idle(work) => work.queue_proofs(&mut nodes), + StorageTrieState::InFlight(in_flight) => in_flight.proofs.append(&mut nodes), + } + } + self.trie.record_revealed_storage_nodes(revealed_nodes); + } + + /// Returns the storage trie state for `address`, checking its storage trie out of the state + /// trie the first time the address is touched by this block. + fn storage_trie_state_mut(&mut self, address: B256) -> &mut StorageTrieState { + let Self { storage, trie, .. } = self; + storage.entry(address).or_insert_with(|| { + StorageTrieState::Idle(Box::new(StorageTrieWork::new( + trie.take_or_create_storage_trie(&address), + ))) + }) + } + fn on_proof_result_message( &mut self, message: ProofResultMessage, @@ -574,7 +755,15 @@ where Ok(result) } + /// Applies buffered input and runs ready storage work, including work buffered while a job + /// was in flight even if no new input messages were received. fn process_new_updates(&mut self) -> SparseTrieResult<()> { + self.apply_new_updates()?; + self.run_ready_storage_work() + } + + /// Applies new account updates and queues new storage updates on their address' entry. + fn apply_new_updates(&mut self) -> SparseTrieResult<()> { if self.pending_updates == 0 { return Ok(()); } @@ -583,32 +772,29 @@ where self.pending_updates = 0; self.initial_updates_applied = true; - // Firstly apply all new storage and account updates to the tries. - self.process_leaf_updates(true)?; + // Queue the new storage updates on their entries; the jobs apply them to the tries. + let Self { storage, trie, new_storage_updates, .. } = self; + for (address, new) in new_storage_updates.drain() { + if new.is_empty() { + continue; + } - for (address, mut new) in self.new_storage_updates.drain() { - match self.storage_updates.entry(address) { - Entry::Vacant(entry) => { - entry.insert(new); // insert the whole map at once, no per-slot loop - } - Entry::Occupied(mut entry) => { - let updates = entry.get_mut(); - for (slot, new) in new.drain() { - match updates.entry(slot) { - Entry::Occupied(mut slot_entry) => { - if new.is_changed() { - slot_entry.insert(new); - } - } - Entry::Vacant(slot_entry) => { - slot_entry.insert(new); - } - } - } + let state = storage.entry(address).or_insert_with(|| { + StorageTrieState::Idle(Box::new(StorageTrieWork::new( + trie.take_or_create_storage_trie(&address), + ))) + }); + match state { + StorageTrieState::Idle(work) => work.queue_updates(new), + StorageTrieState::InFlight(in_flight) => { + merge_leaf_updates(&mut in_flight.updates, new) } } } + // Apply the new account updates to the accounts trie, which is never checked out. + self.process_account_leaf_updates(true)?; + for (address, new) in self.new_account_updates.drain() { match self.account_updates.entry(address) { Entry::Occupied(mut entry) => { @@ -625,59 +811,85 @@ where Ok(()) } - /// Applies all account and storage leaf updates to corresponding tries and collects any new - /// multiproof targets. + /// Runs a pass over every idle storage trie that has something to do: proof nodes to reveal, + /// leaf updates that were not tried against the current state of the trie yet. + /// + /// Each pass owns its address' payload for its whole duration, so the passes are independent + /// of each other and of this thread. Most of them are handed to the rayon pool; the ones that + /// would cost more in handoff than in work stay here, see [`StorageTrieWork::is_target_only`] + /// and [`INLINE_STORAGE_WORK_UNITS`]. #[instrument( level = "trace", target = "engine::tree::payload_processor::sparse_trie", skip_all )] - fn process_leaf_updates(&mut self, new: bool) -> SparseTrieResult<()> { - let storage_updates = - if new { &mut self.new_storage_updates } else { &mut self.storage_updates }; - - // Process all storage updates, skipping tries with no pending updates. - let span = trace_span!("process_storage_leaf_updates").entered(); - for (address, updates) in storage_updates { - if updates.is_empty() { + fn run_ready_storage_work(&mut self) -> SparseTrieResult<()> { + let mut ready = Vec::new(); + let mut job_units = 0; + for (address, state) in &self.storage { + let StorageTrieState::Idle(work) = state else { continue }; + if !work.has_work() { continue; } - let _enter = trace_span!(target: "engine::tree::payload_processor::sparse_trie", parent: &span, "storage_trie_leaf_updates", a=%address).entered(); - - let trie = self.trie.get_or_create_storage_trie_mut(*address); - let fetched = self.fetched_storage_targets.entry(*address).or_default(); - let mut targets = Vec::new(); + if !work.is_target_only() { + job_units += work.proofs.len() + work.pending.len(); + } + ready.push(*address); + } + if ready.is_empty() { + return Ok(()) + } - let updates_len_before = updates.len(); - trie.update_leaves(updates, |path, parent| match fetched.entry(path) { - Entry::Occupied(mut entry) => { - if parent < *entry.get() { - entry.insert(parent); - targets.push(ProofV2Target::new(path).with_parent(parent)); - } - } - Entry::Vacant(entry) => { - entry.insert(parent); - targets.push(ProofV2Target::new(path).with_parent(parent)); - } - })?; - let updates_len_after = updates.len(); - self.storage_cache_hits += (updates_len_before - updates_len_after) as u64; - self.storage_cache_misses += updates_len_after as u64; + let inline_round = job_units <= INLINE_STORAGE_WORK_UNITS; + let new_epoch = self.new_epoch; + let retain_updates = self.trie.retains_updates(); + let started = Instant::now(); + let mut jobs = Vec::new(); + for address in ready { + let state = self.storage.get_mut(&address).expect("entry was just seen"); + let StorageTrieState::Idle(work) = state else { + unreachable!("an idle trie is only checked out from here") + }; - if !targets.is_empty() { - self.pending_targets.extend_storage_targets(address, targets); + if inline_round || work.is_target_only() { + let output = work.run(new_epoch, retain_updates); + self.apply_storage_output(address, output)?; + continue; } - } - drop(span); + let StorageTrieState::Idle(work) = core::mem::replace( + state, + StorageTrieState::InFlight(Box::new(InFlightStorage::new(started))), + ) else { + unreachable!("just matched") + }; + self.storage_in_flight += 1; + jobs.push(StorageTrieJob { address, work }); + } - // Process account trie updates and fill the account targets. - self.process_account_leaf_updates(new)?; + self.spawn_storage_jobs(jobs); Ok(()) } + /// Records what a storage pass produced: the proof targets it discovered and its cache + /// counters. Returns the error the pass stopped at, if any. + fn apply_storage_output( + &mut self, + address: B256, + output: StorageWorkOutput, + ) -> SparseTrieResult<()> { + let StorageWorkOutput { targets, cache_hits, cache_misses, result } = output; + + self.storage_cache_hits += cache_hits; + self.storage_cache_misses += cache_misses; + if !targets.is_empty() { + self.pending_targets.extend_storage_targets(&address, targets); + } + + result + } + /// Invokes `update_leaves` for the accounts trie and collects any new targets. /// /// Returns whether any updates were drained (applied to the trie). @@ -716,96 +928,145 @@ where Ok(updates_len_after < updates_len_before) } - /// Computes storage roots for accounts whose storage updates are fully drained. - /// - /// For each storage trie T that: - /// 1. was modified in the current block, - /// 2. all the storage updates are fully drained, - /// 3. but the storage root hasn't been updated yet, + /// Hands checked out storage tries to the rayon pool in chunks. /// - /// we trigger state root computation on a rayon pool. - fn compute_drained_storage_roots(&mut self) { - struct SendStorageTriePtr(*mut RevealableSparseTrie); - // SAFETY: this wrapper only forwards the pointer across rayon; deref invariants are - // documented at the use site below. - unsafe impl Send for SendStorageTriePtr {} - - let mut tries_to_compute_roots: Vec<(B256, SendStorageTriePtr)> = Vec::new(); - for (address, updates) in &self.storage_updates { - if updates.is_empty() && - let Some(trie) = self.trie.storage_tries_mut().get_mut(address) && - !trie.is_root_cached() - { - tries_to_compute_roots.push((*address, SendStorageTriePtr(trie))); - } - } - - if tries_to_compute_roots.is_empty() { + /// Chunking amortizes the spawn cost over a batch, while each trie is still sent back on its + /// own so promotion does not wait for the rest of the chunk. + fn spawn_storage_jobs(&self, mut jobs: Vec>) { + if jobs.is_empty() { return; } - let parent_span = - debug_span!("compute_drained_storage_roots", n = tries_to_compute_roots.len()); + let parent_span = debug_span!("spawn_storage_jobs", n = jobs.len()); + let chunk_len = storage_job_chunk_len(jobs.len()); let new_epoch = self.new_epoch; - tries_to_compute_roots.into_par_iter().for_each(|(address, SendStorageTriePtr(trie))| { - let span = if tracing::enabled!(tracing::Level::TRACE) { - debug_span!( + let retain_updates = self.trie.retains_updates(); + while !jobs.is_empty() { + let chunk = jobs.split_off(jobs.len().saturating_sub(chunk_len)); + let storage_done_tx = self.storage_done_tx.clone(); + let parent_span = parent_span.clone(); + rayon::spawn(move || { + let _enter = debug_span!( target: "engine::tree::payload_processor::sparse_trie", parent: &parent_span, - "storage_root", - ?address + "storage_jobs", + n = chunk.len(), ) - } else { - debug_span!( - target: "engine::tree::payload_processor::sparse_trie", - parent: &parent_span, - "storage_root", - ) - }; - let _enter = span.entered(); - // SAFETY: - // - pointers are created from `storage_tries_mut().get_mut(address)` above; - // - `storage_updates` is a map, so addresses are unique; - // - we do not insert/remove entries between pointer collection and use, so pointers - // stay valid and map reallocation cannot occur; - // - each pointer is consumed by at most one rayon task, so no aliasing mutable access. - unsafe { - (*trie) - .root(new_epoch) - .expect("updates are drained, trie should be revealed by now") + .entered(); + for job in chunk { + let address = job.address; + let message = match panic::catch_unwind(AssertUnwindSafe(|| { + job.run(new_epoch, retain_updates) + })) { + Ok(done) => StorageJobMessage::Done(done), + Err(payload) => StorageJobMessage::Panicked { address, payload }, + }; + if storage_done_tx.send(message).is_err() { + // Nobody is waiting for the result anymore, drop the rest here. + return; + } + } + }); + } + } + + /// Handles a message from a storage job: takes a finished payload back, or resumes a panic + /// that happened on the job thread here, where it fails this task like an inline panic would. + fn on_storage_job_message(&mut self, message: StorageJobMessage) -> SparseTrieResult<()> { + match message { + StorageJobMessage::Done(done) => self.on_storage_trie_returned(done), + StorageJobMessage::Panicked { address, payload } => { + if self.storage.remove(&address).is_some() { + self.storage_in_flight -= 1; + } + panic::resume_unwind(payload) + } + } + } + + /// Puts a payload back into its entry once its job finished, folding in everything that + /// arrived for the address while it was gone. + fn on_storage_trie_returned(&mut self, done: StorageTrieJobDone) -> SparseTrieResult<()> { + let StorageTrieJobDone { address, work, output } = done; + + let Entry::Occupied(mut entry) = self.storage.entry(address) else { + unreachable!("a returned payload was checked out of its entry") + }; + let StorageTrieState::InFlight(in_flight) = entry.insert(StorageTrieState::Idle(work)) + else { + unreachable!("a checked out address stays in flight until its job returns") + }; + let started = in_flight.started; + let StorageTrieState::Idle(work) = entry.into_mut() else { unreachable!("just inserted") }; + work.take_buffered(*in_flight); + self.storage_in_flight -= 1; + self.metrics.sparse_trie_storage_job_duration_histogram.record(started.elapsed()); + + self.apply_storage_output(address, output) + } + + /// Takes back every payload whose job has already finished, without blocking. + fn drain_returned_storage_tries(&mut self) -> SparseTrieResult<()> { + while let Ok(message) = self.storage_done_rx.try_recv() { + self.on_storage_job_message(message)?; + } + + Ok(()) + } + + /// Puts every storage trie back into the sparse state trie. + /// + /// The final root and everything after it - trie updates, preservation, pruning - reads the + /// storage tries through [`SparseStateTrie`], so no trie may remain in the task's storage map. + fn return_storage_tries(&mut self) { + let mut storage = core::mem::take(&mut self.storage); + for (address, state) in &mut storage { + let StorageTrieState::Idle(work) = state else { + unreachable!("completion must wait for every storage job") }; - }); + self.trie.insert_storage_trie(*address, core::mem::take(&mut work.trie)); + } + // Keep the maps and proof buffers until the result has been published. + self.storage_to_drop = storage; } - /// Iterates through all storage tries for which all updates were processed, computes their - /// storage roots, and promotes corresponding pending account updates into proper leaf updates - /// for accounts trie. + /// Promotes the pending account updates whose storage root is already available into proper + /// leaf updates for the accounts trie. + /// + /// Accounts whose trie is still checked out stay pending and are promoted by a later call, + /// once the payload has come back. #[instrument( level = "trace", target = "engine::tree::payload_processor::sparse_trie", skip_all )] fn promote_pending_account_updates(&mut self) -> SparseTrieResult<()> { - self.process_leaf_updates(false)?; + self.process_account_leaf_updates(false)?; if self.pending_account_updates.is_empty() { return Ok(()); } - self.compute_drained_storage_roots(); - + let new_epoch = self.new_epoch; loop { let span = trace_span!("promote_updates", promoted = tracing::field::Empty).entered(); // Now handle pending account updates that can be upgraded to a proper update. let account_rlp_buf = &mut self.account_rlp_buf; let mut num_promoted = 0; self.pending_account_updates.retain(|addr, account| { - if let Some(updates) = self.storage_updates.get(addr) { - if !updates.is_empty() { + let updated_storage = match self.storage.get_mut(addr) { + // The payload is out with a job, so the root is not readable yet. + Some(StorageTrieState::InFlight(_)) => return true, + Some(StorageTrieState::Idle(work)) if work.updated => Some(work), + _ => None, + }; + + if let Some(work) = updated_storage { + if !work.pending.is_empty() { // If account has pending storage updates, it is still pending. return true; } else if let Some(account) = account.take() { - let storage_root = self.trie.storage_root(addr, self.new_epoch).expect("updates are drained, storage trie should be revealed by now"); + let storage_root = work.trie.root(new_epoch).expect("updates are drained, storage trie should be revealed by now"); let encoded = encode_account_leaf_value(account, storage_root, account_rlp_buf); self.account_updates.insert(*addr, LeafUpdate::Changed(encoded)); num_promoted += 1; @@ -834,7 +1095,14 @@ where (account, storage_root) } else { - (trie_account.map(Into::into), self.trie.storage_root(addr, self.new_epoch).expect("account had storage updates that were applied to its trie, storage root must be revealed by now")) + // An address with no entry was not touched by this block, so its trie, if any, + // is still in the state trie with the root it was preserved with. + let storage_root = match self.storage.get_mut(addr) { + Some(StorageTrieState::Idle(work)) => work.trie.root(new_epoch), + Some(StorageTrieState::InFlight(_)) => unreachable!("returned above"), + None => self.trie.storage_root(addr, new_epoch), + }; + (trie_account.map(Into::into), storage_root.expect("account had storage updates that were applied to its trie, storage root must be revealed by now")) }; let encoded = encode_account_leaf_value(account, storage_root, account_rlp_buf); @@ -907,13 +1175,22 @@ where fn has_pending_sparse_trie_updates(&self) -> bool { !self.account_updates.is_empty() || - self.storage_updates.values().any(|updates| !updates.is_empty()) || - !self.pending_account_updates.is_empty() + !self.pending_account_updates.is_empty() || + self.storage.values().any(|state| state.is_pending()) + } + + /// Returns whether any idle storage trie has a pass to run, which is progress that has not been + /// made yet rather than a stall. + fn has_ready_storage_work(&self) -> bool { + self.storage.values().any(|state| match state { + StorageTrieState::Idle(work) => work.has_work(), + StorageTrieState::InFlight(_) => false, + }) } /// Errors when pending trie updates remain but nothing can deliver them: no update - /// messages are queued, no proof targets are queued or in flight, and no proof results - /// are waiting. + /// messages are queued, no proof targets are queued or in flight, no proof results + /// are waiting, and no storage pass is running or waiting to run. /// /// `updates_queued` is passed in instead of reading `self.updates` directly, because in /// the draining phase the updates channel is not read anymore and may hold ignored late @@ -925,6 +1202,8 @@ where self.pending_targets.is_empty() && self.in_flight_proof_batches == 0 && self.proof_result_rx.is_empty() && + self.storage_in_flight == 0 && + !self.has_ready_storage_work() && self.has_pending_sparse_trie_updates() { const MAX_STALLED_PROOF_TARGETS_TO_LOG: usize = 5; @@ -940,17 +1219,16 @@ where account_targets.truncate(MAX_STALLED_PROOF_TARGETS_TO_LOG); let mut storage_targets = self - .storage_updates + .storage .iter() - .flat_map(|(address, updates)| { - let fetched_targets = self.fetched_storage_targets.get(address); - updates.keys().map(move |target| { - ( - *address, - *target, - fetched_targets.and_then(|targets| targets.get(target)).copied(), - ) - }) + .filter_map(|(address, state)| match state { + StorageTrieState::Idle(work) => Some((address, work)), + StorageTrieState::InFlight(_) => None, + }) + .flat_map(|(address, work)| { + work.pending + .keys() + .map(move |target| (*address, *target, work.fetched.get(target).copied())) }) .collect::>(); storage_targets.sort_unstable(); @@ -973,6 +1251,269 @@ where } } +/// State of one address' storage trie in the sparse trie task. +enum StorageTrieState { + /// Nothing is running for this address, so its payload can be read and handed to a job. + Idle(Box>), + /// A job owns the payload. Everything arriving for the address meanwhile is buffered here + /// and folded back in when the job returns. + InFlight(Box), +} + +impl StorageTrieState { + /// Returns whether the address still owes the state root some work. + fn is_pending(&self) -> bool { + match self { + Self::InFlight(_) => true, + Self::Idle(work) => !work.pending.is_empty() || work.has_work(), + } + } +} + +/// Everything the sparse trie task knows about one storage trie, moved into a job as a whole and +/// handed back by it. +struct StorageTrieWork { + /// The storage trie, checked out of the [`SparseStateTrie`] for the whole run. + trie: RevealableSparseTrie, + /// Leaf updates that are not applied to `trie` yet, either because they are new or because + /// they are blocked on a blinded node. + pending: B256Map, + /// Proof nodes to reveal into `trie` before the next leaf pass. + proofs: Vec, + /// Slots whose proof was already requested, mapped to the broadest requested parent context + /// (an unknown parent sorts before every known parent). + fetched: B256Map, + /// Whether a pass could act on something: proof nodes to reveal, or leaf updates that were + /// not tried against the current state of the trie yet. + /// + /// Blocked updates alone are not enough. Retrying them without a reveal in between cannot + /// apply anything and would only resort the whole set again, and only this trie's own proof + /// nodes can unblock them. + dirty: bool, + /// Whether this address ever had storage leaf updates queued, which is what makes promotion + /// of its account wait for a recomputed storage root. + updated: bool, +} + +impl StorageTrieWork { + fn new(trie: RevealableSparseTrie) -> Self { + Self { + trie, + pending: Default::default(), + proofs: Vec::new(), + fetched: Default::default(), + dirty: false, + updated: false, + } + } + + /// Reveals the queued proof nodes, applies the leaf updates that are not blocked by a blinded + /// node, and recomputes the root once nothing is left to apply. + fn run(&mut self, new_epoch: TrieNodeEpoch, retain_updates: bool) -> StorageWorkOutput { + let Self { trie, pending, proofs, fetched, dirty, .. } = self; + *dirty = false; + let mut output = StorageWorkOutput::default(); + + if !proofs.is_empty() { + output.result = trie.reveal_v2_proof_nodes(proofs, retain_updates); + proofs.clear(); + if output.result.is_err() { + return output + } + } + + if !pending.is_empty() { + let updates_len_before = pending.len(); + let targets = &mut output.targets; + output.result = trie.update_leaves(pending, |path, parent| match fetched.entry(path) { + Entry::Occupied(mut entry) => { + if parent < *entry.get() { + entry.insert(parent); + targets.push(ProofV2Target::new(path).with_parent(parent)); + } + } + Entry::Vacant(entry) => { + entry.insert(parent); + targets.push(ProofV2Target::new(path).with_parent(parent)); + } + }); + output.cache_hits = (updates_len_before - pending.len()) as u64; + output.cache_misses = pending.len() as u64; + if output.result.is_err() { + return output + } + } + + if self.needs_root() { + self.trie.root(new_epoch); + } + + output + } + + /// Queues leaf updates, letting a newly known value win over one that is still queued. + fn queue_updates(&mut self, updates: B256Map) { + if updates.is_empty() { + return + } + + self.updated = true; + self.dirty = true; + merge_leaf_updates(&mut self.pending, updates); + } + + /// Queues proof nodes, draining them out of the batch they arrived in. + fn queue_proofs(&mut self, nodes: &mut Vec) { + if nodes.is_empty() { + return + } + + self.dirty = true; + self.proofs.append(nodes); + } + + /// Folds what arrived while a job owned this payload back into it. + fn take_buffered(&mut self, buffered: InFlightStorage) { + let InFlightStorage { mut proofs, updates, .. } = buffered; + self.queue_proofs(&mut proofs); + self.queue_updates(updates); + } + + /// Returns whether a pass would do anything. + fn has_work(&self) -> bool { + debug_assert!(self.dirty || !self.needs_root(), "a drained pass must hash its trie"); + self.dirty + } + + /// Returns whether the trie was modified since its root was last computed. + /// + /// Only a trie this block changed is hashed here: for the others the root the account leaf + /// already carries is still correct. + fn needs_root(&self) -> bool { + self.updated && + self.pending.is_empty() && + self.trie.is_revealed() && + !self.trie.is_root_cached() + } + + /// Returns whether a pass would only turn pending updates into proof targets. + /// + /// A blind trie has nothing to reveal into and nothing to apply, so running the pass on the + /// task itself keeps the proof request from waiting for a job round trip. + const fn is_target_only(&self) -> bool { + self.proofs.is_empty() && self.trie.is_blind() + } +} + +/// Buffers for one address while a job owns its payload. +struct InFlightStorage { + /// When the payload was handed off, for the round trip histogram. + started: Instant, + /// Proof nodes that arrived for this address while the payload was gone. + proofs: Vec, + /// Leaf updates that were queued for this address while the payload was gone. + updates: B256Map, +} + +impl InFlightStorage { + fn new(started: Instant) -> Self { + Self { started, proofs: Vec::new(), updates: Default::default() } + } +} + +/// A storage payload checked out of its entry and handed to a job. +struct StorageTrieJob { + /// Hashed address the payload belongs to. + address: B256, + /// The checked out payload. + work: Box>, +} + +impl StorageTrieJob { + fn run(mut self, new_epoch: TrieNodeEpoch, retain_updates: bool) -> StorageTrieJobDone { + let output = self.work.run(new_epoch, retain_updates); + StorageTrieJobDone { address: self.address, work: self.work, output } + } +} + +/// What a spawned storage job sends back to the sparse trie task. +enum StorageJobMessage { + /// The job finished and hands its payload back. + Done(StorageTrieJobDone), + /// The job panicked and its trie is lost. The global rayon pool has no panic handler, so a + /// panic escaping a spawned job would abort the process; it is caught and resumed on the task + /// thread instead, where it fails the state root calculation like an inline panic. + Panicked { + /// Hashed address of the payload the job owned. + address: B256, + /// The panic payload, resumed on the task thread. + payload: Box, + }, +} + +/// A storage payload coming back to the sparse trie task after its job finished. +struct StorageTrieJobDone { + /// Hashed address the payload belongs to. + address: B256, + /// The payload itself, to be put back into its entry. + work: Box>, + /// What the pass produced. + output: StorageWorkOutput, +} + +/// What one storage pass produced for the sparse trie task. +struct StorageWorkOutput { + /// Proof targets discovered while applying the leaf updates. + targets: Vec, + /// Leaf updates applied without needing a new proof. + cache_hits: u64, + /// Leaf updates that stayed blocked on a blinded node. + cache_misses: u64, + /// The error the pass stopped at, if any. + result: SparseTrieResult<()>, +} + +impl Default for StorageWorkOutput { + fn default() -> Self { + Self { targets: Vec::new(), cache_hits: 0, cache_misses: 0, result: Ok(()) } + } +} + +/// Merges leaf updates into a queue, letting a newly known value win over a queued one. +fn merge_leaf_updates(queued: &mut B256Map, updates: B256Map) { + if queued.is_empty() { + // Take the whole map at once, no per-slot loop. + *queued = updates; + return + } + + for (slot, update) in updates { + match queued.entry(slot) { + Entry::Occupied(mut entry) => { + if update.is_changed() { + entry.insert(update); + } + } + Entry::Vacant(entry) => { + entry.insert(update); + } + } + } +} + +/// Number of storage tries handed to a single spawned job. +/// +/// Chunking keeps a block with thousands of tiny tries from paying a rayon spawn per trie, while +/// staying small enough that one slow trie only delays the few behind it in its own chunk. Tries +/// are still handed back one at a time, so a chunk does not delay promotion of the ones it +/// already finished. +fn storage_job_chunk_len(tries: usize) -> usize { + /// Upper bound on how many tries a single slow one can hold up. + const MAX_CHUNK_LEN: usize = 32; + + tries.div_ceil(rayon::current_num_threads().max(1) * 4).clamp(1, MAX_CHUNK_LEN) +} + /// Metrics recorded by sparse trie and hashing tasks. #[derive(Metrics, Clone)] #[metrics(scope = "tree.root")] @@ -985,6 +1526,8 @@ pub(super) struct SparseTrieTaskMetrics { pub(super) sparse_trie_channel_wait_duration_histogram: Histogram, /// Histogram of durations spent processing trie updates and promoting pending accounts. pub(super) sparse_trie_process_updates_duration_histogram: Histogram, + /// Histogram of durations storage tries spend checked out for a job on another thread. + pub(super) sparse_trie_storage_job_duration_histogram: Histogram, /// Histogram of sparse trie final update durations. pub(super) sparse_trie_final_update_duration_histogram: Histogram, /// Histogram of sparse trie total durations. @@ -1022,6 +1565,11 @@ const DEFAULT_MAX_TARGETS_FOR_CHUNKING: usize = 300; /// Start proof fetching while the first state-update batch is still arriving. const INITIAL_UPDATE_BATCH_SIZE: usize = 64; +/// A round of storage passes that would do at most this much work - proof nodes to reveal plus +/// leaf updates to apply - runs on the sparse trie task itself, because handing the tries to +/// another thread and waiting for them to come back costs more than the work. +const INLINE_STORAGE_WORK_UNITS: usize = 16; + /// Dispatches work items as a single unit or in chunks based on target size and worker /// availability. #[expect(clippy::too_many_arguments)] @@ -1127,6 +1675,7 @@ mod tests { use reth_db_common::init::init_genesis; use reth_provider::test_utils::create_test_provider_factory; use reth_storage_overlay::{OverlayManager, OverlayStateProviderFactory}; + use reth_trie_common::{ExtensionNode, LeafNode, Nibbles, RlpNode, TrieNodeV2}; use reth_trie_parallel::proof_task::ProofTaskCtx; use reth_trie_sparse::ArenaParallelSparseTrie; @@ -1136,6 +1685,97 @@ mod tests { } } + fn test_task( + runtime: &Runtime, + trie: SparseStateTrie, + ) -> (SparseTrieCacheTask, CrossbeamSender, CrossbeamSender<()>) { + let provider_factory = create_test_provider_factory(); + let anchor_hash = init_genesis(&provider_factory).expect("failed to initialize genesis"); + let state_provider_factory = OverlayStateProviderFactory::new( + provider_factory, + OverlayManager::::default() + .overlay_builder(anchor_hash), + ); + let (proof_result_tx, proof_result_rx) = crossbeam_channel::unbounded(); + let proof_worker_handle = ProofWorkerHandle::new( + runtime, + ProofTaskCtx::new(state_provider_factory), + false, + proof_result_tx.clone(), + ); + let (updates_tx, updates_rx) = crossbeam_channel::unbounded(); + let (cancel_guard, cancel_rx) = crossbeam_channel::bounded(0); + let task = SparseTrieCacheTask::new_with_trie( + runtime, + updates_rx, + cancel_rx, + std::sync::mpsc::channel().0, + proof_worker_handle, + proof_result_tx, + proof_result_rx, + SparseTrieTaskMetrics::default(), + trie, + EMPTY_ROOT_HASH, + TrieNodeEpoch::new(1), + 1, + ); + (task, updates_tx, cancel_guard) + } + + fn receive_storage_job( + task: &SparseTrieCacheTask, + ) -> StorageJobMessage { + task.storage_done_rx + .recv_timeout(std::time::Duration::from_secs(5)) + .expect("storage job did not return") + } + + /// Checks a payload out of its entry the way a spawned job does, without racing one. + fn check_out_storage( + task: &mut SparseTrieCacheTask, + address: B256, + ) -> Box> { + let state = task.storage.get_mut(&address).expect("address must have an entry"); + let StorageTrieState::Idle(work) = core::mem::replace( + state, + StorageTrieState::InFlight(Box::new(InFlightStorage::new(Instant::now()))), + ) else { + panic!("trie must be idle") + }; + task.storage_in_flight += 1; + work + } + + /// Hands a payload checked out by [`check_out_storage`] back, as a finished job would. + fn return_storage( + task: &mut SparseTrieCacheTask, + address: B256, + work: Box>, + ) { + task.on_storage_trie_returned(StorageTrieJobDone { + address, + work, + output: StorageWorkOutput::default(), + }) + .unwrap(); + } + + fn storage_slot_value( + task: &SparseTrieCacheTask, + address: &B256, + slot: &B256, + ) -> Option> { + let StorageTrieState::Idle(work) = task.storage.get(address)? else { return None }; + work.trie.as_revealed_ref()?.get_leaf_value(&Nibbles::unpack(slot)).cloned() + } + + fn storage_root_of(task: &mut SparseTrieCacheTask, address: B256) -> B256 { + let StorageTrieState::Idle(work) = task.storage.get_mut(&address).expect("entry") else { + panic!("payload is out with a job") + }; + work.trie.root(TrieNodeEpoch::new(1)).expect("storage trie must be revealed") + } + #[test] fn test_run_hashing_task_hashed_state_update_forwards() { let (updates_tx, updates_rx) = crossbeam_channel::unbounded(); @@ -1229,6 +1869,427 @@ mod tests { assert_eq!(account_rlp_buf, encoded); } + #[test] + fn checked_out_storage_trie_holds_back_its_updates_until_it_returns() { + let runtime = Runtime::test(); + let trie = SparseStateTrie::default() + .with_accounts_trie(RevealableSparseTrie::::revealed_empty()) + .with_default_storage_trie(RevealableSparseTrie::blind_from( + ArenaParallelSparseTrie::default(), + )) + .with_updates(true); + let (mut task, updates_tx, _cancel_guard) = test_task(&runtime, trie); + + let address = B256::repeat_byte(0x11); + let revealed_slot = B256::repeat_byte(0x22); + let late_slot = B256::repeat_byte(0x33); + let revealed_value = alloy_rlp::encode_fixed_size(&U256::from(7)).to_vec(); + let late_value = alloy_rlp::encode_fixed_size(&U256::from(9)).to_vec(); + let later_value = alloy_rlp::encode_fixed_size(&U256::from(11)).to_vec(); + + let mut state = HashedPostState::default(); + state.accounts.insert(address, Some(Account { nonce: 1, ..Default::default() })); + state.storages.entry(address).or_default().storage.insert(revealed_slot, U256::from(7)); + task.on_hashed_state_update(state); + task.pending_updates = 1; + task.apply_new_updates().unwrap(); + + // Check the payload out by hand so the test does not race a spawned job. + let work = check_out_storage(&mut task, address); + task.finished_state_updates = true; + + assert!(task.has_pending_sparse_trie_updates()); + assert!( + task.ensure_not_stalled(false).is_ok(), + "a checked out payload can still deliver progress" + ); + + // A proof for a checked out address must be buffered, not revealed into a fresh blind + // trie that the returning job would then overwrite. + let leaf = ProofTrieNodeV2 { + path: Nibbles::default(), + node: TrieNodeV2::Leaf(LeafNode::new( + Nibbles::unpack(revealed_slot), + revealed_value.clone(), + )), + masks: None, + }; + task.on_proof_result(DecodedMultiProofV2 { + storage_proofs: B256Map::from_iter([(address, vec![leaf])]), + ..Default::default() + }) + .unwrap(); + assert!(!task.trie.storage_tries_mut().contains_key(&address)); + let StorageTrieState::InFlight(in_flight) = &task.storage[&address] else { + panic!("payload is out with a job") + }; + assert_eq!(in_flight.proofs.len(), 1); + + // The same holds for leaf updates arriving while the payload is gone. + let mut state = HashedPostState::default(); + state.storages.entry(address).or_default().storage.insert(late_slot, U256::from(9)); + task.on_hashed_state_update(state); + task.pending_updates = 1; + task.apply_new_updates().unwrap(); + let StorageTrieState::InFlight(in_flight) = &task.storage[&address] else { + panic!("payload is out with a job") + }; + assert!(in_flight.updates.contains_key(&late_slot)); + + return_storage(&mut task, address, work); + assert_eq!(task.storage_in_flight, 0); + + // Returned work must run even without new input messages. + assert_eq!(task.pending_updates, 0); + task.process_new_updates().unwrap(); + assert_eq!(storage_slot_value(&task, &address, &revealed_slot), Some(revealed_value)); + assert_eq!(storage_slot_value(&task, &address, &late_slot), Some(late_value)); + + let StorageTrieState::Idle(work) = &task.storage[&address] else { + panic!("payload is back") + }; + assert!(work.pending.is_empty()); + assert!(work.trie.is_root_cached(), "a drained payload hashes its trie"); + let root_before = storage_root_of(&mut task, address); + + // An update arriving during a job invalidates the root the previous pass computed. + let work = check_out_storage(&mut task, address); + let mut state = HashedPostState::default(); + state.storages.entry(address).or_default().storage.insert(late_slot, U256::from(11)); + task.on_hashed_state_update(state); + task.pending_updates = 1; + task.apply_new_updates().unwrap(); + return_storage(&mut task, address, work); + + let StorageTrieState::Idle(work) = &task.storage[&address] else { + panic!("payload is back") + }; + assert!(work.has_work(), "a buffered update makes the payload ready again"); + + task.process_new_updates().unwrap(); + assert_eq!(storage_slot_value(&task, &address, &late_slot), Some(later_value)); + assert_ne!(storage_root_of(&mut task, address), root_before); + + drop(updates_tx); + drop(task); + drain_sparse_trie_tasks(&runtime); + } + + #[test] + fn in_flight_storage_updates_keep_latest_values_and_deletions() { + let runtime = Runtime::test(); + let trie = SparseStateTrie::default() + .with_accounts_trie(RevealableSparseTrie::revealed_empty()) + .with_updates(true); + let (mut task, updates_tx, _cancel_guard) = test_task(&runtime, trie); + let address = B256::repeat_byte(0x11); + let removed_slot = B256::repeat_byte(0x20); + let changed_slot = B256::repeat_byte(0x30); + let new_slot = B256::repeat_byte(0x31); + // A nested branch ensures there are persisted branch updates to compare. + let mut sibling_slot = changed_slot; + sibling_slot[1] = 0x40; + let account = Account { nonce: 1, ..Default::default() }; + let parent_leaf = ProofTrieNodeV2 { + path: Nibbles::default(), + node: TrieNodeV2::Leaf(LeafNode::new( + Nibbles::unpack(removed_slot), + alloy_rlp::encode(U256::from(7)), + )), + masks: None, + }; + + let mut state = HashedPostState::default(); + state.accounts.insert(address, Some(account)); + state.storages.entry(address).or_default().storage.extend([ + (removed_slot, U256::from(8)), + (changed_slot, U256::from(9)), + (new_slot, U256::from(1)), + (sibling_slot, U256::from(2)), + ]); + task.on_hashed_state_update(state); + task.pending_updates = 1; + task.apply_new_updates().unwrap(); + + let mut work = check_out_storage(&mut task, address); + let output = work.run(task.new_epoch, true); + assert_eq!(work.pending.len(), 4, "the blind pass keeps its original updates"); + assert_eq!(output.targets.len(), 4); + + // New values and a later prewarm hint arrive while the old values are still pending. + let mut state = HashedPostState::default(); + state + .storages + .entry(address) + .or_default() + .storage + .extend([(removed_slot, U256::ZERO), (changed_slot, U256::from(11))]); + task.on_hashed_state_update(state); + task.pending_updates = 1; + task.apply_new_updates().unwrap(); + task.on_prewarm_targets(MultiProofTargetsV2 { + storage_targets: B256Map::from_iter([( + address, + vec![ProofV2Target::new(removed_slot), ProofV2Target::new(changed_slot)], + )]), + ..Default::default() + }); + task.pending_updates = 1; + task.apply_new_updates().unwrap(); + + task.on_storage_trie_returned(StorageTrieJobDone { address, work, output }).unwrap(); + // Deliver the requested parent proof directly so the interleaving is deterministic. + let (targets, _) = task.pending_targets.take(); + assert_eq!(targets.storage_targets[&address].len(), 4); + task.on_proof_result(DecodedMultiProofV2 { + storage_proofs: B256Map::from_iter([(address, vec![parent_leaf.clone()])]), + ..Default::default() + }) + .unwrap(); + updates_tx.send(StateRootMessage::FinishedStateUpdates).unwrap(); + let outcome = task.run().unwrap(); + + let expected_root = reth_trie_common::root::storage_root_unsorted([ + (changed_slot, U256::from(11)), + (new_slot, U256::from(1)), + (sibling_slot, U256::from(2)), + ]); + assert_eq!( + outcome.state_root, + reth_trie_common::root::state_root_unsorted([( + address, + account.into_trie_account(expected_root), + )]) + ); + + let mut serial = RevealableSparseTrie::::blind(); + serial.reveal_v2_proof_nodes(&mut [parent_leaf], true).unwrap(); + let mut final_updates = B256Map::from_iter([ + (removed_slot, LeafUpdate::Changed(Vec::new())), + (changed_slot, LeafUpdate::Changed(alloy_rlp::encode(U256::from(11)))), + (new_slot, LeafUpdate::Changed(alloy_rlp::encode(U256::from(1)))), + (sibling_slot, LeafUpdate::Changed(alloy_rlp::encode(U256::from(2)))), + ]); + serial + .update_leaves(&mut final_updates, |_, _| panic!("parent is fully revealed")) + .unwrap(); + let (serial_root, serial_updates) = serial.root_with_updates(task.new_epoch).unwrap(); + assert_eq!(serial_root, expected_root); + assert!(!serial_updates.updated_nodes.is_empty()); + let updates = &outcome.trie_updates.storage_tries[&address]; + assert_eq!(updates.storage_nodes, serial_updates.updated_nodes); + assert_eq!(updates.removed_nodes, serial_updates.removed_nodes); + + let (mut reused, _) = task.into_trie_for_reuse(); + assert_eq!(reused.storage_root(&address, TrieNodeEpoch::new(2)), Some(expected_root)); + assert!(reused.take_trie_updates().unwrap().is_empty()); + drop(updates_tx); + drain_sparse_trie_tasks(&runtime); + } + + #[test] + fn unrelated_storage_proofs_do_not_retry_blocked_leaves() { + let runtime = Runtime::test(); + let (mut task, updates_tx, _cancel_guard) = test_task(&runtime, SparseStateTrie::default()); + let address = B256::repeat_byte(0x11); + let slot = B256::repeat_byte(0x22); + let StorageTrieState::Idle(work) = task.storage_trie_state_mut(address) else { + unreachable!() + }; + work.queue_updates(B256Map::from_iter([(slot, LeafUpdate::Changed(vec![1]))])); + task.run_ready_storage_work().unwrap(); + assert_eq!(task.storage_cache_misses, 1); + + task.on_proof_result(DecodedMultiProofV2 { + storage_proofs: B256Map::from_iter([( + B256::repeat_byte(0x33), + vec![ProofTrieNodeV2::empty()], + )]), + ..Default::default() + }) + .unwrap(); + assert_eq!(task.storage_cache_misses, 1, "unrelated reveals cannot unblock this trie"); + assert_eq!(storage_slot_value(&task, &address, &slot), None); + + task.on_proof_result(DecodedMultiProofV2 { + storage_proofs: B256Map::from_iter([(address, vec![ProofTrieNodeV2::empty()])]), + ..Default::default() + }) + .unwrap(); + assert_eq!(storage_slot_value(&task, &address, &slot), Some(vec![1])); + assert_eq!(task.storage_cache_hits, 1); + assert!(!task.has_ready_storage_work()); + + drop(updates_tx); + drop(task); + drain_sparse_trie_tasks(&runtime); + } + + #[test] + fn canceled_task_disconnects_checked_out_storage_jobs() { + let runtime = Runtime::test(); + let trie = SparseStateTrie::default() + .with_default_storage_trie(RevealableSparseTrie::revealed_empty()); + let (mut task, updates_tx, cancel_guard) = test_task(&runtime, trie); + let idle_address = B256::repeat_byte(0x11); + let address = B256::repeat_byte(0x22); + task.storage_trie_state_mut(idle_address); + task.storage_trie_state_mut(address); + let work = check_out_storage(&mut task, address); + let done_tx = task.storage_done_tx.clone(); + let epoch = task.new_epoch; + + drop(cancel_guard); + assert!(matches!(task.run(), Err(StateRootTaskError::Canceled))); + let (trie, _) = task.into_cleared_trie(); + assert_eq!(trie.retained_storage_tries_count(), 1, "only the idle trie can be reused"); + + // Complete the held job after teardown, when its payload must be dropped by the sender. + let done = StorageTrieJob { address, work }.run(epoch, false); + assert!(done_tx.send(StorageJobMessage::Done(done)).is_err()); + drop(updates_tx); + drain_sparse_trie_tasks(&runtime); + } + + #[test] + fn storage_job_panics_are_resumed_on_the_task() { + let runtime = Runtime::test(); + let (mut task, updates_tx, _cancel_guard) = test_task(&runtime, SparseStateTrie::default()); + let address = B256::repeat_byte(0x11); + let StorageTrieState::Idle(work) = task.storage_trie_state_mut(address) else { + unreachable!() + }; + // An unfused extension root is unsupported and deterministically panics in set_root. + work.queue_proofs(&mut vec![ProofTrieNodeV2 { + path: Nibbles::default(), + node: TrieNodeV2::Extension(ExtensionNode::new( + Nibbles::from_nibbles([1]), + RlpNode::word_rlp(&B256::ZERO), + )), + masks: None, + }]); + let work = check_out_storage(&mut task, address); + task.spawn_storage_jobs(vec![StorageTrieJob { address, work }]); + + let message = receive_storage_job(&task); + assert!( + matches!(&message, StorageJobMessage::Panicked { address: returned, .. } if *returned == address) + ); + let panic = panic::catch_unwind(AssertUnwindSafe(|| task.on_storage_job_message(message))); + assert!(panic.is_err()); + assert_eq!(task.storage_in_flight, 0); + assert!(!task.storage.contains_key(&address)); + + drop(updates_tx); + drop(task); + drain_sparse_trie_tasks(&runtime); + } + + #[test] + fn large_storage_batches_run_off_thread() { + let runtime = Runtime::test(); + let trie = SparseStateTrie::default() + .with_accounts_trie(RevealableSparseTrie::::revealed_empty()) + .with_default_storage_trie(RevealableSparseTrie::blind_from( + ArenaParallelSparseTrie::default(), + )) + .with_updates(true); + let (mut task, updates_tx, _cancel_guard) = test_task(&runtime, trie); + + // One leaf per address, enough of them to exceed the inline budget. + let leaves = (0..=INLINE_STORAGE_WORK_UNITS as u8) + .map(|index| { + let address = B256::repeat_byte(0x10 + index); + let slot = B256::repeat_byte(0x80 + index); + let value = alloy_rlp::encode_fixed_size(&U256::from(index + 1)).to_vec(); + (address, slot, value) + }) + .collect::>(); + let storage_proofs = leaves + .iter() + .map(|(address, slot, value)| { + let leaf = ProofTrieNodeV2 { + path: Nibbles::default(), + node: TrieNodeV2::Leaf(LeafNode::new(Nibbles::unpack(slot), value.clone())), + masks: None, + }; + (*address, vec![leaf]) + }) + .collect(); + + task.on_proof_result(DecodedMultiProofV2 { account_proofs: Vec::new(), storage_proofs }) + .unwrap(); + + assert_eq!(task.storage_in_flight, leaves.len()); + assert!(task.has_pending_sparse_trie_updates(), "completion must wait for the reveals"); + + while task.storage_in_flight > 0 { + let message = receive_storage_job(&task); + task.on_storage_job_message(message).unwrap(); + } + for (address, slot, value) in &leaves { + assert_eq!(storage_slot_value(&task, address, slot).as_ref(), Some(value)); + } + + drop(updates_tx); + drop(task); + drain_sparse_trie_tasks(&runtime); + } + + #[test] + fn run_waits_for_storage_tries_hashed_off_thread() { + let runtime = Runtime::test(); + let default_trie = RevealableSparseTrie::::revealed_empty(); + let trie = SparseStateTrie::default() + .with_accounts_trie(default_trie.clone()) + .with_default_storage_trie(default_trie) + .with_updates(true); + let (mut task, updates_tx, _cancel_guard) = test_task(&runtime, trie); + + let accounts = (0..8u8) + .map(|index| { + let address = B256::repeat_byte(0x10 + index); + let account = Account { + nonce: u64::from(index) + 1, + balance: U256::from(index), + bytecode_hash: None, + }; + let storage = (0..4u8) + .map(|slot| { + ( + B256::repeat_byte(0x40 + index * 4 + slot), + U256::from(slot) + U256::from(1), + ) + }) + .collect::>(); + (address, account, storage) + }) + .collect::>(); + + let mut state = HashedPostState::default(); + for (address, account, storage) in &accounts { + state.accounts.insert(*address, Some(*account)); + state.storages.entry(*address).or_default().storage.extend(storage.iter().copied()); + } + updates_tx.send(StateRootMessage::HashedStateUpdate(state)).unwrap(); + updates_tx.send(StateRootMessage::FinishedStateUpdates).unwrap(); + + let outcome = task.run().expect("state root computation should succeed"); + + let expected = accounts.iter().map(|(address, account, storage)| { + let storage_root = + reth_trie_common::root::storage_root_unsorted(storage.iter().copied()); + (*address, account.into_trie_account(storage_root)) + }); + assert_eq!(outcome.state_root, reth_trie_common::root::state_root_unsorted(expected)); + assert_eq!(task.storage_in_flight, 0); + assert!(task.storage.is_empty(), "every storage trie is back in the state trie"); + + drop(updates_tx); + drop(task); + drain_sparse_trie_tasks(&runtime); + } + #[test] fn first_leaf_batch_starts_proofs_before_input_queue_drains() { let runtime = reth_tasks::Runtime::test(); @@ -1418,13 +2479,16 @@ mod tests { task.finished_state_updates = true; task.account_updates.insert(account, LeafUpdate::Touched); - task.storage_updates.entry(account).or_default().insert(slot, LeafUpdate::Touched); task.pending_account_updates.insert(account, None); task.fetched_account_targets.insert(account_target, ProofV2TargetParent::NONE); - task.fetched_storage_targets - .entry(account) - .or_default() - .insert(storage_target, ProofV2TargetParent::new(11)); + + // A storage leaf that stayed blocked after its pass ran, so no pass is ready to run. + let StorageTrieState::Idle(work) = task.storage_trie_state_mut(account) else { + panic!("a fresh trie state is idle") + }; + work.updated = true; + work.pending.insert(slot, LeafUpdate::Touched); + work.fetched.insert(storage_target, ProofV2TargetParent::new(11)); task.in_flight_proof_batches = 1; assert!(task.ensure_not_stalled(false).is_ok()); diff --git a/crates/trie/sparse/src/state.rs b/crates/trie/sparse/src/state.rs index fed07f8e53f..5648f3cd2c9 100644 --- a/crates/trie/sparse/src/state.rs +++ b/crates/trie/sparse/src/state.rs @@ -80,6 +80,11 @@ impl SparseStateTrie { self } + /// Returns whether branch node updates and deletions are retained. + pub const fn retains_updates(&self) -> bool { + self.retain_updates + } + /// Set the accounts trie to the given `RevealableSparseTrie`. pub fn set_accounts_trie(&mut self, trie: RevealableSparseTrie) { self.state = trie; @@ -319,6 +324,36 @@ where any_err } + /// Reveals account trie proof nodes on the calling thread. + /// + /// Unlike [`Self::reveal_decoded_multiproof_v2`] this touches only the account trie, so a + /// caller that owns some of the storage tries itself can reveal the two halves separately. + pub fn reveal_account_proof_nodes( + &mut self, + mut nodes: Vec, + ) -> SparseStateTrieResult<()> { + if nodes.is_empty() { + return Ok(()) + } + + #[cfg(feature = "metrics")] + self.metrics.increment_total_account_nodes(nodes.len() as u64); + + let result = self.state.reveal_v2_proof_nodes(&mut nodes, self.retain_updates); + self.deferred_drops.proof_nodes_bufs.push(nodes); + + Ok(result?) + } + + /// Records storage trie nodes that were revealed into a trie taken out of this state trie, so + /// the reveal metrics stay complete. + pub const fn record_revealed_storage_nodes(&mut self, nodes: usize) { + #[cfg(feature = "metrics")] + self.metrics.increment_total_storage_nodes(nodes as u64); + #[cfg(not(feature = "metrics"))] + let _ = nodes; + } + /// Calculates the hashes of subtries. /// /// If the trie has not been revealed, this function does nothing. From 30e596d42d14ef067d320579029f237718f27908 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Tue, 15 Sep 2026 18:12:21 +0000 Subject: [PATCH 269/294] fix(engine): adjudicate speculative BAL worker failures in transaction order (#26721) Co-authored-by: Matthias Seitz <19890894+mattsse@users.noreply.github.com> --- .../src/tree/payload_processor/bal/execute.rs | 304 +++++++++++++++++- .../payload_processor/bal/ordered_outputs.rs | 104 +++++- .../src/tree/payload_processor/bal/worker.rs | 116 ++++--- .../tree/src/tree/payload_processor/mod.rs | 33 +- 4 files changed, 497 insertions(+), 60 deletions(-) diff --git a/crates/engine/tree/src/tree/payload_processor/bal/execute.rs b/crates/engine/tree/src/tree/payload_processor/bal/execute.rs index aa88f7c7510..b8dac4edd67 100644 --- a/crates/engine/tree/src/tree/payload_processor/bal/execute.rs +++ b/crates/engine/tree/src/tree/payload_processor/bal/execute.rs @@ -4,17 +4,28 @@ //! //! Worker states run transactions speculatively. Each worker gets one fresh cache-filling database //! from `make_db(true)`, installs the received BAL, sets the transaction BAL index for each -//! streamed transaction, and returns uncommitted transaction results. +//! streamed transaction, and returns uncommitted results. Execution errors rebuild the State, +//! EVM, and executor while retaining the worker's database. //! //! The canonical state owns block effects. It runs the normal pre/post block hooks, commits //! worker results in transaction order, tracks block gas admission, and builds the BAL that this //! execution actually produced. //! +//! Recovery and speculative execution failures are deferred to their transaction slot and +//! adjudicated in block order: block-gas admission at an earlier slot takes precedence, otherwise +//! the first failure decides the block's verdict. Workers trust the received BAL before its hash is +//! validated against the rebuilt BAL. An undeclared access therefore short-circuits execution +//! at that slot, and its error may differ from serial execution against canonical state (which +//! can instead fail nonce, balance, or later block checks). Failed transactions are not replayed. +//! //! The rebuilt BAL is returned to the outer payload validator for consensus post-execution //! validation. This module only logs the first divergence between the received BAL and the BAL //! rebuilt from canonical execution. -use super::{ordered_outputs::ordered_worker_outputs, worker, BalExecutionError}; +use super::{ + ordered_outputs::{ordered_worker_outputs, OrderedWorkerOutputError}, + worker, BalExecutionError, +}; use alloy_eip7928::{ bal::{Bal as AlloyBal, DecodedBal}, compute_block_access_list_hash, BlockAccessList, @@ -40,6 +51,9 @@ use std::sync::Arc; use crate::tree::payload_processor::receipt_root_task::IndexedReceipt; /// Executes one block on the BAL path using the runtime's persistent BAL worker pool. +/// +/// The ordered commit loop applies Ethereum block-level gas admission. Executors with different +/// admission rules, such as segment-scoped gas budgets, must align those checks before using it. #[expect(clippy::too_many_arguments, clippy::type_complexity)] pub fn execute_block<'a, Evm, Tx, Err, DB, MakeDb>( runtime: &Runtime, @@ -153,7 +167,29 @@ where let mut senders = Vec::with_capacity(transaction_count); let mut last_sent_len = 0usize; for output in ordered_worker_outputs(&result_rx, transaction_count) { - let output = output?; + let output = match output { + Ok(output) => output, + Err(OrderedWorkerOutputError::Worker(worker::BalWorkerError::Execution { + tx_index, + tx_gas_limit, + source, + })) => { + // Block-gas admission is adjudicated in transaction order and takes + // precedence over the failure at this slot. + gas_tracker.validate_tx_limit(tx_gas_limit)?; + + // The transaction is admitted, so its execution failure is the block's + // verdict: a BAL miss proves the received BAL diverges from this execution. + tracing::debug!( + target: "engine::tree::payload_processor::bal", + tx_index, + err = %source, + "Speculative BAL execution failed; rejecting block in transaction order" + ); + return Err(BalExecutionError::Execution(source)); + } + Err(err) => return Err(err.into()), + }; gas_tracker.validate_tx_limit(output.tx_gas_limit)?; gas_tracker.record_result(output.result.result()); @@ -977,6 +1013,268 @@ mod tests { } } + /// Two funded senders each transferring to a fresh recipient, plus the reference BAL of a + /// block containing only the covered subset of those transfers. + fn two_transfers_with_reference_bal( + evm_config: &EthEvmConfig, + tx_gas_limit: u64, + bal_covers_first: bool, + ) -> ( + CacheDB, + BlockAccessList, + Recovered, + Recovered, + ) { + use alloy_consensus::TxLegacy; + use alloy_primitives::TxKind; + use reth_chainspec::MAINNET; + use reth_ethereum_primitives::Transaction; + use reth_primitives_traits::crypto::secp256k1::public_key_to_address; + use reth_testing_utils::generators::{generate_key, rng, sign_tx_with_key_pair}; + + let recipient = alloy_primitives::Address::from([0xCA; 20]); + let sender_balance = U256::from(alloy_consensus::constants::ETH_TO_WEI); + + let alice_kp = generate_key(&mut rng()); + let alice = public_key_to_address(alice_kp.public_key()); + let bob_kp = generate_key(&mut rng()); + let bob = public_key_to_address(bob_kp.public_key()); + + let mut pre_block_db = system_contracts_db(); + insert_funded(&mut pre_block_db, alice, sender_balance); + insert_funded(&mut pre_block_db, bob, sender_balance); + + let chain_id = MAINNET.chain.id(); + let make_tx = |kp, value| { + sign_tx_with_key_pair( + kp, + Transaction::Legacy(TxLegacy { + chain_id: Some(chain_id), + nonce: 0, + gas_price: 1, + gas_limit: tx_gas_limit, + to: TxKind::Call(recipient), + value: U256::from(value), + input: Default::default(), + }), + ) + }; + let tx1 = Recovered::new_unchecked(make_tx(alice_kp, 100u64), alice); + let tx2 = Recovered::new_unchecked(make_tx(bob_kp, 200u64), bob); + + let reference_block = empty_amsterdam_block(B256::ZERO); + let covered = if bal_covers_first { vec![tx1.clone()] } else { vec![] }; + let reference_bal = + reference_bal_for_block(evm_config, pre_block_db.clone(), &reference_block, covered); + (pre_block_db, reference_bal, tx1, tx2) + } + + #[test] + fn propagates_bal_miss_of_admitted_transaction_as_block_verdict() { + // The BAL covers neither transaction: the first slot's speculative failure is + // adjudicated in block order and becomes the block's verdict. + let (pre_block_db, reference_bal, tx1, tx2) = + two_transfers_with_reference_bal(&EthEvmConfig::mainnet(), 100_000, false); + let bal_hash = alloy_eip7928::compute_block_access_list_hash(&reference_bal); + let block = empty_amsterdam_block(bal_hash); + + let result = run_execute_block( + &Runtime::test(), + EthEvmConfig::mainnet(), + db_factory(pre_block_db), + to_arc_decoded(reference_bal), + &block, + vec![tx1, tx2], + ); + + match result { + Err(BalExecutionError::Execution(err)) => { + let msg = err.to_string(); + assert!(msg.contains("not found in BAL"), "expected BAL miss error, got {msg}"); + } + Err(err) => panic!("expected BAL execution error, got {err:?}"), + Ok(_) => panic!("expected BAL execution error, got Ok"), + } + } + + #[test] + fn admission_rejects_before_speculative_worker_failure_surfaces() { + // tx2 exceeds the remaining block gas AND misses the BAL; block-gas admission is + // adjudicated first, so the verdict is the gas error, not tx2's BAL failure. + let (pre_block_db, reference_bal, tx1, tx2) = + two_transfers_with_reference_bal(&EthEvmConfig::mainnet(), 990_000, true); + let bal_hash = alloy_eip7928::compute_block_access_list_hash(&reference_bal); + let low_gas_block = empty_amsterdam_block_with_gas_limit(bal_hash, 1_000_000); + + let result = run_execute_block( + &Runtime::test(), + EthEvmConfig::mainnet(), + db_factory(pre_block_db), + to_arc_decoded(reference_bal), + &low_gas_block, + vec![tx1, tx2], + ); + + match result { + Err(BalExecutionError::Execution(err)) => assert!( + matches!( + err.as_validation(), + Some(BlockValidationError::TransactionGasLimitMoreThanAvailableBlockGas { .. }) + ), + "expected block gas validation error, got {err:?}" + ), + Err(err) => panic!("expected block gas validation error, got {err:?}"), + Ok(_) => panic!("expected block gas validation error, got Ok"), + } + } + + #[test] + fn worker_reuses_provider_after_failure_and_finishes_earlier_transactions() { + use std::sync::atomic::{AtomicUsize, Ordering}; + + let evm_config = EthEvmConfig::mainnet(); + let (pre_block_db, reference_bal, tx1, tx2) = + two_transfers_with_reference_bal(&evm_config, 100_000, true); + let block = empty_amsterdam_block(B256::ZERO); + let received_bal = convert_alloy_to_revm_bal(&reference_bal.into()).unwrap(); + let setups = AtomicUsize::new(0); + let make_db = |_: bool| { + // Reopening the provider after a speculative error must not introduce an + // unindexed setup failure that overtakes the remaining transaction slots. + if setups.fetch_add(1, Ordering::Relaxed) != 0 { + return Err(BalExecutionError::Provider( + reth_errors::ProviderError::HeaderNotFound(B256::ZERO.into()), + )); + } + Ok(pre_block_db.clone()) + }; + let (tx_tx, tx_rx) = crossbeam_channel::unbounded(); + // Recovery can finish out of order. A single worker must continue after both kinds + // of error to execute slot 0, without reusing the failed slot 1 executor. + tx_tx.send((2, Err(std::io::Error::other("sig fail")))).unwrap(); + tx_tx.send((1, Ok(tx2))).unwrap(); + tx_tx.send((0, Ok(tx1.clone()))).unwrap(); + drop(tx_tx); + let (_abort_guard, abort_rx) = AbortGuard::new(); + let (result_tx, result_rx) = crossbeam_channel::unbounded(); + let runtime = Runtime::test(); + runtime.bal_streaming_pool().in_place_scope(|scope| { + worker::spawn_worker( + scope, + tx_rx, + abort_rx, + result_tx, + &evm_config, + &make_db, + received_bal, + evm_config.evm_env(block.header()).unwrap(), + evm_config.context_for_block(&block).unwrap(), + ); + }); + let mut outputs = ordered_worker_outputs(&result_rx, 3); + let output = outputs.next().unwrap().unwrap(); + assert_eq!(output.index, 0); + assert_eq!(output.signer, tx1.signer()); + let mut state = State::builder().with_database(pre_block_db).build(); + let mut serial = evm_config.executor_for_block(&mut state, &block).unwrap(); + serial.apply_pre_execution_changes().unwrap(); + let expected = serial.execute_transaction_without_commit(tx1).unwrap(); + assert_eq!(output.result.result(), expected.result()); + assert!(matches!( + outputs.next().unwrap(), + Err(OrderedWorkerOutputError::Worker(worker::BalWorkerError::Execution { + tx_index: 1, + .. + })) + )); + let error = outputs.next().unwrap().err().unwrap(); + assert!(matches!( + error, + OrderedWorkerOutputError::Worker(worker::BalWorkerError::Transaction { + tx_index: 2, + .. + }) + )); + assert!(matches!( + BalExecutionError::from(error), + BalExecutionError::Execution(BlockExecutionError::Validation(_)) + )); + assert!(outputs.next().is_none()); + assert_eq!(setups.load(Ordering::Relaxed), 1); + } + + #[test] + fn amsterdam_state_gas_admission_matches_serial_execution() { + let evm_config = EthEvmConfig::new(Arc::new( + reth_chainspec::ChainSpecBuilder::mainnet().amsterdam_activated().build(), + )); + for (tx_gas_limit, complete_bal) in [(500_000, true), (990_000, true), (990_000, false)] { + let (pre_block_db, partial_bal, tx1, tx2) = + two_transfers_with_reference_bal(&evm_config, tx_gas_limit, true); + let reference_block = empty_amsterdam_block(B256::ZERO); + let reference_bal = if complete_bal { + reference_bal_for_block( + &evm_config, + pre_block_db.clone(), + &reference_block, + vec![tx1.clone(), tx2.clone()], + ) + } else { + // State-gas admission must also precede a speculative BAL miss at this slot. + partial_bal + }; + let block = empty_amsterdam_block_with_gas_limit( + compute_block_access_list_hash(&reference_bal), + 1_000_000, + ); + let mut evm_env = evm_config.evm_env(block.header()).unwrap(); + assert!(evm_env.cfg_env.enable_amsterdam_eip8037); + // Both capped regular limits fit. The full limit of the second transaction + // must still fit the state budget consumed by the first account creation. + evm_env.cfg_env.tx_gas_limit_cap = Some(400_000); + let ctx = evm_config.context_for_block(&block).unwrap(); + let mut state = State::builder().with_database(pre_block_db.clone()).build(); + let evm = evm_config.evm_with_env(&mut state, evm_env.clone()); + let mut serial = evm_config.create_executor_with_state(evm, ctx.clone()); + serial.apply_pre_execution_changes().unwrap(); + serial.execute_transaction(tx1.clone()).unwrap(); + assert!(serial.block_state_gas_used > 10_000); + assert!(serial.block_regular_gas_used + 400_000 < 1_000_000); + let serial_result = serial.execute_transaction(tx2.clone()); + + let (receipt_tx, _receipt_rx) = crossbeam_channel::unbounded(); + let parallel_result = execute_block( + &Runtime::test(), + &evm_config, + &|_: bool| Ok(pre_block_db.clone()), + to_arc_decoded(reference_bal), + evm_env, + ctx, + 2, + tx_stream(vec![tx1, tx2]), + receipt_tx, + ); + if tx_gas_limit == 500_000 { + serial_result.unwrap(); + assert_eq!(parallel_result.unwrap().0.receipts, serial.receipts()); + } else { + let serial_error = serial_result.unwrap_err(); + let Some(BlockValidationError::TransactionGasLimitMoreThanAvailableBlockGas { + block_available_gas, + .. + }) = serial_error.as_validation() + else { + panic!("expected serial state-gas admission error, got {serial_error:?}"); + }; + assert_eq!(*block_available_gas, 1_000_000 - serial.block_state_gas_used); + let Err(BalExecutionError::Execution(parallel_error)) = parallel_result else { + panic!("expected parallel state-gas admission error"); + }; + assert_eq!(parallel_error.to_string(), serial_error.to_string()); + } + } + } + #[test] fn shadow_tx_with_revert() { // A tx that reverts in a deployed contract. Both paths must produce identical receipts diff --git a/crates/engine/tree/src/tree/payload_processor/bal/ordered_outputs.rs b/crates/engine/tree/src/tree/payload_processor/bal/ordered_outputs.rs index bed19ef85f4..544c0e6eb4a 100644 --- a/crates/engine/tree/src/tree/payload_processor/bal/ordered_outputs.rs +++ b/crates/engine/tree/src/tree/payload_processor/bal/ordered_outputs.rs @@ -32,12 +32,13 @@ impl From for BalExecutionError { /// state and performs no canonical commit work. /// /// Behavior: -/// - each yielded `Ok` output has the next transaction index -/// - worker errors are forwarded unchanged -/// - closed channels before `total` outputs yield `Err` +/// - each yielded item has the next transaction index, `Ok` and indexed transaction errors alike: a +/// recovery or execution error is a per-transaction result, deferred to its slot so the consumer +/// observes it in block order +/// - non-indexed worker errors are forwarded immediately and exhaust the iterator +/// - closed channels before `total` outputs yield `Err` and exhaust the iterator /// - out-of-bounds and duplicate indices panic because they violate the internal worker/dispatcher /// index invariant -/// - after the first error, the iterator is exhausted pub(super) fn ordered_worker_outputs( result_rx: &Receiver, BalWorkerError>>, total: usize, @@ -47,9 +48,14 @@ pub(super) fn ordered_worker_outputs( struct OrderedWorkerOutputs<'a, R> { result_rx: &'a Receiver, BalWorkerError>>, - pending: Vec>>, + /// Slot per transaction: an out-of-order output (or its indexed transaction error) is parked + /// here until every earlier transaction has been yielded. + pending: Vec, BalWorkerError>>>, + /// Next transaction index to yield. next: usize, + /// Total number of transactions in the block. total: usize, + /// Set on a fatal error; the iterator is exhausted afterwards. failed: bool, } @@ -77,16 +83,27 @@ impl Iterator for OrderedWorkerOutputs<'_, R> { } loop { - if let Some(output) = self.pending[self.next].take() { + if let Some(slot) = self.pending[self.next].take() { self.next += 1; - return Some(Ok(output)); + return Some(slot.map_err(Into::into)); } - let output = match self.result_rx.recv() { - Ok(Ok(output)) => output, + let (index, slot) = match self.result_rx.recv() { + Ok(Ok(output)) => (output.index, Ok(output)), Ok(Err(err)) => { - self.failed = true; - return Some(Err(err.into())); + // Transaction failures are deferred to their transaction's slot so the + // consumer observes them in block order: verdict precedence must follow + // block position, not worker completion order. Errors without an index + // cannot be attributed to a transaction and end the iteration. + let tx_index = match &err { + BalWorkerError::Execution { tx_index, .. } | + BalWorkerError::Transaction { tx_index, .. } => *tx_index, + BalWorkerError::Setup(_) => { + self.failed = true; + return Some(Err(err.into())); + } + }; + (tx_index, Err(err)) } Err(_) => { self.failed = true; @@ -94,7 +111,6 @@ impl Iterator for OrderedWorkerOutputs<'_, R> { } }; - let index = output.index; assert!( index < self.total, "BAL worker returned out-of-bounds transaction index {index}; total={}", @@ -105,7 +121,7 @@ impl Iterator for OrderedWorkerOutputs<'_, R> { "BAL worker returned duplicate transaction index {index}", ); - self.pending[index] = Some(output); + self.pending[index] = Some(slot); } } } @@ -160,6 +176,68 @@ mod tests { assert!(outputs.next().is_none()); } + #[test] + fn defers_indexed_execution_errors_to_their_slot() { + let (tx, rx) = crossbeam_channel::unbounded(); + tx.send(Err(BalWorkerError::Execution { + tx_index: 1, + tx_gas_limit: 42, + source: alloy_evm::block::BlockExecutionError::msg("bal miss"), + })) + .unwrap(); + tx.send(Ok(output(0, 0))).unwrap(); + drop(tx); + + let mut outputs = ordered_worker_outputs(&rx, 2); + + assert_eq!(outputs.next().expect("first item").expect("first output").result, 0); + expect_err_contains(outputs.next().expect("second item"), "transaction 1: bal miss"); + assert!(outputs.next().is_none()); + } + + #[test] + fn continues_past_indexed_execution_errors() { + let (tx, rx) = crossbeam_channel::unbounded(); + tx.send(Err(BalWorkerError::Execution { + tx_index: 0, + tx_gas_limit: 42, + source: alloy_evm::block::BlockExecutionError::msg("bal miss"), + })) + .unwrap(); + tx.send(Ok(output(1, 10))).unwrap(); + drop(tx); + + let mut outputs = ordered_worker_outputs(&rx, 2); + + expect_err_contains(outputs.next().expect("first item"), "transaction 0: bal miss"); + assert_eq!(outputs.next().expect("second item").expect("second output").result, 10); + assert!(outputs.next().is_none()); + } + + #[test] + fn orders_recovery_and_execution_errors_by_transaction_index() { + let (tx, rx) = crossbeam_channel::unbounded(); + tx.send(Err(BalWorkerError::Execution { + tx_index: 2, + tx_gas_limit: 42, + source: alloy_evm::block::BlockExecutionError::msg("bal miss"), + })) + .unwrap(); + tx.send(Err(BalWorkerError::Transaction { + tx_index: 1, + source: std::io::Error::other("sig fail").into(), + })) + .unwrap(); + tx.send(Ok(output(0, 0))).unwrap(); + drop(tx); + + let mut outputs = ordered_worker_outputs(&rx, 3); + assert_eq!(outputs.next().unwrap().unwrap().result, 0); + expect_err_contains(outputs.next().unwrap(), "transaction 1: sig fail"); + expect_err_contains(outputs.next().unwrap(), "transaction 2: bal miss"); + assert!(outputs.next().is_none()); + } + #[test] fn rejects_closed_channel_before_all_outputs_arrive() { let (tx, rx) = crossbeam_channel::unbounded(); diff --git a/crates/engine/tree/src/tree/payload_processor/bal/worker.rs b/crates/engine/tree/src/tree/payload_processor/bal/worker.rs index 09d7cfdf67a..38b03a48ad9 100644 --- a/crates/engine/tree/src/tree/payload_processor/bal/worker.rs +++ b/crates/engine/tree/src/tree/payload_processor/bal/worker.rs @@ -17,21 +17,35 @@ pub(super) enum BalWorkerError { #[error("BAL worker setup failed: {0}")] Setup(#[source] BalExecutionError), /// Transaction recovery or conversion failed before EVM execution. - #[error("BAL worker transaction conversion failed: {0}")] - Transaction(Box), + #[error("BAL worker transaction conversion failed for transaction {tx_index}: {source}")] + Transaction { + /// Index of the transaction that failed. + tx_index: usize, + /// The underlying recovery or conversion error. + #[source] + source: Box, + }, /// EVM transaction execution failed. - #[error("BAL worker EVM execution failed: {0}")] - Execution(BlockExecutionError), + #[error("BAL worker EVM execution failed for transaction {tx_index}: {source}")] + Execution { + /// Index of the transaction that failed. + tx_index: usize, + /// Gas limit of the transaction that failed. + tx_gas_limit: u64, + /// The underlying execution error. + #[source] + source: BlockExecutionError, + }, } impl From for BalExecutionError { fn from(err: BalWorkerError) -> Self { match err { BalWorkerError::Setup(err) => err, - BalWorkerError::Transaction(err) => { - Self::Execution(BlockValidationError::Other(err).into()) + BalWorkerError::Transaction { source, .. } => { + Self::Execution(BlockValidationError::Other(source).into()) } - BalWorkerError::Execution(err) => Self::Execution(err), + BalWorkerError::Execution { source, .. } => Self::Execution(source), } } } @@ -69,42 +83,62 @@ pub(super) fn spawn_worker<'scope, Evm, Tx, Err, DB, MakeDb>( { scope.spawn(move |_| { let worker_result = (|| -> Result<(), BalWorkerError> { - // Create a database with fill_on_miss=true ensuring misses - // are inserted for the other workers. - let database = make_db(true).map_err(BalWorkerError::Setup)?; - let mut worker_state = State::builder() - .with_database(database) - .with_bal(received_bal_revm) - .with_bundle_update() - .build(); - let evm = evm_config.evm_with_env(&mut worker_state, evm_env); - let mut executor = evm_config.create_executor_with_state(evm, ctx.clone()); + // Keep the cache-filling database across executor resets so a speculative failure + // cannot introduce an unindexed provider setup error ahead of its ordered verdict. + let mut database = make_db(true).map_err(BalWorkerError::Setup)?; + 'worker: loop { + let mut worker_state = State::builder() + .with_database(&mut database) + .with_bal(Arc::clone(&received_bal_revm)) + .with_bundle_update() + .build(); + let evm = evm_config.evm_with_env(&mut worker_state, evm_env.clone()); + let mut executor = evm_config.create_executor_with_state(evm, ctx.clone()); - loop { - let (index, tx) = crossbeam_channel::select_biased! { - recv(abort_rx) -> _ => break, - recv(tx_rx) -> msg => match msg { - Ok(ix_tx) => ix_tx, - Err(_) => break, - }, - }; - let tx = tx.map_err(|e| BalWorkerError::Transaction(Box::new(e)))?; - let signer = *tx.signer(); - let tx_gas_limit = tx.tx().gas_limit(); + loop { + let (tx_index, tx) = crossbeam_channel::select_biased! { + recv(abort_rx) -> _ => break 'worker, + recv(tx_rx) -> msg => match msg { + Ok(ix_tx) => ix_tx, + Err(_) => break 'worker, + }, + }; + let tx = match tx { + Ok(tx) => tx, + Err(source) => { + let error = + BalWorkerError::Transaction { tx_index, source: Box::new(source) }; + if result_tx.send(Err(error)).is_err() { + break 'worker; + } + continue; + } + }; + let signer = *tx.signer(); + let tx_gas_limit = tx.tx().gas_limit(); - executor - .evm_mut() - .db_mut() - .set_bal_index(BlockAccessIndex::from_tx_index(index as u64)); - let result = executor - .execute_transaction_without_commit(tx) - .map_err(BalWorkerError::Execution)?; - - if result_tx - .send(Ok(BalWorkerOutput { index, signer, tx_gas_limit, result })) - .is_err() - { - break; + executor + .evm_mut() + .db_mut() + .set_bal_index(BlockAccessIndex::from_tx_index(tx_index as u64)); + let message = match executor.execute_transaction_without_commit(tx) { + Ok(result) => { + Ok(BalWorkerOutput { index: tx_index, signer, tx_gas_limit, result }) + } + Err(source) => { + Err(BalWorkerError::Execution { tx_index, tx_gas_limit, source }) + } + }; + let failed = message.is_err(); + if result_tx.send(message).is_err() { + break 'worker; + } + if failed { + // The executor trait does not guarantee reuse after an error. Rebuild + // its EVM and state before serving more work: the queue can still contain + // earlier transactions whose verdict must precede this failure. + break; + } } } diff --git a/crates/engine/tree/src/tree/payload_processor/mod.rs b/crates/engine/tree/src/tree/payload_processor/mod.rs index 9ef57678448..2fd848295aa 100644 --- a/crates/engine/tree/src/tree/payload_processor/mod.rs +++ b/crates/engine/tree/src/tree/payload_processor/mod.rs @@ -298,13 +298,14 @@ where .enumerate() .try_for_each(|(idx, tx)| { let tx = convert.convert(tx).map(WithTxEnv::new); - let failed = tx.is_err(); if let (Some(prewarm_tx), Ok(tx)) = (&prewarm_tx, &tx) { let _ = prewarm_tx.send((idx, tx.clone())); } let disconnected = execute_tx.send((idx, tx)).is_err(); trace!(target: "engine::tree::payload_processor", idx, "yielded transaction"); - if failed || disconnected { + // Recovery failures are adjudicated in transaction order by + // the BAL commit loop. Earlier slots may still be unconverted. + if disconnected { Err(()) } else { Ok(()) @@ -753,7 +754,7 @@ mod tests { #[test] fn transaction_conversion_stops_on_error() { for (count, bal, fail_at) in - [(10, false, 0), (200, false, 0), (200, false, 4), (200, false, 20), (200, true, 0)] + [(10, false, 0), (10, true, 0), (200, false, 0), (200, false, 4), (200, false, 20)] { let processor = test_processor(); let (_, receiver) = processor.spawn_tx_iterator( @@ -785,6 +786,32 @@ mod tests { } } + #[test] + fn bal_transaction_conversion_preserves_all_error_slots() { + let count = 200; + let processor = test_processor(); + let (_, receiver) = processor.spawn_tx_iterator( + ((0..count).collect::>(), |idx| { + if idx % 3 == 0 { + Err(std::io::Error::other("invalid transaction")) + } else { + Ok(converted_tx()) + } + }), + count, + true, + false, + ); + let mut indices = Vec::new(); + for _ in 0..count { + let (idx, tx) = receiver.recv_timeout(std::time::Duration::from_secs(10)).unwrap(); + assert_eq!(tx.is_err(), idx % 3 == 0); + indices.push(idx); + } + indices.sort_unstable(); + assert_eq!(indices, (0..count).collect::>()); + } + #[test] fn dropping_payload_handle_stops_transaction_conversion() { for (count, bal, pause_at) in From e4eab28843eb5a4ed8dd78317541ede1b48ee8ef Mon Sep 17 00:00:00 2001 From: joshieDo <93316087+joshieDo@users.noreply.github.com> Date: Tue, 15 Sep 2026 21:19:36 +0000 Subject: [PATCH 270/294] fix(prune): include transaction zero in lookup pruning (#27231) --- .../src/segments/user/transaction_lookup.rs | 139 ++++++++++++++++-- 1 file changed, 125 insertions(+), 14 deletions(-) diff --git a/crates/prune/prune/src/segments/user/transaction_lookup.rs b/crates/prune/prune/src/segments/user/transaction_lookup.rs index 8a85655e463..1aed8438302 100644 --- a/crates/prune/prune/src/segments/user/transaction_lookup.rs +++ b/crates/prune/prune/src/segments/user/transaction_lookup.rs @@ -70,21 +70,22 @@ where provider.static_file_provider().get_lowest_range(StaticFileSegment::Transactions) && input .previous_checkpoint - .is_none_or(|checkpoint| checkpoint.block_number < Some(lowest_range.start())) + .is_none_or(|checkpoint| checkpoint.block_number < Some(lowest_range.start())) && + let Some(previous_block) = lowest_range.start().checked_sub(1) && + let Some(body_indices) = provider.block_body_indices(previous_block)? && + let Some(tx_number) = body_indices.next_tx_num().checked_sub(1) { - let new_checkpoint = lowest_range.start().saturating_sub(1); - if let Some(body_indices) = provider.block_body_indices(new_checkpoint)? { - input.previous_checkpoint = Some(PruneCheckpoint { - block_number: Some(new_checkpoint), - tx_number: Some(body_indices.last_tx_num()), - prune_mode: self.mode, - }); - debug!( - target: "pruner", - static_file_checkpoint = ?input.previous_checkpoint, - "Using static file transaction checkpoint as TransactionLookup starting point" - ); - } + // Only skip earlier transactions if any exist. + input.previous_checkpoint = Some(PruneCheckpoint { + block_number: Some(previous_block), + tx_number: Some(tx_number), + prune_mode: self.mode, + }); + debug!( + target: "pruner", + static_file_checkpoint = ?input.previous_checkpoint, + "Using static file transaction checkpoint as TransactionLookup starting point" + ); } let (start, end) = match input.get_next_tx_num_range(provider)? { @@ -317,6 +318,116 @@ mod tests { use reth_testing_utils::generators::{self, random_block_range, BlockRangeParams}; use std::ops::Sub; + #[test] + fn prune_transaction_zero_after_empty_genesis() { + let db = TestStageDB::default(); + let mut rng = generators::rng(); + let blocks = [ + random_block_range( + &mut rng, + 0..=0, + BlockRangeParams { tx_count: 0..1, ..Default::default() }, + ), + random_block_range( + &mut rng, + 1..=2, + BlockRangeParams { tx_count: 1..2, ..Default::default() }, + ), + ] + .concat(); + db.insert_blocks(blocks.iter(), StorageKind::Static).unwrap(); + let hashes = blocks + .iter() + .flat_map(|block| &block.body().transactions) + .map(|tx| *tx.tx_hash()) + .collect::>(); + db.insert_tx_hash_numbers(hashes.iter().enumerate().map(|(id, hash)| (*hash, id as u64))) + .unwrap(); + + let provider = db.factory.database_provider_rw().unwrap(); + let result = TransactionLookup::new(PruneMode::Before(2)) + .prune( + &provider, + PruneInput { + previous_checkpoint: None, + to_block: 1, + limiter: PruneLimiter::default().set_deleted_entries_limit(10), + }, + ) + .unwrap(); + assert_eq!(result.pruned, 1); + assert_eq!(result.checkpoint.unwrap().tx_number, Some(0)); + provider.commit().unwrap(); + assert_eq!(db.table::().unwrap(), vec![(hashes[1], 1)]); + } + + #[test] + fn prune_transaction_zero_when_earlier_blocks_are_empty() { + use reth_provider::{StaticFileProviderFactory, StaticFileWriter}; + use reth_static_file_types::{StaticFileSegment, DEFAULT_BLOCKS_PER_STATIC_FILE}; + + let db = TestStageDB::default(); + let mut rng = generators::rng(); + let first_retained_block = DEFAULT_BLOCKS_PER_STATIC_FILE; + let blocks = [ + random_block_range( + &mut rng, + first_retained_block - 1..=first_retained_block - 1, + BlockRangeParams { tx_count: 0..1, ..Default::default() }, + ), + random_block_range( + &mut rng, + first_retained_block..=first_retained_block + 1, + BlockRangeParams { tx_count: 1..2, ..Default::default() }, + ), + ] + .concat(); + db.insert_blocks(blocks.iter(), StorageKind::Database(None)).unwrap(); + + // Earlier blocks were empty, so this file contains transaction zero. + let static_files = db.factory.static_file_provider(); + { + let mut writer = static_files + .get_writer(first_retained_block, StaticFileSegment::Transactions) + .unwrap(); + for (tx_number, block) in blocks[1..].iter().enumerate() { + writer.append_transaction(tx_number as u64, &block.body().transactions[0]).unwrap(); + writer.increment_block(block.number).unwrap(); + } + writer.commit().unwrap(); + } + assert_eq!( + static_files.get_lowest_range(StaticFileSegment::Transactions).unwrap().start(), + first_retained_block, + ); + + let hashes = blocks[1..] + .iter() + .map(|block| *block.body().transactions[0].tx_hash()) + .collect::>(); + db.insert_tx_hash_numbers(hashes.iter().enumerate().map(|(id, hash)| (*hash, id as u64))) + .unwrap(); + + let provider = db.factory.database_provider_rw().unwrap(); + let result = TransactionLookup::new(PruneMode::Before(first_retained_block + 1)) + .prune( + &provider, + PruneInput { + previous_checkpoint: None, + to_block: first_retained_block, + limiter: PruneLimiter::default().set_deleted_entries_limit(10), + }, + ) + .unwrap(); + assert!(result.progress.is_finished()); + assert_eq!(result.pruned, 1); + let checkpoint = result.checkpoint.unwrap(); + assert_eq!(checkpoint.block_number, Some(first_retained_block)); + assert_eq!(checkpoint.tx_number, Some(0)); + provider.commit().unwrap(); + assert_eq!(db.table::().unwrap(), vec![(hashes[1], 1)]); + } + #[test] fn prune() { let db = TestStageDB::default(); From 412fccc9b5381fcb2444d0e65fb672acc2e07d68 Mon Sep 17 00:00:00 2001 From: Sergey Melnychuk <8093171+sergey-melnychuk@users.noreply.github.com> Date: Wed, 16 Sep 2026 00:07:34 +0000 Subject: [PATCH 271/294] fix(examples): read hashed storage in full-contract-state example (#27261) Co-authored-by: sergey-melnychuk --- examples/full-contract-state/README.md | 9 ++++ examples/full-contract-state/src/main.rs | 66 ++++++++++++++++++------ 2 files changed, 60 insertions(+), 15 deletions(-) diff --git a/examples/full-contract-state/README.md b/examples/full-contract-state/README.md index c0cd4a0def4..cfebb47e907 100644 --- a/examples/full-contract-state/README.md +++ b/examples/full-contract-state/README.md @@ -51,6 +51,15 @@ For efficient storage iteration, the example uses database cursors: - `seek_exact` - Positions cursor at specific key - `next_dup` - Iterates through duplicate entries +### Hashed State +Depending on `cached_storage_settings().use_hashed_state()`, a node keeps +either `PlainStorageState` (keyed by the raw 32-byte slot) or +`HashedStorages` (keyed by `keccak256(slot)`) up to date -- the table that +isn't active for the running node is simply empty, not an error. This +example checks that flag and reads the right table; in hashed mode the +printed keys are `keccak256(slot)`, not the original slot number, since +hashed-only storage doesn't retain a plain-key mirror to recover it from. + ## Output The example will print: diff --git a/examples/full-contract-state/src/main.rs b/examples/full-contract-state/src/main.rs index 22082cdbe23..f887fde5010 100644 --- a/examples/full-contract-state/src/main.rs +++ b/examples/full-contract-state/src/main.rs @@ -7,7 +7,7 @@ //! 3. Get contract bytecode //! 4. Iterate through all storage slots for the contract -use alloy_primitives::{map::B256Map, Address, U256}; +use alloy_primitives::{keccak256, map::B256Map, Address, U256}; use reth_ethereum::{ chainspec::ChainSpecBuilder, node::EthereumNode, @@ -21,7 +21,7 @@ use reth_ethereum::{ providers::ReadOnlyConfig, ProviderResult, }, - storage::{DBProvider, StateProvider}, + storage::{DBProvider, StateProvider, StorageSettingsCache}, }; use std::str::FromStr; @@ -34,12 +34,27 @@ pub struct ContractState { pub account: Account, /// Contract bytecode (None if not a contract or doesn't exist) pub bytecode: Option, - /// All storage slots for the contract + /// All storage slots for the contract. Keyed by `keccak256(slot)` when + /// `hashed` is true, by the raw slot otherwise -- see `hashed`. pub storage: B256Map, + /// Whether `storage` is keyed by `keccak256(slot)` rather than the raw + /// slot. True when the node keeps only hashed state (no `PlainStorageState` + /// mirror), which some storage settings/configurations use to save space. + pub hashed: bool, } -/// Extract the full state of a specific contract -pub fn extract_contract_state( +/// Extract the full state of a specific contract. +/// +/// A node keeps either `PlainStorageState` (keyed by the raw 32-byte slot) +/// or `HashedStorages` (keyed by `keccak256(slot)`) up to date, depending on +/// `cached_storage_settings().use_hashed_state()` -- this is exactly what +/// `StateProvider::storage()` itself branches on internally +/// (`providers/state/latest.rs`). Reading the wrong one doesn't error, it +/// just silently returns zero rows, since the table simply isn't maintained +/// in that mode. Hashed mode has no plain-key mirror to recover the raw slot +/// number from (that's the space it saves), so in that case this returns +/// `keccak256(slot) -> value` pairs instead of `slot -> value`. +pub fn extract_contract_state( provider: &P, state_provider: &dyn StateProvider, contract_address: Address, @@ -51,18 +66,29 @@ pub fn extract_contract_state( let bytecode = state_provider.account_code(&contract_address)?; - let mut storage_cursor = provider.tx_ref().cursor_dup_read::()?; + let hashed = provider.cached_storage_settings().use_hashed_state(); let mut storage = B256Map::default(); - if let Some((_, first_entry)) = storage_cursor.seek_exact(contract_address)? { - storage.insert(first_entry.key, first_entry.value); - - while let Some((_, entry)) = storage_cursor.next_dup()? { - storage.insert(entry.key, entry.value); + if hashed { + let hashed_address = keccak256(contract_address); + let mut cursor = provider.tx_ref().cursor_dup_read::()?; + if let Some((_, first_entry)) = cursor.seek_exact(hashed_address)? { + storage.insert(first_entry.key, first_entry.value); + while let Some((_, entry)) = cursor.next_dup()? { + storage.insert(entry.key, entry.value); + } + } + } else { + let mut cursor = provider.tx_ref().cursor_dup_read::()?; + if let Some((_, first_entry)) = cursor.seek_exact(contract_address)? { + storage.insert(first_entry.key, first_entry.value); + while let Some((_, entry)) = cursor.next_dup()? { + storage.insert(entry.key, entry.value); + } } } - Ok(Some(ContractState { address: contract_address, account, bytecode, storage })) + Ok(Some(ContractState { address: contract_address, account, bytecode, storage, hashed })) } fn main() -> eyre::Result<()> { @@ -88,9 +114,19 @@ fn main() -> eyre::Result<()> { println!("Balance: {}", state.account.balance); println!("Nonce: {}", state.account.nonce); println!("Code hash: {:?}", state.account.bytecode_hash); - println!("Storage slots: {}", state.storage.len()); - for (key, value) in &state.storage { - println!("\t{key}: {value}"); + if state.hashed { + println!( + "Storage slots: {} (hashed-state node -- keys below are keccak256(slot), not the raw slot number)", + state.storage.len() + ); + for (hashed_key, value) in &state.storage { + println!("\tkeccak256(slot)={hashed_key}: {value}"); + } + } else { + println!("Storage slots: {}", state.storage.len()); + for (key, value) in &state.storage { + println!("\t{key}: {value}"); + } } } From 0c8b1c187afdf0f4225a6533ca1b0380ecb16abb Mon Sep 17 00:00:00 2001 From: rm-rf Date: Wed, 16 Sep 2026 01:03:52 +0000 Subject: [PATCH 272/294] fix(txpool): notify promotions from parked insertions (#27259) Co-authored-by: Matthias Seitz <19890894+mattsse@users.noreply.github.com> --- crates/transaction-pool/src/pool/mod.rs | 222 ++++++++++++++++++++- crates/transaction-pool/src/pool/txpool.rs | 1 + 2 files changed, 218 insertions(+), 5 deletions(-) diff --git a/crates/transaction-pool/src/pool/mod.rs b/crates/transaction-pool/src/pool/mod.rs index b4a3897436d..3cd955f21c6 100644 --- a/crates/transaction-pool/src/pool/mod.rs +++ b/crates/transaction-pool/src/pool/mod.rs @@ -742,7 +742,7 @@ where /// /// Performs blob storage operations and sends all notifications. This should be called /// after the pool write lock has been released to avoid blocking pool operations. - fn on_added_transaction(&self, meta: AddedTransactionMeta) { + fn on_added_transaction(&self, mut meta: AddedTransactionMeta) { // Handle blob sidecar storage and notifications for EIP-4844 transactions if let Some(sidecar) = meta.blob_sidecar { let hash = *meta.added.hash(); @@ -766,6 +766,13 @@ where self.on_new_pending_transaction(pending); } + // A parked insertion can promote older transactions when validation observes newer + // account state. Notify those promotions independently of the inserted transaction. + let promoted = meta.added.take_parked_promoted(); + if !promoted.is_empty() { + self.notify_on_transaction_updates(promoted, Vec::new()); + } + // Notify event listeners self.notify_event_listeners(&meta.added); @@ -1473,6 +1480,12 @@ pub enum AddedTransaction { subpool: SubPool, /// The specific reason why the transaction is queued (if applicable). queued_reason: Option, + /// Existing transactions promoted to pending by this insertion. + /// + /// Validation can observe newer account state before pool maintenance applies it. + /// Insertion rechecks the sender's transactions against that state, so lower-nonce + /// transactions can become pending while the inserted transaction remains parked. + promoted: Vec>>, }, } @@ -1493,6 +1506,17 @@ impl AddedTransaction { } } + /// Takes promotions caused by a parked insertion for separate notification. + /// + /// Call before notifying listeners about the inserted transaction. This drains only the + /// parked promotions; subsequent parked event conversion handles the inserted transaction. + pub(crate) fn take_parked_promoted(&mut self) -> Vec>> { + match self { + Self::Parked { promoted, .. } => std::mem::take(promoted), + Self::Pending(_) => Vec::new(), + } + } + /// Returns the discarded transactions if there were any pub(crate) fn discarded_transactions(&self) -> Option<&[Arc>]> { match self { @@ -1682,15 +1706,47 @@ mod tests { use crate::{ blobstore::{BlobStore, InMemoryBlobStore, PooledBlobSidecar}, identifier::SenderId, - test_utils::{testing_pool, MockTransaction, TestPoolBuilder}, + test_utils::{testing_pool, MockTransaction, TestPool, TestPoolBuilder}, validate::ValidTransaction, - BlockInfo, PoolConfig, SubPoolLimit, TransactionOrigin, TransactionPool, - TransactionPoolExt, TransactionValidationOutcome, ValidPoolTransaction, U256, + BlockInfo, FullTransactionEvent, PoolConfig, SubPool, SubPoolLimit, + TransactionListenerKind, TransactionOrigin, TransactionPool, TransactionPoolExt, + TransactionValidationOutcome, ValidPoolTransaction, U256, }; use alloy_consensus::Transaction; use alloy_eips::{eip4844::BlobTransactionSidecar, eip7594::BlobTransactionSidecarVariant}; - use alloy_primitives::Address; + use alloy_primitives::{Address, B256}; + use futures_util::{FutureExt, StreamExt}; use std::{fs, path::PathBuf, sync::Arc}; + use tokio::sync::mpsc::error::TryRecvError; + + /// Inserts a transaction with an explicit validator snapshot, without running maintenance. + fn insert_with_state_nonce(pool: &TestPool, nonce: u64, state_nonce: u64, propagate: bool) { + let transaction = MockTransaction::eip1559() + .with_sender(Address::with_last_byte(1)) + .with_nonce(nonce) + .with_hash(B256::from([nonce as u8; 32])); + pool.pool + .add_transactions( + TransactionOrigin::External, + [TransactionValidationOutcome::Valid { + balance: U256::MAX, + state_nonce, + bytecode_hash: None, + transaction: ValidTransaction::Valid(transaction), + propagate, + authorities: None, + }], + ) + .pop() + .unwrap() + .unwrap(); + } + + fn transaction_nonces(txs: &[Arc>]) -> Vec { + let mut nonces: Vec<_> = txs.iter().map(|tx| tx.transaction.nonce()).collect(); + nonces.sort_unstable(); + nonces + } #[tokio::test] async fn all_transactions_by_sender_across_reclassification() { @@ -1725,6 +1781,162 @@ mod tests { assert_eq!(nonces(&pool.get_transactions_by_sender(sender)), [0, 1, 9]); } + #[test] + fn queued_insertion_notifies_older_promotions() { + let test_pool = testing_pool(); + let pool = &test_pool.pool; + let mut network = pool.add_pending_listener(TransactionListenerKind::PropagateOnly); + let mut all = pool.add_pending_listener(TransactionListenerKind::All); + // Missing nonce 0 keeps both transactions queued. + insert_with_state_nonce(&test_pool, 1, 0, true); + insert_with_state_nonce(&test_pool, 2, 0, true); + let txs = test_pool.all_transactions_by_sender(Address::with_last_byte(1)); + assert!(txs.pending.is_empty()); + assert_eq!(transaction_nonces(&txs.queued), [1, 2]); + assert_eq!(network.try_recv(), Err(TryRecvError::Empty)); + assert_eq!(all.try_recv(), Err(TryRecvError::Empty)); + + let mut full_network = + pool.add_new_transaction_listener(TransactionListenerKind::PropagateOnly); + let mut full_all = pool.add_new_transaction_listener(TransactionListenerKind::All); + let mut events = pool.add_all_transactions_event_listener(); + + // A newer validation result says nonce 0 is mined. Nonces 1 and 2 become + // pending, but inserting 4 leaves a gap at 3. + insert_with_state_nonce(&test_pool, 4, 1, true); + let txs = test_pool.all_transactions_by_sender(Address::with_last_byte(1)); + assert_eq!(transaction_nonces(&txs.pending), [1, 2]); + assert_eq!(transaction_nonces(&txs.queued), [4]); + + let expected = vec![B256::from([1; 32]), B256::from([2; 32])]; + for (kind, listener) in [("network", &mut network), ("all", &mut all)] { + let mut received = Vec::new(); + while let Ok(hash) = listener.try_recv() { + received.push(hash); + } + received.sort_unstable(); + assert_eq!(received, expected, "{kind} pending notifications"); + } + + // Full-transaction subscribers get each promotion once and the inserted + // transaction once, still marked queued. Propagation filtering applies here too. + for (kind, listener) in [("network", &mut full_network), ("all", &mut full_all)] { + let mut received = Vec::new(); + while let Ok(event) = listener.try_recv() { + received.push((*event.transaction.hash(), event.subpool)); + } + received.sort_unstable_by_key(|(hash, _)| *hash); + let expected = vec![ + (B256::from([1; 32]), SubPool::Pending), + (B256::from([2; 32]), SubPool::Pending), + (B256::from([4; 32]), SubPool::Queued), + ]; + assert_eq!(received, expected, "{kind} full transaction notifications"); + } + + let mut pending_events = Vec::new(); + let mut queued_events = Vec::new(); + while let Some(Some(event)) = events.next().now_or_never() { + match event { + FullTransactionEvent::Pending(hash) => pending_events.push(hash), + FullTransactionEvent::Queued(hash, _) => queued_events.push(hash), + other => panic!("unexpected event: {other:?}"), + } + } + pending_events.sort_unstable(); + assert_eq!(pending_events, expected); + assert_eq!(queued_events, vec![B256::from([4; 32])]); + + // A later insertion observing the same state must not notify those promotions again. + insert_with_state_nonce(&test_pool, 5, 1, true); + assert_eq!(network.try_recv(), Err(TryRecvError::Empty)); + assert_eq!(all.try_recv(), Err(TryRecvError::Empty)); + for listener in [&mut full_network, &mut full_all] { + let event = listener.try_recv().unwrap(); + assert_eq!(*event.transaction.hash(), B256::from([5; 32])); + assert_eq!(event.subpool, SubPool::Queued); + assert!(matches!(listener.try_recv(), Err(TryRecvError::Empty))); + } + assert!(matches!( + events.next().now_or_never(), + Some(Some(FullTransactionEvent::Queued(hash, _))) if hash == B256::from([5; 32]) + )); + assert!(events.next().now_or_never().is_none()); + } + + #[test] + fn queued_insertion_promotions_respect_propagation_filter() { + let test_pool = testing_pool(); + let pool = &test_pool.pool; + insert_with_state_nonce(&test_pool, 1, 0, false); + insert_with_state_nonce(&test_pool, 2, 0, true); + + let mut network = pool.add_pending_listener(TransactionListenerKind::PropagateOnly); + let mut all = pool.add_pending_listener(TransactionListenerKind::All); + let mut full_network = + pool.add_new_transaction_listener(TransactionListenerKind::PropagateOnly); + let mut full_all = pool.add_new_transaction_listener(TransactionListenerKind::All); + insert_with_state_nonce(&test_pool, 4, 1, true); + + assert_eq!(network.try_recv().unwrap(), B256::from([2; 32])); + assert_eq!(network.try_recv(), Err(TryRecvError::Empty)); + let mut received = Vec::new(); + while let Ok(hash) = all.try_recv() { + received.push(hash); + } + received.sort_unstable(); + assert_eq!(received, [B256::from([1; 32]), B256::from([2; 32])]); + + for (listener, expected) in [ + ( + &mut full_network, + vec![ + (B256::from([2; 32]), SubPool::Pending), + (B256::from([4; 32]), SubPool::Queued), + ], + ), + ( + &mut full_all, + vec![ + (B256::from([1; 32]), SubPool::Pending), + (B256::from([2; 32]), SubPool::Pending), + (B256::from([4; 32]), SubPool::Queued), + ], + ), + ] { + let mut received = Vec::new(); + while let Ok(event) = listener.try_recv() { + received.push((*event.transaction.hash(), event.subpool)); + } + received.sort_unstable_by_key(|(hash, _)| *hash); + assert_eq!(received, expected); + } + } + + #[test] + fn pending_insertion_notifies_older_promotions() { + let test_pool = testing_pool(); + let pool = &test_pool.pool; + insert_with_state_nonce(&test_pool, 1, 0, true); + insert_with_state_nonce(&test_pool, 2, 0, true); + let mut network = pool.add_pending_listener(TransactionListenerKind::PropagateOnly); + let mut all = pool.add_pending_listener(TransactionListenerKind::All); + + // Filling the gap makes both the older transactions and the insertion pending. + insert_with_state_nonce(&test_pool, 3, 1, true); + let txs = test_pool.all_transactions_by_sender(Address::with_last_byte(1)); + assert_eq!(transaction_nonces(&txs.pending), [1, 2, 3]); + assert!(txs.queued.is_empty()); + for listener in [&mut network, &mut all] { + let mut received = Vec::new(); + while let Ok(hash) = listener.try_recv() { + received.push(hash); + } + received.sort_unstable(); + assert_eq!(received, [B256::from([1; 32]), B256::from([2; 32]), B256::from([3; 32])]); + } + } + #[test] fn test_discard_blobs_on_blob_tx_eviction() { let blobs = { diff --git a/crates/transaction-pool/src/pool/txpool.rs b/crates/transaction-pool/src/pool/txpool.rs index 01608d93fda..493aca1eef1 100644 --- a/crates/transaction-pool/src/pool/txpool.rs +++ b/crates/transaction-pool/src/pool/txpool.rs @@ -869,6 +869,7 @@ impl TxPool { subpool: move_to, replaced, queued_reason, + promoted, } }; From 259f70878a51335beb3050cd36638e42c490841a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A9a=20Narzis?= <78718413+lean-apple@users.noreply.github.com> Date: Wed, 16 Sep 2026 08:20:15 +0000 Subject: [PATCH 273/294] chore: clean up cargo manifest warnings (#27263) --- Cargo.toml | 35 ------------------- crates/node/builder/Cargo.toml | 1 - crates/storage/libmdbx-rs/mdbx-sys/Cargo.toml | 2 ++ .../beacon-api-sidecar-fetcher/Cargo.toml | 2 ++ examples/beacon-api-sse/Cargo.toml | 2 ++ examples/bsc-p2p/Cargo.toml | 2 ++ .../custom-auth-http-middleware/Cargo.toml | 2 ++ examples/custom-beacon-withdrawals/Cargo.toml | 2 ++ examples/custom-dev-node/Cargo.toml | 2 ++ examples/custom-engine-types/Cargo.toml | 2 ++ examples/custom-evm/Cargo.toml | 2 ++ examples/custom-hardforks/Cargo.toml | 2 ++ examples/custom-inspector/Cargo.toml | 2 ++ examples/custom-node-components/Cargo.toml | 2 ++ examples/custom-payload-builder/Cargo.toml | 2 ++ examples/custom-rlpx-subprotocol/Cargo.toml | 2 ++ examples/custom-rpc-middleware/Cargo.toml | 2 ++ examples/custom-state-root/Cargo.toml | 2 ++ examples/db-access/Cargo.toml | 2 ++ examples/exex-subscription/Cargo.toml | 2 ++ examples/exex-test/Cargo.toml | 2 ++ examples/manual-p2p/Cargo.toml | 2 ++ examples/network-proxy/Cargo.toml | 2 ++ examples/network-txpool/Cargo.toml | 2 ++ examples/network/Cargo.toml | 2 ++ examples/node-builder-api/Cargo.toml | 2 ++ examples/node-custom-rpc/Cargo.toml | 1 + examples/node-event-hooks/Cargo.toml | 2 ++ examples/polygon-p2p/Cargo.toml | 2 ++ examples/rpc-db/Cargo.toml | 2 ++ examples/txpool-tracing/Cargo.toml | 2 ++ 31 files changed, 57 insertions(+), 36 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 8a54e15dbe9..53965b57963 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -316,8 +316,6 @@ incremental = false [workspace.dependencies] # reth -reth = { path = "bin/reth" } -reth-storage-rpc-provider = { path = "crates/storage/rpc-provider" } reth-basic-payload-builder = { path = "crates/payload/basic" } reth-chain-state = { path = "crates/chain-state" } reth-chainspec = { path = "crates/chainspec", default-features = false } @@ -365,7 +363,6 @@ reth-evm-ethereum = { path = "crates/ethereum/evm", default-features = false } reth-execution-errors = { path = "crates/evm/execution-errors", default-features = false } reth-execution-types = { path = "crates/evm/execution-types", default-features = false } reth-exex = { path = "crates/exex/exex" } -reth-exex-test-utils = { path = "crates/exex/test-utils" } reth-exex-types = { path = "crates/exex/types" } reth-fs-util = { path = "crates/fs-util" } reth-invalid-block-hooks = { path = "crates/engine/invalid-block-hooks" } @@ -393,7 +390,6 @@ reth-payload-builder = { path = "crates/payload/builder" } reth-payload-builder-primitives = { path = "crates/payload/builder-primitives" } reth-payload-primitives = { path = "crates/payload/primitives" } reth-payload-validator = { path = "crates/payload/validator" } -reth-payload-util = { path = "crates/payload/util" } reth-primitives-traits = { version = "0.7.1", default-features = false } reth-provider = { path = "crates/storage/provider" } reth-prune = { path = "crates/prune/prune" } @@ -402,7 +398,6 @@ reth-revm = { path = "crates/revm", default-features = false } reth-rpc = { path = "crates/rpc/rpc" } reth-rpc-api = { path = "crates/rpc/rpc-api" } reth-rpc-builder = { path = "crates/rpc/rpc-builder" } -reth-rpc-e2e-tests = { path = "crates/rpc/rpc-e2e-tests" } reth-rpc-engine-api = { path = "crates/rpc/rpc-engine-api" } reth-rpc-eth-api = { path = "crates/rpc/rpc-eth-api" } reth-rpc-eth-types = { path = "crates/rpc/rpc-eth-types", default-features = false } @@ -410,7 +405,6 @@ reth-rpc-layer = { path = "crates/rpc/rpc-layer" } reth-rpc-server-types = { path = "crates/rpc/rpc-server-types" } reth-rpc-convert = { path = "crates/rpc/rpc-convert" } reth-rpc-traits = { version = "0.7.0", default-features = false } -reth-snap-sync = { path = "crates/snap-sync" } reth-stages = { path = "crates/stages/stages" } reth-stages-api = { path = "crates/stages/api" } reth-stages-types = { path = "crates/stages/types", default-features = false } @@ -457,10 +451,8 @@ alloy-eips = { version = "2.4.2", default-features = false } alloy-genesis = { version = "2.4.2", default-features = false } alloy-json-rpc = { version = "2.4.2", default-features = false } alloy-network = { version = "2.4.2", default-features = false } -alloy-network-primitives = { version = "2.4.2", default-features = false } alloy-node-bindings = "2.4.2" alloy-provider = { version = "2.4.2", features = ["reqwest", "debug-api"], default-features = false } -alloy-pubsub = { version = "2.4.2", default-features = false } alloy-rpc-client = { version = "2.4.2", default-features = false } alloy-rpc-types = { version = "2.4.2", features = ["eth"], default-features = false } alloy-rpc-types-admin = { version = "2.4.2", default-features = false } @@ -476,9 +468,6 @@ alloy-serde = { version = "2.4.2", default-features = false } alloy-signer = { version = "2.4.2", default-features = false } alloy-signer-local = { version = "2.4.2", default-features = false } alloy-transport = { version = "2.4.2" } -alloy-transport-http = { version = "2.4.2", features = ["reqwest-rustls-tls"], default-features = false } -alloy-transport-ipc = { version = "2.4.2", default-features = false } -alloy-transport-ws = { version = "2.4.2", default-features = false } # misc either = { version = "1.16.0", default-features = false } @@ -494,7 +483,6 @@ blake3 = "1.8" brotli = "8" cfg-if = "1.0" clap = "4" -color-eyre = "0.6" dashmap = "6.2" derive_more = { version = "2", default-features = false, features = ["full"] } dirs-next = "2.0.0" @@ -525,12 +513,9 @@ serde = { version = "1.0", default-features = false } serde_json = { version = "1.0", default-features = false, features = ["alloc"] } serde_with = { version = "3", default-features = false, features = ["macros"] } sha2 = { version = "0.10", default-features = false } -shlex = "1.3" slotmap = "1" smallvec = "1" strum = { version = "0.27", default-features = false } -strum_macros = "0.27" -syn = "2.0" thiserror = { version = "2.0.0", default-features = false } tar = "0.4.46" tracing = { version = "0.1.0", default-features = false, features = ["attributes"] } @@ -540,8 +525,6 @@ zstd = "0.13" byteorder = "1" fixed-cache = { version = "0.1.10", default-features = false, features = ["alloc"] } moka = "0.12" -tar-no-std = { version = "0.4.2", default-features = false } -miniz_oxide = { version = "0.9.0", default-features = false } chrono = "0.4.41" # metrics @@ -551,10 +534,6 @@ metrics-exporter-prometheus = { version = "0.18.3", default-features = false } metrics-process = "2.1.0" metrics-util = { default-features = false, version = "0.20.3" } -# proc-macros -proc-macro2 = "1.0" -quote = "1.0" - # tokio tokio = { version = "1.52.3", default-features = false } tokio-stream = "0.1.11" @@ -589,20 +568,15 @@ jsonrpsee-types = "0.26.0" # http http = "1.0" -http-body = "1.0" http-body-util = "0.1.2" proptest-arbitrary-interop = "0.1.0" # crypto enr = { version = "0.13", default-features = false } -k256 = { version = "0.13", default-features = false, features = ["ecdsa"] } secp256k1 = { version = "0.30", default-features = false, features = ["global-context", "recovery"] } # rand 8 for secp256k1 rand_08 = { package = "rand", version = "0.8" } -# for eip-4844 -c-kzg = "2.1.8" - # config toml = "0.9" @@ -623,11 +597,9 @@ assert_matches = "1.5.0" criterion = { package = "codspeed-criterion-compat", version = "4.3" } insta = "1.47" proptest = "1.7" -proptest-derive = "0.7" similar-asserts = { version = "1.5.0", features = ["serde"] } tempfile = "3.20" test-fuzz = "7" -rstest = "0.26.1" test-case = "3" # ssz encoding @@ -643,7 +615,6 @@ tracy-client = { version = "0.18.0", features = ["demangle"] } snmalloc-rs = { version = "0.3.7", features = ["build_cc"] } aes = "0.8.1" -ahash = "0.8" anyhow = "1.0" bindgen = { version = "0.72", default-features = false } cc = "1.2.62" @@ -654,11 +625,8 @@ crossbeam-channel = "0.5.13" crossbeam-queue = "0.3" crossbeam-utils = "0.8" crossterm = "0.29.0" -csv = "1.3.0" -ctrlc = "3.4" ctr = "0.9.2" data-encoding = "2" -delegate = "0.13" digest = "0.10.5" hash-db = "=0.15.2" hickory-resolver = "0.26.1" @@ -677,7 +645,6 @@ ringbuffer = "0.16.0" rmp-serde = "1.3" roaring = "0.11.3" rolling-file = "0.2.0" -sha3 = "0.10.5" snap = "1.1.1" socket2 = { version = "0.6", default-features = false } sysinfo = { version = "0.38", default-features = false } @@ -688,9 +655,7 @@ tracing-chrome = "0.7.2" tracing-subscriber = { version = "0.3", default-features = false } tracing-tracy = "0.11" triehash = "0.8" -typenum = "1.15.0" vergen = "10.0.1" -visibility = "0.1.1" walkdir = "2.3.3" vergen-git2 = "10.0.1" diff --git a/crates/node/builder/Cargo.toml b/crates/node/builder/Cargo.toml index 524266446a9..a522d3bfa20 100644 --- a/crates/node/builder/Cargo.toml +++ b/crates/node/builder/Cargo.toml @@ -6,7 +6,6 @@ rust-version.workspace = true license.workspace = true homepage.workspace = true repository.workspace = true -readme = "README.md" [lints] workspace = true diff --git a/crates/storage/libmdbx-rs/mdbx-sys/Cargo.toml b/crates/storage/libmdbx-rs/mdbx-sys/Cargo.toml index 1fb1b9b4a3c..8a6082e3372 100644 --- a/crates/storage/libmdbx-rs/mdbx-sys/Cargo.toml +++ b/crates/storage/libmdbx-rs/mdbx-sys/Cargo.toml @@ -11,3 +11,5 @@ repository.workspace = true [build-dependencies] cc.workspace = true bindgen = { workspace = true, features = ["runtime"] } + +[lints] diff --git a/examples/beacon-api-sidecar-fetcher/Cargo.toml b/examples/beacon-api-sidecar-fetcher/Cargo.toml index 90fa08efab8..9ee796a7fcf 100644 --- a/examples/beacon-api-sidecar-fetcher/Cargo.toml +++ b/examples/beacon-api-sidecar-fetcher/Cargo.toml @@ -20,3 +20,5 @@ reqwest.workspace = true serde_json.workspace = true serde.workspace = true thiserror.workspace = true + +[lints] diff --git a/examples/beacon-api-sse/Cargo.toml b/examples/beacon-api-sse/Cargo.toml index 1f30eeebc8b..fc9360da58c 100644 --- a/examples/beacon-api-sse/Cargo.toml +++ b/examples/beacon-api-sse/Cargo.toml @@ -17,3 +17,5 @@ reqwest.workspace = true serde_json.workspace = true tokio = { workspace = true, features = ["time"] } tracing.workspace = true + +[lints] diff --git a/examples/bsc-p2p/Cargo.toml b/examples/bsc-p2p/Cargo.toml index 43e3ab0374f..ba5461ee382 100644 --- a/examples/bsc-p2p/Cargo.toml +++ b/examples/bsc-p2p/Cargo.toml @@ -56,3 +56,5 @@ serde = [ "reth-network-api/serde", "reth-ethereum-primitives/serde", ] + +[lints] diff --git a/examples/custom-auth-http-middleware/Cargo.toml b/examples/custom-auth-http-middleware/Cargo.toml index 29b9d467885..88615ec3d70 100644 --- a/examples/custom-auth-http-middleware/Cargo.toml +++ b/examples/custom-auth-http-middleware/Cargo.toml @@ -14,3 +14,5 @@ jsonrpsee = { workspace = true, features = ["server"] } serde_json.workspace = true tower.workspace = true tracing.workspace = true + +[lints] diff --git a/examples/custom-beacon-withdrawals/Cargo.toml b/examples/custom-beacon-withdrawals/Cargo.toml index bc484fa4542..9421eaf97ee 100644 --- a/examples/custom-beacon-withdrawals/Cargo.toml +++ b/examples/custom-beacon-withdrawals/Cargo.toml @@ -13,3 +13,5 @@ alloy-sol-types.workspace = true alloy-eips.workspace = true eyre.workspace = true + +[lints] diff --git a/examples/custom-dev-node/Cargo.toml b/examples/custom-dev-node/Cargo.toml index ad0ba9aba9c..555e72ffaee 100644 --- a/examples/custom-dev-node/Cargo.toml +++ b/examples/custom-dev-node/Cargo.toml @@ -15,3 +15,5 @@ serde_json.workspace = true alloy-genesis.workspace = true alloy-primitives.workspace = true + +[lints] diff --git a/examples/custom-engine-types/Cargo.toml b/examples/custom-engine-types/Cargo.toml index 46f8f92532a..b06e1eb712b 100644 --- a/examples/custom-engine-types/Cargo.toml +++ b/examples/custom-engine-types/Cargo.toml @@ -19,3 +19,5 @@ eyre.workspace = true tokio.workspace = true thiserror.workspace = true serde.workspace = true + +[lints] diff --git a/examples/custom-evm/Cargo.toml b/examples/custom-evm/Cargo.toml index 84ba29cf5c6..10a34a1fc24 100644 --- a/examples/custom-evm/Cargo.toml +++ b/examples/custom-evm/Cargo.toml @@ -14,3 +14,5 @@ alloy-primitives.workspace = true eyre.workspace = true tokio.workspace = true + +[lints] diff --git a/examples/custom-hardforks/Cargo.toml b/examples/custom-hardforks/Cargo.toml index 78060f6af62..211d2f0ccec 100644 --- a/examples/custom-hardforks/Cargo.toml +++ b/examples/custom-hardforks/Cargo.toml @@ -14,3 +14,5 @@ alloy-consensus.workspace = true alloy-primitives.workspace = true alloy-eips.workspace = true serde = { version = "1.0", features = ["derive"] } + +[lints] diff --git a/examples/custom-inspector/Cargo.toml b/examples/custom-inspector/Cargo.toml index 22bf3c7a246..6b8a468b2d3 100644 --- a/examples/custom-inspector/Cargo.toml +++ b/examples/custom-inspector/Cargo.toml @@ -13,3 +13,5 @@ futures-util.workspace = true alloy-evm.workspace = true alloy-primitives.workspace = true alloy-eips.workspace = true + +[lints] diff --git a/examples/custom-node-components/Cargo.toml b/examples/custom-node-components/Cargo.toml index dd467c09ae2..1b46042b308 100644 --- a/examples/custom-node-components/Cargo.toml +++ b/examples/custom-node-components/Cargo.toml @@ -10,3 +10,5 @@ reth-ethereum = { workspace = true, features = ["node", "pool", "node-api", "cli reth-tracing.workspace = true eyre.workspace = true + +[lints] diff --git a/examples/custom-payload-builder/Cargo.toml b/examples/custom-payload-builder/Cargo.toml index f7a24eb6852..c03a6be5f44 100644 --- a/examples/custom-payload-builder/Cargo.toml +++ b/examples/custom-payload-builder/Cargo.toml @@ -16,3 +16,5 @@ alloy-eips.workspace = true tracing.workspace = true futures-util.workspace = true eyre.workspace = true + +[lints] diff --git a/examples/custom-rlpx-subprotocol/Cargo.toml b/examples/custom-rlpx-subprotocol/Cargo.toml index 06e8b77950c..2f3a700005a 100644 --- a/examples/custom-rlpx-subprotocol/Cargo.toml +++ b/examples/custom-rlpx-subprotocol/Cargo.toml @@ -13,3 +13,5 @@ tokio-stream.workspace = true eyre.workspace = true tracing.workspace = true alloy-primitives.workspace = true + +[lints] diff --git a/examples/custom-rpc-middleware/Cargo.toml b/examples/custom-rpc-middleware/Cargo.toml index 92b5975e43e..605b7170c9a 100644 --- a/examples/custom-rpc-middleware/Cargo.toml +++ b/examples/custom-rpc-middleware/Cargo.toml @@ -12,3 +12,5 @@ clap = { workspace = true, features = ["derive"] } jsonrpsee = { workspace = true, features = ["server", "macros"] } tracing.workspace = true tower.workspace = true + +[lints] diff --git a/examples/custom-state-root/Cargo.toml b/examples/custom-state-root/Cargo.toml index b7601d10b38..c47960bc0f9 100644 --- a/examples/custom-state-root/Cargo.toml +++ b/examples/custom-state-root/Cargo.toml @@ -22,3 +22,5 @@ alloy-genesis.workspace = true alloy-primitives.workspace = true alloy-network.workspace = true alloy-rpc-types-engine.workspace = true + +[lints] diff --git a/examples/db-access/Cargo.toml b/examples/db-access/Cargo.toml index 07687e5c465..09d616f389e 100644 --- a/examples/db-access/Cargo.toml +++ b/examples/db-access/Cargo.toml @@ -10,3 +10,5 @@ reth-ethereum = { workspace = true, features = ["node"] } alloy-primitives.workspace = true eyre.workspace = true + +[lints] diff --git a/examples/exex-subscription/Cargo.toml b/examples/exex-subscription/Cargo.toml index 9593409b215..e0914e24984 100644 --- a/examples/exex-subscription/Cargo.toml +++ b/examples/exex-subscription/Cargo.toml @@ -22,3 +22,5 @@ tokio.workspace = true serde.workspace = true serde_json.workspace = true tracing.workspace = true + +[lints] diff --git a/examples/exex-test/Cargo.toml b/examples/exex-test/Cargo.toml index 27e8910484d..44d52db4564 100644 --- a/examples/exex-test/Cargo.toml +++ b/examples/exex-test/Cargo.toml @@ -17,3 +17,5 @@ tokio = { workspace = true, features = ["full"] } # other dependencies eyre.workspace = true serde_json.workspace = true + +[lints] diff --git a/examples/manual-p2p/Cargo.toml b/examples/manual-p2p/Cargo.toml index 1eb8808f990..ada3ecbc52c 100644 --- a/examples/manual-p2p/Cargo.toml +++ b/examples/manual-p2p/Cargo.toml @@ -19,3 +19,5 @@ futures.workspace = true tokio.workspace = true eyre.workspace = true + +[lints] diff --git a/examples/network-proxy/Cargo.toml b/examples/network-proxy/Cargo.toml index c4f11e93ba2..082b7db2d6d 100644 --- a/examples/network-proxy/Cargo.toml +++ b/examples/network-proxy/Cargo.toml @@ -12,3 +12,5 @@ reth-tracing.workspace = true futures.workspace = true tokio.workspace = true eyre.workspace = true + +[lints] diff --git a/examples/network-txpool/Cargo.toml b/examples/network-txpool/Cargo.toml index 187a7ab9a3a..865f347a962 100644 --- a/examples/network-txpool/Cargo.toml +++ b/examples/network-txpool/Cargo.toml @@ -9,3 +9,5 @@ license.workspace = true reth-ethereum = { workspace = true, features = ["test-utils", "network", "pool", "provider"] } eyre.workspace = true tokio.workspace = true + +[lints] diff --git a/examples/network/Cargo.toml b/examples/network/Cargo.toml index 0786bda5926..4b17701c935 100644 --- a/examples/network/Cargo.toml +++ b/examples/network/Cargo.toml @@ -10,3 +10,5 @@ reth-ethereum = { workspace = true, features = ["test-utils", "network", "provid futures.workspace = true tokio.workspace = true eyre.workspace = true + +[lints] diff --git a/examples/node-builder-api/Cargo.toml b/examples/node-builder-api/Cargo.toml index 287456ec04e..5750e40c669 100644 --- a/examples/node-builder-api/Cargo.toml +++ b/examples/node-builder-api/Cargo.toml @@ -7,3 +7,5 @@ license.workspace = true [dependencies] reth-ethereum = { workspace = true, features = ["node", "pool", "node-api", "cli", "test-utils"] } + +[lints] diff --git a/examples/node-custom-rpc/Cargo.toml b/examples/node-custom-rpc/Cargo.toml index 5b8aa98dbcf..c774c7ad06c 100644 --- a/examples/node-custom-rpc/Cargo.toml +++ b/examples/node-custom-rpc/Cargo.toml @@ -14,3 +14,4 @@ tokio.workspace = true serde_json.workspace = true [dev-dependencies] +[lints] diff --git a/examples/node-event-hooks/Cargo.toml b/examples/node-event-hooks/Cargo.toml index ecb0dd1429e..13ea0893205 100644 --- a/examples/node-event-hooks/Cargo.toml +++ b/examples/node-event-hooks/Cargo.toml @@ -7,3 +7,5 @@ license.workspace = true [dependencies] reth-ethereum = { workspace = true, features = ["node", "cli"] } + +[lints] diff --git a/examples/polygon-p2p/Cargo.toml b/examples/polygon-p2p/Cargo.toml index e452d59cc42..631b0ed39a6 100644 --- a/examples/polygon-p2p/Cargo.toml +++ b/examples/polygon-p2p/Cargo.toml @@ -14,3 +14,5 @@ reth-tracing.workspace = true tokio-stream.workspace = true reth-discv4 = { workspace = true, features = ["test-utils"] } alloy-genesis.workspace = true + +[lints] diff --git a/examples/rpc-db/Cargo.toml b/examples/rpc-db/Cargo.toml index 3928a30f892..8f474b654a6 100644 --- a/examples/rpc-db/Cargo.toml +++ b/examples/rpc-db/Cargo.toml @@ -11,3 +11,5 @@ jsonrpsee.workspace = true reth-ethereum = { workspace = true, features = ["test-utils", "node", "provider", "pool", "network", "rpc"] } tokio = { workspace = true, features = ["full"] } eyre.workspace = true + +[lints] diff --git a/examples/txpool-tracing/Cargo.toml b/examples/txpool-tracing/Cargo.toml index df72dd193f9..0579d2da388 100644 --- a/examples/txpool-tracing/Cargo.toml +++ b/examples/txpool-tracing/Cargo.toml @@ -15,3 +15,5 @@ alloy-network.workspace = true clap = { workspace = true, features = ["derive"] } futures-util.workspace = true eyre.workspace = true + +[lints] From 844ff66d080cf7e786b34af02d2a3db1f217fa2d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A9a=20Narzis?= <78718413+lean-apple@users.noreply.github.com> Date: Wed, 16 Sep 2026 08:20:22 +0000 Subject: [PATCH 274/294] feat(snap-sync): download and persist missing contract code (#27258) --- crates/net/downloaders/src/snap/mod.rs | 36 ++- crates/snap-sync/src/bytecode/download.rs | 326 ++++++++++++++++++++++ crates/snap-sync/src/bytecode/mod.rs | 7 + crates/snap-sync/src/bytecode/store.rs | 239 ++++++++++++++++ crates/snap-sync/src/common/download.rs | 12 + crates/snap-sync/src/lib.rs | 2 + crates/snap-sync/src/test_utils.rs | 27 +- 7 files changed, 642 insertions(+), 7 deletions(-) create mode 100644 crates/snap-sync/src/bytecode/download.rs create mode 100644 crates/snap-sync/src/bytecode/mod.rs create mode 100644 crates/snap-sync/src/bytecode/store.rs diff --git a/crates/net/downloaders/src/snap/mod.rs b/crates/net/downloaders/src/snap/mod.rs index 9c490184e09..78bc3a04683 100644 --- a/crates/net/downloaders/src/snap/mod.rs +++ b/crates/net/downloaders/src/snap/mod.rs @@ -3,7 +3,7 @@ //! //! Persistence and range selection are handled by the snap sync orchestrator. -use alloy_primitives::B256; +use alloy_primitives::{map::B256Set, B256, KECCAK256_EMPTY}; use futures::Future; use reth_eth_wire_types::snap::{ AccountRangeMessage, GetAccountRangeMessage, GetStorageRangesMessage, @@ -136,6 +136,18 @@ impl VerifiedAccountRange { } } + /// Code hashes the accounts reference, once each and in the order they appear. + /// + /// Accounts sharing code therefore yield one hash, so it is downloaded and stored once. + pub fn code_hashes(&self) -> Vec { + let mut seen = B256Set::default(); + self.accounts + .iter() + .map(|(_, account)| account.code_hash) + .filter(|hash| *hash != KECCAK256_EMPTY && seen.insert(*hash)) + .collect() + } + /// Borrows only the accounts that have storage, together with the root that authenticated /// them. /// @@ -496,6 +508,28 @@ mod tests { assert_eq!(batch.range(2..4), None); } + #[test] + fn code_hashes_are_listed_once_in_the_order_the_accounts_reference_them() { + let shared = B256::repeat_byte(0x11); + let mut first = account(1); + first.code_hash = shared; + let mut third = account(3); + third.code_hash = shared; + let mut fourth = account(4); + fourth.code_hash = B256::repeat_byte(0x44); + let accounts = + vec![(key(1), first), (key(2), account(2)), (key(3), third), (key(4), fourth)]; + let range = VerifiedAccountRange { + state_root: root(&accounts), + origin: B256::ZERO, + accounts, + has_more: false, + next: None, + }; + + assert_eq!(range.code_hashes(), vec![shared, B256::repeat_byte(0x44)]); + } + #[test] fn storage_batch_omits_interleaved_accounts_without_storage() { let mut first = account(1); diff --git a/crates/snap-sync/src/bytecode/download.rs b/crates/snap-sync/src/bytecode/download.rs new file mode 100644 index 00000000000..07d94efd9c0 --- /dev/null +++ b/crates/snap-sync/src/bytecode/download.rs @@ -0,0 +1,326 @@ +//! Downloads the code an account range's contracts reference, committing every verified response. + +use crate::{common::DownloadContext, SnapBytecodeStore, SnapSyncError, VerifiedRange}; +use reth_db_api::transaction::DbTxMut; +use reth_downloaders::snap::{BytecodeDownloader, BytecodeOutcome}; +use reth_eth_wire_types::snap::GetByteCodesMessage; +use reth_network_p2p::snap::client::SnapClient; +use reth_network_peers::PeerId; +use reth_storage_api::{ + DBProvider, DatabaseProviderFactory, MetadataProvider, MetadataWriter, StateWriter, +}; +use reth_tasks::Runtime; +use std::fmt; + +/// Default number of code hashes asked for per request. +pub const DEFAULT_CODE_HASHES: usize = 128; + +/// Downloads the code an account range still needs, one request at a time. +/// +/// Code is content addressed, so a hash is requested only while no blob is stored for it, however +/// many accounts reference it and whichever attempt fetched it. +pub struct BytecodeDownload { + context: DownloadContext, + // Code hashes asked for per request. + max_hashes: usize, +} + +impl BytecodeDownload { + /// Creates a download that requests whatever code the store does not already hold. + pub const fn new(client: C, factory: F, runtime: Runtime) -> Self { + Self { + context: DownloadContext::new(client, factory, runtime), + max_hashes: DEFAULT_CODE_HASHES, + } + } + + /// Returns this download asking peers for at most `response_bytes` per response. + pub const fn with_response_bytes(mut self, response_bytes: u64) -> Self { + self.context.set_response_bytes(response_bytes); + self + } + + /// Returns this download asking for at most `max_hashes` code hashes per request, at least one. + pub const fn with_max_hashes(mut self, max_hashes: usize) -> Self { + self.max_hashes = if max_hashes == 0 { 1 } else { max_hashes }; + self + } +} + +impl BytecodeDownload +where + C: SnapClient + Clone + Unpin, + F: DatabaseProviderFactory + Clone + 'static, + F::Provider: MetadataProvider, + F::ProviderRW: MetadataProvider + MetadataWriter + StateWriter + DBProvider, +{ + /// Requests code `range` still needs and commits the verified response. + /// + /// [`BytecodeStep::Complete`] once every hash it references is stored, so the range can commit + /// without supplying code. Whatever a peer leaves out stays missing for the next call. + pub async fn next(&mut self, range: &VerifiedRange) -> Result { + let write = range.write(); + let referenced = range.range().code_hashes(); + if referenced.is_empty() { + return Ok(BytecodeStep::Complete) + } + let limit = self.max_hashes; + let missing = self + .context + .read(move |provider| provider.missing_code(write, &referenced, limit)) + .await?; + if missing.is_empty() { + return Ok(BytecodeStep::Complete) + } + + let request = GetByteCodesMessage { + request_id: self.context.next_request_id(), + hashes: missing, + response_bytes: self.context.response_bytes(), + }; + let downloader = BytecodeDownloader::new( + self.context.client().clone(), + request, + self.context.runtime().clone(), + ) + .expect("missing code is never empty"); + let codes = match downloader.await? { + BytecodeOutcome::Verified(verified) => verified.into_codes(), + BytecodeOutcome::Unavailable { peer_id } => { + return Ok(BytecodeStep::Unavailable { peer_id }) + } + }; + + let persisted = + self.context.commit(move |provider| provider.commit_bytecodes(write, codes)).await?; + Ok(BytecodeStep::Committed { persisted }) + } +} + +impl fmt::Debug for BytecodeDownload { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("BytecodeDownload") + .field("context", &self.context) + .field("max_hashes", &self.max_hashes) + .finish() + } +} + +/// What one request of a [`BytecodeDownload`] produced. +#[derive(Debug)] +pub enum BytecodeStep { + /// A response was committed, storing this many code blobs. + Committed { + /// Blobs the response supplied, at most the hashes it was asked for. + persisted: usize, + }, + /// The peer held none of the requested code, so all of it stays missing. + Unavailable { + /// Peer that answered without the code, so the retry can go elsewhere. + peer_id: PeerId, + }, + /// Every hash the range's accounts reference is stored. + Complete, +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{ + test_utils::{ + account, byte_codes, generation, hashed_factory, key, state_root, verified_range, + ScriptedSnapClient, + }, + SnapAccountStore, SnapAttemptStore, + }; + use alloy_primitives::{keccak256, Bytes, B256}; + use reth_db_api::{tables, transaction::DbTx}; + use reth_eth_wire_types::snap::ByteCodesMessage; + use reth_network_p2p::{error::PeerRequestResult, snap::client::SnapResponse}; + use reth_network_peers::WithPeerId; + use reth_provider::{test_utils::MockNodeTypesWithDB, ProviderFactory}; + use reth_trie_common::TrieAccount; + use std::sync::Arc; + + type Factory = ProviderFactory; + type Download = BytecodeDownload, Factory>; + + fn code(byte: u8) -> Bytes { + Bytes::from(vec![byte; 4]) + } + + fn contract(nonce: u64, code: &Bytes) -> TrieAccount { + let mut contract = account(nonce); + contract.code_hash = keccak256(code); + contract + } + + // Two contracts sharing one blob, a plain account, and a contract with its own blob. + fn accounts() -> Vec<(B256, TrieAccount)> { + vec![ + (key(1), contract(1, &code(1))), + (key(2), account(2)), + (key(3), contract(3, &code(1))), + (key(4), contract(4, &code(2))), + ] + } + + // An attempt that fetched all of `accounts` as one range, not yet committed. + fn started(accounts: &[(B256, TrieAccount)]) -> (Factory, VerifiedRange) { + let factory = hashed_factory(); + let provider = factory.database_provider_rw().unwrap(); + let write = provider.start_snap_attempt(generation(1, state_root(accounts))).unwrap(); + provider.start_account_coverage(write).unwrap(); + provider.commit().unwrap(); + let range = verified_range(accounts, 0..accounts.len(), B256::ZERO, &[]); + (factory, VerifiedRange::new(write, range)) + } + + fn download( + responses: impl IntoIterator>, + factory: Factory, + ) -> (Arc, Download) { + let client = Arc::new(ScriptedSnapClient::new(responses)); + (Arc::clone(&client), BytecodeDownload::new(client, factory, Runtime::test())) + } + + async fn committed(download: &mut Download, range: &VerifiedRange) -> usize { + match download.next(range).await.unwrap() { + BytecodeStep::Committed { persisted } => persisted, + step => panic!("expected a commit, got {step:?}"), + } + } + + fn stored(factory: &Factory, code: &Bytes) -> bool { + let provider = factory.database_provider_ro().unwrap(); + provider.tx_ref().get::(keccak256(code)).unwrap().is_some() + } + + #[tokio::test] + async fn shared_code_is_requested_once_and_persisted_for_every_account() { + let accounts = accounts(); + let (factory, range) = started(&accounts); + let (client, mut download) = + download([byte_codes(1, &[code(1), code(2)])], factory.clone()); + + assert_eq!(committed(&mut download, &range).await, 2); + + assert!(matches!(download.next(&range).await.unwrap(), BytecodeStep::Complete)); + assert_eq!(*client.code_requests(), [vec![keccak256(code(1)), keccak256(code(2))]]); + assert!(stored(&factory, &code(1)) && stored(&factory, &code(2))); + + // With its code stored, the range commits without supplying any. + let provider = factory.database_provider_rw().unwrap(); + let coverage = provider + .commit_account_range(range.write(), range.range(), Default::default(), Vec::new()) + .unwrap(); + provider.commit().unwrap(); + assert!(coverage.is_complete()); + } + + #[tokio::test] + async fn code_already_stored_is_never_requested() { + let accounts = accounts(); + let (factory, range) = started(&accounts); + let (_, mut first) = download([byte_codes(1, &[code(1), code(2)])], factory.clone()); + committed(&mut first, &range).await; + drop(first); + + let (client, mut resumed) = download([], factory); + + assert!(matches!(resumed.next(&range).await.unwrap(), BytecodeStep::Complete)); + assert!(client.code_requests().is_empty()); + } + + #[tokio::test] + async fn code_a_peer_does_not_have_stays_missing() { + let accounts = accounts(); + let (factory, range) = started(&accounts); + let peer = PeerId::random(); + let empty = ByteCodesMessage { request_id: 1, codes: Vec::new() }; + let responses = [ + Ok(WithPeerId::new(peer, SnapResponse::ByteCodes(empty))), + // A peer that holds only the first blob answers with a subsequence. + byte_codes(2, &[code(1)]), + byte_codes(3, &[code(2)]), + ]; + let (client, mut download) = download(responses, factory.clone()); + + let step = download.next(&range).await.unwrap(); + + assert!(matches!(step, BytecodeStep::Unavailable { peer_id } if peer_id == peer)); + assert!(!stored(&factory, &code(1))); + assert_eq!(committed(&mut download, &range).await, 1); + // The unanswered hash is asked for again, the stored one is not. + assert_eq!(committed(&mut download, &range).await, 1); + assert!(matches!(download.next(&range).await.unwrap(), BytecodeStep::Complete)); + assert_eq!( + *client.code_requests(), + [ + vec![keccak256(code(1)), keccak256(code(2))], + vec![keccak256(code(1)), keccak256(code(2))], + vec![keccak256(code(2))], + ] + ); + } + + #[tokio::test] + async fn unresolved_code_prevents_the_range_from_committing() { + let accounts = accounts(); + let (factory, range) = started(&accounts); + let (_, mut download) = download([byte_codes(1, &[code(1)])], factory.clone()); + committed(&mut download, &range).await; + + let provider = factory.database_provider_rw().unwrap(); + let refused = provider.commit_account_range( + range.write(), + range.range(), + Default::default(), + Vec::new(), + ); + + assert!(matches!( + refused, + Err(SnapSyncError::MissingCode { hash }) if hash == keccak256(code(2)) + )); + } + + #[tokio::test] + async fn requests_ask_for_at_most_the_configured_hashes() { + let accounts = accounts(); + let (factory, range) = started(&accounts); + let responses = [byte_codes(1, &[code(1)]), byte_codes(2, &[code(2)])]; + let (client, download) = download(responses, factory); + let mut download = download.with_max_hashes(1); + + committed(&mut download, &range).await; + committed(&mut download, &range).await; + + assert!(matches!(download.next(&range).await.unwrap(), BytecodeStep::Complete)); + assert_eq!(*client.code_requests(), [vec![keccak256(code(1))], vec![keccak256(code(2))]]); + } + + #[tokio::test] + async fn a_range_without_contracts_requests_no_code() { + let accounts = vec![(key(1), account(1)), (key(2), account(2))]; + let (factory, range) = started(&accounts); + let (client, mut download) = download([], factory); + + assert!(matches!(download.next(&range).await.unwrap(), BytecodeStep::Complete)); + assert!(client.code_requests().is_empty()); + } + + #[tokio::test] + async fn code_fetched_before_the_pivot_moved_is_not_committed() { + let accounts = accounts(); + let (factory, range) = started(&accounts); + let (client, mut download) = + download([byte_codes(1, &[code(1), code(2)])], factory.clone()); + let provider = factory.database_provider_rw().unwrap(); + provider.advance_snap_pivot(range.write(), generation(2, state_root(&accounts))).unwrap(); + provider.commit().unwrap(); + + assert!(matches!(download.next(&range).await, Err(SnapSyncError::StaleWrite { .. }))); + assert!(client.code_requests().is_empty()); + } +} diff --git a/crates/snap-sync/src/bytecode/mod.rs b/crates/snap-sync/src/bytecode/mod.rs new file mode 100644 index 00000000000..d8d02984780 --- /dev/null +++ b/crates/snap-sync/src/bytecode/mod.rs @@ -0,0 +1,7 @@ +//! Contract bytecode download coordination and persistence. + +mod download; +mod store; + +pub use download::{BytecodeDownload, BytecodeStep, DEFAULT_CODE_HASHES}; +pub use store::SnapBytecodeStore; diff --git a/crates/snap-sync/src/bytecode/store.rs b/crates/snap-sync/src/bytecode/store.rs new file mode 100644 index 00000000000..b3eeaa7d215 --- /dev/null +++ b/crates/snap-sync/src/bytecode/store.rs @@ -0,0 +1,239 @@ +//! Persists contract code ahead of the account range that commits to it. +//! +//! Code is content addressed, so a blob is authenticated by its hash alone and stays valid however +//! far the pivot moves. Accounts sharing a code hash therefore need it downloaded and stored once. + +use crate::{SnapAttemptStore, SnapSyncError, SnapWrite}; +use alloy_primitives::{keccak256, Bytes, B256}; +use reth_db_api::{ + tables, + transaction::{DbTx, DbTxMut}, + RawKey, RawTable, +}; +use reth_storage_api::{DBProvider, MetadataProvider, StateWriter}; +use revm::{bytecode::Bytecode, database::states::StateChangeset}; + +/// Persistence for the code an attempt's accounts reference. +/// +/// Blanket-implemented over the node's writers, so code joins the caller's transaction and commits +/// with whatever else that transaction carries. +pub trait SnapBytecodeStore { + /// Returns the first `limit` of `hashes` that are not stored yet, so the scan a request costs + /// is bounded by what it can ask for. + fn missing_code( + &self, + write: SnapWrite, + hashes: &[B256], + limit: usize, + ) -> Result, SnapSyncError> + where + Self: DBProvider; + + /// Persists `codes`, each checked against the hash it was downloaded under, and returns how + /// many were written. + fn commit_bytecodes( + &self, + write: SnapWrite, + codes: Vec<(B256, Bytes)>, + ) -> Result + where + Self: StateWriter + DBProvider; +} + +impl SnapBytecodeStore for T { + fn missing_code( + &self, + write: SnapWrite, + hashes: &[B256], + limit: usize, + ) -> Result, SnapSyncError> + where + Self: DBProvider, + { + self.authorize_snap_write(write)?; + let mut missing = Vec::new(); + for hash in hashes { + if missing.len() == limit { + break + } + // Only presence matters, so stored code is not decoded. + if self.tx_ref().get::>(RawKey::new(*hash))?.is_none() { + missing.push(*hash); + } + } + Ok(missing) + } + + // Every check runs before the first write, so a refused response changes nothing. + fn commit_bytecodes( + &self, + write: SnapWrite, + codes: Vec<(B256, Bytes)>, + ) -> Result + where + Self: StateWriter + DBProvider, + { + self.authorize_snap_write(write)?; + let contracts = codes + .into_iter() + .map(|(hash, code)| { + let got = keccak256(&code); + if got != hash { + return Err(SnapSyncError::CodeMismatch { expected: hash, got }) + } + // Code deployed before EIP-3541 can carry the delegation prefix without being + // one, so authenticated bytes that do not parse as a delegation are legacy code. + let code = Bytecode::new_raw_checked(code.clone()) + .unwrap_or_else(|_| Bytecode::new_legacy(code)); + Ok((hash, code)) + }) + .collect::, SnapSyncError>>()?; + + let written = contracts.len(); + self.write_state_changes(StateChangeset { contracts, ..Default::default() })?; + Ok(written) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::test_utils::{account, generation, hashed_factory, key, state_root, verified_range}; + use alloy_primitives::bytes; + use reth_provider::{ + test_utils::MockNodeTypesWithDB, DatabaseProviderFactory, ProviderFactory, + }; + use reth_trie_common::TrieAccount; + + type Factory = ProviderFactory; + + fn code(byte: u8) -> Bytes { + Bytes::from(vec![byte; 4]) + } + + // An account referencing `code`, distinguished by its nonce. + fn contract(nonce: u64, code: &Bytes) -> TrieAccount { + let mut contract = account(nonce); + contract.code_hash = keccak256(code); + contract + } + + fn started(accounts: &[(B256, TrieAccount)]) -> (Factory, SnapWrite) { + let factory = hashed_factory(); + let provider = factory.database_provider_rw().unwrap(); + let write = provider.start_snap_attempt(generation(1, state_root(accounts))).unwrap(); + provider.commit().unwrap(); + (factory, write) + } + + fn is_stored(provider: &impl DBProvider, code: &Bytes) -> bool { + provider.tx_ref().get::(keccak256(code)).unwrap().is_some() + } + + #[test] + fn one_blob_answers_every_account_sharing_its_hash() { + let shared = code(1); + let accounts = vec![ + (key(1), contract(1, &shared)), + (key(2), account(2)), + (key(3), contract(3, &shared)), + (key(4), contract(4, &code(2))), + ]; + let (factory, write) = started(&accounts); + let provider = factory.database_provider_rw().unwrap(); + let referenced = + verified_range(&accounts, 0..accounts.len(), B256::ZERO, &[]).code_hashes(); + + let missing = provider.missing_code(write, &referenced, usize::MAX).unwrap(); + + assert_eq!(missing, [keccak256(&shared), keccak256(code(2))]); + assert_eq!( + provider.commit_bytecodes(write, vec![(missing[0], shared.clone())]).unwrap(), + 1 + ); + // The shared blob answers both accounts, leaving only the other contract's code. + assert_eq!( + provider.missing_code(write, &referenced, usize::MAX).unwrap(), + [keccak256(code(2))] + ); + assert!(is_stored(&provider, &shared)); + } + + #[test] + fn a_scan_stops_once_it_has_the_hashes_a_request_can_carry() { + let hashes: Vec<_> = (1..=4).map(|byte| keccak256(code(byte))).collect(); + let (factory, write) = started(&[(key(1), contract(1, &code(1)))]); + let provider = factory.database_provider_ro().unwrap(); + + assert_eq!(provider.missing_code(write, &hashes, 2).unwrap(), hashes[..2]); + } + + #[test] + fn code_already_stored_is_not_requested_again() { + let stored = code(1); + let accounts = vec![(key(1), contract(1, &stored))]; + let (factory, write) = started(&accounts); + let provider = factory.database_provider_rw().unwrap(); + provider.commit_bytecodes(write, vec![(keccak256(&stored), stored.clone())]).unwrap(); + provider.commit().unwrap(); + + let provider = factory.database_provider_ro().unwrap(); + assert!(provider + .missing_code(write, &[keccak256(&stored)], usize::MAX) + .unwrap() + .is_empty()); + assert!(is_stored(&provider, &stored)); + } + + #[test] + fn code_that_does_not_hash_to_its_requested_hash_is_refused() { + let wanted = code(1); + let accounts = vec![(key(1), contract(1, &wanted))]; + let (factory, write) = started(&accounts); + let provider = factory.database_provider_rw().unwrap(); + let hash = keccak256(&wanted); + + let refused = provider.commit_bytecodes(write, vec![(hash, code(2))]); + + assert!( + matches!(refused, Err(SnapSyncError::CodeMismatch { expected, .. }) if expected == hash) + ); + assert_eq!(provider.missing_code(write, &[hash], usize::MAX).unwrap(), [hash]); + assert!(!is_stored(&provider, &wanted)); + } + + // A delegation is exactly 23 bytes, but EIP-3541 only bars the prefix from London onwards, so + // an older account can reference shorter code that starts with it. + #[test] + fn delegation_shaped_code_that_predates_eip_3541_is_kept_as_legacy_code() { + let historical = bytes!("ef0100"); + let accounts = vec![(key(1), contract(1, &historical))]; + let (factory, write) = started(&accounts); + let provider = factory.database_provider_rw().unwrap(); + + let hash = keccak256(&historical); + assert_eq!(provider.commit_bytecodes(write, vec![(hash, historical.clone())]).unwrap(), 1); + + let stored = provider.tx_ref().get::(hash).unwrap().unwrap(); + assert_eq!(stored.original_bytes(), historical); + assert!(provider.missing_code(write, &[hash], usize::MAX).unwrap().is_empty()); + } + + #[test] + fn code_is_refused_once_the_attempt_no_longer_owns_the_state() { + let wanted = code(1); + let accounts = vec![(key(1), contract(1, &wanted))]; + let (factory, write) = started(&accounts); + let provider = factory.database_provider_rw().unwrap(); + provider.advance_snap_pivot(write, generation(2, B256::repeat_byte(0xcc))).unwrap(); + + let refused = provider.commit_bytecodes(write, vec![(keccak256(&wanted), wanted.clone())]); + + assert!(matches!(refused, Err(SnapSyncError::StaleWrite { .. }))); + assert!(matches!( + provider.missing_code(write, &[keccak256(&wanted)], usize::MAX), + Err(SnapSyncError::StaleWrite { .. }) + )); + assert!(!is_stored(&provider, &wanted)); + } +} diff --git a/crates/snap-sync/src/common/download.rs b/crates/snap-sync/src/common/download.rs index 85c1d342bd8..973b64ac075 100644 --- a/crates/snap-sync/src/common/download.rs +++ b/crates/snap-sync/src/common/download.rs @@ -64,6 +64,18 @@ where F: DatabaseProviderFactory + Clone + 'static, F::ProviderRW: DBProvider, { + /// Runs `read` on the blocking pool, where scans touching many keys belong. + pub(crate) async fn read( + &self, + read: impl FnOnce(&F::Provider) -> Result + Send + 'static, + ) -> Result { + let factory = self.factory.clone(); + self.runtime + .spawn_blocking(move || read(&factory.database_provider_ro()?)) + .await + .map_err(|error| SnapSyncError::Provider(ProviderError::other(error)))? + } + /// Runs `write` in one transaction on the blocking pool, committing only if it succeeds. pub(crate) async fn commit( &self, diff --git a/crates/snap-sync/src/lib.rs b/crates/snap-sync/src/lib.rs index c9157628063..b28cda7b815 100644 --- a/crates/snap-sync/src/lib.rs +++ b/crates/snap-sync/src/lib.rs @@ -39,6 +39,7 @@ mod account; mod attempt; mod bal; +mod bytecode; mod common; mod error; mod generation; @@ -54,6 +55,7 @@ pub use account::{ }; pub use attempt::{SnapAttemptStore, SnapWrite}; pub use bal::{BalStateUpdate, DownloadedAccount}; +pub use bytecode::{BytecodeDownload, BytecodeStep, SnapBytecodeStore, DEFAULT_CODE_HASHES}; pub use common::{DEFAULT_RESPONSE_BYTES, MAX_HASH}; pub use error::SnapSyncError; pub use generation::{SnapGeneration, SnapPhase}; diff --git a/crates/snap-sync/src/test_utils.rs b/crates/snap-sync/src/test_utils.rs index 66d0c6ac48f..c75c3b42f49 100644 --- a/crates/snap-sync/src/test_utils.rs +++ b/crates/snap-sync/src/test_utils.rs @@ -11,8 +11,9 @@ use futures::future::{ready, Ready}; use reth_db_api::{cursor::DbDupCursorRO, tables, transaction::DbTx}; use reth_downloaders::snap::{AccountRangeDownloader, AccountRangeOutcome, VerifiedAccountRange}; use reth_eth_wire_types::snap::{ - AccountData, AccountRangeMessage, GetAccountRangeMessage, GetBlockAccessListsMessage, - GetByteCodesMessage, GetStorageRangesMessage, StorageData, StorageRangesMessage, + AccountData, AccountRangeMessage, ByteCodesMessage, GetAccountRangeMessage, + GetBlockAccessListsMessage, GetByteCodesMessage, GetStorageRangesMessage, StorageData, + StorageRangesMessage, }; use reth_network_p2p::{ download::DownloadClient, @@ -210,6 +211,12 @@ pub(crate) fn storage_ranges( Ok(WithPeerId::new(PeerId::random(), SnapResponse::StorageRanges(message))) } +/// A peer's answer serving `codes`, in the order they were requested. +pub(crate) fn byte_codes(request_id: u64, codes: &[Bytes]) -> PeerRequestResult { + let message = ByteCodesMessage { request_id, codes: codes.to_vec() }; + Ok(WithPeerId::new(PeerId::random(), SnapResponse::ByteCodes(message))) +} + /// Slots persisted for `account`, in key order. pub(crate) fn stored_slots(provider: &impl DBProvider, account: B256) -> Vec<(B256, U256)> { let mut cursor = provider.tx_ref().cursor_dup_read::().unwrap(); @@ -228,6 +235,7 @@ pub(crate) struct ScriptedSnapClient { responses: Mutex>>, origins: Mutex>, storage_requests: Mutex, B256)>>, + code_requests: Mutex>>, } impl ScriptedSnapClient { @@ -238,6 +246,7 @@ impl ScriptedSnapClient { responses: Mutex::new(responses.into_iter().collect()), origins: Mutex::new(Vec::new()), storage_requests: Mutex::new(Vec::new()), + code_requests: Mutex::new(Vec::new()), } } @@ -251,6 +260,11 @@ impl ScriptedSnapClient { self.storage_requests.lock().unwrap() } + /// Hashes of the bytecode requests sent so far. + pub(crate) fn code_requests(&self) -> MutexGuard<'_, Vec>> { + self.code_requests.lock().unwrap() + } + fn next_response(&self) -> Ready> { let response = self.responses.lock().unwrap().pop_front(); ready(response.unwrap_or(Err(RequestError::ChannelClosed))) @@ -297,16 +311,17 @@ impl SnapClient for ScriptedSnapClient { self.next_response() } - fn get_byte_codes(&self, _request: GetByteCodesMessage) -> Self::Output { - unsupported() + fn get_byte_codes(&self, request: GetByteCodesMessage) -> Self::Output { + self.get_byte_codes_with_priority(request, Priority::Normal) } fn get_byte_codes_with_priority( &self, - _request: GetByteCodesMessage, + request: GetByteCodesMessage, _priority: Priority, ) -> Self::Output { - unsupported() + self.code_requests.lock().unwrap().push(request.hashes); + self.next_response() } fn get_block_access_lists_with_priority( From f18dde3a699f6f9267b20c3d4b4751a12ffe7290 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Wed, 16 Sep 2026 08:50:42 +0000 Subject: [PATCH 275/294] refactor(rpc): expose shared eth api settings (#27251) Co-authored-by: Matthias Seitz <19890894+mattsse@users.noreply.github.com> --- crates/rpc/rpc-eth-api/src/helpers/call.rs | 16 +- .../rpc-eth-api/src/helpers/pending_block.rs | 4 +- crates/rpc/rpc-eth-api/src/helpers/state.rs | 4 +- .../rpc-eth-api/src/helpers/transaction.rs | 4 +- crates/rpc/rpc-eth-api/src/types.rs | 4 + crates/rpc/rpc-eth-types/src/lib.rs | 2 + crates/rpc/rpc-eth-types/src/settings.rs | 47 ++++++ crates/rpc/rpc/src/eth/core.rs | 143 +++++++++++++----- crates/rpc/rpc/src/eth/helpers/call.rs | 19 --- .../rpc/rpc/src/eth/helpers/pending_block.rs | 7 +- crates/rpc/rpc/src/eth/helpers/state.rs | 3 - crates/rpc/rpc/src/eth/helpers/transaction.rs | 9 +- 12 files changed, 182 insertions(+), 80 deletions(-) create mode 100644 crates/rpc/rpc-eth-types/src/settings.rs diff --git a/crates/rpc/rpc-eth-api/src/helpers/call.rs b/crates/rpc/rpc-eth-api/src/helpers/call.rs index 2cc96dfbdcf..48249a286fa 100644 --- a/crates/rpc/rpc-eth-api/src/helpers/call.rs +++ b/crates/rpc/rpc-eth-api/src/helpers/call.rs @@ -533,16 +533,24 @@ pub trait Call: /// Returns default gas limit to use for `eth_call` and tracing RPC methods. /// /// Data access in default trait method implementations. - fn call_gas_limit(&self) -> u64; + fn call_gas_limit(&self) -> u64 { + self.eth_api_settings().gas_cap + } /// Returns the maximum number of blocks accepted for `eth_simulateV1`. - fn max_simulate_blocks(&self) -> u64; + fn max_simulate_blocks(&self) -> u64 { + self.eth_api_settings().max_simulate_blocks + } /// Returns whether `eth_simulateV1` should compute state roots. - fn compute_state_root_for_eth_simulate(&self) -> bool; + fn compute_state_root_for_eth_simulate(&self) -> bool { + self.eth_api_settings().compute_state_root_for_eth_simulate + } /// Returns the maximum memory the EVM can allocate per RPC request. - fn evm_memory_limit(&self) -> u64; + fn evm_memory_limit(&self) -> u64 { + self.eth_api_settings().evm_memory_limit + } /// Returns the max gas limit that the caller can afford given a transaction environment. fn caller_gas_allowance( diff --git a/crates/rpc/rpc-eth-api/src/helpers/pending_block.rs b/crates/rpc/rpc-eth-api/src/helpers/pending_block.rs index 60fc361a7cc..4642172c646 100644 --- a/crates/rpc/rpc-eth-api/src/helpers/pending_block.rs +++ b/crates/rpc/rpc-eth-api/src/helpers/pending_block.rs @@ -58,7 +58,9 @@ pub trait LoadPendingBlock: fn pending_env_builder(&self) -> &dyn PendingEnvBuilder; /// Returns the pending block kind - fn pending_block_kind(&self) -> PendingBlockKind; + fn pending_block_kind(&self) -> PendingBlockKind { + self.eth_api_settings().pending_block_kind + } /// Configures the [`PendingBlockEnv`] for the pending block /// diff --git a/crates/rpc/rpc-eth-api/src/helpers/state.rs b/crates/rpc/rpc-eth-api/src/helpers/state.rs index 3fcfd25353d..5679706c53b 100644 --- a/crates/rpc/rpc-eth-api/src/helpers/state.rs +++ b/crates/rpc/rpc-eth-api/src/helpers/state.rs @@ -28,7 +28,9 @@ use std::{collections::HashMap, sync::Arc}; /// Helper methods for `eth_` methods relating to state (accounts). pub trait EthState: LoadState + SpawnBlocking { /// Returns the maximum number of blocks into the past for generating state proofs. - fn max_proof_window(&self) -> u64; + fn max_proof_window(&self) -> u64 { + self.eth_api_settings().eth_proof_window + } /// Validates that the given block is within the configured proof window. /// diff --git a/crates/rpc/rpc-eth-api/src/helpers/transaction.rs b/crates/rpc/rpc-eth-api/src/helpers/transaction.rs index 674cc825fd5..e56fd3e66b9 100644 --- a/crates/rpc/rpc-eth-api/src/helpers/transaction.rs +++ b/crates/rpc/rpc-eth-api/src/helpers/transaction.rs @@ -74,7 +74,9 @@ pub trait EthTransactions: LoadTransaction { } /// Returns the timeout duration for `send_raw_transaction_sync` RPC method. - fn send_raw_transaction_sync_timeout(&self) -> Duration; + fn send_raw_transaction_sync_timeout(&self) -> Duration { + self.eth_api_settings().send_raw_transaction_sync_timeout + } /// Decodes and recovers the transaction and submits it to the pool. /// diff --git a/crates/rpc/rpc-eth-api/src/types.rs b/crates/rpc/rpc-eth-api/src/types.rs index b4b23e01680..643fead6248 100644 --- a/crates/rpc/rpc-eth-api/src/types.rs +++ b/crates/rpc/rpc-eth-api/src/types.rs @@ -4,6 +4,7 @@ use crate::{AsEthApiError, FromEthApiError, RpcNodeCore}; use alloy_rpc_types_eth::Block; use reth_rpc_convert::{RpcConvert, SignableTxRequest}; pub use reth_rpc_convert::{RpcTransaction, RpcTxReq, RpcTypes}; +use reth_rpc_eth_types::EthApiSettings; use reth_storage_api::ProviderTx; use std::error::Error; @@ -29,6 +30,9 @@ pub trait EthApiTypes: Send + Sync + Clone { /// Conversion methods for transaction RPC type. type RpcConvert: RpcConvert; + /// Returns the settings shared by the `eth` RPC helpers. + fn eth_api_settings(&self) -> &EthApiSettings; + /// Returns reference to transaction response builder. fn converter(&self) -> &Self::RpcConvert; } diff --git a/crates/rpc/rpc-eth-types/src/lib.rs b/crates/rpc/rpc-eth-types/src/lib.rs index fec783da17a..beacba6c3c1 100644 --- a/crates/rpc/rpc-eth-types/src/lib.rs +++ b/crates/rpc/rpc-eth-types/src/lib.rs @@ -22,6 +22,7 @@ pub mod id_provider; pub mod logs_utils; pub mod pending_block; pub mod receipt; +pub mod settings; pub mod simulate; pub mod transaction; pub mod tx_forward; @@ -42,5 +43,6 @@ pub use gas_oracle::{ }; pub use id_provider::EthSubscriptionIdProvider; pub use pending_block::{PendingBlock, PendingBlockEnv, PendingBlockEnvOrigin}; +pub use settings::EthApiSettings; pub use transaction::TransactionSource; pub use tx_forward::ForwardConfig; diff --git a/crates/rpc/rpc-eth-types/src/settings.rs b/crates/rpc/rpc-eth-types/src/settings.rs new file mode 100644 index 00000000000..0a3b42bf108 --- /dev/null +++ b/crates/rpc/rpc-eth-types/src/settings.rs @@ -0,0 +1,47 @@ +//! Runtime settings shared by `eth` RPC helpers. + +use crate::{builder::config::PendingBlockKind, RPC_DEFAULT_GAS_CAP}; +use reth_rpc_server_types::constants::{ + DEFAULT_ETH_PROOF_WINDOW, DEFAULT_MAX_SIMULATE_BLOCKS, + RPC_DEFAULT_SEND_RAW_TX_SYNC_TIMEOUT_SECS, +}; +use std::time::Duration; + +/// Settings used when serving `eth` RPC requests. +/// +/// These settings are shared by the API's helper traits so additional settings can be exposed +/// without adding individual trait methods. +#[derive(Debug, Clone, Eq, PartialEq)] +pub struct EthApiSettings { + /// Maximum gas limit for `eth_call` and call tracing RPC methods. + pub gas_cap: u64, + /// Maximum number of blocks for `eth_simulateV1`. + pub max_simulate_blocks: u64, + /// Whether to compute state roots for `eth_simulateV1`. + pub compute_state_root_for_eth_simulate: bool, + /// Maximum number of blocks into the past for generating state proofs. + pub eth_proof_window: u64, + /// Configuration for pending block construction. + pub pending_block_kind: PendingBlockKind, + /// Timeout duration for `send_raw_transaction_sync` RPC method. + pub send_raw_transaction_sync_timeout: Duration, + /// Maximum memory the EVM can allocate per RPC request. + pub evm_memory_limit: u64, + /// Whether to force upcasting EIP-4844 blob sidecars to EIP-7594 format when Osaka is active. + pub force_blob_sidecar_upcasting: bool, +} + +impl Default for EthApiSettings { + fn default() -> Self { + Self { + gas_cap: RPC_DEFAULT_GAS_CAP.into(), + max_simulate_blocks: DEFAULT_MAX_SIMULATE_BLOCKS, + compute_state_root_for_eth_simulate: false, + eth_proof_window: DEFAULT_ETH_PROOF_WINDOW, + pending_block_kind: PendingBlockKind::default(), + send_raw_transaction_sync_timeout: RPC_DEFAULT_SEND_RAW_TX_SYNC_TIMEOUT_SECS, + evm_memory_limit: (1 << 32) - 1, + force_blob_sidecar_upcasting: false, + } + } +} diff --git a/crates/rpc/rpc/src/eth/core.rs b/crates/rpc/rpc/src/eth/core.rs index 7b28e2787aa..a3058de6559 100644 --- a/crates/rpc/rpc/src/eth/core.rs +++ b/crates/rpc/rpc/src/eth/core.rs @@ -21,8 +21,8 @@ use reth_rpc_eth_api::{ EthApiTypes, RpcNodeCore, }; use reth_rpc_eth_types::{ - builder::config::PendingBlockKind, receipt::EthReceiptConverter, EthApiError, EthStateCache, - FeeHistoryCache, GasCap, GasPriceOracle, PendingBlock, + builder::config::PendingBlockKind, receipt::EthReceiptConverter, EthApiError, EthApiSettings, + EthStateCache, FeeHistoryCache, GasCap, GasPriceOracle, PendingBlock, }; use reth_storage_api::{noop::NoopProvider, BlockReaderIdExt, ProviderHeader}; use reth_tasks::{ @@ -141,6 +141,10 @@ where type NetworkTypes = Rpc::Network; type RpcConvert = Rpc; + fn eth_api_settings(&self) -> &EthApiSettings { + self.inner.eth_api_settings() + } + fn converter(&self) -> &Self::RpcConvert { &self.converter } @@ -232,14 +236,8 @@ pub struct EthApiInner { eth_cache: EthStateCache, /// The async gas oracle frontend for gas price suggestions gas_oracle: GasPriceOracle, - /// Maximum gas limit for `eth_call` and call tracing RPC methods. - gas_cap: u64, - /// Maximum number of blocks for `eth_simulateV1`. - max_simulate_blocks: u64, - /// Whether to compute state roots for `eth_simulateV1`. - compute_state_root_for_eth_simulate: bool, - /// The maximum number of blocks into the past for generating state proofs. - eth_proof_window: u64, + /// Settings shared by the `eth` RPC helpers. + settings: EthApiSettings, /// The block number at which the node started starting_block: U256, /// The type that can spawn tasks which would otherwise block. @@ -273,20 +271,8 @@ pub struct EthApiInner { tx_batch_sender: mpsc::UnboundedSender::Transaction>>, - /// Configuration for pending block construction. - pending_block_kind: PendingBlockKind, - - /// Timeout duration for `send_raw_transaction_sync` RPC method. - send_raw_transaction_sync_timeout: Duration, - /// Blob sidecar converter blob_sidecar_converter: BlobSidecarConverter, - - /// Maximum memory the EVM can allocate per RPC request. - evm_memory_limit: u64, - - /// Whether to force upcasting EIP-4844 blob sidecars to EIP-7594 format when Osaka is active. - force_blob_sidecar_upcasting: bool, } impl EthApiInner @@ -342,10 +328,16 @@ where signers, eth_cache, gas_oracle, - gas_cap: gas_cap.into().into(), - max_simulate_blocks, - compute_state_root_for_eth_simulate, - eth_proof_window, + settings: EthApiSettings { + gas_cap: gas_cap.into().into(), + max_simulate_blocks, + compute_state_root_for_eth_simulate, + eth_proof_window, + pending_block_kind, + send_raw_transaction_sync_timeout, + evm_memory_limit, + force_blob_sidecar_upcasting, + }, starting_block, task_spawner, pending_block: Default::default(), @@ -358,11 +350,7 @@ where converter, next_env_builder: Box::new(next_env), tx_batch_sender, - pending_block_kind, - send_raw_transaction_sync_timeout, blob_sidecar_converter: BlobSidecarConverter::new(), - evm_memory_limit, - force_blob_sidecar_upcasting, } } } @@ -429,22 +417,28 @@ where self.components.pool() } + /// Returns the settings shared by the `eth` RPC helpers. + #[inline] + pub const fn eth_api_settings(&self) -> &EthApiSettings { + &self.settings + } + /// Returns the gas cap. #[inline] pub const fn gas_cap(&self) -> u64 { - self.gas_cap + self.settings.gas_cap } /// Returns the `max_simulate_blocks`. #[inline] pub const fn max_simulate_blocks(&self) -> u64 { - self.max_simulate_blocks + self.settings.max_simulate_blocks } /// Returns whether state roots are computed for `eth_simulateV1`. #[inline] pub const fn compute_state_root_for_eth_simulate(&self) -> bool { - self.compute_state_root_for_eth_simulate + self.settings.compute_state_root_for_eth_simulate } /// Returns a handle to the gas oracle. @@ -480,7 +474,7 @@ where /// The maximum number of blocks into the past for generating state proofs. #[inline] pub const fn eth_proof_window(&self) -> u64 { - self.eth_proof_window + self.settings.eth_proof_window } /// Returns reference to [`BlockingTaskGuard`]. @@ -529,7 +523,7 @@ where /// Returns the pending block kind #[inline] pub const fn pending_block_kind(&self) -> PendingBlockKind { - self.pending_block_kind + self.settings.pending_block_kind } /// Returns a handle to the raw transaction forwarder. @@ -541,7 +535,7 @@ where /// Returns the timeout duration for `send_raw_transaction_sync` RPC method. #[inline] pub const fn send_raw_transaction_sync_timeout(&self) -> Duration { - self.send_raw_transaction_sync_timeout + self.settings.send_raw_transaction_sync_timeout } /// Returns a handle to the blob sidecar converter. @@ -553,7 +547,7 @@ where /// Returns the EVM memory limit. #[inline] pub const fn evm_memory_limit(&self) -> u64 { - self.evm_memory_limit + self.settings.evm_memory_limit } /// Returns a reference to the blocking IO request semaphore. @@ -565,7 +559,7 @@ where /// Returns whether to force upcasting EIP-4844 blob sidecars to EIP-7594 format. #[inline] pub const fn force_blob_sidecar_upcasting(&self) -> bool { - self.force_blob_sidecar_upcasting + self.settings.force_blob_sidecar_upcasting } } @@ -626,6 +620,79 @@ mod tests { .build() } + #[tokio::test] + async fn test_eth_api_settings() { + use reth_rpc_eth_api::{ + helpers::{Call, EthState, EthTransactions, LoadPendingBlock}, + EthApiTypes, + }; + use reth_rpc_eth_types::{builder::config::PendingBlockKind, EthApiSettings}; + use std::time::Duration; + + fn settings(api: &impl EthApiTypes) -> &EthApiSettings { + api.eth_api_settings() + } + + let default_api = build_test_eth_api(MockEthProvider::default()); + assert_eq!(settings(&default_api), &EthApiSettings::default()); + + let expected = EthApiSettings { + gas_cap: 123_456, + max_simulate_blocks: 7, + compute_state_root_for_eth_simulate: true, + eth_proof_window: 42, + pending_block_kind: PendingBlockKind::Empty, + send_raw_transaction_sync_timeout: Duration::from_secs(9), + evm_memory_limit: 1 << 20, + force_blob_sidecar_upcasting: true, + }; + let api = EthApiBuilder::new( + MockEthProvider::default(), + testing_pool(), + NoopNetwork::default(), + EthEvmConfig::mainnet(), + ) + .gas_cap(expected.gas_cap.into()) + .max_simulate_blocks(expected.max_simulate_blocks) + .compute_state_root_for_eth_simulate(expected.compute_state_root_for_eth_simulate) + .eth_proof_window(expected.eth_proof_window) + .pending_block_kind(expected.pending_block_kind) + .send_raw_transaction_sync_timeout(expected.send_raw_transaction_sync_timeout) + .evm_memory_limit(expected.evm_memory_limit) + .force_blob_sidecar_upcasting(expected.force_blob_sidecar_upcasting) + .build(); + + assert_eq!(settings(&api), &expected); + assert_eq!(Call::call_gas_limit(&api), expected.gas_cap); + assert_eq!(Call::max_simulate_blocks(&api), expected.max_simulate_blocks); + assert_eq!( + Call::compute_state_root_for_eth_simulate(&api), + expected.compute_state_root_for_eth_simulate + ); + assert_eq!(Call::evm_memory_limit(&api), expected.evm_memory_limit); + assert_eq!(EthState::max_proof_window(&api), expected.eth_proof_window); + assert_eq!(LoadPendingBlock::pending_block_kind(&api), expected.pending_block_kind); + assert_eq!( + EthTransactions::send_raw_transaction_sync_timeout(&api), + expected.send_raw_transaction_sync_timeout + ); + assert_eq!(api.inner.gas_cap(), expected.gas_cap); + assert_eq!(api.inner.max_simulate_blocks(), expected.max_simulate_blocks); + assert_eq!( + api.inner.compute_state_root_for_eth_simulate(), + expected.compute_state_root_for_eth_simulate + ); + assert_eq!(api.inner.eth_proof_window(), expected.eth_proof_window); + assert_eq!(api.inner.pending_block_kind(), expected.pending_block_kind); + assert_eq!( + api.inner.send_raw_transaction_sync_timeout(), + expected.send_raw_transaction_sync_timeout + ); + assert_eq!(api.inner.evm_memory_limit(), expected.evm_memory_limit); + assert_eq!(api.inner.force_blob_sidecar_upcasting(), expected.force_blob_sidecar_upcasting); + assert!(std::ptr::eq(settings(&api), settings(&api.clone()))); + } + // Function to prepare the EthApi with mock data fn prepare_eth_api( newest_block: u64, diff --git a/crates/rpc/rpc/src/eth/helpers/call.rs b/crates/rpc/rpc/src/eth/helpers/call.rs index 4caf6be3200..0ddab24a2f1 100644 --- a/crates/rpc/rpc/src/eth/helpers/call.rs +++ b/crates/rpc/rpc/src/eth/helpers/call.rs @@ -22,25 +22,6 @@ where EthApiError: FromEvmError, Rpc: RpcConvert, { - #[inline] - fn call_gas_limit(&self) -> u64 { - self.inner.gas_cap() - } - - #[inline] - fn max_simulate_blocks(&self) -> u64 { - self.inner.max_simulate_blocks() - } - - #[inline] - fn compute_state_root_for_eth_simulate(&self) -> bool { - self.inner.compute_state_root_for_eth_simulate() - } - - #[inline] - fn evm_memory_limit(&self) -> u64 { - self.inner.evm_memory_limit() - } } impl EstimateCall for EthApi diff --git a/crates/rpc/rpc/src/eth/helpers/pending_block.rs b/crates/rpc/rpc/src/eth/helpers/pending_block.rs index 0e0ae51669f..5c44f48271f 100644 --- a/crates/rpc/rpc/src/eth/helpers/pending_block.rs +++ b/crates/rpc/rpc/src/eth/helpers/pending_block.rs @@ -6,7 +6,7 @@ use reth_rpc_eth_api::{ helpers::{pending_block::PendingEnvBuilder, LoadPendingBlock}, FromEvmError, RpcNodeCore, }; -use reth_rpc_eth_types::{builder::config::PendingBlockKind, EthApiError, PendingBlock}; +use reth_rpc_eth_types::{EthApiError, PendingBlock}; impl LoadPendingBlock for EthApi where @@ -23,9 +23,4 @@ where fn pending_env_builder(&self) -> &dyn PendingEnvBuilder { self.inner.pending_env_builder() } - - #[inline] - fn pending_block_kind(&self) -> PendingBlockKind { - self.inner.pending_block_kind() - } } diff --git a/crates/rpc/rpc/src/eth/helpers/state.rs b/crates/rpc/rpc/src/eth/helpers/state.rs index 41aa273b122..390424db3ed 100644 --- a/crates/rpc/rpc/src/eth/helpers/state.rs +++ b/crates/rpc/rpc/src/eth/helpers/state.rs @@ -14,9 +14,6 @@ where Rpc: RpcConvert, Self: LoadPendingBlock, { - fn max_proof_window(&self) -> u64 { - self.inner.eth_proof_window() - } } impl LoadState for EthApi diff --git a/crates/rpc/rpc/src/eth/helpers/transaction.rs b/crates/rpc/rpc/src/eth/helpers/transaction.rs index eaf2af53043..52473d687b5 100644 --- a/crates/rpc/rpc/src/eth/helpers/transaction.rs +++ b/crates/rpc/rpc/src/eth/helpers/transaction.rs @@ -1,7 +1,5 @@ //! Contains RPC handler implementations specific to transactions -use std::time::Duration; - use crate::EthApi; use alloy_consensus::BlobTransactionValidationError; use alloy_eips::{eip7594::BlobTransactionSidecarVariant, BlockId, Typed2718}; @@ -31,11 +29,6 @@ where self.inner.signers() } - #[inline] - fn send_raw_transaction_sync_timeout(&self) -> Duration { - self.inner.send_raw_transaction_sync_timeout() - } - async fn send_pool_transaction( &self, origin: reth_transaction_pool::TransactionOrigin, @@ -131,6 +124,8 @@ where #[cfg(test)] mod tests { + use std::time::Duration; + use super::*; use crate::eth::helpers::{signer::DevSigner, types::EthRpcConverter}; use alloy_consensus::{ From ff28bad67ec87f5fd6d03db4aacc252413e8b242 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Wed, 16 Sep 2026 05:50:55 -0400 Subject: [PATCH 276/294] refactor(rpc): pass eth api settings directly to constructor (#27268) --- crates/rpc/rpc-eth-types/src/settings.rs | 20 ++++++++--- crates/rpc/rpc/src/eth/builder.rs | 31 ++++++++++------- crates/rpc/rpc/src/eth/core.rs | 43 ++++++++++-------------- 3 files changed, 51 insertions(+), 43 deletions(-) diff --git a/crates/rpc/rpc-eth-types/src/settings.rs b/crates/rpc/rpc-eth-types/src/settings.rs index 0a3b42bf108..fbcbcc8ce1c 100644 --- a/crates/rpc/rpc-eth-types/src/settings.rs +++ b/crates/rpc/rpc-eth-types/src/settings.rs @@ -1,18 +1,25 @@ -//! Runtime settings shared by `eth` RPC helpers. +//! Settings for initializing and serving the `eth` RPC API. use crate::{builder::config::PendingBlockKind, RPC_DEFAULT_GAS_CAP}; use reth_rpc_server_types::constants::{ - DEFAULT_ETH_PROOF_WINDOW, DEFAULT_MAX_SIMULATE_BLOCKS, - RPC_DEFAULT_SEND_RAW_TX_SYNC_TIMEOUT_SECS, + DEFAULT_ETH_PROOF_WINDOW, DEFAULT_MAX_BLOCKING_IO_REQUEST, DEFAULT_MAX_SIMULATE_BLOCKS, + DEFAULT_PROOF_PERMITS, RPC_DEFAULT_SEND_RAW_TX_SYNC_TIMEOUT_SECS, }; use std::time::Duration; -/// Settings used when serving `eth` RPC requests. +/// Settings used when initializing the API and serving `eth` RPC requests. /// /// These settings are shared by the API's helper traits so additional settings can be exposed -/// without adding individual trait methods. +/// without adding individual trait methods. Concurrency and batching limits take effect when the +/// API is constructed. #[derive(Debug, Clone, Eq, PartialEq)] pub struct EthApiSettings { + /// Maximum number of concurrent proof requests. + pub proof_permits: usize, + /// Maximum batch size for transaction insertions. + pub max_batch_size: usize, + /// Maximum number of concurrent blocking IO requests. + pub max_blocking_io_requests: usize, /// Maximum gas limit for `eth_call` and call tracing RPC methods. pub gas_cap: u64, /// Maximum number of blocks for `eth_simulateV1`. @@ -34,6 +41,9 @@ pub struct EthApiSettings { impl Default for EthApiSettings { fn default() -> Self { Self { + proof_permits: DEFAULT_PROOF_PERMITS, + max_batch_size: 1, + max_blocking_io_requests: DEFAULT_MAX_BLOCKING_IO_REQUEST, gas_cap: RPC_DEFAULT_GAS_CAP.into(), max_simulate_blocks: DEFAULT_MAX_SIMULATE_BLOCKS, compute_state_root_for_eth_simulate: false, diff --git a/crates/rpc/rpc/src/eth/builder.rs b/crates/rpc/rpc/src/eth/builder.rs index 8cb0d665953..59a0d8349fe 100644 --- a/crates/rpc/rpc/src/eth/builder.rs +++ b/crates/rpc/rpc/src/eth/builder.rs @@ -11,8 +11,9 @@ use reth_rpc_eth_api::{ }; use reth_rpc_eth_types::{ builder::config::PendingBlockKind, fee_history::fee_history_cache_new_blocks_task, - receipt::EthReceiptConverter, EthStateCache, EthStateCacheConfig, FeeHistoryCache, - FeeHistoryCacheConfig, ForwardConfig, GasCap, GasPriceOracle, GasPriceOracleConfig, + receipt::EthReceiptConverter, EthApiSettings, EthStateCache, EthStateCacheConfig, + FeeHistoryCache, FeeHistoryCacheConfig, ForwardConfig, GasCap, GasPriceOracle, + GasPriceOracleConfig, }; use reth_rpc_server_types::constants::{ DEFAULT_ETH_PROOF_WINDOW, DEFAULT_MAX_BLOCKING_IO_REQUEST, DEFAULT_MAX_SIMULATE_BLOCKS, @@ -557,14 +558,25 @@ where }, ); + let settings = EthApiSettings { + proof_permits, + max_batch_size, + max_blocking_io_requests, + gas_cap: gas_cap.into(), + max_simulate_blocks, + compute_state_root_for_eth_simulate, + eth_proof_window, + pending_block_kind, + send_raw_transaction_sync_timeout, + evm_memory_limit, + force_blob_sidecar_upcasting, + }; + EthApiInner::new( components, eth_cache, gas_oracle, - gas_cap, - max_simulate_blocks, - compute_state_root_for_eth_simulate, - eth_proof_window, + settings, blocking_task_pool.unwrap_or_else(|| { BlockingTaskPool::builder() .thread_name(|i| format!("blocking-{i:02}")) @@ -574,16 +586,9 @@ where }), fee_history_cache, task_spawner, - proof_permits, rpc_converter, next_env, - max_batch_size, - max_blocking_io_requests, - pending_block_kind, raw_tx_forwarder.forwarder_client(), - send_raw_transaction_sync_timeout, - evm_memory_limit, - force_blob_sidecar_upcasting, ) } diff --git a/crates/rpc/rpc/src/eth/core.rs b/crates/rpc/rpc/src/eth/core.rs index a3058de6559..85c5da9a312 100644 --- a/crates/rpc/rpc/src/eth/core.rs +++ b/crates/rpc/rpc/src/eth/core.rs @@ -22,7 +22,7 @@ use reth_rpc_eth_api::{ }; use reth_rpc_eth_types::{ builder::config::PendingBlockKind, receipt::EthReceiptConverter, EthApiError, EthApiSettings, - EthStateCache, FeeHistoryCache, GasCap, GasPriceOracle, PendingBlock, + EthStateCache, FeeHistoryCache, GasPriceOracle, PendingBlock, }; use reth_storage_api::{noop::NoopProvider, BlockReaderIdExt, ProviderHeader}; use reth_tasks::{ @@ -286,23 +286,13 @@ where components: N, eth_cache: EthStateCache, gas_oracle: GasPriceOracle, - gas_cap: impl Into, - max_simulate_blocks: u64, - compute_state_root_for_eth_simulate: bool, - eth_proof_window: u64, + settings: EthApiSettings, blocking_task_pool: BlockingTaskPool, fee_history_cache: FeeHistoryCache>, task_spawner: Runtime, - proof_permits: usize, converter: Rpc, next_env: impl PendingEnvBuilder, - max_batch_size: usize, - max_blocking_io_requests: usize, - pending_block_kind: PendingBlockKind, raw_tx_forwarder: Option, - send_raw_transaction_sync_timeout: Duration, - evm_memory_limit: u64, - force_blob_sidecar_upcasting: bool, ) -> Self { let signers = parking_lot::RwLock::new(Default::default()); // get the block number of the latest block @@ -320,7 +310,7 @@ where // Create tx pool insertion batcher let (processor, tx_batch_sender) = - BatchTxProcessor::new(components.pool().clone(), max_batch_size); + BatchTxProcessor::new(components.pool().clone(), settings.max_batch_size); task_spawner.spawn_critical_task("tx-batcher", processor); Self { @@ -328,23 +318,16 @@ where signers, eth_cache, gas_oracle, - settings: EthApiSettings { - gas_cap: gas_cap.into().into(), - max_simulate_blocks, - compute_state_root_for_eth_simulate, - eth_proof_window, - pending_block_kind, - send_raw_transaction_sync_timeout, - evm_memory_limit, - force_blob_sidecar_upcasting, - }, starting_block, task_spawner, pending_block: Default::default(), blocking_task_pool, fee_history_cache, - blocking_task_guard: BlockingTaskGuard::new(proof_permits), - blocking_io_request_semaphore: Arc::new(Semaphore::new(max_blocking_io_requests)), + blocking_task_guard: BlockingTaskGuard::new(settings.proof_permits), + blocking_io_request_semaphore: Arc::new(Semaphore::new( + settings.max_blocking_io_requests, + )), + settings, raw_tx_sender, raw_tx_forwarder, converter, @@ -637,6 +620,9 @@ mod tests { assert_eq!(settings(&default_api), &EthApiSettings::default()); let expected = EthApiSettings { + proof_permits: 3, + max_batch_size: 5, + max_blocking_io_requests: 7, gas_cap: 123_456, max_simulate_blocks: 7, compute_state_root_for_eth_simulate: true, @@ -652,6 +638,9 @@ mod tests { NoopNetwork::default(), EthEvmConfig::mainnet(), ) + .proof_permits(expected.proof_permits) + .max_batch_size(expected.max_batch_size) + .max_blocking_io_requests(expected.max_blocking_io_requests) .gas_cap(expected.gas_cap.into()) .max_simulate_blocks(expected.max_simulate_blocks) .compute_state_root_for_eth_simulate(expected.compute_state_root_for_eth_simulate) @@ -663,6 +652,10 @@ mod tests { .build(); assert_eq!(settings(&api), &expected); + assert_eq!( + api.inner.blocking_io_request_semaphore().available_permits(), + expected.max_blocking_io_requests + ); assert_eq!(Call::call_gas_limit(&api), expected.gas_cap); assert_eq!(Call::max_simulate_blocks(&api), expected.max_simulate_blocks); assert_eq!( From 18e6014ac023b8049ff8d4a3ffb739a507cc8b04 Mon Sep 17 00:00:00 2001 From: Karl Yu <43113774+0xKarl98@users.noreply.github.com> Date: Wed, 16 Sep 2026 10:37:11 +0000 Subject: [PATCH 277/294] feat(rpc): prewarm BAL cache from local replay (#24097) Co-authored-by: Matthias Seitz <19890894+mattsse@users.noreply.github.com> --- crates/ethereum/node/tests/e2e/rpc.rs | 88 ++++++++- crates/node/builder/src/rpc.rs | 175 +++++++++++++++++- crates/node/core/src/args/rpc_server.rs | 17 ++ crates/node/core/src/args/rpc_state_cache.rs | 22 +++ crates/rpc/rpc-builder/src/config.rs | 2 + crates/rpc/rpc-eth-api/src/helpers/bal.rs | 13 +- crates/rpc/rpc-eth-types/src/cache/config.rs | 11 ++ crates/rpc/rpc-eth-types/src/cache/mod.rs | 56 ++++++ .../rpc-eth-types/src/cache/multi_consumer.rs | 5 + crates/rpc/rpc-eth-types/src/settings.rs | 3 + crates/rpc/rpc/src/eth/builder.rs | 2 + crates/rpc/rpc/src/eth/core.rs | 5 + docs/vocs/docs/pages/cli/reth/node.mdx | 8 + 13 files changed, 401 insertions(+), 6 deletions(-) diff --git a/crates/ethereum/node/tests/e2e/rpc.rs b/crates/ethereum/node/tests/e2e/rpc.rs index 0182d842ce8..3cad83a38b5 100644 --- a/crates/ethereum/node/tests/e2e/rpc.rs +++ b/crates/ethereum/node/tests/e2e/rpc.rs @@ -1,5 +1,7 @@ use crate::utils::{eth_payload_attributes, eth_payload_attributes_amsterdam}; -use alloy_eips::{eip2718::Encodable2718, eip7910::EthConfig, BlockNumberOrTag}; +use alloy_eips::{ + eip2718::Encodable2718, eip7910::EthConfig, eip7928::BlockAccessList, BlockNumberOrTag, +}; use alloy_genesis::Genesis; use alloy_primitives::{Address, Bytes, B256, U256}; use alloy_provider::{ @@ -114,6 +116,90 @@ async fn test_block_access_list_lookup_semantics() -> eyre::Result<()> { Ok(()) } +#[tokio::test(flavor = "multi_thread")] +async fn test_bal_prewarming_for_transaction_replay() -> eyre::Result<()> { + for (cache_computed, prewarm) in [(false, false), (true, false), (false, true)] { + let chain_spec = Arc::new( + ChainSpecBuilder::default() + .chain(MAINNET.chain) + .genesis(serde_json::from_str(include_str!("../assets/genesis.json"))?) + .cancun_activated() + .build(), + ); + let (mut nodes, wallet) = + E2ETestSetupBuilder::::new(1, chain_spec, eth_payload_attributes) + .with_node_config_modifier(move |mut config| { + config.rpc.rpc_state_cache.cache_computed_bals = cache_computed; + config.rpc.rpc_state_cache.prewarm_bals = prewarm.then_some(0); + config + }) + .build() + .await?; + let mut node = nodes.pop().unwrap(); + let provider = ProviderBuilder::new() + .wallet(EthereumWallet::new(wallet.wallet_gen().swap_remove(0))) + .connect_http(node.rpc_url()); + let first = provider + .send_transaction(TransactionRequest::default().to(Address::ZERO).value(U256::from(1))) + .await?; + let second = provider + .send_transaction(TransactionRequest::default().to(Address::ZERO).value(U256::from(2))) + .await?; + node.advance_block().await?; + let receipt = second.get_receipt().await?; + assert_eq!(receipt.transaction_index, Some(1)); + let block_hash = receipt.block_hash.unwrap(); + let client = node.rpc_client().unwrap(); + let cache = node.rpc.inner.eth_api().cache(); + let opts = + serde_json::json!({"tracer": "prestateTracer", "tracerConfig": {"diffMode": true}}); + let hashes = [*first.tx_hash(), receipt.transaction_hash]; + let mut expected = Vec::new(); + if prewarm { + tokio::time::timeout(std::time::Duration::from_secs(10), async { + loop { + let (_, bal) = + cache.get_recovered_block_and_maybe_bal(block_hash).await?.unwrap(); + if bal.is_some() { + return Ok::<_, eyre::Report>(()); + } + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + } + }) + .await??; + } else { + assert!(cache + .get_recovered_block_and_maybe_bal(block_hash) + .await? + .unwrap() + .1 + .is_none()); + for hash in hashes { + expected.push( + client + .request::("debug_traceTransaction", (hash, &opts)) + .await?, + ); + } + } + let bal: BlockAccessList = client.request("eth_getBlockAccessList", (block_hash,)).await?; + assert!(!bal.is_empty()); + let raw: Bytes = client.request("debug_getRawBlockAccessList", (block_hash,)).await?; + assert_eq!(raw.as_ref(), alloy_rlp::encode(&bal)); + let (_, cached_bal) = cache.get_recovered_block_and_maybe_bal(block_hash).await?.unwrap(); + assert_eq!(cached_bal.is_some(), cache_computed || prewarm); + + for (index, hash) in hashes.into_iter().enumerate() { + let actual: serde_json::Value = + client.request("debug_traceTransaction", (hash, &opts)).await?; + if !prewarm { + assert_eq!(actual, expected[index]); + } + } + } + Ok(()) +} + #[tokio::test] async fn test_fee_history() -> eyre::Result<()> { reth_tracing::init_test_tracing(); diff --git a/crates/node/builder/src/rpc.rs b/crates/node/builder/src/rpc.rs index d76fed14791..6cc2b6973b2 100644 --- a/crates/node/builder/src/rpc.rs +++ b/crates/node/builder/src/rpc.rs @@ -16,11 +16,14 @@ use crate::{ invalid_block_hook::InvalidBlockHookExt, txpool_prewarm, ConfigureEngineEvm, ConsensusEngineEvent, ConsensusEngineHandle, }; +use alloy_consensus::BlockHeader; +use alloy_eips::BlockNumberOrTag; use alloy_rpc_types::engine::ClientVersionV1; use alloy_rpc_types_engine::ExecutionData; +use futures::{Stream, StreamExt}; use jsonrpsee::RpcModule; use parking_lot::Mutex; -use reth_chain_state::CanonStateSubscriptions; +use reth_chain_state::{CanonStateNotification, CanonStateSubscriptions}; use reth_chainspec::{ChainSpecProvider, EthChainSpec, EthereumHardforks, Hardforks}; use reth_node_api::{ AddOnsContext, BlockTy, EngineApiValidator, EngineTypes, FullNodeComponents, FullNodeTypes, @@ -32,11 +35,15 @@ use reth_node_core::{ version::{version_metadata, CLIENT_CODE}, }; use reth_payload_builder::{PayloadBuilderHandle, PayloadStore}; +use reth_primitives_traits::NodePrimitives; use reth_rpc::{ eth::{core::EthRpcConverterFor, DevSigner, EthApiTypes, FullEthApiServer}, AdminApi, }; -use reth_rpc_api::{eth::helpers::EthTransactions, IntoEngineApiRpcModule}; +use reth_rpc_api::{ + eth::helpers::{EthTransactions, GetBlockAccessList}, + IntoEngineApiRpcModule, +}; use reth_rpc_builder::{ auth::{AuthRpcModule, AuthServerHandle}, config::RethRpcServerConfig, @@ -1150,6 +1157,13 @@ where cache_new_blocks_task(c, new_canonical_blocks).await; }); + let prewarm_bals = config + .rpc + .eth_config() + .cache + .prewarm_bals + .map(|count| (count, node.provider().canonical_state_stream())); + let eth_config = config.rpc.eth_config().max_batch_size(config.txpool.max_batch_size()); let ctx = EthApiCtx { components: &node, @@ -1159,6 +1173,14 @@ where }; let eth_api = eth_api_builder.build_eth_api(ctx).await?; + if let Some((count, events)) = prewarm_bals { + node.task_executor().spawn_task(prewarm_new_block_bals_task( + eth_api.clone(), + events, + count, + )); + } + let auth_config = config.rpc.auth_server_config(jwt_secret)?; let module_config = config.rpc.transport_rpc_module_config(); debug!(target: "reth::cli", http=?module_config.http(), ws=?module_config.ws(), "Using RPC module config"); @@ -1337,6 +1359,7 @@ impl<'a, N: FullNodeComponents reth_rpc::EthApiBuilder> { reth_rpc::EthApiBuilder::new_with_components(self.components.clone()) .eth_cache(self.cache) + .eth_state_cache_config(self.config.cache) .task_spawner(self.components.task_executor().clone()) .gas_cap(self.config.rpc_gas_cap.into()) .max_simulate_blocks(self.config.rpc_max_simulate_blocks) @@ -1653,3 +1676,151 @@ pub struct EngineShutdownRequest { /// Channel to signal shutdown completion. pub done_tx: oneshot::Sender<()>, } + +/// Prewarms recent and new canonical blocks until a block includes a BAL hash. +async fn prewarm_new_block_bals_task( + eth_api: EthApi, + mut events: impl Stream> + Unpin, + startup_blocks: usize, +) { + let mut startup = match eth_api.recovered_block(BlockNumberOrTag::Latest.into()).await { + Ok(Some(head)) => { + if head.block_access_list_hash().is_some() { + debug!(target: "reth::cli", block_hash = ?head.hash(), "Stopping BAL prewarming: native BALs available"); + return; + } + + // Leave the newest BALs in the cache if the startup range exceeds its capacity. + head.number().saturating_sub(startup_blocks as u64)..head.number() + } + Err(err) => { + debug!(target: "reth::cli", %err, "Failed to load head for BAL prewarming"); + 0..0 + } + Ok(None) => 0..0, + } + .map(|number| number + 1); + + loop { + tokio::select! { + // Observe new blocks (including native BAL activation) between startup replays. + biased; + Some(event) = events.next() => { + let committed = event.committed(); + if let Some(block) = committed.blocks_iter().find(|block| block.block_access_list_hash().is_some()) { + debug!(target: "reth::cli", block_hash = ?block.hash(), "Stopping BAL prewarming: native BALs available"); + return; + } + + for block in committed.blocks_iter() { + if let Err(err) = eth_api.get_block_access_list(block.hash().into()).await { + debug!( + target: "reth::cli", + %err, + block_hash = ?block.hash(), + "Failed to prewarm BAL for canonical block", + ); + } + } + } + Some(number) = async { startup.next() } => { + if let Err(err) = eth_api.get_block_access_list(number.into()).await { + debug!(target: "reth::cli", %err, block_number = number, "Failed to prewarm BAL on startup"); + } + } + else => break, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use alloy_consensus::{Block, BlockBody, Header}; + use alloy_primitives::B256; + use reth_evm_ethereum::EthEvmConfig; + use reth_network_api::noop::NoopNetwork; + use reth_primitives_traits::RecoveredBlock; + use reth_provider::{test_utils::MockEthProvider, Chain, ExecutionOutcome}; + use reth_rpc::EthApiBuilder; + use reth_rpc_eth_types::EthStateCacheConfig; + use reth_transaction_pool::noop::NoopTransactionPool; + + #[tokio::test] + async fn bal_prewarming_startup_range_and_native_stop() { + for (count, native, native_event) in [ + (0, false, false), + (2, false, false), + (10, false, false), + (2, true, false), + (10, false, true), + ] { + let provider = MockEthProvider::default(); + let blocks: Vec<_> = (0..=3) + .map(|number| { + let block = RecoveredBlock::new_unhashed( + Block { + header: Header { + number, + block_access_list_hash: (native && number == 3) + .then_some(B256::ZERO), + ..Default::default() + }, + body: BlockBody::default(), + }, + Vec::new(), + ); + provider.add_block(block.hash(), block.clone_block()); + block + }) + .collect(); + let api = EthApiBuilder::new( + provider, + NoopTransactionPool::default(), + NoopNetwork::default(), + EthEvmConfig::mainnet(), + ) + .eth_state_cache_config(EthStateCacheConfig { + prewarm_bals: Some(count), + ..Default::default() + }) + .build(); + + let chain = Chain::new( + blocks.clone(), + ExecutionOutcome { receipts: vec![vec![]; blocks.len()], ..Default::default() }, + Default::default(), + ); + cache_new_blocks_task( + api.cache().clone(), + futures::stream::iter([reth_chain_state::CanonStateNotification::Commit { + new: Arc::new(chain), + }]), + ) + .await; + let events = futures::stream::iter(native_event.then(|| { + let mut block = blocks.last().unwrap().clone_block(); + block.header.number += 1; + block.header.block_access_list_hash = Some(B256::ZERO); + CanonStateNotification::Commit { + new: Arc::new(Chain::::new( + [RecoveredBlock::new_unhashed(block, Vec::new())], + Default::default(), + Default::default(), + )), + } + })); + prewarm_new_block_bals_task(api.clone(), events, count).await; + for (number, block) in blocks.iter().enumerate() { + assert_eq!( + api.cache().get_bal(block.hash()).await.unwrap().is_some(), + !native && + !native_event && + number != 0 && + number > 3_usize.saturating_sub(count), + "count={count}, native={native}, native_event={native_event}, block={number}", + ); + } + } + } +} diff --git a/crates/node/core/src/args/rpc_server.rs b/crates/node/core/src/args/rpc_server.rs index 74514481335..08eda4a6e47 100644 --- a/crates/node/core/src/args/rpc_server.rs +++ b/crates/node/core/src/args/rpc_server.rs @@ -1038,6 +1038,19 @@ mod tests { args: T, } + #[test] + fn parse_bal_cache_modes() { + for (flag, cache_computed, prewarm) in [ + ("--rpc-cache.cache-computed-bals", true, None), + ("--rpc-cache.prewarm-bals", false, Some(0)), + ("--rpc-cache.prewarm-bals=10", false, Some(10)), + ] { + let args = CommandParser::::parse_from(["reth", flag]).args; + assert_eq!(args.rpc_state_cache.cache_computed_bals, cache_computed); + assert_eq!(args.rpc_state_cache.prewarm_bals, prewarm); + } + } + #[test] fn test_rpc_server_args_parser() { let args = @@ -1221,6 +1234,8 @@ mod tests { max_receipts: 2000, max_headers: 1000, max_bals: 1000, + cache_computed_bals: true, + prewarm_bals: Some(0), max_concurrent_db_requests: 512, max_cached_tx_hashes: 100_000, }, @@ -1313,6 +1328,8 @@ mod tests { "1000", "--rpc-cache.max-bals", "1000", + "--rpc-cache.cache-computed-bals", + "--rpc-cache.prewarm-bals", "--rpc-cache.max-concurrent-db-requests", "512", "--gpo.blocks", diff --git a/crates/node/core/src/args/rpc_state_cache.rs b/crates/node/core/src/args/rpc_state_cache.rs index eea316055d8..0c691e3d001 100644 --- a/crates/node/core/src/args/rpc_state_cache.rs +++ b/crates/node/core/src/args/rpc_state_cache.rs @@ -38,6 +38,24 @@ pub struct RpcStateCacheArgs { )] pub max_bals: u32, + /// Cache block access lists computed by RPC requests for transaction tracing. + #[arg(long = "rpc-cache.cache-computed-bals")] + pub cache_computed_bals: bool, + + /// Replay new canonical blocks to cache block access lists until native BAL support. + /// + /// Optionally replay the latest BLOCKS blocks sequentially on startup. Without a count, only + /// new blocks are prewarmed. Implies --rpc-cache.cache-computed-bals. + /// Prewarming stops when a canonical block with a block access list hash is received. + #[arg( + long = "rpc-cache.prewarm-bals", + value_name = "BLOCKS", + num_args = 0..=1, + default_missing_value = "0", + require_equals = true, + )] + pub prewarm_bals: Option, + /// Max number of concurrent database requests. #[arg( long = "rpc-cache.max-concurrent-db-requests", @@ -60,6 +78,8 @@ impl RpcStateCacheArgs { self.max_receipts = 0; self.max_headers = 0; self.max_bals = 0; + self.cache_computed_bals = false; + self.prewarm_bals = None; } } @@ -70,6 +90,8 @@ impl Default for RpcStateCacheArgs { max_receipts: DEFAULT_RECEIPT_CACHE_MAX_LEN, max_headers: DEFAULT_HEADER_CACHE_MAX_LEN, max_bals: DEFAULT_BAL_CACHE_MAX_LEN, + cache_computed_bals: false, + prewarm_bals: None, max_concurrent_db_requests: DEFAULT_CONCURRENT_DB_REQUESTS, max_cached_tx_hashes: DEFAULT_MAX_CACHED_TX_HASHES, } diff --git a/crates/rpc/rpc-builder/src/config.rs b/crates/rpc/rpc-builder/src/config.rs index 6a6b8f1351a..de5cfa0bd6c 100644 --- a/crates/rpc/rpc-builder/src/config.rs +++ b/crates/rpc/rpc-builder/src/config.rs @@ -126,6 +126,8 @@ impl RethRpcServerConfig for RpcServerArgs { max_blocks: self.rpc_state_cache.max_blocks, max_receipts: self.rpc_state_cache.max_receipts, max_bals: self.rpc_state_cache.max_bals, + cache_computed_bals: self.rpc_state_cache.cache_computed_bals, + prewarm_bals: self.rpc_state_cache.prewarm_bals, max_concurrent_db_requests: self.rpc_state_cache.max_concurrent_db_requests, max_cached_tx_hashes: self.rpc_state_cache.max_cached_tx_hashes, } diff --git a/crates/rpc/rpc-eth-api/src/helpers/bal.rs b/crates/rpc/rpc-eth-api/src/helpers/bal.rs index b6640ff5777..ec68eebf995 100644 --- a/crates/rpc/rpc-eth-api/src/helpers/bal.rs +++ b/crates/rpc/rpc-eth-api/src/helpers/bal.rs @@ -8,6 +8,7 @@ use reth_evm::{block::BlockExecutor, ConfigureEvm, Evm}; use reth_revm::{database::StateProviderDatabase, State}; use reth_rpc_eth_types::{error::FromEthApiError, EthApiError}; use reth_storage_api::StateProviderFactory; +use std::sync::Arc; use crate::{ helpers::{Call, LoadBlock, Trace}, @@ -66,7 +67,6 @@ pub trait GetBlockAccessList: Trace + Call + LoadBlock + RpcNodeCoreExt { executor.apply_pre_execution_changes().map_err(Self::Error::from_eth_err)?; executor.evm_mut().db_mut().bump_bal_index(); - // replay all transactions prior to the targeted transaction for block_tx in block_txs { executor.execute_transaction(block_tx).map_err(Self::Error::from_eth_err)?; executor.evm_mut().db_mut().bump_bal_index(); @@ -76,8 +76,15 @@ pub trait GetBlockAccessList: Trace + Call + LoadBlock + RpcNodeCoreExt { .apply_post_execution_changes() .map_err(|err| EthApiError::Internal(err.into()))?; - let bal = db.take_built_alloy_bal(); - Ok(bal) + let revm_bal = db.take_built_bal().expect("BAL builder configured"); + if !eth_api.eth_api_settings().cache_computed_bals { + return Ok(Some(revm_bal.into_alloy_bal())); + } + + let bal = revm_bal.clone().into_alloy_bal(); + let raw = alloy_rlp::encode(&bal).into(); + eth_api.cache().insert_bal(block.hash(), DecodedBal::new(Arc::new(revm_bal), raw)); + Ok(Some(bal)) }) .await } diff --git a/crates/rpc/rpc-eth-types/src/cache/config.rs b/crates/rpc/rpc-eth-types/src/cache/config.rs index 4fc1b6e90a7..ac534abb39e 100644 --- a/crates/rpc/rpc-eth-types/src/cache/config.rs +++ b/crates/rpc/rpc-eth-types/src/cache/config.rs @@ -23,6 +23,15 @@ pub struct EthStateCacheConfig { /// /// Default is 1000. pub max_bals: u32, + /// Cache BALs computed by RPC requests for transaction tracing. Disabled by default. + #[serde(default)] + pub cache_computed_bals: bool, + /// Prewarm BALs until native BAL support, optionally replaying this many recent blocks on + /// startup. `Some(0)` prewarms only new canonical blocks; `None` disables prewarming. + /// + /// Implies caching of BALs computed by RPC requests. + #[serde(default)] + pub prewarm_bals: Option, /// Max number of concurrent database requests. /// /// Default is 512. @@ -37,6 +46,8 @@ impl Default for EthStateCacheConfig { max_blocks: DEFAULT_BLOCK_CACHE_MAX_LEN, max_receipts: DEFAULT_RECEIPT_CACHE_MAX_LEN, max_bals: DEFAULT_BAL_CACHE_MAX_LEN, + cache_computed_bals: false, + prewarm_bals: None, max_concurrent_db_requests: DEFAULT_CONCURRENT_DB_REQUESTS, max_cached_tx_hashes: DEFAULT_MAX_CACHED_TX_HASHES, } diff --git a/crates/rpc/rpc-eth-types/src/cache/mod.rs b/crates/rpc/rpc-eth-types/src/cache/mod.rs index 3d06e7548a8..ee168055d04 100644 --- a/crates/rpc/rpc-eth-types/src/cache/mod.rs +++ b/crates/rpc/rpc-eth-types/src/cache/mod.rs @@ -96,6 +96,8 @@ impl EthStateCache { max_blocks, max_receipts, max_bals, + cache_computed_bals: _, + prewarm_bals: _, max_concurrent_db_requests, max_cached_tx_hashes, } = config; @@ -257,6 +259,13 @@ impl EthStateCache { .map_err(|_| CacheServiceUnavailable)? .map(|maybe_bal| maybe_bal.map(|cached| cached.0)) } + + /// Inserts a decoded revm BAL into the cache. + pub fn insert_bal(&self, block_hash: B256, bal: DecodedBal>) { + let _ = self + .to_service + .send(CacheAction::InsertBal { block_hash, bal: CachedRevmBal::new(bal) }); + } } /// Thrown when the cache service task dropped. #[derive(Debug, thiserror::Error)] @@ -376,6 +385,11 @@ where } fn on_new_bal(&mut self, block_hash: B256, res: ProviderResult>) { + // A local replay may finish before an older provider lookup returns. + if self.bal_cache.contains_key(&block_hash) { + return + } + if let Some(queued) = self.bal_cache.remove(&block_hash) { for tx in queued { let _ = tx.send(res.clone()); @@ -555,6 +569,9 @@ where CacheAction::BalResult { block_hash, res } => { this.on_new_bal(block_hash, res); } + CacheAction::InsertBal { block_hash, bal } => { + this.on_new_bal(block_hash, Ok(Some(bal))); + } CacheAction::BlockWithSendersResult { block_hash, res } => match res { Ok(Some(block_with_senders)) => { this.on_new_block(block_hash, Ok(Some(block_with_senders))); @@ -650,6 +667,10 @@ enum CacheAction { block_hash: B256, res: ProviderResult>, }, + InsertBal { + block_hash: B256, + bal: CachedRevmBal, + }, CacheNewCanonicalChain { chain_change: ChainChange, }, @@ -889,6 +910,8 @@ mod tests { max_blocks: 4, max_receipts: 4, max_bals: 4, + cache_computed_bals: false, + prewarm_bals: None, max_concurrent_db_requests: 1, max_cached_tx_hashes: 16, }, @@ -942,6 +965,18 @@ mod tests { assert!(service.bal_cache.get(&block_hash).is_none()); } + #[test] + fn late_provider_miss_preserves_generated_bal() { + let mut service = test_service(); + let hash = B256::repeat_byte(0x69); + let (tx, mut rx) = oneshot::channel(); + assert!(service.bal_cache.queue(hash, tx)); + service.on_new_bal(hash, Ok(Some(CachedRevmBal::new(test_decoded_revm_bal())))); + assert!(rx.try_recv().unwrap().unwrap().is_some()); + service.on_new_bal(hash, Ok(None)); + assert!(service.bal_cache.contains_key(&hash)); + } + #[test] fn reorg_forwards_bal_to_queued_requests() { let mut service = test_service(); @@ -997,6 +1032,8 @@ mod tests { max_blocks: 0, max_receipts: 0, max_bals: 4, + cache_computed_bals: false, + prewarm_bals: None, max_concurrent_db_requests: 1, max_cached_tx_hashes: 0, }, @@ -1022,6 +1059,8 @@ mod tests { max_blocks: 4, max_receipts: 0, max_bals: 4, + cache_computed_bals: false, + prewarm_bals: None, max_concurrent_db_requests: 1, max_cached_tx_hashes: 0, }, @@ -1048,6 +1087,21 @@ mod tests { assert_eq!(bal_fetches.load(Ordering::SeqCst), 1); } + #[tokio::test] + async fn insert_bal_populates_cache_without_provider_fetch() { + let fetches = Arc::new(AtomicUsize::default()); + let provider = TestBalProvider::new(fetches.clone()); + let cache = EthStateCache::::spawn_with( + provider, + EthStateCacheConfig { max_bals: 4, ..Default::default() }, + Runtime::test(), + ); + let block_hash = B256::repeat_byte(0x68); + cache.insert_bal(block_hash, test_decoded_revm_bal()); + assert!(cache.get_bal(block_hash).await.unwrap().is_some()); + assert_eq!(fetches.load(Ordering::SeqCst), 0); + } + #[tokio::test] async fn concurrent_get_bal_requests_share_fetch() { let fetches = Arc::new(AtomicUsize::default()); @@ -1058,6 +1112,8 @@ mod tests { max_blocks: 0, max_receipts: 0, max_bals: 4, + cache_computed_bals: false, + prewarm_bals: None, max_concurrent_db_requests: 1, max_cached_tx_hashes: 0, }, diff --git a/crates/rpc/rpc-eth-types/src/cache/multi_consumer.rs b/crates/rpc/rpc-eth-types/src/cache/multi_consumer.rs index 2ac7d37b814..c727bc11faa 100644 --- a/crates/rpc/rpc-eth-types/src/cache/multi_consumer.rs +++ b/crates/rpc/rpc-eth-types/src/cache/multi_consumer.rs @@ -91,6 +91,11 @@ where entry } + /// Checks for a cached value without changing recency or lookup metrics. + pub(crate) fn contains_key(&self, key: &K) -> bool { + self.cache.peek(key).is_some() + } + /// Inserts a new element into the map. /// /// Can fail if the element is rejected by the limiter or if we fail to grow an empty map. diff --git a/crates/rpc/rpc-eth-types/src/settings.rs b/crates/rpc/rpc-eth-types/src/settings.rs index fbcbcc8ce1c..eb150ec0c6f 100644 --- a/crates/rpc/rpc-eth-types/src/settings.rs +++ b/crates/rpc/rpc-eth-types/src/settings.rs @@ -20,6 +20,8 @@ pub struct EthApiSettings { pub max_batch_size: usize, /// Maximum number of concurrent blocking IO requests. pub max_blocking_io_requests: usize, + /// Cache computed block access lists for transaction tracing. + pub cache_computed_bals: bool, /// Maximum gas limit for `eth_call` and call tracing RPC methods. pub gas_cap: u64, /// Maximum number of blocks for `eth_simulateV1`. @@ -44,6 +46,7 @@ impl Default for EthApiSettings { proof_permits: DEFAULT_PROOF_PERMITS, max_batch_size: 1, max_blocking_io_requests: DEFAULT_MAX_BLOCKING_IO_REQUEST, + cache_computed_bals: false, gas_cap: RPC_DEFAULT_GAS_CAP.into(), max_simulate_blocks: DEFAULT_MAX_SIMULATE_BLOCKS, compute_state_root_for_eth_simulate: false, diff --git a/crates/rpc/rpc/src/eth/builder.rs b/crates/rpc/rpc/src/eth/builder.rs index 59a0d8349fe..d54f92037f9 100644 --- a/crates/rpc/rpc/src/eth/builder.rs +++ b/crates/rpc/rpc/src/eth/builder.rs @@ -562,6 +562,8 @@ where proof_permits, max_batch_size, max_blocking_io_requests, + cache_computed_bals: eth_state_cache_config.cache_computed_bals || + eth_state_cache_config.prewarm_bals.is_some(), gas_cap: gas_cap.into(), max_simulate_blocks, compute_state_root_for_eth_simulate, diff --git a/crates/rpc/rpc/src/eth/core.rs b/crates/rpc/rpc/src/eth/core.rs index 85c5da9a312..c6c9533aa05 100644 --- a/crates/rpc/rpc/src/eth/core.rs +++ b/crates/rpc/rpc/src/eth/core.rs @@ -623,6 +623,7 @@ mod tests { proof_permits: 3, max_batch_size: 5, max_blocking_io_requests: 7, + cache_computed_bals: true, gas_cap: 123_456, max_simulate_blocks: 7, compute_state_root_for_eth_simulate: true, @@ -638,6 +639,10 @@ mod tests { NoopNetwork::default(), EthEvmConfig::mainnet(), ) + .eth_state_cache_config(reth_rpc_eth_types::EthStateCacheConfig { + cache_computed_bals: true, + ..Default::default() + }) .proof_permits(expected.proof_permits) .max_batch_size(expected.max_batch_size) .max_blocking_io_requests(expected.max_blocking_io_requests) diff --git a/docs/vocs/docs/pages/cli/reth/node.mdx b/docs/vocs/docs/pages/cli/reth/node.mdx index a0c77f80ee0..79414e8cbc7 100644 --- a/docs/vocs/docs/pages/cli/reth/node.mdx +++ b/docs/vocs/docs/pages/cli/reth/node.mdx @@ -549,6 +549,14 @@ RPC State Cache: [default: 1000] + --rpc-cache.cache-computed-bals + Cache block access lists computed by RPC requests for transaction tracing + + --rpc-cache.prewarm-bals[=] + Replay new canonical blocks to cache block access lists until native BAL support. + + Optionally replay the latest BLOCKS blocks sequentially on startup. Without a count, only new blocks are prewarmed. Implies --rpc-cache.cache-computed-bals. Prewarming stops when a canonical block with a block access list hash is received. + --rpc-cache.max-concurrent-db-requests Max number of concurrent database requests From fc44cfc515085dbb3bc99cecd4a40db4823f878b Mon Sep 17 00:00:00 2001 From: Brian Picciano Date: Wed, 16 Sep 2026 11:41:15 +0000 Subject: [PATCH 278/294] fix(engine): backpressure on state trie frontier (#27269) --- crates/engine/tree/src/tree/mod.rs | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/crates/engine/tree/src/tree/mod.rs b/crates/engine/tree/src/tree/mod.rs index 414ad1834cf..d45cfcab271 100644 --- a/crates/engine/tree/src/tree/mod.rs +++ b/crates/engine/tree/src/tree/mod.rs @@ -504,17 +504,14 @@ where self.incoming_tx.clone() } - /// How many blocks the canonical tip is ahead of the last persisted block. A large gap means - /// persistence is falling behind execution. - const fn persistence_gap(&self) -> u64 { - self.state - .tree_state - .canonical_block_number() - .saturating_sub(self.persistence_state.last_persisted_block.number) + /// How many canonical blocks are retained in memory. A large count means persistence is + /// falling behind execution. + fn persistence_gap(&self) -> u64 { + self.canonical_in_memory_state.canonical_chain().count() as u64 } /// How many blocks beyond the configured in-memory buffer are awaiting persistence. - const fn persistence_backpressure_gap(&self) -> u64 { + fn persistence_backpressure_gap(&self) -> u64 { self.persistence_gap().saturating_sub(self.config.memory_block_buffer_target()) } @@ -522,7 +519,7 @@ where /// /// This is the case when persistence is already running and the number of blocks beyond the /// configured in-memory buffer has reached the configured threshold. - const fn should_backpressure(&self) -> bool { + fn should_backpressure(&self) -> bool { self.persistence_state.in_progress() && self.persistence_backpressure_gap() >= self.config.persistence_backpressure_threshold() @@ -1629,6 +1626,10 @@ where let is_pending = self.state.tree_state.canonical_block_hash() == block.recovered_block().parent_hash(); self.state.tree_state.insert_executed(block.clone()); + self.metrics + .engine + .executed_blocks + .set(self.state.tree_state.block_count() as f64); if is_pending { debug!(target: "engine::tree", pending=?block_num_hash, "updating pending block"); @@ -2940,6 +2941,7 @@ where self.state.tree_state.insert_executed(block); } } + self.metrics.engine.executed_blocks.set(self.state.tree_state.block_count() as f64); } /// This handles downloaded blocks that are shown to be disconnected from the canonical chain. @@ -3558,6 +3560,7 @@ where self.persistence_state.last_persisted_block.hash, num, ); + self.metrics.engine.executed_blocks.set(self.state.tree_state.block_count() as f64); Ok(()) } } From f631095a8c73cedce96f1e979515b7e5475c450c Mon Sep 17 00:00:00 2001 From: Kanan <93033289+kriss39@users.noreply.github.com> Date: Wed, 16 Sep 2026 16:15:12 +0000 Subject: [PATCH 279/294] fix(rpc): derive the next block base fee in eth_callBundle (#27265) Co-authored-by: Matthias Seitz <19890894+mattsse@users.noreply.github.com> --- Cargo.lock | 2 + crates/rpc/rpc/Cargo.toml | 2 + crates/rpc/rpc/src/eth/bundle.rs | 168 ++++++++++++++++++++++++++++++- 3 files changed, 170 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index fad0103d913..3a2a28b3901 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -10058,6 +10058,7 @@ dependencies = [ "reth-consensus", "reth-consensus-common", "reth-db-api", + "reth-db-common", "reth-engine-primitives", "reth-errors", "reth-ethereum-engine-primitives", @@ -10081,6 +10082,7 @@ dependencies = [ "reth-rpc-eth-types", "reth-rpc-server-types", "reth-storage-api", + "reth-storage-overlay", "reth-tasks", "reth-testing-utils", "reth-transaction-pool", diff --git a/crates/rpc/rpc/Cargo.toml b/crates/rpc/rpc/Cargo.toml index fc729b3fb70..9af5fee3909 100644 --- a/crates/rpc/rpc/Cargo.toml +++ b/crates/rpc/rpc/Cargo.toml @@ -97,6 +97,8 @@ reth-ethereum-primitives.workspace = true reth-transaction-pool = { workspace = true, features = ["test-utils"] } reth-provider = { workspace = true, features = ["test-utils"] } reth-db-api.workspace = true +reth-db-common.workspace = true +reth-storage-overlay.workspace = true rand.workspace = true diff --git a/crates/rpc/rpc/src/eth/bundle.rs b/crates/rpc/rpc/src/eth/bundle.rs index 0327d5c7c07..37f8d1f8afc 100644 --- a/crates/rpc/rpc/src/eth/bundle.rs +++ b/crates/rpc/rpc/src/eth/bundle.rs @@ -12,7 +12,11 @@ use reth_rpc_eth_api::{ helpers::{Call, EthTransactions, LoadPendingBlock}, EthCallBundleApiServer, FromEthApiError, FromEvmError, }; -use reth_rpc_eth_types::{utils::recover_raw_transaction, EthApiError, RpcInvalidTransactionError}; +use reth_rpc_eth_types::{ + utils::recover_raw_transaction, EthApiError, PendingBlockEnv, PendingBlockEnvOrigin, + RpcInvalidTransactionError, +}; +use reth_storage_api::BlockReaderIdExt; use reth_tasks::pool::BlockingTaskGuard; use reth_transaction_pool::{ EthBlobTransactionSidecar, EthPoolTransaction, PoolPooledTx, PoolTransaction, TransactionPool, @@ -94,7 +98,24 @@ where let block_id: alloy_rpc_types_eth::BlockId = state_block_number.into(); // Note: the block number is considered the `parent` block: - let (mut evm_env, at) = self.eth_api().evm_env_at(block_id).await?; + let (mut evm_env, at, parent) = if block_id.is_pending() { + let PendingBlockEnv { evm_env, origin } = self.eth_api().pending_block_env_and_cfg()?; + let at = origin.state_block_id(); + let parent = match origin { + PendingBlockEnvOrigin::ActualPending(block, _) => block.clone_sealed_header(), + PendingBlockEnvOrigin::DerivedFromLatest(header) => header, + }; + (evm_env, at, parent) + } else { + let parent = self + .eth_api() + .provider() + .sealed_header_by_id(block_id) + .map_err(Eth::Error::from_eth_err)? + .ok_or(EthApiError::HeaderNotFound(block_id))?; + let evm_env = self.eth_api().evm_env_for_header(&parent)?; + (evm_env, parent.hash().into(), parent) + }; if let Some(coinbase) = coinbase { evm_env.block_env.inner_mut().beneficiary = coinbase; @@ -133,8 +154,17 @@ where // Apply gas limit: default to call gas limit unless user requests a smaller limit evm_env.block_env.inner_mut().gas_limit = gas_limit.unwrap_or(call_gas_limit); + // The bundle is simulated on top of the state block, so default to the next block's base + // fee. if let Some(base_fee) = base_fee { evm_env.block_env.inner_mut().basefee = base_fee.try_into().unwrap_or(u64::MAX); + } else if let Some(next_base_fee) = self + .eth_api() + .provider() + .chain_spec() + .next_block_base_fee(&parent, evm_env.block_env.timestamp().saturating_to()) + { + evm_env.block_env.inner_mut().basefee = next_base_fee; } let state_block_number = evm_env.block_env.number(); @@ -304,3 +334,137 @@ pub enum EthBundleError { #[error("blob gas usage exceeds the limit of {0} gas per block.")] Eip4844BlobGasExceeded(u64), } + +#[cfg(test)] +mod tests { + use super::*; + use crate::EthApiBuilder; + use alloy_consensus::{transaction::SignerRecoverable, TxEip1559}; + use alloy_eips::{eip1559::INITIAL_BASE_FEE, eip2718::Encodable2718, BlockNumberOrTag}; + use alloy_genesis::{Genesis, GenesisAccount}; + use alloy_primitives::{Address, TxKind}; + use reth_chain_state::ExecutedBlock; + use reth_chainspec::ChainSpecBuilder; + use reth_db_common::init::init_genesis; + use reth_ethereum_primitives::Block as EthBlock; + use reth_evm_ethereum::EthEvmConfig; + use reth_network_api::noop::NoopNetwork; + use reth_primitives_traits::RecoveredBlock; + use reth_provider::{ + providers::BlockchainProvider, test_utils::create_test_provider_factory_with_chain_spec, + }; + use reth_storage_overlay::OverlayManager; + use reth_testing_utils::generators::{self, generate_key, sign_tx_with_key_pair}; + use reth_transaction_pool::test_utils::testing_pool; + + /// The bundle is simulated in the block after the state block, so a transaction that only + /// covers the next block's base fee, but not the state block's, must be accepted. + #[tokio::test(flavor = "multi_thread")] + async fn call_bundle_uses_next_block_base_fee() { + let mut rng = generators::rng(); + let key = generate_key(&mut rng); + // genesis is empty, so the next base fee is 7/8 of the initial base fee + let next_base_fee = INITIAL_BASE_FEE - INITIAL_BASE_FEE / 8; + let tx = sign_tx_with_key_pair( + key, + reth_ethereum_primitives::Transaction::Eip1559(TxEip1559 { + chain_id: 1, + nonce: 0, + gas_limit: 21_000, + max_fee_per_gas: (next_base_fee + 1) as u128, + max_priority_fee_per_gas: (next_base_fee + 1) as u128, + to: TxKind::Call(Address::random()), + ..Default::default() + }), + ); + let sender = tx.recover_signer().unwrap(); + + let genesis = Genesis::default() + .with_gas_limit(30_000_000) + .with_base_fee(Some(INITIAL_BASE_FEE as u128)) + .extend_accounts([( + sender, + GenesisAccount::default().with_balance(U256::from(10u128.pow(18))), + )]); + let chain_spec = + Arc::new(ChainSpecBuilder::mainnet().cancun_activated().genesis(genesis).build()); + let overlay_manager = OverlayManager::default(); + let factory = create_test_provider_factory_with_chain_spec(chain_spec) + .with_overlay_manager(overlay_manager.clone()); + init_genesis(&factory).unwrap(); + let provider = BlockchainProvider::new(factory).unwrap(); + + let eth_api = EthApiBuilder::new( + provider.clone(), + testing_pool(), + NoopNetwork::default(), + EthEvmConfig::new(provider.chain_spec()), + ) + .build(); + let api = EthBundle::new(eth_api, BlockingTaskGuard::new(4)); + + let bundle = EthCallBundle { + txs: vec![tx.encoded_2718().into()], + block_number: 1, + state_block_number: BlockNumberOrTag::Number(0), + ..Default::default() + }; + for state_block_number in + [BlockNumberOrTag::Number(0), BlockNumberOrTag::Latest, BlockNumberOrTag::Pending] + { + for base_fee in [None, Some((next_base_fee - 1) as u128)] { + let response = api + .call_bundle(EthCallBundle { state_block_number, base_fee, ..bundle.clone() }) + .await + .unwrap(); + let expected_tip = + (next_base_fee + 1) as u128 - base_fee.unwrap_or(next_base_fee as u128); + assert_bundle_fees(&response, expected_tip); + } + } + + // An actual pending block is not canonical and has its own base fee. + let chain_spec = provider.chain_spec(); + let mut header = chain_spec.genesis_header().clone(); + header.parent_hash = chain_spec.genesis_hash(); + header.number = 1; + header.timestamp = 12; + header.base_fee_per_gas = Some(next_base_fee); + let pending_block = ExecutedBlock { + recovered_block: Arc::new(RecoveredBlock::new_unhashed( + EthBlock { header, body: Default::default() }, + vec![], + )), + ..Default::default() + }; + overlay_manager.insert_block(pending_block.clone()); + provider.canonical_in_memory_state().set_pending_block(pending_block); + let pending_next_base_fee = next_base_fee - next_base_fee / 8; + for base_fee in [None, Some((pending_next_base_fee - 1) as u128)] { + let response = api + .call_bundle(EthCallBundle { + block_number: 2, + state_block_number: BlockNumberOrTag::Pending, + base_fee, + ..bundle.clone() + }) + .await + .unwrap(); + let expected_tip = + (next_base_fee + 1) as u128 - base_fee.unwrap_or(pending_next_base_fee as u128); + assert_bundle_fees(&response, expected_tip); + } + } + + fn assert_bundle_fees(response: &EthCallBundleResponse, expected_tip: u128) { + assert_eq!(response.results.len(), 1); + assert_eq!(response.total_gas_used, 21_000); + let gas_price = U256::from(expected_tip); + let gas_fees = gas_price * U256::from(21_000); + assert_eq!(response.results[0].gas_price, gas_price); + assert_eq!(response.results[0].gas_fees, gas_fees); + assert_eq!(response.gas_fees, gas_fees); + assert_eq!(response.bundle_gas_price, gas_price); + assert_eq!(response.coinbase_diff, gas_fees); + } +} From 423f1736851e401317131d2391b5e0673a7eb5e5 Mon Sep 17 00:00:00 2001 From: Bruce0X <109469941+Bruce039@users.noreply.github.com> Date: Wed, 16 Sep 2026 17:46:12 +0000 Subject: [PATCH 280/294] fix(rpc): simulate mev_simBundle in the block after parentBlock (#27266) Co-authored-by: Matthias Seitz <19890894+mattsse@users.noreply.github.com> --- crates/rpc/rpc/src/eth/sim_bundle.rs | 179 +++++++++++++++++++++++++-- 1 file changed, 166 insertions(+), 13 deletions(-) diff --git a/crates/rpc/rpc/src/eth/sim_bundle.rs b/crates/rpc/rpc/src/eth/sim_bundle.rs index fb4ef5814ca..e1959705f7f 100644 --- a/crates/rpc/rpc/src/eth/sim_bundle.rs +++ b/crates/rpc/rpc/src/eth/sim_bundle.rs @@ -10,6 +10,7 @@ use alloy_rpc_types_mev::{ SimBundleResponse, Validity, }; use jsonrpsee::core::RpcResult; +use reth_errors::RethError; use reth_evm::{ConfigureEvm, Evm}; use reth_primitives_traits::Recovered; use reth_rpc_api::MevSimApiServer; @@ -290,23 +291,41 @@ where let flattened_bundle = self.parse_and_flatten_bundle(&request)?; let block_id = parent_block.unwrap_or(BlockId::Number(BlockNumberOrTag::Latest)); - let (current_block, mut evm_env, current_block_id) = + let (parent, _, parent_block_id) = self.eth_api().evm_env_and_recovered_block_at(block_id).await?; + // The bundle is simulated in the block that builds on the parent, so derive the + // environment of the next block, see + // + let parent_header = parent.clone_sealed_header(); + let mut evm_env = { + let attributes = self + .eth_api() + .pending_env_builder() + .pending_env_attributes(&parent_header, Some(&block_overrides))?; + self.eth_api() + .evm_config() + .next_evm_env(&parent_header, &attributes) + .map_err(RethError::other) + .map_err(EthApiError::from_eth_err)? + }; + let eth_api = self.inner.eth_api.clone(); let sim_response = self .inner .eth_api - .spawn_with_state_at_block(current_block_id, move |_, mut db| { - // Setup environment - let current_block_number = current_block.number(); + .spawn_with_state_at_block(parent_block_id, move |_, mut db| { + // apply overrides before reading the block values they may change + apply_block_overrides(block_overrides, &mut db, evm_env.block_env.inner_mut()); + + let state_block = parent.number(); + let simulated_block_number = evm_env.block_env.number().saturating_to::(); + let simulated_block_timestamp = + evm_env.block_env.timestamp().saturating_to::(); let coinbase = evm_env.block_env.beneficiary(); let basefee = evm_env.block_env.basefee(); - // apply overrides - apply_block_overrides(block_overrides, &mut db, evm_env.block_env.inner_mut()); - let initial_coinbase_balance = DatabaseRef::basic_ref(&db, coinbase) .map_err(EthApiError::from_eth_err)? .map(|acc| acc.balance) @@ -327,8 +346,8 @@ where let max_block_number = item.inclusion.max_block_number().unwrap_or(block_number); - if current_block_number < block_number || - current_block_number > max_block_number + if simulated_block_number < block_number || + simulated_block_number > max_block_number { return Err(EthApiError::InvalidParams( EthSimBundleError::InvalidInclusion.to_string(), @@ -375,9 +394,10 @@ where .map(|inner| { let full_log = Log { inner, - block_hash: Some(current_block.hash()), - block_number: Some(current_block.number()), - block_timestamp: Some(current_block.timestamp()), + // the simulated block has no hash yet + block_hash: None, + block_number: Some(simulated_block_number), + block_timestamp: Some(simulated_block_timestamp), transaction_hash: Some(*item.tx.tx_hash()), transaction_index: Some(tx_index as u64), log_index: Some(log_index), @@ -452,7 +472,7 @@ where Ok(SimBundleResponse { success: true, - state_block: current_block_number, + state_block, error: None, logs: Some(body_logs), gas_used: total_gas_used, @@ -559,6 +579,139 @@ mod tests { use alloy_primitives::Bytes; use alloy_rpc_types_mev::{Inclusion, ProtocolVersion}; + #[tokio::test(flavor = "multi_thread")] + async fn sim_bundle_uses_next_block_env_and_overrides() { + use crate::EthApiBuilder; + use alloy_consensus::{transaction::SignerRecoverable, TxEip1559}; + use alloy_eips::eip2718::Encodable2718; + use alloy_genesis::{Genesis, GenesisAccount}; + use alloy_primitives::{Address, TxKind, B256}; + use alloy_rpc_types_eth::BlockOverrides; + use reth_chainspec::{ChainSpecBuilder, ChainSpecProvider}; + use reth_db_common::init::init_genesis; + use reth_evm_ethereum::EthEvmConfig; + use reth_network_api::noop::NoopNetwork; + use reth_provider::{ + providers::BlockchainProvider, test_utils::create_test_provider_factory_with_chain_spec, + }; + use reth_testing_utils::generators::{self, generate_key, sign_tx_with_key_pair}; + use reth_transaction_pool::test_utils::testing_pool; + + let contract = Address::repeat_byte(0xaa); + let coinbase = Address::repeat_byte(0x11); + let overridden_coinbase = Address::repeat_byte(0x22); + let mut rng = generators::rng(); + let tx = sign_tx_with_key_pair( + generate_key(&mut rng), + reth_ethereum_primitives::Transaction::Eip1559(TxEip1559 { + chain_id: 1, + nonce: 0, + gas_limit: 100_000, + max_fee_per_gas: 2_000_000_000, + max_priority_fee_per_gas: 1_000_000_000, + to: TxKind::Call(contract), + ..Default::default() + }), + ); + let sender = tx.recover_signer().unwrap(); + + // Log NUMBER, TIMESTAMP, BASEFEE and COINBASE as topics, in that order. + let code = Bytes::from_static(&[0x41, 0x48, 0x42, 0x43, 0x5f, 0x5f, 0xa4, 0x00]); + let genesis = Genesis::default() + .with_gas_limit(30_000_000) + .with_timestamp(100) + .with_base_fee(Some(1_000_000_000)) + .with_coinbase(coinbase) + .extend_accounts([ + (sender, GenesisAccount::default().with_balance(U256::from(10u128.pow(18)))), + (contract, GenesisAccount::default().with_code(Some(code))), + ( + overridden_coinbase, + GenesisAccount::default().with_balance(U256::from(10u128.pow(18))), + ), + ]); + let chain_spec = + Arc::new(ChainSpecBuilder::mainnet().cancun_activated().genesis(genesis).build()); + let factory = create_test_provider_factory_with_chain_spec(chain_spec); + init_genesis(&factory).unwrap(); + let provider = BlockchainProvider::new(factory).unwrap(); + + let eth_api = EthApiBuilder::new( + provider.clone(), + testing_pool(), + NoopNetwork::default(), + EthEvmConfig::new(provider.chain_spec()), + ) + .build(); + let api = EthSimBundle::new(eth_api, BlockingTaskGuard::new(4)); + + let bundle = |block| MevSendBundle { + protocol_version: ProtocolVersion::V0_1, + inclusion: Inclusion::at_block(block), + bundle_body: vec![BundleItem::Tx { tx: tx.encoded_2718().into(), can_revert: false }], + validity: None, + privacy: None, + }; + + for (overrides, number, timestamp, base_fee, beneficiary) in [ + (SimBundleOverrides::default(), 1, 112, 875_000_000, coinbase), + ( + SimBundleOverrides { + parent_block: Some(BlockNumberOrTag::Number(0).into()), + block_overrides: BlockOverrides { + number: Some(U256::from(2)), + time: Some(200), + base_fee: Some(U256::from(1_500_000_000)), + coinbase: Some(overridden_coinbase), + ..Default::default() + }, + ..Default::default() + }, + 2, + 200, + 1_500_000_000, + overridden_coinbase, + ), + ] { + let response = api.sim_bundle(bundle(number), overrides.clone()).await.unwrap(); + assert!(response.success); + assert_eq!(response.state_block, 0); + let tip = (2_000_000_000u64 - base_fee).min(1_000_000_000); + assert_eq!(response.profit, U256::from(response.gas_used) * U256::from(tip)); + assert_eq!(response.refundable_value, response.profit); + assert_eq!(response.mev_gas_price, U256::from(tip)); + + let logs = response.logs.unwrap(); + assert_eq!(logs.len(), 1); + let tx_logs = logs[0].tx_logs.as_ref().unwrap(); + assert_eq!(tx_logs.len(), 1); + let log = &tx_logs[0]; + assert_eq!( + log.inner.topics(), + &[ + B256::from(U256::from(number)), + B256::from(U256::from(timestamp)), + B256::from(U256::from(base_fee)), + beneficiary.into_word(), + ] + ); + assert_eq!(log.block_hash, None); + assert_eq!(log.block_number, Some(number)); + assert_eq!(log.block_timestamp, Some(timestamp)); + assert_eq!(log.transaction_hash, Some(*tx.tx_hash())); + assert_eq!(log.transaction_index, Some(0)); + assert_eq!(log.log_index, Some(0)); + + // Nonzero inclusion bounds reach the simulation check instead of failing parsing. + let err = api.sim_bundle(bundle(number + 1), overrides.clone()).await.unwrap_err(); + assert!(err.message().contains("invalid inclusion"), "{err}"); + if number > 1 { + let err = api.sim_bundle(bundle(number - 1), overrides).await.unwrap_err(); + assert!(err.message().contains("invalid inclusion"), "{err}"); + } + } + } + fn create_test_bundle(tx_bytes: Vec) -> MevSendBundle { let body: Vec = tx_bytes.into_iter().map(|tx| BundleItem::Tx { tx, can_revert: false }).collect(); From dfcb10454cabf4ce778e0dedfe79c512b37f2c8c Mon Sep 17 00:00:00 2001 From: Bruce0X <109469941+Bruce039@users.noreply.github.com> Date: Wed, 16 Sep 2026 16:46:32 -0300 Subject: [PATCH 281/294] fix(rpc): return null when sender nonce has no transaction (#27275) Co-authored-by: Matthias Seitz <19890894+mattsse@users.noreply.github.com> --- .../rpc-eth-api/src/helpers/transaction.rs | 51 ++++++------ crates/rpc/rpc/src/eth/core.rs | 80 ++++++++++++++++++- .../storage/provider/src/test_utils/mock.rs | 12 +-- 3 files changed, 111 insertions(+), 32 deletions(-) diff --git a/crates/rpc/rpc-eth-api/src/helpers/transaction.rs b/crates/rpc/rpc-eth-api/src/helpers/transaction.rs index e56fd3e66b9..32f6f044974 100644 --- a/crates/rpc/rpc-eth-api/src/helpers/transaction.rs +++ b/crates/rpc/rpc-eth-api/src/helpers/transaction.rs @@ -434,32 +434,33 @@ pub trait EthTransactions: LoadTransaction { .await?; let block_id = num.into(); - self.recovered_block(block_id) - .await? - .and_then(|block| { - let block_hash = block.hash(); - let block_number = block.number(); - let block_timestamp = block.timestamp(); - let base_fee_per_gas = block.base_fee_per_gas(); - - block - .transactions_with_sender() - .enumerate() - .find(|(_, (signer, tx))| **signer == sender && (*tx).nonce() == nonce) - .map(|(index, (signer, tx))| { - let tx_info = TransactionInfo { - hash: Some(*tx.tx_hash()), - block_hash: Some(block_hash), - block_number: Some(block_number), - block_timestamp: Some(block_timestamp), - base_fee: base_fee_per_gas, - index: Some(index as u64), - }; - Ok(self.converter().fill(tx.clone().with_signer(*signer), tx_info)?) - }) + let Some(block) = self.recovered_block(block_id).await? else { + return Err(EthApiError::HeaderNotFound(block_id).into()) + }; + + let block_hash = block.hash(); + let block_number = block.number(); + let block_timestamp = block.timestamp(); + let base_fee_per_gas = block.base_fee_per_gas(); + + // EIP-7702 authorizations can consume the account's nonce without a transaction + // from that sender, so an existing block may contain no matching transaction. + block + .transactions_with_sender() + .enumerate() + .find(|(_, (signer, tx))| **signer == sender && (*tx).nonce() == nonce) + .map(|(index, (signer, tx))| { + let tx_info = TransactionInfo { + hash: Some(*tx.tx_hash()), + block_hash: Some(block_hash), + block_number: Some(block_number), + block_timestamp: Some(block_timestamp), + base_fee: base_fee_per_gas, + index: Some(index as u64), + }; + Ok(self.converter().fill(tx.clone().with_signer(*signer), tx_info)?) }) - .ok_or(EthApiError::HeaderNotFound(block_id))? - .map(Some) + .transpose() } } diff --git a/crates/rpc/rpc/src/eth/core.rs b/crates/rpc/rpc/src/eth/core.rs index c6c9533aa05..a721dc3c621 100644 --- a/crates/rpc/rpc/src/eth/core.rs +++ b/crates/rpc/rpc/src/eth/core.rs @@ -551,7 +551,7 @@ mod tests { use crate::{eth::helpers::types::EthRpcConverter, EthApi, EthApiBuilder}; use alloy_consensus::{Block, BlockBody, Header}; use alloy_eips::BlockNumberOrTag; - use alloy_primitives::{Signature, B256, U64}; + use alloy_primitives::{Address, Signature, B256, U256, U64}; use alloy_rpc_types::FeeHistory; use alloy_rpc_types_eth::{Bundle, TransactionRequest}; use jsonrpsee_types::error::INVALID_PARAMS_CODE; @@ -562,7 +562,7 @@ mod tests { use reth_evm_ethereum::EthEvmConfig; use reth_network_api::noop::NoopNetwork; use reth_provider::{ - test_utils::{MockEthProvider, NoopProvider}, + test_utils::{ExtendedAccount, MockEthProvider, NoopProvider}, PruneCheckpointReader, StageCheckpointReader, }; use reth_rpc_eth_api::{node::RpcNodeCoreAdapter, EthApiServer}; @@ -603,6 +603,82 @@ mod tests { .build() } + #[tokio::test] + async fn test_transaction_by_sender_and_nonce_without_sender_transaction_returns_none() { + use reth_rpc_eth_api::helpers::EthTransactions; + + let provider = MockEthProvider::default(); + let sender = Address::random(); + provider.add_account(sender, ExtendedAccount::new(1, U256::ZERO)); + + let block_hash = B256::random(); + provider.add_block( + block_hash, + Block { + header: Header { number: 1, ..Default::default() }, + body: BlockBody::default(), + }, + ); + + let api = build_test_eth_api(provider); + let transaction = + EthTransactions::get_transaction_by_sender_and_nonce(&api, sender, 0, false) + .await + .expect("existing block without a sender transaction should not be a header error"); + + assert!(transaction.is_none()); + } + + #[tokio::test] + async fn test_transaction_by_sender_and_nonce_missing_block_returns_error() { + use reth_rpc_eth_api::helpers::EthTransactions; + use reth_rpc_eth_types::EthApiError; + + let provider = MockEthProvider::default(); + let sender = Address::random(); + provider.add_account(sender, ExtendedAccount::new(1, U256::ZERO)); + provider.add_header(B256::random(), Header { number: 1, ..Default::default() }); + + let api = build_test_eth_api(provider); + let error = EthTransactions::get_transaction_by_sender_and_nonce(&api, sender, 0, false) + .await + .unwrap_err(); + + assert!(matches!(error, EthApiError::HeaderNotFound(id) if id == 1.into())); + } + + #[tokio::test] + async fn test_transaction_by_sender_and_nonce_returns_matching_transaction() { + use reth_primitives_traits::SignerRecoverable; + use reth_rpc_eth_api::helpers::EthTransactions; + + let provider = MockEthProvider::default(); + let tx = TransactionSigned::new_unhashed( + reth_ethereum_primitives::Transaction::Legacy(Default::default()), + Signature::test_signature(), + ); + let sender = tx.recover_signer().unwrap(); + provider.add_account(sender, ExtendedAccount::new(1, U256::ZERO)); + let block = Block { + header: Header { number: 1, ..Default::default() }, + body: BlockBody { transactions: vec![tx], ..Default::default() }, + }; + let block_hash = block.header.hash_slow(); + provider.add_block(block_hash, block); + + let api = build_test_eth_api(provider); + let transaction = + EthTransactions::get_transaction_by_sender_and_nonce(&api, sender, 0, false) + .await + .unwrap() + .expect("matching transaction should be returned"); + + assert_eq!(transaction.block_hash, Some(block_hash)); + assert_eq!(transaction.block_number, Some(1)); + assert_eq!(transaction.transaction_index, Some(0)); + assert_eq!(transaction.inner.signer(), sender); + } + #[tokio::test] async fn test_eth_api_settings() { use reth_rpc_eth_api::{ diff --git a/crates/storage/provider/src/test_utils/mock.rs b/crates/storage/provider/src/test_utils/mock.rs index ae1eb506362..55d54e91954 100644 --- a/crates/storage/provider/src/test_utils/mock.rs +++ b/crates/storage/provider/src/test_utils/mock.rs @@ -947,18 +947,20 @@ impl BlockRe fn recovered_block( &self, - _id: BlockHashOrNumber, + id: BlockHashOrNumber, _transaction_kind: TransactionVariant, ) -> ProviderResult>> { - Ok(None) + self.block(id)? + .map(|block| block.try_into_recovered().map_err(ProviderError::other)) + .transpose() } fn sealed_block_with_senders( &self, - _id: BlockHashOrNumber, - _transaction_kind: TransactionVariant, + id: BlockHashOrNumber, + transaction_kind: TransactionVariant, ) -> ProviderResult>> { - Ok(None) + self.recovered_block(id, transaction_kind) } fn block_range(&self, range: RangeInclusive) -> ProviderResult> { From 73a3a00862a8f14f89e30da8de001456f18cfae0 Mon Sep 17 00:00:00 2001 From: Emma Jamieson-Hoare Date: Thu, 17 Sep 2026 13:49:52 +0100 Subject: [PATCH 282/294] chore: bump versions --- Cargo.lock | 230 +++++++++++++++++++-------------------- Cargo.toml | 2 +- docs/vocs/vocs.config.ts | 2 +- 3 files changed, 117 insertions(+), 117 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3a2a28b3901..9a19ff74073 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3344,7 +3344,7 @@ dependencies = [ [[package]] name = "ef-test-runner" -version = "2.5.2" +version = "2.6.0" dependencies = [ "clap", "ef-tests", @@ -3352,7 +3352,7 @@ dependencies = [ [[package]] name = "ef-tests" -version = "2.5.2" +version = "2.6.0" dependencies = [ "alloy-consensus", "alloy-eips", @@ -3805,7 +3805,7 @@ dependencies = [ [[package]] name = "example-full-contract-state" -version = "2.5.2" +version = "2.6.0" dependencies = [ "alloy-primitives", "eyre", @@ -3923,7 +3923,7 @@ dependencies = [ [[package]] name = "exex-subscription" -version = "2.5.2" +version = "2.6.0" dependencies = [ "alloy-primitives", "clap", @@ -7801,7 +7801,7 @@ checksum = "1e061d1b48cb8d38042de4ae0a7a6401009d6143dc80d2e2d6f31f0bdd6470c7" [[package]] name = "reth" -version = "2.5.2" +version = "2.6.0" dependencies = [ "alloy-node-bindings", "alloy-primitives", @@ -7855,7 +7855,7 @@ dependencies = [ [[package]] name = "reth-basic-payload-builder" -version = "2.5.2" +version = "2.6.0" dependencies = [ "alloy-consensus", "alloy-eips", @@ -7881,7 +7881,7 @@ dependencies = [ [[package]] name = "reth-bb" -version = "2.5.2" +version = "2.6.0" dependencies = [ "alloy-consensus", "alloy-eips", @@ -7928,7 +7928,7 @@ dependencies = [ [[package]] name = "reth-chain-state" -version = "2.5.2" +version = "2.6.0" dependencies = [ "alloy-consensus", "alloy-eips", @@ -7959,7 +7959,7 @@ dependencies = [ [[package]] name = "reth-chainspec" -version = "2.5.2" +version = "2.6.0" dependencies = [ "alloy-chains", "alloy-consensus", @@ -7979,7 +7979,7 @@ dependencies = [ [[package]] name = "reth-cli" -version = "2.5.2" +version = "2.6.0" dependencies = [ "alloy-genesis", "clap", @@ -7991,7 +7991,7 @@ dependencies = [ [[package]] name = "reth-cli-commands" -version = "2.5.2" +version = "2.6.0" dependencies = [ "alloy-chains", "alloy-consensus", @@ -8082,7 +8082,7 @@ dependencies = [ [[package]] name = "reth-cli-runner" -version = "2.5.2" +version = "2.6.0" dependencies = [ "reth-tasks", "tokio", @@ -8091,7 +8091,7 @@ dependencies = [ [[package]] name = "reth-cli-util" -version = "2.5.2" +version = "2.6.0" dependencies = [ "alloy-eips", "alloy-primitives", @@ -8145,7 +8145,7 @@ dependencies = [ [[package]] name = "reth-config" -version = "2.5.2" +version = "2.6.0" dependencies = [ "alloy-primitives", "eyre", @@ -8163,7 +8163,7 @@ dependencies = [ [[package]] name = "reth-consensus" -version = "2.5.2" +version = "2.6.0" dependencies = [ "alloy-consensus", "alloy-eip7928", @@ -8176,7 +8176,7 @@ dependencies = [ [[package]] name = "reth-consensus-common" -version = "2.5.2" +version = "2.6.0" dependencies = [ "alloy-consensus", "alloy-eips", @@ -8190,7 +8190,7 @@ dependencies = [ [[package]] name = "reth-consensus-debug-client" -version = "2.5.2" +version = "2.6.0" dependencies = [ "alloy-consensus", "alloy-eips", @@ -8214,7 +8214,7 @@ dependencies = [ [[package]] name = "reth-db" -version = "2.5.2" +version = "2.6.0" dependencies = [ "alloy-consensus", "alloy-primitives", @@ -8249,7 +8249,7 @@ dependencies = [ [[package]] name = "reth-db-api" -version = "2.5.2" +version = "2.6.0" dependencies = [ "alloy-consensus", "alloy-eips", @@ -8278,7 +8278,7 @@ dependencies = [ [[package]] name = "reth-db-common" -version = "2.5.2" +version = "2.6.0" dependencies = [ "alloy-consensus", "alloy-genesis", @@ -8309,7 +8309,7 @@ dependencies = [ [[package]] name = "reth-db-models" -version = "2.5.2" +version = "2.6.0" dependencies = [ "alloy-eips", "alloy-primitives", @@ -8325,7 +8325,7 @@ dependencies = [ [[package]] name = "reth-discv4" -version = "2.5.2" +version = "2.6.0" dependencies = [ "alloy-primitives", "alloy-rlp", @@ -8352,7 +8352,7 @@ dependencies = [ [[package]] name = "reth-discv5" -version = "2.5.2" +version = "2.6.0" dependencies = [ "alloy-primitives", "alloy-rlp", @@ -8377,7 +8377,7 @@ dependencies = [ [[package]] name = "reth-dns-discovery" -version = "2.5.2" +version = "2.6.0" dependencies = [ "alloy-chains", "alloy-primitives", @@ -8405,7 +8405,7 @@ dependencies = [ [[package]] name = "reth-downloaders" -version = "2.5.2" +version = "2.6.0" dependencies = [ "alloy-consensus", "alloy-eips", @@ -8445,7 +8445,7 @@ dependencies = [ [[package]] name = "reth-e2e-test-utils" -version = "2.5.2" +version = "2.6.0" dependencies = [ "alloy-consensus", "alloy-eips", @@ -8503,7 +8503,7 @@ dependencies = [ [[package]] name = "reth-ecies" -version = "2.5.2" +version = "2.6.0" dependencies = [ "aes", "alloy-primitives", @@ -8529,7 +8529,7 @@ dependencies = [ [[package]] name = "reth-engine-local" -version = "2.5.2" +version = "2.6.0" dependencies = [ "alloy-consensus", "alloy-primitives", @@ -8551,7 +8551,7 @@ dependencies = [ [[package]] name = "reth-engine-primitives" -version = "2.5.2" +version = "2.6.0" dependencies = [ "alloy-consensus", "alloy-eips", @@ -8578,7 +8578,7 @@ dependencies = [ [[package]] name = "reth-engine-tree" -version = "2.5.2" +version = "2.6.0" dependencies = [ "alloy-consensus", "alloy-eip7928", @@ -8648,7 +8648,7 @@ dependencies = [ [[package]] name = "reth-engine-util" -version = "2.5.2" +version = "2.6.0" dependencies = [ "alloy-consensus", "alloy-primitives", @@ -8677,7 +8677,7 @@ dependencies = [ [[package]] name = "reth-era" -version = "2.5.2" +version = "2.6.0" dependencies = [ "alloy-consensus", "alloy-eips", @@ -8702,7 +8702,7 @@ dependencies = [ [[package]] name = "reth-era-downloader" -version = "2.5.2" +version = "2.6.0" dependencies = [ "alloy-primitives", "bytes", @@ -8720,7 +8720,7 @@ dependencies = [ [[package]] name = "reth-era-utils" -version = "2.5.2" +version = "2.6.0" dependencies = [ "alloy-consensus", "alloy-primitives", @@ -8751,7 +8751,7 @@ dependencies = [ [[package]] name = "reth-errors" -version = "2.5.2" +version = "2.6.0" dependencies = [ "reth-consensus", "reth-execution-errors", @@ -8761,7 +8761,7 @@ dependencies = [ [[package]] name = "reth-eth-wire" -version = "2.5.2" +version = "2.6.0" dependencies = [ "alloy-chains", "alloy-consensus", @@ -8799,7 +8799,7 @@ dependencies = [ [[package]] name = "reth-eth-wire-types" -version = "2.5.2" +version = "2.6.0" dependencies = [ "alloy-chains", "alloy-consensus", @@ -8827,7 +8827,7 @@ dependencies = [ [[package]] name = "reth-ethereum" -version = "2.5.2" +version = "2.6.0" dependencies = [ "alloy-rpc-types-engine", "alloy-rpc-types-eth", @@ -8867,7 +8867,7 @@ dependencies = [ [[package]] name = "reth-ethereum-cli" -version = "2.5.2" +version = "2.6.0" dependencies = [ "clap", "eyre", @@ -8890,7 +8890,7 @@ dependencies = [ [[package]] name = "reth-ethereum-consensus" -version = "2.5.2" +version = "2.6.0" dependencies = [ "alloy-consensus", "alloy-eips", @@ -8906,7 +8906,7 @@ dependencies = [ [[package]] name = "reth-ethereum-engine-primitives" -version = "2.5.2" +version = "2.6.0" dependencies = [ "alloy-eips", "alloy-primitives", @@ -8922,7 +8922,7 @@ dependencies = [ [[package]] name = "reth-ethereum-forks" -version = "2.5.2" +version = "2.6.0" dependencies = [ "alloy-eip2124", "alloy-hardforks 0.4.8", @@ -8935,7 +8935,7 @@ dependencies = [ [[package]] name = "reth-ethereum-payload-builder" -version = "2.5.2" +version = "2.6.0" dependencies = [ "alloy-consensus", "alloy-eips", @@ -8964,7 +8964,7 @@ dependencies = [ [[package]] name = "reth-ethereum-primitives" -version = "2.5.2" +version = "2.6.0" dependencies = [ "alloy-consensus", "alloy-eips", @@ -8985,7 +8985,7 @@ dependencies = [ [[package]] name = "reth-etl" -version = "2.5.2" +version = "2.6.0" dependencies = [ "alloy-primitives", "rayon", @@ -8995,7 +8995,7 @@ dependencies = [ [[package]] name = "reth-evm" -version = "2.5.2" +version = "2.6.0" dependencies = [ "alloy-consensus", "alloy-eip7928", @@ -9021,7 +9021,7 @@ dependencies = [ [[package]] name = "reth-evm-ethereum" -version = "2.5.2" +version = "2.6.0" dependencies = [ "alloy-consensus", "alloy-eips", @@ -9046,7 +9046,7 @@ dependencies = [ [[package]] name = "reth-execution-cache" -version = "2.5.2" +version = "2.6.0" dependencies = [ "alloy-primitives", "fixed-cache", @@ -9064,7 +9064,7 @@ dependencies = [ [[package]] name = "reth-execution-errors" -version = "2.5.2" +version = "2.6.0" dependencies = [ "alloy-evm", "alloy-primitives", @@ -9076,7 +9076,7 @@ dependencies = [ [[package]] name = "reth-execution-types" -version = "2.5.2" +version = "2.6.0" dependencies = [ "alloy-consensus", "alloy-eips", @@ -9097,7 +9097,7 @@ dependencies = [ [[package]] name = "reth-exex" -version = "2.5.2" +version = "2.6.0" dependencies = [ "alloy-consensus", "alloy-eips", @@ -9142,7 +9142,7 @@ dependencies = [ [[package]] name = "reth-exex-test-utils" -version = "2.5.2" +version = "2.6.0" dependencies = [ "alloy-eips", "eyre", @@ -9173,7 +9173,7 @@ dependencies = [ [[package]] name = "reth-exex-types" -version = "2.5.2" +version = "2.6.0" dependencies = [ "alloy-eips", "alloy-primitives", @@ -9190,7 +9190,7 @@ dependencies = [ [[package]] name = "reth-fs-util" -version = "2.5.2" +version = "2.6.0" dependencies = [ "serde", "serde_json", @@ -9199,7 +9199,7 @@ dependencies = [ [[package]] name = "reth-invalid-block-hooks" -version = "2.5.2" +version = "2.6.0" dependencies = [ "alloy-consensus", "alloy-eips", @@ -9230,7 +9230,7 @@ dependencies = [ [[package]] name = "reth-ipc" -version = "2.5.2" +version = "2.6.0" dependencies = [ "bytes", "futures", @@ -9252,7 +9252,7 @@ dependencies = [ [[package]] name = "reth-libmdbx" -version = "2.5.2" +version = "2.6.0" dependencies = [ "bitflags 2.13.1", "byteorder", @@ -9269,7 +9269,7 @@ dependencies = [ [[package]] name = "reth-mdbx-sys" -version = "2.5.2" +version = "2.6.0" dependencies = [ "bindgen", "cc", @@ -9277,7 +9277,7 @@ dependencies = [ [[package]] name = "reth-metrics" -version = "2.5.2" +version = "2.6.0" dependencies = [ "futures", "libc", @@ -9290,7 +9290,7 @@ dependencies = [ [[package]] name = "reth-net-banlist" -version = "2.5.2" +version = "2.6.0" dependencies = [ "alloy-primitives", "ipnet", @@ -9298,7 +9298,7 @@ dependencies = [ [[package]] name = "reth-net-nat" -version = "2.5.2" +version = "2.6.0" dependencies = [ "futures-util", "if-addrs", @@ -9312,7 +9312,7 @@ dependencies = [ [[package]] name = "reth-network" -version = "2.5.2" +version = "2.6.0" dependencies = [ "alloy-consensus", "alloy-eip7928", @@ -9382,7 +9382,7 @@ dependencies = [ [[package]] name = "reth-network-api" -version = "2.5.2" +version = "2.6.0" dependencies = [ "alloy-consensus", "alloy-primitives", @@ -9406,7 +9406,7 @@ dependencies = [ [[package]] name = "reth-network-p2p" -version = "2.5.2" +version = "2.6.0" dependencies = [ "alloy-consensus", "alloy-eip7928", @@ -9430,7 +9430,7 @@ dependencies = [ [[package]] name = "reth-network-peers" -version = "2.5.2" +version = "2.6.0" dependencies = [ "alloy-primitives", "alloy-rlp", @@ -9447,7 +9447,7 @@ dependencies = [ [[package]] name = "reth-network-types" -version = "2.5.2" +version = "2.6.0" dependencies = [ "alloy-eip2124", "humantime-serde", @@ -9460,7 +9460,7 @@ dependencies = [ [[package]] name = "reth-nippy-jar" -version = "2.5.2" +version = "2.6.0" dependencies = [ "anyhow", "bincode", @@ -9479,7 +9479,7 @@ dependencies = [ [[package]] name = "reth-node-api" -version = "2.5.2" +version = "2.6.0" dependencies = [ "alloy-rpc-types-engine", "eyre", @@ -9502,7 +9502,7 @@ dependencies = [ [[package]] name = "reth-node-builder" -version = "2.5.2" +version = "2.6.0" dependencies = [ "alloy-consensus", "alloy-eips", @@ -9573,7 +9573,7 @@ dependencies = [ [[package]] name = "reth-node-core" -version = "2.5.2" +version = "2.6.0" dependencies = [ "alloy-consensus", "alloy-eips", @@ -9629,7 +9629,7 @@ dependencies = [ [[package]] name = "reth-node-ethereum" -version = "2.5.2" +version = "2.6.0" dependencies = [ "alloy-consensus", "alloy-contract", @@ -9700,7 +9700,7 @@ dependencies = [ [[package]] name = "reth-node-ethstats" -version = "2.5.2" +version = "2.6.0" dependencies = [ "alloy-consensus", "alloy-primitives", @@ -9723,7 +9723,7 @@ dependencies = [ [[package]] name = "reth-node-events" -version = "2.5.2" +version = "2.6.0" dependencies = [ "alloy-consensus", "alloy-eips", @@ -9746,7 +9746,7 @@ dependencies = [ [[package]] name = "reth-node-metrics" -version = "2.5.2" +version = "2.6.0" dependencies = [ "bytes", "eyre", @@ -9776,7 +9776,7 @@ dependencies = [ [[package]] name = "reth-node-types" -version = "2.5.2" +version = "2.6.0" dependencies = [ "reth-chainspec", "reth-db-api", @@ -9787,7 +9787,7 @@ dependencies = [ [[package]] name = "reth-payload-builder" -version = "2.5.2" +version = "2.6.0" dependencies = [ "alloy-consensus", "alloy-primitives", @@ -9810,7 +9810,7 @@ dependencies = [ [[package]] name = "reth-payload-builder-primitives" -version = "2.5.2" +version = "2.6.0" dependencies = [ "pin-project", "reth-payload-primitives", @@ -9821,7 +9821,7 @@ dependencies = [ [[package]] name = "reth-payload-primitives" -version = "2.5.2" +version = "2.6.0" dependencies = [ "alloy-consensus", "alloy-eips", @@ -9845,7 +9845,7 @@ dependencies = [ [[package]] name = "reth-payload-util" -version = "2.5.2" +version = "2.6.0" dependencies = [ "alloy-consensus", "alloy-primitives", @@ -9854,7 +9854,7 @@ dependencies = [ [[package]] name = "reth-payload-validator" -version = "2.5.2" +version = "2.6.0" dependencies = [ "alloy-consensus", "alloy-rpc-types-engine", @@ -9897,7 +9897,7 @@ dependencies = [ [[package]] name = "reth-provider" -version = "2.5.2" +version = "2.6.0" dependencies = [ "alloy-consensus", "alloy-eip7928", @@ -9952,7 +9952,7 @@ dependencies = [ [[package]] name = "reth-prune" -version = "2.5.2" +version = "2.6.0" dependencies = [ "alloy-consensus", "alloy-primitives", @@ -9983,7 +9983,7 @@ dependencies = [ [[package]] name = "reth-prune-types" -version = "2.5.2" +version = "2.6.0" dependencies = [ "alloy-primitives", "arbitrary", @@ -10003,7 +10003,7 @@ dependencies = [ [[package]] name = "reth-revm" -version = "2.5.2" +version = "2.6.0" dependencies = [ "alloy-consensus", "alloy-primitives", @@ -10019,7 +10019,7 @@ dependencies = [ [[package]] name = "reth-rpc" -version = "2.5.2" +version = "2.6.0" dependencies = [ "alloy-consensus", "alloy-dyn-abi", @@ -10101,7 +10101,7 @@ dependencies = [ [[package]] name = "reth-rpc-api" -version = "2.5.2" +version = "2.6.0" dependencies = [ "alloy-eips", "alloy-genesis", @@ -10131,7 +10131,7 @@ dependencies = [ [[package]] name = "reth-rpc-builder" -version = "2.5.2" +version = "2.6.0" dependencies = [ "alloy-eips", "alloy-network", @@ -10189,7 +10189,7 @@ dependencies = [ [[package]] name = "reth-rpc-convert" -version = "2.5.2" +version = "2.6.0" dependencies = [ "alloy-consensus", "alloy-evm", @@ -10209,7 +10209,7 @@ dependencies = [ [[package]] name = "reth-rpc-e2e-tests" -version = "2.5.2" +version = "2.6.0" dependencies = [ "alloy-genesis", "alloy-rpc-types-engine", @@ -10229,7 +10229,7 @@ dependencies = [ [[package]] name = "reth-rpc-engine-api" -version = "2.5.2" +version = "2.6.0" dependencies = [ "alloy-eips", "alloy-primitives", @@ -10265,7 +10265,7 @@ dependencies = [ [[package]] name = "reth-rpc-eth-api" -version = "2.5.2" +version = "2.6.0" dependencies = [ "alloy-consensus", "alloy-dyn-abi", @@ -10311,7 +10311,7 @@ dependencies = [ [[package]] name = "reth-rpc-eth-types" -version = "2.5.2" +version = "2.6.0" dependencies = [ "alloy-chains", "alloy-consensus", @@ -10362,7 +10362,7 @@ dependencies = [ [[package]] name = "reth-rpc-layer" -version = "2.5.2" +version = "2.6.0" dependencies = [ "alloy-rpc-types-engine", "brotli", @@ -10382,7 +10382,7 @@ dependencies = [ [[package]] name = "reth-rpc-server-types" -version = "2.5.2" +version = "2.6.0" dependencies = [ "alloy-primitives", "alloy-rpc-types-engine", @@ -10411,7 +10411,7 @@ dependencies = [ [[package]] name = "reth-snap-sync" -version = "2.5.2" +version = "2.6.0" dependencies = [ "alloy-consensus", "alloy-eip7928", @@ -10444,7 +10444,7 @@ dependencies = [ [[package]] name = "reth-stages" -version = "2.5.2" +version = "2.6.0" dependencies = [ "alloy-consensus", "alloy-eip7928", @@ -10507,7 +10507,7 @@ dependencies = [ [[package]] name = "reth-stages-api" -version = "2.5.2" +version = "2.6.0" dependencies = [ "alloy-eips", "alloy-primitives", @@ -10541,7 +10541,7 @@ dependencies = [ [[package]] name = "reth-stages-types" -version = "2.5.2" +version = "2.6.0" dependencies = [ "alloy-primitives", "arbitrary", @@ -10557,7 +10557,7 @@ dependencies = [ [[package]] name = "reth-static-file" -version = "2.5.2" +version = "2.6.0" dependencies = [ "alloy-primitives", "assert_matches", @@ -10580,7 +10580,7 @@ dependencies = [ [[package]] name = "reth-static-file-types" -version = "2.5.2" +version = "2.6.0" dependencies = [ "alloy-primitives", "clap", @@ -10598,7 +10598,7 @@ dependencies = [ [[package]] name = "reth-storage-api" -version = "2.5.2" +version = "2.6.0" dependencies = [ "alloy-consensus", "alloy-eip7928", @@ -10625,7 +10625,7 @@ dependencies = [ [[package]] name = "reth-storage-errors" -version = "2.5.2" +version = "2.6.0" dependencies = [ "alloy-eips", "alloy-primitives", @@ -10641,7 +10641,7 @@ dependencies = [ [[package]] name = "reth-storage-overlay" -version = "2.5.2" +version = "2.6.0" dependencies = [ "alloy-consensus", "alloy-eips", @@ -10671,7 +10671,7 @@ dependencies = [ [[package]] name = "reth-storage-rpc-provider" -version = "2.5.2" +version = "2.6.0" dependencies = [ "alloy-consensus", "alloy-eips", @@ -10700,7 +10700,7 @@ dependencies = [ [[package]] name = "reth-tasks" -version = "2.5.2" +version = "2.6.0" dependencies = [ "crossbeam-utils", "dashmap", @@ -10720,7 +10720,7 @@ dependencies = [ [[package]] name = "reth-testing-utils" -version = "2.5.2" +version = "2.6.0" dependencies = [ "alloy-consensus", "alloy-eips", @@ -10736,7 +10736,7 @@ dependencies = [ [[package]] name = "reth-tokio-util" -version = "2.5.2" +version = "2.6.0" dependencies = [ "tokio", "tokio-stream", @@ -10745,7 +10745,7 @@ dependencies = [ [[package]] name = "reth-tracing" -version = "2.5.2" +version = "2.6.0" dependencies = [ "clap", "eyre", @@ -10764,7 +10764,7 @@ dependencies = [ [[package]] name = "reth-tracing-otlp" -version = "2.5.2" +version = "2.6.0" dependencies = [ "base64 0.22.1", "clap", @@ -10782,7 +10782,7 @@ dependencies = [ [[package]] name = "reth-transaction-pool" -version = "2.5.2" +version = "2.6.0" dependencies = [ "alloy-consensus", "alloy-eips", @@ -10833,7 +10833,7 @@ dependencies = [ [[package]] name = "reth-trie" -version = "2.5.2" +version = "2.6.0" dependencies = [ "alloy-consensus", "alloy-eips", @@ -10865,7 +10865,7 @@ dependencies = [ [[package]] name = "reth-trie-common" -version = "2.5.2" +version = "2.6.0" dependencies = [ "alloy-consensus", "alloy-eips", @@ -10898,7 +10898,7 @@ dependencies = [ [[package]] name = "reth-trie-db" -version = "2.5.2" +version = "2.6.0" dependencies = [ "alloy-consensus", "alloy-primitives", @@ -10925,7 +10925,7 @@ dependencies = [ [[package]] name = "reth-trie-parallel" -version = "2.5.2" +version = "2.6.0" dependencies = [ "alloy-evm", "alloy-primitives", @@ -10952,7 +10952,7 @@ dependencies = [ [[package]] name = "reth-trie-sparse" -version = "2.5.2" +version = "2.6.0" dependencies = [ "alloy-primitives", "alloy-rlp", diff --git a/Cargo.toml b/Cargo.toml index 53965b57963..1b1be1a7134 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,5 +1,5 @@ [workspace.package] -version = "2.5.2" +version = "2.6.0" edition = "2024" rust-version = "1.95" license = "MIT OR Apache-2.0" diff --git a/docs/vocs/vocs.config.ts b/docs/vocs/vocs.config.ts index 6aa891ac973..b3eddc738f1 100644 --- a/docs/vocs/vocs.config.ts +++ b/docs/vocs/vocs.config.ts @@ -23,7 +23,7 @@ export default defineConfig({ { text: 'Rustdocs', link: '/docs' }, { text: 'GitHub', link: 'https://github.com/paradigmxyz/reth' }, { - text: 'v2.5.2', + text: 'v2.6.0', items: [ { text: 'Releases', From e04d895b7fbc2a3ac1672a38db9ea963469a11b5 Mon Sep 17 00:00:00 2001 From: Brian Picciano Date: Thu, 17 Sep 2026 12:58:50 +0000 Subject: [PATCH 283/294] perf(trie): skip historical trie changesets with complete overlays (#27129) (cherry picked from commit 835e42864d2f5cf375ebeacca3d279c3838971dd) --- .../src/providers/blockchain_provider.rs | 153 +++++++++++++ .../provider/src/providers/state/latest.rs | 25 ++- crates/storage/provider/src/writer/mod.rs | 12 +- crates/storage/storage-overlay/src/builder.rs | 136 +++++++++--- .../storage-overlay/src/changeset_cache.rs | 4 +- crates/storage/storage-overlay/src/manager.rs | 6 +- .../storage/storage-overlay/src/provider.rs | 208 ++++++++++-------- crates/trie/common/src/input.rs | 7 + crates/trie/db/src/proof.rs | 22 +- crates/trie/db/src/storage.rs | 27 ++- 10 files changed, 440 insertions(+), 160 deletions(-) diff --git a/crates/storage/provider/src/providers/blockchain_provider.rs b/crates/storage/provider/src/providers/blockchain_provider.rs index ac11538c2fd..318a374421a 100644 --- a/crates/storage/provider/src/providers/blockchain_provider.rs +++ b/crates/storage/provider/src/providers/blockchain_provider.rs @@ -1080,6 +1080,159 @@ mod tests { sync::Arc, }; + #[test] + fn historical_proofs_complete_masked_trie_rows() -> eyre::Result<()> { + use crate::{StaticFileProviderFactory, StaticFileSegment, StaticFileWriter}; + use reth_storage_api::{StorageSettings, StorageSettingsCache, TrieWriter}; + use reth_trie::{test_utils::TrieTestHarness, MultiProofTargetsV2}; + use revm::state::AccountInfo; + + let address = Address::with_last_byte(1); + let other_address = Address::with_last_byte(2); + let hashed_address = keccak256(address); + let account = Account { balance: U256::from(1), ..Default::default() }; + // Branches 00, 01 and 02 share the cached row at 0. The first two change + // in different blocks; the third keeps proof-v2's branch-collapse path inactive. + let slots: Vec = (0..=2u8) + .flat_map(|prefix| { + (0..u64::MAX) + .map(|slot| B256::from(U256::from(slot))) + .filter(move |slot| keccak256(slot)[0] == prefix) + .take(2) + }) + .collect(); + let mut storage: BTreeMap<_, _> = slots.iter().map(|&slot| (slot, U256::from(1))).collect(); + let mut trie = TrieTestHarness::new( + storage.iter().map(|(slot, value)| (keccak256(slot), *value)).collect(), + ); + let mut blocks = Vec::::new(); + let mut storage_roots = Vec::new(); + for number in 0..=2 { + let mut output = (*ExecutedBlock::::default() + .execution_output) + .clone(); + let (state, storage_updates) = if number == 0 { + ( + HashedPostState::default() + .with_accounts([ + (hashed_address, Some(account)), + (keccak256(other_address), Some(account)), + ]) + .with_storages([( + hashed_address, + HashedStorage::from_iter(trie.storage().iter().map(|(k, v)| (*k, *v))), + )]), + trie.storage_trie_updates().clone(), + ) + } else { + let slot = slots[(number as usize - 1) * 2]; + let value = U256::from(number + 1); + let changes = BTreeMap::from([(keccak256(slot), value)]); + let (_, updates) = trie.get_root_with_updates(&changes); + trie.apply_changeset(changes); + storage.insert(slot, value); + let info = AccountInfo::from_balance(account.balance); + output.state = BundleState::builder(number..=number) + .state_original_account_info(address, info.clone()) + .state_present_account_info(address, info) + .state_storage( + address, + std::iter::once((U256::from_be_bytes(slot.0), (U256::from(1), value))) + .collect(), + ) + .revert_storage( + number, + address, + vec![(U256::from_be_bytes(slot.0), U256::from(1))], + ) + .build(); + ( + HashedPostState::from_hashed_storage( + hashed_address, + HashedStorage::from_iter([(keccak256(slot), value)]), + ), + updates, + ) + }; + let mut updates = TrieUpdates::default(); + updates.storage_tries.insert(hashed_address, storage_updates); + let state_root = reth_trie::test_utils::state_root([ + (address, (account, storage.clone())), + (other_address, (account, BTreeMap::new())), + ]); + storage_roots.push(reth_trie::test_utils::storage_root(storage.clone())); + let block = Block { + header: alloy_consensus::Header { + number, + parent_hash: blocks + .last() + .map(|b| b.recovered_block().hash()) + .unwrap_or_default(), + state_root, + ..Default::default() + }, + ..Default::default() + }; + blocks.push(ExecutedBlock::new( + Arc::new(RecoveredBlock::new_unhashed(block, vec![])), + Arc::new(output), + ComputedTrieData::new( + Arc::new(state.into_sorted()), + Arc::new(updates.into_sorted()), + ), + )); + } + + let factory = create_test_provider_factory(); + factory.set_storage_settings_cache(StorageSettings::v2()); + let writer = factory.provider_rw()?; + writer.insert_block(blocks[0].recovered_block())?; + writer.write_hashed_state(blocks[0].hashed_state_ref())?; + writer.write_trie_updates_sorted(blocks[0].trie_updates_ref())?; + writer.commit()?; + let static_files = factory.static_file_provider(); + static_files.get_writer(0, StaticFileSegment::Receipts)?.increment_block(0)?; + static_files + .get_writer(0, StaticFileSegment::AccountChangeSets)? + .append_account_changeset(vec![], 0)?; + static_files + .get_writer(0, StaticFileSegment::StorageChangeSets)? + .append_storage_changeset(vec![], 0)?; + static_files.commit()?; + let writer = factory.provider_rw()?; + writer.save_blocks(&SaveBlocksInput::new(blocks[1..].to_vec(), 0, 0, 2, 1))?; + writer.commit()?; + factory.overlay_manager().insert_block(blocks[2].clone()); + let provider = BlockchainProvider::new(factory)?; + + for number in [1, 2] { + let state = provider.history_by_block_hash(blocks[number].recovered_block().hash())?; + let root = blocks[number].recovered_block().state_root; + state + .multiproof_v2( + Default::default(), + MultiProofTargetsV2 { + account_targets: vec![keccak256(other_address).into()], + ..Default::default() + }, + )? + .account_proof(other_address, &[])? + .verify(root)?; + assert_eq!(state.state_root(HashedPostState::default())?, root); + assert_eq!( + state.storage_root(address, HashedStorage::default())?, + storage_roots[number] + ); + state + .storage_proof(address, slots[2], HashedStorage::default())? + .verify(storage_roots[number])?; + let proof = state.storage_multiproof(address, &[slots[2]], HashedStorage::default())?; + assert_eq!(proof.root, storage_roots[number]); + proof.storage_proof(slots[2])?.verify(storage_roots[number])?; + } + Ok(()) + } + const TEST_BLOCKS_COUNT: usize = 5; const TEST_TRANSACTIONS_COUNT: u8 = 4; diff --git a/crates/storage/provider/src/providers/state/latest.rs b/crates/storage/provider/src/providers/state/latest.rs index f85b371ea72..788c60e9bd8 100644 --- a/crates/storage/provider/src/providers/state/latest.rs +++ b/crates/storage/provider/src/providers/state/latest.rs @@ -153,7 +153,14 @@ impl StorageRootProvider hashed_storage: HashedStorage, ) -> ProviderResult { reth_trie_db::with_adapter!(self.0, |A| { - >::overlay_root(self.tx(), address, hashed_storage) + let input = TrieInputSorted::from_state( + HashedPostState::from_hashed_storage( + alloy_primitives::keccak256(address), + hashed_storage, + ) + .into_sorted(), + ); + >::overlay_root(self.tx(), address, input) .map_err(|err| ProviderError::Database(err.into())) }) } @@ -182,13 +189,15 @@ impl StorageRootProvider hashed_storage: HashedStorage, ) -> ProviderResult { reth_trie_db::with_adapter!(self.0, |A| { - >::overlay_storage_multiproof( - self.tx(), - address, - slots, - hashed_storage, - ) - .map_err(ProviderError::from) + let input = TrieInputSorted::from_state( + HashedPostState::from_hashed_storage( + alloy_primitives::keccak256(address), + hashed_storage, + ) + .into_sorted(), + ); + >::overlay_storage_multiproof(self.tx(), address, slots, input) + .map_err(ProviderError::from) }) } } diff --git a/crates/storage/provider/src/writer/mod.rs b/crates/storage/provider/src/writer/mod.rs index ef8fb25013b..b5a313c4448 100644 --- a/crates/storage/provider/src/writer/mod.rs +++ b/crates/storage/provider/src/writer/mod.rs @@ -20,6 +20,7 @@ mod tests { use reth_trie::{ test_utils::{state_root, storage_root_prehashed}, HashedPostState, HashedStorage, StateRoot, StorageRoot, StorageRootProgress, + TrieInputSorted, }; use reth_trie_db::{DatabaseStateRoot, DatabaseStorageRoot, LegacyKeyAdapter, PackedKeyAdapter}; use revm::database::{ @@ -1186,18 +1187,25 @@ mod tests { reth_trie_db::DatabaseHashedCursorFactory<&'a TX>, >; let is_v2 = provider_rw.cached_storage_settings().is_v2(); + let input = TrieInputSorted::from_state(HashedPostState::from_hashed_storage( + hashed_address, + updated_storage.clone(), + ).into_sorted()); let storage_root = if is_v2 { TestStorageRoot::<_, _, PackedKeyAdapter>::overlay_root( tx, address, - updated_storage.clone(), + input, ) .unwrap() } else { TestStorageRoot::<_, _, LegacyKeyAdapter>::overlay_root( tx, address, - updated_storage.clone(), + TrieInputSorted::from_state(HashedPostState::from_hashed_storage( + hashed_address, + updated_storage.clone(), + ).into_sorted()), ) .unwrap() }; diff --git a/crates/storage/storage-overlay/src/builder.rs b/crates/storage/storage-overlay/src/builder.rs index d56fc0b4d7f..b78df88c9d3 100644 --- a/crates/storage/storage-overlay/src/builder.rs +++ b/crates/storage/storage-overlay/src/builder.rs @@ -1,4 +1,7 @@ -use crate::{manager::OverlayCacheConfig, OverlayManager}; +use crate::{ + manager::{OverlayCacheConfig, StateTrieOverlayError}, + OverlayManager, +}; use alloy_eips::BlockNumHash; use alloy_primitives::{ map::{AddressMap, AddressSet, B256Map, U256Map}, @@ -16,7 +19,7 @@ use reth_storage_api::{ BlockNumReader, ChangeSetReader, DBProvider, PruneCheckpointReader, StageCheckpointReader, StorageChangeSetReader, StorageSettingsCache, }; -use reth_trie::{updates::TrieUpdatesSorted, HashedPostStateSorted}; +use reth_trie::{updates::TrieUpdatesSorted, HashedPostStateSorted, TrieInputSorted}; use reth_trie_db::DatabaseHashedPostState; use revm::{bytecode::Bytecode, database::BundleState, state::AccountInfo}; use std::{ @@ -29,28 +32,23 @@ use tracing::{debug, debug_span, instrument}; /// Contains the trie and hashed-state data required to initialize an overlay state provider. #[derive(Debug, Clone)] pub struct StateTrieOverlay { - /// Trie updates overlay. - pub trie_updates: Arc, - /// Hashed state overlay. - pub hashed_post_state: Arc, + input: TrieInputSorted, /// Whether construction was skipped because a reused sparse trie covers this range. skipped_for_reused_sparse_trie: bool, } impl StateTrieOverlay { - pub(crate) const fn new( - trie_updates: Arc, - hashed_post_state: Arc, - ) -> Self { - Self { trie_updates, hashed_post_state, skipped_for_reused_sparse_trie: false } + pub(crate) const fn new(input: TrieInputSorted) -> Self { + Self { input, skipped_for_reused_sparse_trie: false } } fn empty() -> Self { - Self { - trie_updates: Arc::new(TrieUpdatesSorted::default()), - hashed_post_state: Arc::new(HashedPostStateSorted::default()), - skipped_for_reused_sparse_trie: true, - } + Self { input: TrieInputSorted::default(), skipped_for_reused_sparse_trie: true } + } + + /// Returns the trie input represented by this overlay. + pub const fn input(&self) -> &TrieInputSorted { + &self.input } pub(crate) const fn skipped_for_reused_sparse_trie(&self) -> bool { @@ -393,11 +391,17 @@ impl OverlayBuilder { } /// Builds the effective state trie overlay for the given provider. + /// + /// Set `trie_changesets` only for consumers that produce [`TrieUpdates`], such as + /// [`StateRootProvider::state_root_with_updates`]. Other consumers, including roots, proofs, + /// multiproofs, and witnesses, should leave it false: complete the cached trie at Finish and + /// invalidate hashed-state revert prefixes instead of querying trie changesets. #[cfg(test)] #[instrument(level = "debug", target = "storage::overlay", skip_all)] fn build_state_trie_overlay( &self, provider: &Provider, + trie_changesets: bool, ) -> ProviderResult where Provider: StageCheckpointReader @@ -409,7 +413,12 @@ impl OverlayBuilder { + StorageSettingsCache, { let (state_trie_tip_block, finish_tip_block) = database_state_frontiers(provider)?; - self.build_state_trie_overlay_at_frontiers(provider, state_trie_tip_block, finish_tip_block) + self.build_state_trie_overlay_at_frontiers( + provider, + state_trie_tip_block, + finish_tip_block, + trie_changesets, + ) } /// Builds the effective state trie overlay using frontiers already read from the provider. @@ -426,6 +435,7 @@ impl OverlayBuilder { provider: &Provider, state_trie_tip_block: BlockNumHash, finish_tip_block: BlockNumHash, + trie_changesets: bool, ) -> ProviderResult where Provider: ChangeSetReader @@ -445,7 +455,7 @@ impl OverlayBuilder { self.anchor_at_parent_with_frontiers(provider, state_trie_tip_block, finish_tip_block)?; // Collect any reverts which are required to bring the DB view back to the anchor hash. - let (trie_updates, hashed_post_state) = match &anchor_for_parent { + let (trie_updates, hashed_post_state, prefix_sets) = match &anchor_for_parent { AnchorForParent::RevertsRequired { anchor, .. } => { let revert_blocks = self.revert_blocks(&anchor_for_parent)?.expect("reverts are required"); @@ -457,7 +467,7 @@ impl OverlayBuilder { "Collecting trie reverts for overlay state provider" ); - let trie_reverts = { + let trie_reverts = if trie_changesets { let _guard = debug_span!(target: "storage::overlay", "retrieving_trie_reverts") .entered(); let start = Instant::now(); @@ -470,6 +480,32 @@ impl OverlayBuilder { )?; retrieve_trie_reverts_duration = start.elapsed(); accumulated_reverts + } else if state_trie_tip_block == finish_tip_block { + retrieve_trie_reverts_duration = Duration::ZERO; + Arc::default() + } else { + // Revert prefixes describe changes since the anchor, not stale hashes in + // masked DB rows. Complete the trie at Finish before applying those prefixes. + let start = Instant::now(); + let finish_state = self + .overlay_manager + .block_state(finish_tip_block.hash) + .ok_or_else(|| { + ProviderError::other(StateTrieOverlayError { + tip_hash: finish_tip_block.hash, + anchor_hash: state_trie_tip_block.hash, + }) + })?; + let (nodes, _) = self + .overlay_manager + .overlay_for_parent( + &finish_state, + state_trie_tip_block.hash, + OverlayCacheConfig::default(), + ) + .map_err(ProviderError::other)?; + retrieve_trie_reverts_duration = start.elapsed(); + nodes }; let mut hashed_state_reverts = { @@ -482,6 +518,12 @@ impl OverlayBuilder { res }; + let prefix_sets = if trie_changesets { + Default::default() + } else { + hashed_state_reverts.construct_prefix_sets() + }; + // Resolve overlays and extend reverts with them. If reverts are empty, use overlays // directly to avoid cloning. let (overlay_trie, overlay_state) = @@ -517,7 +559,7 @@ impl OverlayBuilder { "Reverted to anchor block", ); - (trie_updates, hashed_state_updates) + (trie_updates, hashed_state_updates, prefix_sets) } AnchorForParent::NoReverts { anchor } => { // If no reverts are needed, use the manager overlay directly unless the reused @@ -548,7 +590,7 @@ impl OverlayBuilder { "Built overlay directly from durable frontier" ); - (trie_updates, hashed_post_state) + (trie_updates, hashed_post_state, Default::default()) } }; @@ -561,7 +603,11 @@ impl OverlayBuilder { self.metrics.trie_updates_size.record(trie_updates_total_len as f64); self.metrics.hashed_state_size.record(hashed_state_updates_total_len as f64); - Ok(StateTrieOverlay::new(trie_updates, hashed_post_state)) + Ok(StateTrieOverlay::new(TrieInputSorted::new( + trie_updates, + hashed_post_state, + prefix_sets, + ))) } /// Returns the in-memory execution overlay and the block for historical fallback reads. @@ -892,11 +938,11 @@ mod tests { } fn account_keys(overlay: &StateTrieOverlay) -> Vec { - overlay.hashed_post_state.accounts.iter().map(|(key, _)| *key).collect() + overlay.input().state.accounts.iter().map(|(key, _)| *key).collect() } fn account_node_paths(overlay: &StateTrieOverlay) -> Vec { - overlay.trie_updates.account_nodes_ref().iter().map(|(path, _)| *path).collect() + overlay.input().nodes.account_nodes_ref().iter().map(|(path, _)| *path).collect() } #[test] @@ -1013,7 +1059,7 @@ mod tests { for (parent_index, expected_ids) in [(3, vec![3, 4]), (4, vec![3, 4, 5])] { let overlay = manager .overlay_builder(blocks[parent_index].recovered_block().hash()) - .build_state_trie_overlay(&provider) + .build_state_trie_overlay(&provider, true) .unwrap(); assert_eq!( @@ -1041,11 +1087,11 @@ mod tests { let overlay = manager .overlay_builder(blocks[4].recovered_block().hash()) .with_skip_overlay_for_reused_sparse_trie(blocks[3].recovered_block().hash()) - .build_state_trie_overlay(&provider) + .build_state_trie_overlay(&provider, true) .unwrap(); - assert!(overlay.hashed_post_state.is_empty()); - assert!(overlay.trie_updates.is_empty()); + assert!(overlay.input().state.is_empty()); + assert!(overlay.input().nodes.is_empty()); } #[test] @@ -1056,7 +1102,7 @@ mod tests { let builder = OverlayManager::::default() .overlay_builder(blocks[1].recovered_block().hash()) .with_no_reverts(); - let error = builder.build_state_trie_overlay(&provider).unwrap_err(); + let error = builder.build_state_trie_overlay(&provider, true).unwrap_err(); assert!(error.to_string().contains("reverts are disabled")); } @@ -1077,11 +1123,37 @@ mod tests { Err(ProviderError::BlockHashNotFound(hash)) if hash == parent_hash )); assert!(matches!( - builder.build_state_trie_overlay(&provider), + builder.build_state_trie_overlay(&provider, true), Err(ProviderError::BlockHashNotFound(hash)) if hash == parent_hash )); } + #[test] + fn state_trie_overlay_uses_revert_prefix_sets_without_trie_changesets() { + let (factory, blocks) = setup_frontiers(3, 3); + let provider_rw = factory.provider_rw().unwrap(); + provider_rw + .tx_ref() + .put::( + 3, + AccountBeforeTx { + address: Address::with_last_byte(1), + info: Some(Account::default()), + }, + ) + .unwrap(); + provider_rw.commit().unwrap(); + + let provider = factory.provider().unwrap(); + let overlay = OverlayManager::::default() + .overlay_builder(blocks[1].recovered_block().hash()) + .build_state_trie_overlay(&provider, false) + .unwrap(); + + assert!(overlay.input().nodes.is_empty()); + assert!(!overlay.input().prefix_sets.is_empty()); + } + #[test] fn execution_overlay_marks_historical_fallback() { let (factory, blocks) = setup_frontiers(1, 3); @@ -1276,7 +1348,7 @@ mod tests { let provider = factory.provider().unwrap(); let error = OverlayManager::::default() .overlay_builder(blocks[3].recovered_block().hash()) - .build_state_trie_overlay(&provider) + .build_state_trie_overlay(&provider, true) .unwrap_err(); assert!( @@ -1292,7 +1364,7 @@ mod tests { let parent_hash = blocks[3].recovered_block().hash(); let error = OverlayManager::::default() .overlay_builder(parent_hash) - .build_state_trie_overlay(&provider) + .build_state_trie_overlay(&provider, true) .unwrap_err(); assert!(error.to_string().contains("is after partial state trie frontier")); diff --git a/crates/storage/storage-overlay/src/changeset_cache.rs b/crates/storage/storage-overlay/src/changeset_cache.rs index c2e070fd4b1..69191d65ae9 100644 --- a/crates/storage/storage-overlay/src/changeset_cache.rs +++ b/crates/storage/storage-overlay/src/changeset_cache.rs @@ -403,7 +403,7 @@ impl ChangesetCache { let overlay = overlay_manager .overlay_builder(finish.hash) .with_no_reverts() - .build_state_trie_overlay_at_frontiers(provider, partial_state_trie, finish)?; + .build_state_trie_overlay_at_frontiers(provider, partial_state_trie, finish, true)?; let state_trie_provider = OverlayStateProvider::<&P, N>::new_with_state_trie( provider, overlay, @@ -644,7 +644,7 @@ mod tests { } fn empty_overlay() -> StateTrieOverlay { - StateTrieOverlay::new(Arc::default(), Arc::default()) + StateTrieOverlay::new(TrieInputSorted::default()) } fn insert_test_changesets( diff --git a/crates/storage/storage-overlay/src/manager.rs b/crates/storage/storage-overlay/src/manager.rs index 0e942dacdf7..8eb63374718 100644 --- a/crates/storage/storage-overlay/src/manager.rs +++ b/crates/storage/storage-overlay/src/manager.rs @@ -99,7 +99,7 @@ impl OverlayManager { OverlayBuilder::new(parent_hash, self.block_state(parent_hash), self.clone()) } - fn block_state(&self, parent_hash: B256) -> Option> { + pub(crate) fn block_state(&self, parent_hash: B256) -> Option> { let mut blocks = self.parent_chain(parent_hash).collect::>(); blocks.pop().map(|oldest| { blocks.into_iter().rev().fold(BlockState::new(oldest), |parent, block| { @@ -685,9 +685,9 @@ impl Default for OverlayCacheConfig { #[derive(Debug)] pub(crate) struct StateTrieOverlayError { /// Requested in-memory tip hash. - tip_hash: B256, + pub(crate) tip_hash: B256, /// Requested anchor hash. - anchor_hash: B256, + pub(crate) anchor_hash: B256, } impl fmt::Display for StateTrieOverlayError { diff --git a/crates/storage/storage-overlay/src/provider.rs b/crates/storage/storage-overlay/src/provider.rs index 578e4453560..cf2614c56a7 100644 --- a/crates/storage/storage-overlay/src/provider.rs +++ b/crates/storage/storage-overlay/src/provider.rs @@ -52,7 +52,7 @@ pub struct OverlayStateProviderFactory { factory: F, /// Overlay builder containing the configuration and overlay calculation logic. overlay_builder: OverlayBuilder, - /// A cache mapping `(state_trie_tip, finish_tip)` to [`StateTrieOverlay`]. + /// A cache mapping `(state_trie_tip, finish_tip, trie_changesets)` to [`StateTrieOverlay`]. /// /// Under partial persistence the overlay depends on both durable frontiers, so both hashes are /// part of the cache key. @@ -129,6 +129,7 @@ pub struct OverlayStateProvider { state_trie_overlay_cache: StateTrieOverlayCache, metrics: OverlayStateProviderFactoryMetrics, state_trie_overlay: OnceCell, + state_trie_overlay_with_trie_changesets: OnceCell, execution_overlay: OnceCell, is_v2: bool, } @@ -162,6 +163,7 @@ impl OverlayStateProvider, state_trie_overlay_cache, metrics, state_trie_overlay: OnceCell::new(), + state_trie_overlay_with_trie_changesets: OnceCell::new(), execution_overlay: OnceCell::new(), is_v2, } @@ -179,6 +181,7 @@ impl OverlayStateProvider, state_trie_overlay_cache: Default::default(), metrics: Default::default(), state_trie_overlay: OnceCell::new(), + state_trie_overlay_with_trie_changesets: OnceCell::new(), execution_overlay: OnceCell::from(CachedExecutionOverlay { overlay: execution_overlay, historical_fallback: None, @@ -201,6 +204,7 @@ impl<'a, Provider, N: NodePrimitives> OverlayStateProvider<&'a Provider, N> { state_trie_overlay_cache: Default::default(), metrics: Default::default(), state_trie_overlay: OnceCell::new(), + state_trie_overlay_with_trie_changesets: OnceCell::new(), execution_overlay: OnceCell::new(), is_v2, } @@ -216,7 +220,8 @@ impl<'a, Provider, N: NodePrimitives> OverlayStateProvider<&'a Provider, N> { overlay_builder: None, state_trie_overlay_cache: Default::default(), metrics: Default::default(), - state_trie_overlay: OnceCell::from(state_trie_overlay), + state_trie_overlay: OnceCell::from(state_trie_overlay.clone()), + state_trie_overlay_with_trie_changesets: OnceCell::from(state_trie_overlay), execution_overlay: OnceCell::new(), is_v2, } @@ -232,7 +237,7 @@ where &self.provider } - fn state_trie_overlay(&self) -> ProviderResult<&StateTrieOverlay> + fn state_trie_overlay(&self, trie_changesets: bool) -> ProviderResult<&StateTrieOverlay> where Provider::Target: StageCheckpointReader + PruneCheckpointReader @@ -242,15 +247,17 @@ where + BlockNumReader + StorageSettingsCache, { - if let Some(overlay) = self.state_trie_overlay.get() { + let state_trie_overlay = self.state_trie_overlay_mut(trie_changesets); + if let Some(overlay) = state_trie_overlay.get() { return Ok(overlay) } let (state_trie_tip_block, finish_tip_block) = database_state_frontiers(self.provider())?; - let overlay = match self - .state_trie_overlay_cache - .entry((state_trie_tip_block.hash, finish_tip_block.hash)) - { + let overlay = match self.state_trie_overlay_cache.entry(( + state_trie_tip_block.hash, + finish_tip_block.hash, + trie_changesets, + )) { dashmap::Entry::Occupied(entry) => entry.get().clone(), dashmap::Entry::Vacant(entry) => { self.metrics.state_trie_overlay_cache_misses.increment(1); @@ -262,6 +269,7 @@ where self.provider(), state_trie_tip_block, finish_tip_block, + trie_changesets, )?; if !overlay.skipped_for_reused_sparse_trie() { entry.insert(overlay.clone()); @@ -269,11 +277,23 @@ where overlay } }; - let _ = self.state_trie_overlay.set(overlay); - Ok(self.state_trie_overlay.get().expect("state trie overlay was just initialized")) + let _ = state_trie_overlay.set(overlay); + Ok(state_trie_overlay.get().expect("state trie overlay was just initialized")) + } + + const fn state_trie_overlay_mut(&self, trie_changesets: bool) -> &OnceCell { + if trie_changesets { + &self.state_trie_overlay_with_trie_changesets + } else { + &self.state_trie_overlay + } } - fn build_overlay(&self, input: TrieInputSorted) -> ProviderResult + fn build_overlay( + &self, + input: TrieInputSorted, + trie_changesets: bool, + ) -> ProviderResult where Provider::Target: StageCheckpointReader + PruneCheckpointReader @@ -283,13 +303,14 @@ where + BlockNumReader + StorageSettingsCache, { - let overlay = self.state_trie_overlay()?; + let overlay = self.state_trie_overlay(trie_changesets)?; if overlay.skipped_for_reused_sparse_trie() { return Err(ProviderError::UnsupportedProvider) } - let TrieInputSorted { nodes: input_nodes, state: input_state, prefix_sets } = input; - let mut nodes = Arc::clone(&overlay.trie_updates); - let mut state = Arc::clone(&overlay.hashed_post_state); + let TrieInputSorted { nodes: input_nodes, state: input_state, mut prefix_sets } = input; + let overlay_input = overlay.input(); + let mut nodes = Arc::clone(&overlay_input.nodes); + let mut state = Arc::clone(&overlay_input.state); if !input_nodes.is_empty() { Arc::make_mut(&mut nodes).extend_ref_and_sort(&input_nodes); @@ -298,6 +319,7 @@ where Arc::make_mut(&mut state).extend_ref_and_sort(&input_state); } + prefix_sets.extend_ref(&overlay_input.prefix_sets); Ok(TrieInputSorted::new(nodes, state, prefix_sets)) } @@ -511,16 +533,15 @@ where { fn state_root(&self, hashed_state: HashedPostState) -> ProviderResult { reth_trie_db::with_adapter!(self.provider(), |A| { - let input = self.build_overlay(TrieInputSorted::from_unsorted( - TrieInput::from_state(hashed_state), - ))?; + let input = + self.build_overlay(TrieInputSorted::from_state(hashed_state.into_sorted()), false)?; Ok(>::overlay_root_from_nodes(self.provider().tx(), input)?) }) } fn state_root_from_nodes(&self, input: TrieInput) -> ProviderResult { reth_trie_db::with_adapter!(self.provider(), |A| { - let input = self.build_overlay(TrieInputSorted::from_unsorted(input))?; + let input = self.build_overlay(TrieInputSorted::from_unsorted(input), false)?; Ok( as DatabaseStateRoot<_>>::overlay_root_from_nodes( self.provider().tx(), input, @@ -533,9 +554,8 @@ where hashed_state: HashedPostState, ) -> ProviderResult<(B256, TrieUpdates)> { reth_trie_db::with_adapter!(self.provider(), |A| { - let input = self.build_overlay(TrieInputSorted::from_unsorted( - TrieInput::from_state(hashed_state), - ))?; + let input = + self.build_overlay(TrieInputSorted::from_state(hashed_state.into_sorted()), true)?; Ok(>::overlay_root_from_nodes_with_updates( self.provider().tx(), input, @@ -548,7 +568,7 @@ where input: TrieInput, ) -> ProviderResult<(B256, TrieUpdates)> { reth_trie_db::with_adapter!(self.provider(), |A| { - let input = self.build_overlay(TrieInputSorted::from_unsorted(input))?; + let input = self.build_overlay(TrieInputSorted::from_unsorted(input), true)?; Ok( as DatabaseStateRoot<_>>::overlay_root_from_nodes_with_updates( self.provider().tx(), @@ -576,20 +596,15 @@ where hashed_storage: HashedStorage, ) -> ProviderResult { reth_trie_db::with_adapter!(self.provider(), |A| { - let input = self.build_overlay(TrieInputSorted::from_unsorted( - TrieInput::from_state(HashedPostState::from_hashed_storage( - alloy_primitives::keccak256(address), - hashed_storage, - )), - ))?; - let hashed_storage = input - .state - .account_storages() - .get(&alloy_primitives::keccak256(address)) - .cloned() - .unwrap_or_default() - .into(); - >::overlay_root(self.provider().tx(), address, hashed_storage) + let hashed_address = alloy_primitives::keccak256(address); + let input = self.build_overlay( + TrieInputSorted::from_state( + HashedPostState::from_hashed_storage(hashed_address, hashed_storage) + .into_sorted(), + ), + false, + )?; + >::overlay_root(self.provider().tx(), address, input) .map_err(|err| ProviderError::Database(err.into())) }) } @@ -600,28 +615,9 @@ where slot: B256, hashed_storage: HashedStorage, ) -> ProviderResult { - reth_trie_db::with_adapter!(self.provider(), |A| { - let input = self.build_overlay(TrieInputSorted::from_unsorted( - TrieInput::from_state(HashedPostState::from_hashed_storage( - alloy_primitives::keccak256(address), - hashed_storage, - )), - ))?; - let hashed_storage = input - .state - .account_storages() - .get(&alloy_primitives::keccak256(address)) - .cloned() - .unwrap_or_default() - .into(); - >::overlay_storage_proof( - self.provider().tx(), - address, - slot, - hashed_storage, - ) + self.storage_multiproof(address, &[slot], hashed_storage)? + .storage_proof(slot) .map_err(ProviderError::from) - }) } fn storage_multiproof( @@ -631,24 +627,19 @@ where hashed_storage: HashedStorage, ) -> ProviderResult { reth_trie_db::with_adapter!(self.provider(), |A| { - let input = self.build_overlay(TrieInputSorted::from_unsorted( - TrieInput::from_state(HashedPostState::from_hashed_storage( - alloy_primitives::keccak256(address), - hashed_storage, - )), - ))?; - let hashed_storage = input - .state - .account_storages() - .get(&alloy_primitives::keccak256(address)) - .cloned() - .unwrap_or_default() - .into(); + let hashed_address = alloy_primitives::keccak256(address); + let input = self.build_overlay( + TrieInputSorted::from_state( + HashedPostState::from_hashed_storage(hashed_address, hashed_storage) + .into_sorted(), + ), + false, + )?; >::overlay_storage_multiproof( self.provider().tx(), address, slots, - hashed_storage, + input, ) .map_err(ProviderError::from) }) @@ -674,7 +665,7 @@ where ) -> ProviderResult { reth_trie_db::with_adapter!(self.provider(), |A| { let TrieInputSorted { nodes, state, prefix_sets } = - self.build_overlay(TrieInputSorted::from_unsorted(input))?; + self.build_overlay(TrieInputSorted::from_unsorted(input), false)?; let input = TrieInput::new( Arc::unwrap_or_clone(nodes).into(), Arc::unwrap_or_clone(state).into(), @@ -692,7 +683,7 @@ where ) -> ProviderResult { reth_trie_db::with_adapter!(self.provider(), |A| { let TrieInputSorted { nodes, state, prefix_sets } = - self.build_overlay(TrieInputSorted::from_unsorted(input))?; + self.build_overlay(TrieInputSorted::from_unsorted(input), false)?; let input = TrieInput::new( Arc::unwrap_or_clone(nodes).into(), Arc::unwrap_or_clone(state).into(), @@ -710,7 +701,7 @@ where ) -> ProviderResult { reth_trie_db::with_adapter!(self.provider(), |A| { let TrieInputSorted { nodes, state, prefix_sets } = - self.build_overlay(TrieInputSorted::from_unsorted(input))?; + self.build_overlay(TrieInputSorted::from_unsorted(input), false)?; let input = TrieInput::new( Arc::unwrap_or_clone(nodes).into(), Arc::unwrap_or_clone(state).into(), @@ -729,7 +720,7 @@ where ) -> ProviderResult> { reth_trie_db::with_adapter!(self.provider(), |A| { let TrieInputSorted { nodes, state, prefix_sets } = - self.build_overlay(TrieInputSorted::from_unsorted(input))?; + self.build_overlay(TrieInputSorted::from_unsorted(input), false)?; let witness = TrieWitness::new( InMemoryTrieCursorFactory::new( DatabaseTrieCursorFactory::<_, A>::new(self.provider().tx()), @@ -778,7 +769,7 @@ where return Ok(hashed_state) } - let overlay_state = self.build_overlay(TrieInputSorted::default())?.state; + let overlay_state = self.build_overlay(TrieInputSorted::default(), false)?.state; zero_destroyed_account_storage( &HashedPostStateCursorFactory::new( DatabaseHashedCursorFactory::new(self.provider().tx()), @@ -893,7 +884,7 @@ where Self: 'a; fn account_trie_cursor(&self) -> Result, DatabaseError> { - let overlay = self.state_trie_overlay().map_err(into_database_error)?; + let overlay = self.state_trie_overlay(true).map_err(into_database_error)?; let cursor: Box = if self.is_v2 { Box::new(DatabaseAccountTrieCursor::<_, PackedKeyAdapter>::new( self.provider().tx().cursor_read::()?, @@ -903,14 +894,14 @@ where self.provider().tx().cursor_read::()?, )) }; - Ok(InMemoryTrieCursor::new_account(cursor, &overlay.trie_updates)) + Ok(InMemoryTrieCursor::new_account(cursor, &overlay.input().nodes)) } fn storage_trie_cursor( &self, hashed_address: B256, ) -> Result, DatabaseError> { - let overlay = self.state_trie_overlay().map_err(into_database_error)?; + let overlay = self.state_trie_overlay(true).map_err(into_database_error)?; let cursor: Box = if self.is_v2 { Box::new(DatabaseStorageTrieCursor::<_, PackedKeyAdapter>::new( self.provider().tx().cursor_dup_read::()?, @@ -922,7 +913,7 @@ where hashed_address, )) }; - Ok(InMemoryTrieCursor::new_storage(cursor, &overlay.trie_updates, hashed_address)) + Ok(InMemoryTrieCursor::new_storage(cursor, &overlay.input().nodes, hashed_address)) } } @@ -954,10 +945,10 @@ where Self: 'a; fn hashed_account_cursor(&self) -> Result, DatabaseError> { - let overlay = self.state_trie_overlay().map_err(into_database_error)?; + let overlay = self.state_trie_overlay(true).map_err(into_database_error)?; HashedPostStateCursorFactory::new( DatabaseHashedCursorFactory::new(self.provider().tx()), - &overlay.hashed_post_state, + &overlay.input().state, ) .hashed_account_cursor() } @@ -966,10 +957,10 @@ where &self, hashed_address: B256, ) -> Result, DatabaseError> { - let overlay = self.state_trie_overlay().map_err(into_database_error)?; + let overlay = self.state_trie_overlay(true).map_err(into_database_error)?; HashedPostStateCursorFactory::new( DatabaseHashedCursorFactory::new(self.provider().tx()), - &overlay.hashed_post_state, + &overlay.input().state, ) .hashed_storage_cursor(hashed_address) } @@ -987,7 +978,7 @@ pub(crate) struct OverlayStateProviderFactoryMetrics { state_trie_overlay_cache_misses: Counter, } -type StateTrieOverlayCache = Arc>; +type StateTrieOverlayCache = Arc>; #[derive(Clone, Debug)] struct CachedExecutionOverlay { @@ -1122,11 +1113,11 @@ mod tests { } fn account_keys(overlay: &StateTrieOverlay) -> Vec { - overlay.hashed_post_state.accounts.iter().map(|(key, _)| *key).collect() + overlay.input().state.accounts.iter().map(|(key, _)| *key).collect() } fn account_node_paths(overlay: &StateTrieOverlay) -> Vec { - overlay.trie_updates.account_nodes_ref().iter().map(|(path, _)| *path).collect() + overlay.input().nodes.account_nodes_ref().iter().map(|(path, _)| *path).collect() } #[test] @@ -1142,7 +1133,7 @@ mod tests { ); let provider = state_provider_factory.database_provider_ro().unwrap(); - let first = provider.state_trie_overlay().unwrap().clone(); + let first = provider.state_trie_overlay(false).unwrap().clone(); assert_eq!(account_keys(&first), vec![B256::with_last_byte(3), B256::with_last_byte(4)]); drop(provider); @@ -1158,7 +1149,7 @@ mod tests { provider_rw.commit().unwrap(); let provider = state_provider_factory.database_provider_ro().unwrap(); - let second = provider.state_trie_overlay().unwrap().clone(); + let second = provider.state_trie_overlay(false).unwrap().clone(); assert_eq!(account_keys(&second), vec![B256::with_last_byte(4)]); assert_eq!(account_node_paths(&second), vec![Nibbles::from_nibbles([4])]); assert_eq!(state_provider_factory.state_trie_overlay_cache.len(), 2); @@ -1191,6 +1182,39 @@ mod tests { assert_eq!(state_provider_factory.state_trie_overlay_cache.len(), 1); } + #[test] + fn state_trie_overlay_cache_is_keyed_by_trie_changesets() { + let (factory, blocks) = setup_frontiers(1, 3); + let manager = OverlayManager::default(); + for block in &blocks[2..=3] { + manager.insert_block(block.clone()); + } + let state_provider_factory = OverlayStateProviderFactory::new( + factory, + manager.overlay_builder(blocks[3].recovered_block().hash()), + ); + let provider = state_provider_factory.database_provider_ro().unwrap(); + + provider.state_trie_overlay(false).unwrap(); + provider.state_trie_overlay(true).unwrap(); + + assert_eq!(state_provider_factory.state_trie_overlay_cache.len(), 2); + } + + #[test] + fn supplied_state_trie_overlay_is_available_in_both_modes() { + let (factory, _) = setup_frontiers(1, 1); + let provider = factory.provider().unwrap(); + let provider = OverlayStateProvider::<&_, EthPrimitives>::new_with_state_trie( + &provider, + StateTrieOverlay::new(TrieInputSorted::default()), + false, + ); + + assert!(provider.state_trie_overlay(false).is_ok()); + assert!(provider.state_trie_overlay(true).is_ok()); + } + #[test] fn lazy_overlays_survive_manager_block_removal() { let (factory, blocks) = setup_frontiers(1, 1); @@ -1212,7 +1236,7 @@ mod tests { [blocks[2].recovered_block().num_hash(), blocks[3].recovered_block().num_hash()] ); assert_eq!( - account_keys(provider.state_trie_overlay().unwrap()), + account_keys(provider.state_trie_overlay(false).unwrap()), vec![B256::with_last_byte(3), B256::with_last_byte(4)] ); } @@ -1230,7 +1254,7 @@ mod tests { ); let provider = state_provider_factory.database_provider_ro().unwrap(); - assert!(provider.state_trie_overlay().unwrap().skipped_for_reused_sparse_trie()); + assert!(provider.state_trie_overlay(false).unwrap().skipped_for_reused_sparse_trie()); assert!(state_provider_factory.state_trie_overlay_cache.is_empty()); assert!(matches!( provider.state_root(HashedPostState::default()), diff --git a/crates/trie/common/src/input.rs b/crates/trie/common/src/input.rs index 3d3bc2349d3..d9f8f41f46b 100644 --- a/crates/trie/common/src/input.rs +++ b/crates/trie/common/src/input.rs @@ -160,6 +160,13 @@ impl TrieInputSorted { Self { nodes, state, prefix_sets } } + /// Create new sorted trie input from sorted in-memory state. The prefix sets will be + /// constructed and set automatically. + pub fn from_state(state: HashedPostStateSorted) -> Self { + let prefix_sets = state.construct_prefix_sets(); + Self { nodes: Default::default(), state: Arc::new(state), prefix_sets } + } + /// Create from unsorted [`TrieInput`] by sorting. pub fn from_unsorted(input: TrieInput) -> Self { Self { diff --git a/crates/trie/db/src/proof.rs b/crates/trie/db/src/proof.rs index f362692963d..934c3fecd4e 100644 --- a/crates/trie/db/src/proof.rs +++ b/crates/trie/db/src/proof.rs @@ -7,7 +7,7 @@ use reth_trie::{ proof::{Proof, StorageProof}, trie_cursor::InMemoryTrieCursorFactory, AccountProof, DecodedMultiProofV2, HashedPostStateSorted, HashedStorage, MultiProof, - MultiProofTargets, MultiProofTargetsV2, StorageMultiProof, TrieInput, + MultiProofTargets, MultiProofTargetsV2, StorageMultiProof, TrieInput, TrieInputSorted, }; /// Extends [`Proof`] with operations specific for working with a database transaction. @@ -114,7 +114,7 @@ pub trait DatabaseStorageProof<'a, TX> { tx: &'a TX, address: Address, slots: &[B256], - storage: HashedStorage, + input: TrieInputSorted, ) -> Result; } @@ -158,21 +158,21 @@ impl<'a, TX: DbTx, A: TrieTableAdapter> DatabaseStorageProof<'a, TX> tx: &'a TX, address: Address, slots: &[B256], - storage: HashedStorage, + mut input: TrieInputSorted, ) -> Result { let hashed_address = keccak256(address); let targets = slots.iter().map(keccak256).collect(); - let prefix_set = storage.construct_prefix_set(); - let state_sorted = HashedPostStateSorted::new( - Default::default(), - HashMap::from_iter([(hashed_address, storage.into_sorted())]), - ); StorageProof::new( - DatabaseTrieCursorFactory::<_, A>::new(tx), - HashedPostStateCursorFactory::new(DatabaseHashedCursorFactory::new(tx), &state_sorted), + InMemoryTrieCursorFactory::new( + DatabaseTrieCursorFactory::<_, A>::new(tx), + &input.nodes, + ), + HashedPostStateCursorFactory::new(DatabaseHashedCursorFactory::new(tx), &input.state), address, ) - .with_prefix_set_mut(prefix_set) + .with_prefix_set_mut( + input.prefix_sets.storage_prefix_sets.remove(&hashed_address).unwrap_or_default(), + ) .storage_multiproof(targets) } } diff --git a/crates/trie/db/src/storage.rs b/crates/trie/db/src/storage.rs index 884d6f15565..e634723f1b2 100644 --- a/crates/trie/db/src/storage.rs +++ b/crates/trie/db/src/storage.rs @@ -5,7 +5,8 @@ use reth_execution_errors::StorageRootError; use reth_storage_api::{BlockNumReader, StorageChangeSetReader}; use reth_storage_errors::provider::ProviderResult; use reth_trie::{ - hashed_cursor::HashedPostStateCursorFactory, HashedPostState, HashedStorage, StorageRoot, + hashed_cursor::HashedPostStateCursorFactory, trie_cursor::InMemoryTrieCursorFactory, + HashedStorage, StorageRoot, TrieInputSorted, }; #[cfg(feature = "metrics")] @@ -19,11 +20,11 @@ pub trait DatabaseStorageRoot<'a, TX> { /// Create a new storage root calculator from database transaction and hashed address. fn from_tx_hashed(tx: &'a TX, hashed_address: B256) -> Self; - /// Calculates the storage root for this [`HashedStorage`] and returns it. + /// Calculates the storage root with the given trie input. fn overlay_root( tx: &'a TX, address: Address, - hashed_storage: HashedStorage, + input: TrieInputSorted, ) -> Result; } @@ -85,16 +86,22 @@ impl<'a, TX: DbTx, A: TrieTableAdapter> DatabaseStorageRoot<'a, TX> fn overlay_root( tx: &'a TX, address: Address, - hashed_storage: HashedStorage, + mut input: TrieInputSorted, ) -> Result { - let prefix_set = hashed_storage.construct_prefix_set().freeze(); - let state_sorted = - HashedPostState::from_hashed_storage(keccak256(address), hashed_storage).into_sorted(); + let hashed_address = keccak256(address); StorageRoot::new( - DatabaseTrieCursorFactory::<_, A>::new(tx), - HashedPostStateCursorFactory::new(DatabaseHashedCursorFactory::new(tx), &state_sorted), + InMemoryTrieCursorFactory::new( + DatabaseTrieCursorFactory::<_, A>::new(tx), + &input.nodes, + ), + HashedPostStateCursorFactory::new(DatabaseHashedCursorFactory::new(tx), &input.state), address, - prefix_set, + input + .prefix_sets + .storage_prefix_sets + .remove(&hashed_address) + .unwrap_or_default() + .freeze(), #[cfg(feature = "metrics")] TrieRootMetrics::new(reth_trie::TrieType::Storage), ) From d70cc04dab7d337f96080fad1923ec355750c391 Mon Sep 17 00:00:00 2001 From: Sebastian Stammler Date: Thu, 1 Oct 2026 10:42:37 +0200 Subject: [PATCH 284/294] test(storage-overlay): cover storage roots over a masked state trie Pin that `OverlayStateProvider::storage_root` reads trie nodes through the overlay when partial persistence leaves database storage-trie nodes older than the persisted hashed storage. OP Stack consensus depends on this: the Isthmus withdrawals root is the L2ToL1MessagePasser storage root, computed through `storage_root` when building and validating blocks. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../storage/storage-overlay/src/provider.rs | 180 ++++++++++++++++++ 1 file changed, 180 insertions(+) diff --git a/crates/storage/storage-overlay/src/provider.rs b/crates/storage/storage-overlay/src/provider.rs index cf2614c56a7..af04e75adb4 100644 --- a/crates/storage/storage-overlay/src/provider.rs +++ b/crates/storage/storage-overlay/src/provider.rs @@ -1361,4 +1361,184 @@ mod tests { assert_eq!(provider.basic_account(&address).unwrap(), Some(account)); assert_eq!(provider.storage(address, storage_key).unwrap(), Some(storage)); } + + /// Hashed slot with the given first two bytes, so that the storage trie has stored branch + /// nodes at depth one that cache the hashes of their children. + fn hashed_slot(first: u8, second: u8) -> B256 { + let mut slot = alloy_primitives::keccak256([first, second]); + slot.0[0] = first; + slot.0[1] = second; + slot + } + + /// Writes hashed state and trie updates like `save_blocks` does under partial persistence. + fn write_state_trie( + provider: &(impl reth_storage_api::StateWriter + reth_storage_api::TrieWriter), + batch: &[ExecutedBlock], + mask: &[ExecutedBlock], + ) { + provider + .write_hashed_state(&HashedPostStateSorted::disjointed_merge_batch( + &ExecutedBlock::hashed_state_refs(batch), + &ExecutedBlock::hashed_state_refs(mask), + )) + .unwrap(); + provider + .write_trie_updates_sorted(&TrieUpdatesSorted::disjointed_merge_batch( + &ExecutedBlock::trie_updates_refs(batch), + &ExecutedBlock::trie_updates_refs(mask), + )) + .unwrap(); + } + + /// Storage roots over a partially persisted state trie must use the overlay trie nodes: the + /// database trie tables omit nodes overwritten by the masking suffix, so their cached child + /// hashes can be older than the persisted hashed state. + #[test] + fn storage_root_over_masked_state_trie() { + for settings in [reth_provider::StorageSettings::v1(), reth_provider::StorageSettings::v2()] + { + let address = Address::with_last_byte(0x16); + let hashed_address = alloy_primitives::keccak256(address); + + let base = (0..=u8::MAX) + .flat_map(|first| { + [0x00, 0x80].map(|second| (hashed_slot(first, second), U256::from(1))) + }) + .collect::>(); + let changes: [Vec<(B256, U256)>; 8] = [ + vec![], + base, + vec![ + (hashed_slot(0x10, 0x00), U256::from(2)), + (hashed_slot(0x11, 0x42), U256::from(2)), + (hashed_slot(0x20, 0x00), U256::ZERO), + ], + vec![(hashed_slot(0x30, 0x00), U256::from(3))], + vec![(hashed_slot(0x40, 0x80), U256::from(4))], + vec![ + (hashed_slot(0x30, 0x00), U256::from(5)), + (hashed_slot(0x31, 0x00), U256::from(5)), + (hashed_slot(0x50, 0x00), U256::from(5)), + (hashed_slot(0x60, 0x80), U256::ZERO), + ], + vec![ + (hashed_slot(0x1f, 0x80), U256::ZERO), + (hashed_slot(0x50, 0x00), U256::from(6)), + (hashed_slot(0x70, 0x00), U256::from(6)), + ], + vec![ + (hashed_slot(0x50, 0x00), U256::from(7)), + (hashed_slot(0x80, 0x00), U256::ZERO), + ], + ]; + let block_updates = [ + (hashed_slot(0x90, 0x42), U256::from(1)), + (hashed_slot(0xa0, 0x00), U256::from(8)), + (hashed_slot(0xb0, 0x80), U256::ZERO), + ]; + + // Like the engine's, each block's storage trie updates only contain modified nodes. + let reference = create_test_provider_factory(); + reference.set_storage_settings_cache(settings); + let mut storage_nodes = std::collections::BTreeMap::new(); + let blocks = TestBlockBuilder::eth() + .get_executed_blocks(0..changes.len() as u64) + .zip(&changes) + .map(|(block, changes)| { + let mut hashed_state = HashedPostState::default().with_storages([( + hashed_address, + HashedStorage::from_iter(changes.iter().copied()), + )]); + if block.recovered_block().number == 1 { + hashed_state = hashed_state + .with_accounts([(hashed_address, Some(Account::default()))]); + } + let provider = reference.provider().unwrap(); + let (_, trie_updates) = reth_provider::LatestStateProviderRef::new(&provider) + .state_root_with_updates(hashed_state.clone()) + .unwrap(); + drop(provider); + let hashed_state = hashed_state.into_sorted(); + let trie_updates = trie_updates.into_sorted(); + let provider_rw = reference.provider_rw().unwrap(); + reth_storage_api::StateWriter::write_hashed_state(&*provider_rw, &hashed_state) + .unwrap(); + reth_storage_api::TrieWriter::write_trie_updates_sorted( + &*provider_rw, + &trie_updates, + ) + .unwrap(); + provider_rw.commit().unwrap(); + + let modified_storage_nodes = trie_updates + .storage_tries_ref() + .get(&hashed_address) + .map(|storage_trie| storage_trie.storage_nodes_ref()) + .unwrap_or_default() + .iter() + .filter(|(path, node)| match node { + Some(node) => { + storage_nodes.insert(*path, node.clone()).as_ref() != Some(node) + } + None => storage_nodes.remove(path).is_some(), + }) + .cloned() + .collect(); + let trie_updates = TrieUpdatesSorted::new( + trie_updates.account_nodes_ref().to_vec(), + alloy_primitives::map::B256Map::from_iter([( + hashed_address, + reth_trie::updates::StorageTrieUpdatesSorted { + storage_nodes: modified_storage_nodes, + }, + )]), + ); + ExecutedBlock::new( + Arc::clone(&block.recovered_block), + Arc::clone(&block.execution_output), + ComputedTrieData::new(Arc::new(hashed_state), Arc::new(trie_updates)), + ) + }) + .collect::>(); + + let factory = create_test_provider_factory(); + factory.set_storage_settings_cache(settings); + let provider_rw = factory.provider_rw().unwrap(); + for block in &blocks[..=6] { + provider_rw.insert_block(block.recovered_block()).unwrap(); + } + write_state_trie(&*provider_rw, &blocks[1..=1], &[]); + write_state_trie(&*provider_rw, &blocks[2..=4], &blocks[5..=6]); + provider_rw + .save_stage_checkpoint( + StageId::Finish, + StageCheckpoint::new(6).with_finish_stage_checkpoint(FinishCheckpoint { + partial_state_trie: Some(4), + }), + ) + .unwrap(); + provider_rw.commit().unwrap(); + + let manager = OverlayManager::default(); + for block in &blocks[5..=7] { + manager.insert_block(block.clone()); + } + let state_provider_factory = OverlayStateProviderFactory::new( + factory, + manager.overlay_builder(blocks[7].recovered_block().hash()), + ); + let provider = state_provider_factory.database_provider_ro().unwrap(); + + let mut expected = std::collections::BTreeMap::new(); + for (slot, value) in changes.iter().flatten().chain(&block_updates) { + expected.insert(*slot, *value); + } + expected.retain(|_, value| !value.is_zero()); + + let storage_root = + provider.storage_root(address, HashedStorage::from_iter(block_updates)).unwrap(); + assert_eq!(storage_root, reth_trie::test_utils::storage_root_prehashed(expected)); + } + } } From 0232cc15fef36c2a362e0cfdaf1b7b72cd27f723 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Thu, 24 Sep 2026 13:53:27 +0000 Subject: [PATCH 285/294] fix(engine): validate forkchoice state before applying updates (#27247) Co-authored-by: Matthias Seitz <19890894+mattsse@users.noreply.github.com> (cherry picked from commit a120f00d47e43427816e2fe7e38229fabbb2db87) --- crates/chain-state/src/in_memory.rs | 16 +- crates/engine/tree/src/tree/mod.rs | 73 +++++++++ crates/engine/tree/src/tree/tests.rs | 36 +++++ crates/ethereum/node/tests/e2e/forkchoice.rs | 161 +++++++++++++++++++ crates/ethereum/node/tests/e2e/main.rs | 1 + 5 files changed, 284 insertions(+), 3 deletions(-) create mode 100644 crates/ethereum/node/tests/e2e/forkchoice.rs diff --git a/crates/chain-state/src/in_memory.rs b/crates/chain-state/src/in_memory.rs index ef0fa4fabb3..adf6790bf0f 100644 --- a/crates/chain-state/src/in_memory.rs +++ b/crates/chain-state/src/in_memory.rs @@ -1001,12 +1001,22 @@ impl> NewCanonicalChain { /// Returns the new tip for [`Self::Reorg`] and [`Self::Commit`] variants which commit at least /// 1 new block. pub fn tip(&self) -> &RecoveredBlock { + self.new_blocks().last().expect("non empty blocks").recovered_block() + } + + /// Returns the blocks added to the canonical chain by this update. + pub fn new_blocks(&self) -> &[ExecutedBlock] { match self { - Self::Commit { new } | Self::Reorg { new, .. } => { - new.last().expect("non empty blocks").recovered_block() - } + Self::Commit { new } | Self::Reorg { new, .. } => new, } } + + /// Returns whether the newly canonicalized blocks contain the given hash. + /// + /// This does not include the unchanged canonical prefix before the first new block. + pub fn contains(&self, hash: B256) -> bool { + self.new_blocks().iter().any(|block| block.recovered_block().hash() == hash) + } } #[cfg(test)] diff --git a/crates/engine/tree/src/tree/mod.rs b/crates/engine/tree/src/tree/mod.rs index d45cfcab271..b91e86a65ea 100644 --- a/crates/engine/tree/src/tree/mod.rs +++ b/crates/engine/tree/src/tree/mod.rs @@ -1252,6 +1252,10 @@ where trace!(target: "engine::tree", "fcu head hash is already canonical"); + if !self.is_consistent_forkchoice_state(state, None)? { + return Ok(Some(TreeOutcome::new(OnForkChoiceUpdated::invalid_state()))); + } + // Update the safe and finalized blocks and ensure their values are valid if let Err(outcome) = self.ensure_consistent_forkchoice_state(state) { // safe or finalized hashes are invalid @@ -1320,6 +1324,10 @@ where return Ok(Some(TreeOutcome::new(OnForkChoiceUpdated::too_deep_reorg()))); } + if !self.is_consistent_forkchoice_state(state, None)? { + return Ok(Some(TreeOutcome::new(OnForkChoiceUpdated::invalid_state()))); + } + // We need to effectively unwind the _canonical_ chain to the FCU's head, which is // part of the canonical chain. We need to update the latest block state to reflect // the canonical ancestor. This ensures that state providers and the transaction @@ -1348,6 +1356,10 @@ where // Ensure we can apply a new chain update for the head block if let Some(chain_update) = self.on_new_head(state.head_block_hash)? { + if !self.is_consistent_forkchoice_state(state, Some(&chain_update))? { + return Ok(Some(TreeOutcome::new(OnForkChoiceUpdated::invalid_state()))); + } + let tip = chain_update.tip().clone_sealed_header(); self.on_canonical_chain_update(chain_update); @@ -3373,6 +3385,67 @@ where Ok(canonical) } + /// Checks that nonzero safe and finalized hashes belong to the chain defined by the FCU head. + /// + /// The [Engine API forkchoiceUpdated specification] requires `-38002` when a `VALID` head's + /// safe or finalized hash is outside that chain, and requires all forkchoice state updates to + /// be atomic. In direct FCU processing, this check must therefore run before canonicalizing + /// the head or updating either marker; checking only after canonicalization would leave an + /// invalid reorg applied. + /// + /// `chain_update` describes a proposed commit or reorg that has not yet been applied. Its new + /// blocks and the canonical prefix below its first block form the proposed chain. Without a + /// chain update, the proposed head is already canonical, so only canonical blocks through its + /// height are eligible. For a canonical-prefix hash, the current in-memory or persisted + /// canonical hash must match too: a stale persisted header may still be found by hash while + /// disk reorg cleanup is pending. A zero safe or finalized hash leaves that marker unchanged. + /// Returns `Ok(false)` for an unknown or off-chain hash and propagates provider errors. + /// + /// [Engine API forkchoiceUpdated specification]: https://github.com/ethereum/execution-apis/blob/main/src/engine/paris.md#specification-1 + fn is_consistent_forkchoice_state( + &self, + state: ForkchoiceState, + chain_update: Option<&NewCanonicalChain>, + ) -> ProviderResult { + let canonical_head_number = match chain_update { + Some(chain_update) => { + let new = chain_update.new_blocks(); + // Only the canonical prefix below the new branch remains on the proposed chain. + new.first().expect("non empty chain").block_number() - 1 + } + None => { + let Some(head) = self.find_canonical_header(state.head_block_hash)? else { + return Ok(false) + }; + head.number() + } + }; + + for hash in [state.finalized_block_hash, state.safe_block_hash] { + if hash.is_zero() || chain_update.is_some_and(|update| update.contains(hash)) { + continue + } + let Some(header) = self.find_canonical_header(hash)? else { return Ok(false) }; + if header.number() > canonical_head_number { + return Ok(false) + } + + // A persisted header can still be found by hash while its disk reorg is pending. + // Prefer the in-memory canonical hash at this height over the persisted one. + let canonical_hash = if let Some(hash) = + self.canonical_in_memory_state.hash_by_number(header.number()) + { + Some(hash) + } else { + self.provider.block_hash(header.number())? + }; + if canonical_hash != Some(hash) { + return Ok(false) + } + } + Ok(true) + } + /// Updates the tracked finalized block if we have it. fn update_finalized_block( &self, diff --git a/crates/engine/tree/src/tree/tests.rs b/crates/engine/tree/src/tree/tests.rs index 6830500088a..d78cfe77030 100644 --- a/crates/engine/tree/src/tree/tests.rs +++ b/crates/engine/tree/src/tree/tests.rs @@ -3350,3 +3350,39 @@ async fn test_on_backfill_sync_finished_opstack_retriggers_backfill_to_buffered_ async fn test_on_backfill_sync_finished_eth_retriggers_backfill_to_buffered_finalized() { assert_post_backfill_recheck_retriggers_to_buffered_target(EngineApiKind::Ethereum).await; } + +/// A stale persisted hash must not count as an ancestor after an in-memory reorg. +#[test] +fn test_forkchoice_rejects_stale_persisted_prefix_hash() { + let mut test_harness = TestHarness::new(MAINNET.clone()); + let old: Vec<_> = test_harness.block_builder.get_executed_blocks(0..3).collect(); + test_harness = test_harness.with_blocks(old.clone()); + + let new: Vec<_> = test_harness + .block_builder + .create_fork(old[0].recovered_block(), 2) + .into_iter() + .map(|block| { + ExecutedBlock::new( + Arc::new(block), + Arc::new(BlockExecutionOutput::default()), + ComputedTrieData::default(), + ) + }) + .collect(); + test_harness.tree.canonical_in_memory_state.update_chain(NewCanonicalChain::Reorg { + new: vec![new[0].clone()], + old: old[1..].to_vec(), + }); + + let state = ForkchoiceState { + head_block_hash: new[1].recovered_block().hash(), + safe_block_hash: old[1].recovered_block().hash(), + finalized_block_hash: old[0].recovered_block().hash(), + }; + let update = NewCanonicalChain::Commit { new: vec![new[1].clone()] }; + assert!(!test_harness.tree.is_consistent_forkchoice_state(state, Some(&update)).unwrap()); + + let state = ForkchoiceState { safe_block_hash: new[0].recovered_block().hash(), ..state }; + assert!(test_harness.tree.is_consistent_forkchoice_state(state, Some(&update)).unwrap()); +} diff --git a/crates/ethereum/node/tests/e2e/forkchoice.rs b/crates/ethereum/node/tests/e2e/forkchoice.rs new file mode 100644 index 00000000000..e9ff2ef457f --- /dev/null +++ b/crates/ethereum/node/tests/e2e/forkchoice.rs @@ -0,0 +1,161 @@ +//! Tests for atomic forkchoice state updates via the Engine API. + +use crate::utils::eth_payload_attributes; +use alloy_eips::BlockNumberOrTag; +use alloy_primitives::B256; +use alloy_provider::{Provider, ProviderBuilder}; +use alloy_rpc_types_engine::{ForkchoiceState, PayloadStatusEnum}; +use jsonrpsee_core::client::Error; +use reth_chainspec::{ChainSpecBuilder, MAINNET}; +use reth_e2e_test_utils::E2ETestSetupBuilder; +use reth_node_ethereum::{EthEngineTypes, EthereumNode}; +use reth_rpc_api::{EngineApiClient, TestingBuildBlockRequestV1}; +use reth_rpc_server_types::{RethRpcModule, RpcModuleSelection}; +use std::sync::Arc; + +#[tokio::test] +async fn invalid_forkchoice_preserves_canonical_state() -> eyre::Result<()> { + reth_tracing::init_test_tracing(); + + let chain_spec = Arc::new( + ChainSpecBuilder::default() + .chain(MAINNET.chain) + .genesis(serde_json::from_str(include_str!("../assets/genesis.json")).unwrap()) + .cancun_activated() + .build(), + ); + let (mut nodes, _) = + E2ETestSetupBuilder::::new(2, chain_spec, eth_payload_attributes) + .with_connect_nodes(false) + .with_node_config_modifier(|mut config| { + config.rpc.http_api = + Some(RpcModuleSelection::from([RethRpcModule::Eth, RethRpcModule::Testing])); + config + }) + .build() + .await?; + let node = nodes.pop().unwrap(); + let producer = nodes.pop().unwrap(); + let genesis = node.block_hash(0); + let engine = node.auth_server_handle().http_client(); + let producer_engine = producer.auth_server_handle().http_client(); + let rpc = ProviderBuilder::new().connect_http(node.rpc_url()); + + // Build a1 <- a2 <- a3 and b1 <- b2, both rooted at genesis. Only chain A is canonical. + let mut chains = [vec![genesis], vec![genesis]]; + for (branch, length) in [3, 2].into_iter().enumerate() { + for number in 1..=length { + let envelope = producer + .testing_build_block_v1(TestingBuildBlockRequestV1 { + parent_block_hash: *chains[branch].last().unwrap(), + payload_attributes: eth_payload_attributes(number + branch as u64 * 10), + transactions: vec![], + extra_data: None, + }) + .await?; + let payload = envelope.execution_payload; + let hash = payload.payload_inner.payload_inner.block_hash; + for client in [&producer_engine, &engine] { + let status = EngineApiClient::::new_payload_v3( + client, + payload.clone(), + vec![], + B256::ZERO, + ) + .await?; + assert_eq!(status.status, PayloadStatusEnum::Valid); + } + producer.update_forkchoice(genesis, hash).await?; + chains[branch].push(hash); + } + if branch == 0 { + let status = EngineApiClient::::fork_choice_updated_v3( + &engine, + ForkchoiceState { + head_block_hash: chains[0][3], + safe_block_hash: genesis, + finalized_block_hash: genesis, + }, + None, + ) + .await?; + assert_eq!(status.payload_status.status, PayloadStatusEnum::Valid); + } + } + let [a, b] = chains; + + for (head, safe, finalized) in [ + (b[2], a[2], a[1]), + (b[2], a[2], b[1]), + (b[2], b[1], a[1]), + (b[2], B256::repeat_byte(0xff), b[1]), + (a[3], b[1], a[1]), + (a[2], a[3], genesis), + ] { + let state = ForkchoiceState { + head_block_hash: head, + safe_block_hash: safe, + finalized_block_hash: finalized, + }; + let err = EngineApiClient::::fork_choice_updated_v3(&engine, state, None) + .await + .unwrap_err(); + let Error::Call(err) = err else { panic!("Expected an RPC error, got {err:?}") }; + assert_eq!(err.code(), -38002, "{state:?}"); + + for (tag, hash) in [ + (BlockNumberOrTag::Latest, a[3]), + (BlockNumberOrTag::Safe, genesis), + (BlockNumberOrTag::Finalized, genesis), + (BlockNumberOrTag::Number(1), a[1]), + (BlockNumberOrTag::Number(2), a[2]), + (BlockNumberOrTag::Number(3), a[3]), + ] { + assert_eq!( + rpc.get_block_by_number(tag).await?.unwrap().header.hash, + hash, + "{tag:?} changed after rejecting {state:?}" + ); + } + } + + // Safe/finalized hashes on the proposed branch must be accepted before it is canonical. + let status = EngineApiClient::::fork_choice_updated_v3( + &engine, + ForkchoiceState { + head_block_hash: b[2], + safe_block_hash: b[2], + finalized_block_hash: b[1], + }, + None, + ) + .await?; + assert_eq!(status.payload_status.status, PayloadStatusEnum::Valid); + for (tag, hash) in [ + (BlockNumberOrTag::Latest, b[2]), + (BlockNumberOrTag::Safe, b[2]), + (BlockNumberOrTag::Finalized, b[1]), + (BlockNumberOrTag::Number(1), b[1]), + (BlockNumberOrTag::Number(2), b[2]), + ] { + assert_eq!(rpc.get_block_by_number(tag).await?.unwrap().header.hash, hash); + } + assert!(rpc.get_block_by_number(BlockNumberOrTag::Number(3)).await?.is_none()); + + // Invalid payload attributes must not roll back an otherwise valid forkchoice update. + let err = EngineApiClient::::fork_choice_updated_v3( + &engine, + ForkchoiceState::same_hash(b[2]), + Some(eth_payload_attributes(1)), + ) + .await + .unwrap_err(); + let Error::Call(err) = err else { panic!("Expected an RPC error, got {err:?}") }; + assert_eq!(err.code(), -38003); + assert_eq!( + rpc.get_block_by_number(BlockNumberOrTag::Finalized).await?.unwrap().header.hash, + b[2] + ); + + Ok(()) +} diff --git a/crates/ethereum/node/tests/e2e/main.rs b/crates/ethereum/node/tests/e2e/main.rs index 045998ffedc..7272b4fade8 100644 --- a/crates/ethereum/node/tests/e2e/main.rs +++ b/crates/ethereum/node/tests/e2e/main.rs @@ -5,6 +5,7 @@ mod custom_genesis; mod dev; mod eip6780; mod eth; +mod forkchoice; mod invalid_payload; mod p2p; mod pool; From 1320f9251ab1f029901461ae09b446d08389feae Mon Sep 17 00:00:00 2001 From: banteg <4562643+banteg@users.noreply.github.com> Date: Fri, 25 Sep 2026 09:27:42 +0000 Subject: [PATCH 286/294] fix(engine): don't treat stale persisted fcu head as canonical (#27429) Co-authored-by: Matthias Seitz <19890894+mattsse@users.noreply.github.com> (cherry picked from commit 444e867155c2a7a4c028e479ca7201b0b6871579) --- crates/engine/tree/src/tree/mod.rs | 43 +++--- crates/engine/tree/src/tree/tests.rs | 61 ++++++++- crates/ethereum/node/tests/e2e/forkchoice.rs | 133 ++++++++++++++++++- 3 files changed, 213 insertions(+), 24 deletions(-) diff --git a/crates/engine/tree/src/tree/mod.rs b/crates/engine/tree/src/tree/mod.rs index b91e86a65ea..9e16234ec2e 100644 --- a/crates/engine/tree/src/tree/mod.rs +++ b/crates/engine/tree/src/tree/mod.rs @@ -3372,17 +3372,33 @@ where } /// Attempts to find the header for the given block hash if it is canonical. + /// + /// A persisted header can still be found by hash while its disk reorg is pending, so it is + /// only canonical if it is not above the canonical head and matches the canonical hash at its + /// height. The in-memory canonical hash is preferred over the persisted one. pub fn find_canonical_header( &self, hash: B256, ) -> Result>, ProviderError> { - let mut canonical = self.canonical_in_memory_state.header_by_hash(hash); + if let Some(header) = self.canonical_in_memory_state.header_by_hash(hash) { + return Ok(Some(header)) + } - if canonical.is_none() { - canonical = self.provider.header(hash)?.map(|header| SealedHeader::new(header, hash)); + let Some(header) = self.provider.sealed_header_by_hash(hash)? else { return Ok(None) }; + // Reorged-out headers remain on disk until persistence catches up, so reject old tips above + // the in-memory head and use in-memory hashes to reject old blocks at shared heights. + let number = header.number(); + if number > self.canonical_in_memory_state.get_canonical_block_number() { + return Ok(None) } + let canonical_hash = + if let Some(hash) = self.canonical_in_memory_state.hash_by_number(number) { + Some(hash) + } else { + self.provider.block_hash(number)? + }; - Ok(canonical) + Ok((canonical_hash == Some(hash)).then_some(header)) } /// Checks that nonzero safe and finalized hashes belong to the chain defined by the FCU head. @@ -3396,9 +3412,9 @@ where /// `chain_update` describes a proposed commit or reorg that has not yet been applied. Its new /// blocks and the canonical prefix below its first block form the proposed chain. Without a /// chain update, the proposed head is already canonical, so only canonical blocks through its - /// height are eligible. For a canonical-prefix hash, the current in-memory or persisted - /// canonical hash must match too: a stale persisted header may still be found by hash while - /// disk reorg cleanup is pending. A zero safe or finalized hash leaves that marker unchanged. + /// height are eligible. Canonical-prefix hashes are resolved via + /// [`Self::find_canonical_header`], which rejects stale persisted headers whose disk reorg + /// cleanup is pending. A zero safe or finalized hash leaves that marker unchanged. /// Returns `Ok(false)` for an unknown or off-chain hash and propagates provider errors. /// /// [Engine API forkchoiceUpdated specification]: https://github.com/ethereum/execution-apis/blob/main/src/engine/paris.md#specification-1 @@ -3429,19 +3445,6 @@ where if header.number() > canonical_head_number { return Ok(false) } - - // A persisted header can still be found by hash while its disk reorg is pending. - // Prefer the in-memory canonical hash at this height over the persisted one. - let canonical_hash = if let Some(hash) = - self.canonical_in_memory_state.hash_by_number(header.number()) - { - Some(hash) - } else { - self.provider.block_hash(header.number())? - }; - if canonical_hash != Some(hash) { - return Ok(false) - } } Ok(true) } diff --git a/crates/engine/tree/src/tree/tests.rs b/crates/engine/tree/src/tree/tests.rs index d78cfe77030..e4b88ebc1aa 100644 --- a/crates/engine/tree/src/tree/tests.rs +++ b/crates/engine/tree/src/tree/tests.rs @@ -30,7 +30,9 @@ use reth_ethereum_primitives::{Block, EthPrimitives}; use reth_evm_ethereum::MockEvmConfig; use reth_payload_builder::PayloadServiceCommand; use reth_primitives_traits::Block as _; -use reth_provider::{test_utils::MockEthProvider, BalStoreHandle, InMemoryBalStore, RawBal}; +use reth_provider::{ + test_utils::MockEthProvider, BalStoreHandle, HeaderProvider, InMemoryBalStore, RawBal, +}; use reth_tasks::spawn_os_thread; use reth_trie_common::ComputedTrieData; use std::{ @@ -3386,3 +3388,60 @@ fn test_forkchoice_rejects_stale_persisted_prefix_hash() { let state = ForkchoiceState { safe_block_hash: new[0].recovered_block().hash(), ..state }; assert!(test_harness.tree.is_consistent_forkchoice_state(state, Some(&update)).unwrap()); } + +/// Reorgs from a persisted chain to a sibling branch of `sibling_len` blocks without advancing +/// persistence, then asserts that an FCU back to the reorged-out tip makes it canonical again. +async fn assert_fcu_back_to_reorged_out_head_with_pending_disk_reorg(sibling_len: u64) { + reth_tracing::init_test_tracing(); + let mut test_harness = TestHarness::new(MAINNET.clone()); + let old: Vec<_> = test_harness.block_builder.get_executed_blocks(0..4).collect(); + test_harness = test_harness.with_blocks(old.clone()); + let old_tip = old[3].recovered_block().hash(); + + let new: Vec<_> = test_harness + .block_builder + .create_fork(old[0].recovered_block(), sibling_len) + .into_iter() + .map(|block| { + ExecutedBlock::new( + Arc::new(block), + Arc::new(BlockExecutionOutput::default()), + ComputedTrieData::default(), + ) + }) + .collect(); + for block in &new { + test_harness.tree.state.tree_state.insert_executed(block.clone()); + } + let new_tip = new.last().unwrap().recovered_block().hash(); + + let fcu = |head_block_hash| ForkchoiceState { + head_block_hash, + safe_block_hash: B256::ZERO, + finalized_block_hash: B256::ZERO, + }; + + let outcome = test_harness.tree.on_forkchoice_updated(fcu(new_tip), None).unwrap(); + assert_eq!(outcome.outcome.forkchoice_status(), ForkchoiceStatus::Valid); + assert_eq!(test_harness.tree.canonical_in_memory_state.get_canonical_head().hash(), new_tip); + + // Persistence is not advanced, so the reorged-out blocks are still found on disk. + assert!(test_harness.provider.header(old_tip).unwrap().is_some()); + + let outcome = test_harness.tree.on_forkchoice_updated(fcu(old_tip), None).unwrap(); + assert_eq!(outcome.outcome.forkchoice_status(), ForkchoiceStatus::Valid); + assert_eq!(test_harness.tree.state.tree_state.canonical_block_hash(), old_tip); + assert_eq!(test_harness.tree.canonical_in_memory_state.get_canonical_head().hash(), old_tip); +} + +/// A reorged-out head must become canonical again while its disk reorg is still pending. +#[tokio::test] +async fn test_fcu_back_to_reorged_out_head_with_pending_disk_reorg() { + assert_fcu_back_to_reorged_out_head_with_pending_disk_reorg(3).await; +} + +/// Same as above, but the reorged-out head is above the tip of the shorter sibling branch. +#[tokio::test] +async fn test_fcu_back_to_reorged_out_head_above_shorter_branch_with_pending_disk_reorg() { + assert_fcu_back_to_reorged_out_head_with_pending_disk_reorg(1).await; +} diff --git a/crates/ethereum/node/tests/e2e/forkchoice.rs b/crates/ethereum/node/tests/e2e/forkchoice.rs index e9ff2ef457f..6e0e5806922 100644 --- a/crates/ethereum/node/tests/e2e/forkchoice.rs +++ b/crates/ethereum/node/tests/e2e/forkchoice.rs @@ -1,4 +1,4 @@ -//! Tests for atomic forkchoice state updates via the Engine API. +//! Tests for canonical chain and atomic forkchoice state updates via the Engine API. use crate::utils::eth_payload_attributes; use alloy_eips::BlockNumberOrTag; @@ -7,11 +7,12 @@ use alloy_provider::{Provider, ProviderBuilder}; use alloy_rpc_types_engine::{ForkchoiceState, PayloadStatusEnum}; use jsonrpsee_core::client::Error; use reth_chainspec::{ChainSpecBuilder, MAINNET}; -use reth_e2e_test_utils::E2ETestSetupBuilder; +use reth_e2e_test_utils::{trie::wait_for_persisted_block, E2ETestSetupBuilder}; use reth_node_ethereum::{EthEngineTypes, EthereumNode}; +use reth_provider::{DatabaseProviderFactory, HeaderProvider}; use reth_rpc_api::{EngineApiClient, TestingBuildBlockRequestV1}; use reth_rpc_server_types::{RethRpcModule, RpcModuleSelection}; -use std::sync::Arc; +use std::{sync::Arc, time::Duration}; #[tokio::test] async fn invalid_forkchoice_preserves_canonical_state() -> eyre::Result<()> { @@ -159,3 +160,129 @@ async fn invalid_forkchoice_preserves_canonical_state() -> eyre::Result<()> { Ok(()) } + +/// Restoring a persisted head must update canonical state before disk reorg cleanup completes. +#[tokio::test] +async fn fcu_restores_reorged_out_persisted_head() -> eyre::Result<()> { + assert_fcu_restores_reorged_out_persisted_head(3).await +} + +/// A stale persisted head above a shorter sibling branch must also become canonical again. +#[tokio::test] +async fn fcu_restores_reorged_out_persisted_head_above_shorter_branch() -> eyre::Result<()> { + assert_fcu_restores_reorged_out_persisted_head(1).await +} + +/// Holds the database writer lock across both FCUs so disk cleanup cannot win the race. +async fn assert_fcu_restores_reorged_out_persisted_head(sibling_len: u64) -> eyre::Result<()> { + reth_tracing::init_test_tracing(); + + let chain_spec = Arc::new( + ChainSpecBuilder::default() + .chain(MAINNET.chain) + .genesis(serde_json::from_str(include_str!("../assets/genesis.json")).unwrap()) + .cancun_activated() + .build(), + ); + let (mut nodes, _) = + E2ETestSetupBuilder::::new(2, chain_spec, eth_payload_attributes) + .with_connect_nodes(false) + .with_tree_config_modifier(|config| { + config + .with_num_state_masking_blocks(0) + .with_persistence_threshold(0) + .with_memory_block_buffer_target(0) + }) + .with_node_config_modifier(|mut config| { + config.rpc.http_api = + Some(RpcModuleSelection::from([RethRpcModule::Eth, RethRpcModule::Testing])); + config + }) + .build() + .await?; + let node = nodes.pop().unwrap(); + let producer = nodes.pop().unwrap(); + let genesis = node.block_hash(0); + let engine = node.auth_server_handle().http_client(); + let producer_engine = producer.auth_server_handle().http_client(); + let rpc = ProviderBuilder::new().connect_http(node.rpc_url()); + + // Persist A through block 3, then import a sibling branch without canonicalizing it. + let mut chains = [vec![genesis], vec![genesis]]; + for (branch, length) in [3, sibling_len].into_iter().enumerate() { + for number in 1..=length { + let envelope = producer + .testing_build_block_v1(TestingBuildBlockRequestV1 { + parent_block_hash: *chains[branch].last().unwrap(), + payload_attributes: eth_payload_attributes(number + branch as u64 * 10), + transactions: vec![], + extra_data: None, + }) + .await?; + let payload = envelope.execution_payload; + let hash = payload.payload_inner.payload_inner.block_hash; + for client in [&producer_engine, &engine] { + let status = EngineApiClient::::new_payload_v3( + client, + payload.clone(), + vec![], + B256::ZERO, + ) + .await?; + assert_eq!(status.status, PayloadStatusEnum::Valid); + } + producer.update_forkchoice(genesis, hash).await?; + chains[branch].push(hash); + } + if branch == 0 { + node.update_forkchoice(genesis, chains[0][3]).await?; + wait_for_persisted_block(&node.inner.provider, 3, Duration::from_secs(30)).await?; + } + } + let [a, b] = chains; + + // MDBX allows only one writer. Acquire it off the executor and keep it alive while the + // persistence task attempts to remove A, leaving the old headers visible on disk. + let provider = node.inner.provider.clone(); + let disk_reorg_guard = + tokio::task::spawn_blocking(move || provider.database_provider_rw()).await??; + for head in [*b.last().unwrap(), a[3]] { + let status = tokio::time::timeout( + Duration::from_secs(10), + EngineApiClient::::fork_choice_updated_v3( + &engine, + ForkchoiceState { + head_block_hash: head, + safe_block_hash: B256::ZERO, + finalized_block_hash: B256::ZERO, + }, + None, + ), + ) + .await??; + assert_eq!(status.payload_status.status, PayloadStatusEnum::Valid); + assert_eq!( + node.inner.provider.database_provider_ro()?.sealed_header(3)?.unwrap().hash(), + a[3], + "The old tip must remain on disk throughout both FCUs" + ); + assert_eq!( + rpc.get_block_by_number(BlockNumberOrTag::Latest).await?.unwrap().header.hash, + head, + "A VALID FCU must make the requested sibling tip canonical" + ); + } + for (number, hash) in a.into_iter().enumerate() { + assert_eq!( + rpc.get_block_by_number(BlockNumberOrTag::Number(number as u64)) + .await? + .unwrap() + .header + .hash, + hash + ); + } + drop(disk_reorg_guard); + + Ok(()) +} From 68594096cc4b8a53bce48485c56994d59b21642a Mon Sep 17 00:00:00 2001 From: Derek Cofausper <256792747+decofe@users.noreply.github.com> Date: Thu, 1 Oct 2026 15:44:08 +0000 Subject: [PATCH 287/294] fix(storage): use trie changesets for all overlays (#27634) Co-authored-by: Brian Picciano <933154+mediocregopher@users.noreply.github.com> (cherry picked from commit 078d026239a1977f5d79eeb67fa44d2087ebf7e5) --- crates/storage/storage-overlay/src/provider.rs | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/crates/storage/storage-overlay/src/provider.rs b/crates/storage/storage-overlay/src/provider.rs index af04e75adb4..a856f956b92 100644 --- a/crates/storage/storage-overlay/src/provider.rs +++ b/crates/storage/storage-overlay/src/provider.rs @@ -534,14 +534,14 @@ where fn state_root(&self, hashed_state: HashedPostState) -> ProviderResult { reth_trie_db::with_adapter!(self.provider(), |A| { let input = - self.build_overlay(TrieInputSorted::from_state(hashed_state.into_sorted()), false)?; + self.build_overlay(TrieInputSorted::from_state(hashed_state.into_sorted()), true)?; Ok(>::overlay_root_from_nodes(self.provider().tx(), input)?) }) } fn state_root_from_nodes(&self, input: TrieInput) -> ProviderResult { reth_trie_db::with_adapter!(self.provider(), |A| { - let input = self.build_overlay(TrieInputSorted::from_unsorted(input), false)?; + let input = self.build_overlay(TrieInputSorted::from_unsorted(input), true)?; Ok( as DatabaseStateRoot<_>>::overlay_root_from_nodes( self.provider().tx(), input, @@ -602,7 +602,7 @@ where HashedPostState::from_hashed_storage(hashed_address, hashed_storage) .into_sorted(), ), - false, + true, )?; >::overlay_root(self.provider().tx(), address, input) .map_err(|err| ProviderError::Database(err.into())) @@ -633,7 +633,7 @@ where HashedPostState::from_hashed_storage(hashed_address, hashed_storage) .into_sorted(), ), - false, + true, )?; >::overlay_storage_multiproof( self.provider().tx(), @@ -665,7 +665,7 @@ where ) -> ProviderResult { reth_trie_db::with_adapter!(self.provider(), |A| { let TrieInputSorted { nodes, state, prefix_sets } = - self.build_overlay(TrieInputSorted::from_unsorted(input), false)?; + self.build_overlay(TrieInputSorted::from_unsorted(input), true)?; let input = TrieInput::new( Arc::unwrap_or_clone(nodes).into(), Arc::unwrap_or_clone(state).into(), @@ -683,7 +683,7 @@ where ) -> ProviderResult { reth_trie_db::with_adapter!(self.provider(), |A| { let TrieInputSorted { nodes, state, prefix_sets } = - self.build_overlay(TrieInputSorted::from_unsorted(input), false)?; + self.build_overlay(TrieInputSorted::from_unsorted(input), true)?; let input = TrieInput::new( Arc::unwrap_or_clone(nodes).into(), Arc::unwrap_or_clone(state).into(), @@ -701,7 +701,7 @@ where ) -> ProviderResult { reth_trie_db::with_adapter!(self.provider(), |A| { let TrieInputSorted { nodes, state, prefix_sets } = - self.build_overlay(TrieInputSorted::from_unsorted(input), false)?; + self.build_overlay(TrieInputSorted::from_unsorted(input), true)?; let input = TrieInput::new( Arc::unwrap_or_clone(nodes).into(), Arc::unwrap_or_clone(state).into(), @@ -720,7 +720,7 @@ where ) -> ProviderResult> { reth_trie_db::with_adapter!(self.provider(), |A| { let TrieInputSorted { nodes, state, prefix_sets } = - self.build_overlay(TrieInputSorted::from_unsorted(input), false)?; + self.build_overlay(TrieInputSorted::from_unsorted(input), true)?; let witness = TrieWitness::new( InMemoryTrieCursorFactory::new( DatabaseTrieCursorFactory::<_, A>::new(self.provider().tx()), @@ -769,7 +769,7 @@ where return Ok(hashed_state) } - let overlay_state = self.build_overlay(TrieInputSorted::default(), false)?.state; + let overlay_state = self.build_overlay(TrieInputSorted::default(), true)?.state; zero_destroyed_account_storage( &HashedPostStateCursorFactory::new( DatabaseHashedCursorFactory::new(self.provider().tx()), From 0b1f6f3949290490a5620d217052929c445dbe14 Mon Sep 17 00:00:00 2001 From: Christopher Campbell Date: Wed, 23 Sep 2026 09:49:23 +0000 Subject: [PATCH 288/294] fix(trie): state root task persists EIP-161-empty accounts into HashedAccounts (#27252) Co-authored-by: Claude Co-authored-by: Brian Picciano (cherry picked from commit 695b4f174b495e5a2bbc8779165422a678858bd7) --- crates/trie/parallel/src/state_root_task.rs | 200 +++++++++++++++++++- 1 file changed, 196 insertions(+), 4 deletions(-) diff --git a/crates/trie/parallel/src/state_root_task.rs b/crates/trie/parallel/src/state_root_task.rs index 5eb920317a9..382124aa63a 100644 --- a/crates/trie/parallel/src/state_root_task.rs +++ b/crates/trie/parallel/src/state_root_task.rs @@ -497,9 +497,19 @@ pub fn evm_state_to_hashed_post_state(update: EvmState) -> HashedPostState { trace!(target: "trie::parallel::sparse", ?address, ?hashed_address, "Adding account to state update"); let destroyed = account.is_selfdestructed(); - if account.info != account.original_info() { - let info = if destroyed { None } else { Some(account.info.into()) }; - hashed_state.accounts.insert(hashed_address, info); + // EIP-161: a touched account that ends up empty is deleted, so it must be emitted + // as a removal rather than as an all-zero account. This mirrors what revm does in + // the bundle path (`CacheAccount::touch_empty_eip161`) and what the sibling + // producer for this consumer already does in `send_bal_hashed_state`. + // An address that never existed and still does not exist is not a deletion: revm + // emits no transition for `LoadedNotExisting`. Skipping it matches the bundle + // producer; every `None` here becomes a storage-trie cursor walk in `StateRoot`. + let deleted = destroyed || (account.is_empty() && !account.is_loaded_as_not_existing()); + if deleted { + hashed_state.accounts.insert(hashed_address, None); + } else if account.info != account.original_info() { + // A touched but unchanged account produces no bundle transition either. + hashed_state.accounts.insert(hashed_address, Some(account.info.into())); } let mut changed_storage_iter = account @@ -533,7 +543,7 @@ mod tests { #[test] fn created_selfdestruct_does_not_emit_storage() { let address = Address::repeat_byte(0x01); - let mut account = Account::default(); + let mut account = Account::new_not_existing(TransactionId::ZERO); account.mark_touch(); assert!(account.mark_created_locally()); assert!(account.mark_selfdestructed_locally()); @@ -573,6 +583,188 @@ mod tests { assert!(!hashed_state.storages.contains_key(&hashed_address)); } + /// An account drained to zero balance, with nonce 0 and no code, is EIP-161-empty and + /// canonical execution deletes it. Asserts the converter reports the deletion as `None`, + /// matching `HashedPostState::from_bundle_state`, so `write_hashed_state` removes the + /// `HashedAccounts` row instead of upserting an all-zero one. + #[test] + fn emptied_account_is_deleted() { + let address = Address::repeat_byte(0x05); + let mut account = Account::default(); + // Pre-state: the account exists and holds a balance. + account.info.balance = U256::from(1); + account.set_current_info_as_original(); + // This block drains it. Not selfdestructed: an ordinary value transfer out. + account.mark_touch(); + account.info.balance = U256::ZERO; + assert!(account.is_empty(), "the drained account must be EIP-161-empty"); + assert!(!account.is_selfdestructed()); + + let hashed_state = + evm_state_to_hashed_post_state(EvmState::from_iter([(address, account)])); + + assert_eq!(hashed_state.accounts.get(&keccak256(address)), Some(&None)); + } + + /// A pre-existing EIP-161-empty account that is merely touched must be deleted. revm marks it + /// for removal (`touch_empty_eip161`) and the `BundleState` path reports `None`. Asserts the + /// converter emits the deletion even though `info == original_info()`, so an existing all-zero + /// `HashedAccounts` row can be cleared by a touch. + #[test] + fn touched_preexisting_empty_account_is_deleted() { + let address = Address::repeat_byte(0x06); + // Empty in the pre-state too: nonce 0, balance 0, no code. + let mut account = Account::default(); + account.set_current_info_as_original(); + account.mark_touch(); + assert!(account.is_empty()); + + let hashed_state = + evm_state_to_hashed_post_state(EvmState::from_iter([(address, account)])); + + assert_eq!(hashed_state.accounts.get(&keccak256(address)), Some(&None)); + } + + /// The two `HashedPostState` producers that feed the same consumer must agree. + /// + /// `evm_state_to_hashed_post_state` converts the raw `EvmState` handed to the state hook; + /// `HashedPostState::from_bundle_state` converts the `BundleState` revm produces from the + /// same execution. Since "perf: avoid hashing the state twice" the engine persists whichever + /// one it gets, so a disagreement between them is a disagreement about durable state. + /// + /// Asserts both producers agree for a touched account drained to EIP-161-empty, which revm + /// marks for removal in the bundle path (`CacheAccount::touch_empty_eip161`). + #[test] + fn matches_bundle_state_for_emptied_account() { + use revm::{ + database::{states::bundle_state::BundleRetention, State}, + state::AccountInfo, + DatabaseCommit, + }; + + let address = Address::repeat_byte(0x07); + let pre = AccountInfo { balance: U256::from(1), ..Default::default() }; + + // The EvmState the state hook observes: a funded account drained to empty. + let mut account = Account::from(pre.clone()); + account.mark_touch(); + account.info.balance = U256::ZERO; + let evm_state = EvmState::from_iter([(address, account)]); + + // Same execution, through revm's own bundle machinery. + let mut db = State::builder().with_bundle_update().build(); + db.insert_account(address, pre); + db.commit(evm_state.clone()); + db.merge_transitions(BundleRetention::PlainState); + let bundle = db.take_bundle(); + + let from_bundle = + HashedPostState::from_bundle_state::(bundle.state.iter()); + let from_hook = evm_state_to_hashed_post_state(evm_state); + + assert_eq!( + from_hook.accounts, from_bundle.accounts, + "state-hook and bundle producers disagree about durable account state" + ); + } + + /// An account created during the block whose final info is empty. EIP-161 deletes it, so + /// the bundle producer reports a removal. + #[test] + fn created_empty_account_matches_bundle_state() { + use revm::{ + database::{states::bundle_state::BundleRetention, State}, + DatabaseCommit, + }; + + let address = Address::repeat_byte(0x08); + let mut account = Account::default(); + account.mark_touch(); + assert!(account.mark_created_locally()); + assert!(account.is_empty()); + let evm_state = EvmState::from_iter([(address, account)]); + + let mut db = State::builder().with_bundle_update().build(); + db.commit(evm_state.clone()); + db.merge_transitions(BundleRetention::PlainState); + let bundle = db.take_bundle(); + + let from_bundle = + HashedPostState::from_bundle_state::(bundle.state.iter()); + let from_hook = evm_state_to_hashed_post_state(evm_state); + + assert_eq!( + from_hook.accounts.get(&keccak256(address)).copied().flatten(), + from_bundle.accounts.get(&keccak256(address)).copied().flatten(), + "state-hook and bundle producers disagree about a created-empty account" + ); + } + + /// A zero-value call to an address that never existed leaves it non-existent. revm's bundle + /// path emits no transition for it (`CacheAccount::touch_empty_eip161` returns `None` for + /// `LoadedNotExisting`). Asserts the converter also emits nothing, rather than a deletion + /// that would queue a `HashedAccounts` delete and a storage-trie wipe for an unused address. + #[test] + fn touched_never_existing_account_matches_bundle_state() { + use revm::{ + database::{states::bundle_state::BundleRetention, State}, + DatabaseCommit, + }; + + let address = Address::repeat_byte(0x09); + let mut account = Account::new_not_existing(TransactionId::default()); + account.mark_touch(); + let evm_state = EvmState::from_iter([(address, account)]); + + let mut db = State::builder().with_bundle_update().build(); + db.commit(evm_state.clone()); + db.merge_transitions(BundleRetention::PlainState); + let bundle = db.take_bundle(); + + let from_bundle = + HashedPostState::from_bundle_state::(bundle.state.iter()); + let from_hook = evm_state_to_hashed_post_state(evm_state); + + assert_eq!( + from_hook.accounts, from_bundle.accounts, + "state-hook and bundle producers disagree about a never-existing touched account" + ); + } + + /// An account touched without being changed. revm's bundle producer reports nothing for it. + /// Asserts the converter agrees, so a block full of zero-value calls does not rewrite + /// unchanged `HashedAccounts` rows. + #[test] + fn touched_unchanged_account_matches_bundle_state() { + use revm::{ + database::{states::bundle_state::BundleRetention, State}, + state::AccountInfo, + DatabaseCommit, + }; + + let address = Address::repeat_byte(0x0a); + let pre = AccountInfo { balance: U256::from(7), nonce: 1, ..Default::default() }; + let mut account = Account::from(pre.clone()); + account.mark_touch(); + assert!(!account.is_empty()); + let evm_state = EvmState::from_iter([(address, account)]); + + let mut db = State::builder().with_bundle_update().build(); + db.insert_account(address, pre); + db.commit(evm_state.clone()); + db.merge_transitions(BundleRetention::PlainState); + let bundle = db.take_bundle(); + + let from_bundle = + HashedPostState::from_bundle_state::(bundle.state.iter()); + let from_hook = evm_state_to_hashed_post_state(evm_state); + + assert_eq!( + from_hook.accounts, from_bundle.accounts, + "state-hook and bundle producers disagree about a touched but unchanged account" + ); + } + #[derive(Default)] struct CountingSink { access_hints: AtomicUsize, From 56c3d375fbbb4543ff1eb702138ad01d672bd0e9 Mon Sep 17 00:00:00 2001 From: Brian Picciano Date: Wed, 7 Oct 2026 23:17:37 +0000 Subject: [PATCH 289/294] fix(trie): rewind backward overlay seeks (#27782) (cherry picked from commit 5868ee478841e1eaffc2fae740b60b7d5a920896) --- crates/trie/trie/src/hashed_cursor/post_state.rs | 4 ++++ crates/trie/trie/src/trie_cursor/in_memory.rs | 8 ++++++++ 2 files changed, 12 insertions(+) diff --git a/crates/trie/trie/src/hashed_cursor/post_state.rs b/crates/trie/trie/src/hashed_cursor/post_state.rs index a5e6e593347..d6d6c2fe906 100644 --- a/crates/trie/trie/src/hashed_cursor/post_state.rs +++ b/crates/trie/trie/src/hashed_cursor/post_state.rs @@ -299,6 +299,10 @@ where /// The returned account key is memoized and the cursor remains positioned at that key until /// [`HashedCursor::seek`] or [`HashedCursor::next`] are called. fn seek(&mut self, key: B256) -> Result, DatabaseError> { + // Forward scans can reuse cursor positions; backward seeks and exhausted cursors cannot. + if self.last_key.is_none_or(|last| key < last) { + self.reset(); + } let post_state_entry = self.post_state_cursor.seek(&key).copied().map(|(k, v)| (k, v.into_option())); diff --git a/crates/trie/trie/src/trie_cursor/in_memory.rs b/crates/trie/trie/src/trie_cursor/in_memory.rs index 1d704cc4abf..8db7853e804 100644 --- a/crates/trie/trie/src/trie_cursor/in_memory.rs +++ b/crates/trie/trie/src/trie_cursor/in_memory.rs @@ -238,6 +238,10 @@ impl TrieCursor for InMemoryTrieCursor<'_, C> { &mut self, key: Nibbles, ) -> Result, DatabaseError> { + // Forward scans can reuse cursor positions; backward seeks and exhausted cursors cannot. + if self.last_key.is_none_or(|last| key < last) { + self.reset(); + } let mem_entry = self.in_memory_cursor.seek(&key); if let Some((mem_key, entry_inner)) = mem_entry && @@ -281,6 +285,10 @@ impl TrieCursor for InMemoryTrieCursor<'_, C> { &mut self, key: Nibbles, ) -> Result, DatabaseError> { + // Forward scans can reuse cursor positions; backward seeks and exhausted cursors cannot. + if self.last_key.is_none_or(|last| key < last) { + self.reset(); + } let mem_entry = self.in_memory_cursor.seek(&key); if let Some((mem_key, Some(node))) = mem_entry && From 5f793856dd20b855973d8cdfa593d774e7534338 Mon Sep 17 00:00:00 2001 From: Derek Cofausper <256792747+decofe@users.noreply.github.com> Date: Fri, 9 Oct 2026 00:39:09 +0000 Subject: [PATCH 290/294] fix(trie): remove orphaned branches (#27832) Co-authored-by: Brian Picciano <933154+mediocregopher@users.noreply.github.com> Co-authored-by: Brian Picciano (cherry picked from commit 5e309af68bb064eb594807bf8d2b2e6638b91c27) --- crates/trie/trie/src/trie.rs | 6 + crates/trie/trie/src/walker.rs | 239 ++++++++++++++++++++++++++++++--- 2 files changed, 230 insertions(+), 15 deletions(-) diff --git a/crates/trie/trie/src/trie.rs b/crates/trie/trie/src/trie.rs index bf3144046de..9ff833dc9ce 100644 --- a/crates/trie/trie/src/trie.rs +++ b/crates/trie/trie/src/trie.rs @@ -73,6 +73,9 @@ impl StateRoot { } /// Configures the state root walker to visit all children of changed branch paths. + /// + /// When updates are retained, also removes stored descendants hidden by a changed branch's + /// tree mask. Regenerated branch updates take precedence over these deletions. pub const fn with_walk_all_changed_branch_children(mut self, enabled: bool) -> Self { self.walk_all_changed_branch_children = enabled; self @@ -586,6 +589,9 @@ impl StorageRoot { } /// Configures the storage root walker to visit all children of changed branch paths. + /// + /// When updates are retained, also removes stored descendants hidden by a changed branch's + /// tree mask. Regenerated branch updates take precedence over these deletions. pub const fn with_walk_all_changed_branch_children(mut self, enabled: bool) -> Self { self.walk_all_changed_branch_children = enabled; self diff --git a/crates/trie/trie/src/walker.rs b/crates/trie/trie/src/walker.rs index a8442578575..4702d86b3a0 100644 --- a/crates/trie/trie/src/walker.rs +++ b/crates/trie/trie/src/walker.rs @@ -4,22 +4,10 @@ use crate::{ BranchNodeCompact, Nibbles, }; use alloy_primitives::{map::HashSet, B256}; -use alloy_trie::proof::AddedRemovedKeys; +use alloy_trie::{proof::AddedRemovedKeys, TrieMask}; use reth_storage_errors::db::DatabaseError; use tracing::{instrument, trace}; -#[cfg(test)] -use crate::trie_cursor::{mock::MockTrieCursorFactory, TrieCursorFactory}; - -#[cfg(test)] -use alloy_primitives::map::B256Map; - -#[cfg(test)] -use alloy_trie::TrieMask; - -#[cfg(test)] -use std::collections::BTreeMap; - #[cfg(feature = "metrics")] use crate::metrics::WalkerMetrics; @@ -126,6 +114,10 @@ impl> TrieWalker { } /// Configures the walker to treat every child of a matching branch path as unskippable. + /// + /// When deletions are retained, also collects stored descendants hidden by a matching + /// branch's tree mask. These keys appear in the deletion set even if traversal never visits + /// them. Root calculations can replace these deletions with regenerated branch updates. pub const fn with_walk_all_changed_branch_children(mut self, enabled: bool) -> Self { self.walk_all_changed_branch_children = enabled; self @@ -354,6 +346,8 @@ impl> TrieWalker { return Ok(()) } + self.remove_orphaned_descendants(key, node.tree_mask)?; + // Create a new CursorSubNode and push it to the stack. let subnode = CursorSubNode::new(key, Some(node)); let position = subnode.position(); @@ -415,13 +409,62 @@ impl> TrieWalker { Ok(()) } + + /// Removes cached descendants hidden by a changed branch's tree mask during recovery. + /// + /// For example, a change under `0x3a` can leave stored `0x3c` and `0x3c4...` entries while + /// parent `0x3` has tree-mask bit `c` clear. The walker skips those entries and therefore never + /// records their keys for deletion. + /// + /// Collect the hidden cache keys without using their hashes; regenerated branch updates take + /// precedence over these deletions. The cursor must be positioned on this parent; restore that + /// position afterwards so traversal continues unchanged. + fn remove_orphaned_descendants( + &mut self, + key: Nibbles, + tree_mask: TrieMask, + ) -> Result<(), DatabaseError> { + let missing_children = !tree_mask; + if self.walk_all_changed_branch_children && + !missing_children.is_empty() && + let Some(removed_keys) = &mut self.removed_keys && + self.changes.contains(&key) + { + for nibble in missing_children.iter() { + let mut prefix = key; + prefix.push(nibble); + let mut entry = self.cursor.seek(prefix)?; + #[cfg(feature = "metrics")] + self.metrics.inc_branch_nodes_seeked(); + while let Some((path, _)) = entry && + path.starts_with(&prefix) + { + removed_keys.insert(path); + entry = self.cursor.next()?; + } + } + self.cursor.seek_exact(key)?; + #[cfg(feature = "metrics")] + self.metrics.inc_branch_nodes_seeked(); + } + Ok(()) + } } #[cfg(test)] mod tests { use super::*; - use crate::prefix_set::PrefixSetMut; - use alloy_primitives::B256; + use crate::{ + mock::KeyVisitType, + prefix_set::PrefixSetMut, + progress::StorageRootProgress, + test_utils::{storage_root_prehashed, TrieTestHarness}, + trie_cursor::{mock::MockTrieCursorFactory, TrieCursorFactory}, + updates::StorageTrieUpdates, + StorageRoot, + }; + use alloy_primitives::{map::B256Map, B256, U256}; + use std::collections::BTreeMap; fn branch_node(state_mask: u16, tree_mask: u16, hash_mask: u16) -> BranchNodeCompact { let hash_count = hash_mask.count_ones() as usize; @@ -495,4 +538,170 @@ mod tests { assert_eq!(walker.key().copied(), Some(Nibbles::from_nibbles([0x2, 0x4]))); assert!(!walker.can_skip_current_node); } + + #[test] + fn changed_branch_children_remove_orphans() { + for regenerate_branch in [false, true] { + let mut storage = BTreeMap::from([ + (B256::right_padding_from(&[0x3a]), U256::ONE), + (B256::right_padding_from(&[0x3a, 0x01]), U256::from(10)), + (B256::right_padding_from(&[0x3a, 0x10]), U256::from(7)), + (B256::right_padding_from(&[0x3b]), U256::from(2)), + (B256::right_padding_from(&[0x3c, 0x47]), U256::from(3)), + (B256::right_padding_from(&[0x80]), U256::from(4)), + (B256::right_padding_from(&[0x81]), U256::from(5)), + (B256::right_padding_from(&[0x80, 0x10]), U256::from(8)), + ]); + if regenerate_branch { + storage.insert(B256::right_padding_from(&[0x3c, 0x48]), U256::from(6)); + storage.insert(B256::right_padding_from(&[0x3c, 0x47, 0x10]), U256::from(9)); + } + let mut harness = TrieTestHarness::new(storage.clone()); + let expected_nodes = harness + .storage_trie_updates() + .storage_nodes + .iter() + .map(|(key, node)| (*key, node.clone())) + .collect::>(); + let mut stored_nodes = expected_nodes.clone(); + let parent = Nibbles::from_nibbles([0x3]); + stored_nodes.get_mut(&parent).unwrap().tree_mask &= + !TrieMask::new((1 << 0xa) | (1 << 0xc)); + + // The cached branch at 0x3c and its descendants are unreachable through 0x3's mask. + // Include a descendant which should be regenerated and one which should disappear. + let orphan_paths = [ + Nibbles::from_nibbles([0x3, 0xc]), + Nibbles::from_nibbles([0x3, 0xc, 0x4]), + Nibbles::from_nibbles([0x3, 0xc, 0x4, 0xf]), + Nibbles::from_nibbles([0x3, 0xf, 0x1]), + Nibbles::from_nibbles([0x3, 0x1, 0x4]), + ]; + // The valid branch under 0x3a is also hidden by the mask. Its regenerated update can + // be split off before the walker finishes 0x3, so resumption must not delete it again. + assert!(expected_nodes.contains_key(&Nibbles::from_nibbles([0x3, 0xa]))); + for path in orphan_paths { + let mut stale_node = branch_node(0b11, 0, 0b11); + stale_node.hashes = vec![B256::repeat_byte(0xee); 2].into(); + stored_nodes.insert(path, stale_node); + } + harness.set_trie_nodes(stored_nodes.clone()); + + // A change under sibling 0x3a must also clean up 0x3c and state-mask gaps. + let prefix_set = + PrefixSetMut::from([Nibbles::unpack(B256::right_padding_from(&[0x3a]))]).freeze(); + for (enabled, threshold) in [(false, u64::MAX), (true, u64::MAX), (true, 1)] { + // Also rebuild the neighboring branch in checkpointed runs. Resuming a skipped + // branch after a leaf checkpoint currently fails even without orphan cleanup. + let changes = if threshold == 1 { + PrefixSetMut::from([ + Nibbles::unpack(B256::right_padding_from(&[0x3a])), + Nibbles::unpack(B256::right_padding_from(&[0x80])), + ]) + .freeze() + } else { + prefix_set.clone() + }; + let (root, updates) = + storage_root_with_progress(&harness, changes, enabled, threshold); + + let mut resulting_nodes = stored_nodes.clone(); + for (path, node) in updates.into_sorted().storage_nodes { + if let Some(node) = node { + resulting_nodes.insert(path, node); + } else { + resulting_nodes.remove(&path); + } + } + assert_eq!(root, storage_root_prehashed(storage.clone())); + if enabled { + assert_eq!(resulting_nodes, expected_nodes); + } else { + // Normal traversal ignores hidden descendants, but leaves them in the cache. + assert_ne!(resulting_nodes, expected_nodes); + } + } + } + } + + fn storage_root_with_progress( + harness: &TrieTestHarness, + changes: PrefixSet, + enabled: bool, + threshold: u64, + ) -> (B256, StorageTrieUpdates) { + let mut previous_state = None; + let mut updates = StorageTrieUpdates::default(); + let mut checkpoints = 0; + loop { + let progress = StorageRoot::new_hashed( + harness.trie_cursor_factory(), + harness.hashed_cursor_factory(), + harness.hashed_address(), + changes.clone(), + #[cfg(feature = "metrics")] + crate::metrics::TrieRootMetrics::new(crate::TrieType::Storage), + ) + .with_walk_all_changed_branch_children(enabled) + .with_threshold(threshold) + .with_intermediate_state(previous_state) + .root_with_progress() + .unwrap(); + match progress { + StorageRootProgress::Progress(state, _, new_updates) => { + updates.extend(new_updates); + previous_state = Some(*state); + checkpoints += 1; + } + StorageRootProgress::Complete(root, _, new_updates) => { + updates.extend(new_updates); + if threshold == 1 { + assert!(checkpoints > 1); + } + return (root, updates) + } + } + } + } + + #[test] + fn orphan_cleanup_skips_live_subtrees() { + let parent = Nibbles::from_nibbles([0x3]); + let tree_mask = TrieMask::from_nibble(0xa); + let orphans = [ + Nibbles::from_nibbles([0x3, 0xb]), + Nibbles::from_nibbles([0x3, 0xb, 0x5]), + Nibbles::from_nibbles([0x3, 0xb, 0x5, 0xf]), + ]; + let mut nodes = BTreeMap::from([ + (parent, branch_node((1 << 0xa) | (1 << 0xb), tree_mask.get(), 0)), + (Nibbles::from_nibbles([0x3, 0xa]), branch_node(0b11, 0, 0)), + (Nibbles::from_nibbles([0x4]), branch_node(0b11, 0, 0)), + ]); + for index in 0..128 { + nodes.insert( + Nibbles::from_nibbles([0x3, 0xa, index / 16, index % 16]), + branch_node(0b11, 0, 0), + ); + } + nodes.extend(orphans.into_iter().map(|path| (path, branch_node(0b11, 0, 0)))); + let factory = MockTrieCursorFactory::new(nodes, B256Map::default()); + let changes = PrefixSetMut::from([Nibbles::from_nibbles([0x3, 0xa])]).freeze(); + let mut walker = + TrieWalker::<_>::state_trie(factory.account_trie_cursor().unwrap(), changes) + .with_walk_all_changed_branch_children(true) + .with_deletions_retained(true); + walker.cursor.seek_exact(parent).unwrap(); + factory.visited_account_keys().clear(); + walker.remove_orphaned_descendants(parent, tree_mask).unwrap(); + + assert_eq!(walker.take_removed_keys(), HashSet::from_iter(orphans)); + assert_eq!(walker.cursor.current().unwrap(), Some(parent)); + let next_count = factory + .visited_account_keys() + .iter() + .filter(|visit| matches!(visit.visit_type, KeyVisitType::Next)) + .count(); + assert!(next_count <= 4, "cleanup walked {next_count} entries for 3 orphan nodes"); + } } From cd6b0c3cf71a8b6342efbeaa3acf0b338cb971df Mon Sep 17 00:00:00 2001 From: Derek Cofausper <256792747+decofe@users.noreply.github.com> Date: Thu, 17 Sep 2026 16:54:06 +0000 Subject: [PATCH 291/294] chore(deps): update imbl and remove bitmaps exception (#27286) Co-authored-by: Sergei Shulepov <2205845+pepyakin@users.noreply.github.com> (cherry picked from commit 25993190fae484f6d93a247a935d25db30b3d6fc) --- Cargo.lock | 18 ++++-------------- deny.toml | 2 -- 2 files changed, 4 insertions(+), 16 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9a19ff74073..3c61ed85363 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1756,12 +1756,6 @@ dependencies = [ "serde_core", ] -[[package]] -name = "bitmaps" -version = "3.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1d084b0137aaa901caf9f1e8b21daa6aa24d41cd806e111335541eff9683bd6" - [[package]] name = "bitvec" version = "1.0.1" @@ -5014,13 +5008,12 @@ dependencies = [ [[package]] name = "imbl" -version = "7.0.1" +version = "7.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43ea8d4c37ee560727e824d62804183624d371e632019b0e9e3532bce64a33e5" +checksum = "46bad832b9b463ed9398b8506488cc2e3b897a9d44f13af115f382aac71f4fec" dependencies = [ "arbitrary", "archery", - "bitmaps", "equivalent", "imbl-sized-chunks", "rand_core 0.9.5", @@ -5032,12 +5025,9 @@ dependencies = [ [[package]] name = "imbl-sized-chunks" -version = "0.1.3" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f4241005618a62f8d57b2febd02510fb96e0137304728543dfc5fd6f052c22d" -dependencies = [ - "bitmaps", -] +checksum = "2a0813be332553f857953298749fa19549e8b61b80589757c29b4e2a804fa9c6" [[package]] name = "impl-codec" diff --git a/deny.toml b/deny.toml index 133b02e82a1..be45b22db44 100644 --- a/deny.toml +++ b/deny.toml @@ -12,8 +12,6 @@ ignore = [ "RUSTSEC-2023-0089", # https://rustsec.org/advisories/RUSTSEC-2026-0173 proc-macro-error2 is unmaintained "RUSTSEC-2026-0173", - # https://rustsec.org/advisories/RUSTSEC-2026-0247 bitmaps is unmaintained, transitive dep via imbl - "RUSTSEC-2026-0247", ] # This section is considered when running `cargo deny check bans`. From 8d359d0142719d4042eaf09fcbb7e4a1f89c37ac Mon Sep 17 00:00:00 2001 From: Sebastian Stammler Date: Tue, 15 Sep 2026 15:59:19 +0200 Subject: [PATCH 292/294] ci: run fork validation through CircleCI act Squash the fork CI support from 04e73b7c2bdc8dcf3c1e8961c8d04b861bc15a8e and push-event support from b0d194cad81ddf9dcb954ecee95b22cc2d68b70b into one replayable maintenance commit. Preserve release action pins and advisory policy when resolving the replay. Select the book build job without entering Pages deployment. Refresh the standalone lockfile to patched h2 and rustls releases instead of suppressing RUSTSEC-2026-0258 and RUSTSEC-2026-0285; rustls requires the updated AWS-LC backend. Run cargo deny natively instead of through act (folded in from c279f31455e8b714c1123c19accb1a68bd566f7c): lint.yml's `deny` job calls tempoxyz/ci's reusable deny.yml, which starts with tempoxyz's secure-runner. That action requires a GitHub OIDC token unless the event is a fork pull request, and act never receives one, so the job failed on every push and same-repo PR. Run the same `cargo deny --all-features check all` in a CircleCI job with a digest-pinned Rust image and a checksum-pinned cargo-deny release, and drop `deny` from the act lint matrix. Co-authored-by: Josh Klopfenstein Co-authored-by: falcorocks <14293929+falcorocks@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 (1M context) Maintenance CI: Compare optimism push vectors against v2.6.0, retaining actual PR and merge-queue bases.: --- .circleci/config.yml | 243 ++++++++++++++++++++++++++++++ .circleci/prepare-act-event.sh | 102 +++++++++++++ .github/scripts/check_rv32imac.sh | 2 +- .github/scripts/check_wasm.sh | 2 +- .github/workflows/book.yml | 2 + .github/workflows/compact.yml | 10 +- .github/workflows/e2e.yml | 4 + .github/workflows/integration.yml | 4 + .github/workflows/lint.yml | 20 ++- .github/workflows/stage.yml | 2 + .github/workflows/unit.yml | 6 + 11 files changed, 391 insertions(+), 6 deletions(-) create mode 100644 .circleci/config.yml create mode 100755 .circleci/prepare-act-event.sh diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 00000000000..4ed70d500ab --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,243 @@ +version: 2.1 + +# Runs this fork's validation workflows through pinned act because GitHub Actions is disabled. +# Pull requests and branch pushes run the PR checks; merge queues add stage.yml. Event metadata is +# generated once per CircleCI revision and shared by all shards. +# +# The read-only GitHub token is exposed to PR code, so its CircleCI context must remain +# restricted to trusted contributors. Keep CircleCI auto-cancellation enabled. +# +# The act binary, checksum, and runner image are pinned. Compiler caches are no-ops because +# act's local cache is not reusable across these machines; Pages plumbing is skipped after the +# validation build. Lint and unit jobs are distributed; other workflows run up to four jobs. +# +# Excluded: pr-title.yml, label-pr.yml, and the privileged cyclops-audit.yml integration. + +jobs: + # Replaces lint.yml's `deny` job, which calls tempoxyz/ci's reusable deny.yml. That workflow starts + # with tempoxyz's secure-runner, which needs a GitHub OIDC token that act never receives, so the + # check itself runs natively here. Its policy comes from deny.toml in this repo. + # + # When re-applying this CI patch onto a new reth release, diff lint.yml's `deny:` job and the + # tempoxyz/ci deny.yml it calls against this job (toolchain, cargo-deny version, `check` + # arguments, deny flags), and update both pins here. + cargo-deny: + docker: + - image: rust:1.95-bookworm@sha256:6258907abe69656e41cd992e0b705cdcfabcbbe3db374f92ed2d47121282d4a1 + resource_class: medium + steps: + - checkout + - run: + name: Install pinned cargo-deny + command: | + set -euo pipefail + readonly DENY_VERSION="0.20.2" + readonly DENY_SHA256="9f12ed4c49936e09b48bf862b595cde2fe64fcbd9d74dfacac6131ca824c8d5f" + readonly DENY_ARCHIVE="cargo-deny-${DENY_VERSION}-x86_64-unknown-linux-musl.tar.gz" + curl --fail --silent --show-error --location \ + "https://github.com/EmbarkStudios/cargo-deny/releases/download/${DENY_VERSION}/${DENY_ARCHIVE}" \ + --output "/tmp/${DENY_ARCHIVE}" + echo "${DENY_SHA256} /tmp/${DENY_ARCHIVE}" | sha256sum --check + tar -xzf "/tmp/${DENY_ARCHIVE}" -C /usr/local/bin --strip-components=1 \ + "cargo-deny-${DENY_VERSION}-x86_64-unknown-linux-musl/cargo-deny" + test "$(cargo-deny --version)" = "cargo-deny ${DENY_VERSION}" + - run: + name: cargo deny check + command: cargo deny --log-level warn --all-features check all + + prepare-act-event: + machine: + image: ubuntu-2404:2025.09.1 + resource_class: medium + steps: + - checkout + - run: + name: Snapshot GitHub event + command: | + mkdir -p /tmp/act-event + if ! .circleci/prepare-act-event.sh; then + echo "GitHub event preparation failed; act shards will report the failure" | + tee /tmp/act-event/failed + fi + - persist_to_workspace: + root: /tmp/act-event + paths: + - . + + act: + parameters: + wf: + type: string + job: + type: string + default: "" + concurrent_jobs: + type: integer + default: 4 + machine: + image: ubuntu-2404:2025.09.1 + resource_class: xlarge + steps: + - checkout + - attach_workspace: + at: /tmp/act-event + - run: + name: Check event snapshot + command: | + if [ -f /tmp/act-event/failed ]; then + cat /tmp/act-event/failed >&2 + exit 1 + fi + - run: + name: Install pinned act + command: | + set -euo pipefail + readonly ACT_VERSION="0.2.89-action-cache-fix.1" + readonly ACT_RELEASE="v0.2.89-action-cache-fix.4" + readonly ACT_BINARY="act_Linux_x86_64" + readonly ACT_SHA256="28bfcf1502f59a834b14f2fb23c39e5b253c2234d875592b5aa0a175f9674dce" + curl --fail --silent --show-error --location \ + "https://github.com/ethereum-optimism/act/releases/download/${ACT_RELEASE}/${ACT_BINARY}" \ + --output "/tmp/${ACT_BINARY}" + echo "${ACT_SHA256} /tmp/${ACT_BINARY}" | sha256sum --check + sudo install -m 0755 "/tmp/${ACT_BINARY}" /usr/local/bin/act + test "$(act --version)" = "act version ${ACT_VERSION}" + - run: + name: Stub unsupported non-check actions + command: | + mkdir -p /tmp/noop-action + printf '%s\n' \ + 'name: noop' \ + 'description: disabled service plumbing under act' \ + 'inputs:' \ + ' cache-bin: {required: false}' \ + ' cache-on-failure: {required: false}' \ + ' cache-targets: {required: false}' \ + ' prefix-key: {required: false}' \ + 'runs:' \ + ' using: composite' \ + ' steps:' \ + ' - run: echo "Service plumbing disabled under act"' \ + ' shell: bash' \ + > /tmp/noop-action/action.yml + - run: + name: Run << parameters.wf >> via act + no_output_timeout: 45m + command: | + set -euo pipefail + event_path=/tmp/act-event/github-event.json + event_name="$(jq -er 'if .pull_request then "pull_request" elif .merge_group then "merge_group" elif .ref and .after then "push" else error("unsupported event") end' "$event_path")" + base_ref="$(jq -r '(.pull_request.base.ref // .merge_group.base_ref // "") | sub("^refs/heads/"; "")' "$event_path")" + target_job="<< parameters.job >>" + # Validate documentation without entering Pages deployment on branch pushes. + if [ "<< parameters.wf >>" = "book.yml" ]; then + target_job=build + fi + act_job=() + if [ -n "$target_job" ]; then + case "$target_job" in + crate-checks-*) + partition="${target_job#crate-checks-}" + if [[ ! "$partition" =~ ^[1-9][0-9]*$ ]]; then + echo "invalid crate-checks partition: $partition" >&2 + exit 1 + fi + act_job=(-j crate-checks --matrix "partition:${partition}") + ;; + *) + act_job=(-j "$target_job") + ;; + esac + fi + + act_env=(--env "RUSTC_WRAPPER=" --env "GITHUB_BASE_REF=${base_ref}") + if [ "<< parameters.wf >>" = "unit.yml" ] && [ "$target_job" = "state" ]; then + # EF tests use Rayon internally, so unrestricted nextest processes multiply + # runnable threads and starve valid_blocks past its upstream timeout. + act_env+=(--env "NEXTEST_TEST_THREADS=2") + fi + if [ "$event_name" != "pull_request" ]; then + act_env+=( + --env "GITHUB_REF=refs/heads/${CIRCLE_BRANCH}" + --env "GITHUB_SHA=${CIRCLE_SHA1}" + ) + fi + + act "$event_name" -W ".github/workflows/<< parameters.wf >>" \ + "${act_job[@]}" \ + --eventpath "$event_path" \ + -P "ubuntu-latest=catthehacker/ubuntu@sha256:b839c14c4410998529ec18f951262bdf87a2b23bc1467304d07b491b9455e074" \ + --concurrent-jobs << parameters.concurrent_jobs >> \ + "${act_env[@]}" \ + -s GITHUB_TOKEN="${MISE_GITHUB_TOKEN:?missing restricted read-only GitHub token}" \ + --local-repository "mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba=/tmp/noop-action" \ + --local-repository "Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6=/tmp/noop-action" \ + --local-repository "actions/configure-pages@45bfe0192ca1faeb007ade9deae92b16b8254a0d=/tmp/noop-action" \ + --local-repository "actions/upload-pages-artifact@fc324d3547104276b827a68afc52ff2a11cc49c9=/tmp/noop-action" + +workflows: + reth-act: + jobs: + - prepare-act-event: + context: + - circleci-repo-readonly-authenticated-github-token + - act: + context: + - circleci-repo-readonly-authenticated-github-token + name: "act-<< matrix.wf >>" + requires: + - prepare-act-event + matrix: + parameters: + wf: + - book.yml + - compact.yml + - e2e.yml + - grafana.yml + - integration.yml + - lint-actions.yml + - stage.yml + - act: + wf: lint.yml + concurrent_jobs: 1 + context: + - circleci-repo-readonly-authenticated-github-token + name: "act-lint-<< matrix.job >>" + requires: + - prepare-act-event + matrix: + parameters: + job: + - clippy-binaries + - clippy + - wasm + - riscv + - crate-checks-1 + - crate-checks-2 + - crate-checks-3 + - msrv + - docs + - fmt + - udeps + - book + - typos + - check-toml + - grafana + - no-test-deps + - feature-propagation + - cargo-deny: + name: lint-deny + - act: + wf: unit.yml + concurrent_jobs: 1 + context: + - circleci-repo-readonly-authenticated-github-token + name: "act-unit-<< matrix.job >>" + requires: + - prepare-act-event + matrix: + parameters: + job: + - test + - state + - doc diff --git a/.circleci/prepare-act-event.sh b/.circleci/prepare-act-event.sh new file mode 100755 index 00000000000..99cccb27f3c --- /dev/null +++ b/.circleci/prepare-act-event.sh @@ -0,0 +1,102 @@ +#!/usr/bin/env bash +set -euo pipefail + +readonly OUTPUT_DIR="${ACT_EVENT_OUTPUT_DIR:-/tmp/act-event}" +readonly EVENT_PATH="${OUTPUT_DIR}/github-event.json" + +mkdir -p "$OUTPUT_DIR" +rm -f "$EVENT_PATH" + +: "${CIRCLE_SHA1:?missing CircleCI revision}" +: "${CIRCLE_PROJECT_USERNAME:?missing CircleCI project owner}" +: "${CIRCLE_PROJECT_REPONAME:?missing CircleCI project repository}" + +repository="${CIRCLE_PROJECT_USERNAME}/${CIRCLE_PROJECT_REPONAME}" + +if [[ "${CIRCLE_BRANCH:-}" == gh-readonly-queue/* ]]; then + event_name="merge_group" +elif [[ -n "${CIRCLE_PULL_REQUEST:-}" ]]; then + event_name="pull_request" +else + event_name="push" +fi + +case "$event_name" in + pull_request) + : "${CIRCLE_PULL_REQUEST:?pull-request trigger must provide CIRCLE_PULL_REQUEST}" + pr_number="${CIRCLE_PULL_REQUEST##*/}" + if [[ ! "$pr_number" =~ ^[1-9][0-9]*$ ]]; then + echo "invalid pull-request URL: ${CIRCLE_PULL_REQUEST}" >&2 + exit 1 + fi + + pull_request_path="${OUTPUT_DIR}/pull-request.json" + curl --fail --silent --show-error --location \ + -H "Accept: application/vnd.github+json" \ + -H "Authorization: Bearer ${MISE_GITHUB_TOKEN:?missing restricted read-only GitHub token}" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + "https://api.github.com/repos/${repository}/pulls/${pr_number}" > "$pull_request_path" + + event_head_sha="$(jq -er '.head.sha' "$pull_request_path")" + if [[ "$event_head_sha" != "$CIRCLE_SHA1" ]]; then + echo "PR head does not match CircleCI revision: ${CIRCLE_SHA1} != ${event_head_sha}" >&2 + exit 1 + fi + + jq '{ + action: "synchronize", + number: .number, + pull_request: ., + repository: .base.repo, + sender: .user + }' "$pull_request_path" > "$EVENT_PATH" + base_ref="$(jq -er '.base.ref | select(length > 0)' "$pull_request_path")" + head_ref="$(jq -er '.head.ref | select(length > 0)' "$pull_request_path")" + echo "act event: pull_request #${pr_number}, ${head_ref} -> ${base_ref}" + ;; + push) + : "${CIRCLE_BRANCH:?push trigger must provide CIRCLE_BRANCH}" + jq -n \ + --arg ref "refs/heads/${CIRCLE_BRANCH}" \ + --arg after "$CIRCLE_SHA1" \ + '{ref: $ref, after: $after}' > "$EVENT_PATH" + echo "act event: push ${CIRCLE_BRANCH} at ${CIRCLE_SHA1}" + ;; + merge_group) + : "${CIRCLE_BRANCH:?merge-queue trigger must provide CIRCLE_BRANCH}" + if [[ ! "$CIRCLE_BRANCH" =~ ^gh-readonly-queue/(.+)/pr-([1-9][0-9]*)-[^/]+$ ]]; then + echo "merge_group requires a gh-readonly-queue//pr-- branch, got: ${CIRCLE_BRANCH}" >&2 + exit 1 + fi + base_ref="${BASH_REMATCH[1]}" + + if ! base_sha="$(git rev-parse "${CIRCLE_SHA1}^1" 2>/dev/null)"; then + git fetch --no-tags --depth=2 origin "$CIRCLE_SHA1" + base_sha="$(git rev-parse "${CIRCLE_SHA1}^1")" + fi + + jq -n \ + --arg head_sha "$CIRCLE_SHA1" \ + --arg head_ref "refs/heads/${CIRCLE_BRANCH}" \ + --arg base_sha "$base_sha" \ + --arg base_ref "refs/heads/${base_ref}" \ + --arg repository "$repository" \ + --arg repository_name "$CIRCLE_PROJECT_REPONAME" \ + --arg repository_owner "$CIRCLE_PROJECT_USERNAME" \ + '{ + action: "checks_requested", + merge_group: { + head_sha: $head_sha, + head_ref: $head_ref, + base_sha: $base_sha, + base_ref: $base_ref + }, + repository: { + full_name: $repository, + name: $repository_name, + owner: {login: $repository_owner} + } + }' > "$EVENT_PATH" + echo "act event: merge_group ${CIRCLE_BRANCH} -> ${base_ref} at ${base_sha}" + ;; +esac diff --git a/.github/scripts/check_rv32imac.sh b/.github/scripts/check_rv32imac.sh index 4e477cdbed1..988a95a3309 100755 --- a/.github/scripts/check_rv32imac.sh +++ b/.github/scripts/check_rv32imac.sh @@ -32,7 +32,7 @@ trap 'rm -rf -- "$tmpdir"' EXIT INT TERM for crate in "${crates_to_check[@]}"; do outfile="$tmpdir/$crate.log" - if cargo +stable build -p "$crate" --target riscv32imac-unknown-none-elf --no-default-features --color never >"$outfile" 2>&1; then + if cargo build -p "$crate" --target riscv32imac-unknown-none-elf --no-default-features --color never >"$outfile" 2>&1; then echo "✅ $crate" else echo "❌ $crate" diff --git a/.github/scripts/check_wasm.sh b/.github/scripts/check_wasm.sh index 15a8b8122ff..e4d5a8037a8 100755 --- a/.github/scripts/check_wasm.sh +++ b/.github/scripts/check_wasm.sh @@ -110,7 +110,7 @@ for crate in "${crates[@]}"; do fi outfile="$tmpdir/$crate.log" - if cargo +stable build -p "$crate" --target wasm32-wasip1 --no-default-features --color never >"$outfile" 2>&1; then + if cargo build -p "$crate" --target wasm32-wasip1 --no-default-features --color never >"$outfile" 2>&1; then echo "✅ $crate" else echo "❌ $crate" diff --git a/.github/workflows/book.yml b/.github/workflows/book.yml index 07f26676b62..86a39c9945b 100644 --- a/.github/workflows/book.yml +++ b/.github/workflows/book.yml @@ -40,6 +40,8 @@ jobs: - name: Install Rust nightly uses: dtolnay/rust-toolchain@nightly + with: + toolchain: nightly-2026-07-14 - name: Build docs run: cd docs/vocs && bash scripts/build-cargo-docs.sh diff --git a/.github/workflows/compact.yml b/.github/workflows/compact.yml index 52ad1ff8374..ac60ce324b2 100644 --- a/.github/workflows/compact.yml +++ b/.github/workflows/compact.yml @@ -1,6 +1,6 @@ # Ensures that `Compact` codec changes are backwards compatible. # -# 1) checkout `main` +# 1) checkout the PR base, or the selected release for maintenance-branch pushes # 2) randomly generate and serialize to disk many different type vectors with `Compact` (eg. Header, Transaction, etc) # 3) checkout `pr` # 4) deserialize previously generated test vectors @@ -35,6 +35,8 @@ jobs: - uses: rui314/setup-mold@7e4f20ad28a2e8ca6fd0892ccf72e2abb706b9c3 # v1 - run: .github/scripts/install_llvm.sh ubuntu - uses: dtolnay/rust-toolchain@stable + with: + toolchain: "1.97.0" - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 with: @@ -42,9 +44,11 @@ jobs: - name: Checkout base uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: - ref: ${{ github.base_ref || 'main' }} + # Use v2.6.0's commit: the fork carries its history but does not mirror upstream tags. + # The maintenance branch must decode its release's data, not unreleased main formats. + ref: ${{ github.base_ref || (github.ref == 'refs/heads/optimism' && '73a3a00862a8f14f89e30da8de001456f18cfae0' || 'main') }} persist-credentials: false - # On `main` branch, generates test vectors and serializes them to disk using `Compact`. + # On the comparison base, generate test vectors and serialize them to disk using `Compact`. - name: Generate compact vectors run: | ${{ matrix.bin }} -- test-vectors compact --write diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 6031cd278a5..98f7d55e787 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -35,6 +35,8 @@ jobs: - uses: rui314/setup-mold@7e4f20ad28a2e8ca6fd0892ccf72e2abb706b9c3 # v1 - run: .github/scripts/install_llvm.sh ubuntu - uses: dtolnay/rust-toolchain@stable + with: + toolchain: "1.97.0" - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 - uses: taiki-e/install-action@d438492cf8a250514fa2d34b30bc3c0dc37c65ff # v2.87.8 with: @@ -71,6 +73,8 @@ jobs: - uses: rui314/setup-mold@7e4f20ad28a2e8ca6fd0892ccf72e2abb706b9c3 # v1 - run: .github/scripts/install_llvm.sh ubuntu - uses: dtolnay/rust-toolchain@stable + with: + toolchain: "1.97.0" - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 - uses: taiki-e/install-action@d438492cf8a250514fa2d34b30bc3c0dc37c65ff # v2.87.8 with: diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 3d3ff5abf6d..45567fed0b0 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -42,6 +42,8 @@ jobs: - uses: rui314/setup-mold@7e4f20ad28a2e8ca6fd0892ccf72e2abb706b9c3 # v1 - run: .github/scripts/install_llvm.sh ubuntu - uses: dtolnay/rust-toolchain@stable + with: + toolchain: "1.97.0" - name: Install Geth run: .github/scripts/install_geth.sh - uses: taiki-e/install-action@d438492cf8a250514fa2d34b30bc3c0dc37c65ff # v2.87.8 @@ -86,6 +88,8 @@ jobs: - uses: rui314/setup-mold@7e4f20ad28a2e8ca6fd0892ccf72e2abb706b9c3 # v1 - run: .github/scripts/install_llvm.sh ubuntu - uses: dtolnay/rust-toolchain@stable + with: + toolchain: "1.97.0" - uses: taiki-e/install-action@d438492cf8a250514fa2d34b30bc3c0dc37c65ff # v2.87.8 with: tool: nextest diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index b52ca66799c..1368de51907 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -38,6 +38,7 @@ jobs: - run: .github/scripts/install_llvm.sh ubuntu - uses: dtolnay/rust-toolchain@clippy with: + toolchain: nightly-2026-07-14 components: clippy - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 @@ -62,6 +63,7 @@ jobs: - run: .github/scripts/install_llvm.sh ubuntu - uses: dtolnay/rust-toolchain@nightly with: + toolchain: nightly-2026-07-14 components: clippy - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 @@ -81,8 +83,10 @@ jobs: with: persist-credentials: false - uses: rui314/setup-mold@7e4f20ad28a2e8ca6fd0892ccf72e2abb706b9c3 # v1 + - run: .github/scripts/install_llvm.sh ubuntu - uses: dtolnay/rust-toolchain@stable with: + toolchain: "1.97.0" target: wasm32-wasip1 - uses: taiki-e/install-action@d438492cf8a250514fa2d34b30bc3c0dc37c65ff # v2.87.8 with: @@ -94,7 +98,7 @@ jobs: - uses: dcarbone/install-jq-action@4fcb5062d7ce9bc4382d1a352d19ba3ba2c317c1 # v4.0.1 - name: Run Wasm checks run: | - sudo apt update && sudo apt install gcc-multilib + sudo apt update && sudo apt install --yes gcc-multilib .github/scripts/check_wasm.sh riscv: @@ -109,6 +113,7 @@ jobs: - uses: rui314/setup-mold@7e4f20ad28a2e8ca6fd0892ccf72e2abb706b9c3 # v1 - uses: dtolnay/rust-toolchain@stable with: + toolchain: "1.97.0" target: riscv32imac-unknown-none-elf - uses: taiki-e/install-action@d438492cf8a250514fa2d34b30bc3c0dc37c65ff # v2.87.8 with: @@ -138,6 +143,8 @@ jobs: - uses: rui314/setup-mold@7e4f20ad28a2e8ca6fd0892ccf72e2abb706b9c3 # v1 - run: .github/scripts/install_llvm.sh ubuntu - uses: dtolnay/rust-toolchain@stable + with: + toolchain: "1.97.0" - uses: taiki-e/install-action@d438492cf8a250514fa2d34b30bc3c0dc37c65ff # v2.87.8 with: tool: cargo-hack @@ -183,6 +190,8 @@ jobs: - uses: rui314/setup-mold@7e4f20ad28a2e8ca6fd0892ccf72e2abb706b9c3 # v1 - run: .github/scripts/install_llvm.sh ubuntu - uses: dtolnay/rust-toolchain@nightly + with: + toolchain: nightly-2026-07-14 - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 with: @@ -206,6 +215,7 @@ jobs: - uses: rui314/setup-mold@7e4f20ad28a2e8ca6fd0892ccf72e2abb706b9c3 # v1 - uses: dtolnay/rust-toolchain@nightly with: + toolchain: nightly-2026-07-14 components: rustfmt - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 - name: Run fmt @@ -224,6 +234,8 @@ jobs: - uses: rui314/setup-mold@7e4f20ad28a2e8ca6fd0892ccf72e2abb706b9c3 # v1 - run: .github/scripts/install_llvm.sh ubuntu - uses: dtolnay/rust-toolchain@nightly + with: + toolchain: nightly-2026-07-14 - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 with: @@ -246,6 +258,8 @@ jobs: - uses: rui314/setup-mold@7e4f20ad28a2e8ca6fd0892ccf72e2abb706b9c3 # v1 - run: .github/scripts/install_llvm.sh ubuntu - uses: dtolnay/rust-toolchain@nightly + with: + toolchain: nightly-2026-07-14 - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 with: @@ -308,6 +322,8 @@ jobs: persist-credentials: false - uses: rui314/setup-mold@7e4f20ad28a2e8ca6fd0892ccf72e2abb706b9c3 # v1 - uses: dtolnay/rust-toolchain@stable + with: + toolchain: "1.97.0" - name: Ensure no arbitrary or proptest dependency on default build run: cargo tree --package reth -e=features,no-dev | grep -Eq "arbitrary|proptest" && exit 1 || exit 0 @@ -323,6 +339,8 @@ jobs: persist-credentials: false - run: .github/scripts/install_llvm.sh ubuntu - uses: dtolnay/rust-toolchain@stable + with: + toolchain: "1.97.0" - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 - uses: rui314/setup-mold@7e4f20ad28a2e8ca6fd0892ccf72e2abb706b9c3 # v1 - uses: taiki-e/cache-cargo-install-action@9ee83daaa7b96a6fab930949ecf1122bba04a389 # v3.0.8 diff --git a/.github/workflows/stage.yml b/.github/workflows/stage.yml index bb5c2422482..a235be631cf 100644 --- a/.github/workflows/stage.yml +++ b/.github/workflows/stage.yml @@ -39,6 +39,8 @@ jobs: - uses: rui314/setup-mold@7e4f20ad28a2e8ca6fd0892ccf72e2abb706b9c3 # v1 - run: .github/scripts/install_llvm.sh ubuntu - uses: dtolnay/rust-toolchain@stable + with: + toolchain: "1.97.0" - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 with: diff --git a/.github/workflows/unit.yml b/.github/workflows/unit.yml index 9e59205b8d8..a42d46af854 100644 --- a/.github/workflows/unit.yml +++ b/.github/workflows/unit.yml @@ -42,6 +42,8 @@ jobs: - uses: rui314/setup-mold@7e4f20ad28a2e8ca6fd0892ccf72e2abb706b9c3 # v1 - run: .github/scripts/install_llvm.sh ubuntu - uses: dtolnay/rust-toolchain@stable + with: + toolchain: "1.97.0" - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 with: @@ -94,6 +96,8 @@ jobs: - uses: rui314/setup-mold@7e4f20ad28a2e8ca6fd0892ccf72e2abb706b9c3 # v1 - run: .github/scripts/install_llvm.sh ubuntu - uses: dtolnay/rust-toolchain@stable + with: + toolchain: "1.97.0" - uses: taiki-e/install-action@d438492cf8a250514fa2d34b30bc3c0dc37c65ff # v2.87.8 with: tool: nextest @@ -118,6 +122,8 @@ jobs: - uses: rui314/setup-mold@7e4f20ad28a2e8ca6fd0892ccf72e2abb706b9c3 # v1 - run: .github/scripts/install_llvm.sh ubuntu - uses: dtolnay/rust-toolchain@stable + with: + toolchain: "1.97.0" - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 with: From 785b05757fc7a90b5a6356caeefc63ab98da6dab Mon Sep 17 00:00:00 2001 From: Josh Klopfenstein Date: Fri, 16 Oct 2026 10:48:02 -0700 Subject: [PATCH 293/294] ci: isolate compact builds and refresh the rustls lock Build the compact reader in a separate target directory so act cannot reuse libraries compiled from the comparison base after restoring older source mtimes. Update the fork's rustls lock from 0.23.45 to 0.23.46 for RUSTSEC-2026-0346 through RUSTSEC-2026-0349. The consuming Optimism workspace already resolves 0.23.46. Preserve the published pin as parent. --- .github/workflows/compact.yml | 4 ++++ Cargo.lock | 4 ++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/compact.yml b/.github/workflows/compact.yml index ac60ce324b2..6a80eeedf46 100644 --- a/.github/workflows/compact.yml +++ b/.github/workflows/compact.yml @@ -59,4 +59,8 @@ jobs: persist-credentials: false # On incoming merge try to read and decode previously generated vectors with `Compact` - name: Read vectors + env: + # act restores the head with source mtimes older than the base build's artifacts. + # Keep the builds separate so Cargo cannot reuse libraries from the comparison base. + CARGO_TARGET_DIR: target/compact-head run: ${{ matrix.bin }} -- test-vectors compact --read diff --git a/Cargo.lock b/Cargo.lock index 3c61ed85363..0af8e7f9822 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -11560,9 +11560,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.45" +version = "0.23.46" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d41d731c7d2f962d1ccc364cec258de3c0e93b38c2fb3ba97ac74513048d634" +checksum = "48e13bd8c0e9365c43cfa5c9e8f9ad49d3c8444926c9aac819e0e4dc503c8fdf" dependencies = [ "aws-lc-rs", "log", From d9bd9875fe88dcc0183d78b187ce2ceab0c93e93 Mon Sep 17 00:00:00 2001 From: Josh Klopfenstein Date: Fri, 16 Oct 2026 11:23:10 -0700 Subject: [PATCH 294/294] ci: give the valid-block fixture group a bounded timeout valid_blocks runs the whole ValidBlocks fixture directory but inherits the single-test sixty-second timeout. The CircleCI run terminated it at that boundary on all three attempts while the other 61 groups passed. Give only this exact ef-tests test the ten-minute budget already used by grouped general-state fixtures. All assertions, retries, and other test timeouts remain unchanged. A successful remote run must still prove it finishes within the new bound. --- .config/nextest.toml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.config/nextest.toml b/.config/nextest.toml index 01cb0158619..d07545e2400 100644 --- a/.config/nextest.toml +++ b/.config/nextest.toml @@ -6,6 +6,12 @@ slow-timeout = { period = "30s", terminate-after = 2 } filter = "test(general_state_tests)" slow-timeout = { period = "1m", terminate-after = 10 } +# valid_blocks covers a full fixture directory, like the grouped state tests. +# Keep its CircleCI runtime bounded by the same ten-minute budget. +[[profile.default.overrides]] +filter = "package(=ef-tests) and test(=valid_blocks)" +slow-timeout = { period = "1m", terminate-after = 10 } + [[profile.default.overrides]] filter = "test(eest_fixtures)" slow-timeout = { period = "2m", terminate-after = 10 }