From 5f792261518acbf84301664d70e3248cb53290a1 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Thu, 12 Nov 2026 18:24:23 +0100 Subject: [PATCH] op-acceptance-tests,op-devstack,op-service: cover eth version negotiation and transaction propagation between op-reth and op-geth Add an acceptance test package that peers an op-reth and an op-geth follower as basic peers, requires them to negotiate the highest eth version both advertise, requires a transaction submitted to either to reach the other's pool and a block, and hands op-geth twice as many transactions to announce as op-reth tracks per peer, sent so that op-reth is offered more than its cap, requiring op-reth to keep the session, keep syncing, include at least a cap's worth of the burst, and, once mined transactions have left its pool, account for every burst transaction it pooled as included or still pooled. Both nodes run pool limits well above the burst, the funding leaves room for the L1 data fee, the pair is used only once the sender has caught up with the sequencer, and the op-geth follower runs a genesis without Karst, which op-geth v1.101702.4-rc.2 refuses. The devstack gains basic (untrusted) EL peering, typed admin_peers lookups including handshaking peers, batch transaction submission, logged pool waits including waits for pending, pooled and included transactions and for transactions to leave the pool, offline signing, and op-geth options on the mixed preset; txplan gains a static chain ID option, and an explicit fee cap is no longer recomputed when the tip cap is pinned after it. Co-Authored-By: Claude Fable 5.1 Co-Authored-By: Claude Opus 5.5 --- .../announcement_burst_test.go | 194 +++++++++ .../eth_version_test.go | 44 ++ .../op_reth_opgeth_peering/funding_test.go | 130 ++++++ .../funding_unit_test.go | 56 +++ .../propagation_test.go | 50 +++ .../op_reth_opgeth_peering/system_test.go | 165 +++++++ op-devstack/dsl/eoa.go | 29 ++ op-devstack/dsl/l2_el.go | 8 - op-devstack/dsl/l2_el_p2p.go | 58 +++ op-devstack/dsl/l2_el_txpool.go | 410 ++++++++++++++++++ op-devstack/sysgo/l2_el_opgeth.go | 7 + op-devstack/sysgo/l2_el_p2p_util.go | 249 +++++++++-- op-devstack/sysgo/l2_el_p2p_util_test.go | 100 +++++ op-devstack/sysgo/mixed_runtime.go | 4 +- op-devstack/sysgo/singlechain_build.go | 17 +- op-service/txplan/txplan.go | 14 + op-service/txplan/txplan_test.go | 29 ++ 17 files changed, 1511 insertions(+), 53 deletions(-) create mode 100644 op-acceptance-tests/tests/op_reth_opgeth_peering/announcement_burst_test.go create mode 100644 op-acceptance-tests/tests/op_reth_opgeth_peering/eth_version_test.go create mode 100644 op-acceptance-tests/tests/op_reth_opgeth_peering/funding_test.go create mode 100644 op-acceptance-tests/tests/op_reth_opgeth_peering/funding_unit_test.go create mode 100644 op-acceptance-tests/tests/op_reth_opgeth_peering/propagation_test.go create mode 100644 op-acceptance-tests/tests/op_reth_opgeth_peering/system_test.go create mode 100644 op-devstack/dsl/l2_el_p2p.go create mode 100644 op-devstack/dsl/l2_el_txpool.go diff --git a/op-acceptance-tests/tests/op_reth_opgeth_peering/announcement_burst_test.go b/op-acceptance-tests/tests/op_reth_opgeth_peering/announcement_burst_test.go new file mode 100644 index 00000000000..08f3434b85e --- /dev/null +++ b/op-acceptance-tests/tests/op_reth_opgeth_peering/announcement_burst_test.go @@ -0,0 +1,194 @@ +package op_reth_opgeth_peering + +import ( + "maps" + "testing" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + + "github.com/ethereum-optimism/optimism/op-devstack/devtest" + "github.com/ethereum-optimism/optimism/op-devstack/dsl" + "github.com/ethereum-optimism/optimism/op-devstack/sysgo" + "github.com/ethereum-optimism/optimism/op-service/eth" + "github.com/ethereum-optimism/optimism/op-service/log" + "github.com/ethereum-optimism/optimism/op-service/txplan" +) + +const ( + // announcedHashCap is how many announced transaction hashes op-reth tracks per peer at most + // (reth#26924): a hash a peer announces while that many of its announcements are still + // unfetched is dropped. op-reth fetches while the burst is still arriving, so it stays under + // the cap only if it fetches as fast as op-geth announces. Whether it ever reached the cap is + // not observable through its RPC. + announcedHashCap = 4096 + // burstTransactions is how many transactions op-geth takes in to announce to op-reth. op-reth + // is offered all of them only if op-geth's sends to it keep up with opGethMaxQueuedTxAnns. + burstTransactions = 2 * announcedHashCap + + // opGethMaxQueuedTxAnns is how many hashes op-geth queues for announcement to one peer at + // most (its maxQueuedTxAnns): when more are queued it drops the oldest, so it always announces + // the last opGethMaxQueuedTxAnns it queued, and earlier ones only if it sent them before. + opGethMaxQueuedTxAnns = 4096 + // opGethTxMaxBroadcastSize is the largest transaction op-geth sends to a peer whole (its + // txMaxBroadcastSize); it announces a larger one by hash only, so the peer has to fetch it. + opGethTxMaxBroadcastSize = 4096 + + // announceOnlyDataBytes is the calldata that alone puts a transaction over + // opGethTxMaxBroadcastSize. + announceOnlyDataBytes = opGethTxMaxBroadcastSize + // announceOnlyTxGas covers the EIP-7623 calldata floor of announceOnlyDataBytes zero bytes. + announceOnlyTxGas = 80_000 + // burstSenderFundsGwei pays one announce-only transaction at the pinned fee cap plus its L1 + // data fee. + burstSenderFundsGwei = 20_000_000 + + // burstPoolMaxCount is op-reth's pending and basefee sub-pool size, in transactions, and how + // many transactions op-geth's pool holds at most. Their defaults, 10000 for reth and 6144 for + // op-geth, are too close to or below the burst. + burstPoolMaxCount = 2 * burstTransactions + // burstPoolMaxSizeMB is op-reth's pending and basefee sub-pool size, in MB. The burst takes + // about 34 MB of pool space, more than reth's 20 MB default. + burstPoolMaxSizeMB = 100 + + // burstDrainAttempts bounds, in 1s polls, how long the resumed sequencer may take to include a + // cap's worth of the burst, which fills about five blocks at the devnet's gas limit. + burstDrainAttempts = 90 + // burstLeadAttempts bounds, in 1s polls, how long op-geth may take to hold the burst's first + // transaction as pending. + burstLeadAttempts = 30 + // burstPoolAttempts bounds, in 1s polls, how long op-reth may take to fetch a cap's worth of + // the burst into its pool. + burstPoolAttempts = 60 + // includedDrainAttempts bounds, in 1s polls, how long op-reth may take to clear transactions + // from its pool once it has imported the blocks that include them. + includedDrainAttempts = 60 + // syncAttempts bounds, in 2s polls, how long a follower may take to import blocks the + // sequencer has already built. + syncAttempts = 30 +) + +// announceOnlyTransaction signs the transaction a fresh sender contributes to the burst: too +// large for op-geth to send it whole, so it is announced by hash. +func announceOnlyTransaction(t devtest.T, sender *dsl.EOA) *types.Transaction { + to := sender.Address() + tx := sender.Sign( + sender.PlanOffline(), + txplan.WithStaticNonce(0), + txplan.WithTo(&to), + txplan.WithData(make([]byte, announceOnlyDataBytes)), + txplan.WithGasLimit(announceOnlyTxGas), + txplan.WithGasFeeCap(feeCap), + txplan.WithGasTipCap(tip), + ) + t.Require().Greater(tx.Size(), uint64(opGethTxMaxBroadcastSize), "the transaction must be larger than what op-geth sends whole") + return tx +} + +// verifySessionKept requires after to describe the same session as before, seen from the +// accepting side: a dropped session that the dialing side re-dials shows up there as a new +// remote port. +func verifySessionKept(t devtest.T, before, after sysgo.ELPeerInfo) { + t.Require().True(before.Inbound, "a re-dial shows a new remote port on the accepting side only") + t.Require().Equal(before.RemoteAddress, after.RemoteAddress, "op-reth must keep its session with op-geth; a new remote port means the session was dropped and re-dialed") + t.Require().Equal(before.EthVersion, after.EthVersion, "the session must keep its negotiated eth version") +} + +// transactionsWithHashes returns those of txs whose hash is in hashes. +func transactionsWithHashes(txs []*types.Transaction, hashes map[common.Hash]struct{}) []*types.Transaction { + var selected []*types.Transaction + for _, tx := range txs { + if _, ok := hashes[tx.Hash()]; ok { + selected = append(selected, tx) + } + } + return selected +} + +// verifyBurstReconciled requires every burst transaction op-reth pooled to be either included in +// a block or still in its pool. +func verifyBurstReconciled(t devtest.T, pooled, included, stillPooled map[common.Hash]struct{}) { + var inNeither []common.Hash + for hash := range pooled { + _, isIncluded := included[hash] + _, isStillPooled := stillPooled[hash] + if !isIncluded && !isStillPooled { + inNeither = append(inNeither, hash) + } + } + t.Require().Empty(inNeither, "every burst transaction op-reth pooled must be in a block or still in its pool; %d of %d are in neither", len(inNeither), len(pooled)) +} + +// TestOpRethKeepsItsOpGethPeerThroughAnAnnouncementBurst hands op-geth twice as many transactions +// to announce to op-reth within a few seconds as op-reth tracks announced hashes per peer, sent so +// that op-reth is offered more hashes than that whatever op-geth drops from its own announcement +// queue, while the sequencer is paused. op-reth has to fetch at least a full cap's worth of them +// into its pool, keep its session with op-geth and log no error meanwhile. Once the sequencer +// resumes, op-reth has to keep following the chain until it includes a cap's worth of the burst. +// With the sequencer paused again, op-reth has to clear the burst transactions in blocks up to +// the paused head from its pool, and every burst transaction it was seen pooling has to be either +// in such a block or still in its pool; its pool may keep what a cap-sized fetch downstream leaves +// behind. How many of the burst op-reth pooled, how many were included, how many op-geth offered +// op-reth at least, and how many each follower still holds afterwards, is logged. +func TestOpRethKeepsItsOpGethPeerThroughAnAnnouncementBurst(gt *testing.T) { + t := devtest.ParallelT(gt) + skipUnsupported(t) + sys := newPairSystem(t, sysgo.MixedL2ELOpGeth) + opGeth, opReth := sys.Sender.EL, sys.Receiver.EL + senders, fundedAt := fundAccounts(t, sys, burstTransactions, eth.GWei(burstSenderFundsGwei)) + burst := make([]*types.Transaction, len(senders)) + for i, sender := range senders { + burst[i] = announceOnlyTransaction(t, sender) + } + // Both followers admit a transaction against their own state, which must hold the senders' + // funds. + dsl.CheckAll(t, + opGeth.ReachedFn(eth.Unsafe, fundedAt, syncAttempts), + opReth.ReachedFn(eth.Unsafe, fundedAt, syncAttempts), + ) + sys.Sequencer.CL.StopSequencer() + + sessionBefore := opReth.PeerInfoOf(opGeth) + errorLogs := opReth.RecordLogs(log.LevelError) + // op-geth announces at least the last opGethMaxQueuedTxAnns hashes it queues for op-reth. The + // lead transaction, pending in op-geth before the rest is sent, is queued alone and taken for + // announcement before the rest is queued, as op-geth has nothing else to announce to op-reth, + // its only peer: op-reth is offered more than announcedHashCap hashes of the burst whatever + // op-geth drops. + t.Require().GreaterOrEqual(opGethMaxQueuedTxAnns, announcedHashCap, "op-geth must announce at least a cap's worth of what it queues") + lead, rest := burst[:1], burst[1:] + opGeth.SendTransactions(lead) + opGeth.WaitForPendingTransactions(lead, burstLeadAttempts) + opGeth.SendTransactions(rest) + pooled := opReth.WaitForPooledTransactions(burst, announcedHashCap, burstPoolAttempts) + + verifySessionKept(t, sessionBefore, opReth.PeerInfoOf(opGeth)) + errorLogs.VerifyEmpty("fetching the burst") + t.Logger().Info("op-reth fetched the burst", "burst", len(burst), "pooled", len(pooled)) + + sys.Sequencer.CL.StartSequencer() + // op-reth passes the burst on to the sequencer, which fetches what op-reth only announced + // to it under the same per-peer cap: at least a cap's worth of the burst reaches a block, and + // up to as many transactions as the burst exceeds the cap by can stay behind in op-reth's pool. + // The burst is sent once both followers are past fundedAt, so no block up to it holds any. + included, nextBlock := opReth.WaitForIncludedTransactions(burst, fundedAt+1, announcedHashCap, burstDrainAttempts) + opReth.Advanced(eth.Unsafe, 2) + // Pausing the sequencer again holds op-reth's chain still, so its blocks and its pool are read + // against the same head. + pausedHead := sys.Sequencer.EL.BlockRefByHash(sys.Sequencer.CL.StopSequencer()) + opReth.Reached(eth.Unsafe, pausedHead.Number, syncAttempts) + maps.Copy(included, opReth.IncludedTransactions(burst, nextBlock, pausedHead.Number)) + opReth.WaitForTransactionsOutOfPool(transactionsWithHashes(burst, included), includedDrainAttempts) + stillPooled := opReth.PooledTransactions(burst) + t.Require().Equal(pausedHead.Hash, opReth.BlockRefByLabel(eth.Unsafe).Hash, "op-reth's head must stay at the paused sequencer's head while the burst is reconciled") + verifyBurstReconciled(t, pooled, included, stillPooled) + + verifySessionKept(t, sessionBefore, opReth.PeerInfoOf(opGeth)) + opGeth.VerifyPeeredWith(opReth) + // op-reth learns of a burst transaction only from op-geth, so those it pooled or that reached a + // block are a lower bound on how many op-geth offered it. + offeredAtLeast := maps.Clone(pooled) + maps.Copy(offeredAtLeast, included) + maps.Copy(offeredAtLeast, stillPooled) + t.Logger().Info("Burst included", "burst", len(burst), "offered_at_least", len(offeredAtLeast), "pooled", len(pooled), "included", len(included), "left_in_op_geth", opGeth.TxPoolStatus(), "left_in_op_reth", len(stillPooled)) +} diff --git a/op-acceptance-tests/tests/op_reth_opgeth_peering/eth_version_test.go b/op-acceptance-tests/tests/op_reth_opgeth_peering/eth_version_test.go new file mode 100644 index 00000000000..097a79a7fa9 --- /dev/null +++ b/op-acceptance-tests/tests/op_reth_opgeth_peering/eth_version_test.go @@ -0,0 +1,44 @@ +package op_reth_opgeth_peering + +import ( + "testing" + + "github.com/ethereum-optimism/optimism/op-devstack/devtest" + "github.com/ethereum-optimism/optimism/op-devstack/sysgo" +) + +// highestEthVersion returns the highest eth protocol version among the capabilities a peer +// advertised. +func highestEthVersion(t devtest.T, caps []string) uint64 { + version, err := sysgo.HighestEthVersion(caps) + t.Require().NoError(err, "a peer must advertise an eth version") + return version +} + +// TestOpRethAndOpGethNegotiateTheHighestCommonEthVersion peers an op-reth and an op-geth follower +// as basic peers, with either client dialing, and requires both to report the same negotiated eth +// protocol version through admin_peers: the highest version both advertise. The advertised +// versions are logged, so a run records which version the pair settled on. +func TestOpRethAndOpGethNegotiateTheHighestCommonEthVersion(gt *testing.T) { + t := devtest.ParallelT(gt) + devtest.RunParallel(t, followerCells("%s_dials_%s"), func(t devtest.T, dialer followerCell) { + skipUnsupported(t) + sys := newPairSystem(t, dialer.Kind) + opReth := sys.follower(sysgo.MixedL2ELOpReth).EL + opGeth := sys.follower(sysgo.MixedL2ELOpGeth).EL + + rethReports := opReth.PeerInfoOf(opGeth) + gethReports := opGeth.PeerInfoOf(opReth) + // Each side reports the capabilities the other advertised. + opRethCaps, opGethCaps := gethReports.Caps, rethReports.Caps + t.Logger().Info("Negotiated eth version", + "op_reth_advertises", highestEthVersion(t, opRethCaps), "op_geth_advertises", highestEthVersion(t, opGethCaps), + "op_reth_reports", rethReports.EthVersion, "op_geth_reports", gethReports.EthVersion, + "op_reth_caps", opRethCaps, "op_geth_caps", opGethCaps) + + t.Require().Equal(rethReports.EthVersion, gethReports.EthVersion, "both clients must report the same negotiated eth version") + highestCommon, err := sysgo.HighestCommonEthVersion(opRethCaps, opGethCaps) + t.Require().NoError(err, "the clients must share an eth version") + t.Require().Equal(highestCommon, rethReports.EthVersion, "the pair must negotiate the highest eth version both advertise") + }) +} diff --git a/op-acceptance-tests/tests/op_reth_opgeth_peering/funding_test.go b/op-acceptance-tests/tests/op_reth_opgeth_peering/funding_test.go new file mode 100644 index 00000000000..c2b1252478e --- /dev/null +++ b/op-acceptance-tests/tests/op_reth_opgeth_peering/funding_test.go @@ -0,0 +1,130 @@ +package op_reth_opgeth_peering + +import ( + "math/big" + + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/params" + + "github.com/ethereum-optimism/optimism/op-devstack/devtest" + "github.com/ethereum-optimism/optimism/op-devstack/dsl" + "github.com/ethereum-optimism/optimism/op-service/eth" + "github.com/ethereum-optimism/optimism/op-service/txplan" +) + +const ( + // fanOut is how many transactions one account can hold in a reth pool at once (reth's + // --txpool.max-account-slots default), so how many accounts one account funds per round. + fanOut = 16 + + // feeCapGwei is pinned on every transaction the tests sign offline. It leaves room for the + // base fee to climb through the full blocks a burst produces. + feeCapGwei = 100 + // fundingL1DataFeeAllowanceGwei bounds the L1 data fee of one funding transfer, which a pool + // adds to the transfer's cost when it checks the payer's balance. A devnet charges orders of + // magnitude less. + fundingL1DataFeeAllowanceGwei = 100_000 + // tipGwei is pinned on every transaction the tests sign offline. + tipGwei = 1 + + // fundingRoundAttemptsPerBlock bounds, in 1s polls, how long the sequencer may take to build + // each block a funding round's gas fills. + fundingRoundAttemptsPerBlock = 10 * l2BlockTimeSeconds + // fundingRoundBaseAttempts bounds, in 1s polls, how long the sequencer may take to start + // including a funding round once it is sent. + fundingRoundBaseAttempts = 30 +) + +var ( + feeCap = big.NewInt(feeCapGwei * params.GWei) + // fundingTxCost is what one funding transfer may cost its payer: its L2 execution at the + // pinned fee cap, and its L1 data fee. + fundingTxCost = eth.GWei(feeCapGwei).Mul(params.TxGas).Add(eth.GWei(fundingL1DataFeeAllowanceGwei)) + tip = big.NewInt(tipGwei * params.GWei) +) + +// fundAccounts returns count fresh accounts holding at least amount each, bound to the +// sequencer's EL, and the number of a block by which the sequencer has funded all of them. The +// funder pays up to fanOut distributors in one round, each distributor pays up to fanOut more in +// the next, and so on until a round pays the count accounts, so no account ever has more than +// fanOut transactions in a pool at once. Every round is one batch of offline signed transfers to +// the sequencer, waited on until the sequencer includes all of them. +func fundAccounts(t devtest.T, sys *pairSystem, count int, amount eth.ETH) ([]*dsl.EOA, uint64) { + levels := fundingLevels(count) + amounts := fundingAmounts(levels, amount) + t.Logger().Info("Funding accounts through distributors", "count", count, "levels", levels, "amounts", amounts) + accounts := sys.FunderL2.NewFundedEOAs(levels[0], amounts[0]) + fundedAt := sys.Sequencer.EL.BlockRefByLabel(eth.Unsafe).Number + for level := 1; level < len(levels); level++ { + accounts, fundedAt = fundRound(t, sys, accounts, levels[level], amounts[level]) + } + return accounts, fundedAt +} + +// fundingLevels returns how many accounts each funding round creates, the last being count, with +// every earlier round creating enough payers for the next. +func fundingLevels(count int) []int { + levels := []int{count} + for levels[0] > fanOut { + payers := (levels[0] + fanOut - 1) / fanOut + levels = append([]int{payers}, levels...) + } + return levels +} + +// fundingAmounts returns what an account of each round is funded with: the last round's accounts +// get amount, and a payer gets what it pays out plus the cost of paying it. +func fundingAmounts(levels []int, amount eth.ETH) []eth.ETH { + amounts := make([]eth.ETH, len(levels)) + amounts[len(levels)-1] = amount + for level := len(levels) - 2; level >= 0; level-- { + amounts[level] = amounts[level+1].Add(fundingTxCost).Mul(fanOut) + } + return amounts +} + +// fundingRoundAttempts bounds, in 1s polls, how long the sequencer may take to include a funding +// round of transfers: fundingRoundBaseAttempts plus fundingRoundAttemptsPerBlock for every block +// the round's gas fills at blockGasLimit, a partly filled last block included. +func fundingRoundAttempts(transfers int, blockGasLimit uint64) int { + transfersPerBlock := int(blockGasLimit / params.TxGas) + blocks := (transfers + transfersPerBlock - 1) / transfersPerBlock + return fundingRoundBaseAttempts + blocks*fundingRoundAttemptsPerBlock +} + +// fundRound has each payer fund up to fanOut fresh accounts with amount, in one batch of offline +// signed transfers, and returns the funded accounts once the sequencer includes every transfer, +// with the number of a block by which it has. +func fundRound(t devtest.T, sys *pairSystem, payers []*dsl.EOA, count int, amount eth.ETH) ([]*dsl.EOA, uint64) { + t.Require().GreaterOrEqual(len(payers)*fanOut, count, "not enough payers for the round") + accounts := make([]*dsl.EOA, 0, count) + transfers := make([]*types.Transaction, 0, count) + for i, payer := range payers { + paidBefore := len(transfers) + for nonce := uint64(0); nonce < fanOut && len(accounts) < count; nonce++ { + account := sys.Wallet.NewEOA(sys.Sequencer.EL) + to := account.Address() + transfer := payer.Sign( + payer.PlanOffline(), + txplan.WithStaticNonce(nonce), + txplan.WithTo(&to), + txplan.WithValue(amount), + txplan.WithGasLimit(params.TxGas), + txplan.WithGasFeeCap(feeCap), + txplan.WithGasTipCap(tip), + ) + accounts = append(accounts, account) + transfers = append(transfers, transfer) + } + t.Require().Greater(len(transfers), paidBefore, "payer %d of %d had nothing to pay", i, len(payers)) + } + sentAfter := sys.Sequencer.EL.BlockRefByLabel(eth.Unsafe).Number + blockGasLimit := sys.Sequencer.EL.GasLimitAtBlock(sentAfter) + t.Require().GreaterOrEqual(blockGasLimit, params.TxGas, "a block must hold a funding transfer") + attempts := fundingRoundAttempts(len(transfers), blockGasLimit) + sys.Sequencer.EL.SendTransactions(transfers) + _, nextBlock := sys.Sequencer.EL.WaitForIncludedTransactions(transfers, sentAfter+1, len(transfers), attempts) + fundedAt := nextBlock - 1 + t.Logger().Info("Funded a round of accounts", "count", len(accounts), "payers", len(payers), "amount", amount, "funded_at", fundedAt) + return accounts, fundedAt +} diff --git a/op-acceptance-tests/tests/op_reth_opgeth_peering/funding_unit_test.go b/op-acceptance-tests/tests/op_reth_opgeth_peering/funding_unit_test.go new file mode 100644 index 00000000000..5755ec3633b --- /dev/null +++ b/op-acceptance-tests/tests/op_reth_opgeth_peering/funding_unit_test.go @@ -0,0 +1,56 @@ +package op_reth_opgeth_peering + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/ethereum-optimism/optimism/op-service/eth" +) + +func TestFundingLevelsEndWithTheCountAndKeepEveryRoundWithinTheFanOut(t *testing.T) { + for _, tc := range []struct { + count int + levels []int + }{ + {count: 1, levels: []int{1}}, + {count: 16, levels: []int{16}}, + {count: 17, levels: []int{2, 17}}, + {count: 256, levels: []int{16, 256}}, + {count: 257, levels: []int{2, 17, 257}}, + {count: 4112, levels: []int{2, 17, 257, 4112}}, + {count: 8192, levels: []int{2, 32, 512, 8192}}, + } { + t.Run(fmt.Sprintf("%d_accounts", tc.count), func(t *testing.T) { + require.Equal(t, tc.levels, fundingLevels(tc.count)) + }) + } +} + +func TestFundingAmountsCoverWhatEachRoundPaysOut(t *testing.T) { + // At a 100 gwei fee cap and a 100,000 gwei L1 data fee allowance a transfer costs 2,200,000 + // gwei; a payer of 16 accounts at 1,000 gwei each needs 16 x 2,201,000 gwei, and its own payer + // 16 x (35,216,000 + 2,200,000) gwei. + amounts := fundingAmounts([]int{2, 17, 257}, eth.GWei(1_000)) + require.Equal(t, []eth.ETH{eth.GWei(598_656_000), eth.GWei(35_216_000), eth.GWei(1_000)}, amounts) +} + +func TestFundingRoundAttemptsGrowWithTheBlocksTheRoundFills(t *testing.T) { + const blockGasLimit = 60_000_000 // 2857 transfers per block + for _, tc := range []struct { + name string + transfers int + + attempts int + }{ + {name: "one_transfer", transfers: 1, attempts: fundingRoundBaseAttempts + fundingRoundAttemptsPerBlock}, + {name: "one_full_block", transfers: 2857, attempts: fundingRoundBaseAttempts + fundingRoundAttemptsPerBlock}, + {name: "one_over_a_full_block", transfers: 2858, attempts: fundingRoundBaseAttempts + 2*fundingRoundAttemptsPerBlock}, + {name: "burst_last_round", transfers: 8192, attempts: fundingRoundBaseAttempts + 3*fundingRoundAttemptsPerBlock}, + } { + t.Run(tc.name, func(t *testing.T) { + require.Equal(t, tc.attempts, fundingRoundAttempts(tc.transfers, blockGasLimit)) + }) + } +} diff --git a/op-acceptance-tests/tests/op_reth_opgeth_peering/propagation_test.go b/op-acceptance-tests/tests/op_reth_opgeth_peering/propagation_test.go new file mode 100644 index 00000000000..31b0ac8a070 --- /dev/null +++ b/op-acceptance-tests/tests/op_reth_opgeth_peering/propagation_test.go @@ -0,0 +1,50 @@ +package op_reth_opgeth_peering + +import ( + "testing" + + "github.com/ethereum/go-ethereum/core/types" + + "github.com/ethereum-optimism/optimism/op-devstack/devtest" + "github.com/ethereum-optimism/optimism/op-service/eth" +) + +const ( + // crossingAttempts bounds, in 1s polls, how long a transfer may take to cross the peering + // into the receiver's pool. + crossingAttempts = 30 + // inclusionAttempts bounds, in 1s polls, how long the resumed sequencer may take to include + // the transfer and each follower to import the block that does. + inclusionAttempts = 30 +) + +// TestTransactionCrossesTheOpRethOpGethPeeringIntoABlock submits a transfer to one follower while +// the sequencer is paused and requires the other follower, the sender's only execution-layer +// peer, to hold it in its pool, and then both followers to see it in the same block once the +// sequencer resumes; from op-reth to op-geth and from op-geth to op-reth. +func TestTransactionCrossesTheOpRethOpGethPeeringIntoABlock(gt *testing.T) { + t := devtest.ParallelT(gt) + devtest.RunParallel(t, followerCells("%s_to_%s"), func(t devtest.T, sender followerCell) { + skipUnsupported(t) + sys := newPairSystem(t, sender.Kind) + alice := sys.FunderL2.NewFundedEOA(eth.OneEther).AsEL(sys.Sender.EL) + // Both followers check the transfer against their own state, which must hold the + // funding by then: the sender when it plans the transfer, the receiver when it admits + // it to its pool. + alice.WaitForBalanceAtLeast(eth.OneEther) + alice.AsEL(sys.Receiver.EL).WaitForBalanceAtLeast(eth.OneEther) + bob := sys.Wallet.NewEOA(sys.Sender.EL) + + pausedHead := sys.Sequencer.EL.BlockRefByHash(sys.Sequencer.CL.StopSequencer()) + transfer := alice.Submit(alice.PlanTransfer(bob.Address(), eth.OneHundredthEther)) + sys.Receiver.EL.WaitForPooledTransaction(transfer.Hash(), crossingAttempts) + + sys.Sequencer.CL.StartSequencer() + transfers := []*types.Transaction{transfer} + sys.Receiver.EL.WaitForIncludedTransactions(transfers, pausedHead.Number+1, len(transfers), inclusionAttempts) + sys.Sender.EL.WaitForIncludedTransactions(transfers, pausedHead.Number+1, len(transfers), inclusionAttempts) + onReceiver := sys.Receiver.EL.WaitForReceipt(transfer.Hash()) + onSender := sys.Sender.EL.WaitForReceipt(transfer.Hash()) + t.Require().Equal(onReceiver.BlockHash, onSender.BlockHash, "both followers must see the transfer in the same block") + }) +} diff --git a/op-acceptance-tests/tests/op_reth_opgeth_peering/system_test.go b/op-acceptance-tests/tests/op_reth_opgeth_peering/system_test.go new file mode 100644 index 00000000000..6e3a959570f --- /dev/null +++ b/op-acceptance-tests/tests/op_reth_opgeth_peering/system_test.go @@ -0,0 +1,165 @@ +package op_reth_opgeth_peering + +import ( + "fmt" + + "github.com/ethereum/go-ethereum/eth/ethconfig" + "github.com/ethereum/go-ethereum/node" + + "github.com/ethereum-optimism/optimism/op-devstack/devtest" + "github.com/ethereum-optimism/optimism/op-devstack/dsl" + "github.com/ethereum-optimism/optimism/op-devstack/presets" + "github.com/ethereum-optimism/optimism/op-devstack/sysgo" + "github.com/ethereum-optimism/optimism/op-e2e/e2eutils/geth" + "github.com/ethereum-optimism/optimism/op-service/eth" +) + +const ( + // l2BlockTimeSeconds keeps the runs short. + l2BlockTimeSeconds = 1 + // senderSyncAttempts bounds, in 2s polls, how long the sender may take to catch up with the + // chain once peered. + senderSyncAttempts = 60 +) + +// followerKinds are the two execution clients of the pair; a matrix over them runs the op-reth +// cell first. +var followerKinds = []sysgo.MixedL2ELKind{sysgo.MixedL2ELOpReth, sysgo.MixedL2ELOpGeth} + +// otherKind returns the execution client kind that pairs with kind. +func otherKind(kind sysgo.MixedL2ELKind) sysgo.MixedL2ELKind { + if kind == sysgo.MixedL2ELOpReth { + return sysgo.MixedL2ELOpGeth + } + return sysgo.MixedL2ELOpReth +} + +// followerCell is one cell of a matrix over followerKinds: the follower that acts, and the +// subtest name. +type followerCell struct { + Kind sysgo.MixedL2ELKind + Name string +} + +func (cell followerCell) String() string { + return cell.Name +} + +// followerCells returns a cell per follower kind, named by nameFormat applied to the acting kind +// and the other kind. +func followerCells(nameFormat string) []followerCell { + cells := make([]followerCell, len(followerKinds)) + for i, kind := range followerKinds { + cells[i] = followerCell{Kind: kind, Name: fmt.Sprintf(nameFormat, kind, otherKind(kind))} + } + return cells +} + +type pairNode struct { + CL *dsl.L2CLNode + EL *dsl.L2ELNode + Kind sysgo.MixedL2ELKind +} + +// pairSystem is an op-reth sequencer with an op-reth follower and an op-geth follower. The sender +// follower has the receiver follower as its only execution-layer peer, through a basic peering it +// dialed, while the receiver keeps the trusted peering with the sequencer that the mixed runtime +// sets up. A transaction submitted to the sender can only reach the sequencer through the +// receiver, so its inclusion proves it crossed the peering under test. The sender's consensus +// client is peered with the sequencer's alone, so the sender follows the chain, and it is on the +// chain by the time the system is returned. The op-reth nodes get pending and basefee sub-pools of +// burstPoolMaxSizeMB and burstPoolMaxCount, and op-geth a pool of burstPoolMaxCount. +type pairSystem struct { + FunderL2 *dsl.FunderEOA + Receiver pairNode + Sender pairNode + Sequencer pairNode + Wallet *dsl.HDWallet +} + +// follower returns the pair's follower of the given kind. +func (sys *pairSystem) follower(kind sysgo.MixedL2ELKind) pairNode { + if sys.Sender.Kind == kind { + return sys.Sender + } + return sys.Receiver +} + +// withBurstPoolSlots lets op-geth's pool hold burstPoolMaxCount transactions of one slot each, +// the size of every transaction these tests sign. +func withBurstPoolSlots(ethCfg *ethconfig.Config, _ *node.Config) error { + ethCfg.TxPool.GlobalSlots = burstPoolMaxCount + return nil +} + +func newPairSystem(t devtest.T, sender sysgo.MixedL2ELKind) *pairSystem { + clKind := sysgo.ResolveMixedL2CLKind() + followerSpec := func(kind sysgo.MixedL2ELKind) sysgo.MixedSingleChainNodeSpec { + return sysgo.MixedSingleChainNodeSpec{ + CLKey: string(kind) + "-follower", + CLKind: clKind, + ELKey: string(kind) + "-follower", + ELKind: kind, + // The runtime peers every other node with every other as trusted; the sender is kept + // off that mesh and peered below, with the receiver as its only execution-layer peer. + IsolateFromL2P2P: kind == sender, + } + } + runtime := sysgo.NewMixedSingleChainRuntime(t, sysgo.MixedSingleChainPresetConfig{ + // op-geth refuses Karst blocks, which the default dev genesis activates. + DeployerOptions: []sysgo.DeployerOption{ + sysgo.WithJovianAtGenesis, + sysgo.WithUniformL2BlockTimes(l2BlockTimeSeconds), + }, + NodeSpecs: []sysgo.MixedSingleChainNodeSpec{ + { + CLKey: "sequencer", + CLKind: clKind, + ELKey: "sequencer", + ELKind: sysgo.MixedL2ELOpReth, + IsSequencer: true, + }, + followerSpec(sysgo.MixedL2ELOpReth), + followerSpec(sysgo.MixedL2ELOpGeth), + }, + OpGethOptions: []geth.GethOption{withBurstPoolSlots}, + OpRethOptions: []sysgo.OpRethOption{ + sysgo.OpRethWithExtraArgs( + fmt.Sprintf("--txpool.basefee-max-count=%d", burstPoolMaxCount), + fmt.Sprintf("--txpool.basefee-max-size=%d", burstPoolMaxSizeMB), + fmt.Sprintf("--txpool.pending-max-count=%d", burstPoolMaxCount), + fmt.Sprintf("--txpool.pending-max-size=%d", burstPoolMaxSizeMB), + ), + }, + }) + frontends := presets.NewMixedSingleChainFrontends(t, runtime) + t.Require().Len(frontends.Nodes, 3, "expected the sequencer and the two followers") + + sys := &pairSystem{FunderL2: frontends.FunderL2, Wallet: frontends.Wallet} + for _, node := range frontends.Nodes { + pair := pairNode{CL: node.CL, EL: node.EL, Kind: node.Spec.ELKind} + switch { + case node.Spec.IsSequencer: + sys.Sequencer = pair + case node.Spec.ELKind == sender: + sys.Sender = pair + default: + sys.Receiver = pair + } + } + + sys.Sender.CL.ConnectPeer(sys.Sequencer.CL) + sys.Sender.CL.IsP2PConnected(sys.Sequencer.CL) + sys.Sender.EL.PeerWithBasic(sys.Receiver.EL) + t.Require().Len(sys.Sender.EL.Peers(), 1, "the sender must have the receiver as its only execution-layer peer") + // The sender started behind the chain. It catches up through its consensus-layer peering with + // the sequencer, or through the receiver, its only execution-layer peer, when its consensus + // client leaves the gap to execution-layer sync. A test may plan transactions against the + // sender's state or have it admit them only once it is on the chain. + sys.Sender.EL.Reached(eth.Unsafe, sys.Sequencer.EL.BlockRefByLabel(eth.Unsafe).Number, senderSyncAttempts) + return sys +} + +func skipUnsupported(t devtest.T) { + sysgo.SkipOnOpGeth(t, "the pair's sequencer is this tree's op-reth") +} diff --git a/op-devstack/dsl/eoa.go b/op-devstack/dsl/eoa.go index b001dab80b2..ff2d60b63a2 100644 --- a/op-devstack/dsl/eoa.go +++ b/op-devstack/dsl/eoa.go @@ -170,6 +170,17 @@ func (u *EOA) PlanAuth(code common.Address) txplan.Option { ) } +// PlanOffline creates the tx-planning options to sign a transaction with this Key for its chain +// without reading anything from the EL node. The nonce and gas default to 0 and 21000, so a caller +// pins what differs; the fee cap must be pinned with WithGasFeeCap, since the default fee cap +// reads the base fee from a node. +func (u *EOA) PlanOffline() txplan.Option { + return txplan.Combine( + u.key.Plan(), + txplan.WithStaticChainID(u.ChainID()), + ) +} + // PlanTransfer creates the tx-plan options to perform a transfer // of the given amount of ETH to the given account. func (u *EOA) PlanTransfer(to common.Address, amount eth.ETH) txplan.Option { @@ -186,6 +197,24 @@ func (u *EOA) Transfer(to common.Address, amount eth.ETH) *txplan.PlannedTx { return u.Transact(u.PlanTransfer(to, amount)) } +// Sign plans and signs a tx without submitting it, and returns the signed tx. +func (u *EOA) Sign(opts ...txplan.Option) *types.Transaction { + tx := txplan.NewPlannedTx(txplan.Combine(opts...)) + signed, err := tx.Signed.Eval(u.ctx) + u.require.NoError(err, "must sign") + return signed +} + +// Submit plans, signs and submits a tx, and returns the signed tx without waiting for inclusion. +func (u *EOA) Submit(opts ...txplan.Option) *types.Transaction { + tx := txplan.NewPlannedTx(txplan.Combine(opts...)) + _, err := tx.Submitted.Eval(u.ctx) + u.require.NoError(err, "must submit") + signed, err := tx.Signed.Get() + u.require.NoError(err, "must have signed") + return signed +} + // Transact plans and executes a tx. // The success-state, as defined by the tx-plan options, is required. // The resulting evaluated tx is returned. diff --git a/op-devstack/dsl/l2_el.go b/op-devstack/dsl/l2_el.go index 0206ba647a4..638a45439a5 100644 --- a/op-devstack/dsl/l2_el.go +++ b/op-devstack/dsl/l2_el.go @@ -586,14 +586,6 @@ func (el *L2ELNode) RecordLogs(minLevel slog.Level) *LogRecorder { return recorder } -func (el *L2ELNode) PeerWith(peer *L2ELNode) { - sysgo.ConnectP2P(el.ctx, el.require, el.inner.L2EthClient().RPC(), peer.inner.L2EthClient().RPC()) -} - -func (el *L2ELNode) DisconnectPeerWith(peer *L2ELNode) { - sysgo.DisconnectP2P(el.ctx, el.require, el.inner.L2EthClient().RPC(), peer.inner.L2EthClient().RPC()) -} - // Enode returns the node's enode URL, which identifies it to its peers across restarts. func (el *L2ELNode) Enode() string { var info p2p.NodeInfo diff --git a/op-devstack/dsl/l2_el_p2p.go b/op-devstack/dsl/l2_el_p2p.go new file mode 100644 index 00000000000..df07b1f2fba --- /dev/null +++ b/op-devstack/dsl/l2_el_p2p.go @@ -0,0 +1,58 @@ +package dsl + +import ( + "github.com/ethereum-optimism/optimism/op-devstack/sysgo" + "github.com/ethereum-optimism/optimism/op-service/client" +) + +// PeerWith connects el to peer and registers the peering as trusted on both sides, so the session +// survives reputation penalties, bans and eviction. +func (el *L2ELNode) PeerWith(peer *L2ELNode) { + el.log.Info("Peering with a trusted peer", "node", el, "peer", peer) + sysgo.ConnectP2P(el.ctx, el.require, el.rpc(), peer.rpc()) +} + +// PeerWithBasic connects el to peer as a basic peer: el dials peer and keeps it as a static entry, +// and peer keeps nothing, so peer's side of the session stays subject to reputation penalties, +// bans and eviction and a test can observe how peer manages el, which PeerWith masks. An op-reth +// el exempts its static entry from most reputation penalties and from eviction. +func (el *L2ELNode) PeerWithBasic(peer *L2ELNode) { + el.log.Info("Peering with a basic peer", "node", el, "peer", peer) + sysgo.ConnectP2PBasic(el.ctx, el.require, el.rpc(), peer.rpc()) +} + +// DisconnectPeerWith undoes a PeerWith peering: it drops the trusted entries and the peer on both +// sides and waits for the session to end. It requires the trusted entries, so it is not the +// inverse of PeerWithBasic. +func (el *L2ELNode) DisconnectPeerWith(peer *L2ELNode) { + el.log.Info("Disconnecting from a peer", "node", el, "peer", peer) + sysgo.DisconnectP2P(el.ctx, el.require, el.rpc(), peer.rpc()) +} + +// VerifyPeeredWith requires peer to be among el's connected peers, waiting for it to show up. +func (el *L2ELNode) VerifyPeeredWith(peer *L2ELNode) { + el.log.Info("Expecting a peer to be connected", "node", el, "peer", peer) + sysgo.WaitP2PConnected(el.ctx, el.require, el.rpc(), peer.rpc()) +} + +// PeerInfoOf returns what el reports about its session with peer, once the session has +// negotiated its eth protocol version. +func (el *L2ELNode) PeerInfoOf(peer *L2ELNode) sysgo.ELPeerInfo { + el.log.Info("Reading peer info", "node", el, "peer", peer) + info := sysgo.PeerInfo(el.ctx, el.require, el.rpc(), peer.rpc()) + el.log.Info("Read peer info", "node", el, "peer", peer, "eth_version", info.EthVersion, "caps", info.Caps, + "inbound", info.Inbound, "static", info.Static, "trusted", info.Trusted, "remote_address", info.RemoteAddress) + return info +} + +// Peers returns what el reports about every peer it is connected to, with an EthVersion of 0 for +// a session whose eth handshake is still pending. +func (el *L2ELNode) Peers() []sysgo.ELPeerInfo { + peers, err := sysgo.ListPeers(el.ctx, el.rpc()) + el.require.NoError(err, "failed to list the peers of %s", el) + return peers +} + +func (el *L2ELNode) rpc() client.RPC { + return el.inner.L2EthClient().RPC() +} diff --git a/op-devstack/dsl/l2_el_txpool.go b/op-devstack/dsl/l2_el_txpool.go new file mode 100644 index 00000000000..4eafcf349ea --- /dev/null +++ b/op-devstack/dsl/l2_el_txpool.go @@ -0,0 +1,410 @@ +package dsl + +import ( + "context" + "fmt" + "maps" + "time" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/rpc" + + "github.com/ethereum-optimism/optimism/op-service/eth" + "github.com/ethereum-optimism/optimism/op-service/retry" +) + +const ( + // sendTransactionsBatchSize bounds one eth_sendRawTransaction batch call. It stays under + // op-geth's default limit of 1000 calls per batch, and keeps a batch of 4 KiB transactions + // well inside both clients' request size limits. + sendTransactionsBatchSize = 512 + // sendTransactionsTimeout bounds one batch call: a node validates every transaction of the + // batch before it answers. + sendTransactionsTimeout = time.Minute + // txPoolContentBatchSize bounds one txpool_contentFrom batch call. Its answer for senders of + // 4 KiB transactions stays a few MB, well inside the client's 32 MB websocket read limit. + txPoolContentBatchSize = 512 + // txPoolPollInterval is how often the pool waits look at the pool. + txPoolPollInterval = time.Second +) + +// SendTransactions submits signed transactions to the node's pool in batch calls, back to back, +// and fails on the first one the node refuses. It does not wait for inclusion. +func (el *L2ELNode) SendTransactions(txs []*types.Transaction) { + el.log.Info("Sending transactions", "node", el, "count", len(txs)) + for start := 0; start < len(txs); start += sendTransactionsBatchSize { + batch := txs[start:min(start+sendTransactionsBatchSize, len(txs))] + calls := make([]rpc.BatchElem, len(batch)) + for i, tx := range batch { + raw, err := tx.MarshalBinary() + el.require.NoError(err, "must encode transaction %s", tx.Hash()) + calls[i] = rpc.BatchElem{Method: "eth_sendRawTransaction", Args: []any{hexutil.Bytes(raw)}, Result: new(common.Hash)} + } + ctx, cancel := context.WithTimeout(el.ctx, sendTransactionsTimeout) + err := el.rpc().BatchCallContext(ctx, calls) + cancel() + el.require.NoError(err, "the batch of %d transactions from index %d must be sent to %s", len(batch), start, el) + for i, call := range calls { + el.require.NoError(call.Error, "transaction %s (index %d of %d) must be accepted by %s", batch[i].Hash(), start+i, len(txs), el) + } + } + el.log.Info("Sent transactions", "node", el, "count", len(txs)) +} + +// WaitForPooledTransaction waits, polling once a second up to attempts times, until the node +// holds the transaction in its pool: known to the node and not yet in a block. A transaction the +// node first sees in a block fails the wait. +func (el *L2ELNode) WaitForPooledTransaction(hash common.Hash, attempts int) { + el.log.Info("Waiting for a transaction to be pooled", "node", el, "tx", hash, "attempts", attempts) + var inBlock *hexutil.Big + err := retry.Do0(el.ctx, attempts, &retry.FixedStrategy{Dur: txPoolPollInterval}, func() error { + var tx *struct { + BlockNumber *hexutil.Big `json:"blockNumber"` + } + ctx, cancel := context.WithTimeout(el.ctx, DefaultTimeout) + defer cancel() + if err := el.rpc().CallContext(ctx, &tx, "eth_getTransactionByHash", hash); err != nil { + el.log.Warn("Transaction lookup failed; will retry", "node", el, "tx", hash, "err", err) + return fmt.Errorf("looking up transaction %s on %s: %w", hash, el, err) + } + if tx == nil { + el.log.Info("Transaction not known yet", "node", el, "tx", hash) + return fmt.Errorf("transaction %s is not known to %s yet", hash, el) + } + inBlock = tx.BlockNumber + return nil + }) + el.require.NoError(err, "transaction %s did not reach the pool of %s", hash, el) + el.require.Nilf(inBlock, "transaction %s was first seen in block %s on %s, not in its pool", hash, inBlock, el) + el.log.Info("Transaction is pooled", "node", el, "tx", hash) +} + +// TxPoolStatus is how many transactions a node's pool holds. +type TxPoolStatus struct { + // Pending transactions can go into the next block. + Pending uint64 + // Queued transactions wait for something, mostly an earlier nonce, before they can. + Queued uint64 +} + +func (s TxPoolStatus) Total() uint64 { + return s.Pending + s.Queued +} + +func (s TxPoolStatus) String() string { + return fmt.Sprintf("%d pending, %d queued", s.Pending, s.Queued) +} + +func (el *L2ELNode) txPoolStatus() (TxPoolStatus, error) { + var status struct { + Pending hexutil.Uint64 `json:"pending"` + Queued hexutil.Uint64 `json:"queued"` + } + ctx, cancel := context.WithTimeout(el.ctx, DefaultTimeout) + defer cancel() + if err := el.rpc().CallContext(ctx, &status, "txpool_status"); err != nil { + return TxPoolStatus{}, fmt.Errorf("reading the pool status of %s: %w", el, err) + } + return TxPoolStatus{Pending: uint64(status.Pending), Queued: uint64(status.Queued)}, nil +} + +// TxPoolStatus reads how many transactions the node's pool holds right now. +func (el *L2ELNode) TxPoolStatus() TxPoolStatus { + status, err := el.txPoolStatus() + el.require.NoError(err) + return status +} + +// transactionHashes returns the set of the hashes of txs. +func transactionHashes(txs []*types.Transaction) map[common.Hash]struct{} { + hashes := make(map[common.Hash]struct{}, len(txs)) + for _, tx := range txs { + hashes[tx.Hash()] = struct{}{} + } + return hashes +} + +// collectIncludedTransactions adds to included the hashes of those of wanted that the node's +// canonical block number includes. +func (el *L2ELNode) collectIncludedTransactions(wanted, included map[common.Hash]struct{}, number uint64) error { + el.log.Debug("Fetching block transactions", "node", el, "block", number) + ctx, cancel := context.WithTimeout(el.ctx, DefaultTimeout) + _, blockTxs, err := el.EthClient().InfoAndTxsByNumber(ctx, number) + cancel() + if err != nil { + el.log.Error("Failed to fetch block transactions", "node", el, "block", number, "err", err) + return fmt.Errorf("fetching block %d from %s: %w", number, el, err) + } + var includedInBlock int + for _, tx := range blockTxs { + if _, ok := wanted[tx.Hash()]; ok { + included[tx.Hash()] = struct{}{} + includedInBlock++ + } + } + el.log.Debug("Fetched block transactions", "node", el, "block", number, "transactions", len(blockTxs), "included", includedInBlock) + return nil +} + +// IncludedTransactions returns the hashes of those of txs that the node's canonical blocks from +// fromBlock through toBlock include. +func (el *L2ELNode) IncludedTransactions(txs []*types.Transaction, fromBlock, toBlock uint64) map[common.Hash]struct{} { + el.log.Info("Collecting included transactions", "node", el, "count", len(txs), "from", fromBlock, "to", toBlock) + wanted := transactionHashes(txs) + included := make(map[common.Hash]struct{}) + for number := fromBlock; number <= toBlock; number++ { + el.require.NoError(el.collectIncludedTransactions(wanted, included, number)) + } + el.log.Info("Collected included transactions", "node", el, "count", len(txs), "from", fromBlock, "to", toBlock, "included", len(included)) + return included +} + +// WaitForIncludedTransactions waits until the node's canonical blocks from fromBlock up to its +// unsafe head include at least count of txs, and returns the hashes of those of txs the blocks it +// read include, with the number of the first block it did not read. It reads every block once, +// so a reorg of a block it has read goes unnoticed. +func (el *L2ELNode) WaitForIncludedTransactions(txs []*types.Transaction, fromBlock uint64, count int, attempts int) (map[common.Hash]struct{}, uint64) { + el.log.Info("Waiting for transactions to be included", "node", el, "count", count, "of", len(txs), "from", fromBlock, "attempts", attempts) + wanted := transactionHashes(txs) + included := make(map[common.Hash]struct{}) + nextBlock := fromBlock + err := retry.Do0(el.ctx, attempts, &retry.FixedStrategy{Dur: txPoolPollInterval}, func() error { + head, err := el.blockRefByLabel(eth.Unsafe) + if err != nil { + el.log.Warn("Unsafe head lookup failed; will retry", "node", el, "err", err) + return err + } + for ; nextBlock <= head.Number; nextBlock++ { + if err := el.collectIncludedTransactions(wanted, included, nextBlock); err != nil { + return err + } + } + if len(included) < count { + el.log.Info("Not enough transactions included yet", "node", el, "included", len(included), "count", count, "head", head.Number) + return fmt.Errorf("blocks %d through %d of %s include %d of the transactions, expecting %d", fromBlock, head.Number, el, len(included), count) + } + return nil + }) + el.require.NoError(err, "%s did not include %d of the transactions", el, count) + el.log.Info("Transactions included", "node", el, "included", len(included), "count", count, "through", nextBlock-1) + return included, nextBlock +} + +// pooledTransaction is the part of a txpool_contentFrom entry that identifies the transaction. +type pooledTransaction struct { + Hash common.Hash `json:"hash"` +} + +// senderPoolContent is a txpool_contentFrom answer: one sender's pooled transactions by nonce. +type senderPoolContent struct { + Pending map[string]pooledTransaction `json:"pending"` + Queued map[string]pooledTransaction `json:"queued"` +} + +// txPoolLookup is a set of transactions to look up in a pool by sender. +type txPoolLookup struct { + hashes map[common.Hash]struct{} + // senders are the distinct senders of the transactions, in the order they first sign one. + senders []common.Address +} + +func newTxPoolLookup(txs []*types.Transaction) (txPoolLookup, error) { + lookup := txPoolLookup{hashes: transactionHashes(txs)} + seen := make(map[common.Address]struct{}, len(txs)) + for _, tx := range txs { + sender, err := types.Sender(types.LatestSignerForChainID(tx.ChainId()), tx) + if err != nil { + return txPoolLookup{}, fmt.Errorf("recovering the sender of transaction %s: %w", tx.Hash(), err) + } + if _, ok := seen[sender]; !ok { + seen[sender] = struct{}{} + lookup.senders = append(lookup.senders, sender) + } + } + return lookup, nil +} + +// withSenders returns the lookup narrowed to senders. +func (lookup txPoolLookup) withSenders(senders []common.Address) txPoolLookup { + return txPoolLookup{hashes: lookup.hashes, senders: senders} +} + +// poolHoldings is what a node's pool holds of a lookup's transactions. +type poolHoldings struct { + // holders are the senders of the lookup with at least one of its transactions in the pool. + holders []common.Address + + pending map[common.Hash]struct{} + queued map[common.Hash]struct{} +} + +// all returns the hashes of the held transactions, pending or queued. +func (holdings poolHoldings) all() map[common.Hash]struct{} { + all := maps.Clone(holdings.pending) + maps.Copy(all, holdings.queued) + return all +} + +// pooledTransactions reads what the node's pool holds right now of the lookup's transactions, +// from the pool contents of the lookup's senders, in batch calls. +func (el *L2ELNode) pooledTransactions(lookup txPoolLookup) (poolHoldings, error) { + holdings := poolHoldings{pending: make(map[common.Hash]struct{}), queued: make(map[common.Hash]struct{})} + for start := 0; start < len(lookup.senders); start += txPoolContentBatchSize { + batch := lookup.senders[start:min(start+txPoolContentBatchSize, len(lookup.senders))] + contents := make([]senderPoolContent, len(batch)) + calls := make([]rpc.BatchElem, len(batch)) + for i, sender := range batch { + calls[i] = rpc.BatchElem{Method: "txpool_contentFrom", Args: []any{sender}, Result: &contents[i]} + } + el.log.Debug("Reading the pool contents of senders", "node", el, "from_index", start, "senders", len(batch)) + ctx, cancel := context.WithTimeout(el.ctx, DefaultTimeout) + err := el.rpc().BatchCallContext(ctx, calls) + cancel() + if err != nil { + el.log.Error("Failed to read the pool contents of senders", "node", el, "from_index", start, "senders", len(batch), "err", err) + return poolHoldings{}, fmt.Errorf("reading the pool contents of %d senders from index %d on %s: %w", len(batch), start, el, err) + } + for i, call := range calls { + if call.Error != nil { + el.log.Error("Failed to read the pool contents of a sender", "node", el, "sender", batch[i], "err", call.Error) + return poolHoldings{}, fmt.Errorf("reading the pool contents of sender %s on %s: %w", batch[i], el, call.Error) + } + pendingFound := collectWanted(lookup.hashes, contents[i].Pending, holdings.pending) + queuedFound := collectWanted(lookup.hashes, contents[i].Queued, holdings.queued) + if pendingFound || queuedFound { + holdings.holders = append(holdings.holders, batch[i]) + } + } + el.log.Debug("Read the pool contents of senders", "node", el, "from_index", start, "senders", len(batch)) + } + return holdings, nil +} + +// collectWanted adds to found the hashes of those of subPool's transactions that are in wanted, +// and reports whether there was any. +func collectWanted(wanted map[common.Hash]struct{}, subPool map[string]pooledTransaction, found map[common.Hash]struct{}) bool { + var foundAny bool + for _, tx := range subPool { + if _, ok := wanted[tx.Hash]; ok { + found[tx.Hash] = struct{}{} + foundAny = true + } + } + return foundAny +} + +// PooledTransactions returns the hashes of those of txs that the node's pool holds right now, +// pending or queued. +func (el *L2ELNode) PooledTransactions(txs []*types.Transaction) map[common.Hash]struct{} { + el.log.Info("Reading pooled transactions", "node", el, "count", len(txs)) + lookup, err := newTxPoolLookup(txs) + el.require.NoError(err) + holdings, err := el.pooledTransactions(lookup) + el.require.NoError(err) + pooled := holdings.all() + el.log.Info("Read pooled transactions", "node", el, "count", len(txs), "pooled", len(pooled)) + return pooled +} + +// WaitForPooledTransactions waits, polling once a second up to attempts times, until the node's +// pool holds at least count of txs, pending or queued, and returns the hashes of those it holds +// then. It reads the pool contents only once the pool holds count transactions in all. +func (el *L2ELNode) WaitForPooledTransactions(txs []*types.Transaction, count int, attempts int) map[common.Hash]struct{} { + el.log.Info("Waiting for transactions to be pooled", "node", el, "count", count, "of", len(txs), "attempts", attempts) + lookup, err := newTxPoolLookup(txs) + el.require.NoError(err) + var pooled map[common.Hash]struct{} + err = retry.Do0(el.ctx, attempts, &retry.FixedStrategy{Dur: txPoolPollInterval}, func() error { + status, err := el.txPoolStatus() + if err != nil { + el.log.Warn("Pool status lookup failed; will retry", "node", el, "err", err) + return err + } + if status.Total() < uint64(count) { + el.log.Info("Pool holds too few transactions yet", "node", el, "count", count, "status", status) + return fmt.Errorf("the pool of %s holds %s, expecting at least %d of the transactions", el, status, count) + } + holdings, err := el.pooledTransactions(lookup) + if err != nil { + el.log.Warn("Pool contents lookup failed; will retry", "node", el, "err", err) + return err + } + pooled = holdings.all() + if len(pooled) < count { + el.log.Info("Not enough transactions pooled yet", "node", el, "pooled", len(pooled), "count", count) + return fmt.Errorf("the pool of %s holds %d of the transactions, expecting %d", el, len(pooled), count) + } + return nil + }) + el.require.NoError(err, "the pool of %s did not take %d of the transactions", el, count) + el.log.Info("Transactions pooled", "node", el, "pooled", len(pooled), "count", count) + return pooled +} + +// WaitForPendingTransactions waits, polling once a second up to attempts times, until the node's +// pool holds every one of txs as pending: ready for the next block, not queued behind anything. +// It reads the pool contents only once the pool holds that many pending transactions in all. +func (el *L2ELNode) WaitForPendingTransactions(txs []*types.Transaction, attempts int) { + el.log.Info("Waiting for transactions to be pending", "node", el, "count", len(txs), "attempts", attempts) + lookup, err := newTxPoolLookup(txs) + el.require.NoError(err) + wanted := len(lookup.hashes) + err = retry.Do0(el.ctx, attempts, &retry.FixedStrategy{Dur: txPoolPollInterval}, func() error { + status, err := el.txPoolStatus() + if err != nil { + el.log.Warn("Pool status lookup failed; will retry", "node", el, "err", err) + return err + } + if status.Pending < uint64(wanted) { + el.log.Info("Pool holds too few pending transactions yet", "node", el, "count", wanted, "status", status) + return fmt.Errorf("the pool of %s holds %s, expecting at least %d pending", el, status, wanted) + } + holdings, err := el.pooledTransactions(lookup) + if err != nil { + el.log.Warn("Pool contents lookup failed; will retry", "node", el, "err", err) + return err + } + if len(holdings.pending) < wanted { + el.log.Info("Not every transaction is pending yet", "node", el, "pending", len(holdings.pending), "queued", len(holdings.queued), "count", wanted) + return fmt.Errorf("the pool of %s holds %d of %d transactions as pending", el, len(holdings.pending), wanted) + } + return nil + }) + el.require.NoError(err, "the pool of %s did not hold all %d transactions as pending", el, wanted) + el.log.Info("Transactions pending", "node", el, "count", wanted) +} + +// WaitForTransactionsOutOfPool waits until the node's pool holds none of txs. A node removes the +// transactions of a block it imported from its pool asynchronously to the import. Each poll reads +// only the senders that still held one of txs on the previous poll; once none does, one more +// poll reads every sender, since a transaction can return to the pool. +func (el *L2ELNode) WaitForTransactionsOutOfPool(txs []*types.Transaction, attempts int) { + el.log.Info("Waiting for transactions to leave the pool", "node", el, "count", len(txs), "attempts", attempts) + everySender, err := newTxPoolLookup(txs) + el.require.NoError(err) + lookup, readEverySender := everySender, true + err = retry.Do0(el.ctx, attempts, &retry.FixedStrategy{Dur: txPoolPollInterval}, func() error { + holdings, err := el.pooledTransactions(lookup) + if err != nil { + el.log.Warn("Pool contents lookup failed; will retry", "node", el, "err", err) + return err + } + pooled := len(holdings.pending) + len(holdings.queued) + switch { + case pooled > 0: + el.log.Info("Transactions still pooled", "node", el, "count", len(txs), "pooled", pooled) + lookup, readEverySender = everySender.withSenders(holdings.holders), false + return fmt.Errorf("the pool of %s still holds %d of %d transactions", el, pooled, len(txs)) + case !readEverySender: + el.log.Info("The remaining senders let go of the transactions; reading every sender", "node", el, "count", len(txs)) + lookup, readEverySender = everySender, true + return fmt.Errorf("the pool of %s has not been read for every sender of the transactions yet", el) + default: + return nil + } + }) + el.require.NoError(err, "the pool of %s did not let go of the transactions", el) + el.log.Info("Transactions left the pool", "node", el, "count", len(txs)) +} diff --git a/op-devstack/sysgo/l2_el_opgeth.go b/op-devstack/sysgo/l2_el_opgeth.go index 91819153d58..a510794e09a 100644 --- a/op-devstack/sysgo/l2_el_opgeth.go +++ b/op-devstack/sysgo/l2_el_opgeth.go @@ -30,6 +30,8 @@ type OpGeth struct { supervisorRPC string l2Geth *geth.GethInstance cfg *L2ELConfig + // gethOptions apply after the node's own geth configuration. + gethOptions []geth.GethOption authRPC string userRPC string @@ -122,6 +124,11 @@ func (n *OpGeth) Start() { } nodeCfg.P2P.TrustedNodes = nodes } + for _, option := range n.gethOptions { + if err := option(ethCfg, nodeCfg); err != nil { + return err + } + } return nil }) require.NoError(err) diff --git a/op-devstack/sysgo/l2_el_p2p_util.go b/op-devstack/sysgo/l2_el_p2p_util.go index e0f9b84aa19..0c92fc07575 100644 --- a/op-devstack/sysgo/l2_el_p2p_util.go +++ b/op-devstack/sysgo/l2_el_p2p_util.go @@ -6,6 +6,7 @@ import ( "fmt" "net" "slices" + "strconv" "strings" "time" @@ -28,14 +29,8 @@ type RpcCaller interface { // after a drop comes from the initiator's static entry (geth) or from either // side's trusted entry (reth). func ConnectP2P(ctx context.Context, require *testreq.Assertions, initiator RpcCaller, acceptor RpcCaller) { - var targetInfo p2p.NodeInfo - require.NoError(acceptor.CallContext(ctx, &targetInfo, "admin_nodeInfo"), "get node info") - targetNode, err := enode.ParseV4(targetInfo.Enode) - require.NoError(err, "failed to parse target node") - expectedID := targetNode.ID().String() - - var initiatorInfo p2p.NodeInfo - require.NoError(initiator.CallContext(ctx, &initiatorInfo, "admin_nodeInfo"), "get initiator node info") + targetInfo, expectedID := nodeInfo(ctx, require, acceptor) + initiatorInfo, _ := nodeInfo(ctx, require, initiator) // Dial first, then mark trusted: on reth, admin_addPeer overwrites the peer // kind with Static, so calling it after admin_addTrustedPeer would downgrade @@ -51,15 +46,7 @@ func ConnectP2P(ctx context.Context, require *testreq.Assertions, initiator RpcC } addTrustedPeer(initiator, targetInfo.Enode, "initiator") - waitCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second) - defer cancel() - err = forPeerList(waitCtx, initiator, func(peers []peer) bool { - return slices.ContainsFunc(peers, func(p peer) bool { - peerID := strings.TrimPrefix(strings.ToLower(p.ID), "0x") - return peerID == strings.ToLower(expectedID) - }) - }) - require.NoError(err, "The peer was not connected") + require.NoError(waitForPeerConnected(ctx, initiator, expectedID), "The peer was not connected") // The acceptor's trusted entry is added only once the session is up: reth // dials trusted peers immediately, and two crossed in-flight dials are both @@ -69,6 +56,163 @@ func ConnectP2P(ctx context.Context, require *testreq.Assertions, initiator RpcC addTrustedPeer(acceptor, initiatorInfo.Enode, "acceptor") } +// ConnectP2PBasic creates a p2p peer connection between initiator and acceptor without +// registering it as trusted on either side. The acceptor keeps nothing, so on its side the +// session stays subject to reputation penalties, bans and eviction like a discovered peer's. The +// initiator keeps the acceptor as a static entry and re-dials it after the session drops; a reth +// initiator neither slashes a static peer's reputation for drops, bad announcements, timeouts or +// already seen transactions, nor evicts or forgets it. ConnectP2P is for a peering that must +// survive whatever the test does; this is for a test that observes the acceptor's peer +// management, which a trusted peering masks. +func ConnectP2PBasic(ctx context.Context, require *testreq.Assertions, initiator RpcCaller, acceptor RpcCaller) { + targetInfo, expectedID := nodeInfo(ctx, require, acceptor) + + var peerAdded bool + require.NoError(initiator.CallContext(ctx, &peerAdded, "admin_addPeer", targetInfo.Enode), "add peer") + require.True(peerAdded, "should have added peer successfully") + + require.NoError(waitForPeerConnected(ctx, initiator, expectedID), "The peer was not connected") + + require.False(PeerInfo(ctx, require, initiator, acceptor).Trusted, "the initiator must not hold the acceptor as a trusted peer") + require.False(PeerInfo(ctx, require, acceptor, initiator).Trusted, "the acceptor must not hold the initiator as a trusted peer") +} + +// WaitP2PConnected waits until node lists remote among its connected peers, and fails when it +// does not within the peer wait budget. +func WaitP2PConnected(ctx context.Context, require *testreq.Assertions, node RpcCaller, remote RpcCaller) { + require.NoError(waitForPeerConnected(ctx, node, nodeID(ctx, require, remote)), "The peer is not connected") +} + +// ELPeerInfo is what an execution-layer node reports about one of its peers through admin_peers. +type ELPeerInfo struct { + // Caps are the capabilities the peer advertised in its hello, such as "eth/69". + Caps []string + // EthVersion is the eth protocol version the session negotiated, 0 while the eth handshake + // is still pending. + EthVersion uint64 + ID string + Inbound bool + // LocalAddress and RemoteAddress are the TCP endpoints of the session. Every dial comes from + // a fresh ephemeral port, so a session that was dropped and re-dialed shows a new + // RemoteAddress on the accepting side and a new LocalAddress on the dialing side. + LocalAddress string + RemoteAddress string + Static bool + Trusted bool +} + +// PeerInfo waits until node lists remote with a negotiated eth session and returns what node +// reports about it. It fails when the session is not up within the peer wait budget. +func PeerInfo(ctx context.Context, require *testreq.Assertions, node RpcCaller, remote RpcCaller) ELPeerInfo { + expectedID := nodeID(ctx, require, remote) + found, err := waitForPeer(ctx, node, expectedID, peer.negotiated) + require.NoError(err, "The peer %s has no negotiated eth session", expectedID) + return found.info() +} + +// ListPeers returns what node reports about every peer it is connected to, including those +// whose eth handshake is still pending. +func ListPeers(ctx context.Context, node RpcCaller) ([]ELPeerInfo, error) { + callCtx, cancel := context.WithTimeout(ctx, peerCallTimeout) + defer cancel() + var peers []peer + if err := node.CallContext(callCtx, &peers, "admin_peers"); err != nil { + return nil, fmt.Errorf("listing peers: %w", err) + } + infos := make([]ELPeerInfo, 0, len(peers)) + for _, p := range peers { + infos = append(infos, p.info()) + } + return infos, nil +} + +// HighestEthVersion returns the highest eth protocol version among caps, and an error when caps +// carries none. +func HighestEthVersion(caps []string) (uint64, error) { + versions, err := ethVersions(caps) + if err != nil { + return 0, err + } + if len(versions) == 0 { + return 0, fmt.Errorf("no eth capability in %v", caps) + } + return slices.Max(versions), nil +} + +// HighestCommonEthVersion returns the highest eth protocol version both caps and otherCaps carry, +// which is the version two peers advertising them negotiate, and an error when they share none. +func HighestCommonEthVersion(caps, otherCaps []string) (uint64, error) { + versions, err := ethVersions(caps) + if err != nil { + return 0, err + } + otherVersions, err := ethVersions(otherCaps) + if err != nil { + return 0, err + } + common := slices.DeleteFunc(versions, func(version uint64) bool { return !slices.Contains(otherVersions, version) }) + if len(common) == 0 { + return 0, fmt.Errorf("no common eth capability between %v and %v", caps, otherCaps) + } + return slices.Max(common), nil +} + +// ethVersions parses the eth protocol versions out of caps, ignoring other capabilities. +func ethVersions(caps []string) ([]uint64, error) { + var versions []uint64 + for _, capability := range caps { + versionText, ok := strings.CutPrefix(capability, "eth/") + if !ok { + continue + } + version, err := strconv.ParseUint(versionText, 10, 64) + if err != nil { + return nil, fmt.Errorf("parsing capability %q: %w", capability, err) + } + versions = append(versions, version) + } + return versions, nil +} + +// nodeInfo returns the node's admin_nodeInfo and its identity as admin_peers lists it on its +// peers. +func nodeInfo(ctx context.Context, require *testreq.Assertions, node RpcCaller) (p2p.NodeInfo, string) { + var info p2p.NodeInfo + require.NoError(node.CallContext(ctx, &info, "admin_nodeInfo"), "get node info") + parsed, err := enode.ParseV4(info.Enode) + require.NoError(err, "failed to parse node enode") + return info, parsed.ID().String() +} + +func nodeID(ctx context.Context, require *testreq.Assertions, node RpcCaller) string { + _, id := nodeInfo(ctx, require, node) + return id +} + +// waitForPeerConnected polls node's admin_peers until it lists expectedID. +func waitForPeerConnected(ctx context.Context, node RpcCaller, expectedID string) error { + _, err := waitForPeer(ctx, node, expectedID, func(peer) bool { return true }) + return err +} + +// waitForPeer polls node's admin_peers until it lists expectedID with an entry accept takes, and +// returns that entry. +func waitForPeer(ctx context.Context, node RpcCaller, expectedID string, accept func(peer) bool) (peer, error) { + waitCtx, cancel := context.WithTimeout(ctx, peerWaitTimeout) + defer cancel() + var found peer + err := forPeerList(waitCtx, node, func(peers []peer) bool { + for _, p := range peers { + if p.idMatches(expectedID) && accept(p) { + found = p + return true + } + } + return false + }) + return found, err +} + // forPeerList polls admin_peers on node until cond holds for the returned peer // list. Each poll is individually time-bounded so that a single stalled call — // e.g. the RPC client re-dialing a connection the kernel never answers — cannot @@ -76,7 +220,7 @@ func ConnectP2P(ctx context.Context, require *testreq.Assertions, initiator RpcC // port and typically succeeds). Only attempt timeouts are retried; any other // error still fails fast. func forPeerList(ctx context.Context, node RpcCaller, cond func([]peer) bool) error { - return pollPeerList(ctx, node, time.Second, 5*time.Second, cond) + return pollPeerList(ctx, node, time.Second, peerCallTimeout, cond) } // pollPeerList is forPeerList with the poll interval and per-attempt timeout @@ -121,14 +265,8 @@ func isTimeout(err error) bool { // DisconnectP2P disconnects a p2p peer connection between node1 and node2. func DisconnectP2P(ctx context.Context, require *testreq.Assertions, initiator RpcCaller, acceptor RpcCaller) { - var targetInfo p2p.NodeInfo - require.NoError(acceptor.CallContext(ctx, &targetInfo, "admin_nodeInfo"), "get node info") - targetNode, err := enode.ParseV4(targetInfo.Enode) - require.NoError(err, "failed to parse target node") - expectedID := targetNode.ID().String() - - var initiatorInfo p2p.NodeInfo - require.NoError(initiator.CallContext(ctx, &initiatorInfo, "admin_nodeInfo"), "get initiator node info") + targetInfo, expectedID := nodeInfo(ctx, require, acceptor) + initiatorInfo, _ := nodeInfo(ctx, require, initiator) // Drop the trusted status set up by ConnectP2P first: reth's admin_removePeer // is a no-op on a trusted peer. Trusted removal alone is not enough either, @@ -151,13 +289,10 @@ func DisconnectP2P(ctx context.Context, require *testreq.Assertions, initiator R removePeer(initiator, targetInfo.ENR, "initiator") removePeer(acceptor, initiatorInfo.ENR, "acceptor") - ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + waitCtx, cancel := context.WithTimeout(ctx, peerWaitTimeout) defer cancel() - err = forPeerList(ctx, initiator, func(peers []peer) bool { - return !slices.ContainsFunc(peers, func(p peer) bool { - peerID := strings.TrimPrefix(strings.ToLower(p.ID), "0x") - return peerID == strings.ToLower(expectedID) - }) + err := forPeerList(waitCtx, initiator, func(peers []peer) bool { + return !slices.ContainsFunc(peers, func(p peer) bool { return p.idMatches(expectedID) }) }) require.NoError(err, "The peer was not removed") } @@ -179,13 +314,53 @@ func ForgetP2PPeer(ctx context.Context, require *testreq.Assertions, node RpcCal waitCtx, cancel := context.WithTimeout(ctx, 30*time.Second) defer cancel() err = forPeerList(waitCtx, node, func(peers []peer) bool { - return !slices.ContainsFunc(peers, func(p peer) bool { - return strings.TrimPrefix(strings.ToLower(p.ID), "0x") == forgottenID - }) + return !slices.ContainsFunc(peers, func(p peer) bool { return p.idMatches(forgottenID) }) }) require.NoError(err, "the forgotten peer is still connected") } -type peer struct { - ID string `json:"id"` +const ( + // peerCallTimeout bounds one admin call. + peerCallTimeout = 5 * time.Second + // peerWaitTimeout bounds a wait for a peer list to reach a state. + peerWaitTimeout = 30 * time.Second +) + +// peer is an admin_peers entry as both op-geth and op-reth report it. +type peer p2p.PeerInfo + +func (p peer) idMatches(expectedID string) bool { + return strings.TrimPrefix(strings.ToLower(p.ID), "0x") == strings.ToLower(expectedID) +} + +// ethVersion returns the negotiated eth version, 0 while the handshake is pending. The eth +// protocol entry is an object with the negotiated version once the handshake is done; op-geth +// reports a string such as "handshake" until then. +func (p peer) ethVersion() uint64 { + eth, ok := p.Protocols["eth"].(map[string]any) + if !ok { + return 0 + } + version, ok := eth["version"].(float64) + if !ok || version < 1 { + return 0 + } + return uint64(version) +} + +func (p peer) negotiated() bool { + return p.ethVersion() != 0 +} + +func (p peer) info() ELPeerInfo { + return ELPeerInfo{ + Caps: p.Caps, + EthVersion: p.ethVersion(), + ID: p.ID, + Inbound: p.Network.Inbound, + LocalAddress: p.Network.LocalAddress, + RemoteAddress: p.Network.RemoteAddress, + Static: p.Network.Static, + Trusted: p.Network.Trusted, + } } diff --git a/op-devstack/sysgo/l2_el_p2p_util_test.go b/op-devstack/sysgo/l2_el_p2p_util_test.go index caa1fc05751..726221d60d5 100644 --- a/op-devstack/sysgo/l2_el_p2p_util_test.go +++ b/op-devstack/sysgo/l2_el_p2p_util_test.go @@ -134,3 +134,103 @@ func TestPollPeerListOuterBudgetExpiry(t *testing.T) { require.ErrorContains(t, err, "last admin_peers attempt error") require.GreaterOrEqual(t, node.calls, 2, "attempt timeouts must be retried until the outer budget expires") } + +func TestHighestEthVersionPicksTheHighestEthCapability(t *testing.T) { + for _, tc := range []struct { + name string + caps []string + + fails bool + highest uint64 + }{ + {name: "single", caps: []string{"eth/69"}, highest: 69}, + {name: "several_unordered", caps: []string{"eth/68", "snap/1", "eth/71", "eth/69"}, highest: 71}, + {name: "no_eth", caps: []string{"snap/1"}, fails: true}, + {name: "malformed", caps: []string{"eth/x"}, fails: true}, + } { + t.Run(tc.name, func(t *testing.T) { + highest, err := HighestEthVersion(tc.caps) + if tc.fails { + require.Error(t, err) + return + } + require.NoError(t, err) + require.Equal(t, tc.highest, highest) + }) + } +} + +func TestHighestCommonEthVersionIntersectsTheCapabilities(t *testing.T) { + for _, tc := range []struct { + name string + caps []string + other []string + + common uint64 + fails bool + }{ + {name: "op_reth_develop_with_op_geth", caps: []string{"eth/66", "eth/67", "eth/68", "eth/69"}, other: []string{"eth/69"}, common: 69}, + {name: "op_reth_bump_with_op_geth", caps: []string{"eth/66", "eth/67", "eth/68", "eth/69", "eth/70", "eth/71"}, other: []string{"eth/69"}, common: 69}, + {name: "gap_in_one_side", caps: []string{"eth/68", "eth/71"}, other: []string{"eth/69", "eth/70", "eth/71"}, common: 71}, + {name: "disjoint", caps: []string{"eth/68"}, other: []string{"eth/69"}, fails: true}, + {name: "malformed", caps: []string{"eth/x"}, other: []string{"eth/69"}, fails: true}, + } { + t.Run(tc.name, func(t *testing.T) { + common, err := HighestCommonEthVersion(tc.caps, tc.other) + if tc.fails { + require.Error(t, err) + return + } + require.NoError(t, err) + require.Equal(t, tc.common, common) + }) + } +} + +func TestPeerInfoIsReportedOnceTheEthHandshakeIsDone(t *testing.T) { + for _, tc := range []struct { + name string + entry string + + info ELPeerInfo + negotiated bool + }{ + { + name: "op_geth_before_handshake", + entry: `{"id":"ab","caps":["eth/69"],"network":{"inbound":true},"protocols":{"eth":"handshake"}}`, + info: ELPeerInfo{Caps: []string{"eth/69"}, ID: "ab", Inbound: true}, + negotiated: false, + }, + { + name: "op_geth_with_empty_eth_info", + entry: `{"id":"ab","caps":["eth/69"],"network":{},"protocols":{"eth":{}}}`, + info: ELPeerInfo{Caps: []string{"eth/69"}, ID: "ab"}, + negotiated: false, + }, + { + name: "op_geth_negotiated", + entry: `{"id":"ab","caps":["eth/68","eth/69"],"network":{"localAddress":"127.0.0.1:30303","remoteAddress":"127.0.0.1:50001","inbound":true,"trusted":false,"static":false},"protocols":{"eth":{"version":69,"head":"0x1"}}}`, + info: ELPeerInfo{Caps: []string{"eth/68", "eth/69"}, EthVersion: 69, ID: "ab", Inbound: true, LocalAddress: "127.0.0.1:30303", RemoteAddress: "127.0.0.1:50001"}, + negotiated: true, + }, + { + name: "op_reth_negotiated", + entry: `{"id":"0xcd","caps":["eth/69","eth/71"],"network":{"localAddress":"127.0.0.1:50002","remoteAddress":"127.0.0.1:30304","inbound":false,"trusted":true,"static":true},"protocols":{"eth":{"version":71}}}`, + info: ELPeerInfo{Caps: []string{"eth/69", "eth/71"}, EthVersion: 71, ID: "0xcd", LocalAddress: "127.0.0.1:50002", RemoteAddress: "127.0.0.1:30304", Static: true, Trusted: true}, + negotiated: true, + }, + } { + t.Run(tc.name, func(t *testing.T) { + var entry peer + require.NoError(t, json.Unmarshal([]byte(tc.entry), &entry)) + require.Equal(t, tc.negotiated, entry.negotiated()) + require.Equal(t, tc.info, entry.info()) + }) + } +} + +func TestPeerIDMatchesIgnoresCaseAndPrefix(t *testing.T) { + require.True(t, peer{ID: "0xABcd"}.idMatches("abCD")) + require.True(t, peer{ID: "abcd"}.idMatches("ABCD")) + require.False(t, peer{ID: "abcd"}.idMatches("abce")) +} diff --git a/op-devstack/sysgo/mixed_runtime.go b/op-devstack/sysgo/mixed_runtime.go index ab69c81101f..03c3be95645 100644 --- a/op-devstack/sysgo/mixed_runtime.go +++ b/op-devstack/sysgo/mixed_runtime.go @@ -17,6 +17,7 @@ import ( coredepset "github.com/ethereum-optimism/optimism/op-core/interop/depset" "github.com/ethereum-optimism/optimism/op-devstack/devtest" "github.com/ethereum-optimism/optimism/op-devstack/shared/rustbin" + "github.com/ethereum-optimism/optimism/op-e2e/e2eutils/geth" "github.com/ethereum-optimism/optimism/op-service/endpoint" "github.com/ethereum-optimism/optimism/op-service/eth" "github.com/ethereum-optimism/optimism/op-service/log" @@ -208,6 +209,7 @@ type MixedSingleChainPresetConfig struct { LocalContractArtifactsPath string DeployerOptions []DeployerOption BatcherOptions []BatcherOption + OpGethOptions []geth.GethOption OpRethOptions []OpRethOption // InteropAtGenesis activates the Lagoon hardfork at genesis on the L2 chain and provides // the resulting dependency set to op-node CL startup. Required by any test that exercises @@ -278,7 +280,7 @@ func NewMixedSingleChainRuntime(t devtest.T, cfg MixedSingleChainPresetConfig) * var el L2ELNode switch spec.ELKind { case MixedL2ELOpGeth: - el = startL2ELNode(t, l2Net, jwtPath, jwtSecret, spec.ELKey, identity) + el = startL2ELNode(t, l2Net, jwtPath, jwtSecret, spec.ELKey, identity, cfg.OpGethOptions...) case MixedL2ELOpReth: el = startMixedOpRethNode(t, l2Net, spec.ELKey, jwtPath, jwtSecret, metricsRegistrar, nodeOpRethOpts...) default: diff --git a/op-devstack/sysgo/singlechain_build.go b/op-devstack/sysgo/singlechain_build.go index e2fff1cb089..f822994802b 100644 --- a/op-devstack/sysgo/singlechain_build.go +++ b/op-devstack/sysgo/singlechain_build.go @@ -18,6 +18,7 @@ import ( "github.com/ethereum-optimism/optimism/op-chain-ops/devkeys" "github.com/ethereum-optimism/optimism/op-core/interop/depset" "github.com/ethereum-optimism/optimism/op-devstack/devtest" + "github.com/ethereum-optimism/optimism/op-e2e/e2eutils/geth" "github.com/ethereum-optimism/optimism/op-e2e/e2eutils/intentbuilder" "github.com/ethereum-optimism/optimism/op-node/config" opNodeFlags "github.com/ethereum-optimism/optimism/op-node/flags" @@ -187,6 +188,7 @@ func startL2ELNode( jwtSecret [32]byte, key string, identity *ELNodeIdentity, + gethOptions ...geth.GethOption, ) *OpGeth { cfg := DefaultL2ELConfig() cfg.P2PAddr = "127.0.0.1" @@ -194,13 +196,14 @@ func startL2ELNode( cfg.P2PNodeKeyHex = identity.KeyHex() l2EL := &OpGeth{ - name: key, - p: t, - logger: t.Logger().New("component", "l2el-"+key), - l2Net: l2Net, - jwtPath: jwtPath, - jwtSecret: jwtSecret, - cfg: cfg, + name: key, + p: t, + logger: t.Logger().New("component", "l2el-"+key), + l2Net: l2Net, + jwtPath: jwtPath, + jwtSecret: jwtSecret, + cfg: cfg, + gethOptions: gethOptions, } l2EL.Start() t.Cleanup(l2EL.Stop) diff --git a/op-service/txplan/txplan.go b/op-service/txplan/txplan.go index aad0cdd317b..b71a55fc5f9 100644 --- a/op-service/txplan/txplan.go +++ b/op-service/txplan/txplan.go @@ -149,6 +149,17 @@ func WithStaticNonce(nonce uint64) Option { } } +// WithStaticChainID pins the chain ID instead of reading it from a node, so a transaction whose +// other fields are pinned as well can be signed without any node call. +func WithStaticChainID(chainID eth.ChainID) Option { + return func(tx *PlannedTx) { + tx.ChainID.Set(chainID) + tx.ChainID.Fn(func(ctx context.Context) (eth.ChainID, error) { + return chainID, nil + }) + } +} + func WithAccessList(al types.AccessList) Option { return func(tx *PlannedTx) { tx.AccessList.Set(al) @@ -196,6 +207,9 @@ func WithGasLimit(limit uint64) Option { func WithGasFeeCap(feeCap *big.Int) Option { return func(tx *PlannedTx) { + // The fee cap is explicitly set, so drop the default computation from the tip cap and the + // base fee, which a later change of the tip cap would otherwise trigger again. + tx.GasFeeCap.ResetFnAndDependencies() tx.GasFeeCap.Set(feeCap) } } diff --git a/op-service/txplan/txplan_test.go b/op-service/txplan/txplan_test.go index 05f34020b3b..6c571810f0e 100644 --- a/op-service/txplan/txplan_test.go +++ b/op-service/txplan/txplan_test.go @@ -98,3 +98,32 @@ func TestWithReaderDoesNotUseDefaultFeeCaps(t *testing.T) { require.Equal(t, accessList, reader.msg.AccessList) require.Equal(t, rpc.BlockNumber(7), reader.blockNumber) } + +func TestPinnedFieldsSignWithoutANode(t *testing.T) { + key, err := crypto.GenerateKey() + require.NoError(t, err) + to := common.Address{0x01} + chainID := eth.ChainIDFromUInt64(901) + + // The tip cap is pinned after the fee cap: the pinned fee cap must survive that, instead of + // being recomputed from the tip cap and a block nobody fetched. + ptx := NewPlannedTx( + WithPrivateKey(key), + WithStaticChainID(chainID), + WithStaticNonce(7), + WithTo(&to), + WithGasLimit(21_000), + WithGasFeeCap(big.NewInt(100)), + WithGasTipCap(big.NewInt(3)), + ) + tx, err := ptx.Signed.Eval(context.Background()) + require.NoError(t, err) + + require.Equal(t, chainID.ToBig(), tx.ChainId()) + require.Equal(t, uint64(7), tx.Nonce()) + require.Equal(t, big.NewInt(100), tx.GasFeeCap()) + require.Equal(t, big.NewInt(3), tx.GasTipCap()) + sender, err := types.LatestSignerForChainID(chainID.ToBig()).Sender(tx) + require.NoError(t, err) + require.Equal(t, crypto.PubkeyToAddress(key.PublicKey), sender) +}