Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1,194 @@
package op_reth_opgeth_peering

import (
"maps"
"testing"

"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/types"

"github.com/ethereum-optimism/optimism/op-devstack/devtest"
"github.com/ethereum-optimism/optimism/op-devstack/dsl"
"github.com/ethereum-optimism/optimism/op-devstack/sysgo"
"github.com/ethereum-optimism/optimism/op-service/eth"
"github.com/ethereum-optimism/optimism/op-service/log"
"github.com/ethereum-optimism/optimism/op-service/txplan"
)

const (
// announcedHashCap is how many announced transaction hashes op-reth tracks per peer at most
// (reth#26924): a hash a peer announces while that many of its announcements are still
// unfetched is dropped. op-reth fetches while the burst is still arriving, so it stays under
// the cap only if it fetches as fast as op-geth announces. Whether it ever reached the cap is
// not observable through its RPC.
announcedHashCap = 4096
// burstTransactions is how many transactions op-geth takes in to announce to op-reth. op-reth
// is offered all of them only if op-geth's sends to it keep up with opGethMaxQueuedTxAnns.
burstTransactions = 2 * announcedHashCap

// opGethMaxQueuedTxAnns is how many hashes op-geth queues for announcement to one peer at
// most (its maxQueuedTxAnns): when more are queued it drops the oldest, so it always announces
// the last opGethMaxQueuedTxAnns it queued, and earlier ones only if it sent them before.
opGethMaxQueuedTxAnns = 4096
// opGethTxMaxBroadcastSize is the largest transaction op-geth sends to a peer whole (its
// txMaxBroadcastSize); it announces a larger one by hash only, so the peer has to fetch it.
opGethTxMaxBroadcastSize = 4096

// announceOnlyDataBytes is the calldata that alone puts a transaction over
// opGethTxMaxBroadcastSize.
announceOnlyDataBytes = opGethTxMaxBroadcastSize
// announceOnlyTxGas covers the EIP-7623 calldata floor of announceOnlyDataBytes zero bytes.
announceOnlyTxGas = 80_000
// burstSenderFundsGwei pays one announce-only transaction at the pinned fee cap plus its L1
// data fee.
burstSenderFundsGwei = 20_000_000

// burstPoolMaxCount is op-reth's pending and basefee sub-pool size, in transactions, and how
// many transactions op-geth's pool holds at most. Their defaults, 10000 for reth and 6144 for
// op-geth, are too close to or below the burst.
burstPoolMaxCount = 2 * burstTransactions
// burstPoolMaxSizeMB is op-reth's pending and basefee sub-pool size, in MB. The burst takes
// about 34 MB of pool space, more than reth's 20 MB default.
burstPoolMaxSizeMB = 100

// burstDrainAttempts bounds, in 1s polls, how long the resumed sequencer may take to include a
// cap's worth of the burst, which fills about five blocks at the devnet's gas limit.
burstDrainAttempts = 90
// burstLeadAttempts bounds, in 1s polls, how long op-geth may take to hold the burst's first
// transaction as pending.
burstLeadAttempts = 30
// burstPoolAttempts bounds, in 1s polls, how long op-reth may take to fetch a cap's worth of
// the burst into its pool.
burstPoolAttempts = 60
// includedDrainAttempts bounds, in 1s polls, how long op-reth may take to clear transactions
// from its pool once it has imported the blocks that include them.
includedDrainAttempts = 60
// syncAttempts bounds, in 2s polls, how long a follower may take to import blocks the
// sequencer has already built.
syncAttempts = 30
)

// announceOnlyTransaction signs the transaction a fresh sender contributes to the burst: too
// large for op-geth to send it whole, so it is announced by hash.
func announceOnlyTransaction(t devtest.T, sender *dsl.EOA) *types.Transaction {
to := sender.Address()
tx := sender.Sign(
sender.PlanOffline(),
txplan.WithStaticNonce(0),
txplan.WithTo(&to),
txplan.WithData(make([]byte, announceOnlyDataBytes)),
txplan.WithGasLimit(announceOnlyTxGas),
txplan.WithGasFeeCap(feeCap),
txplan.WithGasTipCap(tip),
)
t.Require().Greater(tx.Size(), uint64(opGethTxMaxBroadcastSize), "the transaction must be larger than what op-geth sends whole")
return tx
}

// verifySessionKept requires after to describe the same session as before, seen from the
// accepting side: a dropped session that the dialing side re-dials shows up there as a new
// remote port.
func verifySessionKept(t devtest.T, before, after sysgo.ELPeerInfo) {
t.Require().True(before.Inbound, "a re-dial shows a new remote port on the accepting side only")
t.Require().Equal(before.RemoteAddress, after.RemoteAddress, "op-reth must keep its session with op-geth; a new remote port means the session was dropped and re-dialed")
t.Require().Equal(before.EthVersion, after.EthVersion, "the session must keep its negotiated eth version")
}

// transactionsWithHashes returns those of txs whose hash is in hashes.
func transactionsWithHashes(txs []*types.Transaction, hashes map[common.Hash]struct{}) []*types.Transaction {
var selected []*types.Transaction
for _, tx := range txs {
if _, ok := hashes[tx.Hash()]; ok {
selected = append(selected, tx)
}
}
return selected
}

// verifyBurstReconciled requires every burst transaction op-reth pooled to be either included in
// a block or still in its pool.
func verifyBurstReconciled(t devtest.T, pooled, included, stillPooled map[common.Hash]struct{}) {
var inNeither []common.Hash
for hash := range pooled {
_, isIncluded := included[hash]
_, isStillPooled := stillPooled[hash]
if !isIncluded && !isStillPooled {
inNeither = append(inNeither, hash)
}
}
t.Require().Empty(inNeither, "every burst transaction op-reth pooled must be in a block or still in its pool; %d of %d are in neither", len(inNeither), len(pooled))
}

// TestOpRethKeepsItsOpGethPeerThroughAnAnnouncementBurst hands op-geth twice as many transactions
// to announce to op-reth within a few seconds as op-reth tracks announced hashes per peer, sent so
// that op-reth is offered more hashes than that whatever op-geth drops from its own announcement
// queue, while the sequencer is paused. op-reth has to fetch at least a full cap's worth of them
// into its pool, keep its session with op-geth and log no error meanwhile. Once the sequencer
// resumes, op-reth has to keep following the chain until it includes a cap's worth of the burst.
// With the sequencer paused again, op-reth has to clear the burst transactions in blocks up to
// the paused head from its pool, and every burst transaction it was seen pooling has to be either
// in such a block or still in its pool; its pool may keep what a cap-sized fetch downstream leaves
// behind. How many of the burst op-reth pooled, how many were included, how many op-geth offered
// op-reth at least, and how many each follower still holds afterwards, is logged.
func TestOpRethKeepsItsOpGethPeerThroughAnAnnouncementBurst(gt *testing.T) {
t := devtest.ParallelT(gt)
skipUnsupported(t)
sys := newPairSystem(t, sysgo.MixedL2ELOpGeth)
opGeth, opReth := sys.Sender.EL, sys.Receiver.EL
senders, fundedAt := fundAccounts(t, sys, burstTransactions, eth.GWei(burstSenderFundsGwei))
burst := make([]*types.Transaction, len(senders))
for i, sender := range senders {
burst[i] = announceOnlyTransaction(t, sender)
}
// Both followers admit a transaction against their own state, which must hold the senders'
// funds.
dsl.CheckAll(t,
opGeth.ReachedFn(eth.Unsafe, fundedAt, syncAttempts),
opReth.ReachedFn(eth.Unsafe, fundedAt, syncAttempts),
)
sys.Sequencer.CL.StopSequencer()

sessionBefore := opReth.PeerInfoOf(opGeth)
errorLogs := opReth.RecordLogs(log.LevelError)
// op-geth announces at least the last opGethMaxQueuedTxAnns hashes it queues for op-reth. The
// lead transaction, pending in op-geth before the rest is sent, is queued alone and taken for
// announcement before the rest is queued, as op-geth has nothing else to announce to op-reth,
// its only peer: op-reth is offered more than announcedHashCap hashes of the burst whatever
// op-geth drops.
t.Require().GreaterOrEqual(opGethMaxQueuedTxAnns, announcedHashCap, "op-geth must announce at least a cap's worth of what it queues")
lead, rest := burst[:1], burst[1:]
opGeth.SendTransactions(lead)
opGeth.WaitForPendingTransactions(lead, burstLeadAttempts)
opGeth.SendTransactions(rest)
pooled := opReth.WaitForPooledTransactions(burst, announcedHashCap, burstPoolAttempts)

verifySessionKept(t, sessionBefore, opReth.PeerInfoOf(opGeth))
errorLogs.VerifyEmpty("fetching the burst")
t.Logger().Info("op-reth fetched the burst", "burst", len(burst), "pooled", len(pooled))

sys.Sequencer.CL.StartSequencer()
// op-reth passes the burst on to the sequencer, which fetches what op-reth only announced
// to it under the same per-peer cap: at least a cap's worth of the burst reaches a block, and
// up to as many transactions as the burst exceeds the cap by can stay behind in op-reth's pool.
// The burst is sent once both followers are past fundedAt, so no block up to it holds any.
included, nextBlock := opReth.WaitForIncludedTransactions(burst, fundedAt+1, announcedHashCap, burstDrainAttempts)
opReth.Advanced(eth.Unsafe, 2)
// Pausing the sequencer again holds op-reth's chain still, so its blocks and its pool are read
// against the same head.
pausedHead := sys.Sequencer.EL.BlockRefByHash(sys.Sequencer.CL.StopSequencer())
opReth.Reached(eth.Unsafe, pausedHead.Number, syncAttempts)
maps.Copy(included, opReth.IncludedTransactions(burst, nextBlock, pausedHead.Number))
opReth.WaitForTransactionsOutOfPool(transactionsWithHashes(burst, included), includedDrainAttempts)
stillPooled := opReth.PooledTransactions(burst)
t.Require().Equal(pausedHead.Hash, opReth.BlockRefByLabel(eth.Unsafe).Hash, "op-reth's head must stay at the paused sequencer's head while the burst is reconciled")
verifyBurstReconciled(t, pooled, included, stillPooled)

verifySessionKept(t, sessionBefore, opReth.PeerInfoOf(opGeth))
opGeth.VerifyPeeredWith(opReth)
// op-reth learns of a burst transaction only from op-geth, so those it pooled or that reached a
// block are a lower bound on how many op-geth offered it.
offeredAtLeast := maps.Clone(pooled)
maps.Copy(offeredAtLeast, included)
maps.Copy(offeredAtLeast, stillPooled)
t.Logger().Info("Burst included", "burst", len(burst), "offered_at_least", len(offeredAtLeast), "pooled", len(pooled), "included", len(included), "left_in_op_geth", opGeth.TxPoolStatus(), "left_in_op_reth", len(stillPooled))
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
package op_reth_opgeth_peering

import (
"testing"

"github.com/ethereum-optimism/optimism/op-devstack/devtest"
"github.com/ethereum-optimism/optimism/op-devstack/sysgo"
)

// highestEthVersion returns the highest eth protocol version among the capabilities a peer
// advertised.
func highestEthVersion(t devtest.T, caps []string) uint64 {
version, err := sysgo.HighestEthVersion(caps)
t.Require().NoError(err, "a peer must advertise an eth version")
return version
}

// TestOpRethAndOpGethNegotiateTheHighestCommonEthVersion peers an op-reth and an op-geth follower
// as basic peers, with either client dialing, and requires both to report the same negotiated eth
// protocol version through admin_peers: the highest version both advertise. The advertised
// versions are logged, so a run records which version the pair settled on.
func TestOpRethAndOpGethNegotiateTheHighestCommonEthVersion(gt *testing.T) {
t := devtest.ParallelT(gt)
devtest.RunParallel(t, followerCells("%s_dials_%s"), func(t devtest.T, dialer followerCell) {
skipUnsupported(t)
sys := newPairSystem(t, dialer.Kind)
opReth := sys.follower(sysgo.MixedL2ELOpReth).EL
opGeth := sys.follower(sysgo.MixedL2ELOpGeth).EL

rethReports := opReth.PeerInfoOf(opGeth)
gethReports := opGeth.PeerInfoOf(opReth)
// Each side reports the capabilities the other advertised.
opRethCaps, opGethCaps := gethReports.Caps, rethReports.Caps
t.Logger().Info("Negotiated eth version",
"op_reth_advertises", highestEthVersion(t, opRethCaps), "op_geth_advertises", highestEthVersion(t, opGethCaps),
"op_reth_reports", rethReports.EthVersion, "op_geth_reports", gethReports.EthVersion,
"op_reth_caps", opRethCaps, "op_geth_caps", opGethCaps)

t.Require().Equal(rethReports.EthVersion, gethReports.EthVersion, "both clients must report the same negotiated eth version")
highestCommon, err := sysgo.HighestCommonEthVersion(opRethCaps, opGethCaps)
t.Require().NoError(err, "the clients must share an eth version")
t.Require().Equal(highestCommon, rethReports.EthVersion, "the pair must negotiate the highest eth version both advertise")
})
}
130 changes: 130 additions & 0 deletions op-acceptance-tests/tests/op_reth_opgeth_peering/funding_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,130 @@
package op_reth_opgeth_peering

import (
"math/big"

"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/params"

"github.com/ethereum-optimism/optimism/op-devstack/devtest"
"github.com/ethereum-optimism/optimism/op-devstack/dsl"
"github.com/ethereum-optimism/optimism/op-service/eth"
"github.com/ethereum-optimism/optimism/op-service/txplan"
)

const (
// fanOut is how many transactions one account can hold in a reth pool at once (reth's
// --txpool.max-account-slots default), so how many accounts one account funds per round.
fanOut = 16

// feeCapGwei is pinned on every transaction the tests sign offline. It leaves room for the
// base fee to climb through the full blocks a burst produces.
feeCapGwei = 100
// fundingL1DataFeeAllowanceGwei bounds the L1 data fee of one funding transfer, which a pool
// adds to the transfer's cost when it checks the payer's balance. A devnet charges orders of
// magnitude less.
fundingL1DataFeeAllowanceGwei = 100_000
// tipGwei is pinned on every transaction the tests sign offline.
tipGwei = 1

// fundingRoundAttemptsPerBlock bounds, in 1s polls, how long the sequencer may take to build
// each block a funding round's gas fills.
fundingRoundAttemptsPerBlock = 10 * l2BlockTimeSeconds
// fundingRoundBaseAttempts bounds, in 1s polls, how long the sequencer may take to start
// including a funding round once it is sent.
fundingRoundBaseAttempts = 30
)

var (
feeCap = big.NewInt(feeCapGwei * params.GWei)
// fundingTxCost is what one funding transfer may cost its payer: its L2 execution at the
// pinned fee cap, and its L1 data fee.
fundingTxCost = eth.GWei(feeCapGwei).Mul(params.TxGas).Add(eth.GWei(fundingL1DataFeeAllowanceGwei))
tip = big.NewInt(tipGwei * params.GWei)
)

// fundAccounts returns count fresh accounts holding at least amount each, bound to the
// sequencer's EL, and the number of a block by which the sequencer has funded all of them. The
// funder pays up to fanOut distributors in one round, each distributor pays up to fanOut more in
// the next, and so on until a round pays the count accounts, so no account ever has more than
// fanOut transactions in a pool at once. Every round is one batch of offline signed transfers to
// the sequencer, waited on until the sequencer includes all of them.
func fundAccounts(t devtest.T, sys *pairSystem, count int, amount eth.ETH) ([]*dsl.EOA, uint64) {
levels := fundingLevels(count)
amounts := fundingAmounts(levels, amount)
t.Logger().Info("Funding accounts through distributors", "count", count, "levels", levels, "amounts", amounts)
accounts := sys.FunderL2.NewFundedEOAs(levels[0], amounts[0])
fundedAt := sys.Sequencer.EL.BlockRefByLabel(eth.Unsafe).Number
for level := 1; level < len(levels); level++ {
accounts, fundedAt = fundRound(t, sys, accounts, levels[level], amounts[level])
}
return accounts, fundedAt
}

// fundingLevels returns how many accounts each funding round creates, the last being count, with
// every earlier round creating enough payers for the next.
func fundingLevels(count int) []int {
levels := []int{count}
for levels[0] > fanOut {
payers := (levels[0] + fanOut - 1) / fanOut
levels = append([]int{payers}, levels...)
}
return levels
}

// fundingAmounts returns what an account of each round is funded with: the last round's accounts
// get amount, and a payer gets what it pays out plus the cost of paying it.
func fundingAmounts(levels []int, amount eth.ETH) []eth.ETH {
amounts := make([]eth.ETH, len(levels))
amounts[len(levels)-1] = amount
for level := len(levels) - 2; level >= 0; level-- {
amounts[level] = amounts[level+1].Add(fundingTxCost).Mul(fanOut)
}
return amounts
}

// fundingRoundAttempts bounds, in 1s polls, how long the sequencer may take to include a funding
// round of transfers: fundingRoundBaseAttempts plus fundingRoundAttemptsPerBlock for every block
// the round's gas fills at blockGasLimit, a partly filled last block included.
func fundingRoundAttempts(transfers int, blockGasLimit uint64) int {
transfersPerBlock := int(blockGasLimit / params.TxGas)
blocks := (transfers + transfersPerBlock - 1) / transfersPerBlock
return fundingRoundBaseAttempts + blocks*fundingRoundAttemptsPerBlock
}

// fundRound has each payer fund up to fanOut fresh accounts with amount, in one batch of offline
// signed transfers, and returns the funded accounts once the sequencer includes every transfer,
// with the number of a block by which it has.
func fundRound(t devtest.T, sys *pairSystem, payers []*dsl.EOA, count int, amount eth.ETH) ([]*dsl.EOA, uint64) {
t.Require().GreaterOrEqual(len(payers)*fanOut, count, "not enough payers for the round")
accounts := make([]*dsl.EOA, 0, count)
transfers := make([]*types.Transaction, 0, count)
for i, payer := range payers {
paidBefore := len(transfers)
for nonce := uint64(0); nonce < fanOut && len(accounts) < count; nonce++ {
account := sys.Wallet.NewEOA(sys.Sequencer.EL)
to := account.Address()
transfer := payer.Sign(
payer.PlanOffline(),
txplan.WithStaticNonce(nonce),
txplan.WithTo(&to),
txplan.WithValue(amount),
txplan.WithGasLimit(params.TxGas),
txplan.WithGasFeeCap(feeCap),
txplan.WithGasTipCap(tip),
)
accounts = append(accounts, account)
transfers = append(transfers, transfer)
}
t.Require().Greater(len(transfers), paidBefore, "payer %d of %d had nothing to pay", i, len(payers))
}
sentAfter := sys.Sequencer.EL.BlockRefByLabel(eth.Unsafe).Number
blockGasLimit := sys.Sequencer.EL.GasLimitAtBlock(sentAfter)
t.Require().GreaterOrEqual(blockGasLimit, params.TxGas, "a block must hold a funding transfer")
attempts := fundingRoundAttempts(len(transfers), blockGasLimit)
sys.Sequencer.EL.SendTransactions(transfers)
_, nextBlock := sys.Sequencer.EL.WaitForIncludedTransactions(transfers, sentAfter+1, len(transfers), attempts)
fundedAt := nextBlock - 1
t.Logger().Info("Funded a round of accounts", "count", len(accounts), "payers", len(payers), "amount", amount, "funded_at", fundedAt)
return accounts, fundedAt
}
Loading
Loading