diff --git a/op-acceptance-tests/tests/proofs_history_match/proofs_history_match_test.go b/op-acceptance-tests/tests/proofs_history_match/proofs_history_match_test.go new file mode 100644 index 00000000000..2d92fe1d76e --- /dev/null +++ b/op-acceptance-tests/tests/proofs_history_match/proofs_history_match_test.go @@ -0,0 +1,177 @@ +package proofs_history_match + +import ( + "fmt" + "testing" + + "github.com/ethereum-optimism/optimism/op-devstack/devtest" + "github.com/ethereum-optimism/optimism/op-devstack/dsl" + "github.com/ethereum-optimism/optimism/op-devstack/presets" + "github.com/ethereum-optimism/optimism/op-devstack/sysgo" + "github.com/ethereum-optimism/optimism/op-service/bigs" + "github.com/ethereum-optimism/optimism/op-service/eth" + "github.com/ethereum-optimism/optimism/op-service/txplan" + "github.com/ethereum/go-ethereum/common" +) + +const ( + // referenceEthProofWindow lets the reference node answer eth_getProof for every block of the + // run from reth's own historical state; reth's default window of zero rejects them all. + referenceEthProofWindow = 1_000_000 + // memoryBlockBufferTarget is reth's --engine.memory-block-buffer-target default. + memoryBlockBufferTarget = 5 + workloadRounds = 4 +) + +// proofsHistoryCase is one cell of the matrix the proofs follower runs under. +type proofsHistoryCase struct { + persistenceThreshold uint64 + stateMaskingBlocks uint64 + verificationInterval uint64 +} + +func (c proofsHistoryCase) name() string { + proofsPath := "index" + if c.verificationInterval != 0 { + proofsPath = "execute" + } + return fmt.Sprintf("threshold_%d-masking_%d-%s", c.persistenceThreshold, c.stateMaskingBlocks, proofsPath) +} + +func (c proofsHistoryCase) opRethArgs() []string { + args := []string{ + fmt.Sprintf("--engine.persistence-threshold=%d", c.persistenceThreshold), + fmt.Sprintf("--proofs-history.verification-interval=%d", c.verificationInterval), + } + if c.stateMaskingBlocks != 0 { + args = append(args, fmt.Sprintf("--engine.num-state-masking-blocks=%d", c.stateMaskingBlocks)) + } + return args +} + +// blocksBeforeLastRound is a chain height by which reth has persisted at least once under the +// case's threshold: persistence starts once more than the threshold of blocks sit in memory and +// keeps the buffer target in memory afterwards. +func (c proofsHistoryCase) blocksBeforeLastRound() uint64 { + return c.persistenceThreshold + 2*memoryBlockBufferTarget +} + +func proofsHistoryMatrix() []proofsHistoryCase { + var cases []proofsHistoryCase + for _, persistenceThreshold := range []uint64{7, 50} { + for _, stateMaskingBlocks := range []uint64{0, 30} { + for _, verificationInterval := range []uint64{0, 1} { + cases = append(cases, proofsHistoryCase{ + persistenceThreshold: persistenceThreshold, + stateMaskingBlocks: stateMaskingBlocks, + verificationInterval: verificationInterval, + }) + } + } + } + return cases +} + +type proofsHistorySystem struct { + Bridge *dsl.StandardBridge + FunderL2 *dsl.FunderEOA + ProofsFollower *dsl.L2ELNode + Reference *dsl.L2ELNode +} + +// newProofsHistorySystem starts a sequencer without proofs history, which serves as the +// independent eth_getProof reference, and a follower running proofs history under the case's +// reth and ExEx settings. +func newProofsHistorySystem(t devtest.T, c proofsHistoryCase) *proofsHistorySystem { + sysgo.SkipOnOpGeth(t, "proofs history is an op-reth feature") + clKind := sysgo.ResolveMixedL2CLKind() + + runtime := sysgo.NewMixedSingleChainRuntime(t, sysgo.MixedSingleChainPresetConfig{ + NodeSpecs: []sysgo.MixedSingleChainNodeSpec{ + { + CLKey: "sequencer", + CLKind: clKind, + ELKey: "sequencer", + ELKind: sysgo.MixedL2ELOpReth, + IsSequencer: true, + OpRethOpts: []sysgo.OpRethOption{ + sysgo.OpRethWithoutProofsHistory(), + sysgo.OpRethWithEthProofWindow(referenceEthProofWindow), + }, + }, + { + CLKey: "follower", + CLKind: clKind, + ELKey: "proofs-follower", + ELKind: sysgo.MixedL2ELOpReth, + OpRethOpts: []sysgo.OpRethOption{sysgo.OpRethWithExtraArgs(c.opRethArgs()...)}, + }, + }, + }) + frontends := presets.NewMixedSingleChainFrontends(t, runtime) + t.Require().Len(frontends.Nodes, 2, "expected the sequencer and the proofs follower") + + var follower *dsl.L2ELNode + for _, node := range frontends.Nodes { + if !node.Spec.IsSequencer { + follower = node.EL + } + } + t.Require().NotNil(follower, "missing the proofs follower") + + return &proofsHistorySystem{ + Bridge: dsl.NewStandardBridge(t, frontends.L2Network, frontends.L1EL), + FunderL2: frontends.FunderL2, + ProofsFollower: follower, + Reference: frontends.L2Network.PrimaryEL(), + } +} + +func includedBlock(t devtest.T, tx *txplan.PlannedTx) uint64 { + receipt, err := tx.Included.Eval(t.Ctx()) + t.Require().NoError(err, "transaction must be included") + return bigs.Uint64Strict(receipt.BlockNumber) +} + +// TestProofsHistoryFollowerServesMatchingHistoricalProofs runs transfers, withdrawals and contract +// writes against a sequencer that serves eth_getProof from its own historical state, then sweeps a +// proofs-history follower from its window's first block to the last workload block: each of the +// follower's proofs must verify against the block header and equal the sequencer's. +func TestProofsHistoryFollowerServesMatchingHistoricalProofs(gt *testing.T) { + t := devtest.ParallelT(gt) + for _, c := range proofsHistoryMatrix() { + t.Run(c.name(), func(t devtest.T) { + t.Parallel() + if c.stateMaskingBlocks != 0 { + t.Skip("state masking is compiled out of reth 2.5.2 on develop and #23046 rejects a nonzero --engine.num-state-masking-blocks; re-enable with ethereum-optimism/protocol-team#457") + } + sys := newProofsHistorySystem(t, c) + + alice := sys.FunderL2.NewFundedEOA(eth.OneEther) + bob := sys.FunderL2.NewFundedEOA(eth.OneEther) + recorder := alice.DeployBlockRecorder() + var lastWorkloadBlock uint64 + for round := range workloadRounds { + if round == workloadRounds-1 { + // The last round lands after reth has persisted at least once, so the sweep spans + // blocks that left reth's memory and blocks that have not. + target := c.blocksBeforeLastRound() + sys.ProofsFollower.Reached(eth.Unsafe, target, int(target)+30) + } + alice.Transfer(bob.Address(), eth.OneHundredthEther) + bob.Transfer(alice.Address(), eth.OneHundredthEther) + sys.Bridge.InitiateWithdrawal(eth.OneHundredthEther, bob) + lastWorkloadBlock = includedBlock(t, alice.Transact(alice.Plan(), txplan.WithTo(&recorder))) + } + + sys.ProofsFollower.VerifyHistoricalProofsMatch(sys.Reference, func(opts *dsl.HistoricalProofsOpts) { + opts.Accounts = []dsl.ProofAccount{ + {Address: alice.Address()}, + {Address: bob.Address()}, + {Address: recorder, StorageKeys: []common.Hash{common.HexToHash("0x0"), common.HexToHash("0x1")}}, + } + opts.LastBlock = lastWorkloadBlock + }) + }) + } +} diff --git a/op-acceptance-tests/tests/proofs_history_reorg/matrix_test.go b/op-acceptance-tests/tests/proofs_history_reorg/matrix_test.go new file mode 100644 index 00000000000..71539671e53 --- /dev/null +++ b/op-acceptance-tests/tests/proofs_history_reorg/matrix_test.go @@ -0,0 +1,60 @@ +package proofs_history_reorg + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestBlocksInRethMemory(t *testing.T) { + for _, tc := range []struct { + name string + threshold uint64 + height uint64 + want uint64 + }{ + {name: "nothing persisted below the first run", threshold: 7, height: 7, want: 7}, + {name: "first run keeps the buffer target", threshold: 7, height: 8, want: 5}, + {name: "grows by one per block", threshold: 7, height: 10, want: 7}, + {name: "next run three blocks later at threshold 7", threshold: 7, height: 11, want: 5}, + {name: "nothing persisted below the first run at threshold 50", threshold: 50, height: 50, want: 50}, + {name: "first run at threshold 50", threshold: 50, height: 51, want: 5}, + {name: "just before the second run at threshold 50", threshold: 50, height: 96, want: 50}, + {name: "second run at threshold 50", threshold: 50, height: 97, want: 5}, + } { + t.Run(tc.name, func(t *testing.T) { + c := reorgCase{persistenceThreshold: tc.threshold} + require.Equal(t, tc.want, c.blocksInRethMemory(tc.height)) + }) + } +} + +func TestInMemoryDepthsFitRethMemoryWithinOnePersistenceCycle(t *testing.T) { + for _, c := range reorgMatrix() { + if c.kind != inMemoryReorg { + continue + } + t.Run(c.name(), func(t *testing.T) { + for forkPoint := uint64(1); forkPoint < 200; forkPoint++ { + padding := uint64(0) + for !c.forkPointInMemory(forkPoint + padding + c.depth) { + padding++ + } + require.Less(t, padding, c.persistenceThreshold, "fork point %d", forkPoint) + } + }) + } +} + +func TestOnDiskDepthsExceedRethMemory(t *testing.T) { + for _, c := range reorgMatrix() { + if c.kind != onDiskReorg { + continue + } + t.Run(c.name(), func(t *testing.T) { + for h := c.persistenceThreshold + 1; h < 400; h++ { + require.False(t, c.forkPointInMemory(h), "height %d", h) + } + }) + } +} diff --git a/op-acceptance-tests/tests/proofs_history_reorg/proofs_history_reorg_test.go b/op-acceptance-tests/tests/proofs_history_reorg/proofs_history_reorg_test.go new file mode 100644 index 00000000000..f46137ebd4f --- /dev/null +++ b/op-acceptance-tests/tests/proofs_history_reorg/proofs_history_reorg_test.go @@ -0,0 +1,565 @@ +package proofs_history_reorg + +import ( + "fmt" + "testing" + "time" + + "github.com/ethereum-optimism/optimism/op-devstack/devtest" + "github.com/ethereum-optimism/optimism/op-devstack/dsl" + "github.com/ethereum-optimism/optimism/op-devstack/presets" + "github.com/ethereum-optimism/optimism/op-devstack/sysgo" + "github.com/ethereum-optimism/optimism/op-service/bigs" + "github.com/ethereum-optimism/optimism/op-service/eth" + safety "github.com/ethereum-optimism/optimism/op-service/eth/safety" + "github.com/ethereum-optimism/optimism/op-service/logpipe" + "github.com/ethereum-optimism/optimism/op-service/txplan" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/crypto" +) + +const ( + // l2BlockTimeSeconds keeps the runs short: the blocks sequenced by op-node before the fork + // point and after the reorg take a block time each. + l2BlockTimeSeconds = 1 + // memoryBlockBufferTarget is reth's --engine.memory-block-buffer-target, the number of newest + // blocks reth keeps in memory after persisting. It is passed explicitly, like the persistence + // threshold, because the reorg depths are chosen relative to it. + memoryBlockBufferTarget = 5 + // postReorgTrafficBlocks is how far past the sequencer's head the chain is driven once the + // follower has adopted the replacement chain, so the sweep also covers blocks the follower + // executed after the reorg. + postReorgTrafficBlocks = 10 + // referenceEthProofWindow lets the sequencer answer eth_getProof for every block of the run + // from reth's own historical state; reth's default window of zero rejects them all. + referenceEthProofWindow = 1_000_000 + + // blockAttempts bounds, in 2s polls, the wait for a node to reach a block the test sequencer + // just committed. + blockAttempts = 30 + // reorgAttempts bounds, in 2s polls, how long the follower may take to adopt the replacement + // chain. op-node drops gossiped blocks that do not extend its unsafe chain; once a gossiped + // block above its head arrives, its unsafe-gap request hands that block to reth, which + // fetches the missing replacement blocks from its EL peer and switches once they connect. + reorgAttempts = 90 + // storeCatchUpTimeout bounds the wait for the follower's proofs store to flush the chain + // head once sequencing stopped; the store flushes buffered blocks after 10s idle. + storeCatchUpTimeout = 60 * time.Second + // syncAttempts bounds, in 2s polls, the wait for the follower to catch up with the sequencer. + syncAttempts = 60 +) + +var ( + recorderNumberSlot = common.HexToHash("0x0") + recorderTimestampSlot = common.HexToHash("0x1") +) + +// reorgWorkload says what the replaced and the replacement blocks change. +type reorgWorkload string + +const ( + // deployedContract: the first replaced block deploys a contract and the replaced blocks + // after it call it. The replacement block re-includes the deployment, so the contract exists + // at the same address on the replacement chain, and calls it right away; the traffic after + // the reorg keeps calling it. The follower must execute calls into code that only a + // replacement block created. + deployedContract reorgWorkload = "deployed_contract" + // plainTransfers: every block moves ether between accounts, nothing else. + plainTransfers reorgWorkload = "plain_transfers" +) + +// reorgKind is how reth resolves the reorg, decided by where the fork point sits relative to +// reth's persisted tip. +type reorgKind string + +const ( + // inMemoryReorg: the fork point is a block reth still holds in memory or the block it + // persisted last, so only reth's in-memory chain is rewound. + inMemoryReorg reorgKind = "in_memory" + // onDiskReorg: the fork point lies below reth's persisted tip, so reth unwinds its database + // before it can extend the fork point. + onDiskReorg reorgKind = "on_disk" +) + +// reorgCase is one cell of the matrix the follower runs under. +type reorgCase struct { + // depth is the number of blocks the reorg replaces. + depth uint64 + kind reorgKind + persistenceThreshold uint64 + stateMaskingBlocks uint64 + workload reorgWorkload +} + +func (c reorgCase) name() string { + return fmt.Sprintf("threshold_%d-masking_%d-%s-depth_%d_%s", + c.persistenceThreshold, c.stateMaskingBlocks, c.workload, c.depth, c.kind) +} + +// engineArgs pin reth's persistence schedule on both nodes, which decides whether a fork point of +// the cell's depth is in memory or on disk. +func (c reorgCase) engineArgs() []string { + return []string{ + fmt.Sprintf("--engine.memory-block-buffer-target=%d", memoryBlockBufferTarget), + fmt.Sprintf("--engine.persistence-threshold=%d", c.persistenceThreshold), + } +} + +// followerArgs put the follower on the re-execution path for every block. +func (c reorgCase) followerArgs() []string { + args := append(c.engineArgs(), "--proofs-history.verification-interval=1") + if c.stateMaskingBlocks != 0 { + args = append(args, fmt.Sprintf("--engine.num-state-masking-blocks=%d", c.stateMaskingBlocks)) + } + return args +} + +// blocksInRethMemory models how many of the newest blocks reth holds in memory at height h: +// every block until the count first exceeds the persistence threshold, then the buffer target +// right after each persistence run, growing by one per block until the next run. +func (c reorgCase) blocksInRethMemory(h uint64) uint64 { + firstPersistence := c.persistenceThreshold + 1 + if h < firstPersistence { + return h + } + blocksBetweenRuns := firstPersistence - memoryBlockBufferTarget + return memoryBlockBufferTarget + (h-firstPersistence)%blocksBetweenRuns +} + +// forkPointInMemory says whether reth, by the model above, still holds the fork point of a +// reorg of the cell's depth in memory when the chain head is at height head. +func (c reorgCase) forkPointInMemory(head uint64) bool { + return c.blocksInRethMemory(head) >= c.depth +} + +// reorgMatrix crosses the reth settings and the workloads with three reorg depths per threshold: +// shallower than the memory block buffer target, so the fork point is among the blocks reth +// never persisted; deeper than it, so the fork point has been eligible for persistence; and +// deeper than reth ever holds in memory, which reth resolves as an on-disk reorg. reth holds +// between the buffer target and the threshold of blocks in memory, so the on-disk depths exceed +// the threshold with a margin. +func reorgMatrix() []reorgCase { + type depthCell struct { + depth uint64 + kind reorgKind + } + depthsByThreshold := []struct { + persistenceThreshold uint64 + depths []depthCell + }{ + {persistenceThreshold: 7, depths: []depthCell{{2, inMemoryReorg}, {6, inMemoryReorg}, {12, onDiskReorg}}}, + {persistenceThreshold: 50, depths: []depthCell{{2, inMemoryReorg}, {6, inMemoryReorg}, {55, onDiskReorg}}}, + } + var cases []reorgCase + for _, threshold := range depthsByThreshold { + for _, stateMaskingBlocks := range []uint64{0, 30} { + for _, workload := range []reorgWorkload{deployedContract, plainTransfers} { + for _, cell := range threshold.depths { + cases = append(cases, reorgCase{ + depth: cell.depth, + kind: cell.kind, + persistenceThreshold: threshold.persistenceThreshold, + stateMaskingBlocks: stateMaskingBlocks, + workload: workload, + }) + } + } + } + } + return cases +} + +// skipUnsupportedCell skips the cells the configured clients cannot run. +func skipUnsupportedCell(t devtest.T, c reorgCase) { + sysgo.SkipOnOpGeth(t, "proofs history is an op-reth feature") + sysgo.SkipOnKonaNode(t, "the test sequencer commits blocks through op-node's opstack_commitBlockV1 and opstack_publishBlockV1 RPCs, which kona-node does not serve") + if c.stateMaskingBlocks != 0 { + t.Skip("state masking is compiled out of reth 2.5.2 on develop and #23046 rejects a nonzero --engine.num-state-masking-blocks; re-enable with ethereum-optimism/protocol-team#457") + } +} + +type reorgSystem struct { + Batcher *dsl.L2Batcher + Follower *dsl.L2ELNode + FunderL2 *dsl.FunderEOA + L2Network *dsl.L2Network + Sequencer *dsl.L2ELNode + SequencerCL *dsl.L2CLNode + TestSequencer *dsl.TestSequencer +} + +// newReorgSystem starts a sequencer without proofs history, which serves as the independent +// eth_getProof reference and whose op-node the test sequencer drives around the reorg, and a +// follower re-executing every block into its proofs store under the cell's reth settings. +func newReorgSystem(t devtest.T, c reorgCase) *reorgSystem { + clKind := sysgo.ResolveMixedL2CLKind() + runtime := sysgo.NewMixedSingleChainRuntime(t, sysgo.MixedSingleChainPresetConfig{ + DeployerOptions: []sysgo.DeployerOption{sysgo.WithUniformL2BlockTimes(l2BlockTimeSeconds)}, + NodeSpecs: []sysgo.MixedSingleChainNodeSpec{ + { + CLKey: "sequencer", + CLKind: clKind, + ELKey: "sequencer", + ELKind: sysgo.MixedL2ELOpReth, + IsSequencer: true, + OpRethOpts: []sysgo.OpRethOption{ + sysgo.OpRethWithoutProofsHistory(), + sysgo.OpRethWithEthProofWindow(referenceEthProofWindow), + sysgo.OpRethWithExtraArgs(c.engineArgs()...), + }, + }, + { + CLKey: "follower", + CLKind: clKind, + ELKey: "proofs-follower", + ELKind: sysgo.MixedL2ELOpReth, + OpRethOpts: []sysgo.OpRethOption{sysgo.OpRethWithExtraArgs(c.followerArgs()...)}, + }, + }, + TestSequencerName: "test-sequencer", + WithTestSequencer: true, + }) + frontends := presets.NewMixedSingleChainFrontends(t, runtime) + t.Require().Len(frontends.Nodes, 2, "expected the sequencer and the proofs follower") + t.Require().NotNil(frontends.TestSequencer, "missing the test sequencer") + + sys := &reorgSystem{ + Batcher: frontends.L2Batcher, + FunderL2: frontends.FunderL2, + L2Network: frontends.L2Network, + TestSequencer: frontends.TestSequencer, + } + for _, node := range frontends.Nodes { + if node.Spec.IsSequencer { + sys.Sequencer, sys.SequencerCL = node.EL, node.CL + } else { + sys.Follower = node.EL + } + } + t.Require().NotNil(sys.Follower, "missing the proofs follower") + return sys +} + +// followerLogs are the follower's op-reth log lines the test reads after the reorg. +type followerLogs struct { + // leftChain is the ExEx handing a block that left reth's chain mid-execution to its sync + // loop; the line exists from the reth 2.6 bump (#23046) on. + leftChain *logpipe.LogRecord + // onDiskReorg is reth resolving a fork point below its persisted tip. + onDiskReorg *logpipe.LogRecord + // panicked is a reth critical task, such as the ExEx, crashing the node. + panicked *logpipe.LogRecord + // parentStateMissing is the ExEx skipping a block whose parent state reth no longer holds, + // leaving it to a later notification. + parentStateMissing *logpipe.LogRecord + // reorgHandled is the proofs engine unwinding blocks it stored and buffering the replacement + // path, as opposed to ignoring a reorg above its stored tip. + reorgHandled *logpipe.LogRecord + // reorgNotified is the ExEx receiving reth's reorg notification. + reorgNotified *logpipe.LogRecord +} + +func recordFollowerLogs(follower *dsl.L2ELNode) followerLogs { + return followerLogs{ + leftChain: follower.RecordLogLines("Block left reth's chain during proofs execution"), + onDiskReorg: follower.RecordLogLines("on-disk reorg detected"), + panicked: follower.RecordLogLines("panicked"), + parentStateMissing: follower.RecordLogLines("Parent state not available in reth; skipping execute_block"), + reorgHandled: follower.RecordLogLines("Handling reorg: unwinding and buffering new path"), + reorgNotified: follower.RecordLogLines("ChainReorged notification received"), + } +} + +// verify requires the follower to have handled the reorg the way the cell expects, and reports +// the lines whose presence depends on timing or on the op-reth version under test. +func (l followerLogs) verify(t devtest.T, c reorgCase) { + t.Require().NotZerof(l.reorgNotified.Count(), "the follower's ExEx never received a reorg notification") + t.Require().NotZerof(l.reorgHandled.Count(), "the follower's proofs engine never unwound the replaced blocks: the reorg did not reach blocks it had stored") + if c.kind == onDiskReorg { + t.Require().NotZerof(l.onDiskReorg.Count(), "expected a fork point below reth's persisted tip, but reth never reported an on-disk reorg; the depth %d is not deeper than what reth held in memory", c.depth) + } else { + t.Require().Zerof(l.onDiskReorg.Count(), "expected the fork point in reth's memory, but reth reported an on-disk reorg: %v", messagesOf(l.onDiskReorg)) + } + t.Logger().Info("Follower reorg handling log lines", + "reorg_notified", l.reorgNotified.Count(), + "reorg_handled", l.reorgHandled.Count(), + "on_disk_reorg", l.onDiskReorg.Count(), + "parent_state_missing_skips", l.parentStateMissing.Count(), + "left_chain_handoffs", l.leftChain.Count()) +} + +func messagesOf(record *logpipe.LogRecord) []string { + var messages []string + for _, entry := range record.Entries() { + messages = append(messages, fmt.Sprintf("%s %v", entry.LogMessage(), entry.LogFields())) + } + return messages +} + +// workload is the accounts the blocks around the reorg change. alice and bob transact before +// the fork point, in the replaced blocks and after the reorg; carol only ever transacts on the +// replacement chain, so the nonces her replacement transactions need are the same on both sides +// of the fork; the deployer's only transaction is the recorder deployment, which the first +// replaced block carries and the replacement block carries again. +type workload struct { + alice *dsl.EOA + bob *dsl.EOA + carol *dsl.EOA + deployer *dsl.EOA + kind reorgWorkload + t devtest.T + + deployment signedTx + recorder common.Address + // setupBlock is the block that completed the funding; the fork point must be above it. + setupBlock uint64 +} + +func newWorkload(t devtest.T, funder *dsl.FunderEOA, kind reorgWorkload) *workload { + w := &workload{ + alice: funder.NewFundedEOA(eth.OneEther), + bob: funder.NewFundedEOA(eth.OneEther), + carol: funder.NewFundedEOA(eth.OneEther), + deployer: funder.NewFundedEOA(eth.OneEther), + kind: kind, + t: t, + } + w.setupBlock = includedBlock(t, w.alice.Transfer(w.bob.Address(), eth.OneGWei)) + if kind == deployedContract { + w.deployment = signTx(t, w.deployer.PlanBlockRecorderDeployment()) + w.recorder = crypto.CreateAddress(w.deployer.Address(), w.deployment.nonce) + t.Logger().Info("Signed the recorder deployment", "address", w.recorder, "tx", w.deployment.hash) + } + return w +} + +// accounts are the accounts the proofs sweep proves at every block. +func (w *workload) accounts() []dsl.ProofAccount { + accounts := []dsl.ProofAccount{ + {Address: w.alice.Address()}, + {Address: w.bob.Address()}, + {Address: w.carol.Address()}, + {Address: w.deployer.Address()}, + } + if w.kind == deployedContract { + accounts = append(accounts, dsl.ProofAccount{Address: w.recorder, StorageKeys: []common.Hash{recorderNumberSlot, recorderTimestampSlot}}) + } + return accounts +} + +// driveUntilBlock sends transactions one after another, each after the previous one was +// included, until one lands in a block at or past target, and returns that block. +func (w *workload) driveUntilBlock(target uint64) uint64 { + for { + if block := includedBlock(w.t, w.alice.Transfer(w.bob.Address(), eth.OneHundredthEther)); block >= target { + return block + } + if block := includedBlock(w.t, w.bob.Transact(w.bobsReply())); block >= target { + return block + } + } +} + +// bobsReply is bob's part of the traffic: a call into the recorder for the deployed-contract +// workload, which before the deployment is a plain transaction to an address without code, a +// transfer back to alice otherwise. +func (w *workload) bobsReply() txplan.Option { + if w.kind == deployedContract { + return txplan.Combine(w.bob.Plan(), txplan.WithTo(&w.recorder)) + } + return w.bob.PlanTransfer(w.alice.Address(), eth.OneHundredthEther) +} + +// replacedBlockTxs are the transactions of the n-th replaced block, 1-based: the recorder +// deployment first and calls into it after that for the deployed-contract workload, transfers +// otherwise. +func (w *workload) replacedBlockTxs(n int) []signedTx { + switch { + case w.kind == deployedContract && n == 1: + return []signedTx{w.deployment, signTx(w.t, txplan.NewPlannedTx(w.alice.PlanTransfer(w.bob.Address(), eth.OneHundredthEther)))} + case w.kind == deployedContract: + return []signedTx{signTx(w.t, txplan.NewPlannedTx(w.alice.Plan(), txplan.WithTo(&w.recorder)))} + default: + return []signedTx{signTx(w.t, txplan.NewPlannedTx(w.alice.PlanTransfer(w.bob.Address(), eth.OneHundredthEther)))} + } +} + +// replacementBlockTxs are the transactions of the replacement block: a transfer from carol, and +// for the deployed-contract workload the same recorder deployment as on the replaced chain +// followed by carol's call into it. carol's nonces are assigned up front, since both of her +// transactions are signed before either is included. +func (w *workload) replacementBlockTxs(int) []signedTx { + nonce := w.carol.PendingNonce() + transfer := signTx(w.t, txplan.NewPlannedTx(w.carol.PlanTransfer(w.bob.Address(), eth.OneHundredthEther), txplan.WithStaticNonce(nonce))) + if w.kind != deployedContract { + return []signedTx{transfer} + } + call := signTx(w.t, txplan.NewPlannedTx(w.carol.Plan(), txplan.WithTo(&w.recorder), txplan.WithStaticNonce(nonce+1))) + return []signedTx{w.deployment, transfer, call} +} + +type signedTx struct { + hash common.Hash + nonce uint64 + raw []byte +} + +func signTx(t devtest.T, tx *txplan.PlannedTx) signedTx { + signed, err := tx.Signed.Eval(t.Ctx()) + t.Require().NoError(err, "transaction must sign") + raw, err := signed.MarshalBinary() + t.Require().NoError(err, "transaction must encode") + return signedTx{hash: signed.Hash(), nonce: signed.Nonce(), raw: raw} +} + +func rawTxs(txs []signedTx) [][]byte { + raws := make([][]byte, len(txs)) + for i, tx := range txs { + raws[i] = tx.raw + } + return raws +} + +func includedBlock(t devtest.T, tx *txplan.PlannedTx) uint64 { + receipt, err := tx.Included.Eval(t.Ctx()) + t.Require().NoError(err, "transaction must be included") + return bigs.Uint64Strict(receipt.BlockNumber) +} + +// sequencedChain is a run of consecutive blocks the test sequencer committed, lowest first, with +// the transactions each of them carries. +type sequencedChain struct { + blocks []eth.L2BlockRef + txs [][]signedTx +} + +func (s sequencedChain) head() eth.L2BlockRef { + return s.blocks[len(s.blocks)-1] +} + +// verifyServedBy requires node to serve every block of the chain with its transactions. +func (s sequencedChain) verifyServedBy(t devtest.T, node *dsl.L2ELNode) { + for i, block := range s.blocks { + t.Require().Equalf(block.Hash, node.BlockRefByNumber(block.Number).Hash, "%s must serve block %d of the chain", node, block.Number) + for _, tx := range s.txs[i] { + node.AssertTxInBlock(block.Number, tx.hash) + } + } +} + +// sequenceChain commits count blocks on parent with the test sequencer, each carrying the +// transactions blockTxs returns for it and none ahead of wall-clock time, and returns them once +// the sequencer's EL serves each. +func sequenceChain(t devtest.T, sys *reorgSystem, parent eth.L2BlockRef, count uint64, blockTxs func(n int) []signedTx) sequencedChain { + chain := sequencedChain{} + for i := range int(count) { + txs := blockTxs(i + 1) + t.Logger().Info("Sequencing a block with the test sequencer", "number", parent.Number+1, "parent", parent, "txs", len(txs)) + sys.TestSequencer.SequenceBlockWithTxsWhenDue(t, sys.L2Network, parent, rawTxs(txs)) + block := sys.Sequencer.BlockRefByNumber(parent.Number + 1) + t.Require().Equalf(parent.Hash, block.ParentHash, "block %d must extend %s", block.Number, parent) + chain.blocks = append(chain.blocks, block) + chain.txs = append(chain.txs, txs) + parent = block + } + return chain +} + +// stopSequencing drives traffic past the workload's setup and stops op-node's sequencer, which +// makes its head the fork point of the reorg. For an in-memory cell the chain is then padded +// with empty blocks until the replaced blocks will leave the fork point in reth's memory by the +// persistence model, so the fork point does not depend on where the stop lands. +func stopSequencing(t devtest.T, sys *reorgSystem, w *workload, c reorgCase) eth.L2BlockRef { + w.driveUntilBlock(w.setupBlock + 1) + forkPoint := sys.Sequencer.BlockRefByHash(sys.SequencerCL.StopSequencer()) + padding := 0 + for c.kind == inMemoryReorg && !c.forkPointInMemory(forkPoint.Number+c.depth) { + t.Require().Lessf(padding, int(c.persistenceThreshold), + "no height keeps a fork point %d blocks deep in reth's memory at threshold %d", c.depth, c.persistenceThreshold) + forkPoint = sequenceChain(t, sys, forkPoint, 1, func(int) []signedTx { return nil }).head() + padding++ + } + t.Logger().Info("Sequencing stopped at the fork point", "fork_point", forkPoint, "depth", c.depth, "kind", c.kind, "padding_blocks", padding) + return forkPoint +} + +// sequenceReplacedChain builds the blocks the reorg will replace on the fork point and waits +// until the follower has imported them and its proofs store has flushed them, so every replaced +// block was re-executed into the store before it is replaced. +func sequenceReplacedChain(t devtest.T, sys *reorgSystem, w *workload, forkPoint eth.L2BlockRef, c reorgCase) sequencedChain { + replaced := sequenceChain(t, sys, forkPoint, c.depth, w.replacedBlockTxs) + sys.Follower.Reached(eth.Unsafe, replaced.head().Number, blockAttempts) + replaced.verifyServedBy(t, sys.Follower) + sys.Follower.WaitForProofsStoreBlock(replaced.head().Number, storeCatchUpTimeout) + return replaced +} + +// sequenceReplacementChain builds the first replacement block on the fork point, which reorgs +// the sequencer's EL, and waits until the sequencer's op-node is on it so it resumes sequencing +// from there. One block is all the test sequencer can build on a deep fork: its timestamp is +// the fork point's plus a block time, which op-node's gossip validation rejects as too old once +// the replaced chain took longer than the gossip age threshold to build, and a block that is +// not published leaves the test sequencer's job sealed. op-node builds the remaining +// replacement blocks, as it does after a real sequencer failover. +func sequenceReplacementChain(t devtest.T, sys *reorgSystem, w *workload, forkPoint eth.L2BlockRef, replaced sequencedChain) sequencedChain { + replacement := sequenceChain(t, sys, forkPoint, 1, w.replacementBlockTxs) + t.Require().NotEqualf(replaced.blocks[0].Hash, replacement.head().Hash, "the replacement block must differ from the replaced block at height %d", replacement.head().Number) + sys.SequencerCL.MatchedUnsafe(sys.Sequencer, syncAttempts) + return replacement +} + +// verifyRecorderCalled requires the recorder's storage on node to hold the number of a block +// after the fork point at block, which only a call on the replacement chain can have written, so +// the sweep's matching proofs of that slot were not vacuous. +func verifyRecorderCalled(t devtest.T, node *dsl.L2ELNode, w *workload, forkPoint eth.L2BlockRef, block uint64) { + ref := node.BlockRefByNumber(block) + value, err := node.EthClient().ReadStorageAt(t.Ctx(), w.recorder, recorderNumberSlot, ref.Hash) + t.Require().NoError(err, "must read the recorder's storage at block %d", block) + t.Require().Greaterf(bigs.Uint64Strict(value.Big()), forkPoint.Number, + "the recorder at block %d must have been called on the replacement chain, but holds block number %s", block, value.Big()) +} + +// TestProofsHistoryFollowerMatchesTheReplacementChainAfterASequencerReorg has a follower that +// re-executes every block into its proofs store watch the sequencer replace blocks the store +// already holds: op-node stops sequencing at the fork point, the test sequencer builds the +// blocks to be replaced on top and the follower executes them, then the test sequencer builds a +// conflicting block on the fork point and op-node resumes on top of it. The follower must adopt +// the replacement chain, keep executing, and serve proofs from the fork point onwards that match +// the sequencer's own historical state. +func TestProofsHistoryFollowerMatchesTheReplacementChainAfterASequencerReorg(gt *testing.T) { + t := devtest.ParallelT(gt) + for _, c := range reorgMatrix() { + t.Run(c.name(), func(t devtest.T) { + t.Parallel() + skipUnsupportedCell(t, c) + sys := newReorgSystem(t, c) + // Blocks that reach L1 are derived back as they were, so the batcher stays off until + // the replacement chain exists and is the one it submits. + sys.Batcher.Stop() + logs := recordFollowerLogs(sys.Follower) + w := newWorkload(t, sys.FunderL2, c.workload) + + forkPoint := stopSequencing(t, sys, w, c) + replaced := sequenceReplacedChain(t, sys, w, forkPoint, c) + replacement := sequenceReplacementChain(t, sys, w, forkPoint, replaced) + sys.Batcher.Start() + sys.SequencerCL.StartSequencer() + + sys.Follower.ReorgExact(replaced.blocks[0], reorgAttempts) + t.Require().Emptyf(messagesOf(logs.panicked), "the follower's op-reth must survive the reorg") + sys.Follower.InSync(sys.Sequencer, safety.LocalUnsafe, syncAttempts) + replacement.verifyServedBy(t, sys.Follower) + lastBlock := w.driveUntilBlock(sys.Sequencer.BlockRefByLabel(eth.Unsafe).Number + postReorgTrafficBlocks) + + sys.Follower.VerifyHistoricalProofsMatch(sys.Sequencer, func(opts *dsl.HistoricalProofsOpts) { + opts.Accounts = w.accounts() + opts.FirstBlock = forkPoint.Number + opts.LastBlock = lastBlock + }) + if c.workload == deployedContract { + verifyRecorderCalled(t, sys.Follower, w, forkPoint, lastBlock) + } + logs.verify(t, c) + }) + } +} diff --git a/op-acceptance-tests/tests/proofs_history_restart/proofs_history_restart_test.go b/op-acceptance-tests/tests/proofs_history_restart/proofs_history_restart_test.go new file mode 100644 index 00000000000..4e4f4a79001 --- /dev/null +++ b/op-acceptance-tests/tests/proofs_history_restart/proofs_history_restart_test.go @@ -0,0 +1,364 @@ +package proofs_history_restart + +import ( + "fmt" + "testing" + "time" + + "github.com/ethereum-optimism/optimism/op-devstack/devtest" + "github.com/ethereum-optimism/optimism/op-devstack/dsl" + "github.com/ethereum-optimism/optimism/op-devstack/presets" + "github.com/ethereum-optimism/optimism/op-devstack/sysgo" + nodeSync "github.com/ethereum-optimism/optimism/op-node/rollup/sync" + "github.com/ethereum-optimism/optimism/op-service/bigs" + "github.com/ethereum-optimism/optimism/op-service/eth" + safety "github.com/ethereum-optimism/optimism/op-service/eth/safety" + "github.com/ethereum-optimism/optimism/op-service/txplan" + "github.com/ethereum/go-ethereum/common" +) + +const ( + // l2BlockTimeSeconds keeps the runs short: the kill point is a block height, so halving the + // block time halves the time spent producing blocks. + l2BlockTimeSeconds = 1 + // memoryBlockBufferTarget is reth's --engine.memory-block-buffer-target. It is passed + // explicitly, like the persistence threshold, so the gap the kill opens between the proofs + // store and reth's datadir does not depend on reth's defaults. + memoryBlockBufferTarget = 5 + // postRestartTrafficBlocks is how far past the head the chain is driven once the follower has + // caught up, so the sweep also covers blocks the proofs store indexed after coming back. + postRestartTrafficBlocks = 10 + // referenceEthProofWindow lets the sequencer answer eth_getProof for every block of the run + // from reth's own historical state; reth's default window of zero rejects them all. + referenceEthProofWindow = 1_000_000 + + // rebuildAttempts bounds how long the restarted sequencer may take to rebuild the heights it + // lost with different contents. + rebuildAttempts = 30 + // resyncAttempts bounds how long the restarted follower may take to catch up with the + // sequencer. A follower whose op-reth crash-loops never does. + resyncAttempts = 60 + // storeCatchUpTimeout bounds the wait for a proofs store flush before the kill. + storeCatchUpTimeout = 60 * time.Second +) + +// killPoint says where, relative to the proofs store's flush cadence, the follower is killed. +type killPoint string + +const ( + // killAfterStoreFlush kills the follower as soon as the store has flushed the target block, so + // the store's tip is the chain head and its lead over reth's datadir is the largest. + killAfterStoreFlush killPoint = "after_flush" + // killMidStoreBuffer kills the follower a couple of blocks after a flush, so the store's tip + // trails the chain head and the store loses blocks of its own on top of reth's. + killMidStoreBuffer killPoint = "mid_buffer" +) + +// restartCase is one cell of the matrix the follower runs under. +type restartCase struct { + killPoint killPoint + persistenceThreshold uint64 + // storeFlushBeforeKill is the proofs store tip the kill waits for. + storeFlushBeforeKill uint64 + syncMode nodeSync.Mode + verificationInterval uint64 +} + +func (c restartCase) name() string { + proofsPath := "index" + if c.verificationInterval != 0 { + proofsPath = "execute" + } + syncMode := "cl_sync" + if c.syncMode == nodeSync.ELSync { + syncMode = "el_sync" + } + return fmt.Sprintf("threshold_%d-%s-%s-kill_%s", c.persistenceThreshold, proofsPath, syncMode, c.killPoint) +} + +// engineArgs pins reth's persistence schedule, which decides how many blocks an unclean stop +// loses. Both nodes get them so the sequencer loses blocks on the same schedule as the follower. +func (c restartCase) engineArgs() []string { + return []string{ + fmt.Sprintf("--engine.memory-block-buffer-target=%d", memoryBlockBufferTarget), + fmt.Sprintf("--engine.persistence-threshold=%d", c.persistenceThreshold), + } +} + +func (c restartCase) followerArgs() []string { + return append(c.engineArgs(), fmt.Sprintf("--proofs-history.verification-interval=%d", c.verificationInterval)) +} + +// restartMatrix pairs the kill point with the persistence threshold rather than crossing them: +// the small threshold gets the mid-buffer kill, where both reth and the store lose blocks, the +// large one the after-flush kill, where the store's lead over reth is largest. The kill heights +// leave reth's datadir behind the store: at threshold 7 reth persists every few blocks, at 50 it +// has persisted once by block 60 (down to block 46) and holds the rest in memory. +func restartMatrix() []restartCase { + var cases []restartCase + for _, schedule := range []struct { + killPoint killPoint + persistenceThreshold uint64 + storeFlushBeforeKill uint64 + }{ + {killPoint: killMidStoreBuffer, persistenceThreshold: 7, storeFlushBeforeKill: 25}, + {killPoint: killAfterStoreFlush, persistenceThreshold: 50, storeFlushBeforeKill: 60}, + } { + for _, verificationInterval := range []uint64{0, 1} { + for _, syncMode := range []nodeSync.Mode{nodeSync.CLSync, nodeSync.ELSync} { + cases = append(cases, restartCase{ + killPoint: schedule.killPoint, + persistenceThreshold: schedule.persistenceThreshold, + storeFlushBeforeKill: schedule.storeFlushBeforeKill, + syncMode: syncMode, + verificationInterval: verificationInterval, + }) + } + } + } + return cases +} + +// skipUnsupportedCell skips the cells the configured clients cannot run. +func skipUnsupportedCell(t devtest.T, c restartCase) { + sysgo.SkipOnOpGeth(t, "proofs history is an op-reth feature") + if c.syncMode == nodeSync.CLSync { + sysgo.SkipOnKonaNode(t, "kona-node has no CL sync mode: it always lets the EL sync the gap") + } +} + +type restartSystem struct { + Batcher *dsl.L2Batcher + Follower *dsl.L2ELNode + FollowerCL *dsl.L2CLNode + FunderL2 *dsl.FunderEOA + Sequencer *dsl.L2ELNode + SequencerCL *dsl.L2CLNode +} + +// newRestartSystem starts a sequencer without proofs history, which serves as the independent +// eth_getProof reference, and a follower running proofs history under the case's reth, ExEx and +// sync-mode settings. +func newRestartSystem(t devtest.T, c restartCase) *restartSystem { + clKind := sysgo.ResolveMixedL2CLKind() + runtime := sysgo.NewMixedSingleChainRuntime(t, sysgo.MixedSingleChainPresetConfig{ + DeployerOptions: []sysgo.DeployerOption{sysgo.WithUniformL2BlockTimes(l2BlockTimeSeconds)}, + NodeSpecs: []sysgo.MixedSingleChainNodeSpec{ + { + CLKey: "sequencer", + CLKind: clKind, + ELKey: "sequencer", + ELKind: sysgo.MixedL2ELOpReth, + IsSequencer: true, + OpRethOpts: []sysgo.OpRethOption{ + sysgo.OpRethWithoutProofsHistory(), + sysgo.OpRethWithEthProofWindow(referenceEthProofWindow), + sysgo.OpRethWithExtraArgs(c.engineArgs()...), + }, + }, + { + CLKey: "follower", + CLKind: clKind, + ELKey: "proofs-follower", + ELKind: sysgo.MixedL2ELOpReth, + L2CLOpts: []sysgo.L2CLOption{sysgo.L2CLVerifierSyncMode(c.syncMode)}, + OpRethOpts: []sysgo.OpRethOption{sysgo.OpRethWithExtraArgs(c.followerArgs()...)}, + }, + }, + }) + frontends := presets.NewMixedSingleChainFrontends(t, runtime) + t.Require().Len(frontends.Nodes, 2, "expected the sequencer and the proofs follower") + + sys := &restartSystem{Batcher: frontends.L2Batcher, FunderL2: frontends.FunderL2} + for _, node := range frontends.Nodes { + if node.Spec.IsSequencer { + sys.Sequencer, sys.SequencerCL = node.EL, node.CL + } else { + sys.Follower, sys.FollowerCL = node.EL, node.CL + } + } + t.Require().NotNil(sys.Follower, "missing the proofs follower") + return sys +} + +// workload changes state in nearly every block: alice and bob transfer back and forth and bob +// writes to a contract, so every block an unclean stop loses carries balance and storage changes +// that a rebuilt block would not repeat. +type workload struct { + alice *dsl.EOA + bob *dsl.EOA + recorder common.Address + // setupBlock is the block that completed the funding and the contract deployment; a rebuilt + // chain must start below it or the workload's accounts no longer exist. + setupBlock uint64 + t devtest.T +} + +func newWorkload(t devtest.T, funder *dsl.FunderEOA) *workload { + w := &workload{ + alice: funder.NewFundedEOA(eth.OneEther), + bob: funder.NewFundedEOA(eth.OneEther), + t: t, + } + w.recorder = w.alice.DeployBlockRecorder() + w.setupBlock = includedBlock(t, w.alice.Transact(w.alice.Plan(), txplan.WithTo(&w.recorder))) + return w +} + +// accounts are the accounts the proofs sweep proves at every block. +func (w *workload) accounts() []dsl.ProofAccount { + return []dsl.ProofAccount{ + {Address: w.alice.Address()}, + {Address: w.bob.Address()}, + {Address: w.recorder, StorageKeys: []common.Hash{common.HexToHash("0x0"), common.HexToHash("0x1")}}, + } +} + +// driveUntilBlock sends transactions one after another until one lands in a block at or past +// target, and returns that block. +func (w *workload) driveUntilBlock(target uint64) uint64 { + for { + w.alice.Transfer(w.bob.Address(), eth.OneHundredthEther) + w.bob.Transfer(w.alice.Address(), eth.OneHundredthEther) + block := includedBlock(w.t, w.bob.Transact(w.bob.Plan(), txplan.WithTo(&w.recorder))) + if block >= target { + return block + } + } +} + +func includedBlock(t devtest.T, tx *txplan.PlannedTx) uint64 { + receipt, err := tx.Included.Eval(t.Ctx()) + t.Require().NoError(err, "transaction must be included") + return bigs.Uint64Strict(receipt.BlockNumber) +} + +// uncleanStop is what the follower held when it was killed: the chain head it had imported and +// the last block its proofs store had flushed, both on the chain of that moment. +type uncleanStop struct { + head eth.L2BlockRef + storeTip eth.L2BlockRef +} + +// killFollower drives the workload until the follower's proofs store has flushed the case's target +// block and kills the follower's op-reth at the case's kill point. Its CL is left to the caller, +// so a test that kills the sequencer too can do so right away. +func killFollower(t devtest.T, sys *restartSystem, w *workload, c restartCase) uncleanStop { + // Stop sending transactions two blocks short of the flush: close enough that the last + // transactions sit above reth's persisted tip, yet with no transaction in flight to delay the + // kill once the flush is observed. + w.driveUntilBlock(c.storeFlushBeforeKill - 2) + window := sys.Follower.WaitForProofsStoreBlock(c.storeFlushBeforeKill, storeCatchUpTimeout) + if c.killPoint == killMidStoreBuffer { + sys.Follower.WaitForBlockNumber(*window.Latest + 2) + } + // Read the store tip again right before the kill: a flush since the wait would have moved it. + window, err := sys.Follower.ProofsSyncStatus() + t.Require().NoError(err, "debug_proofsSyncStatus must answer before the kill") + storeTip := sys.Follower.BlockRefByNumber(*window.Latest) + head := sys.Follower.BlockRefByLabel(eth.Unsafe) + t.Logger().Info("Killing the follower", "head", head, "proofs_store_tip", storeTip, "kill_point", c.killPoint) + + sys.Follower.StopUngracefully() + return uncleanStop{head: head, storeTip: storeTip} +} + +// restartFollower brings the follower's op-reth and CL back and re-peers them with the sequencer, +// since both peerings died with the processes. It returns the block reth resumes from: its datadir +// tip, which must be below the proofs store's tip for the test to exercise the crash gap at all. +func restartFollower(t devtest.T, sys *restartSystem, stop uncleanStop) eth.L2BlockRef { + sys.Follower.Start() + sys.Follower.WaitForOnline() + // Nothing feeds the EL until its CL is back, so its head is what reth had persisted. + persistedTip := sys.Follower.BlockRefByLabel(eth.Unsafe) + t.Logger().Info("Follower op-reth is back on its datadir", "persisted_tip", persistedTip, + "proofs_store_tip", stop.storeTip, "head_at_kill", stop.head) + t.Require().Lessf(persistedTip.Number, stop.storeTip.Number, + "the proofs store (tip %d) must be ahead of reth's datadir (tip %d) after the kill; the kill landed too late for this threshold", + stop.storeTip.Number, persistedTip.Number) + + sys.Follower.PeerWith(sys.Sequencer) + sys.FollowerCL.Start() + sys.FollowerCL.ConnectPeer(sys.SequencerCL) + return persistedTip +} + +// TestProofsHistoryFollowerRecoversFromUncleanShutdown kills a proofs-history follower once its +// store has flushed blocks reth has not yet persisted, restarts it into the same chain and +// requires it to catch up with the sequencer, keep indexing, and serve proofs that match the +// sequencer's for the blocks it lost, the blocks it re-imported and the blocks that followed. +func TestProofsHistoryFollowerRecoversFromUncleanShutdown(gt *testing.T) { + t := devtest.ParallelT(gt) + for _, c := range restartMatrix() { + t.Run(c.name(), func(t devtest.T) { + t.Parallel() + skipUnsupportedCell(t, c) + sys := newRestartSystem(t, c) + w := newWorkload(t, sys.FunderL2) + + stop := killFollower(t, sys, w, c) + sys.FollowerCL.Stop() + persistedTip := restartFollower(t, sys, stop) + + sys.Follower.InSync(sys.Sequencer, safety.LocalUnsafe, resyncAttempts) + lastBlock := w.driveUntilBlock(sys.Sequencer.BlockRefByLabel(eth.Unsafe).Number + postRestartTrafficBlocks) + sys.Follower.VerifyHistoricalProofsMatch(sys.Sequencer, func(opts *dsl.HistoricalProofsOpts) { + opts.Accounts = w.accounts() + opts.FirstBlock = persistedTip.Number + opts.LastBlock = lastBlock + }) + }) + } +} + +// TestProofsHistoryFollowerServesRebuiltBlocksAfterUncleanShutdown kills a proofs-history +// follower, then kills the sequencer's op-reth too while the blocks the follower's store holds are +// still only in the sequencer's memory, so the restarted sequencer rebuilds those heights with +// different contents. The restarted follower must follow the rebuilt chain and serve its proofs, +// not the ones its store recorded for the chain that no longer exists. +func TestProofsHistoryFollowerServesRebuiltBlocksAfterUncleanShutdown(gt *testing.T) { + t := devtest.ParallelT(gt) + for _, c := range restartMatrix() { + t.Run(c.name(), func(t devtest.T) { + t.Parallel() + skipUnsupportedCell(t, c) + t.Skip("fails on develop: the restarted follower's op-reth exits with `Critical task exex panicked: ExEx proofs-history crashed: Parent hash mismatch` at the block after its proofs store's tip, because the store is never reconciled with reth's datadir (ethereum-optimism/protocol-team#464); un-skip with #23358") + sys := newRestartSystem(t, c) + // Blocks that reached L1 would be derived back as they were; blocks that never did + // are re-sequenced, which is the rebuild this test needs. + sys.Batcher.Stop() + w := newWorkload(t, sys.FunderL2) + + // Both op-reths die back to back, so the sequencer still holds the blocks the + // follower's store flushed; the CLs follow once the processes are gone. + stop := killFollower(t, sys, w, c) + sys.Sequencer.StopUngracefully() + sys.FollowerCL.Stop() + sys.SequencerCL.Stop() + + sys.Sequencer.Start() + sys.Sequencer.WaitForOnline() + sequencerPersistedTip := sys.Sequencer.BlockRefByLabel(eth.Unsafe) + t.Logger().Info("Sequencer op-reth is back on its datadir", "persisted_tip", sequencerPersistedTip, + "follower_proofs_store_tip", stop.storeTip, "workload_setup_block", w.setupBlock) + t.Require().Lessf(sequencerPersistedTip.Number, stop.storeTip.Number, + "the sequencer must rebuild blocks the follower's proofs store holds, but its datadir tip %d is not below the store tip %d", + sequencerPersistedTip.Number, stop.storeTip.Number) + t.Require().LessOrEqualf(w.setupBlock, sequencerPersistedTip.Number, + "the workload's funding and contract (block %d) must survive the sequencer's crash, but its datadir tip is %d", + w.setupBlock, sequencerPersistedTip.Number) + sys.SequencerCL.Start() + sys.Sequencer.ReorgTriggered(stop.storeTip, rebuildAttempts) + + persistedTip := restartFollower(t, sys, stop) + + sys.Follower.InSync(sys.Sequencer, safety.LocalUnsafe, resyncAttempts) + lastBlock := w.driveUntilBlock(sys.Sequencer.BlockRefByLabel(eth.Unsafe).Number + postRestartTrafficBlocks) + sys.Follower.VerifyHistoricalProofsMatch(sys.Sequencer, func(opts *dsl.HistoricalProofsOpts) { + opts.Accounts = w.accounts() + opts.FirstBlock = persistedTip.Number + opts.LastBlock = lastBlock + }) + }) + } +} diff --git a/op-devstack/dsl/eoa.go b/op-devstack/dsl/eoa.go index 13ba617abef..598c2315969 100644 --- a/op-devstack/dsl/eoa.go +++ b/op-devstack/dsl/eoa.go @@ -22,6 +22,8 @@ import ( "github.com/ethereum-optimism/optimism/op-service/txplan" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/core/vm" + "github.com/ethereum/go-ethereum/core/vm/program" "github.com/ethereum/go-ethereum/crypto" ) @@ -252,6 +254,28 @@ func (u *EOA) DeployEventLogger() common.Address { return eventLoggerAddress } +// PlanBlockRecorderDeployment plans the deployment of a contract that writes the block number to +// storage slot 0 and the block timestamp to slot 1 on every call, so every call to it changes its +// storage. The transaction is not sent. +func (u *EOA) PlanBlockRecorderDeployment() *txplan.PlannedTx { + runtimeCode := program.New(). + Op(vm.NUMBER).Push(0).Op(vm.SSTORE). + Op(vm.TIMESTAMP).Push(1).Op(vm.SSTORE). + Bytes() + initCode := program.New().ReturnViaCodeCopy(runtimeCode).Bytes() + return txplan.NewPlannedTx(u.Plan(), txplan.WithData(initCode)) +} + +// DeployBlockRecorder sends the deployment PlanBlockRecorderDeployment plans and returns the +// contract's address. +func (u *EOA) DeployBlockRecorder() common.Address { + tx := u.PlanBlockRecorderDeployment() + res, err := tx.Included.Eval(u.ctx) + u.t.Require().NoError(err, "failed to deploy block recorder") + u.log.Info("deployed block recorder", "chainID", tx.ChainID.Value(), "address", res.ContractAddress) + return res.ContractAddress +} + func (u *EOA) DeployWETH() common.Address { // Use the e2e bindings which contain the WETH bytecode tx := txplan.NewPlannedTx(u.Plan(), txplan.WithData(common.FromHex(e2eBindings.WETHBin))) diff --git a/op-devstack/dsl/l2_el.go b/op-devstack/dsl/l2_el.go index a940ac0fb06..afaf4e88070 100644 --- a/op-devstack/dsl/l2_el.go +++ b/op-devstack/dsl/l2_el.go @@ -17,6 +17,7 @@ import ( "github.com/ethereum-optimism/optimism/op-service/bigs" "github.com/ethereum-optimism/optimism/op-service/clock" "github.com/ethereum-optimism/optimism/op-service/eth" + "github.com/ethereum-optimism/optimism/op-service/logpipe" "github.com/ethereum-optimism/optimism/op-service/retry" "github.com/ethereum-optimism/optimism/op-service/txplan" @@ -525,12 +526,31 @@ func (el *L2ELNode) Stop() { lifecycle.Stop() } +// StopUngracefully ends the node's process without a shutdown, the way a crash would, so state +// it held only in memory is lost. Start brings the node back. +func (el *L2ELNode) StopUngracefully() { + el.log.Info("Stopping ungracefully", "name", el.inner.Name()) + lifecycle, ok := el.inner.(stack.UngracefulLifecycle) + el.require.Truef(ok, "L2EL node %s cannot be stopped ungracefully", el.inner.Name()) + lifecycle.StopUngracefully() +} + func (el *L2ELNode) Start() { lifecycle, ok := el.inner.(stack.Lifecycle) el.require.Truef(ok, "L2EL node %s is not lifecycle-controllable", el.inner.Name()) lifecycle.Start() } +// RecordLogLines returns a record of every log line this node's process writes from now on, +// across restarts, whose message contains substring. It fails for a node whose log the devstack +// does not expose, such as an in-process op-geth. +func (el *L2ELNode) RecordLogLines(substring string) *logpipe.LogRecord { + recorder, ok := el.inner.(stack.LogRecorder) + el.require.Truef(ok, "L2EL node %s does not expose its log", el.inner.Name()) + el.log.Info("Recording log lines", "name", el.inner.Name(), "substring", substring) + return recorder.RecordLogLines(substring) +} + func (el *L2ELNode) PeerWith(peer *L2ELNode) { sysgo.ConnectP2P(el.ctx, el.require, el.inner.L2EthClient().RPC(), peer.inner.L2EthClient().RPC()) } diff --git a/op-devstack/dsl/l2_el_proofs.go b/op-devstack/dsl/l2_el_proofs.go new file mode 100644 index 00000000000..047b9949f21 --- /dev/null +++ b/op-devstack/dsl/l2_el_proofs.go @@ -0,0 +1,383 @@ +package dsl + +import ( + "bytes" + "context" + "errors" + "fmt" + "time" + + "github.com/ethereum-optimism/optimism/op-core/predeploys" + "github.com/ethereum-optimism/optimism/op-e2e/e2eutils/wait" + "github.com/ethereum-optimism/optimism/op-service/eth" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/ethdb/memorydb" + "github.com/ethereum/go-ethereum/rlp" + gethrpc "github.com/ethereum/go-ethereum/rpc" + "github.com/ethereum/go-ethereum/trie" +) + +// DefaultProofsCatchUpTimeout bounds how long a proofs-history store may lag behind the chain +// before a proof verification fails. +const DefaultProofsCatchUpTimeout = 30 * time.Second + +// ProofsSyncStatus is the window of blocks an op-reth proofs-history store serves, as reported by +// debug_proofsSyncStatus. Both bounds are nil until the store has indexed its first block. +type ProofsSyncStatus struct { + Earliest *uint64 `json:"earliest"` + Latest *uint64 `json:"latest"` +} + +func (s ProofsSyncStatus) String() string { + if s.Earliest == nil || s.Latest == nil { + return "uninitialized" + } + return fmt.Sprintf("[%d, %d]", *s.Earliest, *s.Latest) +} + +func (s ProofsSyncStatus) serves(block uint64) bool { + return s.Earliest != nil && s.Latest != nil && *s.Earliest <= block && block <= *s.Latest +} + +// ProofAccount names an account to prove, with the storage slots whose proofs are requested. +type ProofAccount struct { + Address common.Address + StorageKeys []common.Hash +} + +// HistoricalProofsOpts configures VerifyHistoricalProofsMatch. +type HistoricalProofsOpts struct { + // Accounts are proven at every sampled block. L2ToL1MessagePasser is always proven and need + // not be listed. + Accounts []ProofAccount + // CatchUpTimeout bounds how long the proofs store and the reference node may lag behind a + // sampled block before the sweep fails. + CatchUpTimeout time.Duration + // FirstBlock and LastBlock bound the sweep. Zero selects the matching edge of the proofs + // window reported by debug_proofsSyncStatus when the sweep starts. A block the store prunes + // while the sweep runs is skipped when FirstBlock is zero and fails the sweep otherwise. + FirstBlock uint64 + LastBlock uint64 + // MaxSampledBlocks bounds how many blocks of the range are compared; zero compares every + // block. The first and last block of the range are always compared, so at least two blocks + // of a multi-block range are. + MaxSampledBlocks int +} + +// ProofsSyncStatus reports the window of blocks this node's proofs-history store serves. +func (el *L2ELNode) ProofsSyncStatus() (ProofsSyncStatus, error) { + ctx, cancel := context.WithTimeout(el.ctx, DefaultTimeout) + defer cancel() + var status ProofsSyncStatus + err := el.inner.L2EthClient().RPC().CallContext(ctx, &status, "debug_proofsSyncStatus") + return status, err +} + +// WaitForProofsStoreBlock waits until this node's proofs-history store serves target, so the RPCs +// answered from the store (eth_getProof, debug_executePayload, debug_executionWitness) can be +// called for it. The store indexes blocks asynchronously, so the chain head leads the store. +func (el *L2ELNode) WaitForProofsStoreBlock(target uint64, timeout time.Duration) ProofsSyncStatus { + ctx, cancel := context.WithTimeout(el.ctx, timeout) + defer cancel() + logger := el.log.With("name", el.inner.Name(), "chain", el.ChainID(), "target", target, "timeout", timeout) + logger.Info("Waiting for proofs store to index block") + + var lastStatus ProofsSyncStatus + var lastErr error + err := wait.For(ctx, 200*time.Millisecond, func() (bool, error) { + status, err := el.ProofsSyncStatus() + if err != nil { + lastErr = err + logger.Warn("debug_proofsSyncStatus failed; will retry", "err", err) + return false, nil + } + lastStatus, lastErr = status, nil + if status.Earliest != nil && status.Latest != nil && *status.Latest >= target { + logger.Info("Proofs store indexed block", "window", status) + return true, nil + } + logger.Info("Proofs store behind target", "window", status) + return false, nil + }) + el.require.NoErrorf(err, "proofs store on %s did not index block %d within %s; last window %s; last error: %v", + el.inner.Name(), target, timeout, lastStatus, lastErr) + return lastStatus +} + +// VerifyHistoricalProofsMatch sweeps the blocks this node's proofs-history store serves and, at +// every sampled block, requires eth_getProof for each configured account to verify against the +// block's state root and to equal the proof served by reference. The L2ToL1MessagePasser proof is +// always included, and from Isthmus on its storage root must equal the header's withdrawalsRoot. +// +// reference must answer eth_getProof from its own historical state rather than from a +// proofs-history store, since two stores can agree on a wrong proof. +func (el *L2ELNode) VerifyHistoricalProofsMatch(reference *L2ELNode, opts ...func(*HistoricalProofsOpts)) { + cfg := HistoricalProofsOpts{CatchUpTimeout: DefaultProofsCatchUpTimeout} + for _, opt := range opts { + opt(&cfg) + } + logger := el.log.With("name", el.inner.Name(), "chain", el.ChainID(), "reference", reference.inner.Name()) + + el.requireIndependentProofReference(reference) + + window := el.WaitForProofsStoreBlock(cfg.LastBlock, cfg.CatchUpTimeout) + first, last := cfg.FirstBlock, cfg.LastBlock + if first == 0 { + first = *window.Earliest + } + if last == 0 { + last = *window.Latest + } + el.require.LessOrEqualf(first, last, "historical proofs sweep range [%d, %d] is empty; proofs window %s", first, last, window) + blocks := sampleBlockNumbers(first, last, cfg.MaxSampledBlocks) + accounts := append([]ProofAccount{{Address: predeploys.L2ToL1MessagePasserAddr}}, cfg.Accounts...) + + logger.Info("Sweeping historical proofs", "window", window, "first", first, "last", last, + "blocks", len(blocks), "accounts", len(accounts)) + pruned := 0 + for _, block := range blocks { + window, err := el.ProofsSyncStatus() + el.require.NoErrorf(err, "debug_proofsSyncStatus failed on %s", el.inner.Name()) + if cfg.FirstBlock == 0 && window.Earliest != nil && block < *window.Earliest { + logger.Info("Block left the proofs window during the sweep; skipping", "block", block, "window", window) + pruned++ + continue + } + el.verifyProofsMatchAt(reference, block, window, accounts, cfg.CatchUpTimeout) + } + logger.Info("Historical proofs match the reference", "first", first, "last", last, + "blocks", len(blocks)-pruned, "pruned_during_sweep", pruned, "accounts", len(accounts)) +} + +// VerifyProofsMatchAt verifies the proofs of the given accounts at one block the same way +// VerifyHistoricalProofsMatch does, after waiting up to DefaultProofsCatchUpTimeout for the proofs +// store to serve the block. Only the listed accounts are proven, and reference is not checked for +// independence. +func (el *L2ELNode) VerifyProofsMatchAt(reference *L2ELNode, block uint64, accounts ...ProofAccount) { + window := el.WaitForProofsStoreBlock(block, DefaultProofsCatchUpTimeout) + el.verifyProofsMatchAt(reference, block, window, accounts, DefaultProofsCatchUpTimeout) +} + +func (el *L2ELNode) verifyProofsMatchAt(reference *L2ELNode, block uint64, window ProofsSyncStatus, accounts []ProofAccount, catchUpTimeout time.Duration) { + el.require.Truef(window.serves(block), "proofs store on %s does not serve block %d: window %s", el.inner.Name(), block, window) + + header := el.awaitHeaderByNumber(block, catchUpTimeout) + referenceHeader := reference.awaitHeaderByNumber(block, catchUpTimeout) + el.require.Equalf(referenceHeader.Hash(), header.Hash(), "block %d: %s and reference %s disagree on the block hash", + block, el.inner.Name(), reference.inner.Name()) + + for _, account := range accounts { + el.verifyAccountProofMatchesAt(reference, header, account, catchUpTimeout) + } + el.log.Info("Historical proofs verified", "name", el.inner.Name(), "block", block, "accounts", len(accounts)) +} + +func (el *L2ELNode) verifyAccountProofMatchesAt(reference *L2ELNode, header eth.BlockInfo, account ProofAccount, catchUpTimeout time.Duration) { + block := header.NumberU64() + proof := el.awaitProof(header, account, catchUpTimeout) + referenceProof := reference.awaitProof(header, account, catchUpTimeout) + NormalizeProofResponse(proof) + NormalizeProofResponse(referenceProof) + + el.require.NoErrorf(VerifyAccountProof(proof, header.Root()), + "block %d account %s: eth_getProof from %s does not verify against state root %s", + block, account.Address, el.inner.Name(), header.Root()) + el.require.NoErrorf(VerifyAccountProof(referenceProof, header.Root()), + "block %d account %s: eth_getProof from reference %s does not verify against state root %s", + block, account.Address, reference.inner.Name(), header.Root()) + el.require.Equalf(referenceProof, proof, "block %d account %s: eth_getProof from %s differs from reference %s", + block, account.Address, el.inner.Name(), reference.inner.Name()) + + // Isthmus made withdrawalsRoot the message passer's storage root. Before that the header + // carries the hash of an empty withdrawals list, which says nothing about the message passer. + if account.Address == predeploys.L2ToL1MessagePasserAddr && header.WithdrawalsRoot() != nil && *header.WithdrawalsRoot() != types.EmptyWithdrawalsHash { + el.require.Equalf(*header.WithdrawalsRoot(), proof.StorageHash, + "block %d: L2ToL1MessagePasser storage root from %s does not equal the header's withdrawalsRoot", block, el.inner.Name()) + } + el.log.Debug("Historical proof matches the reference", "name", el.inner.Name(), "block", block, + "account", account.Address, "storage_keys", len(account.StorageKeys)) +} + +// requireIndependentProofReference fails unless reference lacks debug_proofsSyncStatus, which +// op-reth registers only on a proofs-history node and op-geth never does. +func (el *L2ELNode) requireIndependentProofReference(reference *L2ELNode) { + status, err := reference.ProofsSyncStatus() + var rpcErr gethrpc.Error + el.require.Truef(errors.As(err, &rpcErr) && rpcErr.ErrorCode() == int(eth.MethodNotFound), + "reference %s must serve eth_getProof from its own historical state, not a proofs-history store; debug_proofsSyncStatus returned window %s, err: %v", + reference.inner.Name(), status, err) +} + +func (el *L2ELNode) awaitHeaderByNumber(number uint64, timeout time.Duration) eth.BlockInfo { + ctx, cancel := context.WithTimeout(el.ctx, timeout) + defer cancel() + var header eth.BlockInfo + var lastErr error + err := wait.For(ctx, 200*time.Millisecond, func() (bool, error) { + info, err := el.inner.L2EthClient().InfoByNumber(ctx, number) + if err != nil { + lastErr = err + el.log.Debug("Header lookup failed; will retry", "name", el.inner.Name(), "block", number, "err", err) + return false, nil + } + header = info + return true, nil + }) + el.require.NoErrorf(err, "%s did not serve the header of block %d within %s; last error: %v", el.inner.Name(), number, timeout, lastErr) + return header +} + +// awaitProof fetches eth_getProof by block hash, so a block that leaves the canonical chain while +// the sweep runs fails the lookup rather than returning the proof of its replacement. +func (el *L2ELNode) awaitProof(header eth.BlockInfo, account ProofAccount, timeout time.Duration) *eth.AccountResult { + ctx, cancel := context.WithTimeout(el.ctx, timeout) + defer cancel() + // A nil key list encodes as JSON null, which op-reth rejects where op-geth accepts it. + storageKeys := account.StorageKeys + if storageKeys == nil { + storageKeys = []common.Hash{} + } + var proof *eth.AccountResult + var lastErr error + err := wait.For(ctx, 200*time.Millisecond, func() (bool, error) { + result, err := el.inner.L2EthClient().GetProof(ctx, account.Address, storageKeys, header.Hash().String()) + if err != nil { + lastErr = err + el.log.Warn("eth_getProof failed; will retry", "name", el.inner.Name(), "block", header.NumberU64(), + "account", account.Address, "err", err) + return false, nil + } + proof = result + return true, nil + }) + el.require.NoErrorf(err, "%s did not serve eth_getProof for account %s at block %d within %s; last error: %v", + el.inner.Name(), account.Address, header.NumberU64(), timeout, lastErr) + return proof +} + +// sampleBlockNumbers returns evenly spaced block numbers in [first, last], always including both +// edges: every block when maxSamples is non-positive or covers the range, otherwise +// max(maxSamples, 2) of them. +func sampleBlockNumbers(first, last uint64, maxSamples int) []uint64 { + count := last - first + 1 + if maxSamples <= 0 || uint64(maxSamples) >= count { + blocks := make([]uint64, 0, count) + for block := first; block <= last; block++ { + blocks = append(blocks, block) + } + return blocks + } + if maxSamples < 2 { + maxSamples = 2 + } + blocks := make([]uint64, 0, maxSamples) + for i := range maxSamples { + blocks = append(blocks, first+(count-1)*uint64(i)/uint64(maxSamples-1)) + } + return blocks +} + +// NormalizeProofResponse rewrites an eth_getProof response into one canonical encoding so proofs +// from different clients compare equal: an empty storage proof, which reth encodes as ["0x80"], +// becomes empty; a zero code hash becomes keccak256("") and a zero storage hash becomes the empty +// trie root, which is how reth encodes both; and an empty storage proof list becomes nil. +func NormalizeProofResponse(res *eth.AccountResult) { + for i := range res.StorageProof { + if len(res.StorageProof[i].Proof) == 1 && bytes.Equal(res.StorageProof[i].Proof[0], []byte{0x80}) { + res.StorageProof[i].Proof = []hexutil.Bytes{} + } + } + if len(res.StorageProof) == 0 { + res.StorageProof = nil + } + if res.CodeHash == (common.Hash{}) { + res.CodeHash = crypto.Keccak256Hash(nil) + } + if res.StorageHash == (common.Hash{}) { + res.StorageHash = types.EmptyRootHash + } +} + +// VerifyAccountProof verifies an eth_getProof response against a state root. Unlike +// eth.AccountResult.Verify it accepts an empty storage trie, whose proofs carry no nodes, and a +// non-existence proof for an account the response reports as empty. +func VerifyAccountProof(res *eth.AccountResult, stateRoot common.Hash) error { + if res.StorageHash == types.EmptyRootHash { + for i, entry := range res.StorageProof { + if entry.Value.ToInt().Sign() != 0 { + return fmt.Errorf("storage value %d at key %s is nonzero but the storage trie is empty", i, entry.Key.String()) + } + } + } else { + for i, entry := range res.StorageProof { + db := memorydb.New() + for j, encodedNode := range entry.Proof { + nodeKey := encodedNode + if len(encodedNode) >= 32 { // small MPT nodes are not hashed + nodeKey = crypto.Keccak256(encodedNode) + } + if err := db.Put(nodeKey, encodedNode); err != nil { + return fmt.Errorf("failed to load storage proof node %d of storage value %d into mem db: %w", j, i, err) + } + } + path := crypto.Keccak256(entry.Key) + val, err := trie.VerifyProof(res.StorageHash, path, db) + if err != nil { + return fmt.Errorf("failed to verify storage value %d with key %s (path %x) in storage trie %s: %w", i, entry.Key.String(), path, res.StorageHash, err) + } + if val == nil && entry.Value.ToInt().Cmp(common.Big0) == 0 { // empty storage is zero by default + continue + } + comparison, err := rlp.EncodeToBytes(entry.Value.ToInt().Bytes()) + if err != nil { + return fmt.Errorf("failed to encode storage value %d with key %s (path %x) in storage trie %s: %w", i, entry.Key.String(), path, res.StorageHash, err) + } + if !bytes.Equal(val, comparison) { + return fmt.Errorf("value %d in storage proof does not match proven value at key %s (path %x)", i, entry.Key.String(), path) + } + } + } + + accountClaimed := []any{uint64(res.Nonce), res.Balance.ToInt().Bytes(), res.StorageHash, res.CodeHash} + accountClaimedValue, err := rlp.EncodeToBytes(accountClaimed) + if err != nil { + return fmt.Errorf("failed to encode account from retrieved values: %w", err) + } + + db := memorydb.New() + for i, encodedNode := range res.AccountProof { + nodeKey := encodedNode + if len(encodedNode) >= 32 { // small MPT nodes are not hashed + nodeKey = crypto.Keccak256(encodedNode) + } + if err := db.Put(nodeKey, encodedNode); err != nil { + return fmt.Errorf("failed to load account proof node %d into mem db: %w", i, err) + } + } + path := crypto.Keccak256(res.Address[:]) + accountProofValue, err := trie.VerifyProof(stateRoot, path, db) + if err != nil { + return fmt.Errorf("failed to verify account value with key %s (path %x) in account trie %s: %w", res.Address, path, stateRoot, err) + } + + // A non-existence proof is only valid when the response reports an empty account. + if len(accountProofValue) == 0 { + isEmpty := res.Nonce == 0 && + res.Balance.ToInt().Sign() == 0 && + (res.StorageHash == types.EmptyRootHash || res.StorageHash == common.Hash{}) && + (res.CodeHash == crypto.Keccak256Hash(nil) || res.CodeHash == common.Hash{}) + if isEmpty { + return nil + } + } + + if !bytes.Equal(accountClaimedValue, accountProofValue) { + return fmt.Errorf("account proof does not match the claimed account values:\n"+ + " claimed: %x\n"+ + " proof: %x", accountClaimedValue, accountProofValue) + } + return nil +} diff --git a/op-devstack/dsl/l2_el_proofs_test.go b/op-devstack/dsl/l2_el_proofs_test.go new file mode 100644 index 00000000000..7adf8dddd9f --- /dev/null +++ b/op-devstack/dsl/l2_el_proofs_test.go @@ -0,0 +1,279 @@ +package dsl + +import ( + "math/big" + "testing" + + "github.com/ethereum-optimism/optimism/op-service/eth" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/rlp" + "github.com/ethereum/go-ethereum/trie" + "github.com/ethereum/go-ethereum/trie/trienode" + "github.com/ethereum/go-ethereum/triedb" + "github.com/stretchr/testify/require" +) + +// provenAccount is one account of a synthetic state trie. Zero-valued slots are left out of the +// storage trie, as a client would. +type provenAccount struct { + address common.Address + balance *big.Int + nonce uint64 + slots map[common.Hash]*big.Int +} + +var ( + alice = provenAccount{ + address: common.HexToAddress("0xa11ce"), + balance: big.NewInt(1_000), + nonce: 3, + slots: map[common.Hash]*big.Int{ + common.HexToHash("0x0"): big.NewInt(42), + common.HexToHash("0x1"): big.NewInt(0), + }, + } + bob = provenAccount{ + address: common.HexToAddress("0xb0b"), + balance: big.NewInt(7), + nonce: 1, + } + carol = provenAccount{ + address: common.HexToAddress("0xca201"), + balance: big.NewInt(99), + nonce: 0, + slots: map[common.Hash]*big.Int{common.HexToHash("0x5"): big.NewInt(5)}, + } + nobody = provenAccount{address: common.HexToAddress("0x1234"), balance: big.NewInt(0)} +) + +func newTestTrie() *trie.Trie { + return trie.NewEmpty(triedb.NewDatabase(rawdb.NewMemoryDatabase(), nil)) +} + +func proveKey(t *testing.T, tr *trie.Trie, key []byte) []hexutil.Bytes { + var nodes trienode.ProofList + require.NoError(t, tr.Prove(crypto.Keccak256(key), &nodes)) + proof := make([]hexutil.Bytes, len(nodes)) + for i, node := range nodes { + proof[i] = hexutil.Bytes(node) + } + return proof +} + +func storageTrieOf(t *testing.T, account provenAccount) *trie.Trie { + storageTrie := newTestTrie() + for slot, value := range account.slots { + if value.Sign() == 0 { + continue + } + encoded, err := rlp.EncodeToBytes(value.Bytes()) + require.NoError(t, err) + storageTrie.MustUpdate(crypto.Keccak256(slot.Bytes()), encoded) + } + return storageTrie +} + +// stateTrieOf builds a state trie holding every account in accounts. +func stateTrieOf(t *testing.T, accounts ...provenAccount) *trie.Trie { + stateTrie := newTestTrie() + for _, account := range accounts { + encoded, err := rlp.EncodeToBytes([]any{account.nonce, account.balance.Bytes(), storageTrieOf(t, account).Hash(), crypto.Keccak256Hash(nil)}) + require.NoError(t, err) + stateTrie.MustUpdate(crypto.Keccak256(account.address.Bytes()), encoded) + } + return stateTrie +} + +// proofOf returns what eth_getProof would answer for account against stateTrie, whether or not +// the trie holds the account. +func proofOf(t *testing.T, stateTrie *trie.Trie, account provenAccount) *eth.AccountResult { + storageTrie := storageTrieOf(t, account) + result := ð.AccountResult{ + AccountProof: proveKey(t, stateTrie, account.address.Bytes()), + Address: account.address, + Balance: (*hexutil.Big)(account.balance), + CodeHash: crypto.Keccak256Hash(nil), + Nonce: hexutil.Uint64(account.nonce), + StorageHash: storageTrie.Hash(), + } + for slot, value := range account.slots { + result.StorageProof = append(result.StorageProof, eth.StorageProofEntry{ + Key: slot.Bytes(), + Value: (hexutil.Big)(*value), + Proof: proveKey(t, storageTrie, slot.Bytes()), + }) + } + return result +} + +func TestVerifyAccountProof(t *testing.T) { + stateTrie := stateTrieOf(t, alice, bob, carol) + stateRoot := stateTrie.Hash() + + for _, tc := range []struct { + name string + proof func() *eth.AccountResult + root common.Hash + wantErr string + }{ + {name: "account with storage verifies", proof: func() *eth.AccountResult { return proofOf(t, stateTrie, alice) }, root: stateRoot}, + {name: "account without storage verifies", proof: func() *eth.AccountResult { return proofOf(t, stateTrie, bob) }, root: stateRoot}, + {name: "absent account reported empty verifies", proof: func() *eth.AccountResult { return proofOf(t, stateTrie, nobody) }, root: stateRoot}, + { + name: "zero slot of an empty storage trie verifies", + proof: func() *eth.AccountResult { + proof := proofOf(t, stateTrie, bob) + proof.StorageProof = []eth.StorageProofEntry{{Key: common.HexToHash("0x0").Bytes(), Proof: []hexutil.Bytes{}}} + return proof + }, + root: stateRoot, + }, + { + name: "tampered balance fails", + proof: func() *eth.AccountResult { + proof := proofOf(t, stateTrie, alice) + proof.Balance = (*hexutil.Big)(big.NewInt(1_001)) + return proof + }, + root: stateRoot, + wantErr: "account proof does not match the claimed account values", + }, + { + name: "tampered storage value fails", + proof: func() *eth.AccountResult { + proof := proofOf(t, stateTrie, carol) + proof.StorageProof[0].Value = (hexutil.Big)(*big.NewInt(6)) + return proof + }, + root: stateRoot, + wantErr: "does not match proven value", + }, + { + name: "nonzero slot of an empty storage trie fails", + proof: func() *eth.AccountResult { + proof := proofOf(t, stateTrie, bob) + proof.StorageProof = []eth.StorageProofEntry{{Key: common.HexToHash("0x0").Bytes(), Value: (hexutil.Big)(*big.NewInt(42)), Proof: []hexutil.Bytes{}}} + return proof + }, + root: stateRoot, + wantErr: "is nonzero but the storage trie is empty", + }, + { + name: "absent account claiming a balance fails", + proof: func() *eth.AccountResult { + proof := proofOf(t, stateTrie, nobody) + proof.Balance = (*hexutil.Big)(big.NewInt(1)) + return proof + }, + root: stateRoot, + wantErr: "account proof does not match the claimed account values", + }, + { + name: "wrong state root fails", + proof: func() *eth.AccountResult { return proofOf(t, stateTrie, alice) }, + root: common.HexToHash("0xdead"), + wantErr: "failed to verify account value", + }, + } { + t.Run(tc.name, func(t *testing.T) { + err := VerifyAccountProof(tc.proof(), tc.root) + if tc.wantErr == "" { + require.NoError(t, err) + return + } + require.ErrorContains(t, err, tc.wantErr) + }) + } +} + +func TestNormalizeProofResponse(t *testing.T) { + emptyCodeHash := crypto.Keccak256Hash(nil) + someHash := common.HexToHash("0xabc") + key := common.HexToHash("0x1").Bytes() + + for _, tc := range []struct { + name string + in eth.AccountResult + want eth.AccountResult + }{ + { + name: "reth empty storage proof becomes empty", + in: eth.AccountResult{CodeHash: someHash, StorageHash: someHash, StorageProof: []eth.StorageProofEntry{{Key: key, Proof: []hexutil.Bytes{{0x80}}}}}, + want: eth.AccountResult{CodeHash: someHash, StorageHash: someHash, StorageProof: []eth.StorageProofEntry{{Key: key, Proof: []hexutil.Bytes{}}}}, + }, + { + name: "zero code hash becomes the empty code hash", + in: eth.AccountResult{StorageHash: someHash}, + want: eth.AccountResult{CodeHash: emptyCodeHash, StorageHash: someHash}, + }, + { + name: "zero storage hash becomes the empty trie root", + in: eth.AccountResult{CodeHash: someHash}, + want: eth.AccountResult{CodeHash: someHash, StorageHash: types.EmptyRootHash}, + }, + { + name: "empty storage proof list becomes nil", + in: eth.AccountResult{CodeHash: someHash, StorageHash: someHash, StorageProof: []eth.StorageProofEntry{}}, + want: eth.AccountResult{CodeHash: someHash, StorageHash: someHash}, + }, + { + name: "non-empty values are kept", + in: eth.AccountResult{CodeHash: someHash, StorageHash: someHash, StorageProof: []eth.StorageProofEntry{{Key: key, Proof: []hexutil.Bytes{{0x80}, {0x81}}}}}, + want: eth.AccountResult{CodeHash: someHash, StorageHash: someHash, StorageProof: []eth.StorageProofEntry{{Key: key, Proof: []hexutil.Bytes{{0x80}, {0x81}}}}}, + }, + } { + t.Run(tc.name, func(t *testing.T) { + got := tc.in + NormalizeProofResponse(&got) + require.Equal(t, tc.want, got) + }) + } +} + +func TestProofsSyncStatusServes(t *testing.T) { + window := func(earliest, latest uint64) ProofsSyncStatus { + return ProofsSyncStatus{Earliest: &earliest, Latest: &latest} + } + for _, tc := range []struct { + name string + status ProofsSyncStatus + block uint64 + want bool + }{ + {name: "uninitialized store serves nothing", status: ProofsSyncStatus{}, block: 0, want: false}, + {name: "earliest edge is served", status: window(5, 9), block: 5, want: true}, + {name: "latest edge is served", status: window(5, 9), block: 9, want: true}, + {name: "block below earliest is not served", status: window(5, 9), block: 4, want: false}, + {name: "block above latest is not served", status: window(5, 9), block: 10, want: false}, + } { + t.Run(tc.name, func(t *testing.T) { + require.Equal(t, tc.want, tc.status.serves(tc.block)) + }) + } +} + +func TestSampleBlockNumbers(t *testing.T) { + for _, tc := range []struct { + name string + first uint64 + last uint64 + maxSamples int + want []uint64 + }{ + {name: "zero bound returns every block", first: 0, last: 4, maxSamples: 0, want: []uint64{0, 1, 2, 3, 4}}, + {name: "single block", first: 5, last: 5, maxSamples: 0, want: []uint64{5}}, + {name: "bound above count returns every block", first: 0, last: 4, maxSamples: 20, want: []uint64{0, 1, 2, 3, 4}}, + {name: "bound of two keeps the edges", first: 0, last: 9, maxSamples: 2, want: []uint64{0, 9}}, + {name: "bound of one still keeps both edges", first: 0, last: 9, maxSamples: 1, want: []uint64{0, 9}}, + {name: "samples are evenly spaced", first: 0, last: 100, maxSamples: 5, want: []uint64{0, 25, 50, 75, 100}}, + {name: "uneven spacing rounds down and keeps the last block", first: 10, last: 13, maxSamples: 3, want: []uint64{10, 11, 13}}, + } { + t.Run(tc.name, func(t *testing.T) { + require.Equal(t, tc.want, sampleBlockNumbers(tc.first, tc.last, tc.maxSamples)) + }) + } +} diff --git a/op-devstack/dsl/sequencer.go b/op-devstack/dsl/sequencer.go index 857c8712e7c..20315ddc545 100644 --- a/op-devstack/dsl/sequencer.go +++ b/op-devstack/dsl/sequencer.go @@ -7,6 +7,7 @@ import ( "github.com/ethereum-optimism/optimism/op-devstack/devtest" "github.com/ethereum-optimism/optimism/op-devstack/stack" + "github.com/ethereum-optimism/optimism/op-service/clock" "github.com/ethereum-optimism/optimism/op-service/eth" "github.com/ethereum-optimism/optimism/op-service/txplan" "github.com/ethereum-optimism/optimism/op-test-sequencer/sequencer/seqtypes" @@ -109,9 +110,29 @@ func (s *TestSequencer) SequenceBlockWithTxs(t devtest.T, chainID eth.ChainID, p require.NoError(t, ca.Sign(ctx)) require.NoError(t, ca.Commit(ctx)) - // Publish is optional - it broadcasts via P2P which may not be enabled in tests. + // Publish is optional: it broadcasts the block over P2P, which may not be enabled in tests. // The block is already committed and canonical at this point. - _ = ca.Publish(ctx) // ignore publish errors + if err := ca.Publish(ctx); err != nil { + s.log.Warn("Block committed but not published; its job stays sealed, so no further block can be sequenced on this chain", + "chain", chainID, "err", err) + return + } + // Every step of the job is done, so Next only closes it out, which lets the next block on + // this chain start. + require.NoError(t, ca.Next(ctx), "failed to close out the sequenced block") +} + +// SequenceBlockWithTxsWhenDue is SequenceBlockWithTxs for a chain whose head stands still: it +// waits until wall-clock time reaches the new block's timestamp, which its height fixes, before +// building. op-node rejects a gossiped payload more than 5s ahead of wall-clock time, on publish +// as well as on receipt, so a block built early would never reach the other nodes. +func (s *TestSequencer) SequenceBlockWithTxsWhenDue(t devtest.T, network *L2Network, parent eth.L2BlockRef, rawTxs [][]byte) { + due := time.Unix(int64(network.TimestampForBlockNum(parent.Number+1)), 0) + if wait := time.Until(due); wait > 0 { + s.log.Info("Waiting for the next block's timestamp before sequencing it", "block", parent.Number+1, "wait", wait) + require.NoError(t, clock.SystemClock.SleepCtx(t.Ctx(), wait)) // nosemgrep: flake-sleep-in-test -- the chain stands still; wall-clock time is the only thing to wait on + } + s.SequenceBlockWithTxs(t, network.ChainID(), parent.Hash, rawTxs) } // SequenceBlockWithPlannedTxs assigns consecutive nonces and includes the planned transactions in one block. diff --git a/op-devstack/presets/rpc_frontends.go b/op-devstack/presets/rpc_frontends.go index b6008086dac..9631aff65eb 100644 --- a/op-devstack/presets/rpc_frontends.go +++ b/op-devstack/presets/rpc_frontends.go @@ -19,6 +19,7 @@ import ( opclient "github.com/ethereum-optimism/optimism/op-service/client" "github.com/ethereum-optimism/optimism/op-service/eth" "github.com/ethereum-optimism/optimism/op-service/locks" + "github.com/ethereum-optimism/optimism/op-service/logpipe" "github.com/ethereum-optimism/optimism/op-service/sources" "github.com/ethereum-optimism/optimism/op-service/testreq" "github.com/ethereum-optimism/optimism/op-sync-tester/synctester" @@ -155,9 +156,15 @@ type l2ELFrontend struct { l2EngineClient *sources.EngineClient lifecycle stack.Lifecycle control stack.ControlledLifecycle + ungraceful stack.UngracefulLifecycle + logRecorder stack.LogRecorder } -var _ stack.L2ELNode = (*l2ELFrontend)(nil) +var ( + _ stack.L2ELNode = (*l2ELFrontend)(nil) + _ stack.LogRecorder = (*l2ELFrontend)(nil) + _ stack.UngracefulLifecycle = (*l2ELFrontend)(nil) +) func newPresetL2ELNode(t devtest.T, name string, chainID eth.ChainID, userRPCCl opclient.RPC, userRPC string, engineRPCCl opclient.RPC, rollupCfg *rollup.Config) *l2ELFrontend { t.Require().NotNil(rollupCfg, "rollup config must be configured") @@ -193,6 +200,16 @@ func (r *l2ELFrontend) Stop() { r.lifecycle.Stop() } +func (r *l2ELFrontend) StopUngracefully() { + r.require().NotNil(r.ungraceful, "L2EL node %s cannot be stopped ungracefully", r.Name()) + r.ungraceful.StopUngracefully() +} + +func (r *l2ELFrontend) RecordLogLines(substring string) *logpipe.LogRecord { + r.require().NotNil(r.logRecorder, "L2EL node %s does not expose its log", r.Name()) + return r.logRecorder.RecordLogLines(substring) +} + func (r *l2ELFrontend) StartControlled(ctx context.Context) error { if r.control == nil { return fmt.Errorf("L2EL node %s is not controllable", r.Name()) diff --git a/op-devstack/presets/sysgo_runtime.go b/op-devstack/presets/sysgo_runtime.go index 3ae12a67ebe..18b7dcac546 100644 --- a/op-devstack/presets/sysgo_runtime.go +++ b/op-devstack/presets/sysgo_runtime.go @@ -66,6 +66,12 @@ func newL2ELFrontend(t devtest.T, name string, chainID eth.ChainID, userRPC stri if control, ok := lifecycle[0].(stack.ControlledLifecycle); ok { l2EL.control = control } + if ungraceful, ok := lifecycle[0].(stack.UngracefulLifecycle); ok { + l2EL.ungraceful = ungraceful + } + if recorder, ok := lifecycle[0].(stack.LogRecorder); ok { + l2EL.logRecorder = recorder + } } return l2EL } diff --git a/op-devstack/stack/lifecycle.go b/op-devstack/stack/lifecycle.go index f26d3b7e605..22278e6444f 100644 --- a/op-devstack/stack/lifecycle.go +++ b/op-devstack/stack/lifecycle.go @@ -12,3 +12,10 @@ type ControlledLifecycle interface { StopControlled(ctx context.Context) error Running() bool } + +// UngracefulLifecycle is implemented by components that can be stopped the way a process crash +// would stop them: no shutdown runs, so state held only in process memory is lost. Writes already +// handed to the OS survive, so this does not model a power loss. +type UngracefulLifecycle interface { + StopUngracefully() +} diff --git a/op-devstack/stack/logs.go b/op-devstack/stack/logs.go new file mode 100644 index 00000000000..6169e2ccb9b --- /dev/null +++ b/op-devstack/stack/logs.go @@ -0,0 +1,10 @@ +package stack + +import "github.com/ethereum-optimism/optimism/op-service/logpipe" + +// LogRecorder is implemented by components whose process log a test can watch. +type LogRecorder interface { + // RecordLogLines returns a record of every log entry the component writes from now on, across + // restarts, whose message contains substring. + RecordLogLines(substring string) *logpipe.LogRecord +} diff --git a/op-devstack/sysgo/l2_cl.go b/op-devstack/sysgo/l2_cl.go index dc7539eb004..7de36189195 100644 --- a/op-devstack/sysgo/l2_cl.go +++ b/op-devstack/sysgo/l2_cl.go @@ -57,6 +57,15 @@ func L2CLFollowSource(source string) L2CLOption { }) } +// L2CLVerifierSyncMode sets the sync mode a verifier runs: CL sync has op-node fill unsafe gaps +// from its peers, EL sync hands the forkchoice target to the EL and lets it sync from its own +// peers. +func L2CLVerifierSyncMode(mode nodeSync.Mode) L2CLOption { + return L2CLOptionFn(func(p devtest.T, _ ComponentTarget, cfg *L2CLConfig) { + cfg.VerifierSyncMode = mode + }) +} + func DefaultL2CLConfig() *L2CLConfig { return &L2CLConfig{ SequencerSyncMode: nodeSync.CLSync, diff --git a/op-devstack/sysgo/l2_cl_test.go b/op-devstack/sysgo/l2_cl_test.go new file mode 100644 index 00000000000..1479ba12049 --- /dev/null +++ b/op-devstack/sysgo/l2_cl_test.go @@ -0,0 +1,17 @@ +package sysgo + +import ( + "testing" + + "github.com/stretchr/testify/require" + + nodeSync "github.com/ethereum-optimism/optimism/op-node/rollup/sync" +) + +func TestL2CLVerifierSyncMode(t *testing.T) { + cfg := &L2CLConfig{SequencerSyncMode: nodeSync.CLSync, VerifierSyncMode: nodeSync.CLSync} + L2CLVerifierSyncMode(nodeSync.ELSync).Apply(nil, ComponentTarget{}, cfg) + + require.Equal(t, nodeSync.ELSync, cfg.VerifierSyncMode) + require.Equal(t, nodeSync.CLSync, cfg.SequencerSyncMode, "the sequencer keeps its own sync mode") +} diff --git a/op-devstack/sysgo/l2_el_opreth.go b/op-devstack/sysgo/l2_el_opreth.go index 959204e81be..9783a5d81bb 100644 --- a/op-devstack/sysgo/l2_el_opreth.go +++ b/op-devstack/sysgo/l2_el_opreth.go @@ -8,6 +8,7 @@ import ( "sync" "github.com/ethereum-optimism/optimism/op-devstack/devtest" + "github.com/ethereum-optimism/optimism/op-devstack/stack" "github.com/ethereum-optimism/optimism/op-service/eth" "github.com/ethereum-optimism/optimism/op-service/log" "github.com/ethereum-optimism/optimism/op-service/logpipe" @@ -27,6 +28,9 @@ type OpRethConfig struct { // runtime otherwise enables proofs-history on every op-reth node; some CLI-superset binaries // reject --proofs-history in the mode under test. DisableProofsHistory bool + // ProofsHistoryWindow is the number of blocks the proofs-history store keeps. Zero keeps the + // mixed runtime's default window. + ProofsHistoryWindow uint64 } // DefaultOpRethConfig returns a zero-valued OpRethConfig that callers can mutate via OpRethOptions. @@ -89,6 +93,22 @@ func OpRethWithoutProofsHistory() OpRethOption { }) } +// OpRethWithProofsHistoryWindow sets the number of blocks the proofs-history store keeps. The mixed +// runtime passes --proofs-history.window itself and op-reth rejects a repeated flag, so the window +// cannot be set through OpRethWithExtraArgs. +func OpRethWithProofsHistoryWindow(blocks uint64) OpRethOption { + return OpRethOptionFn(func(p devtest.T, _ ComponentTarget, cfg *OpRethConfig) { + cfg.ProofsHistoryWindow = blocks + }) +} + +// OpRethWithEthProofWindow lets eth_getProof answer for blocks up to the given distance behind the +// head from reth's own historical state. reth's default window is zero, so a node without proofs +// history rejects every historical proof request unless it gets this option. +func OpRethWithEthProofWindow(blocks uint64) OpRethOption { + return OpRethWithExtraArgs(fmt.Sprintf("--rpc.eth-proof-window=%d", blocks)) +} + // OpRethWithInteropURL wires the op-reth node to the given interop filter HTTP endpoint. // An empty interopURL is a no-op so callers can pass the value unconditionally. func OpRethWithInteropURL(interopURL string) OpRethOption { @@ -123,9 +143,24 @@ type OpReth struct { sub *SubProcess l2MetricsRegistrar L2MetricsRegistrar + + // logRecords outlives the sub-process so a record registered before a restart keeps + // collecting afterwards. + logRecords logpipe.LogRecords } -var _ L2ELNode = (*OpReth)(nil) +var ( + _ L2ELNode = (*OpReth)(nil) + _ stack.LogRecorder = (*OpReth)(nil) + _ stack.UngracefulLifecycle = (*OpReth)(nil) +) + +// RecordLogLines returns a record of every line op-reth writes to stdout or stderr from now on, +// across restarts, whose message, or whose raw text for a line that is not a structured log +// entry, contains substring. Lines below the level the test log keeps are recorded too. +func (n *OpReth) RecordLogLines(substring string) *logpipe.LogRecord { + return n.logRecords.Record(substring) +} func (n *OpReth) Start() { n.mu.Lock() @@ -201,10 +236,12 @@ func (n *OpReth) Start() { e := logpipe.ParseRustStructuredLogs(line) logOut(e) onLogEntry(e) + n.logRecords.Observe(e) }) stdErrLogs := logpipe.LogCallback(func(line []byte) { e := logpipe.ParseRustStructuredLogs(line) logErr(e) + n.logRecords.Observe(e) }) n.sub = NewSubProcess(n.p, stdOutLogs, stdErrLogs) @@ -239,6 +276,22 @@ func (n *OpReth) Stop() { n.sub = nil } +// StopUngracefully ends the op-reth process with SIGKILL, the way a crash would, so whatever it +// had not yet handed to the OS for its datadir is lost. Start brings the node back on the same +// datadir. +func (n *OpReth) StopUngracefully() { + n.mu.Lock() + defer n.mu.Unlock() + if n.sub == nil { + n.p.Logger().Warn("op-reth already stopped") + return + } + n.p.Logger().Info("Stopping op-reth ungracefully", "name", n.name, "chain", n.chainID) + n.clearProxyUpstreams() + n.p.Require().NoError(n.sub.Kill(), "Must kill") + n.sub = nil +} + // Callers must hold n.mu. func (n *OpReth) clearProxyUpstreams() { if n.userProxy != nil { diff --git a/op-devstack/sysgo/l2_el_opreth_test.go b/op-devstack/sysgo/l2_el_opreth_test.go new file mode 100644 index 00000000000..10a27dfe175 --- /dev/null +++ b/op-devstack/sysgo/l2_el_opreth_test.go @@ -0,0 +1,53 @@ +package sysgo + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +func argsWithPrefix(args []string, prefix string) []string { + var matches []string + for _, arg := range args { + if strings.HasPrefix(arg, prefix) { + matches = append(matches, arg) + } + } + return matches +} + +func TestProofsHistoryNodeArgs(t *testing.T) { + for _, tc := range []struct { + name string + window uint64 + wantWindow string + }{ + {name: "zero window uses the runtime default", window: 0, wantWindow: "--proofs-history.window=10000"}, + {name: "explicit window replaces the default", window: 25, wantWindow: "--proofs-history.window=25"}, + } { + t.Run(tc.name, func(t *testing.T) { + args := proofsHistoryNodeArgs(&OpRethConfig{ProofsHistoryWindow: tc.window}, "/proofs") + + require.Contains(t, args, "--proofs-history") + require.Contains(t, args, "--proofs-history.storage-path=/proofs") + require.Equal(t, []string{tc.wantWindow}, argsWithPrefix(args, "--proofs-history.window="), + "op-reth rejects a repeated flag, so the window must appear exactly once") + }) + } +} + +func TestOpRethWithProofsHistoryWindow(t *testing.T) { + cfg := DefaultOpRethConfig() + OpRethWithProofsHistoryWindow(25).Apply(nil, ComponentTarget{}, cfg) + + require.Equal(t, uint64(25), cfg.ProofsHistoryWindow) + require.Empty(t, cfg.ExtraArgs, "the window is a config field, not an extra arg") +} + +func TestOpRethWithEthProofWindow(t *testing.T) { + cfg := DefaultOpRethConfig() + OpRethWithEthProofWindow(1_000_000).Apply(nil, ComponentTarget{}, cfg) + + require.Equal(t, []string{"--rpc.eth-proof-window=1000000"}, cfg.ExtraArgs) +} diff --git a/op-devstack/sysgo/mixed_runtime.go b/op-devstack/sysgo/mixed_runtime.go index bdb6e8b0df6..b25b948954c 100644 --- a/op-devstack/sysgo/mixed_runtime.go +++ b/op-devstack/sysgo/mixed_runtime.go @@ -8,6 +8,7 @@ import ( "os" "os/exec" "path/filepath" + "strconv" "strings" "github.com/ethereum/go-ethereum/common/hexutil" @@ -176,6 +177,9 @@ type MixedSingleChainNodeSpec struct { // by deriving from L1. Used to exercise the derivation/force-build path (FCU-with-attributes) // rather than the consolidation path. Defaults to false (fully peered). IsolateFromL2P2P bool + // L2CLOpts are per-node op-node options, e.g. L2CLVerifierSyncMode for a follower that must + // EL-sync. Ignored for non-op-node CL kinds. + L2CLOpts []L2CLOption // OpRethOpts are per-node op-reth options applied AFTER the shared cfg.OpRethOptions, only to // this node's EL. Use for options that must not reach every node — e.g. a per-node binary // override (OpRethWithBinary) or a flag a single node should receive while its peers reject it. @@ -286,10 +290,10 @@ func NewMixedSingleChainRuntime(t devtest.T, cfg MixedSingleChainPresetConfig) * IsSequencer: spec.IsSequencer, NoDiscovery: true, DependencySet: depSet, - L2CLOptions: []L2CLOption{L2CLOptionFn(func(_ devtest.T, _ ComponentTarget, cfg *L2CLConfig) { + L2CLOptions: append([]L2CLOption{L2CLOptionFn(func(_ devtest.T, _ ComponentTarget, cfg *L2CLConfig) { cfg.SafeDBPath = safeDBPath cfg.VerifierL1Confs = spec.VerifierL1Confs - })}, + })}, spec.L2CLOpts...), }) case MixedL2CLKona: cl = startMixedKonaNode( @@ -390,6 +394,24 @@ func mixedNodeRefs(nodes []mixedSingleChainNode) []MixedSingleChainNodeRefs { return out } +// defaultMixedProofsHistoryWindow is the proofs-history window an op-reth node gets unless +// OpRethWithProofsHistoryWindow overrides it. +const defaultMixedProofsHistoryWindow uint64 = 10_000 + +// proofsHistoryNodeArgs returns the node flags that enable proofs history on the store at +// storagePath. +func proofsHistoryNodeArgs(cfg *OpRethConfig, storagePath string) []string { + window := cfg.ProofsHistoryWindow + if window == 0 { + window = defaultMixedProofsHistoryWindow + } + return []string{ + "--proofs-history", + "--proofs-history.storage-path=" + storagePath, + "--proofs-history.window=" + strconv.FormatUint(window, 10), + } +} + // buildMixedOpRethNode constructs an OpReth node without starting it. func buildMixedOpRethNode( t devtest.T, @@ -488,12 +510,7 @@ func buildMixedOpRethNode( initOut, initErr := exec.Command(execPath, initProofsArgs...).CombinedOutput() t.Require().NoError(initErr, "must init op-reth proof history: %s", string(initOut)) - args = append( - args, - "--proofs-history", - "--proofs-history.window=10000", - "--proofs-history.storage-path="+proofHistoryDir, - ) + args = append(args, proofsHistoryNodeArgs(opRethCfg, proofHistoryDir)...) } args = append(args, opRethCfg.ExtraArgs...) diff --git a/op-devstack/sysgo/subproc.go b/op-devstack/sysgo/subproc.go index f86f582922f..7c248a6aec9 100644 --- a/op-devstack/sysgo/subproc.go +++ b/op-devstack/sysgo/subproc.go @@ -142,6 +142,40 @@ func (sp *SubProcess) stop(ctx context.Context, interrupt bool, interruptWait ti return sp.completeStopLocked(interrupt, sp.waitErr) } +// Kill ends the process with SIGKILL, so it runs none of its shutdown logic, and waits for it to +// exit. It is the unclean counterpart of Stop: whatever the process held only in its own memory +// is lost, while writes it had already handed to the OS survive, so this is a crash, not a +// power loss. +func (sp *SubProcess) Kill() error { + sp.mu.Lock() + defer sp.mu.Unlock() + if sp.cmd == nil { + return nil // already stopped + } + + select { + case <-sp.exited: + // A process that died on its own is a different event from the kill a caller asked for. + sp.p.Logger().Warn("Sub-process exited before it could be killed", "exit", sp.waitErr) + sp.releaseLocked() + return nil + default: + } + + sp.p.Logger().Info("Killing sub-process", "pid", sp.cmd.Process.Pid) + // The process may have exited between the check and the signal, which Kill reports as + // ErrProcessDone; the wait below then returns at once. + if err := sp.cmd.Process.Kill(); err != nil && !errors.Is(err, os.ErrProcessDone) { + return fmt.Errorf("failed to kill sub-process: %w", err) + } + if !waitProcess(context.Background(), sp.exited, controlledKillWait) { + return fmt.Errorf("sub-process did not exit within %s of SIGKILL", controlledKillWait) + } + sp.p.Logger().Info("Sub-process killed", "exit", sp.waitErr) + sp.releaseLocked() + return nil +} + func (sp *SubProcess) completeStopLocked(interrupt bool, waitErr error) error { var exitErr *exec.ExitError if waitErr != nil && !(interrupt && errors.As(waitErr, &exitErr)) { @@ -151,7 +185,12 @@ func (sp *SubProcess) completeStopLocked(interrupt bool, waitErr error) error { } else { sp.p.Logger().Info("Sub-process gracefully exited") } + sp.releaseLocked() + return nil +} +// releaseLocked closes the log pipes and forgets the exited process so Start can run a new one. +func (sp *SubProcess) releaseLocked() { if sp.stdOutProc != nil { _ = sp.stdOutProc.Close() sp.stdOutProc = nil @@ -161,7 +200,6 @@ func (sp *SubProcess) completeStopLocked(interrupt bool, waitErr error) error { sp.stdErrProc = nil } sp.cmd = nil - return nil } func waitProcess(ctx context.Context, exited <-chan struct{}, timeout time.Duration) bool { diff --git a/op-devstack/sysgo/subproc_test.go b/op-devstack/sysgo/subproc_test.go index e0173db840b..c398279989c 100644 --- a/op-devstack/sysgo/subproc_test.go +++ b/op-devstack/sysgo/subproc_test.go @@ -85,6 +85,43 @@ func TestSubProcessConcurrentStop(gt *testing.T) { require.NoError(gt, sp.Stop(false)) } +func newTestSubProcess(gt *testing.T) (*SubProcess, *testlog.CapturingHandler) { + tLog := testlog.Logger(gt, log.LevelInfo) + logger, capt := testlog.CaptureLogger(gt, log.LevelInfo) + p := devtest.NewP(context.Background(), logger, func(bool) { panic("fail") }, func() { panic("skip") }) + gt.Cleanup(p.Close) + logCallback := logpipe.LogCallback(func(line []byte) { + logger.Info(string(line)) + tLog.Info("Sub-process logged message", "line", string(line)) + }) + return NewSubProcess(p, logCallback, logCallback), capt +} + +func TestSubProcessKillEndsProcessThatIgnoresInterrupts(gt *testing.T) { + sp, capt := newTestSubProcess(gt) + require.NoError(gt, sp.Start("/bin/sh", []string{"-c", "trap '' INT TERM; exec /bin/sleep 10000000000"}, []string{})) + + require.NoError(gt, sp.Kill()) + + require.Nil(gt, capt.FindLog(testlog.NewMessageFilter("Sending interrupt")), "Kill must not interrupt first") + require.NotNil(gt, capt.FindLog(testlog.NewMessageFilter("Sub-process killed"))) + + require.NoError(gt, sp.Start("/bin/echo", []string{"restart ok"}, []string{}), "the sub-process must restart after a kill") + require.NoError(gt, sp.Stop(false)) + require.NotNil(gt, capt.FindLog(testlog.NewMessageFilter("restart ok"))) +} + +func TestSubProcessKillOfExitedProcess(gt *testing.T) { + sp, capt := newTestSubProcess(gt) + require.NoError(gt, sp.Start("/bin/echo", []string{"done"}, []string{})) + <-sp.Exited() + + require.NoError(gt, sp.Kill()) + require.NotNil(gt, capt.FindLog(testlog.NewMessageFilter("Sub-process exited before it could be killed"))) + require.Nil(gt, capt.FindLog(testlog.NewMessageFilter("Sub-process killed")), "an exit of its own must not read as a kill") + require.NoError(gt, sp.Kill(), "a second kill is a no-op") +} + // testEcho tests that we can handle a sub-process that completes on its own func testEcho(gt *testing.T, capt *testlog.CapturingHandler, sp *SubProcess) { require.NoError(gt, sp.Start("/bin/echo", []string{"hello world"}, []string{})) diff --git a/op-service/logpipe/pipe.go b/op-service/logpipe/pipe.go index c68e53d21eb..27fbdb7d4a6 100644 --- a/op-service/logpipe/pipe.go +++ b/op-service/logpipe/pipe.go @@ -23,15 +23,32 @@ type StructuredRustLogEntry struct { Target string } +// A line that is not structured JSON, such as a panic hook's output, becomes an entry with this +// message and the raw line in this field. +const ( + unparsedLineField = "line" + unparsedLineMessage = "Invalid JSON" +) + +// UnparsedLine returns the raw text of a line that was not structured JSON, and false for an +// entry that was. +func (e StructuredRustLogEntry) UnparsedLine() (string, bool) { + if e.Message != unparsedLineMessage { + return "", false + } + line, ok := e.Fields[unparsedLineField].(string) + return line, ok +} + func ParseRustStructuredLogs(line []byte) LogEntry { dec := json.NewDecoder(bytes.NewReader(line)) dec.UseNumber() // to preserve number formatting var e rawRustJSONLog if err := dec.Decode(&e); err != nil { return StructuredRustLogEntry{ - Message: "Invalid JSON", + Message: unparsedLineMessage, Level: slog.LevelWarn, - Fields: map[string]any{"line": string(line)}, + Fields: map[string]any{unparsedLineField: string(line)}, } } lvl, err := log.LevelFromString(e.Level) diff --git a/op-service/logpipe/record.go b/op-service/logpipe/record.go new file mode 100644 index 00000000000..7916ee99518 --- /dev/null +++ b/op-service/logpipe/record.go @@ -0,0 +1,80 @@ +package logpipe + +import ( + "strings" + "sync" +) + +// LogRecord keeps the entries of a log stream whose text contains a substring: the message of a +// structured entry, the raw line of an unstructured one. Observing and reading are safe from +// different goroutines, so a test can read a record while the stream is still being written. +type LogRecord struct { + substring string + + mu sync.Mutex + entries []LogEntry +} + +func newLogRecord(substring string) *LogRecord { + return &LogRecord{substring: substring} +} + +// Observe keeps e when its text contains the record's substring. +func (r *LogRecord) Observe(e LogEntry) { + if !strings.Contains(entryText(e), r.substring) { + return + } + r.mu.Lock() + defer r.mu.Unlock() + r.entries = append(r.entries, e) +} + +func entryText(e LogEntry) string { + if structured, ok := e.(StructuredRustLogEntry); ok { + if line, unparsed := structured.UnparsedLine(); unparsed { + return line + } + } + return e.LogMessage() +} + +// Entries returns the kept entries in the order they were observed. +func (r *LogRecord) Entries() []LogEntry { + r.mu.Lock() + defer r.mu.Unlock() + return append([]LogEntry(nil), r.entries...) +} + +// Count is the number of kept entries. +func (r *LogRecord) Count() int { + r.mu.Lock() + defer r.mu.Unlock() + return len(r.entries) +} + +// LogRecords fans one log stream out to every record registered on it. The zero value is ready +// to use. +type LogRecords struct { + mu sync.Mutex + records []*LogRecord +} + +// Record registers a new record for substring and returns it. Entries observed before the call +// are not part of it. +func (s *LogRecords) Record(substring string) *LogRecord { + record := newLogRecord(substring) + s.mu.Lock() + defer s.mu.Unlock() + s.records = append(s.records, record) + return record +} + +// Observe offers e to every registered record. +func (s *LogRecords) Observe(e LogEntry) { + s.mu.Lock() + records := append([]*LogRecord(nil), s.records...) + s.mu.Unlock() + for _, record := range records { + record.Observe(e) + } +} diff --git a/op-service/logpipe/record_test.go b/op-service/logpipe/record_test.go new file mode 100644 index 00000000000..b61a5ae1d16 --- /dev/null +++ b/op-service/logpipe/record_test.go @@ -0,0 +1,60 @@ +package logpipe + +import ( + "encoding/json" + "testing" + + "github.com/stretchr/testify/require" +) + +func rustLogLine(message string) []byte { + return []byte(`{"timestamp":"2026-01-01T00:00:00Z","level":"DEBUG","fields":{"message":"` + message + `","block":7},"target":"engine::tree"}`) +} + +func TestLogRecordKeepsMatchingEntriesInOrder(t *testing.T) { + record := newLogRecord("reorg") + record.Observe(ParseRustStructuredLogs(rustLogLine("on-disk reorg detected"))) + record.Observe(ParseRustStructuredLogs(rustLogLine("Block executed"))) + record.Observe(ParseRustStructuredLogs(rustLogLine("Handling reorg: unwinding and buffering new path"))) + + require.Equal(t, 2, record.Count()) + entries := record.Entries() + require.Len(t, entries, 2) + require.Equal(t, "on-disk reorg detected", entries[0].LogMessage()) + require.Equal(t, "Handling reorg: unwinding and buffering new path", entries[1].LogMessage()) + require.Equal(t, json.Number("7"), entries[0].FieldValue("block"), "entries keep their fields") +} + +func TestLogRecordMatchesTheRawTextOfUnstructuredLines(t *testing.T) { + record := newLogRecord("panicked at") + record.Observe(ParseRustStructuredLogs([]byte("thread 'main' panicked at src/lib.rs:7:9:"))) + record.Observe(ParseRustStructuredLogs([]byte("some other plain line"))) + record.Observe(ParseRustStructuredLogs(rustLogLine("Invalid JSON"))) + + require.Equal(t, 1, record.Count(), "the raw line is matched, not the placeholder message of an unstructured entry") + line, unparsed := record.Entries()[0].(StructuredRustLogEntry).UnparsedLine() + require.True(t, unparsed) + require.Equal(t, "thread 'main' panicked at src/lib.rs:7:9:", line) +} + +func TestLogRecordEntriesIsACopy(t *testing.T) { + record := newLogRecord("") + record.Observe(ParseRustStructuredLogs(rustLogLine("first"))) + entries := record.Entries() + record.Observe(ParseRustStructuredLogs(rustLogLine("second"))) + + require.Len(t, entries, 1, "entries returned earlier must not grow with the record") + require.Equal(t, 2, record.Count()) +} + +func TestLogRecordsFanOutFromRegistration(t *testing.T) { + var records LogRecords + records.Observe(ParseRustStructuredLogs(rustLogLine("before registration"))) + reorgs := records.Record("reorg") + everything := records.Record("") + records.Observe(ParseRustStructuredLogs(rustLogLine("on-disk reorg detected"))) + records.Observe(ParseRustStructuredLogs(rustLogLine("Block executed"))) + + require.Equal(t, 1, reorgs.Count()) + require.Equal(t, 2, everything.Count(), "a record only sees entries observed after it was registered") +} diff --git a/rust/op-reth/tests/proofs/core/account_proofs_test.go b/rust/op-reth/tests/proofs/core/account_proofs_test.go index 1553c7c45a2..4ee952643e8 100644 --- a/rust/op-reth/tests/proofs/core/account_proofs_test.go +++ b/rust/op-reth/tests/proofs/core/account_proofs_test.go @@ -4,10 +4,10 @@ import ( "testing" "github.com/ethereum-optimism/optimism/op-devstack/devtest" + "github.com/ethereum-optimism/optimism/op-devstack/dsl" "github.com/ethereum-optimism/optimism/op-service/bigs" "github.com/ethereum-optimism/optimism/op-service/eth" "github.com/ethereum-optimism/optimism/rust/op-reth/tests/proofs/utils" - "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" "github.com/stretchr/testify/require" ) @@ -40,8 +40,7 @@ func TestL2MultipleTransactionsInDifferentBlocks(gt *testing.T) { require.Equal(t, types.ReceiptStatusSuccessful, receipt1.Status) t.Logf("Transaction 1 included in block: %d", bigs.Uint64Strict(receipt1.BlockNumber)) - utils.WaitForProofsStoreBlock(t, sys.RethWithProofL2ELNode().Escape().L2EthClient(), bigs.Uint64Strict(receipt1.BlockNumber)) - utils.FetchAndVerifyProofs(t, sys, accounts[0].Address(), []common.Hash{}, bigs.Uint64Strict(receipt1.BlockNumber)) + sys.RethWithProofL2ELNode().VerifyProofsMatchAt(sys.L2ELSequencerNode(), bigs.Uint64Strict(receipt1.BlockNumber), dsl.ProofAccount{Address: accounts[0].Address()}) sys.L2ELSequencerNode().WaitForBlockNumber(currentBlock.Number + 1) // Block 2: Send transaction from second account @@ -55,11 +54,10 @@ func TestL2MultipleTransactionsInDifferentBlocks(gt *testing.T) { require.Equal(t, types.ReceiptStatusSuccessful, receipt2.Status) t.Logf("Transaction 2 included in block: %d", bigs.Uint64Strict(receipt2.BlockNumber)) - utils.WaitForProofsStoreBlock(t, sys.RethWithProofL2ELNode().Escape().L2EthClient(), bigs.Uint64Strict(receipt2.BlockNumber)) - utils.FetchAndVerifyProofs(t, sys, accounts[1].Address(), []common.Hash{}, bigs.Uint64Strict(receipt2.BlockNumber)) + sys.RethWithProofL2ELNode().VerifyProofsMatchAt(sys.L2ELSequencerNode(), bigs.Uint64Strict(receipt2.BlockNumber), dsl.ProofAccount{Address: accounts[1].Address()}) // Also verify we can get proofs for account 0 at block 2 (different block height) - utils.FetchAndVerifyProofs(t, sys, accounts[0].Address(), []common.Hash{}, bigs.Uint64Strict(receipt2.BlockNumber)) + sys.RethWithProofL2ELNode().VerifyProofsMatchAt(sys.L2ELSequencerNode(), bigs.Uint64Strict(receipt2.BlockNumber), dsl.ProofAccount{Address: accounts[0].Address()}) } // TestL2MultipleTransactionsInSingleBlock tests 2 different accounts sending transactions @@ -100,21 +98,20 @@ func TestL2MultipleTransactionsInSingleBlock(gt *testing.T) { receipt1Block := bigs.Uint64Strict(receipt1.BlockNumber) t.Logf("Transaction 1 included in block %d", receipt1Block) - utils.WaitForProofsStoreBlock(t, sys.RethWithProofL2ELNode().Escape().L2EthClient(), max(receipt0Block, receipt1Block)) // Txns can land in the same or different blocks depending on timing. if receipt0Block == receipt1Block { t.Logf("Both transactions included in the same L2 block: %d", receipt0Block) // Verify both proofs against the same block state root - utils.FetchAndVerifyProofs(t, sys, accounts[0].Address(), []common.Hash{}, receipt0Block) - utils.FetchAndVerifyProofs(t, sys, accounts[1].Address(), []common.Hash{}, receipt0Block) + sys.RethWithProofL2ELNode().VerifyProofsMatchAt(sys.L2ELSequencerNode(), receipt0Block, dsl.ProofAccount{Address: accounts[0].Address()}) + sys.RethWithProofL2ELNode().VerifyProofsMatchAt(sys.L2ELSequencerNode(), receipt0Block, dsl.ProofAccount{Address: accounts[1].Address()}) } else { t.Logf("Transactions in different blocks: %d and %d", receipt0Block, receipt1Block) // Different blocks: verify each proof's merkle root matches its respective block's state root - utils.FetchAndVerifyProofs(t, sys, accounts[0].Address(), []common.Hash{}, receipt0Block) - utils.FetchAndVerifyProofs(t, sys, accounts[1].Address(), []common.Hash{}, receipt1Block) + sys.RethWithProofL2ELNode().VerifyProofsMatchAt(sys.L2ELSequencerNode(), receipt0Block, dsl.ProofAccount{Address: accounts[0].Address()}) + sys.RethWithProofL2ELNode().VerifyProofsMatchAt(sys.L2ELSequencerNode(), receipt1Block, dsl.ProofAccount{Address: accounts[1].Address()}) } t.Logf("Proof for account 0 and 1 verified successfully") diff --git a/rust/op-reth/tests/proofs/core/execute_payload_test.go b/rust/op-reth/tests/proofs/core/execute_payload_test.go index d9d59c3a331..fa67bc00bf2 100644 --- a/rust/op-reth/tests/proofs/core/execute_payload_test.go +++ b/rust/op-reth/tests/proofs/core/execute_payload_test.go @@ -4,6 +4,7 @@ import ( "testing" "github.com/ethereum-optimism/optimism/op-devstack/devtest" + "github.com/ethereum-optimism/optimism/op-devstack/dsl" "github.com/ethereum-optimism/optimism/op-service/eth" "github.com/ethereum-optimism/optimism/op-service/txplan" "github.com/ethereum-optimism/optimism/rust/op-reth/tests/proofs/utils" @@ -22,7 +23,7 @@ func TestExecutePayloadSuccess(gt *testing.T) { if err != nil { gt.Fatal(err) } - utils.WaitForProofsStoreBlock(t, opRethELNode.Escape().L2EthClient(), seqHead.NumberU64()) + opRethELNode.WaitForProofsStoreBlock(seqHead.NumberU64(), dsl.DefaultProofsCatchUpTimeout) plannedTxOption := user.PlanTransfer(user.Address(), eth.OneWei) plannedTx := txplan.NewPlannedTx(plannedTxOption) @@ -45,7 +46,7 @@ func TestExecutePayloadSuccess(gt *testing.T) { // ChainCommitted notifications asynchronously, so the EL head can advance before // the store is ready. debug_executePayload reads from the ExEx store and will // fail with "no state found" if it hasn't caught up yet. - utils.WaitForProofsStoreBlock(t, opRethELNode.Escape().L2EthClient(), lastBlock.NumberU64()) + opRethELNode.WaitForProofsStoreBlock(lastBlock.NumberU64(), dsl.DefaultProofsCatchUpTimeout) blockTime := lastBlock.Time() + 1 gasLimit := eth.Uint64Quantity(lastBlock.GasLimit()) @@ -90,7 +91,7 @@ func TestExecutePayloadWithInvalidParentHash(gt *testing.T) { if err != nil { gt.Fatal(err) } - utils.WaitForProofsStoreBlock(t, opRethELNode.Escape().L2EthClient(), seqHead.NumberU64()) + opRethELNode.WaitForProofsStoreBlock(seqHead.NumberU64(), dsl.DefaultProofsCatchUpTimeout) plannedTxOption := user.PlanTransfer(user.Address(), eth.OneWei) plannedTx := txplan.NewPlannedTx(plannedTxOption) @@ -110,7 +111,7 @@ func TestExecutePayloadWithInvalidParentHash(gt *testing.T) { } // Wait for the proofs ExEx store to index the parent block (same race as TestExecutePayloadSuccess). - utils.WaitForProofsStoreBlock(t, opRethELNode.Escape().L2EthClient(), lastBlock.NumberU64()) + opRethELNode.WaitForProofsStoreBlock(lastBlock.NumberU64(), dsl.DefaultProofsCatchUpTimeout) blockTime := lastBlock.Time() + 1 gasLimit := eth.Uint64Quantity(lastBlock.GasLimit()) diff --git a/rust/op-reth/tests/proofs/core/execution_witness_test.go b/rust/op-reth/tests/proofs/core/execution_witness_test.go index 3ad3dd2caec..153d016b615 100644 --- a/rust/op-reth/tests/proofs/core/execution_witness_test.go +++ b/rust/op-reth/tests/proofs/core/execution_witness_test.go @@ -5,6 +5,7 @@ import ( "testing" "github.com/ethereum-optimism/optimism/op-devstack/devtest" + "github.com/ethereum-optimism/optimism/op-devstack/dsl" "github.com/ethereum-optimism/optimism/op-service/bigs" "github.com/ethereum-optimism/optimism/op-service/eth" "github.com/ethereum-optimism/optimism/rust/op-reth/tests/proofs/utils" @@ -49,7 +50,7 @@ func TestDebugExecutionWitness(gt *testing.T) { require.Equal(t, types.ReceiptStatusSuccessful, receipt.Status) t.Logf("Transaction included in block: %d", bigs.Uint64Strict(receipt.BlockNumber)) - utils.WaitForProofsStoreBlock(t, opRethELNode.Escape().L2EthClient(), bigs.Uint64Strict(receipt.BlockNumber)) + opRethELNode.WaitForProofsStoreBlock(bigs.Uint64Strict(receipt.BlockNumber), dsl.DefaultProofsCatchUpTimeout) l2RethClient := opRethELNode.Escape().L2EthClient() // Get the block to inspect the state changes diff --git a/rust/op-reth/tests/proofs/core/resyncing_test.go b/rust/op-reth/tests/proofs/core/resyncing_test.go index 539b1c6a1f1..faf269b4a7f 100644 --- a/rust/op-reth/tests/proofs/core/resyncing_test.go +++ b/rust/op-reth/tests/proofs/core/resyncing_test.go @@ -6,11 +6,11 @@ import ( "time" "github.com/ethereum-optimism/optimism/op-devstack/devtest" + "github.com/ethereum-optimism/optimism/op-devstack/dsl" "github.com/ethereum-optimism/optimism/op-e2e/e2eutils/wait" "github.com/ethereum-optimism/optimism/op-service/bigs" "github.com/ethereum-optimism/optimism/op-service/eth" "github.com/ethereum-optimism/optimism/rust/op-reth/tests/proofs/utils" - "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" "github.com/stretchr/testify/require" ) @@ -61,12 +61,10 @@ func TestResyncing(gt *testing.T) { }) require.NoError(gt, err, "Validator L2 CL failed to resync to latest block") - utils.WaitForProofsStoreBlock(t, sys.L2ELValidatorNode().Escape().L2EthClient(), blockNumbers[len(blockNumbers)-1]) - t.Logf("Fetching and verifying proofs for transactions produced while node was down") // verify the proofs for the transactions produced while the node was down for _, blockNumber := range blockNumbers { - utils.FetchAndVerifyProofs(t, sys, bob.Address(), []common.Hash{}, blockNumber) - utils.FetchAndVerifyProofs(t, sys, alice.Address(), []common.Hash{}, blockNumber) + sys.RethWithProofL2ELNode().VerifyProofsMatchAt(sys.L2ELSequencerNode(), blockNumber, dsl.ProofAccount{Address: bob.Address()}) + sys.RethWithProofL2ELNode().VerifyProofsMatchAt(sys.L2ELSequencerNode(), blockNumber, dsl.ProofAccount{Address: alice.Address()}) } } diff --git a/rust/op-reth/tests/proofs/core/simple_storage_test.go b/rust/op-reth/tests/proofs/core/simple_storage_test.go index f9f8e39a431..07f72822c8a 100644 --- a/rust/op-reth/tests/proofs/core/simple_storage_test.go +++ b/rust/op-reth/tests/proofs/core/simple_storage_test.go @@ -5,6 +5,7 @@ import ( "testing" "github.com/ethereum-optimism/optimism/op-devstack/devtest" + "github.com/ethereum-optimism/optimism/op-devstack/dsl" "github.com/ethereum-optimism/optimism/op-service/bigs" "github.com/ethereum-optimism/optimism/op-service/eth" "github.com/ethereum-optimism/optimism/rust/op-reth/tests/proofs/utils" @@ -21,9 +22,8 @@ func TestStorageProofUsingSimpleStorageContract(gt *testing.T) { contract, receipt := utils.DeploySimpleStorage(t, user) t.Logf("contract deployed at address %s in L2 block %d", contract.Address().Hex(), bigs.Uint64Strict(receipt.BlockNumber)) - utils.WaitForProofsStoreBlock(t, sys.RethWithProofL2ELNode().Escape().L2EthClient(), bigs.Uint64Strict(receipt.BlockNumber)) // fetch and verify initial proof (should be zeroed storage) - utils.FetchAndVerifyProofs(t, sys, contract.Address(), []common.Hash{common.HexToHash("0x0")}, bigs.Uint64Strict(receipt.BlockNumber)) + sys.RethWithProofL2ELNode().VerifyProofsMatchAt(sys.L2ELSequencerNode(), bigs.Uint64Strict(receipt.BlockNumber), dsl.ProofAccount{Address: contract.Address(), StorageKeys: []common.Hash{common.HexToHash("0x0")}}) type caseEntry struct { Block uint64 @@ -49,15 +49,14 @@ func TestStorageProofUsingSimpleStorageContract(gt *testing.T) { }) t.Logf("reset setValue transaction included in L2 block %d", callRes.BlockNumber) - utils.WaitForProofsStoreBlock(t, sys.RethWithProofL2ELNode().Escape().L2EthClient(), bigs.Uint64Strict(callRes.BlockNumber)) // for each case, get proof and verify for _, c := range cases { - utils.FetchAndVerifyProofs(t, sys, contract.Address(), []common.Hash{common.HexToHash("0x0")}, c.Block) + sys.RethWithProofL2ELNode().VerifyProofsMatchAt(sys.L2ELSequencerNode(), c.Block, dsl.ProofAccount{Address: contract.Address(), StorageKeys: []common.Hash{common.HexToHash("0x0")}}) } // test with non-existent storage slot nonExistentSlot := common.HexToHash("0xdeadbeef") - utils.FetchAndVerifyProofs(t, sys, contract.Address(), []common.Hash{nonExistentSlot}, cases[len(cases)-1].Block) + sys.RethWithProofL2ELNode().VerifyProofsMatchAt(sys.L2ELSequencerNode(), cases[len(cases)-1].Block, dsl.ProofAccount{Address: contract.Address(), StorageKeys: []common.Hash{nonExistentSlot}}) } func TestStorageProofUsingMultiStorageContract(gt *testing.T) { @@ -70,9 +69,8 @@ func TestStorageProofUsingMultiStorageContract(gt *testing.T) { contract, receipt := utils.DeployMultiStorage(t, user) t.Logf("contract deployed at address %s in L2 block %d", contract.Address().Hex(), bigs.Uint64Strict(receipt.BlockNumber)) - utils.WaitForProofsStoreBlock(t, sys.RethWithProofL2ELNode().Escape().L2EthClient(), bigs.Uint64Strict(receipt.BlockNumber)) // fetch and verify initial proof (should be zeroed storage) - utils.FetchAndVerifyProofs(t, sys, contract.Address(), []common.Hash{common.HexToHash("0x0"), common.HexToHash("0x1")}, bigs.Uint64Strict(receipt.BlockNumber)) + sys.RethWithProofL2ELNode().VerifyProofsMatchAt(sys.L2ELSequencerNode(), bigs.Uint64Strict(receipt.BlockNumber), dsl.ProofAccount{Address: contract.Address(), StorageKeys: []common.Hash{common.HexToHash("0x0"), common.HexToHash("0x1")}}) // set multiple storage slots type caseEntry struct { @@ -107,7 +105,6 @@ func TestStorageProofUsingMultiStorageContract(gt *testing.T) { }) t.Logf("reset setValues transaction included in L2 block %d", callRes.BlockNumber) - utils.WaitForProofsStoreBlock(t, sys.RethWithProofL2ELNode().Escape().L2EthClient(), bigs.Uint64Strict(callRes.BlockNumber)) // for each case, get proof and verify for _, c := range cases { var slots []common.Hash @@ -115,7 +112,7 @@ func TestStorageProofUsingMultiStorageContract(gt *testing.T) { slots = append(slots, slot) } - utils.FetchAndVerifyProofs(t, sys, contract.Address(), slots, c.Block) + sys.RethWithProofL2ELNode().VerifyProofsMatchAt(sys.L2ELSequencerNode(), c.Block, dsl.ProofAccount{Address: contract.Address(), StorageKeys: slots}) } } @@ -151,8 +148,6 @@ func TestTokenVaultStorageProofs(gt *testing.T) { deactBlock := bigs.Uint64Strict(deactRes.BlockNumber) t.Logf("deactivateAllowance included in block %d", deactBlock) - utils.WaitForProofsStoreBlock(t, sys.RethWithProofL2ELNode().Escape().L2EthClient(), deactBlock) - // balance slot for user balanceSlot := contract.GetBalanceSlot(userAddr) // nested allowance slot owner=user, spender=spenderAddr @@ -163,11 +158,11 @@ func TestTokenVaultStorageProofs(gt *testing.T) { // fetch & verify proofs at appropriate blocks // balance after deposit (depositBlock) t.Logf("Verifying balance slot %s at deposit block %d", balanceSlot.Hex(), depositBlock) - utils.FetchAndVerifyProofs(t, sys, contract.Address(), []common.Hash{balanceSlot, depositor0Slot}, depositBlock) + sys.RethWithProofL2ELNode().VerifyProofsMatchAt(sys.L2ELSequencerNode(), depositBlock, dsl.ProofAccount{Address: contract.Address(), StorageKeys: []common.Hash{balanceSlot, depositor0Slot}}) // allowance after approve (approveBlock) t.Logf("Verifying allowance slot %s at approve block %d", allowanceSlot.Hex(), approveBlock) - utils.FetchAndVerifyProofs(t, sys, contract.Address(), []common.Hash{allowanceSlot}, approveBlock) + sys.RethWithProofL2ELNode().VerifyProofsMatchAt(sys.L2ELSequencerNode(), approveBlock, dsl.ProofAccount{Address: contract.Address(), StorageKeys: []common.Hash{allowanceSlot}}) // after deactivation, allowance should be zero at deactBlock t.Logf("Verifying allowance slot %s at deactivate block %d", allowanceSlot.Hex(), deactBlock) - utils.FetchAndVerifyProofs(t, sys, contract.Address(), []common.Hash{allowanceSlot}, deactBlock) + sys.RethWithProofL2ELNode().VerifyProofsMatchAt(sys.L2ELSequencerNode(), deactBlock, dsl.ProofAccount{Address: contract.Address(), StorageKeys: []common.Hash{allowanceSlot}}) } diff --git a/rust/op-reth/tests/proofs/prune/prune_test.go b/rust/op-reth/tests/proofs/prune/prune_test.go index d38c5872ab2..245d88556df 100644 --- a/rust/op-reth/tests/proofs/prune/prune_test.go +++ b/rust/op-reth/tests/proofs/prune/prune_test.go @@ -5,11 +5,10 @@ import ( "time" "github.com/ethereum-optimism/optimism/op-devstack/devtest" - "github.com/ethereum-optimism/optimism/op-service/apis" + "github.com/ethereum-optimism/optimism/op-devstack/dsl" "github.com/ethereum-optimism/optimism/op-service/bigs" "github.com/ethereum-optimism/optimism/op-service/eth" "github.com/ethereum-optimism/optimism/rust/op-reth/tests/proofs/utils" - "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" "github.com/stretchr/testify/require" ) @@ -35,7 +34,6 @@ func TestPruneProofStorageWithGetProofConsistency(gt *testing.T) { var pruneDetectTimeout = 5 * time.Minute opRethELNode := sys.RethWithProofL2ELNode() - ethClient := opRethELNode.Escape().EthClient() // ----------------------------- // (1) Create tx + validate proof pre-prune @@ -67,20 +65,17 @@ func TestPruneProofStorageWithGetProofConsistency(gt *testing.T) { } t.Logf("Target block for proof validation (pre-prune): %d", targetBlock) - // Make sure the proof store has indexed the target block before verifying. - utils.WaitForProofsStoreBlock(t, ethClient, targetBlock) - // Pre-prune proof verification at targetBlock. // This verifies the proof against the block's state root (efficient correctness check). t.Logf("Pre-prune: verifying getProof proofs at block %d", targetBlock) - utils.FetchAndVerifyProofs(t, sys, accounts[0].Address(), []common.Hash{}, targetBlock) - utils.FetchAndVerifyProofs(t, sys, accounts[1].Address(), []common.Hash{}, targetBlock) + sys.RethWithProofL2ELNode().VerifyProofsMatchAt(sys.L2ELSequencerNode(), targetBlock, dsl.ProofAccount{Address: accounts[0].Address()}) + sys.RethWithProofL2ELNode().VerifyProofsMatchAt(sys.L2ELSequencerNode(), targetBlock, dsl.ProofAccount{Address: accounts[1].Address()}) t.Log("Pre-prune: proofs verified successfully") // ----------------------------- // (2) Wait until targetBlock is pruned (earliest > targetBlock) // ----------------------------- - initialStatus := getProofSyncStatus(t, ethClient) + initialStatus := getProofSyncStatus(t, opRethELNode) t.Logf("Initial proofs sync status: earliest=%d latest=%d", initialStatus.Earliest, initialStatus.Latest) // Ensure we advance far enough that pruning *can* move earliest past targetBlock. @@ -88,7 +83,7 @@ func TestPruneProofStorageWithGetProofConsistency(gt *testing.T) { requiredLatest := targetBlock + proofWindow if initialStatus.Latest < requiredLatest { t.Logf("Waiting for chain to advance to at least block %d so pruning can pass targetBlock", requiredLatest) - utils.WaitForProofsStoreBlock(t, ethClient, requiredLatest) + opRethELNode.WaitForProofsStoreBlock(requiredLatest, dsl.DefaultProofsCatchUpTimeout) } t.Logf("Waiting for pruner to advance earliest past targetBlock=%d ...", targetBlock) @@ -100,7 +95,7 @@ func TestPruneProofStorageWithGetProofConsistency(gt *testing.T) { t.Errorf("Timed out waiting for prune: earliest did not advance past targetBlock=%d within %s", targetBlock, pruneDetectTimeout) } - prunedStatus = getProofSyncStatus(t, ethClient) + prunedStatus = getProofSyncStatus(t, opRethELNode) t.Logf("Polling proofs sync status: earliest=%d latest=%d (target=%d)", prunedStatus.Earliest, prunedStatus.Latest, targetBlock) // This is the key condition: the specific block we validated earlier is now out of window. @@ -119,8 +114,8 @@ func TestPruneProofStorageWithGetProofConsistency(gt *testing.T) { // (3) Post-prune consistency checks for getProof // ----------------------------- t.Logf("Post-prune: expecting getProof verification to succeed at new earliest block=%d", prunedStatus.Earliest) - utils.FetchAndVerifyProofs(t, sys, accounts[0].Address(), []common.Hash{}, prunedStatus.Earliest) - utils.FetchAndVerifyProofs(t, sys, accounts[1].Address(), []common.Hash{}, prunedStatus.Earliest) + sys.RethWithProofL2ELNode().VerifyProofsMatchAt(sys.L2ELSequencerNode(), prunedStatus.Earliest, dsl.ProofAccount{Address: accounts[0].Address()}) + sys.RethWithProofL2ELNode().VerifyProofsMatchAt(sys.L2ELSequencerNode(), prunedStatus.Earliest, dsl.ProofAccount{Address: accounts[1].Address()}) t.Log("Post-prune: getProof consistency checks passed") } @@ -129,11 +124,9 @@ type proofSyncStatus struct { Latest uint64 `json:"latest"` } -func getProofSyncStatus(t devtest.T, client apis.EthClient) proofSyncStatus { - var result proofSyncStatus - err := client.RPC().CallContext(t.Ctx(), &result, "debug_proofsSyncStatus") - if err != nil { - t.Errorf("debug_proofsSyncStatus call failed: %v", err) - } - return result +func getProofSyncStatus(t devtest.T, node *dsl.L2ELNode) proofSyncStatus { + status, err := node.ProofsSyncStatus() + require.NoError(t, err, "debug_proofsSyncStatus call failed") + require.NotNil(t, status.Latest, "proofs store is not initialized") + return proofSyncStatus{Earliest: *status.Earliest, Latest: *status.Latest} } diff --git a/rust/op-reth/tests/proofs/reorg/reorg_test.go b/rust/op-reth/tests/proofs/reorg/reorg_test.go deleted file mode 100644 index fcafc889793..00000000000 --- a/rust/op-reth/tests/proofs/reorg/reorg_test.go +++ /dev/null @@ -1,204 +0,0 @@ -package reorg - -import ( - "context" - "math/big" - "testing" - "time" - - "github.com/ethereum-optimism/optimism/op-devstack/devtest" - "github.com/ethereum-optimism/optimism/op-e2e/e2eutils/wait" - "github.com/ethereum-optimism/optimism/op-service/bigs" - "github.com/ethereum-optimism/optimism/op-service/eth" - "github.com/ethereum-optimism/optimism/op-service/txplan" - "github.com/ethereum-optimism/optimism/op-test-sequencer/sequencer/seqtypes" - "github.com/ethereum-optimism/optimism/rust/op-reth/tests/proofs/utils" - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core/types" - "github.com/stretchr/testify/require" -) - -func TestReorgUsingAccountProof(gt *testing.T) { - t := devtest.SerialT(gt) - ctx := t.Ctx() - - sys := utils.NewMixedOpProofPreset(t) - l := sys.Log - - ia := sys.TestSequencer.Escape().ControlAPI(sys.L2Chain.ChainID()) - - // stop batcher on chain A - sys.L2Batcher.Stop() - - // two EOAs for a sample transfer tx used later in a conflicting block - alice := sys.FunderL2.NewFundedEOA(eth.OneHundredthEther) - bob := sys.FunderL2.NewFundedEOA(eth.OneHundredthEther) - - user := sys.FunderL2.NewFundedEOA(eth.OneEther) - contract, deployBlock := utils.DeploySimpleStorage(t, user) - t.Logf("SimpleStorage deployed at %s block=%d", contract.Address().Hex(), bigs.Uint64Strict(deployBlock.BlockNumber)) - - time.Sleep(12 * time.Second) - divergenceHead := sys.L2Chain.WaitForBlock() - // build up some blocks that will be reorged away - - type caseEntry struct { - Block uint64 - addr common.Address - slots []common.Hash - } - var cases []caseEntry - - // deploy another contract in the reorged blocks - { - rContract, rDeployBlock := utils.DeploySimpleStorage(t, user) - t.Logf("Reorg SimpleStorage deployed at %s block=%d", rContract.Address().Hex(), bigs.Uint64Strict(rDeployBlock.BlockNumber)) - - cases = append(cases, caseEntry{ - Block: bigs.Uint64Strict(rDeployBlock.BlockNumber), - addr: rContract.Address(), - slots: []common.Hash{common.HexToHash("0x0")}, - }) - } - - for i := 0; i < 3; i++ { - tx := alice.Transfer(bob.Address(), eth.OneGWei) - receipt, err := tx.Included.Eval(ctx) - require.NoError(gt, err) - require.Equal(gt, types.ReceiptStatusSuccessful, receipt.Status) - - cases = append(cases, caseEntry{ - Block: bigs.Uint64Strict(receipt.BlockNumber), - addr: alice.Address(), - slots: []common.Hash{}, - }) - cases = append(cases, caseEntry{ - Block: bigs.Uint64Strict(receipt.BlockNumber), - addr: bob.Address(), - slots: []common.Hash{}, - }) - - // also include the contract account in the proofs to verify - val := big.NewInt(int64(i * 10)) - callRes := contract.SetValue(user, val) - - cases = append(cases, caseEntry{ - Block: bigs.Uint64Strict(callRes.BlockNumber), - addr: contract.Address(), - slots: []common.Hash{common.HexToHash("0x0")}, - }) - } - - sys.L2CLSequencer.StopSequencer() - - var divergenceBlockNumber uint64 - var originalRef eth.L2BlockRef - // prepare and sequence a conflicting block for the L2A chain - { - divergenceBlockRef := sys.L2ELSequencerNode().BlockRefByNumber(divergenceHead.Number) - - l.Info("Expect to reorg the chain on block", "number", divergenceBlockRef.Number, "head", divergenceHead, "parent", divergenceBlockRef.ParentID().Hash) - divergenceBlockNumber = divergenceBlockRef.Number - originalRef = divergenceBlockRef - - parentOfDivergenceHead := divergenceBlockRef.ParentID() - - l.Info("Sequencing a conflicting block", "divergenceBlockRef", divergenceBlockRef, "parent", parentOfDivergenceHead) - - // sequence a conflicting block with a simple transfer tx, based on the parent of the parent of the unsafe head - { - err := ia.New(ctx, seqtypes.BuildOpts{ - Parent: parentOfDivergenceHead.Hash, - L1Origin: nil, - }) - require.NoError(t, err, "Expected to be able to create a new block job for sequencing on op-test-sequencer, but got error") - - // include simple transfer tx in opened block - { - to := bob.PlanTransfer(alice.Address(), eth.OneGWei) - opt := txplan.Combine(to) - ptx := txplan.NewPlannedTx(opt) - signed_tx, err := ptx.Signed.Eval(ctx) - require.NoError(t, err, "Expected to be able to evaluate a planned transaction on op-test-sequencer, but got error") - txdata, err := signed_tx.MarshalBinary() - require.NoError(t, err, "Expected to be able to marshal a signed transaction on op-test-sequencer, but got error") - - err = ia.IncludeTx(ctx, txdata) - require.NoError(t, err, "Expected to be able to include a signed transaction on op-test-sequencer, but got error") - - cases = append(cases, caseEntry{ - Block: divergenceHead.Number, - addr: alice.Address(), - slots: []common.Hash{}, - }) - cases = append(cases, caseEntry{ - Block: divergenceHead.Number, - addr: bob.Address(), - slots: []common.Hash{}, - }) - } - - err = ia.Next(ctx) - require.NoError(t, err, "Expected to be able to call Next() after New() on op-test-sequencer, but got error") - } - } - - // start batcher on chain A - sys.L2Batcher.Start() - - // sequence a second block with op-test-sequencer (no L1 origin override) - { - l.Info("Sequencing with op-test-sequencer (no L1 origin override)") - err := ia.New(ctx, seqtypes.BuildOpts{ - Parent: sys.L2ELSequencerNode().BlockRefByLabel(eth.Unsafe).Hash, - L1Origin: nil, - }) - require.NoError(t, err, "Expected to be able to create a new block job for sequencing on op-test-sequencer, but got error") - time.Sleep(2 * time.Second) - - err = ia.Next(ctx) - require.NoError(t, err, "Expected to be able to call Next() after New() on op-test-sequencer, but got error") - time.Sleep(2 * time.Second) - } - - // continue sequencing with consensus node (op-node) - sys.L2CLSequencer.StartSequencer() - - for i := 0; i < 3; i++ { - sys.L2Chain.WaitForBlock() - } - - latestBlock := sys.L2Chain.WaitForBlock() - utils.WaitForProofsStoreBlock(t, sys.RethWithProofL2ELNode().Escape().L2EthClient(), latestBlock.Number) - - // verify that the L2A validator has reorged and reached the latest block. Bounded so a - // validator that never converges fails here instead of consuming the package timeout. - reorgCtx, cancelReorg := context.WithTimeout(t.Ctx(), 5*time.Minute) - defer cancelReorg() - err := wait.For(reorgCtx, 2*time.Second, func() (bool, error) { - blockRef, err := sys.L2ELValidatorNode().Escape().EthClient().BlockRefByNumber(ctx, latestBlock.Number) - if err != nil { - // this could happen if the validator is still syncing after reorg - l.Warn("Error fetching block reference from validator", "error", err) - return false, nil - } - return blockRef.Hash == latestBlock.Hash, nil - }) - require.NoError(t, err, "Expected block hash to match latest block hash on validator") - - reorgedRef_A, err := sys.L2ELSequencerNode().Escape().EthClient().BlockRefByNumber(ctx, divergenceBlockNumber) - require.NoError(t, err, "Expected to be able to call BlockRefByNumber API, but got error") - - l.Info("Reorged chain on divergence block number (prior the reorg)", "number", divergenceBlockNumber, "head", originalRef.Hash, "parent", originalRef.ParentID().Hash) - l.Info("Reorged chain on divergence block number (after the reorg)", "number", divergenceBlockNumber, "head", reorgedRef_A.Hash, "parent", reorgedRef_A.ParentID().Hash) - require.NotEqual(t, originalRef.Hash, reorgedRef_A.Hash, "Expected to get different heads on divergence block number, but got the same hash, so no reorg happened on chain A") - require.Equal(t, originalRef.ParentID().Hash, reorgedRef_A.ParentHash, "Expected to get same parent hashes on divergence block number, but got different hashes") - - time.Sleep(10 * time.Second) - - // verify that the accounts involved in the conflicting blocks - for i, c := range cases { - l.Info("Verifying proof", "case", i, "addr", c.addr.Hex(), "block", c.Block) - utils.FetchAndVerifyProofs(t, sys, c.addr, c.slots, c.Block) - } -} diff --git a/rust/op-reth/tests/proofs/utils/proof.go b/rust/op-reth/tests/proofs/utils/proof.go deleted file mode 100644 index 67afbd658c2..00000000000 --- a/rust/op-reth/tests/proofs/utils/proof.go +++ /dev/null @@ -1,166 +0,0 @@ -package utils - -import ( - "bytes" - "fmt" - "time" - - "github.com/ethereum-optimism/optimism/op-devstack/devtest" - "github.com/ethereum-optimism/optimism/op-service/eth" - "github.com/ethereum-optimism/optimism/op-service/retry" - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/common/hexutil" - "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/crypto" - "github.com/ethereum/go-ethereum/ethdb/memorydb" - "github.com/ethereum/go-ethereum/rlp" - "github.com/ethereum/go-ethereum/trie" - "github.com/stretchr/testify/require" -) - -// NormalizeProofResponse standardizes an AccountResult obtained from eth_getProof -// across different client implementations (e.g., Geth, Reth) so that they can be -// compared meaningfully in tests. -// -// Ethereum clients may encode empty or zeroed data structures differently while -// still representing the same logical state. For example: -// - An empty storage proof may appear as [] (Geth) or ["0x80"] (Reth). -// -// This function normalizes such differences by: -// - Converting single-element proofs containing "0x80" to an empty proof slice. -func NormalizeProofResponse(res *eth.AccountResult) { - for i := range res.StorageProof { - if len(res.StorageProof[i].Proof) == 1 && bytes.Equal(res.StorageProof[i].Proof[0], []byte{0x80}) { - res.StorageProof[i].Proof = []hexutil.Bytes{} - } - } - - // Normalize empty CodeHash - // Geth returns 0x0000000000000000000000000000000000000000000000000000000000000000 - // Reth returns 0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470 - if res.CodeHash == (common.Hash{}) { - res.CodeHash = crypto.Keccak256Hash(nil) - } - - // Normalize empty StorageHash - // Geth returns 0x0000000000000000000000000000000000000000000000000000000000000000 - // Reth returns 0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421 - if res.StorageHash == (common.Hash{}) { - res.StorageHash = types.EmptyRootHash - } -} - -// VerifyProof verifies an account and its storage proofs against a given state root. -// -// This function extends the standard behavior of go-ethereum’s AccountResult.Verify() -// by gracefully handling the case where the account’s storage trie root is empty -// (0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421). -func VerifyProof(res *eth.AccountResult, stateRoot common.Hash) error { - // Skip storage proof verification if the storage trie is empty. - if res.StorageHash != types.EmptyRootHash { - for i, entry := range res.StorageProof { - // load all MPT nodes into a DB - db := memorydb.New() - for j, encodedNode := range entry.Proof { - nodeKey := encodedNode - if len(encodedNode) >= 32 { // small MPT nodes are not hashed - nodeKey = crypto.Keccak256(encodedNode) - } - if err := db.Put(nodeKey, encodedNode); err != nil { - return fmt.Errorf("failed to load storage proof node %d of storage value %d into mem db: %w", j, i, err) - } - } - path := crypto.Keccak256(entry.Key) - val, err := trie.VerifyProof(res.StorageHash, path, db) - if err != nil { - return fmt.Errorf("failed to verify storage value %d with key %s (path %x) in storage trie %s: %w", i, entry.Key.String(), path, res.StorageHash, err) - } - if val == nil && entry.Value.ToInt().Cmp(common.Big0) == 0 { // empty storage is zero by default - continue - } - comparison, err := rlp.EncodeToBytes(entry.Value.ToInt().Bytes()) - if err != nil { - return fmt.Errorf("failed to encode storage value %d with key %s (path %x) in storage trie %s: %w", i, entry.Key.String(), path, res.StorageHash, err) - } - if !bytes.Equal(val, comparison) { - return fmt.Errorf("value %d in storage proof does not match proven value at key %s (path %x)", i, entry.Key.String(), path) - } - } - } - - accountClaimed := []any{uint64(res.Nonce), res.Balance.ToInt().Bytes(), res.StorageHash, res.CodeHash} - accountClaimedValue, err := rlp.EncodeToBytes(accountClaimed) - if err != nil { - return fmt.Errorf("failed to encode account from retrieved values: %w", err) - } - - // create a db with all account trie nodes - db := memorydb.New() - for i, encodedNode := range res.AccountProof { - nodeKey := encodedNode - if len(encodedNode) >= 32 { // small MPT nodes are not hashed - nodeKey = crypto.Keccak256(encodedNode) - } - if err := db.Put(nodeKey, encodedNode); err != nil { - return fmt.Errorf("failed to load account proof node %d into mem db: %w", i, err) - } - } - path := crypto.Keccak256(res.Address[:]) - accountProofValue, err := trie.VerifyProof(stateRoot, path, db) - if err != nil { - return fmt.Errorf("failed to verify account value with key %s (path %x) in account trie %s: %w", res.Address, path, stateRoot, err) - } - - // If the proof demonstrates non-existence (nil value), we must check if the RPC claimed the account is empty. - if len(accountProofValue) == 0 { - isEmpty := res.Nonce == 0 && - res.Balance.ToInt().Sign() == 0 && - (res.StorageHash == types.EmptyRootHash || res.StorageHash == common.Hash{}) && - (res.CodeHash == crypto.Keccak256Hash(nil) || res.CodeHash == common.Hash{}) - - if isEmpty { - return nil - } - } - - if !bytes.Equal(accountClaimedValue, accountProofValue) { - return fmt.Errorf("L1 RPC is tricking us, account proof does not match provided deserialized values:\n"+ - " claimed: %x\n"+ - " proof: %x", accountClaimedValue, accountProofValue) - } - return nil -} - -func fetchProofWithRetry(t devtest.T, source string, block uint64, fetch func() (*eth.AccountResult, error)) *eth.AccountResult { - // Each op-reth node persists proof state asynchronously. Readiness on one node does not imply - // that its peer can serve the same historical block yet, so retry the exact RPC on each node. - proof, err := retry.Do(t.Ctx(), 50, &retry.FixedStrategy{Dur: 200 * time.Millisecond}, fetch) - require.NoError(t, err, "failed to get proof from %s at block %d", source, block) - return proof -} - -// FetchAndVerifyProofs fetches account proofs from both L2EL and L2ELB for the given address -func FetchAndVerifyProofs(t devtest.T, sys *MixedOpProofPreset, address common.Address, slots []common.Hash, block uint64) { - ctx := t.Ctx() - blockInfo, err := sys.L2ELSequencerNode().Escape().L2EthClient().InfoByNumber(ctx, block) - require.NoError(t, err, "failed to get block info for block %d", block) - - blockTag := hexutil.Uint64(block).String() - seqProofRes := fetchProofWithRetry(t, "L2EL", block, func() (*eth.AccountResult, error) { - return sys.L2ELSequencerNode().Escape().L2EthClient().GetProof(ctx, address, slots, blockTag) - }) - valProofRes := fetchProofWithRetry(t, "L2ELB", block, func() (*eth.AccountResult, error) { - return sys.L2ELValidatorNode().Escape().L2EthClient().GetProof(ctx, address, slots, blockTag) - }) - - NormalizeProofResponse(seqProofRes) - NormalizeProofResponse(valProofRes) - - require.Equal(t, seqProofRes, valProofRes, "sequencer and validator proofs should match") - - err = VerifyProof(seqProofRes, blockInfo.Root()) - require.NoError(t, err, "geth proof verification failed at block %d", block) - - err = VerifyProof(valProofRes, blockInfo.Root()) - require.NoError(t, err, "reth proof verification failed at block %d", block) -} diff --git a/rust/op-reth/tests/proofs/utils/utils.go b/rust/op-reth/tests/proofs/utils/utils.go index 10e79d4ffc1..e47fdc47e15 100644 --- a/rust/op-reth/tests/proofs/utils/utils.go +++ b/rust/op-reth/tests/proofs/utils/utils.go @@ -4,11 +4,9 @@ import ( "encoding/json" "os" "strings" - "time" "github.com/ethereum-optimism/optimism/op-devstack/devtest" "github.com/ethereum-optimism/optimism/op-devstack/dsl" - "github.com/ethereum-optimism/optimism/op-service/apis" "github.com/ethereum-optimism/optimism/op-service/txplan" "github.com/ethereum/go-ethereum/accounts/abi" "github.com/ethereum/go-ethereum/common" @@ -51,33 +49,6 @@ func LoadArtifact(t devtest.T, artifactPath string) (abi.ABI, []byte) { return parsedABI, common.FromHex(binHex) } -// WaitForProofsStoreBlock polls the op-reth debug_proofsSyncStatus RPC until the -// proofs ExEx store has indexed at least up to targetBlock. The ExEx processes -// ChainCommitted notifications asynchronously, so the EL head can advance before -// the proofs store has caught up. Any RPC that depends on OpStateProviderFactory -// (e.g. debug_executePayload) will fail with "no state found" if called before -// the store is ready. -func WaitForProofsStoreBlock(t devtest.T, client apis.EthClient, targetBlock uint64) { - type syncStatus struct { - Earliest *uint64 `json:"earliest"` - Latest *uint64 `json:"latest"` - } - require.Eventually(t, func() bool { - var status syncStatus - err := client.RPC().CallContext(t.Ctx(), &status, "debug_proofsSyncStatus") - if err != nil { - t.Logf("debug_proofsSyncStatus call failed (retrying): %v", err) - return false - } - if status.Latest == nil { - t.Logf("proofs store not yet initialized, waiting...") - return false - } - t.Logf("proofs store status: latest=%d target=%d", *status.Latest, targetBlock) - return *status.Latest >= targetBlock - }, 30*time.Second, 200*time.Millisecond, "proofs store did not index block %d in time", targetBlock) -} - // DeployContract deploys the contract creation bytecode from the given artifact. // user must provide a Plan() method compatible with txplan.NewPlannedTx (kept generic). func DeployContract(t devtest.T, user *dsl.EOA, bin []byte) (common.Address, *types.Receipt) {