diff --git a/op-acceptance-tests/tests/op_reth_stale_forkchoice/datadir_wipe_test.go b/op-acceptance-tests/tests/op_reth_stale_forkchoice/datadir_wipe_test.go new file mode 100644 index 00000000000..2c71959f640 --- /dev/null +++ b/op-acceptance-tests/tests/op_reth_stale_forkchoice/datadir_wipe_test.go @@ -0,0 +1,67 @@ +package op_reth_stale_forkchoice + +import ( + "testing" + + "github.com/ethereum-optimism/optimism/op-devstack/devtest" + "github.com/ethereum-optimism/optimism/op-devstack/sysgo" + "github.com/ethereum-optimism/optimism/op-service/eth" + safety "github.com/ethereum-optimism/optimism/op-service/eth/safety" +) + +const ( + // finalityAttempts bounds, in 2s polls, how long the follower may take to finalize a block + // above genesis, which takes L1 finality: 20 L1 blocks of 6s. + finalityAttempts = 120 + // preWipeChainBlocks is how tall the chain is at least when the follower's datadir is wiped, + // so syncing it again is a download of many blocks rather than of a handful. + preWipeChainBlocks = 60 + // refinalizeAttempts bounds, in 2s polls, how long the follower may take to be finalized + // again where it was: after its reset a consensus client finalizes only blocks it derived + // since, so this can take about two L1 finality delays. + refinalizeAttempts = 210 + // resyncFromScratchAttempts bounds, in 2s polls, how long the follower may take to sync the + // whole chain again from an empty datadir and to be safe where it was. + resyncFromScratchAttempts = 120 +) + +// TestFollowerResyncsFromScratchUnderConsensusClientThatKeptItsHeads wipes the follower's +// op-reth datadir while its consensus client keeps running, so the first forkchoice updates the +// empty node receives name a head, a safe block and a finalized block it has not downloaded, and +// requires the follower to sync the chain again from genesis: to hold the sequencer's unsafe +// head, and to be safe and finalized at least where it was before the wipe. +func TestFollowerResyncsFromScratchUnderConsensusClientThatKeptItsHeads(gt *testing.T) { + t := devtest.ParallelT(gt) + devtest.RunParallel(t, syncModeMatrix(), func(t devtest.T, c syncModeCase) { + skipUnsupportedCell(t, c) + sys := newStaleForkchoiceSystem(t, systemConfig{ + persistenceThreshold: defaultPersistenceThreshold, + syncMode: c.syncMode, + }) + sys.Follower.EL.Reached(eth.Unsafe, preWipeChainBlocks, chainBuildUpAttempts) + // The consensus client must hold a safe and a finalized block above genesis, which + // an empty datadir does not have. + sys.Follower.EL.Reached(eth.Finalized, 1, finalityAttempts) + before := readEngineLabels(sys.Follower.EL) + + t.Logger().Info("Wiping the follower's datadir under its consensus client", "head", before.head, "safe", before.safe, "finalized", before.finalized) + sys.Follower.EL.Stop() + sys.Follower.EL.WipeDatadir() + sys.Follower.EL.Start() + sys.Follower.EL.WaitForOnline() + t.Require().Less(sys.Follower.EL.BlockRefByLabel(eth.Unsafe).Number, before.head.Number, "the wipe must drop the follower's chain") + sys.Follower.EL.PeerWith(sys.Sequencer.EL) + + sys.Follower.EL.InSync(sys.Sequencer.EL, safety.LocalUnsafe, resyncFromScratchAttempts) + sys.Follower.EL.Reached(eth.Safe, before.safe.Number, resyncFromScratchAttempts) + + sysgo.SkipOnKonaNode(t, "the follower is unsafe and safe where it was again, but it is not reliably finalized again "+ + "(#24066): after the wipe its kona-node resets on `Consolidate(MissingUnsafeL2Block(N))` every few seconds, which "+ + "clears its finalization queue each time, and in CI a finalize task hit `Finalize(BlockNotSafe)`, which kona-node "+ + "treats as fatal (`Failed to start rollup node service: EngineTask(Finalize(BlockNotSafe))`), so the finalized wait "+ + "failed with `operation failed permanently after 210 attempts: expected head for label=finalized to advance to "+ + "target=59, but got current=0`") + sys.Follower.EL.Reached(eth.Finalized, before.finalized.Number, refinalizeAttempts) + sys.Follower.CL.InSync(sys.Sequencer.CL, safety.LocalSafe, resyncAttempts) + }) +} diff --git a/op-acceptance-tests/tests/op_reth_stale_forkchoice/l1_reorg_test.go b/op-acceptance-tests/tests/op_reth_stale_forkchoice/l1_reorg_test.go new file mode 100644 index 00000000000..03927f1c9d5 --- /dev/null +++ b/op-acceptance-tests/tests/op_reth_stale_forkchoice/l1_reorg_test.go @@ -0,0 +1,67 @@ +package op_reth_stale_forkchoice + +import ( + "testing" + + "github.com/ethereum-optimism/optimism/op-devstack/devtest" + "github.com/ethereum-optimism/optimism/op-devstack/sysgo" + "github.com/ethereum-optimism/optimism/op-service/eth" + safety "github.com/ethereum-optimism/optimism/op-service/eth/safety" +) + +const ( + // l1ReorgAttempts bounds, in 7s polls, how long the L1 node may take to switch to the block + // the test sequencer built on the reorged block's parent. + l1ReorgAttempts = 10 + // postReorgSafeBlocks is how far past its pre-reorg height the follower's safe head must get + // again on the new L1 chain. + postReorgSafeBlocks = 5 + // safeRecoveryAttempts bounds, in 2s polls, how long the follower may take after the L1 reorg + // to replace its safe block and to be safe past the pre-reorg height again: the batcher must + // notice the reorg and post again, and the follower's derivation must reset and catch up. + safeRecoveryAttempts = 90 +) + +// TestFollowerSafeHeadAdvancesAgainAfterL1ReorgRewindsIt reorgs out the L1 block the follower's +// safe head derives from while the follower's consensus client runs, so the client resets and +// sends forkchoice updates whose safe hash moves back below the unsafe chain its engine still +// holds, and requires the follower to replace the safe block that derived from the reorged L1 +// block, to derive the new L1 chain past where its safe head was, and to be in sync with the +// sequencer again at the safe and unsafe heads. +func TestFollowerSafeHeadAdvancesAgainAfterL1ReorgRewindsIt(gt *testing.T) { + t := devtest.ParallelT(gt) + devtest.RunParallel(t, syncModeMatrix(), func(t devtest.T, c syncModeCase) { + skipUnsupportedCell(t, c) + sysgo.SkipOnKonaNode(t, "fails on develop (#18676): the kona-node sequencer resets its derivation on the L1 reorg "+ + "(`Derivation pipeline is being reset: L1 reorg detected`) but keeps building on the unsafe chain whose L1 origin was "+ + "reorged out, so nothing replaces that chain, every node rejects the old batches (`batch is for different L1 chain, "+ + "epoch hash does not match`) and the follower's safe block is never replaced: `operation failed permanently after 90 "+ + "attempts: expected head to reorg 0x3f26…d122:20, but got 0x3f26…d122:20`; the L1-reorg tests in sync/elsync/reorg "+ + "skip on kona-node for the same reason (#22275)") + sys := newStaleForkchoiceSystem(t, systemConfig{ + persistenceThreshold: defaultPersistenceThreshold, + syncMode: c.syncMode, + withTestSequencer: true, + }) + l1ChainID := sys.L1Network.ChainID() + sys.L1Network.WaitForBlock() + // Only the test sequencer builds L1 from here on, so the reorg below is the only fork. + sys.L1CL.Stop() + startL1 := sys.L1EL.BlockRefByLabel(eth.Unsafe) + safeBefore := sys.TestSequencer.BuildL1UntilSafeOriginPast(t, l1ChainID, sys.Follower.EL, startL1.Number) + + l1Origin := sys.L1EL.BlockRefByNumber(safeBefore.L1Origin.Number) + t.Logger().Info("Reorging out the L1 block the follower's safe head derives from", "safe", safeBefore, "l1_origin", l1Origin) + sys.TestSequencer.SequenceBlock(t, l1ChainID, l1Origin.ParentHash) + sys.L1CL.Start() + sys.L1EL.ReorgTriggered(l1Origin, l1ReorgAttempts) + l1After := sys.L1EL.BlockRefByNumber(l1Origin.Number) + t.Logger().Info("L1 reorged", "before", l1Origin, "after", l1After) + + sys.Follower.EL.ReorgTriggered(safeBefore, safeRecoveryAttempts) + sys.Follower.EL.WaitL1OriginHash(eth.Safe, l1After.ID(), safeRecoveryAttempts) + sys.Follower.EL.Reached(eth.Safe, safeBefore.Number+postReorgSafeBlocks, safeRecoveryAttempts) + sys.Follower.EL.InSync(sys.Sequencer.EL, safety.LocalSafe, resyncAttempts) + sys.Follower.EL.InSync(sys.Sequencer.EL, safety.LocalUnsafe, resyncAttempts) + }) +} diff --git a/op-acceptance-tests/tests/op_reth_stale_forkchoice/raw_forkchoice_test.go b/op-acceptance-tests/tests/op_reth_stale_forkchoice/raw_forkchoice_test.go new file mode 100644 index 00000000000..418a0b1d61b --- /dev/null +++ b/op-acceptance-tests/tests/op_reth_stale_forkchoice/raw_forkchoice_test.go @@ -0,0 +1,150 @@ +package op_reth_stale_forkchoice + +import ( + "fmt" + "testing" + "time" + + "github.com/ethereum-optimism/optimism/op-devstack/devtest" + "github.com/ethereum-optimism/optimism/op-devstack/dsl" + "github.com/ethereum-optimism/optimism/op-devstack/sysgo" + "github.com/ethereum-optimism/optimism/op-node/rollup/derive" + nodeSync "github.com/ethereum-optimism/optimism/op-node/rollup/sync" + "github.com/ethereum-optimism/optimism/op-service/eth" + "github.com/ethereum/go-ethereum/common" +) + +const ( + // replacedPersistedTimeout bounds how long the frozen follower may take to report the replaced + // block persisted: it already holds every block it needs, so reth only finishes the saves that + // are due. + replacedPersistedTimeout = 30 * time.Second + // syncBackAttempts bounds, in 2s polls, how long the follower may take to download the head + // of a forkchoice update it answered SYNCING from the sequencer and to make it canonical. + syncBackAttempts = 30 +) + +// freezeFollower stops the batcher, so nothing above the sequencer's head at that moment can +// become safe, lets the follower get blocksAboveSafe blocks past that head, and stops its +// consensus client, so the follower's engine answers only the forkchoice updates the test sends. +// It returns the follower's frozen head. +func freezeFollower(t devtest.T, sys *staleForkchoiceSystem, blocksAboveSafe uint64) eth.L2BlockRef { + sys.Batcher.Stop() + safeBound := sys.Sequencer.EL.BlockRefByLabel(eth.Unsafe) + sys.Follower.EL.Reached(eth.Unsafe, safeBound.Number+blocksAboveSafe, chainBuildUpAttempts) + sys.Follower.CL.Stop() + head := sys.Follower.EL.BlockRefByLabel(eth.Unsafe) + t.Logger().Info("Follower frozen", "head", head, "safe", sys.Follower.EL.BlockRefByLabel(eth.Safe), "safe_bound", safeBound) + return head +} + +// TestForkchoiceUpdateNamingReorgedOutFinalizedIsRejected builds a sibling of the follower's +// head and sends a forkchoice update that names the head the sibling replaces as finalized, and +// requires the engine to refuse it with the engine API's -38002 "Invalid forkchoice state" error +// and to leave its head, safe and finalized blocks as they were: first while the update is the one +// that would reorg to the sibling, then again once the sibling is the head. +func TestForkchoiceUpdateNamingReorgedOutFinalizedIsRejected(gt *testing.T) { + t := devtest.ParallelT(gt) + skipUnsupported(t) + sys := newStaleForkchoiceSystem(t, systemConfig{ + persistenceThreshold: defaultPersistenceThreshold, + syncMode: nodeSync.CLSync, + }) + replaced := freezeFollower(t, sys, 1) + sibling := sys.Follower.EL.BuildSiblingBlock(replaced.Number) + siblingHash := sibling.ExecutionPayload.BlockHash + sys.Follower.EL.NewPayloadRaw(sibling).IsValid() + labels := readEngineLabels(sys.Follower.EL) + t.Require().Less(labels.safe.Number, replaced.Number, "the safe head must be below the replaced block") + requireReplacedRefusedAsFinalized(t, sys.Follower.EL, siblingHash, replaced, labels) + + sys.Follower.EL.ForkchoiceUpdateRaw(siblingHash, labels.safe.Hash, labels.finalized.Hash, nil).IsValid() + labels = readEngineLabels(sys.Follower.EL) + t.Require().Equal(siblingHash, labels.head.Hash, "the sibling must be the head") + requireReplacedRefusedAsFinalized(t, sys.Follower.EL, siblingHash, replaced, labels) +} + +// requireReplacedRefusedAsFinalized sends a forkchoice update to the sibling that names the block +// it replaced as finalized, and requires the engine to refuse it and to keep labels. +func requireReplacedRefusedAsFinalized(t devtest.T, el *dsl.L2ELNode, siblingHash common.Hash, replaced eth.L2BlockRef, labels engineLabels) { + t.Logger().Info("Naming the replaced block as finalized", "head", labels.head, "safe", labels.safe, "finalized", labels.finalized, "replaced", replaced, "sibling", siblingHash) + el.ForkchoiceUpdateRaw(siblingHash, labels.safe.Hash, replaced.Hash, nil).IsInvalidForkchoiceState() + t.Require().Equal(labels, readEngineLabels(el), "a refused forkchoice update must leave the head, safe and finalized blocks as they were") +} + +// persistedHeadCase is one cell of the matrix the follower is reorged below its in-memory +// window under. +type persistedHeadCase struct { + persistenceThreshold uint64 +} + +func (c persistedHeadCase) String() string { + return fmt.Sprintf("threshold_%d", c.persistenceThreshold) +} + +// forkDepth is how far below the frozen head the replaced block sits. Reth persists once more +// than the threshold of blocks sit in memory and keeps the newest of them up to the memory block +// buffer target, so a block this deep is due for persistence without further blocks arriving. +func (c persistedHeadCase) forkDepth() uint64 { + return c.persistenceThreshold + sysgo.OpRethPinnedMemoryBlockBufferTarget + 2 +} + +func persistedHeadMatrix() []persistedHeadCase { + return []persistedHeadCase{{persistenceThreshold: 7}, {persistenceThreshold: 50}} +} + +// TestForkchoiceUpdateToPersistedReorgedOutHeadIsAppliedOrSynced waits until the follower +// reports a block persisted, reorgs it to a sibling of that block, and sends a forkchoice update +// back to the replaced block as soon as the reorg is answered. The replaced block is not +// canonical, so the engine must not take it for the head it already has: it either reorgs back to +// it and answers VALID, or answers SYNCING and the follower downloads it from the sequencer, which +// still holds it. Either way the follower ends on the replaced block. +func TestForkchoiceUpdateToPersistedReorgedOutHeadIsAppliedOrSynced(gt *testing.T) { + t := devtest.ParallelT(gt) + devtest.RunParallel(t, persistedHeadMatrix(), func(t devtest.T, c persistedHeadCase) { + skipUnsupported(t) + sys := newStaleForkchoiceSystem(t, systemConfig{ + persistenceThreshold: c.persistenceThreshold, + syncMode: nodeSync.CLSync, + }) + persistedBlocks := sys.Follower.EL.TrackPersistedBlock() + head := freezeFollower(t, sys, c.forkDepth()+1) + replaced := sys.Follower.EL.BlockRefByNumber(head.Number - c.forkDepth()) + persistedBlocks.AwaitPersisted(replaced.Number, replacedPersistedTimeout) + sibling := sys.Follower.EL.BuildSiblingBlock(replaced.Number) + siblingHash := sibling.ExecutionPayload.BlockHash + sys.Follower.EL.NewPayloadRaw(sibling).IsValid() + labels := readEngineLabels(sys.Follower.EL) + t.Require().Less(labels.safe.Number, replaced.Number, "the safe head must be below the replaced block") + + t.Logger().Info("Reorging the follower below its persisted block", "head", head, "replaced", replaced, "sibling", siblingHash, "safe", labels.safe, "finalized", labels.finalized) + reorged := sys.Follower.EL.ForkchoiceUpdateRaw(siblingHash, labels.safe.Hash, labels.finalized.Hash, nil).IsValid() + t.Require().NotNil(reorged.Result.PayloadStatus.LatestValidHash, "a VALID answer to the reorg must name its latest valid block") + t.Require().Equal(siblingHash, *reorged.Result.PayloadStatus.LatestValidHash, "the engine must have made the sibling the head") + // Reth removes the replaced blocks from disk on its persistence thread after answering the + // reorg and offers no way to hold that off. Only a VALID answer, which needs the update back + // to arrive before that removal commits, checks that a persisted header counts as canonical + // only when it matches the canonical hash (reth#27429); after the removal reth answers + // SYNCING, which this test accepts, so a run can pass without exercising that check. No RPC + // runs between the two updates, to keep the VALID window as wide as it can be. + answer := sys.Follower.EL.ForkchoiceUpdateRaw(replaced.Hash, labels.safe.Hash, labels.finalized.Hash, nil) + t.Require().NoError(answer.Err, "the update back to the replaced block must be answered") + status := answer.Result.PayloadStatus.Status + headAfter := sys.Follower.EL.BlockRefByLabel(eth.Unsafe) + t.Logger().Info("Engine answered the update back to the replaced block", "status", status, + "latest_valid_hash", answer.Result.PayloadStatus.LatestValidHash, "head", headAfter) + t.Require().Contains([]eth.ExecutePayloadStatus{eth.ExecutionValid, eth.ExecutionSyncing}, status, + "the replaced block is valid, so the engine either applies the update or syncs to it") + if status == eth.ExecutionValid { + t.Logger().Info("Engine reorged back to the persisted block itself, which exercises reth#27429") + t.Require().Equal(replaced, headAfter, "a VALID answer must mean the head is the requested block") + } else { + t.Logger().Info("Engine answered SYNCING, so this run does not exercise reth#27429; the follower downloads the replaced block from the sequencer") + } + siblingRef, err := derive.PayloadToBlockRef(sys.L2Network.Escape().RollupConfig(), sibling.ExecutionPayload) + t.Require().NoError(err, "the sibling must carry an L1 info deposit") + // A SYNCING answer makes the replaced block a sync target the follower downloads from the sequencer. + sys.Follower.EL.ReorgExact(siblingRef, syncBackAttempts) + t.Require().Equal(replaced, sys.Follower.EL.BlockRefByLabel(eth.Unsafe), "the follower must be back on the replaced block") + }) +} diff --git a/op-acceptance-tests/tests/op_reth_stale_forkchoice/system_test.go b/op-acceptance-tests/tests/op_reth_stale_forkchoice/system_test.go new file mode 100644 index 00000000000..a1d8e4f275d --- /dev/null +++ b/op-acceptance-tests/tests/op_reth_stale_forkchoice/system_test.go @@ -0,0 +1,162 @@ +package op_reth_stale_forkchoice + +import ( + "github.com/ethereum-optimism/optimism/op-devstack/devtest" + "github.com/ethereum-optimism/optimism/op-devstack/dsl" + "github.com/ethereum-optimism/optimism/op-devstack/presets" + "github.com/ethereum-optimism/optimism/op-devstack/sysgo" + nodeSync "github.com/ethereum-optimism/optimism/op-node/rollup/sync" + "github.com/ethereum-optimism/optimism/op-service/eth" +) + +const ( + // chainBuildUpAttempts bounds, in 2s polls, how long the follower may take to hold the blocks + // a scenario needs before it starts: at most about a minute of 1s blocks. + chainBuildUpAttempts = 90 + // defaultPersistenceThreshold is reth's --engine.persistence-threshold for the scenarios that + // do not vary it: reth 2.5's default, passed explicitly so the persistence schedule does not + // move with the binary. + defaultPersistenceThreshold = 7 + // l2BlockTimeSeconds keeps the runs short: the chain lengths the scenarios need are block + // counts, so halving the block time halves the time spent producing blocks. + l2BlockTimeSeconds = 1 + // resyncAttempts bounds, in 2s polls, how long the follower may take to be in sync with the + // sequencer again once it has recovered. + resyncAttempts = 60 +) + +const ( + followerKey = "follower" + sequencerKey = "sequencer" +) + +// l2Node is one L2 node of the system: its execution layer and the consensus client driving it. +type l2Node struct { + CL *dsl.L2CLNode + EL *dsl.L2ELNode +} + +// systemConfig is what varies between the systems the scenarios run on. +type systemConfig struct { + persistenceThreshold uint64 + // syncMode is how the follower's consensus client fills an unsafe gap: CL sync has op-node + // fetch the blocks from its peers, EL sync hands the target to the EL. kona-node has no CL + // sync mode and ignores this. + syncMode nodeSync.Mode + // withTestSequencer adds a test sequencer that builds L1 blocks on a chosen parent, which is + // how a scenario reorgs L1. + withTestSequencer bool +} + +type staleForkchoiceSystem struct { + Batcher *dsl.L2Batcher + Follower l2Node + L1CL *dsl.L1CLNode + L1EL *dsl.L1ELNode + L1Network *dsl.L1Network + L2Network *dsl.L2Network + Sequencer l2Node + TestSequencer *dsl.TestSequencer +} + +// newStaleForkchoiceSystem starts an op-reth sequencer and an op-reth follower, both without +// proofs history and on the configured persistence schedule, with the follower's consensus +// client in the configured sync mode. +func newStaleForkchoiceSystem(t devtest.T, cfg systemConfig) *staleForkchoiceSystem { + clKind := sysgo.ResolveMixedL2CLKind() + opRethOpts := []sysgo.OpRethOption{ + sysgo.OpRethWithoutProofsHistory(), + sysgo.OpRethWithPersistenceSchedule(sysgo.OpRethPersistenceSchedule{ + MemoryBlockBufferTarget: sysgo.OpRethPinnedMemoryBlockBufferTarget, + PersistenceThreshold: cfg.persistenceThreshold, + }), + } + runtime := sysgo.NewMixedSingleChainRuntime(t, sysgo.MixedSingleChainPresetConfig{ + DeployerOptions: []sysgo.DeployerOption{sysgo.WithUniformL2BlockTimes(l2BlockTimeSeconds)}, + NodeSpecs: []sysgo.MixedSingleChainNodeSpec{ + { + CLKey: sequencerKey, + CLKind: clKind, + ELKey: sequencerKey, + ELKind: sysgo.MixedL2ELOpReth, + IsSequencer: true, + OpRethOpts: opRethOpts, + }, + { + CLKey: followerKey, + CLKind: clKind, + ELKey: followerKey, + ELKind: sysgo.MixedL2ELOpReth, + L2CLOpts: []sysgo.L2CLOption{sysgo.L2CLVerifierSyncMode(cfg.syncMode)}, + OpRethOpts: opRethOpts, + }, + }, + WithTestSequencer: cfg.withTestSequencer, + }) + frontends := presets.NewMixedSingleChainFrontends(t, runtime) + t.Require().Len(frontends.Nodes, 2, "expected the sequencer and the follower") + + sys := &staleForkchoiceSystem{ + Batcher: frontends.L2Batcher, + L1CL: frontends.L1CL, + L1EL: frontends.L1EL, + L1Network: frontends.L1Network, + L2Network: frontends.L2Network, + TestSequencer: frontends.TestSequencer, + } + for _, node := range frontends.Nodes { + switch node.Spec.ELKey { + case sequencerKey: + sys.Sequencer = l2Node{CL: node.CL, EL: node.EL} + case followerKey: + sys.Follower = l2Node{CL: node.CL, EL: node.EL} + } + } + t.Require().NotNil(sys.Sequencer.EL, "missing the sequencer") + t.Require().NotNil(sys.Follower.EL, "missing the follower") + return sys +} + +// syncModeCase is one cell of the matrix the follower's consensus client runs under. +type syncModeCase struct { + syncMode nodeSync.Mode +} + +func (c syncModeCase) String() string { + if c.syncMode == nodeSync.ELSync { + return "el_sync" + } + return "cl_sync" +} + +func syncModeMatrix() []syncModeCase { + return []syncModeCase{{syncMode: nodeSync.CLSync}, {syncMode: nodeSync.ELSync}} +} + +// skipUnsupportedCell skips the cells the configured clients cannot run. +func skipUnsupportedCell(t devtest.T, c syncModeCase) { + skipUnsupported(t) + if c.syncMode == nodeSync.CLSync { + sysgo.SkipOnKonaNode(t, "kona-node has no CL sync mode: it always lets the EL sync the gap") + } +} + +func skipUnsupported(t devtest.T) { + sysgo.SkipOnOpGeth(t, "the forkchoice updates under test are answered by op-reth") +} + +// engineLabels are an execution engine's head, safe and finalized blocks, read together so a +// forkchoice update can be required to have changed none of them. +type engineLabels struct { + finalized eth.L2BlockRef + head eth.L2BlockRef + safe eth.L2BlockRef +} + +func readEngineLabels(el *dsl.L2ELNode) engineLabels { + return engineLabels{ + finalized: el.BlockRefByLabel(eth.Finalized), + head: el.BlockRefByLabel(eth.Unsafe), + safe: el.BlockRefByLabel(eth.Safe), + } +} diff --git a/op-devstack/dsl/engine.go b/op-devstack/dsl/engine.go index 741c9f4e895..417e2324f19 100644 --- a/op-devstack/dsl/engine.go +++ b/op-devstack/dsl/engine.go @@ -8,6 +8,7 @@ import ( "github.com/ethereum-optimism/optimism/op-devstack/devtest" "github.com/ethereum-optimism/optimism/op-service/eth" "github.com/ethereum-optimism/optimism/op-service/retry" + "github.com/ethereum/go-ethereum/rpc" ) type NewPayloadResult struct { @@ -69,6 +70,19 @@ func (r *ForkchoiceUpdateResult) IsValid() *ForkchoiceUpdateResult { return r } +// IsInvalidForkchoiceState requires the update to have been refused with the engine API's +// -38002 "Invalid forkchoice state" error, which an execution engine answers when a nonzero safe +// or finalized hash is not on the head's chain, with any detail a client adds to the message. A +// refused update carries no payload status. +func (r *ForkchoiceUpdateResult) IsInvalidForkchoiceState() *ForkchoiceUpdateResult { + r.T.Require().Nil(r.Result, "a forkchoice update with an invalid state must be refused with an error, not answered with a status") + var rpcErr rpc.Error + r.T.Require().ErrorAs(r.Err, &rpcErr, "expected an engine error code, got: %v", r.Err) + r.T.Require().Equal(eth.InvalidForkchoiceState, eth.ErrorCode(rpcErr.ErrorCode()), "unexpected engine error: %v", r.Err) + r.T.Require().Contains(rpcErr.Error(), "Invalid forkchoice state", "unexpected engine error message") + return r +} + func (r *ForkchoiceUpdateResult) WaitUntilValid(attempts int) *ForkchoiceUpdateResult { tryCnt := 0 err := retry.Do0(r.T.Ctx(), attempts, &retry.FixedStrategy{Dur: 1 * time.Second}, diff --git a/op-devstack/dsl/l2_el.go b/op-devstack/dsl/l2_el.go index 768dee0685f..0206ba647a4 100644 --- a/op-devstack/dsl/l2_el.go +++ b/op-devstack/dsl/l2_el.go @@ -267,6 +267,7 @@ func (el *L2ELNode) ReachedWithProgressFn(label, progressLabel eth.BlockLabel, t } } +// blockRefByNumber returns the block ref at num, or the lookup error instead of failing the test. func (el *L2ELNode) blockRefByNumber(num uint64) (eth.L2BlockRef, error) { ctx, cancel := context.WithTimeout(el.ctx, DefaultTimeout) defer cancel() @@ -547,6 +548,16 @@ func (el *L2ELNode) StopUngracefully() { lifecycle.StopUngracefully() } +// WipeDatadir empties the node's storage and initializes it afresh, so the next Start runs the +// node on an empty chain while whatever drives it keeps its own view of the chain. The node must +// be stopped. +func (el *L2ELNode) WipeDatadir() { + el.log.Info("Wiping datadir", "name", el.inner.Name()) + wipeable, ok := el.inner.(stack.DatadirWipeable) + el.require.Truef(ok, "L2EL node %s cannot wipe its datadir", el.inner.Name()) + wipeable.WipeDatadir() +} + func (el *L2ELNode) Start() { lifecycle, ok := el.inner.(stack.Lifecycle) el.require.Truef(ok, "L2EL node %s is not lifecycle-controllable", el.inner.Name()) @@ -618,6 +629,14 @@ func (el *L2ELNode) NewPayloadRaw(payload *eth.ExecutionPayloadEnvelope) *NewPay return &NewPayloadResult{T: el.t, Status: status, Err: err} } +// GetPayload fetches the payload a forkchoice update with attributes started building. +func (el *L2ELNode) GetPayload(info eth.PayloadInfo) *eth.ExecutionPayloadEnvelope { + el.log.Info("GetPayload", "payload_id", info.ID, "timestamp", info.Timestamp, "node", el) + payload, err := el.inner.L2EngineClient().GetPayload(el.ctx, info) + el.require.NoError(err, "failed to get payload %s", info.ID) + return payload +} + // ForkchoiceUpdate fetches FCU target hashes from the reference EL node, and FCU update with attributes func (el *L2ELNode) ForkchoiceUpdate(refNode *L2ELNode, unsafe, safe, finalized uint64, attr *eth.PayloadAttributes) *ForkchoiceUpdateResult { unsafeHash := refNode.BlockRefByNumber(unsafe).Hash diff --git a/op-devstack/dsl/l2_el_sibling.go b/op-devstack/dsl/l2_el_sibling.go new file mode 100644 index 00000000000..b17b98543ee --- /dev/null +++ b/op-devstack/dsl/l2_el_sibling.go @@ -0,0 +1,64 @@ +package dsl + +import ( + "github.com/ethereum-optimism/optimism/op-core/eip1559" + "github.com/ethereum-optimism/optimism/op-service/eth" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/crypto" +) + +// BuildSiblingBlock has the engine build a block that can replace the canonical block at number: +// it extends the same parent from the attributes that block was built from, so it carries the +// same transactions and timestamp, and differs in its prevRandao and so in its hash. The payload +// is returned without being inserted. The build is started by a forkchoice update that names the +// parent as head with zero safe and finalized hashes, since reth 2.6 refuses a nonzero safe or +// finalized hash above the head it is given; naming a canonical ancestor leaves the head where +// it is, which is required. +func (el *L2ELNode) BuildSiblingBlock(number uint64) *eth.ExecutionPayloadEnvelope { + headBefore := el.BlockRefByLabel(eth.Unsafe) + replaced := el.PayloadByNumber(number) + parent := replaced.ExecutionPayload.ParentHash + attrs := siblingPayloadAttributes(replaced) + el.log.Info("Building a sibling block", "number", number, "replaced", replaced.ExecutionPayload.BlockHash, "parent", parent, "node", el) + started := el.ForkchoiceUpdateRaw(parent, common.Hash{}, common.Hash{}, attrs) + started.IsValid() + el.require.NotNilf(started.Result.PayloadID, "the engine must start building on %s", parent) + sibling := el.GetPayload(eth.PayloadInfo{ID: *started.Result.PayloadID, Timestamp: uint64(attrs.Timestamp)}) + el.require.Equal(parent, sibling.ExecutionPayload.ParentHash, "the sibling must extend the replaced block's parent") + el.require.Equal(number, uint64(sibling.ExecutionPayload.BlockNumber), "the sibling must sit at the replaced block's height") + el.require.NotEqual(replaced.ExecutionPayload.BlockHash, sibling.ExecutionPayload.BlockHash, "the sibling must differ from the block it replaces") + canonicalAtHeadBefore, err := el.blockRefByNumber(headBefore.Number) + el.require.NoError(err, "the head from before the build must still be canonical") + el.require.Equal(headBefore.Hash, canonicalAtHeadBefore.Hash, "building the sibling must not reorg the head away") + el.log.Info("Built a sibling block", "number", number, "sibling", sibling.ExecutionPayload.BlockHash, "replaced", replaced.ExecutionPayload.BlockHash, "node", el) + return sibling +} + +// siblingPayloadAttributes returns the attributes that rebuild payload on its parent with a +// different prevRandao. They force the payload's transactions in and take nothing from the +// transaction pool, and they carry the fee recipient, gas limit and EIP-1559 parameters the +// payload was built with, which an execution engine requires to match the active forks. +func siblingPayloadAttributes(payload *eth.ExecutionPayloadEnvelope) *eth.PayloadAttributes { + block := payload.ExecutionPayload + gasLimit := block.GasLimit + attrs := ð.PayloadAttributes{ + ParentBeaconBlockRoot: payload.ParentBeaconBlockRoot, + PrevRandao: eth.Bytes32(crypto.Keccak256Hash(block.PrevRandao[:])), + SuggestedFeeRecipient: block.FeeRecipient, + Timestamp: block.Timestamp, + Withdrawals: block.Withdrawals, + + GasLimit: &gasLimit, + NoTxPool: true, + Transactions: block.Transactions, + } + // Holocene puts the EIP-1559 parameters into extraData, Jovian adds the minimum base fee; + // before Holocene extraData is empty and the attributes carry neither. + if len(block.ExtraData) > 0 { + denominator, elasticity, minBaseFee := eip1559.DecodeJovianExtraData(block.ExtraData) + params := eth.Bytes8(eip1559.EncodeHolocene1559Params(denominator, elasticity)) + attrs.EIP1559Params = ¶ms + attrs.MinBaseFee = minBaseFee + } + return attrs +} diff --git a/op-devstack/dsl/l2_el_sibling_test.go b/op-devstack/dsl/l2_el_sibling_test.go new file mode 100644 index 00000000000..84900c1c6ea --- /dev/null +++ b/op-devstack/dsl/l2_el_sibling_test.go @@ -0,0 +1,72 @@ +package dsl + +import ( + "testing" + + "github.com/ethereum-optimism/optimism/op-core/eip1559" + "github.com/ethereum-optimism/optimism/op-service/eth" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + "github.com/stretchr/testify/require" +) + +func siblingSourcePayload(extraData []byte) *eth.ExecutionPayloadEnvelope { + beaconRoot := common.HexToHash("0xbeac") + return ð.ExecutionPayloadEnvelope{ + ExecutionPayload: ð.ExecutionPayload{ + BlockHash: common.HexToHash("0xb10c"), + BlockNumber: 42, + ExtraData: extraData, + FeeRecipient: common.HexToAddress("0xfee"), + GasLimit: 30_000_000, + ParentHash: common.HexToHash("0x9a"), + PrevRandao: eth.Bytes32(common.HexToHash("0x1")), + Timestamp: 1_700_000_000, + Transactions: []eth.Data{{0x7e, 0x01}, {0x02, 0x02}}, + Withdrawals: &types.Withdrawals{}, + }, + ParentBeaconBlockRoot: &beaconRoot, + } +} + +func TestSiblingPayloadAttributesRebuildTheBlockWithAnotherPrevRandao(t *testing.T) { + t.Parallel() + payload := siblingSourcePayload(nil) + attrs := siblingPayloadAttributes(payload) + block := payload.ExecutionPayload + require.Equal(t, block.Timestamp, attrs.Timestamp) + require.NotEqual(t, block.PrevRandao, attrs.PrevRandao) + require.Equal(t, block.FeeRecipient, attrs.SuggestedFeeRecipient) + require.Equal(t, block.Withdrawals, attrs.Withdrawals) + require.Equal(t, payload.ParentBeaconBlockRoot, attrs.ParentBeaconBlockRoot) + require.Equal(t, block.Transactions, attrs.Transactions) + require.True(t, attrs.NoTxPool, "the sibling must carry the replaced block's transactions and nothing from the pool") + require.Equal(t, block.GasLimit, *attrs.GasLimit) +} + +func TestSiblingPayloadAttributesCarryTheExtraDataParameters(t *testing.T) { + t.Parallel() + minBaseFee := uint64(7) + for _, tc := range []struct { + name string + eip1559Params *eth.Bytes8 + extraData []byte + minBaseFee *uint64 + }{ + {name: "pre-holocene"}, + {name: "holocene", eip1559Params: siblingParams(250, 6), extraData: eip1559.EncodeHoloceneExtraData(250, 6)}, + {name: "jovian", eip1559Params: siblingParams(250, 6), extraData: eip1559.EncodeJovianExtraData(250, 6, minBaseFee), minBaseFee: &minBaseFee}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + attrs := siblingPayloadAttributes(siblingSourcePayload(tc.extraData)) + require.Equal(t, tc.eip1559Params, attrs.EIP1559Params) + require.Equal(t, tc.minBaseFee, attrs.MinBaseFee) + }) + } +} + +func siblingParams(denominator, elasticity uint64) *eth.Bytes8 { + params := eth.Bytes8(eip1559.EncodeHolocene1559Params(denominator, elasticity)) + return ¶ms +} diff --git a/op-devstack/presets/rpc_frontends.go b/op-devstack/presets/rpc_frontends.go index 38d04744220..4b294fc2548 100644 --- a/op-devstack/presets/rpc_frontends.go +++ b/op-devstack/presets/rpc_frontends.go @@ -160,6 +160,7 @@ type l2ELFrontend struct { binarySwitch stack.BinarySwitchable logs stack.LogObservable + wipeable stack.DatadirWipeable } var ( @@ -167,6 +168,7 @@ var ( _ stack.UngracefulLifecycle = (*l2ELFrontend)(nil) _ stack.BinarySwitchable = (*l2ELFrontend)(nil) _ stack.LogObservable = (*l2ELFrontend)(nil) + _ stack.DatadirWipeable = (*l2ELFrontend)(nil) ) func newPresetL2ELNode(t devtest.T, name string, chainID eth.ChainID, userRPCCl opclient.RPC, userRPC string, engineRPCCl opclient.RPC, rollupCfg *rollup.Config) *l2ELFrontend { @@ -213,6 +215,11 @@ func (r *l2ELFrontend) SwitchBinary(execPath string) { r.binarySwitch.SwitchBinary(execPath) } +func (r *l2ELFrontend) WipeDatadir() { + r.require().NotNil(r.wipeable, "L2EL node %s cannot wipe its datadir", r.Name()) + r.wipeable.WipeDatadir() +} + func (r *l2ELFrontend) ObserveLogs(observer func(logpipe.LogEntry)) func() { r.require().NotNil(r.logs, "L2EL node %s does not expose its logs", r.Name()) return r.logs.ObserveLogs(observer) diff --git a/op-devstack/presets/sysgo_runtime.go b/op-devstack/presets/sysgo_runtime.go index e7455f3a554..ec9635f553c 100644 --- a/op-devstack/presets/sysgo_runtime.go +++ b/op-devstack/presets/sysgo_runtime.go @@ -75,6 +75,9 @@ func newL2ELFrontend(t devtest.T, name string, chainID eth.ChainID, userRPC stri if logs, ok := lifecycle[0].(stack.LogObservable); ok { l2EL.logs = logs } + if wipeable, ok := lifecycle[0].(stack.DatadirWipeable); ok { + l2EL.wipeable = wipeable + } } return l2EL } diff --git a/op-devstack/stack/lifecycle.go b/op-devstack/stack/lifecycle.go index 21997a796f7..49542d99136 100644 --- a/op-devstack/stack/lifecycle.go +++ b/op-devstack/stack/lifecycle.go @@ -27,3 +27,11 @@ type BinarySwitchable interface { // stopped. SwitchBinary(execPath string) } + +// DatadirWipeable is implemented by components that keep their chain in a datadir and can empty +// it, so that their next Start runs the same configuration on an empty chain. +type DatadirWipeable interface { + // WipeDatadir removes everything the component has stored and initializes its storage afresh. + // The component must be stopped. + WipeDatadir() +} diff --git a/op-devstack/sysgo/l2_el_opreth.go b/op-devstack/sysgo/l2_el_opreth.go index 6564e5b4773..1e7ed26b86e 100644 --- a/op-devstack/sysgo/l2_el_opreth.go +++ b/op-devstack/sysgo/l2_el_opreth.go @@ -4,6 +4,8 @@ import ( "context" "fmt" "net/url" + "os" + "os/exec" "slices" "strings" "sync" @@ -172,6 +174,11 @@ type OpReth struct { authRPC string userRPC string + chainConfigPath string + dataDirPath string + // proofsHistoryDir is the proofs store's path, empty when proofs history is disabled. + proofsHistoryDir string + authProxy *tcpproxy.Proxy userProxy *tcpproxy.Proxy @@ -194,6 +201,7 @@ var ( _ stack.UngracefulLifecycle = (*OpReth)(nil) _ stack.BinarySwitchable = (*OpReth)(nil) _ stack.LogObservable = (*OpReth)(nil) + _ stack.DatadirWipeable = (*OpReth)(nil) ) func (n *OpReth) Start() { @@ -419,6 +427,49 @@ func (n *OpReth) SwitchBinary(execPath string) { n.execPath = execPath } +// WipeDatadir empties the node's datadir and, when enabled, its proofs store, and initializes +// both afresh, so the next Start runs the node's binary and arguments on an empty chain, the way +// an operator replaces a node's storage. The node must be stopped. +func (n *OpReth) WipeDatadir() { + n.mu.Lock() + defer n.mu.Unlock() + n.p.Require().Truef(n.sub == nil, "op-reth %s must be stopped before its datadir is wiped", n.name) + n.p.Require().NotEmptyf(n.dataDirPath, "op-reth %s has no datadir to wipe", n.name) + n.p.Logger().Info("Wiping op-reth datadir", "name", n.name, "chain", n.chainID, "datadir", n.dataDirPath, "proofs_store", n.proofsHistoryDir) + n.p.Require().NoError(os.RemoveAll(n.dataDirPath), "must remove datadir") + n.p.Require().NoError(os.MkdirAll(n.dataDirPath, 0o755), "must create datadir") + initOpRethDatadir(n.p, n.execPath, n.dataDirPath, n.chainConfigPath) + if n.proofsHistoryDir != "" { + n.p.Require().NoError(os.RemoveAll(n.proofsHistoryDir), "must remove proofs store") + initOpRethProofsStore(n.p, n.execPath, n.dataDirPath, n.chainConfigPath, n.proofsHistoryDir) + } + n.p.Logger().Info("Wiped op-reth datadir", "name", n.name, "chain", n.chainID) +} + +// initOpRethDatadir writes the chain's genesis into an empty datadir. +func initOpRethDatadir(t devtest.T, execPath, dataDirPath, chainConfigPath string) { + initArgs := []string{ + "init", + "--datadir=" + dataDirPath, + "--chain=" + chainConfigPath, + } + initOut, err := exec.CommandContext(t.Ctx(), execPath, initArgs...).CombinedOutput() + t.Require().NoError(err, "must init op-reth node: %s", string(initOut)) +} + +// initOpRethProofsStore creates an empty proofs-history store at storagePath for the datadir. +func initOpRethProofsStore(t devtest.T, execPath, dataDirPath, chainConfigPath, storagePath string) { + initProofsArgs := []string{ + "proofs", + "init", + "--datadir=" + dataDirPath, + "--chain=" + chainConfigPath, + "--proofs-history.storage-path=" + storagePath, + } + initOut, err := exec.CommandContext(t.Ctx(), execPath, initProofsArgs...).CombinedOutput() + t.Require().NoError(err, "must init op-reth proof history: %s", string(initOut)) +} + // ObserveLogs registers observer for every stdout and stderr entry the op-reth process emits, // across restarts, until the returned function is called. func (n *OpReth) ObserveLogs(observer func(logpipe.LogEntry)) func() { diff --git a/op-devstack/sysgo/mixed_runtime.go b/op-devstack/sysgo/mixed_runtime.go index e8dc747a2e3..ab69c81101f 100644 --- a/op-devstack/sysgo/mixed_runtime.go +++ b/op-devstack/sysgo/mixed_runtime.go @@ -6,7 +6,6 @@ import ( "encoding/json" "fmt" "os" - "os/exec" "path/filepath" "strings" @@ -470,32 +469,17 @@ func buildMixedOpRethNode( args = append(args, "--metrics=127.0.0.1:0") } - initArgs := []string{ - "init", - "--datadir=" + dataDirPath, - "--chain=" + chainConfigPath, - } - initOut, err := exec.Command(execPath, initArgs...).CombinedOutput() - t.Require().NoError(err, "must init op-reth node: %s", string(initOut)) + initOpRethDatadir(t, execPath, dataDirPath, chainConfigPath) + var proofsHistoryDir string if !opRethCfg.DisableProofsHistory { - proofHistoryDir := filepath.Join(tempDir, "proof-history") - - initProofsArgs := []string{ - "proofs", - "init", - "--datadir=" + dataDirPath, - "--chain=" + chainConfigPath, - "--proofs-history.storage-path=" + proofHistoryDir, - } - initOut, initErr := exec.Command(execPath, initProofsArgs...).CombinedOutput() - t.Require().NoError(initErr, "must init op-reth proof history: %s", string(initOut)) - + proofsHistoryDir = filepath.Join(tempDir, "proof-history") + initOpRethProofsStore(t, execPath, dataDirPath, chainConfigPath, proofsHistoryDir) args = append( args, "--proofs-history", "--proofs-history.window=10000", - "--proofs-history.storage-path="+proofHistoryDir, + "--proofs-history.storage-path="+proofsHistoryDir, ) } @@ -508,6 +492,9 @@ func buildMixedOpRethNode( jwtSecret: jwtSecret, authRPC: "", userRPC: "", + chainConfigPath: chainConfigPath, + dataDirPath: dataDirPath, + proofsHistoryDir: proofsHistoryDir, execPath: execPath, args: args, env: []string{},