diff --git a/op-acceptance-tests/tests/proofs_history_match/proofs_history_match_test.go b/op-acceptance-tests/tests/proofs_history_match/proofs_history_match_test.go index ab9dcb87bea..c44c728c0ee 100644 --- a/op-acceptance-tests/tests/proofs_history_match/proofs_history_match_test.go +++ b/op-acceptance-tests/tests/proofs_history_match/proofs_history_match_test.go @@ -9,12 +9,9 @@ import ( "github.com/ethereum-optimism/optimism/op-devstack/dsl" "github.com/ethereum-optimism/optimism/op-devstack/presets" "github.com/ethereum-optimism/optimism/op-devstack/sysgo" - "github.com/ethereum-optimism/optimism/op-service/bigs" "github.com/ethereum-optimism/optimism/op-service/eth" "github.com/ethereum-optimism/optimism/op-service/txplan" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core/vm" - "github.com/ethereum/go-ethereum/core/vm/program" ) const ( @@ -99,27 +96,6 @@ func newProofsHistorySystem(t devtest.T, c proofsHistoryCase) *proofsHistorySyst } } -// deployBlockRecorder deploys a contract that writes the block number to slot 0 and the block -// timestamp to slot 1 on every call, so every call changes its storage. -func deployBlockRecorder(t devtest.T, deployer *dsl.EOA) common.Address { - runtimeCode := program.New(). - Op(vm.NUMBER).Push(0).Op(vm.SSTORE). - Op(vm.TIMESTAMP).Push(1).Op(vm.SSTORE). - Bytes() - initCode := program.New().ReturnViaCodeCopy(runtimeCode).Bytes() - - tx := deployer.Transact(deployer.Plan(), txplan.WithData(initCode)) - receipt, err := tx.Included.Eval(t.Ctx()) - t.Require().NoError(err, "block recorder deployment must be included") - return receipt.ContractAddress -} - -func includedBlock(t devtest.T, tx *txplan.PlannedTx) uint64 { - receipt, err := tx.Included.Eval(t.Ctx()) - t.Require().NoError(err, "transaction must be included") - return bigs.Uint64Strict(receipt.BlockNumber) -} - // TestProofsHistoryFollowerServesMatchingHistoricalProofs runs transfers, withdrawals and contract // writes against a sequencer that serves eth_getProof from its own historical state, then sweeps a // proofs-history follower from its window's first block to the last workload block: each of the @@ -132,7 +108,7 @@ func TestProofsHistoryFollowerServesMatchingHistoricalProofs(gt *testing.T) { alice := sys.FunderL2.NewFundedEOA(eth.OneEther) bob := sys.FunderL2.NewFundedEOA(eth.OneEther) - recorder := deployBlockRecorder(t, alice) + recorder := alice.DeployBlockRecorder() var lastWorkloadBlock uint64 for round := range workloadRounds { if round == workloadRounds-1 { @@ -144,7 +120,7 @@ func TestProofsHistoryFollowerServesMatchingHistoricalProofs(gt *testing.T) { alice.Transfer(bob.Address(), eth.OneHundredthEther) bob.Transfer(alice.Address(), eth.OneHundredthEther) sys.Bridge.InitiateWithdrawal(eth.OneHundredthEther, bob) - lastWorkloadBlock = includedBlock(t, alice.Transact(alice.Plan(), txplan.WithTo(&recorder))) + lastWorkloadBlock = dsl.IncludedBlockNumber(t, alice.Transact(alice.Plan(), txplan.WithTo(&recorder))) } sys.ProofsFollower.VerifyHistoricalProofsMatch(sys.Reference, func(opts *dsl.HistoricalProofsOpts) { diff --git a/op-acceptance-tests/tests/proofs_history_restart/proofs_history_restart_test.go b/op-acceptance-tests/tests/proofs_history_restart/proofs_history_restart_test.go new file mode 100644 index 00000000000..a2a38b0fe05 --- /dev/null +++ b/op-acceptance-tests/tests/proofs_history_restart/proofs_history_restart_test.go @@ -0,0 +1,470 @@ +package proofs_history_restart + +import ( + "errors" + "fmt" + "testing" + "time" + + "github.com/ethereum-optimism/optimism/op-devstack/devtest" + "github.com/ethereum-optimism/optimism/op-devstack/dsl" + "github.com/ethereum-optimism/optimism/op-devstack/presets" + "github.com/ethereum-optimism/optimism/op-devstack/sysgo" + nodeSync "github.com/ethereum-optimism/optimism/op-node/rollup/sync" + "github.com/ethereum-optimism/optimism/op-service/eth" + safety "github.com/ethereum-optimism/optimism/op-service/eth/safety" + "github.com/ethereum-optimism/optimism/op-service/log" + "github.com/ethereum-optimism/optimism/op-service/retry" + "github.com/ethereum-optimism/optimism/op-service/testlog" +) + +const ( + // l2BlockTimeSeconds keeps the runs short: the kill point is a block height, so halving the + // block time halves the time spent producing blocks. + l2BlockTimeSeconds = 1 + // memoryBlockBufferTarget is reth's --engine.memory-block-buffer-target. It is passed + // explicitly, like the persistence threshold, so the gap the kill opens between the proofs + // store and reth's datadir does not depend on reth's defaults. + memoryBlockBufferTarget = 5 + // postRestartTrafficBlocks is how far past the head the chain is driven once the follower has + // caught up, so the sweep also covers blocks the proofs store indexed after coming back. + postRestartTrafficBlocks = 10 + // proofsStoreFlushBlocks is how many blocks op-reth's proofs store buffers before it flushes + // them. op-reth has no flag for it. + proofsStoreFlushBlocks = 5 + + // followerImportAttempts bounds the wait for the follower to import the block the sequencer + // was held at. + followerImportAttempts = 10 + // killPointHoldAttempts bounds how often the chain is held before the held head fits the case's + // kill point. + killPointHoldAttempts = 3 + // rebuildAttempts bounds how long the restarted sequencer may take to rebuild the heights it + // lost with different contents. + rebuildAttempts = 30 + // resyncAttempts bounds how long the restarted follower may take to catch up with the + // sequencer. A follower whose op-reth crash-loops never does. + resyncAttempts = 60 + // storeCatchUpTimeout bounds the wait for a proofs store flush before the kill. + storeCatchUpTimeout = 60 * time.Second +) + +// killPoint says where, relative to the proofs store's flush cadence, the follower is killed. +type killPoint string + +const ( + // killAfterStoreFlush kills the follower as soon as the store has flushed the target block and + // one more workload transaction has landed, so the store's tip is near the chain head and its + // lead over reth's datadir is the largest. + killAfterStoreFlush killPoint = "after_flush" + // killMidStoreBuffer kills the follower a couple of blocks after a flush, so the store's tip + // trails the chain head and the store loses blocks of its own on top of reth's. + killMidStoreBuffer killPoint = "mid_buffer" +) + +// syncPath says how the restarted follower's CL brings its EL from reth's datadir tip back to +// the sequencer's head. +type syncPath string + +const ( + // clSync has op-node insert the missing blocks itself, fetched from its peers. + clSync syncPath = "cl_sync" + // elSync runs op-node in EL sync mode with post-finalization EL sync, as op-node runs it + // against op-reth, so it hands the gap to op-reth even though op-reth already holds a + // finalized block past genesis. + elSync syncPath = "el_sync" + // elSyncAfterFinality runs op-node in EL sync mode with post-finalization EL sync turned off, + // as op-node runs it against op-geth. The follower's first EL sync finalized a block past + // genesis, so the restarted op-node finds it and goes straight to CL sync. + elSyncAfterFinality syncPath = "el_sync_after_finality" +) + +func (p syncPath) followerL2CLOpts(clKind sysgo.MixedL2CLKind) []sysgo.L2CLOption { + switch p { + case elSync: + opts := []sysgo.L2CLOption{sysgo.L2CLVerifierSyncMode(nodeSync.ELSync)} + // kona-node always EL-syncs the gap and rejects op-node's switch for it. + if clKind == sysgo.MixedL2CLOpNode { + opts = append(opts, sysgo.L2CLPostFinalizationELSync(true)) + } + return opts + case elSyncAfterFinality: + return []sysgo.L2CLOption{sysgo.L2CLVerifierSyncMode(nodeSync.ELSync), sysgo.L2CLPostFinalizationELSync(false)} + default: + return []sysgo.L2CLOption{sysgo.L2CLVerifierSyncMode(nodeSync.CLSync)} + } +} + +// restartCase is one cell of the matrix the follower runs under. +type restartCase struct { + killPoint killPoint + persistenceThreshold uint64 + // storeFlushPastSetup is how far past the workload's setup block the proofs store must have + // flushed before the kill. It exceeds persistenceThreshold+1, the most reth's persisted tip + // trails its head by, so reth has persisted the setup block by the kill. + storeFlushPastSetup uint64 + syncPath syncPath + verificationInterval uint64 +} + +func (c restartCase) String() string { + proofsPath := "index" + if c.verificationInterval != 0 { + proofsPath = "execute" + } + return fmt.Sprintf("threshold_%d-%s-%s-kill_%s", c.persistenceThreshold, proofsPath, c.syncPath, c.killPoint) +} + +// engineArgs pins reth's persistence schedule, which decides how many blocks an unclean stop +// loses. Both nodes get them so the sequencer loses blocks on the same schedule as the follower. +func (c restartCase) engineArgs() []string { + return []string{ + fmt.Sprintf("--engine.memory-block-buffer-target=%d", memoryBlockBufferTarget), + fmt.Sprintf("--engine.persistence-threshold=%d", c.persistenceThreshold), + } +} + +func (c restartCase) followerArgs() []string { + return append(c.engineArgs(), fmt.Sprintf("--proofs-history.verification-interval=%d", c.verificationInterval)) +} + +// killSchedule pairs the kill point with the persistence threshold rather than crossing them: the +// small threshold gets the mid-buffer kill, where both reth and the store lose blocks, the large +// one the after-flush kill, where the store's lead over reth is largest. Reth persists only up to +// memoryBlockBufferTarget blocks below its head, so its datadir trails the store at the kill: at +// threshold 7 by a few blocks, at 50 by up to 50 blocks it holds only in memory. +type killSchedule struct { + killPoint killPoint + persistenceThreshold uint64 + storeFlushPastSetup uint64 +} + +var ( + afterStoreFlushSchedule = killSchedule{killPoint: killAfterStoreFlush, persistenceThreshold: 50, storeFlushPastSetup: 55} + midStoreBufferSchedule = killSchedule{killPoint: killMidStoreBuffer, persistenceThreshold: 7, storeFlushPastSetup: 17} +) + +// restartMatrix crosses the sync paths with the index proofs path only, and runs the execute +// proofs path once, under EL sync, the one sync path that runs on every CL kind and the one that +// hands the gap to reth's own backfill rather than inserting it block by block. +func restartMatrix(schedule killSchedule) []restartCase { + cell := func(path syncPath, verificationInterval uint64) restartCase { + return restartCase{ + killPoint: schedule.killPoint, + persistenceThreshold: schedule.persistenceThreshold, + storeFlushPastSetup: schedule.storeFlushPastSetup, + syncPath: path, + verificationInterval: verificationInterval, + } + } + return []restartCase{ + cell(clSync, 0), + cell(elSync, 0), + cell(elSyncAfterFinality, 0), + cell(elSync, 1), + } +} + +// skipUnsupportedCell skips the cells the configured clients cannot run. +func skipUnsupportedCell(t devtest.T, c restartCase) { + sysgo.SkipOnOpGeth(t, "proofs history is an op-reth feature") + switch c.syncPath { + case clSync: + sysgo.SkipOnKonaNode(t, "kona-node has no CL sync mode: it always lets the EL sync the gap") + case elSyncAfterFinality: + sysgo.SkipOnKonaNode(t, "kona-node EL-syncs the gap whatever it has finalized, so this cell would repeat el_sync") + } +} + +type restartSystem struct { + Batcher *dsl.L2Batcher + Follower *dsl.L2ELNode + FollowerCL *dsl.L2CLNode + FollowerCLKey string + FollowerEnode string + FunderL2 *dsl.FunderEOA + Logs *testlog.CapturingHandler + Sequencer *dsl.L2ELNode + SequencerCL *dsl.L2CLNode +} + +// logCapturingT hands the system a logger that also captures every record it passes on, so a +// cell can check which sync path the follower's op-node logged. +type logCapturingT struct { + devtest.T + logger log.Logger +} + +func (t logCapturingT) Logger() log.Logger { return t.logger } + +// newRestartSystem starts a sequencer without proofs history, which serves as the independent +// eth_getProof reference, and a follower running proofs history under the case's reth, ExEx and +// sync-path settings. +func newRestartSystem(t devtest.T, c restartCase) *restartSystem { + logs, ok := testlog.WrapCaptureLogger(t.Logger().Handler()).(*testlog.CapturingHandler) + t.Require().True(ok, "the log capture must wrap the test's handler") + t = logCapturingT{T: t, logger: log.NewLogger(logs).WithContext(t.Ctx())} + clKind := sysgo.ResolveMixedL2CLKind() + frontends := presets.NewProofsHistoryFollowerSystem(t, func(cfg *presets.ProofsHistoryFollowerConfig) { + cfg.DeployerOptions = []sysgo.DeployerOption{sysgo.WithUniformL2BlockTimes(l2BlockTimeSeconds)} + cfg.FollowerL2CLOpts = c.syncPath.followerL2CLOpts(clKind) + cfg.FollowerOpRethOpts = []sysgo.OpRethOption{sysgo.OpRethWithExtraArgs(c.followerArgs()...)} + cfg.SequencerOpRethOpts = []sysgo.OpRethOption{sysgo.OpRethWithExtraArgs(c.engineArgs()...)} + }) + return &restartSystem{ + Batcher: frontends.L2Batcher, + Follower: frontends.Follower.EL, + FollowerCL: frontends.Follower.CL, + FollowerCLKey: frontends.Follower.Spec.CLKey, + FollowerEnode: frontends.Follower.EL.Enode(), + FunderL2: frontends.FunderL2, + Logs: logs, + Sequencer: frontends.Sequencer.EL, + SequencerCL: frontends.Sequencer.CL, + } +} + +// uncleanStop is what the follower held when it was killed: the chain head it had imported, the +// block that held the workload's last transaction, both on the chain of that moment, and the last +// block its proofs store had flushed. +type uncleanStop struct { + head eth.L2BlockRef + lastWorkloadBlock eth.L2BlockRef + storeTip uint64 +} + +// killFollower drives the workload until the follower's proofs store has flushed the case's target +// block, holds the chain at the case's kill point right after one more workload transaction and +// kills the follower's op-reth there. The sequencer is left stopped and the follower's CL running. +func killFollower(t devtest.T, sys *restartSystem, w *dsl.BlockRecorderWorkload, c restartCase) uncleanStop { + storeFlushTarget := w.SetupBlock() + c.storeFlushPastSetup + w.DriveUntilBlock(storeFlushTarget) + window := sys.Follower.WaitForProofsStoreBlock(storeFlushTarget, storeCatchUpTimeout) + lastWorkloadTarget := *window.Latest + if c.killPoint == killMidStoreBuffer { + lastWorkloadTarget += 2 + } + stop := holdChainAtKillPoint(t, sys, w, c, w.DriveUntilBlock(lastWorkloadTarget)) + t.Logger().Info("Killing the follower", "head", stop.head, "proofs_store_tip", stop.storeTip, + "last_workload_block", stop.lastWorkloadBlock, "kill_point", c.killPoint) + sys.Follower.StopUngracefully() + return stop +} + +// holdChainAtKillPoint holds the chain at a head that fits the case's kill point. Reth persists +// only up to memoryBlockBufferTarget blocks below its head, so a head fewer blocks than that past +// the workload's last transaction keeps that block out of reth's datadir; a mid-buffer kill also +// needs a head the proofs store has not flushed. While the held head misses either, the sequencer +// resumes for one more workload transaction and the chain is held again. +func holdChainAtKillPoint(t devtest.T, sys *restartSystem, w *dsl.BlockRecorderWorkload, c restartCase, lastWorkloadBlockNumber uint64) uncleanStop { + stop, err := retry.Do(t.Ctx(), killPointHoldAttempts, retry.Fixed(0), func() (uncleanStop, error) { + stop := uncleanStop{head: holdChain(t, sys)} + stop.lastWorkloadBlock = sys.Follower.BlockRefByNumber(lastWorkloadBlockNumber) + // Read last, right before the kill: op-reth flushes a buffer left idle for 10s, and a flush + // of the held head turns a mid-buffer kill into an after-flush one. + stop.storeTip = proofsStoreTip(t, sys.Follower) + mismatch := killPointMismatch(c, stop) + if mismatch == "" { + return stop, nil + } + t.Logger().Info("Held head does not fit the kill point; resuming the sequencer for another workload transaction", + "head", stop.head, "proofs_store_tip", stop.storeTip, "last_workload_block", stop.lastWorkloadBlock, + "mismatch", mismatch) + sys.SequencerCL.StartSequencer() + lastWorkloadBlockNumber = w.DriveUntilBlock(stop.head.Number + 1) + return uncleanStop{}, errors.New(mismatch) + }) + t.Require().NoErrorf(err, "the chain never held at a head fitting the %s kill point", c.killPoint) + return stop +} + +// killPointMismatch says why the held chain does not fit the case's kill point, or returns "" when +// it does. +func killPointMismatch(c restartCase, stop uncleanStop) string { + if blocksPastWorkload := stop.head.Number - stop.lastWorkloadBlock.Number; blocksPastWorkload >= memoryBlockBufferTarget { + return fmt.Sprintf("the held head %d is %d blocks past the workload's last block %d, so reth may persist that block", + stop.head.Number, blocksPastWorkload, stop.lastWorkloadBlock.Number) + } + if c.killPoint == killMidStoreBuffer && stop.storeTip >= stop.head.Number { + return fmt.Sprintf("the proofs store flushed the held head %d (store tip %d)", stop.head.Number, stop.storeTip) + } + return "" +} + +// holdChain stops block production, waits for the follower to import the last block and returns +// that head once the proofs store can no longer flush on block count: it flushes once +// proofsStoreFlushBlocks blocks are buffered, so a tip within that many blocks of the held head +// stays put until op-reth's 10s idle flush. +func holdChain(t devtest.T, sys *restartSystem) eth.L2BlockRef { + heldHead := sys.Sequencer.BlockRefByHash(sys.SequencerCL.StopSequencer()) + sys.Follower.Reached(eth.Unsafe, heldHead.Number, followerImportAttempts) + head := sys.Follower.BlockRefByLabel(eth.Unsafe) + t.Require().Equalf(heldHead.ID(), head.ID(), "the follower must hold the sequencer's head %s, but holds %s", heldHead, head) + sys.Follower.WaitForProofsStoreBlock(head.Number-(proofsStoreFlushBlocks-1), storeCatchUpTimeout) + return head +} + +func proofsStoreTip(t devtest.T, el *dsl.L2ELNode) uint64 { + status, err := el.ProofsSyncStatus(t.Ctx()) + t.Require().NoErrorf(err, "debug_proofsSyncStatus on %s failed", el) + t.Require().NotNilf(status.Latest, "the proofs store on %s has indexed no block", el) + return *status.Latest +} + +// restartFollower brings the follower's op-reth and CL back and re-peers them with the sequencer, +// since both peerings died with the processes. It returns the block reth resumes from, its datadir +// tip, which must be below the proofs store's tip for the test to exercise the crash gap at all. +// It clears the captured logs before the follower's CL starts, so the follower op-node records +// left in them are the restarted node's. +func restartFollower(t devtest.T, sys *restartSystem, stop uncleanStop) eth.L2BlockRef { + // The follower keeps its node ID but listens on a new port, so the sequencer's trusted entry + // would keep redialing the old one, and reach the restarted follower if the OS reuses that + // port: a redial that crosses the follower's own dial makes reth tear down both sessions. + sys.Sequencer.ForgetPeer(sys.FollowerEnode) + sys.Follower.Start() + sys.Follower.WaitForOnline() + // Nothing feeds the EL until its CL is back, so its head is what reth had persisted. + persistedTip := sys.Follower.BlockRefByLabel(eth.Unsafe) + t.Logger().Info("Follower op-reth is back on its datadir", "persisted_tip", persistedTip, + "proofs_store_tip", stop.storeTip, "head_at_kill", stop.head) + t.Require().Lessf(persistedTip.Number, stop.storeTip, + "the proofs store (tip %d) must be ahead of reth's datadir (tip %d) after the kill; the kill landed too late for this threshold", + stop.storeTip, persistedTip.Number) + + sys.Follower.PeerWith(sys.Sequencer) + sys.Logs.Clear() + sys.FollowerCL.Start() + sys.FollowerCL.ConnectPeer(sys.SequencerCL) + return persistedTip +} + +// verifyFollowerSyncPath requires the restarted op-node to have logged the sync path the cell +// claims: el_sync must finish an EL sync, el_sync_after_finality must skip it for CL sync. op-node +// logs either outcome no later than its first block past the head it caught up to, so once the +// follower has indexed the post-restart traffic the record is in the capture. kona-node is not +// checked: it always EL-syncs and logs neither. +func verifyFollowerSyncPath(t devtest.T, sys *restartSystem, c restartCase) { + if sysgo.ResolveMixedL2CLKind() != sysgo.MixedL2CLOpNode { + return + } + var expectedMessage string + switch c.syncPath { + case elSync: + expectedMessage = "Finished EL sync" + case elSyncAfterFinality: + expectedMessage = "Skipping EL sync" + default: + return + } + record := sys.Logs.FindLog( + testlog.NewAttributesFilter("component", "l2cl-"+sys.FollowerCLKey), + testlog.NewMessageContainsFilter(expectedMessage), + ) + t.Require().NotNilf(record, "the restarted follower op-node never logged %q, so it did not take the %s path", expectedMessage, c.syncPath) +} + +// Each kill schedule is a top-level test of its own, so its four cells, which each start a devnet +// and run one after another, get a go test timeout to themselves. + +func TestProofsHistoryFollowerRecoversFromUncleanShutdownAfterStoreFlush(gt *testing.T) { + testFollowerRecoversFromUncleanShutdown(gt, afterStoreFlushSchedule) +} + +func TestProofsHistoryFollowerRecoversFromUncleanShutdownMidStoreBuffer(gt *testing.T) { + testFollowerRecoversFromUncleanShutdown(gt, midStoreBufferSchedule) +} + +func TestProofsHistoryFollowerServesRebuiltBlocksAfterUncleanShutdownAfterStoreFlush(gt *testing.T) { + testFollowerServesRebuiltBlocksAfterUncleanShutdown(gt, afterStoreFlushSchedule) +} + +func TestProofsHistoryFollowerServesRebuiltBlocksAfterUncleanShutdownMidStoreBuffer(gt *testing.T) { + testFollowerServesRebuiltBlocksAfterUncleanShutdown(gt, midStoreBufferSchedule) +} + +// testFollowerRecoversFromUncleanShutdown kills a proofs-history follower once its store has +// flushed blocks reth has not yet persisted, restarts it into the same chain and requires it to +// catch up with the sequencer, keep indexing, and serve proofs that match the sequencer's for the +// blocks it lost, the blocks it re-imported and the blocks that followed. The follower catches up +// by CL sync, by EL sync, or by the CL sync an EL sync config without post-finalization EL sync +// falls back to once a block past genesis is finalized. +func testFollowerRecoversFromUncleanShutdown(gt *testing.T, schedule killSchedule) { + t := devtest.ParallelT(gt) + devtest.RunParallel(t, restartMatrix(schedule), func(t devtest.T, c restartCase) { + skipUnsupportedCell(t, c) + sys := newRestartSystem(t, c) + w := dsl.NewBlockRecorderWorkload(t, sys.FunderL2) + + stop := killFollower(t, sys, w, c) + sys.SequencerCL.StartSequencer() + sys.FollowerCL.Stop() + persistedTip := restartFollower(t, sys, stop) + + sys.Follower.InSync(sys.Sequencer, safety.LocalUnsafe, resyncAttempts) + lastBlock := w.DriveUntilBlock(sys.Sequencer.BlockRefByLabel(eth.Unsafe).Number + postRestartTrafficBlocks) + sys.Follower.VerifyHistoricalProofsMatch(sys.Sequencer, func(opts *dsl.HistoricalProofsOpts) { + opts.Accounts = w.Accounts() + opts.FirstBlock = persistedTip.Number + opts.LastBlock = lastBlock + }) + verifyFollowerSyncPath(t, sys, c) + }) +} + +// testFollowerServesRebuiltBlocksAfterUncleanShutdown kills a proofs-history follower, then kills +// the sequencer's op-reth too while the blocks the follower's store holds are still only in the +// sequencer's memory, so the restarted sequencer rebuilds those heights with different contents. +// The restarted follower must follow the rebuilt chain and serve its proofs, not the ones its +// store recorded for the chain that no longer exists. +func testFollowerServesRebuiltBlocksAfterUncleanShutdown(gt *testing.T, schedule killSchedule) { + t := devtest.ParallelT(gt) + devtest.RunParallel(t, restartMatrix(schedule), func(t devtest.T, c restartCase) { + skipUnsupportedCell(t, c) + sys := newRestartSystem(t, c) + // Blocks that reached L1 would be derived back as they were; blocks that never did + // are re-sequenced, which is the rebuild this test needs. + sys.Batcher.Stop() + w := dsl.NewBlockRecorderWorkload(t, sys.FunderL2) + + // Both op-reths die back to back on the held chain, so the sequencer still holds the + // blocks the follower's store flushed; the CLs follow once the processes are gone. The + // restarted sequencer CL sequences again. + stop := killFollower(t, sys, w, c) + sys.Sequencer.StopUngracefully() + sys.FollowerCL.Stop() + sys.SequencerCL.Stop() + + sys.Sequencer.Start() + sys.Sequencer.WaitForOnline() + sequencerPersistedTip := sys.Sequencer.BlockRefByLabel(eth.Unsafe) + t.Logger().Info("Sequencer op-reth is back on its datadir", "persisted_tip", sequencerPersistedTip, + "follower_proofs_store_tip", stop.storeTip, "last_workload_block", stop.lastWorkloadBlock, + "workload_setup_block", w.SetupBlock()) + t.Require().Lessf(sequencerPersistedTip.Number, stop.storeTip, + "the sequencer must rebuild blocks the follower's proofs store holds, but its datadir tip %d is not below the store tip %d", + sequencerPersistedTip.Number, stop.storeTip) + // A lost block without a workload transaction can come back identical, so only losing the + // workload's last block guarantees the rebuilt chain differs. + t.Require().Lessf(sequencerPersistedTip.Number, stop.lastWorkloadBlock.Number, + "the block %d that held the workload's last transaction must be among the sequencer's lost blocks, but its datadir tip is %d", + stop.lastWorkloadBlock.Number, sequencerPersistedTip.Number) + t.Require().LessOrEqualf(w.SetupBlock(), sequencerPersistedTip.Number, + "the workload's funding and contract (block %d) must survive the sequencer's crash, but its datadir tip is %d", + w.SetupBlock(), sequencerPersistedTip.Number) + sys.SequencerCL.Start() + sys.Sequencer.ReorgTriggered(stop.lastWorkloadBlock, rebuildAttempts) + + persistedTip := restartFollower(t, sys, stop) + + sys.Follower.InSync(sys.Sequencer, safety.LocalUnsafe, resyncAttempts) + lastBlock := w.DriveUntilBlock(sys.Sequencer.BlockRefByLabel(eth.Unsafe).Number + postRestartTrafficBlocks) + sys.Follower.VerifyHistoricalProofsMatch(sys.Sequencer, func(opts *dsl.HistoricalProofsOpts) { + opts.Accounts = w.Accounts() + // The follower's datadir can outlast the sequencer's, so start at whichever comes + // first: its own tip or the first height the sequencer rebuilt. + opts.FirstBlock = min(persistedTip.Number, sequencerPersistedTip.Number+1) + opts.LastBlock = lastBlock + }) + verifyFollowerSyncPath(t, sys, c) + }) +} diff --git a/op-devstack/dsl/block_recorder_workload.go b/op-devstack/dsl/block_recorder_workload.go new file mode 100644 index 00000000000..1d2d7a64e71 --- /dev/null +++ b/op-devstack/dsl/block_recorder_workload.go @@ -0,0 +1,89 @@ +package dsl + +import ( + "time" + + "github.com/ethereum-optimism/optimism/op-devstack/devtest" + "github.com/ethereum-optimism/optimism/op-service/eth" + "github.com/ethereum-optimism/optimism/op-service/retry" + "github.com/ethereum-optimism/optimism/op-service/txplan" + "github.com/ethereum/go-ethereum/common" +) + +// BlockRecorderWorkload changes state in nearly every block: a writer and a counterparty transfer +// back and forth and the writer calls a block recorder contract, so every block it fills carries +// balance and storage changes that a rebuilt block would not repeat. +type BlockRecorderWorkload struct { + commonImpl + + counterparty *EOA + inclusionPoll txplan.Option + recorder common.Address + // setupBlock is the block that completed the funding and the contract deployment; a chain that + // lacks it lacks the workload's accounts. + setupBlock uint64 + writer *EOA +} + +// NewBlockRecorderWorkload funds two senders, deploys a block recorder and calls it once. +func NewBlockRecorderWorkload(t devtest.T, funder *FunderEOA) *BlockRecorderWorkload { + w := &BlockRecorderWorkload{ + commonImpl: commonFromT(t), + counterparty: funder.NewFundedEOA(eth.OneEther), + inclusionPoll: txplan.Combine(), + writer: funder.NewFundedEOA(eth.OneEther), + } + w.recorder = w.writer.DeployBlockRecorder() + w.setupBlock = IncludedBlockNumber(t, w.writer.Transact(w.writer.Plan(), txplan.WithTo(&w.recorder))) + w.log.Info("Block recorder workload set up", "recorder", w.recorder, "setup_block", w.setupBlock) + return w +} + +// WithFreshSenders returns a workload that writes to the same contract from two newly funded +// accounts. A node that reorgs puts the transactions of the blocks it drops back into its pool, so +// new transactions from the original senders could race those for the same nonces. +func (w *BlockRecorderWorkload) WithFreshSenders(funder *FunderEOA) *BlockRecorderWorkload { + fresh := *w + fresh.counterparty = funder.NewFundedEOA(eth.OneEther) + fresh.writer = funder.NewFundedEOA(eth.OneEther) + return &fresh +} + +// WithInclusionPoll returns a workload that polls el for its transactions' receipts every +// interval, up to maxAttempts times, so DriveUntilBlock returns within about interval of the block +// that held its last write rather than after the default exponential backoff. +func (w *BlockRecorderWorkload) WithInclusionPoll(el *L2ELNode, interval time.Duration, maxAttempts int) *BlockRecorderWorkload { + polled := *w + polled.inclusionPoll = txplan.WithRetryInclusion(el.EthClient(), maxAttempts, retry.Fixed(interval)) + return &polled +} + +// Accounts are the senders and the recorder with the storage slots it writes. +func (w *BlockRecorderWorkload) Accounts() []ProofAccount { + return append(w.SenderAccounts(), + ProofAccount{Address: w.recorder, StorageKeys: []common.Hash{common.HexToHash("0x0"), common.HexToHash("0x1")}}) +} + +func (w *BlockRecorderWorkload) SenderAccounts() []ProofAccount { + return []ProofAccount{{Address: w.counterparty.Address()}, {Address: w.writer.Address()}} +} + +func (w *BlockRecorderWorkload) SetupBlock() uint64 { + return w.setupBlock +} + +// DriveUntilBlock sends transactions one after another until a contract write lands in a block at +// or past target, and returns that block. +func (w *BlockRecorderWorkload) DriveUntilBlock(target uint64) uint64 { + w.log.Info("Driving the block recorder workload", "target", target) + for { + w.counterparty.Transact(w.counterparty.PlanTransfer(w.writer.Address(), eth.OneHundredthEther), w.inclusionPoll) + w.writer.Transact(w.writer.PlanTransfer(w.counterparty.Address(), eth.OneHundredthEther), w.inclusionPoll) + write := w.writer.Transact(w.writer.Plan(), txplan.WithTo(&w.recorder), w.inclusionPoll) + block := IncludedBlockNumber(w.t, write) + if block >= target { + w.log.Info("Block recorder workload reached target", "target", target, "last_write_block", block) + return block + } + } +} diff --git a/op-devstack/dsl/eoa.go b/op-devstack/dsl/eoa.go index 13ba617abef..3efb159f57d 100644 --- a/op-devstack/dsl/eoa.go +++ b/op-devstack/dsl/eoa.go @@ -11,6 +11,7 @@ import ( "github.com/ethereum-optimism/optimism/op-acceptance-tests/tests/interop" messages "github.com/ethereum-optimism/optimism/op-core/interop/messages" "github.com/ethereum-optimism/optimism/op-core/predeploys" + "github.com/ethereum-optimism/optimism/op-devstack/devtest" e2eBindings "github.com/ethereum-optimism/optimism/op-e2e/bindings" "github.com/ethereum-optimism/optimism/op-service/bigs" "github.com/ethereum-optimism/optimism/op-service/eth" @@ -22,6 +23,8 @@ import ( "github.com/ethereum-optimism/optimism/op-service/txplan" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/core/vm" + "github.com/ethereum/go-ethereum/core/vm/program" "github.com/ethereum/go-ethereum/crypto" ) @@ -252,6 +255,30 @@ func (u *EOA) DeployEventLogger() common.Address { return eventLoggerAddress } +// DeployBlockRecorder deploys a contract that writes the block number to storage slot 0 and the +// block timestamp to slot 1 on every call, so every call to it changes its storage. +func (u *EOA) DeployBlockRecorder() common.Address { + runtimeCode := program.New(). + Op(vm.NUMBER).Push(0).Op(vm.SSTORE). + Op(vm.TIMESTAMP).Push(1).Op(vm.SSTORE). + Bytes() + initCode := program.New().ReturnViaCodeCopy(runtimeCode).Bytes() + tx := txplan.NewPlannedTx(u.Plan(), txplan.WithData(initCode)) + _, err := tx.Success.Eval(u.ctx) + u.t.Require().NoError(err, "block recorder deployment must succeed") + res, err := tx.Included.Eval(u.ctx) + u.t.Require().NoError(err, "failed to read the block recorder deployment receipt") + u.log.Info("deployed block recorder", "chainID", tx.ChainID.Value(), "address", res.ContractAddress) + return res.ContractAddress +} + +// IncludedBlockNumber waits for tx to be included and returns the number of the block holding it. +func IncludedBlockNumber(t devtest.T, tx *txplan.PlannedTx) uint64 { + receipt, err := tx.Included.Eval(t.Ctx()) + t.Require().NoError(err, "transaction must be included") + return bigs.Uint64Strict(receipt.BlockNumber) +} + func (u *EOA) DeployWETH() common.Address { // Use the e2e bindings which contain the WETH bytecode tx := txplan.NewPlannedTx(u.Plan(), txplan.WithData(common.FromHex(e2eBindings.WETHBin))) diff --git a/op-devstack/dsl/l2_el.go b/op-devstack/dsl/l2_el.go index a940ac0fb06..dce2ef7001f 100644 --- a/op-devstack/dsl/l2_el.go +++ b/op-devstack/dsl/l2_el.go @@ -23,6 +23,7 @@ import ( safety "github.com/ethereum-optimism/optimism/op-service/eth/safety" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/p2p" ) var emptyHash = common.Hash{} @@ -525,6 +526,15 @@ func (el *L2ELNode) Stop() { lifecycle.Stop() } +// StopUngracefully ends the node's process without a shutdown, the way a crash would, so state +// it held only in memory is lost. Start brings the node back. +func (el *L2ELNode) StopUngracefully() { + el.log.Info("Stopping ungracefully", "name", el.inner.Name()) + lifecycle, ok := el.inner.(stack.UngracefulLifecycle) + el.require.Truef(ok, "L2EL node %s cannot be stopped ungracefully", el.inner.Name()) + lifecycle.StopUngracefully() +} + func (el *L2ELNode) Start() { lifecycle, ok := el.inner.(stack.Lifecycle) el.require.Truef(ok, "L2EL node %s is not lifecycle-controllable", el.inner.Name()) @@ -539,6 +549,22 @@ func (el *L2ELNode) DisconnectPeerWith(peer *L2ELNode) { sysgo.DisconnectP2P(el.ctx, el.require, el.inner.L2EthClient().RPC(), peer.inner.L2EthClient().RPC()) } +// Enode returns the node's enode URL, which identifies it to its peers across restarts. +func (el *L2ELNode) Enode() string { + var info p2p.NodeInfo + el.require.NoErrorf(el.inner.L2EthClient().RPC().CallContext(el.ctx, &info, "admin_nodeInfo"), + "admin_nodeInfo on %s failed", el) + return info.Enode +} + +// ForgetPeer drops this node's peer entry for peerEnode, trusted status included, and waits until +// the peer is no longer connected. The peer may be down. +func (el *L2ELNode) ForgetPeer(peerEnode string) { + el.log.Info("Forgetting peer", "node", el, "peer_enode", peerEnode) + sysgo.ForgetP2PPeer(el.ctx, el.require, el.inner.L2EthClient().RPC(), peerEnode) + el.log.Info("Forgot peer", "node", el, "peer_enode", peerEnode) +} + func (el *L2ELNode) PayloadByNumber(number uint64) *eth.ExecutionPayloadEnvelope { payload, err := el.inner.L2EthClient().PayloadByNumber(el.ctx, number) el.require.NoError(err, "failed to get payload") diff --git a/op-devstack/presets/proofs_history_follower.go b/op-devstack/presets/proofs_history_follower.go index 7500b653fdd..3ca19442f8f 100644 --- a/op-devstack/presets/proofs_history_follower.go +++ b/op-devstack/presets/proofs_history_follower.go @@ -13,6 +13,7 @@ const proofsHistoryReferenceEthProofWindow = 1_000_000 // starts. type ProofsHistoryFollowerConfig struct { DeployerOptions []sysgo.DeployerOption + FollowerL2CLOpts []sysgo.L2CLOption FollowerOpRethOpts []sysgo.OpRethOption // SequencerOpRethOpts apply after the options that turn the sequencer's proofs history off and // open its eth_getProof window. @@ -57,6 +58,7 @@ func NewProofsHistoryFollowerSystem(t devtest.T, configure ...func(*ProofsHistor CLKind: clKind, ELKey: "proofs-follower", ELKind: sysgo.MixedL2ELOpReth, + L2CLOpts: cfg.FollowerL2CLOpts, OpRethOpts: cfg.FollowerOpRethOpts, }, }, diff --git a/op-devstack/presets/rpc_frontends.go b/op-devstack/presets/rpc_frontends.go index b6008086dac..d0a41c6182b 100644 --- a/op-devstack/presets/rpc_frontends.go +++ b/op-devstack/presets/rpc_frontends.go @@ -155,9 +155,13 @@ type l2ELFrontend struct { l2EngineClient *sources.EngineClient lifecycle stack.Lifecycle control stack.ControlledLifecycle + ungraceful stack.UngracefulLifecycle } -var _ stack.L2ELNode = (*l2ELFrontend)(nil) +var ( + _ stack.L2ELNode = (*l2ELFrontend)(nil) + _ stack.UngracefulLifecycle = (*l2ELFrontend)(nil) +) func newPresetL2ELNode(t devtest.T, name string, chainID eth.ChainID, userRPCCl opclient.RPC, userRPC string, engineRPCCl opclient.RPC, rollupCfg *rollup.Config) *l2ELFrontend { t.Require().NotNil(rollupCfg, "rollup config must be configured") @@ -193,6 +197,11 @@ func (r *l2ELFrontend) Stop() { r.lifecycle.Stop() } +func (r *l2ELFrontend) StopUngracefully() { + r.require().NotNil(r.ungraceful, "L2EL node %s cannot be stopped ungracefully", r.Name()) + r.ungraceful.StopUngracefully() +} + func (r *l2ELFrontend) StartControlled(ctx context.Context) error { if r.control == nil { return fmt.Errorf("L2EL node %s is not controllable", r.Name()) diff --git a/op-devstack/presets/sysgo_runtime.go b/op-devstack/presets/sysgo_runtime.go index 3ae12a67ebe..ddf7a42fe27 100644 --- a/op-devstack/presets/sysgo_runtime.go +++ b/op-devstack/presets/sysgo_runtime.go @@ -66,6 +66,9 @@ func newL2ELFrontend(t devtest.T, name string, chainID eth.ChainID, userRPC stri if control, ok := lifecycle[0].(stack.ControlledLifecycle); ok { l2EL.control = control } + if ungraceful, ok := lifecycle[0].(stack.UngracefulLifecycle); ok { + l2EL.ungraceful = ungraceful + } } return l2EL } diff --git a/op-devstack/stack/lifecycle.go b/op-devstack/stack/lifecycle.go index f26d3b7e605..22278e6444f 100644 --- a/op-devstack/stack/lifecycle.go +++ b/op-devstack/stack/lifecycle.go @@ -12,3 +12,10 @@ type ControlledLifecycle interface { StopControlled(ctx context.Context) error Running() bool } + +// UngracefulLifecycle is implemented by components that can be stopped the way a process crash +// would stop them: no shutdown runs, so state held only in process memory is lost. Writes already +// handed to the OS survive, so this does not model a power loss. +type UngracefulLifecycle interface { + StopUngracefully() +} diff --git a/op-devstack/sysgo/l2_cl.go b/op-devstack/sysgo/l2_cl.go index dc7539eb004..8e48895a89c 100644 --- a/op-devstack/sysgo/l2_cl.go +++ b/op-devstack/sysgo/l2_cl.go @@ -40,6 +40,10 @@ type L2CLConfig struct { // OffsetELSafe retracts safe and finalized from the EL-sync tip by floor(OffsetELSafe / L2BlockTime) blocks. OffsetELSafe time.Duration + // SupportsPostFinalizationELSync lets an op-node in EL sync mode EL-sync even when its EL + // already has a finalized block past genesis. When false, such a node goes straight to CL sync. + // It is off unless set, whatever the EL, while production op-node turns it on for op-reth. + SupportsPostFinalizationELSync bool // AltDAServerURL is the DA server the node reads alt-DA inputs from. Empty disables alt-DA. AltDAServerURL string @@ -57,6 +61,24 @@ func L2CLFollowSource(source string) L2CLOption { }) } +// L2CLVerifierSyncMode sets the sync mode a verifier runs: CL sync has op-node fill unsafe gaps +// from its peers, EL sync hands the forkchoice target to the EL and lets it sync from its own +// peers. +func L2CLVerifierSyncMode(mode nodeSync.Mode) L2CLOption { + return L2CLOptionFn(func(p devtest.T, _ ComponentTarget, cfg *L2CLConfig) { + cfg.VerifierSyncMode = mode + }) +} + +// L2CLPostFinalizationELSync sets whether an op-node in EL sync mode EL-syncs a restart even though +// its EL has a finalized block past genesis. Production op-node enables it for an op-reth or +// op-erigon EL and disables it for op-geth. +func L2CLPostFinalizationELSync(enabled bool) L2CLOption { + return L2CLOptionFn(func(p devtest.T, _ ComponentTarget, cfg *L2CLConfig) { + cfg.SupportsPostFinalizationELSync = enabled + }) +} + func DefaultL2CLConfig() *L2CLConfig { return &L2CLConfig{ SequencerSyncMode: nodeSync.CLSync, diff --git a/op-devstack/sysgo/l2_cl_kona_config_test.go b/op-devstack/sysgo/l2_cl_kona_config_test.go index d70dc2b7690..07a1f1e469a 100644 --- a/op-devstack/sysgo/l2_cl_kona_config_test.go +++ b/op-devstack/sysgo/l2_cl_kona_config_test.go @@ -140,6 +140,7 @@ func TestKonaL2CLConfigRejectsUnsupportedOptions(t *testing.T) { {"role override", true, func(cfg *L2CLConfig) { cfg.IsSequencer = false }, "unsupported L2CL option"}, {"sequencer sync", true, func(cfg *L2CLConfig) { cfg.SequencerSyncMode = nodeSync.Mode(99) }, "unknown effective sync mode"}, {"verifier sync", false, func(cfg *L2CLConfig) { cfg.VerifierSyncMode = nodeSync.Mode(99) }, "unknown effective sync mode"}, + {"post-finalization EL sync", false, func(cfg *L2CLConfig) { cfg.SupportsPostFinalizationELSync = true }, "unsupported L2CL option"}, {"verifier conductor", false, func(cfg *L2CLConfig) { cfg.ConductorRPC = "http://conductor:8547" }, "unsupported L2CL option"}, {"zero conductor timeout", true, func(cfg *L2CLConfig) { cfg.ConductorRPC = "http://conductor:8547" }, "timeout must be positive"}, {"fractional conductor timeout", true, func(cfg *L2CLConfig) { diff --git a/op-devstack/sysgo/l2_cl_test.go b/op-devstack/sysgo/l2_cl_test.go new file mode 100644 index 00000000000..f09da243a71 --- /dev/null +++ b/op-devstack/sysgo/l2_cl_test.go @@ -0,0 +1,30 @@ +package sysgo + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/require" + + nodeSync "github.com/ethereum-optimism/optimism/op-node/rollup/sync" +) + +func TestL2CLVerifierSyncMode(t *testing.T) { + cfg := &L2CLConfig{SequencerSyncMode: nodeSync.CLSync, VerifierSyncMode: nodeSync.CLSync} + L2CLVerifierSyncMode(nodeSync.ELSync).Apply(nil, ComponentTarget{}, cfg) + + require.Equal(t, nodeSync.ELSync, cfg.VerifierSyncMode) + require.Equal(t, nodeSync.CLSync, cfg.SequencerSyncMode, "the sequencer keeps its own sync mode") +} + +func TestL2CLPostFinalizationELSync(t *testing.T) { + require.False(t, DefaultL2CLConfig().SupportsPostFinalizationELSync, "op-node skips EL sync past finality by default") + for _, enabled := range []bool{false, true} { + t.Run(fmt.Sprintf("enabled=%t", enabled), func(t *testing.T) { + cfg := &L2CLConfig{SupportsPostFinalizationELSync: !enabled} + L2CLPostFinalizationELSync(enabled).Apply(nil, ComponentTarget{}, cfg) + + require.Equal(t, enabled, cfg.SupportsPostFinalizationELSync) + }) + } +} diff --git a/op-devstack/sysgo/l2_el_opreth.go b/op-devstack/sysgo/l2_el_opreth.go index 61a6ea5adf3..de8510295de 100644 --- a/op-devstack/sysgo/l2_el_opreth.go +++ b/op-devstack/sysgo/l2_el_opreth.go @@ -8,6 +8,7 @@ import ( "sync" "github.com/ethereum-optimism/optimism/op-devstack/devtest" + "github.com/ethereum-optimism/optimism/op-devstack/stack" "github.com/ethereum-optimism/optimism/op-service/eth" "github.com/ethereum-optimism/optimism/op-service/log" "github.com/ethereum-optimism/optimism/op-service/logpipe" @@ -132,7 +133,10 @@ type OpReth struct { l2MetricsRegistrar L2MetricsRegistrar } -var _ L2ELNode = (*OpReth)(nil) +var ( + _ L2ELNode = (*OpReth)(nil) + _ stack.UngracefulLifecycle = (*OpReth)(nil) +) func (n *OpReth) Start() { n.mu.Lock() @@ -246,6 +250,23 @@ func (n *OpReth) Stop() { n.sub = nil } +// StopUngracefully ends the op-reth process with SIGKILL, the way a crash would, so whatever it +// had not yet handed to the OS for its datadir is lost. It fails the test if op-reth had already +// exited on its own, since that crash is not the stop the test asked for. Start brings the node +// back on the same datadir. +func (n *OpReth) StopUngracefully() { + n.mu.Lock() + defer n.mu.Unlock() + if n.sub == nil { + n.p.Logger().Warn("op-reth already stopped") + return + } + n.p.Logger().Info("Stopping op-reth ungracefully", "name", n.name, "chain", n.chainID) + n.clearProxyUpstreams() + n.p.Require().NoError(n.sub.Kill(), "failed to kill op-reth") + n.sub = nil +} + // Callers must hold n.mu. func (n *OpReth) clearProxyUpstreams() { if n.userProxy != nil { diff --git a/op-devstack/sysgo/l2_el_p2p_util.go b/op-devstack/sysgo/l2_el_p2p_util.go index a0ccee24176..e0f9b84aa19 100644 --- a/op-devstack/sysgo/l2_el_p2p_util.go +++ b/op-devstack/sysgo/l2_el_p2p_util.go @@ -162,6 +162,30 @@ func DisconnectP2P(ctx context.Context, require *testreq.Assertions, initiator R require.NoError(err, "The peer was not removed") } +// ForgetP2PPeer drops node's peer entry for peerEnode and waits until node no longer lists that +// peer. It needs only node to be reachable, so it can forget a peer that is down. The trusted +// status goes first: reth's admin_removePeer leaves a trusted entry in place. Reth answers both +// removals with true whether or not it held the entry, so only admin_peers shows the outcome. +func ForgetP2PPeer(ctx context.Context, require *testreq.Assertions, node RpcCaller, peerEnode string) { + forgottenNode, err := enode.ParseV4(peerEnode) + require.NoError(err, "failed to parse the enode of the peer to forget") + forgottenID := forgottenNode.ID().String() + + for _, method := range []string{"admin_removeTrustedPeer", "admin_removePeer"} { + var removed bool + require.NoErrorf(node.CallContext(ctx, &removed, method, peerEnode), "%s failed", method) + } + + waitCtx, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + err = forPeerList(waitCtx, node, func(peers []peer) bool { + return !slices.ContainsFunc(peers, func(p peer) bool { + return strings.TrimPrefix(strings.ToLower(p.ID), "0x") == forgottenID + }) + }) + require.NoError(err, "the forgotten peer is still connected") +} + type peer struct { ID string `json:"id"` } diff --git a/op-devstack/sysgo/mixed_runtime.go b/op-devstack/sysgo/mixed_runtime.go index bdb6e8b0df6..fa910730607 100644 --- a/op-devstack/sysgo/mixed_runtime.go +++ b/op-devstack/sysgo/mixed_runtime.go @@ -176,6 +176,10 @@ type MixedSingleChainNodeSpec struct { // by deriving from L1. Used to exercise the derivation/force-build path (FCU-with-attributes) // rather than the consolidation path. Defaults to false (fully peered). IsolateFromL2P2P bool + // L2CLOpts are per-node L2 CL options applied after the runtime's own, e.g. + // L2CLVerifierSyncMode for a follower that must EL-sync. kona-node rejects the options it + // cannot honor. + L2CLOpts []L2CLOption // OpRethOpts are per-node op-reth options applied AFTER the shared cfg.OpRethOptions, only to // this node's EL. Use for options that must not reach every node — e.g. a per-node binary // override (OpRethWithBinary) or a flag a single node should receive while its peers reject it. @@ -286,10 +290,10 @@ func NewMixedSingleChainRuntime(t devtest.T, cfg MixedSingleChainPresetConfig) * IsSequencer: spec.IsSequencer, NoDiscovery: true, DependencySet: depSet, - L2CLOptions: []L2CLOption{L2CLOptionFn(func(_ devtest.T, _ ComponentTarget, cfg *L2CLConfig) { + L2CLOptions: append([]L2CLOption{L2CLOptionFn(func(_ devtest.T, _ ComponentTarget, cfg *L2CLConfig) { cfg.SafeDBPath = safeDBPath cfg.VerifierL1Confs = spec.VerifierL1Confs - })}, + })}, spec.L2CLOpts...), }) case MixedL2CLKona: cl = startMixedKonaNode( @@ -304,12 +308,12 @@ func NewMixedSingleChainRuntime(t devtest.T, cfg MixedSingleChainPresetConfig) * spec.ELKey, konaNodeStartConfig{ DependencySet: depSet, - L2CLConfig: resolveKonaL2CLConfig(t, spec.CLKey, l2Net.ChainID(), spec.IsSequencer, "", []L2CLOption{ + L2CLConfig: resolveKonaL2CLConfig(t, spec.CLKey, l2Net.ChainID(), spec.IsSequencer, "", append([]L2CLOption{ L2CLOptionFn(func(_ devtest.T, _ ComponentTarget, cfg *L2CLConfig) { cfg.SafeDBPath = safeDBPath cfg.VerifierL1Confs = spec.VerifierL1Confs }), - }), + }, spec.L2CLOpts...)), }, ) default: diff --git a/op-devstack/sysgo/singlechain_build.go b/op-devstack/sysgo/singlechain_build.go index 5e8a3533be1..e2fff1cb089 100644 --- a/op-devstack/sysgo/singlechain_build.go +++ b/op-devstack/sysgo/singlechain_build.go @@ -415,7 +415,7 @@ func startL2CLNode( Sync: nodeSync.Config{ SyncMode: syncMode, SkipSyncStartCheck: false, - SupportsPostFinalizationELSync: false, + SupportsPostFinalizationELSync: cfg.SupportsPostFinalizationELSync, L2FollowSourceEndpoint: cfg.FollowSource, // Mirror op-node/service.go: a follow-mode sequencer needs a single // initial engine reset to trigger block building (TryInitialResetEngineForSequencer). diff --git a/op-devstack/sysgo/subproc.go b/op-devstack/sysgo/subproc.go index f86f582922f..1f216412584 100644 --- a/op-devstack/sysgo/subproc.go +++ b/op-devstack/sysgo/subproc.go @@ -7,6 +7,7 @@ import ( "os" "os/exec" "sync" + "syscall" "time" "github.com/ethereum-optimism/optimism/op-devstack/devtest" @@ -142,6 +143,62 @@ func (sp *SubProcess) stop(ctx context.Context, interrupt bool, interruptWait ti return sp.completeStopLocked(interrupt, sp.waitErr) } +// Kill ends the process with SIGKILL, so none of its shutdown logic runs: a crash, not a power +// loss. A process that had already exited on its own is released, and Kill returns that exit as +// an error. +func (sp *SubProcess) Kill() error { + sp.mu.Lock() + defer sp.mu.Unlock() + if sp.cmd == nil { + return nil // already stopped + } + + select { + case <-sp.exited: + return sp.releaseExitedBeforeKillLocked() + default: + } + + sp.p.Logger().Info("Killing sub-process", "pid", sp.cmd.Process.Pid) + killErr := sp.cmd.Process.Kill() + if killErr != nil && !errors.Is(killErr, os.ErrProcessDone) { + sp.p.Logger().Error("Failed to kill sub-process", "pid", sp.cmd.Process.Pid, "err", killErr) + return fmt.Errorf("failed to kill sub-process: %w", killErr) + } + if !waitProcess(context.Background(), sp.exited, controlledKillWait) { + sp.p.Logger().Error("Sub-process did not exit after SIGKILL", "pid", sp.cmd.Process.Pid, "wait", controlledKillWait) + return fmt.Errorf("sub-process did not exit within %s of SIGKILL", controlledKillWait) + } + // The process can also exit on its own after the check above: Wait either reaps it before the + // signal (ErrProcessDone) or reaps it after the signal landed on the zombie, changing nothing. + if errors.Is(killErr, os.ErrProcessDone) || !terminatedBySIGKILL(sp.waitErr) { + return sp.releaseExitedBeforeKillLocked() + } + sp.p.Logger().Info("Sub-process killed", "exit", sp.waitErr) + sp.releaseLocked() + return nil +} + +// releaseExitedBeforeKillLocked releases a process that exited on its own and returns the error +// naming that exit. +func (sp *SubProcess) releaseExitedBeforeKillLocked() error { + sp.p.Logger().Error("Sub-process exited before it could be killed", "exit", sp.waitErr) + sp.releaseLocked() + if sp.waitErr == nil { + return errors.New("sub-process exited with status 0 before it could be killed") + } + return fmt.Errorf("sub-process exited before it could be killed: %w", sp.waitErr) +} + +func terminatedBySIGKILL(waitErr error) bool { + var exitErr *exec.ExitError + if !errors.As(waitErr, &exitErr) { + return false + } + status, ok := exitErr.Sys().(syscall.WaitStatus) + return ok && status.Signaled() && status.Signal() == syscall.SIGKILL +} + func (sp *SubProcess) completeStopLocked(interrupt bool, waitErr error) error { var exitErr *exec.ExitError if waitErr != nil && !(interrupt && errors.As(waitErr, &exitErr)) { @@ -151,7 +208,12 @@ func (sp *SubProcess) completeStopLocked(interrupt bool, waitErr error) error { } else { sp.p.Logger().Info("Sub-process gracefully exited") } + sp.releaseLocked() + return nil +} +// releaseLocked closes the log pipes and forgets the exited process so Start can run a new one. +func (sp *SubProcess) releaseLocked() { if sp.stdOutProc != nil { _ = sp.stdOutProc.Close() sp.stdOutProc = nil @@ -161,7 +223,6 @@ func (sp *SubProcess) completeStopLocked(interrupt bool, waitErr error) error { sp.stdErrProc = nil } sp.cmd = nil - return nil } func waitProcess(ctx context.Context, exited <-chan struct{}, timeout time.Duration) bool { diff --git a/op-devstack/sysgo/subproc_test.go b/op-devstack/sysgo/subproc_test.go index e0173db840b..fb8955b78c4 100644 --- a/op-devstack/sysgo/subproc_test.go +++ b/op-devstack/sysgo/subproc_test.go @@ -2,6 +2,9 @@ package sysgo import ( "context" + "io" + "os" + "os/exec" "testing" "time" @@ -64,7 +67,7 @@ func TestSubProcessConcurrentStop(gt *testing.T) { }) sp := NewSubProcess(p, logCallback, logCallback) - require.NoError(gt, sp.Start("/bin/sh", []string{"-c", "trap '' INT; exec /bin/sleep 10000000000"}, []string{})) + require.NoError(gt, sp.Start("/bin/sh", []string{"-c", "trap '' INT; exec /bin/sleep 3600"}, []string{})) start := make(chan struct{}) errCh := make(chan error, 2) @@ -85,6 +88,108 @@ func TestSubProcessConcurrentStop(gt *testing.T) { require.NoError(gt, sp.Stop(false)) } +func newTestSubProcess(gt *testing.T) (*SubProcess, *testlog.CapturingHandler) { + tLog := testlog.Logger(gt, log.LevelInfo) + logger, capt := testlog.CaptureLogger(gt, log.LevelInfo) + p := devtest.NewP(context.Background(), logger, func(bool) { panic("fail") }, func() { panic("skip") }) + gt.Cleanup(p.Close) + logCallback := logpipe.LogCallback(func(line []byte) { + logger.Info(string(line)) + tLog.Info("Sub-process logged message", "line", string(line)) + }) + return NewSubProcess(p, logCallback, logCallback), capt +} + +func TestSubProcessKillEndsProcessThatIgnoresInterrupts(gt *testing.T) { + sp, capt := newTestSubProcess(gt) + require.NoError(gt, sp.Start("/bin/sh", []string{"-c", "trap '' INT TERM; exec /bin/sleep 3600"}, []string{})) + + require.NoError(gt, sp.Kill()) + + require.Nil(gt, capt.FindLog(testlog.NewMessageFilter("Sending interrupt")), "Kill must not interrupt first") + require.NotNil(gt, capt.FindLog(testlog.NewMessageFilter("Sub-process killed"))) + + require.NoError(gt, sp.Start("/bin/echo", []string{"restart ok"}, []string{}), "the sub-process must restart after a kill") + require.NoError(gt, sp.Stop(false)) + require.NotNil(gt, capt.FindLog(testlog.NewMessageFilter("restart ok"))) +} + +func TestSubProcessKillOfExitedProcess(gt *testing.T) { + sp, capt := newTestSubProcess(gt) + require.NoError(gt, sp.Start("/bin/echo", []string{"done"}, []string{})) + <-sp.Exited() + + require.ErrorContains(gt, sp.Kill(), "exited with status 0 before it could be killed", "an exit of its own must fail the kill") + require.NotNil(gt, capt.FindLog(testlog.NewMessageFilter("Sub-process exited before it could be killed"))) + require.Nil(gt, capt.FindLog(testlog.NewMessageFilter("Sub-process killed")), "an exit of its own must not read as a kill") + require.NoError(gt, sp.Kill(), "a second kill is a no-op") +} + +func TestSubProcessKillOfCrashedProcess(gt *testing.T) { + sp, _ := newTestSubProcess(gt) + require.NoError(gt, sp.Start("/bin/sh", []string{"-c", "exit 3"}, []string{})) + <-sp.Exited() + + err := sp.Kill() + var exitErr *exec.ExitError + require.ErrorAs(gt, err, &exitErr, "the kill must report the exit it found") + require.Equal(gt, 3, exitErr.ExitCode()) + require.NoError(gt, sp.Start("/bin/echo", []string{"restart ok"}, []string{}), "the sub-process must restart after a failed kill") + require.NoError(gt, sp.Stop(false)) +} + +func TestSubProcessKillOfProcessThatExitedBeforeItWasReaped(gt *testing.T) { + sp, capt := newTestSubProcess(gt) + childExitedReader, childStdout, err := os.Pipe() + require.NoError(gt, err) + defer childExitedReader.Close() + cmd := exec.Command("/bin/sh", "-c", "exit 3") + cmd.Stdout = childStdout + require.NoError(gt, cmd.Start()) + require.NoError(gt, childStdout.Close()) + // The pipe reaches EOF once the child has exited, while it stays unreaped until reap closes. + _, err = io.ReadAll(childExitedReader) + require.NoError(gt, err) + + reap := make(chan struct{}) + sp.cmd, sp.exited = cmd, make(chan struct{}) + go func() { + <-reap + sp.waitErr = cmd.Wait() + close(sp.exited) + }() + killErrCh := make(chan error, 1) + go func() { killErrCh <- sp.Kill() }() + require.Eventually(gt, func() bool { + return capt.FindLog(testlog.NewMessageFilter("Killing sub-process")) != nil + }, 5*time.Second, 10*time.Millisecond, "the kill must reach the unreaped process") + close(reap) + + killErr := <-killErrCh + var exitErr *exec.ExitError + require.ErrorAs(gt, killErr, &exitErr, "the kill must report the exit that preceded it") + require.Equal(gt, 3, exitErr.ExitCode()) + require.Nil(gt, capt.FindLog(testlog.NewMessageFilter("Sub-process killed")), "an exit of its own must not read as a kill") +} + +func TestTerminatedBySIGKILL(gt *testing.T) { + for _, testCase := range []struct { + name string + script string + want bool + }{ + {name: "exit_0", script: "exit 0", want: false}, + {name: "exit_3", script: "exit 3", want: false}, + {name: "sigkill", script: "kill -KILL $$", want: true}, + {name: "sigterm", script: "kill -TERM $$", want: false}, + } { + gt.Run(testCase.name, func(gt *testing.T) { + waitErr := exec.Command("/bin/sh", "-c", testCase.script).Run() + require.Equal(gt, testCase.want, terminatedBySIGKILL(waitErr), "wait result: %v", waitErr) + }) + } +} + // testEcho tests that we can handle a sub-process that completes on its own func testEcho(gt *testing.T, capt *testlog.CapturingHandler, sp *SubProcess) { require.NoError(gt, sp.Start("/bin/echo", []string{"hello world"}, []string{}))