From 7f21b304202ef22b8910fb185830d2437480cb28 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Thu, 12 Nov 2026 18:24:23 +0100 Subject: [PATCH] op-acceptance-tests,op-devstack: cover proofs history across an unclean shutdown SubProcess.Kill sends SIGKILL, waits a bounded time for the exit and fails unless the process died of that signal, so a crash before a deliberate kill fails the caller; OpReth.StopUngracefully, stack.UngracefulLifecycle and L2ELNode.StopUngracefully expose it. MixedSingleChainNodeSpec gains per-node L2CLOpts, with L2CLVerifierSyncMode and L2CLPostFinalizationELSync, and kona-node rejects options it cannot honor. The DSL gains Enode and ForgetPeer, so a node can drop a peer's trusted entry before it dials again, and BlockRecorderWorkload, the transfer and block-recorder load the restart, upgrade and crash tests share. Four top-level tests, one per test and kill schedule, run four cells each: the index proofs path under cl_sync, el_sync (post-finalization EL sync on) and el_sync_after_finality, and the execute path under el_sync. A proofs-history follower is held at a block its store has not flushed and killed there, restarted, and must catch up and serve proofs matching the sequencer's across the gap; under op-node the EL-sync cells also require op-node's log to show the sync path the cell names. In the rebuilt-chain tests both op-reths are killed back to back so the lost heights, which include the last workload block, are rebuilt with different contents, and the follower must serve the rebuilt chain's proofs from below the sequencer's persisted tip. Co-Authored-By: Claude Fable 5.1 Co-Authored-By: Claude Opus 5.5 --- .../proofs_history_match_test.go | 28 +- .../proofs_history_restart_test.go | 470 ++++++++++++++++++ op-devstack/dsl/block_recorder_workload.go | 89 ++++ op-devstack/dsl/eoa.go | 27 + op-devstack/dsl/l2_el.go | 26 + .../presets/proofs_history_follower.go | 2 + op-devstack/presets/rpc_frontends.go | 11 +- op-devstack/presets/sysgo_runtime.go | 3 + op-devstack/stack/lifecycle.go | 7 + op-devstack/sysgo/l2_cl.go | 22 + op-devstack/sysgo/l2_cl_kona_config_test.go | 1 + op-devstack/sysgo/l2_cl_test.go | 30 ++ op-devstack/sysgo/l2_el_opreth.go | 23 +- op-devstack/sysgo/l2_el_p2p_util.go | 24 + op-devstack/sysgo/mixed_runtime.go | 12 +- op-devstack/sysgo/singlechain_build.go | 2 +- op-devstack/sysgo/subproc.go | 63 ++- op-devstack/sysgo/subproc_test.go | 107 +++- 18 files changed, 912 insertions(+), 35 deletions(-) create mode 100644 op-acceptance-tests/tests/proofs_history_restart/proofs_history_restart_test.go create mode 100644 op-devstack/dsl/block_recorder_workload.go create mode 100644 op-devstack/sysgo/l2_cl_test.go diff --git a/op-acceptance-tests/tests/proofs_history_match/proofs_history_match_test.go b/op-acceptance-tests/tests/proofs_history_match/proofs_history_match_test.go index ab9dcb87bea..c44c728c0ee 100644 --- a/op-acceptance-tests/tests/proofs_history_match/proofs_history_match_test.go +++ b/op-acceptance-tests/tests/proofs_history_match/proofs_history_match_test.go @@ -9,12 +9,9 @@ import ( "github.com/ethereum-optimism/optimism/op-devstack/dsl" "github.com/ethereum-optimism/optimism/op-devstack/presets" "github.com/ethereum-optimism/optimism/op-devstack/sysgo" - "github.com/ethereum-optimism/optimism/op-service/bigs" "github.com/ethereum-optimism/optimism/op-service/eth" "github.com/ethereum-optimism/optimism/op-service/txplan" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core/vm" - "github.com/ethereum/go-ethereum/core/vm/program" ) const ( @@ -99,27 +96,6 @@ func newProofsHistorySystem(t devtest.T, c proofsHistoryCase) *proofsHistorySyst } } -// deployBlockRecorder deploys a contract that writes the block number to slot 0 and the block -// timestamp to slot 1 on every call, so every call changes its storage. -func deployBlockRecorder(t devtest.T, deployer *dsl.EOA) common.Address { - runtimeCode := program.New(). - Op(vm.NUMBER).Push(0).Op(vm.SSTORE). - Op(vm.TIMESTAMP).Push(1).Op(vm.SSTORE). - Bytes() - initCode := program.New().ReturnViaCodeCopy(runtimeCode).Bytes() - - tx := deployer.Transact(deployer.Plan(), txplan.WithData(initCode)) - receipt, err := tx.Included.Eval(t.Ctx()) - t.Require().NoError(err, "block recorder deployment must be included") - return receipt.ContractAddress -} - -func includedBlock(t devtest.T, tx *txplan.PlannedTx) uint64 { - receipt, err := tx.Included.Eval(t.Ctx()) - t.Require().NoError(err, "transaction must be included") - return bigs.Uint64Strict(receipt.BlockNumber) -} - // TestProofsHistoryFollowerServesMatchingHistoricalProofs runs transfers, withdrawals and contract // writes against a sequencer that serves eth_getProof from its own historical state, then sweeps a // proofs-history follower from its window's first block to the last workload block: each of the @@ -132,7 +108,7 @@ func TestProofsHistoryFollowerServesMatchingHistoricalProofs(gt *testing.T) { alice := sys.FunderL2.NewFundedEOA(eth.OneEther) bob := sys.FunderL2.NewFundedEOA(eth.OneEther) - recorder := deployBlockRecorder(t, alice) + recorder := alice.DeployBlockRecorder() var lastWorkloadBlock uint64 for round := range workloadRounds { if round == workloadRounds-1 { @@ -144,7 +120,7 @@ func TestProofsHistoryFollowerServesMatchingHistoricalProofs(gt *testing.T) { alice.Transfer(bob.Address(), eth.OneHundredthEther) bob.Transfer(alice.Address(), eth.OneHundredthEther) sys.Bridge.InitiateWithdrawal(eth.OneHundredthEther, bob) - lastWorkloadBlock = includedBlock(t, alice.Transact(alice.Plan(), txplan.WithTo(&recorder))) + lastWorkloadBlock = dsl.IncludedBlockNumber(t, alice.Transact(alice.Plan(), txplan.WithTo(&recorder))) } sys.ProofsFollower.VerifyHistoricalProofsMatch(sys.Reference, func(opts *dsl.HistoricalProofsOpts) { diff --git a/op-acceptance-tests/tests/proofs_history_restart/proofs_history_restart_test.go b/op-acceptance-tests/tests/proofs_history_restart/proofs_history_restart_test.go new file mode 100644 index 00000000000..a2a38b0fe05 --- /dev/null +++ b/op-acceptance-tests/tests/proofs_history_restart/proofs_history_restart_test.go @@ -0,0 +1,470 @@ +package proofs_history_restart + +import ( + "errors" + "fmt" + "testing" + "time" + + "github.com/ethereum-optimism/optimism/op-devstack/devtest" + "github.com/ethereum-optimism/optimism/op-devstack/dsl" + "github.com/ethereum-optimism/optimism/op-devstack/presets" + "github.com/ethereum-optimism/optimism/op-devstack/sysgo" + nodeSync "github.com/ethereum-optimism/optimism/op-node/rollup/sync" + "github.com/ethereum-optimism/optimism/op-service/eth" + safety "github.com/ethereum-optimism/optimism/op-service/eth/safety" + "github.com/ethereum-optimism/optimism/op-service/log" + "github.com/ethereum-optimism/optimism/op-service/retry" + "github.com/ethereum-optimism/optimism/op-service/testlog" +) + +const ( + // l2BlockTimeSeconds keeps the runs short: the kill point is a block height, so halving the + // block time halves the time spent producing blocks. + l2BlockTimeSeconds = 1 + // memoryBlockBufferTarget is reth's --engine.memory-block-buffer-target. It is passed + // explicitly, like the persistence threshold, so the gap the kill opens between the proofs + // store and reth's datadir does not depend on reth's defaults. + memoryBlockBufferTarget = 5 + // postRestartTrafficBlocks is how far past the head the chain is driven once the follower has + // caught up, so the sweep also covers blocks the proofs store indexed after coming back. + postRestartTrafficBlocks = 10 + // proofsStoreFlushBlocks is how many blocks op-reth's proofs store buffers before it flushes + // them. op-reth has no flag for it. + proofsStoreFlushBlocks = 5 + + // followerImportAttempts bounds the wait for the follower to import the block the sequencer + // was held at. + followerImportAttempts = 10 + // killPointHoldAttempts bounds how often the chain is held before the held head fits the case's + // kill point. + killPointHoldAttempts = 3 + // rebuildAttempts bounds how long the restarted sequencer may take to rebuild the heights it + // lost with different contents. + rebuildAttempts = 30 + // resyncAttempts bounds how long the restarted follower may take to catch up with the + // sequencer. A follower whose op-reth crash-loops never does. + resyncAttempts = 60 + // storeCatchUpTimeout bounds the wait for a proofs store flush before the kill. + storeCatchUpTimeout = 60 * time.Second +) + +// killPoint says where, relative to the proofs store's flush cadence, the follower is killed. +type killPoint string + +const ( + // killAfterStoreFlush kills the follower as soon as the store has flushed the target block and + // one more workload transaction has landed, so the store's tip is near the chain head and its + // lead over reth's datadir is the largest. + killAfterStoreFlush killPoint = "after_flush" + // killMidStoreBuffer kills the follower a couple of blocks after a flush, so the store's tip + // trails the chain head and the store loses blocks of its own on top of reth's. + killMidStoreBuffer killPoint = "mid_buffer" +) + +// syncPath says how the restarted follower's CL brings its EL from reth's datadir tip back to +// the sequencer's head. +type syncPath string + +const ( + // clSync has op-node insert the missing blocks itself, fetched from its peers. + clSync syncPath = "cl_sync" + // elSync runs op-node in EL sync mode with post-finalization EL sync, as op-node runs it + // against op-reth, so it hands the gap to op-reth even though op-reth already holds a + // finalized block past genesis. + elSync syncPath = "el_sync" + // elSyncAfterFinality runs op-node in EL sync mode with post-finalization EL sync turned off, + // as op-node runs it against op-geth. The follower's first EL sync finalized a block past + // genesis, so the restarted op-node finds it and goes straight to CL sync. + elSyncAfterFinality syncPath = "el_sync_after_finality" +) + +func (p syncPath) followerL2CLOpts(clKind sysgo.MixedL2CLKind) []sysgo.L2CLOption { + switch p { + case elSync: + opts := []sysgo.L2CLOption{sysgo.L2CLVerifierSyncMode(nodeSync.ELSync)} + // kona-node always EL-syncs the gap and rejects op-node's switch for it. + if clKind == sysgo.MixedL2CLOpNode { + opts = append(opts, sysgo.L2CLPostFinalizationELSync(true)) + } + return opts + case elSyncAfterFinality: + return []sysgo.L2CLOption{sysgo.L2CLVerifierSyncMode(nodeSync.ELSync), sysgo.L2CLPostFinalizationELSync(false)} + default: + return []sysgo.L2CLOption{sysgo.L2CLVerifierSyncMode(nodeSync.CLSync)} + } +} + +// restartCase is one cell of the matrix the follower runs under. +type restartCase struct { + killPoint killPoint + persistenceThreshold uint64 + // storeFlushPastSetup is how far past the workload's setup block the proofs store must have + // flushed before the kill. It exceeds persistenceThreshold+1, the most reth's persisted tip + // trails its head by, so reth has persisted the setup block by the kill. + storeFlushPastSetup uint64 + syncPath syncPath + verificationInterval uint64 +} + +func (c restartCase) String() string { + proofsPath := "index" + if c.verificationInterval != 0 { + proofsPath = "execute" + } + return fmt.Sprintf("threshold_%d-%s-%s-kill_%s", c.persistenceThreshold, proofsPath, c.syncPath, c.killPoint) +} + +// engineArgs pins reth's persistence schedule, which decides how many blocks an unclean stop +// loses. Both nodes get them so the sequencer loses blocks on the same schedule as the follower. +func (c restartCase) engineArgs() []string { + return []string{ + fmt.Sprintf("--engine.memory-block-buffer-target=%d", memoryBlockBufferTarget), + fmt.Sprintf("--engine.persistence-threshold=%d", c.persistenceThreshold), + } +} + +func (c restartCase) followerArgs() []string { + return append(c.engineArgs(), fmt.Sprintf("--proofs-history.verification-interval=%d", c.verificationInterval)) +} + +// killSchedule pairs the kill point with the persistence threshold rather than crossing them: the +// small threshold gets the mid-buffer kill, where both reth and the store lose blocks, the large +// one the after-flush kill, where the store's lead over reth is largest. Reth persists only up to +// memoryBlockBufferTarget blocks below its head, so its datadir trails the store at the kill: at +// threshold 7 by a few blocks, at 50 by up to 50 blocks it holds only in memory. +type killSchedule struct { + killPoint killPoint + persistenceThreshold uint64 + storeFlushPastSetup uint64 +} + +var ( + afterStoreFlushSchedule = killSchedule{killPoint: killAfterStoreFlush, persistenceThreshold: 50, storeFlushPastSetup: 55} + midStoreBufferSchedule = killSchedule{killPoint: killMidStoreBuffer, persistenceThreshold: 7, storeFlushPastSetup: 17} +) + +// restartMatrix crosses the sync paths with the index proofs path only, and runs the execute +// proofs path once, under EL sync, the one sync path that runs on every CL kind and the one that +// hands the gap to reth's own backfill rather than inserting it block by block. +func restartMatrix(schedule killSchedule) []restartCase { + cell := func(path syncPath, verificationInterval uint64) restartCase { + return restartCase{ + killPoint: schedule.killPoint, + persistenceThreshold: schedule.persistenceThreshold, + storeFlushPastSetup: schedule.storeFlushPastSetup, + syncPath: path, + verificationInterval: verificationInterval, + } + } + return []restartCase{ + cell(clSync, 0), + cell(elSync, 0), + cell(elSyncAfterFinality, 0), + cell(elSync, 1), + } +} + +// skipUnsupportedCell skips the cells the configured clients cannot run. +func skipUnsupportedCell(t devtest.T, c restartCase) { + sysgo.SkipOnOpGeth(t, "proofs history is an op-reth feature") + switch c.syncPath { + case clSync: + sysgo.SkipOnKonaNode(t, "kona-node has no CL sync mode: it always lets the EL sync the gap") + case elSyncAfterFinality: + sysgo.SkipOnKonaNode(t, "kona-node EL-syncs the gap whatever it has finalized, so this cell would repeat el_sync") + } +} + +type restartSystem struct { + Batcher *dsl.L2Batcher + Follower *dsl.L2ELNode + FollowerCL *dsl.L2CLNode + FollowerCLKey string + FollowerEnode string + FunderL2 *dsl.FunderEOA + Logs *testlog.CapturingHandler + Sequencer *dsl.L2ELNode + SequencerCL *dsl.L2CLNode +} + +// logCapturingT hands the system a logger that also captures every record it passes on, so a +// cell can check which sync path the follower's op-node logged. +type logCapturingT struct { + devtest.T + logger log.Logger +} + +func (t logCapturingT) Logger() log.Logger { return t.logger } + +// newRestartSystem starts a sequencer without proofs history, which serves as the independent +// eth_getProof reference, and a follower running proofs history under the case's reth, ExEx and +// sync-path settings. +func newRestartSystem(t devtest.T, c restartCase) *restartSystem { + logs, ok := testlog.WrapCaptureLogger(t.Logger().Handler()).(*testlog.CapturingHandler) + t.Require().True(ok, "the log capture must wrap the test's handler") + t = logCapturingT{T: t, logger: log.NewLogger(logs).WithContext(t.Ctx())} + clKind := sysgo.ResolveMixedL2CLKind() + frontends := presets.NewProofsHistoryFollowerSystem(t, func(cfg *presets.ProofsHistoryFollowerConfig) { + cfg.DeployerOptions = []sysgo.DeployerOption{sysgo.WithUniformL2BlockTimes(l2BlockTimeSeconds)} + cfg.FollowerL2CLOpts = c.syncPath.followerL2CLOpts(clKind) + cfg.FollowerOpRethOpts = []sysgo.OpRethOption{sysgo.OpRethWithExtraArgs(c.followerArgs()...)} + cfg.SequencerOpRethOpts = []sysgo.OpRethOption{sysgo.OpRethWithExtraArgs(c.engineArgs()...)} + }) + return &restartSystem{ + Batcher: frontends.L2Batcher, + Follower: frontends.Follower.EL, + FollowerCL: frontends.Follower.CL, + FollowerCLKey: frontends.Follower.Spec.CLKey, + FollowerEnode: frontends.Follower.EL.Enode(), + FunderL2: frontends.FunderL2, + Logs: logs, + Sequencer: frontends.Sequencer.EL, + SequencerCL: frontends.Sequencer.CL, + } +} + +// uncleanStop is what the follower held when it was killed: the chain head it had imported, the +// block that held the workload's last transaction, both on the chain of that moment, and the last +// block its proofs store had flushed. +type uncleanStop struct { + head eth.L2BlockRef + lastWorkloadBlock eth.L2BlockRef + storeTip uint64 +} + +// killFollower drives the workload until the follower's proofs store has flushed the case's target +// block, holds the chain at the case's kill point right after one more workload transaction and +// kills the follower's op-reth there. The sequencer is left stopped and the follower's CL running. +func killFollower(t devtest.T, sys *restartSystem, w *dsl.BlockRecorderWorkload, c restartCase) uncleanStop { + storeFlushTarget := w.SetupBlock() + c.storeFlushPastSetup + w.DriveUntilBlock(storeFlushTarget) + window := sys.Follower.WaitForProofsStoreBlock(storeFlushTarget, storeCatchUpTimeout) + lastWorkloadTarget := *window.Latest + if c.killPoint == killMidStoreBuffer { + lastWorkloadTarget += 2 + } + stop := holdChainAtKillPoint(t, sys, w, c, w.DriveUntilBlock(lastWorkloadTarget)) + t.Logger().Info("Killing the follower", "head", stop.head, "proofs_store_tip", stop.storeTip, + "last_workload_block", stop.lastWorkloadBlock, "kill_point", c.killPoint) + sys.Follower.StopUngracefully() + return stop +} + +// holdChainAtKillPoint holds the chain at a head that fits the case's kill point. Reth persists +// only up to memoryBlockBufferTarget blocks below its head, so a head fewer blocks than that past +// the workload's last transaction keeps that block out of reth's datadir; a mid-buffer kill also +// needs a head the proofs store has not flushed. While the held head misses either, the sequencer +// resumes for one more workload transaction and the chain is held again. +func holdChainAtKillPoint(t devtest.T, sys *restartSystem, w *dsl.BlockRecorderWorkload, c restartCase, lastWorkloadBlockNumber uint64) uncleanStop { + stop, err := retry.Do(t.Ctx(), killPointHoldAttempts, retry.Fixed(0), func() (uncleanStop, error) { + stop := uncleanStop{head: holdChain(t, sys)} + stop.lastWorkloadBlock = sys.Follower.BlockRefByNumber(lastWorkloadBlockNumber) + // Read last, right before the kill: op-reth flushes a buffer left idle for 10s, and a flush + // of the held head turns a mid-buffer kill into an after-flush one. + stop.storeTip = proofsStoreTip(t, sys.Follower) + mismatch := killPointMismatch(c, stop) + if mismatch == "" { + return stop, nil + } + t.Logger().Info("Held head does not fit the kill point; resuming the sequencer for another workload transaction", + "head", stop.head, "proofs_store_tip", stop.storeTip, "last_workload_block", stop.lastWorkloadBlock, + "mismatch", mismatch) + sys.SequencerCL.StartSequencer() + lastWorkloadBlockNumber = w.DriveUntilBlock(stop.head.Number + 1) + return uncleanStop{}, errors.New(mismatch) + }) + t.Require().NoErrorf(err, "the chain never held at a head fitting the %s kill point", c.killPoint) + return stop +} + +// killPointMismatch says why the held chain does not fit the case's kill point, or returns "" when +// it does. +func killPointMismatch(c restartCase, stop uncleanStop) string { + if blocksPastWorkload := stop.head.Number - stop.lastWorkloadBlock.Number; blocksPastWorkload >= memoryBlockBufferTarget { + return fmt.Sprintf("the held head %d is %d blocks past the workload's last block %d, so reth may persist that block", + stop.head.Number, blocksPastWorkload, stop.lastWorkloadBlock.Number) + } + if c.killPoint == killMidStoreBuffer && stop.storeTip >= stop.head.Number { + return fmt.Sprintf("the proofs store flushed the held head %d (store tip %d)", stop.head.Number, stop.storeTip) + } + return "" +} + +// holdChain stops block production, waits for the follower to import the last block and returns +// that head once the proofs store can no longer flush on block count: it flushes once +// proofsStoreFlushBlocks blocks are buffered, so a tip within that many blocks of the held head +// stays put until op-reth's 10s idle flush. +func holdChain(t devtest.T, sys *restartSystem) eth.L2BlockRef { + heldHead := sys.Sequencer.BlockRefByHash(sys.SequencerCL.StopSequencer()) + sys.Follower.Reached(eth.Unsafe, heldHead.Number, followerImportAttempts) + head := sys.Follower.BlockRefByLabel(eth.Unsafe) + t.Require().Equalf(heldHead.ID(), head.ID(), "the follower must hold the sequencer's head %s, but holds %s", heldHead, head) + sys.Follower.WaitForProofsStoreBlock(head.Number-(proofsStoreFlushBlocks-1), storeCatchUpTimeout) + return head +} + +func proofsStoreTip(t devtest.T, el *dsl.L2ELNode) uint64 { + status, err := el.ProofsSyncStatus(t.Ctx()) + t.Require().NoErrorf(err, "debug_proofsSyncStatus on %s failed", el) + t.Require().NotNilf(status.Latest, "the proofs store on %s has indexed no block", el) + return *status.Latest +} + +// restartFollower brings the follower's op-reth and CL back and re-peers them with the sequencer, +// since both peerings died with the processes. It returns the block reth resumes from, its datadir +// tip, which must be below the proofs store's tip for the test to exercise the crash gap at all. +// It clears the captured logs before the follower's CL starts, so the follower op-node records +// left in them are the restarted node's. +func restartFollower(t devtest.T, sys *restartSystem, stop uncleanStop) eth.L2BlockRef { + // The follower keeps its node ID but listens on a new port, so the sequencer's trusted entry + // would keep redialing the old one, and reach the restarted follower if the OS reuses that + // port: a redial that crosses the follower's own dial makes reth tear down both sessions. + sys.Sequencer.ForgetPeer(sys.FollowerEnode) + sys.Follower.Start() + sys.Follower.WaitForOnline() + // Nothing feeds the EL until its CL is back, so its head is what reth had persisted. + persistedTip := sys.Follower.BlockRefByLabel(eth.Unsafe) + t.Logger().Info("Follower op-reth is back on its datadir", "persisted_tip", persistedTip, + "proofs_store_tip", stop.storeTip, "head_at_kill", stop.head) + t.Require().Lessf(persistedTip.Number, stop.storeTip, + "the proofs store (tip %d) must be ahead of reth's datadir (tip %d) after the kill; the kill landed too late for this threshold", + stop.storeTip, persistedTip.Number) + + sys.Follower.PeerWith(sys.Sequencer) + sys.Logs.Clear() + sys.FollowerCL.Start() + sys.FollowerCL.ConnectPeer(sys.SequencerCL) + return persistedTip +} + +// verifyFollowerSyncPath requires the restarted op-node to have logged the sync path the cell +// claims: el_sync must finish an EL sync, el_sync_after_finality must skip it for CL sync. op-node +// logs either outcome no later than its first block past the head it caught up to, so once the +// follower has indexed the post-restart traffic the record is in the capture. kona-node is not +// checked: it always EL-syncs and logs neither. +func verifyFollowerSyncPath(t devtest.T, sys *restartSystem, c restartCase) { + if sysgo.ResolveMixedL2CLKind() != sysgo.MixedL2CLOpNode { + return + } + var expectedMessage string + switch c.syncPath { + case elSync: + expectedMessage = "Finished EL sync" + case elSyncAfterFinality: + expectedMessage = "Skipping EL sync" + default: + return + } + record := sys.Logs.FindLog( + testlog.NewAttributesFilter("component", "l2cl-"+sys.FollowerCLKey), + testlog.NewMessageContainsFilter(expectedMessage), + ) + t.Require().NotNilf(record, "the restarted follower op-node never logged %q, so it did not take the %s path", expectedMessage, c.syncPath) +} + +// Each kill schedule is a top-level test of its own, so its four cells, which each start a devnet +// and run one after another, get a go test timeout to themselves. + +func TestProofsHistoryFollowerRecoversFromUncleanShutdownAfterStoreFlush(gt *testing.T) { + testFollowerRecoversFromUncleanShutdown(gt, afterStoreFlushSchedule) +} + +func TestProofsHistoryFollowerRecoversFromUncleanShutdownMidStoreBuffer(gt *testing.T) { + testFollowerRecoversFromUncleanShutdown(gt, midStoreBufferSchedule) +} + +func TestProofsHistoryFollowerServesRebuiltBlocksAfterUncleanShutdownAfterStoreFlush(gt *testing.T) { + testFollowerServesRebuiltBlocksAfterUncleanShutdown(gt, afterStoreFlushSchedule) +} + +func TestProofsHistoryFollowerServesRebuiltBlocksAfterUncleanShutdownMidStoreBuffer(gt *testing.T) { + testFollowerServesRebuiltBlocksAfterUncleanShutdown(gt, midStoreBufferSchedule) +} + +// testFollowerRecoversFromUncleanShutdown kills a proofs-history follower once its store has +// flushed blocks reth has not yet persisted, restarts it into the same chain and requires it to +// catch up with the sequencer, keep indexing, and serve proofs that match the sequencer's for the +// blocks it lost, the blocks it re-imported and the blocks that followed. The follower catches up +// by CL sync, by EL sync, or by the CL sync an EL sync config without post-finalization EL sync +// falls back to once a block past genesis is finalized. +func testFollowerRecoversFromUncleanShutdown(gt *testing.T, schedule killSchedule) { + t := devtest.ParallelT(gt) + devtest.RunParallel(t, restartMatrix(schedule), func(t devtest.T, c restartCase) { + skipUnsupportedCell(t, c) + sys := newRestartSystem(t, c) + w := dsl.NewBlockRecorderWorkload(t, sys.FunderL2) + + stop := killFollower(t, sys, w, c) + sys.SequencerCL.StartSequencer() + sys.FollowerCL.Stop() + persistedTip := restartFollower(t, sys, stop) + + sys.Follower.InSync(sys.Sequencer, safety.LocalUnsafe, resyncAttempts) + lastBlock := w.DriveUntilBlock(sys.Sequencer.BlockRefByLabel(eth.Unsafe).Number + postRestartTrafficBlocks) + sys.Follower.VerifyHistoricalProofsMatch(sys.Sequencer, func(opts *dsl.HistoricalProofsOpts) { + opts.Accounts = w.Accounts() + opts.FirstBlock = persistedTip.Number + opts.LastBlock = lastBlock + }) + verifyFollowerSyncPath(t, sys, c) + }) +} + +// testFollowerServesRebuiltBlocksAfterUncleanShutdown kills a proofs-history follower, then kills +// the sequencer's op-reth too while the blocks the follower's store holds are still only in the +// sequencer's memory, so the restarted sequencer rebuilds those heights with different contents. +// The restarted follower must follow the rebuilt chain and serve its proofs, not the ones its +// store recorded for the chain that no longer exists. +func testFollowerServesRebuiltBlocksAfterUncleanShutdown(gt *testing.T, schedule killSchedule) { + t := devtest.ParallelT(gt) + devtest.RunParallel(t, restartMatrix(schedule), func(t devtest.T, c restartCase) { + skipUnsupportedCell(t, c) + sys := newRestartSystem(t, c) + // Blocks that reached L1 would be derived back as they were; blocks that never did + // are re-sequenced, which is the rebuild this test needs. + sys.Batcher.Stop() + w := dsl.NewBlockRecorderWorkload(t, sys.FunderL2) + + // Both op-reths die back to back on the held chain, so the sequencer still holds the + // blocks the follower's store flushed; the CLs follow once the processes are gone. The + // restarted sequencer CL sequences again. + stop := killFollower(t, sys, w, c) + sys.Sequencer.StopUngracefully() + sys.FollowerCL.Stop() + sys.SequencerCL.Stop() + + sys.Sequencer.Start() + sys.Sequencer.WaitForOnline() + sequencerPersistedTip := sys.Sequencer.BlockRefByLabel(eth.Unsafe) + t.Logger().Info("Sequencer op-reth is back on its datadir", "persisted_tip", sequencerPersistedTip, + "follower_proofs_store_tip", stop.storeTip, "last_workload_block", stop.lastWorkloadBlock, + "workload_setup_block", w.SetupBlock()) + t.Require().Lessf(sequencerPersistedTip.Number, stop.storeTip, + "the sequencer must rebuild blocks the follower's proofs store holds, but its datadir tip %d is not below the store tip %d", + sequencerPersistedTip.Number, stop.storeTip) + // A lost block without a workload transaction can come back identical, so only losing the + // workload's last block guarantees the rebuilt chain differs. + t.Require().Lessf(sequencerPersistedTip.Number, stop.lastWorkloadBlock.Number, + "the block %d that held the workload's last transaction must be among the sequencer's lost blocks, but its datadir tip is %d", + stop.lastWorkloadBlock.Number, sequencerPersistedTip.Number) + t.Require().LessOrEqualf(w.SetupBlock(), sequencerPersistedTip.Number, + "the workload's funding and contract (block %d) must survive the sequencer's crash, but its datadir tip is %d", + w.SetupBlock(), sequencerPersistedTip.Number) + sys.SequencerCL.Start() + sys.Sequencer.ReorgTriggered(stop.lastWorkloadBlock, rebuildAttempts) + + persistedTip := restartFollower(t, sys, stop) + + sys.Follower.InSync(sys.Sequencer, safety.LocalUnsafe, resyncAttempts) + lastBlock := w.DriveUntilBlock(sys.Sequencer.BlockRefByLabel(eth.Unsafe).Number + postRestartTrafficBlocks) + sys.Follower.VerifyHistoricalProofsMatch(sys.Sequencer, func(opts *dsl.HistoricalProofsOpts) { + opts.Accounts = w.Accounts() + // The follower's datadir can outlast the sequencer's, so start at whichever comes + // first: its own tip or the first height the sequencer rebuilt. + opts.FirstBlock = min(persistedTip.Number, sequencerPersistedTip.Number+1) + opts.LastBlock = lastBlock + }) + verifyFollowerSyncPath(t, sys, c) + }) +} diff --git a/op-devstack/dsl/block_recorder_workload.go b/op-devstack/dsl/block_recorder_workload.go new file mode 100644 index 00000000000..1d2d7a64e71 --- /dev/null +++ b/op-devstack/dsl/block_recorder_workload.go @@ -0,0 +1,89 @@ +package dsl + +import ( + "time" + + "github.com/ethereum-optimism/optimism/op-devstack/devtest" + "github.com/ethereum-optimism/optimism/op-service/eth" + "github.com/ethereum-optimism/optimism/op-service/retry" + "github.com/ethereum-optimism/optimism/op-service/txplan" + "github.com/ethereum/go-ethereum/common" +) + +// BlockRecorderWorkload changes state in nearly every block: a writer and a counterparty transfer +// back and forth and the writer calls a block recorder contract, so every block it fills carries +// balance and storage changes that a rebuilt block would not repeat. +type BlockRecorderWorkload struct { + commonImpl + + counterparty *EOA + inclusionPoll txplan.Option + recorder common.Address + // setupBlock is the block that completed the funding and the contract deployment; a chain that + // lacks it lacks the workload's accounts. + setupBlock uint64 + writer *EOA +} + +// NewBlockRecorderWorkload funds two senders, deploys a block recorder and calls it once. +func NewBlockRecorderWorkload(t devtest.T, funder *FunderEOA) *BlockRecorderWorkload { + w := &BlockRecorderWorkload{ + commonImpl: commonFromT(t), + counterparty: funder.NewFundedEOA(eth.OneEther), + inclusionPoll: txplan.Combine(), + writer: funder.NewFundedEOA(eth.OneEther), + } + w.recorder = w.writer.DeployBlockRecorder() + w.setupBlock = IncludedBlockNumber(t, w.writer.Transact(w.writer.Plan(), txplan.WithTo(&w.recorder))) + w.log.Info("Block recorder workload set up", "recorder", w.recorder, "setup_block", w.setupBlock) + return w +} + +// WithFreshSenders returns a workload that writes to the same contract from two newly funded +// accounts. A node that reorgs puts the transactions of the blocks it drops back into its pool, so +// new transactions from the original senders could race those for the same nonces. +func (w *BlockRecorderWorkload) WithFreshSenders(funder *FunderEOA) *BlockRecorderWorkload { + fresh := *w + fresh.counterparty = funder.NewFundedEOA(eth.OneEther) + fresh.writer = funder.NewFundedEOA(eth.OneEther) + return &fresh +} + +// WithInclusionPoll returns a workload that polls el for its transactions' receipts every +// interval, up to maxAttempts times, so DriveUntilBlock returns within about interval of the block +// that held its last write rather than after the default exponential backoff. +func (w *BlockRecorderWorkload) WithInclusionPoll(el *L2ELNode, interval time.Duration, maxAttempts int) *BlockRecorderWorkload { + polled := *w + polled.inclusionPoll = txplan.WithRetryInclusion(el.EthClient(), maxAttempts, retry.Fixed(interval)) + return &polled +} + +// Accounts are the senders and the recorder with the storage slots it writes. +func (w *BlockRecorderWorkload) Accounts() []ProofAccount { + return append(w.SenderAccounts(), + ProofAccount{Address: w.recorder, StorageKeys: []common.Hash{common.HexToHash("0x0"), common.HexToHash("0x1")}}) +} + +func (w *BlockRecorderWorkload) SenderAccounts() []ProofAccount { + return []ProofAccount{{Address: w.counterparty.Address()}, {Address: w.writer.Address()}} +} + +func (w *BlockRecorderWorkload) SetupBlock() uint64 { + return w.setupBlock +} + +// DriveUntilBlock sends transactions one after another until a contract write lands in a block at +// or past target, and returns that block. +func (w *BlockRecorderWorkload) DriveUntilBlock(target uint64) uint64 { + w.log.Info("Driving the block recorder workload", "target", target) + for { + w.counterparty.Transact(w.counterparty.PlanTransfer(w.writer.Address(), eth.OneHundredthEther), w.inclusionPoll) + w.writer.Transact(w.writer.PlanTransfer(w.counterparty.Address(), eth.OneHundredthEther), w.inclusionPoll) + write := w.writer.Transact(w.writer.Plan(), txplan.WithTo(&w.recorder), w.inclusionPoll) + block := IncludedBlockNumber(w.t, write) + if block >= target { + w.log.Info("Block recorder workload reached target", "target", target, "last_write_block", block) + return block + } + } +} diff --git a/op-devstack/dsl/eoa.go b/op-devstack/dsl/eoa.go index 13ba617abef..3efb159f57d 100644 --- a/op-devstack/dsl/eoa.go +++ b/op-devstack/dsl/eoa.go @@ -11,6 +11,7 @@ import ( "github.com/ethereum-optimism/optimism/op-acceptance-tests/tests/interop" messages "github.com/ethereum-optimism/optimism/op-core/interop/messages" "github.com/ethereum-optimism/optimism/op-core/predeploys" + "github.com/ethereum-optimism/optimism/op-devstack/devtest" e2eBindings "github.com/ethereum-optimism/optimism/op-e2e/bindings" "github.com/ethereum-optimism/optimism/op-service/bigs" "github.com/ethereum-optimism/optimism/op-service/eth" @@ -22,6 +23,8 @@ import ( "github.com/ethereum-optimism/optimism/op-service/txplan" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/core/vm" + "github.com/ethereum/go-ethereum/core/vm/program" "github.com/ethereum/go-ethereum/crypto" ) @@ -252,6 +255,30 @@ func (u *EOA) DeployEventLogger() common.Address { return eventLoggerAddress } +// DeployBlockRecorder deploys a contract that writes the block number to storage slot 0 and the +// block timestamp to slot 1 on every call, so every call to it changes its storage. +func (u *EOA) DeployBlockRecorder() common.Address { + runtimeCode := program.New(). + Op(vm.NUMBER).Push(0).Op(vm.SSTORE). + Op(vm.TIMESTAMP).Push(1).Op(vm.SSTORE). + Bytes() + initCode := program.New().ReturnViaCodeCopy(runtimeCode).Bytes() + tx := txplan.NewPlannedTx(u.Plan(), txplan.WithData(initCode)) + _, err := tx.Success.Eval(u.ctx) + u.t.Require().NoError(err, "block recorder deployment must succeed") + res, err := tx.Included.Eval(u.ctx) + u.t.Require().NoError(err, "failed to read the block recorder deployment receipt") + u.log.Info("deployed block recorder", "chainID", tx.ChainID.Value(), "address", res.ContractAddress) + return res.ContractAddress +} + +// IncludedBlockNumber waits for tx to be included and returns the number of the block holding it. +func IncludedBlockNumber(t devtest.T, tx *txplan.PlannedTx) uint64 { + receipt, err := tx.Included.Eval(t.Ctx()) + t.Require().NoError(err, "transaction must be included") + return bigs.Uint64Strict(receipt.BlockNumber) +} + func (u *EOA) DeployWETH() common.Address { // Use the e2e bindings which contain the WETH bytecode tx := txplan.NewPlannedTx(u.Plan(), txplan.WithData(common.FromHex(e2eBindings.WETHBin))) diff --git a/op-devstack/dsl/l2_el.go b/op-devstack/dsl/l2_el.go index a940ac0fb06..dce2ef7001f 100644 --- a/op-devstack/dsl/l2_el.go +++ b/op-devstack/dsl/l2_el.go @@ -23,6 +23,7 @@ import ( safety "github.com/ethereum-optimism/optimism/op-service/eth/safety" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/p2p" ) var emptyHash = common.Hash{} @@ -525,6 +526,15 @@ func (el *L2ELNode) Stop() { lifecycle.Stop() } +// StopUngracefully ends the node's process without a shutdown, the way a crash would, so state +// it held only in memory is lost. Start brings the node back. +func (el *L2ELNode) StopUngracefully() { + el.log.Info("Stopping ungracefully", "name", el.inner.Name()) + lifecycle, ok := el.inner.(stack.UngracefulLifecycle) + el.require.Truef(ok, "L2EL node %s cannot be stopped ungracefully", el.inner.Name()) + lifecycle.StopUngracefully() +} + func (el *L2ELNode) Start() { lifecycle, ok := el.inner.(stack.Lifecycle) el.require.Truef(ok, "L2EL node %s is not lifecycle-controllable", el.inner.Name()) @@ -539,6 +549,22 @@ func (el *L2ELNode) DisconnectPeerWith(peer *L2ELNode) { sysgo.DisconnectP2P(el.ctx, el.require, el.inner.L2EthClient().RPC(), peer.inner.L2EthClient().RPC()) } +// Enode returns the node's enode URL, which identifies it to its peers across restarts. +func (el *L2ELNode) Enode() string { + var info p2p.NodeInfo + el.require.NoErrorf(el.inner.L2EthClient().RPC().CallContext(el.ctx, &info, "admin_nodeInfo"), + "admin_nodeInfo on %s failed", el) + return info.Enode +} + +// ForgetPeer drops this node's peer entry for peerEnode, trusted status included, and waits until +// the peer is no longer connected. The peer may be down. +func (el *L2ELNode) ForgetPeer(peerEnode string) { + el.log.Info("Forgetting peer", "node", el, "peer_enode", peerEnode) + sysgo.ForgetP2PPeer(el.ctx, el.require, el.inner.L2EthClient().RPC(), peerEnode) + el.log.Info("Forgot peer", "node", el, "peer_enode", peerEnode) +} + func (el *L2ELNode) PayloadByNumber(number uint64) *eth.ExecutionPayloadEnvelope { payload, err := el.inner.L2EthClient().PayloadByNumber(el.ctx, number) el.require.NoError(err, "failed to get payload") diff --git a/op-devstack/presets/proofs_history_follower.go b/op-devstack/presets/proofs_history_follower.go index 7500b653fdd..3ca19442f8f 100644 --- a/op-devstack/presets/proofs_history_follower.go +++ b/op-devstack/presets/proofs_history_follower.go @@ -13,6 +13,7 @@ const proofsHistoryReferenceEthProofWindow = 1_000_000 // starts. type ProofsHistoryFollowerConfig struct { DeployerOptions []sysgo.DeployerOption + FollowerL2CLOpts []sysgo.L2CLOption FollowerOpRethOpts []sysgo.OpRethOption // SequencerOpRethOpts apply after the options that turn the sequencer's proofs history off and // open its eth_getProof window. @@ -57,6 +58,7 @@ func NewProofsHistoryFollowerSystem(t devtest.T, configure ...func(*ProofsHistor CLKind: clKind, ELKey: "proofs-follower", ELKind: sysgo.MixedL2ELOpReth, + L2CLOpts: cfg.FollowerL2CLOpts, OpRethOpts: cfg.FollowerOpRethOpts, }, }, diff --git a/op-devstack/presets/rpc_frontends.go b/op-devstack/presets/rpc_frontends.go index b6008086dac..d0a41c6182b 100644 --- a/op-devstack/presets/rpc_frontends.go +++ b/op-devstack/presets/rpc_frontends.go @@ -155,9 +155,13 @@ type l2ELFrontend struct { l2EngineClient *sources.EngineClient lifecycle stack.Lifecycle control stack.ControlledLifecycle + ungraceful stack.UngracefulLifecycle } -var _ stack.L2ELNode = (*l2ELFrontend)(nil) +var ( + _ stack.L2ELNode = (*l2ELFrontend)(nil) + _ stack.UngracefulLifecycle = (*l2ELFrontend)(nil) +) func newPresetL2ELNode(t devtest.T, name string, chainID eth.ChainID, userRPCCl opclient.RPC, userRPC string, engineRPCCl opclient.RPC, rollupCfg *rollup.Config) *l2ELFrontend { t.Require().NotNil(rollupCfg, "rollup config must be configured") @@ -193,6 +197,11 @@ func (r *l2ELFrontend) Stop() { r.lifecycle.Stop() } +func (r *l2ELFrontend) StopUngracefully() { + r.require().NotNil(r.ungraceful, "L2EL node %s cannot be stopped ungracefully", r.Name()) + r.ungraceful.StopUngracefully() +} + func (r *l2ELFrontend) StartControlled(ctx context.Context) error { if r.control == nil { return fmt.Errorf("L2EL node %s is not controllable", r.Name()) diff --git a/op-devstack/presets/sysgo_runtime.go b/op-devstack/presets/sysgo_runtime.go index 3ae12a67ebe..ddf7a42fe27 100644 --- a/op-devstack/presets/sysgo_runtime.go +++ b/op-devstack/presets/sysgo_runtime.go @@ -66,6 +66,9 @@ func newL2ELFrontend(t devtest.T, name string, chainID eth.ChainID, userRPC stri if control, ok := lifecycle[0].(stack.ControlledLifecycle); ok { l2EL.control = control } + if ungraceful, ok := lifecycle[0].(stack.UngracefulLifecycle); ok { + l2EL.ungraceful = ungraceful + } } return l2EL } diff --git a/op-devstack/stack/lifecycle.go b/op-devstack/stack/lifecycle.go index f26d3b7e605..22278e6444f 100644 --- a/op-devstack/stack/lifecycle.go +++ b/op-devstack/stack/lifecycle.go @@ -12,3 +12,10 @@ type ControlledLifecycle interface { StopControlled(ctx context.Context) error Running() bool } + +// UngracefulLifecycle is implemented by components that can be stopped the way a process crash +// would stop them: no shutdown runs, so state held only in process memory is lost. Writes already +// handed to the OS survive, so this does not model a power loss. +type UngracefulLifecycle interface { + StopUngracefully() +} diff --git a/op-devstack/sysgo/l2_cl.go b/op-devstack/sysgo/l2_cl.go index dc7539eb004..8e48895a89c 100644 --- a/op-devstack/sysgo/l2_cl.go +++ b/op-devstack/sysgo/l2_cl.go @@ -40,6 +40,10 @@ type L2CLConfig struct { // OffsetELSafe retracts safe and finalized from the EL-sync tip by floor(OffsetELSafe / L2BlockTime) blocks. OffsetELSafe time.Duration + // SupportsPostFinalizationELSync lets an op-node in EL sync mode EL-sync even when its EL + // already has a finalized block past genesis. When false, such a node goes straight to CL sync. + // It is off unless set, whatever the EL, while production op-node turns it on for op-reth. + SupportsPostFinalizationELSync bool // AltDAServerURL is the DA server the node reads alt-DA inputs from. Empty disables alt-DA. AltDAServerURL string @@ -57,6 +61,24 @@ func L2CLFollowSource(source string) L2CLOption { }) } +// L2CLVerifierSyncMode sets the sync mode a verifier runs: CL sync has op-node fill unsafe gaps +// from its peers, EL sync hands the forkchoice target to the EL and lets it sync from its own +// peers. +func L2CLVerifierSyncMode(mode nodeSync.Mode) L2CLOption { + return L2CLOptionFn(func(p devtest.T, _ ComponentTarget, cfg *L2CLConfig) { + cfg.VerifierSyncMode = mode + }) +} + +// L2CLPostFinalizationELSync sets whether an op-node in EL sync mode EL-syncs a restart even though +// its EL has a finalized block past genesis. Production op-node enables it for an op-reth or +// op-erigon EL and disables it for op-geth. +func L2CLPostFinalizationELSync(enabled bool) L2CLOption { + return L2CLOptionFn(func(p devtest.T, _ ComponentTarget, cfg *L2CLConfig) { + cfg.SupportsPostFinalizationELSync = enabled + }) +} + func DefaultL2CLConfig() *L2CLConfig { return &L2CLConfig{ SequencerSyncMode: nodeSync.CLSync, diff --git a/op-devstack/sysgo/l2_cl_kona_config_test.go b/op-devstack/sysgo/l2_cl_kona_config_test.go index d70dc2b7690..07a1f1e469a 100644 --- a/op-devstack/sysgo/l2_cl_kona_config_test.go +++ b/op-devstack/sysgo/l2_cl_kona_config_test.go @@ -140,6 +140,7 @@ func TestKonaL2CLConfigRejectsUnsupportedOptions(t *testing.T) { {"role override", true, func(cfg *L2CLConfig) { cfg.IsSequencer = false }, "unsupported L2CL option"}, {"sequencer sync", true, func(cfg *L2CLConfig) { cfg.SequencerSyncMode = nodeSync.Mode(99) }, "unknown effective sync mode"}, {"verifier sync", false, func(cfg *L2CLConfig) { cfg.VerifierSyncMode = nodeSync.Mode(99) }, "unknown effective sync mode"}, + {"post-finalization EL sync", false, func(cfg *L2CLConfig) { cfg.SupportsPostFinalizationELSync = true }, "unsupported L2CL option"}, {"verifier conductor", false, func(cfg *L2CLConfig) { cfg.ConductorRPC = "http://conductor:8547" }, "unsupported L2CL option"}, {"zero conductor timeout", true, func(cfg *L2CLConfig) { cfg.ConductorRPC = "http://conductor:8547" }, "timeout must be positive"}, {"fractional conductor timeout", true, func(cfg *L2CLConfig) { diff --git a/op-devstack/sysgo/l2_cl_test.go b/op-devstack/sysgo/l2_cl_test.go new file mode 100644 index 00000000000..f09da243a71 --- /dev/null +++ b/op-devstack/sysgo/l2_cl_test.go @@ -0,0 +1,30 @@ +package sysgo + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/require" + + nodeSync "github.com/ethereum-optimism/optimism/op-node/rollup/sync" +) + +func TestL2CLVerifierSyncMode(t *testing.T) { + cfg := &L2CLConfig{SequencerSyncMode: nodeSync.CLSync, VerifierSyncMode: nodeSync.CLSync} + L2CLVerifierSyncMode(nodeSync.ELSync).Apply(nil, ComponentTarget{}, cfg) + + require.Equal(t, nodeSync.ELSync, cfg.VerifierSyncMode) + require.Equal(t, nodeSync.CLSync, cfg.SequencerSyncMode, "the sequencer keeps its own sync mode") +} + +func TestL2CLPostFinalizationELSync(t *testing.T) { + require.False(t, DefaultL2CLConfig().SupportsPostFinalizationELSync, "op-node skips EL sync past finality by default") + for _, enabled := range []bool{false, true} { + t.Run(fmt.Sprintf("enabled=%t", enabled), func(t *testing.T) { + cfg := &L2CLConfig{SupportsPostFinalizationELSync: !enabled} + L2CLPostFinalizationELSync(enabled).Apply(nil, ComponentTarget{}, cfg) + + require.Equal(t, enabled, cfg.SupportsPostFinalizationELSync) + }) + } +} diff --git a/op-devstack/sysgo/l2_el_opreth.go b/op-devstack/sysgo/l2_el_opreth.go index 61a6ea5adf3..de8510295de 100644 --- a/op-devstack/sysgo/l2_el_opreth.go +++ b/op-devstack/sysgo/l2_el_opreth.go @@ -8,6 +8,7 @@ import ( "sync" "github.com/ethereum-optimism/optimism/op-devstack/devtest" + "github.com/ethereum-optimism/optimism/op-devstack/stack" "github.com/ethereum-optimism/optimism/op-service/eth" "github.com/ethereum-optimism/optimism/op-service/log" "github.com/ethereum-optimism/optimism/op-service/logpipe" @@ -132,7 +133,10 @@ type OpReth struct { l2MetricsRegistrar L2MetricsRegistrar } -var _ L2ELNode = (*OpReth)(nil) +var ( + _ L2ELNode = (*OpReth)(nil) + _ stack.UngracefulLifecycle = (*OpReth)(nil) +) func (n *OpReth) Start() { n.mu.Lock() @@ -246,6 +250,23 @@ func (n *OpReth) Stop() { n.sub = nil } +// StopUngracefully ends the op-reth process with SIGKILL, the way a crash would, so whatever it +// had not yet handed to the OS for its datadir is lost. It fails the test if op-reth had already +// exited on its own, since that crash is not the stop the test asked for. Start brings the node +// back on the same datadir. +func (n *OpReth) StopUngracefully() { + n.mu.Lock() + defer n.mu.Unlock() + if n.sub == nil { + n.p.Logger().Warn("op-reth already stopped") + return + } + n.p.Logger().Info("Stopping op-reth ungracefully", "name", n.name, "chain", n.chainID) + n.clearProxyUpstreams() + n.p.Require().NoError(n.sub.Kill(), "failed to kill op-reth") + n.sub = nil +} + // Callers must hold n.mu. func (n *OpReth) clearProxyUpstreams() { if n.userProxy != nil { diff --git a/op-devstack/sysgo/l2_el_p2p_util.go b/op-devstack/sysgo/l2_el_p2p_util.go index a0ccee24176..e0f9b84aa19 100644 --- a/op-devstack/sysgo/l2_el_p2p_util.go +++ b/op-devstack/sysgo/l2_el_p2p_util.go @@ -162,6 +162,30 @@ func DisconnectP2P(ctx context.Context, require *testreq.Assertions, initiator R require.NoError(err, "The peer was not removed") } +// ForgetP2PPeer drops node's peer entry for peerEnode and waits until node no longer lists that +// peer. It needs only node to be reachable, so it can forget a peer that is down. The trusted +// status goes first: reth's admin_removePeer leaves a trusted entry in place. Reth answers both +// removals with true whether or not it held the entry, so only admin_peers shows the outcome. +func ForgetP2PPeer(ctx context.Context, require *testreq.Assertions, node RpcCaller, peerEnode string) { + forgottenNode, err := enode.ParseV4(peerEnode) + require.NoError(err, "failed to parse the enode of the peer to forget") + forgottenID := forgottenNode.ID().String() + + for _, method := range []string{"admin_removeTrustedPeer", "admin_removePeer"} { + var removed bool + require.NoErrorf(node.CallContext(ctx, &removed, method, peerEnode), "%s failed", method) + } + + waitCtx, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + err = forPeerList(waitCtx, node, func(peers []peer) bool { + return !slices.ContainsFunc(peers, func(p peer) bool { + return strings.TrimPrefix(strings.ToLower(p.ID), "0x") == forgottenID + }) + }) + require.NoError(err, "the forgotten peer is still connected") +} + type peer struct { ID string `json:"id"` } diff --git a/op-devstack/sysgo/mixed_runtime.go b/op-devstack/sysgo/mixed_runtime.go index bdb6e8b0df6..fa910730607 100644 --- a/op-devstack/sysgo/mixed_runtime.go +++ b/op-devstack/sysgo/mixed_runtime.go @@ -176,6 +176,10 @@ type MixedSingleChainNodeSpec struct { // by deriving from L1. Used to exercise the derivation/force-build path (FCU-with-attributes) // rather than the consolidation path. Defaults to false (fully peered). IsolateFromL2P2P bool + // L2CLOpts are per-node L2 CL options applied after the runtime's own, e.g. + // L2CLVerifierSyncMode for a follower that must EL-sync. kona-node rejects the options it + // cannot honor. + L2CLOpts []L2CLOption // OpRethOpts are per-node op-reth options applied AFTER the shared cfg.OpRethOptions, only to // this node's EL. Use for options that must not reach every node — e.g. a per-node binary // override (OpRethWithBinary) or a flag a single node should receive while its peers reject it. @@ -286,10 +290,10 @@ func NewMixedSingleChainRuntime(t devtest.T, cfg MixedSingleChainPresetConfig) * IsSequencer: spec.IsSequencer, NoDiscovery: true, DependencySet: depSet, - L2CLOptions: []L2CLOption{L2CLOptionFn(func(_ devtest.T, _ ComponentTarget, cfg *L2CLConfig) { + L2CLOptions: append([]L2CLOption{L2CLOptionFn(func(_ devtest.T, _ ComponentTarget, cfg *L2CLConfig) { cfg.SafeDBPath = safeDBPath cfg.VerifierL1Confs = spec.VerifierL1Confs - })}, + })}, spec.L2CLOpts...), }) case MixedL2CLKona: cl = startMixedKonaNode( @@ -304,12 +308,12 @@ func NewMixedSingleChainRuntime(t devtest.T, cfg MixedSingleChainPresetConfig) * spec.ELKey, konaNodeStartConfig{ DependencySet: depSet, - L2CLConfig: resolveKonaL2CLConfig(t, spec.CLKey, l2Net.ChainID(), spec.IsSequencer, "", []L2CLOption{ + L2CLConfig: resolveKonaL2CLConfig(t, spec.CLKey, l2Net.ChainID(), spec.IsSequencer, "", append([]L2CLOption{ L2CLOptionFn(func(_ devtest.T, _ ComponentTarget, cfg *L2CLConfig) { cfg.SafeDBPath = safeDBPath cfg.VerifierL1Confs = spec.VerifierL1Confs }), - }), + }, spec.L2CLOpts...)), }, ) default: diff --git a/op-devstack/sysgo/singlechain_build.go b/op-devstack/sysgo/singlechain_build.go index 5e8a3533be1..e2fff1cb089 100644 --- a/op-devstack/sysgo/singlechain_build.go +++ b/op-devstack/sysgo/singlechain_build.go @@ -415,7 +415,7 @@ func startL2CLNode( Sync: nodeSync.Config{ SyncMode: syncMode, SkipSyncStartCheck: false, - SupportsPostFinalizationELSync: false, + SupportsPostFinalizationELSync: cfg.SupportsPostFinalizationELSync, L2FollowSourceEndpoint: cfg.FollowSource, // Mirror op-node/service.go: a follow-mode sequencer needs a single // initial engine reset to trigger block building (TryInitialResetEngineForSequencer). diff --git a/op-devstack/sysgo/subproc.go b/op-devstack/sysgo/subproc.go index f86f582922f..1f216412584 100644 --- a/op-devstack/sysgo/subproc.go +++ b/op-devstack/sysgo/subproc.go @@ -7,6 +7,7 @@ import ( "os" "os/exec" "sync" + "syscall" "time" "github.com/ethereum-optimism/optimism/op-devstack/devtest" @@ -142,6 +143,62 @@ func (sp *SubProcess) stop(ctx context.Context, interrupt bool, interruptWait ti return sp.completeStopLocked(interrupt, sp.waitErr) } +// Kill ends the process with SIGKILL, so none of its shutdown logic runs: a crash, not a power +// loss. A process that had already exited on its own is released, and Kill returns that exit as +// an error. +func (sp *SubProcess) Kill() error { + sp.mu.Lock() + defer sp.mu.Unlock() + if sp.cmd == nil { + return nil // already stopped + } + + select { + case <-sp.exited: + return sp.releaseExitedBeforeKillLocked() + default: + } + + sp.p.Logger().Info("Killing sub-process", "pid", sp.cmd.Process.Pid) + killErr := sp.cmd.Process.Kill() + if killErr != nil && !errors.Is(killErr, os.ErrProcessDone) { + sp.p.Logger().Error("Failed to kill sub-process", "pid", sp.cmd.Process.Pid, "err", killErr) + return fmt.Errorf("failed to kill sub-process: %w", killErr) + } + if !waitProcess(context.Background(), sp.exited, controlledKillWait) { + sp.p.Logger().Error("Sub-process did not exit after SIGKILL", "pid", sp.cmd.Process.Pid, "wait", controlledKillWait) + return fmt.Errorf("sub-process did not exit within %s of SIGKILL", controlledKillWait) + } + // The process can also exit on its own after the check above: Wait either reaps it before the + // signal (ErrProcessDone) or reaps it after the signal landed on the zombie, changing nothing. + if errors.Is(killErr, os.ErrProcessDone) || !terminatedBySIGKILL(sp.waitErr) { + return sp.releaseExitedBeforeKillLocked() + } + sp.p.Logger().Info("Sub-process killed", "exit", sp.waitErr) + sp.releaseLocked() + return nil +} + +// releaseExitedBeforeKillLocked releases a process that exited on its own and returns the error +// naming that exit. +func (sp *SubProcess) releaseExitedBeforeKillLocked() error { + sp.p.Logger().Error("Sub-process exited before it could be killed", "exit", sp.waitErr) + sp.releaseLocked() + if sp.waitErr == nil { + return errors.New("sub-process exited with status 0 before it could be killed") + } + return fmt.Errorf("sub-process exited before it could be killed: %w", sp.waitErr) +} + +func terminatedBySIGKILL(waitErr error) bool { + var exitErr *exec.ExitError + if !errors.As(waitErr, &exitErr) { + return false + } + status, ok := exitErr.Sys().(syscall.WaitStatus) + return ok && status.Signaled() && status.Signal() == syscall.SIGKILL +} + func (sp *SubProcess) completeStopLocked(interrupt bool, waitErr error) error { var exitErr *exec.ExitError if waitErr != nil && !(interrupt && errors.As(waitErr, &exitErr)) { @@ -151,7 +208,12 @@ func (sp *SubProcess) completeStopLocked(interrupt bool, waitErr error) error { } else { sp.p.Logger().Info("Sub-process gracefully exited") } + sp.releaseLocked() + return nil +} +// releaseLocked closes the log pipes and forgets the exited process so Start can run a new one. +func (sp *SubProcess) releaseLocked() { if sp.stdOutProc != nil { _ = sp.stdOutProc.Close() sp.stdOutProc = nil @@ -161,7 +223,6 @@ func (sp *SubProcess) completeStopLocked(interrupt bool, waitErr error) error { sp.stdErrProc = nil } sp.cmd = nil - return nil } func waitProcess(ctx context.Context, exited <-chan struct{}, timeout time.Duration) bool { diff --git a/op-devstack/sysgo/subproc_test.go b/op-devstack/sysgo/subproc_test.go index e0173db840b..fb8955b78c4 100644 --- a/op-devstack/sysgo/subproc_test.go +++ b/op-devstack/sysgo/subproc_test.go @@ -2,6 +2,9 @@ package sysgo import ( "context" + "io" + "os" + "os/exec" "testing" "time" @@ -64,7 +67,7 @@ func TestSubProcessConcurrentStop(gt *testing.T) { }) sp := NewSubProcess(p, logCallback, logCallback) - require.NoError(gt, sp.Start("/bin/sh", []string{"-c", "trap '' INT; exec /bin/sleep 10000000000"}, []string{})) + require.NoError(gt, sp.Start("/bin/sh", []string{"-c", "trap '' INT; exec /bin/sleep 3600"}, []string{})) start := make(chan struct{}) errCh := make(chan error, 2) @@ -85,6 +88,108 @@ func TestSubProcessConcurrentStop(gt *testing.T) { require.NoError(gt, sp.Stop(false)) } +func newTestSubProcess(gt *testing.T) (*SubProcess, *testlog.CapturingHandler) { + tLog := testlog.Logger(gt, log.LevelInfo) + logger, capt := testlog.CaptureLogger(gt, log.LevelInfo) + p := devtest.NewP(context.Background(), logger, func(bool) { panic("fail") }, func() { panic("skip") }) + gt.Cleanup(p.Close) + logCallback := logpipe.LogCallback(func(line []byte) { + logger.Info(string(line)) + tLog.Info("Sub-process logged message", "line", string(line)) + }) + return NewSubProcess(p, logCallback, logCallback), capt +} + +func TestSubProcessKillEndsProcessThatIgnoresInterrupts(gt *testing.T) { + sp, capt := newTestSubProcess(gt) + require.NoError(gt, sp.Start("/bin/sh", []string{"-c", "trap '' INT TERM; exec /bin/sleep 3600"}, []string{})) + + require.NoError(gt, sp.Kill()) + + require.Nil(gt, capt.FindLog(testlog.NewMessageFilter("Sending interrupt")), "Kill must not interrupt first") + require.NotNil(gt, capt.FindLog(testlog.NewMessageFilter("Sub-process killed"))) + + require.NoError(gt, sp.Start("/bin/echo", []string{"restart ok"}, []string{}), "the sub-process must restart after a kill") + require.NoError(gt, sp.Stop(false)) + require.NotNil(gt, capt.FindLog(testlog.NewMessageFilter("restart ok"))) +} + +func TestSubProcessKillOfExitedProcess(gt *testing.T) { + sp, capt := newTestSubProcess(gt) + require.NoError(gt, sp.Start("/bin/echo", []string{"done"}, []string{})) + <-sp.Exited() + + require.ErrorContains(gt, sp.Kill(), "exited with status 0 before it could be killed", "an exit of its own must fail the kill") + require.NotNil(gt, capt.FindLog(testlog.NewMessageFilter("Sub-process exited before it could be killed"))) + require.Nil(gt, capt.FindLog(testlog.NewMessageFilter("Sub-process killed")), "an exit of its own must not read as a kill") + require.NoError(gt, sp.Kill(), "a second kill is a no-op") +} + +func TestSubProcessKillOfCrashedProcess(gt *testing.T) { + sp, _ := newTestSubProcess(gt) + require.NoError(gt, sp.Start("/bin/sh", []string{"-c", "exit 3"}, []string{})) + <-sp.Exited() + + err := sp.Kill() + var exitErr *exec.ExitError + require.ErrorAs(gt, err, &exitErr, "the kill must report the exit it found") + require.Equal(gt, 3, exitErr.ExitCode()) + require.NoError(gt, sp.Start("/bin/echo", []string{"restart ok"}, []string{}), "the sub-process must restart after a failed kill") + require.NoError(gt, sp.Stop(false)) +} + +func TestSubProcessKillOfProcessThatExitedBeforeItWasReaped(gt *testing.T) { + sp, capt := newTestSubProcess(gt) + childExitedReader, childStdout, err := os.Pipe() + require.NoError(gt, err) + defer childExitedReader.Close() + cmd := exec.Command("/bin/sh", "-c", "exit 3") + cmd.Stdout = childStdout + require.NoError(gt, cmd.Start()) + require.NoError(gt, childStdout.Close()) + // The pipe reaches EOF once the child has exited, while it stays unreaped until reap closes. + _, err = io.ReadAll(childExitedReader) + require.NoError(gt, err) + + reap := make(chan struct{}) + sp.cmd, sp.exited = cmd, make(chan struct{}) + go func() { + <-reap + sp.waitErr = cmd.Wait() + close(sp.exited) + }() + killErrCh := make(chan error, 1) + go func() { killErrCh <- sp.Kill() }() + require.Eventually(gt, func() bool { + return capt.FindLog(testlog.NewMessageFilter("Killing sub-process")) != nil + }, 5*time.Second, 10*time.Millisecond, "the kill must reach the unreaped process") + close(reap) + + killErr := <-killErrCh + var exitErr *exec.ExitError + require.ErrorAs(gt, killErr, &exitErr, "the kill must report the exit that preceded it") + require.Equal(gt, 3, exitErr.ExitCode()) + require.Nil(gt, capt.FindLog(testlog.NewMessageFilter("Sub-process killed")), "an exit of its own must not read as a kill") +} + +func TestTerminatedBySIGKILL(gt *testing.T) { + for _, testCase := range []struct { + name string + script string + want bool + }{ + {name: "exit_0", script: "exit 0", want: false}, + {name: "exit_3", script: "exit 3", want: false}, + {name: "sigkill", script: "kill -KILL $$", want: true}, + {name: "sigterm", script: "kill -TERM $$", want: false}, + } { + gt.Run(testCase.name, func(gt *testing.T) { + waitErr := exec.Command("/bin/sh", "-c", testCase.script).Run() + require.Equal(gt, testCase.want, terminatedBySIGKILL(waitErr), "wait result: %v", waitErr) + }) + } +} + // testEcho tests that we can handle a sub-process that completes on its own func testEcho(gt *testing.T, capt *testlog.CapturingHandler, sp *SubProcess) { require.NoError(gt, sp.Start("/bin/echo", []string{"hello world"}, []string{}))