diff --git a/op-acceptance-tests/tests/proofs_history_init/proofs_history_init_test.go b/op-acceptance-tests/tests/proofs_history_init/proofs_history_init_test.go new file mode 100644 index 00000000000..52bb2d2fa64 --- /dev/null +++ b/op-acceptance-tests/tests/proofs_history_init/proofs_history_init_test.go @@ -0,0 +1,421 @@ +package proofs_history_init + +import ( + "fmt" + "testing" + "time" + + "github.com/ethereum-optimism/optimism/op-devstack/devtest" + "github.com/ethereum-optimism/optimism/op-devstack/dsl" + "github.com/ethereum-optimism/optimism/op-devstack/presets" + "github.com/ethereum-optimism/optimism/op-devstack/stack" + "github.com/ethereum-optimism/optimism/op-devstack/sysgo" + "github.com/ethereum-optimism/optimism/op-service/bigs" + "github.com/ethereum-optimism/optimism/op-service/eth" + safety "github.com/ethereum-optimism/optimism/op-service/eth/safety" + "github.com/ethereum-optimism/optimism/op-service/txplan" + "github.com/ethereum/go-ethereum/common" +) + +const ( + // l2BlockTimeSeconds keeps the runs short: the chain has to grow past a window before the + // init and past another one for the prune, so halving the block time halves the run. + l2BlockTimeSeconds = 1 + // memoryBlockBufferTarget is reth's --engine.memory-block-buffer-target. It is passed + // explicitly, like the persistence threshold, so how far reth's datadir trails its head does + // not depend on reth's defaults. + memoryBlockBufferTarget = 5 + // postRestartTrafficBlocks is how far past the head the chain is driven once the follower is + // back, so the sweeps also cover blocks the store indexed live after the init. + postRestartTrafficBlocks = 10 + // proofsHistoryWindow is the window the init backfills behind its anchor and the node keeps + // afterwards: small enough that pruning moves the window within the test, large enough that + // the backfill spans blocks reth has persisted and blocks it still held in memory. + proofsHistoryWindow = 30 + // referenceEthProofWindow lets the sequencer answer eth_getProof for every block of the run + // from reth's own historical state; reth's default window of zero rejects them all. + referenceEthProofWindow = 1_000_000 + // windowMargin is how far past a full window the init's anchor must sit, so the backfill + // target lies above genesis and the store's earliest block is that target, not the clamp. + windowMargin = 5 + + // pruneTimeout bounds how long the store may take to prune the backfilled blocks once the + // chain is a window past the anchor. + pruneTimeout = 90 * time.Second + // resyncAttempts bounds how long the restarted follower may take to catch up with the + // sequencer. + resyncAttempts = 60 + // storeCatchUpTimeout bounds how long the restarted node may take to serve its seeded window. + storeCatchUpTimeout = 60 * time.Second +) + +// followerSpec says how the follower's op-reth runs. Its proofs history is off until a test seeds +// the store with proofs init. +type followerSpec struct { + persistenceThreshold uint64 + stateMaskingBlocks uint64 + verificationInterval uint64 +} + +func (s followerSpec) proofsPath() string { + if s.verificationInterval != 0 { + return "execute" + } + return "index" +} + +// engineArgs pin reth's persistence schedule, which decides how far the datadir trails the head +// at the init. Both nodes get them so the sequencer persists on the same schedule as the follower. +func (s followerSpec) engineArgs() []string { + args := []string{ + fmt.Sprintf("--engine.memory-block-buffer-target=%d", memoryBlockBufferTarget), + fmt.Sprintf("--engine.persistence-threshold=%d", s.persistenceThreshold), + } + if s.stateMaskingBlocks != 0 { + args = append(args, fmt.Sprintf("--engine.num-state-masking-blocks=%d", s.stateMaskingBlocks)) + } + return args +} + +func (s followerSpec) followerArgs() []string { + return append(s.engineArgs(), fmt.Sprintf("--proofs-history.verification-interval=%d", s.verificationInterval)) +} + +// headWithPersistedWindow is a chain height by which reth's datadir holds more than a window of +// blocks under the spec's threshold, whatever reth still has in memory: reth persists nothing +// until more than the threshold of blocks sit in memory, and from then on its datadir trails the +// head by at most the threshold. +func (s followerSpec) headWithPersistedWindow() uint64 { + return s.persistenceThreshold + proofsHistoryWindow + windowMargin +} + +// name identifies a guard cell: the guards are about reth's datadir, so the cells vary what +// shapes it, the persistence schedule and state masking. +func (s followerSpec) name() string { + return fmt.Sprintf("threshold_%d-masking_%d", s.persistenceThreshold, s.stateMaskingBlocks) +} + +// initCase is one cell of the matrix the operator path runs under. +type initCase struct { + plainBackfill bool + spec followerSpec +} + +func (c initCase) name() string { + backfill := "snapshot" + if c.plainBackfill { + backfill = "plain" + } + return fmt.Sprintf("threshold_%d-%s-backfill_%s", c.spec.persistenceThreshold, c.spec.proofsPath(), backfill) +} + +func initMatrix() []initCase { + var cases []initCase + for _, persistenceThreshold := range []uint64{7, 50} { + for _, verificationInterval := range []uint64{0, 1} { + for _, plainBackfill := range []bool{false, true} { + cases = append(cases, initCase{ + plainBackfill: plainBackfill, + spec: followerSpec{persistenceThreshold: persistenceThreshold, verificationInterval: verificationInterval}, + }) + } + } + } + return cases +} + +func guardMatrix() []followerSpec { + var cases []followerSpec + for _, persistenceThreshold := range []uint64{7, 50} { + for _, stateMaskingBlocks := range []uint64{0, 30} { + cases = append(cases, followerSpec{persistenceThreshold: persistenceThreshold, stateMaskingBlocks: stateMaskingBlocks}) + } + } + return cases +} + +// skipMaskingCell skips a cell that needs state masking, which develop cannot run. +func skipMaskingCell(t devtest.T, spec followerSpec) { + if spec.stateMaskingBlocks != 0 { + t.Skip("state masking is compiled out of reth 2.5.2 on develop and #23046 rejects a nonzero --engine.num-state-masking-blocks; re-enable with ethereum-optimism/protocol-team#457, where the init against a killed node must fail with InitializeSourceStateTrieLagging until a clean restart repairs the datadir") + } +} + +type initSystem struct { + Follower *dsl.L2ELNode + FollowerCL *dsl.L2CLNode + FunderL2 *dsl.FunderEOA + Sequencer *dsl.L2ELNode + SequencerCL *dsl.L2CLNode +} + +// newInitSystem starts a sequencer without proofs history, which serves as the independent +// eth_getProof reference, and a follower without proofs history under the spec's reth and ExEx +// settings, configured with the window its store gets once a test seeds it. +func newInitSystem(t devtest.T, spec followerSpec) *initSystem { + sysgo.SkipOnOpGeth(t, "proofs history is an op-reth feature") + clKind := sysgo.ResolveMixedL2CLKind() + runtime := sysgo.NewMixedSingleChainRuntime(t, sysgo.MixedSingleChainPresetConfig{ + DeployerOptions: []sysgo.DeployerOption{sysgo.WithUniformL2BlockTimes(l2BlockTimeSeconds)}, + NodeSpecs: []sysgo.MixedSingleChainNodeSpec{ + { + CLKey: "sequencer", + CLKind: clKind, + ELKey: "sequencer", + ELKind: sysgo.MixedL2ELOpReth, + IsSequencer: true, + OpRethOpts: []sysgo.OpRethOption{ + sysgo.OpRethWithoutProofsHistory(), + sysgo.OpRethWithEthProofWindow(referenceEthProofWindow), + sysgo.OpRethWithExtraArgs(spec.engineArgs()...), + }, + }, + { + CLKey: "follower", + CLKind: clKind, + ELKey: "follower", + ELKind: sysgo.MixedL2ELOpReth, + OpRethOpts: []sysgo.OpRethOption{ + sysgo.OpRethWithoutProofsHistory(), + sysgo.OpRethWithProofsHistoryWindow(proofsHistoryWindow), + sysgo.OpRethWithExtraArgs(spec.followerArgs()...), + }, + }, + }, + }) + frontends := presets.NewMixedSingleChainFrontends(t, runtime) + t.Require().Len(frontends.Nodes, 2, "expected the sequencer and the follower") + + sys := &initSystem{FunderL2: frontends.FunderL2} + for _, node := range frontends.Nodes { + if node.Spec.IsSequencer { + sys.Sequencer, sys.SequencerCL = node.EL, node.CL + } else { + sys.Follower, sys.FollowerCL = node.EL, node.CL + } + } + t.Require().NotNil(sys.Follower, "missing the follower") + return sys +} + +// workload changes state in nearly every block: alice and bob transfer back and forth and bob +// writes to a contract, so every block the backfill reconstructs carries balance and storage +// changes. +type workload struct { + alice *dsl.EOA + bob *dsl.EOA + recorder common.Address + t devtest.T +} + +func newWorkload(t devtest.T, funder *dsl.FunderEOA) *workload { + w := &workload{ + alice: funder.NewFundedEOA(eth.OneEther), + bob: funder.NewFundedEOA(eth.OneEther), + t: t, + } + w.recorder = w.alice.DeployBlockRecorder() + return w +} + +// accounts are the accounts the proofs sweeps prove at every block. +func (w *workload) accounts() []dsl.ProofAccount { + return []dsl.ProofAccount{ + {Address: w.alice.Address()}, + {Address: w.bob.Address()}, + {Address: w.recorder, StorageKeys: []common.Hash{common.HexToHash("0x0"), common.HexToHash("0x1")}}, + } +} + +// driveUntilBlock sends transactions one after another until one lands in a block at or past +// target, and returns that block. +func (w *workload) driveUntilBlock(target uint64) uint64 { + for { + w.alice.Transfer(w.bob.Address(), eth.OneHundredthEther) + w.bob.Transfer(w.alice.Address(), eth.OneHundredthEther) + block := includedBlock(w.t, w.bob.Transact(w.bob.Plan(), txplan.WithTo(&w.recorder))) + if block >= target { + return block + } + } +} + +func includedBlock(t devtest.T, tx *txplan.PlannedTx) uint64 { + receipt, err := tx.Included.Eval(t.Ctx()) + t.Require().NoError(err, "transaction must be included") + return bigs.Uint64Strict(receipt.BlockNumber) +} + +// driveChainTo runs the workload until the chain reaches target and the follower has imported +// all of it, and returns the follower's head. +func driveChainTo(sys *initSystem, w *workload, target uint64) eth.L2BlockRef { + w.driveUntilBlock(target) + sys.Follower.InSync(sys.Sequencer, safety.LocalUnsafe, resyncAttempts) + return sys.Follower.BlockRefByLabel(eth.Unsafe) +} + +// restartFollowerEL brings the follower's op-reth back on its datadir, now with proofs history, +// and returns the datadir tip it resumed from. Nothing feeds the node until its CL is back, so +// the tip is what reth had persisted, and the store serves exactly what the init seeded. +func restartFollowerEL(t devtest.T, sys *initSystem) eth.L2BlockRef { + sys.Follower.Start() + sys.Follower.WaitForOnline() + tip := sys.Follower.BlockRefByLabel(eth.Unsafe) + t.Logger().Info("Follower op-reth is back on its datadir with proofs history", "datadir_tip", tip) + return tip +} + +// reconnectFollower re-peers the follower's op-reth with the sequencer, brings its CL back and +// waits until the follower has caught up with the sequencer. +func reconnectFollower(sys *initSystem) { + sys.Follower.PeerWith(sys.Sequencer) + sys.FollowerCL.Start() + sys.FollowerCL.ConnectPeer(sys.SequencerCL) + sys.Follower.InSync(sys.Sequencer, safety.LocalUnsafe, resyncAttempts) +} + +// verifySeededWindow requires the store seeded at anchor to span exactly one window behind it and +// sweeps that window against the sequencer. It runs before the follower's CL is back, so nothing +// extends or prunes the store meanwhile. +func verifySeededWindow(t devtest.T, sys *initSystem, w *workload, anchor eth.L2BlockRef) { + t.Require().GreaterOrEqualf(anchor.Number, uint64(proofsHistoryWindow), + "the anchor %s must be at least a window (%d blocks) above genesis for the backfill to span a full window", anchor, proofsHistoryWindow) + window := sys.Follower.WaitForProofsStoreBlock(anchor.Number, storeCatchUpTimeout) + t.Require().Equalf(anchor.Number, *window.Latest, "the store must be anchored at the datadir tip %s; store window %s", anchor, window) + t.Require().Equalf(anchor.Number-proofsHistoryWindow, *window.Earliest, + "the backfill must extend the store one window (%d blocks) behind the anchor %s; store window %s", proofsHistoryWindow, anchor, window) + sys.Follower.VerifyHistoricalProofsMatch(sys.Sequencer, func(opts *dsl.HistoricalProofsOpts) { + opts.Accounts = w.accounts() + opts.FirstBlock = *window.Earliest + opts.LastBlock = anchor.Number + }) +} + +// verifyLiveWindow drives traffic past the current head and sweeps the window the store serves +// then, which now holds blocks the store indexed live. +func verifyLiveWindow(sys *initSystem, w *workload) { + lastBlock := w.driveUntilBlock(sys.Sequencer.BlockRefByLabel(eth.Unsafe).Number + postRestartTrafficBlocks) + sys.Follower.VerifyHistoricalProofsMatch(sys.Sequencer, func(opts *dsl.HistoricalProofsOpts) { + opts.Accounts = w.accounts() + opts.LastBlock = lastBlock + }) +} + +// TestFollowerServesTheWholeWindowAfterProofsInitWithBackfill runs the operator path: a follower +// that synced without proofs history is stopped cleanly, `op-reth proofs init` seeds its store +// at the datadir tip and backfills a window behind it, and the restarted follower must serve +// proofs for the whole window that verify against the headers and match the sequencer's, then +// keep doing so once pruning has moved the window past the init's anchor. +func TestFollowerServesTheWholeWindowAfterProofsInitWithBackfill(gt *testing.T) { + t := devtest.ParallelT(gt) + for _, c := range initMatrix() { + t.Run(c.name(), func(t devtest.T) { + t.Parallel() + sys := newInitSystem(t, c.spec) + w := newWorkload(t, sys.FunderL2) + // A clean stop persists what reth holds in memory, so the datadir tip is the head + // whatever the threshold, and a window plus margin of blocks is enough. + head := driveChainTo(sys, w, proofsHistoryWindow+windowMargin) + + sys.FollowerCL.Stop() + sys.Follower.Stop() + report := sys.Follower.InitProofsHistory(func(init *stack.ProofsHistoryInit) { + init.PlainBackfill = c.plainBackfill + }) + t.Logger().Info("Proofs init seeded the follower's store", "cell", c.name(), "init_duration", report.Duration, + "head_at_stop", head, "window", proofsHistoryWindow) + + anchor := restartFollowerEL(t, sys) + verifySeededWindow(t, sys, w, anchor) + + reconnectFollower(sys) + lastBlock := w.driveUntilBlock(anchor.Number + proofsHistoryWindow + windowMargin) + window := sys.Follower.WaitForProofsStorePrunedPast(anchor.Number, pruneTimeout) + t.Logger().Info("Pruning moved the window past the init's anchor", "anchor", anchor, "window", window) + sys.Follower.VerifyHistoricalProofsMatch(sys.Sequencer, func(opts *dsl.HistoricalProofsOpts) { + opts.Accounts = w.accounts() + opts.LastBlock = lastBlock + }) + }) + } +} + +// TestProofsInitSeedsAConsistentStoreFromAKilledNodeDatadir kills a follower that synced without +// proofs history, so reth's datadir trails the head by the blocks it still held in memory, and +// runs `op-reth proofs init` against that datadir. The init must seed a store anchored at the +// persisted tip with a full window behind it, and once the follower is back it must serve that +// window and the blocks it re-imports with proofs that match the sequencer's. +func TestProofsInitSeedsAConsistentStoreFromAKilledNodeDatadir(gt *testing.T) { + t := devtest.ParallelT(gt) + for _, spec := range guardMatrix() { + t.Run(spec.name(), func(t devtest.T) { + t.Parallel() + skipMaskingCell(t, spec) + sys := newInitSystem(t, spec) + w := newWorkload(t, sys.FunderL2) + head := driveChainTo(sys, w, spec.headWithPersistedWindow()) + + sys.Follower.StopUngracefully() + sys.FollowerCL.Stop() + report := sys.Follower.InitProofsHistory() + t.Logger().Info("Proofs init seeded the killed follower's store", "cell", spec.name(), "init_duration", report.Duration, "head_at_kill", head) + + persistedTip := restartFollowerEL(t, sys) + t.Require().Lessf(persistedTip.Number, head.Number, + "the kill must leave reth's datadir (tip %d) behind the head %d, or the init ran against a datadir no different from a clean stop", + persistedTip.Number, head.Number) + verifySeededWindow(t, sys, w, persistedTip) + + reconnectFollower(sys) + verifyLiveWindow(sys, w) + }) + } +} + +// TestProofsInitAgainstARunningNodeLeavesAStoreThatMatchesTheChain runs `op-reth proofs init` +// against the datadir of a follower that is still running and persisting, which an operator can +// do by mistake. Whatever the init makes of that datadir, the store the follower ends up running +// on must describe the chain: after a clean stop, a seeding of a store the init refused, and a +// restart with proofs history, every proof of the window must verify against its header and match +// the sequencer's. +func TestProofsInitAgainstARunningNodeLeavesAStoreThatMatchesTheChain(gt *testing.T) { + t := devtest.ParallelT(gt) + for _, spec := range guardMatrix() { + t.Run(spec.name(), func(t devtest.T) { + t.Parallel() + skipMaskingCell(t, spec) + sys := newInitSystem(t, spec) + w := newWorkload(t, sys.FunderL2) + // The init reads reth's persisted tip, so the chain is driven until that tip is a + // window above genesis under the spec's threshold. + head := driveChainTo(sys, w, spec.headWithPersistedWindow()) + + report, err := sys.Follower.InitProofsHistoryRaw() + if err != nil { + // The init may refuse a datadir whose tip moved between its two reads (#23046); + // any other failure is a bug. + t.Require().ErrorContainsf(err, "reth database tip moved", + "proofs init against a running node may only fail because reth's tip moved (#23046); it failed with: %v", err) + } + t.Logger().Info("Proofs init ran against the running follower", "cell", spec.name(), "init_duration", report.Duration, + "head_at_init", head, "refused", err != nil) + + sys.FollowerCL.Stop() + sys.Follower.Stop() + if err != nil { + sys.Follower.InitProofsHistory() + } + + tip := restartFollowerEL(t, sys) + window := sys.Follower.WaitForProofsStoreBlock(tip.Number, storeCatchUpTimeout) + t.Logger().Info("Follower store caught up with its datadir", "datadir_tip", tip, "window", window) + sys.Follower.VerifyHistoricalProofsMatch(sys.Sequencer, func(opts *dsl.HistoricalProofsOpts) { + opts.Accounts = w.accounts() + opts.LastBlock = tip.Number + }) + + reconnectFollower(sys) + verifyLiveWindow(sys, w) + }) + } +} diff --git a/op-devstack/dsl/l2_el_proofs.go b/op-devstack/dsl/l2_el_proofs.go index 047b9949f21..44dc1de51e9 100644 --- a/op-devstack/dsl/l2_el_proofs.go +++ b/op-devstack/dsl/l2_el_proofs.go @@ -8,6 +8,7 @@ import ( "time" "github.com/ethereum-optimism/optimism/op-core/predeploys" + "github.com/ethereum-optimism/optimism/op-devstack/stack" "github.com/ethereum-optimism/optimism/op-e2e/e2eutils/wait" "github.com/ethereum-optimism/optimism/op-service/eth" "github.com/ethereum/go-ethereum/common" @@ -107,6 +108,73 @@ func (el *L2ELNode) WaitForProofsStoreBlock(target uint64, timeout time.Duration return lastStatus } +// WaitForProofsStorePrunedPast waits until this node's proofs-history store no longer serves +// block, because pruning moved the store's earliest block past it. The store prunes to its +// window behind the latest block it indexed, so the chain has to be more than a window past +// block first. +func (el *L2ELNode) WaitForProofsStorePrunedPast(block uint64, timeout time.Duration) ProofsSyncStatus { + ctx, cancel := context.WithTimeout(el.ctx, timeout) + defer cancel() + logger := el.log.With("name", el.inner.Name(), "chain", el.ChainID(), "block", block, "timeout", timeout) + logger.Info("Waiting for proofs store to prune past block") + + var lastStatus ProofsSyncStatus + var lastErr error + err := wait.For(ctx, 200*time.Millisecond, func() (bool, error) { + status, err := el.ProofsSyncStatus() + if err != nil { + lastErr = err + logger.Warn("debug_proofsSyncStatus failed; will retry", "err", err) + return false, nil + } + lastStatus, lastErr = status, nil + if status.Earliest != nil && *status.Earliest > block { + logger.Info("Proofs store pruned past block", "window", status) + return true, nil + } + logger.Info("Proofs store still serves block", "window", status) + return false, nil + }) + el.require.NoErrorf(err, "proofs store on %s did not prune past block %d within %s; last window %s; last error: %v", + el.inner.Name(), block, timeout, lastStatus, lastErr) + return lastStatus +} + +// InitProofsHistory seeds this node's proofs-history store from its datadir with +// `op-reth proofs init`, the way an operator does for a node that synced without proofs history, +// and makes the node run with proofs history from its next Start. It fails the test if the node +// is running or the init fails; InitProofsHistoryRaw does neither. +func (el *L2ELNode) InitProofsHistory(opts ...func(*stack.ProofsHistoryInit)) stack.ProofsHistoryInitReport { + if control, ok := el.inner.(stack.ControlledLifecycle); ok { + el.require.Falsef(control.Running(), "stop %s before seeding its proofs-history store", el.inner.Name()) + } + report, err := el.InitProofsHistoryRaw(opts...) + el.require.NoErrorf(err, "proofs init on %s must seed the store", el.inner.Name()) + return report +} + +// InitProofsHistoryRaw runs `op-reth proofs init` against this node's datadir whether or not the +// node is running and returns the outcome, failure included, for the test to assert on. The +// options start from op-reth's defaults: a backfill through a trie-state snapshot and the node's +// configured window. +func (el *L2ELNode) InitProofsHistoryRaw(opts ...func(*stack.ProofsHistoryInit)) (stack.ProofsHistoryInitReport, error) { + initializer, ok := el.inner.(stack.ProofsHistoryInitializer) + el.require.Truef(ok, "L2EL node %s cannot seed a proofs-history store", el.inner.Name()) + var init stack.ProofsHistoryInit + for _, opt := range opts { + opt(&init) + } + logger := el.log.With("name", el.inner.Name(), "chain", el.ChainID(), "plain_backfill", init.PlainBackfill) + logger.Info("Seeding the proofs-history store from the datadir") + report, err := initializer.InitProofsHistory(init) + if err != nil { + logger.Warn("Proofs init did not seed the store", "duration", report.Duration) + return report, err + } + logger.Info("Proofs init seeded the store", "duration", report.Duration) + return report, nil +} + // VerifyHistoricalProofsMatch sweeps the blocks this node's proofs-history store serves and, at // every sampled block, requires eth_getProof for each configured account to verify against the // block's state root and to equal the proof served by reference. The L2ToL1MessagePasser proof is diff --git a/op-devstack/presets/rpc_frontends.go b/op-devstack/presets/rpc_frontends.go index 9631aff65eb..a192020970a 100644 --- a/op-devstack/presets/rpc_frontends.go +++ b/op-devstack/presets/rpc_frontends.go @@ -158,12 +158,14 @@ type l2ELFrontend struct { control stack.ControlledLifecycle ungraceful stack.UngracefulLifecycle logRecorder stack.LogRecorder + proofsInit stack.ProofsHistoryInitializer } var ( - _ stack.L2ELNode = (*l2ELFrontend)(nil) - _ stack.LogRecorder = (*l2ELFrontend)(nil) - _ stack.UngracefulLifecycle = (*l2ELFrontend)(nil) + _ stack.L2ELNode = (*l2ELFrontend)(nil) + _ stack.LogRecorder = (*l2ELFrontend)(nil) + _ stack.ProofsHistoryInitializer = (*l2ELFrontend)(nil) + _ stack.UngracefulLifecycle = (*l2ELFrontend)(nil) ) func newPresetL2ELNode(t devtest.T, name string, chainID eth.ChainID, userRPCCl opclient.RPC, userRPC string, engineRPCCl opclient.RPC, rollupCfg *rollup.Config) *l2ELFrontend { @@ -210,6 +212,11 @@ func (r *l2ELFrontend) RecordLogLines(substring string) *logpipe.LogRecord { return r.logRecorder.RecordLogLines(substring) } +func (r *l2ELFrontend) InitProofsHistory(init stack.ProofsHistoryInit) (stack.ProofsHistoryInitReport, error) { + r.require().NotNil(r.proofsInit, "L2EL node %s cannot seed a proofs-history store", r.Name()) + return r.proofsInit.InitProofsHistory(init) +} + func (r *l2ELFrontend) StartControlled(ctx context.Context) error { if r.control == nil { return fmt.Errorf("L2EL node %s is not controllable", r.Name()) diff --git a/op-devstack/presets/sysgo_runtime.go b/op-devstack/presets/sysgo_runtime.go index 18b7dcac546..36d968e0879 100644 --- a/op-devstack/presets/sysgo_runtime.go +++ b/op-devstack/presets/sysgo_runtime.go @@ -72,6 +72,9 @@ func newL2ELFrontend(t devtest.T, name string, chainID eth.ChainID, userRPC stri if recorder, ok := lifecycle[0].(stack.LogRecorder); ok { l2EL.logRecorder = recorder } + if initializer, ok := lifecycle[0].(stack.ProofsHistoryInitializer); ok { + l2EL.proofsInit = initializer + } } return l2EL } diff --git a/op-devstack/stack/proofs_history.go b/op-devstack/stack/proofs_history.go new file mode 100644 index 00000000000..de86665f5ce --- /dev/null +++ b/op-devstack/stack/proofs_history.go @@ -0,0 +1,34 @@ +package stack + +import "time" + +// ProofsHistoryInit is one run of `op-reth proofs init` against a node's datadir: it anchors a +// proofs-history store at the datadir's tip, backfills the store Window blocks behind the anchor, +// and makes the node run with proofs history from its next Start on. The zero value is the +// operator's plain invocation. +type ProofsHistoryInit struct { + // PlainBackfill reads every backfilled block's changes from reth's database instead of going + // through the trie-state snapshot op-reth builds by default. + PlainBackfill bool + // Window is the number of blocks the backfill extends behind the anchor and the window the + // node keeps once proofs history is on. Zero keeps the window the node was configured with. + Window uint64 +} + +// ProofsHistoryInitReport is what a proofs-history init run produced. +type ProofsHistoryInitReport struct { + // Duration is how long the subcommand ran, backfill included. + Duration time.Duration + // Output is the subcommand's combined stdout and stderr. + Output string +} + +// ProofsHistoryInitializer is implemented by execution-layer nodes whose proofs-history store can +// be seeded from their datadir after they were built, the way an operator seeds one for a node +// that has synced without proofs history. +type ProofsHistoryInitializer interface { + // InitProofsHistory runs the init against the node's datadir, whether or not the node is + // running, and enables proofs history on the node's next Start when it succeeds. The error of + // a failed run carries the subcommand's output. + InitProofsHistory(init ProofsHistoryInit) (ProofsHistoryInitReport, error) +} diff --git a/op-devstack/sysgo/l2_el_opreth.go b/op-devstack/sysgo/l2_el_opreth.go index 9783a5d81bb..ee0c21b3894 100644 --- a/op-devstack/sysgo/l2_el_opreth.go +++ b/op-devstack/sysgo/l2_el_opreth.go @@ -4,8 +4,11 @@ import ( "context" "fmt" "net/url" + "os/exec" + "strconv" "strings" "sync" + "time" "github.com/ethereum-optimism/optimism/op-devstack/devtest" "github.com/ethereum-optimism/optimism/op-devstack/stack" @@ -24,9 +27,10 @@ type OpRethConfig struct { // (any binary that accepts every op-reth subcommand/flag, plus optionally its own additive // flags) may be selected via OpRethWithBinary. Binary string - // DisableProofsHistory skips the proofs-history init + runtime flags for this node. The mixed - // runtime otherwise enables proofs-history on every op-reth node; some CLI-superset binaries - // reject --proofs-history in the mode under test. + // DisableProofsHistory builds the node without a seeded proofs-history store and without the + // proofs-history runtime flags, until InitProofsHistory seeds the store. The mixed runtime + // otherwise enables proofs history on every op-reth node; some CLI-superset binaries reject + // --proofs-history in the mode under test. DisableProofsHistory bool // ProofsHistoryWindow is the number of blocks the proofs-history store keeps. Zero keeps the // mixed runtime's default window. @@ -84,9 +88,9 @@ func OpRethWithBinary(binary string) OpRethOption { }) } -// OpRethWithoutProofsHistory disables the proofs-history subsystem for this node. The mixed runtime -// enables proofs-history on every op-reth node by default; use this for a CLI-superset binary that -// rejects --proofs-history in the mode under test. +// OpRethWithoutProofsHistory builds the node without proofs history, which InitProofsHistory can +// turn on later. The mixed runtime enables it on every op-reth node by default; use this for a +// CLI-superset binary that rejects --proofs-history in the mode under test. func OpRethWithoutProofsHistory() OpRethOption { return OpRethOptionFn(func(p devtest.T, _ ComponentTarget, cfg *OpRethConfig) { cfg.DisableProofsHistory = true @@ -134,10 +138,23 @@ type OpReth struct { userProxy *tcpproxy.Proxy execPath string - args []string + // args are the `node` arguments without the proofs-history flags, which Start appends while + // proofsHistoryEnabled is set. + args []string // Each entry is of the form "key=value". env []string + chainConfigPath string + dataDirPath string + + // proofsHistoryEnabled says the store at proofsHistoryStoragePath is seeded, so the node runs + // with proofs history from its next start. + proofsHistoryEnabled bool + proofsHistoryStoragePath string + // proofsHistoryWindow is the window proofs init backfills and the node keeps; zero means the + // mixed runtime's default. + proofsHistoryWindow uint64 + p devtest.T sub *SubProcess @@ -150,9 +167,10 @@ type OpReth struct { } var ( - _ L2ELNode = (*OpReth)(nil) - _ stack.LogRecorder = (*OpReth)(nil) - _ stack.UngracefulLifecycle = (*OpReth)(nil) + _ L2ELNode = (*OpReth)(nil) + _ stack.LogRecorder = (*OpReth)(nil) + _ stack.ProofsHistoryInitializer = (*OpReth)(nil) + _ stack.UngracefulLifecycle = (*OpReth)(nil) ) // RecordLogLines returns a record of every line op-reth writes to stdout or stderr from now on, @@ -245,7 +263,7 @@ func (n *OpReth) Start() { }) n.sub = NewSubProcess(n.p, stdOutLogs, stdErrLogs) - err := n.sub.Start(n.execPath, n.args, n.env) + err := n.sub.Start(n.execPath, n.nodeArgs(), n.env) n.p.Require().NoError(err, "Must start") var userRPCAddr, authRPCAddr string @@ -292,6 +310,83 @@ func (n *OpReth) StopUngracefully() { n.sub = nil } +// nodeArgs are the `node` arguments of the next start. Callers must hold n.mu. +func (n *OpReth) nodeArgs() []string { + if !n.proofsHistoryEnabled { + return n.args + } + args := append([]string{}, n.args...) + return append(args, proofsHistoryNodeArgs(n.proofsHistoryWindow, n.proofsHistoryStoragePath)...) +} + +// InitProofsHistory runs `op-reth proofs init` against this node's datadir and, when it succeeds, +// makes the next Start run with proofs history on the seeded store. It does not stop the node: +// what init does to a datadir the node is still writing to is the caller's concern. +func (n *OpReth) InitProofsHistory(init stack.ProofsHistoryInit) (stack.ProofsHistoryInitReport, error) { + n.mu.Lock() + defer n.mu.Unlock() + window := n.proofsHistoryWindow + if init.Window != 0 { + window = init.Window + } + args := proofsHistoryInitArgs(n.chainConfigPath, n.dataDirPath, n.proofsHistoryStoragePath, window, init.PlainBackfill) + logger := n.p.Logger().New("name", n.name, "chain", n.chainID, "running", n.sub != nil) + logger.Info("Running op-reth proofs init", "cmd", n.execPath, "args", args) + started := time.Now() + output, err := exec.CommandContext(n.p.Ctx(), n.execPath, args...).CombinedOutput() + report := stack.ProofsHistoryInitReport{Duration: time.Since(started), Output: string(output)} + if err != nil { + logger.Warn("op-reth proofs init failed", "duration", report.Duration, "err", err, "output", report.Output) + return report, fmt.Errorf("op-reth proofs init failed after %s: %w\n%s", report.Duration, err, report.Output) + } + n.proofsHistoryEnabled = true + n.proofsHistoryWindow = window + logger.Info("op-reth proofs init succeeded; proofs history is on from the next start", + "duration", report.Duration, "window", resolveProofsHistoryWindow(window), "storage_path", n.proofsHistoryStoragePath) + logger.Debug("op-reth proofs init output", "output", report.Output) + return report, nil +} + +// defaultMixedProofsHistoryWindow is the proofs-history window an op-reth node gets unless +// OpRethWithProofsHistoryWindow or a ProofsHistoryInit overrides it. +const defaultMixedProofsHistoryWindow uint64 = 10_000 + +func resolveProofsHistoryWindow(window uint64) uint64 { + if window == 0 { + return defaultMixedProofsHistoryWindow + } + return window +} + +// proofsHistoryNodeArgs returns the node flags that enable proofs history on the store at +// storagePath. +func proofsHistoryNodeArgs(window uint64, storagePath string) []string { + return []string{ + "--proofs-history", + "--proofs-history.storage-path=" + storagePath, + "--proofs-history.window=" + strconv.FormatUint(resolveProofsHistoryWindow(window), 10), + } +} + +// proofsHistoryInitArgs returns the `proofs init` invocation that seeds the store at storagePath +// from the datadir and backfills it window blocks behind the datadir's tip. op-reth backfills +// through a trie-state snapshot unless told not to, so the snapshot path gets the operator's +// invocation with no flag. +func proofsHistoryInitArgs(chainConfigPath, dataDirPath, storagePath string, window uint64, plainBackfill bool) []string { + args := []string{ + "proofs", + "init", + "--chain=" + chainConfigPath, + "--datadir=" + dataDirPath, + "--proofs-history.storage-path=" + storagePath, + "--proofs-history.window=" + strconv.FormatUint(resolveProofsHistoryWindow(window), 10), + } + if plainBackfill { + args = append(args, "--proofs-history.use-snapshot=false") + } + return args +} + // Callers must hold n.mu. func (n *OpReth) clearProxyUpstreams() { if n.userProxy != nil { diff --git a/op-devstack/sysgo/l2_el_opreth_test.go b/op-devstack/sysgo/l2_el_opreth_test.go index 10a27dfe175..4fd9339542e 100644 --- a/op-devstack/sysgo/l2_el_opreth_test.go +++ b/op-devstack/sysgo/l2_el_opreth_test.go @@ -1,10 +1,16 @@ package sysgo import ( + "fmt" + "os" + "path/filepath" "strings" "testing" "github.com/stretchr/testify/require" + + "github.com/ethereum-optimism/optimism/op-devstack/devtest" + "github.com/ethereum-optimism/optimism/op-devstack/stack" ) func argsWithPrefix(args []string, prefix string) []string { @@ -27,7 +33,7 @@ func TestProofsHistoryNodeArgs(t *testing.T) { {name: "explicit window replaces the default", window: 25, wantWindow: "--proofs-history.window=25"}, } { t.Run(tc.name, func(t *testing.T) { - args := proofsHistoryNodeArgs(&OpRethConfig{ProofsHistoryWindow: tc.window}, "/proofs") + args := proofsHistoryNodeArgs(tc.window, "/proofs") require.Contains(t, args, "--proofs-history") require.Contains(t, args, "--proofs-history.storage-path=/proofs") @@ -37,6 +43,31 @@ func TestProofsHistoryNodeArgs(t *testing.T) { } } +func TestProofsHistoryInitArgs(t *testing.T) { + for _, tc := range []struct { + name string + plainBackfill bool + window uint64 + wantSnapshotFlag []string + wantWindow string + }{ + {name: "snapshot backfill is op-reth's default and gets no flag", plainBackfill: false, window: 30, wantSnapshotFlag: nil, wantWindow: "--proofs-history.window=30"}, + {name: "plain backfill turns the snapshot off", plainBackfill: true, window: 30, wantSnapshotFlag: []string{"--proofs-history.use-snapshot=false"}, wantWindow: "--proofs-history.window=30"}, + {name: "zero window uses the runtime default", plainBackfill: false, window: 0, wantSnapshotFlag: nil, wantWindow: "--proofs-history.window=10000"}, + } { + t.Run(tc.name, func(t *testing.T) { + args := proofsHistoryInitArgs("/genesis.json", "/data", "/proofs", tc.window, tc.plainBackfill) + + require.Equal(t, []string{"proofs", "init"}, args[:2]) + require.Contains(t, args, "--chain=/genesis.json") + require.Contains(t, args, "--datadir=/data") + require.Contains(t, args, "--proofs-history.storage-path=/proofs") + require.Equal(t, []string{tc.wantWindow}, argsWithPrefix(args, "--proofs-history.window=")) + require.Equal(t, tc.wantSnapshotFlag, argsWithPrefix(args, "--proofs-history.use-snapshot")) + }) + } +} + func TestOpRethWithProofsHistoryWindow(t *testing.T) { cfg := DefaultOpRethConfig() OpRethWithProofsHistoryWindow(25).Apply(nil, ComponentTarget{}, cfg) @@ -51,3 +82,66 @@ func TestOpRethWithEthProofWindow(t *testing.T) { require.Equal(t, []string{"--rpc.eth-proof-window=1000000"}, cfg.ExtraArgs) } + +func TestOpRethNodeArgsCarryProofsHistoryOnlyOnceEnabled(t *testing.T) { + baseArgs := []string{"node", "--datadir=/data"} + n := &OpReth{args: baseArgs, proofsHistoryStoragePath: "/proofs", proofsHistoryWindow: 25} + + require.Equal(t, baseArgs, n.nodeArgs()) + + n.proofsHistoryEnabled = true + require.Equal(t, append([]string{"node", "--datadir=/data"}, proofsHistoryNodeArgs(25, "/proofs")...), n.nodeArgs()) + require.Equal(t, baseArgs, n.args, "the base arguments must stay free of the proofs-history flags") +} + +func TestInitProofsHistoryEnablesProofsHistoryOnSuccessOnly(t *testing.T) { + for _, tc := range []struct { + name string + exitCode int + window uint64 + wantEnabled bool + wantWindow uint64 + }{ + {name: "success enables proofs history with the init's window", exitCode: 0, window: 30, wantEnabled: true, wantWindow: 30}, + {name: "success keeps the configured window when the init names none", exitCode: 0, window: 0, wantEnabled: true, wantWindow: 25}, + {name: "failure leaves proofs history off and the window untouched", exitCode: 1, window: 30, wantEnabled: false, wantWindow: 25}, + } { + t.Run(tc.name, func(t *testing.T) { + n := newTestOpReth(t, fakeOpReth(t, tc.exitCode)) + + report, err := n.InitProofsHistory(stack.ProofsHistoryInit{PlainBackfill: true, Window: tc.window}) + + if tc.exitCode == 0 { + require.NoError(t, err) + } else { + require.ErrorContains(t, err, "op-reth proofs init failed") + require.ErrorContains(t, err, "proofs init --chain=/genesis.json", "the error must carry the subcommand's output") + } + require.Contains(t, report.Output, "--proofs-history.use-snapshot=false", "the fake binary echoes the arguments it got") + require.Positive(t, report.Duration) + require.Equal(t, tc.wantEnabled, n.proofsHistoryEnabled) + require.Equal(t, tc.wantWindow, n.proofsHistoryWindow) + }) + } +} + +// fakeOpReth writes a script that echoes its arguments and exits with exitCode, standing in for +// the op-reth binary. +func fakeOpReth(t *testing.T, exitCode int) string { + path := filepath.Join(t.TempDir(), "op-reth") + script := fmt.Sprintf("#!/bin/sh\necho \"$@\"\nexit %d\n", exitCode) + require.NoError(t, os.WriteFile(path, []byte(script), 0o755)) + return path +} + +func newTestOpReth(t *testing.T, execPath string) *OpReth { + return &OpReth{ + chainConfigPath: "/genesis.json", + dataDirPath: "/data", + execPath: execPath, + name: "el", + p: devtest.SerialT(t), + proofsHistoryStoragePath: "/proofs", + proofsHistoryWindow: 25, + } +} diff --git a/op-devstack/sysgo/mixed_runtime.go b/op-devstack/sysgo/mixed_runtime.go index b25b948954c..765b8533610 100644 --- a/op-devstack/sysgo/mixed_runtime.go +++ b/op-devstack/sysgo/mixed_runtime.go @@ -8,7 +8,6 @@ import ( "os" "os/exec" "path/filepath" - "strconv" "strings" "github.com/ethereum/go-ethereum/common/hexutil" @@ -19,6 +18,7 @@ import ( coredepset "github.com/ethereum-optimism/optimism/op-core/interop/depset" "github.com/ethereum-optimism/optimism/op-devstack/devtest" "github.com/ethereum-optimism/optimism/op-devstack/shared/rustbin" + "github.com/ethereum-optimism/optimism/op-devstack/stack" "github.com/ethereum-optimism/optimism/op-service/endpoint" "github.com/ethereum-optimism/optimism/op-service/eth" "github.com/ethereum-optimism/optimism/op-service/log" @@ -394,25 +394,9 @@ func mixedNodeRefs(nodes []mixedSingleChainNode) []MixedSingleChainNodeRefs { return out } -// defaultMixedProofsHistoryWindow is the proofs-history window an op-reth node gets unless -// OpRethWithProofsHistoryWindow overrides it. -const defaultMixedProofsHistoryWindow uint64 = 10_000 - -// proofsHistoryNodeArgs returns the node flags that enable proofs history on the store at -// storagePath. -func proofsHistoryNodeArgs(cfg *OpRethConfig, storagePath string) []string { - window := cfg.ProofsHistoryWindow - if window == 0 { - window = defaultMixedProofsHistoryWindow - } - return []string{ - "--proofs-history", - "--proofs-history.storage-path=" + storagePath, - "--proofs-history.window=" + strconv.FormatUint(window, 10), - } -} - -// buildMixedOpRethNode constructs an OpReth node without starting it. +// buildMixedOpRethNode constructs an OpReth node without starting it. Unless the options disable +// proofs history, its store is seeded from the genesis datadir so the node runs with proofs +// history from its first start. func buildMixedOpRethNode( t devtest.T, l2Net *L2Network, @@ -497,25 +481,9 @@ func buildMixedOpRethNode( initOut, err := exec.Command(execPath, initArgs...).CombinedOutput() t.Require().NoError(err, "must init op-reth node: %s", string(initOut)) - if !opRethCfg.DisableProofsHistory { - proofHistoryDir := filepath.Join(tempDir, "proof-history") - - initProofsArgs := []string{ - "proofs", - "init", - "--datadir=" + dataDirPath, - "--chain=" + chainConfigPath, - "--proofs-history.storage-path=" + proofHistoryDir, - } - initOut, initErr := exec.Command(execPath, initProofsArgs...).CombinedOutput() - t.Require().NoError(initErr, "must init op-reth proof history: %s", string(initOut)) - - args = append(args, proofsHistoryNodeArgs(opRethCfg, proofHistoryDir)...) - } - args = append(args, opRethCfg.ExtraArgs...) - return &OpReth{ + node := &OpReth{ name: key, chainID: l2Net.ChainID(), jwtPath: jwtPath, @@ -527,7 +495,17 @@ func buildMixedOpRethNode( env: []string{}, p: t, l2MetricsRegistrar: metricsRegistrar, + + chainConfigPath: chainConfigPath, + dataDirPath: dataDirPath, + proofsHistoryStoragePath: filepath.Join(tempDir, "proof-history"), + proofsHistoryWindow: opRethCfg.ProofsHistoryWindow, } + if !opRethCfg.DisableProofsHistory { + _, err = node.InitProofsHistory(stack.ProofsHistoryInit{}) + t.Require().NoError(err, "must init op-reth proof history") + } + return node } func startMixedOpRethNode(