From 2080ccc82d369deb915682d714f93c3c2703b6d1 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Wed, 14 Oct 2026 11:33:13 -0700 Subject: [PATCH 1/3] op-devstack,op-acceptance-tests: cover the proofs init path with backfill Co-Authored-By: Claude Fable 5.1 (cherry picked from commit 8e22eadfbf84299b915cc62954f31f1617ebef08) (cherry picked from commit dde6412af5e5d2ec17988d5cf4d7291421ccd1fb) --- .../proofs_history_init_test.go | 411 ++++++++++++++++++ op-devstack/dsl/l2_el_proofs.go | 68 +++ op-devstack/presets/rpc_frontends.go | 13 +- op-devstack/presets/sysgo_runtime.go | 3 + op-devstack/stack/proofs_history.go | 33 ++ op-devstack/sysgo/l2_el_opreth.go | 118 ++++- op-devstack/sysgo/l2_el_opreth_test.go | 96 +++- op-devstack/sysgo/mixed_runtime.go | 72 ++- 8 files changed, 752 insertions(+), 62 deletions(-) create mode 100644 op-acceptance-tests/tests/proofs_history_init/proofs_history_init_test.go create mode 100644 op-devstack/stack/proofs_history.go diff --git a/op-acceptance-tests/tests/proofs_history_init/proofs_history_init_test.go b/op-acceptance-tests/tests/proofs_history_init/proofs_history_init_test.go new file mode 100644 index 00000000000..a913e0f92a1 --- /dev/null +++ b/op-acceptance-tests/tests/proofs_history_init/proofs_history_init_test.go @@ -0,0 +1,411 @@ +package proofs_history_init + +import ( + "fmt" + "testing" + "time" + + "github.com/ethereum-optimism/optimism/op-devstack/devtest" + "github.com/ethereum-optimism/optimism/op-devstack/dsl" + "github.com/ethereum-optimism/optimism/op-devstack/presets" + "github.com/ethereum-optimism/optimism/op-devstack/stack" + "github.com/ethereum-optimism/optimism/op-devstack/sysgo" + "github.com/ethereum-optimism/optimism/op-service/bigs" + "github.com/ethereum-optimism/optimism/op-service/eth" + safety "github.com/ethereum-optimism/optimism/op-service/eth/safety" + "github.com/ethereum-optimism/optimism/op-service/txplan" + "github.com/ethereum/go-ethereum/common" +) + +const ( + // l2BlockTimeSeconds keeps the runs short: the chain has to grow past a window before the + // init and past another one for the prune, so halving the block time halves the run. + l2BlockTimeSeconds = 1 + // memoryBlockBufferTarget is reth's --engine.memory-block-buffer-target. It is passed + // explicitly, like the persistence threshold, so how far reth's datadir trails its head does + // not depend on reth's defaults. + memoryBlockBufferTarget = 5 + // proofsHistoryWindow is the window the init backfills behind its anchor and the node keeps + // afterwards: small enough that pruning moves the window within the test, large enough that + // the backfill spans blocks reth has persisted and blocks it still held in memory. + proofsHistoryWindow = 30 + // windowMargin is how far past a full window the chain is driven before the init, so the + // backfill target lies above the blocks that funded the workload. + windowMargin = 5 + // postRestartTrafficBlocks is how far past the head the chain is driven once the follower is + // back, so the sweeps also cover blocks the store indexed live after the init. + postRestartTrafficBlocks = 10 + // referenceEthProofWindow lets the sequencer answer eth_getProof for every block of the run + // from reth's own historical state; reth's default window of zero rejects them all. + referenceEthProofWindow = 1_000_000 + + // pruneTimeout bounds how long the store may take to prune the backfilled blocks once the + // chain is a window past the anchor. + pruneTimeout = 90 * time.Second + // resyncAttempts bounds how long the restarted follower may take to catch up with the + // sequencer. + resyncAttempts = 60 + // storeCatchUpTimeout bounds how long the restarted node may take to serve its seeded window. + storeCatchUpTimeout = 60 * time.Second +) + +// followerSpec says how the follower's op-reth runs. Its proofs history is off until a test seeds +// the store with proofs init. +type followerSpec struct { + persistenceThreshold uint64 + stateMaskingBlocks uint64 + verificationInterval uint64 +} + +func (s followerSpec) proofsPath() string { + if s.verificationInterval != 0 { + return "execute" + } + return "index" +} + +// engineArgs pin reth's persistence schedule, which decides how far the datadir trails the head +// at the init. Both nodes get them so the sequencer persists on the same schedule as the follower. +func (s followerSpec) engineArgs() []string { + args := []string{ + fmt.Sprintf("--engine.memory-block-buffer-target=%d", memoryBlockBufferTarget), + fmt.Sprintf("--engine.persistence-threshold=%d", s.persistenceThreshold), + } + if s.stateMaskingBlocks != 0 { + args = append(args, fmt.Sprintf("--engine.num-state-masking-blocks=%d", s.stateMaskingBlocks)) + } + return args +} + +func (s followerSpec) followerArgs() []string { + return append(s.engineArgs(), fmt.Sprintf("--proofs-history.verification-interval=%d", s.verificationInterval)) +} + +// initCase is one cell of the matrix the operator path runs under. +type initCase struct { + spec followerSpec + useSnapshot bool +} + +func (c initCase) name() string { + backfill := "snapshot" + if !c.useSnapshot { + backfill = "plain" + } + return fmt.Sprintf("threshold_%d-%s-backfill_%s", c.spec.persistenceThreshold, c.spec.proofsPath(), backfill) +} + +func initMatrix() []initCase { + var cases []initCase + for _, persistenceThreshold := range []uint64{7, 50} { + for _, verificationInterval := range []uint64{0, 1} { + for _, useSnapshot := range []bool{true, false} { + cases = append(cases, initCase{ + spec: followerSpec{persistenceThreshold: persistenceThreshold, verificationInterval: verificationInterval}, + useSnapshot: useSnapshot, + }) + } + } + } + return cases +} + +// guardCase is one cell of the matrix the init guards run under. The guards are about reth's +// datadir, so the cells vary what shapes it: the persistence schedule and state masking. +type guardCase struct { + spec followerSpec +} + +func (c guardCase) name() string { + return fmt.Sprintf("threshold_%d-masking_%d", c.spec.persistenceThreshold, c.spec.stateMaskingBlocks) +} + +func guardMatrix() []guardCase { + var cases []guardCase + for _, persistenceThreshold := range []uint64{7, 50} { + for _, stateMaskingBlocks := range []uint64{0, 30} { + cases = append(cases, guardCase{spec: followerSpec{persistenceThreshold: persistenceThreshold, stateMaskingBlocks: stateMaskingBlocks}}) + } + } + return cases +} + +// skipMaskingCell skips a cell that needs state masking, which develop cannot run. +func skipMaskingCell(t devtest.T, spec followerSpec) { + if spec.stateMaskingBlocks != 0 { + t.Skip("state masking is compiled out of reth 2.5.2 on develop and #23046 rejects a nonzero --engine.num-state-masking-blocks; re-enable with ethereum-optimism/protocol-team#457, where the init against a killed node must fail with InitializeSourceStateTrieLagging until a clean restart repairs the datadir") + } +} + +type initSystem struct { + Follower *dsl.L2ELNode + FollowerCL *dsl.L2CLNode + FunderL2 *dsl.FunderEOA + Sequencer *dsl.L2ELNode + SequencerCL *dsl.L2CLNode +} + +// newInitSystem starts a sequencer without proofs history, which serves as the independent +// eth_getProof reference, and a follower without proofs history under the spec's reth and ExEx +// settings, configured with the window its store gets once a test seeds it. +func newInitSystem(t devtest.T, spec followerSpec) *initSystem { + sysgo.SkipOnOpGeth(t, "proofs history is an op-reth feature") + clKind := sysgo.ResolveMixedL2CLKind() + runtime := sysgo.NewMixedSingleChainRuntime(t, sysgo.MixedSingleChainPresetConfig{ + DeployerOptions: []sysgo.DeployerOption{sysgo.WithUniformL2BlockTimes(l2BlockTimeSeconds)}, + NodeSpecs: []sysgo.MixedSingleChainNodeSpec{ + { + CLKey: "sequencer", + CLKind: clKind, + ELKey: "sequencer", + ELKind: sysgo.MixedL2ELOpReth, + IsSequencer: true, + OpRethOpts: []sysgo.OpRethOption{ + sysgo.OpRethWithoutProofsHistory(), + sysgo.OpRethWithEthProofWindow(referenceEthProofWindow), + sysgo.OpRethWithExtraArgs(spec.engineArgs()...), + }, + }, + { + CLKey: "follower", + CLKind: clKind, + ELKey: "follower", + ELKind: sysgo.MixedL2ELOpReth, + OpRethOpts: []sysgo.OpRethOption{ + sysgo.OpRethWithoutProofsHistory(), + sysgo.OpRethWithProofsHistoryWindow(proofsHistoryWindow), + sysgo.OpRethWithExtraArgs(spec.followerArgs()...), + }, + }, + }, + }) + frontends := presets.NewMixedSingleChainFrontends(t, runtime) + t.Require().Len(frontends.Nodes, 2, "expected the sequencer and the follower") + + sys := &initSystem{FunderL2: frontends.FunderL2} + for _, node := range frontends.Nodes { + if node.Spec.IsSequencer { + sys.Sequencer, sys.SequencerCL = node.EL, node.CL + } else { + sys.Follower, sys.FollowerCL = node.EL, node.CL + } + } + t.Require().NotNil(sys.Follower, "missing the follower") + return sys +} + +// workload changes state in nearly every block: alice and bob transfer back and forth and bob +// writes to a contract, so every block the backfill reconstructs carries balance and storage +// changes. +type workload struct { + alice *dsl.EOA + bob *dsl.EOA + recorder common.Address + t devtest.T +} + +func newWorkload(t devtest.T, funder *dsl.FunderEOA) *workload { + w := &workload{ + alice: funder.NewFundedEOA(eth.OneEther), + bob: funder.NewFundedEOA(eth.OneEther), + t: t, + } + w.recorder = w.alice.DeployBlockRecorder() + return w +} + +// accounts are the accounts the proofs sweeps prove at every block. +func (w *workload) accounts() []dsl.ProofAccount { + return []dsl.ProofAccount{ + {Address: w.alice.Address()}, + {Address: w.bob.Address()}, + {Address: w.recorder, StorageKeys: []common.Hash{common.HexToHash("0x0"), common.HexToHash("0x1")}}, + } +} + +// driveUntilBlock sends transactions one after another until one lands in a block at or past +// target, and returns that block. +func (w *workload) driveUntilBlock(target uint64) uint64 { + for { + w.alice.Transfer(w.bob.Address(), eth.OneHundredthEther) + w.bob.Transfer(w.alice.Address(), eth.OneHundredthEther) + block := includedBlock(w.t, w.bob.Transact(w.bob.Plan(), txplan.WithTo(&w.recorder))) + if block >= target { + return block + } + } +} + +func includedBlock(t devtest.T, tx *txplan.PlannedTx) uint64 { + receipt, err := tx.Included.Eval(t.Ctx()) + t.Require().NoError(err, "transaction must be included") + return bigs.Uint64Strict(receipt.BlockNumber) +} + +// driveChainPastWindow runs the workload until the chain is more than a window long and the +// follower has imported all of it, and returns the follower's head. +func driveChainPastWindow(sys *initSystem, w *workload) eth.L2BlockRef { + w.driveUntilBlock(proofsHistoryWindow + windowMargin) + sys.Follower.InSync(sys.Sequencer, safety.LocalUnsafe, resyncAttempts) + return sys.Follower.BlockRefByLabel(eth.Unsafe) +} + +// restartFollowerEL brings the follower's op-reth back on its datadir, now with proofs history, +// and returns the datadir tip it resumed from. Nothing feeds the node until its CL is back, so +// the tip is what reth had persisted, and the store serves exactly what the init seeded. +func restartFollowerEL(t devtest.T, sys *initSystem) eth.L2BlockRef { + sys.Follower.Start() + sys.Follower.WaitForOnline() + tip := sys.Follower.BlockRefByLabel(eth.Unsafe) + t.Logger().Info("Follower op-reth is back on its datadir with proofs history", "datadir_tip", tip) + return tip +} + +// reconnectFollower re-peers the follower's op-reth with the sequencer, brings its CL back and +// waits until the follower has caught up with the sequencer. +func reconnectFollower(sys *initSystem) { + sys.Follower.PeerWith(sys.Sequencer) + sys.FollowerCL.Start() + sys.FollowerCL.ConnectPeer(sys.SequencerCL) + sys.Follower.InSync(sys.Sequencer, safety.LocalUnsafe, resyncAttempts) +} + +// verifySeededWindow requires the store seeded at anchor to span exactly one window behind it and +// sweeps that window against the sequencer. It runs before the follower's CL is back, so nothing +// extends or prunes the store meanwhile. +func verifySeededWindow(t devtest.T, sys *initSystem, w *workload, anchor eth.L2BlockRef) { + window := sys.Follower.WaitForProofsStoreBlock(anchor.Number, storeCatchUpTimeout) + t.Require().Equalf(anchor.Number, *window.Latest, "the store must be anchored at the datadir tip %s; store window %s", anchor, window) + t.Require().Equalf(anchor.Number-proofsHistoryWindow, *window.Earliest, + "the backfill must extend the store one window (%d blocks) behind the anchor %s; store window %s", proofsHistoryWindow, anchor, window) + sys.Follower.VerifyHistoricalProofsMatch(sys.Sequencer, func(opts *dsl.HistoricalProofsOpts) { + opts.Accounts = w.accounts() + opts.FirstBlock = *window.Earliest + opts.LastBlock = anchor.Number + }) +} + +// verifyLiveWindow drives traffic past the current head and sweeps the window the store serves +// then, which now holds blocks the store indexed live. +func verifyLiveWindow(sys *initSystem, w *workload) { + lastBlock := w.driveUntilBlock(sys.Sequencer.BlockRefByLabel(eth.Unsafe).Number + postRestartTrafficBlocks) + sys.Follower.VerifyHistoricalProofsMatch(sys.Sequencer, func(opts *dsl.HistoricalProofsOpts) { + opts.Accounts = w.accounts() + opts.LastBlock = lastBlock + }) +} + +// TestFollowerServesTheWholeWindowAfterProofsInitWithBackfill runs the operator path: a follower +// that synced without proofs history is stopped cleanly, `op-reth proofs init` seeds its store +// at the datadir tip and backfills a window behind it, and the restarted follower must serve +// proofs for the whole window that verify against the headers and match the sequencer's, then +// keep doing so once pruning has moved the window past the init's anchor. +func TestFollowerServesTheWholeWindowAfterProofsInitWithBackfill(gt *testing.T) { + t := devtest.ParallelT(gt) + for _, c := range initMatrix() { + t.Run(c.name(), func(t devtest.T) { + t.Parallel() + sys := newInitSystem(t, c.spec) + w := newWorkload(t, sys.FunderL2) + head := driveChainPastWindow(sys, w) + + sys.FollowerCL.Stop() + sys.Follower.Stop() + report := sys.Follower.InitProofsHistory(func(init *stack.ProofsHistoryInit) { + init.UseSnapshot = c.useSnapshot + }) + t.Logger().Info("Proofs init seeded the follower's store", "cell", c.name(), "init_duration", report.Duration, + "head_at_stop", head, "window", proofsHistoryWindow) + + anchor := restartFollowerEL(t, sys) + verifySeededWindow(t, sys, w, anchor) + + reconnectFollower(sys) + lastBlock := w.driveUntilBlock(anchor.Number + proofsHistoryWindow + windowMargin) + window := sys.Follower.WaitForProofsStorePrunedPast(anchor.Number, pruneTimeout) + t.Logger().Info("Pruning moved the window past the init's anchor", "anchor", anchor, "window", window) + sys.Follower.VerifyHistoricalProofsMatch(sys.Sequencer, func(opts *dsl.HistoricalProofsOpts) { + opts.Accounts = w.accounts() + opts.LastBlock = lastBlock + }) + }) + } +} + +// TestProofsInitSeedsAConsistentStoreFromAKilledNodeDatadir kills a follower that synced without +// proofs history, so reth's datadir trails the head by the blocks it still held in memory, and +// runs `op-reth proofs init` against that datadir. The init must seed a store anchored at the +// persisted tip with a full window behind it, and once the follower is back it must serve that +// window and the blocks it re-imports with proofs that match the sequencer's. +func TestProofsInitSeedsAConsistentStoreFromAKilledNodeDatadir(gt *testing.T) { + t := devtest.ParallelT(gt) + for _, c := range guardMatrix() { + t.Run(c.name(), func(t devtest.T) { + t.Parallel() + skipMaskingCell(t, c.spec) + sys := newInitSystem(t, c.spec) + w := newWorkload(t, sys.FunderL2) + head := driveChainPastWindow(sys, w) + + sys.Follower.StopUngracefully() + sys.FollowerCL.Stop() + report := sys.Follower.InitProofsHistory() + t.Logger().Info("Proofs init seeded the killed follower's store", "cell", c.name(), "init_duration", report.Duration, "head_at_kill", head) + + persistedTip := restartFollowerEL(t, sys) + t.Require().Lessf(persistedTip.Number, head.Number, + "the kill must leave reth's datadir (tip %d) behind the head %d, or the init ran against a datadir no different from a clean stop", + persistedTip.Number, head.Number) + verifySeededWindow(t, sys, w, persistedTip) + + reconnectFollower(sys) + verifyLiveWindow(sys, w) + }) + } +} + +// TestProofsInitAgainstARunningNodeNeverSeedsAStoreThatMatchesNoBlock runs `op-reth proofs init` +// against the datadir of a follower that is still running and persisting, which an operator can +// do by mistake. Whatever the init makes of that datadir, the store the follower ends up running +// on must describe the chain: after a clean stop, a seeding of a store the init refused, and a +// restart with proofs history, every proof of the window must verify against its header and match +// the sequencer's. +func TestProofsInitAgainstARunningNodeNeverSeedsAStoreThatMatchesNoBlock(gt *testing.T) { + t := devtest.ParallelT(gt) + for _, c := range guardMatrix() { + t.Run(c.name(), func(t devtest.T) { + t.Parallel() + skipMaskingCell(t, c.spec) + sys := newInitSystem(t, c.spec) + w := newWorkload(t, sys.FunderL2) + head := driveChainPastWindow(sys, w) + + report, err := sys.Follower.TryInitProofsHistory() + if err != nil { + // #23046 refuses the init when reth persisted past the anchor between the init's + // two database reads. develop has no such guard, so any failure is unexpected. + t.Require().ErrorContainsf(err, "reth database tip moved", + "proofs init against a running node may only fail because reth's tip moved (#23046); it failed with: %v", err) + } + t.Logger().Info("Proofs init ran against the running follower", "cell", c.name(), "init_duration", report.Duration, + "head_at_init", head, "refused", err != nil) + + sys.FollowerCL.Stop() + sys.Follower.Stop() + if err != nil { + sys.Follower.InitProofsHistory() + } + + tip := restartFollowerEL(t, sys) + window := sys.Follower.WaitForProofsStoreBlock(tip.Number, storeCatchUpTimeout) + t.Logger().Info("Follower store caught up with its datadir", "datadir_tip", tip, "window", window) + sys.Follower.VerifyHistoricalProofsMatch(sys.Sequencer, func(opts *dsl.HistoricalProofsOpts) { + opts.Accounts = w.accounts() + opts.LastBlock = tip.Number + }) + + reconnectFollower(sys) + verifyLiveWindow(sys, w) + }) + } +} diff --git a/op-devstack/dsl/l2_el_proofs.go b/op-devstack/dsl/l2_el_proofs.go index 047b9949f21..920bcceec25 100644 --- a/op-devstack/dsl/l2_el_proofs.go +++ b/op-devstack/dsl/l2_el_proofs.go @@ -8,6 +8,7 @@ import ( "time" "github.com/ethereum-optimism/optimism/op-core/predeploys" + "github.com/ethereum-optimism/optimism/op-devstack/stack" "github.com/ethereum-optimism/optimism/op-e2e/e2eutils/wait" "github.com/ethereum-optimism/optimism/op-service/eth" "github.com/ethereum/go-ethereum/common" @@ -107,6 +108,73 @@ func (el *L2ELNode) WaitForProofsStoreBlock(target uint64, timeout time.Duration return lastStatus } +// WaitForProofsStorePrunedPast waits until this node's proofs-history store no longer serves +// block, because pruning moved the store's earliest block past it. The store prunes to its +// window behind the latest block it indexed, so the chain has to be more than a window past +// block first. +func (el *L2ELNode) WaitForProofsStorePrunedPast(block uint64, timeout time.Duration) ProofsSyncStatus { + ctx, cancel := context.WithTimeout(el.ctx, timeout) + defer cancel() + logger := el.log.With("name", el.inner.Name(), "chain", el.ChainID(), "block", block, "timeout", timeout) + logger.Info("Waiting for proofs store to prune past block") + + var lastStatus ProofsSyncStatus + var lastErr error + err := wait.For(ctx, 200*time.Millisecond, func() (bool, error) { + status, err := el.ProofsSyncStatus() + if err != nil { + lastErr = err + logger.Warn("debug_proofsSyncStatus failed; will retry", "err", err) + return false, nil + } + lastStatus, lastErr = status, nil + if status.Earliest != nil && *status.Earliest > block { + logger.Info("Proofs store pruned past block", "window", status) + return true, nil + } + logger.Info("Proofs store still serves block", "window", status) + return false, nil + }) + el.require.NoErrorf(err, "proofs store on %s did not prune past block %d within %s; last window %s; last error: %v", + el.inner.Name(), block, timeout, lastStatus, lastErr) + return lastStatus +} + +// InitProofsHistory seeds this node's proofs-history store from its datadir with +// `op-reth proofs init`, the way an operator does for a node that synced without proofs history, +// and makes the node run with proofs history from its next Start. The node must be stopped: a +// test of what init does to a datadir the node still writes to uses TryInitProofsHistory. +func (el *L2ELNode) InitProofsHistory(opts ...func(*stack.ProofsHistoryInit)) stack.ProofsHistoryInitReport { + if control, ok := el.inner.(stack.ControlledLifecycle); ok { + el.require.Falsef(control.Running(), "stop %s before seeding its proofs-history store", el.inner.Name()) + } + report, err := el.TryInitProofsHistory(opts...) + el.require.NoErrorf(err, "proofs init on %s must seed the store", el.inner.Name()) + return report +} + +// TryInitProofsHistory is InitProofsHistory for a test that asserts on the init's outcome itself: +// it runs whether or not the node is running and returns the failure instead of failing the test. +// The store backfills through a trie-state snapshot and keeps the node's configured window unless +// the options say otherwise. +func (el *L2ELNode) TryInitProofsHistory(opts ...func(*stack.ProofsHistoryInit)) (stack.ProofsHistoryInitReport, error) { + initializer, ok := el.inner.(stack.ProofsHistoryInitializer) + el.require.Truef(ok, "L2EL node %s cannot seed a proofs-history store", el.inner.Name()) + init := stack.ProofsHistoryInit{UseSnapshot: true} + for _, opt := range opts { + opt(&init) + } + logger := el.log.With("name", el.inner.Name(), "chain", el.ChainID(), "window", init.Window, "use_snapshot", init.UseSnapshot) + logger.Info("Seeding the proofs-history store from the datadir") + report, err := initializer.InitProofsHistory(init) + if err != nil { + logger.Warn("Proofs init did not seed the store", "duration", report.Duration, "err", err) + return report, err + } + logger.Info("Proofs init seeded the store", "duration", report.Duration) + return report, nil +} + // VerifyHistoricalProofsMatch sweeps the blocks this node's proofs-history store serves and, at // every sampled block, requires eth_getProof for each configured account to verify against the // block's state root and to equal the proof served by reference. The L2ToL1MessagePasser proof is diff --git a/op-devstack/presets/rpc_frontends.go b/op-devstack/presets/rpc_frontends.go index 9631aff65eb..a192020970a 100644 --- a/op-devstack/presets/rpc_frontends.go +++ b/op-devstack/presets/rpc_frontends.go @@ -158,12 +158,14 @@ type l2ELFrontend struct { control stack.ControlledLifecycle ungraceful stack.UngracefulLifecycle logRecorder stack.LogRecorder + proofsInit stack.ProofsHistoryInitializer } var ( - _ stack.L2ELNode = (*l2ELFrontend)(nil) - _ stack.LogRecorder = (*l2ELFrontend)(nil) - _ stack.UngracefulLifecycle = (*l2ELFrontend)(nil) + _ stack.L2ELNode = (*l2ELFrontend)(nil) + _ stack.LogRecorder = (*l2ELFrontend)(nil) + _ stack.ProofsHistoryInitializer = (*l2ELFrontend)(nil) + _ stack.UngracefulLifecycle = (*l2ELFrontend)(nil) ) func newPresetL2ELNode(t devtest.T, name string, chainID eth.ChainID, userRPCCl opclient.RPC, userRPC string, engineRPCCl opclient.RPC, rollupCfg *rollup.Config) *l2ELFrontend { @@ -210,6 +212,11 @@ func (r *l2ELFrontend) RecordLogLines(substring string) *logpipe.LogRecord { return r.logRecorder.RecordLogLines(substring) } +func (r *l2ELFrontend) InitProofsHistory(init stack.ProofsHistoryInit) (stack.ProofsHistoryInitReport, error) { + r.require().NotNil(r.proofsInit, "L2EL node %s cannot seed a proofs-history store", r.Name()) + return r.proofsInit.InitProofsHistory(init) +} + func (r *l2ELFrontend) StartControlled(ctx context.Context) error { if r.control == nil { return fmt.Errorf("L2EL node %s is not controllable", r.Name()) diff --git a/op-devstack/presets/sysgo_runtime.go b/op-devstack/presets/sysgo_runtime.go index 18b7dcac546..36d968e0879 100644 --- a/op-devstack/presets/sysgo_runtime.go +++ b/op-devstack/presets/sysgo_runtime.go @@ -72,6 +72,9 @@ func newL2ELFrontend(t devtest.T, name string, chainID eth.ChainID, userRPC stri if recorder, ok := lifecycle[0].(stack.LogRecorder); ok { l2EL.logRecorder = recorder } + if initializer, ok := lifecycle[0].(stack.ProofsHistoryInitializer); ok { + l2EL.proofsInit = initializer + } } return l2EL } diff --git a/op-devstack/stack/proofs_history.go b/op-devstack/stack/proofs_history.go new file mode 100644 index 00000000000..97152c39dd1 --- /dev/null +++ b/op-devstack/stack/proofs_history.go @@ -0,0 +1,33 @@ +package stack + +import "time" + +// ProofsHistoryInit is one run of `op-reth proofs init` against a node's datadir: it anchors a +// proofs-history store at the datadir's tip, backfills the store Window blocks behind the anchor, +// and makes the node run with proofs history from its next Start on. +type ProofsHistoryInit struct { + // UseSnapshot backfills through a trie-state snapshot, as op-reth does by default; false reads + // every backfilled block's changes from reth's database instead. + UseSnapshot bool + // Window is the number of blocks the backfill extends behind the anchor and the window the + // node keeps once proofs history is on. Zero keeps the window the node was configured with. + Window uint64 +} + +// ProofsHistoryInitReport is what a proofs-history init run produced. +type ProofsHistoryInitReport struct { + // Duration is how long the subcommand ran, backfill included. + Duration time.Duration + // Output is the subcommand's combined stdout and stderr. + Output string +} + +// ProofsHistoryInitializer is implemented by execution-layer nodes whose proofs-history store can +// be seeded from their datadir after they were built, the way an operator seeds one for a node +// that has synced without proofs history. +type ProofsHistoryInitializer interface { + // InitProofsHistory runs the init against the node's datadir, whether or not the node is + // running, and enables proofs history on the node's next Start when it succeeds. The error of + // a failed run carries the subcommand's output. + InitProofsHistory(init ProofsHistoryInit) (ProofsHistoryInitReport, error) +} diff --git a/op-devstack/sysgo/l2_el_opreth.go b/op-devstack/sysgo/l2_el_opreth.go index 9783a5d81bb..0de2e23826d 100644 --- a/op-devstack/sysgo/l2_el_opreth.go +++ b/op-devstack/sysgo/l2_el_opreth.go @@ -4,8 +4,11 @@ import ( "context" "fmt" "net/url" + "os/exec" + "strconv" "strings" "sync" + "time" "github.com/ethereum-optimism/optimism/op-devstack/devtest" "github.com/ethereum-optimism/optimism/op-devstack/stack" @@ -24,9 +27,10 @@ type OpRethConfig struct { // (any binary that accepts every op-reth subcommand/flag, plus optionally its own additive // flags) may be selected via OpRethWithBinary. Binary string - // DisableProofsHistory skips the proofs-history init + runtime flags for this node. The mixed - // runtime otherwise enables proofs-history on every op-reth node; some CLI-superset binaries - // reject --proofs-history in the mode under test. + // DisableProofsHistory builds the node without a seeded proofs-history store and without the + // proofs-history runtime flags. The mixed runtime otherwise enables proofs history on every + // op-reth node; some CLI-superset binaries reject --proofs-history in the mode under test, and + // a test of the operator path seeds the store later with InitProofsHistory. DisableProofsHistory bool // ProofsHistoryWindow is the number of blocks the proofs-history store keeps. Zero keeps the // mixed runtime's default window. @@ -84,9 +88,10 @@ func OpRethWithBinary(binary string) OpRethOption { }) } -// OpRethWithoutProofsHistory disables the proofs-history subsystem for this node. The mixed runtime -// enables proofs-history on every op-reth node by default; use this for a CLI-superset binary that -// rejects --proofs-history in the mode under test. +// OpRethWithoutProofsHistory builds the node without proofs history. The mixed runtime enables it +// on every op-reth node by default; use this for a CLI-superset binary that rejects +// --proofs-history in the mode under test, or for a node whose store a test seeds later with +// InitProofsHistory. func OpRethWithoutProofsHistory() OpRethOption { return OpRethOptionFn(func(p devtest.T, _ ComponentTarget, cfg *OpRethConfig) { cfg.DisableProofsHistory = true @@ -134,10 +139,23 @@ type OpReth struct { userProxy *tcpproxy.Proxy execPath string - args []string + // args are the `node` arguments without the proofs-history flags, which Start appends while + // proofsHistoryEnabled is set. + args []string // Each entry is of the form "key=value". env []string + chainConfigPath string + dataDirPath string + + // proofsHistoryEnabled says the store at proofsHistoryStoragePath is seeded, so the node runs + // with proofs history from its next start. + proofsHistoryEnabled bool + proofsHistoryStoragePath string + // proofsHistoryWindow is the window proofs init backfills and the node keeps; zero means the + // mixed runtime's default. + proofsHistoryWindow uint64 + p devtest.T sub *SubProcess @@ -150,9 +168,10 @@ type OpReth struct { } var ( - _ L2ELNode = (*OpReth)(nil) - _ stack.LogRecorder = (*OpReth)(nil) - _ stack.UngracefulLifecycle = (*OpReth)(nil) + _ L2ELNode = (*OpReth)(nil) + _ stack.LogRecorder = (*OpReth)(nil) + _ stack.ProofsHistoryInitializer = (*OpReth)(nil) + _ stack.UngracefulLifecycle = (*OpReth)(nil) ) // RecordLogLines returns a record of every line op-reth writes to stdout or stderr from now on, @@ -245,7 +264,7 @@ func (n *OpReth) Start() { }) n.sub = NewSubProcess(n.p, stdOutLogs, stdErrLogs) - err := n.sub.Start(n.execPath, n.args, n.env) + err := n.sub.Start(n.execPath, n.nodeArgs(), n.env) n.p.Require().NoError(err, "Must start") var userRPCAddr, authRPCAddr string @@ -292,6 +311,83 @@ func (n *OpReth) StopUngracefully() { n.sub = nil } +// nodeArgs are the `node` arguments of the next start. Callers must hold n.mu. +func (n *OpReth) nodeArgs() []string { + if !n.proofsHistoryEnabled { + return n.args + } + args := append([]string{}, n.args...) + return append(args, proofsHistoryNodeArgs(n.proofsHistoryWindow, n.proofsHistoryStoragePath)...) +} + +// InitProofsHistory runs `op-reth proofs init` against this node's datadir and, when it succeeds, +// makes the next Start run with proofs history on the seeded store. It does not stop the node: +// what init does to a datadir the node is still writing to is the caller's concern. +func (n *OpReth) InitProofsHistory(init stack.ProofsHistoryInit) (stack.ProofsHistoryInitReport, error) { + n.mu.Lock() + defer n.mu.Unlock() + window := n.proofsHistoryWindow + if init.Window != 0 { + window = init.Window + } + args := proofsHistoryInitArgs(n.chainConfigPath, n.dataDirPath, n.proofsHistoryStoragePath, window, init.UseSnapshot) + logger := n.p.Logger().New("name", n.name, "chain", n.chainID, "running", n.sub != nil) + logger.Info("Running op-reth proofs init", "cmd", n.execPath, "args", args) + started := time.Now() + output, err := exec.Command(n.execPath, args...).CombinedOutput() + report := stack.ProofsHistoryInitReport{Duration: time.Since(started), Output: string(output)} + if err != nil { + logger.Warn("op-reth proofs init failed", "duration", report.Duration, "err", err, "output", report.Output) + return report, fmt.Errorf("op-reth proofs init failed after %s: %w\n%s", report.Duration, err, report.Output) + } + n.proofsHistoryEnabled = true + n.proofsHistoryWindow = window + logger.Info("op-reth proofs init succeeded; proofs history is on from the next start", + "duration", report.Duration, "window", resolveProofsHistoryWindow(window), "storage_path", n.proofsHistoryStoragePath) + logger.Debug("op-reth proofs init output", "output", report.Output) + return report, nil +} + +// defaultMixedProofsHistoryWindow is the proofs-history window an op-reth node gets unless +// OpRethWithProofsHistoryWindow or a ProofsHistoryInit overrides it. +const defaultMixedProofsHistoryWindow uint64 = 10_000 + +func resolveProofsHistoryWindow(window uint64) uint64 { + if window == 0 { + return defaultMixedProofsHistoryWindow + } + return window +} + +// proofsHistoryNodeArgs returns the node flags that enable proofs history on the store at +// storagePath. +func proofsHistoryNodeArgs(window uint64, storagePath string) []string { + return []string{ + "--proofs-history", + "--proofs-history.storage-path=" + storagePath, + "--proofs-history.window=" + strconv.FormatUint(resolveProofsHistoryWindow(window), 10), + } +} + +// proofsHistoryInitArgs returns the `proofs init` invocation that seeds the store at storagePath +// from the datadir and backfills it window blocks behind the datadir's tip. op-reth backfills +// through a trie-state snapshot unless told not to, so the snapshot path gets the operator's +// invocation with no flag. +func proofsHistoryInitArgs(chainConfigPath, dataDirPath, storagePath string, window uint64, useSnapshot bool) []string { + args := []string{ + "proofs", + "init", + "--chain=" + chainConfigPath, + "--datadir=" + dataDirPath, + "--proofs-history.storage-path=" + storagePath, + "--proofs-history.window=" + strconv.FormatUint(resolveProofsHistoryWindow(window), 10), + } + if !useSnapshot { + args = append(args, "--proofs-history.use-snapshot=false") + } + return args +} + // Callers must hold n.mu. func (n *OpReth) clearProxyUpstreams() { if n.userProxy != nil { diff --git a/op-devstack/sysgo/l2_el_opreth_test.go b/op-devstack/sysgo/l2_el_opreth_test.go index 10a27dfe175..336348014f6 100644 --- a/op-devstack/sysgo/l2_el_opreth_test.go +++ b/op-devstack/sysgo/l2_el_opreth_test.go @@ -1,10 +1,16 @@ package sysgo import ( + "fmt" + "os" + "path/filepath" "strings" "testing" "github.com/stretchr/testify/require" + + "github.com/ethereum-optimism/optimism/op-devstack/devtest" + "github.com/ethereum-optimism/optimism/op-devstack/stack" ) func argsWithPrefix(args []string, prefix string) []string { @@ -27,7 +33,7 @@ func TestProofsHistoryNodeArgs(t *testing.T) { {name: "explicit window replaces the default", window: 25, wantWindow: "--proofs-history.window=25"}, } { t.Run(tc.name, func(t *testing.T) { - args := proofsHistoryNodeArgs(&OpRethConfig{ProofsHistoryWindow: tc.window}, "/proofs") + args := proofsHistoryNodeArgs(tc.window, "/proofs") require.Contains(t, args, "--proofs-history") require.Contains(t, args, "--proofs-history.storage-path=/proofs") @@ -37,6 +43,31 @@ func TestProofsHistoryNodeArgs(t *testing.T) { } } +func TestProofsHistoryInitArgs(t *testing.T) { + for _, tc := range []struct { + name string + useSnapshot bool + window uint64 + wantSnapshotFlag []string + wantWindow string + }{ + {name: "snapshot backfill is op-reth's default and gets no flag", useSnapshot: true, window: 30, wantSnapshotFlag: nil, wantWindow: "--proofs-history.window=30"}, + {name: "plain backfill turns the snapshot off", useSnapshot: false, window: 30, wantSnapshotFlag: []string{"--proofs-history.use-snapshot=false"}, wantWindow: "--proofs-history.window=30"}, + {name: "zero window uses the runtime default", useSnapshot: true, window: 0, wantSnapshotFlag: nil, wantWindow: "--proofs-history.window=10000"}, + } { + t.Run(tc.name, func(t *testing.T) { + args := proofsHistoryInitArgs("/genesis.json", "/data", "/proofs", tc.window, tc.useSnapshot) + + require.Equal(t, []string{"proofs", "init"}, args[:2]) + require.Contains(t, args, "--chain=/genesis.json") + require.Contains(t, args, "--datadir=/data") + require.Contains(t, args, "--proofs-history.storage-path=/proofs") + require.Equal(t, []string{tc.wantWindow}, argsWithPrefix(args, "--proofs-history.window=")) + require.Equal(t, tc.wantSnapshotFlag, argsWithPrefix(args, "--proofs-history.use-snapshot")) + }) + } +} + func TestOpRethWithProofsHistoryWindow(t *testing.T) { cfg := DefaultOpRethConfig() OpRethWithProofsHistoryWindow(25).Apply(nil, ComponentTarget{}, cfg) @@ -51,3 +82,66 @@ func TestOpRethWithEthProofWindow(t *testing.T) { require.Equal(t, []string{"--rpc.eth-proof-window=1000000"}, cfg.ExtraArgs) } + +func TestOpRethNodeArgsCarryProofsHistoryOnlyOnceEnabled(t *testing.T) { + baseArgs := []string{"node", "--datadir=/data"} + n := &OpReth{args: baseArgs, proofsHistoryStoragePath: "/proofs", proofsHistoryWindow: 25} + + require.Equal(t, baseArgs, n.nodeArgs()) + + n.proofsHistoryEnabled = true + require.Equal(t, append([]string{"node", "--datadir=/data"}, proofsHistoryNodeArgs(25, "/proofs")...), n.nodeArgs()) + require.Equal(t, baseArgs, n.args, "the base arguments must stay free of the proofs-history flags") +} + +func TestInitProofsHistoryEnablesProofsHistoryOnSuccessOnly(t *testing.T) { + for _, tc := range []struct { + name string + exitCode int + window uint64 + wantEnabled bool + wantWindow uint64 + }{ + {name: "success enables proofs history with the init's window", exitCode: 0, window: 30, wantEnabled: true, wantWindow: 30}, + {name: "success keeps the configured window when the init names none", exitCode: 0, window: 0, wantEnabled: true, wantWindow: 25}, + {name: "failure leaves proofs history off and the window untouched", exitCode: 1, window: 30, wantEnabled: false, wantWindow: 25}, + } { + t.Run(tc.name, func(t *testing.T) { + n := newTestOpReth(t, fakeOpReth(t, tc.exitCode)) + + report, err := n.InitProofsHistory(stack.ProofsHistoryInit{UseSnapshot: false, Window: tc.window}) + + if tc.exitCode == 0 { + require.NoError(t, err) + } else { + require.ErrorContains(t, err, "op-reth proofs init failed") + require.ErrorContains(t, err, "proofs init --chain=/genesis.json", "the error must carry the subcommand's output") + } + require.Contains(t, report.Output, "--proofs-history.use-snapshot=false", "the fake binary echoes the arguments it got") + require.Positive(t, report.Duration) + require.Equal(t, tc.wantEnabled, n.proofsHistoryEnabled) + require.Equal(t, tc.wantWindow, n.proofsHistoryWindow) + }) + } +} + +// fakeOpReth writes a script that echoes its arguments and exits with exitCode, standing in for +// the op-reth binary. +func fakeOpReth(t *testing.T, exitCode int) string { + path := filepath.Join(t.TempDir(), "op-reth") + script := fmt.Sprintf("#!/bin/sh\necho \"$@\"\nexit %d\n", exitCode) + require.NoError(t, os.WriteFile(path, []byte(script), 0o755)) + return path +} + +func newTestOpReth(t *testing.T, execPath string) *OpReth { + return &OpReth{ + chainConfigPath: "/genesis.json", + dataDirPath: "/data", + execPath: execPath, + name: "el", + p: devtest.SerialT(t), + proofsHistoryStoragePath: "/proofs", + proofsHistoryWindow: 25, + } +} diff --git a/op-devstack/sysgo/mixed_runtime.go b/op-devstack/sysgo/mixed_runtime.go index b25b948954c..083796e6203 100644 --- a/op-devstack/sysgo/mixed_runtime.go +++ b/op-devstack/sysgo/mixed_runtime.go @@ -8,7 +8,6 @@ import ( "os" "os/exec" "path/filepath" - "strconv" "strings" "github.com/ethereum/go-ethereum/common/hexutil" @@ -19,6 +18,7 @@ import ( coredepset "github.com/ethereum-optimism/optimism/op-core/interop/depset" "github.com/ethereum-optimism/optimism/op-devstack/devtest" "github.com/ethereum-optimism/optimism/op-devstack/shared/rustbin" + "github.com/ethereum-optimism/optimism/op-devstack/stack" "github.com/ethereum-optimism/optimism/op-service/endpoint" "github.com/ethereum-optimism/optimism/op-service/eth" "github.com/ethereum-optimism/optimism/op-service/log" @@ -394,25 +394,10 @@ func mixedNodeRefs(nodes []mixedSingleChainNode) []MixedSingleChainNodeRefs { return out } -// defaultMixedProofsHistoryWindow is the proofs-history window an op-reth node gets unless -// OpRethWithProofsHistoryWindow overrides it. -const defaultMixedProofsHistoryWindow uint64 = 10_000 +// buildMixedOpRethNode constructs an OpReth node without starting it. Unless the options disable +// proofs history, its store is seeded from the genesis datadir so the node runs with proofs +// history from its first start. -// proofsHistoryNodeArgs returns the node flags that enable proofs history on the store at -// storagePath. -func proofsHistoryNodeArgs(cfg *OpRethConfig, storagePath string) []string { - window := cfg.ProofsHistoryWindow - if window == 0 { - window = defaultMixedProofsHistoryWindow - } - return []string{ - "--proofs-history", - "--proofs-history.storage-path=" + storagePath, - "--proofs-history.window=" + strconv.FormatUint(window, 10), - } -} - -// buildMixedOpRethNode constructs an OpReth node without starting it. func buildMixedOpRethNode( t devtest.T, l2Net *L2Network, @@ -497,37 +482,30 @@ func buildMixedOpRethNode( initOut, err := exec.Command(execPath, initArgs...).CombinedOutput() t.Require().NoError(err, "must init op-reth node: %s", string(initOut)) - if !opRethCfg.DisableProofsHistory { - proofHistoryDir := filepath.Join(tempDir, "proof-history") - - initProofsArgs := []string{ - "proofs", - "init", - "--datadir=" + dataDirPath, - "--chain=" + chainConfigPath, - "--proofs-history.storage-path=" + proofHistoryDir, - } - initOut, initErr := exec.Command(execPath, initProofsArgs...).CombinedOutput() - t.Require().NoError(initErr, "must init op-reth proof history: %s", string(initOut)) - - args = append(args, proofsHistoryNodeArgs(opRethCfg, proofHistoryDir)...) - } - args = append(args, opRethCfg.ExtraArgs...) - return &OpReth{ - name: key, - chainID: l2Net.ChainID(), - jwtPath: jwtPath, - jwtSecret: jwtSecret, - authRPC: "", - userRPC: "", - execPath: execPath, - args: args, - env: []string{}, - p: t, - l2MetricsRegistrar: metricsRegistrar, + node := &OpReth{ + name: key, + chainID: l2Net.ChainID(), + jwtPath: jwtPath, + jwtSecret: jwtSecret, + authRPC: "", + userRPC: "", + execPath: execPath, + args: args, + env: []string{}, + p: t, + l2MetricsRegistrar: metricsRegistrar, + chainConfigPath: chainConfigPath, + dataDirPath: dataDirPath, + proofsHistoryStoragePath: filepath.Join(tempDir, "proof-history"), + proofsHistoryWindow: opRethCfg.ProofsHistoryWindow, } + if !opRethCfg.DisableProofsHistory { + _, err := node.InitProofsHistory(stack.ProofsHistoryInit{UseSnapshot: true}) + t.Require().NoError(err, "must init op-reth proof history") + } + return node } func startMixedOpRethNode( From cfc1c02d52596afdb694a88474cacbc41f17471a Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Wed, 14 Oct 2026 11:46:57 -0700 Subject: [PATCH 2/3] op-acceptance-tests: drive the proofs init guard tests past reth's first persistence Co-Authored-By: Claude Fable 5.1 (cherry picked from commit c9576b8d0fc61db9d67646365f541848bfb8c582) (cherry picked from commit ce3a7c42a019dff799492ffbc051fd66c8edcb39) --- .../proofs_history_init_test.go | 28 ++++++++++++++----- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/op-acceptance-tests/tests/proofs_history_init/proofs_history_init_test.go b/op-acceptance-tests/tests/proofs_history_init/proofs_history_init_test.go index a913e0f92a1..3b3b29854ed 100644 --- a/op-acceptance-tests/tests/proofs_history_init/proofs_history_init_test.go +++ b/op-acceptance-tests/tests/proofs_history_init/proofs_history_init_test.go @@ -81,6 +81,14 @@ func (s followerSpec) followerArgs() []string { return append(s.engineArgs(), fmt.Sprintf("--proofs-history.verification-interval=%d", s.verificationInterval)) } +// headWithPersistedWindow is a chain height by which reth's datadir holds more than a window of +// blocks under the spec's threshold, whatever reth still has in memory: reth persists nothing +// until more than the threshold of blocks sit in memory, and from then on its datadir trails the +// head by at most the threshold. +func (s followerSpec) headWithPersistedWindow() uint64 { + return s.persistenceThreshold + proofsHistoryWindow + windowMargin +} + // initCase is one cell of the matrix the operator path runs under. type initCase struct { spec followerSpec @@ -242,10 +250,10 @@ func includedBlock(t devtest.T, tx *txplan.PlannedTx) uint64 { return bigs.Uint64Strict(receipt.BlockNumber) } -// driveChainPastWindow runs the workload until the chain is more than a window long and the -// follower has imported all of it, and returns the follower's head. -func driveChainPastWindow(sys *initSystem, w *workload) eth.L2BlockRef { - w.driveUntilBlock(proofsHistoryWindow + windowMargin) +// driveChainTo runs the workload until the chain reaches target and the follower has imported +// all of it, and returns the follower's head. +func driveChainTo(sys *initSystem, w *workload, target uint64) eth.L2BlockRef { + w.driveUntilBlock(target) sys.Follower.InSync(sys.Sequencer, safety.LocalUnsafe, resyncAttempts) return sys.Follower.BlockRefByLabel(eth.Unsafe) } @@ -274,6 +282,8 @@ func reconnectFollower(sys *initSystem) { // sweeps that window against the sequencer. It runs before the follower's CL is back, so nothing // extends or prunes the store meanwhile. func verifySeededWindow(t devtest.T, sys *initSystem, w *workload, anchor eth.L2BlockRef) { + t.Require().GreaterOrEqualf(anchor.Number, uint64(proofsHistoryWindow), + "the anchor %s must be at least a window (%d blocks) above genesis for the backfill to span a full window", anchor, proofsHistoryWindow) window := sys.Follower.WaitForProofsStoreBlock(anchor.Number, storeCatchUpTimeout) t.Require().Equalf(anchor.Number, *window.Latest, "the store must be anchored at the datadir tip %s; store window %s", anchor, window) t.Require().Equalf(anchor.Number-proofsHistoryWindow, *window.Earliest, @@ -307,7 +317,9 @@ func TestFollowerServesTheWholeWindowAfterProofsInitWithBackfill(gt *testing.T) t.Parallel() sys := newInitSystem(t, c.spec) w := newWorkload(t, sys.FunderL2) - head := driveChainPastWindow(sys, w) + // A clean stop persists what reth holds in memory, so the datadir tip is the head + // whatever the threshold, and a window plus margin of blocks is enough. + head := driveChainTo(sys, w, proofsHistoryWindow+windowMargin) sys.FollowerCL.Stop() sys.Follower.Stop() @@ -345,7 +357,7 @@ func TestProofsInitSeedsAConsistentStoreFromAKilledNodeDatadir(gt *testing.T) { skipMaskingCell(t, c.spec) sys := newInitSystem(t, c.spec) w := newWorkload(t, sys.FunderL2) - head := driveChainPastWindow(sys, w) + head := driveChainTo(sys, w, c.spec.headWithPersistedWindow()) sys.Follower.StopUngracefully() sys.FollowerCL.Stop() @@ -378,7 +390,9 @@ func TestProofsInitAgainstARunningNodeNeverSeedsAStoreThatMatchesNoBlock(gt *tes skipMaskingCell(t, c.spec) sys := newInitSystem(t, c.spec) w := newWorkload(t, sys.FunderL2) - head := driveChainPastWindow(sys, w) + // The init reads reth's persisted tip, so the chain is driven until that tip is a + // window above genesis under the spec's threshold. + head := driveChainTo(sys, w, c.spec.headWithPersistedWindow()) report, err := sys.Follower.TryInitProofsHistory() if err != nil { From f4bf9f52274563df4e0fe520a37908079338f1df Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Wed, 14 Oct 2026 11:56:26 -0700 Subject: [PATCH 3/3] op-devstack,op-acceptance-tests: address review of the proofs init seam and tests Co-Authored-By: Claude Fable 5.1 (cherry picked from commit a5c44cf371d903840bbee312b936ab22e3dff99c) (cherry picked from commit 21eb4663e3b719779d601419af2fdec85a8c0e8a) --- .../proofs_history_init_test.go | 82 +++++++++---------- op-devstack/dsl/l2_el_proofs.go | 22 ++--- op-devstack/stack/proofs_history.go | 9 +- op-devstack/sysgo/l2_el_opreth.go | 21 +++-- op-devstack/sysgo/l2_el_opreth_test.go | 12 +-- op-devstack/sysgo/mixed_runtime.go | 26 +++--- 6 files changed, 84 insertions(+), 88 deletions(-) diff --git a/op-acceptance-tests/tests/proofs_history_init/proofs_history_init_test.go b/op-acceptance-tests/tests/proofs_history_init/proofs_history_init_test.go index 3b3b29854ed..52bb2d2fa64 100644 --- a/op-acceptance-tests/tests/proofs_history_init/proofs_history_init_test.go +++ b/op-acceptance-tests/tests/proofs_history_init/proofs_history_init_test.go @@ -25,19 +25,19 @@ const ( // explicitly, like the persistence threshold, so how far reth's datadir trails its head does // not depend on reth's defaults. memoryBlockBufferTarget = 5 + // postRestartTrafficBlocks is how far past the head the chain is driven once the follower is + // back, so the sweeps also cover blocks the store indexed live after the init. + postRestartTrafficBlocks = 10 // proofsHistoryWindow is the window the init backfills behind its anchor and the node keeps // afterwards: small enough that pruning moves the window within the test, large enough that // the backfill spans blocks reth has persisted and blocks it still held in memory. proofsHistoryWindow = 30 - // windowMargin is how far past a full window the chain is driven before the init, so the - // backfill target lies above the blocks that funded the workload. - windowMargin = 5 - // postRestartTrafficBlocks is how far past the head the chain is driven once the follower is - // back, so the sweeps also cover blocks the store indexed live after the init. - postRestartTrafficBlocks = 10 // referenceEthProofWindow lets the sequencer answer eth_getProof for every block of the run // from reth's own historical state; reth's default window of zero rejects them all. referenceEthProofWindow = 1_000_000 + // windowMargin is how far past a full window the init's anchor must sit, so the backfill + // target lies above genesis and the store's earliest block is that target, not the clamp. + windowMargin = 5 // pruneTimeout bounds how long the store may take to prune the backfilled blocks once the // chain is a window past the anchor. @@ -89,15 +89,21 @@ func (s followerSpec) headWithPersistedWindow() uint64 { return s.persistenceThreshold + proofsHistoryWindow + windowMargin } +// name identifies a guard cell: the guards are about reth's datadir, so the cells vary what +// shapes it, the persistence schedule and state masking. +func (s followerSpec) name() string { + return fmt.Sprintf("threshold_%d-masking_%d", s.persistenceThreshold, s.stateMaskingBlocks) +} + // initCase is one cell of the matrix the operator path runs under. type initCase struct { - spec followerSpec - useSnapshot bool + plainBackfill bool + spec followerSpec } func (c initCase) name() string { backfill := "snapshot" - if !c.useSnapshot { + if c.plainBackfill { backfill = "plain" } return fmt.Sprintf("threshold_%d-%s-backfill_%s", c.spec.persistenceThreshold, c.spec.proofsPath(), backfill) @@ -107,10 +113,10 @@ func initMatrix() []initCase { var cases []initCase for _, persistenceThreshold := range []uint64{7, 50} { for _, verificationInterval := range []uint64{0, 1} { - for _, useSnapshot := range []bool{true, false} { + for _, plainBackfill := range []bool{false, true} { cases = append(cases, initCase{ - spec: followerSpec{persistenceThreshold: persistenceThreshold, verificationInterval: verificationInterval}, - useSnapshot: useSnapshot, + plainBackfill: plainBackfill, + spec: followerSpec{persistenceThreshold: persistenceThreshold, verificationInterval: verificationInterval}, }) } } @@ -118,21 +124,11 @@ func initMatrix() []initCase { return cases } -// guardCase is one cell of the matrix the init guards run under. The guards are about reth's -// datadir, so the cells vary what shapes it: the persistence schedule and state masking. -type guardCase struct { - spec followerSpec -} - -func (c guardCase) name() string { - return fmt.Sprintf("threshold_%d-masking_%d", c.spec.persistenceThreshold, c.spec.stateMaskingBlocks) -} - -func guardMatrix() []guardCase { - var cases []guardCase +func guardMatrix() []followerSpec { + var cases []followerSpec for _, persistenceThreshold := range []uint64{7, 50} { for _, stateMaskingBlocks := range []uint64{0, 30} { - cases = append(cases, guardCase{spec: followerSpec{persistenceThreshold: persistenceThreshold, stateMaskingBlocks: stateMaskingBlocks}}) + cases = append(cases, followerSpec{persistenceThreshold: persistenceThreshold, stateMaskingBlocks: stateMaskingBlocks}) } } return cases @@ -324,7 +320,7 @@ func TestFollowerServesTheWholeWindowAfterProofsInitWithBackfill(gt *testing.T) sys.FollowerCL.Stop() sys.Follower.Stop() report := sys.Follower.InitProofsHistory(func(init *stack.ProofsHistoryInit) { - init.UseSnapshot = c.useSnapshot + init.PlainBackfill = c.plainBackfill }) t.Logger().Info("Proofs init seeded the follower's store", "cell", c.name(), "init_duration", report.Duration, "head_at_stop", head, "window", proofsHistoryWindow) @@ -351,18 +347,18 @@ func TestFollowerServesTheWholeWindowAfterProofsInitWithBackfill(gt *testing.T) // window and the blocks it re-imports with proofs that match the sequencer's. func TestProofsInitSeedsAConsistentStoreFromAKilledNodeDatadir(gt *testing.T) { t := devtest.ParallelT(gt) - for _, c := range guardMatrix() { - t.Run(c.name(), func(t devtest.T) { + for _, spec := range guardMatrix() { + t.Run(spec.name(), func(t devtest.T) { t.Parallel() - skipMaskingCell(t, c.spec) - sys := newInitSystem(t, c.spec) + skipMaskingCell(t, spec) + sys := newInitSystem(t, spec) w := newWorkload(t, sys.FunderL2) - head := driveChainTo(sys, w, c.spec.headWithPersistedWindow()) + head := driveChainTo(sys, w, spec.headWithPersistedWindow()) sys.Follower.StopUngracefully() sys.FollowerCL.Stop() report := sys.Follower.InitProofsHistory() - t.Logger().Info("Proofs init seeded the killed follower's store", "cell", c.name(), "init_duration", report.Duration, "head_at_kill", head) + t.Logger().Info("Proofs init seeded the killed follower's store", "cell", spec.name(), "init_duration", report.Duration, "head_at_kill", head) persistedTip := restartFollowerEL(t, sys) t.Require().Lessf(persistedTip.Number, head.Number, @@ -376,32 +372,32 @@ func TestProofsInitSeedsAConsistentStoreFromAKilledNodeDatadir(gt *testing.T) { } } -// TestProofsInitAgainstARunningNodeNeverSeedsAStoreThatMatchesNoBlock runs `op-reth proofs init` +// TestProofsInitAgainstARunningNodeLeavesAStoreThatMatchesTheChain runs `op-reth proofs init` // against the datadir of a follower that is still running and persisting, which an operator can // do by mistake. Whatever the init makes of that datadir, the store the follower ends up running // on must describe the chain: after a clean stop, a seeding of a store the init refused, and a // restart with proofs history, every proof of the window must verify against its header and match // the sequencer's. -func TestProofsInitAgainstARunningNodeNeverSeedsAStoreThatMatchesNoBlock(gt *testing.T) { +func TestProofsInitAgainstARunningNodeLeavesAStoreThatMatchesTheChain(gt *testing.T) { t := devtest.ParallelT(gt) - for _, c := range guardMatrix() { - t.Run(c.name(), func(t devtest.T) { + for _, spec := range guardMatrix() { + t.Run(spec.name(), func(t devtest.T) { t.Parallel() - skipMaskingCell(t, c.spec) - sys := newInitSystem(t, c.spec) + skipMaskingCell(t, spec) + sys := newInitSystem(t, spec) w := newWorkload(t, sys.FunderL2) // The init reads reth's persisted tip, so the chain is driven until that tip is a // window above genesis under the spec's threshold. - head := driveChainTo(sys, w, c.spec.headWithPersistedWindow()) + head := driveChainTo(sys, w, spec.headWithPersistedWindow()) - report, err := sys.Follower.TryInitProofsHistory() + report, err := sys.Follower.InitProofsHistoryRaw() if err != nil { - // #23046 refuses the init when reth persisted past the anchor between the init's - // two database reads. develop has no such guard, so any failure is unexpected. + // The init may refuse a datadir whose tip moved between its two reads (#23046); + // any other failure is a bug. t.Require().ErrorContainsf(err, "reth database tip moved", "proofs init against a running node may only fail because reth's tip moved (#23046); it failed with: %v", err) } - t.Logger().Info("Proofs init ran against the running follower", "cell", c.name(), "init_duration", report.Duration, + t.Logger().Info("Proofs init ran against the running follower", "cell", spec.name(), "init_duration", report.Duration, "head_at_init", head, "refused", err != nil) sys.FollowerCL.Stop() diff --git a/op-devstack/dsl/l2_el_proofs.go b/op-devstack/dsl/l2_el_proofs.go index 920bcceec25..44dc1de51e9 100644 --- a/op-devstack/dsl/l2_el_proofs.go +++ b/op-devstack/dsl/l2_el_proofs.go @@ -142,33 +142,33 @@ func (el *L2ELNode) WaitForProofsStorePrunedPast(block uint64, timeout time.Dura // InitProofsHistory seeds this node's proofs-history store from its datadir with // `op-reth proofs init`, the way an operator does for a node that synced without proofs history, -// and makes the node run with proofs history from its next Start. The node must be stopped: a -// test of what init does to a datadir the node still writes to uses TryInitProofsHistory. +// and makes the node run with proofs history from its next Start. It fails the test if the node +// is running or the init fails; InitProofsHistoryRaw does neither. func (el *L2ELNode) InitProofsHistory(opts ...func(*stack.ProofsHistoryInit)) stack.ProofsHistoryInitReport { if control, ok := el.inner.(stack.ControlledLifecycle); ok { el.require.Falsef(control.Running(), "stop %s before seeding its proofs-history store", el.inner.Name()) } - report, err := el.TryInitProofsHistory(opts...) + report, err := el.InitProofsHistoryRaw(opts...) el.require.NoErrorf(err, "proofs init on %s must seed the store", el.inner.Name()) return report } -// TryInitProofsHistory is InitProofsHistory for a test that asserts on the init's outcome itself: -// it runs whether or not the node is running and returns the failure instead of failing the test. -// The store backfills through a trie-state snapshot and keeps the node's configured window unless -// the options say otherwise. -func (el *L2ELNode) TryInitProofsHistory(opts ...func(*stack.ProofsHistoryInit)) (stack.ProofsHistoryInitReport, error) { +// InitProofsHistoryRaw runs `op-reth proofs init` against this node's datadir whether or not the +// node is running and returns the outcome, failure included, for the test to assert on. The +// options start from op-reth's defaults: a backfill through a trie-state snapshot and the node's +// configured window. +func (el *L2ELNode) InitProofsHistoryRaw(opts ...func(*stack.ProofsHistoryInit)) (stack.ProofsHistoryInitReport, error) { initializer, ok := el.inner.(stack.ProofsHistoryInitializer) el.require.Truef(ok, "L2EL node %s cannot seed a proofs-history store", el.inner.Name()) - init := stack.ProofsHistoryInit{UseSnapshot: true} + var init stack.ProofsHistoryInit for _, opt := range opts { opt(&init) } - logger := el.log.With("name", el.inner.Name(), "chain", el.ChainID(), "window", init.Window, "use_snapshot", init.UseSnapshot) + logger := el.log.With("name", el.inner.Name(), "chain", el.ChainID(), "plain_backfill", init.PlainBackfill) logger.Info("Seeding the proofs-history store from the datadir") report, err := initializer.InitProofsHistory(init) if err != nil { - logger.Warn("Proofs init did not seed the store", "duration", report.Duration, "err", err) + logger.Warn("Proofs init did not seed the store", "duration", report.Duration) return report, err } logger.Info("Proofs init seeded the store", "duration", report.Duration) diff --git a/op-devstack/stack/proofs_history.go b/op-devstack/stack/proofs_history.go index 97152c39dd1..de86665f5ce 100644 --- a/op-devstack/stack/proofs_history.go +++ b/op-devstack/stack/proofs_history.go @@ -4,11 +4,12 @@ import "time" // ProofsHistoryInit is one run of `op-reth proofs init` against a node's datadir: it anchors a // proofs-history store at the datadir's tip, backfills the store Window blocks behind the anchor, -// and makes the node run with proofs history from its next Start on. +// and makes the node run with proofs history from its next Start on. The zero value is the +// operator's plain invocation. type ProofsHistoryInit struct { - // UseSnapshot backfills through a trie-state snapshot, as op-reth does by default; false reads - // every backfilled block's changes from reth's database instead. - UseSnapshot bool + // PlainBackfill reads every backfilled block's changes from reth's database instead of going + // through the trie-state snapshot op-reth builds by default. + PlainBackfill bool // Window is the number of blocks the backfill extends behind the anchor and the window the // node keeps once proofs history is on. Zero keeps the window the node was configured with. Window uint64 diff --git a/op-devstack/sysgo/l2_el_opreth.go b/op-devstack/sysgo/l2_el_opreth.go index 0de2e23826d..ee0c21b3894 100644 --- a/op-devstack/sysgo/l2_el_opreth.go +++ b/op-devstack/sysgo/l2_el_opreth.go @@ -28,9 +28,9 @@ type OpRethConfig struct { // flags) may be selected via OpRethWithBinary. Binary string // DisableProofsHistory builds the node without a seeded proofs-history store and without the - // proofs-history runtime flags. The mixed runtime otherwise enables proofs history on every - // op-reth node; some CLI-superset binaries reject --proofs-history in the mode under test, and - // a test of the operator path seeds the store later with InitProofsHistory. + // proofs-history runtime flags, until InitProofsHistory seeds the store. The mixed runtime + // otherwise enables proofs history on every op-reth node; some CLI-superset binaries reject + // --proofs-history in the mode under test. DisableProofsHistory bool // ProofsHistoryWindow is the number of blocks the proofs-history store keeps. Zero keeps the // mixed runtime's default window. @@ -88,10 +88,9 @@ func OpRethWithBinary(binary string) OpRethOption { }) } -// OpRethWithoutProofsHistory builds the node without proofs history. The mixed runtime enables it -// on every op-reth node by default; use this for a CLI-superset binary that rejects -// --proofs-history in the mode under test, or for a node whose store a test seeds later with -// InitProofsHistory. +// OpRethWithoutProofsHistory builds the node without proofs history, which InitProofsHistory can +// turn on later. The mixed runtime enables it on every op-reth node by default; use this for a +// CLI-superset binary that rejects --proofs-history in the mode under test. func OpRethWithoutProofsHistory() OpRethOption { return OpRethOptionFn(func(p devtest.T, _ ComponentTarget, cfg *OpRethConfig) { cfg.DisableProofsHistory = true @@ -330,11 +329,11 @@ func (n *OpReth) InitProofsHistory(init stack.ProofsHistoryInit) (stack.ProofsHi if init.Window != 0 { window = init.Window } - args := proofsHistoryInitArgs(n.chainConfigPath, n.dataDirPath, n.proofsHistoryStoragePath, window, init.UseSnapshot) + args := proofsHistoryInitArgs(n.chainConfigPath, n.dataDirPath, n.proofsHistoryStoragePath, window, init.PlainBackfill) logger := n.p.Logger().New("name", n.name, "chain", n.chainID, "running", n.sub != nil) logger.Info("Running op-reth proofs init", "cmd", n.execPath, "args", args) started := time.Now() - output, err := exec.Command(n.execPath, args...).CombinedOutput() + output, err := exec.CommandContext(n.p.Ctx(), n.execPath, args...).CombinedOutput() report := stack.ProofsHistoryInitReport{Duration: time.Since(started), Output: string(output)} if err != nil { logger.Warn("op-reth proofs init failed", "duration", report.Duration, "err", err, "output", report.Output) @@ -373,7 +372,7 @@ func proofsHistoryNodeArgs(window uint64, storagePath string) []string { // from the datadir and backfills it window blocks behind the datadir's tip. op-reth backfills // through a trie-state snapshot unless told not to, so the snapshot path gets the operator's // invocation with no flag. -func proofsHistoryInitArgs(chainConfigPath, dataDirPath, storagePath string, window uint64, useSnapshot bool) []string { +func proofsHistoryInitArgs(chainConfigPath, dataDirPath, storagePath string, window uint64, plainBackfill bool) []string { args := []string{ "proofs", "init", @@ -382,7 +381,7 @@ func proofsHistoryInitArgs(chainConfigPath, dataDirPath, storagePath string, win "--proofs-history.storage-path=" + storagePath, "--proofs-history.window=" + strconv.FormatUint(resolveProofsHistoryWindow(window), 10), } - if !useSnapshot { + if plainBackfill { args = append(args, "--proofs-history.use-snapshot=false") } return args diff --git a/op-devstack/sysgo/l2_el_opreth_test.go b/op-devstack/sysgo/l2_el_opreth_test.go index 336348014f6..4fd9339542e 100644 --- a/op-devstack/sysgo/l2_el_opreth_test.go +++ b/op-devstack/sysgo/l2_el_opreth_test.go @@ -46,17 +46,17 @@ func TestProofsHistoryNodeArgs(t *testing.T) { func TestProofsHistoryInitArgs(t *testing.T) { for _, tc := range []struct { name string - useSnapshot bool + plainBackfill bool window uint64 wantSnapshotFlag []string wantWindow string }{ - {name: "snapshot backfill is op-reth's default and gets no flag", useSnapshot: true, window: 30, wantSnapshotFlag: nil, wantWindow: "--proofs-history.window=30"}, - {name: "plain backfill turns the snapshot off", useSnapshot: false, window: 30, wantSnapshotFlag: []string{"--proofs-history.use-snapshot=false"}, wantWindow: "--proofs-history.window=30"}, - {name: "zero window uses the runtime default", useSnapshot: true, window: 0, wantSnapshotFlag: nil, wantWindow: "--proofs-history.window=10000"}, + {name: "snapshot backfill is op-reth's default and gets no flag", plainBackfill: false, window: 30, wantSnapshotFlag: nil, wantWindow: "--proofs-history.window=30"}, + {name: "plain backfill turns the snapshot off", plainBackfill: true, window: 30, wantSnapshotFlag: []string{"--proofs-history.use-snapshot=false"}, wantWindow: "--proofs-history.window=30"}, + {name: "zero window uses the runtime default", plainBackfill: false, window: 0, wantSnapshotFlag: nil, wantWindow: "--proofs-history.window=10000"}, } { t.Run(tc.name, func(t *testing.T) { - args := proofsHistoryInitArgs("/genesis.json", "/data", "/proofs", tc.window, tc.useSnapshot) + args := proofsHistoryInitArgs("/genesis.json", "/data", "/proofs", tc.window, tc.plainBackfill) require.Equal(t, []string{"proofs", "init"}, args[:2]) require.Contains(t, args, "--chain=/genesis.json") @@ -109,7 +109,7 @@ func TestInitProofsHistoryEnablesProofsHistoryOnSuccessOnly(t *testing.T) { t.Run(tc.name, func(t *testing.T) { n := newTestOpReth(t, fakeOpReth(t, tc.exitCode)) - report, err := n.InitProofsHistory(stack.ProofsHistoryInit{UseSnapshot: false, Window: tc.window}) + report, err := n.InitProofsHistory(stack.ProofsHistoryInit{PlainBackfill: true, Window: tc.window}) if tc.exitCode == 0 { require.NoError(t, err) diff --git a/op-devstack/sysgo/mixed_runtime.go b/op-devstack/sysgo/mixed_runtime.go index 083796e6203..765b8533610 100644 --- a/op-devstack/sysgo/mixed_runtime.go +++ b/op-devstack/sysgo/mixed_runtime.go @@ -397,7 +397,6 @@ func mixedNodeRefs(nodes []mixedSingleChainNode) []MixedSingleChainNodeRefs { // buildMixedOpRethNode constructs an OpReth node without starting it. Unless the options disable // proofs history, its store is seeded from the genesis datadir so the node runs with proofs // history from its first start. - func buildMixedOpRethNode( t devtest.T, l2Net *L2Network, @@ -485,24 +484,25 @@ func buildMixedOpRethNode( args = append(args, opRethCfg.ExtraArgs...) node := &OpReth{ - name: key, - chainID: l2Net.ChainID(), - jwtPath: jwtPath, - jwtSecret: jwtSecret, - authRPC: "", - userRPC: "", - execPath: execPath, - args: args, - env: []string{}, - p: t, - l2MetricsRegistrar: metricsRegistrar, + name: key, + chainID: l2Net.ChainID(), + jwtPath: jwtPath, + jwtSecret: jwtSecret, + authRPC: "", + userRPC: "", + execPath: execPath, + args: args, + env: []string{}, + p: t, + l2MetricsRegistrar: metricsRegistrar, + chainConfigPath: chainConfigPath, dataDirPath: dataDirPath, proofsHistoryStoragePath: filepath.Join(tempDir, "proof-history"), proofsHistoryWindow: opRethCfg.ProofsHistoryWindow, } if !opRethCfg.DisableProofsHistory { - _, err := node.InitProofsHistory(stack.ProofsHistoryInit{UseSnapshot: true}) + _, err = node.InitProofsHistory(stack.ProofsHistoryInit{}) t.Require().NoError(err, "must init op-reth proof history") } return node