diff --git a/op-acceptance-tests/tests/safeheaddb_clsync/safeheaddb_test.go b/op-acceptance-tests/tests/safeheaddb_clsync/safeheaddb_test.go index 66f034d5e60..f5d3e3472bc 100644 --- a/op-acceptance-tests/tests/safeheaddb_clsync/safeheaddb_test.go +++ b/op-acceptance-tests/tests/safeheaddb_clsync/safeheaddb_test.go @@ -8,24 +8,13 @@ import ( "github.com/ethereum-optimism/optimism/op-devstack/presets" "github.com/ethereum-optimism/optimism/op-devstack/sysgo" "github.com/ethereum-optimism/optimism/op-node/rollup/sync" + "github.com/ethereum-optimism/optimism/op-service/eth" safety "github.com/ethereum-optimism/optimism/op-service/eth/safety" ) func TestPreserveDatabaseOnCLResync(gt *testing.T) { t := devtest.ParallelT(gt) - // Example error with op-reth: - // - // assertions.go:387: ERROR[03-31|09:40:50.281] - // assertions.go:387: Error Trace: /optimism/op-devstack/sysgo/l2_el_p2p_util.go:61 - // assertions.go:387: /optimism/op-devstack/dsl/l2_el.go:281 - // assertions.go:387: /optimism/op-acceptance-tests/tests/safeheaddb_clsync/safeheaddb_test.go:42 - // assertions.go:387: Error: Received unexpected error: - // assertions.go:387: context deadline exceeded - // assertions.go:387: Test: TestPreserveDatabaseOnCLResync - // assertions.go:387: Messages: The peer was not connected - // assertions.go:387: - sysgo.SkipOnOpReth(t, "not supported (peering issue)") sys := presets.NewSingleChainMultiNode(t, presets.WithGlobalL2CLOption(sysgo.L2CLOptionFn(func(p devtest.T, _ sysgo.ComponentTarget, cfg *sysgo.L2CLConfig) { cfg.SequencerSyncMode = sync.CLSync @@ -42,15 +31,17 @@ func TestPreserveDatabaseOnCLResync(gt *testing.T) { sys.L2CLB.InSync(sys.L2CL, safety.LocalSafe, 30) sys.L2CLB.VerifySafeHeadDatabaseMatches(sys.L2CL) - // Stop the verifier node. Since the sysgo EL uses in-memory storage this also wipes its database. - // With the EL reset to genesis, when the CL restarts it will use EL sync to resync the chain rather than - // deriving it from L1. + // Stop the verifier node and wipe its EL database. + // With the EL reset to genesis and the verifier in CL sync mode, the restarted CL resets to genesis and + // derives the chain again from L1, re-recording safe heads that are already in its database. sys.L2ELB.Stop() + sys.L2ELB.Wipe() sys.L2CLB.Stop() sys.L2CL.Advanced(safety.LocalSafe, 3, 30) sys.L2ELB.Start() + t.Require().Zero(sys.L2ELB.BlockRefByLabel(eth.Unsafe).Number, "EL resync must start from genesis") sys.L2CLB.Start() sys.L2ELB.PeerWith(sys.L2EL) diff --git a/op-acceptance-tests/tests/safeheaddb_elsync/safeheaddb_test.go b/op-acceptance-tests/tests/safeheaddb_elsync/safeheaddb_test.go index c7929eee690..017e6da6fb9 100644 --- a/op-acceptance-tests/tests/safeheaddb_elsync/safeheaddb_test.go +++ b/op-acceptance-tests/tests/safeheaddb_elsync/safeheaddb_test.go @@ -8,6 +8,7 @@ import ( "github.com/ethereum-optimism/optimism/op-devstack/presets" "github.com/ethereum-optimism/optimism/op-devstack/sysgo" "github.com/ethereum-optimism/optimism/op-node/rollup/sync" + "github.com/ethereum-optimism/optimism/op-service/eth" safety "github.com/ethereum-optimism/optimism/op-service/eth/safety" ) @@ -26,18 +27,6 @@ func newSingleChainMultiNodeELSync(t devtest.T) *presets.SingleChainMultiNode { func TestTruncateDatabaseOnELResync(gt *testing.T) { t := devtest.ParallelT(gt) - // Example error with op-reth: - // - // assertions.go:387: ERROR[03-31|09:41:28.788] - // assertions.go:387: Error Trace: /optimism/op-devstack/sysgo/l2_el_p2p_util.go:61 - // assertions.go:387: /optimism/op-devstack/dsl/l2_el.go:281 - // assertions.go:387: /optimism/op-acceptance-tests/tests/safeheaddb_elsync/safeheaddb_test.go:44 - // assertions.go:387: Error: Received unexpected error: - // assertions.go:387: dial tcp 127.0.0.1:55994: i/o timeout - // assertions.go:387: Test: TestTruncateDatabaseOnELResync - // assertions.go:387: Messages: The peer was not connected - // assertions.go:387: - sysgo.SkipOnOpReth(t, "not supported (peering issue)") sys := newSingleChainMultiNodeELSync(t) dsl.CheckAll(t, @@ -47,15 +36,17 @@ func TestTruncateDatabaseOnELResync(gt *testing.T) { sys.L2CLB.InSync(sys.L2CL, safety.LocalSafe, 30) sys.L2CLB.VerifySafeHeadDatabaseMatches(sys.L2CL) - // Stop the verifier node. Since the sysgo EL uses in-memory storage this also wipes its database. + // Stop the verifier node and wipe its EL database. // With the EL reset to genesis, when the CL restarts it will use EL sync to resync the chain rather than // deriving it from L1. sys.L2ELB.Stop() + sys.L2ELB.Wipe() sys.L2CLB.Stop() sys.L2CL.Advanced(safety.LocalSafe, 3, 30) sys.L2ELB.Start() + t.Require().Zero(sys.L2ELB.BlockRefByLabel(eth.Unsafe).Number, "EL resync must start from genesis") sys.L2CLB.Start() // Wait for the restarted EL to be ready to accept RPC/P2P connections // before attempting to peer. Without this, PeerWith may waste its 30s diff --git a/op-acceptance-tests/tests/sync/elsync/gap_clp2p/sync_test.go b/op-acceptance-tests/tests/sync/elsync/gap_clp2p/sync_test.go index a5623a6e98f..e1d57aa5f04 100644 --- a/op-acceptance-tests/tests/sync/elsync/gap_clp2p/sync_test.go +++ b/op-acceptance-tests/tests/sync/elsync/gap_clp2p/sync_test.go @@ -50,13 +50,14 @@ func TestCLUnsafeNotRewoundOnInvalidDuringELSync(gt *testing.T) { t := devtest.ParallelT(gt) // Example error with op-reth: // - // assertions.go:387: ERROR[03-31|09:41:58.089] - // assertions.go:387: Error Trace: /optimism/op-devstack/dsl/l2_cl.go:279 - // assertions.go:387: /optimism/op-acceptance-tests/tests/sync/elsync/gap_clp2p/sync_test.go:96 - // assertions.go:387: Error: Received unexpected error: - // assertions.go:387: expected head not to advance: unsafe - // assertions.go:387: Test: TestCLUnsafeNotRewoundOnInvalidDuringELSync - // assertions.go:387: + // Error Trace: /optimism/op-devstack/dsl/l2_el.go:822 + // /optimism/op-acceptance-tests/tests/sync/elsync/gap_clp2p/sync_test.go:113 + // Error: Not equal: + // expected: 0x5 + // actual : 0x6 + // Test: TestCLUnsafeNotRewoundOnInvalidDuringELSync + // + // TODO(#23317): the CL unsafe head advances onto the injected invalid payload; remove once fixed. sysgo.SkipOnOpReth(t, "not supported") sys := newGapCLP2PSystem(t) logger := t.Logger() @@ -69,6 +70,7 @@ func TestCLUnsafeNotRewoundOnInvalidDuringELSync(gt *testing.T) { sys.L2CLB.Stop() // Wipe out L2ELB state to start from genesis sys.L2ELB.Stop() + sys.L2ELB.Wipe() sys.L2ELB.Start() sys.L2CLB.Start() diff --git a/op-acceptance-tests/tests/sync/elsync/offset_el_safe/sync_test.go b/op-acceptance-tests/tests/sync/elsync/offset_el_safe/sync_test.go index fd1d0506858..55a2cad6aeb 100644 --- a/op-acceptance-tests/tests/sync/elsync/offset_el_safe/sync_test.go +++ b/op-acceptance-tests/tests/sync/elsync/offset_el_safe/sync_test.go @@ -25,7 +25,8 @@ import ( // 5. Assert safe/finalized are behind unsafe. func TestELSyncSafeRetractedByOffset(gt *testing.T) { t := devtest.ParallelT(gt) - sysgo.SkipOnOpReth(t, "not supported (peering issue)") + // TODO(#23828): remove once kona-node supports OffsetELSafe. + sysgo.SkipOnKonaNode(t, "kona-node does not support OffsetELSafe") sys := newOffsetELSafeSystem(t) require := t.Require() logger := t.Logger() @@ -39,6 +40,7 @@ func TestELSyncSafeRetractedByOffset(gt *testing.T) { // Stop verifier and wipe its EL to force a full EL sync on restart. sys.L2ELB.Stop() + sys.L2ELB.Wipe() sys.L2CLB.Stop() // Stop the batcher so new sequencer blocks are unsafe-only. @@ -50,7 +52,9 @@ func TestELSyncSafeRetractedByOffset(gt *testing.T) { sys.L2CL.Advanced(safety.LocalUnsafe, 3, 30) sys.L2ELB.Start() + t.Require().Zero(sys.L2ELB.BlockRefByLabel(eth.Unsafe).Number, "EL resync must start from genesis") sys.L2CLB.Start() + sys.L2ELB.WaitForOnline() sys.L2ELB.PeerWith(sys.L2EL) // Wait for the verifier CL to advance LocalSafe, which confirms: @@ -59,7 +63,9 @@ func TestELSyncSafeRetractedByOffset(gt *testing.T) { // - Derivation began processing old batched data // Because the batcher is stopped, derivation can only reach the last batched // block — the gap between safe and unsafe is permanent. - sys.L2CLB.Advanced(safety.LocalSafe, 1, 30) + // A full EL sync after a wipe is slower than the initial sync, so allow the + // same budget as TestTruncateDatabaseOnELResync. + sys.L2CLB.Advanced(safety.LocalSafe, 1, 90) unsafeHead := sys.L2ELB.BlockRefByLabel(eth.Unsafe) safeHead := sys.L2ELB.SafeHead().BlockRef diff --git a/op-devstack/dsl/l2_el.go b/op-devstack/dsl/l2_el.go index 6adfb21a85b..1d57022c733 100644 --- a/op-devstack/dsl/l2_el.go +++ b/op-devstack/dsl/l2_el.go @@ -531,6 +531,15 @@ func (el *L2ELNode) Start() { lifecycle.Start() } +// Wipe discards the stopped node's chain data so the next Start begins from genesis. +// Stop alone does not guarantee this: op-reth keeps its datadir across restarts. +func (el *L2ELNode) Wipe() { + el.log.Info("Wiping", "name", el.inner.Name()) + wipeable, ok := el.inner.(stack.Wipeable) + el.require.Truef(ok, "L2EL node %s is not wipeable", el.inner.Name()) + wipeable.Wipe() +} + func (el *L2ELNode) PeerWith(peer *L2ELNode) { sysgo.ConnectP2P(el.ctx, el.require, el.inner.L2EthClient().RPC(), peer.inner.L2EthClient().RPC()) } diff --git a/op-devstack/presets/rpc_frontends.go b/op-devstack/presets/rpc_frontends.go index 07ff63aba8a..c5ad40fd5a6 100644 --- a/op-devstack/presets/rpc_frontends.go +++ b/op-devstack/presets/rpc_frontends.go @@ -156,6 +156,7 @@ type l2ELFrontend struct { l2EngineClient *sources.EngineClient lifecycle stack.Lifecycle control stack.ControlledLifecycle + wipeable stack.Wipeable } var _ stack.L2ELNode = (*l2ELFrontend)(nil) @@ -195,6 +196,11 @@ func (r *l2ELFrontend) Stop() { r.lifecycle.Stop() } +func (r *l2ELFrontend) Wipe() { + r.require().NotNil(r.wipeable, "L2EL node %s is not wipeable", r.Name()) + r.wipeable.Wipe() +} + func (r *l2ELFrontend) StartControlled(ctx context.Context) error { if r.control == nil { return fmt.Errorf("L2EL node %s is not controllable", r.Name()) diff --git a/op-devstack/presets/sysgo_runtime.go b/op-devstack/presets/sysgo_runtime.go index 3ae12a67ebe..74438651c36 100644 --- a/op-devstack/presets/sysgo_runtime.go +++ b/op-devstack/presets/sysgo_runtime.go @@ -66,6 +66,9 @@ func newL2ELFrontend(t devtest.T, name string, chainID eth.ChainID, userRPC stri if control, ok := lifecycle[0].(stack.ControlledLifecycle); ok { l2EL.control = control } + if wipeable, ok := lifecycle[0].(stack.Wipeable); ok { + l2EL.wipeable = wipeable + } } return l2EL } diff --git a/op-devstack/stack/lifecycle.go b/op-devstack/stack/lifecycle.go index f26d3b7e605..ca61de4155b 100644 --- a/op-devstack/stack/lifecycle.go +++ b/op-devstack/stack/lifecycle.go @@ -7,6 +7,14 @@ type Lifecycle interface { Stop() } +// Wipeable is implemented by nodes whose chain data can be discarded, so that the +// next Start begins from genesis. Stop alone does not guarantee this: whether a +// node keeps its data across a restart depends on the implementation. +type Wipeable interface { + // Wipe discards the node's chain data. The node must be stopped. + Wipe() +} + type ControlledLifecycle interface { StartControlled(ctx context.Context) error StopControlled(ctx context.Context) error diff --git a/op-devstack/sysgo/l2_el_opgeth.go b/op-devstack/sysgo/l2_el_opgeth.go index 91819153d58..43727f48c41 100644 --- a/op-devstack/sysgo/l2_el_opgeth.go +++ b/op-devstack/sysgo/l2_el_opgeth.go @@ -13,6 +13,7 @@ import ( "github.com/ethereum/go-ethereum/p2p/enode" "github.com/ethereum-optimism/optimism/op-devstack/devtest" + "github.com/ethereum-optimism/optimism/op-devstack/stack" "github.com/ethereum-optimism/optimism/op-e2e/e2eutils/geth" "github.com/ethereum-optimism/optimism/op-service/log" "github.com/ethereum-optimism/optimism/op-service/testutils/tcpproxy" @@ -39,6 +40,7 @@ type OpGeth struct { } var _ L2ELNode = (*OpGeth)(nil) +var _ stack.Wipeable = (*OpGeth)(nil) func (n *OpGeth) UserRPC() string { return n.userRPC @@ -145,6 +147,14 @@ func (n *OpGeth) Stop() { n.l2Geth = nil } +// Wipe is a no-op beyond checking that the node is stopped: op-geth runs with +// in-memory storage, so Stop already discards its chain data. +func (n *OpGeth) Wipe() { + n.mu.Lock() + defer n.mu.Unlock() + n.p.Require().Nil(n.l2Geth, "op-geth %s must be stopped before wiping", n.name) +} + // Callers must hold n.mu. func (n *OpGeth) clearProxyUpstreams() { if n.userProxy != nil { diff --git a/op-devstack/sysgo/l2_el_opreth.go b/op-devstack/sysgo/l2_el_opreth.go index 959204e81be..d627287b20a 100644 --- a/op-devstack/sysgo/l2_el_opreth.go +++ b/op-devstack/sysgo/l2_el_opreth.go @@ -4,10 +4,13 @@ import ( "context" "fmt" "net/url" + "os" + "os/exec" "strings" "sync" "github.com/ethereum-optimism/optimism/op-devstack/devtest" + "github.com/ethereum-optimism/optimism/op-devstack/stack" "github.com/ethereum-optimism/optimism/op-service/eth" "github.com/ethereum-optimism/optimism/op-service/log" "github.com/ethereum-optimism/optimism/op-service/logpipe" @@ -118,6 +121,11 @@ type OpReth struct { // Each entry is of the form "key=value". env []string + // dataDirs lists every directory holding chain data, including --datadir; + // initCmds populate them from genesis. + dataDirs []string + initCmds [][]string + p devtest.T sub *SubProcess @@ -126,6 +134,7 @@ type OpReth struct { } var _ L2ELNode = (*OpReth)(nil) +var _ stack.Wipeable = (*OpReth)(nil) func (n *OpReth) Start() { n.mu.Lock() @@ -239,6 +248,30 @@ func (n *OpReth) Stop() { n.sub = nil } +// Wipe deletes the node's chain data and re-initializes it from genesis, so the +// next Start begins with an empty chain. Unlike op-geth, op-reth keeps its +// datadir across Stop/Start. +func (n *OpReth) Wipe() { + n.mu.Lock() + defer n.mu.Unlock() + n.p.Require().Nil(n.sub, "op-reth %s must be stopped before wiping", n.name) + n.p.Logger().Info("Wiping op-reth", "name", n.name, "chain", n.chainID) + n.initData() +} + +// initData resets dataDirs and runs initCmds. Callers must hold n.mu or have exclusive access. +func (n *OpReth) initData() { + require := n.p.Require() + for _, dir := range n.dataDirs { + require.NoError(os.RemoveAll(dir), "must remove %s", dir) + require.NoError(os.MkdirAll(dir, 0o755), "must create %s", dir) + } + for _, args := range n.initCmds { + out, err := exec.CommandContext(n.p.Ctx(), n.execPath, args...).CombinedOutput() + require.NoError(err, "must run op-reth %s: %s", strings.Join(args, " "), string(out)) + } +} + // Callers must hold n.mu. func (n *OpReth) clearProxyUpstreams() { if n.userProxy != nil { diff --git a/op-devstack/sysgo/mixed_runtime.go b/op-devstack/sysgo/mixed_runtime.go index f221c92335a..7216c20fbd1 100644 --- a/op-devstack/sysgo/mixed_runtime.go +++ b/op-devstack/sysgo/mixed_runtime.go @@ -6,7 +6,6 @@ import ( "encoding/json" "fmt" "os" - "os/exec" "path/filepath" "strings" @@ -408,7 +407,6 @@ func buildMixedOpRethNode( t.Require().NoError(os.WriteFile(chainConfigPath, data, 0o640), "must write genesis file") dataDirPath := filepath.Join(tempDir, "data") - t.Require().NoError(os.MkdirAll(dataDirPath, 0o755), "must create datadir") logDirPath := filepath.Join(tempDir, "logs") t.Require().NoError(os.MkdirAll(logDirPath, 0o755), "must create logs dir") @@ -467,26 +465,23 @@ func buildMixedOpRethNode( args = append(args, "--metrics=127.0.0.1:0") } - initArgs := []string{ + dataDirs := []string{dataDirPath} + initCmds := [][]string{{ "init", "--datadir=" + dataDirPath, "--chain=" + chainConfigPath, - } - initOut, err := exec.Command(execPath, initArgs...).CombinedOutput() - t.Require().NoError(err, "must init op-reth node: %s", string(initOut)) + }} if !opRethCfg.DisableProofsHistory { proofHistoryDir := filepath.Join(tempDir, "proof-history") - - initProofsArgs := []string{ + dataDirs = append(dataDirs, proofHistoryDir) + initCmds = append(initCmds, []string{ "proofs", "init", "--datadir=" + dataDirPath, "--chain=" + chainConfigPath, "--proofs-history.storage-path=" + proofHistoryDir, - } - initOut, initErr := exec.Command(execPath, initProofsArgs...).CombinedOutput() - t.Require().NoError(initErr, "must init op-reth proof history: %s", string(initOut)) + }) args = append( args, @@ -498,7 +493,7 @@ func buildMixedOpRethNode( args = append(args, opRethCfg.ExtraArgs...) - return &OpReth{ + node := &OpReth{ name: key, chainID: l2Net.ChainID(), jwtPath: jwtPath, @@ -508,9 +503,13 @@ func buildMixedOpRethNode( execPath: execPath, args: args, env: []string{}, + dataDirs: dataDirs, + initCmds: initCmds, p: t, l2MetricsRegistrar: metricsRegistrar, } + node.initData() + return node } func startMixedOpRethNode(