Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 6 additions & 15 deletions op-acceptance-tests/tests/safeheaddb_clsync/safeheaddb_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,24 +8,13 @@ import (
"github.com/ethereum-optimism/optimism/op-devstack/presets"
"github.com/ethereum-optimism/optimism/op-devstack/sysgo"
"github.com/ethereum-optimism/optimism/op-node/rollup/sync"
"github.com/ethereum-optimism/optimism/op-service/eth"

safety "github.com/ethereum-optimism/optimism/op-service/eth/safety"
)

func TestPreserveDatabaseOnCLResync(gt *testing.T) {
t := devtest.ParallelT(gt)
// Example error with op-reth:
//
// assertions.go:387: ERROR[03-31|09:40:50.281]
// assertions.go:387: Error Trace: /optimism/op-devstack/sysgo/l2_el_p2p_util.go:61
// assertions.go:387: /optimism/op-devstack/dsl/l2_el.go:281
// assertions.go:387: /optimism/op-acceptance-tests/tests/safeheaddb_clsync/safeheaddb_test.go:42
// assertions.go:387: Error: Received unexpected error:
// assertions.go:387: context deadline exceeded
// assertions.go:387: Test: TestPreserveDatabaseOnCLResync
// assertions.go:387: Messages: The peer was not connected
// assertions.go:387:
sysgo.SkipOnOpReth(t, "not supported (peering issue)")
sys := presets.NewSingleChainMultiNode(t,
presets.WithGlobalL2CLOption(sysgo.L2CLOptionFn(func(p devtest.T, _ sysgo.ComponentTarget, cfg *sysgo.L2CLConfig) {
cfg.SequencerSyncMode = sync.CLSync
Expand All @@ -42,15 +31,17 @@ func TestPreserveDatabaseOnCLResync(gt *testing.T) {
sys.L2CLB.InSync(sys.L2CL, safety.LocalSafe, 30)
sys.L2CLB.VerifySafeHeadDatabaseMatches(sys.L2CL)

// Stop the verifier node. Since the sysgo EL uses in-memory storage this also wipes its database.
// With the EL reset to genesis, when the CL restarts it will use EL sync to resync the chain rather than
// deriving it from L1.
// Stop the verifier node and wipe its EL database.
// With the EL reset to genesis and the verifier in CL sync mode, the restarted CL resets to genesis and
// derives the chain again from L1, re-recording safe heads that are already in its database.
sys.L2ELB.Stop()
sys.L2ELB.Wipe()
Comment thread
joshklop marked this conversation as resolved.
Resolved
sys.L2CLB.Stop()

sys.L2CL.Advanced(safety.LocalSafe, 3, 30)

sys.L2ELB.Start()
t.Require().Zero(sys.L2ELB.BlockRefByLabel(eth.Unsafe).Number, "EL resync must start from genesis")
sys.L2CLB.Start()
sys.L2ELB.PeerWith(sys.L2EL)

Expand Down
17 changes: 4 additions & 13 deletions op-acceptance-tests/tests/safeheaddb_elsync/safeheaddb_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import (
"github.com/ethereum-optimism/optimism/op-devstack/presets"
"github.com/ethereum-optimism/optimism/op-devstack/sysgo"
"github.com/ethereum-optimism/optimism/op-node/rollup/sync"
"github.com/ethereum-optimism/optimism/op-service/eth"

safety "github.com/ethereum-optimism/optimism/op-service/eth/safety"
)
Expand All @@ -26,18 +27,6 @@ func newSingleChainMultiNodeELSync(t devtest.T) *presets.SingleChainMultiNode {

func TestTruncateDatabaseOnELResync(gt *testing.T) {
t := devtest.ParallelT(gt)
// Example error with op-reth:
//
// assertions.go:387: ERROR[03-31|09:41:28.788]
// assertions.go:387: Error Trace: /optimism/op-devstack/sysgo/l2_el_p2p_util.go:61
// assertions.go:387: /optimism/op-devstack/dsl/l2_el.go:281
// assertions.go:387: /optimism/op-acceptance-tests/tests/safeheaddb_elsync/safeheaddb_test.go:44
// assertions.go:387: Error: Received unexpected error:
// assertions.go:387: dial tcp 127.0.0.1:55994: i/o timeout
// assertions.go:387: Test: TestTruncateDatabaseOnELResync
// assertions.go:387: Messages: The peer was not connected
// assertions.go:387:
sysgo.SkipOnOpReth(t, "not supported (peering issue)")
sys := newSingleChainMultiNodeELSync(t)

dsl.CheckAll(t,
Expand All @@ -47,15 +36,17 @@ func TestTruncateDatabaseOnELResync(gt *testing.T) {
sys.L2CLB.InSync(sys.L2CL, safety.LocalSafe, 30)
sys.L2CLB.VerifySafeHeadDatabaseMatches(sys.L2CL)

// Stop the verifier node. Since the sysgo EL uses in-memory storage this also wipes its database.
// Stop the verifier node and wipe its EL database.
// With the EL reset to genesis, when the CL restarts it will use EL sync to resync the chain rather than
// deriving it from L1.
sys.L2ELB.Stop()
sys.L2ELB.Wipe()
Comment thread
joshklop marked this conversation as resolved.
Resolved
sys.L2CLB.Stop()

sys.L2CL.Advanced(safety.LocalSafe, 3, 30)

sys.L2ELB.Start()
t.Require().Zero(sys.L2ELB.BlockRefByLabel(eth.Unsafe).Number, "EL resync must start from genesis")
sys.L2CLB.Start()
// Wait for the restarted EL to be ready to accept RPC/P2P connections
// before attempting to peer. Without this, PeerWith may waste its 30s
Expand Down
16 changes: 9 additions & 7 deletions op-acceptance-tests/tests/sync/elsync/gap_clp2p/sync_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -50,13 +50,14 @@ func TestCLUnsafeNotRewoundOnInvalidDuringELSync(gt *testing.T) {
t := devtest.ParallelT(gt)
// Example error with op-reth:
//
// assertions.go:387: ERROR[03-31|09:41:58.089]
// assertions.go:387: Error Trace: /optimism/op-devstack/dsl/l2_cl.go:279
// assertions.go:387: /optimism/op-acceptance-tests/tests/sync/elsync/gap_clp2p/sync_test.go:96
// assertions.go:387: Error: Received unexpected error:
// assertions.go:387: expected head not to advance: unsafe
// assertions.go:387: Test: TestCLUnsafeNotRewoundOnInvalidDuringELSync
// assertions.go:387:
// Error Trace: /optimism/op-devstack/dsl/l2_el.go:822
// /optimism/op-acceptance-tests/tests/sync/elsync/gap_clp2p/sync_test.go:113
// Error: Not equal:
// expected: 0x5
// actual : 0x6
// Test: TestCLUnsafeNotRewoundOnInvalidDuringELSync
//
// TODO(#23317): the CL unsafe head advances onto the injected invalid payload; remove once fixed.
sysgo.SkipOnOpReth(t, "not supported")
sys := newGapCLP2PSystem(t)
logger := t.Logger()
Expand All @@ -69,6 +70,7 @@ func TestCLUnsafeNotRewoundOnInvalidDuringELSync(gt *testing.T) {
sys.L2CLB.Stop()
// Wipe out L2ELB state to start from genesis
sys.L2ELB.Stop()
sys.L2ELB.Wipe()
Comment thread
joshklop marked this conversation as resolved.
Resolved
sys.L2ELB.Start()
sys.L2CLB.Start()

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,8 @@ import (
// 5. Assert safe/finalized are behind unsafe.
func TestELSyncSafeRetractedByOffset(gt *testing.T) {
t := devtest.ParallelT(gt)
sysgo.SkipOnOpReth(t, "not supported (peering issue)")
// TODO(#23828): remove once kona-node supports OffsetELSafe.
sysgo.SkipOnKonaNode(t, "kona-node does not support OffsetELSafe")
sys := newOffsetELSafeSystem(t)
require := t.Require()
logger := t.Logger()
Expand All @@ -39,6 +40,7 @@ func TestELSyncSafeRetractedByOffset(gt *testing.T) {

// Stop verifier and wipe its EL to force a full EL sync on restart.
sys.L2ELB.Stop()
sys.L2ELB.Wipe()
Comment thread
joshklop marked this conversation as resolved.
Resolved
sys.L2CLB.Stop()

// Stop the batcher so new sequencer blocks are unsafe-only.
Expand All @@ -50,7 +52,9 @@ func TestELSyncSafeRetractedByOffset(gt *testing.T) {
sys.L2CL.Advanced(safety.LocalUnsafe, 3, 30)

sys.L2ELB.Start()
t.Require().Zero(sys.L2ELB.BlockRefByLabel(eth.Unsafe).Number, "EL resync must start from genesis")
sys.L2CLB.Start()
sys.L2ELB.WaitForOnline()
sys.L2ELB.PeerWith(sys.L2EL)

// Wait for the verifier CL to advance LocalSafe, which confirms:
Expand All @@ -59,7 +63,9 @@ func TestELSyncSafeRetractedByOffset(gt *testing.T) {
// - Derivation began processing old batched data
// Because the batcher is stopped, derivation can only reach the last batched
// block — the gap between safe and unsafe is permanent.
sys.L2CLB.Advanced(safety.LocalSafe, 1, 30)
// A full EL sync after a wipe is slower than the initial sync, so allow the
// same budget as TestTruncateDatabaseOnELResync.
sys.L2CLB.Advanced(safety.LocalSafe, 1, 90)

unsafeHead := sys.L2ELB.BlockRefByLabel(eth.Unsafe)
safeHead := sys.L2ELB.SafeHead().BlockRef
Expand Down
9 changes: 9 additions & 0 deletions op-devstack/dsl/l2_el.go
Original file line number Diff line number Diff line change
Expand Up @@ -531,6 +531,15 @@ func (el *L2ELNode) Start() {
lifecycle.Start()
}

// Wipe discards the stopped node's chain data so the next Start begins from genesis.
// Stop alone does not guarantee this: op-reth keeps its datadir across restarts.
func (el *L2ELNode) Wipe() {
el.log.Info("Wiping", "name", el.inner.Name())
wipeable, ok := el.inner.(stack.Wipeable)
Comment thread
joshklop marked this conversation as resolved.
Resolved
el.require.Truef(ok, "L2EL node %s is not wipeable", el.inner.Name())
wipeable.Wipe()
}

func (el *L2ELNode) PeerWith(peer *L2ELNode) {
sysgo.ConnectP2P(el.ctx, el.require, el.inner.L2EthClient().RPC(), peer.inner.L2EthClient().RPC())
}
Expand Down
6 changes: 6 additions & 0 deletions op-devstack/presets/rpc_frontends.go
Original file line number Diff line number Diff line change
Expand Up @@ -156,6 +156,7 @@ type l2ELFrontend struct {
l2EngineClient *sources.EngineClient
lifecycle stack.Lifecycle
control stack.ControlledLifecycle
wipeable stack.Wipeable
Comment thread
joshklop marked this conversation as resolved.
Resolved
}

var _ stack.L2ELNode = (*l2ELFrontend)(nil)
Expand Down Expand Up @@ -195,6 +196,11 @@ func (r *l2ELFrontend) Stop() {
r.lifecycle.Stop()
}

func (r *l2ELFrontend) Wipe() {
r.require().NotNil(r.wipeable, "L2EL node %s is not wipeable", r.Name())
r.wipeable.Wipe()
}

func (r *l2ELFrontend) StartControlled(ctx context.Context) error {
if r.control == nil {
return fmt.Errorf("L2EL node %s is not controllable", r.Name())
Expand Down
3 changes: 3 additions & 0 deletions op-devstack/presets/sysgo_runtime.go
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,9 @@ func newL2ELFrontend(t devtest.T, name string, chainID eth.ChainID, userRPC stri
if control, ok := lifecycle[0].(stack.ControlledLifecycle); ok {
l2EL.control = control
}
if wipeable, ok := lifecycle[0].(stack.Wipeable); ok {
l2EL.wipeable = wipeable
}
}
return l2EL
}
Expand Down
8 changes: 8 additions & 0 deletions op-devstack/stack/lifecycle.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,14 @@ type Lifecycle interface {
Stop()
}

// Wipeable is implemented by nodes whose chain data can be discarded, so that the
// next Start begins from genesis. Stop alone does not guarantee this: whether a
// node keeps its data across a restart depends on the implementation.
type Wipeable interface {
// Wipe discards the node's chain data. The node must be stopped.
Wipe()
}

type ControlledLifecycle interface {
StartControlled(ctx context.Context) error
StopControlled(ctx context.Context) error
Expand Down
10 changes: 10 additions & 0 deletions op-devstack/sysgo/l2_el_opgeth.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ import (
"github.com/ethereum/go-ethereum/p2p/enode"

"github.com/ethereum-optimism/optimism/op-devstack/devtest"
"github.com/ethereum-optimism/optimism/op-devstack/stack"
"github.com/ethereum-optimism/optimism/op-e2e/e2eutils/geth"
"github.com/ethereum-optimism/optimism/op-service/log"
"github.com/ethereum-optimism/optimism/op-service/testutils/tcpproxy"
Expand All @@ -39,6 +40,7 @@ type OpGeth struct {
}

var _ L2ELNode = (*OpGeth)(nil)
var _ stack.Wipeable = (*OpGeth)(nil)

func (n *OpGeth) UserRPC() string {
return n.userRPC
Expand Down Expand Up @@ -145,6 +147,14 @@ func (n *OpGeth) Stop() {
n.l2Geth = nil
}

// Wipe is a no-op beyond checking that the node is stopped: op-geth runs with
// in-memory storage, so Stop already discards its chain data.
func (n *OpGeth) Wipe() {
Comment thread
joshklop marked this conversation as resolved.
Resolved
n.mu.Lock()
defer n.mu.Unlock()
n.p.Require().Nil(n.l2Geth, "op-geth %s must be stopped before wiping", n.name)
Comment thread
joshklop marked this conversation as resolved.
Resolved
}

// Callers must hold n.mu.
func (n *OpGeth) clearProxyUpstreams() {
if n.userProxy != nil {
Expand Down
33 changes: 33 additions & 0 deletions op-devstack/sysgo/l2_el_opreth.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,10 +4,13 @@ import (
"context"
"fmt"
"net/url"
"os"
"os/exec"
"strings"
"sync"

"github.com/ethereum-optimism/optimism/op-devstack/devtest"
"github.com/ethereum-optimism/optimism/op-devstack/stack"
"github.com/ethereum-optimism/optimism/op-service/eth"
"github.com/ethereum-optimism/optimism/op-service/log"
"github.com/ethereum-optimism/optimism/op-service/logpipe"
Expand Down Expand Up @@ -118,6 +121,11 @@ type OpReth struct {
// Each entry is of the form "key=value".
env []string

// dataDirs lists every directory holding chain data, including --datadir;
// initCmds populate them from genesis.
dataDirs []string
Comment thread
joshklop marked this conversation as resolved.
Resolved
initCmds [][]string

p devtest.T

sub *SubProcess
Expand All @@ -126,6 +134,7 @@ type OpReth struct {
}

var _ L2ELNode = (*OpReth)(nil)
var _ stack.Wipeable = (*OpReth)(nil)

func (n *OpReth) Start() {
n.mu.Lock()
Expand Down Expand Up @@ -239,6 +248,30 @@ func (n *OpReth) Stop() {
n.sub = nil
}

// Wipe deletes the node's chain data and re-initializes it from genesis, so the
// next Start begins with an empty chain. Unlike op-geth, op-reth keeps its
// datadir across Stop/Start.
func (n *OpReth) Wipe() {
n.mu.Lock()
defer n.mu.Unlock()
n.p.Require().Nil(n.sub, "op-reth %s must be stopped before wiping", n.name)
n.p.Logger().Info("Wiping op-reth", "name", n.name, "chain", n.chainID)
n.initData()
}

// initData resets dataDirs and runs initCmds. Callers must hold n.mu or have exclusive access.
func (n *OpReth) initData() {
require := n.p.Require()
for _, dir := range n.dataDirs {
require.NoError(os.RemoveAll(dir), "must remove %s", dir)
require.NoError(os.MkdirAll(dir, 0o755), "must create %s", dir)
}
for _, args := range n.initCmds {
out, err := exec.CommandContext(n.p.Ctx(), n.execPath, args...).CombinedOutput()
Comment thread
wiz-0f98cca50a[bot] marked this conversation as resolved.
Resolved
require.NoError(err, "must run op-reth %s: %s", strings.Join(args, " "), string(out))
}
}

// Callers must hold n.mu.
func (n *OpReth) clearProxyUpstreams() {
if n.userProxy != nil {
Expand Down
23 changes: 11 additions & 12 deletions op-devstack/sysgo/mixed_runtime.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@ import (
"encoding/json"
"fmt"
"os"
"os/exec"
"path/filepath"
"strings"

Expand Down Expand Up @@ -408,7 +407,6 @@ func buildMixedOpRethNode(
t.Require().NoError(os.WriteFile(chainConfigPath, data, 0o640), "must write genesis file")

dataDirPath := filepath.Join(tempDir, "data")
t.Require().NoError(os.MkdirAll(dataDirPath, 0o755), "must create datadir")

logDirPath := filepath.Join(tempDir, "logs")
t.Require().NoError(os.MkdirAll(logDirPath, 0o755), "must create logs dir")
Expand Down Expand Up @@ -467,26 +465,23 @@ func buildMixedOpRethNode(
args = append(args, "--metrics=127.0.0.1:0")
}

initArgs := []string{
dataDirs := []string{dataDirPath}
initCmds := [][]string{{
"init",
"--datadir=" + dataDirPath,
"--chain=" + chainConfigPath,
}
initOut, err := exec.Command(execPath, initArgs...).CombinedOutput()
t.Require().NoError(err, "must init op-reth node: %s", string(initOut))
}}

if !opRethCfg.DisableProofsHistory {
proofHistoryDir := filepath.Join(tempDir, "proof-history")

initProofsArgs := []string{
dataDirs = append(dataDirs, proofHistoryDir)
initCmds = append(initCmds, []string{
"proofs",
"init",
"--datadir=" + dataDirPath,
"--chain=" + chainConfigPath,
"--proofs-history.storage-path=" + proofHistoryDir,
}
initOut, initErr := exec.Command(execPath, initProofsArgs...).CombinedOutput()
t.Require().NoError(initErr, "must init op-reth proof history: %s", string(initOut))
})

args = append(
args,
Expand All @@ -498,7 +493,7 @@ func buildMixedOpRethNode(

args = append(args, opRethCfg.ExtraArgs...)

return &OpReth{
node := &OpReth{
name: key,
chainID: l2Net.ChainID(),
jwtPath: jwtPath,
Expand All @@ -508,9 +503,13 @@ func buildMixedOpRethNode(
execPath: execPath,
args: args,
env: []string{},
dataDirs: dataDirs,
initCmds: initCmds,
p: t,
l2MetricsRegistrar: metricsRegistrar,
}
node.initData()
return node
}

func startMixedOpRethNode(
Expand Down
Loading