Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1,67 @@
package op_reth_stale_forkchoice

import (
"testing"

"github.com/ethereum-optimism/optimism/op-devstack/devtest"
"github.com/ethereum-optimism/optimism/op-devstack/sysgo"
"github.com/ethereum-optimism/optimism/op-service/eth"
safety "github.com/ethereum-optimism/optimism/op-service/eth/safety"
)

const (
// finalityAttempts bounds, in 2s polls, how long the follower may take to finalize a block
// above genesis, which takes L1 finality: 20 L1 blocks of 6s.
finalityAttempts = 120
// preWipeChainBlocks is how tall the chain is at least when the follower's datadir is wiped,
// so syncing it again is a download of many blocks rather than of a handful.
preWipeChainBlocks = 60
// refinalizeAttempts bounds, in 2s polls, how long the follower may take to be finalized
// again where it was: after its reset a consensus client finalizes only blocks it derived
// since, so this can take about two L1 finality delays.
refinalizeAttempts = 210
// resyncFromScratchAttempts bounds, in 2s polls, how long the follower may take to sync the
// whole chain again from an empty datadir and to be safe where it was.
resyncFromScratchAttempts = 120
)

// TestFollowerResyncsFromScratchUnderConsensusClientThatKeptItsHeads wipes the follower's
// op-reth datadir while its consensus client keeps running, so the first forkchoice updates the
// empty node receives name a head, a safe block and a finalized block it has not downloaded, and
// requires the follower to sync the chain again from genesis: to hold the sequencer's unsafe
// head, and to be safe and finalized at least where it was before the wipe.
func TestFollowerResyncsFromScratchUnderConsensusClientThatKeptItsHeads(gt *testing.T) {
t := devtest.ParallelT(gt)
devtest.RunParallel(t, syncModeMatrix(), func(t devtest.T, c syncModeCase) {
skipUnsupportedCell(t, c)
sys := newStaleForkchoiceSystem(t, systemConfig{
persistenceThreshold: defaultPersistenceThreshold,
syncMode: c.syncMode,
})
sys.Follower.EL.Reached(eth.Unsafe, preWipeChainBlocks, chainBuildUpAttempts)
// The consensus client must hold a safe and a finalized block above genesis, which
// an empty datadir does not have.
sys.Follower.EL.Reached(eth.Finalized, 1, finalityAttempts)
before := readEngineLabels(sys.Follower.EL)

t.Logger().Info("Wiping the follower's datadir under its consensus client", "head", before.head, "safe", before.safe, "finalized", before.finalized)
sys.Follower.EL.Stop()
sys.Follower.EL.WipeDatadir()
sys.Follower.EL.Start()
sys.Follower.EL.WaitForOnline()
t.Require().Less(sys.Follower.EL.BlockRefByLabel(eth.Unsafe).Number, before.head.Number, "the wipe must drop the follower's chain")
sys.Follower.EL.PeerWith(sys.Sequencer.EL)

sys.Follower.EL.InSync(sys.Sequencer.EL, safety.LocalUnsafe, resyncFromScratchAttempts)
sys.Follower.EL.Reached(eth.Safe, before.safe.Number, resyncFromScratchAttempts)

sysgo.SkipOnKonaNode(t, "the follower is unsafe and safe where it was again, but it is not reliably finalized again "+
"(#24066): after the wipe its kona-node resets on `Consolidate(MissingUnsafeL2Block(N))` every few seconds, which "+
"clears its finalization queue each time, and in CI a finalize task hit `Finalize(BlockNotSafe)`, which kona-node "+
"treats as fatal (`Failed to start rollup node service: EngineTask(Finalize(BlockNotSafe))`), so the finalized wait "+
"failed with `operation failed permanently after 210 attempts: expected head for label=finalized to advance to "+
"target=59, but got current=0`")
sys.Follower.EL.Reached(eth.Finalized, before.finalized.Number, refinalizeAttempts)
sys.Follower.CL.InSync(sys.Sequencer.CL, safety.LocalSafe, resyncAttempts)
})
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,67 @@
package op_reth_stale_forkchoice

import (
"testing"

"github.com/ethereum-optimism/optimism/op-devstack/devtest"
"github.com/ethereum-optimism/optimism/op-devstack/sysgo"
"github.com/ethereum-optimism/optimism/op-service/eth"
safety "github.com/ethereum-optimism/optimism/op-service/eth/safety"
)

const (
// l1ReorgAttempts bounds, in 7s polls, how long the L1 node may take to switch to the block
// the test sequencer built on the reorged block's parent.
l1ReorgAttempts = 10
// postReorgSafeBlocks is how far past its pre-reorg height the follower's safe head must get
// again on the new L1 chain.
postReorgSafeBlocks = 5
// safeRecoveryAttempts bounds, in 2s polls, how long the follower may take after the L1 reorg
// to replace its safe block and to be safe past the pre-reorg height again: the batcher must
// notice the reorg and post again, and the follower's derivation must reset and catch up.
safeRecoveryAttempts = 90
)

// TestFollowerSafeHeadAdvancesAgainAfterL1ReorgRewindsIt reorgs out the L1 block the follower's
// safe head derives from while the follower's consensus client runs, so the client resets and
// sends forkchoice updates whose safe hash moves back below the unsafe chain its engine still
// holds, and requires the follower to replace the safe block that derived from the reorged L1
// block, to derive the new L1 chain past where its safe head was, and to be in sync with the
// sequencer again at the safe and unsafe heads.
func TestFollowerSafeHeadAdvancesAgainAfterL1ReorgRewindsIt(gt *testing.T) {
t := devtest.ParallelT(gt)
devtest.RunParallel(t, syncModeMatrix(), func(t devtest.T, c syncModeCase) {
skipUnsupportedCell(t, c)
sysgo.SkipOnKonaNode(t, "fails on develop (#18676): the kona-node sequencer resets its derivation on the L1 reorg "+
"(`Derivation pipeline is being reset: L1 reorg detected`) but keeps building on the unsafe chain whose L1 origin was "+
"reorged out, so nothing replaces that chain, every node rejects the old batches (`batch is for different L1 chain, "+
"epoch hash does not match`) and the follower's safe block is never replaced: `operation failed permanently after 90 "+
"attempts: expected head to reorg 0x3f26…d122:20, but got 0x3f26…d122:20`; the L1-reorg tests in sync/elsync/reorg "+
"skip on kona-node for the same reason (#22275)")
sys := newStaleForkchoiceSystem(t, systemConfig{
persistenceThreshold: defaultPersistenceThreshold,
syncMode: c.syncMode,
withTestSequencer: true,
})
l1ChainID := sys.L1Network.ChainID()
sys.L1Network.WaitForBlock()
// Only the test sequencer builds L1 from here on, so the reorg below is the only fork.
sys.L1CL.Stop()
startL1 := sys.L1EL.BlockRefByLabel(eth.Unsafe)
safeBefore := sys.TestSequencer.BuildL1UntilSafeOriginPast(t, l1ChainID, sys.Follower.EL, startL1.Number)

l1Origin := sys.L1EL.BlockRefByNumber(safeBefore.L1Origin.Number)
t.Logger().Info("Reorging out the L1 block the follower's safe head derives from", "safe", safeBefore, "l1_origin", l1Origin)
sys.TestSequencer.SequenceBlock(t, l1ChainID, l1Origin.ParentHash)
sys.L1CL.Start()
sys.L1EL.ReorgTriggered(l1Origin, l1ReorgAttempts)
l1After := sys.L1EL.BlockRefByNumber(l1Origin.Number)
t.Logger().Info("L1 reorged", "before", l1Origin, "after", l1After)

sys.Follower.EL.ReorgTriggered(safeBefore, safeRecoveryAttempts)
sys.Follower.EL.WaitL1OriginHash(eth.Safe, l1After.ID(), safeRecoveryAttempts)
sys.Follower.EL.Reached(eth.Safe, safeBefore.Number+postReorgSafeBlocks, safeRecoveryAttempts)
sys.Follower.EL.InSync(sys.Sequencer.EL, safety.LocalSafe, resyncAttempts)
sys.Follower.EL.InSync(sys.Sequencer.EL, safety.LocalUnsafe, resyncAttempts)
})
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,150 @@
package op_reth_stale_forkchoice

import (
"fmt"
"testing"
"time"

"github.com/ethereum-optimism/optimism/op-devstack/devtest"
"github.com/ethereum-optimism/optimism/op-devstack/dsl"
"github.com/ethereum-optimism/optimism/op-devstack/sysgo"
"github.com/ethereum-optimism/optimism/op-node/rollup/derive"
nodeSync "github.com/ethereum-optimism/optimism/op-node/rollup/sync"
"github.com/ethereum-optimism/optimism/op-service/eth"
"github.com/ethereum/go-ethereum/common"
)

const (
// replacedPersistedTimeout bounds how long the frozen follower may take to report the replaced
// block persisted: it already holds every block it needs, so reth only finishes the saves that
// are due.
replacedPersistedTimeout = 30 * time.Second
// syncBackAttempts bounds, in 2s polls, how long the follower may take to download the head
// of a forkchoice update it answered SYNCING from the sequencer and to make it canonical.
syncBackAttempts = 30
)

// freezeFollower stops the batcher, so nothing above the sequencer's head at that moment can
// become safe, lets the follower get blocksAboveSafe blocks past that head, and stops its
// consensus client, so the follower's engine answers only the forkchoice updates the test sends.
// It returns the follower's frozen head.
func freezeFollower(t devtest.T, sys *staleForkchoiceSystem, blocksAboveSafe uint64) eth.L2BlockRef {
sys.Batcher.Stop()
safeBound := sys.Sequencer.EL.BlockRefByLabel(eth.Unsafe)
sys.Follower.EL.Reached(eth.Unsafe, safeBound.Number+blocksAboveSafe, chainBuildUpAttempts)
sys.Follower.CL.Stop()
head := sys.Follower.EL.BlockRefByLabel(eth.Unsafe)
t.Logger().Info("Follower frozen", "head", head, "safe", sys.Follower.EL.BlockRefByLabel(eth.Safe), "safe_bound", safeBound)
return head
}

// TestForkchoiceUpdateNamingReorgedOutFinalizedIsRejected builds a sibling of the follower's
// head and sends a forkchoice update that names the head the sibling replaces as finalized, and
// requires the engine to refuse it with the engine API's -38002 "Invalid forkchoice state" error
// and to leave its head, safe and finalized blocks as they were: first while the update is the one
// that would reorg to the sibling, then again once the sibling is the head.
func TestForkchoiceUpdateNamingReorgedOutFinalizedIsRejected(gt *testing.T) {
t := devtest.ParallelT(gt)
skipUnsupported(t)
sys := newStaleForkchoiceSystem(t, systemConfig{
persistenceThreshold: defaultPersistenceThreshold,
syncMode: nodeSync.CLSync,
})
replaced := freezeFollower(t, sys, 1)
sibling := sys.Follower.EL.BuildSiblingBlock(replaced.Number)
siblingHash := sibling.ExecutionPayload.BlockHash
sys.Follower.EL.NewPayloadRaw(sibling).IsValid()
labels := readEngineLabels(sys.Follower.EL)
t.Require().Less(labels.safe.Number, replaced.Number, "the safe head must be below the replaced block")
requireReplacedRefusedAsFinalized(t, sys.Follower.EL, siblingHash, replaced, labels)

sys.Follower.EL.ForkchoiceUpdateRaw(siblingHash, labels.safe.Hash, labels.finalized.Hash, nil).IsValid()
labels = readEngineLabels(sys.Follower.EL)
t.Require().Equal(siblingHash, labels.head.Hash, "the sibling must be the head")
requireReplacedRefusedAsFinalized(t, sys.Follower.EL, siblingHash, replaced, labels)
}

// requireReplacedRefusedAsFinalized sends a forkchoice update to the sibling that names the block
// it replaced as finalized, and requires the engine to refuse it and to keep labels.
func requireReplacedRefusedAsFinalized(t devtest.T, el *dsl.L2ELNode, siblingHash common.Hash, replaced eth.L2BlockRef, labels engineLabels) {
t.Logger().Info("Naming the replaced block as finalized", "head", labels.head, "safe", labels.safe, "finalized", labels.finalized, "replaced", replaced, "sibling", siblingHash)
el.ForkchoiceUpdateRaw(siblingHash, labels.safe.Hash, replaced.Hash, nil).IsInvalidForkchoiceState()
t.Require().Equal(labels, readEngineLabels(el), "a refused forkchoice update must leave the head, safe and finalized blocks as they were")
}

// persistedHeadCase is one cell of the matrix the follower is reorged below its in-memory
// window under.
type persistedHeadCase struct {
persistenceThreshold uint64
}

func (c persistedHeadCase) String() string {
return fmt.Sprintf("threshold_%d", c.persistenceThreshold)
}

// forkDepth is how far below the frozen head the replaced block sits. Reth persists once more
// than the threshold of blocks sit in memory and keeps the newest of them up to the memory block
// buffer target, so a block this deep is due for persistence without further blocks arriving.
func (c persistedHeadCase) forkDepth() uint64 {
return c.persistenceThreshold + sysgo.OpRethPinnedMemoryBlockBufferTarget + 2
}

func persistedHeadMatrix() []persistedHeadCase {
return []persistedHeadCase{{persistenceThreshold: 7}, {persistenceThreshold: 50}}
}

// TestForkchoiceUpdateToPersistedReorgedOutHeadIsAppliedOrSynced waits until the follower
// reports a block persisted, reorgs it to a sibling of that block, and sends a forkchoice update
// back to the replaced block as soon as the reorg is answered. The replaced block is not
// canonical, so the engine must not take it for the head it already has: it either reorgs back to
// it and answers VALID, or answers SYNCING and the follower downloads it from the sequencer, which
// still holds it. Either way the follower ends on the replaced block.
func TestForkchoiceUpdateToPersistedReorgedOutHeadIsAppliedOrSynced(gt *testing.T) {
t := devtest.ParallelT(gt)
devtest.RunParallel(t, persistedHeadMatrix(), func(t devtest.T, c persistedHeadCase) {
skipUnsupported(t)
sys := newStaleForkchoiceSystem(t, systemConfig{
persistenceThreshold: c.persistenceThreshold,
syncMode: nodeSync.CLSync,
})
persistedBlocks := sys.Follower.EL.TrackPersistedBlock()
head := freezeFollower(t, sys, c.forkDepth()+1)
replaced := sys.Follower.EL.BlockRefByNumber(head.Number - c.forkDepth())
persistedBlocks.AwaitPersisted(replaced.Number, replacedPersistedTimeout)
sibling := sys.Follower.EL.BuildSiblingBlock(replaced.Number)
siblingHash := sibling.ExecutionPayload.BlockHash
sys.Follower.EL.NewPayloadRaw(sibling).IsValid()
labels := readEngineLabels(sys.Follower.EL)
t.Require().Less(labels.safe.Number, replaced.Number, "the safe head must be below the replaced block")

t.Logger().Info("Reorging the follower below its persisted block", "head", head, "replaced", replaced, "sibling", siblingHash, "safe", labels.safe, "finalized", labels.finalized)
reorged := sys.Follower.EL.ForkchoiceUpdateRaw(siblingHash, labels.safe.Hash, labels.finalized.Hash, nil).IsValid()
t.Require().NotNil(reorged.Result.PayloadStatus.LatestValidHash, "a VALID answer to the reorg must name its latest valid block")
t.Require().Equal(siblingHash, *reorged.Result.PayloadStatus.LatestValidHash, "the engine must have made the sibling the head")
// Reth removes the replaced blocks from disk on its persistence thread after answering the
// reorg and offers no way to hold that off. Only a VALID answer, which needs the update back
// to arrive before that removal commits, checks that a persisted header counts as canonical
// only when it matches the canonical hash (reth#27429); after the removal reth answers
// SYNCING, which this test accepts, so a run can pass without exercising that check. No RPC
// runs between the two updates, to keep the VALID window as wide as it can be.
answer := sys.Follower.EL.ForkchoiceUpdateRaw(replaced.Hash, labels.safe.Hash, labels.finalized.Hash, nil)
t.Require().NoError(answer.Err, "the update back to the replaced block must be answered")
status := answer.Result.PayloadStatus.Status
headAfter := sys.Follower.EL.BlockRefByLabel(eth.Unsafe)
t.Logger().Info("Engine answered the update back to the replaced block", "status", status,
"latest_valid_hash", answer.Result.PayloadStatus.LatestValidHash, "head", headAfter)
t.Require().Contains([]eth.ExecutePayloadStatus{eth.ExecutionValid, eth.ExecutionSyncing}, status,
"the replaced block is valid, so the engine either applies the update or syncs to it")
if status == eth.ExecutionValid {
t.Logger().Info("Engine reorged back to the persisted block itself, which exercises reth#27429")
t.Require().Equal(replaced, headAfter, "a VALID answer must mean the head is the requested block")
} else {
t.Logger().Info("Engine answered SYNCING, so this run does not exercise reth#27429; the follower downloads the replaced block from the sequencer")
}
siblingRef, err := derive.PayloadToBlockRef(sys.L2Network.Escape().RollupConfig(), sibling.ExecutionPayload)
t.Require().NoError(err, "the sibling must carry an L1 info deposit")
// A SYNCING answer makes the replaced block a sync target the follower downloads from the sequencer.
sys.Follower.EL.ReorgExact(siblingRef, syncBackAttempts)
t.Require().Equal(replaced, sys.Follower.EL.BlockRefByLabel(eth.Unsafe), "the follower must be back on the replaced block")
})
}
Loading
Loading