From ec0255719a60f4741186cd3cd095cc166b03a87c Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Fri, 6 Nov 2026 20:36:50 -0800 Subject: [PATCH 1/6] op-devstack,op-acceptance-tests: run the follow-source tests against kona-node The follow_l2 tests skipped on kona-node because the single-chain test sequencer requires an op-node CL, and the two-verifiers preset applied DEVSTACK_L2CL_KIND to every node. The preset now always runs the sequencer and the deriving verifier "b" on op-node, and only the follow-source verifier "c" takes the env-selected CL. Under kona-node that checks kona's follow-source mode against an op-node upstream; under op-node every node keeps its CL. The preset also no longer starts a proposer or challenger, which no follow_l2 test uses, and accepts only the options that runtime honors. The single-chain CL starters take the CL kind as an argument instead of reading DEVSTACK_L2CL_KIND, each preset resolves it once, and an unknown kind, the empty string included, fails instead of running op-node. Presets with interop at genesis pin their sequencer to op-node, which they already required. Co-Authored-By: Claude Opus 5.5 Co-Authored-By: Claude Fable 5.1 --- .../tests/sync/follow_l2/sync_test.go | 42 +---------- op-devstack/presets/option_validation.go | 9 +++ .../presets/singlechain_twoverifiers.go | 2 +- op-devstack/sysgo/mixed_runtime.go | 2 +- op-devstack/sysgo/singlechain_build.go | 38 ++++------ op-devstack/sysgo/singlechain_build_test.go | 28 +++++++- op-devstack/sysgo/singlechain_interop.go | 14 +++- op-devstack/sysgo/singlechain_runtime.go | 61 +++++++++++++--- op-devstack/sysgo/singlechain_variants.go | 69 ++++++++++++++----- .../sysgo/singlechain_variants_test.go | 31 +++++++++ 10 files changed, 200 insertions(+), 96 deletions(-) create mode 100644 op-devstack/sysgo/singlechain_variants_test.go diff --git a/op-acceptance-tests/tests/sync/follow_l2/sync_test.go b/op-acceptance-tests/tests/sync/follow_l2/sync_test.go index 5a6babfbaad..a779a90df55 100644 --- a/op-acceptance-tests/tests/sync/follow_l2/sync_test.go +++ b/op-acceptance-tests/tests/sync/follow_l2/sync_test.go @@ -8,7 +8,6 @@ import ( "github.com/ethereum-optimism/optimism/op-devstack/devtest" "github.com/ethereum-optimism/optimism/op-devstack/dsl" "github.com/ethereum-optimism/optimism/op-devstack/presets" - "github.com/ethereum-optimism/optimism/op-devstack/sysgo" "github.com/ethereum-optimism/optimism/op-service/eth" safety "github.com/ethereum-optimism/optimism/op-service/eth/safety" @@ -17,18 +16,6 @@ import ( func TestFollowL2_Safe_Finalized_CurrentL1(gt *testing.T) { t := devtest.ParallelT(gt) - // Example error with kona-node: - // - // assertions.go:387: ERROR[03-31|11:33:11.255] - // assertions.go:387: Error Trace: /optimism/op-devstack/sysgo/singlechain_variants.go:143 - // assertions.go:387: /optimism/op-devstack/sysgo/singlechain_variants.go:53 - // assertions.go:387: /optimism/op-devstack/presets/singlechain_twoverifiers.go:24 - // assertions.go:387: /optimism/op-acceptance-tests/tests/sync/follow_l2/setup_test.go:24 - // assertions.go:387: /optimism/op-acceptance-tests/tests/sync/follow_l2/sync_test.go:18 - // assertions.go:387: Error: Should be true - // assertions.go:387: Test: TestFollowL2_Safe_Finalized_CurrentL1 - // assertions.go:387: Messages: single-chain test sequencer requires an op-node CL node - sysgo.SkipOnKonaNode(t, "not supported") sys := newSingleChainTwoVerifiersFollowL2(t) logger := t.Logger() @@ -36,9 +23,8 @@ func TestFollowL2_Safe_Finalized_CurrentL1(gt *testing.T) { attempts := 70 target := uint64(3) - // L2CL is the sequencer with CL follow source, derivation disabled - // L2CLB is the verifier without follow source, derivation enabled - // L2CLC is the verifier with CL follow source, derivation disabled + // L2CL is the sequencer and L2CLB the deriving verifier, both op-node by the preset. + // L2CLC follows L2CLB with derivation disabled and runs the CL kind DEVSTACK_L2CL_KIND selects. // All verifiers must eventually advance unsafe, safe, finalized checkMatchedAll := func(lvl safety.Level) { dsl.CheckAll(t, @@ -71,18 +57,6 @@ func TestFollowL2_Safe_Finalized_CurrentL1(gt *testing.T) { func TestFollowL2_ReorgRecovery(gt *testing.T) { t := devtest.ParallelT(gt) - // Example error with kona-node: - // - // assertions.go:387: ERROR[03-31|11:31:11.567] - // assertions.go:387: Error Trace: /optimism/op-devstack/sysgo/singlechain_variants.go:143 - // assertions.go:387: /optimism/op-devstack/sysgo/singlechain_variants.go:53 - // assertions.go:387: /optimism/op-devstack/presets/singlechain_twoverifiers.go:24 - // assertions.go:387: /optimism/op-acceptance-tests/tests/sync/follow_l2/setup_test.go:24 - // assertions.go:387: /optimism/op-acceptance-tests/tests/sync/follow_l2/sync_test.go:60 - // assertions.go:387: Error: Should be true - // assertions.go:387: Test: TestFollowL2_ReorgRecovery - // assertions.go:387: Messages: single-chain test sequencer requires an op-node CL node - sysgo.SkipOnKonaNode(t, "not supported") sys := newSingleChainTwoVerifiersFollowL2(t) require := t.Require() logger := t.Logger() @@ -170,18 +144,6 @@ func TestFollowL2_ReorgRecovery(gt *testing.T) { func TestFollowL2_WithoutCLP2P(gt *testing.T) { t := devtest.ParallelT(gt) - // Example error with kona-node: - // - // assertions.go:387: ERROR[03-31|11:27:57.797] - // assertions.go:387: Error Trace: /optimism/op-devstack/sysgo/singlechain_variants.go:143 - // assertions.go:387: /optimism/op-devstack/sysgo/singlechain_variants.go:53 - // assertions.go:387: /optimism/op-devstack/presets/singlechain_twoverifiers.go:24 - // assertions.go:387: /optimism/op-acceptance-tests/tests/sync/follow_l2/setup_test.go:24 - // assertions.go:387: /optimism/op-acceptance-tests/tests/sync/follow_l2/sync_test.go:136 - // assertions.go:387: Error: Should be true - // assertions.go:387: Test: TestFollowL2_WithoutCLP2P - // assertions.go:387: Messages: single-chain test sequencer requires an op-node CL nod - sysgo.SkipOnKonaNode(t, "not supported") // Without CL P2P, L2CLC learns of new blocks only from its follow source, polled every two // L2 blocks. When the source's local-safe head is past L2CLC's unsafe head, op-node has its // EL sync to it, gets SYNCING back and resets local-safe to the finalized head. It recovers diff --git a/op-devstack/presets/option_validation.go b/op-devstack/presets/option_validation.go index 0ba613ce813..4a0357b38c3 100644 --- a/op-devstack/presets/option_validation.go +++ b/op-devstack/presets/option_validation.go @@ -164,6 +164,15 @@ const minimalPresetSupportedOptionKinds = optionKindDeployer | optionKindProofValidation | optionKindSkipHonestProposer +// The two-verifiers preset starts no proposer or challenger, so it takes no fault-proof options. +const singleChainTwoVerifiersPresetSupportedOptionKinds = optionKindDeployer | + optionKindBatcher | + optionKindOpReth | + optionKindGlobalL2CL | + optionKindL1EL | + optionKindTimeTravel | + optionKindAfterBuild + // minimalNoFaultProofsPresetSupportedOptionKinds adds alt-DA, which only this preset supports (see WithAltDA). const minimalNoFaultProofsPresetSupportedOptionKinds = minimalPresetSupportedOptionKinds | optionKindAltDA diff --git a/op-devstack/presets/singlechain_twoverifiers.go b/op-devstack/presets/singlechain_twoverifiers.go index 4de57d3c2b9..0047044a7f8 100644 --- a/op-devstack/presets/singlechain_twoverifiers.go +++ b/op-devstack/presets/singlechain_twoverifiers.go @@ -20,7 +20,7 @@ type SingleChainTwoVerifiers struct { // // The target is created from the runtime plus any additional preset options. func NewSingleChainTwoVerifiersWithoutCheck(t devtest.T, opts ...Option) *SingleChainTwoVerifiers { - presetCfg, presetOpts := collectSupportedPresetConfig(t, "NewSingleChainTwoVerifiersWithoutCheck", opts, minimalPresetSupportedOptionKinds) + presetCfg, presetOpts := collectSupportedPresetConfig(t, "NewSingleChainTwoVerifiersWithoutCheck", opts, singleChainTwoVerifiersPresetSupportedOptionKinds) out := singleChainTwoVerifiersFromRuntime(t, sysgo.NewSingleChainTwoVerifiersRuntimeWithConfig(t, presetCfg)) presetOpts.applyPreset(out) return out diff --git a/op-devstack/sysgo/mixed_runtime.go b/op-devstack/sysgo/mixed_runtime.go index 8a792ecfb40..61b55027d91 100644 --- a/op-devstack/sysgo/mixed_runtime.go +++ b/op-devstack/sysgo/mixed_runtime.go @@ -72,7 +72,7 @@ func devstackL2CLKind() MixedL2CLKind { // ResolveMixedL2CLKind returns the L2 CL kind requested via DEVSTACK_L2CL_KIND, // defaulting to op-node when the variable is unset. Tests that build explicit // MixedSingleChainNodeSpec lists use this so they honor the same env-driven CL -// selection as the higher-level single-chain presets (see startL2CLForKey). +// selection as the higher-level single-chain presets. func ResolveMixedL2CLKind() MixedL2CLKind { if kind := devstackL2CLKind(); kind != "" { return kind diff --git a/op-devstack/sysgo/singlechain_build.go b/op-devstack/sysgo/singlechain_build.go index 616d1fed364..9e2f58c0e7b 100644 --- a/op-devstack/sysgo/singlechain_build.go +++ b/op-devstack/sysgo/singlechain_build.go @@ -91,11 +91,7 @@ func applyConfigPrefundedL2(t devtest.T, keys devkeys.Keys, l1ChainID, l2ChainID l1Config.WithPrefundedAccount(addrFor(devkeys.SystemConfigOwner), *millionEth) } -// startL2CLForKey starts an L2 CL node for the given key, respecting DEVSTACK_L2CL_KIND. -// This is the env-aware dispatch point for preset runtimes that don't build explicit node -// specs. Runtimes constructed from explicit MixedSingleChainNodeSpec.CLKind values (e.g. -// NewMixedSingleChainRuntime) don't route through here; they resolve the env up front via -// ResolveMixedL2CLKind instead. +// startL2CLForKey starts an L2 CL node of the given kind for the given key. func startL2CLForKey( t devtest.T, keys devkeys.Keys, @@ -105,17 +101,18 @@ func startL2CLForKey( l1CL *L1CLNode, l2EL L2ELNode, jwtSecret [32]byte, + clKind MixedL2CLKind, clKey, elKey string, isSequencer bool, followSource string, l2CLOpts []L2CLOption, ) L2CLNode { - switch devstackL2CLKind() { + switch clKind { case MixedL2CLKona: return startMixedKonaNode(t, keys, l1Net, l2Net, l1EL, l1CL, l2EL, clKey, elKey, konaNodeStartConfig{ L2CLConfig: resolveKonaL2CLConfig(t, clKey, l2Net.ChainID(), isSequencer, followSource, l2CLOpts), }) - default: // op-node + case MixedL2CLOpNode: return startL2CLNode(t, keys, l1Net, l2Net, l1EL, l1CL, l2EL, jwtSecret, l2CLNodeStartConfig{ Key: clKey, IsSequencer: isSequencer, @@ -123,10 +120,13 @@ func startL2CLForKey( L2FollowSource: followSource, L2CLOptions: l2CLOpts, }) + default: + t.Require().FailNow("unsupported L2 CL kind", "unknown L2 CL kind %q", clKind) + return nil // unreachable } } -// startStoppedSequencerCL starts the env-selected sequencer CL in the stopped +// startStoppedSequencerCL starts a sequencer CL of the given kind in the stopped // state required while a conductor cluster is being assembled. func startStoppedSequencerCL( t devtest.T, @@ -137,10 +137,11 @@ func startStoppedSequencerCL( l1CL *L1CLNode, l2EL L2ELNode, jwtSecret [32]byte, + clKind MixedL2CLKind, key string, l2CLOpts []L2CLOption, ) L2CLNode { - switch devstackL2CLKind() { + switch clKind { case MixedL2CLKona: return startMixedKonaNode( t, keys, l1Net, l2Net, l1EL, l1CL, l2EL, key, key, konaNodeStartConfig{ @@ -148,7 +149,7 @@ func startStoppedSequencerCL( SequencerStopped: true, }, ) - default: // op-node + case MixedL2CLOpNode: return startL2CLNode(t, keys, l1Net, l2Net, l1EL, l1CL, l2EL, jwtSecret, l2CLNodeStartConfig{ Key: key, IsSequencer: true, @@ -156,6 +157,9 @@ func startStoppedSequencerCL( L2CLOptions: l2CLOpts, SequencerStopped: true, }) + default: + t.Require().FailNow("unsupported L2 CL kind", "unknown L2 CL kind %q", clKind) + return nil // unreachable } } @@ -201,20 +205,6 @@ func connectL2CLPeers(t devtest.T, logger log.Logger, l2CL1, l2CL2 L2CLNode) { require.True(ok2, "peer register invalid (cl2 missing cl1)") } -func startSequencerCL( - t devtest.T, - keys devkeys.Keys, - l1Net *L1Network, - l2Net *L2Network, - l1EL L1ELNode, - l1CL *L1CLNode, - l2EL L2ELNode, - jwtSecret [32]byte, - l2CLOpts []L2CLOption, -) L2CLNode { - return startL2CLForKey(t, keys, l1Net, l2Net, l1EL, l1CL, l2EL, jwtSecret, "sequencer", "sequencer", true, "", l2CLOpts) -} - type l2CLNodeStartConfig struct { Key string IsSequencer bool diff --git a/op-devstack/sysgo/singlechain_build_test.go b/op-devstack/sysgo/singlechain_build_test.go index 07dada44d10..e69392ec74d 100644 --- a/op-devstack/sysgo/singlechain_build_test.go +++ b/op-devstack/sysgo/singlechain_build_test.go @@ -12,7 +12,8 @@ import ( ) func TestKonaOptionsRejectedBeforeStartup(t *testing.T) { - t.Setenv("DEVSTACK_L2CL_KIND", string(MixedL2CLKona)) + // The env asks for op-node, so a kona-node rejection shows the explicit kind picks the CL. + t.Setenv("DEVSTACK_L2CL_KIND", string(MixedL2CLOpNode)) t.Setenv("KONA_NODE_P2P_NO_DISCOVERY", "true") for _, stopped := range []bool{false, true} { t.Run(fmt.Sprintf("stopped=%t", stopped), func(t *testing.T) { @@ -24,12 +25,33 @@ func TestKonaOptionsRejectedBeforeStartup(t *testing.T) { // Nil startup dependencies ensure validation runs before starting any process or RPC. require.PanicsWithValue(t, "Kona option rejected", func() { if stopped { - startStoppedSequencerCL(dt, nil, nil, network, nil, nil, nil, [32]byte{}, "node", opts) + startStoppedSequencerCL(dt, nil, nil, network, nil, nil, nil, [32]byte{}, MixedL2CLKona, "node", opts) } else { - startL2CLForKey(dt, nil, nil, network, nil, nil, nil, [32]byte{}, "node", "node", true, "", opts) + startL2CLForKey(dt, nil, nil, network, nil, nil, nil, [32]byte{}, MixedL2CLKona, "node", "node", true, "", opts) } }) require.Contains(t, dt.failure, "unsupported L2CL option for kona-node") }) } } + +// A kind the starters do not know, including the empty string, fails before anything starts +// instead of silently running op-node. +func TestUnknownL2CLKindRejectedBeforeStartup(t *testing.T) { + for _, kind := range []MixedL2CLKind{"", "kona", "Kona-Node"} { + for _, stopped := range []bool{false, true} { + t.Run(fmt.Sprintf("kind=%q/stopped=%t", kind, stopped), func(t *testing.T) { + dt := &rejectedKonaOptionTest{T: devtest.SerialT(t)} + network := &L2Network{chainID: eth.ChainIDFromUInt64(901)} + require.PanicsWithValue(t, "Kona option rejected", func() { + if stopped { + startStoppedSequencerCL(dt, nil, nil, network, nil, nil, nil, [32]byte{}, kind, "node", nil) + } else { + startL2CLForKey(dt, nil, nil, network, nil, nil, nil, [32]byte{}, kind, "node", "node", true, "", nil) + } + }) + require.Contains(t, dt.failure, "unknown L2 CL kind") + }) + } + } +} diff --git a/op-devstack/sysgo/singlechain_interop.go b/op-devstack/sysgo/singlechain_interop.go index c3bad3d2f2b..80ba236837d 100644 --- a/op-devstack/sysgo/singlechain_interop.go +++ b/op-devstack/sysgo/singlechain_interop.go @@ -31,8 +31,10 @@ func startSingleChainInteropPrimaryNoSupernode( jwtPath string, jwtSecret [32]byte, cfg PresetConfig, + sequencerCLKind MixedL2CLKind, ) singleChainPrimaryRuntime { t.Require().NotNil(world.Interop, "single-chain interop runtime requires interop support") + requireOpNodeSequencerCLKind(t, sequencerCLKind) l2EL := startSequencerEL(t, world.L2Network, jwtPath, jwtSecret, ResolveMixedL2ELOpts(t)...) l2CL := startL2CLNode(t, keys, world.L1Network, world.L2Network, l1EL, l1CL, l2EL, jwtSecret, l2CLNodeStartConfig{ @@ -54,10 +56,12 @@ func startSingleChainInteropPrimaryNoSupernode( func NewMinimalInteropNoSupernodeRuntime(t devtest.T) *SingleChainRuntime { return newSingleChainRuntimeWithConfig(t, PresetConfig{}, singleChainRuntimeSpec{ BuildWorld: newSingleChainInteropWorldNoSupernode, + SequencerCLKind: MixedL2CLOpNode, StartPrimary: startSingleChainInteropPrimaryNoSupernode, + StartBatcher: true, - StartProposer: true, StartChallenger: false, + StartProposer: true, }) } @@ -70,10 +74,12 @@ func NewSingleChainInteropNoSupernodeSuperRootRuntimeWithConfig(t devtest.T, cfg cfg.AddedGameTypes = append(cfg.AddedGameTypes, gameTypes.SuperCannonKonaGameType) return newSingleChainRuntimeWithConfig(t, cfg, singleChainRuntimeSpec{ BuildWorld: newSingleChainInteropWorldNoSupernode, + SequencerCLKind: MixedL2CLOpNode, StartPrimary: startDefaultSingleChainPrimary, + StartBatcher: true, - StartProposer: false, StartChallenger: true, + StartProposer: false, }) } @@ -88,9 +94,11 @@ func NewSingleChainInteropNoSupernodeZKDisputeRuntimeWithConfig(t devtest.T, cfg cfg.AddedGameTypes = append(cfg.AddedGameTypes, gameTypes.ZKDisputeGameType) return newSingleChainRuntimeWithConfig(t, cfg, singleChainRuntimeSpec{ BuildWorld: newSingleChainInteropWorldNoSupernode, + SequencerCLKind: MixedL2CLOpNode, StartPrimary: startDefaultSingleChainPrimary, + StartBatcher: true, - StartProposer: false, StartChallenger: true, + StartProposer: false, }) } diff --git a/op-devstack/sysgo/singlechain_runtime.go b/op-devstack/sysgo/singlechain_runtime.go index de6501560fa..98487eeac8b 100644 --- a/op-devstack/sysgo/singlechain_runtime.go +++ b/op-devstack/sysgo/singlechain_runtime.go @@ -45,13 +45,43 @@ type singleChainPrimaryRuntime struct { CL L2CLNode } +type singleChainPrimaryStartFunc func( + t devtest.T, + keys devkeys.Keys, + world singleChainRuntimeWorld, + l1EL *L1Geth, + l1CL *L1CLNode, + jwtPath string, + jwtSecret [32]byte, + cfg PresetConfig, + sequencerCLKind MixedL2CLKind, +) singleChainPrimaryRuntime + type singleChainRuntimeSpec struct { BuildWorld func(t devtest.T, keys devkeys.Keys, cfg PresetConfig, startL1 func(*L1Network) (*L1Geth, *L1CLNode)) singleChainRuntimeWorld - StartPrimary func(t devtest.T, keys devkeys.Keys, world singleChainRuntimeWorld, l1EL *L1Geth, l1CL *L1CLNode, jwtPath string, jwtSecret [32]byte, cfg PresetConfig) singleChainPrimaryRuntime + SequencerCLKind MixedL2CLKind + StartPrimary singleChainPrimaryStartFunc + StartBatcher bool - StartProposer bool StartChallenger bool - TestSequencer string + StartProposer bool + + TestSequencer string +} + +// requireOpNodeSequencerCLKind fails unless the sequencer runs on op-node, which the single-chain +// interop sequencer requires. +func requireOpNodeSequencerCLKind(t devtest.T, sequencerCLKind MixedL2CLKind) { + t.Require().Equal(MixedL2CLOpNode, sequencerCLKind, "single-chain interop sequencer requires op-node") +} + +// resolveSequencerCLKind returns the L2 CL kind the environment requests for the sequencer, +// pinned to op-node when the chain runs interop at genesis. +func resolveSequencerCLKind(cfg PresetConfig) MixedL2CLKind { + if cfg.InteropAtGenesis { + return MixedL2CLOpNode + } + return ResolveMixedL2CLKind() } func newSingleChainNodeRuntime(name string, isSequencer bool, el L2ELNode, cl L2CLNode) *SingleChainNodeRuntime { @@ -94,6 +124,7 @@ func startDefaultSingleChainPrimary( jwtPath string, jwtSecret [32]byte, cfg PresetConfig, + sequencerCLKind MixedL2CLKind, ) singleChainPrimaryRuntime { safeDBPath := filepath.Join(t.TempDirWithPrefix("l2-safe-db-"+world.L2Network.ChainID().String()), "safe-head.db") l2CLOptions := []L2CLOption{L2CLOptionFn(func(_ devtest.T, _ ComponentTarget, cfg *L2CLConfig) { @@ -104,6 +135,7 @@ func startDefaultSingleChainPrimary( sequencerELOpts := append(append([]OpRethOption{}, ResolveMixedL2ELOpts(t)...), cfg.OpRethOptions...) l2EL := startSequencerEL(t, world.L2Network, jwtPath, jwtSecret, sequencerELOpts...) if world.Interop != nil { + requireOpNodeSequencerCLKind(t, sequencerCLKind) l2CL := startL2CLNode(t, keys, world.L1Network, world.L2Network, l1EL, l1CL, l2EL, jwtSecret, l2CLNodeStartConfig{ Key: "sequencer", IsSequencer: true, @@ -113,7 +145,7 @@ func startDefaultSingleChainPrimary( }) return singleChainPrimaryRuntime{EL: l2EL, CL: l2CL} } - l2CL := startSequencerCL(t, keys, world.L1Network, world.L2Network, l1EL, l1CL, l2EL, jwtSecret, l2CLOptions) + l2CL := startL2CLForKey(t, keys, world.L1Network, world.L2Network, l1EL, l1CL, l2EL, jwtSecret, sequencerCLKind, "sequencer", "sequencer", true, "", l2CLOptions) return singleChainPrimaryRuntime{ EL: l2EL, CL: l2CL, @@ -132,6 +164,7 @@ func startStoppedSingleChainPrimary( jwtPath string, jwtSecret [32]byte, cfg PresetConfig, + sequencerCLKind MixedL2CLKind, ) singleChainPrimaryRuntime { t.Require().Nil(world.Interop, "conductor primary does not support interop") safeDBPath := filepath.Join(t.TempDirWithPrefix("l2-safe-db-"+world.L2Network.ChainID().String()), "safe-head.db") @@ -144,7 +177,7 @@ func startStoppedSingleChainPrimary( l2EL := startSequencerEL(t, world.L2Network, jwtPath, jwtSecret, sequencerELOpts...) l2CL := startStoppedSequencerCL( t, keys, world.L1Network, world.L2Network, l1EL, l1CL, l2EL, jwtSecret, - "sequencer", l2CLOptions, + sequencerCLKind, "sequencer", l2CLOptions, ) return singleChainPrimaryRuntime{EL: l2EL, CL: l2CL} } @@ -174,7 +207,7 @@ func newSingleChainRuntimeWithConfig(t devtest.T, cfg PresetConfig, spec singleC return l1EL, l1CL }) - primary := spec.StartPrimary(t, keys, world, l1EL, l1CL, jwtPath, jwtSecret, cfg) + primary := spec.StartPrimary(t, keys, world, l1EL, l1CL, jwtPath, jwtSecret, cfg, spec.SequencerCLKind) primaryNode := newSingleChainNodeRuntime("sequencer", true, primary.EL, primary.CL) var l2Batcher *L2Batcher @@ -239,12 +272,18 @@ func NewMinimalRuntime(t devtest.T) *SingleChainRuntime { } func NewMinimalRuntimeWithConfig(t devtest.T, cfg PresetConfig) *SingleChainRuntime { + return newMinimalRuntime(t, cfg, resolveSequencerCLKind(cfg)) +} + +func newMinimalRuntime(t devtest.T, cfg PresetConfig, sequencerCLKind MixedL2CLKind) *SingleChainRuntime { return newSingleChainRuntimeWithConfig(t, cfg, singleChainRuntimeSpec{ BuildWorld: newDefaultSingleChainWorld, + SequencerCLKind: sequencerCLKind, StartPrimary: startDefaultSingleChainPrimary, + StartBatcher: true, - StartProposer: true, StartChallenger: true, + StartProposer: true, }) } @@ -254,12 +293,18 @@ func NewMinimalRuntimeWithConfig(t devtest.T, cfg PresetConfig) *SingleChainRunt // fault proofs. Skipping the challenger also avoids requiring cannon prestate // artifacts, which are expensive to build locally. func NewMinimalNoFaultProofsRuntimeWithConfig(t devtest.T, cfg PresetConfig) *SingleChainRuntime { + return newMinimalNoFaultProofsRuntime(t, cfg, resolveSequencerCLKind(cfg)) +} + +func newMinimalNoFaultProofsRuntime(t devtest.T, cfg PresetConfig, sequencerCLKind MixedL2CLKind) *SingleChainRuntime { return newSingleChainRuntimeWithConfig(t, cfg, singleChainRuntimeSpec{ BuildWorld: newDefaultSingleChainWorld, + SequencerCLKind: sequencerCLKind, StartPrimary: startDefaultSingleChainPrimary, + StartBatcher: true, - StartProposer: false, StartChallenger: false, + StartProposer: false, }) } diff --git a/op-devstack/sysgo/singlechain_variants.go b/op-devstack/sysgo/singlechain_variants.go index e6787b9c2f4..5d9819bbf1b 100644 --- a/op-devstack/sysgo/singlechain_variants.go +++ b/op-devstack/sysgo/singlechain_variants.go @@ -32,15 +32,17 @@ func NewSingleChainMultiNodeRuntime(t devtest.T, withP2P bool) *SingleChainRunti } func NewSingleChainMultiNodeRuntimeWithConfig(t devtest.T, withP2P bool, cfg PresetConfig) *SingleChainRuntime { - return addSingleChainVerifier(t, NewMinimalRuntimeWithConfig(t, cfg), withP2P, cfg) + clKind := ResolveMixedL2CLKind() + return addSingleChainVerifier(t, newMinimalRuntime(t, cfg, resolveSequencerCLKind(cfg)), clKind, withP2P, cfg) } func NewSingleChainWithIsolatedVerifierRuntimeWithConfig(t devtest.T, cfg PresetConfig) *SingleChainRuntime { - return addSingleChainVerifier(t, NewMinimalNoFaultProofsRuntimeWithConfig(t, cfg), false, cfg) + clKind := ResolveMixedL2CLKind() + return addSingleChainVerifier(t, newMinimalNoFaultProofsRuntime(t, cfg, resolveSequencerCLKind(cfg)), clKind, false, cfg) } -func addSingleChainVerifier(t devtest.T, runtime *SingleChainRuntime, withP2P bool, cfg PresetConfig) *SingleChainRuntime { - nodeB := addSingleChainOpNode(t, runtime, "b", false, "", cfg.GlobalL2CLOptions...) +func addSingleChainVerifier(t devtest.T, runtime *SingleChainRuntime, clKind MixedL2CLKind, withP2P bool, cfg PresetConfig) *SingleChainRuntime { + nodeB := addSingleChainNode(t, runtime, "b", clKind, false, "", cfg.GlobalL2CLOptions...) if withP2P { connectSingleChainNodes(t, runtime.L2EL, runtime.L2CL, nodeB) } @@ -52,11 +54,22 @@ func NewSingleChainTwoVerifiersRuntime(t devtest.T) *SingleChainRuntime { return NewSingleChainTwoVerifiersRuntimeWithConfig(t, PresetConfig{}) } +// NewSingleChainTwoVerifiersRuntimeWithConfig adds two verifiers to a minimal runtime without +// fault proofs: "b" derives from L1, and "c" follows "b" as its follow source. The CL kind of each +// node comes from newSingleChainTwoVerifiersCLKinds. func NewSingleChainTwoVerifiersRuntimeWithConfig(t devtest.T, cfg PresetConfig) *SingleChainRuntime { - runtime := NewSingleChainMultiNodeRuntimeWithConfig(t, true, cfg) + requestedCLKind := ResolveMixedL2CLKind() + clKinds := newSingleChainTwoVerifiersCLKinds(requestedCLKind) + t.Logger().Info("Pinned the L2 CL kind of the sequencer and verifier b; verifier c uses the requested kind", + "requestedKind", requestedCLKind, + "b", clKinds.DerivingVerifier, + "c", clKinds.FollowSourceVerifier, + "sequencer", clKinds.Sequencer, + ) + runtime := addSingleChainVerifier(t, newMinimalNoFaultProofsRuntime(t, cfg, clKinds.Sequencer), clKinds.DerivingVerifier, true, cfg) nodeB := runtime.Nodes["b"] t.Require().NotNil(nodeB, "missing single-chain node b") - nodeC := addSingleChainOpNode(t, runtime, "c", false, nodeB.CL.UserRPC(), cfg.GlobalL2CLOptions...) + nodeC := addSingleChainNode(t, runtime, "c", clKinds.FollowSourceVerifier, false, nodeB.CL.UserRPC(), cfg.GlobalL2CLOptions...) connectSingleChainNodes(t, runtime.L2EL, runtime.L2CL, nodeC) connectSingleChainNodes(t, nodeB.EL, nodeB.CL, nodeC) @@ -66,6 +79,24 @@ func NewSingleChainTwoVerifiersRuntimeWithConfig(t devtest.T, cfg PresetConfig) return runtime } +type singleChainTwoVerifiersCLKinds struct { + DerivingVerifier MixedL2CLKind + FollowSourceVerifier MixedL2CLKind + Sequencer MixedL2CLKind +} + +// newSingleChainTwoVerifiersCLKinds runs the sequencer and the deriving verifier on op-node +// whatever the requested kind, and only the follow-source verifier on the requested kind, so a +// kona-node run exercises kona's follow-source mode against an op-node upstream rather than kona +// sequencing or derivation. +func newSingleChainTwoVerifiersCLKinds(requested MixedL2CLKind) singleChainTwoVerifiersCLKinds { + return singleChainTwoVerifiersCLKinds{ + DerivingVerifier: MixedL2CLOpNode, + FollowSourceVerifier: requested, + Sequencer: MixedL2CLOpNode, + } +} + func NewSimpleWithSyncTesterRuntime(t devtest.T) *SingleChainRuntime { return NewSimpleWithSyncTesterRuntimeWithConfig(t, PresetConfig{}) } @@ -116,6 +147,7 @@ func NewMinimalWithConductorsRuntime(t devtest.T) *SingleChainRuntime { } func NewMinimalWithConductorsRuntimeWithConfig(t devtest.T, cfg PresetConfig) *SingleChainRuntime { + clKind := ResolveMixedL2CLKind() // The CL and conductor depend on each other's RPC endpoints. Allocate a // stable proxy for each conductor before starting its CL, then attach the // conductor once its dynamically allocated RPC port is known. @@ -140,22 +172,24 @@ func NewMinimalWithConductorsRuntimeWithConfig(t devtest.T, cfg PresetConfig) *S // challenger, and rust e2e jobs do not build cannon artifacts. runtime := newSingleChainRuntimeWithConfig(t, cfg, singleChainRuntimeSpec{ BuildWorld: newDefaultSingleChainWorld, + SequencerCLKind: clKind, StartPrimary: startStoppedSingleChainPrimary, + StartBatcher: true, - StartProposer: true, StartChallenger: false, + StartProposer: true, }) // Every conductor member sequences once leadership reaches it, so a binary override has to // apply to all of them or it would silently stop taking effect on transfer. The primary already // received these options from startStoppedSingleChainPrimary. candidateELOpts := append(append([]OpRethOption{}, ResolveMixedL2ELOpts(t)...), cfg.OpRethOptions...) - nodeB := addStoppedSingleChainSequencerNode(t, runtime, "b", candidateELOpts, cfg.GlobalL2CLOptions...) - nodeC := addStoppedSingleChainSequencerNode(t, runtime, "c", candidateELOpts, cfg.GlobalL2CLOptions...) + nodeB := addStoppedSingleChainSequencerNode(t, runtime, "b", clKind, candidateELOpts, cfg.GlobalL2CLOptions...) + nodeC := addStoppedSingleChainSequencerNode(t, runtime, "c", clKind, candidateELOpts, cfg.GlobalL2CLOptions...) // A non-candidate node, always stock: it never sequences, so it independently validates every // block the candidates build. Without it an overridden binary on all three members could accept // its own invalid blocks and the run would still pass. - nodeVerifier := addSingleChainOpNode(t, runtime, "verifier", false, "", cfg.GlobalL2CLOptions...) + nodeVerifier := addSingleChainNode(t, runtime, "verifier", clKind, false, "", cfg.GlobalL2CLOptions...) healthCheck := conductorHealthCheckConfig(cfg.ConductorFastHealthChecks) conductorA := startConductorNode(t, "sequencer", runtime.L2Network, runtime.L2CL, conductorRPCProxies["sequencer"], runtime.L2EL, true, false, healthCheck) @@ -233,24 +267,26 @@ func replaceSingleChainTestSequencer(t devtest.T, runtime *SingleChainRuntime, n runtime.TestSequencer = newTestSequencerRuntime(testSequencer, name) } -func addSingleChainOpNode( +func addSingleChainNode( t devtest.T, runtime *SingleChainRuntime, name string, + clKind MixedL2CLKind, isSequencer bool, followSource string, l2Opts ...L2CLOption, ) *SingleChainNodeRuntime { - return addSingleChainOpNodeWithELOpts(t, runtime, name, isSequencer, followSource, nil, l2Opts...) + return addSingleChainNodeWithELOpts(t, runtime, name, clKind, isSequencer, followSource, nil, l2Opts...) } -// addSingleChainOpNodeWithELOpts adds a node whose EL takes op-reth options. Passing none yields a +// addSingleChainNodeWithELOpts adds a node whose EL takes op-reth options. Passing none yields a // stock op-reth, which is what lets a preset keep an independent check on blocks built by an // overridden binary. -func addSingleChainOpNodeWithELOpts( +func addSingleChainNodeWithELOpts( t devtest.T, runtime *SingleChainRuntime, name string, + clKind MixedL2CLKind, isSequencer bool, followSource string, elOpts []OpRethOption, @@ -259,7 +295,7 @@ func addSingleChainOpNodeWithELOpts( jwtPath := runtime.L2EL.JWTPath() jwtSecret := readJWTSecretFromPath(t, jwtPath) l2EL := startMixedOpRethNode(t, runtime.L2Network, name, jwtPath, jwtSecret, nil, elOpts...) - l2CL := startL2CLForKey(t, runtime.Keys, runtime.L1Network, runtime.L2Network, runtime.L1EL, runtime.L1CL, l2EL, jwtSecret, name, name, isSequencer, followSource, l2Opts) + l2CL := startL2CLForKey(t, runtime.Keys, runtime.L1Network, runtime.L2Network, runtime.L1EL, runtime.L1CL, l2EL, jwtSecret, clKind, name, name, isSequencer, followSource, l2Opts) node := newSingleChainNodeRuntime(name, isSequencer, l2EL, l2CL) runtime.Nodes[name] = node return node @@ -271,6 +307,7 @@ func addStoppedSingleChainSequencerNode( t devtest.T, runtime *SingleChainRuntime, name string, + clKind MixedL2CLKind, elOpts []OpRethOption, l2Opts ...L2CLOption, ) *SingleChainNodeRuntime { @@ -279,7 +316,7 @@ func addStoppedSingleChainSequencerNode( l2EL := startMixedOpRethNode(t, runtime.L2Network, name, jwtPath, jwtSecret, nil, elOpts...) l2CL := startStoppedSequencerCL( t, runtime.Keys, runtime.L1Network, runtime.L2Network, runtime.L1EL, runtime.L1CL, - l2EL, jwtSecret, name, l2Opts, + l2EL, jwtSecret, clKind, name, l2Opts, ) node := newSingleChainNodeRuntime(name, true, l2EL, l2CL) runtime.Nodes[name] = node diff --git a/op-devstack/sysgo/singlechain_variants_test.go b/op-devstack/sysgo/singlechain_variants_test.go new file mode 100644 index 00000000000..cd10d25e2c3 --- /dev/null +++ b/op-devstack/sysgo/singlechain_variants_test.go @@ -0,0 +1,31 @@ +package sysgo + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestSingleChainTwoVerifiersCLKinds(t *testing.T) { + clKinds := func(sequencer, derivingVerifier, followSourceVerifier MixedL2CLKind) singleChainTwoVerifiersCLKinds { + return singleChainTwoVerifiersCLKinds{ + DerivingVerifier: derivingVerifier, + FollowSourceVerifier: followSourceVerifier, + Sequencer: sequencer, + } + } + for _, testCase := range []struct { + name string + requestedKind MixedL2CLKind + want singleChainTwoVerifiersCLKinds + }{ + {name: "empty", requestedKind: "", want: clKinds(MixedL2CLOpNode, MixedL2CLOpNode, MixedL2CLOpNode)}, + {name: "op-node", requestedKind: MixedL2CLOpNode, want: clKinds(MixedL2CLOpNode, MixedL2CLOpNode, MixedL2CLOpNode)}, + {name: "kona-node", requestedKind: MixedL2CLKona, want: clKinds(MixedL2CLOpNode, MixedL2CLOpNode, MixedL2CLKona)}, + } { + t.Run(testCase.name, func(t *testing.T) { + t.Setenv("DEVSTACK_L2CL_KIND", string(testCase.requestedKind)) + require.Equal(t, testCase.want, newSingleChainTwoVerifiersCLKinds(ResolveMixedL2CLKind())) + }) + } +} From d5994d3b4e76780bf16adfa59df04032d4d86cf9 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Fri, 13 Nov 2026 00:55:02 +0100 Subject: [PATCH 2/6] op-acceptance-tests: require follow_l2 recovery past the reorged block TestFollowL2_ReorgRecovery could pass with the follow-source verifier and the deriving verifier both reset to genesis. The follow-source verifier must now hold the reorged block as local-safe before the L1 reorg, replace it in its EL with the deriving verifier's block, and move its local-safe head past that height. TestFollowL2_WithoutCLP2P also checks that the finalized head follows the deriving verifier after reconnecting. Co-Authored-By: Claude Opus 5.5 --- .../tests/sync/follow_l2/sync_test.go | 28 +++++++++++++++++ .../sysgo/singlechain_variants_test.go | 31 ------------------- 2 files changed, 28 insertions(+), 31 deletions(-) delete mode 100644 op-devstack/sysgo/singlechain_variants_test.go diff --git a/op-acceptance-tests/tests/sync/follow_l2/sync_test.go b/op-acceptance-tests/tests/sync/follow_l2/sync_test.go index a779a90df55..2da8d1ea3bd 100644 --- a/op-acceptance-tests/tests/sync/follow_l2/sync_test.go +++ b/op-acceptance-tests/tests/sync/follow_l2/sync_test.go @@ -100,6 +100,14 @@ func TestFollowL2_ReorgRecovery(gt *testing.T) { // Make sure verifier safe head is also advanced from reorgL2Block or matched sys.L2ELB.Reached(eth.Safe, l2BlockBeforeReorg.Number, 3) + // The follow-source verifier must hold the block about to be reorged as local-safe, so the + // recovery checks below start from a real safe head rather than from genesis. + require.NoError(sys.L2CLC.ReachedWithProgressFn( + safety.LocalSafe, safety.LocalUnsafe, l2BlockBeforeReorg.Number, + 2*time.Minute, 30*time.Second, + )()) + require.Equal(l2BlockBeforeReorg.Hash, sys.L2ELC.BlockRefByNumber(l2BlockBeforeReorg.Number).Hash, + "follow-source verifier must hold the deriving verifier's block before the L1 reorg") // Reorg L1 block which safe block L1 Origin points to l1BlockBeforeReorg := sys.L1EL.BlockRefByNumber(l2BlockBeforeReorg.L1Origin.Number) @@ -133,6 +141,18 @@ func TestFollowL2_ReorgRecovery(gt *testing.T) { }, 60*time.Second, 2*time.Second) logger.Info("Triggered L2 reorg", "l2", l2BlockAfterReorg) + sys.L2ELC.ReorgTriggered(l2BlockBeforeReorg, 60) + // A local-safe head past the reorged height shows the follow-source verifier recovered; the + // in-sync checks below also pass when both verifiers sit at genesis. Waiting for the EL reorg + // first means the unsafe head that bounds this wait is already climbing on the new chain. + require.NoError(sys.L2CLC.ReachedWithProgressFn( + safety.LocalSafe, safety.LocalUnsafe, l2BlockBeforeReorg.Number+1, + 5*time.Minute, time.Minute, + )()) + require.Equal(sys.L2ELB.BlockRefByNumber(l2BlockBeforeReorg.Number).Hash, + sys.L2ELC.BlockRefByNumber(l2BlockBeforeReorg.Number).Hash, + "follow-source verifier must replace the reorged block with the deriving verifier's") + attempts := 30 dsl.CheckAll(t, sys.L2CL.InSyncFn(sys.L2CLB, safety.LocalUnsafe, attempts), @@ -217,6 +237,14 @@ func TestFollowL2_WithoutCLP2P(gt *testing.T) { sys.L2CLC.InSyncFn(sys.L2CLB, safety.LocalUnsafe, attempts), ) + // The deriving verifier finalizes past genesis first, so the comparison cannot pass with both + // nodes at genesis. + require.NoError(sys.L2CLB.ReachedWithProgressFn( + safety.Finalized, safety.LocalSafe, 1, + 5*time.Minute, time.Minute, + )()) + sys.L2CLC.InSync(sys.L2CLB, safety.Finalized, attempts) + t.Cleanup(func() { sys.L2CLC.ConnectPeer(sys.L2CLB) sys.L2CLB.ConnectPeer(sys.L2CLC) diff --git a/op-devstack/sysgo/singlechain_variants_test.go b/op-devstack/sysgo/singlechain_variants_test.go deleted file mode 100644 index cd10d25e2c3..00000000000 --- a/op-devstack/sysgo/singlechain_variants_test.go +++ /dev/null @@ -1,31 +0,0 @@ -package sysgo - -import ( - "testing" - - "github.com/stretchr/testify/require" -) - -func TestSingleChainTwoVerifiersCLKinds(t *testing.T) { - clKinds := func(sequencer, derivingVerifier, followSourceVerifier MixedL2CLKind) singleChainTwoVerifiersCLKinds { - return singleChainTwoVerifiersCLKinds{ - DerivingVerifier: derivingVerifier, - FollowSourceVerifier: followSourceVerifier, - Sequencer: sequencer, - } - } - for _, testCase := range []struct { - name string - requestedKind MixedL2CLKind - want singleChainTwoVerifiersCLKinds - }{ - {name: "empty", requestedKind: "", want: clKinds(MixedL2CLOpNode, MixedL2CLOpNode, MixedL2CLOpNode)}, - {name: "op-node", requestedKind: MixedL2CLOpNode, want: clKinds(MixedL2CLOpNode, MixedL2CLOpNode, MixedL2CLOpNode)}, - {name: "kona-node", requestedKind: MixedL2CLKona, want: clKinds(MixedL2CLOpNode, MixedL2CLOpNode, MixedL2CLKona)}, - } { - t.Run(testCase.name, func(t *testing.T) { - t.Setenv("DEVSTACK_L2CL_KIND", string(testCase.requestedKind)) - require.Equal(t, testCase.want, newSingleChainTwoVerifiersCLKinds(ResolveMixedL2CLKind())) - }) - } -} From aafe0f0151b3d9c4c4123ea276fd5d3e9dd6a293 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Fri, 13 Nov 2026 00:55:03 +0100 Subject: [PATCH 3/6] op-devstack: validate DEVSTACK_L2CL_KIND when it is resolved ResolveMixedL2CLKind takes the test handle and fails on a value other than op-node or kona-node, so an invalid kind fails before any node starts instead of when the first node of that kind starts. The two-verifiers runtime reads the follow-source verifier's kind into a local in place of singleChainTwoVerifiersCLKinds, the multi-node runtimes call the exported minimal-runtime constructors, the CL-kind starters use FailNowf, and the README says that some presets pin nodes to op-node. Co-Authored-By: Claude Opus 5.5 --- op-acceptance-tests/tests/sdm/init_test.go | 4 +- op-devstack/README.md | 2 +- op-devstack/sysgo/mixed_runtime.go | 18 +++++--- op-devstack/sysgo/mixed_runtime_test.go | 28 +++++++++++++ op-devstack/sysgo/singlechain_build.go | 4 +- op-devstack/sysgo/singlechain_runtime.go | 8 ++-- op-devstack/sysgo/singlechain_variants.go | 49 ++++++---------------- 7 files changed, 62 insertions(+), 51 deletions(-) diff --git a/op-acceptance-tests/tests/sdm/init_test.go b/op-acceptance-tests/tests/sdm/init_test.go index db96beb2240..643f7b6caec 100644 --- a/op-acceptance-tests/tests/sdm/init_test.go +++ b/op-acceptance-tests/tests/sdm/init_test.go @@ -43,7 +43,7 @@ func newSDMRethSystemWithIsolatedVerifier(t devtest.T) *sdmtest.RethSystem { // derivation actor stays in AwaitingELSyncCompletion, the EL gets no engine calls, and the // safe head never leaves genesis. kona-node has no L1-only bootstrap of the force-build path // today, so this test cannot pass under kona-node and is op-node-only for now. - if sysgo.ResolveMixedL2CLKind() == sysgo.MixedL2CLKona { + if sysgo.ResolveMixedL2CLKind(t) == sysgo.MixedL2CLKona { t.Skip("isolated-verifier force-build path is not supported by kona-node (no L1-only EL-sync bootstrap); op-node only") } return buildSDMRethSystem(t, true, true, true, "", nil) @@ -84,7 +84,7 @@ func buildSDMRethSystem( ) *sdmtest.RethSystem { // Honor DEVSTACK_L2CL_KIND so the kona acceptance suite exercises this test with // kona-node on both the sequencer and verifier (defaults to op-node when unset). - clKind := sysgo.ResolveMixedL2CLKind() + clKind := sysgo.ResolveMixedL2CLKind(t) sequencerKey := "sequencer-op-reth" // The two sequencer paths are disjoint, and neither is an extension point for a diff --git a/op-devstack/README.md b/op-devstack/README.md index d2246ed928d..5dfab54b194 100644 --- a/op-devstack/README.md +++ b/op-devstack/README.md @@ -110,7 +110,7 @@ and returns a typed output that the test then may use. - `DEVSTACK_MONOREPO_ROOT`: Absolute path to a monorepo checkout. Used by out-of-tree acceptance suites (which depend on this module by git rev) so contract-artifact resolution can find `packages/contracts-bedrock` outside the cwd-relative walk. In-tree runs don't need it. ### Rust stack env vars: -- `DEVSTACK_L2CL_KIND=kona-node` to select kona-node as default L2 CL node +- `DEVSTACK_L2CL_KIND=kona-node` to select kona-node as default L2 CL node. Any value other than `op-node` or `kona-node` fails the test before a node starts. Some presets pin nodes to op-node whatever the variable says, such as the sequencer of a chain with interop at genesis, and the sequencer and deriving verifier of the two-verifiers preset. - `RUST_BINARY_PATH_KONA_NODE=/home/USERHERE/projects/kona/target/debug/kona-node` to select the kona-node executable to run (skips the build; the path must exist) - `RUST_BINARY_PATH_OP_RETH=/home/USERHERE/projects/reth/target/release/op-reth` to select the op-reth executable to run (skips the build; the path must exist) diff --git a/op-devstack/sysgo/mixed_runtime.go b/op-devstack/sysgo/mixed_runtime.go index 61b55027d91..08bc3aba0d6 100644 --- a/op-devstack/sysgo/mixed_runtime.go +++ b/op-devstack/sysgo/mixed_runtime.go @@ -70,14 +70,20 @@ func devstackL2CLKind() MixedL2CLKind { } // ResolveMixedL2CLKind returns the L2 CL kind requested via DEVSTACK_L2CL_KIND, -// defaulting to op-node when the variable is unset. Tests that build explicit -// MixedSingleChainNodeSpec lists use this so they honor the same env-driven CL -// selection as the higher-level single-chain presets. -func ResolveMixedL2CLKind() MixedL2CLKind { - if kind := devstackL2CLKind(); kind != "" { +// defaulting to op-node when the variable is unset, and fails the test on any other +// value before a node starts. Tests that build explicit MixedSingleChainNodeSpec lists +// use this so they honor the same env-driven CL selection as the higher-level +// single-chain presets. +func ResolveMixedL2CLKind(t devtest.T) MixedL2CLKind { + switch kind := devstackL2CLKind(); kind { + case "": + return MixedL2CLOpNode + case MixedL2CLOpNode, MixedL2CLKona: return kind + default: + t.Require().FailNowf("unsupported L2 CL kind", "DEVSTACK_L2CL_KIND=%q, want %q or %q", kind, MixedL2CLOpNode, MixedL2CLKona) + return "" // unreachable } - return MixedL2CLOpNode } const ( diff --git a/op-devstack/sysgo/mixed_runtime_test.go b/op-devstack/sysgo/mixed_runtime_test.go index ae5124255dd..5c97f9ca742 100644 --- a/op-devstack/sysgo/mixed_runtime_test.go +++ b/op-devstack/sysgo/mixed_runtime_test.go @@ -66,3 +66,31 @@ func TestResolveMixedL2ELOptsReadsEnv(t *testing.T) { require.Equal(t, []string{"--extra.bind=127.0.0.1:0"}, cfg.ExtraArgs) require.True(t, cfg.DisableProofsHistory) } + +func TestResolveMixedL2CLKind(t *testing.T) { + for _, tc := range []struct { + env string + want MixedL2CLKind + }{ + {env: "", want: MixedL2CLOpNode}, + {env: "op-node", want: MixedL2CLOpNode}, + {env: "kona-node", want: MixedL2CLKona}, + } { + t.Run(tc.env, func(t *testing.T) { + t.Setenv("DEVSTACK_L2CL_KIND", tc.env) + require.Equal(t, tc.want, ResolveMixedL2CLKind(devtest.SerialT(t))) + }) + } +} + +// An unknown kind fails at resolution, before any preset starts a node of that kind. +func TestResolveMixedL2CLKindRejectsUnknownKind(t *testing.T) { + for _, kind := range []string{"kona", "Kona-Node", "op-geth"} { + t.Run(kind, func(t *testing.T) { + t.Setenv("DEVSTACK_L2CL_KIND", kind) + dt := &rejectedKonaOptionTest{T: devtest.SerialT(t)} + require.PanicsWithValue(t, "Kona option rejected", func() { ResolveMixedL2CLKind(dt) }) + require.Contains(t, dt.failure, "DEVSTACK_L2CL_KIND") + }) + } +} diff --git a/op-devstack/sysgo/singlechain_build.go b/op-devstack/sysgo/singlechain_build.go index 9e2f58c0e7b..68ba9535293 100644 --- a/op-devstack/sysgo/singlechain_build.go +++ b/op-devstack/sysgo/singlechain_build.go @@ -121,7 +121,7 @@ func startL2CLForKey( L2CLOptions: l2CLOpts, }) default: - t.Require().FailNow("unsupported L2 CL kind", "unknown L2 CL kind %q", clKind) + t.Require().FailNowf("unsupported L2 CL kind", "unknown L2 CL kind %q", clKind) return nil // unreachable } } @@ -158,7 +158,7 @@ func startStoppedSequencerCL( SequencerStopped: true, }) default: - t.Require().FailNow("unsupported L2 CL kind", "unknown L2 CL kind %q", clKind) + t.Require().FailNowf("unsupported L2 CL kind", "unknown L2 CL kind %q", clKind) return nil // unreachable } } diff --git a/op-devstack/sysgo/singlechain_runtime.go b/op-devstack/sysgo/singlechain_runtime.go index 98487eeac8b..f3c827358f2 100644 --- a/op-devstack/sysgo/singlechain_runtime.go +++ b/op-devstack/sysgo/singlechain_runtime.go @@ -77,11 +77,11 @@ func requireOpNodeSequencerCLKind(t devtest.T, sequencerCLKind MixedL2CLKind) { // resolveSequencerCLKind returns the L2 CL kind the environment requests for the sequencer, // pinned to op-node when the chain runs interop at genesis. -func resolveSequencerCLKind(cfg PresetConfig) MixedL2CLKind { +func resolveSequencerCLKind(t devtest.T, cfg PresetConfig) MixedL2CLKind { if cfg.InteropAtGenesis { return MixedL2CLOpNode } - return ResolveMixedL2CLKind() + return ResolveMixedL2CLKind(t) } func newSingleChainNodeRuntime(name string, isSequencer bool, el L2ELNode, cl L2CLNode) *SingleChainNodeRuntime { @@ -272,7 +272,7 @@ func NewMinimalRuntime(t devtest.T) *SingleChainRuntime { } func NewMinimalRuntimeWithConfig(t devtest.T, cfg PresetConfig) *SingleChainRuntime { - return newMinimalRuntime(t, cfg, resolveSequencerCLKind(cfg)) + return newMinimalRuntime(t, cfg, resolveSequencerCLKind(t, cfg)) } func newMinimalRuntime(t devtest.T, cfg PresetConfig, sequencerCLKind MixedL2CLKind) *SingleChainRuntime { @@ -293,7 +293,7 @@ func newMinimalRuntime(t devtest.T, cfg PresetConfig, sequencerCLKind MixedL2CLK // fault proofs. Skipping the challenger also avoids requiring cannon prestate // artifacts, which are expensive to build locally. func NewMinimalNoFaultProofsRuntimeWithConfig(t devtest.T, cfg PresetConfig) *SingleChainRuntime { - return newMinimalNoFaultProofsRuntime(t, cfg, resolveSequencerCLKind(cfg)) + return newMinimalNoFaultProofsRuntime(t, cfg, resolveSequencerCLKind(t, cfg)) } func newMinimalNoFaultProofsRuntime(t devtest.T, cfg PresetConfig, sequencerCLKind MixedL2CLKind) *SingleChainRuntime { diff --git a/op-devstack/sysgo/singlechain_variants.go b/op-devstack/sysgo/singlechain_variants.go index 5d9819bbf1b..b63b01a1e05 100644 --- a/op-devstack/sysgo/singlechain_variants.go +++ b/op-devstack/sysgo/singlechain_variants.go @@ -32,13 +32,13 @@ func NewSingleChainMultiNodeRuntime(t devtest.T, withP2P bool) *SingleChainRunti } func NewSingleChainMultiNodeRuntimeWithConfig(t devtest.T, withP2P bool, cfg PresetConfig) *SingleChainRuntime { - clKind := ResolveMixedL2CLKind() - return addSingleChainVerifier(t, newMinimalRuntime(t, cfg, resolveSequencerCLKind(cfg)), clKind, withP2P, cfg) + clKind := ResolveMixedL2CLKind(t) + return addSingleChainVerifier(t, NewMinimalRuntimeWithConfig(t, cfg), clKind, withP2P, cfg) } func NewSingleChainWithIsolatedVerifierRuntimeWithConfig(t devtest.T, cfg PresetConfig) *SingleChainRuntime { - clKind := ResolveMixedL2CLKind() - return addSingleChainVerifier(t, newMinimalNoFaultProofsRuntime(t, cfg, resolveSequencerCLKind(cfg)), clKind, false, cfg) + clKind := ResolveMixedL2CLKind(t) + return addSingleChainVerifier(t, NewMinimalNoFaultProofsRuntimeWithConfig(t, cfg), clKind, false, cfg) } func addSingleChainVerifier(t devtest.T, runtime *SingleChainRuntime, clKind MixedL2CLKind, withP2P bool, cfg PresetConfig) *SingleChainRuntime { @@ -55,21 +55,16 @@ func NewSingleChainTwoVerifiersRuntime(t devtest.T) *SingleChainRuntime { } // NewSingleChainTwoVerifiersRuntimeWithConfig adds two verifiers to a minimal runtime without -// fault proofs: "b" derives from L1, and "c" follows "b" as its follow source. The CL kind of each -// node comes from newSingleChainTwoVerifiersCLKinds. +// fault proofs: "b" derives from L1, and "c" follows "b" as its follow source. func NewSingleChainTwoVerifiersRuntimeWithConfig(t devtest.T, cfg PresetConfig) *SingleChainRuntime { - requestedCLKind := ResolveMixedL2CLKind() - clKinds := newSingleChainTwoVerifiersCLKinds(requestedCLKind) - t.Logger().Info("Pinned the L2 CL kind of the sequencer and verifier b; verifier c uses the requested kind", - "requestedKind", requestedCLKind, - "b", clKinds.DerivingVerifier, - "c", clKinds.FollowSourceVerifier, - "sequencer", clKinds.Sequencer, - ) - runtime := addSingleChainVerifier(t, newMinimalNoFaultProofsRuntime(t, cfg, clKinds.Sequencer), clKinds.DerivingVerifier, true, cfg) + // Only "c" takes the requested CL kind; the sequencer and "b" stay on op-node, so a kona-node + // run exercises kona's follow-source mode against an op-node upstream. + followKind := ResolveMixedL2CLKind(t) + t.Logger().Info("Starting two-verifier runtime", "b", MixedL2CLOpNode, "c", followKind, "sequencer", MixedL2CLOpNode) + runtime := addSingleChainVerifier(t, newMinimalNoFaultProofsRuntime(t, cfg, MixedL2CLOpNode), MixedL2CLOpNode, true, cfg) nodeB := runtime.Nodes["b"] t.Require().NotNil(nodeB, "missing single-chain node b") - nodeC := addSingleChainNode(t, runtime, "c", clKinds.FollowSourceVerifier, false, nodeB.CL.UserRPC(), cfg.GlobalL2CLOptions...) + nodeC := addSingleChainNode(t, runtime, "c", followKind, false, nodeB.CL.UserRPC(), cfg.GlobalL2CLOptions...) connectSingleChainNodes(t, runtime.L2EL, runtime.L2CL, nodeC) connectSingleChainNodes(t, nodeB.EL, nodeB.CL, nodeC) @@ -79,24 +74,6 @@ func NewSingleChainTwoVerifiersRuntimeWithConfig(t devtest.T, cfg PresetConfig) return runtime } -type singleChainTwoVerifiersCLKinds struct { - DerivingVerifier MixedL2CLKind - FollowSourceVerifier MixedL2CLKind - Sequencer MixedL2CLKind -} - -// newSingleChainTwoVerifiersCLKinds runs the sequencer and the deriving verifier on op-node -// whatever the requested kind, and only the follow-source verifier on the requested kind, so a -// kona-node run exercises kona's follow-source mode against an op-node upstream rather than kona -// sequencing or derivation. -func newSingleChainTwoVerifiersCLKinds(requested MixedL2CLKind) singleChainTwoVerifiersCLKinds { - return singleChainTwoVerifiersCLKinds{ - DerivingVerifier: MixedL2CLOpNode, - FollowSourceVerifier: requested, - Sequencer: MixedL2CLOpNode, - } -} - func NewSimpleWithSyncTesterRuntime(t devtest.T) *SingleChainRuntime { return NewSimpleWithSyncTesterRuntimeWithConfig(t, PresetConfig{}) } @@ -147,7 +124,7 @@ func NewMinimalWithConductorsRuntime(t devtest.T) *SingleChainRuntime { } func NewMinimalWithConductorsRuntimeWithConfig(t devtest.T, cfg PresetConfig) *SingleChainRuntime { - clKind := ResolveMixedL2CLKind() + clKind := ResolveMixedL2CLKind(t) // The CL and conductor depend on each other's RPC endpoints. Allocate a // stable proxy for each conductor before starting its CL, then attach the // conductor once its dynamically allocated RPC port is known. @@ -186,7 +163,7 @@ func NewMinimalWithConductorsRuntimeWithConfig(t devtest.T, cfg PresetConfig) *S nodeB := addStoppedSingleChainSequencerNode(t, runtime, "b", clKind, candidateELOpts, cfg.GlobalL2CLOptions...) nodeC := addStoppedSingleChainSequencerNode(t, runtime, "c", clKind, candidateELOpts, cfg.GlobalL2CLOptions...) - // A non-candidate node, always stock: it never sequences, so it independently validates every + // A non-candidate node with stock op-reth: it never sequences, so it independently validates every // block the candidates build. Without it an overridden binary on all three members could accept // its own invalid blocks and the run would still pass. nodeVerifier := addSingleChainNode(t, runtime, "verifier", clKind, false, "", cfg.GlobalL2CLOptions...) From 7bdee17022acf50261e6592988c0309ab956aa8c Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Wed, 11 Nov 2026 18:16:47 +0100 Subject: [PATCH 4/6] kona-engine,kona-node-service: keep a stale delegated safe head from moving safe below finality A stale delegate snapshot can report a safe head below kona-node's finalized head, and kona-node applies it, moving safe behind its own finality. Kona-node now drops a delegate status whose labels are out of order (a zero local-safe head counts as unreported) or that breaks the engine's applied finality, and the engine skips a delegated safe head below, or conflicting with, established finality. A delegated safe head below safe is ignored, and forwarded only once per engine view, unless an EL that applied the last forkchoice holds a different block at that height, which proves divergence and reorgs the engine above finality. Labels left with unsafe below safe are repaired by the next delegated head above finality. Repeated rejection warnings are deduplicated. A rejected delegated forkchoice update resets the engine as on develop; recovery is #24065 (#24061). Known window: after a delegate L1 reorg whose fork point is above its new safe head, safe stays on the old fork until the delegate passes it. Closes #23910. BREAKING CHANGE: `ConsolidateTaskError` gains `BreaksFinality` and `FailedToFetchDelegatedSafeBlock`; `FinalizeTaskError::{BehindFinalizedHead, ConflictsWithFinalizedHead}` become `BreaksFinality(FinalityViolation)`; `FinalityViolation` is exported; `EngineState` gains `el_applied_last_forkchoice`. Co-Authored-By: Claude Opus 5.5 --- rust/kona/crates/node/engine/src/lib.rs | 2 +- .../kona/crates/node/engine/src/state/core.rs | 16 +- .../crates/node/engine/src/state/finality.rs | 126 ++++ rust/kona/crates/node/engine/src/state/mod.rs | 3 + .../src/task_queue/tasks/consolidate/error.rs | 14 +- .../src/task_queue/tasks/consolidate/task.rs | 578 +++++++++++++++--- .../src/task_queue/tasks/finalize/error.rs | 25 +- .../src/task_queue/tasks/finalize/task.rs | 27 +- .../task_queue/tasks/finalize/task_test.rs | 15 +- .../task_queue/tasks/synchronize/helper.rs | 2 + .../engine/src/test_utils/engine_state.rs | 14 +- .../crates/node/engine/src/test_utils/rpc.rs | 10 + .../src/actors/derivation/delegated/actor.rs | 180 +++--- .../src/actors/derivation/delegated/tests.rs | 98 +++ .../node/service/src/actors/engine/actor.rs | 2 +- 15 files changed, 925 insertions(+), 187 deletions(-) create mode 100644 rust/kona/crates/node/engine/src/state/finality.rs diff --git a/rust/kona/crates/node/engine/src/lib.rs b/rust/kona/crates/node/engine/src/lib.rs index c1f955096bb..e8fd1f9305d 100644 --- a/rust/kona/crates/node/engine/src/lib.rs +++ b/rust/kona/crates/node/engine/src/lib.rs @@ -65,7 +65,7 @@ mod versions; pub use versions::{EngineForkchoiceVersion, EngineGetPayloadVersion, EngineNewPayloadVersion}; mod state; -pub use state::{EngineState, EngineSyncState, EngineSyncStateUpdate}; +pub use state::{EngineState, EngineSyncState, EngineSyncStateUpdate, FinalityViolation}; mod kinds; pub use kinds::EngineKind; diff --git a/rust/kona/crates/node/engine/src/state/core.rs b/rust/kona/crates/node/engine/src/state/core.rs index 7a34ce3f626..67204d7d81f 100644 --- a/rust/kona/crates/node/engine/src/state/core.rs +++ b/rust/kona/crates/node/engine/src/state/core.rs @@ -1,6 +1,7 @@ //! The internal state of the engine controller. use crate::Metrics; +use alloy_eips::BlockNumHash; use alloy_rpc_types_engine::ForkchoiceState; use kona_protocol::L2BlockInfo; use serde::{Deserialize, Serialize}; @@ -54,6 +55,12 @@ impl EngineSyncState { self.finalized_head } + /// Returns the id of the finalized head once finality is established. The default finalized + /// head means it is not yet, so no block pins finality. + pub fn established_finalized_head(&self) -> Option { + (self.finalized_head != L2BlockInfo::default()).then(|| self.finalized_head.block_info.id()) + } + /// Creates a `ForkchoiceState` /// /// - `head_block` = `unsafe_head` @@ -127,8 +134,10 @@ pub struct EngineSyncStateUpdate { /// The chain state viewed by the engine controller. #[derive(Default, Debug, Copy, Clone, PartialEq, Eq)] pub struct EngineState { - /// The sync state of the engine. - pub sync_state: EngineSyncState, + /// Whether the execution layer answered `VALID` to the last forkchoice update the engine sent, + /// so its canonical chain is the engine's. Unlike `el_sync_finished`, a later `SYNCING` answer + /// clears it. + pub el_applied_last_forkchoice: bool, /// Whether or not the EL has finished syncing. pub el_sync_finished: bool, @@ -139,6 +148,9 @@ pub struct EngineState { /// because engine may forgot backupUnsafeHead or backupUnsafeHead is not part /// of the chain. pub need_fcu_call_backup_unsafe_reorg: bool, + + /// The sync state of the engine. + pub sync_state: EngineSyncState, } impl EngineState { diff --git a/rust/kona/crates/node/engine/src/state/finality.rs b/rust/kona/crates/node/engine/src/state/finality.rs new file mode 100644 index 00000000000..94363c549bf --- /dev/null +++ b/rust/kona/crates/node/engine/src/state/finality.rs @@ -0,0 +1,126 @@ +//! Checks that keep a label from moving below finality. + +use crate::EngineSyncState; +use alloy_eips::BlockNumHash; +use thiserror::Error; + +/// A block that a label cannot move to, because it would move the label below finality. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Error)] +pub enum FinalityViolation { + /// The block is below the finalized head. + #[error("Block {target} is behind the finalized head {finalized:?}")] + BehindFinalizedHead { + /// The finalized head. + finalized: BlockNumHash, + /// The number of the block. + target: u64, + }, + /// The block is a different block at the finalized head's height. + #[error("Block {target:?} conflicts with the finalized head {finalized:?}")] + ConflictsWithFinalizedHead { + /// The finalized head. + finalized: BlockNumHash, + /// The block. + target: BlockNumHash, + }, +} + +impl EngineSyncState { + /// Rejects a block number below the established finalized head. + pub fn ensure_not_behind_finality(&self, target: u64) -> Result<(), FinalityViolation> { + match self.established_finalized_head() { + Some(finalized) if target < finalized.number => { + Err(FinalityViolation::BehindFinalizedHead { finalized, target }) + } + _ => Ok(()), + } + } + + /// Rejects a different block at the established finalized head's height. Before finality is + /// established no block pins it, so any block, including the configured genesis, passes. + pub fn ensure_no_conflict_with_finality( + &self, + target: BlockNumHash, + ) -> Result<(), FinalityViolation> { + match self.established_finalized_head() { + Some(finalized) + if target.number == finalized.number && target.hash != finalized.hash => + { + Err(FinalityViolation::ConflictsWithFinalizedHead { finalized, target }) + } + _ => Ok(()), + } + } + + /// Rejects a block below the established finalized head or conflicting with it. + pub fn ensure_respects_finality(&self, target: BlockNumHash) -> Result<(), FinalityViolation> { + self.ensure_not_behind_finality(target.number)?; + self.ensure_no_conflict_with_finality(target) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::EngineSyncStateUpdate; + use alloy_primitives::B256; + use kona_protocol::{BlockInfo, L2BlockInfo}; + + const FINALIZED_NUMBER: u64 = 5; + + fn block(number: u64, hash: u8) -> L2BlockInfo { + L2BlockInfo { + block_info: BlockInfo { hash: B256::repeat_byte(hash), number, ..Default::default() }, + ..Default::default() + } + } + + fn sync_state_finalized_at(finalized_head: L2BlockInfo) -> EngineSyncState { + EngineSyncState::default().apply_update(EngineSyncStateUpdate { + finalized_head: Some(finalized_head), + ..Default::default() + }) + } + + /// The finalized head at 5 once finality is established, and the default head before. + fn finalized_head(is_finality_established: bool) -> L2BlockInfo { + if is_finality_established { block(FINALIZED_NUMBER, 1) } else { L2BlockInfo::default() } + } + + /// The violation expected for `target` checked against the finalized head at 5. + fn behind(target: L2BlockInfo) -> Option { + Some(FinalityViolation::BehindFinalizedHead { + finalized: block(FINALIZED_NUMBER, 1).block_info.id(), + target: target.block_info.number, + }) + } + + /// The violation expected for `target` conflicting with the finalized head at 5. + fn conflicting(target: L2BlockInfo) -> Option { + Some(FinalityViolation::ConflictsWithFinalizedHead { + finalized: block(FINALIZED_NUMBER, 1).block_info.id(), + target: target.block_info.id(), + }) + } + + /// A block below the established finalized head, or a different block at its height, breaks + /// finality. Before finality is established no block pins it, so a different block at the + /// default head's height, genesis, passes. + #[rstest::rstest] + #[case::above(true, block(FINALIZED_NUMBER + 1, 2), None)] + #[case::at(true, block(FINALIZED_NUMBER, 1), None)] + #[case::below(true, block(FINALIZED_NUMBER - 1, 2), behind(block(FINALIZED_NUMBER - 1, 2)))] + #[case::conflicting(true, block(FINALIZED_NUMBER, 2), conflicting(block(FINALIZED_NUMBER, 2)))] + #[case::not_established(false, block(0, 2), None)] + fn block_is_checked_against_established_finality( + #[case] is_finality_established: bool, + #[case] target: L2BlockInfo, + #[case] expected_violation: Option, + ) { + let sync_state = sync_state_finalized_at(finalized_head(is_finality_established)); + + let result = sync_state.ensure_respects_finality(target.block_info.id()); + + assert_eq!(result.err(), expected_violation); + } +} diff --git a/rust/kona/crates/node/engine/src/state/mod.rs b/rust/kona/crates/node/engine/src/state/mod.rs index b61100917e6..a3b0cfe6f51 100644 --- a/rust/kona/crates/node/engine/src/state/mod.rs +++ b/rust/kona/crates/node/engine/src/state/mod.rs @@ -2,3 +2,6 @@ mod core; pub use core::{EngineState, EngineSyncState, EngineSyncStateUpdate}; + +mod finality; +pub use finality::FinalityViolation; diff --git a/rust/kona/crates/node/engine/src/task_queue/tasks/consolidate/error.rs b/rust/kona/crates/node/engine/src/task_queue/tasks/consolidate/error.rs index c5bd5dbcb0f..2ce1b187a76 100644 --- a/rust/kona/crates/node/engine/src/task_queue/tasks/consolidate/error.rs +++ b/rust/kona/crates/node/engine/src/task_queue/tasks/consolidate/error.rs @@ -1,7 +1,7 @@ //! Contains error types for the [`crate::ConsolidateTask`]. use crate::{ - BuildTaskError, EngineTaskError, SealTaskError, SynchronizeTaskError, + BuildTaskError, EngineTaskError, FinalityViolation, SealTaskError, SynchronizeTaskError, task_queue::tasks::{BuildAndImportError, task::EngineTaskErrorSeverity}, }; use thiserror::Error; @@ -9,6 +9,13 @@ use thiserror::Error; /// An error that occurs when running the [`crate::ConsolidateTask`]. #[derive(Debug, Error)] pub enum ConsolidateTaskError { + /// A delegated safe head would move the safe labels below finality. + #[error("Delegated safe head breaks finality: {0}")] + BreaksFinality(#[from] FinalityViolation), + /// Failed to fetch the execution layer's block at the height of a delegated safe head below the + /// safe head. + #[error("Failed to fetch the block at delegated safe head height {0}")] + FailedToFetchDelegatedSafeBlock(u64), /// The unsafe L2 block is missing. #[error("Unsafe L2 block is missing {0}")] MissingUnsafeL2Block(u64), @@ -38,8 +45,11 @@ impl From for ConsolidateTaskError { impl EngineTaskError for ConsolidateTaskError { fn severity(&self) -> EngineTaskErrorSeverity { match self { + Self::BreaksFinality(_) => EngineTaskErrorSeverity::Skip, Self::MissingUnsafeL2Block(_) => EngineTaskErrorSeverity::Reset, - Self::FailedToFetchUnsafeL2Block => EngineTaskErrorSeverity::Temporary, + Self::FailedToFetchDelegatedSafeBlock(_) | Self::FailedToFetchUnsafeL2Block => { + EngineTaskErrorSeverity::Temporary + } Self::BuildTaskFailed(inner) => inner.severity(), Self::SealTaskFailed(inner) => inner.severity(), Self::ForkchoiceUpdateFailed(inner) => inner.severity(), diff --git a/rust/kona/crates/node/engine/src/task_queue/tasks/consolidate/task.rs b/rust/kona/crates/node/engine/src/task_queue/tasks/consolidate/task.rs index d3183cdafa5..00f31457729 100644 --- a/rust/kona/crates/node/engine/src/task_queue/tasks/consolidate/task.rs +++ b/rust/kona/crates/node/engine/src/task_queue/tasks/consolidate/task.rs @@ -100,6 +100,159 @@ impl ConsolidateTask { Ok(()) } + /// This provides symmetric fallback behavior with `build_and_import`. + async fn reconcile_to_safe_head( + &self, + state: &mut EngineState, + safe_l2: &L2BlockInfo, + ) -> Result<(), ConsolidateTaskError> { + warn!( + target: "engine", + safe_l2 = %safe_l2, + "Apply safe head" + ); + + // We intentionally set the unsafe head to safe_l2 to ensure the engine observes a + // self-consistent head state. This is required to correctly handle reorgs (where unsafe + // may be ahead on a non-canonical fork) and to trigger EL sync when the local unsafe head + // lags behind the safe head. + self.apply_delegated_safe_head( + state, + safe_l2, + EngineSyncStateUpdate { + local_safe_head: Some(*safe_l2), + safe_head: Some(*safe_l2), + unsafe_head: Some(*safe_l2), + ..Default::default() + }, + "Updated safe head via follow safe", + ) + .await + } + + /// Moves the safe labels to a delegated safe head that the execution layer does not have yet, + /// leaving the unsafe head in place. + /// + /// A block missing below the unsafe head means the execution layer is still syncing toward the + /// unsafe head. Resetting would restart that sync from the execution layer's old labels, and + /// moving the unsafe head back would retarget it. Like op-node's follow-source mode, only the + /// safe labels follow the delegate, and the forkchoice update carries them to the execution + /// layer. + async fn follow_safe_head_during_el_sync( + &self, + state: &mut EngineState, + safe_l2: &L2BlockInfo, + ) -> Result<(), ConsolidateTaskError> { + warn!( + target: "engine", + safe_l2 = %safe_l2, + unsafe_number = state.sync_state.unsafe_head().block_info.number, + "Execution layer does not have the delegated safe head yet; keeping its sync target" + ); + + self.apply_delegated_safe_head( + state, + safe_l2, + EngineSyncStateUpdate { + local_safe_head: Some(*safe_l2), + safe_head: Some(*safe_l2), + ..Default::default() + }, + "Updated safe head while the execution layer syncs", + ) + .await + } + + /// Applies `update`, whose labels follow the delegated `safe_l2`, and always sends its + /// forkchoice update: the execution layer may have answered `SYNCING` to the same labels + /// before, or lost them in a restart. + async fn apply_delegated_safe_head( + &self, + state: &mut EngineState, + safe_l2: &L2BlockInfo, + update: EngineSyncStateUpdate, + outcome: &'static str, + ) -> Result<(), ConsolidateTaskError> { + synchronize_unconditionally(self.client.as_ref(), state, update) + .await + .map_err(|e| { + warn!(target: "engine", ?e, safe_l2 = %safe_l2, "Applying the delegated safe head failed"); + e + })?; + + info!( + target: "engine", + hash = %safe_l2.block_info.hash, + number = safe_l2.block_info.number, + "{outcome}" + ); + + Ok(()) + } + + /// Follows a delegated safe head below the safe head. Only a different block at its height in + /// a synced execution layer proves the delegate diverged and reorgs the engine onto it; the + /// same block (a lagging delegate) or a missing one leaves the labels. After a delegate L1 + /// reorg whose fork point is above the head it reports, the labels stay on the abandoned + /// blocks until it reports a head above that point. + async fn follow_delegated_safe_head_behind_safe_head( + &self, + state: &mut EngineState, + safe_l2: &L2BlockInfo, + ) -> Result<(), ConsolidateTaskError> { + // Until the execution layer applies the engine's forkchoice, its canonical block at that + // height may predate the engine's own labels, so it proves nothing about the delegate. + if !state.el_applied_last_forkchoice { + debug!( + target: "engine", + safe_l2 = %safe_l2, + "Ignoring a delegated safe head behind the safe head while the execution layer syncs" + ); + return Ok(()); + } + debug!( + target: "engine", + safe_l2 = %safe_l2, + "Looking up the execution layer's block at a delegated safe head behind the safe head" + ); + let el_block = + self.client.get_l2_block(safe_l2.block_info.number.into()).await.map_err(|err| { + warn!( + target: "engine", + ?err, + safe_l2 = %safe_l2, + "Failed to look up the execution layer's block at a delegated safe head" + ); + ConsolidateTaskError::FailedToFetchDelegatedSafeBlock(safe_l2.block_info.number) + })?; + debug!( + target: "engine", + safe_l2 = %safe_l2, + el_hash = ?el_block.as_ref().map(|block| block.header.hash), + "Looked up the execution layer's block at a delegated safe head" + ); + match el_block { + Some(el_block) if el_block.header.hash != safe_l2.block_info.hash => { + debug!( + target: "engine", + safe_l2 = %safe_l2, + el_hash = %el_block.header.hash, + "Delegated safe head behind the safe head diverges from the execution layer" + ); + self.reconcile_to_safe_head(state, safe_l2).await + } + _ => { + debug!( + target: "engine", + safe_l2 = %safe_l2, + safe_number = state.sync_state.safe_head().block_info.number, + "Ignoring a delegated safe head behind the safe head" + ); + Ok(()) + } + } + } + /// Handles the fallback case when the block doesn't match the input or does not exist. async fn reconcile_unsafe_to_safe( &self, @@ -110,37 +263,7 @@ impl ConsolidateTask { self.build_and_import_attributes(state, attributes).await } ConsolidateInput::BlockInfo(safe_l2) => { - warn!(target: "engine", safe_l2 = %safe_l2, "Apply safe head"); - - // We intentionally set the unsafe head to safe_l2 to ensure the engine observes a - // self-consistent head state. This is required to correctly handle reorgs (where - // unsafe may be ahead on a non-canonical fork) and to trigger EL sync when the - // local unsafe head lags behind the safe head. The forkchoice update is always - // sent: the execution layer may have answered `SYNCING` to the same labels before, - // or lost them in a restart. - synchronize_unconditionally( - self.client.as_ref(), - state, - EngineSyncStateUpdate { - local_safe_head: Some(*safe_l2), - safe_head: Some(*safe_l2), - unsafe_head: Some(*safe_l2), - ..Default::default() - }, - ) - .await - .map_err(|e| { - warn!(target: "engine", ?e, safe_l2 = %safe_l2, "Applying the delegated safe head failed"); - e - })?; - - info!( - target: "engine", - hash = %safe_l2.block_info.hash, - number = safe_l2.block_info.number, - "Updated safe head via follow safe" - ); - Ok(()) + self.reconcile_to_safe_head(state, safe_l2).await } } } @@ -160,39 +283,7 @@ impl ConsolidateTask { Ok(Some(block)) => block, Ok(None) => { if let ConsolidateInput::BlockInfo(safe_l2) = &self.input { - // A block missing below the unsafe head means the execution layer is still - // syncing toward the unsafe head. Resetting would restart that sync from the - // execution layer's old labels, and moving the unsafe head back would retarget - // it. Like op-node's follow-source mode, only the safe labels follow the - // delegate, and the forkchoice update, sent even when unchanged, carries them - // to the execution layer. - warn!( - target: "engine", - safe_l2 = %safe_l2, - unsafe_number = state.sync_state.unsafe_head().block_info.number, - "Execution layer does not have the delegated safe head yet; keeping its sync target" - ); - synchronize_unconditionally( - self.client.as_ref(), - state, - EngineSyncStateUpdate { - local_safe_head: Some(*safe_l2), - safe_head: Some(*safe_l2), - ..Default::default() - }, - ) - .await - .map_err(|e| { - warn!(target: "engine", ?e, safe_l2 = %safe_l2, "Applying the delegated safe head failed"); - e - })?; - info!( - target: "engine", - hash = %safe_l2.block_info.hash, - number = safe_l2.block_info.number, - "Updated safe head while the execution layer syncs" - ); - return Ok(()); + return self.follow_safe_head_during_el_sync(state, safe_l2).await; } warn!(target: "engine", "Received `None` block for {}", block_num); return Err(ConsolidateTaskError::MissingUnsafeL2Block(block_num)); @@ -278,6 +369,14 @@ impl ConsolidateTask { } } + // A different block from an execution layer that has not applied the engine's forkchoice + // may be from before its sync, so it does not prove the delegate diverged. + if let ConsolidateInput::BlockInfo(safe_l2) = &self.input && + !state.el_applied_last_forkchoice + { + return self.follow_safe_head_during_el_sync(state, safe_l2).await; + } + debug!( target: "engine", input = ?self.input, @@ -307,6 +406,19 @@ impl ExecutableEngineTask for ConsolidateTask { // injected safe head is ahead of the EngineActor's unsafe head, we reconcile the unsafe chain // up to the safe head instead of consolidating. async fn execute(&self, state: &mut EngineState) -> Result<(), ConsolidateTaskError> { + if let ConsolidateInput::BlockInfo(safe_l2) = &self.input { + state.sync_state.ensure_respects_finality(safe_l2.block_info.id())?; + // Labels with the unsafe head below the safe head take the paths below, which move the + // labels back to a consistent state; ignoring a lagging delegate would keep them broken + // until a finalization above the unsafe head fails. + let are_labels_consistent = state.sync_state.unsafe_head().block_info.number >= + state.sync_state.safe_head().block_info.number; + if are_labels_consistent && + safe_l2.block_info.number < state.sync_state.safe_head().block_info.number + { + return self.follow_delegated_safe_head_behind_safe_head(state, safe_l2).await; + } + } let safe_head_number = match &self.input { ConsolidateInput::Attributes { .. } => state.sync_state.safe_head().block_info.number, ConsolidateInput::BlockInfo(safe_block_info) => safe_block_info.block_info.number, @@ -323,7 +435,8 @@ impl ExecutableEngineTask for ConsolidateTask { mod tests { use super::*; use crate::{ - EngineSyncState, EngineTaskError, ImportedBlockSink, NoopBlockSink, SealTaskError, + EngineSyncState, EngineTaskError, FinalityViolation, ImportedBlockSink, NoopBlockSink, + SealTaskError, test_utils::{ TestAttributesBuilder, TestEngineStateBuilder, l1_info_deposit, test_block_info, test_engine_client, @@ -547,10 +660,9 @@ mod tests { el_has_block.then_some(block) } - /// The safe head before consolidation: either already the delegated target, as a `SYNCING` - /// answer or an earlier poll leaves it, or an older block. - fn initial_safe_head(starts_at_target: bool, target: L2BlockInfo) -> L2BlockInfo { - if starts_at_target { target } else { test_block_info(5) } + /// Either `target` itself or an older block, for a label that may already sit at the target. + fn target_or_older_block(is_at_target: bool, target: L2BlockInfo) -> L2BlockInfo { + if is_at_target { target } else { test_block_info(5) } } /// Below the unsafe head, a delegated safe head always reaches the execution layer: when the @@ -570,8 +682,10 @@ mod tests { ) { let (block, cfg) = genesis_pinned_block(10); let delegated_safe_head = block_info_of(&cfg, &block); - let mut state = - engine_state_with_safe_head(initial_safe_head(starts_at_target, delegated_safe_head)); + let mut state = engine_state_with_safe_head(target_or_older_block( + starts_at_target, + delegated_safe_head, + )); let expected_sync_state = with_safe_head(&state, delegated_safe_head); let (client, l1, l2) = test_engine_client(cfg.clone()); l2.expect_params( @@ -692,4 +806,326 @@ mod tests { l1.assert_finished(); l2.assert_finished(); } + + /// Where a delegated safe head sits against the finalized head. + #[derive(Clone, Copy, Debug)] + enum TargetAgainstFinalized { + Below, + ConflictingAt, + } + + impl TargetAgainstFinalized { + fn delegated_safe_head(self, finalized: L2BlockInfo) -> L2BlockInfo { + let number = finalized.block_info.number; + match self { + Self::Below => test_block_info(number - 1), + Self::ConflictingAt => L2BlockInfo { + block_info: BlockInfo { + hash: !finalized.block_info.hash, + ..finalized.block_info + }, + ..finalized + }, + } + } + } + + /// The outcome of checking a delegated safe head against finality. + #[derive(Clone, Copy, Debug, PartialEq, Eq)] + enum FinalityCheck { + Accepted, + BehindFinalizedHead, + ConflictsWithFinalizedHead, + } + + impl FinalityCheck { + fn of(result: &Result<(), ConsolidateTaskError>) -> Self { + match result { + Ok(()) => Self::Accepted, + Err(ConsolidateTaskError::BreaksFinality( + FinalityViolation::BehindFinalizedHead { .. }, + )) => Self::BehindFinalizedHead, + Err(ConsolidateTaskError::BreaksFinality( + FinalityViolation::ConflictsWithFinalizedHead { .. }, + )) => Self::ConflictsWithFinalizedHead, + Err(err) => panic!("unexpected error {err:?}"), + } + } + } + + /// A delegated safe head that breaks finality, as a stale delegate snapshot reports, is skipped + /// before the execution layer is asked anything, and leaves every label where it was. + #[rstest::rstest] + #[case::below(TargetAgainstFinalized::Below, FinalityCheck::BehindFinalizedHead)] + #[case::conflicting( + TargetAgainstFinalized::ConflictingAt, + FinalityCheck::ConflictsWithFinalizedHead + )] + #[tokio::test] + async fn delegated_safe_head_breaking_finality_is_skipped( + #[case] target: TargetAgainstFinalized, + #[case] expected: FinalityCheck, + ) { + let cfg = Arc::new(RollupConfig::default()); + let mut state = engine_state_with_safe_head(test_block_info(12)); + let state_before = state; + let delegated_safe_head = target.delegated_safe_head(state.sync_state.finalized_head()); + let (client, l1, l2) = test_engine_client(cfg.clone()); + let task = ConsolidateTask::new( + Arc::new(client), + cfg, + ConsolidateInput::BlockInfo(delegated_safe_head), + Arc::new(NoopBlockSink), + ); + + let result = task.execute(&mut state).await; + + assert_eq!(FinalityCheck::of(&result), expected); + assert_eq!(result.unwrap_err().severity(), crate::EngineTaskErrorSeverity::Skip); + assert_eq!(state, state_before); + l1.assert_finished(); + l2.assert_finished(); + } + + /// A delegated safe head from a lagging delegate, between the finalized head, inclusive, and + /// the safe head, does not move the safe labels back, whether or not the execution layer has + /// its block, and sends no forkchoice update. + #[rstest::rstest] + #[case::el_has_block(true, false)] + #[case::el_missing_block(false, false)] + #[case::at_finalized_head(true, true)] + #[tokio::test] + async fn lagging_delegated_safe_head_leaves_labels_unchanged( + #[case] el_has_block: bool, + #[case] finalized_at_target: bool, + ) { + let (block, cfg) = genesis_pinned_block(10); + let delegated_safe_head = block_info_of(&cfg, &block); + let mut state = TestEngineStateBuilder::new() + .with_unsafe_head(test_block_info(20)) + .with_safe_head(test_block_info(12)) + .with_finalized_head(target_or_older_block(finalized_at_target, delegated_safe_head)) + .build(); + let state_before = state; + let (client, l1, l2) = test_engine_client(cfg.clone()); + l2.expect_block_header_by_number(10, el_block(el_has_block, block)); + let sink = Arc::new(RecordingSink::default()); + let task = ConsolidateTask::new( + Arc::new(client), + cfg, + ConsolidateInput::BlockInfo(delegated_safe_head), + sink.clone(), + ); + + task.execute(&mut state).await.unwrap(); + + assert_eq!(state, state_before); + assert!(sink.0.lock().unwrap().is_empty()); + l1.assert_finished(); + l2.assert_finished(); + } + + /// A delegated safe head behind the safe head whose block differs from the execution layer's + /// means the delegate's chain diverged above it, so the engine still reorgs onto it, without + /// moving finalized. + #[tokio::test] + async fn diverged_delegated_safe_head_behind_safe_head_reorgs_the_engine() { + let (block, cfg) = genesis_pinned_block(10); + let delegated_safe_head = block_info_of(&cfg, &block); + let mut state = engine_state_with_safe_head(test_block_info(12)); + let finalized_before = state.sync_state.finalized_head(); + let expected_sync_state = state.sync_state.apply_update(EngineSyncStateUpdate { + local_safe_head: Some(delegated_safe_head), + safe_head: Some(delegated_safe_head), + unsafe_head: Some(delegated_safe_head), + ..Default::default() + }); + let (client, l1, l2) = test_engine_client(cfg.clone()); + l2.expect_block_header_by_number(10, Some(other_chain_block(10))); + l2.expect_forkchoice_updated( + expected_sync_state.create_forkchoice_state(), + PayloadStatusEnum::Valid, + ); + let task = ConsolidateTask::new( + Arc::new(client), + cfg, + ConsolidateInput::BlockInfo(delegated_safe_head), + Arc::new(NoopBlockSink), + ); + + task.execute(&mut state).await.unwrap(); + + assert_eq!(state.sync_state, expected_sync_state); + assert_eq!(state.sync_state.finalized_head(), finalized_before); + l1.assert_finished(); + l2.assert_finished(); + } + + /// A delegated safe head behind the safe head is not judged against an execution layer that has + /// not applied the engine's last forkchoice, whether it is in its first sync or syncing again + /// after answering `SYNCING`: its canonical block there may predate the engine's labels, so no + /// lookup is made and the labels stay. + #[rstest::rstest] + #[case::initial_sync(false, false)] + #[case::syncing_again(true, false)] + #[tokio::test] + async fn delegated_safe_head_behind_safe_head_is_ignored_while_the_el_syncs( + #[case] el_sync_finished: bool, + #[case] el_applied_last_forkchoice: bool, + ) { + let (block, cfg) = genesis_pinned_block(10); + let delegated_safe_head = block_info_of(&cfg, &block); + let mut state = TestEngineStateBuilder::new() + .with_unsafe_head(test_block_info(20)) + .with_safe_head(test_block_info(12)) + .with_finalized_head(test_block_info(5)) + .with_el_sync_finished(el_sync_finished) + .with_el_applied_last_forkchoice(el_applied_last_forkchoice) + .build(); + let state_before = state; + let (client, l1, l2) = test_engine_client(cfg.clone()); + let task = ConsolidateTask::new( + Arc::new(client), + cfg, + ConsolidateInput::BlockInfo(delegated_safe_head), + Arc::new(NoopBlockSink), + ); + + task.execute(&mut state).await.unwrap(); + + assert_eq!(state, state_before); + l1.assert_finished(); + l2.assert_finished(); + } + + /// An empty block at `number` whose hash differs from [`genesis_pinned_block`]'s at the same + /// height, standing for a block on another chain. + fn other_chain_block(number: u64) -> RpcBlock { + RpcBlock::new( + RpcHeader::new(alloy_consensus::Header { number, timestamp: 1, ..Default::default() }), + BlockTransactions::Full(vec![]), + ) + } + + /// A reorg onto a diverged delegated safe head that the execution layer refuses as an invalid + /// forkchoice state resets the engine, which rebuilds the labels from the execution layer. + #[tokio::test] + async fn rejected_reorg_onto_a_delegated_safe_head_resets_the_engine() { + let (block, cfg) = genesis_pinned_block(10); + let delegated_safe_head = block_info_of(&cfg, &block); + let mut state = engine_state_with_safe_head(test_block_info(12)); + let state_before = state; + let (client, l1, l2) = test_engine_client(cfg.clone()); + l2.expect_block_header_by_number(10, Some(other_chain_block(10))); + l2.expect_error_code( + "engine_forkchoiceUpdatedV3", + i64::from(alloy_rpc_types_engine::INVALID_FORK_CHOICE_STATE_ERROR), + ); + let task = ConsolidateTask::new( + Arc::new(client), + cfg, + ConsolidateInput::BlockInfo(delegated_safe_head), + Arc::new(NoopBlockSink), + ); + + let err = task.execute(&mut state).await.unwrap_err(); + + assert_eq!(err.severity(), crate::EngineTaskErrorSeverity::Reset, "got {err:?}"); + assert_eq!(state, state_before); + l1.assert_finished(); + l2.assert_finished(); + } + + /// Labels with the unsafe head rewound below the safe head are moved back into order by any + /// delegated safe head above finality instead of being kept: at or above the unsafe head the + /// engine reorgs onto it, and below it the matching block consolidates. Keeping them would let + /// a later finalization above the unsafe head fail fatally. + #[rstest::rstest] + #[case::at_or_above_unsafe_head(12, false)] + #[case::below_unsafe_head(8, true)] + #[tokio::test] + async fn delegated_safe_head_repairs_an_unsafe_head_below_the_safe_head( + #[case] delegated_number: u64, + #[case] is_below_unsafe_head: bool, + ) { + let (block, cfg) = genesis_pinned_block(delegated_number); + let delegated_safe_head = block_info_of(&cfg, &block); + let mut state = TestEngineStateBuilder::new() + .with_unsafe_head(test_block_info(10)) + .with_safe_head(test_block_info(15)) + .with_finalized_head(test_block_info(5)) + .build(); + let expected_sync_state = state.sync_state.apply_update(EngineSyncStateUpdate { + local_safe_head: Some(delegated_safe_head), + safe_head: Some(delegated_safe_head), + unsafe_head: (!is_below_unsafe_head).then_some(delegated_safe_head), + ..Default::default() + }); + let (client, l1, l2) = test_engine_client(cfg.clone()); + if is_below_unsafe_head { + l2.expect_params( + "eth_getBlockByNumber", + serde_json::json!([format!("{delegated_number:#x}"), true]), + block, + ); + } + l2.expect_forkchoice_updated( + expected_sync_state.create_forkchoice_state(), + PayloadStatusEnum::Valid, + ); + let task = ConsolidateTask::new( + Arc::new(client), + cfg, + ConsolidateInput::BlockInfo(delegated_safe_head), + Arc::new(NoopBlockSink), + ); + + task.execute(&mut state).await.unwrap(); + + assert_eq!(state.sync_state, expected_sync_state); + assert!( + state.sync_state.unsafe_head().block_info.number >= + state.sync_state.safe_head().block_info.number + ); + l1.assert_finished(); + l2.assert_finished(); + } + + /// A delegated safe head whose block differs from the execution layer's is not treated as a + /// divergence while the execution layer has not applied the engine's last forkchoice: only the + /// safe labels move to it, and the unsafe head stays the sync target. + #[tokio::test] + async fn mismatched_delegated_safe_head_does_not_reorg_while_the_el_syncs() { + let (block, cfg) = genesis_pinned_block(15); + let delegated_safe_head = block_info_of(&cfg, &block); + let mut state = TestEngineStateBuilder::new() + .with_unsafe_head(test_block_info(20)) + .with_safe_head(test_block_info(12)) + .with_finalized_head(test_block_info(5)) + .with_el_applied_last_forkchoice(false) + .build(); + let expected_sync_state = with_safe_head(&state, delegated_safe_head); + let (client, l1, l2) = test_engine_client(cfg.clone()); + l2.expect_params( + "eth_getBlockByNumber", + serde_json::json!(["0xf", true]), + other_chain_block(15), + ); + l2.expect_forkchoice_updated( + expected_sync_state.create_forkchoice_state(), + PayloadStatusEnum::Syncing, + ); + let task = ConsolidateTask::new( + Arc::new(client), + cfg, + ConsolidateInput::BlockInfo(delegated_safe_head), + Arc::new(NoopBlockSink), + ); + + task.execute(&mut state).await.unwrap(); + + assert_eq!(state.sync_state, expected_sync_state); + l1.assert_finished(); + l2.assert_finished(); + } } diff --git a/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/error.rs b/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/error.rs index 92eb39d2e96..3e07f755697 100644 --- a/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/error.rs +++ b/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/error.rs @@ -1,7 +1,8 @@ //! Contains error types for the [`crate::FinalizeTask`]. use crate::{ - EngineTaskError, SynchronizeTaskError, task_queue::tasks::task::EngineTaskErrorSeverity, + EngineTaskError, FinalityViolation, SynchronizeTaskError, + task_queue::tasks::task::EngineTaskErrorSeverity, }; use alloy_eips::BlockNumHash; use alloy_transport::{RpcError, TransportErrorKind}; @@ -11,14 +12,9 @@ use thiserror::Error; /// An error that occurs when running the [`crate::FinalizeTask`]. #[derive(Debug, Error)] pub enum FinalizeTaskError { - /// The target is below the finalized head, and finality never moves backward. - #[error("Finalization target {target} is behind the finalized head {finalized}")] - BehindFinalizedHead { - /// The number of the finalized head. - finalized: u64, - /// The number of the finalization target. - target: u64, - }, + /// The target would move the finalized head backward, or onto another chain. + #[error("Finalization target breaks finality: {0}")] + BreaksFinality(#[from] FinalityViolation), /// The block to finalize was not found. #[error("The block to finalize was not found: Number {0}")] BlockNotFound(u64), @@ -40,14 +36,6 @@ pub enum FinalizeTaskError { /// The number of the block the engine returned. returned: u64, }, - /// The target is a different block at the finalized head's height. - #[error("Finalization target {target:?} conflicts with the finalized head {finalized:?}")] - ConflictsWithFinalizedHead { - /// The finalized head. - finalized: BlockNumHash, - /// The finalization target. - target: BlockNumHash, - }, /// The forkchoice update call to finalize the block failed. #[error(transparent)] ForkchoiceUpdateFailed(#[from] SynchronizeTaskError), @@ -78,9 +66,8 @@ pub enum FinalizeTaskError { impl EngineTaskError for FinalizeTaskError { fn severity(&self) -> EngineTaskErrorSeverity { match self { - Self::BehindFinalizedHead { .. } | Self::BlockNotSafe { .. } | - Self::ConflictsWithFinalizedHead { .. } | + Self::BreaksFinality(_) | Self::NotCanonical { .. } | Self::NotYetAvailable { .. } => EngineTaskErrorSeverity::Skip, Self::BlockNotFound(_) | Self::BlockNumberMismatch { .. } | Self::FromBlock(_) => { diff --git a/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/task.rs b/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/task.rs index 30a08a102bd..d89e9c0c0b9 100644 --- a/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/task.rs +++ b/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/task.rs @@ -1,8 +1,8 @@ //! A task for finalizing an L2 block. use crate::{ - EngineClient, EngineState, ExecutableEngineTask, FinalizeBlockId, FinalizeTaskError, - state::EngineSyncStateUpdate, task_queue::synchronize, + EngineClient, EngineState, EngineSyncState, ExecutableEngineTask, FinalizeBlockId, + FinalizeTaskError, state::EngineSyncStateUpdate, task_queue::synchronize, }; use alloy_eips::BlockNumHash; use async_trait::async_trait; @@ -42,19 +42,14 @@ impl FinalizeTask { /// Whether `target` is the finalized head itself; a different block at its height is an error. fn is_already_finalized( - finalized_head: &L2BlockInfo, + sync_state: &EngineSyncState, target: BlockNumHash, ) -> Result { - let finalized = finalized_head.block_info.id(); - if target == finalized { + if target == sync_state.finalized_head().block_info.id() { debug!(target: "engine", ?target, "Target is already finalized"); return Ok(true); } - // The default head means finality has not been established yet, so any target, including - // the configured genesis, may initialize it. - if *finalized_head != L2BlockInfo::default() && target.number == finalized.number { - return Err(FinalizeTaskError::ConflictsWithFinalizedHead { finalized, target }); - } + sync_state.ensure_no_conflict_with_finality(target)?; Ok(false) } @@ -101,19 +96,13 @@ impl ExecutableEngineTask for FinalizeTask { type Error = FinalizeTaskError; async fn execute(&self, state: &mut EngineState) -> Result<(), FinalizeTaskError> { - let finalized_head = state.sync_state.finalized_head(); let target_number = self.block_id.number(); // A lagging source can report an older finalized block, or one from another chain. A // pinned hash is enough to detect either without asking the execution layer. - if target_number < finalized_head.block_info.number { - return Err(FinalizeTaskError::BehindFinalizedHead { - finalized: finalized_head.block_info.number, - target: target_number, - }); - } + state.sync_state.ensure_not_behind_finality(target_number)?; if let FinalizeBlockId::ByHash(target) = self.block_id && - Self::is_already_finalized(&finalized_head, target)? + Self::is_already_finalized(&state.sync_state, target)? { return Ok(()); } @@ -132,7 +121,7 @@ impl ExecutableEngineTask for FinalizeTask { // A by-number target only learns its hash from the lookup. if matches!(self.block_id, FinalizeBlockId::ByNumber(_)) && - Self::is_already_finalized(&finalized_head, block_info.block_info.id())? + Self::is_already_finalized(&state.sync_state, block_info.block_info.id())? { return Ok(()); } diff --git a/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/task_test.rs b/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/task_test.rs index 45420631095..5708a52c150 100644 --- a/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/task_test.rs +++ b/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/task_test.rs @@ -1,7 +1,7 @@ use crate::{ EngineState, EngineSyncState, EngineSyncStateUpdate, EngineTaskError, EngineTaskErrorSeverity::{self, Critical, Skip, Temporary}, - ExecutableEngineTask, FinalizeBlockId, FinalizeTask, FinalizeTaskError, + ExecutableEngineTask, FinalityViolation, FinalizeBlockId, FinalizeTask, FinalizeTaskError, test_utils::{ RpcMock, TestEngineStateBuilder, TestRpcBlock as RpcBlock, test_block_info, test_engine_client, test_l2_block, test_rpc_block, @@ -103,11 +103,20 @@ async fn finalize( } const fn is_behind(err: &FinalizeTaskError) -> bool { - matches!(err, FinalizeTaskError::BehindFinalizedHead { finalized: FINALIZED, target: STALE }) + matches!( + err, + FinalizeTaskError::BreaksFinality(FinalityViolation::BehindFinalizedHead { + finalized: BlockNumHash { number: FINALIZED, .. }, + target: STALE, + }) + ) } const fn is_conflict(err: &FinalizeTaskError) -> bool { - matches!(err, FinalizeTaskError::ConflictsWithFinalizedHead { .. }) + matches!( + err, + FinalizeTaskError::BreaksFinality(FinalityViolation::ConflictsWithFinalizedHead { .. }) + ) } const fn is_not_canonical(err: &FinalizeTaskError) -> bool { diff --git a/rust/kona/crates/node/engine/src/task_queue/tasks/synchronize/helper.rs b/rust/kona/crates/node/engine/src/task_queue/tasks/synchronize/helper.rs index a9bc15146ff..fc28ccf0026 100644 --- a/rust/kona/crates/node/engine/src/task_queue/tasks/synchronize/helper.rs +++ b/rust/kona/crates/node/engine/src/task_queue/tasks/synchronize/helper.rs @@ -100,6 +100,7 @@ async fn send_forkchoice( match &valid_response.payload_status.status { PayloadStatusEnum::Valid => { + state.el_applied_last_forkchoice = true; if !state.el_sync_finished { info!(target: "engine", "Finished execution layer sync."); state.el_sync_finished = true; @@ -107,6 +108,7 @@ async fn send_forkchoice( } // If we're not building a new payload, we're driving EL sync. PayloadStatusEnum::Syncing => { + state.el_applied_last_forkchoice = false; debug!(target: "engine", "Attempting to update forkchoice state while EL syncing") } // Other codes are not expected. diff --git a/rust/kona/crates/node/engine/src/test_utils/engine_state.rs b/rust/kona/crates/node/engine/src/test_utils/engine_state.rs index 60542823b96..1ed3c5b8178 100644 --- a/rust/kona/crates/node/engine/src/test_utils/engine_state.rs +++ b/rust/kona/crates/node/engine/src/test_utils/engine_state.rs @@ -10,6 +10,7 @@ pub struct TestEngineStateBuilder { local_safe_head: Option, safe_head: Option, finalized_head: Option, + el_applied_last_forkchoice: bool, el_sync_finished: bool, } @@ -33,6 +34,7 @@ impl TestEngineStateBuilder { local_safe_head: None, safe_head: None, finalized_head: None, + el_applied_last_forkchoice: true, el_sync_finished: true, } } @@ -55,13 +57,22 @@ impl TestEngineStateBuilder { self } - /// Sets whether EL sync is finished + /// Sets whether EL sync is finished. An execution layer still syncing has not applied the last + /// forkchoice either, so this sets both flags. #[allow(dead_code)] pub const fn with_el_sync_finished(mut self, finished: bool) -> Self { + self.el_applied_last_forkchoice = finished; self.el_sync_finished = finished; self } + /// Sets whether the execution layer applied the last forkchoice update. + #[allow(dead_code)] + pub const fn with_el_applied_last_forkchoice(mut self, applied: bool) -> Self { + self.el_applied_last_forkchoice = applied; + self + } + /// Builds the `EngineState` pub fn build(self) -> EngineState { let mut state = EngineState::default(); @@ -74,6 +85,7 @@ impl TestEngineStateBuilder { finalized_head: Some(self.finalized_head.unwrap_or(self.unsafe_head)), }); + state.el_applied_last_forkchoice = self.el_applied_last_forkchoice; state.el_sync_finished = self.el_sync_finished; state } diff --git a/rust/kona/crates/node/engine/src/test_utils/rpc.rs b/rust/kona/crates/node/engine/src/test_utils/rpc.rs index 0bd0bfef0c6..cda70c224a6 100644 --- a/rust/kona/crates/node/engine/src/test_utils/rpc.rs +++ b/rust/kona/crates/node/engine/src/test_utils/rpc.rs @@ -70,6 +70,16 @@ impl RpcMock { ); } + /// Expects a header-only `eth_getBlockByNumber` call for `number` and answers it with `block`, + /// or with null for `None`. + pub fn expect_block_header_by_number(&self, number: u64, block: Option) { + self.expect_params( + "eth_getBlockByNumber", + serde_json::json!([format!("{number:#x}"), false]), + block, + ); + } + /// Expects an `engine_forkchoiceUpdatedV3` call for `forkchoice` without payload attributes and /// answers it with `status`. pub fn expect_forkchoice_updated( diff --git a/rust/kona/crates/node/service/src/actors/derivation/delegated/actor.rs b/rust/kona/crates/node/service/src/actors/derivation/delegated/actor.rs index 480019925a0..37dc0e00185 100644 --- a/rust/kona/crates/node/service/src/actors/derivation/delegated/actor.rs +++ b/rust/kona/crates/node/service/src/actors/derivation/delegated/actor.rs @@ -6,7 +6,7 @@ use alloy_primitives::BlockHash; use async_trait::async_trait; use jsonrpsee::core::ClientError; use kona_derive::{ChainProvider, Signal}; -use kona_engine::{EngineState, FinalizeBlockId}; +use kona_engine::{EngineState, FinalityViolation, FinalizeBlockId}; use kona_protocol::{BlockInfo, L2BlockInfo, SyncStatus}; use kona_rpc::RollupNodeApiClient; use thiserror::Error; @@ -77,6 +77,15 @@ where has_engine_sync_completed: bool, /// Ticker driving periodic polls of the derivation delegate. delegated_derivation_ticker: time::Interval, + + /// A safe head behind the engine's that was forwarded while the execution layer applied the + /// engine's forkchoice, with the engine's safe head at the time. The engine checks such a head + /// against its execution layer once; until either changes, sending it again repeats that + /// lookup to the same answer. + forwarded_lagging_safe_head: Option<(BlockNumHash, BlockNumHash)>, + /// The last rejection of a delegate status, so a delegate stuck on one inconsistency is + /// warned about once rather than on every poll. + last_rejection: Option, } impl @@ -107,6 +116,8 @@ where engine_l2_safe_head: L2BlockInfo::default(), has_engine_sync_completed: false, delegated_derivation_ticker, + forwarded_lagging_safe_head: None, + last_rejection: None, } } } @@ -175,58 +186,44 @@ where Ok(()) } - /// The id of an established finalized head. The default head means finality has not been - /// established yet, so any finalized block from the delegate may initialize it. - fn established_finalized_l2(finalized_head: L2BlockInfo) -> Option { - (finalized_head != L2BlockInfo::default()).then(|| finalized_head.block_info.id()) + /// Rejects a status whose labels are out of order: its finalized block ahead of its safe head, + /// or its safe head ahead of its local safe head. No consistent delegate reports either, and a + /// finalized block ahead of the safe head would let a stale safe head sit below finality. A + /// zero local safe head is a label the delegate has not derived or does not report, so it does + /// not order anything. + fn validate_label_order(status: &SyncStatus) -> Result<(), DerivationDelegationError> { + let finalized = status.finalized_l2.block_info.number; + let safe = status.safe_l2.block_info.number; + let local_safe = if status.local_safe_l2 == L2BlockInfo::default() { + safe + } else { + status.local_safe_l2.block_info.number + }; + if finalized > safe { + return Err(DerivationDelegationError::FinalizedAheadOfSafe { finalized, safe }); + } + if safe > local_safe { + return Err(DerivationDelegationError::SafeAheadOfLocalSafe { local_safe, safe }); + } + Ok(()) } - /// Rejects a finalized L2 block behind, or at the same height as but different from, the - /// engine's. Either means the delegate lags or follows another chain, so none of its status - /// applies: its safe head would move the engine back or onto that chain too. + /// Verifies that the labels reported by the derivation delegate are in order, that its + /// finalized L2 block does not move back from the engine's, and that its L1 info is consistent + /// with the canonical L1 chain. /// /// Only a fork at the engine's finalized height is caught here. A chain that forked below it /// and reports a higher finalized block passes, and the execution layer rejects the forkchoice /// update that would move onto it. - fn validate_finalized_l2_progress( - &self, - reported: BlockNumHash, + async fn validate_sync_status( + &mut self, + v: &SyncStatus, ) -> Result<(), DerivationDelegationError> { - let engine_finalized_head = self.engine_state_rx.borrow().sync_state.finalized_head(); - let Some(engine) = Self::established_finalized_l2(engine_finalized_head) else { - return Ok(()); - }; - if reported.number < engine.number { - return Err(DerivationDelegationError::FinalizedL2Regressed { engine, reported }); - } - if reported.number == engine.number && reported.hash != engine.hash { - return Err(DerivationDelegationError::FinalizedL2Conflict { engine, reported }); - } - Ok(()) - } - - /// Verifies that the derivation delegate's finalized L2 block is not above its safe L2 block - /// and does not move back from the engine's, and that its L1 info is consistent with the - /// canonical L1 chain. - async fn validate_sync_status(&mut self, v: &SyncStatus) -> bool { - // A finalized block above the safe head is an inconsistent snapshot; the engine would only - // refuse to finalize it. - if v.finalized_l2.block_info.number > v.safe_l2.block_info.number { - let err = DerivationDelegationError::FinalizedAheadOfSafe { - finalized: v.finalized_l2.block_info.number, - safe: v.safe_l2.block_info.number, - }; - warn!(target: "derivation", error = %err, "Inconsistent sync status from delegate"); - return false; - } - if let Err(err) = self.validate_finalized_l2_progress(v.finalized_l2.block_info.id()) { - warn!( - target: "derivation", - error = %err, - "Finalized L2 inconsistency detected at sync status from delegate" - ); - return false; - } + Self::validate_label_order(v)?; + self.engine_state_rx + .borrow() + .sync_state + .ensure_respects_finality(v.finalized_l2.block_info.id())?; let checks = [ ("L1 Origin of Safe L2", v.safe_l2.l1_origin.number, v.safe_l2.l1_origin.hash), @@ -237,18 +234,52 @@ where ), ("Current L1", v.current_l1.number, v.current_l1.hash), ]; + let mut validated_l1_blocks = Vec::with_capacity(checks.len()); for (context, number, hash) in checks { - if let Err(err) = self.validate_l1_block(context, number, hash).await { - warn!( - target: "derivation", - context = context, - error = %err, - "L1 inconsistency detected at sync status from delegate" - ); - return false; + // Labels often share an L1 origin, and one lookup proves it canonical for all of them. + if validated_l1_blocks.contains(&(number, hash)) { + continue; } + self.validate_l1_block(context, number, hash).await?; + validated_l1_blocks.push((number, hash)); + } + Ok(()) + } + + /// Logs a rejected delegate status at `warn!` when the reason changed, and at `debug!` while + /// the delegate keeps reporting the same inconsistency. + fn log_rejected_status(&mut self, err: &DerivationDelegationError) { + let reason = err.to_string(); + if self.last_rejection.as_ref() == Some(&reason) { + debug!(target: "derivation", error = %err, "Sync status from delegate rejected again"); + } else { + warn!(target: "derivation", error = %err, "Rejected sync status from delegate"); + self.last_rejection = Some(reason); } - true + } + + /// Whether the delegated safe head is the lagging one already forwarded against the same + /// engine safe head, so the engine would repeat its lookup to the same answer. + fn is_lagging_safe_head_forwarded(&self, safe_l2: &L2BlockInfo) -> bool { + self.lagging_safe_head_to_remember(safe_l2) == self.forwarded_lagging_safe_head && + self.forwarded_lagging_safe_head.is_some() + } + + /// The pair to remember for a delegated safe head behind the engine's consistent safe head + /// while its execution layer applied the engine's forkchoice, or `None` for any other head, + /// which the engine must always see. + fn lagging_safe_head_to_remember( + &self, + safe_l2: &L2BlockInfo, + ) -> Option<(BlockNumHash, BlockNumHash)> { + let engine_state = self.engine_state_rx.borrow(); + let engine_safe_head = engine_state.sync_state.safe_head().block_info; + let is_consistent = + engine_state.sync_state.unsafe_head().block_info.number >= engine_safe_head.number; + (engine_state.el_applied_last_forkchoice && + is_consistent && + safe_l2.block_info.number < engine_safe_head.number) + .then(|| (safe_l2.block_info.id(), engine_safe_head.id())) } /// Fetches, validates, and applies sync status from the derivation delegate. @@ -265,18 +296,30 @@ where } }; - if !self.validate_sync_status(&sync_status).await { + if let Err(err) = self.validate_sync_status(&sync_status).await { // Validation failures here are expected to be transient, so we skip processing // this sync status and continue delegating derivation instead of treating it as // fatal. + self.log_rejected_status(&err); return Ok(()); } + self.last_rejection = None; self.derivation_origin_tx.send_replace(Some(sync_status.current_l1)); - self.engine_client - .send_safe_l2_signal(sync_status.safe_l2.into()) - .await - .map_err(|e| DerivationError::Sender(Box::new(e)))?; + if self.is_lagging_safe_head_forwarded(&sync_status.safe_l2) { + debug!( + target: "derivation", + safe_l2 = ?sync_status.safe_l2, + "Not forwarding a lagging safe head the engine already checked" + ); + } else { + self.forwarded_lagging_safe_head = + self.lagging_safe_head_to_remember(&sync_status.safe_l2); + self.engine_client + .send_safe_l2_signal(sync_status.safe_l2.into()) + .await + .map_err(|e| DerivationError::Sender(Box::new(e)))?; + } // Delegated polling supplies `(number, hash)`. Carry the hash through so the engine // finalizes the specific block we were asked to, not whatever it happens to have at the @@ -329,17 +372,14 @@ where #[derive(Error, Debug)] enum DerivationDelegationError { - /// The delegate's finalized L2 block is above its safe L2 block. + /// The delegate's finalized L2 block is ahead of its own safe head. #[error("Finalized L2 {finalized} is ahead of safe L2 {safe}")] FinalizedAheadOfSafe { finalized: u64, safe: u64 }, - /// The delegate's finalized L2 block is a different block at the engine's finalized height. - #[error("Finalized L2 {reported:?} conflicts with the engine's finalized L2 {engine:?}")] - FinalizedL2Conflict { engine: BlockNumHash, reported: BlockNumHash }, - - /// The delegate's finalized L2 block is behind the engine's. - #[error("Finalized L2 {reported:?} is behind the engine's finalized L2 {engine:?}")] - FinalizedL2Regressed { engine: BlockNumHash, reported: BlockNumHash }, + /// The delegate's finalized L2 block is behind the engine's, or a different block at its + /// height. + #[error("Finalized L2 moves the engine's finality back: {0}")] + FinalizedL2BreaksFinality(#[from] FinalityViolation), /// The L1 provider returned an error (network, RPC, etc.) #[error("L1 provider error: {0}")] @@ -348,4 +388,8 @@ enum DerivationDelegationError { /// The hash provided by the derivation delegation does not match the canonical chain. #[error("L1 inconsistency in {context} at block {number}: expected {expected}, got {actual}")] L1ValidationFailed { context: String, number: u64, expected: BlockHash, actual: BlockHash }, + + /// The delegate's safe head is ahead of its own local safe head. + #[error("Safe L2 {safe} is ahead of local safe L2 {local_safe}")] + SafeAheadOfLocalSafe { local_safe: u64, safe: u64 }, } diff --git a/rust/kona/crates/node/service/src/actors/derivation/delegated/tests.rs b/rust/kona/crates/node/service/src/actors/derivation/delegated/tests.rs index fea52b068d1..b586f878488 100644 --- a/rust/kona/crates/node/service/src/actors/derivation/delegated/tests.rs +++ b/rust/kona/crates/node/service/src/actors/derivation/delegated/tests.rs @@ -88,6 +88,15 @@ impl Harness { self.engine_state.send_modify(|state| state.sync_state = sync_state); } + /// Publishes the engine's labels along with whether its execution layer applied the engine's + /// last forkchoice. + fn publish_engine_state(&self, sync_state: EngineSyncState, el_applied_last_forkchoice: bool) { + self.engine_state.send_modify(|state| { + state.sync_state = sync_state; + state.el_applied_last_forkchoice = el_applied_last_forkchoice; + }); + } + /// The engine requests sent since the last call, in order. fn forwarded(&mut self) -> Vec { std::iter::from_fn(|| self.engine_requests.try_recv().ok()).map(Forwarded::from).collect() @@ -367,3 +376,92 @@ async fn poll_rejects_status_whose_finalized_is_ahead_of_its_safe( assert_eq!(harness.forwarded(), forwarded_if(accepted, &status)); } + +/// A status whose labels are out of order is rejected whole: a finalized block ahead of the safe +/// head would let a stale snapshot's safe head sit below finality, and a safe head ahead of the +/// local safe head is not a state any delegate reaches. +#[rstest] +#[case::ordered(15, 20, 20, true)] +#[case::finalized_at_safe(20, 20, 20, true)] +#[case::safe_behind_local_safe(15, 18, 20, true)] +#[case::finalized_ahead_of_safe(16, 15, 20, false)] +#[case::safe_ahead_of_local_safe(15, 21, 20, false)] +#[tokio::test(start_paused = true)] +async fn poll_rejects_status_with_labels_out_of_order( + #[case] finalized: u64, + #[case] safe: u64, + #[case] local_safe: u64, + #[case] accepted: bool, +) { + let status = SyncStatus { + finalized_l2: l2_block(finalized, FINALIZED_ORIGIN), + local_safe_l2: l2_block(local_safe, SAFE_ORIGIN), + safe_l2: l2_block(safe, SAFE_ORIGIN), + ..delegate_status() + }; + let mut harness = Harness::new([Ok(status.clone())]); + harness.complete_engine_sync().await; + + harness.actor.step().await.unwrap(); + + assert_eq!(harness.forwarded(), forwarded_if(accepted, &status)); + assert_eq!(harness.current_l1.borrow().is_some(), accepted); +} + +/// A zero local safe head is a label the delegate has not derived or does not report, so it does +/// not put the safe head out of order and the status is forwarded. +#[tokio::test(start_paused = true)] +async fn unreported_local_safe_head_does_not_order_the_labels() { + let status = SyncStatus { local_safe_l2: L2BlockInfo::default(), ..delegate_status() }; + let mut harness = Harness::new([Ok(status.clone())]); + harness.complete_engine_sync().await; + + harness.actor.step().await.unwrap(); + + assert_eq!(harness.forwarded(), forwarded_for(&status)); +} + +/// An engine sync state with consistent labels: unsafe at 30, safe at `safe`, finalized at 15. +fn engine_safe_at(safe: u64) -> EngineSyncState { + EngineSyncState::default().apply_update(EngineSyncStateUpdate { + finalized_head: Some(l2_block(15, FINALIZED_ORIGIN)), + local_safe_head: Some(l2_block(safe, SAFE_ORIGIN)), + safe_head: Some(l2_block(safe, SAFE_ORIGIN)), + unsafe_head: Some(l2_block(30, SAFE_ORIGIN)), + }) +} + +/// What a poll of `status` forwards when its safe head is held back: the finalized block alone. +fn finalized_only(status: &SyncStatus) -> Vec { + vec![Forwarded::Finalized(FinalizeBlockId::ByHash(status.finalized_l2.block_info.id()))] +} + +/// A safe head behind the engine's is forwarded once while the execution layer applied the +/// engine's forkchoice, since the engine's answer to it cannot change; finalized still goes out +/// on every poll. A moved engine safe head, or an execution layer that did not apply the last +/// forkchoice, sends it again. +#[rstest] +#[case::same_engine_view(true, 25, finalized_only)] +#[case::el_not_applied(false, 25, forwarded_for)] +#[case::engine_safe_moved(true, 26, forwarded_for)] +#[tokio::test(start_paused = true)] +async fn lagging_safe_head_is_forwarded_once_per_engine_view( + #[case] el_applied_last_forkchoice: bool, + #[case] engine_safe_on_second_poll: u64, + #[case] second_poll_forwards: fn(&SyncStatus) -> Vec, +) { + let status = delegate_status(); + let mut harness = Harness::new([Ok(status.clone()), Ok(status.clone())]); + harness.complete_engine_sync_with(engine_safe_at(25)).await; + harness.publish_engine_state(engine_safe_at(25), el_applied_last_forkchoice); + harness.actor.step().await.unwrap(); + assert_eq!(harness.forwarded(), forwarded_for(&status)); + harness.publish_engine_state( + engine_safe_at(engine_safe_on_second_poll), + el_applied_last_forkchoice, + ); + + harness.actor.step().await.unwrap(); + + assert_eq!(harness.forwarded(), second_poll_forwards(&status)); +} diff --git a/rust/kona/crates/node/service/src/actors/engine/actor.rs b/rust/kona/crates/node/service/src/actors/engine/actor.rs index f09047db8a2..b45bfa62238 100644 --- a/rust/kona/crates/node/service/src/actors/engine/actor.rs +++ b/rust/kona/crates/node/service/src/actors/engine/actor.rs @@ -235,7 +235,7 @@ where self.send_derivation_actor_safe_head_if_updated().await?; if !self.el_sync_complete && self.engine.state().el_sync_finished { - if self.engine.state().sync_state.finalized_head() == L2BlockInfo::default() { + if self.engine.state().sync_state.established_finalized_head().is_none() { self.begin_reset(); } else { self.mark_el_sync_complete_and_notify_derivation_actor().await?; From 9ad3406b912d69a34dbc0134113989f0571ec474 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Wed, 11 Nov 2026 18:16:47 +0100 Subject: [PATCH 5/6] kona-engine,kona-node-service: import delegated local-safe and cross-safe heads separately Under interop op-node imports a delegate's local-safe and cross-safe heads separately; kona-node copies the delegated safe head into both, so its forkchoice safe hash can run ahead of cross-chain verification. Kona-node now follows local-safe for the EL lookup, EL sync and reorgs, and moves the safe label to cross-safe only when it advances. Only the unsafe head and established finality count as proof of the chain; other heights are checked against an EL that applied the last forkchoice, and only at or below the unsafe head. A head that does not extend the unsafe chain reorgs directly, moving safe to cross-safe, clamped to finalized, unless the EL holds the current safe label, which a catch-up then keeps; only a conflict with finalized skips the update. While the EL has not applied the last forkchoice, safe only moves forward and nothing reorgs. Inconsistent heads are rejected, and the local-safe L1 origin must be canonical. Before interop both heads are one block, so non-interop chains see no change. A rejected update resets the engine; recovery is #24065. Closes #23911. BREAKING CHANGE: `ConsolidateInput::BlockInfo` is replaced by `Delegated(Box)`, built with `DelegatedSafeHeads::try_new`, which returns `InconsistentSafeHeadsError`; the `From` conversions are removed. Co-Authored-By: Claude Opus 5.5 --- .../kona-node/configuration.mdx | 20 + .../kona-node/design/engine.mdx | 2 +- .../bin/node/src/flags/engine/providers.rs | 3 + rust/kona/crates/node/engine/src/lib.rs | 9 +- .../tasks/consolidate/delegated_safe_heads.rs | 140 ++ .../src/task_queue/tasks/consolidate/error.rs | 3 +- .../src/task_queue/tasks/consolidate/mod.rs | 3 + .../src/task_queue/tasks/consolidate/task.rs | 1625 ++++++++++++----- .../node/engine/src/task_queue/tasks/mod.rs | 5 +- .../engine/src/test_utils/engine_state.rs | 15 +- .../crates/node/engine/src/test_utils/misc.rs | 12 + .../crates/node/engine/src/test_utils/mod.rs | 2 +- .../src/actors/derivation/delegated/actor.rs | 136 +- .../src/actors/derivation/delegated/tests.rs | 54 +- .../service/src/actors/engine/queue_tests.rs | 22 +- .../node/service/src/actors/engine/tests.rs | 9 +- 16 files changed, 1530 insertions(+), 530 deletions(-) create mode 100644 rust/kona/crates/node/engine/src/task_queue/tasks/consolidate/delegated_safe_heads.rs diff --git a/docs/public-docs/node-operators/kona-node/configuration.mdx b/docs/public-docs/node-operators/kona-node/configuration.mdx index d851596373c..1b4adb066f9 100644 --- a/docs/public-docs/node-operators/kona-node/configuration.mdx +++ b/docs/public-docs/node-operators/kona-node/configuration.mdx @@ -251,6 +251,26 @@ The RPC server starts once. If it stops unexpectedly, the node shuts down with a `kona-node` does not run a supervisor RPC server, so there is no `--supervisor.*` flag group. `--interop.dependency-set` is the node's only interop flag. +## Derivation Delegation Arguments + +| Flag | Env | Description | Default | +|------|-----|-------------|---------| +| `--l2-follow-source ` (alias `--l2.follow.source`) | `KONA_NODE_L2_FOLLOW_SOURCE` | RPC URL of an OP Stack consensus client exposing `optimism_syncStatus`. When set, `kona-node` derives nothing itself and follows that client's safe and finalized heads instead. Cannot be combined with `--safedb.path`. | - | + +Like op-node's follow-source mode, `kona-node` imports the source's local-safe and cross-safe +heads separately. The local-safe head is the block the execution client must have, or sync to, +and becomes the local-safe label; the cross-safe head becomes the safe label that forkchoice +updates carry, and finalization never goes above it. The safe label moves back to a lower +cross-safe head only when the source's chain has diverged from the node's, or when the unsafe head +moves to a source block whose ancestry the node cannot check and the execution client does not hold +the current safe label, and never below the finalized head. While the execution client has not +applied kona-node's last forkchoice update, for example while it is still syncing, its canonical +blocks do not count as evidence: a delegated block it lacks, or one that differs from its block, +moves the safe label only forward. +Before interop both heads are the same block. +Under interop, `kona-node` verifies no cross-chain dependencies itself, so its safe label is only +as safe as the source's cross-safe head: the source must verify them. + ## RPC Trust Flags The `--l1-trust-rpc` and `--l2-trust-rpc` flags control whether Kona verifies the block hashes of RPC responses. For guidance on when to disable trust and worked examples for trusted, untrusted, and mixed provider setups, see [configure RPC trust](/node-operators/kona-node/run/rpc-trust). diff --git a/docs/public-docs/node-operators/kona-node/design/engine.mdx b/docs/public-docs/node-operators/kona-node/design/engine.mdx index a8472c5fc4b..77003e49cc1 100644 --- a/docs/public-docs/node-operators/kona-node/design/engine.mdx +++ b/docs/public-docs/node-operators/kona-node/design/engine.mdx @@ -145,7 +145,7 @@ pub struct ConsolidateTask { pub client: Arc, /// The RollupConfig. pub cfg: Arc, - /// The input for consolidation (either attributes or block info). + /// The input for consolidation (either derived attributes or delegated safe heads). pub input: ConsolidateInput, /// Where to hand the decoded block once the engine has canonicalized it. pub block_sink: Arc, diff --git a/rust/kona/bin/node/src/flags/engine/providers.rs b/rust/kona/bin/node/src/flags/engine/providers.rs index 36bbc728ea0..f3fe5d762ae 100644 --- a/rust/kona/bin/node/src/flags/engine/providers.rs +++ b/rust/kona/bin/node/src/flags/engine/providers.rs @@ -103,6 +103,9 @@ impl Default for L2ClientArgs { #[derive(Clone, Debug, Default, clap::Args)] pub struct DerivationDelegateArgs { /// The source must be an OP Stack L2 CL RPC exposing `optimism_syncStatus`. + /// + /// Its local-safe and cross-safe heads become this node's local-safe and safe labels. Under + /// interop this node verifies no cross-chain dependencies itself, so the source must. #[arg(long, visible_alias = "l2.follow.source", env = "KONA_NODE_L2_FOLLOW_SOURCE")] pub l2_follow_source: Option, } diff --git a/rust/kona/crates/node/engine/src/lib.rs b/rust/kona/crates/node/engine/src/lib.rs index e8fd1f9305d..1f64ad82ac1 100644 --- a/rust/kona/crates/node/engine/src/lib.rs +++ b/rust/kona/crates/node/engine/src/lib.rs @@ -40,10 +40,11 @@ extern crate tracing; mod task_queue; pub use task_queue::{ BuildTask, BuildTaskError, CanonicalizeTask, CanonicalizeTaskError, ConsolidateInput, - ConsolidateTask, ConsolidateTaskError, Engine, EngineBuildError, EngineResetError, EngineTask, - EngineTaskError, EngineTaskErrorSeverity, EngineTaskErrors, ExecutableEngineTask, - FinalizeBlockId, FinalizeTask, FinalizeTaskError, InsertTask, InsertTaskError, SealTask, - SealTaskError, SynchronizeTaskError, + ConsolidateTask, ConsolidateTaskError, DelegatedSafeHeads, Engine, EngineBuildError, + EngineResetError, EngineTask, EngineTaskError, EngineTaskErrorSeverity, EngineTaskErrors, + ExecutableEngineTask, FinalizeBlockId, FinalizeTask, FinalizeTaskError, + InconsistentSafeHeadsError, InsertTask, InsertTaskError, SealTask, SealTaskError, + SynchronizeTaskError, }; mod attributes; diff --git a/rust/kona/crates/node/engine/src/task_queue/tasks/consolidate/delegated_safe_heads.rs b/rust/kona/crates/node/engine/src/task_queue/tasks/consolidate/delegated_safe_heads.rs new file mode 100644 index 00000000000..200c72b469c --- /dev/null +++ b/rust/kona/crates/node/engine/src/task_queue/tasks/consolidate/delegated_safe_heads.rs @@ -0,0 +1,140 @@ +//! The safe heads a derivation delegate reports. + +use alloy_eips::BlockNumHash; +use kona_protocol::L2BlockInfo; +use thiserror::Error; + +/// The safe heads a derivation delegate reports, imported as separate labels. +/// +/// Before interop both are the same block. Under interop the cross-safe head can lag the +/// local-safe head while the delegate verifies cross-chain dependencies, and only the cross-safe +/// head may reach the execution layer as its safe block. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct DelegatedSafeHeads { + local_safe_head: L2BlockInfo, + safe_head: L2BlockInfo, +} + +/// Safe heads that no consistent delegate reports. +#[derive(Debug, Clone, Copy, Error, PartialEq, Eq)] +pub enum InconsistentSafeHeadsError { + /// The cross-safe head is ahead of the local-safe head. + #[error("Safe head {safe} is ahead of local safe head {local_safe}")] + SafeAheadOfLocalSafe { + /// The number of the delegate's local-safe head. + local_safe: u64, + /// The number of the delegate's cross-safe head. + safe: u64, + }, + /// The cross-safe head is a different block at the local-safe head's height. + #[error("Safe head {safe:?} conflicts with local safe head {local_safe:?}")] + SafeConflictsWithLocalSafe { + /// The delegate's local-safe head. + local_safe: BlockNumHash, + /// The delegate's cross-safe head. + safe: BlockNumHash, + }, +} + +impl DelegatedSafeHeads { + /// Pairs the delegate's local-safe head, derived from L1 with cross-chain dependencies not + /// verified yet, with its cross-safe head, which must be at or below it on the same chain: not + /// ahead of it, and not a different block at its height. + pub fn try_new( + local_safe_head: L2BlockInfo, + safe_head: L2BlockInfo, + ) -> Result { + let local_safe = local_safe_head.block_info.id(); + let safe = safe_head.block_info.id(); + if safe.number > local_safe.number { + return Err(InconsistentSafeHeadsError::SafeAheadOfLocalSafe { + local_safe: local_safe.number, + safe: safe.number, + }); + } + if safe.number == local_safe.number && safe.hash != local_safe.hash { + return Err(InconsistentSafeHeadsError::SafeConflictsWithLocalSafe { + local_safe, + safe, + }); + } + Ok(Self { local_safe_head, safe_head }) + } + + /// The delegate's local-safe head. + pub const fn local_safe_head(&self) -> L2BlockInfo { + self.local_safe_head + } + + /// The delegate's cross-safe head. + pub const fn safe_head(&self) -> L2BlockInfo { + self.safe_head + } + + /// The cross-safe head when it is above `current_safe_head`, and `None` otherwise: a + /// cross-safe head at or below the safe label leaves it where it is. + pub(crate) const fn advancing_safe_head( + &self, + current_safe_head: &L2BlockInfo, + ) -> Option { + if self.safe_head.block_info.number > current_safe_head.block_info.number { + Some(self.safe_head) + } else { + None + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::test_utils::test_block_info; + use alloy_primitives::B256; + use kona_protocol::BlockInfo; + use rstest::rstest; + + const LOCAL_SAFE_HEAD: L2BlockInfo = L2BlockInfo { + block_info: BlockInfo { + hash: B256::repeat_byte(0x10), + number: 10, + parent_hash: B256::ZERO, + timestamp: 0, + }, + l1_origin: BlockNumHash { hash: B256::ZERO, number: 0 }, + seq_num: 0, + }; + + /// A different block at the height of `block`. + fn conflicting_block(block: L2BlockInfo) -> L2BlockInfo { + L2BlockInfo { + block_info: BlockInfo { hash: !block.block_info.hash, ..block.block_info }, + ..block + } + } + + #[rstest] + #[case::cross_safe_lags(test_block_info(8), true)] + #[case::same_block(LOCAL_SAFE_HEAD, true)] + #[case::conflicting_at_same_height(conflicting_block(LOCAL_SAFE_HEAD), false)] + #[case::cross_safe_ahead(test_block_info(11), false)] + fn try_new_rejects_cross_safe_off_the_local_safe_chain( + #[case] safe_head: L2BlockInfo, + #[case] accepted: bool, + ) { + let heads = DelegatedSafeHeads::try_new(LOCAL_SAFE_HEAD, safe_head); + + assert_eq!(heads.is_ok(), accepted, "got {heads:?}"); + } + + /// Only a cross-safe head strictly above the safe label advances it. + #[rstest] + #[case::above(8, true)] + #[case::same_height(7, false)] + #[case::below(6, false)] + fn advancing_safe_head_requires_a_higher_block(#[case] safe: u64, #[case] advances: bool) { + let heads = + DelegatedSafeHeads::try_new(test_block_info(10), test_block_info(safe)).unwrap(); + + assert_eq!(heads.advancing_safe_head(&test_block_info(7)).is_some(), advances); + } +} diff --git a/rust/kona/crates/node/engine/src/task_queue/tasks/consolidate/error.rs b/rust/kona/crates/node/engine/src/task_queue/tasks/consolidate/error.rs index 2ce1b187a76..0c9869ab7ae 100644 --- a/rust/kona/crates/node/engine/src/task_queue/tasks/consolidate/error.rs +++ b/rust/kona/crates/node/engine/src/task_queue/tasks/consolidate/error.rs @@ -12,8 +12,7 @@ pub enum ConsolidateTaskError { /// A delegated safe head would move the safe labels below finality. #[error("Delegated safe head breaks finality: {0}")] BreaksFinality(#[from] FinalityViolation), - /// Failed to fetch the execution layer's block at the height of a delegated safe head below the - /// safe head. + /// Failed to fetch the execution layer's block at the height of a delegated safe head. #[error("Failed to fetch the block at delegated safe head height {0}")] FailedToFetchDelegatedSafeBlock(u64), /// The unsafe L2 block is missing. diff --git a/rust/kona/crates/node/engine/src/task_queue/tasks/consolidate/mod.rs b/rust/kona/crates/node/engine/src/task_queue/tasks/consolidate/mod.rs index 21b125bd135..110983b601f 100644 --- a/rust/kona/crates/node/engine/src/task_queue/tasks/consolidate/mod.rs +++ b/rust/kona/crates/node/engine/src/task_queue/tasks/consolidate/mod.rs @@ -1,5 +1,8 @@ //! Consolidation Task +mod delegated_safe_heads; +pub use delegated_safe_heads::{DelegatedSafeHeads, InconsistentSafeHeadsError}; + mod error; pub use error::ConsolidateTaskError; diff --git a/rust/kona/crates/node/engine/src/task_queue/tasks/consolidate/task.rs b/rust/kona/crates/node/engine/src/task_queue/tasks/consolidate/task.rs index 00f31457729..f9d6c5dd2cc 100644 --- a/rust/kona/crates/node/engine/src/task_queue/tasks/consolidate/task.rs +++ b/rust/kona/crates/node/engine/src/task_queue/tasks/consolidate/task.rs @@ -1,10 +1,12 @@ //! A task to consolidate the engine state. use crate::{ - ConsolidateTaskError, EngineClient, EngineState, ExecutableEngineTask, ImportedBlockSink, + ConsolidateTaskError, DelegatedSafeHeads, EngineClient, EngineState, ExecutableEngineTask, + FinalityViolation, ImportedBlockSink, state::EngineSyncStateUpdate, task_queue::{build_and_import, synchronize, synchronize_unconditionally}, }; +use alloy_primitives::B256; use alloy_rpc_types_eth::Block; use async_trait::async_trait; use kona_genesis::RollupConfig; @@ -12,18 +14,18 @@ use kona_protocol::{L2BlockInfo, OpAttributesWithParent}; use op_alloy_rpc_types::Transaction; use std::sync::Arc; -/// Input for consolidation - either derived attributes or safe L2 block +/// Input for consolidation - either derived attributes or delegated safe heads #[derive(Debug, Clone)] pub enum ConsolidateInput { /// Consolidate based on derived attributes. Attributes(Box), - /// Derivation Delegation: consolidate based on safe L2 block info. - BlockInfo(L2BlockInfo), + /// Derivation Delegation: consolidate onto the safe heads a derivation delegate reports. + Delegated(Box), } -impl From for ConsolidateInput { - fn from(v: L2BlockInfo) -> Self { - Self::BlockInfo(v) +impl From for ConsolidateInput { + fn from(heads: DelegatedSafeHeads) -> Self { + Self::Delegated(Box::new(heads)) } } @@ -38,7 +40,7 @@ impl ConsolidateInput { const fn l2_block_number(&self) -> u64 { match self { Self::Attributes(attributes) => attributes.block_number(), - Self::BlockInfo(info) => info.block_info.number, + Self::Delegated(heads) => heads.local_safe_head().block_info.number, } } @@ -48,7 +50,7 @@ impl ConsolidateInput { Self::Attributes(attributes) => { crate::AttributesMatch::check(cfg, attributes, block).is_match() } - Self::BlockInfo(info) => block.header.hash == info.block_info.hash, + Self::Delegated(heads) => block.header.hash == heads.local_safe_head().block_info.hash, } } @@ -59,14 +61,14 @@ impl ConsolidateInput { } /// The [`ConsolidateTask`] attempts to consolidate the engine state -/// using the specified payload attributes or block info. +/// using the specified derived attributes or delegated safe heads. #[derive(Debug, Clone)] pub struct ConsolidateTask { /// The engine client. pub client: Arc, /// The [`RollupConfig`]. pub cfg: Arc, - /// The input for consolidation (either attributes or block info). + /// The input for consolidation (either derived attributes or delegated safe heads). pub input: ConsolidateInput, /// Where to hand the decoded block once the engine has canonicalized it. pub block_sink: Arc, @@ -100,29 +102,167 @@ impl ConsolidateTask { Ok(()) } - /// This provides symmetric fallback behavior with `build_and_import`. - async fn reconcile_to_safe_head( + /// The hash of the execution layer's canonical block at `number`, or `None` when it has none + /// there. Only the hash is needed, so the block is fetched without its transactions. + async fn el_block_hash(&self, number: u64) -> Result, ConsolidateTaskError> { + debug!(target: "engine", number, "Looking up the execution layer's block hash"); + let block = self.client.get_l2_block(number.into()).await.map_err(|err| { + warn!(target: "engine", ?err, number, "Failed to look up the execution layer's block"); + ConsolidateTaskError::FailedToFetchDelegatedSafeBlock(number) + })?; + let el_hash = block.map(|block| block.header.hash); + debug!(target: "engine", number, ?el_hash, "Looked up the execution layer's block hash"); + Ok(el_hash) + } + + /// The hash of the engine's unsafe or established finalized head at `number`, if one sits at + /// that height. Only these two pin the engine's chain: the unsafe head is the head the + /// execution layer follows, and finality cannot move, while the safe labels can stay on blocks + /// that a delegate's reorg abandoned. + fn chain_pinning_label_hash_at(state: &EngineState, number: u64) -> Option { + let sync_state = &state.sync_state; + Some(sync_state.unsafe_head()) + .filter(|unsafe_head| *unsafe_head != L2BlockInfo::default()) + .map(|unsafe_head| unsafe_head.block_info.id()) + .into_iter() + .chain(sync_state.established_finalized_head()) + .find(|label| label.number == number) + .map(|label| label.hash) + } + + /// The safe label when the cross-safe head cannot be checked against the engine's chain: the + /// cross-safe head only when it respects finality and advances the label. + fn unverified_advancing_safe_head( + state: &EngineState, + heads: &DelegatedSafeHeads, + ) -> Option { + Self::check_cross_safe_finality(state, heads) + .ok() + .and_then(|()| heads.advancing_safe_head(&state.sync_state.safe_head())) + } + + /// The one check of the delegated cross-safe head against finality. + fn check_cross_safe_finality( + state: &EngineState, + heads: &DelegatedSafeHeads, + ) -> Result<(), FinalityViolation> { + state.sync_state.ensure_respects_finality(heads.safe_head().block_info.id()) + } + + /// The safe label when the unsafe head moves onto a chain the current safe label is not proven + /// to be on: the cross-safe head, clamped to the finalized head; a conflict with finality + /// skips. + fn safe_label_off_engine_chain( + state: &EngineState, + heads: &DelegatedSafeHeads, + ) -> Result { + match Self::check_cross_safe_finality(state, heads) { + Ok(()) => Ok(heads.safe_head()), + Err(FinalityViolation::BehindFinalizedHead { .. }) => { + Ok(state.sync_state.finalized_head()) + } + Err(violation @ FinalityViolation::ConflictsWithFinalizedHead { .. }) => { + Err(violation.into()) + } + } + } + + /// The safe label for delegated safe heads whose local-safe head is on the engine's chain: the + /// cross-safe head when it advances the label, or when it replaces, at the label's height, a + /// safe label that a reorg left on abandoned blocks, and `None`, which keeps the label. + async fn safe_label_on_engine_chain( + &self, + state: &EngineState, + heads: &DelegatedSafeHeads, + ) -> Result, ConsolidateTaskError> { + if let Err(violation) = Self::check_cross_safe_finality(state, heads) { + debug!(target: "engine", %violation, ?heads, "Delegated cross-safe head breaks finality; keeping the safe label"); + return Ok(None); + } + let safe = state.sync_state.safe_head(); + let cross_safe = heads.safe_head(); + // Below the safe label the cross-safe head can neither advance nor replace it. + if cross_safe.block_info.number < safe.block_info.number { + return Ok(None); + } + + let is_known_on_engine_chain = cross_safe.block_info.id() == + heads.local_safe_head().block_info.id() || + cross_safe.block_info.id() == safe.block_info.id(); + let chain_hash = if is_known_on_engine_chain { + Some(cross_safe.block_info.hash) + } else { + self.engine_chain_hash_at(state, cross_safe.block_info.number).await? + }; + if chain_hash.is_some_and(|hash| hash != cross_safe.block_info.hash) { + debug!(target: "engine", ?heads, "Delegated cross-safe head is not on the chain of its local-safe head; keeping the safe label"); + return Ok(None); + } + let is_proven_on_engine_chain = chain_hash.is_some(); + let replaces_stale_safe_label = is_proven_on_engine_chain && + cross_safe.block_info.number == safe.block_info.number && + cross_safe.block_info.hash != safe.block_info.hash; + Ok(if replaces_stale_safe_label { + Some(cross_safe) + } else { + heads.advancing_safe_head(&safe) + }) + } + + /// The hash of the engine's chain at `number`: its unsafe or finalized head when one sits + /// there, and otherwise the execution layer's canonical block. `None` when nothing can tell. + async fn engine_chain_hash_at( + &self, + state: &EngineState, + number: u64, + ) -> Result, ConsolidateTaskError> { + if let Some(label_hash) = Self::chain_pinning_label_hash_at(state, number) { + return Ok(Some(label_hash)); + } + self.el_canonical_hash_at(state, number).await + } + + /// The hash of the execution layer's canonical block at `number`, only while that chain is the + /// engine's: the execution layer applied the engine's last forkchoice, and `number` is at or + /// below the unsafe head. Otherwise its canonical blocks may predate a sync, or sit above a + /// rewound unsafe head, and prove nothing, so `None` without a lookup. + async fn el_canonical_hash_at( + &self, + state: &EngineState, + number: u64, + ) -> Result, ConsolidateTaskError> { + let is_el_canonical_chain_current = state.el_applied_last_forkchoice && + number <= state.sync_state.unsafe_head().block_info.number; + if is_el_canonical_chain_current { self.el_block_hash(number).await } else { Ok(None) } + } + + /// Moves the unsafe and local-safe heads to the delegated local-safe head, and the safe label + /// to `safe_head`. + async fn reconcile_to_delegated_safe_heads( &self, state: &mut EngineState, - safe_l2: &L2BlockInfo, + heads: &DelegatedSafeHeads, + safe_head: Option, ) -> Result<(), ConsolidateTaskError> { - warn!( + debug!( target: "engine", - safe_l2 = %safe_l2, + local_safe_l2 = %heads.local_safe_head(), + safe_l2 = %heads.safe_head(), + ?safe_head, "Apply safe head" ); - // We intentionally set the unsafe head to safe_l2 to ensure the engine observes a - // self-consistent head state. This is required to correctly handle reorgs (where unsafe + // We intentionally set the unsafe head to the local-safe head to ensure the engine observes + // a self-consistent head state. This is required to correctly handle reorgs (where unsafe // may be ahead on a non-canonical fork) and to trigger EL sync when the local unsafe head // lags behind the safe head. - self.apply_delegated_safe_head( + self.apply_delegated_safe_heads( state, - safe_l2, + heads, EngineSyncStateUpdate { - local_safe_head: Some(*safe_l2), - safe_head: Some(*safe_l2), - unsafe_head: Some(*safe_l2), + local_safe_head: Some(heads.local_safe_head()), + safe_head, + unsafe_head: Some(heads.local_safe_head()), ..Default::default() }, "Updated safe head via follow safe", @@ -130,32 +270,86 @@ impl ConsolidateTask { .await } - /// Moves the safe labels to a delegated safe head that the execution layer does not have yet, - /// leaving the unsafe head in place. - /// - /// A block missing below the unsafe head means the execution layer is still syncing toward the - /// unsafe head. Resetting would restart that sync from the execution layer's old labels, and - /// moving the unsafe head back would retarget it. Like op-node's follow-source mode, only the - /// safe labels follow the delegate, and the forkchoice update carries them to the execution - /// layer. + /// Reorgs the engine onto delegated safe heads whose chain differs from the engine's, moving + /// the safe label to the cross-safe head even when lower, since the old safe block may be on + /// the abandoned chain, or to the finalized head when the cross-safe head is below it. + async fn reorg_onto_delegated_safe_heads( + &self, + state: &mut EngineState, + heads: &DelegatedSafeHeads, + ) -> Result<(), ConsolidateTaskError> { + let safe_head = Self::safe_label_off_engine_chain(state, heads)?; + warn!( + target: "engine", + local_safe_l2 = %heads.local_safe_head(), + %safe_head, + "Reorging onto the delegate's chain" + ); + self.reconcile_to_delegated_safe_heads(state, heads, Some(safe_head)).await + } + + /// Moves the engine to delegated safe heads at or above the unsafe head. Only the unsafe head + /// itself or its child extends the engine's chain; any other local-safe head has unknown + /// ancestry and reorgs, so the safe label follows it and stays an ancestor of the new head. + async fn extend_onto_delegated_safe_heads( + &self, + state: &mut EngineState, + heads: &DelegatedSafeHeads, + ) -> Result<(), ConsolidateTaskError> { + let unsafe_head = state.sync_state.unsafe_head().block_info; + let local_safe = heads.local_safe_head().block_info; + let is_at_unsafe_height = local_safe.number == unsafe_head.number; + if is_at_unsafe_height && local_safe.hash != unsafe_head.hash { + return self.reorg_onto_delegated_safe_heads(state, heads).await; + } + let is_child_of_unsafe_head = local_safe.number == unsafe_head.number + 1 && + local_safe.parent_hash == unsafe_head.hash; + let is_unproven_extension = !is_at_unsafe_height && !is_child_of_unsafe_head; + let would_lower_safe_label = + heads.safe_head().block_info.number < state.sync_state.safe_head().block_info.number; + if is_unproven_extension && + !(would_lower_safe_label && self.is_safe_label_on_el_chain(state).await?) + { + return self.reorg_onto_delegated_safe_heads(state, heads).await; + } + + let safe_head = self.safe_label_on_engine_chain(state, heads).await?; + self.reconcile_to_delegated_safe_heads(state, heads, safe_head).await + } + + /// Whether the execution layer's canonical chain holds the engine's safe label. A local-safe + /// head several blocks above the unsafe head has ancestry the engine cannot check, but a + /// delegate catching up builds on the safe label the execution layer holds, so that label stays + /// rather than dropping to a lower cross-safe head. + async fn is_safe_label_on_el_chain( + &self, + state: &EngineState, + ) -> Result { + let safe = state.sync_state.safe_head().block_info; + Ok(self.el_canonical_hash_at(state, safe.number).await? == Some(safe.hash)) + } + + /// Moves the safe labels to a delegated local-safe head that the syncing execution layer does + /// not have yet, keeping the unsafe head as its sync target; the cross-safe head only advances. async fn follow_safe_head_during_el_sync( &self, state: &mut EngineState, - safe_l2: &L2BlockInfo, + heads: &DelegatedSafeHeads, ) -> Result<(), ConsolidateTaskError> { warn!( target: "engine", - safe_l2 = %safe_l2, + local_safe_l2 = %heads.local_safe_head(), unsafe_number = state.sync_state.unsafe_head().block_info.number, "Execution layer does not have the delegated safe head yet; keeping its sync target" ); - self.apply_delegated_safe_head( + let safe_head = Self::unverified_advancing_safe_head(state, heads); + self.apply_delegated_safe_heads( state, - safe_l2, + heads, EngineSyncStateUpdate { - local_safe_head: Some(*safe_l2), - safe_head: Some(*safe_l2), + local_safe_head: Some(heads.local_safe_head()), + safe_head, ..Default::default() }, "Updated safe head while the execution layer syncs", @@ -163,97 +357,101 @@ impl ConsolidateTask { .await } - /// Applies `update`, whose labels follow the delegated `safe_l2`, and always sends its + /// Applies `update`, whose labels follow the delegated `heads`, and always sends its /// forkchoice update: the execution layer may have answered `SYNCING` to the same labels /// before, or lost them in a restart. - async fn apply_delegated_safe_head( + async fn apply_delegated_safe_heads( &self, state: &mut EngineState, - safe_l2: &L2BlockInfo, + heads: &DelegatedSafeHeads, update: EngineSyncStateUpdate, outcome: &'static str, ) -> Result<(), ConsolidateTaskError> { - synchronize_unconditionally(self.client.as_ref(), state, update) - .await - .map_err(|e| { - warn!(target: "engine", ?e, safe_l2 = %safe_l2, "Applying the delegated safe head failed"); - e - })?; + synchronize_unconditionally(self.client.as_ref(), state, update).await.map_err(|e| { + warn!(target: "engine", ?e, ?heads, "Applying the delegated safe heads failed"); + e + })?; info!( target: "engine", - hash = %safe_l2.block_info.hash, - number = safe_l2.block_info.number, + local_safe_hash = %heads.local_safe_head().block_info.hash, + local_safe_number = heads.local_safe_head().block_info.number, + safe_hash = %state.sync_state.safe_head().block_info.hash, + safe_number = state.sync_state.safe_head().block_info.number, "{outcome}" ); Ok(()) } - /// Follows a delegated safe head below the safe head. Only a different block at its height in - /// a synced execution layer proves the delegate diverged and reorgs the engine onto it; the - /// same block (a lagging delegate) or a missing one leaves the labels. After a delegate L1 - /// reorg whose fork point is above the head it reports, the labels stay on the abandoned - /// blocks until it reports a head above that point. - async fn follow_delegated_safe_head_behind_safe_head( + /// Follows delegated safe heads below the engine's local-safe head. Only a different block at + /// that height in a synced execution layer proves divergence and reorgs; otherwise the + /// cross-safe head may only move the safe label. After a delegate L1 reorg whose fork point is + /// above its reported head, the labels stay on the abandoned blocks until it passes that point. + async fn follow_lagging_delegated_safe_heads( &self, state: &mut EngineState, - safe_l2: &L2BlockInfo, + heads: &DelegatedSafeHeads, ) -> Result<(), ConsolidateTaskError> { - // Until the execution layer applies the engine's forkchoice, its canonical block at that - // height may predate the engine's own labels, so it proves nothing about the delegate. - if !state.el_applied_last_forkchoice { + let local_safe = heads.local_safe_head().block_info; + let el_hash = self.el_canonical_hash_at(state, local_safe.number).await?; + let safe_head = match el_hash { + Some(el_hash) if el_hash != local_safe.hash => { + debug!(target: "engine", ?heads, %el_hash, "Delegated local-safe head behind the engine's diverges from the execution layer"); + return self.reorg_onto_delegated_safe_heads(state, heads).await; + } + Some(_) => self.safe_label_on_engine_chain(state, heads).await?, + None => Self::unverified_advancing_safe_head(state, heads), + }; + let Some(safe_head) = safe_head else { debug!( target: "engine", - safe_l2 = %safe_l2, - "Ignoring a delegated safe head behind the safe head while the execution layer syncs" + ?heads, + local_safe_number = state.sync_state.local_safe_head().block_info.number, + "Ignoring delegated safe heads behind the local-safe head" ); return Ok(()); - } - debug!( - target: "engine", - safe_l2 = %safe_l2, - "Looking up the execution layer's block at a delegated safe head behind the safe head" - ); - let el_block = - self.client.get_l2_block(safe_l2.block_info.number.into()).await.map_err(|err| { - warn!( - target: "engine", - ?err, - safe_l2 = %safe_l2, - "Failed to look up the execution layer's block at a delegated safe head" - ); - ConsolidateTaskError::FailedToFetchDelegatedSafeBlock(safe_l2.block_info.number) - })?; - debug!( - target: "engine", - safe_l2 = %safe_l2, - el_hash = ?el_block.as_ref().map(|block| block.header.hash), - "Looked up the execution layer's block at a delegated safe head" - ); - match el_block { - Some(el_block) if el_block.header.hash != safe_l2.block_info.hash => { - debug!( - target: "engine", - safe_l2 = %safe_l2, - el_hash = %el_block.header.hash, - "Delegated safe head behind the safe head diverges from the execution layer" - ); - self.reconcile_to_safe_head(state, safe_l2).await + }; + self.apply_delegated_safe_heads( + state, + heads, + EngineSyncStateUpdate { safe_head: Some(safe_head), ..Default::default() }, + "Updated safe head from a delegate behind the local-safe head", + ) + .await + } + + /// The safe label after consolidating onto `consolidated`, which the execution layer has: that + /// block for derived attributes, and the judged cross-safe head for a delegate. + async fn consolidated_safe_head( + &self, + state: &EngineState, + consolidated: L2BlockInfo, + ) -> Result, ConsolidateTaskError> { + let ConsolidateInput::Delegated(heads) = &self.input else { + return Ok(Some(consolidated)); + }; + self.safe_label_on_engine_chain(state, heads).await + } + + /// Handles a block that does not match the input: derived attributes build their own block, + /// and delegated safe heads reorg the engine onto the delegate's chain. + async fn reconcile_mismatch( + &self, + state: &mut EngineState, + ) -> Result<(), ConsolidateTaskError> { + match &self.input { + ConsolidateInput::Attributes(attributes) => { + self.build_and_import_attributes(state, attributes).await } - _ => { - debug!( - target: "engine", - safe_l2 = %safe_l2, - safe_number = state.sync_state.safe_head().block_info.number, - "Ignoring a delegated safe head behind the safe head" - ); - Ok(()) + ConsolidateInput::Delegated(heads) => { + self.reorg_onto_delegated_safe_heads(state, heads).await } } } - /// Handles the fallback case when the block doesn't match the input or does not exist. + /// Handles input at or above the unsafe head: derived attributes build their block, and + /// delegated safe heads move the engine onto them. async fn reconcile_unsafe_to_safe( &self, state: &mut EngineState, @@ -262,8 +460,8 @@ impl ConsolidateTask { ConsolidateInput::Attributes(attributes) => { self.build_and_import_attributes(state, attributes).await } - ConsolidateInput::BlockInfo(safe_l2) => { - self.reconcile_to_safe_head(state, safe_l2).await + ConsolidateInput::Delegated(heads) => { + self.extend_onto_delegated_safe_heads(state, heads).await } } } @@ -282,8 +480,8 @@ impl ConsolidateTask { let block = match self.client.l2_block_by_label(block_num.into()).await { Ok(Some(block)) => block, Ok(None) => { - if let ConsolidateInput::BlockInfo(safe_l2) = &self.input { - return self.follow_safe_head_during_el_sync(state, safe_l2).await; + if let ConsolidateInput::Delegated(heads) = &self.input { + return self.follow_safe_head_during_el_sync(state, heads).await; } warn!(target: "engine", "Received `None` block for {}", block_num); return Err(ConsolidateTaskError::MissingUnsafeL2Block(block_num)); @@ -294,98 +492,107 @@ impl ConsolidateTask { }; let block_hash = block.header.hash; - if self.input.is_consistent_with_block(&self.cfg, &block) { - trace!( + if !self.input.is_consistent_with_block(&self.cfg, &block) { + // A different block from an execution layer that has not applied the engine's + // forkchoice may be from before its sync, so it does not prove the delegate diverged. + if let ConsolidateInput::Delegated(heads) = &self.input && + !state.el_applied_last_forkchoice + { + return self.follow_safe_head_during_el_sync(state, heads).await; + } + debug!( target: "engine", input = ?self.input, block_hash = %block_hash, - "Consolidating engine state", + "ConsolidateInput mismatch! Initiating reorg", ); - let consensus_block = - block.into_consensus().map_transactions(|t| t.inner.inner.into_inner()); - match L2BlockInfo::from_block_and_genesis(&consensus_block, &self.cfg.genesis) { - // A span batch's blocks before its last one only move the safe label; the last - // block's forkchoice update delivers them all. This avoids sending a FCU call for - // every block in the span batch. - Ok(block_info) if self.input.is_intermediate_span_block() => { - // The next attributes built are this block's child, and ask for its config. - self.block_sink.block_imported(consensus_block, block_info); - - // Apply a transient update to the safe head. - state.sync_state = state.sync_state.apply_update(EngineSyncStateUpdate { - safe_head: Some(block_info), - local_safe_head: Some(block_info), - ..Default::default() - }); - - info!( - target: "engine", - hash = %block_info.block_info.hash, - number = block_info.block_info.number, - "Updated safe head via L1 consolidation" - ); - - return Ok(()); - } - Ok(block_info) => { - // The next attributes built are this block's child, and ask for its config. - self.block_sink.block_imported(consensus_block, block_info); - - let update = EngineSyncStateUpdate { - safe_head: Some(block_info), - local_safe_head: Some(block_info), - ..Default::default() - }; - let synchronized = match &self.input { - // Derived attributes move the safe head on every consolidation, so an - // unchanged forkchoice was already sent. - ConsolidateInput::Attributes(_) => { - synchronize(self.client.as_ref(), state, update).await - } - // A delegated safe head can repeat a forkchoice the execution layer - // answered `SYNCING` or lost, so it is always sent. - ConsolidateInput::BlockInfo(_) => { - synchronize_unconditionally(self.client.as_ref(), state, update).await - } - }; - synchronized.map_err(|e| { - warn!(target: "engine", ?e, "Consolidation failed"); - e - })?; - - info!( - target: "engine", - hash = %block_info.block_info.hash, - number = block_info.block_info.number, - "Updated safe head via L1 consolidation" - ); - - return Ok(()); - } - Err(e) => { - // Continue on to build the block since we failed to construct the block info. - warn!(target: "engine", ?e, "Failed to construct L2BlockInfo, proceeding to build task"); - } - } - } - - // A different block from an execution layer that has not applied the engine's forkchoice - // may be from before its sync, so it does not prove the delegate diverged. - if let ConsolidateInput::BlockInfo(safe_l2) = &self.input && - !state.el_applied_last_forkchoice - { - return self.follow_safe_head_during_el_sync(state, safe_l2).await; + return self.reconcile_mismatch(state).await; } - debug!( + trace!( target: "engine", input = ?self.input, block_hash = %block_hash, - "ConsolidateInput mismatch! Initiating reorg", + "Consolidating engine state", + ); + let consensus_block = + block.into_consensus().map_transactions(|t| t.inner.inner.into_inner()); + let block_info = match ( + L2BlockInfo::from_block_and_genesis(&consensus_block, &self.cfg.genesis), + &self.input, + ) { + (Ok(block_info), _) => block_info, + // The block matches the delegated local-safe head by hash, so the delegate's info + // describes it, and the block stays where it is on the engine's chain. + (Err(e), ConsolidateInput::Delegated(heads)) => { + warn!(target: "engine", ?e, "Failed to construct L2BlockInfo, using the delegated local-safe head's"); + heads.local_safe_head() + } + (Err(e), ConsolidateInput::Attributes(attributes)) => { + // Continue on to build the block since we failed to construct the block info. + warn!(target: "engine", ?e, "Failed to construct L2BlockInfo, proceeding to build task"); + return self.build_and_import_attributes(state, attributes).await; + } + }; + + // A span batch's blocks before its last one only move the safe label; the last block's + // forkchoice update delivers them all. This avoids sending a FCU call for every block in + // the span batch. + if self.input.is_intermediate_span_block() { + // The next attributes built are this block's child, and ask for its config. + self.block_sink.block_imported(consensus_block, block_info); + + // Apply a transient update to the safe head. + state.sync_state = state.sync_state.apply_update(EngineSyncStateUpdate { + local_safe_head: Some(block_info), + safe_head: Some(block_info), + ..Default::default() + }); + + info!( + target: "engine", + hash = %block_info.block_info.hash, + number = block_info.block_info.number, + "Updated safe head via L1 consolidation" + ); + + return Ok(()); + } + + let safe_head = self.consolidated_safe_head(state, block_info).await?; + // The next attributes built are this block's child, and ask for its config. + self.block_sink.block_imported(consensus_block, block_info); + + let update = EngineSyncStateUpdate { + local_safe_head: Some(block_info), + safe_head, + ..Default::default() + }; + let synchronized = match &self.input { + // Derived attributes move the safe head on every consolidation, so an unchanged + // forkchoice was already sent. + ConsolidateInput::Attributes(_) => { + synchronize(self.client.as_ref(), state, update).await + } + // A delegated safe head can repeat a forkchoice the execution layer answered `SYNCING` + // or lost, so it is always sent. + ConsolidateInput::Delegated(_) => { + synchronize_unconditionally(self.client.as_ref(), state, update).await + } + }; + synchronized.map_err(|e| { + warn!(target: "engine", ?e, "Consolidation failed"); + e + })?; + + info!( + target: "engine", + hash = %block_info.block_info.hash, + number = block_info.block_info.number, + "Updated safe head via L1 consolidation" ); - // Handle mismatch case - called when consistency check fails - // or when L2BlockInfo construction fails in Attributes branch - self.reconcile_unsafe_to_safe(state).await + + Ok(()) } } @@ -401,29 +608,36 @@ impl ExecutableEngineTask for ConsolidateTask { // DerivationActor and EngineActor coordinate both safe and unsafe heads. In this case, we // consolidate as long as the unsafe head has not fallen behind. // - // - `BlockInfo`: The safe head is injected externally by the DerivationActor while delegating - // derivation, and is not coordinated with the EngineActor's safe/unsafe heads. If the - // injected safe head is ahead of the EngineActor's unsafe head, we reconcile the unsafe chain - // up to the safe head instead of consolidating. + // - `Delegated`: The safe heads are injected externally by the DerivationActor while delegating + // derivation, and are not coordinated with the EngineActor's heads. A local-safe head below + // the engine's comes from a lagging or diverged delegate; one below the unsafe head is + // consolidated against the execution layer's block; one at or above the unsafe head moves the + // unsafe chain onto it. async fn execute(&self, state: &mut EngineState) -> Result<(), ConsolidateTaskError> { - if let ConsolidateInput::BlockInfo(safe_l2) = &self.input { - state.sync_state.ensure_respects_finality(safe_l2.block_info.id())?; - // Labels with the unsafe head below the safe head take the paths below, which move the - // labels back to a consistent state; ignoring a lagging delegate would keep them broken - // until a finalization above the unsafe head fails. - let are_labels_consistent = state.sync_state.unsafe_head().block_info.number >= - state.sync_state.safe_head().block_info.number; - if are_labels_consistent && - safe_l2.block_info.number < state.sync_state.safe_head().block_info.number + let ConsolidateInput::Delegated(heads) = &self.input else { + return if state.sync_state.safe_head().block_info.number < + state.sync_state.unsafe_head().block_info.number { - return self.follow_delegated_safe_head_behind_safe_head(state, safe_l2).await; - } - } - let safe_head_number = match &self.input { - ConsolidateInput::Attributes { .. } => state.sync_state.safe_head().block_info.number, - ConsolidateInput::BlockInfo(safe_block_info) => safe_block_info.block_info.number, + self.consolidate(state).await + } else { + self.reconcile_unsafe_to_safe(state).await + }; }; - if safe_head_number < state.sync_state.unsafe_head().block_info.number { + + let local_safe_number = heads.local_safe_head().block_info.number; + state.sync_state.ensure_respects_finality(heads.local_safe_head().block_info.id())?; + // Labels with the unsafe head below the safe labels take the paths below, which move them + // back into order; ignoring a lagging delegate would keep them broken until a finalization + // above the unsafe head fails. + let unsafe_number = state.sync_state.unsafe_head().block_info.number; + let are_labels_consistent = unsafe_number >= + state.sync_state.local_safe_head().block_info.number && + unsafe_number >= state.sync_state.safe_head().block_info.number; + if are_labels_consistent && + local_safe_number < state.sync_state.local_safe_head().block_info.number + { + self.follow_lagging_delegated_safe_heads(state, heads).await + } else if local_safe_number < state.sync_state.unsafe_head().block_info.number { self.consolidate(state).await } else { self.reconcile_unsafe_to_safe(state).await @@ -435,11 +649,11 @@ impl ExecutableEngineTask for ConsolidateTask { mod tests { use super::*; use crate::{ - EngineSyncState, EngineTaskError, FinalityViolation, ImportedBlockSink, NoopBlockSink, - SealTaskError, + EngineSyncState, EngineTaskError, EngineTaskErrorSeverity, FinalityViolation, + ImportedBlockSink, NoopBlockSink, SealTaskError, test_utils::{ - TestAttributesBuilder, TestEngineStateBuilder, l1_info_deposit, test_block_info, - test_engine_client, + RpcMock, TestAttributesBuilder, TestEngineStateBuilder, delegated_safe_heads, + l1_info_deposit, test_block_info, test_engine_client, }, }; use alloy_consensus::transaction::Recovered; @@ -481,7 +695,7 @@ mod tests { }; let sink = Arc::new(RecordingSink::default()); let task = - ConsolidateTask::new(client, cfg, ConsolidateInput::BlockInfo(safe_l2), sink.clone()); + ConsolidateTask::new(client, cfg, delegated_safe_heads(safe_l2, safe_l2), sink.clone()); task.consolidate(&mut EngineState::default()).await.unwrap(); @@ -622,6 +836,7 @@ mod tests { fn engine_state_with_safe_head(safe_head: L2BlockInfo) -> EngineState { TestEngineStateBuilder::new() .with_unsafe_head(test_block_info(20)) + .with_local_safe_head(safe_head) .with_safe_head(safe_head) .with_finalized_head(test_block_info(5)) .build() @@ -660,9 +875,10 @@ mod tests { el_has_block.then_some(block) } - /// Either `target` itself or an older block, for a label that may already sit at the target. - fn target_or_older_block(is_at_target: bool, target: L2BlockInfo) -> L2BlockInfo { - if is_at_target { target } else { test_block_info(5) } + /// The safe head before consolidation: either already the delegated target, as a `SYNCING` + /// answer or an earlier poll leaves it, or an older block. + fn initial_safe_head(starts_at_target: bool, target: L2BlockInfo) -> L2BlockInfo { + if starts_at_target { target } else { test_block_info(5) } } /// Below the unsafe head, a delegated safe head always reaches the execution layer: when the @@ -682,10 +898,8 @@ mod tests { ) { let (block, cfg) = genesis_pinned_block(10); let delegated_safe_head = block_info_of(&cfg, &block); - let mut state = engine_state_with_safe_head(target_or_older_block( - starts_at_target, - delegated_safe_head, - )); + let mut state = + engine_state_with_safe_head(initial_safe_head(starts_at_target, delegated_safe_head)); let expected_sync_state = with_safe_head(&state, delegated_safe_head); let (client, l1, l2) = test_engine_client(cfg.clone()); l2.expect_params( @@ -698,7 +912,7 @@ mod tests { let task = ConsolidateTask::new( Arc::new(client), cfg, - ConsolidateInput::BlockInfo(delegated_safe_head), + delegated_safe_heads(delegated_safe_head, delegated_safe_head), sink.clone(), ); @@ -807,7 +1021,7 @@ mod tests { l2.assert_finished(); } - /// Where a delegated safe head sits against the finalized head. + /// Where a delegated local-safe head sits against the finalized head. #[derive(Clone, Copy, Debug)] enum TargetAgainstFinalized { Below, @@ -830,302 +1044,835 @@ mod tests { } } - /// The outcome of checking a delegated safe head against finality. - #[derive(Clone, Copy, Debug, PartialEq, Eq)] - enum FinalityCheck { - Accepted, - BehindFinalizedHead, - ConflictsWithFinalizedHead, - } - - impl FinalityCheck { - fn of(result: &Result<(), ConsolidateTaskError>) -> Self { - match result { - Ok(()) => Self::Accepted, - Err(ConsolidateTaskError::BreaksFinality( - FinalityViolation::BehindFinalizedHead { .. }, - )) => Self::BehindFinalizedHead, - Err(ConsolidateTaskError::BreaksFinality( - FinalityViolation::ConflictsWithFinalizedHead { .. }, - )) => Self::ConflictsWithFinalizedHead, - Err(err) => panic!("unexpected error {err:?}"), - } - } + fn is_behind_finality(result: &Result<(), ConsolidateTaskError>) -> bool { + matches!( + result, + Err(ConsolidateTaskError::BreaksFinality( + FinalityViolation::BehindFinalizedHead { .. } + )) + ) + } + + fn is_conflicting_with_finality(result: &Result<(), ConsolidateTaskError>) -> bool { + matches!( + result, + Err(ConsolidateTaskError::BreaksFinality( + FinalityViolation::ConflictsWithFinalizedHead { .. } + )) + ) } - /// A delegated safe head that breaks finality, as a stale delegate snapshot reports, is skipped - /// before the execution layer is asked anything, and leaves every label where it was. + /// A delegated local-safe head that breaks finality, as a stale delegate snapshot reports, is + /// skipped before the execution layer is asked anything, and leaves every label where it was. #[rstest::rstest] - #[case::below(TargetAgainstFinalized::Below, FinalityCheck::BehindFinalizedHead)] - #[case::conflicting( - TargetAgainstFinalized::ConflictingAt, - FinalityCheck::ConflictsWithFinalizedHead - )] + #[case::below(TargetAgainstFinalized::Below, is_behind_finality)] + #[case::conflicting(TargetAgainstFinalized::ConflictingAt, is_conflicting_with_finality)] #[tokio::test] - async fn delegated_safe_head_breaking_finality_is_skipped( + async fn delegated_local_safe_head_breaking_finality_is_skipped( #[case] target: TargetAgainstFinalized, - #[case] expected: FinalityCheck, + #[case] is_expected: fn(&Result<(), ConsolidateTaskError>) -> bool, ) { - let cfg = Arc::new(RollupConfig::default()); let mut state = engine_state_with_safe_head(test_block_info(12)); let state_before = state; - let delegated_safe_head = target.delegated_safe_head(state.sync_state.finalized_head()); + let delegated = target.delegated_safe_head(state.sync_state.finalized_head()); + + let result = run_delegated(&mut state, delegated, delegated, |_| {}).await; + + assert!(is_expected(&result), "got {result:?}"); + assert_eq!(result.unwrap_err().severity(), crate::EngineTaskErrorSeverity::Skip); + assert_eq!(state, state_before); + } + + /// A block at `number` on the chain identified by `chain`, and its info. Blocks on different + /// chains at the same height have different hashes. + fn chain_block(number: u64, chain: u64) -> (RpcBlock, L2BlockInfo) { + let header = RpcHeader::new(alloy_consensus::Header { + number, + timestamp: chain, + ..Default::default() + }); + let info = L2BlockInfo { + block_info: BlockInfo { hash: header.hash, number, ..Default::default() }, + ..Default::default() + }; + (RpcBlock::new(header, BlockTransactions::Full(vec![])), info) + } + + /// The engine's chain. + const ENGINE_CHAIN: u64 = 0; + /// A chain that diverged from the engine's. + const OTHER_CHAIN: u64 = 1; + + fn engine_chain_block(number: u64) -> L2BlockInfo { + chain_block(number, ENGINE_CHAIN).1 + } + + fn other_chain_block(number: u64) -> L2BlockInfo { + chain_block(number, OTHER_CHAIN).1 + } + + /// What the execution layer holds at a height: the engine chain's block, another chain's, or + /// nothing. + #[derive(Clone, Copy, Debug)] + enum ElHas { + EngineChain, + Missing, + OtherChain, + } + + impl ElHas { + fn block(self, number: u64) -> Option> { + match self { + Self::EngineChain => Some(chain_block(number, ENGINE_CHAIN).0), + Self::Missing => None, + Self::OtherChain => Some(chain_block(number, OTHER_CHAIN).0), + } + } + } + + /// An engine state with the unsafe head at 20 and the finalized head at 5 on the engine's + /// chain, and its local-safe and safe heads at the given heights, which differ under interop. + fn engine_state_with_safe_labels(local_safe: u64, safe: u64) -> EngineState { + TestEngineStateBuilder::new() + .with_unsafe_head(engine_chain_block(20)) + .with_local_safe_head(engine_chain_block(local_safe)) + .with_safe_head(engine_chain_block(safe)) + .with_finalized_head(engine_chain_block(5)) + .build() + } + + /// Runs a consolidation of the delegated `local_safe` and `cross_safe` heads against an + /// execution layer that `expect_engine_calls` scripts, and checks every scripted call was made. + /// Genesis is pinned to the local-safe head, so its empty block decodes without an L1-info + /// deposit. + async fn run_delegated( + state: &mut EngineState, + local_safe: L2BlockInfo, + cross_safe: L2BlockInfo, + expect_engine_calls: impl FnOnce(&RpcMock), + ) -> Result<(), ConsolidateTaskError> { + let cfg = Arc::new(RollupConfig { + genesis: ChainGenesis { l2: local_safe.block_info.id(), ..Default::default() }, + ..Default::default() + }); let (client, l1, l2) = test_engine_client(cfg.clone()); + expect_engine_calls(&l2); let task = ConsolidateTask::new( Arc::new(client), cfg, - ConsolidateInput::BlockInfo(delegated_safe_head), + delegated_safe_heads(local_safe, cross_safe), Arc::new(NoopBlockSink), ); - let result = task.execute(&mut state).await; + let result = task.execute(state).await; - assert_eq!(FinalityCheck::of(&result), expected); - assert_eq!(result.unwrap_err().severity(), crate::EngineTaskErrorSeverity::Skip); - assert_eq!(state, state_before); l1.assert_finished(); l2.assert_finished(); + result + } + + /// The labels after the engine moves its local-safe head, and its safe head to `safe`. + fn labels_after( + state: &EngineState, + local_safe: L2BlockInfo, + safe: L2BlockInfo, + ) -> EngineSyncState { + state.sync_state.apply_update(EngineSyncStateUpdate { + local_safe_head: Some(local_safe), + safe_head: Some(safe), + ..Default::default() + }) + } + + /// The labels after a reorg onto the delegated heads. + fn labels_after_reorg( + state: &EngineState, + local_safe: L2BlockInfo, + cross_safe: L2BlockInfo, + ) -> EngineSyncState { + state.sync_state.apply_update(EngineSyncStateUpdate { + local_safe_head: Some(local_safe), + safe_head: Some(cross_safe), + unsafe_head: Some(local_safe), + ..Default::default() + }) + } + + /// Below the unsafe head, a delegated local-safe head on the engine's chain that the execution + /// layer has becomes the local-safe label, and a cross-safe head on that chain above the safe + /// label moves it once the execution layer's block at its height confirms it. + #[tokio::test] + async fn advancing_cross_safe_head_moves_the_safe_label() { + let (local_safe, cross_safe) = (engine_chain_block(10), engine_chain_block(8)); + let mut state = engine_state_with_safe_labels(9, 7); + let expected = labels_after(&state, local_safe, cross_safe); + + run_delegated(&mut state, local_safe, cross_safe, |l2| { + l2.expect_block_by_number(10, ElHas::EngineChain.block(10)); + l2.expect_block_header_by_number(8, ElHas::EngineChain.block(8)); + l2.expect_forkchoice_updated( + expected.create_forkchoice_state(), + PayloadStatusEnum::Valid, + ); + }) + .await + .unwrap(); + + assert_eq!(state.sync_state, expected); + } + + /// The safe label's own block is known, so a cross-safe head there needs no lookup. + fn cross_safe_at_safe_label(l2: &RpcMock) { + l2.expect_block_by_number(10, ElHas::EngineChain.block(10)); + } + + /// A cross-safe head behind the safe label can neither advance nor replace it, so it is not + /// looked up. + fn cross_safe_behind_safe_label(l2: &RpcMock) { + l2.expect_block_by_number(10, ElHas::EngineChain.block(10)); + } + + /// The execution layer still syncs toward the local-safe head, so the cross-safe head is not + /// looked up. + fn el_syncing_cross_safe_behind_safe_label(l2: &RpcMock) { + l2.expect_block_by_number(10, ElHas::Missing.block(10)); + } + + /// The execution layer has another block at the cross-safe height than the delegate's. + fn cross_safe_off_the_local_safe_chain(l2: &RpcMock) { + l2.expect_block_by_number(10, ElHas::EngineChain.block(10)); + l2.expect_block_header_by_number(8, ElHas::OtherChain.block(8)); } - /// A delegated safe head from a lagging delegate, between the finalized head, inclusive, and - /// the safe head, does not move the safe labels back, whether or not the execution layer has - /// its block, and sends no forkchoice update. + /// A cross-safe head at or below the safe label leaves it, so a load-balanced delegate whose + /// cross-safe head lags does not drop the forkchoice update's safe block; so does one that is + /// not on the chain of its own local-safe head, which the execution layer has. #[rstest::rstest] - #[case::el_has_block(true, false)] - #[case::el_missing_block(false, false)] - #[case::at_finalized_head(true, true)] + #[case::at_safe_label(7, cross_safe_at_safe_label)] + #[case::behind_safe_label(6, cross_safe_behind_safe_label)] + #[case::el_syncing_behind_safe_label(6, el_syncing_cross_safe_behind_safe_label)] + #[case::off_the_local_safe_chain(8, cross_safe_off_the_local_safe_chain)] #[tokio::test] - async fn lagging_delegated_safe_head_leaves_labels_unchanged( - #[case] el_has_block: bool, - #[case] finalized_at_target: bool, + async fn cross_safe_head_not_advancing_keeps_the_safe_label( + #[case] cross_safe_number: u64, + #[case] expect_lookups: fn(&RpcMock), ) { - let (block, cfg) = genesis_pinned_block(10); - let delegated_safe_head = block_info_of(&cfg, &block); - let mut state = TestEngineStateBuilder::new() - .with_unsafe_head(test_block_info(20)) - .with_safe_head(test_block_info(12)) - .with_finalized_head(target_or_older_block(finalized_at_target, delegated_safe_head)) - .build(); - let state_before = state; - let (client, l1, l2) = test_engine_client(cfg.clone()); - l2.expect_block_header_by_number(10, el_block(el_has_block, block)); - let sink = Arc::new(RecordingSink::default()); - let task = ConsolidateTask::new( - Arc::new(client), - cfg, - ConsolidateInput::BlockInfo(delegated_safe_head), - sink.clone(), - ); + let local_safe = engine_chain_block(10); + let mut state = engine_state_with_safe_labels(9, 7); + let expected = labels_after(&state, local_safe, state.sync_state.safe_head()); - task.execute(&mut state).await.unwrap(); + run_delegated(&mut state, local_safe, engine_chain_block(cross_safe_number), |l2| { + expect_lookups(l2); + l2.expect_forkchoice_updated( + expected.create_forkchoice_state(), + PayloadStatusEnum::Valid, + ); + }) + .await + .unwrap(); - assert_eq!(state, state_before); - assert!(sink.0.lock().unwrap().is_empty()); - l1.assert_finished(); - l2.assert_finished(); + assert_eq!(state.sync_state, expected); } - /// A delegated safe head behind the safe head whose block differs from the execution layer's - /// means the delegate's chain diverged above it, so the engine still reorgs onto it, without - /// moving finalized. + /// A cross-safe head behind, or conflicting with, finality leaves the safe label without a + /// lookup, while the local-safe head from the same status still moves. + #[rstest::rstest] + #[case::behind_finalized(engine_chain_block(4))] + #[case::conflicting_with_finalized(other_chain_block(5))] #[tokio::test] - async fn diverged_delegated_safe_head_behind_safe_head_reorgs_the_engine() { - let (block, cfg) = genesis_pinned_block(10); - let delegated_safe_head = block_info_of(&cfg, &block); - let mut state = engine_state_with_safe_head(test_block_info(12)); - let finalized_before = state.sync_state.finalized_head(); - let expected_sync_state = state.sync_state.apply_update(EngineSyncStateUpdate { - local_safe_head: Some(delegated_safe_head), - safe_head: Some(delegated_safe_head), - unsafe_head: Some(delegated_safe_head), + async fn cross_safe_head_breaking_finality_keeps_only_the_safe_label( + #[case] cross_safe: L2BlockInfo, + ) { + let local_safe = engine_chain_block(10); + let mut state = engine_state_with_safe_labels(9, 7); + let expected = labels_after(&state, local_safe, state.sync_state.safe_head()); + + run_delegated(&mut state, local_safe, cross_safe, |l2| { + l2.expect_block_by_number(10, ElHas::EngineChain.block(10)); + l2.expect_forkchoice_updated( + expected.create_forkchoice_state(), + PayloadStatusEnum::Valid, + ); + }) + .await + .unwrap(); + + assert_eq!(state.sync_state, expected); + } + + /// The execution layer has the lagging local-safe head at 10, and confirms the cross-safe head + /// at 9. + fn el_has_local_safe_10_and_cross_safe_9(l2: &RpcMock) { + l2.expect_block_header_by_number(10, ElHas::EngineChain.block(10)); + l2.expect_block_header_by_number(9, ElHas::EngineChain.block(9)); + } + + /// The execution layer lacks the lagging local-safe head at 10, so the cross-safe head is not + /// looked up. + fn el_missing_local_safe_10(l2: &RpcMock) { + l2.expect_block_header_by_number(10, ElHas::Missing.block(10)); + } + + /// A delegated local-safe head behind the engine's whose block the execution layer has, or + /// lacks, from a lagging delegate, leaves the local-safe and unsafe heads in place. Its + /// cross-safe head still moves the safe label when above it. + #[rstest::rstest] + #[case::el_has_local_safe(el_has_local_safe_10_and_cross_safe_9)] + #[case::el_missing_local_safe(el_missing_local_safe_10)] + #[tokio::test] + async fn lagging_delegate_moves_only_an_advancing_safe_label( + #[case] expect_lookups: fn(&RpcMock), + ) { + let (local_safe, cross_safe) = (engine_chain_block(10), engine_chain_block(9)); + let mut state = engine_state_with_safe_labels(12, 8); + let expected = state.sync_state.apply_update(EngineSyncStateUpdate { + safe_head: Some(cross_safe), ..Default::default() }); - let (client, l1, l2) = test_engine_client(cfg.clone()); - l2.expect_block_header_by_number(10, Some(other_chain_block(10))); - l2.expect_forkchoice_updated( - expected_sync_state.create_forkchoice_state(), - PayloadStatusEnum::Valid, - ); - let task = ConsolidateTask::new( - Arc::new(client), - cfg, - ConsolidateInput::BlockInfo(delegated_safe_head), - Arc::new(NoopBlockSink), - ); - task.execute(&mut state).await.unwrap(); + run_delegated(&mut state, local_safe, cross_safe, |l2| { + expect_lookups(l2); + l2.expect_forkchoice_updated( + expected.create_forkchoice_state(), + PayloadStatusEnum::Valid, + ); + }) + .await + .unwrap(); - assert_eq!(state.sync_state, expected_sync_state); - assert_eq!(state.sync_state.finalized_head(), finalized_before); - l1.assert_finished(); - l2.assert_finished(); + assert_eq!(state.sync_state, expected); } - /// A delegated safe head behind the safe head is not judged against an execution layer that has - /// not applied the engine's last forkchoice, whether it is in its first sync or syncing again - /// after answering `SYNCING`: its canonical block there may predate the engine's labels, so no - /// lookup is made and the labels stay. + /// A delegate before interop, whose heads are the same block, that lags the engine leaves every + /// label where it is and sends no forkchoice update, whether or not the execution layer has the + /// block. #[rstest::rstest] - #[case::initial_sync(false, false)] - #[case::syncing_again(true, false)] + #[case::el_has_block(10, ElHas::EngineChain)] + #[case::el_missing_block(10, ElHas::Missing)] + #[case::at_finalized_head(5, ElHas::EngineChain)] #[tokio::test] - async fn delegated_safe_head_behind_safe_head_is_ignored_while_the_el_syncs( - #[case] el_sync_finished: bool, - #[case] el_applied_last_forkchoice: bool, + async fn lagging_delegate_before_interop_changes_nothing( + #[case] delegated_number: u64, + #[case] el_at_local_safe: ElHas, ) { - let (block, cfg) = genesis_pinned_block(10); - let delegated_safe_head = block_info_of(&cfg, &block); - let mut state = TestEngineStateBuilder::new() - .with_unsafe_head(test_block_info(20)) - .with_safe_head(test_block_info(12)) - .with_finalized_head(test_block_info(5)) - .with_el_sync_finished(el_sync_finished) - .with_el_applied_last_forkchoice(el_applied_last_forkchoice) - .build(); + let delegated = engine_chain_block(delegated_number); + let mut state = engine_state_with_safe_labels(12, 12); let state_before = state; - let (client, l1, l2) = test_engine_client(cfg.clone()); - let task = ConsolidateTask::new( - Arc::new(client), - cfg, - ConsolidateInput::BlockInfo(delegated_safe_head), - Arc::new(NoopBlockSink), - ); - task.execute(&mut state).await.unwrap(); + run_delegated(&mut state, delegated, delegated, |l2| { + l2.expect_block_header_by_number( + delegated_number, + el_at_local_safe.block(delegated_number), + ); + }) + .await + .unwrap(); assert_eq!(state, state_before); - l1.assert_finished(); - l2.assert_finished(); } - /// An empty block at `number` whose hash differs from [`genesis_pinned_block`]'s at the same - /// height, standing for a block on another chain. - fn other_chain_block(number: u64) -> RpcBlock { - RpcBlock::new( - RpcHeader::new(alloy_consensus::Header { number, timestamp: 1, ..Default::default() }), - BlockTransactions::Full(vec![]), - ) + /// A lagging delegate whose cross-safe head is behind the safe label leaves every label where + /// it is and sends no forkchoice update, without looking the cross-safe head up. + #[tokio::test] + async fn lagging_delegate_with_cross_safe_head_behind_the_safe_label_changes_nothing() { + let mut state = engine_state_with_safe_labels(12, 12); + let state_before = state; + + run_delegated(&mut state, engine_chain_block(10), engine_chain_block(7), |l2| { + l2.expect_block_header_by_number(10, ElHas::EngineChain.block(10)); + }) + .await + .unwrap(); + + assert_eq!(state, state_before); } - /// A reorg onto a diverged delegated safe head that the execution layer refuses as an invalid - /// forkchoice state resets the engine, which rebuilds the labels from the execution layer. + /// A lagging delegate whose cross-safe head conflicts with the finalized head leaves every + /// label where it is: the cross-safe head cannot become the safe label. #[tokio::test] - async fn rejected_reorg_onto_a_delegated_safe_head_resets_the_engine() { - let (block, cfg) = genesis_pinned_block(10); - let delegated_safe_head = block_info_of(&cfg, &block); - let mut state = engine_state_with_safe_head(test_block_info(12)); + async fn lagging_delegate_with_cross_safe_head_conflicting_with_finality_changes_nothing() { + let mut state = engine_state_with_safe_labels(12, 12); let state_before = state; - let (client, l1, l2) = test_engine_client(cfg.clone()); - l2.expect_block_header_by_number(10, Some(other_chain_block(10))); - l2.expect_error_code( - "engine_forkchoiceUpdatedV3", - i64::from(alloy_rpc_types_engine::INVALID_FORK_CHOICE_STATE_ERROR), - ); - let task = ConsolidateTask::new( - Arc::new(client), - cfg, - ConsolidateInput::BlockInfo(delegated_safe_head), - Arc::new(NoopBlockSink), - ); - let err = task.execute(&mut state).await.unwrap_err(); + run_delegated(&mut state, engine_chain_block(10), other_chain_block(5), |l2| { + l2.expect_block_header_by_number(10, ElHas::EngineChain.block(10)); + }) + .await + .unwrap(); - assert_eq!(err.severity(), crate::EngineTaskErrorSeverity::Reset, "got {err:?}"); assert_eq!(state, state_before); - l1.assert_finished(); - l2.assert_finished(); } - /// Labels with the unsafe head rewound below the safe head are moved back into order by any - /// delegated safe head above finality instead of being kept: at or above the unsafe head the - /// engine reorgs onto it, and below it the matching block consolidates. Keeping them would let - /// a later finalization above the unsafe head fail fatally. + /// A lagging delegate whose local-safe block the execution layer lacks cannot be judged against + /// the execution layer's canonical blocks, which may predate its sync, so a cross-safe head off + /// them is not looked up and reorgs nothing; not above the safe label, it changes nothing. + #[tokio::test] + async fn lagging_delegate_missing_from_the_el_never_reorgs() { + let mut state = engine_state_with_safe_labels(12, 12); + let state_before = state; + + run_delegated(&mut state, other_chain_block(10), other_chain_block(8), |l2| { + l2.expect_block_header_by_number(10, ElHas::Missing.block(10)); + }) + .await + .unwrap(); + + assert_eq!(state, state_before); + } + + /// A lagging delegate is not judged against an execution layer that has not applied the + /// engine's last forkchoice, even after its first sync finished: its canonical block at the + /// local-safe height may predate the engine's labels, so no lookup is made and a delegate on + /// another chain reorgs nothing. + #[tokio::test] + async fn lagging_delegate_is_not_judged_against_a_syncing_el() { + let mut state = engine_state_with_safe_labels(12, 12); + state.el_applied_last_forkchoice = false; + let state_before = state; + + run_delegated(&mut state, other_chain_block(10), other_chain_block(8), |_| {}) + .await + .unwrap(); + + assert_eq!(state, state_before); + } + + /// Lagging delegate: the execution layer has another block at its local-safe height. + fn lagging_local_safe_off_the_el_chain(l2: &RpcMock) { + l2.expect_block_header_by_number(10, ElHas::EngineChain.block(10)); + } + + /// Below the unsafe head: the execution layer has another block at the local-safe height. + fn local_safe_off_the_el_chain(l2: &RpcMock) { + l2.expect_block_by_number(18, ElHas::EngineChain.block(18)); + } + + /// At or above the unsafe head: the local-safe head is another block at the unsafe height. + const fn local_safe_off_the_unsafe_head(_: &RpcMock) {} + + /// Above the unsafe head but not its child: the local-safe head's ancestry is unknown, and the + /// execution layer does not hold the engine's safe label either. + fn local_safe_above_the_unsafe_head(l2: &RpcMock) { + l2.expect_block_header_by_number(12, ElHas::OtherChain.block(12)); + } + + /// Delegated heads on a chain that diverged from the engine's, which an execution-layer block + /// or the unsafe head at the local-safe height proves, or above the unsafe head without being + /// its child while the execution layer does not hold the safe label, reorg the engine onto + /// them: the unsafe and local-safe heads move to the local-safe head, and the safe label to the + /// cross-safe head even when lower, since the old safe block is on the abandoned chain. + /// Finalized stays. #[rstest::rstest] - #[case::at_or_above_unsafe_head(12, false)] - #[case::below_unsafe_head(8, true)] + #[case::lagging_local_safe(10, lagging_local_safe_off_the_el_chain)] + #[case::local_safe_below_unsafe(18, local_safe_off_the_el_chain)] + #[case::local_safe_at_unsafe(20, local_safe_off_the_unsafe_head)] + #[case::local_safe_above_unsafe(25, local_safe_above_the_unsafe_head)] #[tokio::test] - async fn delegated_safe_head_repairs_an_unsafe_head_below_the_safe_head( - #[case] delegated_number: u64, - #[case] is_below_unsafe_head: bool, + async fn delegated_heads_off_the_engine_chain_reorg_the_engine( + #[case] local_safe_number: u64, + #[case] expect_lookups: fn(&RpcMock), ) { - let (block, cfg) = genesis_pinned_block(delegated_number); - let delegated_safe_head = block_info_of(&cfg, &block); + let (local_safe, cross_safe) = (other_chain_block(local_safe_number), other_chain_block(8)); + let mut state = engine_state_with_safe_labels(12, 12); + let finalized_before = state.sync_state.finalized_head(); + let expected = labels_after_reorg(&state, local_safe, cross_safe); + + run_delegated(&mut state, local_safe, cross_safe, |l2| { + expect_lookups(l2); + l2.expect_forkchoice_updated( + expected.create_forkchoice_state(), + PayloadStatusEnum::Valid, + ); + }) + .await + .unwrap(); + + assert_eq!(state.sync_state, expected); + assert_eq!(state.sync_state.finalized_head(), finalized_before); + } + + /// Delegated heads at or above the unsafe head on the engine's chain move the unsafe and + /// local-safe heads to the local-safe head, and a cross-safe head on that chain above the safe + /// label moves it. + #[rstest::rstest] + #[case::at_unsafe_head(engine_chain_block(20))] + #[case::child_of_unsafe_head(child_of_engine_unsafe_head())] + #[tokio::test] + async fn delegated_heads_extending_the_engine_chain_move_an_advancing_safe_label( + #[case] local_safe: L2BlockInfo, + ) { + let cross_safe = engine_chain_block(17); + let mut state = engine_state_with_safe_labels(15, 15); + let expected = labels_after_reorg(&state, local_safe, cross_safe); + + run_delegated(&mut state, local_safe, cross_safe, |l2| { + l2.expect_block_header_by_number(17, ElHas::EngineChain.block(17)); + l2.expect_forkchoice_updated( + expected.create_forkchoice_state(), + PayloadStatusEnum::Valid, + ); + }) + .await + .unwrap(); + + assert_eq!(state.sync_state, expected); + } + + /// A safe label left on blocks a reorg abandoned does not prove the engine's chain: the + /// execution layer confirms the cross-safe head at the safe label's height, and the safe label + /// moves to it, keeping it an ancestor of the unsafe head. + #[tokio::test] + async fn cross_safe_head_replaces_a_stale_safe_label_at_its_height() { + let (local_safe, cross_safe) = (engine_chain_block(20), engine_chain_block(12)); let mut state = TestEngineStateBuilder::new() - .with_unsafe_head(test_block_info(10)) - .with_safe_head(test_block_info(15)) - .with_finalized_head(test_block_info(5)) + .with_unsafe_head(engine_chain_block(20)) + .with_local_safe_head(other_chain_block(12)) + .with_safe_head(other_chain_block(12)) + .with_finalized_head(engine_chain_block(5)) + .build(); + let expected = labels_after_reorg(&state, local_safe, cross_safe); + + run_delegated(&mut state, local_safe, cross_safe, |l2| { + l2.expect_block_header_by_number(12, ElHas::EngineChain.block(12)); + l2.expect_forkchoice_updated( + expected.create_forkchoice_state(), + PayloadStatusEnum::Valid, + ); + }) + .await + .unwrap(); + + assert_eq!(state.sync_state, expected); + } + + /// While the execution layer has not finished syncing, its canonical blocks may be from before + /// the sync, so the cross-safe head is not checked against them and only advances the safe + /// label. + #[tokio::test] + async fn cross_safe_head_is_not_judged_against_a_syncing_el() { + let (local_safe, cross_safe) = (engine_chain_block(20), engine_chain_block(17)); + let mut state = TestEngineStateBuilder::new() + .with_unsafe_head(engine_chain_block(20)) + .with_safe_head(engine_chain_block(15)) + .with_finalized_head(engine_chain_block(5)) + .with_el_sync_finished(false) .build(); - let expected_sync_state = state.sync_state.apply_update(EngineSyncStateUpdate { - local_safe_head: Some(delegated_safe_head), - safe_head: Some(delegated_safe_head), - unsafe_head: (!is_below_unsafe_head).then_some(delegated_safe_head), + let expected = labels_after_reorg(&state, local_safe, cross_safe); + + run_delegated(&mut state, local_safe, cross_safe, |l2| { + l2.expect_forkchoice_updated( + expected.create_forkchoice_state(), + PayloadStatusEnum::Valid, + ); + }) + .await + .unwrap(); + + assert_eq!(state.sync_state, expected); + } + + /// The engine chain's block at 21, the child of the engine's unsafe head at 20. + fn child_of_engine_unsafe_head() -> L2BlockInfo { + L2BlockInfo { + block_info: BlockInfo { + parent_hash: engine_chain_block(20).block_info.hash, + ..engine_chain_block(21).block_info + }, + ..Default::default() + } + } + + /// A delegated local-safe head that is the unsafe head's child extends the engine's chain, so + /// a cross-safe head below the safe label keeps it. + #[tokio::test] + async fn child_of_the_unsafe_head_keeps_the_safe_label() { + let local_safe = child_of_engine_unsafe_head(); + let mut state = engine_state_with_safe_labels(15, 15); + let expected = state.sync_state.apply_update(EngineSyncStateUpdate { + local_safe_head: Some(local_safe), + unsafe_head: Some(local_safe), ..Default::default() }); - let (client, l1, l2) = test_engine_client(cfg.clone()); - if is_below_unsafe_head { - l2.expect_params( - "eth_getBlockByNumber", - serde_json::json!([format!("{delegated_number:#x}"), true]), - block, + + run_delegated(&mut state, local_safe, engine_chain_block(12), |l2| { + l2.expect_forkchoice_updated( + expected.create_forkchoice_state(), + PayloadStatusEnum::Valid, ); - } - l2.expect_forkchoice_updated( - expected_sync_state.create_forkchoice_state(), - PayloadStatusEnum::Valid, + }) + .await + .unwrap(); + + assert_eq!(state.sync_state, expected); + } + + /// A reorg whose cross-safe head is below the finalized head moves the safe label to the + /// finalized head, which the delegate's chain shares with the engine's. + #[tokio::test] + async fn reorg_with_cross_safe_head_behind_finality_moves_safe_to_finalized() { + let local_safe = other_chain_block(20); + let mut state = engine_state_with_safe_labels(15, 15); + let expected = labels_after_reorg(&state, local_safe, state.sync_state.finalized_head()); + + run_delegated(&mut state, local_safe, other_chain_block(4), |l2| { + l2.expect_forkchoice_updated( + expected.create_forkchoice_state(), + PayloadStatusEnum::Valid, + ); + }) + .await + .unwrap(); + + assert_eq!(state.sync_state, expected); + } + + /// Delegated heads at the unsafe head on the engine's chain, as when the engine has no unsafe + /// gossip, keep the safe label for a cross-safe head at or below it, so a lagging backend + /// behind a load balancer does not move safe back. + #[rstest::rstest] + #[case::at_safe_label(15, no_lookup)] + #[case::behind_safe_label(12, no_lookup)] + #[tokio::test] + async fn delegated_heads_at_the_unsafe_head_keep_the_safe_label( + #[case] cross_safe_number: u64, + #[case] expect_cross_safe_lookup: fn(&RpcMock), + ) { + let local_safe = engine_chain_block(20); + let mut state = engine_state_with_safe_labels(15, 15); + let expected = state.sync_state.apply_update(EngineSyncStateUpdate { + local_safe_head: Some(local_safe), + unsafe_head: Some(local_safe), + ..Default::default() + }); + + run_delegated(&mut state, local_safe, engine_chain_block(cross_safe_number), |l2| { + expect_cross_safe_lookup(l2); + l2.expect_forkchoice_updated( + expected.create_forkchoice_state(), + PayloadStatusEnum::Valid, + ); + }) + .await + .unwrap(); + + assert_eq!(state.sync_state, expected); + } + + /// A cross-safe head at one of the engine's labels is not looked up. + const fn no_lookup(_: &RpcMock) {} + + /// The execution layer has the engine chain's block at 15, which the safe label alone cannot + /// prove, since it can stay on blocks a reorg abandoned. + fn cross_safe_15_on_the_engine_chain(l2: &RpcMock) { + l2.expect_block_header_by_number(15, ElHas::EngineChain.block(15)); + } + + fn expect_invalid_forkchoice_state(l2: &RpcMock) { + l2.expect_error_code( + "engine_forkchoiceUpdatedV3", + i64::from(alloy_rpc_types_engine::INVALID_FORK_CHOICE_STATE_ERROR), ); + } + + /// A forkchoice update for delegated safe heads, whether a reorg off the engine's chain or an + /// extension of it, that the execution layer refuses as an invalid forkchoice state resets the + /// engine, which rebuilds the labels from the execution layer. + #[rstest::rstest] + #[case::reorg(other_chain_block(20))] + #[case::extension(engine_chain_block(25))] + #[tokio::test] + async fn rejected_forkchoice_for_delegated_heads_resets_the_engine( + #[case] delegated: L2BlockInfo, + ) { + let mut state = engine_state_with_safe_labels(15, 15); + let state_before = state; + + let err = run_delegated(&mut state, delegated, delegated, expect_invalid_forkchoice_state) + .await + .unwrap_err(); + + assert_eq!(err.severity(), EngineTaskErrorSeverity::Reset, "got {err:?}"); + assert_eq!(state, state_before); + } + + /// A reorg whose cross-safe head breaks finality is skipped before the execution layer is + /// asked: the cross-safe head cannot become the safe label, and the old safe label would sit + /// above the new unsafe head. + #[tokio::test] + async fn reorg_with_cross_safe_head_breaking_finality_is_skipped() { + let mut state = engine_state_with_safe_labels(15, 15); + let state_before = state; + + let err = run_delegated(&mut state, other_chain_block(20), other_chain_block(5), |_| {}) + .await + .unwrap_err(); + + assert!(matches!(err, ConsolidateTaskError::BreaksFinality(_)), "got {err:?}"); + assert_eq!(err.severity(), crate::EngineTaskErrorSeverity::Skip); + assert_eq!(state, state_before); + } + + /// While the execution layer syncs, its canonical blocks are still those from before the sync, + /// so a cross-safe head is not compared with them: it only advances the safe label, and never + /// reorgs the engine or retargets the sync. + #[rstest::rstest] + #[case::other_chain_below_safe_label(other_chain_block(12), engine_chain_block(15))] + #[case::advancing(engine_chain_block(16), engine_chain_block(16))] + #[tokio::test] + async fn el_sync_moves_the_safe_label_only_forward_without_lookups( + #[case] cross_safe: L2BlockInfo, + #[case] expected_safe: L2BlockInfo, + ) { + let local_safe = engine_chain_block(18); + let mut state = engine_state_with_safe_labels(15, 15); + let expected = labels_after(&state, local_safe, expected_safe); + + run_delegated(&mut state, local_safe, cross_safe, |l2| { + l2.expect_block_by_number(18, ElHas::Missing.block(18)); + l2.expect_forkchoice_updated( + expected.create_forkchoice_state(), + PayloadStatusEnum::Valid, + ); + }) + .await + .unwrap(); + + assert_eq!(state.sync_state, expected); + } + + /// A cross-safe head that is the engine's safe label leaves it in place without asking the + /// execution layer. One that differs from the safe label at its height is checked against the + /// execution layer, which has the engine chain's block there, so the safe label stays too. + #[rstest::rstest] + #[case::at_safe_label(engine_chain_block(15), no_lookup)] + #[case::conflicting_with_safe_label(other_chain_block(15), cross_safe_15_on_the_engine_chain)] + #[tokio::test] + async fn cross_safe_head_off_the_engine_chain_keeps_the_safe_label( + #[case] cross_safe: L2BlockInfo, + #[case] expect_cross_safe_lookup: fn(&RpcMock), + ) { + let local_safe = engine_chain_block(18); + let mut state = engine_state_with_safe_labels(15, 15); + let expected = labels_after(&state, local_safe, engine_chain_block(15)); + + run_delegated(&mut state, local_safe, cross_safe, |l2| { + l2.expect_block_by_number(18, ElHas::EngineChain.block(18)); + expect_cross_safe_lookup(l2); + l2.expect_forkchoice_updated( + expected.create_forkchoice_state(), + PayloadStatusEnum::Valid, + ); + }) + .await + .unwrap(); + + assert_eq!(state.sync_state, expected); + } + + /// A delegated local-safe head whose block the execution layer has, but whose info cannot be + /// built from it, is consolidated with the delegate's info instead of reorging the unsafe head + /// back to it. + #[tokio::test] + async fn undecodable_delegated_block_is_consolidated_without_a_reorg() { + let (block, local_safe) = chain_block(18, ENGINE_CHAIN); + let cfg = Arc::new(RollupConfig::default()); + let mut state = engine_state_with_safe_labels(15, 15); + let expected = labels_after(&state, local_safe, local_safe); + let (client, l1, l2) = test_engine_client(cfg.clone()); + l2.expect_block_by_number(18, Some(block)); + l2.expect_forkchoice_updated(expected.create_forkchoice_state(), PayloadStatusEnum::Valid); let task = ConsolidateTask::new( Arc::new(client), cfg, - ConsolidateInput::BlockInfo(delegated_safe_head), + delegated_safe_heads(local_safe, local_safe), Arc::new(NoopBlockSink), ); task.execute(&mut state).await.unwrap(); - assert_eq!(state.sync_state, expected_sync_state); - assert!( - state.sync_state.unsafe_head().block_info.number >= - state.sync_state.safe_head().block_info.number - ); + assert_eq!(state.sync_state, expected); l1.assert_finished(); l2.assert_finished(); } - /// A delegated safe head whose block differs from the execution layer's is not treated as a - /// divergence while the execution layer has not applied the engine's last forkchoice: only the - /// safe labels move to it, and the unsafe head stays the sync target. + /// A delegated block that differs from the execution layer's is not a divergence while the + /// execution layer has not applied the engine's last forkchoice: only the safe labels move, the + /// cross-safe head only advancing, and the unsafe head stays the sync target. #[tokio::test] - async fn mismatched_delegated_safe_head_does_not_reorg_while_the_el_syncs() { - let (block, cfg) = genesis_pinned_block(15); - let delegated_safe_head = block_info_of(&cfg, &block); + async fn delegated_block_off_a_syncing_el_moves_only_the_safe_labels() { + let (local_safe, cross_safe) = (engine_chain_block(18), engine_chain_block(17)); + let mut state = engine_state_with_safe_labels(15, 15); + state.el_applied_last_forkchoice = false; + let expected = labels_after(&state, local_safe, cross_safe); + + run_delegated(&mut state, local_safe, cross_safe, |l2| { + l2.expect_block_by_number(18, ElHas::OtherChain.block(18)); + l2.expect_forkchoice_updated( + expected.create_forkchoice_state(), + PayloadStatusEnum::Syncing, + ); + }) + .await + .unwrap(); + + assert_eq!(state.sync_state, expected); + } + + /// A delegate catching up several blocks above the unsafe head keeps the engine's safe label + /// when the execution layer holds it, rather than dropping it to a lower cross-safe head; only + /// a safe label the execution layer does not hold falls to the cross-safe head. + #[rstest::rstest] + #[case::el_holds_safe_label(ElHas::EngineChain, 15)] + #[case::el_off_safe_label(ElHas::OtherChain, 12)] + #[tokio::test] + async fn catch_up_above_the_unsafe_head_keeps_a_safe_label_the_el_holds( + #[case] el_at_safe_label: ElHas, + #[case] expected_safe: u64, + ) { + let (local_safe, cross_safe) = (engine_chain_block(23), engine_chain_block(12)); + let mut state = engine_state_with_safe_labels(15, 15); + let expected = state.sync_state.apply_update(EngineSyncStateUpdate { + local_safe_head: Some(local_safe), + safe_head: Some(engine_chain_block(expected_safe)), + unsafe_head: Some(local_safe), + ..Default::default() + }); + + run_delegated(&mut state, local_safe, cross_safe, |l2| { + l2.expect_block_header_by_number(15, el_at_safe_label.block(15)); + l2.expect_forkchoice_updated( + expected.create_forkchoice_state(), + PayloadStatusEnum::Valid, + ); + }) + .await + .unwrap(); + + assert_eq!(state.sync_state, expected); + } + + /// Labels with the unsafe head rewound below the safe labels are moved back into order by + /// delegated heads above finality instead of being kept as a lagging delegate's; the execution + /// layer is not asked about heights above the rewound unsafe head. + #[tokio::test] + async fn delegated_heads_repair_an_unsafe_head_below_the_safe_labels() { + let (local_safe, cross_safe) = (engine_chain_block(12), engine_chain_block(12)); let mut state = TestEngineStateBuilder::new() - .with_unsafe_head(test_block_info(20)) - .with_safe_head(test_block_info(12)) - .with_finalized_head(test_block_info(5)) - .with_el_applied_last_forkchoice(false) + .with_unsafe_head(engine_chain_block(10)) + .with_safe_head(engine_chain_block(15)) + .with_finalized_head(engine_chain_block(5)) .build(); - let expected_sync_state = with_safe_head(&state, delegated_safe_head); - let (client, l1, l2) = test_engine_client(cfg.clone()); - l2.expect_params( - "eth_getBlockByNumber", - serde_json::json!(["0xf", true]), - other_chain_block(15), - ); - l2.expect_forkchoice_updated( - expected_sync_state.create_forkchoice_state(), - PayloadStatusEnum::Syncing, - ); - let task = ConsolidateTask::new( - Arc::new(client), - cfg, - ConsolidateInput::BlockInfo(delegated_safe_head), - Arc::new(NoopBlockSink), - ); + let expected = labels_after_reorg(&state, local_safe, cross_safe); - task.execute(&mut state).await.unwrap(); + run_delegated(&mut state, local_safe, cross_safe, |l2| { + l2.expect_forkchoice_updated( + expected.create_forkchoice_state(), + PayloadStatusEnum::Valid, + ); + }) + .await + .unwrap(); - assert_eq!(state.sync_state, expected_sync_state); - l1.assert_finished(); - l2.assert_finished(); + assert_eq!(state.sync_state, expected); } } diff --git a/rust/kona/crates/node/engine/src/task_queue/tasks/mod.rs b/rust/kona/crates/node/engine/src/task_queue/tasks/mod.rs index 9eddaa76d00..12ae6c8ba16 100644 --- a/rust/kona/crates/node/engine/src/task_queue/tasks/mod.rs +++ b/rust/kona/crates/node/engine/src/task_queue/tasks/mod.rs @@ -25,7 +25,10 @@ mod canonicalize; pub use canonicalize::{CanonicalizeTask, CanonicalizeTaskError}; mod consolidate; -pub use consolidate::{ConsolidateInput, ConsolidateTask, ConsolidateTaskError}; +pub use consolidate::{ + ConsolidateInput, ConsolidateTask, ConsolidateTaskError, DelegatedSafeHeads, + InconsistentSafeHeadsError, +}; mod finalize; pub use finalize::{FinalizeBlockId, FinalizeTask, FinalizeTaskError}; diff --git a/rust/kona/crates/node/engine/src/test_utils/engine_state.rs b/rust/kona/crates/node/engine/src/test_utils/engine_state.rs index 1ed3c5b8178..a1399be192e 100644 --- a/rust/kona/crates/node/engine/src/test_utils/engine_state.rs +++ b/rust/kona/crates/node/engine/src/test_utils/engine_state.rs @@ -3,7 +3,10 @@ use alloy_eips::BlockNumHash; use alloy_primitives::{B256, b256}; use kona_protocol::{BlockInfo, L2BlockInfo}; -/// Builder for creating test `EngineState` instances with sensible defaults +/// Builder for creating test `EngineState` instances with sensible defaults. +/// +/// An unset local-safe head equals the safe head, as on every chain before interop; tests of the +/// local-safe and safe labels apart set it with [`Self::with_local_safe_head`]. #[derive(Debug)] pub struct TestEngineStateBuilder { unsafe_head: L2BlockInfo, @@ -45,6 +48,12 @@ impl TestEngineStateBuilder { self } + /// Sets the local-safe head, which otherwise follows the safe head + pub const fn with_local_safe_head(mut self, block: L2BlockInfo) -> Self { + self.local_safe_head = Some(block); + self + } + /// Sets the safe head pub const fn with_safe_head(mut self, block: L2BlockInfo) -> Self { self.safe_head = Some(block); @@ -80,7 +89,9 @@ impl TestEngineStateBuilder { // Set unsafe head (required) state.sync_state = state.sync_state.apply_update(EngineSyncStateUpdate { unsafe_head: Some(self.unsafe_head), - local_safe_head: Some(self.local_safe_head.unwrap_or(self.unsafe_head)), + local_safe_head: Some( + self.local_safe_head.or(self.safe_head).unwrap_or(self.unsafe_head), + ), safe_head: Some(self.safe_head.unwrap_or(self.unsafe_head)), finalized_head: Some(self.finalized_head.unwrap_or(self.unsafe_head)), }); diff --git a/rust/kona/crates/node/engine/src/test_utils/misc.rs b/rust/kona/crates/node/engine/src/test_utils/misc.rs index fd77bf7ccf4..c7675444b0d 100644 --- a/rust/kona/crates/node/engine/src/test_utils/misc.rs +++ b/rust/kona/crates/node/engine/src/test_utils/misc.rs @@ -1,3 +1,4 @@ +use crate::{ConsolidateInput, DelegatedSafeHeads}; use alloy_eips::BlockNumHash; use alloy_primitives::B256; use kona_protocol::{BlockInfo, L2BlockInfo}; @@ -15,3 +16,14 @@ pub fn test_block_info(number: u64) -> L2BlockInfo { seq_num: 0, } } + +/// Delegated safe heads as a consolidation input. Pass the same block twice for the heads of a +/// delegate before interop. +pub fn delegated_safe_heads( + local_safe_head: L2BlockInfo, + safe_head: L2BlockInfo, +) -> ConsolidateInput { + DelegatedSafeHeads::try_new(local_safe_head, safe_head) + .expect("the cross-safe head must be at or below the local-safe head, on its chain") + .into() +} diff --git a/rust/kona/crates/node/engine/src/test_utils/mod.rs b/rust/kona/crates/node/engine/src/test_utils/mod.rs index 89d57ac5892..bfde4784cac 100644 --- a/rust/kona/crates/node/engine/src/test_utils/mod.rs +++ b/rust/kona/crates/node/engine/src/test_utils/mod.rs @@ -14,4 +14,4 @@ mod engine_state; pub use engine_state::TestEngineStateBuilder; mod misc; -pub use misc::test_block_info; +pub use misc::{delegated_safe_heads, test_block_info}; diff --git a/rust/kona/crates/node/service/src/actors/derivation/delegated/actor.rs b/rust/kona/crates/node/service/src/actors/derivation/delegated/actor.rs index 37dc0e00185..b3d64c7202b 100644 --- a/rust/kona/crates/node/service/src/actors/derivation/delegated/actor.rs +++ b/rust/kona/crates/node/service/src/actors/derivation/delegated/actor.rs @@ -6,7 +6,9 @@ use alloy_primitives::BlockHash; use async_trait::async_trait; use jsonrpsee::core::ClientError; use kona_derive::{ChainProvider, Signal}; -use kona_engine::{EngineState, FinalityViolation, FinalizeBlockId}; +use kona_engine::{ + DelegatedSafeHeads, EngineState, FinalityViolation, FinalizeBlockId, InconsistentSafeHeadsError, +}; use kona_protocol::{BlockInfo, L2BlockInfo, SyncStatus}; use kona_rpc::RollupNodeApiClient; use thiserror::Error; @@ -48,6 +50,14 @@ impl SyncStatusSource for C { /// /// Once validated, the actor sends the derived safe and finalized L2 info /// to the [`NodeActor`] responsible for the execution sub-routine. +/// +/// The delegate's local-safe and cross-safe heads are imported +/// separately: the local-safe head is the block the execution layer must have, or sync to, and +/// sets the engine's local-safe label; the cross-safe head sets the safe label that forkchoice +/// updates carry, and finalization never goes above it. Before interop both are the same block. +/// Under interop the delegate must verify cross-chain dependencies, for example op-node with a +/// supervisor, since this node checks none itself: its safe label is only as safe as the +/// delegate's cross-safe head. #[derive(Debug)] pub struct DelegateDerivationActor where @@ -78,11 +88,11 @@ where /// Ticker driving periodic polls of the derivation delegate. delegated_derivation_ticker: time::Interval, - /// A safe head behind the engine's that was forwarded while the execution layer applied the - /// engine's forkchoice, with the engine's safe head at the time. The engine checks such a head - /// against its execution layer once; until either changes, sending it again repeats that - /// lookup to the same answer. - forwarded_lagging_safe_head: Option<(BlockNumHash, BlockNumHash)>, + /// Safe heads behind the engine's that were forwarded while the execution layer applied the + /// engine's forkchoice, with the engine's local-safe and safe heads at the time. The engine + /// checks such heads against its execution layer once; until either side changes, sending + /// them again repeats that lookup to the same answer. + forwarded_lagging_safe_heads: Option<(DelegatedSafeHeads, BlockNumHash, BlockNumHash)>, /// The last rejection of a delegate status, so a delegate stuck on one inconsistency is /// warned about once rather than on every poll. last_rejection: Option, @@ -116,7 +126,7 @@ where engine_l2_safe_head: L2BlockInfo::default(), has_engine_sync_completed: false, delegated_derivation_ticker, - forwarded_lagging_safe_head: None, + forwarded_lagging_safe_heads: None, last_rejection: None, } } @@ -186,31 +196,34 @@ where Ok(()) } + /// The delegate's local-safe head, or its safe head when it reports a zero local-safe head: + /// `SyncStatus` zeroes a label the delegate has not derived, and a zero head carries nothing. + fn reported_local_safe_head(status: &SyncStatus) -> L2BlockInfo { + if status.local_safe_l2 == L2BlockInfo::default() { + status.safe_l2 + } else { + status.local_safe_l2 + } + } + /// Rejects a status whose labels are out of order: its finalized block ahead of its safe head, - /// or its safe head ahead of its local safe head. No consistent delegate reports either, and a - /// finalized block ahead of the safe head would let a stale safe head sit below finality. A - /// zero local safe head is a label the delegate has not derived or does not report, so it does - /// not order anything. - fn validate_label_order(status: &SyncStatus) -> Result<(), DerivationDelegationError> { + /// or its safe head ahead of its local safe head or a different block at its height. No + /// consistent delegate reports any of these, and a finalized block ahead of the safe head would + /// let a stale safe head sit below finality. Returns its safe heads otherwise. + fn validate_label_order( + status: &SyncStatus, + ) -> Result { let finalized = status.finalized_l2.block_info.number; let safe = status.safe_l2.block_info.number; - let local_safe = if status.local_safe_l2 == L2BlockInfo::default() { - safe - } else { - status.local_safe_l2.block_info.number - }; if finalized > safe { return Err(DerivationDelegationError::FinalizedAheadOfSafe { finalized, safe }); } - if safe > local_safe { - return Err(DerivationDelegationError::SafeAheadOfLocalSafe { local_safe, safe }); - } - Ok(()) + Ok(DelegatedSafeHeads::try_new(Self::reported_local_safe_head(status), status.safe_l2)?) } /// Verifies that the labels reported by the derivation delegate are in order, that its /// finalized L2 block does not move back from the engine's, and that its L1 info is consistent - /// with the canonical L1 chain. + /// with the canonical L1 chain. Returns its safe heads when it passes. /// /// Only a fork at the engine's finalized height is caught here. A chain that forked below it /// and reports a higher finalized block passes, and the execution layer rejects the forkchoice @@ -218,14 +231,16 @@ where async fn validate_sync_status( &mut self, v: &SyncStatus, - ) -> Result<(), DerivationDelegationError> { - Self::validate_label_order(v)?; + ) -> Result { + let safe_heads = Self::validate_label_order(v)?; self.engine_state_rx .borrow() .sync_state .ensure_respects_finality(v.finalized_l2.block_info.id())?; + let local_safe = safe_heads.local_safe_head(); let checks = [ + ("L1 Origin of Local Safe L2", local_safe.l1_origin.number, local_safe.l1_origin.hash), ("L1 Origin of Safe L2", v.safe_l2.l1_origin.number, v.safe_l2.l1_origin.hash), ( "L1 Origin of Finalized L2", @@ -243,7 +258,7 @@ where self.validate_l1_block(context, number, hash).await?; validated_l1_blocks.push((number, hash)); } - Ok(()) + Ok(safe_heads) } /// Logs a rejected delegate status at `warn!` when the reason changed, and at `debug!` while @@ -258,28 +273,23 @@ where } } - /// Whether the delegated safe head is the lagging one already forwarded against the same - /// engine safe head, so the engine would repeat its lookup to the same answer. - fn is_lagging_safe_head_forwarded(&self, safe_l2: &L2BlockInfo) -> bool { - self.lagging_safe_head_to_remember(safe_l2) == self.forwarded_lagging_safe_head && - self.forwarded_lagging_safe_head.is_some() - } - - /// The pair to remember for a delegated safe head behind the engine's consistent safe head - /// while its execution layer applied the engine's forkchoice, or `None` for any other head, - /// which the engine must always see. - fn lagging_safe_head_to_remember( + /// What to remember for safe heads behind the engine's consistent local-safe head while its + /// execution layer applied the engine's forkchoice, or `None` for any other heads, which the + /// engine must always see. + fn lagging_safe_heads_to_remember( &self, - safe_l2: &L2BlockInfo, - ) -> Option<(BlockNumHash, BlockNumHash)> { + safe_heads: DelegatedSafeHeads, + ) -> Option<(DelegatedSafeHeads, BlockNumHash, BlockNumHash)> { let engine_state = self.engine_state_rx.borrow(); - let engine_safe_head = engine_state.sync_state.safe_head().block_info; - let is_consistent = - engine_state.sync_state.unsafe_head().block_info.number >= engine_safe_head.number; + let labels = &engine_state.sync_state; + let engine_local_safe = labels.local_safe_head().block_info; + let engine_safe = labels.safe_head().block_info; + let are_labels_consistent = labels.unsafe_head().block_info.number >= engine_safe.number && + labels.unsafe_head().block_info.number >= engine_local_safe.number; (engine_state.el_applied_last_forkchoice && - is_consistent && - safe_l2.block_info.number < engine_safe_head.number) - .then(|| (safe_l2.block_info.id(), engine_safe_head.id())) + are_labels_consistent && + safe_heads.local_safe_head().block_info.number < engine_local_safe.number) + .then(|| (safe_heads, engine_local_safe.id(), engine_safe.id())) } /// Fetches, validates, and applies sync status from the derivation delegate. @@ -296,27 +306,30 @@ where } }; - if let Err(err) = self.validate_sync_status(&sync_status).await { - // Validation failures here are expected to be transient, so we skip processing - // this sync status and continue delegating derivation instead of treating it as - // fatal. - self.log_rejected_status(&err); - return Ok(()); - } + let safe_heads = match self.validate_sync_status(&sync_status).await { + Ok(safe_heads) => safe_heads, + Err(err) => { + // Validation failures here are expected to be transient, so we skip processing + // this sync status and continue delegating derivation instead of treating it as + // fatal. + self.log_rejected_status(&err); + return Ok(()); + } + }; self.last_rejection = None; self.derivation_origin_tx.send_replace(Some(sync_status.current_l1)); - if self.is_lagging_safe_head_forwarded(&sync_status.safe_l2) { + let lagging_safe_heads = self.lagging_safe_heads_to_remember(safe_heads); + if lagging_safe_heads.is_some() && lagging_safe_heads == self.forwarded_lagging_safe_heads { debug!( target: "derivation", - safe_l2 = ?sync_status.safe_l2, - "Not forwarding a lagging safe head the engine already checked" + ?safe_heads, + "Not forwarding lagging safe heads the engine already checked" ); } else { - self.forwarded_lagging_safe_head = - self.lagging_safe_head_to_remember(&sync_status.safe_l2); + self.forwarded_lagging_safe_heads = lagging_safe_heads; self.engine_client - .send_safe_l2_signal(sync_status.safe_l2.into()) + .send_safe_l2_signal(safe_heads.into()) .await .map_err(|e| DerivationError::Sender(Box::new(e)))?; } @@ -333,6 +346,7 @@ where debug!( target: "derivation", + local_safe_l2 = ?sync_status.local_safe_l2, safe_l2 = ?sync_status.safe_l2, finalized_l2 = ?sync_status.finalized_l2, "Processed sync status from delegate" @@ -381,6 +395,10 @@ enum DerivationDelegationError { #[error("Finalized L2 moves the engine's finality back: {0}")] FinalizedL2BreaksFinality(#[from] FinalityViolation), + /// The delegate's safe heads are inconsistent with each other. + #[error(transparent)] + InconsistentSafeHeads(#[from] InconsistentSafeHeadsError), + /// The L1 provider returned an error (network, RPC, etc.) #[error("L1 provider error: {0}")] L1Provider(String), @@ -388,8 +406,4 @@ enum DerivationDelegationError { /// The hash provided by the derivation delegation does not match the canonical chain. #[error("L1 inconsistency in {context} at block {number}: expected {expected}, got {actual}")] L1ValidationFailed { context: String, number: u64, expected: BlockHash, actual: BlockHash }, - - /// The delegate's safe head is ahead of its own local safe head. - #[error("Safe L2 {safe} is ahead of local safe L2 {local_safe}")] - SafeAheadOfLocalSafe { local_safe: u64, safe: u64 }, } diff --git a/rust/kona/crates/node/service/src/actors/derivation/delegated/tests.rs b/rust/kona/crates/node/service/src/actors/derivation/delegated/tests.rs index b586f878488..738f6d4397b 100644 --- a/rust/kona/crates/node/service/src/actors/derivation/delegated/tests.rs +++ b/rust/kona/crates/node/service/src/actors/derivation/delegated/tests.rs @@ -5,7 +5,8 @@ use alloy_primitives::B256; use async_trait::async_trait; use kona_derive::test_utils::TestChainProvider; use kona_engine::{ - ConsolidateInput, EngineState, EngineSyncState, EngineSyncStateUpdate, FinalizeBlockId, + ConsolidateInput, DelegatedSafeHeads, EngineState, EngineSyncState, EngineSyncStateUpdate, + FinalizeBlockId, }; use kona_protocol::{BlockInfo, L2BlockInfo, SyncStatus}; use rstest::rstest; @@ -108,25 +109,27 @@ impl Harness { enum Forwarded { Finalized(FinalizeBlockId), Other(String), - Safe(L2BlockInfo), + Safe(Box), } impl From for Forwarded { fn from(request: EngineActorRequest) -> Self { match request { EngineActorRequest::ProcessFinalizedL2Block(finalized) => Self::Finalized(*finalized), - EngineActorRequest::ProcessSafeL2Signal(ConsolidateInput::BlockInfo(safe)) => { - Self::Safe(safe) + EngineActorRequest::ProcessSafeL2Signal(ConsolidateInput::Delegated(heads)) => { + Self::Safe(heads) } other => Self::Other(format!("{other:?}")), } } } -/// What one poll of `status` forwards: the safe head, then the finalized block pinned by hash. +/// What one poll of `status` forwards: the safe heads, then the finalized block pinned by hash. fn forwarded_for(status: &SyncStatus) -> Vec { vec![ - Forwarded::Safe(status.safe_l2), + Forwarded::Safe(Box::new( + DelegatedSafeHeads::try_new(status.local_safe_l2, status.safe_l2).unwrap(), + )), Forwarded::Finalized(FinalizeBlockId::ByHash(status.finalized_l2.block_info.id())), ] } @@ -270,6 +273,7 @@ enum DelegateReply { CurrentL1NotCanonical, FetchFailed, FinalizedOriginNotCanonical, + LocalSafeOriginNotCanonical, SafeOriginNotCanonical, Valid, } @@ -287,6 +291,9 @@ impl DelegateReply { Self::FinalizedOriginNotCanonical => { status.finalized_l2.l1_origin = non_canonical(status.finalized_l2.l1_origin) } + Self::LocalSafeOriginNotCanonical => { + status.local_safe_l2.l1_origin = non_canonical(status.local_safe_l2.l1_origin) + } Self::SafeOriginNotCanonical => { status.safe_l2.l1_origin = non_canonical(status.safe_l2.l1_origin) } @@ -315,6 +322,7 @@ async fn polls_wait_for_engine_sync_completion() { #[rstest] #[case::valid(DelegateReply::Valid, true)] #[case::fetch_failed(DelegateReply::FetchFailed, false)] +#[case::local_safe_origin_not_canonical(DelegateReply::LocalSafeOriginNotCanonical, false)] #[case::safe_origin_not_canonical(DelegateReply::SafeOriginNotCanonical, false)] #[case::finalized_origin_not_canonical(DelegateReply::FinalizedOriginNotCanonical, false)] #[case::current_l1_not_canonical(DelegateReply::CurrentL1NotCanonical, false)] @@ -408,6 +416,33 @@ async fn poll_rejects_status_with_labels_out_of_order( assert_eq!(harness.current_l1.borrow().is_some(), accepted); } +/// The delegate's local-safe and cross-safe heads are forwarded as separate labels, so a cross-safe +/// head that lags the local-safe head under interop is not raised to it, and before interop both +/// carry the same block. +#[rstest] +#[case::pre_interop(20, 20)] +#[case::cross_safe_lags_local_safe(18, 20)] +#[tokio::test(start_paused = true)] +async fn poll_forwards_local_safe_and_cross_safe_heads_separately( + #[case] safe: u64, + #[case] local_safe: u64, +) { + let status = SyncStatus { + local_safe_l2: l2_block(local_safe, SAFE_ORIGIN), + safe_l2: l2_block(safe, SAFE_ORIGIN), + ..delegate_status() + }; + let mut harness = Harness::new([Ok(status.clone())]); + harness.complete_engine_sync().await; + + harness.actor.step().await.unwrap(); + + let expected_heads = + DelegatedSafeHeads::try_new(l2_block(local_safe, SAFE_ORIGIN), l2_block(safe, SAFE_ORIGIN)) + .unwrap(); + assert_eq!(harness.forwarded()[0], Forwarded::Safe(Box::new(expected_heads))); +} + /// A zero local safe head is a label the delegate has not derived or does not report, so it does /// not put the safe head out of order and the status is forwarded. #[tokio::test(start_paused = true)] @@ -418,7 +453,8 @@ async fn unreported_local_safe_head_does_not_order_the_labels() { harness.actor.step().await.unwrap(); - assert_eq!(harness.forwarded(), forwarded_for(&status)); + let reported_as_safe = SyncStatus { local_safe_l2: status.safe_l2, ..status }; + assert_eq!(harness.forwarded(), forwarded_for(&reported_as_safe)); } /// An engine sync state with consistent labels: unsafe at 30, safe at `safe`, finalized at 15. @@ -436,7 +472,7 @@ fn finalized_only(status: &SyncStatus) -> Vec { vec![Forwarded::Finalized(FinalizeBlockId::ByHash(status.finalized_l2.block_info.id()))] } -/// A safe head behind the engine's is forwarded once while the execution layer applied the +/// Safe heads behind the engine's are forwarded once while the execution layer applied the /// engine's forkchoice, since the engine's answer to it cannot change; finalized still goes out /// on every poll. A moved engine safe head, or an execution layer that did not apply the last /// forkchoice, sends it again. @@ -445,7 +481,7 @@ fn finalized_only(status: &SyncStatus) -> Vec { #[case::el_not_applied(false, 25, forwarded_for)] #[case::engine_safe_moved(true, 26, forwarded_for)] #[tokio::test(start_paused = true)] -async fn lagging_safe_head_is_forwarded_once_per_engine_view( +async fn lagging_safe_heads_are_forwarded_once_per_engine_view( #[case] el_applied_last_forkchoice: bool, #[case] engine_safe_on_second_poll: u64, #[case] second_poll_forwards: fn(&SyncStatus) -> Vec, diff --git a/rust/kona/crates/node/service/src/actors/engine/queue_tests.rs b/rust/kona/crates/node/service/src/actors/engine/queue_tests.rs index cd3c695f383..defb95a85b2 100644 --- a/rust/kona/crates/node/service/src/actors/engine/queue_tests.rs +++ b/rust/kona/crates/node/service/src/actors/engine/queue_tests.rs @@ -5,11 +5,10 @@ use alloy_rpc_types_engine::{ ExecutionPayloadV1, ForkchoiceState, PayloadStatus, PayloadStatusEnum, }; use kona_engine::{ - ConsolidateInput, EngineState, EngineSyncStateUpdate, FinalizeBlockId, NoopBlockSink, - ResetRequest, + EngineState, EngineSyncStateUpdate, FinalizeBlockId, NoopBlockSink, ResetRequest, test_utils::{ - RpcMock, TestEngineStateBuilder, TestRpcBlock, expect_sync_start, test_block_info, - test_engine_client, test_l2_block, test_rpc_block, + RpcMock, TestEngineStateBuilder, TestRpcBlock, delegated_safe_heads, expect_sync_start, + test_block_info, test_engine_client, test_l2_block, test_rpc_block, }, }; use op_alloy_consensus::{OpBlock, OpTxEnvelope}; @@ -118,7 +117,8 @@ async fn delegated_safe_head_is_sent_on_every_poll() { // accepts the next poll. for _ in 0..3 { requests - .send(EngineActorRequest::ProcessSafeL2Signal(ConsolidateInput::BlockInfo( + .send(EngineActorRequest::ProcessSafeL2Signal(delegated_safe_heads( + delegated_safe_head, delegated_safe_head, ))) .await @@ -143,6 +143,7 @@ async fn delegated_poll_during_el_sync_moves_safe_labels_and_skips_finalization( let (client, l1, l2) = test_engine_client(config.clone()); let state = TestEngineStateBuilder::new() .with_unsafe_head(test_block_info(20)) + .with_local_safe_head(test_block_info(5)) .with_safe_head(test_block_info(5)) .with_finalized_head(test_block_info(3)) .build(); @@ -175,7 +176,8 @@ async fn delegated_poll_during_el_sync_moves_safe_labels_and_skips_finalization( // One poll: the safe head, then the finalized block pinned by hash. Each step drains the // previous request's task and then accepts the next request. requests - .send(EngineActorRequest::ProcessSafeL2Signal(ConsolidateInput::BlockInfo( + .send(EngineActorRequest::ProcessSafeL2Signal(delegated_safe_heads( + delegated_safe_head, delegated_safe_head, ))) .await @@ -319,7 +321,13 @@ async fn finalization_target_survives_resets_below_it(#[case] target_kind: Targe requests.send(target_kind.finalize_request(target)).await.unwrap(); actor.step().await.unwrap(); assert!(second_reset_result.try_recv().unwrap().is_ok()); - requests.send(EngineActorRequest::ProcessSafeL2Signal(restored_safe.into())).await.unwrap(); + requests + .send(EngineActorRequest::ProcessSafeL2Signal(delegated_safe_heads( + restored_safe, + restored_safe, + ))) + .await + .unwrap(); actor.step().await.unwrap(); assert_eq!(actor.engine.len(), 1, "only the safe head update may remain queued"); assert!(!actor.reset_pending); diff --git a/rust/kona/crates/node/service/src/actors/engine/tests.rs b/rust/kona/crates/node/service/src/actors/engine/tests.rs index b61712d95b5..4d43b98d03a 100644 --- a/rust/kona/crates/node/service/src/actors/engine/tests.rs +++ b/rust/kona/crates/node/service/src/actors/engine/tests.rs @@ -4,7 +4,7 @@ use alloy_primitives::B256; use alloy_rpc_types_engine::{ForkchoiceUpdated, PayloadStatus, PayloadStatusEnum}; use kona_engine::{ Engine, EngineActorRequest, EngineState, EngineSyncStateUpdate, FinalizeBlockId, NoopBlockSink, - test_utils::{TestAttributesBuilder, test_engine_client}, + test_utils::{TestAttributesBuilder, delegated_safe_heads, test_engine_client}, }; use kona_genesis::RollupConfig; use kona_protocol::{BlockInfo, L2BlockInfo}; @@ -75,7 +75,7 @@ async fn invalid_forkchoice_resets_and_resumes(#[case] rebuild: bool, #[case] ho .build() .into() } else { - l2_block(1).into() + delegated_safe_heads(l2_block(1), l2_block(1)) }; requests.send(EngineActorRequest::ProcessSafeL2Signal(signal)).await.unwrap(); actor.step().await.unwrap(); @@ -98,7 +98,10 @@ async fn invalid_forkchoice_resets_and_resumes(#[case] rebuild: bool, #[case] ho ForkchoiceUpdated::new(PayloadStatus::from_status(PayloadStatusEnum::Valid)), ); let recovered = l2_block(2); - requests.send(EngineActorRequest::ProcessSafeL2Signal(recovered.into())).await.unwrap(); + requests + .send(EngineActorRequest::ProcessSafeL2Signal(delegated_safe_heads(recovered, recovered))) + .await + .unwrap(); actor.step().await.unwrap(); assert_eq!(state_rx.borrow().sync_state.unsafe_head(), genesis, "publish the reset head"); assert_eq!(state_rx.borrow().sync_state.safe_head(), genesis); From 8854cc00aef8a250185c0e0b64c15ea0d54515e5 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Wed, 11 Nov 2026 18:16:48 +0100 Subject: [PATCH 6/6] kona-node-service,op-acceptance-tests: exercise delegated derivation recovery end to end MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit #23947 runs the follow-source tests against kona-node, but they pass on develop too, so catch-up, SYNCING→VALID, stale finality and reset recovery are still unexercised. This adds end-to-end scenarios that run the real delegate and engine actors against a scripted delegate and a strict Engine API mock that checks every forkchoice update and lookup by hash, one per #545 completion criterion, plus local-safe ahead of cross-safe and a reorg that the EL first answers `SYNCING`. Each step waits for the engine's published labels and ends with a quiet window longer than the engine's longest retry, the poll cadence and the published derivation origin are asserted, and a stopped actor reports its own panic. A follow_l2 acceptance test pauses the sequencer once kona-node follows its source (pausing earlier strands it at genesis, #24030), waits on both layers with progress-bounded waits, and compares kona-node's labels with its EL and the sequencer's chain by hash. The scenarios found the forwarded-versus-applied finalized bug fixed in #23899. Production code changes only in retry and poll-interval constants becoming module-level. Closes #23912. Gaps that need real-EL fault injection are listed on ethereum-optimism/protocol-team#545. Co-Authored-By: Claude Opus 5.5 --- .../sync/follow_l2/paused_sequencer_test.go | 65 ++ .../src/actors/derivation/delegated/mod.rs | 6 + .../actors/derivation/delegated/scenarios.rs | 647 ++++++++++++++++++ .../actors/derivation/delegated/test_utils.rs | 114 +++ .../src/actors/derivation/delegated/tests.rs | 62 +- .../node/service/src/actors/engine/actor.rs | 15 +- .../node/service/src/actors/engine/mod.rs | 2 + 7 files changed, 875 insertions(+), 36 deletions(-) create mode 100644 op-acceptance-tests/tests/sync/follow_l2/paused_sequencer_test.go create mode 100644 rust/kona/crates/node/service/src/actors/derivation/delegated/scenarios.rs create mode 100644 rust/kona/crates/node/service/src/actors/derivation/delegated/test_utils.rs diff --git a/op-acceptance-tests/tests/sync/follow_l2/paused_sequencer_test.go b/op-acceptance-tests/tests/sync/follow_l2/paused_sequencer_test.go new file mode 100644 index 00000000000..3c2eb1e7547 --- /dev/null +++ b/op-acceptance-tests/tests/sync/follow_l2/paused_sequencer_test.go @@ -0,0 +1,65 @@ +package follow_l2 + +import ( + "testing" + "time" + + "github.com/ethereum-optimism/optimism/op-devstack/devtest" + "github.com/ethereum-optimism/optimism/op-devstack/dsl" + "github.com/ethereum-optimism/optimism/op-service/eth" + + safety "github.com/ethereum-optimism/optimism/op-service/eth/safety" +) + +// TestFollowL2_LabelsMatchELByHashWithSequencerPaused stops the sequencer, waits until the +// follow-source verifier is safe and finalized at the paused head, and then requires its +// consensus-layer labels to name the same blocks, by hash, as its execution layer's labels and +// the sequencer's paused head. With the sequencer paused the labels stop moving, so the two +// layers are compared at rest rather than racing new blocks. +func TestFollowL2_LabelsMatchELByHashWithSequencerPaused(gt *testing.T) { + t := devtest.ParallelT(gt) + sys := newSingleChainTwoVerifiersFollowL2(t) + require := t.Require() + // With the sequencer paused only the waited label itself can move, and it moves in jumps: when a + // batch lands on L1 for local-safe, and when that L1 block finalizes, about two minutes later, + // for finalized. The stall bound covers one such jump; a label stuck past it fails fast. + const maxWait = 8 * time.Minute + const stallTimeout = 4 * time.Minute + + // Local-safe above genesis means kona finished engine sync and follows the source. Pausing + // earlier strands kona-node at genesis (#24030), where every label would match trivially. + require.NoError(sys.L2CLC.ReachedWithProgressFn(safety.LocalSafe, safety.LocalUnsafe, 1, maxWait, stallTimeout)()) + + pausedHead := sys.L2EL.BlockRefByHash(sys.L2CL.StopSequencer()) + t.Logger().Info("Paused the sequencer", "head", pausedHead) + for _, label := range []struct { + consensusLayer safety.Level + executionLayer eth.BlockLabel + }{ + {safety.LocalSafe, eth.Safe}, + {safety.Finalized, eth.Finalized}, + } { + dsl.CheckAll(t, + sys.L2CLC.ReachedWithProgressFn(label.consensusLayer, label.consensusLayer, pausedHead.Number, maxWait, stallTimeout), + sys.L2ELC.ReachedWithProgressFn(label.executionLayer, label.executionLayer, pausedHead.Number, maxWait, stallTimeout), + ) + require.Equal(pausedHead.ID(), sys.L2CLC.HeadBlockRef(label.consensusLayer).ID(), "%s label must reach the paused head itself", label.consensusLayer) + } + + status := sys.L2CLC.SyncStatus() + require.Equal(status.SafeL2, status.LocalSafeL2, "delegated local-safe and safe must match outside interop") + for _, label := range []struct { + consensusLayer eth.L2BlockRef + executionLayer eth.BlockLabel + }{ + {status.UnsafeL2, eth.Unsafe}, + {status.SafeL2, eth.Safe}, + {status.FinalizedL2, eth.Finalized}, + } { + require.Equal(pausedHead, label.consensusLayer, "%s label must stop at the paused head", label.executionLayer) + require.Equal(label.consensusLayer, sys.L2ELC.BlockRefByLabel(label.executionLayer), "%s label differs between the layers", label.executionLayer) + } + // Every label is the paused head, so it is on the sequencer's chain as long as the sequencer + // still has the paused head as its own head. + require.Equal(pausedHead, sys.L2EL.BlockRefByLabel(eth.Unsafe), "the sequencer's head moved while paused") +} diff --git a/rust/kona/crates/node/service/src/actors/derivation/delegated/mod.rs b/rust/kona/crates/node/service/src/actors/derivation/delegated/mod.rs index c965fa802f6..061edc10154 100644 --- a/rust/kona/crates/node/service/src/actors/derivation/delegated/mod.rs +++ b/rust/kona/crates/node/service/src/actors/derivation/delegated/mod.rs @@ -1,5 +1,11 @@ mod actor; pub use actor::{DelegateDerivationActor, SyncStatusSource}; +#[cfg(test)] +mod scenarios; + +#[cfg(test)] +mod test_utils; + #[cfg(test)] mod tests; diff --git a/rust/kona/crates/node/service/src/actors/derivation/delegated/scenarios.rs b/rust/kona/crates/node/service/src/actors/derivation/delegated/scenarios.rs new file mode 100644 index 00000000000..9cc35c2835f --- /dev/null +++ b/rust/kona/crates/node/service/src/actors/derivation/delegated/scenarios.rs @@ -0,0 +1,647 @@ +//! Delegated derivation end to end: the real actors against a scripted delegate and a strict EL. + +use super::{ + DelegateDerivationActor, + actor::DERIVATION_DELEGATE_POLL_INTERVAL as POLL_INTERVAL, + test_utils::{ScriptedDelegate, sync_status}, +}; +use crate::{ + EngineActor, EngineActorRequest, NodeActor, QueuedDerivationEngineClient, + QueuedEngineDerivationClient, ResetRequest, + actors::engine::{INITIAL_RETRY_DELAY, MAX_RETRY_DELAY}, +}; +use alloy_eips::BlockNumHash; +use alloy_primitives::B256; +use alloy_rpc_types_engine::PayloadStatusEnum; +use kona_derive::test_utils::TestChainProvider; +use kona_engine::{ + Engine, EngineState, EngineSyncState, EngineSyncStateUpdate, NoopBlockSink, + test_utils::{ + RpcMock, TestEngineStateBuilder, expect_sync_start, test_engine_client, test_l2_block, + test_rpc_block, + }, +}; +use kona_genesis::RollupConfig; +use kona_protocol::{BlockInfo, L2BlockInfo, SyncStatus}; +use op_alloy_consensus::OpBlock; +use std::sync::Arc; +use tokio::{ + sync::{mpsc, watch}, + task::{JoinError, JoinHandle}, + time::{self, Duration, Instant}, +}; + +/// How long the engine may take to reach the labels a step expects. +const LABELS_DEADLINE: Duration = Duration::from_secs(1); + +/// Long enough for any task the engine left queued to retry with an unexpected request. +const QUIET_TIME: Duration = MAX_RETRY_DELAY.saturating_add(INITIAL_RETRY_DELAY); + +/// Height of the canonical chain the scenarios draw blocks from. +const CHAIN_LENGTH: u64 = 40; + +/// The L1 origin every test L2 block reports, which the delegate actor checks is canonical. +const L1_ORIGIN: BlockInfo = + BlockInfo { hash: B256::ZERO, number: 0, parent_hash: B256::ZERO, timestamp: 0 }; + +/// The canonical L1 block a delegate reports as its `current_l1` when its safe head is at +/// `number`, so each accepted poll publishes a distinct derivation origin. +fn l1_block(number: u64) -> BlockInfo { + if number == 0 { + return L1_ORIGIN; + } + BlockInfo { hash: B256::with_last_byte(number as u8), number, ..Default::default() } +} + +/// A canonical L2 chain of decodable blocks, linked by parent hash from the configured genesis. +struct TestChain { + blocks: Vec, + config: Arc, + infos: Vec, +} + +impl TestChain { + fn new() -> Self { + let genesis = OpBlock::default(); + let mut config = RollupConfig::default(); + config.genesis.l2 = BlockNumHash { hash: genesis.header.hash_slow(), number: 0 }; + let mut blocks = vec![genesis]; + for number in 1..=CHAIN_LENGTH { + let parent_hash = blocks[number as usize - 1].header.hash_slow(); + blocks.push(test_l2_block(number, parent_hash, 0)); + } + let infos = blocks.iter().map(|block| l2_block_info(block, &config)).collect(); + Self { blocks, config: Arc::new(config), infos } + } + + fn block(&self, number: u64) -> L2BlockInfo { + self.infos[number as usize] + } + + /// A block at `number` on a fork that leaves the canonical chain right below it. + fn fork_block(&self, number: u64) -> L2BlockInfo { + let parent_hash = self.block(number - 1).block_info.hash; + l2_block_info(&test_l2_block(number, parent_hash, 1), &self.config) + } + + fn el_serves(&self, el: &RpcMock, number: u64) { + el.expect_block_by_number(number, Some(test_rpc_block(&self.blocks[number as usize]))); + } + + fn el_serves_header(&self, el: &RpcMock, number: u64) { + el.expect_block_header_by_number( + number, + Some(test_rpc_block(&self.blocks[number as usize])), + ); + } + + fn el_lacks(&self, el: &RpcMock, number: u64) { + el.expect_block_by_number(number, None); + } + + /// Expects the reset's sync start to find the execution layer's labels all at genesis. + fn el_reset_to_genesis(&self, l1: &RpcMock, el: &RpcMock) { + let genesis = self.block(0); + expect_sync_start( + l1, + el, + &test_rpc_block(&self.blocks[0]), + all_at(genesis).sync_state.create_forkchoice_state(), + ); + } +} + +fn l2_block_info(block: &OpBlock, config: &RollupConfig) -> L2BlockInfo { + L2BlockInfo::from_block_and_genesis(block, &config.genesis).unwrap() +} + +/// A synced engine whose local-safe and safe heads are the same block, as before interop. +fn synced_engine( + finalized: L2BlockInfo, + safe: L2BlockInfo, + unsafe_head: L2BlockInfo, +) -> EngineState { + TestEngineStateBuilder::new() + .with_finalized_head(finalized) + .with_safe_head(safe) + .with_unsafe_head(unsafe_head) + .build() +} + +fn all_at(block: L2BlockInfo) -> EngineState { + synced_engine(block, block, block) +} + +/// The label moves the scenarios expect of the engine. +trait LabelMoves { + fn with_finalized(self, finalized: L2BlockInfo) -> Self; + + /// Local-safe and safe both at `safe`, as before interop. + fn with_safe(self, safe: L2BlockInfo) -> Self; + + fn with_safe_heads(self, local_safe: L2BlockInfo, safe: L2BlockInfo) -> Self; + + /// The unsafe head moved, forward or back, to a delegated safe head. + fn with_safe_and_unsafe(self, safe: L2BlockInfo) -> Self; +} + +impl LabelMoves for EngineSyncState { + fn with_finalized(self, finalized: L2BlockInfo) -> Self { + self.apply_update(EngineSyncStateUpdate { + finalized_head: Some(finalized), + ..Default::default() + }) + } + + fn with_safe(self, safe: L2BlockInfo) -> Self { + self.with_safe_heads(safe, safe) + } + + fn with_safe_heads(self, local_safe: L2BlockInfo, safe: L2BlockInfo) -> Self { + self.apply_update(EngineSyncStateUpdate { + local_safe_head: Some(local_safe), + safe_head: Some(safe), + ..Default::default() + }) + } + + fn with_safe_and_unsafe(self, safe: L2BlockInfo) -> Self { + self.with_safe(safe) + .apply_update(EngineSyncStateUpdate { unsafe_head: Some(safe), ..Default::default() }) + } +} + +fn pre_interop_status(safe: L2BlockInfo, finalized: L2BlockInfo) -> SyncStatus { + sync_status(l1_block(safe.block_info.number), safe, safe, finalized) +} + +/// A delegated kona-node whose delegate and engine actors run as tasks, as in production. +struct DelegatedNode { + answered_polls: watch::Receiver, + answered_polls_seen: u32, + delegate: ScriptedDelegate, + delegate_task: JoinHandle<()>, + derivation_origin: watch::Receiver>, + el: RpcMock, + engine_requests: mpsc::Sender, + engine_state: watch::Receiver, + engine_task: JoinHandle<()>, + l1: RpcMock, + polls: watch::Receiver, + polls_checked: u32, + polls_checked_at: Instant, +} + +impl DelegatedNode { + /// Starts the node at `initial`; the delegate first polls once the engine reports sync + /// complete. + fn start(chain: &TestChain, initial: EngineState) -> Self { + let (client, l1, el) = test_engine_client(chain.config.clone()); + let (derivation_request_tx, derivation_request_rx) = mpsc::channel(32); + let (engine_requests, engine_request_rx) = mpsc::channel(32); + let (engine_state_tx, engine_state) = watch::channel(initial); + let engine = EngineActor::new( + Arc::new(client), + chain.config.clone(), + QueuedEngineDerivationClient { derivation_actor_request_tx: derivation_request_tx }, + Engine::new(initial, engine_state_tx), + None, + engine_request_rx, + Arc::new(NoopBlockSink), + ); + + let mut canonical_l1 = TestChainProvider::default(); + for number in 0..=CHAIN_LENGTH { + canonical_l1.insert_block(number, l1_block(number)); + } + let delegate = ScriptedDelegate::lenient(); + let (derivation_origin_tx, derivation_origin) = watch::channel(None); + let delegate_actor = DelegateDerivationActor::new( + QueuedDerivationEngineClient::new(engine_requests.clone()), + derivation_request_rx, + delegate.clone(), + canonical_l1, + derivation_origin_tx, + engine_state.clone(), + ); + + Self { + answered_polls: delegate.answered_polls(), + answered_polls_seen: 0, + delegate_task: run(delegate_actor, "delegate"), + derivation_origin, + el, + engine_requests, + engine_state, + engine_task: run(engine, "engine"), + l1, + polls: delegate.polls(), + polls_checked: 0, + polls_checked_at: Instant::now(), + delegate, + } + } + + /// Scripts the next poll's reply: `safe` as both the local-safe and the safe head. + fn delegate_reports(&self, safe: L2BlockInfo, finalized: L2BlockInfo) { + self.delegate.reports(pre_interop_status(safe, finalized)); + } + + /// Requests an engine reset, which runs on the engine's next step. + async fn request_reset(&self) { + let (result_tx, _result_rx) = mpsc::channel(1); + self.engine_requests + .send(EngineActorRequest::Reset(Box::new(ResetRequest { result_tx }))) + .await + .unwrap(); + } + + /// Waits for the delegate to answer the scripted poll. + async fn next_poll(&mut self) { + if time::timeout(POLL_INTERVAL * 2, self.answered_polls.changed()).await.is_err() { + self.assert_running().await; + panic!("the delegate did not answer a poll within {:?}", POLL_INTERVAL * 2); + } + let answered = *self.answered_polls.borrow_and_update(); + assert_eq!(answered, self.answered_polls_seen + 1, "the scenario missed a delegate poll"); + self.answered_polls_seen = answered; + self.assert_poll_cadence(); + } + + /// Fails if the delegate was polled more often than once per poll interval since the last + /// check. Unscripted polls fail and change nothing, so only their count can catch an actor that + /// polls too often. + fn assert_poll_cadence(&mut self) { + let polls = *self.polls.borrow(); + let elapsed = Instant::now() - self.polls_checked_at; + let allowed = elapsed.as_nanos() / POLL_INTERVAL.as_nanos() + 1; + let polled = u128::from(polls - self.polls_checked); + assert!( + polled <= allowed, + "the delegate was polled {polled} times in {elapsed:?}, at most {allowed} expected" + ); + self.polls_checked = polls; + self.polls_checked_at = Instant::now(); + } + + /// Waits `duration`, failing if the delegate answers a poll meanwhile and its requests mix with + /// these. Polls with no reply scripted fail and change nothing. + async fn wait_without_answered_poll(&mut self, duration: Duration) { + time::sleep(duration).await; + assert_eq!( + *self.answered_polls.borrow(), + self.answered_polls_seen, + "the delegate answered a poll during a {duration:?} wait" + ); + self.assert_poll_cadence(); + self.assert_running().await; + } + + /// Fails with every stopped actor's panic, so a secondary failure cannot hide the first. + async fn assert_running(&mut self) { + let mut stopped = Vec::new(); + for (name, task) in + [("engine", &mut self.engine_task), ("delegate", &mut self.delegate_task)] + { + if task.is_finished() { + stopped.push(format!("{name}: {}", stop_reason(task.await))); + } + } + assert!(stopped.is_empty(), "actors stopped:\n{}", stopped.join("\n")); + } + + /// Waits for the engine to reach `expected`, then asserts it, and every scripted reply and + /// request consumed, once any task the engine left queued has had time to retry. + async fn assert_labels(&mut self, expected: EngineSyncState) { + let reached = time::timeout( + LABELS_DEADLINE, + self.engine_state.wait_for(|state| state.sync_state == expected), + ) + .await + .is_ok_and(|watched| watched.is_ok()); + if !reached { + self.assert_running().await; + panic!( + "the engine did not reach the expected labels within {LABELS_DEADLINE:?}: {:?}", + self.engine_state.borrow().sync_state + ); + } + self.wait_without_answered_poll(QUIET_TIME).await; + self.delegate.assert_finished(); + self.l1.assert_finished(); + self.el.assert_finished(); + assert_eq!(self.engine_state.borrow().sync_state, expected); + } + + /// Asserts the derivation origin the delegate actor last published. + fn assert_origin(&self, expected: Option) { + assert_eq!(*self.derivation_origin.borrow(), expected, "published derivation origin"); + } + + /// Polls canonical `safe` and `finalized`, each applied by a forkchoice update; returns the + /// labels. A safe head below the unsafe head is looked up first; one at it extends the chain. + async fn poll_applies_safe_and_finalized( + &mut self, + chain: &TestChain, + labels: EngineSyncState, + safe: u64, + finalized: u64, + ) -> EngineSyncState { + self.delegate_reports(chain.block(safe), chain.block(finalized)); + if safe < labels.unsafe_head().block_info.number { + chain.el_serves(&self.el, safe); + } + let labels = labels.with_safe(chain.block(safe)); + self.expect_forkchoice(labels, PayloadStatusEnum::Valid); + chain.el_serves(&self.el, finalized); + let labels = labels.with_finalized(chain.block(finalized)); + self.expect_forkchoice(labels, PayloadStatusEnum::Valid); + self.next_poll().await; + self.assert_labels(labels).await; + self.assert_origin(Some(l1_block(safe))); + labels + } + + fn expect_forkchoice(&self, labels: EngineSyncState, status: PayloadStatusEnum) { + self.el.expect_forkchoice_updated(labels.create_forkchoice_state(), status); + } +} + +impl Drop for DelegatedNode { + /// Stops both actors, also when a scenario panics, so their later failures cannot interleave + /// with its message. + fn drop(&mut self) { + self.engine_task.abort(); + self.delegate_task.abort(); + } +} + +/// Why an actor task stopped: its panic message, or the join error. +fn stop_reason(result: Result<(), JoinError>) -> String { + match result { + Ok(()) => "returned".to_string(), + Err(err) if err.is_panic() => { + let payload = err.into_panic(); + payload + .downcast_ref::() + .cloned() + .or_else(|| payload.downcast_ref::<&str>().map(|message| (*message).to_string())) + .unwrap_or_else(|| "a panic without a message".to_string()) + } + Err(err) => err.to_string(), + } +} + +/// Steps `actor` until it fails, which ends the task with a panic carrying the error. +fn run(mut actor: Actor, name: &'static str) -> JoinHandle<()> +where + Actor: NodeActor + Send + 'static, + Actor::Error: std::fmt::Debug, +{ + tokio::spawn(async move { + loop { + if let Err(err) = actor.step().await { + panic!("{name} actor stopped: {err:?}"); + } + } + }) +} + +/// With no unsafe blocks, safe and finalized follow the delegate by hash and match the last FCU. +#[tokio::test(start_paused = true)] +async fn safe_and_finalized_progress_by_hash_with_unsafe_production_paused() { + let chain = TestChain::new(); + let initial = synced_engine(chain.block(5), chain.block(10), chain.block(30)); + let mut node = DelegatedNode::start(&chain, initial); + + let mut labels = initial.sync_state; + for (safe, finalized) in [(20, 10), (25, 20), (30, 25)] { + labels = node.poll_applies_safe_and_finalized(&chain, labels, safe, finalized).await; + } +} + +/// A delegated safe head on another branch reorgs the EL onto it: the EL syncs the branch while +/// the shared ancestor finalizes, then validates it on the next poll. +#[tokio::test(start_paused = true)] +async fn delegated_safe_head_on_another_branch_reorgs_the_el_onto_it() { + let chain = TestChain::new(); + let initial = synced_engine(chain.block(5), chain.block(10), chain.block(30)); + let (fork_safe, shared_finalized) = (chain.fork_block(25), chain.block(20)); + let reorged = initial.sync_state.with_safe_and_unsafe(fork_safe); + let finalized_on_branch = reorged.with_finalized(shared_finalized); + let mut node = DelegatedNode::start(&chain, initial); + + // The EL still has the canonical block at 25, which proves the divergence, and syncs the + // branch. + node.delegate_reports(fork_safe, shared_finalized); + chain.el_serves(&node.el, fork_safe.block_info.number); + node.expect_forkchoice(reorged, PayloadStatusEnum::Syncing); + chain.el_serves(&node.el, shared_finalized.block_info.number); + node.expect_forkchoice(finalized_on_branch, PayloadStatusEnum::Syncing); + node.next_poll().await; + node.assert_labels(finalized_on_branch).await; + + // Once the EL has the branch, the same target extends the engine's chain and is validated. + node.delegate_reports(fork_safe, shared_finalized); + node.expect_forkchoice(finalized_on_branch, PayloadStatusEnum::Valid); + node.next_poll().await; + node.assert_labels(finalized_on_branch).await; +} + +/// An unchanged target is resent through `SYNCING` until `VALID`, then finalizes once. +#[tokio::test(start_paused = true)] +async fn source_ahead_catch_up_retries_unchanged_target_through_syncing_until_valid() { + let chain = TestChain::new(); + let initial = synced_engine(chain.block(5), chain.block(10), chain.block(10)); + let (target_safe, target_finalized) = (chain.block(30), chain.block(20)); + let catching_up = initial.sync_state.with_safe_and_unsafe(target_safe); + let caught_up = catching_up.with_finalized(target_finalized); + let mut node = DelegatedNode::start(&chain, initial); + + for _ in 0..2 { + node.delegate_reports(target_safe, target_finalized); + node.expect_forkchoice(catching_up, PayloadStatusEnum::Syncing); + chain.el_lacks(&node.el, target_finalized.block_info.number); + node.next_poll().await; + node.assert_labels(catching_up).await; + } + + node.delegate_reports(target_safe, target_finalized); + node.expect_forkchoice(catching_up, PayloadStatusEnum::Valid); + chain.el_serves(&node.el, target_finalized.block_info.number); + node.expect_forkchoice(caught_up, PayloadStatusEnum::Valid); + node.next_poll().await; + node.assert_labels(caught_up).await; + + node.delegate_reports(target_safe, target_finalized); + node.expect_forkchoice(caught_up, PayloadStatusEnum::Valid); + node.next_poll().await; + node.assert_labels(caught_up).await; +} + +/// A stale or diverged finalized reference from the delegate. +#[derive(Clone, Copy, Debug)] +enum StaleFinalized { + /// A different block above the engine's finalized head, at a height the execution layer has. + ConflictingAbove, + /// A different block at the engine's finalized height. + ConflictingAtFinalizedHeight, + /// A block below the engine's finalized head, from a delegate that lags. + Regressed, +} + +impl StaleFinalized { + /// The status a stale delegate reports while the engine is safe at 20 and finalized at 15. + fn status(self, chain: &TestChain) -> SyncStatus { + match self { + Self::ConflictingAbove => pre_interop_status(chain.block(20), chain.fork_block(17)), + Self::ConflictingAtFinalizedHeight => { + pre_interop_status(chain.block(20), chain.fork_block(15)) + } + Self::Regressed => pre_interop_status(chain.block(18), chain.block(12)), + } + } + + /// What the EL is asked: only a conflicting target above finality reaches the engine. + fn expect_engine_requests( + self, + chain: &TestChain, + node: &DelegatedNode, + labels: EngineSyncState, + ) { + match self { + Self::ConflictingAbove => { + chain.el_serves(&node.el, 20); + node.expect_forkchoice(labels, PayloadStatusEnum::Valid); + chain.el_serves(&node.el, 17); + } + Self::ConflictingAtFinalizedHeight | Self::Regressed => {} + } + } +} + +/// Starts a node safe at 20 whose first poll finalizes block 15. +async fn start_finalized_at_15(chain: &TestChain) -> (DelegatedNode, EngineSyncState) { + let initial = synced_engine(chain.block(10), chain.block(20), chain.block(30)); + let mut node = DelegatedNode::start(chain, initial); + let finalized_at_15 = + node.poll_applies_safe_and_finalized(chain, initial.sync_state, 20, 15).await; + (node, finalized_at_15) +} + +/// Stale or conflicting finality never moves finalized; the next canonical one applies. +#[rstest::rstest] +#[case::conflicting_above(StaleFinalized::ConflictingAbove, 25, 20)] +#[case::conflicting_above_then_canonical_at_its_height(StaleFinalized::ConflictingAbove, 20, 17)] +#[case::conflicting_at_finalized_height(StaleFinalized::ConflictingAtFinalizedHeight, 25, 20)] +#[case::regressed(StaleFinalized::Regressed, 25, 20)] +#[tokio::test(start_paused = true)] +async fn stale_source_finalized_reference_never_moves_finalized_back( + #[case] stale: StaleFinalized, + #[case] recovered_safe: u64, + #[case] recovered_finalized: u64, +) { + let chain = TestChain::new(); + let (mut node, finalized_at_15) = start_finalized_at_15(&chain).await; + + node.delegate.reports(stale.status(&chain)); + stale.expect_engine_requests(&chain, &node, finalized_at_15); + node.next_poll().await; + node.assert_labels(finalized_at_15).await; + node.assert_origin(Some(l1_block(20))); + + node.poll_applies_safe_and_finalized( + &chain, + finalized_at_15, + recovered_safe, + recovered_finalized, + ) + .await; +} + +/// Local-safe and cross-safe labels move separately; finalizing above cross-safe is rejected whole. +#[tokio::test(start_paused = true)] +async fn local_safe_ahead_of_cross_safe_moves_each_label_separately_and_finalizes_below_cross_safe() +{ + let chain = TestChain::new(); + let initial = synced_engine(chain.block(5), chain.block(10), chain.block(35)); + let mut node = DelegatedNode::start(&chain, initial); + + node.delegate.reports(sync_status( + L1_ORIGIN, + chain.block(25), + chain.block(20), + chain.block(18), + )); + chain.el_serves(&node.el, 25); + chain.el_serves_header(&node.el, 20); + let cross_safe_at_20 = initial.sync_state.with_safe_heads(chain.block(25), chain.block(20)); + node.expect_forkchoice(cross_safe_at_20, PayloadStatusEnum::Valid); + chain.el_serves(&node.el, 18); + let finalized_at_18 = cross_safe_at_20.with_finalized(chain.block(18)); + node.expect_forkchoice(finalized_at_18, PayloadStatusEnum::Valid); + node.next_poll().await; + node.assert_labels(finalized_at_18).await; + + node.delegate.reports(sync_status( + L1_ORIGIN, + chain.block(30), + chain.block(22), + chain.block(25), + )); + node.next_poll().await; + node.assert_labels(finalized_at_18).await; + + node.delegate.reports(sync_status( + L1_ORIGIN, + chain.block(32), + chain.block(28), + chain.block(25), + )); + chain.el_serves(&node.el, 32); + chain.el_serves_header(&node.el, 28); + let cross_safe_at_28 = finalized_at_18.with_safe_heads(chain.block(32), chain.block(28)); + node.expect_forkchoice(cross_safe_at_28, PayloadStatusEnum::Valid); + chain.el_serves(&node.el, 25); + let finalized_at_25 = cross_safe_at_28.with_finalized(chain.block(25)); + node.expect_forkchoice(finalized_at_25, PayloadStatusEnum::Valid); + node.next_poll().await; + node.assert_labels(finalized_at_25).await; +} + +/// From a wiped EL, a reset returns to genesis and the target finalizes once the EL has it. +#[tokio::test(start_paused = true)] +async fn reset_to_genesis_while_finalized_is_missing_finalizes_once_the_el_has_it() { + let chain = TestChain::new(); + let genesis = all_at(chain.block(0)).sync_state; + let (target_safe, target_finalized) = (chain.block(30), chain.block(20)); + let catching_up = genesis.with_safe_and_unsafe(target_safe); + let caught_up = catching_up.with_finalized(target_finalized); + let mut node = DelegatedNode::start(&chain, EngineState::default()); + + // The startup reset finds only genesis, and the first poll starts the EL's sync. + chain.el_reset_to_genesis(&node.l1, &node.el); + node.delegate_reports(target_safe, target_finalized); + node.expect_forkchoice(catching_up, PayloadStatusEnum::Syncing); + chain.el_lacks(&node.el, target_finalized.block_info.number); + node.request_reset().await; + node.next_poll().await; + node.assert_labels(catching_up).await; + + chain.el_reset_to_genesis(&node.l1, &node.el); + node.request_reset().await; + node.assert_labels(genesis).await; + + node.delegate_reports(target_safe, target_finalized); + node.expect_forkchoice(catching_up, PayloadStatusEnum::Syncing); + chain.el_lacks(&node.el, target_finalized.block_info.number); + node.next_poll().await; + node.assert_labels(catching_up).await; + + node.delegate_reports(target_safe, target_finalized); + node.expect_forkchoice(catching_up, PayloadStatusEnum::Valid); + chain.el_serves(&node.el, target_finalized.block_info.number); + node.expect_forkchoice(caught_up, PayloadStatusEnum::Valid); + node.next_poll().await; + node.assert_labels(caught_up).await; +} diff --git a/rust/kona/crates/node/service/src/actors/derivation/delegated/test_utils.rs b/rust/kona/crates/node/service/src/actors/derivation/delegated/test_utils.rs new file mode 100644 index 00000000000..2f6104e5b1e --- /dev/null +++ b/rust/kona/crates/node/service/src/actors/derivation/delegated/test_utils.rs @@ -0,0 +1,114 @@ +//! A scripted derivation delegate and sync status builder shared by the delegated actor's tests. + +use super::SyncStatusSource; +use async_trait::async_trait; +use kona_protocol::{BlockInfo, L2BlockInfo, SyncStatus}; +use std::{ + collections::VecDeque, + sync::{Arc, Mutex}, +}; +use tokio::sync::watch; + +pub(super) type SyncStatusReply = Result; + +/// What a poll with no reply scripted does. +#[derive(Clone, Copy, Debug)] +enum UnscriptedPoll { + /// Fails like an unreachable delegate, which the actor skips. + Fails, + /// Panics, so a test that scripts every poll catches an extra one. + Panics, +} + +/// Answers each poll with the next scripted reply. It counts every poll, and separately the polls +/// it answered with a sync status. +#[derive(Clone, Debug)] +pub(super) struct ScriptedDelegate { + answered_polls: watch::Sender, + polls: watch::Sender, + replies: Arc>>, + unscripted_poll: UnscriptedPoll, +} + +impl ScriptedDelegate { + /// A delegate whose every poll must be scripted. + pub(super) fn strict(replies: impl IntoIterator) -> Self { + Self::new(replies, UnscriptedPoll::Panics) + } + + /// A delegate that fails the polls nothing is scripted for, so waits can span them. + pub(super) fn lenient() -> Self { + Self::new([], UnscriptedPoll::Fails) + } + + fn new( + replies: impl IntoIterator, + unscripted_poll: UnscriptedPoll, + ) -> Self { + Self { + answered_polls: watch::Sender::new(0), + polls: watch::Sender::new(0), + replies: Arc::new(Mutex::new(replies.into_iter().collect())), + unscripted_poll, + } + } + + /// Scripts `status` as the reply to the next unanswered poll. + pub(super) fn reports(&self, status: SyncStatus) { + self.replies.lock().unwrap().push_back(Ok(status)); + } + + /// Counts the polls answered with a sync status. + pub(super) fn answered_polls(&self) -> watch::Receiver { + self.answered_polls.subscribe() + } + + /// Counts every poll, answered or not. + pub(super) fn polls(&self) -> watch::Receiver { + self.polls.subscribe() + } + + pub(super) fn assert_finished(&self) { + let unanswered = self.replies.lock().unwrap().len(); + assert_eq!(unanswered, 0, "{unanswered} scripted sync status replies were never polled"); + } +} + +#[async_trait] +impl SyncStatusSource for ScriptedDelegate { + type Error = &'static str; + + async fn sync_status(&self) -> SyncStatusReply { + self.polls.send_modify(|polls| *polls += 1); + let Some(reply) = self.replies.lock().unwrap().pop_front() else { + return match self.unscripted_poll { + UnscriptedPoll::Fails => Err("no sync status scripted"), + UnscriptedPoll::Panics => panic!("unexpected sync status poll"), + }; + }; + if reply.is_ok() { + self.answered_polls.send_modify(|polls| *polls += 1); + } + reply + } +} + +/// A sync status with these labels, its unsafe head at the local-safe head. +pub(super) fn sync_status( + current_l1: BlockInfo, + local_safe: L2BlockInfo, + safe: L2BlockInfo, + finalized: L2BlockInfo, +) -> SyncStatus { + SyncStatus { + current_l1, + current_l1_finalized: BlockInfo::default(), + finalized_l1: BlockInfo::default(), + finalized_l2: finalized, + head_l1: BlockInfo::default(), + local_safe_l2: local_safe, + safe_l1: BlockInfo::default(), + safe_l2: safe, + unsafe_l2: local_safe, + } +} diff --git a/rust/kona/crates/node/service/src/actors/derivation/delegated/tests.rs b/rust/kona/crates/node/service/src/actors/derivation/delegated/tests.rs index 738f6d4397b..b9e522b66a7 100644 --- a/rust/kona/crates/node/service/src/actors/derivation/delegated/tests.rs +++ b/rust/kona/crates/node/service/src/actors/derivation/delegated/tests.rs @@ -1,8 +1,11 @@ -use super::{DelegateDerivationActor, SyncStatusSource, actor::DERIVATION_DELEGATE_POLL_INTERVAL}; +use super::{ + DelegateDerivationActor, + actor::DERIVATION_DELEGATE_POLL_INTERVAL, + test_utils::{ScriptedDelegate, SyncStatusReply, sync_status}, +}; use crate::{DerivationActorRequest, EngineActorRequest, NodeActor, QueuedDerivationEngineClient}; use alloy_eips::BlockNumHash; use alloy_primitives::B256; -use async_trait::async_trait; use kona_derive::test_utils::TestChainProvider; use kona_engine::{ ConsolidateInput, DelegatedSafeHeads, EngineState, EngineSyncState, EngineSyncStateUpdate, @@ -10,37 +13,24 @@ use kona_engine::{ }; use kona_protocol::{BlockInfo, L2BlockInfo, SyncStatus}; use rstest::rstest; -use std::{collections::VecDeque, sync::Mutex}; use tokio::{ sync::{mpsc, watch}, time::{self, Instant}, }; +type HarnessActor = + DelegateDerivationActor; + const SAFE_ORIGIN: u64 = 1; const FINALIZED_ORIGIN: u64 = 2; const CURRENT_L1: u64 = 3; -type SyncStatusReply = Result; - -/// Answers each sync status poll with the next scripted reply. -#[derive(Debug)] -struct ScriptedDelegate(Mutex>); - -#[async_trait] -impl SyncStatusSource for ScriptedDelegate { - type Error = &'static str; - - async fn sync_status(&self) -> SyncStatusReply { - self.0.lock().unwrap().pop_front().expect("unexpected sync status poll") - } -} - /// A delegate actor over a canonical L1 chain, with the engine requests it sends and the /// `current_l1` it publishes. struct Harness { - actor: - DelegateDerivationActor, + actor: HarnessActor, current_l1: watch::Receiver>, + delegate: ScriptedDelegate, derivation_requests: mpsc::Sender, engine_requests: mpsc::Receiver, engine_state: watch::Sender, @@ -56,15 +46,16 @@ impl Harness { let (engine_request_tx, engine_requests) = mpsc::channel(8); let (current_l1_tx, current_l1) = watch::channel(None); let (engine_state, engine_state_rx) = watch::channel(EngineState::default()); - let actor = DelegateDerivationActor::new( + let delegate = ScriptedDelegate::strict(replies); + let actor = HarnessActor::new( QueuedDerivationEngineClient::new(engine_request_tx), derivation_request_rx, - ScriptedDelegate(Mutex::new(replies.into_iter().collect())), + delegate.clone(), l1, current_l1_tx, engine_state_rx, ); - Self { actor, current_l1, derivation_requests, engine_requests, engine_state } + Self { actor, current_l1, delegate, derivation_requests, engine_requests, engine_state } } async fn complete_engine_sync(&mut self) { @@ -104,6 +95,14 @@ impl Harness { } } +impl Drop for Harness { + fn drop(&mut self) { + if !std::thread::panicking() { + self.delegate.assert_finished(); + } + } +} + /// The parts of an engine request that the delegate actor decides. #[derive(Debug, PartialEq, Eq)] enum Forwarded { @@ -155,18 +154,17 @@ fn l2_block(number: u64, l1_origin: u64) -> L2BlockInfo { } } -/// A sync status whose safe origin, finalized origin and `current_l1` are all canonical. +/// A sync status whose safe origin, finalized origin and `current_l1` are all canonical, with an +/// unsafe head above its safe heads. fn delegate_status() -> SyncStatus { SyncStatus { - current_l1: canonical_l1_block(CURRENT_L1), - current_l1_finalized: BlockInfo::default(), - finalized_l1: BlockInfo::default(), - finalized_l2: l2_block(15, FINALIZED_ORIGIN), - head_l1: BlockInfo::default(), - local_safe_l2: l2_block(20, SAFE_ORIGIN), - safe_l1: BlockInfo::default(), - safe_l2: l2_block(20, SAFE_ORIGIN), unsafe_l2: l2_block(25, SAFE_ORIGIN), + ..sync_status( + canonical_l1_block(CURRENT_L1), + l2_block(20, SAFE_ORIGIN), + l2_block(20, SAFE_ORIGIN), + l2_block(15, FINALIZED_ORIGIN), + ) } } diff --git a/rust/kona/crates/node/service/src/actors/engine/actor.rs b/rust/kona/crates/node/service/src/actors/engine/actor.rs index b45bfa62238..546358ca87c 100644 --- a/rust/kona/crates/node/service/src/actors/engine/actor.rs +++ b/rust/kona/crates/node/service/src/actors/engine/actor.rs @@ -20,6 +20,13 @@ use tokio::{ #[path = "queue_tests.rs"] mod queue_tests; +/// The delay before the first retry after a dependency failure; a clean drain resets the backoff +/// to it. +pub(crate) const INITIAL_RETRY_DELAY: Duration = Duration::from_millis(100); + +/// The cap on the doubling retry delay, keeping recovery responsive. +pub(crate) const MAX_RETRY_DELAY: Duration = Duration::from_secs(5); + /// Responsible for managing the operations sent to the execution layer's Engine API. To accomplish /// this, it uses the [`Engine`] task queue to order Engine API interactions based off of /// the [`Ord`] implementation of [`EngineTask`]. @@ -92,7 +99,7 @@ where reset_pending: false, reset_waiters: Vec::new(), retry_at: Instant::now(), - retry_delay: Duration::from_millis(100), + retry_delay: INITIAL_RETRY_DELAY, } } @@ -110,7 +117,7 @@ where fn defer_retry(&mut self) { self.retry_at = Instant::now() + self.retry_delay; - self.retry_delay = (self.retry_delay * 2).min(Duration::from_secs(5)); + self.retry_delay = (self.retry_delay * 2).min(MAX_RETRY_DELAY); } fn begin_reset(&mut self) { @@ -126,7 +133,7 @@ where match self.reset().await { Ok(()) => { self.reset_pending = false; - self.retry_delay = Duration::from_millis(100); + self.retry_delay = INITIAL_RETRY_DELAY; if !self.el_sync_complete && self.engine.state().el_sync_finished { self.mark_el_sync_complete_and_notify_derivation_actor().await?; } @@ -192,7 +199,7 @@ where match self.engine.drain().await { Ok(_) => { trace!(target: "engine", "[ENGINE] tasks drained"); - self.retry_delay = Duration::from_millis(100); + self.retry_delay = INITIAL_RETRY_DELAY; } Err(err) => { match err.severity() { diff --git a/rust/kona/crates/node/service/src/actors/engine/mod.rs b/rust/kona/crates/node/service/src/actors/engine/mod.rs index ef8308ab059..e13242fa0dd 100644 --- a/rust/kona/crates/node/service/src/actors/engine/mod.rs +++ b/rust/kona/crates/node/service/src/actors/engine/mod.rs @@ -2,6 +2,8 @@ mod actor; pub use actor::EngineActor; +#[cfg(test)] +pub(crate) use actor::{INITIAL_RETRY_DELAY, MAX_RETRY_DELAY}; mod client; pub use client::{EngineDerivationClient, QueuedEngineDerivationClient};