diff --git a/docs/public-docs/node-operators/kona-node/configuration.mdx b/docs/public-docs/node-operators/kona-node/configuration.mdx index d851596373c..1b4adb066f9 100644 --- a/docs/public-docs/node-operators/kona-node/configuration.mdx +++ b/docs/public-docs/node-operators/kona-node/configuration.mdx @@ -251,6 +251,26 @@ The RPC server starts once. If it stops unexpectedly, the node shuts down with a `kona-node` does not run a supervisor RPC server, so there is no `--supervisor.*` flag group. `--interop.dependency-set` is the node's only interop flag. +## Derivation Delegation Arguments + +| Flag | Env | Description | Default | +|------|-----|-------------|---------| +| `--l2-follow-source ` (alias `--l2.follow.source`) | `KONA_NODE_L2_FOLLOW_SOURCE` | RPC URL of an OP Stack consensus client exposing `optimism_syncStatus`. When set, `kona-node` derives nothing itself and follows that client's safe and finalized heads instead. Cannot be combined with `--safedb.path`. | - | + +Like op-node's follow-source mode, `kona-node` imports the source's local-safe and cross-safe +heads separately. The local-safe head is the block the execution client must have, or sync to, +and becomes the local-safe label; the cross-safe head becomes the safe label that forkchoice +updates carry, and finalization never goes above it. The safe label moves back to a lower +cross-safe head only when the source's chain has diverged from the node's, or when the unsafe head +moves to a source block whose ancestry the node cannot check and the execution client does not hold +the current safe label, and never below the finalized head. While the execution client has not +applied kona-node's last forkchoice update, for example while it is still syncing, its canonical +blocks do not count as evidence: a delegated block it lacks, or one that differs from its block, +moves the safe label only forward. +Before interop both heads are the same block. +Under interop, `kona-node` verifies no cross-chain dependencies itself, so its safe label is only +as safe as the source's cross-safe head: the source must verify them. + ## RPC Trust Flags The `--l1-trust-rpc` and `--l2-trust-rpc` flags control whether Kona verifies the block hashes of RPC responses. For guidance on when to disable trust and worked examples for trusted, untrusted, and mixed provider setups, see [configure RPC trust](/node-operators/kona-node/run/rpc-trust). diff --git a/docs/public-docs/node-operators/kona-node/design/engine.mdx b/docs/public-docs/node-operators/kona-node/design/engine.mdx index a8472c5fc4b..77003e49cc1 100644 --- a/docs/public-docs/node-operators/kona-node/design/engine.mdx +++ b/docs/public-docs/node-operators/kona-node/design/engine.mdx @@ -145,7 +145,7 @@ pub struct ConsolidateTask { pub client: Arc, /// The RollupConfig. pub cfg: Arc, - /// The input for consolidation (either attributes or block info). + /// The input for consolidation (either derived attributes or delegated safe heads). pub input: ConsolidateInput, /// Where to hand the decoded block once the engine has canonicalized it. pub block_sink: Arc, diff --git a/op-acceptance-tests/tests/sdm/init_test.go b/op-acceptance-tests/tests/sdm/init_test.go index db96beb2240..643f7b6caec 100644 --- a/op-acceptance-tests/tests/sdm/init_test.go +++ b/op-acceptance-tests/tests/sdm/init_test.go @@ -43,7 +43,7 @@ func newSDMRethSystemWithIsolatedVerifier(t devtest.T) *sdmtest.RethSystem { // derivation actor stays in AwaitingELSyncCompletion, the EL gets no engine calls, and the // safe head never leaves genesis. kona-node has no L1-only bootstrap of the force-build path // today, so this test cannot pass under kona-node and is op-node-only for now. - if sysgo.ResolveMixedL2CLKind() == sysgo.MixedL2CLKona { + if sysgo.ResolveMixedL2CLKind(t) == sysgo.MixedL2CLKona { t.Skip("isolated-verifier force-build path is not supported by kona-node (no L1-only EL-sync bootstrap); op-node only") } return buildSDMRethSystem(t, true, true, true, "", nil) @@ -84,7 +84,7 @@ func buildSDMRethSystem( ) *sdmtest.RethSystem { // Honor DEVSTACK_L2CL_KIND so the kona acceptance suite exercises this test with // kona-node on both the sequencer and verifier (defaults to op-node when unset). - clKind := sysgo.ResolveMixedL2CLKind() + clKind := sysgo.ResolveMixedL2CLKind(t) sequencerKey := "sequencer-op-reth" // The two sequencer paths are disjoint, and neither is an extension point for a diff --git a/op-acceptance-tests/tests/sync/follow_l2/paused_sequencer_test.go b/op-acceptance-tests/tests/sync/follow_l2/paused_sequencer_test.go new file mode 100644 index 00000000000..3c2eb1e7547 --- /dev/null +++ b/op-acceptance-tests/tests/sync/follow_l2/paused_sequencer_test.go @@ -0,0 +1,65 @@ +package follow_l2 + +import ( + "testing" + "time" + + "github.com/ethereum-optimism/optimism/op-devstack/devtest" + "github.com/ethereum-optimism/optimism/op-devstack/dsl" + "github.com/ethereum-optimism/optimism/op-service/eth" + + safety "github.com/ethereum-optimism/optimism/op-service/eth/safety" +) + +// TestFollowL2_LabelsMatchELByHashWithSequencerPaused stops the sequencer, waits until the +// follow-source verifier is safe and finalized at the paused head, and then requires its +// consensus-layer labels to name the same blocks, by hash, as its execution layer's labels and +// the sequencer's paused head. With the sequencer paused the labels stop moving, so the two +// layers are compared at rest rather than racing new blocks. +func TestFollowL2_LabelsMatchELByHashWithSequencerPaused(gt *testing.T) { + t := devtest.ParallelT(gt) + sys := newSingleChainTwoVerifiersFollowL2(t) + require := t.Require() + // With the sequencer paused only the waited label itself can move, and it moves in jumps: when a + // batch lands on L1 for local-safe, and when that L1 block finalizes, about two minutes later, + // for finalized. The stall bound covers one such jump; a label stuck past it fails fast. + const maxWait = 8 * time.Minute + const stallTimeout = 4 * time.Minute + + // Local-safe above genesis means kona finished engine sync and follows the source. Pausing + // earlier strands kona-node at genesis (#24030), where every label would match trivially. + require.NoError(sys.L2CLC.ReachedWithProgressFn(safety.LocalSafe, safety.LocalUnsafe, 1, maxWait, stallTimeout)()) + + pausedHead := sys.L2EL.BlockRefByHash(sys.L2CL.StopSequencer()) + t.Logger().Info("Paused the sequencer", "head", pausedHead) + for _, label := range []struct { + consensusLayer safety.Level + executionLayer eth.BlockLabel + }{ + {safety.LocalSafe, eth.Safe}, + {safety.Finalized, eth.Finalized}, + } { + dsl.CheckAll(t, + sys.L2CLC.ReachedWithProgressFn(label.consensusLayer, label.consensusLayer, pausedHead.Number, maxWait, stallTimeout), + sys.L2ELC.ReachedWithProgressFn(label.executionLayer, label.executionLayer, pausedHead.Number, maxWait, stallTimeout), + ) + require.Equal(pausedHead.ID(), sys.L2CLC.HeadBlockRef(label.consensusLayer).ID(), "%s label must reach the paused head itself", label.consensusLayer) + } + + status := sys.L2CLC.SyncStatus() + require.Equal(status.SafeL2, status.LocalSafeL2, "delegated local-safe and safe must match outside interop") + for _, label := range []struct { + consensusLayer eth.L2BlockRef + executionLayer eth.BlockLabel + }{ + {status.UnsafeL2, eth.Unsafe}, + {status.SafeL2, eth.Safe}, + {status.FinalizedL2, eth.Finalized}, + } { + require.Equal(pausedHead, label.consensusLayer, "%s label must stop at the paused head", label.executionLayer) + require.Equal(label.consensusLayer, sys.L2ELC.BlockRefByLabel(label.executionLayer), "%s label differs between the layers", label.executionLayer) + } + // Every label is the paused head, so it is on the sequencer's chain as long as the sequencer + // still has the paused head as its own head. + require.Equal(pausedHead, sys.L2EL.BlockRefByLabel(eth.Unsafe), "the sequencer's head moved while paused") +} diff --git a/op-acceptance-tests/tests/sync/follow_l2/sync_test.go b/op-acceptance-tests/tests/sync/follow_l2/sync_test.go index 5a6babfbaad..2da8d1ea3bd 100644 --- a/op-acceptance-tests/tests/sync/follow_l2/sync_test.go +++ b/op-acceptance-tests/tests/sync/follow_l2/sync_test.go @@ -8,7 +8,6 @@ import ( "github.com/ethereum-optimism/optimism/op-devstack/devtest" "github.com/ethereum-optimism/optimism/op-devstack/dsl" "github.com/ethereum-optimism/optimism/op-devstack/presets" - "github.com/ethereum-optimism/optimism/op-devstack/sysgo" "github.com/ethereum-optimism/optimism/op-service/eth" safety "github.com/ethereum-optimism/optimism/op-service/eth/safety" @@ -17,18 +16,6 @@ import ( func TestFollowL2_Safe_Finalized_CurrentL1(gt *testing.T) { t := devtest.ParallelT(gt) - // Example error with kona-node: - // - // assertions.go:387: ERROR[03-31|11:33:11.255] - // assertions.go:387: Error Trace: /optimism/op-devstack/sysgo/singlechain_variants.go:143 - // assertions.go:387: /optimism/op-devstack/sysgo/singlechain_variants.go:53 - // assertions.go:387: /optimism/op-devstack/presets/singlechain_twoverifiers.go:24 - // assertions.go:387: /optimism/op-acceptance-tests/tests/sync/follow_l2/setup_test.go:24 - // assertions.go:387: /optimism/op-acceptance-tests/tests/sync/follow_l2/sync_test.go:18 - // assertions.go:387: Error: Should be true - // assertions.go:387: Test: TestFollowL2_Safe_Finalized_CurrentL1 - // assertions.go:387: Messages: single-chain test sequencer requires an op-node CL node - sysgo.SkipOnKonaNode(t, "not supported") sys := newSingleChainTwoVerifiersFollowL2(t) logger := t.Logger() @@ -36,9 +23,8 @@ func TestFollowL2_Safe_Finalized_CurrentL1(gt *testing.T) { attempts := 70 target := uint64(3) - // L2CL is the sequencer with CL follow source, derivation disabled - // L2CLB is the verifier without follow source, derivation enabled - // L2CLC is the verifier with CL follow source, derivation disabled + // L2CL is the sequencer and L2CLB the deriving verifier, both op-node by the preset. + // L2CLC follows L2CLB with derivation disabled and runs the CL kind DEVSTACK_L2CL_KIND selects. // All verifiers must eventually advance unsafe, safe, finalized checkMatchedAll := func(lvl safety.Level) { dsl.CheckAll(t, @@ -71,18 +57,6 @@ func TestFollowL2_Safe_Finalized_CurrentL1(gt *testing.T) { func TestFollowL2_ReorgRecovery(gt *testing.T) { t := devtest.ParallelT(gt) - // Example error with kona-node: - // - // assertions.go:387: ERROR[03-31|11:31:11.567] - // assertions.go:387: Error Trace: /optimism/op-devstack/sysgo/singlechain_variants.go:143 - // assertions.go:387: /optimism/op-devstack/sysgo/singlechain_variants.go:53 - // assertions.go:387: /optimism/op-devstack/presets/singlechain_twoverifiers.go:24 - // assertions.go:387: /optimism/op-acceptance-tests/tests/sync/follow_l2/setup_test.go:24 - // assertions.go:387: /optimism/op-acceptance-tests/tests/sync/follow_l2/sync_test.go:60 - // assertions.go:387: Error: Should be true - // assertions.go:387: Test: TestFollowL2_ReorgRecovery - // assertions.go:387: Messages: single-chain test sequencer requires an op-node CL node - sysgo.SkipOnKonaNode(t, "not supported") sys := newSingleChainTwoVerifiersFollowL2(t) require := t.Require() logger := t.Logger() @@ -126,6 +100,14 @@ func TestFollowL2_ReorgRecovery(gt *testing.T) { // Make sure verifier safe head is also advanced from reorgL2Block or matched sys.L2ELB.Reached(eth.Safe, l2BlockBeforeReorg.Number, 3) + // The follow-source verifier must hold the block about to be reorged as local-safe, so the + // recovery checks below start from a real safe head rather than from genesis. + require.NoError(sys.L2CLC.ReachedWithProgressFn( + safety.LocalSafe, safety.LocalUnsafe, l2BlockBeforeReorg.Number, + 2*time.Minute, 30*time.Second, + )()) + require.Equal(l2BlockBeforeReorg.Hash, sys.L2ELC.BlockRefByNumber(l2BlockBeforeReorg.Number).Hash, + "follow-source verifier must hold the deriving verifier's block before the L1 reorg") // Reorg L1 block which safe block L1 Origin points to l1BlockBeforeReorg := sys.L1EL.BlockRefByNumber(l2BlockBeforeReorg.L1Origin.Number) @@ -159,6 +141,18 @@ func TestFollowL2_ReorgRecovery(gt *testing.T) { }, 60*time.Second, 2*time.Second) logger.Info("Triggered L2 reorg", "l2", l2BlockAfterReorg) + sys.L2ELC.ReorgTriggered(l2BlockBeforeReorg, 60) + // A local-safe head past the reorged height shows the follow-source verifier recovered; the + // in-sync checks below also pass when both verifiers sit at genesis. Waiting for the EL reorg + // first means the unsafe head that bounds this wait is already climbing on the new chain. + require.NoError(sys.L2CLC.ReachedWithProgressFn( + safety.LocalSafe, safety.LocalUnsafe, l2BlockBeforeReorg.Number+1, + 5*time.Minute, time.Minute, + )()) + require.Equal(sys.L2ELB.BlockRefByNumber(l2BlockBeforeReorg.Number).Hash, + sys.L2ELC.BlockRefByNumber(l2BlockBeforeReorg.Number).Hash, + "follow-source verifier must replace the reorged block with the deriving verifier's") + attempts := 30 dsl.CheckAll(t, sys.L2CL.InSyncFn(sys.L2CLB, safety.LocalUnsafe, attempts), @@ -170,18 +164,6 @@ func TestFollowL2_ReorgRecovery(gt *testing.T) { func TestFollowL2_WithoutCLP2P(gt *testing.T) { t := devtest.ParallelT(gt) - // Example error with kona-node: - // - // assertions.go:387: ERROR[03-31|11:27:57.797] - // assertions.go:387: Error Trace: /optimism/op-devstack/sysgo/singlechain_variants.go:143 - // assertions.go:387: /optimism/op-devstack/sysgo/singlechain_variants.go:53 - // assertions.go:387: /optimism/op-devstack/presets/singlechain_twoverifiers.go:24 - // assertions.go:387: /optimism/op-acceptance-tests/tests/sync/follow_l2/setup_test.go:24 - // assertions.go:387: /optimism/op-acceptance-tests/tests/sync/follow_l2/sync_test.go:136 - // assertions.go:387: Error: Should be true - // assertions.go:387: Test: TestFollowL2_WithoutCLP2P - // assertions.go:387: Messages: single-chain test sequencer requires an op-node CL nod - sysgo.SkipOnKonaNode(t, "not supported") // Without CL P2P, L2CLC learns of new blocks only from its follow source, polled every two // L2 blocks. When the source's local-safe head is past L2CLC's unsafe head, op-node has its // EL sync to it, gets SYNCING back and resets local-safe to the finalized head. It recovers @@ -255,6 +237,14 @@ func TestFollowL2_WithoutCLP2P(gt *testing.T) { sys.L2CLC.InSyncFn(sys.L2CLB, safety.LocalUnsafe, attempts), ) + // The deriving verifier finalizes past genesis first, so the comparison cannot pass with both + // nodes at genesis. + require.NoError(sys.L2CLB.ReachedWithProgressFn( + safety.Finalized, safety.LocalSafe, 1, + 5*time.Minute, time.Minute, + )()) + sys.L2CLC.InSync(sys.L2CLB, safety.Finalized, attempts) + t.Cleanup(func() { sys.L2CLC.ConnectPeer(sys.L2CLB) sys.L2CLB.ConnectPeer(sys.L2CLC) diff --git a/op-devstack/README.md b/op-devstack/README.md index d2246ed928d..5dfab54b194 100644 --- a/op-devstack/README.md +++ b/op-devstack/README.md @@ -110,7 +110,7 @@ and returns a typed output that the test then may use. - `DEVSTACK_MONOREPO_ROOT`: Absolute path to a monorepo checkout. Used by out-of-tree acceptance suites (which depend on this module by git rev) so contract-artifact resolution can find `packages/contracts-bedrock` outside the cwd-relative walk. In-tree runs don't need it. ### Rust stack env vars: -- `DEVSTACK_L2CL_KIND=kona-node` to select kona-node as default L2 CL node +- `DEVSTACK_L2CL_KIND=kona-node` to select kona-node as default L2 CL node. Any value other than `op-node` or `kona-node` fails the test before a node starts. Some presets pin nodes to op-node whatever the variable says, such as the sequencer of a chain with interop at genesis, and the sequencer and deriving verifier of the two-verifiers preset. - `RUST_BINARY_PATH_KONA_NODE=/home/USERHERE/projects/kona/target/debug/kona-node` to select the kona-node executable to run (skips the build; the path must exist) - `RUST_BINARY_PATH_OP_RETH=/home/USERHERE/projects/reth/target/release/op-reth` to select the op-reth executable to run (skips the build; the path must exist) diff --git a/op-devstack/presets/option_validation.go b/op-devstack/presets/option_validation.go index 0ba613ce813..4a0357b38c3 100644 --- a/op-devstack/presets/option_validation.go +++ b/op-devstack/presets/option_validation.go @@ -164,6 +164,15 @@ const minimalPresetSupportedOptionKinds = optionKindDeployer | optionKindProofValidation | optionKindSkipHonestProposer +// The two-verifiers preset starts no proposer or challenger, so it takes no fault-proof options. +const singleChainTwoVerifiersPresetSupportedOptionKinds = optionKindDeployer | + optionKindBatcher | + optionKindOpReth | + optionKindGlobalL2CL | + optionKindL1EL | + optionKindTimeTravel | + optionKindAfterBuild + // minimalNoFaultProofsPresetSupportedOptionKinds adds alt-DA, which only this preset supports (see WithAltDA). const minimalNoFaultProofsPresetSupportedOptionKinds = minimalPresetSupportedOptionKinds | optionKindAltDA diff --git a/op-devstack/presets/singlechain_twoverifiers.go b/op-devstack/presets/singlechain_twoverifiers.go index 4de57d3c2b9..0047044a7f8 100644 --- a/op-devstack/presets/singlechain_twoverifiers.go +++ b/op-devstack/presets/singlechain_twoverifiers.go @@ -20,7 +20,7 @@ type SingleChainTwoVerifiers struct { // // The target is created from the runtime plus any additional preset options. func NewSingleChainTwoVerifiersWithoutCheck(t devtest.T, opts ...Option) *SingleChainTwoVerifiers { - presetCfg, presetOpts := collectSupportedPresetConfig(t, "NewSingleChainTwoVerifiersWithoutCheck", opts, minimalPresetSupportedOptionKinds) + presetCfg, presetOpts := collectSupportedPresetConfig(t, "NewSingleChainTwoVerifiersWithoutCheck", opts, singleChainTwoVerifiersPresetSupportedOptionKinds) out := singleChainTwoVerifiersFromRuntime(t, sysgo.NewSingleChainTwoVerifiersRuntimeWithConfig(t, presetCfg)) presetOpts.applyPreset(out) return out diff --git a/op-devstack/sysgo/mixed_runtime.go b/op-devstack/sysgo/mixed_runtime.go index 8a792ecfb40..08bc3aba0d6 100644 --- a/op-devstack/sysgo/mixed_runtime.go +++ b/op-devstack/sysgo/mixed_runtime.go @@ -70,14 +70,20 @@ func devstackL2CLKind() MixedL2CLKind { } // ResolveMixedL2CLKind returns the L2 CL kind requested via DEVSTACK_L2CL_KIND, -// defaulting to op-node when the variable is unset. Tests that build explicit -// MixedSingleChainNodeSpec lists use this so they honor the same env-driven CL -// selection as the higher-level single-chain presets (see startL2CLForKey). -func ResolveMixedL2CLKind() MixedL2CLKind { - if kind := devstackL2CLKind(); kind != "" { +// defaulting to op-node when the variable is unset, and fails the test on any other +// value before a node starts. Tests that build explicit MixedSingleChainNodeSpec lists +// use this so they honor the same env-driven CL selection as the higher-level +// single-chain presets. +func ResolveMixedL2CLKind(t devtest.T) MixedL2CLKind { + switch kind := devstackL2CLKind(); kind { + case "": + return MixedL2CLOpNode + case MixedL2CLOpNode, MixedL2CLKona: return kind + default: + t.Require().FailNowf("unsupported L2 CL kind", "DEVSTACK_L2CL_KIND=%q, want %q or %q", kind, MixedL2CLOpNode, MixedL2CLKona) + return "" // unreachable } - return MixedL2CLOpNode } const ( diff --git a/op-devstack/sysgo/mixed_runtime_test.go b/op-devstack/sysgo/mixed_runtime_test.go index ae5124255dd..5c97f9ca742 100644 --- a/op-devstack/sysgo/mixed_runtime_test.go +++ b/op-devstack/sysgo/mixed_runtime_test.go @@ -66,3 +66,31 @@ func TestResolveMixedL2ELOptsReadsEnv(t *testing.T) { require.Equal(t, []string{"--extra.bind=127.0.0.1:0"}, cfg.ExtraArgs) require.True(t, cfg.DisableProofsHistory) } + +func TestResolveMixedL2CLKind(t *testing.T) { + for _, tc := range []struct { + env string + want MixedL2CLKind + }{ + {env: "", want: MixedL2CLOpNode}, + {env: "op-node", want: MixedL2CLOpNode}, + {env: "kona-node", want: MixedL2CLKona}, + } { + t.Run(tc.env, func(t *testing.T) { + t.Setenv("DEVSTACK_L2CL_KIND", tc.env) + require.Equal(t, tc.want, ResolveMixedL2CLKind(devtest.SerialT(t))) + }) + } +} + +// An unknown kind fails at resolution, before any preset starts a node of that kind. +func TestResolveMixedL2CLKindRejectsUnknownKind(t *testing.T) { + for _, kind := range []string{"kona", "Kona-Node", "op-geth"} { + t.Run(kind, func(t *testing.T) { + t.Setenv("DEVSTACK_L2CL_KIND", kind) + dt := &rejectedKonaOptionTest{T: devtest.SerialT(t)} + require.PanicsWithValue(t, "Kona option rejected", func() { ResolveMixedL2CLKind(dt) }) + require.Contains(t, dt.failure, "DEVSTACK_L2CL_KIND") + }) + } +} diff --git a/op-devstack/sysgo/singlechain_build.go b/op-devstack/sysgo/singlechain_build.go index 616d1fed364..68ba9535293 100644 --- a/op-devstack/sysgo/singlechain_build.go +++ b/op-devstack/sysgo/singlechain_build.go @@ -91,11 +91,7 @@ func applyConfigPrefundedL2(t devtest.T, keys devkeys.Keys, l1ChainID, l2ChainID l1Config.WithPrefundedAccount(addrFor(devkeys.SystemConfigOwner), *millionEth) } -// startL2CLForKey starts an L2 CL node for the given key, respecting DEVSTACK_L2CL_KIND. -// This is the env-aware dispatch point for preset runtimes that don't build explicit node -// specs. Runtimes constructed from explicit MixedSingleChainNodeSpec.CLKind values (e.g. -// NewMixedSingleChainRuntime) don't route through here; they resolve the env up front via -// ResolveMixedL2CLKind instead. +// startL2CLForKey starts an L2 CL node of the given kind for the given key. func startL2CLForKey( t devtest.T, keys devkeys.Keys, @@ -105,17 +101,18 @@ func startL2CLForKey( l1CL *L1CLNode, l2EL L2ELNode, jwtSecret [32]byte, + clKind MixedL2CLKind, clKey, elKey string, isSequencer bool, followSource string, l2CLOpts []L2CLOption, ) L2CLNode { - switch devstackL2CLKind() { + switch clKind { case MixedL2CLKona: return startMixedKonaNode(t, keys, l1Net, l2Net, l1EL, l1CL, l2EL, clKey, elKey, konaNodeStartConfig{ L2CLConfig: resolveKonaL2CLConfig(t, clKey, l2Net.ChainID(), isSequencer, followSource, l2CLOpts), }) - default: // op-node + case MixedL2CLOpNode: return startL2CLNode(t, keys, l1Net, l2Net, l1EL, l1CL, l2EL, jwtSecret, l2CLNodeStartConfig{ Key: clKey, IsSequencer: isSequencer, @@ -123,10 +120,13 @@ func startL2CLForKey( L2FollowSource: followSource, L2CLOptions: l2CLOpts, }) + default: + t.Require().FailNowf("unsupported L2 CL kind", "unknown L2 CL kind %q", clKind) + return nil // unreachable } } -// startStoppedSequencerCL starts the env-selected sequencer CL in the stopped +// startStoppedSequencerCL starts a sequencer CL of the given kind in the stopped // state required while a conductor cluster is being assembled. func startStoppedSequencerCL( t devtest.T, @@ -137,10 +137,11 @@ func startStoppedSequencerCL( l1CL *L1CLNode, l2EL L2ELNode, jwtSecret [32]byte, + clKind MixedL2CLKind, key string, l2CLOpts []L2CLOption, ) L2CLNode { - switch devstackL2CLKind() { + switch clKind { case MixedL2CLKona: return startMixedKonaNode( t, keys, l1Net, l2Net, l1EL, l1CL, l2EL, key, key, konaNodeStartConfig{ @@ -148,7 +149,7 @@ func startStoppedSequencerCL( SequencerStopped: true, }, ) - default: // op-node + case MixedL2CLOpNode: return startL2CLNode(t, keys, l1Net, l2Net, l1EL, l1CL, l2EL, jwtSecret, l2CLNodeStartConfig{ Key: key, IsSequencer: true, @@ -156,6 +157,9 @@ func startStoppedSequencerCL( L2CLOptions: l2CLOpts, SequencerStopped: true, }) + default: + t.Require().FailNowf("unsupported L2 CL kind", "unknown L2 CL kind %q", clKind) + return nil // unreachable } } @@ -201,20 +205,6 @@ func connectL2CLPeers(t devtest.T, logger log.Logger, l2CL1, l2CL2 L2CLNode) { require.True(ok2, "peer register invalid (cl2 missing cl1)") } -func startSequencerCL( - t devtest.T, - keys devkeys.Keys, - l1Net *L1Network, - l2Net *L2Network, - l1EL L1ELNode, - l1CL *L1CLNode, - l2EL L2ELNode, - jwtSecret [32]byte, - l2CLOpts []L2CLOption, -) L2CLNode { - return startL2CLForKey(t, keys, l1Net, l2Net, l1EL, l1CL, l2EL, jwtSecret, "sequencer", "sequencer", true, "", l2CLOpts) -} - type l2CLNodeStartConfig struct { Key string IsSequencer bool diff --git a/op-devstack/sysgo/singlechain_build_test.go b/op-devstack/sysgo/singlechain_build_test.go index 07dada44d10..e69392ec74d 100644 --- a/op-devstack/sysgo/singlechain_build_test.go +++ b/op-devstack/sysgo/singlechain_build_test.go @@ -12,7 +12,8 @@ import ( ) func TestKonaOptionsRejectedBeforeStartup(t *testing.T) { - t.Setenv("DEVSTACK_L2CL_KIND", string(MixedL2CLKona)) + // The env asks for op-node, so a kona-node rejection shows the explicit kind picks the CL. + t.Setenv("DEVSTACK_L2CL_KIND", string(MixedL2CLOpNode)) t.Setenv("KONA_NODE_P2P_NO_DISCOVERY", "true") for _, stopped := range []bool{false, true} { t.Run(fmt.Sprintf("stopped=%t", stopped), func(t *testing.T) { @@ -24,12 +25,33 @@ func TestKonaOptionsRejectedBeforeStartup(t *testing.T) { // Nil startup dependencies ensure validation runs before starting any process or RPC. require.PanicsWithValue(t, "Kona option rejected", func() { if stopped { - startStoppedSequencerCL(dt, nil, nil, network, nil, nil, nil, [32]byte{}, "node", opts) + startStoppedSequencerCL(dt, nil, nil, network, nil, nil, nil, [32]byte{}, MixedL2CLKona, "node", opts) } else { - startL2CLForKey(dt, nil, nil, network, nil, nil, nil, [32]byte{}, "node", "node", true, "", opts) + startL2CLForKey(dt, nil, nil, network, nil, nil, nil, [32]byte{}, MixedL2CLKona, "node", "node", true, "", opts) } }) require.Contains(t, dt.failure, "unsupported L2CL option for kona-node") }) } } + +// A kind the starters do not know, including the empty string, fails before anything starts +// instead of silently running op-node. +func TestUnknownL2CLKindRejectedBeforeStartup(t *testing.T) { + for _, kind := range []MixedL2CLKind{"", "kona", "Kona-Node"} { + for _, stopped := range []bool{false, true} { + t.Run(fmt.Sprintf("kind=%q/stopped=%t", kind, stopped), func(t *testing.T) { + dt := &rejectedKonaOptionTest{T: devtest.SerialT(t)} + network := &L2Network{chainID: eth.ChainIDFromUInt64(901)} + require.PanicsWithValue(t, "Kona option rejected", func() { + if stopped { + startStoppedSequencerCL(dt, nil, nil, network, nil, nil, nil, [32]byte{}, kind, "node", nil) + } else { + startL2CLForKey(dt, nil, nil, network, nil, nil, nil, [32]byte{}, kind, "node", "node", true, "", nil) + } + }) + require.Contains(t, dt.failure, "unknown L2 CL kind") + }) + } + } +} diff --git a/op-devstack/sysgo/singlechain_interop.go b/op-devstack/sysgo/singlechain_interop.go index c3bad3d2f2b..80ba236837d 100644 --- a/op-devstack/sysgo/singlechain_interop.go +++ b/op-devstack/sysgo/singlechain_interop.go @@ -31,8 +31,10 @@ func startSingleChainInteropPrimaryNoSupernode( jwtPath string, jwtSecret [32]byte, cfg PresetConfig, + sequencerCLKind MixedL2CLKind, ) singleChainPrimaryRuntime { t.Require().NotNil(world.Interop, "single-chain interop runtime requires interop support") + requireOpNodeSequencerCLKind(t, sequencerCLKind) l2EL := startSequencerEL(t, world.L2Network, jwtPath, jwtSecret, ResolveMixedL2ELOpts(t)...) l2CL := startL2CLNode(t, keys, world.L1Network, world.L2Network, l1EL, l1CL, l2EL, jwtSecret, l2CLNodeStartConfig{ @@ -54,10 +56,12 @@ func startSingleChainInteropPrimaryNoSupernode( func NewMinimalInteropNoSupernodeRuntime(t devtest.T) *SingleChainRuntime { return newSingleChainRuntimeWithConfig(t, PresetConfig{}, singleChainRuntimeSpec{ BuildWorld: newSingleChainInteropWorldNoSupernode, + SequencerCLKind: MixedL2CLOpNode, StartPrimary: startSingleChainInteropPrimaryNoSupernode, + StartBatcher: true, - StartProposer: true, StartChallenger: false, + StartProposer: true, }) } @@ -70,10 +74,12 @@ func NewSingleChainInteropNoSupernodeSuperRootRuntimeWithConfig(t devtest.T, cfg cfg.AddedGameTypes = append(cfg.AddedGameTypes, gameTypes.SuperCannonKonaGameType) return newSingleChainRuntimeWithConfig(t, cfg, singleChainRuntimeSpec{ BuildWorld: newSingleChainInteropWorldNoSupernode, + SequencerCLKind: MixedL2CLOpNode, StartPrimary: startDefaultSingleChainPrimary, + StartBatcher: true, - StartProposer: false, StartChallenger: true, + StartProposer: false, }) } @@ -88,9 +94,11 @@ func NewSingleChainInteropNoSupernodeZKDisputeRuntimeWithConfig(t devtest.T, cfg cfg.AddedGameTypes = append(cfg.AddedGameTypes, gameTypes.ZKDisputeGameType) return newSingleChainRuntimeWithConfig(t, cfg, singleChainRuntimeSpec{ BuildWorld: newSingleChainInteropWorldNoSupernode, + SequencerCLKind: MixedL2CLOpNode, StartPrimary: startDefaultSingleChainPrimary, + StartBatcher: true, - StartProposer: false, StartChallenger: true, + StartProposer: false, }) } diff --git a/op-devstack/sysgo/singlechain_runtime.go b/op-devstack/sysgo/singlechain_runtime.go index de6501560fa..f3c827358f2 100644 --- a/op-devstack/sysgo/singlechain_runtime.go +++ b/op-devstack/sysgo/singlechain_runtime.go @@ -45,13 +45,43 @@ type singleChainPrimaryRuntime struct { CL L2CLNode } +type singleChainPrimaryStartFunc func( + t devtest.T, + keys devkeys.Keys, + world singleChainRuntimeWorld, + l1EL *L1Geth, + l1CL *L1CLNode, + jwtPath string, + jwtSecret [32]byte, + cfg PresetConfig, + sequencerCLKind MixedL2CLKind, +) singleChainPrimaryRuntime + type singleChainRuntimeSpec struct { BuildWorld func(t devtest.T, keys devkeys.Keys, cfg PresetConfig, startL1 func(*L1Network) (*L1Geth, *L1CLNode)) singleChainRuntimeWorld - StartPrimary func(t devtest.T, keys devkeys.Keys, world singleChainRuntimeWorld, l1EL *L1Geth, l1CL *L1CLNode, jwtPath string, jwtSecret [32]byte, cfg PresetConfig) singleChainPrimaryRuntime + SequencerCLKind MixedL2CLKind + StartPrimary singleChainPrimaryStartFunc + StartBatcher bool - StartProposer bool StartChallenger bool - TestSequencer string + StartProposer bool + + TestSequencer string +} + +// requireOpNodeSequencerCLKind fails unless the sequencer runs on op-node, which the single-chain +// interop sequencer requires. +func requireOpNodeSequencerCLKind(t devtest.T, sequencerCLKind MixedL2CLKind) { + t.Require().Equal(MixedL2CLOpNode, sequencerCLKind, "single-chain interop sequencer requires op-node") +} + +// resolveSequencerCLKind returns the L2 CL kind the environment requests for the sequencer, +// pinned to op-node when the chain runs interop at genesis. +func resolveSequencerCLKind(t devtest.T, cfg PresetConfig) MixedL2CLKind { + if cfg.InteropAtGenesis { + return MixedL2CLOpNode + } + return ResolveMixedL2CLKind(t) } func newSingleChainNodeRuntime(name string, isSequencer bool, el L2ELNode, cl L2CLNode) *SingleChainNodeRuntime { @@ -94,6 +124,7 @@ func startDefaultSingleChainPrimary( jwtPath string, jwtSecret [32]byte, cfg PresetConfig, + sequencerCLKind MixedL2CLKind, ) singleChainPrimaryRuntime { safeDBPath := filepath.Join(t.TempDirWithPrefix("l2-safe-db-"+world.L2Network.ChainID().String()), "safe-head.db") l2CLOptions := []L2CLOption{L2CLOptionFn(func(_ devtest.T, _ ComponentTarget, cfg *L2CLConfig) { @@ -104,6 +135,7 @@ func startDefaultSingleChainPrimary( sequencerELOpts := append(append([]OpRethOption{}, ResolveMixedL2ELOpts(t)...), cfg.OpRethOptions...) l2EL := startSequencerEL(t, world.L2Network, jwtPath, jwtSecret, sequencerELOpts...) if world.Interop != nil { + requireOpNodeSequencerCLKind(t, sequencerCLKind) l2CL := startL2CLNode(t, keys, world.L1Network, world.L2Network, l1EL, l1CL, l2EL, jwtSecret, l2CLNodeStartConfig{ Key: "sequencer", IsSequencer: true, @@ -113,7 +145,7 @@ func startDefaultSingleChainPrimary( }) return singleChainPrimaryRuntime{EL: l2EL, CL: l2CL} } - l2CL := startSequencerCL(t, keys, world.L1Network, world.L2Network, l1EL, l1CL, l2EL, jwtSecret, l2CLOptions) + l2CL := startL2CLForKey(t, keys, world.L1Network, world.L2Network, l1EL, l1CL, l2EL, jwtSecret, sequencerCLKind, "sequencer", "sequencer", true, "", l2CLOptions) return singleChainPrimaryRuntime{ EL: l2EL, CL: l2CL, @@ -132,6 +164,7 @@ func startStoppedSingleChainPrimary( jwtPath string, jwtSecret [32]byte, cfg PresetConfig, + sequencerCLKind MixedL2CLKind, ) singleChainPrimaryRuntime { t.Require().Nil(world.Interop, "conductor primary does not support interop") safeDBPath := filepath.Join(t.TempDirWithPrefix("l2-safe-db-"+world.L2Network.ChainID().String()), "safe-head.db") @@ -144,7 +177,7 @@ func startStoppedSingleChainPrimary( l2EL := startSequencerEL(t, world.L2Network, jwtPath, jwtSecret, sequencerELOpts...) l2CL := startStoppedSequencerCL( t, keys, world.L1Network, world.L2Network, l1EL, l1CL, l2EL, jwtSecret, - "sequencer", l2CLOptions, + sequencerCLKind, "sequencer", l2CLOptions, ) return singleChainPrimaryRuntime{EL: l2EL, CL: l2CL} } @@ -174,7 +207,7 @@ func newSingleChainRuntimeWithConfig(t devtest.T, cfg PresetConfig, spec singleC return l1EL, l1CL }) - primary := spec.StartPrimary(t, keys, world, l1EL, l1CL, jwtPath, jwtSecret, cfg) + primary := spec.StartPrimary(t, keys, world, l1EL, l1CL, jwtPath, jwtSecret, cfg, spec.SequencerCLKind) primaryNode := newSingleChainNodeRuntime("sequencer", true, primary.EL, primary.CL) var l2Batcher *L2Batcher @@ -239,12 +272,18 @@ func NewMinimalRuntime(t devtest.T) *SingleChainRuntime { } func NewMinimalRuntimeWithConfig(t devtest.T, cfg PresetConfig) *SingleChainRuntime { + return newMinimalRuntime(t, cfg, resolveSequencerCLKind(t, cfg)) +} + +func newMinimalRuntime(t devtest.T, cfg PresetConfig, sequencerCLKind MixedL2CLKind) *SingleChainRuntime { return newSingleChainRuntimeWithConfig(t, cfg, singleChainRuntimeSpec{ BuildWorld: newDefaultSingleChainWorld, + SequencerCLKind: sequencerCLKind, StartPrimary: startDefaultSingleChainPrimary, + StartBatcher: true, - StartProposer: true, StartChallenger: true, + StartProposer: true, }) } @@ -254,12 +293,18 @@ func NewMinimalRuntimeWithConfig(t devtest.T, cfg PresetConfig) *SingleChainRunt // fault proofs. Skipping the challenger also avoids requiring cannon prestate // artifacts, which are expensive to build locally. func NewMinimalNoFaultProofsRuntimeWithConfig(t devtest.T, cfg PresetConfig) *SingleChainRuntime { + return newMinimalNoFaultProofsRuntime(t, cfg, resolveSequencerCLKind(t, cfg)) +} + +func newMinimalNoFaultProofsRuntime(t devtest.T, cfg PresetConfig, sequencerCLKind MixedL2CLKind) *SingleChainRuntime { return newSingleChainRuntimeWithConfig(t, cfg, singleChainRuntimeSpec{ BuildWorld: newDefaultSingleChainWorld, + SequencerCLKind: sequencerCLKind, StartPrimary: startDefaultSingleChainPrimary, + StartBatcher: true, - StartProposer: false, StartChallenger: false, + StartProposer: false, }) } diff --git a/op-devstack/sysgo/singlechain_variants.go b/op-devstack/sysgo/singlechain_variants.go index e6787b9c2f4..b63b01a1e05 100644 --- a/op-devstack/sysgo/singlechain_variants.go +++ b/op-devstack/sysgo/singlechain_variants.go @@ -32,15 +32,17 @@ func NewSingleChainMultiNodeRuntime(t devtest.T, withP2P bool) *SingleChainRunti } func NewSingleChainMultiNodeRuntimeWithConfig(t devtest.T, withP2P bool, cfg PresetConfig) *SingleChainRuntime { - return addSingleChainVerifier(t, NewMinimalRuntimeWithConfig(t, cfg), withP2P, cfg) + clKind := ResolveMixedL2CLKind(t) + return addSingleChainVerifier(t, NewMinimalRuntimeWithConfig(t, cfg), clKind, withP2P, cfg) } func NewSingleChainWithIsolatedVerifierRuntimeWithConfig(t devtest.T, cfg PresetConfig) *SingleChainRuntime { - return addSingleChainVerifier(t, NewMinimalNoFaultProofsRuntimeWithConfig(t, cfg), false, cfg) + clKind := ResolveMixedL2CLKind(t) + return addSingleChainVerifier(t, NewMinimalNoFaultProofsRuntimeWithConfig(t, cfg), clKind, false, cfg) } -func addSingleChainVerifier(t devtest.T, runtime *SingleChainRuntime, withP2P bool, cfg PresetConfig) *SingleChainRuntime { - nodeB := addSingleChainOpNode(t, runtime, "b", false, "", cfg.GlobalL2CLOptions...) +func addSingleChainVerifier(t devtest.T, runtime *SingleChainRuntime, clKind MixedL2CLKind, withP2P bool, cfg PresetConfig) *SingleChainRuntime { + nodeB := addSingleChainNode(t, runtime, "b", clKind, false, "", cfg.GlobalL2CLOptions...) if withP2P { connectSingleChainNodes(t, runtime.L2EL, runtime.L2CL, nodeB) } @@ -52,11 +54,17 @@ func NewSingleChainTwoVerifiersRuntime(t devtest.T) *SingleChainRuntime { return NewSingleChainTwoVerifiersRuntimeWithConfig(t, PresetConfig{}) } +// NewSingleChainTwoVerifiersRuntimeWithConfig adds two verifiers to a minimal runtime without +// fault proofs: "b" derives from L1, and "c" follows "b" as its follow source. func NewSingleChainTwoVerifiersRuntimeWithConfig(t devtest.T, cfg PresetConfig) *SingleChainRuntime { - runtime := NewSingleChainMultiNodeRuntimeWithConfig(t, true, cfg) + // Only "c" takes the requested CL kind; the sequencer and "b" stay on op-node, so a kona-node + // run exercises kona's follow-source mode against an op-node upstream. + followKind := ResolveMixedL2CLKind(t) + t.Logger().Info("Starting two-verifier runtime", "b", MixedL2CLOpNode, "c", followKind, "sequencer", MixedL2CLOpNode) + runtime := addSingleChainVerifier(t, newMinimalNoFaultProofsRuntime(t, cfg, MixedL2CLOpNode), MixedL2CLOpNode, true, cfg) nodeB := runtime.Nodes["b"] t.Require().NotNil(nodeB, "missing single-chain node b") - nodeC := addSingleChainOpNode(t, runtime, "c", false, nodeB.CL.UserRPC(), cfg.GlobalL2CLOptions...) + nodeC := addSingleChainNode(t, runtime, "c", followKind, false, nodeB.CL.UserRPC(), cfg.GlobalL2CLOptions...) connectSingleChainNodes(t, runtime.L2EL, runtime.L2CL, nodeC) connectSingleChainNodes(t, nodeB.EL, nodeB.CL, nodeC) @@ -116,6 +124,7 @@ func NewMinimalWithConductorsRuntime(t devtest.T) *SingleChainRuntime { } func NewMinimalWithConductorsRuntimeWithConfig(t devtest.T, cfg PresetConfig) *SingleChainRuntime { + clKind := ResolveMixedL2CLKind(t) // The CL and conductor depend on each other's RPC endpoints. Allocate a // stable proxy for each conductor before starting its CL, then attach the // conductor once its dynamically allocated RPC port is known. @@ -140,22 +149,24 @@ func NewMinimalWithConductorsRuntimeWithConfig(t devtest.T, cfg PresetConfig) *S // challenger, and rust e2e jobs do not build cannon artifacts. runtime := newSingleChainRuntimeWithConfig(t, cfg, singleChainRuntimeSpec{ BuildWorld: newDefaultSingleChainWorld, + SequencerCLKind: clKind, StartPrimary: startStoppedSingleChainPrimary, + StartBatcher: true, - StartProposer: true, StartChallenger: false, + StartProposer: true, }) // Every conductor member sequences once leadership reaches it, so a binary override has to // apply to all of them or it would silently stop taking effect on transfer. The primary already // received these options from startStoppedSingleChainPrimary. candidateELOpts := append(append([]OpRethOption{}, ResolveMixedL2ELOpts(t)...), cfg.OpRethOptions...) - nodeB := addStoppedSingleChainSequencerNode(t, runtime, "b", candidateELOpts, cfg.GlobalL2CLOptions...) - nodeC := addStoppedSingleChainSequencerNode(t, runtime, "c", candidateELOpts, cfg.GlobalL2CLOptions...) + nodeB := addStoppedSingleChainSequencerNode(t, runtime, "b", clKind, candidateELOpts, cfg.GlobalL2CLOptions...) + nodeC := addStoppedSingleChainSequencerNode(t, runtime, "c", clKind, candidateELOpts, cfg.GlobalL2CLOptions...) - // A non-candidate node, always stock: it never sequences, so it independently validates every + // A non-candidate node with stock op-reth: it never sequences, so it independently validates every // block the candidates build. Without it an overridden binary on all three members could accept // its own invalid blocks and the run would still pass. - nodeVerifier := addSingleChainOpNode(t, runtime, "verifier", false, "", cfg.GlobalL2CLOptions...) + nodeVerifier := addSingleChainNode(t, runtime, "verifier", clKind, false, "", cfg.GlobalL2CLOptions...) healthCheck := conductorHealthCheckConfig(cfg.ConductorFastHealthChecks) conductorA := startConductorNode(t, "sequencer", runtime.L2Network, runtime.L2CL, conductorRPCProxies["sequencer"], runtime.L2EL, true, false, healthCheck) @@ -233,24 +244,26 @@ func replaceSingleChainTestSequencer(t devtest.T, runtime *SingleChainRuntime, n runtime.TestSequencer = newTestSequencerRuntime(testSequencer, name) } -func addSingleChainOpNode( +func addSingleChainNode( t devtest.T, runtime *SingleChainRuntime, name string, + clKind MixedL2CLKind, isSequencer bool, followSource string, l2Opts ...L2CLOption, ) *SingleChainNodeRuntime { - return addSingleChainOpNodeWithELOpts(t, runtime, name, isSequencer, followSource, nil, l2Opts...) + return addSingleChainNodeWithELOpts(t, runtime, name, clKind, isSequencer, followSource, nil, l2Opts...) } -// addSingleChainOpNodeWithELOpts adds a node whose EL takes op-reth options. Passing none yields a +// addSingleChainNodeWithELOpts adds a node whose EL takes op-reth options. Passing none yields a // stock op-reth, which is what lets a preset keep an independent check on blocks built by an // overridden binary. -func addSingleChainOpNodeWithELOpts( +func addSingleChainNodeWithELOpts( t devtest.T, runtime *SingleChainRuntime, name string, + clKind MixedL2CLKind, isSequencer bool, followSource string, elOpts []OpRethOption, @@ -259,7 +272,7 @@ func addSingleChainOpNodeWithELOpts( jwtPath := runtime.L2EL.JWTPath() jwtSecret := readJWTSecretFromPath(t, jwtPath) l2EL := startMixedOpRethNode(t, runtime.L2Network, name, jwtPath, jwtSecret, nil, elOpts...) - l2CL := startL2CLForKey(t, runtime.Keys, runtime.L1Network, runtime.L2Network, runtime.L1EL, runtime.L1CL, l2EL, jwtSecret, name, name, isSequencer, followSource, l2Opts) + l2CL := startL2CLForKey(t, runtime.Keys, runtime.L1Network, runtime.L2Network, runtime.L1EL, runtime.L1CL, l2EL, jwtSecret, clKind, name, name, isSequencer, followSource, l2Opts) node := newSingleChainNodeRuntime(name, isSequencer, l2EL, l2CL) runtime.Nodes[name] = node return node @@ -271,6 +284,7 @@ func addStoppedSingleChainSequencerNode( t devtest.T, runtime *SingleChainRuntime, name string, + clKind MixedL2CLKind, elOpts []OpRethOption, l2Opts ...L2CLOption, ) *SingleChainNodeRuntime { @@ -279,7 +293,7 @@ func addStoppedSingleChainSequencerNode( l2EL := startMixedOpRethNode(t, runtime.L2Network, name, jwtPath, jwtSecret, nil, elOpts...) l2CL := startStoppedSequencerCL( t, runtime.Keys, runtime.L1Network, runtime.L2Network, runtime.L1EL, runtime.L1CL, - l2EL, jwtSecret, name, l2Opts, + l2EL, jwtSecret, clKind, name, l2Opts, ) node := newSingleChainNodeRuntime(name, true, l2EL, l2CL) runtime.Nodes[name] = node diff --git a/rust/kona/bin/node/src/flags/engine/providers.rs b/rust/kona/bin/node/src/flags/engine/providers.rs index 36bbc728ea0..f3fe5d762ae 100644 --- a/rust/kona/bin/node/src/flags/engine/providers.rs +++ b/rust/kona/bin/node/src/flags/engine/providers.rs @@ -103,6 +103,9 @@ impl Default for L2ClientArgs { #[derive(Clone, Debug, Default, clap::Args)] pub struct DerivationDelegateArgs { /// The source must be an OP Stack L2 CL RPC exposing `optimism_syncStatus`. + /// + /// Its local-safe and cross-safe heads become this node's local-safe and safe labels. Under + /// interop this node verifies no cross-chain dependencies itself, so the source must. #[arg(long, visible_alias = "l2.follow.source", env = "KONA_NODE_L2_FOLLOW_SOURCE")] pub l2_follow_source: Option, } diff --git a/rust/kona/crates/node/engine/src/lib.rs b/rust/kona/crates/node/engine/src/lib.rs index c1f955096bb..1f64ad82ac1 100644 --- a/rust/kona/crates/node/engine/src/lib.rs +++ b/rust/kona/crates/node/engine/src/lib.rs @@ -40,10 +40,11 @@ extern crate tracing; mod task_queue; pub use task_queue::{ BuildTask, BuildTaskError, CanonicalizeTask, CanonicalizeTaskError, ConsolidateInput, - ConsolidateTask, ConsolidateTaskError, Engine, EngineBuildError, EngineResetError, EngineTask, - EngineTaskError, EngineTaskErrorSeverity, EngineTaskErrors, ExecutableEngineTask, - FinalizeBlockId, FinalizeTask, FinalizeTaskError, InsertTask, InsertTaskError, SealTask, - SealTaskError, SynchronizeTaskError, + ConsolidateTask, ConsolidateTaskError, DelegatedSafeHeads, Engine, EngineBuildError, + EngineResetError, EngineTask, EngineTaskError, EngineTaskErrorSeverity, EngineTaskErrors, + ExecutableEngineTask, FinalizeBlockId, FinalizeTask, FinalizeTaskError, + InconsistentSafeHeadsError, InsertTask, InsertTaskError, SealTask, SealTaskError, + SynchronizeTaskError, }; mod attributes; @@ -65,7 +66,7 @@ mod versions; pub use versions::{EngineForkchoiceVersion, EngineGetPayloadVersion, EngineNewPayloadVersion}; mod state; -pub use state::{EngineState, EngineSyncState, EngineSyncStateUpdate}; +pub use state::{EngineState, EngineSyncState, EngineSyncStateUpdate, FinalityViolation}; mod kinds; pub use kinds::EngineKind; diff --git a/rust/kona/crates/node/engine/src/state/core.rs b/rust/kona/crates/node/engine/src/state/core.rs index 7a34ce3f626..67204d7d81f 100644 --- a/rust/kona/crates/node/engine/src/state/core.rs +++ b/rust/kona/crates/node/engine/src/state/core.rs @@ -1,6 +1,7 @@ //! The internal state of the engine controller. use crate::Metrics; +use alloy_eips::BlockNumHash; use alloy_rpc_types_engine::ForkchoiceState; use kona_protocol::L2BlockInfo; use serde::{Deserialize, Serialize}; @@ -54,6 +55,12 @@ impl EngineSyncState { self.finalized_head } + /// Returns the id of the finalized head once finality is established. The default finalized + /// head means it is not yet, so no block pins finality. + pub fn established_finalized_head(&self) -> Option { + (self.finalized_head != L2BlockInfo::default()).then(|| self.finalized_head.block_info.id()) + } + /// Creates a `ForkchoiceState` /// /// - `head_block` = `unsafe_head` @@ -127,8 +134,10 @@ pub struct EngineSyncStateUpdate { /// The chain state viewed by the engine controller. #[derive(Default, Debug, Copy, Clone, PartialEq, Eq)] pub struct EngineState { - /// The sync state of the engine. - pub sync_state: EngineSyncState, + /// Whether the execution layer answered `VALID` to the last forkchoice update the engine sent, + /// so its canonical chain is the engine's. Unlike `el_sync_finished`, a later `SYNCING` answer + /// clears it. + pub el_applied_last_forkchoice: bool, /// Whether or not the EL has finished syncing. pub el_sync_finished: bool, @@ -139,6 +148,9 @@ pub struct EngineState { /// because engine may forgot backupUnsafeHead or backupUnsafeHead is not part /// of the chain. pub need_fcu_call_backup_unsafe_reorg: bool, + + /// The sync state of the engine. + pub sync_state: EngineSyncState, } impl EngineState { diff --git a/rust/kona/crates/node/engine/src/state/finality.rs b/rust/kona/crates/node/engine/src/state/finality.rs new file mode 100644 index 00000000000..94363c549bf --- /dev/null +++ b/rust/kona/crates/node/engine/src/state/finality.rs @@ -0,0 +1,126 @@ +//! Checks that keep a label from moving below finality. + +use crate::EngineSyncState; +use alloy_eips::BlockNumHash; +use thiserror::Error; + +/// A block that a label cannot move to, because it would move the label below finality. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Error)] +pub enum FinalityViolation { + /// The block is below the finalized head. + #[error("Block {target} is behind the finalized head {finalized:?}")] + BehindFinalizedHead { + /// The finalized head. + finalized: BlockNumHash, + /// The number of the block. + target: u64, + }, + /// The block is a different block at the finalized head's height. + #[error("Block {target:?} conflicts with the finalized head {finalized:?}")] + ConflictsWithFinalizedHead { + /// The finalized head. + finalized: BlockNumHash, + /// The block. + target: BlockNumHash, + }, +} + +impl EngineSyncState { + /// Rejects a block number below the established finalized head. + pub fn ensure_not_behind_finality(&self, target: u64) -> Result<(), FinalityViolation> { + match self.established_finalized_head() { + Some(finalized) if target < finalized.number => { + Err(FinalityViolation::BehindFinalizedHead { finalized, target }) + } + _ => Ok(()), + } + } + + /// Rejects a different block at the established finalized head's height. Before finality is + /// established no block pins it, so any block, including the configured genesis, passes. + pub fn ensure_no_conflict_with_finality( + &self, + target: BlockNumHash, + ) -> Result<(), FinalityViolation> { + match self.established_finalized_head() { + Some(finalized) + if target.number == finalized.number && target.hash != finalized.hash => + { + Err(FinalityViolation::ConflictsWithFinalizedHead { finalized, target }) + } + _ => Ok(()), + } + } + + /// Rejects a block below the established finalized head or conflicting with it. + pub fn ensure_respects_finality(&self, target: BlockNumHash) -> Result<(), FinalityViolation> { + self.ensure_not_behind_finality(target.number)?; + self.ensure_no_conflict_with_finality(target) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::EngineSyncStateUpdate; + use alloy_primitives::B256; + use kona_protocol::{BlockInfo, L2BlockInfo}; + + const FINALIZED_NUMBER: u64 = 5; + + fn block(number: u64, hash: u8) -> L2BlockInfo { + L2BlockInfo { + block_info: BlockInfo { hash: B256::repeat_byte(hash), number, ..Default::default() }, + ..Default::default() + } + } + + fn sync_state_finalized_at(finalized_head: L2BlockInfo) -> EngineSyncState { + EngineSyncState::default().apply_update(EngineSyncStateUpdate { + finalized_head: Some(finalized_head), + ..Default::default() + }) + } + + /// The finalized head at 5 once finality is established, and the default head before. + fn finalized_head(is_finality_established: bool) -> L2BlockInfo { + if is_finality_established { block(FINALIZED_NUMBER, 1) } else { L2BlockInfo::default() } + } + + /// The violation expected for `target` checked against the finalized head at 5. + fn behind(target: L2BlockInfo) -> Option { + Some(FinalityViolation::BehindFinalizedHead { + finalized: block(FINALIZED_NUMBER, 1).block_info.id(), + target: target.block_info.number, + }) + } + + /// The violation expected for `target` conflicting with the finalized head at 5. + fn conflicting(target: L2BlockInfo) -> Option { + Some(FinalityViolation::ConflictsWithFinalizedHead { + finalized: block(FINALIZED_NUMBER, 1).block_info.id(), + target: target.block_info.id(), + }) + } + + /// A block below the established finalized head, or a different block at its height, breaks + /// finality. Before finality is established no block pins it, so a different block at the + /// default head's height, genesis, passes. + #[rstest::rstest] + #[case::above(true, block(FINALIZED_NUMBER + 1, 2), None)] + #[case::at(true, block(FINALIZED_NUMBER, 1), None)] + #[case::below(true, block(FINALIZED_NUMBER - 1, 2), behind(block(FINALIZED_NUMBER - 1, 2)))] + #[case::conflicting(true, block(FINALIZED_NUMBER, 2), conflicting(block(FINALIZED_NUMBER, 2)))] + #[case::not_established(false, block(0, 2), None)] + fn block_is_checked_against_established_finality( + #[case] is_finality_established: bool, + #[case] target: L2BlockInfo, + #[case] expected_violation: Option, + ) { + let sync_state = sync_state_finalized_at(finalized_head(is_finality_established)); + + let result = sync_state.ensure_respects_finality(target.block_info.id()); + + assert_eq!(result.err(), expected_violation); + } +} diff --git a/rust/kona/crates/node/engine/src/state/mod.rs b/rust/kona/crates/node/engine/src/state/mod.rs index b61100917e6..a3b0cfe6f51 100644 --- a/rust/kona/crates/node/engine/src/state/mod.rs +++ b/rust/kona/crates/node/engine/src/state/mod.rs @@ -2,3 +2,6 @@ mod core; pub use core::{EngineState, EngineSyncState, EngineSyncStateUpdate}; + +mod finality; +pub use finality::FinalityViolation; diff --git a/rust/kona/crates/node/engine/src/task_queue/tasks/consolidate/delegated_safe_heads.rs b/rust/kona/crates/node/engine/src/task_queue/tasks/consolidate/delegated_safe_heads.rs new file mode 100644 index 00000000000..200c72b469c --- /dev/null +++ b/rust/kona/crates/node/engine/src/task_queue/tasks/consolidate/delegated_safe_heads.rs @@ -0,0 +1,140 @@ +//! The safe heads a derivation delegate reports. + +use alloy_eips::BlockNumHash; +use kona_protocol::L2BlockInfo; +use thiserror::Error; + +/// The safe heads a derivation delegate reports, imported as separate labels. +/// +/// Before interop both are the same block. Under interop the cross-safe head can lag the +/// local-safe head while the delegate verifies cross-chain dependencies, and only the cross-safe +/// head may reach the execution layer as its safe block. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct DelegatedSafeHeads { + local_safe_head: L2BlockInfo, + safe_head: L2BlockInfo, +} + +/// Safe heads that no consistent delegate reports. +#[derive(Debug, Clone, Copy, Error, PartialEq, Eq)] +pub enum InconsistentSafeHeadsError { + /// The cross-safe head is ahead of the local-safe head. + #[error("Safe head {safe} is ahead of local safe head {local_safe}")] + SafeAheadOfLocalSafe { + /// The number of the delegate's local-safe head. + local_safe: u64, + /// The number of the delegate's cross-safe head. + safe: u64, + }, + /// The cross-safe head is a different block at the local-safe head's height. + #[error("Safe head {safe:?} conflicts with local safe head {local_safe:?}")] + SafeConflictsWithLocalSafe { + /// The delegate's local-safe head. + local_safe: BlockNumHash, + /// The delegate's cross-safe head. + safe: BlockNumHash, + }, +} + +impl DelegatedSafeHeads { + /// Pairs the delegate's local-safe head, derived from L1 with cross-chain dependencies not + /// verified yet, with its cross-safe head, which must be at or below it on the same chain: not + /// ahead of it, and not a different block at its height. + pub fn try_new( + local_safe_head: L2BlockInfo, + safe_head: L2BlockInfo, + ) -> Result { + let local_safe = local_safe_head.block_info.id(); + let safe = safe_head.block_info.id(); + if safe.number > local_safe.number { + return Err(InconsistentSafeHeadsError::SafeAheadOfLocalSafe { + local_safe: local_safe.number, + safe: safe.number, + }); + } + if safe.number == local_safe.number && safe.hash != local_safe.hash { + return Err(InconsistentSafeHeadsError::SafeConflictsWithLocalSafe { + local_safe, + safe, + }); + } + Ok(Self { local_safe_head, safe_head }) + } + + /// The delegate's local-safe head. + pub const fn local_safe_head(&self) -> L2BlockInfo { + self.local_safe_head + } + + /// The delegate's cross-safe head. + pub const fn safe_head(&self) -> L2BlockInfo { + self.safe_head + } + + /// The cross-safe head when it is above `current_safe_head`, and `None` otherwise: a + /// cross-safe head at or below the safe label leaves it where it is. + pub(crate) const fn advancing_safe_head( + &self, + current_safe_head: &L2BlockInfo, + ) -> Option { + if self.safe_head.block_info.number > current_safe_head.block_info.number { + Some(self.safe_head) + } else { + None + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::test_utils::test_block_info; + use alloy_primitives::B256; + use kona_protocol::BlockInfo; + use rstest::rstest; + + const LOCAL_SAFE_HEAD: L2BlockInfo = L2BlockInfo { + block_info: BlockInfo { + hash: B256::repeat_byte(0x10), + number: 10, + parent_hash: B256::ZERO, + timestamp: 0, + }, + l1_origin: BlockNumHash { hash: B256::ZERO, number: 0 }, + seq_num: 0, + }; + + /// A different block at the height of `block`. + fn conflicting_block(block: L2BlockInfo) -> L2BlockInfo { + L2BlockInfo { + block_info: BlockInfo { hash: !block.block_info.hash, ..block.block_info }, + ..block + } + } + + #[rstest] + #[case::cross_safe_lags(test_block_info(8), true)] + #[case::same_block(LOCAL_SAFE_HEAD, true)] + #[case::conflicting_at_same_height(conflicting_block(LOCAL_SAFE_HEAD), false)] + #[case::cross_safe_ahead(test_block_info(11), false)] + fn try_new_rejects_cross_safe_off_the_local_safe_chain( + #[case] safe_head: L2BlockInfo, + #[case] accepted: bool, + ) { + let heads = DelegatedSafeHeads::try_new(LOCAL_SAFE_HEAD, safe_head); + + assert_eq!(heads.is_ok(), accepted, "got {heads:?}"); + } + + /// Only a cross-safe head strictly above the safe label advances it. + #[rstest] + #[case::above(8, true)] + #[case::same_height(7, false)] + #[case::below(6, false)] + fn advancing_safe_head_requires_a_higher_block(#[case] safe: u64, #[case] advances: bool) { + let heads = + DelegatedSafeHeads::try_new(test_block_info(10), test_block_info(safe)).unwrap(); + + assert_eq!(heads.advancing_safe_head(&test_block_info(7)).is_some(), advances); + } +} diff --git a/rust/kona/crates/node/engine/src/task_queue/tasks/consolidate/error.rs b/rust/kona/crates/node/engine/src/task_queue/tasks/consolidate/error.rs index c5bd5dbcb0f..0c9869ab7ae 100644 --- a/rust/kona/crates/node/engine/src/task_queue/tasks/consolidate/error.rs +++ b/rust/kona/crates/node/engine/src/task_queue/tasks/consolidate/error.rs @@ -1,7 +1,7 @@ //! Contains error types for the [`crate::ConsolidateTask`]. use crate::{ - BuildTaskError, EngineTaskError, SealTaskError, SynchronizeTaskError, + BuildTaskError, EngineTaskError, FinalityViolation, SealTaskError, SynchronizeTaskError, task_queue::tasks::{BuildAndImportError, task::EngineTaskErrorSeverity}, }; use thiserror::Error; @@ -9,6 +9,12 @@ use thiserror::Error; /// An error that occurs when running the [`crate::ConsolidateTask`]. #[derive(Debug, Error)] pub enum ConsolidateTaskError { + /// A delegated safe head would move the safe labels below finality. + #[error("Delegated safe head breaks finality: {0}")] + BreaksFinality(#[from] FinalityViolation), + /// Failed to fetch the execution layer's block at the height of a delegated safe head. + #[error("Failed to fetch the block at delegated safe head height {0}")] + FailedToFetchDelegatedSafeBlock(u64), /// The unsafe L2 block is missing. #[error("Unsafe L2 block is missing {0}")] MissingUnsafeL2Block(u64), @@ -38,8 +44,11 @@ impl From for ConsolidateTaskError { impl EngineTaskError for ConsolidateTaskError { fn severity(&self) -> EngineTaskErrorSeverity { match self { + Self::BreaksFinality(_) => EngineTaskErrorSeverity::Skip, Self::MissingUnsafeL2Block(_) => EngineTaskErrorSeverity::Reset, - Self::FailedToFetchUnsafeL2Block => EngineTaskErrorSeverity::Temporary, + Self::FailedToFetchDelegatedSafeBlock(_) | Self::FailedToFetchUnsafeL2Block => { + EngineTaskErrorSeverity::Temporary + } Self::BuildTaskFailed(inner) => inner.severity(), Self::SealTaskFailed(inner) => inner.severity(), Self::ForkchoiceUpdateFailed(inner) => inner.severity(), diff --git a/rust/kona/crates/node/engine/src/task_queue/tasks/consolidate/mod.rs b/rust/kona/crates/node/engine/src/task_queue/tasks/consolidate/mod.rs index 21b125bd135..110983b601f 100644 --- a/rust/kona/crates/node/engine/src/task_queue/tasks/consolidate/mod.rs +++ b/rust/kona/crates/node/engine/src/task_queue/tasks/consolidate/mod.rs @@ -1,5 +1,8 @@ //! Consolidation Task +mod delegated_safe_heads; +pub use delegated_safe_heads::{DelegatedSafeHeads, InconsistentSafeHeadsError}; + mod error; pub use error::ConsolidateTaskError; diff --git a/rust/kona/crates/node/engine/src/task_queue/tasks/consolidate/task.rs b/rust/kona/crates/node/engine/src/task_queue/tasks/consolidate/task.rs index d3183cdafa5..f9d6c5dd2cc 100644 --- a/rust/kona/crates/node/engine/src/task_queue/tasks/consolidate/task.rs +++ b/rust/kona/crates/node/engine/src/task_queue/tasks/consolidate/task.rs @@ -1,10 +1,12 @@ //! A task to consolidate the engine state. use crate::{ - ConsolidateTaskError, EngineClient, EngineState, ExecutableEngineTask, ImportedBlockSink, + ConsolidateTaskError, DelegatedSafeHeads, EngineClient, EngineState, ExecutableEngineTask, + FinalityViolation, ImportedBlockSink, state::EngineSyncStateUpdate, task_queue::{build_and_import, synchronize, synchronize_unconditionally}, }; +use alloy_primitives::B256; use alloy_rpc_types_eth::Block; use async_trait::async_trait; use kona_genesis::RollupConfig; @@ -12,18 +14,18 @@ use kona_protocol::{L2BlockInfo, OpAttributesWithParent}; use op_alloy_rpc_types::Transaction; use std::sync::Arc; -/// Input for consolidation - either derived attributes or safe L2 block +/// Input for consolidation - either derived attributes or delegated safe heads #[derive(Debug, Clone)] pub enum ConsolidateInput { /// Consolidate based on derived attributes. Attributes(Box), - /// Derivation Delegation: consolidate based on safe L2 block info. - BlockInfo(L2BlockInfo), + /// Derivation Delegation: consolidate onto the safe heads a derivation delegate reports. + Delegated(Box), } -impl From for ConsolidateInput { - fn from(v: L2BlockInfo) -> Self { - Self::BlockInfo(v) +impl From for ConsolidateInput { + fn from(heads: DelegatedSafeHeads) -> Self { + Self::Delegated(Box::new(heads)) } } @@ -38,7 +40,7 @@ impl ConsolidateInput { const fn l2_block_number(&self) -> u64 { match self { Self::Attributes(attributes) => attributes.block_number(), - Self::BlockInfo(info) => info.block_info.number, + Self::Delegated(heads) => heads.local_safe_head().block_info.number, } } @@ -48,7 +50,7 @@ impl ConsolidateInput { Self::Attributes(attributes) => { crate::AttributesMatch::check(cfg, attributes, block).is_match() } - Self::BlockInfo(info) => block.header.hash == info.block_info.hash, + Self::Delegated(heads) => block.header.hash == heads.local_safe_head().block_info.hash, } } @@ -59,14 +61,14 @@ impl ConsolidateInput { } /// The [`ConsolidateTask`] attempts to consolidate the engine state -/// using the specified payload attributes or block info. +/// using the specified derived attributes or delegated safe heads. #[derive(Debug, Clone)] pub struct ConsolidateTask { /// The engine client. pub client: Arc, /// The [`RollupConfig`]. pub cfg: Arc, - /// The input for consolidation (either attributes or block info). + /// The input for consolidation (either derived attributes or delegated safe heads). pub input: ConsolidateInput, /// Where to hand the decoded block once the engine has canonicalized it. pub block_sink: Arc, @@ -100,7 +102,356 @@ impl ConsolidateTask { Ok(()) } - /// Handles the fallback case when the block doesn't match the input or does not exist. + /// The hash of the execution layer's canonical block at `number`, or `None` when it has none + /// there. Only the hash is needed, so the block is fetched without its transactions. + async fn el_block_hash(&self, number: u64) -> Result, ConsolidateTaskError> { + debug!(target: "engine", number, "Looking up the execution layer's block hash"); + let block = self.client.get_l2_block(number.into()).await.map_err(|err| { + warn!(target: "engine", ?err, number, "Failed to look up the execution layer's block"); + ConsolidateTaskError::FailedToFetchDelegatedSafeBlock(number) + })?; + let el_hash = block.map(|block| block.header.hash); + debug!(target: "engine", number, ?el_hash, "Looked up the execution layer's block hash"); + Ok(el_hash) + } + + /// The hash of the engine's unsafe or established finalized head at `number`, if one sits at + /// that height. Only these two pin the engine's chain: the unsafe head is the head the + /// execution layer follows, and finality cannot move, while the safe labels can stay on blocks + /// that a delegate's reorg abandoned. + fn chain_pinning_label_hash_at(state: &EngineState, number: u64) -> Option { + let sync_state = &state.sync_state; + Some(sync_state.unsafe_head()) + .filter(|unsafe_head| *unsafe_head != L2BlockInfo::default()) + .map(|unsafe_head| unsafe_head.block_info.id()) + .into_iter() + .chain(sync_state.established_finalized_head()) + .find(|label| label.number == number) + .map(|label| label.hash) + } + + /// The safe label when the cross-safe head cannot be checked against the engine's chain: the + /// cross-safe head only when it respects finality and advances the label. + fn unverified_advancing_safe_head( + state: &EngineState, + heads: &DelegatedSafeHeads, + ) -> Option { + Self::check_cross_safe_finality(state, heads) + .ok() + .and_then(|()| heads.advancing_safe_head(&state.sync_state.safe_head())) + } + + /// The one check of the delegated cross-safe head against finality. + fn check_cross_safe_finality( + state: &EngineState, + heads: &DelegatedSafeHeads, + ) -> Result<(), FinalityViolation> { + state.sync_state.ensure_respects_finality(heads.safe_head().block_info.id()) + } + + /// The safe label when the unsafe head moves onto a chain the current safe label is not proven + /// to be on: the cross-safe head, clamped to the finalized head; a conflict with finality + /// skips. + fn safe_label_off_engine_chain( + state: &EngineState, + heads: &DelegatedSafeHeads, + ) -> Result { + match Self::check_cross_safe_finality(state, heads) { + Ok(()) => Ok(heads.safe_head()), + Err(FinalityViolation::BehindFinalizedHead { .. }) => { + Ok(state.sync_state.finalized_head()) + } + Err(violation @ FinalityViolation::ConflictsWithFinalizedHead { .. }) => { + Err(violation.into()) + } + } + } + + /// The safe label for delegated safe heads whose local-safe head is on the engine's chain: the + /// cross-safe head when it advances the label, or when it replaces, at the label's height, a + /// safe label that a reorg left on abandoned blocks, and `None`, which keeps the label. + async fn safe_label_on_engine_chain( + &self, + state: &EngineState, + heads: &DelegatedSafeHeads, + ) -> Result, ConsolidateTaskError> { + if let Err(violation) = Self::check_cross_safe_finality(state, heads) { + debug!(target: "engine", %violation, ?heads, "Delegated cross-safe head breaks finality; keeping the safe label"); + return Ok(None); + } + let safe = state.sync_state.safe_head(); + let cross_safe = heads.safe_head(); + // Below the safe label the cross-safe head can neither advance nor replace it. + if cross_safe.block_info.number < safe.block_info.number { + return Ok(None); + } + + let is_known_on_engine_chain = cross_safe.block_info.id() == + heads.local_safe_head().block_info.id() || + cross_safe.block_info.id() == safe.block_info.id(); + let chain_hash = if is_known_on_engine_chain { + Some(cross_safe.block_info.hash) + } else { + self.engine_chain_hash_at(state, cross_safe.block_info.number).await? + }; + if chain_hash.is_some_and(|hash| hash != cross_safe.block_info.hash) { + debug!(target: "engine", ?heads, "Delegated cross-safe head is not on the chain of its local-safe head; keeping the safe label"); + return Ok(None); + } + let is_proven_on_engine_chain = chain_hash.is_some(); + let replaces_stale_safe_label = is_proven_on_engine_chain && + cross_safe.block_info.number == safe.block_info.number && + cross_safe.block_info.hash != safe.block_info.hash; + Ok(if replaces_stale_safe_label { + Some(cross_safe) + } else { + heads.advancing_safe_head(&safe) + }) + } + + /// The hash of the engine's chain at `number`: its unsafe or finalized head when one sits + /// there, and otherwise the execution layer's canonical block. `None` when nothing can tell. + async fn engine_chain_hash_at( + &self, + state: &EngineState, + number: u64, + ) -> Result, ConsolidateTaskError> { + if let Some(label_hash) = Self::chain_pinning_label_hash_at(state, number) { + return Ok(Some(label_hash)); + } + self.el_canonical_hash_at(state, number).await + } + + /// The hash of the execution layer's canonical block at `number`, only while that chain is the + /// engine's: the execution layer applied the engine's last forkchoice, and `number` is at or + /// below the unsafe head. Otherwise its canonical blocks may predate a sync, or sit above a + /// rewound unsafe head, and prove nothing, so `None` without a lookup. + async fn el_canonical_hash_at( + &self, + state: &EngineState, + number: u64, + ) -> Result, ConsolidateTaskError> { + let is_el_canonical_chain_current = state.el_applied_last_forkchoice && + number <= state.sync_state.unsafe_head().block_info.number; + if is_el_canonical_chain_current { self.el_block_hash(number).await } else { Ok(None) } + } + + /// Moves the unsafe and local-safe heads to the delegated local-safe head, and the safe label + /// to `safe_head`. + async fn reconcile_to_delegated_safe_heads( + &self, + state: &mut EngineState, + heads: &DelegatedSafeHeads, + safe_head: Option, + ) -> Result<(), ConsolidateTaskError> { + debug!( + target: "engine", + local_safe_l2 = %heads.local_safe_head(), + safe_l2 = %heads.safe_head(), + ?safe_head, + "Apply safe head" + ); + + // We intentionally set the unsafe head to the local-safe head to ensure the engine observes + // a self-consistent head state. This is required to correctly handle reorgs (where unsafe + // may be ahead on a non-canonical fork) and to trigger EL sync when the local unsafe head + // lags behind the safe head. + self.apply_delegated_safe_heads( + state, + heads, + EngineSyncStateUpdate { + local_safe_head: Some(heads.local_safe_head()), + safe_head, + unsafe_head: Some(heads.local_safe_head()), + ..Default::default() + }, + "Updated safe head via follow safe", + ) + .await + } + + /// Reorgs the engine onto delegated safe heads whose chain differs from the engine's, moving + /// the safe label to the cross-safe head even when lower, since the old safe block may be on + /// the abandoned chain, or to the finalized head when the cross-safe head is below it. + async fn reorg_onto_delegated_safe_heads( + &self, + state: &mut EngineState, + heads: &DelegatedSafeHeads, + ) -> Result<(), ConsolidateTaskError> { + let safe_head = Self::safe_label_off_engine_chain(state, heads)?; + warn!( + target: "engine", + local_safe_l2 = %heads.local_safe_head(), + %safe_head, + "Reorging onto the delegate's chain" + ); + self.reconcile_to_delegated_safe_heads(state, heads, Some(safe_head)).await + } + + /// Moves the engine to delegated safe heads at or above the unsafe head. Only the unsafe head + /// itself or its child extends the engine's chain; any other local-safe head has unknown + /// ancestry and reorgs, so the safe label follows it and stays an ancestor of the new head. + async fn extend_onto_delegated_safe_heads( + &self, + state: &mut EngineState, + heads: &DelegatedSafeHeads, + ) -> Result<(), ConsolidateTaskError> { + let unsafe_head = state.sync_state.unsafe_head().block_info; + let local_safe = heads.local_safe_head().block_info; + let is_at_unsafe_height = local_safe.number == unsafe_head.number; + if is_at_unsafe_height && local_safe.hash != unsafe_head.hash { + return self.reorg_onto_delegated_safe_heads(state, heads).await; + } + let is_child_of_unsafe_head = local_safe.number == unsafe_head.number + 1 && + local_safe.parent_hash == unsafe_head.hash; + let is_unproven_extension = !is_at_unsafe_height && !is_child_of_unsafe_head; + let would_lower_safe_label = + heads.safe_head().block_info.number < state.sync_state.safe_head().block_info.number; + if is_unproven_extension && + !(would_lower_safe_label && self.is_safe_label_on_el_chain(state).await?) + { + return self.reorg_onto_delegated_safe_heads(state, heads).await; + } + + let safe_head = self.safe_label_on_engine_chain(state, heads).await?; + self.reconcile_to_delegated_safe_heads(state, heads, safe_head).await + } + + /// Whether the execution layer's canonical chain holds the engine's safe label. A local-safe + /// head several blocks above the unsafe head has ancestry the engine cannot check, but a + /// delegate catching up builds on the safe label the execution layer holds, so that label stays + /// rather than dropping to a lower cross-safe head. + async fn is_safe_label_on_el_chain( + &self, + state: &EngineState, + ) -> Result { + let safe = state.sync_state.safe_head().block_info; + Ok(self.el_canonical_hash_at(state, safe.number).await? == Some(safe.hash)) + } + + /// Moves the safe labels to a delegated local-safe head that the syncing execution layer does + /// not have yet, keeping the unsafe head as its sync target; the cross-safe head only advances. + async fn follow_safe_head_during_el_sync( + &self, + state: &mut EngineState, + heads: &DelegatedSafeHeads, + ) -> Result<(), ConsolidateTaskError> { + warn!( + target: "engine", + local_safe_l2 = %heads.local_safe_head(), + unsafe_number = state.sync_state.unsafe_head().block_info.number, + "Execution layer does not have the delegated safe head yet; keeping its sync target" + ); + + let safe_head = Self::unverified_advancing_safe_head(state, heads); + self.apply_delegated_safe_heads( + state, + heads, + EngineSyncStateUpdate { + local_safe_head: Some(heads.local_safe_head()), + safe_head, + ..Default::default() + }, + "Updated safe head while the execution layer syncs", + ) + .await + } + + /// Applies `update`, whose labels follow the delegated `heads`, and always sends its + /// forkchoice update: the execution layer may have answered `SYNCING` to the same labels + /// before, or lost them in a restart. + async fn apply_delegated_safe_heads( + &self, + state: &mut EngineState, + heads: &DelegatedSafeHeads, + update: EngineSyncStateUpdate, + outcome: &'static str, + ) -> Result<(), ConsolidateTaskError> { + synchronize_unconditionally(self.client.as_ref(), state, update).await.map_err(|e| { + warn!(target: "engine", ?e, ?heads, "Applying the delegated safe heads failed"); + e + })?; + + info!( + target: "engine", + local_safe_hash = %heads.local_safe_head().block_info.hash, + local_safe_number = heads.local_safe_head().block_info.number, + safe_hash = %state.sync_state.safe_head().block_info.hash, + safe_number = state.sync_state.safe_head().block_info.number, + "{outcome}" + ); + + Ok(()) + } + + /// Follows delegated safe heads below the engine's local-safe head. Only a different block at + /// that height in a synced execution layer proves divergence and reorgs; otherwise the + /// cross-safe head may only move the safe label. After a delegate L1 reorg whose fork point is + /// above its reported head, the labels stay on the abandoned blocks until it passes that point. + async fn follow_lagging_delegated_safe_heads( + &self, + state: &mut EngineState, + heads: &DelegatedSafeHeads, + ) -> Result<(), ConsolidateTaskError> { + let local_safe = heads.local_safe_head().block_info; + let el_hash = self.el_canonical_hash_at(state, local_safe.number).await?; + let safe_head = match el_hash { + Some(el_hash) if el_hash != local_safe.hash => { + debug!(target: "engine", ?heads, %el_hash, "Delegated local-safe head behind the engine's diverges from the execution layer"); + return self.reorg_onto_delegated_safe_heads(state, heads).await; + } + Some(_) => self.safe_label_on_engine_chain(state, heads).await?, + None => Self::unverified_advancing_safe_head(state, heads), + }; + let Some(safe_head) = safe_head else { + debug!( + target: "engine", + ?heads, + local_safe_number = state.sync_state.local_safe_head().block_info.number, + "Ignoring delegated safe heads behind the local-safe head" + ); + return Ok(()); + }; + self.apply_delegated_safe_heads( + state, + heads, + EngineSyncStateUpdate { safe_head: Some(safe_head), ..Default::default() }, + "Updated safe head from a delegate behind the local-safe head", + ) + .await + } + + /// The safe label after consolidating onto `consolidated`, which the execution layer has: that + /// block for derived attributes, and the judged cross-safe head for a delegate. + async fn consolidated_safe_head( + &self, + state: &EngineState, + consolidated: L2BlockInfo, + ) -> Result, ConsolidateTaskError> { + let ConsolidateInput::Delegated(heads) = &self.input else { + return Ok(Some(consolidated)); + }; + self.safe_label_on_engine_chain(state, heads).await + } + + /// Handles a block that does not match the input: derived attributes build their own block, + /// and delegated safe heads reorg the engine onto the delegate's chain. + async fn reconcile_mismatch( + &self, + state: &mut EngineState, + ) -> Result<(), ConsolidateTaskError> { + match &self.input { + ConsolidateInput::Attributes(attributes) => { + self.build_and_import_attributes(state, attributes).await + } + ConsolidateInput::Delegated(heads) => { + self.reorg_onto_delegated_safe_heads(state, heads).await + } + } + } + + /// Handles input at or above the unsafe head: derived attributes build their block, and + /// delegated safe heads move the engine onto them. async fn reconcile_unsafe_to_safe( &self, state: &mut EngineState, @@ -109,38 +460,8 @@ impl ConsolidateTask { ConsolidateInput::Attributes(attributes) => { self.build_and_import_attributes(state, attributes).await } - ConsolidateInput::BlockInfo(safe_l2) => { - warn!(target: "engine", safe_l2 = %safe_l2, "Apply safe head"); - - // We intentionally set the unsafe head to safe_l2 to ensure the engine observes a - // self-consistent head state. This is required to correctly handle reorgs (where - // unsafe may be ahead on a non-canonical fork) and to trigger EL sync when the - // local unsafe head lags behind the safe head. The forkchoice update is always - // sent: the execution layer may have answered `SYNCING` to the same labels before, - // or lost them in a restart. - synchronize_unconditionally( - self.client.as_ref(), - state, - EngineSyncStateUpdate { - local_safe_head: Some(*safe_l2), - safe_head: Some(*safe_l2), - unsafe_head: Some(*safe_l2), - ..Default::default() - }, - ) - .await - .map_err(|e| { - warn!(target: "engine", ?e, safe_l2 = %safe_l2, "Applying the delegated safe head failed"); - e - })?; - - info!( - target: "engine", - hash = %safe_l2.block_info.hash, - number = safe_l2.block_info.number, - "Updated safe head via follow safe" - ); - Ok(()) + ConsolidateInput::Delegated(heads) => { + self.extend_onto_delegated_safe_heads(state, heads).await } } } @@ -159,40 +480,8 @@ impl ConsolidateTask { let block = match self.client.l2_block_by_label(block_num.into()).await { Ok(Some(block)) => block, Ok(None) => { - if let ConsolidateInput::BlockInfo(safe_l2) = &self.input { - // A block missing below the unsafe head means the execution layer is still - // syncing toward the unsafe head. Resetting would restart that sync from the - // execution layer's old labels, and moving the unsafe head back would retarget - // it. Like op-node's follow-source mode, only the safe labels follow the - // delegate, and the forkchoice update, sent even when unchanged, carries them - // to the execution layer. - warn!( - target: "engine", - safe_l2 = %safe_l2, - unsafe_number = state.sync_state.unsafe_head().block_info.number, - "Execution layer does not have the delegated safe head yet; keeping its sync target" - ); - synchronize_unconditionally( - self.client.as_ref(), - state, - EngineSyncStateUpdate { - local_safe_head: Some(*safe_l2), - safe_head: Some(*safe_l2), - ..Default::default() - }, - ) - .await - .map_err(|e| { - warn!(target: "engine", ?e, safe_l2 = %safe_l2, "Applying the delegated safe head failed"); - e - })?; - info!( - target: "engine", - hash = %safe_l2.block_info.hash, - number = safe_l2.block_info.number, - "Updated safe head while the execution layer syncs" - ); - return Ok(()); + if let ConsolidateInput::Delegated(heads) = &self.input { + return self.follow_safe_head_during_el_sync(state, heads).await; } warn!(target: "engine", "Received `None` block for {}", block_num); return Err(ConsolidateTaskError::MissingUnsafeL2Block(block_num)); @@ -203,90 +492,107 @@ impl ConsolidateTask { }; let block_hash = block.header.hash; - if self.input.is_consistent_with_block(&self.cfg, &block) { - trace!( + if !self.input.is_consistent_with_block(&self.cfg, &block) { + // A different block from an execution layer that has not applied the engine's + // forkchoice may be from before its sync, so it does not prove the delegate diverged. + if let ConsolidateInput::Delegated(heads) = &self.input && + !state.el_applied_last_forkchoice + { + return self.follow_safe_head_during_el_sync(state, heads).await; + } + debug!( target: "engine", input = ?self.input, block_hash = %block_hash, - "Consolidating engine state", + "ConsolidateInput mismatch! Initiating reorg", ); - let consensus_block = - block.into_consensus().map_transactions(|t| t.inner.inner.into_inner()); - match L2BlockInfo::from_block_and_genesis(&consensus_block, &self.cfg.genesis) { - // A span batch's blocks before its last one only move the safe label; the last - // block's forkchoice update delivers them all. This avoids sending a FCU call for - // every block in the span batch. - Ok(block_info) if self.input.is_intermediate_span_block() => { - // The next attributes built are this block's child, and ask for its config. - self.block_sink.block_imported(consensus_block, block_info); - - // Apply a transient update to the safe head. - state.sync_state = state.sync_state.apply_update(EngineSyncStateUpdate { - safe_head: Some(block_info), - local_safe_head: Some(block_info), - ..Default::default() - }); - - info!( - target: "engine", - hash = %block_info.block_info.hash, - number = block_info.block_info.number, - "Updated safe head via L1 consolidation" - ); - - return Ok(()); - } - Ok(block_info) => { - // The next attributes built are this block's child, and ask for its config. - self.block_sink.block_imported(consensus_block, block_info); - - let update = EngineSyncStateUpdate { - safe_head: Some(block_info), - local_safe_head: Some(block_info), - ..Default::default() - }; - let synchronized = match &self.input { - // Derived attributes move the safe head on every consolidation, so an - // unchanged forkchoice was already sent. - ConsolidateInput::Attributes(_) => { - synchronize(self.client.as_ref(), state, update).await - } - // A delegated safe head can repeat a forkchoice the execution layer - // answered `SYNCING` or lost, so it is always sent. - ConsolidateInput::BlockInfo(_) => { - synchronize_unconditionally(self.client.as_ref(), state, update).await - } - }; - synchronized.map_err(|e| { - warn!(target: "engine", ?e, "Consolidation failed"); - e - })?; - - info!( - target: "engine", - hash = %block_info.block_info.hash, - number = block_info.block_info.number, - "Updated safe head via L1 consolidation" - ); - - return Ok(()); - } - Err(e) => { - // Continue on to build the block since we failed to construct the block info. - warn!(target: "engine", ?e, "Failed to construct L2BlockInfo, proceeding to build task"); - } - } + return self.reconcile_mismatch(state).await; } - debug!( + trace!( target: "engine", input = ?self.input, block_hash = %block_hash, - "ConsolidateInput mismatch! Initiating reorg", + "Consolidating engine state", ); - // Handle mismatch case - called when consistency check fails - // or when L2BlockInfo construction fails in Attributes branch - self.reconcile_unsafe_to_safe(state).await + let consensus_block = + block.into_consensus().map_transactions(|t| t.inner.inner.into_inner()); + let block_info = match ( + L2BlockInfo::from_block_and_genesis(&consensus_block, &self.cfg.genesis), + &self.input, + ) { + (Ok(block_info), _) => block_info, + // The block matches the delegated local-safe head by hash, so the delegate's info + // describes it, and the block stays where it is on the engine's chain. + (Err(e), ConsolidateInput::Delegated(heads)) => { + warn!(target: "engine", ?e, "Failed to construct L2BlockInfo, using the delegated local-safe head's"); + heads.local_safe_head() + } + (Err(e), ConsolidateInput::Attributes(attributes)) => { + // Continue on to build the block since we failed to construct the block info. + warn!(target: "engine", ?e, "Failed to construct L2BlockInfo, proceeding to build task"); + return self.build_and_import_attributes(state, attributes).await; + } + }; + + // A span batch's blocks before its last one only move the safe label; the last block's + // forkchoice update delivers them all. This avoids sending a FCU call for every block in + // the span batch. + if self.input.is_intermediate_span_block() { + // The next attributes built are this block's child, and ask for its config. + self.block_sink.block_imported(consensus_block, block_info); + + // Apply a transient update to the safe head. + state.sync_state = state.sync_state.apply_update(EngineSyncStateUpdate { + local_safe_head: Some(block_info), + safe_head: Some(block_info), + ..Default::default() + }); + + info!( + target: "engine", + hash = %block_info.block_info.hash, + number = block_info.block_info.number, + "Updated safe head via L1 consolidation" + ); + + return Ok(()); + } + + let safe_head = self.consolidated_safe_head(state, block_info).await?; + // The next attributes built are this block's child, and ask for its config. + self.block_sink.block_imported(consensus_block, block_info); + + let update = EngineSyncStateUpdate { + local_safe_head: Some(block_info), + safe_head, + ..Default::default() + }; + let synchronized = match &self.input { + // Derived attributes move the safe head on every consolidation, so an unchanged + // forkchoice was already sent. + ConsolidateInput::Attributes(_) => { + synchronize(self.client.as_ref(), state, update).await + } + // A delegated safe head can repeat a forkchoice the execution layer answered `SYNCING` + // or lost, so it is always sent. + ConsolidateInput::Delegated(_) => { + synchronize_unconditionally(self.client.as_ref(), state, update).await + } + }; + synchronized.map_err(|e| { + warn!(target: "engine", ?e, "Consolidation failed"); + e + })?; + + info!( + target: "engine", + hash = %block_info.block_info.hash, + number = block_info.block_info.number, + "Updated safe head via L1 consolidation" + ); + + Ok(()) } } @@ -302,16 +608,36 @@ impl ExecutableEngineTask for ConsolidateTask { // DerivationActor and EngineActor coordinate both safe and unsafe heads. In this case, we // consolidate as long as the unsafe head has not fallen behind. // - // - `BlockInfo`: The safe head is injected externally by the DerivationActor while delegating - // derivation, and is not coordinated with the EngineActor's safe/unsafe heads. If the - // injected safe head is ahead of the EngineActor's unsafe head, we reconcile the unsafe chain - // up to the safe head instead of consolidating. + // - `Delegated`: The safe heads are injected externally by the DerivationActor while delegating + // derivation, and are not coordinated with the EngineActor's heads. A local-safe head below + // the engine's comes from a lagging or diverged delegate; one below the unsafe head is + // consolidated against the execution layer's block; one at or above the unsafe head moves the + // unsafe chain onto it. async fn execute(&self, state: &mut EngineState) -> Result<(), ConsolidateTaskError> { - let safe_head_number = match &self.input { - ConsolidateInput::Attributes { .. } => state.sync_state.safe_head().block_info.number, - ConsolidateInput::BlockInfo(safe_block_info) => safe_block_info.block_info.number, + let ConsolidateInput::Delegated(heads) = &self.input else { + return if state.sync_state.safe_head().block_info.number < + state.sync_state.unsafe_head().block_info.number + { + self.consolidate(state).await + } else { + self.reconcile_unsafe_to_safe(state).await + }; }; - if safe_head_number < state.sync_state.unsafe_head().block_info.number { + + let local_safe_number = heads.local_safe_head().block_info.number; + state.sync_state.ensure_respects_finality(heads.local_safe_head().block_info.id())?; + // Labels with the unsafe head below the safe labels take the paths below, which move them + // back into order; ignoring a lagging delegate would keep them broken until a finalization + // above the unsafe head fails. + let unsafe_number = state.sync_state.unsafe_head().block_info.number; + let are_labels_consistent = unsafe_number >= + state.sync_state.local_safe_head().block_info.number && + unsafe_number >= state.sync_state.safe_head().block_info.number; + if are_labels_consistent && + local_safe_number < state.sync_state.local_safe_head().block_info.number + { + self.follow_lagging_delegated_safe_heads(state, heads).await + } else if local_safe_number < state.sync_state.unsafe_head().block_info.number { self.consolidate(state).await } else { self.reconcile_unsafe_to_safe(state).await @@ -323,10 +649,11 @@ impl ExecutableEngineTask for ConsolidateTask { mod tests { use super::*; use crate::{ - EngineSyncState, EngineTaskError, ImportedBlockSink, NoopBlockSink, SealTaskError, + EngineSyncState, EngineTaskError, EngineTaskErrorSeverity, FinalityViolation, + ImportedBlockSink, NoopBlockSink, SealTaskError, test_utils::{ - TestAttributesBuilder, TestEngineStateBuilder, l1_info_deposit, test_block_info, - test_engine_client, + RpcMock, TestAttributesBuilder, TestEngineStateBuilder, delegated_safe_heads, + l1_info_deposit, test_block_info, test_engine_client, }, }; use alloy_consensus::transaction::Recovered; @@ -368,7 +695,7 @@ mod tests { }; let sink = Arc::new(RecordingSink::default()); let task = - ConsolidateTask::new(client, cfg, ConsolidateInput::BlockInfo(safe_l2), sink.clone()); + ConsolidateTask::new(client, cfg, delegated_safe_heads(safe_l2, safe_l2), sink.clone()); task.consolidate(&mut EngineState::default()).await.unwrap(); @@ -509,6 +836,7 @@ mod tests { fn engine_state_with_safe_head(safe_head: L2BlockInfo) -> EngineState { TestEngineStateBuilder::new() .with_unsafe_head(test_block_info(20)) + .with_local_safe_head(safe_head) .with_safe_head(safe_head) .with_finalized_head(test_block_info(5)) .build() @@ -584,7 +912,7 @@ mod tests { let task = ConsolidateTask::new( Arc::new(client), cfg, - ConsolidateInput::BlockInfo(delegated_safe_head), + delegated_safe_heads(delegated_safe_head, delegated_safe_head), sink.clone(), ); @@ -692,4 +1020,859 @@ mod tests { l1.assert_finished(); l2.assert_finished(); } + + /// Where a delegated local-safe head sits against the finalized head. + #[derive(Clone, Copy, Debug)] + enum TargetAgainstFinalized { + Below, + ConflictingAt, + } + + impl TargetAgainstFinalized { + fn delegated_safe_head(self, finalized: L2BlockInfo) -> L2BlockInfo { + let number = finalized.block_info.number; + match self { + Self::Below => test_block_info(number - 1), + Self::ConflictingAt => L2BlockInfo { + block_info: BlockInfo { + hash: !finalized.block_info.hash, + ..finalized.block_info + }, + ..finalized + }, + } + } + } + + fn is_behind_finality(result: &Result<(), ConsolidateTaskError>) -> bool { + matches!( + result, + Err(ConsolidateTaskError::BreaksFinality( + FinalityViolation::BehindFinalizedHead { .. } + )) + ) + } + + fn is_conflicting_with_finality(result: &Result<(), ConsolidateTaskError>) -> bool { + matches!( + result, + Err(ConsolidateTaskError::BreaksFinality( + FinalityViolation::ConflictsWithFinalizedHead { .. } + )) + ) + } + + /// A delegated local-safe head that breaks finality, as a stale delegate snapshot reports, is + /// skipped before the execution layer is asked anything, and leaves every label where it was. + #[rstest::rstest] + #[case::below(TargetAgainstFinalized::Below, is_behind_finality)] + #[case::conflicting(TargetAgainstFinalized::ConflictingAt, is_conflicting_with_finality)] + #[tokio::test] + async fn delegated_local_safe_head_breaking_finality_is_skipped( + #[case] target: TargetAgainstFinalized, + #[case] is_expected: fn(&Result<(), ConsolidateTaskError>) -> bool, + ) { + let mut state = engine_state_with_safe_head(test_block_info(12)); + let state_before = state; + let delegated = target.delegated_safe_head(state.sync_state.finalized_head()); + + let result = run_delegated(&mut state, delegated, delegated, |_| {}).await; + + assert!(is_expected(&result), "got {result:?}"); + assert_eq!(result.unwrap_err().severity(), crate::EngineTaskErrorSeverity::Skip); + assert_eq!(state, state_before); + } + + /// A block at `number` on the chain identified by `chain`, and its info. Blocks on different + /// chains at the same height have different hashes. + fn chain_block(number: u64, chain: u64) -> (RpcBlock, L2BlockInfo) { + let header = RpcHeader::new(alloy_consensus::Header { + number, + timestamp: chain, + ..Default::default() + }); + let info = L2BlockInfo { + block_info: BlockInfo { hash: header.hash, number, ..Default::default() }, + ..Default::default() + }; + (RpcBlock::new(header, BlockTransactions::Full(vec![])), info) + } + + /// The engine's chain. + const ENGINE_CHAIN: u64 = 0; + /// A chain that diverged from the engine's. + const OTHER_CHAIN: u64 = 1; + + fn engine_chain_block(number: u64) -> L2BlockInfo { + chain_block(number, ENGINE_CHAIN).1 + } + + fn other_chain_block(number: u64) -> L2BlockInfo { + chain_block(number, OTHER_CHAIN).1 + } + + /// What the execution layer holds at a height: the engine chain's block, another chain's, or + /// nothing. + #[derive(Clone, Copy, Debug)] + enum ElHas { + EngineChain, + Missing, + OtherChain, + } + + impl ElHas { + fn block(self, number: u64) -> Option> { + match self { + Self::EngineChain => Some(chain_block(number, ENGINE_CHAIN).0), + Self::Missing => None, + Self::OtherChain => Some(chain_block(number, OTHER_CHAIN).0), + } + } + } + + /// An engine state with the unsafe head at 20 and the finalized head at 5 on the engine's + /// chain, and its local-safe and safe heads at the given heights, which differ under interop. + fn engine_state_with_safe_labels(local_safe: u64, safe: u64) -> EngineState { + TestEngineStateBuilder::new() + .with_unsafe_head(engine_chain_block(20)) + .with_local_safe_head(engine_chain_block(local_safe)) + .with_safe_head(engine_chain_block(safe)) + .with_finalized_head(engine_chain_block(5)) + .build() + } + + /// Runs a consolidation of the delegated `local_safe` and `cross_safe` heads against an + /// execution layer that `expect_engine_calls` scripts, and checks every scripted call was made. + /// Genesis is pinned to the local-safe head, so its empty block decodes without an L1-info + /// deposit. + async fn run_delegated( + state: &mut EngineState, + local_safe: L2BlockInfo, + cross_safe: L2BlockInfo, + expect_engine_calls: impl FnOnce(&RpcMock), + ) -> Result<(), ConsolidateTaskError> { + let cfg = Arc::new(RollupConfig { + genesis: ChainGenesis { l2: local_safe.block_info.id(), ..Default::default() }, + ..Default::default() + }); + let (client, l1, l2) = test_engine_client(cfg.clone()); + expect_engine_calls(&l2); + let task = ConsolidateTask::new( + Arc::new(client), + cfg, + delegated_safe_heads(local_safe, cross_safe), + Arc::new(NoopBlockSink), + ); + + let result = task.execute(state).await; + + l1.assert_finished(); + l2.assert_finished(); + result + } + + /// The labels after the engine moves its local-safe head, and its safe head to `safe`. + fn labels_after( + state: &EngineState, + local_safe: L2BlockInfo, + safe: L2BlockInfo, + ) -> EngineSyncState { + state.sync_state.apply_update(EngineSyncStateUpdate { + local_safe_head: Some(local_safe), + safe_head: Some(safe), + ..Default::default() + }) + } + + /// The labels after a reorg onto the delegated heads. + fn labels_after_reorg( + state: &EngineState, + local_safe: L2BlockInfo, + cross_safe: L2BlockInfo, + ) -> EngineSyncState { + state.sync_state.apply_update(EngineSyncStateUpdate { + local_safe_head: Some(local_safe), + safe_head: Some(cross_safe), + unsafe_head: Some(local_safe), + ..Default::default() + }) + } + + /// Below the unsafe head, a delegated local-safe head on the engine's chain that the execution + /// layer has becomes the local-safe label, and a cross-safe head on that chain above the safe + /// label moves it once the execution layer's block at its height confirms it. + #[tokio::test] + async fn advancing_cross_safe_head_moves_the_safe_label() { + let (local_safe, cross_safe) = (engine_chain_block(10), engine_chain_block(8)); + let mut state = engine_state_with_safe_labels(9, 7); + let expected = labels_after(&state, local_safe, cross_safe); + + run_delegated(&mut state, local_safe, cross_safe, |l2| { + l2.expect_block_by_number(10, ElHas::EngineChain.block(10)); + l2.expect_block_header_by_number(8, ElHas::EngineChain.block(8)); + l2.expect_forkchoice_updated( + expected.create_forkchoice_state(), + PayloadStatusEnum::Valid, + ); + }) + .await + .unwrap(); + + assert_eq!(state.sync_state, expected); + } + + /// The safe label's own block is known, so a cross-safe head there needs no lookup. + fn cross_safe_at_safe_label(l2: &RpcMock) { + l2.expect_block_by_number(10, ElHas::EngineChain.block(10)); + } + + /// A cross-safe head behind the safe label can neither advance nor replace it, so it is not + /// looked up. + fn cross_safe_behind_safe_label(l2: &RpcMock) { + l2.expect_block_by_number(10, ElHas::EngineChain.block(10)); + } + + /// The execution layer still syncs toward the local-safe head, so the cross-safe head is not + /// looked up. + fn el_syncing_cross_safe_behind_safe_label(l2: &RpcMock) { + l2.expect_block_by_number(10, ElHas::Missing.block(10)); + } + + /// The execution layer has another block at the cross-safe height than the delegate's. + fn cross_safe_off_the_local_safe_chain(l2: &RpcMock) { + l2.expect_block_by_number(10, ElHas::EngineChain.block(10)); + l2.expect_block_header_by_number(8, ElHas::OtherChain.block(8)); + } + + /// A cross-safe head at or below the safe label leaves it, so a load-balanced delegate whose + /// cross-safe head lags does not drop the forkchoice update's safe block; so does one that is + /// not on the chain of its own local-safe head, which the execution layer has. + #[rstest::rstest] + #[case::at_safe_label(7, cross_safe_at_safe_label)] + #[case::behind_safe_label(6, cross_safe_behind_safe_label)] + #[case::el_syncing_behind_safe_label(6, el_syncing_cross_safe_behind_safe_label)] + #[case::off_the_local_safe_chain(8, cross_safe_off_the_local_safe_chain)] + #[tokio::test] + async fn cross_safe_head_not_advancing_keeps_the_safe_label( + #[case] cross_safe_number: u64, + #[case] expect_lookups: fn(&RpcMock), + ) { + let local_safe = engine_chain_block(10); + let mut state = engine_state_with_safe_labels(9, 7); + let expected = labels_after(&state, local_safe, state.sync_state.safe_head()); + + run_delegated(&mut state, local_safe, engine_chain_block(cross_safe_number), |l2| { + expect_lookups(l2); + l2.expect_forkchoice_updated( + expected.create_forkchoice_state(), + PayloadStatusEnum::Valid, + ); + }) + .await + .unwrap(); + + assert_eq!(state.sync_state, expected); + } + + /// A cross-safe head behind, or conflicting with, finality leaves the safe label without a + /// lookup, while the local-safe head from the same status still moves. + #[rstest::rstest] + #[case::behind_finalized(engine_chain_block(4))] + #[case::conflicting_with_finalized(other_chain_block(5))] + #[tokio::test] + async fn cross_safe_head_breaking_finality_keeps_only_the_safe_label( + #[case] cross_safe: L2BlockInfo, + ) { + let local_safe = engine_chain_block(10); + let mut state = engine_state_with_safe_labels(9, 7); + let expected = labels_after(&state, local_safe, state.sync_state.safe_head()); + + run_delegated(&mut state, local_safe, cross_safe, |l2| { + l2.expect_block_by_number(10, ElHas::EngineChain.block(10)); + l2.expect_forkchoice_updated( + expected.create_forkchoice_state(), + PayloadStatusEnum::Valid, + ); + }) + .await + .unwrap(); + + assert_eq!(state.sync_state, expected); + } + + /// The execution layer has the lagging local-safe head at 10, and confirms the cross-safe head + /// at 9. + fn el_has_local_safe_10_and_cross_safe_9(l2: &RpcMock) { + l2.expect_block_header_by_number(10, ElHas::EngineChain.block(10)); + l2.expect_block_header_by_number(9, ElHas::EngineChain.block(9)); + } + + /// The execution layer lacks the lagging local-safe head at 10, so the cross-safe head is not + /// looked up. + fn el_missing_local_safe_10(l2: &RpcMock) { + l2.expect_block_header_by_number(10, ElHas::Missing.block(10)); + } + + /// A delegated local-safe head behind the engine's whose block the execution layer has, or + /// lacks, from a lagging delegate, leaves the local-safe and unsafe heads in place. Its + /// cross-safe head still moves the safe label when above it. + #[rstest::rstest] + #[case::el_has_local_safe(el_has_local_safe_10_and_cross_safe_9)] + #[case::el_missing_local_safe(el_missing_local_safe_10)] + #[tokio::test] + async fn lagging_delegate_moves_only_an_advancing_safe_label( + #[case] expect_lookups: fn(&RpcMock), + ) { + let (local_safe, cross_safe) = (engine_chain_block(10), engine_chain_block(9)); + let mut state = engine_state_with_safe_labels(12, 8); + let expected = state.sync_state.apply_update(EngineSyncStateUpdate { + safe_head: Some(cross_safe), + ..Default::default() + }); + + run_delegated(&mut state, local_safe, cross_safe, |l2| { + expect_lookups(l2); + l2.expect_forkchoice_updated( + expected.create_forkchoice_state(), + PayloadStatusEnum::Valid, + ); + }) + .await + .unwrap(); + + assert_eq!(state.sync_state, expected); + } + + /// A delegate before interop, whose heads are the same block, that lags the engine leaves every + /// label where it is and sends no forkchoice update, whether or not the execution layer has the + /// block. + #[rstest::rstest] + #[case::el_has_block(10, ElHas::EngineChain)] + #[case::el_missing_block(10, ElHas::Missing)] + #[case::at_finalized_head(5, ElHas::EngineChain)] + #[tokio::test] + async fn lagging_delegate_before_interop_changes_nothing( + #[case] delegated_number: u64, + #[case] el_at_local_safe: ElHas, + ) { + let delegated = engine_chain_block(delegated_number); + let mut state = engine_state_with_safe_labels(12, 12); + let state_before = state; + + run_delegated(&mut state, delegated, delegated, |l2| { + l2.expect_block_header_by_number( + delegated_number, + el_at_local_safe.block(delegated_number), + ); + }) + .await + .unwrap(); + + assert_eq!(state, state_before); + } + + /// A lagging delegate whose cross-safe head is behind the safe label leaves every label where + /// it is and sends no forkchoice update, without looking the cross-safe head up. + #[tokio::test] + async fn lagging_delegate_with_cross_safe_head_behind_the_safe_label_changes_nothing() { + let mut state = engine_state_with_safe_labels(12, 12); + let state_before = state; + + run_delegated(&mut state, engine_chain_block(10), engine_chain_block(7), |l2| { + l2.expect_block_header_by_number(10, ElHas::EngineChain.block(10)); + }) + .await + .unwrap(); + + assert_eq!(state, state_before); + } + + /// A lagging delegate whose cross-safe head conflicts with the finalized head leaves every + /// label where it is: the cross-safe head cannot become the safe label. + #[tokio::test] + async fn lagging_delegate_with_cross_safe_head_conflicting_with_finality_changes_nothing() { + let mut state = engine_state_with_safe_labels(12, 12); + let state_before = state; + + run_delegated(&mut state, engine_chain_block(10), other_chain_block(5), |l2| { + l2.expect_block_header_by_number(10, ElHas::EngineChain.block(10)); + }) + .await + .unwrap(); + + assert_eq!(state, state_before); + } + + /// A lagging delegate whose local-safe block the execution layer lacks cannot be judged against + /// the execution layer's canonical blocks, which may predate its sync, so a cross-safe head off + /// them is not looked up and reorgs nothing; not above the safe label, it changes nothing. + #[tokio::test] + async fn lagging_delegate_missing_from_the_el_never_reorgs() { + let mut state = engine_state_with_safe_labels(12, 12); + let state_before = state; + + run_delegated(&mut state, other_chain_block(10), other_chain_block(8), |l2| { + l2.expect_block_header_by_number(10, ElHas::Missing.block(10)); + }) + .await + .unwrap(); + + assert_eq!(state, state_before); + } + + /// A lagging delegate is not judged against an execution layer that has not applied the + /// engine's last forkchoice, even after its first sync finished: its canonical block at the + /// local-safe height may predate the engine's labels, so no lookup is made and a delegate on + /// another chain reorgs nothing. + #[tokio::test] + async fn lagging_delegate_is_not_judged_against_a_syncing_el() { + let mut state = engine_state_with_safe_labels(12, 12); + state.el_applied_last_forkchoice = false; + let state_before = state; + + run_delegated(&mut state, other_chain_block(10), other_chain_block(8), |_| {}) + .await + .unwrap(); + + assert_eq!(state, state_before); + } + + /// Lagging delegate: the execution layer has another block at its local-safe height. + fn lagging_local_safe_off_the_el_chain(l2: &RpcMock) { + l2.expect_block_header_by_number(10, ElHas::EngineChain.block(10)); + } + + /// Below the unsafe head: the execution layer has another block at the local-safe height. + fn local_safe_off_the_el_chain(l2: &RpcMock) { + l2.expect_block_by_number(18, ElHas::EngineChain.block(18)); + } + + /// At or above the unsafe head: the local-safe head is another block at the unsafe height. + const fn local_safe_off_the_unsafe_head(_: &RpcMock) {} + + /// Above the unsafe head but not its child: the local-safe head's ancestry is unknown, and the + /// execution layer does not hold the engine's safe label either. + fn local_safe_above_the_unsafe_head(l2: &RpcMock) { + l2.expect_block_header_by_number(12, ElHas::OtherChain.block(12)); + } + + /// Delegated heads on a chain that diverged from the engine's, which an execution-layer block + /// or the unsafe head at the local-safe height proves, or above the unsafe head without being + /// its child while the execution layer does not hold the safe label, reorg the engine onto + /// them: the unsafe and local-safe heads move to the local-safe head, and the safe label to the + /// cross-safe head even when lower, since the old safe block is on the abandoned chain. + /// Finalized stays. + #[rstest::rstest] + #[case::lagging_local_safe(10, lagging_local_safe_off_the_el_chain)] + #[case::local_safe_below_unsafe(18, local_safe_off_the_el_chain)] + #[case::local_safe_at_unsafe(20, local_safe_off_the_unsafe_head)] + #[case::local_safe_above_unsafe(25, local_safe_above_the_unsafe_head)] + #[tokio::test] + async fn delegated_heads_off_the_engine_chain_reorg_the_engine( + #[case] local_safe_number: u64, + #[case] expect_lookups: fn(&RpcMock), + ) { + let (local_safe, cross_safe) = (other_chain_block(local_safe_number), other_chain_block(8)); + let mut state = engine_state_with_safe_labels(12, 12); + let finalized_before = state.sync_state.finalized_head(); + let expected = labels_after_reorg(&state, local_safe, cross_safe); + + run_delegated(&mut state, local_safe, cross_safe, |l2| { + expect_lookups(l2); + l2.expect_forkchoice_updated( + expected.create_forkchoice_state(), + PayloadStatusEnum::Valid, + ); + }) + .await + .unwrap(); + + assert_eq!(state.sync_state, expected); + assert_eq!(state.sync_state.finalized_head(), finalized_before); + } + + /// Delegated heads at or above the unsafe head on the engine's chain move the unsafe and + /// local-safe heads to the local-safe head, and a cross-safe head on that chain above the safe + /// label moves it. + #[rstest::rstest] + #[case::at_unsafe_head(engine_chain_block(20))] + #[case::child_of_unsafe_head(child_of_engine_unsafe_head())] + #[tokio::test] + async fn delegated_heads_extending_the_engine_chain_move_an_advancing_safe_label( + #[case] local_safe: L2BlockInfo, + ) { + let cross_safe = engine_chain_block(17); + let mut state = engine_state_with_safe_labels(15, 15); + let expected = labels_after_reorg(&state, local_safe, cross_safe); + + run_delegated(&mut state, local_safe, cross_safe, |l2| { + l2.expect_block_header_by_number(17, ElHas::EngineChain.block(17)); + l2.expect_forkchoice_updated( + expected.create_forkchoice_state(), + PayloadStatusEnum::Valid, + ); + }) + .await + .unwrap(); + + assert_eq!(state.sync_state, expected); + } + + /// A safe label left on blocks a reorg abandoned does not prove the engine's chain: the + /// execution layer confirms the cross-safe head at the safe label's height, and the safe label + /// moves to it, keeping it an ancestor of the unsafe head. + #[tokio::test] + async fn cross_safe_head_replaces_a_stale_safe_label_at_its_height() { + let (local_safe, cross_safe) = (engine_chain_block(20), engine_chain_block(12)); + let mut state = TestEngineStateBuilder::new() + .with_unsafe_head(engine_chain_block(20)) + .with_local_safe_head(other_chain_block(12)) + .with_safe_head(other_chain_block(12)) + .with_finalized_head(engine_chain_block(5)) + .build(); + let expected = labels_after_reorg(&state, local_safe, cross_safe); + + run_delegated(&mut state, local_safe, cross_safe, |l2| { + l2.expect_block_header_by_number(12, ElHas::EngineChain.block(12)); + l2.expect_forkchoice_updated( + expected.create_forkchoice_state(), + PayloadStatusEnum::Valid, + ); + }) + .await + .unwrap(); + + assert_eq!(state.sync_state, expected); + } + + /// While the execution layer has not finished syncing, its canonical blocks may be from before + /// the sync, so the cross-safe head is not checked against them and only advances the safe + /// label. + #[tokio::test] + async fn cross_safe_head_is_not_judged_against_a_syncing_el() { + let (local_safe, cross_safe) = (engine_chain_block(20), engine_chain_block(17)); + let mut state = TestEngineStateBuilder::new() + .with_unsafe_head(engine_chain_block(20)) + .with_safe_head(engine_chain_block(15)) + .with_finalized_head(engine_chain_block(5)) + .with_el_sync_finished(false) + .build(); + let expected = labels_after_reorg(&state, local_safe, cross_safe); + + run_delegated(&mut state, local_safe, cross_safe, |l2| { + l2.expect_forkchoice_updated( + expected.create_forkchoice_state(), + PayloadStatusEnum::Valid, + ); + }) + .await + .unwrap(); + + assert_eq!(state.sync_state, expected); + } + + /// The engine chain's block at 21, the child of the engine's unsafe head at 20. + fn child_of_engine_unsafe_head() -> L2BlockInfo { + L2BlockInfo { + block_info: BlockInfo { + parent_hash: engine_chain_block(20).block_info.hash, + ..engine_chain_block(21).block_info + }, + ..Default::default() + } + } + + /// A delegated local-safe head that is the unsafe head's child extends the engine's chain, so + /// a cross-safe head below the safe label keeps it. + #[tokio::test] + async fn child_of_the_unsafe_head_keeps_the_safe_label() { + let local_safe = child_of_engine_unsafe_head(); + let mut state = engine_state_with_safe_labels(15, 15); + let expected = state.sync_state.apply_update(EngineSyncStateUpdate { + local_safe_head: Some(local_safe), + unsafe_head: Some(local_safe), + ..Default::default() + }); + + run_delegated(&mut state, local_safe, engine_chain_block(12), |l2| { + l2.expect_forkchoice_updated( + expected.create_forkchoice_state(), + PayloadStatusEnum::Valid, + ); + }) + .await + .unwrap(); + + assert_eq!(state.sync_state, expected); + } + + /// A reorg whose cross-safe head is below the finalized head moves the safe label to the + /// finalized head, which the delegate's chain shares with the engine's. + #[tokio::test] + async fn reorg_with_cross_safe_head_behind_finality_moves_safe_to_finalized() { + let local_safe = other_chain_block(20); + let mut state = engine_state_with_safe_labels(15, 15); + let expected = labels_after_reorg(&state, local_safe, state.sync_state.finalized_head()); + + run_delegated(&mut state, local_safe, other_chain_block(4), |l2| { + l2.expect_forkchoice_updated( + expected.create_forkchoice_state(), + PayloadStatusEnum::Valid, + ); + }) + .await + .unwrap(); + + assert_eq!(state.sync_state, expected); + } + + /// Delegated heads at the unsafe head on the engine's chain, as when the engine has no unsafe + /// gossip, keep the safe label for a cross-safe head at or below it, so a lagging backend + /// behind a load balancer does not move safe back. + #[rstest::rstest] + #[case::at_safe_label(15, no_lookup)] + #[case::behind_safe_label(12, no_lookup)] + #[tokio::test] + async fn delegated_heads_at_the_unsafe_head_keep_the_safe_label( + #[case] cross_safe_number: u64, + #[case] expect_cross_safe_lookup: fn(&RpcMock), + ) { + let local_safe = engine_chain_block(20); + let mut state = engine_state_with_safe_labels(15, 15); + let expected = state.sync_state.apply_update(EngineSyncStateUpdate { + local_safe_head: Some(local_safe), + unsafe_head: Some(local_safe), + ..Default::default() + }); + + run_delegated(&mut state, local_safe, engine_chain_block(cross_safe_number), |l2| { + expect_cross_safe_lookup(l2); + l2.expect_forkchoice_updated( + expected.create_forkchoice_state(), + PayloadStatusEnum::Valid, + ); + }) + .await + .unwrap(); + + assert_eq!(state.sync_state, expected); + } + + /// A cross-safe head at one of the engine's labels is not looked up. + const fn no_lookup(_: &RpcMock) {} + + /// The execution layer has the engine chain's block at 15, which the safe label alone cannot + /// prove, since it can stay on blocks a reorg abandoned. + fn cross_safe_15_on_the_engine_chain(l2: &RpcMock) { + l2.expect_block_header_by_number(15, ElHas::EngineChain.block(15)); + } + + fn expect_invalid_forkchoice_state(l2: &RpcMock) { + l2.expect_error_code( + "engine_forkchoiceUpdatedV3", + i64::from(alloy_rpc_types_engine::INVALID_FORK_CHOICE_STATE_ERROR), + ); + } + + /// A forkchoice update for delegated safe heads, whether a reorg off the engine's chain or an + /// extension of it, that the execution layer refuses as an invalid forkchoice state resets the + /// engine, which rebuilds the labels from the execution layer. + #[rstest::rstest] + #[case::reorg(other_chain_block(20))] + #[case::extension(engine_chain_block(25))] + #[tokio::test] + async fn rejected_forkchoice_for_delegated_heads_resets_the_engine( + #[case] delegated: L2BlockInfo, + ) { + let mut state = engine_state_with_safe_labels(15, 15); + let state_before = state; + + let err = run_delegated(&mut state, delegated, delegated, expect_invalid_forkchoice_state) + .await + .unwrap_err(); + + assert_eq!(err.severity(), EngineTaskErrorSeverity::Reset, "got {err:?}"); + assert_eq!(state, state_before); + } + + /// A reorg whose cross-safe head breaks finality is skipped before the execution layer is + /// asked: the cross-safe head cannot become the safe label, and the old safe label would sit + /// above the new unsafe head. + #[tokio::test] + async fn reorg_with_cross_safe_head_breaking_finality_is_skipped() { + let mut state = engine_state_with_safe_labels(15, 15); + let state_before = state; + + let err = run_delegated(&mut state, other_chain_block(20), other_chain_block(5), |_| {}) + .await + .unwrap_err(); + + assert!(matches!(err, ConsolidateTaskError::BreaksFinality(_)), "got {err:?}"); + assert_eq!(err.severity(), crate::EngineTaskErrorSeverity::Skip); + assert_eq!(state, state_before); + } + + /// While the execution layer syncs, its canonical blocks are still those from before the sync, + /// so a cross-safe head is not compared with them: it only advances the safe label, and never + /// reorgs the engine or retargets the sync. + #[rstest::rstest] + #[case::other_chain_below_safe_label(other_chain_block(12), engine_chain_block(15))] + #[case::advancing(engine_chain_block(16), engine_chain_block(16))] + #[tokio::test] + async fn el_sync_moves_the_safe_label_only_forward_without_lookups( + #[case] cross_safe: L2BlockInfo, + #[case] expected_safe: L2BlockInfo, + ) { + let local_safe = engine_chain_block(18); + let mut state = engine_state_with_safe_labels(15, 15); + let expected = labels_after(&state, local_safe, expected_safe); + + run_delegated(&mut state, local_safe, cross_safe, |l2| { + l2.expect_block_by_number(18, ElHas::Missing.block(18)); + l2.expect_forkchoice_updated( + expected.create_forkchoice_state(), + PayloadStatusEnum::Valid, + ); + }) + .await + .unwrap(); + + assert_eq!(state.sync_state, expected); + } + + /// A cross-safe head that is the engine's safe label leaves it in place without asking the + /// execution layer. One that differs from the safe label at its height is checked against the + /// execution layer, which has the engine chain's block there, so the safe label stays too. + #[rstest::rstest] + #[case::at_safe_label(engine_chain_block(15), no_lookup)] + #[case::conflicting_with_safe_label(other_chain_block(15), cross_safe_15_on_the_engine_chain)] + #[tokio::test] + async fn cross_safe_head_off_the_engine_chain_keeps_the_safe_label( + #[case] cross_safe: L2BlockInfo, + #[case] expect_cross_safe_lookup: fn(&RpcMock), + ) { + let local_safe = engine_chain_block(18); + let mut state = engine_state_with_safe_labels(15, 15); + let expected = labels_after(&state, local_safe, engine_chain_block(15)); + + run_delegated(&mut state, local_safe, cross_safe, |l2| { + l2.expect_block_by_number(18, ElHas::EngineChain.block(18)); + expect_cross_safe_lookup(l2); + l2.expect_forkchoice_updated( + expected.create_forkchoice_state(), + PayloadStatusEnum::Valid, + ); + }) + .await + .unwrap(); + + assert_eq!(state.sync_state, expected); + } + + /// A delegated local-safe head whose block the execution layer has, but whose info cannot be + /// built from it, is consolidated with the delegate's info instead of reorging the unsafe head + /// back to it. + #[tokio::test] + async fn undecodable_delegated_block_is_consolidated_without_a_reorg() { + let (block, local_safe) = chain_block(18, ENGINE_CHAIN); + let cfg = Arc::new(RollupConfig::default()); + let mut state = engine_state_with_safe_labels(15, 15); + let expected = labels_after(&state, local_safe, local_safe); + let (client, l1, l2) = test_engine_client(cfg.clone()); + l2.expect_block_by_number(18, Some(block)); + l2.expect_forkchoice_updated(expected.create_forkchoice_state(), PayloadStatusEnum::Valid); + let task = ConsolidateTask::new( + Arc::new(client), + cfg, + delegated_safe_heads(local_safe, local_safe), + Arc::new(NoopBlockSink), + ); + + task.execute(&mut state).await.unwrap(); + + assert_eq!(state.sync_state, expected); + l1.assert_finished(); + l2.assert_finished(); + } + + /// A delegated block that differs from the execution layer's is not a divergence while the + /// execution layer has not applied the engine's last forkchoice: only the safe labels move, the + /// cross-safe head only advancing, and the unsafe head stays the sync target. + #[tokio::test] + async fn delegated_block_off_a_syncing_el_moves_only_the_safe_labels() { + let (local_safe, cross_safe) = (engine_chain_block(18), engine_chain_block(17)); + let mut state = engine_state_with_safe_labels(15, 15); + state.el_applied_last_forkchoice = false; + let expected = labels_after(&state, local_safe, cross_safe); + + run_delegated(&mut state, local_safe, cross_safe, |l2| { + l2.expect_block_by_number(18, ElHas::OtherChain.block(18)); + l2.expect_forkchoice_updated( + expected.create_forkchoice_state(), + PayloadStatusEnum::Syncing, + ); + }) + .await + .unwrap(); + + assert_eq!(state.sync_state, expected); + } + + /// A delegate catching up several blocks above the unsafe head keeps the engine's safe label + /// when the execution layer holds it, rather than dropping it to a lower cross-safe head; only + /// a safe label the execution layer does not hold falls to the cross-safe head. + #[rstest::rstest] + #[case::el_holds_safe_label(ElHas::EngineChain, 15)] + #[case::el_off_safe_label(ElHas::OtherChain, 12)] + #[tokio::test] + async fn catch_up_above_the_unsafe_head_keeps_a_safe_label_the_el_holds( + #[case] el_at_safe_label: ElHas, + #[case] expected_safe: u64, + ) { + let (local_safe, cross_safe) = (engine_chain_block(23), engine_chain_block(12)); + let mut state = engine_state_with_safe_labels(15, 15); + let expected = state.sync_state.apply_update(EngineSyncStateUpdate { + local_safe_head: Some(local_safe), + safe_head: Some(engine_chain_block(expected_safe)), + unsafe_head: Some(local_safe), + ..Default::default() + }); + + run_delegated(&mut state, local_safe, cross_safe, |l2| { + l2.expect_block_header_by_number(15, el_at_safe_label.block(15)); + l2.expect_forkchoice_updated( + expected.create_forkchoice_state(), + PayloadStatusEnum::Valid, + ); + }) + .await + .unwrap(); + + assert_eq!(state.sync_state, expected); + } + + /// Labels with the unsafe head rewound below the safe labels are moved back into order by + /// delegated heads above finality instead of being kept as a lagging delegate's; the execution + /// layer is not asked about heights above the rewound unsafe head. + #[tokio::test] + async fn delegated_heads_repair_an_unsafe_head_below_the_safe_labels() { + let (local_safe, cross_safe) = (engine_chain_block(12), engine_chain_block(12)); + let mut state = TestEngineStateBuilder::new() + .with_unsafe_head(engine_chain_block(10)) + .with_safe_head(engine_chain_block(15)) + .with_finalized_head(engine_chain_block(5)) + .build(); + let expected = labels_after_reorg(&state, local_safe, cross_safe); + + run_delegated(&mut state, local_safe, cross_safe, |l2| { + l2.expect_forkchoice_updated( + expected.create_forkchoice_state(), + PayloadStatusEnum::Valid, + ); + }) + .await + .unwrap(); + + assert_eq!(state.sync_state, expected); + } } diff --git a/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/error.rs b/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/error.rs index 92eb39d2e96..3e07f755697 100644 --- a/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/error.rs +++ b/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/error.rs @@ -1,7 +1,8 @@ //! Contains error types for the [`crate::FinalizeTask`]. use crate::{ - EngineTaskError, SynchronizeTaskError, task_queue::tasks::task::EngineTaskErrorSeverity, + EngineTaskError, FinalityViolation, SynchronizeTaskError, + task_queue::tasks::task::EngineTaskErrorSeverity, }; use alloy_eips::BlockNumHash; use alloy_transport::{RpcError, TransportErrorKind}; @@ -11,14 +12,9 @@ use thiserror::Error; /// An error that occurs when running the [`crate::FinalizeTask`]. #[derive(Debug, Error)] pub enum FinalizeTaskError { - /// The target is below the finalized head, and finality never moves backward. - #[error("Finalization target {target} is behind the finalized head {finalized}")] - BehindFinalizedHead { - /// The number of the finalized head. - finalized: u64, - /// The number of the finalization target. - target: u64, - }, + /// The target would move the finalized head backward, or onto another chain. + #[error("Finalization target breaks finality: {0}")] + BreaksFinality(#[from] FinalityViolation), /// The block to finalize was not found. #[error("The block to finalize was not found: Number {0}")] BlockNotFound(u64), @@ -40,14 +36,6 @@ pub enum FinalizeTaskError { /// The number of the block the engine returned. returned: u64, }, - /// The target is a different block at the finalized head's height. - #[error("Finalization target {target:?} conflicts with the finalized head {finalized:?}")] - ConflictsWithFinalizedHead { - /// The finalized head. - finalized: BlockNumHash, - /// The finalization target. - target: BlockNumHash, - }, /// The forkchoice update call to finalize the block failed. #[error(transparent)] ForkchoiceUpdateFailed(#[from] SynchronizeTaskError), @@ -78,9 +66,8 @@ pub enum FinalizeTaskError { impl EngineTaskError for FinalizeTaskError { fn severity(&self) -> EngineTaskErrorSeverity { match self { - Self::BehindFinalizedHead { .. } | Self::BlockNotSafe { .. } | - Self::ConflictsWithFinalizedHead { .. } | + Self::BreaksFinality(_) | Self::NotCanonical { .. } | Self::NotYetAvailable { .. } => EngineTaskErrorSeverity::Skip, Self::BlockNotFound(_) | Self::BlockNumberMismatch { .. } | Self::FromBlock(_) => { diff --git a/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/task.rs b/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/task.rs index 30a08a102bd..d89e9c0c0b9 100644 --- a/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/task.rs +++ b/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/task.rs @@ -1,8 +1,8 @@ //! A task for finalizing an L2 block. use crate::{ - EngineClient, EngineState, ExecutableEngineTask, FinalizeBlockId, FinalizeTaskError, - state::EngineSyncStateUpdate, task_queue::synchronize, + EngineClient, EngineState, EngineSyncState, ExecutableEngineTask, FinalizeBlockId, + FinalizeTaskError, state::EngineSyncStateUpdate, task_queue::synchronize, }; use alloy_eips::BlockNumHash; use async_trait::async_trait; @@ -42,19 +42,14 @@ impl FinalizeTask { /// Whether `target` is the finalized head itself; a different block at its height is an error. fn is_already_finalized( - finalized_head: &L2BlockInfo, + sync_state: &EngineSyncState, target: BlockNumHash, ) -> Result { - let finalized = finalized_head.block_info.id(); - if target == finalized { + if target == sync_state.finalized_head().block_info.id() { debug!(target: "engine", ?target, "Target is already finalized"); return Ok(true); } - // The default head means finality has not been established yet, so any target, including - // the configured genesis, may initialize it. - if *finalized_head != L2BlockInfo::default() && target.number == finalized.number { - return Err(FinalizeTaskError::ConflictsWithFinalizedHead { finalized, target }); - } + sync_state.ensure_no_conflict_with_finality(target)?; Ok(false) } @@ -101,19 +96,13 @@ impl ExecutableEngineTask for FinalizeTask { type Error = FinalizeTaskError; async fn execute(&self, state: &mut EngineState) -> Result<(), FinalizeTaskError> { - let finalized_head = state.sync_state.finalized_head(); let target_number = self.block_id.number(); // A lagging source can report an older finalized block, or one from another chain. A // pinned hash is enough to detect either without asking the execution layer. - if target_number < finalized_head.block_info.number { - return Err(FinalizeTaskError::BehindFinalizedHead { - finalized: finalized_head.block_info.number, - target: target_number, - }); - } + state.sync_state.ensure_not_behind_finality(target_number)?; if let FinalizeBlockId::ByHash(target) = self.block_id && - Self::is_already_finalized(&finalized_head, target)? + Self::is_already_finalized(&state.sync_state, target)? { return Ok(()); } @@ -132,7 +121,7 @@ impl ExecutableEngineTask for FinalizeTask { // A by-number target only learns its hash from the lookup. if matches!(self.block_id, FinalizeBlockId::ByNumber(_)) && - Self::is_already_finalized(&finalized_head, block_info.block_info.id())? + Self::is_already_finalized(&state.sync_state, block_info.block_info.id())? { return Ok(()); } diff --git a/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/task_test.rs b/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/task_test.rs index 45420631095..5708a52c150 100644 --- a/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/task_test.rs +++ b/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/task_test.rs @@ -1,7 +1,7 @@ use crate::{ EngineState, EngineSyncState, EngineSyncStateUpdate, EngineTaskError, EngineTaskErrorSeverity::{self, Critical, Skip, Temporary}, - ExecutableEngineTask, FinalizeBlockId, FinalizeTask, FinalizeTaskError, + ExecutableEngineTask, FinalityViolation, FinalizeBlockId, FinalizeTask, FinalizeTaskError, test_utils::{ RpcMock, TestEngineStateBuilder, TestRpcBlock as RpcBlock, test_block_info, test_engine_client, test_l2_block, test_rpc_block, @@ -103,11 +103,20 @@ async fn finalize( } const fn is_behind(err: &FinalizeTaskError) -> bool { - matches!(err, FinalizeTaskError::BehindFinalizedHead { finalized: FINALIZED, target: STALE }) + matches!( + err, + FinalizeTaskError::BreaksFinality(FinalityViolation::BehindFinalizedHead { + finalized: BlockNumHash { number: FINALIZED, .. }, + target: STALE, + }) + ) } const fn is_conflict(err: &FinalizeTaskError) -> bool { - matches!(err, FinalizeTaskError::ConflictsWithFinalizedHead { .. }) + matches!( + err, + FinalizeTaskError::BreaksFinality(FinalityViolation::ConflictsWithFinalizedHead { .. }) + ) } const fn is_not_canonical(err: &FinalizeTaskError) -> bool { diff --git a/rust/kona/crates/node/engine/src/task_queue/tasks/mod.rs b/rust/kona/crates/node/engine/src/task_queue/tasks/mod.rs index 9eddaa76d00..12ae6c8ba16 100644 --- a/rust/kona/crates/node/engine/src/task_queue/tasks/mod.rs +++ b/rust/kona/crates/node/engine/src/task_queue/tasks/mod.rs @@ -25,7 +25,10 @@ mod canonicalize; pub use canonicalize::{CanonicalizeTask, CanonicalizeTaskError}; mod consolidate; -pub use consolidate::{ConsolidateInput, ConsolidateTask, ConsolidateTaskError}; +pub use consolidate::{ + ConsolidateInput, ConsolidateTask, ConsolidateTaskError, DelegatedSafeHeads, + InconsistentSafeHeadsError, +}; mod finalize; pub use finalize::{FinalizeBlockId, FinalizeTask, FinalizeTaskError}; diff --git a/rust/kona/crates/node/engine/src/task_queue/tasks/synchronize/helper.rs b/rust/kona/crates/node/engine/src/task_queue/tasks/synchronize/helper.rs index a9bc15146ff..fc28ccf0026 100644 --- a/rust/kona/crates/node/engine/src/task_queue/tasks/synchronize/helper.rs +++ b/rust/kona/crates/node/engine/src/task_queue/tasks/synchronize/helper.rs @@ -100,6 +100,7 @@ async fn send_forkchoice( match &valid_response.payload_status.status { PayloadStatusEnum::Valid => { + state.el_applied_last_forkchoice = true; if !state.el_sync_finished { info!(target: "engine", "Finished execution layer sync."); state.el_sync_finished = true; @@ -107,6 +108,7 @@ async fn send_forkchoice( } // If we're not building a new payload, we're driving EL sync. PayloadStatusEnum::Syncing => { + state.el_applied_last_forkchoice = false; debug!(target: "engine", "Attempting to update forkchoice state while EL syncing") } // Other codes are not expected. diff --git a/rust/kona/crates/node/engine/src/test_utils/engine_state.rs b/rust/kona/crates/node/engine/src/test_utils/engine_state.rs index 60542823b96..a1399be192e 100644 --- a/rust/kona/crates/node/engine/src/test_utils/engine_state.rs +++ b/rust/kona/crates/node/engine/src/test_utils/engine_state.rs @@ -3,13 +3,17 @@ use alloy_eips::BlockNumHash; use alloy_primitives::{B256, b256}; use kona_protocol::{BlockInfo, L2BlockInfo}; -/// Builder for creating test `EngineState` instances with sensible defaults +/// Builder for creating test `EngineState` instances with sensible defaults. +/// +/// An unset local-safe head equals the safe head, as on every chain before interop; tests of the +/// local-safe and safe labels apart set it with [`Self::with_local_safe_head`]. #[derive(Debug)] pub struct TestEngineStateBuilder { unsafe_head: L2BlockInfo, local_safe_head: Option, safe_head: Option, finalized_head: Option, + el_applied_last_forkchoice: bool, el_sync_finished: bool, } @@ -33,6 +37,7 @@ impl TestEngineStateBuilder { local_safe_head: None, safe_head: None, finalized_head: None, + el_applied_last_forkchoice: true, el_sync_finished: true, } } @@ -43,6 +48,12 @@ impl TestEngineStateBuilder { self } + /// Sets the local-safe head, which otherwise follows the safe head + pub const fn with_local_safe_head(mut self, block: L2BlockInfo) -> Self { + self.local_safe_head = Some(block); + self + } + /// Sets the safe head pub const fn with_safe_head(mut self, block: L2BlockInfo) -> Self { self.safe_head = Some(block); @@ -55,13 +66,22 @@ impl TestEngineStateBuilder { self } - /// Sets whether EL sync is finished + /// Sets whether EL sync is finished. An execution layer still syncing has not applied the last + /// forkchoice either, so this sets both flags. #[allow(dead_code)] pub const fn with_el_sync_finished(mut self, finished: bool) -> Self { + self.el_applied_last_forkchoice = finished; self.el_sync_finished = finished; self } + /// Sets whether the execution layer applied the last forkchoice update. + #[allow(dead_code)] + pub const fn with_el_applied_last_forkchoice(mut self, applied: bool) -> Self { + self.el_applied_last_forkchoice = applied; + self + } + /// Builds the `EngineState` pub fn build(self) -> EngineState { let mut state = EngineState::default(); @@ -69,11 +89,14 @@ impl TestEngineStateBuilder { // Set unsafe head (required) state.sync_state = state.sync_state.apply_update(EngineSyncStateUpdate { unsafe_head: Some(self.unsafe_head), - local_safe_head: Some(self.local_safe_head.unwrap_or(self.unsafe_head)), + local_safe_head: Some( + self.local_safe_head.or(self.safe_head).unwrap_or(self.unsafe_head), + ), safe_head: Some(self.safe_head.unwrap_or(self.unsafe_head)), finalized_head: Some(self.finalized_head.unwrap_or(self.unsafe_head)), }); + state.el_applied_last_forkchoice = self.el_applied_last_forkchoice; state.el_sync_finished = self.el_sync_finished; state } diff --git a/rust/kona/crates/node/engine/src/test_utils/misc.rs b/rust/kona/crates/node/engine/src/test_utils/misc.rs index fd77bf7ccf4..c7675444b0d 100644 --- a/rust/kona/crates/node/engine/src/test_utils/misc.rs +++ b/rust/kona/crates/node/engine/src/test_utils/misc.rs @@ -1,3 +1,4 @@ +use crate::{ConsolidateInput, DelegatedSafeHeads}; use alloy_eips::BlockNumHash; use alloy_primitives::B256; use kona_protocol::{BlockInfo, L2BlockInfo}; @@ -15,3 +16,14 @@ pub fn test_block_info(number: u64) -> L2BlockInfo { seq_num: 0, } } + +/// Delegated safe heads as a consolidation input. Pass the same block twice for the heads of a +/// delegate before interop. +pub fn delegated_safe_heads( + local_safe_head: L2BlockInfo, + safe_head: L2BlockInfo, +) -> ConsolidateInput { + DelegatedSafeHeads::try_new(local_safe_head, safe_head) + .expect("the cross-safe head must be at or below the local-safe head, on its chain") + .into() +} diff --git a/rust/kona/crates/node/engine/src/test_utils/mod.rs b/rust/kona/crates/node/engine/src/test_utils/mod.rs index 89d57ac5892..bfde4784cac 100644 --- a/rust/kona/crates/node/engine/src/test_utils/mod.rs +++ b/rust/kona/crates/node/engine/src/test_utils/mod.rs @@ -14,4 +14,4 @@ mod engine_state; pub use engine_state::TestEngineStateBuilder; mod misc; -pub use misc::test_block_info; +pub use misc::{delegated_safe_heads, test_block_info}; diff --git a/rust/kona/crates/node/engine/src/test_utils/rpc.rs b/rust/kona/crates/node/engine/src/test_utils/rpc.rs index 0bd0bfef0c6..cda70c224a6 100644 --- a/rust/kona/crates/node/engine/src/test_utils/rpc.rs +++ b/rust/kona/crates/node/engine/src/test_utils/rpc.rs @@ -70,6 +70,16 @@ impl RpcMock { ); } + /// Expects a header-only `eth_getBlockByNumber` call for `number` and answers it with `block`, + /// or with null for `None`. + pub fn expect_block_header_by_number(&self, number: u64, block: Option) { + self.expect_params( + "eth_getBlockByNumber", + serde_json::json!([format!("{number:#x}"), false]), + block, + ); + } + /// Expects an `engine_forkchoiceUpdatedV3` call for `forkchoice` without payload attributes and /// answers it with `status`. pub fn expect_forkchoice_updated( diff --git a/rust/kona/crates/node/service/src/actors/derivation/delegated/actor.rs b/rust/kona/crates/node/service/src/actors/derivation/delegated/actor.rs index 480019925a0..b3d64c7202b 100644 --- a/rust/kona/crates/node/service/src/actors/derivation/delegated/actor.rs +++ b/rust/kona/crates/node/service/src/actors/derivation/delegated/actor.rs @@ -6,7 +6,9 @@ use alloy_primitives::BlockHash; use async_trait::async_trait; use jsonrpsee::core::ClientError; use kona_derive::{ChainProvider, Signal}; -use kona_engine::{EngineState, FinalizeBlockId}; +use kona_engine::{ + DelegatedSafeHeads, EngineState, FinalityViolation, FinalizeBlockId, InconsistentSafeHeadsError, +}; use kona_protocol::{BlockInfo, L2BlockInfo, SyncStatus}; use kona_rpc::RollupNodeApiClient; use thiserror::Error; @@ -48,6 +50,14 @@ impl SyncStatusSource for C { /// /// Once validated, the actor sends the derived safe and finalized L2 info /// to the [`NodeActor`] responsible for the execution sub-routine. +/// +/// The delegate's local-safe and cross-safe heads are imported +/// separately: the local-safe head is the block the execution layer must have, or sync to, and +/// sets the engine's local-safe label; the cross-safe head sets the safe label that forkchoice +/// updates carry, and finalization never goes above it. Before interop both are the same block. +/// Under interop the delegate must verify cross-chain dependencies, for example op-node with a +/// supervisor, since this node checks none itself: its safe label is only as safe as the +/// delegate's cross-safe head. #[derive(Debug)] pub struct DelegateDerivationActor where @@ -77,6 +87,15 @@ where has_engine_sync_completed: bool, /// Ticker driving periodic polls of the derivation delegate. delegated_derivation_ticker: time::Interval, + + /// Safe heads behind the engine's that were forwarded while the execution layer applied the + /// engine's forkchoice, with the engine's local-safe and safe heads at the time. The engine + /// checks such heads against its execution layer once; until either side changes, sending + /// them again repeats that lookup to the same answer. + forwarded_lagging_safe_heads: Option<(DelegatedSafeHeads, BlockNumHash, BlockNumHash)>, + /// The last rejection of a delegate status, so a delegate stuck on one inconsistency is + /// warned about once rather than on every poll. + last_rejection: Option, } impl @@ -107,6 +126,8 @@ where engine_l2_safe_head: L2BlockInfo::default(), has_engine_sync_completed: false, delegated_derivation_ticker, + forwarded_lagging_safe_heads: None, + last_rejection: None, } } } @@ -175,60 +196,51 @@ where Ok(()) } - /// The id of an established finalized head. The default head means finality has not been - /// established yet, so any finalized block from the delegate may initialize it. - fn established_finalized_l2(finalized_head: L2BlockInfo) -> Option { - (finalized_head != L2BlockInfo::default()).then(|| finalized_head.block_info.id()) + /// The delegate's local-safe head, or its safe head when it reports a zero local-safe head: + /// `SyncStatus` zeroes a label the delegate has not derived, and a zero head carries nothing. + fn reported_local_safe_head(status: &SyncStatus) -> L2BlockInfo { + if status.local_safe_l2 == L2BlockInfo::default() { + status.safe_l2 + } else { + status.local_safe_l2 + } } - /// Rejects a finalized L2 block behind, or at the same height as but different from, the - /// engine's. Either means the delegate lags or follows another chain, so none of its status - /// applies: its safe head would move the engine back or onto that chain too. + /// Rejects a status whose labels are out of order: its finalized block ahead of its safe head, + /// or its safe head ahead of its local safe head or a different block at its height. No + /// consistent delegate reports any of these, and a finalized block ahead of the safe head would + /// let a stale safe head sit below finality. Returns its safe heads otherwise. + fn validate_label_order( + status: &SyncStatus, + ) -> Result { + let finalized = status.finalized_l2.block_info.number; + let safe = status.safe_l2.block_info.number; + if finalized > safe { + return Err(DerivationDelegationError::FinalizedAheadOfSafe { finalized, safe }); + } + Ok(DelegatedSafeHeads::try_new(Self::reported_local_safe_head(status), status.safe_l2)?) + } + + /// Verifies that the labels reported by the derivation delegate are in order, that its + /// finalized L2 block does not move back from the engine's, and that its L1 info is consistent + /// with the canonical L1 chain. Returns its safe heads when it passes. /// /// Only a fork at the engine's finalized height is caught here. A chain that forked below it /// and reports a higher finalized block passes, and the execution layer rejects the forkchoice /// update that would move onto it. - fn validate_finalized_l2_progress( - &self, - reported: BlockNumHash, - ) -> Result<(), DerivationDelegationError> { - let engine_finalized_head = self.engine_state_rx.borrow().sync_state.finalized_head(); - let Some(engine) = Self::established_finalized_l2(engine_finalized_head) else { - return Ok(()); - }; - if reported.number < engine.number { - return Err(DerivationDelegationError::FinalizedL2Regressed { engine, reported }); - } - if reported.number == engine.number && reported.hash != engine.hash { - return Err(DerivationDelegationError::FinalizedL2Conflict { engine, reported }); - } - Ok(()) - } - - /// Verifies that the derivation delegate's finalized L2 block is not above its safe L2 block - /// and does not move back from the engine's, and that its L1 info is consistent with the - /// canonical L1 chain. - async fn validate_sync_status(&mut self, v: &SyncStatus) -> bool { - // A finalized block above the safe head is an inconsistent snapshot; the engine would only - // refuse to finalize it. - if v.finalized_l2.block_info.number > v.safe_l2.block_info.number { - let err = DerivationDelegationError::FinalizedAheadOfSafe { - finalized: v.finalized_l2.block_info.number, - safe: v.safe_l2.block_info.number, - }; - warn!(target: "derivation", error = %err, "Inconsistent sync status from delegate"); - return false; - } - if let Err(err) = self.validate_finalized_l2_progress(v.finalized_l2.block_info.id()) { - warn!( - target: "derivation", - error = %err, - "Finalized L2 inconsistency detected at sync status from delegate" - ); - return false; - } + async fn validate_sync_status( + &mut self, + v: &SyncStatus, + ) -> Result { + let safe_heads = Self::validate_label_order(v)?; + self.engine_state_rx + .borrow() + .sync_state + .ensure_respects_finality(v.finalized_l2.block_info.id())?; + let local_safe = safe_heads.local_safe_head(); let checks = [ + ("L1 Origin of Local Safe L2", local_safe.l1_origin.number, local_safe.l1_origin.hash), ("L1 Origin of Safe L2", v.safe_l2.l1_origin.number, v.safe_l2.l1_origin.hash), ( "L1 Origin of Finalized L2", @@ -237,18 +249,47 @@ where ), ("Current L1", v.current_l1.number, v.current_l1.hash), ]; + let mut validated_l1_blocks = Vec::with_capacity(checks.len()); for (context, number, hash) in checks { - if let Err(err) = self.validate_l1_block(context, number, hash).await { - warn!( - target: "derivation", - context = context, - error = %err, - "L1 inconsistency detected at sync status from delegate" - ); - return false; + // Labels often share an L1 origin, and one lookup proves it canonical for all of them. + if validated_l1_blocks.contains(&(number, hash)) { + continue; } + self.validate_l1_block(context, number, hash).await?; + validated_l1_blocks.push((number, hash)); + } + Ok(safe_heads) + } + + /// Logs a rejected delegate status at `warn!` when the reason changed, and at `debug!` while + /// the delegate keeps reporting the same inconsistency. + fn log_rejected_status(&mut self, err: &DerivationDelegationError) { + let reason = err.to_string(); + if self.last_rejection.as_ref() == Some(&reason) { + debug!(target: "derivation", error = %err, "Sync status from delegate rejected again"); + } else { + warn!(target: "derivation", error = %err, "Rejected sync status from delegate"); + self.last_rejection = Some(reason); } - true + } + + /// What to remember for safe heads behind the engine's consistent local-safe head while its + /// execution layer applied the engine's forkchoice, or `None` for any other heads, which the + /// engine must always see. + fn lagging_safe_heads_to_remember( + &self, + safe_heads: DelegatedSafeHeads, + ) -> Option<(DelegatedSafeHeads, BlockNumHash, BlockNumHash)> { + let engine_state = self.engine_state_rx.borrow(); + let labels = &engine_state.sync_state; + let engine_local_safe = labels.local_safe_head().block_info; + let engine_safe = labels.safe_head().block_info; + let are_labels_consistent = labels.unsafe_head().block_info.number >= engine_safe.number && + labels.unsafe_head().block_info.number >= engine_local_safe.number; + (engine_state.el_applied_last_forkchoice && + are_labels_consistent && + safe_heads.local_safe_head().block_info.number < engine_local_safe.number) + .then(|| (safe_heads, engine_local_safe.id(), engine_safe.id())) } /// Fetches, validates, and applies sync status from the derivation delegate. @@ -265,18 +306,33 @@ where } }; - if !self.validate_sync_status(&sync_status).await { - // Validation failures here are expected to be transient, so we skip processing - // this sync status and continue delegating derivation instead of treating it as - // fatal. - return Ok(()); - } + let safe_heads = match self.validate_sync_status(&sync_status).await { + Ok(safe_heads) => safe_heads, + Err(err) => { + // Validation failures here are expected to be transient, so we skip processing + // this sync status and continue delegating derivation instead of treating it as + // fatal. + self.log_rejected_status(&err); + return Ok(()); + } + }; + self.last_rejection = None; self.derivation_origin_tx.send_replace(Some(sync_status.current_l1)); - self.engine_client - .send_safe_l2_signal(sync_status.safe_l2.into()) - .await - .map_err(|e| DerivationError::Sender(Box::new(e)))?; + let lagging_safe_heads = self.lagging_safe_heads_to_remember(safe_heads); + if lagging_safe_heads.is_some() && lagging_safe_heads == self.forwarded_lagging_safe_heads { + debug!( + target: "derivation", + ?safe_heads, + "Not forwarding lagging safe heads the engine already checked" + ); + } else { + self.forwarded_lagging_safe_heads = lagging_safe_heads; + self.engine_client + .send_safe_l2_signal(safe_heads.into()) + .await + .map_err(|e| DerivationError::Sender(Box::new(e)))?; + } // Delegated polling supplies `(number, hash)`. Carry the hash through so the engine // finalizes the specific block we were asked to, not whatever it happens to have at the @@ -290,6 +346,7 @@ where debug!( target: "derivation", + local_safe_l2 = ?sync_status.local_safe_l2, safe_l2 = ?sync_status.safe_l2, finalized_l2 = ?sync_status.finalized_l2, "Processed sync status from delegate" @@ -329,17 +386,18 @@ where #[derive(Error, Debug)] enum DerivationDelegationError { - /// The delegate's finalized L2 block is above its safe L2 block. + /// The delegate's finalized L2 block is ahead of its own safe head. #[error("Finalized L2 {finalized} is ahead of safe L2 {safe}")] FinalizedAheadOfSafe { finalized: u64, safe: u64 }, - /// The delegate's finalized L2 block is a different block at the engine's finalized height. - #[error("Finalized L2 {reported:?} conflicts with the engine's finalized L2 {engine:?}")] - FinalizedL2Conflict { engine: BlockNumHash, reported: BlockNumHash }, + /// The delegate's finalized L2 block is behind the engine's, or a different block at its + /// height. + #[error("Finalized L2 moves the engine's finality back: {0}")] + FinalizedL2BreaksFinality(#[from] FinalityViolation), - /// The delegate's finalized L2 block is behind the engine's. - #[error("Finalized L2 {reported:?} is behind the engine's finalized L2 {engine:?}")] - FinalizedL2Regressed { engine: BlockNumHash, reported: BlockNumHash }, + /// The delegate's safe heads are inconsistent with each other. + #[error(transparent)] + InconsistentSafeHeads(#[from] InconsistentSafeHeadsError), /// The L1 provider returned an error (network, RPC, etc.) #[error("L1 provider error: {0}")] diff --git a/rust/kona/crates/node/service/src/actors/derivation/delegated/mod.rs b/rust/kona/crates/node/service/src/actors/derivation/delegated/mod.rs index c965fa802f6..061edc10154 100644 --- a/rust/kona/crates/node/service/src/actors/derivation/delegated/mod.rs +++ b/rust/kona/crates/node/service/src/actors/derivation/delegated/mod.rs @@ -1,5 +1,11 @@ mod actor; pub use actor::{DelegateDerivationActor, SyncStatusSource}; +#[cfg(test)] +mod scenarios; + +#[cfg(test)] +mod test_utils; + #[cfg(test)] mod tests; diff --git a/rust/kona/crates/node/service/src/actors/derivation/delegated/scenarios.rs b/rust/kona/crates/node/service/src/actors/derivation/delegated/scenarios.rs new file mode 100644 index 00000000000..9cc35c2835f --- /dev/null +++ b/rust/kona/crates/node/service/src/actors/derivation/delegated/scenarios.rs @@ -0,0 +1,647 @@ +//! Delegated derivation end to end: the real actors against a scripted delegate and a strict EL. + +use super::{ + DelegateDerivationActor, + actor::DERIVATION_DELEGATE_POLL_INTERVAL as POLL_INTERVAL, + test_utils::{ScriptedDelegate, sync_status}, +}; +use crate::{ + EngineActor, EngineActorRequest, NodeActor, QueuedDerivationEngineClient, + QueuedEngineDerivationClient, ResetRequest, + actors::engine::{INITIAL_RETRY_DELAY, MAX_RETRY_DELAY}, +}; +use alloy_eips::BlockNumHash; +use alloy_primitives::B256; +use alloy_rpc_types_engine::PayloadStatusEnum; +use kona_derive::test_utils::TestChainProvider; +use kona_engine::{ + Engine, EngineState, EngineSyncState, EngineSyncStateUpdate, NoopBlockSink, + test_utils::{ + RpcMock, TestEngineStateBuilder, expect_sync_start, test_engine_client, test_l2_block, + test_rpc_block, + }, +}; +use kona_genesis::RollupConfig; +use kona_protocol::{BlockInfo, L2BlockInfo, SyncStatus}; +use op_alloy_consensus::OpBlock; +use std::sync::Arc; +use tokio::{ + sync::{mpsc, watch}, + task::{JoinError, JoinHandle}, + time::{self, Duration, Instant}, +}; + +/// How long the engine may take to reach the labels a step expects. +const LABELS_DEADLINE: Duration = Duration::from_secs(1); + +/// Long enough for any task the engine left queued to retry with an unexpected request. +const QUIET_TIME: Duration = MAX_RETRY_DELAY.saturating_add(INITIAL_RETRY_DELAY); + +/// Height of the canonical chain the scenarios draw blocks from. +const CHAIN_LENGTH: u64 = 40; + +/// The L1 origin every test L2 block reports, which the delegate actor checks is canonical. +const L1_ORIGIN: BlockInfo = + BlockInfo { hash: B256::ZERO, number: 0, parent_hash: B256::ZERO, timestamp: 0 }; + +/// The canonical L1 block a delegate reports as its `current_l1` when its safe head is at +/// `number`, so each accepted poll publishes a distinct derivation origin. +fn l1_block(number: u64) -> BlockInfo { + if number == 0 { + return L1_ORIGIN; + } + BlockInfo { hash: B256::with_last_byte(number as u8), number, ..Default::default() } +} + +/// A canonical L2 chain of decodable blocks, linked by parent hash from the configured genesis. +struct TestChain { + blocks: Vec, + config: Arc, + infos: Vec, +} + +impl TestChain { + fn new() -> Self { + let genesis = OpBlock::default(); + let mut config = RollupConfig::default(); + config.genesis.l2 = BlockNumHash { hash: genesis.header.hash_slow(), number: 0 }; + let mut blocks = vec![genesis]; + for number in 1..=CHAIN_LENGTH { + let parent_hash = blocks[number as usize - 1].header.hash_slow(); + blocks.push(test_l2_block(number, parent_hash, 0)); + } + let infos = blocks.iter().map(|block| l2_block_info(block, &config)).collect(); + Self { blocks, config: Arc::new(config), infos } + } + + fn block(&self, number: u64) -> L2BlockInfo { + self.infos[number as usize] + } + + /// A block at `number` on a fork that leaves the canonical chain right below it. + fn fork_block(&self, number: u64) -> L2BlockInfo { + let parent_hash = self.block(number - 1).block_info.hash; + l2_block_info(&test_l2_block(number, parent_hash, 1), &self.config) + } + + fn el_serves(&self, el: &RpcMock, number: u64) { + el.expect_block_by_number(number, Some(test_rpc_block(&self.blocks[number as usize]))); + } + + fn el_serves_header(&self, el: &RpcMock, number: u64) { + el.expect_block_header_by_number( + number, + Some(test_rpc_block(&self.blocks[number as usize])), + ); + } + + fn el_lacks(&self, el: &RpcMock, number: u64) { + el.expect_block_by_number(number, None); + } + + /// Expects the reset's sync start to find the execution layer's labels all at genesis. + fn el_reset_to_genesis(&self, l1: &RpcMock, el: &RpcMock) { + let genesis = self.block(0); + expect_sync_start( + l1, + el, + &test_rpc_block(&self.blocks[0]), + all_at(genesis).sync_state.create_forkchoice_state(), + ); + } +} + +fn l2_block_info(block: &OpBlock, config: &RollupConfig) -> L2BlockInfo { + L2BlockInfo::from_block_and_genesis(block, &config.genesis).unwrap() +} + +/// A synced engine whose local-safe and safe heads are the same block, as before interop. +fn synced_engine( + finalized: L2BlockInfo, + safe: L2BlockInfo, + unsafe_head: L2BlockInfo, +) -> EngineState { + TestEngineStateBuilder::new() + .with_finalized_head(finalized) + .with_safe_head(safe) + .with_unsafe_head(unsafe_head) + .build() +} + +fn all_at(block: L2BlockInfo) -> EngineState { + synced_engine(block, block, block) +} + +/// The label moves the scenarios expect of the engine. +trait LabelMoves { + fn with_finalized(self, finalized: L2BlockInfo) -> Self; + + /// Local-safe and safe both at `safe`, as before interop. + fn with_safe(self, safe: L2BlockInfo) -> Self; + + fn with_safe_heads(self, local_safe: L2BlockInfo, safe: L2BlockInfo) -> Self; + + /// The unsafe head moved, forward or back, to a delegated safe head. + fn with_safe_and_unsafe(self, safe: L2BlockInfo) -> Self; +} + +impl LabelMoves for EngineSyncState { + fn with_finalized(self, finalized: L2BlockInfo) -> Self { + self.apply_update(EngineSyncStateUpdate { + finalized_head: Some(finalized), + ..Default::default() + }) + } + + fn with_safe(self, safe: L2BlockInfo) -> Self { + self.with_safe_heads(safe, safe) + } + + fn with_safe_heads(self, local_safe: L2BlockInfo, safe: L2BlockInfo) -> Self { + self.apply_update(EngineSyncStateUpdate { + local_safe_head: Some(local_safe), + safe_head: Some(safe), + ..Default::default() + }) + } + + fn with_safe_and_unsafe(self, safe: L2BlockInfo) -> Self { + self.with_safe(safe) + .apply_update(EngineSyncStateUpdate { unsafe_head: Some(safe), ..Default::default() }) + } +} + +fn pre_interop_status(safe: L2BlockInfo, finalized: L2BlockInfo) -> SyncStatus { + sync_status(l1_block(safe.block_info.number), safe, safe, finalized) +} + +/// A delegated kona-node whose delegate and engine actors run as tasks, as in production. +struct DelegatedNode { + answered_polls: watch::Receiver, + answered_polls_seen: u32, + delegate: ScriptedDelegate, + delegate_task: JoinHandle<()>, + derivation_origin: watch::Receiver>, + el: RpcMock, + engine_requests: mpsc::Sender, + engine_state: watch::Receiver, + engine_task: JoinHandle<()>, + l1: RpcMock, + polls: watch::Receiver, + polls_checked: u32, + polls_checked_at: Instant, +} + +impl DelegatedNode { + /// Starts the node at `initial`; the delegate first polls once the engine reports sync + /// complete. + fn start(chain: &TestChain, initial: EngineState) -> Self { + let (client, l1, el) = test_engine_client(chain.config.clone()); + let (derivation_request_tx, derivation_request_rx) = mpsc::channel(32); + let (engine_requests, engine_request_rx) = mpsc::channel(32); + let (engine_state_tx, engine_state) = watch::channel(initial); + let engine = EngineActor::new( + Arc::new(client), + chain.config.clone(), + QueuedEngineDerivationClient { derivation_actor_request_tx: derivation_request_tx }, + Engine::new(initial, engine_state_tx), + None, + engine_request_rx, + Arc::new(NoopBlockSink), + ); + + let mut canonical_l1 = TestChainProvider::default(); + for number in 0..=CHAIN_LENGTH { + canonical_l1.insert_block(number, l1_block(number)); + } + let delegate = ScriptedDelegate::lenient(); + let (derivation_origin_tx, derivation_origin) = watch::channel(None); + let delegate_actor = DelegateDerivationActor::new( + QueuedDerivationEngineClient::new(engine_requests.clone()), + derivation_request_rx, + delegate.clone(), + canonical_l1, + derivation_origin_tx, + engine_state.clone(), + ); + + Self { + answered_polls: delegate.answered_polls(), + answered_polls_seen: 0, + delegate_task: run(delegate_actor, "delegate"), + derivation_origin, + el, + engine_requests, + engine_state, + engine_task: run(engine, "engine"), + l1, + polls: delegate.polls(), + polls_checked: 0, + polls_checked_at: Instant::now(), + delegate, + } + } + + /// Scripts the next poll's reply: `safe` as both the local-safe and the safe head. + fn delegate_reports(&self, safe: L2BlockInfo, finalized: L2BlockInfo) { + self.delegate.reports(pre_interop_status(safe, finalized)); + } + + /// Requests an engine reset, which runs on the engine's next step. + async fn request_reset(&self) { + let (result_tx, _result_rx) = mpsc::channel(1); + self.engine_requests + .send(EngineActorRequest::Reset(Box::new(ResetRequest { result_tx }))) + .await + .unwrap(); + } + + /// Waits for the delegate to answer the scripted poll. + async fn next_poll(&mut self) { + if time::timeout(POLL_INTERVAL * 2, self.answered_polls.changed()).await.is_err() { + self.assert_running().await; + panic!("the delegate did not answer a poll within {:?}", POLL_INTERVAL * 2); + } + let answered = *self.answered_polls.borrow_and_update(); + assert_eq!(answered, self.answered_polls_seen + 1, "the scenario missed a delegate poll"); + self.answered_polls_seen = answered; + self.assert_poll_cadence(); + } + + /// Fails if the delegate was polled more often than once per poll interval since the last + /// check. Unscripted polls fail and change nothing, so only their count can catch an actor that + /// polls too often. + fn assert_poll_cadence(&mut self) { + let polls = *self.polls.borrow(); + let elapsed = Instant::now() - self.polls_checked_at; + let allowed = elapsed.as_nanos() / POLL_INTERVAL.as_nanos() + 1; + let polled = u128::from(polls - self.polls_checked); + assert!( + polled <= allowed, + "the delegate was polled {polled} times in {elapsed:?}, at most {allowed} expected" + ); + self.polls_checked = polls; + self.polls_checked_at = Instant::now(); + } + + /// Waits `duration`, failing if the delegate answers a poll meanwhile and its requests mix with + /// these. Polls with no reply scripted fail and change nothing. + async fn wait_without_answered_poll(&mut self, duration: Duration) { + time::sleep(duration).await; + assert_eq!( + *self.answered_polls.borrow(), + self.answered_polls_seen, + "the delegate answered a poll during a {duration:?} wait" + ); + self.assert_poll_cadence(); + self.assert_running().await; + } + + /// Fails with every stopped actor's panic, so a secondary failure cannot hide the first. + async fn assert_running(&mut self) { + let mut stopped = Vec::new(); + for (name, task) in + [("engine", &mut self.engine_task), ("delegate", &mut self.delegate_task)] + { + if task.is_finished() { + stopped.push(format!("{name}: {}", stop_reason(task.await))); + } + } + assert!(stopped.is_empty(), "actors stopped:\n{}", stopped.join("\n")); + } + + /// Waits for the engine to reach `expected`, then asserts it, and every scripted reply and + /// request consumed, once any task the engine left queued has had time to retry. + async fn assert_labels(&mut self, expected: EngineSyncState) { + let reached = time::timeout( + LABELS_DEADLINE, + self.engine_state.wait_for(|state| state.sync_state == expected), + ) + .await + .is_ok_and(|watched| watched.is_ok()); + if !reached { + self.assert_running().await; + panic!( + "the engine did not reach the expected labels within {LABELS_DEADLINE:?}: {:?}", + self.engine_state.borrow().sync_state + ); + } + self.wait_without_answered_poll(QUIET_TIME).await; + self.delegate.assert_finished(); + self.l1.assert_finished(); + self.el.assert_finished(); + assert_eq!(self.engine_state.borrow().sync_state, expected); + } + + /// Asserts the derivation origin the delegate actor last published. + fn assert_origin(&self, expected: Option) { + assert_eq!(*self.derivation_origin.borrow(), expected, "published derivation origin"); + } + + /// Polls canonical `safe` and `finalized`, each applied by a forkchoice update; returns the + /// labels. A safe head below the unsafe head is looked up first; one at it extends the chain. + async fn poll_applies_safe_and_finalized( + &mut self, + chain: &TestChain, + labels: EngineSyncState, + safe: u64, + finalized: u64, + ) -> EngineSyncState { + self.delegate_reports(chain.block(safe), chain.block(finalized)); + if safe < labels.unsafe_head().block_info.number { + chain.el_serves(&self.el, safe); + } + let labels = labels.with_safe(chain.block(safe)); + self.expect_forkchoice(labels, PayloadStatusEnum::Valid); + chain.el_serves(&self.el, finalized); + let labels = labels.with_finalized(chain.block(finalized)); + self.expect_forkchoice(labels, PayloadStatusEnum::Valid); + self.next_poll().await; + self.assert_labels(labels).await; + self.assert_origin(Some(l1_block(safe))); + labels + } + + fn expect_forkchoice(&self, labels: EngineSyncState, status: PayloadStatusEnum) { + self.el.expect_forkchoice_updated(labels.create_forkchoice_state(), status); + } +} + +impl Drop for DelegatedNode { + /// Stops both actors, also when a scenario panics, so their later failures cannot interleave + /// with its message. + fn drop(&mut self) { + self.engine_task.abort(); + self.delegate_task.abort(); + } +} + +/// Why an actor task stopped: its panic message, or the join error. +fn stop_reason(result: Result<(), JoinError>) -> String { + match result { + Ok(()) => "returned".to_string(), + Err(err) if err.is_panic() => { + let payload = err.into_panic(); + payload + .downcast_ref::() + .cloned() + .or_else(|| payload.downcast_ref::<&str>().map(|message| (*message).to_string())) + .unwrap_or_else(|| "a panic without a message".to_string()) + } + Err(err) => err.to_string(), + } +} + +/// Steps `actor` until it fails, which ends the task with a panic carrying the error. +fn run(mut actor: Actor, name: &'static str) -> JoinHandle<()> +where + Actor: NodeActor + Send + 'static, + Actor::Error: std::fmt::Debug, +{ + tokio::spawn(async move { + loop { + if let Err(err) = actor.step().await { + panic!("{name} actor stopped: {err:?}"); + } + } + }) +} + +/// With no unsafe blocks, safe and finalized follow the delegate by hash and match the last FCU. +#[tokio::test(start_paused = true)] +async fn safe_and_finalized_progress_by_hash_with_unsafe_production_paused() { + let chain = TestChain::new(); + let initial = synced_engine(chain.block(5), chain.block(10), chain.block(30)); + let mut node = DelegatedNode::start(&chain, initial); + + let mut labels = initial.sync_state; + for (safe, finalized) in [(20, 10), (25, 20), (30, 25)] { + labels = node.poll_applies_safe_and_finalized(&chain, labels, safe, finalized).await; + } +} + +/// A delegated safe head on another branch reorgs the EL onto it: the EL syncs the branch while +/// the shared ancestor finalizes, then validates it on the next poll. +#[tokio::test(start_paused = true)] +async fn delegated_safe_head_on_another_branch_reorgs_the_el_onto_it() { + let chain = TestChain::new(); + let initial = synced_engine(chain.block(5), chain.block(10), chain.block(30)); + let (fork_safe, shared_finalized) = (chain.fork_block(25), chain.block(20)); + let reorged = initial.sync_state.with_safe_and_unsafe(fork_safe); + let finalized_on_branch = reorged.with_finalized(shared_finalized); + let mut node = DelegatedNode::start(&chain, initial); + + // The EL still has the canonical block at 25, which proves the divergence, and syncs the + // branch. + node.delegate_reports(fork_safe, shared_finalized); + chain.el_serves(&node.el, fork_safe.block_info.number); + node.expect_forkchoice(reorged, PayloadStatusEnum::Syncing); + chain.el_serves(&node.el, shared_finalized.block_info.number); + node.expect_forkchoice(finalized_on_branch, PayloadStatusEnum::Syncing); + node.next_poll().await; + node.assert_labels(finalized_on_branch).await; + + // Once the EL has the branch, the same target extends the engine's chain and is validated. + node.delegate_reports(fork_safe, shared_finalized); + node.expect_forkchoice(finalized_on_branch, PayloadStatusEnum::Valid); + node.next_poll().await; + node.assert_labels(finalized_on_branch).await; +} + +/// An unchanged target is resent through `SYNCING` until `VALID`, then finalizes once. +#[tokio::test(start_paused = true)] +async fn source_ahead_catch_up_retries_unchanged_target_through_syncing_until_valid() { + let chain = TestChain::new(); + let initial = synced_engine(chain.block(5), chain.block(10), chain.block(10)); + let (target_safe, target_finalized) = (chain.block(30), chain.block(20)); + let catching_up = initial.sync_state.with_safe_and_unsafe(target_safe); + let caught_up = catching_up.with_finalized(target_finalized); + let mut node = DelegatedNode::start(&chain, initial); + + for _ in 0..2 { + node.delegate_reports(target_safe, target_finalized); + node.expect_forkchoice(catching_up, PayloadStatusEnum::Syncing); + chain.el_lacks(&node.el, target_finalized.block_info.number); + node.next_poll().await; + node.assert_labels(catching_up).await; + } + + node.delegate_reports(target_safe, target_finalized); + node.expect_forkchoice(catching_up, PayloadStatusEnum::Valid); + chain.el_serves(&node.el, target_finalized.block_info.number); + node.expect_forkchoice(caught_up, PayloadStatusEnum::Valid); + node.next_poll().await; + node.assert_labels(caught_up).await; + + node.delegate_reports(target_safe, target_finalized); + node.expect_forkchoice(caught_up, PayloadStatusEnum::Valid); + node.next_poll().await; + node.assert_labels(caught_up).await; +} + +/// A stale or diverged finalized reference from the delegate. +#[derive(Clone, Copy, Debug)] +enum StaleFinalized { + /// A different block above the engine's finalized head, at a height the execution layer has. + ConflictingAbove, + /// A different block at the engine's finalized height. + ConflictingAtFinalizedHeight, + /// A block below the engine's finalized head, from a delegate that lags. + Regressed, +} + +impl StaleFinalized { + /// The status a stale delegate reports while the engine is safe at 20 and finalized at 15. + fn status(self, chain: &TestChain) -> SyncStatus { + match self { + Self::ConflictingAbove => pre_interop_status(chain.block(20), chain.fork_block(17)), + Self::ConflictingAtFinalizedHeight => { + pre_interop_status(chain.block(20), chain.fork_block(15)) + } + Self::Regressed => pre_interop_status(chain.block(18), chain.block(12)), + } + } + + /// What the EL is asked: only a conflicting target above finality reaches the engine. + fn expect_engine_requests( + self, + chain: &TestChain, + node: &DelegatedNode, + labels: EngineSyncState, + ) { + match self { + Self::ConflictingAbove => { + chain.el_serves(&node.el, 20); + node.expect_forkchoice(labels, PayloadStatusEnum::Valid); + chain.el_serves(&node.el, 17); + } + Self::ConflictingAtFinalizedHeight | Self::Regressed => {} + } + } +} + +/// Starts a node safe at 20 whose first poll finalizes block 15. +async fn start_finalized_at_15(chain: &TestChain) -> (DelegatedNode, EngineSyncState) { + let initial = synced_engine(chain.block(10), chain.block(20), chain.block(30)); + let mut node = DelegatedNode::start(chain, initial); + let finalized_at_15 = + node.poll_applies_safe_and_finalized(chain, initial.sync_state, 20, 15).await; + (node, finalized_at_15) +} + +/// Stale or conflicting finality never moves finalized; the next canonical one applies. +#[rstest::rstest] +#[case::conflicting_above(StaleFinalized::ConflictingAbove, 25, 20)] +#[case::conflicting_above_then_canonical_at_its_height(StaleFinalized::ConflictingAbove, 20, 17)] +#[case::conflicting_at_finalized_height(StaleFinalized::ConflictingAtFinalizedHeight, 25, 20)] +#[case::regressed(StaleFinalized::Regressed, 25, 20)] +#[tokio::test(start_paused = true)] +async fn stale_source_finalized_reference_never_moves_finalized_back( + #[case] stale: StaleFinalized, + #[case] recovered_safe: u64, + #[case] recovered_finalized: u64, +) { + let chain = TestChain::new(); + let (mut node, finalized_at_15) = start_finalized_at_15(&chain).await; + + node.delegate.reports(stale.status(&chain)); + stale.expect_engine_requests(&chain, &node, finalized_at_15); + node.next_poll().await; + node.assert_labels(finalized_at_15).await; + node.assert_origin(Some(l1_block(20))); + + node.poll_applies_safe_and_finalized( + &chain, + finalized_at_15, + recovered_safe, + recovered_finalized, + ) + .await; +} + +/// Local-safe and cross-safe labels move separately; finalizing above cross-safe is rejected whole. +#[tokio::test(start_paused = true)] +async fn local_safe_ahead_of_cross_safe_moves_each_label_separately_and_finalizes_below_cross_safe() +{ + let chain = TestChain::new(); + let initial = synced_engine(chain.block(5), chain.block(10), chain.block(35)); + let mut node = DelegatedNode::start(&chain, initial); + + node.delegate.reports(sync_status( + L1_ORIGIN, + chain.block(25), + chain.block(20), + chain.block(18), + )); + chain.el_serves(&node.el, 25); + chain.el_serves_header(&node.el, 20); + let cross_safe_at_20 = initial.sync_state.with_safe_heads(chain.block(25), chain.block(20)); + node.expect_forkchoice(cross_safe_at_20, PayloadStatusEnum::Valid); + chain.el_serves(&node.el, 18); + let finalized_at_18 = cross_safe_at_20.with_finalized(chain.block(18)); + node.expect_forkchoice(finalized_at_18, PayloadStatusEnum::Valid); + node.next_poll().await; + node.assert_labels(finalized_at_18).await; + + node.delegate.reports(sync_status( + L1_ORIGIN, + chain.block(30), + chain.block(22), + chain.block(25), + )); + node.next_poll().await; + node.assert_labels(finalized_at_18).await; + + node.delegate.reports(sync_status( + L1_ORIGIN, + chain.block(32), + chain.block(28), + chain.block(25), + )); + chain.el_serves(&node.el, 32); + chain.el_serves_header(&node.el, 28); + let cross_safe_at_28 = finalized_at_18.with_safe_heads(chain.block(32), chain.block(28)); + node.expect_forkchoice(cross_safe_at_28, PayloadStatusEnum::Valid); + chain.el_serves(&node.el, 25); + let finalized_at_25 = cross_safe_at_28.with_finalized(chain.block(25)); + node.expect_forkchoice(finalized_at_25, PayloadStatusEnum::Valid); + node.next_poll().await; + node.assert_labels(finalized_at_25).await; +} + +/// From a wiped EL, a reset returns to genesis and the target finalizes once the EL has it. +#[tokio::test(start_paused = true)] +async fn reset_to_genesis_while_finalized_is_missing_finalizes_once_the_el_has_it() { + let chain = TestChain::new(); + let genesis = all_at(chain.block(0)).sync_state; + let (target_safe, target_finalized) = (chain.block(30), chain.block(20)); + let catching_up = genesis.with_safe_and_unsafe(target_safe); + let caught_up = catching_up.with_finalized(target_finalized); + let mut node = DelegatedNode::start(&chain, EngineState::default()); + + // The startup reset finds only genesis, and the first poll starts the EL's sync. + chain.el_reset_to_genesis(&node.l1, &node.el); + node.delegate_reports(target_safe, target_finalized); + node.expect_forkchoice(catching_up, PayloadStatusEnum::Syncing); + chain.el_lacks(&node.el, target_finalized.block_info.number); + node.request_reset().await; + node.next_poll().await; + node.assert_labels(catching_up).await; + + chain.el_reset_to_genesis(&node.l1, &node.el); + node.request_reset().await; + node.assert_labels(genesis).await; + + node.delegate_reports(target_safe, target_finalized); + node.expect_forkchoice(catching_up, PayloadStatusEnum::Syncing); + chain.el_lacks(&node.el, target_finalized.block_info.number); + node.next_poll().await; + node.assert_labels(catching_up).await; + + node.delegate_reports(target_safe, target_finalized); + node.expect_forkchoice(catching_up, PayloadStatusEnum::Valid); + chain.el_serves(&node.el, target_finalized.block_info.number); + node.expect_forkchoice(caught_up, PayloadStatusEnum::Valid); + node.next_poll().await; + node.assert_labels(caught_up).await; +} diff --git a/rust/kona/crates/node/service/src/actors/derivation/delegated/test_utils.rs b/rust/kona/crates/node/service/src/actors/derivation/delegated/test_utils.rs new file mode 100644 index 00000000000..2f6104e5b1e --- /dev/null +++ b/rust/kona/crates/node/service/src/actors/derivation/delegated/test_utils.rs @@ -0,0 +1,114 @@ +//! A scripted derivation delegate and sync status builder shared by the delegated actor's tests. + +use super::SyncStatusSource; +use async_trait::async_trait; +use kona_protocol::{BlockInfo, L2BlockInfo, SyncStatus}; +use std::{ + collections::VecDeque, + sync::{Arc, Mutex}, +}; +use tokio::sync::watch; + +pub(super) type SyncStatusReply = Result; + +/// What a poll with no reply scripted does. +#[derive(Clone, Copy, Debug)] +enum UnscriptedPoll { + /// Fails like an unreachable delegate, which the actor skips. + Fails, + /// Panics, so a test that scripts every poll catches an extra one. + Panics, +} + +/// Answers each poll with the next scripted reply. It counts every poll, and separately the polls +/// it answered with a sync status. +#[derive(Clone, Debug)] +pub(super) struct ScriptedDelegate { + answered_polls: watch::Sender, + polls: watch::Sender, + replies: Arc>>, + unscripted_poll: UnscriptedPoll, +} + +impl ScriptedDelegate { + /// A delegate whose every poll must be scripted. + pub(super) fn strict(replies: impl IntoIterator) -> Self { + Self::new(replies, UnscriptedPoll::Panics) + } + + /// A delegate that fails the polls nothing is scripted for, so waits can span them. + pub(super) fn lenient() -> Self { + Self::new([], UnscriptedPoll::Fails) + } + + fn new( + replies: impl IntoIterator, + unscripted_poll: UnscriptedPoll, + ) -> Self { + Self { + answered_polls: watch::Sender::new(0), + polls: watch::Sender::new(0), + replies: Arc::new(Mutex::new(replies.into_iter().collect())), + unscripted_poll, + } + } + + /// Scripts `status` as the reply to the next unanswered poll. + pub(super) fn reports(&self, status: SyncStatus) { + self.replies.lock().unwrap().push_back(Ok(status)); + } + + /// Counts the polls answered with a sync status. + pub(super) fn answered_polls(&self) -> watch::Receiver { + self.answered_polls.subscribe() + } + + /// Counts every poll, answered or not. + pub(super) fn polls(&self) -> watch::Receiver { + self.polls.subscribe() + } + + pub(super) fn assert_finished(&self) { + let unanswered = self.replies.lock().unwrap().len(); + assert_eq!(unanswered, 0, "{unanswered} scripted sync status replies were never polled"); + } +} + +#[async_trait] +impl SyncStatusSource for ScriptedDelegate { + type Error = &'static str; + + async fn sync_status(&self) -> SyncStatusReply { + self.polls.send_modify(|polls| *polls += 1); + let Some(reply) = self.replies.lock().unwrap().pop_front() else { + return match self.unscripted_poll { + UnscriptedPoll::Fails => Err("no sync status scripted"), + UnscriptedPoll::Panics => panic!("unexpected sync status poll"), + }; + }; + if reply.is_ok() { + self.answered_polls.send_modify(|polls| *polls += 1); + } + reply + } +} + +/// A sync status with these labels, its unsafe head at the local-safe head. +pub(super) fn sync_status( + current_l1: BlockInfo, + local_safe: L2BlockInfo, + safe: L2BlockInfo, + finalized: L2BlockInfo, +) -> SyncStatus { + SyncStatus { + current_l1, + current_l1_finalized: BlockInfo::default(), + finalized_l1: BlockInfo::default(), + finalized_l2: finalized, + head_l1: BlockInfo::default(), + local_safe_l2: local_safe, + safe_l1: BlockInfo::default(), + safe_l2: safe, + unsafe_l2: local_safe, + } +} diff --git a/rust/kona/crates/node/service/src/actors/derivation/delegated/tests.rs b/rust/kona/crates/node/service/src/actors/derivation/delegated/tests.rs index fea52b068d1..b9e522b66a7 100644 --- a/rust/kona/crates/node/service/src/actors/derivation/delegated/tests.rs +++ b/rust/kona/crates/node/service/src/actors/derivation/delegated/tests.rs @@ -1,45 +1,36 @@ -use super::{DelegateDerivationActor, SyncStatusSource, actor::DERIVATION_DELEGATE_POLL_INTERVAL}; +use super::{ + DelegateDerivationActor, + actor::DERIVATION_DELEGATE_POLL_INTERVAL, + test_utils::{ScriptedDelegate, SyncStatusReply, sync_status}, +}; use crate::{DerivationActorRequest, EngineActorRequest, NodeActor, QueuedDerivationEngineClient}; use alloy_eips::BlockNumHash; use alloy_primitives::B256; -use async_trait::async_trait; use kona_derive::test_utils::TestChainProvider; use kona_engine::{ - ConsolidateInput, EngineState, EngineSyncState, EngineSyncStateUpdate, FinalizeBlockId, + ConsolidateInput, DelegatedSafeHeads, EngineState, EngineSyncState, EngineSyncStateUpdate, + FinalizeBlockId, }; use kona_protocol::{BlockInfo, L2BlockInfo, SyncStatus}; use rstest::rstest; -use std::{collections::VecDeque, sync::Mutex}; use tokio::{ sync::{mpsc, watch}, time::{self, Instant}, }; +type HarnessActor = + DelegateDerivationActor; + const SAFE_ORIGIN: u64 = 1; const FINALIZED_ORIGIN: u64 = 2; const CURRENT_L1: u64 = 3; -type SyncStatusReply = Result; - -/// Answers each sync status poll with the next scripted reply. -#[derive(Debug)] -struct ScriptedDelegate(Mutex>); - -#[async_trait] -impl SyncStatusSource for ScriptedDelegate { - type Error = &'static str; - - async fn sync_status(&self) -> SyncStatusReply { - self.0.lock().unwrap().pop_front().expect("unexpected sync status poll") - } -} - /// A delegate actor over a canonical L1 chain, with the engine requests it sends and the /// `current_l1` it publishes. struct Harness { - actor: - DelegateDerivationActor, + actor: HarnessActor, current_l1: watch::Receiver>, + delegate: ScriptedDelegate, derivation_requests: mpsc::Sender, engine_requests: mpsc::Receiver, engine_state: watch::Sender, @@ -55,15 +46,16 @@ impl Harness { let (engine_request_tx, engine_requests) = mpsc::channel(8); let (current_l1_tx, current_l1) = watch::channel(None); let (engine_state, engine_state_rx) = watch::channel(EngineState::default()); - let actor = DelegateDerivationActor::new( + let delegate = ScriptedDelegate::strict(replies); + let actor = HarnessActor::new( QueuedDerivationEngineClient::new(engine_request_tx), derivation_request_rx, - ScriptedDelegate(Mutex::new(replies.into_iter().collect())), + delegate.clone(), l1, current_l1_tx, engine_state_rx, ); - Self { actor, current_l1, derivation_requests, engine_requests, engine_state } + Self { actor, current_l1, delegate, derivation_requests, engine_requests, engine_state } } async fn complete_engine_sync(&mut self) { @@ -88,36 +80,55 @@ impl Harness { self.engine_state.send_modify(|state| state.sync_state = sync_state); } + /// Publishes the engine's labels along with whether its execution layer applied the engine's + /// last forkchoice. + fn publish_engine_state(&self, sync_state: EngineSyncState, el_applied_last_forkchoice: bool) { + self.engine_state.send_modify(|state| { + state.sync_state = sync_state; + state.el_applied_last_forkchoice = el_applied_last_forkchoice; + }); + } + /// The engine requests sent since the last call, in order. fn forwarded(&mut self) -> Vec { std::iter::from_fn(|| self.engine_requests.try_recv().ok()).map(Forwarded::from).collect() } } +impl Drop for Harness { + fn drop(&mut self) { + if !std::thread::panicking() { + self.delegate.assert_finished(); + } + } +} + /// The parts of an engine request that the delegate actor decides. #[derive(Debug, PartialEq, Eq)] enum Forwarded { Finalized(FinalizeBlockId), Other(String), - Safe(L2BlockInfo), + Safe(Box), } impl From for Forwarded { fn from(request: EngineActorRequest) -> Self { match request { EngineActorRequest::ProcessFinalizedL2Block(finalized) => Self::Finalized(*finalized), - EngineActorRequest::ProcessSafeL2Signal(ConsolidateInput::BlockInfo(safe)) => { - Self::Safe(safe) + EngineActorRequest::ProcessSafeL2Signal(ConsolidateInput::Delegated(heads)) => { + Self::Safe(heads) } other => Self::Other(format!("{other:?}")), } } } -/// What one poll of `status` forwards: the safe head, then the finalized block pinned by hash. +/// What one poll of `status` forwards: the safe heads, then the finalized block pinned by hash. fn forwarded_for(status: &SyncStatus) -> Vec { vec![ - Forwarded::Safe(status.safe_l2), + Forwarded::Safe(Box::new( + DelegatedSafeHeads::try_new(status.local_safe_l2, status.safe_l2).unwrap(), + )), Forwarded::Finalized(FinalizeBlockId::ByHash(status.finalized_l2.block_info.id())), ] } @@ -143,18 +154,17 @@ fn l2_block(number: u64, l1_origin: u64) -> L2BlockInfo { } } -/// A sync status whose safe origin, finalized origin and `current_l1` are all canonical. +/// A sync status whose safe origin, finalized origin and `current_l1` are all canonical, with an +/// unsafe head above its safe heads. fn delegate_status() -> SyncStatus { SyncStatus { - current_l1: canonical_l1_block(CURRENT_L1), - current_l1_finalized: BlockInfo::default(), - finalized_l1: BlockInfo::default(), - finalized_l2: l2_block(15, FINALIZED_ORIGIN), - head_l1: BlockInfo::default(), - local_safe_l2: l2_block(20, SAFE_ORIGIN), - safe_l1: BlockInfo::default(), - safe_l2: l2_block(20, SAFE_ORIGIN), unsafe_l2: l2_block(25, SAFE_ORIGIN), + ..sync_status( + canonical_l1_block(CURRENT_L1), + l2_block(20, SAFE_ORIGIN), + l2_block(20, SAFE_ORIGIN), + l2_block(15, FINALIZED_ORIGIN), + ) } } @@ -261,6 +271,7 @@ enum DelegateReply { CurrentL1NotCanonical, FetchFailed, FinalizedOriginNotCanonical, + LocalSafeOriginNotCanonical, SafeOriginNotCanonical, Valid, } @@ -278,6 +289,9 @@ impl DelegateReply { Self::FinalizedOriginNotCanonical => { status.finalized_l2.l1_origin = non_canonical(status.finalized_l2.l1_origin) } + Self::LocalSafeOriginNotCanonical => { + status.local_safe_l2.l1_origin = non_canonical(status.local_safe_l2.l1_origin) + } Self::SafeOriginNotCanonical => { status.safe_l2.l1_origin = non_canonical(status.safe_l2.l1_origin) } @@ -306,6 +320,7 @@ async fn polls_wait_for_engine_sync_completion() { #[rstest] #[case::valid(DelegateReply::Valid, true)] #[case::fetch_failed(DelegateReply::FetchFailed, false)] +#[case::local_safe_origin_not_canonical(DelegateReply::LocalSafeOriginNotCanonical, false)] #[case::safe_origin_not_canonical(DelegateReply::SafeOriginNotCanonical, false)] #[case::finalized_origin_not_canonical(DelegateReply::FinalizedOriginNotCanonical, false)] #[case::current_l1_not_canonical(DelegateReply::CurrentL1NotCanonical, false)] @@ -367,3 +382,120 @@ async fn poll_rejects_status_whose_finalized_is_ahead_of_its_safe( assert_eq!(harness.forwarded(), forwarded_if(accepted, &status)); } + +/// A status whose labels are out of order is rejected whole: a finalized block ahead of the safe +/// head would let a stale snapshot's safe head sit below finality, and a safe head ahead of the +/// local safe head is not a state any delegate reaches. +#[rstest] +#[case::ordered(15, 20, 20, true)] +#[case::finalized_at_safe(20, 20, 20, true)] +#[case::safe_behind_local_safe(15, 18, 20, true)] +#[case::finalized_ahead_of_safe(16, 15, 20, false)] +#[case::safe_ahead_of_local_safe(15, 21, 20, false)] +#[tokio::test(start_paused = true)] +async fn poll_rejects_status_with_labels_out_of_order( + #[case] finalized: u64, + #[case] safe: u64, + #[case] local_safe: u64, + #[case] accepted: bool, +) { + let status = SyncStatus { + finalized_l2: l2_block(finalized, FINALIZED_ORIGIN), + local_safe_l2: l2_block(local_safe, SAFE_ORIGIN), + safe_l2: l2_block(safe, SAFE_ORIGIN), + ..delegate_status() + }; + let mut harness = Harness::new([Ok(status.clone())]); + harness.complete_engine_sync().await; + + harness.actor.step().await.unwrap(); + + assert_eq!(harness.forwarded(), forwarded_if(accepted, &status)); + assert_eq!(harness.current_l1.borrow().is_some(), accepted); +} + +/// The delegate's local-safe and cross-safe heads are forwarded as separate labels, so a cross-safe +/// head that lags the local-safe head under interop is not raised to it, and before interop both +/// carry the same block. +#[rstest] +#[case::pre_interop(20, 20)] +#[case::cross_safe_lags_local_safe(18, 20)] +#[tokio::test(start_paused = true)] +async fn poll_forwards_local_safe_and_cross_safe_heads_separately( + #[case] safe: u64, + #[case] local_safe: u64, +) { + let status = SyncStatus { + local_safe_l2: l2_block(local_safe, SAFE_ORIGIN), + safe_l2: l2_block(safe, SAFE_ORIGIN), + ..delegate_status() + }; + let mut harness = Harness::new([Ok(status.clone())]); + harness.complete_engine_sync().await; + + harness.actor.step().await.unwrap(); + + let expected_heads = + DelegatedSafeHeads::try_new(l2_block(local_safe, SAFE_ORIGIN), l2_block(safe, SAFE_ORIGIN)) + .unwrap(); + assert_eq!(harness.forwarded()[0], Forwarded::Safe(Box::new(expected_heads))); +} + +/// A zero local safe head is a label the delegate has not derived or does not report, so it does +/// not put the safe head out of order and the status is forwarded. +#[tokio::test(start_paused = true)] +async fn unreported_local_safe_head_does_not_order_the_labels() { + let status = SyncStatus { local_safe_l2: L2BlockInfo::default(), ..delegate_status() }; + let mut harness = Harness::new([Ok(status.clone())]); + harness.complete_engine_sync().await; + + harness.actor.step().await.unwrap(); + + let reported_as_safe = SyncStatus { local_safe_l2: status.safe_l2, ..status }; + assert_eq!(harness.forwarded(), forwarded_for(&reported_as_safe)); +} + +/// An engine sync state with consistent labels: unsafe at 30, safe at `safe`, finalized at 15. +fn engine_safe_at(safe: u64) -> EngineSyncState { + EngineSyncState::default().apply_update(EngineSyncStateUpdate { + finalized_head: Some(l2_block(15, FINALIZED_ORIGIN)), + local_safe_head: Some(l2_block(safe, SAFE_ORIGIN)), + safe_head: Some(l2_block(safe, SAFE_ORIGIN)), + unsafe_head: Some(l2_block(30, SAFE_ORIGIN)), + }) +} + +/// What a poll of `status` forwards when its safe head is held back: the finalized block alone. +fn finalized_only(status: &SyncStatus) -> Vec { + vec![Forwarded::Finalized(FinalizeBlockId::ByHash(status.finalized_l2.block_info.id()))] +} + +/// Safe heads behind the engine's are forwarded once while the execution layer applied the +/// engine's forkchoice, since the engine's answer to it cannot change; finalized still goes out +/// on every poll. A moved engine safe head, or an execution layer that did not apply the last +/// forkchoice, sends it again. +#[rstest] +#[case::same_engine_view(true, 25, finalized_only)] +#[case::el_not_applied(false, 25, forwarded_for)] +#[case::engine_safe_moved(true, 26, forwarded_for)] +#[tokio::test(start_paused = true)] +async fn lagging_safe_heads_are_forwarded_once_per_engine_view( + #[case] el_applied_last_forkchoice: bool, + #[case] engine_safe_on_second_poll: u64, + #[case] second_poll_forwards: fn(&SyncStatus) -> Vec, +) { + let status = delegate_status(); + let mut harness = Harness::new([Ok(status.clone()), Ok(status.clone())]); + harness.complete_engine_sync_with(engine_safe_at(25)).await; + harness.publish_engine_state(engine_safe_at(25), el_applied_last_forkchoice); + harness.actor.step().await.unwrap(); + assert_eq!(harness.forwarded(), forwarded_for(&status)); + harness.publish_engine_state( + engine_safe_at(engine_safe_on_second_poll), + el_applied_last_forkchoice, + ); + + harness.actor.step().await.unwrap(); + + assert_eq!(harness.forwarded(), second_poll_forwards(&status)); +} diff --git a/rust/kona/crates/node/service/src/actors/engine/actor.rs b/rust/kona/crates/node/service/src/actors/engine/actor.rs index f09047db8a2..546358ca87c 100644 --- a/rust/kona/crates/node/service/src/actors/engine/actor.rs +++ b/rust/kona/crates/node/service/src/actors/engine/actor.rs @@ -20,6 +20,13 @@ use tokio::{ #[path = "queue_tests.rs"] mod queue_tests; +/// The delay before the first retry after a dependency failure; a clean drain resets the backoff +/// to it. +pub(crate) const INITIAL_RETRY_DELAY: Duration = Duration::from_millis(100); + +/// The cap on the doubling retry delay, keeping recovery responsive. +pub(crate) const MAX_RETRY_DELAY: Duration = Duration::from_secs(5); + /// Responsible for managing the operations sent to the execution layer's Engine API. To accomplish /// this, it uses the [`Engine`] task queue to order Engine API interactions based off of /// the [`Ord`] implementation of [`EngineTask`]. @@ -92,7 +99,7 @@ where reset_pending: false, reset_waiters: Vec::new(), retry_at: Instant::now(), - retry_delay: Duration::from_millis(100), + retry_delay: INITIAL_RETRY_DELAY, } } @@ -110,7 +117,7 @@ where fn defer_retry(&mut self) { self.retry_at = Instant::now() + self.retry_delay; - self.retry_delay = (self.retry_delay * 2).min(Duration::from_secs(5)); + self.retry_delay = (self.retry_delay * 2).min(MAX_RETRY_DELAY); } fn begin_reset(&mut self) { @@ -126,7 +133,7 @@ where match self.reset().await { Ok(()) => { self.reset_pending = false; - self.retry_delay = Duration::from_millis(100); + self.retry_delay = INITIAL_RETRY_DELAY; if !self.el_sync_complete && self.engine.state().el_sync_finished { self.mark_el_sync_complete_and_notify_derivation_actor().await?; } @@ -192,7 +199,7 @@ where match self.engine.drain().await { Ok(_) => { trace!(target: "engine", "[ENGINE] tasks drained"); - self.retry_delay = Duration::from_millis(100); + self.retry_delay = INITIAL_RETRY_DELAY; } Err(err) => { match err.severity() { @@ -235,7 +242,7 @@ where self.send_derivation_actor_safe_head_if_updated().await?; if !self.el_sync_complete && self.engine.state().el_sync_finished { - if self.engine.state().sync_state.finalized_head() == L2BlockInfo::default() { + if self.engine.state().sync_state.established_finalized_head().is_none() { self.begin_reset(); } else { self.mark_el_sync_complete_and_notify_derivation_actor().await?; diff --git a/rust/kona/crates/node/service/src/actors/engine/mod.rs b/rust/kona/crates/node/service/src/actors/engine/mod.rs index ef8308ab059..e13242fa0dd 100644 --- a/rust/kona/crates/node/service/src/actors/engine/mod.rs +++ b/rust/kona/crates/node/service/src/actors/engine/mod.rs @@ -2,6 +2,8 @@ mod actor; pub use actor::EngineActor; +#[cfg(test)] +pub(crate) use actor::{INITIAL_RETRY_DELAY, MAX_RETRY_DELAY}; mod client; pub use client::{EngineDerivationClient, QueuedEngineDerivationClient}; diff --git a/rust/kona/crates/node/service/src/actors/engine/queue_tests.rs b/rust/kona/crates/node/service/src/actors/engine/queue_tests.rs index cd3c695f383..defb95a85b2 100644 --- a/rust/kona/crates/node/service/src/actors/engine/queue_tests.rs +++ b/rust/kona/crates/node/service/src/actors/engine/queue_tests.rs @@ -5,11 +5,10 @@ use alloy_rpc_types_engine::{ ExecutionPayloadV1, ForkchoiceState, PayloadStatus, PayloadStatusEnum, }; use kona_engine::{ - ConsolidateInput, EngineState, EngineSyncStateUpdate, FinalizeBlockId, NoopBlockSink, - ResetRequest, + EngineState, EngineSyncStateUpdate, FinalizeBlockId, NoopBlockSink, ResetRequest, test_utils::{ - RpcMock, TestEngineStateBuilder, TestRpcBlock, expect_sync_start, test_block_info, - test_engine_client, test_l2_block, test_rpc_block, + RpcMock, TestEngineStateBuilder, TestRpcBlock, delegated_safe_heads, expect_sync_start, + test_block_info, test_engine_client, test_l2_block, test_rpc_block, }, }; use op_alloy_consensus::{OpBlock, OpTxEnvelope}; @@ -118,7 +117,8 @@ async fn delegated_safe_head_is_sent_on_every_poll() { // accepts the next poll. for _ in 0..3 { requests - .send(EngineActorRequest::ProcessSafeL2Signal(ConsolidateInput::BlockInfo( + .send(EngineActorRequest::ProcessSafeL2Signal(delegated_safe_heads( + delegated_safe_head, delegated_safe_head, ))) .await @@ -143,6 +143,7 @@ async fn delegated_poll_during_el_sync_moves_safe_labels_and_skips_finalization( let (client, l1, l2) = test_engine_client(config.clone()); let state = TestEngineStateBuilder::new() .with_unsafe_head(test_block_info(20)) + .with_local_safe_head(test_block_info(5)) .with_safe_head(test_block_info(5)) .with_finalized_head(test_block_info(3)) .build(); @@ -175,7 +176,8 @@ async fn delegated_poll_during_el_sync_moves_safe_labels_and_skips_finalization( // One poll: the safe head, then the finalized block pinned by hash. Each step drains the // previous request's task and then accepts the next request. requests - .send(EngineActorRequest::ProcessSafeL2Signal(ConsolidateInput::BlockInfo( + .send(EngineActorRequest::ProcessSafeL2Signal(delegated_safe_heads( + delegated_safe_head, delegated_safe_head, ))) .await @@ -319,7 +321,13 @@ async fn finalization_target_survives_resets_below_it(#[case] target_kind: Targe requests.send(target_kind.finalize_request(target)).await.unwrap(); actor.step().await.unwrap(); assert!(second_reset_result.try_recv().unwrap().is_ok()); - requests.send(EngineActorRequest::ProcessSafeL2Signal(restored_safe.into())).await.unwrap(); + requests + .send(EngineActorRequest::ProcessSafeL2Signal(delegated_safe_heads( + restored_safe, + restored_safe, + ))) + .await + .unwrap(); actor.step().await.unwrap(); assert_eq!(actor.engine.len(), 1, "only the safe head update may remain queued"); assert!(!actor.reset_pending); diff --git a/rust/kona/crates/node/service/src/actors/engine/tests.rs b/rust/kona/crates/node/service/src/actors/engine/tests.rs index b61712d95b5..4d43b98d03a 100644 --- a/rust/kona/crates/node/service/src/actors/engine/tests.rs +++ b/rust/kona/crates/node/service/src/actors/engine/tests.rs @@ -4,7 +4,7 @@ use alloy_primitives::B256; use alloy_rpc_types_engine::{ForkchoiceUpdated, PayloadStatus, PayloadStatusEnum}; use kona_engine::{ Engine, EngineActorRequest, EngineState, EngineSyncStateUpdate, FinalizeBlockId, NoopBlockSink, - test_utils::{TestAttributesBuilder, test_engine_client}, + test_utils::{TestAttributesBuilder, delegated_safe_heads, test_engine_client}, }; use kona_genesis::RollupConfig; use kona_protocol::{BlockInfo, L2BlockInfo}; @@ -75,7 +75,7 @@ async fn invalid_forkchoice_resets_and_resumes(#[case] rebuild: bool, #[case] ho .build() .into() } else { - l2_block(1).into() + delegated_safe_heads(l2_block(1), l2_block(1)) }; requests.send(EngineActorRequest::ProcessSafeL2Signal(signal)).await.unwrap(); actor.step().await.unwrap(); @@ -98,7 +98,10 @@ async fn invalid_forkchoice_resets_and_resumes(#[case] rebuild: bool, #[case] ho ForkchoiceUpdated::new(PayloadStatus::from_status(PayloadStatusEnum::Valid)), ); let recovered = l2_block(2); - requests.send(EngineActorRequest::ProcessSafeL2Signal(recovered.into())).await.unwrap(); + requests + .send(EngineActorRequest::ProcessSafeL2Signal(delegated_safe_heads(recovered, recovered))) + .await + .unwrap(); actor.step().await.unwrap(); assert_eq!(state_rx.borrow().sync_state.unsafe_head(), genesis, "publish the reset head"); assert_eq!(state_rx.borrow().sync_state.safe_head(), genesis);