From 94fe82dcfa2f38d6b5154b66f4b1a15dd477b4cb Mon Sep 17 00:00:00 2001 From: Josh Klopfenstein Date: Sun, 8 Nov 2026 16:16:44 -0800 Subject: [PATCH] op-node: keep follow-source heads when upstream CurrentL1 is not on our L1 yet A light node checks the upstream CurrentL1 against its own L1 RPC. When that RPC is a few blocks behind the upstream's (for example a consensus-aware L1 proxyd whose head moves back while a lagging backend rejoins), the block is not found and the whole poll was rejected and counted as error_l1_lookup, discarding safe and finalized heads that were already verified. Treat a not-found CurrentL1 like an upstream that reports none: apply the verified heads, leave CurrentL1 unchanged, and count it in the new follow_source_current_l1_not_found_total metric. A CurrentL1 hash mismatch or any other lookup error still rejects the poll. Co-Authored-By: Claude Opus 5.5 --- op-node/metrics/metrics.go | 10 ++ op-node/rollup/driver/driver.go | 56 ++++--- op-node/rollup/driver/follow_upstream_test.go | 152 +++++++++++++++++- 3 files changed, 191 insertions(+), 27 deletions(-) diff --git a/op-node/metrics/metrics.go b/op-node/metrics/metrics.go index 2b93b390d8e..19c65ab8f74 100644 --- a/op-node/metrics/metrics.go +++ b/op-node/metrics/metrics.go @@ -35,6 +35,7 @@ type Metricer interface { SetSequencerState(active bool) RecordPipelineReset() RecordFollowSourceRequest(result string) + RecordFollowSourceCurrentL1NotFound() RecordFollowSourceReorg(action string) RecordSuperAuthorityReorgSignal(reason string) RecordSequencingError() @@ -92,6 +93,7 @@ type Metrics struct { L2FollowSourceCache *metrics.CacheMetrics FollowSourceRequests *prometheus.CounterVec + FollowSourceCurrentL1Miss prometheus.Counter FollowSourceReorgs *prometheus.CounterVec SuperAuthorityReorgSignals *prometheus.CounterVec @@ -222,6 +224,7 @@ func NewMetrics(procName string, labels prometheus.Labels) *Metrics { Name: "follow_source_requests_total", Help: "Count of follow source requests by result", }, []string{"result"}), + FollowSourceCurrentL1Miss: factory.NewCounter(prometheus.CounterOpts{Namespace: ns, Name: "follow_source_current_l1_not_found_total", Help: "Count of follow source polls whose upstream CurrentL1 was not found on our L1 chain and was dropped"}), FollowSourceReorgs: factory.NewCounterVec(prometheus.CounterOpts{Namespace: ns, Name: "follow_source_reorgs_total", Help: "Count of follow source reorg decisions by action"}, []string{"action"}), SuperAuthorityReorgSignals: factory.NewCounterVec(prometheus.CounterOpts{Namespace: ns, Name: "super_authority_reorg_signals_total", Help: "Count of super authority reorg signals by reason"}, []string{"reason"}), @@ -580,6 +583,10 @@ func (m *Metrics) RecordFollowSourceRequest(result string) { m.FollowSourceRequests.WithLabelValues(result).Inc() } +func (m *Metrics) RecordFollowSourceCurrentL1NotFound() { + m.FollowSourceCurrentL1Miss.Inc() +} + func (m *Metrics) RecordFollowSourceReorg(action string) { m.FollowSourceReorgs.WithLabelValues(action).Inc() } @@ -736,6 +743,9 @@ func (n *noopMetricer) RecordPipelineReset() { func (n *noopMetricer) RecordFollowSourceRequest(result string) { } +func (n *noopMetricer) RecordFollowSourceCurrentL1NotFound() { +} + func (n *noopMetricer) RecordFollowSourceReorg(action string) { } diff --git a/op-node/rollup/driver/driver.go b/op-node/rollup/driver/driver.go index e8e752895c4..10609951d94 100644 --- a/op-node/rollup/driver/driver.go +++ b/op-node/rollup/driver/driver.go @@ -2,9 +2,11 @@ package driver import ( "context" + "errors" "fmt" "time" + "github.com/ethereum/go-ethereum" "github.com/ethereum/go-ethereum/common" gosync "sync" @@ -457,9 +459,11 @@ func (s *Driver) startFollowUpstreamFetch(resultCh chan<- *sources.FollowStatus) // followUpstream fetches and validates external safe, finalized, and CurrentL1 // references when derivation is disabled (UnsafeOnly). It returns nil if the -// fetch fails or the external state is inconsistent; a non-nil status has -// passed all checks. It performs no engine or event side effects — the driver -// event loop applies those when it receives the returned status. +// fetch fails or the external state is inconsistent; the heads of a non-nil +// status have passed all checks. If our L1 chain does not have the external +// CurrentL1 yet, it returns the status with CurrentL1 cleared. +// It performs no engine or event side effects — the driver event loop applies +// those when it receives the returned status. func (s *Driver) followUpstream() *sources.FollowStatus { status, err := s.upstreamFollowSource.GetFollowStatus(s.driverCtx) if err != nil { @@ -530,22 +534,36 @@ func (s *Driver) followUpstream() *sources.FollowStatus { if (status.CurrentL1 == eth.L1BlockRef{}) { s.log.Debug("Follow Upstream: CurrentL1 not available") - } else { - eCurrentL1, err := s.upstreamFollowSource.L1BlockRefByNumber(s.driverCtx, status.CurrentL1.Number) - if err != nil { - s.log.Warn("Follow Upstream: Failed to look up external currentL1", "err", err) - s.metrics.RecordFollowSourceRequest("error_l1_lookup") - return nil - } - if eCurrentL1.Hash != status.CurrentL1.Hash { - s.log.Warn( - "Follow Upstream: Invalid external CurrentL1: L1 head mismatch", - "actual", eCurrentL1, - "expected", status.CurrentL1, - ) - s.metrics.RecordFollowSourceRequest("error_l1_mismatch") - return nil - } + return status + } + eCurrentL1, err := s.upstreamFollowSource.L1BlockRefByNumber(s.driverCtx, status.CurrentL1.Number) + if errors.Is(err, ethereum.NotFound) { + // Our L1 RPC is behind the upstream's: it does not have this block yet, for example + // while a load-balanced L1 endpoint serves from a lagging backend. The L1 origins of + // the heads are verified, so apply the heads and drop only CurrentL1, which the event + // loop then leaves unchanged. The L1 blocks between the local-safe origin and + // CurrentL1 stay unverified until a later poll finds a CurrentL1 on our chain; if the + // upstream is on an L1 fork, that poll fails with a hash mismatch. + s.log.Info("Follow Upstream: External CurrentL1 not found on our L1 chain yet, not updating CurrentL1", + "currentL1", status.CurrentL1) + s.metrics.RecordFollowSourceCurrentL1NotFound() + withoutCurrentL1 := *status + withoutCurrentL1.CurrentL1 = eth.L1BlockRef{} + return &withoutCurrentL1 + } + if err != nil { + s.log.Warn("Follow Upstream: Failed to look up external currentL1", "err", err) + s.metrics.RecordFollowSourceRequest("error_l1_lookup") + return nil + } + if eCurrentL1.Hash != status.CurrentL1.Hash { + s.log.Warn( + "Follow Upstream: Invalid external CurrentL1: L1 head mismatch", + "actual", eCurrentL1, + "expected", status.CurrentL1, + ) + s.metrics.RecordFollowSourceRequest("error_l1_mismatch") + return nil } return status } diff --git a/op-node/rollup/driver/follow_upstream_test.go b/op-node/rollup/driver/follow_upstream_test.go index 185404647d8..a1bd9f72506 100644 --- a/op-node/rollup/driver/follow_upstream_test.go +++ b/op-node/rollup/driver/follow_upstream_test.go @@ -3,6 +3,7 @@ package driver import ( "context" "errors" + "fmt" "testing" "time" @@ -10,6 +11,7 @@ import ( "github.com/ethereum-optimism/optimism/op-service/log" "github.com/ethereum-optimism/optimism/op-service/sources" "github.com/ethereum-optimism/optimism/op-service/testlog" + "github.com/ethereum/go-ethereum" "github.com/ethereum/go-ethereum/common" "github.com/stretchr/testify/require" ) @@ -41,23 +43,157 @@ func (s *blockingUpstreamFollowSource) L1BlockRefByNumber(_ context.Context, num // methods panic via the nil embedded interface if called. type followMetricsStub struct { Metrics - results []string + results []string + currentL1NotFound int } func (m *followMetricsStub) RecordFollowSourceRequest(result string) { m.results = append(m.results, result) } -func TestStartFollowUpstreamFetchIsAsync(t *testing.T) { - ref := func(number uint64) eth.BlockID { - return eth.BlockID{Number: number, Hash: common.Hash{byte(number)}} +func (m *followMetricsStub) RecordFollowSourceCurrentL1NotFound() { + m.currentL1NotFound++ +} + +func l1Ref(number uint64) eth.L1BlockRef { + return eth.L1BlockRef{Number: number, Hash: common.Hash{byte(number)}} +} + +// stubUpstreamFollowSource serves a fixed status and answers L1 lookups from our +// side of the chain: blocks in l1 are found, numbers in l1Errs fail, and any +// other number is not found. +type stubUpstreamFollowSource struct { + status *sources.FollowStatus + l1 map[uint64]eth.L1BlockRef + l1Errs map[uint64]error +} + +func (s *stubUpstreamFollowSource) GetFollowStatus(context.Context) (*sources.FollowStatus, error) { + return s.status, nil +} + +func (s *stubUpstreamFollowSource) L1BlockRefByNumber(_ context.Context, number uint64) (eth.L1BlockRef, error) { + if err, ok := s.l1Errs[number]; ok { + return eth.L1BlockRef{}, err } + if ref, ok := s.l1[number]; ok { + return ref, nil + } + return eth.L1BlockRef{}, fmt.Errorf("failed to fetch header by num %d: %w", number, ethereum.NotFound) +} + +// TestFollowUpstreamCurrentL1 checks that an upstream CurrentL1 our L1 chain does +// not have yet only drops CurrentL1: the verified heads are still applied, and the +// request is not counted as an error. Any other CurrentL1 failure still rejects +// the whole status. +func TestFollowUpstreamCurrentL1(t *testing.T) { + ourL1 := map[uint64]eth.L1BlockRef{1: l1Ref(1), 2: l1Ref(2), 3: l1Ref(3), 4: l1Ref(4)} + newStatus := func(currentL1 eth.L1BlockRef) *sources.FollowStatus { + return &sources.FollowStatus{ + LocalSafeL2: eth.L2BlockRef{Number: 3, L1Origin: l1Ref(3).ID()}, + SafeL2: eth.L2BlockRef{Number: 2, L1Origin: l1Ref(2).ID()}, + FinalizedL2: eth.L2BlockRef{Number: 1, L1Origin: l1Ref(1).ID()}, + CurrentL1: currentL1, + } + } + + tests := []struct { + name string + currentL1 eth.L1BlockRef + l1Errs map[uint64]error + wantRejected string // request result if the status is rejected, empty if accepted + wantCurrentL1 eth.L1BlockRef + wantNotFound int + }{ + { + name: "verified", + currentL1: l1Ref(4), + wantCurrentL1: l1Ref(4), + }, + { + name: "not reported by upstream", + currentL1: eth.L1BlockRef{}, + wantCurrentL1: eth.L1BlockRef{}, + }, + { + name: "ahead of our L1 view", + currentL1: l1Ref(5), + wantCurrentL1: eth.L1BlockRef{}, + wantNotFound: 1, + }, + { + name: "L1 lookup fails", + currentL1: l1Ref(4), + l1Errs: map[uint64]error{4: errors.New("connection refused")}, + wantRejected: "error_l1_lookup", + }, + { + name: "not on our L1 chain", + currentL1: eth.L1BlockRef{Number: 4, Hash: common.Hash{0xff}}, + wantRejected: "error_l1_mismatch", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + upstream := newStatus(tt.currentL1) + metrics := &followMetricsStub{} + driver := &Driver{ + driverCtx: context.Background(), + upstreamFollowSource: &stubUpstreamFollowSource{status: upstream, l1: ourL1, l1Errs: tt.l1Errs}, + log: testlog.Logger(t, log.LevelError), + metrics: metrics, + } + + status := driver.followUpstream() + + require.Equal(t, tt.wantNotFound, metrics.currentL1NotFound) + require.Equal(t, tt.currentL1, upstream.CurrentL1, "must not modify the upstream status") + if tt.wantRejected != "" { + require.Nil(t, status) + require.Equal(t, []string{tt.wantRejected}, metrics.results) + return + } + require.NotNil(t, status) + require.Equal(t, upstream.LocalSafeL2, status.LocalSafeL2) + require.Equal(t, upstream.SafeL2, status.SafeL2) + require.Equal(t, upstream.FinalizedL2, status.FinalizedL2) + require.Equal(t, tt.wantCurrentL1, status.CurrentL1) + require.Empty(t, metrics.results, "the event loop records the request result") + }) + } +} + +// TestFollowUpstreamHeadOriginNotFound checks that the heads, unlike CurrentL1, +// must all be verified: if the L1 origin of one is not found, nothing is applied. +func TestFollowUpstreamHeadOriginNotFound(t *testing.T) { + metrics := &followMetricsStub{} + driver := &Driver{ + driverCtx: context.Background(), + upstreamFollowSource: &stubUpstreamFollowSource{ + status: &sources.FollowStatus{ + LocalSafeL2: eth.L2BlockRef{Number: 3, L1Origin: l1Ref(3).ID()}, + SafeL2: eth.L2BlockRef{Number: 2, L1Origin: l1Ref(2).ID()}, + FinalizedL2: eth.L2BlockRef{Number: 1, L1Origin: l1Ref(1).ID()}, + CurrentL1: l1Ref(4), + }, + l1: map[uint64]eth.L1BlockRef{1: l1Ref(1), 2: l1Ref(2), 4: l1Ref(4)}, + }, + log: testlog.Logger(t, log.LevelError), + metrics: metrics, + } + + require.Nil(t, driver.followUpstream()) + require.Equal(t, []string{"error_l1_lookup"}, metrics.results) + require.Zero(t, metrics.currentL1NotFound) +} + +func TestStartFollowUpstreamFetchIsAsync(t *testing.T) { source := &blockingUpstreamFollowSource{ status: &sources.FollowStatus{ - LocalSafeL2: eth.L2BlockRef{Number: 3, L1Origin: ref(1)}, - SafeL2: eth.L2BlockRef{Number: 2, L1Origin: ref(2)}, - FinalizedL2: eth.L2BlockRef{Number: 1, L1Origin: ref(3)}, - CurrentL1: eth.L1BlockRef{Number: 4, Hash: common.Hash{4}}, + LocalSafeL2: eth.L2BlockRef{Number: 3, L1Origin: l1Ref(1).ID()}, + SafeL2: eth.L2BlockRef{Number: 2, L1Origin: l1Ref(2).ID()}, + FinalizedL2: eth.L2BlockRef{Number: 1, L1Origin: l1Ref(3).ID()}, + CurrentL1: l1Ref(4), }, started: make(chan struct{}, 1), release: make(chan struct{}),