Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions op-node/metrics/metrics.go
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ type Metricer interface {
SetSequencerState(active bool)
RecordPipelineReset()
RecordFollowSourceRequest(result string)
RecordFollowSourceCurrentL1NotFound()
RecordFollowSourceReorg(action string)
RecordSuperAuthorityReorgSignal(reason string)
RecordSequencingError()
Expand Down Expand Up @@ -92,6 +93,7 @@ type Metrics struct {

L2FollowSourceCache *metrics.CacheMetrics
FollowSourceRequests *prometheus.CounterVec
FollowSourceCurrentL1Miss prometheus.Counter
FollowSourceReorgs *prometheus.CounterVec
SuperAuthorityReorgSignals *prometheus.CounterVec

Expand Down Expand Up @@ -222,6 +224,7 @@ func NewMetrics(procName string, labels prometheus.Labels) *Metrics {
Name: "follow_source_requests_total",
Help: "Count of follow source requests by result",
}, []string{"result"}),
FollowSourceCurrentL1Miss: factory.NewCounter(prometheus.CounterOpts{Namespace: ns, Name: "follow_source_current_l1_not_found_total", Help: "Count of follow source polls whose upstream CurrentL1 was not found on our L1 chain and was dropped"}),
FollowSourceReorgs: factory.NewCounterVec(prometheus.CounterOpts{Namespace: ns, Name: "follow_source_reorgs_total", Help: "Count of follow source reorg decisions by action"}, []string{"action"}),
SuperAuthorityReorgSignals: factory.NewCounterVec(prometheus.CounterOpts{Namespace: ns, Name: "super_authority_reorg_signals_total", Help: "Count of super authority reorg signals by reason"}, []string{"reason"}),

Expand Down Expand Up @@ -580,6 +583,10 @@ func (m *Metrics) RecordFollowSourceRequest(result string) {
m.FollowSourceRequests.WithLabelValues(result).Inc()
}

func (m *Metrics) RecordFollowSourceCurrentL1NotFound() {
m.FollowSourceCurrentL1Miss.Inc()
}

func (m *Metrics) RecordFollowSourceReorg(action string) {
m.FollowSourceReorgs.WithLabelValues(action).Inc()
}
Expand Down Expand Up @@ -736,6 +743,9 @@ func (n *noopMetricer) RecordPipelineReset() {
func (n *noopMetricer) RecordFollowSourceRequest(result string) {
}

func (n *noopMetricer) RecordFollowSourceCurrentL1NotFound() {
}

func (n *noopMetricer) RecordFollowSourceReorg(action string) {
}

Expand Down
56 changes: 37 additions & 19 deletions op-node/rollup/driver/driver.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,11 @@ package driver

import (
"context"
"errors"
"fmt"
"time"

"github.com/ethereum/go-ethereum"
"github.com/ethereum/go-ethereum/common"

gosync "sync"
Expand Down Expand Up @@ -457,9 +459,11 @@ func (s *Driver) startFollowUpstreamFetch(resultCh chan<- *sources.FollowStatus)

// followUpstream fetches and validates external safe, finalized, and CurrentL1
// references when derivation is disabled (UnsafeOnly). It returns nil if the
// fetch fails or the external state is inconsistent; a non-nil status has
// passed all checks. It performs no engine or event side effects — the driver
// event loop applies those when it receives the returned status.
// fetch fails or the external state is inconsistent; the heads of a non-nil
// status have passed all checks. If our L1 chain does not have the external
// CurrentL1 yet, it returns the status with CurrentL1 cleared.
// It performs no engine or event side effects — the driver event loop applies
// those when it receives the returned status.
func (s *Driver) followUpstream() *sources.FollowStatus {
status, err := s.upstreamFollowSource.GetFollowStatus(s.driverCtx)
if err != nil {
Expand Down Expand Up @@ -530,22 +534,36 @@ func (s *Driver) followUpstream() *sources.FollowStatus {

if (status.CurrentL1 == eth.L1BlockRef{}) {
s.log.Debug("Follow Upstream: CurrentL1 not available")
} else {
eCurrentL1, err := s.upstreamFollowSource.L1BlockRefByNumber(s.driverCtx, status.CurrentL1.Number)
if err != nil {
s.log.Warn("Follow Upstream: Failed to look up external currentL1", "err", err)
s.metrics.RecordFollowSourceRequest("error_l1_lookup")
return nil
}
if eCurrentL1.Hash != status.CurrentL1.Hash {
s.log.Warn(
"Follow Upstream: Invalid external CurrentL1: L1 head mismatch",
"actual", eCurrentL1,
"expected", status.CurrentL1,
)
s.metrics.RecordFollowSourceRequest("error_l1_mismatch")
return nil
}
return status
}
eCurrentL1, err := s.upstreamFollowSource.L1BlockRefByNumber(s.driverCtx, status.CurrentL1.Number)
if errors.Is(err, ethereum.NotFound) {
// Our L1 RPC is behind the upstream's: it does not have this block yet, for example
// while a load-balanced L1 endpoint serves from a lagging backend. The L1 origins of
// the heads are verified, so apply the heads and drop only CurrentL1, which the event
// loop then leaves unchanged. The L1 blocks between the local-safe origin and
// CurrentL1 stay unverified until a later poll finds a CurrentL1 on our chain; if the
// upstream is on an L1 fork, that poll fails with a hash mismatch.
s.log.Info("Follow Upstream: External CurrentL1 not found on our L1 chain yet, not updating CurrentL1",
"currentL1", status.CurrentL1)
s.metrics.RecordFollowSourceCurrentL1NotFound()
withoutCurrentL1 := *status
withoutCurrentL1.CurrentL1 = eth.L1BlockRef{}
return &withoutCurrentL1
}
if err != nil {
s.log.Warn("Follow Upstream: Failed to look up external currentL1", "err", err)
s.metrics.RecordFollowSourceRequest("error_l1_lookup")
return nil
}
if eCurrentL1.Hash != status.CurrentL1.Hash {
s.log.Warn(
"Follow Upstream: Invalid external CurrentL1: L1 head mismatch",
"actual", eCurrentL1,
"expected", status.CurrentL1,
)
s.metrics.RecordFollowSourceRequest("error_l1_mismatch")
return nil
}
return status
}
152 changes: 144 additions & 8 deletions op-node/rollup/driver/follow_upstream_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,13 +3,15 @@ package driver
import (
"context"
"errors"
"fmt"
"testing"
"time"

"github.com/ethereum-optimism/optimism/op-service/eth"
"github.com/ethereum-optimism/optimism/op-service/log"
"github.com/ethereum-optimism/optimism/op-service/sources"
"github.com/ethereum-optimism/optimism/op-service/testlog"
"github.com/ethereum/go-ethereum"
"github.com/ethereum/go-ethereum/common"
"github.com/stretchr/testify/require"
)
Expand Down Expand Up @@ -41,23 +43,157 @@ func (s *blockingUpstreamFollowSource) L1BlockRefByNumber(_ context.Context, num
// methods panic via the nil embedded interface if called.
type followMetricsStub struct {
Metrics
results []string
results []string
currentL1NotFound int
}

func (m *followMetricsStub) RecordFollowSourceRequest(result string) {
m.results = append(m.results, result)
}

func TestStartFollowUpstreamFetchIsAsync(t *testing.T) {
ref := func(number uint64) eth.BlockID {
return eth.BlockID{Number: number, Hash: common.Hash{byte(number)}}
func (m *followMetricsStub) RecordFollowSourceCurrentL1NotFound() {
m.currentL1NotFound++
}

func l1Ref(number uint64) eth.L1BlockRef {
return eth.L1BlockRef{Number: number, Hash: common.Hash{byte(number)}}
}

// stubUpstreamFollowSource serves a fixed status and answers L1 lookups from our
// side of the chain: blocks in l1 are found, numbers in l1Errs fail, and any
// other number is not found.
type stubUpstreamFollowSource struct {
status *sources.FollowStatus
l1 map[uint64]eth.L1BlockRef
l1Errs map[uint64]error
}

func (s *stubUpstreamFollowSource) GetFollowStatus(context.Context) (*sources.FollowStatus, error) {
return s.status, nil
}

func (s *stubUpstreamFollowSource) L1BlockRefByNumber(_ context.Context, number uint64) (eth.L1BlockRef, error) {
if err, ok := s.l1Errs[number]; ok {
return eth.L1BlockRef{}, err
}
if ref, ok := s.l1[number]; ok {
return ref, nil
}
return eth.L1BlockRef{}, fmt.Errorf("failed to fetch header by num %d: %w", number, ethereum.NotFound)
}

// TestFollowUpstreamCurrentL1 checks that an upstream CurrentL1 our L1 chain does
// not have yet only drops CurrentL1: the verified heads are still applied, and the
// request is not counted as an error. Any other CurrentL1 failure still rejects
// the whole status.
func TestFollowUpstreamCurrentL1(t *testing.T) {
ourL1 := map[uint64]eth.L1BlockRef{1: l1Ref(1), 2: l1Ref(2), 3: l1Ref(3), 4: l1Ref(4)}
newStatus := func(currentL1 eth.L1BlockRef) *sources.FollowStatus {
return &sources.FollowStatus{
LocalSafeL2: eth.L2BlockRef{Number: 3, L1Origin: l1Ref(3).ID()},
SafeL2: eth.L2BlockRef{Number: 2, L1Origin: l1Ref(2).ID()},
FinalizedL2: eth.L2BlockRef{Number: 1, L1Origin: l1Ref(1).ID()},
CurrentL1: currentL1,
}
}

tests := []struct {
name string
currentL1 eth.L1BlockRef
l1Errs map[uint64]error
wantRejected string // request result if the status is rejected, empty if accepted
wantCurrentL1 eth.L1BlockRef
wantNotFound int
}{
{
name: "verified",
currentL1: l1Ref(4),
wantCurrentL1: l1Ref(4),
},
{
name: "not reported by upstream",
currentL1: eth.L1BlockRef{},
wantCurrentL1: eth.L1BlockRef{},
},
{
name: "ahead of our L1 view",
currentL1: l1Ref(5),
wantCurrentL1: eth.L1BlockRef{},
wantNotFound: 1,
},
{
name: "L1 lookup fails",
currentL1: l1Ref(4),
l1Errs: map[uint64]error{4: errors.New("connection refused")},
wantRejected: "error_l1_lookup",
},
{
name: "not on our L1 chain",
currentL1: eth.L1BlockRef{Number: 4, Hash: common.Hash{0xff}},
wantRejected: "error_l1_mismatch",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
upstream := newStatus(tt.currentL1)
metrics := &followMetricsStub{}
driver := &Driver{
driverCtx: context.Background(),
upstreamFollowSource: &stubUpstreamFollowSource{status: upstream, l1: ourL1, l1Errs: tt.l1Errs},
log: testlog.Logger(t, log.LevelError),
metrics: metrics,
}

status := driver.followUpstream()

require.Equal(t, tt.wantNotFound, metrics.currentL1NotFound)
require.Equal(t, tt.currentL1, upstream.CurrentL1, "must not modify the upstream status")
if tt.wantRejected != "" {
require.Nil(t, status)
require.Equal(t, []string{tt.wantRejected}, metrics.results)
return
}
require.NotNil(t, status)
require.Equal(t, upstream.LocalSafeL2, status.LocalSafeL2)
require.Equal(t, upstream.SafeL2, status.SafeL2)
require.Equal(t, upstream.FinalizedL2, status.FinalizedL2)
require.Equal(t, tt.wantCurrentL1, status.CurrentL1)
require.Empty(t, metrics.results, "the event loop records the request result")
})
}
}

// TestFollowUpstreamHeadOriginNotFound checks that the heads, unlike CurrentL1,
// must all be verified: if the L1 origin of one is not found, nothing is applied.
func TestFollowUpstreamHeadOriginNotFound(t *testing.T) {
metrics := &followMetricsStub{}
driver := &Driver{
driverCtx: context.Background(),
upstreamFollowSource: &stubUpstreamFollowSource{
status: &sources.FollowStatus{
LocalSafeL2: eth.L2BlockRef{Number: 3, L1Origin: l1Ref(3).ID()},
SafeL2: eth.L2BlockRef{Number: 2, L1Origin: l1Ref(2).ID()},
FinalizedL2: eth.L2BlockRef{Number: 1, L1Origin: l1Ref(1).ID()},
CurrentL1: l1Ref(4),
},
l1: map[uint64]eth.L1BlockRef{1: l1Ref(1), 2: l1Ref(2), 4: l1Ref(4)},
},
log: testlog.Logger(t, log.LevelError),
metrics: metrics,
}

require.Nil(t, driver.followUpstream())
require.Equal(t, []string{"error_l1_lookup"}, metrics.results)
require.Zero(t, metrics.currentL1NotFound)
}

func TestStartFollowUpstreamFetchIsAsync(t *testing.T) {
source := &blockingUpstreamFollowSource{
status: &sources.FollowStatus{
LocalSafeL2: eth.L2BlockRef{Number: 3, L1Origin: ref(1)},
SafeL2: eth.L2BlockRef{Number: 2, L1Origin: ref(2)},
FinalizedL2: eth.L2BlockRef{Number: 1, L1Origin: ref(3)},
CurrentL1: eth.L1BlockRef{Number: 4, Hash: common.Hash{4}},
LocalSafeL2: eth.L2BlockRef{Number: 3, L1Origin: l1Ref(1).ID()},
SafeL2: eth.L2BlockRef{Number: 2, L1Origin: l1Ref(2).ID()},
FinalizedL2: eth.L2BlockRef{Number: 1, L1Origin: l1Ref(3).ID()},
CurrentL1: l1Ref(4),
},
started: make(chan struct{}, 1),
release: make(chan struct{}),
Expand Down
Loading