From 40c2fba04939b852fae5c7865ca51e11d2e1cfa3 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Wed, 14 Oct 2026 06:26:58 -0700 Subject: [PATCH 1/8] op-devstack: add a windowed proof oracle to the L2 EL DSL Add L2ELNode.VerifyHistoricalProofsMatch, which sweeps the blocks a proofs-history store serves and, at every sampled block, verifies eth_getProof for a configured account set against the header's state root and against an independent reference node, requiring the L2ToL1MessagePasser storage root to equal the header's withdrawalsRoot. VerifyProofsMatchAt is the single-block form; ProofsSyncStatus and WaitForProofsStoreBlock expose debug_proofsSyncStatus. Move NormalizeProofResponse and VerifyProof out of the op-reth proofs e2e utils into the DSL (as VerifyAccountProof) and switch that suite to the DSL methods. Add OpRethConfig.ProofsHistoryWindow with OpRethWithProofsHistoryWindow so a test can shrink a node's proofs window, and OpRethWithEthProofWindow so a node without proofs history can serve historical eth_getProof. Add the proofs_history_match acceptance test, which runs transfers and contract writes and sweeps a proofs follower against a sequencer without proofs history over the persistence-threshold x proofs-path matrix. Co-Authored-By: Claude Fable 5.1 (cherry picked from commit 42fd696271446b2da387b0e53e0fe4e667f03022) (cherry picked from commit 021f7115a6dfd9ccb7b624bcf14a7c0df7e13492) --- .../proofs_history_match_test.go | 167 ++++++++ op-devstack/dsl/l2_el_proofs.go | 362 ++++++++++++++++++ op-devstack/dsl/l2_el_proofs_test.go | 243 ++++++++++++ op-devstack/sysgo/l2_el_opreth.go | 19 + op-devstack/sysgo/l2_el_opreth_test.go | 53 +++ op-devstack/sysgo/mixed_runtime.go | 26 +- .../tests/proofs/core/account_proofs_test.go | 22 +- .../tests/proofs/core/execute_payload_test.go | 9 +- .../proofs/core/execution_witness_test.go | 3 +- .../tests/proofs/core/resyncing_test.go | 8 +- .../tests/proofs/core/simple_storage_test.go | 27 +- rust/op-reth/tests/proofs/prune/prune_test.go | 32 +- rust/op-reth/tests/proofs/reorg/reorg_test.go | 5 +- rust/op-reth/tests/proofs/utils/proof.go | 166 -------- rust/op-reth/tests/proofs/utils/utils.go | 29 -- 15 files changed, 917 insertions(+), 254 deletions(-) create mode 100644 op-acceptance-tests/tests/proofs_history_match/proofs_history_match_test.go create mode 100644 op-devstack/dsl/l2_el_proofs.go create mode 100644 op-devstack/dsl/l2_el_proofs_test.go create mode 100644 op-devstack/sysgo/l2_el_opreth_test.go delete mode 100644 rust/op-reth/tests/proofs/utils/proof.go diff --git a/op-acceptance-tests/tests/proofs_history_match/proofs_history_match_test.go b/op-acceptance-tests/tests/proofs_history_match/proofs_history_match_test.go new file mode 100644 index 00000000000..b2a8ca6dc42 --- /dev/null +++ b/op-acceptance-tests/tests/proofs_history_match/proofs_history_match_test.go @@ -0,0 +1,167 @@ +package proofs_history_match + +import ( + "fmt" + "testing" + + "github.com/ethereum-optimism/optimism/op-devstack/devtest" + "github.com/ethereum-optimism/optimism/op-devstack/dsl" + "github.com/ethereum-optimism/optimism/op-devstack/presets" + "github.com/ethereum-optimism/optimism/op-devstack/sysgo" + "github.com/ethereum-optimism/optimism/op-service/eth" + "github.com/ethereum-optimism/optimism/op-service/txplan" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/vm" + "github.com/ethereum/go-ethereum/core/vm/program" +) + +const ( + // referenceEthProofWindow lets the reference node answer eth_getProof for every block of the + // run from reth's own historical state; reth's default window of zero rejects them all. + referenceEthProofWindow = 1_000_000 + workloadRounds = 4 +) + +// proofsHistoryCase is one cell of the matrix the proofs follower runs under. +type proofsHistoryCase struct { + persistenceThreshold uint64 + stateMaskingBlocks uint64 + verificationInterval uint64 +} + +func (c proofsHistoryCase) name() string { + proofsPath := "index" + if c.verificationInterval != 0 { + proofsPath = "execute" + } + return fmt.Sprintf("threshold_%d-masking_%d-%s", c.persistenceThreshold, c.stateMaskingBlocks, proofsPath) +} + +func (c proofsHistoryCase) opRethArgs() []string { + args := []string{ + fmt.Sprintf("--engine.persistence-threshold=%d", c.persistenceThreshold), + fmt.Sprintf("--proofs-history.verification-interval=%d", c.verificationInterval), + } + if c.stateMaskingBlocks != 0 { + args = append(args, fmt.Sprintf("--engine.num-state-masking-blocks=%d", c.stateMaskingBlocks)) + } + return args +} + +func proofsHistoryMatrix() []proofsHistoryCase { + var cases []proofsHistoryCase + for _, persistenceThreshold := range []uint64{7, 50} { + for _, stateMaskingBlocks := range []uint64{0, 30} { + for _, verificationInterval := range []uint64{0, 1} { + cases = append(cases, proofsHistoryCase{ + persistenceThreshold: persistenceThreshold, + stateMaskingBlocks: stateMaskingBlocks, + verificationInterval: verificationInterval, + }) + } + } + } + return cases +} + +type proofsHistorySystem struct { + FunderL2 *dsl.FunderEOA + ProofsFollower *dsl.L2ELNode + Reference *dsl.L2ELNode +} + +// newProofsHistorySystem starts a sequencer without proofs history, which serves as the +// independent eth_getProof reference, and a follower running proofs history under the case's +// reth and ExEx settings. +func newProofsHistorySystem(t devtest.T, c proofsHistoryCase) *proofsHistorySystem { + sysgo.SkipOnOpGeth(t, "proofs history is an op-reth feature") + clKind := sysgo.ResolveMixedL2CLKind() + + runtime := sysgo.NewMixedSingleChainRuntime(t, sysgo.MixedSingleChainPresetConfig{ + NodeSpecs: []sysgo.MixedSingleChainNodeSpec{ + { + CLKey: "sequencer", + CLKind: clKind, + ELKey: "sequencer", + ELKind: sysgo.MixedL2ELOpReth, + IsSequencer: true, + OpRethOpts: []sysgo.OpRethOption{ + sysgo.OpRethWithoutProofsHistory(), + sysgo.OpRethWithEthProofWindow(referenceEthProofWindow), + }, + }, + { + CLKey: "follower", + CLKind: clKind, + ELKey: "proofs-follower", + ELKind: sysgo.MixedL2ELOpReth, + OpRethOpts: []sysgo.OpRethOption{sysgo.OpRethWithExtraArgs(c.opRethArgs()...)}, + }, + }, + }) + frontends := presets.NewMixedSingleChainFrontends(t, runtime) + t.Require().Len(frontends.Nodes, 2, "expected the sequencer and the proofs follower") + + var follower *dsl.L2ELNode + for _, node := range frontends.Nodes { + if !node.Spec.IsSequencer { + follower = node.EL + } + } + t.Require().NotNil(follower, "missing the proofs follower") + + return &proofsHistorySystem{ + FunderL2: frontends.FunderL2, + ProofsFollower: follower, + Reference: frontends.L2Network.PrimaryEL(), + } +} + +// deployBlockRecorder deploys a contract that writes the block number to slot 0 and the block +// timestamp to slot 1 on every call, so every call changes its storage. +func deployBlockRecorder(t devtest.T, deployer *dsl.EOA) common.Address { + runtimeCode := program.New(). + Op(vm.NUMBER).Push(0).Op(vm.SSTORE). + Op(vm.TIMESTAMP).Push(1).Op(vm.SSTORE). + Bytes() + initCode := program.New().ReturnViaCodeCopy(runtimeCode).Bytes() + + tx := deployer.Transact(deployer.Plan(), txplan.WithData(initCode)) + receipt, err := tx.Included.Eval(t.Ctx()) + t.Require().NoError(err, "block recorder deployment must be included") + return receipt.ContractAddress +} + +// TestProofsHistoryFollowerServesMatchingHistoricalProofs runs transfers and contract writes +// against a sequencer that serves eth_getProof from its own historical state, then sweeps every +// block of a proofs-history follower's window: each of the follower's proofs must verify against +// the block header and equal the sequencer's. +func TestProofsHistoryFollowerServesMatchingHistoricalProofs(gt *testing.T) { + t := devtest.ParallelT(gt) + for _, c := range proofsHistoryMatrix() { + t.Run(c.name(), func(t devtest.T) { + t.Parallel() + if c.stateMaskingBlocks != 0 { + t.Skipf("state masking is compiled out of the reth 2.5.2 on develop and #23046 rejects a nonzero --engine.num-state-masking-blocks; re-enable with ethereum-optimism/protocol-team#457") + } + sys := newProofsHistorySystem(t, c) + + alice := sys.FunderL2.NewFundedEOA(eth.OneEther) + bob := sys.FunderL2.NewFundedEOA(eth.OneEther) + recorder := deployBlockRecorder(t, alice) + for range workloadRounds { + alice.Transfer(bob.Address(), eth.OneHundredthEther) + bob.Transfer(alice.Address(), eth.OneHundredthEther) + alice.Transact(alice.Plan(), txplan.WithTo(&recorder)) + } + + sys.ProofsFollower.VerifyHistoricalProofsMatch(sys.Reference, func(opts *dsl.HistoricalProofsOpts) { + opts.Accounts = []dsl.ProofAccount{ + {Address: alice.Address()}, + {Address: bob.Address()}, + {Address: recorder, StorageKeys: []common.Hash{common.HexToHash("0x0"), common.HexToHash("0x1")}}, + } + }) + }) + } +} diff --git a/op-devstack/dsl/l2_el_proofs.go b/op-devstack/dsl/l2_el_proofs.go new file mode 100644 index 00000000000..804006f1f6c --- /dev/null +++ b/op-devstack/dsl/l2_el_proofs.go @@ -0,0 +1,362 @@ +package dsl + +import ( + "bytes" + "context" + "fmt" + "time" + + "github.com/ethereum-optimism/optimism/op-core/predeploys" + "github.com/ethereum-optimism/optimism/op-e2e/e2eutils/wait" + "github.com/ethereum-optimism/optimism/op-service/eth" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/ethdb/memorydb" + "github.com/ethereum/go-ethereum/rlp" + "github.com/ethereum/go-ethereum/trie" +) + +// DefaultProofsCatchUpTimeout bounds how long a proofs-history store may lag behind the chain +// before a proof verification fails. +const DefaultProofsCatchUpTimeout = 30 * time.Second + +// ProofsSyncStatus is the window of blocks an op-reth proofs-history store serves, as reported by +// debug_proofsSyncStatus. Both bounds are nil until the store has indexed its first block. +type ProofsSyncStatus struct { + Earliest *uint64 `json:"earliest"` + Latest *uint64 `json:"latest"` +} + +func (s ProofsSyncStatus) String() string { + if s.Earliest == nil || s.Latest == nil { + return "uninitialized" + } + return fmt.Sprintf("[%d, %d]", *s.Earliest, *s.Latest) +} + +func (s ProofsSyncStatus) serves(block uint64) bool { + return s.Earliest != nil && s.Latest != nil && *s.Earliest <= block && block <= *s.Latest +} + +// ProofAccount names an account to prove, with the storage slots whose proofs are requested. +type ProofAccount struct { + Address common.Address + StorageKeys []common.Hash +} + +// HistoricalProofsOpts configures VerifyHistoricalProofsMatch. +type HistoricalProofsOpts struct { + // Accounts are proven at every sampled block. L2ToL1MessagePasser is always proven and need + // not be listed. + Accounts []ProofAccount + // CatchUpTimeout bounds how long the proofs store and the reference node may lag behind a + // sampled block before the sweep fails. + CatchUpTimeout time.Duration + // FirstBlock and LastBlock bound the sweep. Zero selects the matching edge of the proofs + // window reported by debug_proofsSyncStatus when the sweep starts. + FirstBlock uint64 + LastBlock uint64 + // MaxSampledBlocks bounds how many blocks of the range are compared; zero compares every + // block. The first and last block of the range are always compared. + MaxSampledBlocks int +} + +// ProofsSyncStatus reports the window of blocks this node's proofs-history store serves. +func (el *L2ELNode) ProofsSyncStatus() (ProofsSyncStatus, error) { + ctx, cancel := context.WithTimeout(el.ctx, DefaultTimeout) + defer cancel() + var status ProofsSyncStatus + err := el.inner.L2EthClient().RPC().CallContext(ctx, &status, "debug_proofsSyncStatus") + return status, err +} + +// WaitForProofsStoreBlock waits until this node's proofs-history store serves target, so the RPCs +// answered from the store (eth_getProof, debug_executePayload, debug_executionWitness) can be +// called for it. The store indexes blocks asynchronously, so the chain head leads the store. +func (el *L2ELNode) WaitForProofsStoreBlock(target uint64, timeout time.Duration) ProofsSyncStatus { + ctx, cancel := context.WithTimeout(el.ctx, timeout) + defer cancel() + logger := el.log.With("name", el.inner.Name(), "chain", el.ChainID(), "target", target, "timeout", timeout) + logger.Info("Waiting for proofs store to index block") + + var lastStatus ProofsSyncStatus + var lastErr error + err := wait.For(ctx, 200*time.Millisecond, func() (bool, error) { + status, err := el.ProofsSyncStatus() + if err != nil { + lastErr = err + logger.Warn("debug_proofsSyncStatus failed; will retry", "err", err) + return false, nil + } + lastStatus, lastErr = status, nil + if status.Latest != nil && *status.Latest >= target { + logger.Info("Proofs store indexed block", "window", status) + return true, nil + } + logger.Info("Proofs store behind target", "window", status) + return false, nil + }) + el.require.NoErrorf(err, "proofs store on %s did not index block %d within %s; last window %s; last error: %v", + el.inner.Name(), target, timeout, lastStatus, lastErr) + return lastStatus +} + +// VerifyHistoricalProofsMatch sweeps the blocks this node's proofs-history store serves and, at +// every sampled block, requires eth_getProof for each configured account to verify against the +// block's state root and to equal the proof served by reference. The L2ToL1MessagePasser proof is +// always included, and its storage root must equal the header's withdrawalsRoot where the header +// carries one. +// +// reference must answer eth_getProof from its own historical state rather than from a +// proofs-history store, since two stores can agree on a wrong proof. +func (el *L2ELNode) VerifyHistoricalProofsMatch(reference *L2ELNode, opts ...func(*HistoricalProofsOpts)) { + cfg := HistoricalProofsOpts{CatchUpTimeout: DefaultProofsCatchUpTimeout} + for _, opt := range opts { + opt(&cfg) + } + logger := el.log.With("name", el.inner.Name(), "chain", el.ChainID(), "reference", reference.inner.Name()) + + el.requireIndependentProofReference(reference) + + window := el.WaitForProofsStoreBlock(cfg.LastBlock, cfg.CatchUpTimeout) + first, last := cfg.FirstBlock, cfg.LastBlock + if first == 0 { + first = *window.Earliest + } + if last == 0 { + last = *window.Latest + } + el.require.LessOrEqualf(first, last, "historical proofs sweep range [%d, %d] is empty; proofs window %s", first, last, window) + blocks := sampleBlockNumbers(first, last, cfg.MaxSampledBlocks) + accounts := append([]ProofAccount{{Address: predeploys.L2ToL1MessagePasserAddr}}, cfg.Accounts...) + + logger.Info("Sweeping historical proofs", "window", window, "first", first, "last", last, + "blocks", len(blocks), "accounts", len(accounts)) + for _, block := range blocks { + el.verifyProofsMatchAt(reference, block, accounts, cfg.CatchUpTimeout) + } + logger.Info("Historical proofs match the reference", "first", first, "last", last, + "blocks", len(blocks), "accounts", len(accounts)) +} + +// VerifyProofsMatchAt is VerifyHistoricalProofsMatch for one block and an explicit account list, +// waiting up to DefaultProofsCatchUpTimeout for the proofs store to serve the block. +func (el *L2ELNode) VerifyProofsMatchAt(reference *L2ELNode, block uint64, accounts ...ProofAccount) { + el.WaitForProofsStoreBlock(block, DefaultProofsCatchUpTimeout) + el.verifyProofsMatchAt(reference, block, accounts, DefaultProofsCatchUpTimeout) +} + +func (el *L2ELNode) verifyProofsMatchAt(reference *L2ELNode, block uint64, accounts []ProofAccount, catchUpTimeout time.Duration) { + window, err := el.ProofsSyncStatus() + el.require.NoErrorf(err, "debug_proofsSyncStatus failed on %s", el.inner.Name()) + el.require.Truef(window.serves(block), "proofs store on %s does not serve block %d: window %s", el.inner.Name(), block, window) + + header := el.awaitHeaderByNumber(block, catchUpTimeout) + referenceHeader := reference.awaitHeaderByNumber(block, catchUpTimeout) + el.require.Equalf(referenceHeader.Hash(), header.Hash(), "block %d: %s and reference %s disagree on the block hash", + block, el.inner.Name(), reference.inner.Name()) + + for _, account := range accounts { + el.verifyAccountProofMatchesAt(reference, header, account, catchUpTimeout) + } + el.log.Info("Historical proofs verified", "name", el.inner.Name(), "block", block, "accounts", len(accounts)) +} + +func (el *L2ELNode) verifyAccountProofMatchesAt(reference *L2ELNode, header eth.BlockInfo, account ProofAccount, catchUpTimeout time.Duration) { + block := header.NumberU64() + proof := el.awaitProof(header, account, catchUpTimeout) + referenceProof := reference.awaitProof(header, account, catchUpTimeout) + NormalizeProofResponse(proof) + NormalizeProofResponse(referenceProof) + + el.require.NoErrorf(VerifyAccountProof(proof, header.Root()), + "block %d account %s: eth_getProof from %s does not verify against state root %s", + block, account.Address, el.inner.Name(), header.Root()) + el.require.NoErrorf(VerifyAccountProof(referenceProof, header.Root()), + "block %d account %s: eth_getProof from reference %s does not verify against state root %s", + block, account.Address, reference.inner.Name(), header.Root()) + el.require.Equalf(referenceProof, proof, "block %d account %s: eth_getProof from %s differs from reference %s", + block, account.Address, el.inner.Name(), reference.inner.Name()) + + if account.Address == predeploys.L2ToL1MessagePasserAddr && header.WithdrawalsRoot() != nil { + el.require.Equalf(*header.WithdrawalsRoot(), proof.StorageHash, + "block %d: L2ToL1MessagePasser storage root from %s does not equal the header's withdrawalsRoot", block, el.inner.Name()) + } + el.log.Debug("Historical proof matches the reference", "name", el.inner.Name(), "block", block, + "account", account.Address, "storage_keys", len(account.StorageKeys)) +} + +// requireIndependentProofReference fails when reference serves eth_getProof from a proofs-history +// store, which every node that exposes a proofs window does. +func (el *L2ELNode) requireIndependentProofReference(reference *L2ELNode) { + status, err := reference.ProofsSyncStatus() + if err != nil { + el.log.Debug("Reference node has no proofs-history store", "reference", reference.inner.Name(), "err", err) + return + } + el.require.Nilf(status.Latest, "reference %s runs proofs history (window %s); the sweep needs a reference that serves eth_getProof from its own historical state", + reference.inner.Name(), status) +} + +func (el *L2ELNode) awaitHeaderByNumber(number uint64, timeout time.Duration) eth.BlockInfo { + ctx, cancel := context.WithTimeout(el.ctx, timeout) + defer cancel() + var header eth.BlockInfo + var lastErr error + err := wait.For(ctx, 200*time.Millisecond, func() (bool, error) { + info, err := el.inner.L2EthClient().InfoByNumber(ctx, number) + if err != nil { + lastErr = err + el.log.Debug("Header lookup failed; will retry", "name", el.inner.Name(), "block", number, "err", err) + return false, nil + } + header = info + return true, nil + }) + el.require.NoErrorf(err, "%s did not serve the header of block %d within %s; last error: %v", el.inner.Name(), number, timeout, lastErr) + return header +} + +// awaitProof fetches eth_getProof by block hash, so a block that leaves the canonical chain while +// the sweep runs fails the lookup rather than returning the proof of its replacement. +func (el *L2ELNode) awaitProof(header eth.BlockInfo, account ProofAccount, timeout time.Duration) *eth.AccountResult { + ctx, cancel := context.WithTimeout(el.ctx, timeout) + defer cancel() + var proof *eth.AccountResult + var lastErr error + err := wait.For(ctx, 200*time.Millisecond, func() (bool, error) { + result, err := el.inner.L2EthClient().GetProof(ctx, account.Address, account.StorageKeys, header.Hash().String()) + if err != nil { + lastErr = err + el.log.Debug("eth_getProof failed; will retry", "name", el.inner.Name(), "block", header.NumberU64(), + "account", account.Address, "err", err) + return false, nil + } + proof = result + return true, nil + }) + el.require.NoErrorf(err, "%s did not serve eth_getProof for account %s at block %d within %s; last error: %v", + el.inner.Name(), account.Address, header.NumberU64(), timeout, lastErr) + return proof +} + +// sampleBlockNumbers returns at most maxSamples evenly spaced block numbers in [first, last], +// always including both edges. A non-positive maxSamples returns every block. +func sampleBlockNumbers(first, last uint64, maxSamples int) []uint64 { + count := last - first + 1 + if maxSamples <= 0 || uint64(maxSamples) >= count { + blocks := make([]uint64, 0, count) + for block := first; block <= last; block++ { + blocks = append(blocks, block) + } + return blocks + } + if maxSamples < 2 { + maxSamples = 2 + } + blocks := make([]uint64, 0, maxSamples) + for i := range maxSamples { + blocks = append(blocks, first+(count-1)*uint64(i)/uint64(maxSamples-1)) + } + return blocks +} + +// NormalizeProofResponse rewrites an eth_getProof response into one canonical encoding so proofs +// from different clients compare equal: an empty storage proof, which reth encodes as ["0x80"], +// becomes empty; a zero code hash becomes keccak256("") and a zero storage hash becomes the empty +// trie root, which is how reth encodes both; storage keys are left-padded to 32 bytes; and an +// absent storage proof list becomes nil. +func NormalizeProofResponse(res *eth.AccountResult) { + for i := range res.StorageProof { + if len(res.StorageProof[i].Proof) == 1 && bytes.Equal(res.StorageProof[i].Proof[0], []byte{0x80}) { + res.StorageProof[i].Proof = []hexutil.Bytes{} + } + res.StorageProof[i].Key = common.BytesToHash(res.StorageProof[i].Key).Bytes() + } + if len(res.StorageProof) == 0 { + res.StorageProof = nil + } + if res.CodeHash == (common.Hash{}) { + res.CodeHash = crypto.Keccak256Hash(nil) + } + if res.StorageHash == (common.Hash{}) { + res.StorageHash = types.EmptyRootHash + } +} + +// VerifyAccountProof verifies an eth_getProof response against a state root. Unlike +// eth.AccountResult.Verify it accepts an empty storage trie, whose proofs carry no nodes, and a +// non-existence proof for an account the response reports as empty. +func VerifyAccountProof(res *eth.AccountResult, stateRoot common.Hash) error { + if res.StorageHash != types.EmptyRootHash { + for i, entry := range res.StorageProof { + db := memorydb.New() + for j, encodedNode := range entry.Proof { + nodeKey := encodedNode + if len(encodedNode) >= 32 { // small MPT nodes are not hashed + nodeKey = crypto.Keccak256(encodedNode) + } + if err := db.Put(nodeKey, encodedNode); err != nil { + return fmt.Errorf("failed to load storage proof node %d of storage value %d into mem db: %w", j, i, err) + } + } + path := crypto.Keccak256(entry.Key) + val, err := trie.VerifyProof(res.StorageHash, path, db) + if err != nil { + return fmt.Errorf("failed to verify storage value %d with key %s (path %x) in storage trie %s: %w", i, entry.Key.String(), path, res.StorageHash, err) + } + if val == nil && entry.Value.ToInt().Cmp(common.Big0) == 0 { // empty storage is zero by default + continue + } + comparison, err := rlp.EncodeToBytes(entry.Value.ToInt().Bytes()) + if err != nil { + return fmt.Errorf("failed to encode storage value %d with key %s (path %x) in storage trie %s: %w", i, entry.Key.String(), path, res.StorageHash, err) + } + if !bytes.Equal(val, comparison) { + return fmt.Errorf("value %d in storage proof does not match proven value at key %s (path %x)", i, entry.Key.String(), path) + } + } + } + + accountClaimed := []any{uint64(res.Nonce), res.Balance.ToInt().Bytes(), res.StorageHash, res.CodeHash} + accountClaimedValue, err := rlp.EncodeToBytes(accountClaimed) + if err != nil { + return fmt.Errorf("failed to encode account from retrieved values: %w", err) + } + + db := memorydb.New() + for i, encodedNode := range res.AccountProof { + nodeKey := encodedNode + if len(encodedNode) >= 32 { // small MPT nodes are not hashed + nodeKey = crypto.Keccak256(encodedNode) + } + if err := db.Put(nodeKey, encodedNode); err != nil { + return fmt.Errorf("failed to load account proof node %d into mem db: %w", i, err) + } + } + path := crypto.Keccak256(res.Address[:]) + accountProofValue, err := trie.VerifyProof(stateRoot, path, db) + if err != nil { + return fmt.Errorf("failed to verify account value with key %s (path %x) in account trie %s: %w", res.Address, path, stateRoot, err) + } + + // A non-existence proof is only valid when the response reports an empty account. + if len(accountProofValue) == 0 { + isEmpty := res.Nonce == 0 && + res.Balance.ToInt().Sign() == 0 && + (res.StorageHash == types.EmptyRootHash || res.StorageHash == common.Hash{}) && + (res.CodeHash == crypto.Keccak256Hash(nil) || res.CodeHash == common.Hash{}) + if isEmpty { + return nil + } + } + + if !bytes.Equal(accountClaimedValue, accountProofValue) { + return fmt.Errorf("account proof does not match the claimed account values:\n"+ + " claimed: %x\n"+ + " proof: %x", accountClaimedValue, accountProofValue) + } + return nil +} diff --git a/op-devstack/dsl/l2_el_proofs_test.go b/op-devstack/dsl/l2_el_proofs_test.go new file mode 100644 index 00000000000..c89339582c9 --- /dev/null +++ b/op-devstack/dsl/l2_el_proofs_test.go @@ -0,0 +1,243 @@ +package dsl + +import ( + "math/big" + "testing" + + "github.com/ethereum-optimism/optimism/op-service/eth" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/rlp" + "github.com/ethereum/go-ethereum/trie" + "github.com/ethereum/go-ethereum/trie/trienode" + "github.com/ethereum/go-ethereum/triedb" + "github.com/stretchr/testify/require" +) + +// provenAccount is one account of a synthetic state trie. Zero-valued slots are left out of the +// storage trie, as a client would. +type provenAccount struct { + address common.Address + balance *big.Int + nonce uint64 + slots map[common.Hash]*big.Int +} + +var ( + alice = provenAccount{ + address: common.HexToAddress("0xa11ce"), + balance: big.NewInt(1_000), + nonce: 3, + slots: map[common.Hash]*big.Int{ + common.HexToHash("0x0"): big.NewInt(42), + common.HexToHash("0x1"): big.NewInt(0), + }, + } + bob = provenAccount{ + address: common.HexToAddress("0xb0b"), + balance: big.NewInt(7), + nonce: 1, + } + carol = provenAccount{ + address: common.HexToAddress("0xca201"), + balance: big.NewInt(99), + nonce: 0, + slots: map[common.Hash]*big.Int{common.HexToHash("0x5"): big.NewInt(5)}, + } + nobody = provenAccount{address: common.HexToAddress("0x1234"), balance: big.NewInt(0)} +) + +func newTestTrie() *trie.Trie { + return trie.NewEmpty(triedb.NewDatabase(rawdb.NewMemoryDatabase(), nil)) +} + +func proveKey(t *testing.T, tr *trie.Trie, key []byte) []hexutil.Bytes { + var nodes trienode.ProofList + require.NoError(t, tr.Prove(crypto.Keccak256(key), &nodes)) + proof := make([]hexutil.Bytes, len(nodes)) + for i, node := range nodes { + proof[i] = hexutil.Bytes(node) + } + return proof +} + +func storageTrieOf(t *testing.T, account provenAccount) *trie.Trie { + storageTrie := newTestTrie() + for slot, value := range account.slots { + if value.Sign() == 0 { + continue + } + encoded, err := rlp.EncodeToBytes(value.Bytes()) + require.NoError(t, err) + storageTrie.MustUpdate(crypto.Keccak256(slot.Bytes()), encoded) + } + return storageTrie +} + +// stateTrieOf builds a state trie holding every account in accounts. +func stateTrieOf(t *testing.T, accounts ...provenAccount) *trie.Trie { + stateTrie := newTestTrie() + for _, account := range accounts { + encoded, err := rlp.EncodeToBytes([]any{account.nonce, account.balance.Bytes(), storageTrieOf(t, account).Hash(), crypto.Keccak256Hash(nil)}) + require.NoError(t, err) + stateTrie.MustUpdate(crypto.Keccak256(account.address.Bytes()), encoded) + } + return stateTrie +} + +// proofOf returns what eth_getProof would answer for account against stateTrie, whether or not +// the trie holds the account. +func proofOf(t *testing.T, stateTrie *trie.Trie, account provenAccount) *eth.AccountResult { + storageTrie := storageTrieOf(t, account) + result := ð.AccountResult{ + AccountProof: proveKey(t, stateTrie, account.address.Bytes()), + Address: account.address, + Balance: (*hexutil.Big)(account.balance), + CodeHash: crypto.Keccak256Hash(nil), + Nonce: hexutil.Uint64(account.nonce), + StorageHash: storageTrie.Hash(), + } + for slot, value := range account.slots { + result.StorageProof = append(result.StorageProof, eth.StorageProofEntry{ + Key: slot.Bytes(), + Value: (hexutil.Big)(*value), + Proof: proveKey(t, storageTrie, slot.Bytes()), + }) + } + return result +} + +func TestVerifyAccountProof(t *testing.T) { + stateTrie := stateTrieOf(t, alice, bob, carol) + stateRoot := stateTrie.Hash() + + for _, tc := range []struct { + name string + proof func() *eth.AccountResult + root common.Hash + wantErr string + }{ + {name: "account with storage verifies", proof: func() *eth.AccountResult { return proofOf(t, stateTrie, alice) }, root: stateRoot}, + {name: "account without storage verifies", proof: func() *eth.AccountResult { return proofOf(t, stateTrie, bob) }, root: stateRoot}, + {name: "absent account reported empty verifies", proof: func() *eth.AccountResult { return proofOf(t, stateTrie, nobody) }, root: stateRoot}, + { + name: "tampered balance fails", + proof: func() *eth.AccountResult { + proof := proofOf(t, stateTrie, alice) + proof.Balance = (*hexutil.Big)(big.NewInt(1_001)) + return proof + }, + root: stateRoot, + wantErr: "account proof does not match the claimed account values", + }, + { + name: "tampered storage value fails", + proof: func() *eth.AccountResult { + proof := proofOf(t, stateTrie, carol) + proof.StorageProof[0].Value = (hexutil.Big)(*big.NewInt(6)) + return proof + }, + root: stateRoot, + wantErr: "does not match proven value", + }, + { + name: "absent account claiming a balance fails", + proof: func() *eth.AccountResult { + proof := proofOf(t, stateTrie, nobody) + proof.Balance = (*hexutil.Big)(big.NewInt(1)) + return proof + }, + root: stateRoot, + wantErr: "account proof does not match the claimed account values", + }, + { + name: "wrong state root fails", + proof: func() *eth.AccountResult { return proofOf(t, stateTrie, alice) }, + root: common.HexToHash("0xdead"), + wantErr: "failed to verify account value", + }, + } { + t.Run(tc.name, func(t *testing.T) { + err := VerifyAccountProof(tc.proof(), tc.root) + if tc.wantErr == "" { + require.NoError(t, err) + return + } + require.ErrorContains(t, err, tc.wantErr) + }) + } +} + +func TestNormalizeProofResponse(t *testing.T) { + emptyCodeHash := crypto.Keccak256Hash(nil) + someHash := common.HexToHash("0xabc") + paddedKey := common.HexToHash("0x1").Bytes() + + for _, tc := range []struct { + name string + in eth.AccountResult + want eth.AccountResult + }{ + { + name: "reth empty storage proof becomes empty", + in: eth.AccountResult{CodeHash: someHash, StorageHash: someHash, StorageProof: []eth.StorageProofEntry{{Key: paddedKey, Proof: []hexutil.Bytes{{0x80}}}}}, + want: eth.AccountResult{CodeHash: someHash, StorageHash: someHash, StorageProof: []eth.StorageProofEntry{{Key: paddedKey, Proof: []hexutil.Bytes{}}}}, + }, + { + name: "short storage key is padded to 32 bytes", + in: eth.AccountResult{CodeHash: someHash, StorageHash: someHash, StorageProof: []eth.StorageProofEntry{{Key: []byte{0x1}, Proof: []hexutil.Bytes{}}}}, + want: eth.AccountResult{CodeHash: someHash, StorageHash: someHash, StorageProof: []eth.StorageProofEntry{{Key: paddedKey, Proof: []hexutil.Bytes{}}}}, + }, + { + name: "zero code hash becomes the empty code hash", + in: eth.AccountResult{StorageHash: someHash}, + want: eth.AccountResult{CodeHash: emptyCodeHash, StorageHash: someHash}, + }, + { + name: "zero storage hash becomes the empty trie root", + in: eth.AccountResult{CodeHash: someHash}, + want: eth.AccountResult{CodeHash: someHash, StorageHash: types.EmptyRootHash}, + }, + { + name: "empty storage proof list becomes nil", + in: eth.AccountResult{CodeHash: someHash, StorageHash: someHash, StorageProof: []eth.StorageProofEntry{}}, + want: eth.AccountResult{CodeHash: someHash, StorageHash: someHash}, + }, + { + name: "non-empty values are kept", + in: eth.AccountResult{CodeHash: someHash, StorageHash: someHash, StorageProof: []eth.StorageProofEntry{{Key: paddedKey, Proof: []hexutil.Bytes{{0x80}, {0x81}}}}}, + want: eth.AccountResult{CodeHash: someHash, StorageHash: someHash, StorageProof: []eth.StorageProofEntry{{Key: paddedKey, Proof: []hexutil.Bytes{{0x80}, {0x81}}}}}, + }, + } { + t.Run(tc.name, func(t *testing.T) { + got := tc.in + NormalizeProofResponse(&got) + require.Equal(t, tc.want, got) + }) + } +} + +func TestSampleBlockNumbers(t *testing.T) { + for _, tc := range []struct { + name string + first uint64 + last uint64 + maxSamples int + want []uint64 + }{ + {name: "zero bound returns every block", first: 0, last: 4, maxSamples: 0, want: []uint64{0, 1, 2, 3, 4}}, + {name: "single block", first: 5, last: 5, maxSamples: 0, want: []uint64{5}}, + {name: "bound above count returns every block", first: 0, last: 4, maxSamples: 20, want: []uint64{0, 1, 2, 3, 4}}, + {name: "bound of two keeps the edges", first: 0, last: 9, maxSamples: 2, want: []uint64{0, 9}}, + {name: "bound of one still keeps both edges", first: 0, last: 9, maxSamples: 1, want: []uint64{0, 9}}, + {name: "samples are evenly spaced", first: 0, last: 100, maxSamples: 5, want: []uint64{0, 25, 50, 75, 100}}, + {name: "uneven spacing rounds down and keeps the last block", first: 10, last: 13, maxSamples: 3, want: []uint64{10, 11, 13}}, + } { + t.Run(tc.name, func(t *testing.T) { + require.Equal(t, tc.want, sampleBlockNumbers(tc.first, tc.last, tc.maxSamples)) + }) + } +} diff --git a/op-devstack/sysgo/l2_el_opreth.go b/op-devstack/sysgo/l2_el_opreth.go index 959204e81be..7a3736f3d6f 100644 --- a/op-devstack/sysgo/l2_el_opreth.go +++ b/op-devstack/sysgo/l2_el_opreth.go @@ -27,6 +27,9 @@ type OpRethConfig struct { // runtime otherwise enables proofs-history on every op-reth node; some CLI-superset binaries // reject --proofs-history in the mode under test. DisableProofsHistory bool + // ProofsHistoryWindow is the number of blocks the proofs-history store keeps. Zero keeps the + // mixed runtime's default window. + ProofsHistoryWindow uint64 } // DefaultOpRethConfig returns a zero-valued OpRethConfig that callers can mutate via OpRethOptions. @@ -89,6 +92,22 @@ func OpRethWithoutProofsHistory() OpRethOption { }) } +// OpRethWithProofsHistoryWindow sets the number of blocks the proofs-history store keeps. The mixed +// runtime passes --proofs-history.window itself and op-reth rejects a repeated flag, so the window +// cannot be set through OpRethWithExtraArgs. +func OpRethWithProofsHistoryWindow(blocks uint64) OpRethOption { + return OpRethOptionFn(func(p devtest.T, _ ComponentTarget, cfg *OpRethConfig) { + cfg.ProofsHistoryWindow = blocks + }) +} + +// OpRethWithEthProofWindow lets eth_getProof answer for blocks up to the given distance behind the +// head from reth's own historical state. reth's default window is zero, so a node without proofs +// history rejects every historical proof request unless it gets this option. +func OpRethWithEthProofWindow(blocks uint64) OpRethOption { + return OpRethWithExtraArgs(fmt.Sprintf("--rpc.eth-proof-window=%d", blocks)) +} + // OpRethWithInteropURL wires the op-reth node to the given interop filter HTTP endpoint. // An empty interopURL is a no-op so callers can pass the value unconditionally. func OpRethWithInteropURL(interopURL string) OpRethOption { diff --git a/op-devstack/sysgo/l2_el_opreth_test.go b/op-devstack/sysgo/l2_el_opreth_test.go new file mode 100644 index 00000000000..10a27dfe175 --- /dev/null +++ b/op-devstack/sysgo/l2_el_opreth_test.go @@ -0,0 +1,53 @@ +package sysgo + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +func argsWithPrefix(args []string, prefix string) []string { + var matches []string + for _, arg := range args { + if strings.HasPrefix(arg, prefix) { + matches = append(matches, arg) + } + } + return matches +} + +func TestProofsHistoryNodeArgs(t *testing.T) { + for _, tc := range []struct { + name string + window uint64 + wantWindow string + }{ + {name: "zero window uses the runtime default", window: 0, wantWindow: "--proofs-history.window=10000"}, + {name: "explicit window replaces the default", window: 25, wantWindow: "--proofs-history.window=25"}, + } { + t.Run(tc.name, func(t *testing.T) { + args := proofsHistoryNodeArgs(&OpRethConfig{ProofsHistoryWindow: tc.window}, "/proofs") + + require.Contains(t, args, "--proofs-history") + require.Contains(t, args, "--proofs-history.storage-path=/proofs") + require.Equal(t, []string{tc.wantWindow}, argsWithPrefix(args, "--proofs-history.window="), + "op-reth rejects a repeated flag, so the window must appear exactly once") + }) + } +} + +func TestOpRethWithProofsHistoryWindow(t *testing.T) { + cfg := DefaultOpRethConfig() + OpRethWithProofsHistoryWindow(25).Apply(nil, ComponentTarget{}, cfg) + + require.Equal(t, uint64(25), cfg.ProofsHistoryWindow) + require.Empty(t, cfg.ExtraArgs, "the window is a config field, not an extra arg") +} + +func TestOpRethWithEthProofWindow(t *testing.T) { + cfg := DefaultOpRethConfig() + OpRethWithEthProofWindow(1_000_000).Apply(nil, ComponentTarget{}, cfg) + + require.Equal(t, []string{"--rpc.eth-proof-window=1000000"}, cfg.ExtraArgs) +} diff --git a/op-devstack/sysgo/mixed_runtime.go b/op-devstack/sysgo/mixed_runtime.go index bdb6e8b0df6..e448dbaa4ef 100644 --- a/op-devstack/sysgo/mixed_runtime.go +++ b/op-devstack/sysgo/mixed_runtime.go @@ -8,6 +8,7 @@ import ( "os" "os/exec" "path/filepath" + "strconv" "strings" "github.com/ethereum/go-ethereum/common/hexutil" @@ -390,6 +391,24 @@ func mixedNodeRefs(nodes []mixedSingleChainNode) []MixedSingleChainNodeRefs { return out } +// defaultMixedProofsHistoryWindow is the proofs-history window an op-reth node gets unless +// OpRethWithProofsHistoryWindow overrides it. +const defaultMixedProofsHistoryWindow uint64 = 10_000 + +// proofsHistoryNodeArgs returns the node flags that enable proofs history on the store at +// storagePath. +func proofsHistoryNodeArgs(cfg *OpRethConfig, storagePath string) []string { + window := cfg.ProofsHistoryWindow + if window == 0 { + window = defaultMixedProofsHistoryWindow + } + return []string{ + "--proofs-history", + "--proofs-history.storage-path=" + storagePath, + "--proofs-history.window=" + strconv.FormatUint(window, 10), + } +} + // buildMixedOpRethNode constructs an OpReth node without starting it. func buildMixedOpRethNode( t devtest.T, @@ -488,12 +507,7 @@ func buildMixedOpRethNode( initOut, initErr := exec.Command(execPath, initProofsArgs...).CombinedOutput() t.Require().NoError(initErr, "must init op-reth proof history: %s", string(initOut)) - args = append( - args, - "--proofs-history", - "--proofs-history.window=10000", - "--proofs-history.storage-path="+proofHistoryDir, - ) + args = append(args, proofsHistoryNodeArgs(opRethCfg, proofHistoryDir)...) } args = append(args, opRethCfg.ExtraArgs...) diff --git a/rust/op-reth/tests/proofs/core/account_proofs_test.go b/rust/op-reth/tests/proofs/core/account_proofs_test.go index 1553c7c45a2..a5dee48b887 100644 --- a/rust/op-reth/tests/proofs/core/account_proofs_test.go +++ b/rust/op-reth/tests/proofs/core/account_proofs_test.go @@ -4,10 +4,10 @@ import ( "testing" "github.com/ethereum-optimism/optimism/op-devstack/devtest" + "github.com/ethereum-optimism/optimism/op-devstack/dsl" "github.com/ethereum-optimism/optimism/op-service/bigs" "github.com/ethereum-optimism/optimism/op-service/eth" "github.com/ethereum-optimism/optimism/rust/op-reth/tests/proofs/utils" - "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" "github.com/stretchr/testify/require" ) @@ -40,8 +40,8 @@ func TestL2MultipleTransactionsInDifferentBlocks(gt *testing.T) { require.Equal(t, types.ReceiptStatusSuccessful, receipt1.Status) t.Logf("Transaction 1 included in block: %d", bigs.Uint64Strict(receipt1.BlockNumber)) - utils.WaitForProofsStoreBlock(t, sys.RethWithProofL2ELNode().Escape().L2EthClient(), bigs.Uint64Strict(receipt1.BlockNumber)) - utils.FetchAndVerifyProofs(t, sys, accounts[0].Address(), []common.Hash{}, bigs.Uint64Strict(receipt1.BlockNumber)) + sys.RethWithProofL2ELNode().WaitForProofsStoreBlock(bigs.Uint64Strict(receipt1.BlockNumber), dsl.DefaultProofsCatchUpTimeout) + sys.RethWithProofL2ELNode().VerifyProofsMatchAt(sys.L2ELSequencerNode(), bigs.Uint64Strict(receipt1.BlockNumber), dsl.ProofAccount{Address: accounts[0].Address()}) sys.L2ELSequencerNode().WaitForBlockNumber(currentBlock.Number + 1) // Block 2: Send transaction from second account @@ -55,11 +55,11 @@ func TestL2MultipleTransactionsInDifferentBlocks(gt *testing.T) { require.Equal(t, types.ReceiptStatusSuccessful, receipt2.Status) t.Logf("Transaction 2 included in block: %d", bigs.Uint64Strict(receipt2.BlockNumber)) - utils.WaitForProofsStoreBlock(t, sys.RethWithProofL2ELNode().Escape().L2EthClient(), bigs.Uint64Strict(receipt2.BlockNumber)) - utils.FetchAndVerifyProofs(t, sys, accounts[1].Address(), []common.Hash{}, bigs.Uint64Strict(receipt2.BlockNumber)) + sys.RethWithProofL2ELNode().WaitForProofsStoreBlock(bigs.Uint64Strict(receipt2.BlockNumber), dsl.DefaultProofsCatchUpTimeout) + sys.RethWithProofL2ELNode().VerifyProofsMatchAt(sys.L2ELSequencerNode(), bigs.Uint64Strict(receipt2.BlockNumber), dsl.ProofAccount{Address: accounts[1].Address()}) // Also verify we can get proofs for account 0 at block 2 (different block height) - utils.FetchAndVerifyProofs(t, sys, accounts[0].Address(), []common.Hash{}, bigs.Uint64Strict(receipt2.BlockNumber)) + sys.RethWithProofL2ELNode().VerifyProofsMatchAt(sys.L2ELSequencerNode(), bigs.Uint64Strict(receipt2.BlockNumber), dsl.ProofAccount{Address: accounts[0].Address()}) } // TestL2MultipleTransactionsInSingleBlock tests 2 different accounts sending transactions @@ -100,21 +100,21 @@ func TestL2MultipleTransactionsInSingleBlock(gt *testing.T) { receipt1Block := bigs.Uint64Strict(receipt1.BlockNumber) t.Logf("Transaction 1 included in block %d", receipt1Block) - utils.WaitForProofsStoreBlock(t, sys.RethWithProofL2ELNode().Escape().L2EthClient(), max(receipt0Block, receipt1Block)) + sys.RethWithProofL2ELNode().WaitForProofsStoreBlock(max(receipt0Block, receipt1Block), dsl.DefaultProofsCatchUpTimeout) // Txns can land in the same or different blocks depending on timing. if receipt0Block == receipt1Block { t.Logf("Both transactions included in the same L2 block: %d", receipt0Block) // Verify both proofs against the same block state root - utils.FetchAndVerifyProofs(t, sys, accounts[0].Address(), []common.Hash{}, receipt0Block) - utils.FetchAndVerifyProofs(t, sys, accounts[1].Address(), []common.Hash{}, receipt0Block) + sys.RethWithProofL2ELNode().VerifyProofsMatchAt(sys.L2ELSequencerNode(), receipt0Block, dsl.ProofAccount{Address: accounts[0].Address()}) + sys.RethWithProofL2ELNode().VerifyProofsMatchAt(sys.L2ELSequencerNode(), receipt0Block, dsl.ProofAccount{Address: accounts[1].Address()}) } else { t.Logf("Transactions in different blocks: %d and %d", receipt0Block, receipt1Block) // Different blocks: verify each proof's merkle root matches its respective block's state root - utils.FetchAndVerifyProofs(t, sys, accounts[0].Address(), []common.Hash{}, receipt0Block) - utils.FetchAndVerifyProofs(t, sys, accounts[1].Address(), []common.Hash{}, receipt1Block) + sys.RethWithProofL2ELNode().VerifyProofsMatchAt(sys.L2ELSequencerNode(), receipt0Block, dsl.ProofAccount{Address: accounts[0].Address()}) + sys.RethWithProofL2ELNode().VerifyProofsMatchAt(sys.L2ELSequencerNode(), receipt1Block, dsl.ProofAccount{Address: accounts[1].Address()}) } t.Logf("Proof for account 0 and 1 verified successfully") diff --git a/rust/op-reth/tests/proofs/core/execute_payload_test.go b/rust/op-reth/tests/proofs/core/execute_payload_test.go index d9d59c3a331..fa67bc00bf2 100644 --- a/rust/op-reth/tests/proofs/core/execute_payload_test.go +++ b/rust/op-reth/tests/proofs/core/execute_payload_test.go @@ -4,6 +4,7 @@ import ( "testing" "github.com/ethereum-optimism/optimism/op-devstack/devtest" + "github.com/ethereum-optimism/optimism/op-devstack/dsl" "github.com/ethereum-optimism/optimism/op-service/eth" "github.com/ethereum-optimism/optimism/op-service/txplan" "github.com/ethereum-optimism/optimism/rust/op-reth/tests/proofs/utils" @@ -22,7 +23,7 @@ func TestExecutePayloadSuccess(gt *testing.T) { if err != nil { gt.Fatal(err) } - utils.WaitForProofsStoreBlock(t, opRethELNode.Escape().L2EthClient(), seqHead.NumberU64()) + opRethELNode.WaitForProofsStoreBlock(seqHead.NumberU64(), dsl.DefaultProofsCatchUpTimeout) plannedTxOption := user.PlanTransfer(user.Address(), eth.OneWei) plannedTx := txplan.NewPlannedTx(plannedTxOption) @@ -45,7 +46,7 @@ func TestExecutePayloadSuccess(gt *testing.T) { // ChainCommitted notifications asynchronously, so the EL head can advance before // the store is ready. debug_executePayload reads from the ExEx store and will // fail with "no state found" if it hasn't caught up yet. - utils.WaitForProofsStoreBlock(t, opRethELNode.Escape().L2EthClient(), lastBlock.NumberU64()) + opRethELNode.WaitForProofsStoreBlock(lastBlock.NumberU64(), dsl.DefaultProofsCatchUpTimeout) blockTime := lastBlock.Time() + 1 gasLimit := eth.Uint64Quantity(lastBlock.GasLimit()) @@ -90,7 +91,7 @@ func TestExecutePayloadWithInvalidParentHash(gt *testing.T) { if err != nil { gt.Fatal(err) } - utils.WaitForProofsStoreBlock(t, opRethELNode.Escape().L2EthClient(), seqHead.NumberU64()) + opRethELNode.WaitForProofsStoreBlock(seqHead.NumberU64(), dsl.DefaultProofsCatchUpTimeout) plannedTxOption := user.PlanTransfer(user.Address(), eth.OneWei) plannedTx := txplan.NewPlannedTx(plannedTxOption) @@ -110,7 +111,7 @@ func TestExecutePayloadWithInvalidParentHash(gt *testing.T) { } // Wait for the proofs ExEx store to index the parent block (same race as TestExecutePayloadSuccess). - utils.WaitForProofsStoreBlock(t, opRethELNode.Escape().L2EthClient(), lastBlock.NumberU64()) + opRethELNode.WaitForProofsStoreBlock(lastBlock.NumberU64(), dsl.DefaultProofsCatchUpTimeout) blockTime := lastBlock.Time() + 1 gasLimit := eth.Uint64Quantity(lastBlock.GasLimit()) diff --git a/rust/op-reth/tests/proofs/core/execution_witness_test.go b/rust/op-reth/tests/proofs/core/execution_witness_test.go index 3ad3dd2caec..153d016b615 100644 --- a/rust/op-reth/tests/proofs/core/execution_witness_test.go +++ b/rust/op-reth/tests/proofs/core/execution_witness_test.go @@ -5,6 +5,7 @@ import ( "testing" "github.com/ethereum-optimism/optimism/op-devstack/devtest" + "github.com/ethereum-optimism/optimism/op-devstack/dsl" "github.com/ethereum-optimism/optimism/op-service/bigs" "github.com/ethereum-optimism/optimism/op-service/eth" "github.com/ethereum-optimism/optimism/rust/op-reth/tests/proofs/utils" @@ -49,7 +50,7 @@ func TestDebugExecutionWitness(gt *testing.T) { require.Equal(t, types.ReceiptStatusSuccessful, receipt.Status) t.Logf("Transaction included in block: %d", bigs.Uint64Strict(receipt.BlockNumber)) - utils.WaitForProofsStoreBlock(t, opRethELNode.Escape().L2EthClient(), bigs.Uint64Strict(receipt.BlockNumber)) + opRethELNode.WaitForProofsStoreBlock(bigs.Uint64Strict(receipt.BlockNumber), dsl.DefaultProofsCatchUpTimeout) l2RethClient := opRethELNode.Escape().L2EthClient() // Get the block to inspect the state changes diff --git a/rust/op-reth/tests/proofs/core/resyncing_test.go b/rust/op-reth/tests/proofs/core/resyncing_test.go index 539b1c6a1f1..15a195accd9 100644 --- a/rust/op-reth/tests/proofs/core/resyncing_test.go +++ b/rust/op-reth/tests/proofs/core/resyncing_test.go @@ -6,11 +6,11 @@ import ( "time" "github.com/ethereum-optimism/optimism/op-devstack/devtest" + "github.com/ethereum-optimism/optimism/op-devstack/dsl" "github.com/ethereum-optimism/optimism/op-e2e/e2eutils/wait" "github.com/ethereum-optimism/optimism/op-service/bigs" "github.com/ethereum-optimism/optimism/op-service/eth" "github.com/ethereum-optimism/optimism/rust/op-reth/tests/proofs/utils" - "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" "github.com/stretchr/testify/require" ) @@ -61,12 +61,12 @@ func TestResyncing(gt *testing.T) { }) require.NoError(gt, err, "Validator L2 CL failed to resync to latest block") - utils.WaitForProofsStoreBlock(t, sys.L2ELValidatorNode().Escape().L2EthClient(), blockNumbers[len(blockNumbers)-1]) + sys.L2ELValidatorNode().WaitForProofsStoreBlock(blockNumbers[len(blockNumbers)-1], dsl.DefaultProofsCatchUpTimeout) t.Logf("Fetching and verifying proofs for transactions produced while node was down") // verify the proofs for the transactions produced while the node was down for _, blockNumber := range blockNumbers { - utils.FetchAndVerifyProofs(t, sys, bob.Address(), []common.Hash{}, blockNumber) - utils.FetchAndVerifyProofs(t, sys, alice.Address(), []common.Hash{}, blockNumber) + sys.RethWithProofL2ELNode().VerifyProofsMatchAt(sys.L2ELSequencerNode(), blockNumber, dsl.ProofAccount{Address: bob.Address()}) + sys.RethWithProofL2ELNode().VerifyProofsMatchAt(sys.L2ELSequencerNode(), blockNumber, dsl.ProofAccount{Address: alice.Address()}) } } diff --git a/rust/op-reth/tests/proofs/core/simple_storage_test.go b/rust/op-reth/tests/proofs/core/simple_storage_test.go index f9f8e39a431..2c497e84c53 100644 --- a/rust/op-reth/tests/proofs/core/simple_storage_test.go +++ b/rust/op-reth/tests/proofs/core/simple_storage_test.go @@ -5,6 +5,7 @@ import ( "testing" "github.com/ethereum-optimism/optimism/op-devstack/devtest" + "github.com/ethereum-optimism/optimism/op-devstack/dsl" "github.com/ethereum-optimism/optimism/op-service/bigs" "github.com/ethereum-optimism/optimism/op-service/eth" "github.com/ethereum-optimism/optimism/rust/op-reth/tests/proofs/utils" @@ -21,9 +22,9 @@ func TestStorageProofUsingSimpleStorageContract(gt *testing.T) { contract, receipt := utils.DeploySimpleStorage(t, user) t.Logf("contract deployed at address %s in L2 block %d", contract.Address().Hex(), bigs.Uint64Strict(receipt.BlockNumber)) - utils.WaitForProofsStoreBlock(t, sys.RethWithProofL2ELNode().Escape().L2EthClient(), bigs.Uint64Strict(receipt.BlockNumber)) + sys.RethWithProofL2ELNode().WaitForProofsStoreBlock(bigs.Uint64Strict(receipt.BlockNumber), dsl.DefaultProofsCatchUpTimeout) // fetch and verify initial proof (should be zeroed storage) - utils.FetchAndVerifyProofs(t, sys, contract.Address(), []common.Hash{common.HexToHash("0x0")}, bigs.Uint64Strict(receipt.BlockNumber)) + sys.RethWithProofL2ELNode().VerifyProofsMatchAt(sys.L2ELSequencerNode(), bigs.Uint64Strict(receipt.BlockNumber), dsl.ProofAccount{Address: contract.Address(), StorageKeys: []common.Hash{common.HexToHash("0x0")}}) type caseEntry struct { Block uint64 @@ -49,15 +50,15 @@ func TestStorageProofUsingSimpleStorageContract(gt *testing.T) { }) t.Logf("reset setValue transaction included in L2 block %d", callRes.BlockNumber) - utils.WaitForProofsStoreBlock(t, sys.RethWithProofL2ELNode().Escape().L2EthClient(), bigs.Uint64Strict(callRes.BlockNumber)) + sys.RethWithProofL2ELNode().WaitForProofsStoreBlock(bigs.Uint64Strict(callRes.BlockNumber), dsl.DefaultProofsCatchUpTimeout) // for each case, get proof and verify for _, c := range cases { - utils.FetchAndVerifyProofs(t, sys, contract.Address(), []common.Hash{common.HexToHash("0x0")}, c.Block) + sys.RethWithProofL2ELNode().VerifyProofsMatchAt(sys.L2ELSequencerNode(), c.Block, dsl.ProofAccount{Address: contract.Address(), StorageKeys: []common.Hash{common.HexToHash("0x0")}}) } // test with non-existent storage slot nonExistentSlot := common.HexToHash("0xdeadbeef") - utils.FetchAndVerifyProofs(t, sys, contract.Address(), []common.Hash{nonExistentSlot}, cases[len(cases)-1].Block) + sys.RethWithProofL2ELNode().VerifyProofsMatchAt(sys.L2ELSequencerNode(), cases[len(cases)-1].Block, dsl.ProofAccount{Address: contract.Address(), StorageKeys: []common.Hash{nonExistentSlot}}) } func TestStorageProofUsingMultiStorageContract(gt *testing.T) { @@ -70,9 +71,9 @@ func TestStorageProofUsingMultiStorageContract(gt *testing.T) { contract, receipt := utils.DeployMultiStorage(t, user) t.Logf("contract deployed at address %s in L2 block %d", contract.Address().Hex(), bigs.Uint64Strict(receipt.BlockNumber)) - utils.WaitForProofsStoreBlock(t, sys.RethWithProofL2ELNode().Escape().L2EthClient(), bigs.Uint64Strict(receipt.BlockNumber)) + sys.RethWithProofL2ELNode().WaitForProofsStoreBlock(bigs.Uint64Strict(receipt.BlockNumber), dsl.DefaultProofsCatchUpTimeout) // fetch and verify initial proof (should be zeroed storage) - utils.FetchAndVerifyProofs(t, sys, contract.Address(), []common.Hash{common.HexToHash("0x0"), common.HexToHash("0x1")}, bigs.Uint64Strict(receipt.BlockNumber)) + sys.RethWithProofL2ELNode().VerifyProofsMatchAt(sys.L2ELSequencerNode(), bigs.Uint64Strict(receipt.BlockNumber), dsl.ProofAccount{Address: contract.Address(), StorageKeys: []common.Hash{common.HexToHash("0x0"), common.HexToHash("0x1")}}) // set multiple storage slots type caseEntry struct { @@ -107,7 +108,7 @@ func TestStorageProofUsingMultiStorageContract(gt *testing.T) { }) t.Logf("reset setValues transaction included in L2 block %d", callRes.BlockNumber) - utils.WaitForProofsStoreBlock(t, sys.RethWithProofL2ELNode().Escape().L2EthClient(), bigs.Uint64Strict(callRes.BlockNumber)) + sys.RethWithProofL2ELNode().WaitForProofsStoreBlock(bigs.Uint64Strict(callRes.BlockNumber), dsl.DefaultProofsCatchUpTimeout) // for each case, get proof and verify for _, c := range cases { var slots []common.Hash @@ -115,7 +116,7 @@ func TestStorageProofUsingMultiStorageContract(gt *testing.T) { slots = append(slots, slot) } - utils.FetchAndVerifyProofs(t, sys, contract.Address(), slots, c.Block) + sys.RethWithProofL2ELNode().VerifyProofsMatchAt(sys.L2ELSequencerNode(), c.Block, dsl.ProofAccount{Address: contract.Address(), StorageKeys: slots}) } } @@ -151,7 +152,7 @@ func TestTokenVaultStorageProofs(gt *testing.T) { deactBlock := bigs.Uint64Strict(deactRes.BlockNumber) t.Logf("deactivateAllowance included in block %d", deactBlock) - utils.WaitForProofsStoreBlock(t, sys.RethWithProofL2ELNode().Escape().L2EthClient(), deactBlock) + sys.RethWithProofL2ELNode().WaitForProofsStoreBlock(deactBlock, dsl.DefaultProofsCatchUpTimeout) // balance slot for user balanceSlot := contract.GetBalanceSlot(userAddr) @@ -163,11 +164,11 @@ func TestTokenVaultStorageProofs(gt *testing.T) { // fetch & verify proofs at appropriate blocks // balance after deposit (depositBlock) t.Logf("Verifying balance slot %s at deposit block %d", balanceSlot.Hex(), depositBlock) - utils.FetchAndVerifyProofs(t, sys, contract.Address(), []common.Hash{balanceSlot, depositor0Slot}, depositBlock) + sys.RethWithProofL2ELNode().VerifyProofsMatchAt(sys.L2ELSequencerNode(), depositBlock, dsl.ProofAccount{Address: contract.Address(), StorageKeys: []common.Hash{balanceSlot, depositor0Slot}}) // allowance after approve (approveBlock) t.Logf("Verifying allowance slot %s at approve block %d", allowanceSlot.Hex(), approveBlock) - utils.FetchAndVerifyProofs(t, sys, contract.Address(), []common.Hash{allowanceSlot}, approveBlock) + sys.RethWithProofL2ELNode().VerifyProofsMatchAt(sys.L2ELSequencerNode(), approveBlock, dsl.ProofAccount{Address: contract.Address(), StorageKeys: []common.Hash{allowanceSlot}}) // after deactivation, allowance should be zero at deactBlock t.Logf("Verifying allowance slot %s at deactivate block %d", allowanceSlot.Hex(), deactBlock) - utils.FetchAndVerifyProofs(t, sys, contract.Address(), []common.Hash{allowanceSlot}, deactBlock) + sys.RethWithProofL2ELNode().VerifyProofsMatchAt(sys.L2ELSequencerNode(), deactBlock, dsl.ProofAccount{Address: contract.Address(), StorageKeys: []common.Hash{allowanceSlot}}) } diff --git a/rust/op-reth/tests/proofs/prune/prune_test.go b/rust/op-reth/tests/proofs/prune/prune_test.go index d38c5872ab2..6c7d52d925d 100644 --- a/rust/op-reth/tests/proofs/prune/prune_test.go +++ b/rust/op-reth/tests/proofs/prune/prune_test.go @@ -5,11 +5,10 @@ import ( "time" "github.com/ethereum-optimism/optimism/op-devstack/devtest" - "github.com/ethereum-optimism/optimism/op-service/apis" + "github.com/ethereum-optimism/optimism/op-devstack/dsl" "github.com/ethereum-optimism/optimism/op-service/bigs" "github.com/ethereum-optimism/optimism/op-service/eth" "github.com/ethereum-optimism/optimism/rust/op-reth/tests/proofs/utils" - "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" "github.com/stretchr/testify/require" ) @@ -35,7 +34,6 @@ func TestPruneProofStorageWithGetProofConsistency(gt *testing.T) { var pruneDetectTimeout = 5 * time.Minute opRethELNode := sys.RethWithProofL2ELNode() - ethClient := opRethELNode.Escape().EthClient() // ----------------------------- // (1) Create tx + validate proof pre-prune @@ -68,19 +66,19 @@ func TestPruneProofStorageWithGetProofConsistency(gt *testing.T) { t.Logf("Target block for proof validation (pre-prune): %d", targetBlock) // Make sure the proof store has indexed the target block before verifying. - utils.WaitForProofsStoreBlock(t, ethClient, targetBlock) + opRethELNode.WaitForProofsStoreBlock(targetBlock, dsl.DefaultProofsCatchUpTimeout) // Pre-prune proof verification at targetBlock. // This verifies the proof against the block's state root (efficient correctness check). t.Logf("Pre-prune: verifying getProof proofs at block %d", targetBlock) - utils.FetchAndVerifyProofs(t, sys, accounts[0].Address(), []common.Hash{}, targetBlock) - utils.FetchAndVerifyProofs(t, sys, accounts[1].Address(), []common.Hash{}, targetBlock) + sys.RethWithProofL2ELNode().VerifyProofsMatchAt(sys.L2ELSequencerNode(), targetBlock, dsl.ProofAccount{Address: accounts[0].Address()}) + sys.RethWithProofL2ELNode().VerifyProofsMatchAt(sys.L2ELSequencerNode(), targetBlock, dsl.ProofAccount{Address: accounts[1].Address()}) t.Log("Pre-prune: proofs verified successfully") // ----------------------------- // (2) Wait until targetBlock is pruned (earliest > targetBlock) // ----------------------------- - initialStatus := getProofSyncStatus(t, ethClient) + initialStatus := getProofSyncStatus(t, opRethELNode) t.Logf("Initial proofs sync status: earliest=%d latest=%d", initialStatus.Earliest, initialStatus.Latest) // Ensure we advance far enough that pruning *can* move earliest past targetBlock. @@ -88,7 +86,7 @@ func TestPruneProofStorageWithGetProofConsistency(gt *testing.T) { requiredLatest := targetBlock + proofWindow if initialStatus.Latest < requiredLatest { t.Logf("Waiting for chain to advance to at least block %d so pruning can pass targetBlock", requiredLatest) - utils.WaitForProofsStoreBlock(t, ethClient, requiredLatest) + opRethELNode.WaitForProofsStoreBlock(requiredLatest, dsl.DefaultProofsCatchUpTimeout) } t.Logf("Waiting for pruner to advance earliest past targetBlock=%d ...", targetBlock) @@ -100,7 +98,7 @@ func TestPruneProofStorageWithGetProofConsistency(gt *testing.T) { t.Errorf("Timed out waiting for prune: earliest did not advance past targetBlock=%d within %s", targetBlock, pruneDetectTimeout) } - prunedStatus = getProofSyncStatus(t, ethClient) + prunedStatus = getProofSyncStatus(t, opRethELNode) t.Logf("Polling proofs sync status: earliest=%d latest=%d (target=%d)", prunedStatus.Earliest, prunedStatus.Latest, targetBlock) // This is the key condition: the specific block we validated earlier is now out of window. @@ -119,8 +117,8 @@ func TestPruneProofStorageWithGetProofConsistency(gt *testing.T) { // (3) Post-prune consistency checks for getProof // ----------------------------- t.Logf("Post-prune: expecting getProof verification to succeed at new earliest block=%d", prunedStatus.Earliest) - utils.FetchAndVerifyProofs(t, sys, accounts[0].Address(), []common.Hash{}, prunedStatus.Earliest) - utils.FetchAndVerifyProofs(t, sys, accounts[1].Address(), []common.Hash{}, prunedStatus.Earliest) + sys.RethWithProofL2ELNode().VerifyProofsMatchAt(sys.L2ELSequencerNode(), prunedStatus.Earliest, dsl.ProofAccount{Address: accounts[0].Address()}) + sys.RethWithProofL2ELNode().VerifyProofsMatchAt(sys.L2ELSequencerNode(), prunedStatus.Earliest, dsl.ProofAccount{Address: accounts[1].Address()}) t.Log("Post-prune: getProof consistency checks passed") } @@ -129,11 +127,9 @@ type proofSyncStatus struct { Latest uint64 `json:"latest"` } -func getProofSyncStatus(t devtest.T, client apis.EthClient) proofSyncStatus { - var result proofSyncStatus - err := client.RPC().CallContext(t.Ctx(), &result, "debug_proofsSyncStatus") - if err != nil { - t.Errorf("debug_proofsSyncStatus call failed: %v", err) - } - return result +func getProofSyncStatus(t devtest.T, node *dsl.L2ELNode) proofSyncStatus { + status, err := node.ProofsSyncStatus() + require.NoError(t, err, "debug_proofsSyncStatus call failed") + require.NotNil(t, status.Latest, "proofs store is not initialized") + return proofSyncStatus{Earliest: *status.Earliest, Latest: *status.Latest} } diff --git a/rust/op-reth/tests/proofs/reorg/reorg_test.go b/rust/op-reth/tests/proofs/reorg/reorg_test.go index fcafc889793..e78c3176103 100644 --- a/rust/op-reth/tests/proofs/reorg/reorg_test.go +++ b/rust/op-reth/tests/proofs/reorg/reorg_test.go @@ -7,6 +7,7 @@ import ( "time" "github.com/ethereum-optimism/optimism/op-devstack/devtest" + "github.com/ethereum-optimism/optimism/op-devstack/dsl" "github.com/ethereum-optimism/optimism/op-e2e/e2eutils/wait" "github.com/ethereum-optimism/optimism/op-service/bigs" "github.com/ethereum-optimism/optimism/op-service/eth" @@ -169,7 +170,7 @@ func TestReorgUsingAccountProof(gt *testing.T) { } latestBlock := sys.L2Chain.WaitForBlock() - utils.WaitForProofsStoreBlock(t, sys.RethWithProofL2ELNode().Escape().L2EthClient(), latestBlock.Number) + sys.RethWithProofL2ELNode().WaitForProofsStoreBlock(latestBlock.Number, dsl.DefaultProofsCatchUpTimeout) // verify that the L2A validator has reorged and reached the latest block. Bounded so a // validator that never converges fails here instead of consuming the package timeout. @@ -199,6 +200,6 @@ func TestReorgUsingAccountProof(gt *testing.T) { // verify that the accounts involved in the conflicting blocks for i, c := range cases { l.Info("Verifying proof", "case", i, "addr", c.addr.Hex(), "block", c.Block) - utils.FetchAndVerifyProofs(t, sys, c.addr, c.slots, c.Block) + sys.RethWithProofL2ELNode().VerifyProofsMatchAt(sys.L2ELSequencerNode(), c.Block, dsl.ProofAccount{Address: c.addr, StorageKeys: c.slots}) } } diff --git a/rust/op-reth/tests/proofs/utils/proof.go b/rust/op-reth/tests/proofs/utils/proof.go deleted file mode 100644 index 67afbd658c2..00000000000 --- a/rust/op-reth/tests/proofs/utils/proof.go +++ /dev/null @@ -1,166 +0,0 @@ -package utils - -import ( - "bytes" - "fmt" - "time" - - "github.com/ethereum-optimism/optimism/op-devstack/devtest" - "github.com/ethereum-optimism/optimism/op-service/eth" - "github.com/ethereum-optimism/optimism/op-service/retry" - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/common/hexutil" - "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/crypto" - "github.com/ethereum/go-ethereum/ethdb/memorydb" - "github.com/ethereum/go-ethereum/rlp" - "github.com/ethereum/go-ethereum/trie" - "github.com/stretchr/testify/require" -) - -// NormalizeProofResponse standardizes an AccountResult obtained from eth_getProof -// across different client implementations (e.g., Geth, Reth) so that they can be -// compared meaningfully in tests. -// -// Ethereum clients may encode empty or zeroed data structures differently while -// still representing the same logical state. For example: -// - An empty storage proof may appear as [] (Geth) or ["0x80"] (Reth). -// -// This function normalizes such differences by: -// - Converting single-element proofs containing "0x80" to an empty proof slice. -func NormalizeProofResponse(res *eth.AccountResult) { - for i := range res.StorageProof { - if len(res.StorageProof[i].Proof) == 1 && bytes.Equal(res.StorageProof[i].Proof[0], []byte{0x80}) { - res.StorageProof[i].Proof = []hexutil.Bytes{} - } - } - - // Normalize empty CodeHash - // Geth returns 0x0000000000000000000000000000000000000000000000000000000000000000 - // Reth returns 0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470 - if res.CodeHash == (common.Hash{}) { - res.CodeHash = crypto.Keccak256Hash(nil) - } - - // Normalize empty StorageHash - // Geth returns 0x0000000000000000000000000000000000000000000000000000000000000000 - // Reth returns 0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421 - if res.StorageHash == (common.Hash{}) { - res.StorageHash = types.EmptyRootHash - } -} - -// VerifyProof verifies an account and its storage proofs against a given state root. -// -// This function extends the standard behavior of go-ethereum’s AccountResult.Verify() -// by gracefully handling the case where the account’s storage trie root is empty -// (0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421). -func VerifyProof(res *eth.AccountResult, stateRoot common.Hash) error { - // Skip storage proof verification if the storage trie is empty. - if res.StorageHash != types.EmptyRootHash { - for i, entry := range res.StorageProof { - // load all MPT nodes into a DB - db := memorydb.New() - for j, encodedNode := range entry.Proof { - nodeKey := encodedNode - if len(encodedNode) >= 32 { // small MPT nodes are not hashed - nodeKey = crypto.Keccak256(encodedNode) - } - if err := db.Put(nodeKey, encodedNode); err != nil { - return fmt.Errorf("failed to load storage proof node %d of storage value %d into mem db: %w", j, i, err) - } - } - path := crypto.Keccak256(entry.Key) - val, err := trie.VerifyProof(res.StorageHash, path, db) - if err != nil { - return fmt.Errorf("failed to verify storage value %d with key %s (path %x) in storage trie %s: %w", i, entry.Key.String(), path, res.StorageHash, err) - } - if val == nil && entry.Value.ToInt().Cmp(common.Big0) == 0 { // empty storage is zero by default - continue - } - comparison, err := rlp.EncodeToBytes(entry.Value.ToInt().Bytes()) - if err != nil { - return fmt.Errorf("failed to encode storage value %d with key %s (path %x) in storage trie %s: %w", i, entry.Key.String(), path, res.StorageHash, err) - } - if !bytes.Equal(val, comparison) { - return fmt.Errorf("value %d in storage proof does not match proven value at key %s (path %x)", i, entry.Key.String(), path) - } - } - } - - accountClaimed := []any{uint64(res.Nonce), res.Balance.ToInt().Bytes(), res.StorageHash, res.CodeHash} - accountClaimedValue, err := rlp.EncodeToBytes(accountClaimed) - if err != nil { - return fmt.Errorf("failed to encode account from retrieved values: %w", err) - } - - // create a db with all account trie nodes - db := memorydb.New() - for i, encodedNode := range res.AccountProof { - nodeKey := encodedNode - if len(encodedNode) >= 32 { // small MPT nodes are not hashed - nodeKey = crypto.Keccak256(encodedNode) - } - if err := db.Put(nodeKey, encodedNode); err != nil { - return fmt.Errorf("failed to load account proof node %d into mem db: %w", i, err) - } - } - path := crypto.Keccak256(res.Address[:]) - accountProofValue, err := trie.VerifyProof(stateRoot, path, db) - if err != nil { - return fmt.Errorf("failed to verify account value with key %s (path %x) in account trie %s: %w", res.Address, path, stateRoot, err) - } - - // If the proof demonstrates non-existence (nil value), we must check if the RPC claimed the account is empty. - if len(accountProofValue) == 0 { - isEmpty := res.Nonce == 0 && - res.Balance.ToInt().Sign() == 0 && - (res.StorageHash == types.EmptyRootHash || res.StorageHash == common.Hash{}) && - (res.CodeHash == crypto.Keccak256Hash(nil) || res.CodeHash == common.Hash{}) - - if isEmpty { - return nil - } - } - - if !bytes.Equal(accountClaimedValue, accountProofValue) { - return fmt.Errorf("L1 RPC is tricking us, account proof does not match provided deserialized values:\n"+ - " claimed: %x\n"+ - " proof: %x", accountClaimedValue, accountProofValue) - } - return nil -} - -func fetchProofWithRetry(t devtest.T, source string, block uint64, fetch func() (*eth.AccountResult, error)) *eth.AccountResult { - // Each op-reth node persists proof state asynchronously. Readiness on one node does not imply - // that its peer can serve the same historical block yet, so retry the exact RPC on each node. - proof, err := retry.Do(t.Ctx(), 50, &retry.FixedStrategy{Dur: 200 * time.Millisecond}, fetch) - require.NoError(t, err, "failed to get proof from %s at block %d", source, block) - return proof -} - -// FetchAndVerifyProofs fetches account proofs from both L2EL and L2ELB for the given address -func FetchAndVerifyProofs(t devtest.T, sys *MixedOpProofPreset, address common.Address, slots []common.Hash, block uint64) { - ctx := t.Ctx() - blockInfo, err := sys.L2ELSequencerNode().Escape().L2EthClient().InfoByNumber(ctx, block) - require.NoError(t, err, "failed to get block info for block %d", block) - - blockTag := hexutil.Uint64(block).String() - seqProofRes := fetchProofWithRetry(t, "L2EL", block, func() (*eth.AccountResult, error) { - return sys.L2ELSequencerNode().Escape().L2EthClient().GetProof(ctx, address, slots, blockTag) - }) - valProofRes := fetchProofWithRetry(t, "L2ELB", block, func() (*eth.AccountResult, error) { - return sys.L2ELValidatorNode().Escape().L2EthClient().GetProof(ctx, address, slots, blockTag) - }) - - NormalizeProofResponse(seqProofRes) - NormalizeProofResponse(valProofRes) - - require.Equal(t, seqProofRes, valProofRes, "sequencer and validator proofs should match") - - err = VerifyProof(seqProofRes, blockInfo.Root()) - require.NoError(t, err, "geth proof verification failed at block %d", block) - - err = VerifyProof(valProofRes, blockInfo.Root()) - require.NoError(t, err, "reth proof verification failed at block %d", block) -} diff --git a/rust/op-reth/tests/proofs/utils/utils.go b/rust/op-reth/tests/proofs/utils/utils.go index 10e79d4ffc1..e47fdc47e15 100644 --- a/rust/op-reth/tests/proofs/utils/utils.go +++ b/rust/op-reth/tests/proofs/utils/utils.go @@ -4,11 +4,9 @@ import ( "encoding/json" "os" "strings" - "time" "github.com/ethereum-optimism/optimism/op-devstack/devtest" "github.com/ethereum-optimism/optimism/op-devstack/dsl" - "github.com/ethereum-optimism/optimism/op-service/apis" "github.com/ethereum-optimism/optimism/op-service/txplan" "github.com/ethereum/go-ethereum/accounts/abi" "github.com/ethereum/go-ethereum/common" @@ -51,33 +49,6 @@ func LoadArtifact(t devtest.T, artifactPath string) (abi.ABI, []byte) { return parsedABI, common.FromHex(binHex) } -// WaitForProofsStoreBlock polls the op-reth debug_proofsSyncStatus RPC until the -// proofs ExEx store has indexed at least up to targetBlock. The ExEx processes -// ChainCommitted notifications asynchronously, so the EL head can advance before -// the proofs store has caught up. Any RPC that depends on OpStateProviderFactory -// (e.g. debug_executePayload) will fail with "no state found" if called before -// the store is ready. -func WaitForProofsStoreBlock(t devtest.T, client apis.EthClient, targetBlock uint64) { - type syncStatus struct { - Earliest *uint64 `json:"earliest"` - Latest *uint64 `json:"latest"` - } - require.Eventually(t, func() bool { - var status syncStatus - err := client.RPC().CallContext(t.Ctx(), &status, "debug_proofsSyncStatus") - if err != nil { - t.Logf("debug_proofsSyncStatus call failed (retrying): %v", err) - return false - } - if status.Latest == nil { - t.Logf("proofs store not yet initialized, waiting...") - return false - } - t.Logf("proofs store status: latest=%d target=%d", *status.Latest, targetBlock) - return *status.Latest >= targetBlock - }, 30*time.Second, 200*time.Millisecond, "proofs store did not index block %d in time", targetBlock) -} - // DeployContract deploys the contract creation bytecode from the given artifact. // user must provide a Plan() method compatible with txplan.NewPlannedTx (kept generic). func DeployContract(t devtest.T, user *dsl.EOA, bin []byte) (common.Address, *types.Receipt) { From 8ef29f2b8c8faffd891be06d29c6d11e38255060 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Wed, 14 Oct 2026 06:35:19 -0700 Subject: [PATCH 2/8] op-devstack: send empty storage key lists to eth_getProof op-reth rejects a null storage key list where op-geth accepts it, so awaitProof always sends a list. Retries are logged at warn level so a failing proof lookup is diagnosable from the test log, and the acceptance test pins the sweep's last block to the last workload block so the whole workload is covered and the catch-up wait is exercised. Co-Authored-By: Claude Fable 5.1 (cherry picked from commit 9c4ce60b09157a2747c0fbbaa468b836c4f8d2f9) (cherry picked from commit ed29c772af8efc2f2f4a9675b57687d14b7591e5) --- .../proofs_history_match_test.go | 13 +++++++++++-- op-devstack/dsl/l2_el_proofs.go | 9 +++++++-- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/op-acceptance-tests/tests/proofs_history_match/proofs_history_match_test.go b/op-acceptance-tests/tests/proofs_history_match/proofs_history_match_test.go index b2a8ca6dc42..838a0790664 100644 --- a/op-acceptance-tests/tests/proofs_history_match/proofs_history_match_test.go +++ b/op-acceptance-tests/tests/proofs_history_match/proofs_history_match_test.go @@ -8,6 +8,7 @@ import ( "github.com/ethereum-optimism/optimism/op-devstack/dsl" "github.com/ethereum-optimism/optimism/op-devstack/presets" "github.com/ethereum-optimism/optimism/op-devstack/sysgo" + "github.com/ethereum-optimism/optimism/op-service/bigs" "github.com/ethereum-optimism/optimism/op-service/eth" "github.com/ethereum-optimism/optimism/op-service/txplan" "github.com/ethereum/go-ethereum/common" @@ -132,6 +133,12 @@ func deployBlockRecorder(t devtest.T, deployer *dsl.EOA) common.Address { return receipt.ContractAddress } +func includedBlock(t devtest.T, tx *txplan.PlannedTx) uint64 { + receipt, err := tx.Included.Eval(t.Ctx()) + t.Require().NoError(err, "transaction must be included") + return bigs.Uint64Strict(receipt.BlockNumber) +} + // TestProofsHistoryFollowerServesMatchingHistoricalProofs runs transfers and contract writes // against a sequencer that serves eth_getProof from its own historical state, then sweeps every // block of a proofs-history follower's window: each of the follower's proofs must verify against @@ -142,17 +149,18 @@ func TestProofsHistoryFollowerServesMatchingHistoricalProofs(gt *testing.T) { t.Run(c.name(), func(t devtest.T) { t.Parallel() if c.stateMaskingBlocks != 0 { - t.Skipf("state masking is compiled out of the reth 2.5.2 on develop and #23046 rejects a nonzero --engine.num-state-masking-blocks; re-enable with ethereum-optimism/protocol-team#457") + t.Skipf("state masking is compiled out of reth 2.5.2 on develop and #23046 rejects a nonzero --engine.num-state-masking-blocks; re-enable with ethereum-optimism/protocol-team#457") } sys := newProofsHistorySystem(t, c) alice := sys.FunderL2.NewFundedEOA(eth.OneEther) bob := sys.FunderL2.NewFundedEOA(eth.OneEther) recorder := deployBlockRecorder(t, alice) + var lastWorkloadBlock uint64 for range workloadRounds { alice.Transfer(bob.Address(), eth.OneHundredthEther) bob.Transfer(alice.Address(), eth.OneHundredthEther) - alice.Transact(alice.Plan(), txplan.WithTo(&recorder)) + lastWorkloadBlock = includedBlock(t, alice.Transact(alice.Plan(), txplan.WithTo(&recorder))) } sys.ProofsFollower.VerifyHistoricalProofsMatch(sys.Reference, func(opts *dsl.HistoricalProofsOpts) { @@ -161,6 +169,7 @@ func TestProofsHistoryFollowerServesMatchingHistoricalProofs(gt *testing.T) { {Address: bob.Address()}, {Address: recorder, StorageKeys: []common.Hash{common.HexToHash("0x0"), common.HexToHash("0x1")}}, } + opts.LastBlock = lastWorkloadBlock }) }) } diff --git a/op-devstack/dsl/l2_el_proofs.go b/op-devstack/dsl/l2_el_proofs.go index 804006f1f6c..09aabe01b99 100644 --- a/op-devstack/dsl/l2_el_proofs.go +++ b/op-devstack/dsl/l2_el_proofs.go @@ -224,13 +224,18 @@ func (el *L2ELNode) awaitHeaderByNumber(number uint64, timeout time.Duration) et func (el *L2ELNode) awaitProof(header eth.BlockInfo, account ProofAccount, timeout time.Duration) *eth.AccountResult { ctx, cancel := context.WithTimeout(el.ctx, timeout) defer cancel() + // A nil key list encodes as JSON null, which op-reth rejects where op-geth accepts it. + storageKeys := account.StorageKeys + if storageKeys == nil { + storageKeys = []common.Hash{} + } var proof *eth.AccountResult var lastErr error err := wait.For(ctx, 200*time.Millisecond, func() (bool, error) { - result, err := el.inner.L2EthClient().GetProof(ctx, account.Address, account.StorageKeys, header.Hash().String()) + result, err := el.inner.L2EthClient().GetProof(ctx, account.Address, storageKeys, header.Hash().String()) if err != nil { lastErr = err - el.log.Debug("eth_getProof failed; will retry", "name", el.inner.Name(), "block", header.NumberU64(), + el.log.Warn("eth_getProof failed; will retry", "name", el.inner.Name(), "block", header.NumberU64(), "account", account.Address, "err", err) return false, nil } From ea1e60aa6c7067ee01a20347cc98b87ce9910806 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Wed, 14 Oct 2026 06:51:32 -0700 Subject: [PATCH 3/8] op-devstack: harden the historical proof oracle Re-read the proofs window at every swept block and skip blocks the pruner removed meanwhile when the caller did not pin FirstBlock, so a small proofs window does not race the sweep. Gate the withdrawalsRoot check on a non-empty header value, since pre-Isthmus headers carry the empty withdrawals hash. Reject a nonzero storage value claimed under an empty storage root. Require debug_proofsSyncStatus to be unknown on the reference rather than merely failing. Drop the storage-key padding the client already enforces. The acceptance test adds a withdrawal per round so the message passer's storage root changes inside the swept range, and waits for reth to pass its persistence threshold before the last round so the threshold-50 cells persist at least once. The proofs e2e suite drops the waits that VerifyProofsMatchAt performs itself. Co-Authored-By: Claude Fable 5.1 (cherry picked from commit 25e9f7e7dfddc7e6d72777eca27876e8eab5af0a) (cherry picked from commit 36688d25fe3f86682b1243f3983e0f42daeaa543) --- .../proofs_history_match_test.go | 30 ++++++-- op-devstack/dsl/l2_el_proofs.go | 74 +++++++++++-------- op-devstack/dsl/l2_el_proofs_test.go | 56 +++++++++++--- .../tests/proofs/core/account_proofs_test.go | 3 - .../tests/proofs/core/resyncing_test.go | 2 - .../tests/proofs/core/simple_storage_test.go | 6 -- rust/op-reth/tests/proofs/prune/prune_test.go | 3 - rust/op-reth/tests/proofs/reorg/reorg_test.go | 1 - 8 files changed, 115 insertions(+), 60 deletions(-) diff --git a/op-acceptance-tests/tests/proofs_history_match/proofs_history_match_test.go b/op-acceptance-tests/tests/proofs_history_match/proofs_history_match_test.go index 838a0790664..c5d3c15fbd4 100644 --- a/op-acceptance-tests/tests/proofs_history_match/proofs_history_match_test.go +++ b/op-acceptance-tests/tests/proofs_history_match/proofs_history_match_test.go @@ -20,6 +20,8 @@ const ( // referenceEthProofWindow lets the reference node answer eth_getProof for every block of the // run from reth's own historical state; reth's default window of zero rejects them all. referenceEthProofWindow = 1_000_000 + // memoryBlockBufferTarget is reth's --engine.memory-block-buffer-target default. + memoryBlockBufferTarget = 5 workloadRounds = 4 ) @@ -49,6 +51,13 @@ func (c proofsHistoryCase) opRethArgs() []string { return args } +// blocksBeforeLastRound is a chain height by which reth has persisted at least once under the +// case's threshold: persistence starts once more than the threshold of blocks sit in memory and +// keeps the buffer target in memory afterwards. +func (c proofsHistoryCase) blocksBeforeLastRound() uint64 { + return c.persistenceThreshold + 2*memoryBlockBufferTarget +} + func proofsHistoryMatrix() []proofsHistoryCase { var cases []proofsHistoryCase for _, persistenceThreshold := range []uint64{7, 50} { @@ -66,6 +75,7 @@ func proofsHistoryMatrix() []proofsHistoryCase { } type proofsHistorySystem struct { + Bridge *dsl.StandardBridge FunderL2 *dsl.FunderEOA ProofsFollower *dsl.L2ELNode Reference *dsl.L2ELNode @@ -112,6 +122,7 @@ func newProofsHistorySystem(t devtest.T, c proofsHistoryCase) *proofsHistorySyst t.Require().NotNil(follower, "missing the proofs follower") return &proofsHistorySystem{ + Bridge: dsl.NewStandardBridge(t, frontends.L2Network, frontends.L1EL), FunderL2: frontends.FunderL2, ProofsFollower: follower, Reference: frontends.L2Network.PrimaryEL(), @@ -139,17 +150,17 @@ func includedBlock(t devtest.T, tx *txplan.PlannedTx) uint64 { return bigs.Uint64Strict(receipt.BlockNumber) } -// TestProofsHistoryFollowerServesMatchingHistoricalProofs runs transfers and contract writes -// against a sequencer that serves eth_getProof from its own historical state, then sweeps every -// block of a proofs-history follower's window: each of the follower's proofs must verify against -// the block header and equal the sequencer's. +// TestProofsHistoryFollowerServesMatchingHistoricalProofs runs transfers, withdrawals and contract +// writes against a sequencer that serves eth_getProof from its own historical state, then sweeps a +// proofs-history follower from its window's first block to the last workload block: each of the +// follower's proofs must verify against the block header and equal the sequencer's. func TestProofsHistoryFollowerServesMatchingHistoricalProofs(gt *testing.T) { t := devtest.ParallelT(gt) for _, c := range proofsHistoryMatrix() { t.Run(c.name(), func(t devtest.T) { t.Parallel() if c.stateMaskingBlocks != 0 { - t.Skipf("state masking is compiled out of reth 2.5.2 on develop and #23046 rejects a nonzero --engine.num-state-masking-blocks; re-enable with ethereum-optimism/protocol-team#457") + t.Skip("state masking is compiled out of reth 2.5.2 on develop and #23046 rejects a nonzero --engine.num-state-masking-blocks; re-enable with ethereum-optimism/protocol-team#457") } sys := newProofsHistorySystem(t, c) @@ -157,9 +168,16 @@ func TestProofsHistoryFollowerServesMatchingHistoricalProofs(gt *testing.T) { bob := sys.FunderL2.NewFundedEOA(eth.OneEther) recorder := deployBlockRecorder(t, alice) var lastWorkloadBlock uint64 - for range workloadRounds { + for round := range workloadRounds { + if round == workloadRounds-1 { + // The last round lands after reth has persisted at least once, so the sweep spans + // blocks that left reth's memory and blocks that have not. + target := c.blocksBeforeLastRound() + sys.ProofsFollower.Reached(eth.Unsafe, target, int(target)+30) + } alice.Transfer(bob.Address(), eth.OneHundredthEther) bob.Transfer(alice.Address(), eth.OneHundredthEther) + sys.Bridge.InitiateWithdrawal(eth.OneHundredthEther, bob) lastWorkloadBlock = includedBlock(t, alice.Transact(alice.Plan(), txplan.WithTo(&recorder))) } diff --git a/op-devstack/dsl/l2_el_proofs.go b/op-devstack/dsl/l2_el_proofs.go index 09aabe01b99..047b9949f21 100644 --- a/op-devstack/dsl/l2_el_proofs.go +++ b/op-devstack/dsl/l2_el_proofs.go @@ -3,6 +3,7 @@ package dsl import ( "bytes" "context" + "errors" "fmt" "time" @@ -15,6 +16,7 @@ import ( "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/ethdb/memorydb" "github.com/ethereum/go-ethereum/rlp" + gethrpc "github.com/ethereum/go-ethereum/rpc" "github.com/ethereum/go-ethereum/trie" ) @@ -55,11 +57,13 @@ type HistoricalProofsOpts struct { // sampled block before the sweep fails. CatchUpTimeout time.Duration // FirstBlock and LastBlock bound the sweep. Zero selects the matching edge of the proofs - // window reported by debug_proofsSyncStatus when the sweep starts. + // window reported by debug_proofsSyncStatus when the sweep starts. A block the store prunes + // while the sweep runs is skipped when FirstBlock is zero and fails the sweep otherwise. FirstBlock uint64 LastBlock uint64 // MaxSampledBlocks bounds how many blocks of the range are compared; zero compares every - // block. The first and last block of the range are always compared. + // block. The first and last block of the range are always compared, so at least two blocks + // of a multi-block range are. MaxSampledBlocks int } @@ -91,7 +95,7 @@ func (el *L2ELNode) WaitForProofsStoreBlock(target uint64, timeout time.Duration return false, nil } lastStatus, lastErr = status, nil - if status.Latest != nil && *status.Latest >= target { + if status.Earliest != nil && status.Latest != nil && *status.Latest >= target { logger.Info("Proofs store indexed block", "window", status) return true, nil } @@ -106,8 +110,7 @@ func (el *L2ELNode) WaitForProofsStoreBlock(target uint64, timeout time.Duration // VerifyHistoricalProofsMatch sweeps the blocks this node's proofs-history store serves and, at // every sampled block, requires eth_getProof for each configured account to verify against the // block's state root and to equal the proof served by reference. The L2ToL1MessagePasser proof is -// always included, and its storage root must equal the header's withdrawalsRoot where the header -// carries one. +// always included, and from Isthmus on its storage root must equal the header's withdrawalsRoot. // // reference must answer eth_getProof from its own historical state rather than from a // proofs-history store, since two stores can agree on a wrong proof. @@ -134,23 +137,31 @@ func (el *L2ELNode) VerifyHistoricalProofsMatch(reference *L2ELNode, opts ...fun logger.Info("Sweeping historical proofs", "window", window, "first", first, "last", last, "blocks", len(blocks), "accounts", len(accounts)) + pruned := 0 for _, block := range blocks { - el.verifyProofsMatchAt(reference, block, accounts, cfg.CatchUpTimeout) + window, err := el.ProofsSyncStatus() + el.require.NoErrorf(err, "debug_proofsSyncStatus failed on %s", el.inner.Name()) + if cfg.FirstBlock == 0 && window.Earliest != nil && block < *window.Earliest { + logger.Info("Block left the proofs window during the sweep; skipping", "block", block, "window", window) + pruned++ + continue + } + el.verifyProofsMatchAt(reference, block, window, accounts, cfg.CatchUpTimeout) } logger.Info("Historical proofs match the reference", "first", first, "last", last, - "blocks", len(blocks), "accounts", len(accounts)) + "blocks", len(blocks)-pruned, "pruned_during_sweep", pruned, "accounts", len(accounts)) } -// VerifyProofsMatchAt is VerifyHistoricalProofsMatch for one block and an explicit account list, -// waiting up to DefaultProofsCatchUpTimeout for the proofs store to serve the block. +// VerifyProofsMatchAt verifies the proofs of the given accounts at one block the same way +// VerifyHistoricalProofsMatch does, after waiting up to DefaultProofsCatchUpTimeout for the proofs +// store to serve the block. Only the listed accounts are proven, and reference is not checked for +// independence. func (el *L2ELNode) VerifyProofsMatchAt(reference *L2ELNode, block uint64, accounts ...ProofAccount) { - el.WaitForProofsStoreBlock(block, DefaultProofsCatchUpTimeout) - el.verifyProofsMatchAt(reference, block, accounts, DefaultProofsCatchUpTimeout) + window := el.WaitForProofsStoreBlock(block, DefaultProofsCatchUpTimeout) + el.verifyProofsMatchAt(reference, block, window, accounts, DefaultProofsCatchUpTimeout) } -func (el *L2ELNode) verifyProofsMatchAt(reference *L2ELNode, block uint64, accounts []ProofAccount, catchUpTimeout time.Duration) { - window, err := el.ProofsSyncStatus() - el.require.NoErrorf(err, "debug_proofsSyncStatus failed on %s", el.inner.Name()) +func (el *L2ELNode) verifyProofsMatchAt(reference *L2ELNode, block uint64, window ProofsSyncStatus, accounts []ProofAccount, catchUpTimeout time.Duration) { el.require.Truef(window.serves(block), "proofs store on %s does not serve block %d: window %s", el.inner.Name(), block, window) header := el.awaitHeaderByNumber(block, catchUpTimeout) @@ -180,7 +191,9 @@ func (el *L2ELNode) verifyAccountProofMatchesAt(reference *L2ELNode, header eth. el.require.Equalf(referenceProof, proof, "block %d account %s: eth_getProof from %s differs from reference %s", block, account.Address, el.inner.Name(), reference.inner.Name()) - if account.Address == predeploys.L2ToL1MessagePasserAddr && header.WithdrawalsRoot() != nil { + // Isthmus made withdrawalsRoot the message passer's storage root. Before that the header + // carries the hash of an empty withdrawals list, which says nothing about the message passer. + if account.Address == predeploys.L2ToL1MessagePasserAddr && header.WithdrawalsRoot() != nil && *header.WithdrawalsRoot() != types.EmptyWithdrawalsHash { el.require.Equalf(*header.WithdrawalsRoot(), proof.StorageHash, "block %d: L2ToL1MessagePasser storage root from %s does not equal the header's withdrawalsRoot", block, el.inner.Name()) } @@ -188,16 +201,14 @@ func (el *L2ELNode) verifyAccountProofMatchesAt(reference *L2ELNode, header eth. "account", account.Address, "storage_keys", len(account.StorageKeys)) } -// requireIndependentProofReference fails when reference serves eth_getProof from a proofs-history -// store, which every node that exposes a proofs window does. +// requireIndependentProofReference fails unless reference lacks debug_proofsSyncStatus, which +// op-reth registers only on a proofs-history node and op-geth never does. func (el *L2ELNode) requireIndependentProofReference(reference *L2ELNode) { status, err := reference.ProofsSyncStatus() - if err != nil { - el.log.Debug("Reference node has no proofs-history store", "reference", reference.inner.Name(), "err", err) - return - } - el.require.Nilf(status.Latest, "reference %s runs proofs history (window %s); the sweep needs a reference that serves eth_getProof from its own historical state", - reference.inner.Name(), status) + var rpcErr gethrpc.Error + el.require.Truef(errors.As(err, &rpcErr) && rpcErr.ErrorCode() == int(eth.MethodNotFound), + "reference %s must serve eth_getProof from its own historical state, not a proofs-history store; debug_proofsSyncStatus returned window %s, err: %v", + reference.inner.Name(), status, err) } func (el *L2ELNode) awaitHeaderByNumber(number uint64, timeout time.Duration) eth.BlockInfo { @@ -247,8 +258,9 @@ func (el *L2ELNode) awaitProof(header eth.BlockInfo, account ProofAccount, timeo return proof } -// sampleBlockNumbers returns at most maxSamples evenly spaced block numbers in [first, last], -// always including both edges. A non-positive maxSamples returns every block. +// sampleBlockNumbers returns evenly spaced block numbers in [first, last], always including both +// edges: every block when maxSamples is non-positive or covers the range, otherwise +// max(maxSamples, 2) of them. func sampleBlockNumbers(first, last uint64, maxSamples int) []uint64 { count := last - first + 1 if maxSamples <= 0 || uint64(maxSamples) >= count { @@ -271,14 +283,12 @@ func sampleBlockNumbers(first, last uint64, maxSamples int) []uint64 { // NormalizeProofResponse rewrites an eth_getProof response into one canonical encoding so proofs // from different clients compare equal: an empty storage proof, which reth encodes as ["0x80"], // becomes empty; a zero code hash becomes keccak256("") and a zero storage hash becomes the empty -// trie root, which is how reth encodes both; storage keys are left-padded to 32 bytes; and an -// absent storage proof list becomes nil. +// trie root, which is how reth encodes both; and an empty storage proof list becomes nil. func NormalizeProofResponse(res *eth.AccountResult) { for i := range res.StorageProof { if len(res.StorageProof[i].Proof) == 1 && bytes.Equal(res.StorageProof[i].Proof[0], []byte{0x80}) { res.StorageProof[i].Proof = []hexutil.Bytes{} } - res.StorageProof[i].Key = common.BytesToHash(res.StorageProof[i].Key).Bytes() } if len(res.StorageProof) == 0 { res.StorageProof = nil @@ -295,7 +305,13 @@ func NormalizeProofResponse(res *eth.AccountResult) { // eth.AccountResult.Verify it accepts an empty storage trie, whose proofs carry no nodes, and a // non-existence proof for an account the response reports as empty. func VerifyAccountProof(res *eth.AccountResult, stateRoot common.Hash) error { - if res.StorageHash != types.EmptyRootHash { + if res.StorageHash == types.EmptyRootHash { + for i, entry := range res.StorageProof { + if entry.Value.ToInt().Sign() != 0 { + return fmt.Errorf("storage value %d at key %s is nonzero but the storage trie is empty", i, entry.Key.String()) + } + } + } else { for i, entry := range res.StorageProof { db := memorydb.New() for j, encodedNode := range entry.Proof { diff --git a/op-devstack/dsl/l2_el_proofs_test.go b/op-devstack/dsl/l2_el_proofs_test.go index c89339582c9..7adf8dddd9f 100644 --- a/op-devstack/dsl/l2_el_proofs_test.go +++ b/op-devstack/dsl/l2_el_proofs_test.go @@ -123,6 +123,15 @@ func TestVerifyAccountProof(t *testing.T) { {name: "account with storage verifies", proof: func() *eth.AccountResult { return proofOf(t, stateTrie, alice) }, root: stateRoot}, {name: "account without storage verifies", proof: func() *eth.AccountResult { return proofOf(t, stateTrie, bob) }, root: stateRoot}, {name: "absent account reported empty verifies", proof: func() *eth.AccountResult { return proofOf(t, stateTrie, nobody) }, root: stateRoot}, + { + name: "zero slot of an empty storage trie verifies", + proof: func() *eth.AccountResult { + proof := proofOf(t, stateTrie, bob) + proof.StorageProof = []eth.StorageProofEntry{{Key: common.HexToHash("0x0").Bytes(), Proof: []hexutil.Bytes{}}} + return proof + }, + root: stateRoot, + }, { name: "tampered balance fails", proof: func() *eth.AccountResult { @@ -143,6 +152,16 @@ func TestVerifyAccountProof(t *testing.T) { root: stateRoot, wantErr: "does not match proven value", }, + { + name: "nonzero slot of an empty storage trie fails", + proof: func() *eth.AccountResult { + proof := proofOf(t, stateTrie, bob) + proof.StorageProof = []eth.StorageProofEntry{{Key: common.HexToHash("0x0").Bytes(), Value: (hexutil.Big)(*big.NewInt(42)), Proof: []hexutil.Bytes{}}} + return proof + }, + root: stateRoot, + wantErr: "is nonzero but the storage trie is empty", + }, { name: "absent account claiming a balance fails", proof: func() *eth.AccountResult { @@ -174,7 +193,7 @@ func TestVerifyAccountProof(t *testing.T) { func TestNormalizeProofResponse(t *testing.T) { emptyCodeHash := crypto.Keccak256Hash(nil) someHash := common.HexToHash("0xabc") - paddedKey := common.HexToHash("0x1").Bytes() + key := common.HexToHash("0x1").Bytes() for _, tc := range []struct { name string @@ -183,13 +202,8 @@ func TestNormalizeProofResponse(t *testing.T) { }{ { name: "reth empty storage proof becomes empty", - in: eth.AccountResult{CodeHash: someHash, StorageHash: someHash, StorageProof: []eth.StorageProofEntry{{Key: paddedKey, Proof: []hexutil.Bytes{{0x80}}}}}, - want: eth.AccountResult{CodeHash: someHash, StorageHash: someHash, StorageProof: []eth.StorageProofEntry{{Key: paddedKey, Proof: []hexutil.Bytes{}}}}, - }, - { - name: "short storage key is padded to 32 bytes", - in: eth.AccountResult{CodeHash: someHash, StorageHash: someHash, StorageProof: []eth.StorageProofEntry{{Key: []byte{0x1}, Proof: []hexutil.Bytes{}}}}, - want: eth.AccountResult{CodeHash: someHash, StorageHash: someHash, StorageProof: []eth.StorageProofEntry{{Key: paddedKey, Proof: []hexutil.Bytes{}}}}, + in: eth.AccountResult{CodeHash: someHash, StorageHash: someHash, StorageProof: []eth.StorageProofEntry{{Key: key, Proof: []hexutil.Bytes{{0x80}}}}}, + want: eth.AccountResult{CodeHash: someHash, StorageHash: someHash, StorageProof: []eth.StorageProofEntry{{Key: key, Proof: []hexutil.Bytes{}}}}, }, { name: "zero code hash becomes the empty code hash", @@ -208,8 +222,8 @@ func TestNormalizeProofResponse(t *testing.T) { }, { name: "non-empty values are kept", - in: eth.AccountResult{CodeHash: someHash, StorageHash: someHash, StorageProof: []eth.StorageProofEntry{{Key: paddedKey, Proof: []hexutil.Bytes{{0x80}, {0x81}}}}}, - want: eth.AccountResult{CodeHash: someHash, StorageHash: someHash, StorageProof: []eth.StorageProofEntry{{Key: paddedKey, Proof: []hexutil.Bytes{{0x80}, {0x81}}}}}, + in: eth.AccountResult{CodeHash: someHash, StorageHash: someHash, StorageProof: []eth.StorageProofEntry{{Key: key, Proof: []hexutil.Bytes{{0x80}, {0x81}}}}}, + want: eth.AccountResult{CodeHash: someHash, StorageHash: someHash, StorageProof: []eth.StorageProofEntry{{Key: key, Proof: []hexutil.Bytes{{0x80}, {0x81}}}}}, }, } { t.Run(tc.name, func(t *testing.T) { @@ -220,6 +234,28 @@ func TestNormalizeProofResponse(t *testing.T) { } } +func TestProofsSyncStatusServes(t *testing.T) { + window := func(earliest, latest uint64) ProofsSyncStatus { + return ProofsSyncStatus{Earliest: &earliest, Latest: &latest} + } + for _, tc := range []struct { + name string + status ProofsSyncStatus + block uint64 + want bool + }{ + {name: "uninitialized store serves nothing", status: ProofsSyncStatus{}, block: 0, want: false}, + {name: "earliest edge is served", status: window(5, 9), block: 5, want: true}, + {name: "latest edge is served", status: window(5, 9), block: 9, want: true}, + {name: "block below earliest is not served", status: window(5, 9), block: 4, want: false}, + {name: "block above latest is not served", status: window(5, 9), block: 10, want: false}, + } { + t.Run(tc.name, func(t *testing.T) { + require.Equal(t, tc.want, tc.status.serves(tc.block)) + }) + } +} + func TestSampleBlockNumbers(t *testing.T) { for _, tc := range []struct { name string diff --git a/rust/op-reth/tests/proofs/core/account_proofs_test.go b/rust/op-reth/tests/proofs/core/account_proofs_test.go index a5dee48b887..4ee952643e8 100644 --- a/rust/op-reth/tests/proofs/core/account_proofs_test.go +++ b/rust/op-reth/tests/proofs/core/account_proofs_test.go @@ -40,7 +40,6 @@ func TestL2MultipleTransactionsInDifferentBlocks(gt *testing.T) { require.Equal(t, types.ReceiptStatusSuccessful, receipt1.Status) t.Logf("Transaction 1 included in block: %d", bigs.Uint64Strict(receipt1.BlockNumber)) - sys.RethWithProofL2ELNode().WaitForProofsStoreBlock(bigs.Uint64Strict(receipt1.BlockNumber), dsl.DefaultProofsCatchUpTimeout) sys.RethWithProofL2ELNode().VerifyProofsMatchAt(sys.L2ELSequencerNode(), bigs.Uint64Strict(receipt1.BlockNumber), dsl.ProofAccount{Address: accounts[0].Address()}) sys.L2ELSequencerNode().WaitForBlockNumber(currentBlock.Number + 1) @@ -55,7 +54,6 @@ func TestL2MultipleTransactionsInDifferentBlocks(gt *testing.T) { require.Equal(t, types.ReceiptStatusSuccessful, receipt2.Status) t.Logf("Transaction 2 included in block: %d", bigs.Uint64Strict(receipt2.BlockNumber)) - sys.RethWithProofL2ELNode().WaitForProofsStoreBlock(bigs.Uint64Strict(receipt2.BlockNumber), dsl.DefaultProofsCatchUpTimeout) sys.RethWithProofL2ELNode().VerifyProofsMatchAt(sys.L2ELSequencerNode(), bigs.Uint64Strict(receipt2.BlockNumber), dsl.ProofAccount{Address: accounts[1].Address()}) // Also verify we can get proofs for account 0 at block 2 (different block height) @@ -100,7 +98,6 @@ func TestL2MultipleTransactionsInSingleBlock(gt *testing.T) { receipt1Block := bigs.Uint64Strict(receipt1.BlockNumber) t.Logf("Transaction 1 included in block %d", receipt1Block) - sys.RethWithProofL2ELNode().WaitForProofsStoreBlock(max(receipt0Block, receipt1Block), dsl.DefaultProofsCatchUpTimeout) // Txns can land in the same or different blocks depending on timing. if receipt0Block == receipt1Block { t.Logf("Both transactions included in the same L2 block: %d", receipt0Block) diff --git a/rust/op-reth/tests/proofs/core/resyncing_test.go b/rust/op-reth/tests/proofs/core/resyncing_test.go index 15a195accd9..faf269b4a7f 100644 --- a/rust/op-reth/tests/proofs/core/resyncing_test.go +++ b/rust/op-reth/tests/proofs/core/resyncing_test.go @@ -61,8 +61,6 @@ func TestResyncing(gt *testing.T) { }) require.NoError(gt, err, "Validator L2 CL failed to resync to latest block") - sys.L2ELValidatorNode().WaitForProofsStoreBlock(blockNumbers[len(blockNumbers)-1], dsl.DefaultProofsCatchUpTimeout) - t.Logf("Fetching and verifying proofs for transactions produced while node was down") // verify the proofs for the transactions produced while the node was down for _, blockNumber := range blockNumbers { diff --git a/rust/op-reth/tests/proofs/core/simple_storage_test.go b/rust/op-reth/tests/proofs/core/simple_storage_test.go index 2c497e84c53..07f72822c8a 100644 --- a/rust/op-reth/tests/proofs/core/simple_storage_test.go +++ b/rust/op-reth/tests/proofs/core/simple_storage_test.go @@ -22,7 +22,6 @@ func TestStorageProofUsingSimpleStorageContract(gt *testing.T) { contract, receipt := utils.DeploySimpleStorage(t, user) t.Logf("contract deployed at address %s in L2 block %d", contract.Address().Hex(), bigs.Uint64Strict(receipt.BlockNumber)) - sys.RethWithProofL2ELNode().WaitForProofsStoreBlock(bigs.Uint64Strict(receipt.BlockNumber), dsl.DefaultProofsCatchUpTimeout) // fetch and verify initial proof (should be zeroed storage) sys.RethWithProofL2ELNode().VerifyProofsMatchAt(sys.L2ELSequencerNode(), bigs.Uint64Strict(receipt.BlockNumber), dsl.ProofAccount{Address: contract.Address(), StorageKeys: []common.Hash{common.HexToHash("0x0")}}) @@ -50,7 +49,6 @@ func TestStorageProofUsingSimpleStorageContract(gt *testing.T) { }) t.Logf("reset setValue transaction included in L2 block %d", callRes.BlockNumber) - sys.RethWithProofL2ELNode().WaitForProofsStoreBlock(bigs.Uint64Strict(callRes.BlockNumber), dsl.DefaultProofsCatchUpTimeout) // for each case, get proof and verify for _, c := range cases { sys.RethWithProofL2ELNode().VerifyProofsMatchAt(sys.L2ELSequencerNode(), c.Block, dsl.ProofAccount{Address: contract.Address(), StorageKeys: []common.Hash{common.HexToHash("0x0")}}) @@ -71,7 +69,6 @@ func TestStorageProofUsingMultiStorageContract(gt *testing.T) { contract, receipt := utils.DeployMultiStorage(t, user) t.Logf("contract deployed at address %s in L2 block %d", contract.Address().Hex(), bigs.Uint64Strict(receipt.BlockNumber)) - sys.RethWithProofL2ELNode().WaitForProofsStoreBlock(bigs.Uint64Strict(receipt.BlockNumber), dsl.DefaultProofsCatchUpTimeout) // fetch and verify initial proof (should be zeroed storage) sys.RethWithProofL2ELNode().VerifyProofsMatchAt(sys.L2ELSequencerNode(), bigs.Uint64Strict(receipt.BlockNumber), dsl.ProofAccount{Address: contract.Address(), StorageKeys: []common.Hash{common.HexToHash("0x0"), common.HexToHash("0x1")}}) @@ -108,7 +105,6 @@ func TestStorageProofUsingMultiStorageContract(gt *testing.T) { }) t.Logf("reset setValues transaction included in L2 block %d", callRes.BlockNumber) - sys.RethWithProofL2ELNode().WaitForProofsStoreBlock(bigs.Uint64Strict(callRes.BlockNumber), dsl.DefaultProofsCatchUpTimeout) // for each case, get proof and verify for _, c := range cases { var slots []common.Hash @@ -152,8 +148,6 @@ func TestTokenVaultStorageProofs(gt *testing.T) { deactBlock := bigs.Uint64Strict(deactRes.BlockNumber) t.Logf("deactivateAllowance included in block %d", deactBlock) - sys.RethWithProofL2ELNode().WaitForProofsStoreBlock(deactBlock, dsl.DefaultProofsCatchUpTimeout) - // balance slot for user balanceSlot := contract.GetBalanceSlot(userAddr) // nested allowance slot owner=user, spender=spenderAddr diff --git a/rust/op-reth/tests/proofs/prune/prune_test.go b/rust/op-reth/tests/proofs/prune/prune_test.go index 6c7d52d925d..245d88556df 100644 --- a/rust/op-reth/tests/proofs/prune/prune_test.go +++ b/rust/op-reth/tests/proofs/prune/prune_test.go @@ -65,9 +65,6 @@ func TestPruneProofStorageWithGetProofConsistency(gt *testing.T) { } t.Logf("Target block for proof validation (pre-prune): %d", targetBlock) - // Make sure the proof store has indexed the target block before verifying. - opRethELNode.WaitForProofsStoreBlock(targetBlock, dsl.DefaultProofsCatchUpTimeout) - // Pre-prune proof verification at targetBlock. // This verifies the proof against the block's state root (efficient correctness check). t.Logf("Pre-prune: verifying getProof proofs at block %d", targetBlock) diff --git a/rust/op-reth/tests/proofs/reorg/reorg_test.go b/rust/op-reth/tests/proofs/reorg/reorg_test.go index e78c3176103..6c6a1af6fd7 100644 --- a/rust/op-reth/tests/proofs/reorg/reorg_test.go +++ b/rust/op-reth/tests/proofs/reorg/reorg_test.go @@ -170,7 +170,6 @@ func TestReorgUsingAccountProof(gt *testing.T) { } latestBlock := sys.L2Chain.WaitForBlock() - sys.RethWithProofL2ELNode().WaitForProofsStoreBlock(latestBlock.Number, dsl.DefaultProofsCatchUpTimeout) // verify that the L2A validator has reorged and reached the latest block. Bounded so a // validator that never converges fails here instead of consuming the package timeout. From 3c98724402ff3dfcadd9d715a6395ff7f6c24e31 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Wed, 14 Oct 2026 07:34:52 -0700 Subject: [PATCH 4/8] op-devstack: add an unclean op-reth stop and per-node L2 CL options to the mixed runtime SubProcess.Kill sends SIGKILL without an interrupt and waits a bounded time for the exit, so a test can end an op-reth process the way a crash or a power loss would, losing whatever it had not written to its datadir. OpReth.StopUngracefully exposes it, stack.UngracefulLifecycle lets the preset frontend forward it, and L2ELNode.StopUngracefully is the DSL method. Stop stays graceful. MixedSingleChainNodeSpec gains L2CLOpts, per-node op-node options passed to startL2CLNode, and L2CLVerifierSyncMode is the option a follower needs to EL-sync; the mixed runtime passed no CL options before, so every follower ran CL sync. Co-Authored-By: Claude Fable 5.1 (cherry picked from commit b9386d0f44c981128bd8b2aca3ca9232c2a1d9bf) (cherry picked from commit 13cc581abae9ed7047c503af0c162250c90853b3) --- op-devstack/dsl/l2_el.go | 10 ++++++++ op-devstack/presets/rpc_frontends.go | 11 +++++++- op-devstack/presets/sysgo_runtime.go | 3 +++ op-devstack/stack/lifecycle.go | 6 +++++ op-devstack/sysgo/l2_cl.go | 8 ++++++ op-devstack/sysgo/l2_cl_test.go | 17 +++++++++++++ op-devstack/sysgo/l2_el_opreth.go | 22 +++++++++++++++- op-devstack/sysgo/mixed_runtime.go | 7 +++-- op-devstack/sysgo/subproc.go | 35 ++++++++++++++++++++++++- op-devstack/sysgo/subproc_test.go | 38 ++++++++++++++++++++++++++++ 10 files changed, 152 insertions(+), 5 deletions(-) create mode 100644 op-devstack/sysgo/l2_cl_test.go diff --git a/op-devstack/dsl/l2_el.go b/op-devstack/dsl/l2_el.go index a940ac0fb06..432a7a9fb81 100644 --- a/op-devstack/dsl/l2_el.go +++ b/op-devstack/dsl/l2_el.go @@ -525,6 +525,16 @@ func (el *L2ELNode) Stop() { lifecycle.Stop() } +// StopUngracefully ends the node's process without a shutdown, the way a crash or a power loss +// would, so whatever it had not yet written to disk is lost. Start brings it back on the same +// data directory. +func (el *L2ELNode) StopUngracefully() { + el.log.Info("Stopping ungracefully", "name", el.inner.Name()) + lifecycle, ok := el.inner.(stack.UngracefulLifecycle) + el.require.Truef(ok, "L2EL node %s cannot be stopped ungracefully", el.inner.Name()) + lifecycle.StopUngracefully() +} + func (el *L2ELNode) Start() { lifecycle, ok := el.inner.(stack.Lifecycle) el.require.Truef(ok, "L2EL node %s is not lifecycle-controllable", el.inner.Name()) diff --git a/op-devstack/presets/rpc_frontends.go b/op-devstack/presets/rpc_frontends.go index b6008086dac..d0a41c6182b 100644 --- a/op-devstack/presets/rpc_frontends.go +++ b/op-devstack/presets/rpc_frontends.go @@ -155,9 +155,13 @@ type l2ELFrontend struct { l2EngineClient *sources.EngineClient lifecycle stack.Lifecycle control stack.ControlledLifecycle + ungraceful stack.UngracefulLifecycle } -var _ stack.L2ELNode = (*l2ELFrontend)(nil) +var ( + _ stack.L2ELNode = (*l2ELFrontend)(nil) + _ stack.UngracefulLifecycle = (*l2ELFrontend)(nil) +) func newPresetL2ELNode(t devtest.T, name string, chainID eth.ChainID, userRPCCl opclient.RPC, userRPC string, engineRPCCl opclient.RPC, rollupCfg *rollup.Config) *l2ELFrontend { t.Require().NotNil(rollupCfg, "rollup config must be configured") @@ -193,6 +197,11 @@ func (r *l2ELFrontend) Stop() { r.lifecycle.Stop() } +func (r *l2ELFrontend) StopUngracefully() { + r.require().NotNil(r.ungraceful, "L2EL node %s cannot be stopped ungracefully", r.Name()) + r.ungraceful.StopUngracefully() +} + func (r *l2ELFrontend) StartControlled(ctx context.Context) error { if r.control == nil { return fmt.Errorf("L2EL node %s is not controllable", r.Name()) diff --git a/op-devstack/presets/sysgo_runtime.go b/op-devstack/presets/sysgo_runtime.go index 3ae12a67ebe..ddf7a42fe27 100644 --- a/op-devstack/presets/sysgo_runtime.go +++ b/op-devstack/presets/sysgo_runtime.go @@ -66,6 +66,9 @@ func newL2ELFrontend(t devtest.T, name string, chainID eth.ChainID, userRPC stri if control, ok := lifecycle[0].(stack.ControlledLifecycle); ok { l2EL.control = control } + if ungraceful, ok := lifecycle[0].(stack.UngracefulLifecycle); ok { + l2EL.ungraceful = ungraceful + } } return l2EL } diff --git a/op-devstack/stack/lifecycle.go b/op-devstack/stack/lifecycle.go index f26d3b7e605..c235cb4f557 100644 --- a/op-devstack/stack/lifecycle.go +++ b/op-devstack/stack/lifecycle.go @@ -12,3 +12,9 @@ type ControlledLifecycle interface { StopControlled(ctx context.Context) error Running() bool } + +// UngracefulLifecycle is implemented by components that can be stopped the way a crash or a +// power loss would stop them: without a shutdown, so whatever they held only in memory is lost. +type UngracefulLifecycle interface { + StopUngracefully() +} diff --git a/op-devstack/sysgo/l2_cl.go b/op-devstack/sysgo/l2_cl.go index dc7539eb004..0158dc47942 100644 --- a/op-devstack/sysgo/l2_cl.go +++ b/op-devstack/sysgo/l2_cl.go @@ -57,6 +57,14 @@ func L2CLFollowSource(source string) L2CLOption { }) } +// L2CLVerifierSyncMode sets the sync mode a verifier runs: CL sync has op-node fill unsafe gaps +// from its peers, EL sync hands the forkchoice target to the EL and lets it sync from its own. +func L2CLVerifierSyncMode(mode nodeSync.Mode) L2CLOption { + return L2CLOptionFn(func(p devtest.T, _ ComponentTarget, cfg *L2CLConfig) { + cfg.VerifierSyncMode = mode + }) +} + func DefaultL2CLConfig() *L2CLConfig { return &L2CLConfig{ SequencerSyncMode: nodeSync.CLSync, diff --git a/op-devstack/sysgo/l2_cl_test.go b/op-devstack/sysgo/l2_cl_test.go new file mode 100644 index 00000000000..fd41ee8b393 --- /dev/null +++ b/op-devstack/sysgo/l2_cl_test.go @@ -0,0 +1,17 @@ +package sysgo + +import ( + "testing" + + "github.com/stretchr/testify/require" + + nodeSync "github.com/ethereum-optimism/optimism/op-node/rollup/sync" +) + +func TestL2CLVerifierSyncMode(t *testing.T) { + cfg := DefaultL2CLConfig() + L2CLVerifierSyncMode(nodeSync.ELSync).Apply(nil, ComponentTarget{}, cfg) + + require.Equal(t, nodeSync.ELSync, cfg.VerifierSyncMode) + require.Equal(t, nodeSync.CLSync, cfg.SequencerSyncMode, "the sequencer keeps its own sync mode") +} diff --git a/op-devstack/sysgo/l2_el_opreth.go b/op-devstack/sysgo/l2_el_opreth.go index 7a3736f3d6f..6a31e5d0ea8 100644 --- a/op-devstack/sysgo/l2_el_opreth.go +++ b/op-devstack/sysgo/l2_el_opreth.go @@ -8,6 +8,7 @@ import ( "sync" "github.com/ethereum-optimism/optimism/op-devstack/devtest" + "github.com/ethereum-optimism/optimism/op-devstack/stack" "github.com/ethereum-optimism/optimism/op-service/eth" "github.com/ethereum-optimism/optimism/op-service/log" "github.com/ethereum-optimism/optimism/op-service/logpipe" @@ -144,7 +145,10 @@ type OpReth struct { l2MetricsRegistrar L2MetricsRegistrar } -var _ L2ELNode = (*OpReth)(nil) +var ( + _ L2ELNode = (*OpReth)(nil) + _ stack.UngracefulLifecycle = (*OpReth)(nil) +) func (n *OpReth) Start() { n.mu.Lock() @@ -258,6 +262,22 @@ func (n *OpReth) Stop() { n.sub = nil } +// StopUngracefully ends the op-reth process with SIGKILL, the way a crash or a power loss would, +// so whatever it had not yet written to its datadir is lost. Start brings the node back on the +// same datadir. +func (n *OpReth) StopUngracefully() { + n.mu.Lock() + defer n.mu.Unlock() + if n.sub == nil { + n.p.Logger().Warn("op-reth already stopped") + return + } + n.p.Logger().Info("Stopping op-reth ungracefully", "name", n.name, "chain", n.chainID) + n.clearProxyUpstreams() + n.p.Require().NoError(n.sub.Kill(), "Must kill") + n.sub = nil +} + // Callers must hold n.mu. func (n *OpReth) clearProxyUpstreams() { if n.userProxy != nil { diff --git a/op-devstack/sysgo/mixed_runtime.go b/op-devstack/sysgo/mixed_runtime.go index e448dbaa4ef..b25b948954c 100644 --- a/op-devstack/sysgo/mixed_runtime.go +++ b/op-devstack/sysgo/mixed_runtime.go @@ -177,6 +177,9 @@ type MixedSingleChainNodeSpec struct { // by deriving from L1. Used to exercise the derivation/force-build path (FCU-with-attributes) // rather than the consolidation path. Defaults to false (fully peered). IsolateFromL2P2P bool + // L2CLOpts are per-node op-node options, e.g. L2CLVerifierSyncMode for a follower that must + // EL-sync. Ignored for non-op-node CL kinds. + L2CLOpts []L2CLOption // OpRethOpts are per-node op-reth options applied AFTER the shared cfg.OpRethOptions, only to // this node's EL. Use for options that must not reach every node — e.g. a per-node binary // override (OpRethWithBinary) or a flag a single node should receive while its peers reject it. @@ -287,10 +290,10 @@ func NewMixedSingleChainRuntime(t devtest.T, cfg MixedSingleChainPresetConfig) * IsSequencer: spec.IsSequencer, NoDiscovery: true, DependencySet: depSet, - L2CLOptions: []L2CLOption{L2CLOptionFn(func(_ devtest.T, _ ComponentTarget, cfg *L2CLConfig) { + L2CLOptions: append([]L2CLOption{L2CLOptionFn(func(_ devtest.T, _ ComponentTarget, cfg *L2CLConfig) { cfg.SafeDBPath = safeDBPath cfg.VerifierL1Confs = spec.VerifierL1Confs - })}, + })}, spec.L2CLOpts...), }) case MixedL2CLKona: cl = startMixedKonaNode( diff --git a/op-devstack/sysgo/subproc.go b/op-devstack/sysgo/subproc.go index f86f582922f..baf1bb588fb 100644 --- a/op-devstack/sysgo/subproc.go +++ b/op-devstack/sysgo/subproc.go @@ -142,6 +142,35 @@ func (sp *SubProcess) stop(ctx context.Context, interrupt bool, interruptWait ti return sp.completeStopLocked(interrupt, sp.waitErr) } +// Kill ends the process with SIGKILL, so it runs none of its shutdown logic, and waits for it to +// exit. It is the unclean counterpart of Stop: whatever the process held only in memory is lost. +func (sp *SubProcess) Kill() error { + sp.mu.Lock() + defer sp.mu.Unlock() + if sp.cmd == nil { + return nil // already stopped + } + + select { + case <-sp.exited: + // already exited; nothing to kill + default: + sp.p.Logger().Info("Killing sub-process", "pid", sp.cmd.Process.Pid) + // The process may have exited between the check and the signal, which Kill reports as + // ErrProcessDone; the wait below then returns at once. + if err := sp.cmd.Process.Kill(); err != nil && !errors.Is(err, os.ErrProcessDone) { + return fmt.Errorf("failed to kill sub-process: %w", err) + } + } + + if !waitProcess(context.Background(), sp.exited, controlledKillWait) { + return fmt.Errorf("sub-process did not exit within %s of SIGKILL", controlledKillWait) + } + sp.p.Logger().Info("Sub-process killed", "exit", sp.waitErr) + sp.releaseLocked() + return nil +} + func (sp *SubProcess) completeStopLocked(interrupt bool, waitErr error) error { var exitErr *exec.ExitError if waitErr != nil && !(interrupt && errors.As(waitErr, &exitErr)) { @@ -151,7 +180,12 @@ func (sp *SubProcess) completeStopLocked(interrupt bool, waitErr error) error { } else { sp.p.Logger().Info("Sub-process gracefully exited") } + sp.releaseLocked() + return nil +} +// releaseLocked closes the log pipes and forgets the exited process so Start can run a new one. +func (sp *SubProcess) releaseLocked() { if sp.stdOutProc != nil { _ = sp.stdOutProc.Close() sp.stdOutProc = nil @@ -161,7 +195,6 @@ func (sp *SubProcess) completeStopLocked(interrupt bool, waitErr error) error { sp.stdErrProc = nil } sp.cmd = nil - return nil } func waitProcess(ctx context.Context, exited <-chan struct{}, timeout time.Duration) bool { diff --git a/op-devstack/sysgo/subproc_test.go b/op-devstack/sysgo/subproc_test.go index e0173db840b..221b929c16f 100644 --- a/op-devstack/sysgo/subproc_test.go +++ b/op-devstack/sysgo/subproc_test.go @@ -85,6 +85,44 @@ func TestSubProcessConcurrentStop(gt *testing.T) { require.NoError(gt, sp.Stop(false)) } +func newTestSubProcess(gt *testing.T) (*SubProcess, *testlog.CapturingHandler) { + tLog := testlog.Logger(gt, log.LevelInfo) + logger, capt := testlog.CaptureLogger(gt, log.LevelInfo) + p := devtest.NewP(context.Background(), logger, func(bool) { panic("fail") }, func() { panic("skip") }) + gt.Cleanup(p.Close) + logCallback := logpipe.LogCallback(func(line []byte) { + logger.Info(string(line)) + tLog.Info("Sub-process logged message", "line", string(line)) + }) + return NewSubProcess(p, logCallback, logCallback), capt +} + +func TestSubProcessKillEndsProcessThatIgnoresInterrupts(gt *testing.T) { + sp, capt := newTestSubProcess(gt) + require.NoError(gt, sp.Start("/bin/sh", []string{"-c", "trap '' INT TERM; exec /bin/sleep 10000000000"}, []string{})) + + start := time.Now() + require.NoError(gt, sp.Kill()) + require.Less(gt, time.Since(start), controlledKillWait, "SIGKILL must not wait for the interrupt grace period") + + require.Nil(gt, capt.FindLog(testlog.NewMessageFilter("Sending interrupt")), "Kill must not interrupt first") + require.NotNil(gt, capt.FindLog(testlog.NewMessageFilter("Sub-process killed"))) + + require.NoError(gt, sp.Start("/bin/echo", []string{"restart ok"}, []string{}), "the sub-process must restart after a kill") + require.NoError(gt, sp.Stop(false)) + require.NotNil(gt, capt.FindLog(testlog.NewMessageFilter("restart ok"))) +} + +func TestSubProcessKillOfExitedProcess(gt *testing.T) { + sp, capt := newTestSubProcess(gt) + require.NoError(gt, sp.Start("/bin/echo", []string{"done"}, []string{})) + <-sp.Exited() + + require.NoError(gt, sp.Kill()) + require.NotNil(gt, capt.FindLog(testlog.NewMessageFilter("Sub-process killed"))) + require.NoError(gt, sp.Kill(), "a second kill is a no-op") +} + // testEcho tests that we can handle a sub-process that completes on its own func testEcho(gt *testing.T, capt *testlog.CapturingHandler, sp *SubProcess) { require.NoError(gt, sp.Start("/bin/echo", []string{"hello world"}, []string{})) From 0c0c42c735471234dc841222bd2eaf7684e5930b Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Wed, 14 Oct 2026 07:34:52 -0700 Subject: [PATCH 5/8] op-acceptance-tests: cover proofs history across an unclean shutdown Two tests over threshold {7, 50} x proofs path {index, execute} x restart sync mode {CL, EL}: a proofs-history follower is killed once its store has flushed blocks reth has not persisted, restarted, and must catch up and serve proofs matching the sequencer's across the gap; in the second test the sequencer's op-reth is killed too, so the lost heights are rebuilt with different contents and the follower must serve the rebuilt chain's proofs rather than its store's. The second test fails on develop with the ParentHashMismatch crash of ethereum-optimism/protocol-team#464 in every cell and is skipped naming it. Co-Authored-By: Claude Fable 5.1 (cherry picked from commit 0462ddc9652285aa23357685d1952429f02f70ef) (cherry picked from commit 3053fcf35f4ae1819ac4f8e78db10518dc1769c1) --- .../proofs_history_restart_test.go | 367 ++++++++++++++++++ 1 file changed, 367 insertions(+) create mode 100644 op-acceptance-tests/tests/proofs_history_restart/proofs_history_restart_test.go diff --git a/op-acceptance-tests/tests/proofs_history_restart/proofs_history_restart_test.go b/op-acceptance-tests/tests/proofs_history_restart/proofs_history_restart_test.go new file mode 100644 index 00000000000..677f078b6e8 --- /dev/null +++ b/op-acceptance-tests/tests/proofs_history_restart/proofs_history_restart_test.go @@ -0,0 +1,367 @@ +package proofs_history_restart + +import ( + "fmt" + "testing" + "time" + + "github.com/ethereum-optimism/optimism/op-devstack/devtest" + "github.com/ethereum-optimism/optimism/op-devstack/dsl" + "github.com/ethereum-optimism/optimism/op-devstack/presets" + "github.com/ethereum-optimism/optimism/op-devstack/sysgo" + nodeSync "github.com/ethereum-optimism/optimism/op-node/rollup/sync" + "github.com/ethereum-optimism/optimism/op-service/bigs" + "github.com/ethereum-optimism/optimism/op-service/eth" + safety "github.com/ethereum-optimism/optimism/op-service/eth/safety" + "github.com/ethereum-optimism/optimism/op-service/txplan" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/vm" + "github.com/ethereum/go-ethereum/core/vm/program" +) + +const ( + // l2BlockTimeSeconds keeps the runs short: the kill point is a block height, so halving the + // block time halves the time spent producing blocks. + l2BlockTimeSeconds = 1 + // memoryBlockBufferTarget is reth's --engine.memory-block-buffer-target. It is passed + // explicitly, like the persistence threshold, so the gap the kill opens between the proofs + // store and reth's datadir does not depend on reth's defaults. + memoryBlockBufferTarget = 5 + // referenceEthProofWindow lets the sequencer answer eth_getProof for every block of the run + // from reth's own historical state; reth's default window of zero rejects them all. + referenceEthProofWindow = 1_000_000 + // postRestartTrafficBlocks is how far past the head the chain is driven once the follower has + // caught up, so the sweep also covers blocks the proofs store indexed after coming back. + postRestartTrafficBlocks = 10 + + // storeCatchUpTimeout bounds the wait for a proofs store flush before the kill. + storeCatchUpTimeout = 60 * time.Second + // resyncAttempts bounds, in 2s polls, how long the restarted follower may take to catch up with + // the sequencer. A follower whose op-reth crash-loops never does. + resyncAttempts = 60 + // rebuildAttempts bounds, in 2s polls, how long the restarted sequencer may take to rebuild the + // heights it lost with different contents. + rebuildAttempts = 30 +) + +// killPoint says where, relative to the proofs store's flush cadence, the follower is killed. +type killPoint string + +const ( + // killAfterStoreFlush kills the follower as soon as the store has flushed the target block, so + // the store's tip is the chain head and its lead over reth's datadir is the largest. + killAfterStoreFlush killPoint = "after_flush" + // killMidStoreBuffer kills the follower a couple of blocks after a flush, so the store's tip + // trails the chain head and the store loses blocks of its own on top of reth's. + killMidStoreBuffer killPoint = "mid_buffer" +) + +// restartCase is one cell of the matrix the follower runs under. +type restartCase struct { + killPoint killPoint + persistenceThreshold uint64 + // storeFlushBeforeKill is the proofs store tip the kill waits for. It leaves reth's persisted + // tip behind it: at threshold 7 reth persists every few blocks, at 50 it has persisted + // exactly once by block 60 and holds the rest in memory. + storeFlushBeforeKill uint64 + syncMode nodeSync.Mode + verificationInterval uint64 +} + +func (c restartCase) name() string { + proofsPath := "index" + if c.verificationInterval != 0 { + proofsPath = "execute" + } + syncMode := "cl_sync" + if c.syncMode == nodeSync.ELSync { + syncMode = "el_sync" + } + return fmt.Sprintf("threshold_%d-%s-%s-kill_%s", c.persistenceThreshold, proofsPath, syncMode, c.killPoint) +} + +// engineArgs pins reth's persistence schedule, which decides how many blocks an unclean stop +// loses. Both nodes get them so the sequencer loses blocks on the same schedule as the follower. +func (c restartCase) engineArgs() []string { + return []string{ + fmt.Sprintf("--engine.memory-block-buffer-target=%d", memoryBlockBufferTarget), + fmt.Sprintf("--engine.persistence-threshold=%d", c.persistenceThreshold), + } +} + +func (c restartCase) followerArgs() []string { + return append(c.engineArgs(), fmt.Sprintf("--proofs-history.verification-interval=%d", c.verificationInterval)) +} + +// restartMatrix pairs the kill point with the persistence threshold rather than crossing them: +// the small threshold gets the mid-buffer kill, where both reth and the store lose blocks, the +// large one the after-flush kill, where the store's lead over reth is largest. +func restartMatrix() []restartCase { + var cases []restartCase + for _, schedule := range []struct { + killPoint killPoint + persistenceThreshold uint64 + storeFlushBeforeKill uint64 + }{ + {killPoint: killMidStoreBuffer, persistenceThreshold: 7, storeFlushBeforeKill: 25}, + {killPoint: killAfterStoreFlush, persistenceThreshold: 50, storeFlushBeforeKill: 60}, + } { + for _, verificationInterval := range []uint64{0, 1} { + for _, syncMode := range []nodeSync.Mode{nodeSync.CLSync, nodeSync.ELSync} { + cases = append(cases, restartCase{ + killPoint: schedule.killPoint, + persistenceThreshold: schedule.persistenceThreshold, + storeFlushBeforeKill: schedule.storeFlushBeforeKill, + syncMode: syncMode, + verificationInterval: verificationInterval, + }) + } + } + } + return cases +} + +// skipUnsupportedCell skips the cells the configured clients cannot run. +func skipUnsupportedCell(t devtest.T, c restartCase) { + sysgo.SkipOnOpGeth(t, "proofs history is an op-reth feature") + if c.syncMode == nodeSync.CLSync { + sysgo.SkipOnKonaNode(t, "kona-node has no CL sync mode: it always lets the EL sync the gap") + } +} + +type restartSystem struct { + Batcher *dsl.L2Batcher + Follower *dsl.L2ELNode + FollowerCL *dsl.L2CLNode + FunderL2 *dsl.FunderEOA + Sequencer *dsl.L2ELNode + SequencerCL *dsl.L2CLNode +} + +// newRestartSystem starts a sequencer without proofs history, which serves as the independent +// eth_getProof reference, and a follower running proofs history under the case's reth, ExEx and +// sync-mode settings. +func newRestartSystem(t devtest.T, c restartCase) *restartSystem { + clKind := sysgo.ResolveMixedL2CLKind() + runtime := sysgo.NewMixedSingleChainRuntime(t, sysgo.MixedSingleChainPresetConfig{ + DeployerOptions: []sysgo.DeployerOption{sysgo.WithUniformL2BlockTimes(l2BlockTimeSeconds)}, + NodeSpecs: []sysgo.MixedSingleChainNodeSpec{ + { + CLKey: "sequencer", + CLKind: clKind, + ELKey: "sequencer", + ELKind: sysgo.MixedL2ELOpReth, + IsSequencer: true, + OpRethOpts: []sysgo.OpRethOption{ + sysgo.OpRethWithoutProofsHistory(), + sysgo.OpRethWithEthProofWindow(referenceEthProofWindow), + sysgo.OpRethWithExtraArgs(c.engineArgs()...), + }, + }, + { + CLKey: "follower", + CLKind: clKind, + ELKey: "proofs-follower", + ELKind: sysgo.MixedL2ELOpReth, + L2CLOpts: []sysgo.L2CLOption{sysgo.L2CLVerifierSyncMode(c.syncMode)}, + OpRethOpts: []sysgo.OpRethOption{sysgo.OpRethWithExtraArgs(c.followerArgs()...)}, + }, + }, + }) + frontends := presets.NewMixedSingleChainFrontends(t, runtime) + t.Require().Len(frontends.Nodes, 2, "expected the sequencer and the proofs follower") + + sys := &restartSystem{Batcher: frontends.L2Batcher, FunderL2: frontends.FunderL2} + for _, node := range frontends.Nodes { + if node.Spec.IsSequencer { + sys.Sequencer, sys.SequencerCL = node.EL, node.CL + } else { + sys.Follower, sys.FollowerCL = node.EL, node.CL + } + } + t.Require().NotNil(sys.Follower, "missing the proofs follower") + return sys +} + +// traffic changes state in nearly every block: alice and bob transfer back and forth and bob +// writes to a contract, so every block an unclean stop loses carries balance and storage changes +// that a rebuilt block would not repeat. +type traffic struct { + alice *dsl.EOA + bob *dsl.EOA + recorder common.Address + t devtest.T +} + +func newTraffic(t devtest.T, funder *dsl.FunderEOA) *traffic { + alice := funder.NewFundedEOA(eth.OneEther) + return &traffic{ + alice: alice, + bob: funder.NewFundedEOA(eth.OneEther), + recorder: deployBlockRecorder(t, alice), + t: t, + } +} + +// deployBlockRecorder deploys a contract that writes the block number to slot 0 and the block +// timestamp to slot 1 on every call, so every call changes its storage. +func deployBlockRecorder(t devtest.T, deployer *dsl.EOA) common.Address { + runtimeCode := program.New(). + Op(vm.NUMBER).Push(0).Op(vm.SSTORE). + Op(vm.TIMESTAMP).Push(1).Op(vm.SSTORE). + Bytes() + initCode := program.New().ReturnViaCodeCopy(runtimeCode).Bytes() + + tx := deployer.Transact(deployer.Plan(), txplan.WithData(initCode)) + receipt, err := tx.Included.Eval(t.Ctx()) + t.Require().NoError(err, "block recorder deployment must be included") + return receipt.ContractAddress +} + +// accounts are the accounts the proofs sweep proves at every block. +func (w *traffic) accounts() []dsl.ProofAccount { + return []dsl.ProofAccount{ + {Address: w.alice.Address()}, + {Address: w.bob.Address()}, + {Address: w.recorder, StorageKeys: []common.Hash{common.HexToHash("0x0"), common.HexToHash("0x1")}}, + } +} + +// driveUntilBlock sends transactions one after another until one lands in a block at or past +// target, and returns that block. +func (w *traffic) driveUntilBlock(target uint64) uint64 { + for { + w.alice.Transfer(w.bob.Address(), eth.OneHundredthEther) + w.bob.Transfer(w.alice.Address(), eth.OneHundredthEther) + block := includedBlock(w.t, w.bob.Transact(w.bob.Plan(), txplan.WithTo(&w.recorder))) + if block >= target { + return block + } + } +} + +func includedBlock(t devtest.T, tx *txplan.PlannedTx) uint64 { + receipt, err := tx.Included.Eval(t.Ctx()) + t.Require().NoError(err, "transaction must be included") + return bigs.Uint64Strict(receipt.BlockNumber) +} + +// uncleanStop is what the follower held when it was killed: the chain head it had imported and +// the last block its proofs store had flushed, both on the chain of that moment. +type uncleanStop struct { + head eth.L2BlockRef + storeTip eth.L2BlockRef +} + +// killFollower drives traffic until the follower's proofs store has flushed the case's target +// block, kills the follower's op-reth at the case's kill point and then stops its CL. +func killFollower(t devtest.T, sys *restartSystem, w *traffic, c restartCase) uncleanStop { + // Stop sending transactions two blocks short of the flush: close enough that the last + // transactions sit above reth's persisted tip, yet with no transaction in flight to delay the + // kill once the flush is observed. + w.driveUntilBlock(c.storeFlushBeforeKill - 2) + window := sys.Follower.WaitForProofsStoreBlock(c.storeFlushBeforeKill, storeCatchUpTimeout) + storeTip := sys.Follower.BlockRefByNumber(*window.Latest) + if c.killPoint == killMidStoreBuffer { + sys.Follower.WaitForBlockNumber(storeTip.Number + 2) + } + head := sys.Follower.BlockRefByLabel(eth.Unsafe) + t.Logger().Info("Killing the follower", "head", head, "proofs_store_tip", storeTip, "kill_point", c.killPoint) + + sys.Follower.StopUngracefully() + sys.FollowerCL.Stop() + return uncleanStop{head: head, storeTip: storeTip} +} + +// restartFollower brings the follower's op-reth and CL back and re-peers them with the sequencer, +// since both peerings died with the processes. It returns the block reth resumes from: its datadir +// tip, which must be below the proofs store's tip for the test to exercise the crash gap at all. +func restartFollower(t devtest.T, sys *restartSystem, stop uncleanStop) eth.L2BlockRef { + sys.Follower.Start() + sys.Follower.WaitForOnline() + // Nothing feeds the EL until its CL is back, so its head is what reth had persisted. + persistedTip := sys.Follower.BlockRefByLabel(eth.Unsafe) + t.Logger().Info("Follower op-reth is back on its datadir", "persisted_tip", persistedTip, + "proofs_store_tip", stop.storeTip, "head_at_kill", stop.head) + t.Require().Lessf(persistedTip.Number, stop.storeTip.Number, + "the proofs store (tip %d) must be ahead of reth's datadir (tip %d) after the kill; the kill landed too late for this threshold", + stop.storeTip.Number, persistedTip.Number) + + sys.Follower.PeerWith(sys.Sequencer) + sys.FollowerCL.Start() + sys.FollowerCL.ConnectPeer(sys.SequencerCL) + return persistedTip +} + +// TestProofsHistoryFollowerRecoversFromUncleanShutdown kills a proofs-history follower once its +// store has flushed blocks reth has not yet persisted, restarts it into the same chain and +// requires it to catch up with the sequencer, keep indexing, and serve proofs that match the +// sequencer's for the blocks it lost, the blocks it re-imported and the blocks that followed. +func TestProofsHistoryFollowerRecoversFromUncleanShutdown(gt *testing.T) { + t := devtest.ParallelT(gt) + for _, c := range restartMatrix() { + t.Run(c.name(), func(t devtest.T) { + t.Parallel() + skipUnsupportedCell(t, c) + sys := newRestartSystem(t, c) + w := newTraffic(t, sys.FunderL2) + + stop := killFollower(t, sys, w, c) + persistedTip := restartFollower(t, sys, stop) + + sys.Follower.InSync(sys.Sequencer, safety.LocalUnsafe, resyncAttempts) + lastBlock := w.driveUntilBlock(sys.Sequencer.BlockRefByLabel(eth.Unsafe).Number + postRestartTrafficBlocks) + sys.Follower.VerifyHistoricalProofsMatch(sys.Sequencer, func(opts *dsl.HistoricalProofsOpts) { + opts.Accounts = w.accounts() + opts.FirstBlock = persistedTip.Number + opts.LastBlock = lastBlock + }) + }) + } +} + +// TestProofsHistoryFollowerServesRebuiltBlocksAfterUncleanShutdown kills a proofs-history +// follower, then kills the sequencer's op-reth too while the blocks the follower's store holds are +// still only in the sequencer's memory, so the restarted sequencer rebuilds those heights with +// different contents. The restarted follower must follow the rebuilt chain and serve its proofs, +// not the ones its store recorded for the chain that no longer exists. +func TestProofsHistoryFollowerServesRebuiltBlocksAfterUncleanShutdown(gt *testing.T) { + t := devtest.ParallelT(gt) + for _, c := range restartMatrix() { + t.Run(c.name(), func(t devtest.T) { + t.Parallel() + skipUnsupportedCell(t, c) + t.Skip("fails on develop: the restarted follower's op-reth exits with `Critical task exex panicked: ExEx proofs-history crashed: Parent hash mismatch` at the block after its proofs store's tip, because the store is never reconciled with reth's datadir (ethereum-optimism/protocol-team#464); un-skip with #23358") + sys := newRestartSystem(t, c) + // Blocks that reached L1 would be derived back as they were; blocks that never did + // are re-sequenced, which is the rebuild this test needs. + sys.Batcher.Stop() + w := newTraffic(t, sys.FunderL2) + + stop := killFollower(t, sys, w, c) + + sys.Sequencer.StopUngracefully() + sys.SequencerCL.Stop() + sys.Sequencer.Start() + sys.Sequencer.WaitForOnline() + sequencerPersistedTip := sys.Sequencer.BlockRefByLabel(eth.Unsafe) + t.Logger().Info("Sequencer op-reth is back on its datadir", "persisted_tip", sequencerPersistedTip, + "follower_proofs_store_tip", stop.storeTip) + t.Require().Lessf(sequencerPersistedTip.Number, stop.storeTip.Number, + "the sequencer must rebuild blocks the follower's proofs store holds, but its datadir tip %d is not below the store tip %d", + sequencerPersistedTip.Number, stop.storeTip.Number) + sys.SequencerCL.Start() + sys.Sequencer.ReorgTriggered(stop.storeTip, rebuildAttempts) + + persistedTip := restartFollower(t, sys, stop) + + sys.Follower.InSync(sys.Sequencer, safety.LocalUnsafe, resyncAttempts) + lastBlock := w.driveUntilBlock(sys.Sequencer.BlockRefByLabel(eth.Unsafe).Number + postRestartTrafficBlocks) + sys.Follower.VerifyHistoricalProofsMatch(sys.Sequencer, func(opts *dsl.HistoricalProofsOpts) { + opts.Accounts = w.accounts() + opts.FirstBlock = persistedTip.Number + opts.LastBlock = lastBlock + }) + }) + } +} From 35282e533542cf3452287b655a815409d3426583 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Wed, 14 Oct 2026 07:55:37 -0700 Subject: [PATCH 6/8] op-devstack,op-acceptance-tests: address review of the unclean-shutdown tests Kill reaps a process that died on its own with a warning instead of reporting a kill, so a crash before the kill stays visible in the logs. The rebuild test kills both op-reths back to back before stopping the CLs, so the sequencer's datadir tip cannot catch up with the follower's store tip while the follower's CL shuts down, re-reads the store tip right before the kill, and requires the workload's funding block to survive the sequencer's crash. The NUMBER/TIMESTAMP recorder contract moves into the DSL as EOA.DeployBlockRecorder, shared by both proofs-history packages. Comments no longer claim SIGKILL models a power loss. Co-Authored-By: Claude Fable 5.1 (cherry picked from commit c474193c6dac3f0e1286580a95725ba2cc379f4c) (cherry picked from commit b96b619824ac56c77b80ad963dfa9d208d1f20d4) --- .../proofs_history_match_test.go | 19 +--- .../proofs_history_restart_test.go | 101 +++++++++--------- op-devstack/dsl/eoa.go | 17 +++ op-devstack/dsl/l2_el.go | 5 +- op-devstack/stack/lifecycle.go | 5 +- op-devstack/sysgo/l2_cl.go | 3 +- op-devstack/sysgo/l2_cl_test.go | 2 +- op-devstack/sysgo/l2_el_opreth.go | 6 +- op-devstack/sysgo/subproc.go | 21 ++-- op-devstack/sysgo/subproc_test.go | 5 +- 10 files changed, 93 insertions(+), 91 deletions(-) diff --git a/op-acceptance-tests/tests/proofs_history_match/proofs_history_match_test.go b/op-acceptance-tests/tests/proofs_history_match/proofs_history_match_test.go index c5d3c15fbd4..2d92fe1d76e 100644 --- a/op-acceptance-tests/tests/proofs_history_match/proofs_history_match_test.go +++ b/op-acceptance-tests/tests/proofs_history_match/proofs_history_match_test.go @@ -12,8 +12,6 @@ import ( "github.com/ethereum-optimism/optimism/op-service/eth" "github.com/ethereum-optimism/optimism/op-service/txplan" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core/vm" - "github.com/ethereum/go-ethereum/core/vm/program" ) const ( @@ -129,21 +127,6 @@ func newProofsHistorySystem(t devtest.T, c proofsHistoryCase) *proofsHistorySyst } } -// deployBlockRecorder deploys a contract that writes the block number to slot 0 and the block -// timestamp to slot 1 on every call, so every call changes its storage. -func deployBlockRecorder(t devtest.T, deployer *dsl.EOA) common.Address { - runtimeCode := program.New(). - Op(vm.NUMBER).Push(0).Op(vm.SSTORE). - Op(vm.TIMESTAMP).Push(1).Op(vm.SSTORE). - Bytes() - initCode := program.New().ReturnViaCodeCopy(runtimeCode).Bytes() - - tx := deployer.Transact(deployer.Plan(), txplan.WithData(initCode)) - receipt, err := tx.Included.Eval(t.Ctx()) - t.Require().NoError(err, "block recorder deployment must be included") - return receipt.ContractAddress -} - func includedBlock(t devtest.T, tx *txplan.PlannedTx) uint64 { receipt, err := tx.Included.Eval(t.Ctx()) t.Require().NoError(err, "transaction must be included") @@ -166,7 +149,7 @@ func TestProofsHistoryFollowerServesMatchingHistoricalProofs(gt *testing.T) { alice := sys.FunderL2.NewFundedEOA(eth.OneEther) bob := sys.FunderL2.NewFundedEOA(eth.OneEther) - recorder := deployBlockRecorder(t, alice) + recorder := alice.DeployBlockRecorder() var lastWorkloadBlock uint64 for round := range workloadRounds { if round == workloadRounds-1 { diff --git a/op-acceptance-tests/tests/proofs_history_restart/proofs_history_restart_test.go b/op-acceptance-tests/tests/proofs_history_restart/proofs_history_restart_test.go index 677f078b6e8..4e4f4a79001 100644 --- a/op-acceptance-tests/tests/proofs_history_restart/proofs_history_restart_test.go +++ b/op-acceptance-tests/tests/proofs_history_restart/proofs_history_restart_test.go @@ -15,8 +15,6 @@ import ( safety "github.com/ethereum-optimism/optimism/op-service/eth/safety" "github.com/ethereum-optimism/optimism/op-service/txplan" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core/vm" - "github.com/ethereum/go-ethereum/core/vm/program" ) const ( @@ -27,21 +25,21 @@ const ( // explicitly, like the persistence threshold, so the gap the kill opens between the proofs // store and reth's datadir does not depend on reth's defaults. memoryBlockBufferTarget = 5 - // referenceEthProofWindow lets the sequencer answer eth_getProof for every block of the run - // from reth's own historical state; reth's default window of zero rejects them all. - referenceEthProofWindow = 1_000_000 // postRestartTrafficBlocks is how far past the head the chain is driven once the follower has // caught up, so the sweep also covers blocks the proofs store indexed after coming back. postRestartTrafficBlocks = 10 + // referenceEthProofWindow lets the sequencer answer eth_getProof for every block of the run + // from reth's own historical state; reth's default window of zero rejects them all. + referenceEthProofWindow = 1_000_000 + // rebuildAttempts bounds how long the restarted sequencer may take to rebuild the heights it + // lost with different contents. + rebuildAttempts = 30 + // resyncAttempts bounds how long the restarted follower may take to catch up with the + // sequencer. A follower whose op-reth crash-loops never does. + resyncAttempts = 60 // storeCatchUpTimeout bounds the wait for a proofs store flush before the kill. storeCatchUpTimeout = 60 * time.Second - // resyncAttempts bounds, in 2s polls, how long the restarted follower may take to catch up with - // the sequencer. A follower whose op-reth crash-loops never does. - resyncAttempts = 60 - // rebuildAttempts bounds, in 2s polls, how long the restarted sequencer may take to rebuild the - // heights it lost with different contents. - rebuildAttempts = 30 ) // killPoint says where, relative to the proofs store's flush cadence, the follower is killed. @@ -60,9 +58,7 @@ const ( type restartCase struct { killPoint killPoint persistenceThreshold uint64 - // storeFlushBeforeKill is the proofs store tip the kill waits for. It leaves reth's persisted - // tip behind it: at threshold 7 reth persists every few blocks, at 50 it has persisted - // exactly once by block 60 and holds the rest in memory. + // storeFlushBeforeKill is the proofs store tip the kill waits for. storeFlushBeforeKill uint64 syncMode nodeSync.Mode verificationInterval uint64 @@ -95,7 +91,9 @@ func (c restartCase) followerArgs() []string { // restartMatrix pairs the kill point with the persistence threshold rather than crossing them: // the small threshold gets the mid-buffer kill, where both reth and the store lose blocks, the -// large one the after-flush kill, where the store's lead over reth is largest. +// large one the after-flush kill, where the store's lead over reth is largest. The kill heights +// leave reth's datadir behind the store: at threshold 7 reth persists every few blocks, at 50 it +// has persisted once by block 60 (down to block 46) and holds the rest in memory. func restartMatrix() []restartCase { var cases []restartCase for _, schedule := range []struct { @@ -183,43 +181,32 @@ func newRestartSystem(t devtest.T, c restartCase) *restartSystem { return sys } -// traffic changes state in nearly every block: alice and bob transfer back and forth and bob +// workload changes state in nearly every block: alice and bob transfer back and forth and bob // writes to a contract, so every block an unclean stop loses carries balance and storage changes // that a rebuilt block would not repeat. -type traffic struct { +type workload struct { alice *dsl.EOA bob *dsl.EOA recorder common.Address - t devtest.T + // setupBlock is the block that completed the funding and the contract deployment; a rebuilt + // chain must start below it or the workload's accounts no longer exist. + setupBlock uint64 + t devtest.T } -func newTraffic(t devtest.T, funder *dsl.FunderEOA) *traffic { - alice := funder.NewFundedEOA(eth.OneEther) - return &traffic{ - alice: alice, - bob: funder.NewFundedEOA(eth.OneEther), - recorder: deployBlockRecorder(t, alice), - t: t, +func newWorkload(t devtest.T, funder *dsl.FunderEOA) *workload { + w := &workload{ + alice: funder.NewFundedEOA(eth.OneEther), + bob: funder.NewFundedEOA(eth.OneEther), + t: t, } -} - -// deployBlockRecorder deploys a contract that writes the block number to slot 0 and the block -// timestamp to slot 1 on every call, so every call changes its storage. -func deployBlockRecorder(t devtest.T, deployer *dsl.EOA) common.Address { - runtimeCode := program.New(). - Op(vm.NUMBER).Push(0).Op(vm.SSTORE). - Op(vm.TIMESTAMP).Push(1).Op(vm.SSTORE). - Bytes() - initCode := program.New().ReturnViaCodeCopy(runtimeCode).Bytes() - - tx := deployer.Transact(deployer.Plan(), txplan.WithData(initCode)) - receipt, err := tx.Included.Eval(t.Ctx()) - t.Require().NoError(err, "block recorder deployment must be included") - return receipt.ContractAddress + w.recorder = w.alice.DeployBlockRecorder() + w.setupBlock = includedBlock(t, w.alice.Transact(w.alice.Plan(), txplan.WithTo(&w.recorder))) + return w } // accounts are the accounts the proofs sweep proves at every block. -func (w *traffic) accounts() []dsl.ProofAccount { +func (w *workload) accounts() []dsl.ProofAccount { return []dsl.ProofAccount{ {Address: w.alice.Address()}, {Address: w.bob.Address()}, @@ -229,7 +216,7 @@ func (w *traffic) accounts() []dsl.ProofAccount { // driveUntilBlock sends transactions one after another until one lands in a block at or past // target, and returns that block. -func (w *traffic) driveUntilBlock(target uint64) uint64 { +func (w *workload) driveUntilBlock(target uint64) uint64 { for { w.alice.Transfer(w.bob.Address(), eth.OneHundredthEther) w.bob.Transfer(w.alice.Address(), eth.OneHundredthEther) @@ -253,23 +240,26 @@ type uncleanStop struct { storeTip eth.L2BlockRef } -// killFollower drives traffic until the follower's proofs store has flushed the case's target -// block, kills the follower's op-reth at the case's kill point and then stops its CL. -func killFollower(t devtest.T, sys *restartSystem, w *traffic, c restartCase) uncleanStop { +// killFollower drives the workload until the follower's proofs store has flushed the case's target +// block and kills the follower's op-reth at the case's kill point. Its CL is left to the caller, +// so a test that kills the sequencer too can do so right away. +func killFollower(t devtest.T, sys *restartSystem, w *workload, c restartCase) uncleanStop { // Stop sending transactions two blocks short of the flush: close enough that the last // transactions sit above reth's persisted tip, yet with no transaction in flight to delay the // kill once the flush is observed. w.driveUntilBlock(c.storeFlushBeforeKill - 2) window := sys.Follower.WaitForProofsStoreBlock(c.storeFlushBeforeKill, storeCatchUpTimeout) - storeTip := sys.Follower.BlockRefByNumber(*window.Latest) if c.killPoint == killMidStoreBuffer { - sys.Follower.WaitForBlockNumber(storeTip.Number + 2) + sys.Follower.WaitForBlockNumber(*window.Latest + 2) } + // Read the store tip again right before the kill: a flush since the wait would have moved it. + window, err := sys.Follower.ProofsSyncStatus() + t.Require().NoError(err, "debug_proofsSyncStatus must answer before the kill") + storeTip := sys.Follower.BlockRefByNumber(*window.Latest) head := sys.Follower.BlockRefByLabel(eth.Unsafe) t.Logger().Info("Killing the follower", "head", head, "proofs_store_tip", storeTip, "kill_point", c.killPoint) sys.Follower.StopUngracefully() - sys.FollowerCL.Stop() return uncleanStop{head: head, storeTip: storeTip} } @@ -304,9 +294,10 @@ func TestProofsHistoryFollowerRecoversFromUncleanShutdown(gt *testing.T) { t.Parallel() skipUnsupportedCell(t, c) sys := newRestartSystem(t, c) - w := newTraffic(t, sys.FunderL2) + w := newWorkload(t, sys.FunderL2) stop := killFollower(t, sys, w, c) + sys.FollowerCL.Stop() persistedTip := restartFollower(t, sys, stop) sys.Follower.InSync(sys.Sequencer, safety.LocalUnsafe, resyncAttempts) @@ -336,20 +327,26 @@ func TestProofsHistoryFollowerServesRebuiltBlocksAfterUncleanShutdown(gt *testin // Blocks that reached L1 would be derived back as they were; blocks that never did // are re-sequenced, which is the rebuild this test needs. sys.Batcher.Stop() - w := newTraffic(t, sys.FunderL2) + w := newWorkload(t, sys.FunderL2) + // Both op-reths die back to back, so the sequencer still holds the blocks the + // follower's store flushed; the CLs follow once the processes are gone. stop := killFollower(t, sys, w, c) - sys.Sequencer.StopUngracefully() + sys.FollowerCL.Stop() sys.SequencerCL.Stop() + sys.Sequencer.Start() sys.Sequencer.WaitForOnline() sequencerPersistedTip := sys.Sequencer.BlockRefByLabel(eth.Unsafe) t.Logger().Info("Sequencer op-reth is back on its datadir", "persisted_tip", sequencerPersistedTip, - "follower_proofs_store_tip", stop.storeTip) + "follower_proofs_store_tip", stop.storeTip, "workload_setup_block", w.setupBlock) t.Require().Lessf(sequencerPersistedTip.Number, stop.storeTip.Number, "the sequencer must rebuild blocks the follower's proofs store holds, but its datadir tip %d is not below the store tip %d", sequencerPersistedTip.Number, stop.storeTip.Number) + t.Require().LessOrEqualf(w.setupBlock, sequencerPersistedTip.Number, + "the workload's funding and contract (block %d) must survive the sequencer's crash, but its datadir tip is %d", + w.setupBlock, sequencerPersistedTip.Number) sys.SequencerCL.Start() sys.Sequencer.ReorgTriggered(stop.storeTip, rebuildAttempts) diff --git a/op-devstack/dsl/eoa.go b/op-devstack/dsl/eoa.go index 13ba617abef..f7ce1e89bff 100644 --- a/op-devstack/dsl/eoa.go +++ b/op-devstack/dsl/eoa.go @@ -22,6 +22,8 @@ import ( "github.com/ethereum-optimism/optimism/op-service/txplan" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/core/vm" + "github.com/ethereum/go-ethereum/core/vm/program" "github.com/ethereum/go-ethereum/crypto" ) @@ -252,6 +254,21 @@ func (u *EOA) DeployEventLogger() common.Address { return eventLoggerAddress } +// DeployBlockRecorder deploys a contract that writes the block number to storage slot 0 and the +// block timestamp to slot 1 on every call, so every call to it changes its storage. +func (u *EOA) DeployBlockRecorder() common.Address { + runtimeCode := program.New(). + Op(vm.NUMBER).Push(0).Op(vm.SSTORE). + Op(vm.TIMESTAMP).Push(1).Op(vm.SSTORE). + Bytes() + initCode := program.New().ReturnViaCodeCopy(runtimeCode).Bytes() + tx := txplan.NewPlannedTx(u.Plan(), txplan.WithData(initCode)) + res, err := tx.Included.Eval(u.ctx) + u.t.Require().NoError(err, "failed to deploy block recorder") + u.log.Info("deployed block recorder", "chainID", tx.ChainID.Value(), "address", res.ContractAddress) + return res.ContractAddress +} + func (u *EOA) DeployWETH() common.Address { // Use the e2e bindings which contain the WETH bytecode tx := txplan.NewPlannedTx(u.Plan(), txplan.WithData(common.FromHex(e2eBindings.WETHBin))) diff --git a/op-devstack/dsl/l2_el.go b/op-devstack/dsl/l2_el.go index 432a7a9fb81..bbbae68386a 100644 --- a/op-devstack/dsl/l2_el.go +++ b/op-devstack/dsl/l2_el.go @@ -525,9 +525,8 @@ func (el *L2ELNode) Stop() { lifecycle.Stop() } -// StopUngracefully ends the node's process without a shutdown, the way a crash or a power loss -// would, so whatever it had not yet written to disk is lost. Start brings it back on the same -// data directory. +// StopUngracefully ends the node's process without a shutdown, the way a crash would, so state +// it held only in memory is lost. Start brings the node back. func (el *L2ELNode) StopUngracefully() { el.log.Info("Stopping ungracefully", "name", el.inner.Name()) lifecycle, ok := el.inner.(stack.UngracefulLifecycle) diff --git a/op-devstack/stack/lifecycle.go b/op-devstack/stack/lifecycle.go index c235cb4f557..22278e6444f 100644 --- a/op-devstack/stack/lifecycle.go +++ b/op-devstack/stack/lifecycle.go @@ -13,8 +13,9 @@ type ControlledLifecycle interface { Running() bool } -// UngracefulLifecycle is implemented by components that can be stopped the way a crash or a -// power loss would stop them: without a shutdown, so whatever they held only in memory is lost. +// UngracefulLifecycle is implemented by components that can be stopped the way a process crash +// would stop them: no shutdown runs, so state held only in process memory is lost. Writes already +// handed to the OS survive, so this does not model a power loss. type UngracefulLifecycle interface { StopUngracefully() } diff --git a/op-devstack/sysgo/l2_cl.go b/op-devstack/sysgo/l2_cl.go index 0158dc47942..7de36189195 100644 --- a/op-devstack/sysgo/l2_cl.go +++ b/op-devstack/sysgo/l2_cl.go @@ -58,7 +58,8 @@ func L2CLFollowSource(source string) L2CLOption { } // L2CLVerifierSyncMode sets the sync mode a verifier runs: CL sync has op-node fill unsafe gaps -// from its peers, EL sync hands the forkchoice target to the EL and lets it sync from its own. +// from its peers, EL sync hands the forkchoice target to the EL and lets it sync from its own +// peers. func L2CLVerifierSyncMode(mode nodeSync.Mode) L2CLOption { return L2CLOptionFn(func(p devtest.T, _ ComponentTarget, cfg *L2CLConfig) { cfg.VerifierSyncMode = mode diff --git a/op-devstack/sysgo/l2_cl_test.go b/op-devstack/sysgo/l2_cl_test.go index fd41ee8b393..1479ba12049 100644 --- a/op-devstack/sysgo/l2_cl_test.go +++ b/op-devstack/sysgo/l2_cl_test.go @@ -9,7 +9,7 @@ import ( ) func TestL2CLVerifierSyncMode(t *testing.T) { - cfg := DefaultL2CLConfig() + cfg := &L2CLConfig{SequencerSyncMode: nodeSync.CLSync, VerifierSyncMode: nodeSync.CLSync} L2CLVerifierSyncMode(nodeSync.ELSync).Apply(nil, ComponentTarget{}, cfg) require.Equal(t, nodeSync.ELSync, cfg.VerifierSyncMode) diff --git a/op-devstack/sysgo/l2_el_opreth.go b/op-devstack/sysgo/l2_el_opreth.go index 6a31e5d0ea8..524c43d88b6 100644 --- a/op-devstack/sysgo/l2_el_opreth.go +++ b/op-devstack/sysgo/l2_el_opreth.go @@ -262,9 +262,9 @@ func (n *OpReth) Stop() { n.sub = nil } -// StopUngracefully ends the op-reth process with SIGKILL, the way a crash or a power loss would, -// so whatever it had not yet written to its datadir is lost. Start brings the node back on the -// same datadir. +// StopUngracefully ends the op-reth process with SIGKILL, the way a crash would, so whatever it +// had not yet handed to the OS for its datadir is lost. Start brings the node back on the same +// datadir. func (n *OpReth) StopUngracefully() { n.mu.Lock() defer n.mu.Unlock() diff --git a/op-devstack/sysgo/subproc.go b/op-devstack/sysgo/subproc.go index baf1bb588fb..7c248a6aec9 100644 --- a/op-devstack/sysgo/subproc.go +++ b/op-devstack/sysgo/subproc.go @@ -143,7 +143,9 @@ func (sp *SubProcess) stop(ctx context.Context, interrupt bool, interruptWait ti } // Kill ends the process with SIGKILL, so it runs none of its shutdown logic, and waits for it to -// exit. It is the unclean counterpart of Stop: whatever the process held only in memory is lost. +// exit. It is the unclean counterpart of Stop: whatever the process held only in its own memory +// is lost, while writes it had already handed to the OS survive, so this is a crash, not a +// power loss. func (sp *SubProcess) Kill() error { sp.mu.Lock() defer sp.mu.Unlock() @@ -153,16 +155,19 @@ func (sp *SubProcess) Kill() error { select { case <-sp.exited: - // already exited; nothing to kill + // A process that died on its own is a different event from the kill a caller asked for. + sp.p.Logger().Warn("Sub-process exited before it could be killed", "exit", sp.waitErr) + sp.releaseLocked() + return nil default: - sp.p.Logger().Info("Killing sub-process", "pid", sp.cmd.Process.Pid) - // The process may have exited between the check and the signal, which Kill reports as - // ErrProcessDone; the wait below then returns at once. - if err := sp.cmd.Process.Kill(); err != nil && !errors.Is(err, os.ErrProcessDone) { - return fmt.Errorf("failed to kill sub-process: %w", err) - } } + sp.p.Logger().Info("Killing sub-process", "pid", sp.cmd.Process.Pid) + // The process may have exited between the check and the signal, which Kill reports as + // ErrProcessDone; the wait below then returns at once. + if err := sp.cmd.Process.Kill(); err != nil && !errors.Is(err, os.ErrProcessDone) { + return fmt.Errorf("failed to kill sub-process: %w", err) + } if !waitProcess(context.Background(), sp.exited, controlledKillWait) { return fmt.Errorf("sub-process did not exit within %s of SIGKILL", controlledKillWait) } diff --git a/op-devstack/sysgo/subproc_test.go b/op-devstack/sysgo/subproc_test.go index 221b929c16f..c398279989c 100644 --- a/op-devstack/sysgo/subproc_test.go +++ b/op-devstack/sysgo/subproc_test.go @@ -101,9 +101,7 @@ func TestSubProcessKillEndsProcessThatIgnoresInterrupts(gt *testing.T) { sp, capt := newTestSubProcess(gt) require.NoError(gt, sp.Start("/bin/sh", []string{"-c", "trap '' INT TERM; exec /bin/sleep 10000000000"}, []string{})) - start := time.Now() require.NoError(gt, sp.Kill()) - require.Less(gt, time.Since(start), controlledKillWait, "SIGKILL must not wait for the interrupt grace period") require.Nil(gt, capt.FindLog(testlog.NewMessageFilter("Sending interrupt")), "Kill must not interrupt first") require.NotNil(gt, capt.FindLog(testlog.NewMessageFilter("Sub-process killed"))) @@ -119,7 +117,8 @@ func TestSubProcessKillOfExitedProcess(gt *testing.T) { <-sp.Exited() require.NoError(gt, sp.Kill()) - require.NotNil(gt, capt.FindLog(testlog.NewMessageFilter("Sub-process killed"))) + require.NotNil(gt, capt.FindLog(testlog.NewMessageFilter("Sub-process exited before it could be killed"))) + require.Nil(gt, capt.FindLog(testlog.NewMessageFilter("Sub-process killed")), "an exit of its own must not read as a kill") require.NoError(gt, sp.Kill(), "a second kill is a no-op") } From 23b2cda019fd1bda9630f29a6889906473961eaa Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Wed, 14 Oct 2026 10:13:15 -0700 Subject: [PATCH 7/8] op-devstack,op-service: record op-reth log lines and pace the test sequencer Add a log recorder a test can register on an op-reth node: logpipe.LogRecord keeps every entry whose message, or raw text for an unstructured line, contains a substring, sysgo.OpReth feeds it from both pipes across restarts, and the preset frontend and dsl.L2ELNode expose it as RecordLogLines. dsl.EOA.PlanBlockRecorderDeployment returns the recorder deployment as a planned transaction so a test can keep its signed bytes. dsl.TestSequencer.SequenceBlockWithTxs closes out the job with Next after a successful publish, so consecutive blocks can be sequenced on one chain, and SequenceBlockWithTxsWhenDue builds a block no earlier than its timestamp, which op-node's gossip validation needs on publish and on receipt. Co-Authored-By: Claude Fable 5.1 (cherry picked from commit af71eab3168a2c283943fe4d26cc5f089e11769a) (cherry picked from commit 25c2917ba75d2d605b262e9191cd256f815a6c8f) --- op-devstack/dsl/eoa.go | 15 ++++-- op-devstack/dsl/l2_el.go | 11 ++++ op-devstack/dsl/sequencer.go | 25 ++++++++- op-devstack/presets/rpc_frontends.go | 8 +++ op-devstack/presets/sysgo_runtime.go | 3 ++ op-devstack/stack/logs.go | 10 ++++ op-devstack/sysgo/l2_el_opreth.go | 14 +++++ op-service/logpipe/pipe.go | 21 +++++++- op-service/logpipe/record.go | 80 ++++++++++++++++++++++++++++ op-service/logpipe/record_test.go | 60 +++++++++++++++++++++ 10 files changed, 239 insertions(+), 8 deletions(-) create mode 100644 op-devstack/stack/logs.go create mode 100644 op-service/logpipe/record.go create mode 100644 op-service/logpipe/record_test.go diff --git a/op-devstack/dsl/eoa.go b/op-devstack/dsl/eoa.go index f7ce1e89bff..598c2315969 100644 --- a/op-devstack/dsl/eoa.go +++ b/op-devstack/dsl/eoa.go @@ -254,15 +254,22 @@ func (u *EOA) DeployEventLogger() common.Address { return eventLoggerAddress } -// DeployBlockRecorder deploys a contract that writes the block number to storage slot 0 and the -// block timestamp to slot 1 on every call, so every call to it changes its storage. -func (u *EOA) DeployBlockRecorder() common.Address { +// PlanBlockRecorderDeployment plans the deployment of a contract that writes the block number to +// storage slot 0 and the block timestamp to slot 1 on every call, so every call to it changes its +// storage. The transaction is not sent. +func (u *EOA) PlanBlockRecorderDeployment() *txplan.PlannedTx { runtimeCode := program.New(). Op(vm.NUMBER).Push(0).Op(vm.SSTORE). Op(vm.TIMESTAMP).Push(1).Op(vm.SSTORE). Bytes() initCode := program.New().ReturnViaCodeCopy(runtimeCode).Bytes() - tx := txplan.NewPlannedTx(u.Plan(), txplan.WithData(initCode)) + return txplan.NewPlannedTx(u.Plan(), txplan.WithData(initCode)) +} + +// DeployBlockRecorder sends the deployment PlanBlockRecorderDeployment plans and returns the +// contract's address. +func (u *EOA) DeployBlockRecorder() common.Address { + tx := u.PlanBlockRecorderDeployment() res, err := tx.Included.Eval(u.ctx) u.t.Require().NoError(err, "failed to deploy block recorder") u.log.Info("deployed block recorder", "chainID", tx.ChainID.Value(), "address", res.ContractAddress) diff --git a/op-devstack/dsl/l2_el.go b/op-devstack/dsl/l2_el.go index bbbae68386a..afaf4e88070 100644 --- a/op-devstack/dsl/l2_el.go +++ b/op-devstack/dsl/l2_el.go @@ -17,6 +17,7 @@ import ( "github.com/ethereum-optimism/optimism/op-service/bigs" "github.com/ethereum-optimism/optimism/op-service/clock" "github.com/ethereum-optimism/optimism/op-service/eth" + "github.com/ethereum-optimism/optimism/op-service/logpipe" "github.com/ethereum-optimism/optimism/op-service/retry" "github.com/ethereum-optimism/optimism/op-service/txplan" @@ -540,6 +541,16 @@ func (el *L2ELNode) Start() { lifecycle.Start() } +// RecordLogLines returns a record of every log line this node's process writes from now on, +// across restarts, whose message contains substring. It fails for a node whose log the devstack +// does not expose, such as an in-process op-geth. +func (el *L2ELNode) RecordLogLines(substring string) *logpipe.LogRecord { + recorder, ok := el.inner.(stack.LogRecorder) + el.require.Truef(ok, "L2EL node %s does not expose its log", el.inner.Name()) + el.log.Info("Recording log lines", "name", el.inner.Name(), "substring", substring) + return recorder.RecordLogLines(substring) +} + func (el *L2ELNode) PeerWith(peer *L2ELNode) { sysgo.ConnectP2P(el.ctx, el.require, el.inner.L2EthClient().RPC(), peer.inner.L2EthClient().RPC()) } diff --git a/op-devstack/dsl/sequencer.go b/op-devstack/dsl/sequencer.go index 857c8712e7c..20315ddc545 100644 --- a/op-devstack/dsl/sequencer.go +++ b/op-devstack/dsl/sequencer.go @@ -7,6 +7,7 @@ import ( "github.com/ethereum-optimism/optimism/op-devstack/devtest" "github.com/ethereum-optimism/optimism/op-devstack/stack" + "github.com/ethereum-optimism/optimism/op-service/clock" "github.com/ethereum-optimism/optimism/op-service/eth" "github.com/ethereum-optimism/optimism/op-service/txplan" "github.com/ethereum-optimism/optimism/op-test-sequencer/sequencer/seqtypes" @@ -109,9 +110,29 @@ func (s *TestSequencer) SequenceBlockWithTxs(t devtest.T, chainID eth.ChainID, p require.NoError(t, ca.Sign(ctx)) require.NoError(t, ca.Commit(ctx)) - // Publish is optional - it broadcasts via P2P which may not be enabled in tests. + // Publish is optional: it broadcasts the block over P2P, which may not be enabled in tests. // The block is already committed and canonical at this point. - _ = ca.Publish(ctx) // ignore publish errors + if err := ca.Publish(ctx); err != nil { + s.log.Warn("Block committed but not published; its job stays sealed, so no further block can be sequenced on this chain", + "chain", chainID, "err", err) + return + } + // Every step of the job is done, so Next only closes it out, which lets the next block on + // this chain start. + require.NoError(t, ca.Next(ctx), "failed to close out the sequenced block") +} + +// SequenceBlockWithTxsWhenDue is SequenceBlockWithTxs for a chain whose head stands still: it +// waits until wall-clock time reaches the new block's timestamp, which its height fixes, before +// building. op-node rejects a gossiped payload more than 5s ahead of wall-clock time, on publish +// as well as on receipt, so a block built early would never reach the other nodes. +func (s *TestSequencer) SequenceBlockWithTxsWhenDue(t devtest.T, network *L2Network, parent eth.L2BlockRef, rawTxs [][]byte) { + due := time.Unix(int64(network.TimestampForBlockNum(parent.Number+1)), 0) + if wait := time.Until(due); wait > 0 { + s.log.Info("Waiting for the next block's timestamp before sequencing it", "block", parent.Number+1, "wait", wait) + require.NoError(t, clock.SystemClock.SleepCtx(t.Ctx(), wait)) // nosemgrep: flake-sleep-in-test -- the chain stands still; wall-clock time is the only thing to wait on + } + s.SequenceBlockWithTxs(t, network.ChainID(), parent.Hash, rawTxs) } // SequenceBlockWithPlannedTxs assigns consecutive nonces and includes the planned transactions in one block. diff --git a/op-devstack/presets/rpc_frontends.go b/op-devstack/presets/rpc_frontends.go index d0a41c6182b..9631aff65eb 100644 --- a/op-devstack/presets/rpc_frontends.go +++ b/op-devstack/presets/rpc_frontends.go @@ -19,6 +19,7 @@ import ( opclient "github.com/ethereum-optimism/optimism/op-service/client" "github.com/ethereum-optimism/optimism/op-service/eth" "github.com/ethereum-optimism/optimism/op-service/locks" + "github.com/ethereum-optimism/optimism/op-service/logpipe" "github.com/ethereum-optimism/optimism/op-service/sources" "github.com/ethereum-optimism/optimism/op-service/testreq" "github.com/ethereum-optimism/optimism/op-sync-tester/synctester" @@ -156,10 +157,12 @@ type l2ELFrontend struct { lifecycle stack.Lifecycle control stack.ControlledLifecycle ungraceful stack.UngracefulLifecycle + logRecorder stack.LogRecorder } var ( _ stack.L2ELNode = (*l2ELFrontend)(nil) + _ stack.LogRecorder = (*l2ELFrontend)(nil) _ stack.UngracefulLifecycle = (*l2ELFrontend)(nil) ) @@ -202,6 +205,11 @@ func (r *l2ELFrontend) StopUngracefully() { r.ungraceful.StopUngracefully() } +func (r *l2ELFrontend) RecordLogLines(substring string) *logpipe.LogRecord { + r.require().NotNil(r.logRecorder, "L2EL node %s does not expose its log", r.Name()) + return r.logRecorder.RecordLogLines(substring) +} + func (r *l2ELFrontend) StartControlled(ctx context.Context) error { if r.control == nil { return fmt.Errorf("L2EL node %s is not controllable", r.Name()) diff --git a/op-devstack/presets/sysgo_runtime.go b/op-devstack/presets/sysgo_runtime.go index ddf7a42fe27..18b7dcac546 100644 --- a/op-devstack/presets/sysgo_runtime.go +++ b/op-devstack/presets/sysgo_runtime.go @@ -69,6 +69,9 @@ func newL2ELFrontend(t devtest.T, name string, chainID eth.ChainID, userRPC stri if ungraceful, ok := lifecycle[0].(stack.UngracefulLifecycle); ok { l2EL.ungraceful = ungraceful } + if recorder, ok := lifecycle[0].(stack.LogRecorder); ok { + l2EL.logRecorder = recorder + } } return l2EL } diff --git a/op-devstack/stack/logs.go b/op-devstack/stack/logs.go new file mode 100644 index 00000000000..6169e2ccb9b --- /dev/null +++ b/op-devstack/stack/logs.go @@ -0,0 +1,10 @@ +package stack + +import "github.com/ethereum-optimism/optimism/op-service/logpipe" + +// LogRecorder is implemented by components whose process log a test can watch. +type LogRecorder interface { + // RecordLogLines returns a record of every log entry the component writes from now on, across + // restarts, whose message contains substring. + RecordLogLines(substring string) *logpipe.LogRecord +} diff --git a/op-devstack/sysgo/l2_el_opreth.go b/op-devstack/sysgo/l2_el_opreth.go index 524c43d88b6..9783a5d81bb 100644 --- a/op-devstack/sysgo/l2_el_opreth.go +++ b/op-devstack/sysgo/l2_el_opreth.go @@ -143,13 +143,25 @@ type OpReth struct { sub *SubProcess l2MetricsRegistrar L2MetricsRegistrar + + // logRecords outlives the sub-process so a record registered before a restart keeps + // collecting afterwards. + logRecords logpipe.LogRecords } var ( _ L2ELNode = (*OpReth)(nil) + _ stack.LogRecorder = (*OpReth)(nil) _ stack.UngracefulLifecycle = (*OpReth)(nil) ) +// RecordLogLines returns a record of every line op-reth writes to stdout or stderr from now on, +// across restarts, whose message, or whose raw text for a line that is not a structured log +// entry, contains substring. Lines below the level the test log keeps are recorded too. +func (n *OpReth) RecordLogLines(substring string) *logpipe.LogRecord { + return n.logRecords.Record(substring) +} + func (n *OpReth) Start() { n.mu.Lock() defer n.mu.Unlock() @@ -224,10 +236,12 @@ func (n *OpReth) Start() { e := logpipe.ParseRustStructuredLogs(line) logOut(e) onLogEntry(e) + n.logRecords.Observe(e) }) stdErrLogs := logpipe.LogCallback(func(line []byte) { e := logpipe.ParseRustStructuredLogs(line) logErr(e) + n.logRecords.Observe(e) }) n.sub = NewSubProcess(n.p, stdOutLogs, stdErrLogs) diff --git a/op-service/logpipe/pipe.go b/op-service/logpipe/pipe.go index c68e53d21eb..27fbdb7d4a6 100644 --- a/op-service/logpipe/pipe.go +++ b/op-service/logpipe/pipe.go @@ -23,15 +23,32 @@ type StructuredRustLogEntry struct { Target string } +// A line that is not structured JSON, such as a panic hook's output, becomes an entry with this +// message and the raw line in this field. +const ( + unparsedLineField = "line" + unparsedLineMessage = "Invalid JSON" +) + +// UnparsedLine returns the raw text of a line that was not structured JSON, and false for an +// entry that was. +func (e StructuredRustLogEntry) UnparsedLine() (string, bool) { + if e.Message != unparsedLineMessage { + return "", false + } + line, ok := e.Fields[unparsedLineField].(string) + return line, ok +} + func ParseRustStructuredLogs(line []byte) LogEntry { dec := json.NewDecoder(bytes.NewReader(line)) dec.UseNumber() // to preserve number formatting var e rawRustJSONLog if err := dec.Decode(&e); err != nil { return StructuredRustLogEntry{ - Message: "Invalid JSON", + Message: unparsedLineMessage, Level: slog.LevelWarn, - Fields: map[string]any{"line": string(line)}, + Fields: map[string]any{unparsedLineField: string(line)}, } } lvl, err := log.LevelFromString(e.Level) diff --git a/op-service/logpipe/record.go b/op-service/logpipe/record.go new file mode 100644 index 00000000000..7916ee99518 --- /dev/null +++ b/op-service/logpipe/record.go @@ -0,0 +1,80 @@ +package logpipe + +import ( + "strings" + "sync" +) + +// LogRecord keeps the entries of a log stream whose text contains a substring: the message of a +// structured entry, the raw line of an unstructured one. Observing and reading are safe from +// different goroutines, so a test can read a record while the stream is still being written. +type LogRecord struct { + substring string + + mu sync.Mutex + entries []LogEntry +} + +func newLogRecord(substring string) *LogRecord { + return &LogRecord{substring: substring} +} + +// Observe keeps e when its text contains the record's substring. +func (r *LogRecord) Observe(e LogEntry) { + if !strings.Contains(entryText(e), r.substring) { + return + } + r.mu.Lock() + defer r.mu.Unlock() + r.entries = append(r.entries, e) +} + +func entryText(e LogEntry) string { + if structured, ok := e.(StructuredRustLogEntry); ok { + if line, unparsed := structured.UnparsedLine(); unparsed { + return line + } + } + return e.LogMessage() +} + +// Entries returns the kept entries in the order they were observed. +func (r *LogRecord) Entries() []LogEntry { + r.mu.Lock() + defer r.mu.Unlock() + return append([]LogEntry(nil), r.entries...) +} + +// Count is the number of kept entries. +func (r *LogRecord) Count() int { + r.mu.Lock() + defer r.mu.Unlock() + return len(r.entries) +} + +// LogRecords fans one log stream out to every record registered on it. The zero value is ready +// to use. +type LogRecords struct { + mu sync.Mutex + records []*LogRecord +} + +// Record registers a new record for substring and returns it. Entries observed before the call +// are not part of it. +func (s *LogRecords) Record(substring string) *LogRecord { + record := newLogRecord(substring) + s.mu.Lock() + defer s.mu.Unlock() + s.records = append(s.records, record) + return record +} + +// Observe offers e to every registered record. +func (s *LogRecords) Observe(e LogEntry) { + s.mu.Lock() + records := append([]*LogRecord(nil), s.records...) + s.mu.Unlock() + for _, record := range records { + record.Observe(e) + } +} diff --git a/op-service/logpipe/record_test.go b/op-service/logpipe/record_test.go new file mode 100644 index 00000000000..b61a5ae1d16 --- /dev/null +++ b/op-service/logpipe/record_test.go @@ -0,0 +1,60 @@ +package logpipe + +import ( + "encoding/json" + "testing" + + "github.com/stretchr/testify/require" +) + +func rustLogLine(message string) []byte { + return []byte(`{"timestamp":"2026-01-01T00:00:00Z","level":"DEBUG","fields":{"message":"` + message + `","block":7},"target":"engine::tree"}`) +} + +func TestLogRecordKeepsMatchingEntriesInOrder(t *testing.T) { + record := newLogRecord("reorg") + record.Observe(ParseRustStructuredLogs(rustLogLine("on-disk reorg detected"))) + record.Observe(ParseRustStructuredLogs(rustLogLine("Block executed"))) + record.Observe(ParseRustStructuredLogs(rustLogLine("Handling reorg: unwinding and buffering new path"))) + + require.Equal(t, 2, record.Count()) + entries := record.Entries() + require.Len(t, entries, 2) + require.Equal(t, "on-disk reorg detected", entries[0].LogMessage()) + require.Equal(t, "Handling reorg: unwinding and buffering new path", entries[1].LogMessage()) + require.Equal(t, json.Number("7"), entries[0].FieldValue("block"), "entries keep their fields") +} + +func TestLogRecordMatchesTheRawTextOfUnstructuredLines(t *testing.T) { + record := newLogRecord("panicked at") + record.Observe(ParseRustStructuredLogs([]byte("thread 'main' panicked at src/lib.rs:7:9:"))) + record.Observe(ParseRustStructuredLogs([]byte("some other plain line"))) + record.Observe(ParseRustStructuredLogs(rustLogLine("Invalid JSON"))) + + require.Equal(t, 1, record.Count(), "the raw line is matched, not the placeholder message of an unstructured entry") + line, unparsed := record.Entries()[0].(StructuredRustLogEntry).UnparsedLine() + require.True(t, unparsed) + require.Equal(t, "thread 'main' panicked at src/lib.rs:7:9:", line) +} + +func TestLogRecordEntriesIsACopy(t *testing.T) { + record := newLogRecord("") + record.Observe(ParseRustStructuredLogs(rustLogLine("first"))) + entries := record.Entries() + record.Observe(ParseRustStructuredLogs(rustLogLine("second"))) + + require.Len(t, entries, 1, "entries returned earlier must not grow with the record") + require.Equal(t, 2, record.Count()) +} + +func TestLogRecordsFanOutFromRegistration(t *testing.T) { + var records LogRecords + records.Observe(ParseRustStructuredLogs(rustLogLine("before registration"))) + reorgs := records.Record("reorg") + everything := records.Record("") + records.Observe(ParseRustStructuredLogs(rustLogLine("on-disk reorg detected"))) + records.Observe(ParseRustStructuredLogs(rustLogLine("Block executed"))) + + require.Equal(t, 1, reorgs.Count()) + require.Equal(t, 2, everything.Count(), "a record only sees entries observed after it was registered") +} From 7d1f04cb52245c18e4073ee28b623ceabdf79d68 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Wed, 14 Oct 2026 10:13:17 -0700 Subject: [PATCH 8/8] op-acceptance-tests: cover proofs re-execution across a sequencer reorg A follower re-executing every block into its proofs store watches op-node's sequencer stop at a fork point, the test sequencer build the blocks the reorg replaces (a recorder deployment and calls into it, or transfers) and then one conflicting block, and op-node resume on top of it. The follower must adopt the replacement chain, keep executing and serve proofs from the fork point on that match the sequencer's historical state, over reth threshold {7, 50} x masking {0, 30} x depth {in memory shallow, in memory deep, on disk}, with reth's own on-disk reorg line asserted per cell. The dormant rust/op-reth/tests/proofs/reorg test, which CI never ran, is superseded and removed. Co-Authored-By: Claude Fable 5.1 (cherry picked from commit b591b623f0153941526348cbdd5fca346a5a8cd2) (cherry picked from commit aa393bd9f3396df8df088e07f6ba4bb382a2da2b) --- .../tests/proofs_history_reorg/matrix_test.go | 60 ++ .../proofs_history_reorg_test.go | 565 ++++++++++++++++++ rust/op-reth/tests/proofs/reorg/reorg_test.go | 204 ------- 3 files changed, 625 insertions(+), 204 deletions(-) create mode 100644 op-acceptance-tests/tests/proofs_history_reorg/matrix_test.go create mode 100644 op-acceptance-tests/tests/proofs_history_reorg/proofs_history_reorg_test.go delete mode 100644 rust/op-reth/tests/proofs/reorg/reorg_test.go diff --git a/op-acceptance-tests/tests/proofs_history_reorg/matrix_test.go b/op-acceptance-tests/tests/proofs_history_reorg/matrix_test.go new file mode 100644 index 00000000000..71539671e53 --- /dev/null +++ b/op-acceptance-tests/tests/proofs_history_reorg/matrix_test.go @@ -0,0 +1,60 @@ +package proofs_history_reorg + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestBlocksInRethMemory(t *testing.T) { + for _, tc := range []struct { + name string + threshold uint64 + height uint64 + want uint64 + }{ + {name: "nothing persisted below the first run", threshold: 7, height: 7, want: 7}, + {name: "first run keeps the buffer target", threshold: 7, height: 8, want: 5}, + {name: "grows by one per block", threshold: 7, height: 10, want: 7}, + {name: "next run three blocks later at threshold 7", threshold: 7, height: 11, want: 5}, + {name: "nothing persisted below the first run at threshold 50", threshold: 50, height: 50, want: 50}, + {name: "first run at threshold 50", threshold: 50, height: 51, want: 5}, + {name: "just before the second run at threshold 50", threshold: 50, height: 96, want: 50}, + {name: "second run at threshold 50", threshold: 50, height: 97, want: 5}, + } { + t.Run(tc.name, func(t *testing.T) { + c := reorgCase{persistenceThreshold: tc.threshold} + require.Equal(t, tc.want, c.blocksInRethMemory(tc.height)) + }) + } +} + +func TestInMemoryDepthsFitRethMemoryWithinOnePersistenceCycle(t *testing.T) { + for _, c := range reorgMatrix() { + if c.kind != inMemoryReorg { + continue + } + t.Run(c.name(), func(t *testing.T) { + for forkPoint := uint64(1); forkPoint < 200; forkPoint++ { + padding := uint64(0) + for !c.forkPointInMemory(forkPoint + padding + c.depth) { + padding++ + } + require.Less(t, padding, c.persistenceThreshold, "fork point %d", forkPoint) + } + }) + } +} + +func TestOnDiskDepthsExceedRethMemory(t *testing.T) { + for _, c := range reorgMatrix() { + if c.kind != onDiskReorg { + continue + } + t.Run(c.name(), func(t *testing.T) { + for h := c.persistenceThreshold + 1; h < 400; h++ { + require.False(t, c.forkPointInMemory(h), "height %d", h) + } + }) + } +} diff --git a/op-acceptance-tests/tests/proofs_history_reorg/proofs_history_reorg_test.go b/op-acceptance-tests/tests/proofs_history_reorg/proofs_history_reorg_test.go new file mode 100644 index 00000000000..f46137ebd4f --- /dev/null +++ b/op-acceptance-tests/tests/proofs_history_reorg/proofs_history_reorg_test.go @@ -0,0 +1,565 @@ +package proofs_history_reorg + +import ( + "fmt" + "testing" + "time" + + "github.com/ethereum-optimism/optimism/op-devstack/devtest" + "github.com/ethereum-optimism/optimism/op-devstack/dsl" + "github.com/ethereum-optimism/optimism/op-devstack/presets" + "github.com/ethereum-optimism/optimism/op-devstack/sysgo" + "github.com/ethereum-optimism/optimism/op-service/bigs" + "github.com/ethereum-optimism/optimism/op-service/eth" + safety "github.com/ethereum-optimism/optimism/op-service/eth/safety" + "github.com/ethereum-optimism/optimism/op-service/logpipe" + "github.com/ethereum-optimism/optimism/op-service/txplan" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/crypto" +) + +const ( + // l2BlockTimeSeconds keeps the runs short: the blocks sequenced by op-node before the fork + // point and after the reorg take a block time each. + l2BlockTimeSeconds = 1 + // memoryBlockBufferTarget is reth's --engine.memory-block-buffer-target, the number of newest + // blocks reth keeps in memory after persisting. It is passed explicitly, like the persistence + // threshold, because the reorg depths are chosen relative to it. + memoryBlockBufferTarget = 5 + // postReorgTrafficBlocks is how far past the sequencer's head the chain is driven once the + // follower has adopted the replacement chain, so the sweep also covers blocks the follower + // executed after the reorg. + postReorgTrafficBlocks = 10 + // referenceEthProofWindow lets the sequencer answer eth_getProof for every block of the run + // from reth's own historical state; reth's default window of zero rejects them all. + referenceEthProofWindow = 1_000_000 + + // blockAttempts bounds, in 2s polls, the wait for a node to reach a block the test sequencer + // just committed. + blockAttempts = 30 + // reorgAttempts bounds, in 2s polls, how long the follower may take to adopt the replacement + // chain. op-node drops gossiped blocks that do not extend its unsafe chain; once a gossiped + // block above its head arrives, its unsafe-gap request hands that block to reth, which + // fetches the missing replacement blocks from its EL peer and switches once they connect. + reorgAttempts = 90 + // storeCatchUpTimeout bounds the wait for the follower's proofs store to flush the chain + // head once sequencing stopped; the store flushes buffered blocks after 10s idle. + storeCatchUpTimeout = 60 * time.Second + // syncAttempts bounds, in 2s polls, the wait for the follower to catch up with the sequencer. + syncAttempts = 60 +) + +var ( + recorderNumberSlot = common.HexToHash("0x0") + recorderTimestampSlot = common.HexToHash("0x1") +) + +// reorgWorkload says what the replaced and the replacement blocks change. +type reorgWorkload string + +const ( + // deployedContract: the first replaced block deploys a contract and the replaced blocks + // after it call it. The replacement block re-includes the deployment, so the contract exists + // at the same address on the replacement chain, and calls it right away; the traffic after + // the reorg keeps calling it. The follower must execute calls into code that only a + // replacement block created. + deployedContract reorgWorkload = "deployed_contract" + // plainTransfers: every block moves ether between accounts, nothing else. + plainTransfers reorgWorkload = "plain_transfers" +) + +// reorgKind is how reth resolves the reorg, decided by where the fork point sits relative to +// reth's persisted tip. +type reorgKind string + +const ( + // inMemoryReorg: the fork point is a block reth still holds in memory or the block it + // persisted last, so only reth's in-memory chain is rewound. + inMemoryReorg reorgKind = "in_memory" + // onDiskReorg: the fork point lies below reth's persisted tip, so reth unwinds its database + // before it can extend the fork point. + onDiskReorg reorgKind = "on_disk" +) + +// reorgCase is one cell of the matrix the follower runs under. +type reorgCase struct { + // depth is the number of blocks the reorg replaces. + depth uint64 + kind reorgKind + persistenceThreshold uint64 + stateMaskingBlocks uint64 + workload reorgWorkload +} + +func (c reorgCase) name() string { + return fmt.Sprintf("threshold_%d-masking_%d-%s-depth_%d_%s", + c.persistenceThreshold, c.stateMaskingBlocks, c.workload, c.depth, c.kind) +} + +// engineArgs pin reth's persistence schedule on both nodes, which decides whether a fork point of +// the cell's depth is in memory or on disk. +func (c reorgCase) engineArgs() []string { + return []string{ + fmt.Sprintf("--engine.memory-block-buffer-target=%d", memoryBlockBufferTarget), + fmt.Sprintf("--engine.persistence-threshold=%d", c.persistenceThreshold), + } +} + +// followerArgs put the follower on the re-execution path for every block. +func (c reorgCase) followerArgs() []string { + args := append(c.engineArgs(), "--proofs-history.verification-interval=1") + if c.stateMaskingBlocks != 0 { + args = append(args, fmt.Sprintf("--engine.num-state-masking-blocks=%d", c.stateMaskingBlocks)) + } + return args +} + +// blocksInRethMemory models how many of the newest blocks reth holds in memory at height h: +// every block until the count first exceeds the persistence threshold, then the buffer target +// right after each persistence run, growing by one per block until the next run. +func (c reorgCase) blocksInRethMemory(h uint64) uint64 { + firstPersistence := c.persistenceThreshold + 1 + if h < firstPersistence { + return h + } + blocksBetweenRuns := firstPersistence - memoryBlockBufferTarget + return memoryBlockBufferTarget + (h-firstPersistence)%blocksBetweenRuns +} + +// forkPointInMemory says whether reth, by the model above, still holds the fork point of a +// reorg of the cell's depth in memory when the chain head is at height head. +func (c reorgCase) forkPointInMemory(head uint64) bool { + return c.blocksInRethMemory(head) >= c.depth +} + +// reorgMatrix crosses the reth settings and the workloads with three reorg depths per threshold: +// shallower than the memory block buffer target, so the fork point is among the blocks reth +// never persisted; deeper than it, so the fork point has been eligible for persistence; and +// deeper than reth ever holds in memory, which reth resolves as an on-disk reorg. reth holds +// between the buffer target and the threshold of blocks in memory, so the on-disk depths exceed +// the threshold with a margin. +func reorgMatrix() []reorgCase { + type depthCell struct { + depth uint64 + kind reorgKind + } + depthsByThreshold := []struct { + persistenceThreshold uint64 + depths []depthCell + }{ + {persistenceThreshold: 7, depths: []depthCell{{2, inMemoryReorg}, {6, inMemoryReorg}, {12, onDiskReorg}}}, + {persistenceThreshold: 50, depths: []depthCell{{2, inMemoryReorg}, {6, inMemoryReorg}, {55, onDiskReorg}}}, + } + var cases []reorgCase + for _, threshold := range depthsByThreshold { + for _, stateMaskingBlocks := range []uint64{0, 30} { + for _, workload := range []reorgWorkload{deployedContract, plainTransfers} { + for _, cell := range threshold.depths { + cases = append(cases, reorgCase{ + depth: cell.depth, + kind: cell.kind, + persistenceThreshold: threshold.persistenceThreshold, + stateMaskingBlocks: stateMaskingBlocks, + workload: workload, + }) + } + } + } + } + return cases +} + +// skipUnsupportedCell skips the cells the configured clients cannot run. +func skipUnsupportedCell(t devtest.T, c reorgCase) { + sysgo.SkipOnOpGeth(t, "proofs history is an op-reth feature") + sysgo.SkipOnKonaNode(t, "the test sequencer commits blocks through op-node's opstack_commitBlockV1 and opstack_publishBlockV1 RPCs, which kona-node does not serve") + if c.stateMaskingBlocks != 0 { + t.Skip("state masking is compiled out of reth 2.5.2 on develop and #23046 rejects a nonzero --engine.num-state-masking-blocks; re-enable with ethereum-optimism/protocol-team#457") + } +} + +type reorgSystem struct { + Batcher *dsl.L2Batcher + Follower *dsl.L2ELNode + FunderL2 *dsl.FunderEOA + L2Network *dsl.L2Network + Sequencer *dsl.L2ELNode + SequencerCL *dsl.L2CLNode + TestSequencer *dsl.TestSequencer +} + +// newReorgSystem starts a sequencer without proofs history, which serves as the independent +// eth_getProof reference and whose op-node the test sequencer drives around the reorg, and a +// follower re-executing every block into its proofs store under the cell's reth settings. +func newReorgSystem(t devtest.T, c reorgCase) *reorgSystem { + clKind := sysgo.ResolveMixedL2CLKind() + runtime := sysgo.NewMixedSingleChainRuntime(t, sysgo.MixedSingleChainPresetConfig{ + DeployerOptions: []sysgo.DeployerOption{sysgo.WithUniformL2BlockTimes(l2BlockTimeSeconds)}, + NodeSpecs: []sysgo.MixedSingleChainNodeSpec{ + { + CLKey: "sequencer", + CLKind: clKind, + ELKey: "sequencer", + ELKind: sysgo.MixedL2ELOpReth, + IsSequencer: true, + OpRethOpts: []sysgo.OpRethOption{ + sysgo.OpRethWithoutProofsHistory(), + sysgo.OpRethWithEthProofWindow(referenceEthProofWindow), + sysgo.OpRethWithExtraArgs(c.engineArgs()...), + }, + }, + { + CLKey: "follower", + CLKind: clKind, + ELKey: "proofs-follower", + ELKind: sysgo.MixedL2ELOpReth, + OpRethOpts: []sysgo.OpRethOption{sysgo.OpRethWithExtraArgs(c.followerArgs()...)}, + }, + }, + TestSequencerName: "test-sequencer", + WithTestSequencer: true, + }) + frontends := presets.NewMixedSingleChainFrontends(t, runtime) + t.Require().Len(frontends.Nodes, 2, "expected the sequencer and the proofs follower") + t.Require().NotNil(frontends.TestSequencer, "missing the test sequencer") + + sys := &reorgSystem{ + Batcher: frontends.L2Batcher, + FunderL2: frontends.FunderL2, + L2Network: frontends.L2Network, + TestSequencer: frontends.TestSequencer, + } + for _, node := range frontends.Nodes { + if node.Spec.IsSequencer { + sys.Sequencer, sys.SequencerCL = node.EL, node.CL + } else { + sys.Follower = node.EL + } + } + t.Require().NotNil(sys.Follower, "missing the proofs follower") + return sys +} + +// followerLogs are the follower's op-reth log lines the test reads after the reorg. +type followerLogs struct { + // leftChain is the ExEx handing a block that left reth's chain mid-execution to its sync + // loop; the line exists from the reth 2.6 bump (#23046) on. + leftChain *logpipe.LogRecord + // onDiskReorg is reth resolving a fork point below its persisted tip. + onDiskReorg *logpipe.LogRecord + // panicked is a reth critical task, such as the ExEx, crashing the node. + panicked *logpipe.LogRecord + // parentStateMissing is the ExEx skipping a block whose parent state reth no longer holds, + // leaving it to a later notification. + parentStateMissing *logpipe.LogRecord + // reorgHandled is the proofs engine unwinding blocks it stored and buffering the replacement + // path, as opposed to ignoring a reorg above its stored tip. + reorgHandled *logpipe.LogRecord + // reorgNotified is the ExEx receiving reth's reorg notification. + reorgNotified *logpipe.LogRecord +} + +func recordFollowerLogs(follower *dsl.L2ELNode) followerLogs { + return followerLogs{ + leftChain: follower.RecordLogLines("Block left reth's chain during proofs execution"), + onDiskReorg: follower.RecordLogLines("on-disk reorg detected"), + panicked: follower.RecordLogLines("panicked"), + parentStateMissing: follower.RecordLogLines("Parent state not available in reth; skipping execute_block"), + reorgHandled: follower.RecordLogLines("Handling reorg: unwinding and buffering new path"), + reorgNotified: follower.RecordLogLines("ChainReorged notification received"), + } +} + +// verify requires the follower to have handled the reorg the way the cell expects, and reports +// the lines whose presence depends on timing or on the op-reth version under test. +func (l followerLogs) verify(t devtest.T, c reorgCase) { + t.Require().NotZerof(l.reorgNotified.Count(), "the follower's ExEx never received a reorg notification") + t.Require().NotZerof(l.reorgHandled.Count(), "the follower's proofs engine never unwound the replaced blocks: the reorg did not reach blocks it had stored") + if c.kind == onDiskReorg { + t.Require().NotZerof(l.onDiskReorg.Count(), "expected a fork point below reth's persisted tip, but reth never reported an on-disk reorg; the depth %d is not deeper than what reth held in memory", c.depth) + } else { + t.Require().Zerof(l.onDiskReorg.Count(), "expected the fork point in reth's memory, but reth reported an on-disk reorg: %v", messagesOf(l.onDiskReorg)) + } + t.Logger().Info("Follower reorg handling log lines", + "reorg_notified", l.reorgNotified.Count(), + "reorg_handled", l.reorgHandled.Count(), + "on_disk_reorg", l.onDiskReorg.Count(), + "parent_state_missing_skips", l.parentStateMissing.Count(), + "left_chain_handoffs", l.leftChain.Count()) +} + +func messagesOf(record *logpipe.LogRecord) []string { + var messages []string + for _, entry := range record.Entries() { + messages = append(messages, fmt.Sprintf("%s %v", entry.LogMessage(), entry.LogFields())) + } + return messages +} + +// workload is the accounts the blocks around the reorg change. alice and bob transact before +// the fork point, in the replaced blocks and after the reorg; carol only ever transacts on the +// replacement chain, so the nonces her replacement transactions need are the same on both sides +// of the fork; the deployer's only transaction is the recorder deployment, which the first +// replaced block carries and the replacement block carries again. +type workload struct { + alice *dsl.EOA + bob *dsl.EOA + carol *dsl.EOA + deployer *dsl.EOA + kind reorgWorkload + t devtest.T + + deployment signedTx + recorder common.Address + // setupBlock is the block that completed the funding; the fork point must be above it. + setupBlock uint64 +} + +func newWorkload(t devtest.T, funder *dsl.FunderEOA, kind reorgWorkload) *workload { + w := &workload{ + alice: funder.NewFundedEOA(eth.OneEther), + bob: funder.NewFundedEOA(eth.OneEther), + carol: funder.NewFundedEOA(eth.OneEther), + deployer: funder.NewFundedEOA(eth.OneEther), + kind: kind, + t: t, + } + w.setupBlock = includedBlock(t, w.alice.Transfer(w.bob.Address(), eth.OneGWei)) + if kind == deployedContract { + w.deployment = signTx(t, w.deployer.PlanBlockRecorderDeployment()) + w.recorder = crypto.CreateAddress(w.deployer.Address(), w.deployment.nonce) + t.Logger().Info("Signed the recorder deployment", "address", w.recorder, "tx", w.deployment.hash) + } + return w +} + +// accounts are the accounts the proofs sweep proves at every block. +func (w *workload) accounts() []dsl.ProofAccount { + accounts := []dsl.ProofAccount{ + {Address: w.alice.Address()}, + {Address: w.bob.Address()}, + {Address: w.carol.Address()}, + {Address: w.deployer.Address()}, + } + if w.kind == deployedContract { + accounts = append(accounts, dsl.ProofAccount{Address: w.recorder, StorageKeys: []common.Hash{recorderNumberSlot, recorderTimestampSlot}}) + } + return accounts +} + +// driveUntilBlock sends transactions one after another, each after the previous one was +// included, until one lands in a block at or past target, and returns that block. +func (w *workload) driveUntilBlock(target uint64) uint64 { + for { + if block := includedBlock(w.t, w.alice.Transfer(w.bob.Address(), eth.OneHundredthEther)); block >= target { + return block + } + if block := includedBlock(w.t, w.bob.Transact(w.bobsReply())); block >= target { + return block + } + } +} + +// bobsReply is bob's part of the traffic: a call into the recorder for the deployed-contract +// workload, which before the deployment is a plain transaction to an address without code, a +// transfer back to alice otherwise. +func (w *workload) bobsReply() txplan.Option { + if w.kind == deployedContract { + return txplan.Combine(w.bob.Plan(), txplan.WithTo(&w.recorder)) + } + return w.bob.PlanTransfer(w.alice.Address(), eth.OneHundredthEther) +} + +// replacedBlockTxs are the transactions of the n-th replaced block, 1-based: the recorder +// deployment first and calls into it after that for the deployed-contract workload, transfers +// otherwise. +func (w *workload) replacedBlockTxs(n int) []signedTx { + switch { + case w.kind == deployedContract && n == 1: + return []signedTx{w.deployment, signTx(w.t, txplan.NewPlannedTx(w.alice.PlanTransfer(w.bob.Address(), eth.OneHundredthEther)))} + case w.kind == deployedContract: + return []signedTx{signTx(w.t, txplan.NewPlannedTx(w.alice.Plan(), txplan.WithTo(&w.recorder)))} + default: + return []signedTx{signTx(w.t, txplan.NewPlannedTx(w.alice.PlanTransfer(w.bob.Address(), eth.OneHundredthEther)))} + } +} + +// replacementBlockTxs are the transactions of the replacement block: a transfer from carol, and +// for the deployed-contract workload the same recorder deployment as on the replaced chain +// followed by carol's call into it. carol's nonces are assigned up front, since both of her +// transactions are signed before either is included. +func (w *workload) replacementBlockTxs(int) []signedTx { + nonce := w.carol.PendingNonce() + transfer := signTx(w.t, txplan.NewPlannedTx(w.carol.PlanTransfer(w.bob.Address(), eth.OneHundredthEther), txplan.WithStaticNonce(nonce))) + if w.kind != deployedContract { + return []signedTx{transfer} + } + call := signTx(w.t, txplan.NewPlannedTx(w.carol.Plan(), txplan.WithTo(&w.recorder), txplan.WithStaticNonce(nonce+1))) + return []signedTx{w.deployment, transfer, call} +} + +type signedTx struct { + hash common.Hash + nonce uint64 + raw []byte +} + +func signTx(t devtest.T, tx *txplan.PlannedTx) signedTx { + signed, err := tx.Signed.Eval(t.Ctx()) + t.Require().NoError(err, "transaction must sign") + raw, err := signed.MarshalBinary() + t.Require().NoError(err, "transaction must encode") + return signedTx{hash: signed.Hash(), nonce: signed.Nonce(), raw: raw} +} + +func rawTxs(txs []signedTx) [][]byte { + raws := make([][]byte, len(txs)) + for i, tx := range txs { + raws[i] = tx.raw + } + return raws +} + +func includedBlock(t devtest.T, tx *txplan.PlannedTx) uint64 { + receipt, err := tx.Included.Eval(t.Ctx()) + t.Require().NoError(err, "transaction must be included") + return bigs.Uint64Strict(receipt.BlockNumber) +} + +// sequencedChain is a run of consecutive blocks the test sequencer committed, lowest first, with +// the transactions each of them carries. +type sequencedChain struct { + blocks []eth.L2BlockRef + txs [][]signedTx +} + +func (s sequencedChain) head() eth.L2BlockRef { + return s.blocks[len(s.blocks)-1] +} + +// verifyServedBy requires node to serve every block of the chain with its transactions. +func (s sequencedChain) verifyServedBy(t devtest.T, node *dsl.L2ELNode) { + for i, block := range s.blocks { + t.Require().Equalf(block.Hash, node.BlockRefByNumber(block.Number).Hash, "%s must serve block %d of the chain", node, block.Number) + for _, tx := range s.txs[i] { + node.AssertTxInBlock(block.Number, tx.hash) + } + } +} + +// sequenceChain commits count blocks on parent with the test sequencer, each carrying the +// transactions blockTxs returns for it and none ahead of wall-clock time, and returns them once +// the sequencer's EL serves each. +func sequenceChain(t devtest.T, sys *reorgSystem, parent eth.L2BlockRef, count uint64, blockTxs func(n int) []signedTx) sequencedChain { + chain := sequencedChain{} + for i := range int(count) { + txs := blockTxs(i + 1) + t.Logger().Info("Sequencing a block with the test sequencer", "number", parent.Number+1, "parent", parent, "txs", len(txs)) + sys.TestSequencer.SequenceBlockWithTxsWhenDue(t, sys.L2Network, parent, rawTxs(txs)) + block := sys.Sequencer.BlockRefByNumber(parent.Number + 1) + t.Require().Equalf(parent.Hash, block.ParentHash, "block %d must extend %s", block.Number, parent) + chain.blocks = append(chain.blocks, block) + chain.txs = append(chain.txs, txs) + parent = block + } + return chain +} + +// stopSequencing drives traffic past the workload's setup and stops op-node's sequencer, which +// makes its head the fork point of the reorg. For an in-memory cell the chain is then padded +// with empty blocks until the replaced blocks will leave the fork point in reth's memory by the +// persistence model, so the fork point does not depend on where the stop lands. +func stopSequencing(t devtest.T, sys *reorgSystem, w *workload, c reorgCase) eth.L2BlockRef { + w.driveUntilBlock(w.setupBlock + 1) + forkPoint := sys.Sequencer.BlockRefByHash(sys.SequencerCL.StopSequencer()) + padding := 0 + for c.kind == inMemoryReorg && !c.forkPointInMemory(forkPoint.Number+c.depth) { + t.Require().Lessf(padding, int(c.persistenceThreshold), + "no height keeps a fork point %d blocks deep in reth's memory at threshold %d", c.depth, c.persistenceThreshold) + forkPoint = sequenceChain(t, sys, forkPoint, 1, func(int) []signedTx { return nil }).head() + padding++ + } + t.Logger().Info("Sequencing stopped at the fork point", "fork_point", forkPoint, "depth", c.depth, "kind", c.kind, "padding_blocks", padding) + return forkPoint +} + +// sequenceReplacedChain builds the blocks the reorg will replace on the fork point and waits +// until the follower has imported them and its proofs store has flushed them, so every replaced +// block was re-executed into the store before it is replaced. +func sequenceReplacedChain(t devtest.T, sys *reorgSystem, w *workload, forkPoint eth.L2BlockRef, c reorgCase) sequencedChain { + replaced := sequenceChain(t, sys, forkPoint, c.depth, w.replacedBlockTxs) + sys.Follower.Reached(eth.Unsafe, replaced.head().Number, blockAttempts) + replaced.verifyServedBy(t, sys.Follower) + sys.Follower.WaitForProofsStoreBlock(replaced.head().Number, storeCatchUpTimeout) + return replaced +} + +// sequenceReplacementChain builds the first replacement block on the fork point, which reorgs +// the sequencer's EL, and waits until the sequencer's op-node is on it so it resumes sequencing +// from there. One block is all the test sequencer can build on a deep fork: its timestamp is +// the fork point's plus a block time, which op-node's gossip validation rejects as too old once +// the replaced chain took longer than the gossip age threshold to build, and a block that is +// not published leaves the test sequencer's job sealed. op-node builds the remaining +// replacement blocks, as it does after a real sequencer failover. +func sequenceReplacementChain(t devtest.T, sys *reorgSystem, w *workload, forkPoint eth.L2BlockRef, replaced sequencedChain) sequencedChain { + replacement := sequenceChain(t, sys, forkPoint, 1, w.replacementBlockTxs) + t.Require().NotEqualf(replaced.blocks[0].Hash, replacement.head().Hash, "the replacement block must differ from the replaced block at height %d", replacement.head().Number) + sys.SequencerCL.MatchedUnsafe(sys.Sequencer, syncAttempts) + return replacement +} + +// verifyRecorderCalled requires the recorder's storage on node to hold the number of a block +// after the fork point at block, which only a call on the replacement chain can have written, so +// the sweep's matching proofs of that slot were not vacuous. +func verifyRecorderCalled(t devtest.T, node *dsl.L2ELNode, w *workload, forkPoint eth.L2BlockRef, block uint64) { + ref := node.BlockRefByNumber(block) + value, err := node.EthClient().ReadStorageAt(t.Ctx(), w.recorder, recorderNumberSlot, ref.Hash) + t.Require().NoError(err, "must read the recorder's storage at block %d", block) + t.Require().Greaterf(bigs.Uint64Strict(value.Big()), forkPoint.Number, + "the recorder at block %d must have been called on the replacement chain, but holds block number %s", block, value.Big()) +} + +// TestProofsHistoryFollowerMatchesTheReplacementChainAfterASequencerReorg has a follower that +// re-executes every block into its proofs store watch the sequencer replace blocks the store +// already holds: op-node stops sequencing at the fork point, the test sequencer builds the +// blocks to be replaced on top and the follower executes them, then the test sequencer builds a +// conflicting block on the fork point and op-node resumes on top of it. The follower must adopt +// the replacement chain, keep executing, and serve proofs from the fork point onwards that match +// the sequencer's own historical state. +func TestProofsHistoryFollowerMatchesTheReplacementChainAfterASequencerReorg(gt *testing.T) { + t := devtest.ParallelT(gt) + for _, c := range reorgMatrix() { + t.Run(c.name(), func(t devtest.T) { + t.Parallel() + skipUnsupportedCell(t, c) + sys := newReorgSystem(t, c) + // Blocks that reach L1 are derived back as they were, so the batcher stays off until + // the replacement chain exists and is the one it submits. + sys.Batcher.Stop() + logs := recordFollowerLogs(sys.Follower) + w := newWorkload(t, sys.FunderL2, c.workload) + + forkPoint := stopSequencing(t, sys, w, c) + replaced := sequenceReplacedChain(t, sys, w, forkPoint, c) + replacement := sequenceReplacementChain(t, sys, w, forkPoint, replaced) + sys.Batcher.Start() + sys.SequencerCL.StartSequencer() + + sys.Follower.ReorgExact(replaced.blocks[0], reorgAttempts) + t.Require().Emptyf(messagesOf(logs.panicked), "the follower's op-reth must survive the reorg") + sys.Follower.InSync(sys.Sequencer, safety.LocalUnsafe, syncAttempts) + replacement.verifyServedBy(t, sys.Follower) + lastBlock := w.driveUntilBlock(sys.Sequencer.BlockRefByLabel(eth.Unsafe).Number + postReorgTrafficBlocks) + + sys.Follower.VerifyHistoricalProofsMatch(sys.Sequencer, func(opts *dsl.HistoricalProofsOpts) { + opts.Accounts = w.accounts() + opts.FirstBlock = forkPoint.Number + opts.LastBlock = lastBlock + }) + if c.workload == deployedContract { + verifyRecorderCalled(t, sys.Follower, w, forkPoint, lastBlock) + } + logs.verify(t, c) + }) + } +} diff --git a/rust/op-reth/tests/proofs/reorg/reorg_test.go b/rust/op-reth/tests/proofs/reorg/reorg_test.go deleted file mode 100644 index 6c6a1af6fd7..00000000000 --- a/rust/op-reth/tests/proofs/reorg/reorg_test.go +++ /dev/null @@ -1,204 +0,0 @@ -package reorg - -import ( - "context" - "math/big" - "testing" - "time" - - "github.com/ethereum-optimism/optimism/op-devstack/devtest" - "github.com/ethereum-optimism/optimism/op-devstack/dsl" - "github.com/ethereum-optimism/optimism/op-e2e/e2eutils/wait" - "github.com/ethereum-optimism/optimism/op-service/bigs" - "github.com/ethereum-optimism/optimism/op-service/eth" - "github.com/ethereum-optimism/optimism/op-service/txplan" - "github.com/ethereum-optimism/optimism/op-test-sequencer/sequencer/seqtypes" - "github.com/ethereum-optimism/optimism/rust/op-reth/tests/proofs/utils" - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core/types" - "github.com/stretchr/testify/require" -) - -func TestReorgUsingAccountProof(gt *testing.T) { - t := devtest.SerialT(gt) - ctx := t.Ctx() - - sys := utils.NewMixedOpProofPreset(t) - l := sys.Log - - ia := sys.TestSequencer.Escape().ControlAPI(sys.L2Chain.ChainID()) - - // stop batcher on chain A - sys.L2Batcher.Stop() - - // two EOAs for a sample transfer tx used later in a conflicting block - alice := sys.FunderL2.NewFundedEOA(eth.OneHundredthEther) - bob := sys.FunderL2.NewFundedEOA(eth.OneHundredthEther) - - user := sys.FunderL2.NewFundedEOA(eth.OneEther) - contract, deployBlock := utils.DeploySimpleStorage(t, user) - t.Logf("SimpleStorage deployed at %s block=%d", contract.Address().Hex(), bigs.Uint64Strict(deployBlock.BlockNumber)) - - time.Sleep(12 * time.Second) - divergenceHead := sys.L2Chain.WaitForBlock() - // build up some blocks that will be reorged away - - type caseEntry struct { - Block uint64 - addr common.Address - slots []common.Hash - } - var cases []caseEntry - - // deploy another contract in the reorged blocks - { - rContract, rDeployBlock := utils.DeploySimpleStorage(t, user) - t.Logf("Reorg SimpleStorage deployed at %s block=%d", rContract.Address().Hex(), bigs.Uint64Strict(rDeployBlock.BlockNumber)) - - cases = append(cases, caseEntry{ - Block: bigs.Uint64Strict(rDeployBlock.BlockNumber), - addr: rContract.Address(), - slots: []common.Hash{common.HexToHash("0x0")}, - }) - } - - for i := 0; i < 3; i++ { - tx := alice.Transfer(bob.Address(), eth.OneGWei) - receipt, err := tx.Included.Eval(ctx) - require.NoError(gt, err) - require.Equal(gt, types.ReceiptStatusSuccessful, receipt.Status) - - cases = append(cases, caseEntry{ - Block: bigs.Uint64Strict(receipt.BlockNumber), - addr: alice.Address(), - slots: []common.Hash{}, - }) - cases = append(cases, caseEntry{ - Block: bigs.Uint64Strict(receipt.BlockNumber), - addr: bob.Address(), - slots: []common.Hash{}, - }) - - // also include the contract account in the proofs to verify - val := big.NewInt(int64(i * 10)) - callRes := contract.SetValue(user, val) - - cases = append(cases, caseEntry{ - Block: bigs.Uint64Strict(callRes.BlockNumber), - addr: contract.Address(), - slots: []common.Hash{common.HexToHash("0x0")}, - }) - } - - sys.L2CLSequencer.StopSequencer() - - var divergenceBlockNumber uint64 - var originalRef eth.L2BlockRef - // prepare and sequence a conflicting block for the L2A chain - { - divergenceBlockRef := sys.L2ELSequencerNode().BlockRefByNumber(divergenceHead.Number) - - l.Info("Expect to reorg the chain on block", "number", divergenceBlockRef.Number, "head", divergenceHead, "parent", divergenceBlockRef.ParentID().Hash) - divergenceBlockNumber = divergenceBlockRef.Number - originalRef = divergenceBlockRef - - parentOfDivergenceHead := divergenceBlockRef.ParentID() - - l.Info("Sequencing a conflicting block", "divergenceBlockRef", divergenceBlockRef, "parent", parentOfDivergenceHead) - - // sequence a conflicting block with a simple transfer tx, based on the parent of the parent of the unsafe head - { - err := ia.New(ctx, seqtypes.BuildOpts{ - Parent: parentOfDivergenceHead.Hash, - L1Origin: nil, - }) - require.NoError(t, err, "Expected to be able to create a new block job for sequencing on op-test-sequencer, but got error") - - // include simple transfer tx in opened block - { - to := bob.PlanTransfer(alice.Address(), eth.OneGWei) - opt := txplan.Combine(to) - ptx := txplan.NewPlannedTx(opt) - signed_tx, err := ptx.Signed.Eval(ctx) - require.NoError(t, err, "Expected to be able to evaluate a planned transaction on op-test-sequencer, but got error") - txdata, err := signed_tx.MarshalBinary() - require.NoError(t, err, "Expected to be able to marshal a signed transaction on op-test-sequencer, but got error") - - err = ia.IncludeTx(ctx, txdata) - require.NoError(t, err, "Expected to be able to include a signed transaction on op-test-sequencer, but got error") - - cases = append(cases, caseEntry{ - Block: divergenceHead.Number, - addr: alice.Address(), - slots: []common.Hash{}, - }) - cases = append(cases, caseEntry{ - Block: divergenceHead.Number, - addr: bob.Address(), - slots: []common.Hash{}, - }) - } - - err = ia.Next(ctx) - require.NoError(t, err, "Expected to be able to call Next() after New() on op-test-sequencer, but got error") - } - } - - // start batcher on chain A - sys.L2Batcher.Start() - - // sequence a second block with op-test-sequencer (no L1 origin override) - { - l.Info("Sequencing with op-test-sequencer (no L1 origin override)") - err := ia.New(ctx, seqtypes.BuildOpts{ - Parent: sys.L2ELSequencerNode().BlockRefByLabel(eth.Unsafe).Hash, - L1Origin: nil, - }) - require.NoError(t, err, "Expected to be able to create a new block job for sequencing on op-test-sequencer, but got error") - time.Sleep(2 * time.Second) - - err = ia.Next(ctx) - require.NoError(t, err, "Expected to be able to call Next() after New() on op-test-sequencer, but got error") - time.Sleep(2 * time.Second) - } - - // continue sequencing with consensus node (op-node) - sys.L2CLSequencer.StartSequencer() - - for i := 0; i < 3; i++ { - sys.L2Chain.WaitForBlock() - } - - latestBlock := sys.L2Chain.WaitForBlock() - - // verify that the L2A validator has reorged and reached the latest block. Bounded so a - // validator that never converges fails here instead of consuming the package timeout. - reorgCtx, cancelReorg := context.WithTimeout(t.Ctx(), 5*time.Minute) - defer cancelReorg() - err := wait.For(reorgCtx, 2*time.Second, func() (bool, error) { - blockRef, err := sys.L2ELValidatorNode().Escape().EthClient().BlockRefByNumber(ctx, latestBlock.Number) - if err != nil { - // this could happen if the validator is still syncing after reorg - l.Warn("Error fetching block reference from validator", "error", err) - return false, nil - } - return blockRef.Hash == latestBlock.Hash, nil - }) - require.NoError(t, err, "Expected block hash to match latest block hash on validator") - - reorgedRef_A, err := sys.L2ELSequencerNode().Escape().EthClient().BlockRefByNumber(ctx, divergenceBlockNumber) - require.NoError(t, err, "Expected to be able to call BlockRefByNumber API, but got error") - - l.Info("Reorged chain on divergence block number (prior the reorg)", "number", divergenceBlockNumber, "head", originalRef.Hash, "parent", originalRef.ParentID().Hash) - l.Info("Reorged chain on divergence block number (after the reorg)", "number", divergenceBlockNumber, "head", reorgedRef_A.Hash, "parent", reorgedRef_A.ParentID().Hash) - require.NotEqual(t, originalRef.Hash, reorgedRef_A.Hash, "Expected to get different heads on divergence block number, but got the same hash, so no reorg happened on chain A") - require.Equal(t, originalRef.ParentID().Hash, reorgedRef_A.ParentHash, "Expected to get same parent hashes on divergence block number, but got different hashes") - - time.Sleep(10 * time.Second) - - // verify that the accounts involved in the conflicting blocks - for i, c := range cases { - l.Info("Verifying proof", "case", i, "addr", c.addr.Hex(), "block", c.Block) - sys.RethWithProofL2ELNode().VerifyProofsMatchAt(sys.L2ELSequencerNode(), c.Block, dsl.ProofAccount{Address: c.addr, StorageKeys: c.slots}) - } -}