From 269d7631f39e67f7a159d775f1a1feaa7a504b98 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Wed, 11 Nov 2026 14:32:32 +0100 Subject: [PATCH 1/6] kona-node,safedb,op-acceptance-tests: validate persistence and proof consumers Run the SafeDB acceptance cases that skipped kona-node against both consensus clients: SafeDB history across restart and resync, Cannon execution-step proofs through game resolution with time travel, and proposer and dispute-monitor consumers of super and output roots. The two SafeDB resync cases skip on op-reth, so a shard-0 CI step runs them on op-geth, pinned to Jovian at genesis, after the sharded suite. The step fails when either case skips or does not pass. Document the kona-node SafeDB RPCs' historical availability and the SafeDB RPC error contract. Co-Authored-By: Claude Fable 5.1 Co-Authored-By: Claude Opus 5.5 --- .circleci/continue/main.yml | 32 +++ .../node-operators/kona-node/rpc/rollup.mdx | 262 +++++++++++++----- .../tests/proofs/cannon_kona/setup_test.go | 4 +- .../tests/proofs/cannon_kona/step_test.go | 18 +- .../safedb_proof_consumers/consumers_test.go | 95 +++++++ .../safeheaddb_clsync/safeheaddb_test.go | 6 +- .../safeheaddb_elsync/safeheaddb_test.go | 3 +- rust/kona/crates/node/safedb/README.md | 13 + 8 files changed, 341 insertions(+), 92 deletions(-) create mode 100644 op-acceptance-tests/tests/safedb_proof_consumers/consumers_test.go diff --git a/.circleci/continue/main.yml b/.circleci/continue/main.yml index 3e3a9ebc32e..7fc31856458 100644 --- a/.circleci/continue/main.yml +++ b/.circleci/continue/main.yml @@ -2182,6 +2182,38 @@ jobs: LOG_DIR=$(ls -td -- logs/* | head -1) cat "$LOG_DIR"/failed/*.log || true when: on_fail + # The two resync cases below SkipOnOpReth, so the op-reth run above never + # exercises them. They run here with op-geth, after the sharded suite so a + # resync failure cannot skip shard 0's share of it, and the step fails on + # any skip: buildL2Genesis skips op-geth on Karst chains, which would + # otherwise turn a fork-default change into silent loss of coverage. + - run: + name: Run SafeDB resync tests with op-geth + command: | + if [ "${CIRCLE_NODE_INDEX:-0}" != "0" ]; then + exit 0 + fi + RESYNC_TESTS="TestTruncateDatabaseOnELResync TestPreserveDatabaseOnCLResync" + EVENTS_FILE=op-acceptance-tests/logs/safedb-op-geth/raw_go_events.log + mkdir -p "$(dirname "$EVENTS_FILE")" op-acceptance-tests/results + DEVSTACK_L2EL_KIND=op-geth LOG_LEVEL=info gotestsum \ + --format testname \ + --jsonfile "$EVENTS_FILE" \ + --junitfile op-acceptance-tests/results/safedb-op-geth.xml \ + -- -count=1 -p=1 -parallel=1 -timeout=10m \ + -run "^($(echo "$RESYNC_TESTS" | tr ' ' '|'))$" \ + ./op-acceptance-tests/tests/safeheaddb_elsync \ + ./op-acceptance-tests/tests/safeheaddb_clsync + if grep -q '"Action":"skip","Package":"[^"]*","Test":"' "$EVENTS_FILE"; then + echo "SafeDB resync tests were skipped under op-geth" >&2 + exit 1 + fi + for test_name in $RESYNC_TESTS; do + if ! grep -q "\"Action\":\"pass\",\"Package\":\"[^\"]*\",\"Test\":\"$test_name\"" "$EVENTS_FILE"; then + echo "SafeDB resync test $test_name did not pass under op-geth" >&2 + exit 1 + fi + done # Save the module cache for future runs - go-save-cache: namespace: op-acceptance-tests diff --git a/docs/public-docs/node-operators/kona-node/rpc/rollup.mdx b/docs/public-docs/node-operators/kona-node/rpc/rollup.mdx index 59becec189c..9c7a3b3e654 100644 --- a/docs/public-docs/node-operators/kona-node/rpc/rollup.mdx +++ b/docs/public-docs/node-operators/kona-node/rpc/rollup.mdx @@ -120,6 +120,136 @@ Returns the output root at a specific block number, including the L2 block refer } ``` +## `optimism_rollupConfig` + +Returns the rollup configuration parameters that define the rollup chain's behavior and properties. + +| Client | Method invocation | +| ------ | ---------------------------------------------- | +| RPC | `{"method": "optimism_rollupConfig", "params": []}` | + +### Returns + +`RollupConfig` - An object containing the complete rollup configuration: +- `genesis` (`ChainGenesis`): The genesis state of the rollup +- `blockTime` (`number`): The block time of the L2 in seconds +- `maxSequencerDrift` (`number`): Maximum sequencer drift in seconds +- `seqWindowSize` (`number`): The sequencer window size +- `channelTimeout` (`number`): Number of L1 blocks between channel open/close +- `graniteChannelTimeout` (`number`): Channel timeout after Granite hardfork +- `l1ChainId` (`number`): The L1 chain ID +- `l2ChainId` (`number`): The L2 chain ID +- `batchInboxAddress` (`string`): L1 address where batches are sent +- `depositContractAddress` (`string`): L1 address for deposits +- `l1SystemConfigAddress` (`string`): L1 address for system config +- `protocolVersionsAddress` (`string`): L1 address for protocol versions +- Additional configuration fields for hardforks, fees, and interoperability + +### Example + +```js +// > {"jsonrpc":"2.0","id":1,"method":"optimism_rollupConfig","params":[]} +{ + "jsonrpc": "2.0", + "id": 1, + "result": { + "genesis": { + "l1": { + "hash": "0x1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef", + "number": 18000000 + }, + "l2": { + "hash": "0xabcdef1234567890abcdef1234567890abcdef1234567890abcdef1234567890", + "number": 0, + "parentHash": "0x0000000000000000000000000000000000000000000000000000000000000000", + "timestamp": 1699000000, + "l1Origin": { + "hash": "0x1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef", + "number": 18000000 + }, + "sequenceNumber": 0 + }, + "l2Time": 1699000000, + "systemConfig": { + "batcherAddr": "0x1234567890123456789012345678901234567890", + "overhead": "0x00000000000000000000000000000000000000000000000000000000000000bc", + "scalar": "0x00000000000000000000000000000000000000000000000000000000000f4240", + "gasLimit": 30000000 + } + }, + "blockTime": 2, + "maxSequencerDrift": 600, + "seqWindowSize": 3600, + "channelTimeout": 300, + "graniteChannelTimeout": 50, + "l1ChainId": 1, + "l2ChainId": 10, + "batchInboxAddress": "0xff00000000000000000000000000000000000010", + "depositContractAddress": "0xbEb5Fc579115071764c7423A4f12eDde41f106Ed", + "l1SystemConfigAddress": "0x229047fed2591dbec1eF1118d64F7aF3dB9EB290", + "protocolVersionsAddress": "0x8062AbC286f5e7D9428a0Ccb9AbD71e50d93b935", + "superchainConfigAddress": "0x95703e0982140D16f8ebA6d158FccEde42f04a4C", + "blobs_data": 1710374400, + "interopMessageExpiryWindow": 3600, + "chainOpConfig": { + "canyon_denominator": 250, + "canyon_elasticity": 6 + } + } +} +``` + +## `optimism_safeHeadAtL1Block` + +Returns the most recent recorded L2 safe head whose derivation source L1 block is at or before the requested L1 block. Enable persistence with `--safedb.path` or `KONA_NODE_SAFEDB_PATH`. + +| Client | Method invocation | +| ------ | ------------------------------------------------------------------ | +| RPC | `{"method": "optimism_safeHeadAtL1Block", "params": [l1BlockNumber]}` | + +### Parameters + +- `l1BlockNumber` (`string`): An L1 block number encoded as a hexadecimal quantity, such as `"0x64"`. Block tags are not supported. + +### Returns + +An object containing: + +- `l1Block`: The recorded derivation source L1 block, with `hash` and `number` fields. +- `safeHead`: The corresponding L2 safe head, with `hash` and `number` fields. + +Both block numbers in the response are JSON integers. The recorded L1 block can precede the requested block because records are sparse. A request above the recorded tip returns the latest record; it does not establish that derivation has processed the requested L1 block. + +### Errors + +| Condition | Code | Message | +| --- | --- | --- | +| No record exists at or before the requested L1 block | `-32000` | `not found` | +| SafeDB is disabled | `-32000` | Contains `safe head database not enabled` | +| Invalid parameter encoding | `-32602` | Invalid parameters | + +A disabled database is distinct from an enabled database with no matching record. Database read failures also return an RPC error; callers must not interpret a failed lookup as proof of historical safety. + +### Example + +```js +// > {"jsonrpc":"2.0","id":1,"method":"optimism_safeHeadAtL1Block","params":["0x64"]} +{ + "jsonrpc": "2.0", + "id": 1, + "result": { + "l1Block": { + "hash": "0x1111111111111111111111111111111111111111111111111111111111111111", + "number": 98 + }, + "safeHead": { + "hash": "0x2222222222222222222222222222222222222222222222222222222222222222", + "number": 42 + } + } +} +``` + ## `optimism_syncStatus` Returns the current synchronization status of the rollup node, including information about L1 and L2 block states. @@ -217,104 +347,92 @@ Returns the current synchronization status of the rollup node, including informa } ``` -## `optimism_rollupConfig` +## `optimism_version` -Returns the rollup configuration parameters that define the rollup chain's behavior and properties. +Returns the software version of the Kona rollup node. -| Client | Method invocation | -| ------ | ---------------------------------------------- | -| RPC | `{"method": "optimism_rollupConfig", "params": []}` | +| Client | Method invocation | +| ------ | ---------------------------------------- | +| RPC | `{"method": "optimism_version", "params": []}` | ### Returns -`RollupConfig` - An object containing the complete rollup configuration: -- `genesis` (`ChainGenesis`): The genesis state of the rollup -- `blockTime` (`number`): The block time of the L2 in seconds -- `maxSequencerDrift` (`number`): Maximum sequencer drift in seconds -- `seqWindowSize` (`number`): The sequencer window size -- `channelTimeout` (`number`): Number of L1 blocks between channel open/close -- `graniteChannelTimeout` (`number`): Channel timeout after Granite hardfork -- `l1ChainId` (`number`): The L1 chain ID -- `l2ChainId` (`number`): The L2 chain ID -- `batchInboxAddress` (`string`): L1 address where batches are sent -- `depositContractAddress` (`string`): L1 address for deposits -- `l1SystemConfigAddress` (`string`): L1 address for system config -- `protocolVersionsAddress` (`string`): L1 address for protocol versions -- Additional configuration fields for hardforks, fees, and interoperability +`string` - The version string of the Kona software (e.g., "0.1.0") ### Example ```js -// > {"jsonrpc":"2.0","id":1,"method":"optimism_rollupConfig","params":[]} +// > {"jsonrpc":"2.0","id":1,"method":"optimism_version","params":[]} { "jsonrpc": "2.0", "id": 1, - "result": { - "genesis": { - "l1": { - "hash": "0x1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef", - "number": 18000000 - }, - "l2": { - "hash": "0xabcdef1234567890abcdef1234567890abcdef1234567890abcdef1234567890", - "number": 0, - "parentHash": "0x0000000000000000000000000000000000000000000000000000000000000000", - "timestamp": 1699000000, - "l1Origin": { - "hash": "0x1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef", - "number": 18000000 - }, - "sequenceNumber": 0 - }, - "l2Time": 1699000000, - "systemConfig": { - "batcherAddr": "0x1234567890123456789012345678901234567890", - "overhead": "0x00000000000000000000000000000000000000000000000000000000000000bc", - "scalar": "0x00000000000000000000000000000000000000000000000000000000000f4240", - "gasLimit": 30000000 - } - }, - "blockTime": 2, - "maxSequencerDrift": 600, - "seqWindowSize": 3600, - "channelTimeout": 300, - "graniteChannelTimeout": 50, - "l1ChainId": 1, - "l2ChainId": 10, - "batchInboxAddress": "0xff00000000000000000000000000000000000010", - "depositContractAddress": "0xbEb5Fc579115071764c7423A4f12eDde41f106Ed", - "l1SystemConfigAddress": "0x229047fed2591dbec1eF1118d64F7aF3dB9EB290", - "protocolVersionsAddress": "0x8062AbC286f5e7D9428a0Ccb9AbD71e50d93b935", - "superchainConfigAddress": "0x95703e0982140D16f8ebA6d158FccEde42f04a4C", - "blobs_data": 1710374400, - "interopMessageExpiryWindow": 3600, - "chainOpConfig": { - "canyon_denominator": 250, - "canyon_elasticity": 6 - } - } + "result": "0.1.0" } ``` -## `optimism_version` +## `superroot_atTimestamp` -Returns the software version of the Kona rollup node. +Returns a pre-interop, single-chain super root and its L1 derivation requirements. This method uses the `superroot` namespace rather than the `optimism` namespace. SafeDB must be enabled and contain the relevant history for a non-genesis block that is locally safe. -| Client | Method invocation | -| ------ | ---------------------------------------- | -| RPC | `{"method": "optimism_version", "params": []}` | +| Client | Method invocation | +| ------ | ---------------------------------------------------------- | +| RPC | `{"method": "superroot_atTimestamp", "params": [timestamp]}` | + +### Parameters + +- `timestamp` (`string`): An L2 timestamp encoded as a hexadecimal quantity. The rollup configuration maps the timestamp to an L2 block number. ### Returns -`string` - The version string of the Kona software (e.g., "0.1.0") +The response reports derivation progress even when the requested block is not yet locally safe: + +- `chain_ids`: The single chain ID, encoded as a decimal string. +- `current_l1`: The derivation cursor, with `hash` and `number` fields. Every L1 block strictly below this cursor has been fully processed. Consumers checking an L1 scope `X` must wait for `current_l1.number > X`. +- `finalized_timestamp`: The finalized L2 head timestamp, as a JSON integer. +- `local_safe_timestamp`: The locally safe L2 head timestamp, as a JSON integer. +- `optimistic_at_timestamp`: A map keyed by decimal chain ID. Once the requested block is locally safe, its entry contains `output`, `output_root`, and the SafeDB-backed `required_l1` block. +- `safe_timestamp`: The safe L2 head timestamp, as a JSON integer. + +When the block is locally safe, `data` contains: + +- `super`: The version 1 preimage, containing `chains` and a hexadecimal `timestamp`. Each chain entry contains `ChainID` as a decimal string and `Output` as an output-root hash. +- `super_root`: The super-root hash. +- `verified_required_l1`: The SafeDB-backed L1 block required to derive the output. For the single-chain response, it equals that chain's `required_l1`. + +The `output` preimage contains `blockHash`, `messagePasserStorageRoot`, and `stateRoot`. L1 block references contain a hash and a JSON integer block number. Genesis has a zero L1 requirement. + +When the requested block is above local-safe, `optimistic_at_timestamp` is empty and `data` is omitted. Consumers must wait for a response containing the requested output rather than treating a progress-only response as a verified super root. + +### Errors + +- A timestamp before L2 genesis returns `-32602`. +- History not yet recorded returns `-32000` with a message containing `l1 at safe head not found`. +- History before the available range returns `-32000` with a message containing `l1 at safe head history unavailable`. Waiting for more derivation does not restore that missing historical range. +- A disabled database returns `-32000` with a message containing `safe head database not enabled` when a non-genesis historical lookup is required. +- If history changes repeatedly while the output is read, the request returns `-32000` with `safe head history changed during query; retry`. +- Database and engine read failures return RPC errors rather than an unverified root. + +SafeDB lookup messages for this method include the operation prefix `l1AtSafeHead@`. They differ from the bare `not found` message returned by `optimism_safeHeadAtL1Block`. ### Example +A progress-only response, returned while the requested block is not yet locally safe: + ```js -// > {"jsonrpc":"2.0","id":1,"method":"optimism_version","params":[]} +// > {"jsonrpc":"2.0","id":1,"method":"superroot_atTimestamp","params":["0x6553f100"]} { "jsonrpc": "2.0", "id": 1, - "result": "0.1.0" + "result": { + "chain_ids": ["901"], + "current_l1": { + "hash": "0x1111111111111111111111111111111111111111111111111111111111111111", + "number": 100 + }, + "finalized_timestamp": 1699999900, + "local_safe_timestamp": 1699999998, + "optimistic_at_timestamp": {}, + "safe_timestamp": 1699999998 + } } ``` diff --git a/op-acceptance-tests/tests/proofs/cannon_kona/setup_test.go b/op-acceptance-tests/tests/proofs/cannon_kona/setup_test.go index b84663e040f..d049d5aecce 100644 --- a/op-acceptance-tests/tests/proofs/cannon_kona/setup_test.go +++ b/op-acceptance-tests/tests/proofs/cannon_kona/setup_test.go @@ -11,9 +11,7 @@ func cannonOpts() []presets.Option { return []presets.Option{ presets.WithGameTypeAdded(gameTypes.CannonKonaGameType), presets.WithDeployerOptions(sysgo.WithJovianAtGenesis), - presets.WithGlobalL2CLOption(sysgo.L2CLOptionFn(func(p devtest.T, _ sysgo.ComponentTarget, cfg *sysgo.L2CLConfig) { - cfg.SafeDBPath = p.TempDir() - })), + presets.WithTimeTravelEnabled(), } } diff --git a/op-acceptance-tests/tests/proofs/cannon_kona/step_test.go b/op-acceptance-tests/tests/proofs/cannon_kona/step_test.go index bf49e843ac3..f3a6ed5bdc3 100644 --- a/op-acceptance-tests/tests/proofs/cannon_kona/step_test.go +++ b/op-acceptance-tests/tests/proofs/cannon_kona/step_test.go @@ -2,10 +2,11 @@ package cannon_kona import ( "testing" + "time" + gameTypes "github.com/ethereum-optimism/optimism/op-challenger/game/types" "github.com/ethereum-optimism/optimism/op-devstack/devtest" "github.com/ethereum-optimism/optimism/op-devstack/dsl/proofs" - "github.com/ethereum-optimism/optimism/op-devstack/sysgo" "github.com/ethereum-optimism/optimism/op-service/eth" safety "github.com/ethereum-optimism/optimism/op-service/eth/safety" @@ -13,19 +14,6 @@ import ( func TestExecuteStep_CannonKona(gt *testing.T) { t := devtest.ParallelT(gt) - // Example error with kona-node: - // - // assertions.go:387: ERROR[03-31|10:33:38.442] - // assertions.go:387: Error Trace: /optimism/op-devstack/dsl/proofs/game_helper.go:263 - // assertions.go:387: /optimism/op-devstack/dsl/proofs/game_helper.go:196 - // assertions.go:387: /optimism/op-devstack/dsl/proofs/fault_dispute_game.go:124 - // assertions.go:387: /optimism/op-acceptance-tests/tests/proofs/cannon/step_test.go:40 - // assertions.go:387: /.local/share/mise/installs/go/1.24.13/src/runtime/asm_arm64.s:1223 - // assertions.go:387: Error: Received unexpected error: - // assertions.go:387: failed to get safe head at L1 block 0xe926b953777546729b2607011590354a6a42dc35e36b7ea68c9dd4b470261541:6: not found - // assertions.go:387: Test: TestExecuteStep_CannonKona - // assertions.go:387: Messages: Failed to get honest root claim - sysgo.SkipOnKonaNode(t, "not supported") sys := newSystem(t) l1User := sys.FunderL1.NewFundedEOA(eth.ThousandEther) @@ -39,4 +27,6 @@ func TestExecuteStep_CannonKona(gt *testing.T) { claim = game.DisputeToStep(l1User, claim, 1000) // Skip down to max depth game.LogGameData() claim.WaitForCountered() + sys.AdvanceTime(game.MaxClockDuration() + time.Second) + game.WaitForGameStatus(gameTypes.GameStatusDefenderWon) } diff --git a/op-acceptance-tests/tests/safedb_proof_consumers/consumers_test.go b/op-acceptance-tests/tests/safedb_proof_consumers/consumers_test.go new file mode 100644 index 00000000000..c8607d8ced6 --- /dev/null +++ b/op-acceptance-tests/tests/safedb_proof_consumers/consumers_test.go @@ -0,0 +1,95 @@ +package safedb_proof_consumers + +import ( + "testing" + + gameTypes "github.com/ethereum-optimism/optimism/op-challenger/game/types" + "github.com/ethereum-optimism/optimism/op-core/devfeatures" + "github.com/ethereum-optimism/optimism/op-devstack/devtest" + "github.com/ethereum-optimism/optimism/op-devstack/dsl" + "github.com/ethereum-optimism/optimism/op-devstack/dsl/disputemon" + "github.com/ethereum-optimism/optimism/op-devstack/dsl/proofs" + "github.com/ethereum-optimism/optimism/op-devstack/presets" + "github.com/ethereum-optimism/optimism/op-devstack/sysgo" + "github.com/ethereum-optimism/optimism/op-service/eth" + "github.com/ethereum/go-ethereum/common" + + safety "github.com/ethereum-optimism/optimism/op-service/eth/safety" +) + +func TestProposerPublishesSuperRoot(gt *testing.T) { + t := devtest.ParallelT(gt) + system := presets.NewMinimal(t, + presets.WithDeployerOptions( + sysgo.WithJovianAtGenesis, + sysgo.WithUseInterop(), + sysgo.WithDevFeatureEnabled(devfeatures.SuperRootGamesMigrationFlag), + ), + ) + + game := system.DisputeGameFactory().WaitForGame() + game.VerifyGameType(gameTypes.SuperPermissionedGameType) + dsl.NewOpNodeSuperRoots(system.L2CL).AssertSuperRootAtTimestamp(game.L2SequenceNumber(), game.RootClaimValue()) +} + +func TestDisputeMonitorClassifiesOutputRoots(gt *testing.T) { + for _, testCase := range []struct { + name string + + agreedRoots int + disagreedRoots int + gameOptions []proofs.GameOpt + }{ + {name: "valid", agreedRoots: 1}, + {name: "invalid", disagreedRoots: 1, gameOptions: []proofs.GameOpt{proofs.WithRootClaim(common.HexToHash("0xdead"))}}, + } { + gt.Run(testCase.name, func(gt *testing.T) { + t := devtest.ParallelT(gt) + system := newOutputRootSystem(t) + proposer := system.FunderL1.NewFundedEOA(eth.OneEther) + gameOptions := append([]proofs.GameOpt{proofs.WithL2SequenceNumber(3)}, testCase.gameOptions...) + system.DisputeGameFactory().StartCannonKonaGame(proposer, gameOptions...) + + monitor := system.StartDisputeMon() + monitor.VerifyState( + disputemon.GameCount(gameTypes.CannonKonaGameType, 1), + disputemon.FailedGames(0), + disputemon.AgreedRoots(testCase.agreedRoots), + disputemon.DisagreedRoots(testCase.disagreedRoots), + ) + }) + } +} + +func TestDisputeMonitorRejectsOutputRootNotSafeAtGameL1Head(gt *testing.T) { + t := devtest.ParallelT(gt) + system := newOutputRootSystem(t) + proposer := system.FunderL1.NewFundedEOA(eth.OneEther) + system.L2CL.Reached(safety.LocalSafe, 3, 30) + system.L2Batcher.Stop() + system.L2CL.Advanced(safety.LocalUnsafe, 3, 30) + blockNumber := system.L2CL.HeadBlockRef(safety.LocalUnsafe).Number + + game := system.DisputeGameFactory().StartCannonKonaGame(proposer, + proofs.WithL2SequenceNumber(blockNumber), + proofs.WithFutureProposal(), + ) + system.L2Batcher.Start() + system.L2CL.Reached(safety.LocalSafe, blockNumber, 30) + + monitor := system.StartDisputeMon() + monitor.VerifyState( + disputemon.GameCount(gameTypes.CannonKonaGameType, 1), + disputemon.FailedGames(0), + disputemon.AgreedRoots(0), + disputemon.DisagreedRoots(1), + disputemon.InvalidProposalObserved(game), + ) +} + +func newOutputRootSystem(t devtest.T) *presets.Minimal { + return presets.NewMinimalNoFaultProofs(t, + presets.WithDeployerOptions(sysgo.WithJovianAtGenesis), + presets.WithGameTypeAdded(gameTypes.CannonKonaGameType), + ) +} diff --git a/op-acceptance-tests/tests/safeheaddb_clsync/safeheaddb_test.go b/op-acceptance-tests/tests/safeheaddb_clsync/safeheaddb_test.go index 66f034d5e60..4a1598254ef 100644 --- a/op-acceptance-tests/tests/safeheaddb_clsync/safeheaddb_test.go +++ b/op-acceptance-tests/tests/safeheaddb_clsync/safeheaddb_test.go @@ -27,6 +27,7 @@ func TestPreserveDatabaseOnCLResync(gt *testing.T) { // assertions.go:387: sysgo.SkipOnOpReth(t, "not supported (peering issue)") sys := presets.NewSingleChainMultiNode(t, + presets.WithDeployerOptions(sysgo.WithJovianAtGenesis), presets.WithGlobalL2CLOption(sysgo.L2CLOptionFn(func(p devtest.T, _ sysgo.ComponentTarget, cfg *sysgo.L2CLConfig) { cfg.SequencerSyncMode = sync.CLSync cfg.VerifierSyncMode = sync.CLSync @@ -43,8 +44,8 @@ func TestPreserveDatabaseOnCLResync(gt *testing.T) { sys.L2CLB.VerifySafeHeadDatabaseMatches(sys.L2CL) // Stop the verifier node. Since the sysgo EL uses in-memory storage this also wipes its database. - // With the EL reset to genesis, when the CL restarts it will use EL sync to resync the chain rather than - // deriving it from L1. + // With the EL reset to genesis, the restarted CL stays in CL sync mode and re-derives the chain from L1, + // so unlike the EL sync case it must not truncate its safe head database. sys.L2ELB.Stop() sys.L2CLB.Stop() @@ -52,6 +53,7 @@ func TestPreserveDatabaseOnCLResync(gt *testing.T) { sys.L2ELB.Start() sys.L2CLB.Start() + sys.L2ELB.WaitForOnline() sys.L2ELB.PeerWith(sys.L2EL) sys.L2CLB.InSync(sys.L2CL, safety.LocalSafe, 30) diff --git a/op-acceptance-tests/tests/safeheaddb_elsync/safeheaddb_test.go b/op-acceptance-tests/tests/safeheaddb_elsync/safeheaddb_test.go index c7929eee690..9fb062347d1 100644 --- a/op-acceptance-tests/tests/safeheaddb_elsync/safeheaddb_test.go +++ b/op-acceptance-tests/tests/safeheaddb_elsync/safeheaddb_test.go @@ -17,6 +17,7 @@ func newSingleChainMultiNodeELSync(t devtest.T) *presets.SingleChainMultiNode { // uses a 4x CrossSafe budget (see singlechain_from_runtime.go), which is // what this test needs. return presets.NewSingleChainMultiNode(t, + presets.WithDeployerOptions(sysgo.WithJovianAtGenesis), presets.WithGlobalL2CLOption(sysgo.L2CLOptionFn(func(p devtest.T, _ sysgo.ComponentTarget, cfg *sysgo.L2CLConfig) { cfg.VerifierSyncMode = sync.ELSync cfg.SafeDBPath = p.TempDir() @@ -85,7 +86,7 @@ func TestNotTruncateDatabaseOnRestartWithExistingDatabase(gt *testing.T) { preRestartSafeBlock := sys.L2CLB.SafeHeadDBFloor() sys.L2CLB.VerifySafeHeadDatabaseMatches(sys.L2CL, dsl.WithMinRequiredL2Block(preRestartSafeBlock)) - // Restart the verifier op-node, but not the EL so the existing chain data is not deleted. + // Keep the EL's chain data while restarting the verifier CL. sys.L2CLB.Stop() sys.L2CL.Advanced(safety.LocalSafe, 3, 30) diff --git a/rust/kona/crates/node/safedb/README.md b/rust/kona/crates/node/safedb/README.md index 3639948bd13..2e9b27ac88f 100644 --- a/rust/kona/crates/node/safedb/README.md +++ b/rust/kona/crates/node/safedb/README.md @@ -19,6 +19,19 @@ missed. See [`SafeDb::safe_head_updated`]. The layout carries no compatibility guarantee with any other implementation and is free to change. +## RPC error contract + +The node's `optimism_safeHeadAtL1Block` and pre-interop `superroot_atTimestamp` handlers +translate `SafeDB` failures into JSON-RPC errors with code `-32000`. A missing L1 lookup +record has the bare message `not found`. The reverse lookup used by `superroot_atTimestamp` +distinguishes `l1 at safe head not found` (not recorded yet) from +`l1 at safe head history unavailable` (before the available history), with an operation +prefix of `l1AtSafeHead@`. A disabled database has a message containing +`safe head database not enabled`, prefixed with the attempted operation. These conditions +remain distinct so clients can identify unavailable historical coverage without treating a +disabled database as an empty one. Other storage failures return an error with operation +context rather than a successful historical answer. + The crate exposes: - [`SafeDb`] — the database trait. From 5707a2205cae88e33f51944162d8fb5d0f011f67 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Wed, 11 Nov 2026 16:55:29 +0100 Subject: [PATCH 2/6] ci,op-acceptance-tests,op-devstack,kona-node: tighten SafeDB consumer checks The op-geth resync step accepts a test's "--- PASS"/"--- SKIP" output line when test2json drops the result event, and sets no_output_timeout above go's -timeout so a hang ends in a goroutine dump. The proposer test checks the super root's output root against the node's output at that block, and that verified_required_l1 is the first SafeDB record with that block safe. The dispute-monitor agreement test checks the SafeDB record at the game's L1 head directly, since dispute-mon also agrees when SafeDB is unreadable. The invalid-root case, which never reads SafeDB, is removed. The DSL gains SuperRootQuerier.AwaitSuperRootAtTimestamp and L1ELNode.BlockRefByHash for these checks. Only the ELSync resync test pins Jovian at genesis, so the restart test on the same preset runs on the default fork. The SafeDB RPC error contract is documented only in the kona-node RPC docs. Co-Authored-By: Claude Opus 5.5 --- .circleci/continue/main.yml | 8 +- .../safedb_proof_consumers/consumers_test.go | 80 ++++++++++++------- .../safeheaddb_elsync/safeheaddb_test.go | 11 +-- op-devstack/dsl/l1_el.go | 9 +++ op-devstack/dsl/super_root_source.go | 10 ++- rust/kona/crates/node/safedb/README.md | 13 --- 6 files changed, 82 insertions(+), 49 deletions(-) diff --git a/.circleci/continue/main.yml b/.circleci/continue/main.yml index 7fc31856458..8edf9a04327 100644 --- a/.circleci/continue/main.yml +++ b/.circleci/continue/main.yml @@ -2189,6 +2189,8 @@ jobs: # otherwise turn a fork-default change into silent loss of coverage. - run: name: Run SafeDB resync tests with op-geth + # Above go's -timeout, so a hang ends in go's goroutine dump, not a silent kill. + no_output_timeout: 15m command: | if [ "${CIRCLE_NODE_INDEX:-0}" != "0" ]; then exit 0 @@ -2204,12 +2206,14 @@ jobs: -run "^($(echo "$RESYNC_TESTS" | tr ' ' '|'))$" \ ./op-acceptance-tests/tests/safeheaddb_elsync \ ./op-acceptance-tests/tests/safeheaddb_clsync - if grep -q '"Action":"skip","Package":"[^"]*","Test":"' "$EVENTS_FILE"; then + # test2json can drop a result event and leave a \u0016 marker on the result output line, + # so the "--- PASS"/"--- SKIP" output lines count as well as the events. + if grep -qE '"Action":"skip","Package":"[^"]*","Test":"|"Output":"(\\u0016)?--- SKIP: ' "$EVENTS_FILE"; then echo "SafeDB resync tests were skipped under op-geth" >&2 exit 1 fi for test_name in $RESYNC_TESTS; do - if ! grep -q "\"Action\":\"pass\",\"Package\":\"[^\"]*\",\"Test\":\"$test_name\"" "$EVENTS_FILE"; then + if ! grep -qE "\"Action\":\"pass\",\"Package\":\"[^\"]*\",\"Test\":\"$test_name\"|\"Output\":\"(\\\\u0016)?--- PASS: $test_name \(" "$EVENTS_FILE"; then echo "SafeDB resync test $test_name did not pass under op-geth" >&2 exit 1 fi diff --git a/op-acceptance-tests/tests/safedb_proof_consumers/consumers_test.go b/op-acceptance-tests/tests/safedb_proof_consumers/consumers_test.go index c8607d8ced6..8986ac402a3 100644 --- a/op-acceptance-tests/tests/safedb_proof_consumers/consumers_test.go +++ b/op-acceptance-tests/tests/safedb_proof_consumers/consumers_test.go @@ -1,6 +1,7 @@ package safedb_proof_consumers import ( + "errors" "testing" gameTypes "github.com/ethereum-optimism/optimism/op-challenger/game/types" @@ -11,6 +12,7 @@ import ( "github.com/ethereum-optimism/optimism/op-devstack/dsl/proofs" "github.com/ethereum-optimism/optimism/op-devstack/presets" "github.com/ethereum-optimism/optimism/op-devstack/sysgo" + "github.com/ethereum-optimism/optimism/op-node/node/safedb" "github.com/ethereum-optimism/optimism/op-service/eth" "github.com/ethereum/go-ethereum/common" @@ -19,6 +21,7 @@ import ( func TestProposerPublishesSuperRoot(gt *testing.T) { t := devtest.ParallelT(gt) + require := t.Require() system := presets.NewMinimal(t, presets.WithDeployerOptions( sysgo.WithJovianAtGenesis, @@ -29,36 +32,38 @@ func TestProposerPublishesSuperRoot(gt *testing.T) { game := system.DisputeGameFactory().WaitForGame() game.VerifyGameType(gameTypes.SuperPermissionedGameType) - dsl.NewOpNodeSuperRoots(system.L2CL).AssertSuperRootAtTimestamp(game.L2SequenceNumber(), game.RootClaimValue()) + response := dsl.NewOpNodeSuperRoots(system.L2CL).AwaitSuperRootAtTimestamp(game.L2SequenceNumber()) + require.Equal(game.RootClaimValue(), common.Hash(eth.SuperRoot(response.Data.Super))) + + chainOutput, ok := response.OptimisticAtTimestamp[system.L2Chain.ChainID()] + require.True(ok, "super root response must include the L2 chain") + block := system.L2EL.BlockRefByHash(chainOutput.Output.BlockHash) + // The genesis block's required L1 is fixed at 0 without a SafeDB read. + require.NotZero(block.Number, "the game must commit past L2 genesis") + require.Equal(system.L2CL.OutputAtBlock(block.Number).OutputRoot, chainOutput.OutputRoot) + requireFirstL1WithSafeHead(t, system.L2CL, block.Number, response.Data.VerifiedRequiredL1) } -func TestDisputeMonitorClassifiesOutputRoots(gt *testing.T) { - for _, testCase := range []struct { - name string - - agreedRoots int - disagreedRoots int - gameOptions []proofs.GameOpt - }{ - {name: "valid", agreedRoots: 1}, - {name: "invalid", disagreedRoots: 1, gameOptions: []proofs.GameOpt{proofs.WithRootClaim(common.HexToHash("0xdead"))}}, - } { - gt.Run(testCase.name, func(gt *testing.T) { - t := devtest.ParallelT(gt) - system := newOutputRootSystem(t) - proposer := system.FunderL1.NewFundedEOA(eth.OneEther) - gameOptions := append([]proofs.GameOpt{proofs.WithL2SequenceNumber(3)}, testCase.gameOptions...) - system.DisputeGameFactory().StartCannonKonaGame(proposer, gameOptions...) - - monitor := system.StartDisputeMon() - monitor.VerifyState( - disputemon.GameCount(gameTypes.CannonKonaGameType, 1), - disputemon.FailedGames(0), - disputemon.AgreedRoots(testCase.agreedRoots), - disputemon.DisagreedRoots(testCase.disagreedRoots), - ) - }) - } +func TestDisputeMonitorAgreesWithSafeOutputRoot(gt *testing.T) { + t := devtest.ParallelT(gt) + require := t.Require() + system := newOutputRootSystem(t) + proposer := system.FunderL1.NewFundedEOA(eth.OneEther) + game := system.DisputeGameFactory().StartCannonKonaGame(proposer, proofs.WithL2SequenceNumber(3)) + + monitor := system.StartDisputeMon() + monitor.VerifyState( + disputemon.GameCount(gameTypes.CannonKonaGameType, 1), + disputemon.FailedGames(0), + disputemon.AgreedRoots(1), + disputemon.DisagreedRoots(0), + ) + + // dispute-mon also agrees when it cannot read the SafeDB, so check the record it relies on. + l1Head := system.L1EL.BlockRefByHash(game.L1Head()) + safeHead, err := system.L2CL.SafeHeadAtL1Block(l1Head.Number) + require.NoError(err, "SafeDB must answer at the game's L1 head %d", l1Head.Number) + require.GreaterOrEqual(safeHead.SafeHead.Number, game.L2SequenceNumber()) } func TestDisputeMonitorRejectsOutputRootNotSafeAtGameL1Head(gt *testing.T) { @@ -93,3 +98,22 @@ func newOutputRootSystem(t devtest.T) *presets.Minimal { presets.WithGameTypeAdded(gameTypes.CannonKonaGameType), ) } + +// requireFirstL1WithSafeHead asserts that requiredL1 is the first L1 block at which the +// SafeDB records l2Block as safe. +func requireFirstL1WithSafeHead(t devtest.T, cl *dsl.L2CLNode, l2Block uint64, requiredL1 eth.BlockID) { + require := t.Require() + require.NotZero(requiredL1.Number, "required L1 for L2 block %d", l2Block) + + atRequired, err := cl.SafeHeadAtL1Block(requiredL1.Number) + require.NoError(err, "SafeDB must answer at the required L1 block %d", requiredL1.Number) + require.Equal(requiredL1, atRequired.L1Block, "required L1 must be a SafeDB record") + require.GreaterOrEqual(atRequired.SafeHead.Number, l2Block) + + beforeRequired, err := cl.SafeHeadAtL1Block(requiredL1.Number - 1) + if errors.Is(err, safedb.ErrNotFound) { + return + } + require.NoError(err, "SafeDB must answer below the required L1 block %d", requiredL1.Number) + require.Less(beforeRequired.SafeHead.Number, l2Block, "L2 block %d was already safe before the required L1 block", l2Block) +} diff --git a/op-acceptance-tests/tests/safeheaddb_elsync/safeheaddb_test.go b/op-acceptance-tests/tests/safeheaddb_elsync/safeheaddb_test.go index 9fb062347d1..bde22f221ca 100644 --- a/op-acceptance-tests/tests/safeheaddb_elsync/safeheaddb_test.go +++ b/op-acceptance-tests/tests/safeheaddb_elsync/safeheaddb_test.go @@ -12,17 +12,17 @@ import ( safety "github.com/ethereum-optimism/optimism/op-service/eth/safety" ) -func newSingleChainMultiNodeELSync(t devtest.T) *presets.SingleChainMultiNode { +func newSingleChainMultiNodeELSync(t devtest.T, extraOptions ...presets.Option) *presets.SingleChainMultiNode { // The default preset sync check auto-detects ELSync mode on node B and // uses a 4x CrossSafe budget (see singlechain_from_runtime.go), which is // what this test needs. - return presets.NewSingleChainMultiNode(t, - presets.WithDeployerOptions(sysgo.WithJovianAtGenesis), + options := append([]presets.Option{ presets.WithGlobalL2CLOption(sysgo.L2CLOptionFn(func(p devtest.T, _ sysgo.ComponentTarget, cfg *sysgo.L2CLConfig) { cfg.VerifierSyncMode = sync.ELSync cfg.SafeDBPath = p.TempDir() })), - ) + }, extraOptions...) + return presets.NewSingleChainMultiNode(t, options...) } func TestTruncateDatabaseOnELResync(gt *testing.T) { @@ -39,7 +39,8 @@ func TestTruncateDatabaseOnELResync(gt *testing.T) { // assertions.go:387: Messages: The peer was not connected // assertions.go:387: sysgo.SkipOnOpReth(t, "not supported (peering issue)") - sys := newSingleChainMultiNodeELSync(t) + // op-geth cannot run Karst chains, and this test only runs on op-geth. + sys := newSingleChainMultiNodeELSync(t, presets.WithDeployerOptions(sysgo.WithJovianAtGenesis)) dsl.CheckAll(t, sys.L2CL.AdvancedFn(safety.LocalSafe, 1, 30), diff --git a/op-devstack/dsl/l1_el.go b/op-devstack/dsl/l1_el.go index 0210e6a1fb7..071418695c4 100644 --- a/op-devstack/dsl/l1_el.go +++ b/op-devstack/dsl/l1_el.go @@ -11,6 +11,7 @@ import ( "github.com/ethereum-optimism/optimism/op-service/clock" "github.com/ethereum-optimism/optimism/op-service/eth" "github.com/ethereum-optimism/optimism/op-service/retry" + "github.com/ethereum/go-ethereum/common" ) // L1ELNode wraps a stack.L1ELNode interface for DSL operations @@ -90,6 +91,14 @@ func (el *L1ELNode) BlockRefByNumber(number uint64) eth.L1BlockRef { return block } +func (el *L1ELNode) BlockRefByHash(hash common.Hash) eth.L1BlockRef { + ctx, cancel := context.WithTimeout(el.ctx, DefaultTimeout) + defer cancel() + block, err := el.inner.EthClient().BlockRefByHash(ctx, hash) + el.require.NoError(err, "block not found using block hash %s", hash) + return block +} + // ReorgTriggeredFn returns a lambda that checks that a L1 reorg occurred on the expected block // Composable with other lambdas to wait in parallel func (el *L1ELNode) ReorgTriggeredFn(target eth.L1BlockRef, attempts int) CheckFunc { diff --git a/op-devstack/dsl/super_root_source.go b/op-devstack/dsl/super_root_source.go index a00936848a0..074a4015af5 100644 --- a/op-devstack/dsl/super_root_source.go +++ b/op-devstack/dsl/super_root_source.go @@ -85,16 +85,24 @@ func (s *SuperRootQuerier) AssertSuperRootAtTimestamp(l2SequenceNumber uint64, r // AwaitValidatedTimestamp waits for the super-root at the given timestamp to be available. func (s *SuperRootQuerier) AwaitValidatedTimestamp(timestamp uint64) { + s.AwaitSuperRootAtTimestamp(timestamp) +} + +// AwaitSuperRootAtTimestamp waits for the super-root at the given timestamp to be available and returns it. +func (s *SuperRootQuerier) AwaitSuperRootAtTimestamp(timestamp uint64) eth.SuperRootAtTimestampResponse { ctx, cancel := context.WithTimeout(s.ctx, 5*DefaultTimeout) defer cancel() + var resp eth.SuperRootAtTimestampResponse err := wait.For(ctx, 1*time.Second, func() (bool, error) { - resp, err := s.api.SuperRootAtTimestamp(ctx, timestamp) + var err error + resp, err = s.api.SuperRootAtTimestamp(ctx, timestamp) if err != nil { return false, nil // Ignore transient errors. } return resp.Data != nil, nil }) s.require.NoError(err, "super-root at timestamp %d was not validated in time", timestamp) + return resp } // AwaitOptimisticBlockAtTimestamp waits for one chain's optimistic output to become available. diff --git a/rust/kona/crates/node/safedb/README.md b/rust/kona/crates/node/safedb/README.md index 2e9b27ac88f..3639948bd13 100644 --- a/rust/kona/crates/node/safedb/README.md +++ b/rust/kona/crates/node/safedb/README.md @@ -19,19 +19,6 @@ missed. See [`SafeDb::safe_head_updated`]. The layout carries no compatibility guarantee with any other implementation and is free to change. -## RPC error contract - -The node's `optimism_safeHeadAtL1Block` and pre-interop `superroot_atTimestamp` handlers -translate `SafeDB` failures into JSON-RPC errors with code `-32000`. A missing L1 lookup -record has the bare message `not found`. The reverse lookup used by `superroot_atTimestamp` -distinguishes `l1 at safe head not found` (not recorded yet) from -`l1 at safe head history unavailable` (before the available history), with an operation -prefix of `l1AtSafeHead@`. A disabled database has a message containing -`safe head database not enabled`, prefixed with the attempted operation. These conditions -remain distinct so clients can identify unavailable historical coverage without treating a -disabled database as an empty one. Other storage failures return an error with operation -context rather than a successful historical answer. - The crate exposes: - [`SafeDb`] — the database trait. From 276f4c20ccb5d85c8ee82d3889ebd199c70b3668 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Wed, 11 Nov 2026 17:24:17 +0100 Subject: [PATCH 3/6] op-acceptance-tests,op-devstack: wait for a super-root game past L2 genesis The first super-root game can commit to the L2 genesis block, whose required L1 is fixed at 0 without a SafeDB read. The proposer test waits for the first game at or after the second block's timestamp, using the new DisputeGameFactory.WaitForGameFrom. Co-Authored-By: Claude Opus 5.5 --- .../safedb_proof_consumers/consumers_test.go | 6 ++-- .../dsl/proofs/dispute_game_factory.go | 29 +++++++++++++++++++ 2 files changed, 32 insertions(+), 3 deletions(-) diff --git a/op-acceptance-tests/tests/safedb_proof_consumers/consumers_test.go b/op-acceptance-tests/tests/safedb_proof_consumers/consumers_test.go index 8986ac402a3..c2a758353cb 100644 --- a/op-acceptance-tests/tests/safedb_proof_consumers/consumers_test.go +++ b/op-acceptance-tests/tests/safedb_proof_consumers/consumers_test.go @@ -30,7 +30,9 @@ func TestProposerPublishesSuperRoot(gt *testing.T) { ), ) - game := system.DisputeGameFactory().WaitForGame() + // The genesis block's required L1 is fixed at 0 without a SafeDB read, so wait for a game past it. + rollupConfig := system.L2Chain.Escape().RollupConfig() + game := system.DisputeGameFactory().WaitForGameFrom(rollupConfig.Genesis.L2Time + rollupConfig.BlockTime) game.VerifyGameType(gameTypes.SuperPermissionedGameType) response := dsl.NewOpNodeSuperRoots(system.L2CL).AwaitSuperRootAtTimestamp(game.L2SequenceNumber()) require.Equal(game.RootClaimValue(), common.Hash(eth.SuperRoot(response.Data.Super))) @@ -38,8 +40,6 @@ func TestProposerPublishesSuperRoot(gt *testing.T) { chainOutput, ok := response.OptimisticAtTimestamp[system.L2Chain.ChainID()] require.True(ok, "super root response must include the L2 chain") block := system.L2EL.BlockRefByHash(chainOutput.Output.BlockHash) - // The genesis block's required L1 is fixed at 0 without a SafeDB read. - require.NotZero(block.Number, "the game must commit past L2 genesis") require.Equal(system.L2CL.OutputAtBlock(block.Number).OutputRoot, chainOutput.OutputRoot) requireFirstL1WithSafeHead(t, system.L2CL, block.Number, response.Data.VerifiedRequiredL1) } diff --git a/op-devstack/dsl/proofs/dispute_game_factory.go b/op-devstack/dsl/proofs/dispute_game_factory.go index d44ef2aa0c1..707a80c709c 100644 --- a/op-devstack/dsl/proofs/dispute_game_factory.go +++ b/op-devstack/dsl/proofs/dispute_game_factory.go @@ -248,6 +248,35 @@ func (f *DisputeGameFactory) WaitForGame() *FaultDisputeGame { return f.GameAtIndex(initialCount) } +// WaitForGameFrom waits for a new game whose L2 sequence number is at least minSequenceNumber, +// skipping new games below it. +func (f *DisputeGameFactory) WaitForGameFrom(minSequenceNumber uint64) *FaultDisputeGame { + nextIndex := f.GameCount() + timedCtx, cancel := context.WithTimeout(f.t.Ctx(), 10*time.Minute) + defer cancel() + var lastReadErr error + var found *FaultDisputeGame + err := wait.For(timedCtx, time.Second, func() (bool, error) { + count, readErr := contractio.Read(f.dgf.GameCount(), timedCtx) + lastReadErr = readErr + if readErr != nil { + f.log.Debug("Game count unavailable while waiting for new game", "next", nextIndex, "err", readErr) + return false, nil + } + for ; nextIndex < count.Int64(); nextIndex++ { + game := f.GameAtIndex(nextIndex) + if game.L2SequenceNumber() >= minSequenceNumber { + found = game + return true, nil + } + f.log.Info("Skipping game below L2 sequence number", "index", nextIndex, "gameSequenceNumber", game.L2SequenceNumber(), "minSequenceNumber", minSequenceNumber) + } + return false, nil + }) + f.require.NoErrorf(err, "dispute game factory created no new game at or above L2 sequence number %d; last read error: %v", minSequenceNumber, lastReadErr) + return found +} + // WaitForZKGameAtIndex waits until the factory holds a ZK dispute game at the // given index, then returns that game. func (f *DisputeGameFactory) WaitForZKGameAtIndex(idx int64) *ZKGame { From 55c3fc3256e7ca528c7e29e66163bc100dbc629a Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Wed, 11 Nov 2026 18:11:04 +0100 Subject: [PATCH 4/6] ci,op-acceptance-tests,op-devstack: harden SafeDB resync and super-root checks DisputeGameFactory gains WaitForSuperGameFrom, which retries transient reads and checks the super game type; it and WaitForGame share one polling loop. SuperRootQuerier.AwaitSuperRootAtTimestamp reports the last query error on timeout, and AwaitSuperRootMatching asserts the root claim on the awaited response. L2CLNode.SafeHeadRecordAtL1Block returns nil when the SafeDB has no record, so tests need no op-node import. TestTruncateDatabaseOnELResync asserts that the SafeDB floor moves past its pre-resync value. The CL-resync test's comment states what it can observe, and its package is named safeheaddb_clsync. The op-geth EL step runs both packages in parallel with a 20m go timeout and names the CL kind in its failure messages. Co-Authored-By: Claude Opus 5.5 --- .circleci/continue/main.yml | 10 +-- .../safedb_proof_consumers/consumers_test.go | 22 +++---- .../safeheaddb_clsync/safeheaddb_test.go | 6 +- .../safeheaddb_elsync/safeheaddb_test.go | 3 + op-devstack/dsl/l2_cl.go | 6 ++ .../dsl/proofs/dispute_game_factory.go | 65 ++++++++++++------- op-devstack/dsl/super_root_source.go | 20 +++++- 7 files changed, 82 insertions(+), 50 deletions(-) diff --git a/.circleci/continue/main.yml b/.circleci/continue/main.yml index 8edf9a04327..a09fa18b2ff 100644 --- a/.circleci/continue/main.yml +++ b/.circleci/continue/main.yml @@ -2188,9 +2188,9 @@ jobs: # any skip: buildL2Genesis skips op-geth on Karst chains, which would # otherwise turn a fork-default change into silent loss of coverage. - run: - name: Run SafeDB resync tests with op-geth + name: Run SafeDB resync tests with an op-geth EL # Above go's -timeout, so a hang ends in go's goroutine dump, not a silent kill. - no_output_timeout: 15m + no_output_timeout: 25m command: | if [ "${CIRCLE_NODE_INDEX:-0}" != "0" ]; then exit 0 @@ -2202,19 +2202,19 @@ jobs: --format testname \ --jsonfile "$EVENTS_FILE" \ --junitfile op-acceptance-tests/results/safedb-op-geth.xml \ - -- -count=1 -p=1 -parallel=1 -timeout=10m \ + -- -count=1 -p=2 -parallel=1 -timeout=20m \ -run "^($(echo "$RESYNC_TESTS" | tr ' ' '|'))$" \ ./op-acceptance-tests/tests/safeheaddb_elsync \ ./op-acceptance-tests/tests/safeheaddb_clsync # test2json can drop a result event and leave a \u0016 marker on the result output line, # so the "--- PASS"/"--- SKIP" output lines count as well as the events. if grep -qE '"Action":"skip","Package":"[^"]*","Test":"|"Output":"(\\u0016)?--- SKIP: ' "$EVENTS_FILE"; then - echo "SafeDB resync tests were skipped under op-geth" >&2 + echo "SafeDB resync tests were skipped with an op-geth EL and $DEVSTACK_L2CL_KIND" >&2 exit 1 fi for test_name in $RESYNC_TESTS; do if ! grep -qE "\"Action\":\"pass\",\"Package\":\"[^\"]*\",\"Test\":\"$test_name\"|\"Output\":\"(\\\\u0016)?--- PASS: $test_name \(" "$EVENTS_FILE"; then - echo "SafeDB resync test $test_name did not pass under op-geth" >&2 + echo "SafeDB resync test $test_name did not pass with an op-geth EL and $DEVSTACK_L2CL_KIND" >&2 exit 1 fi done diff --git a/op-acceptance-tests/tests/safedb_proof_consumers/consumers_test.go b/op-acceptance-tests/tests/safedb_proof_consumers/consumers_test.go index c2a758353cb..ad2e1c8840e 100644 --- a/op-acceptance-tests/tests/safedb_proof_consumers/consumers_test.go +++ b/op-acceptance-tests/tests/safedb_proof_consumers/consumers_test.go @@ -1,7 +1,6 @@ package safedb_proof_consumers import ( - "errors" "testing" gameTypes "github.com/ethereum-optimism/optimism/op-challenger/game/types" @@ -12,9 +11,7 @@ import ( "github.com/ethereum-optimism/optimism/op-devstack/dsl/proofs" "github.com/ethereum-optimism/optimism/op-devstack/presets" "github.com/ethereum-optimism/optimism/op-devstack/sysgo" - "github.com/ethereum-optimism/optimism/op-node/node/safedb" "github.com/ethereum-optimism/optimism/op-service/eth" - "github.com/ethereum/go-ethereum/common" safety "github.com/ethereum-optimism/optimism/op-service/eth/safety" ) @@ -31,11 +28,9 @@ func TestProposerPublishesSuperRoot(gt *testing.T) { ) // The genesis block's required L1 is fixed at 0 without a SafeDB read, so wait for a game past it. - rollupConfig := system.L2Chain.Escape().RollupConfig() - game := system.DisputeGameFactory().WaitForGameFrom(rollupConfig.Genesis.L2Time + rollupConfig.BlockTime) + game := system.DisputeGameFactory().WaitForSuperGameFrom(system.L2Chain.TimestampForBlockNum(1)) game.VerifyGameType(gameTypes.SuperPermissionedGameType) - response := dsl.NewOpNodeSuperRoots(system.L2CL).AwaitSuperRootAtTimestamp(game.L2SequenceNumber()) - require.Equal(game.RootClaimValue(), common.Hash(eth.SuperRoot(response.Data.Super))) + response := dsl.NewOpNodeSuperRoots(system.L2CL).AwaitSuperRootMatching(game.L2SequenceNumber(), game.RootClaimValue()) chainOutput, ok := response.OptimisticAtTimestamp[system.L2Chain.ChainID()] require.True(ok, "super root response must include the L2 chain") @@ -61,8 +56,8 @@ func TestDisputeMonitorAgreesWithSafeOutputRoot(gt *testing.T) { // dispute-mon also agrees when it cannot read the SafeDB, so check the record it relies on. l1Head := system.L1EL.BlockRefByHash(game.L1Head()) - safeHead, err := system.L2CL.SafeHeadAtL1Block(l1Head.Number) - require.NoError(err, "SafeDB must answer at the game's L1 head %d", l1Head.Number) + safeHead := system.L2CL.SafeHeadRecordAtL1Block(l1Head.Number) + require.NotNil(safeHead, "SafeDB must have a record at the game's L1 head %d", l1Head.Number) require.GreaterOrEqual(safeHead.SafeHead.Number, game.L2SequenceNumber()) } @@ -105,15 +100,14 @@ func requireFirstL1WithSafeHead(t devtest.T, cl *dsl.L2CLNode, l2Block uint64, r require := t.Require() require.NotZero(requiredL1.Number, "required L1 for L2 block %d", l2Block) - atRequired, err := cl.SafeHeadAtL1Block(requiredL1.Number) - require.NoError(err, "SafeDB must answer at the required L1 block %d", requiredL1.Number) + atRequired := cl.SafeHeadRecordAtL1Block(requiredL1.Number) + require.NotNil(atRequired, "SafeDB must have a record at the required L1 block %d", requiredL1.Number) require.Equal(requiredL1, atRequired.L1Block, "required L1 must be a SafeDB record") require.GreaterOrEqual(atRequired.SafeHead.Number, l2Block) - beforeRequired, err := cl.SafeHeadAtL1Block(requiredL1.Number - 1) - if errors.Is(err, safedb.ErrNotFound) { + beforeRequired := cl.SafeHeadRecordAtL1Block(requiredL1.Number - 1) + if beforeRequired == nil { return } - require.NoError(err, "SafeDB must answer below the required L1 block %d", requiredL1.Number) require.Less(beforeRequired.SafeHead.Number, l2Block, "L2 block %d was already safe before the required L1 block", l2Block) } diff --git a/op-acceptance-tests/tests/safeheaddb_clsync/safeheaddb_test.go b/op-acceptance-tests/tests/safeheaddb_clsync/safeheaddb_test.go index 4a1598254ef..6792798785b 100644 --- a/op-acceptance-tests/tests/safeheaddb_clsync/safeheaddb_test.go +++ b/op-acceptance-tests/tests/safeheaddb_clsync/safeheaddb_test.go @@ -1,4 +1,4 @@ -package safeheaddb_elsync +package safeheaddb_clsync import ( "testing" @@ -44,8 +44,8 @@ func TestPreserveDatabaseOnCLResync(gt *testing.T) { sys.L2CLB.VerifySafeHeadDatabaseMatches(sys.L2CL) // Stop the verifier node. Since the sysgo EL uses in-memory storage this also wipes its database. - // With the EL reset to genesis, the restarted CL stays in CL sync mode and re-derives the chain from L1, - // so unlike the EL sync case it must not truncate its safe head database. + // Re-deriving from L1 writes the same records again, so this cannot tell a preserved safe head + // database from a rebuilt one; it checks that history back to startSafeBlock is still served. sys.L2ELB.Stop() sys.L2CLB.Stop() diff --git a/op-acceptance-tests/tests/safeheaddb_elsync/safeheaddb_test.go b/op-acceptance-tests/tests/safeheaddb_elsync/safeheaddb_test.go index bde22f221ca..c8383c0c122 100644 --- a/op-acceptance-tests/tests/safeheaddb_elsync/safeheaddb_test.go +++ b/op-acceptance-tests/tests/safeheaddb_elsync/safeheaddb_test.go @@ -48,6 +48,7 @@ func TestTruncateDatabaseOnELResync(gt *testing.T) { sys.L2CLB.InSync(sys.L2CL, safety.LocalSafe, 30) sys.L2CLB.VerifySafeHeadDatabaseMatches(sys.L2CL) + preResyncFloor := sys.L2CLB.SafeHeadDBFloor() // Stop the verifier node. Since the sysgo EL uses in-memory storage this also wipes its database. // With the EL reset to genesis, when the CL restarts it will use EL sync to resync the chain rather than @@ -72,6 +73,8 @@ func TestTruncateDatabaseOnELResync(gt *testing.T) { sys.L2CLB.Advanced(safety.LocalSafe, 1, 90) // At least one safe head db update after resync sys.L2CLB.VerifySafeHeadDatabaseMatches(sys.L2CL) + // EL sync adopts heads that were not derived from L1, so the records from before it must be gone. + t.Require().Greater(sys.L2CLB.SafeHeadDBFloor(), preResyncFloor, "EL resync must truncate the safe head database") } func TestNotTruncateDatabaseOnRestartWithExistingDatabase(gt *testing.T) { diff --git a/op-devstack/dsl/l2_cl.go b/op-devstack/dsl/l2_cl.go index 602858b10bb..992d45d7b48 100644 --- a/op-devstack/dsl/l2_cl.go +++ b/op-devstack/dsl/l2_cl.go @@ -563,6 +563,12 @@ func (cl *L2CLNode) SafeHeadAtL1Block(l1BlockNum uint64) (*eth.SafeHeadResponse, return cl.inner.RollupAPI().SafeHeadAtL1Block(ctx, l1BlockNum) } +// SafeHeadRecordAtL1Block returns the safe head recorded at or before the given L1 block, or nil +// when the SafeDB has no record there. Any other error fails the test. +func (cl *L2CLNode) SafeHeadRecordAtL1Block(l1BlockNum uint64) *eth.SafeHeadResponse { + return cl.safeHeadAtL1Block(l1BlockNum) +} + func (cl *L2CLNode) safeHeadAtL1Block(l1BlockNum uint64) *eth.SafeHeadResponse { resp, err := cl.SafeHeadAtL1Block(l1BlockNum) if errors.Is(err, safedb.ErrNotFound) { diff --git a/op-devstack/dsl/proofs/dispute_game_factory.go b/op-devstack/dsl/proofs/dispute_game_factory.go index 707a80c709c..810383f87e5 100644 --- a/op-devstack/dsl/proofs/dispute_game_factory.go +++ b/op-devstack/dsl/proofs/dispute_game_factory.go @@ -3,6 +3,7 @@ package proofs import ( "context" "encoding/binary" + "errors" "math" "math/big" "net/url" @@ -229,52 +230,66 @@ func (f *DisputeGameFactory) GameArgs(gameType gameTypes.GameType) []byte { } func (f *DisputeGameFactory) WaitForGame() *FaultDisputeGame { - initialCount := f.GameCount() - timedCtx, cancel := context.WithTimeout(f.t.Ctx(), 10*time.Minute) - defer cancel() - var lastReadErr error - err := wait.For(timedCtx, time.Second, func() (bool, error) { - count, readErr := contractio.Read(f.dgf.GameCount(), timedCtx) - lastReadErr = readErr - if readErr != nil { - f.log.Debug("Game count unavailable while waiting for new game", "current", initialCount, "err", readErr) + idx, err := f.waitForNewGame(func(context.Context, int64) (bool, error) { return true, nil }) + f.require.NoError(err, "dispute game factory did not create a new game") + return f.GameAtIndex(idx) +} + +// WaitForSuperGameFrom waits for a new super game whose L2 timestamp is at least minTimestamp, +// skipping new games before it. +func (f *DisputeGameFactory) WaitForSuperGameFrom(minTimestamp uint64) *SuperFaultDisputeGame { + idx, err := f.waitForNewGame(func(ctx context.Context, idx int64) (bool, error) { + gameInfo, err := contractio.Read(f.dgf.GameAtIndex(big.NewInt(idx)), ctx) + if err != nil { + return false, err + } + game := bindings.NewFaultDisputeGame(bindings.WithClient(f.ethClient), bindings.WithTo(gameInfo.Proxy), bindings.WithTest(f.t)) + timestamp, err := contractio.Read(game.L2SequenceNumber(), ctx) + if err != nil { + return false, err + } + if bigs.Uint64Strict(timestamp) < minTimestamp { + f.log.Info("Skipping game before timestamp", "index", idx, "gameTimestamp", timestamp, "minTimestamp", minTimestamp) return false, nil } - gameCount := count.Int64() - f.t.Logf("waiting for new game. current=%d new=%d", initialCount, gameCount) - return gameCount > initialCount, nil + return true, nil }) - f.require.NoErrorf(err, "dispute game factory did not create a new game beyond count %d; last read error: %v", initialCount, lastReadErr) - return f.GameAtIndex(initialCount) + f.require.NoErrorf(err, "dispute game factory did not create a new game at or after timestamp %d", minTimestamp) + return f.SuperGameAtIndex(idx) } -// WaitForGameFrom waits for a new game whose L2 sequence number is at least minSequenceNumber, -// skipping new games below it. -func (f *DisputeGameFactory) WaitForGameFrom(minSequenceNumber uint64) *FaultDisputeGame { +// waitForNewGame checks games created after the call, in index order, until isWanted accepts +// one, and returns its index. Read errors are retried until the timeout. +func (f *DisputeGameFactory) waitForNewGame(isWanted func(ctx context.Context, idx int64) (bool, error)) (int64, error) { nextIndex := f.GameCount() timedCtx, cancel := context.WithTimeout(f.t.Ctx(), 10*time.Minute) defer cancel() var lastReadErr error - var found *FaultDisputeGame err := wait.For(timedCtx, time.Second, func() (bool, error) { count, readErr := contractio.Read(f.dgf.GameCount(), timedCtx) - lastReadErr = readErr if readErr != nil { + lastReadErr = readErr f.log.Debug("Game count unavailable while waiting for new game", "next", nextIndex, "err", readErr) return false, nil } + f.log.Info("Waiting for new game", "next", nextIndex, "count", count) for ; nextIndex < count.Int64(); nextIndex++ { - game := f.GameAtIndex(nextIndex) - if game.L2SequenceNumber() >= minSequenceNumber { - found = game + wanted, readErr := isWanted(timedCtx, nextIndex) + if readErr != nil { + lastReadErr = readErr + f.log.Debug("Game unavailable while waiting for new game", "index", nextIndex, "err", readErr) + return false, nil + } + if wanted { return true, nil } - f.log.Info("Skipping game below L2 sequence number", "index", nextIndex, "gameSequenceNumber", game.L2SequenceNumber(), "minSequenceNumber", minSequenceNumber) } return false, nil }) - f.require.NoErrorf(err, "dispute game factory created no new game at or above L2 sequence number %d; last read error: %v", minSequenceNumber, lastReadErr) - return found + if err != nil { + return 0, errors.Join(err, lastReadErr) + } + return nextIndex, nil } // WaitForZKGameAtIndex waits until the factory holds a ZK dispute game at the diff --git a/op-devstack/dsl/super_root_source.go b/op-devstack/dsl/super_root_source.go index 074a4015af5..a382ab64d6e 100644 --- a/op-devstack/dsl/super_root_source.go +++ b/op-devstack/dsl/super_root_source.go @@ -77,7 +77,18 @@ func (s *SuperRootQuerier) SuperRootAtTimestamp(timestamp uint64) eth.SuperRootA // AssertSuperRootAtTimestamp asserts that the super-root at the given timestamp matches the expected root claim. func (s *SuperRootQuerier) AssertSuperRootAtTimestamp(l2SequenceNumber uint64, rootClaim common.Hash) { - resp := s.SuperRootAtTimestamp(l2SequenceNumber) + s.requireSuperRootMatches(s.SuperRootAtTimestamp(l2SequenceNumber), l2SequenceNumber, rootClaim) +} + +// AwaitSuperRootMatching waits for the super-root at the given timestamp, asserts that it matches +// the expected root claim, and returns the response. +func (s *SuperRootQuerier) AwaitSuperRootMatching(l2SequenceNumber uint64, rootClaim common.Hash) eth.SuperRootAtTimestampResponse { + resp := s.AwaitSuperRootAtTimestamp(l2SequenceNumber) + s.requireSuperRootMatches(resp, l2SequenceNumber, rootClaim) + return resp +} + +func (s *SuperRootQuerier) requireSuperRootMatches(resp eth.SuperRootAtTimestampResponse, l2SequenceNumber uint64, rootClaim common.Hash) { s.require.NotNilf(resp.Data, "super root does not exist at time %d", l2SequenceNumber) superRoot := eth.SuperRoot(resp.Data.Super) s.require.Equal(superRoot[:], rootClaim[:]) @@ -93,15 +104,18 @@ func (s *SuperRootQuerier) AwaitSuperRootAtTimestamp(timestamp uint64) eth.Super ctx, cancel := context.WithTimeout(s.ctx, 5*DefaultTimeout) defer cancel() var resp eth.SuperRootAtTimestampResponse + var lastErr error err := wait.For(ctx, 1*time.Second, func() (bool, error) { var err error resp, err = s.api.SuperRootAtTimestamp(ctx, timestamp) if err != nil { - return false, nil // Ignore transient errors. + lastErr = err + s.log.Debug("Super-root query failed, retrying", "timestamp", timestamp, "err", err) + return false, nil // Retry: some errors are transient. } return resp.Data != nil, nil }) - s.require.NoError(err, "super-root at timestamp %d was not validated in time", timestamp) + s.require.NoError(err, "super-root at timestamp %d was not validated in time; last error: %v", timestamp, lastErr) return resp } From a1d1821edea09427938ec5bef96396f8c71831bb Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Thu, 12 Nov 2026 14:29:34 +0100 Subject: [PATCH 5/6] op-acceptance-tests: drop the SafeDB floor assertion from the EL-resync test Wiping the EL makes the CL reset to genesis, which leaves a genesis record, so the SafeDB floor drops to 0 instead of moving past its pre-resync value on both op-node and kona-node. The floor does not show whether EL sync truncated the database. Co-Authored-By: Claude Opus 5.5 --- op-acceptance-tests/tests/safeheaddb_elsync/safeheaddb_test.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/op-acceptance-tests/tests/safeheaddb_elsync/safeheaddb_test.go b/op-acceptance-tests/tests/safeheaddb_elsync/safeheaddb_test.go index c8383c0c122..bde22f221ca 100644 --- a/op-acceptance-tests/tests/safeheaddb_elsync/safeheaddb_test.go +++ b/op-acceptance-tests/tests/safeheaddb_elsync/safeheaddb_test.go @@ -48,7 +48,6 @@ func TestTruncateDatabaseOnELResync(gt *testing.T) { sys.L2CLB.InSync(sys.L2CL, safety.LocalSafe, 30) sys.L2CLB.VerifySafeHeadDatabaseMatches(sys.L2CL) - preResyncFloor := sys.L2CLB.SafeHeadDBFloor() // Stop the verifier node. Since the sysgo EL uses in-memory storage this also wipes its database. // With the EL reset to genesis, when the CL restarts it will use EL sync to resync the chain rather than @@ -73,8 +72,6 @@ func TestTruncateDatabaseOnELResync(gt *testing.T) { sys.L2CLB.Advanced(safety.LocalSafe, 1, 90) // At least one safe head db update after resync sys.L2CLB.VerifySafeHeadDatabaseMatches(sys.L2CL) - // EL sync adopts heads that were not derived from L1, so the records from before it must be gone. - t.Require().Greater(sys.L2CLB.SafeHeadDBFloor(), preResyncFloor, "EL resync must truncate the safe head database") } func TestNotTruncateDatabaseOnRestartWithExistingDatabase(gt *testing.T) { From 68cf64ea2d5bd99e8162214998e0e1f9463c81ca Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Thu, 12 Nov 2026 17:10:14 +0100 Subject: [PATCH 6/6] ci: drop the op-geth SafeDB resync step TestTruncateDatabaseOnELResync and TestPreserveDatabaseOnCLResync run in the normal op-reth acceptance jobs since #23827, and op-geth is being removed from op-devstack (#20257, #21744). Co-Authored-By: Claude Opus 5.5 --- .circleci/continue/main.yml | 36 ------------------------------------ 1 file changed, 36 deletions(-) diff --git a/.circleci/continue/main.yml b/.circleci/continue/main.yml index be29a18afc2..d8095eb5994 100644 --- a/.circleci/continue/main.yml +++ b/.circleci/continue/main.yml @@ -2255,42 +2255,6 @@ jobs: LOG_DIR=$(ls -td -- logs/* | head -1) cat "$LOG_DIR"/failed/*.log || true when: on_fail - # The two resync cases below SkipOnOpReth, so the op-reth run above never - # exercises them. They run here with op-geth, after the sharded suite so a - # resync failure cannot skip shard 0's share of it, and the step fails on - # any skip: buildL2Genesis skips op-geth on Karst chains, which would - # otherwise turn a fork-default change into silent loss of coverage. - - run: - name: Run SafeDB resync tests with an op-geth EL - # Above go's -timeout, so a hang ends in go's goroutine dump, not a silent kill. - no_output_timeout: 25m - command: | - if [ "${CIRCLE_NODE_INDEX:-0}" != "0" ]; then - exit 0 - fi - RESYNC_TESTS="TestTruncateDatabaseOnELResync TestPreserveDatabaseOnCLResync" - EVENTS_FILE=op-acceptance-tests/logs/safedb-op-geth/raw_go_events.log - mkdir -p "$(dirname "$EVENTS_FILE")" op-acceptance-tests/results - DEVSTACK_L2EL_KIND=op-geth LOG_LEVEL=info gotestsum \ - --format testname \ - --jsonfile "$EVENTS_FILE" \ - --junitfile op-acceptance-tests/results/safedb-op-geth.xml \ - -- -count=1 -p=2 -parallel=1 -timeout=20m \ - -run "^($(echo "$RESYNC_TESTS" | tr ' ' '|'))$" \ - ./op-acceptance-tests/tests/safeheaddb_elsync \ - ./op-acceptance-tests/tests/safeheaddb_clsync - # test2json can drop a result event and leave a \u0016 marker on the result output line, - # so the "--- PASS"/"--- SKIP" output lines count as well as the events. - if grep -qE '"Action":"skip","Package":"[^"]*","Test":"|"Output":"(\\u0016)?--- SKIP: ' "$EVENTS_FILE"; then - echo "SafeDB resync tests were skipped with an op-geth EL and $DEVSTACK_L2CL_KIND" >&2 - exit 1 - fi - for test_name in $RESYNC_TESTS; do - if ! grep -qE "\"Action\":\"pass\",\"Package\":\"[^\"]*\",\"Test\":\"$test_name\"|\"Output\":\"(\\\\u0016)?--- PASS: $test_name \(" "$EVENTS_FILE"; then - echo "SafeDB resync test $test_name did not pass with an op-geth EL and $DEVSTACK_L2CL_KIND" >&2 - exit 1 - fi - done # Save the module cache for future runs - go-save-cache: namespace: op-acceptance-tests