From 5702c3985fcfbae41c76e37e378292cdffaf99c4 Mon Sep 17 00:00:00 2001 From: Teddy Knox Date: Wed, 18 Nov 2026 13:56:38 -0500 Subject: [PATCH 1/3] rust,op-reth: restore the teardown-SIGSEGV retry and cover the two-node tests The #20973 stopgap retries a test that dies with SIGSEGV during teardown, after its body has passed. Its filter named `p2p_version::peers_negotiate_eth_69` exactly. #23046 renamed that test to `peers_negotiate_eth_71`, so the override has matched no test since then: a crash fails the job again, with no retry and no Slack alert. The same crash has now been seen on a second test of the same binary, `invalid_post_exec::test_buffered_malformed_post_exec_child_is_invalid`. Both tests start two nodes in one process with `setup(2)`. #20973 scoped the crash to the p2p test because it was then the only such test; #23186 added two more. The override now covers the three two-node tests of `reth-optimism-node::it`: - The p2p test, by an anchored regex on the eth version. The next version bump renames the test again and no longer disables the retry. - The two `invalid_post_exec` tests, by exact name. One has crashed. The other builds its two nodes the same way and has no crash on record. The crash has not been seen on a one-node test, so those tests stay under the no-retries policy. The two `invalid_post_exec` tests now install the fatal-signal handler, as the p2p test does. A retried crash then leaves the faulting stack in the JUnit report, which is what the Slack alert points to. The CircleCI flaky-retry step is unchanged. It greps the JUnit report for ` --- .circleci/continue/rust-ci.yml | 6 +-- rust/.config/nextest.toml | 50 +++++++++++++------ .../crates/node/tests/it/invalid_post_exec.rs | 10 ++++ .../crates/node/tests/it/p2p_version.rs | 3 +- 4 files changed, 50 insertions(+), 19 deletions(-) diff --git a/.circleci/continue/rust-ci.yml b/.circleci/continue/rust-ci.yml index 688dd990308..d87dd1c8a1f 100644 --- a/.circleci/continue/rust-ci.yml +++ b/.circleci/continue/rust-ci.yml @@ -344,15 +344,15 @@ jobs: path: <>/target/nextest/default/junit.xml destination: junit when: always - # A retried flake (e.g. the eth69 teardown SIGSEGV, #20973) passes on rerun and + # A retried flake (e.g. the teardown SIGSEGV, #20973) passes on rerun and # leaves the job GREEN, so nothing else surfaces it — CircleCI does not parse # nextest's element, so store_test_results/Insights stay blind. Detect # it ourselves and ping Slack. Every occurrence is a rare, precious data point (the # crash has never reproduced locally), so alert on every branch, not just develop — # a stack lost on a PR run is a stack lost. Fork PRs get no SLACK_ACCESS_TOKEN and # self-skip. Only tests with a retry override can emit , so today this - # fires only for the one tracked flake. This step must never fail the job (the retry - # passing is the whole point). + # fires only for the tests that the #20973 override in rust/.config/nextest.toml names. + # This step must never fail the job (the retry passing is the whole point). - run: name: Alert Slack on a flaky retry when: always diff --git a/rust/.config/nextest.toml b/rust/.config/nextest.toml index 9cb49f19920..7ab2bd8d993 100644 --- a/rust/.config/nextest.toml +++ b/rust/.config/nextest.toml @@ -2,7 +2,7 @@ [profile.default] # No test retries by default: a flaky test should fail loudly and get fixed, not -# be retried away. The sole exception is the eth69 teardown-SIGSEGV override at the +# be retried away. The sole exception is the teardown-SIGSEGV override at the # bottom of this file (#20973) — a narrowly-scoped stopgap for a crash that happens # *after* the test has already passed. # Flag tests slower than 30s and terminate any that hang past ~2 minutes. @@ -10,7 +10,7 @@ slow-timeout = { period = "30s", terminate-after = 4 } # Emit a JUnit report so CircleCI `store_test_results` can surface individual test # outcomes (written to `target/nextest/default/junit.xml`). The cargo-test job also -# greps it for a `` to detect the eth69 retry below and alert Slack. +# greps it for a `` to detect a retry from that override and alert Slack. [profile.default.junit] path = "junit.xml" @@ -36,23 +36,43 @@ threads-required = "num-test-threads" filter = "binary(e2e_testsuite)" slow-timeout = { period = "2m", terminate-after = 3 } -# `p2p_version::peers_negotiate_eth_69` intermittently dies with SIGSEGV *after* its body -# has passed — the assertions succeed and nextest prints `test result: ok`, then the binary -# segfaults during teardown, failing the run purely on the signal exit and ejecting unrelated -# PRs from the merge queue. Retrying is sound *because* the crash is teardown-only: nextest -# reruns in a fresh process, re-establishing every assertion rather than papering over a -# failure. This deliberately contradicts the no-retries policy above — it is a stopgap for -# an unresolved crash tracked in #20973; remove it once that is fixed. +# A `reth-optimism-node::it` test that starts two nodes in one process intermittently dies with +# SIGSEGV *after* its body has passed — the assertions succeed and nextest prints +# `test result: ok`, then the binary segfaults during teardown, failing the run purely on the +# signal exit and ejecting unrelated PRs from the merge queue. Retrying is sound *because* the +# crash is teardown-only: nextest reruns in a fresh process, re-establishing every assertion +# rather than papering over a failure. This deliberately contradicts the no-retries policy +# above — it is a stopgap for an unresolved crash tracked in #20973; remove it once that is +# fixed. +# +# Scope: the three tests of that binary that build two nodes with `E2ETestSetupBuilder` +# (`setup(2)`, `setup_lagoon(2)`). The crash has been seen on two of them: on the p2p test +# repeatedly, and on `test_buffered_malformed_post_exec_child_is_invalid`. The third test has +# the same shape. The crash has not been seen on a one-node test, so the override does not +# cover the rest of the binary. If you add a two-node test to that binary, add it here and call +# `crash_backtrace::install()` in it. # # The retry turns the red job green, so the cargo-test CI job greps this JUnit report for the # `` and pings Slack (CircleCI does not parse that element, so its Tests tab and -# Insights stay blind — the detection is ours). +# Insights stay blind — the detection is ours). Each test below installs the fatal-signal +# handler in `tests/it/crash_backtrace.rs`, so the failed attempt in that report carries the +# faulting stack. # -# `test(=exact)`, not `binary_id(...)`: nextest validates override filters against the whole +# `test(...)`, not `binary_id(...)`: nextest validates override filters against the whole # workspace binary namespace, so a future crate/binary rename would hard-error every run in -# `rust/`; the exact-match predicate degrades to a no-op instead. The `=` is load-bearing — -# bare `test(...)` is a substring match, so a sibling like `peers_negotiate_eth_69_disconnect` -# would silently inherit the retry. +# `rust/`; a `test(...)` predicate that matches nothing degrades to a no-op instead. That no-op +# is silent: this override was dead from #23046, which renamed `peers_negotiate_eth_69`, until +# the filter below replaced the exact name. +# - The p2p test name carries the eth wire version, and a reth update that bumps the version +# renames the test. The regex follows that rename. Its `^` and `$` are load-bearing — an +# unanchored regex is a substring match, so a sibling like `peers_negotiate_eth_71_disconnect` +# would silently inherit the retry. +# - The other two names are exact; the `=` is load-bearing for the same reason. If you rename +# one of those tests, rename it here. [[profile.default.overrides]] -filter = 'test(=p2p_version::peers_negotiate_eth_69)' +filter = ''' +test(/^p2p_version::peers_negotiate_eth_\d+$/) +| test(=invalid_post_exec::test_buffered_malformed_post_exec_child_is_invalid) +| test(=invalid_post_exec::test_lagoon_buffered_child_with_wrong_post_exec_block_number_is_invalid) +''' retries = 2 diff --git a/rust/op-reth/crates/node/tests/it/invalid_post_exec.rs b/rust/op-reth/crates/node/tests/it/invalid_post_exec.rs index 1d880f2bbd9..46337e3e59d 100644 --- a/rust/op-reth/crates/node/tests/it/invalid_post_exec.rs +++ b/rust/op-reth/crates/node/tests/it/invalid_post_exec.rs @@ -159,6 +159,11 @@ async fn test_lagoon_new_payload_with_wrong_post_exec_block_number_is_invalid() #[tokio::test] async fn test_lagoon_buffered_child_with_wrong_post_exec_block_number_is_invalid() -> eyre::Result<()> { + // A test that starts two nodes in one process intermittently segfaults during teardown, + // after its body has passed (ethereum-optimism/optimism#20973). The handler outlives the + // body, so it reports the faulting thread and stack when that happens. + // `rust/.config/nextest.toml` retries this test by name. + crate::crash_backtrace::install(); reth_tracing::init_test_tracing(); let (mut nodes, wallet) = setup_lagoon(2).await?; @@ -213,6 +218,11 @@ async fn test_new_payload_with_oversized_post_exec_entries_is_invalid() -> eyre: #[tokio::test] async fn test_buffered_malformed_post_exec_child_is_invalid() -> eyre::Result<()> { + // A test that starts two nodes in one process intermittently segfaults during teardown, + // after its body has passed (ethereum-optimism/optimism#20973). The handler outlives the + // body, so it reports the faulting thread and stack when that happens. + // `rust/.config/nextest.toml` retries this test by name. + crate::crash_backtrace::install(); reth_tracing::init_test_tracing(); let (mut nodes, wallet) = setup(2).await?; diff --git a/rust/op-reth/crates/node/tests/it/p2p_version.rs b/rust/op-reth/crates/node/tests/it/p2p_version.rs index c61d64a2642..acd73bea99d 100644 --- a/rust/op-reth/crates/node/tests/it/p2p_version.rs +++ b/rust/op-reth/crates/node/tests/it/p2p_version.rs @@ -17,7 +17,8 @@ const EXPECTED_ETH_VERSION: u64 = 71; async fn peers_negotiate_eth_71() -> eyre::Result<()> { // This test intermittently segfaults during teardown, after the body below has // passed (ethereum-optimism/optimism#20973). The handler outlives the body, so it - // reports the faulting thread and stack when that happens. + // reports the faulting thread and stack when that happens. `rust/.config/nextest.toml` + // retries this test by name. crate::crash_backtrace::install(); reth_tracing::init_test_tracing(); From 7a2ed8f4d9a63094103e3913f66aa8e472455fe0 Mon Sep 17 00:00:00 2001 From: Teddy Knox Date: Wed, 18 Nov 2026 15:16:35 -0500 Subject: [PATCH 2/3] rust,op-reth: retry only the tests on which the teardown SIGSEGV was seen The override also covered `test_lagoon_buffered_child_with_wrong_post_exec_block_number_is_invalid`, which starts two nodes like the two tests that crashed but has no crash on record. A retry for it stretches a stopgap that is an exception to the no-retries policy. The override now covers a test only after the crash has been seen on it. That test keeps the fatal-signal handler. A crash there fails the job and leaves the faulting stack, which is the evidence needed to add the test to the override. Co-Authored-By: Claude Fable 5.2 --- rust/.config/nextest.toml | 17 ++++++++--------- .../crates/node/tests/it/invalid_post_exec.rs | 4 ++-- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/rust/.config/nextest.toml b/rust/.config/nextest.toml index 7ab2bd8d993..1a7a113501f 100644 --- a/rust/.config/nextest.toml +++ b/rust/.config/nextest.toml @@ -45,12 +45,12 @@ slow-timeout = { period = "2m", terminate-after = 3 } # above — it is a stopgap for an unresolved crash tracked in #20973; remove it once that is # fixed. # -# Scope: the three tests of that binary that build two nodes with `E2ETestSetupBuilder` -# (`setup(2)`, `setup_lagoon(2)`). The crash has been seen on two of them: on the p2p test -# repeatedly, and on `test_buffered_malformed_post_exec_child_is_invalid`. The third test has -# the same shape. The crash has not been seen on a one-node test, so the override does not -# cover the rest of the binary. If you add a two-node test to that binary, add it here and call -# `crash_backtrace::install()` in it. +# Scope: the two tests on which the crash has been seen — the p2p test repeatedly, and +# `test_buffered_malformed_post_exec_child_is_invalid`. Both call `setup(2)`. The override +# covers a test only after the crash has been seen on it. `invalid_post_exec` has a third +# two-node test (`setup_lagoon(2)`) with no crash on record; it stays under the no-retries +# policy, as do the one-node tests. That third test installs the fatal-signal handler, so a +# crash there fails the job with a stack. Add the test here if that happens. # # The retry turns the red job green, so the cargo-test CI job greps this JUnit report for the # `` and pings Slack (CircleCI does not parse that element, so its Tests tab and @@ -67,12 +67,11 @@ slow-timeout = { period = "2m", terminate-after = 3 } # renames the test. The regex follows that rename. Its `^` and `$` are load-bearing — an # unanchored regex is a substring match, so a sibling like `peers_negotiate_eth_71_disconnect` # would silently inherit the retry. -# - The other two names are exact; the `=` is load-bearing for the same reason. If you rename -# one of those tests, rename it here. +# - The other name is exact; the `=` is load-bearing for the same reason. If you rename that +# test, rename it here. [[profile.default.overrides]] filter = ''' test(/^p2p_version::peers_negotiate_eth_\d+$/) | test(=invalid_post_exec::test_buffered_malformed_post_exec_child_is_invalid) -| test(=invalid_post_exec::test_lagoon_buffered_child_with_wrong_post_exec_block_number_is_invalid) ''' retries = 2 diff --git a/rust/op-reth/crates/node/tests/it/invalid_post_exec.rs b/rust/op-reth/crates/node/tests/it/invalid_post_exec.rs index 46337e3e59d..52e20dab619 100644 --- a/rust/op-reth/crates/node/tests/it/invalid_post_exec.rs +++ b/rust/op-reth/crates/node/tests/it/invalid_post_exec.rs @@ -161,8 +161,8 @@ async fn test_lagoon_buffered_child_with_wrong_post_exec_block_number_is_invalid -> eyre::Result<()> { // A test that starts two nodes in one process intermittently segfaults during teardown, // after its body has passed (ethereum-optimism/optimism#20973). The handler outlives the - // body, so it reports the faulting thread and stack when that happens. - // `rust/.config/nextest.toml` retries this test by name. + // body, so it reports the faulting thread and stack if that happens here. This test has + // no crash on record, so `rust/.config/nextest.toml` does not retry it. crate::crash_backtrace::install(); reth_tracing::init_test_tracing(); From aaa20457f4fd135502ccf039ebb65ad3910f3373 Mon Sep 17 00:00:00 2001 From: Teddy Knox Date: Wed, 18 Nov 2026 15:28:01 -0500 Subject: [PATCH 3/3] rust: reduce the retry fix to the renamed test name Keep only the change that restores the #20973 stopgap: the override filter names `peers_negotiate_eth_71`, the name the test has had since #23046. Take out the rest of this branch: the second test in the override, the crash handler in the `invalid_post_exec` tests, and the comment edits in `nextest.toml` and `rust-ci.yml`. Co-Authored-By: Claude Fable 5.2 --- .circleci/continue/rust-ci.yml | 6 +-- rust/.config/nextest.toml | 49 ++++++------------- .../crates/node/tests/it/invalid_post_exec.rs | 10 ---- .../crates/node/tests/it/p2p_version.rs | 3 +- 4 files changed, 19 insertions(+), 49 deletions(-) diff --git a/.circleci/continue/rust-ci.yml b/.circleci/continue/rust-ci.yml index d87dd1c8a1f..688dd990308 100644 --- a/.circleci/continue/rust-ci.yml +++ b/.circleci/continue/rust-ci.yml @@ -344,15 +344,15 @@ jobs: path: <>/target/nextest/default/junit.xml destination: junit when: always - # A retried flake (e.g. the teardown SIGSEGV, #20973) passes on rerun and + # A retried flake (e.g. the eth69 teardown SIGSEGV, #20973) passes on rerun and # leaves the job GREEN, so nothing else surfaces it — CircleCI does not parse # nextest's element, so store_test_results/Insights stay blind. Detect # it ourselves and ping Slack. Every occurrence is a rare, precious data point (the # crash has never reproduced locally), so alert on every branch, not just develop — # a stack lost on a PR run is a stack lost. Fork PRs get no SLACK_ACCESS_TOKEN and # self-skip. Only tests with a retry override can emit , so today this - # fires only for the tests that the #20973 override in rust/.config/nextest.toml names. - # This step must never fail the job (the retry passing is the whole point). + # fires only for the one tracked flake. This step must never fail the job (the retry + # passing is the whole point). - run: name: Alert Slack on a flaky retry when: always diff --git a/rust/.config/nextest.toml b/rust/.config/nextest.toml index 1a7a113501f..b7dc8ced905 100644 --- a/rust/.config/nextest.toml +++ b/rust/.config/nextest.toml @@ -2,7 +2,7 @@ [profile.default] # No test retries by default: a flaky test should fail loudly and get fixed, not -# be retried away. The sole exception is the teardown-SIGSEGV override at the +# be retried away. The sole exception is the eth69 teardown-SIGSEGV override at the # bottom of this file (#20973) — a narrowly-scoped stopgap for a crash that happens # *after* the test has already passed. # Flag tests slower than 30s and terminate any that hang past ~2 minutes. @@ -10,7 +10,7 @@ slow-timeout = { period = "30s", terminate-after = 4 } # Emit a JUnit report so CircleCI `store_test_results` can surface individual test # outcomes (written to `target/nextest/default/junit.xml`). The cargo-test job also -# greps it for a `` to detect a retry from that override and alert Slack. +# greps it for a `` to detect the eth69 retry below and alert Slack. [profile.default.junit] path = "junit.xml" @@ -36,42 +36,23 @@ threads-required = "num-test-threads" filter = "binary(e2e_testsuite)" slow-timeout = { period = "2m", terminate-after = 3 } -# A `reth-optimism-node::it` test that starts two nodes in one process intermittently dies with -# SIGSEGV *after* its body has passed — the assertions succeed and nextest prints -# `test result: ok`, then the binary segfaults during teardown, failing the run purely on the -# signal exit and ejecting unrelated PRs from the merge queue. Retrying is sound *because* the -# crash is teardown-only: nextest reruns in a fresh process, re-establishing every assertion -# rather than papering over a failure. This deliberately contradicts the no-retries policy -# above — it is a stopgap for an unresolved crash tracked in #20973; remove it once that is -# fixed. -# -# Scope: the two tests on which the crash has been seen — the p2p test repeatedly, and -# `test_buffered_malformed_post_exec_child_is_invalid`. Both call `setup(2)`. The override -# covers a test only after the crash has been seen on it. `invalid_post_exec` has a third -# two-node test (`setup_lagoon(2)`) with no crash on record; it stays under the no-retries -# policy, as do the one-node tests. That third test installs the fatal-signal handler, so a -# crash there fails the job with a stack. Add the test here if that happens. +# `p2p_version::peers_negotiate_eth_69` intermittently dies with SIGSEGV *after* its body +# has passed — the assertions succeed and nextest prints `test result: ok`, then the binary +# segfaults during teardown, failing the run purely on the signal exit and ejecting unrelated +# PRs from the merge queue. Retrying is sound *because* the crash is teardown-only: nextest +# reruns in a fresh process, re-establishing every assertion rather than papering over a +# failure. This deliberately contradicts the no-retries policy above — it is a stopgap for +# an unresolved crash tracked in #20973; remove it once that is fixed. # # The retry turns the red job green, so the cargo-test CI job greps this JUnit report for the # `` and pings Slack (CircleCI does not parse that element, so its Tests tab and -# Insights stay blind — the detection is ours). Each test below installs the fatal-signal -# handler in `tests/it/crash_backtrace.rs`, so the failed attempt in that report carries the -# faulting stack. +# Insights stay blind — the detection is ours). # -# `test(...)`, not `binary_id(...)`: nextest validates override filters against the whole +# `test(=exact)`, not `binary_id(...)`: nextest validates override filters against the whole # workspace binary namespace, so a future crate/binary rename would hard-error every run in -# `rust/`; a `test(...)` predicate that matches nothing degrades to a no-op instead. That no-op -# is silent: this override was dead from #23046, which renamed `peers_negotiate_eth_69`, until -# the filter below replaced the exact name. -# - The p2p test name carries the eth wire version, and a reth update that bumps the version -# renames the test. The regex follows that rename. Its `^` and `$` are load-bearing — an -# unanchored regex is a substring match, so a sibling like `peers_negotiate_eth_71_disconnect` -# would silently inherit the retry. -# - The other name is exact; the `=` is load-bearing for the same reason. If you rename that -# test, rename it here. +# `rust/`; the exact-match predicate degrades to a no-op instead. The `=` is load-bearing — +# bare `test(...)` is a substring match, so a sibling like `peers_negotiate_eth_69_disconnect` +# would silently inherit the retry. [[profile.default.overrides]] -filter = ''' -test(/^p2p_version::peers_negotiate_eth_\d+$/) -| test(=invalid_post_exec::test_buffered_malformed_post_exec_child_is_invalid) -''' +filter = 'test(=p2p_version::peers_negotiate_eth_71)' retries = 2 diff --git a/rust/op-reth/crates/node/tests/it/invalid_post_exec.rs b/rust/op-reth/crates/node/tests/it/invalid_post_exec.rs index 52e20dab619..1d880f2bbd9 100644 --- a/rust/op-reth/crates/node/tests/it/invalid_post_exec.rs +++ b/rust/op-reth/crates/node/tests/it/invalid_post_exec.rs @@ -159,11 +159,6 @@ async fn test_lagoon_new_payload_with_wrong_post_exec_block_number_is_invalid() #[tokio::test] async fn test_lagoon_buffered_child_with_wrong_post_exec_block_number_is_invalid() -> eyre::Result<()> { - // A test that starts two nodes in one process intermittently segfaults during teardown, - // after its body has passed (ethereum-optimism/optimism#20973). The handler outlives the - // body, so it reports the faulting thread and stack if that happens here. This test has - // no crash on record, so `rust/.config/nextest.toml` does not retry it. - crate::crash_backtrace::install(); reth_tracing::init_test_tracing(); let (mut nodes, wallet) = setup_lagoon(2).await?; @@ -218,11 +213,6 @@ async fn test_new_payload_with_oversized_post_exec_entries_is_invalid() -> eyre: #[tokio::test] async fn test_buffered_malformed_post_exec_child_is_invalid() -> eyre::Result<()> { - // A test that starts two nodes in one process intermittently segfaults during teardown, - // after its body has passed (ethereum-optimism/optimism#20973). The handler outlives the - // body, so it reports the faulting thread and stack when that happens. - // `rust/.config/nextest.toml` retries this test by name. - crate::crash_backtrace::install(); reth_tracing::init_test_tracing(); let (mut nodes, wallet) = setup(2).await?; diff --git a/rust/op-reth/crates/node/tests/it/p2p_version.rs b/rust/op-reth/crates/node/tests/it/p2p_version.rs index acd73bea99d..c61d64a2642 100644 --- a/rust/op-reth/crates/node/tests/it/p2p_version.rs +++ b/rust/op-reth/crates/node/tests/it/p2p_version.rs @@ -17,8 +17,7 @@ const EXPECTED_ETH_VERSION: u64 = 71; async fn peers_negotiate_eth_71() -> eyre::Result<()> { // This test intermittently segfaults during teardown, after the body below has // passed (ethereum-optimism/optimism#20973). The handler outlives the body, so it - // reports the faulting thread and stack when that happens. `rust/.config/nextest.toml` - // retries this test by name. + // reports the faulting thread and stack when that happens. crate::crash_backtrace::install(); reth_tracing::init_test_tracing();