From a45565fa1bdf2c8a35a4a4df04c2613fb6674346 Mon Sep 17 00:00:00 2001 From: Josh Klopfenstein Date: Fri, 16 Oct 2026 13:10:32 -0700 Subject: [PATCH 1/2] ci: balance acceptance shards with incomplete timing history --- .circleci/continue/main.yml | 3 + docs/ai/ci-ops.md | 14 +++ op-acceptance-tests/justfile | 3 + ops/scripts/junit-shard-timings.py | 33 +++++ ops/scripts/shard-tests.sh | 5 +- ops/scripts/test-acceptance-timings.py | 162 +++++++++++++++++++++++++ 6 files changed, 219 insertions(+), 1 deletion(-) create mode 100644 ops/scripts/junit-shard-timings.py create mode 100644 ops/scripts/test-acceptance-timings.py diff --git a/.circleci/continue/main.yml b/.circleci/continue/main.yml index f36f8e70534..d6160a8241f 100644 --- a/.circleci/continue/main.yml +++ b/.circleci/continue/main.yml @@ -2049,6 +2049,9 @@ jobs: - utils/checkout-with-mise: checkout-method: blobless enable-mise-cache: true + - run: + name: Test acceptance test timing helpers + command: python3 ops/scripts/test-acceptance-timings.py - apt-install: extra_flags: "--no-install-recommends" packages: "eatmydata" diff --git a/docs/ai/ci-ops.md b/docs/ai/ci-ops.md index b2459f883dd..fdbcfa9afb9 100644 --- a/docs/ai/ci-ops.md +++ b/docs/ai/ci-ops.md @@ -63,6 +63,20 @@ branch that only added a new `op-core/types` package. The diff touched nothing i `op-deployer`; the failures were a data-dependent flake already fixed on `develop` by #21396. Rebasing made CI green with no change to the branch's own work. +## Acceptance package timeouts with imbalanced shards + +A late-starting acceptance test can fail during setup because earlier tests exhausted +its Go package's shared timeout. Compare raw Go `start`/`cont` events with the +package deadline: `devtest.SerialT` cancels its context three seconds before that +deadline. The assertion that happens to see cancellation need not be the slow test. + +Check the shard logs for `No timing found` warnings and the `Downloading previous +test results` step for the source job. A canceled source run can supply partial +history: in [#23471](https://github.com/ethereum-optimism/optimism/pull/23471), all 64 +unknown tests landed on one node and exhausted the ZK package timeout. Keep a +meaningful missing-timing estimate in `ops/scripts/shard-tests.sh`, and include +parallel subtest costs in the top-level timings uploaded for the next split. + ## Missing `op-core/superchain` bundle (`superchain-configs.zip`) A CI-only compile error like: diff --git a/op-acceptance-tests/justfile b/op-acceptance-tests/justfile index e048d226f27..9deb3b238fb 100644 --- a/op-acceptance-tests/justfile +++ b/op-acceptance-tests/justfile @@ -163,6 +163,9 @@ acceptance-test: set -e cp "$LOG_DIR/all.log" "$LOG_DIR/summary.log" + # The splitter lists top-level tests, whose Go elapsed time can exclude + # parallel subtests. Include their cost in the history uploaded to CircleCI. + python3 "{{REPO_ROOT}}/ops/scripts/junit-shard-timings.py" "$JUNIT_FILE" exit "$TEST_STATUS" clean: diff --git a/ops/scripts/junit-shard-timings.py b/ops/scripts/junit-shard-timings.py new file mode 100644 index 00000000000..69e4518fa9f --- /dev/null +++ b/ops/scripts/junit-shard-timings.py @@ -0,0 +1,33 @@ +#!/usr/bin/env python3 +"""Include parallel subtest costs in gotestsum JUnit timings used for sharding.""" + +import sys +import xml.etree.ElementTree as ET + + +def aggregate_timings(tree): + for suite in tree.iter("testsuite"): + cases = {case.attrib["name"]: case for case in suite.findall("testcase")} + elapsed = {name: float(case.get("time", "0")) for name, case in cases.items()} + names = set() + for name in cases: + parts = name.split("/") + names.update("/".join(parts[:i]) for i in range(1, len(parts) + 1)) + + children = {} + for name in sorted(names, key=lambda name: name.count("/"), reverse=True): + # Go excludes time spent waiting for parallel subtests from a + # parent's elapsed time. Serial subtests are already included, so + # take the larger value rather than adding the parent's time. + duration = max(elapsed.get(name, 0), children.get(name, 0)) + if name in cases and elapsed[name] >= 0 and duration > elapsed[name]: + cases[name].set("time", f"{duration:.6f}") + if "/" in name: + parent = name.rsplit("/", 1)[0] + children[parent] = children.get(parent, 0) + duration + + +if __name__ == "__main__": + tree = ET.parse(sys.argv[1]) + aggregate_timings(tree) + tree.write(sys.argv[1], encoding="utf-8", xml_declaration=True) diff --git a/ops/scripts/shard-tests.sh b/ops/scripts/shard-tests.sh index 0c83ec89c34..26d32ea980a 100755 --- a/ops/scripts/shard-tests.sh +++ b/ops/scripts/shard-tests.sh @@ -30,4 +30,7 @@ if [ -z "${CIRCLE_NODE_TOTAL:-}" ] || [ "${CIRCLE_NODE_TOTAL:-1}" -le 1 ]; then exit 0 fi -circleci tests split --split-by=timings --timings-type=testname "$ALL_TESTS_FILE" +# Canceled runs can leave partial history. CircleCI's tiny default weight for +# missing timings can put every unknown test on the same otherwise-light shard. +# Give unknown acceptance tests a realistic setup/runtime budget instead. +circleci tests split --split-by=timings --timings-type=testname --time-default=1m "$ALL_TESTS_FILE" diff --git a/ops/scripts/test-acceptance-timings.py b/ops/scripts/test-acceptance-timings.py new file mode 100644 index 00000000000..333bdc54d60 --- /dev/null +++ b/ops/scripts/test-acceptance-timings.py @@ -0,0 +1,162 @@ +#!/usr/bin/env python3 +"""Regression checks for acceptance-test sharding and its JUnit timing history.""" + +import importlib.util +import json +import os +from pathlib import Path +import subprocess +import sys +import tempfile +import unittest +import xml.etree.ElementTree as ET + +SCRIPT_DIR = Path(__file__).resolve().parent +spec = importlib.util.spec_from_file_location("junit_shard_timings", SCRIPT_DIR / "junit-shard-timings.py") +timings = importlib.util.module_from_spec(spec) +spec.loader.exec_module(timings) + + +class JUnitTimingsTest(unittest.TestCase): + def normalize(self, xml): + tree = ET.fromstring(xml) + timings.aggregate_timings(tree) + return tree + + def test_parallel_subtests_contribute_to_top_level_timing(self): + # These are the zero parent and real subtest timings from canceled job + # 5717816, whose partial history overloaded the next run's ZK shard. + tree = self.normalize(''' + + + + ''') + self.assertAlmostEqual(float(tree.find(".//testcase").get("time")), 113.46) + + def test_nested_subtests_are_not_counted_twice(self): + tree = self.normalize(''' + + + + + + ''') + cases = {c.get("name"): float(c.get("time")) for c in tree.iter("testcase")} + self.assertEqual(cases["TestRoot"], 35) + self.assertEqual(cases["TestRoot/nested"], 30) + before = ET.tostring(tree) + timings.aggregate_timings(tree) + self.assertEqual(ET.tostring(tree), before) + + def test_serial_parent_keeps_its_inclusive_elapsed_time(self): + tree = self.normalize(''' + + + ''') + self.assertEqual(tree.find(".//testcase").get("time"), "100") + + def test_missing_intermediate_results_and_distinct_packages(self): + tree = self.normalize(''' + + + + + + + + + ''') + self.assertEqual([float(s.find("testcase").get("time")) for s in tree], [30, 10]) + + def test_cli_preserves_outcomes_and_unknown_failure_duration(self): + xml = ''' + details + logs + + ''' + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "results.xml" + path.write_text(xml) + subprocess.run([sys.executable, str(SCRIPT_DIR / "junit-shard-timings.py"), str(path)], check=True) + tree = ET.parse(path) + suite = tree.find("testsuite") + self.assertEqual(suite.attrib, {"name": "zk", "tests": "3", "failures": "1", "time": "120"}) + root, child, aborted = suite.findall("testcase") + self.assertEqual(float(root.get("time")), 30) + self.assertEqual(root.find("failure").text, "details") + self.assertEqual(child.find("system-out").text, "logs") + self.assertEqual(aborted.get("time"), "-1") + self.assertEqual(aborted.find("skipped").get("message"), "aborted") + + +class ShardTestsTest(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.directory = Path(self.temp.name) + # A test double for CircleCI's weighted splitter. Unequal known shard + # loads reproduce why tiny missing weights all fit on one light shard. + # The real CircleCI agent is available only inside CircleCI jobs. + circleci = self.directory / "circleci" + circleci.write_text(f'''#!{sys.executable} +import json, os, sys +args = sys.argv[1:] +assert args[:2] == ["tests", "split"] +assert "--split-by=timings" in args and "--timings-type=testname" in args +weight = 0.0001 +for arg in args: + if arg.startswith("--time-default="): + value = arg.split("=", 1)[1] + weight = float(value[:-1]) * {{"s": 1, "m": 60}}[value[-1]] +history = json.loads(os.environ["TEST_TIMINGS"]) +names = open(args[-1]).read().splitlines() +loads = [0] * int(os.environ["CIRCLE_NODE_TOTAL"]) +shards = [[] for _ in loads] +for name in sorted(names, key=lambda name: (-history.get(name, weight), name)): + node = min(range(len(loads)), key=loads.__getitem__) + loads[node] += history.get(name, weight) + shards[node].append(name) +print("\\n".join(shards[int(os.environ["CIRCLE_NODE_INDEX"])])) +''') + circleci.chmod(0o755) + + def run_sharder(self, names, node=0, total=8, history=None): + env = dict(os.environ, PATH=f"{self.directory}{os.pathsep}{os.environ['PATH']}", + CIRCLE_NODE_TOTAL=str(total), CIRCLE_NODE_INDEX=str(node), + TEST_TIMINGS=json.dumps(history or {})) + # Include the Go package summary and other non-test lines emitted by -list. + listing = "\n".join(names + ["ok example.com/tests 0.001s", "? example.com/empty [no test files]"]) + return subprocess.run(["bash", str(SCRIPT_DIR / "shard-tests.sh")], input=listing, + text=True, capture_output=True, env=env) + + def test_partial_canceled_run_history_does_not_concentrate_unknown_tests(self): + known = {f"TestKnown{i}": 1000 + i * 10 for i in range(8)} + unknown = [f"TestUnknown{i:02}" for i in range(64)] + names = list(known) + unknown + shards = [] + for node in range(8): + result = self.run_sharder(names, node=node, history=known) + self.assertEqual(result.returncode, 0, result.stderr) + shards.append(result.stdout.splitlines()) + self.assertCountEqual([name for shard in shards for name in shard], names) + self.assertLessEqual(max(len(set(shard) & set(unknown)) for shard in shards), 9) + + def test_empty_history_covers_every_test_once(self): + names = [f"TestNew{i:02}" for i in range(64)] + shards = [self.run_sharder(names, node=node).stdout.splitlines() for node in range(8)] + self.assertCountEqual([name for shard in shards for name in shard], names) + self.assertTrue(all(len(shard) == 8 for shard in shards)) + + def test_single_node_passes_all_test_names_through(self): + result = self.run_sharder(["TestOne", "TestTwo"], total=1) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(result.stdout.splitlines(), ["TestOne", "TestTwo"]) + + def test_empty_discovery_fails_instead_of_skipping_the_suite(self): + result = self.run_sharder([]) + self.assertNotEqual(result.returncode, 0) + self.assertIn("no tests to shard", result.stderr) + + +if __name__ == "__main__": + unittest.main() From 695d4c938005579618578ef5cdea8a994aa1c4d1 Mon Sep 17 00:00:00 2001 From: Josh Klopfenstein Date: Fri, 16 Oct 2026 13:39:18 -0700 Subject: [PATCH 2/2] ci: limit the shard fix to missing timing estimates --- .circleci/continue/main.yml | 3 - docs/ai/ci-ops.md | 14 --- op-acceptance-tests/justfile | 3 - ops/scripts/junit-shard-timings.py | 33 ----- ops/scripts/test-acceptance-timings.py | 162 ------------------------- 5 files changed, 215 deletions(-) delete mode 100644 ops/scripts/junit-shard-timings.py delete mode 100644 ops/scripts/test-acceptance-timings.py diff --git a/.circleci/continue/main.yml b/.circleci/continue/main.yml index d6160a8241f..f36f8e70534 100644 --- a/.circleci/continue/main.yml +++ b/.circleci/continue/main.yml @@ -2049,9 +2049,6 @@ jobs: - utils/checkout-with-mise: checkout-method: blobless enable-mise-cache: true - - run: - name: Test acceptance test timing helpers - command: python3 ops/scripts/test-acceptance-timings.py - apt-install: extra_flags: "--no-install-recommends" packages: "eatmydata" diff --git a/docs/ai/ci-ops.md b/docs/ai/ci-ops.md index fdbcfa9afb9..b2459f883dd 100644 --- a/docs/ai/ci-ops.md +++ b/docs/ai/ci-ops.md @@ -63,20 +63,6 @@ branch that only added a new `op-core/types` package. The diff touched nothing i `op-deployer`; the failures were a data-dependent flake already fixed on `develop` by #21396. Rebasing made CI green with no change to the branch's own work. -## Acceptance package timeouts with imbalanced shards - -A late-starting acceptance test can fail during setup because earlier tests exhausted -its Go package's shared timeout. Compare raw Go `start`/`cont` events with the -package deadline: `devtest.SerialT` cancels its context three seconds before that -deadline. The assertion that happens to see cancellation need not be the slow test. - -Check the shard logs for `No timing found` warnings and the `Downloading previous -test results` step for the source job. A canceled source run can supply partial -history: in [#23471](https://github.com/ethereum-optimism/optimism/pull/23471), all 64 -unknown tests landed on one node and exhausted the ZK package timeout. Keep a -meaningful missing-timing estimate in `ops/scripts/shard-tests.sh`, and include -parallel subtest costs in the top-level timings uploaded for the next split. - ## Missing `op-core/superchain` bundle (`superchain-configs.zip`) A CI-only compile error like: diff --git a/op-acceptance-tests/justfile b/op-acceptance-tests/justfile index 9deb3b238fb..e048d226f27 100644 --- a/op-acceptance-tests/justfile +++ b/op-acceptance-tests/justfile @@ -163,9 +163,6 @@ acceptance-test: set -e cp "$LOG_DIR/all.log" "$LOG_DIR/summary.log" - # The splitter lists top-level tests, whose Go elapsed time can exclude - # parallel subtests. Include their cost in the history uploaded to CircleCI. - python3 "{{REPO_ROOT}}/ops/scripts/junit-shard-timings.py" "$JUNIT_FILE" exit "$TEST_STATUS" clean: diff --git a/ops/scripts/junit-shard-timings.py b/ops/scripts/junit-shard-timings.py deleted file mode 100644 index 69e4518fa9f..00000000000 --- a/ops/scripts/junit-shard-timings.py +++ /dev/null @@ -1,33 +0,0 @@ -#!/usr/bin/env python3 -"""Include parallel subtest costs in gotestsum JUnit timings used for sharding.""" - -import sys -import xml.etree.ElementTree as ET - - -def aggregate_timings(tree): - for suite in tree.iter("testsuite"): - cases = {case.attrib["name"]: case for case in suite.findall("testcase")} - elapsed = {name: float(case.get("time", "0")) for name, case in cases.items()} - names = set() - for name in cases: - parts = name.split("/") - names.update("/".join(parts[:i]) for i in range(1, len(parts) + 1)) - - children = {} - for name in sorted(names, key=lambda name: name.count("/"), reverse=True): - # Go excludes time spent waiting for parallel subtests from a - # parent's elapsed time. Serial subtests are already included, so - # take the larger value rather than adding the parent's time. - duration = max(elapsed.get(name, 0), children.get(name, 0)) - if name in cases and elapsed[name] >= 0 and duration > elapsed[name]: - cases[name].set("time", f"{duration:.6f}") - if "/" in name: - parent = name.rsplit("/", 1)[0] - children[parent] = children.get(parent, 0) + duration - - -if __name__ == "__main__": - tree = ET.parse(sys.argv[1]) - aggregate_timings(tree) - tree.write(sys.argv[1], encoding="utf-8", xml_declaration=True) diff --git a/ops/scripts/test-acceptance-timings.py b/ops/scripts/test-acceptance-timings.py deleted file mode 100644 index 333bdc54d60..00000000000 --- a/ops/scripts/test-acceptance-timings.py +++ /dev/null @@ -1,162 +0,0 @@ -#!/usr/bin/env python3 -"""Regression checks for acceptance-test sharding and its JUnit timing history.""" - -import importlib.util -import json -import os -from pathlib import Path -import subprocess -import sys -import tempfile -import unittest -import xml.etree.ElementTree as ET - -SCRIPT_DIR = Path(__file__).resolve().parent -spec = importlib.util.spec_from_file_location("junit_shard_timings", SCRIPT_DIR / "junit-shard-timings.py") -timings = importlib.util.module_from_spec(spec) -spec.loader.exec_module(timings) - - -class JUnitTimingsTest(unittest.TestCase): - def normalize(self, xml): - tree = ET.fromstring(xml) - timings.aggregate_timings(tree) - return tree - - def test_parallel_subtests_contribute_to_top_level_timing(self): - # These are the zero parent and real subtest timings from canceled job - # 5717816, whose partial history overloaded the next run's ZK shard. - tree = self.normalize(''' - - - - ''') - self.assertAlmostEqual(float(tree.find(".//testcase").get("time")), 113.46) - - def test_nested_subtests_are_not_counted_twice(self): - tree = self.normalize(''' - - - - - - ''') - cases = {c.get("name"): float(c.get("time")) for c in tree.iter("testcase")} - self.assertEqual(cases["TestRoot"], 35) - self.assertEqual(cases["TestRoot/nested"], 30) - before = ET.tostring(tree) - timings.aggregate_timings(tree) - self.assertEqual(ET.tostring(tree), before) - - def test_serial_parent_keeps_its_inclusive_elapsed_time(self): - tree = self.normalize(''' - - - ''') - self.assertEqual(tree.find(".//testcase").get("time"), "100") - - def test_missing_intermediate_results_and_distinct_packages(self): - tree = self.normalize(''' - - - - - - - - - ''') - self.assertEqual([float(s.find("testcase").get("time")) for s in tree], [30, 10]) - - def test_cli_preserves_outcomes_and_unknown_failure_duration(self): - xml = ''' - details - logs - - ''' - with tempfile.TemporaryDirectory() as directory: - path = Path(directory) / "results.xml" - path.write_text(xml) - subprocess.run([sys.executable, str(SCRIPT_DIR / "junit-shard-timings.py"), str(path)], check=True) - tree = ET.parse(path) - suite = tree.find("testsuite") - self.assertEqual(suite.attrib, {"name": "zk", "tests": "3", "failures": "1", "time": "120"}) - root, child, aborted = suite.findall("testcase") - self.assertEqual(float(root.get("time")), 30) - self.assertEqual(root.find("failure").text, "details") - self.assertEqual(child.find("system-out").text, "logs") - self.assertEqual(aborted.get("time"), "-1") - self.assertEqual(aborted.find("skipped").get("message"), "aborted") - - -class ShardTestsTest(unittest.TestCase): - def setUp(self): - self.temp = tempfile.TemporaryDirectory() - self.addCleanup(self.temp.cleanup) - self.directory = Path(self.temp.name) - # A test double for CircleCI's weighted splitter. Unequal known shard - # loads reproduce why tiny missing weights all fit on one light shard. - # The real CircleCI agent is available only inside CircleCI jobs. - circleci = self.directory / "circleci" - circleci.write_text(f'''#!{sys.executable} -import json, os, sys -args = sys.argv[1:] -assert args[:2] == ["tests", "split"] -assert "--split-by=timings" in args and "--timings-type=testname" in args -weight = 0.0001 -for arg in args: - if arg.startswith("--time-default="): - value = arg.split("=", 1)[1] - weight = float(value[:-1]) * {{"s": 1, "m": 60}}[value[-1]] -history = json.loads(os.environ["TEST_TIMINGS"]) -names = open(args[-1]).read().splitlines() -loads = [0] * int(os.environ["CIRCLE_NODE_TOTAL"]) -shards = [[] for _ in loads] -for name in sorted(names, key=lambda name: (-history.get(name, weight), name)): - node = min(range(len(loads)), key=loads.__getitem__) - loads[node] += history.get(name, weight) - shards[node].append(name) -print("\\n".join(shards[int(os.environ["CIRCLE_NODE_INDEX"])])) -''') - circleci.chmod(0o755) - - def run_sharder(self, names, node=0, total=8, history=None): - env = dict(os.environ, PATH=f"{self.directory}{os.pathsep}{os.environ['PATH']}", - CIRCLE_NODE_TOTAL=str(total), CIRCLE_NODE_INDEX=str(node), - TEST_TIMINGS=json.dumps(history or {})) - # Include the Go package summary and other non-test lines emitted by -list. - listing = "\n".join(names + ["ok example.com/tests 0.001s", "? example.com/empty [no test files]"]) - return subprocess.run(["bash", str(SCRIPT_DIR / "shard-tests.sh")], input=listing, - text=True, capture_output=True, env=env) - - def test_partial_canceled_run_history_does_not_concentrate_unknown_tests(self): - known = {f"TestKnown{i}": 1000 + i * 10 for i in range(8)} - unknown = [f"TestUnknown{i:02}" for i in range(64)] - names = list(known) + unknown - shards = [] - for node in range(8): - result = self.run_sharder(names, node=node, history=known) - self.assertEqual(result.returncode, 0, result.stderr) - shards.append(result.stdout.splitlines()) - self.assertCountEqual([name for shard in shards for name in shard], names) - self.assertLessEqual(max(len(set(shard) & set(unknown)) for shard in shards), 9) - - def test_empty_history_covers_every_test_once(self): - names = [f"TestNew{i:02}" for i in range(64)] - shards = [self.run_sharder(names, node=node).stdout.splitlines() for node in range(8)] - self.assertCountEqual([name for shard in shards for name in shard], names) - self.assertTrue(all(len(shard) == 8 for shard in shards)) - - def test_single_node_passes_all_test_names_through(self): - result = self.run_sharder(["TestOne", "TestTwo"], total=1) - self.assertEqual(result.returncode, 0, result.stderr) - self.assertEqual(result.stdout.splitlines(), ["TestOne", "TestTwo"]) - - def test_empty_discovery_fails_instead_of_skipping_the_suite(self): - result = self.run_sharder([]) - self.assertNotEqual(result.returncode, 0) - self.assertIn("no tests to shard", result.stderr) - - -if __name__ == "__main__": - unittest.main()