From fa7b23acf10a6a3206c0fa220bbcf51296050301 Mon Sep 17 00:00:00 2001 From: Marcus Kainth Date: Fri, 25 Sep 2026 13:58:05 +0100 Subject: [PATCH] ci: pack the native suites into four groups The simulation groups were packed when a single test took 14 to 21 minutes. Running every arm of a suite in one session and cutting the tic statement's analysis brought the slowest test down to 227 s, and six simulation groups now finish their suites in 78 to 261 s each on the six main runs 36126756550 to 36132815983, while emulator takes 210 to 246 s. Each group is a runner, and the runners queue against the account's concurrent-job limit. The native crate's suites are repacked into four groups from each test's worst time across those six runs. Each group is modelled as longest test first over four slots, and suites are placed and swapped until the worst group stops falling. The crate's non-simulation suites stay in the catch-all group, native-sim-d, because the lettered lists name only simulation suites. Worst-time model per group: a 253 s, b 251 s, c 252 s, d 253 s (medians: 223, 219, 224, 232 s). The same model reproduced the observed suites step of every group in those runs within 4 to 17%. Five groups would model at 227 s, the length of sim_plat_live's longest test, for one more runner. The timeouts are re-derived from the same runs: - nextest's cutoff drops from 2,400 s to 720 s (120 s x 6). The slowest test's worst was 227 s, and the cutoff sits above FIRST_TIC_TIMEOUT's 600 s plus a load, so a first tic that never comes fails with the session's own message. - The test jobs' timeout-minutes drops from 45 to 15: about twice the slowest test job seen (7.1 minutes), and above nextest's 12-minute cutoff. --- .config/nextest.toml | 19 +++++----- .github/workflows/ci.yml | 25 ++++++------- scripts/test-group.sh | 78 ++++++++++++++++------------------------ 3 files changed, 50 insertions(+), 72 deletions(-) diff --git a/.config/nextest.toml b/.config/nextest.toml index 402e2e5..a3457c7 100644 --- a/.config/nextest.toml +++ b/.config/nextest.toml @@ -4,13 +4,12 @@ # at once share nothing but the server. [profile.default] fail-fast = false -# A live suite's first session analyses a statement that takes a few minutes -# on a CI runner, so a test is slow only past this, and one that opens more -# than one session runs well past it before it is cut off. The cutoff is -# 2,400 s, about 1.8x the slowest test on record: sim_missile_live's own -# single test, 1,234 s at the median of five main runs and 1,347 s at its -# worst. A test's own time moves up to 35% run to run, so the margin is -# sized against the worst rather than the median. A test past the cutoff is -# killed, and the failure reads as a timeout rather than as anything about -# the test, which is why this is not tight. -slow-timeout = { period = "300s", terminate-after = 8 } +# A test is reported slow every 120 s and killed at 720 s. The slowest test +# in the six main runs 36126756550 to 36132815983 is sim_plat_live's +# a_crossing_spawns_the_plat_ev_do_plat_spawns_for_it, 215 s at the median +# and 227 s at its worst, so the cutoff is about 3x the worst. It sits above +# FIRST_TIC_TIMEOUT (600 s, native/src/resident/stream.rs) plus a load, so +# a first tic that never arrives fails with the session's own message +# rather than with nextest's. A test killed here reads as a timeout and +# says nothing about what the test checks. +slow-timeout = { period = "120s", terminate-after = 6 } diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ad6a125..fa286a5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -188,18 +188,13 @@ jobs: runs-on: ${{ vars.CLICKDOOM_TEST_RUNNER || 'ubuntu-latest' }} permissions: contents: read - # A backstop against a job that hangs, not a budget. Packed over six - # groups every group's own tests schedule in 19.8 to 20.6 minutes on - # the median of five main runs, and a job runs 3.4 minutes longer than - # its tests at the median, 8.8 at the worst seen. That says nothing - # about the worst a runner produces: a test's own time moves 17% run to - # run at the median and 35% at the extreme, a slow runner has taken a - # whole group 30% past those medians, and the slowest group on record - # ran 41 minutes. So this sits above the observed worst rather than - # above the modelled one. A group that is merely slow has to finish and - # report; cancelling it reads as a timeout and says nothing about any - # test. - timeout-minutes: 45 + # A backstop against a job that hangs. The slowest test job in the six + # main runs 36126756550 to 36132815983 took 7.1 minutes from its start + # to its end, and this is about twice that. It also sits above + # nextest's cutoff in .config/nextest.toml, 12 minutes, so a test that + # hangs is killed by nextest and reported by name before the job is + # cancelled. + timeout-minutes: 15 env: TEST_THREADS: ${{ vars.CLICKDOOM_TEST_THREADS || '4' }} strategy: @@ -258,13 +253,13 @@ jobs: permissions: contents: read # The same backstop as `test`. - timeout-minutes: 45 + timeout-minutes: 15 env: TEST_THREADS: ${{ vars.CLICKDOOM_TEST_THREADS || '4' }} strategy: fail-fast: false matrix: - group: [native-sim-a, native-sim-b, native-sim-c, native-sim-d, native-sim-e, native-sim-f] + group: [native-sim-a, native-sim-b, native-sim-c, native-sim-d] name: test (${{ matrix.group }}) steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -322,7 +317,7 @@ jobs: # against the hash the probe recorded beside that frame's row. # # The renderer's pixels are pinned by `native/tests/render_live.rs` in - # the native-sim-e group. This job checks the load and render path around + # the native-sim-d group. This job checks the load and render path around # them and the fixture's own metadata. No ROM is built: the target's # prerequisites are `up` and `build-clickdoom`. runs-on: ubuntu-latest diff --git a/scripts/test-group.sh b/scripts/test-group.sh index 38d7363..c7bd3e6 100755 --- a/scripts/test-group.sh +++ b/scripts/test-group.sh @@ -17,21 +17,18 @@ # emulator and the driver's emulation side, then the ROM # suites that need a release build. The driver's # connection and stream suites run here too -# native-sim-a the compact, missile, missile-wall, player-frames and -# refire suites -# native-sim-b the door, missile-kill-drop, move and player-damage -# suites -# native-sim-c the gunshot-kill-drop, pain, punch, shot, thinker- -# order and troop suites -# native-sim-d the floor, hearing, justattacked, lights, thrust -# and tic suites -# native-sim-e every native crate suite the other native-sim groups -# do not name: the blast, claw, cost, fall, gunshot, -# hitscan, input, kills-and-drops, missile-same-target, -# setup, spawn, throw, traverse and use suites, and the +# native-sim-a the aim, claw, gunshot-kill-drop, hitscan, kills-and- +# drops, missile-kill-drop, missile-wall, plat, removed, +# sight, spawn, throw and tic suites +# native-sim-b the blast, cost, damage, door, impact, noise, player- +# damage, punch, setup and use suites +# native-sim-c the compact, fall, floor, input, justattacked, missile- +# same-target, move, pain, player-frames, refire, traverse +# and troop suites +# native-sim-d every native crate suite the other native-sim groups +# do not name: the gunshot, hearing, lights, missile, +# parity, shot, thinker-order and thrust suites, and the # crate's loader, renderer, session and table suites -# native-sim-f the aim, damage, impact, noise, parity, plat, -# removed and sight suites # driver-native the driver's native_* suites (load, render, demo, play, # diff and session). The connection and stream suites run # in `emulator`, where nothing runs beside them @@ -43,7 +40,7 @@ set -euo pipefail cd "$(dirname "$0")/.." -groups=(emulator native-sim-a native-sim-b native-sim-c native-sim-d native-sim-e native-sim-f driver-native) +groups=(emulator native-sim-a native-sim-b native-sim-c native-sim-d driver-native) verb=run if [ "${1-}" = --list ]; then @@ -99,26 +96,23 @@ else fi # The simulation suites the lettered groups name, one per line so a move -# between groups is one line of diff. native-sim-e is every native crate +# between groups is one line of diff. native-sim-d is every native crate # suite not listed here, so a suite added to a lettered group has to be # added here in the same commit or it runs twice, and a suite added to none -# of them runs in native-sim-e rather than in nothing. +# of them runs in native-sim-d rather than in nothing. # # The packing is over each group's own thread schedule, longest test first -# over TEST_THREADS, taken on the median of every test across five main -# runs. A group's cost is not its total: a test cannot be split, so a group -# holding one very long test costs at least that test however little else it -# carries. Five of the suites here are a single test of 14 to 21 minutes. +# over TEST_THREADS, taken on each test's worst time across six main runs. +# A group's cost is not its total: a test cannot be split, so a group +# holding one very long test costs at least that test however little else +# it carries. # -# native-sim-e, by name: sim_blast_live, sim_claw_live, sim_fall_live, sim_gunshot_live, -# sim_hitscan_live, sim_input_live, sim_kills_and_drops_live, -# sim_missile_same_target_live, sim_setup_live, sim_spawn_live, -# sim_throw_live, sim_traverse_live, sim_use_live. -sim_a='binary(sim_compact_live) | binary(sim_missile_live) | binary(sim_missile_wall_live) | binary(sim_player_frames_live) | binary(sim_refire_live)' -sim_b='binary(sim_door_live) | binary(sim_missile_kill_drop_live) | binary(sim_move_live) | binary(sim_player_damage_live)' -sim_c='binary(sim_gunshot_kill_drop_live) | binary(sim_pain_live) | binary(sim_punch_live) | binary(sim_shot_live) | binary(sim_thinker_order_live) | binary(sim_troop_live)' -sim_d='binary(sim_floor_live) | binary(sim_hearing_live) | binary(sim_justattacked_live) | binary(sim_lights_live) | binary(sim_thrust_live) | binary(sim_tic_live)' -sim_f='binary(sim_aim_live) | binary(sim_damage_live) | binary(sim_impact_live) | binary(sim_noise_live) | binary(sim_parity_live) | binary(sim_plat_live) | binary(sim_removed_live) | binary(sim_sight_live)' +# native-sim-d, by name: sim_gunshot_live, sim_hearing_live, +# sim_lights_live, sim_missile_live, sim_parity_live, sim_shot_live, +# sim_thinker_order_live, sim_thrust_live. +sim_a='binary(sim_aim_live) | binary(sim_claw_live) | binary(sim_gunshot_kill_drop_live) | binary(sim_hitscan_live) | binary(sim_kills_and_drops_live) | binary(sim_missile_kill_drop_live) | binary(sim_missile_wall_live) | binary(sim_plat_live) | binary(sim_removed_live) | binary(sim_sight_live) | binary(sim_spawn_live) | binary(sim_throw_live) | binary(sim_tic_live)' +sim_b='binary(sim_blast_live) | binary(sim_cost_live) | binary(sim_damage_live) | binary(sim_door_live) | binary(sim_impact_live) | binary(sim_noise_live) | binary(sim_player_damage_live) | binary(sim_punch_live) | binary(sim_setup_live) | binary(sim_use_live)' +sim_c='binary(sim_compact_live) | binary(sim_fall_live) | binary(sim_floor_live) | binary(sim_input_live) | binary(sim_justattacked_live) | binary(sim_missile_same_target_live) | binary(sim_move_live) | binary(sim_pain_live) | binary(sim_player_frames_live) | binary(sim_refire_live) | binary(sim_traverse_live) | binary(sim_troop_live)' case "$group" in --check) @@ -163,12 +157,12 @@ case "$group" in run_rom \ -E 'binary(reference_trace) | binary(demo3_parity) | binary(rom_symbols) | binary(probe_fixture)' ;; - # A test that opens a session pays the tic statement's analysis, about - # three minutes on a standard runner, and the analysis runs on one - # thread, so the simulation groups run TEST_THREADS tests at a time - # (four unless set, which a standard runner's four cores fill) and the - # suites that open sessions are spread over the groups by their - # measured length. A runner with more cores sets TEST_THREADS higher. + # A test that opens a session pays the tic statement's analysis, and + # the analysis runs on one thread, so the simulation groups run + # TEST_THREADS tests at a time (four unless set, which a standard + # runner's four cores fill) and the suites that open sessions are spread + # over the groups by their measured length. A runner with more cores + # sets TEST_THREADS higher. native-sim-a) # shellcheck disable=SC2086 run_native $live --test-threads "${TEST_THREADS:-4}" \ @@ -187,17 +181,7 @@ case "$group" in native-sim-d) # shellcheck disable=SC2086 run_native $live --test-threads "${TEST_THREADS:-4}" \ - -E "package(clickdoom-native) and ($sim_d)" - ;; - native-sim-e) - # shellcheck disable=SC2086 - run_native $live --test-threads "${TEST_THREADS:-4}" \ - -E "package(clickdoom-native) and not ($sim_a | $sim_b | $sim_c | $sim_d | $sim_f)" - ;; - native-sim-f) - # shellcheck disable=SC2086 - run_native $live --test-threads "${TEST_THREADS:-4}" \ - -E "package(clickdoom-native) and ($sim_f)" + -E "package(clickdoom-native) and not ($sim_a | $sim_b | $sim_c)" ;; driver-native) # The driver's sessions pay the same analysis, so this runs as many