diff --git a/.config/nextest.toml b/.config/nextest.toml index 402e2e5..a3457c7 100644 --- a/.config/nextest.toml +++ b/.config/nextest.toml @@ -4,13 +4,12 @@ # at once share nothing but the server. [profile.default] fail-fast = false -# A live suite's first session analyses a statement that takes a few minutes -# on a CI runner, so a test is slow only past this, and one that opens more -# than one session runs well past it before it is cut off. The cutoff is -# 2,400 s, about 1.8x the slowest test on record: sim_missile_live's own -# single test, 1,234 s at the median of five main runs and 1,347 s at its -# worst. A test's own time moves up to 35% run to run, so the margin is -# sized against the worst rather than the median. A test past the cutoff is -# killed, and the failure reads as a timeout rather than as anything about -# the test, which is why this is not tight. -slow-timeout = { period = "300s", terminate-after = 8 } +# A test is reported slow every 120 s and killed at 720 s. The slowest test +# in the six main runs 36126756550 to 36132815983 is sim_plat_live's +# a_crossing_spawns_the_plat_ev_do_plat_spawns_for_it, 215 s at the median +# and 227 s at its worst, so the cutoff is about 3x the worst. It sits above +# FIRST_TIC_TIMEOUT (600 s, native/src/resident/stream.rs) plus a load, so +# a first tic that never arrives fails with the session's own message +# rather than with nextest's. A test killed here reads as a timeout and +# says nothing about what the test checks. +slow-timeout = { period = "120s", terminate-after = 6 } diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ad6a125..fa286a5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -188,18 +188,13 @@ jobs: runs-on: ${{ vars.CLICKDOOM_TEST_RUNNER || 'ubuntu-latest' }} permissions: contents: read - # A backstop against a job that hangs, not a budget. Packed over six - # groups every group's own tests schedule in 19.8 to 20.6 minutes on - # the median of five main runs, and a job runs 3.4 minutes longer than - # its tests at the median, 8.8 at the worst seen. That says nothing - # about the worst a runner produces: a test's own time moves 17% run to - # run at the median and 35% at the extreme, a slow runner has taken a - # whole group 30% past those medians, and the slowest group on record - # ran 41 minutes. So this sits above the observed worst rather than - # above the modelled one. A group that is merely slow has to finish and - # report; cancelling it reads as a timeout and says nothing about any - # test. - timeout-minutes: 45 + # A backstop against a job that hangs. The slowest test job in the six + # main runs 36126756550 to 36132815983 took 7.1 minutes from its start + # to its end, and this is about twice that. It also sits above + # nextest's cutoff in .config/nextest.toml, 12 minutes, so a test that + # hangs is killed by nextest and reported by name before the job is + # cancelled. + timeout-minutes: 15 env: TEST_THREADS: ${{ vars.CLICKDOOM_TEST_THREADS || '4' }} strategy: @@ -258,13 +253,13 @@ jobs: permissions: contents: read # The same backstop as `test`. - timeout-minutes: 45 + timeout-minutes: 15 env: TEST_THREADS: ${{ vars.CLICKDOOM_TEST_THREADS || '4' }} strategy: fail-fast: false matrix: - group: [native-sim-a, native-sim-b, native-sim-c, native-sim-d, native-sim-e, native-sim-f] + group: [native-sim-a, native-sim-b, native-sim-c, native-sim-d] name: test (${{ matrix.group }}) steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -322,7 +317,7 @@ jobs: # against the hash the probe recorded beside that frame's row. # # The renderer's pixels are pinned by `native/tests/render_live.rs` in - # the native-sim-e group. This job checks the load and render path around + # the native-sim-d group. This job checks the load and render path around # them and the fixture's own metadata. No ROM is built: the target's # prerequisites are `up` and `build-clickdoom`. runs-on: ubuntu-latest diff --git a/scripts/test-group.sh b/scripts/test-group.sh index 38d7363..c7bd3e6 100755 --- a/scripts/test-group.sh +++ b/scripts/test-group.sh @@ -17,21 +17,18 @@ # emulator and the driver's emulation side, then the ROM # suites that need a release build. The driver's # connection and stream suites run here too -# native-sim-a the compact, missile, missile-wall, player-frames and -# refire suites -# native-sim-b the door, missile-kill-drop, move and player-damage -# suites -# native-sim-c the gunshot-kill-drop, pain, punch, shot, thinker- -# order and troop suites -# native-sim-d the floor, hearing, justattacked, lights, thrust -# and tic suites -# native-sim-e every native crate suite the other native-sim groups -# do not name: the blast, claw, cost, fall, gunshot, -# hitscan, input, kills-and-drops, missile-same-target, -# setup, spawn, throw, traverse and use suites, and the +# native-sim-a the aim, claw, gunshot-kill-drop, hitscan, kills-and- +# drops, missile-kill-drop, missile-wall, plat, removed, +# sight, spawn, throw and tic suites +# native-sim-b the blast, cost, damage, door, impact, noise, player- +# damage, punch, setup and use suites +# native-sim-c the compact, fall, floor, input, justattacked, missile- +# same-target, move, pain, player-frames, refire, traverse +# and troop suites +# native-sim-d every native crate suite the other native-sim groups +# do not name: the gunshot, hearing, lights, missile, +# parity, shot, thinker-order and thrust suites, and the # crate's loader, renderer, session and table suites -# native-sim-f the aim, damage, impact, noise, parity, plat, -# removed and sight suites # driver-native the driver's native_* suites (load, render, demo, play, # diff and session). The connection and stream suites run # in `emulator`, where nothing runs beside them @@ -43,7 +40,7 @@ set -euo pipefail cd "$(dirname "$0")/.." -groups=(emulator native-sim-a native-sim-b native-sim-c native-sim-d native-sim-e native-sim-f driver-native) +groups=(emulator native-sim-a native-sim-b native-sim-c native-sim-d driver-native) verb=run if [ "${1-}" = --list ]; then @@ -99,26 +96,23 @@ else fi # The simulation suites the lettered groups name, one per line so a move -# between groups is one line of diff. native-sim-e is every native crate +# between groups is one line of diff. native-sim-d is every native crate # suite not listed here, so a suite added to a lettered group has to be # added here in the same commit or it runs twice, and a suite added to none -# of them runs in native-sim-e rather than in nothing. +# of them runs in native-sim-d rather than in nothing. # # The packing is over each group's own thread schedule, longest test first -# over TEST_THREADS, taken on the median of every test across five main -# runs. A group's cost is not its total: a test cannot be split, so a group -# holding one very long test costs at least that test however little else it -# carries. Five of the suites here are a single test of 14 to 21 minutes. +# over TEST_THREADS, taken on each test's worst time across six main runs. +# A group's cost is not its total: a test cannot be split, so a group +# holding one very long test costs at least that test however little else +# it carries. # -# native-sim-e, by name: sim_blast_live, sim_claw_live, sim_fall_live, sim_gunshot_live, -# sim_hitscan_live, sim_input_live, sim_kills_and_drops_live, -# sim_missile_same_target_live, sim_setup_live, sim_spawn_live, -# sim_throw_live, sim_traverse_live, sim_use_live. -sim_a='binary(sim_compact_live) | binary(sim_missile_live) | binary(sim_missile_wall_live) | binary(sim_player_frames_live) | binary(sim_refire_live)' -sim_b='binary(sim_door_live) | binary(sim_missile_kill_drop_live) | binary(sim_move_live) | binary(sim_player_damage_live)' -sim_c='binary(sim_gunshot_kill_drop_live) | binary(sim_pain_live) | binary(sim_punch_live) | binary(sim_shot_live) | binary(sim_thinker_order_live) | binary(sim_troop_live)' -sim_d='binary(sim_floor_live) | binary(sim_hearing_live) | binary(sim_justattacked_live) | binary(sim_lights_live) | binary(sim_thrust_live) | binary(sim_tic_live)' -sim_f='binary(sim_aim_live) | binary(sim_damage_live) | binary(sim_impact_live) | binary(sim_noise_live) | binary(sim_parity_live) | binary(sim_plat_live) | binary(sim_removed_live) | binary(sim_sight_live)' +# native-sim-d, by name: sim_gunshot_live, sim_hearing_live, +# sim_lights_live, sim_missile_live, sim_parity_live, sim_shot_live, +# sim_thinker_order_live, sim_thrust_live. +sim_a='binary(sim_aim_live) | binary(sim_claw_live) | binary(sim_gunshot_kill_drop_live) | binary(sim_hitscan_live) | binary(sim_kills_and_drops_live) | binary(sim_missile_kill_drop_live) | binary(sim_missile_wall_live) | binary(sim_plat_live) | binary(sim_removed_live) | binary(sim_sight_live) | binary(sim_spawn_live) | binary(sim_throw_live) | binary(sim_tic_live)' +sim_b='binary(sim_blast_live) | binary(sim_cost_live) | binary(sim_damage_live) | binary(sim_door_live) | binary(sim_impact_live) | binary(sim_noise_live) | binary(sim_player_damage_live) | binary(sim_punch_live) | binary(sim_setup_live) | binary(sim_use_live)' +sim_c='binary(sim_compact_live) | binary(sim_fall_live) | binary(sim_floor_live) | binary(sim_input_live) | binary(sim_justattacked_live) | binary(sim_missile_same_target_live) | binary(sim_move_live) | binary(sim_pain_live) | binary(sim_player_frames_live) | binary(sim_refire_live) | binary(sim_traverse_live) | binary(sim_troop_live)' case "$group" in --check) @@ -163,12 +157,12 @@ case "$group" in run_rom \ -E 'binary(reference_trace) | binary(demo3_parity) | binary(rom_symbols) | binary(probe_fixture)' ;; - # A test that opens a session pays the tic statement's analysis, about - # three minutes on a standard runner, and the analysis runs on one - # thread, so the simulation groups run TEST_THREADS tests at a time - # (four unless set, which a standard runner's four cores fill) and the - # suites that open sessions are spread over the groups by their - # measured length. A runner with more cores sets TEST_THREADS higher. + # A test that opens a session pays the tic statement's analysis, and + # the analysis runs on one thread, so the simulation groups run + # TEST_THREADS tests at a time (four unless set, which a standard + # runner's four cores fill) and the suites that open sessions are spread + # over the groups by their measured length. A runner with more cores + # sets TEST_THREADS higher. native-sim-a) # shellcheck disable=SC2086 run_native $live --test-threads "${TEST_THREADS:-4}" \ @@ -187,17 +181,7 @@ case "$group" in native-sim-d) # shellcheck disable=SC2086 run_native $live --test-threads "${TEST_THREADS:-4}" \ - -E "package(clickdoom-native) and ($sim_d)" - ;; - native-sim-e) - # shellcheck disable=SC2086 - run_native $live --test-threads "${TEST_THREADS:-4}" \ - -E "package(clickdoom-native) and not ($sim_a | $sim_b | $sim_c | $sim_d | $sim_f)" - ;; - native-sim-f) - # shellcheck disable=SC2086 - run_native $live --test-threads "${TEST_THREADS:-4}" \ - -E "package(clickdoom-native) and ($sim_f)" + -E "package(clickdoom-native) and not ($sim_a | $sim_b | $sim_c)" ;; driver-native) # The driver's sessions pay the same analysis, so this runs as many