From 712e4eba584b29077270a66b94cb30d10c639c1a Mon Sep 17 00:00:00 2001 From: Marcus Kainth Date: Fri, 25 Sep 2026 11:30:27 +0100 Subject: [PATCH] ci: split the test archive by crate build-tests made one archive holding every crate's test binaries and one holding the reference emulator's release ROM suites, one after the other, and every test job downloaded both. A simulation group needs only the native crate's suites. build-tests is now a matrix of two archives built side by side: native.tar.zst with the native crate's suites and workspace.tar.zst with every other crate's. The ROM suites archive moves to build-refemu, which already builds the reference emulator in release and is off the path to the test jobs. Each group downloads only what scripts/test-group.sh runs it from: the simulation groups the native archive, the emulator group the workspace and ROM suites archives, and native-rest both, since it runs the native crate's other suites and the driver's native_* suites. native-rest runs them as two nextest invocations, one per archive, at the same two threads. Measured on the throwaway run 36122853256 with a warm cache, no debug info and the native crate's suites at opt-level 0, the native archive built in 48 s (119 MB) and the other crates' in 94 s (72 MB), against 2 m 16 s (206 MB) for the single archive under the same settings. Each group selects the same tests as before: listing every group's filter over the workspace before the split and over the two archives after gives the same test lists for all eight groups, 953 tests in all, none in two groups. --- .github/workflows/ci.yml | 70 ++++++++++++++++++++++++++++++++-------- scripts/test-group.sh | 40 ++++++++++++++--------- 2 files changed, 81 insertions(+), 29 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8b558d4..5043b8e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -95,18 +95,33 @@ jobs: with: persist-credentials: false - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 + - uses: taiki-e/install-action@3f74d7c16a4242f1c95561e98edc25d36adb4375 # v2.87.12 + with: + tool: cargo-nextest@0.9.140 - name: Build the reference emulator run: cargo build --locked --release -p refemu + - name: Archive the reference emulator's ROM suites + # In release, the build they need. The `emulator` group runs them. + run: | + mkdir -p target/nextest + cargo nextest archive --locked --release -p refemu --features rom-tests \ + --archive-file target/nextest/rom-suites.tar.zst - name: Upload refemu artifact uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: refemu path: target/release/refemu + - name: Upload the ROM suites archive + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: rom-suites + path: target/nextest/rom-suites.tar.zst build-tests: - # Every test binary, compiled once into two nextest archives: the - # workspace with the live suites, and the reference emulator's release - # ROM suites. The test jobs run from the archives and compile nothing. + # The workspace's test binaries with the live suites, compiled once into + # two nextest archives, one per matrix entry: the native crate's suites, + # and every other crate's. The test jobs run from the archives and + # compile nothing. runs-on: ubuntu-latest permissions: contents: read @@ -115,30 +130,44 @@ jobs: # its panic message with the file and line; a backtrace names # functions only. Set for the job so rust-cache keys on it. CARGO_PROFILE_TEST_DEBUG: "0" + strategy: + fail-fast: false + matrix: + include: + - archive: native + packages: -p clickdoom-native + - archive: workspace + packages: --workspace --exclude clickdoom-native + name: build-tests (${{ matrix.archive }}) steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 + with: + key: ${{ matrix.archive }} - uses: taiki-e/install-action@3f74d7c16a4242f1c95561e98edc25d36adb4375 # v2.87.12 with: tool: cargo-nextest@0.9.140 - name: Archive the suites + env: + ARCHIVE: ${{ matrix.archive }} + PACKAGES: ${{ matrix.packages }} run: | mkdir -p target/nextest - cargo nextest archive --locked --workspace --features clickhouse-tests \ - --archive-file target/nextest/tests.tar.zst - cargo nextest archive --locked --release -p refemu --features rom-tests \ - --archive-file target/nextest/rom-suites.tar.zst - - name: Upload the archives + # shellcheck disable=SC2086 # $PACKAGES is a list of flags + cargo nextest archive --locked $PACKAGES --features clickhouse-tests \ + --archive-file "target/nextest/$ARCHIVE.tar.zst" + - name: Upload the archive uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: - name: test-archives - path: target/nextest/*.tar.zst + name: ${{ matrix.archive }}-tests + path: target/nextest/${{ matrix.archive }}.tar.zst test: # Every suite, in the groups scripts/test-group.sh names, one runner per - # group with its own ClickHouse, run from the archives build-tests made. + # group with its own ClickHouse, run from the archives build-tests and + # build-refemu made. # Throwaway databases only; never `clickdoom`. `make test` runs the same # suites in one pass locally. needs: [build-tests, build-rom, build-refemu] @@ -176,10 +205,25 @@ jobs: tool: cargo-nextest@0.9.140 - name: Start the pinned ClickHouse run: make up - - name: Download the test archives + # Each group downloads the archives scripts/test-group.sh runs it + # from, and a group that lacks one fails on the missing file. + - name: Download the native crate's suites + if: matrix.group != 'emulator' + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: native-tests + path: target/nextest + - name: Download the other crates' suites + if: matrix.group == 'emulator' || matrix.group == 'native-rest' + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: workspace-tests + path: target/nextest + - name: Download the ROM suites + if: matrix.group == 'emulator' uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: - name: test-archives + name: rom-suites path: target/nextest - name: Download ROM artifact uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 diff --git a/scripts/test-group.sh b/scripts/test-group.sh index 7295497..b6079db 100755 --- a/scripts/test-group.sh +++ b/scripts/test-group.sh @@ -38,25 +38,30 @@ cd "$(dirname "$0")/.." group="${1-}" # With NEXTEST_ARCHIVE_DIR set, the suites come pre-built from -# `cargo nextest archive` (tests.tar.zst for the workspace with the live -# suites, rom-suites.tar.zst for the reference emulator's release ROM -# suites) and nothing is compiled here. Without it, each run builds what it -# needs. +# `cargo nextest archive` and nothing is compiled here: native.tar.zst holds +# the native crate's suites, workspace.tar.zst every other crate's with the +# live suites, and rom-suites.tar.zst the reference emulator's release ROM +# suites. Each group names the archives it runs from. Without it, each run +# builds what it needs. archive="${NEXTEST_ARCHIVE_DIR-}" if [ -n "$archive" ]; then # Extracted over the workspace rather than into a temporary directory: # a test that runs the driver binary reaches it by the path compiled in # at build time, which is the workspace's own target directory. extract=(--workspace-remap . --extract-to . --extract-overwrite) - run() { - cargo nextest run --archive-file "$archive/tests.tar.zst" "${extract[@]}" "$@" + run_native() { + cargo nextest run --archive-file "$archive/native.tar.zst" "${extract[@]}" "$@" + } + run_workspace() { + cargo nextest run --archive-file "$archive/workspace.tar.zst" "${extract[@]}" "$@" } run_rom() { cargo nextest run --archive-file "$archive/rom-suites.tar.zst" "${extract[@]}" "$@" } live="" else - run() { cargo nextest run --locked "$@"; } + run_native() { cargo nextest run --locked "$@"; } + run_workspace() { cargo nextest run --locked "$@"; } run_rom() { cargo nextest run --locked --release -p refemu --features rom-tests "$@"; } live="--workspace --features clickhouse-tests" fi @@ -90,7 +95,7 @@ case "$group" in # them would warm or cool, and the connection suite counts the # server's connections, which a neighbour's session would move. # shellcheck disable=SC2086 # $live is a list of flags or nothing - run $live --test-threads 1 \ + run_workspace $live --test-threads 1 \ -E 'not package(clickdoom-native) and (not binary(/^native_/) or binary(native_connections_live))' # The ROM suites are the reference emulator's, so only it is built # in release. @@ -105,38 +110,41 @@ case "$group" in # measured length. A runner with more cores sets TEST_THREADS higher. native-sim-a) # shellcheck disable=SC2086 - run $live --test-threads "${TEST_THREADS:-4}" \ + run_native $live --test-threads "${TEST_THREADS:-4}" \ -E "package(clickdoom-native) and ($sim_a)" ;; native-sim-b) # shellcheck disable=SC2086 - run $live --test-threads "${TEST_THREADS:-4}" \ + run_native $live --test-threads "${TEST_THREADS:-4}" \ -E "package(clickdoom-native) and ($sim_b)" ;; native-sim-c) # shellcheck disable=SC2086 - run $live --test-threads "${TEST_THREADS:-4}" \ + run_native $live --test-threads "${TEST_THREADS:-4}" \ -E "package(clickdoom-native) and ($sim_c)" ;; native-sim-d) # shellcheck disable=SC2086 - run $live --test-threads "${TEST_THREADS:-4}" \ + run_native $live --test-threads "${TEST_THREADS:-4}" \ -E "package(clickdoom-native) and ($sim_d)" ;; native-sim-e) # shellcheck disable=SC2086 - run $live --test-threads "${TEST_THREADS:-4}" \ + run_native $live --test-threads "${TEST_THREADS:-4}" \ -E "package(clickdoom-native) and binary(/^sim_/) and not ($sim_a | $sim_b | $sim_c | $sim_d | $sim_f)" ;; native-sim-f) # shellcheck disable=SC2086 - run $live --test-threads "${TEST_THREADS:-4}" \ + run_native $live --test-threads "${TEST_THREADS:-4}" \ -E "package(clickdoom-native) and ($sim_f)" ;; native-rest) # shellcheck disable=SC2086 - run $live --test-threads 2 \ - -E '(package(clickdoom-native) and not binary(/^sim_/)) or (package(clickdoom-driver) and binary(/^native_/) and not binary(native_connections_live))' + run_native $live --test-threads 2 \ + -E 'package(clickdoom-native) and not binary(/^sim_/)' + # shellcheck disable=SC2086 + run_workspace $live --test-threads 2 \ + -E 'package(clickdoom-driver) and binary(/^native_/) and not binary(native_connections_live)' ;; *) echo "usage: scripts/test-group.sh emulator|native-sim-a|native-sim-b|native-sim-c|native-sim-d|native-sim-e|native-sim-f|native-rest" >&2