diff --git a/.github/workflows/cutile-rs.yml b/.github/workflows/cutile-rs.yml index 11d0496217..521a54ef34 100644 --- a/.github/workflows/cutile-rs.yml +++ b/.github/workflows/cutile-rs.yml @@ -155,8 +155,11 @@ jobs: command -v tileiras tileiras --help | grep -q "sm_120" + - uses: $/cutile-rs/.github/actions/setup - uses: actions/checkout@v7 - - uses: ./cutile-rs/.github/actions/setup + + - name: Mark checkout as safe + run: git config --global --add safe.directory /__w/cuda-rust/cuda-rust # Cheapest checks first so a formatting or lint slip fails in seconds. - name: Format check @@ -184,6 +187,71 @@ jobs: - name: CPU only tests run: ./scripts/run_cpu_tests.sh + # nextest archives carry test executables, Cargo metadata, and build + # outputs needed to run on another machine without rebuilding Rust. + # Keep both workspaces separate, then bundle them in one downloadable tar. + - uses: taiki-e/install-action@9114bf4d891761788c546334fd37538eae1bf8b3 # v2.87.16 + name: Install cargo-nextest + with: + tool: cargo-nextest + + - name: Archive Tile runtime test packages + run: | + mkdir -p "$RUNNER_TEMP/cutile-gpu-tests" + cargo nextest archive --locked --features experimental-tune \ + --archive-file "$RUNNER_TEMP/cutile-gpu-tests/tile-tests.tar.zst" + + - name: Archive shared-host runtime test packages + working-directory: ${{ github.workspace }} + run: | + cargo nextest archive --locked \ + -p cuda-bindings -p cuda-core -p cuda-async \ + --archive-file "$RUNNER_TEMP/cutile-gpu-tests/host-tests.tar.zst" + + - name: Bundle runtime test archives + run: | + cp "$(command -v cargo-nextest)" "$RUNNER_TEMP/cutile-gpu-tests/cargo-nextest" + git rev-parse HEAD > "$RUNNER_TEMP/cutile-gpu-tests/revision.txt" + cargo nextest --version > "$RUNNER_TEMP/cutile-gpu-tests/nextest-version.txt" + tar -cf "$RUNNER_TEMP/cutile-gpu-tests.tar" \ + -C "$RUNNER_TEMP/cutile-gpu-tests" . + + - name: Upload runtime test packages + uses: actions/upload-artifact@v4 + with: + name: cutile-gpu-tests + path: ${{ runner.temp }}/cutile-gpu-tests.tar + if-no-files-found: error + compression-level: 0 # The nested nextest archives are already compressed. + retention-days: 1 + + gpu-tests: + needs: build + runs-on: linux-amd64-gpu-rtxpro6000-latest-1 + timeout-minutes: 30 + container: + image: nvidia/cuda:13.3.0-devel-ubuntu24.04 + env: + CUDA_TOOLKIT_PATH: /usr/local/cuda + CUTILE_TILEIRAS_PATH: /usr/local/cuda/bin/tileiras + steps: + - uses: actions/checkout@v7 + + - uses: actions/download-artifact@v4 + with: + name: cutile-gpu-tests + path: ${{ runner.temp }}/gpu-tests + + - name: Extract runtime test packages + run: | + tar -xf "$RUNNER_TEMP/gpu-tests/cutile-gpu-tests.tar" -C "$RUNNER_TEMP/gpu-tests" + test "$(cat "$RUNNER_TEMP/gpu-tests/revision.txt")" = "$GITHUB_SHA" + nvidia-smi + "$CUTILE_TILEIRAS_PATH" --version + + - name: GPU runtime tests + run: ./scripts/run_gpu_tests.sh --archive-dir "$RUNNER_TEMP/gpu-tests" + # Breaking API changes must ship with the right version bump: compare each # publishable crate's API against its latest crates.io release and fail the # PR when a break arrives without one (the 0.3.0 -> 0.3.1 lesson). Same diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 93c51116b7..ab909717cc 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -192,6 +192,17 @@ cutile-rs/scripts/run_all.sh # both of the above, then the examples and Run `cutile-rs/scripts/run_all.sh` before marking a Tile pull request ready for review. + +The GPU script also accepts `--archive-dir DIR` to run prebuilt tests without +Cargo or a Rust toolchain. The directory must contain `cargo-nextest`, +`tile-tests.tar.zst`, and `host-tests.tar.zst`, as bundled by CI. This mode runs +tests serially and uses the checkout for source fixtures. +See the archive, bundle, and extract steps in +[the Tile CI workflow](.github/workflows/cutile-rs.yml) for how to produce this +directory. Extract `cutile-gpu-tests.tar` first and use a checkout matching its +`revision.txt`; `cargo-nextest` must be executable. The script's opening comments +describe the directory layout and the contents of each test archive. + CI runs the same suites in `.github/workflows/cutile-rs.yml`. Because the Tile lanes use NVIDIA self-hosted runners, that workflow triggers on the `pull-request/` branches that copy-pr-bot mirrors rather than on the pull diff --git a/cutile-rs/.github/actions/setup/action.yml b/cutile-rs/.github/actions/setup/action.yml index 004f7e0fdc..0706d39695 100644 --- a/cutile-rs/.github/actions/setup/action.yml +++ b/cutile-rs/.github/actions/setup/action.yml @@ -19,8 +19,8 @@ runs: using: composite steps: - name: Validate inputs + working-directory: ${{ github.workspace }} shell: bash - working-directory: cutile-rs env: RUST_COMPONENTS: ${{ inputs.components }} RUST_CACHE: ${{ inputs.cache }} @@ -35,8 +35,8 @@ runs: fi - name: Install build dependencies + working-directory: ${{ github.workspace }} shell: bash - working-directory: cutile-rs run: | apt-get update -qq apt-get install -y -qq --no-install-recommends build-essential curl \ @@ -47,7 +47,8 @@ runs: # nightly lives in cuda-oxide/rust-toolchain.toml. - name: Install Rust shell: bash - working-directory: cutile-rs + # relative path to land at the rust-toolchain.toml regardless of use of $ or ./ syntax + working-directory: ${{ github.action_path }}/../../.. env: RUST_COMPONENTS: ${{ inputs.components }} run: | diff --git a/cutile-rs/scripts/run_gpu_tests.sh b/cutile-rs/scripts/run_gpu_tests.sh index b40b2254a4..cb94864231 100755 --- a/cutile-rs/scripts/run_gpu_tests.sh +++ b/cutile-rs/scripts/run_gpu_tests.sh @@ -7,6 +7,154 @@ set -u source "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/test_runner_common.sh" +# With no arguments, build and run locally. Archive mode needs only the bundled +# cargo-nextest executable, the two archives, and this checkout for fixtures. +# After extracting CI's cutile-gpu-tests.tar, --archive-dir points to: +# DIR/cargo-nextest executable used to create the archives +# DIR/tile-tests.tar.zst Tile workspace, with experimental-tune enabled +# DIR/host-tests.tar.zst cuda-bindings, cuda-core, and cuda-async workspace tests +# CI also bundles revision.txt and nextest-version.txt; the workflow checks the +# revision against the checkout before invoking this script. See the archive, +# bundle, and extract steps in .github/workflows/cutile-rs.yml at the git root. +# Both archives use nextest's standard format, including target/nextest metadata. +usage() { + echo "Usage: $0 [--archive-dir DIR]" +} + +ARCHIVE_DIR="" +case "$#" in + 0) ;; + 1) + if [[ "$1" == --help || "$1" == -h ]]; then + usage + exit 0 + fi + usage >&2 + exit 2 + ;; + 2) + if [[ "$1" != --archive-dir || ! -d "$2" ]]; then + usage >&2 + exit 2 + fi + ARCHIVE_DIR="$(cd "$2" && pwd)" + for archive in tile-tests.tar.zst host-tests.tar.zst; do + if [[ ! -f "$ARCHIVE_DIR/$archive" ]]; then + echo "Missing test archive: $ARCHIVE_DIR/$archive" >&2 + exit 2 + fi + done + if [[ ! -x "$ARCHIVE_DIR/cargo-nextest" ]]; then + echo "Missing executable: $ARCHIVE_DIR/cargo-nextest" >&2 + exit 2 + fi + ;; + *) usage >&2; exit 2 ;; +esac + +if [[ -n "$ARCHIVE_DIR" ]]; then + # Extract each workspace once, rather than unpacking it for every test target. + EXTRACT_DIR="$(mktemp -d)" || exit 1 + trap 'rm -rf "$EXTRACT_DIR"' EXIT + for workspace in tile host; do + mkdir -p "$EXTRACT_DIR/$workspace" || exit 1 + workspace_root="$REPO_ROOT" + if [[ "$workspace" == host ]]; then + workspace_root="$GIT_ROOT" + fi + "$ARCHIVE_DIR/cargo-nextest" nextest list --list-type binaries-only \ + --archive-file "$ARCHIVE_DIR/$workspace-tests.tar.zst" \ + --extract-to "$EXTRACT_DIR/$workspace" \ + --workspace-remap "$workspace_root" > /dev/null || exit 1 + done +fi + +# Keep suite selection below independent of the runner. Target is an integration +# test name, "lib" for unit tests, or "all" for the whole package. +# shellcheck disable=SC2329 # Invoked through run_gpu_step/run_step. +gpu_test() ( + local workspace="$1" package="$2" target="$3" + shift 3 + local workspace_root="$REPO_ROOT" + if [[ "$workspace" == host ]]; then + workspace_root="$GIT_ROOT" + fi + cd "$workspace_root" || return 1 + + local expression="package(=$package)" + local cargo_args=(test -p "$package") + local libtest_args=() + local nextest_args=(--test-threads 1 --no-fail-fast) + case "$target" in + lib) cargo_args+=(--lib); expression+=" & kind(lib)" ;; + all) ;; + *) cargo_args+=(--test "$target"); expression+=" & binary(=$target)" ;; + esac + while [[ $# -gt 0 ]]; do + case "$1" in + --filter) + cargo_args+=("$2") + expression+=" & test($2)" + shift 2 ;; + --skip) + libtest_args+=(--skip "$2") + expression+=" & not test($2)" + shift 2 ;; + --ignored) + libtest_args+=(--ignored) + nextest_args+=(--run-ignored only) + shift ;; + --serial) + libtest_args+=(--test-threads=1) + shift ;; + --features) + cargo_args+=(--features "$2") + shift 2 ;; + *) echo "Unknown GPU test option: $1" >&2; return 2 ;; + esac + done + + if [[ -n "$ARCHIVE_DIR" ]]; then + # Serialize GPU tests so unrelated processes cannot compete for resources. + local target_dir="$EXTRACT_DIR/$workspace/target" + "$ARCHIVE_DIR/cargo-nextest" nextest run \ + --cargo-metadata "$target_dir/nextest/cargo-metadata.json" \ + --binaries-metadata "$target_dir/nextest/binaries-metadata.json" \ + --target-dir-remap "$target_dir" \ + --workspace-remap "$workspace_root" "${nextest_args[@]}" -E "$expression" + else + if [[ ${#libtest_args[@]} -gt 0 ]]; then + cargo_args+=(-- "${libtest_args[@]}") + fi + cargo "${cargo_args[@]}" + fi +) + +# shellcheck disable=SC2329 # Invoked through run_gpu_step/run_step. +submission_test() ( + if [[ "$1" == default ]]; then + # Local runs honor developer overrides; CI exercises the actual default. + if [[ -n "$ARCHIVE_DIR" ]]; then + unset CUDA_ASYNC_SPIN_BUDGET_US + fi + else + export CUDA_ASYNC_SPIN_BUDGET_US="$1" + fi + gpu_test tile cutile submission_lifetimes --serial +) + +run_gpu_step() { + if [[ "${GITHUB_ACTIONS:-}" == true ]]; then + echo "::group::$1" + fi + local status=0 + run_step "$@" || status=$? + if [[ "${GITHUB_ACTIONS:-}" == true ]]; then + echo "::endgroup::" + fi + return "$status" +} + print_header "Running GPU tests" for test_target in \ @@ -20,34 +168,34 @@ for test_target in \ tensor_reinterpret \ tensor_views do - run_step \ + run_gpu_step \ "cutile GPU integration test ${test_target}" \ - cargo test -p cutile --features experimental-tune --test "$test_target" + gpu_test tile cutile "$test_target" --features experimental-tune done -run_step \ +run_gpu_step \ "cutile GPU integration test control_flow_ops runtime cases" \ - cargo test -p cutile --test control_flow_ops -- --skip compile_ + gpu_test tile cutile control_flow_ops --skip compile_ -run_step \ +run_gpu_step \ "cutile GPU integration test tensor_and_matrix_ops runtime cases" \ - cargo test -p cutile --test tensor_and_matrix_ops execute_ + gpu_test tile cutile tensor_and_matrix_ops --filter execute_ -run_step \ +run_gpu_step \ "cutile GPU integration test type_conversion_ops runtime cases" \ - cargo test -p cutile --test type_conversion_ops execute_ + gpu_test tile cutile type_conversion_ops --filter execute_ -run_step \ +run_gpu_step \ "cutile GPU integration test specialization_bits runtime cases" \ - cargo test -p cutile --test specialization_bits raw_pointer_launch + gpu_test tile cutile specialization_bits --filter raw_pointer_launch -run_step \ +run_gpu_step \ "cutile GPU warmup and disk-cache tests" \ - cargo test -p cutile --test warmup_suite + gpu_test tile cutile warmup_suite -run_step \ +run_gpu_step \ "cutile GPU aggregate tests" \ - cargo test -p cutile --test gpu + gpu_test tile cutile gpu # Submission-lifetime tests: their own binary, single-threaded. Each test gates # a stream with a blocking host function, and any context-wide synchronize in @@ -59,29 +207,29 @@ run_step \ # in the inline cuStreamQuery spin; 0 forces the reactor path, a large budget # forces the spin path to resolve against the tests' blocking Gates. Under a # Gate, `Pending` is the only correct first-poll outcome on either path. -run_step \ +run_gpu_step \ "cutile GPU submission-lifetime tests" \ - cargo test -p cutile --test submission_lifetimes -- --test-threads=1 + submission_test default -run_step \ +run_gpu_step \ "cutile GPU submission-lifetime tests (reactor path, CUDA_ASYNC_SPIN_BUDGET_US=0)" \ - env CUDA_ASYNC_SPIN_BUDGET_US=0 cargo test -p cutile --test submission_lifetimes -- --test-threads=1 + submission_test 0 -run_step \ +run_gpu_step \ "cutile GPU submission-lifetime tests (spin path, CUDA_ASYNC_SPIN_BUDGET_US=200000)" \ - env CUDA_ASYNC_SPIN_BUDGET_US=200000 cargo test -p cutile --test submission_lifetimes -- --test-threads=1 + submission_test 200000 -run_step \ +run_gpu_step \ "cutile cross-file debug source kernel" \ - cargo test -p cutile --test debug_info kernel_executes_in_all_debug_modes -- --ignored + gpu_test tile cutile debug_info --filter kernel_executes_in_all_debug_modes --ignored -run_step \ +run_gpu_step \ "cuda-core GPU integration test vmm" \ - host_cargo test -p cuda-core --test vmm + gpu_test host cuda-core vmm -run_step \ +run_gpu_step \ "cuda-core GPU integration test vmm_multicast" \ - host_cargo test -p cuda-core --test vmm_multicast + gpu_test host cuda-core vmm_multicast # The SIMT integration tests carried over from cuda-oxide. simt_vmm_p2p # exercises the real two-GPU P2P path (it self-skips on single-device @@ -99,20 +247,20 @@ for test_target in \ simt_vmm_multicast \ simt_vmm_p2p do - run_step \ + run_gpu_step \ "cuda-core GPU integration test ${test_target}" \ - host_cargo test -p cuda-core --test "$test_target" + gpu_test host cuda-core "$test_target" done -run_step \ +run_gpu_step \ "cuda-async unit tests (with live driver)" \ - host_cargo test -p cuda-async --lib + gpu_test host cuda-async lib # Unfiltered: includes the GPU-requiring `cuda_tests` module that the CPU # path skips. -run_step \ +run_gpu_step \ "cuda-bindings tests (with live driver)" \ - host_cargo test -p cuda-bindings + gpu_test host cuda-bindings all for test_target in \ concurrent_capture \ @@ -124,9 +272,9 @@ for test_target in \ pool_allocation \ reactor_correctness do - run_step \ + run_gpu_step \ "cuda-async GPU integration test ${test_target}" \ - host_cargo test -p cuda-async --test "$test_target" + gpu_test host cuda-async "$test_target" done print_summary_and_exit \