From a48c53e8c8cea5a5edb3e31783f45fe6667edc75 Mon Sep 17 00:00:00 2001 From: Canberk Pitirli Date: Wed, 28 Oct 2026 23:56:02 +0300 Subject: [PATCH] perf(cuda-oxide): lower f32 max/min to native max.f32/min.f32 Refs #1594. Co-authored-by: imanolgo <858793+ImanolGo@users.noreply.github.com> Signed-off-by: Canberk Pitirli --- .../crates/dialect-mir/src/rust_intrinsics.rs | 7 +- .../crates/mir-lower/src/convert/ops/call.rs | 65 +- .../tests/lowering_test/math_conversions.rs | 111 +++ .../examples/float_minmax/Cargo.lock | 761 ++++++++++++++++++ .../examples/float_minmax/Cargo.toml | 21 + .../examples/float_minmax/src/main.rs | 249 ++++++ 6 files changed, 1198 insertions(+), 16 deletions(-) create mode 100644 cuda-oxide/crates/rustc-codegen-cuda/examples/float_minmax/Cargo.lock create mode 100644 cuda-oxide/crates/rustc-codegen-cuda/examples/float_minmax/Cargo.toml create mode 100644 cuda-oxide/crates/rustc-codegen-cuda/examples/float_minmax/src/main.rs diff --git a/cuda-oxide/crates/dialect-mir/src/rust_intrinsics.rs b/cuda-oxide/crates/dialect-mir/src/rust_intrinsics.rs index 43d7241f52..f17e34d74a 100644 --- a/cuda-oxide/crates/dialect-mir/src/rust_intrinsics.rs +++ b/cuda-oxide/crates/dialect-mir/src/rust_intrinsics.rs @@ -288,9 +288,10 @@ pub fn is_known_placeholder(callee: &str) -> bool { /// [`is_backend_dependent_libdevice_placeholder`]. /// /// The `max`/`min` placeholders (`maximum_number_nsz_*` / -/// `minimum_number_nsz_*`) are in NEITHER list: they lower to an ordered -/// compare/select expansion under every intrinsic backend, so they never -/// produce a `__nv_*` call. (`llvm.maxnum`/`llvm.minnum` are unusable +/// `minimum_number_nsz_*`) are in NEITHER list: they never produce a +/// `__nv_*` call. Under the LLVM NVPTX backend f32 lowers to the native +/// `llvm.nvvm.fmax.f` / `llvm.nvvm.fmin.f`; f64 and the libNVVM backend use an +/// ordered compare/select expansion. (`llvm.maxnum`/`llvm.minnum` are unusable /// because under LLVM 21 they propagate signaling NaNs, contradicting /// Rust's ignore-any-NaN contract; see #390.) pub fn is_libdevice_backed_placeholder(callee: &str) -> bool { diff --git a/cuda-oxide/crates/mir-lower/src/convert/ops/call.rs b/cuda-oxide/crates/mir-lower/src/convert/ops/call.rs index c5ae9f0325..4d5617f78b 100644 --- a/cuda-oxide/crates/mir-lower/src/convert/ops/call.rs +++ b/cuda-oxide/crates/mir-lower/src/convert/ops/call.rs @@ -362,6 +362,19 @@ impl RustFloatMathIntrinsic { } } + /// Whether this min/max lowers to a native NVVM intrinsic under the + /// active intrinsic backend, instead of the compare/select expansion. + /// + /// Only f32 has one (`llvm.nvvm.fmax.f` / `llvm.nvvm.fmin.f`), and only the + /// LLVM NVPTX backend accepts it; libNVVM keeps the expansion. + fn has_native_minmax(self, ctx: &Context) -> bool { + matches!(self, Self::MaxNumNszF32 | Self::MinNumNszF32) + && matches!( + crate::context::lowering_options(ctx).intrinsic_backend, + crate::IntrinsicBackend::LlvmNvptx + ) + } + /// CUDA libdevice function name for this Rust math intrinsic. fn libdevice_name( self, @@ -409,10 +422,11 @@ impl RustFloatMathIntrinsic { Self::CopysignF64 => Ok("__nv_copysign"), // `f32::max` / `f32::min` (and the f64 forms) call the `_nsz` // intrinsics, i.e. IEEE-754 maxNum/minNum with the "no signed - // zero" relaxation. They expand to an ordered compare/select - // sequence under every intrinsic backend (the caller routes them - // through `lower_float_minmax_nsz` before any symbol lookup), so - // they must never reach a libdevice name. + // zero" relaxation. f32 lowers to a native NVVM intrinsic under + // the LLVM NVPTX backend (see `llvm_intrinsic_name`); everything + // else expands to an ordered compare/select sequence (the caller + // routes it through `lower_float_minmax_nsz` before any symbol + // lookup). Neither path may reach a libdevice name. Self::MaxNumNszF32 | Self::MaxNumNszF64 | Self::MinNumNszF32 | Self::MinNumNszF64 => { pliron::input_err!( loc, @@ -515,11 +529,13 @@ impl RustFloatMathIntrinsic { /// Using LLVM intrinsics avoids a libdevice function call and removes the /// libNVVM dependency for kernels that only use these operations. /// - /// Scope: `max`/`min` deliberately return `None` here. They cannot use - /// `llvm.maxnum`/`llvm.minnum` at all, because under LLVM 21 those - /// propagate signaling NaNs and so contradict Rust's contract; they lower - /// as an ordered compare/select instead (`lower_float_minmax_nsz`, see - /// #390) and never reach this table. + /// Scope: f32 `max`/`min` map to `llvm.nvvm.fmax.f` / `llvm.nvvm.fmin.f`, + /// the non-`.NaN`, non-`.ftz` forms that select `max.f32` / `min.f32`. + /// They cannot use `llvm.maxnum`/`llvm.minnum`, because under LLVM 21 those + /// propagate signaling NaNs and so contradict Rust's contract (#390). f64 + /// has no NVVM equivalent, so f64 `max`/`min` return `None` here and lower + /// as an ordered compare/select instead (`lower_float_minmax_nsz`). The + /// libNVVM backend never uses this table and keeps that expansion for f32. /// /// Naming: pliron identifiers cannot contain dots, so intrinsic symbols /// use the underscore encoding (`llvm_floor_f32`) that the textual `.ll` @@ -551,6 +567,11 @@ impl RustFloatMathIntrinsic { Self::Fabs => fabs_llvm_intrinsic_name(ctx, result_ty, loc).map(Some), Self::CopysignF32 => Ok(Some("llvm_copysign_f32")), Self::CopysignF64 => Ok(Some("llvm_copysign_f64")), + // f32 min/max: the non-`.NaN`, non-`.ftz` NVVM forms select + // `max.f32` / `min.f32`. f64 has no NVVM equivalent and returns + // `None`; see the scope note above. + Self::MaxNumNszF32 => Ok(Some("llvm_nvvm_fmax_f")), + Self::MinNumNszF32 => Ok(Some("llvm_nvvm_fmin_f")), _ => Ok(None), } } @@ -1291,7 +1312,12 @@ fn convert_rust_float_math_intrinsic( return lower_fast_binop(ctx, rewriter, op, &args, binop, loc); } - if let Some(predicate) = intrinsic.minmax_nsz_predicate() { + // f32 `max`/`min` with a native NVVM intrinsic fall through to the common + // intrinsic-call path below; every other min/max keeps the compare/select + // expansion. + if let Some(predicate) = intrinsic.minmax_nsz_predicate() + && !intrinsic.has_native_minmax(ctx) + { return lower_float_minmax_nsz(ctx, rewriter, op, &args, predicate); } @@ -2453,13 +2479,26 @@ mod tests { Some("llvm_roundeven_f64") ); - // max/min return None: LLVM 21's maxnum/minnum propagate signaling - // NaNs, which contradicts Rust's contract. These use compare/select - // lowering instead and never reach the intrinsic tables (see #390). + // f32 max/min use the native NVVM intrinsics (`max.f32` / `min.f32`). + // LLVM 21's maxnum/minnum propagate signaling NaNs, which contradicts + // Rust's contract, so they are not used (see #390). assert_eq!( RustFloatMathIntrinsic::MaxNumNszF32 .llvm_intrinsic_name(&ctx, f32_ty, loc.clone()) .unwrap(), + Some("llvm_nvvm_fmax_f") + ); + assert_eq!( + RustFloatMathIntrinsic::MinNumNszF32 + .llvm_intrinsic_name(&ctx, f32_ty, loc.clone()) + .unwrap(), + Some("llvm_nvvm_fmin_f") + ); + // f64 has no NVVM equivalent and keeps the compare/select expansion. + assert_eq!( + RustFloatMathIntrinsic::MaxNumNszF64 + .llvm_intrinsic_name(&ctx, f64_ty, loc.clone()) + .unwrap(), None ); assert_eq!( diff --git a/cuda-oxide/crates/mir-lower/tests/lowering_test/math_conversions.rs b/cuda-oxide/crates/mir-lower/tests/lowering_test/math_conversions.rs index 95fbf7bd31..520e75e54a 100644 --- a/cuda-oxide/crates/mir-lower/tests/lowering_test/math_conversions.rs +++ b/cuda-oxide/crates/mir-lower/tests/lowering_test/math_conversions.rs @@ -1706,3 +1706,114 @@ fn test_scalar_arithmetic_libnvvm_uses_exact_inline_ptx() -> Result<(), anyhow:: ); Ok(()) } + +/// Lower `max` and `min` calls on both f32 and f64 operands under `backend`. +fn lower_float_minmax( + backend: mir_lower::IntrinsicBackend, +) -> Result<(Context, pliron::context::Ptr), anyhow::Error> { + use dialect_mir::rust_intrinsics; + use pliron::builtin::attributes::StringAttr; + use pliron::builtin::types::{FP32Type, FP64Type}; + use pliron::r#type::TypeHandle; + + let mut ctx = make_test_ctx(); + let f32_ty: TypeHandle = FP32Type::get(&ctx).into(); + let f64_ty: TypeHandle = FP64Type::get(&ctx).into(); + let (module_ptr, entry) = build_test_kernel(&mut ctx, vec![f32_ty, f32_ty, f64_ty, f64_ty]); + let f32_args = [ + entry.deref(&ctx).get_argument(0), + entry.deref(&ctx).get_argument(1), + ]; + let f64_args = [ + entry.deref(&ctx).get_argument(2), + entry.deref(&ctx).get_argument(3), + ]; + + for (callee, args, result_ty) in [ + (rust_intrinsics::CALLEE_MAXNUM_NSZ_F32, f32_args, f32_ty), + (rust_intrinsics::CALLEE_MINNUM_NSZ_F32, f32_args, f32_ty), + (rust_intrinsics::CALLEE_MAXNUM_NSZ_F64, f64_args, f64_ty), + (rust_intrinsics::CALLEE_MINNUM_NSZ_F64, f64_args, f64_ty), + ] { + let call_ptr = Operation::new( + &mut ctx, + mir::MirCallOp::get_concrete_op_info(), + vec![result_ty], + args.to_vec(), + vec![], + 0, + ); + let call = mir::MirCallOp::new(call_ptr); + call.set_attr_callee(&ctx, StringAttr::new(callee.to_string())); + call_ptr.insert_at_back(entry, &ctx); + } + append_return(&mut ctx, entry); + + mir_lower::lower_mir_to_llvm_with_options( + &mut ctx, + module_ptr, + mir_lower::LoweringOptions { + intrinsic_backend: backend, + ..Default::default() + }, + ) + .map_err(|error| anyhow::anyhow!("{error}"))?; + Ok((ctx, module_ptr)) +} + +/// Direct callee names, `fcmp` count and `select` count of a lowered body. +fn minmax_lowering_shape( + ctx: &Context, + body: &[pliron::context::Ptr], +) -> (Vec, usize, usize) { + let mut callees = Vec::new(); + let (mut fcmps, mut selects) = (0, 0); + for &op in body { + assert!( + Operation::get_op::(op, ctx).is_none(), + "max/min placeholder mir.call must not survive MIR lowering" + ); + if let Some(call) = Operation::get_op::(op, ctx) + && let CallOpCallable::Direct(sym) = call.callee(ctx) + { + callees.push(sym.to_string()); + } + fcmps += usize::from(Operation::get_op::(op, ctx).is_some()); + selects += usize::from(Operation::get_op::(op, ctx).is_some()); + } + (callees, fcmps, selects) +} + +#[test] +fn test_float_minmax_llvm_nvptx_f32_uses_nvvm_intrinsics_f64_uses_compare_select() +-> Result<(), anyhow::Error> { + let (ctx, module_ptr) = lower_float_minmax(mir_lower::IntrinsicBackend::LlvmNvptx)?; + let (callees, fcmps, selects) = + minmax_lowering_shape(&ctx, &lowered_kernel_body(&ctx, module_ptr)); + + // f32 max/min: one native NVVM call each, no compare/select. + assert_eq!( + callees, + ["llvm_nvvm_fmax_f", "llvm_nvvm_fmin_f"], + "f32 max/min must call the non-.NaN, non-.ftz NVVM intrinsics" + ); + // f64 max/min: the NaN-ignoring expansion, two fcmp + two select each. + assert_eq!(fcmps, 4, "only the two f64 ops keep the compare/select"); + assert_eq!(selects, 4, "only the two f64 ops keep the compare/select"); + Ok(()) +} + +#[test] +fn test_float_minmax_libnvvm_keeps_compare_select_for_both_widths() -> Result<(), anyhow::Error> { + let (ctx, module_ptr) = lower_float_minmax(mir_lower::IntrinsicBackend::LibNvvm)?; + let (callees, fcmps, selects) = + minmax_lowering_shape(&ctx, &lowered_kernel_body(&ctx, module_ptr)); + + assert!( + callees.is_empty(), + "libNVVM max/min must not call an intrinsic, found {callees:?}" + ); + assert_eq!(fcmps, 8, "all four ops expand to two fcmp each"); + assert_eq!(selects, 8, "all four ops expand to two select each"); + Ok(()) +} diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/float_minmax/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/float_minmax/Cargo.lock new file mode 100644 index 0000000000..6cfd632901 --- /dev/null +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/float_minmax/Cargo.lock @@ -0,0 +1,761 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "aho-corasick" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" +dependencies = [ + "memchr", +] + +[[package]] +name = "anyhow" +version = "1.0.102" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" + +[[package]] +name = "bindgen" +version = "0.69.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "271383c67ccabffb7381723dea0672a673f292304fcb45c01cc648c7a8d58088" +dependencies = [ + "bitflags", + "cexpr", + "clang-sys", + "itertools", + "lazy_static", + "lazycell", + "log", + "prettyplease", + "proc-macro2", + "quote", + "regex", + "rustc-hash", + "shlex", + "syn", + "which", +] + +[[package]] +name = "bitflags" +version = "2.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4388bee8683e3d04af747c73422af53102d2bd24d9eadb6cbc100baef4b43f8" + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "cexpr" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766" +dependencies = [ + "nom", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "clang-sys" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b023947811758c97c59bf9d1c188fd619ad4718dcaa767947df1cadb14f39f4" +dependencies = [ + "glob", + "libc", + "libloading", +] + +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "crc32fast" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "crunchy" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" + +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "cuda-artifact-finalizer" +version = "0.2.1" +dependencies = [ + "libnvvm-sys", + "nvjitlink-sys", + "serde", + "sha2", + "thiserror", +] + +[[package]] +name = "cuda-bindings" +version = "0.5.0" +dependencies = [ + "bindgen", + "libloading", + "prettyplease", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "cuda-core" +version = "0.5.0" +dependencies = [ + "anyhow", + "cuda-bindings", + "cuda-core-derive", + "half", + "oxide-artifacts", +] + +[[package]] +name = "cuda-core-derive" +version = "0.5.0" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "cuda-device" +version = "0.2.1" +dependencies = [ + "cuda-macros", +] + +[[package]] +name = "cuda-host" +version = "0.2.1" +dependencies = [ + "cuda-artifact-finalizer", + "cuda-core", + "cuda-device", + "cuda-macros", + "half", + "libc", + "oxide-artifacts", + "ptx-parse", + "sha2", + "thiserror", +] + +[[package]] +name = "cuda-macros" +version = "0.2.1" +dependencies = [ + "oxide-tile-export", + "proc-macro2", + "quote", + "reserved-oxide-symbols", + "syn", +] + +[[package]] +name = "cuda-target-spec" +version = "0.1.0" + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", +] + +[[package]] +name = "either" +version = "1.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91622ff5e7162018101f2fea40d6ebf4a78bbe5a49736a2020649edf9693679e" + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "float_minmax" +version = "0.1.0" +dependencies = [ + "cuda-core", + "cuda-device", + "cuda-host", + "libm", +] + +[[package]] +name = "foldhash" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "glob" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280" + +[[package]] +name = "half" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" +dependencies = [ + "cfg-if", + "crunchy", + "zerocopy", +] + +[[package]] +name = "hashbrown" +version = "0.15.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" +dependencies = [ + "foldhash", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + +[[package]] +name = "home" +version = "0.5.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc627f471c528ff0c4a49e1d5e60450c8f6461dd6d10ba9dcd3a61d3dff7728d" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "indexmap" +version = "2.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" +dependencies = [ + "equivalent", + "hashbrown 0.17.1", +] + +[[package]] +name = "itertools" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + +[[package]] +name = "lazycell" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" + +[[package]] +name = "libc" +version = "0.2.186" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" + +[[package]] +name = "libloading" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7c4b02199fee7c5d21a5ae7d8cfa79a6ef5bb2fc834d6e9058e89c825efdc55" +dependencies = [ + "cfg-if", + "windows-link", +] + +[[package]] +name = "libm" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" + +[[package]] +name = "libnvvm-sys" +version = "0.2.1" +dependencies = [ + "cuda-target-spec", + "libloading", + "thiserror", +] + +[[package]] +name = "linux-raw-sys" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab" + +[[package]] +name = "log" +version = "0.4.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + +[[package]] +name = "nvjitlink-sys" +version = "0.2.1" +dependencies = [ + "libloading", + "thiserror", +] + +[[package]] +name = "object" +version = "0.36.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62948e14d923ea95ea2c7c86c71013138b66525b86bdc08d2dcc262bdb497b87" +dependencies = [ + "crc32fast", + "hashbrown 0.15.5", + "indexmap", + "memchr", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "oxide-artifacts" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" +dependencies = [ + "object", +] + +[[package]] +name = "oxide-tile-export" +version = "0.2.1" +dependencies = [ + "serde", + "serde_json", + "thiserror", +] + +[[package]] +name = "prettyplease" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" +dependencies = [ + "proc-macro2", + "syn", +] + +[[package]] +name = "proc-macro2" +version = "1.0.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "ptx-parse" +version = "0.1.0" + +[[package]] +name = "quote" +version = "1.0.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "regex" +version = "1.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e10754a14b9137dd7b1e3e5b0493cc9171fdd105e0ab477f51b72e7f3ac0e276" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + +[[package]] +name = "reserved-oxide-symbols" +version = "0.2.1" + +[[package]] +name = "rustc-hash" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" + +[[package]] +name = "rustix" +version = "0.38.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.59.0", +] + +[[package]] +name = "serde" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.154" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7e9cc8b1b85264074fbcc02a88680c4096b1e47df8f739dceb03bf482f04bd6" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + +[[package]] +name = "syn" +version = "2.0.117" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "thiserror" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "which" +version = "4.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87ba24419a2078cd2b0f2ede2691b6c66d8e47836da3b6db8265ebad47afbfc7" +dependencies = [ + "either", + "home", + "once_cell", + "rustix", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-sys" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "zerocopy" +version = "0.8.52" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce1022995ff5ff5d841ad7d994facc23098cd40152f2c1d11cd607c6f530653f" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.52" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ae7f38b72ec2a254e2b87ef277cf2cd4fb97cbebf944faa6f33354da0867930" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/float_minmax/Cargo.toml b/cuda-oxide/crates/rustc-codegen-cuda/examples/float_minmax/Cargo.toml new file mode 100644 index 0000000000..730aba3255 --- /dev/null +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/float_minmax/Cargo.toml @@ -0,0 +1,21 @@ +[package] +name = "float_minmax" +version = "0.1.0" +edition = "2024" +license = "Apache-2.0" + +[workspace] + +# f32 `max`/`min` lower to the native PTX `max.f32` / `min.f32`. +# Checks Rust's NaN contract (a numeric operand wins over a quiet or +# signaling NaN in either position) and that subnormals survive. +# +# Run with: +# cargo oxide run float_minmax +# cargo oxide inspect float_minmax # PTX has max.f32 / min.f32, no setp/selp + +[dependencies] +cuda-device = { path = "../../../cuda-device" } +cuda-host = { path = "../../../cuda-host" } +cuda-core = { path = "../../../../../cuda-core" } +libm = "0.2" diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/float_minmax/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/float_minmax/src/main.rs new file mode 100644 index 0000000000..39b9d9f3ef --- /dev/null +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/float_minmax/src/main.rs @@ -0,0 +1,249 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +//! f32 `max` / `min` on the device (#1594). +//! +//! `f32::max`, `f32::min`, `libm::fmaxf` and `libm::fminf` lower to the native +//! PTX `max.f32` / `min.f32` (through `llvm.nvvm.fmax.f` / `llvm.nvvm.fmin.f`) +//! instead of a `setp` + `selp` pair. `cargo oxide inspect float_minmax` shows +//! the PTX. +//! +//! The host runs every pair of a value set that covers quiet and signaling +//! NaNs of both signs, infinities, signed zeros and subnormals, so each NaN +//! lands in both operand positions, and checks Rust's contract: +//! +//! - one operand is NaN: the other operand is returned bit for bit; +//! - both are NaN: the result is a NaN; +//! - otherwise: the larger (smaller) value, subnormals preserved. +//! +//! `max(-0.0, +0.0)` is unspecified in Rust, so the signed-zero results are +//! printed and only required to be a zero. +//! +//! Run with: +//! cargo oxide run float_minmax + +use cuda_core::simt::LaunchConfig; +use cuda_core::{CudaContext, DeviceBuffer}; +use cuda_device::{DisjointSlice, cuda_module, kernel, thread}; + +#[cuda_module] +mod kernels { + use super::*; + + #[kernel] + pub fn std_max(a: &[f32], b: &[f32], mut out: DisjointSlice) { + let idx = thread::index_1d(); + let i = idx.get(); + if let Some(o) = out.get_mut(idx) { + *o = a[i].max(b[i]); + } + } + + #[kernel] + pub fn std_min(a: &[f32], b: &[f32], mut out: DisjointSlice) { + let idx = thread::index_1d(); + let i = idx.get(); + if let Some(o) = out.get_mut(idx) { + *o = a[i].min(b[i]); + } + } + + #[kernel] + pub fn libm_max(a: &[f32], b: &[f32], mut out: DisjointSlice) { + let idx = thread::index_1d(); + let i = idx.get(); + if let Some(o) = out.get_mut(idx) { + *o = libm::fmaxf(a[i], b[i]); + } + } + + #[kernel] + pub fn libm_min(a: &[f32], b: &[f32], mut out: DisjointSlice) { + let idx = thread::index_1d(); + let i = idx.get(); + if let Some(o) = out.get_mut(idx) { + *o = libm::fminf(a[i], b[i]); + } + } + + /// Both operations in one kernel, as in the report for #1594. + #[kernel] + pub fn std_minmax(a: &[f32], b: &[f32], mut out: DisjointSlice) { + let idx = thread::index_1d(); + let i = idx.get(); + if let Some(o) = out.get_mut(idx) { + *o = a[i].max(b[i]) + a[i].min(b[i]); + } + } + + #[kernel] + pub fn libm_minmax(a: &[f32], b: &[f32], mut out: DisjointSlice) { + let idx = thread::index_1d(); + let i = idx.get(); + if let Some(o) = out.get_mut(idx) { + *o = libm::fmaxf(a[i], b[i]) + libm::fminf(a[i], b[i]); + } + } +} + +/// Bit patterns: numbers, infinities, signed zeros, subnormals, and quiet and +/// signaling NaNs of both signs. +const VALUES: [u32; 16] = [ + 0x3f80_0000, // 1.0 + 0xbf80_0000, // -1.0 + 0x0000_0000, // +0.0 + 0x8000_0000, // -0.0 + 0x7f80_0000, // +inf + 0xff80_0000, // -inf + 0x7f7f_ffff, // f32::MAX + 0x0080_0000, // smallest normal + 0x0000_0001, // smallest subnormal + 0x8000_0001, // -smallest subnormal + 0x007f_ffff, // largest subnormal + 0x7fc0_0000, // +qNaN + 0xffc0_0000, // -qNaN + 0x7fa0_0000, // +sNaN + 0xffa0_0000, // -sNaN + 0x7fc0_1234, // qNaN with a payload +]; + +fn is_zero(x: f32) -> bool { + x == 0.0 +} + +/// Check one result against Rust's contract. Returns an error description. +fn check(is_max: bool, x: f32, y: f32, r: f32) -> Result<(), String> { + let (xb, yb, rb) = (x.to_bits(), y.to_bits(), r.to_bits()); + if x.is_nan() && y.is_nan() { + return if r.is_nan() { + Ok(()) + } else { + Err(format!("both NaN, got {rb:#010x}")) + }; + } + if x.is_nan() || y.is_nan() { + let want = if x.is_nan() { yb } else { xb }; + return if rb == want { + Ok(()) + } else { + Err(format!("one NaN, want {want:#010x}, got {rb:#010x}")) + }; + } + if is_zero(x) && is_zero(y) { + return if is_zero(r) { + Ok(()) + } else { + Err(format!("zeros, got non-zero {rb:#010x}")) + }; + } + let want = match (is_max, x > y) { + (true, true) | (false, false) => xb, + _ => yb, + }; + if rb == want { + Ok(()) + } else { + Err(format!("want {want:#010x}, got {rb:#010x}")) + } +} + +fn main() { + println!("=== float_minmax: f32 max/min on the device ==="); + + let ctx = CudaContext::new(0).expect("Failed to create CUDA context"); + let stream = ctx.default_stream(); + + let (mut a, mut b) = (Vec::new(), Vec::new()); + for &x in &VALUES { + for &y in &VALUES { + a.push(f32::from_bits(x)); + b.push(f32::from_bits(y)); + } + } + let n = a.len(); + + let a_dev = DeviceBuffer::from_host(&stream, &a).unwrap(); + let b_dev = DeviceBuffer::from_host(&stream, &b).unwrap(); + let module = kernels::load(&ctx).expect("Failed to load embedded CUDA module"); + let cfg = LaunchConfig::for_num_elems(n as u32); + + macro_rules! run { + ($kernel:ident) => {{ + let mut out = DeviceBuffer::::zeroed(&stream, n).unwrap(); + // SAFETY: the launch is 1-D, one thread per element, and the + // buffers cover the kernel's accesses. + unsafe { module.$kernel(&stream, cfg, &a_dev, &b_dev, &mut out) } + .expect(concat!(stringify!($kernel), " launch failed")); + out.to_host_vec(&stream).unwrap() + }}; + } + + let mut ok = true; + for (name, is_max, out) in [ + ("std_max", true, run!(std_max)), + ("std_min", false, run!(std_min)), + ("libm_max", true, run!(libm_max)), + ("libm_min", false, run!(libm_min)), + ] { + let mut bad = 0; + for i in 0..n { + if let Err(why) = check(is_max, a[i], b[i], out[i]) { + if bad < 5 { + println!( + "{name}: x={:#010x} y={:#010x}: {why}", + a[i].to_bits(), + b[i].to_bits() + ); + } + bad += 1; + } + } + println!("{name}: {} of {n} pairs match", n - bad); + ok &= bad == 0; + + if name.starts_with("std") { + // Pair (x, y) is stored at index xi * VALUES.len() + yi. + let at = |xi: usize, yi: usize| out[xi * VALUES.len() + yi].to_bits(); + println!( + " signed zero: ({name})(-0.0, +0.0) = {:#010x}, (+0.0, -0.0) = {:#010x}", + at(3, 2), + at(2, 3) + ); + } + } + + // The combined kernels: `max + min`. Skip pairs with a NaN; infinities of + // opposite sign make the sum NaN on host and device alike. + for (name, out) in [ + ("std_minmax", run!(std_minmax)), + ("libm_minmax", run!(libm_minmax)), + ] { + let mut bad = 0; + for i in 0..n { + let (x, y) = (a[i], b[i]); + if x.is_nan() || y.is_nan() { + continue; + } + let want = x + y; + let same = if want.is_nan() { + out[i].is_nan() + } else { + out[i] == want + }; + if !same { + bad += 1; + } + } + println!("{name}: {bad} mismatches among the non-NaN pairs"); + ok &= bad == 0; + } + + if ok { + println!("SUCCESS: f32 max/min follow Rust's NaN contract"); + } else { + println!("FAIL: device max/min diverge from Rust's contract"); + std::process::exit(1); + } +}