From f4ae0285840ca8b190f0000e96175cc4a60d7719 Mon Sep 17 00:00:00 2001 From: renjithravindrankannath Date: Fri, 10 Jul 2026 18:56:33 -0700 Subject: [PATCH 01/28] Py-torch 2.9, 2.11 and ci build for rocm --- .ci/gitlab/configs/linux/ci.yaml | 2 + .../builtin/packages/hipblaslt/package.py | 2 +- .../builtin/packages/py_llvmlite/package.py | 4 ++ .../PR152569-Update-spack-includes-2.5.patch | 5 +- .../PR152569-Update-spack-includes-2.7.patch | 15 +++-- .../builtin/packages/py_torch/package.py | 63 ++++++++++++++++--- stacks/ml-linux-x86_64/spack.yaml | 39 ++++++------ 7 files changed, 95 insertions(+), 35 deletions(-) diff --git a/.ci/gitlab/configs/linux/ci.yaml b/.ci/gitlab/configs/linux/ci.yaml index 9add0f89153..67d2b921d5c 100644 --- a/.ci/gitlab/configs/linux/ci.yaml +++ b/.ci/gitlab/configs/linux/ci.yaml @@ -13,6 +13,8 @@ ci: - wrf build-job: tags: [ "spack", "huge" ] + # 24h for long ROCm/ML rebuilds; GitLab project/runner max must allow this + timeout: 1440 minutes variables: CI_JOB_SIZE: huge SPACK_BUILD_JOBS: "12" diff --git a/repos/spack_repo/builtin/packages/hipblaslt/package.py b/repos/spack_repo/builtin/packages/hipblaslt/package.py index ca3a8bd0997..9d43b3d1176 100644 --- a/repos/spack_repo/builtin/packages/hipblaslt/package.py +++ b/repos/spack_repo/builtin/packages/hipblaslt/package.py @@ -71,7 +71,7 @@ def url_for_version(self, version): depends_on("cmake@3.25.2:", type="build", when="@6.2.0:") depends_on("python@3.7:") - depends_on("python@3.8:3.13.2", when="@6.4:") + depends_on("python@3.8:3.13", when="@6.4:") for ver in [ "6.0.0", diff --git a/repos/spack_repo/builtin/packages/py_llvmlite/package.py b/repos/spack_repo/builtin/packages/py_llvmlite/package.py index dc8271b85b0..5de4d9ff6a3 100644 --- a/repos/spack_repo/builtin/packages/py_llvmlite/package.py +++ b/repos/spack_repo/builtin/packages/py_llvmlite/package.py @@ -66,6 +66,10 @@ class PyLlvmlite(PythonPackage): depends_on("llvm@14", when="@0.41:0.43") depends_on("llvm@11:14", when="@0.40") depends_on("llvm@11", when="@0.37:0.39") + # When the spec has both llvm@20 and hwloc+rocm (e.g. llvm build-depends on hwloc), + # hwloc with older rocm versions conflicts with llvm@20 while linking. + # Hence adding hwloc~rocm dependency for llvmlite@0.45:. + depends_on("hwloc ~rocm", when="@0.45:") for t in [ "arm:", "ppc:", diff --git a/repos/spack_repo/builtin/packages/py_torch/PR152569-Update-spack-includes-2.5.patch b/repos/spack_repo/builtin/packages/py_torch/PR152569-Update-spack-includes-2.5.patch index 2e7a80bcbe8..2c35aafac2f 100644 --- a/repos/spack_repo/builtin/packages/py_torch/PR152569-Update-spack-includes-2.5.patch +++ b/repos/spack_repo/builtin/packages/py_torch/PR152569-Update-spack-includes-2.5.patch @@ -25,10 +25,10 @@ index 9be7f37..39d0f24 100644 endif() diff --git a/cmake/public/LoadHIP.cmake b/cmake/public/LoadHIP.cmake -index 1c0d3a2..e0de4b1 100644 +index 1c0d3a2..83f9f9d 100644 --- a/cmake/public/LoadHIP.cmake +++ b/cmake/public/LoadHIP.cmake -@@ -167,6 +167,10 @@ if(HIP_FOUND) +@@ -167,6 +167,11 @@ if(HIP_FOUND) find_package_and_print_version(hipsolver REQUIRED) find_package_and_print_version(hiprtc REQUIRED) @@ -36,6 +36,7 @@ index 1c0d3a2..e0de4b1 100644 + list(APPEND ROCM_INCLUDE ${rocprim_INCLUDE_DIR}) + list(APPEND ROCM_INCLUDE ${hipcub_INCLUDE_DIR}) + list(APPEND ROCM_INCLUDE ${rocRAND_INCLUDE_DIR}) ++ list(APPEND ROCM_INCLUDE $ENV{AOTRITON_INSTALLED_PREFIX}/include) find_library(PYTORCH_HIP_LIBRARIES amdhip64 HINTS ${ROCM_PATH}/lib) # TODO: miopen_LIBRARIES should return fullpath to the library file, diff --git a/repos/spack_repo/builtin/packages/py_torch/PR152569-Update-spack-includes-2.7.patch b/repos/spack_repo/builtin/packages/py_torch/PR152569-Update-spack-includes-2.7.patch index 4392e00d76a..af876735594 100644 --- a/repos/spack_repo/builtin/packages/py_torch/PR152569-Update-spack-includes-2.7.patch +++ b/repos/spack_repo/builtin/packages/py_torch/PR152569-Update-spack-includes-2.7.patch @@ -1,5 +1,5 @@ diff --git a/caffe2/CMakeLists.txt b/caffe2/CMakeLists.txt -index d2d23b7..620a89f 100644 +index d2d23b7ab65..620a89f65cb 100644 --- a/caffe2/CMakeLists.txt +++ b/caffe2/CMakeLists.txt @@ -1379,13 +1379,6 @@ if(USE_ROCM) @@ -26,7 +26,7 @@ index d2d23b7..620a89f 100644 endif() diff --git a/cmake/public/LoadHIP.cmake b/cmake/public/LoadHIP.cmake -index 58c74dd..d3e1ad4 100644 +index 58c74ddda35..54f96871372 100644 --- a/cmake/public/LoadHIP.cmake +++ b/cmake/public/LoadHIP.cmake @@ -26,12 +26,6 @@ else() @@ -78,7 +78,15 @@ index 58c74dd..d3e1ad4 100644 find_package_and_print_version(amd_comgr REQUIRED) find_package_and_print_version(rocrand REQUIRED) find_package_and_print_version(hiprand REQUIRED) -@@ -171,7 +168,11 @@ if(HIP_FOUND) +@@ -157,6 +154,7 @@ if(HIP_FOUND) + find_package_and_print_version(hipcub REQUIRED) + find_package_and_print_version(rocthrust REQUIRED) + find_package_and_print_version(hipsolver REQUIRED) ++ list(APPEND ROCM_INCLUDE_DIRS $ENV{AOTRITON_INSTALLED_PREFIX}/include) + # workaround cmake 4 build issue + if(CMAKE_VERSION VERSION_GREATER_EQUAL "4.0.0") + message(WARNING "Work around hiprtc cmake failure for cmake >= 4") +@@ -171,7 +169,11 @@ if(HIP_FOUND) if(UNIX) find_package_and_print_version(rccl) find_package_and_print_version(hsa-runtime64 REQUIRED) @@ -89,4 +97,3 @@ index 58c74dd..d3e1ad4 100644 + if(UNIX) # roctx is part of roctracer find_library(ROCM_ROCTX_LIB roctx64 HINTS ${ROCM_PATH}/lib) - diff --git a/repos/spack_repo/builtin/packages/py_torch/package.py b/repos/spack_repo/builtin/packages/py_torch/package.py index 2cab759e303..e6638a6ef93 100644 --- a/repos/spack_repo/builtin/packages/py_torch/package.py +++ b/repos/spack_repo/builtin/packages/py_torch/package.py @@ -97,7 +97,8 @@ class PyTorch(PythonPackage, CudaPackage, ROCmPackage): variant("rocm", default=False, description="Use ROCm") variant("cudnn", default=not is_darwin, description="Use cuDNN", when="+cuda") variant("fbgemm", default=True, description="Use FBGEMM (quantized 8-bit server operators)") - variant("kineto", default=True, description="Use Kineto profiling library", when="@1.8:") + variant("kineto", default=True, description="Use Kineto profiling library", when="@1.8:2.10") + variant("kineto", default=False, description="Disable Kineto from 2.11 on rocm", when="@2.11:") variant("magma", default=not is_darwin, description="Use MAGMA", when="+cuda") variant("metal", default=is_darwin, description="Use Metal for Caffe2 iOS build") variant( @@ -107,7 +108,13 @@ class PyTorch(PythonPackage, CudaPackage, ROCmPackage): when="@1.12: platform=darwin", ) variant("nccl", default=True, description="Use NCCL", when="+cuda platform=linux") - variant("nccl", default=True, description="Use NCCL", when="+rocm platform=linux") + variant("nccl", default=True, description="Use NCCL", when="@:2.10 +rocm platform=linux") + variant( + "nccl", + default=False, + description="Disable NCCL from 2.11 on rocm", + when="@2.11: +rocm platform=linux", + ) # Requires AVX2: https://discuss.pytorch.org/t/107518 variant("nnpack", default=True, description="Use NNPACK", when="target=x86_64_v3:") variant("numa", default=True, description="Use NUMA", when="platform=linux") @@ -144,6 +151,7 @@ class PyTorch(PythonPackage, CudaPackage, ROCmPackage): conflicts("+gloo+rocm") conflicts("+rocm", when="@2.3", msg="Rocm doesn't support py-torch 2.3 release") conflicts("+rocm", when="@2.4", msg="Rocm doesn't support py-torch 2.4 release") + conflicts("+rocm", when="@2.8", msg="Rocm doesn't support py-torch 2.8 release") conflicts("+tensorpipe", when="+rocm ^hip@:5.1", msg="TensorPipe not supported until ROCm 5.2") conflicts("+breakpad", when="target=ppc64:") conflicts("+breakpad", when="target=ppc64le:") @@ -344,7 +352,8 @@ class PyTorch(PythonPackage, CudaPackage, ROCmPackage): depends_on("valgrind", when="+valgrind") with when("+rocm"): depends_on("hsa-rocr-dev") - depends_on("hip") + depends_on("hip@7.0:", when="@2.9:") + depends_on("hip@:6.4", when="@:2.7") depends_on("rccl", when="+nccl") depends_on("rocprim") depends_on("hipcub") @@ -356,14 +365,21 @@ class PyTorch(PythonPackage, CudaPackage, ROCmPackage): depends_on("hiprand") depends_on("hipsolver") depends_on("rocm-core") + depends_on("amdsmi", when="@2.12") depends_on("rocfft") depends_on("rocblas") depends_on("miopen-hip") + depends_on("composable-kernel") + depends_on("hipblaslt") + # Ensure hipblaslt version for 2.9+ + depends_on("hipblaslt@7.0:", when="@2.9:") depends_on("rocminfo") - depends_on("aotriton@0.8.1b", when="@2.5:2.6") - depends_on("aotriton@0.9.1b", when="@2.7:") - depends_on("composable-kernel@:6.3.2", when="@2.5") - depends_on("composable-kernel@6.3.2:", when="@2.6:") + depends_on("hipsparselt@7.0:", when="@2.9:") + depends_on("aotriton@0.8b", when="@2.5:2.6") + depends_on("aotriton@0.9.2b", when="@2.7") + depends_on("aotriton@0.10b", when="@2.8:2.10") + depends_on("aotriton@0.11b", when="@2.11:") + depends_on("mpi", when="+mpi") depends_on("ucc", when="+ucc") depends_on("ucx", when="+ucc") @@ -622,6 +638,17 @@ def patch(self): "torch_global_deps PROPERTIES LINKER_LANGUAGE CXX", "caffe2/CMakeLists.txt", ) + if self.spec.satisfies("@2.5:+rocm"): + filter_file( + "find_library(ROCM_ROCTX_LIB roctx64 HINTS ${ROCM_PATH}/lib)", + "find_library(ROCM_ROCTX_LIB roctx64 HINTS ${ROCM_PATH}/lib)\n" + "if(DEFINED ENV{ROCTRACER_INCLUDE_DIR})\n" + " set(ROCTRACER_INCLUDE_DIR $ENV{ROCTRACER_INCLUDE_DIR} CACHE PATH " + '"Roctracer include directory" FORCE)\n' + "endif()", + "cmake/public/LoadHIP.cmake", + string=True, + ) if self.spec.satisfies("@2.1:2.7+rocm"): filter_file( "${ROCM_INCLUDE_DIRS}/rocm-core/rocm_version.h", @@ -719,6 +746,11 @@ def enable_or_disable(variant, keyword="USE", var=None): enable_or_disable("rocm") if "+rocm" in self.spec: + # So libtorch_hip.so and dependents find ROCm/runtime libs at runtime and + # during binary cache relocation (avoids "=> not found" for e.g. + # libamdhip64.so.6, libhsa-runtime64.so.1). + for lib_dir in [self.spec["hip"].prefix.lib, self.spec["hsa-rocr-dev"].prefix.lib]: + env.append_flags("LDFLAGS", "-Wl,-rpath," + lib_dir) env.set("PYTORCH_ROCM_ARCH", ";".join(self.spec.variants["amdgpu_target"].value)) env.set("HSA_PATH", self.spec["hsa-rocr-dev"].prefix) env.set("ROCBLAS_PATH", self.spec["rocblas"].prefix) @@ -739,6 +771,7 @@ def enable_or_disable(variant, keyword="USE", var=None): if self.spec.satisfies("@2.5:"): env.set("TORCHINDUCTOR_CK_DIR", self.spec["composable-kernel"].prefix) env.set("AOTRITON_INSTALLED_PREFIX", self.spec["aotriton"].prefix) + env.prepend_path("CPATH", self.spec["aotriton"].prefix.include) if self.spec.satisfies("^hip@5.2.0:"): env.set("CMAKE_MODULE_PATH", self.spec["hip"].prefix.lib.cmake.hip) @@ -862,6 +895,22 @@ def enable_or_disable(variant, keyword="USE", var=None): def setup_run_environment(self, env: EnvironmentModifications) -> None: self.torch_cuda_arch_list(env) + if "+rocm" in self.spec: + env.prepend_path("LD_LIBRARY_PATH", self.spec["hip"].prefix.lib) + + def setup_dependent_build_environment(self, env, dependent_spec): + if "+rocm" in self.spec: + env.prepend_path("LD_LIBRARY_PATH", self.spec["hip"].prefix.lib) + # PyTorch headers (e.g. c10/util/complex.h) include ; + # dependents need rocthrust include so HIP device builds can find it. + env.set("THRUST_PATH", self.spec["rocthrust"].prefix) + env.prepend_path("CPATH", self.spec["rocthrust"].prefix.include) + + def setup_dependent_run_environment(self, env, dependent_spec): + """So dependents (e.g. py-torch-nvidia-apex, py-torchaudio) can find + libamdhip64.so when importing torch or running code that uses ROCm.""" + if "+rocm" in self.spec: + env.prepend_path("LD_LIBRARY_PATH", self.spec["hip"].prefix.lib) @run_before("install") def build_amd(self): diff --git a/stacks/ml-linux-x86_64/spack.yaml b/stacks/ml-linux-x86_64/spack.yaml index 50a030f9d9f..bd08e39b60d 100644 --- a/stacks/ml-linux-x86_64/spack.yaml +++ b/stacks/ml-linux-x86_64/spack.yaml @@ -28,21 +28,21 @@ spack: - py-keras backend=jax - py-keras backend=torch - torch_specs: - - py-botorch - - py-gpytorch - - py-kornia - - py-lightning - - py-pytorch-lightning - - py-segmentation-models-pytorch - - py-timm + #- py-botorch + #- py-gpytorch + #- py-kornia + #- py-lightning + #- py-pytorch-lightning + #- py-segmentation-models-pytorch + #- py-timm - py-torch - - py-torch-geometric - - py-torch-nvidia-apex - - py-torchaudio - - py-torchgeo - - py-torchmetrics - - py-torchvision - - py-vector-quantize-pytorch + #- py-torch-geometric + #- py-torch-nvidia-apex + #- py-torchaudio + #- py-torchgeo + #- py-torchmetrics + #- py-torchvision + #- py-vector-quantize-pytorch - scikit_learn_specs: - py-scikit-learn - tensorboard_specs: @@ -113,15 +113,11 @@ spack: - $jax_specs - matrix: - [$keras_specs] - exclude: - - py-keras backend=torch - # Does not yet support Spack-installed ROCm - # - $torch_specs + - $torch_specs - $scikit_learn_specs - $tensorboard_specs - $tensorflow_specs - # Does not yet support Spack-installed ROCm - # - $xgboost_specs + - $xgboost_specs override: packages: all: @@ -139,8 +135,9 @@ spack: - target=x86_64_v3 - ~cuda - +rocm + - ~distributed - amdgpu_target=gfx90a - - ~flash_attention + - +flash_attention - "%c,cxx=gcc" ci: From 4e910f5cef22711ee37967f8576808defeaa04c9 Mon Sep 17 00:00:00 2001 From: renjithravindrankannath Date: Thu, 16 Jul 2026 15:31:03 -0400 Subject: [PATCH 02/28] Removing temp fix since py-llvmlite support llvm@20 and 22 now --- repos/spack_repo/builtin/packages/py_llvmlite/package.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/repos/spack_repo/builtin/packages/py_llvmlite/package.py b/repos/spack_repo/builtin/packages/py_llvmlite/package.py index 5de4d9ff6a3..dc8271b85b0 100644 --- a/repos/spack_repo/builtin/packages/py_llvmlite/package.py +++ b/repos/spack_repo/builtin/packages/py_llvmlite/package.py @@ -66,10 +66,6 @@ class PyLlvmlite(PythonPackage): depends_on("llvm@14", when="@0.41:0.43") depends_on("llvm@11:14", when="@0.40") depends_on("llvm@11", when="@0.37:0.39") - # When the spec has both llvm@20 and hwloc+rocm (e.g. llvm build-depends on hwloc), - # hwloc with older rocm versions conflicts with llvm@20 while linking. - # Hence adding hwloc~rocm dependency for llvmlite@0.45:. - depends_on("hwloc ~rocm", when="@0.45:") for t in [ "arm:", "ppc:", From e7925ba95a716cd8a031867a1e026e3a28efca8f Mon Sep 17 00:00:00 2001 From: renjithravindrankannath Date: Fri, 17 Jul 2026 02:50:36 -0400 Subject: [PATCH 03/28] py-torch 2.12 and 2.13 are not updated for rocm --- repos/spack_repo/builtin/packages/py_torch/package.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/repos/spack_repo/builtin/packages/py_torch/package.py b/repos/spack_repo/builtin/packages/py_torch/package.py index e6638a6ef93..90aedb118b6 100644 --- a/repos/spack_repo/builtin/packages/py_torch/package.py +++ b/repos/spack_repo/builtin/packages/py_torch/package.py @@ -152,6 +152,8 @@ class PyTorch(PythonPackage, CudaPackage, ROCmPackage): conflicts("+rocm", when="@2.3", msg="Rocm doesn't support py-torch 2.3 release") conflicts("+rocm", when="@2.4", msg="Rocm doesn't support py-torch 2.4 release") conflicts("+rocm", when="@2.8", msg="Rocm doesn't support py-torch 2.8 release") + conflicts("+rocm", when="@2.12", msg="Rocm doesn't support py-torch 2.12 release") + conflicts("+rocm", when="@2.13", msg="Rocm doesn't support py-torch 2.13 release") conflicts("+tensorpipe", when="+rocm ^hip@:5.1", msg="TensorPipe not supported until ROCm 5.2") conflicts("+breakpad", when="target=ppc64:") conflicts("+breakpad", when="target=ppc64le:") From 1efca42b29146ba1b124f93205ef4af518d927f9 Mon Sep 17 00:00:00 2001 From: renjithravindrankannath Date: Wed, 22 Jul 2026 03:26:01 -0400 Subject: [PATCH 04/28] hwloc ~rocm for py-llvmlite --- stacks/ml-linux-x86_64/spack.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/stacks/ml-linux-x86_64/spack.yaml b/stacks/ml-linux-x86_64/spack.yaml index bd08e39b60d..d0ae8c755f3 100644 --- a/stacks/ml-linux-x86_64/spack.yaml +++ b/stacks/ml-linux-x86_64/spack.yaml @@ -139,6 +139,9 @@ spack: - amdgpu_target=gfx90a - +flash_attention - "%c,cxx=gcc" + hwloc: + require: + - ~rocm ci: pipeline-gen: From 9a393704a1195d2ad7db00d95dcddddbc26fba79 Mon Sep 17 00:00:00 2001 From: renjithravindrankannath Date: Wed, 22 Jul 2026 11:25:43 -0400 Subject: [PATCH 05/28] Parsing error fix --- stacks/ml-linux-x86_64/spack.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stacks/ml-linux-x86_64/spack.yaml b/stacks/ml-linux-x86_64/spack.yaml index d0ae8c755f3..868fd087861 100644 --- a/stacks/ml-linux-x86_64/spack.yaml +++ b/stacks/ml-linux-x86_64/spack.yaml @@ -139,7 +139,7 @@ spack: - amdgpu_target=gfx90a - +flash_attention - "%c,cxx=gcc" - hwloc: + hwloc: require: - ~rocm From bbb619aa0d888d02504a116ebb46bec4cbec6b52 Mon Sep 17 00:00:00 2001 From: renjithravindrankannath Date: Tue, 28 Jul 2026 17:34:46 -0400 Subject: [PATCH 06/28] Reducing timeout to 8 hours --- .ci/gitlab/configs/linux/ci.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.ci/gitlab/configs/linux/ci.yaml b/.ci/gitlab/configs/linux/ci.yaml index 67d2b921d5c..68887ba89a4 100644 --- a/.ci/gitlab/configs/linux/ci.yaml +++ b/.ci/gitlab/configs/linux/ci.yaml @@ -13,8 +13,8 @@ ci: - wrf build-job: tags: [ "spack", "huge" ] - # 24h for long ROCm/ML rebuilds; GitLab project/runner max must allow this - timeout: 1440 minutes + # 8hr for long ROCm/ML rebuilds; GitLab project/runner max must allow this + timeout: 480 minutes variables: CI_JOB_SIZE: huge SPACK_BUILD_JOBS: "12" From ebb44c540a9ab5cfd2af87d19159529e6a262f4c Mon Sep 17 00:00:00 2001 From: renjithravindrankannath Date: Wed, 29 Jul 2026 15:37:26 -0400 Subject: [PATCH 07/28] ck build requres extra 5.75 hrs --- .ci/gitlab/configs/linux/ci.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.ci/gitlab/configs/linux/ci.yaml b/.ci/gitlab/configs/linux/ci.yaml index 68887ba89a4..6c6899fe311 100644 --- a/.ci/gitlab/configs/linux/ci.yaml +++ b/.ci/gitlab/configs/linux/ci.yaml @@ -13,8 +13,8 @@ ci: - wrf build-job: tags: [ "spack", "huge" ] - # 8hr for long ROCm/ML rebuilds; GitLab project/runner max must allow this - timeout: 480 minutes + # 13.75hr for long ROCm/ML rebuilds; GitLab project/runner max must allow this + timeout: 825 minutes variables: CI_JOB_SIZE: huge SPACK_BUILD_JOBS: "12" From 4d070ed6142fc953bb71d399f897681a8158562b Mon Sep 17 00:00:00 2001 From: renjithravindrankannath Date: Thu, 30 Jul 2026 16:03:30 -0400 Subject: [PATCH 08/28] Reverting to 24 hr temporarily to complete ck build --- .ci/gitlab/configs/linux/ci.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.ci/gitlab/configs/linux/ci.yaml b/.ci/gitlab/configs/linux/ci.yaml index 6c6899fe311..67d2b921d5c 100644 --- a/.ci/gitlab/configs/linux/ci.yaml +++ b/.ci/gitlab/configs/linux/ci.yaml @@ -13,8 +13,8 @@ ci: - wrf build-job: tags: [ "spack", "huge" ] - # 13.75hr for long ROCm/ML rebuilds; GitLab project/runner max must allow this - timeout: 825 minutes + # 24h for long ROCm/ML rebuilds; GitLab project/runner max must allow this + timeout: 1440 minutes variables: CI_JOB_SIZE: huge SPACK_BUILD_JOBS: "12" From b7fd78d47d8757bb0c9d84c1309094458bdd0971 Mon Sep 17 00:00:00 2001 From: Zack Galbreath Date: Mon, 3 Aug 2026 09:29:38 -0400 Subject: [PATCH 09/28] Set timeout to 12 hours Signed-off-by: Zack Galbreath --- .ci/gitlab/configs/linux/ci.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.ci/gitlab/configs/linux/ci.yaml b/.ci/gitlab/configs/linux/ci.yaml index 67d2b921d5c..ed8ea2be4a4 100644 --- a/.ci/gitlab/configs/linux/ci.yaml +++ b/.ci/gitlab/configs/linux/ci.yaml @@ -13,8 +13,8 @@ ci: - wrf build-job: tags: [ "spack", "huge" ] - # 24h for long ROCm/ML rebuilds; GitLab project/runner max must allow this - timeout: 1440 minutes + # 12h for long ROCm/ML rebuilds (maximum allowed for AWS runners) + timeout: 720 minutes variables: CI_JOB_SIZE: huge SPACK_BUILD_JOBS: "12" From 0533ec8bd6de9eca326460fb1b9b21c92139d3dd Mon Sep 17 00:00:00 2001 From: renjithravindrankannath Date: Thu, 6 Aug 2026 20:52:24 -0400 Subject: [PATCH 10/28] Enabling py-botorch --- stacks/ml-linux-x86_64/spack.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stacks/ml-linux-x86_64/spack.yaml b/stacks/ml-linux-x86_64/spack.yaml index 868fd087861..1a6b0ad2847 100644 --- a/stacks/ml-linux-x86_64/spack.yaml +++ b/stacks/ml-linux-x86_64/spack.yaml @@ -28,7 +28,7 @@ spack: - py-keras backend=jax - py-keras backend=torch - torch_specs: - #- py-botorch + - py-botorch #- py-gpytorch #- py-kornia #- py-lightning From efa47f2fbccda214d194c2de0bca0fe7f3375d62 Mon Sep 17 00:00:00 2001 From: renjithravindrankannath Date: Fri, 7 Aug 2026 10:03:40 -0400 Subject: [PATCH 11/28] Enabling py-gpytorch --- stacks/ml-linux-x86_64/spack.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stacks/ml-linux-x86_64/spack.yaml b/stacks/ml-linux-x86_64/spack.yaml index 1a6b0ad2847..14fa885a710 100644 --- a/stacks/ml-linux-x86_64/spack.yaml +++ b/stacks/ml-linux-x86_64/spack.yaml @@ -29,7 +29,7 @@ spack: - py-keras backend=torch - torch_specs: - py-botorch - #- py-gpytorch + - py-gpytorch #- py-kornia #- py-lightning #- py-pytorch-lightning From df77cd68e2d34a6dfda7afbf7512a439e79b60a6 Mon Sep 17 00:00:00 2001 From: renjithravindrankannath Date: Fri, 7 Aug 2026 17:20:01 -0400 Subject: [PATCH 12/28] Enabling py-kornia --- stacks/ml-linux-x86_64/spack.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stacks/ml-linux-x86_64/spack.yaml b/stacks/ml-linux-x86_64/spack.yaml index 14fa885a710..07ec5363986 100644 --- a/stacks/ml-linux-x86_64/spack.yaml +++ b/stacks/ml-linux-x86_64/spack.yaml @@ -30,7 +30,7 @@ spack: - torch_specs: - py-botorch - py-gpytorch - #- py-kornia + - py-kornia #- py-lightning #- py-pytorch-lightning #- py-segmentation-models-pytorch From a8af3ea6ac22df940c502d13ae8434979cb30a33 Mon Sep 17 00:00:00 2001 From: renjithravindrankannath Date: Fri, 7 Aug 2026 22:28:31 -0400 Subject: [PATCH 13/28] Enabling py-lightning --- stacks/ml-linux-x86_64/spack.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stacks/ml-linux-x86_64/spack.yaml b/stacks/ml-linux-x86_64/spack.yaml index 07ec5363986..cfad927e7d7 100644 --- a/stacks/ml-linux-x86_64/spack.yaml +++ b/stacks/ml-linux-x86_64/spack.yaml @@ -31,7 +31,7 @@ spack: - py-botorch - py-gpytorch - py-kornia - #- py-lightning + - py-lightning #- py-pytorch-lightning #- py-segmentation-models-pytorch #- py-timm From 804f4030f188ab21edd1b67f67be200cd47560d5 Mon Sep 17 00:00:00 2001 From: afzpatel Date: Sat, 8 Aug 2026 09:49:17 -0700 Subject: [PATCH 14/28] Adding py-pytorch-lightning --- stacks/ml-linux-x86_64/spack.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stacks/ml-linux-x86_64/spack.yaml b/stacks/ml-linux-x86_64/spack.yaml index cfad927e7d7..1259284e521 100644 --- a/stacks/ml-linux-x86_64/spack.yaml +++ b/stacks/ml-linux-x86_64/spack.yaml @@ -32,7 +32,7 @@ spack: - py-gpytorch - py-kornia - py-lightning - #- py-pytorch-lightning + - py-pytorch-lightning #- py-segmentation-models-pytorch #- py-timm - py-torch From e20a3ab3b059eac6b52d632a8b47a05efe5a2deb Mon Sep 17 00:00:00 2001 From: afzpatel Date: Sun, 9 Aug 2026 01:49:53 -0700 Subject: [PATCH 15/28] Adding py-segmentation-models-pytorch --- stacks/ml-linux-x86_64/spack.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stacks/ml-linux-x86_64/spack.yaml b/stacks/ml-linux-x86_64/spack.yaml index 1259284e521..030bfa703c0 100644 --- a/stacks/ml-linux-x86_64/spack.yaml +++ b/stacks/ml-linux-x86_64/spack.yaml @@ -33,7 +33,7 @@ spack: - py-kornia - py-lightning - py-pytorch-lightning - #- py-segmentation-models-pytorch + - py-segmentation-models-pytorch #- py-timm - py-torch #- py-torch-geometric From 0d32093004d81ac4021692f01993c5f9865069da Mon Sep 17 00:00:00 2001 From: renjithravindrankannath Date: Sun, 9 Aug 2026 22:00:26 -0400 Subject: [PATCH 16/28] Fix duplicate symbol error when building with ROCm --- .../packages/py_torchvision/package.py | 3 ++ ...-0.26.0-rocm-vision-duplicate-symbol.patch | 29 +++++++++++++++++++ 2 files changed, 32 insertions(+) create mode 100644 repos/spack_repo/builtin/packages/py_torchvision/torchvision-0.26.0-rocm-vision-duplicate-symbol.patch diff --git a/repos/spack_repo/builtin/packages/py_torchvision/package.py b/repos/spack_repo/builtin/packages/py_torchvision/package.py index 3b46f39a243..31d3855bdcb 100644 --- a/repos/spack_repo/builtin/packages/py_torchvision/package.py +++ b/repos/spack_repo/builtin/packages/py_torchvision/package.py @@ -61,6 +61,9 @@ class PyTorchvision(PythonPackage): version("0.9.0", sha256="9351ed92aded632f8c7f59dfadac13c191a834babe682f5785ea47e6fcf6b472") version("0.8.2", sha256="9a866c3c8feb23b3221ce261e6153fc65a98ce9ceaa71ccad017016945c178bf") + # Fix duplicate symbol error when building with ROCm + patch("torchvision-0.26.0-rocm-vision-duplicate-symbol.patch", when="@0.26.0 ^py-torch+rocm") + desc = "Enable support for native encoding/decoding of {} formats in torchvision.io" variant("png", default=True, description=desc.format("PNG")) variant("jpeg", default=True, description=desc.format("JPEG")) diff --git a/repos/spack_repo/builtin/packages/py_torchvision/torchvision-0.26.0-rocm-vision-duplicate-symbol.patch b/repos/spack_repo/builtin/packages/py_torchvision/torchvision-0.26.0-rocm-vision-duplicate-symbol.patch new file mode 100644 index 00000000000..50983b868b8 --- /dev/null +++ b/repos/spack_repo/builtin/packages/py_torchvision/torchvision-0.26.0-rocm-vision-duplicate-symbol.patch @@ -0,0 +1,29 @@ +From: Spack Project +Date: Sat, 9 Aug 2026 14:00:00 -0700 +Subject: [PATCH] Fix duplicate symbol error when building with ROCm + +When building torchvision with ROCm support, the hipify process converts +torchvision/csrc/vision.cpp to vision_hip.cpp, but the build system then +compiles and links BOTH vision.cpp and vision_hip.cpp together. This causes +a linker error due to duplicate definition of vision::cuda_version(). + +The root cause is that hipify_python.hipify() is processing vision.cpp even +though the includes pattern is "torchvision/csrc/ops/cuda/*" which should not +match files in torchvision/csrc/ directory. + +This patch explicitly excludes vision.cpp from hipification using the ignores +parameter, ensuring only the files under ops/cuda/ are converted to HIP. + +Upstream issue: https://github.com/pytorch/vision/issues/TBD +Fixes build failure in CI job: https://gitlab.spack.io/spack/spack-packages/-/jobs/23839237 + +--- setup.py.orig 2026-08-09 13:59:24.211961011 -0700 ++++ setup.py 2026-08-09 14:01:13.976910706 -0700 +@@ -165,6 +165,7 @@ + project_directory=str(ROOT_DIR), + output_directory=str(ROOT_DIR), + includes="torchvision/csrc/ops/cuda/*", ++ ignores=("torchvision/csrc/vision.cpp",), + show_detailed=True, + is_pytorch_extension=True, + ) From e8b38fe429191e26208968290ccb5657a50180c2 Mon Sep 17 00:00:00 2001 From: renjithravindrankannath Date: Sun, 9 Aug 2026 22:53:42 -0400 Subject: [PATCH 17/28] Fix patch failure --- ...-0.26.0-rocm-vision-duplicate-symbol.patch | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/repos/spack_repo/builtin/packages/py_torchvision/torchvision-0.26.0-rocm-vision-duplicate-symbol.patch b/repos/spack_repo/builtin/packages/py_torchvision/torchvision-0.26.0-rocm-vision-duplicate-symbol.patch index 50983b868b8..d8dc916eda2 100644 --- a/repos/spack_repo/builtin/packages/py_torchvision/torchvision-0.26.0-rocm-vision-duplicate-symbol.patch +++ b/repos/spack_repo/builtin/packages/py_torchvision/torchvision-0.26.0-rocm-vision-duplicate-symbol.patch @@ -1,22 +1,3 @@ -From: Spack Project -Date: Sat, 9 Aug 2026 14:00:00 -0700 -Subject: [PATCH] Fix duplicate symbol error when building with ROCm - -When building torchvision with ROCm support, the hipify process converts -torchvision/csrc/vision.cpp to vision_hip.cpp, but the build system then -compiles and links BOTH vision.cpp and vision_hip.cpp together. This causes -a linker error due to duplicate definition of vision::cuda_version(). - -The root cause is that hipify_python.hipify() is processing vision.cpp even -though the includes pattern is "torchvision/csrc/ops/cuda/*" which should not -match files in torchvision/csrc/ directory. - -This patch explicitly excludes vision.cpp from hipification using the ignores -parameter, ensuring only the files under ops/cuda/ are converted to HIP. - -Upstream issue: https://github.com/pytorch/vision/issues/TBD -Fixes build failure in CI job: https://gitlab.spack.io/spack/spack-packages/-/jobs/23839237 - --- setup.py.orig 2026-08-09 13:59:24.211961011 -0700 +++ setup.py 2026-08-09 14:01:13.976910706 -0700 @@ -165,6 +165,7 @@ From 883559827fc0d083688bdb7c9931ff7b3ffdcc7c Mon Sep 17 00:00:00 2001 From: renjithravindrankannath Date: Mon, 10 Aug 2026 03:26:05 -0400 Subject: [PATCH 18/28] Fix patch failures --- .../torchvision-0.26.0-rocm-vision-duplicate-symbol.patch | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/repos/spack_repo/builtin/packages/py_torchvision/torchvision-0.26.0-rocm-vision-duplicate-symbol.patch b/repos/spack_repo/builtin/packages/py_torchvision/torchvision-0.26.0-rocm-vision-duplicate-symbol.patch index d8dc916eda2..1a704bfab8e 100644 --- a/repos/spack_repo/builtin/packages/py_torchvision/torchvision-0.26.0-rocm-vision-duplicate-symbol.patch +++ b/repos/spack_repo/builtin/packages/py_torchvision/torchvision-0.26.0-rocm-vision-duplicate-symbol.patch @@ -1,6 +1,8 @@ ---- setup.py.orig 2026-08-09 13:59:24.211961011 -0700 -+++ setup.py 2026-08-09 14:01:13.976910706 -0700 -@@ -165,6 +165,7 @@ +diff --git a/setup.py b/setup.py +index 7bf83fc..dff1d89 100644 +--- a/setup.py ++++ b/setup.py +@@ -165,6 +165,7 @@ def make_C_extension(): project_directory=str(ROOT_DIR), output_directory=str(ROOT_DIR), includes="torchvision/csrc/ops/cuda/*", From 61f0596dd596ce25c1da61162b03d262aa3e1417 Mon Sep 17 00:00:00 2001 From: renjithravindrankannath Date: Tue, 11 Aug 2026 14:37:07 -0400 Subject: [PATCH 19/28] Update torchvision with rocm dependecies --- .../builtin/packages/py_torch/package.py | 10 +++++++++- .../builtin/packages/py_torchvision/package.py | 17 +++++++++++++++++ ...on-0.26.0-rocm-vision-duplicate-symbol.patch | 15 +++++++++++++-- 3 files changed, 39 insertions(+), 3 deletions(-) diff --git a/repos/spack_repo/builtin/packages/py_torch/package.py b/repos/spack_repo/builtin/packages/py_torch/package.py index 90aedb118b6..bded37964c0 100644 --- a/repos/spack_repo/builtin/packages/py_torch/package.py +++ b/repos/spack_repo/builtin/packages/py_torch/package.py @@ -751,8 +751,16 @@ def enable_or_disable(variant, keyword="USE", var=None): # So libtorch_hip.so and dependents find ROCm/runtime libs at runtime and # during binary cache relocation (avoids "=> not found" for e.g. # libamdhip64.so.6, libhsa-runtime64.so.1). - for lib_dir in [self.spec["hip"].prefix.lib, self.spec["hsa-rocr-dev"].prefix.lib]: + for lib_dir in [ + self.spec["hip"].prefix.lib, + self.spec["hsa-rocr-dev"].prefix.lib, + self.spec["rocm-smi-lib"].prefix.lib, + ]: env.append_flags("LDFLAGS", "-Wl,-rpath," + lib_dir) + # Link to rocm-smi-lib which provides rsmi_* symbols used by libtorch_hip.so + env.append_flags( + "LDFLAGS", "-L{} -lrocm_smi64".format(self.spec["rocm-smi-lib"].prefix.lib) + ) env.set("PYTORCH_ROCM_ARCH", ";".join(self.spec.variants["amdgpu_target"].value)) env.set("HSA_PATH", self.spec["hsa-rocr-dev"].prefix) env.set("ROCBLAS_PATH", self.spec["rocblas"].prefix) diff --git a/repos/spack_repo/builtin/packages/py_torchvision/package.py b/repos/spack_repo/builtin/packages/py_torchvision/package.py index 31d3855bdcb..4e321156399 100644 --- a/repos/spack_repo/builtin/packages/py_torchvision/package.py +++ b/repos/spack_repo/builtin/packages/py_torchvision/package.py @@ -210,6 +210,16 @@ def setup_build_environment(self, env: EnvironmentModifications) -> None: include.extend(query.headers.directories) library.extend(query.libs.directories) + # When building with ROCm, add all ROCm library include paths for HIP compilation + # PyTorch headers transitively include many ROCm headers that extensions need + if "^py-torch+rocm" in self.spec: + rocm_deps = ["rocthrust", "rocprim", "hipsparse", "hipblas", "hipblas-common", + "hipblaslt", "hipfft", "hiprand", "hipsolver", "rocblas", + "rocsparse", "rocsolver", "rocfft"] + for dep in rocm_deps: + if dep in self.spec: + include.append(self.spec[dep].prefix.include) + # CONTRIBUTING.md says to use TORCHVISION_INCLUDE and TORCHVISION_LIBRARY, but # these do not work for older releases. Build uses a mix of Spack's compiler wrapper # and the actual compiler, so this is needed to get parts of the build working. @@ -218,3 +228,10 @@ def setup_build_environment(self, env: EnvironmentModifications) -> None: env.set("TORCHVISION_LIBRARY", ":".join(library)) env.set("CPATH", ":".join(include)) env.set("LIBRARY_PATH", ":".join(library)) + + # For ROCm builds, also prepend ROCm includes to ensure hipcc can find them + if "^py-torch+rocm" in self.spec: + for dep in rocm_deps: + if dep in self.spec: + env.prepend_path("CPATH", self.spec[dep].prefix.include) + env.prepend_path("CPLUS_INCLUDE_PATH", self.spec[dep].prefix.include) diff --git a/repos/spack_repo/builtin/packages/py_torchvision/torchvision-0.26.0-rocm-vision-duplicate-symbol.patch b/repos/spack_repo/builtin/packages/py_torchvision/torchvision-0.26.0-rocm-vision-duplicate-symbol.patch index 1a704bfab8e..21cd51f0168 100644 --- a/repos/spack_repo/builtin/packages/py_torchvision/torchvision-0.26.0-rocm-vision-duplicate-symbol.patch +++ b/repos/spack_repo/builtin/packages/py_torchvision/torchvision-0.26.0-rocm-vision-duplicate-symbol.patch @@ -1,8 +1,19 @@ diff --git a/setup.py b/setup.py -index 7bf83fc..dff1d89 100644 +index 7bf83fc..a9c8e5d 100644 --- a/setup.py +++ b/setup.py -@@ -165,6 +165,7 @@ def make_C_extension(): +@@ -153,6 +153,10 @@ def make_C_extension(): + + list(CSRS_DIR.glob("ops/quantized/cpu/*.cpp")) + ) + mps_sources = list(CSRS_DIR.glob("ops/mps/*.mm")) ++ ++ # Exclude vision_hip.cpp from main sources when building with ROCm to avoid duplicate symbols ++ if IS_ROCM: ++ sources = [s for s in sources if "vision_hip.cpp" not in str(s)] + + if IS_ROCM: + from torch.utils.hipify import hipify_python +@@ -161,6 +165,7 @@ def make_C_extension(): project_directory=str(ROOT_DIR), output_directory=str(ROOT_DIR), includes="torchvision/csrc/ops/cuda/*", From 1f64d43d566a5b2e08db7ca87459d2ebc3372e3c Mon Sep 17 00:00:00 2001 From: renjithravindrankannath <94420380+renjithravindrankannath@users.noreply.github.com> Date: Tue, 11 Aug 2026 19:13:56 +0000 Subject: [PATCH 20/28] [@spackbot] updating style on behalf of renjithravindrankannath --- .../builtin/packages/py_torchvision/package.py | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/repos/spack_repo/builtin/packages/py_torchvision/package.py b/repos/spack_repo/builtin/packages/py_torchvision/package.py index 4e321156399..ee66a1335a6 100644 --- a/repos/spack_repo/builtin/packages/py_torchvision/package.py +++ b/repos/spack_repo/builtin/packages/py_torchvision/package.py @@ -213,9 +213,21 @@ def setup_build_environment(self, env: EnvironmentModifications) -> None: # When building with ROCm, add all ROCm library include paths for HIP compilation # PyTorch headers transitively include many ROCm headers that extensions need if "^py-torch+rocm" in self.spec: - rocm_deps = ["rocthrust", "rocprim", "hipsparse", "hipblas", "hipblas-common", - "hipblaslt", "hipfft", "hiprand", "hipsolver", "rocblas", - "rocsparse", "rocsolver", "rocfft"] + rocm_deps = [ + "rocthrust", + "rocprim", + "hipsparse", + "hipblas", + "hipblas-common", + "hipblaslt", + "hipfft", + "hiprand", + "hipsolver", + "rocblas", + "rocsparse", + "rocsolver", + "rocfft", + ] for dep in rocm_deps: if dep in self.spec: include.append(self.spec[dep].prefix.include) From edbc83d877d5db54f5113b4c5487002569a284d7 Mon Sep 17 00:00:00 2001 From: renjithravindrankannath Date: Tue, 11 Aug 2026 20:29:08 -0400 Subject: [PATCH 21/28] Enabling py-torch-geometric --- stacks/ml-linux-x86_64/spack.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/stacks/ml-linux-x86_64/spack.yaml b/stacks/ml-linux-x86_64/spack.yaml index 030bfa703c0..6e35cc5b934 100644 --- a/stacks/ml-linux-x86_64/spack.yaml +++ b/stacks/ml-linux-x86_64/spack.yaml @@ -34,14 +34,14 @@ spack: - py-lightning - py-pytorch-lightning - py-segmentation-models-pytorch - #- py-timm + - py-timm - py-torch - #- py-torch-geometric + - py-torch-geometric #- py-torch-nvidia-apex #- py-torchaudio #- py-torchgeo - #- py-torchmetrics - #- py-torchvision + - py-torchmetrics + - py-torchvision #- py-vector-quantize-pytorch - scikit_learn_specs: - py-scikit-learn From f52abe37a9e50924f658345c52bcfc662f167f5b Mon Sep 17 00:00:00 2001 From: renjithravindrankannath Date: Wed, 12 Aug 2026 21:06:58 -0400 Subject: [PATCH 22/28] Enable py-torchaudio --- stacks/ml-linux-x86_64/spack.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stacks/ml-linux-x86_64/spack.yaml b/stacks/ml-linux-x86_64/spack.yaml index 6e35cc5b934..0d81ca1a456 100644 --- a/stacks/ml-linux-x86_64/spack.yaml +++ b/stacks/ml-linux-x86_64/spack.yaml @@ -38,7 +38,7 @@ spack: - py-torch - py-torch-geometric #- py-torch-nvidia-apex - #- py-torchaudio + - py-torchaudio #- py-torchgeo - py-torchmetrics - py-torchvision From bf1cb0d111e9813aeb8287817c5c165898b6e565 Mon Sep 17 00:00:00 2001 From: renjithravindrankannath Date: Thu, 13 Aug 2026 13:41:51 -0400 Subject: [PATCH 23/28] Enable torchgeo and vector-quantize-pytorch --- stacks/ml-linux-x86_64/spack.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/stacks/ml-linux-x86_64/spack.yaml b/stacks/ml-linux-x86_64/spack.yaml index 0d81ca1a456..2a6283e615c 100644 --- a/stacks/ml-linux-x86_64/spack.yaml +++ b/stacks/ml-linux-x86_64/spack.yaml @@ -39,10 +39,10 @@ spack: - py-torch-geometric #- py-torch-nvidia-apex - py-torchaudio - #- py-torchgeo + - py-torchgeo - py-torchmetrics - py-torchvision - #- py-vector-quantize-pytorch + - py-vector-quantize-pytorch - scikit_learn_specs: - py-scikit-learn - tensorboard_specs: From 9f816704a1bac0ec64370b96bb283e56c3a95d87 Mon Sep 17 00:00:00 2001 From: renjithravindrankannath Date: Thu, 13 Aug 2026 19:17:15 -0400 Subject: [PATCH 24/28] Enable py-torch-nvidia-apex --- stacks/ml-linux-x86_64/spack.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stacks/ml-linux-x86_64/spack.yaml b/stacks/ml-linux-x86_64/spack.yaml index 2a6283e615c..142bc9ea884 100644 --- a/stacks/ml-linux-x86_64/spack.yaml +++ b/stacks/ml-linux-x86_64/spack.yaml @@ -37,7 +37,7 @@ spack: - py-timm - py-torch - py-torch-geometric - #- py-torch-nvidia-apex + - py-torch-nvidia-apex - py-torchaudio - py-torchgeo - py-torchmetrics From a540774259632e2953a2fae3fdfc9f7d0ff0ff46 Mon Sep 17 00:00:00 2001 From: renjithravindrankannath Date: Fri, 14 Aug 2026 03:09:03 -0400 Subject: [PATCH 25/28] Moving py-torch-nvidia-apex to ml-linux-x86_64-cuda group --- stacks/ml-linux-x86_64/spack.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stacks/ml-linux-x86_64/spack.yaml b/stacks/ml-linux-x86_64/spack.yaml index 142bc9ea884..530b720931b 100644 --- a/stacks/ml-linux-x86_64/spack.yaml +++ b/stacks/ml-linux-x86_64/spack.yaml @@ -37,7 +37,6 @@ spack: - py-timm - py-torch - py-torch-geometric - - py-torch-nvidia-apex - py-torchaudio - py-torchgeo - py-torchmetrics @@ -84,6 +83,7 @@ spack: - $scikit_learn_specs - $tensorboard_specs - $tensorflow_specs + - py-torch-nvidia-apex # xgboost requires older cuda, which doesn't support newer GCC # - $xgboost_specs override: From 5e444ca604de0650bf6132e55ddb4d7475ffcdb8 Mon Sep 17 00:00:00 2001 From: renjithravindrankannath Date: Wed, 19 Aug 2026 22:27:14 -0400 Subject: [PATCH 26/28] No kineto by default for rocm only. Adding rocm-smi-lib dependency --- repos/spack_repo/builtin/packages/py_torch/package.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/repos/spack_repo/builtin/packages/py_torch/package.py b/repos/spack_repo/builtin/packages/py_torch/package.py index 469baea806b..7a26cd09116 100644 --- a/repos/spack_repo/builtin/packages/py_torch/package.py +++ b/repos/spack_repo/builtin/packages/py_torch/package.py @@ -98,7 +98,8 @@ class PyTorch(PythonPackage, CudaPackage, ROCmPackage): variant("cudnn", default=not is_darwin, description="Use cuDNN", when="+cuda") variant("fbgemm", default=True, description="Use FBGEMM (quantized 8-bit server operators)") variant("kineto", default=True, description="Use Kineto profiling library", when="@1.8:2.10") - variant("kineto", default=False, description="Disable Kineto from 2.11 on rocm", when="@2.11:") + variant("kineto", default=True, description="Use Kineto profiling library", when="@2.11: ~rocm") + variant("kineto", default=False, description="Disable Kineto from 2.11 on ROCm", when="@2.11: +rocm") variant("magma", default=not is_darwin, description="Use MAGMA", when="+cuda") variant("metal", default=is_darwin, description="Use Metal for Caffe2 iOS build") variant( @@ -370,9 +371,9 @@ class PyTorch(PythonPackage, CudaPackage, ROCmPackage): depends_on("amdsmi", when="@2.12") depends_on("rocfft") depends_on("rocblas") - depends_on("miopen-hip") depends_on("composable-kernel") depends_on("hipblaslt") + depends_on("rocm-smi-lib") # Ensure hipblaslt version for 2.9+ depends_on("hipblaslt@7.0:", when="@2.9:") depends_on("rocminfo") From b86d19d89599cf94b1d6e87d985fc4e83854a20e Mon Sep 17 00:00:00 2001 From: renjithravindrankannath <94420380+renjithravindrankannath@users.noreply.github.com> Date: Thu, 20 Aug 2026 02:33:00 +0000 Subject: [PATCH 27/28] [@spackbot] updating style on behalf of renjithravindrankannath --- repos/spack_repo/builtin/packages/py_torch/package.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/repos/spack_repo/builtin/packages/py_torch/package.py b/repos/spack_repo/builtin/packages/py_torch/package.py index 7a26cd09116..5d51ef688c7 100644 --- a/repos/spack_repo/builtin/packages/py_torch/package.py +++ b/repos/spack_repo/builtin/packages/py_torch/package.py @@ -98,8 +98,15 @@ class PyTorch(PythonPackage, CudaPackage, ROCmPackage): variant("cudnn", default=not is_darwin, description="Use cuDNN", when="+cuda") variant("fbgemm", default=True, description="Use FBGEMM (quantized 8-bit server operators)") variant("kineto", default=True, description="Use Kineto profiling library", when="@1.8:2.10") - variant("kineto", default=True, description="Use Kineto profiling library", when="@2.11: ~rocm") - variant("kineto", default=False, description="Disable Kineto from 2.11 on ROCm", when="@2.11: +rocm") + variant( + "kineto", default=True, description="Use Kineto profiling library", when="@2.11: ~rocm" + ) + variant( + "kineto", + default=False, + description="Disable Kineto from 2.11 on ROCm", + when="@2.11: +rocm", + ) variant("magma", default=not is_darwin, description="Use MAGMA", when="+cuda") variant("metal", default=is_darwin, description="Use Metal for Caffe2 iOS build") variant( From 5f1867e6cf3986d159e0f25b021dcb6ae5fde76b Mon Sep 17 00:00:00 2001 From: renjithravindrankannath Date: Thu, 20 Aug 2026 12:33:17 -0400 Subject: [PATCH 28/28] Adding miopen-hip that was mistakenly removed --- repos/spack_repo/builtin/packages/py_torch/package.py | 1 + 1 file changed, 1 insertion(+) diff --git a/repos/spack_repo/builtin/packages/py_torch/package.py b/repos/spack_repo/builtin/packages/py_torch/package.py index 5d51ef688c7..80aa14c82e5 100644 --- a/repos/spack_repo/builtin/packages/py_torch/package.py +++ b/repos/spack_repo/builtin/packages/py_torch/package.py @@ -378,6 +378,7 @@ class PyTorch(PythonPackage, CudaPackage, ROCmPackage): depends_on("amdsmi", when="@2.12") depends_on("rocfft") depends_on("rocblas") + depends_on("miopen-hip") depends_on("composable-kernel") depends_on("hipblaslt") depends_on("rocm-smi-lib")