From aa1864cf11b63612eb82404be5a34eee5dc9b3e5 Mon Sep 17 00:00:00 2001 From: renjithravindrankannath Date: Fri, 8 May 2026 13:43:30 -0400 Subject: [PATCH 1/5] To check rocm py_torch ci build --- .ci/gitlab/configs/linux/ci.yaml | 2 ++ stacks/ml-linux-x86_64/spack.yaml | 9 ++++----- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/.ci/gitlab/configs/linux/ci.yaml b/.ci/gitlab/configs/linux/ci.yaml index 9add0f89153..67d2b921d5c 100644 --- a/.ci/gitlab/configs/linux/ci.yaml +++ b/.ci/gitlab/configs/linux/ci.yaml @@ -13,6 +13,8 @@ ci: - wrf build-job: tags: [ "spack", "huge" ] + # 24h for long ROCm/ML rebuilds; GitLab project/runner max must allow this + timeout: 1440 minutes variables: CI_JOB_SIZE: huge SPACK_BUILD_JOBS: "12" diff --git a/stacks/ml-linux-x86_64/spack.yaml b/stacks/ml-linux-x86_64/spack.yaml index 50a030f9d9f..59d7fd1c164 100644 --- a/stacks/ml-linux-x86_64/spack.yaml +++ b/stacks/ml-linux-x86_64/spack.yaml @@ -115,13 +115,11 @@ spack: - [$keras_specs] exclude: - py-keras backend=torch - # Does not yet support Spack-installed ROCm - # - $torch_specs + - $torch_specs - $scikit_learn_specs - $tensorboard_specs - $tensorflow_specs - # Does not yet support Spack-installed ROCm - # - $xgboost_specs + - $xgboost_specs override: packages: all: @@ -139,8 +137,9 @@ spack: - target=x86_64_v3 - ~cuda - +rocm + - ~distributed - amdgpu_target=gfx90a - - ~flash_attention + - +flash_attention - "%c,cxx=gcc" ci: From dd71e8daabec80b8d6b80bdf4e782cdcd446fde4 Mon Sep 17 00:00:00 2001 From: renjithravindrankannath Date: Mon, 11 May 2026 17:00:22 -0400 Subject: [PATCH 2/5] removing unwanted $ in the spec string --- stacks/ml-linux-x86_64/spack.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stacks/ml-linux-x86_64/spack.yaml b/stacks/ml-linux-x86_64/spack.yaml index 59d7fd1c164..aae674943fb 100644 --- a/stacks/ml-linux-x86_64/spack.yaml +++ b/stacks/ml-linux-x86_64/spack.yaml @@ -115,7 +115,7 @@ spack: - [$keras_specs] exclude: - py-keras backend=torch - - $torch_specs + - torch_specs - $scikit_learn_specs - $tensorboard_specs - $tensorflow_specs From d7710b7f3905cb7209643cfd1d565dcab7f7301b Mon Sep 17 00:00:00 2001 From: renjithravindrankannath Date: Mon, 11 May 2026 17:33:29 -0400 Subject: [PATCH 3/5] exclude is not required --- stacks/ml-linux-x86_64/spack.yaml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/stacks/ml-linux-x86_64/spack.yaml b/stacks/ml-linux-x86_64/spack.yaml index aae674943fb..04cd2007c65 100644 --- a/stacks/ml-linux-x86_64/spack.yaml +++ b/stacks/ml-linux-x86_64/spack.yaml @@ -113,9 +113,7 @@ spack: - $jax_specs - matrix: - [$keras_specs] - exclude: - - py-keras backend=torch - - torch_specs + - $torch_specs - $scikit_learn_specs - $tensorboard_specs - $tensorflow_specs From 55cfb81fe73fa6e9f6c7405e2953bf600e46a222 Mon Sep 17 00:00:00 2001 From: renjithravindrankannath Date: Tue, 12 May 2026 16:18:08 -0400 Subject: [PATCH 4/5] Updating aotriton-llvm version dependency --- repos/spack_repo/builtin/packages/aotriton/package.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/repos/spack_repo/builtin/packages/aotriton/package.py b/repos/spack_repo/builtin/packages/aotriton/package.py index ff1ce88e3ac..ec70a55bc49 100644 --- a/repos/spack_repo/builtin/packages/aotriton/package.py +++ b/repos/spack_repo/builtin/packages/aotriton/package.py @@ -61,7 +61,7 @@ class Aotriton(CMakePackage): # build llvm version with mlir with the commit that matches inside the llvm-hash.txt depends_on("aotriton-llvm@0.10", when="@0.10b:") - depends_on("aotriton-llvm@0.9", when="@0.9b") + depends_on("aotriton-llvm@0.9", when="@0.9b:0.9.2b") depends_on("aotriton-llvm@0.8", when="@0.8b") conflicts("^openssl@3.3.0") From d9a7c5c12851c7b8d61e4a2d3299fe6361dec385 Mon Sep 17 00:00:00 2001 From: renjithravindrankannath Date: Tue, 12 May 2026 16:31:16 -0400 Subject: [PATCH 5/5] hwloc with older rocm versions conflicts with llvm@20 while linking --- repos/spack_repo/builtin/packages/py_llvmlite/package.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/repos/spack_repo/builtin/packages/py_llvmlite/package.py b/repos/spack_repo/builtin/packages/py_llvmlite/package.py index a7c26e39c73..20453e596f3 100644 --- a/repos/spack_repo/builtin/packages/py_llvmlite/package.py +++ b/repos/spack_repo/builtin/packages/py_llvmlite/package.py @@ -61,6 +61,10 @@ class PyLlvmlite(PythonPackage): depends_on("llvm@14", when="@0.41:0.43") depends_on("llvm@11:14", when="@0.40") depends_on("llvm@11", when="@0.37:0.39") + # When the spec has both llvm@20 and hwloc+rocm (e.g. llvm build-depends on hwloc), + # hwloc with older rocm versions conflicts with llvm@20 while linking. + # Hence adding hwloc~rocm dependency for llvmlite@0.45:. + depends_on("hwloc ~rocm", when="@0.45:") for t in [ "arm:", "ppc:",