From bc03d258e6f4d210c71ea72e297599a65d86c8b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CHenrique?= Date: Thu, 16 Apr 2026 18:02:48 +0100 Subject: [PATCH 1/3] added MI300X build script Linting More linting --- scripts/build_csd3_mi300x.sh | 168 +++++++++++++++++++++++++++++++++++ 1 file changed, 168 insertions(+) create mode 100755 scripts/build_csd3_mi300x.sh diff --git a/scripts/build_csd3_mi300x.sh b/scripts/build_csd3_mi300x.sh new file mode 100755 index 0000000..9dec85b --- /dev/null +++ b/scripts/build_csd3_mi300x.sh @@ -0,0 +1,168 @@ +#!/bin/bash +#SBATCH --job-name=moose_hip_build_ampere +#SBATCH --output=moose_hip_build_log.txt +#SBATCH --error=moose_hip_build_err.txt +#SBATCH --account= +#SBATCH --partition=ukaea-mi300x +#SBATCH --gres=gpu:1 +#SBATCH --nodes=1 +#SBATCH --ntasks=1 +#SBATCH --time=04:00:00 + +## This script builds MOOSE with MFEM configured for HIP support. It is designed for the MI300X partition on the CSD3 machine +## To run, substitute with your project account before submitting the job + +export METHOD=dbg +export MOOSE_JOBS=32 +export HIP_ARCH=gfx942 + +module purge +module load rhel9/default-amdgpu +module load openmpi/5.0.9/llvm-amdgpu-7.1.1/awm5kxiz +module load gcc/15.2.0 + +export CC=mpicc +export CXX=mpicxx +export FC=mpifort +export HIPCXXFLAGS="-std=c++17" +export HIPFLAGS="-std=c++17" +export NO_STOP_MESSAGE=1 +export ROCM_PATH=/usr/local/software/spack/csd3-rhel9/externals/rocm-7.1.1 +export LIBRARY_PATH="${ROCM_PATH}/llvm/lib:${LIBRARY_PATH}" +export LD_LIBRARY_PATH="${ROCM_PATH}/llvm/lib:${LD_LIBRARY_PATH}" +export PATH="${ROCM_PATH}/llvm/bin:${PATH}" + +BUILD_DIR=$(pwd)/moose_hip +export BUILD_DIR + +mkdir -p "${BUILD_DIR}" +cd "${BUILD_DIR}" || exit + +git clone --depth=2 https://github.com/spack/spack.git +# shellcheck source=/dev/null +source spack/share/spack/setup-env.sh +spack install gcc +spack env create amd-env +spack env activate amd-env + +cat >>"$(spack location -e amd-env)/spack.yaml" <<'EOF' + packages: + openmpi: + externals: + - spec: openmpi@5.0.9 + modules: + - openmpi/5.0.9 + buildable: false +EOF + +spack add cmake \ + python \ + py-pandas \ + py-pyaml \ + py-packaging \ + py-jinja2 \ + py-setuptools \ + py-deepdiff \ + py-xmltodict \ + openblas \ + metis \ + parmetis \ + libtirpc \ + hdf5+mpi+hl + +spack concretize -f +spack install -j ${MOOSE_JOBS} + +git clone https://github.com/xiaoyeli/superlu_dist.git +cd superlu_dist || exit +mkdir build +cd build || exit + +cmake .. \ + -DCMAKE_C_COMPILER=mpicc \ + -DCMAKE_CXX_COMPILER=mpicxx \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX=../install \ + -DTPL_ENABLE_HIPLIB=TRUE \ + -DCMAKE_HIP_ARCHITECTURES=${HIP_ARCH} \ + -DTPL_ENABLE_INTERNAL_BLASLIB=OFF \ + -DTPL_BLAS_LIBRARIES="$(spack location -i openblas)/lib/libopenblas.so" \ + -DTPL_ENABLE_LAPACKLIB=ON \ + -DTPL_LAPACK_LIBRARIES="$(spack location -i openblas)/lib/libopenblas.so" \ + -DTPL_PARMETIS_INCLUDE_DIRS="$(spack location -i parmetis)/include;$(spack location -i metis)/include" \ + -DTPL_PARMETIS_LIBRARIES="$(spack location -i parmetis)/lib/libparmetis.so;$(spack location -i metis)/lib/libmetis.so" \ + -DBUILD_SHARED_LIBS=ON + +make -j ${MOOSE_JOBS} +make install +SUPERLU_DIR=$(realpath ../install) +export SUPERLU_DIR + +cd "${BUILD_DIR}" || exit + +git clone https://github.com/hypre-space/hypre.git +cd hypre || exit +mkdir build +cd build || exit + +cmake ../src/ \ + -DHYPRE_ENABLE_HIP=ON \ + -DCMAKE_HIP_ARCHITECTURES=${HIP_ARCH} \ + -DHYPRE_ENABLE_GPU_AWARE_MPI=ON \ + -DMPI_INCLUDE_DIR="$(spack location -i openmpi)/include" \ + -DHYPRE_BUILD_UMPIRE=ON \ + -DCMAKE_INSTALL_PREFIX=../install + +make -j ${MOOSE_JOBS} +make install +HYPRE_DIR=$(realpath ../install) +export HYPRE_DIR + +cd "${BUILD_DIR}" || exit +git clone https://github.com/Heinrich-BR/moose.git +cd moose || exit +git switch HenriqueBR/lld_llvm_fix # Only here until fix gets merged + +./scripts/update_and_rebuild_petsc.sh --with-hip \ + --with-hip-arch=${HIP_ARCH} \ + --with-metis-include="$(spack location -i metis)/include" \ + --with-metis-lib="$(spack location -i metis)/lib/libmetis.so" \ + --with-parmetis-include="$(spack location -i parmetis)/include" \ + --with-parmetis-lib="$(spack location -i parmetis)/lib/libparmetis.so" \ + --with-hdf5-include="$(spack location -i hdf5)/include" \ + --with-hdf5-lib="$(spack location -i hdf5)/lib/libhdf5.so" \ + --download-openblas-make-options='DYNAMIC_ARCH=1 FC=gfortran' \ + --download-hdf5=0 \ + --download-metis=0 \ + --download-parmetis=0 \ + --download-kokkos=0 \ + --download-kokkos-kernels=0 \ + --download-strumpack=0 \ + --download-superlu_dist=0 \ + --download-hypre=0 \ + --download-openblas=1 \ + --with-hypre=1 \ + --with-hypre-dir="${HYPRE_DIR}" \ + --with-superlu_dist=1 \ + --with-superlu_dist-dir="${SUPERLU_DIR}" \ + --with-64-bit-indices=0 \ + --CXXFLAGS+="-include cstring" \ + LIBS="-lhiprtc" + +LIBTIRPC_DIR=$(spack location -i libtirpc) +export LIBTIRPC_DIR +export CPPFLAGS="-I${LIBTIRPC_DIR}/include/tirpc ${CPPFLAGS}" +export LDFLAGS="-L${LIBTIRPC_DIR}/lib -ltirpc ${LDFLAGS}" +export PKG_CONFIG_PATH="${LIBTIRPC_DIR}/lib/pkgconfig:${PKG_CONFIG_PATH}" + +./scripts/update_and_rebuild_libmesh.sh CC=mpicc CXX=mpicxx FC=mpifort F77=mpifort +./scripts/update_and_rebuild_conduit.sh +./scripts/update_and_rebuild_wasp.sh +./scripts/update_and_rebuild_mfem.sh -DMFEM_USE_HIP=YES -DHIP_ARCH=${HIP_ARCH} -DSuperLUDist_DIR="${SUPERLU_DIR}" -DHDF5_DIR="$(spack location -i hdf5)" + +./configure --with-mfem + +cd framework || exit +make -j ${MOOSE_JOBS} +cd ../test || exit +make -j ${MOOSE_JOBS} From 9dd658869036af38a9bbe3ca760d8e21eb9a658b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CHenrique?= Date: Fri, 17 Apr 2026 10:11:04 +0100 Subject: [PATCH 2/3] Changed method to opt --- scripts/build_csd3_mi300x.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/build_csd3_mi300x.sh b/scripts/build_csd3_mi300x.sh index 9dec85b..f1cc1d5 100755 --- a/scripts/build_csd3_mi300x.sh +++ b/scripts/build_csd3_mi300x.sh @@ -12,7 +12,7 @@ ## This script builds MOOSE with MFEM configured for HIP support. It is designed for the MI300X partition on the CSD3 machine ## To run, substitute with your project account before submitting the job -export METHOD=dbg +export METHOD=opt export MOOSE_JOBS=32 export HIP_ARCH=gfx942 From 0bfc0a360f8ac6c3702803e88a2a06804bea3400 Mon Sep 17 00:00:00 2001 From: Henrique BR <60271325+Heinrich-BR@users.noreply.github.com> Date: Tue, 18 Aug 2026 09:41:09 +0100 Subject: [PATCH 3/3] Apply suggestions from code review Co-authored-by: Sean Baccas --- scripts/build_csd3_mi300x.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/build_csd3_mi300x.sh b/scripts/build_csd3_mi300x.sh index f1cc1d5..f5999e8 100755 --- a/scripts/build_csd3_mi300x.sh +++ b/scripts/build_csd3_mi300x.sh @@ -38,10 +38,10 @@ export BUILD_DIR mkdir -p "${BUILD_DIR}" cd "${BUILD_DIR}" || exit -git clone --depth=2 https://github.com/spack/spack.git +git clone --depth=1 --branch v1.1.1 https://github.com/spack/spack.git # shellcheck source=/dev/null source spack/share/spack/setup-env.sh -spack install gcc +spack install gcc@15.2.0 spack env create amd-env spack env activate amd-env