diff --git a/tests/integration/cuda/ci-test-integration.sh b/tests/integration/cuda/ci-test-integration.sh index f3bfe2bc9..37c6f138f 100755 --- a/tests/integration/cuda/ci-test-integration.sh +++ b/tests/integration/cuda/ci-test-integration.sh @@ -4,19 +4,50 @@ set -e echo "Run Integration Test ${PROTEUS_CI_INTEGRATION_TEST}" -# Use pre-installed LLVM 20 on matrix. -export LLVM_INSTALL_DIR=/usr/lib64/llvm20 -echo "LLVM_INSTALL_DIR ${LLVM_INSTALL_DIR}" +# LLVM/Clang version installed from conda-forge (replaces the previously assumed +# system install at /usr/lib64/llvm20). +PROTEUS_CI_LLVM_VERSION=20.1.8 +PYTHON_VERSION=3.12 # Load cuda module. ml load cuda/12 +install_miniforge() { + local miniforge_dir="${1}" + + mkdir -p "${miniforge_dir}" + wget -q --tries=5 --retry-connrefused --wait=5 \ + "https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-Linux-$(uname -m).sh" \ + -O "${miniforge_dir}/miniforge.sh" + bash "${miniforge_dir}/miniforge.sh" -b -u -p "${miniforge_dir}" + rm "${miniforge_dir}/miniforge.sh" + source "${miniforge_dir}/etc/profile.d/conda.sh" +} + export TEST_DIR=${CI_PROJECT_DIR}/tests/integration/cuda/${PROTEUS_CI_INTEGRATION_TEST} mkdir -p /tmp/proteus-ci-$(basename ${TEST_DIR})-${CI_JOB_ID} pushd /tmp/proteus-ci-$(basename ${TEST_DIR})-${CI_JOB_ID} -rm -rf build-proteus install-proteus build install +rm -rf build-proteus install-proteus build install miniforge3 + +# Install Clang/LLVM through conda. +install_miniforge miniforge3 +# Use an older version of gcc to avoid issues with detecting Clang as the CUDA +# compiler. +conda create -y -q -n proteus --override-channels -c conda-forge \ + python=${PYTHON_VERSION} clang=${PROTEUS_CI_LLVM_VERSION} clangxx=${PROTEUS_CI_LLVM_VERSION} \ + pip pybind11 clangdev=${PROTEUS_CI_LLVM_VERSION} llvmdev=${PROTEUS_CI_LLVM_VERSION} \ + lit=${PROTEUS_CI_LLVM_VERSION} mlir=${PROTEUS_CI_LLVM_VERSION} gcc=12 gxx=12 +conda activate proteus + +export LLVM_INSTALL_DIR=$(llvm-config --prefix) +echo "LLVM_INSTALL_DIR ${LLVM_INSTALL_DIR}" + +# Make the conda LLVM discoverable by the test sub-project's find_package(proteus) +# call. CMake reads CMAKE_PREFIX_PATH from the environment. +export CMAKE_PREFIX_PATH="${CONDA_PREFIX}:${CONDA_PREFIX}/lib/cmake" + bash ${TEST_DIR}/ci-build-and-run.sh popd diff --git a/tests/integration/cuda/make-shared_proteus/Makefile b/tests/integration/cuda/make-shared_proteus/Makefile index e6be8f652..8a8a790eb 100644 --- a/tests/integration/cuda/make-shared_proteus/Makefile +++ b/tests/integration/cuda/make-shared_proteus/Makefile @@ -9,6 +9,7 @@ all: -Wl,-rpath,${PROTEUS_INSTALL_DIR}/lib64 \ -l:libproteus.so \ -l:libproteus_cudart_builtins.a \ + -L${CUDA_HOME}/lib64/stubs \ -L${CUDA_HOME}/lib64 \ -lcuda \ -lcudart diff --git a/tests/integration/cuda/make-static_proteus/Makefile b/tests/integration/cuda/make-static_proteus/Makefile index c98920aef..f89cc060a 100644 --- a/tests/integration/cuda/make-static_proteus/Makefile +++ b/tests/integration/cuda/make-static_proteus/Makefile @@ -10,6 +10,7 @@ all: -l:libproteus.a \ -l:libproteus_cudart_builtins.a \ ${LLVM_LIBFLAGS} \ + -L${CUDA_HOME}/lib64/stubs \ -L${CUDA_HOME}/lib64 \ -lcuda \ -lcudart \ diff --git a/tests/integration/cuda/make-static_proteus/ci-build-and-run.sh b/tests/integration/cuda/make-static_proteus/ci-build-and-run.sh index 4fdb44343..eee884cea 100644 --- a/tests/integration/cuda/make-static_proteus/ci-build-and-run.sh +++ b/tests/integration/cuda/make-static_proteus/ci-build-and-run.sh @@ -19,7 +19,7 @@ LLVM_LIBS+=" "$(${LLVM_INSTALL_DIR}/bin/llvm-config --system-libs) cp -r ${TEST_DIR}/* . export PROTEUS_INSTALL_DIR=${PWD}/install-proteus -export LLVM_LIBFLAGS="-L ${LLVM_LIBDIR} ${LLVM_LIBS} -llldCommon -llldELF" +export LLVM_LIBFLAGS="-L ${LLVM_LIBDIR} ${LLVM_LIBS}" echo "LLVM_LIBFLAGS ${LLVM_LIBFLAGS}" make -j ./main diff --git a/tests/integration/cuda/mpi-shared-cache/ci-build-and-run.sh b/tests/integration/cuda/mpi-shared-cache/ci-build-and-run.sh index 6c206f03a..e89509051 100755 --- a/tests/integration/cuda/mpi-shared-cache/ci-build-and-run.sh +++ b/tests/integration/cuda/mpi-shared-cache/ci-build-and-run.sh @@ -23,7 +23,32 @@ run_mpi() { fi } -# Build proteus with MPI enabled. +# Build proteus with MPI enabled. Conda's linker does not follow the default +# Matrix MVAPICH transitive library search paths during CMake's MPI link probe, +# so make those paths explicit without changing the MPI selected by the module. +MPI_RPATH_LINK_DIRS=() +if [[ -n "${CONDA_PREFIX}" ]]; then + MPI_RPATH_LINK_DIRS+=( + "${CONDA_PREFIX}/lib" + "${CONDA_PREFIX}/x86_64-conda-linux-gnu/lib" + ) +fi +MPI_RPATH_LINK_DIRS+=( + /lib64 + /usr/tce/backend/installations/linux-rhel8-x86_64/gcc-10.3.1/intel-oneapi-compilers-2022.1.0-43xp3r52jx2q2rkf3ctzvskqu572xbky/compiler/2022.1.0/linux/compiler/lib/intel64_lin + /usr/tce/backend/installations/linux-rhel8-x86_64/intel-2021.6.0/mvapich2-2.3.7-2575ifqlr5fbj34wdlj2fo2tmqdrehia/lib + /usr/tce/backend/installations/linux-rhel8-x86_64/gcc-8.5.0/zlib-1.2.13-pwvjwcyuqiscffdc2x2lr7ti355xwohq/lib +) +MPI_RPATH_LINK_FLAGS="" +for dir in "${MPI_RPATH_LINK_DIRS[@]}"; do + if [[ -d "${dir}" ]]; then + MPI_RPATH_LINK_FLAGS+="${MPI_RPATH_LINK_FLAGS:+ }-Wl,-rpath-link,${dir}" + fi +done +MPI_CMAKE_ARGS=( + -DCMAKE_EXE_LINKER_FLAGS="${MPI_RPATH_LINK_FLAGS}" +) + cmake -S ${CI_PROJECT_DIR} -B build-proteus \ -DLLVM_INSTALL_DIR="${LLVM_INSTALL_DIR}" \ -DCMAKE_C_COMPILER="${LLVM_INSTALL_DIR}/bin/clang" \ @@ -32,7 +57,8 @@ cmake -S ${CI_PROJECT_DIR} -B build-proteus \ -DPROTEUS_ENABLE_CUDA=on \ -DPROTEUS_ENABLE_MPI=on \ -DENABLE_TESTS=off \ - -DBUILD_SHARED=off + -DBUILD_SHARED=off \ + "${MPI_CMAKE_ARGS[@]}" pushd build-proteus make -j install @@ -44,7 +70,8 @@ cmake -S ${TEST_DIR} -B build \ -DCMAKE_CXX_COMPILER="${LLVM_INSTALL_DIR}/bin/clang++" \ -DCMAKE_CUDA_COMPILER="${LLVM_INSTALL_DIR}/bin/clang++" \ -DCMAKE_CUDA_ARCHITECTURES=90 \ - -Dproteus_DIR="${PWD}/install-proteus" + -Dproteus_DIR="${PWD}/install-proteus" \ + "${MPI_CMAKE_ARGS[@]}" pushd build make -j popd