diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index ecc944eb..63875da3 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -47,12 +47,6 @@ variables: # Required information about GitHub repository GITHUB_PROJECT_NAME: "Mneme" GITHUB_PROJECT_ORG: "Olympus-HPC" -# Set the build-and-test command. -# Nested variables are allowed and useful to customize the job command. We -# prevent variable expansion so that you can define them at job level. - JOB_CMD: - value: "scripts/gitlab/ci-build-test.sh" - expand: false # Override the pattern describing branches that will skip the "draft PR filter # test". Add protected branches here. See default value in # preliminary-ignore-draft-pr.yml. @@ -65,8 +59,7 @@ variables: stages: - prerequisites - build-and-test - - test-integration - - run-benchmarks + - integration-laghos # Template for jobs triggering a build-and-test sub-pipeline: .build-and-test: @@ -88,30 +81,15 @@ stages: forward: pipeline_variables: true -.test-integration: - stage: test-integration +.integration-laghos: + stage: integration-laghos trigger: include: - local: '.gitlab/custom-jobs-and-variables.yml' - project: 'radiuss/radiuss-shared-ci' ref: 'v2025.06.0' file: 'pipelines/${CI_MACHINE}.yml' - - local: '.gitlab/jobs/${CI_MACHINE}-integration.yml' - strategy: depend - forward: - pipeline_variables: true - -.run-benchmarks: - stage: run-benchmarks - trigger: - include: - - local: '.gitlab/custom-jobs-and-variables.yml' - - project: 'radiuss/radiuss-shared-ci' - ref: 'v2025.06.0' - file: 'pipelines/${CI_MACHINE}.yml' - # Add your jobs - # you can use a local file - - local: '.gitlab/jobs/${CI_MACHINE}-benchmarks.yml' + - local: '.gitlab/jobs/${CI_MACHINE}-laghos.yml' strategy: depend forward: pipeline_variables: true diff --git a/.gitlab/custom-jobs-and-variables.yml b/.gitlab/custom-jobs-and-variables.yml index bee64029..2677bbac 100644 --- a/.gitlab/custom-jobs-and-variables.yml +++ b/.gitlab/custom-jobs-and-variables.yml @@ -12,14 +12,6 @@ variables: # named so that we are sure to retrieve it and avoid collisions. ALLOC_NAME: ${CI_PROJECT_NAME}_ci_${CI_PIPELINE_ID} -# Ruby -# Arguments for top level allocation -# RUBY_SHARED_ALLOC: "--exclusive --reservation=ci --time=10 --nodes=1" -# Arguments for job level allocation -# RUBY_JOB_ALLOC: "--reservation=ci --nodes=1" -# Add variables that should apply to all the jobs on a machine: -# RUBY_MY_VAR: "..." - # Poodle # Arguments for top level allocation POODLE_SHARED_ALLOC: "--exclusive --partition=pdebug --time=10 --nodes=1" @@ -40,22 +32,16 @@ variables: # Tioga # Arguments for top level allocation # OPTIONAL: "-o per-resource.count=2" allows to get 2 jobs running on each node. - TIOGA_SHARED_ALLOC: "--queue=pci --exclusive --time-limit=15m --nodes=1" + TIOGA_SHARED_ALLOC: "--queue=pci --exclusive --time-limit=90m --nodes=1" # Arguments for job level allocation TIOGA_JOB_ALLOC: "--nodes=1 --begin-time=+5s" # Add variables that should apply to all the jobs on a machine: # TIOGA_MY_VAR: "..." -# Lassen uses a different job scheduler (spectrum lsf) that does not allow -# pre-allocation the same way slurm does. Arguments for job level allocation - LASSEN_JOB_ALLOC: "1 -W 60 -q pci" -# Add variables that should apply to all the jobs on a machine: -# LASSEN_MY_VAR: "..." - # Tuo # Arguments for top level allocation # OPTIONAL: "-o per-resource.count=2" allows to get 2 jobs running on each node. - TUOLUMNE_SHARED_ALLOC: "--queue=pci --exclusive --time-limit=15m --nodes=1" + TUOLUMNE_SHARED_ALLOC: "--queue=pci --exclusive --time-limit=90m --nodes=1" # Arguments for job level allocation TUOLUMNE_JOB_ALLOC: "--nodes=1 --begin-time=+5s" # Add variables that should apply to all the jobs on a machine: diff --git a/.gitlab/jobs/lassen.yml b/.gitlab/jobs/lassen.yml deleted file mode 100644 index f48996e5..00000000 --- a/.gitlab/jobs/lassen.yml +++ /dev/null @@ -1,60 +0,0 @@ -############################################################################### -# Copyright (c) 2022-23, Lawrence Livermore National Security, LLC and RADIUSS -# project contributors. See the COPYRIGHT file for details. -# -# SPDX-License-Identifier: (MIT) -############################################################################### - -# We require project to define their job command using a variable (JOB_CMD). -# In customization/gitlab-ci.yml, we encourage to define this variable as -# non-expandable, so that project can use nested variables to configure the job -# command. The caveat is that the reproducer here cannot capture the -# definition of these variables in a generic fashion. By overriding the -# following section, projects can specify the variables to define in the -# reproducer to exactly reproduce the CI build. -.lassen_reproducer_vars: - script: - - echo -e "Running on Lassen\n" - -# With GitLab CI, included files cannot be empty. -# TODO: remove when you have at least on job defined. -variables: - INCLUDED_FILE_CANNOT_BE_EMPTY: "True" - -############### -# Explanations: -############### -# RADIUSS Shared CI provides a pipeline for each machine, where a template job -# is provided. Each of your jobs must extend this template to be added to the -# list of jobs running on the associated machine. -# -# The job template then expects you to define the "JOB_CMD" variable with the -# one line command used to trigger the build and test of your project. -# -# We suggest that you set your command in such a way that you can then -# customize it per job with variables. E.g.: -# "./path/to/my_ci_script ${A_VARIABLE}" - -## Adding jobs defined by the project. -## Note: placing the extends section first allows you to override part of the -## shared implementation if needed (and if you know what you are doing). -#: -# extends: .job_on_lassen -# variables: -# : "" - -.base-job: - extends: .job_on_lassen - before_script: - - echo "=== Start CI Build and Test===" - after_script: - - rm -rf ${CI_BUILDS_DIR} - - echo "=== End CI Build and Test===" - -.build-variants: - parallel: - matrix: - - MNEME_CI_ENABLE_DEBUG: ["on", "off"] - -build-and-test-lassen: - extends: [.base-job, .build-variants] diff --git a/.gitlab/jobs/tioga-laghos.yml b/.gitlab/jobs/tioga-laghos.yml new file mode 100644 index 00000000..2734ceb2 --- /dev/null +++ b/.gitlab/jobs/tioga-laghos.yml @@ -0,0 +1,27 @@ +############################################################################### +# Copyright (c) 2022-23, Lawrence Livermore National Security, LLC and RADIUSS +# project contributors. See the COPYRIGHT file for details. +# +# SPDX-License-Identifier: (MIT) +############################################################################### + +.tioga_reproducer_vars: + script: + - echo -e "Running on tioga\n" + +.base-job: + extends: .job_on_tioga + before_script: + - echo "=== Start CI Build and Test===" + after_script: + - rm -rf ${CI_BUILDS_DIR} + - echo "=== End CI Build and Test===" + +.build-variants: + parallel: + matrix: + - MNEME_CI_ENABLE_DEBUG: ["on", "off"] + MNEME_CI_ROCM_VERSION: ["6.2.1", "6.3.1", "6.4.2"] + +integration-laghos-tioga: + extends: [.base-job, .build-variants] diff --git a/.gitlab/jobs/tuolumne-laghos.yml b/.gitlab/jobs/tuolumne-laghos.yml new file mode 100644 index 00000000..3579e69d --- /dev/null +++ b/.gitlab/jobs/tuolumne-laghos.yml @@ -0,0 +1,27 @@ +############################################################################### +# Copyright (c) 2022-23, Lawrence Livermore National Security, LLC and RADIUSS +# project contributors. See the COPYRIGHT file for details. +# +# SPDX-License-Identifier: (MIT) +############################################################################### + +.tuolumne_reproducer_vars: + script: + - echo -e "Running on tuolumne\n" + +.base-job: + extends: .job_on_tuolumne + before_script: + - echo "=== Start CI Build and Test===" + after_script: + - rm -rf ${CI_BUILDS_DIR} + - echo "=== End CI Build and Test===" + +.build-variants: + parallel: + matrix: + - MNEME_CI_ENABLE_DEBUG: ["on", "off"] + MNEME_CI_ROCM_VERSION: ["6.2.1", "6.3.1", "6.4.2"] + +integration-laghos-tuolumne: + extends: [.base-job, .build-variants] diff --git a/.gitlab/subscribed-pipelines.yml b/.gitlab/subscribed-pipelines.yml index 8cb46a0a..e6dfacc7 100644 --- a/.gitlab/subscribed-pipelines.yml +++ b/.gitlab/subscribed-pipelines.yml @@ -29,18 +29,6 @@ # Comment the jobs for machines you don’t need. ### - ## RUBY - #ruby-up-check: - # variables: - # CI_MACHINE: "ruby" - # extends: [.machine-check] - # - #ruby-build-and-test: - # variables: - # CI_MACHINE: "ruby" - # needs: [ruby-up-check] - # ## [Shared jobs scenario] - # TIOGA tioga-up-check: variables: @@ -50,25 +38,23 @@ tioga-up-check: tioga-build-and-test: variables: CI_MACHINE: "tioga" + JOB_CMD: + value: "scripts/gitlab/ci-build-test.sh" + expand: false needs: [tioga-up-check] ## [Shared jobs scenario] ## (See Umpire CI setup for an example). #needs: [tioga-up-check, generate-job-lists] extends: [.build-and-test] -# # LASSEN -# lassen-up-check: -# variables: -# CI_MACHINE: "lassen" -# extends: [.machine-check] - -# lassen-build-and-test: -# variables: -# CI_MACHINE: "lassen" -# needs: [lassen-up-check] -# extends: [.build-and-test] -# # #needs: [lassen-up-check, generate-job-lists] -# # +tioga-integration-laghos: + variables: + CI_MACHINE: "tioga" + JOB_CMD: + value: "scripts/gitlab/ci-integration-laghos.sh" + expand: false + needs: [tioga-up-check] + extends: [.integration-laghos] # TUOLUMNE tuolumne-up-check: @@ -79,8 +65,20 @@ tuolumne-up-check: tuolumne-build-and-test: variables: CI_MACHINE: "tuolumne" + JOB_CMD: + value: "scripts/gitlab/ci-build-test.sh" + expand: false needs: [tuolumne-up-check] ## [Shared jobs scenario] ## (See Umpire CI setup for an example). #needs: [tuolumne-up-check, generate-job-lists] extends: [.build-and-test] + +tuolumne-integration-laghos: + variables: + CI_MACHINE: "tuolumne" + JOB_CMD: + value: "scripts/gitlab/ci-integration-tests.sh" + expand: false + needs: [tuolumne-up-check] + extends: [.integration-laghos] diff --git a/python/mneme/db.py b/python/mneme/db.py index c9364d58..17053422 100644 --- a/python/mneme/db.py +++ b/python/mneme/db.py @@ -179,7 +179,7 @@ def add(self, src_ir: str, dst_ir: str, exp: Experiment): best_speedup, ) - if self._best > exp.exec_time: + if exp.exec_time is not None and self._best > exp.exec_time: self._best = exp.exec_time self._experiments[_hash] = exp.executed diff --git a/scripts/gitlab/ci-build-test.sh b/scripts/gitlab/ci-build-test.sh index 9b2ce266..0f6201c7 100755 --- a/scripts/gitlab/ci-build-test.sh +++ b/scripts/gitlab/ci-build-test.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash set -e @@ -6,21 +6,24 @@ temp_dir=$(pwd) #$(mktemp -d) echo "Temporary directory created at: $temp_dir" host=$(hostname) host=${host//[0-9]/} -mkdir -p ${temp_dir}/build-${host}; build_dir=${temp_dir}/build-${host} +mkdir -p ${build_dir} installDir="/dev/shm/install" +start_test=$(date +'%s') build_proteus() { echo "Building PROTEUS" - git clone --depth 1 git@github.com:Olympus-HPC/proteus.git + if [[ ! -d "proteus" ]]; then + git clone --depth 1 https://github.com/Olympus-HPC/proteus.git + fi pushd proteus PROTEUS_ENABLE_HIP=$1 PROTEUS_ENABLE_CUDA=$2 PROTEUS_INSTALL_DIR=$3 LINK_SHARED_LLVM=$4 echo "Proteus: ENABLE_HIP: $PROTEUS_ENABLE_HIP ENABLE_CUDA: $PROTEUS_ENABLE_CUDA" - mkdir build-proteus-${host} + mkdir -p build-proteus-${host} pushd build-proteus-${host} cmake .. \ -DBUILD_SHARED=Off \ @@ -42,10 +45,12 @@ build_proteus() { build_spdlog() { echo "Building SPDLOG" - git clone --depth 1 --branch v1.15.0 --single-branch https://github.com/gabime/spdlog.git + if [[ ! -d "spdlog" ]]; then + git clone --depth 1 --branch v1.15.0 --single-branch https://github.com/gabime/spdlog.git + fi pushd spdlog SPDLOG_INSTALL_DIR=$1 - mkdir build-spdlog-${host} + mkdir -p build-spdlog-${host} pushd build-spdlog-${host} cmake \ -DCMAKE_C_COMPILER=${LLVM_INSTALL_DIR}/bin/clang \ @@ -114,6 +119,14 @@ elif [[ "$SYS_TYPE" == "toss_4_x86_64_ib_cray" ]]; then ml load rocm/${MNEME_CI_ROCM_VERSION} export LLVM_INSTALL_DIR=${ROCM_PATH}/llvm echo "LLVM INSTALL DIR is ${LLVM_INSTALL_DIR}" +export ROCM_ARCH=$(rocm_agent_enumerator | sed -n 2p) + +if [ -z "${ROCM_ARCH}" ]; then + echo "ROCM_ARCH is not set or is empty" + exit +else + echo "ROCM_ARCH = ${ROCM_ARCH}" +fi build_proteus "ON" "OFF" $installDir OFF echo "After proteus Current directory is $(pwd)" diff --git a/scripts/gitlab/ci-integration-laghos.sh b/scripts/gitlab/ci-integration-laghos.sh new file mode 100755 index 00000000..31efc729 --- /dev/null +++ b/scripts/gitlab/ci-integration-laghos.sh @@ -0,0 +1,227 @@ +#!/usr/bin/env bash + +set -e + +temp_dir=$(pwd) +echo "Temporary directory created at: $temp_dir" +host=$(hostname) +host=${host//[0-9]/} +build_dir=${temp_dir}/build-laghos-${host} +mkdir -p ${build_dir} +installDir="/dev/shm/install" + +start_test=$(date +'%s') + +build_hypre(){ + LOCAL_DIR=$1 + hypre_version=$2 + if [ ! -d hypre ]; then + git clone --depth 1 --branch $hypre_version https://github.com/hypre-space/hypre.git + fi + + pushd hypre + pushd src + rocm_path=$(realpath $(dirname $(which hipcc))/../) + rocm_mpi_path=$(realpath $(dirname $(which mpicc))/../) + make distclean 2>/dev/null || true + CUFLAGS="-fpass-plugin=${LOCAL_DIR}/lib64/libregdeviceir.so -O3 -std=c++14 -x hip --offload-arch=${ROCM_ARCH}" CC=mpicc CXX=mpicxx CXXFLAGS="std=c++17 -fPIC" CFLAGS="-fPIC" ./configure \ + --prefix=${LOCAL_DIR} \ + --with-extra-ldpath="${LOCAL_DIR}/lib64/" \ + --with-MPI-libs="mpi mpich mneme_shallow" \ + --with-MPI-lib-dirs=${rocm_mpi_path}/lib \ + --with-MPI-include=${rocm_mpi_path}/include \ + --enable-fortran \ + --with-hip + + make -j 4 + make check || true + make install + popd + popd +} + +build_metis(){ + LOCAL_DIR=$1 + currDir=$(pwd) + if [ ! -d metis ]; then + git clone --depth 1 https://github.com/mfem/tpls.git + tar xzf tpls/metis-4.0.3.tar.gz + mv metis-4.0.3 metis + rm -rf tpls + fi + + pushd metis + # The Makefile in metis is broken + sed -i 's/^CC = cc$/CC ?= cc/' Makefile.in + CC=amdclang CXX=amdclang++ CPP=amdclang++ make -C Lib OPTFLAGS=-Wno-error=implicit-function-declaration + cp libmetis.a $LOCAL_DIR/lib/ + popd +} + +build_mfem(){ + LOCAL_DIR=$1 + mfem_version=$2 + if [[ ! -d "mfem" ]]; then + git clone --branch ${mfem_version} --depth 1 https://github.com/mfem/mfem.git + fi + pushd mfem + make distclean + CXX=mpicxx make phip HIP_ARCH=${ROCM_ARCH} HIP_FLAGS="-fpass-plugin=${LOCAL_DIR}/lib64/libregdeviceir.so" METIS_DIR=$LOCAL_DIR/lib -j MPICXX=mpicxx HYPRE_OPT=-I${LOCAL_DIR}/include HYPRE_LIB=-L${LOCAL_DIR}/lib + make -j 4 + make install + popd +} + +build_laghos() { + LOCAL_DIR=$1 + + echo "Building HYPRE" + build_hypre ${LOCAL_DIR} v2.32.0 + echo "Building METIS" + build_metis ${LOCAL_DIR} + echo "Building MFEM" + build_mfem ${LOCAL_DIR} v4.7 + + echo "Building Laghos" + if [[ ! -d "laghos" ]]; then + git clone --depth 1 https://github.com/CEED/Laghos.git laghos + fi + pushd laghos + # Laghos makefile needs some modifications to work properly with Mneme + # sed -i 's|^MFEM_DIR ?= \.\./mfem$|MFEM_DIR ?= deps/mfem/|' makefile + sed -i 's/^LAGHOS_LIBS = \$(MFEM_LIBS) \$(MFEM_EXT_LIBS)$/LAGHOS_LIBS = \$(MFEM_LIBS) \$(MFEM_EXT_LIBS) -lHYPRE -lrocsparse -lrocrand/' makefile + sed -i 's/cd \$( Run Laghos integration tests"; + BENCHMARKS="scripts/gitlab/ci-integration-laghos.sh" +else + echo "=> Invalid command. Available commands: /run-{laghos}" + exit 0 +fi + +RESULTS_COMMIT="Artifacts PR ${PR_ID} commit ${CI_COMMIT_SHORT_SHA}" + +# Run the selected benchmark +SECONDS=0 +bash -c "${BENCHMARKS}" +END_TIME=$SECONDS +COMMENT="${RESULTS_COMMIT}: ${COMMENTS_BODY} ran in ${END_TIME} seconds\n

" + +# Post the comment to the GitHub PR. +curl -L -X POST \ + -H "Accept: application/vnd.github+json" \ + -H "Authorization: Bearer $GITHUB_TOKEN" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + "https://api.github.com/repos/Olympus-HPC/Mneme/issues/${PR_ID}/comments" \ + -d "{\"body\": \"${COMMENT}\"}" diff --git a/setup.py b/setup.py index 17bbc092..92ff62e7 100644 --- a/setup.py +++ b/setup.py @@ -47,6 +47,9 @@ def initialize_options(self): self.has_nvidia = "On" if has_nvidia_gpu() else "Off" self.has_amd = "On" if has_amd_gpu() else "Off" self.llvm_dir = os.getenv("LLVM_INSTALL_DIR", None) + + self.mneme_enable_debug = os.getenv("MNEME_ENABLE_DEBUG", "Off") + if self.has_amd == "On": self.cxx = f"{self.llvm_dir}/bin/amdclang++" self.cc = f"{self.llvm_dir}/bin/amdclang" @@ -164,6 +167,7 @@ def build_mneme(self, proteus_dir, spdlog_dir): f"-DLLVM_INSTALL_DIR={self.llvm_dir}", f"-DMNEME_ENABLE_HIP={self.has_amd}", "-DMNEME_ENABLE_TESTS=On", + f"-DMNEME_ENABLE_DEBUG={self.mneme_enable_debug}", "-DMNEME_ENABLE_AUTOTUNE=On", "-DCMAKE_INSTALL_RPATH_USE_LINK_PATH=On", f"-Dproteus_DIR={proteus_dir}", diff --git a/src/python/profile/rocr_profile.cpp b/src/python/profile/rocr_profile.cpp index 2c358232..21579d54 100644 --- a/src/python/profile/rocr_profile.cpp +++ b/src/python/profile/rocr_profile.cpp @@ -236,7 +236,7 @@ int tool_init(rocprofiler_client_finalize_t /*fini*/, void *data_v) { nullptr, 0, instance.getBuffer())); // Service to pick up kernel name -> kernel id and use it when profiling - const rocprofiler_tracing_operation_t ops[] = { + rocprofiler_tracing_operation_t ops[] = { ROCPROFILER_CODE_OBJECT_DEVICE_KERNEL_SYMBOL_REGISTER, ROCPROFILER_CODE_OBJECT_LOAD}; CHECK_ROCP(rocprofiler_configure_callback_tracing_service(