From b261e3a815d2123641d5b591a22ff60dd6ae55e9 Mon Sep 17 00:00:00 2001 From: Loic Pottier Date: Wed, 24 Sep 2025 16:03:39 -0700 Subject: [PATCH 01/21] Added Laghos integration tests for Tuo and Tioga Signed-off-by: Loic Pottier --- .gitlab/custom-jobs-and-variables.yml | 8 - .gitlab/jobs/tioga.yml | 2 + .gitlab/jobs/tuolumne.yml | 2 + .gitlab/subscribed-pipelines.yml | 12 -- scripts/gitlab/ci-build-test.sh | 216 +++++++++++++++++++++++++- 5 files changed, 213 insertions(+), 27 deletions(-) diff --git a/.gitlab/custom-jobs-and-variables.yml b/.gitlab/custom-jobs-and-variables.yml index bee64029..c1d8c04b 100644 --- a/.gitlab/custom-jobs-and-variables.yml +++ b/.gitlab/custom-jobs-and-variables.yml @@ -12,14 +12,6 @@ variables: # named so that we are sure to retrieve it and avoid collisions. ALLOC_NAME: ${CI_PROJECT_NAME}_ci_${CI_PIPELINE_ID} -# Ruby -# Arguments for top level allocation -# RUBY_SHARED_ALLOC: "--exclusive --reservation=ci --time=10 --nodes=1" -# Arguments for job level allocation -# RUBY_JOB_ALLOC: "--reservation=ci --nodes=1" -# Add variables that should apply to all the jobs on a machine: -# RUBY_MY_VAR: "..." - # Poodle # Arguments for top level allocation POODLE_SHARED_ALLOC: "--exclusive --partition=pdebug --time=10 --nodes=1" diff --git a/.gitlab/jobs/tioga.yml b/.gitlab/jobs/tioga.yml index a5e93cbe..c6d6a614 100644 --- a/.gitlab/jobs/tioga.yml +++ b/.gitlab/jobs/tioga.yml @@ -50,6 +50,8 @@ variables: after_script: - rm -rf ${CI_BUILDS_DIR} - echo "=== End CI Build and Test===" + variables: + MNEME_CI_TEST_LAGHOS: "on" .build-variants: parallel: diff --git a/.gitlab/jobs/tuolumne.yml b/.gitlab/jobs/tuolumne.yml index 77c7ae0e..4a74d4d7 100644 --- a/.gitlab/jobs/tuolumne.yml +++ b/.gitlab/jobs/tuolumne.yml @@ -50,6 +50,8 @@ variables: after_script: - rm -rf ${CI_BUILDS_DIR} - echo "=== End CI Build and Test===" + variables: + MNEME_CI_TEST_LAGHOS: "on" .build-variants: parallel: diff --git a/.gitlab/subscribed-pipelines.yml b/.gitlab/subscribed-pipelines.yml index 8cb46a0a..4a72318c 100644 --- a/.gitlab/subscribed-pipelines.yml +++ b/.gitlab/subscribed-pipelines.yml @@ -29,18 +29,6 @@ # Comment the jobs for machines you don’t need. ### - ## RUBY - #ruby-up-check: - # variables: - # CI_MACHINE: "ruby" - # extends: [.machine-check] - # - #ruby-build-and-test: - # variables: - # CI_MACHINE: "ruby" - # needs: [ruby-up-check] - # ## [Shared jobs scenario] - # TIOGA tioga-up-check: variables: diff --git a/scripts/gitlab/ci-build-test.sh b/scripts/gitlab/ci-build-test.sh index 9b2ce266..e72fce04 100755 --- a/scripts/gitlab/ci-build-test.sh +++ b/scripts/gitlab/ci-build-test.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash set -e @@ -6,21 +6,176 @@ temp_dir=$(pwd) #$(mktemp -d) echo "Temporary directory created at: $temp_dir" host=$(hostname) host=${host//[0-9]/} -mkdir -p ${temp_dir}/build-${host}; build_dir=${temp_dir}/build-${host} +mkdir -p ${build_dir} installDir="/dev/shm/install" +build_hypre(){ + LOCAL_DIR=$1 + hypre_version=$2 + if [ ! -d hypre ]; then + git clone --depth 1 --branch $hypre_version https://github.com/hypre-space/hypre.git + fi + + pushd hypre + pushd src + rocm_path=$(realpath $(dirname $(which hipcc))/../) + rocm_mpi_path=$(realpath $(dirname $(which mpicc))/../) + make distclean 2>/dev/null || true + CUFLAGS="-fpass-plugin=${LOCAL_DIR}/lib64/libregdeviceir.so -O3 -std=c++14 -x hip --offload-arch=${ROCM_ARCH}" CC=mpicc CXX=mpicxx CXXFLAGS="std=c++17 -fPIC" CFLAGS="-fPIC" ./configure \ + --prefix=$LOCAL_DIR \ + --with-extra-ldpath="${LOCAL_DIR}/lib64/" \ + --with-MPI-libs="mpi mpich mneme_shallow" \ + --with-MPI-lib-dirs=${rocm_mpi_path}/lib \ + --with-MPI-include=${rocm_mpi_path}/include \ + --enable-fortran \ + --with-hip + + make -j 4 + make check || true + make install + popd + popd +} + +build_metis(){ + LOCAL_DIR=$1 + currDir=$(pwd) + if [ ! -d metis ]; then + git clone --depth 1 https://github.com/mfem/tpls.git + tar xzf tpls/metis-4.0.3.tar.gz + mv metis-4.0.3 metis + rm -rf tpls + fi + + pushd metis + # The Makefile in metis is broken + sed -i 's/^CC = cc$/CC ?= cc/' Makefile.in + CC=amdclang CXX=amdclang++ CPP=amdclang++ make -C Lib OPTFLAGS=-Wno-error=implicit-function-declaration + cp libmetis.a $LOCAL_DIR/lib/ + popd +} + +build_mfem(){ + LOCAL_DIR=$1 + mfem_version=$2 + if [[ ! -d "mfem" ]]; then + git clone --branch ${mfem_version} --depth 1 https://github.com/mfem/mfem.git + fi + pushd mfem + make distclean + CXX=mpicxx make phip HIP_ARCH=${ROCM_ARCH} HIP_FLAGS="-fpass-plugin=${LOCAL_DIR}/lib64/libregdeviceir.so" METIS_DIR=$LOCAL_DIR/lib -j MPICXX=mpicxx HYPRE_OPT=-I${LOCAL_DIR}/include HYPRE_LIB=-L${LOCAL_DIR}/lib + make -j 4 + make install + popd +} + +build_laghos() { + LOCAL_DIR=$1 + + echo "Building HYPRE" + build_hypre ${LOCAL_DIR} v2.32.0 + echo "Building METIS" + build_metis ${LOCAL_DIR} + echo "Building MFEM" + build_mfem ${LOCAL_DIR} v4.7 + + echo "Building Laghos" + if [[ ! -d "laghos" ]]; then + git clone --depth 1 https://github.com/CEED/Laghos.git laghos + fi + pushd laghos + # Laghos makefile needs some modifications to work properly with Mneme + # sed -i 's|^MFEM_DIR ?= \.\./mfem$|MFEM_DIR ?= deps/mfem/|' makefile + sed -i 's/^LAGHOS_LIBS = \$(MFEM_LIBS) \$(MFEM_EXT_LIBS)$/LAGHOS_LIBS = \$(MFEM_LIBS) \$(MFEM_EXT_LIBS) -lHYPRE -lrocsparse -lrocrand/' makefile + sed -i 's/cd \$( Date: Wed, 24 Sep 2025 16:26:23 -0700 Subject: [PATCH 02/21] Reactivated commented tests Signed-off-by: Loic Pottier --- scripts/gitlab/ci-build-test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/gitlab/ci-build-test.sh b/scripts/gitlab/ci-build-test.sh index e72fce04..87610f9e 100755 --- a/scripts/gitlab/ci-build-test.sh +++ b/scripts/gitlab/ci-build-test.sh @@ -302,7 +302,7 @@ fi make -j 10 echo "### TESTING ###" -# ctest --output-on-failure +ctest --output-on-failure echo "### TESTING ###" make -j 10 install From 1251471cc6d8c0bd76014405e68047b393ef518e Mon Sep 17 00:00:00 2001 From: Loic Pottier Date: Wed, 24 Sep 2025 17:15:07 -0700 Subject: [PATCH 03/21] Automatic selection of a kernel and setting MNEME_LOG_LEVEL to critical to avoid log limit on GitLab runners Signed-off-by: Loic Pottier --- scripts/gitlab/ci-build-test.sh | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/scripts/gitlab/ci-build-test.sh b/scripts/gitlab/ci-build-test.sh index 87610f9e..48baf496 100755 --- a/scripts/gitlab/ci-build-test.sh +++ b/scripts/gitlab/ci-build-test.sh @@ -23,7 +23,7 @@ build_hypre(){ rocm_mpi_path=$(realpath $(dirname $(which mpicc))/../) make distclean 2>/dev/null || true CUFLAGS="-fpass-plugin=${LOCAL_DIR}/lib64/libregdeviceir.so -O3 -std=c++14 -x hip --offload-arch=${ROCM_ARCH}" CC=mpicc CXX=mpicxx CXXFLAGS="std=c++17 -fPIC" CFLAGS="-fPIC" ./configure \ - --prefix=$LOCAL_DIR \ + --prefix=${LOCAL_DIR} \ --with-extra-ldpath="${LOCAL_DIR}/lib64/" \ --with-MPI-libs="mpi mpich mneme_shallow" \ --with-MPI-lib-dirs=${rocm_mpi_path}/lib \ @@ -109,7 +109,7 @@ run_laghos() { TF=0.0033 CMD="${INSTALL_DIR}/laghos -p 1 -dim 2 -pa -tf ${TF} -d hip -rs ${RS}" - export MNEME_LOG_LEVEL=debug + export MNEME_LOG_LEVEL=critical export MNEME_PAGE_SIZE=16 # export AMD_LOG_LEVEL=4 @@ -141,7 +141,7 @@ run_mneme_laghos() { --prune \ --internalize \ --num-trials 2 \ - --iterations 3 \ + --iterations 2 \ --seed 0 \ --no-specialize @@ -156,7 +156,7 @@ run_mneme_laghos() { --prune \ --internalize \ --num-trials 2 \ - --iterations 3 \ + --iterations 4 \ --seed 0 \ --no-specialize @@ -340,8 +340,9 @@ if [[ "${MNEME_CI_TEST_LAGHOS}" == "on" || "${MNEME_CI_TEST_LAGHOS}" == "On" || run_laghos ${installDir} ${OUTPUT_DIR} echo "### TESTING Laghos ###" # Find the JSON file containing the kernel - JSON_RECORD="10242537283821721753.json" - KERNEL_ID="15099428936510216301" + # We select one JSON file + JSON_RECORD=$(find run-mneme/ -maxdepth 1 -name '*.json' -printf '%f' -quit) + KERNEL_ID=$(jq -r '.instances | keys[]' ${OUTPUT_DIR}/${JSON_RECORD} | head -n 1) DB_STORE="mneme-result" run_mneme_laghos ${OUTPUT_DIR}/${JSON_RECORD} ${KERNEL_ID} ${DB_STORE} fi From 4550a38c7d2271a66ab814144c3c00dd20f99421 Mon Sep 17 00:00:00 2001 From: Loic Pottier Date: Thu, 25 Sep 2025 11:20:48 -0700 Subject: [PATCH 04/21] Temporarily deactivating Tioga and most of the test to focus on Tuo+ROCm 6.3.1 Signed-off-by: Loic Pottier --- .gitlab/custom-jobs-and-variables.yml | 4 ++-- .gitlab/jobs/tuolumne.yml | 10 ++++++++-- .gitlab/subscribed-pipelines.yml | 26 +++++++++++++------------- scripts/gitlab/ci-build-test.sh | 2 +- 4 files changed, 24 insertions(+), 18 deletions(-) diff --git a/.gitlab/custom-jobs-and-variables.yml b/.gitlab/custom-jobs-and-variables.yml index c1d8c04b..ecb5e314 100644 --- a/.gitlab/custom-jobs-and-variables.yml +++ b/.gitlab/custom-jobs-and-variables.yml @@ -32,7 +32,7 @@ variables: # Tioga # Arguments for top level allocation # OPTIONAL: "-o per-resource.count=2" allows to get 2 jobs running on each node. - TIOGA_SHARED_ALLOC: "--queue=pci --exclusive --time-limit=15m --nodes=1" + TIOGA_SHARED_ALLOC: "--queue=pci --exclusive --time-limit=25m --nodes=1" # Arguments for job level allocation TIOGA_JOB_ALLOC: "--nodes=1 --begin-time=+5s" # Add variables that should apply to all the jobs on a machine: @@ -47,7 +47,7 @@ variables: # Tuo # Arguments for top level allocation # OPTIONAL: "-o per-resource.count=2" allows to get 2 jobs running on each node. - TUOLUMNE_SHARED_ALLOC: "--queue=pci --exclusive --time-limit=15m --nodes=1" + TUOLUMNE_SHARED_ALLOC: "--queue=pci --exclusive --time-limit=25m --nodes=1" # Arguments for job level allocation TUOLUMNE_JOB_ALLOC: "--nodes=1 --begin-time=+5s" # Add variables that should apply to all the jobs on a machine: diff --git a/.gitlab/jobs/tuolumne.yml b/.gitlab/jobs/tuolumne.yml index 4a74d4d7..76a86c11 100644 --- a/.gitlab/jobs/tuolumne.yml +++ b/.gitlab/jobs/tuolumne.yml @@ -53,11 +53,17 @@ variables: variables: MNEME_CI_TEST_LAGHOS: "on" +# .build-variants: +# parallel: +# matrix: +# - MNEME_CI_ENABLE_DEBUG: ["on", "off"] +# MNEME_CI_ROCM_VERSION: ["6.2.1", "6.3.1", "6.4.2"] + .build-variants: parallel: matrix: - - MNEME_CI_ENABLE_DEBUG: ["on", "off"] - MNEME_CI_ROCM_VERSION: ["6.2.1", "6.3.1", "6.4.2"] + - MNEME_CI_ENABLE_DEBUG: ["on"] + MNEME_CI_ROCM_VERSION: ["6.3.1"] build-and-test-tuolumne: extends: [.base-job, .build-variants] diff --git a/.gitlab/subscribed-pipelines.yml b/.gitlab/subscribed-pipelines.yml index 4a72318c..ddaf3d49 100644 --- a/.gitlab/subscribed-pipelines.yml +++ b/.gitlab/subscribed-pipelines.yml @@ -29,20 +29,20 @@ # Comment the jobs for machines you don’t need. ### -# TIOGA -tioga-up-check: - variables: - CI_MACHINE: "tioga" - extends: [.machine-check] +# # TIOGA +# tioga-up-check: +# variables: +# CI_MACHINE: "tioga" +# extends: [.machine-check] -tioga-build-and-test: - variables: - CI_MACHINE: "tioga" - needs: [tioga-up-check] - ## [Shared jobs scenario] - ## (See Umpire CI setup for an example). - #needs: [tioga-up-check, generate-job-lists] - extends: [.build-and-test] +# tioga-build-and-test: +# variables: +# CI_MACHINE: "tioga" +# needs: [tioga-up-check] +# ## [Shared jobs scenario] +# ## (See Umpire CI setup for an example). +# #needs: [tioga-up-check, generate-job-lists] +# extends: [.build-and-test] # # LASSEN # lassen-up-check: diff --git a/scripts/gitlab/ci-build-test.sh b/scripts/gitlab/ci-build-test.sh index 48baf496..ab4a43d9 100755 --- a/scripts/gitlab/ci-build-test.sh +++ b/scripts/gitlab/ci-build-test.sh @@ -271,7 +271,7 @@ elif [[ "$SYS_TYPE" == "toss_4_x86_64_ib_cray" ]]; then ml load rocm/${MNEME_CI_ROCM_VERSION} export LLVM_INSTALL_DIR=${ROCM_PATH}/llvm echo "LLVM INSTALL DIR is ${LLVM_INSTALL_DIR}" -export ROCM_ARCH=$(rocm_agent_enumerator | sed -n 1p) +export ROCM_ARCH=$(rocm_agent_enumerator | sed -n 2p) if [ -z "${ROCM_ARCH}" ]; then echo "ROCM_ARCH is not set or is empty" From 81bfeea9f3ced59667fdc999c1eec0216cc63a2c Mon Sep 17 00:00:00 2001 From: Loic Pottier Date: Thu, 25 Sep 2025 12:41:58 -0700 Subject: [PATCH 05/21] Reactivating all tests Signed-off-by: Loic Pottier --- .gitlab/jobs/tuolumne.yml | 10 ++----- .gitlab/subscribed-pipelines.yml | 50 ++++++++++++++++---------------- 2 files changed, 27 insertions(+), 33 deletions(-) diff --git a/.gitlab/jobs/tuolumne.yml b/.gitlab/jobs/tuolumne.yml index 76a86c11..4a74d4d7 100644 --- a/.gitlab/jobs/tuolumne.yml +++ b/.gitlab/jobs/tuolumne.yml @@ -53,17 +53,11 @@ variables: variables: MNEME_CI_TEST_LAGHOS: "on" -# .build-variants: -# parallel: -# matrix: -# - MNEME_CI_ENABLE_DEBUG: ["on", "off"] -# MNEME_CI_ROCM_VERSION: ["6.2.1", "6.3.1", "6.4.2"] - .build-variants: parallel: matrix: - - MNEME_CI_ENABLE_DEBUG: ["on"] - MNEME_CI_ROCM_VERSION: ["6.3.1"] + - MNEME_CI_ENABLE_DEBUG: ["on", "off"] + MNEME_CI_ROCM_VERSION: ["6.2.1", "6.3.1", "6.4.2"] build-and-test-tuolumne: extends: [.base-job, .build-variants] diff --git a/.gitlab/subscribed-pipelines.yml b/.gitlab/subscribed-pipelines.yml index ddaf3d49..836a8199 100644 --- a/.gitlab/subscribed-pipelines.yml +++ b/.gitlab/subscribed-pipelines.yml @@ -29,34 +29,34 @@ # Comment the jobs for machines you don’t need. ### -# # TIOGA -# tioga-up-check: -# variables: -# CI_MACHINE: "tioga" -# extends: [.machine-check] +# TIOGA +tioga-up-check: + variables: + CI_MACHINE: "tioga" + extends: [.machine-check] -# tioga-build-and-test: -# variables: -# CI_MACHINE: "tioga" -# needs: [tioga-up-check] -# ## [Shared jobs scenario] -# ## (See Umpire CI setup for an example). -# #needs: [tioga-up-check, generate-job-lists] -# extends: [.build-and-test] +tioga-build-and-test: + variables: + CI_MACHINE: "tioga" + needs: [tioga-up-check] + ## [Shared jobs scenario] + ## (See Umpire CI setup for an example). + #needs: [tioga-up-check, generate-job-lists] + extends: [.build-and-test] -# # LASSEN -# lassen-up-check: -# variables: -# CI_MACHINE: "lassen" -# extends: [.machine-check] +# LASSEN +lassen-up-check: + variables: + CI_MACHINE: "lassen" + extends: [.machine-check] -# lassen-build-and-test: -# variables: -# CI_MACHINE: "lassen" -# needs: [lassen-up-check] -# extends: [.build-and-test] -# # #needs: [lassen-up-check, generate-job-lists] -# # +lassen-build-and-test: + variables: + CI_MACHINE: "lassen" + needs: [lassen-up-check] + extends: [.build-and-test] +# #needs: [lassen-up-check, generate-job-lists] +# # TUOLUMNE tuolumne-up-check: From f441eedc7440af167d400f0ef6399728b4411bef Mon Sep 17 00:00:00 2001 From: Loic Pottier Date: Thu, 25 Sep 2025 16:49:05 -0700 Subject: [PATCH 06/21] Testing two concurrent jobs Signed-off-by: Loic Pottier --- .gitlab/jobs/tuolumne.yml | 8 ++++++- .gitlab/subscribed-pipelines.yml | 39 ++++++++++++++++---------------- scripts/gitlab/ci-build-test.sh | 5 ++++ 3 files changed, 31 insertions(+), 21 deletions(-) diff --git a/.gitlab/jobs/tuolumne.yml b/.gitlab/jobs/tuolumne.yml index 4a74d4d7..a23583ba 100644 --- a/.gitlab/jobs/tuolumne.yml +++ b/.gitlab/jobs/tuolumne.yml @@ -53,11 +53,17 @@ variables: variables: MNEME_CI_TEST_LAGHOS: "on" +# .build-variants: +# parallel: +# matrix: +# - MNEME_CI_ENABLE_DEBUG: ["on", "off"] +# MNEME_CI_ROCM_VERSION: ["6.2.1", "6.3.1", "6.4.2"] + .build-variants: parallel: matrix: - MNEME_CI_ENABLE_DEBUG: ["on", "off"] - MNEME_CI_ROCM_VERSION: ["6.2.1", "6.3.1", "6.4.2"] + MNEME_CI_ROCM_VERSION: ["6.3.1"] build-and-test-tuolumne: extends: [.base-job, .build-variants] diff --git a/.gitlab/subscribed-pipelines.yml b/.gitlab/subscribed-pipelines.yml index 836a8199..b899d61a 100644 --- a/.gitlab/subscribed-pipelines.yml +++ b/.gitlab/subscribed-pipelines.yml @@ -35,28 +35,27 @@ tioga-up-check: CI_MACHINE: "tioga" extends: [.machine-check] -tioga-build-and-test: - variables: - CI_MACHINE: "tioga" - needs: [tioga-up-check] - ## [Shared jobs scenario] - ## (See Umpire CI setup for an example). - #needs: [tioga-up-check, generate-job-lists] - extends: [.build-and-test] +# tioga-build-and-test: +# variables: +# CI_MACHINE: "tioga" +# needs: [tioga-up-check] +# ## [Shared jobs scenario] +# ## (See Umpire CI setup for an example). +# #needs: [tioga-up-check, generate-job-lists] +# extends: [.build-and-test] -# LASSEN -lassen-up-check: - variables: - CI_MACHINE: "lassen" - extends: [.machine-check] +# # LASSEN +# lassen-up-check: +# variables: +# CI_MACHINE: "lassen" +# extends: [.machine-check] -lassen-build-and-test: - variables: - CI_MACHINE: "lassen" - needs: [lassen-up-check] - extends: [.build-and-test] -# #needs: [lassen-up-check, generate-job-lists] -# +# lassen-build-and-test: +# variables: +# CI_MACHINE: "lassen" +# needs: [lassen-up-check] +# extends: [.build-and-test] +# # #needs: [lassen-up-check, generate-job-lists] # TUOLUMNE tuolumne-up-check: diff --git a/scripts/gitlab/ci-build-test.sh b/scripts/gitlab/ci-build-test.sh index ab4a43d9..221fb626 100755 --- a/scripts/gitlab/ci-build-test.sh +++ b/scripts/gitlab/ci-build-test.sh @@ -10,6 +10,8 @@ build_dir=${temp_dir}/build-${host} mkdir -p ${build_dir} installDir="/dev/shm/install" +start_test=$(date +'%s') + build_hypre(){ LOCAL_DIR=$1 hypre_version=$2 @@ -346,3 +348,6 @@ if [[ "${MNEME_CI_TEST_LAGHOS}" == "on" || "${MNEME_CI_TEST_LAGHOS}" == "On" || DB_STORE="mneme-result" run_mneme_laghos ${OUTPUT_DIR}/${JSON_RECORD} ${KERNEL_ID} ${DB_STORE} fi + +end_test=$(($(date +'%s') - $start_test)) +echo "Build and test took $end_test seconds" From 2bcaaa7db6eea539e37d9455ebd78d077eea83e9 Mon Sep 17 00:00:00 2001 From: Loic Pottier Date: Thu, 25 Sep 2025 21:59:03 -0700 Subject: [PATCH 07/21] Removing const identifier which breaks with ROCm 6.2.1 Signed-off-by: Loic Pottier --- src/python/profile/rocr_profile.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/python/profile/rocr_profile.cpp b/src/python/profile/rocr_profile.cpp index 2c358232..21579d54 100644 --- a/src/python/profile/rocr_profile.cpp +++ b/src/python/profile/rocr_profile.cpp @@ -236,7 +236,7 @@ int tool_init(rocprofiler_client_finalize_t /*fini*/, void *data_v) { nullptr, 0, instance.getBuffer())); // Service to pick up kernel name -> kernel id and use it when profiling - const rocprofiler_tracing_operation_t ops[] = { + rocprofiler_tracing_operation_t ops[] = { ROCPROFILER_CODE_OBJECT_DEVICE_KERNEL_SYMBOL_REGISTER, ROCPROFILER_CODE_OBJECT_LOAD}; CHECK_ROCP(rocprofiler_configure_callback_tracing_service( From 5d6d09adf1d26b1f080d887044cc7fdef78c976e Mon Sep 17 00:00:00 2001 From: Loic Pottier Date: Thu, 25 Sep 2025 22:00:56 -0700 Subject: [PATCH 08/21] reactivating ROCm 6.2.1 tests Signed-off-by: Loic Pottier --- .gitlab/custom-jobs-and-variables.yml | 4 ++-- .gitlab/jobs/tuolumne.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.gitlab/custom-jobs-and-variables.yml b/.gitlab/custom-jobs-and-variables.yml index ecb5e314..cda3a931 100644 --- a/.gitlab/custom-jobs-and-variables.yml +++ b/.gitlab/custom-jobs-and-variables.yml @@ -32,7 +32,7 @@ variables: # Tioga # Arguments for top level allocation # OPTIONAL: "-o per-resource.count=2" allows to get 2 jobs running on each node. - TIOGA_SHARED_ALLOC: "--queue=pci --exclusive --time-limit=25m --nodes=1" + TIOGA_SHARED_ALLOC: "--queue=pci --exclusive --time-limit=30m --nodes=1" # Arguments for job level allocation TIOGA_JOB_ALLOC: "--nodes=1 --begin-time=+5s" # Add variables that should apply to all the jobs on a machine: @@ -47,7 +47,7 @@ variables: # Tuo # Arguments for top level allocation # OPTIONAL: "-o per-resource.count=2" allows to get 2 jobs running on each node. - TUOLUMNE_SHARED_ALLOC: "--queue=pci --exclusive --time-limit=25m --nodes=1" + TUOLUMNE_SHARED_ALLOC: "--queue=pci --exclusive --time-limit=30m --nodes=1" # Arguments for job level allocation TUOLUMNE_JOB_ALLOC: "--nodes=1 --begin-time=+5s" # Add variables that should apply to all the jobs on a machine: diff --git a/.gitlab/jobs/tuolumne.yml b/.gitlab/jobs/tuolumne.yml index a23583ba..1e1c8e41 100644 --- a/.gitlab/jobs/tuolumne.yml +++ b/.gitlab/jobs/tuolumne.yml @@ -63,7 +63,7 @@ variables: parallel: matrix: - MNEME_CI_ENABLE_DEBUG: ["on", "off"] - MNEME_CI_ROCM_VERSION: ["6.3.1"] + MNEME_CI_ROCM_VERSION: ["6.2.1", "6.3.1"] build-and-test-tuolumne: extends: [.base-job, .build-variants] From 70b79fd3b02e1c5216a6f45384df8bf874875706 Mon Sep 17 00:00:00 2001 From: Loic Pottier Date: Fri, 26 Sep 2025 15:39:37 -0700 Subject: [PATCH 09/21] Separate integration tests with Laghos in its own pipeline Signed-off-by: Loic Pottier --- .gitlab-ci.yml | 30 +--- .gitlab/custom-jobs-and-variables.yml | 4 +- .gitlab/jobs/tioga-laghos.yml | 27 +++ .gitlab/jobs/tioga.yml | 2 - .gitlab/jobs/tuolumne-laghos.yml | 27 +++ .gitlab/jobs/tuolumne.yml | 10 +- .gitlab/subscribed-pipelines.yml | 65 +++++-- scripts/gitlab/ci-build-test.sh | 195 -------------------- scripts/gitlab/ci-integration-laghos.sh | 227 ++++++++++++++++++++++++ setup.py | 4 + 10 files changed, 338 insertions(+), 253 deletions(-) create mode 100644 .gitlab/jobs/tioga-laghos.yml create mode 100644 .gitlab/jobs/tuolumne-laghos.yml create mode 100755 scripts/gitlab/ci-integration-laghos.sh diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index ecc944eb..63875da3 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -47,12 +47,6 @@ variables: # Required information about GitHub repository GITHUB_PROJECT_NAME: "Mneme" GITHUB_PROJECT_ORG: "Olympus-HPC" -# Set the build-and-test command. -# Nested variables are allowed and useful to customize the job command. We -# prevent variable expansion so that you can define them at job level. - JOB_CMD: - value: "scripts/gitlab/ci-build-test.sh" - expand: false # Override the pattern describing branches that will skip the "draft PR filter # test". Add protected branches here. See default value in # preliminary-ignore-draft-pr.yml. @@ -65,8 +59,7 @@ variables: stages: - prerequisites - build-and-test - - test-integration - - run-benchmarks + - integration-laghos # Template for jobs triggering a build-and-test sub-pipeline: .build-and-test: @@ -88,30 +81,15 @@ stages: forward: pipeline_variables: true -.test-integration: - stage: test-integration +.integration-laghos: + stage: integration-laghos trigger: include: - local: '.gitlab/custom-jobs-and-variables.yml' - project: 'radiuss/radiuss-shared-ci' ref: 'v2025.06.0' file: 'pipelines/${CI_MACHINE}.yml' - - local: '.gitlab/jobs/${CI_MACHINE}-integration.yml' - strategy: depend - forward: - pipeline_variables: true - -.run-benchmarks: - stage: run-benchmarks - trigger: - include: - - local: '.gitlab/custom-jobs-and-variables.yml' - - project: 'radiuss/radiuss-shared-ci' - ref: 'v2025.06.0' - file: 'pipelines/${CI_MACHINE}.yml' - # Add your jobs - # you can use a local file - - local: '.gitlab/jobs/${CI_MACHINE}-benchmarks.yml' + - local: '.gitlab/jobs/${CI_MACHINE}-laghos.yml' strategy: depend forward: pipeline_variables: true diff --git a/.gitlab/custom-jobs-and-variables.yml b/.gitlab/custom-jobs-and-variables.yml index cda3a931..c1d8c04b 100644 --- a/.gitlab/custom-jobs-and-variables.yml +++ b/.gitlab/custom-jobs-and-variables.yml @@ -32,7 +32,7 @@ variables: # Tioga # Arguments for top level allocation # OPTIONAL: "-o per-resource.count=2" allows to get 2 jobs running on each node. - TIOGA_SHARED_ALLOC: "--queue=pci --exclusive --time-limit=30m --nodes=1" + TIOGA_SHARED_ALLOC: "--queue=pci --exclusive --time-limit=15m --nodes=1" # Arguments for job level allocation TIOGA_JOB_ALLOC: "--nodes=1 --begin-time=+5s" # Add variables that should apply to all the jobs on a machine: @@ -47,7 +47,7 @@ variables: # Tuo # Arguments for top level allocation # OPTIONAL: "-o per-resource.count=2" allows to get 2 jobs running on each node. - TUOLUMNE_SHARED_ALLOC: "--queue=pci --exclusive --time-limit=30m --nodes=1" + TUOLUMNE_SHARED_ALLOC: "--queue=pci --exclusive --time-limit=15m --nodes=1" # Arguments for job level allocation TUOLUMNE_JOB_ALLOC: "--nodes=1 --begin-time=+5s" # Add variables that should apply to all the jobs on a machine: diff --git a/.gitlab/jobs/tioga-laghos.yml b/.gitlab/jobs/tioga-laghos.yml new file mode 100644 index 00000000..2734ceb2 --- /dev/null +++ b/.gitlab/jobs/tioga-laghos.yml @@ -0,0 +1,27 @@ +############################################################################### +# Copyright (c) 2022-23, Lawrence Livermore National Security, LLC and RADIUSS +# project contributors. See the COPYRIGHT file for details. +# +# SPDX-License-Identifier: (MIT) +############################################################################### + +.tioga_reproducer_vars: + script: + - echo -e "Running on tioga\n" + +.base-job: + extends: .job_on_tioga + before_script: + - echo "=== Start CI Build and Test===" + after_script: + - rm -rf ${CI_BUILDS_DIR} + - echo "=== End CI Build and Test===" + +.build-variants: + parallel: + matrix: + - MNEME_CI_ENABLE_DEBUG: ["on", "off"] + MNEME_CI_ROCM_VERSION: ["6.2.1", "6.3.1", "6.4.2"] + +integration-laghos-tioga: + extends: [.base-job, .build-variants] diff --git a/.gitlab/jobs/tioga.yml b/.gitlab/jobs/tioga.yml index c6d6a614..a5e93cbe 100644 --- a/.gitlab/jobs/tioga.yml +++ b/.gitlab/jobs/tioga.yml @@ -50,8 +50,6 @@ variables: after_script: - rm -rf ${CI_BUILDS_DIR} - echo "=== End CI Build and Test===" - variables: - MNEME_CI_TEST_LAGHOS: "on" .build-variants: parallel: diff --git a/.gitlab/jobs/tuolumne-laghos.yml b/.gitlab/jobs/tuolumne-laghos.yml new file mode 100644 index 00000000..3579e69d --- /dev/null +++ b/.gitlab/jobs/tuolumne-laghos.yml @@ -0,0 +1,27 @@ +############################################################################### +# Copyright (c) 2022-23, Lawrence Livermore National Security, LLC and RADIUSS +# project contributors. See the COPYRIGHT file for details. +# +# SPDX-License-Identifier: (MIT) +############################################################################### + +.tuolumne_reproducer_vars: + script: + - echo -e "Running on tuolumne\n" + +.base-job: + extends: .job_on_tuolumne + before_script: + - echo "=== Start CI Build and Test===" + after_script: + - rm -rf ${CI_BUILDS_DIR} + - echo "=== End CI Build and Test===" + +.build-variants: + parallel: + matrix: + - MNEME_CI_ENABLE_DEBUG: ["on", "off"] + MNEME_CI_ROCM_VERSION: ["6.2.1", "6.3.1", "6.4.2"] + +integration-laghos-tuolumne: + extends: [.base-job, .build-variants] diff --git a/.gitlab/jobs/tuolumne.yml b/.gitlab/jobs/tuolumne.yml index 1e1c8e41..77c7ae0e 100644 --- a/.gitlab/jobs/tuolumne.yml +++ b/.gitlab/jobs/tuolumne.yml @@ -50,20 +50,12 @@ variables: after_script: - rm -rf ${CI_BUILDS_DIR} - echo "=== End CI Build and Test===" - variables: - MNEME_CI_TEST_LAGHOS: "on" - -# .build-variants: -# parallel: -# matrix: -# - MNEME_CI_ENABLE_DEBUG: ["on", "off"] -# MNEME_CI_ROCM_VERSION: ["6.2.1", "6.3.1", "6.4.2"] .build-variants: parallel: matrix: - MNEME_CI_ENABLE_DEBUG: ["on", "off"] - MNEME_CI_ROCM_VERSION: ["6.2.1", "6.3.1"] + MNEME_CI_ROCM_VERSION: ["6.2.1", "6.3.1", "6.4.2"] build-and-test-tuolumne: extends: [.base-job, .build-variants] diff --git a/.gitlab/subscribed-pipelines.yml b/.gitlab/subscribed-pipelines.yml index b899d61a..6c7ae92e 100644 --- a/.gitlab/subscribed-pipelines.yml +++ b/.gitlab/subscribed-pipelines.yml @@ -35,27 +35,42 @@ tioga-up-check: CI_MACHINE: "tioga" extends: [.machine-check] -# tioga-build-and-test: -# variables: -# CI_MACHINE: "tioga" -# needs: [tioga-up-check] -# ## [Shared jobs scenario] -# ## (See Umpire CI setup for an example). -# #needs: [tioga-up-check, generate-job-lists] -# extends: [.build-and-test] +tioga-build-and-test: + variables: + CI_MACHINE: "tioga" + JOB_CMD: + value: "scripts/gitlab/ci-build-test.sh" + expand: false + needs: [tioga-up-check] + ## [Shared jobs scenario] + ## (See Umpire CI setup for an example). + #needs: [tioga-up-check, generate-job-lists] + extends: [.build-and-test] + +tioga-integration-laghos: + variables: + CI_MACHINE: "tioga" + JOB_CMD: + value: "scripts/gitlab/ci-integration-laghos.sh" + expand: false + needs: [tioga-up-check] + extends: [.integration-laghos] -# # LASSEN -# lassen-up-check: -# variables: -# CI_MACHINE: "lassen" -# extends: [.machine-check] +# LASSEN +lassen-up-check: + variables: + CI_MACHINE: "lassen" + extends: [.machine-check] -# lassen-build-and-test: -# variables: -# CI_MACHINE: "lassen" -# needs: [lassen-up-check] -# extends: [.build-and-test] -# # #needs: [lassen-up-check, generate-job-lists] +lassen-build-and-test: + variables: + CI_MACHINE: "lassen" + JOB_CMD: + value: "scripts/gitlab/ci-build-test.sh" + expand: false + needs: [lassen-up-check] + extends: [.build-and-test] +# #needs: [lassen-up-check, generate-job-lists] # TUOLUMNE tuolumne-up-check: @@ -66,8 +81,20 @@ tuolumne-up-check: tuolumne-build-and-test: variables: CI_MACHINE: "tuolumne" + JOB_CMD: + value: "scripts/gitlab/ci-build-test.sh" + expand: false needs: [tuolumne-up-check] ## [Shared jobs scenario] ## (See Umpire CI setup for an example). #needs: [tuolumne-up-check, generate-job-lists] extends: [.build-and-test] + +tuolumne-integration-laghos: + variables: + CI_MACHINE: "tuolumne" + JOB_CMD: + value: "scripts/gitlab/ci-integration-laghos.sh" + expand: false + needs: [tuolumne-up-check] + extends: [.integration-laghos] diff --git a/scripts/gitlab/ci-build-test.sh b/scripts/gitlab/ci-build-test.sh index 221fb626..0f6201c7 100755 --- a/scripts/gitlab/ci-build-test.sh +++ b/scripts/gitlab/ci-build-test.sh @@ -12,160 +12,6 @@ installDir="/dev/shm/install" start_test=$(date +'%s') -build_hypre(){ - LOCAL_DIR=$1 - hypre_version=$2 - if [ ! -d hypre ]; then - git clone --depth 1 --branch $hypre_version https://github.com/hypre-space/hypre.git - fi - - pushd hypre - pushd src - rocm_path=$(realpath $(dirname $(which hipcc))/../) - rocm_mpi_path=$(realpath $(dirname $(which mpicc))/../) - make distclean 2>/dev/null || true - CUFLAGS="-fpass-plugin=${LOCAL_DIR}/lib64/libregdeviceir.so -O3 -std=c++14 -x hip --offload-arch=${ROCM_ARCH}" CC=mpicc CXX=mpicxx CXXFLAGS="std=c++17 -fPIC" CFLAGS="-fPIC" ./configure \ - --prefix=${LOCAL_DIR} \ - --with-extra-ldpath="${LOCAL_DIR}/lib64/" \ - --with-MPI-libs="mpi mpich mneme_shallow" \ - --with-MPI-lib-dirs=${rocm_mpi_path}/lib \ - --with-MPI-include=${rocm_mpi_path}/include \ - --enable-fortran \ - --with-hip - - make -j 4 - make check || true - make install - popd - popd -} - -build_metis(){ - LOCAL_DIR=$1 - currDir=$(pwd) - if [ ! -d metis ]; then - git clone --depth 1 https://github.com/mfem/tpls.git - tar xzf tpls/metis-4.0.3.tar.gz - mv metis-4.0.3 metis - rm -rf tpls - fi - - pushd metis - # The Makefile in metis is broken - sed -i 's/^CC = cc$/CC ?= cc/' Makefile.in - CC=amdclang CXX=amdclang++ CPP=amdclang++ make -C Lib OPTFLAGS=-Wno-error=implicit-function-declaration - cp libmetis.a $LOCAL_DIR/lib/ - popd -} - -build_mfem(){ - LOCAL_DIR=$1 - mfem_version=$2 - if [[ ! -d "mfem" ]]; then - git clone --branch ${mfem_version} --depth 1 https://github.com/mfem/mfem.git - fi - pushd mfem - make distclean - CXX=mpicxx make phip HIP_ARCH=${ROCM_ARCH} HIP_FLAGS="-fpass-plugin=${LOCAL_DIR}/lib64/libregdeviceir.so" METIS_DIR=$LOCAL_DIR/lib -j MPICXX=mpicxx HYPRE_OPT=-I${LOCAL_DIR}/include HYPRE_LIB=-L${LOCAL_DIR}/lib - make -j 4 - make install - popd -} - -build_laghos() { - LOCAL_DIR=$1 - - echo "Building HYPRE" - build_hypre ${LOCAL_DIR} v2.32.0 - echo "Building METIS" - build_metis ${LOCAL_DIR} - echo "Building MFEM" - build_mfem ${LOCAL_DIR} v4.7 - - echo "Building Laghos" - if [[ ! -d "laghos" ]]; then - git clone --depth 1 https://github.com/CEED/Laghos.git laghos - fi - pushd laghos - # Laghos makefile needs some modifications to work properly with Mneme - # sed -i 's|^MFEM_DIR ?= \.\./mfem$|MFEM_DIR ?= deps/mfem/|' makefile - sed -i 's/^LAGHOS_LIBS = \$(MFEM_LIBS) \$(MFEM_EXT_LIBS)$/LAGHOS_LIBS = \$(MFEM_LIBS) \$(MFEM_EXT_LIBS) -lHYPRE -lrocsparse -lrocrand/' makefile - sed -i 's/cd \$(/dev/null || true + CUFLAGS="-fpass-plugin=${LOCAL_DIR}/lib64/libregdeviceir.so -O3 -std=c++14 -x hip --offload-arch=${ROCM_ARCH}" CC=mpicc CXX=mpicxx CXXFLAGS="std=c++17 -fPIC" CFLAGS="-fPIC" ./configure \ + --prefix=${LOCAL_DIR} \ + --with-extra-ldpath="${LOCAL_DIR}/lib64/" \ + --with-MPI-libs="mpi mpich mneme_shallow" \ + --with-MPI-lib-dirs=${rocm_mpi_path}/lib \ + --with-MPI-include=${rocm_mpi_path}/include \ + --enable-fortran \ + --with-hip + + make -j 4 + make check || true + make install + popd + popd +} + +build_metis(){ + LOCAL_DIR=$1 + currDir=$(pwd) + if [ ! -d metis ]; then + git clone --depth 1 https://github.com/mfem/tpls.git + tar xzf tpls/metis-4.0.3.tar.gz + mv metis-4.0.3 metis + rm -rf tpls + fi + + pushd metis + # The Makefile in metis is broken + sed -i 's/^CC = cc$/CC ?= cc/' Makefile.in + CC=amdclang CXX=amdclang++ CPP=amdclang++ make -C Lib OPTFLAGS=-Wno-error=implicit-function-declaration + cp libmetis.a $LOCAL_DIR/lib/ + popd +} + +build_mfem(){ + LOCAL_DIR=$1 + mfem_version=$2 + if [[ ! -d "mfem" ]]; then + git clone --branch ${mfem_version} --depth 1 https://github.com/mfem/mfem.git + fi + pushd mfem + make distclean + CXX=mpicxx make phip HIP_ARCH=${ROCM_ARCH} HIP_FLAGS="-fpass-plugin=${LOCAL_DIR}/lib64/libregdeviceir.so" METIS_DIR=$LOCAL_DIR/lib -j MPICXX=mpicxx HYPRE_OPT=-I${LOCAL_DIR}/include HYPRE_LIB=-L${LOCAL_DIR}/lib + make -j 4 + make install + popd +} + +build_laghos() { + LOCAL_DIR=$1 + + echo "Building HYPRE" + build_hypre ${LOCAL_DIR} v2.32.0 + echo "Building METIS" + build_metis ${LOCAL_DIR} + echo "Building MFEM" + build_mfem ${LOCAL_DIR} v4.7 + + echo "Building Laghos" + if [[ ! -d "laghos" ]]; then + git clone --depth 1 https://github.com/CEED/Laghos.git laghos + fi + pushd laghos + # Laghos makefile needs some modifications to work properly with Mneme + # sed -i 's|^MFEM_DIR ?= \.\./mfem$|MFEM_DIR ?= deps/mfem/|' makefile + sed -i 's/^LAGHOS_LIBS = \$(MFEM_LIBS) \$(MFEM_EXT_LIBS)$/LAGHOS_LIBS = \$(MFEM_LIBS) \$(MFEM_EXT_LIBS) -lHYPRE -lrocsparse -lrocrand/' makefile + sed -i 's/cd \$( Date: Fri, 26 Sep 2025 16:40:39 -0700 Subject: [PATCH 10/21] Giving more time to the runners Signed-off-by: Loic Pottier --- .gitlab/custom-jobs-and-variables.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab/custom-jobs-and-variables.yml b/.gitlab/custom-jobs-and-variables.yml index c1d8c04b..81653555 100644 --- a/.gitlab/custom-jobs-and-variables.yml +++ b/.gitlab/custom-jobs-and-variables.yml @@ -32,7 +32,7 @@ variables: # Tioga # Arguments for top level allocation # OPTIONAL: "-o per-resource.count=2" allows to get 2 jobs running on each node. - TIOGA_SHARED_ALLOC: "--queue=pci --exclusive --time-limit=15m --nodes=1" + TIOGA_SHARED_ALLOC: "--queue=pci --exclusive --time-limit=60m --nodes=1" # Arguments for job level allocation TIOGA_JOB_ALLOC: "--nodes=1 --begin-time=+5s" # Add variables that should apply to all the jobs on a machine: @@ -47,7 +47,7 @@ variables: # Tuo # Arguments for top level allocation # OPTIONAL: "-o per-resource.count=2" allows to get 2 jobs running on each node. - TUOLUMNE_SHARED_ALLOC: "--queue=pci --exclusive --time-limit=15m --nodes=1" + TUOLUMNE_SHARED_ALLOC: "--queue=pci --exclusive --time-limit=60m --nodes=1" # Arguments for job level allocation TUOLUMNE_JOB_ALLOC: "--nodes=1 --begin-time=+5s" # Add variables that should apply to all the jobs on a machine: From 0dbc1239074f5353944508ddf84826415c025452 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Pottier?= <48072795+lpottier@users.noreply.github.com> Date: Fri, 26 Sep 2025 17:14:17 -0700 Subject: [PATCH 11/21] Missing guard in `db.py/add` (#57) * Added guard against exp being None in db.py/add * Deactivated Lassen tests --------- Signed-off-by: Loic Pottier --- .gitlab/subscribed-pipelines.yml | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/.gitlab/subscribed-pipelines.yml b/.gitlab/subscribed-pipelines.yml index 6c7ae92e..be885200 100644 --- a/.gitlab/subscribed-pipelines.yml +++ b/.gitlab/subscribed-pipelines.yml @@ -56,21 +56,21 @@ tioga-integration-laghos: needs: [tioga-up-check] extends: [.integration-laghos] -# LASSEN -lassen-up-check: - variables: - CI_MACHINE: "lassen" - extends: [.machine-check] +# # LASSEN +# lassen-up-check: +# variables: +# CI_MACHINE: "lassen" +# extends: [.machine-check] -lassen-build-and-test: - variables: - CI_MACHINE: "lassen" - JOB_CMD: - value: "scripts/gitlab/ci-build-test.sh" - expand: false - needs: [lassen-up-check] - extends: [.build-and-test] -# #needs: [lassen-up-check, generate-job-lists] +# lassen-build-and-test: +# variables: +# CI_MACHINE: "lassen" +# JOB_CMD: +# value: "scripts/gitlab/ci-build-test.sh" +# expand: false +# needs: [lassen-up-check] +# extends: [.build-and-test] +# # #needs: [lassen-up-check, generate-job-lists] # TUOLUMNE tuolumne-up-check: From 9351e0b7b7a12c431f5361965d15c0852a0e43a6 Mon Sep 17 00:00:00 2001 From: Loic Pottier Date: Thu, 25 Sep 2025 11:20:48 -0700 Subject: [PATCH 12/21] Deactivating Lassen tests Signed-off-by: Loic Pottier --- .gitlab/custom-jobs-and-variables.yml | 14 +++++++------- .gitlab/jobs/tuolumne.yml | 10 ++++++++-- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/.gitlab/custom-jobs-and-variables.yml b/.gitlab/custom-jobs-and-variables.yml index 81653555..b892f54e 100644 --- a/.gitlab/custom-jobs-and-variables.yml +++ b/.gitlab/custom-jobs-and-variables.yml @@ -32,22 +32,22 @@ variables: # Tioga # Arguments for top level allocation # OPTIONAL: "-o per-resource.count=2" allows to get 2 jobs running on each node. - TIOGA_SHARED_ALLOC: "--queue=pci --exclusive --time-limit=60m --nodes=1" + TIOGA_SHARED_ALLOC: "--queue=pci --exclusive --time-limit=90m --nodes=1" # Arguments for job level allocation TIOGA_JOB_ALLOC: "--nodes=1 --begin-time=+5s" # Add variables that should apply to all the jobs on a machine: # TIOGA_MY_VAR: "..." -# Lassen uses a different job scheduler (spectrum lsf) that does not allow -# pre-allocation the same way slurm does. Arguments for job level allocation - LASSEN_JOB_ALLOC: "1 -W 60 -q pci" -# Add variables that should apply to all the jobs on a machine: -# LASSEN_MY_VAR: "..." +# # Lassen uses a different job scheduler (spectrum lsf) that does not allow +# # pre-allocation the same way slurm does. Arguments for job level allocation +# LASSEN_JOB_ALLOC: "1 -W 60 -q pci" +# # Add variables that should apply to all the jobs on a machine: +# # LASSEN_MY_VAR: "..." # Tuo # Arguments for top level allocation # OPTIONAL: "-o per-resource.count=2" allows to get 2 jobs running on each node. - TUOLUMNE_SHARED_ALLOC: "--queue=pci --exclusive --time-limit=60m --nodes=1" + TUOLUMNE_SHARED_ALLOC: "--queue=pci --exclusive --time-limit=90m --nodes=1" # Arguments for job level allocation TUOLUMNE_JOB_ALLOC: "--nodes=1 --begin-time=+5s" # Add variables that should apply to all the jobs on a machine: diff --git a/.gitlab/jobs/tuolumne.yml b/.gitlab/jobs/tuolumne.yml index 77c7ae0e..c868e78e 100644 --- a/.gitlab/jobs/tuolumne.yml +++ b/.gitlab/jobs/tuolumne.yml @@ -51,11 +51,17 @@ variables: - rm -rf ${CI_BUILDS_DIR} - echo "=== End CI Build and Test===" +# .build-variants: +# parallel: +# matrix: +# - MNEME_CI_ENABLE_DEBUG: ["on", "off"] +# MNEME_CI_ROCM_VERSION: ["6.2.1", "6.3.1", "6.4.2"] + .build-variants: parallel: matrix: - - MNEME_CI_ENABLE_DEBUG: ["on", "off"] - MNEME_CI_ROCM_VERSION: ["6.2.1", "6.3.1", "6.4.2"] + - MNEME_CI_ENABLE_DEBUG: ["on"] + MNEME_CI_ROCM_VERSION: ["6.3.1"] build-and-test-tuolumne: extends: [.base-job, .build-variants] From 22da3daf4b313261b32528be0f1638da731f8459 Mon Sep 17 00:00:00 2001 From: Loic Pottier Date: Fri, 26 Sep 2025 18:26:16 -0700 Subject: [PATCH 13/21] Fixed rebase mistake Signed-off-by: Loic Pottier --- .gitlab/jobs/tuolumne.yml | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/.gitlab/jobs/tuolumne.yml b/.gitlab/jobs/tuolumne.yml index c868e78e..77c7ae0e 100644 --- a/.gitlab/jobs/tuolumne.yml +++ b/.gitlab/jobs/tuolumne.yml @@ -51,17 +51,11 @@ variables: - rm -rf ${CI_BUILDS_DIR} - echo "=== End CI Build and Test===" -# .build-variants: -# parallel: -# matrix: -# - MNEME_CI_ENABLE_DEBUG: ["on", "off"] -# MNEME_CI_ROCM_VERSION: ["6.2.1", "6.3.1", "6.4.2"] - .build-variants: parallel: matrix: - - MNEME_CI_ENABLE_DEBUG: ["on"] - MNEME_CI_ROCM_VERSION: ["6.3.1"] + - MNEME_CI_ENABLE_DEBUG: ["on", "off"] + MNEME_CI_ROCM_VERSION: ["6.2.1", "6.3.1", "6.4.2"] build-and-test-tuolumne: extends: [.base-job, .build-variants] From 8fb7d649303fe2e2701876d9d495b7a9bbd40d00 Mon Sep 17 00:00:00 2001 From: Loic Pottier Date: Fri, 26 Sep 2025 20:00:16 -0700 Subject: [PATCH 14/21] Cleaning up Lassen-related tests Signed-off-by: Loic Pottier --- .gitlab/custom-jobs-and-variables.yml | 6 --- .gitlab/jobs/lassen.yml | 60 --------------------------- .gitlab/subscribed-pipelines.yml | 16 ------- 3 files changed, 82 deletions(-) delete mode 100644 .gitlab/jobs/lassen.yml diff --git a/.gitlab/custom-jobs-and-variables.yml b/.gitlab/custom-jobs-and-variables.yml index b892f54e..2677bbac 100644 --- a/.gitlab/custom-jobs-and-variables.yml +++ b/.gitlab/custom-jobs-and-variables.yml @@ -38,12 +38,6 @@ variables: # Add variables that should apply to all the jobs on a machine: # TIOGA_MY_VAR: "..." -# # Lassen uses a different job scheduler (spectrum lsf) that does not allow -# # pre-allocation the same way slurm does. Arguments for job level allocation -# LASSEN_JOB_ALLOC: "1 -W 60 -q pci" -# # Add variables that should apply to all the jobs on a machine: -# # LASSEN_MY_VAR: "..." - # Tuo # Arguments for top level allocation # OPTIONAL: "-o per-resource.count=2" allows to get 2 jobs running on each node. diff --git a/.gitlab/jobs/lassen.yml b/.gitlab/jobs/lassen.yml deleted file mode 100644 index f48996e5..00000000 --- a/.gitlab/jobs/lassen.yml +++ /dev/null @@ -1,60 +0,0 @@ -############################################################################### -# Copyright (c) 2022-23, Lawrence Livermore National Security, LLC and RADIUSS -# project contributors. See the COPYRIGHT file for details. -# -# SPDX-License-Identifier: (MIT) -############################################################################### - -# We require project to define their job command using a variable (JOB_CMD). -# In customization/gitlab-ci.yml, we encourage to define this variable as -# non-expandable, so that project can use nested variables to configure the job -# command. The caveat is that the reproducer here cannot capture the -# definition of these variables in a generic fashion. By overriding the -# following section, projects can specify the variables to define in the -# reproducer to exactly reproduce the CI build. -.lassen_reproducer_vars: - script: - - echo -e "Running on Lassen\n" - -# With GitLab CI, included files cannot be empty. -# TODO: remove when you have at least on job defined. -variables: - INCLUDED_FILE_CANNOT_BE_EMPTY: "True" - -############### -# Explanations: -############### -# RADIUSS Shared CI provides a pipeline for each machine, where a template job -# is provided. Each of your jobs must extend this template to be added to the -# list of jobs running on the associated machine. -# -# The job template then expects you to define the "JOB_CMD" variable with the -# one line command used to trigger the build and test of your project. -# -# We suggest that you set your command in such a way that you can then -# customize it per job with variables. E.g.: -# "./path/to/my_ci_script ${A_VARIABLE}" - -## Adding jobs defined by the project. -## Note: placing the extends section first allows you to override part of the -## shared implementation if needed (and if you know what you are doing). -#: -# extends: .job_on_lassen -# variables: -# : "" - -.base-job: - extends: .job_on_lassen - before_script: - - echo "=== Start CI Build and Test===" - after_script: - - rm -rf ${CI_BUILDS_DIR} - - echo "=== End CI Build and Test===" - -.build-variants: - parallel: - matrix: - - MNEME_CI_ENABLE_DEBUG: ["on", "off"] - -build-and-test-lassen: - extends: [.base-job, .build-variants] diff --git a/.gitlab/subscribed-pipelines.yml b/.gitlab/subscribed-pipelines.yml index be885200..f913ec0b 100644 --- a/.gitlab/subscribed-pipelines.yml +++ b/.gitlab/subscribed-pipelines.yml @@ -56,22 +56,6 @@ tioga-integration-laghos: needs: [tioga-up-check] extends: [.integration-laghos] -# # LASSEN -# lassen-up-check: -# variables: -# CI_MACHINE: "lassen" -# extends: [.machine-check] - -# lassen-build-and-test: -# variables: -# CI_MACHINE: "lassen" -# JOB_CMD: -# value: "scripts/gitlab/ci-build-test.sh" -# expand: false -# needs: [lassen-up-check] -# extends: [.build-and-test] -# # #needs: [lassen-up-check, generate-job-lists] - # TUOLUMNE tuolumne-up-check: variables: From 25049269e08a68fb2e820ac5e226b26e97f00e51 Mon Sep 17 00:00:00 2001 From: Loic Pottier Date: Sun, 28 Sep 2025 14:56:11 -0700 Subject: [PATCH 15/21] Fixed a missing guard against None value in Mneme DB Signed-off-by: Loic Pottier --- python/mneme/db.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/mneme/db.py b/python/mneme/db.py index c9364d58..17053422 100644 --- a/python/mneme/db.py +++ b/python/mneme/db.py @@ -179,7 +179,7 @@ def add(self, src_ir: str, dst_ir: str, exp: Experiment): best_speedup, ) - if self._best > exp.exec_time: + if exp.exec_time is not None and self._best > exp.exec_time: self._best = exp.exec_time self._experiments[_hash] = exp.executed From 115d2f34da0f480e38f992d8d7494d6d52b420a4 Mon Sep 17 00:00:00 2001 From: Loic Pottier Date: Tue, 30 Sep 2025 17:01:26 -0700 Subject: [PATCH 16/21] Tests with laghos can now be trigered with /run-laghos Signed-off-by: Loic Pottier --- .gitlab/subscribed-pipelines.yml | 2 +- scripts/gitlab/ci-integration-laghos.sh | 12 ++++---- scripts/gitlab/ci-integration-tests.sh | 37 +++++++++++++++++++++++++ 3 files changed, 44 insertions(+), 7 deletions(-) create mode 100755 scripts/gitlab/ci-integration-tests.sh diff --git a/.gitlab/subscribed-pipelines.yml b/.gitlab/subscribed-pipelines.yml index f913ec0b..e6dfacc7 100644 --- a/.gitlab/subscribed-pipelines.yml +++ b/.gitlab/subscribed-pipelines.yml @@ -78,7 +78,7 @@ tuolumne-integration-laghos: variables: CI_MACHINE: "tuolumne" JOB_CMD: - value: "scripts/gitlab/ci-integration-laghos.sh" + value: "scripts/gitlab/ci-integration-tests.sh" expand: false needs: [tuolumne-up-check] extends: [.integration-laghos] diff --git a/scripts/gitlab/ci-integration-laghos.sh b/scripts/gitlab/ci-integration-laghos.sh index 27d22b58..31efc729 100755 --- a/scripts/gitlab/ci-integration-laghos.sh +++ b/scripts/gitlab/ci-integration-laghos.sh @@ -139,7 +139,6 @@ run_mneme_laghos() { --db-dir ${DB_STORE} \ --tuner-type optuna \ --search-sampler QMCSampler \ - --suffix "ci" \ --prune \ --internalize \ --num-trials 2 \ @@ -148,19 +147,20 @@ run_mneme_laghos() { --no-specialize # Useful to start another run of Mneme to test more features (Mneme reuses previous runs etc) + # Run with --specialize + echo "Running Mneme with --specialize: $JSON_RECORD / $KERNEL_ID" mneme tune \ -db ${JSON_RECORD} \ -rid ${KERNEL_ID} \ --db-dir ${DB_STORE} \ --tuner-type optuna \ --search-sampler QMCSampler \ - --suffix "ci" \ --prune \ --internalize \ --num-trials 2 \ - --iterations 4 \ + --iterations 3 \ --seed 0 \ - --no-specialize + --specialize duration=$SECONDS echo "Mneme optimization: $((duration / 60)) minutes and $((duration % 60)) seconds elapsed." @@ -183,8 +183,8 @@ if [[ "$SYS_TYPE" == "toss_4_x86_64_ib_cray" ]]; then export LLVM_INSTALL_DIR=${ROCM_PATH} # Instll Mneme python bindings if [[ ! -d "${installDir}/mneme-env" || ! -f "${installDir}/mneme-env/bin/activate" ]]; then - python3 -m venv ${installDir}/mneme-env - echo "Created virtual env ${installDir}/mneme-env" + python3 -m venv ${installDir}/mneme-env + echo "Created virtual env ${installDir}/mneme-env" fi source ${installDir}/mneme-env/bin/activate echo "activated virtual env ${installDir}/mneme-env" diff --git a/scripts/gitlab/ci-integration-tests.sh b/scripts/gitlab/ci-integration-tests.sh new file mode 100755 index 00000000..9b867cca --- /dev/null +++ b/scripts/gitlab/ci-integration-tests.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash + +set -e + +echo "CI_COMMIT_REF_NAME ${CI_COMMIT_REF_NAME}" +# Fetch the PR ID from the branch name. +PR_INFO=$(curl -s -L -H "Authorization: Bearer $GITHUB_TOKEN" \ + -H "Accept: application/vnd.github+json" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + "https://api.github.com/repos/Olympus-HPC/Mneme/pulls?head=Olympus-HPC:${CI_COMMIT_REF_NAME}") + +# Check if PR exists. +if [ -z "${PR_INFO}" ] || [ "$(echo "$PR_INFO" | jq length)" = "0" ]; then + echo "No PR found for ref ${CI_COMMIT_REF_NAME}, exit" + exit 0 +fi + +# Extract PR number. +PR_ID=$(echo "${PR_INFO}" | jq -r '.[0].number') +echo "Processing PR ${PR_ID}" + +COMMENTS_INFO=$(curl -L \ + -H "Accept: application/vnd.github+json" \ + -H "Authorization: Bearer $GITHUB_TOKEN" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + "https://api.github.com/repos/Olympus-HPC/Mneme/issues/${PR_ID}/comments") +COMMENTS_BODY=$(echo ${COMMENTS_INFO} | jq -r '.[].body') +if [[ "${COMMENTS_BODY}" == *"/run-laghos"* ]]; then + echo "=> Run Laghos integration tests"; + BENCHMARKS="scripts/gitlab/ci-integration-laghos.sh" +else + echo "=> Invalid command. Available commands: /run-{laghos}" + exit 0 +fi + +# Run the selected benchmarks +(bash -c "${BENCHMARKS}") \ No newline at end of file From 5c629d354524ad235264ec5c663c680b05fd7c76 Mon Sep 17 00:00:00 2001 From: Loic Pottier Date: Wed, 1 Oct 2025 17:15:01 -0700 Subject: [PATCH 17/21] Empty commit to trigger the CI From e3a531fdee4da659ecdf65942b676bd079c2a45d Mon Sep 17 00:00:00 2001 From: Loic Pottier Date: Fri, 3 Oct 2025 14:10:18 -0700 Subject: [PATCH 18/21] Empty commit to trigger the CI From f02e42c1190c364a154ba767b1bfcfb8466b9429 Mon Sep 17 00:00:00 2001 From: Loic Pottier Date: Fri, 3 Oct 2025 14:31:10 -0700 Subject: [PATCH 19/21] Added time measurements and debug print Signed-off-by: Loic Pottier --- scripts/gitlab/ci-integration-tests.sh | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/scripts/gitlab/ci-integration-tests.sh b/scripts/gitlab/ci-integration-tests.sh index 9b867cca..7aa5278e 100755 --- a/scripts/gitlab/ci-integration-tests.sh +++ b/scripts/gitlab/ci-integration-tests.sh @@ -9,6 +9,8 @@ PR_INFO=$(curl -s -L -H "Authorization: Bearer $GITHUB_TOKEN" \ -H "X-GitHub-Api-Version: 2022-11-28" \ "https://api.github.com/repos/Olympus-HPC/Mneme/pulls?head=Olympus-HPC:${CI_COMMIT_REF_NAME}") +echo "PR_INFO = ${PR_INFO}" + # Check if PR exists. if [ -z "${PR_INFO}" ] || [ "$(echo "$PR_INFO" | jq length)" = "0" ]; then echo "No PR found for ref ${CI_COMMIT_REF_NAME}, exit" @@ -33,5 +35,18 @@ else exit 0 fi -# Run the selected benchmarks -(bash -c "${BENCHMARKS}") \ No newline at end of file +RESULTS_COMMIT="Artifacts PR ${PR_ID} commit ${CI_COMMIT_SHORT_SHA}" + +# Run the selected benchmark +SECONDS=0 +bash -c "${BENCHMARKS}" +END_TIME=$SECONDS +COMMENT="${RESULTS_COMMIT}: ${COMMENTS_BODY} ran in ${END_TIME} seconds\n

" + +# Post the comment to the GitHub PR. +curl -L -X POST \ + -H "Accept: application/vnd.github+json" \ + -H "Authorization: Bearer $GITHUB_TOKEN" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + "https://api.github.com/repos/Olympus-HPC/Mneme/issues/${PR_ID}/comments" \ + -d "{\"body\": \"${COMMENT}\"}" From 188cd1a9313bd4e2a81c2416c2e7bb6199c09afc Mon Sep 17 00:00:00 2001 From: Loic Pottier Date: Tue, 7 Oct 2025 14:55:05 -0700 Subject: [PATCH 20/21] Trigger a new CI with a new token From b6f5758fe00146a2aea47bd3a1d01ea3b75671b1 Mon Sep 17 00:00:00 2001 From: Loic Pottier Date: Fri, 10 Oct 2025 14:43:17 -0700 Subject: [PATCH 21/21] Empty commit to trigger the CI Signed-off-by: Loic Pottier