Added Laghos integration tests for Tuo and Tioga

.gitlab-ci.yml

@@ -47,12 +47,6 @@ variables:
 # Required information about GitHub repository
   GITHUB_PROJECT_NAME: "Mneme"
   GITHUB_PROJECT_ORG: "Olympus-HPC"
-# Set the build-and-test command.
-# Nested variables are allowed and useful to customize the job command. We
-# prevent variable expansion so that you can define them at job level.
-  JOB_CMD:
-    value: "scripts/gitlab/ci-build-test.sh"
-    expand: false
 # Override the pattern describing branches that will skip the "draft PR filter
 # test".  Add protected branches here. See default value in
 # preliminary-ignore-draft-pr.yml.
@@ -65,8 +59,7 @@ variables:
 stages:
   - prerequisites
   - build-and-test
-  - test-integration
-  - run-benchmarks
+  - integration-laghos
 
 # Template for jobs triggering a build-and-test sub-pipeline:
 .build-and-test:
@@ -88,30 +81,15 @@ stages:
     forward:
       pipeline_variables: true
 
-.test-integration:
-  stage: test-integration
+.integration-laghos:
+  stage: integration-laghos
   trigger:
     include:
       - local: '.gitlab/custom-jobs-and-variables.yml'
       - project: 'radiuss/radiuss-shared-ci'
         ref: 'v2025.06.0'
         file: 'pipelines/${CI_MACHINE}.yml'
-      - local: '.gitlab/jobs/${CI_MACHINE}-integration.yml'
-    strategy: depend
-    forward:
-      pipeline_variables: true
-
-.run-benchmarks:
-  stage: run-benchmarks
-  trigger:
-    include:
-      - local: '.gitlab/custom-jobs-and-variables.yml'
-      - project: 'radiuss/radiuss-shared-ci'
-        ref: 'v2025.06.0'
-        file: 'pipelines/${CI_MACHINE}.yml'
-      # Add your jobs
-      # you can use a local file
-      - local: '.gitlab/jobs/${CI_MACHINE}-benchmarks.yml'
+      - local: '.gitlab/jobs/${CI_MACHINE}-laghos.yml'
     strategy: depend
     forward:
       pipeline_variables: true

.gitlab/custom-jobs-and-variables.yml

@@ -12,14 +12,6 @@ variables:
 # named so that we are sure to retrieve it and avoid collisions.
   ALLOC_NAME: ${CI_PROJECT_NAME}_ci_${CI_PIPELINE_ID}
 
-# Ruby
-# Arguments for top level allocation
-#  RUBY_SHARED_ALLOC: "--exclusive --reservation=ci --time=10 --nodes=1"
-# Arguments for job level allocation
-#  RUBY_JOB_ALLOC: "--reservation=ci --nodes=1"
-# Add variables that should apply to all the jobs on a machine:
-#  RUBY_MY_VAR: "..."
-
 # Poodle
 # Arguments for top level allocation
   POODLE_SHARED_ALLOC: "--exclusive --partition=pdebug --time=10 --nodes=1"
@@ -40,22 +32,16 @@ variables:
 # Tioga
 # Arguments for top level allocation
 # OPTIONAL: "-o per-resource.count=2" allows to get 2 jobs running on each node.
-  TIOGA_SHARED_ALLOC: "--queue=pci --exclusive --time-limit=15m --nodes=1"
+  TIOGA_SHARED_ALLOC: "--queue=pci --exclusive --time-limit=90m --nodes=1"
 # Arguments for job level allocation
   TIOGA_JOB_ALLOC: "--nodes=1 --begin-time=+5s"
 # Add variables that should apply to all the jobs on a machine:
 #  TIOGA_MY_VAR: "..."
 
-# Lassen uses a different job scheduler (spectrum lsf) that does not allow
-# pre-allocation the same way slurm does. Arguments for job level allocation
-  LASSEN_JOB_ALLOC: "1 -W 60 -q pci"
-# Add variables that should apply to all the jobs on a machine:
-#  LASSEN_MY_VAR: "..."
-
 # Tuo
 # Arguments for top level allocation
 # OPTIONAL: "-o per-resource.count=2" allows to get 2 jobs running on each node.
-  TUOLUMNE_SHARED_ALLOC: "--queue=pci --exclusive --time-limit=15m --nodes=1"
+  TUOLUMNE_SHARED_ALLOC: "--queue=pci --exclusive --time-limit=90m --nodes=1"
 # Arguments for job level allocation
   TUOLUMNE_JOB_ALLOC: "--nodes=1 --begin-time=+5s"
 # Add variables that should apply to all the jobs on a machine:

.gitlab/jobs/tioga-laghos.yml

@@ -0,0 +1,27 @@
+###############################################################################
+# Copyright (c) 2022-23, Lawrence Livermore National Security, LLC and RADIUSS
+# project contributors. See the COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (MIT)
+###############################################################################
+
+.tioga_reproducer_vars:
+  script:
+    - echo -e "Running on tioga\n"
+
+.base-job:
+  extends: .job_on_tioga
+  before_script:
+    - echo "=== Start CI Build and Test==="
+  after_script:
+    - rm -rf ${CI_BUILDS_DIR}
+    - echo "=== End CI Build and Test==="
+
+.build-variants:
+  parallel:
+    matrix:
+      - MNEME_CI_ENABLE_DEBUG: ["on", "off"]
+        MNEME_CI_ROCM_VERSION: ["6.2.1", "6.3.1", "6.4.2"]
+
+integration-laghos-tioga:
+  extends: [.base-job, .build-variants]

.gitlab/jobs/tuolumne-laghos.yml

@@ -0,0 +1,27 @@
+###############################################################################
+# Copyright (c) 2022-23, Lawrence Livermore National Security, LLC and RADIUSS
+# project contributors. See the COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (MIT)
+###############################################################################
+
+.tuolumne_reproducer_vars:
+  script:
+    - echo -e "Running on tuolumne\n"
+
+.base-job:
+  extends: .job_on_tuolumne
+  before_script:
+    - echo "=== Start CI Build and Test==="
+  after_script:
+    - rm -rf ${CI_BUILDS_DIR}
+    - echo "=== End CI Build and Test==="
+
+.build-variants:
+  parallel:
+    matrix:
+      - MNEME_CI_ENABLE_DEBUG: ["on", "off"]
+        MNEME_CI_ROCM_VERSION: ["6.2.1", "6.3.1", "6.4.2"]
+
+integration-laghos-tuolumne:
+  extends: [.base-job, .build-variants]

.gitlab/subscribed-pipelines.yml

@@ -29,18 +29,6 @@
 # Comment the jobs for machines you don’t need.
 ###
 
-      ## RUBY
-      #ruby-up-check:
-      #  variables:
-      #    CI_MACHINE: "ruby"
-      #  extends: [.machine-check]
-      #
-      #ruby-build-and-test:
-      #  variables:
-      #    CI_MACHINE: "ruby"
-      #  needs: [ruby-up-check]
-      #  ## [Shared jobs scenario]
-
 # TIOGA
 tioga-up-check:
   variables:
@@ -50,25 +38,23 @@ tioga-up-check:
 tioga-build-and-test:
   variables:
     CI_MACHINE: "tioga"
+    JOB_CMD:
+      value: "scripts/gitlab/ci-build-test.sh"
+      expand: false
   needs: [tioga-up-check]
   ## [Shared jobs scenario]
   ## (See Umpire CI setup for an example).
   #needs: [tioga-up-check, generate-job-lists]
   extends: [.build-and-test]
 
-# # LASSEN
-# lassen-up-check:
-#   variables:
-#     CI_MACHINE: "lassen"
-#   extends: [.machine-check]
-
-# lassen-build-and-test:
-#   variables:
-#     CI_MACHINE: "lassen"
-#   needs: [lassen-up-check]
-#   extends: [.build-and-test]
-# #  #needs: [lassen-up-check, generate-job-lists]
-# #
+tioga-integration-laghos:
+  variables:
+    CI_MACHINE: "tioga"
+    JOB_CMD:
+      value: "scripts/gitlab/ci-integration-laghos.sh"
+      expand: false
+  needs: [tioga-up-check]
+  extends: [.integration-laghos]
 
 # TUOLUMNE
 tuolumne-up-check:
@@ -79,8 +65,20 @@ tuolumne-up-check:
 tuolumne-build-and-test:
   variables:
     CI_MACHINE: "tuolumne"
+    JOB_CMD:
+      value: "scripts/gitlab/ci-build-test.sh"
+      expand: false
   needs: [tuolumne-up-check]
   ## [Shared jobs scenario]
   ## (See Umpire CI setup for an example).
   #needs: [tuolumne-up-check, generate-job-lists]
   extends: [.build-and-test]
+
+tuolumne-integration-laghos:
+  variables:
+    CI_MACHINE: "tuolumne"
+    JOB_CMD:
+      value: "scripts/gitlab/ci-integration-tests.sh"
+      expand: false
+  needs: [tuolumne-up-check]
+  extends: [.integration-laghos]

python/mneme/db.py

@@ -179,7 +179,7 @@ def add(self, src_ir: str, dst_ir: str, exp: Experiment):
                 best_speedup,
             )
 
-        if self._best > exp.exec_time:
+        if exp.exec_time is not None and self._best > exp.exec_time:
             self._best = exp.exec_time
 
         self._experiments[_hash] = exp.executed

scripts/gitlab/ci-build-test.sh

@@ -1,26 +1,29 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 set -e
 
 temp_dir=$(pwd) #$(mktemp -d)
 echo "Temporary directory created at: $temp_dir"
 host=$(hostname)
 host=${host//[0-9]/}
-mkdir -p ${temp_dir}/build-${host};
 build_dir=${temp_dir}/build-${host}
+mkdir -p ${build_dir}
 installDir="/dev/shm/install"
 
+start_test=$(date +'%s')
 
 build_proteus() {
   echo "Building PROTEUS"
-  git clone --depth 1 git@github.com:Olympus-HPC/proteus.git
+  if [[ ! -d "proteus" ]]; then
+    git clone --depth 1 https://github.com/Olympus-HPC/proteus.git
+  fi
   pushd proteus 
   PROTEUS_ENABLE_HIP=$1
   PROTEUS_ENABLE_CUDA=$2
   PROTEUS_INSTALL_DIR=$3
   LINK_SHARED_LLVM=$4
   echo "Proteus: ENABLE_HIP: $PROTEUS_ENABLE_HIP ENABLE_CUDA: $PROTEUS_ENABLE_CUDA"
-  mkdir build-proteus-${host}
+  mkdir -p build-proteus-${host}
   pushd build-proteus-${host}
   cmake .. \
   -DBUILD_SHARED=Off \
@@ -42,10 +45,12 @@ build_proteus() {
 
 build_spdlog() {
   echo "Building SPDLOG"
-  git clone --depth 1 --branch v1.15.0  --single-branch https://github.com/gabime/spdlog.git
+  if [[ ! -d "spdlog" ]]; then
+    git clone --depth 1 --branch v1.15.0  --single-branch https://github.com/gabime/spdlog.git
+  fi
   pushd spdlog
   SPDLOG_INSTALL_DIR=$1
-  mkdir build-spdlog-${host}
+  mkdir -p build-spdlog-${host}
   pushd build-spdlog-${host}
   cmake \
   -DCMAKE_C_COMPILER=${LLVM_INSTALL_DIR}/bin/clang \
@@ -114,6 +119,14 @@ elif [[ "$SYS_TYPE" == "toss_4_x86_64_ib_cray" ]]; then
 ml load rocm/${MNEME_CI_ROCM_VERSION}
 export LLVM_INSTALL_DIR=${ROCM_PATH}/llvm
 echo "LLVM INSTALL DIR is ${LLVM_INSTALL_DIR}"
+export ROCM_ARCH=$(rocm_agent_enumerator | sed -n 2p)
+
+if [ -z "${ROCM_ARCH}" ]; then
+  echo "ROCM_ARCH is not set or is empty"
+  exit
+else
+  echo "ROCM_ARCH = ${ROCM_ARCH}"
+fi
 
 build_proteus "ON" "OFF" $installDir OFF
 echo "After proteus Current directory is $(pwd)"