WIP: TPL build for czmatrix support

.github/workflows/ci-tests.yml

@@ -12,9 +12,9 @@ concurrency:
   cancel-in-progress: ${{ github.ref != 'refs/heads/develop' }}
 
 env:
-  CLANG_DOCKER_IMAGE: seracllnl/tpls:clang-19_07-02-26_18h-01m
+  CLANG_DOCKER_IMAGE: seracllnl/tpls:clang-19_08-28-26_19h-02m
   CUDA_DOCKER_IMAGE: seracllnl/tpls:cuda-12_01-09-26_02h-37m
-  GCC_DOCKER_IMAGE: seracllnl/tpls:gcc-14_07-02-26_18h-06m
+  GCC_DOCKER_IMAGE: seracllnl/tpls:gcc-14_08-28-26_19h-02m
 
 jobs:
   # Hacky solution to reference env variables outside of `run` steps https://stackoverflow.com/a/74217028

.gitlab-ci.yml

@@ -75,10 +75,8 @@ stages:
   rules:
     - if: $SMITH_CI_WORKFLOW_TYPE == "comparison"
   dependencies:
-    # TODO re-enable with CZMatrix
-    # - blueos-clang_14_0_5-benchmarks
-    - toss4-gcc_13_3_1-benchmarks
-    - toss4-llvm_19_1_3-benchmarks
+    - dane-gcc_13_3_1-benchmarks
+    - dane-llvm_19_1_3-benchmarks
 
 ####
 # Templates
@@ -109,7 +107,7 @@ stages:
     # Build TPLs, src, and run unit tests
     - echo -e "section_start:$(date +%s):tpl_build_and_unit_test\r\e[0K
       TPL Build and Unit Test ${CI_PROJECT_NAME}"
-    - ${ALLOC_COMMAND} python3 scripts/llnl/build_tpls.py -v ${SPEC} --directory=${FULL_BUILD_ROOT} --short-path
+    - ${ALLOC_COMMAND} python3 scripts/llnl/build_tpls.py -v --spec="${SPEC}" --directory=${FULL_BUILD_ROOT} --short-path
     - echo -e "section_end:$(date +%s):tpl_build_and_unit_test\r\e[0K"
   artifacts:
     expire_in: 2 weeks
@@ -165,9 +163,8 @@ stages:
 
 # This is where jobs are included for each system
 include:
-  # TODO re-enable with CZMatrix
-  # - local: .gitlab/build_blueos.yml
-  - local: .gitlab/build_toss4.yml
-  - local: .gitlab/build_toss4_cray.yml
+  - local: .gitlab/build_dane.yml
+  - local: .gitlab/build_matrix.yml
+  - local: .gitlab/build_tuolumne.yml
   - project: 'lc-templates/id_tokens'
     file: 'id_tokens.yml'

.gitlab/build_toss4.yml → .gitlab/build_dane.yml

@@ -1,14 +1,14 @@
 ####
-# This is the shared configuration of jobs for toss4
-.on_toss4:
+# This is the shared configuration of jobs for dane
+.on_dane:
   variables:
     SCHEDULER_PARAMETERS: "--reservation=ci --exclusive=user --deadline=now+${ALLOC_DEADLINE}minutes -N ${ALLOC_NODES} -t ${ALLOC_TIME} -A ${ALLOC_BANK}"
     ALLOC_DEADLINE: "180" # Allocation times must be less than this value
   tags:
     - batch
     - dane
   rules:
-    - if: '$CI_COMMIT_BRANCH =~ /_qnone/ || $ON_TOSS4 == "OFF"' #run except if ...
+    - if: '$CI_COMMIT_BRANCH =~ /_qnone/ || $ON_DANE == "OFF"' #run except if ...
       when: never
   before_script:
     # We could also extract the CMake executable location from the hostconfig in common_build_functions
@@ -18,32 +18,32 @@
 
 ####
 # Templates
-.src_build_on_toss4:
-  extends: [.src_build_script, .on_toss4, .src_workflow]
+.src_build_on_dane:
+  extends: [.src_build_script, .on_dane, .src_workflow]
 
 # NOTE: the asan build is just a src build with a different pipeline schedule, so re-use src build script.
-.asan_build_on_toss4:
-  extends: [.src_build_script, .on_toss4, .asan_workflow]
+.asan_build_on_dane:
+  extends: [.src_build_script, .on_dane, .asan_workflow]
 
-.examples_build_on_toss4:
-  extends: [.examples_build_script, .on_toss4, .examples_workflow]
+.examples_build_on_dane:
+  extends: [.examples_build_script, .on_dane, .examples_workflow]
 
-.tpl_build_on_toss4:
-  extends: [.tpl_build_script, .on_toss4, .tpl_workflow]
+.tpl_build_on_dane:
+  extends: [.tpl_build_script, .on_dane, .tpl_workflow]
   before_script:
     # LC version of pip is ancient
     - if [[ $(python3 -c 'import pip; print(pip.__version__ < "19.3")') == "True" ]]; then python3 -m pip install --user --upgrade pip; fi
 
-.benchmarks_build_on_toss4:
-  extends: [.benchmarks_build_script, .on_toss4, .benchmarks_workflow]
+.benchmarks_build_on_dane:
+  extends: [.benchmarks_build_script, .on_dane, .benchmarks_workflow]
 
-.comparison_build_on_toss4:
-  extends: [.comparison_build_script, .on_toss4, .comparison_workflow]
+.comparison_build_on_dane:
+  extends: [.comparison_build_script, .on_dane, .comparison_workflow]
 
 ####
 # Build jobs
 
-toss4-llvm_19_1_3-src:
+dane-llvm_19_1_3-src:
   variables:
     COMPILER: "llvm@19.1.3"
     HOST_CONFIG: "dane-toss_4_x86_64_ib-${COMPILER}.cmake"
@@ -53,108 +53,106 @@ toss4-llvm_19_1_3-src:
     # ALLOC_NODES: "2"
     ALLOC_NODES: "1"
     ALLOC_TIME: "30"
-  extends: .src_build_on_toss4
+  extends: .src_build_on_dane
 
-toss4-llvm_19_1_3-src-codevelop:
+dane-llvm_19_1_3-src-codevelop:
   variables:
     COMPILER: "llvm@19.1.3"
     HOST_CONFIG: "dane-toss_4_x86_64_ib-${COMPILER}.cmake"
     EXTRA_CMAKE_OPTIONS: "-DENABLE_BENCHMARKS=ON -DENABLE_DOCS=OFF -DCMAKE_BUILD_TYPE=Debug -DSMITH_ENABLE_CODEVELOP=ON"
     EXTRA_BUILD_OPTIONS: "--skip-install"
     ALLOC_NODES: "1"
     ALLOC_TIME: "30"
-  extends: .src_build_on_toss4
+  extends: .src_build_on_dane
 
-toss4-gcc_13_3_1-src:
+dane-gcc_13_3_1-src:
   variables:
     COMPILER: "gcc@13.3.1"
     HOST_CONFIG: "dane-toss_4_x86_64_ib-${COMPILER}.cmake"
     EXTRA_CMAKE_OPTIONS: "-DENABLE_BENCHMARKS=ON -DENABLE_DOCS=OFF -DCMAKE_BUILD_TYPE=Debug"
     ALLOC_NODES: "1"
     ALLOC_TIME: "30"
-  extends: .src_build_on_toss4
+  extends: .src_build_on_dane
 
-toss4-gcc_13_3_1-src-no-tribol:
+dane-gcc_13_3_1-src-no-tribol:
   variables:
     COMPILER: "gcc@13.3.1"
     HOST_CONFIG: "dane-toss_4_x86_64_ib-${COMPILER}.cmake"
     EXTRA_CMAKE_OPTIONS: "-DENABLE_BENCHMARKS=ON -DENABLE_DOCS=OFF -DCMAKE_BUILD_TYPE=Debug -UTRIBOL_DIR"
     ALLOC_NODES: "1"
     ALLOC_TIME: "30"
-  extends: .src_build_on_toss4
+  extends: .src_build_on_dane
 
-toss4-gcc_13_3_1-src-no-optional-solvers:
+dane-gcc_13_3_1-src-no-optional-solvers:
   variables:
     COMPILER: "gcc@13.3.1"
     HOST_CONFIG: "dane-toss_4_x86_64_ib-${COMPILER}.cmake"
     EXTRA_CMAKE_OPTIONS: "-DENABLE_BENCHMARKS=ON -DENABLE_DOCS=OFF -DCMAKE_BUILD_TYPE=Debug -USUNDIALS_DIR -UPETSC_DIR"
     ALLOC_NODES: "1"
     ALLOC_TIME: "30"
-  extends: .src_build_on_toss4
+  extends: .src_build_on_dane
 
-toss4-llvm_19_1_3-src-asan:
+dane-llvm_19_1_3-src-asan:
   variables:
     COMPILER: "llvm@19.1.3"
     HOST_CONFIG: "dane-toss_4_x86_64_ib-${COMPILER}.cmake"
     EXTRA_CMAKE_OPTIONS: "-DENABLE_DOCS=OFF -DCMAKE_BUILD_TYPE=Debug -DENABLE_ASAN=ON"
     LSAN_OPTIONS: "suppressions=${CI_PROJECT_DIR}/suppressions.asan"
     ALLOC_NODES: "1"
     ALLOC_TIME: "30"
-  extends: .asan_build_on_toss4
+  extends: .asan_build_on_dane
 
-toss4-llvm_19_1_3-tpl:
+dane-llvm_19_1_3-tpl:
   variables:
-    COMPILER: "llvm@19.1.3"
-    SPEC: "--spec %${COMPILER}"
+    SPEC: "%clang_19"
     ALLOC_NODES: "1"
     ALLOC_TIME: "45"
-  extends: .tpl_build_on_toss4
+  extends: .tpl_build_on_dane
 
-toss4-gcc_13_3_1-tpl:
+dane-gcc_13_3_1-tpl:
   variables:
-    COMPILER: "gcc@13.3.1"
-    SPEC: "--spec %${COMPILER}"
+    SPEC: "%gcc_13"
     ALLOC_NODES: "1"
     ALLOC_TIME: "45"
-  extends: .tpl_build_on_toss4
+  extends: .tpl_build_on_dane
 
-toss4-llvm_19_1_3-examples:
+dane-llvm_19_1_3-examples:
   variables:
     COMPILER: "llvm@19.1.3"
     HOST_CONFIG: "dane-toss_4_x86_64_ib-${COMPILER}.cmake"
     EXTRA_CMAKE_OPTIONS: "-DCMAKE_BUILD_TYPE=Debug"
     ALLOC_NODES: "1"
     ALLOC_TIME: "30"
-  extends: .examples_build_on_toss4
+  extends: .examples_build_on_dane
 
-toss4-gcc_13_3_1-examples:
+dane-gcc_13_3_1-examples:
   variables:
     COMPILER: "gcc@13.3.1"
     HOST_CONFIG: "dane-toss_4_x86_64_ib-${COMPILER}.cmake"
     EXTRA_CMAKE_OPTIONS: "-DCMAKE_BUILD_TYPE=Debug"
     ALLOC_NODES: "1"
     ALLOC_TIME: "30"
-  extends: .examples_build_on_toss4
+  extends: .examples_build_on_dane
 
-toss4-llvm_19_1_3-benchmarks:
+dane-llvm_19_1_3-benchmarks:
   variables:
     COMPILER: "llvm@19.1.3"
     HOST_CONFIG: "dane-toss_4_x86_64_ib-${COMPILER}.cmake"
     ALLOC_NODES: "1"
     ALLOC_TIME: "75"
-  extends: .benchmarks_build_on_toss4
+  extends: .benchmarks_build_on_dane
 
-toss4-gcc_13_3_1-benchmarks:
+dane-gcc_13_3_1-benchmarks:
   variables:
     COMPILER: "gcc@13.3.1"
     HOST_CONFIG: "dane-toss_4_x86_64_ib-${COMPILER}.cmake"
     ALLOC_NODES: "1"
     ALLOC_TIME: "75"
-  extends: .benchmarks_build_on_toss4
+  extends: .benchmarks_build_on_dane
 
 # This comparison job simply runs Hatchet and compares against benchmarks generated across all configurations
-toss4-comparison:
+dane-comparison:
   variables:
     ALLOC_NODES: "1"
     ALLOC_TIME: "30"
-  extends: .comparison_build_on_toss4
+  extends: .comparison_build_on_dane

.gitlab/build_matrix.yml

@@ -0,0 +1,52 @@
+####
+# This is the shared configuration of jobs for matrix
+.on_matrix:
+  variables:
+    SCHEDULER_PARAMETERS: "--partition=pci --exclusive=user --deadline=now+2hour -N1 -t ${ALLOC_TIME}"
+  tags:
+    - batch
+    - matrix
+  rules:
+    - if: '$CI_COMMIT_BRANCH =~ /_qnone/ || $ON_MATRIX == "OFF"' #run except if ...
+      when: never
+  before_script:
+    - module load cmake/3.30.5
+
+####
+# Template
+.src_build_on_matrix:
+  extends: [.src_build_script, .on_matrix, .src_workflow]
+
+.tpl_build_on_matrix:
+  extends: [.tpl_build_script, .on_matrix, .tpl_workflow]
+
+####
+# PR Build jobs
+matrix-llvm_19_1_3_cuda-src:
+  variables:
+    COMPILER: "llvm@19.1.3_cuda"
+    HOST_CONFIG: "matrix-toss_4_x86_64_ib-${COMPILER}.cmake"
+    ALLOC_TIME: "30"
+  extends: .src_build_on_matrix
+
+# NOTE: Reducing load put on Matrix, as it has limited resources
+# matrix-gcc_13_3_1_cuda-src:
+#   variables:
+#     COMPILER: "gcc@13.3.1_cuda"
+#     HOST_CONFIG: "matrix-toss_4_x86_64_ib-${COMPILER}.cmake"
+#     ALLOC_TIME: "30"
+# extends: .src_build_on_matrix
+
+####
+# TPL Build jobs
+matrix-llvm_19_1_3_cuda-tpl:
+  variables:
+    SPEC: "~devtools~profiling+adiak+caliper~enzyme cuda_arch=90 %clang_19"
+    ALLOC_TIME: "30"
+  extends: .src_build_on_matrix
+
+# matrix-gcc_13_3_1_cuda-tpl:
+#   variables:
+#     SPEC: "~devtools~profiling+adiak+caliper~enzyme cuda_arch=90 %gcc_13"
+#     ALLOC_TIME: "45"
+#   extends: .tpl_build_on_matrix