From cf333b4018dd829d5ebd3a7865655e894a0ca479 Mon Sep 17 00:00:00 2001 From: Adam Dempsey Date: Fri, 7 Aug 2026 08:31:05 -0600 Subject: [PATCH 1/8] Blindly try adding darwin amd CI. Env is probably not correct yet --- CMakePresets.json | 23 +++++++++++++++++++++++ env/bash | 17 +++++++++++++++++ tst/launch_ci_runner.py | 6 ++++++ 3 files changed, 46 insertions(+) diff --git a/CMakePresets.json b/CMakePresets.json index 3fe06fe1..4e7d0fec 100644 --- a/CMakePresets.json +++ b/CMakePresets.json @@ -57,6 +57,29 @@ "Kokkos_ARCH_VOLTA70": "ON" } }, + { + "name": "darwin-mi250-debug", + "cacheVariables": { + "CMAKE_MAKE_PROGRAM": "$env{MAKE_PROGRAM}", + "CMAKE_BUILD_TYPE": "Debug", + "Kokkos_ENABLE_DEBUG_BOUNDS_CHECK": "ON", + "CMAKE_C_COMPILER": "amdclang", + "CMAKE_CXX_COMPILER": "amdclang++", + "ARTEMIS_ENABLE_HIP": "ON", + "Kokkos_ARCH_AMD_GFX90A": "ON" + } + }, + { + "name": "darwin-mi250-release", + "cacheVariables": { + "CMAKE_MAKE_PROGRAM": "$env{MAKE_PROGRAM}", + "CMAKE_BUILD_TYPE": "Release", + "CMAKE_C_COMPILER": "amdclang", + "CMAKE_CXX_COMPILER": "amdclang++", + "ARTEMIS_ENABLE_HIP": "ON", + "Kokkos_ARCH_AMD_GFX90A": "ON" + } + }, { "name": "chicoma-cpu-debug", "cacheVariables": { diff --git a/env/bash b/env/bash index ab3ee9b7..67d391f4 100644 --- a/env/bash +++ b/env/bash @@ -45,6 +45,7 @@ else # Catch-all for Darwin echo "Supported partitions are" echo " skylake-gold" echo " volta-x86" + echo " mi250" PARTITION="darwin-fe" elif [[ $SLURM_JOB_PARTITION == "power9-rhel7" ]]; then PARTITION="darwin-power9-rhel7" @@ -52,6 +53,8 @@ else # Catch-all for Darwin PARTITION="darwin-skylake-gold" elif [[ $SLURM_JOB_PARTITION == "volta-x86" ]]; then PARTITION="darwin-volta-x86" + elif [[ $SLURM_JOB_PARTITION == "mi250" ]]; then + PARTITION="darwin-mi250" fi fi @@ -163,6 +166,20 @@ elif [[ $PARTITION == "darwin-volta-x86" ]]; then module list export ARTEMIS_SUITE=darwin-gpu echo "...setup SUCCEEDED" +elif [[ $PARTITION == "darwin-mi250" ]]; then + module purge + module load clang/12.0.1 + module load rocm + module load openmpi + module load gcc/12.2.0 + module load hdf5/1.12.2 + module load miniconda3/py311_23.11.0 + source /usr/projects/jovian/dependencies/python/volta-x86-py311/bin/activate + shorten_prompt + module load cmake/3.26.3 + module list + export ARTEMIS_SUITE=darwin-mi250 + echo "...setup SUCCEEDED" elif [[ $PARTITION == "venado-gh" ]]; then module unload cray-libsci module load PrgEnv-gnu diff --git a/tst/launch_ci_runner.py b/tst/launch_ci_runner.py index 28fa8cde..c41053b8 100755 --- a/tst/launch_ci_runner.py +++ b/tst/launch_ci_runner.py @@ -152,6 +152,12 @@ def run_tests_in_temp_dir( subprocess.run(run_cmd) + run_cmd = ["./darwin_amd_ci.py", str(args.pr_number)] + if args.output_dir: + run_cmd.append("--output_dir=" + str(args.output_dir)) + + subprocess.run(run_cmd) + run_cmd = ["./darwin_gpu_ci.py", str(args.pr_number)] if args.output_dir: run_cmd.append("--output_dir=" + str(args.output_dir)) From 0e06ed328f31f29e367e93dc16602f6b4f2d27a3 Mon Sep 17 00:00:00 2001 From: Adam Dempsey Date: Fri, 7 Aug 2026 08:34:02 -0600 Subject: [PATCH 2/8] Actually add the script --- tst/darwin_amd_ci.py | 145 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 145 insertions(+) create mode 100644 tst/darwin_amd_ci.py diff --git a/tst/darwin_amd_ci.py b/tst/darwin_amd_ci.py new file mode 100644 index 00000000..9cbcf960 --- /dev/null +++ b/tst/darwin_amd_ci.py @@ -0,0 +1,145 @@ +#!/usr/bin/env python3 +# ======================================================================================== +# (C) (or copyright) 2026. Triad National Security, LLC. All rights reserved. +# +# This program was produced under U.S. Government contract 89233218CNA000001 for Los +# Alamos National Laboratory (LANL), which is operated by Triad National Security, LLC +# for the U.S. Department of Energy/National Nuclear Security Administration. All rights +# in the program are reserved by Triad National Security, LLC, and the U.S. Department +# of Energy/National Nuclear Security Administration. The Government is granted for +# itself and others acting on its behalf a nonexclusive, paid-up, irrevocable worldwide +# license in this material to reproduce, prepare derivative works, distribute copies to +# the public, perform publicly and display publicly, and to permit others to do so. +# ======================================================================================== + +import argparse +import fnmatch +import os +import shlex +import socket +import subprocess +import sys +from datetime import datetime + +from launch_ci_runner import * + +GITHUB_TOKEN = os.environ.get("ARTEMIS_GITHUB_TOKEN") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="Run CI tasks with optional Slurm submission." + ) + parser.add_argument( + "pr_number", type=int, help="Pull request number for the CI run." + ) + parser.add_argument( + "--submission", + action="store_true", + help="Flag to indicate the script is running as a Slurm submission job.", + ) + parser.add_argument( + "--output_dir", + type=str, + default=None, + help="Output directory created when launching submission script", + ) + args = parser.parse_args() + + pr_info = get_pr_info(args.pr_number) + head_repo = pr_info["head"]["repo"]["clone_url"] + head_ref = pr_info["head"]["ref"] + commit_sha = pr_info["head"]["sha"] + + context = "Continuous Integration / darwin_mi250" + test_suite = "gpu.suite" + suffix = "amd" + + if args.submission: + update_status(commit_sha, "pending", "CI Slurm job running...", context) + test_success = run_tests_in_temp_dir( + args.pr_number, head_repo, head_ref, args.output_dir, test_suite, suffix + ) + if test_success: + update_status(commit_sha, "success", "All tests passed.", context) + else: + update_status(commit_sha, "failure", "Tests failed.", context) + else: + hostname = socket.gethostname() + cluster = os.getenv("SLURM_CLUSTER_NAME") + if not fnmatch.fnmatch(hostname, "darwin-fe*"): + if cluster is None or cluster.lower() != "darwin": + print("ERROR script must be run from Darwin!") + sys.exit(1) + + try: + job_name = f"artemis_ci_darwin_mi250_PR{args.pr_number}" + squeue_command = ( + f"squeue --name={shlex.quote(job_name)} --user=$(whoami) " + "--noheader --format=%i" + ) + squeue_result = subprocess.run( + squeue_command, + shell=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + ) + + job_ids = squeue_result.stdout.strip().split() + if job_ids: + print("Canceling jobs:") + for job_id in job_ids: + print(f" {job_id}") + subprocess.run( + ["scancel"] + job_ids, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + ) + + current_date_time = datetime.now().strftime("%Y-%m-%d_%H:%M:%S") + output_dir = os.path.join( + "/usr", + "projects", + "jovian", + "ci", + "artemis", + f"pr_{args.pr_number}", + current_date_time, + ) + subprocess.run(["mkdir", "-p", output_dir], check=True) + + sbatch_command = [ + "sbatch", + f"--job-name={job_name}", + f"--output={os.path.join(output_dir, job_name)}_%j.out", + f"--error={os.path.join(output_dir, job_name)}_%j.out", + "--partition=mi250", + "--time=04:00:00", + "--wrap", + f"python3 {sys.argv[0]} {args.pr_number} --submission --output_dir {output_dir}", + ] + result = subprocess.run( + sbatch_command, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=True, + universal_newlines=True, + ) + print(result.stdout.strip()) + update_status(commit_sha, "pending", "CI SLURM job submitted...", context) + except Exception as err: + update_status( + commit_sha, + "failure", + "SLURM job submission failed with error: " + repr(err), + context, + ) + finally: + update_status( + commit_sha, + "failure", + "SLURM job submission didn't complete successfully", + context, + ) From f4b631695703cefbf9ca89bb4ddfca265c0809b7 Mon Sep 17 00:00:00 2001 From: Adam Dempsey Date: Fri, 7 Aug 2026 09:24:25 -0600 Subject: [PATCH 3/8] gcc toolchain --- CMakePresets.json | 2 ++ env/bash | 5 ++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/CMakePresets.json b/CMakePresets.json index 4e7d0fec..63cd15df 100644 --- a/CMakePresets.json +++ b/CMakePresets.json @@ -65,6 +65,7 @@ "Kokkos_ENABLE_DEBUG_BOUNDS_CHECK": "ON", "CMAKE_C_COMPILER": "amdclang", "CMAKE_CXX_COMPILER": "amdclang++", + "CMAKE_CXX_FLAGS": "--gcc-toolchain=$env{GCC_ROOT}", "ARTEMIS_ENABLE_HIP": "ON", "Kokkos_ARCH_AMD_GFX90A": "ON" } @@ -76,6 +77,7 @@ "CMAKE_BUILD_TYPE": "Release", "CMAKE_C_COMPILER": "amdclang", "CMAKE_CXX_COMPILER": "amdclang++", + "CMAKE_CXX_FLAGS": "--gcc-toolchain=$env{GCC_ROOT}", "ARTEMIS_ENABLE_HIP": "ON", "Kokkos_ARCH_AMD_GFX90A": "ON" } diff --git a/env/bash b/env/bash index 67d391f4..e697ca1e 100644 --- a/env/bash +++ b/env/bash @@ -168,16 +168,15 @@ elif [[ $PARTITION == "darwin-volta-x86" ]]; then echo "...setup SUCCEEDED" elif [[ $PARTITION == "darwin-mi250" ]]; then module purge - module load clang/12.0.1 + module load openmpi/4.1.6-gcc_13.2.0 module load rocm - module load openmpi - module load gcc/12.2.0 module load hdf5/1.12.2 module load miniconda3/py311_23.11.0 source /usr/projects/jovian/dependencies/python/volta-x86-py311/bin/activate shorten_prompt module load cmake/3.26.3 module list + export GCC_ROOT="$(dirname "$(dirname "$(readlink -f "$(command -v g++)")")")" export ARTEMIS_SUITE=darwin-mi250 echo "...setup SUCCEEDED" elif [[ $PARTITION == "venado-gh" ]]; then From 60111f240fb3021c937fa9593d5282847b140cbc Mon Sep 17 00:00:00 2001 From: Adam Dempsey Date: Fri, 7 Aug 2026 10:03:03 -0600 Subject: [PATCH 4/8] Why limit to 4 cores --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 93668954..c3a9ef12 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -69,7 +69,7 @@ jobs: mkdir -p build cd build cmake --preset=cpu-release ../ - make -j 4 + make -j ctest -L unit --output-on-failure regression-tests: @@ -97,7 +97,7 @@ jobs: mkdir -p build cd build cmake --preset=cpu-release ../../ - make -j 4 + make -j cd .. python3 run_tests.py regression.suite \ --save_build \ From e2ddcc6a4bd8df7c2ee6c82b050f50bbde3a1b87 Mon Sep 17 00:00:00 2001 From: Adam Dempsey Date: Fri, 7 Aug 2026 10:04:01 -0600 Subject: [PATCH 5/8] Rename suite to darwin-amd --- CMakePresets.json | 4 ++-- env/bash | 4 ++-- tst/darwin_amd_ci.py | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/CMakePresets.json b/CMakePresets.json index 63cd15df..a5653442 100644 --- a/CMakePresets.json +++ b/CMakePresets.json @@ -58,7 +58,7 @@ } }, { - "name": "darwin-mi250-debug", + "name": "darwin-amd-debug", "cacheVariables": { "CMAKE_MAKE_PROGRAM": "$env{MAKE_PROGRAM}", "CMAKE_BUILD_TYPE": "Debug", @@ -71,7 +71,7 @@ } }, { - "name": "darwin-mi250-release", + "name": "darwin-amd-release", "cacheVariables": { "CMAKE_MAKE_PROGRAM": "$env{MAKE_PROGRAM}", "CMAKE_BUILD_TYPE": "Release", diff --git a/env/bash b/env/bash index e697ca1e..98906082 100644 --- a/env/bash +++ b/env/bash @@ -53,7 +53,7 @@ else # Catch-all for Darwin PARTITION="darwin-skylake-gold" elif [[ $SLURM_JOB_PARTITION == "volta-x86" ]]; then PARTITION="darwin-volta-x86" - elif [[ $SLURM_JOB_PARTITION == "mi250" ]]; then + elif [[ $SLURM_JOB_PARTITION == "shared-gpu-amd-mi250" ]]; then PARTITION="darwin-mi250" fi fi @@ -177,7 +177,7 @@ elif [[ $PARTITION == "darwin-mi250" ]]; then module load cmake/3.26.3 module list export GCC_ROOT="$(dirname "$(dirname "$(readlink -f "$(command -v g++)")")")" - export ARTEMIS_SUITE=darwin-mi250 + export ARTEMIS_SUITE=darwin-amd echo "...setup SUCCEEDED" elif [[ $PARTITION == "venado-gh" ]]; then module unload cray-libsci diff --git a/tst/darwin_amd_ci.py b/tst/darwin_amd_ci.py index 9cbcf960..c80c514f 100644 --- a/tst/darwin_amd_ci.py +++ b/tst/darwin_amd_ci.py @@ -115,8 +115,8 @@ f"--job-name={job_name}", f"--output={os.path.join(output_dir, job_name)}_%j.out", f"--error={os.path.join(output_dir, job_name)}_%j.out", - "--partition=mi250", - "--time=04:00:00", + "--partition=shared-gpu-amd-mi250", + "--time=02:00:00", "--wrap", f"python3 {sys.argv[0]} {args.pr_number} --submission --output_dir {output_dir}", ] From 4a21ceacda8fe8744f18034aa3ed5865be213bb8 Mon Sep 17 00:00:00 2001 From: Adam Dempsey Date: Fri, 7 Aug 2026 12:47:50 -0600 Subject: [PATCH 6/8] Make +x --- tst/darwin_amd_ci.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 tst/darwin_amd_ci.py diff --git a/tst/darwin_amd_ci.py b/tst/darwin_amd_ci.py old mode 100644 new mode 100755 From 2be8aa1b8245e061334730b52a68d6cdeb042116 Mon Sep 17 00:00:00 2001 From: Adam Dempsey Date: Fri, 7 Aug 2026 12:52:55 -0600 Subject: [PATCH 7/8] Make amd look exactly like the others --- tst/darwin_amd_ci.py | 60 ++++++++++++++++++++++++++++++++------------ 1 file changed, 44 insertions(+), 16 deletions(-) diff --git a/tst/darwin_amd_ci.py b/tst/darwin_amd_ci.py index c80c514f..94ecfea7 100755 --- a/tst/darwin_amd_ci.py +++ b/tst/darwin_amd_ci.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # ======================================================================================== -# (C) (or copyright) 2026. Triad National Security, LLC. All rights reserved. +# (C) (or copyright) 2024. Triad National Security, LLC. All rights reserved. # # This program was produced under U.S. Government contract 89233218CNA000001 for Los # Alamos National Laboratory (LANL), which is operated by Triad National Security, LLC @@ -12,17 +12,24 @@ # the public, perform publicly and display publicly, and to permit others to do so. # ======================================================================================== -import argparse +# This file was created in part or in whole by one of OpenAI's generative AI models + +import subprocess +import socket import fnmatch import os -import shlex -import socket -import subprocess +import requests import sys +import json +import subprocess +import argparse +import tempfile +import shlex from datetime import datetime - from launch_ci_runner import * +# The personal access token (PAT) with 'repo:status' permission +# Store your token securely and do not hardcode it in the script GITHUB_TOKEN = os.environ.get("ARTEMIS_GITHUB_TOKEN") @@ -46,38 +53,50 @@ ) args = parser.parse_args() + # Fetch PR information pr_info = get_pr_info(args.pr_number) head_repo = pr_info["head"]["repo"]["clone_url"] head_ref = pr_info["head"]["ref"] commit_sha = pr_info["head"]["sha"] + # gpu context context = "Continuous Integration / darwin_mi250" test_suite = "gpu.suite" - suffix = "amd" + suffix = "gpu" if args.submission: + # Update github PR status to indicate we have begun testing update_status(commit_sha, "pending", "CI Slurm job running...", context) + + # Run the tests in a temporary directory test_success = run_tests_in_temp_dir( args.pr_number, head_repo, head_ref, args.output_dir, test_suite, suffix ) + + # Update github PR status to indicate that testing has concluded if test_success: update_status(commit_sha, "success", "All tests passed.", context) else: update_status(commit_sha, "failure", "Tests failed.", context) else: + # Check that we are on the right system hostname = socket.gethostname() cluster = os.getenv("SLURM_CLUSTER_NAME") + if not fnmatch.fnmatch(hostname, "darwin-fe*"): + # if we are on a backend if cluster is None or cluster.lower() != "darwin": print("ERROR script must be run from Darwin!") sys.exit(1) + # Execute the sbatch command try: + # Submit batch job with ci_runner script that will checkout and build the code and run + # tests job_name = f"artemis_ci_darwin_mi250_PR{args.pr_number}" - squeue_command = ( - f"squeue --name={shlex.quote(job_name)} --user=$(whoami) " - "--noheader --format=%i" - ) + + # Clean up existing jobs for same PR + squeue_command = f"squeue --name={shlex.quote(job_name)} --user=$(whoami) --noheader --format=%i" squeue_result = subprocess.run( squeue_command, shell=True, @@ -87,17 +106,22 @@ ) job_ids = squeue_result.stdout.strip().split() - if job_ids: + if len(job_ids) >= 1: print("Canceling jobs:") for job_id in job_ids: print(f" {job_id}") - subprocess.run( - ["scancel"] + job_ids, + + # Use scancel to cancel the jobs + scancel_command = ["scancel"] + job_ids + scancel_result = subprocess.run( + scancel_command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True, ) + # Build output path and create directory if necessary + username = os.getenv("USER") current_date_time = datetime.now().strftime("%Y-%m-%d_%H:%M:%S") output_dir = os.path.join( "/usr", @@ -110,13 +134,14 @@ ) subprocess.run(["mkdir", "-p", output_dir], check=True) + # Create subprocess command for submitting CI job, and submit sbatch_command = [ "sbatch", f"--job-name={job_name}", f"--output={os.path.join(output_dir, job_name)}_%j.out", f"--error={os.path.join(output_dir, job_name)}_%j.out", "--partition=shared-gpu-amd-mi250", - "--time=02:00:00", + "--time=04:00:00", "--wrap", f"python3 {sys.argv[0]} {args.pr_number} --submission --output_dir {output_dir}", ] @@ -128,8 +153,11 @@ universal_newlines=True, ) print(result.stdout.strip()) + + # Update PR status that we have successfully submitted to SLURM job update_status(commit_sha, "pending", "CI SLURM job submitted...", context) except Exception as err: + # Update PR status that we have failed to submit the SLURM job update_status( commit_sha, "failure", @@ -140,6 +168,6 @@ update_status( commit_sha, "failure", - "SLURM job submission didn't complete successfully", + "SLURM job submission didn't complete sucessfully", context, ) From 81c59a1e7493220f3172f25bfdab421284af1032 Mon Sep 17 00:00:00 2001 From: Adam Dempsey Date: Fri, 7 Aug 2026 12:54:32 -0600 Subject: [PATCH 8/8] Revert --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c3a9ef12..93668954 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -69,7 +69,7 @@ jobs: mkdir -p build cd build cmake --preset=cpu-release ../ - make -j + make -j 4 ctest -L unit --output-on-failure regression-tests: @@ -97,7 +97,7 @@ jobs: mkdir -p build cd build cmake --preset=cpu-release ../../ - make -j + make -j 4 cd .. python3 run_tests.py regression.suite \ --save_build \