diff --git a/CMakePresets.json b/CMakePresets.json index 3fe06fe1..a5653442 100644 --- a/CMakePresets.json +++ b/CMakePresets.json @@ -57,6 +57,31 @@ "Kokkos_ARCH_VOLTA70": "ON" } }, + { + "name": "darwin-amd-debug", + "cacheVariables": { + "CMAKE_MAKE_PROGRAM": "$env{MAKE_PROGRAM}", + "CMAKE_BUILD_TYPE": "Debug", + "Kokkos_ENABLE_DEBUG_BOUNDS_CHECK": "ON", + "CMAKE_C_COMPILER": "amdclang", + "CMAKE_CXX_COMPILER": "amdclang++", + "CMAKE_CXX_FLAGS": "--gcc-toolchain=$env{GCC_ROOT}", + "ARTEMIS_ENABLE_HIP": "ON", + "Kokkos_ARCH_AMD_GFX90A": "ON" + } + }, + { + "name": "darwin-amd-release", + "cacheVariables": { + "CMAKE_MAKE_PROGRAM": "$env{MAKE_PROGRAM}", + "CMAKE_BUILD_TYPE": "Release", + "CMAKE_C_COMPILER": "amdclang", + "CMAKE_CXX_COMPILER": "amdclang++", + "CMAKE_CXX_FLAGS": "--gcc-toolchain=$env{GCC_ROOT}", + "ARTEMIS_ENABLE_HIP": "ON", + "Kokkos_ARCH_AMD_GFX90A": "ON" + } + }, { "name": "chicoma-cpu-debug", "cacheVariables": { diff --git a/env/bash b/env/bash index ab3ee9b7..98906082 100644 --- a/env/bash +++ b/env/bash @@ -45,6 +45,7 @@ else # Catch-all for Darwin echo "Supported partitions are" echo " skylake-gold" echo " volta-x86" + echo " mi250" PARTITION="darwin-fe" elif [[ $SLURM_JOB_PARTITION == "power9-rhel7" ]]; then PARTITION="darwin-power9-rhel7" @@ -52,6 +53,8 @@ else # Catch-all for Darwin PARTITION="darwin-skylake-gold" elif [[ $SLURM_JOB_PARTITION == "volta-x86" ]]; then PARTITION="darwin-volta-x86" + elif [[ $SLURM_JOB_PARTITION == "shared-gpu-amd-mi250" ]]; then + PARTITION="darwin-mi250" fi fi @@ -163,6 +166,19 @@ elif [[ $PARTITION == "darwin-volta-x86" ]]; then module list export ARTEMIS_SUITE=darwin-gpu echo "...setup SUCCEEDED" +elif [[ $PARTITION == "darwin-mi250" ]]; then + module purge + module load openmpi/4.1.6-gcc_13.2.0 + module load rocm + module load hdf5/1.12.2 + module load miniconda3/py311_23.11.0 + source /usr/projects/jovian/dependencies/python/volta-x86-py311/bin/activate + shorten_prompt + module load cmake/3.26.3 + module list + export GCC_ROOT="$(dirname "$(dirname "$(readlink -f "$(command -v g++)")")")" + export ARTEMIS_SUITE=darwin-amd + echo "...setup SUCCEEDED" elif [[ $PARTITION == "venado-gh" ]]; then module unload cray-libsci module load PrgEnv-gnu diff --git a/tst/darwin_amd_ci.py b/tst/darwin_amd_ci.py new file mode 100755 index 00000000..94ecfea7 --- /dev/null +++ b/tst/darwin_amd_ci.py @@ -0,0 +1,173 @@ +#!/usr/bin/env python3 +# ======================================================================================== +# (C) (or copyright) 2024. Triad National Security, LLC. All rights reserved. +# +# This program was produced under U.S. Government contract 89233218CNA000001 for Los +# Alamos National Laboratory (LANL), which is operated by Triad National Security, LLC +# for the U.S. Department of Energy/National Nuclear Security Administration. All rights +# in the program are reserved by Triad National Security, LLC, and the U.S. Department +# of Energy/National Nuclear Security Administration. The Government is granted for +# itself and others acting on its behalf a nonexclusive, paid-up, irrevocable worldwide +# license in this material to reproduce, prepare derivative works, distribute copies to +# the public, perform publicly and display publicly, and to permit others to do so. +# ======================================================================================== + +# This file was created in part or in whole by one of OpenAI's generative AI models + +import subprocess +import socket +import fnmatch +import os +import requests +import sys +import json +import subprocess +import argparse +import tempfile +import shlex +from datetime import datetime +from launch_ci_runner import * + +# The personal access token (PAT) with 'repo:status' permission +# Store your token securely and do not hardcode it in the script +GITHUB_TOKEN = os.environ.get("ARTEMIS_GITHUB_TOKEN") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="Run CI tasks with optional Slurm submission." + ) + parser.add_argument( + "pr_number", type=int, help="Pull request number for the CI run." + ) + parser.add_argument( + "--submission", + action="store_true", + help="Flag to indicate the script is running as a Slurm submission job.", + ) + parser.add_argument( + "--output_dir", + type=str, + default=None, + help="Output directory created when launching submission script", + ) + args = parser.parse_args() + + # Fetch PR information + pr_info = get_pr_info(args.pr_number) + head_repo = pr_info["head"]["repo"]["clone_url"] + head_ref = pr_info["head"]["ref"] + commit_sha = pr_info["head"]["sha"] + + # gpu context + context = "Continuous Integration / darwin_mi250" + test_suite = "gpu.suite" + suffix = "gpu" + + if args.submission: + # Update github PR status to indicate we have begun testing + update_status(commit_sha, "pending", "CI Slurm job running...", context) + + # Run the tests in a temporary directory + test_success = run_tests_in_temp_dir( + args.pr_number, head_repo, head_ref, args.output_dir, test_suite, suffix + ) + + # Update github PR status to indicate that testing has concluded + if test_success: + update_status(commit_sha, "success", "All tests passed.", context) + else: + update_status(commit_sha, "failure", "Tests failed.", context) + else: + # Check that we are on the right system + hostname = socket.gethostname() + cluster = os.getenv("SLURM_CLUSTER_NAME") + + if not fnmatch.fnmatch(hostname, "darwin-fe*"): + # if we are on a backend + if cluster is None or cluster.lower() != "darwin": + print("ERROR script must be run from Darwin!") + sys.exit(1) + + # Execute the sbatch command + try: + # Submit batch job with ci_runner script that will checkout and build the code and run + # tests + job_name = f"artemis_ci_darwin_mi250_PR{args.pr_number}" + + # Clean up existing jobs for same PR + squeue_command = f"squeue --name={shlex.quote(job_name)} --user=$(whoami) --noheader --format=%i" + squeue_result = subprocess.run( + squeue_command, + shell=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + ) + + job_ids = squeue_result.stdout.strip().split() + if len(job_ids) >= 1: + print("Canceling jobs:") + for job_id in job_ids: + print(f" {job_id}") + + # Use scancel to cancel the jobs + scancel_command = ["scancel"] + job_ids + scancel_result = subprocess.run( + scancel_command, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + ) + + # Build output path and create directory if necessary + username = os.getenv("USER") + current_date_time = datetime.now().strftime("%Y-%m-%d_%H:%M:%S") + output_dir = os.path.join( + "/usr", + "projects", + "jovian", + "ci", + "artemis", + f"pr_{args.pr_number}", + current_date_time, + ) + subprocess.run(["mkdir", "-p", output_dir], check=True) + + # Create subprocess command for submitting CI job, and submit + sbatch_command = [ + "sbatch", + f"--job-name={job_name}", + f"--output={os.path.join(output_dir, job_name)}_%j.out", + f"--error={os.path.join(output_dir, job_name)}_%j.out", + "--partition=shared-gpu-amd-mi250", + "--time=04:00:00", + "--wrap", + f"python3 {sys.argv[0]} {args.pr_number} --submission --output_dir {output_dir}", + ] + result = subprocess.run( + sbatch_command, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=True, + universal_newlines=True, + ) + print(result.stdout.strip()) + + # Update PR status that we have successfully submitted to SLURM job + update_status(commit_sha, "pending", "CI SLURM job submitted...", context) + except Exception as err: + # Update PR status that we have failed to submit the SLURM job + update_status( + commit_sha, + "failure", + "SLURM job submission failed with error: " + repr(err), + context, + ) + finally: + update_status( + commit_sha, + "failure", + "SLURM job submission didn't complete sucessfully", + context, + ) diff --git a/tst/launch_ci_runner.py b/tst/launch_ci_runner.py index 28fa8cde..c41053b8 100755 --- a/tst/launch_ci_runner.py +++ b/tst/launch_ci_runner.py @@ -152,6 +152,12 @@ def run_tests_in_temp_dir( subprocess.run(run_cmd) + run_cmd = ["./darwin_amd_ci.py", str(args.pr_number)] + if args.output_dir: + run_cmd.append("--output_dir=" + str(args.output_dir)) + + subprocess.run(run_cmd) + run_cmd = ["./darwin_gpu_ci.py", str(args.pr_number)] if args.output_dir: run_cmd.append("--output_dir=" + str(args.output_dir))